diff --git a/.arcconfig b/.arcconfig new file mode 100644 index 0000000..c76e1b1 --- /dev/null +++ b/.arcconfig @@ -0,0 +1,11 @@ +{ + "project_id" : "hbase", + "conduit_uri" : "https://reviews.facebook.net/", + "copyright_holder" : "Apache Software Foundation", + "phutil_libraries" : { + "arclib" : ".arc_jira_lib" + }, + "arcanist_configuration" : "ArcJIRAConfiguration", + "jira_project" : "HBASE", + "jira_api_url" : "https://issues.apache.org/jira/si/" +} diff --git a/.gitignore b/.gitignore index 0f182a0..225dd76 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ -*.class - -# Package Files # -*.jar -*.war -*.ear +/.arc_jira_lib +/.classpath +/.externalToolBuilders +/.project +/.settings +/build +/.idea/ +/logs +/target +*.iml +*.orig +*~ diff --git a/CHANGES.txt b/CHANGES.txt new file mode 100644 index 0000000..02dfcf8 --- /dev/null +++ b/CHANGES.txt @@ -0,0 +1,5698 @@ +HBase Change Log + +Release 0.94.8 - 5/22/2013 +Sub-task + + [HBASE-8381] - TestTableInputFormatScan on Hadoop 2 fails because YARN kills our applications + [HBASE-8399] - TestTableInputFormatScan2#testScanFromConfiguration fails on hadoop2 profile + +Bug + + [HBASE-7122] - Proper warning message when opening a log file with no entries (idle cluster) + [HBASE-7210] - Backport HBASE-6059 to 0.94 + [HBASE-7921] - TestHFileBlock.testGzipCompression should ignore the block checksum + [HBASE-8282] - User triggered flushes does not allow compaction to get triggered even if compaction criteria is met + [HBASE-8327] - Consolidate class loaders + [HBASE-8354] - Backport HBASE-7878 'recoverFileLease does not check return value of recoverLease' to 0.94 + [HBASE-8355] - BaseRegionObserver#pre(Compact|Flush|Store)ScannerOpen returns null + [HBASE-8377] - IntegrationTestBigLinkedList calculates wrap for linked list size incorrectly + [HBASE-8379] - bin/graceful_stop.sh does not return the balancer to original state + [HBASE-8385] - [SNAPSHOTS]: Restore fails to restore snapshot of a deleted table + [HBASE-8389] - HBASE-8354 forces Namenode into loop with lease recovery requests + [HBASE-8413] - Snapshot verify region will always fail if the HFile has been archived + [HBASE-8451] - MetricsMBeanBase has concurrency issues in init + [HBASE-8455] - Update ExportSnapshot to reflect changes in HBASE-7419 + [HBASE-8464] - FastDiffEncoder - valueOffset calculation is incorrect + [HBASE-8483] - HConnectionManager can leak ZooKeeper connections when using deleteStaleConnection + [HBASE-8493] - Backport HBASE-8422, 'Master won't go down', to 0.94 + [HBASE-8503] - Backport hbase-8483 "HConnectionManager can leak ZooKeeper connections when using deleteStaleConnection" to 0.94 + [HBASE-8505] - References to split daughters should not be deleted separately from parent META entry + [HBASE-8509] - ZKUtil#createWithParents won't set data during znode creation when parent folder doesn't exit + [HBASE-8513] - [0.94] Fix class files with CRLF endings + [HBASE-8516] - FSUtils.create() fail with ViewFS + [HBASE-8525] - Use sleep multilier when choosing sinks in ReplicationSource + [HBASE-8530] - Refine error message from ExportSnapshot when there is leftover snapshot in target cluster + [HBASE-8538] - HBaseAdmin#isTableEnabled() should check table existence before checking zk state. + [HBASE-8539] - Double(or tripple ...) ZooKeeper listeners of the same type when Master recovers from ZK SessionExpiredException + [HBASE-8540] - SnapshotFileCache logs too many times if snapshot dir doesn't exists + [HBASE-8547] - Fix java.lang.RuntimeException: Cached an already cached block + [HBASE-8550] - 0.94 ChaosMonkey grep for master is too broad + [HBASE-8563] - Double count of read requests for Gets + [HBASE-8588] - [Documentation]: Add information about adding REST and Thrift API kerberos principals to HBase ACL table + +Improvement + + [HBASE-5930] - Limits the amount of time an edit can live in the memstore. + [HBASE-6870] - HTable#coprocessorExec always scan the whole table + [HBASE-8345] - Add all available resources in o.a.h.h.rest.RootResource and VersionResource to o.a.h.h.rest.client.RemoteAdmin + [HBASE-8350] - enable ChaosMonkey to run commands as different users + [HBASE-8367] - LoadIncrementalHFiles does not return an error code nor throw Exception when failures occur due to timeouts + [HBASE-8383] - Support lib/*jar inside coprocessor jar + [HBASE-8405] - Add more custom options to how ClusterManager runs commands + [HBASE-8446] - Allow parallel snapshot of different tables + +New Feature + + [HBASE-7965] - Port table locking to 0.94 (HBASE-7305, HBASE-7546, HBASE-7933) + [HBASE-8415] - DisabledRegionSplitPolicy + +Task + + [HBASE-8574] - Add how to rename a table in the docbook + +Test + + [HBASE-8508] - improve unit-test coverage of package org.apache.hadoop.hbase.metrics.file + + +Release 0.94.7 - 4/24/2013 +Sub-task + + [HBASE-7615] - Add metrics for snapshots + [HBASE-7801] - Allow a deferred sync option per Mutation. + [HBASE-8210] - Backport the LoadTest portions of HBASE-7383 + [HBASE-8316] - JoinedHeap for non essential column families should reseek instead of seek + +Bug + + [HBASE-7401] - Remove warning message about running 'hbase migrate' + [HBASE-7658] - grant with an empty string as permission should throw an exception + [HBASE-7817] - Suggested JDWP debug options in hbase-env.sh are wrong + [HBASE-7824] - Improve master start up time when there is log splitting work + [HBASE-7925] - Back port HBASE-6881 into 0.94 + [HBASE-7961] - truncate on disabled table should throw TableNotEnabledException. + [HBASE-8014] - Backport HBASE-6915 to 0.94. + [HBASE-8030] - znode path of online region servers is hard coded in rolling_restart.sh + [HBASE-8044] - split/flush/compact/major_compact from hbase shell does not work for region key with \x format + [HBASE-8081] - Backport HBASE-7213 (separate hlog for meta tables) to 0.94 + [HBASE-8092] - bulk assignment in 0.94 doesn't handle ZK errors very well + [HBASE-8096] - [replication] NPE while replicating a log that is acquiring a new block from HDFS + [HBASE-8118] - TestTablePermission depends on the execution order + [HBASE-8125] - HBASE-7435 breaks BuiltInGzipDecompressor on Hadoop < 1.0.x + [HBASE-8127] - Region of a disabling or disabled table could be stuck in transition state when RS dies during Master initialization + [HBASE-8128] - HTable#put improvements + [HBASE-8131] - Create table handler needs to handle failure cases. + [HBASE-8142] - Sporadic TestZKProcedureControllers failures on trunk + [HBASE-8146] - IntegrationTestBigLinkedList does not work on distributed setup + [HBASE-8150] - server should not produce RAITE for already-opening region in 0.94 (because master retry logic handles this case poorly) + [HBASE-8151] - Decode memstoreTS in HFileReaderV2 only when necessary + [HBASE-8158] - Backport HBASE-8140 "TableMapReduceUtils#addDependencyJar fails when nested inside another MR job" + [HBASE-8160] - HMaster#move doesn't check if master initialized + [HBASE-8166] - Avoid writing the memstoreTS into HFiles when possible + [HBASE-8169] - TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS may fail due to regions randomly assigned to a RS + [HBASE-8170] - HbaseAdmin.createTable cannot handle creating three regions + [HBASE-8176] - Backport HBASE-5335 "Dynamic Schema Configurations" to 0.94 + [HBASE-8179] - JSON formatting for cluster status is sort of broken + [HBASE-8188] - Avoid unnecessary row compare in StoreScanner + [HBASE-8192] - Logic errror causes infinite loop in HRegion.bulkLoadHFiles(List) + [HBASE-8207] - Replication could have data loss when machine name contains hyphen "-" + [HBASE-8208] - In some situations data is not replicated to slaves when deferredLogSync is enabled + [HBASE-8211] - Support for NN HA for 0.94 + [HBASE-8212] - Introduce a new separator instead of hyphen('-') for renaming recovered queues' znodes + [HBASE-8213] - global authorization may lose efficacy + [HBASE-8215] - Removing existing .regioninfo in writeRegioninfoOnFilesystem + [HBASE-8222] - User class should implement equals() and hashCode() + [HBASE-8225] - [replication] minor code bug when registering ReplicationLogCleaner + [HBASE-8226] - HBaseTestingUtility#waitUntilAllRegionsAssigned won't return if it counts "too many" regions + [HBASE-8229] - Replication code logs like crazy if a target table cannot be found. + [HBASE-8230] - Possible NPE on regionserver abort if replication service has not been started + [HBASE-8231] - delete tests in table_tests.rb(TestShell) always running on empty table. + [HBASE-8232] - TestAccessController occasionally fails with IndexOutOfBoundsException + [HBASE-8246] - Backport HBASE-6318 to 0.94 where SplitLogWorker exits due to ConcurrentModificationException + [HBASE-8259] - Snapshot backport in 0.94.6 breaks rolling restarts + [HBASE-8266] - Master cannot start if TableNotFoundException is thrown while partial table recovery + [HBASE-8270] - Backport HBASE-8097 'MetaServerShutdownHandler may potentially keep bumping up DeadServer.numProcessing' to 0.94 + [HBASE-8274] - Backport to 94: HBASE-7488 Implement HConnectionManager.locateRegions which is currently returning null + [HBASE-8276] - Backport hbase-6738 to 0.94 "Too aggressive task resubmission from the distributed log manager" + [HBASE-8285] - HBaseClient never recovers for single HTable.get() calls with no retries when regions move + [HBASE-8288] - HBaseFileSystem: Refactoring and correct semantics for createPath methods + [HBASE-8303] - Increse the test timeout to 60s when they are less than 20s + [HBASE-8313] - Add Bloom filter testing for HFileOutputFormat + [HBASE-8326] - mapreduce.TestTableInputFormatScan times out frequently + [HBASE-8352] - Rename '.snapshot' directory + [HBASE-8427] - Apache Rat is incorrectly excluding test source files + +Improvement + + [HBASE-7410] - [snapshots] add snapshot/clone/restore/export docs to ref guide + [HBASE-7599] - Port HBASE-6066 (low hanging read path improvements) to 0.94 + [HBASE-8148] - Allow IPC to bind on a specific address + [HBASE-8152] - Avoid creating empty reference file when splitkey is outside the key range of a store file + [HBASE-8174] - Backport HBASE-8161(setting blocking file count on table level doesn't work) to 0.94 + [HBASE-8198] - Backport HBASE-8063(Filter HFiles based on first/last key) into 0.94 + [HBASE-8199] - Eliminate exception for ExportSnapshot against the null table snapshot (with no data in) + [HBASE-8209] - Improve LoadTest extensibility + +New Feature + + [HBASE-1936] - ClassLoader that loads from hdfs; useful adding filters to classpath without having to restart services + [HBASE-7415] - [snapshots] Add task information to snapshot operation + +Task + + [HBASE-7929] - Reapply hbase-7507 "Make memstore flush be able to retry after exception" to 0.94 branch. + +Test + + [HBASE-8106] - Test to check replication log znodes move is done correctly + [HBASE-8260] - create generic integration test for trunk and 94 that is more deterministic, can be run for longer and is less aggressive + + +Release 0.94.6.1 - 4/13/2013 +Bug + + [HBASE-8259] - Snapshot backport in 0.94.6 breaks rolling restarts + + +Release 0.94.6 - 3/14/2013 +Sub-task + + [HBASE-7944] - Replication leaks file reader resource & not reset currentNbOperations + +Bug + + [HBASE-6132] - ColumnCountGetFilter & PageFilter not working with FilterList + [HBASE-6347] - -ROOT- and .META. are stale in table.jsp if they moved + [HBASE-6748] - Endless recursive of deleteNode happened in SplitLogManager#DeleteAsyncCallback + [HBASE-7111] - hbase zkcli will not start if the zookeeper server chosen to connect to is unavailable + [HBASE-7153] - print gc option in hbase-env.sh affects hbase zkcli + [HBASE-7507] - Make memstore flush be able to retry after exception + [HBASE-7521] - fix HBASE-6060 (regions stuck in opening state) in 0.94 + [HBASE-7624] - Backport HBASE-5359 and HBASE-7596 to 0.94 + [HBASE-7671] - Flushing memstore again after last failure could cause data loss + [HBASE-7700] - TestColumnSeeking is mathematically bound to fail + [HBASE-7723] - Remove NameNode URI from ZK splitlogs + [HBASE-7725] - Add ability to create custom compaction request + [HBASE-7761] - MemStore.USEMSLAB_DEFAULT is false, hbase-default.xml says it's true + [HBASE-7763] - Compactions not sorting based on size anymore. + [HBASE-7768] - zkcluster in local mode not seeing configurations in hbase-{site|default}.xml + [HBASE-7777] - HBCK check for lingering split parents should check for child regions + [HBASE-7813] - Bug in BulkDeleteEndpoint kills entire rows on COLUMN/VERSION Deletes + [HBASE-7814] - Port HBASE-6963 'unable to run hbck on a secure cluster' to 0.94 + [HBASE-7829] - zookeeper kerberos conf keytab and principal parameters interchanged + [HBASE-7832] - Use User.getShortName() in FSUtils + [HBASE-7833] - 0.94 does not compile with Hadoop-0.20.205 and 0.22.0 + [HBASE-7851] - Include the guava classes as a dependency for jobs using mapreduce.TableMapReduceUtil + [HBASE-7866] - TestSplitTransactionOnCluster.testSplitBeforeSettingSplittingInZK failed 3 times in a row + [HBASE-7867] - setPreallocSize is different with COMMENT in setupTestEnv in MiniZooKeeperCluster.java + [HBASE-7869] - Provide way to not start LogSyncer thread + [HBASE-7876] - Got exception when manually triggers a split on an empty region + [HBASE-7883] - Update memstore size when removing the entries in append operation + [HBASE-7884] - ByteBloomFilter's performance can be improved by avoiding multiplication when generating hash + [HBASE-7913] - Secure Rest server should login before getting an instance of Rest servlet + [HBASE-7914] - Port the fix of HBASE-6748 into 0.94 branch + [HBASE-7915] - Secure ThriftServer needs to login before calling HBaseHandler + [HBASE-7916] - HMaster uses wrong InetSocketAddress parameter to throw exception + [HBASE-7919] - Wrong key is used in ServerManager#getServerConnection() to retrieve from Map serverConnections + [HBASE-7920] - Move isFamilyEssential(byte[] name) out of Filter interface in 0.94 + [HBASE-7945] - Remove flaky TestCatalogTrackerOnCluster + [HBASE-7986] - [REST] Make HTablePool size configurable + [HBASE-7991] - Backport HBASE-6479 'HFileReaderV1 caching the same parent META block could cause server abort when splitting' to 0.94 + [HBASE-8007] - Adopt TestLoadAndVerify from BigTop + [HBASE-8019] - Port HBASE-7779 '[snapshot 130201 merge] Fix TestMultiParallel' to 0.94 + [HBASE-8025] - zkcli fails when SERVER_GC_OPTS is enabled + [HBASE-8040] - Race condition in AM after HBASE-7521 (only 0.94) + [HBASE-8055] - Null check missing in StoreFile.Reader.getMaxTimestamp() + [HBASE-8061] - Missing test from TestFlushSnapshotFromClient in 0.94 + [HBASE-8069] - TestHLog is dependent on the execution order + [HBASE-8085] - Backport the fix for Bytes.toStringBinary() into 94 (HBASE-6991) + [HBASE-8099] - ReplicationZookeeper.copyQueuesFromRSUsingMulti should not return any queues if it failed to execute. + [HBASE-8103] - Fix pom so 0.94 can generate site reports + +Improvement + + [HBASE-7818] - add region level metrics readReqeustCount and writeRequestCount + [HBASE-7827] - Improve the speed of Hbase Thirft Batch mutation for deletes + [HBASE-8031] - Adopt goraci as an Integration test + +New Feature + + [HBASE-4210] - Allow coprocessor to interact with batches per region sent from a client + [HBASE-7360] - Snapshot 0.94 Backport + +Task + + [HBASE-8088] - Versioning site: part one, put stake in the ground for 0.94 by copying current versions of book and site + [HBASE-8090] - Versioning site; part two, publish 0.94 site and add link from main site + + +Release 0.94.5 - 2/7/2013 +Sub-task + + [HBASE-2611] - Handle RS that fails while processing the failure of another one + [HBASE-7626] - Backport portions of HBASE-7460 to 0.94 + [HBASE-7687] - TestCatalogTracker.testServerNotRunningIOException fails occasionally + [HBASE-7738] - REST server should publish metrics that are available via HTTP + +Bug + + [HBASE-5458] - Thread safety issues with Compression.Algorithm.GZ and CompressionTest + [HBASE-6513] - Test errors when building on MacOS + [HBASE-6824] - Introduce ${hbase.local.dir} and save coprocessor jars there + [HBASE-7034] - Bad version, failed OPENING to OPENED but master thinks it is open anyways + [HBASE-7293] - [replication] Remove dead sinks from ReplicationSource.currentPeers and pick new ones + [HBASE-7423] - HFileArchiver should not use the configuration from the Filesystem + [HBASE-7468] - TestSplitTransactionOnCluster hangs frequently + [HBASE-7476] - HBase shell count command doesn't escape binary output + [HBASE-7497] - TestDistributedLogSplitting.testDelayedDeleteOnFailure times out occasionally + [HBASE-7498] - Make REST server thread pool size configurable + [HBASE-7499] - TestScannerTimeout timeout is too aggressive. + [HBASE-7502] - TestScannerTimeout fails on snapshot branch + [HBASE-7504] - -ROOT- may be offline forever after FullGC of RS + [HBASE-7505] - Server will hang when stopping cluster, caused by waiting for split threads + [HBASE-7506] - Judgment of carrying ROOT/META will become wrong when expiring server + [HBASE-7513] - HDFSBlocksDistribution shouldn't send NPEs when something goes wrong + [HBASE-7515] - Store.loadStoreFiles should close opened files if there's an exception + [HBASE-7524] - hbase-policy.xml is improperly set thus all rules in it can be by-passed + [HBASE-7530] - [replication] Work around HDFS-4380 else we get NPEs + [HBASE-7531] - [replication] NPE in SequenceFileLogReader because ReplicationSource doesn't nullify the reader + [HBASE-7534] - [replication] TestReplication.queueFailover can fail because HBaseTestingUtility.createMultiRegions is dangerous + [HBASE-7545] - [replication] Break out TestReplication into manageable classes + [HBASE-7549] - Make HTableInterface#batch() javadoc proper + [HBASE-7550] - Synchronization problem in AssignmentManager + [HBASE-7551] - nodeChildrenChange event may happen after the transition to RS_ZK_REGION_SPLITTING in SplitTransaction causing the SPLIT event to be missed in the master side. + [HBASE-7562] - ZKUtil: missing "else condition" in multi processing + [HBASE-7575] - FSUtils#getTableStoreFilePathMap should all ignore non-table folders + [HBASE-7578] - TestCatalogTracker hangs occasionally + [HBASE-7581] - TestAccessController depends on the execution order + [HBASE-7584] - Improve TestAccessController.testAppend + [HBASE-7587] - Fix two findbugs warning in RowResource + [HBASE-7592] - HConnectionManager.getHTableDescriptor() compares too much + [HBASE-7602] - TestFromClientSide.testPoolBehavior is incorrect + [HBASE-7617] - TestHRegionOnCluster.testDataCorrectnessReplayingRecoveredEdits still fails occasionally. + [HBASE-7628] - Port HBASE-6509 fast-forwarding FuzzyRowFilter to 0.94 + [HBASE-7643] - HFileArchiver.resolveAndArchive() race condition may lead to snapshot data loss + [HBASE-7644] - Port HBASE-4802 'Disable show table metrics in bulk loader' to 0.94 + [HBASE-7646] - Make forkedProcessTimeoutInSeconds configurable + [HBASE-7647] - 0.94 hfiles v2.1 are not backwards compatible with HFilev2.0 + [HBASE-7648] - TestAcidGuarantees.testMixedAtomicity hangs sometimes + [HBASE-7654] - Add List getCoprocessors() to HTableDescriptor + [HBASE-7669] - ROOT region wouldn't be handled by PRI-IPC-Handler + [HBASE-7681] - Address some recent random test failures + [HBASE-7684] - NullPointerException in SecureClient when Call is cleaned up due to RPC timeout + [HBASE-7685] - Closing socket connection can't be removed from SecureClient + [HBASE-7693] - Hostname returned by TableInputFormatBase.reverseDNS contains trailing period + [HBASE-7694] - Secure HBase should use replication call queue + [HBASE-7698] - race between RS shutdown thread and openregionhandler causes region to get stuck + [HBASE-7702] - Adding filtering to Import jobs + [HBASE-7715] - FSUtils#waitOnSafeMode can incorrectly loop on standby NN + [HBASE-7717] - Wait until regions are assigned in TestSplitTransactionOnCluster + [HBASE-7728] - deadlock occurs between hlog roller and hlog syncer + [HBASE-7729] - TestCatalogTrackerOnCluster.testbadOriginalRootLocation fails occasionally + [HBASE-7730] - HBaseAdmin#synchronousBalanceSwitch is not compatible with 0.92 + [HBASE-7731] - Append/Increment methods in HRegion don't check whether the table is readonly or not + [HBASE-7740] - Recheck matching row for joined scanners + [HBASE-7771] - Secure HBase Client in MR job causes tasks to wait forever + [HBASE-7772] - clusterId is not set in conf properly if only TableMapReduceUtil.initCredentials() is called + [HBASE-7776] - Use ErrorReporter/Log instead of System.out in hbck + [HBASE-7785] - rolling-restart.sh script unable to check expiration of master znode + [HBASE-7793] - Port HBASE-5564 Bulkload is discarding duplicate records to 0.94 + +Improvement + + [HBASE-3996] - Support multiple tables and scanners as input to the mapper in map/reduce jobs + [HBASE-5416] - Improve performance of scans with some kind of filters. + [HBASE-5498] - Secure Bulk Load + [HBASE-5664] - CP hooks in Scan flow for fast forward when filter filters out a row + [HBASE-7441] - Make ClusterManager in IntegrationTestingUtility pluggable + [HBASE-7540] - Make znode dump to print a dump of replication znodes + [HBASE-7561] - Display the total number of regions for a given table on the master webUI + [HBASE-7757] - Add web UI to REST server and Thrift server + +New Feature + + [HBASE-6669] - Add BigDecimalColumnInterpreter for doing aggregations using AggregationClient + [HBASE-7748] - Add DelimitedKeyPrefixRegionSplitPolicy + +Wish + + [HBASE-7705] - Make the method getCurrentPoolSize of HTablePool public + + +Release 0.94.4 - 1/2/2013 +Sub-task + + [HBASE-3776] - Add Bloom Filter Support to HFileOutputFormat + [HBASE-6206] - Large tests fail with jdk1.7 + [HBASE-7009] - Port HBaseCluster interface/tests to 0.94 + [HBASE-7042] - Master Coprocessor Endpoint + [HBASE-7282] - Backport Compaction Tool to 0.94 + [HBASE-7331] - Add access control for region open and close, row locking, and stopping the regionserver + [HBASE-7336] - HFileBlock.readAtOffset does not work well with multiple threads + [HBASE-7371] - Blocksize in TestHFileBlock is unintentionally small + [HBASE-7399] - Health check chore for HMaster + [HBASE-7406] - Example health checker script + [HBASE-7431] - TestSplitTransactionOnCluster tests still flaky + [HBASE-7438] - TestSplitTransactionOnCluster has too many infinite loops + +Bug + + [HBASE-6175] - TestFSUtils flaky on hdfs getFileStatus method + [HBASE-6317] - Master clean start up and Partially enabled tables make region assignment inconsistent. + [HBASE-6327] - HLog can be null when create table + [HBASE-6423] - Writes should not block reads on blocking updates to memstores + [HBASE-7091] - support custom GC options in hbase-env.sh + [HBASE-7158] - Allow CopyTable to identify the source cluster (for replication scenarios) + [HBASE-7165] - TestSplitLogManager.testUnassignedTimeout is flaky + [HBASE-7166] - TestSplitTransactionOnCluster tests are flaky + [HBASE-7172] - TestSplitLogManager.testVanishingTaskZNode() fails when run individually and is flaky + [HBASE-7177] - TestZooKeeperScanPolicyObserver.testScanPolicyObserver is flaky + [HBASE-7180] - RegionScannerImpl.next() is inefficient. + [HBASE-7205] - Coprocessor classloader is replicated for all regions in the HRegionServer + [HBASE-7214] - CleanerChore logs too much, so much so it obscures all else that is going on + [HBASE-7230] - port HBASE-7109 integration tests on cluster are not getting picked up from distribution to 0.94 + [HBASE-7235] - TestMasterObserver is flaky + [HBASE-7251] - Avoid flood logs during client disconnect during batch get operation + [HBASE-7252] - TestSizeBasedThrottler fails occasionally + [HBASE-7259] - Deadlock in HBaseClient when KeeperException occured + [HBASE-7260] - Upgrade hadoop 1 dependency to hadoop 1.1.1 + [HBASE-7273] - Upgrade zookeeper dependency to 3.4.5 for 0.94 + [HBASE-7279] - Avoid copying the rowkey in RegionScanner, StoreScanner, and ScanQueryMatcher + [HBASE-7300] - HbckTestingUtil needs to keep a static executor to lower the number of threads used + [HBASE-7301] - Force ipv4 for unit tests + [HBASE-7307] - MetaReader.tableExists should not return false if the specified table regions has been split + [HBASE-7338] - Fix flaky condition for org.apache.hadoop.hbase.TestRegionRebalancing.testRebalanceOnRegionServerNumberChange + [HBASE-7342] - Split operation without split key incorrectly finds the middle key in off-by-one error + [HBASE-7343] - Fix flaky condition for TestDrainingServer + [HBASE-7357] - HBaseClient and HBaseServer should use hbase.security.authentication when negotiating authentication + [HBASE-7376] - Acquiring readLock does not apply timeout in HRegion#flushcache + [HBASE-7398] - [0.94 UNIT TESTS] TestAssignmentManager fails frequently on CentOS 5 + [HBASE-7412] - Fix how HTableDescriptor handles default max file size and flush size + [HBASE-7417] - TestReplication is flaky + [HBASE-7421] - TestHFileCleaner->testHFileCleaning has an aggressive timeout + [HBASE-7422] - MasterFS doesn't set configuration for internal FileSystem + [HBASE-7432] - TestHBaseFsck prevents testsuite from finishing + [HBASE-7435] - BuiltInGzipDecompressor is only released during full GC + [HBASE-7440] - ReplicationZookeeper#addPeer is racy + [HBASE-7442] - HBase remote CopyTable not working when security enabled + [HBASE-7455] - Increase timeouts in TestReplication and TestSplitLogWorker + [HBASE-7464] - [REST] Sending HTML for errors is unhelpful + [HBASE-7466] - Fix junit dependency typo in 0.94 + [HBASE-7467] - CleanerChore checkAndDeleteDirectory not deleting empty directories + [HBASE-7483] - TestHRegionOnCluster and TestSplitTransactionOnCluster are racy with HBaseAdmin.move() + [HBASE-7485] - TestSplitLogManager is still flaky on windows + +Improvement + + [HBASE-4791] - Allow Secure Zookeeper JAAS configuration to be programmatically set (rather than only by reading JAAS configuration file) + [HBASE-5616] - Make compaction code standalone + [HBASE-5693] - When creating a region, the master initializes it and creates a memstore within the master server + [HBASE-5778] - Fix HLog compression's incompatibilities + [HBASE-5888] - Clover profile in build + [HBASE-6585] - Audit log messages should contain info about the higher level operation being executed + [HBASE-6775] - Use ZK.multi when available for HBASE-6710 0.92/0.94 compatibility fix + [HBASE-7190] - Add an option to hbck to check only meta and assignment + [HBASE-7197] - Add multi get to RemoteHTable + [HBASE-7199] - hbck should check lingering reference hfile and have option to sideline them automatically + [HBASE-7204] - Make hbck ErrorReporter pluggable + [HBASE-7231] - port HBASE-7200 create integration test for balancing regions and killing region servers to 0.94 + [HBASE-7249] - add test name filter to IntegrationTestsDriver + [HBASE-7328] - IntegrationTestRebalanceAndKillServersTargeted supercedes IntegrationTestRebalanceAndKillServers, remove + [HBASE-7351] - Periodic health check chore + [HBASE-7359] - [REST] 'accessToken' in RemoteHTable is vestigial + [HBASE-7374] - Expose master table operations for coprocessors by way of MasterServices + [HBASE-7377] - Clean up TestHBase7051 + [HBASE-7381] - Lightweight data transfer for Class Result + [HBASE-7469] - [REST] Share a HBaseAdmin instance + [HBASE-7472] - [REST] Support MIME type application/protobuf + +Task + + [HBASE-5258] - Move coprocessors set out of RegionLoad + [HBASE-7170] - [0.94 branch] Allow HConnectionImplementation to reconnect to master multiple times + [HBASE-7283] - Backport HBASE-6564 + HBASE-7202 to 0.94 + [HBASE-7341] - Deprecate RowLocks in 0.94 + + +Release 0.94.3 - 11/12/2012 +Sub-task + + [HBASE-4913] - Per-CF compaction Via the Shell + [HBASE-6305] - TestLocalHBaseCluster hangs with hadoop 2.0/0.23 builds. + [HBASE-6925] - Change socket write size from 8K to 64K for HBaseServer + [HBASE-6996] - HRegion.mutateRowsWithLocks should call checkResources/checkReadOnly + [HBASE-7076] - Add test that increment/append properly integrate with MVCC + [HBASE-7077] - Test for: CheckAndPut should properly read MVCC + [HBASE-7078] - Add a test that append is atomic + +Bug + + [HBASE-6389] - Modify the conditions to ensure that Master waits for sufficient number of Region Servers before starting region assignments + [HBASE-6583] - Enhance Hbase load test tool to automatically create column families if not present + [HBASE-6665] - ROOT region should not be splitted even with META row as explicit split key + [HBASE-6700] - [replication] empty znodes created during queue failovers aren't deleted + [HBASE-6728] - [89-fb] prevent OOM possibility due to per connection responseQueue being unbounded + [HBASE-6733] - [0.92 UNIT TESTS] TestReplication.queueFailover occasionally fails [Part-2] + [HBASE-6796] - Backport HBASE-5547, Don't delete HFiles in backup mode. + [HBASE-6843] - loading lzo error when using coprocessor + [HBASE-6846] - BitComparator bug - ArrayIndexOutOfBoundsException + [HBASE-6904] - In the HBase shell, an error is thrown that states replication-related znodes already exist + [HBASE-6958] - TestAssignmentManager sometimes fails + [HBASE-6974] - Metric for blocked updates + [HBASE-6978] - Minor typo in ReplicationSource SocketTimeoutException error handling + [HBASE-7017] - Backport "[replication] The replication-executor should make sure the file that it is replicating is closed before declaring success on that file" to 0.94 + [HBASE-7018] - Fix and Improve TableDescriptor caching for bulk assignment + [HBASE-7021] - Default to Hadoop 1.0.4 in 0.94 and add Hadoop 1.1 profile + [HBASE-7037] - ReplicationPeer logs at WARN level aborting server instead of at FATAL + [HBASE-7048] - Regionsplitter requires the hadoop config path to be in hbase classpath + [HBASE-7051] - CheckAndPut should properly read MVCC + [HBASE-7060] - Region load balancing by table does not handle the case where a table's region count is lower than the number of the RS in the cluster + [HBASE-7069] - HTable.batch does not have to be synchronized + [HBASE-7086] - Enhance ResourceChecker to log stack trace for potentially hanging threads + [HBASE-7095] - Cannot set 'lenAsVal' for KeyOnlyFilter from shell + [HBASE-7103] - Need to fail split if SPLIT znode is deleted even before the split is completed. + [HBASE-7143] - TestMetaMigrationRemovingHTD fails when used with Hadoop 0.23/2.x + +Improvement + + [HBASE-5257] - Allow INCLUDE_AND_NEXT_COL in filters and use it in ColumnPaginationFilter + [HBASE-5314] - Gracefully rolling restart region servers in rolling-restart.sh + [HBASE-5898] - Consider double-checked locking for block cache lock + [HBASE-6852] - SchemaMetrics.updateOnCacheHit costs too much while full scanning a table with all of its fields + [HBASE-6942] - Endpoint implementation for bulk deletion of data + [HBASE-6951] - Allow the master info server to be started in a read only mode. + [HBASE-7073] - OperationMetrics needs to cache the value of hbase.metrics.exposeOperationTimes + [HBASE-7089] - Allow filter to be specified for Get from HBase shell + [HBASE-7097] - Log message in SecureServer.class uses wrong class name + [HBASE-7151] - Better log message for Per-CF compactions + +Task + + [HBASE-6032] - Port HFileBlockIndex improvement from HBASE-5987 + [HBASE-7016] - port HBASE-6518 'Bytes.toBytesBinary() incorrect trailing backslash escape' to 0.94 + [HBASE-7020] - Backport HBASE-6336 Split point should not be equal to start row or end row + [HBASE-7038] - Port HBASE-5970 Improve the AssignmentManager#updateTimer and speed up handling opened event to 0.94 + [HBASE-7040] - Port HBASE-5867 Improve Compaction Throttle Default to 0.94 + [HBASE-7053] - port blockcache configurability (part of HBASE-6312, and HBASE-7033) to 0.94 + [HBASE-7087] - Add to NOTICE.txt a note on jamon being MPL + +Test + + [HBASE-5984] - TestLogRolling.testLogRollOnPipelineRestart failed with HADOOP 2.0.0 + [HBASE-7142] - TestSplitLogManager#testDeadWorker may fail because of hard limit on the TimeoutMonitor's timeout period + + +Release 0.94.2 - 10/08/2012 +Sub-task + + [HBASE-6257] - Avoid unnecessary flush & compact on Meta in admin.rb. + [HBASE-6496] - Example ZK based scan policy + [HBASE-6792] - Remove interface audience annotations in 0.94/0.92 introduced by HBASE-6516 + +Bug + + [HBASE-4565] - Maven HBase build broken on cygwin with copynativelib.sh call. + [HBASE-5292] - getsize per-CF metric incorrectly counts compaction related reads as well + [HBASE-5549] - Master can fail if ZooKeeper session expires + [HBASE-5997] - Fix concerns raised in HBASE-5922 related to HalfStoreFileReader + [HBASE-6165] - Replication can overrun .META. scans on cluster re-start + [HBASE-6211] - Put latencies in jmx + [HBASE-6263] - Use default mode for HBase Thrift gateway if not specified + [HBASE-6268] - Can't enable a table on a 0.94 cluster from a 0.92 client + [HBASE-6299] - RS starting region open while failing ack to HMaster.sendRegionOpen() causes inconsistency in HMaster's region state and a series of successive problems + [HBASE-6321] - ReplicationSource dies reading the peer's id + [HBASE-6340] - HBase RPC should allow protocol extension with common interfaces. + [HBASE-6359] - KeyValue may return incorrect values after readFields() + [HBASE-6364] - Powering down the server host holding the .META. table causes HBase Client to take excessively long to recover and connect to reassigned .META. table + [HBASE-6378] - the javadoc of setEnabledTable maybe not describe accurately + [HBASE-6432] - HRegionServer doesn't properly set clusterId in conf + [HBASE-6437] - Avoid admin.balance during master initialize + [HBASE-6438] - RegionAlreadyInTransitionException needs to give more info to avoid assignment inconsistencies + [HBASE-6447] - Common TestZooKeeper failures on jenkins: testMasterSessionExpired and testCreateSilentIsReallySilent + [HBASE-6450] - HBase startup should be with MALLOC_MAX_ARENA set + [HBASE-6460] - hbck "-repairHoles" usage inconsistent with "-fixHdfsOrphans" + [HBASE-6471] - Performance regression caused by HBASE-4054 + [HBASE-6478] - TestClassLoading.testClassLoadingFromLibDirInJar occasionally fails + [HBASE-6488] - HBase wont run on IPv6 on OSes that use zone-indexes + [HBASE-6503] - HBase Shell Documentation For DROP Is Outdated + [HBASE-6504] - Adding GC details prevents HBase from starting in non-distributed mode + [HBASE-6512] - Incorrect OfflineMetaRepair log class name + [HBASE-6514] - unknown metrics type: org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram + [HBASE-6516] - hbck cannot detect any IOException while ".tableinfo" file is missing + [HBASE-6520] - MSLab May cause the Bytes.toLong not work correctly for increment + [HBASE-6525] - bin/replication/copy_tables_desc.rb references non-existent class + [HBASE-6529] - With HFile v2, the region server will always perform an extra copy of source files + [HBASE-6537] - Race between balancer and disable table can lead to inconsistent cluster + [HBASE-6552] - TestAcidGuarantees system test should flush more aggressively + [HBASE-6561] - Gets/Puts with many columns send the RegionServer into an "endless" loop + [HBASE-6565] - Coprocessor exec result Map is not thread safe + [HBASE-6576] - HBaseAdmin.createTable should wait until the table is enabled + [HBASE-6579] - Unnecessary KV order check in StoreScanner + [HBASE-6587] - Region would be assigned twice in the case of all RS offline + [HBASE-6596] - Revert HBASE-5022; it undoes HBC.create + [HBASE-6602] - Region Server Dynamic Metrics can cause high cpu usage. + [HBASE-6603] - RegionMetricsStorage.incrNumericMetric is called too often + [HBASE-6608] - Fix for HBASE-6160, META entries from daughters can be deleted before parent entries, shouldn't compare HRegionInfo's + [HBASE-6615] - hbase.rs.evictblocksonclose seems to be ineffective + [HBASE-6616] - test failure in TestDelayedRpc#testTooManyDelayedRpcs + [HBASE-6621] - Reduce calls to Bytes.toInt + [HBASE-6623] - [replication] replication metrics value AgeOfLastShippedOp is not set correctly + [HBASE-6631] - TestHMasterRPCException in 0.92 failed twice on socket timeout + [HBASE-6632] - [0.92 UNIT TESTS] testCreateTableRPCTimeOut sets rpc timeout to 1500ms and leaves it (testHundredsOfTable fails w/ 1500ms timeout) + [HBASE-6638] - Move DaemonThreadFactory into Threads (0.94) + [HBASE-6641] - more message with DoNotRetryIOException in client + [HBASE-6647] - [performance regression] appendNoSync/HBASE-4528 doesn't take deferred log flush into account + [HBASE-6648] - [0.92 UNIT TESTS] TestMasterObserver.testRegionTransitionOperations fails occasionally + [HBASE-6649] - [0.92 UNIT TESTS] TestReplication.queueFailover occasionally fails [Part-1] + [HBASE-6662] - Region server incorrectly reports its own address as master's address + [HBASE-6663] - NPE race in HConnection if zookeeper is reset + [HBASE-6671] - Kerberos authenticated super user should be able to retrieve proxied delegation tokens + [HBASE-6679] - RegionServer aborts due to race between compaction and split + [HBASE-6685] - Thrift DemoClient.pl got NullPointerException + [HBASE-6686] - HFile Quarantine fails with missing dirs in hadoop 2.0 + [HBASE-6688] - folder referred by thrift demo app instructions is outdated + [HBASE-6710] - 0.92/0.94 compatibility issues due to HBASE-5206 + [HBASE-6711] - Avoid local results copy in StoreScanner + [HBASE-6713] - Stopping META/ROOT RS may take 50mins when some region is splitting + [HBASE-6714] - TestMultiSlaveReplication#testMultiSlaveReplication may fail + [HBASE-6734] - Code duplication in LoadIncrementalHFiles + [HBASE-6757] - Very inefficient behaviour of scan using FilterList + [HBASE-6762] - HBASE-6340 broke SecureRPCEngine + [HBASE-6769] - HRS.multi eats NoSuchColumnFamilyException since HBASE-5021 + [HBASE-6784] - TestCoprocessorScanPolicy is sometimes flaky when run locally + [HBASE-6803] - script hbase should add JAVA_LIBRARY_PATH to LD_LIBRARY_PATH + [HBASE-6839] - Operations may be executed without holding rowLock + [HBASE-6842] - the jar used in coprocessor is not deleted in local which will exhaust the space of /tmp + [HBASE-6844] - upgrade 0.23 version dependency in 0.94 + [HBASE-6847] - HBASE-6649 broke replication + [HBASE-6851] - Race condition in TableAuthManager.updateGlobalCache() + [HBASE-6853] - IllegalArgument Exception is thrown when an empty region is spliitted. + [HBASE-6854] - Deletion of SPLITTING node on split rollback should clear the region from RIT + [HBASE-6868] - Skip checksum is broke; are we double-checksumming by default? + [HBASE-6871] - HFileBlockIndex Write Error in HFile V2 due to incorrect split into intermediate index blocks + [HBASE-6888] - HBase scripts ignore any HBASE_OPTS set in the environment + [HBASE-6889] - Ignore source control files with apache-rat + [HBASE-6900] - RegionScanner.reseek() creates NPE when a flush or compaction happens before the reseek. + [HBASE-6901] - Store file compactSelection throws ArrayIndexOutOfBoundsException + [HBASE-6906] - TestHBaseFsck#testQuarantine* tests are flakey due to TableNotEnabledException + [HBASE-6912] - Filters are not properly applied in certain cases + [HBASE-6916] - HBA logs at info level errors that won't show in the shell + [HBASE-6920] - On timeout connecting to master, client can get stuck and never make progress + [HBASE-6927] - WrongFS using HRegionInfo.getTableDesc() and different fs for hbase.root and fs.defaultFS + [HBASE-6946] - JavaDoc missing from release tarballs + +Improvement + + [HBASE-3271] - Allow .META. table to be exported + [HBASE-5582] - "No HServerInfo found for" should be a WARNING message + [HBASE-5631] - hbck should handle case where .tableinfo file is missing. + [HBASE-5714] - Add write permissions check before any hbck run that modifies hdfs. + [HBASE-5728] - Methods Missing in HTableInterface + [HBASE-6286] - Upgrade maven-compiler-plugin to 2.5.1 + [HBASE-6291] - Don't retry increments on an invalid cell + [HBASE-6308] - Coprocessors should be loaded in a custom ClassLoader to prevent dependency conflicts with HBase + [HBASE-6373] - Add more context information to audit log messages + [HBASE-6444] - Expose the ability to set custom HTTP Request Headers for the REST client used by RemoteHTable + [HBASE-6458] - new comparator twice in checkAndPut, just reuse the first one + [HBASE-6522] - Expose locks and leases to Coprocessors + [HBASE-6586] - Quarantine Corrupted HFiles with hbck + [HBASE-6643] - Accept encoded region name in compacting/spliting region from shell + [HBASE-6644] - HBaseAdmin.createTable should wait more till table is enabled. + [HBASE-6860] - [replication] HBASE-6550 is too aggressive, DDOSes .META. + [HBASE-6914] - Scans/Gets/Mutations don't give a good error if the table is disabled. + +New Feature + + [HBASE-6427] - Pluggable compaction and scan policies via coprocessors + [HBASE-6505] - Allow shared RegionObserver state + [HBASE-6550] - Refactoring ReplicationSink to make it more responsive of cluster health + +Task + + [HBASE-5042] - TestReadWriteConsistencyControl should be renamed + [HBASE-6288] - In hbase-daemons.sh, description of the default backup-master file path is wrong + [HBASE-6538] - Remove copy_table.rb script + +Test + + [HBASE-6507] - [hbck] TestHBaseFsck ran into TableNotEnabledException + [HBASE-6593] - TestAdmin times out sometimes + + +Release 0.94.1 - 7/24/2012 +Sub-task + + [HBASE-5342] - Grant/Revoke global permissions + [HBASE-5372] - Table mutation operations should check table level rights, not global rights + [HBASE-5385] - Delete table/column should delete stored permissions on -acl- table + [HBASE-5659] - TestAtomicOperation.testMultiRowMutationMultiThreads is still failing occasionally + [HBASE-6061] - Fix ACL "Admin" Table inconsistent permission check + [HBASE-6062] - preCheckAndPut/Delete() checks for READ when also a WRITE is performed + [HBASE-6092] - Authorize flush, split, compact operations in AccessController + [HBASE-6157] - Revoke of Global permission is not taking effect without restart. + [HBASE-6181] - TestStoreFile fails with jdk1.7 + [HBASE-6188] - Remove the concept of table owner + [HBASE-6209] - ACL Corrections for AccessControllerProtocol apis + [HBASE-6224] - add Pre and Post coprocessor hooks for BulkLoad + [HBASE-6238] - Grant on META not taking effect + [HBASE-6252] - TABLE ADMIN should be allowed to relocate regions + [HBASE-6253] - Do not allow user to disable or drop ACL table + [HBASE-6292] - Compact can skip the security access control + [HBASE-6355] - Allow HBase to compile against JDK7 + +Bug + + [HBASE-4379] - [hbck] Does not complain about tables with no end region [Z,] + [HBASE-4470] - ServerNotRunningException coming out of assignRootAndMeta kills the Master + [HBASE-4891] - HTable.ClientScanner needs to clone the Scan object + [HBASE-5546] - Master assigns region in the original region server when opening region failed + [HBASE-5722] - NPE in ZKUtil#getChildDataAndWatchForNewChildren when ZK not available or NW down. + [HBASE-5733] - AssignmentManager#processDeadServersAndRegionsInTransition can fail with NPE. + [HBASE-5741] - ImportTsv does not check for table existence + [HBASE-5757] - TableInputFormat should handle as many errors as possible + [HBASE-5806] - Handle split region related failures on master restart and RS restart + [HBASE-5840] - Open Region FAILED_OPEN doesn't clear the TaskMonitor Status, keeps showing the old status + [HBASE-5853] - java.lang.RuntimeException: readObject can't find class org.apache.hadoop.hdfs.protocol.HdfsFileStatus + [HBASE-5874] - When 'fs.default.name' not configured, the hbck tool and Merge tool throw IllegalArgumentException. + [HBASE-5875] - Process RIT and Master restart may remove an online server considering it as a dead server + [HBASE-5876] - TestImportExport has been failing against hadoop 0.23 profile + [HBASE-5883] - Backup master is going down due to connection refused exception + [HBASE-5894] - Table deletion failed but HBaseAdmin#deletetable reports it as success + [HBASE-5902] - Some scripts are not executable + [HBASE-5909] - SlabStats should be a daemon thread + [HBASE-5916] - RS restart just before master intialization we make the cluster non operative + [HBASE-5918] - Master will block forever at startup if root server dies between assigning root and assigning meta + [HBASE-5922] - HalfStoreFileReader seekBefore causes StackOverflowError + [HBASE-5927] - SSH and DisableTableHandler happening together does not clear the znode of the region and RIT map. + [HBASE-5928] - Hbck shouldn't npe when there are no tables. + [HBASE-5955] - Guava 11 drops MapEvictionListener and Hadoop 2.0.0-alpha requires it + [HBASE-5963] - ClassCastException: FileSystem$Cache$ClientFinalizer cannot be cast to Thread + [HBASE-5964] - HFileSystem: "No FileSystem for scheme: hdfs" + [HBASE-5966] - MapReduce based tests broken on Hadoop 2.0.0-alpha + [HBASE-5975] - Failed suppression of fs shutdown hook with Hadoop 2.0.0 + [HBASE-5986] - Clients can see holes in the META table when regions are being split + [HBASE-6002] - Possible chance of resource leak in HlogSplitter + [HBASE-6011] - Unable to start master in local mode + [HBASE-6016] - ServerShutdownHandler#processDeadRegion could return false for disabling table regions + [HBASE-6018] - hbck fails with a RejectedExecutionException when >50 regions present + [HBASE-6021] - NullPointerException when running LoadTestTool without specifying compression type + [HBASE-6029] - HBCK doesn't recover Balance switch if exception occurs in onlineHbck() + [HBASE-6046] - Master retry on ZK session expiry causes inconsistent region assignments. + [HBASE-6047] - Put.has() can't determine result correctly + [HBASE-6049] - Serializing "List" containing null elements will cause NullPointerException in HbaseObjectWritable.writeObject() + [HBASE-6050] - HLogSplitter renaming recovered.edits and CJ removing the parent directory race, making the HBCK think cluster is inconsistent. + [HBASE-6056] - Restore hbase-default version check + [HBASE-6065] - Log for flush would append a non-sequential edit in the hlog, leading to possible data loss + [HBASE-6068] - Secure HBase cluster : Client not able to call some admin APIs + [HBASE-6069] - TableInputFormatBase#createRecordReader() doesn't initialize TableRecordReader which causes NPE + [HBASE-6070] - AM.nodeDeleted and SSH races creating problems for regions under SPLIT + [HBASE-6088] - Region splitting not happened for long time due to ZK exception while creating RS_ZK_SPLITTING node + [HBASE-6089] - SSH and AM.joinCluster causes Concurrent Modification exception. + [HBASE-6095] - ActiveMasterManager NullPointerException + [HBASE-6115] - NullPointerException is thrown when root and meta table regions are assigning to another RS. + [HBASE-6122] - Backup master does not become Active master after ZK exception + [HBASE-6126] - Fix broke TestLocalHBaseCluster in 0.92/0.94 + [HBASE-6133] - TestRestartCluster failing in 0.92 + [HBASE-6141] - InterfaceAudience breaks 0.94 on older versions of hadoop + [HBASE-6146] - Disabling of Catalog tables should not be allowed + [HBASE-6158] - Data loss if the words 'merges' or 'splits' are used as Column Family name + [HBASE-6160] - META entries from daughters can be deleted before parent entries + [HBASE-6164] - Correct the bug in block encoding usage in bulkload + [HBASE-6185] - Update javadoc for ConstantSizeRegionSplitPolicy class + [HBASE-6195] - Increment data will be lost when the memstore is flushed + [HBASE-6200] - KeyComparator.compareWithoutRow can be wrong when families have the same prefix + [HBASE-6210] - Backport HBASE-6197 to 0.94 + [HBASE-6227] - SSH and cluster startup causes data loss + [HBASE-6229] - AM.assign() should not set table state to ENABLED directly. + [HBASE-6236] - Offline meta repair fails if the HBase base mount point is on a different cluster/volume than its parent in a ViewFS or similar FS + [HBASE-6237] - Fix race on ACL table creation in TestTablePermissions + [HBASE-6240] - Race in HCM.getMaster stalls clients + [HBASE-6246] - Admin.move without specifying destination does not go through AccessController + [HBASE-6248] - Jetty init may fail if directory name contains "master" + [HBASE-6265] - Calling getTimestamp() on a KV in cp.prePut() causes KV not to be flushed + [HBASE-6269] - Lazyseek should use the maxSequenseId StoreFile's KeyValue as the latest KeyValue + [HBASE-6281] - Assignment need not be called for disabling table regions during clean cluster start up. + [HBASE-6284] - Introduce HRegion#doMiniBatchMutation() + [HBASE-6293] - HMaster does not go down while splitting logs even if explicit shutdown is called. + [HBASE-6303] - HCD.setCompressionType should use Enum support for storing compression types as strings + [HBASE-6311] - Data error after majorCompaction caused by keeping MVCC for opened scanners + [HBASE-6313] - Client hangs because the client is not notified + [HBASE-6319] - ReplicationSource can call terminate on itself and deadlock + [HBASE-6325] - [replication] Race in ReplicationSourceManager.init can initiate a failover even if the node is alive + [HBASE-6326] - Avoid nested retry loops in HConnectionManager + [HBASE-6328] - FSHDFSUtils#recoverFileLease tries to rethrow InterruptedException but actually shallows it + [HBASE-6329] - Stopping META regionserver when splitting region could cause daughter region to be assigned twice + [HBASE-6337] - [MTTR] Remove renaming tmp log file in SplitLogManager + [HBASE-6357] - Failed distributed log splitting stuck on master web UI + [HBASE-6369] - HTable is not closed in AggregationClient + [HBASE-6375] - Master may be using a stale list of region servers for creating assignment plan during startup + [HBASE-6377] - HBASE-5533 metrics miss all operations submitted via MultiAction + [HBASE-6380] - bulkload should update the store.storeSize + [HBASE-6392] - UnknownRegionException blocks hbck from sideline big overlap regions + [HBASE-6394] - verifyrep MR job map tasks throws NullPointerException + [HBASE-6397] - [hbck] print out bulk load commands for sidelined regions if necessary + [HBASE-6406] - TestReplicationPeer.testResetZooKeeperSession and TestZooKeeper.testClientSessionExpired fail frequently + [HBASE-6420] - Gracefully shutdown logsyncer + [HBASE-6426] - Add Hadoop 2.0.x profile to 0.92+ + [HBASE-6440] - SplitLogManager - log the exception when failed to finish split log file + [HBASE-6443] - HLogSplitter should ignore 0 length files + [HBASE-6445] - rat check fails if hs_err_pid26514.log dropped in tests + +Improvement + + [HBASE-4720] - Implement atomic update operations (checkAndPut, checkAndDelete) for REST client/server + [HBASE-5360] - [uberhbck] Add options for how to handle offline split parents. + [HBASE-5630] - hbck should disable the balancer using synchronousBalanceSwitch. + [HBASE-5802] - Change the default metrics class to NullContextWithUpdateThread + [HBASE-5838] - Add an LZ4 compression option to HFile + [HBASE-5887] - Make TestAcidGuarantees usable for system testing. + [HBASE-5892] - [hbck] Refactor parallel WorkItem* to Futures. + [HBASE-5913] - Speed up the full scan of META + [HBASE-5973] - Add ability for potentially long-running IPC calls to abort if client disconnects + [HBASE-6010] - Security audit logger configuration for log4j + [HBASE-6013] - Polish sharp edges from CopyTable + [HBASE-6022] - Include Junit in the libs when packaging so that TestAcidGaurntee can run + [HBASE-6023] - Normalize security audit logging level with Hadoop + [HBASE-6040] - Use block encoding and HBase handled checksum verification in bulk loading using HFileOutputFormat + [HBASE-6067] - HBase won't start when hbase.rootdir uses ViewFileSystem + [HBASE-6114] - CacheControl flags should be tunable per table schema per CF + [HBASE-6124] - Backport HBASE-6033 to 0.90, 0.92 and 0.94 + [HBASE-6161] - Log Error when thrift server fails to start up. + [HBASE-6173] - hbck check specified tables only + [HBASE-6207] - Add jitter to client retry timer + [HBASE-6214] - Backport HBASE-5998 to 94.1 + [HBASE-6244] - [REST] Result generators do not need to query table schema + [HBASE-6247] - [REST] HTablePool.putTable is deprecated + [HBASE-6267] - hbase.store.delete.expired.storefile should be true by default + [HBASE-6283] - [region_mover.rb] Add option to exclude list of hosts on unload instead of just assuming the source node. + [HBASE-6314] - Fast fail behavior for unauthenticated user + [HBASE-6332] - Improve POM for better integration with downstream ivy projects + [HBASE-6334] - TestImprovement for TestHRegion.testWritesWhileGetting + [HBASE-6341] - Publicly expose HConnectionKey + [HBASE-6363] - HBaseConfiguration can carry a main method that dumps XML output for debug purposes + [HBASE-6382] - Upgrade Jersey to 1.8 to match Hadoop 1 and 2 + [HBASE-6384] - hbck should group together those sidelined regions need to be bulk loaded later + [HBASE-6433] - Improve HBaseServer#getRemoteAddress by utilizing HBaseServer.Connection.hostAddress + +New Feature + + [HBASE-2730] - Expose RS work queue contents on web UI + [HBASE-4956] - Control direct memory buffer consumption by HBaseClient + [HBASE-5609] - Add the ability to pass additional information for slow query logging + [HBASE-5886] - Add new metric for possible data loss due to puts without WAL + [HBASE-6044] - copytable: remove rs.* parameters + +Task + + [HBASE-6001] - Upgrade slf4j to 1.6.1 + [HBASE-6034] - Upgrade Hadoop dependencies + [HBASE-6077] - Document the most common secure RPC troubleshooting resolutions + [HBASE-6129] - Backport of Add Increment Coalescing in thrift. + [HBASE-6131] - Add attribution for code added by HBASE-5533 metrics + +Test + + [HBASE-5985] - TestMetaMigrationRemovingHTD failed with HADOOP 2.0.0 + + +Release 0.94.0 - 5/1/2012 +Sub-task + + [HBASE-4343] - Get the TestAcidGuarantee unit test to fail consistently + [HBASE-4345] - Ensure that Scanners that read from the storefiles respect MVCC + [HBASE-4346] - Optimise the storage that we use for storing MVCC information. + [HBASE-4485] - Eliminate window of missing Data + [HBASE-4517] - Document new replication features in 0.92 + [HBASE-4544] - Rename RWCC to MVCC + [HBASE-4594] - Ensure that KV's newer than the oldest-living-scanner is not accounted for the maxVersions during flush/compaction. + [HBASE-4661] - Ability to export the list of files for a some or all column families for a given region + [HBASE-4682] - Support deleted rows using Import/Export + [HBASE-4908] - HBase cluster test tool (port from 0.89-fb) + [HBASE-4911] - Clean shutdown + [HBASE-4979] - Setting KEEP_DELETE_CELLS fails in shell + [HBASE-4981] - add raw scan support to shell + [HBASE-4998] - Support deleted rows in CopyTable + [HBASE-5005] - Add DEFAULT_MIN_VERSIONS to HColumnDescriptor.DEFAULT_VALUES + [HBASE-5058] - Allow HBaseAdmin to use an existing connection + [HBASE-5096] - Replication does not handle deletes correctly. + [HBASE-5118] - Fix Scan documentation + [HBASE-5143] - Fix config typo in pluggable load balancer factory + [HBASE-5203] - Group atomic put/delete operation into a single WALEdit to handle region server failures. + [HBASE-5266] - Add documentation for ColumnRangeFilter + [HBASE-5346] - Fix testColumnFamilyCompression and test_TIMERANGE in TestHFileOutputFormat + [HBASE-5368] - Move PrefixSplitKeyPolicy out of the src/test into src, so it is accessible in HBase installs + [HBASE-5371] - Introduce AccessControllerProtocol.checkPermissions(Permission[] permissons) API + [HBASE-5413] - Rename RowMutation to RowMutations + [HBASE-5431] - Improve delete marker handling in Import M/R jobs + [HBASE-5460] - Add protobuf as M/R dependency jar + [HBASE-5497] - Add protobuf as M/R dependency jar (mapred) + [HBASE-5523] - Fix Delete Timerange logic for KEEP_DELETED_CELLS + [HBASE-5541] - Avoid holding the rowlock during HLog sync in HRegion.mutateRowWithLocks + [HBASE-5638] - Backport to 0.90 and 0.92 - NPE reading ZK config in HBase + [HBASE-5641] - decayingSampleTick1 prevents HBase from shutting down. + [HBASE-5793] - TestHBaseFsck#TestNoHdfsTable test hangs after client retries increased + +Bug + + [HBASE-2856] - TestAcidGuarantee broken on trunk + [HBASE-3443] - ICV optimization to look in memstore first and then store files (HBASE-3082) does not work when deletes are in the mix + [HBASE-3690] - Option to Exclude Bulk Import Files from Minor Compaction + [HBASE-3987] - Fix a NullPointerException on a failure to load Bloom filter data + [HBASE-4065] - TableOutputFormat ignores failure to create table instance + [HBASE-4078] - Silent Data Offlining During HDFS Flakiness + [HBASE-4105] - Stargate does not support Content-Type: application/json and Content-Encoding: gzip in parallel + [HBASE-4116] - [stargate] StringIndexOutOfBoundsException in row spec parse + [HBASE-4326] - Tests that use HBaseTestingUtility.startMiniCluster(n) should shutdown with HBaseTestingUtility.shutdownMiniCluster. + [HBASE-4397] - -ROOT-, .META. tables stay offline for too long in recovery phase after all RSs are shutdown at the same time + [HBASE-4398] - If HRegionPartitioner is used in MapReduce, client side configurations are overwritten by hbase-site.xml. + [HBASE-4476] - Compactions must fail if column tracker gets columns out of order + [HBASE-4496] - HFile V2 does not honor setCacheBlocks when scanning. + [HBASE-4607] - Split log worker should terminate properly when waiting for znode + [HBASE-4609] - ThriftServer.getRegionInfo() is expecting old ServerName format, need to use new Addressing class instead + [HBASE-4610] - Port HBASE-3380 (Master failover can split logs of live servers) to 92/trunk (definitely bring in config params, decide if we need to do more to fix the bug) + [HBASE-4626] - Filters unnecessarily copy byte arrays... + [HBASE-4645] - Edits Log recovery losing data across column families + [HBASE-4648] - Bytes.toBigDecimal() doesn't use offset + [HBASE-4658] - Put attributes are not exposed via the ThriftServer + [HBASE-4673] - NPE in HFileReaderV2.close during major compaction when hfile.block.cache.size is set to 0 + [HBASE-4679] - Thrift null mutation error + [HBASE-4691] - Remove more unnecessary byte[] copies from KeyValues + [HBASE-4729] - Clash between region unassign and splitting kills the master + [HBASE-4745] - LRU Statistics thread should be daemon + [HBASE-4769] - Abort RegionServer Immediately on OOME + [HBASE-4776] - HLog.closed should be checked inside of updateLock + [HBASE-4778] - Don't ignore corrupt StoreFiles when opening a region + [HBASE-4790] - Occasional TestDistributedLogSplitting failure + [HBASE-4792] - SplitRegionHandler doesn't care if it deletes the znode or not, leaves the parent region stuck offline + [HBASE-4795] - Fix TestHFileBlock when running on a 32-bit JVM + [HBASE-4797] - [availability] Skip recovered.edits files with edits we know older than what region currently has + [HBASE-4805] - Allow better control of resource consumption in HTable + [HBASE-4819] - TestShell broke in trunk; typo + [HBASE-4825] - TestRegionServersMetrics and TestZKLeaderManager are not categorized (small/medium/large) + [HBASE-4826] - Modify hbasetests.sh to take into account the new pom.xml with surefire + [HBASE-4832] - TestRegionServerCoprocessorExceptionWithAbort fails if the region server stops too fast + [HBASE-4853] - HBASE-4789 does overzealous pruning of seqids + [HBASE-4874] - Run tests with non-secure random, some tests hang otherwise + [HBASE-4878] - Master crash when splitting hlog may cause data loss + [HBASE-4886] - truncate fails in HBase shell + [HBASE-4890] - fix possible NPE in HConnectionManager + [HBASE-4932] - Block cache can be mistakenly instantiated by tools + [HBASE-4936] - Cached HRegionInterface connections crash when getting UnknownHost exceptions + [HBASE-4937] - Error in Quick Start Shell Exercises + [HBASE-4942] - HMaster is unable to start of HFile V1 is used + [HBASE-4946] - HTable.coprocessorExec (and possibly coprocessorProxy) does not work with dynamically loaded coprocessors (from hdfs or local system), because the RPC system tries to deserialize an unknown class. + [HBASE-4993] - Performance regression in minicluster creation + [HBASE-5003] - If the master is started with a wrong root dir, it gets stuck and can't be killed + [HBASE-5010] - Filter HFiles based on TTL + [HBASE-5015] - Remove some leaks in tests due to lack of HTable.close() + [HBASE-5026] - Add coprocessor hook to HRegionServer.ScannerListener.leaseExpired() + [HBASE-5027] - HConnection.create(final Connection conf) does not clone, it creates a new Configuration reading *.xmls and then does a merge. + [HBASE-5038] - Some tests leak connections + [HBASE-5041] - Major compaction on non existing table does not throw error + [HBASE-5051] - HBaseTestingUtility#getHBaseAdmin() creates a new HBaseAdmin instance at each call + [HBASE-5053] - HCM Tests leak connections + [HBASE-5055] - Build against hadoop 0.22 broken + [HBASE-5068] - RC1 can not build its hadoop-0.23 profile + [HBASE-5085] - fix test-patch script from setting the ulimit + [HBASE-5088] - A concurrency issue on SoftValueSortedMap + [HBASE-5091] - [replication] Update replication doc to reflect current znode structure + [HBASE-5097] - RegionObserver implementation whose preScannerOpen and postScannerOpen Impl return null can stall the system initialization through NPE + [HBASE-5099] - ZK event thread waiting for root region assignment may block server shutdown handler for the region sever the root region was on + [HBASE-5100] - Rollback of split could cause closed region to be opened again + [HBASE-5103] - Fix improper master znode deserialization + [HBASE-5120] - Timeout monitor races with table disable handler + [HBASE-5121] - MajorCompaction may affect scan's correctness + [HBASE-5141] - Memory leak in MonitoredRPCHandlerImpl + [HBASE-5152] - Region is on service before completing initialization when doing rollback of split, it will affect read correctness + [HBASE-5163] - TestLogRolling#testLogRollOnDatanodeDeath fails sometimes on Jenkins or hadoop QA ("The directory is already locked.") + [HBASE-5172] - HTableInterface should extend java.io.Closeable + [HBASE-5176] - AssignmentManager#getRegion: logging nit adds a redundant '+' + [HBASE-5182] - TBoundedThreadPoolServer threadKeepAliveTimeSec is not configured properly + [HBASE-5195] - [Coprocessors] preGet hook does not allow overriding or wrapping filter on incoming Get + [HBASE-5196] - Failure in region split after PONR could cause region hole + [HBASE-5200] - AM.ProcessRegionInTransition() and AM.handleRegion() race thus leaving the region assignment inconsistent + [HBASE-5206] - Port HBASE-5155 to 0.92, 0.94, and TRUNK + [HBASE-5212] - Fix test TestTableMapReduce against 0.23. + [HBASE-5213] - "hbase master stop" does not bring down backup masters + [HBASE-5221] - bin/hbase script doesn't look for Hadoop jars in the right place in trunk layout + [HBASE-5228] - [REST] Rip out "transform" feature + [HBASE-5267] - Add a configuration to disable the slab cache by default + [HBASE-5271] - Result.getValue and Result.getColumnLatest return the wrong column. + [HBASE-5278] - HBase shell script refers to removed "migrate" functionality + [HBASE-5281] - Should a failure in creating an unassigned node abort the master? + [HBASE-5282] - Possible file handle leak with truncated HLog file. + [HBASE-5283] - Request counters may become negative for heavily loaded regions + [HBASE-5286] - bin/hbase's logic of adding Hadoop jar files to the classpath is fragile when presented with split packaged Hadoop 0.23 installation + [HBASE-5288] - Security source code dirs missing from 0.92.0 release tarballs. + [HBASE-5290] - [FindBugs] Synchronization on boxed primitive + [HBASE-5292] - getsize per-CF metric incorrectly counts compaction related reads as well + [HBASE-5317] - Fix TestHFileOutputFormat to work against hadoop 0.23 + [HBASE-5327] - Print a message when an invalid hbase.rootdir is passed + [HBASE-5331] - Off by one bug in util.HMerge + [HBASE-5345] - CheckAndPut doesn't work when value is empty byte[] + [HBASE-5348] - Constraint configuration loaded with bloat + [HBASE-5350] - Fix jamon generated package names + [HBASE-5351] - hbase completebulkload to a new table fails in a race + [HBASE-5364] - Fix source files missing licenses in 0.92 and trunk + [HBASE-5384] - Up heap used by hadoopqa + [HBASE-5387] - Reuse compression streams in HFileBlock.Writer + [HBASE-5398] - HBase shell disable_all/enable_all/drop_all promp wrong tables for confirmation + [HBASE-5415] - FSTableDescriptors should handle random folders in hbase.root.dir better + [HBASE-5420] - TestImportTsv does not shut down MR Cluster correctly (fails against 0.23 hadoop) + [HBASE-5423] - Regionserver may block forever on waitOnAllRegionsToClose when aborting + [HBASE-5425] - Punt on the timeout doesn't work in BulkEnabler#waitUntilDone (master's EnableTableHandler) + [HBASE-5437] - HRegionThriftServer does not start because of a bug in HbaseHandlerMetricsProxy + [HBASE-5466] - Opening a table also opens the metatable and never closes it. + [HBASE-5470] - Make DataBlockEncodingTool work correctly with no native compression codecs loaded + [HBASE-5473] - Metrics does not push pread time + [HBASE-5477] - Cannot build RPM for hbase-0.92.0 + [HBASE-5480] - Fixups to MultithreadedTableMapper for Hadoop 0.23.2+ + [HBASE-5481] - Uncaught UnknownHostException prevents HBase from starting + [HBASE-5484] - Spelling mistake in error message in HMasterCommandLine + [HBASE-5485] - LogCleaner refers to non-existant SnapshotLogCleaner + [HBASE-5488] - OfflineMetaRepair doesn't support hadoop 0.20's fs.default.name property + [HBASE-5499] - dev-support/test-patch.sh does not have execute perms + [HBASE-5502] - region_mover.rb fails to load regions back to original server for regions only containing empty tables. + [HBASE-5507] - ThriftServerRunner.HbaseHandler.getRegionInfo() and getTableRegions() do not use ByteBuffer correctly + [HBASE-5514] - Compile against hadoop 0.24-SNAPSHOT + [HBASE-5522] - hbase 0.92 test artifacts are missing from Maven central + [HBASE-5524] - Add a couple of more filters to our rat exclusion set + [HBASE-5529] - MR test failures becuase MALLOC_ARENA_MAX is not set + [HBASE-5531] - Maven hadoop profile (version 23) needs to be updated with latest 23 snapshot + [HBASE-5535] - Make the functions in task monitor synchronized + [HBASE-5537] - MXBean shouldn't have a dependence on InterfaceStability until 0.96 + [HBASE-5545] - region can't be opened for a long time. Because the creating File failed. + [HBASE-5552] - Clean up our jmx view; its a bit of a mess + [HBASE-5562] - test-patch.sh reports a javadoc warning when there are no new javadoc warnings + [HBASE-5563] - HRegionInfo#compareTo should compare regionId as well + [HBASE-5567] - test-patch.sh has logic error in findbugs check + [HBASE-5568] - Multi concurrent flushcache() for one region could cause data loss + [HBASE-5569] - Do not collect deleted KVs when they are still in use by a scanner. + [HBASE-5574] - DEFAULT_MAX_FILE_SIZE defaults to a negative value + [HBASE-5579] - A Delete Version could mask other values + [HBASE-5581] - Creating a table with invalid syntax does not give an error message when it fails + [HBASE-5586] - [replication] NPE in ReplicationSource when creating a stream to an inexistent cluster + [HBASE-5596] - Few minor bugs from HBASE-5209 + [HBASE-5597] - Findbugs check in test-patch.sh always fails + [HBASE-5603] - rolling-restart.sh script hangs when attempting to detect expiration of /hbase/master znode. + [HBASE-5606] - SplitLogManger async delete node hangs log splitting when ZK connection is lost + [HBASE-5611] - Replayed edits from regions that failed to open during recovery aren't removed from the global MemStore size + [HBASE-5613] - ThriftServer getTableRegions does not return serverName and port + [HBASE-5623] - Race condition when rolling the HLog and hlogFlush + [HBASE-5624] - Aborting regionserver when splitting region, may cause daughter region not assigned by ServerShutdownHandler. + [HBASE-5633] - NPE reading ZK config in HBase + [HBASE-5635] - If getTaskList() returns null, splitlogWorker would go down and it won't serve any requests + [HBASE-5636] - TestTableMapReduce doesn't work properly. + [HBASE-5639] - The logic used in waiting for region servers during startup is broken + [HBASE-5656] - LoadIncrementalHFiles createTable should detect and set compression algorithm + [HBASE-5663] - MultithreadedTableMapper doesn't work. + [HBASE-5665] - Repeated split causes HRegionServer failures and breaks table + [HBASE-5669] - AggregationClient fails validation for open stoprow scan + [HBASE-5680] - Improve compatibility warning about HBase with Hadoop 0.23.x + [HBASE-5689] - Skipping RecoveredEdits may cause data loss + [HBASE-5690] - compression does not work in Store.java of 0.94 + [HBASE-5694] - getRowsWithColumnsTs() in Thrift service handles timestamps incorrectly + [HBASE-5701] - Put RegionServerDynamicStatistics under RegionServer in MBean hierarchy rather than have it as a peer. + [HBASE-5717] - Scanner metrics are only reported if you get to the end of a scanner + [HBASE-5720] - HFileDataBlockEncoderImpl uses wrong header size when reading HFiles with no checksums + [HBASE-5722] - NPE in ZKUtil#getChildDataAndWatchForNewChildren when ZK not available or NW down. + [HBASE-5724] - Row cache of KeyValue should be cleared in readFields(). + [HBASE-5736] - ThriftServerRunner.HbaseHandler.mutateRow() does not use ByteBuffer correctly + [HBASE-5743] - Support GIT patches + [HBASE-5773] - HtablePool constructor not reading config files in certain cases + [HBASE-5780] - Fix race in HBase regionserver startup vs ZK SASL authentication + [HBASE-5781] - Zookeeper session got closed while trying to assign the region to RS using hbck -fix + [HBASE-5782] - Edits can be appended out of seqid order since HBASE-4487 + [HBASE-5787] - Table owner can't disable/delete his/her own table + [HBASE-5795] - HServerLoad$RegionLoad breaks 0.92<->0.94 compatibility + [HBASE-5825] - TestHLog not running any tests; fix + [HBASE-5833] - 0.92 build has been failing pretty consistently on TestMasterFailover.... + [HBASE-5848] - Create table with EMPTY_START_ROW passed as splitKey causes the HMaster to abort + [HBASE-5849] - On first cluster startup, RS aborts if root znode is not available + [HBASE-5850] - Refuse operations from Admin before master is initialized - fix for all branches. + [HBASE-5857] - RIT map in RS not getting cleared while region opening + [HBASE-5861] - Hadoop 23 compilation broken due to tests introduced in HBASE-5604 + [HBASE-5864] - Error while reading from hfile in 0.94 + [HBASE-5865] - test-util.sh broken with unittest updates + [HBASE-5866] - Canary in tool package but says its in tools. + [HBASE-5871] - Usability regression, we don't parse compression algos anymore + [HBASE-5873] - TimeOut Monitor thread should be started after atleast one region server registers. + [HBASE-5884] - MapReduce package info has broken link to bulk-loads + [HBASE-5885] - Invalid HFile block magic on Local file System + [HBASE-5893] - Allow spaces in coprocessor conf (aka trim() className) + [HBASE-5897] - prePut coprocessor hook causing substantial CPU usage + [HBASE-5908] - TestHLogSplit.testTralingGarbageCorruptionFileSkipErrorsPasses should not use append to corrupt the HLog + [HBASE-6265] - Calling getTimestamp() on a KV in cp.prePut() causes KV not to be flushed + [HBASE-6357] - Failed distributed log splitting stuck on master web UI + +Improvement + + [HBASE-1744] - Thrift server to match the new java api. + [HBASE-2418] - add support for ZooKeeper authentication + [HBASE-3373] - Allow regions to be load-balanced by table + [HBASE-3433] - Remove the KV copy of every KV in Scan; introduced by HBASE-3232 + [HBASE-3512] - Coprocessors: Shell support for listing currently loaded coprocessor set + [HBASE-3565] - Add metrics to keep track of slow HLog appends + [HBASE-3763] - Add Bloom Block Index Support + [HBASE-3850] - Log more details when a scanner lease expires + [HBASE-3924] - Improve Shell's CLI help + [HBASE-3949] - Add "Master" link to RegionServer pages + [HBASE-4058] - Extend TestHBaseFsck with a complete .META. recovery scenario + [HBASE-4062] - Multi-column scanner unit test + [HBASE-4070] - [Coprocessors] Improve region server metrics to report loaded coprocessors to master + [HBASE-4076] - hbase should pick up HADOOP_CONF_DIR on its classpath + [HBASE-4131] - Make the Replication Service pluggable via a standard interface definition + [HBASE-4132] - Extend the WALActionsListener API to accomodate log archival + [HBASE-4145] - Provide metrics for hbase client + [HBASE-4213] - Support for fault tolerant, instant schema updates with out master's intervention (i.e with out enable/disable and bulk assign/unassign) through ZK. + [HBASE-4218] - Data Block Encoding of KeyValues (aka delta encoding / prefix compression) + [HBASE-4365] - Add a decent heuristic for region size + [HBASE-4418] - Show all the hbase configuration in the web ui + [HBASE-4439] - Move ClientScanner out of HTable + [HBASE-4440] - add an option to presplit table to PerformanceEvaluation + [HBASE-4461] - Expose getRowOrBefore via Thrift + [HBASE-4463] - Run more aggressive compactions during off peak hours + [HBASE-4465] - Lazy-seek optimization for StoreFile scanners + [HBASE-4469] - Avoid top row seek by looking up ROWCOL bloomfilter + [HBASE-4480] - Testing script to simplify local testing + [HBASE-4487] - The increment operation can release the rowlock before sync-ing the Hlog + [HBASE-4489] - Better key splitting in RegionSplitter + [HBASE-4519] - 25s sleep when expiring sessions in tests + [HBASE-4522] - Make hbase-site-custom.xml override the hbase-site.xml + [HBASE-4528] - The put operation can release the rowlock before sync-ing the Hlog + [HBASE-4532] - Avoid top row seek by dedicated bloom filter for delete family bloom filter + [HBASE-4542] - add filter info to slow query logging + [HBASE-4554] - Allow set/unset coprocessor table attributes from shell. + [HBASE-4568] - Make zk dump jsp response more quickly + [HBASE-4585] - Avoid next operations (and instead reseek) when current kv is deleted + [HBASE-4591] - TTL for old HLogs should be calculated from last modification time. + [HBASE-4612] - Allow ColumnPrefixFilter to support multiple prefixes + [HBASE-4627] - Ability to specify a custom start/end to RegionSplitter + [HBASE-4628] - Enhance Table Create Presplit Functionality within the HBase Shell + [HBASE-4640] - Catch ClosedChannelException and document it + [HBASE-4657] - Improve the efficiency of our MR jobs with a few configurations + [HBASE-4669] - Add an option of using round-robin assignment for enabling table + [HBASE-4696] - HRegionThriftServer' might have to indefinitely do redirtects + [HBASE-4704] - A JRuby script for identifying active master + [HBASE-4737] - Categorize the tests into small/medium/large; allow small tests to be run in parallel within a single JVM + [HBASE-4746] - Use a random ZK client port in unit tests so we can run them in parallel + [HBASE-4752] - Don't create an unnecessary LinkedList when evicting from the BlockCache + [HBASE-4760] - Add Developer Debug Options to HBase Config + [HBASE-4761] - Add Developer Debug Options to HBase Config + [HBASE-4764] - naming errors for TestHLogUtils and SoftValueSortedMapTest + [HBASE-4779] - TestHTablePool, TestScanWithBloomError, TestRegionSplitCalculator are not tagged and TestPoolMap should not use TestSuite + [HBASE-4780] - Lower mini cluster shutdown time in HRegionServer#waitOnAllRegionsToClose and ServerManager#letRegionServersShutdown + [HBASE-4781] - Pom update to use the new versions of surefire & junit + [HBASE-4783] - Improve RowCounter to count rows in a specific key range. + [HBASE-4787] - Make corePool as a configurable parameter in HTable + [HBASE-4798] - Sleeps and synchronisation improvements for tests + [HBASE-4809] - Per-CF set RPC metrics + [HBASE-4820] - Distributed log splitting coding enhancement to make it easier to understand, no semantics change + [HBASE-4847] - Activate single jvm for small tests on jenkins + [HBASE-4863] - Make Thrift server thread pool bounded and add a command-line UI test + [HBASE-4884] - Allow environment overrides for various HBase processes + [HBASE-4933] - Ability to calculate the blockcache hit ratio for the last few minutes + [HBASE-4938] - Create a HRegion.getScanner public method that allows reading from a specified readPoint + [HBASE-4940] - hadoop-metrics.properties can include configuration of the "rest" context for ganglia + [HBASE-4957] - Clean up some log messages, code in RecoverableZooKeeper + [HBASE-4964] - Add builddate, make less sections in toc, and add header and footer customizations + [HBASE-4965] - Monitor the open file descriptors and the threads counters during the unit tests + [HBASE-4970] - Add a parameter so that keepAliveTime of Htable thread pool can be changed + [HBASE-4971] - Useless sleeps in TestTimestampsFilter and TestMultipleTimestamps + [HBASE-4973] - On failure, HBaseAdmin sleeps one time too many + [HBASE-4989] - Metrics to measure sequential reads and random reads separately + [HBASE-4995] - Increase zk maxClientCnxns to give us some head room + [HBASE-5014] - PutSortReducer should adhere to memory limits + [HBASE-5017] - Bump the default hfile.block.cache.size because of HFileV2 + [HBASE-5021] - Enforce upper bound on timestamp + [HBASE-5033] - Opening/Closing store in parallel to reduce region open/close time + [HBASE-5064] - utilize surefire tests parallelization + [HBASE-5072] - Support Max Value for Per-Store Metrics + [HBASE-5074] - support checksums in HBase block cache + [HBASE-5134] - Remove getRegionServerWithoutRetries and getRegionServerWithRetries from HConnection Interface + [HBASE-5166] - MultiThreaded Table Mapper analogous to MultiThreaded Mapper in hadoop + [HBASE-5167] - We shouldn't be injecting 'Killing [daemon]' into logs, when we aren't doing that. + [HBASE-5186] - Add metrics to ThriftServer + [HBASE-5189] - Add metrics to keep track of region-splits in RS + [HBASE-5190] - Limit the IPC queue size based on calls' payload size + [HBASE-5193] - Use TBoundedThreadPoolServer in HRegionThriftServer + [HBASE-5197] - [replication] Handle socket timeouts in ReplicationSource to prevent DDOS + [HBASE-5199] - Delete out of TTL store files before compaction selection + [HBASE-5201] - Utilize TThreadedSelectorServer and remove redundant code in ThriftServer and HRegionThriftServer + [HBASE-5209] - HConnection/HMasterInterface should allow for way to get hostname of currently active master in multi-master HBase setup + [HBASE-5246] - Regenerate code with thrift 0.8.0 + [HBASE-5255] - Use singletons for OperationStatus to save memory + [HBASE-5259] - Normalize the RegionLocation in TableInputFormat by the reverse DNS lookup. + [HBASE-5297] - Update metrics numOpenConnections and callQueueLen directly in HBaseServer + [HBASE-5298] - Add thrift metrics to thrift2 + [HBASE-5304] - Pluggable split key policy + [HBASE-5310] - HConnectionManager server cache key enhancement + [HBASE-5325] - Expose basic information about the master-status through jmx beans + [HBASE-5332] - Deterministic Compaction Jitter + [HBASE-5358] - HBaseObjectWritable should be able to serialize/deserialize generic arrays + [HBASE-5363] - Automatically run rat check on mvn release builds + [HBASE-5388] - Tune HConnectionManager#getCachedLocation method + [HBASE-5393] - Consider splitting after flushing + [HBASE-5394] - Add ability to include Protobufs in HbaseObjectWritable + [HBASE-5395] - CopyTable needs to use GenericOptionsParser + [HBASE-5411] - Add more metrics for ThriftMetrics + [HBASE-5421] - use hadoop-client/hadoop-minicluster artifacts for Hadoop 0.23 build + [HBASE-5428] - Allow for custom filters to be registered within the Thrift interface + [HBASE-5433] - [REST] Add metrics to keep track of success/failure count + [HBASE-5434] - [REST] Include more metrics in cluster status request + [HBASE-5436] - Right-size the map when reading attributes. + [HBASE-5439] - Fix some performance findbugs issues + [HBASE-5440] - Allow Import to optionally use HFileOutputFormat + [HBASE-5442] - Use builder pattern in StoreFile and HFile + [HBASE-5454] - Refuse operations from Admin before master is initialized + [HBASE-5464] - Log warning message when thrift calls throw exceptions + [HBASE-5483] - Allow configurable host to bind to for starting REST server from commandline + [HBASE-5489] - Add HTable accessor to get regions for a key range + [HBASE-5508] - Add an option to allow test output to show on the terminal + [HBASE-5520] - Support reseek() at RegionScanner + [HBASE-5533] - Add more metrics to HBase + [HBASE-5551] - Some functions should not be used by customer code and must be deprecated in 0.94 + [HBASE-5560] - Avoid RegionServer GC caused by timed-out calls + [HBASE-5588] - Deprecate/remove AssignmentManager#clearRegionFromTransition + [HBASE-5589] - Add of the offline call to the Master Interface + [HBASE-5592] - Make it easier to get a table from shell + [HBASE-5618] - SplitLogManager - prevent unnecessary attempts to resubmits + [HBASE-5670] - Have Mutation implement the Row interface. + [HBASE-5671] - hbase.metrics.showTableName should be true by default + [HBASE-5682] - Allow HConnectionImplementation to recover from ZK connection loss (for 0.94 only) + [HBASE-5706] - "Dropping fs latency stats since buffer is full" spam + [HBASE-5712] - Parallelize load of .regioninfo files in diagnostic/repair portion of hbck. + [HBASE-5734] - Change hbck sideline root + [HBASE-5735] - Clearer warning message when connecting a non-secure HBase client to a secure HBase server + [HBASE-5737] - Minor Improvements related to balancer. + [HBASE-5748] - Enable lib directory in jar file for coprocessor + [HBASE-5770] - Add a clock skew warning threshold + [HBASE-5775] - ZKUtil doesn't handle deleteRecurisively cleanly + [HBASE-5823] - Hbck should be able to print help + [HBASE-5862] - After Region Close remove the Operation Metrics. + [HBASE-5863] - Improve the graceful_stop.sh CLI help (especially about reloads) + [HBASE-6173] - hbck check specified tables only + [HBASE-5360] - [uberhbck] Add options for how to handle offline split parents. + +New Feature + + [HBASE-2947] - MultiIncrement/MultiAppend (MultiGet functionality for increments and appends) + [HBASE-3134] - [replication] Add the ability to enable/disable streams + [HBASE-3584] - Allow atomic put/delete in one call + [HBASE-3856] - Build a tree structure data block index inside of the HFile + [HBASE-4102] - atomicAppend: A put that appends to the latest version of a cell; i.e. reads current value then adds the bytes offered by the client to the tail and writes out a new entry + [HBASE-4219] - Add Per-Column Family Metrics + [HBASE-4393] - Implement a canary monitoring program + [HBASE-4460] - Support running an embedded ThriftServer within a RegionServer + [HBASE-4536] - Allow CF to retain deleted rows + [HBASE-4608] - HLog Compression + [HBASE-4629] - enable automated patch testing for hbase + [HBASE-4683] - Always cache index and bloom blocks + [HBASE-4698] - Let the HFile Pretty Printer print all the key values for a specific row. + [HBASE-4768] - Per-(table, columnFamily) metrics with configurable table name inclusion + [HBASE-5128] - [uber hbck] Online automated repair of table integrity and region consistency problems + [HBASE-5177] - HTable needs a non cached version of getRegionLocation + [HBASE-5229] - Provide basic building blocks for "multi-row" local transactions. + [HBASE-5526] - Configurable file and directory based umask + [HBASE-5599] - [hbck] handle NO_VERSION_FILE and SHOULD_NOT_BE_DEPLOYED inconsistencies + [HBASE-5604] - M/R tool to replay WAL files + [HBASE-5719] - Enhance hbck to sideline overlapped mega regions + +Task + + [HBASE-4256] - Intra-row scanning (part deux) + [HBASE-4429] - Provide synchronous balanceSwitch() + [HBASE-4611] - Add support for Phabricator/Differential as an alternative code review tool + [HBASE-4712] - Document rules for writing tests + [HBASE-4751] - Make TestAdmin#testEnableTableRoundRobinAssignment friendly to concurrent tests + [HBASE-4968] - Add to troubleshooting workaround for direct buffer oome's. + [HBASE-5011] - Move test-util.sh from src/test/bin to dev-tools + [HBASE-5084] - Allow different HTable instances to share one ExecutorService + [HBASE-5111] - Upgrade zookeeper to 3.4.2 release + [HBASE-5173] - Commit hbase-4480 findHangingTest.sh script under dev-support + [HBASE-5256] - Use WritableUtils.readVInt() in RegionLoad.readFields() + [HBASE-5264] - Add 0.92.0 upgrade guide + [HBASE-5294] - Make sure javadoc is included in tarball bundle when we release + [HBASE-5400] - Some tests does not have annotations for (Small|Medium|Large)Tests + [HBASE-5427] - Upgrade our zk to 3.4.3 + [HBASE-5511] - More doc on maven release process + [HBASE-5715] - Revert 'Instant schema alter' for now, HBASE-4213 + [HBASE-5721] - Update bundled hadoop to be 1.0.2 (it was just released) + [HBASE-5758] - Forward port "HBASE-4109 Hostname returned via reverse dns lookup contains trailing period if configured interface is not 'default'" + [HBASE-5836] - Backport per region metrics from HBASE-3614 to 0.94.1 + +Test + + [HBASE-4516] - HFile-level load tester with compaction and random-read workloads + [HBASE-4534] - A new unit test for lazy seek and StoreScanner in general + [HBASE-4545] - TestHLog doesn't clean up after itself + [HBASE-4772] - Utility to Create StoreFiles + [HBASE-4808] - Test to Ensure Expired Deletes Don't Override Puts + [HBASE-4864] - TestMasterObserver#testRegionTransitionOperations occasionally fails + [HBASE-4868] - TestOfflineMetaRebuildBase#testMetaRebuild occasionally fails + [HBASE-5150] - Failure in a thread may not fail a test, clean up log splitting test + [HBASE-5223] - TestMetaReaderEditor is missing call to CatalogTracker.stop() + [HBASE-5455] - Add test to avoid unintentional reordering of items in HbaseObjectWritable + [HBASE-5792] - HLog Performance Evaluation Tool + + +Release 0.92.1 - Unreleased + BUG FIXES + HBASE-5176 AssignmentManager#getRegion: logging nit adds a redundant '+' (Karthik K) + HBASE-5237 Addendum for HBASE-5160 and HBASE-4397 (Ram) + HBASE-5235 HLogSplitter writer thread's streams not getting closed when any + of the writer threads has exceptions. (Ram) + HBASE-5243 LogSyncerThread not getting shutdown waiting for the interrupted flag (Ram) + HBASE-5255 Use singletons for OperationStatus to save memory (Benoit) + HBASE-5345 CheckAndPut doesn't work when value is empty byte[] (Evert Arckens) + HBASE-5466 Opening a table also opens the metatable and never closes it + (Ashley Taylor) + + TESTS + HBASE-5223 TestMetaReaderEditor is missing call to CatalogTracker.stop() + +Release 0.92.0 - 01/23/2012 + INCOMPATIBLE CHANGES + HBASE-2002 Coprocessors: Client side support; Support RPC interface + changes at runtime (Gary Helmling via Andrew Purtell) + HBASE-3677 Generate a globally unique cluster ID (changed + ClusterStatus serialization) + HBASE-3762 HTableFactory.releaseHTableInterface() should throw IOException + instead of wrapping in RuntimeException (Ted Yu via garyh) + HBASE-3629 Update our thrift to 0.6 (Moaz Reyad) + HBASE-1502 Remove need for heartbeats in HBase + HBASE-451 Remove HTableDescriptor from HRegionInfo (Subbu M Iyer) + HBASE-451 Remove HTableDescriptor from HRegionInfo + addendum that fixes TestTableMapReduce + HBASE-3534 Action should not store or serialize regionName (Ted Yu) + HBASE-4197 RegionServer expects all scanner to be subclasses of + HRegion.RegionScanner (Lars Hofhansl) + HBASE-4233 Update protobuf dependency to 2.4.0a (todd) + HBASE-4299 Update to Avro 1.5.3 and use Avro Maven plugin to generate + Avro classes. (Alejandro Abdelnur) + HBASE-4369 Deprecate HConnection#getZookeeperWatcher in prep for HBASE-1762 + HBASE-4247 Add isAborted method to the Abortable interface + (Akash Ashok) + HBASE-4503 Purge deprecated HBaseClusterTestCase + HBASE-4374 Up default regions size from 256M to 1G + HBASE-4648 Bytes.toBigDecimal() doesn't use offset (Bryan Keller via Lars H) + HBASE-4715 Remove stale broke .rb scripts from bin dir + HBASE-3433 Remove the KV copy of every KV in Scan; introduced by HBASE-3232 (Lars H) + HBASE-5017 Bump the default hfile.block.cache.size because of HFileV2 + + BUG FIXES + HBASE-3280 YouAreDeadException being swallowed in HRS getMaster + HBASE-3282 Need to retain DeadServers to ensure we don't allow + previously expired RS instances to rejoin cluster + HBASE-3283 NPE in AssignmentManager if processing shutdown of RS who + doesn't have any regions assigned to it + HBASE-3173 HBase 2984 breaks ability to specify BLOOMFILTER & + COMPRESSION via shell + HBASE-3310 Failing creating/altering table with compression agrument from + the HBase shell (Igor Ranitovic via Stack) + HBASE-3317 Javadoc and Throws Declaration for Bytes.incrementBytes() is + Wrong (Ed Kohlwey via Stack) + HBASE-1888 KeyValue methods throw NullPointerException instead of + IllegalArgumentException during parameter sanity check + HBASE-3337 Restore HBCK fix of unassignment and dupe assignment for new + master + HBASE-3332 Regions stuck in transition after RS failure + HBASE-3418 Increment operations can break when qualifiers are split + between memstore/snapshot and storefiles + HBASE-3403 Region orphaned after failure during split + HBASE-3492 NPE while splitting table with empty column family store + HBASE-3400 Coprocessor Support for Generic Interfaces + (Ed Kohlwey via Gary Helmling) + HBASE-3552 Coprocessors are unable to load if RegionServer is launched + using a different classloader than system default + HBASE-3578 TableInputFormat does not setup the configuration for HBase + mapreduce jobs correctly (Dan Harvey via Stack) + HBASE-3601 TestMasterFailover broken in TRUNK + HBASE-3605 Fix balancer log message + HBASE-3538 Column families allow to have slashes in name (Ian Knome via Stack) + HBASE-3313 Table name isn't checked in isTableEnabled/isTableDisabled + (Ted Yu via Stack) + HBASE-3514 Speedup HFile.Writer append (Matteo Bertozzi via Ryan) + HBASE-3665 tighten assertions for testBloomFilterSize + HBASE-3662 REST server does not respect client supplied max versions when + creating scanner + HBASE-3641 LruBlockCache.CacheStats.getHitCount() is not using the + correct variable + HBASE-3532 HRegion#equals is broken (Ted Yu via Stack) + HBASE-3697 Admin actions that use MetaReader to iterate regions need to + skip offline ones + HBASE-3583 Coprocessors: scannerNext and scannerClose hooks are called + when HRegionInterface#get is invoked (Mingjie Lai via + Andrew Purtell) + HBASE-3688 Setters of class HTableDescriptor do not work properly + HBASE-3702 Fix NPE in Exec method parameter serialization + HBASE-3709 HFile compression not sharing configuration + HBASE-3711 importtsv fails if rowkey length exceeds MAX_ROW_LENGTH + (Kazuki Ohta via todd) + HBASE-3716 Intermittent TestRegionRebalancing failure + (Ted Yu via Stack) + HBASE-3712 HTable.close() doesn't shutdown thread pool + (Ted Yu via Stack) + HBASE-3238 HBase needs to have the CREATE permission on the parent of its + ZooKeeper parent znode (Alex Newman via Stack) + HBASE-3728 NPE in HTablePool.closeTablePool (Ted Yu via Stack) + HBASE-3733 MemStoreFlusher.flushOneForGlobalPressure() shouldn't + be using TreeSet for HRegion (Ted Yu via J-D) + HBASE-3739 HMaster.getProtocolVersion() should distinguish + HMasterInterface and HMasterRegionInterface versions + HBASE-3723 Major compact should be done when there is only one storefile + and some keyvalue is outdated (Zhou Shuaifeng via Stack) + HBASE-3624 Only one coprocessor of each priority can be loaded for a table + HBASE-3598 Broken formatting in LRU stats output (Erik Onnen) + HBASE-3758 Delete triggers pre/postScannerOpen upcalls of RegionObserver + (Mingjie Lai via garyh) + HBASE-3790 Fix NPE in ExecResult.write() with null return value + HBASE-3781 hbase shell cannot start "NoMethodError: undefined method + `close' for nil:NilClass" (Mikael Sitruk) + HBASE-3802 Redundant list creation in HRegion + HBASE-3788 Two error handlings in AssignmentManager.setOfflineInZooKeeper() + (Ted Yu) + HBASE-3800 HMaster is not able to start due to AlreadyCreatedException + HBASE-3806 distributed log splitting double escapes task names + (Prakash Khemani) + HBASE-3819 TestSplitLogWorker has too many SLWs running -- makes for + contention and occasional failures + HBASE-3210 HBASE-1921 for the new master + HBASE-3827 hbase-1502, removing heartbeats, broke master joining a running + cluster and was returning master hostname for rs to use + HBASE-3829 TestMasterFailover failures in jenkins + HBASE-3843 splitLogWorker starts too early (Prakash Khemani) + HBASE-3838 RegionCoprocesorHost.preWALRestore throws npe in case there is + no RegionObserver registered (Himanshu Vashishtha) + HBASE-3847 Turn off DEBUG logging of RPCs in WriteableRPCEngine on TRUNK + HBASE-3777 Redefine Identity Of HBase Configuration (Karthick Sankarachary) + HBASE-3849 Fix master ui; hbase-1502 broke requests/second + HBASE-3853 Fix TestInfoServers to pass after HBASE-3835 (todd) + HBASE-3862 Race conditions in aggregate calculation (John Heitmann) + HBASE-3865 Failing TestWALReplay + HBASE-3864 Rename of hfile.min.blocksize.size in HBASE-2899 reverted in + HBASE-1861 (Aaron T. Myers) + HBASE-3876 TestCoprocessorInterface.testCoprocessorInterface broke on + jenkins and local + HBASE-3897 Docs (notsoquick guide) suggest invalid XML (Philip Zeyliger) + HBASE-3898 TestSplitTransactionOnCluster broke in TRUNK + HBASE-3826 Minor compaction needs to check if still over + compactionThreshold after compacting (Nicolas Spiegelberg) + HBASE-3912 [Stargate] Columns not handle by Scan + HBASE-3903 A successful write to client write-buffer may be lost or not + visible (Doug Meil) + HBASE-3894 Thread contention over row locks set monitor (Dave Latham) + HBASE-3959 hadoop-snappy version in the pom.xml is incorrect + (Alejandro Abdelnur) + HBASE-3971 Compression.java uses ClassLoader.getSystemClassLoader() + to load codec (Alejandro Abdelnur) + HBASE-3979 Trivial fixes in code, document (Ming Ma) + HBASE-3794 Ability to Discard Bad HTable Puts + HBASE-3923 HBASE-1502 Broke Shell's status 'simple' and 'detailed' + HBASE-3978 Rowlock lease renew doesn't work when custom coprocessor + indicates to bypass default action (Ming Ma) + HBASE-3963 Schedule all log-spliiting at startup all at once (mingjian) + HBASE-3983 list command in shell seems broken + HBASE-3793 HBASE-3468 Broke checkAndPut with null value (Ming Ma) + HBASE-3889 NPE in Distributed Log Splitting (Anirudh Todi) + HBASE-4000 You can't specify split points when you create a table in + the shell (Joey Echeverria) + HBASE-4029 Inappropriate checking of Logging Mode in HRegionServer + (Akash Ashok via Ted Yu) + HBASE-4037 Add timeout annotations to preempt surefire killing + all tests + HBASE-4024 Major compaction may not be triggered, even though region + server log says it is triggered (Ted Yu) + HBASE-4016 HRegion.incrementColumnValue() doesn't have a consistent + behavior when the field that we are incrementing is less + than 8 bytes long (Li Pi) + HBASE-4012 Further optimize byte comparison methods (Ted Yu) + HBASE-4037 Add timeout annotations to preempt surefire killing + all tests - TestFullLogReconstruction + HBASE-4051 [Coprocessors] Table coprocessor loaded twice when region is + initialized + HBASE-4059 If a region is split during RS shutdown process, the daughter + regions are NOT made online by master + HBASE-3904 HBA.createTable(final HTableDescriptor desc, byte [][] splitKeys) + should be synchronous + HBASE-4053 Most of the regions were added into AssignmentManager#servers twice + HBASE-4061 getTableDirs is missing directories to skip + HBASE-3867 when cluster is stopped and server which hosted meta region is + removed from cluster, master breaks down after restarting cluster. + HBASE-4074 When a RS has hostname with uppercase letter, there are two + RS entries in master (Weihua via Ted Yu) + HBASE-4077 Deadlock if WrongRegionException is thrown from getLock in + HRegion.delete (Adam Warrington via Ted Yu) + HBASE-3893 HRegion.internalObtainRowLock shouldn't wait forever + HBASE-4075 A bug in TestZKBasedOpenCloseRegion (Jieshan Bean via Ted Yu) + HBASE-4087 HBaseAdmin should perform validation of connection it holds + HBASE-4052 Enabling a table after master switch does not allow table scan, + throwing NotServingRegionException (ramkrishna via Ted Yu) + HBASE-4112 Creating table may throw NullPointerException (Jinchao via Ted Yu) + HBASE-4093 When verifyAndAssignRoot throws exception, the deadServers state + cannot be changed (fulin wang via Ted Yu) + HBASE-4118 method regionserver.MemStore#updateColumnValue: the check for + qualifier and family is missing (N Keywal via Ted Yu) + HBASE-4127 Don't modify table's name away in HBaseAdmin + HBASE-4105 Stargate does not support Content-Type: application/json and + Content-Encoding: gzip in parallel + HBASE-4116 [stargate] StringIndexOutOfBoundsException in row spec parse + (Allan Yan) + HBASE-3845 data loss because lastSeqWritten can miss memstore edits + (Prakash Khemani and ramkrishna.s.vasudevan) + HBASE-4083 If Enable table is not completed and is partial, then scanning of + the table is not working (ramkrishna.s.vasudevan) + HBASE-4138 If zookeeper.znode.parent is not specifed explicitly in Client + code then HTable object loops continuously waiting for the root region + by using /hbase as the base node.(ramkrishna.s.vasudevan) + HBASE-4032 HBASE-451 improperly breaks public API HRegionInfo#getTableDesc + HBASE-4003 Cleanup Calls Conservatively On Timeout (Karthick) + HBASE-3857 Fix TestHFileBlock.testBlockHeapSize test failure (Mikhail) + HBASE-4150 Don't enforce pool size limit with ThreadLocalPool + (Karthick Sankarachary via garyh) + HBASE-4171 HBase shell broken in trunk (Lars Hofhansl) + HBASE-4162 Fix TestHRegionInfo.testGetSetOfHTD: delete /tmp/hbase- + if it already exists (Mikhail Bautin) + HBASE-4179 Failed to run RowCounter on top of Hadoop branch-0.22 + (Michael Weng) + HBASE-4181 HConnectionManager can't find cached HRegionInterface and makes clients + work very slow (Jia Liu) + HBASE-4156 ZKConfig defaults clientPort improperly (Michajlo Matijkiw) + HBASE-4184 CatalogJanitor doesn't work properly when "fs.default.name" isn't + set in config file (Ming Ma) + HBASE-4186 No region is added to regionsInTransitionInRS + HBASE-4194 RegionSplitter: Split on under-loaded region servers first + HBASE-2399 Forced splits only act on the first family in a table (Ming Ma) + HBASE-4211 Do init-sizing of the StringBuilder making a ServerName + (Benoît Sigoure) + HBASE-4175 Fix FSUtils.createTableDescriptor() (Ramkrishna) + HBASE-4008 Problem while stopping HBase (Akash Ashok) + HBASE-4065 TableOutputFormat ignores failure to create table instance + (Brock Noland) + HBASE-4167 Potential leak of HTable instances when using HTablePool with + PoolType.ThreadLocal (Karthick Sankarachary) + HBASE-4239 HBASE-4012 introduced duplicate variable Bytes.LONG_BYTES + HBASE-4225 NoSuchColumnFamilyException in multi doesn't say which family + is bad (Ramkrishna Vasudevan) + HBASE-4220 Lots of DNS queries from client + HBASE-4253 Intermittent test failure because of missing config parameter in new + HTable(tablename) (Ramkrishna) + HBASE-4217 HRS.closeRegion should be able to close regions with only + the encoded name (ramkrishna.s.vasudevan) + HBASE-3229 HBASE-3229 Table creation, though using "async" call to master, + can actually run for a while and cause RPC timeout (Ming Ma) + HBASE-4252 TestLogRolling's low-probability failure (Jieshan Bean) + HBASE-4278 Race condition in Slab.java that occurs due to spinlock unlocking + early (Li Pi) + HBASE-4269 Add tests and restore semantics to TableInputFormat/TableRecordReader + (Jonathan Hsieh) + HBASE-4290 HLogSplitter doesn't mark its MonitoredTask as complete in + non-distributed case (todd) + HBASE-4303 HRegionInfo.toString has bad quoting (todd) + HBASE-4307 race condition in CacheTestUtils (Li Pi) + HBASE-4310 SlabCache metrics bugfix (Li Pi) + HBASE-4283 HBaseAdmin never recovers from restarted cluster (Lars Hofhansl) + HBASE-4315 RPC logging too verbose (todd) + HBASE-4273 java.lang.NullPointerException when a table is being disabled and + HMaster restarts (Ming Ma) + HBASE-4027 Off Heap Cache never creates Slabs (Li Pi) + HBASE-4265 zookeeper.KeeperException$NodeExistsException if HMaster restarts + while table is being disabled (Ming Ma) + HBASE-4338 Package build for rpm and deb are broken (Eric Yang) + HBASE-4309 slow query log metrics spewing warnings (Riley Patterson) + HBASE-4302 Only run Snappy compression tests if Snappy is available + (Alejandro Abdelnur via todd) + HBASE-4271 Clean up coprocessor handling of table operations + (Ming Ma via garyh) + HBASE-4341 HRS#closeAllRegions should take care of HRS#onlineRegions's + weak consistency (Jieshan Bean) + HBASE-4297 TableMapReduceUtil overwrites user supplied options + (Jan Lukavsky) + HBASE-4015 Refactor the TimeoutMonitor to make it less racy + (ramkrishna.s.vasudevan) + HBASE-4350 Fix a Bloom filter bug introduced by HFile v2 and + TestMultiColumnScanner that caught it (Mikhail Bautin) + HBASE-4007 distributed log splitting can get indefinitely stuck + (Prakash Khemani) + HBASE-4301 META migration from 0.90 to trunk fails (Subbu Iyer) + HBASE-4331 Bypassing default actions in prePut fails sometimes with + HTable client (Lars Hofhansl via garyh) + HBASE-4340 Hbase can't balance if ServerShutdownHandler encountered + exception (Jinchao Gao) + HBASE-4394 Add support for seeking hints to FilterList + HBASE-4406 TestOpenRegionHandler failing after HBASE-4287 (todd) + HBASE-4330 Fix races in slab cache (Li Pi & Todd) + HBASE-4383 SlabCache reports negative heap sizes (Li Pi) + HBASE-4351 If from Admin we try to unassign a region forcefully, + though a valid region name is given the master is not able + to identify the region to unassign (Ramkrishna) + HBASE-4363 [replication] ReplicationSource won't close if failing + to contact the sink (JD and Lars Hofhansl) + HBASE-4390 [replication] ReplicationSource's UncaughtExceptionHandler + shouldn't join + HBASE-4395 EnableTableHandler races with itself + HBASE-4414 Region splits by size not being triggered + HBASE-4322 HBASE-4322 [hbck] Update checkIntegrity/checkRegionChain + to present more accurate region split problem + (Jon Hseih) + HBASE-4417 HBaseAdmin.checkHBaseAvailable() doesn't close ZooKeeper connections + (Stefan Seelmann) + HBASE-4195 Possible inconsistency in a memstore read after a reseek, + possible performance improvement (nkeywal) + HBASE-4420 MasterObserver preMove() and postMove() should throw + IOException instead of UnknownRegionException + HBASE-4419 Resolve build warning messages (Praveen Patibandia) + HBASE-4428 Two methods in CacheTestUtils don't call setDaemon() on the threads + HBASE-4400 .META. getting stuck if RS hosting it is dead and znode state is in + RS_ZK_REGION_OPENED (Ramkrishna) + HBASE-3421 Very wide rows -- 30M plus -- cause us OOME (Nate Putnam) + HBASE-4153 Handle RegionAlreadyInTransitionException in AssignmentManager + (Ramkrishna) + HBASE-4452 Possibility of RS opening a region though tickleOpening fails due to + znode version mismatch (Ramkrishna) + HBASE-4446 Rolling restart RSs scenario, regions could stay in OPENING state + (Ming Ma) + HBASE-4468 Wrong resource name in an error massage: webapps instead of + hbase-webapps (nkeywal) + HBASE-4472 MiniHBaseCluster.shutdown() doesn't work if no active master + HBASE-4455 Rolling restart RSs scenario, -ROOT-, .META. regions are lost in + AssignmentManager (Ming Ma) + HBASE-4513 NOTICES.txt refers to Facebook for Thrift + HBASE-3130 [replication] ReplicationSource can't recover from session + expired on remote clusters (Chris Trezzo via JD) + HBASE-4212 TestMasterFailover fails occasionally (Gao Jinchao) + HBASE-4412 No need to retry scan operation on the same server in case of + RegionServerStoppedException (Ming Ma) + HBASE-4476 Compactions must fail if column tracker gets columns out of order + (Mikhail Bautin) + HBASE-4209 The HBase hbase-daemon.sh SIGKILLs master when stopping it + (Roman Shaposhnik) + HBASE-4496 HFile V2 does not honor setCacheBlocks when scanning (Lars and Mikhail) + HBASE-4531 hbase-4454 failsafe broke mvn site; back it out or fix + (Akash Ashok) + HBASE-4334 HRegion.get never validates row (Lars Hofhansl) + HBASE-4494 AvroServer:: get fails with NPE on a non-existent row + (Kay Kay) + HBASE-4481 TestMergeTool failed in 0.92 build 20 + HBASE-4386 Fix a potential NPE in TaskMonitor (todd) + HBASE-4402 Retaining locality after restart broken + HBASE-4482 Race Condition Concerning Eviction in SlabCache (Li Pi) + HBASE-4547 TestAdmin failing in 0.92 because .tableinfo not found + HBASE-4540 OpenedRegionHandler is not enforcing atomicity of the operation + it is performing(Ram) + HBASE-4335 Splits can create temporary holes in .META. that confuse clients + and regionservers (Lars H) + HBASE-4555 TestShell seems passed, but actually errors seen in test output + file (Mingjie Lai) + HBASE-4582 Store.java cleanup (failing TestHeapSize and has warnings) + HBASE-4556 Fix all incorrect uses of InternalScanner.next(...) (Lars H) + HBASE-4078 Validate store files after flush/compaction + HBASE-3417 CacheOnWrite is using the temporary output path for block + names, need to use a more consistent block naming scheme (jgray) + HBASE-4551 Fix pom and some test cases to compile and run against + Hadoop 0.23 (todd) + HBASE-3446 ProcessServerShutdown fails if META moves, orphaning lots of + regions + HBASE-4589 CacheOnWrite broken in some cases because it can conflict + with evictOnClose (jgray) + HBASE-4579 CST.requestCompaction semantics changed, logs are now + spammed when too many store files + HBASE-4620 I broke the build when I submitted HBASE-3581 (Send length + of the rpc response) + HBASE-4621 TestAvroServer fails quite often intermittently (Akash Ashok) + HBASE-4378 [hbck] Does not complain about regions with startkey==endkey. + (Jonathan Hsieh) + HBASE-4459 HbaseObjectWritable code is a byte, we will eventually run out of codes + HBASE-4430 Disable TestSlabCache and TestSingleSizedCache temporarily to + see if these are cause of build box failure though all tests + pass (Li Pi) + HBASE-4510 Check and workaround usage of internal HDFS APIs in HBase + (Harsh) + HBASE-4595 HFilePrettyPrinter Scanned kv count always 0 (Matteo Bertozzi) + HBASE-4580 Some invalid zk nodes were created when a clean cluster restarts + (Gaojinchao) + HBASE-4588 The floating point arithmetic to validate memory allocation + configurations need to be done as integers (dhruba) + HBASE-4647 RAT finds about 40 files missing licenses + HBASE-4642 Add Apache License Header + HBASE-4591 TTL for old HLogs should be calculated from last modification time. + HBASE-4578 NPE when altering a table that has moving regions (gaojinchao) + HBASE-4070 Improve region server metrics to report loaded coprocessors to + master (Eugene Koontz via apurtell) + HBASE-3512 Shell support for listing currently loaded coprocessors (Eugene + Koontz via apurtell) + HBASE-4670 Fix javadoc warnings + HBASE-4367 Deadlock in MemStore flusher due to JDK internally synchronizing + on current thread + HBASE-4645 Edits Log recovery losing data across column families + HBASE-4634 "test.build.data" property overused leading to write data at the + wrong place (nkeywal) + HBASE-4388 Second start after migration from 90 to trunk crashes + HBASE-4685 TestDistributedLogSplitting.testOrphanLogCreation failing because + of ArithmeticException: / by zero. + HBASE-4300 Start of new-version master fails if old master's znode is + hanging around + HBASE-4679 Thrift null mutation error + HBASE-4304 requestsPerSecond counter stuck at 0 (Li Pi) + HBASE-4692 HBASE-4300 broke the build + HBASE-4641 Block cache can be mistakenly instantiated on Master (jgray) + HBASE-4687 regionserver may miss zk-heartbeats to master when replaying + edits at region open (prakash via jgray) + HBASE-4701 TestMasterObserver fails up on jenkins + HBASE-4700 TestSplitTransactionOnCluster fails on occasion when it tries + to move a region + HBASE-4613 hbase.util.Threads#threadDumpingIsAlive sleeps 1 second, + slowing down the shutdown by 0.5s + HBASE-4552 multi-CF bulk load is not atomic across column families (Jonathan Hsieh) + HBASE-4710 UnknownProtocolException should abort client retries + HBASE-4695 WAL logs get deleted before region server can fully flush + (gaojinchao) + HBASE-4708 Revert safemode related pieces of hbase-4510 (Harsh J) + HBASE-3515 [replication] ReplicationSource can miss a log after RS comes out of GC + HBASE-4713 Raise debug level to warn on ExecutionException in + HConnectionManager$HConnectionImplementation (Lucian George Iordache) + HBASE-4716 Improve locking for single column family bulk load + HBASE-4609 ThriftServer.getRegionInfo() is expecting old ServerName format, need to + use new Addressing class instead (Jonathan Gray) + HBASE-4719 HBase script assumes pre-Hadoop 0.21 layout of jar files + (Roman Shposhnik) + HBASE-4553 The update of .tableinfo is not atomic; we remove then rename + HBASE-4725 NPE in AM#updateTimers + HBASE-4745 LRU statistics thread should be a daemon + HBASE-4749 TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS + occasionally fails + HBASE-4753 org.apache.hadoop.hbase.regionserver.TestHRegionInfo#testGetSetOfHTD + throws NPE on trunk (nkeywal) + HBASE-4754 FSTableDescriptors.getTableInfoPath() should handle FileNotFoundException + HBASE-4740 [bulk load] the HBASE-4552 API can't tell if errors on region server are recoverable + (Jonathan Hsieh) + HBASE-4741 Online schema change doesn't return errors + HBASE-4734 [bulk load] Warn if bulk load directory contained no files + HBASE-4723 Loads of NotAllMetaRegionsOnlineException traces when starting + the master + HBASE-4511 There is data loss when master failovers + HBASE-4577 Region server reports storefileSizeMB bigger than + storefileUncompressedSizeMB (gaojinchao) + HBASE-4478 Improve AssignmentManager.handleRegion so that it can process certain ZK state + in the case of RS offline + HBASE-4777 Write back to client 'incompatible' if we show up with wrong version + HBASE-4775 Remove -ea from all but tests; enable it if you need it testing + HBASE-4784 Void return types not handled correctly for CoprocessorProtocol + methods + HBASE-4792 SplitRegionHandler doesn't care if it deletes the znode or not, + leaves the parent region stuck offline + HBASE-4793 HBase shell still using deprecated methods removed in HBASE-4436 + HBASE-4801 alter_status shell prints sensible message at completion + HBASE-4796 Race between SplitRegionHandlers for the same region kills the master + HBASE-4816 Regionserver wouldn't go down because split happened exactly at same + time we issued bulk user region close call on our way out + HBASE-4815 Disable online altering by default, create a config for it + HBASE-4623 Remove @deprecated Scan methods in 0.90 from TRUNK and 0.92 + HBASE-4842 [hbck] Fix intermittent failures on TestHBaseFsck.testHBaseFsck + (Jon Hsieh) + HBASE-4308 Race between RegionOpenedHandler and AssignmentManager (Ram) + HBASE-4857 Recursive loop on KeeperException in + AuthenticationTokenSecretManager/ZKLeaderManager + HBASE-4739 Master dying while going to close a region can leave it in transition + forever (Gao Jinchao) + HBASE-4855 SplitLogManager hangs on cluster restart due to batch.installed doubly counted + HBASE-4877 TestHCM failing sporadically on jenkins and always for me on an + ubuntu machine + HBASE-4878 Master crash when splitting hlog may cause data loss (Chunhui Shen) + HBASE-4945 NPE in HRegion.bulkLoadHFiles (Andrew P and Lars H) + HBASE-4942 HMaster is unable to start of HFile V1 is used (Honghua Zhu) + HBASE-4610 Port HBASE-3380 (Master failover can split logs of live servers) to 92/trunk + HBASE-4946 HTable.coprocessorExec (and possibly coprocessorProxy) does not work with + dynamically loaded coprocessors (Andrei Dragomir) + HBASE-5026 Add coprocessor hook to HRegionServer.ScannerListener.leaseExpired() + HBASE-4935 hbase 0.92.0 doesn't work going against 0.20.205.0, its packaged hadoop + HBASE-5078 DistributedLogSplitter failing to split file because it has edits for + lots of regions + HBASE-5077 SplitLogWorker fails to let go of a task, kills the RS + HBASE-5096 Replication does not handle deletes correctly. (Lars H) + HBASE-5103 Fix improper master znode deserialization (Jonathan Hsieh) + HBASE-5099 ZK event thread waiting for root region assignment may block server + shutdown handler for the region sever the root region was on (Jimmy) + HBASE-5100 Rollback of split could cause closed region to be opened again (Chunhui) + HBASE-4397 -ROOT-, .META. tables stay offline for too long in recovery phase after all RSs + are shutdown at the same time (Ming Ma) + HBASE-5094 The META can hold an entry for a region with a different server name from the one + actually in the AssignmentManager thus making the region inaccessible. (Ram) + HBASE-5081 Distributed log splitting deleteNode races against splitLog retry (Prakash) + HBASE-4357 Region stayed in transition - in closing state (Ming Ma) + HBASE-5088 A concurrency issue on SoftValueSortedMap (Jieshan Bean and Lars H) + HBASE-5152 Region is on service before completing initialization when doing rollback of split, + it will affect read correctness (Chunhui) + HBASE-5137 MasterFileSystem.splitLog() should abort even if waitOnSafeMode() throws IOException(Ted) + HBASE-5121 MajorCompaction may affect scan's correctness (chunhui shen and Lars H) + HBASE-5143 Fix config typo in pluggable load balancer factory (Harsh J) + HBASE-5196 Failure in region split after PONR could cause region hole (Jimmy Xiang) + + TESTS + HBASE-4450 test for number of blocks read: to serve as baseline for expected + blocks read and for catching regressions (Kannan) + HBASE-4492 TestRollingRestart fails intermittently (Ted Yu and Ram) + HBASE-4512 JVMClusterUtil throwing wrong exception when master thread cannot be created (Ram) + HBASE-4479 TestMasterFailover failure in Hbase-0.92#17(Ram) + HBASE-4651 ConcurrentModificationException might be thrown in + TestHCM.testConnectionUniqueness (Jinchao) + HBASE-4518 TestServerCustomProtocol fails intermittently + HBASE-4790 Occasional TestDistributedLogSplitting failure (Jinchao) + HBASE-4864 TestMasterObserver#testRegionTransitionOperations occasionally + fails (Gao Jinchao) + HBASE-4868 TestOfflineMetaRebuildBase#testMetaRebuild occasionally fails + (Gao Jinchao) + HBASE-4874 Run tests with non-secure random, some tests hang otherwise (Lars H) + HBASE-5112 TestReplication#queueFailover flaky due to potentially + uninitialized Scan (Jimmy Xiang) + HBASE-5113 TestDrainingServer expects round robin region assignment but misses a + config parameter + HBASE-5105 TestImportTsv failed with hadoop 0.22 (Ming Ma) + + IMPROVEMENTS + HBASE-3290 Max Compaction Size (Nicolas Spiegelberg via Stack) + HBASE-3292 Expose block cache hit/miss/evict counts into region server + metrics + HBASE-2936 Differentiate between daemon & restart sleep periods + HBASE-3316 Add support for Java Serialization to HbaseObjectWritable + (Ed Kohlwey via Stack) + HBASE-1861 Multi-Family support for bulk upload tools + HBASE-3308 SplitTransaction.splitStoreFiles slows splits a lot + HBASE-3328 Added Admin API to specify explicit split points + HBASE-3377 Upgrade Jetty to 6.1.26 + HBASE-3393 Update Avro gateway to use Avro 1.4.1 and the new + server.join() method (Jeff Hammerbacher via Stack) + HBASE-3433 KeyValue API to explicitly distinguish between deep & shallow + copies + HBASE-3522 Unbundle our RPC versioning; rather than a global for all 4 + Interfaces -- region, master, region to master, and + coprocesssors -- instead version each individually + HBASE-3520 Update our bundled hadoop from branch-0.20-append to latest + (rpc version 43) + HBASE-3563 [site] Add one-page-only version of hbase doc + HBASE-3564 DemoClient.pl - a demo client in Perl + HBASE-3560 the hbase-default entry of "hbase.defaults.for.version" + causes tests not to run via not-maven + HBASE-3513 upgrade thrift to 0.5.0 and use mvn version + HBASE-3533 Allow HBASE_LIBRARY_PATH env var to specify extra locations + of native lib + HBASE-3631 CLONE - HBase 2984 breaks ability to specify BLOOMFILTER & + COMPRESSION via shell + HBASE-3630 DemoClient.Java is outdated (Moaz Reyed via Stack) + HBASE-3618 Add to HBase book, 'schema' chapter - pre-creating regions and + key types (Doug Meil via Stack) + HBASE-2495 Allow record filtering with selected row key values in HBase + Export (Subbu M Iyer via Stack) + HBASE-3440 Clean out load_table.rb and make sure all roads lead to + completebulkload tool (Vidhyashankar Venkataraman via Stack) + HBASE-3653 Parallelize Server Requests on HBase Client + HBASE-3657 reduce copying of HRegionInfo's (Ted Yu via Stack) + HBASE-3422 Balancer will try to rebalance thousands of regions in one go; + needs an upper bound added (Ted Yu via Stack) + HBASE-3676 Update region server load for AssignmentManager through + regionServerReport() (Ted Yu via Stack) + HBASE-3468 Enhance checkAndPut and checkAndDelete with comparators + HBASE-3683 NMapInputFormat should use a different config param for + number of maps + HBASE-3673 Reduce HTable Pool Contention Using Concurrent Collections + (Karthick Sankarachary via Stack) + HBASE-3474 HFileOutputFormat to use column family's compression algorithm + HBASE-3541 REST Multi Gets (Elliott Clark via Stack) + HBASE-3052 Add ability to have multiple ZK servers in a quorum in + MiniZooKeeperCluster for test writing (Liyin Tang via Stack) + HBASE-3693 isMajorCompaction() check triggers lots of listStatus DFS RPC + calls from HBase (Liyin Tang via Stack) + HBASE-3717 deprecate HTable isTableEnabled() methods in favor of + HBaseAdmin methods (David Butler via Stack) + HBASE-3720 Book.xml - porting conceptual-view / physical-view sections of + HBaseArchitecture wiki (Doug Meil via Stack) + HBASE-3705 Allow passing timestamp into importtsv (Andy Sautins via Stack) + HBASE-3715 Book.xml - adding architecture section on client, adding section + on spec-ex under mapreduce (Doug Meil via Stack) + HBASE-3684 Support column range filter (Jerry Chen via Stack) + HBASE-3647 Distinguish read and write request count in region + (Ted Yu via Stack) + HBASE-3704 Show per region request count in table.jsp + (Ted Yu via Stack) + HBASE-3694 high multiput latency due to checking global mem store size + in a synchronized function (Liyin Tang via Stack) + HBASE-3710 Book.xml - fill out descriptions of metrics + (Doug Meil via Stack) + HBASE-3738 Book.xml - expanding Architecture Client section + (Doug Meil via Stack) + HBASE-3587 Eliminate use of read-write lock to guard loaded + coprocessor collection + HBASE-3729 Get cells via shell with a time range predicate + (Ted Yu via Stack) + HBASE-3764 Book.xml - adding 2 FAQs (SQL and arch question) + HBASE-3770 Make FilterList accept var arg Filters in its constructor + as a convenience (Erik Onnen via Stack) + HBASE-3769 TableMapReduceUtil is inconsistent with other table-related + classes that accept byte[] as a table name (Erik Onnen via Stack) + HBASE-3768 Add best practice to book for loading row key only + (Erik Onnen via Stack) + HBASE-3765 metrics.xml - small format change and adding nav to hbase + book metrics section (Doug Meil) + HBASE-3759 Eliminate use of ThreadLocals for CoprocessorEnvironment + bypass() and complete() + HBASE-3701 revisit ArrayList creation (Ted Yu via Stack) + HBASE-3753 Book.xml - architecture, adding more Store info (Doug Meil) + HBASE-3784 book.xml - adding small subsection in architecture/client on + filters (Doug Meil) + HBASE-3785 book.xml - moving WAL into architecture section, plus adding + more description on what it does (Doug Meil) + HBASE-3699 Make RegionServerServices and MasterServices extend Server + (Erik Onnen) + HBASE-3757 Upgrade to ZK 3.3.3 + HBASE-3609 Improve the selection of regions to balance; part 2 (Ted Yu) + HBASE-2939 Allow Client-Side Connection Pooling (Karthik Sankarachary) + HBASE-3798 [REST] Allow representation to elide row key and column key + HBASE-3812 Tidy up naming consistency and documentation in coprocessor + framework (Mingjie Lai) + HBASE-1512 Support aggregate functions (Himanshu Vashishtha) + HBASE-3796 Per-Store Enties in Compaction Queue + HBASE-3670 Fix error handling in get(List gets) + (Harsh J Chouraria) + HBASE-3835 Switch master and region server pages to Jamon-based templates + HBASE-3721 Speedup LoadIncrementalHFiles (Ted Yu) + HBASE-3855 Performance degradation of memstore because reseek is linear + (dhruba borthakur) + HBASE-3797 StoreFile Level Compaction Locking + HBASE-1476 Multithreaded Compactions + HBASE-3877 Determine Proper Defaults for Compaction ThreadPools + HBASE-3880 Make mapper function in ImportTSV plug-able (Bill Graham) + HBASE-2938 HBASE-2938 Add Thread-Local Behavior To HTable Pool + (Karthick Sankarachary) + HBASE-3811 Allow adding attributes to Scan (Alex Baranau) + HBASE-3841 HTable and HTableInterface docs are inconsistent with + one another (Harsh J Chouraria) + HBASE-2937 Facilitate Timeouts In HBase Client (Karthick Sankarachary) + HBASE-3921 Allow adding arbitrary blobs to Put (dhruba borthakur) + HBASE-3931 Allow adding attributes to Get + HBASE-3942 The thrift scannerOpen functions should support row caching + (Adam Worthington) + HBASE-2556 Add convenience method to HBaseAdmin to get a collection of + HRegionInfo objects for each table (Ming Ma) + HBASE-3952 Guava snuck back in as a dependency via hbase-3777 + HBASE-3808 Implement Executor.toString for master handlers at least + (Brock Noland) + HBASE-3873 Mavenize Hadoop Snappy JAR/SOs project dependencies + (Alejandro Abdelnur) + HBASE-3941 "hbase version" command line should print version info + (Jolly Chen) + HBASE-3961 Add Delete.setWriteToWAL functionality (Bruno Dumon) + HBASE-3928 Some potential performance improvements to Bytes/KeyValue + HBASE-3982 Improvements to TestHFileSeek + HBASE-3940 HBase daemons should log version info at startup and possibly + periodically (Li Pi) + HBASE-3789 Cleanup the locking contention in the master + HBASE-3927 Display total uncompressed byte size of a region in web UI + HBASE-4011 New MasterObserver hook: post startup of active master + HBASE-3994 SplitTransaction has a window where clients can + get RegionOfflineException + HBASE-4010 HMaster.createTable could be heavily optimized + HBASE-3506 Ability to disable, drop and enable tables using regex expression + (Joey Echeverria via Ted Yu) + HBASE-3516 Coprocessors: add test cases for loading coprocessor jars + (Mingjie Lai via garyh) + HBASE-4036 Implementing a MultipleColumnPrefixFilter (Anirudh Todi) + HBASE-4048 [Coprocessors] Support configuration of coprocessor at load time + HBASE-3240 Improve documentation of importtsv and bulk loads. + (Aaron T. Myers via todd) + HBASE-4054 Usability improvement to HTablePool (Daniel Iancu) + HBASE-4079 HTableUtil - helper class for loading data (Doug Meil via Ted Yu) + HBASE-3871 Speedup LoadIncrementalHFiles by parallelizing HFile splitting + HBASE-4081 Issues with HRegion.compactStores methods (Ming Ma) + HBASE-3465 Hbase should use a HADOOP_HOME environment variable if available + (Alejandro Abdelnur) + HBASE-3899 enhance HBase RPC to support free-ing up server handler threads + even if response is not ready (Vlad Dogaru) + HBASE-4142 Advise against large batches in javadoc for HTable#put(List) + HBASE-4139 [stargate] Update ScannerModel with support for filter package + additions + HBASE-1938 Make in-memory table scanning faster (nkeywal) + HBASE-4143 HTable.doPut(List) should check the writebuffer length every so often + (Doug Meil via Ted Yu) + HBASE-3065 Retry all 'retryable' zk operations; e.g. connection loss (Liyin Tang) + HBASE-3810 Registering a coprocessor in HTableDescriptor should be easier + (Mingjie Lai via garyh) + HBASE-4158 Upgrade pom.xml to surefire 2.9 (Aaron Kushner & Mikhail) + HBASE-3899 Add ability for delayed RPC calls to set return value + immediately at call return. (Vlad Dogaru via todd) + HBASE-4169 FSUtils LeaseRecovery for non HDFS FileSystems (Lohit Vijayarenu) + HBASE-3807 Fix units in RS UI metrics (subramanian raghunathan) + HBASE-4193 Enhance RPC debug logging to provide more details on + call contents + HBASE-4190 Coprocessors: pull up some cp constants from cp package to + o.a.h.h.HConstants (Mingjie Lai) + HBASE-4227 Modify the webUI so that default values of column families are + not shown (Nileema Shingte) + HBASE-4229 Replace Jettison JSON encoding with Jackson in HLogPrettyPrinter + (Riley Patterson) + HBASE-4230 Compaction threads need names + HBASE-4236 Don't lock the stream while serializing the response (Benoit Sigoure) + HBASE-4237 Directly remove the call being handled from the map of outstanding RPCs + (Benoit Sigoure) + HBASE-4199 blockCache summary - backend (Doug Meil) + HBASE-4240 Allow Loadbalancer to be pluggable + HBASE-4244 Refactor bin/hbase help + HBASE-4241 Optimize flushing of the Memstore (Lars Hofhansl) + HBASE-4248 Enhancements for Filter Language exposing HBase filters through + the Thrift API (Anirudh Todi) + HBASE-3900 Expose progress of a major compaction in UI and/or in shell + (Brad Anderson) + HBASE-4291 Improve display of regions in transition in UI to be more + readable (todd) + HBASE-4281 Add facility to dump current state of all executors (todd) + HBASE-4275 RS should communicate fatal "aborts" back to the master (todd) + HBASE-4263 New config property for user-table only RegionObservers + (Lars Hofhansl) + HBASE-4257 Limit the number of regions in transitions displayed on + master webpage. (todd) + HBASE-1730 Online Schema Changes + HBASE-4206 jenkins hash implementation uses longs unnecessarily + (Ron Yang) + HBASE-3842 Refactor Coprocessor Compaction API + HBASE-4312 Deploy new hbase logo + HBASE-4327 Compile HBase against hadoop 0.22 (Joep Rottinghuis) + HBASE-4339 Improve eclipse documentation and project file generation + (Eric Charles) + HBASE-4342 Update Thrift to 0.7.0 (Moaz Reyad) + HBASE-4260 Expose a command to manually trigger an HLog roll + (ramkrishna.s.vasudevan) + HBASE-4347 Remove duplicated code from Put, Delete, Get, Scan, MultiPut + (Lars Hofhansl) + HBASE-4359 Show dead RegionServer names in the HMaster info page + (Harsh J) + HBASE-4287 If region opening fails, change region in transition into + a FAILED_OPEN state so that it can be retried quickly. (todd) + HBASE-4381 Refactor split decisions into a split policy class. (todd) + HBASE-4373 HBaseAdmin.assign() does not use force flag (Ramkrishna) + HBASE-4425 Provide access to RpcServer instance from RegionServerServices + HBASE-4411 When copying tables/CFs, allow CF names to be changed + (David Revell) + HBASE-4424 Provide coprocessors access to createTable() via + MasterServices + HBASE-4432 Enable/Disable off heap cache with config (Li Pi) + HBASE-4434 seek optimization: don't do eager HFile Scanner + next() unless the next KV is needed + (Kannan Muthukkaruppan) + HBASE-4280 [replication] ReplicationSink can deadlock itself via handlers + HBASE-4014 Coprocessors: Flag the presence of coprocessors in logged + exceptions (Eugene Koontz) + HBASE-4449 LoadIncrementalHFiles should be able to handle CFs with blooms + (David Revell) + HBASE-4454 Add failsafe plugin to build and rename integration tests + (Jesse Yates) + HBASE-4499 [replication] Source shouldn't update ZK if it didn't progress + (Chris Trezzo via JD) + HBASE-2794 Utilize ROWCOL bloom filter if multiple columns within same family + are requested in a Get (Mikhail Bautin) + HBASE-4487 The increment operation can release the rowlock before sync-ing + the Hlog (dhruba borthakur) + HBASE-4526 special case for stopping master in hbase-daemon.sh is no longer + required (Roman Shaposhnik) + HBASE-4520 Better handling of Bloom filter type discrepancy between HFile + and CF config (Mikhail Bautin) + HBASE-4558 Refactor TestOpenedRegionHandler and TestOpenRegionHandler.(Ram) + HBASE-4558 Addendum for TestMasterFailover (Ram) - Breaks the build + HBASE-4568 Make zk dump jsp response faster + HBASE-4606 Remove spam in HCM and fix a list.size == 0 + HBASE-3581 hbase rpc should send size of response + HBASE-4585 Avoid seek operation when current kv is deleted(Liyin Tang) + HBASE-4486 Improve Javadoc for HTableDescriptor (Akash Ashok) + HBASE-4604 hbase.client.TestHTablePool could start a single + cluster instead of one per method (nkeywal) + HBASE-3929 Add option to HFile tool to produce basic stats (Matteo + Bertozzi and todd via todd) + HBASE-4694 Some cleanup of log messages in RS and M + HBASE-4603 Uneeded sleep time for tests in + hbase.master.ServerManager#waitForRegionServers (nkeywal) + HBASE-4703 Improvements in tests (nkeywal) + HBASE-4611 Add support for Phabricator/Differential as an alternative code review tool + HBASE-3939 Some crossports of Hadoop IPC fixes + HBASE-4756 Enable tab-completion in HBase shell (Ryan Thiessen) + HBASE-4759 Migrate from JUnit 4.8.2 to JUnit 4.10 (nkeywal) + HBASE-4554 Allow set/unset coprocessor table attributes from shell + (Mingjie Lai) + HBASE-4779 TestHTablePool, TestScanWithBloomError, TestRegionSplitCalculator are + not tagged and TestPoolMap should not use TestSuite (N Keywal) + HBASE-4805 Allow better control of resource consumption in HTable (Lars H) + HBASE-4903 Return a result from RegionObserver.preIncrement + (Daniel Gómez Ferro via Lars H) + HBASE-4683 Always cache index and bloom blocks + + TASKS + HBASE-3559 Move report of split to master OFF the heartbeat channel + HBASE-3573 Move shutdown messaging OFF hearbeat; prereq for fix of + hbase-1502 + HBASE-3071 Graceful decommissioning of a regionserver + HBASE-3970 Address HMaster crash/failure half way through meta migration + (Subbu M Iyer) + HBASE-4013 Make ZooKeeperListener Abstract (Akash Ashok via Ted Yu) + HBASE-4025 Server startup fails during startup due to failure in loading + all table descriptors. (Subbu Iyer via Ted Yu) + HBASE-4017 BlockCache interface should be truly modular (Li Pi) + HBASE-4152 Rename o.a.h.h.regionserver.wal.WALObserver to + o.a.h.h.regionserver.wal.WALActionsListener + HBASE-4039 Users should be able to choose custom TableInputFormats without + modifying TableMapReduceUtil.initTableMapperJob() (Brock Noland) + HBASE-4185 Add doc for new hfilev2 format + HBASE-4315 RS requestsPerSecond counter seems to be off (subramanian raghunathan) + HBASE-4289 Move spinlock to SingleSizeCache rather than the slab allocator + (Li Pi) + HBASE-4296 Deprecate HTable[Interface].getRowOrBefore(...) (Lars Hofhansl) + HBASE-2195 Support cyclic replication (Lars Hofhansl) + HBASE-2196 Support more than one slave cluster (Lars Hofhansl) + HBASE-4429 Provide synchronous balanceSwitch() + HBASE-4437 Update hadoop in 0.92 (0.20.205?) + HBASE-4656 Note how dfs.support.append has to be enabled in 0.20.205.0 + clusters + HBASE-4699 Cleanup the UIs + HBASE-4552 Remove trivial 0.90 deprecated code from 0.92 and trunk. + (Jonathan Hsieh) + HBASE-4714 Don't ship w/ icms enabled by default + HBASE-4747 Upgrade maven surefire plugin to 2.10 + HBASE-4288 "Server not running" exception during meta verification causes RS abort + HBASE-4856 Upgrade zookeeper to 3.4.0 release + HBASE-5111 Upgrade zookeeper to 3.4.2 release + HBASE-5125 Upgrade hadoop to 1.0.0 + + NEW FEATURES + HBASE-2001 Coprocessors: Colocate user code with regions (Mingjie Lai via + Andrew Purtell) + HBASE-3287 Add option to cache blocks on hfile write and evict blocks on + hfile close + HBASE-3335 Add BitComparator for filtering (Nathaniel Cook via Stack) + HBASE-3260 Coprocessors: Add explicit lifecycle management + HBASE-3256 Coprocessors: Coprocessor host and observer for HMaster + HBASE-3345 Coprocessors: Allow observers to completely override base + function + HBASE-2824 A filter that randomly includes rows based on a configured + chance (Ferdy via Andrew Purtell) + HBASE-3455 Add memstore-local allocation buffers to combat heap + fragmentation in the region server. Enabled by default as of + 0.91 + HBASE-3257 Coprocessors: Extend server side API to include HLog operations + (Mingjie Lai via Andrew Purtell) + HBASE-3606 Create an package integration project (Eric Yang via Ryan) + HBASE-3488 Add CellCounter to count multiple versions of rows + (Subbu M. Iyer via Stack) + HBASE-1364 [performance] Distributed splitting of regionserver commit logs + (Prakash Khemani) + HBASE-3836 Add facility to track currently progressing actions and + workflows. (todd) + HBASE-3837 Show regions in transition on the master web page (todd) + HBASE-3839 Add monitoring of currently running tasks to the master and + RS web UIs + HBASE-3691 Add compressor support for 'snappy', google's compressor + (Nichole Treadway and Nicholas Telford) + HBASE-2233 Support both Hadoop 0.20 and 0.22 + HBASE-3857 Change the HFile Format (Mikhail & Liyin) + HBASE-4114 Metrics for HFile HDFS block locality (Ming Ma) + HBASE-4176 Exposing HBase Filters to the Thrift API (Anirudh Todi) + HBASE-4221 Changes necessary to build and run against Hadoop 0.23 + (todd) + HBASE-4071 Data GC: Remove all versions > TTL EXCEPT the last + written version (Lars Hofhansl) + HBASE-4242 Add documentation for HBASE-4071 (Lars Hofhansl) + HBASE-4027 Enable direct byte buffers LruBlockCache (Li Pi) + HBASE-4117 Slow Query Log and Client Operation Fingerprints + (Riley Patterson) + HBASE-4292 Add a debugging dump servlet to the master and regionserver + (todd) + HBASE-4057 Implement HBase version of "show processlist" (Riley Patterson) + HBASE-4219 Per Column Family Metrics + HBASE-4219 Addendum for failure of TestHFileBlock + HBASE-4377 [hbck] Offline rebuild .META. from fs data only + (Jonathan Hsieh) + HBASE-4298 Support to drain RS nodes through ZK (Aravind Gottipati) + HBASE-2742 Provide strong authentication with a secure RPC engine + HBASE-3025 Coprocessor based access control + +Release 0.90.7 - Unreleased + + BUG FIXES + HBASE-5271 Result.getValue and Result.getColumnLatest return the wrong column (Ghais Issa) + +Release 0.90.6 - Unreleased + + BUG FIXES + HBASE-4970 Add a parameter so that keepAliveTime of Htable thread pool can be changed (gaojinchao) + HBASE-5060 HBase client is blocked forever (Jinchao) + HBASE-5009 Failure of creating split dir if it already exists prevents splits from happening further + HBASE-5041 Major compaction on non existing table does not throw error (Shrijeet) + HBASE-5327 Print a message when an invalid hbase.rootdir is passed (Jimmy Xiang) + +Release 0.90.5 - Released + + BUG FIXES + HBASE-4160 HBase shell move and online may be unusable if region name + or server includes binary-encoded data (Jonathan Hsieh) + HBASE-4168 A client continues to try and connect to a powered down + regionserver (Anirudh Todi) + HBASE-4196 TableRecordReader may skip first row of region (Ming Ma) + HBASE-4170 createTable java doc needs to be improved (Mubarak Seyed) + HBASE-4144 RS does not abort if the initialization of RS fails + (ramkrishna.s.vasudevan) + HBASE-4148 HFileOutputFormat doesn't fill in TIMERANGE_KEY metadata + (Jonathan Hsieh) + HBASE-4159 HBaseServer - IPC Reader threads are not daemons (Douglas + Campbell) + HBASE-4095 Hlog may not be rolled in a long time if checkLowReplication's + request of LogRoll is blocked (Jieshan Bean) + HBASE-4253 TestScannerTimeOut.test3686a and TestHTablePool. + testReturnDifferentTable() failure because of using new + HTable(tablename) (ramkrishna.s.vasudevan) + HBASE-4124 ZK restarted while a region is being assigned, new active HM + re-assigns it but the RS warns 'already online on this server' + (Gaojinchao) + HBASE-4294 HLogSplitter sleeps with 1-second granularity (todd) + HBASE-4270 IOE ignored during flush-on-close causes dataloss + HBASE-4180 HBase should check the isSecurityEnabled flag before login + HBASE-4325 Improve error message when using STARTROW for meta scans + (Jonathan Hsieh) + HBASE-4238 CatalogJanitor can clear a daughter that split before + processing its parent + HBASE-4445 Not passing --config when checking if distributed mode or not + HBASE-4453 TestReplication failing up on builds.a.o because already + running zk with new format root servername + HBASE-4387 Error while syncing: DFSOutputStream is closed + (Lars Hofhansl) + HBASE-4295 rowcounter does not return the correct number of rows in + certain circumstances (David Revell) + HBASE-4515 User.getCurrent() can fail to initialize the current user + HBASE-4473 NPE when executors are down but events are still coming in + HBASE-4537 TestUser imports breaking build against secure Hadoop + HBASE-4501 [replication] Shutting down a stream leaves recovered + sources running + HBASE-4563 When error occurs in this.parent.close(false) of split, + the split region cannot write or read (bluedavy via Lars H) + HBASE-4570. Fix a race condition that could cause inconsistent results + from scans during concurrent writes. (todd and Jonathan Jsieh + via todd) + HBASE-4562 When split doing offlineParentInMeta encounters error, it'll + cause data loss (bluedavy via Lars H) + HBASE-4800 Result.compareResults is incorrect (James Taylor and Lars H) + HBASE-4848 TestScanner failing because hostname can't be null + HBASE-4862 Splitting hlog and opening region concurrently may cause data loss + (Chunhui Shen) + HBASE-4773 HBaseAdmin may leak ZooKeeper connections (Xufeng) + + IMPROVEMENT + HBASE-4205 Enhance HTable javadoc (Eric Charles) + HBASE-4222 Make HLog more resilient to write pipeline failures + HBASE-4293 More verbose logging in ServerShutdownHandler for meta/root + cases (todd) + HBASE-4276 AssignmentManager debug logs should be at INFO level for + META/ROOT regions (todd) + HBASE-4323 Add debug logging when AssignmentManager can't make a plan + for a region (todd) + HBASE-4313 Refactor TestHBaseFsck to make adding individual hbck tests + easier (Jonathan Hsieh) + HBASE-4272. Add -metaonly flag to hbck feature to only inspect and try + to repair META and ROOT. (todd) + HBASE-4321. Add a more comprehensive region split calculator for future use + in hbck. (Jonathan Hsieh) + HBASE-4384 Hard to tell what causes failure in CloseRegionHandler#getCurrentVersion + (Harsh J) + HBASE-4375 [hbck] Add region coverage visualization to hbck + (Jonathan Hsieh) + HBASE-4506 [hbck] Allow HBaseFsck to be instantiated without connecting + (Jonathan Hsieh) + HBASE-4509 [hbck] Improve region map output + (Jonathan Hsieh) + HBASE-4806 Fix logging message in HbaseObjectWritable + (Jonathan Hsieh via todd) + +Release 0.90.4 - August 10, 2011 + + BUG FIXES + HBASE-3878 Hbase client throws NoSuchElementException (Ted Yu) + HBASE-3881 Add disable balancer in graceful_stop.sh script + HBASE-3895 Fix order of parameters after HBASE-1511 + HBASE-3874 ServerShutdownHandler fails on NPE if a plan has a random + region assignment + HBASE-3902 Add Bytes.toBigDecimal and Bytes.toBytes(BigDecimal) + (Vaibhav Puranik) + HBASE-3820 Splitlog() executed while the namenode was in safemode may + cause data-loss (Jieshan Bean) + HBASE-3905 HBaseAdmin.createTableAsync() should check for invalid split + keys (Ted Yu) + HBASE-3908 TableSplit not implementing "hashCode" problem (Daniel Iancu) + HBASE-3915 Binary row keys in hbck and other miscellaneous binary key + display issues + HBASE-3914 ROOT region appeared in two regionserver's onlineRegions at + the same time (Jieshan Bean) + HBASE-3934 MemStoreFlusher.getMemStoreLimit() doesn't honor defaultLimit + (Ted Yu) + HBASE-3946 The splitted region can be online again while the standby + hmaster becomes the active one (Jieshan Bean) + HBASE-3916 Fix the default bind address of ThriftServer to be wildcard + instead of localhost. (Li Pi) + HBASE-3985 Same Region could be picked out twice in LoadBalance + (Jieshan Bean) + HBASE-3987 Fix a NullPointerException on a failure to load Bloom filter data + (Mikhail Bautin) + HBASE-3948 Improve split/compact result page for RegionServer status page + (Li Pi) + HBASE-3988 Infinite loop for secondary master (Liyin Tang) + HBASE-3995 HBASE-3946 broke TestMasterFailover + HBASE-2077 NullPointerException with an open scanner that expired causing + an immediate region server shutdown -- part 2. + HBASE-4005 close_region bugs + HBASE-4028 Hmaster crashes caused by splitting log. + (gaojinchao via Ted Yu) + HBASE-4035 Fix local-master-backup.sh - parameter order wrong + (Lars George via Ted Yu) + HBASE-4020 "testWritesWhileGetting" unit test needs to be fixed. + (Vandana Ayyalasomayajula via Ted Yu) + HBASE-3984 CT.verifyRegionLocation isn't doing a very good check, + can delay cluster recovery + HBASE-4045 [replication] NPE in ReplicationSource when ZK is gone + HBASE-4034 HRegionServer should be stopped even if no META regions + are hosted by the HRegionServer (Akash Ashok) + HBASE-4033 The shutdown RegionServer could be added to + AssignmentManager.servers again (Jieshan Bean) + HBASE-4088 npes in server shutdown + HBASE-3872 Hole in split transaction rollback; edits to .META. need + to be rolled back even if it seems like they didn't make it + HBASE-4101 Regionserver Deadlock (ramkrishna.s.vasudevan) + HBASE-4115 HBase shell assign and unassign unusable if region name + includes binary-encoded data (Ryan Brush) + HBASE-4126 Make timeoutmonitor timeout after 30 minutes instead of 3 + HBASE-4129 HBASE-3872 added a warn message 'CatalogJanitor: Daughter regiondir + does not exist' that is triggered though its often legit that daughter + is not present + + IMPROVEMENT + HBASE-3882 hbase-config.sh needs to be updated so it can auto-detects the + sun jre provided by RHEL6 (Roman Shaposhnik) + HBASE-3920 HLog hbase.regionserver.flushlogentries no longer supported + (Dave Latham) + HBASE-3919 More places output binary data to text (Dave Latham) + HBASE-3873 HBase IRB shell: Don't pretty-print the output when stdout + isn't a TTY (Benoît Sigoure) + HBASE-3969 Outdated data can not be cleaned in time (Zhou Shuaifeng) + HBASE-3968 HLog Pretty Printer (Riley Patterson) + +Release 0.90.3 - May 19th, 2011 + + BUG FIXES + HBASE-3746 Clean up CompressionTest to not directly reference + DistributedFileSystem (todd) + HBASE-3734 HBaseAdmin creates new configurations in getCatalogTracker + HBASE-3756 Can't move META or ROOT from shell + HBASE-3740 hbck doesn't reset the number of errors when retrying + HBASE-3744 createTable blocks until all regions are out of transition + (Ted Yu via Stack) + HBASE-3750 HTablePool.putTable() should call releaseHTableInterface() + for discarded tables (Ted Yu via garyh) + HBASE-3755 Catch zk's ConnectionLossException and augment error + message with more help + HBASE-3722 A lot of data is lost when name node crashed (gaojinchao) + HBASE-3771 All jsp pages don't clean their HBA + HBASE-3685 when multiple columns are combined with TimestampFilter, only + one column is returned (Jerry Chen) + HBASE-3708 createAndFailSilent is not so silent; leaves lots of logging + in ensemble logs (Dmitriy Ryaboy) + HBASE-3783 hbase-0.90.2.jar exists in hbase root and in 'lib/' + HBASE-3539 Improve shell help to reflect all possible options + (Harsh J Chouraria) + HBASE-3817 HBase Shell has an issue accepting FILTER for the 'scan' command. + (Harsh J Chouraria) + HBASE-3634 Fix JavaDoc for put(List puts) in HTableInterface + (Harsh J Chouraria) + HBASE-3749 Master can't exit when open port failed (gaojinchao) + HBASE-3794 TestRpcMetrics fails on machine where region server is running + (Alex Newman) + HBASE-3741 Make HRegionServer aware of the regions it's opening/closing + HBASE-3597 ageOfLastAppliedOp should update after cluster replication + failures + HBASE-3821 "NOT flushing memstore for region" keep on printing for half + an hour (zhoushuaifeng) + + IMPROVEMENTS + HBASE-3747 ReplicationSource should differanciate remote and local exceptions + HBASE-3652 Speed up tests by lowering some sleeps + HBASE-3767 Improve how HTable handles threads used for multi actions + HBASE-3795 Remove the "Cache hit for row" message + HBASE-3580 Remove RS from DeadServer when new instance checks in + HBASE-2470 Add Scan.setTimeRange() support in Shell (Harsh J Chouraria) + HBASE-3805 Log RegionState that are processed too late in the master + HBASE-3695 Some improvements to Hbck to test the entire region chain in + Meta and provide better error reporting (Marc Limotte) + HBASE-3813 Change RPC callQueue size from 'handlerCount * + MAX_QUEUE_SIZE_PER_HANDLER;' + HBASE-3860 HLog shouldn't create a new HBC when rolling + + TASKS + HBASE-3748 Add rolling of thrift/rest daemons to graceful_stop.sh script + HBASE-3846 Set RIT timeout higher + +Release 0.90.2 - 20110408 + + BUG FIXES + HBASE-3545 Possible liveness issue with MasterServerAddress in + HRegionServer getMaster (Greg Bowyer via Stack) + HBASE-3548 Fix type in documentation of pseudo distributed mode + HBASE-3553 HTable ThreadPoolExecutor does not properly initialize + for hbase.htable.threads.max threads + (Himanshu Vashishtha via garyh) + HBASE-3566 writeToWAL is not serialized for increment operation + HBASE-3576 MasterAddressTracker is registered to ZooKeeperWatcher twice + HBASE-3561 OPTS arguments are duplicated + HBASE-3572 memstore lab can leave half inited data structs (bad!) + HBASE-3589 test jar should not include mapred-queues.xml and + log4j.properties + HBASE-3593 DemoClient.cpp is outdated + HBASE-3591 completebulkload doesn't honor generic -D options + HBASE-3594 Rest server fails because of missing asm jar + HBASE-3582 Allow HMaster and HRegionServer to login from keytab + when on secure Hadoop + HBASE-3608 MemstoreFlusher error message doesnt include exception! + HBASE-1960 Master should wait for DFS to come up when creating + hbase.version; use alternate strategy for waiting for DNs + HBASE-3612 HBaseAdmin::isTableAvailable returns true when the table does + not exit + HBASE-3626 Update instructions in thrift demo files (Moaz Reyad via Stack) + HBASE-3633 ZKUtil::createSetData should only create a node when it + nonexists (Guanpeng Xu via Stack) + HBASE-3636 a bug about deciding whether this key is a new key for the ROWCOL + bloomfilter (Liyin Tang via Stack) + HBASE-3639 FSUtils.getRootDir should qualify path + HBASE-3648 [replication] failover is sloppy with znodes + HBASE-3613 NPE in MemStoreFlusher + HBASE-3650 HBA.delete can return too fast + HBASE-3659 Fix TestHLog to pass on newer versions of Hadoop + HBASE-3595 get_counter broken in shell + HBASE-3664 [replication] Adding a slave when there's none may kill the cluster + HBASE-3671 Split report before we finish parent region open; workaround + till 0.92; Race between split and OPENED processing + HBASE-3674 Treat ChecksumException as we would a ParseException splitting + logs; else we replay split on every restart + HBASE-3621 The timeout handler in AssignmentManager does an RPC while + holding lock on RIT; a big no-no (Ted Yu via Stack) + HBASE-3575 Update rename table script + HBASE-3687 Bulk assign on startup should handle a ServerNotRunningException + HBASE-3617 NoRouteToHostException during balancing will cause Master abort + (Ted Yu via Stack) + HBASE-3668 CatalogTracker.waitForMeta can wait forever and totally stall a RS + HBASE-3627 NPE in EventHandler when region already reassigned + HBASE-3660 HMaster will exit when starting with stale data in cached locations + such as -ROOT- or .META. + HBASE-3654 Weird blocking between getOnlineRegion and createRegionLoad + (Subbu M Iyer via Stack) + HBASE-3666 TestScannerTimeout fails occasionally + HBASE-3497 TableMapReduceUtil.initTableReducerJob broken due to setConf + method in TableOutputFormat + HBASE-3686 ClientScanner skips too many rows on recovery if using scanner + caching (Sean Sechrist via Stack) + + IMPROVEMENTS + HBASE-3542 MultiGet methods in Thrift + HBASE-3586 Improve the selection of regions to balance (Ted Yu via Andrew + Purtell) + HBASE-3603 Remove -XX:+HeapDumpOnOutOfMemoryError autodump of heap option + on OOME + HBASE-3285 Hlog recovery takes too much time + HBASE-3623 Allow non-XML representable separator characters in the ImportTSV tool + (Harsh J Chouraria via Stack) + HBASE-3620 Make HBCK utility faster + HBASE-3625 improve/fix support excluding Tests via Maven -D property + (Alejandro Abdelnur via todd) + HBASE-3437 Support Explict Split Points from the Shell + HBASE-3448 RegionSplitter, utility class to manually split tables + HBASE-3610 Improve RegionSplitter performance + HBASE-3496 HFile CLI Improvements + HBASE-3596 [replication] Wait a few seconds before transferring queues + HBASE-3600 Update our jruby to 1.6.0 + HBASE-3640 [replication] Transferring queues shouldn't be done inline with RS startup + HBASE-3658 Alert when heap is over committed (Subbu M Iyer via Stack) + HBASE-3681 Check the sloppiness of the region load before balancing (Ted Yu via JD) + HBASE-3703 hbase-config.sh needs to be updated so it can auto-detect + the sun jdk provided by RHEL6 (Bruno Mahe via todd) + +Release 0.90.1 - February 9th, 2011 + + NEW FEATURES + HBASE-3455 Add memstore-local allocation buffers to combat heap + fragmentation in the region server. Experimental / disabled + by default in 0.90.1 + + BUG FIXES + HBASE-3445 Master crashes on data that was moved from different host + HBASE-3449 Server shutdown handlers deadlocked waiting for META + HBASE-3456 Fix hardcoding of 20 second socket timeout down in HBaseClient + HBASE-3476 HFile -m option need not scan key values + (Prakash Khemani via Lars George) + HBASE-3481 max seq id in flushed file can be larger than its correct value + causing data loss during recovery + HBASE-3493 HMaster sometimes hangs during initialization due to missing + notify call (Bruno Dumon via Stack) + HBASE-3483 Memstore lower limit should trigger asynchronous flushes + HBASE-3494 checkAndPut implementation doesnt verify row param and writable + row are the same + HBASE-3416 For intra-row scanning, the update readers notification resets + the query matcher and can lead to incorrect behavior + HBASE-3495 Shell is failing on subsequent split calls + HBASE-3502 Can't open region because can't open .regioninfo because + AlreadyBeingCreatedException + HBASE-3501 Remove the deletion limit in LogCleaner + HBASE-3500 Documentation update for replicatio + HBASE-3419 If re-transition to OPENING during log replay fails, server + aborts. Instead, should just cancel region open. + HBASE-3524 NPE from CompactionChecker + HBASE-3531 When under global memstore pressure, dont try to flush + unflushable regions. + HBASE-3550 FilterList reports false positives (Bill Graham via Andrew + Purtell) + + IMPROVEMENTS + HBASE-3305 Allow round-robin distribution for table created with + multiple regions (ted yu via jgray) + HBASE-3508 LruBlockCache statistics thread should have a name + HBASE-3511 Allow rolling restart to apply to only RS or only masters + HBASE-3510 Add thread name for IPC reader threads + HBASE-3509 Add metric for flush queue length + HBASE-3517 Store build version in hbase-default.xml and verify at runtime + +Release 0.90.0 - January 19th, 2011 + INCOMPATIBLE CHANGES + HBASE-1822 Remove the deprecated APIs + HBASE-1848 Fixup shell for HBASE-1822 + HBASE-1854 Remove the Region Historian + HBASE-1930 Put.setTimeStamp misleading (doesn't change timestamp on + existing KeyValues, not copied in copy constructor) + (Dave Latham via Stack) + HBASE-1360 move up to Thrift 0.2.0 (Kay Kay and Lars Francke via Stack) + HBASE-2212 Refactor out lucene dependencies from HBase + (Kay Kay via Stack) + HBASE-2219 stop using code mapping for method names in the RPC + HBASE-1728 Column family scoping and cluster identification + HBASE-2099 Move build to Maven (Paul Smith via Stack) + HBASE-2260 Remove all traces of Ant and Ivy (Lars Francke via Stack) + HBASE-2255 take trunk back to hadoop 0.20 + HBASE-2378 Bulk insert with multiple reducers broken due to improper + ImmutableBytesWritable comparator (Todd Lipcon via Stack) + HBASE-2392 Upgrade to ZooKeeper 3.3.0 + HBASE-2294 Enumerate ACID properties of HBase in a well defined spec + (Todd Lipcon via Stack) + HBASE-2541 Remove transactional contrib (Clint Morgan via Stack) + HBASE-2542 Fold stargate contrib into core + HBASE-2565 Remove contrib module from hbase + HBASE-2397 Bytes.toStringBinary escapes printable chars + HBASE-2771 Update our hadoop jar to be latest from 0.20-append branch + HBASE-2803 Remove remaining Get code from Store.java,etc + HBASE-2553 Revisit IncrementColumnValue implementation in 0.22 + HBASE-2692 Master rewrite and cleanup for 0.90 + (Karthik Ranganathan, Jon Gray & Stack) + HBASE-2961 Close zookeeper when done with it (HCM, Master, and RS) + HBASE-2641 HBASE-2641 Refactor HLog splitLog, hbase-2437 continued; + break out split code as new classes + (James Kennedy via Stack) + + BUG FIXES + HBASE-1791 Timeout in IndexRecordWriter (Bradford Stephens via Andrew + Purtell) + HBASE-1737 Regions unbalanced when adding new node (recommit) + HBASE-1792 [Regression] Cannot save timestamp in the future + HBASE-1793 [Regression] HTable.get/getRow with a ts is broken + HBASE-1698 Review documentation for o.a.h.h.mapreduce + HBASE-1798 [Regression] Unable to delete a row in the future + HBASE-1790 filters are not working correctly (HBASE-1710 HBASE-1807 too) + HBASE-1779 ThriftServer logged error if getVer() result is empty + HBASE-1778 Improve PerformanceEvaluation (Schubert Zhang via Stack) + HBASE-1751 Fix KeyValue javadoc on getValue for client-side + HBASE-1795 log recovery doesnt reset the max sequence id, new logfiles can + get tossed as 'duplicates' + HBASE-1794 recovered log files are not inserted into the storefile map + HBASE-1824 [stargate] default timestamp should be LATEST_TIMESTAMP + HBASE-1740 ICV has a subtle race condition only visible under high load + HBASE-1808 [stargate] fix how columns are specified for scanners + HBASE-1828 CompareFilters are broken from client-side + HBASE-1836 test of indexed hbase broken + HBASE-1838 [javadoc] Add javadoc to Delete explaining behavior when no + timestamp provided + HBASE-1821 Filtering by SingleColumnValueFilter bug + HBASE-1840 RowLock fails when used with IndexTable + (Keith Thomas via Stack) + HBASE-818 HFile code review and refinement (Schubert Zhang via Stack) + HBASE-1830 HbaseObjectWritable methods should allow null HBCs + for when Writable is not Configurable (Stack via jgray) + HBASE-1847 Delete latest of a null qualifier when non-null qualifiers + exist throws a RuntimeException + HBASE-1850 src/examples/mapred do not compile after HBASE-1822 + HBASE-1853 Each time around the regionserver core loop, we clear the + messages to pass master, even if we failed to deliver them + HBASE-1815 HBaseClient can get stuck in an infinite loop while attempting + to contact a failed regionserver + HBASE-1856 HBASE-1765 broke MapReduce when using Result.list() + (Lars George via Stack) + HBASE-1857 WrongRegionException when setting region online after .META. + split (Cosmin Lehane via Stack) + HBASE-1809 NPE thrown in BoundedRangeFileInputStream + HBASE-1859 Misc shell fixes patch (Kyle Oba via Stack) + HBASE-1865 0.20.0 TableInputFormatBase NPE + HBASE-1866 Scan(Scan) copy constructor does not copy value of + cacheBlocks + HBASE-1869 IndexedTable delete fails when used in conjunction with + RowLock (Keith Thomas via Stack) + HBASE-1858 Master can't split logs created by THBase (Clint Morgan via + Andrew Purtell) + HBASE-1871 Wrong type used in TableMapReduceUtil.initTableReduceJob() + (Lars George via Stack) + HBASE-1883 HRegion passes the wrong minSequenceNumber to + doReconstructionLog (Clint Morgan via Stack) + HBASE-1878 BaseScanner results can't be trusted at all (Related to + hbase-1784) + HBASE-1831 Scanning API must be reworked to allow for fully functional + Filters client-side + HBASE-1890 hbase-1506 where assignment is done at regionserver doesn't + work + HBASE-1889 ClassNotFoundException on trunk for REST + HBASE-1905 Remove unused config. hbase.hstore.blockCache.blockSize + HBASE-1906 FilterList of prefix and columnvalue not working properly with + deletes and multiple values + HBASE-1896 WhileMatchFilter.reset should call encapsulated filter reset + HBASE-1912 When adding a secondary index to an existing table, it will + cause NPE during re-indexing (Mingjui Ray Liao via Andrew + Purtell) + HBASE-1916 FindBugs and javac warnings cleanup + HBASE-1908 ROOT not reassigned if only one regionserver left + HBASE-1915 HLog.sync is called way too often, needs to be only called one + time per RPC + HBASE-1777 column length is not checked before saved to memstore + HBASE-1925 IllegalAccessError: Has not been initialized (getMaxSequenceId) + HBASE-1929 If hbase-default.xml is not in CP, zk session timeout is 10 + seconds! + HBASE-1927 Scanners not closed properly in certain circumstances + HBASE-1934 NullPointerException in ClientScanner (Andrew Purtell via Stack) + HBASE-1946 Unhandled exception at regionserver (Dmitriy Lyfar via Stack) + HBASE-1682 IndexedRegion does not properly handle deletes + (Andrew McCall via Clint Morgan and Stack) + HBASE-1953 Overhaul of overview.html (html fixes, typos, consistency) - + no content changes (Lars Francke via Stack) + HBASE-1954 Transactional scans do not see newest put (Clint Morgan via + Stack) + HBASE-1919 code: HRS.delete seems to ignore exceptions it shouldnt + HBASE-1951 Stack overflow when calling HTable.checkAndPut() + when deleting a lot of values + HBASE-1781 Weird behavior of WildcardColumnTracker.checkColumn(), + looks like recursive loop + HBASE-1949 KeyValue expiration by Time-to-Live during major compaction is + broken (Gary Helmling via Stack) + HBASE-1957 Get-s can't set a Filter + HBASE-1928 ROOT and META tables stay in transition state (making the system + not usable) if the designated regionServer dies before the + assignment is complete (Yannis Pavlidis via Stack) + HBASE-1962 Bulk loading script makes regions incorrectly (loadtable.rb) + HBASE-1966 Apply the fix from site/ to remove the forrest dependency on + Java 5 + HBASE-1967 [Transactional] client.TestTransactions.testPutPutScan fails + sometimes -- Temporary fix + HBASE-1841 If multiple of same key in an hfile and they span blocks, may + miss the earlier keys on a lookup + (Schubert Zhang via Stack) + HBASE-1977 Add ts and allow setting VERSIONS when scanning in shell + HBASE-1979 MurmurHash does not yield the same results as the reference C++ + implementation when size % 4 >= 2 (Olivier Gillet via Andrew + Purtell) + HBASE-1999 When HTable goes away, close zk session in shutdown hook or + something... + HBASE-1997 zk tick time bounds maximum zk session time + HBASE-2003 [shell] deleteall ignores column if specified + HBASE-2018 Updates to .META. blocked under high MemStore load + HBASE-1994 Master will lose hlog entries while splitting if region has + empty oldlogfile.log (Lars George via Stack) + HBASE-2022 NPE in housekeeping kills RS + HBASE-2034 [Bulk load tools] loadtable.rb calls an undefined method + 'descendingIterator' (Ching-Shen Chen via Stack) + HBASE-2033 Shell scan 'limit' is off by one + HBASE-2040 Fixes to group commit + HBASE-2047 Example command in the "Getting Started" + documentation doesn't work (Benoit Sigoure via JD) + HBASE-2048 Small inconsistency in the "Example API Usage" + (Benoit Sigoure via JD) + HBASE-2044 HBASE-1822 removed not-deprecated APIs + HBASE-1960 Master should wait for DFS to come up when creating + hbase.version + HBASE-2054 memstore size 0 is >= than blocking -2.0g size + HBASE-2064 Cannot disable a table if at the same the Master is moving + its regions around + HBASE-2065 Cannot disable a table if any of its region is opening + at the same time + HBASE-2026 NPE in StoreScanner on compaction + HBASE-2072 fs.automatic.close isn't passed to FileSystem + HBASE-2075 Master requires HDFS superuser privileges due to waitOnSafeMode + HBASE-2077 NullPointerException with an open scanner that expired causing + an immediate region server shutdown (Sam Pullara via JD) + HBASE-2078 Add JMX settings as commented out lines to hbase-env.sh + (Lars George via JD) + HBASE-2082 TableInputFormat is ignoring input scan's stop row setting + (Scott Wang via Andrew Purtell) + HBASE-2068 MetricsRate is missing "registry" parameter + (Lars George and Gary Helmling via Stack) + HBASE-2093 [stargate] RowSpec parse bug + HBASE-2114 Can't start HBase in trunk (JD and Kay Kay via JD) + HBASE-2115 ./hbase shell would not launch due to missing jruby dependency + (Kay Kay via JD) + HBASE-2101 KeyValueSortReducer collapses all values to last passed + HBASE-2119 Fix top-level NOTICES.txt file. Its stale. + HBASE-2120 [stargate] Unable to delete column families (Greg Lu via Andrew + Purtell) + HBASE-2123 Remove 'master' command-line option from PE + HBASE-2024 [stargate] Deletes not working as expected (Greg Lu via Andrew + Purtell) + HBASE-2122 [stargate] Initializing scanner column families doesn't work + (Greg Lu via Andrew Purtell) + HBASE-2124 Useless exception in HMaster on startup + HBASE-2127 randomWrite mode of PerformanceEvaluation benchmark program + writes only to a small range of keys (Kannan Muthukkaruppan + via Stack) + HBASE-2126 Fix build break - ec2 (Kay Kay via JD) + HBASE-2134 Ivy nit regarding checking with latest snapshots (Kay Kay via + Andrew Purtell) + HBASE-2138 unknown metrics type (Stack via JD) + HBASE-2137 javadoc warnings from 'javadoc' target (Kay Kay via Stack) + HBASE-2135 ant javadoc complains about missing classe (Kay Kay via Stack) + HBASE-2130 bin/* scripts - not to include lib/test/**/*.jar + (Kay Kay via Stack) + HBASE-2140 findbugs issues - 2 performance warnings as suggested by + findbugs (Kay Kay via Stack) + HBASE-2139 findbugs task in build.xml (Kay Kay via Stack) + HBASE-2147 run zookeeper in the same jvm as master during non-distributed + mode + HBASE-65 Thrift Server should have an option to bind to ip address + (Lars Francke via Stack) + HBASE-2146 RPC related metrics are missing in 0.20.3 since recent changes + (Gary Helmling via Lars George) + HBASE-2150 Deprecated HBC(Configuration) constructor doesn't call this() + HBASE-2154 Fix Client#next(int) javadoc + HBASE-2152 Add default jmxremote.{access|password} files into conf + (Lars George and Gary Helmling via Stack) + HBASE-2156 HBASE-2037 broke Scan - only a test for trunk + HBASE-2057 Cluster won't stop (Gary Helmling and JD via JD) + HBASE-2160 Can't put with ts in shell + HBASE-2144 Now does \x20 for spaces + HBASE-2163 ZK dependencies - explicitly add them until ZK artifacts are + published to mvn repository (Kay Kay via Stack) + HBASE-2164 Ivy nit - clean up configs (Kay Kay via Stack) + HBASE-2184 Calling HTable.getTableDescriptor().* on a full cluster takes + a long time (Cristian Ivascu via Stack) + HBASE-2193 Better readability of - hbase.regionserver.lease.period + (Kay Kay via Stack) + HBASE-2199 hbase.client.tableindexed.IndexSpecification, lines 72-73 + should be reversed (Adrian Popescu via Stack) + HBASE-2224 Broken build: TestGetRowVersions.testGetRowMultipleVersions + HBASE-2129 ant tar build broken since switch to Ivy (Kay Kay via Stack) + HBASE-2226 HQuorumPeerTest doesnt run because it doesnt start with the + word Test + HBASE-2230 SingleColumnValueFilter has an ungaurded debug log message + HBASE-2258 The WhileMatchFilter doesn't delegate the call to filterRow() + HBASE-2259 StackOverflow in ExplicitColumnTracker when row has many columns + HBASE-2268 [stargate] Failed tests and DEBUG output is dumped to console + since move to Mavenized build + HBASE-2276 Hbase Shell hcd() method is broken by the replication scope + parameter (Alexey Kovyrin via Lars George) + HBASE-2244 META gets inconsistent in a number of crash scenarios + HBASE-2284 fsWriteLatency metric may be incorrectly reported + (Kannan Muthukkaruppan via Stack) + HBASE-2063 For hfileoutputformat, on timeout/failure/kill clean up + half-written hfile (Ruslan Salyakhov via Stack) + HBASE-2281 Hbase shell does not work when started from the build dir + (Alexey Kovyrin via Stack) + HBASE-2293 CME in RegionManager#isMetaServer + HBASE-2261 The javadoc in WhileMatchFilter and it's tests in TestFilter + are not accurate/wrong + HBASE-2299 [EC2] mapreduce fixups for PE + HBASE-2295 Row locks may deadlock with themselves + (dhruba borthakur via Stack) + HBASE-2308 Fix the bin/rename_table.rb script, make it work again + HBASE-2307 hbase-2295 changed hregion size, testheapsize broke... fix it + HBASE-2269 PerformanceEvaluation "--nomapred" may assign duplicate random + seed over multiple testing threads (Tatsuya Kawano via Stack) + HBASE-2287 TypeError in shell (Alexey Kovyrin via Stack) + HBASE-2023 Client sync block can cause 1 thread of a multi-threaded client + to block all others (Karthik Ranganathan via Stack) + HBASE-2305 Client port for ZK has no default (Suraj Varma via Stack) + HBASE-2323 filter.RegexStringComparator does not work with certain bytes + (Benoit Sigoure via Stack) + HBASE-2313 Nit-pick about hbase-2279 shell fixup, if you do get with + non-existant column family, throws lots of exceptions + (Alexey Kovyrin via Stack) + HBASE-2334 Slimming of Maven dependency tree - improves assembly build + speed (Paul Smith via Stack) + HBASE-2336 Fix build broken with HBASE-2334 (Lars Francke via Lars George) + HBASE-2283 row level atomicity (Kannan Muthukkaruppan via Stack) + HBASE-2355 Unsynchronized logWriters map is mutated from several threads in + HLog splitting (Todd Lipcon via Andrew Purtell) + HBASE-2358 Store doReconstructionLog will fail if oldlogfile.log is empty + and won't load region (Cosmin Lehene via Stack) + HBASE-2370 saveVersion.sh doesnt properly grab the git revision + HBASE-2373 Remove confusing log message of how "BaseScanner GET got + different address/startcode than SCAN" + HBASE-2361 WALEdit broke replication scope + HBASE-2365 Double-assignment around split + HBASE-2398 NPE in HLog.append when calling writer.getLength + (Kannan Muthukkaruppan via Stack) + HBASE-2410 spurious warnings from util.Sleeper + HBASE-2335 mapred package docs don't say zookeeper jar is a dependent + HBASE-2417 HCM.locateRootRegion fails hard on "Connection refused" + HBASE-2346 Usage of FilterList slows down scans + HBASE-2341 ZK settings for initLimit/syncLimit should not have been removed + from hbase-default.xml + HBASE-2439 HBase can get stuck if updates to META are blocked + (Kannan Muthukkaruppan via Stack) + HBASE-2451 .META. by-passes cache; BLOCKCACHE=>'false' + HBASE-2453 Revisit compaction policies after HBASE-2248 commit + (Jonathan Gray via Stack) + HBASE-2458 Client stuck in TreeMap,remove (Todd Lipcon via Stack) + HBASE-2460 add_table.rb deletes any tables for which the target table name + is a prefix (Todd Lipcon via Stack) + HBASE-2463 Various Bytes.* functions silently ignore invalid arguments + (Benoit Sigoure via Stack) + HBASE-2443 IPC client can throw NPE if socket creation fails + (Todd Lipcon via Stack) + HBASE-2447 LogSyncer.addToSyncQueue doesn't check if syncer is still + running before waiting (Todd Lipcon via Stack) + HBASE-2494 Does not apply new.name parameter to CopyTable + (Yoonsik Oh via Stack) + HBASE-2481 Client is not getting UnknownScannerExceptions; they are + being eaten (Jean-Daniel Cryans via Stack) + HBASE-2448 Scanner threads are interrupted without acquiring lock properly + (Todd Lipcon via Stack) + HBASE-2491 master.jsp uses absolute links to table.jsp. This broke when + master.jsp moved under webapps/master(Cristian Ivascu via Stack) + HBASE-2487 Uncaught exceptions in receiving IPC responses orphan clients + (Todd Lipcon via Stack) + HBASE-2497 ProcessServerShutdown throws NullPointerException for offline + regiond (Miklos Kurucz via Stack) + HBASE-2499 Race condition when disabling a table leaves regions in transition + HBASE-2489 Make the "Filesystem needs to be upgraded" error message more + useful (Benoit Sigoure via Stack) + HBASE-2482 regions in transition do not get reassigned by master when RS + crashes (Todd Lipcon via Stack) + HBASE-2513 hbase-2414 added bug where we'd tight-loop if no root available + HBASE-2503 PriorityQueue isn't thread safe, KeyValueHeap uses it that way + HBASE-2431 Master does not respect generation stamps, may result in meta + getting permanently offlined + HBASE-2515 ChangeTableState considers split&&offline regions as being served + HBASE-2544 Forward port branch 0.20 WAL to TRUNK + HBASE-2546 Specify default filesystem in both the new and old way (needed + if we are to run on 0.20 and 0.21 hadoop) + HBASE-1895 HConstants.MAX_ROW_LENGTH is incorrectly 64k, should be 32k + HBASE-1968 Give clients access to the write buffer + HBASE-2028 Add HTable.incrementColumnValue support to shell + (Lars George via Andrew Purtell) + HBASE-2138 unknown metrics type + HBASE-2551 Forward port fixes that are in branch but not in trunk (part of + the merge of old 0.20 into TRUNK task) -- part 1. + HBASE-2474 Bug in HBASE-2248 - mixed version reads (not allowed by spec) + HBASE-2509 NPEs in various places, HRegion.get, HRS.close + HBASE-2344 InfoServer and hence HBase Master doesn't fully start if you + have HADOOP-6151 patch (Kannan Muthukkaruppan via Stack) + HBASE-2382 Don't rely on fs.getDefaultReplication() to roll HLogs + (Nicolas Spiegelberg via Stack) + HBASE-2415 Disable META splitting in 0.20 (Todd Lipcon via Stack) + HBASE-2421 Put hangs for 10 retries on failed region servers + HBASE-2442 Log lease recovery catches IOException too widely + (Todd Lipcon via Stack) + HBASE-2457 RS gets stuck compacting region ad infinitum + HBASE-2562 bin/hbase doesn't work in-situ in maven + (Todd Lipcon via Stack) + HBASE-2449 Local HBase does not stop properly + HBASE-2539 Cannot start ZK before the rest in tests anymore + HBASE-2561 Scanning .META. while split in progress yields + IllegalArgumentException (Todd Lipcon via Stack) + HBASE-2572 hbase/bin/set_meta_block_caching.rb:72: can't convert + Java::JavaLang::String into String (TypeError) - little + issue with script + HBASE-2483 Some tests do not use ephemeral ports + HBASE-2573 client.HConnectionManager$TableServers logs non-printable + binary bytes (Benoît Sigoure via Stack) + HBASE-2576 TestHRegion.testDelete_mixed() failing on hudson + HBASE-2581 Bloom commit broke some tests... fix + HBASE-2582 TestTableSchemaModel not passing after commit of blooms + HBASE-2583 Make webapps work in distributed mode again and make webapps + deploy at / instead of at /webapps/master/master.jsp + HBASE-2590 Failed parse of branch element in saveVersion.sh + HBASE-2591 HBASE-2587 hardcoded the port that dfscluster runs on + HBASE-2519 StoreFileScanner.seek swallows IOEs (Todd Lipcon via Stack) + HBASE-2516 Ugly IOE when region is being closed; rather, should NSRE + (Daniel Ploeg via Stack) + HBASE-2589 TestHRegion.testWritesWhileScanning flaky on trunk + (Todd Lipcon via Stack) + HBASE-2590 Failed parse of branch element in saveVersion.sh + (Benoît Sigoure via Stack) + HBASE-2586 Move hbase webapps to a hbase-webapps dir (Todd Lipcon via + Andrew Purtell) + HBASE-2610 ValueFilter copy pasted javadoc from QualifierFilter + HBASE-2619 HBase shell 'alter' command cannot set table properties to False + (Christo Wilson via Stack) + HBASE-2621 Fix bad link to HFile documentation in javadoc + (Jeff Hammerbacher via Todd Lipcon) + HBASE-2371 Fix 'list' command in shell (Alexey Kovyrin via Todd Lipcon) + HBASE-2620 REST tests don't use ephemeral ports + HBASE-2635 ImmutableBytesWritable ignores offset in several cases + HBASE-2654 Add additional maven repository temporarily to fetch Guava + HBASE-2560 Fix IllegalArgumentException when manually splitting table + from web UI + HBASE-2657 TestTableResource is broken in trunk + HBASE-2662 TestScannerResource.testScannerResource broke in trunk + HBASE-2667 TestHLog.testSplit failing in trunk (Cosmin and Stack) + HBASE-2614 killing server in TestMasterTransitions causes NPEs and test deadlock + HBASE-2615 M/R on bulk imported tables + HBASE-2676 TestInfoServers should use ephemeral ports + HBASE-2616 TestHRegion.testWritesWhileGetting flaky on trunk + HBASE-2684 TestMasterWrongRS flaky in trunk + HBASE-2691 LeaseStillHeldException totally ignored by RS, wrongly named + HBASE-2703 ui not working in distributed context + HBASE-2710 Shell should use default terminal width when autodetection fails + (Kannan Muthukkaruppan via Todd Lipcon) + HBASE-2712 Cached region location that went stale won't recover if + asking for first row + HBASE-2732 TestZooKeeper was broken, HBASE-2691 showed it + HBASE-2670 Provide atomicity for readers even when new insert has + same timestamp as current row. + HBASE-2733 Replacement of LATEST_TIMESTAMP with real timestamp was broken + by HBASE-2353. + HBASE-2734 TestFSErrors should catch all types of exceptions, not just RTE + HBASE-2738 TestTimeRangeMapRed updated now that we keep multiple cells with + same timestamp in MemStore + HBASE-2725 Shutdown hook management is gone in trunk; restore + HBASE-2740 NPE in ReadWriteConsistencyControl + HBASE-2752 Don't retry forever when waiting on too many store files + HBASE-2737 CME in ZKW introduced in HBASE-2694 (Karthik Ranganathan via JD) + HBASE-2756 MetaScanner.metaScan doesn't take configurations + HBASE-2656 HMaster.getRegionTableClosest should not return null for closed + regions + HBASE-2760 Fix MetaScanner TableNotFoundException when scanning starting at + the first row in a table. + HBASE-1025 Reconstruction log playback has no bounds on memory used + HBASE-2757 Fix flaky TestFromClientSide test by forcing region assignment + HBASE-2741 HBaseExecutorService needs to be multi-cluster friendly + (Karthik Ranganathan via JD) + HBASE-2769 Fix typo in warning message for HBaseConfiguration + HBASE-2768 Fix teardown order in TestFilter + HBASE-2763 Cross-port HADOOP-6833 IPC parameter leak bug + HBASE-2758 META region stuck in RS2ZK_REGION_OPENED state + (Karthik Ranganathan via jgray) + HBASE-2767 Fix reflection in tests that was made incompatible by HDFS-1209 + HBASE-2617 Load balancer falls into pathological state if one server under + average - slop; endless churn + HBASE-2729 Interrupted or failed memstore flushes should not corrupt the + region + HBASE-2772 Scan doesn't recover from region server failure + HBASE-2775 Update of hadoop jar in HBASE-2771 broke TestMultiClusters + HBASE-2774 Spin in ReadWriteConsistencyControl eating CPU (load > 40) and + no progress running YCSB on clean cluster startup + HBASE-2785 TestScannerTimeout.test2772 is flaky + HBASE-2787 PE is confused about flushCommits + HBASE-2707 Can't recover from a dead ROOT server if any exceptions happens + during log splitting + HBASE-2501 Refactor StoreFile Code + HBASE-2806 DNS hiccups cause uncaught NPE in HServerAddress#getBindAddress + (Benoit Sigoure via Stack) + HBASE-2806 (small compile fix via jgray) + HBASE-2797 Another NPE in ReadWriteConsistencyControl + HBASE-2831 Fix '$bin' path duplication in setup scripts + (Nicolas Spiegelberg via Stack) + HBASE-2781 ZKW.createUnassignedRegion doesn't make sure existing znode is + in the right state (Karthik Ranganathan via JD) + HBASE-2727 Splits writing one file only is untenable; need dir of recovered + edits ordered by sequenceid + HBASE-2843 Readd bloomfilter test over zealously removed by HBASE-2625 + HBASE-2846 Make rest server be same as thrift and avro servers + HBASE-1511 Pseudo distributed mode in LocalHBaseCluster + (Nicolas Spiegelberg via Stack) + HBASE-2851 Remove testDynamicBloom() unit test + (Nicolas Spiegelberg via Stack) + HBASE-2853 TestLoadIncrementalHFiles fails on TRUNK + HBASE-2854 broken tests on trunk + HBASE-2859 Cleanup deprecated stuff in TestHLog (Alex Newman via Stack) + HBASE-2858 TestReplication.queueFailover fails half the time + HBASE-2863 HBASE-2553 removed an important edge case + HBASE-2866 Region permanently offlined + HBASE-2849 HBase clients cannot recover when their ZooKeeper session + becomes invalid (Benôit Sigoure via Stack) + HBASE-2876 HBase hbck: false positive error reported for parent regions + that are in offline state in meta after a split + HBASE-2815 not able to run the test suite in background because TestShell + gets suspended on tty output (Alexey Kovyrin via Stack) + HBASE-2852 Bloom filter NPE (pranav via jgray) + HBASE-2820 hbck throws an error if HBase root dir isn't on the default FS + HBASE-2884 TestHFileOutputFormat flaky when map tasks generate identical + data + HBASE-2890 Initialize RPC JMX metrics on startup (Gary Helmling via Stack) + HBASE-2755 Duplicate assignment of a region after region server recovery + (Kannan Muthukkaruppan via Stack) + HBASE-2892 Replication metrics aren't updated + HBASE-2461 Split doesn't handle IOExceptions when creating new region + reference files + HBASE-2871 Make "start|stop" commands symmetric for Master & Cluster + (Nicolas Spiegelberg via Stack) + HBASE-2901 HBASE-2461 broke build + HBASE-2823 Entire Row Deletes not stored in Row+Col Bloom + (Alexander Georgiev via Stack) + HBASE-2897 RowResultGenerator should handle NoSuchColumnFamilyException + HBASE-2905 NPE when inserting mass data via REST interface (Sandy Yin via + Andrew Purtell) + HBASE-2908 Wrong order of null-check [in TIF] (Libor Dener via Stack) + HBASE-2909 SoftValueSortedMap is broken, can generate NPEs + HBASE-2919 initTableReducerJob: Unused method parameter + (Libor Dener via Stack) + HBASE-2923 Deadlock between HRegion.internalFlushCache and close + HBASE-2927 BaseScanner gets stale HRegionInfo in some race cases + HBASE-2928 Fault in logic in BinaryPrefixComparator leads to + ArrayIndexOutOfBoundsException (pranav via jgray) + HBASE-2924 TestLogRolling doesn't use the right HLog half the time + HBASE-2931 Do not throw RuntimeExceptions in RPC/HbaseObjectWritable + code, ensure we log and rethrow as IOE + (Karthik Ranganathan via Stack) + HBASE-2915 Deadlock between HRegion.ICV and HRegion.close + HBASE-2920 HTable.checkAndPut/Delete doesn't handle null values + HBASE-2944 cannot alter bloomfilter setting for a column family from + hbase shell (Kannan via jgray) + HBASE-2948 bin/hbase shell broken (after hbase-2692) + (Sebastian Bauer via Stack) + HBASE-2954 Fix broken build caused by hbase-2692 commit + HBASE-2918 SequenceFileLogWriter doesnt make it clear if there is no + append by config or by missing lib/feature + HBASE-2799 "Append not enabled" warning should not show if hbase + root dir isn't on DFS + HBASE-2943 major_compact (and other admin commands) broken for .META. + HBASE-2643 Figure how to deal with eof splitting logs + (Nicolas Spiegelberg via Stack) + HBASE-2925 LRU of HConnectionManager.HBASE_INSTANCES breaks if + HBaseConfiguration is changed + (Robert Mahfoud via Stack) + HBASE-2964 Deadlock when RS tries to RPC to itself inside SplitTransaction + HBASE-1485 Wrong or indeterminate behavior when there are duplicate + versions of a column (pranav via jgray) + HBASE-2967 Failed split: IOE 'File is Corrupt!' -- sync length not being + written out to SequenceFile + HBASE-2969 missing sync in HTablePool.getTable() + (Guilherme Mauro Germoglio Barbosa via Stack) + HBASE-2973 NPE in LogCleaner + HBASE-2974 LoadBalancer ArithmeticException: / by zero + HBASE-2975 DFSClient names in master and RS should be unique + HBASE-2978 LoadBalancer IndexOutOfBoundsException + HBASE-2983 TestHLog unit test is mis-comparing an assertion + (Alex Newman via Todd Lipcon) + HBASE-2986 multi writable can npe causing client hang + HBASE-2979 Fix failing TestMultParrallel in hudson build + HBASE-2899 hfile.min.blocksize.size ignored/documentation wrong + HBASE-3006 Reading compressed HFile blocks causes way too many DFS RPC + calls severly impacting performance + (Kannan Muthukkaruppan via Stack) + HBASE-3010 Can't start/stop/start... cluster using new master + HBASE-3015 recovered.edits files not deleted if it only contain edits that + have already been flushed; hurts perf for all future opens of + the region + HBASE-3018 Bulk assignment on startup runs serially through the cluster + servers assigning in bulk to one at a time + HBASE-3023 NPE processing server crash in MetaReader. getServerUserRegions + HBASE-3024 NPE processing server crash in MetaEditor.addDaughter + HBASE-3026 Fixup of "missing" daughters on split is too aggressive + HBASE-3003 ClassSize constants dont use 'final' + HBASE-3002 Fix zookeepers.sh to work properly with strange JVM options + HBASE-3028 No basescanner means no GC'ing of split, offlined parent regions + HBASE-2989 [replication] RSM won't cleanup after locking if 0 peers + HBASE-2992 [replication] MalformedObjectNameException in ReplicationMetrics + HBASE-3037 When new master joins running cluster does "Received report from + unknown server -- telling it to STOP_REGIONSERVER. + HBASE-3039 Stuck in regionsInTransition because rebalance came in at same + time as a split + HBASE-3042 Use LO4J in SequenceFileLogReader + (Nicolas Spiegelberg via Stack) + HBASE-2995 Incorrect dependency on Log class from Jetty + HBASE-3038 WALReaderFSDataInputStream.getPos() fails if Filesize > MAX_INT + (Nicolas Spiegelberg via Stack) + HBASE-3047 If new master crashes, restart is messy + HBASE-3054 Remore TestEmptyMetaInfo; it doesn't make sense any more. + HBASE-3056 Fix ordering in ZKWatcher constructor to prevent weird race + condition + HBASE-3057 Race condition when closing regions that causes flakiness in + TestRestartCluster + HBASE-3058 Fix REST tests on trunk + HBASE-3068 IllegalStateException when new server comes online, is given + 200 regions to open and 200th region gets timed out of regions + in transition + HBASE-3064 Long sleeping in HConnectionManager after thread is interrupted + (Bruno Dumon via Stack) + HBASE-2753 Remove sorted() methods from Result now that Gets are Scans + HBASE-3059 TestReadWriteConsistencyControl occasionally hangs (Hairong + via Ryan) + HBASE-2906 [rest/stargate] URI decoding in RowResource + HBASE-3008 Memstore.updateColumnValue passes wrong flag to heapSizeChange + (Causes memstore size to go negative) + HBASE-3089 REST tests are broken locally and up in hudson + HBASE-3062 ZooKeeper KeeperException$ConnectionLossException is a + "recoverable" exception; we should retry a while on server + startup at least. + HBASE-3074 Zookeeper test failing on hudson + HBASE-3089 REST tests are broken locally and up in hudson + HBASE-3085 TestSchemaResource broken on TRUNK up on HUDSON + HBASE-3080 TestAdmin hanging on hudson + HBASE-3063 TestThriftServer failing in TRUNK + HBASE-3094 Fixes for miscellaneous broken tests + HBASE-3060 [replication] Reenable replication on trunk with unit tests + HBASE-3041 [replication] ReplicationSink shouldn't kill the whole RS when + it fails to replicate + HBASE-3044 [replication] ReplicationSource won't cleanup logs if there's + nothing to replicate + HBASE-3113 Don't reassign regions if cluster is being shutdown + HBASE-2933 Skip EOF Errors during Log Recovery + (Nicolas Spiegelberg via Stack) + HBASE-3081 Log Splitting & Replay: Distinguish between Network IOE and + Parsing IOE (Nicolas Spiegelberg via Stack) + HBASE-3098 TestMetaReaderEditor is broken in TRUNK; hangs + HBASE-3110 TestReplicationSink failing in TRUNK up on Hudson + HBASE-3101 bin assembly doesn't include -tests or -source jars + HBASE-3121 [rest] Do not perform cache control when returning results + HBASE-2669 HCM.shutdownHook causes data loss with + hbase.client.write.buffer != 0 + HBASE-2985 HRegionServer.multi() no longer calls HRegion.put(List) when + possible + HBASE-3031 CopyTable MR job named "Copy Table" in Driver + HBASE-2658 REST (stargate) TableRegionModel Regions need to be updated to + work w/ new region naming convention from HBASE-2531 + HBASE-3140 Rest schema modification throw null pointer exception + (David Worms via Stack) + HBASE-2998 rolling-restart.sh shouldn't rely on zoo.cfg + HBASE-3145 importtsv fails when the line contains no data + (Kazuki Ohta via Todd Lipcon) + HBASE-2984 [shell] Altering a family shouldn't reset to default unchanged + attributes + HBASE-3143 Adding the tests' hbase-site.xml to the jar breaks some clients + HBASE-3139 Server shutdown processor stuck because meta not online + HBASE-3136 Stale reads from ZK can break the atomic CAS operations we + have in ZKAssign + HBASE-2753 Remove sorted() methods from Result now that Gets are Scans + HBASE-3147 Regions stuck in transition after rolling restart, perpetual + timeout handling but nothing happens + HBASE-3158 Bloom File Writes Broken if keySize is large + (Nicolas Spiegelberg via Stack) + HBASE-3155 HFile.appendMetaBlock() uses wrong comparator + (Nicolas Spiegelberg via Stack) + HBASE-3012 TOF doesn't take zk client port for remote clusters + HBASE-3159 Double play of OpenedRegionHandler for a single region + and assorted fixes around this + TestRollingRestart added + HBASE-3160 Use more intelligent priorities for PriorityCompactionQueue + (Nicolas Spiegelberg via Stack) + HBASE-3172 Reverse order of AssignmentManager and MetaNodeTracker in + ZooKeeperWatcher + HBASE-2406 Define semantics of cell timestamps/versions + HBASE-3175 Commit of HBASE-3160 broke TestPriorityCompactionQueue up on + hudson (nicolas via jgray) + HBASE-3163 If we timeout PENDING_CLOSE and send another closeRegion RPC, + need to handle NSRE from RS (comes as a RemoteException) + HBASE-3164 Handle case where we open META, ROOT has been closed but + znode location not deleted yet, and try to update META + location in ROOT + HBASE-2006 Documentation of hbase-site.xml parameters + HBASE-2672 README.txt should contain basic information like how to run + or build HBase + HBASE-3179 Enable ReplicationLogsCleaner only if replication is, + and fix its test + HBASE-3185 User-triggered compactions are triggering splits! + HBASE-1932 Encourage use of 'lzo' compression... add the wiki page to + getting started + HBASE-3151 NPE when trying to read regioninfo from .META. + HBASE-3191 FilterList with MUST_PASS_ONE and SCVF isn't working + (Stefan Seelmann via Stack) + HBASE-2471 Splitting logs, we'll make an output file though the + region no longer exists + HBASE-3095 Client needs to reconnect if it expires its zk session + HBASE-2935 Refactor "Corrupt Data" Tests in TestHLogSplit + (Alex Newman via Stack) + HBASE-3202 Closing a region, if we get a ConnectException, handle + it rather than abort + HBASE-3198 Log rolling archives files prematurely + HBASE-3203 We can get an order to open a region while shutting down + and it'll hold up regionserver shutdown + HBASE-3204 Reenable deferred log flush + HBASE-3195 [rest] Fix TestTransform breakage on Hudson + HBASE-3205 TableRecordReaderImpl.restart NPEs when first next is restarted + HBASE-3208 HLog.findMemstoresWithEditsOlderThan needs to look for edits + that are equal to too + HBASE-3141 Master RPC server needs to be started before an RS can check in + HBASE-3112 Enable and disable of table needs a bit of loving in new master + HBASE-3207 If we get IOException when closing a region, we should still + remove it from online regions and complete the close in ZK + HBASE-3199 large response handling: some fixups and cleanups + HBASE-3212 More testing of enable/disable uncovered base condition not in + place; i.e. that only one enable/disable runs at a time + HBASE-2898 MultiPut makes proper error handling impossible and leads to + corrupted data + HBASE-3213 If do abort of backup master will get NPE instead of graceful + abort + HBASE-3214 TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS is + failing (Gary via jgray) + HBASE-3216 Move HBaseFsck from client to util + HBASE-3219 Split parents are reassigned on restart and on disable/enable + HBASE-3222 Regionserver region listing in UI is no longer ordered + HBASE-3221 Race between splitting and disabling + HBASE-3224 NPE in KeyValue$KVComparator.compare when compacting + HBASE-3233 Fix Long Running Stats + HBASE-3232 Fix KeyOnlyFilter + Add Value Length (Nicolas via Ryan) + HBASE-3235 Intermittent incrementColumnValue failure in TestHRegion + (Gary via Ryan) + HBASE-3241 check to see if we exceeded hbase.regionserver.maxlogs limit is + incorrect (Kannan Muthukkaruppan via JD) + HBASE-3239 Handle null regions to flush in HLog.cleanOldLogs (Kannan + Muthukkaruppan via JD) + HBASE-3237 Split request accepted -- BUT CURRENTLY A NOOP + HBASE-3252 TestZooKeeperNodeTracker sometimes fails due to a race condition + in test notification (Gary Helmling via Andrew Purtell) + HBASE-3253 Thrift's missing from all the repositories in pom.xml + HBASE-3258 EOF when version file is empty + HBASE-3259 Can't kill the region servers when they wait on the master or + the cluster state znode + HBASE-3249 Typing 'help shutdown' in the shell shouldn't shutdown the cluster + HBASE-3262 TestHMasterRPCException uses non-ephemeral port for master + HBASE-3272 Remove no longer used options + HBASE-3269 HBase table truncate semantics seems broken as "disable" table + is now async by default + HBASE-3275 [rest] No gzip/deflate content encoding support + HBASE-3261 NPE out of HRS.run at startup when clock is out of sync + HBASE-3277 HBase Shell zk_dump command broken + HBASE-3267 close_region shell command breaks region + HBASE-3265 Regionservers waiting for ROOT while Master waiting for RegionServers + HBASE-3263 Stack overflow in AssignmentManager + HBASE-3234 hdfs-724 "breaks" TestHBaseTestingUtility multiClusters + HBASE-3286 Master passes IP and not hostname back to region server + HBASE-3297 If rows in .META. with no HRegionInfo cell, then hbck fails read + of .META. + HBASE-3294 WARN org.apache.hadoop.hbase.regionserver.Store: Not in set + (double-remove?) org.apache.hadoop.hbase.regionserver.StoreScanner@76607d3d + HBASE-3299 If failed open, we don't output the IOE + HBASE-3291 If split happens while regionserver is going down, we can stick open. + HBASE-3295 Dropping a 1k+ regions table likely ends in a client socket timeout + and it's very confusing + HBASE-3301 Treat java.net.SocketTimeoutException same as ConnectException + assigning/unassigning regions + HBASE-3296 Newly created table ends up disabled instead of assigned + HBASE-3304 Get spurious master fails during bootup + HBASE-3298 Regionserver can close during a split causing double assignment + HBASE-3309 " Not running balancer because dead regionserver processing" is a lie + HBASE-3314 [shell] 'move' is broken + HBASE-3315 Add debug output for when balancer makes bad balance + HBASE-3278 AssertionError in LoadBalancer + HBASE-3318 Split rollback leaves parent with writesEnabled=false + HBASE-3334 Refresh our hadoop jar because of HDFS-1520 + HBASE-3347 Can't truncate/disable table that has rows in .META. that have empty + info:regioninfo column + HBASE-3321 Replication.join shouldn't clear the logs znode + HBASE-3352 enabling a non-existent table from shell prints no error + HBASE-3353 table.jsp doesn't handle entries in META without server info + HBASE-3351 ReplicationZookeeper goes to ZK every time a znode is modified + HBASE-3326 Replication state's znode should be created else it + defaults to false + HBASE-3355 Stopping a stopped cluster leaks an HMaster + HBASE-3356 Add more checks in replication if RS is stopped + HBASE-3358 Recovered replication queue wait on themselves when terminating + HBASE-3359 LogRoller not added as a WAL listener when replication is enabled + HBASE-3360 ReplicationLogCleaner is enabled by default in 0.90 -- causes NPE + HBASE-3363 ReplicationSink should batch delete + HBASE-3365 EOFE contacting crashed RS causes Master abort + HBASE-3362 If .META. offline between OPENING and OPENED, then wrong server + location in .META. is possible + HBASE-3368 Split message can come in before region opened message; results + in 'Region has been PENDING_CLOSE for too long' cycle + HBASE-3366 WALObservers should be notified before the lock + HBASE-3367 Failed log split not retried + HBASE-3370 ReplicationSource.openReader fails to locate HLogs when they + aren't split yet + HBASE-3371 Race in TestReplication can make it fail + HBASE-3323 OOME in master splitting logs + HBASE-3374 Our jruby jar has *GPL jars in it; fix + HBASE-3343 Server not shutting down after losing log lease + HBASE-3381 Interrupt of a region open comes across as a successful open + HBASE-3386 NPE in TableRecordReaderImpl.restart + HBASE-3388 NPE processRegionInTransition(AssignmentManager.java:264) + doing rolling-restart.sh + HBASE-3383 [0.90RC1] bin/hbase script displays "no such file" warning on + target/cached_classpath.txt + HBASE-3344 Master aborts after RPC to server that was shutting down + HBASE-3408 AssignmentManager NullPointerException + HBASE-3402 Web UI shows two META regions + HBASE-3409 Failed server shutdown processing when retrying hlog split + HBASE-3412 HLogSplitter should handle missing HLogs + HBASE-3420 Handling a big rebalance, we can queue multiple instances of + a Close event; messes up state + HBASE-3423 hbase-env.sh over-rides HBASE_OPTS incorrectly (Ted Dunning via + Andrew Purtell) + HBASE-3407 hbck should pause between fixing and re-checking state + HBASE-3401 Region IPC operations should be high priority + HBASE-3430 hbase-daemon.sh should clean up PID files on process stop + + + IMPROVEMENTS + HBASE-1760 Cleanup TODOs in HTable + HBASE-1759 Ability to specify scanner caching on a per-scan basis + (Ken Weiner via jgray) + HBASE-1763 Put writeToWAL methods do not have proper getter/setter names + (second commit to fix compile error in hregion) + HBASE-1770 HTable.setWriteBufferSize does not flush the writeBuffer when + its size is set to a value lower than its current size. + (Mathias via jgray) + HBASE-1771 PE sequentialWrite is 7x slower because of + MemStoreFlusher#checkStoreFileCount + HBASE-1758 Extract interface out of HTable (Vaibhav Puranik via Andrew + Purtell) + HBASE-1776 Make rowcounter enum public + HBASE-1276 [testing] Upgrade to JUnit 4.x and use @BeforeClass + annotations to optimize tests + HBASE-1800 Too many ZK connections + HBASE-1819 Update to 0.20.1 hadoop and zk 3.2.1 + HBASE-1820 Update jruby from 1.2 to 1.3.1 + HBASE-1687 bin/hbase script doesn't allow for different memory settings + for each daemon type + HBASE-1823 Ability for Scanners to bypass the block cache + HBASE-1827 Add disabling block cache scanner flag to the shell + HBASE-1835 Add more delete tests + HBASE-1574 Client and server APIs to do batch deletes + HBASE-1833 hfile.main fixes + HBASE-1684 Backup (Export/Import) contrib tool for 0.20 + HBASE-1860 Change HTablePool#createHTable from private to protected + HBASE-48 Bulk load tools + HBASE-1855 HMaster web application doesn't show the region end key in the + table detail page (Andrei Dragomir via Stack) + HBASE-1870 Bytes.toFloat(byte[], int) is marked private + HBASE-1874 Client Scanner mechanism that is used for HbaseAdmin methods + (listTables, tableExists), is very slow if the client is far + away from the HBase cluster (Andrei Dragomir via Stack) + HBASE-1879 ReadOnly transactions generate WAL activity (Clint Morgan via + Stack) + HBASE-1875 Compression test utility + HBASE-1832 Faster enable/disable/delete + HBASE-1481 Add fast row key only scanning + HBASE-1506 [performance] Make splits faster + HBASE-1722 Add support for exporting HBase metrics via JMX + (Gary Helming via Stack) + HBASE-1899 Use scanner caching in shell count + HBASE-1887 Update hbase trunk to latests on hadoop 0.21 branch so we can + all test sync/append + HBASE-1902 Let PerformanceEvaluation support setting tableName and compress + algorithm (Schubert Zhang via Stack) + HBASE-1885 Simplify use of IndexedTable outside Java API + (Kevin Patterson via Stack) + HBASE-1903 Enable DEBUG by default + HBASE-1907 Version all client writables + HBASE-1914 hlog should be able to set replication level for the log + indendently from any other files + HBASE-1537 Intra-row scanning + HBASE-1918 Don't do DNS resolving in .META. scanner for each row + HBASE-1756 Refactor HLog (changing package first) + HBASE-1926 Remove unused xmlenc jar from trunk + HBASE-1936 HLog group commit + HBASE-1921 When the Master's session times out and there's only one, + cluster is wedged + HBASE-1942 Update hadoop jars in trunk; update to r831142 + HBASE-1943 Remove AgileJSON; unused + HBASE-1944 Add a "deferred log flush" attribute to HTD + HBASE-1945 Remove META and ROOT memcache size bandaid + HBASE-1947 If HBase starts/stops often in less than 24 hours, + you end up with lots of store files + HBASE-1829 Make use of start/stop row in TableInputFormat + (Lars George via Stack) + HBASE-1867 Tool to regenerate an hbase table from the data files + HBASE-1904 Add tutorial for installing HBase on Windows using Cygwin as + a test and development environment (Wim Van Leuven via Stack) + HBASE-1963 Output to multiple tables from Hadoop MR without use of HTable + (Kevin Peterson via Andrew Purtell) + HBASE-1975 SingleColumnValueFilter: Add ability to match the value of + previous versions of the specified column + (Jeremiah Jacquet via Stack) + HBASE-1971 Unit test the full WAL replay cycle + HBASE-1970 Export does one version only; make it configurable how many + it does + HBASE-1987 The Put object has no simple read methods for checking what + has already been added (Ryan Smith via Stack) + HBASE-1985 change HTable.delete(ArrayList) to HTable.delete(List) + HBASE-1958 Remove "# TODO: PUT BACK !!! "${HADOOP_HOME}"/bin/hadoop + dfsadmin -safemode wait" + HBASE-2011 Add zktop like output to HBase's master UI (Lars George via + Andrew Purtell) + HBASE-1995 Add configurable max value size check (Lars George via Andrew + Purtell) + HBASE-2017 Set configurable max value size check to 10MB + HBASE-2029 Reduce shell exception dump on console + (Lars George and J-D via Stack) + HBASE-2027 HConnectionManager.HBASE_INSTANCES leaks TableServers + (Dave Latham via Stack) + HBASE-2013 Add useful helpers to HBaseTestingUtility.java (Lars George + via J-D) + HBASE-2031 When starting HQuorumPeer, try to match on more than 1 address + HBASE-2043 Shell's scan broken + HBASE-2044 HBASE-1822 removed not-deprecated APIs + HBASE-2049 Cleanup HLog binary log output (Dave Latham via Stack) + HBASE-2052 Make hbase more 'live' when comes to noticing table creation, + splits, etc., for 0.20.3 + HBASE-2059 Break out WAL reader and writer impl from HLog + HBASE-2060 Missing closing tag in mapreduce package info (Lars George via + Andrew Purtell) + HBASE-2028 Add HTable.incrementColumnValue support to shell (Lars George + via Andrew Purtell) + HBASE-2062 Metrics documentation outdated (Lars George via JD) + HBASE-2045 Update trunk and branch zk to just-release 3.2.2. + HBASE-2074 Improvements to the hadoop-config script (Bassam Tabbara via + Stack) + HBASE-2076 Many javadoc warnings + HBASE-2068 MetricsRate is missing "registry" parameter (Lars George via JD) + HBASE-2025 0.20.2 accessed from older client throws + UndeclaredThrowableException; frustrates rolling upgrade + HBASE-2081 Set the retries higher in shell since client pause is lower + HBASE-1956 Export HDFS read and write latency as a metric + HBASE-2036 Use Configuration instead of HBaseConfiguration (Enis Soztutar + via Stack) + HBASE-2085 StringBuffer -> StringBuilder - conversion of references as + necessary (Kay Kay via Stack) + HBASE-2052 Upper bound of outstanding WALs can be overrun + HBASE-2086 Job(configuration,String) deprecated (Kay Kay via Stack) + HBASE-1996 Configure scanner buffer in bytes instead of number of rows + (Erik Rozendaal and Dave Latham via Stack) + HBASE-2090 findbugs issues (Kay Kay via Stack) + HBASE-2089 HBaseConfiguration() ctor. deprecated (Kay Kay via Stack) + HBASE-2035 Binary values are formatted wrong in shell + HBASE-2095 TIF shuold support more confs for the scanner (Bassam Tabbara + via Andrew Purtell) + HBASE-2107 Upgrading Lucene 2.2 to Lucene 3.0.0 (Kay Kay via Stack) + HBASE-2111 Move to ivy broke our being able to run in-place; i.e. + ./bin/start-hbase.sh in a checkout + HBASE-2136 Forward-port the old mapred package + HBASE-2133 Increase default number of client handlers + HBASE-2109 status 'simple' should show total requests per second, also + the requests/sec is wrong as is + HBASE-2151 Remove onelab and include generated thrift classes in javadoc + (Lars Francke via Stack) + HBASE-2149 hbase.regionserver.global.memstore.lowerLimit is too low + HBASE-2157 LATEST_TIMESTAMP not replaced by current timestamp in KeyValue + (bulk loading) + HBASE-2153 Publish generated HTML documentation for Thrift on the website + (Lars Francke via Stack) + HBASE-1373 Update Thrift to use compact/framed protocol (Lars Francke via + Stack) + HBASE-2172 Add constructor to Put for row key and timestamp + (Lars Francke via Stack) + HBASE-2178 Hooks for replication + HBASE-2180 Bad random read performance from synchronizing + hfile.fddatainputstream + HBASE-2194 HTable - put(Put) , put(ListHRS problem, regions are + not reassigned + HBASE-1568 Client doesnt consult old row filter interface in + filterSaysStop() - could result in NPE or excessive scanning + HBASE-1564 in UI make host addresses all look the same -- not IP sometimes + and host at others + HBASE-1567 cant serialize new filters + HBASE-1585 More binary key/value log output cleanup + (Lars George via Stack) + HBASE-1563 incrementColumnValue does not write to WAL (Jon Gray via Stack) + HBASE-1569 rare race condition can take down a regionserver + HBASE-1450 Scripts passed to hbase shell do not have shell context set up + for them + HBASE-1566 using Scan(startRow,stopRow) will cause you to iterate the + entire table + HBASE-1560 TIF can't seem to find one region + HBASE-1580 Store scanner does not consult filter.filterRow at end of scan + (Clint Morgan via Stack) + HBASE-1437 broken links in hbase.org + HBASE-1582 Translate ColumnValueFilter and RowFilterSet to the new Filter + interface + HBASE-1594 Fix scan addcolumns after hbase-1385 commit (broke hudson build) + HBASE-1595 hadoop-default.xml and zoo.cfg in hbase jar + HBASE-1602 HRegionServer won't go down since we added in new LruBlockCache + HBASE-1608 TestCachedBlockQueue failing on some jvms (Jon Gray via Stack) + HBASE-1615 HBASE-1597 introduced a bug when compacting after a split + (Jon Gray via Stack) + HBASE-1616 Unit test of compacting referenced StoreFiles (Jon Gray via + Stack) + HBASE-1618 Investigate further into the MemStoreFlusher StoreFile limit + (Jon Gray via Stack) + HBASE-1625 Adding check to Put.add(KeyValue), to see that it has the same + row as when instantiated (Erik Holstad via Stack) + HBASE-1629 HRS unable to contact master + HBASE-1633 Can't delete in TRUNK shell; makes it hard doing admin repairs + HBASE-1641 Stargate build.xml causes error in Eclipse + HBASE-1627 TableInputFormatBase#nextKeyValue catches the wrong exception + (Doğacan Güney via Stack) + HBASE-1644 Result.row is cached in getRow; this breaks MapReduce + (Doğacan Güney via Stack) + HBASE-1639 clean checkout with empty hbase-site.xml, zk won't start + HBASE-1646 Scan-s can't set a Filter (Doğacan Güney via Stack) + HBASE-1649 ValueFilter may not reset its internal state + (Doğacan Güney via Stack) + HBASE-1651 client is broken, it requests ROOT region location from ZK too + much + HBASE-1650 HBASE-1551 broke the ability to manage non-regionserver + start-up/shut down. ie: you cant start/stop thrift on a cluster + anymore + HBASE-1658 Remove UI refresh -- its annoying + HBASE-1659 merge tool doesnt take binary regions with \x escape format + HBASE-1663 Request compaction only once instead of every time 500ms each + time we cycle the hstore.getStorefilesCount() > + this.blockingStoreFilesNumber loop + HBASE-1058 Disable 1058 on catalog tables + HBASE-1583 Start/Stop of large cluster untenable + HBASE-1668 hbase-1609 broke TestHRegion.testScanSplitOnRegion unit test + HBASE-1669 need dynamic extensibility of HBaseRPC code maps and interface + lists (Clint Morgan via Stack) + HBASE-1359 After a large truncating table HBase becomes unresponsive + HBASE-1215 0.19.0 -> 0.20.0 migration (hfile, HCD changes, HSK changes) + HBASE-1689 Fix javadoc warnings and add overview on client classes to + client package + HBASE-1680 FilterList writable only works for HBaseObjectWritable + defined types (Clint Morgan via Stack and Jon Gray) + HBASE-1607 transactions / indexing fixes: trx deletes not handeled, index + scan can't specify stopRow (Clint Morgan via Stack) + HBASE-1693 NPE close_region ".META." in shell + HBASE-1706 META row with missing HRI breaks UI + HBASE-1709 Thrift getRowWithColumns doesn't accept column-family only + (Mathias Lehmann via Stack) + HBASE-1692 Web UI is extremely slow / freezes up if you have many tables + HBASE-1686 major compaction can create empty store files, causing AIOOB + when trying to read + HBASE-1705 Thrift server: deletes in mutateRow/s don't delete + (Tim Sell and Ryan Rawson via Stack) + HBASE-1703 ICVs across /during a flush can cause multiple keys with the + same TS (bad) + HBASE-1671 HBASE-1609 broke scanners riding across splits + HBASE-1717 Put on client-side uses passed-in byte[]s rather than always + using copies + HBASE-1647 Filter#filterRow is called too often, filters rows it shouldn't + have (Doğacan Güney via Ryan Rawson and Stack) + HBASE-1718 Reuse of KeyValue during log replay could cause the wrong + data to be used + HBASE-1573 Holes in master state change; updated startcode and server + go into .META. but catalog scanner just got old values (redux) + HBASE-1534 Got ZooKeeper event, state: Disconnected on HRS and then NPE + on reinit + HBASE-1725 Old TableMap interface's definitions are not generic enough + (Doğacan Güney via Stack) + HBASE-1732 Flag to disable regionserver restart + HBASE-1727 HTD and HCD versions need update + HBASE-1604 HBaseClient.getConnection() may return a broken connection + without throwing an exception (Eugene Kirpichov via Stack) + HBASE-1737 Regions unbalanced when adding new node + HBASE-1739 hbase-1683 broke splitting; only split three logs no matter + what N was + HBASE-1745 [tools] Tool to kick region out of inTransistion + HBASE-1757 REST server runs out of fds + HBASE-1768 REST server has upper limit of 5k PUT + HBASE-1766 Add advanced features to HFile.main() to be able to analyze + storefile problems + HBASE-1761 getclosest doesn't understand delete family; manifests as + "HRegionInfo was null or empty in .META" A.K.A the BS problem + HBASE-1738 Scanner doesnt reset when a snapshot is created, could miss + new updates into the 'kvset' (active part) + HBASE-1767 test zookeeper broken in trunk and 0.20 branch; broken on + hudson too + HBASE-1780 HTable.flushCommits clears write buffer in finally clause + HBASE-1784 Missing rows after medium intensity insert + HBASE-1809 NPE thrown in BoundedRangeFileInputStream + HBASE-1810 ConcurrentModificationException in region assignment + (Mathias Herberts via Stack) + HBASE-1804 Puts are permitted (and stored) when including an appended colon + HBASE-1715 Compaction failure in ScanWildcardColumnTracker.checkColumn + HBASE-2352 Small values for hbase.client.retries.number and + ipc.client.connect.max.retries breaks long ops in hbase shell + (Alexey Kovyrin via Stack) + HBASE-2531 32-bit encoding of regionnames waaaaaaayyyyy too susceptible to + hash clashes (Kannan Muthukkaruppan via Stack) + + IMPROVEMENTS + HBASE-1089 Add count of regions on filesystem to master UI; add percentage + online as difference between whats open and whats on filesystem + (Samuel Guo via Stack) + HBASE-1130 PrefixRowFilter (Michael Gottesman via Stack) + HBASE-1139 Update Clover in build.xml + HBASE-876 There are a large number of Java warnings in HBase; part 1, + part 2, part 3, part 4, part 5, part 6, part 7 and part 8 + (Evgeny Ryabitskiy via Stack) + HBASE-896 Update jruby from 1.1.2 to 1.1.6 + HBASE-1031 Add the Zookeeper jar + HBASE-1142 Cleanup thrift server; remove Text and profuse DEBUG messaging + (Tim Sell via Stack) + HBASE-1064 HBase REST xml/json improvements (Brian Beggs working of + initial Michael Gottesman work via Stack) + HBASE-5121 Fix shell usage for format.width + HBASE-845 HCM.isTableEnabled doesn't really tell if it is, or not + HBASE-903 [shell] Can't set table descriptor attributes when I alter a + table + HBASE-1166 saveVersion.sh doesn't work with git (Nitay Joffe via Stack) + HBASE-1167 JSP doesn't work in a git checkout (Nitay Joffe via Andrew + Purtell) + HBASE-1178 Add shutdown command to shell + HBASE-1184 HColumnDescriptor is too restrictive with family names + (Toby White via Andrew Purtell) + HBASE-1180 Add missing import statements to SampleUploader and remove + unnecessary @Overrides (Ryan Smith via Andrew Purtell) + HBASE-1191 ZooKeeper ensureParentExists calls fail + on absolute path (Nitay Joffe via Jean-Daniel Cryans) + HBASE-1187 After disabling/enabling a table, the regions seems to + be assigned to only 1-2 region servers + HBASE-1210 Allow truncation of output for scan and get commands in shell + (Lars George via Stack) + HBASE-1221 When using ant -projecthelp to build HBase not all the important + options show up (Erik Holstad via Stack) + HBASE-1189 Changing the map type used internally for HbaseMapWritable + (Erik Holstad via Stack) + HBASE-1188 Memory size of Java Objects - Make cacheable objects implement + HeapSize (Erik Holstad via Stack) + HBASE-1230 Document installation of HBase on Windows + HBASE-1241 HBase additions to ZooKeeper part 1 (Nitay Joffe via JD) + HBASE-1231 Today, going from a RowResult to a BatchUpdate reqiures some + data processing even though they are pretty much the same thing + (Erik Holstad via Stack) + HBASE-1240 Would be nice if RowResult could be comparable + (Erik Holstad via Stack) + HBASE-803 Atomic increment operations (Ryan Rawson and Jon Gray via Stack) + Part 1 and part 2 -- fix for a crash. + HBASE-1252 Make atomic increment perform a binary increment + (Jonathan Gray via Stack) + HBASE-1258,1259 ganglia metrics for 'requests' is confusing + (Ryan Rawson via Stack) + HBASE-1265 HLogEdit static constants should be final (Nitay Joffe via + Stack) + HBASE-1244 ZooKeeperWrapper constants cleanup (Nitay Joffe via Stack) + HBASE-1262 Eclipse warnings, including performance related things like + synthetic accessors (Nitay Joffe via Stack) + HBASE-1273 ZooKeeper WARN spits out lots of useless messages + (Nitay Joffe via Stack) + HBASE-1285 Forcing compactions should be available via thrift + (Tim Sell via Stack) + HBASE-1186 Memory-aware Maps with LRU eviction for cell cache + (Jonathan Gray via Andrew Purtell) + HBASE-1205 RegionServers should find new master when a new master comes up + (Nitay Joffe via Andrew Purtell) + HBASE-1309 HFile rejects key in Memcache with empty value + HBASE-1331 Lower the default scanner caching value + HBASE-1235 Add table enabled status to shell and UI + (Lars George via Stack) + HBASE-1333 RowCounter updates + HBASE-1195 If HBase directory exists but version file is inexistent, still + proceed with bootstrapping (Evgeny Ryabitskiy via Stack) + HBASE-1301 HTable.getRow() returns null if the row does no exist + (Rong-en Fan via Stack) + HBASE-1176 Javadocs in HBA should be clear about which functions are + asynchronous and which are synchronous + (Evgeny Ryabitskiy via Stack) + HBASE-1260 Bytes utility class changes: remove usage of ByteBuffer and + provide additional ByteBuffer primitives (Jon Gray via Stack) + HBASE-1183 New MR splitting algorithm and other new features need a way to + split a key range in N chunks (Jon Gray via Stack) + HBASE-1350 New method in HTable.java to return start and end keys for + regions in a table (Vimal Mathew via Stack) + HBASE-1271 Allow multiple tests to run on one machine + (Evgeny Ryabitskiy via Stack) + HBASE-1112 we will lose data if the table name happens to be the logs' dir + name (Samuel Guo via Stack) + HBASE-889 The current Thrift API does not allow a new scanner to be + created without supplying a column list unlike the other APIs. + (Tim Sell via Stack) + HBASE-1341 HTable pooler + HBASE-1379 re-enable LZO using hadoop-gpl-compression library + (Ryan Rawson via Stack) + HBASE-1383 hbase shell needs to warn on deleting multi-region table + HBASE-1286 Thrift should support next(nbRow) like functionality + (Alex Newman via Stack) + HBASE-1392 change how we build/configure lzocodec (Ryan Rawson via Stack) + HBASE-1397 Better distribution in the PerformanceEvaluation MapReduce + when rows run to the Billions + HBASE-1393 Narrow synchronization in HLog + HBASE-1404 minor edit of regionserver logging messages + HBASE-1405 Threads.shutdown has unnecessary branch + HBASE-1407 Changing internal structure of ImmutableBytesWritable + contructor (Erik Holstad via Stack) + HBASE-1345 Remove distributed mode from MiniZooKeeper (Nitay Joffe via + Stack) + HBASE-1414 Add server status logging chore to ServerManager + HBASE-1379 Make KeyValue implement Writable + (Erik Holstad and Jon Gray via Stack) + HBASE-1380 Make KeyValue implement HeapSize + (Erik Holstad and Jon Gray via Stack) + HBASE-1413 Fall back to filesystem block size default if HLog blocksize is + not specified + HBASE-1417 Cleanup disorientating RPC message + HBASE-1424 have shell print regioninfo and location on first load if + DEBUG enabled + HBASE-1008 [performance] The replay of logs on server crash takes way too + long + HBASE-1394 Uploads sometimes fall to 0 requests/second (Binding up on + HLog#append?) + HBASE-1429 Allow passing of a configuration object to HTablePool + HBASE-1432 LuceneDocumentWrapper is not public + HBASE-1401 close HLog (and open new one) if there hasnt been edits in N + minutes/hours + HBASE-1420 add abliity to add and remove (table) indexes on existing + tables (Clint Morgan via Stack) + HBASE-1430 Read the logs in batches during log splitting to avoid OOME + HBASE-1017 Region balancing does not bring newly added node within + acceptable range (Evgeny Ryabitskiy via Stack) + HBASE-1454 HBaseAdmin.getClusterStatus + HBASE-1236 Improve readability of table descriptions in the UI + (Lars George and Alex Newman via Stack) + HBASE-1455 Update DemoClient.py for thrift 1.0 (Tim Sell via Stack) + HBASE-1464 Add hbase.regionserver.logroll.period to hbase-default + HBASE-1192 LRU-style map for the block cache (Jon Gray and Ryan Rawson + via Stack) + HBASE-1466 Binary keys are not first class citizens + (Ryan Rawson via Stack) + HBASE-1445 Add the ability to start a master from any machine + HBASE-1474 Add zk attributes to list of attributes + in master and regionserver UIs + HBASE-1448 Add a node in ZK to tell all masters to shutdown + HBASE-1478 Remove hbase master options from shell (Nitay Joffe via Stack) + HBASE-1462 hclient still seems to depend on master + HBASE-1143 region count erratic in master UI + HBASE-1490 Update ZooKeeper library + HBASE-1489 Basic git ignores for people who use git and eclipse + HBASE-1453 Add HADOOP-4681 to our bundled hadoop, add to 'gettting started' + recommendation that hbase users backport + HBASE-1507 iCMS as default JVM + HBASE-1509 Add explanation to shell "help" command on how to use binarykeys + (Lars George via Stack) + HBASE-1514 hfile inspection tool + HBASE-1329 Visibility into ZooKeeper + HBASE-867 If millions of columns in a column family, hbase scanner won't + come up (Jonathan Gray via Stack) + HBASE-1538 Up zookeeper timeout from 10 seconds to 30 seconds to cut down + on hbase-user traffic + HBASE-1539 prevent aborts due to missing zoo.cfg + HBASE-1488 Fix TestThriftServer and re-enable it + HBASE-1541 Scanning multiple column families in the presence of deleted + families results in bad scans + HBASE-1540 Client delete unit test, define behavior + (Jonathan Gray via Stack) + HBASE-1552 provide version running on cluster via getClusterStatus + HBASE-1550 hbase-daemon.sh stop should provide more information when stop + command fails + HBASE-1515 Address part of config option hbase.regionserver unnecessary + HBASE-1532 UI Visibility into ZooKeeper + HBASE-1572 Zookeeper log4j property set to ERROR on default, same output + when cluster working and not working (Jon Gray via Stack) + HBASE-1576 TIF needs to be able to set scanner caching size for smaller + row tables & performance + HBASE-1577 Move memcache to ConcurrentSkipListMap from + ConcurrentSkipListSet + HBASE-1578 Change the name of the in-memory updates from 'memcache' to + 'memtable' or.... + HBASE-1562 How to handle the setting of 32 bit versus 64 bit machines + (Erik Holstad via Stack) + HBASE-1584 Put add methods should return this for ease of use (Be + consistant with Get) (Clint Morgan via Stack) + HBASE-1581 Run major compaction on .META. when table is dropped or + truncated + HBASE-1587 Update ganglia config and doc to account for ganglia 3.1 and + hadoop-4675 + HBASE-1589 Up zk maxClientCnxns from default of 10 to 20 or 30 or so + HBASE-1385 Revamp TableInputFormat, needs updating to match hadoop 0.20.x + AND remove bit where we can make < maps than regions + (Lars George via Stack) + HBASE-1596 Remove WatcherWrapper and have all users of Zookeeper provide a + Watcher + HBASE-1597 Prevent unnecessary caching of blocks during compactions + (Jon Gray via Stack) + HBASE-1607 Redo MemStore heap sizing to be accurate, testable, and more + like new LruBlockCache (Jon Gray via Stack) + HBASE-1218 Implement in-memory column (Jon Gray via Stack) + HBASE-1606 Remove zoo.cfg, put config options into hbase-site.xml + HBASE-1575 HMaster does not handle ZK session expiration + HBASE-1620 Need to use special StoreScanner constructor for major + compactions (passed sf, no caching, etc) (Jon Gray via Stack) + HBASE-1624 Don't sort Puts if only one in list in HCM#processBatchOfRows + HBASE-1626 Allow emitting Deletes out of new TableReducer + (Lars George via Stack) + HBASE-1551 HBase should manage multiple node ZooKeeper quorum + HBASE-1637 Delete client class methods should return itself like Put, Get, + Scan (Jon Gray via Nitay) + HBASE-1640 Allow passing arguments to jruby script run when run by hbase + shell + HBASE-698 HLog recovery is not performed after master failure + HBASE-1643 ScanDeleteTracker takes comparator but it unused + HBASE-1603 MR failed "RetriesExhaustedException: Trying to contact region + server Some server for region TestTable..." -- deubugging + HBASE-1470 hbase and HADOOP-4379, dhruba's flush/sync + HBASE-1632 Write documentation for configuring/managing ZooKeeper + HBASE-1662 Tool to run major compaction on catalog regions when hbase is + shutdown + HBASE-1665 expose more load information to the client side + HBASE-1609 We wait on leases to expire before regionserver goes down. + Rather, just let client fail + HBASE-1655 Usability improvements to HTablePool (Ken Weiner via jgray) + HBASE-1688 Improve javadocs in Result and KeyValue + HBASE-1694 Add TOC to 'Getting Started', add references to THBase and + ITHBase + HBASE-1699 Remove hbrep example as it's too out of date + (Tim Sell via Stack) + HBASE-1683 OOME on master splitting logs; stuck, won't go down + HBASE-1704 Better zk error when failed connect + HBASE-1714 Thrift server: prefix scan API + HBASE-1719 hold a reference to the region in stores instead of only the + region info + HBASE-1743 [debug tool] Add regionsInTransition list to ClusterStatus + detailed output + HBASE-1772 Up the default ZK session timeout from 30seconds to 60seconds + HBASE-2625 Make testDynamicBloom()'s "randomness" deterministic + (Nicolas Spiegelberg via Stack) + + OPTIMIZATIONS + HBASE-1412 Change values for delete column and column family in KeyValue + HBASE-1535 Add client ability to perform mutations without the WAL + (Jon Gray via Stack) + HBASE-1460 Concurrent LRU Block Cache (Jon Gray via Stack) + HBASE-1635 PerformanceEvaluation should use scanner prefetching + +Release 0.19.0 - 01/21/2009 + INCOMPATIBLE CHANGES + HBASE-885 TableMap and TableReduce should be interfaces + (Doğacan Güney via Stack) + HBASE-905 Remove V5 migration classes from 0.19.0 (Jean-Daniel Cryans via + Jim Kellerman) + HBASE-852 Cannot scan all families in a row with a LIMIT, STARTROW, etc. + (Izaak Rubin via Stack) + HBASE-953 Enable BLOCKCACHE by default [WAS -> Reevaluate HBASE-288 block + caching work....?] -- Update your hbase-default.xml file! + HBASE-636 java6 as a requirement + HBASE-994 IPC interfaces with different versions can cause problems + HBASE-1028 If key does not exist, return null in getRow rather than an + empty RowResult + HBASE-1134 OOME in HMaster when HBaseRPC is older than 0.19 + + BUG FIXES + HBASE-891 HRS.validateValuesLength throws IOE, gets caught in the retries + HBASE-892 Cell iteration is broken (Doğacan Güney via Jim Kellerman) + HBASE-898 RowResult.containsKey(String) doesn't work + (Doğacan Güney via Jim Kellerman) + HBASE-906 [shell] Truncates output + HBASE-912 PE is broken when other tables exist + HBASE-853 [shell] Cannot describe meta tables (Izaak Rubin via Stack) + HBASE-844 Can't pass script to hbase shell + HBASE-837 Add unit tests for ThriftServer.HBaseHandler (Izaak Rubin via + Stack) + HBASE-913 Classes using log4j directly + HBASE-914 MSG_REPORT_CLOSE has a byte array for a message + HBASE-918 Region balancing during startup makes cluster unstable + HBASE-921 region close and open processed out of order; makes for + disagreement between master and regionserver on region state + HBASE-925 HRS NPE on way out if no master to connect to + HBASE-928 NPE throwing RetriesExhaustedException + HBASE-924 Update hadoop in lib on 0.18 hbase branch to 0.18.1 + HBASE-929 Clarify that ttl in HColumnDescriptor is seconds + HBASE-930 RegionServer stuck: HLog: Could not append. Requesting close of + log java.io.IOException: Could not get block locations + HBASE-926 If no master, regionservers should hang out rather than fail on + connection and shut themselves down + HBASE-919 Master and Region Server need to provide root region location if + they are using HTable + With J-D's one line patch, test cases now appear to work and + PerformanceEvaluation works as before. + HBASE-939 NPE in HStoreKey + HBASE-945 Be consistent in use of qualified/unqualified mapfile paths + HBASE-946 Row with 55k deletes timesout scanner lease + HBASE-950 HTable.commit no longer works with existing RowLocks though it's + still in API + HBASE-952 Deadlock in HRegion.batchUpdate + HBASE-954 Don't reassign root region until ProcessServerShutdown has split + the former region server's log + HBASE-957 PerformanceEvaluation tests if table exists by comparing + descriptors + HBASE-728, HBASE-956, HBASE-955 Address thread naming, which threads are + Chores, vs Threads, make HLog manager the write ahead log and + not extend it to provided optional HLog sync operations. + HBASE-970 Update the copy/rename scripts to go against change API + HBASE-966 HBASE-748 misses some writes + HBASE-971 Fix the failing tests on Hudson + HBASE-973 [doc] In getting started, make it clear that hbase needs to + create its directory in hdfs + HBASE-963 Fix the retries in HTable.flushCommit + HBASE-969 Won't when storefile > 2G. + HBASE-976 HADOOP 0.19.0 RC0 is broke; replace with HEAD of branch-0.19 + HBASE-977 Arcane HStoreKey comparator bug + HBASE-979 REST web app is not started automatically + HBASE-980 Undo core of HBASE-975, caching of start and end row + HBASE-982 Deleting a column in MapReduce fails (Doğacan Güney via + Stack) + HBASE-984 Fix javadoc warnings + HBASE-985 Fix javadoc warnings + HBASE-951 Either shut down master or let it finish cleanup + HBASE-964 Startup stuck "waiting for root region" + HBASE-964, HBASE-678 provide for safe-mode without locking up HBase "waiting + for root region" + HBASE-990 NoSuchElementException in flushSomeRegions; took two attempts. + HBASE-602 HBase Crash when network card has a IPv6 address + HBASE-996 Migration script to up the versions in catalog tables + HBASE-991 Update the mapred package document examples so they work with + TRUNK/0.19.0. + HBASE-1003 If cell exceeds TTL but not VERSIONs, will not be removed during + major compaction + HBASE-1005 Regex and string comparison operators for ColumnValueFilter + HBASE-910 Scanner misses columns / rows when the scanner is obtained + during a memcache flush + HBASE-1009 Master stuck in loop wanting to assign but regions are closing + HBASE-1016 Fix example in javadoc overvie + HBASE-1021 hbase metrics FileContext not working + HBASE-1023 Check global flusher + HBASE-1036 HBASE-1028 broke Thrift + HBASE-1037 Some test cases failing on Windows/Cygwin but not UNIX/Linux + HBASE-1041 Migration throwing NPE + HBASE-1042 OOME but we don't abort; two part commit. + HBASE-927 We don't recover if HRS hosting -ROOT-/.META. goes down + HBASE-1029 REST wiki documentation incorrect + (Sishen Freecity via Stack) + HBASE-1043 Removing @Override attributes where they are no longer needed. + (Ryan Smith via Jim Kellerman) + HBASE-927 We don't recover if HRS hosting -ROOT-/.META. goes down - + (fix bug in createTable which caused tests to fail) + HBASE-1039 Compaction fails if bloomfilters are enabled + HBASE-1027 Make global flusher check work with percentages rather than + hard code memory sizes + HBASE-1000 Sleeper.sleep does not go back to sleep when interrupted + and no stop flag given. + HBASE-900 Regionserver memory leak causing OOME during relatively + modest bulk importing; part 1 and part 2 + HBASE-1054 Index NPE on scanning (Clint Morgan via Andrew Purtell) + HBASE-1052 Stopping a HRegionServer with unflushed cache causes data loss + from org.apache.hadoop.hbase.DroppedSnapshotException + HBASE-1059 ConcurrentModificationException in notifyChangedReadersObservers + HBASE-1063 "File separator problem on Windows" (Max Lehn via Stack) + HBASE-1068 TestCompaction broken on hudson + HBASE-1067 TestRegionRebalancing broken by running of hdfs shutdown thread + HBASE-1070 Up default index interval in TRUNK and branch + HBASE-1045 Hangup by regionserver causes write to fail + HBASE-1079 Dumb NPE in ServerCallable hides the RetriesExhausted exception + HBASE-782 The DELETE key in the hbase shell deletes the wrong character + (Tim Sell via Stack) + HBASE-543, HBASE-1046, HBase-1051 A region's state is kept in several places + in the master opening the possibility for race conditions + HBASE-1087 DFS failures did not shutdown regionserver + HBASE-1072 Change Thread.join on exit to a timed Thread.join + HBASE-1098 IllegalStateException: Cannot set a region to be closed it it + was not already marked as closing + HBASE-1100 HBASE-1062 broke TestForceSplit + HBASE-1191 shell tools -> close_region does not work for regions that did + not deploy properly on startup + HBASE-1093 NPE in HStore#compact + HBASE-1097 SequenceFile.Reader keeps around buffer whose size is that of + largest item read -> results in lots of dead heap + HBASE-1107 NPE in HStoreScanner.updateReaders + HBASE-1083 Will keep scheduling major compactions if last time one ran, we + didn't. + HBASE-1101 NPE in HConnectionManager$TableServers.processBatchOfRows + HBASE-1099 Regions assigned while master is splitting logs of recently + crashed server; regionserver tries to execute incomplete log + HBASE-1104, HBASE-1098, HBASE-1096: Doubly-assigned regions redux, + IllegalStateException: Cannot set a region to be closed it it was + not already marked as closing, Does not recover if HRS carrying + -ROOT- goes down + HBASE-1114 Weird NPEs compacting + HBASE-1116 generated web.xml and svn don't play nice together + HBASE-1119 ArrayOutOfBoundsException in HStore.compact + HBASE-1121 Cluster confused about where -ROOT- is + HBASE-1125 IllegalStateException: Cannot set a region to be closed if it was + not already marked as pending close + HBASE-1124 Balancer kicks in way too early + HBASE-1127 OOME running randomRead PE + HBASE-1132 Can't append to HLog, can't roll log, infinite cycle (another + spin on HBASE-930) + + IMPROVEMENTS + HBASE-901 Add a limit to key length, check key and value length on client side + HBASE-890 Alter table operation and also related changes in REST interface + (Sishen Freecity via Stack) + HBASE-894 [shell] Should be able to copy-paste table description to create + new table (Sishen Freecity via Stack) + HBASE-886, HBASE-895 Sort the tables in the web UI, [shell] 'list' command + should emit a sorted list of tables (Krzysztof Szlapinski via Stack) + HBASE-884 Double and float converters for Bytes class + (Doğacan Güney via Stack) + HBASE-908 Add approximate counting to CountingBloomFilter + (Andrzej Bialecki via Stack) + HBASE-920 Make region balancing sloppier + HBASE-902 Add force compaction and force split operations to UI and Admin + HBASE-942 Add convenience methods to RowFilterSet + (Clint Morgan via Stack) + HBASE-943 to ColumnValueFilter: add filterIfColumnMissing property, add + SubString operator (Clint Morgan via Stack) + HBASE-937 Thrift getRow does not support specifying columns + (Doğacan Güney via Stack) + HBASE-959 Be able to get multiple RowResult at one time from client side + (Sishen Freecity via Stack) + HBASE-936 REST Interface: enable get number of rows from scanner interface + (Sishen Freecity via Stack) + HBASE-960 REST interface: more generic column family configure and also + get Rows using offset and limit (Sishen Freecity via Stack) + HBASE-817 Hbase/Shell Truncate + HBASE-949 Add an HBase Manual + HBASE-839 Update hadoop libs in hbase; move hbase TRUNK on to an hadoop + 0.19.0 RC + HBASE-785 Remove InfoServer, use HADOOP-3824 StatusHttpServer + instead (requires hadoop 0.19) + HBASE-81 When a scanner lease times out, throw a more "user friendly" exception + HBASE-978 Remove BloomFilterDescriptor. It is no longer used. + HBASE-975 Improve MapFile performance for start and end key + HBASE-961 Delete multiple columns by regular expression + (Samuel Guo via Stack) + HBASE-722 Shutdown and Compactions + HBASE-983 Declare Perl namespace in Hbase.thrift + HBASE-987 We need a Hbase Partitioner for TableMapReduceUtil.initTableReduceJob + MR Jobs (Billy Pearson via Stack) + HBASE-993 Turn off logging of every catalog table row entry on every scan + HBASE-992 Up the versions kept by catalog tables; currently 1. Make it 10? + HBASE-998 Narrow getClosestRowBefore by passing column family + HBASE-999 Up versions on historian and keep history of deleted regions for a + while rather than delete immediately + HBASE-938 Major compaction period is not checked periodically + HBASE-947 [Optimization] Major compaction should remove deletes as well as + the deleted cell + HBASE-675 Report correct server hosting a table split for assignment to + for MR Jobs + HBASE-927 We don't recover if HRS hosting -ROOT-/.META. goes down + HBASE-1013 Add debugging around commit log cleanup + HBASE-972 Update hbase trunk to use released hadoop 0.19.0 + HBASE-1022 Add storefile index size to hbase metrics + HBASE-1026 Tests in mapred are failing + HBASE-1020 Regionserver OOME handler should dump vital stats + HBASE-1018 Regionservers should report detailed health to master + HBASE-1034 Remove useless TestToString unit test + HBASE-1030 Bit of polish on HBASE-1018 + HBASE-847 new API: HTable.getRow with numVersion specified + (Doğacan Güney via Stack) + HBASE-1048 HLog: Found 0 logs to remove out of total 1450; oldest + outstanding seqnum is 162297053 fr om region -ROOT-,,0 + HBASE-1055 Better vm stats on startup + HBASE-1065 Minor logging improvements in the master + HBASE-1053 bring recent rpc changes down from hadoop + HBASE-1056 [migration] enable blockcaching on .META. table + HBASE-1069 Show whether HRegion major compacts or not in INFO level + HBASE-1066 Master should support close/open/reassignment/enable/disable + operations on individual regions + HBASE-1062 Compactions at (re)start on a large table can overwhelm DFS + HBASE-1102 boolean HTable.exists() + HBASE-1105 Remove duplicated code in HCM, add javadoc to RegionState, etc. + HBASE-1106 Expose getClosestRowBefore in HTable + (Michael Gottesman via Stack) + HBASE-1082 Administrative functions for table/region maintenance + HBASE-1090 Atomic Check And Save in HTable (Michael Gottesman via Stack) + HBASE-1137 Add not on xceivers count to overview documentation + + NEW FEATURES + HBASE-875 Use MurmurHash instead of JenkinsHash [in bloomfilters] + (Andrzej Bialecki via Stack) + HBASE-625 Metrics support for cluster load history: emissions and graphs + HBASE-883 Secondary indexes (Clint Morgan via Andrew Purtell) + HBASE-728 Support for HLog appends + + OPTIMIZATIONS + HBASE-748 Add an efficient way to batch update many rows + HBASE-887 Fix a hotspot in scanners + HBASE-967 [Optimization] Cache cell maximum length (HCD.getMaxValueLength); + its used checking batch size + HBASE-940 Make the TableOutputFormat batching-aware + HBASE-576 Investigate IPC performance + +Release 0.18.0 - September 21st, 2008 + + INCOMPATIBLE CHANGES + HBASE-697 Thrift idl needs update/edit to match new 0.2 API (and to fix bugs) + (Tim Sell via Stack) + HBASE-822 Update thrift README and HBase.thrift to use thrift 20080411 + Updated all other languages examples (only python went in) + + BUG FIXES + HBASE-881 Fixed bug when Master tries to reassign split or offline regions + from a dead server + HBASE-860 Fixed Bug in IndexTableReduce where it concerns writing lucene + index fields. + HBASE-805 Remove unnecessary getRow overloads in HRS (Jonathan Gray via + Jim Kellerman) (Fix whitespace diffs in HRegionServer) + HBASE-811 HTD is not fully copyable (Andrew Purtell via Jim Kellerman) + HBASE-729 Client region/metadata cache should have a public method for + invalidating entries (Andrew Purtell via Stack) + HBASE-819 Remove DOS-style ^M carriage returns from all code where found + (Jonathan Gray via Jim Kellerman) + HBASE-818 Deadlock running 'flushSomeRegions' (Andrew Purtell via Stack) + HBASE-820 Need mainline to flush when 'Blocking updates' goes up. + (Jean-Daniel Cryans via Stack) + HBASE-821 UnknownScanner happens too often (Jean-Daniel Cryans via Stack) + HBASE-813 Add a row counter in the new shell (Jean-Daniel Cryans via Stack) + HBASE-824 Bug in Hlog we print array of byes for region name + (Billy Pearson via Stack) + HBASE-825 Master logs showing byte [] in place of string in logging + (Billy Pearson via Stack) + HBASE-808,809 MAX_VERSIONS not respected, and Deletall doesn't and inserts + after delete don't work as expected + (Jean-Daniel Cryans via Stack) + HBASE-831 committing BatchUpdate with no row should complain + (Andrew Purtell via Jim Kellerman) + HBASE-833 Doing an insert with an unknown family throws a NPE in HRS + HBASE-810 Prevent temporary deadlocks when, during a scan with write + operations, the region splits (Jean-Daniel Cryans via Jim + Kellerman) + HBASE-843 Deleting and recreating a table in a single process does not work + (Jonathan Gray via Jim Kellerman) + HBASE-849 Speed improvement in JenkinsHash (Andrzej Bialecki via Stack) + HBASE-552 Bloom filter bugs (Andrzej Bialecki via Jim Kellerman) + HBASE-762 deleteFamily takes timestamp, should only take row and family. + Javadoc describes both cases but only implements the timestamp + case. (Jean-Daniel Cryans via Jim Kellerman) + HBASE-768 This message 'java.io.IOException: Install 0.1.x of hbase and run + its migration first' is useless (Jean-Daniel Cryans via Jim + Kellerman) + HBASE-826 Delete table followed by recreation results in honked table + HBASE-834 'Major' compactions and upper bound on files we compact at any + one time (Billy Pearson via Stack) + HBASE-836 Update thrift examples to work with changed IDL (HBASE-697) + (Toby White via Stack) + HBASE-854 hbase-841 broke build on hudson? - makes sure that proxies are + closed. (Andrew Purtell via Jim Kellerman) + HBASE-855 compaction can return less versions then we should in some cases + (Billy Pearson via Stack) + HBASE-832 Problem with row keys beginnig with characters < than ',' and + the region location cache + HBASE-864 Deadlock in regionserver + HBASE-865 Fix javadoc warnings (Rong-En Fan via Jim Kellerman) + HBASE-872 Getting exceptions in shell when creating/disabling tables + HBASE-868 Incrementing binary rows cause strange behavior once table + splits (Jonathan Gray via Stack) + HBASE-877 HCM is unable to find table with multiple regions which contains + binary (Jonathan Gray via Stack) + + IMPROVEMENTS + HBASE-801 When a table haven't disable, shell could response in a "user + friendly" way. + HBASE-816 TableMap should survive USE (Andrew Purtell via Stack) + HBASE-812 Compaction needs little better skip algo (Daniel Leffel via Stack) + HBASE-806 Change HbaseMapWritable and RowResult to implement SortedMap + instead of Map (Jonathan Gray via Stack) + HBASE-795 More Table operation in TableHandler for REST interface: part 1 + (Sishen Freecity via Stack) + HBASE-795 More Table operation in TableHandler for REST interface: part 2 + (Sishen Freecity via Stack) + HBASE-830 Debugging HCM.locateRegionInMeta is painful + HBASE-784 Base hbase-0.3.0 on hadoop-0.18 + HBASE-841 Consolidate multiple overloaded methods in HRegionInterface, + HRegionServer (Jean-Daniel Cryans via Jim Kellerman) + HBASE-840 More options on the row query in REST interface + (Sishen Freecity via Stack) + HBASE-874 deleting a table kills client rpc; no subsequent communication if + shell or thrift server, etc. (Jonathan Gray via Jim Kellerman) + HBASE-871 Major compaction periodicity should be specifyable at the column + family level, not cluster wide (Jonathan Gray via Stack) + HBASE-465 Fix javadoc for all public declarations + HBASE-882 The BatchUpdate class provides, put(col, cell) and delete(col) + but no get() (Ryan Smith via Stack and Jim Kellerman) + + NEW FEATURES + HBASE-787 Postgresql to HBase table replication example (Tim Sell via Stack) + HBASE-798 Provide Client API to explicitly lock and unlock rows (Jonathan + Gray via Jim Kellerman) + HBASE-798 Add missing classes: UnknownRowLockException and RowLock which + were present in previous versions of the patches for this issue, + but not in the version that was committed. Also fix a number of + compilation problems that were introduced by patch. + HBASE-669 MultiRegion transactions with Optimistic Concurrency Control + (Clint Morgan via Stack) + HBASE-842 Remove methods that have Text as a parameter and were deprecated + in 0.2.1 (Jean-Daniel Cryans via Jim Kellerman) + + OPTIMIZATIONS + +Release 0.2.0 - August 8, 2008. + + INCOMPATIBLE CHANGES + HBASE-584 Names in the filter interface are confusing (Clint Morgan via + Jim Kellerman) (API change for filters) + HBASE-601 Just remove deprecated methods in HTable; 0.2 is not backward + compatible anyways + HBASE-82 Row keys should be array of bytes + HBASE-76 Purge servers of Text (Done as part of HBASE-82 commit). + HBASE-487 Replace hql w/ a hbase-friendly jirb or jython shell + Part 1: purge of hql and added raw jirb in its place. + HBASE-521 Improve client scanner interface + HBASE-288 Add in-memory caching of data. Required update of hadoop to + 0.17.0-dev.2008-02-07_12-01-58. (Tom White via Stack) + HBASE-696 Make bloomfilter true/false and self-sizing + HBASE-720 clean up inconsistencies around deletes (Izaak Rubin via Stack) + HBASE-796 Deprecates Text methods from HTable + (Michael Gottesman via Stack) + + BUG FIXES + HBASE-574 HBase does not load hadoop native libs (Rong-En Fan via Stack) + HBASE-598 Loggging, no .log file; all goes into .out + HBASE-622 Remove StaticTestEnvironment and put a log4j.properties in src/test + HBASE-624 Master will shut down if number of active region servers is zero + even if shutdown was not requested + HBASE-629 Split reports incorrect elapsed time + HBASE-623 Migration script for hbase-82 + HBASE-630 Default hbase.rootdir is garbage + HBASE-589 Remove references to deprecated methods in Hadoop once + hadoop-0.17.0 is released + HBASE-638 Purge \r from src + HBASE-644 DroppedSnapshotException but RegionServer doesn't restart + HBASE-641 Improve master split logging + HBASE-642 Splitting log in a hostile environment -- bad hdfs -- we drop + write-ahead-log edits + HBASE-646 EOFException opening HStoreFile info file (spin on HBASE-645and 550) + HBASE-648 If mapfile index is empty, run repair + HBASE-640 TestMigrate failing on hudson + HBASE-651 Table.commit should throw NoSuchColumnFamilyException if column + family doesn't exist + HBASE-649 API polluted with default and protected access data members and methods + HBASE-650 Add String versions of get, scanner, put in HTable + HBASE-656 Do not retry exceptions such as unknown scanner or illegal argument + HBASE-659 HLog#cacheFlushLock not cleared; hangs a region + HBASE-663 Incorrect sequence number for cache flush + HBASE-655 Need programmatic way to add column family: need programmatic way + to enable/disable table + HBASE-654 API HTable.getMetadata().addFamily shouldn't be exposed to user + HBASE-666 UnmodifyableHRegionInfo gives the wrong encoded name + HBASE-668 HBASE-533 broke build + HBASE-670 Historian deadlocks if regionserver is at global memory boundary + and is hosting .META. + HBASE-665 Server side scanner doesn't honor stop row + HBASE-662 UI in table.jsp gives META locations, not the table's regions + location (Jean-Daniel Cryans via Stack) + HBASE-676 Bytes.getInt returns a long (Clint Morgan via Stack) + HBASE-680 Config parameter hbase.io.index.interval should be + hbase.index.interval, according to HBaseMapFile.HbaseWriter + (LN via Stack) + HBASE-682 Unnecessary iteration in HMemcache.internalGet? got much better + reading performance after break it (LN via Stack) + HBASE-686 MemcacheScanner didn't return the first row(if it exists), + because HScannerInterface's output incorrect (LN via Jim Kellerman) + HBASE-691 get* and getScanner are different in how they treat column parameter + HBASE-694 HStore.rowAtOrBeforeFromMapFile() fails to locate the row if # of mapfiles >= 2 + (Rong-En Fan via Bryan) + HBASE-652 dropping table fails silently if table isn't disabled + HBASE-683 can not get svn revision # at build time if locale is not english + (Rong-En Fan via Stack) + HBASE-699 Fix TestMigrate up on Hudson + HBASE-615 Region balancer oscillates during cluster startup + HBASE-613 Timestamp-anchored scanning fails to find all records + HBASE-681 NPE in Memcache + HBASE-701 Showing bytes in log when should be String + HBASE-702 deleteall doesn't + HBASE-704 update new shell docs and commands on help menu + HBASE-709 Deadlock while rolling WAL-log while finishing flush + HBASE-710 If clocks are way off, then we can have daughter split come + before rather than after its parent in .META. + HBASE-714 Showing bytes in log when should be string (2) + HBASE-627 Disable table doesn't work reliably + HBASE-716 TestGet2.testGetClosestBefore fails with hadoop-0.17.1 + HBASE-715 Base HBase 0.2 on Hadoop 0.17.1 + HBASE-718 hbase shell help info + HBASE-717 alter table broke with new shell returns InvalidColumnNameException + HBASE-573 HBase does not read hadoop-*.xml for dfs configuration after + moving out hadoop/contrib + HBASE-11 Unexpected exits corrupt DFS + HBASE-12 When hbase regionserver restarts, it says "impossible state for + createLease()" + HBASE-575 master dies with stack overflow error if rootdir isn't qualified + HBASE-582 HBase 554 forgot to clear results on each iteration caused by a filter + (Clint Morgan via Stack) + HBASE-532 Odd interaction between HRegion.get, HRegion.deleteAll and compactions + HBASE-10 HRegionServer hangs upon exit due to DFSClient Exception + HBASE-595 RowFilterInterface.rowProcessed() is called *before* fhe final + filtering decision is made (Clint Morgan via Stack) + HBASE-586 HRegion runs HStore memcache snapshotting -- fix it so only HStore + knows about workings of memcache + HBASE-588 Still a 'hole' in scanners, even after HBASE-532 + HBASE-604 Don't allow CLASSPATH from environment pollute the hbase CLASSPATH + HBASE-608 HRegionServer::getThisIP() checks hadoop config var for dns interface name + (Jim R. Wilson via Stack) + HBASE-609 Master doesn't see regionserver edits because of clock skew + HBASE-607 MultiRegionTable.makeMultiRegionTable is not deterministic enough + for regression tests + HBASE-405 TIF and TOF use log4j directly rather than apache commons-logging + HBASE-618 We always compact if 2 files, regardless of the compaction threshold setting + HBASE-619 Fix 'logs' link in UI + HBASE-478 offlining of table does not run reliably + HBASE-453 undeclared throwable exception from HTable.get + HBASE-620 testmergetool failing in branch and trunk since hbase-618 went in + HBASE-550 EOF trying to read reconstruction log stops region deployment + HBASE-551 Master stuck splitting server logs in shutdown loop; on each + iteration, edits are aggregated up into the millions + HBASE-505 Region assignments should never time out so long as the region + server reports that it is processing the open request + HBASE-561 HBase package does not include LICENSE.txt nor build.xml + HBASE-563 TestRowFilterAfterWrite erroneously sets master address to + 0.0.0.0:60100 rather than relying on conf + HBASE-507 Use Callable pattern to sleep between retries + HBASE-564 Don't do a cache flush if there are zero entries in the cache. + HBASE-554 filters generate StackOverflowException + HBASE-567 Reused BatchUpdate instances accumulate BatchOperations + HBASE-577 NPE getting scanner + HBASE-19 CountingBloomFilter can overflow its storage + (Stu Hood and Bryan Duxbury via Stack) + HBASE-28 thrift put/mutateRow methods need to throw IllegalArgument + exceptions (Dave Simpson via Bryan Duxbury via Stack) + HBASE-2 hlog numbers should wrap around when they reach 999 + (Bryan Duxbury via Stack) + HBASE-421 TestRegionServerExit broken + HBASE-426 hbase can't find remote filesystem + HBASE-437 Clear Command should use system.out (Edward Yoon via Stack) + HBASE-434, HBASE-435 TestTableIndex and TestTableMapReduce failed in Hudson builds + HBASE-446 Fully qualified hbase.rootdir doesn't work + HBASE-438 XMLOutputter state should be initialized. (Edward Yoon via Stack) + HBASE-8 Delete table does not remove the table directory in the FS + HBASE-428 Under continuous upload of rows, WrongRegionExceptions are thrown + that reach the client even after retries + HBASE-460 TestMigrate broken when HBase moved to subproject + HBASE-462 Update migration tool + HBASE-473 When a table is deleted, master sends multiple close messages to + the region server + HBASE-490 Doubly-assigned .META.; master uses one and clients another + HBASE-492 hbase TRUNK does not build against hadoop TRUNK + HBASE-496 impossible state for createLease writes 400k lines in about 15mins + HBASE-472 Passing on edits, we dump all to log + HBASE-495 No server address listed in .META. + HBASE-433 HBASE-251 Region server should delete restore log after successful + restore, Stuck replaying the edits of crashed machine. + HBASE-27 hregioninfo cell empty in meta table + HBASE-501 Empty region server address in info:server entry and a + startcode of -1 in .META. + HBASE-516 HStoreFile.finalKey does not update the final key if it is not + the top region of a split region + HBASE-525 HTable.getRow(Text) does not work (Clint Morgan via Bryan Duxbury) + HBASE-524 Problems with getFull + HBASE-528 table 'does not exist' when it does + HBASE-531 Merge tool won't merge two overlapping regions (port HBASE-483 to + trunk) + HBASE-537 Wait for hdfs to exit safe mode + HBASE-476 RegexpRowFilter behaves incorectly when there are multiple store + files (Clint Morgan via Jim Kellerman) + HBASE-527 RegexpRowFilter does not work when there are columns from + multiple families (Clint Morgan via Jim Kellerman) + HBASE-534 Double-assignment at SPLIT-time + HBASE-712 midKey found compacting is the first, not necessarily the optimal + HBASE-719 Find out why users have network problems in HBase and not in Hadoop + and HConnectionManager (Jean-Daniel Cryans via Stack) + HBASE-703 Invalid regions listed by regionserver.jsp (Izaak Rubin via Stack) + HBASE-674 Memcache size unreliable + HBASE-726 Unit tests won't run because of a typo (Sebastien Rainville via Stack) + HBASE-727 Client caught in an infinite loop when trying to connect to cached + server locations (Izaak Rubin via Stack) + HBASE-732 shell formatting error with the describe command + (Izaak Rubin via Stack) + HBASE-731 delete, deletefc in HBase shell do not work correctly + (Izaak Rubin via Stack) + HBASE-734 scan '.META.', {LIMIT => 10} crashes (Izaak Rubin via Stack) + HBASE-736 Should have HTable.deleteAll(String row) and HTable.deleteAll(Text row) + (Jean-Daniel Cryans via Stack) + HBASE-740 ThriftServer getting table names incorrectly (Tim Sell via Stack) + HBASE-742 Rename getMetainfo in HTable as getTableDescriptor + HBASE-739 HBaseAdmin.createTable() using old HTableDescription doesn't work + (Izaak Rubin via Stack) + HBASE-744 BloomFilter serialization/deserialization broken + HBASE-742 Column length limit is not enforced (Jean-Daniel Cryans via Stack) + HBASE-737 Scanner: every cell in a row has the same timestamp + HBASE-700 hbase.io.index.interval need be configuratable in column family + (Andrew Purtell via Stack) + HBASE-62 Allow user add arbitrary key/value pairs to table and column + descriptors (Andrew Purtell via Stack) + HBASE-34 Set memcache flush size per column (Andrew Purtell via Stack) + HBASE-42 Set region split size on table creation (Andrew Purtell via Stack) + HBASE-43 Add a read-only attribute to columns (Andrew Purtell via Stack) + HBASE-424 Should be able to enable/disable .META. table + HBASE-679 Regionserver addresses are still not right in the new tables page + HBASE-758 Throwing IOE read-only when should be throwing NSRE + HBASE-743 bin/hbase migrate upgrade fails when redo logs exists + HBASE-754 The JRuby shell documentation is wrong in "get" and "put" + (Jean-Daniel Cryans via Stack) + HBASE-756 In HBase shell, the put command doesn't process the timestamp + (Jean-Daniel Cryans via Stack) + HBASE-757 REST mangles table names (Sishen via Stack) + HBASE-706 On OOME, regionserver sticks around and doesn't go down with cluster + (Jean-Daniel Cryans via Stack) + HBASE-759 TestMetaUtils failing on hudson + HBASE-761 IOE: Stream closed exception all over logs + HBASE-763 ClassCastException from RowResult.get(String) + (Andrew Purtell via Stack) + HBASE-764 The name of column request has padding zero using REST interface + (Sishen Freecity via Stack) + HBASE-750 NPE caused by StoreFileScanner.updateReaders + HBASE-769 TestMasterAdmin fails throwing RegionOfflineException when we're + expecting IllegalStateException + HBASE-766 FileNotFoundException trying to load HStoreFile 'data' + HBASE-770 Update HBaseRPC to match hadoop 0.17 RPC + HBASE-780 Can't scan '.META.' from new shell + HBASE-424 Should be able to enable/disable .META. table + HBASE-771 Names legal in 0.1 are not in 0.2; breaks migration + HBASE-788 Div by zero in Master.jsp (Clint Morgan via Jim Kellerman) + HBASE-791 RowCount doesn't work (Jean-Daniel Cryans via Stack) + HBASE-751 dfs exception and regionserver stuck during heavy write load + HBASE-793 HTable.getStartKeys() ignores table names when matching columns + (Andrew Purtell and Dru Jensen via Stack) + HBASE-790 During import, single region blocks requests for >10 minutes, + thread dumps, throws out pending requests, and continues + (Jonathan Gray via Stack) + + IMPROVEMENTS + HBASE-559 MR example job to count table rows + HBASE-596 DemoClient.py (Ivan Begtin via Stack) + HBASE-581 Allow adding filters to TableInputFormat (At same time, ensure TIF + is subclassable) (David Alves via Stack) + HBASE-603 When an exception bubbles out of getRegionServerWithRetries, wrap + the exception with a RetriesExhaustedException + HBASE-600 Filters have excessive DEBUG logging + HBASE-611 regionserver should do basic health check before reporting + alls-well to the master + HBASE-614 Retiring regions is not used; exploit or remove + HBASE-538 Improve exceptions that come out on client-side + HBASE-569 DemoClient.php (Jim R. Wilson via Stack) + HBASE-522 Where new Text(string) might be used in client side method calls, + add an overload that takes String (Done as part of HBASE-82) + HBASE-570 Remove HQL unit test (Done as part of HBASE-82 commit). + HBASE-626 Use Visitor pattern in MetaRegion to reduce code clones in HTable + and HConnectionManager (Jean-Daniel Cryans via Stack) + HBASE-621 Make MAX_VERSIONS work like TTL: In scans and gets, check + MAX_VERSIONs setting and return that many only rather than wait on + compaction (Jean-Daniel Cryans via Stack) + HBASE-504 Allow HMsg's carry a payload: e.g. exception that happened over + on the remote side. + HBASE-583 RangeRowFilter/ColumnValueFilter to allow choice of rows based on + a (lexicographic) comparison to column's values + (Clint Morgan via Stack) + HBASE-579 Add hadoop 0.17.x + HBASE-660 [Migration] addColumn/deleteColumn functionality in MetaUtils + HBASE-632 HTable.getMetadata is very inefficient + HBASE-671 New UI page displaying all regions in a table should be sorted + HBASE-672 Sort regions in the regionserver UI + HBASE-677 Make HTable, HRegion, HRegionServer, HStore, and HColumnDescriptor + subclassable (Clint Morgan via Stack) + HBASE-682 Regularize toString + HBASE-672 Sort regions in the regionserver UI + HBASE-469 Streamline HStore startup and compactions + HBASE-544 Purge startUpdate from internal code and test cases + HBASE-557 HTable.getRow() should receive RowResult objects + HBASE-452 "region offline" should throw IOException, not IllegalStateException + HBASE-541 Update hadoop jars. + HBASE-523 package-level javadoc should have example client + HBASE-415 Rewrite leases to use DelayedBlockingQueue instead of polling + HBASE-35 Make BatchUpdate public in the API + HBASE-409 Add build path to svn:ignore list (Edward Yoon via Stack) + HBASE-408 Add .classpath and .project to svn:ignore list + (Edward Yoon via Stack) + HBASE-410 Speed up the test suite (make test timeout 5 instead of 15 mins). + HBASE-281 Shell should allow deletions in .META. and -ROOT- tables + (Edward Yoon & Bryan Duxbury via Stack) + HBASE-56 Unnecessary HQLClient Object creation in a shell loop + (Edward Yoon via Stack) + HBASE-3 rest server: configure number of threads for jetty + (Bryan Duxbury via Stack) + HBASE-416 Add apache-style logging to REST server and add setting log + level, etc. + HBASE-406 Remove HTable and HConnection close methods + (Bryan Duxbury via Stack) + HBASE-418 Move HMaster and related classes into master package + (Bryan Duxbury via Stack) + HBASE-410 Speed up the test suite - Apparently test timeout was too + aggressive for Hudson. TestLogRolling timed out even though it + was operating properly. Change test timeout to 10 minutes. + HBASE-436 website: http://hadoop.apache.org/hbase + HBASE-417 Factor TableOperation and subclasses into separate files from + HMaster (Bryan Duxbury via Stack) + HBASE-440 Add optional log roll interval so that log files are garbage + collected + HBASE-407 Keep HRegionLocation information in LRU structure + HBASE-444 hbase is very slow at determining table is not present + HBASE-438 XMLOutputter state should be initialized. + HBASE-414 Move client classes into client package + HBASE-79 When HBase needs to be migrated, it should display a message on + stdout, not just in the logs + HBASE-461 Simplify leases. + HBASE-419 Move RegionServer and related classes into regionserver package + HBASE-457 Factor Master into Master, RegionManager, and ServerManager + HBASE-464 HBASE-419 introduced javadoc errors + HBASE-468 Move HStoreKey back to o.a.h.h + HBASE-442 Move internal classes out of HRegionServer + HBASE-466 Move HMasterInterface, HRegionInterface, and + HMasterRegionInterface into o.a.h.h.ipc + HBASE-479 Speed up TestLogRolling + HBASE-480 Tool to manually merge two regions + HBASE-477 Add support for an HBASE_CLASSPATH + HBASE-443 Move internal classes out of HStore + HBASE-515 At least double default timeouts between regionserver and master + HBASE-529 RegionServer needs to recover if datanode goes down + HBASE-456 Clearly state which ports need to be opened in order to run HBase + HBASE-536 Remove MiniDFS startup from MiniHBaseCluster + HBASE-521 Improve client scanner interface + HBASE-562 Move Exceptions to subpackages (Jean-Daniel Cryans via Stack) + HBASE-631 HTable.getRow() for only a column family + (Jean-Daniel Cryans via Stack) + HBASE-731 Add a meta refresh tag to the Web ui for master and region server + (Jean-Daniel Cryans via Stack) + HBASE-735 hbase shell doesn't trap CTRL-C signal (Jean-Daniel Cryans via Stack) + HBASE-730 On startup, rinse STARTCODE and SERVER from .META. + (Jean-Daniel Cryans via Stack) + HBASE-738 overview.html in need of updating (Izaak Rubin via Stack) + HBASE-745 scaling of one regionserver, improving memory and cpu usage (partial) + (LN via Stack) + HBASE-746 Batching row mutations via thrift (Tim Sell via Stack) + HBASE-772 Up default lease period from 60 to 120 seconds + HBASE-779 Test changing hbase.hregion.memcache.block.multiplier to 2 + HBASE-783 For single row, single family retrieval, getRow() works half + as fast as getScanner().next() (Jean-Daniel Cryans via Stack) + HBASE-789 add clover coverage report targets (Rong-en Fan via Stack) + + NEW FEATURES + HBASE-47 Option to set TTL for columns in hbase + (Andrew Purtell via Bryan Duxbury and Stack) + HBASE-23 UI listing regions should be sorted by address and show additional + region state (Jean-Daniel Cryans via Stack) + HBASE-639 Add HBaseAdmin.getTableDescriptor function + HBASE-533 Region Historian + HBASE-487 Replace hql w/ a hbase-friendly jirb or jython shell + HBASE-548 Tool to online single region + HBASE-71 Master should rebalance region assignments periodically + HBASE-512 Add configuration for global aggregate memcache size + HBASE-40 Add a method of getting multiple (but not all) cells for a row + at once + HBASE-506 When an exception has to escape ServerCallable due to exhausted + retries, show all the exceptions that lead to this situation + HBASE-747 Add a simple way to do batch updates of many rows (Jean-Daniel + Cryans via JimK) + HBASE-733 Enhance Cell so that it can contain multiple values at multiple + timestamps + HBASE-511 Do exponential backoff in clients on NSRE, WRE, ISE, etc. + (Andrew Purtell via Jim Kellerman) + + OPTIMIZATIONS + HBASE-430 Performance: Scanners and getRow return maps with duplicate data + +Release 0.1.3 - 07/25/2008 + + BUG FIXES + HBASE-644 DroppedSnapshotException but RegionServer doesn't restart + HBASE-645 EOFException opening region (HBASE-550 redux) + HBASE-641 Improve master split logging + HBASE-642 Splitting log in a hostile environment -- bad hdfs -- we drop + write-ahead-log edits + HBASE-646 EOFException opening HStoreFile info file (spin on HBASE-645 and 550) + HBASE-648 If mapfile index is empty, run repair + HBASE-659 HLog#cacheFlushLock not cleared; hangs a region + HBASE-663 Incorrect sequence number for cache flush + HBASE-652 Dropping table fails silently if table isn't disabled + HBASE-674 Memcache size unreliable + HBASE-665 server side scanner doesn't honor stop row + HBASE-681 NPE in Memcache (Clint Morgan via Jim Kellerman) + HBASE-680 config parameter hbase.io.index.interval should be + hbase.index.interval, accroding to HBaseMapFile.HbaseWriter + (LN via Stack) + HBASE-684 unnecessary iteration in HMemcache.internalGet? got much better + reading performance after break it (LN via Stack) + HBASE-686 MemcacheScanner didn't return the first row(if it exists), + because HScannerInterface's output incorrect (LN via Jim Kellerman) + HBASE-613 Timestamp-anchored scanning fails to find all records + HBASE-709 Deadlock while rolling WAL-log while finishing flush + HBASE-707 High-load import of data into single table/family never triggers split + HBASE-710 If clocks are way off, then we can have daughter split come + before rather than after its parent in .META. + +Release 0.1.2 - 05/13/2008 + + BUG FIXES + HBASE-577 NPE getting scanner + HBASE-574 HBase does not load hadoop native libs (Rong-En Fan via Stack). + HBASE-11 Unexpected exits corrupt DFS - best we can do until we have at + least a subset of HADOOP-1700 + HBASE-573 HBase does not read hadoop-*.xml for dfs configuration after + moving out hadoop/contrib + HBASE-12 when hbase regionserver restarts, it says "impossible state for + createLease()" + HBASE-575 master dies with stack overflow error if rootdir isn't qualified + HBASE-500 Regionserver stuck on exit + HBASE-582 HBase 554 forgot to clear results on each iteration caused by a filter + (Clint Morgan via Stack) + HBASE-532 Odd interaction between HRegion.get, HRegion.deleteAll and compactions + HBASE-590 HBase migration tool does not get correct FileSystem or root + directory if configuration is not correct + HBASE-595 RowFilterInterface.rowProcessed() is called *before* fhe final + filtering decision is made (Clint Morgan via Stack) + HBASE-586 HRegion runs HStore memcache snapshotting -- fix it so only HStore + knows about workings of memcache + HBASE-572 Backport HBASE-512 to 0.1 branch + HBASE-588 Still a 'hole' in scanners, even after HBASE-532 + HBASE-604 Don't allow CLASSPATH from environment pollute the hbase CLASSPATH + HBASE-608 HRegionServer::getThisIP() checks hadoop config var for dns interface name + (Jim R. Wilson via Stack) + HBASE-609 Master doesn't see regionserver edits because of clock skew + HBASE-607 MultiRegionTable.makeMultiRegionTable is not deterministic enough + for regression tests + HBASE-478 offlining of table does not run reliably + HBASE-618 We always compact if 2 files, regardless of the compaction threshold setting + HBASE-619 Fix 'logs' link in UI + HBASE-620 testmergetool failing in branch and trunk since hbase-618 went in + + IMPROVEMENTS + HBASE-559 MR example job to count table rows + HBASE-578 Upgrade branch to 0.16.3 hadoop. + HBASE-596 DemoClient.py (Ivan Begtin via Stack) + + +Release 0.1.1 - 04/11/2008 + + BUG FIXES + HBASE-550 EOF trying to read reconstruction log stops region deployment + HBASE-551 Master stuck splitting server logs in shutdown loop; on each + iteration, edits are aggregated up into the millions + HBASE-505 Region assignments should never time out so long as the region + server reports that it is processing the open request + HBASE-552 Fix bloom filter bugs (Andrzej Bialecki via Jim Kellerman) + HBASE-507 Add sleep between retries + HBASE-555 Only one Worker in HRS; on startup, if assigned tens of regions, + havoc of reassignments because open processing is done in series + HBASE-547 UI shows hadoop version, not hbase version + HBASE-561 HBase package does not include LICENSE.txt nor build.xml + HBASE-556 Add 0.16.2 to hbase branch -- if it works + HBASE-563 TestRowFilterAfterWrite erroneously sets master address to + 0.0.0.0:60100 rather than relying on conf + HBASE-554 filters generate StackOverflowException (Clint Morgan via + Jim Kellerman) + HBASE-567 Reused BatchUpdate instances accumulate BatchOperations + + NEW FEATURES + HBASE-548 Tool to online single region + +Release 0.1.0 + + INCOMPATIBLE CHANGES + HADOOP-2750 Deprecated methods startBatchUpdate, commitBatch, abortBatch, + and renewLease have been removed from HTable (Bryan Duxbury via + Jim Kellerman) + HADOOP-2786 Move hbase out of hadoop core + HBASE-403 Fix build after move of hbase in svn + HBASE-494 Up IPC version on 0.1 branch so we cannot mistakenly connect + with a hbase from 0.16.0 + + NEW FEATURES + HBASE-506 When an exception has to escape ServerCallable due to exhausted retries, + show all the exceptions that lead to this situation + + OPTIMIZATIONS + + BUG FIXES + HADOOP-2731 Under load, regions become extremely large and eventually cause + region servers to become unresponsive + HADOOP-2693 NPE in getClosestRowBefore (Bryan Duxbury & Stack) + HADOOP-2599 Some minor improvements to changes in HADOOP-2443 + (Bryan Duxbury & Stack) + HADOOP-2773 Master marks region offline when it is recovering from a region + server death + HBASE-425 Fix doc. so it accomodates new hbase untethered context + HBase-421 TestRegionServerExit broken + HBASE-426 hbase can't find remote filesystem + HBASE-446 Fully qualified hbase.rootdir doesn't work + HBASE-428 Under continuous upload of rows, WrongRegionExceptions are + thrown that reach the client even after retries + HBASE-490 Doubly-assigned .META.; master uses one and clients another + HBASE-496 impossible state for createLease writes 400k lines in about 15mins + HBASE-472 Passing on edits, we dump all to log + HBASE-79 When HBase needs to be migrated, it should display a message on + stdout, not just in the logs + HBASE-495 No server address listed in .META. + HBASE-433 HBASE-251 Region server should delete restore log after successful + restore, Stuck replaying the edits of crashed machine. + HBASE-27 hregioninfo cell empty in meta table + HBASE-501 Empty region server address in info:server entry and a + startcode of -1 in .META. + HBASE-516 HStoreFile.finalKey does not update the final key if it is not + the top region of a split region + HBASE-524 Problems with getFull + HBASE-514 table 'does not exist' when it does + HBASE-537 Wait for hdfs to exit safe mode + HBASE-534 Double-assignment at SPLIT-time + + IMPROVEMENTS + HADOOP-2555 Refactor the HTable#get and HTable#getRow methods to avoid + repetition of retry-on-failure logic (thanks to Peter Dolan and + Bryan Duxbury) + HBASE-281 Shell should allow deletions in .META. and -ROOT- tables + HBASE-480 Tool to manually merge two regions + HBASE-477 Add support for an HBASE_CLASSPATH + HBASE-515 At least double default timeouts between regionserver and master + HBASE-482 package-level javadoc should have example client or at least + point at the FAQ + HBASE-497 RegionServer needs to recover if datanode goes down + HBASE-456 Clearly state which ports need to be opened in order to run HBase + HBASE-483 Merge tool won't merge two overlapping regions + HBASE-476 RegexpRowFilter behaves incorectly when there are multiple store + files (Clint Morgan via Jim Kellerman) + HBASE-527 RegexpRowFilter does not work when there are columns from + multiple families (Clint Morgan via Jim Kellerman) + +Release 0.16.0 + + 2008/02/04 HBase is now a subproject of Hadoop. The first HBase release as + a subproject will be release 0.1.0 which will be equivalent to + the version of HBase included in Hadoop 0.16.0. In order to + accomplish this, the HBase portion of HBASE-288 (formerly + HADOOP-1398) has been backed out. Once 0.1.0 is frozen (depending + mostly on changes to infrastructure due to becoming a sub project + instead of a contrib project), this patch will re-appear on HBase + trunk. + + INCOMPATIBLE CHANGES + HADOOP-2056 A table with row keys containing colon fails to split regions + HADOOP-2079 Fix generated HLog, HRegion names + HADOOP-2495 Minor performance improvements: Slim-down BatchOperation, etc. + HADOOP-2506 Remove the algebra package + HADOOP-2519 Performance improvements: Customized RPC serialization + HADOOP-2478 Restructure how HBase lays out files in the file system (phase 1) + (test input data) + HADOOP-2478 Restructure how HBase lays out files in the file system (phase 2) + Includes migration tool org.apache.hadoop.hbase.util.Migrate + HADOOP-2558 org.onelab.filter.BloomFilter class uses 8X the memory it should + be using + + NEW FEATURES + HADOOP-2061 Add new Base64 dialects + HADOOP-2084 Add a LocalHBaseCluster + HADOOP-2068 RESTful interface (Bryan Duxbury via Stack) + HADOOP-2316 Run REST servlet outside of master + (Bryan Duxbury & Stack) + HADOOP-1550 No means of deleting a'row' (Bryan Duxbuery via Stack) + HADOOP-2384 Delete all members of a column family on a specific row + (Bryan Duxbury via Stack) + HADOOP-2395 Implement "ALTER TABLE ... CHANGE column" operation + (Bryan Duxbury via Stack) + HADOOP-2240 Truncate for hbase (Edward Yoon via Stack) + HADOOP-2389 Provide multiple language bindings for HBase (Thrift) + (David Simpson via Stack) + + OPTIMIZATIONS + HADOOP-2479 Save on number of Text object creations + HADOOP-2485 Make mapfile index interval configurable (Set default to 32 + instead of 128) + HADOOP-2553 Don't make Long objects calculating hbase type hash codes + HADOOP-2377 Holding open MapFile.Readers is expensive, so use less of them + HADOOP-2407 Keeping MapFile.Reader open is expensive: Part 2 + HADOOP-2533 Performance: Scanning, just creating MapWritable in next + consumes >20% CPU + HADOOP-2443 Keep lazy cache of regions in client rather than an + 'authoritative' list (Bryan Duxbury via Stack) + HADOOP-2600 Performance: HStore.getRowKeyAtOrBefore should use + MapFile.Reader#getClosest (before) + (Bryan Duxbury via Stack) + + BUG FIXES + HADOOP-2059 In tests, exceptions in min dfs shutdown should not fail test + (e.g. nightly #272) + HADOOP-2064 TestSplit assertion and NPE failures (Patch build #952 and #953) + HADOOP-2124 Use of `hostname` does not work on Cygwin in some cases + HADOOP-2083 TestTableIndex failed in #970 and #956 + HADOOP-2109 Fixed race condition in processing server lease timeout. + HADOOP-2137 hql.jsp : The character 0x19 is not valid + HADOOP-2109 Fix another race condition in processing dead servers, + Fix error online meta regions: was using region name and not + startKey as key for map.put. Change TestRegionServerExit to + always kill the region server for the META region. This makes + the test more deterministic and getting META reassigned was + problematic. + HADOOP-2155 Method expecting HBaseConfiguration throws NPE when given Configuration + HADOOP-2156 BufferUnderflowException for un-named HTableDescriptors + HADOOP-2161 getRow() is orders of magnitudes slower than get(), even on rows + with one column (Clint Morgan and Stack) + HADOOP-2040 Hudson hangs AFTER test has finished + HADOOP-2274 Excess synchronization introduced by HADOOP-2139 negatively + impacts performance + HADOOP-2196 Fix how hbase sits in hadoop 'package' product + HADOOP-2276 Address regression caused by HADOOP-2274, fix HADOOP-2173 (When + the master times out a region servers lease, the region server + may not restart) + HADOOP-2253 getRow can return HBASE::DELETEVAL cells + (Bryan Duxbury via Stack) + HADOOP-2295 Fix assigning a region to multiple servers + HADOOP-2234 TableInputFormat erroneously aggregates map values + HADOOP-2308 null regioninfo breaks meta scanner + HADOOP-2304 Abbreviated symbol parsing error of dir path in jar command + (Edward Yoon via Stack) + HADOOP-2320 Committed TestGet2 is managled (breaks build). + HADOOP-2322 getRow(row, TS) client interface not properly connected + HADOOP-2309 ConcurrentModificationException doing get of all region start keys + HADOOP-2321 TestScanner2 does not release resources which sometimes cause the + test to time out + HADOOP-2315 REST servlet doesn't treat / characters in row key correctly + (Bryan Duxbury via Stack) + HADOOP-2332 Meta table data selection in Hbase Shell + (Edward Yoon via Stack) + HADOOP-2347 REST servlet not thread safe but run in a threaded manner + (Bryan Duxbury via Stack) + HADOOP-2365 Result of HashFunction.hash() contains all identical values + HADOOP-2362 Leaking hdfs file handle on region split + HADOOP-2338 Fix NullPointerException in master server. + HADOOP-2380 REST servlet throws NPE when any value node has an empty string + (Bryan Duxbury via Stack) + HADOOP-2350 Scanner api returns null row names, or skips row names if + different column families do not have entries for some rows + HADOOP-2283 AlreadyBeingCreatedException (Was: Stuck replay of failed + regionserver edits) + HADOOP-2392 TestRegionServerExit has new failure mode since HADOOP-2338 + HADOOP-2324 Fix assertion failures in TestTableMapReduce + HADOOP-2396 NPE in HMaster.cancelLease + HADOOP-2397 The only time that a meta scanner should try to recover a log is + when the master is starting + HADOOP-2417 Fix critical shutdown problem introduced by HADOOP-2338 + HADOOP-2418 Fix assertion failures in TestTableMapReduce, TestTableIndex, + and TestTableJoinMapReduce + HADOOP-2414 Fix ArrayIndexOutOfBoundsException in bloom filters. + HADOOP-2430 Master will not shut down if there are no active region servers + HADOOP-2199 Add tools for going from hregion filename to region name in logs + HADOOP-2441 Fix build failures in TestHBaseCluster + HADOOP-2451 End key is incorrectly assigned in many region splits + HADOOP-2455 Error in Help-string of CREATE command (Edward Yoon via Stack) + HADOOP-2465 When split parent regions are cleaned up, not all the columns are + deleted + HADOOP-2468 TestRegionServerExit failed in Hadoop-Nightly #338 + HADOOP-2467 scanner truncates resultset when > 1 column families + HADOOP-2503 REST Insert / Select encoding issue (Bryan Duxbury via Stack) + HADOOP-2505 formatter classes missing apache license + HADOOP-2504 REST servlet method for deleting a scanner was not properly + mapped (Bryan Duxbury via Stack) + HADOOP-2507 REST servlet does not properly base64 row keys and column names + (Bryan Duxbury via Stack) + HADOOP-2530 Missing type in new hbase custom RPC serializer + HADOOP-2490 Failure in nightly #346 (Added debugging of hudson failures). + HADOOP-2558 fixes for build up on hudson (part 1, part 2, part 3, part 4) + HADOOP-2500 Unreadable region kills region servers + HADOOP-2579 Initializing a new HTable object against a nonexistent table + throws a NoServerForRegionException instead of a + TableNotFoundException when a different table has been created + previously (Bryan Duxbury via Stack) + HADOOP-2587 Splits blocked by compactions cause region to be offline for + duration of compaction. + HADOOP-2592 Scanning, a region can let out a row that its not supposed + to have + HADOOP-2493 hbase will split on row when the start and end row is the + same cause data loss (Bryan Duxbury via Stack) + HADOOP-2629 Shell digests garbage without complaint + HADOOP-2619 Compaction errors after a region splits + HADOOP-2621 Memcache flush flushing every 60 secs with out considering + the max memcache size + HADOOP-2584 Web UI displays an IOException instead of the Tables + HADOOP-2650 Remove Writables.clone and use WritableUtils.clone from + hadoop instead + HADOOP-2668 Documentation and improved logging so fact that hbase now + requires migration comes as less of a surprise + HADOOP-2686 Removed tables stick around in .META. + HADOOP-2688 IllegalArgumentException processing a shutdown stops + server going down and results in millions of lines of output + HADOOP-2706 HBase Shell crash + HADOOP-2712 under load, regions won't split + HADOOP-2675 Options not passed to rest/thrift + HADOOP-2722 Prevent unintentional thread exit in region server and master + HADOOP-2718 Copy Constructor HBaseConfiguration(Configuration) will override + hbase configurations if argumant is not an instance of + HBaseConfiguration. + HADOOP-2753 Back out 2718; programmatic config works but hbase*xml conf + is overridden + HADOOP-2718 Copy Constructor HBaseConfiguration(Configuration) will override + hbase configurations if argumant is not an instance of + HBaseConfiguration (Put it back again). + HADOOP-2631 2443 breaks HTable.getStartKeys when there is more than one + table or table you are enumerating isn't the first table + Delete empty file: src/contrib/hbase/src/java/org/apache/hadoop/hbase/mapred/ + TableOutputCollector.java per Nigel Daley + + IMPROVEMENTS + HADOOP-2401 Add convenience put method that takes writable + (Johan Oskarsson via Stack) + HADOOP-2074 Simple switch to enable DEBUG level-logging in hbase + HADOOP-2088 Make hbase runnable in $HADOOP_HOME/build(/contrib/hbase) + HADOOP-2126 Use Bob Jenkins' hash for bloom filters + HADOOP-2157 Make Scanners implement Iterable + HADOOP-2176 Htable.deleteAll documentation is ambiguous + HADOOP-2139 (phase 1) Increase parallelism in region servers. + HADOOP-2267 [Hbase Shell] Change the prompt's title from 'hbase' to 'hql'. + (Edward Yoon via Stack) + HADOOP-2139 (phase 2) Make region server more event driven + HADOOP-2289 Useless efforts of looking for the non-existant table in select + command. + (Edward Yoon via Stack) + HADOOP-2257 Show a total of all requests and regions on the web ui + (Paul Saab via Stack) + HADOOP-2261 HTable.abort no longer throws exception if there is no active update. + HADOOP-2287 Make hbase unit tests take less time to complete. + HADOOP-2262 Retry n times instead of n**2 times. + HADOOP-1608 Relational Algrebra Operators + (Edward Yoon via Stack) + HADOOP-2198 HTable should have method to return table metadata + HADOOP-2296 hbase shell: phantom columns show up from select command + HADOOP-2297 System.exit() Handling in hbase shell jar command + (Edward Yoon via Stack) + HADOOP-2224 Add HTable.getRow(ROW, ts) + (Bryan Duxbury via Stack) + HADOOP-2339 Delete command with no WHERE clause + (Edward Yoon via Stack) + HADOOP-2299 Support inclusive scans (Bryan Duxbury via Stack) + HADOOP-2333 Client side retries happen at the wrong level + HADOOP-2357 Compaction cleanup; less deleting + prevent possible file leaks + HADOOP-2392 TestRegionServerExit has new failure mode since HADOOP-2338 + HADOOP-2370 Allow column families with an unlimited number of versions + (Edward Yoon via Stack) + HADOOP-2047 Add an '--master=X' and '--html' command-line parameters to shell + (Edward Yoon via Stack) + HADOOP-2351 If select command returns no result, it doesn't need to show the + header information (Edward Yoon via Stack) + HADOOP-2285 Add being able to shutdown regionservers (Dennis Kubes via Stack) + HADOOP-2458 HStoreFile.writeSplitInfo should just call + HStoreFile.Reference.write + HADOOP-2471 Add reading/writing MapFile to PerformanceEvaluation suite + HADOOP-2522 Separate MapFile benchmark from PerformanceEvaluation + (Tom White via Stack) + HADOOP-2502 Insert/Select timestamp, Timestamp data type in HQL + (Edward Yoon via Stack) + HADOOP-2450 Show version (and svn revision) in hbase web ui + HADOOP-2472 Range selection using filter (Edward Yoon via Stack) + HADOOP-2548 Make TableMap and TableReduce generic + (Frederik Hedberg via Stack) + HADOOP-2557 Shell count function (Edward Yoon via Stack) + HADOOP-2589 Change an classes/package name from Shell to hql + (Edward Yoon via Stack) + HADOOP-2545 hbase rest server should be started with hbase-daemon.sh + HADOOP-2525 Same 2 lines repeated 11 million times in HMaster log upon + HMaster shutdown + HADOOP-2616 hbase not spliting when the total size of region reaches max + region size * 1.5 + HADOOP-2643 Make migration tool smarter. + +Release 0.15.1 +Branch 0.15 + + INCOMPATIBLE CHANGES + HADOOP-1931 Hbase scripts take --ARG=ARG_VALUE when should be like hadoop + and do ---ARG ARG_VALUE + + NEW FEATURES + HADOOP-1768 FS command using Hadoop FsShell operations + (Edward Yoon via Stack) + HADOOP-1784 Delete: Fix scanners and gets so they work properly in presence + of deletes. Added a deleteAll to remove all cells equal to or + older than passed timestamp. Fixed compaction so deleted cells + do not make it out into compacted output. Ensure also that + versions > column max are dropped compacting. + HADOOP-1720 Addition of HQL (Hbase Query Language) support in Hbase Shell. + The old shell syntax has been replaced by HQL, a small SQL-like + set of operators, for creating, altering, dropping, inserting, + deleting, and selecting, etc., data in hbase. + (Inchul Song and Edward Yoon via Stack) + HADOOP-1913 Build a Lucene index on an HBase table + (Ning Li via Stack) + HADOOP-1957 Web UI with report on cluster state and basic browsing of tables + + OPTIMIZATIONS + + BUG FIXES + HADOOP-1527 Region server won't start because logdir exists + HADOOP-1723 If master asks region server to shut down, by-pass return of + shutdown message + HADOOP-1729 Recent renaming or META tables breaks hbase shell + HADOOP-1730 unexpected null value causes META scanner to exit (silently) + HADOOP-1747 On a cluster, on restart, regions multiply assigned + HADOOP-1776 Fix for sporadic compaction failures closing and moving + compaction result + HADOOP-1780 Regions are still being doubly assigned + HADOOP-1797 Fix NPEs in MetaScanner constructor + HADOOP-1799 Incorrect classpath in binary version of Hadoop + HADOOP-1805 Region server hang on exit + HADOOP-1785 TableInputFormat.TableRecordReader.next has a bug + (Ning Li via Stack) + HADOOP-1800 output should default utf8 encoding + HADOOP-1801 When hdfs is yanked out from under hbase, hbase should go down gracefully + HADOOP-1813 OOME makes zombie of region server + HADOOP-1814 TestCleanRegionServerExit fails too often on Hudson + HADOOP-1820 Regionserver creates hlogs without bound + (reverted 2007/09/25) (Fixed 2007/09/30) + HADOOP-1821 Replace all String.getBytes() with String.getBytes("UTF-8") + HADOOP-1832 listTables() returns duplicate tables + HADOOP-1834 Scanners ignore timestamp passed on creation + HADOOP-1847 Many HBase tests do not fail well. + HADOOP-1847 Many HBase tests do not fail well. (phase 2) + HADOOP-1870 Once file system failure has been detected, don't check it again + and get on with shutting down the hbase cluster. + HADOOP-1888 NullPointerException in HMemcacheScanner (reprise) + HADOOP-1903 Possible data loss if Exception happens between snapshot and + flush to disk. + HADOOP-1920 Wrapper scripts broken when hadoop in one location and hbase in + another + HADOOP-1923, HADOOP-1924 a) tests fail sporadically because set up and tear + down is inconsistent b) TestDFSAbort failed in nightly #242 + HADOOP-1929 Add hbase-default.xml to hbase jar + HADOOP-1941 StopRowFilter throws NPE when passed null row + HADOOP-1966 Make HBase unit tests more reliable in the Hudson environment. + HADOOP-1975 HBase tests failing with java.lang.NumberFormatException + HADOOP-1990 Regression test instability affects nightly and patch builds + HADOOP-1996 TestHStoreFile fails on windows if run multiple times + HADOOP-1937 When the master times out a region server's lease, it is too + aggressive in reclaiming the server's log. + HADOOP-2004 webapp hql formatting bugs + HADOOP_2011 Make hbase daemon scripts take args in same order as hadoop + daemon scripts + HADOOP-2017 TestRegionServerAbort failure in patch build #903 and + nightly #266 + HADOOP-2029 TestLogRolling fails too often in patch and nightlies + HADOOP-2038 TestCleanRegionExit failed in patch build #927 + + IMPROVEMENTS + HADOOP-1737 Make HColumnDescriptor data publically members settable + HADOOP-1746 Clean up findbugs warnings + HADOOP-1757 Bloomfilters: single argument constructor, use enum for bloom + filter types + HADOOP-1760 Use new MapWritable and SortedMapWritable classes from + org.apache.hadoop.io + HADOOP-1793 (Phase 1) Remove TestHClient (Phase2) remove HClient. + HADOOP-1794 Remove deprecated APIs + HADOOP-1802 Startup scripts should wait until hdfs as cleared 'safe mode' + HADOOP-1833 bin/stop_hbase.sh returns before it completes + (Izaak Rubin via Stack) + HADOOP-1835 Updated Documentation for HBase setup/installation + (Izaak Rubin via Stack) + HADOOP-1868 Make default configuration more responsive + HADOOP-1884 Remove useless debugging log messages from hbase.mapred + HADOOP-1856 Add Jar command to hbase shell using Hadoop RunJar util + (Edward Yoon via Stack) + HADOOP-1928 Have master pass the regionserver the filesystem to use + HADOOP-1789 Output formatting + HADOOP-1960 If a region server cannot talk to the master before its lease + times out, it should shut itself down + HADOOP-2035 Add logo to webapps + + +Below are the list of changes before 2007-08-18 + + 1. HADOOP-1384. HBase omnibus patch. (jimk, Vuk Ercegovac, and Michael Stack) + 2. HADOOP-1402. Fix javadoc warnings in hbase contrib. (Michael Stack) + 3. HADOOP-1404. HBase command-line shutdown failing (Michael Stack) + 4. HADOOP-1397. Replace custom hbase locking with + java.util.concurrent.locks.ReentrantLock (Michael Stack) + 5. HADOOP-1403. HBase reliability - make master and region server more fault + tolerant. + 6. HADOOP-1418. HBase miscellaneous: unit test for HClient, client to do + 'Performance Evaluation', etc. + 7. HADOOP-1420, HADOOP-1423. Findbugs changes, remove reference to removed + class HLocking. + 8. HADOOP-1424. TestHBaseCluster fails with IllegalMonitorStateException. Fix + regression introduced by HADOOP-1397. + 9. HADOOP-1426. Make hbase scripts executable + add test classes to CLASSPATH. + 10. HADOOP-1430. HBase shutdown leaves regionservers up. + 11. HADOOP-1392. Part1: includes create/delete table; enable/disable table; + add/remove column. + 12. HADOOP-1392. Part2: includes table compaction by merging adjacent regions + that have shrunk in size. + 13. HADOOP-1445 Support updates across region splits and compactions + 14. HADOOP-1460 On shutdown IOException with complaint 'Cannot cancel lease + that is not held' + 15. HADOOP-1421 Failover detection, split log files. + For the files modified, also clean up javadoc, class, field and method + visibility (HADOOP-1466) + 16. HADOOP-1479 Fix NPE in HStore#get if store file only has keys < passed key. + 17. HADOOP-1476 Distributed version of 'Performance Evaluation' script + 18. HADOOP-1469 Asychronous table creation + 19. HADOOP-1415 Integrate BSD licensed bloom filter implementation. + 20. HADOOP-1465 Add cluster stop/start scripts for hbase + 21. HADOOP-1415 Provide configurable per-column bloom filters - part 2. + 22. HADOOP-1498. Replace boxed types with primitives in many places. + 23. HADOOP-1509. Made methods/inner classes in HRegionServer and HClient protected + instead of private for easier extension. Also made HRegion and HRegionInfo public too. + Added an hbase-default.xml property for specifying what HRegionInterface extension to use + for proxy server connection. (James Kennedy via Jim Kellerman) + 24. HADOOP-1534. [hbase] Memcache scanner fails if start key not present + 25. HADOOP-1537. Catch exceptions in testCleanRegionServerExit so we can see + what is failing. + 26. HADOOP-1543 [hbase] Add HClient.tableExists + 27. HADOOP-1519 [hbase] map/reduce interface for HBase. (Vuk Ercegovac and + Jim Kellerman) + 28. HADOOP-1523 Hung region server waiting on write locks + 29. HADOOP-1560 NPE in MiniHBaseCluster on Windows + 30. HADOOP-1531 Add RowFilter to HRegion.HScanner + Adds a row filtering interface and two implemenentations: A page scanner, + and a regex row/column-data matcher. (James Kennedy via Stack) + 31. HADOOP-1566 Key-making utility + 32. HADOOP-1415 Provide configurable per-column bloom filters. + HADOOP-1466 Clean up visibility and javadoc issues in HBase. + 33. HADOOP-1538 Provide capability for client specified time stamps in HBase + HADOOP-1466 Clean up visibility and javadoc issues in HBase. + 34. HADOOP-1589 Exception handling in HBase is broken over client server connections + 35. HADOOP-1375 a simple parser for hbase (Edward Yoon via Stack) + 36. HADOOP-1600 Update license in HBase code + 37. HADOOP-1589 Exception handling in HBase is broken over client server + 38. HADOOP-1574 Concurrent creates of a table named 'X' all succeed + 39. HADOOP-1581 Un-openable tablename bug + 40. HADOOP-1607 [shell] Clear screen command (Edward Yoon via Stack) + 41. HADOOP-1614 [hbase] HClient does not protect itself from simultaneous updates + 42. HADOOP-1468 Add HBase batch update to reduce RPC overhead + 43. HADOOP-1616 Sporadic TestTable failures + 44. HADOOP-1615 Replacing thread notification-based queue with + java.util.concurrent.BlockingQueue in HMaster, HRegionServer + 45. HADOOP-1606 Updated implementation of RowFilterSet, RowFilterInterface + (Izaak Rubin via Stack) + 46. HADOOP-1579 Add new WhileMatchRowFilter and StopRowFilter filters + (Izaak Rubin via Stack) + 47. HADOOP-1637 Fix to HScanner to Support Filters, Add Filter Tests to + TestScanner2 (Izaak Rubin via Stack) + 48. HADOOP-1516 HClient fails to readjust when ROOT or META redeployed on new + region server + 49. HADOOP-1646 RegionServer OOME's under sustained, substantial loading by + 10 concurrent clients + 50. HADOOP-1468 Add HBase batch update to reduce RPC overhead (restrict batches + to a single row at a time) + 51. HADOOP-1528 HClient for multiple tables (phase 1) (James Kennedy & JimK) + 52. HADOOP-1528 HClient for multiple tables (phase 2) all HBase client side code + (except TestHClient and HBaseShell) have been converted to use the new client + side objects (HTable/HBaseAdmin/HConnection) instead of HClient. + 53. HADOOP-1528 HClient for multiple tables - expose close table function + 54. HADOOP-1466 Clean up warnings, visibility and javadoc issues in HBase. + 55. HADOOP-1662 Make region splits faster + 56. HADOOP-1678 On region split, master should designate which host should + serve daughter splits. Phase 1: Master balances load for new regions and + when a region server fails. + 57. HADOOP-1678 On region split, master should designate which host should + serve daughter splits. Phase 2: Master assigns children of split region + instead of HRegionServer serving both children. + 58. HADOOP-1710 All updates should be batch updates + 59. HADOOP-1711 HTable API should use interfaces instead of concrete classes as + method parameters and return values + 60. HADOOP-1644 Compactions should not block updates + 60. HADOOP-1672 HBase Shell should use new client classes + (Edward Yoon via Stack). + 61. HADOOP-1709 Make HRegionInterface more like that of HTable + HADOOP-1725 Client find of table regions should not include offlined, split parents += diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 0000000..337c93b --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,20 @@ +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +In addition, this product includes software developed by: + +Jamon (http://www.jamon.org/) is a text template engine for Java used by our +UI. It uses the Mozilla Public License (http://www.mozilla.org/MPL/) +See the tail of http://www.jamon.org/About.html + +JUnit (http://www.junit.org/) included under the Common Public License v1.0. See +the full text here: http://junit.sourceforge.net/cpl-v10.html + +JRuby (http://jruby.org) is tri-licensed. We include it under terms of the +Common Public License v1.0. + +JRuby itself includes libraries variously licensed. See its COPYING document +for details: https://github.com/jruby/jruby/blob/master/COPYING + +The JRuby community went out of their way to make JRuby compatible with Apache +projects: See https://issues.apache.org/jira/browse/HBASE-3374) diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..7cc60da --- /dev/null +++ b/README.txt @@ -0,0 +1,30 @@ +Apache HBase [1] is an open-source, distributed, versioned, column-oriented +store modeled after Google' Bigtable: A Distributed Storage System for +Structured Data by Chang et al.[2] Just as Bigtable leverages the distributed +data storage provided by the Google File System, HBase provides Bigtable-like +capabilities on top of Apache Hadoop [3]. + +To get started using HBase, the full documentation for this release can be +found under the doc/ directory that accompanies this README. Using a browser, +open the docs/index.html to view the project home page (or browse to [1]). +The hbase 'book' at docs/book.html has a 'quick start' section and is where you +should being your exploration of the hbase project. + +The latest HBase can be downloaded from an Apache Mirror [4]. + +The source code can be found at [5] + +The HBase issue tracker is at [6] + +Apache HBase is made available under the Apache License, version 2.0 [7] + +The HBase mailing lists and archives are listed here [8]. + +1. http://hbase.apache.org +2. http://labs.google.com/papers/bigtable.html +3. http://hadoop.apache.org +4. http://www.apache.org/dyn/closer.cgi/hbase/ +5. http://hbase.apache.org/docs/current/source-repository.html +6. http://hbase.apache.org/docs/current/issue-tracking.html +7. http://hbase.apache.org/docs/current/license.html +8. http://hbase.apache.org/docs/current/mail-lists.html diff --git a/bin/get-active-master.rb b/bin/get-active-master.rb new file mode 100644 index 0000000..8887a45 --- /dev/null +++ b/bin/get-active-master.rb @@ -0,0 +1,45 @@ +#!/usr/bin/env hbase-jruby +# Copyright 2011 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Prints the hostname of the machine running the active master. + +include Java +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.ServerName +import org.apache.hadoop.hbase.zookeeper.ZKUtil +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher + +# disable debug/info logging on this script for clarity +log_level = org.apache.log4j.Level::ERROR +org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level) +org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level) + +config = HBaseConfiguration.create + +zk = ZooKeeperWatcher.new(config, 'get-active-master', nil) +begin + master_address = ZKUtil.getData(zk, zk.masterAddressZNode) + if master_address + puts ServerName.parseVersionedServerName(master_address).getHostname() + else + puts 'Master not running' + end +ensure + zk.close() +end + diff --git a/bin/graceful_stop.sh b/bin/graceful_stop.sh new file mode 100644 index 0000000..9a62440 --- /dev/null +++ b/bin/graceful_stop.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2011 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Move regions off a server then stop it. Optionally restart and reload. +# Turn off the balancer before running this script. +function usage { + echo "Usage: graceful_stop.sh [--config ] [--restart [--reload]] [--thrift] [--rest] " + echo " thrift If we should stop/start thrift before/after the hbase stop/start" + echo " rest If we should stop/start rest before/after the hbase stop/start" + echo " restart If we should restart after graceful stop" + echo " reload Move offloaded regions back on to the restarted server" + echo " debug Print helpful debug information" + echo " hostname Hostname of server we are to stop" + exit 1 +} + +if [ $# -lt 1 ]; then + usage +fi + +bin=`dirname "$0"` +bin=`cd "$bin">/dev/null; pwd` +# This will set HBASE_HOME, etc. +. "$bin"/hbase-config.sh +# Get arguments +restart= +reload= +debug= +thrift= +rest= +while [ $# -gt 0 ] +do + case "$1" in + --thrift) thrift=true; shift;; + --rest) rest=true; shift;; + --restart) restart=true; shift;; + --reload) reload=true; shift;; + --debug) debug="--debug"; shift;; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# "$@" contains the rest. Must be at least the hostname left. +if [ $# -lt 1 ]; then + usage +fi + +hostname=$1 +filename="/tmp/$hostname" +# Run the region mover script. +echo "Disabling balancer! (if required)" +HBASE_BALANCER_STATE=`echo 'balance_switch false' | "$bin"/hbase --config ${HBASE_CONF_DIR} shell | tail -3 | head -1` +echo "Previous balancer state was $HBASE_BALANCER_STATE" +echo "Unloading $hostname region(s)" +HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug unload $hostname +echo "Unloaded $hostname region(s)" +# Stop the server. Have to put hostname into its own little file for hbase-daemons.sh +hosts="/tmp/$(basename $0).$$.tmp" +echo $hostname >> $hosts +if [ "$thrift" != "" ]; then + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop thrift +fi +if [ "$rest" != "" ]; then + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop rest +fi +"$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop regionserver +if [ "$restart" != "" ]; then + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start regionserver + if [ "$thrift" != "" ]; then + # -b 0.0.0.0 says listen on all interfaces rather than just default. + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start thrift -b 0.0.0.0 + fi + if [ "$rest" != "" ]; then + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start rest + fi + if [ "$reload" != "" ]; then + echo "Reloading $hostname region(s)" + HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug load $hostname + echo "Reloaded $hostname region(s)" + fi +fi + +if [ $HBASE_BALANCER_STATE != "false" ]; then + echo "Restoring balancer state to" $HBASE_BALANCER_STATE + echo "balance_switch $HBASE_BALANCER_STATE" | "$bin"/hbase --config ${HBASE_CONF_DIR} shell &> /dev/null +fi + +# Cleanup tmp files. +trap "rm -f "/tmp/$(basename $0).*.tmp" &> /dev/null" EXIT diff --git a/bin/hbase b/bin/hbase new file mode 100644 index 0000000..15e109b --- /dev/null +++ b/bin/hbase @@ -0,0 +1,356 @@ +#! /usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# The hbase command script. Based on the hadoop command script putting +# in hbase classes, libs and configurations ahead of hadoop's. +# +# TODO: Narrow the amount of duplicated code. +# +# Environment Variables: +# +# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. +# +# HBASE_CLASSPATH Extra Java CLASSPATH entries. +# +# HBASE_HEAPSIZE The maximum amount of heap to use, in MB. +# Default is 1000. +# +# HBASE_LIBRARY_PATH HBase additions to JAVA_LIBRARY_PATH for adding +# native libaries. +# +# HBASE_OPTS Extra Java runtime options. +# +# HBASE_CONF_DIR Alternate conf dir. Default is ${HBASE_HOME}/conf. +# +# HBASE_ROOT_LOGGER The root appender. Default is INFO,console +# +# MAVEN_HOME Where mvn is installed. +# +# JRUBY_HOME JRuby path: $JRUBY_HOME/lib/jruby.jar should exist. +# Defaults to the jar packaged with HBase. +# +# JRUBY_OPTS Extra options (eg '--1.9') passed to the hbase shell. +# Empty by default. +# +bin=`dirname "$0"` +bin=`cd "$bin">/dev/null; pwd` + +# This will set HBASE_HOME, etc. +. "$bin"/hbase-config.sh + +cygwin=false +case "`uname`" in +CYGWIN*) cygwin=true;; +esac + +# Detect if we are in hbase sources dir +in_dev_env=false +if [ -d "${HBASE_HOME}/target" ]; then + in_dev_env=true +fi + +# if no args specified, show usage +if [ $# = 0 ]; then + echo "Usage: hbase " + echo "where an option from one of these categories:" + echo "" + echo "DBA TOOLS" + echo " shell run the HBase shell" + echo " hbck run the hbase 'fsck' tool" + echo " hlog write-ahead-log analyzer" + echo " hfile store file analyzer" + echo " zkcli run the ZooKeeper shell" + echo "" + echo "PROCESS MANAGEMENT" + echo " master run an HBase HMaster node" + echo " regionserver run an HBase HRegionServer node" + echo " zookeeper run a Zookeeper server" + echo " rest run an HBase REST server" + echo " thrift run the HBase Thrift server" + echo " thrift2 run the HBase Thrift2 server" + echo " avro run an HBase Avro server" + echo "" + echo "PACKAGE MANAGEMENT" + echo " classpath dump hbase CLASSPATH" + echo " version print the version" + echo "" + echo " or" + echo " CLASSNAME run the class named CLASSNAME" + echo "Most commands print help when invoked w/o parameters." + exit 1 +fi + +# get arguments +COMMAND=$1 +shift + +JAVA=$JAVA_HOME/bin/java +JAVA_HEAP_MAX=-Xmx1000m + +MVN="mvn" +if [ "$MAVEN_HOME" != "" ]; then + MVN=${MAVEN_HOME}/bin/mvn +fi + +# override default settings for this command, if applicable +if [ -f "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" ]; then + . "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" +fi + +# check envvars which might override default args +if [ "$HBASE_HEAPSIZE" != "" ]; then + #echo "run with heapsize $HBASE_HEAPSIZE" + JAVA_HEAP_MAX="-Xmx""$HBASE_HEAPSIZE""m" + #echo $JAVA_HEAP_MAX +fi + +# so that filenames w/ spaces are handled correctly in loops below +IFS= + +# CLASSPATH initially contains $HBASE_CONF_DIR +CLASSPATH="${HBASE_CONF_DIR}" +CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar + +add_maven_deps_to_classpath() { + # Need to generate classpath from maven pom. This is costly so generate it + # and cache it. Save the file into our target dir so a mvn clean will get + # clean it up and force us create a new one. + f="${HBASE_HOME}/target/cached_classpath.txt" + if [ ! -f "${f}" ] + then + ${MVN} -f "${HBASE_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + fi + CLASSPATH=${CLASSPATH}:`cat "${f}"` +} + +add_maven_main_classes_to_classpath() { + if [ -d "$HBASE_HOME/target/classes" ]; then + CLASSPATH=${CLASSPATH}:$HBASE_HOME/target/classes + fi +} + +add_maven_test_classes_to_classpath() { + # For developers, add hbase classes to CLASSPATH + f="$HBASE_HOME/target/test-classes" + if [ -d "${f}" ]; then + CLASSPATH=${CLASSPATH}:${f} + fi +} + +# Add maven target directory +if $in_dev_env; then + add_maven_deps_to_classpath + add_maven_main_classes_to_classpath + add_maven_test_classes_to_classpath +fi + +# For releases, add hbase & webapps to CLASSPATH +# Webapps must come first else it messes up Jetty +if [ -d "$HBASE_HOME/hbase-webapps" ]; then + CLASSPATH=${CLASSPATH}:$HBASE_HOME +fi +if [ -d "$HBASE_HOME/target/hbase-webapps" ]; then + CLASSPATH="${CLASSPATH}:${HBASE_HOME}/target" +fi +for f in $HBASE_HOME/hbase*.jar; do + if [[ $f = *sources.jar ]] + then + : # Skip sources.jar + elif [ -f $f ] + then + CLASSPATH=${CLASSPATH}:$f; + fi +done + +# Add libs to CLASSPATH +for f in $HBASE_HOME/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +# Add user-specified CLASSPATH last +if [ "$HBASE_CLASSPATH" != "" ]; then + CLASSPATH=${CLASSPATH}:${HBASE_CLASSPATH} +fi + +# default log directory & file +if [ "$HBASE_LOG_DIR" = "" ]; then + HBASE_LOG_DIR="$HBASE_HOME/logs" +fi +if [ "$HBASE_LOGFILE" = "" ]; then + HBASE_LOGFILE='hbase.log' +fi + +# cygwin path translation +if $cygwin; then + CLASSPATH=`cygpath -p -w "$CLASSPATH"` + HBASE_HOME=`cygpath -d "$HBASE_HOME"` + HBASE_LOG_DIR=`cygpath -d "$HBASE_LOG_DIR"` +fi + +function append_path() { + if [ -z "$1" ]; then + echo $2 + else + echo $1:$2 + fi +} + +JAVA_PLATFORM="" + +#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH +HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) +if [ -f ${HADOOP_IN_PATH} ]; then + HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ + org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) + if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then + JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") + fi + CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) +fi + +if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then + if [ -z $JAVA_PLATFORM ]; then + JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` + fi + if [ -d "$HBASE_HOME/build/native" ]; then + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib) + fi + + if [ -d "${HBASE_HOME}/lib/native" ]; then + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM}) + fi +fi + +# cygwin path translation +if $cygwin; then + JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` +fi + +# restore ordinary behaviour +unset IFS + +#Set the right GC options based on the what we are running +declare -a server_cmds=("master" "regionserver" "thrift" "thrift2" "rest" "avro" "zookeeper") +for cmd in ${server_cmds[@]}; do + if [[ $cmd == $COMMAND ]]; then + server=true + break + fi +done + +if [[ $server ]]; then + HBASE_OPTS="$HBASE_OPTS $SERVER_GC_OPTS" +else + HBASE_OPTS="$HBASE_OPTS $CLIENT_GC_OPTS" +fi + +# figure out which class to run +if [ "$COMMAND" = "shell" ] ; then + # eg export JRUBY_HOME=/usr/local/share/jruby + if [ "$JRUBY_HOME" != "" ] ; then + CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH" + HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib" + fi + CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" +elif [ "$COMMAND" = "hbck" ] ; then + CLASS='org.apache.hadoop.hbase.util.HBaseFsck' +elif [ "$COMMAND" = "hlog" ] ; then + CLASS='org.apache.hadoop.hbase.regionserver.wal.HLogPrettyPrinter' +elif [ "$COMMAND" = "hfile" ] ; then + CLASS='org.apache.hadoop.hbase.io.hfile.HFile' +elif [ "$COMMAND" = "zkcli" ] ; then + # ZooKeeperMainServerArg returns '-server HOST:PORT' or empty string. + SERVER_ARG=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServerArg` + CLASS="org.apache.zookeeper.ZooKeeperMain ${SERVER_ARG}" + +elif [ "$COMMAND" = "master" ] ; then + CLASS='org.apache.hadoop.hbase.master.HMaster' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_MASTER_OPTS" + fi +elif [ "$COMMAND" = "regionserver" ] ; then + CLASS='org.apache.hadoop.hbase.regionserver.HRegionServer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_REGIONSERVER_OPTS" + fi +elif [ "$COMMAND" = "thrift" ] ; then + CLASS='org.apache.hadoop.hbase.thrift.ThriftServer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" + fi +elif [ "$COMMAND" = "thrift2" ] ; then + CLASS='org.apache.hadoop.hbase.thrift2.ThriftServer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" + fi +elif [ "$COMMAND" = "rest" ] ; then + CLASS='org.apache.hadoop.hbase.rest.Main' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_REST_OPTS" + fi +elif [ "$COMMAND" = "avro" ] ; then + CLASS='org.apache.hadoop.hbase.avro.AvroServer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_AVRO_OPTS" + fi +elif [ "$COMMAND" = "zookeeper" ] ; then + CLASS='org.apache.hadoop.hbase.zookeeper.HQuorumPeer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_ZOOKEEPER_OPTS" + fi + +elif [ "$COMMAND" = "classpath" ] ; then + echo $CLASSPATH + exit 0 +elif [ "$COMMAND" = "version" ] ; then + CLASS='org.apache.hadoop.hbase.util.VersionInfo' +else + CLASS=$COMMAND +fi + +# Have JVM dump heap if we run out of memory. Files will be 'launch directory' +# and are named like the following: java_pid21612.hprof. Apparently it doesn't +# 'cost' to have this flag enabled. Its a 1.6 flag only. See: +# http://blogs.sun.com/alanb/entry/outofmemoryerror_looks_a_bit_better +HBASE_OPTS="$HBASE_OPTS -Dhbase.log.dir=$HBASE_LOG_DIR" +HBASE_OPTS="$HBASE_OPTS -Dhbase.log.file=$HBASE_LOGFILE" +HBASE_OPTS="$HBASE_OPTS -Dhbase.home.dir=$HBASE_HOME" +HBASE_OPTS="$HBASE_OPTS -Dhbase.id.str=$HBASE_IDENT_STRING" +HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger=${HBASE_ROOT_LOGGER:-INFO,console}" +if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + HBASE_OPTS="$HBASE_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH" +fi + +# Enable security logging on the master and regionserver only +if [ "$COMMAND" = "master" ] || [ "$COMMAND" = "regionserver" ]; then + HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,DRFAS}" +else + HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}" +fi + +# Exec unless HBASE_NOEXEC is set. +if [ "${HBASE_NOEXEC}" != "" ]; then + "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" +else + exec "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" +fi diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh new file mode 100644 index 0000000..966744a --- /dev/null +++ b/bin/hbase-config.sh @@ -0,0 +1,120 @@ +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# included in all the hbase scripts with source command +# should not be executable directly +# also should not be passed any arguments, since we need original $* +# Modelled after $HADOOP_HOME/bin/hadoop-env.sh. + +# resolve links - "${BASH_SOURCE-$0}" may be a softlink + +this="${BASH_SOURCE-$0}" +while [ -h "$this" ]; do + ls=`ls -ld "$this"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + this="$link" + else + this=`dirname "$this"`/"$link" + fi +done + +# convert relative path to absolute path +bin=`dirname "$this"` +script=`basename "$this"` +bin=`cd "$bin">/dev/null; pwd` +this="$bin/$script" + +# the root of the hbase installation +if [ -z "$HBASE_HOME" ]; then + export HBASE_HOME=`dirname "$this"`/.. +fi + +#check to see if the conf dir or hbase home are given as an optional arguments +while [ $# -gt 1 ] +do + if [ "--config" = "$1" ] + then + shift + confdir=$1 + shift + HBASE_CONF_DIR=$confdir + elif [ "--hosts" = "$1" ] + then + shift + hosts=$1 + shift + HBASE_REGIONSERVERS=$hosts + else + # Presume we are at end of options and break + break + fi +done + +# Allow alternate hbase conf dir location. +HBASE_CONF_DIR="${HBASE_CONF_DIR:-$HBASE_HOME/conf}" +# List of hbase regions servers. +HBASE_REGIONSERVERS="${HBASE_REGIONSERVERS:-$HBASE_CONF_DIR/regionservers}" +# List of hbase secondary masters. +HBASE_BACKUP_MASTERS="${HBASE_BACKUP_MASTERS:-$HBASE_CONF_DIR/backup-masters}" + +# Source the hbase-env.sh. Will have JAVA_HOME defined. +# HBASE-7817 - Source the hbase-env.sh only if it has not already been done. HBASE_ENV_INIT keeps track of it. +if [ -z "$HBASE_ENV_INIT" ] && [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then + . "${HBASE_CONF_DIR}/hbase-env.sh" + export HBASE_ENV_INIT="true" +fi + +# Newer versions of glibc use an arena memory allocator that causes virtual +# memory usage to explode. Tune the variable down to prevent vmem explosion. +export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4} + +if [ -z "$JAVA_HOME" ]; then + for candidate in \ + /usr/lib/jvm/java-6-sun \ + /usr/lib/jvm/java-1.6.0-sun-1.6.0.*/jre \ + /usr/lib/jvm/java-1.6.0-sun-1.6.0.* \ + /usr/lib/j2sdk1.6-sun \ + /usr/java/jdk1.6* \ + /usr/java/jre1.6* \ + /Library/Java/Home ; do + if [ -e $candidate/bin/java ]; then + export JAVA_HOME=$candidate + break + fi + done + # if we didn't set it + if [ -z "$JAVA_HOME" ]; then + cat 1>&2 < http://java.sun.com/javase/downloads/ < | +| | +| HBase requires Java 1.6 or later. | +| NOTE: This script will find Sun Java whether you install using the | +| binary or the RPM based installer. | ++======================================================================+ +EOF + exit 1 + fi +fi diff --git a/bin/hbase-daemon.sh b/bin/hbase-daemon.sh new file mode 100644 index 0000000..6f68090 --- /dev/null +++ b/bin/hbase-daemon.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Runs a Hadoop hbase command as a daemon. +# +# Environment Variables +# +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HBASE_LOG_DIR Where log files are stored. PWD by default. +# HBASE_PID_DIR The pid files are stored. /tmp by default. +# HBASE_IDENT_STRING A string representing this instance of hadoop. $USER by default +# HBASE_NICENESS The scheduling priority for daemons. Defaults to 0. +# +# Modelled after $HADOOP_HOME/bin/hadoop-daemon.sh + +usage="Usage: hbase-daemon.sh [--config ]\ + (start|stop|restart) \ + " + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# get arguments +startStop=$1 +shift + +command=$1 +shift + +hbase_rotate_log () +{ + log=$1; + num=5; + if [ -n "$2" ]; then + num=$2 + fi + if [ -f "$log" ]; then # rotate logs + while [ $num -gt 1 ]; do + prev=`expr $num - 1` + [ -f "$log.$prev" ] && mv -f "$log.$prev" "$log.$num" + num=$prev + done + mv -f "$log" "$log.$num"; + fi +} + +wait_until_done () +{ + p=$1 + cnt=${HBASE_SLAVE_TIMEOUT:-300} + origcnt=$cnt + while kill -0 $p > /dev/null 2>&1; do + if [ $cnt -gt 1 ]; then + cnt=`expr $cnt - 1` + sleep 1 + else + echo "Process did not complete after $origcnt seconds, killing." + kill -9 $p + exit 1 + fi + done + return 0 +} + +# get log directory +if [ "$HBASE_LOG_DIR" = "" ]; then + export HBASE_LOG_DIR="$HBASE_HOME/logs" +fi +mkdir -p "$HBASE_LOG_DIR" + +if [ "$HBASE_PID_DIR" = "" ]; then + HBASE_PID_DIR=/tmp +fi + +if [ "$HBASE_IDENT_STRING" = "" ]; then + export HBASE_IDENT_STRING="$USER" +fi + +# Some variables +# Work out java location so can print version into log. +if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME +fi +if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 +fi +JAVA=$JAVA_HOME/bin/java +export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME +export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log +export HBASE_ROOT_LOGGER="INFO,DRFA" +export HBASE_SECURITY_LOGGER="INFO,DRFAS" +logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out +loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc +loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" +pid=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.pid + +if [ -n "$SERVER_GC_OPTS" ]; then + export SERVER_GC_OPTS=${SERVER_GC_OPTS/"-Xloggc:"/"-Xloggc:${loggc}"} +fi +if [ -n "$CLIENT_GC_OPTS" ]; then + export CLIENT_GC_OPTS=${CLIENT_GC_OPTS/"-Xloggc:"/"-Xloggc:${loggc}"} +fi + +# Set default scheduling priority +if [ "$HBASE_NICENESS" = "" ]; then + export HBASE_NICENESS=0 +fi + +case $startStop in + + (start) + mkdir -p "$HBASE_PID_DIR" + if [ -f $pid ]; then + if kill -0 `cat $pid` > /dev/null 2>&1; then + echo $command running as process `cat $pid`. Stop it first. + exit 1 + fi + fi + + hbase_rotate_log $logout + hbase_rotate_log $loggc + echo starting $command, logging to $logout + # Add to the command log file vital stats on our environment. + echo "`date` Starting $command on `hostname`" >> $loglog + echo "`ulimit -a`" >> $loglog 2>&1 + nohup nice -n $HBASE_NICENESS "$HBASE_HOME"/bin/hbase \ + --config "${HBASE_CONF_DIR}" \ + $command "$@" $startStop > "$logout" 2>&1 < /dev/null & + echo $! > $pid + sleep 1; head "$logout" + ;; + + (stop) + if [ -f $pid ]; then + # kill -0 == see if the PID exists + if kill -0 `cat $pid` > /dev/null 2>&1; then + echo -n stopping $command + echo "`date` Terminating $command" >> $loglog + kill `cat $pid` > /dev/null 2>&1 + while kill -0 `cat $pid` > /dev/null 2>&1; do + echo -n "." + sleep 1; + done + rm $pid + echo + else + retval=$? + echo no $command to stop because kill -0 of pid `cat $pid` failed with status $retval + fi + else + echo no $command to stop because no pid file $pid + fi + ;; + + (restart) + thiscmd=$0 + args=$@ + # stop the command + $thiscmd --config "${HBASE_CONF_DIR}" stop $command $args & + wait_until_done $! + # wait a user-specified sleep period + sp=${HBASE_RESTART_SLEEP:-3} + if [ $sp -gt 0 ]; then + sleep $sp + fi + # start the command + $thiscmd --config "${HBASE_CONF_DIR}" start $command $args & + wait_until_done $! + ;; + + (*) + echo $usage + exit 1 + ;; + +esac diff --git a/bin/hbase-daemons.sh b/bin/hbase-daemons.sh new file mode 100644 index 0000000..843eaaa --- /dev/null +++ b/bin/hbase-daemons.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a hbase command on all slave hosts. +# Modelled after $HADOOP_HOME/bin/hadoop-daemons.sh + +usage="Usage: hbase-daemons.sh [--config ] \ + [--hosts regionserversfile] [start|stop] command args..." + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. $bin/hbase-config.sh + +remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@" +args="--hosts ${HBASE_REGIONSERVERS} --config ${HBASE_CONF_DIR} $remote_cmd" + +command=$2 +case $command in + (zookeeper) + exec "$bin/zookeepers.sh" $args + ;; + (master-backup) + exec "$bin/master-backup.sh" $args + ;; + (*) + exec "$bin/regionservers.sh" $args + ;; +esac + diff --git a/bin/hbase-jruby b/bin/hbase-jruby new file mode 100644 index 0000000..37bce46 --- /dev/null +++ b/bin/hbase-jruby @@ -0,0 +1,22 @@ +#!/bin/bash +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + + +`dirname $0`/hbase org.jruby.Main $* + diff --git a/bin/hirb.rb b/bin/hirb.rb new file mode 100644 index 0000000..32a51b3 --- /dev/null +++ b/bin/hirb.rb @@ -0,0 +1,188 @@ +# +# Copyright 2009 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# File passed to org.jruby.Main by bin/hbase. Pollutes jirb with hbase imports +# and hbase commands and then loads jirb. Outputs a banner that tells user +# where to find help, shell version, and loads up a custom hirb. + +# TODO: Add 'debug' support (client-side logs show in shell). Add it as +# command-line option and as command. +# TODO: Interrupt a table creation or a connection to a bad master. Currently +# has to time out. Below we've set down the retries for rpc and hbase but +# still can be annoying (And there seem to be times when we'll retry for +# ever regardless) +# TODO: Add support for listing and manipulating catalog tables, etc. +# TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes + +# Run the java magic include and import basic HBase types that will help ease +# hbase hacking. +include Java + +# Some goodies for hirb. Should these be left up to the user's discretion? +require 'irb/completion' + +# Add the $HBASE_HOME/lib/ruby OR $HBASE_HOME/src/main/ruby/lib directory +# to the ruby load path so I can load up my HBase ruby modules +if File.exists?(File.join(File.dirname(__FILE__), "..", "lib", "ruby", "hbase.rb")) + $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "lib", "ruby") +else + $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "src", "main", "ruby") +end + +# +# FIXME: Switch args processing to getopt +# +# See if there are args for this shell. If any, read and then strip from ARGV +# so they don't go through to irb. Output shell 'usage' if user types '--help' +cmdline_help = </dev/null && pwd` + +if [ $# -lt 2 ]; then + S=`basename "${BASH_SOURCE-$0}"` + echo "Usage: $S [start|stop] offset(s)" + echo "" + echo " e.g. $S start 1" + exit +fi + +# sanity check: make sure your master opts don't use ports [i.e. JMX/DBG] +export HBASE_MASTER_OPTS=" " + +run_master () { + DN=$2 + export HBASE_IDENT_STRING="$USER-$DN" + HBASE_MASTER_ARGS="\ + -D hbase.master.port=`expr 60000 + $DN` \ + -D hbase.master.info.port=`expr 60010 + $DN` \ + --backup" + "$bin"/hbase-daemon.sh $1 master $HBASE_MASTER_ARGS +} + +cmd=$1 +shift; + +for i in $* +do + run_master $cmd $i +done diff --git a/bin/local-regionservers.sh b/bin/local-regionservers.sh new file mode 100644 index 0000000..a4d5a1d --- /dev/null +++ b/bin/local-regionservers.sh @@ -0,0 +1,54 @@ +#!/bin/sh +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# This is used for starting multiple regionservers on the same machine. +# run it from hbase-dir/ just like 'bin/hbase' +# Supports up to 100 regionservers (limitation = overlapping ports) + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin" >/dev/null && pwd` + +if [ $# -lt 2 ]; then + S=`basename "${BASH_SOURCE-$0}"` + echo "Usage: $S [start|stop] offset(s)" + echo "" + echo " e.g. $S start 1 2" + exit +fi + +# sanity check: make sure your regionserver opts don't use ports [i.e. JMX/DBG] +export HBASE_REGIONSERVER_OPTS=" " + +run_regionserver () { + DN=$2 + export HBASE_IDENT_STRING="$USER-$DN" + HBASE_REGIONSERVER_ARGS="\ + -D hbase.regionserver.port=`expr 60200 + $DN` \ + -D hbase.regionserver.info.port=`expr 60300 + $DN`" + "$bin"/hbase-daemon.sh $1 regionserver $HBASE_REGIONSERVER_ARGS +} + +cmd=$1 +shift; + +for i in $* +do + run_regionserver $cmd $i +done diff --git a/bin/master-backup.sh b/bin/master-backup.sh new file mode 100644 index 0000000..114757c --- /dev/null +++ b/bin/master-backup.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2010 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all backup master hosts. +# +# Environment Variables +# +# HBASE_BACKUP_MASTERS File naming remote hosts. +# Default is ${HBASE_CONF_DIR}/backup-masters +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage="Usage: $0 [--config ] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# If the master backup file is specified in the command line, +# then it takes precedence over the definition in +# hbase-env.sh. Save it here. +HOSTLIST=$HBASE_BACKUP_MASTERS + +if [ "$HOSTLIST" = "" ]; then + if [ "$HBASE_BACKUP_MASTERS" = "" ]; then + export HOSTLIST="${HBASE_CONF_DIR}/backup-masters" + else + export HOSTLIST="${HBASE_BACKUP_MASTERS}" + fi +fi + + +args=${@// /\\ } +args=${args/master-backup/master} + +if [ -f $HOSTLIST ]; then + for hmaster in `cat "$HOSTLIST"`; do + ssh $HBASE_SSH_OPTS $hmaster $"$args --backup" \ + 2>&1 | sed "s/^/$hmaster: /" & + if [ "$HBASE_SLAVE_SLEEP" != "" ]; then + sleep $HBASE_SLAVE_SLEEP + fi + done +fi + +wait diff --git a/bin/region_mover.rb b/bin/region_mover.rb new file mode 100644 index 0000000..5c6bea0 --- /dev/null +++ b/bin/region_mover.rb @@ -0,0 +1,475 @@ +# Copyright 2011 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Moves regions. Will confirm region access in current location and will +# not move a new region until successful confirm of region loading in new +# location. Presumes balancer is disabled when we run (not harmful if its +# on but this script and balancer will end up fighting each other). +# Does not work for case of multiple regionservers all running on the +# one node. +require 'optparse' +include Java +import org.apache.hadoop.hbase.HConstants +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.client.HBaseAdmin +import org.apache.hadoop.hbase.client.Get +import org.apache.hadoop.hbase.client.Scan +import org.apache.hadoop.hbase.client.HTable +import org.apache.hadoop.hbase.client.HConnectionManager +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.HServerAddress +import org.apache.hadoop.hbase.util.Bytes +import org.apache.hadoop.hbase.util.Writables +import org.apache.hadoop.conf.Configuration +import org.apache.commons.logging.Log +import org.apache.commons.logging.LogFactory + +# Name of this script +NAME = "region_mover" + +# Get root table reference +def getRootTable(config) + # Keep meta reference in ruby global + if not $ROOT + $ROOT = HTable.new(config, HConstants::ROOT_TABLE_NAME) + end + return $ROOT +end + +# Get meta table reference +def getMetaTable(config) + # Keep meta reference in ruby global + if not $META + $META = HTable.new(config, HConstants::META_TABLE_NAME) + end + return $META +end + +# Get table instance. +# Maintains cache of table instances. +def getTable(config, name) + # Keep dictionary of tables in ruby global + if not $TABLES + $TABLES = {} + end + key = Bytes.toString(name) + if not $TABLES[key] + $TABLES[key] = HTable.new(config, name) + end + return $TABLES[key] +end + + +# Returns true if passed region is still on 'original' when we look at .META. +def isSameServer(admin, r, original) + server = getServerNameForRegion(admin, r) + return false unless server + return true unless original + return server == original +end + +class RubyAbortable + include org.apache.hadoop.hbase.Abortable + def abort(why, e) + puts "ABORTED! why=" + why + ", e=" + e.to_s + end +end + +# Get servername that is up in .META.; this is hostname + port + startcode comma-delimited. +# Can return nil +def getServerNameForRegion(admin, r) + if r.isRootRegion() + # Hack + tracker = org.apache.hadoop.hbase.zookeeper.RootRegionTracker.new(admin.getConnection().getZooKeeperWatcher(), RubyAbortable.new()) + tracker.start() + while not tracker.isLocationAvailable() + sleep 0.1 + end + # Make a fake servername by appending ',' + rootServer = tracker.getRootRegionLocation().toString() + "," + tracker.stop() + return rootServer + end + table = nil + if r.isMetaRegion() + table = getRootTable(admin.getConfiguration()) + else + table = getMetaTable(admin.getConfiguration()) + end + g = Get.new(r.getRegionName()) + g.addColumn(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) + g.addColumn(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) + result = table.get(g) + server = result.getValue(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) + startcode = result.getValue(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) + return nil unless server + return java.lang.String.new(Bytes.toString(server)).replaceFirst(":", ",") + "," + Bytes.toLong(startcode).to_s +end + +# Trys to scan a row from passed region +# Throws exception if can't +def isSuccessfulScan(admin, r) + scan = Scan.new(r.getStartKey()) + scan.setBatch(1) + scan.setCaching(1) + scan.setFilter(FirstKeyOnlyFilter.new()) + table = getTable(admin.getConfiguration(), r.getTableName()) + scanner = table.getScanner(scan) + begin + results = scanner.next() + # We might scan into next region, this might be an empty table. + # But if no exception, presume scanning is working. + ensure + scanner.close() + table.close() + end +end + +# Check region has moved successful and is indeed hosted on another server +# Wait until that is the case. +def move(admin, r, newServer, original) + # Now move it. Do it in a loop so can retry if fail. Have seen issue where + # we tried move region but failed and retry put it back on old location; + # retry in this case. + retries = admin.getConfiguration.getInt("hbase.move.retries.max", 5) + count = 0 + same = true + while count < retries and same + if count > 0 + $LOG.info("Retry " + count.to_s + " of maximum " + retries.to_s) + end + count = count + 1 + begin + admin.move(Bytes.toBytes(r.getEncodedName()), Bytes.toBytes(newServer)) + rescue java.lang.reflect.UndeclaredThrowableException => e + $LOG.info("Exception moving " + r.getEncodedName() + + "; split/moved? Continuing: " + e) + return + end + # Wait till its up on new server before moving on + maxWaitInSeconds = admin.getConfiguration.getInt("hbase.move.wait.max", 60) + maxWait = Time.now + maxWaitInSeconds + while Time.now < maxWait + same = isSameServer(admin, r, original) + break unless same + sleep 0.1 + end + end + raise RuntimeError, "Region stuck on #{original}, newserver=#{newServer}" if same + # Assert can Scan from new location. + isSuccessfulScan(admin, r) +end + +# Return the hostname portion of a servername (all up to first ',') +def getHostnamePortFromServerName(serverName) + parts = serverName.split(',') + return parts[0] + ":" + parts[1] +end + +# Return the hostname:port out of a servername (all up to first ',') +def getHostnameFromServerName(serverName) + return serverName.split(',')[0] +end + +# Return array of servernames where servername is hostname+port+startcode +# comma-delimited +def getServers(admin) + serverInfos = admin.getClusterStatus().getServerInfo() + servers = [] + for server in serverInfos + servers << server.getServerName() + end + return servers +end + +# Remove the servername whose hostname portion matches from the passed +# array of servers. Returns as side-effect the servername removed. +def stripServer(servers, hostname) + count = servers.length + servername = nil + for server in servers + if getHostnameFromServerName(server) == hostname + servername = servers.delete(server) + end + end + # Check server to exclude is actually present + raise RuntimeError, "Server %s not online" % hostname unless servers.length < count + return servername +end + +# Returns a new serverlist that excludes the servername whose hostname portion +# matches from the passed array of servers. +def stripExcludes(servers, excludefile) + excludes = readExcludes(excludefile) + servers = servers.find_all{|server| !excludes.contains(getHostnameFromServerName(server)) } + # return updated servers list + return servers +end + + +# Return servername that matches passed hostname +def getServerName(servers, hostname) + servername = nil + for server in servers + if getHostnameFromServerName(server) == hostname + servername = server + break + end + end + raise ArgumentError, "Server %s not online" % hostname unless servername + return servername +end + +# Create a logger and disable the DEBUG-level annoying client logging +def configureLogging(options) + apacheLogger = LogFactory.getLog(NAME) + # Configure log4j to not spew so much + unless (options[:debug]) + logger = org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase.client") + logger.setLevel(org.apache.log4j.Level::INFO) + end + return apacheLogger +end + +# Get configuration instance +def getConfiguration() + config = HBaseConfiguration.create() + # No prefetching on .META. + config.setInt("hbase.client.prefetch.limit", 1) + # Make a config that retries at short intervals many times + config.setInt("hbase.client.pause", 500) + config.setInt("hbase.client.retries.number", 100) + return config +end + +# Now get list of regions on targetServer +def getRegions(config, servername) + connection = HConnectionManager::getConnection(config) + parts = servername.split(',') + rs = connection.getHRegionConnection(parts[0], parts[1].to_i) + return rs.getOnlineRegions() +end + +def deleteFile(filename) + f = java.io.File.new(filename) + f.delete() if f.exists() +end + +# Write HRegionInfo to file +# Need to serialize in case non-printable characters. +# Format is count of regionnames followed by serialized regionnames. +def writeFile(filename, regions) + fos = java.io.FileOutputStream.new(filename) + dos = java.io.DataOutputStream.new(fos) + # Write out a count of region names + dos.writeInt(regions.size()) + # Write actual region names. + for r in regions + bytes = Writables.getBytes(r) + Bytes.writeByteArray(dos, bytes) + end + dos.close() +end + +# See writeFile above. +# Returns array of HRegionInfos +def readFile(filename) + f = java.io.File.new(filename) + return java.util.ArrayList.new() unless f.exists() + fis = java.io.FileInputStream.new(f) + dis = java.io.DataInputStream.new(fis) + # Read count of regions + count = dis.readInt() + regions = java.util.ArrayList.new(count) + index = 0 + while index < count + regions.add(Writables.getHRegionInfo(Bytes.readByteArray(dis))) + index = index + 1 + end + dis.close() + return regions +end + +# Move regions off the passed hostname +def unloadRegions(options, hostname) + # Get configuration + config = getConfiguration() + # Clean up any old files. + filename = getFilename(options, hostname) + deleteFile(filename) + # Get an admin instance + admin = HBaseAdmin.new(config) + servers = getServers(admin) + # Remove the server we are unloading from from list of servers. + # Side-effect is the servername that matches this hostname + servername = stripServer(servers, hostname) + + # Remove the servers in our exclude list from list of servers. + servers = stripExcludes(servers, options[:excludesFile]) + puts "Valid region move targets: ", servers + movedRegions = java.util.ArrayList.new() + while true + rs = getRegions(config, servername) + break if rs.length == 0 + count = 0 + $LOG.info("Moving " + rs.length.to_s + " region(s) from " + servername + + " during this cycle"); + for r in rs + # Get a random server to move the region to. + server = servers[rand(servers.length)] + $LOG.info("Moving region " + r.getEncodedName() + " (" + count.to_s + + " of " + rs.length.to_s + ") to server=" + server); + count = count + 1 + # Assert we can scan region in its current location + isSuccessfulScan(admin, r) + # Now move it. + move(admin, r, server, servername) + movedRegions.add(r) + end + end + if movedRegions.size() > 0 + # Write out file of regions moved + writeFile(filename, movedRegions) + $LOG.info("Wrote list of moved regions to " + filename) + end +end + +# Move regions to the passed hostname +def loadRegions(options, hostname) + # Get configuration + config = getConfiguration() + # Get an admin instance + admin = HBaseAdmin.new(config) + filename = getFilename(options, hostname) + regions = readFile(filename) + return if regions.isEmpty() + servername = nil + # Wait till server is up + maxWaitInSeconds = admin.getConfiguration.getInt("hbase.serverstart.wait.max", 180) + maxWait = Time.now + maxWaitInSeconds + while Time.now < maxWait + servers = getServers(admin) + begin + servername = getServerName(servers, hostname) + rescue ArgumentError => e + $LOG.info("hostname=" + hostname.to_s + " is not up yet, waiting"); + end + break if servername + sleep 0.5 + end + $LOG.info("Moving " + regions.size().to_s + " regions to " + servername) + count = 0 + for r in regions + exists = false + begin + isSuccessfulScan(admin, r) + exists = true + rescue org.apache.hadoop.hbase.NotServingRegionException => e + $LOG.info("Failed scan of " + e.message) + end + count = count + 1 + next unless exists + currentServer = getServerNameForRegion(admin, r) + if currentServer and currentServer == servername + $LOG.info("Region " + r.getRegionNameAsString() + " (" + count.to_s + + " of " + regions.length.to_s + ") already on target server=" + servername) + next + end + $LOG.info("Moving region " + r.getEncodedName() + " (" + count.to_s + + " of " + regions.length.to_s + ") to server=" + servername); + move(admin, r, servername, currentServer) + end +end + +# Returns an array of hosts to exclude as region move targets +def readExcludes(filename) + if filename == nil + return java.util.ArrayList.new() + end + if ! File.exist?(filename) + puts "Error: Unable to read host exclude file: ", filename + raise RuntimeError + end + + f = File.new(filename, "r") + # Read excluded hosts list + excludes = java.util.ArrayList.new() + while (line = f.gets) + line.strip! # do an inplace drop of pre and post whitespaces + excludes.add(line) unless line.empty? # exclude empty lines + end + puts "Excluding hosts as region move targets: ", excludes + f.close + + return excludes +end + +def getFilename(options, targetServer) + filename = options[:file] + if not filename + filename = "/tmp/" + targetServer + end + return filename +end + + +# Do command-line parsing +options = {} +optparse = OptionParser.new do |opts| + opts.banner = "Usage: #{NAME}.rb [options] load|unload " + opts.separator 'Load or unload regions by moving one at a time' + options[:file] = nil + opts.on('-f', '--filename=FILE', 'File to save regions list into unloading, or read from loading; default /tmp/') do |file| + options[:file] = file + end + opts.on('-h', '--help', 'Display usage information') do + puts opts + exit + end + options[:debug] = false + opts.on('-d', '--debug', 'Display extra debug logging') do + options[:debug] = true + end + opts.on('-x', '--excludefile=FILE', 'File with hosts-per-line to exclude as unload targets; default excludes only target host; useful for rack decommisioning.') do |file| + options[:excludesFile] = file + end +end +optparse.parse! + +# Check ARGVs +if ARGV.length < 2 + puts optparse + exit 1 +end +hostname = ARGV[1] +if not hostname + opts optparse + exit 2 +end +# Create a logger and save it to ruby global +$LOG = configureLogging(options) +case ARGV[0] + when 'load' + loadRegions(options, hostname) + when 'unload' + unloadRegions(options, hostname) + else + puts optparse + exit 3 +end diff --git a/bin/region_status.rb b/bin/region_status.rb new file mode 100644 index 0000000..5ccc8c2 --- /dev/null +++ b/bin/region_status.rb @@ -0,0 +1,149 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# View the current status of all regions on an HBase cluster. This is +# predominantly used to determined if all the regions in META have been +# onlined yet on startup. +# +# To use this script, run: +# +# ${HBASE_HOME}/bin/hbase org.jruby.Main region_status.rb [wait] [--table ] + + +require 'optparse' + +usage = 'Usage : ./hbase org.jruby.Main region_status.rb [wait]' + + '[--table ]\n' +OptionParser.new do |o| + o.banner = usage + o.on('-t', '--table TABLENAME', 'Only process TABLENAME') do |tablename| + $tablename = tablename + end + o.on('-h', '--help', 'Display help message') { puts o; exit } + o.parse! +end + +SHOULD_WAIT = ARGV[0] == 'wait' +if ARGV[0] and not SHOULD_WAIT + print usage + exit 1 +end + + +require 'java' + +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.HConstants +import org.apache.hadoop.hbase.MasterNotRunningException +import org.apache.hadoop.hbase.client.HBaseAdmin +import org.apache.hadoop.hbase.client.HTable +import org.apache.hadoop.hbase.client.Scan +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter +import org.apache.hadoop.hbase.util.Bytes +import org.apache.hadoop.hbase.util.Writables + +# disable debug logging on this script for clarity +log_level = org.apache.log4j.Level::ERROR +org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level) +org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level) + +config = HBaseConfiguration.create +config.set 'fs.default.name', config.get(HConstants::HBASE_DIR) + +# wait until the master is running +admin = nil +while true + begin + admin = HBaseAdmin.new config + break + rescue MasterNotRunningException => e + print 'Waiting for master to start...\n' + sleep 1 + end +end + +meta_count = 0 +server_count = 0 + +# scan META to see how many regions we should have +if $tablename.nil? + scan = Scan.new +else + tableNameMetaPrefix = $tablename + HConstants::META_ROW_DELIMITER.chr + scan = Scan.new( + (tableNameMetaPrefix + HConstants::META_ROW_DELIMITER.chr).to_java_bytes + ) +end +scan.cache_blocks = false +scan.caching = 10 +scan.setFilter(FirstKeyOnlyFilter.new) +INFO = 'info'.to_java_bytes +REGION_INFO = 'regioninfo'.to_java_bytes +scan.addColumn INFO, REGION_INFO +table = nil +iter = nil +while true + begin + table = HTable.new config, '.META.'.to_java_bytes + scanner = table.getScanner(scan) + iter = scanner.iterator + break + rescue IOException => ioe + print "Exception trying to scan META: #{ioe}" + sleep 1 + end +end +while iter.hasNext + result = iter.next + rowid = Bytes.toString(result.getRow()) + rowidStr = java.lang.String.new(rowid) + if not $tablename.nil? and not rowidStr.startsWith(tableNameMetaPrefix) + # Gone too far, break + break + end + region = Writables.getHRegionInfo result.getValue(INFO, REGION_INFO) + if not region.isOffline + # only include regions that should be online + meta_count += 1 + end +end +scanner.close +# If we're trying to see the status of all HBase tables, we need to include the +# -ROOT- & .META. tables, that are not included in our scan +if $tablename.nil? + meta_count += 2 +end + +# query the master to see how many regions are on region servers +if not $tablename.nil? + $tableq = HTable.new config, $tablename.to_java_bytes +end +while true + if $tablename.nil? + server_count = admin.getClusterStatus().getRegionsCount() + else + server_count = $tableq.getRegionsInfo().size() + end + print "Region Status: #{server_count} / #{meta_count}\n" + if SHOULD_WAIT and server_count < meta_count + #continue this loop until server & meta count match + sleep 10 + else + break + end +end + +exit server_count == meta_count ? 0 : 1 diff --git a/bin/regionservers.sh b/bin/regionservers.sh new file mode 100644 index 0000000..9759f2b --- /dev/null +++ b/bin/regionservers.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all regionserver hosts. +# +# Environment Variables +# +# HBASE_REGIONSERVERS File naming remote hosts. +# Default is ${HADOOP_CONF_DIR}/regionservers +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage="Usage: regionservers [--config ] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# If the regionservers file is specified in the command line, +# then it takes precedence over the definition in +# hbase-env.sh. Save it here. +HOSTLIST=$HBASE_REGIONSERVERS + +if [ "$HOSTLIST" = "" ]; then + if [ "$HBASE_REGIONSERVERS" = "" ]; then + export HOSTLIST="${HBASE_CONF_DIR}/regionservers" + else + export HOSTLIST="${HBASE_REGIONSERVERS}" + fi +fi + +for regionserver in `cat "$HOSTLIST"`; do + if ${HBASE_SLAVE_PARALLEL:-true}; then + ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ + 2>&1 | sed "s/^/$regionserver: /" & + else # run each command serially + ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ + 2>&1 | sed "s/^/$regionserver: /" + fi + if [ "$HBASE_SLAVE_SLEEP" != "" ]; then + sleep $HBASE_SLAVE_SLEEP + fi +done + +wait diff --git a/bin/replication/copy_tables_desc.rb b/bin/replication/copy_tables_desc.rb new file mode 100644 index 0000000..13cfaff --- /dev/null +++ b/bin/replication/copy_tables_desc.rb @@ -0,0 +1,74 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Script to recreate all tables from one cluster to another +# To see usage for this script, run: +# +# ${HBASE_HOME}/bin/hbase org.jruby.Main copy_tables_desc.rb +# + +include Java +import org.apache.commons.logging.LogFactory +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.HConstants +import org.apache.hadoop.hbase.EmptyWatcher +import org.apache.hadoop.hbase.client.HBaseAdmin +import org.apache.hadoop.hbase.HTableDescriptor +import org.apache.hadoop.conf.Configuration + +# Name of this script +NAME = "copy_tables_desc" + +# Print usage for this script +def usage + puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent slave_zookeeper.quorum.peers:clientport:znode_parent' % NAME + exit! +end + +if ARGV.size != 2 + usage +end + +LOG = LogFactory.getLog(NAME) + +parts1 = ARGV[0].split(":") + +parts2 = ARGV[1].split(":") + +c1 = HBaseConfiguration.create() +c1.set(HConstants::ZOOKEEPER_QUORUM, parts1[0]) +c1.set("hbase.zookeeper.property.clientPort", parts1[1]) +c1.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts1[2]) + +admin1 = HBaseAdmin.new(c1) + +c2 = HBaseConfiguration.create() +c2.set(HConstants::ZOOKEEPER_QUORUM, parts2[0]) +c2.set("hbase.zookeeper.property.clientPort", parts2[1]) +c2.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts2[2]) + +admin2 = HBaseAdmin.new(c2) + +for t in admin1.listTables() + admin2.createTable(t) +end + + +puts "All descriptions were copied" diff --git a/bin/rolling-restart.sh b/bin/rolling-restart.sh new file mode 100644 index 0000000..23f8d32 --- /dev/null +++ b/bin/rolling-restart.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all regionserver hosts. +# +# Environment Variables +# +# HBASE_REGIONSERVERS File naming remote hosts. +# Default is ${HADOOP_CONF_DIR}/regionservers +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HADOOP_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. +# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage="Usage: $0 [--config ] [--rs-only] [--master-only] [--graceful]" + +bin=`dirname "$0"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# start hbase daemons +errCode=$? +if [ $errCode -ne 0 ] +then + exit $errCode +fi + +function usage() { + echo $usage + exit 1 +} + + +RR_RS=1 +RR_MASTER=1 +RR_GRACEFUL=0 + +for x in "$@" ; do + case "$x" in + --rs-only|-r) + RR_RS=1 + RR_MASTER=0 + RR_GRACEFUL=0 + ;; + --master-only) + RR_RS=0 + RR_MASTER=1 + RR_GRACEFUL=0 + ;; + --graceful) + RR_RS=0 + RR_MASTER=0 + RR_GRACEFUL=1 + ;; + *) + echo Bad argument: $x + usage + exit 1 + ;; + esac +done + +# quick function to get a value from the HBase config file +# HBASE-6504 - only take the first line of the output in case verbose gc is on +distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` +if [ "$distMode" == 'false' ]; then + if [ $RR_RS -ne 1 ] || [ $RR_MASTER -ne 1 ]; then + echo Cant do selective rolling restart if not running distributed + exit 1 + fi + "$bin"/hbase-daemon.sh restart master +else + zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` + if [ "$zparent" == "null" ]; then zparent="/hbase"; fi + + if [ $RR_MASTER -eq 1 ]; then + # stop all masters before re-start to avoid races for master znode + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup + + # make sure the master znode has been deleted before continuing + zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master` + if [ "$zmaster" == "null" ]; then zmaster="master"; fi + zmaster=$zparent/$zmaster + echo -n "Waiting for Master ZNode ${zmaster} to expire" + while ! "$bin"/hbase zkcli stat $zmaster 2>&1 | grep "Node does not exist"; do + echo -n "." + sleep 1 + done + echo #force a newline + + # all masters are down, now restart + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_BACKUP_MASTERS}" start master-backup + + echo "Wait a minute for master to come up join cluster" + sleep 60 + + # Master joing cluster will start in cleaning out regions in transition. + # Wait until the master has cleaned out regions in transition before + # giving it a bunch of work to do; master is vulnerable during startup + zunassigned=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.unassigned` + if [ "$zunassigned" == "null" ]; then zunassigned="unassigned"; fi + zunassigned="$zparent/$zunassigned" + echo -n "Waiting for ${zunassigned} to empty" + while true ; do + unassigned=`$bin/hbase zkcli stat ${zunassigned} 2>&1 |grep -e 'numChildren = '|sed -e 's,numChildren = ,,'` + if test 0 -eq ${unassigned} + then + break + else + echo -n " ${unassigned}" + fi + sleep 1 + done + fi + + if [ $RR_RS -eq 1 ]; then + # unlike the masters, roll all regionservers one-at-a-time + export HBASE_SLAVE_PARALLEL=false + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_REGIONSERVERS}" restart regionserver + fi + + if [ $RR_GRACEFUL -eq 1 ]; then + # gracefully restart all online regionservers + zkrs=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.rs` + if [ "$zkrs" == "null" ]; then zkrs="rs"; fi + zkrs="$zparent/$zkrs" + online_regionservers=`$bin/hbase zkcli ls $zkrs 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"` + for rs in $online_regionservers + do + rs_parts=(${rs//,/ }) + hostname=${rs_parts[0]} + echo "Gracefully restarting: $hostname" + "$bin"/graceful_stop.sh --config "${HBASE_CONF_DIR}" --restart --reload --debug "$hostname" + sleep 1 + done + fi +fi diff --git a/bin/start-hbase.sh b/bin/start-hbase.sh new file mode 100644 index 0000000..6f3cd90 --- /dev/null +++ b/bin/start-hbase.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Modelled after $HADOOP_HOME/bin/start-hbase.sh. + +# Start hadoop hbase daemons. +# Run this on master node. +usage="Usage: start-hbase.sh" + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# start hbase daemons +errCode=$? +if [ $errCode -ne 0 ] +then + exit $errCode +fi + +# HBASE-6504 - only take the first line of the output in case verbose gc is on +distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` + + +if [ "$distMode" == 'false' ] +then + "$bin"/hbase-daemon.sh start master +else + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_REGIONSERVERS}" start regionserver + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_BACKUP_MASTERS}" start master-backup +fi diff --git a/bin/stop-hbase.sh b/bin/stop-hbase.sh new file mode 100644 index 0000000..e9749fe --- /dev/null +++ b/bin/stop-hbase.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Modelled after $HADOOP_HOME/bin/stop-hbase.sh. + +# Stop hadoop hbase daemons. Run this on master node. + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# variables needed for stop command +if [ "$HBASE_LOG_DIR" = "" ]; then + export HBASE_LOG_DIR="$HBASE_HOME/logs" +fi +mkdir -p "$HBASE_LOG_DIR" + +if [ "$HBASE_IDENT_STRING" = "" ]; then + export HBASE_IDENT_STRING="$USER" +fi + +export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-master-$HOSTNAME +export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log +logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out +loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" +pid=${HBASE_PID_DIR:-/tmp}/hbase-$HBASE_IDENT_STRING-master.pid + +echo -n stopping hbase +echo "`date` Stopping hbase (via master)" >> $loglog + +nohup nice -n ${HBASE_NICENESS:-0} "$HBASE_HOME"/bin/hbase \ + --config "${HBASE_CONF_DIR}" \ + master stop "$@" > "$logout" 2>&1 < /dev/null & + +while kill -0 `cat $pid` > /dev/null 2>&1; do + echo -n "." + sleep 1; +done +# Add a CR after we're done w/ dots. +echo + +# distributed == false means that the HMaster will kill ZK when it exits +# HBASE-6504 - only take the first line of the output in case verbose gc is on +distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` +if [ "$distMode" == 'true' ] +then + # TODO: store backup masters in ZooKeeper and have the primary send them a shutdown message + # stop any backup masters + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup + + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper +fi diff --git a/bin/zookeepers.sh b/bin/zookeepers.sh new file mode 100644 index 0000000..89a214e --- /dev/null +++ b/bin/zookeepers.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2009 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all zookeeper hosts. +# +# Environment Variables +# +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HBASE_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage="Usage: zookeepers [--config ] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +if [ "$HBASE_MANAGES_ZK" = "" ]; then + HBASE_MANAGES_ZK=true +fi + +if [ "$HBASE_MANAGES_ZK" = "true" ]; then + hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool | grep '^ZK host:' | sed 's,^ZK host:,,'` + cmd=$"${@// /\\ }" + for zookeeper in $hosts; do + ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" & + if [ "$HBASE_SLAVE_SLEEP" != "" ]; then + sleep $HBASE_SLAVE_SLEEP + fi + done +fi + +wait diff --git a/conf/hadoop-metrics.properties b/conf/hadoop-metrics.properties new file mode 100644 index 0000000..2060e22 --- /dev/null +++ b/conf/hadoop-metrics.properties @@ -0,0 +1,70 @@ +# See http://wiki.apache.org/hadoop/GangliaMetrics +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html +# GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia +# meta daemon (gmetad) service is running + +# Configuration of the "hbase" context for NullContextWithUpdateThread +# NullContextWithUpdateThread is a null context which has a thread calling +# periodically when monitoring is started. This keeps the data sampled +# correctly. +hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread +hbase.period=10 + +# Configuration of the "hbase" context for file +# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext +# hbase.fileName=/tmp/metrics_hbase.log + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + +# Configuration of the "hbase" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext +# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +# hbase.period=10 +# hbase.servers=GMETADHOST_IP:8649 + +# Configuration of the "jvm" context for null +jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread +jvm.period=10 + +# Configuration of the "jvm" context for file +# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext +# jvm.fileName=/tmp/metrics_jvm.log + +# Configuration of the "jvm" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext +# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +# jvm.period=10 +# jvm.servers=GMETADHOST_IP:8649 + +# Configuration of the "rpc" context for null +rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread +rpc.period=10 + +# Configuration of the "rpc" context for file +# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext +# rpc.fileName=/tmp/metrics_rpc.log + +# Configuration of the "rpc" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext +# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +# rpc.period=10 +# rpc.servers=GMETADHOST_IP:8649 + +# Configuration of the "rest" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext +# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +# rest.period=10 +# rest.servers=GMETADHOST_IP:8649 diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh new file mode 100644 index 0000000..7c74010 --- /dev/null +++ b/conf/hbase-env.sh @@ -0,0 +1,117 @@ +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Set environment variables here. + +# This script sets variables multiple times over the course of starting an hbase process, +# so try to keep things idempotent unless you want to take an even deeper look +# into the startup scripts (bin/hbase, etc.) + +# The java implementation to use. Java 1.6 required. +# export JAVA_HOME=/usr/java/jdk1.6.0/ + +# Extra Java CLASSPATH elements. Optional. +# export HBASE_CLASSPATH= + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="-XX:+UseConcMarkSweepGC" + +# Uncomment one of the below three options to enable java garbage collection logging for the server-side processes. + +# This enables basic gc logging to the .out file. +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# Uncomment one of the below three options to enable java garbage collection logging for the client processes. + +# This enables basic gc logging to the .out file. +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# Uncomment below if you intend to use the EXPERIMENTAL off heap cache. +# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=" +# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value. + + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers + +# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. +# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +# export HBASE_LOG_DIR=${HBASE_HOME}/logs + +# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +# export HBASE_PID_DIR=/var/hadoop/pids + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +# export HBASE_MANAGES_ZK=true diff --git a/conf/hbase-policy.xml b/conf/hbase-policy.xml new file mode 100644 index 0000000..e45f23c --- /dev/null +++ b/conf/hbase-policy.xml @@ -0,0 +1,53 @@ + + + + + + + security.client.protocol.acl + * + ACL for HRegionInterface protocol implementations (ie. + clients talking to HRegionServers) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.admin.protocol.acl + * + ACL for HMasterInterface protocol implementation (ie. + clients talking to HMaster for admin operations). + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.masterregion.protocol.acl + * + ACL for HMasterRegionInterface protocol implementations + (for HRegionServers communicating with HMaster) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + diff --git a/conf/hbase-site.xml b/conf/hbase-site.xml new file mode 100644 index 0000000..af4c300 --- /dev/null +++ b/conf/hbase-site.xml @@ -0,0 +1,25 @@ + + + + + diff --git a/conf/log4j.properties b/conf/log4j.properties new file mode 100644 index 0000000..24d7b2b --- /dev/null +++ b/conf/log4j.properties @@ -0,0 +1,75 @@ +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.security.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# Debugging Pattern format +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + +# +# Security audit appender +# +hbase.security.log.file=SecurityAuth.audit +log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFAS.File=${hbase.log.dir}/${hbase.security.log.file} +log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.category.SecurityLogger=${hbase.security.logger} +log4j.additivity.SecurityLogger=false +#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE + +# +# Null Appender +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# Custom Logging levels + +log4j.logger.org.apache.zookeeper=INFO +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.hbase=DEBUG +# Make these two classes INFO-level. Make them DEBUG to see more zk debug. +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO +#log4j.logger.org.apache.hadoop.dfs=DEBUG +# Set this class to log INFO only otherwise its OTT + +# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) +#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG + +# Uncomment the below if you want to remove logging of client region caching' +# and scan of .META. messages +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO +# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO diff --git a/conf/regionservers b/conf/regionservers new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/conf/regionservers @@ -0,0 +1 @@ +localhost diff --git a/dev-support/findHangingTest.sh b/dev-support/findHangingTest.sh new file mode 100644 index 0000000..4518c68 --- /dev/null +++ b/dev-support/findHangingTest.sh @@ -0,0 +1,41 @@ +#!/bin/bash +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## +# script to find hanging test from Jenkins build output +# usage: ./findHangingTest.sh +# +`curl -k -o jenkins.out "$1"` +expecting=Running +cat jenkins.out | while read line; do + if [[ "$line" =~ "Running org.apache.hadoop" ]]; then + if [[ "$expecting" =~ "Running" ]]; then + expecting=Tests + else + echo "Hanging test: $prevLine" + fi + fi + if [[ "$line" =~ "Tests run" ]]; then + expecting=Running + fi + if [[ "$line" =~ "Forking command line" ]]; then + a=$line + else + prevLine=$line + fi +done +rm jenkins.out diff --git a/dev-support/hbasetests.sh b/dev-support/hbasetests.sh new file mode 100644 index 0000000..e129bd4 --- /dev/null +++ b/dev-support/hbasetests.sh @@ -0,0 +1,428 @@ +#!/usr/bin/env bash +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## +# +# This script: +# - analyse the content of the .java test file to split them between +# small/medium/large +# - launch the small tests in a single maven, with surefire +# parallelisation activated +# - launch the medium & large in two maven, parallelized +# - the flaky tests are run at the end, not parallelized +# - present a small report of the global results +# - copy the failed test reports with prefix 'fail_' and a timestamp +# to protect them from a later deletion by maven +# - if configured for, relaunch the tests in errors +# +# +# Caveats: +# - multiple maven are launch, hence there can be recompilation +# between the tests if a file is modified. For non flaky tests and +# parallelization, the frame is the time to execute the small tests, +# so it's around 4 minutes. +# - Note that surefire is buggy, and the results presented while +# running may be wrong. For example, it can says that a class tests +# have 5 errors. When you look at the file it wrote, it says that the +# 2 tests are ok, and in the class there are actually two tests +# methods, not five. If you generate the report at the end with +# surefire-report it's fine however. +# +######################################### parameters + +#mvn test -Dtest=org.apache.hadoop.hbase.regionserver.TestScanWithBloomError $* + +#exit + +#set to 0 to run only developpers tests (small & medium categories) +runAllTests=0 + +#set to 1 to replay the failed tests. Previous reports are kept in +# fail_ files +replayFailed=0 + +#set to 0 to run all medium & large tests in a single maven operation +# instead of two +parallelMaven=1 + +#harcoded list of tests that often fail. We don't want to add any +# complexity around then so there are not run in parallel but after +# the others +#The ',' at the end is mandatory +flakyTests= +#org.apache.hadoop.hbase.mapreduce.TestTableInputFormatScan,org.apache.hadoop.hbase.catalog.TestMetaReaderEditorNoCluster,org.apache.hadoop.hbase.catalog.TestMetaReaderEditor,org.apache.hadoop.hbase.mapreduce.TestHFileOutputFormat,org.apache.hadoop.hbase.mapred.TestTableMapReduce,org.apache.hadoop.hbase.coprocessor.TestMasterCoprocessorExceptionWithAbort,org.apache.hadoop.hbase.coprocessor.TestMasterCoprocessorExceptionWithRemove,org.apache.hadoop.hbase.client.TestAdmin,org.apache.hadoop.hbase.master.TestMasterFailover,org.apache.hadoop.hbase.regionserver.wal.TestLogRolling,org.apache.hadoop.hbase.master.TestDistributedLogSplitting,org.apache.hadoop.hbase.master.TestMasterRestartAfterDisablingTable,org.apache.hadoop.hbase.TestGlobalMemStoreSize, + +######################################### Internal parameters +#directory used for surefire & the source code. +#They should not need to be modified +#The final / is mandatory +rootTestClassDirectory="./src/test/java/" +surefireReportDirectory="./target/surefire-reports/" + +#variable to use to debug the script without launching the tests +mvnCommand="mvn " +#mvnCommand="echo $mvnCommand" + +######################################### Functions +#get the list of the process considered as dead +# i.e.: in the same group as the script and with a ppid of 1 +# We do this because surefire can leave some dead process, so +# we will jstack them and kill them +function createListDeadProcess { + id=$$ + listDeadProcess="" + + #list of the process with a ppid of 1 + sonProcess=`ps -o pid= --ppid 1` + + #then the process with a pgid of the script + for pId in $sonProcess + do + pgid=`ps -o pgid= --pid $pId | sed 's/ //g'` + if [ "$pgid" == "$id" ] + then + listDeadProcess="$pId $listDeadProcess" + fi + done +} + +#kill the java sub process, if any, with a kill and a kill -9 +#When maven/surefire fails, it lefts some process with a ppid==1 +#we're going to find them with the pgid, print the stack and kill them. +function cleanProcess { + id=$$ + + createListDeadProcess + for pId in $listDeadProcess + do + echo "$pId survived, I will kill if it's a java process. 'ps' says:" + ps -fj --pid $pId + name=`ps -o comm= --pid $pId` + if [ "$name" == "java" ] + then + echo "$pId, java sub process of $id, is still running, killing it with a standard kill" + echo "Stack for $pId before kill:" + jstack -F -l $pId + kill $pId + echo "kill sent, waiting for 30 seconds" + sleep 30 + son=`ps -o pid= --pid $pId | wc -l` + if (test $son -gt 0) + then + echo "$pId, java sub process of $id, is still running after a standard kill, using kill -9 now" + echo "Stack for $pId before kill -9:" + jstack -F -l $pId + kill -9 $pId + echo "kill sent, waiting for 2 seconds" + sleep 2 + echo "Process $pId killed by kill -9" + else + echo "Process $pId killed by standard kill -15" + fi + else + echo "$pId is not a java process (it's $name), I don't kill it." + fi + done + + createListDeadProcess + if (test ${#listDeadProcess} -gt 0) + then + echo "There are still $sonProcess for process $id left." + else + echo "Process $id clean, no son process left" + fi +} + +#count the number of ',' in a string +# used to calculate the number of class +#write $count +function countClasses { + cars=`echo $1 | sed 's/[^,]//g' | wc -c ` + count=$((cars - 1)) +} + + +######################################### script +echo "Starting Script. Possible parameters are: runAllTests, replayFailed, nonParallelMaven" +echo "Other parameters are sent to maven" + +#We will use this value at the end to calculate the execution time +startTime=`date +%s` + +#look in the arguments if we override default values +for arg in "$@" +do + if [ $arg == "runAllTests" ] + then + runAllTests=1 + else + if [ $arg == "replayFailed" ] + then + replayFailed=1 + else + if [ $arg == "nonParallelMaven" ] + then + parallelMaven=0 + else + args=$args" $arg" + fi + fi + fi +done + + + +testsList=$(find $rootTestClassDirectory -name "Test*.java") + + +#for all java test files, let see if they contain the pattern +# to recognize the category +for testFile in $testsList +do + lenPath=$((${#rootTestClassDirectory})) + len=$((${#testFile} - $lenPath - 5)) # len(".java") == 5 + + shortTestFile=${testFile:lenPath:$len} + testName=$(echo $shortTestFile | sed 's/\//\./g') + + #The ',' is used in the grep pattern as we don't want to catch + # partial name + isFlaky=$((`echo $flakyTests | grep "$testName," | wc -l`)) + + if (test $isFlaky -eq 0) + then + isSmall=0 + isMedium=0 + isLarge=0 + + # determine the category of the test by greping into the source code + isMedium=`grep "@Category" $testFile | grep "MediumTests.class" | wc -l` + if (test $isMedium -eq 0) + then + isLarge=`grep "@Category" $testFile | grep "LargeTests.class" | wc -l` + if (test $isLarge -eq 0) + then + isSmall=`grep "@Category" $testFile | grep "SmallTests.class" | wc -l` + if (test $isSmall -eq 0) + then + echo "$testName is not categorized, so it won't be tested" + else + #sanity check on small tests + isStrange=`grep "\.startMini" $testFile | wc -l` + if (test $isStrange -gt 0) + then + echo "$testFile is categorized as 'small' but contains a .startMini string. Keep it as small anyway, but it's strange." + fi + fi + fi + fi + + #put the test in the right list + if (test $isSmall -gt 0) + then + smallList="$smallList,$testName" + fi + if (test $isMedium -gt 0) + then + mediumList="$mediumList,$testName" + fi + if (test $isLarge -gt 0) + then + largeList="$largeList,$testName" + fi + + fi +done + +#remove the ',' at the beginning +smallList=${smallList:1:${#smallList}} +mediumList=${mediumList:1:${#mediumList}} +largeList=${largeList:1:${#largeList}} + +countClasses $smallList +echo "There are $count small tests" + +countClasses $mediumList +echo "There are $count medium tests" + +countClasses $largeList +echo "There are $count large tests" + + + + +#do we launch only dev or all tests? +if (test $runAllTests -eq 1) +then + echo "Running all tests, small, medium and large" + longList="$mediumList,$largeList" +else + echo "Running developper tests only, small and medium categories" + longList=$mediumList +fi + +#medium and large test can be run in //, so we're +#going to create two lists +nextList=1 +for testClass in `echo $longList | sed 's/,/ /g'` +do + if (test $nextList -eq 1) + then + nextList=2 + runList1=$runList1,$testClass + else + nextList=1 + runList2=$runList2,$testClass + fi +done + +#remove the ',' at the beginning +runList1=${runList1:1:${#runList1}} +runList2=${runList2:1:${#runList2}} + +#now we can run the tests, at last! + +echo "Running small tests with one maven instance, in parallel" +#echo Small tests are $smallList +$mvnCommand -P singleJVMTests test -Dtest=$smallList $args +cleanProcess + +exeTime=$(((`date +%s` - $startTime)/60)) +echo "Small tests executed after $exeTime minutes" + +if (test $parallelMaven -gt 0) +then + echo "Running tests with two maven instances in parallel" + $mvnCommand -P localTests test -Dtest=$runList1 $args & + + #give some time to the fist process if there is anything to compile + sleep 30 + $mvnCommand -P localTests test -Dtest=$runList2 $args + + #wait for forked process to finish + wait + + cleanProcess + + exeTime=$(((`date +%s` - $startTime)/60)) + echo "Medium and large (if selected) tests executed after $exeTime minutes" + + #now the flaky tests, alone, if the list is not empty + # we test on size greater then 5 to remove any "," effect + if (test $runAllTests -eq 1 && test ${#flakyTests} -gt 5) + then + echo "Running flaky tests" + $mvnCommand -P localTests test -Dtest=$flakyTests $args + cleanProcess + exeTime=$(((`date +%s` - $startTime)/60)) + echo "Flaky tests executed after $exeTime minutes" + fi +else + echo "Running tests with a single maven instance, no parallelization" + $mvnCommand -P localTests test -Dtest=$runList1,$runList2,$flakyTests $args + cleanProcess + exeTime=$(((`date +%s` - $startTime)/60)) + echo "Single maven instance tests executed after $exeTime minutes" +fi + +#let's analyze the results +fullRunList="$smallList,$longList" + +if (test $runAllTests -eq 1) +then + fullRunList="$fullRunList,$flakyTests" +fi + +#single timestamp to ensure files uniquess. +timestamp=`date +%s` + +#some counters, initialized because they may not be touched +# in the loop +errorCounter=0 +sucessCounter=0 +notFinishedCounter=0 + +for testClass in `echo $fullRunList | sed 's/,/ /g'` +do + reportFile=$surefireReportDirectory/$testClass.txt + outputReportFile=$surefireReportDirectory/$testClass-output.txt + + if [ -s $reportFile ]; + then + isError=`grep FAILURE $reportFile | wc -l` + if (test $isError -gt 0) + then + errorList="$errorList,$testClass" + errorCounter=$(($errorCounter + 1)) + + #let's copy the files if we want to use it later + cp $reportFile "$surefireReportDirectory/fail_$timestamp.$testClass.txt" + if [ -s $reportFile ]; + then + cp $outputReportFile "$surefireReportDirectory/fail_$timestamp.$testClass"-output.txt"" + fi + else + + sucessCounter=$(($sucessCounter +1)) + fi + else + #report file does not exist or is empty => the test didn't finish + notFinishedCounter=$(($notFinishedCounter + 1)) + notFinishedList="$notFinishedList,$testClass" + fi +done + +#list of all tests that failed +replayList="$notFinishedList""$errorList" + +#remove the ',' at the beginning +notFinishedList=${notFinishedList:1:${#notFinishedList}} +errorList=${errorList:1:${#errorList}} +replayList=${replayList:1:${#replayList}} + +#make it simpler to read by removing the org.* stuff from the name +notFinishedPresList=`echo $notFinishedList | sed 's/org.apache.hadoop.hbase.//g' | sed 's/,/, /g'` +errorPresList=`echo $errorList | sed 's/org.apache.hadoop.hbase.//g' | sed 's/,/, /g'` + + +#calculate the execution time +curTime=`date +%s` +exeTime=$((($curTime - $startTime)/60)) + +echo "##########################" +echo "$sucessCounter tests executed successfully" +echo "$errorCounter tests are in error" +echo "$notFinishedCounter tests didn't finish" +echo +echo "Tests in error are: $errorPresList" +echo "Tests that didn't finish are: $notFinishedPresList" +echo +echo "Execution time in minutes: $exeTime" +echo "##########################" + + +if (test ${#replayList} -gt 0) +then + if (test $replayFailed -gt 0) + then + echo "Replaying all tests that failed" + $mvnCommand -P localTests test -Dtest=$replayList $args + echo "Replaying done" + fi +fi + +exit diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh new file mode 100644 index 0000000..74a7128 --- /dev/null +++ b/dev-support/smart-apply-patch.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +PATCH_FILE=$1 +if [ -z "$PATCH_FILE" ]; then + echo usage: $0 patch-file + exit 1 +fi + +PATCH=${PATCH:-patch} # allow overriding patch binary + +# Cleanup handler for temporary files +TOCLEAN="" +cleanup() { + rm $TOCLEAN + exit $1 +} +trap "cleanup 1" HUP INT QUIT TERM + +# Allow passing "-" for stdin patches +if [ "$PATCH_FILE" == "-" ]; then + PATCH_FILE=/tmp/tmp.in.$$ + cat /dev/fd/0 > $PATCH_FILE + TOCLEAN="$TOCLEAN $PATCH_FILE" +fi + +# Come up with a list of changed files into $TMP +TMP=/tmp/tmp.paths.$$ +TOCLEAN="$TOCLEAN $TMP" + +if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then + PLEVEL=0 + #if the patch applied at P0 there is the possability that all we are doing + # is adding new files and they would apply anywhere. So try to guess the + # correct place to put those files. + + TMP2=/tmp/tmp.paths.2.$$ + TOCLEAN="$TOCLEAN $TMP2" + + grep '^patching file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2 + + #first off check that all of the files do not exist + FOUND_ANY=0 + for CHECK_FILE in $(cat $TMP2) + do + if [[ -f $CHECK_FILE ]]; then + FOUND_ANY=1 + fi + done + + if [[ "$FOUND_ANY" = "0" ]]; then + #all of the files are new files so we have to guess where the correct place to put it is. + + # if all of the lines start with a/ or b/, then this is a git patch that + # was generated without --no-prefix + if ! grep -qv '^a/\|^b/' $TMP2 ; then + echo Looks like this is a git patch. Stripping a/ and b/ prefixes + echo and incrementing PLEVEL + PLEVEL=$[$PLEVEL + 1] + sed -i -e 's,^[ab]/,,' $TMP2 + fi + fi +elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then + PLEVEL=1 +elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then + PLEVEL=2 +else + echo "The patch does not appear to apply with p0 to p2"; + cleanup 1; +fi + +echo Going to apply patch with: $PATCH -p$PLEVEL +$PATCH -p$PLEVEL -E < $PATCH_FILE + +cleanup $? diff --git a/dev-support/test-patch.properties b/dev-support/test-patch.properties new file mode 100644 index 0000000..afe21d4 --- /dev/null +++ b/dev-support/test-patch.properties @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MAVEN_OPTS="-Xmx3g" + +# The number of acceptable warning for *all* modules +# Please update the per-module test-patch.properties if you update this file. + +OK_RELEASEAUDIT_WARNINGS=84 +OK_FINDBUGS_WARNINGS=768 +OK_JAVADOC_WARNINGS=169 diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh new file mode 100644 index 0000000..bdef3ab --- /dev/null +++ b/dev-support/test-patch.sh @@ -0,0 +1,740 @@ +#!/usr/bin/env bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +#set -x + +### Setup some variables. +### SVN_REVISION and BUILD_URL are set by Hudson if it is run by patch process +### Read variables from properties file +bindir=$(dirname $0) + +# Defaults +if [ -z "$MAVEN_HOME" ]; then + MVN=mvn +else + MVN=$MAVEN_HOME/bin/mvn +fi + +PROJECT_NAME=HBase +JENKINS=false +PATCH_DIR=/tmp +SUPPORT_DIR=/tmp +BASEDIR=$(pwd) + +PS=${PS:-ps} +AWK=${AWK:-awk} +WGET=${WGET:-wget} +SVN=${SVN:-svn} +GREP=${GREP:-grep} +PATCH=${PATCH:-patch} +JIRACLI=${JIRA:-jira} +FINDBUGS_HOME=${FINDBUGS_HOME} +FORREST_HOME=${FORREST_HOME} +ECLIPSE_HOME=${ECLIPSE_HOME} + +############################################################################### +printUsage() { + echo "Usage: $0 [options] patch-file | defect-number" + echo + echo "Where:" + echo " patch-file is a local patch file containing the changes to test" + echo " defect-number is a JIRA defect number (e.g. 'HADOOP-1234') to test (Jenkins only)" + echo + echo "Options:" + echo "--patch-dir= The directory for working and output files (default '/tmp')" + echo "--basedir= The directory to apply the patch to (default current directory)" + echo "--mvn-cmd= The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')" + echo "--ps-cmd= The 'ps' command to use (default 'ps')" + echo "--awk-cmd= The 'awk' command to use (default 'awk')" + echo "--svn-cmd= The 'svn' command to use (default 'svn')" + echo "--grep-cmd= The 'grep' command to use (default 'grep')" + echo "--patch-cmd= The 'patch' command to use (default 'patch')" + echo "--findbugs-home= Findbugs home directory (default FINDBUGS_HOME environment variable)" + echo "--forrest-home= Forrest home directory (default FORREST_HOME environment variable)" + echo "--dirty-workspace Allow the local SVN workspace to have uncommitted changes" + echo + echo "Jenkins-only options:" + echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)" + echo "--support-dir= The directory to find support files in" + echo "--wget-cmd= The 'wget' command to use (default 'wget')" + echo "--jira-cmd= The 'jira' command to use (default 'jira')" + echo "--jira-password= The password for the 'jira' command" + echo "--eclipse-home= Eclipse home directory (default ECLIPSE_HOME environment variable)" +} + +############################################################################### +parseArgs() { + for i in $* + do + case $i in + --jenkins) + JENKINS=true + ;; + --patch-dir=*) + PATCH_DIR=${i#*=} + ;; + --support-dir=*) + SUPPORT_DIR=${i#*=} + ;; + --basedir=*) + BASEDIR=${i#*=} + ;; + --mvn-cmd=*) + MVN=${i#*=} + ;; + --ps-cmd=*) + PS=${i#*=} + ;; + --awk-cmd=*) + AWK=${i#*=} + ;; + --wget-cmd=*) + WGET=${i#*=} + ;; + --svn-cmd=*) + SVN=${i#*=} + ;; + --grep-cmd=*) + GREP=${i#*=} + ;; + --patch-cmd=*) + PATCH=${i#*=} + ;; + --jira-cmd=*) + JIRACLI=${i#*=} + ;; + --jira-password=*) + JIRA_PASSWD=${i#*=} + ;; + --findbugs-home=*) + FINDBUGS_HOME=${i#*=} + ;; + --forrest-home=*) + FORREST_HOME=${i#*=} + ;; + --eclipse-home=*) + ECLIPSE_HOME=${i#*=} + ;; + --dirty-workspace) + DIRTY_WORKSPACE=true + ;; + *) + PATCH_OR_DEFECT=$i + ;; + esac + done + if [ -z "$PATCH_OR_DEFECT" ]; then + printUsage + exit 1 + fi + if [[ $JENKINS == "true" ]] ; then + echo "Running in Jenkins mode" + defect=$PATCH_OR_DEFECT + ECLIPSE_PROPERTY="-Declipse.home=$ECLIPSE_HOME" + else + echo "Running in developer mode" + JENKINS=false + ### PATCH_FILE contains the location of the patchfile + PATCH_FILE=$PATCH_OR_DEFECT + if [[ ! -e "$PATCH_FILE" ]] ; then + echo "Unable to locate the patch file $PATCH_FILE" + cleanupAndExit 0 + fi + ### Check if $PATCH_DIR exists. If it does not exist, create a new directory + if [[ ! -e "$PATCH_DIR" ]] ; then + mkdir "$PATCH_DIR" + if [[ $? == 0 ]] ; then + echo "$PATCH_DIR has been created" + else + echo "Unable to create $PATCH_DIR" + cleanupAndExit 0 + fi + fi + ### Obtain the patch filename to append it to the version number + defect=`basename $PATCH_FILE` + fi +} + +############################################################################### +checkout () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Testing patch for ${defect}." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + ### When run by a developer, if the workspace contains modifications, do not continue + ### unless the --dirty-workspace option was set + status=`$SVN stat --ignore-externals | sed -e '/^X[ ]*/D'` + if [[ $JENKINS == "false" ]] ; then + if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then + echo "ERROR: can't run in a workspace that contains the following modifications" + echo "$status" + cleanupAndExit 1 + fi + echo + else + cd $BASEDIR + $SVN revert -R . + rm -rf `$SVN status --no-ignore` + $SVN update + fi + return $? +} + +############################################################################### +setup () { + ### Download latest patch file (ignoring .htm and .html) when run from patch process + if [[ $JENKINS == "true" ]] ; then + $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect + if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then + echo "$defect is not \"Patch Available\". Exiting." + cleanupAndExit 0 + fi + relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira | $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'` + patchURL="http://issues.apache.org${relativePatchURL}" + patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'` + echo "$defect patch is being downloaded at `date` from" + echo "$patchURL" + $WGET -q -O $PATCH_DIR/patch $patchURL + VERSION=${SVN_REVISION}_${defect}_PATCH-${patchNum} + JIRA_COMMENT="Here are the results of testing the latest attachment + $patchURL + against trunk revision ${SVN_REVISION}." + + ### Copy in any supporting files needed by this process + cp -r $SUPPORT_DIR/lib/* ./lib + #PENDING: cp -f $SUPPORT_DIR/etc/checkstyle* ./src/test + ### Copy the patch file to $PATCH_DIR + else + VERSION=PATCH-${defect} + cp $PATCH_FILE $PATCH_DIR/patch + if [[ $? == 0 ]] ; then + echo "Patch file $PATCH_FILE copied to $PATCH_DIR" + else + echo "Could not copy $PATCH_FILE to $PATCH_DIR" + cleanupAndExit 0 + fi + fi + . $BASEDIR/dev-support/test-patch.properties + ### exit if warnings are NOT defined in the properties file + if [ -z "$OK_FINDBUGS_WARNINGS" ] || [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[ -z $OK_RELEASEAUDIT_WARNINGS ]] ; then + echo "Please define the following properties in test-patch.properties file" + echo "OK_FINDBUGS_WARNINGS" + echo "OK_RELEASEAUDIT_WARNINGS" + echo "OK_JAVADOC_WARNINGS" + cleanupAndExit 1 + fi + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Pre-build trunk to verify trunk stability and javac warnings" + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1" + export MAVEN_OPTS="${MAVEN_OPTS}" + $MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1 + if [[ $? != 0 ]] ; then + echo "Trunk compilation is broken?" + cleanupAndExit 1 + fi +} + +############################################################################### +### Check for @author tags in the patch +checkAuthor () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Checking there are no @author tags in the patch." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + authorTags=`$GREP -c -i '@author' $PATCH_DIR/patch` + echo "There appear to be $authorTags @author tags in the patch." + if [[ $authorTags != 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + -1 @author. The patch appears to contain $authorTags @author tags which the Hadoop community has agreed to not allow in code contributions." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + +1 @author. The patch does not contain any @author tags." + return 0 +} + +############################################################################### +### Check for tests in the patch +checkTests () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Checking there are new or changed tests in the patch." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + testReferences=`$GREP -c -i '/test' $PATCH_DIR/patch` + echo "There appear to be $testReferences test files referenced in the patch." + if [[ $testReferences == 0 ]] ; then + if [[ $JENKINS == "true" ]] ; then + patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira` + if [[ $patchIsDoc != 0 ]] ; then + echo "The patch appears to be a documentation patch that doesn't require tests." + JIRA_COMMENT="$JIRA_COMMENT + + +0 tests included. The patch appears to be a documentation patch that doesn't require tests." + return 0 + fi + fi + JIRA_COMMENT="$JIRA_COMMENT + + -1 tests included. The patch doesn't appear to include any new or modified tests. + Please justify why no new tests are needed for this patch. + Also please list what manual steps were performed to verify this patch." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + +1 tests included. The patch appears to include $testReferences new or modified tests." + return 0 +} + +############################################################################### +### Attempt to apply the patch +applyPatch () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Applying patch." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + + $PATCH -p0 -E < $PATCH_DIR/patch + if [[ $? != 0 ]] ; then + echo "PATCH APPLICATION FAILED" + JIRA_COMMENT="$JIRA_COMMENT + + -1 patch. The patch command could not apply the patch." + return 1 + fi + return 0 +} + +############################################################################### +### Check there are no javadoc warnings +checkJavadocWarnings () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched javadoc warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN clean compile javadoc:javadoc -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1" + export MAVEN_OPTS="${MAVEN_OPTS}" + $MVN clean compile javadoc:javadoc -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1 + javadocWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavadocWarnings.txt | $AWK '/Javadoc Warnings/,EOF' | $GREP warning | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` + echo "" + echo "" + echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build." + + ### if current warnings greater than OK_JAVADOC_WARNINGS + if [[ $javadocWarnings -gt $OK_JAVADOC_WARNINGS ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + -1 javadoc. The javadoc tool appears to have generated `expr $(($javadocWarnings-$OK_JAVADOC_WARNINGS))` warning messages." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + +1 javadoc. The javadoc tool did not generate any warning messages." + return 0 +} + +############################################################################### +### Check there are no changes in the number of Javac warnings +checkJavacWarnings () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched javac warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavacWarnings.txt 2>&1" + export MAVEN_OPTS="${MAVEN_OPTS}" + $MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavacWarnings.txt 2>&1 + if [[ $? != 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + -1 javac. The patch appears to cause mvn compile goal to fail." + return 1 + fi + ### Compare trunk and patch javac warning numbers + if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then + trunkJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` + patchJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` + echo "There appear to be $trunkJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch." + if [[ $patchJavacWarnings != "" && $trunkJavacWarnings != "" ]] ; then + if [[ $patchJavacWarnings -gt $trunkJavacWarnings ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + -1 javac. The applied patch generated $patchJavacWarnings javac compiler warnings (more than the trunk's current $trunkJavacWarnings warnings)." + return 1 + fi + fi + fi + JIRA_COMMENT="$JIRA_COMMENT + + +1 javac. The applied patch does not increase the total number of javac compiler warnings." + return 0 +} + +############################################################################### +### Check there are no changes in the number of release audit (RAT) warnings +checkReleaseAuditWarnings () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched release audit warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1" + export MAVEN_OPTS="${MAVEN_OPTS}" + $MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1 + find $BASEDIR -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt + + ### Compare trunk and patch release audit warning numbers + if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then + patchReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt` + echo "" + echo "" + echo "There appear to be $OK_RELEASEAUDIT_WARNINGS release audit warnings before the patch and $patchReleaseAuditWarnings release audit warnings after applying the patch." + if [[ $patchReleaseAuditWarnings != "" && $OK_RELEASEAUDIT_WARNINGS != "" ]] ; then + if [[ $patchReleaseAuditWarnings -gt $OK_RELEASEAUDIT_WARNINGS ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + -1 release audit. The applied patch generated $patchReleaseAuditWarnings release audit warnings (more than the trunk's current $OK_RELEASEAUDIT_WARNINGS warnings)." + $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt + echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt + JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/trunk/patchprocess/patchReleaseAuditProblems.txt +$JIRA_COMMENT_FOOTER" + return 1 + fi + fi + fi + JIRA_COMMENT="$JIRA_COMMENT + + +1 release audit. The applied patch does not increase the total number of release audit warnings." + return 0 +} + +############################################################################### +### Check there are no changes in the number of Checkstyle warnings +checkStyle () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched checkstyle warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "THIS IS NOT IMPLEMENTED YET" + echo "" + echo "" + echo "$MVN compile checkstyle:checkstyle -D${PROJECT_NAME}PatchProcess" + export MAVEN_OPTS="${MAVEN_OPTS}" + $MVN compile checkstyle:checkstyle -D${PROJECT_NAME}PatchProcess + + JIRA_COMMENT_FOOTER="Checkstyle results: $BUILD_URL/artifact/trunk/build/test/checkstyle-errors.html +$JIRA_COMMENT_FOOTER" + ### TODO: calculate actual patchStyleErrors +# patchStyleErrors=0 +# if [[ $patchStyleErrors != 0 ]] ; then +# JIRA_COMMENT="$JIRA_COMMENT +# +# -1 checkstyle. The patch generated $patchStyleErrors code style errors." +# return 1 +# fi +# JIRA_COMMENT="$JIRA_COMMENT +# +# +1 checkstyle. The patch generated 0 code style errors." + return 0 +} + +############################################################################### +### Check there are no changes in the number of Findbugs warnings +checkFindbugsWarnings () { + findbugs_version=`${FINDBUGS_HOME}/bin/findbugs -version` + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched Findbugs warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess" + export MAVEN_OPTS="${MAVEN_OPTS}" + $MVN clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess < /dev/null + + if [ $? != 0 ] ; then + JIRA_COMMENT="$JIRA_COMMENT + + -1 findbugs. The patch appears to cause Findbugs (version ${findbugs_version}) to fail." + return 1 + fi + + findbugsWarnings=0 + for file in $(find $BASEDIR -name findbugsXml.xml) + do + relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix + if [ ! $relative_file == "target/findbugsXml.xml" ]; then + module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path + module_suffix=`basename ${module_suffix}` + fi + + cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml + $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ + $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml + newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'` + echo "Found $newFindbugsWarnings Findbugs warnings ($file)" + findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings)) + $FINDBUGS_HOME/bin/convertXmlToText -html \ + $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html + JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/patchprocess/newPatchFindbugsWarnings${module_suffix}.html +$JIRA_COMMENT_FOOTER" + done + + ### if current warnings greater than OK_FINDBUGS_WARNINGS + if [[ $findbugsWarnings -gt $OK_FINDBUGS_WARNINGS ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + -1 findbugs. The patch appears to introduce `expr $(($findbugsWarnings-$OK_FINDBUGS_WARNINGS))` new Findbugs (version ${findbugs_version}) warnings." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + +1 findbugs. The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings." + return 0 +} + +############################################################################### +### Run the tests +runTests () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Running tests." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + + failed_tests="" + ### Kill any rogue build processes from the last attempt + $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null + echo "$MVN clean test -P runAllTests -D${PROJECT_NAME}PatchProcess -Dsurefire.secondPartThreadCount=4" + export MAVEN_OPTS="${MAVEN_OPTS}" + ulimit -a + $MVN clean test -P runAllTests -D${PROJECT_NAME}PatchProcess -Dsurefire.secondPartThreadCount=4 + if [[ $? != 0 ]] ; then + ### Find and format names of failed tests + failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E " /dev/null + + #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults" + #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults + echo "NOP" + return 0 + if [[ $? != 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + -1 system test framework. The patch failed system test framework compile." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + +1 system test framework. The patch passed system test framework compile." + return 0 +} + +############################################################################### +### Submit a comment to the defect's Jira +submitJiraComment () { + local result=$1 + ### Do not output the value of JIRA_COMMENT_FOOTER when run by a developer + if [[ $JENKINS == "false" ]] ; then + JIRA_COMMENT_FOOTER="" + fi + if [[ $result == 0 ]] ; then + comment="+1 overall. $JIRA_COMMENT + +$JIRA_COMMENT_FOOTER" + else + comment="-1 overall. $JIRA_COMMENT + +$JIRA_COMMENT_FOOTER" + fi + ### Output the test result to the console + echo " + + + +$comment" + + if [[ $JENKINS == "true" ]] ; then + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Adding comment to Jira." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + ### Update Jira with a comment + export USER=hudson + $JIRACLI -s https://issues.apache.org/jira -a addcomment -u hadoopqa -p $JIRA_PASSWD --comment "$comment" --issue $defect + $JIRACLI -s https://issues.apache.org/jira -a logout -u hadoopqa -p $JIRA_PASSWD + fi +} + +############################################################################### +### Cleanup files +cleanupAndExit () { + local result=$1 + if [[ $JENKINS == "true" ]] ; then + if [ -e "$PATCH_DIR" ] ; then + mv $PATCH_DIR $BASEDIR + fi + fi + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Finished build." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + exit $result +} + +############################################################################### +############################################################################### +############################################################################### + +JIRA_COMMENT="" +JIRA_COMMENT_FOOTER="Console output: $BUILD_URL/console + +This message is automatically generated." + +### Check if arguments to the script have been specified properly or not +parseArgs $@ +cd $BASEDIR + +checkout +RESULT=$? +if [[ $JENKINS == "true" ]] ; then + if [[ $RESULT != 0 ]] ; then + exit 100 + fi +fi +setup +checkAuthor +RESULT=$? +checkTests +(( RESULT = RESULT + $? )) +applyPatch +if [[ $? != 0 ]] ; then + submitJiraComment 1 + cleanupAndExit 1 +fi +checkJavadocWarnings +(( RESULT = RESULT + $? )) +checkJavacWarnings +(( RESULT = RESULT + $? )) +### Checkstyle not implemented yet +#checkStyle +#(( RESULT = RESULT + $? )) +checkFindbugsWarnings +(( RESULT = RESULT + $? )) +checkReleaseAuditWarnings +(( RESULT = RESULT + $? )) +### Do not call these when run by a developer +if [[ $JENKINS == "true" ]] ; then + runTests + (( RESULT = RESULT + $? )) +JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/ +$JIRA_COMMENT_FOOTER" +fi +submitJiraComment $RESULT +cleanupAndExit $RESULT diff --git a/dev-support/test-util.sh b/dev-support/test-util.sh new file mode 100644 index 0000000..3101e10 --- /dev/null +++ b/dev-support/test-util.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +usage() +{ +cat << EOF +usage: $0 [options] [test-name...] + +Run a set of hbase tests. Individual tests may be specified on the +command line or in a file using -f, with one test per line. Runs all +tests by default. Each specified tests should include the fully +qualified package name. + +options: + -h Show this message + -c Run 'mvn clean' before running the tests + -f FILE Run the additional tests listed in the FILE + -u Only run unit tests. Default is to run + unit and integration tests + -n N Run each test N times. Default = 1. + -s N Print N slowest tests + -H Print which tests are hanging (if any) + -e Echo the maven call before running. Default: not enabled + -r Runs remotely, on the build server. Default: not enabled +EOF +} + +echoUsage=0 +server=0 +testFile= +doClean="" +testType=verify +numIters=1 +showSlowest= +showHanging= + +# normalize path refs for surefire +if [[ "$0" != /* ]]; then + # relative path + scriptDir=`pwd`/$(dirname $0) +else + # absolute path + scriptDir=$(dirname $0) +fi +testDir=$scriptDir/../../../target/surefire-reports + +while getopts "hcerHun:s:f:" OPTION +do + case $OPTION in + h) + usage + exit 0 + ;; + c) + doClean="clean" + ;; + H) + showHanging=1 + ;; + u) + testType=test + ;; + n) + numIters=$OPTARG + ;; + s) + showSlowest=$OPTARG + ;; + f) + testFile=$OPTARG + ;; + e) + echoUsage=1 + ;; + r) + server=1 + ;; + ?) + usage + exit 1 + esac +done + +testIdx=0 + +# add tests specified in a file +if [ ! -z $testFile ]; then + exec 3<$testFile + + while read <&3 line ; do + if [ ! -z "$line" ]; then + test[$testIdx]="$line" + fi + testIdx=$(($testIdx+1)) + done +fi + +# add tests specified on cmd line +if [ ! -z $BASH_ARGC ]; then + shift $(($OPTIND - 1)) + for (( i = $OPTIND; i <= $BASH_ARGC; i++ )) + do + test[$testIdx]=$1; + testIdx=$(($testIdx+1)) + shift + done +fi + +echo "Running tests..." +numTests=${#test[@]} + +for (( i = 1 ; i <= $numIters; i++ )) +do + if [[ $numTests > 0 ]]; then + #Now loop through each test + for (( j = 0; j < $numTests; j++ )) + do + # Create the general command + cmd="nice -10 mvn $doClean $testType -Dtest=${test[$j]}" + + # Add that is should run locally, if not on the server + if [ ${server} -eq 0 ]; then + cmd="${cmd} -P localTests" + fi + + # Print the command, if we should + if [ ${echoUsage} -eq 1 ]; then + echo "${cmd}" + fi + + # Run the command + $cmd + + if [ $? -ne 0 ]; then + echo "${test[$j]} failed, iteration: $i" + exit 1 + fi + done + else + echo "EXECUTING ALL TESTS" + # Create the general command + cmd="nice -10 mvn $doClean $testType" + + # Add that is should run locally, if not on the server + if [ ${server} -eq 0 ]; then + cmd="${cmd} -P localTests" + fi + + # Print the command, if we should + if [ ${echoUsage} -eq 1 ]; then + echo "${cmd}" + fi + + #now run the command + $cmd + fi +done + +# Print a report of the slowest running tests +if [ ! -z $showSlowest ]; then + + testNameIdx=0 + for (( i = 0; i < ${#test[@]}; i++ )) + do + testNames[$i]=$testDir/'TEST-'${test[$i]}'.xml' + done + + echo "Slowest $showSlowest tests:" + + awk '//,""); \ + split($7,testname,"="); \ + split($3,time,"="); \ + print testname[2]"\t"time[2]}' \ + ${testNames[@]} \ + | sort -k2,2rn \ + | head -n $showSlowest +fi + +# Print a report of tests that hung +if [ ! -z $showHanging ]; then + echo "Hanging tests:" + find $testDir -type f -name *.xml -size 0k \ + | xargs basename \ + | cut -d"." -f -1 +fi diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..1c80fc7 --- /dev/null +++ b/pom.xml @@ -0,0 +1,2591 @@ + + + + 4.0.0 + + + + org.apache + apache + 8 + + + + org.apache.hbase + hbase + jar + 0.94.8 + HBase + + HBase is the &lt;a href="http://hadoop.apache.org"&rt;Hadoop</a&rt; database. Use it when you need + random, realtime read/write access to your Big Data. + This project's goal is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters + of commodity hardware. + + http://hbase.apache.org + + + scm:svn:http://svn.apache.org/repos/asf/hbase/trunk + scm:svn:https://svn.apache.org/repos/asf/hbase/trunk + http://svn.apache.org/viewvc/hbase/trunk + + + + JIRA + http://issues.apache.org/jira/browse/HBASE + + + + hudson + http://hudson.zones.apache.org/hudson/view/HBase/job/HBase-TRUNK/ + + + + + User List + user-subscribe@hbase.apache.org + user-unsubscribe@hbase.apache.org + user@hbase.apache.org + http://mail-archives.apache.org/mod_mbox/hbase-user/ + + http://hbase.apache.org/mail/user/ + http://dir.gmane.org/gmane.comp.java.hadoop.hbase.user + http://search-hadoop.com/?q=&fc_project=HBase + + + + Developer List + dev-subscribe@hbase.apache.org + dev-unsubscribe@hbase.apache.org + dev@hbase.apache.org + http://mail-archives.apache.org/mod_mbox/hbase-dev/ + + http://hbase.apache.org/mail/dev/ + http://dir.gmane.org/gmane.comp.java.hadoop.hbase.devel + http://search-hadoop.com/?q=&fc_project=HBase + + + + Commits List + commits-subscribe@hbase.apache.org + commits-unsubscribe@hbase.apache.org + http://mail-archives.apache.org/mod_mbox/hbase-commits/ + + http://hbase.apache.org/mail/commits/ + + + + Issues List + issues-subscribe@hbase.apache.org + issues-unsubscribe@hbase.apache.org + http://mail-archives.apache.org/mod_mbox/hbase-issues/ + + http://hbase.apache.org/mail/issues/ + + + + + + + apurtell + Andrew Purtell + apurtell@apache.org + -8 + Intel + http://www.intel.com + + + dmeil + Doug Meil + dmeil@apache.org + -5 + Explorys + http://www.explorys.com + + + garyh + Gary Helmling + garyh@apache.org + -8 + Twitter + http://www.twitter.com + + + jdcryans + Jean-Daniel Cryans + jdcryans@apache.org + -8 + StumbleUpon + http://www.stumbleupon.com + + + jgray + Jonathan Gray + jgray@fb.com + -8 + Facebook + http://www.facebook.com + + + jmhsieh + Jonathan Hsieh + jmhsieh@apache.org + -8 + Cloudera + http://www.cloudera.com + + + kannan + Kannan Muthukkaruppan + kannan@fb.com + -8 + Facebook + http://www.facebook.com + + + karthik + Karthik Ranganathan + kranganathan@fb.com + -8 + Facebook + http://www.facebook.com + + + larsgeorge + Lars George + larsgeorge@apache.org + +1 + Cloudera + http://www.cloudera.com/ + + + larsh + Lars Hofhansl + larsh@apache.org + -8 + Salesforce.com + http://www.salesforce.com/ + + + mbautin + Mikhail Bautin + mbautin@apache.org + -8 + Facebook + http://www.facebook.com + + + nspiegelberg + Nicolas Spiegelberg + nspiegelberg@fb.com + -8 + Facebook + http://www.facebook.com + + + rawson + Ryan Rawson + rawson@apache.org + -8 + CX + http://www.cx.com + + + stack + Michael Stack + stack@apache.org + -8 + StumbleUpon + http://www.stumbleupon.com/ + + + tedyu + Ted Yu + yuzhihong@gmail.com + -8 + CarrierIQ + http://www.carrieriq.com + + + todd + Todd Lipcon + todd@apache.org + -8 + Cloudera + http://www.cloudera.com + + + ramkrishna + Ramkrishna S Vasudevan + ramkrishna@apache.org + +5 + Intel + http://www.intel.in + + + + + + apache release + https://repository.apache.org/content/repositories/releases/ + + + apache non-releases + Apache non-releases + http://people.apache.org/~stack/m2/repository + + false + + + true + + + + java.net + Java.Net + http://download.java.net/maven/2/ + + false + + + true + + + + codehaus + Codehaus Public + http://repository.codehaus.org/ + + false + + + true + + + + repository.jboss.org + http://repository.jboss.org/nexus/content/groups/public-jboss/ + + false + + + + ghelmling.testing + Gary Helmling test repo + http://people.apache.org/~garyh/mvn/ + + true + + + true + + + + + + + ghelmling.testing + Gary Helmling test repo + http://people.apache.org/~garyh/mvn/ + + true + + + true + + + + + + + + + + org.apache.maven.plugins + maven-release-plugin + + + + apache-release + + -Dmaven.test.skip.exec + + + + maven-compiler-plugin + 2.5.1 + + ${compileSource} + ${compileSource} + true + false + -Xlint:-options + + + + maven-surefire-plugin + ${surefire.version} + + + + org.apache.maven.surefire + ${surefire.provider} + ${surefire.version} + + + + + ${surefire.timeout} + -enableassertions -Xmx1900m -Djava.security.egd=file:/dev/./urandom + ${test.output.tofile} + + + + org.apache.maven.plugins + maven-site-plugin + 3.2 + + + org.apache.maven.plugins + maven-failsafe-plugin + ${surefire.version} + + + org.apache.maven.surefire + ${surefire.provider} + ${surefire.version} + + + + + ${integrationtest.include} + + + ${unittest.include} + **/*$* + + ${test.output.tofile} + + ${env.LD_LIBRARY_PATH}:${project.build.directory}/nativelib + ${env.DYLD_LIBRARY_PATH}:${project.build.directory}/nativelib + 4 + + + + + integration-test + + integration-test + + + + verify + + verify + + + + + + maven-clean-plugin + + + + + build + + + + + + maven-surefire-report-plugin + ${surefire.version} + + + org.apache.avro + avro-maven-plugin + ${avro.version} + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.avro + avro-maven-plugin + [1.5.3,) + + protocol + schema + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + [1.3,) + + run + + + + + + + + + org.jamon + jamon-maven-plugin + [2.3.4,) + + translate + + + + + + + + + + + + org.apache.rat + apache-rat-plugin + 0.8 + + + **/*.log + **/.* + **/*.tgz + **/*.orig + test/** + **/8e8ab58dcf39412da19833fcd8f687ac + **/.git/** + **/target/** + **/CHANGES.txt + **/generated/** + **/conf/* + **/*.avpr + **/*.svg + **/*.vm + **/control + **/conffile + docs/* + + **/src/site/resources/css/freebsd_docbook.css + + .git/** + .svn/** + + + + + + + + + + src/main/resources/ + + hbase-default.xml + + + + ${project.build.directory} + + hbase-webapps/** + + + + + + src/test/resources + + hbase-site.xml + + + + + + + maven-site-plugin + + UTF-8 + UTF-8 + src/site/site.vm + + + + org.apache.avro + avro-maven-plugin + + + generate-avro-sources + generate-sources + + schema + protocol + + + + + ${project.build.directory}/generated-sources/java + + + + org.codehaus.mojo + xml-maven-plugin + 1.0-beta-3 + + + + transform + + pre-site + + + + + + ${basedir}/src/main/resources/ + + hbase-default.xml + + ${basedir}/src/main/xslt/configuration_to_docbook_section.xsl + ${basedir}/target/site/ + + + + + + com.agilejava.docbkx + docbkx-maven-plugin + 2.0.13 + + + multipage + pre-site + + true + true + true + true + 100 + true + true + ${basedir}/target/site/book/ + ../css/freebsd_docbook.css + src/docbkx/customization.xsl + ../images/ + 2 + yes + + + generate-html + + + + onepage + pre-site + + true + true + 100 + true + true + ${basedir}/target/site/ + css/freebsd_docbook.css + images/ + 2 + yes + + + generate-html + + + + + + org.docbook + docbook-xml + 4.4 + runtime + + + + + maven-assembly-plugin + + gnu + false + + src/assembly/all.xml + + + + + tarball + package + + single + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + prepare-package + + test-jar + + + + + + + org/apache/hadoop/hbase/mapreduce/Driver + + + + + org/apache/jute/** + org/apache/zookeeper/** + **/*.jsp + hbase-site.xml + log4j.properties + mapred-queues.xml + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + prepare-package + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${surefire.skipFirstPart} + ${surefire.firstPartForkMode} + ${surefire.firstPartParallel} + false + ${surefire.firstPartThreadCount} + classes + ${surefire.firstPartGroups} + false + + + + secondPartTestsExecution + test + test + + ${surefire.skipSecondPart} + false + perThread + false + ${surefire.secondPartThreadCount} + classes + ${surefire.secondPartGroups} + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + false + always + + 1800 + -enableassertions -Xmx1900m + -Djava.security.egd=file:/dev/./urandom -Djava.net.preferIPv4Stack=true + false + + + + maven-antrun-plugin + 1.6 + + + arc-setup + initialize + + + + + + + + + + + + + + + + + + + + + + + + run + + + + generate + generate-sources + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + run + + + + process-resources + + + + + + + + + + + + run + + + + package + package + + + + + + + + + which cygpath 2> /dev/null + if [ $? = 1 ]; then + BUILD_DIR="${project.build.directory}" + else + BUILD_DIR=`cygpath --unix '${project.build.directory}'` + fi + if [ `ls $BUILD_DIR/nativelib | wc -l` -ne 0 ]; then + cp -PR $BUILD_DIR/nativelib/lib* $BUILD_DIR/${project.build.finalName}/${project.build.finalName}/lib/native/${build.platform} + fi + + + + + + + + which cygpath 2> /dev/null + if [ $? = 1 ]; then + BUILD_DIR="${project.build.directory}" + else + BUILD_DIR=`cygpath --unix '${project.build.directory}'` + fi + + cd $BUILD_DIR/${project.build.finalName} + tar czf $BUILD_DIR/${project.build.finalName}.tar.gz ${project.build.finalName} + + + + + + + + + run + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + jspcSource-packageInfo-Avro-source + generate-sources + + add-source + + + + ${project.build.directory}/generated-jamon + ${project.build.directory}/generated-sources/java + + + + + src-index + generate-sources + + add-source + + + + ${project.basedir}/secondaryindex/src/main/java + + + + + test-index + generate-sources + + add-test-source + + + + ${project.basedir}/secondaryindex/src/test/java + + + + + + + org.jamon + jamon-maven-plugin + 2.3.4 + + + generate-sources + + translate + + + src/main/jamon + target/generated-jamon + + + + + + org.apache.maven.plugins + maven-eclipse-plugin + 2.8 + + + org.jamon.project.jamonnature + + + org.jamon.project.templateBuilder + org.eclipse.jdt.core.javabuilder + org.jamon.project.markerUpdater + + + + .settings/org.jamon.prefs + # now + eclipse.preferences.version=1 + templateSourceDir=src/main/jamon + templateOutputDir=target/generated-jamon + + + + + + + + + + + yyyy-MM-dd'T'HH:mm + + + ${maven.build.timestamp} + + 1.6 + + + 1.5.3 + 1.2 + 1.4 + 3.1 + 2.1 + 2.5 + 1.1.1 + 2.1 + 1.6 + 2.1.2 + 11.0.2 + 1.8.8 + 5.5.23 + 2.1 + 6.1.26 + 6.1.14 + 1.8 + 1.6.5 + 4.10-HBASE-1 + 1.4.3 + 1.2.16 + 1.8.5 + 2.4.0a + 1.0.1 + 0.8.0 + 3.4.5 + 0.0.1-SNAPSHOT + 2.6.3 + + /usr + /etc/hbase + /var/log/hbase + /var/run/hbase + 1 + + 0.91.0 + ${project.artifactId}-${project.version} + + + **/Test*.java + **/IntegrationTest*.java + + 2.12-TRUNK-HBASE-2 + surefire-junit47 + + + false + false + + once + classes + 1 + 2 + + org.apache.hadoop.hbase.SmallTests + org.apache.hadoop.hbase.MediumTests + true + 900 + + + + + + + + + + com.yammer.metrics + metrics-core + ${metrics-core.version} + + + com.google.guava + guava + ${guava.version} + + + commons-cli + commons-cli + ${commons-cli.version} + + + commons-configuration + commons-configuration + ${commons-configuration.version} + + + com.github.stephenc.high-scale-lib + high-scale-lib + 1.1.1 + + + commons-codec + commons-codec + ${commons-codec.version} + + + commons-httpclient + commons-httpclient + ${commons-httpclient.version} + + + commons-io + commons-io + ${commons-io.version} + + + commons-lang + commons-lang + ${commons-lang.version} + + + commons-logging + commons-logging + ${commons-logging.version} + + + log4j + log4j + ${log4j.version} + + + org.apache.avro + avro + ${avro.version} + + + com.thoughtworks.paranamer + paranamer + + + com.thoughtworks.paranamer + paranamer-ant + + + + + org.apache.avro + avro-ipc + ${avro.version} + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + jline + jline + + + + + org.apache.thrift + libthrift + ${thrift.version} + + + org.slf4j + slf4j-simple + + + + + org.jruby + jruby-complete + ${jruby.version} + + + org.mortbay.jetty + jetty + ${jetty.version} + + + org.mortbay.jetty + servlet-api + + + + + org.mortbay.jetty + jetty-util + ${jetty.version} + + + org.mortbay.jetty + jsp-2.1 + ${jetty.jspapi.version} + + + ant + ant + + + + + org.mortbay.jetty + jsp-api-2.1 + ${jetty.jspapi.version} + + + org.mortbay.jetty + servlet-api-2.5 + ${jetty.jspapi.version} + + + + org.codehaus.jackson + jackson-core-asl + ${jackson.version} + + + org.codehaus.jackson + jackson-mapper-asl + ${jackson.version} + + + org.codehaus.jackson + jackson-jaxrs + ${jackson.version} + + + org.codehaus.jackson + jackson-xc + ${jackson.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + + + + tomcat + jasper-compiler + ${jasper.version} + runtime + + + javax.servlet + jsp-api + + + javax.servlet + servlet-api + + + ant + ant + + + + + tomcat + jasper-runtime + ${jasper.version} + runtime + + + javax.servlet + servlet-api + + + + + + org.jamon + jamon-runtime + 2.3.1 + + + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + com.sun.jersey + jersey-core + ${jersey.version} + + + com.sun.jersey + jersey-json + ${jersey.version} + + + com.sun.jersey + jersey-server + ${jersey.version} + + + javax.xml.bind + jaxb-api + ${jaxb-api.version} + + + javax.xml.stream + stax-api + + + + + stax + stax-api + ${stax-api.version} + + + + + junit + junit + ${junit.version} + runtime + + true + + + org.mockito + mockito-all + ${mockito-all.version} + test + + + org.apache.commons + commons-math + ${commons-math.version} + test + + + + + + + rpm + + + + maven-antrun-plugin + 1.6 + + + build-rpm + package + + + + + + + + + + run + + + + + + maven-javadoc-plugin + 2.6.1 + + true + + + + prepare-package + + javadoc + + + + + + + + + deb + + + + maven-antrun-plugin + 1.6 + + + build-deb + package + + + + + + + + + + + run + + + + + + org.vafer + jdeb + 0.8 + + + + + + + + + os.linux + + false + + Linux + + + + ${os.name}-${os.arch}-${sun.arch.data.model} + + + + os.mac + + + Mac + + + + Mac_OS_X-${sun.arch.data.model} + + + + os.windows + + + Windows + + + + cygwin + + + + + + release + + + + org.apache.rat + apache-rat-plugin + + + package + + check + + + + + + maven-javadoc-plugin + 2.6.1 + + true + + + + prepare-package + + javadoc + + + + + + + + + + hadoop-snappy + + false + + snappy + + + + + org.apache.hadoop + hadoop-snappy + ${hadoop-snappy.version} + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + get-hadoop-snappy-native + generate-resources + + copy + + + + + org.apache.hadoop + hadoop-snappy + ${hadoop-snappy.version} + ${build.platform} + tar + false + ${project.build.directory}/nativelib + hadoop-snappy-nativelibs.tar + + + + + + + + + + + + + hadoop-1.0 + + + !hadoop.profile + + + + 1.0.4 + 1.4.3 + + + + org.apache.hadoop + hadoop-core + ${hadoop.version} + true + + + hsqldb + hsqldb + + + net.sf.kosmosfs + kfs + + + org.eclipse.jdt + core + + + net.java.dev.jets3t + jets3t + + + oro + oro + + + + + org.apache.hadoop + hadoop-test + ${hadoop.version} + true + test + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-test-resource + + add-test-resource + + + + + src/test/resources + + hbase-site.xml + + + + + + + + + + + + + + security + + 1.0.4 + + + ${project.artifactId}-${project.version}-security + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + + add-source + + + + ${project.basedir}/security/src/main/java + + + + + add-test-source + + add-test-source + + + + ${project.basedir}/security/src/test/java + + + + + add-test-resource + + add-test-resource + + + + + ${project.basedir}/security/src/test/resources + + hbase-site.xml + + + + + + + + + + + + + + + hadoop-1.1 + + + hadoop.profile + 1.1 + + + + 1.1.2 + 1.4.3 + + + + org.apache.hadoop + hadoop-core + ${hadoop.version} + true + + + hsqldb + hsqldb + + + net.sf.kosmosfs + kfs + + + org.eclipse.jdt + core + + + net.java.dev.jets3t + jets3t + + + oro + oro + + + + + org.apache.hadoop + hadoop-test + ${hadoop.version} + true + test + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-test-resource + + add-test-resource + + + + + src/test/resources + + hbase-site.xml + + + + + + + + + + + + + + hadoop-0.22 + + + hadoop.profile + 22 + + + + 0.22.0 + 1.6.1 + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + true + + + + hsqldb + hsqldb + + + net.sf.kosmosfs + kfs + + + org.eclipse.jdt + core + + + net.java.dev.jets3t + jets3t + + + oro + oro + + + jdiff + jdiff + + + org.apache.lucene + lucene-core + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + true + + + + hsqldb + hsqldb + + + net.sf.kosmosfs + kfs + + + org.eclipse.jdt + core + + + net.java.dev.jets3t + jets3t + + + oro + oro + + + jdiff + jdiff + + + org.apache.lucene + lucene-core + + + + + org.apache.hadoop + hadoop-mapred + ${hadoop.version} + true + + + + hsqldb + hsqldb + + + net.sf.kosmosfs + kfs + + + org.eclipse.jdt + core + + + net.java.dev.jets3t + jets3t + + + oro + oro + + + jdiff + jdiff + + + org.apache.lucene + lucene-core + + + + + + org.apache.hadoop + hadoop-common-test + ${hadoop.version} + true + test + + + org.apache.hadoop + hadoop-hdfs-test + ${hadoop.version} + true + test + + + org.apache.hadoop + hadoop-mapred-test + ${hadoop.version} + true + test + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-test-resource + + add-test-resource + + + + + src/test/resources + + hbase-site.xml + + + + + + + + + + + + + + hadoop-0.23 + + + hadoop.profile + 23 + + + + 0.23.3 + 1.6.1 + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + org.apache.hadoop + hadoop-annotations + ${hadoop.version} + + + + org.apache.hadoop + hadoop-minicluster + ${hadoop.version} + compile + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-test-resource + + add-test-resource + + + + + src/test/resources + + hbase-site.xml + + + + + + + + + maven-dependency-plugin + + + create-mrapp-generated-classpath + generate-test-resources + + build-classpath + + + + ${project.build.directory}/test-classes/mrapp-generated-classpath + + + + + + + + + + + hadoop-0.24 + + + hadoop.profile + 24 + + + + 0.24.0-SNAPSHOT + 1.6.1 + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-annotations + ${hadoop.version} + + + + org.apache.hadoop + hadoop-minicluster + ${hadoop.version} + compile + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-test-resource + + add-test-resource + + + + + src/test/resources + + hbase-site.xml + + + + + + + + + maven-dependency-plugin + + + create-mrapp-generated-classpath + generate-test-resources + + build-classpath + + + + ${project.build.directory}/test-classes/mrapp-generated-classpath + + + + + + + + + + + hadoop-2.0 + + + hadoop.profile + 2.0 + + + + 2.0.0-alpha + 1.6.1 + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-annotations + ${hadoop.version} + + + + org.apache.hadoop + hadoop-minicluster + ${hadoop.version} + compile + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-test-resource + + add-test-resource + + + + + src/test/resources + + hbase-site.xml + + + + + + + + + maven-dependency-plugin + + + create-mrapp-generated-classpath + generate-test-resources + + build-classpath + + + + ${project.build.directory}/test-classes/mrapp-generated-classpath + + + + + + + + + + + + nonParallelTests + + false + + + always + none + 1 + + + + + parallelTests + + false + + + once + classes + 1 + + + + + singleJVMTests + + false + + + once + none + 1 + + false + true + + + + + + runSmallTests + + false + + + once + none + 1 + + false + true + org.apache.hadoop.hbase.SmallTests + + + + + + runMediumTests + + false + + + always + false + true + org.apache.hadoop.hbase.MediumTests + + + + + + runLargeTests + + false + + + always + false + true + org.apache.hadoop.hbase.LargeTests + + + + + + runDevTests + + false + + + once + none + 1 + + false + false + org.apache.hadoop.hbase.SmallTests + org.apache.hadoop.hbase.MediumTests + + + + + runAllTests + + false + + + once + none + 1 + 4 + + false + false + org.apache.hadoop.hbase.SmallTests + org.apache.hadoop.hbase.MediumTests,org.apache.hadoop.hbase.LargeTests + + + + + skipSurefireTests + + false + + + true + true + + + + + localTests + + false + + + surefire-junit4 + 2.10 + + always + false + true + + + + + + + clover + + false + + clover + + + + ${user.home}/.clover.license + 2.6.3 + + + + + com.atlassian.maven.plugins + maven-clover2-plugin + ${clover.version} + + true + true + 50% + true + true + + **/generated/** + + + + + clover-setup + process-sources + + setup + + + + clover + site + + clover + + + + + + + + + + + + + + maven-project-info-reports-plugin + 2.6 + + + + project-team + mailing-list + cim + issue-tracking + license + scm + index + + + + + + + maven-javadoc-plugin + 2.6.1 + + true + + + + default + + javadoc + + + + + + + + org.apache.maven.plugins + maven-jxr-plugin + 2.1 + + + + org.apache.rat + apache-rat-plugin + 0.8 + + + org.apache.maven.plugins + maven-surefire-report-plugin + 2.7.2 + + + integration-tests + + report-only + + + failsafe-report + + ${project.build.directory}/failsafe-reports + + + + + + + + + + hbase.apache.org + HBase Website at hbase.apache.org + + file:///tmp + + + diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Column.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Column.java new file mode 100644 index 0000000..df20919 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Column.java @@ -0,0 +1,95 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.Serializable; + +import org.apache.hadoop.hbase.util.Bytes; + +public class Column implements Serializable { + private static final long serialVersionUID = -1958705310924323448L; + + private byte[] cf; + private byte[] qualifier; + private ValuePartition valuePartition = null; + + public Column() { + } + + public Column(byte[] cf, byte[] qualifier) { + this.cf = cf; + this.qualifier = qualifier; + } + + public Column(byte[] cf, byte[] qualifier, ValuePartition vp) { + this.cf = cf; + this.qualifier = qualifier; + this.valuePartition = vp; + } + + public void setFamily(byte[] cf) { + this.cf = cf; + } + + public void setQualifier(byte[] qualifier) { + this.qualifier = qualifier; + } + + public byte[] getFamily() { + return cf; + } + + public byte[] getQualifier() { + return qualifier; + } + + public ValuePartition getValuePartition() { + return this.valuePartition; + } + + public void setValuePartition(ValuePartition vp) { + this.valuePartition = vp; + } + + public boolean equals(Object obj) { + if (!(obj instanceof Column)) return false; + Column that = (Column) obj; + if (!(Bytes.equals(this.cf, that.cf))) return false; + if (!(Bytes.equals(this.qualifier, that.qualifier))) return false; + if (valuePartition == null && that.valuePartition == null) { + return true; + } else if (valuePartition != null && that.valuePartition != null) { + return valuePartition.equals(that.valuePartition); + } else { + return false; + } + } + + public int hashCode() { + int result = Bytes.hashCode(this.cf); + result ^= Bytes.hashCode(this.qualifier); + if (valuePartition != null) result ^= valuePartition.hashCode(); + return result; + } + + public String toString() { + return Bytes.toString(this.cf) + " : " + Bytes.toString(this.qualifier); + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ColumnQualifier.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ColumnQualifier.java new file mode 100644 index 0000000..a02bda4 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ColumnQualifier.java @@ -0,0 +1,212 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.index.ValuePartition.PartitionType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableComparable; + +/** + * A ColumnQualifier contains information about column family name,qualifier and maximum value + * length used in index specification. maValueLength is used to make sure the common pattern for all + * the rowkeys in the index table. ColumnQualifier is used as input for index specification to + * specify column family and column qualifier on which we are going to do indexing. + */ +public class ColumnQualifier implements WritableComparable { + + private byte[] cfBytes; + + private byte[] qualifierBytes; + + private int maxValueLength; + + private ValuePartition valuePartition = null; + + private ValueType type; + + public ColumnQualifier() { + // Dummy constructor which is needed for the readFields + } + + public ColumnQualifier(String cf, String qualifier) { + this(Bytes.toBytes(cf), Bytes.toBytes(qualifier)); + } + + public ColumnQualifier(byte[] cf, byte[] qualifier) { + this(cf, qualifier, ValueType.String, 0, null); + } + + public ColumnQualifier(String cf, String qualifier, ValueType type, int maxValueLength) { + this(Bytes.toBytes(cf), Bytes.toBytes(qualifier), type, maxValueLength, null); + } + + public ColumnQualifier(String cf, String qualifier, ValueType type, int maxValueLength, + ValuePartition vp) { + this(Bytes.toBytes(cf), Bytes.toBytes(qualifier), type, maxValueLength, vp); + } + + public ColumnQualifier(byte[] cf, byte[] qualifier, ValueType type, int maxValueLength, + ValuePartition vp) { + this.cfBytes = cf; + this.qualifierBytes = qualifier; + this.type = type; + this.maxValueLength = maxValueLength; + this.valuePartition = vp; + } + + /** + * @return Column Family as string + */ + public String getColumnFamilyString() { + return Bytes.toString(this.cfBytes); + } + + /** + * @return Column qualifier as string + */ + public String getQualifierString() { + return Bytes.toString(this.qualifierBytes); + } + + /** + * @return Column family as byte array + */ + public byte[] getColumnFamily() { + return this.cfBytes; + } + + /** + * @return Column qualifier as byte array + */ + public byte[] getQualifier() { + return this.qualifierBytes; + } + + public ValuePartition getValuePartition() { + return valuePartition; + } + + /** + * @param DataInput Stream + * @throws IOException + */ + public void readFields(DataInput in) throws IOException { + this.cfBytes = Bytes.readByteArray(in); + this.qualifierBytes = Bytes.readByteArray(in); + this.type = ValueType.valueOf(Bytes.toString(Bytes.readByteArray(in))); + this.maxValueLength = in.readInt(); + PartitionType p = PartitionType.valueOf(in.readUTF()); + if (p.equals(PartitionType.SEPARATOR)) { + valuePartition = new SeparatorPartition(); + } else if (p.equals(PartitionType.SPATIAL)) { + valuePartition = new SpatialPartition(); + } + if (valuePartition != null) { + valuePartition.readFields(in); + } + } + + /** + * @param DataOutput stream + * @throws IOException + */ + public void write(DataOutput out) throws IOException { + Bytes.writeByteArray(out, this.cfBytes); + Bytes.writeByteArray(out, this.qualifierBytes); + Bytes.writeByteArray(out, Bytes.toBytes(this.type.name())); + out.writeInt(maxValueLength); + if (valuePartition == null) { + out.writeUTF(PartitionType.NONE.name()); + } else { + out.writeUTF(valuePartition.getPartitionType().name()); + valuePartition.write(out); + } + } + + /** + * @param ColumnQualifier with whom to compare + * @return return true if both objects are equal otherwise false + */ + @Override + public boolean equals(Object cq) { + if (this == cq) { + return true; + } + if (false == (cq instanceof ColumnQualifier)) { + return false; + } + return this.compareTo((ColumnQualifier) cq) == 0; + } + + /** + * return hashcode of object + */ + public int hashCode() { + int result = Bytes.hashCode(this.cfBytes); + result ^= Bytes.hashCode(this.qualifierBytes); + result ^= this.maxValueLength; + if (valuePartition != null) result ^= valuePartition.hashCode(); + return result; + } + + /** + * @param IndexSpecification + * @return int + */ + @Override + public int compareTo(ColumnQualifier cq) { + int diff = 0; + diff = Bytes.compareTo(this.cfBytes, cq.cfBytes); + if (0 == diff) { + diff = Bytes.compareTo(this.qualifierBytes, cq.qualifierBytes); + if (0 == diff) { + if (valuePartition != null && cq.valuePartition != null) { + return valuePartition.compareTo(cq.valuePartition); + } else if (valuePartition == null && cq.valuePartition == null) { + return 0; + } else { + return 1; + } + } + } + return diff; + } + + public int getMaxValueLength() { + return this.maxValueLength; + } + + public ValueType getType() { + return this.type; + } + + public enum ValueType { + String, Int, Float, Long, Double, Short, Byte, Char + }; + + // TODO - Include valuePartition also into this + public String toString() { + return "CF : " + getColumnFamilyString() + ",Qualifier : " + getQualifierString(); + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Constants.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Constants.java new file mode 100644 index 0000000..a110520 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Constants.java @@ -0,0 +1,51 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Constants contains all constants used in indexing + */ +public class Constants { + + public static final int DEFAULT_NUM_RETRIES = 10; + + public static final long DEFAULT_PAUSE = 1000; + + public static final int DEFAULT_RETRY_LONGER_MULTIPLIER = 10; + + public static final byte[] IDX_COL_FAMILY = Bytes.toBytes("d"); + + public static final byte[] IDX_COL_QUAL = new byte[0]; + + public static final String INDEX_TABLE_SUFFIX = "_idx"; + + public static final int DEF_MAX_INDEX_NAME_LENGTH = 18; + + /** + * While scan the index(s) to be used can be explicitly passed from client application. Use this + * as the name to pass it in attributes + * @see Scan#setAttribute(String, byte[]) + */ + public static final String INDEX_EXPRESSION = "indexExpression"; + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/GroupingCondition.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/GroupingCondition.java new file mode 100644 index 0000000..32d20c8 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/GroupingCondition.java @@ -0,0 +1,24 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +public enum GroupingCondition { + AND, OR +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/IndexSpecification.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/IndexSpecification.java new file mode 100644 index 0000000..ead2713 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/IndexSpecification.java @@ -0,0 +1,344 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.EOFException; +import java.io.IOException; +import java.util.LinkedHashSet; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableComparable; +import org.mortbay.log.Log; + +/** + * An IndexSpecification should have a unique name and can be created on 1 or more columns (Here + * column refers to columnfamily + qualifier) For each of such column a ColumnQualifier is provided + * which takes the column details. This includes the column family name and qualifier name. The + * additional columns are those columns in the main table whose value also will be captured in the + * secondary index table. Index Specfication name should not start with '.' and '-'. Can contain + * alphanumerics and '-','.','_'. + */ + +public class IndexSpecification implements WritableComparable { + + private byte[] name; + + private Set indexColumns = new LinkedHashSet(1); + + private ColumnQualifier lastColumn = null; + + private int totalValueLength = 0; + + private long ttl = -1; + + private int maxVersions = -1; + + // Empty constructor for serialization and deserialization. + public IndexSpecification() { + } + + /** + * @param name should not start with '.' and '-'. Can contain alphanumerics and '-','.','_'. + * @throws IllegalArgumentException if invalid table name is provided + */ + public IndexSpecification(String name) { + validateIndexSpecification(Bytes.toBytes(name)); + this.name = Bytes.toBytes(name); + } + + private void validateIndexSpecification(byte[] indexSpecName) { + // throws IllegalArgException if invalid table name is provided + HTableDescriptor.isLegalTableName(indexSpecName); + } + + /** + * @param name should not start with '.' and '-'. Can contain alphanumerics and '-','.','_'. + * @throws IllegalArgumentException if invalid table name is provided + */ + public IndexSpecification(byte[] name) { + validateIndexSpecification(name); + this.name = name; + } + + /** + * @return index name + */ + public String getName() { + return Bytes.toString(this.name); + } + + /** + * @param cf column family + * @param qualifier + * @param type - If type is specified as null then by default ValueType will be taken as String. + * @param maxValueLength + * @throws IllegalArgumentException If column family name and/or qualifier is null or blanks.
+ * If column family name starts with '.',contains control characters or colons. + * @see ValueType + */ + public void addIndexColumn(HColumnDescriptor cf, String qualifier, ValueType type, + int maxValueLength) throws IllegalArgumentException { + type = checkForType(type); + isValidFamilyAndQualifier(cf, qualifier); + maxValueLength = getMaxLength(type, maxValueLength); + ColumnQualifier cq = new ColumnQualifier(cf.getNameAsString(), qualifier, type, maxValueLength); + isNotDuplicateEntry(cq); + formMinTTL(cf); + formMaxVersions(cf); + internalAdd(cq); + } + + private ValueType checkForType(ValueType type) { + if (type == null) { + type = ValueType.String; + } + return type; + } + + private int getMaxLength(ValueType type, int maxValueLength) { + if ((type == ValueType.Int || type == ValueType.Float) && maxValueLength != 4) { + Log.warn("With integer or float datatypes, the maxValueLength has to be 4 bytes"); + return 4; + } + if ((type == ValueType.Double || type == ValueType.Long) && maxValueLength != 8) { + Log.warn("With Double and Long datatypes, the maxValueLength has to be 8 bytes"); + return 8; + } + if ((type == ValueType.Short || type == ValueType.Char) && maxValueLength != 2) { + Log.warn("With Short and Char datatypes, the maxValueLength has to be 2 bytes"); + return 2; + } + if (type == ValueType.Byte && maxValueLength != 1) { + Log.warn("With Byte datatype, the maxValueLength has to be 1 bytes"); + return 1; + } + if (type == ValueType.String && maxValueLength == 0) { + Log.warn("With String datatype, the minimun value length is 2"); + maxValueLength = 2; + } + return maxValueLength; + } + + /** + * @param cf Column Family + * @param qualifier Column Qualifier + * @param ValuePortion vp + * @param type Data Type + * @param maxValueLength + * @throws IllegalArgumentException + */ + public void addIndexColumn(HColumnDescriptor cf, String qualifier, ValuePartition vp, + ValueType type, int maxValueLength) throws IllegalArgumentException { + checkForType(type); + isValidFamilyAndQualifier(cf, qualifier); + maxValueLength = getMaxLength(type, maxValueLength); + ColumnQualifier cq = + new ColumnQualifier(cf.getNameAsString(), qualifier, type, maxValueLength, vp); + isNotDuplicateEntry(cq); + formMinTTL(cf); + formMaxVersions(cf); + internalAdd(cq); + } + + private void formMinTTL(HColumnDescriptor cf) { + int timeToLive = cf.getTimeToLive(); + if (ttl == -1) { + ttl = timeToLive; + } else if (timeToLive != HConstants.FOREVER && timeToLive != -1) { + if (timeToLive < ttl) { + ttl = timeToLive; + } + } + } + + private void formMaxVersions(HColumnDescriptor cf) { + int maxVersion = cf.getMaxVersions(); + if (maxVersions == -1) { + maxVersions = maxVersion; + } else if (maxVersion != HConstants.FOREVER && maxVersion != -1) { + if (maxVersion < maxVersions) { + maxVersions = maxVersion; + } + } + } + + private void internalAdd(ColumnQualifier cq) { + indexColumns.add(cq); + lastColumn = cq; + totalValueLength += cq.getMaxValueLength(); + } + + /** + * @return List of column specifiers + */ + public Set getIndexColumns() { + return this.indexColumns; + } + + /** + * @param cf column family + * @param qualifier + * @throws IllegalArgumentException If column family name and/or qualifier is null or blanks + * @throws IllegalArgumentException If column family name starts with '.',contains control + * characters or colons + */ + private void isValidFamilyAndQualifier(HColumnDescriptor cf, String qualifier) { + if (null == cf || null == qualifier) { + throw new IllegalArgumentException("Column family/qualifier should not be null."); + } + if (StringUtils.isBlank(cf.getNameAsString()) || StringUtils.isBlank(qualifier)) { + throw new IllegalArgumentException("Column family/qualifier should not be blank."); + } + } + + /** + * @param ColumnQualifier to check duplicate entry + */ + private void isNotDuplicateEntry(ColumnQualifier c) { + if (this.getIndexColumns().contains(c)) { + throw new IllegalArgumentException("Duplicate column family and qualifier " + + "combination should not be present."); + } + } + + /** + * @param Data Input Stream + * @throws IOException + */ + public void readFields(DataInput in) throws IOException { + this.name = Bytes.readByteArray(in); + try { + HTableDescriptor.isLegalTableName(this.name); + } catch (IllegalArgumentException e) { + String msg = + "Received unexpected data while parsing the column qualifiers :" + + Bytes.toString(this.name) + "."; + Log.warn(msg + " Could be an non-indexed table."); + throw new EOFException(msg); + } + int indexColsSize = in.readInt(); + indexColumns.clear(); + for (int i = 0; i < indexColsSize; i++) { + ColumnQualifier cq = new ColumnQualifier(); + // Need to revisit this place. May be some other valid value though invalid + // comes up. + try { + cq.readFields(in); + } catch (IllegalArgumentException e) { + throw new EOFException("Received unexpected data while parsing the column qualifiers."); + } + internalAdd(cq); + } + this.maxVersions = in.readInt(); + this.ttl = in.readLong(); + } + + /** + * @param Data Output Stream + * @throws IOException + */ + public void write(DataOutput out) throws IOException { + Bytes.writeByteArray(out, this.name); + out.writeInt(this.indexColumns.size()); + for (ColumnQualifier cq : this.indexColumns) { + cq.write(out); + } + out.writeInt(maxVersions); + out.writeLong(ttl); + } + + /** + * @param IndexSpecification + * @return int + */ + public int compareTo(IndexSpecification o) { + return 0; + } + + public String toString() { + return "Index : " + getName() + ",Index Columns : " + indexColumns; + } + + public boolean equals(Object obj) { + if (obj instanceof IndexSpecification) { + IndexSpecification other = (IndexSpecification) obj; + return Bytes.equals(this.name, other.name); + } + return false; + } + + public int hashCode() { + return Bytes.hashCode(this.name); + } + + public ColumnQualifier getLastColumn() { + return this.lastColumn; + } + + public boolean contains(byte[] family) { + for (ColumnQualifier qual : indexColumns) { + if (Bytes.equals(family, qual.getColumnFamily())) { + return true; + } + } + return false; + } + + public boolean contains(byte[] family, byte[] qualifier) { + if (qualifier == null || qualifier.length == 0) { + return contains(family); + } + for (ColumnQualifier qual : indexColumns) { + if (Bytes.equals(family, qual.getColumnFamily()) + && Bytes.equals(qualifier, qual.getQualifier())) { + return true; + } + } + return false; + } + + public int getTotalValueLength() { + return totalValueLength; + } + + /** + * Return the minimum of the timeToLive specified for the column families in the specifed index + * @return + */ + public long getTTL() { + return this.ttl; + } + + /** + * Return the minimum of the maxVersion specified for the column families in the specified index + * @return + */ + public int getMaxVersions() { + return this.maxVersions; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/IndexedHTableDescriptor.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/IndexedHTableDescriptor.java new file mode 100644 index 0000000..1c32827 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/IndexedHTableDescriptor.java @@ -0,0 +1,128 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.EOFException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HTableDescriptor; + +/** + * IndexedHTabledDescriptor is extension of HTableDescriptor. This contains indices to specify index + * name and column details. There can be one or more indices on one table. For each of the index on + * the table and IndexSpecification is to be created and added to the indices. + */ +public class IndexedHTableDescriptor extends HTableDescriptor { + + private static final Log LOG = LogFactory.getLog(IndexedHTableDescriptor.class); + + private List indices = new ArrayList(1); + + public IndexedHTableDescriptor() { + + } + + public IndexedHTableDescriptor(String tableName) { + super(tableName); + } + + public IndexedHTableDescriptor(byte[] tableName) { + super(tableName); + } + + /** + * @param IndexSpecification to be added to indices + * @throws IllegalArgumentException if duplicate indexes for same table + */ + public void addIndex(IndexSpecification iSpec) throws IllegalArgumentException { + String indexName = iSpec.getName(); + if (null == indexName) { + String message = "Index name should not be null in Index Specification."; + LOG.error(message); + throw new IllegalArgumentException(message); + } + if (true == StringUtils.isBlank(indexName)) { + String message = "Index name should not be blank in Index Specification."; + LOG.error(message); + throw new IllegalArgumentException(message); + } + if (indexName.length() > Constants.DEF_MAX_INDEX_NAME_LENGTH) { + String message = + "Index name length should not more than " + Constants.DEF_MAX_INDEX_NAME_LENGTH + '.'; + LOG.error(message); + throw new IllegalArgumentException(message); + } + for (IndexSpecification is : indices) { + if (is.getName().equals(indexName)) { + String message = "Duplicate index names should not be present for same table."; + LOG.error(message); + throw new IllegalArgumentException(message); + } + } + indices.add(iSpec); + } + + /** + * @return IndexSpecification list + */ + public List getIndices() { + return (new ArrayList(this.indices)); + } + + /** + * @param DataOutput stream + */ + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeInt(this.indices.size()); + for (IndexSpecification index : indices) { + index.write(out); + } + } + + /** + * @param DataInput stream + * @throws IOException + */ + public void readFields(DataInput in) throws IOException { + try { + super.readFields(in); + int indicesSize = in.readInt(); + indices.clear(); + for (int i = 0; i < indicesSize; i++) { + IndexSpecification is = new IndexSpecification(); + is.readFields(in); + this.indices.add(is); + } + } catch (EOFException e) { + LOG.warn("Error reading feilds from the descriptor " + this.getNameAsString()); + throw e; + } + + } + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SecIndexLoadBalancer.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SecIndexLoadBalancer.java new file mode 100644 index 0000000..683f03f --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SecIndexLoadBalancer.java @@ -0,0 +1,627 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * This class is an extension of the load balancer class. It allows to colocate the regions of the + * actual table and the regions of the indexed table. roundRobinAssignment, retainAssignment -> + * index regions will follow the actual table regions. randomAssignment -> either index table or + * actual table region will follow each other based on which ever comes first. In case of master + * failover there is a chance that the znodes of the index table and actual table are left behind. + * Then in that scenario we may get randomAssignment for either the actual table region first or the + * index table region first. + */ + +public class SecIndexLoadBalancer implements LoadBalancer { + + private static final Log LOG = LogFactory.getLog(SecIndexLoadBalancer.class); + + private LoadBalancer delegator; + + private MasterServices master; + + private static final Random RANDOM = new Random(System.currentTimeMillis()); + + private Map> regionLocation = + new ConcurrentHashMap>(); + + @Override + public Configuration getConf() { + return this.delegator.getConf(); + } + + @Override + public void setConf(Configuration configuration) { + this.delegator.setConf(configuration); + } + + @Override + public void setClusterStatus(ClusterStatus st) { + this.delegator.setClusterStatus(st); + + } + + public Map> getRegionLocation() { + return regionLocation; + } + + @Override + public void setMasterServices(MasterServices masterServices) { + this.master = masterServices; + this.delegator.setMasterServices(masterServices); + } + + public void setDelegator(LoadBalancer defaultLoadBalancer) { + this.delegator = defaultLoadBalancer; + } + + @Override + public List balanceCluster(Map> clusterState) { + synchronized (this.regionLocation) { + Map> userClusterState = + new HashMap>(1); + Map> indexClusterState = + new HashMap>(1); + boolean balanceByTable = + this.master.getConfiguration().getBoolean("hbase.master.loadbalance.bytable", true); + + String tableName = null; + if (LOG.isDebugEnabled()) { + LOG.debug("Seperating user and index regions of each region server in the cluster."); + } + if (balanceByTable) { + // Check and modify the regionLocation map based on values of cluster state because we will + // call balancer only when the cluster is in stable state and reliable. + Map regionMap = null; + for (Entry> serverVsRegionList : clusterState.entrySet()) { + ServerName sn = serverVsRegionList.getKey(); + List regionInfos = serverVsRegionList.getValue(); + if (regionInfos.isEmpty()) { + continue; + } + // Just get the table name from any one of the values in the regioninfo list + if (null == tableName) { + tableName = regionInfos.get(0).getTableNameAsString(); + regionMap = this.regionLocation.get(tableName); + } + if (regionMap != null) { + for (HRegionInfo hri : regionInfos) { + updateServer(regionMap, sn, hri); + } + } + } + } else { + for (Entry> serverVsRegionList : clusterState.entrySet()) { + ServerName sn = serverVsRegionList.getKey(); + List regionsInfos = serverVsRegionList.getValue(); + List idxRegionsToBeMoved = new ArrayList(); + List uRegionsToBeMoved = new ArrayList(); + for (HRegionInfo hri : regionsInfos) { + if (hri.isMetaRegion() || hri.isRootRegion()) { + continue; + } + tableName = hri.getTableNameAsString(); + // table name may change every time thats why always need to get table entries. + Map regionMap = this.regionLocation.get(tableName); + if (regionMap != null) { + updateServer(regionMap, sn, hri); + } + if (tableName.endsWith(Constants.INDEX_TABLE_SUFFIX)) { + idxRegionsToBeMoved.add(hri); + continue; + } + uRegionsToBeMoved.add(hri); + + } + // there may be dummy entries here if assignments by table is set + userClusterState.put(sn, uRegionsToBeMoved); + indexClusterState.put(sn, idxRegionsToBeMoved); + } + } + /* + * In case of table wise balancing if balanceCluster called for index table then no user + * regions available. At that time skip default balancecluster call and get region plan from + * region location map if exist. + */ + // TODO : Needs refactoring here + List regionPlanList = null; + + if (balanceByTable && (false == tableName.endsWith(Constants.INDEX_TABLE_SUFFIX))) { + regionPlanList = this.delegator.balanceCluster(clusterState); + // regionPlanList is null means skipping balancing. + if (null == regionPlanList) { + if (LOG.isDebugEnabled()) { + LOG.debug("User region plan is null."); + } + return null; + } else { + saveRegionPlanList(regionPlanList); + return regionPlanList; + } + } else if (balanceByTable && (true == tableName.endsWith(Constants.INDEX_TABLE_SUFFIX))) { + regionPlanList = new ArrayList(1); + String actualTableName = extractActualTableName(tableName); + Map regionMap = regionLocation.get(actualTableName); + // no previous region plan for user table. + if (null == regionMap) { + if (LOG.isDebugEnabled()) { + LOG.debug("No user table region plans present for index table " + tableName + '.'); + } + return null; + } + for (Entry e : regionMap.entrySet()) { + regionPlanList.add(new RegionPlan(e.getKey(), null, e.getValue())); + } + // for preparing the index plan + List indexPlanList = new ArrayList(1); + // copy of region plan to iterate. + List regionPlanListCopy = new ArrayList(regionPlanList); + if (LOG.isDebugEnabled()) { + LOG.debug("Preparing index region plans from user region plans for table " + tableName + + "."); + } + return prepareIndexPlan(clusterState, indexPlanList, regionPlanListCopy); + } else { + regionPlanList = this.delegator.balanceCluster(userClusterState); + if (null == regionPlanList) { + if (LOG.isDebugEnabled()) { + LOG.debug("User region plan is null."); + } + regionPlanList = new ArrayList(1); + } else { + saveRegionPlanList(regionPlanList); + } + List userRegionPlans = new ArrayList(1); + + for (Entry> tableVsRegions : this.regionLocation + .entrySet()) { + Map regionMap = regionLocation.get(tableVsRegions.getKey()); + // no previous region plan for user table. + if (null == regionMap) { + if (LOG.isDebugEnabled()) { + LOG.debug("No user table region plans present for index table " + tableName + '.'); + } + } else { + for (Entry e : regionMap.entrySet()) { + userRegionPlans.add(new RegionPlan(e.getKey(), null, e.getValue())); + } + } + } + List regionPlanListCopy = new ArrayList(userRegionPlans); + if (LOG.isDebugEnabled()) { + LOG.debug("Preparing index region plans from user region plans for whole cluster."); + } + return prepareIndexPlan(indexClusterState, regionPlanList, regionPlanListCopy); + } + } + } + + private void updateServer(Map regionMap, ServerName sn, HRegionInfo hri) { + ServerName existingServer = regionMap.get(hri); + if (!sn.equals(existingServer)) { + if (LOG.isDebugEnabled()) { + LOG.debug("There is a mismatch in the existing server name for the region " + hri + + ". Replacing the server " + existingServer + " with " + sn + "."); + } + regionMap.put(hri, sn); + } + } + + // Creates the index region plan based on the corresponding user region plan + private List prepareIndexPlan(Map> indexClusterState, + List regionPlanList, List regionPlanListCopy) { + if (LOG.isDebugEnabled()) { + LOG.debug("Entered prepareIndexPlan"); + } + OUTER_LOOP: for (RegionPlan regionPlan : regionPlanListCopy) { + HRegionInfo hri = regionPlan.getRegionInfo(); + + MIDDLE_LOOP: for (Entry> serverVsRegionList : indexClusterState + .entrySet()) { + List indexRegions = serverVsRegionList.getValue(); + ServerName server = serverVsRegionList.getKey(); + if (regionPlan.getDestination().equals(server)) { + // desination server in the region plan is new and should not be same with this + // server in index cluster state.thats why skipping regions check in this server + continue MIDDLE_LOOP; + } + String actualTableName = null; + + for (HRegionInfo indexRegionInfo : indexRegions) { + String indexTableName = indexRegionInfo.getTableNameAsString(); + actualTableName = extractActualTableName(indexTableName); + if (false == hri.getTableNameAsString().equals(actualTableName)) { + continue; + } + if (0 != Bytes.compareTo(hri.getStartKey(), indexRegionInfo.getStartKey())) { + continue; + } + RegionPlan rp = new RegionPlan(indexRegionInfo, server, regionPlan.getDestination()); + if (LOG.isDebugEnabled()) { + LOG.debug("Selected server " + regionPlan.getDestination() + + " as destination for region " + indexRegionInfo.getRegionNameAsString() + + "from user region plan."); + } + + putRegionPlan(indexRegionInfo, regionPlan.getDestination()); + regionPlanList.add(rp); + continue OUTER_LOOP; + } + } + } + regionPlanListCopy.clear(); + // if no user regions to balance then return newly formed index region plan. + if (LOG.isDebugEnabled()) { + LOG.debug("Exited prepareIndexPlan"); + } + return regionPlanList; + } + + private void saveRegionPlanList(List regionPlanList) { + for (RegionPlan regionPlan : regionPlanList) { + HRegionInfo hri = regionPlan.getRegionInfo(); + if (LOG.isDebugEnabled()) { + LOG.debug("Saving region plan of region " + hri.getRegionNameAsString() + '.'); + } + putRegionPlan(hri, regionPlan.getDestination()); + } + } + + @Override + public Map> roundRobinAssignment(List regions, + List servers) { + List userRegions = new ArrayList(1); + List indexRegions = new ArrayList(1); + for (HRegionInfo hri : regions) { + seperateUserAndIndexRegion(hri, userRegions, indexRegions); + } + Map> bulkPlan = null; + if (false == userRegions.isEmpty()) { + bulkPlan = this.delegator.roundRobinAssignment(userRegions, servers); + if (null == bulkPlan) { + if (LOG.isDebugEnabled()) { + LOG.debug("No region plan for user regions."); + } + return null; + } + synchronized (this.regionLocation) { + savePlan(bulkPlan); + } + } + bulkPlan = prepareIndexRegionPlan(indexRegions, bulkPlan, servers); + return bulkPlan; + } + + private void seperateUserAndIndexRegion(HRegionInfo hri, List userRegions, + List indexRegions) { + if (hri.getTableNameAsString().endsWith(Constants.INDEX_TABLE_SUFFIX)) { + indexRegions.add(hri); + return; + } + userRegions.add(hri); + } + + private String extractActualTableName(String indexTableName) { + int endIndex = indexTableName.length() - Constants.INDEX_TABLE_SUFFIX.length(); + return indexTableName.substring(0, endIndex); + } + + private Map> prepareIndexRegionPlan(List indexRegions, + Map> bulkPlan, List servers) { + if (null != indexRegions && false == indexRegions.isEmpty()) { + if (null == bulkPlan) { + bulkPlan = new ConcurrentHashMap>(1); + } + for (HRegionInfo hri : indexRegions) { + if (LOG.isDebugEnabled()) { + LOG.debug("Preparing region plan for index region " + hri.getRegionNameAsString() + '.'); + } + ServerName destServer = getDestServerForIdxRegion(hri); + List destServerRegions = null; + if (null == destServer) { + destServer = this.randomAssignment(hri, servers); + } + if (null != destServer) { + destServerRegions = bulkPlan.get(destServer); + if (null == destServerRegions) { + destServerRegions = new ArrayList(1); + bulkPlan.put(destServer, destServerRegions); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Server " + destServer + " selected for region " + + hri.getRegionNameAsString() + '.'); + } + destServerRegions.add(hri); + } + } + } + return bulkPlan; + } + + private ServerName getDestServerForIdxRegion(HRegionInfo hri) { + // Every time we calculate the table name because in case of master restart the index regions + // may be coming for different index tables. + String indexTableName = hri.getTableNameAsString(); + String actualTableName = extractActualTableName(indexTableName); + synchronized (this.regionLocation) { + Map regionMap = regionLocation.get(actualTableName); + if (null == regionMap) { + // Can this case come + return null; + } + for (Map.Entry e : regionMap.entrySet()) { + HRegionInfo uHri = e.getKey(); + if (0 == Bytes.compareTo(uHri.getStartKey(), hri.getStartKey())) { + // put index region location if corresponding user region found in regionLocation map. + putRegionPlan(hri, e.getValue()); + return e.getValue(); + } + } + } + return null; + } + + private void savePlan(Map> bulkPlan) { + for (Entry> e : bulkPlan.entrySet()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Saving user regions' plans for server " + e.getKey() + '.'); + } + for (HRegionInfo hri : e.getValue()) { + putRegionPlan(hri, e.getKey()); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Saved user regions' plans for server " + e.getKey() + '.'); + } + } + } + + @Override + public Map> retainAssignment(Map regions, + List servers) { + + Map userRegionsMap = new ConcurrentHashMap(1); + List indexRegions = new ArrayList(1); + for (Entry e : regions.entrySet()) { + seperateUserAndIndexRegion(e, userRegionsMap, indexRegions, servers); + } + Map> bulkPlan = null; + if (false == userRegionsMap.isEmpty()) { + bulkPlan = this.delegator.retainAssignment(userRegionsMap, servers); + if (null == bulkPlan) { + if (LOG.isDebugEnabled()) { + LOG.debug("Empty region plan for user regions."); + } + return null; + } + synchronized (this.regionLocation) { + savePlan(bulkPlan); + } + } + bulkPlan = prepareIndexRegionPlan(indexRegions, bulkPlan, servers); + return bulkPlan; + } + + private void seperateUserAndIndexRegion(Entry e, + Map userRegionsMap, List indexRegions, + List servers) { + HRegionInfo hri = e.getKey(); + if (hri.getTableNameAsString().endsWith(Constants.INDEX_TABLE_SUFFIX)) { + indexRegions.add(hri); + return; + } + if (e.getValue() == null) { + userRegionsMap.put(hri, servers.get(RANDOM.nextInt(servers.size()))); + } else { + userRegionsMap.put(hri, e.getValue()); + } + } + + @Override + public Map immediateAssignment(List regions, + List servers) { + return this.delegator.immediateAssignment(regions, servers); + } + + @Override + public ServerName randomAssignment(HRegionInfo regionInfo, List servers) { + if (regionInfo.isMetaTable()) { + // if the region is root or meta table region no need to check for any region plan. + return this.delegator.randomAssignment(regionInfo, servers); + } + ServerName sn = null; + try { + sn = getServerNameFromMap(regionInfo, servers); + } catch (IOException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Not able to get server name.", e); + } + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while getting region and location details.", e); + } + } + if (sn == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("No server found for region " + regionInfo.getRegionNameAsString() + '.'); + } + sn = getRandomServer(regionInfo, servers); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Destination server for region " + regionInfo.getRegionNameAsString() + " is " + + ((sn == null) ? "null" : sn.toString()) + '.'); + } + return sn; + } + + private ServerName getRandomServer(HRegionInfo regionInfo, List servers) { + ServerName sn = null; + String tableName = regionInfo.getTableNameAsString(); + if (true == IndexUtils.isIndexTable(tableName)) { + String actualTableName = extractActualTableName(tableName); + sn = + this.delegator.randomAssignment(new HRegionInfo(Bytes.toBytes(actualTableName), + regionInfo.getStartKey(), regionInfo.getEndKey()), servers); + } else { + sn = this.delegator.randomAssignment(regionInfo, servers); + } + if (sn == null) { + return null; + } + synchronized (this.regionLocation) { + putRegionPlan(regionInfo, sn); + } + return sn; + } + + private ServerName getServerNameFromMap(HRegionInfo regionInfo, List onlineServers) + throws IOException, InterruptedException { + String tableNameOfCurrentRegion = regionInfo.getTableNameAsString(); + String correspondingTableName = null; + if (false == tableNameOfCurrentRegion.endsWith(Constants.INDEX_TABLE_SUFFIX)) { + // if the region is user region need to check whether index region plan available or not. + correspondingTableName = tableNameOfCurrentRegion + Constants.INDEX_TABLE_SUFFIX; + } else { + // if the region is index region need to check whether user region plan available or not. + correspondingTableName = extractActualTableName(tableNameOfCurrentRegion); + } + synchronized (this.regionLocation) { + + // skip if its in both index and user and same server + // I will always have the regionMapWithServerLocation for the correspondingTableName already + // populated. + // Only on the first time both the regionMapWithServerLocation and actualRegionMap may be + // null. + Map regionMapWithServerLocation = + this.regionLocation.get(correspondingTableName); + Map actualRegionMap = + this.regionLocation.get(tableNameOfCurrentRegion); + + if (null != regionMapWithServerLocation) { + for (Entry iHri : regionMapWithServerLocation.entrySet()) { + if (0 == Bytes.compareTo(iHri.getKey().getStartKey(), regionInfo.getStartKey())) { + ServerName previousServer = null; + if (null != actualRegionMap) { + previousServer = actualRegionMap.get(regionInfo); + } + ServerName sn = iHri.getValue(); + if (null != previousServer) { + // if servername of index region and user region are same in regionLocation clean + // previous plans and return null + if (previousServer.equals(sn)) { + regionMapWithServerLocation.remove(iHri.getKey()); + actualRegionMap.remove(regionInfo); + if (LOG.isDebugEnabled()) { + LOG.debug("Both user region plan and index region plan " + + "in regionLocation are same for the region." + + regionInfo.getRegionNameAsString() + " The location is " + sn + + ". Hence clearing from regionLocation."); + } + return null; + } + } + if (sn != null && onlineServers.contains(sn)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Updating the region " + regionInfo.getRegionNameAsString() + + " with server " + sn); + } + putRegionPlan(regionInfo, sn); + return sn; + } else if (sn != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("The location " + sn + " of region " + + iHri.getKey().getRegionNameAsString() + + " is not in online. Selecting other region server."); + } + return null; + } + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No region plans in regionLocation for table " + correspondingTableName); + } + } + return null; + } + } + + public void putRegionPlan(HRegionInfo regionInfo, ServerName sn) { + String tableName = regionInfo.getTableNameAsString(); + synchronized (this.regionLocation) { + Map regionMap = this.regionLocation.get(tableName); + if (null == regionMap) { + if (LOG.isDebugEnabled()) { + LOG.debug("No regions of table " + tableName + " in the region plan."); + } + regionMap = new ConcurrentHashMap(1); + this.regionLocation.put(tableName, regionMap); + } + regionMap.put(regionInfo, sn); + } + } + + public void clearTableRegionPlans(String tableName) { + if (LOG.isDebugEnabled()) { + LOG.debug("Clearing regions plans from regionLocation for table " + tableName); + } + synchronized (this.regionLocation) { + this.regionLocation.remove(tableName); + } + } + + public void clearRegionInfoFromRegionPlan(HRegionInfo regionInfo) { + String tableName = regionInfo.getTableNameAsString(); + synchronized (this.regionLocation) { + Map regionMap = this.regionLocation.get(tableName); + if (null == regionMap) { + if (LOG.isDebugEnabled()) { + LOG.debug("No regions of table " + tableName + " in the region plan."); + } + } else { + regionMap.remove(regionInfo); + if (LOG.isDebugEnabled()) { + LOG.debug("The regioninfo " + regionInfo + " removed from the region plan"); + } + } + } + } + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SeparatorPartition.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SeparatorPartition.java new file mode 100644 index 0000000..151fd80 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SeparatorPartition.java @@ -0,0 +1,166 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.util.Bytes; + +/** + * A column value is composed of many values separated using some known separator. Part of the + * column value to be indexed. This class specified how to get that value part. Takes the separator + * so as to split the value and the value position in the split. Note that the position index starts + * from '1' + */ +public class SeparatorPartition implements ValuePartition { + + private static final long serialVersionUID = -3409814164480687975L; + + private byte[] separator; + + private int position; + + public SeparatorPartition() { + + } + + public SeparatorPartition(String separator, int position) { + if ((null == separator || separator.length() == 0)) { + throw new IllegalArgumentException("Separator cannot be null"); + } + if ((null != separator) && position == 0) { + throw new IllegalArgumentException("With separator ,the position cannot be zero."); + } + this.separator = Bytes.toBytes(separator); + this.position = position; + } + + public SeparatorPartition(byte[] separator, int position) { + this.separator = separator; + this.position = position; + } + + @Override + public PartitionType getPartitionType() { + return PartitionType.SEPARATOR; + } + + public byte[] getSeparator() { + return this.separator; + } + + public int getPosition() { + return this.position; + } + + @Override + public byte[] getPartOfValue(byte[] value) { + // TODO check this method.. Seems so much of code! + int sepLastKnownPosition = -1; + int sepCurrPositon = -1; + int separatorOccurences = 0; + byte[] kvSubset = new byte[separator.length]; + for (int i = 0; i < value.length;) { + if ((value.length - i) >= separator.length) { + System.arraycopy(value, i, kvSubset, 0, separator.length); + if (Bytes.equals(kvSubset, separator)) { + separatorOccurences++; + sepLastKnownPosition = sepCurrPositon; + sepCurrPositon = i; + i += separator.length; + } else { + i++; + } + if (separatorOccurences < this.position) { + continue; + } + break; + } + break; + } + if (separatorOccurences < this.position - 1) { + return new byte[0]; + } + byte valuePart[] = null; + if (separatorOccurences == this.position - 1) { + if (sepCurrPositon == -1) { + valuePart = value; + } else { + valuePart = new byte[value.length - sepCurrPositon - separator.length]; + System.arraycopy(value, sepCurrPositon + separator.length, valuePart, 0, valuePart.length); + } + return valuePart; + } else if (separatorOccurences == this.position) { + if (sepLastKnownPosition == -1) { + valuePart = new byte[sepCurrPositon]; + System.arraycopy(value, 0, valuePart, 0, valuePart.length); + } else { + valuePart = new byte[sepCurrPositon - sepLastKnownPosition - separator.length]; + System.arraycopy(value, sepLastKnownPosition + separator.length, valuePart, 0, + valuePart.length); + } + return valuePart; + } + return valuePart; + } + + @Override + public void write(DataOutput out) throws IOException { + Bytes.writeByteArray(out, this.separator); + out.writeInt(this.position); + } + + @Override + public void readFields(DataInput in) throws IOException { + this.separator = Bytes.readByteArray(in); + this.position = in.readInt(); + } + + @Override + public int compareTo(ValuePartition vp) { + if (!(vp instanceof SeparatorPartition)) return 1; + SeparatorPartition sp = (SeparatorPartition) vp; + int diff = Bytes.compareTo(this.separator, sp.separator); + if (diff == 0) return this.position - sp.position; + return diff; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } else if (that instanceof SeparatorPartition) { + SeparatorPartition sp = (SeparatorPartition) that; + return Bytes.compareTo(this.separator, sp.getSeparator()) == 0 + && this.position == sp.getPosition(); + } + return false; + } + + @Override + public int hashCode() { + int result = 13; + result ^= Bytes.hashCode(this.separator); + result ^= this.position; + return result; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SpatialPartition.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SpatialPartition.java new file mode 100644 index 0000000..b2adcb7 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/SpatialPartition.java @@ -0,0 +1,114 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * Column value is composed of different values. The value to be indexed is in a known offset + */ +public class SpatialPartition implements ValuePartition { + + private static final long serialVersionUID = 4154246616417056121L; + + private int offset; + + private int length; + + public SpatialPartition() { + + } + + public SpatialPartition(int offset, int length) { + if (offset < 0 || length < 1) { + throw new IllegalArgumentException("offset/length cannot be les than 1"); + } + this.offset = offset; + this.length = length; + } + + public int getOffset() { + return offset; + } + + public int getLength() { + return length; + } + + @Override + public PartitionType getPartitionType() { + return PartitionType.SPATIAL; + } + + @Override + public byte[] getPartOfValue(byte[] value) { + if (this.offset >= value.length) { + return new byte[0]; + } else { + int valueLength = + (this.offset + this.length > value.length) ? value.length - this.offset : this.length; + byte[] valuePart = new byte[valueLength]; + System.arraycopy(value, this.offset, valuePart, 0, valueLength); + return valuePart; + } + } + + @Override + public int compareTo(ValuePartition vp) { + if (!(vp instanceof SpatialPartition)) return 1; + SpatialPartition sp = (SpatialPartition) vp; + int diff = this.offset - sp.offset; + if (diff == 0) return this.length - sp.length; + return diff; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } else if (that instanceof SpatialPartition) { + SpatialPartition sp = (SpatialPartition) that; + return this.offset == sp.getOffset() && this.length == sp.getLength(); + } + return false; + } + + @Override + public int hashCode() { + int result = 13; + result ^= this.offset; + result ^= this.length; + return result; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(this.offset); + out.writeInt(this.length); + } + + @Override + public void readFields(DataInput in) throws IOException { + this.offset = in.readInt(); + this.length = in.readInt(); + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ValuePartition.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ValuePartition.java new file mode 100644 index 0000000..d414c61 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ValuePartition.java @@ -0,0 +1,38 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.Serializable; + +import org.apache.hadoop.io.WritableComparable; + +/** + * Used to specify the part of a column which needs to be indexed. + */ +public interface ValuePartition extends WritableComparable, Serializable { + + public enum PartitionType { + SEPARATOR, SPATIAL, NONE + } + + public PartitionType getPartitionType(); + + public byte[] getPartOfValue(byte[] value); +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/EqualsExpression.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/EqualsExpression.java new file mode 100644 index 0000000..59a89a8 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/EqualsExpression.java @@ -0,0 +1,53 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.io.Serializable; + +import org.apache.hadoop.hbase.index.Column; + +/** + * Can be used to specify an equals condition on a column associated with an index. + */ +public class EqualsExpression implements Serializable { + + private static final long serialVersionUID = -7130697408286943018L; + + private Column column; + private byte[] value; + + public EqualsExpression(Column column, byte[] value) { + this.column = column; + this.value = value; + } + + public Column getColumn() { + return this.column; + } + + public byte[] getValue() { + return value; + } + + @Override + public String toString() { + return "EqualsExpression : Column[" + this.column + ']'; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexExpression.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexExpression.java new file mode 100644 index 0000000..8acf703 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexExpression.java @@ -0,0 +1,39 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.io.Serializable; + +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.Constants; + +/** + * While scan the index(s) to be used can be explicitly passed from client application. Objects of + * IndexExpression can specify the index(s) details. + * @see SingleIndexExpression + * @see MultiIndexExpression + * @see NoIndexExpression + * @see Scan#setAttribute(String, byte[]) + * @see Constants#INDEX_EXPRESSION + * @see IndexUtils#toBytes(IndexExpression) + */ +public interface IndexExpression extends Serializable { + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexUtils.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexUtils.java new file mode 100644 index 0000000..4c4a6ad --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexUtils.java @@ -0,0 +1,67 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +import org.apache.hadoop.hbase.client.Scan; + +/** + * Client side utility class for using Secondary Index. + */ +public class IndexUtils { + + private IndexUtils() { + // Avoid instantiation of this class. + } + + /** + * Utility to convert IndexExpression into byte[]. Can be used to pass the IndexExpression in the + * Scan attributes. + * @see Scan#setAttribute(String, byte[]) + * @param indexExpression + * @return byte[] + * @throws IOException + */ + public static byte[] toBytes(IndexExpression indexExpression) throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(bos); + oos.writeObject(indexExpression); + return bos.toByteArray(); + } + + /** + * Creates back IndexExpression from byte[] + * @param bytes + * @return + * @throws IOException + * @throws ClassNotFoundException + */ + public static IndexExpression toIndexExpression(byte[] bytes) throws IOException, + ClassNotFoundException { + ByteArrayInputStream bis = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bis); + return (IndexExpression) ois.readObject(); + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/MultiIndexExpression.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/MultiIndexExpression.java new file mode 100644 index 0000000..ce16a20 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/MultiIndexExpression.java @@ -0,0 +1,58 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.GroupingCondition; + +/** + * Can be used to group a set of {@link SingleIndexExpression} with AND/OR {@link GroupingCondition} + * @see Scan#setAttribute(String, byte[]) + * @see Constants#INDEX_EXPRESSION + * @see IndexUtils#toBytes(IndexExpression) + */ +public class MultiIndexExpression implements IndexExpression { + + private static final long serialVersionUID = 5322668904124942100L; + + private List indexExpressions = new ArrayList(); + + private GroupingCondition groupingCondition; + + public MultiIndexExpression(GroupingCondition condition) { + this.groupingCondition = condition; + } + + public GroupingCondition getGroupingCondition() { + return this.groupingCondition; + } + + public void addIndexExpression(IndexExpression indexExpression) { + this.indexExpressions.add(indexExpression); + } + + public List getIndexExpressions() { + return this.indexExpressions; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/NoIndexExpression.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/NoIndexExpression.java new file mode 100644 index 0000000..98ba8b9 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/NoIndexExpression.java @@ -0,0 +1,36 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.Constants; + +/** + * Pass this as the index expression via Scan attribute when a scan query should not use any index + * which is possible to be used + * @see Scan#setAttribute(String, byte[]) + * @see Constants#INDEX_EXPRESSION + * @see IndexUtils#toBytes(IndexExpression) + */ +public class NoIndexExpression implements IndexExpression { + + private static final long serialVersionUID = 5445463806972787598L; + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/RangeExpression.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/RangeExpression.java new file mode 100644 index 0000000..d98ec18 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/RangeExpression.java @@ -0,0 +1,87 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.io.Serializable; + +import org.apache.hadoop.hbase.index.Column; + +/** + * Can be used to specify a range condition on a column associated with an index. When the range is + * non closed at one end (to specific upper bound but only lower bound) pass the corresponding bound + * value as null. + */ +public class RangeExpression implements Serializable { + + private static final long serialVersionUID = 8772267632040419734L; + + private Column column; + private byte[] lowerBoundValue; + private byte[] upperBoundValue; + private boolean lowerBoundInclusive; + private boolean upperBoundInclusive; + + public Column getColumn() { + return column; + } + + public byte[] getLowerBoundValue() { + return lowerBoundValue; + } + + public byte[] getUpperBoundValue() { + return upperBoundValue; + } + + public boolean isLowerBoundInclusive() { + return lowerBoundInclusive; + } + + public boolean isUpperBoundInclusive() { + return upperBoundInclusive; + } + + /** + * When the range is non closed at one end (to specific upper bound but only lower bound) pass the + * corresponding bound value as null. + * @param column + * @param lowerBoundValue + * @param upperBoundValue + * @param lowerBoundInclusive + * @param upperBoundInclusive + */ + public RangeExpression(Column column, byte[] lowerBoundValue, byte[] upperBoundValue, + boolean lowerBoundInclusive, boolean upperBoundInclusive) { + if (column == null || (lowerBoundValue == null && upperBoundValue == null)) { + throw new IllegalArgumentException(); + } + this.column = column; + this.lowerBoundValue = lowerBoundValue; + this.upperBoundValue = upperBoundValue; + this.lowerBoundInclusive = lowerBoundInclusive; + this.upperBoundInclusive = upperBoundInclusive; + } + + @Override + public String toString() { + return "RangeExpression : Column[" + this.column + "], lowerBoundInclusive : " + + this.lowerBoundInclusive + ", upperBoundInclusive : " + this.upperBoundInclusive; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/SingleIndexExpression.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/SingleIndexExpression.java new file mode 100644 index 0000000..97f9c4e --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/SingleIndexExpression.java @@ -0,0 +1,78 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.Constants; + +/** + * This can specify one index details to be used. The index can be single column or multi column + * index. This index can be used for equals or range condition on columns. For a multi column index, + * there can be 0 or more equals condition associated with this index in the scan and at max one + * range condition (on a column). For an index the columns are in an order. There can not be any + * equals condition specified on columns coming after the range conditioned column as per the + * columns order in the index specification. + * @see Scan#setAttribute(String, byte[]) + * @see Constants#INDEX_EXPRESSION + * @see IndexUtils#toBytes(IndexExpression) + */ +public class SingleIndexExpression implements IndexExpression { + + private static final long serialVersionUID = 893160134306193043L; + + private String indexName; + + private List equalsExpressions = new ArrayList(); + + private RangeExpression rangeExpression; + + public SingleIndexExpression(String indexName) { + this.indexName = indexName; + } + + public String getIndexName() { + return indexName; + } + + /** + * This is expected to be called in the order of columns specified in the Index. If index is on + * columns cf1:c1, cf1:c2 and cf2:c3 when creating the SingleIndexExpression call this method in + * the same order as of above + * @param equalsExpression + */ + public void addEqualsExpression(EqualsExpression equalsExpression) { + this.equalsExpressions.add(equalsExpression); + } + + public List getEqualsExpressions() { + return this.equalsExpressions; + } + + public void setRangeExpression(RangeExpression rangeExpression) { + this.rangeExpression = rangeExpression; + } + + public RangeExpression getRangeExpression() { + return this.rangeExpression; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/master/IndexMasterObserver.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/master/IndexMasterObserver.java new file mode 100644 index 0000000..509dd78 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/master/IndexMasterObserver.java @@ -0,0 +1,740 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.master; + +import java.io.EOFException; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableDescriptors; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.MasterObserverExt; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.ColumnQualifier; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.SecIndexLoadBalancer; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.master.handler.CreateTableHandler; +import org.apache.hadoop.hbase.master.handler.DeleteTableHandler; +import org.apache.hadoop.hbase.master.handler.DisableTableHandler; +import org.apache.hadoop.hbase.master.handler.EnableTableHandler; +import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZKTable; + +/** + * Defines of coprocessor hooks(to support secondary indexing) of operations on + * {@link org.apache.hadoop.hbase.master.HMaster} process. + */ + +public class IndexMasterObserver extends BaseMasterObserver implements MasterObserverExt { + + private static final Log LOG = LogFactory.getLog(IndexMasterObserver.class.getName()); + + /* + * (non-Javadoc) + * @see org.apache.hadoop.hbase.coprocessor.BaseMasterObserver#preCreateTable(org + * .apache.hadoop.hbase.coprocessor.ObserverContext, org.apache.hadoop.hbase.HTableDescriptor, + * org.apache.hadoop.hbase.HRegionInfo[]) + */ + @Override + public void preCreateTable(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + LOG.info("Entered into preCreateTable."); + MasterServices master = ctx.getEnvironment().getMasterServices(); + if (desc instanceof IndexedHTableDescriptor) { + Map> indexColDetails = + new HashMap>(); + String tableName = desc.getNameAsString(); + checkEndsWithIndexSuffix(tableName); + String indexTableName = IndexUtils.getIndexTableName(tableName); + List indices = ((IndexedHTableDescriptor) desc).getIndices(); + // Even if indices list is empty,it will create index table also. + if (indices.isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Empty indices. Index table may not created" + + " if master goes down in between user table creation"); + } + } + LOG.trace("Checking whether column families in " + + "index specification are in actual table column familes."); + for (IndexSpecification iSpec : indices) { + checkColumnsForValidityAndConsistency(desc, iSpec, indexColDetails); + } + LOG.trace("Column families in index specifications " + "are in actual table column familes."); + + boolean isTableExists = MetaReader.tableExists(master.getCatalogTracker(), tableName); + boolean isIndexTableExists = + MetaReader.tableExists(master.getCatalogTracker(), indexTableName); + + if (isTableExists && isIndexTableExists) { + throw new TableExistsException("Table " + tableName + " already exist."); + } else if (isIndexTableExists) { + disableAndDeleteTable(master, indexTableName); + } + } + LOG.info("Exiting from preCreateTable."); + } + + @Override + public void postModifyTableHandler(ObserverContext ctx, + byte[] tableName, HTableDescriptor htd) throws IOException { + String table = Bytes.toString(tableName); + MasterServices master = ctx.getEnvironment().getMasterServices(); + List> tableRegionsAndLocations = null; + LOG.info("Entering postModifyTable for the table " + table); + if (htd instanceof IndexedHTableDescriptor) { + TableDescriptors tableDescriptors = master.getTableDescriptors(); + Map allTableDesc = tableDescriptors.getAll(); + String indexTableName = IndexUtils.getIndexTableName(tableName); + if (allTableDesc.containsKey(indexTableName)) { + // Do table modification + List indices = ((IndexedHTableDescriptor) htd).getIndices(); + if (indices.isEmpty()) { + LOG.error("Empty indices are passed to modify the table " + Bytes.toString(tableName)); + return; + } + IndexManager idxManager = IndexManager.getInstance(); + idxManager.removeIndices(table); + idxManager.addIndexForTable(table, indices); + LOG.info("Successfully updated the indexes for the table " + table + " to " + indices); + } else { + try { + tableRegionsAndLocations = + MetaReader.getTableRegionsAndLocations(master.getCatalogTracker(), tableName, true); + } catch (InterruptedException e) { + LOG.error("Exception while trying to create index table for the existing table " + table); + return; + } + if (tableRegionsAndLocations != null) { + HRegionInfo[] regionInfo = new HRegionInfo[tableRegionsAndLocations.size()]; + for (int i = 0; i < tableRegionsAndLocations.size(); i++) { + regionInfo[i] = tableRegionsAndLocations.get(i).getFirst(); + } + + byte[][] splitKeys = IndexUtils.getSplitKeys(regionInfo); + IndexedHTableDescriptor iDesc = (IndexedHTableDescriptor) htd; + createSecondaryIndexTable(iDesc, splitKeys, master, true); + } + } + } + LOG.info("Exiting postModifyTable for the table " + table); + } + + private void checkColumnsForValidityAndConsistency(HTableDescriptor desc, + IndexSpecification iSpec, Map> indexColDetails) + throws IOException { + for (ColumnQualifier cq : iSpec.getIndexColumns()) { + if (null == desc.getFamily(cq.getColumnFamily())) { + String message = + "Column family " + cq.getColumnFamilyString() + " in index specification " + + iSpec.getName() + " not in Column families of table " + desc.getNameAsString() + + '.'; + LOG.error(message); + IllegalArgumentException ie = new IllegalArgumentException(message); + throw new IOException(ie); + } + Column column = new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition()); + ValueType type = cq.getType(); + int maxlength = cq.getMaxValueLength(); + Pair colDetail = indexColDetails.get(column); + if (null != colDetail) { + if (!colDetail.getFirst().equals(type) || colDetail.getSecond() != maxlength) { + throw new IOException("ValueType/max value length of column " + column + + " not consistent across the indices"); + } + } else { + indexColDetails.put(column, new Pair(type, maxlength)); + } + } + } + + private void checkEndsWithIndexSuffix(String tableName) throws IOException { + if (tableName.endsWith(Constants.INDEX_TABLE_SUFFIX)) { + String message = + "User table name should not be ends with " + Constants.INDEX_TABLE_SUFFIX + '.'; + LOG.error(message); + IllegalArgumentException ie = new IllegalArgumentException(message); + throw new IOException(ie); + } + } + + private void disableAndDeleteTable(MasterServices master, String tableName) throws IOException { + byte[] tableNameInBytes = Bytes.toBytes(tableName); + LOG.error(tableName + " already exists. Disabling and deleting table " + tableName + '.'); + boolean disabled = master.getAssignmentManager().getZKTable().isDisabledTable(tableName); + if (false == disabled) { + LOG.info("Disabling table " + tableName + '.'); + new DisableTableHandler(master, tableNameInBytes, master.getCatalogTracker(), + master.getAssignmentManager(), false).process(); + if (false == master.getAssignmentManager().getZKTable().isDisabledTable(tableName)) { + throw new IOException("Table " + tableName + " not disabled."); + } + } + LOG.info("Disabled table " + tableName + '.'); + LOG.info("Deleting table " + tableName + '.'); + new DeleteTableHandler(tableNameInBytes, master, master).process(); + if (true == MetaReader.tableExists(master.getCatalogTracker(), tableName)) { + throw new IOException("Table " + tableName + " not deleted."); + } + LOG.info("Deleted table " + tableName + '.'); + } + + @Override + public void postCreateTableHandler(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + LOG.info("Entered into postCreateTableHandler of table " + desc.getNameAsString() + '.'); + if (desc instanceof IndexedHTableDescriptor) { + MasterServices master = ctx.getEnvironment().getMasterServices(); + byte[][] splitKeys = IndexUtils.getSplitKeys(regions); + // In case of post call for the index table creation, it wont be + // IndexedHTableDescriptor + IndexedHTableDescriptor iDesc = (IndexedHTableDescriptor) desc; + createSecondaryIndexTable(iDesc, splitKeys, master, false); + // if there is any user scenarios + // we can add index datails to index manager + } + LOG.info("Exiting from postCreateTableHandler of table " + desc.getNameAsString() + '.'); + } + + /** + * @param IndexedHTableDescriptor iDesc + * @param HRegionInfo [] regions + * @param MasterServices master + * @throws NotAllMetaRegionsOnlineException + * @throws IOException + */ + private void createSecondaryIndexTable(IndexedHTableDescriptor iDesc, byte[][] splitKeys, + MasterServices master, boolean disableTable) throws NotAllMetaRegionsOnlineException, + IOException { + String indexTableName = IndexUtils.getIndexTableName(iDesc.getNameAsString()); + LOG.info("Creating secondary index table " + indexTableName + " for table " + + iDesc.getNameAsString() + '.'); + HTableDescriptor indexTableDesc = new HTableDescriptor(indexTableName); + HColumnDescriptor columnDescriptor = new HColumnDescriptor(Constants.IDX_COL_FAMILY); + String dataBlockEncodingAlgo = + master.getConfiguration().get("index.data.block.encoding.algo", "NONE"); + DataBlockEncoding[] values = DataBlockEncoding.values(); + + for (DataBlockEncoding dataBlockEncoding : values) { + if (dataBlockEncoding.toString().equals(dataBlockEncodingAlgo)) { + columnDescriptor.setDataBlockEncoding(dataBlockEncoding); + } + } + + indexTableDesc.addFamily(columnDescriptor); + indexTableDesc.setValue(HTableDescriptor.SPLIT_POLICY, + ConstantSizeRegionSplitPolicy.class.getName()); + indexTableDesc.setMaxFileSize(Long.MAX_VALUE); + LOG.info("Setting the split policy for the Index Table " + indexTableName + + " as ConstantSizeRegionSplitPolicy with maxFileSize as " + Long.MAX_VALUE + '.'); + HRegionInfo[] newRegions = getHRegionInfos(indexTableDesc, splitKeys); + new CreateTableHandler(master, master.getMasterFileSystem(), master.getServerManager(), + indexTableDesc, master.getConfiguration(), newRegions, master.getCatalogTracker(), + master.getAssignmentManager()).process(); + // Disable the index table so that when we enable the main table both can be enabled + if (disableTable) { + new DisableTableHandler(master, Bytes.toBytes(indexTableName), master.getCatalogTracker(), + master.getAssignmentManager(), false).process(); + } + LOG.info("Created secondary index table " + indexTableName + " for table " + + iDesc.getNameAsString() + '.'); + + } + + private HRegionInfo[] getHRegionInfos(HTableDescriptor hTableDescriptor, byte[][] splitKeys) { + HRegionInfo[] hRegionInfos = null; + if (splitKeys == null || splitKeys.length == 0) { + hRegionInfos = new HRegionInfo[] { new HRegionInfo(hTableDescriptor.getName(), null, null) }; + } else { + int numRegions = splitKeys.length + 1; + hRegionInfos = new HRegionInfo[numRegions]; + byte[] startKey = null; + byte[] endKey = null; + for (int i = 0; i < numRegions; i++) { + endKey = (i == splitKeys.length) ? null : splitKeys[i]; + hRegionInfos[i] = new HRegionInfo(hTableDescriptor.getName(), startKey, endKey); + startKey = endKey; + } + } + return hRegionInfos; + } + + @Override + public void preAssign(ObserverContext ctx, HRegionInfo hri) + throws IOException { + boolean isRegionInTransition = checkRegionInTransition(ctx, hri); + if (isRegionInTransition) { + LOG.info("Not calling assign for region " + hri.getRegionNameAsString() + + "because the region is already in transition."); + ctx.bypass(); + return; + } + } + + @Override + public void postAssign(ObserverContext ctx, HRegionInfo regionInfo) + throws IOException { + LOG.info("Entering into postAssign of region " + regionInfo.getRegionNameAsString() + '.'); + + if (false == regionInfo.getTableNameAsString().endsWith(Constants.INDEX_TABLE_SUFFIX)) { + MasterServices master = ctx.getEnvironment().getMasterServices(); + AssignmentManager am = master.getAssignmentManager(); + // waiting until user region is removed from transition. + long timeout = + master.getConfiguration() + .getLong("hbase.bulk.assignment.waiton.empty.rit", 5 * 60 * 1000); + Set regionSet = new HashSet(1); + regionSet.add(regionInfo); + try { + am.waitUntilNoRegionsInTransition(timeout, regionSet); + am.waitForAssignment(regionInfo, timeout); + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while region in assignment."); + } + } + ServerName sn = am.getRegionServerOfRegion(regionInfo); + String indexTableName = IndexUtils.getIndexTableName(regionInfo.getTableNameAsString()); + List tableRegions = am.getRegionsOfTable(Bytes.toBytes(indexTableName)); + for (HRegionInfo hRegionInfo : tableRegions) { + if (0 == Bytes.compareTo(hRegionInfo.getStartKey(), regionInfo.getStartKey())) { + am.addPlan(hRegionInfo.getEncodedName(), new RegionPlan(hRegionInfo, null, sn)); + LOG.info("Assigning region " + hRegionInfo.getRegionNameAsString() + " to server " + sn + + '.'); + am.assign(hRegionInfo, true, false, false); + } + } + } + LOG.info("Exiting from postAssign " + regionInfo.getRegionNameAsString() + '.'); + } + + @Override + public void preUnassign(ObserverContext ctx, HRegionInfo hri, + boolean force) throws IOException { + boolean isRegionInTransition = checkRegionInTransition(ctx, hri); + if (isRegionInTransition) { + LOG.info("Not calling move for region because region" + hri.getRegionNameAsString() + + " is already in transition."); + ctx.bypass(); + return; + } + } + + @Override + public void preMove(ObserverContext ctx, HRegionInfo hri, + ServerName srcServer, ServerName destServer) throws IOException { + boolean isRegionInTransition = checkRegionInTransition(ctx, hri); + if (isRegionInTransition) { + LOG.info("Not calling move for region " + hri.getRegionNameAsString() + + "because the region is already in transition."); + ctx.bypass(); + return; + } + } + + // This is just an additional precaution. This cannot ensure 100% that the RIT regions + // will not be picked up. + // Because the RIT map that is taken here is the copy of original RIT map and there is + // no sync mechanism also. + private boolean checkRegionInTransition(ObserverContext ctx, + HRegionInfo hri) { + MasterServices master = ctx.getEnvironment().getMasterServices(); + AssignmentManager am = master.getAssignmentManager(); + boolean isRegionInTransition = false; + String tableName = hri.getTableNameAsString(); + if (false == IndexUtils.isIndexTable(tableName)) { + NavigableMap regionsInTransition = am.getRegionsInTransition(); + RegionState regionState = regionsInTransition.get(hri.getEncodedName()); + if (regionState != null) { + isRegionInTransition = true; + } else { + String indexTableName = IndexUtils.getIndexTableName(tableName); + for (Entry region : regionsInTransition.entrySet()) { + HRegionInfo regionInfo = region.getValue().getRegion(); + if (indexTableName.equals(regionInfo.getTableNameAsString())) { + if (Bytes.compareTo(hri.getStartKey(), regionInfo.getStartKey()) == 0) { + isRegionInTransition = true; + break; + } + } + } + } + } + return isRegionInTransition; + } + + @Override + public void postMove(ObserverContext ctx, HRegionInfo regionInfo, + ServerName srcServer, ServerName destServer) throws IOException { + LOG.info("Entering into postMove " + regionInfo.getRegionNameAsString() + '.'); + if (false == regionInfo.getTableNameAsString().endsWith(Constants.INDEX_TABLE_SUFFIX)) { + MasterServices master = ctx.getEnvironment().getMasterServices(); + AssignmentManager am = master.getAssignmentManager(); + // waiting until user region is removed from transition. + long timeout = + master.getConfiguration() + .getLong("hbase.bulk.assignment.waiton.empty.rit", 5 * 60 * 1000); + Set regionSet = new HashSet(1); + regionSet.add(regionInfo); + try { + am.waitUntilNoRegionsInTransition(timeout, regionSet); + am.waitForAssignment(regionInfo, timeout); + destServer = am.getRegionServerOfRegion(regionInfo); + am.putRegionPlan(regionInfo, destServer); + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while region in assignment."); + } + } + String indexTableName = IndexUtils.getIndexTableName(regionInfo.getTableNameAsString()); + List tableRegions = am.getRegionsOfTable(Bytes.toBytes(indexTableName)); + for (HRegionInfo indexRegionInfo : tableRegions) { + if (0 == Bytes.compareTo(indexRegionInfo.getStartKey(), regionInfo.getStartKey())) { + LOG.info("Assigning region " + indexRegionInfo.getRegionNameAsString() + "from " + + srcServer + " to server " + destServer + '.'); + am.putRegionPlan(indexRegionInfo, destServer); + am.addPlan(indexRegionInfo.getEncodedName(), new RegionPlan(indexRegionInfo, null, + destServer)); + am.unassign(indexRegionInfo); + /* + * ((HMaster) master).move(indexRegionInfo.getEncodedNameAsBytes(), + * Bytes.toBytes(destServer.getServerName())); + */ + } + } + } + LOG.info("Exiting from postMove " + regionInfo.getRegionNameAsString() + '.'); + } + + @Override + public void postDisableTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + LOG.info("Entered into postDisableTableHandler of table " + Bytes.toString(tableName)); + MasterServices master = ctx.getEnvironment().getMasterServices(); + AssignmentManager am = master.getAssignmentManager(); + try { + if (false == IndexUtils.isIndexTable(Bytes.toString(tableName))) { + String indexTableName = IndexUtils.getIndexTableName(tableName); + // Index table may not present following three cases. + // 1) Index details are not specified during table creation then index table wont be + // created. + // 2) Even we specify index details if master restarted in the middle of user table creation + // corresponding index table wont be created. But without creating index table user table + // wont + // be disabled. No need to call disable for index table at that time. + // 3) Index table may be deleted but this wont happen without deleting user table. + if (true == am.getZKTable().isTablePresent(indexTableName)) { + long timeout = + master.getConfiguration().getLong("hbase.bulk.assignment.waiton.empty.rit", + 5 * 60 * 1000); + // Both user table and index table should not be in enabling/disabling state at a time. + // If disable is progress for user table then index table should be in ENABLED state. + // If enable is progress for index table wait until table enabled. + waitUntilTableEnabled(timeout, indexTableName, am.getZKTable()); + if (waitUntilTableEnabled(timeout, indexTableName, am.getZKTable())) { + new DisableTableHandler(master, Bytes.toBytes(indexTableName), + master.getCatalogTracker(), am, false).process(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Table " + indexTableName + " not in ENABLED state to disable."); + } + } + } + } + } finally { + // clear user table region plans in secondary index load balancer. + clearRegionPlans((HMaster) master, Bytes.toString(tableName)); + } + LOG.info("Exiting from postDisableTableHandler of table " + Bytes.toString(tableName)); + } + + private void clearRegionPlans(HMaster master, String tableName) { + ((SecIndexLoadBalancer) master.getBalancer()).clearTableRegionPlans(tableName); + } + + @Override + public void postEnableTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + LOG.info("Entered into postEnableTableHandler of table " + Bytes.toString(tableName)); + if (false == IndexUtils.isIndexTable(Bytes.toString(tableName))) { + MasterServices master = ctx.getEnvironment().getMasterServices(); + AssignmentManager am = master.getAssignmentManager(); + String indexTableName = IndexUtils.getIndexTableName(tableName); + // Index table may not present in three cases + // 1) Index details are not specified during table creation then index table wont be created. + // 2) Even we specify index details if master restarted in the middle of user table creation + // corresponding index table wont be created. Then no need to call enable for index table + // because it will be created as part of preMasterInitialization and enable. + // 3) Index table may be deleted but this wont happen without deleting user table. + if (true == am.getZKTable().isTablePresent(indexTableName)) { + long timeout = + master.getConfiguration().getLong("hbase.bulk.assignment.waiton.empty.rit", + 5 * 60 * 1000); + // Both user table and index table should not be in enabling/disabling state at a time. + // If enable is progress for user table then index table should be in disabled state. + // If disable is progress for index table wait until table disabled. + if (waitUntilTableDisabled(timeout, indexTableName, am.getZKTable())) { + new EnableTableHandler(master, Bytes.toBytes(indexTableName), master.getCatalogTracker(), + am, false).process(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Table " + indexTableName + " not in DISABLED state to enable."); + } + } + } + } + LOG.info("Exiting from postEnableTableHandler of table " + Bytes.toString(tableName)); + + } + + private boolean waitUntilTableDisabled(long timeout, String tableName, ZKTable zk) { + long startTime = System.currentTimeMillis(); + long remaining = timeout; + boolean disabled = false; + while (!(disabled = zk.isDisabledTable(tableName)) && remaining > 0) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while waiting for table" + tableName + " set to DISABLED."); + } + } + remaining = timeout - (System.currentTimeMillis() - startTime); + } + if (remaining <= 0) { + return disabled; + } else { + return true; + } + } + + private boolean waitUntilTableEnabled(long timeout, String tableName, ZKTable zk) { + long startTime = System.currentTimeMillis(); + long remaining = timeout; + boolean enabled = false; + while (!(enabled = zk.isEnabledTable(tableName)) && remaining > 0) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while waiting for table " + tableName + "state set to ENABLED."); + } + } + remaining = timeout - (System.currentTimeMillis() - startTime); + } + if (remaining <= 0) { + return enabled; + } else { + return true; + } + + } + + @Override + public void postDeleteTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + LOG.info("Entered into postDeleteTableHandler of table " + Bytes.toString(tableName) + '.'); + MasterServices master = ctx.getEnvironment().getMasterServices(); + String indexTableName = IndexUtils.getIndexTableName(tableName); + boolean indexTablePresent = + master.getAssignmentManager().getZKTable().isTablePresent(indexTableName); + // Not checking for disabled state because before deleting user table both user and index table + // should be disabled. + if ((false == IndexUtils.isIndexTable(Bytes.toString(tableName))) && indexTablePresent) { + new DeleteTableHandler(Bytes.toBytes(indexTableName), master, master).process(); + } + LOG.info("Exiting from postDeleteTableHandler of table " + Bytes.toString(tableName) + '.'); + } + + @Override + public void preMasterInitialization(ObserverContext ctx) + throws IOException { + LOG.info("Entering into preMasterInitialization."); + MasterServices master = ctx.getEnvironment().getMasterServices(); + AssignmentManager am = master.getAssignmentManager(); + ZKTable zkTable = am.getZKTable(); + long timeout = + master.getConfiguration().getLong("hbase.bulk.assignment.waiton.empty.rit", 5 * 60 * 1000); + try { + am.waitUntilNoRegionsInTransition(timeout); + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while waiting for the regions in transition to complete.", e); + } + } + + TableDescriptors tableDescriptors = master.getTableDescriptors(); + Map descMap = tableDescriptors.getAll(); + Collection htds = descMap.values(); + IndexedHTableDescriptor iHtd = null; + Configuration conf = master.getConfiguration(); + FileSystem fs = FSUtils.getCurrentFileSystem(conf); + Path rootPath = FSUtils.getRootDir(conf); + for (HTableDescriptor htd : htds) { + if (false == htd.getNameAsString().endsWith(Constants.INDEX_TABLE_SUFFIX)) { + FSDataInputStream fsDataInputStream = null; + try { + Path path = FSUtils.getTablePath(rootPath, htd.getName()); + FileStatus status = getTableInfoPath(fs, path); + if (null == status) { + return; + } + fsDataInputStream = fs.open(status.getPath()); + iHtd = new IndexedHTableDescriptor(); + iHtd.readFields(fsDataInputStream); + } catch (EOFException e) { + if (LOG.isDebugEnabled()) { + LOG.debug(iHtd.getNameAsString() + " is normal table and not an indexed table.", e); + } + } catch (IOException i) { + throw i; + } finally { + if (null != fsDataInputStream) { + fsDataInputStream.close(); + } + } + if (false == iHtd.getIndices().isEmpty()) { + String tableName = iHtd.getNameAsString(); + String indexTableName = IndexUtils.getIndexTableName(tableName); + boolean tableExists = MetaReader.tableExists(master.getCatalogTracker(), tableName); + boolean indexTableExists = + MetaReader.tableExists(master.getCatalogTracker(), indexTableName); + if ((true == tableExists) && (false == indexTableExists)) { + LOG.info("Table has index specification details but " + "no corresponding index table."); + List regions = + MetaReader.getTableRegions(master.getCatalogTracker(), iHtd.getName()); + HRegionInfo[] regionsArray = new HRegionInfo[regions.size()]; + byte[][] splitKeys = IndexUtils.getSplitKeys(regions.toArray(regionsArray)); + createSecondaryIndexTable(iHtd, splitKeys, master, false); + } else if (true == tableExists && true == indexTableExists) { + // If both tables are present both should be in same state in zookeeper. If tables are + // partially enabled or disabled they will be processed as part of recovery + // enabling/disabling tables. + // if user table is in ENABLED state and index table is in DISABLED state means master + // restarted as soon as user table enabled. So here we need to enable index table. + if (zkTable.isEnabledTable(tableName) && zkTable.isDisabledTable(indexTableName)) { + new EnableTableHandler(master, Bytes.toBytes(indexTableName), + master.getCatalogTracker(), am, false).process(); + } else if (zkTable.isDisabledTable(tableName) && zkTable.isEnabledTable(indexTableName)) { + // If user table is in DISABLED state and index table is in ENABLED state means master + // restarted as soon as user table disabled. So here we need to disable index table. + new DisableTableHandler(master, Bytes.toBytes(indexTableName), + master.getCatalogTracker(), am, false).process(); + // clear index table region plans in secondary index load balancer. + clearRegionPlans((HMaster) master, indexTableName); + + } + } + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Balancing after master initialization."); + } + + try { + master.getAssignmentManager().waitUntilNoRegionsInTransition(timeout); + } catch (InterruptedException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted while waiting for the regions in transition to complete.", e); + } + } + ((HMaster) master).balanceInternals(); + LOG.info("Exiting from preMasterInitialization."); + } + + public static FileStatus getTableInfoPath(final FileSystem fs, final Path tabledir) + throws IOException { + FileStatus[] status = FSUtils.listStatus(fs, tabledir, new PathFilter() { + @Override + public boolean accept(Path p) { + // Accept any file that starts with TABLEINFO_NAME + return p.getName().startsWith(FSTableDescriptors.TABLEINFO_NAME); + } + }); + if (status == null || status.length < 1) return null; + Arrays.sort(status, new FileStatusFileNameComparator()); + if (status.length > 1) { + // Clean away old versions of .tableinfo + for (int i = 1; i < status.length; i++) { + Path p = status[i].getPath(); + // Clean up old versions + if (!fs.delete(p, false)) { + LOG.warn("Failed cleanup of " + p); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Cleaned up old tableinfo file " + p); + } + } + } + } + return status[0]; + } + + /** + * Compare {@link FileStatus} instances by {@link Path#getName()}. Returns in reverse order. + */ + private static class FileStatusFileNameComparator implements Comparator { + @Override + public int compare(FileStatus left, FileStatus right) { + return -left.compareTo(right); + } + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/BackwardSeekableRegionScanner.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/BackwardSeekableRegionScanner.java new file mode 100644 index 0000000..0786347 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/BackwardSeekableRegionScanner.java @@ -0,0 +1,150 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionScanner; + +public class BackwardSeekableRegionScanner implements SeekAndReadRegionScanner { + + private ReInitializableRegionScanner delegator; + + private Scan scan; + + private HRegion hRegion; + + private byte[] startRow; + + private boolean closed = false; + + public BackwardSeekableRegionScanner(ReInitializableRegionScanner delegator, Scan scan, + HRegion hRegion, byte[] startRow) { + this.delegator = delegator; + this.scan = scan; + this.hRegion = hRegion; + this.startRow = startRow; + } + + Scan getScan() { + return scan; + } + + byte[] getStartRow() { + return startRow; + } + + // For testing. + RegionScanner getDelegator() { + return delegator; + } + + @Override + public HRegionInfo getRegionInfo() { + return this.delegator.getRegionInfo(); + } + + @Override + public boolean isFilterDone() { + return this.delegator.isFilterDone(); + } + + @Override + public synchronized void close() throws IOException { + this.delegator.close(); + closed = true; + } + + @Override + public boolean isClosed() { + return closed; + } + + @Override + public synchronized boolean next(List results) throws IOException { + return next(results, this.scan.getBatch()); + } + + @Override + public boolean next(List result, int limit) throws IOException { + return false; + } + + @Override + public synchronized boolean reseek(byte[] row) throws IOException { + return this.delegator.reseek(row); + } + + @Override + public void addSeekPoints(List seekPoints) { + this.delegator.addSeekPoints(seekPoints); + } + + @Override + public boolean seekToNextPoint() throws IOException { + return this.delegator.seekToNextPoint(); + } + + @Override + public byte[] getLatestSeekpoint() { + return this.delegator.getLatestSeekpoint(); + } + + @Override + public boolean next(List results, String metric) throws IOException { + return next(results, this.scan.getBatch(), metric); + } + + @Override + public boolean next(List result, int limit, String metric) throws IOException { + boolean hasNext = false; + try { + if (this.delegator.isClosed()) return false; + hasNext = this.delegator.next(result, limit, metric); + } catch (SeekUnderValueException e) { + Scan newScan = new Scan(this.scan); + // Start from the point where we got stopped because of seek backward + newScan.setStartRow(getLatestSeekpoint()); + this.delegator.reInit(this.hRegion.getScanner(newScan)); + hasNext = next(result, limit, metric); + } + return hasNext; + } + + @Override + public long getMvccReadPoint() { + return this.delegator.getMvccReadPoint(); + } + + @Override + public boolean nextRaw(List result, String metric) throws IOException { + return this.delegator.nextRaw(result, metric); + } + + @Override + public boolean nextRaw(List result, int limit, String metric) throws IOException { + return this.delegator.next(result, limit, metric); + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueDetail.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueDetail.java new file mode 100644 index 0000000..4bf9ff7 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueDetail.java @@ -0,0 +1,101 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.util.Bytes; + +public class FilterColumnValueDetail { + protected byte[] cf; + protected byte[] qualifier; + protected byte[] value; + protected CompareOp compareOp; + protected Column column; + protected int maxValueLength; + protected ValueType valueType; + + public FilterColumnValueDetail(byte[] cf, byte[] qualifier, byte[] value, CompareOp compareOp) { + this.cf = cf; + this.qualifier = qualifier; + this.value = value; + this.compareOp = compareOp; + this.column = new Column(this.cf, this.qualifier); + } + + public FilterColumnValueDetail(byte[] cf, byte[] qualifier, byte[] value, + ValuePartition valuePartition, CompareOp compareOp) { + this.cf = cf; + this.qualifier = qualifier; + this.value = value; + this.compareOp = compareOp; + this.column = new Column(this.cf, this.qualifier, valuePartition); + } + + public FilterColumnValueDetail(Column column, byte[] value, CompareOp compareOp) { + this.cf = column.getFamily(); + this.qualifier = column.getQualifier(); + this.value = value; + this.compareOp = compareOp; + this.column = column; + } + + public boolean equals(Object obj) { + if (!(obj instanceof FilterColumnValueDetail)) return false; + FilterColumnValueDetail that = (FilterColumnValueDetail) obj; + if (!this.column.equals(that.column)) { + return false; + } + // Need to check. + if (this.value != null && that.value != null) { + if (!(Bytes.equals(this.value, that.value))) return false; + } else if (this.value == null && that.value == null) { + return true; + } else { + return false; + } + return true; + } + + public int hashCode() { + return this.column.hashCode(); + } + + public String toString() { + return String.format("%s (%s, %s, %s, %s, %s)", this.getClass().getSimpleName(), + Bytes.toStringBinary(this.cf), Bytes.toStringBinary(this.qualifier), this.valueType.name(), + this.compareOp.name(), Bytes.toStringBinary(this.value)); + } + + public Column getColumn() { + return this.column; + } + + public byte[] getValue() { + return this.value; + } + + protected void setValue(byte[] value) { + this.value = value; + } + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueRange.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueRange.java new file mode 100644 index 0000000..5976c3d --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueRange.java @@ -0,0 +1,98 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.util.Bytes; + +public class FilterColumnValueRange extends FilterColumnValueDetail { + private CompareOp upperBoundCompareOp; + private byte[] upperBoundValue; + + public FilterColumnValueRange(byte[] cf, byte[] qualifier, byte[] lowerBoundValue, + CompareOp lowerBoundCompareOp, byte[] upperBoundValue, CompareOp upperBoundCompareOp) { + super(cf, qualifier, lowerBoundValue, lowerBoundCompareOp); + this.upperBoundCompareOp = upperBoundCompareOp; + this.upperBoundValue = upperBoundValue; + } + + public FilterColumnValueRange(byte[] cf, byte[] qualifier, ValuePartition vp, + byte[] lowerBoundValue, CompareOp lowerBoundCompareOp, byte[] upperBoundValue, + CompareOp upperBoundCompareOp) { + super(cf, qualifier, lowerBoundValue, vp, lowerBoundCompareOp); + this.upperBoundCompareOp = upperBoundCompareOp; + this.upperBoundValue = upperBoundValue; + } + + public FilterColumnValueRange(Column column, byte[] lowerBoundValue, + CompareOp lowerBoundCompareOp, byte[] upperBoundValue, CompareOp upperBoundCompareOp) { + super(column, lowerBoundValue, lowerBoundCompareOp); + this.upperBoundCompareOp = upperBoundCompareOp; + this.upperBoundValue = upperBoundValue; + } + + // No need to have the hashCode() and equals(Object obj) implementation here. Super class + // implementation checks for the CF name and qualifier name which is sufficient. + + public String toString() { + return String.format("%s (%s, %s, %s, %s, %s, %s, %s)", this.getClass().getSimpleName(), Bytes + .toStringBinary(this.cf), Bytes.toStringBinary(this.qualifier), this.valueType.name(), + this.compareOp.name(), Bytes.toStringBinary(this.value), + this.upperBoundCompareOp == null ? "" : this.upperBoundCompareOp.name(), + this.upperBoundValue == null ? "" : Bytes.toStringBinary(this.upperBoundValue)); + } + + public CompareOp getUpperBoundCompareOp() { + return this.upperBoundCompareOp; + } + + public byte[] getUpperBoundValue() { + return this.upperBoundValue; + } + + // The equals method is a bit tricky. + // Needs one more eye on this + @Override + public boolean equals(Object other) { + if (this == other) return true; + FilterColumnValueRange fcvr = null; + if (other instanceof FilterColumnValueRange && super.equals(other)) { + fcvr = (FilterColumnValueRange) other; + if (this.upperBoundValue != null && fcvr.getUpperBoundValue() != null) { + if (Bytes.compareTo(this.upperBoundValue, fcvr.getUpperBoundValue()) != 0) { + return false; + } + } else if (this.upperBoundValue == null && fcvr.getUpperBoundValue() == null) { + return true; + } else { + return false; + } + } + return false; + } + + @Override + public int hashCode() { + return super.hashCode(); + } + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterGroupingWorker.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterGroupingWorker.java new file mode 100644 index 0000000..8e56285 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterGroupingWorker.java @@ -0,0 +1,528 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.index.filter.SingleColumnRangeFilter; +import org.apache.hadoop.hbase.index.filter.SingleColumnValuePartitionFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +/** + * This class does the grouping work for different filterlists. Like if we i have 3 different filter + * list and all have MUST_PASS_ALL condition finally this class groups the filters lists into one + * filter list with all as MUST_PASS_ALL. Also it checks if the combination of the filters list is + * given correctly. For eg: If i have c1 > 10 and c1 < 5. This is wrong combination. + */ +public class FilterGroupingWorker { + + private static final Log LOG = LogFactory.getLog(FilterGroupingWorker.class); + + private Map> colWithOperators = + new HashMap>(); + private Map> colWithOperatorsOfOR = new HashMap>(); + + public Filter group(Filter filter) { + if (filter instanceof FilterList) { + FilterList fList = (FilterList) filter; + // We need to create a new FL here taking up only the filters of our interest + FilterList newFList = new FilterList(fList.getOperator()); + List filters = fList.getFilters(); + if (fList.getOperator() == Operator.MUST_PASS_ONE) { + for (Filter subFilter : filters) { + Filter resultFilter = handleFilterWithinOR(subFilter); + // If result filter is not SingleColumnValueFilter or filter list that means OR branch is + // having different type of filter other than SCVF. In that case we should not consider + // the OR branch for scanning. + if (resultFilter instanceof FilterList) { + newFList.addFilter(resultFilter); + } else if (resultFilter != null) { + // This means OR filter list have at least one filter other than SCVF(may be other + // child OR branches). + return null; + } + } + addORColsToFinalList(newFList); + if (newFList.getFilters().isEmpty()) { + return null; + } + return newFList; + } else { + // AND condition as long as the condition is AND in one sub tree all those can be + // grouped under one AND parent(new one). + for (Filter subFilter : filters) { + Filter group = handleFilterWithinAND(subFilter); + // group is null means, all are AND conditions and will be handled at once with the + // below createFinalFilter + if (group != null) { + newFList.addFilter(group); + } + } + addANDColsToFinalList(newFList); + if (newFList.getFilters().isEmpty()) { + return null; + } + return newFList; + } + } else if (filter instanceof SingleColumnValueFilter + || filter instanceof SingleColumnRangeFilter) { + return filter; + } + return null; + } + + private Filter handleFilterWithinAND(Filter filter) { + if (filter instanceof FilterList) { + FilterList fList = (FilterList) filter; + if (fList.getOperator() == Operator.MUST_PASS_ONE) { + return new FilterGroupingWorker().group(fList); + } else { + List filters = fList.getFilters(); + for (Filter subFilter : filters) { + handleFilterWithinAND(subFilter); + } + } + } else if (filter instanceof SingleColumnValueFilter) { + handleScvf((SingleColumnValueFilter) filter); + } // TODO when we expose SingleColumnRangeFilter to handle that also here. + return null; + } + + /** + * Since you can use Filter Lists as children of Filter Lists, you can create a hierarchy of + * filters to be evaluated. In the hierarchy if OR branch having any filter type other than SCVF + * as child then we should not consider the branch for scanning because we cannot fetch seek + * points from other type of filters without column and value details. Ex: AND AND + * __________|_______ | | | --> SCVF OR SCVF _______|______ | | ROWFILTER SVCF If the OR is root + * then we should skip index table scanning for this filter. OR _______|______ --> null | | + * ROWFILTER SVCF If the OR is child of another OR branch then parent OR branch will be excluded + * for scanning. Ex: AND AND __________|_______ | | | --> SCVF OR SCVF _______|______ | | OR SVCF + * _______|______ | | ROWFILTER SVCF + * @param filter + * @return if filter is filter list with AND condition then we will return AND branch after + * grouping. if filter is filter list with OR condition return null if no children is of + * type other than SCVF or filter list else return different filter. if filter is SCVF + * then return null. returning null means we are combining the filter(s) with children of + * parent OR filter to perform optimizations. + */ + private Filter handleFilterWithinOR(Filter filter) { + if (filter instanceof FilterList) { + FilterList fList = (FilterList) filter; + if (fList.getOperator() == Operator.MUST_PASS_ONE) { + List filters = fList.getFilters(); + Filter resultFilter = null; + for (Filter subFilter : filters) { + // If this OR branch in the filter list have filter type other than SCVF we should report + // it to parent by returning the other type of filter in such a way that the branch will + // be skipped from index scan. + resultFilter = handleFilterWithinOR(subFilter); + if (resultFilter == null || (resultFilter instanceof FilterList)) { + continue; + } else { + return resultFilter; + } + } + return null; + } else { + return new FilterGroupingWorker().group(fList); + } + } else if (filter instanceof SingleColumnValueFilter) { + handleScvfOfOR((SingleColumnValueFilter) filter); + return null; + }// TODO when we expose SingleColumnRangeFilter to handle that also here. + // filter other than SingleColumnValueFilter. + return filter; + } + + private void handleScvfOfOR(SingleColumnValueFilter scvf) { + ValuePartition vp = null; + if (scvf instanceof SingleColumnValuePartitionFilter) { + vp = ((SingleColumnValuePartitionFilter) scvf).getValuePartition(); + } + Column key = new Column(scvf.getFamily(), scvf.getQualifier(), vp); + if (colWithOperatorsOfOR.get(key) == null) { + List valueList = new ArrayList(1); + valueList.add(new Value(scvf.getOperator(), scvf.getComparator().getValue(), scvf)); + colWithOperatorsOfOR.put(key, valueList); + } else { + List valueList = colWithOperatorsOfOR.get(key); + Iterator valueListItr = valueList.iterator(); + CompareOp operator = scvf.getOperator(); + byte[] value = scvf.getComparator().getValue(); + Value prevValueObj = null; + while (valueListItr.hasNext()) { + prevValueObj = valueListItr.next(); + // TODO As Anoop said we may have to check the Value type also.. + // We can not compare and validate this way. btw "a" and "K". + // Only in case of Numeric col type we can have this check. + byte[] prevValue = prevValueObj.getValue(); + int result = Bytes.compareTo(prevValue, value); + CompareOp prevOperator = prevValueObj.getOperator(); + switch (operator) { + case GREATER: + if (prevOperator == CompareOp.GREATER || prevOperator == CompareOp.GREATER_OR_EQUAL) { + if (result > 0) { + valueListItr.remove(); + } else { + // Already you found less or equal value than present value means present filter will + // return subset of previous filter. No need to add it to list. + return; + } + } else if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) { + // Need to handle conditions like previous is c1<5 and current c1>2. In these cases we + // can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to + // list. + } else if (prevOperator == CompareOp.EQUAL) { + if (result > 0) { + valueListItr.remove(); + } else if (result == 0) { + // remove this entry and convert GREATER to GREATER_OR_EQUAL + valueListItr.remove(); + SingleColumnValueFilter newScvf = null; + if (vp == null) { + newScvf = + new SingleColumnValueFilter(scvf.getFamily(), scvf.getQualifier(), + CompareOp.GREATER_OR_EQUAL, scvf.getComparator()); + } else { + newScvf = + new SingleColumnValuePartitionFilter(scvf.getFamily(), scvf.getQualifier(), + CompareOp.GREATER_OR_EQUAL, scvf.getComparator(), vp); + } + Value newValue = new Value(CompareOp.GREATER_OR_EQUAL, prevValue, newScvf); + valueList.add(newValue); + return; + } + } + break; + case GREATER_OR_EQUAL: + if (prevOperator == CompareOp.GREATER || prevOperator == CompareOp.GREATER_OR_EQUAL) { + if (result >= 0) { + valueListItr.remove(); + } else { + // Already you found less value than present value means present filter will + // return subset of previous filter. No need to add it to list. + return; + } + } else if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) { + // Need to handle conditions like previous is c1<5 and current c1>2. In these cases we + // can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to + // list. + } else if (prevOperator == CompareOp.EQUAL) { + if (result >= 0) { + valueListItr.remove(); + } + } + break; + case LESS: + if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) { + if (result < 0) { + valueListItr.remove(); + } else { + // Already you found less or equal value than present value means present filter will + // return subset of previous filter. No need to add it to list. + return; + } + } else if (prevOperator == CompareOp.GREATER + || prevOperator == CompareOp.GREATER_OR_EQUAL) { + // Need to handle conditions like previous is c1<5 and current c1>2. In these cases we + // can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to + // list. + } else if (prevOperator == CompareOp.EQUAL) { + if (result < 0) { + valueListItr.remove(); + } else if (result == 0) { + // remove this entry and convert LESS to LESS_OR_EQUAL + valueListItr.remove(); + SingleColumnValueFilter newScvf = null; + if (vp == null) { + newScvf = + new SingleColumnValueFilter(scvf.getFamily(), scvf.getQualifier(), + CompareOp.LESS_OR_EQUAL, scvf.getComparator()); + } else { + newScvf = + new SingleColumnValuePartitionFilter(scvf.getFamily(), scvf.getQualifier(), + CompareOp.LESS_OR_EQUAL, scvf.getComparator(), vp); + } + Value newValue = new Value(CompareOp.LESS_OR_EQUAL, prevValue, newScvf); + valueList.add(newValue); + return; + } + } + break; + case LESS_OR_EQUAL: + if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) { + if (result <= 0) { + valueListItr.remove(); + } else { + // Already you found less or equal value than present value means present filter will + // return subset of previous filter. No need to add it to list. + return; + } + } else if (prevOperator == CompareOp.GREATER + || prevOperator == CompareOp.GREATER_OR_EQUAL) { + // Need to handle conditions like previous is c1<5 and current c1>2. In these cases we + // can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to + // list. + } else if (prevOperator == CompareOp.EQUAL) { + // If we dont want to do conversion we can add into first if condition. + if (result <= 0) { + valueListItr.remove(); + } else if (result == 0) { + // remove this entry and convert GREATER to GREATER_OR_EQUAL + } + } + break; + case EQUAL: + if (prevOperator == CompareOp.GREATER) { + if (result < 0) { + return; + } else if (result == 0) { + valueListItr.remove(); + SingleColumnValueFilter newScvf = null; + if (vp == null) { + newScvf = + new SingleColumnValueFilter(scvf.getFamily(), scvf.getQualifier(), + CompareOp.GREATER_OR_EQUAL, scvf.getComparator()); + } else { + newScvf = + new SingleColumnValuePartitionFilter(scvf.getFamily(), scvf.getQualifier(), + CompareOp.GREATER_OR_EQUAL, scvf.getComparator(), vp); + } + Value newValue = new Value(CompareOp.GREATER_OR_EQUAL, prevValue, newScvf); + valueList.add(newValue); + return; + } + } else if (prevOperator == CompareOp.GREATER_OR_EQUAL) { + if (result <= 0) { + return; + } + } else if (prevOperator == CompareOp.LESS) { + if (result > 0) { + return; + } else if (result == 0) { + valueListItr.remove(); + SingleColumnValueFilter newScvf = null; + if (vp == null) { + newScvf = + new SingleColumnValueFilter(scvf.getFamily(), scvf.getQualifier(), + CompareOp.LESS_OR_EQUAL, scvf.getComparator()); + } else { + newScvf = + new SingleColumnValuePartitionFilter(scvf.getFamily(), scvf.getQualifier(), + CompareOp.LESS_OR_EQUAL, scvf.getComparator(), vp); + } + Value newValue = new Value(CompareOp.LESS_OR_EQUAL, prevValue, newScvf); + valueList.add(newValue); + return; + } + } else if (prevOperator == CompareOp.LESS_OR_EQUAL) { + if (result >= 0) { + return; + } + } else if (prevOperator == CompareOp.EQUAL) { + if (result == 0) { + // Already same filter exists with same condiftion. + return; + } + } + break; + case NOT_EQUAL: + case NO_OP: + // Need to check this + break; + } + } + valueList.add(new Value(scvf.getOperator(), scvf.getComparator().getValue(), scvf)); + } + } + + private void handleScvf(SingleColumnValueFilter scvf) { + ValuePartition vp = null; + if (scvf instanceof SingleColumnValuePartitionFilter) { + vp = ((SingleColumnValuePartitionFilter) scvf).getValuePartition(); + } + Column column = new Column(scvf.getFamily(), scvf.getQualifier(), vp); + Pair pair = colWithOperators.get(column); + if (pair == null) { + pair = new Pair(); + // The first operator should be set here + pair.setFirst(new Value(scvf.getOperator(), scvf.getComparator().getValue(), scvf)); + colWithOperators.put(column, pair); + } else { + if (pair.getFirst() != null && pair.getSecond() == null) { + // TODO As Anoop said we may have to check the Value type also.. + // We can not compare and validate this way. btw "a" and "K". + // Only in case of Numeric col type we can have this check. + byte[] curBoundValue = scvf.getComparator().getValue(); + byte[] prevBoundValue = pair.getFirst().getValue(); + int result = Bytes.compareTo(prevBoundValue, curBoundValue); + CompareOp curBoundOperator = scvf.getOperator(); + CompareOp prevBoundOperator = pair.getFirst().getOperator(); + switch (curBoundOperator) { + case GREATER: + case GREATER_OR_EQUAL: + if (prevBoundOperator == CompareOp.GREATER + || prevBoundOperator == CompareOp.GREATER_OR_EQUAL) { + LOG.warn("Wrong usage. It should be < > || > <. Cannot be > >"); + if (result > 1) { + pair.setFirst(new Value(curBoundOperator, curBoundValue, scvf)); + } + pair.setSecond(null); + } else if (prevBoundOperator == CompareOp.LESS + || prevBoundOperator == CompareOp.LESS_OR_EQUAL) { + if (result < 1) { + LOG.warn("Possible wrong usage as there cannot be a value < 10 and > 20"); + pair.setFirst(null); + pair.setSecond(null); + } else { + pair.setSecond(new Value(curBoundOperator, curBoundValue, scvf)); + } + } else if (prevBoundOperator == CompareOp.EQUAL) { + LOG.warn("Use the equal operator and ignore the current one"); + pair.setSecond(null); + } + break; + case LESS: + case LESS_OR_EQUAL: + if (prevBoundOperator == CompareOp.LESS || prevBoundOperator == CompareOp.LESS_OR_EQUAL) { + LOG.warn("Wrong usage. It should be < > || > <. Cannot be > >"); + if (result < 1) { + pair.setFirst(new Value(curBoundOperator, curBoundValue, scvf)); + } + pair.setSecond(null); + } else if (prevBoundOperator == CompareOp.GREATER + || prevBoundOperator == CompareOp.GREATER_OR_EQUAL) { + if (result > 1) { + LOG.warn("Possible wrong usage as there cannot be a value < 10 and > 20"); + pair.setFirst(null); + pair.setSecond(null); + } else { + pair.setSecond(new Value(curBoundOperator, curBoundValue, scvf)); + } + } else if (prevBoundOperator == CompareOp.EQUAL) { + LOG.warn("Use the EQUAL operator only and ignore the current one."); + pair.setSecond(null); + } + break; + case EQUAL: + // For equal condition give priority to equals only.. + // If the prevOperator is also == and the current is also == + // take the second one.(Currently) + if (prevBoundOperator == CompareOp.LESS || prevBoundOperator == CompareOp.LESS_OR_EQUAL + || prevBoundOperator == CompareOp.EQUAL || prevBoundOperator == CompareOp.GREATER + || prevBoundOperator == CompareOp.GREATER_OR_EQUAL) { + pair.setFirst(new Value(curBoundOperator, curBoundValue, scvf)); + pair.setSecond(null); + } + break; + case NOT_EQUAL: + case NO_OP: + // Need to check this + break; + } + } else { + LOG.warn("Am getting an extra comparison coming for the same col family." + + "I cannot have 3 conditions on the same column"); + pair.setFirst(null); + pair.setSecond(null); + } + } + } + + private void addANDColsToFinalList(FilterList filterList) { + for (Entry> entry : colWithOperators.entrySet()) { + Pair value = entry.getValue(); + if (value.getFirst() != null && value.getSecond() != null) { + // Here we are introducing a new Filter + SingleColumnRangeFilter rangeFltr = + new SingleColumnRangeFilter(entry.getKey().getFamily(), entry.getKey().getQualifier(), + entry.getKey().getValuePartition(), value.getFirst().getValue(), value.getFirst() + .getOperator(), value.getSecond().getValue(), value.getSecond().getOperator()); + filterList.addFilter(rangeFltr); + } else if (value.getFirst() != null) { + if (value.getFirst().getOperator() == CompareOp.EQUAL) { + filterList.addFilter(value.getFirst().getFilter()); + } else { + SingleColumnRangeFilter rangeFltr = + new SingleColumnRangeFilter(entry.getKey().getFamily(), + entry.getKey().getQualifier(), entry.getKey().getValuePartition(), value + .getFirst().getValue(), value.getFirst().getOperator(), null, null); + filterList.addFilter(rangeFltr); + } + } + } + } + + private void addORColsToFinalList(FilterList filterList) { + for (Entry> entry : colWithOperatorsOfOR.entrySet()) { + List valueList = entry.getValue(); + for (Value value : valueList) { + if (value.getOperator() == CompareOp.EQUAL) { + filterList.addFilter(value.getFilter()); + } else { + SingleColumnRangeFilter rangeFltr = + new SingleColumnRangeFilter(entry.getKey().getFamily(), + entry.getKey().getQualifier(), entry.getKey().getValuePartition(), + value.getValue(), value.getOperator(), null, null); + filterList.addFilter(rangeFltr); + } + } + } + } + + private static class Value { + private CompareOp operator; + private byte[] value; + private Filter filter; + + public Value(CompareOp operator, byte[] value, Filter filter) { + this.operator = operator; + this.value = value; + this.filter = filter; + } + + public CompareOp getOperator() { + return this.operator; + } + + public byte[] getValue() { + return this.value; + } + + public Filter getFilter() { + return this.filter; + } + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterNode.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterNode.java new file mode 100644 index 0000000..79d0298 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterNode.java @@ -0,0 +1,35 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.util.Pair; + +public interface FilterNode { + Map, IndexSpecification> getIndexToUse(); + + Map>> getPossibleUseIndices(); + + Map>> getPossibleFutureUseIndices(); +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexFilterNode.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexFilterNode.java new file mode 100644 index 0000000..f56d86a --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexFilterNode.java @@ -0,0 +1,103 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.util.Pair; + +public class IndexFilterNode implements LeafFilterNode { + + private IndexSpecification indexToUse; + // all possible indices which can be used. This includes the selected indexToUse also. + // This contains the an integer as the second item in the Pair. This is the relative overhead + // in scanning the index region. The lesser the value the lesser the overhead in scanning the + // index region. This will be set with the number of columns in the index specification. + private List> possibleUseIndices; + + private List> possibleFutureUseIndices; + + private FilterColumnValueDetail filterColumnValueDetail; + + @Override + public Map>> getPossibleFutureUseIndices() { + // TODO avoid create of Map instance all the time... + Map>> reply = + new HashMap>>(); + reply.put(filterColumnValueDetail.getColumn(), possibleFutureUseIndices); + return reply; + } + + public IndexFilterNode(IndexSpecification indexToUse, + List> possibleUseIndices, + List> possibleFutureUseIndices, + FilterColumnValueDetail filterColumnValueDetail) { + this.indexToUse = indexToUse; + this.possibleUseIndices = possibleUseIndices; + this.possibleFutureUseIndices = possibleFutureUseIndices; + this.filterColumnValueDetail = filterColumnValueDetail; + } + + /** + * all possible indices which can be used. This includes the selected indexToUse also. This + * contains the an integer as the second item in the Pair. This is the relative overhead in + * scanning the index region. The lesser the value the lesser the overhead in scanning the index + * region. This will be set with the number of columns in the index specification. + * @return + */ + @Override + public Map>> getPossibleUseIndices() { + // TODO avoid create of Map instance all the time... + Map>> reply = + new HashMap>>(); + reply.put(filterColumnValueDetail.getColumn(), possibleUseIndices); + return reply; + } + + @Override + public Map, IndexSpecification> getIndexToUse() { + // TODO avoid create of Map instance all the time... + Map, IndexSpecification> reply = + new HashMap, IndexSpecification>(); + List key = new ArrayList(1); + key.add(filterColumnValueDetail); + reply.put(key, indexToUse); + return reply; + } + + @Override + public IndexSpecification getBestIndex() { + return this.indexToUse; + } + + @Override + public FilterColumnValueDetail getFilterColumnValueDetail() { + return this.filterColumnValueDetail; + } + + public void setFilterColumnValueDetail(FilterColumnValueDetail filterColumnValueDetail) { + this.filterColumnValueDetail = filterColumnValueDetail; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionObserver.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionObserver.java new file mode 100644 index 0000000..1c0f867 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionObserver.java @@ -0,0 +1,860 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HConstants.OperationStatusCode; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.RegionObserverExt; +import org.apache.hadoop.hbase.index.ColumnQualifier; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.OperationStatus; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.SplitTransaction; +import org.apache.hadoop.hbase.regionserver.SplitTransaction.SplitInfo; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.PairOfSameType; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; + +public class IndexRegionObserver extends BaseRegionObserver implements RegionObserverExt { + + private static final Log LOG = LogFactory.getLog(IndexRegionObserver.class); + + // variable will be set to true in test case for testing + // All below public static fields are used for testing. + static boolean isTestingEnabled = false; + + public static boolean isSeekpointAddded = false; + + public static boolean isIndexedFlowUsed = false; + + public static List seekPoints = null; + + public static List seekPointsForMultipleIndices = null; + + private Map scannerMap = + new ConcurrentHashMap(); + + private IndexManager indexManager = IndexManager.getInstance(); + + public static final ThreadLocal threadLocal = new ThreadLocal() { + @Override + protected IndexEdits initialValue() { + return new IndexEdits(); + } + }; + + @Override + public void postOpen(ObserverContext contx) { + byte[] tableName = contx.getEnvironment().getRegion().getTableDesc().getName(); + String tableNameStr = Bytes.toString(tableName); + if (IndexUtils.isCatalogTable(tableName) || IndexUtils.isIndexTable(tableNameStr)) { + return; + } + LOG.trace("Entering postOpen for the table " + tableNameStr); + this.indexManager.incrementRegionCount(tableNameStr); + List list = indexManager.getIndicesForTable(tableNameStr); + if (null != list) { + LOG.trace("Index Manager already contains an entry for the table " + + ". Hence returning from postOpen"); + return; + } + RegionServerServices rss = contx.getEnvironment().getRegionServerServices(); + Configuration conf = rss.getConfiguration(); + IndexedHTableDescriptor tableDescriptor = null; + try { + tableDescriptor = IndexUtils.getIndexedHTableDescriptor(tableName, conf); + } catch (IOException e) { + rss.abort("Some unidentified scenario while reading from the " + + "table descriptor . Aborting RegionServer", e); + } + if (tableDescriptor != null) { + list = tableDescriptor.getIndices(); + if (list != null && list.size() > 0) { + indexManager.addIndexForTable(tableNameStr, list); + LOG.trace("Added index Specification in the Manager for the " + tableNameStr); + } else { + list = new ArrayList(); + indexManager.addIndexForTable(tableNameStr, list); + LOG.trace("Added index Specification in the Manager for the " + tableNameStr); + } + } + LOG.trace("Exiting postOpen for the table " + tableNameStr); + } + + @Override + public void preBatchMutate(final ObserverContext ctx, + final List> mutationVsBatchOp, final WALEdit edit) + throws IOException { + HRegionServer rs = (HRegionServer) ctx.getEnvironment().getRegionServerServices(); + HRegion userRegion = ctx.getEnvironment().getRegion(); + HTableDescriptor userTableDesc = userRegion.getTableDesc(); + String tableName = userTableDesc.getNameAsString(); + if (IndexUtils.isCatalogTable(userTableDesc.getName()) || IndexUtils.isIndexTable(tableName)) { + return; + } + List indices = indexManager.getIndicesForTable(tableName); + if (indices == null || indices.isEmpty()) { + LOG.trace("skipping preBatchMutate for the table " + tableName + " as there are no indices"); + return; + } + LOG.trace("Entering preBatchMutate for the table " + tableName); + LOG.trace("Indices for the table " + tableName + " are: " + indices); + HRegion indexRegion = getIndexTableRegion(tableName, userRegion, rs); + // Storing this found HRegion in the index table within the thread locale. + IndexEdits indexEdits = threadLocal.get(); + indexEdits.indexRegion = indexRegion; + for (Pair mutation : mutationVsBatchOp) { + if (mutation.getSecond().getOperationStatusCode() != OperationStatusCode.NOT_RUN) { + continue; + } + // only for successful puts + Mutation m = mutation.getFirst(); + if (m instanceof Put) { + try { + prepareIndexMutations(indices, userRegion, m, tableName, indexRegion); + } catch (IOException e) { + mutation.setSecond(new OperationStatus(OperationStatusCode.SANITY_CHECK_FAILURE, e + .getMessage())); + } + } else if (m instanceof Delete) { + prepareIndexMutations(indices, userRegion, m, tableName, indexRegion); + } + } + indexEdits.setUpdateLocked(); + indexRegion.updateLock(); + LOG.trace("Exiting preBatchMutate for the table " + tableName); + } + + private HRegion getIndexTableRegion(String tableName, HRegion userRegion, HRegionServer rs) + throws IOException { + String indexTableName = IndexUtils.getIndexTableName(tableName); + Collection idxTabRegions = rs.getOnlineRegions(Bytes.toBytes(indexTableName)); + for (HRegion idxTabRegion : idxTabRegions) { + // TODO start key check is enough? May be we can check for the + // possibility for N-1 Mapping? + if (Bytes.equals(idxTabRegion.getStartKey(), userRegion.getStartKey())) { + return idxTabRegion; + } + } + // No corresponding index region found in the RS online regions list! + LOG.warn("Index Region not found on the region server . " + + "So skipping the put. Need Balancing"); + // TODO give a proper Exception msg + throw new IOException(); + } + + private void prepareIndexMutations(List indices, HRegion userRegion, + Mutation mutation, String tableName, HRegion indexRegion) throws IOException { + IndexEdits indexEdits = threadLocal.get(); + if (mutation instanceof Put) { + for (IndexSpecification index : indices) { + // Handle each of the index + Mutation indexPut = IndexUtils.prepareIndexPut((Put) mutation, index, indexRegion); + if (null != indexPut) { + // This mutation can be null when the user table mutation is not + // containing all of the indexed col value. + indexEdits.add(indexPut); + } + } + } else if (mutation instanceof Delete) { + Collection indexDeletes = + prepareIndexDeletes((Delete) mutation, userRegion, indices, indexRegion); + indexEdits.addAll(indexDeletes); + } else { + // TODO : Log or throw exception + } + } + + Collection prepareIndexDeletes(Delete delete, HRegion userRegion, + List indexSpecs, HRegion indexRegion) throws IOException { + Collection indexDeletes = new LinkedHashSet(); + for (Entry> entry : delete.getFamilyMap().entrySet()) { + for (KeyValue kv : entry.getValue()) { + indexDeletes.addAll(getIndexDeletes(indexSpecs, userRegion, indexRegion, kv)); + } + } + return indexDeletes; + } + + private static Collection getIndexDeletes(List indexSpecs, + HRegion userRegion, HRegion indexRegion, KeyValue deleteKV) throws IOException { + Collection indexDeletes = new LinkedHashSet(); + List indicesToUpdate = new LinkedList(); + Multimap groupedKV = + doGetAndGroupByTS(indexSpecs, userRegion, deleteKV, indicesToUpdate); + + // There can be multiple index kvs for each user kv + // So, prepare all resultant index delete kvs for this user delete kv + for (Entry> entry : groupedKV.asMap().entrySet()) { + for (IndexSpecification index : indicesToUpdate) { + ByteArrayBuilder indexRow = + IndexUtils.getIndexRowKeyHeader(index, indexRegion.getStartKey(), deleteKV.getRow()); + boolean update = false; + for (ColumnQualifier cq : index.getIndexColumns()) { + KeyValue kvFound = null; + for (KeyValue kv : entry.getValue()) { + if (Bytes.equals(cq.getColumnFamily(), kv.getFamily()) + && Bytes.equals(cq.getQualifier(), kv.getQualifier())) { + kvFound = kv; + update = true; + break; + } + } + if (kvFound == null) { + indexRow.position(indexRow.position() + cq.getMaxValueLength()); + } else { + IndexUtils.updateRowKeyForKV(cq, kvFound, indexRow); + } + } + if (update) { + // Append the actual row key at the end of the index row key. + indexRow.put(deleteKV.getRow()); + Delete idxDelete = new Delete(indexRow.array()); + if (deleteKV.isDeleteType()) { + idxDelete + .deleteColumn(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, entry.getKey()); + } else { + idxDelete.deleteFamily(Constants.IDX_COL_FAMILY, entry.getKey()); + } + idxDelete.setWriteToWAL(false); + indexDeletes.add(idxDelete); + } + } + } + return indexDeletes; + } + + private static Multimap doGetAndGroupByTS(List indexSpecs, + HRegion userRegion, KeyValue deleteKV, List indicesToConsider) + throws IOException { + + Get get = new Get(deleteKV.getRow()); + long maxTS = HConstants.LATEST_TIMESTAMP; + + if (deleteKV.getTimestamp() < maxTS) { + // Add +1 to make the current get includes the timestamp + maxTS = deleteKV.getTimestamp() + 1; + } + get.setTimeRange(HConstants.OLDEST_TIMESTAMP, maxTS); + + for (IndexSpecification index : indexSpecs) { + // Get all indices involves this family/qualifier + if (index.contains(deleteKV.getFamily(), deleteKV.getQualifier())) { + indicesToConsider.add(index); + for (ColumnQualifier cq : index.getIndexColumns()) { + get.addColumn(cq.getColumnFamily(), cq.getQualifier()); + } + } + } + if (deleteKV.isDeleteType()) { + get.setMaxVersions(1); + } else if (deleteKV.isDeleteColumnOrFamily()) { + get.setMaxVersions(); + } + List userKVs = userRegion.get(get, 0).list(); + + // Group KV based on timestamp + Multimap groupedKV = HashMultimap.create(); + + if (userKVs != null) { + for (KeyValue userKV : userKVs) { + groupedKV.put(userKV.getTimestamp(), userKV); + } + } + return groupedKV; + } + + // collection of edits for index table's memstore and WAL + public static class IndexEdits { + private WALEdit walEdit = new WALEdit(); + private HRegion indexRegion; + private boolean updatesLocked = false; + + /** + * Collection of mutations with locks. Locks will be null always as they not yet acquired for + * index table. + * @see HRegion#batchMutate(Pair[]) + */ + private List> mutations = new ArrayList>(); + + public WALEdit getWALEdit() { + return this.walEdit; + } + + public boolean isUpdatesLocked() { + return this.updatesLocked; + } + + public void setUpdateLocked() { + updatesLocked = true; + } + + public void add(Mutation mutation) { + // Check if WAL is disabled + for (List kvs : mutation.getFamilyMap().values()) { + for (KeyValue kv : kvs) { + this.walEdit.add(kv); + } + } + // There is no lock acquired for index table. So, set it to null + this.mutations.add(new Pair(mutation, null)); + } + + public void addAll(Collection mutations) { + for (Mutation mutation : mutations) { + add(mutation); + } + } + + public List> getIndexMutations() { + return this.mutations; + } + + public HRegion getRegion() { + return this.indexRegion; + } + } + + @Override + public void postBatchMutate(final ObserverContext ctx, + final List mutations, WALEdit walEdit) { + HTableDescriptor userTableDesc = ctx.getEnvironment().getRegion().getTableDesc(); + String tableName = userTableDesc.getNameAsString(); + if (IndexUtils.isCatalogTable(userTableDesc.getName()) || IndexUtils.isIndexTable(tableName)) { + return; + } + List indices = indexManager.getIndicesForTable(tableName); + if (indices == null || indices.isEmpty()) { + LOG.trace("skipping postBatchMutate for the table " + tableName + " as there are no indices"); + return; + } + LOG.trace("Entering postBatchMutate for the table " + tableName); + IndexEdits indexEdits = threadLocal.get(); + List> indexMutations = indexEdits.getIndexMutations(); + + if (indexMutations.size() == 0) { + return; + } + HRegion hr = indexEdits.getRegion(); + LOG.trace("Updating index table " + hr.getRegionInfo().getTableNameAsString()); + try { + hr.batchMutateForIndex(indexMutations.toArray(new Pair[indexMutations.size()])); + } catch (IOException e) { + // TODO This can come? If so we need to revert the actual put + // and make the op failed. + LOG.error("Error putting data into the index region", e); + } + LOG.trace("Exiting postBatchMutate for the table " + tableName); + } + + @Override + public void postCompleteBatchMutate(final ObserverContext ctx, + List mutations) throws IOException { + IndexEdits indexEdits = threadLocal.get(); + if (indexEdits != null) { + if (indexEdits.isUpdatesLocked()) { + indexEdits.getRegion().releaseLock(); + } + + } + threadLocal.remove(); + + } + + @Override + public boolean postFilterRow(ObserverContext ctx, + InternalScanner s, byte[] currentRow) throws IOException { + String tableName = ctx.getEnvironment().getRegion().getTableDesc().getNameAsString(); + if (IndexUtils.isIndexTable(tableName)) { + return true; + } + SeekAndReadRegionScanner bsrs = SeekAndReadRegionScannerHolder.getRegionScanner(); + if (bsrs != null) { + while (false == bsrs.seekToNextPoint()) { + SeekPointFetcher seekPointFetcher = scannerMap.get(bsrs); + if (null != seekPointFetcher) { + List seekPoints = new ArrayList(1); + seekPointFetcher.nextSeekPoints(seekPoints, 1); + // TODO use return boolean? + if (seekPoints.isEmpty()) { + LOG.trace("No seekpoints are remaining hence returning.. "); + return false; + } + bsrs.addSeekPoints(seekPoints); + if (isTestingEnabled) { + setSeekPoints(seekPoints); + setSeekpointAdded(true); + addSeekPoints(seekPoints); + } + } else { + // This will happen for a region with no index + break; + } + } + } + return true; + } + + public RegionScanner postScannerOpen(ObserverContext e, Scan scan, + RegionScanner s) { + HRegion region = e.getEnvironment().getRegion(); + String tableName = region.getTableDesc().getNameAsString(); + HRegionServer rs = (HRegionServer) e.getEnvironment().getRegionServerServices(); + // If the passed region is a region from an indexed table + SeekAndReadRegionScanner bsrs = null; + + try { + List indexlist = IndexManager.getInstance().getIndicesForTable(tableName); + if (indexlist != null) { + if (indexlist == null || indexlist.isEmpty()) { + // Not an indexed table. Just return. + return s; + } + LOG.trace("Entering postScannerOpen for the table " + tableName); + Collection onlineRegions = rs.getOnlineRegionsLocalContext(); + for (HRegion onlineIdxRegion : onlineRegions) { + if (IndexUtils.isCatalogTable(Bytes.toBytes(onlineIdxRegion.getTableDesc() + .getNameAsString()))) { + continue; + } + if (onlineIdxRegion.equals(region)) { + continue; + } + if (Bytes.equals(onlineIdxRegion.getStartKey(), region.getStartKey()) + && Bytes.equals(Bytes.toBytes(IndexUtils.getIndexTableName(region.getTableDesc() + .getNameAsString())), onlineIdxRegion.getTableDesc().getName())) { + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + IndexRegionScanner indexScanner = + mapper.evaluate(scan, indexlist, onlineIdxRegion.getStartKey(), onlineIdxRegion, + tableName); + if (indexScanner == null) return s; + SeekPointFetcher spf = new SeekPointFetcher(indexScanner); + ReInitializableRegionScanner reinitializeScanner = + new ReInitializableRegionScannerImpl(s, scan, spf); + bsrs = new BackwardSeekableRegionScanner(reinitializeScanner, scan, region, null); + scannerMap.put(bsrs, spf); + LOG.trace("Scanner Map has " + scannerMap); + break; + } + } + LOG.trace("Exiting postScannerOpen for the table " + tableName); + } + } catch (Exception ex) { + LOG.error("Exception occured in postScannerOpen for the table " + tableName, ex); + } + if (bsrs != null) { + return bsrs; + } else { + return s; + } + } + + public boolean preScannerNext(ObserverContext e, InternalScanner s, + List results, int nbRows, boolean hasMore) throws IOException { + HRegion region = e.getEnvironment().getRegion(); + String tableName = region.getTableDesc().getNameAsString(); + try { + if (s instanceof SeekAndReadRegionScanner) { + LOG.trace("Entering preScannerNext for the table " + tableName); + BackwardSeekableRegionScanner bsrs = (BackwardSeekableRegionScanner) s; + SeekAndReadRegionScannerHolder.setRegionScanner(bsrs); + SeekPointFetcher spf = scannerMap.get(bsrs); + List seekPoints = null; + if (spf != null) { + if (isTestingEnabled) { + setIndexedFlowUsed(true); + } + seekPoints = new ArrayList(); + spf.nextSeekPoints(seekPoints, nbRows); + } + if (seekPoints == null || seekPoints.isEmpty()) { + LOG.trace("No seekpoints are remaining hence returning.. "); + SeekAndReadRegionScannerHolder.removeRegionScanner(); + e.bypass(); + return false; + } + bsrs.addSeekPoints(seekPoints); + // This setting is just for testing purpose + if (isTestingEnabled) { + setSeekPoints(seekPoints); + setSeekpointAdded(true); + addSeekPoints(seekPoints); + } + LOG.trace("Exiting preScannerNext for the table " + tableName); + } + } catch (Exception ex) { + LOG.error("Exception occured in preScannerNext for the table " + tableName + ex); + } + return true; + } + + @Override + public boolean postScannerNext(ObserverContext e, + InternalScanner s, List results, int limit, boolean hasMore) throws IOException { + if (s instanceof SeekAndReadRegionScanner) { + SeekAndReadRegionScannerHolder.removeRegionScanner(); + } + return true; + } + + @Override + public void preScannerClose(ObserverContext e, InternalScanner s) + throws IOException { + if (s instanceof BackwardSeekableRegionScanner) { + scannerMap.remove((RegionScanner) s); + } + } + + @Override + public SplitInfo preSplitBeforePONR(ObserverContext e, + byte[] splitKey) throws IOException { + RegionCoprocessorEnvironment environment = e.getEnvironment(); + HRegionServer rs = (HRegionServer) environment.getRegionServerServices(); + HRegion region = environment.getRegion(); + String userTableName = region.getTableDesc().getNameAsString(); + LOG.trace("Entering preSplitBeforePONR for the table " + userTableName + " for the region " + + region.getRegionInfo()); + String indexTableName = IndexUtils.getIndexTableName(userTableName); + if (indexManager.getIndicesForTable(userTableName) != null) { + HRegion indexRegion = null; + SplitTransaction st = null; + try { + indexRegion = getIndexRegion(rs, region.getStartKey(), indexTableName); + if (null != indexRegion) { + LOG.info("Flushing the cache for the index table " + indexTableName + " for the region " + + indexRegion.getRegionInfo()); + indexRegion.flushcache(); + if (LOG.isInfoEnabled()) { + LOG.info("Forcing split for the index table " + indexTableName + " with split key " + + Bytes.toString(splitKey)); + } + st = new SplitTransaction(indexRegion, splitKey); + if (!st.prepare()) { + LOG.error("Prepare for the index table " + indexTableName + + " failed. So returning null. "); + return null; + } + indexRegion.forceSplit(splitKey); + PairOfSameType daughterRegions = st.stepsBeforeAddingPONR(rs, rs, false); + SplitInfo splitInfo = splitThreadLocal.get(); + splitInfo.setDaughtersAndTransaction(daughterRegions, st); + LOG.info("Daughter regions created for the index table " + indexTableName + + " for the region " + indexRegion.getRegionInfo()); + return splitInfo; + } else { + LOG.error("IndexRegion for the table " + indexTableName + " is null. So returning null. "); + return null; + } + } catch (Exception ex) { + LOG.error("Error while spliting the indexTabRegion or not able to get the indexTabRegion:" + + indexRegion != null ? indexRegion.getRegionName() : "", ex); + st.rollback(rs, rs); + return null; + } + } + LOG.trace("Indexes for the table " + userTableName + + " are null. So returning the empty SplitInfo"); + return new SplitInfo(); + } + + @Override + public void preSplit(ObserverContext e) throws IOException { + if (splitThreadLocal != null) { + splitThreadLocal.remove(); + splitThreadLocal.set(new SplitInfo()); + } + } + + private HRegion getIndexRegion(HRegionServer rs, byte[] startKey, String indexTableName) + throws IOException { + List indexTabRegions = rs.getOnlineRegions(Bytes.toBytes(indexTableName)); + for (HRegion indexRegion : indexTabRegions) { + if (Bytes.equals(startKey, indexRegion.getStartKey())) { + return indexRegion; + } + } + return null; + } + + @Override + public void postSplit(ObserverContext e, HRegion l, HRegion r) + throws IOException { + RegionCoprocessorEnvironment environment = e.getEnvironment(); + HRegionServer rs = (HRegionServer) environment.getRegionServerServices(); + HRegion region = environment.getRegion(); + String userTableName = region.getTableDesc().getNameAsString(); + String indexTableName = IndexUtils.getIndexTableName(userTableName); + if (IndexUtils.isIndexTable(userTableName)) { + return; + } + LOG.trace("Entering postSplit for the table " + userTableName + " for the region " + + region.getRegionInfo()); + IndexManager indexManager = IndexManager.getInstance(); + SplitTransaction splitTransaction = null; + if (indexManager.getIndicesForTable(userTableName) != null) { + try { + SplitInfo splitInfo = splitThreadLocal.get(); + splitTransaction = splitInfo.getSplitTransaction(); + PairOfSameType daughters = splitInfo.getDaughters(); + if (splitTransaction != null && daughters != null) { + splitTransaction.stepsAfterPONR(rs, rs, daughters); + LOG.info("Daughter regions are opened and split transaction finished for zknodes for index table " + + indexTableName + " for the region " + region.getRegionInfo()); + } + } catch (Exception ex) { + String msg = + "Splitting of index region has failed in stepsAfterPONR stage so aborting the server"; + LOG.error(msg, ex); + rs.abort(msg); + } + } + } + + // A thread local variable used to get the splitted region information of the index region. + // This is needed becuase in order to do the PONR entry we need the info of the index + // region's daughter entries. + public static final ThreadLocal splitThreadLocal = new ThreadLocal() { + protected SplitInfo initialValue() { + return null; + }; + }; + + @Override + public void preRollBack(ObserverContext ctx) throws IOException { + RegionCoprocessorEnvironment environment = ctx.getEnvironment(); + HRegionServer rs = (HRegionServer) environment.getRegionServerServices(); + HRegion region = environment.getRegion(); + String userTableName = region.getTableDesc().getNameAsString(); + if (IndexUtils.isIndexTable(userTableName)) { + return; + } + LOG.trace("Entering preRollBack for the table " + userTableName + " for the region " + + region.getRegionInfo()); + SplitInfo splitInfo = splitThreadLocal.get(); + SplitTransaction splitTransaction = splitInfo.getSplitTransaction(); + try { + if (splitTransaction != null) { + splitTransaction.rollback(rs, rs); + LOG.info("preRollBack successfully done for the table " + userTableName + + " for the region " + region.getRegionInfo()); + } + } catch (Exception e) { + LOG.error( + "Error while rolling back the split failure for index region " + + splitTransaction.getParent(), e); + rs.abort("Abort; we got an error during rollback of index"); + } + } + + // For testing to check whether final step of seek point is added + public static void setSeekpointAdded(boolean isSeekpointAddded) { + IndexRegionObserver.isSeekpointAddded = isSeekpointAddded; + } + + // For testing + public static boolean getSeekpointAdded() { + return isSeekpointAddded; + } + + // For testing to ensure indexed flow is used or not + public static void setIndexedFlowUsed(boolean isIndexedFlowUsed) { + IndexRegionObserver.isIndexedFlowUsed = isIndexedFlowUsed; + } + + // For testing + public static boolean getIndexedFlowUsed() { + return isIndexedFlowUsed; + } + + // For testing + static List getSeekpoints() { + return seekPoints; + } + + // For testing to ensure cache size is returned correctly + public static void setSeekPoints(List seekPoints) { + IndexRegionObserver.seekPoints = seekPoints; + } + + public static void setIsTestingEnabled(boolean isTestingEnabled) { + IndexRegionObserver.isTestingEnabled = isTestingEnabled; + } + + public static void addSeekPoints(List seekPoints) { + if (seekPoints == null) { + IndexRegionObserver.seekPointsForMultipleIndices = null; + return; + } + if (IndexRegionObserver.seekPointsForMultipleIndices == null) { + IndexRegionObserver.seekPointsForMultipleIndices = new ArrayList(); + } + IndexRegionObserver.seekPointsForMultipleIndices.addAll(seekPoints); + } + + public static List getMultipleSeekPoints() { + return IndexRegionObserver.seekPointsForMultipleIndices; + } + + private static class SeekAndReadRegionScannerHolder { + private static ThreadLocal holder = + new ThreadLocal(); + + public static void setRegionScanner(SeekAndReadRegionScanner scanner) { + holder.set(scanner); + } + + public static SeekAndReadRegionScanner getRegionScanner() { + return holder.get(); + } + + public static void removeRegionScanner() { + holder.remove(); + } + } + + @Override + public InternalScanner preCompactScannerOpen(ObserverContext c, + Store store, List scanners, ScanType scanType, long earliestPutTs, + InternalScanner s) throws IOException { + HRegionServer rs = (HRegionServer) c.getEnvironment().getRegionServerServices(); + if (!store.getTableName().contains(Constants.INDEX_TABLE_SUFFIX)) { + // Not an index table + return null; + } + long smallestReadPoint = c.getEnvironment().getRegion().getSmallestReadPoint(); + String actualTableName = IndexUtils.getActualTableNameFromIndexTableName(store.getTableName()); + TTLStoreScanner ttlStoreScanner = + new TTLStoreScanner(store, smallestReadPoint, earliestPutTs, scanType, scanners, + new TTLExpiryChecker(), actualTableName, rs); + return ttlStoreScanner; + } + + @Override + public void postClose(ObserverContext e, boolean abortRequested) { + HRegion region = e.getEnvironment().getRegion(); + byte[] tableName = region.getRegionInfo().getTableName(); + if (IndexUtils.isCatalogTable(tableName) || IndexUtils.isIndexTable(tableName)) { + return; + } + if (splitThreadLocal.get() == null) { + this.indexManager.decrementRegionCount(Bytes.toString(tableName), true); + } else { + this.indexManager.decrementRegionCount(Bytes.toString(tableName), false); + } + } + + private boolean isValidIndexMutation(HTableDescriptor userTableDesc, String tableName) { + if (IndexUtils.isCatalogTable(userTableDesc.getName()) || IndexUtils.isIndexTable(tableName)) { + return false; + } + List indices = indexManager.getIndicesForTable(tableName); + if (indices == null || indices.isEmpty()) { + LOG.trace("skipping preBatchMutate for the table " + tableName + " as there are no indices"); + return false; + } + return true; + } + + private void acquireLockOnIndexRegion(String tableName, HRegion userRegion, HRegionServer rs) + throws IOException { + HRegion indexRegion = getIndexTableRegion(tableName, userRegion, rs); + indexRegion.checkResources(); + indexRegion.startRegionOperation(); + } + + @Override + public void postCloseRegionOperation(ObserverContext e) + throws IOException { + HRegionServer rs = (HRegionServer) e.getEnvironment().getRegionServerServices(); + HRegion userRegion = e.getEnvironment().getRegion(); + HTableDescriptor userTableDesc = userRegion.getTableDesc(); + String tableName = userTableDesc.getNameAsString(); + if (!isValidIndexMutation(userTableDesc, tableName)) { + // Ideally need not release any lock because in the preStartRegionOperationHook we would not + // have + // acquired + // any lock on the index region + return; + } + HRegion indexRegion = getIndexTableRegion(tableName, userRegion, rs); + // This check for isClosed and isClosing is needed because we should not unlock + // when the index region lock would have already been released before throwing NSRE + + // TODO : What is the scenario that i may get an IllegalMonitorStateException + if (!indexRegion.isClosed() || !indexRegion.isClosing()) { + indexRegion.closeRegionOperation(); + } + } + + @Override + public void postStartRegionOperation(ObserverContext e) + throws IOException { + HRegionServer rs = (HRegionServer) e.getEnvironment().getRegionServerServices(); + HRegion userRegion = e.getEnvironment().getRegion(); + HTableDescriptor userTableDesc = userRegion.getTableDesc(); + String tableName = userTableDesc.getNameAsString(); + if (!isValidIndexMutation(userTableDesc, tableName)) { + return; + } + acquireLockOnIndexRegion(tableName, userRegion, rs); + + } + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScanner.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScanner.java new file mode 100644 index 0000000..7941337 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScanner.java @@ -0,0 +1,38 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import org.apache.hadoop.hbase.regionserver.RegionScanner; + +public interface IndexRegionScanner extends RegionScanner { + + public void advance(); + + public void setRangeFlag(boolean range); + + public boolean isRange(); + + public void setScannerIndex(int index); + + public int getScannerIndex(); + + public boolean hasChildScanners(); + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForAND.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForAND.java new file mode 100644 index 0000000..10da954 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForAND.java @@ -0,0 +1,317 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +public class IndexRegionScannerForAND implements IndexRegionScanner { + + private static final Log LOG = LogFactory.getLog(IndexRegionScannerForAND.class); + + private List scanners = null; + + private Map, Integer>> rowCache = + new HashMap, Integer>>(); + + private int scannersCount = 0; + + boolean hasRangeScanners = false; + + private int scannerIndex = -1; + + public IndexRegionScannerForAND(List scanners) { + this.scanners = scanners; + scannersCount = scanners.size(); + } + + @Override + public void advance() { + for (IndexRegionScanner scn : this.scanners) { + scn.advance(); + } + } + + @Override + public HRegionInfo getRegionInfo() { + return null; + } + + @Override + public boolean isFilterDone() { + return false; + } + + @Override + public boolean hasChildScanners() { + return scannersCount != 0; + }; + + @Override + public void setRangeFlag(boolean range) { + hasRangeScanners = range; + } + + @Override + public boolean isRange() { + return this.hasRangeScanners; + } + + @Override + public void setScannerIndex(int index) { + scannerIndex = index; + } + + @Override + public int getScannerIndex() { + return scannerIndex; + } + + @Override + public synchronized boolean reseek(byte[] row) throws IOException { + // ideally reseek on AND may not come as AND cannot be a child of another AND. + if (this.scanners.isEmpty()) return false; + for (IndexRegionScanner scn : this.scanners) { + boolean reseek = scn.reseek(row); + if (!reseek) return false; + } + return true; + } + + @Override + public synchronized void close() throws IOException { + for (IndexRegionScanner scn : this.scanners) { + scn.close(); + } + this.scanners.clear(); + } + + @Override + public synchronized boolean next(List results) throws IOException { + if (this.scanners != null && !this.scanners.isEmpty()) { + List> valueList = + new ArrayList>(); + byte[] maxRowKey = null; + while (results.size() < 1) { + List intermediateResult = new ArrayList(); + boolean haveSameRows = true; + KeyValue kv = null; + Iterator scnItr = this.scanners.iterator(); + while (scnItr.hasNext()) { + IndexRegionScanner scn = scnItr.next(); + if (!hasRangeScanners) { + if (checkForScanner(valueList, scn.getScannerIndex())) continue; + } + boolean hasMore = scn.next(intermediateResult); + if (!hasRangeScanners) { + if (intermediateResult != null && !intermediateResult.isEmpty()) { + byte[] rowKey = IndexUtils.getRowKeyFromKV(intermediateResult.get(0)); + if (maxRowKey == null) { + maxRowKey = rowKey; + } else { + int result = Bytes.compareTo(maxRowKey, rowKey); + if (haveSameRows) haveSameRows = (result == 0); + maxRowKey = result > 0 ? maxRowKey : rowKey; + } + if (kv == null) kv = intermediateResult.get(0); + intermediateResult.clear(); + valueList.add(new Pair(rowKey, scn)); + } + } else { + if (!intermediateResult.isEmpty()) { + boolean matching = + checkAndPutMatchingEntry(intermediateResult.get(0), scn.getScannerIndex()); + if (matching) { + results.addAll(intermediateResult); + } + intermediateResult.clear(); + } + } + if (!hasMore) { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing scanner " + scn + " from the list."); + } + scn.close(); + scnItr.remove(); + if (hasRangeScanners) { + // TODO: we can remove unnecessary rows(which never become matching entries) from + // cache on scanner close. + if (this.scanners.isEmpty()) return false; + } + if (results.size() > 0) { + break; + } + } + if (results.size() > 0) { + return !this.scanners.isEmpty(); + } + } + if (!hasRangeScanners) { + if (haveSameRows && valueList.size() == scannersCount) { + if (kv != null) results.add(kv); + return this.scanners.size() == scannersCount; + } else if (haveSameRows && valueList.size() != scannersCount) { + close(); + return false; + } else { + // In case of AND if the reseek on any one scanner returns false + // we can close the entire scanners in the AND subtree + if (!reseekTheScanners(valueList, maxRowKey)) { + close(); + return false; + } + } + } + } + if (hasRangeScanners) { + return true; + } else { + return this.scanners.size() == scannersCount; + } + } + return false; + } + + private boolean checkForScanner(List> valueList, int scnIndex) { + for (Pair value : valueList) { + if (value.getSecond().getScannerIndex() == scnIndex) { + return true; + } + } + return false; + } + + private boolean checkAndPutMatchingEntry(KeyValue intermediateResult, int index) { + String rowKeyFromKV = Bytes.toString(IndexUtils.getRowKeyFromKV(intermediateResult)); + Pair, Integer> countPair = rowCache.get(rowKeyFromKV); + if (countPair == null) { + // If present scanners count is not equal to actual scanners count,no need to put row key into + // cache because this will never become matching entry. + if (this.scanners.size() == scannersCount) { + List scannerFlags = new ArrayList(scannerIndex); + for (int i = 0; i < scannersCount; i++) { + scannerFlags.add(false); + } + countPair = new Pair, Integer>(scannerFlags, 0); + // TODO verify + // rowCache.put(rowKeyFromKV, countPair); + } else { + return false; + } + } + Boolean indexFlag = countPair.getFirst().get(index); + Integer countObj = countPair.getSecond(); + // If count is equal to scanner count before increment means its returned already. skip the + // result. + if (countObj == scannersCount) { + return false; + } + if (!indexFlag) { + countObj++; + countPair.getFirst().set(index, true); + countPair.setSecond(countObj); + rowCache.put(rowKeyFromKV, countPair); + } + if (countObj == scannersCount) { + return true; + } else { + // If the difference between actual scanner count(scannerCount) and number of scanners have + // this row(countObj) is more than present scanners size then remove row from cache because + // this never be maching entry. + if ((scannersCount - countObj) > this.scanners.size()) { + rowCache.remove(rowKeyFromKV); + return false; + } + } + return false; + } + + private boolean reseekTheScanners(List> valueList, + byte[] maxRowKey) throws IOException { + Iterator> itr = valueList.iterator(); + while (itr.hasNext()) { + Pair rowVsScanner = itr.next(); + IndexRegionScanner scn = rowVsScanner.getSecond(); + // We need to call reseek on OR scanner even the last returned row key is equal or more than + // max row key to set reseek flag. + if (scn instanceof IndexRegionScannerForOR) { + rowVsScanner.getSecond().reseek(maxRowKey); + itr.remove(); + continue; + } + if (Bytes.compareTo(rowVsScanner.getFirst(), maxRowKey) < 0) { + if (!scn.reseek(maxRowKey)) { + return false; + } + itr.remove(); + } + } + return true; + } + + @Override + public boolean next(List result, int limit) throws IOException { + // TODO Auto-generated method stub + return false; + } + + @Override + public long getMvccReadPoint() { + // TODO Implement RegionScanner.getMvccReadPoint + return 0; + } + + @Override + public boolean nextRaw(List result, String metric) throws IOException { + // TODO Implement RegionScanner.nextRaw + return false; + } + + @Override + public boolean nextRaw(List result, int limit, String metric) throws IOException { + // TODO Implement RegionScanner.nextRaw + return false; + } + + @Override + public boolean next(List results, String metric) throws IOException { + // TODO Implement InternalScanner.next + return false; + } + + @Override + public boolean next(List result, int limit, String metric) throws IOException { + // TODO Implement InternalScanner.next + return false; + } + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForOR.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForOR.java new file mode 100644 index 0000000..b1dfcbe --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForOR.java @@ -0,0 +1,335 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +public class IndexRegionScannerForOR implements IndexRegionScanner { + + private static final Log LOG = LogFactory.getLog(IndexRegionScannerForOR.class); + + private List scanners = null; + + // private Pair lastReturnedValue = null; + + private byte[] lastReturnedValue = null; + + // Cache to store the values retrieved by range scans + private Map rowCache = new HashMap(); + + private Map> valueMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); + + private boolean hasRangeScanners = false; + + private boolean isRootScanner = false; + + private int scannerIndex = -1; + + private boolean firstScan = true; + + private boolean reseeked = false; + + public IndexRegionScannerForOR(List scanners) { + this.scanners = scanners; + } + + @Override + public void advance() { + if (this.lastReturnedValue != null) { + // this.lastReturnedValue.getFirst().advance(); + } + this.lastReturnedValue = null; + } + + @Override + public HRegionInfo getRegionInfo() { + return null; + } + + @Override + public boolean isFilterDone() { + return false; + } + + @Override + public void setRangeFlag(boolean range) { + hasRangeScanners = range; + } + + public void setRootFlag(boolean isRootScanner) { + this.isRootScanner = isRootScanner; + } + + @Override + public boolean isRange() { + return this.hasRangeScanners; + } + + @Override + public void setScannerIndex(int index) { + scannerIndex = index; + } + + @Override + public int getScannerIndex() { + return scannerIndex; + } + + @Override + public boolean hasChildScanners() { + return !scanners.isEmpty(); + } + + @Override + public synchronized boolean reseek(byte[] row) throws IOException { + boolean success = false; + if (!valueMap.isEmpty()) { + Iterator>> itr = + valueMap.entrySet().iterator(); + while (itr.hasNext()) { + Entry> entry = itr.next(); + IndexRegionScanner scn = entry.getValue().getFirst(); + if (Bytes.compareTo(entry.getKey(), row) < 0) { + // If the reseek does not retrieve any row then it means we have reached the end of the + // scan. So this scanner can be safely removed. + if (!scn.reseek(row)) { + removeScanner(scn.getScannerIndex()); + } + itr.remove(); + } else { + break; + } + } + } + reseeked = true; + this.lastReturnedValue = null; + return success; + } + + private void removeScanner(int scnIndex) { + Iterator itr = this.scanners.iterator(); + while (itr.hasNext()) { + if (itr.next().getScannerIndex() == scnIndex) { + itr.remove(); + break; + } + } + } + + @Override + public synchronized void close() throws IOException { + for (IndexRegionScanner scn : this.scanners) { + scn.close(); + } + this.valueMap.clear(); + this.scanners.clear(); + this.lastReturnedValue = null; + } + + @Override + public synchronized boolean next(List results) throws IOException { + OUTER: while (results.size() < 1) { + List intermediateResult = new ArrayList(); + if (hasRangeScanners || firstScan || reseeked) { + Iterator scnItr = this.scanners.iterator(); + INNER: while (scnItr.hasNext()) { + IndexRegionScanner scn = scnItr.next(); + if (reseeked) { + boolean exists = checkForScanner(scn.getScannerIndex()); + if (exists) continue; + } + boolean hasMore = scn.next(intermediateResult); + if (!hasRangeScanners) { + if (intermediateResult != null && intermediateResult.size() > 0) { + byte[] rowKeyFromKV = IndexUtils.getRowKeyFromKV(intermediateResult.get(0)); + while (valueMap.containsKey(rowKeyFromKV)) { + intermediateResult.clear(); + hasMore = scn.next(intermediateResult); + if (!intermediateResult.isEmpty()) { + rowKeyFromKV = IndexUtils.getRowKeyFromKV(intermediateResult.get(0)); + } else { + break; + } + } + if (!hasMore && intermediateResult.isEmpty()) { + // Allow other scanners to scan. Nothing to do. + } else { + valueMap.put(rowKeyFromKV, new Pair(scn, + intermediateResult.get(0))); + intermediateResult.clear(); + } + } + } + if (!hasMore) { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing scanner " + scn + " from the list."); + } + scn.close(); + scnItr.remove(); + } + if (hasRangeScanners) { + if (!intermediateResult.isEmpty()) { + String rowKey = Bytes.toString(IndexUtils.getRowKeyFromKV(intermediateResult.get(0))); + if (isRootScanner && !rowCache.containsKey(rowKey)) { + rowCache.put(rowKey, false); + results.addAll(intermediateResult); + return !this.scanners.isEmpty(); + } else if (isRootScanner) { + // dont add to results because already exists and scan for other entry. + intermediateResult.clear(); + continue OUTER; + } else { + results.addAll(intermediateResult); + return !this.scanners.isEmpty(); + } + } + } + } + if (firstScan) firstScan = false; + if (reseeked) reseeked = false; + } else { + // Scan on previous scanner which returned minimum values. + Entry> minEntry = null; + if (!valueMap.isEmpty()) { + minEntry = valueMap.entrySet().iterator().next(); + IndexRegionScanner scn = minEntry.getValue().getFirst(); + if (Bytes.compareTo(lastReturnedValue, minEntry.getKey()) == 0) { + valueMap.remove(minEntry.getKey()); + boolean hasMore = scn.next(intermediateResult); + byte[] rowKeyFromKV = null; + if (!intermediateResult.isEmpty()) { + rowKeyFromKV = IndexUtils.getRowKeyFromKV(intermediateResult.get(0)); + while (valueMap.containsKey(rowKeyFromKV)) { + intermediateResult.clear(); + if (hasMore) { + hasMore = minEntry.getValue().getFirst().next(intermediateResult); + } + if (intermediateResult.isEmpty()) { + rowKeyFromKV = null; + scn.close(); + removeScanner(scn.getScannerIndex()); + break; + } + rowKeyFromKV = IndexUtils.getRowKeyFromKV(intermediateResult.get(0)); + } + } + if (rowKeyFromKV != null) { + valueMap.put(rowKeyFromKV, new Pair(scn, + intermediateResult.get(0))); + intermediateResult.clear(); + } + } + } else { + return false; + } + } + if (!valueMap.isEmpty()) { + Entry> minEntry = + valueMap.entrySet().iterator().next(); + lastReturnedValue = minEntry.getKey(); + results.add(minEntry.getValue().getSecond()); + return true; + } else { + return false; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug(results.size() + " seek points obtained. Values: " + + (!results.isEmpty() ? Bytes.toString(results.get(0).getRow()) : 0)); + } + return !results.isEmpty(); + } + + private boolean checkForScanner(int scnIndex) { + Collection> scanners = valueMap.values(); + for (Pair scn : scanners) { + if (scnIndex == scn.getFirst().getScannerIndex()) { + return true; + } + } + return false; + } + + /* + * private void getTheSmallerValue(KeyValue keyValue, IndexRegionScanner scn ) { byte[] currentRow + * = getRowKeyFromKV(keyValue); if(this.lastReturnedValue == null){ + * this.lastReturnedValue.setFirst(new Pair(currentRow,scn.getScannerIndex())); + * this.lastReturnedValue.setSecond(keyValue); } else { byte[] lastRow = + * getRowKeyFromKV(this.lastReturnedValue.getSecond()); if(Bytes.compareTo(currentRow, lastRow) < + * 0){ this.lastReturnedValue.setFirst(new Pair(currentRow,scn.getScannerIndex())); this.lastReturnedValue.setSecond(keyValue); } // + * TODO: In case of equal need to check what to do? if(Bytes.compareTo(currentRow, lastRow) == 0){ + * scn.advance(); } } } + */ + + @Override + public boolean next(List result, int limit) throws IOException { + // TODO Auto-generated method stub + return false; + } + + @Override + public long getMvccReadPoint() { + // TODO Implement RegionScanner.getMvccReadPoint + return 0; + } + + @Override + public boolean nextRaw(List result, String metric) throws IOException { + // TODO Implement RegionScanner.nextRaw + return false; + } + + @Override + public boolean nextRaw(List result, int limit, String metric) throws IOException { + // TODO Implement RegionScanner.nextRaw + return false; + } + + @Override + public boolean next(List results, String metric) throws IOException { + // TODO Implement InternalScanner.next + return false; + } + + @Override + public boolean next(List result, int limit, String metric) throws IOException { + // TODO Implement InternalScanner.next + return false; + } + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafFilterNode.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafFilterNode.java new file mode 100644 index 0000000..ea7f06b --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafFilterNode.java @@ -0,0 +1,30 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import org.apache.hadoop.hbase.index.IndexSpecification; + +interface LeafFilterNode extends FilterNode { + FilterColumnValueDetail getFilterColumnValueDetail(); + + void setFilterColumnValueDetail(FilterColumnValueDetail filterColumnValueDetail); + + IndexSpecification getBestIndex(); +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafIndexRegionScanner.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafIndexRegionScanner.java new file mode 100644 index 0000000..161a220 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafIndexRegionScanner.java @@ -0,0 +1,185 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; + +public class LeafIndexRegionScanner implements IndexRegionScanner { + + private static final Log LOG = LogFactory.getLog(LeafIndexRegionScanner.class); + + private RegionScanner delegator = null; + + private KeyValue currentKV = null; + + private boolean hadMore = true; + + private final IndexSpecification index; + + private boolean isRangeScanner = false; + + private TTLExpiryChecker ttlExpiryChecker; + + private int scannerIndex = -1; + + public LeafIndexRegionScanner(IndexSpecification index, RegionScanner delegator, + TTLExpiryChecker ttlExpiryChecker) { + this.delegator = delegator; + this.index = index; + this.ttlExpiryChecker = ttlExpiryChecker; + } + + @Override + public void advance() { + this.currentKV = null; + } + + @Override + public HRegionInfo getRegionInfo() { + return this.delegator.getRegionInfo(); + } + + @Override + public boolean isFilterDone() { + return this.delegator.isFilterDone(); + } + + @Override + public void setRangeFlag(boolean range) { + isRangeScanner = range; + } + + @Override + public boolean isRange() { + return this.isRangeScanner; + } + + @Override + public void setScannerIndex(int index) { + scannerIndex = index; + } + + @Override + public int getScannerIndex() { + return scannerIndex; + } + + @Override + public boolean hasChildScanners() { + return false; + } + + // TODO the passing row to be the full key in the index table. + // The callee need to take care of this creation.. + @Override + public synchronized boolean reseek(byte[] row) throws IOException { + if (!hadMore) return false; + byte[] targetRowKey = createRowKeyForReseek(row); + return this.delegator.reseek(targetRowKey); + } + + private byte[] createRowKeyForReseek(byte[] targetRow) { + byte[] curRK = this.currentKV.getRow(); + byte[] curValue = this.currentKV.getValue(); + short actualTabRKOffset = Bytes.toShort(curValue, 2); + byte[] newRowKey = new byte[actualTabRKOffset + targetRow.length]; + System.arraycopy(curRK, 0, newRowKey, 0, actualTabRKOffset); + System.arraycopy(targetRow, 0, newRowKey, actualTabRKOffset, targetRow.length); + return newRowKey; + } + + @Override + public synchronized void close() throws IOException { + this.delegator.close(); + } + + @Override + public synchronized boolean next(List results) throws IOException { + boolean hasMore = false; + do { + // this check here will prevent extra next call when in the previous + // next last row was fetched and after that an advance was called on this + // scanner. So instead of making a next call again we can return from here. + if (!this.hadMore) return false; + hasMore = this.delegator.next(results); + if (results != null && results.size() > 0) { + KeyValue kv = results.get(0); + if (this.ttlExpiryChecker.checkIfTTLExpired(this.index.getTTL(), kv.getTimestamp())) { + results.clear(); + LOG.info("The ttl has expired for the kv " + kv); + } else { + if (!isRangeScanner) { + // This is need to reseek in case of EQUAL scanners. + this.currentKV = kv; + break; + } + } + } + } while (results.size() < 1 && hasMore); + this.hadMore = hasMore; + return hasMore; + } + + @Override + public boolean next(List result, int limit) throws IOException { + // We wont call this method at all.. As we have only CF:qualifier per row in index table. + throw new UnsupportedOperationException("Use only next(List results) method."); + } + + @Override + public long getMvccReadPoint() { + // TODO Implement RegionScanner.getMvccReadPoint + return 0; + } + + @Override + public boolean nextRaw(List result, String metric) throws IOException { + // TODO Implement RegionScanner.nextRaw + return false; + } + + @Override + public boolean nextRaw(List result, int limit, String metric) throws IOException { + // TODO Implement RegionScanner.nextRaw + return false; + } + + @Override + public boolean next(List results, String metric) throws IOException { + // TODO Implement InternalScanner.next + return false; + } + + @Override + public boolean next(List result, int limit, String metric) throws IOException { + // TODO Implement InternalScanner.next + return false; + } + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NoIndexFilterNode.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NoIndexFilterNode.java new file mode 100644 index 0000000..dfed816 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NoIndexFilterNode.java @@ -0,0 +1,45 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.util.Pair; + +public class NoIndexFilterNode implements FilterNode { + + @Override + public Map, IndexSpecification> getIndexToUse() { + return null; + } + + @Override + public Map>> getPossibleUseIndices() { + return null; + } + + @Override + public Map>> getPossibleFutureUseIndices() { + return null; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NonLeafFilterNode.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NonLeafFilterNode.java new file mode 100644 index 0000000..d493a27 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NonLeafFilterNode.java @@ -0,0 +1,82 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.GroupingCondition; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.util.Pair; + +public class NonLeafFilterNode implements FilterNode { + + private List filterNodes = new ArrayList(); + + private GroupingCondition groupingCondition; + + private Map, IndexSpecification> indicesToUse = + new HashMap, IndexSpecification>(); + + public NonLeafFilterNode(GroupingCondition condition) { + this.groupingCondition = condition; + } + + public GroupingCondition getGroupingCondition() { + return groupingCondition; + } + + public List getFilterNodes() { + return filterNodes; + } + + public void addFilterNode(FilterNode filterNode) { + this.filterNodes.add(filterNode); + } + + public void addIndicesToUse(FilterColumnValueDetail f, IndexSpecification i) { + List key = new ArrayList(1); + key.add(f); + this.indicesToUse.put(key, i); + } + + public void addIndicesToUse(List lf, IndexSpecification i) { + this.indicesToUse.put(lf, i); + } + + @Override + public Map, IndexSpecification> getIndexToUse() { + return this.indicesToUse; + } + + @Override + public Map>> getPossibleUseIndices() { + return null; + } + + @Override + public Map>> getPossibleFutureUseIndices() { + // There is no question of future use possible indices on a non leaf node. + return null; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/PossibleIndexFilterNode.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/PossibleIndexFilterNode.java new file mode 100644 index 0000000..3b46687 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/PossibleIndexFilterNode.java @@ -0,0 +1,74 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.util.Pair; + +public class PossibleIndexFilterNode implements LeafFilterNode { + + private List> possibleFutureUseIndices; + + private FilterColumnValueDetail filterColumnValueDetail; + + public PossibleIndexFilterNode(List> possibleFutureUseIndices, + FilterColumnValueDetail filterColumnValueDetail) { + this.possibleFutureUseIndices = possibleFutureUseIndices; + this.filterColumnValueDetail = filterColumnValueDetail; + } + + @Override + public Map>> getPossibleFutureUseIndices() { + // TODO avoid create of Map instance all the time... + Map>> reply = + new HashMap>>(); + reply.put(filterColumnValueDetail.getColumn(), possibleFutureUseIndices); + return reply; + } + + @Override + public Map, IndexSpecification> getIndexToUse() { + return null; + } + + public Map>> getPossibleUseIndices() { + return null; + } + + @Override + public FilterColumnValueDetail getFilterColumnValueDetail() { + return this.filterColumnValueDetail; + } + + @Override + public void setFilterColumnValueDetail(FilterColumnValueDetail filterColumnValueDetail) { + this.filterColumnValueDetail = filterColumnValueDetail; + } + + @Override + public IndexSpecification getBestIndex() { + return null; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScanner.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScanner.java new file mode 100644 index 0000000..a1528a8 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScanner.java @@ -0,0 +1,31 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; + +import org.apache.hadoop.hbase.regionserver.RegionScanner; + +public interface ReInitializableRegionScanner extends SeekAndReadRegionScanner { + + // TODO better name + void reInit(RegionScanner rs) throws IOException; + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScannerImpl.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScannerImpl.java new file mode 100644 index 0000000..40bddaa --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScannerImpl.java @@ -0,0 +1,186 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; + +// TODO better name? +public class ReInitializableRegionScannerImpl implements ReInitializableRegionScanner { + + private RegionScanner delegator; + + private int batch; + + private byte[] lastSeekedRowKey; + + private byte[] lastSeekAttemptedRowKey; + + private static final Log LOG = LogFactory.getLog(ReInitializableRegionScannerImpl.class); + + // Can this be queue? + // Criteria for the selection to be additional heap overhead wrt the object + // used cost in operation + private Set seekPoints; + + private SeekPointFetcher seekPointFetcher; + + private boolean closed = false; + + public ReInitializableRegionScannerImpl(RegionScanner delegator, Scan scan, + SeekPointFetcher seekPointFetcher) { + this.delegator = delegator; + this.batch = scan.getBatch(); + this.seekPoints = new TreeSet(Bytes.BYTES_COMPARATOR); + this.seekPointFetcher = seekPointFetcher; + } + + @Override + public HRegionInfo getRegionInfo() { + return this.delegator.getRegionInfo(); + } + + @Override + public boolean isFilterDone() { + return this.delegator.isFilterDone(); + } + + @Override + public synchronized void close() throws IOException { + try { + this.delegator.close(); + } finally { + this.seekPointFetcher.close(); + } + closed = true; + } + + @Override + public synchronized boolean next(List results) throws IOException { + return next(results, this.batch); + } + + public boolean isClosed() { + return closed; + } + + @Override + public synchronized boolean next(List result, int limit) throws IOException { + return next(result, limit, null); + } + + @Override + public void addSeekPoints(List seekPoints) { + // well this add will do the sorting and remove duplicates. :) + for (byte[] seekPoint : seekPoints) { + this.seekPoints.add(seekPoint); + } + } + + @Override + public boolean seekToNextPoint() throws IOException { + // At this class level if seek is called it must be forward seek. + // call reseek() directly with the next seek point + Iterator spIterator = this.seekPoints.iterator(); + if (spIterator.hasNext()) { + this.lastSeekAttemptedRowKey = spIterator.next(); + if (null != this.lastSeekedRowKey + && Bytes.BYTES_COMPARATOR.compare(this.lastSeekedRowKey, this.lastSeekAttemptedRowKey) > 0) { + throw new SeekUnderValueException(); + } + spIterator.remove(); + LOG.trace("Next seek point " + Bytes.toString(this.lastSeekAttemptedRowKey)); + boolean reseekResult = closed ? false : this.reseek(this.lastSeekAttemptedRowKey); + if (!reseekResult) return false; + this.lastSeekedRowKey = this.lastSeekAttemptedRowKey; + return true; + } + return false; + } + + @Override + public synchronized boolean reseek(byte[] row) throws IOException { + return this.delegator.reseek(row); + } + + @Override + public void reInit(RegionScanner rs) throws IOException { + this.delegator.close(); + this.delegator = rs; + this.lastSeekedRowKey = null; + this.lastSeekAttemptedRowKey = null; + this.closed = false; + } + + @Override + public byte[] getLatestSeekpoint() { + return this.lastSeekAttemptedRowKey; + } + + @Override + public long getMvccReadPoint() { + return this.delegator.getMvccReadPoint(); + } + + @Override + public boolean nextRaw(List result, String metric) throws IOException { + return this.delegator.nextRaw(result, metric); + } + + @Override + public boolean nextRaw(List result, int limit, String metric) throws IOException { + return this.delegator.nextRaw(result, limit, metric); + } + + @Override + public boolean next(List results, String metric) throws IOException { + // TODO Implement InternalScanner.next + return false; + } + + @Override + public boolean next(List result, int limit, String metric) throws IOException { + // Before every next call seek to the appropriate position. + if (closed) return false; + while (!seekToNextPoint()) { + List seekPoints = new ArrayList(1); + // TODO Do we need to fetch more seekpoints here? + if (!closed) this.seekPointFetcher.nextSeekPoints(seekPoints, 1); + if (seekPoints.isEmpty()) { + // nothing further to fetch from the index table. + return false; + } + this.seekPoints.addAll(seekPoints); + } + return closed ? false : this.delegator.next(result, limit, metric); + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ScanFilterEvaluator.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ScanFilterEvaluator.java new file mode 100644 index 0000000..6bdabd4 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ScanFilterEvaluator.java @@ -0,0 +1,1191 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.ColumnQualifier; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.GroupingCondition; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.index.client.EqualsExpression; +import org.apache.hadoop.hbase.index.client.IndexExpression; +import org.apache.hadoop.hbase.index.client.MultiIndexExpression; +import org.apache.hadoop.hbase.index.client.NoIndexExpression; +import org.apache.hadoop.hbase.index.client.RangeExpression; +import org.apache.hadoop.hbase.index.client.SingleIndexExpression; +import org.apache.hadoop.hbase.index.filter.SingleColumnRangeFilter; +import org.apache.hadoop.hbase.index.filter.SingleColumnValuePartitionFilter; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +public class ScanFilterEvaluator { + + private static final Log LOG = LogFactory.getLog(ScanFilterEvaluator.class); + + private static final Set EMPTY_INDEX_SET = new HashSet(0); + + // TODO we can make this class singleton? This cache will be shared by all regions. + // Then need to change the Map to include the table info also. A Map + private Map colFilterNodeCache = + new ConcurrentHashMap(); + + /** + * This method will evaluate the filter(s) used along with the user table scan against the indices + * available on the table and will provide a list of scans we would like to perform on the index + * table corresponding to this user scan. + * @param scan + * @param indices + * @param regionStartKey + * @param idxRegion + * @param tableName + * @return + * @throws IOException + */ + public IndexRegionScanner evaluate(Scan scan, List indices, + byte[] regionStartKey, HRegion idxRegion, String tableName) throws IOException { + Filter filter = scan.getFilter(); + byte[] indexExpBytes = scan.getAttribute(Constants.INDEX_EXPRESSION); + if (filter == null) { + // When the index(s) to be used is passed through scan attributes and Scan not having any + // filters it is invalid + if (indexExpBytes != null) { + LOG.warn("Passed an Index expression along with the Scan but without any filters on Scan!" + + " The index wont be used"); + } + return null; + } + FilterNode node = null; + IndexRegionScanner indexRegionScanner = null; + if (indexExpBytes != null) { + // Which index(s) to be used is already passed by the user. + try { + IndexExpression indexExpression = + org.apache.hadoop.hbase.index.client.IndexUtils.toIndexExpression(indexExpBytes); + if (indexExpression instanceof NoIndexExpression) { + // Client side says not to use any index for this Scan. + LOG.info("NoIndexExpression is passed as the index to be used for this Scan." + + " No possible index will be used."); + return null; + } + Map nameVsIndex = new HashMap(); + for (IndexSpecification index : indices) { + nameVsIndex.put(index.getName(), index); + } + node = convertIdxExpToFilterNode(indexExpression, nameVsIndex, tableName); + } catch (Exception e) { + LOG.error("There is an Exception in getting IndexExpression from Scan attribute!" + + " The index won't be used", e); + } + } else { + Filter newFilter = doFiltersRestruct(filter); + if (newFilter != null) { + node = evalFilterForIndexSelection(newFilter, indices); + } + } + if (node != null) { + indexRegionScanner = + createIndexScannerScheme(node, regionStartKey, scan.getStartRow(), scan.getStopRow(), + idxRegion, tableName); + // TODO - This check and set looks ugly. Correct me.. + if (indexRegionScanner instanceof IndexRegionScannerForOR) { + ((IndexRegionScannerForOR) indexRegionScanner).setRootFlag(true); + } + if (indexRegionScanner != null) { + indexRegionScanner.setScannerIndex(0); + if (indexRegionScanner.hasChildScanners()) { + return indexRegionScanner; + } else { + return null; + } + } + } + return indexRegionScanner; + } + + private FilterNode convertIdxExpToFilterNode(IndexExpression indexExpression, + Map nameVsIndex, String tableName) { + if (indexExpression instanceof MultiIndexExpression) { + MultiIndexExpression mie = (MultiIndexExpression) indexExpression; + NonLeafFilterNode nlfn = new NonLeafFilterNode(mie.getGroupingCondition()); + for (IndexExpression ie : mie.getIndexExpressions()) { + FilterNode fn = convertIdxExpToFilterNode(ie, nameVsIndex, tableName); + nlfn.addFilterNode(fn); + } + return nlfn; + } else { + // SingleIndexExpression + SingleIndexExpression sie = (SingleIndexExpression) indexExpression; + IndexSpecification index = nameVsIndex.get(sie.getIndexName()); + if (index == null) { + throw new RuntimeException("No index:" + sie.getIndexName() + " added for table:" + + tableName); + } + Map colVsCQ = new HashMap(); + for (ColumnQualifier cq : index.getIndexColumns()) { + colVsCQ + .put(new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition()), cq); + } + // TODO -- seems some thing wrong with IndexFilterNode! Ideally I need to create that here. + NonLeafFilterNode nlfn = new NonLeafFilterNode(null); + List fcvds = new ArrayList(); + // Here by we expect that the equals expressions are given the order of columns in index. + // TODO add reordering if needed? + for (EqualsExpression ee : sie.getEqualsExpressions()) { + ColumnQualifier cq = colVsCQ.get(ee.getColumn()); + if (cq == null) { + throw new RuntimeException("The column:[" + ee.getColumn() + "] is not a part of index: " + + sie.getIndexName()); + } + FilterColumnValueDetail fcvd = + new FilterColumnValueDetail(ee.getColumn(), ee.getValue(), CompareOp.EQUAL); + fcvd.maxValueLength = cq.getMaxValueLength(); + fcvd.valueType = cq.getType(); + fcvds.add(fcvd); + } + // RangeExpression to come after the EqualsExpressions + RangeExpression re = sie.getRangeExpression(); + if (re != null) { + ColumnQualifier cq = colVsCQ.get(re.getColumn()); + if (cq == null) { + throw new RuntimeException("The column:[" + re.getColumn() + "] is not a part of index: " + + sie.getIndexName()); + } + CompareOp lowerBoundCompareOp = + re.isLowerBoundInclusive() ? CompareOp.GREATER_OR_EQUAL : CompareOp.GREATER; + CompareOp upperBoundCompareOp = + re.isUpperBoundInclusive() ? CompareOp.LESS_OR_EQUAL : CompareOp.LESS; + if (re.getLowerBoundValue() == null) { + lowerBoundCompareOp = null; + } + if (re.getUpperBoundValue() == null) { + upperBoundCompareOp = null; + } + FilterColumnValueRange fcvr = + new FilterColumnValueRange(re.getColumn(), re.getLowerBoundValue(), + lowerBoundCompareOp, re.getUpperBoundValue(), upperBoundCompareOp); + fcvr.maxValueLength = cq.getMaxValueLength(); + fcvr.valueType = cq.getType(); + fcvds.add(fcvr); + } + nlfn.addIndicesToUse(fcvds, index); + return nlfn; + } + } + + IndexRegionScanner createIndexScannerScheme(FilterNode node, byte[] regionStartKey, + byte[] startRow, byte[] stopRow, HRegion indexRegion, String userTableName) + throws IOException { + List scanners = new ArrayList(); + IndexRegionScanner idxScanner = null; + int scannerIndex = -1; + if (node instanceof NonLeafFilterNode) { + boolean hasRangeScanner = false; + NonLeafFilterNode nlfNode = (NonLeafFilterNode) node; + Map, IndexSpecification> indicesToUse = nlfNode.getIndexToUse(); + for (Entry, IndexSpecification> entry : indicesToUse.entrySet()) { + List fcvdList = entry.getKey(); + byte[] indexName = Bytes.toBytes(entry.getValue().getName()); + ByteArrayBuilder indexNameBuilder = + ByteArrayBuilder.allocate(IndexUtils.getMaxIndexNameLength()); + indexNameBuilder.put(indexName); + Scan scan = createScan(regionStartKey, indexName, fcvdList, startRow, stopRow); + boolean isRange = isHavingRangeFilters(fcvdList); + if (!hasRangeScanner && isRange) hasRangeScanner = isRange; + createRegionScanner(indexRegion, userTableName, scanners, indexNameBuilder, scan, isRange, + ++scannerIndex); + } + for (FilterNode fn : nlfNode.getFilterNodes()) { + IndexRegionScanner childIndexScaner = + createIndexScannerScheme(fn, regionStartKey, startRow, stopRow, indexRegion, + userTableName); + childIndexScaner.setScannerIndex(++scannerIndex); + scanners.add(childIndexScaner); + if (!hasRangeScanner) hasRangeScanner = childIndexScaner.isRange(); + } + idxScanner = createScannerForNonLeafNode(scanners, nlfNode.getGroupingCondition()); + idxScanner.setRangeFlag(hasRangeScanner); + return idxScanner; + } else if (node instanceof PossibleIndexFilterNode) { + LOG.info("No index can be used for the column " + + ((PossibleIndexFilterNode) node).getFilterColumnValueDetail().getColumn()); + return null; + } else if (node instanceof IndexFilterNode) { + // node is IndexFilterNode + IndexFilterNode ifNode = (IndexFilterNode) node; + // There will be only one entry in this Map + List filterColsDetails = + ifNode.getIndexToUse().keySet().iterator().next(); + byte[] indexName = Bytes.toBytes(ifNode.getBestIndex().getName()); + ByteArrayBuilder indexNameBuilder = + ByteArrayBuilder.allocate(IndexUtils.getMaxIndexNameLength()); + indexNameBuilder.put(indexName); + Scan scan = createScan(regionStartKey, indexName, filterColsDetails, startRow, stopRow); + boolean isRange = isHavingRangeFilters(filterColsDetails); + createRegionScanner(indexRegion, userTableName, scanners, indexNameBuilder, scan, isRange, + ++scannerIndex); + idxScanner = createScannerForNonLeafNode(scanners, null); + idxScanner.setRangeFlag(isRange); + return idxScanner; + } + return null; + } + + private boolean isHavingRangeFilters(List fcvdList) { + for (FilterColumnValueDetail fcvd : fcvdList) { + if (fcvd instanceof FilterColumnValueRange) { + return true; + } + } + return false; + } + + private IndexRegionScanner createScannerForNonLeafNode(List scanners, + GroupingCondition condition) { + IndexRegionScanner idxScanner; + if (condition == GroupingCondition.OR) { + idxScanner = new IndexRegionScannerForOR(scanners); + } else { + idxScanner = new IndexRegionScannerForAND(scanners); + } + return idxScanner; + } + + private void createRegionScanner(HRegion indexRegion, String userTableName, + List scanners, ByteArrayBuilder indexNameBuilder, Scan scan, + boolean isRange, int scannerIndex) throws IOException { + RegionScanner scannerForIndexRegion = indexRegion.getScanner(scan); + LeafIndexRegionScanner leafIndexRegionScanner = + new LeafIndexRegionScanner(IndexManager.getInstance().getIndex(userTableName, + indexNameBuilder.array()), scannerForIndexRegion, new TTLExpiryChecker()); + leafIndexRegionScanner.setScannerIndex(scannerIndex); + leafIndexRegionScanner.setRangeFlag(isRange); + scanners.add(leafIndexRegionScanner); + } + + private Scan createScan(byte[] regionStartKey, byte[] indexName, + List filterColsDetails, byte[] startRow, byte[] stopRow) { + Scan scan = new Scan(); + // common key is the regionStartKey + indexName + byte[] commonKey = createCommonKeyForIndex(regionStartKey, indexName); + scan.setStartRow(createStartOrStopKeyForIndexScan(filterColsDetails, commonKey, startRow, true)); + // Find the end key for the scan + scan.setStopRow(createStartOrStopKeyForIndexScan(filterColsDetails, commonKey, stopRow, false)); + return scan; + } + + private byte[] createCommonKeyForIndex(byte[] regionStartKey, byte[] indexName) { + // Format for index table rowkey [Startkey for the index region] + [one 0 byte]+ + // [Index name] + [Padding for the max index name] + .... + int commonKeyLength = regionStartKey.length + 1 + IndexUtils.getMaxIndexNameLength(); + ByteArrayBuilder builder = ByteArrayBuilder.allocate(commonKeyLength); + // Adding the startkey for the index region and single 0 Byte. + builder.put(regionStartKey); + builder.position(builder.position() + 1); + // Adding the index name and the padding needed + builder.put(indexName); + // No need to add the padding bytes specifically. In the array all the bytes will be 0s. + return builder.array(); + } + + // When it comes here, only the last column in the colDetails will be a range. + // Others will be exact value. EQUALS condition. + private byte[] createStartOrStopKeyForIndexScan(List colDetails, + byte[] commonKey, byte[] userTabRowKey, boolean isStartKey) { + int totalValueLen = 0; + for (FilterColumnValueDetail fcvd : colDetails) { + totalValueLen += fcvd.maxValueLength; + } + int userTabRowKeyLen = userTabRowKey == null ? 0 : userTabRowKey.length; + ByteArrayBuilder builder = + ByteArrayBuilder.allocate(commonKey.length + totalValueLen + userTabRowKeyLen); + builder.put(commonKey); + for (int i = 0; i < colDetails.size(); i++) { + FilterColumnValueDetail fcvd = colDetails.get(i); + if (i == colDetails.size() - 1) { + // This is the last column in the colDetails. Here the range check can come + if (isStartKey) { + if (fcvd.compareOp == null) { + // This can happen when the condition is a range condition and only upper bound is + // specified. ie. c1<100 + // Now the column value can be specified as 0 bytes. Just we need to advance the byte[] + assert fcvd instanceof FilterColumnValueRange; + assert fcvd.value == null; + builder.position(builder.position() + fcvd.maxValueLength); + } else if (fcvd.compareOp.equals(CompareOp.EQUAL) + || fcvd.compareOp.equals(CompareOp.GREATER_OR_EQUAL)) { + copyColumnValueToKey(builder, fcvd.value, fcvd.maxValueLength, fcvd.valueType); + } else if (fcvd.compareOp.equals(CompareOp.GREATER)) { + // We can go with the value + 1 + // For eg : if type is int and value is 45, make startkey considering value as 46 + // If type is String and value is 'ad' make startkey considering value as 'ae' + + copyColumnValueToKey(builder, fcvd.value, fcvd.maxValueLength, fcvd.valueType); + IndexUtils.incrementValue(builder.array(), false); + } + } else { + CompareOp op = fcvd.compareOp; + byte[] value = fcvd.value; + if (fcvd instanceof FilterColumnValueRange) { + op = ((FilterColumnValueRange) fcvd).getUpperBoundCompareOp(); + value = ((FilterColumnValueRange) fcvd).getUpperBoundValue(); + } + if (op == null) { + // This can happen when the condition is a range condition and only lower bound is + // specified. ie. c1>=10 + assert fcvd instanceof FilterColumnValueRange; + assert value == null; + // Here the stop row is already partially built. As there is no upper bound all the + // possibles values for that column we need to consider. Well the max value for a + // column with maxValueLength=10 will a byte array of 10 bytes with all bytes as FF. + // But we can put this FF bytes into the stop row because the stop row will not be + // considered by the scanner. So we need to increment this by 1. + // We can increment the byte[] created until now by 1. + byte[] stopRowTillNow = builder.array(0, builder.position()); + stopRowTillNow = IndexUtils.incrementValue(stopRowTillNow, true); + // Now we need to copy back this incremented value to the builder. + builder.position(0); + builder.put(stopRowTillNow); + // Now just advance the builder pos by fcvd.maxValueLength as we need all 0 bytes + builder.position(builder.position() + fcvd.maxValueLength); + } else if (op.equals(CompareOp.EQUAL) || op.equals(CompareOp.LESS_OR_EQUAL)) { + copyColumnValueToKey(builder, value, fcvd.maxValueLength, fcvd.valueType); + IndexUtils.incrementValue(builder.array(), false); + + } else if (op.equals(CompareOp.LESS)) { + copyColumnValueToKey(builder, value, fcvd.maxValueLength, fcvd.valueType); + } + } + } else { + // For all other columns other than the last column the condition must be EQUALS + copyColumnValueToKey(builder, fcvd.value, fcvd.maxValueLength, fcvd.valueType); + } + } + if (userTabRowKeyLen > 0) { + builder.put(userTabRowKey); + } + + return builder.array(); + } + + private void copyColumnValueToKey(ByteArrayBuilder builder, byte[] colValue, int maxValueLength, + ValueType valueType) { + colValue = IndexUtils.changeValueAccToDataType(colValue, valueType); + builder.put(colValue); + int paddingLength = maxValueLength - colValue.length; + builder.position(builder.position() + paddingLength); + } + + // This method will do the reorder and restructuring of the filters so as the finding of + // suitable index for the query will be easier for the remaining steps + // This will do 2 things basically + // 1.Group the AND condition on SingleColumnValueFilter together so as to make a range condition. + // In this process it will validate the correctness of the range and will avoid the invalid + // things. + // 2.Move the adjacent AND conditions together within on filter list. If N number of adjacent + // filters or filter lists are there and all are combined using AND condition then it is same + // as all within one list. + // Default access for testability + Filter doFiltersRestruct(Filter filter) { + if (filter instanceof SingleColumnValueFilter) { + ValuePartition vp = null; + if (filter instanceof SingleColumnValuePartitionFilter) { + vp = ((SingleColumnValuePartitionFilter) filter).getValuePartition(); + } + SingleColumnValueFilter scvf = (SingleColumnValueFilter) filter; + if (scvf.getOperator().equals(CompareOp.LESS) + || scvf.getOperator().equals(CompareOp.LESS_OR_EQUAL) + || scvf.getOperator().equals(CompareOp.GREATER) + || scvf.getOperator().equals(CompareOp.GREATER_OR_EQUAL)) { + return new SingleColumnRangeFilter(scvf.getFamily(), scvf.getQualifier(), vp, scvf + .getComparator().getValue(), scvf.getOperator(), null, null); + } + } + FilterGroupingWorker groupWorker = new FilterGroupingWorker(); + return groupWorker.group(filter); + } + + // For a multi column index, the index look up with a value range can be supported only on the + // last column. The previous columns lookups to be exact value look up. + // Index idx1 on columns c1 and c2. A lookup condition with c1=x and c2 btw x and z can be + // supported using the index idx1. But if the lookup is c1 btw a and k and c2... we can better + // not support using idx1 there. (May be check later we can support this) + // In this case 2 indices idx1 and idx2 on col1 and col2 respectively can be used. + // In the above case a lookup only on c1, either it is a range lookup or single value lookup + // we can make use of the multi column index on c1 and c2. + // In this case if there is an index on c1 alone, we better use that. + // But a single column lookup on column c1 and there is an index on c2 and c1 , we can not use + // this index. It is equivalent to c1=x and c2 btw min to max. + // Default access for testability + FilterNode evalFilterForIndexSelection(Filter filter, List indices) { + if (filter instanceof FilterList) { + FilterList fList = (FilterList) filter; + GroupingCondition condition = + (fList.getOperator() == Operator.MUST_PASS_ALL) ? GroupingCondition.AND + : GroupingCondition.OR; + NonLeafFilterNode nonLeafFilterNode = new NonLeafFilterNode(condition); + List filters = fList.getFilters(); + for (Filter fltr : filters) { + FilterNode node = evalFilterForIndexSelection(fltr, indices); + nonLeafFilterNode.addFilterNode(node); + } + return handleNonLeafFilterNode(nonLeafFilterNode); + } else if (filter instanceof SingleColumnValueFilter) { + // Check for the availability of index + return selectBestFitAndPossibleIndicesForSCVF(indices, (SingleColumnValueFilter) filter); + } else if (filter instanceof SingleColumnRangeFilter) { + return selectBestFitAndPossibleIndicesForSCRF(indices, (SingleColumnRangeFilter) filter); + } + return new NoIndexFilterNode(); + } + + private FilterNode selectBestFitAndPossibleIndicesForSCRF(List indices, + SingleColumnRangeFilter filter) { + FilterColumnValueRange range = + new FilterColumnValueRange(filter.getFamily(), filter.getQualifier(), + filter.getValuePartition(), filter.getLowerBoundValue(), filter.getLowerBoundOp(), + filter.getUpperBoundValue(), filter.getUpperBoundOp()); + return selectBestFitIndexForColumn(indices, range); + } + + private FilterNode handleNonLeafFilterNode(NonLeafFilterNode nonLeafFilterNode) { + // check whether this node can be changed to a NoIndexFilterNode + // This we can do when the condition in nonLeafFilterNode is OR and any of the + // child node is NoIndexFilterNode + if (nonLeafFilterNode.getGroupingCondition() == GroupingCondition.OR) { + return handleORCondition(nonLeafFilterNode); + } else { + // AND condition + return handleANDCondition(nonLeafFilterNode); + } + } + + private FilterNode handleORCondition(NonLeafFilterNode nonLeafFilterNode) { + Iterator nonLeafFilterNodeItr = nonLeafFilterNode.getFilterNodes().iterator(); + while (nonLeafFilterNodeItr.hasNext()) { + FilterNode filterNode = nonLeafFilterNodeItr.next(); + if (filterNode instanceof IndexFilterNode) { + FilterColumnValueDetail filterColumnValueDetail = + ((IndexFilterNode) filterNode).getFilterColumnValueDetail(); + IndexSpecification indexToUse = ((IndexFilterNode) filterNode).getBestIndex(); + nonLeafFilterNode.addIndicesToUse(filterColumnValueDetail, indexToUse); + nonLeafFilterNodeItr.remove(); + } else if (filterNode instanceof PossibleIndexFilterNode + || filterNode instanceof NoIndexFilterNode) { + // The moment an OR condition contains a column on which there is no index which can be + // used, the entire OR node becomes as non indexed. + return new NoIndexFilterNode(); + } + // A NonLeafFilterNode under the OR node need to be kept as it is. + } + return nonLeafFilterNode; + } + + private FilterNode handleANDCondition(NonLeafFilterNode nonLeafFilterNode) { + Map leafNodes = new HashMap(); + Iterator filterNodesIterator = nonLeafFilterNode.getFilterNodes().iterator(); + while (filterNodesIterator.hasNext()) { + FilterNode filterNode = filterNodesIterator.next(); + if (filterNode instanceof LeafFilterNode) { + LeafFilterNode lfNode = (LeafFilterNode) filterNode; + leafNodes.put(lfNode.getFilterColumnValueDetail().column, lfNode); + filterNodesIterator.remove(); + } else if (filterNode instanceof NoIndexFilterNode) { + filterNodesIterator.remove(); + } + // Any NonLeafFilterNode under this NonLeafFilterNode will be kept as it is. + // This will be a OR condition corresponding node. + } + // This below method will consider all the leafNodes just under that and will try to + // finalize one or more index to be used for those col. It will try to get the best + // combination minimizing the number of indices to be used. If I have say 5 leaf cols + // under this AND node and there is one index on these 5 cols, well I can use that one + // index. If not will try to find indices which can be applied on the subsets of these + // 5 cols, say one on 3 cols and other on 2 cols + if (!leafNodes.isEmpty()) { + Map, IndexSpecification> colVsIndex = finalizeIndexForLeafNodes(leafNodes); + if (LOG.isDebugEnabled()) { + LOG.debug("Index(s) which will be used for columns " + leafNodes.keySet() + " : " + + colVsIndex); + } + if (colVsIndex != null) { + addIndicesToNonLeafAndNode(colVsIndex, nonLeafFilterNode, leafNodes); + } + } + return nonLeafFilterNode; + } + + // Here also 2 many loops we can avoid this + @SuppressWarnings("unchecked") + private Map, IndexSpecification> finalizeIndexForLeafNodes( + Map leafNodes) { + // go with the breakups and check + // suppose there are 5 cols under the AND condition and are c1,c2,c3,c4,c5 + // There can be different break ups for the cols possible. + // [5],[4,1],[3,2],[3,1,1],[2,2,1],[2,1,1,1],[1,1,1,1,1] + // In each of these breakup also we can get many columns combinations. + // Except in first and last where either all cols in one group or 1 column only. + // For [4,1] there can be 5c1 combinations possible. + Set>>> colBreakUps = getColsBreakUps(leafNodes.keySet()); + ColBreakUpIndexDetails bestColBreakUpIndexDetails = null; + for (List>> colBreakUp : colBreakUps) { + ColBreakUpIndexDetails colBreakUpIndexDetails = + findBestIndicesForColSplitsInBreakUp(colBreakUp, leafNodes); + if (colBreakUpIndexDetails == null) { + continue; + } + if (colBreakUpIndexDetails.isBestIndex) { + // This means this is THE best index. It solves all the columns and exactly those cols only + // there as part of the indices too.. What else we need... + bestColBreakUpIndexDetails = colBreakUpIndexDetails; + break; + } else { + if (bestColBreakUpIndexDetails == null + || isIndicesGroupBetterThanCurBest(colBreakUpIndexDetails, bestColBreakUpIndexDetails)) { + bestColBreakUpIndexDetails = colBreakUpIndexDetails; + } + } + } + // TODO some more logging of the output.. + return bestColBreakUpIndexDetails == null ? null + : bestColBreakUpIndexDetails.bestIndicesForBreakUp; + } + + private void addIndicesToNonLeafAndNode(Map, IndexSpecification> colsVsIndex, + NonLeafFilterNode nonLeafFilterNode, Map leafNodes) { + for (Entry, IndexSpecification> entry : colsVsIndex.entrySet()) { + List cols = entry.getKey(); + int colsSize = cols.size(); + IndexSpecification index = entry.getValue(); + // The FilterColumnValueDetail for cols need to be in the same order as that of cols + // in the index. This order will be important for creating the start/stop keys for + // index scan. + List fcvds = new ArrayList(colsSize); + int i = 0; + for (ColumnQualifier cq : index.getIndexColumns()) { + FilterColumnValueDetail fcvd = + leafNodes.get( + new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition())) + .getFilterColumnValueDetail(); + assert fcvd != null; + fcvds.add(fcvd); + i++; + if (i == colsSize) { + // The selected index might be on more cols than those we are interested in now. + // All those will be towards the end. + break; + } + } + LOG.info("Index using for the columns " + cols + " : " + index); + nonLeafFilterNode.addIndicesToUse(fcvds, index); + } + } + + private static class ColBreakUpIndexDetails { + private Map, IndexSpecification> bestIndicesForBreakUp; + private int bestIndicesCardinality = -1; + private int colsResolvedByBestIndices = -1; + private boolean isBestIndex = false; + } + + private ColBreakUpIndexDetails findBestIndicesForColSplitsInBreakUp( + List>> colBreakUpCombs, Map leafNodes) { + int totalColsInBreakup = leafNodes.size(); + ColBreakUpIndexDetails bestColBreakUpIndexDetails = null; + // List>> breakUp contains different combinations of the columns in a + // particular break up case. Say for 3 cols, a,b,c and the breakup is [2,1], this list + // contains combinations [[a,b],[c]],[[b,c],[a]],[[a,c],[b]] + for (List> colBreakUpComb : colBreakUpCombs) { + if (LOG.isDebugEnabled()) { + LOG.debug("Checking for the best index(s) for the cols combination : " + colBreakUpComb); + } + Map, IndexSpecification> colsVsIndex = + new HashMap, IndexSpecification>(); + for (List cols : colBreakUpComb) { + IndexSpecification index = findBestIndex(cols, leafNodes); + if (LOG.isDebugEnabled()) { + LOG.debug("Best index selected for the cols " + cols + " : " + index); + } + if (index != null) { + colsVsIndex.put(cols, index); + } + } + if (colsVsIndex.size() == 0) { + // For this cols breakup not even single index we are able to find which solves atleast + // one column. Just continue with the next combination. + if (LOG.isDebugEnabled()) { + LOG.debug("Not even one index found for the cols combination : " + colBreakUpComb); + } + continue; + } + int netIndicesCardinality = 0; + int colsResolved = 0; + for (Entry, IndexSpecification> entry : colsVsIndex.entrySet()) { + colsResolved += entry.getKey().size(); + netIndicesCardinality += entry.getValue().getIndexColumns().size(); + } + // Am I THE real best combination + if (colBreakUpComb.size() == colsVsIndex.size()) { + assert colsResolved == totalColsInBreakup; + // Means we have index for all the cols combinations + // Now check the net cardinality of all the indices + if (netIndicesCardinality == totalColsInBreakup) { + // This is the best + LOG.info("Got indices combination for the cols " + colsVsIndex + " which is THE BEST!"); + ColBreakUpIndexDetails bestDetails = new ColBreakUpIndexDetails(); + bestDetails.bestIndicesForBreakUp = colsVsIndex; + bestDetails.isBestIndex = true; + // colsResolved and netIndicesCardinality not needed in this case + return bestDetails; + } + } + // Checking whether this colsIndexMap combination better than the previous best. + ColBreakUpIndexDetails curDetails = new ColBreakUpIndexDetails(); + curDetails.bestIndicesForBreakUp = colsVsIndex; + curDetails.colsResolvedByBestIndices = colsResolved; + curDetails.bestIndicesCardinality = netIndicesCardinality; + if (bestColBreakUpIndexDetails == null + || isIndicesGroupBetterThanCurBest(curDetails, bestColBreakUpIndexDetails)) { + bestColBreakUpIndexDetails = curDetails; + } + } + return bestColBreakUpIndexDetails; + } + + private boolean isIndicesGroupBetterThanCurBest(ColBreakUpIndexDetails curColBreakUpIndexDetails, + ColBreakUpIndexDetails bestColBreakUpIndexDetails) { + // Conditions to decide whether current one is better than the current best + // 1. The total number of cols all the indices resolve + // 2. The number of indices which resolves the cols + // 3. The net cardinality of all the indices + // TODO later can check for the dynamic metric data about indices and decide which can be used. + int colsResolvedByCurIndices = curColBreakUpIndexDetails.colsResolvedByBestIndices; + int colsResolvedByBestIndices = bestColBreakUpIndexDetails.colsResolvedByBestIndices; + if (colsResolvedByCurIndices > colsResolvedByBestIndices) { + // The more cols it resolves the better. + return true; + } else if (colsResolvedByCurIndices == colsResolvedByBestIndices) { + int indicesInCurComb = curColBreakUpIndexDetails.bestIndicesForBreakUp.size(); + int indicesInBestComb = bestColBreakUpIndexDetails.bestIndicesForBreakUp.size(); + if (indicesInCurComb < indicesInBestComb) { + // The lesser the number of indices the better. + return true; + } else if (indicesInCurComb == indicesInBestComb) { + int curIndicesCardinality = curColBreakUpIndexDetails.bestIndicesCardinality; + int bestIndicesCardinality = bestColBreakUpIndexDetails.bestIndicesCardinality; + if (curIndicesCardinality < bestIndicesCardinality) { + // The lesser the cardinality the better. + return true; + } + } + } + return false; + } + + private IndexSpecification + findBestIndex(List cols, Map leafNodes) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to find a best index for the cols : " + cols); + } + Set indicesToUse = getPossibleIndicesForCols(cols, leafNodes); + // indicesToUse will never come as null.... + if (LOG.isDebugEnabled()) { + LOG.debug("Possible indices for cols " + cols + " : " + indicesToUse); + } + IndexSpecification bestIndex = null; + int bestIndexCardinality = -1; + for (IndexSpecification index : indicesToUse) { + if (isIndexSuitable(index, cols, leafNodes)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Index " + index + " seems to be suitable for the columns " + cols); + } + if (index.getIndexColumns().size() == cols.size()) { + // Yea we got the best index. Juts return this. No need to loop through and check + // with other indices + return index; + } + // Compare this index with the current best. This will be better if its cardinality + // is better(lesser) than the current best's + // TODO pluggable interface to decide which index to be used when both this and current + // best index having same cardinality. + if (bestIndex == null || index.getIndexColumns().size() < bestIndexCardinality) { + bestIndex = index; + bestIndexCardinality = index.getIndexColumns().size(); + } + } + } + return bestIndex; + } + + private Set getPossibleIndicesForCols(List cols, + Map leafNodes) { + // When there are more than one range conditioned columns, no need to check with any of the + // index. We will not get any matched index for all cols. + int noOfRangeConds = 0; + for (Column col : cols) { + LeafFilterNode leafFilterNode = leafNodes.get(col); + if (leafFilterNode != null + && leafFilterNode.getFilterColumnValueDetail() instanceof FilterColumnValueRange) { + noOfRangeConds++; + } + if (noOfRangeConds > 1) { + return EMPTY_INDEX_SET; + } + } + // For a given column with a range condition an index can be its possible index or + // future possible index iff this column is the last column in the index + // Suppose an index on c1&c2 and the column c1 is having of range 10-20, then this index + // can not be used for this range condition. [provided some condition is there on c2 also] + // AND(c1=10,c2=20,c3 btw[10,20]) + Set indicesToUse = new HashSet(); + for (Column col : cols) { + LeafFilterNode lfn = leafNodes.get(col); + if (lfn != null) { + FilterColumnValueDetail filterColumnValueDetail = lfn.getFilterColumnValueDetail(); + IndexSpecification bestIndex = lfn.getBestIndex(); + if (bestIndex != null) { + indicesToUse.add(bestIndex); + } + Map>> possibleUseIndices = + lfn.getPossibleUseIndices(); + if (possibleUseIndices != null) { + List> possibleIndices = + possibleUseIndices.get(filterColumnValueDetail.getColumn()); + if (possibleIndices != null) { + for (Pair pair : possibleIndices) { + indicesToUse.add(pair.getFirst()); + } + } + } + Map>> possibleFutureUseIndices = + lfn.getPossibleFutureUseIndices(); + if (possibleFutureUseIndices != null) { + List> possibleIndices = + possibleFutureUseIndices.get(filterColumnValueDetail.getColumn()); + if (possibleIndices != null) { + for (Pair pair : possibleIndices) { + indicesToUse.add(pair.getFirst()); + } + } + } + } + } + return indicesToUse; + } + + // Whether the passed index is suitable to be used for the given columns. + // index is suitable when the it contains all the columns. + // Also index should not contain any other column before any of the column + // in the passed cols list + private boolean isIndexSuitable(IndexSpecification index, List cols, + Map leafNodes) { + int matchedCols = 0; + for (ColumnQualifier cq : index.getIndexColumns()) { + Column column = new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition()); + if (cols.contains(column)) { + matchedCols++; + // leafNodes.get(column) will never be null.. Don't worry + if (leafNodes.get(column).getFilterColumnValueDetail() instanceof FilterColumnValueRange) { + // When the condition on the column is a range condition, we need to ensure in this index + // 1. The column is the last column + // or + // 2. There are no columns in this index which is part of the cols list + if (matchedCols != cols.size()) { + return false; + } + } + } else { + if (matchedCols != cols.size()) { + return false; + } + } + if (matchedCols == cols.size()) { + return true; + } + } + return false; + } + + @SuppressWarnings("unchecked") + Set>>> getColsBreakUps(Set columns) { + int totalColsCount = columns.size(); + // Breakups should be sorted such that the best one come before the others. The best one is the + // one with lower number of splits. If the number of splits same, then the which contains more + // cols in one split. + // eg: When there are total 5 cols, then split [7] is better than [4,1] and [4,1] is better + // than [3,2] + Comparator>>> comparator = new Comparator>>>() { + @Override + public int compare(List>> l1, List>> l2) { + List> firstColsCombl1 = l1.get(0); + List> firstColsCombl2 = l2.get(0); + int result = firstColsCombl1.size() - firstColsCombl2.size(); + if (result != 0) return result; + int size = Math.min(firstColsCombl1.size(), firstColsCombl2.size()); + for (int i = 0; i < size; i++) { + int colsSizeInSplit1 = firstColsCombl1.get(i).size(); + int colsSizeInSplit2 = firstColsCombl2.get(i).size(); + result = colsSizeInSplit1 - colsSizeInSplit2; + if (result != 0) return -result; + } + return 0; + } + }; + Set>>> listOfCols = new TreeSet>>>(comparator); + ColCombination comb = new ColCombination(); + ArrayList breakUp = null; + int firstItemColsCount = totalColsCount; + while (firstItemColsCount >= 1) { + breakUp = new ArrayList(); + breakUp.add(firstItemColsCount); + int totalColsCountInBreakUp = firstItemColsCount; + while (totalColsCountInBreakUp < totalColsCount) { + // Fill with 1... + breakUp.add(1); + totalColsCountInBreakUp++; + } + listOfCols.add(makeColCombination(columns, breakUp, comb)); + // now try to combine the after 1st entry 1s into bigger items. + // But this combined value never should be more than the 1st value. + // group last 2 items and create a new breakup.. + while (true) { + // In order to do this group there should be atleast 3 elements in the breakUp list. + if (breakUp.size() < 3) { + break; + } + breakUp = (ArrayList) breakUp.clone(); + int lastElem = breakUp.remove(breakUp.size() - 1); + int secondLastElem = breakUp.remove(breakUp.size() - 1); + // Add this element to the current last. ie. the old second last element + // Well this element must be 1 only. + int newLastElem = lastElem + secondLastElem; + if (newLastElem > firstItemColsCount) { + break; + } + breakUp.add(newLastElem); + listOfCols.add(makeColCombination(columns, breakUp, comb)); + } + firstItemColsCount--; + } + return listOfCols; + } + + private List>> makeColCombination(Set columns, List breakUp, + ColCombination comb) { + int size = breakUp.size(); + return comb.formColCombinations(size, columns, breakUp); + } + + /** + * Generates the column combination + */ + private static class ColCombination { + + public List>> formColCombinations(int breakUpSize, Set columns, + List breakUp) { + // If no of cols and break up is same it will be all 1s combination. So just return the list + List>> finalCols = new ArrayList>>(); + ArrayList> possibleCols = new ArrayList>(); + LinkedList copyOfColumns = new LinkedList(columns); + int lastIndex = 0; + boolean allOnes = false; + // for every break up find the combination + for (Integer r : breakUp) { + if (possibleCols.size() == 0) { + possibleCols = + (ArrayList>) createCombination(copyOfColumns, r, possibleCols); + copyOfColumns = new LinkedList(columns); + if (columns.size() == breakUpSize) { + // for [1 1 1 1 1] + allOnes = true; + break; + } + } else { + int colSize = possibleCols.size(); + List> cloneOfPossibleCols = new ArrayList>(possibleCols); + + for (int i = lastIndex; i < colSize; i++) { + + List> possibleCols1 = new ArrayList>(); + List col = clone(cloneOfPossibleCols, i); + copyOfColumns.removeAll(cloneOfPossibleCols.get(i)); + possibleCols1 = createCombination(copyOfColumns, r, possibleCols); + + copyOfColumns = new LinkedList(columns); + boolean cloned = true; + for (List set : possibleCols1) { + if (cloned == false) { + col = clone(cloneOfPossibleCols, i); + } + col.addAll(set); + possibleCols.add(col); + + cloned = false; + } + } + lastIndex = colSize; + } + } + // Optimize here + if (!allOnes) { + for (int i = lastIndex; i < possibleCols.size(); i++) { + List> subList = new ArrayList>(); + int prev = 0; + for (int j = 0; j < breakUp.size(); j++) { + int index = breakUp.get(j); + subList.add(possibleCols.get(i).subList(prev, prev + index)); + prev = prev + index; + + } + finalCols.add(subList); + } + } else { + finalCols.add(possibleCols); + } + return finalCols; + } + + private List clone(List> cloneOfPossibleCols, int i) { + return (List) ((ArrayList) cloneOfPossibleCols.get(i)).clone(); + } + + private List> createCombination(LinkedList copyOfColumns, int r, + ArrayList> possibleCols2) { + int j; + int k = 0; + List> possibleCols = new ArrayList>(); + ArrayList colList = new ArrayList(); + int size = copyOfColumns.size(); + for (int i = 0; i < (1 << size); i++) { + if (r != findNoOfBitsSet(i)) { + continue; + } + j = i; + k = 0; + while (j != 0) { + if (j % 2 == 1) { + colList.add(copyOfColumns.get(k)); + } + j /= 2; + k++; + } + possibleCols.add((List) colList.clone()); + colList.clear(); + } + return possibleCols; + } + + private int findNoOfBitsSet(int n) { + int count = 0; + while (n != 0) { + n &= n - 1; + count++; + } + return count; + } + } + + // This is at the ColumnValueFilter level. That means value for one cf:qualifier(let me call + // this column as col1) is provided in the condition + // Any index on the table which contain the column col1 as the 1st index column in it can be + // selected for the usage. If the table having an index on col1 & col2 we can consider this + // index as col1 is the 1st column in the index columns list. But if the index is on col2 & col1 + // ie. 1st col in the index cols list is not col1, we wont be able to use that index. + // Now comes the question of best fit index. When there are more than one possible index to be + // used, we need to go with one index. Others are only possible candidates to be considered. + // Suppose there is an index on col1&col2 and another on col1 alone. Here to consider the 2nd + // index will be better as that will be having lesser data in the index table, that we need to + // scan. So exact match is always better. + // Suppose there is an index on col1&col2 and another on col1&col2&col3, it this case it would + // be better to go with index on col1&col2 as this will be giving lesser data to be scanned. + // To be general, if there are more possible indices which can be used, we can select the one + // which is having lesser number of index cols in it. + private FilterNode selectBestFitAndPossibleIndicesForSCVF(List indices, + SingleColumnValueFilter filter) { + if (CompareOp.NOT_EQUAL == filter.getOperator() || CompareOp.NO_OP == filter.getOperator()) { + return new NoIndexFilterNode(); + } + FilterColumnValueDetail detail = null; + if (filter instanceof SingleColumnValuePartitionFilter) { + SingleColumnValuePartitionFilter escvf = (SingleColumnValuePartitionFilter) filter; + detail = + new FilterColumnValueDetail(escvf.getFamily(), escvf.getQualifier(), escvf + .getComparator().getValue(), escvf.getValuePartition(), escvf.getOperator()); + } else { + detail = + new FilterColumnValueDetail(filter.getFamily(), filter.getQualifier(), filter + .getComparator().getValue(), filter.getOperator()); + } + return selectBestFitIndexForColumn(indices, detail); + } + + private FilterNode selectBestFitIndexForColumn(List indices, + FilterColumnValueDetail detail) { + FilterNode filterNode = + this.colFilterNodeCache.get(new ColumnWithValue(detail.getColumn(), detail.getValue())); + if (filterNode != null) { + // This column is being already evaluated to find the best an possible indices + // No need to work on this + return filterNode; + } + List> possibleUseIndices = + new ArrayList>(); + List> possibleFutureUseIndices = + new ArrayList>(); + IndexSpecification bestFitIndex = null; + int bestFitIndexColsSize = Integer.MAX_VALUE; + for (IndexSpecification index : indices) { + Set indexColumns = index.getIndexColumns(); + // Don't worry . This Set is LinkedHashSet. So this will maintain the order. + Iterator indexColumnsIterator = indexColumns.iterator(); + ColumnQualifier firstCQInIndex = indexColumnsIterator.next(); + Column firstColInIndex = + new Column(firstCQInIndex.getColumnFamily(), firstCQInIndex.getQualifier(), + firstCQInIndex.getValuePartition()); + if (firstColInIndex.equals(detail.column)) { + possibleUseIndices.add(new Pair(index, indexColumns.size())); + // When we have condition on col1 and we have indices on col1&Col2 and col1&col3 + // which one we should select? We dont know the data related details of every index. + // So select any one. May be the 1st come selected. + // TODO later we can add more intelligence and then add index level data details + if (indexColumns.size() < bestFitIndexColsSize) { + bestFitIndex = index; + bestFitIndexColsSize = indexColumns.size(); + } + detail.maxValueLength = firstCQInIndex.getMaxValueLength(); + detail.valueType = firstCQInIndex.getType(); + } + // This index might be useful at a topper level.... + // I will explain in detail + // When the filter from customer is coming this way as shown below + // & + // ___|___ + // | | + // c1=10 c2=5 + // Suppose we have an index on c2&c1 only on the table, when we check for c1=10 node + // we will not find any index for that as index is not having c1 as the 1st column. + // For c2=5 node, the index is a possible option. Now when we consider the & node, + // the index seems perfect match. That is why even for the c1=10 node also, we can not + // sat it is a NoIndexFilterNode, if there are any indices on the table which contain c1 + // as one column in the index columns. + else { + ColumnQualifier cq = null; + Column column = null; + while (indexColumnsIterator.hasNext()) { + cq = indexColumnsIterator.next(); + column = new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition()); + if (column.equals(detail.column)) { + possibleFutureUseIndices.add(new Pair(index, indexColumns + .size())); + detail.maxValueLength = firstCQInIndex.getMaxValueLength(); + detail.valueType = firstCQInIndex.getType(); + break; + } + } + } + } + if (null != bestFitIndex) { + filterNode = + new IndexFilterNode(bestFitIndex, possibleUseIndices, possibleFutureUseIndices, detail); + } else if (!possibleFutureUseIndices.isEmpty()) { + // when bestFitIndex is null possibleUseIndices will be empty for sure + filterNode = new PossibleIndexFilterNode(possibleFutureUseIndices, detail); + } else { + filterNode = new NoIndexFilterNode(); + } + byte[] val = null; + if (detail instanceof FilterColumnValueRange) { + // Probably we have LESS condition + if (detail.getValue() == null) { + val = ((FilterColumnValueRange) detail).getUpperBoundValue(); + } else { + val = detail.getValue(); + } + } + this.colFilterNodeCache.put(new ColumnWithValue(detail.getColumn(), val), filterNode); + return filterNode; + } + + // TODO toString() in the VO classes + + public static class ColumnWithValue { + private Column column; + private byte[] value; + + public ColumnWithValue(Column column, byte[] value) { + this.column = column; + this.value = value; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ColumnWithValue)) return false; + ColumnWithValue that = (ColumnWithValue) obj; + Column col = that.getColumn(); + boolean equals = this.column.equals(col); + if (equals && Bytes.equals(this.value, that.getValue())) { + return true; + } else { + return false; + } + } + + public Column getColumn() { + return this.column; + } + + public byte[] getValue() { + return this.value; + } + + @Override + public int hashCode() { + return this.column.hashCode(); + } + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekAndReadRegionScanner.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekAndReadRegionScanner.java new file mode 100644 index 0000000..12ed288 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekAndReadRegionScanner.java @@ -0,0 +1,42 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.regionserver.RegionScanner; + +public interface SeekAndReadRegionScanner extends RegionScanner { + + void addSeekPoints(List seekPoints); + + byte[] getLatestSeekpoint(); + + /** + * This will make the scanner to reseek to the next seek point. + * @return True when seeked to the next point. False when there is no further seek points. + * @throws IOException + */ + boolean seekToNextPoint() throws IOException; + + public boolean isClosed(); + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekPointFetcher.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekPointFetcher.java new file mode 100644 index 0000000..6c8885c --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekPointFetcher.java @@ -0,0 +1,86 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; + +// TODO better name +public class SeekPointFetcher { + + private static final Log LOG = LogFactory.getLog(SeekPointFetcher.class); + private RegionScanner indexRegionScanner; + + public SeekPointFetcher(RegionScanner indexRegionScanner) { + this.indexRegionScanner = indexRegionScanner; + } + + /** + * Fetches the next N seek points for the scan. + * @param seekPoints + * @param noOfSeekPoints + * @return false when the scan on the index table for having no more rows remaining. + * @throws IOException + */ + public synchronized boolean nextSeekPoints(List seekPoints, int noOfSeekPoints) + throws IOException { + boolean hasMore = true; + List indexScanResult = new ArrayList(); + for (int i = 0; i < noOfSeekPoints; i++) { + hasMore = indexRegionScanner.next(indexScanResult); + if (indexScanResult.size() > 0) { + populateSeekPointsWithTableRowKey(seekPoints, indexScanResult.get(0)); + } + indexScanResult.clear(); + if (hasMore == false) break; + } + // TODO log the seekpoints INFO level. + return hasMore; + } + + private void populateSeekPointsWithTableRowKey(List seekPoints, KeyValue kv) { + byte[] row = kv.getRow(); + // Row key of the index table entry = region startkey + index name + column value(s) + // + actual table rowkey. + // Every row in the index table will have exactly one KV in that. The value will be + // 4 bytes. First 2 bytes specify length of the region start key bytes part in the + // rowkey. Last 2 bytes specify the offset to the actual table rowkey part within the + // index table rowkey. + byte[] value = kv.getValue(); + short actualRowKeyOffset = Bytes.toShort(value, 2); + if (LOG.isTraceEnabled()) { + LOG.trace("row value for the index table " + Bytes.toString(row)); + } + byte[] actualTableRowKey = new byte[row.length - actualRowKeyOffset]; + System.arraycopy(row, actualRowKeyOffset, actualTableRowKey, 0, actualTableRowKey.length); + seekPoints.add(actualTableRowKey); + } + + public synchronized void close() throws IOException { + this.indexRegionScanner.close(); + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekUnderValueException.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekUnderValueException.java new file mode 100644 index 0000000..77e2f94 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/SeekUnderValueException.java @@ -0,0 +1,29 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; + +// TODO better name? +public class SeekUnderValueException extends IOException { + + private static final long serialVersionUID = -4319585454456479286L; + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TTLExpiryChecker.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TTLExpiryChecker.java new file mode 100644 index 0000000..047e9cd --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TTLExpiryChecker.java @@ -0,0 +1,40 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import org.apache.hadoop.hbase.HConstants; + +public class TTLExpiryChecker { + + public boolean checkIfTTLExpired(long ttl, long timestamp) { + if (ttl == HConstants.FOREVER) { + return false; + } else if (ttl == -1) { + return false; + } else { + // second -> ms adjust for user data + ttl *= 1000; + } + if ((System.currentTimeMillis() - timestamp) > ttl) { + return true; + } + return false; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TTLStoreScanner.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TTLStoreScanner.java new file mode 100644 index 0000000..c84ed5f --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TTLStoreScanner.java @@ -0,0 +1,161 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreScanner; +import org.apache.hadoop.hbase.util.Bytes; + +public class TTLStoreScanner implements InternalScanner { + private InternalScanner delegate; + private Store store; + private long smallestReadPoint; + private long earliestTS; + private ScanType type; // (This should be scan type) + private static final Log LOG = LogFactory.getLog(TTLStoreScanner.class); + + private TTLExpiryChecker ttlExpiryChecker; + private String actualTableName; + private HRegionServer rs; + private Boolean userRegionAvailable = null; + + public TTLStoreScanner(Store store, long smallestReadPoint, long earliestTS, ScanType type, + List scanners, TTLExpiryChecker ttlExpiryChecker, + String actualTableName, HRegionServer rs) throws IOException { + this.store = store; + this.smallestReadPoint = smallestReadPoint; + this.earliestTS = earliestTS; + this.type = type; + Scan scan = new Scan(); + scan.setMaxVersions(store.getFamily().getMaxVersions()); + delegate = + new StoreScanner(store, store.getScanInfo(), scan, scanners, type, this.smallestReadPoint, + this.earliestTS); + this.ttlExpiryChecker = ttlExpiryChecker; + this.actualTableName = actualTableName; + this.rs = rs; + } + + @Override + public boolean next(List results) throws IOException { + return this.next(results, 1); + } + + @Override + public boolean next(List result, int limit) throws IOException { + boolean next = this.delegate.next(result, limit); + // Ideally here i should get only one result(i.e) only one kv + for (Iterator iterator = result.iterator(); iterator.hasNext();) { + KeyValue kv = (KeyValue) iterator.next(); + byte[] indexNameInBytes = formIndexNameFromKV(kv); + // From the indexname get the TTL + IndexSpecification index = + IndexManager.getInstance().getIndex(this.actualTableName, indexNameInBytes); + HRegion hRegion = store.getHRegion(); + if (this.type == ScanType.MAJOR_COMPACT) { + if (this.userRegionAvailable == null) { + this.userRegionAvailable = + isUserTableRegionAvailable(hRegion.getTableDesc().getNameAsString(), hRegion, this.rs); + } + // If index is null probably index is been dropped through drop index + // If user region not available it may be due to the reason that the user region has not yet + // opened but + // the index region has opened. + // Its better not to avoid the kv here, and write it during this current compaction. + // Anyway later compaction will avoid it. May lead to false positives but better than + // data loss + if (null == index && userRegionAvailable) { + // Remove the dropped index from the results + LOG.info("The index has been removed for the kv " + kv); + iterator.remove(); + continue; + } + } + if (index != null) { + boolean ttlExpired = + this.ttlExpiryChecker.checkIfTTLExpired(index.getTTL(), kv.getTimestamp()); + if (ttlExpired) { + result.clear(); + LOG.info("The ttl has expired for the kv " + kv); + return false; + } + } + } + return next; + } + + @Override + public void close() throws IOException { + this.delegate.close(); + } + + private byte[] formIndexNameFromKV(KeyValue kv) { + byte[] rowKey = kv.getRow(); + // First two bytes are going to be the + ByteArrayBuilder keyBuilder = ByteArrayBuilder.allocate(rowKey.length); + // Start from 2nd offset because the first 2 bytes corresponds to the rowkeylength + keyBuilder.put(rowKey, 0, rowKey.length); + int indexOf = com.google.common.primitives.Bytes.indexOf(keyBuilder.array(), new byte[1]); + return keyBuilder.array(indexOf + 1, IndexUtils.getMaxIndexNameLength()); + } + + private boolean isUserTableRegionAvailable(String indexTableName, HRegion indexRegion, + HRegionServer rs) { + Collection userRegions = rs.getOnlineRegions(Bytes.toBytes(this.actualTableName)); + for (HRegion userRegion : userRegions) { + // TODO start key check is enough? May be we can check for the + // possibility for N-1 Mapping? + if (Bytes.equals(userRegion.getStartKey(), indexRegion.getStartKey())) { + return true; + } + } + return false; + } + + @Override + public boolean next(List results, String metric) throws IOException { + // TODO Implement InternalScanner.next + return false; + } + + @Override + public boolean next(List result, int limit, String metric) throws IOException { + // TODO Implement InternalScanner.next + return false; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/wal/IndexWALObserver.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/wal/IndexWALObserver.java new file mode 100644 index 0000000..0dea9db --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/wal/IndexWALObserver.java @@ -0,0 +1,119 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.wal; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.WALCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.WALObserver; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver.IndexEdits; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; + +public class IndexWALObserver implements WALObserver { + + private static final Log LOG = LogFactory.getLog(IndexWALObserver.class); + + private IndexManager indexManager = IndexManager.getInstance(); + + @Override + public boolean preWALWrite(ObserverContext ctx, HRegionInfo info, + HLogKey logKey, WALEdit logEdit) throws IOException { + String tableNameStr = info.getTableNameAsString(); + if (IndexUtils.isCatalogTable(info.getTableName()) || IndexUtils.isIndexTable(tableNameStr)) { + return true; + } + List indices = indexManager.getIndicesForTable(tableNameStr); + if (indices != null && !indices.isEmpty()) { + LOG.trace("Entering preWALWrite for the table " + tableNameStr); + String indexTableName = IndexUtils.getIndexTableName(tableNameStr); + IndexEdits iEdits = IndexRegionObserver.threadLocal.get(); + WALEdit indexWALEdit = iEdits.getWALEdit(); + // This size will be 0 when none of the Mutations to the user table to be indexed. + // or write to WAL is disabled for the Mutations + if (indexWALEdit.getKeyValues().size() == 0) { + return true; + } + LOG.trace("Adding indexWALEdits into WAL for table " + tableNameStr); + HRegion indexRegion = iEdits.getRegion(); + // TS in all KVs within WALEdit will be the same. So considering the 1st one. + Long time = indexWALEdit.getKeyValues().get(0).getTimestamp(); + ctx.getEnvironment() + .getWAL() + .appendNoSync(indexRegion.getRegionInfo(), Bytes.toBytes(indexTableName), indexWALEdit, + logKey.getClusterId(), time, indexRegion.getTableDesc()); + LOG.trace("Exiting preWALWrite for the table " + tableNameStr); + } + return true; + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + } + + @Override + public void stop(CoprocessorEnvironment env) throws IOException { + } + + @Override + public void postWALWrite(ObserverContext ctx, HRegionInfo info, + HLogKey logKey, WALEdit logEdit) throws IOException { + } + + /* + * HLog log =ctx.getEnvironment().getWAL(); List kvList = logEdit.getKeyValues(); String + * userTableName = info.getEncodedName(); List indices = + * IndexManager.getInstance().getIndicesForTable(userTableName); if(null != indices){ + * handleWalEdit(userTableName, log, kvList, indices, info); } return false; } private void + * handleWalEdit(String userTableName, HLog log, List kvList, List + * indices, HRegionInfo info) { HashMap>> myMap = new + * HashMap>>(); for (KeyValue kv : kvList) { for + * (IndexSpecification idx : indices) { Set colSet = idx.getIndexColumns(); for + * (ColumnQualifier col : colSet) { if (Bytes.equals(kv.getFamily(), col.getColumnFamily()) && + * Bytes.equals(kv.getQualifier(), col.getQualifier())) { Map> + * mapOfCfToListOfKV = myMap.get(kv.getRow()); if (mapOfCfToListOfKV == null) { mapOfCfToListOfKV + * = new HashMap>(); myMap.put(kv.getRow(), mapOfCfToListOfKV); } // + * listOfKVs.add(kv); List listOfKV = mapOfCfToListOfKV.get(idx.getName().getBytes()); + * if(listOfKV == null ){ listOfKV = new ArrayList(); + * mapOfCfToListOfKV.put(idx.getName().getBytes(), listOfKV); } listOfKV.add(kv.clone()); } } } } + * createIndexWalEdit(myMap, indices, info); } private void createIndexWalEdit( HashMap>> myMap, List indices, HRegionInfo info) { int + * totalValueLength = 0; byte[] primaryRowKey = null; byte[] prStartKey = info.getStartKey(); for + * (Map.Entry>> mainEntry : myMap .entrySet()) { Map> idxMap = mainEntry.getValue(); for(IndexSpecification index : indices){ byte[] + * name = index.getName().getBytes(); if(null != idxMap.get(name)){ Set colSet = + * index.getIndexColumns(); for (ColumnQualifier c : colSet) { totalValueLength = totalValueLength + * + c.getMaxValueLength(); } primaryRowKey = mainEntry.getKey(); int rowLength = + * prStartKey.length + name.length; rowLength += totalValueLength; rowLength += + * primaryRowKey.length; } } } } + */ +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/filter/SingleColumnRangeFilter.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/filter/SingleColumnRangeFilter.java new file mode 100644 index 0000000..acb174e --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/filter/SingleColumnRangeFilter.java @@ -0,0 +1,145 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.filter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.FilterBase; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.util.Bytes; + +// TODO make it a full fledged Filter implementation +/** + * This is internal class. Not exposed as a Filter implementation + */ +public class SingleColumnRangeFilter extends FilterBase { + + private byte[] family; + + private byte[] qualifier; + + private CompareOp lowerCompareOp; + + private CompareOp upperCompareop; + + private byte[] upperBoundValue; + + private byte[] lowerBoundValue; + + private ValuePartition valuePartition = null; + + /** + * When the range is having only one bound pass that value and condition as the boundValue1 and + * boundOp1. Here boundValue2 and boundOp2 can be passed as null + * @param cf + * @param qualifier + * @param boundValue1 + * @param boundOp1 + * @param boundValue2 + * @param boundOp2 + */ + public SingleColumnRangeFilter(byte[] cf, byte[] qualifier, byte[] boundValue1, + CompareOp boundOp1, byte[] boundValue2, CompareOp boundOp2) { + this.family = cf; + this.qualifier = qualifier; + // CompareOp.LESS or CompareOp.LESS_OR_EQUAL will be the upper bound + if (boundOp1.equals(CompareOp.LESS) || boundOp1.equals(CompareOp.LESS_OR_EQUAL)) { + this.upperCompareop = boundOp1; + this.upperBoundValue = boundValue1; + this.lowerCompareOp = boundOp2; + this.lowerBoundValue = boundValue2; + } else { + this.upperCompareop = boundOp2; + this.upperBoundValue = boundValue2; + this.lowerCompareOp = boundOp1; + this.lowerBoundValue = boundValue1; + } + } + + public SingleColumnRangeFilter(byte[] cf, byte[] qualifier, ValuePartition vp, + byte[] boundValue1, CompareOp boundOp1, byte[] boundValue2, CompareOp boundOp2) { + this(cf, qualifier, boundValue1, boundOp1, boundValue2, boundOp2); + this.valuePartition = vp; + } + + @Override + public void readFields(DataInput datainput) throws IOException { + + } + + @Override + public void write(DataOutput dataoutput) throws IOException { + + } + + public void setUpperBoundValue(byte[] upperBoundValue, CompareOp upperOp) { + this.upperBoundValue = upperBoundValue; + this.upperCompareop = upperOp; + } + + public void setLowerBoundValue(byte[] lowerBoundValue, CompareOp lowerOp) { + this.lowerBoundValue = lowerBoundValue; + this.lowerCompareOp = lowerOp; + } + + public byte[] getFamily() { + return this.family; + } + + public byte[] getQualifier() { + return this.qualifier; + } + + public CompareOp getUpperBoundOp() { + return this.upperCompareop; + } + + public CompareOp getLowerBoundOp() { + return this.lowerCompareOp; + } + + public byte[] getLowerBoundValue() { + return this.lowerBoundValue; + } + + public byte[] getUpperBoundValue() { + return this.upperBoundValue; + } + + public ValuePartition getValuePartition() { + return valuePartition; + } + + public void setValuePartition(ValuePartition valuePartition) { + this.valuePartition = valuePartition; + } + + public String toString() { + return String.format("%s (%s, %s, %s, %s, %s, %s)", this.getClass().getSimpleName(), Bytes + .toStringBinary(this.family), Bytes.toStringBinary(this.qualifier), + this.lowerCompareOp == null ? "" : this.lowerCompareOp.name(), + this.lowerBoundValue == null ? "" : Bytes.toStringBinary(this.lowerBoundValue), + this.upperCompareop == null ? "" : this.upperCompareop.name(), + this.upperBoundValue == null ? "" : Bytes.toStringBinary(this.upperBoundValue)); + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/filter/SingleColumnValuePartitionFilter.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/filter/SingleColumnValuePartitionFilter.java new file mode 100644 index 0000000..c69d081 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/filter/SingleColumnValuePartitionFilter.java @@ -0,0 +1,153 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.filter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; +import org.apache.hadoop.hbase.index.SeparatorPartition; +import org.apache.hadoop.hbase.index.SpatialPartition; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.index.ValuePartition.PartitionType; + +/** + * This filter is used to filter cells based on a part of it's value. It takes a + * {@link CompareFilter.CompareOp} operator (equal, greater, not equal, etc), and either a byte [] + * value or a WritableByteArrayComparable. + *

+ * You must also specify a family and qualifier. Only the value of this column will be tested. When + * using this filter on a {@link Scan} with specified inputs, the column to be tested should also be + * added as input (otherwise the filter will regard the column as missing). + *

+ * To prevent the entire row from being emitted if the column is not found on a row, use + * {@link #setFilterIfMissing}. Otherwise, if the column is found, the entire row will be emitted + * only if the value passes. If the value fails, the row will be filtered out. + *

+ * In order to test values of previous versions (timestamps), set {@link #setLatestVersionOnly} to + * false. The default is true, meaning that only the latest version's value is tested and all + * previous versions are ignored. + */ +public class SingleColumnValuePartitionFilter extends SingleColumnValueFilter { + + private boolean foundColumn = false; + private boolean matchedColumn = false; + private ValuePartition valuePartition = null; + + /** + * Writable constructor, do not use. + */ + public SingleColumnValuePartitionFilter() { + } + + public SingleColumnValuePartitionFilter(final byte[] family, final byte[] qualifier, + final CompareOp compareOp, final byte[] value, ValuePartition vp) { + this(family, qualifier, compareOp, new BinaryComparator(value), vp); + } + + public SingleColumnValuePartitionFilter(final byte[] family, final byte[] qualifier, + final CompareOp compareOp, final WritableByteArrayComparable comparator, ValuePartition vp) { + super(family, qualifier, compareOp, comparator); + this.valuePartition = vp; + } + + public ValuePartition getValuePartition() { + return valuePartition; + } + + public boolean filterRow() { + // If column was found, return false if it was matched, true if it was not + // If column not found, return true if we filter if missing, false if not + return this.foundColumn ? !this.matchedColumn : this.getFilterIfMissing(); + } + + public void reset() { + foundColumn = false; + matchedColumn = false; + } + + public ReturnCode filterKeyValue(KeyValue keyValue) { + if (this.matchedColumn) { + // We already found and matched the single column, all keys now pass + return ReturnCode.INCLUDE; + } else if (this.getLatestVersionOnly() && this.foundColumn) { + // We found but did not match the single column, skip to next row + return ReturnCode.NEXT_ROW; + } + if (!keyValue.matchingColumn(this.columnFamily, this.columnQualifier)) { + return ReturnCode.INCLUDE; + } + foundColumn = true; + byte[] value = valuePartition.getPartOfValue(keyValue.getValue()); + if (filterColumnValue(value, 0, value.length)) { + return this.getLatestVersionOnly() ? ReturnCode.NEXT_ROW : ReturnCode.INCLUDE; + } + this.matchedColumn = true; + return ReturnCode.INCLUDE; + } + + private boolean filterColumnValue(final byte[] data, final int offset, final int length) { + int compareResult = this.getComparator().compareTo(data, offset, length); + switch (this.getOperator()) { + case LESS: + return compareResult <= 0; + case LESS_OR_EQUAL: + return compareResult < 0; + case EQUAL: + return compareResult != 0; + case NOT_EQUAL: + return compareResult == 0; + case GREATER_OR_EQUAL: + return compareResult > 0; + case GREATER: + return compareResult >= 0; + default: + throw new RuntimeException("Unknown Compare op " + this.getOperator().name()); + } + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeUTF(valuePartition.getPartitionType().name()); + valuePartition.write(out); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + PartitionType p = PartitionType.valueOf(in.readUTF()); + if (p.equals(PartitionType.SEPARATOR)) { + valuePartition = new SeparatorPartition(); + } else if (p.equals(PartitionType.SPATIAL)) { + valuePartition = new SpatialPartition(); + } + if (valuePartition != null) { + valuePartition.readFields(in); + } + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/io/IndexHalfStoreFileReader.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/io/IndexHalfStoreFileReader.java new file mode 100644 index 0000000..bb39de2 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/io/IndexHalfStoreFileReader.java @@ -0,0 +1,433 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.io; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * A facade for a {@link org.apache.hadoop.hbase.io.hfile.HFile.Reader} that serves up either the + * top or bottom half of a HFile where 'bottom' is the first half of the file containing the keys + * that sort lowest and 'top' is the second half of the file with keys that sort greater than those + * of the bottom half. The top includes the split files midkey, of the key that follows if it does + * not exist in the file. + *

+ * This type works in tandem with the {@link Reference} type. This class is used reading while + * Reference is used writing. + *

+ * This file is not splitable. Calls to {@link #midkey()} return null. + */ +// TODO check some implementations here. Seems buggy!! +public class IndexHalfStoreFileReader extends StoreFile.Reader { + private static final int ROW_KEY_LENGTH = 2; + private final boolean top; + // This is the key we split around. Its the first possible entry on a row: + // i.e. empty column and a timestamp of LATEST_TIMESTAMP. + private final byte[] splitkey; + private final byte[] splitRow; + + /** + * @param p + * @param cacheConf + * @param r + * @throws IOException + */ + public IndexHalfStoreFileReader(final FileSystem fs, final Path p, final CacheConfig cacheConf, + final Reference r, DataBlockEncoding preferredEncodingInCache) throws IOException { + super(fs, p, cacheConf, preferredEncodingInCache); + this.splitkey = r.getSplitKey(); + // Is it top or bottom half? + this.top = Reference.isTopFileRegion(r.getFileRegion()); + this.splitRow = KeyValue.createKeyValueFromKey(splitkey).getRow(); + } + + protected boolean isTop() { + return this.top; + } + + @Override + public HFileScanner getScanner(final boolean cacheBlocks, final boolean pread, + final boolean isCompaction) { + final HFileScanner s = super.getScanner(cacheBlocks, pread, isCompaction); + return new HFileScanner() { + final HFileScanner delegate = s; + public boolean atEnd = false; + + public ByteBuffer getKey() { + if (atEnd) { + return null; + } + if (!top) { + return delegate.getKey(); + } + // If it is top store file replace the StartKey of the Key with SplitKey + return getChangedKey(delegate.getKeyValue()); + } + + private ByteBuffer getChangedKey(KeyValue kv) { + // new KeyValue(row, family, qualifier, timestamp, type, value) + byte[] newRowkey = getNewRowkeyByRegionStartKeyReplacedWithSplitKey(kv); + KeyValue newKv = + new KeyValue(newRowkey, kv.getFamily(), kv.getQualifier(), kv.getTimestamp(), + Type.codeToType(kv.getType()), null); + ByteBuffer keyBuffer = ByteBuffer.wrap(newKv.getKey()); + return keyBuffer; + } + + private byte[] getNewRowkeyByRegionStartKeyReplacedWithSplitKey(KeyValue kv) { + // TODO any other way when Delete type? + if (KeyValue.isDelete(kv.getType()) && kv.getValue().length == 0) { + return replaceDeleteKeyWithSplitKey(kv.getRow()); + } + byte[] original = kv.getRow(); + byte[] value = kv.getValue(); + int lenOfRegionStartKey = Bytes.toShort(value, 0); // 1st 2 bytes length of the region + int lenOfRemainingKey = original.length - lenOfRegionStartKey; + byte[] keyReplacedStartKey = new byte[lenOfRemainingKey + splitRow.length]; + System.arraycopy(splitRow, 0, keyReplacedStartKey, 0, splitRow.length); + System.arraycopy(original, lenOfRegionStartKey, keyReplacedStartKey, splitRow.length, + lenOfRemainingKey); + return keyReplacedStartKey; + } + + public String getKeyString() { + if (atEnd) { + return null; + } + return Bytes.toStringBinary(getKey()); + } + + public ByteBuffer getValue() { + if (atEnd) { + return null; + } + if (!top) { + return delegate.getValue(); + } + // If it is top store file change the value corresponding to the changed key like + // [first 2 bytes]StartKey length replace with SplitKey length + // [last 2 bytes]ActualRowKey offset add with difference of SplitKey & StartKey + byte[] changedValue = getChangedValue(delegate.getKeyValue().getValue()); + return ByteBuffer.wrap(changedValue); + } + + private byte[] getChangedValue(byte[] value) { + if (value.length == 0) return value; // The value can be empty when the KV type is DELETE. + int lenghtOfTheStartKey = Bytes.toShort(value, 0); + int offsetOfActualKey = Bytes.toShort(value, 2); + offsetOfActualKey = offsetOfActualKey + (splitRow.length - lenghtOfTheStartKey); + byte[] changedValue = new byte[4]; + System + .arraycopy(Bytes.toBytes((short) splitRow.length), 0, changedValue, 0, ROW_KEY_LENGTH); + System.arraycopy(Bytes.toBytes((short) offsetOfActualKey), 0, changedValue, ROW_KEY_LENGTH, + ROW_KEY_LENGTH); + return changedValue; + } + + public String getValueString() { + if (atEnd) { + return null; + } + return Bytes.toStringBinary(getValue()); + } + + public KeyValue getKeyValue() { + if (atEnd) { + return null; + } + KeyValue kv = delegate.getKeyValue(); + if (!top) { + return kv; + } + // If it is a top store file change the StartKey with SplitKey in Key + // and produce the new value corresponding to the change in key + byte[] changedKey = getNewRowkeyByRegionStartKeyReplacedWithSplitKey(kv); + byte[] changedValue = getChangedValue(kv.getValue()); + KeyValue changedKv = + new KeyValue(changedKey, kv.getFamily(), kv.getQualifier(), kv.getTimestamp(), + Type.codeToType(kv.getType()), changedValue); + return changedKv; + } + + public boolean next() throws IOException { + if (atEnd) { + return false; + } + // TODO check what will be returned when next moves the cursor to the last entry + // in the file + while (true) { + boolean b = delegate.next(); + if (!b) { + atEnd = true; + return b; + } + // We need to check whether the current KV pointed by this reader is corresponding to + // this split or not. + // In case of top store file if the ActualRowKey >= SplitKey + // In case of bottom store file if the ActualRowKey < Splitkey + if (isSatisfiedMidKeyCondition(delegate.getKeyValue())) { + return true; + } + } + } + + public boolean seekBefore(byte[] key) throws IOException { + return seekBefore(key, 0, key.length); + } + + public boolean seekBefore(byte[] key, int offset, int length) throws IOException { + if (top) { + byte[] fk = getFirstKey(); + // This will be null when the file is empty in which we can not seekBefore to any key + if (fk == null) { + return false; + } + if (getComparator().compare(key, offset, length, fk, 0, fk.length) <= 0) { + return false; + } + KeyValue replacedKey = getKeyPresentInHFiles(key); + return this.delegate.seekBefore(replacedKey.getBuffer(), replacedKey.getKeyOffset(), + replacedKey.getKeyLength()); + } else { + // The equals sign isn't strictly necessary just here to be consistent with seekTo + if (getComparator().compare(key, offset, length, splitkey, 0, splitkey.length) >= 0) { + return this.delegate.seekBefore(splitkey, 0, splitkey.length); + } + } + return this.delegate.seekBefore(key, offset, length); + } + + public boolean seekTo() throws IOException { + boolean b = delegate.seekTo(); + if (!b) { + atEnd = true; + return b; + } + while (true) { + // We need to check the first occurrence of satisfying the condition + // In case of top store file if the ActualRowKey >= SplitKey + // In case of bottom store file if the ActualRowKey < Splitkey + if (isSatisfiedMidKeyCondition(delegate.getKeyValue())) { + return true; + } + b = delegate.next(); + if (!b) { + return b; + } + } + } + + public int seekTo(byte[] key) throws IOException { + return seekTo(key, 0, key.length); + } + + public int seekTo(byte[] key, int offset, int length) throws IOException { + if (top) { + if (getComparator().compare(key, offset, length, splitkey, 0, splitkey.length) < 0) { + return -1; + } + KeyValue replacedKey = getKeyPresentInHFiles(key); + + int seekTo = + delegate.seekTo(replacedKey.getBuffer(), replacedKey.getKeyOffset(), + replacedKey.getKeyLength()); + return seekTo; + /* + * if (seekTo == 0 || seekTo == -1) { return seekTo; } else if (seekTo == 1) { boolean + * next = this.next(); } + */ + } else { + if (getComparator().compare(key, offset, length, splitkey, 0, splitkey.length) >= 0) { + // we would place the scanner in the second half. + // it might be an error to return false here ever... + boolean res = delegate.seekBefore(splitkey, 0, splitkey.length); + if (!res) { + throw new IOException( + "Seeking for a key in bottom of file, but key exists in top of file, failed on seekBefore(midkey)"); + } + return 1; + } + } + return delegate.seekTo(key, offset, length); + } + + public int reseekTo(byte[] key) throws IOException { + return reseekTo(key, 0, key.length); + } + + public int reseekTo(byte[] key, int offset, int length) throws IOException { + if (top) { + if (getComparator().compare(key, offset, length, splitkey, 0, splitkey.length) < 0) { + return -1; + } + KeyValue replacedKey = getKeyPresentInHFiles(key); + return delegate.reseekTo(replacedKey.getBuffer(), replacedKey.getKeyOffset(), + replacedKey.getKeyLength()); + } else { + if (getComparator().compare(key, offset, length, splitkey, 0, splitkey.length) >= 0) { + // we would place the scanner in the second half. + // it might be an error to return false here ever... + boolean res = delegate.seekBefore(splitkey, 0, splitkey.length); + if (!res) { + throw new IOException( + "Seeking for a key in bottom of file, but key exists in top of file, failed on seekBefore(midkey)"); + } + return 1; + } + } + return delegate.reseekTo(key, offset, length); + } + + public org.apache.hadoop.hbase.io.hfile.HFile.Reader getReader() { + return this.delegate.getReader(); + } + + // TODO: Need to change as per IndexHalfStoreFileReader + public boolean isSeeked() { + return this.delegate.isSeeked(); + } + }; + } + + private boolean isSatisfiedMidKeyCondition(KeyValue kv) { + if (KeyValue.isDelete(kv.getType()) && kv.getValue().length == 0) { + // In case of a Delete type KV, let it be going to both the daughter regions. + // No problems in doing so. In the correct daughter region where it belongs to, this delete + // tomb will really delete a KV. In the other it will just hang around there with no actual + // kv coming for which this is a delete tomb. :) + return true; + } + byte[] row = kv.getRow(); + int offsetToActuRowKey = Bytes.toShort(kv.getValue(), ROW_KEY_LENGTH); + int actuRowKeyLength = (row.length - offsetToActuRowKey); + byte[] actuRowKey = new byte[actuRowKeyLength]; + System.arraycopy(row, offsetToActuRowKey, actuRowKey, 0, actuRowKeyLength); + int compareResult = Bytes.compareTo(actuRowKey, splitRow); + if (top) { + if (compareResult >= 0) { + return true; + } + } else { + if (compareResult < 0) { + return true; + } + } + return false; + } + + // In case of top half store, the passed key will be with the start key of the daughter region. + // But in the actual HFiles, the key will be with the start key of the old parent region. + // In order to make the real seek in the HFiles, we need to build the old key. + private KeyValue getKeyPresentInHFiles(byte[] key) { + KeyValue keyValue = new KeyValue(key); + KeyValue keyValCopy = keyValue.shallowCopy(); + int rowLength = keyValCopy.getRowLength(); + int rowOffset = keyValCopy.getRowOffset(); + byte[] row = keyValCopy.getRow(); + int rowIndex = com.google.common.primitives.Bytes.indexOf(row, new byte[1]); + byte[] firstKey = getFirstKey(); + // In firstkey first 2 bytes will reperesent the key length so don't consider it + byte[] actualFirstKey = new byte[firstKey.length - ROW_KEY_LENGTH]; + // copy from 2nd position of firstkey + System.arraycopy(firstKey, ROW_KEY_LENGTH, actualFirstKey, 0, firstKey.length - ROW_KEY_LENGTH); + // Get the main table start key using the one byte as separator + int firstindex = com.google.common.primitives.Bytes.indexOf(actualFirstKey, new byte[1]); + byte[] startRow = new byte[firstindex]; + System.arraycopy(actualFirstKey, 0, startRow, 0, firstindex); + + // This comes incase of deletefamily + if (top && 0 == keyValCopy.getValueLength() + && keyValCopy.getTimestamp() == HConstants.LATEST_TIMESTAMP + && Bytes.compareTo(row, splitRow) == 0 && keyValCopy.isDeleteFamily()) { + KeyValue createFirstDeleteFamilyOnRow = + KeyValue.createFirstDeleteFamilyOnRow(startRow, keyValCopy.getFamily()); + return createFirstDeleteFamilyOnRow; + } + + byte[] rowAfterSplitKey = new byte[row.length - rowIndex]; + byte[] afterRow = new byte[key.length - (rowOffset + rowLength)]; + byte[] replacedKey = + new byte[rowAfterSplitKey.length + afterRow.length + firstindex + ROW_KEY_LENGTH]; + + // copy the bytes after split key til the row end + System.arraycopy(row, rowIndex, rowAfterSplitKey, 0, row.length - rowIndex); + // Copy the bytes after row till end + System.arraycopy(key, rowOffset + rowLength, afterRow, 0, + (key.length - (rowOffset + rowLength))); + + short length = (short) (rowAfterSplitKey.length + firstindex); + byte[] rowKeyLengthBytes = Bytes.toBytes(length); + // This is for padding the row length to the first 2 byte positions + System.arraycopy(rowKeyLengthBytes, 0, replacedKey, 0, rowKeyLengthBytes.length); + // Copy the actualFirstKey till firstIndex to replacedKey.. This will be the start key of main + // table + System.arraycopy(actualFirstKey, 0, replacedKey, ROW_KEY_LENGTH, firstindex); + // Now copy the rowAfterSplitKey + System.arraycopy(rowAfterSplitKey, 0, replacedKey, firstindex + rowKeyLengthBytes.length, + rowAfterSplitKey.length); + // Now copy the afterRow part + System.arraycopy(afterRow, 0, replacedKey, firstindex + rowAfterSplitKey.length + + rowKeyLengthBytes.length, afterRow.length); + return KeyValue.createKeyValueFromKey(replacedKey); + } + + public byte[] getLastKey() { + // This method wont get used for the index region. There is no need to call getClosestRowBefore + // on the index table. Also this is a split region. Can not be further split + throw new UnsupportedOperationException("Method is not implemented!"); + } + + private byte[] replaceDeleteKeyWithSplitKey(byte[] key) { + int lenOfRegionStartKeyPart = com.google.common.primitives.Bytes.indexOf(getFirstKey(), new byte[1]); + int remainingKeyLen = key.length - lenOfRegionStartKeyPart; + byte[] replacedKey = new byte[remainingKeyLen + splitRow.length]; + System.arraycopy(splitRow, 0, replacedKey, 0, splitRow.length); + System.arraycopy(key, lenOfRegionStartKeyPart, replacedKey, splitRow.length, remainingKeyLen); + return replacedKey; + } + + public byte[] midkey() throws IOException { + // Returns null to indicate file is not splitable. + return null; + } + + @Override + public byte[] getFirstKey() { + return super.getFirstKey(); + } + + @Override + public boolean passesKeyRangeFilter(Scan scan) { + return true; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/manager/IndexManager.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/manager/IndexManager.java new file mode 100644 index 0000000..14c4ace --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/manager/IndexManager.java @@ -0,0 +1,149 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.manager; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * IndexManager manages the index details of each table. + */ +public class IndexManager { + + // manager is Singleton object + private static IndexManager manager = new IndexManager(); + + private Map> tableVsIndices = + new ConcurrentHashMap>(); + + private ConcurrentHashMap tableVsNumberOfRegions = + new ConcurrentHashMap(); + + // TODO one DS is enough + private Map> tableIndexMap = + new ConcurrentHashMap>(); + + private IndexManager() { + + } + + /** + * @return IndexManager instance + */ + public static IndexManager getInstance() { + return manager; + } + + /** + * @param table name on which index applying + * @param IndexSpecification list of table + */ + public void addIndexForTable(String tableName, List indexList) { + this.tableVsIndices.put(tableName, indexList); + // TODO the inner map needs to be thread safe when we support dynamic index add/remove + Map indexMap = + new TreeMap(Bytes.BYTES_COMPARATOR); + for (IndexSpecification index : indexList) { + ByteArrayBuilder keyBuilder = ByteArrayBuilder.allocate(IndexUtils.getMaxIndexNameLength()); + keyBuilder.put(Bytes.toBytes(index.getName())); + indexMap.put(keyBuilder.array(), index); + } + this.tableIndexMap.put(tableName, indexMap); + } + + /** + * @param table name on which index applying + * @return IndexSpecification list for the table or return null if no index for the table + */ + public List getIndicesForTable(String tableName) { + return this.tableVsIndices.get(tableName); + } + + /** + * @param tableName on which index applying + */ + public void removeIndices(String tableName) { + this.tableVsIndices.remove(tableName); + this.tableIndexMap.remove(tableName); + } + + /** + * @param tableName + * @param indexName + * @return + */ + public IndexSpecification getIndex(String tableName, byte[] indexName) { + Map indices = this.tableIndexMap.get(tableName); + if (indices != null) { + return indices.get(indexName); + } + return null; + } + + public void incrementRegionCount(String tableName) { + AtomicInteger count = this.tableVsNumberOfRegions.get(tableName); + // Here synchronization is needed for the first time count operation to be + // initialized + if (null == count) { + synchronized (tableVsNumberOfRegions) { + count = this.tableVsNumberOfRegions.get(tableName); + if (null == count) { + count = new AtomicInteger(0); + this.tableVsNumberOfRegions.put(tableName, count); + } + } + } + count.incrementAndGet(); + + } + + public void decrementRegionCount(String tableName, boolean removeIndices) { + // Need not be synchronized here because anyway the decrement operator + // will work atomically. Ultimately atleast one thread will see the count + // to be 0 which should be sufficient to remove the indices + AtomicInteger count = this.tableVsNumberOfRegions.get(tableName); + if (null != count) { + int next = count.decrementAndGet(); + if (next == 0) { + this.tableVsNumberOfRegions.remove(tableName); + if (removeIndices) { + this.removeIndices(tableName); + } + } + } + } + + // API needed for test cases. + public int getTableRegionCount(String tableName) { + AtomicInteger count = this.tableVsNumberOfRegions.get(tableName); + if (count != null) { + return count.get(); + } + return 0; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexCreationMapper.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexCreationMapper.java new file mode 100644 index 0000000..828f524 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexCreationMapper.java @@ -0,0 +1,63 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.TableMapper; + +public class IndexCreationMapper extends TableMapper { + public void map(ImmutableBytesWritable row, Result value, Context context) throws IOException, + InterruptedException { + Configuration conf = context.getConfiguration(); + resultToPut(context, row, value, conf); + } + + private void resultToPut(Context context, ImmutableBytesWritable key, Result result, + Configuration conf) throws IOException, InterruptedException { + byte[] row = key.get(); + Put put = null; + for (KeyValue kv : result.raw()) { + if (kv.isDelete()) { + // Skipping delete records as any way the deletes will mask the actual + // puts + continue; + } else { + if (put == null) { + put = new Put(row); + } + put.add(kv); + } + } + if (put != null) { + List indexPuts = IndexMapReduceUtil.getIndexPut(put, conf); + for (Put idxPut : indexPuts) { + context.write(new ImmutableBytesWritable(idxPut.getRow()), idxPut); + } + } + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexHFileOutputFormat.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexHFileOutputFormat.java new file mode 100644 index 0000000..07f30d2 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexHFileOutputFormat.java @@ -0,0 +1,290 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; +import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder; +import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; + +public class IndexHFileOutputFormat extends FileOutputFormat { + + static Log LOG = LogFactory.getLog(IndexHFileOutputFormat.class); + static final String COMPRESSION_CONF_KEY = "hbase.hfileoutputformat.families.compression"; + private static final String DATABLOCK_ENCODING_CONF_KEY = + "hbase.mapreduce.hfileoutputformat.datablock.encoding"; + TimeRangeTracker trt = new TimeRangeTracker(); + + public static void configureIncrementalLoad(Job job, HTable table) throws IOException { + HFileOutputFormat.configureIncrementalLoad(job, table); + // Override OutputFormatClass + job.setOutputFormatClass(IndexHFileOutputFormat.class); + } + + public RecordWriter getRecordWriter( + final TaskAttemptContext context) throws IOException, InterruptedException { + + // Get the path of the temporary output file + final Path outputPath = FileOutputFormat.getOutputPath(context); + final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath(); + + final Configuration conf = context.getConfiguration(); + final FileSystem fs = outputdir.getFileSystem(conf); + + // These configs. are from hbase-*.xml + final long maxsize = + conf.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE); + final int blocksize = + conf.getInt("hbase.mapreduce.hfileoutputformat.blocksize", HFile.DEFAULT_BLOCKSIZE); + // Invented config. Add to hbase-*.xml if other than default compression. + final String defaultCompression = + conf.get("hfile.compression", Compression.Algorithm.NONE.getName()); + final boolean compactionExclude = + conf.getBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", false); + + final boolean indexedTable = conf.getBoolean(IndexMapReduceUtil.INDEX_IS_INDEXED_TABLE, false); + + final Path indexDir = new Path(outputdir, IndexMapReduceUtil.INDEX_DATA_DIR); + + final FileSystem indexFs = indexDir.getFileSystem(conf); + + if (indexedTable) { + if (!indexFs.exists(indexDir)) { + indexFs.mkdirs(indexDir); + } + } + + // create a map from column family to the compression algorithm + final Map compressionMap = createFamilyCompressionMap(conf); + + String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_CONF_KEY); + final HFileDataBlockEncoder encoder; + if (dataBlockEncodingStr == null) { + encoder = NoOpDataBlockEncoder.INSTANCE; + } else { + try { + encoder = new HFileDataBlockEncoderImpl(DataBlockEncoding.valueOf(dataBlockEncodingStr)); + } catch (IllegalArgumentException ex) { + throw new RuntimeException("Invalid data block encoding type configured for the param " + + DATABLOCK_ENCODING_CONF_KEY + " : " + dataBlockEncodingStr); + } + } + + return new RecordWriter() { + // Map of families to writers and how much has been output on the writer. + private final Map writers = new TreeMap( + Bytes.BYTES_COMPARATOR); + private byte[] previousRow = HConstants.EMPTY_BYTE_ARRAY; + private final byte[] now = Bytes.toBytes(System.currentTimeMillis()); + private boolean rollRequested = false; + + public void write(ImmutableBytesWritable row, KeyValue kv) throws IOException { + // null input == user explicitly wants to flush + if (row == null && kv == null) { + rollWriters(); + return; + } + boolean indexed = false; + + byte[] rowKey = kv.getRow(); + long length = kv.getLength(); + byte[] family = kv.getFamily(); + byte[] qualifier = kv.getQualifier(); + + if (Bytes.equals(family, Constants.IDX_COL_FAMILY) + && Bytes.equals(qualifier, Constants.IDX_COL_QUAL)) { + indexed = true; + } + + WriterLength wl = null; + if (indexed) { + wl = this.writers.get(Bytes.toBytes(IndexMapReduceUtil.INDEX_DATA_DIR)); + } else { + wl = this.writers.get(family); + } + + // If this is a new column family, verify that the directory exists + if (wl == null) { + if (indexed) { + indexFs.mkdirs(new Path(indexDir, Bytes.toString(family))); + } else { + fs.mkdirs(new Path(outputdir, Bytes.toString(family))); + } + } + + // If any of the HFiles for the column families has reached + // maxsize, we need to roll all the writers + if (wl != null && wl.written + length >= maxsize) { + this.rollRequested = true; + } + + // This can only happen once a row is finished though + if (rollRequested && Bytes.compareTo(this.previousRow, rowKey) != 0) { + rollWriters(); + } + + // create a new HLog writer, if necessary + if (wl == null || wl.writer == null) { + wl = getNewWriter(family, conf, indexed); + } + + // we now have the proper HLog writer. full steam ahead + kv.updateLatestStamp(this.now); + trt.includeTimestamp(kv); + wl.writer.append(kv); + wl.written += length; + + // Copy the row so we know when a row transition. + this.previousRow = rowKey; + } + + private void rollWriters() throws IOException { + for (WriterLength wl : this.writers.values()) { + if (wl.writer != null) { + LOG.info("Writer=" + wl.writer.getPath() + + ((wl.written == 0) ? "" : ", wrote=" + wl.written)); + close(wl.writer); + } + wl.writer = null; + wl.written = 0; + } + this.rollRequested = false; + } + + /* + * Create a new HFile.Writer. + * @param family + * @return A WriterLength, containing a new HFile.Writer. + * @throws IOException + */ + private WriterLength getNewWriter(byte[] family, Configuration conf, boolean indexData) + throws IOException { + WriterLength wl = new WriterLength(); + + Path familydir = null; + + String compression = compressionMap.get(family); + compression = compression == null ? defaultCompression : compression; + + if (indexData) { + familydir = new Path(indexDir, Bytes.toString(family)); + wl.writer = + HFile.getWriterFactoryNoCache(conf) + .withPath(indexFs, StoreFile.getUniqueFile(indexFs, familydir)) + .withBlockSize(blocksize).withCompression(compression) + .withComparator(KeyValue.KEY_COMPARATOR).withDataBlockEncoder(encoder) + .withChecksumType(Store.getChecksumType(conf)) + .withBytesPerChecksum(Store.getBytesPerChecksum(conf)).create(); + this.writers.put(Bytes.toBytes(IndexMapReduceUtil.INDEX_DATA_DIR), wl); + } else { + familydir = new Path(outputdir, Bytes.toString(family)); + wl.writer = + HFile.getWriterFactoryNoCache(conf) + .withPath(fs, StoreFile.getUniqueFile(fs, familydir)).withBlockSize(blocksize) + .withCompression(compression).withComparator(KeyValue.KEY_COMPARATOR) + .withDataBlockEncoder(encoder).withChecksumType(Store.getChecksumType(conf)) + .withBytesPerChecksum(Store.getBytesPerChecksum(conf)).create(); + this.writers.put(family, wl); + } + + return wl; + } + + private void close(final HFile.Writer w) throws IOException { + if (w != null) { + w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis())); + w.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY, + Bytes.toBytes(context.getTaskAttemptID().toString())); + w.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY, Bytes.toBytes(true)); + w.appendFileInfo(StoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY, + Bytes.toBytes(compactionExclude)); + w.appendFileInfo(StoreFile.TIMERANGE_KEY, WritableUtils.toByteArray(trt)); + w.close(); + } + } + + public void close(TaskAttemptContext c) throws IOException, InterruptedException { + for (WriterLength wl : this.writers.values()) { + close(wl.writer); + } + } + }; + + } + + /** + * Run inside the task to deserialize column family to compression algorithm map from the + * configuration. Package-private for unit tests only. + * @return a map from column family to the name of the configured compression algorithm + */ + static Map createFamilyCompressionMap(Configuration conf) { + Map compressionMap = new TreeMap(Bytes.BYTES_COMPARATOR); + String compressionConf = conf.get(COMPRESSION_CONF_KEY, ""); + for (String familyConf : compressionConf.split("&")) { + String[] familySplit = familyConf.split("="); + if (familySplit.length != 2) { + continue; + } + + try { + compressionMap.put(Bytes.toBytes(URLDecoder.decode(familySplit[0], "UTF-8")), + URLDecoder.decode(familySplit[1], "UTF-8")); + } catch (UnsupportedEncodingException e) { + // will not happen with UTF-8 encoding + throw new AssertionError(e); + } + } + return compressionMap; + } + + static class WriterLength { + long written = 0; + HFile.Writer writer = null; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexImportTsv.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexImportTsv.java new file mode 100644 index 0000000..d5e8683 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexImportTsv.java @@ -0,0 +1,448 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.PutSortReducer; +import org.apache.hadoop.hbase.mapreduce.TableInputFormat; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.util.Base64; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.util.GenericOptionsParser; + +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; + +public class IndexImportTsv { + + final static String NAME = "indeximporttsv"; + final static String MAPPER_CONF_KEY = "importtsv.mapper.class"; + final static String SKIP_LINES_CONF_KEY = "importtsv.skip.bad.lines"; + final static String BULK_OUTPUT_CONF_KEY = "importtsv.bulk.output"; + final static String COLUMNS_CONF_KEY = "importtsv.columns"; + final static String SEPARATOR_CONF_KEY = "importtsv.separator"; + final static String TIMESTAMP_CONF_KEY = "importtsv.timestamp"; + final static String DEFAULT_SEPARATOR = "\t"; + final static Class DEFAULT_MAPPER = IndexTsvImporterMapper.class; + + private static HBaseAdmin hbaseAdmin; + + static class TsvParser { + /** + * Column families and qualifiers mapped to the TSV columns + */ + private final byte[][] families; + private final byte[][] qualifiers; + + private final byte separatorByte; + + private int rowKeyColumnIndex; + + private int maxColumnCount; + + // Default value must be negative + public static final int DEFAULT_TIMESTAMP_COLUMN_INDEX = -1; + + private int timestampKeyColumnIndex = DEFAULT_TIMESTAMP_COLUMN_INDEX; + + public static String ROWKEY_COLUMN_SPEC = "HBASE_ROW_KEY"; + + public static String TIMESTAMPKEY_COLUMN_SPEC = "HBASE_TS_KEY"; + + /** + * @param columnsSpecification the list of columns to parser out, comma separated. The row key + * should be the special token TsvParser.ROWKEY_COLUMN_SPEC + */ + public TsvParser(String columnsSpecification, String separatorStr) { + // Configure separator + byte[] separator = Bytes.toBytes(separatorStr); + Preconditions.checkArgument(separator.length == 1, + "TsvParser only supports single-byte separators"); + separatorByte = separator[0]; + + // Configure columns + ArrayList columnStrings = + Lists.newArrayList(Splitter.on(',').trimResults().split(columnsSpecification)); + + maxColumnCount = columnStrings.size(); + families = new byte[maxColumnCount][]; + qualifiers = new byte[maxColumnCount][]; + + for (int i = 0; i < columnStrings.size(); i++) { + String str = columnStrings.get(i); + if (ROWKEY_COLUMN_SPEC.equals(str)) { + rowKeyColumnIndex = i; + continue; + } + + if (TIMESTAMPKEY_COLUMN_SPEC.equals(str)) { + timestampKeyColumnIndex = i; + continue; + } + + String[] parts = str.split(":", 2); + if (parts.length == 1) { + families[i] = Bytes.toBytes(str); + qualifiers[i] = HConstants.EMPTY_BYTE_ARRAY; + } else { + families[i] = Bytes.toBytes(parts[0]); + qualifiers[i] = Bytes.toBytes(parts[1]); + } + } + } + + public int getRowKeyColumnIndex() { + return rowKeyColumnIndex; + } + + public boolean hasTimestamp() { + return timestampKeyColumnIndex != DEFAULT_TIMESTAMP_COLUMN_INDEX; + } + + public int getTimestampKeyColumnIndex() { + return timestampKeyColumnIndex; + } + + public byte[] getFamily(int idx) { + return families[idx]; + } + + public byte[] getQualifier(int idx) { + return qualifiers[idx]; + } + + public ParsedLine parse(byte[] lineBytes, int length) throws BadTsvLineException { + // Enumerate separator offsets + ArrayList tabOffsets = new ArrayList(maxColumnCount); + for (int i = 0; i < length; i++) { + if (lineBytes[i] == separatorByte) { + tabOffsets.add(i); + } + } + if (tabOffsets.isEmpty()) { + throw new BadTsvLineException("No delimiter"); + } + + tabOffsets.add(length); + + if (tabOffsets.size() > maxColumnCount) { + throw new BadTsvLineException("Excessive columns"); + } else if (tabOffsets.size() <= getRowKeyColumnIndex()) { + throw new BadTsvLineException("No row key"); + } else if (hasTimestamp() && tabOffsets.size() <= getTimestampKeyColumnIndex()) { + throw new BadTsvLineException("No timestamp"); + } + return new ParsedLine(tabOffsets, lineBytes); + } + + class ParsedLine { + private final ArrayList tabOffsets; + private byte[] lineBytes; + + ParsedLine(ArrayList tabOffsets, byte[] lineBytes) { + this.tabOffsets = tabOffsets; + this.lineBytes = lineBytes; + } + + public int getRowKeyOffset() { + return getColumnOffset(rowKeyColumnIndex); + } + + public int getRowKeyLength() { + return getColumnLength(rowKeyColumnIndex); + } + + public long getTimestamp(long ts) throws BadTsvLineException { + // Return ts if HBASE_TS_KEY is not configured in column spec + if (!hasTimestamp()) { + return ts; + } + + try { + return Long.parseLong(Bytes.toString(lineBytes, getColumnOffset(timestampKeyColumnIndex), + getColumnLength(timestampKeyColumnIndex))); + } catch (NumberFormatException nfe) { + // treat this record as bad record + throw new BadTsvLineException("Invalid timestamp"); + } + } + + public int getColumnOffset(int idx) { + if (idx > 0) return tabOffsets.get(idx - 1) + 1; + else return 0; + } + + public int getColumnLength(int idx) { + return tabOffsets.get(idx) - getColumnOffset(idx); + } + + public int getColumnCount() { + return tabOffsets.size(); + } + + public byte[] getLineBytes() { + return lineBytes; + } + } + + public static class BadTsvLineException extends Exception { + public BadTsvLineException(String err) { + super(err); + } + + private static final long serialVersionUID = 1L; + } + } + + /** + * Sets up the actual job. + * @param conf The current configuration. + * @param args The command line parameters. + * @return The newly created job. + * @throws IOException When setting up the job fails. + * @throws InterruptedException + */ + public static Job createSubmittableJob(Configuration conf, String[] args) throws IOException, + ClassNotFoundException, InterruptedException { + + // Support non-XML supported characters + // by re-encoding the passed separator as a Base64 string. + String actualSeparator = conf.get(SEPARATOR_CONF_KEY); + if (actualSeparator != null) { + conf.set(SEPARATOR_CONF_KEY, Base64.encodeBytes(actualSeparator.getBytes())); + } + + // See if a non-default Mapper was set + String mapperClassName = conf.get(MAPPER_CONF_KEY); + Class mapperClass = mapperClassName != null ? Class.forName(mapperClassName) : DEFAULT_MAPPER; + + String tableName = args[0]; + Path inputDir = new Path(args[1]); + + String input = conf.get(IndexUtils.TABLE_INPUT_COLS); + if (!doesTableExist(tableName)) { + createTable(conf, tableName); + if (input != null) { + IndexUtils.createIndexTable(tableName, conf, null); + } + } + + conf.set(TableInputFormat.INPUT_TABLE, tableName); + boolean indexedTable = IndexMapReduceUtil.isIndexedTable(conf); + conf.setBoolean(IndexMapReduceUtil.INDEX_IS_INDEXED_TABLE, indexedTable); + Job job = new Job(conf, NAME + "_" + tableName); + job.setJarByClass(mapperClass); + FileInputFormat.setInputPaths(job, inputDir); + job.setInputFormatClass(TextInputFormat.class); + job.setMapperClass(mapperClass); + + String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); + if (hfileOutPath != null) { + HTable table = new HTable(conf, tableName); + job.setReducerClass(PutSortReducer.class); + Path outputDir = new Path(hfileOutPath); + FileOutputFormat.setOutputPath(job, outputDir); + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + job.setMapOutputValueClass(Put.class); + IndexHFileOutputFormat.configureIncrementalLoad(job, table); + } else { + // No reducers. Just write straight to table. Call initTableReducerJob + // to set up the TableOutputFormat. + TableMapReduceUtil.initTableReducerJob(tableName, null, job); + job.setNumReduceTasks(0); + } + + TableMapReduceUtil.addDependencyJars(job); + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + com.google.common.base.Function.class /* Guava used by TsvParser */); + return job; + } + + static boolean doesTableExist(String tableName) throws IOException { + return hbaseAdmin.tableExists(Bytes.toBytes(tableName)); + } + + static void createTable(Configuration conf, String tableName) throws IOException { + HTableDescriptor htd = new HTableDescriptor(Bytes.toBytes(tableName)); + String columns[] = conf.getStrings(COLUMNS_CONF_KEY); + Set cfSet = new HashSet(); + for (String aColumn : columns) { + if (TsvParser.ROWKEY_COLUMN_SPEC.equals(aColumn)) continue; + // we are only concerned with the first one (in case this is a cf:cq) + cfSet.add(aColumn.split(":", 2)[0]); + } + for (String cf : cfSet) { + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes(cf)); + htd.addFamily(hcd); + } + hbaseAdmin.createTable(htd); + } + + /* + * @param errorMsg Error message. Can be null. + */ + private static void usage(final String errorMsg) { + if (errorMsg != null && errorMsg.length() > 0) { + System.err.println("ERROR: " + errorMsg); + } + String usage = + "Usage: " + + NAME + + " -Dimporttsv.columns=a,b,c \n" + + "\n" + + "Imports the given input directory of TSV data into the specified table.\n" + + "\n" + + "The column names of the TSV data must be specified using the -Dimporttsv.columns\n" + + "option. This option takes the form of comma-separated column names, where each\n" + + "column name is either a simple column family, or a columnfamily:qualifier. The special\n" + + "column name HBASE_ROW_KEY is used to designate that this column should be used\n" + + "as the row key for each imported record. You must specify exactly one column\n" + + "to be the row key, and you must specify a column name for every column that exists in the\n" + + "input data. Another special column HBASE_TS_KEY designates that this column should be\n" + + "used as timestamp for each record. Unlike HBASE_ROW_KEY, HBASE_TS_KEY is optional.\n" + + "You must specify atmost one column as timestamp key for each imported record.\n" + + "Record with invalid timestamps (blank, non-numeric) will be treated as bad record.\n" + + "Note: if you use this option, then 'importtsv.timestamp' option will be ignored.\n" + + "\n" + + "By default importtsv will load data directly into HBase. To instead generate\n" + + "HFiles of data to prepare for a bulk data load, pass the option:\n" + + " -D" + + BULK_OUTPUT_CONF_KEY + + "=/path/for/output\n" + + " Note: if you do not use this option, then the target table must already exist in HBase\n" + + "\n" + + "Other options that may be specified with -D include:\n" + + " -D" + + SKIP_LINES_CONF_KEY + + "=false - fail if encountering an invalid line\n" + + " '-D" + + SEPARATOR_CONF_KEY + + "=|' - eg separate on pipes instead of tabs\n" + + " -D" + + TIMESTAMP_CONF_KEY + + "=currentTimeAsLong - use the specified timestamp for the import\n" + + " -D" + + MAPPER_CONF_KEY + + "=my.Mapper - A user-defined Mapper to use instead of " + + DEFAULT_MAPPER.getName() + + "\n" + + "For performance consider the following options:\n" + + " -Dmapred.map.tasks.speculative.execution=false\n" + + " -Dmapred.reduce.tasks.speculative.execution=false\n" + + " -Dtable.columns.index='IDX1=>cf1:[q1->datatype& length],[q2]," + + "[q3];cf2:[q1->datatype&length],[q2->datatype&length],[q3->datatype& lenght]#IDX2=>cf1:q5,q5'" + + " The format used here is: \n" + " IDX1 - Index name\n" + + " cf1 - Columnfamilyname\n" + " q1 - qualifier\n" + + " datatype - datatype (Int, String, Double, Float)\n" + + " length - length of the value\n" + + " The columnfamily should be seperated by ';'\n" + + " The qualifier and the datatype and its length should be enclosed in '[]'.\n" + + " The qualifier details are specified using '->' following qualifer name and the" + + " details are seperated by '&'\n" + + " If the qualifier details are not specified default values are used.\n" + + " # is used to seperate between two index details"; + System.err.println(usage); + } + + /** + * Used only by test method + * @param conf + */ + static void createHbaseAdmin(Configuration conf) throws IOException { + hbaseAdmin = new HBaseAdmin(conf); + } + + /** + * Main entry point. + * @param args The command line parameters. + * @throws Exception When running the job fails. + */ + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); + if (otherArgs.length < 2) { + usage("Wrong number of arguments: " + otherArgs.length); + System.exit(-1); + } + + // Make sure columns are specified + String columns[] = conf.getStrings(COLUMNS_CONF_KEY); + if (columns == null) { + usage("No columns specified. Please specify with -D" + COLUMNS_CONF_KEY + "=..."); + System.exit(-1); + } + + // Make sure they specify exactly one column as the row key + int rowkeysFound = 0; + for (String col : columns) { + if (col.equals(TsvParser.ROWKEY_COLUMN_SPEC)) rowkeysFound++; + } + if (rowkeysFound != 1) { + usage("Must specify exactly one column as " + TsvParser.ROWKEY_COLUMN_SPEC); + System.exit(-1); + } + + // Make sure we have at most one column as the timestamp key + int tskeysFound = 0; + for (String col : columns) { + if (col.equals(TsvParser.TIMESTAMPKEY_COLUMN_SPEC)) tskeysFound++; + } + if (tskeysFound > 1) { + usage("Must specify at most one column as " + TsvParser.TIMESTAMPKEY_COLUMN_SPEC); + System.exit(-1); + } + + // Make sure one or more columns are specified excluding rowkey and timestamp key + if (columns.length - (rowkeysFound + tskeysFound) < 1) { + usage("One or more columns in addition to the row key and timestamp(optional) are required"); + System.exit(-1); + } + + // If timestamp option is not specified, use current system time. + long timstamp = conf.getLong(TIMESTAMP_CONF_KEY, System.currentTimeMillis()); + + // Set it back to replace invalid timestamp (non-numeric) with current system time + conf.setLong(TIMESTAMP_CONF_KEY, timstamp); + + hbaseAdmin = new HBaseAdmin(conf); + Job job = createSubmittableJob(conf, otherArgs); + System.exit(job.waitForCompletion(true) ? 0 : 1); + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexLoadIncrementalHFile.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexLoadIncrementalHFile.java new file mode 100644 index 0000000..e77537c --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexLoadIncrementalHFile.java @@ -0,0 +1,682 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Deque; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.ServerCallable; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.HalfStoreFileReader; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.io.Reference.Range; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * Tool to load the output of HFileOutputFormat into an existing table. + * @see #usage() + */ +public class IndexLoadIncrementalHFile extends Configured implements Tool { + + private static Log LOG = LogFactory.getLog(IndexLoadIncrementalHFile.class); + static AtomicLong regionCount = new AtomicLong(0); + private HBaseAdmin hbAdmin; + private Configuration cfg; + + public static String NAME = "completebulkload"; + + public IndexLoadIncrementalHFile(Configuration conf) throws Exception { + super(conf); + this.cfg = conf; + this.hbAdmin = new HBaseAdmin(conf); + } + + private void usage() { + System.err.println("usage: " + NAME + " /path/to/hfileoutputformat-output " + "tablename"); + } + + /** + * Represents an HFile waiting to be loaded. An queue is used in this class in order to support + * the case where a region has split during the process of the load. When this happens, the HFile + * is split into two physical parts across the new region boundary, and each part is added back + * into the queue. The import process finishes when the queue is empty. + */ + static class LoadQueueItem { + final byte[] family; + final Path hfilePath; + + public LoadQueueItem(byte[] family, Path hfilePath) { + this.family = family; + this.hfilePath = hfilePath; + } + + public String toString() { + return "family:" + Bytes.toString(family) + " path:" + hfilePath.toString(); + } + } + + /** + * Walk the given directory for all HFiles, and return a Queue containing all such files. + */ + private void discoverLoadQueue(Deque ret, Path hfofDir) throws IOException { + FileSystem fs = hfofDir.getFileSystem(getConf()); + + if (!fs.exists(hfofDir)) { + throw new FileNotFoundException("HFileOutputFormat dir " + hfofDir + " not found"); + } + + FileStatus[] familyDirStatuses = fs.listStatus(hfofDir); + if (familyDirStatuses == null) { + throw new FileNotFoundException("No families found in " + hfofDir); + } + + for (FileStatus stat : familyDirStatuses) { + if (!stat.isDir()) { + LOG.warn("Skipping non-directory " + stat.getPath()); + continue; + } + Path familyDir = stat.getPath(); + // Skip _logs and .index, etc + if (familyDir.getName().startsWith("_") + || familyDir.getName().startsWith(IndexMapReduceUtil.INDEX_DATA_DIR)) continue; + byte[] family = Bytes.toBytes(familyDir.getName()); + Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); + for (Path hfile : hfiles) { + if (hfile.getName().startsWith("_")) continue; + ret.add(new LoadQueueItem(family, hfile)); + } + } + } + + /** + * Perform a bulk load of the given directory into the given pre-existing table. This method is + * not threadsafe. + * @param hfofDir the directory that was provided as the output path of a job using + * HFileOutputFormat + * @param table the table to load into + * @throws TableNotFoundException if table does not yet exist + */ + public void doBulkLoad(Path hfofDir, final HTable table) throws TableNotFoundException, + IOException { + final HConnection conn = table.getConnection(); + + if (!conn.isTableAvailable(table.getTableName())) { + throw new TableNotFoundException("Table " + Bytes.toStringBinary(table.getTableName()) + + "is not currently available."); + } + + // initialize thread pools + int nrThreads = + cfg.getInt("hbase.loadincremental.threads.max", Runtime.getRuntime().availableProcessors()); + ThreadFactoryBuilder builder = new ThreadFactoryBuilder(); + builder.setNameFormat("LoadIncrementalHFiles-%1$d"); + ExecutorService pool = + new ThreadPoolExecutor(nrThreads, nrThreads, 60, TimeUnit.SECONDS, + new LinkedBlockingQueue(), builder.build()); + ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true); + + // LQI queue does not need to be threadsafe -- all operations on this queue + // happen in this thread + Deque queue = new LinkedList(); + try { + discoverLoadQueue(queue, hfofDir); + int count = 0; + + if (queue.isEmpty()) { + LOG.warn("Bulk load operation did not find any files to load in " + "directory " + + hfofDir.toUri() + ". Does it contain files in " + + "subdirectories that correspond to column family names?"); + return; + } + + if (queue.isEmpty()) { + LOG.warn("Bulk load operation did not find any files to load in " + "directory " + + hfofDir.toUri() + ". Does it contain files in " + + "subdirectories that correspond to column family names?"); + } + + // Assumes that region splits can happen while this occurs. + while (!queue.isEmpty()) { + // need to reload split keys each iteration. + final Pair startEndKeys = table.getStartEndKeys(); + if (count != 0) { + LOG.info("Split occured while grouping HFiles, retry attempt " + +count + " with " + + queue.size() + " files remaining to group or split"); + } + + int maxRetries = cfg.getInt("hbase.bulkload.retries.number", 0); + if (maxRetries != 0 && count >= maxRetries) { + LOG.error("Retry attempted " + count + " times without completing, bailing out"); + return; + } + count++; + + // Using ByteBuffer for byte[] equality semantics + Multimap regionGroups = + groupOrSplitPhase(table, pool, queue, startEndKeys); + + bulkLoadPhase(table, conn, pool, queue, regionGroups); + + // NOTE: The next iteration's split / group could happen in parallel to + // atomic bulkloads assuming that there are splits and no merges, and + // that we can atomically pull out the groups we want to retry. + } + + } finally { + pool.shutdown(); + if (queue != null && !queue.isEmpty()) { + StringBuilder err = new StringBuilder(); + err.append("-------------------------------------------------\n"); + err.append("Bulk load aborted with some files not yet loaded:\n"); + err.append("-------------------------------------------------\n"); + for (LoadQueueItem q : queue) { + err.append(" ").append(q.hfilePath).append('\n'); + } + LOG.error(err); + } + } + } + + /** + * This takes the LQI's grouped by likely regions and attempts to bulk load them. Any failures are + * re-queued for another pass with the groupOrSplitPhase. + */ + protected void bulkLoadPhase(final HTable table, final HConnection conn, ExecutorService pool, + Deque queue, final Multimap regionGroups) + throws IOException { + // atomically bulk load the groups. + Set>> loadingFutures = new HashSet>>(); + for (Entry> e : regionGroups.asMap().entrySet()) { + final byte[] first = e.getKey().array(); + final Collection lqis = e.getValue(); + + final Callable> call = new Callable>() { + public List call() throws Exception { + List toRetry = + tryAtomicRegionLoad(conn, table.getTableName(), first, lqis); + return toRetry; + } + }; + loadingFutures.add(pool.submit(call)); + } + + // get all the results. + for (Future> future : loadingFutures) { + try { + List toRetry = future.get(); + + // LQIs that are requeued to be regrouped. + queue.addAll(toRetry); + + } catch (ExecutionException e1) { + Throwable t = e1.getCause(); + if (t instanceof IOException) { + // At this point something unrecoverable has happened. + // TODO Implement bulk load recovery + throw new IOException("BulkLoad encountered an unrecoverable problem", t); + } + LOG.error("Unexpected execution exception during bulk load", e1); + throw new IllegalStateException(t); + } catch (InterruptedException e1) { + LOG.error("Unexpected interrupted exception during bulk load", e1); + throw new IllegalStateException(e1); + } + } + } + + /** + * @return A Multimap that groups LQI by likely bulk load region targets. + */ + private Multimap + groupOrSplitPhase(final HTable table, ExecutorService pool, Deque queue, + final Pair startEndKeys) throws IOException { + // need synchronized only within this scope of this + // phase because of the puts that happen in futures. + Multimap rgs = HashMultimap.create(); + final Multimap regionGroups = Multimaps.synchronizedMultimap(rgs); + + // drain LQIs and figure out bulk load groups + Set>> splittingFutures = new HashSet>>(); + while (!queue.isEmpty()) { + final LoadQueueItem item = queue.remove(); + + final Callable> call = new Callable>() { + public List call() throws Exception { + List splits = groupOrSplit(regionGroups, item, table, startEndKeys); + return splits; + } + }; + splittingFutures.add(pool.submit(call)); + } + // get all the results. All grouping and splitting must finish before + // we can attempt the atomic loads. + for (Future> lqis : splittingFutures) { + try { + List splits = lqis.get(); + if (splits != null) { + queue.addAll(splits); + } + } catch (ExecutionException e1) { + Throwable t = e1.getCause(); + if (t instanceof IOException) { + LOG.error("IOException during splitting", e1); + throw (IOException) t; // would have been thrown if not parallelized, + } + LOG.error("Unexpected execution exception during splitting", e1); + throw new IllegalStateException(t); + } catch (InterruptedException e1) { + LOG.error("Unexpected interrupted exception during splitting", e1); + throw new IllegalStateException(e1); + } + } + return regionGroups; + } + + // unique file name for the table + String getUniqueName(byte[] tableName) { + String name = Bytes.toStringBinary(tableName) + "," + regionCount.incrementAndGet(); + return name; + } + + protected List splitStoreFile(final LoadQueueItem item, final HTable table, + byte[] startKey, byte[] splitKey) throws IOException { + final Path hfilePath = item.hfilePath; + + // We use a '_' prefix which is ignored when walking directory trees + // above. + final Path tmpDir = new Path(item.hfilePath.getParent(), "_tmp"); + + LOG.info("HFile at " + hfilePath + " no longer fits inside a single " + "region. Splitting..."); + + String uniqueName = getUniqueName(table.getTableName()); + HColumnDescriptor familyDesc = table.getTableDescriptor().getFamily(item.family); + Path botOut = new Path(tmpDir, uniqueName + ".bottom"); + Path topOut = new Path(tmpDir, uniqueName + ".top"); + splitStoreFile(getConf(), hfilePath, familyDesc, splitKey, botOut, topOut); + + // Add these back at the *front* of the queue, so there's a lower + // chance that the region will just split again before we get there. + List lqis = new ArrayList(2); + lqis.add(new LoadQueueItem(item.family, botOut)); + lqis.add(new LoadQueueItem(item.family, topOut)); + + LOG.info("Successfully split into new HFiles " + botOut + " and " + topOut); + return lqis; + } + + /** + * Attempt to assign the given load queue item into its target region group. If the hfile boundary + * no longer fits into a region, physically splits the hfile such that the new bottom half will + * fit and returns the list of LQI's corresponding to the resultant hfiles. protected for testing + */ + protected List groupOrSplit(Multimap regionGroups, + final LoadQueueItem item, final HTable table, final Pair startEndKeys) + throws IOException { + final Path hfilePath = item.hfilePath; + final FileSystem fs = hfilePath.getFileSystem(getConf()); + HFile.Reader hfr = HFile.createReader(fs, hfilePath, new CacheConfig(getConf())); + final byte[] first, last; + try { + hfr.loadFileInfo(); + first = hfr.getFirstRowKey(); + last = hfr.getLastRowKey(); + } finally { + hfr.close(); + } + + LOG.info("Trying to load hfile=" + hfilePath + " first=" + Bytes.toStringBinary(first) + + " last=" + Bytes.toStringBinary(last)); + if (first == null || last == null) { + assert first == null && last == null; + // TODO what if this is due to a bad HFile? + LOG.info("hfile " + hfilePath + " has no entries, skipping"); + return null; + } + if (Bytes.compareTo(first, last) > 0) { + throw new IllegalArgumentException("Invalid range: " + Bytes.toStringBinary(first) + " > " + + Bytes.toStringBinary(last)); + } + int idx = Arrays.binarySearch(startEndKeys.getFirst(), first, Bytes.BYTES_COMPARATOR); + if (idx < 0) { + // not on boundary, returns -(insertion index). Calculate region it + // would be in. + idx = -(idx + 1) - 1; + } + final int indexForCallable = idx; + boolean lastKeyInRange = + Bytes.compareTo(last, startEndKeys.getSecond()[idx]) < 0 + || Bytes.equals(startEndKeys.getSecond()[idx], HConstants.EMPTY_BYTE_ARRAY); + if (!lastKeyInRange) { + List lqis = + splitStoreFile(item, table, startEndKeys.getFirst()[indexForCallable], + startEndKeys.getSecond()[indexForCallable]); + return lqis; + } + + // group regions. + regionGroups.put(ByteBuffer.wrap(startEndKeys.getFirst()[idx]), item); + return null; + } + + /** + * Attempts to do an atomic load of many hfiles into a region. If it fails, it returns a list of + * hfiles that need to be retried. If it is successful it will return an empty list. NOTE: To + * maintain row atomicity guarantees, region server callable should succeed atomically and fails + * atomically. Protected for testing. + * @return empty list if success, list of items to retry on recoverable failure + */ + protected List tryAtomicRegionLoad(final HConnection conn, byte[] tableName, + final byte[] first, Collection lqis) throws IOException { + + final List> famPaths = new ArrayList>(lqis.size()); + for (LoadQueueItem lqi : lqis) { + famPaths.add(Pair.newPair(lqi.family, lqi.hfilePath.toString())); + } + + final ServerCallable svrCallable = + new ServerCallable(conn, tableName, first) { + @Override + public Boolean call() throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("Going to connect to server " + location + " for row " + + Bytes.toStringBinary(row)); + } + byte[] regionName = location.getRegionInfo().getRegionName(); + return server.bulkLoadHFiles(famPaths, regionName); + } + }; + + try { + List toRetry = new ArrayList(); + boolean success = svrCallable.withRetries(); + if (!success) { + LOG.warn("Attempt to bulk load region containing " + Bytes.toStringBinary(first) + + " into table " + Bytes.toStringBinary(tableName) + " with files " + lqis + + " failed. This is recoverable and they will be retried."); + toRetry.addAll(lqis); // return lqi's to retry + } + // success + return toRetry; + } catch (IOException e) { + LOG.error("Encountered unrecoverable error from region server", e); + throw e; + } + } + + /** + * Split a storefile into a top and bottom half, maintaining the metadata, recreating bloom + * filters, etc. + */ + static void splitStoreFile(Configuration conf, Path inFile, HColumnDescriptor familyDesc, + byte[] splitKey, Path bottomOut, Path topOut) throws IOException { + // Open reader with no block cache, and not in-memory + Reference topReference = new Reference(splitKey, Range.top); + Reference bottomReference = new Reference(splitKey, Range.bottom); + + copyHFileHalf(conf, inFile, topOut, topReference, familyDesc); + copyHFileHalf(conf, inFile, bottomOut, bottomReference, familyDesc); + } + + /** + * Copy half of an HFile into a new HFile. + */ + private static void copyHFileHalf(Configuration conf, Path inFile, Path outFile, + Reference reference, HColumnDescriptor familyDescriptor) throws IOException { + FileSystem fs = inFile.getFileSystem(conf); + CacheConfig cacheConf = new CacheConfig(conf); + HalfStoreFileReader halfReader = null; + StoreFile.Writer halfWriter = null; + HFileDataBlockEncoder dataBlockEncoder = + new HFileDataBlockEncoderImpl(familyDescriptor.getDataBlockEncodingOnDisk(), + familyDescriptor.getDataBlockEncoding()); + try { + halfReader = + new HalfStoreFileReader(fs, inFile, cacheConf, reference, DataBlockEncoding.NONE); + Map fileInfo = halfReader.loadFileInfo(); + + int blocksize = familyDescriptor.getBlocksize(); + Algorithm compression = familyDescriptor.getCompression(); + BloomType bloomFilterType = familyDescriptor.getBloomFilterType(); + + halfWriter = + new StoreFile.WriterBuilder(conf, cacheConf, fs, blocksize).withFilePath(outFile) + .withCompression(compression).withDataBlockEncoder(dataBlockEncoder) + .withBloomType(bloomFilterType).withChecksumType(Store.getChecksumType(conf)) + .withBytesPerChecksum(Store.getBytesPerChecksum(conf)).build(); + HFileScanner scanner = halfReader.getScanner(false, false, false); + scanner.seekTo(); + do { + KeyValue kv = scanner.getKeyValue(); + halfWriter.append(kv); + } while (scanner.next()); + + for (Map.Entry entry : fileInfo.entrySet()) { + if (shouldCopyHFileMetaKey(entry.getKey())) { + halfWriter.appendFileInfo(entry.getKey(), entry.getValue()); + } + } + } finally { + if (halfWriter != null) halfWriter.close(); + if (halfReader != null) halfReader.close(cacheConf.shouldEvictOnClose()); + } + } + + private static boolean shouldCopyHFileMetaKey(byte[] key) { + return !HFile.isReservedFileInfoKey(key); + } + + private boolean doesTableExist(String tableName) throws Exception { + return hbAdmin.tableExists(tableName); + } + + /* + * Infers region boundaries for a new table. Parameter: bdryMap is a map between keys to an + * integer belonging to {+1, -1} If a key is a start key of a file, then it maps to +1 If a key is + * an end key of a file, then it maps to -1 Algo: 1) Poll on the keys in order: a) Keep adding the + * mapped values to these keys (runningSum) b) Each time runningSum reaches 0, add the start Key + * from when the runningSum had started to a boundary list. 2) Return the boundary list. + */ + public static byte[][] inferBoundaries(TreeMap bdryMap) { + ArrayList keysArray = new ArrayList(); + int runningValue = 0; + byte[] currStartKey = null; + boolean firstBoundary = true; + + for (Map.Entry item : bdryMap.entrySet()) { + if (runningValue == 0) currStartKey = item.getKey(); + runningValue += item.getValue(); + if (runningValue == 0) { + if (!firstBoundary) keysArray.add(currStartKey); + firstBoundary = false; + } + } + + return keysArray.toArray(new byte[0][0]); + } + + /* + * If the table is created for the first time, then "completebulkload" reads the files twice. More + * modifications necessary if we want to avoid doing it. + */ + private void createTable(String tableName, String dirPath) throws Exception { + Path hfofDir = new Path(dirPath); + FileSystem fs = hfofDir.getFileSystem(getConf()); + + if (!fs.exists(hfofDir)) { + throw new FileNotFoundException("HFileOutputFormat dir " + hfofDir + " not found"); + } + + FileStatus[] familyDirStatuses = fs.listStatus(hfofDir); + if (familyDirStatuses == null) { + throw new FileNotFoundException("No families found in " + hfofDir); + } + + HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = null; + + // Add column families + // Build a set of keys + byte[][] keys = null; + TreeMap map = new TreeMap(Bytes.BYTES_COMPARATOR); + + for (FileStatus stat : familyDirStatuses) { + if (!stat.isDir()) { + LOG.warn("Skipping non-directory " + stat.getPath()); + continue; + } + Path familyDir = stat.getPath(); + // Skip _logs & .index etc + if (familyDir.getName().startsWith("_")) continue; + + if (familyDir.getName().startsWith(IndexMapReduceUtil.INDEX_DATA_DIR)) { + LOG.warn("Ignoring all the HFile specific to " + tableName + " indexed data."); + continue; + } + + byte[] family = Bytes.toBytes(familyDir.getName()); + + hcd = new HColumnDescriptor(family); + htd.addFamily(hcd); + + Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); + for (Path hfile : hfiles) { + if (hfile.getName().startsWith("_")) continue; + HFile.Reader reader = HFile.createReader(fs, hfile, new CacheConfig(getConf())); + final byte[] first, last; + try { + if (hcd.getCompressionType() != reader.getCompressionAlgorithm()) { + hcd.setCompressionType(reader.getCompressionAlgorithm()); + LOG.info("Setting compression " + hcd.getCompressionType().name() + " for family " + + hcd.toString()); + } + reader.loadFileInfo(); + first = reader.getFirstRowKey(); + last = reader.getLastRowKey(); + + LOG.info("Trying to figure out region boundaries hfile=" + hfile + " first=" + + Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last)); + + // To eventually infer start key-end key boundaries + Integer value = map.containsKey(first) ? (Integer) map.get(first) : 0; + map.put(first, value + 1); + + value = map.containsKey(last) ? (Integer) map.get(last) : 0; + map.put(last, value - 1); + } finally { + reader.close(); + } + } + } + + keys = IndexLoadIncrementalHFile.inferBoundaries(map); + this.hbAdmin.createTable(htd, keys); + + LOG.info("Table " + tableName + " is available!!"); + } + + @Override + public int run(String[] args) throws Exception { + if (args.length != 2) { + usage(); + return -1; + } + + String dirPath = args[0]; + String tableName = args[1]; + + boolean tableExists = this.doesTableExist(tableName); + if (!tableExists) this.createTable(tableName, dirPath); + + String indexTableName = IndexUtils.getIndexTableName(tableName); + boolean indexedTable = this.hbAdmin.isTableAvailable(indexTableName); + if (indexedTable) { + // load the index data to the indextable + Path indxhfDir = new Path(dirPath, IndexMapReduceUtil.INDEX_DATA_DIR); + HTable idxTable = new HTable(this.cfg, indexTableName); + doBulkLoad(indxhfDir, idxTable); + } + + Path hfofDir = new Path(dirPath); + HTable table = new HTable(this.cfg, tableName); + doBulkLoad(hfofDir, table); + + return 0; + } + + public static void main(String[] args) throws Exception { + int ret = ToolRunner.run(new IndexLoadIncrementalHFile(HBaseConfiguration.create()), args); + System.exit(ret); + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexMapReduceUtil.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexMapReduceUtil.java new file mode 100644 index 0000000..6ab261b --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexMapReduceUtil.java @@ -0,0 +1,131 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.mapreduce.TableInputFormat; +import org.apache.hadoop.hbase.util.Bytes; + +public class IndexMapReduceUtil { + + public static final String INDEX_DATA_DIR = ".index"; + + public static final String INDEX_IS_INDEXED_TABLE = "indeximporttsv.isindexedtable"; + + static Log LOG = LogFactory.getLog(IndexMapReduceUtil.class); + + private static IndexedHTableDescriptor hTableDescriptor = null; + private static String tblName = null; + + private static byte[][] startKeys = null; + + public static IndexedHTableDescriptor getTableDescriptor(String tableName, Configuration conf) + throws IOException { + if (hTableDescriptor == null || !tableName.equals(hTableDescriptor.getNameAsString())) { + hTableDescriptor = IndexUtils.getIndexedHTableDescriptor(Bytes.toBytes(tableName), conf); + } + return hTableDescriptor; + } + + public static boolean isIndexedTable(String tableName, Configuration conf) throws IOException { + IndexedHTableDescriptor tableDescriptor = getTableDescriptor(tableName, conf); + return tableDescriptor != null; + } + + public static boolean isIndexedTable(Configuration conf) throws IOException { + String tableName = conf.get(TableInputFormat.INPUT_TABLE); + return isIndexedTable(tableName, conf); + } + + public static List getIndexPut(Put userPut, Configuration conf) throws IOException { + String tableName = conf.get(TableInputFormat.INPUT_TABLE); + IndexedHTableDescriptor tableDescriptor = getTableDescriptor(tableName, conf); + List indexPuts = new ArrayList(); + if (tableDescriptor != null) { + List indices = tableDescriptor.getIndices(); + for (IndexSpecification index : indices) { + byte[] startkey = getStartKey(conf, tableName, userPut.getRow()); + Put indexPut = IndexUtils.prepareIndexPut(userPut, index, startkey); + if (indexPut != null) { + indexPuts.add(indexPut); + } + } + } + return indexPuts; + } + + public static List getIndexDelete(Delete userDelete, Configuration conf) + throws IOException { + String tableName = conf.get(TableInputFormat.INPUT_TABLE); + IndexedHTableDescriptor tableDescriptor = getTableDescriptor(tableName, conf); + List indexDeletes = new ArrayList(); + if (tableDescriptor != null) { + List indices = tableDescriptor.getIndices(); + for (IndexSpecification index : indices) { + byte[] startkey = getStartKey(conf, tableName, userDelete.getRow()); + Delete indexDelete = IndexUtils.prepareIndexDelete(userDelete, index, startkey); + if (indexDelete != null) { + indexDeletes.add(indexDelete); + } + } + } + return indexDeletes; + } + + public static byte[] getStartKey(Configuration conf, String tableName, byte[] row) + throws IOException { + + if (startKeys == null || startKeys.length == 0 || !tableName.equals(tblName)) { + tblName = tableName; + HTable table = null; + try { + table = new HTable(conf, tableName); + startKeys = table.getStartKeys(); + } finally { + if (table != null) { + table.close(); + } + } + } + if (startKeys.length != 0) { + for (int i = 0; i < (startKeys.length - 1); i++) { + int diff = Bytes.compareTo(row, startKeys[i]); + if (diff == 0 || (diff > 0 && Bytes.compareTo(row, startKeys[i + 1]) < 0)) { + return startKeys[i]; + } + } + return startKeys[startKeys.length - 1]; + } + return null; + } + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexTsvImporterMapper.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexTsvImporterMapper.java new file mode 100644 index 0000000..de720f6 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/IndexTsvImporterMapper.java @@ -0,0 +1,181 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Base64; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.Mapper; + +public class IndexTsvImporterMapper extends Mapper { + + /** Timestamp for all inserted rows */ + private long ts; + + /** Column seperator */ + private String separator; + + /** Should skip bad lines */ + private boolean skipBadLines; + private Counter badLineCount; + + private IndexImportTsv.TsvParser parser; + + private String hfileOutPath; + + private boolean indexedTable; + + public long getTs() { + return ts; + } + + public boolean getSkipBadLines() { + return skipBadLines; + } + + public Counter getBadLineCount() { + return badLineCount; + } + + public void incrementBadLineCount(int count) { + this.badLineCount.increment(count); + } + + /** + * Handles initializing this class with objects specific to it (i.e., the parser). Common + * initialization that might be leveraged by a subsclass is done in doSetup. Hence a + * subclass may choose to override this method and call doSetup as well before + * handling it's own custom params. + * @param context + */ + @Override + protected void setup(Context context) { + doSetup(context); + + Configuration conf = context.getConfiguration(); + + parser = new IndexImportTsv.TsvParser(conf.get(IndexImportTsv.COLUMNS_CONF_KEY), separator); + if (parser.getRowKeyColumnIndex() == -1) { + throw new RuntimeException("No row key column specified"); + } + } + + /** + * Handles common parameter initialization that a subclass might want to leverage. + * @param context + */ + protected void doSetup(Context context) { + Configuration conf = context.getConfiguration(); + + // If a custom separator has been used, + // decode it back from Base64 encoding. + separator = conf.get(IndexImportTsv.SEPARATOR_CONF_KEY); + if (separator == null) { + separator = IndexImportTsv.DEFAULT_SEPARATOR; + } else { + separator = new String(Base64.decode(separator)); + } + + // Should never get 0 as we are setting this to a valid value in job configuration. + ts = conf.getLong(IndexImportTsv.TIMESTAMP_CONF_KEY, 0); + + skipBadLines = context.getConfiguration().getBoolean(IndexImportTsv.SKIP_LINES_CONF_KEY, true); + badLineCount = context.getCounter("ImportTsv", "Bad Lines"); + hfileOutPath = conf.get(IndexImportTsv.BULK_OUTPUT_CONF_KEY); + indexedTable = conf.getBoolean(IndexMapReduceUtil.INDEX_IS_INDEXED_TABLE, false); + } + + /** + * Convert a line of TSV text into an HBase table row. + */ + @Override + public void map(LongWritable offset, Text value, Context context) throws IOException { + byte[] lineBytes = value.getBytes(); + + try { + IndexImportTsv.TsvParser.ParsedLine parsed = parser.parse(lineBytes, value.getLength()); + ImmutableBytesWritable rowKey = + new ImmutableBytesWritable(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength()); + // Retrieve timestamp if exists + ts = parsed.getTimestamp(ts); + Configuration conf = context.getConfiguration(); + Put put = new Put(rowKey.copyBytes()); + for (int i = 0; i < parsed.getColumnCount(); i++) { + if (i == parser.getRowKeyColumnIndex() || i == parser.getTimestampKeyColumnIndex()) { + continue; + } + KeyValue kv = + new KeyValue(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(), + parser.getFamily(i), 0, parser.getFamily(i).length, parser.getQualifier(i), 0, + parser.getQualifier(i).length, ts, KeyValue.Type.Put, lineBytes, + parsed.getColumnOffset(i), parsed.getColumnLength(i)); + put.add(kv); + } + + if (indexedTable && hfileOutPath != null) { + List indexPuts = null; + try { + // Genrate Index entry for the index put + indexPuts = IndexMapReduceUtil.getIndexPut(put, conf); + } catch (IOException e) { + if (skipBadLines) { + System.err.println("Bad line at offset: " + offset.get() + ":\n" + e.getMessage()); + incrementBadLineCount(1); + return; + } else { + throw e; + } + } + for (Put usrPut : indexPuts) { + context.write(new ImmutableBytesWritable(usrPut.getRow()), usrPut); + } + } + // Write user table put + context.write(rowKey, put); + + } catch (IndexImportTsv.TsvParser.BadTsvLineException badLine) { + if (skipBadLines) { + System.err.println("Bad line at offset: " + offset.get() + ":\n" + badLine.getMessage()); + incrementBadLineCount(1); + return; + } else { + throw new IOException(badLine); + } + } catch (IllegalArgumentException e) { + if (skipBadLines) { + System.err.println("Bad line at offset: " + offset.get() + ":\n" + e.getMessage()); + incrementBadLineCount(1); + return; + } else { + throw new IOException(e); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/TableIndexer.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/TableIndexer.java new file mode 100644 index 0000000..66896b1 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/mapreduce/TableIndexer.java @@ -0,0 +1,158 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat; +import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer; +import org.apache.hadoop.hbase.mapreduce.TableInputFormat; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.util.GenericOptionsParser; + +public class TableIndexer { + + private static final String TABLE_NAME_TO_INDEX = "tablename.to.index"; + final static String BULK_OUTPUT_CONF_KEY = "import.bulk.output"; + + private final static int DEFAULT_CACHING = 500; + private final static int DEFAULT_VERSIONS = 1; + + // This can be a comma seperated list + // We can pass like + // IDX1=>cf1:[q1->datatype& + // length],[q2],[q3];cf2:[q1->datatype&length],[q2->datatype&length],[q3->datatype& + // lenght]#IDX2=>cf1:q5,q5 + + private static Map> cfs = new HashMap>(); + + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); + if (otherArgs.length < 2) { + System.out.println("Caching and Versions not specified"); + System.exit(-1); + } + int caching = -1; + int versions = -1; + try { + caching = Integer.parseInt(otherArgs[0]); + } catch (NumberFormatException nfe) { + caching = DEFAULT_CACHING; + } + try { + versions = Integer.parseInt(otherArgs[1]); + } catch (NumberFormatException nfe) { + versions = DEFAULT_VERSIONS; + } + + String[] tableName = conf.getStrings(TABLE_NAME_TO_INDEX); + if (tableName == null) { + System.out + .println("Wrong usage. Usage is pass the table -Dindex.tablename='table1' " + + "-Dtable.columns.index='IDX1=>cf1:[q1->datatype& length],[q2]," + + "[q3];cf2:[q1->datatype&length],[q2->datatype&length],[q3->datatype& lenght]#IDX2=>cf1:q5,q5'"); + System.out.println("The format used here is: "); + System.out.println("IDX1 - Index name"); + System.out.println("cf1 - Columnfamilyname"); + System.out.println("q1 - qualifier"); + System.out.println("datatype - datatype (Int, String, Double, Float)"); + System.out.println("length - length of the value"); + System.out.println("The columnfamily should be seperated by ';'"); + System.out + .println("The qualifier and the datatype and its length should be enclosed in '[]'." + + " The qualifier details are specified using '->' following qualifer name and the details are seperated by '&'"); + System.out.println("If the qualifier details are not specified default values are used."); + System.out.println("# is used to seperate between two index details"); + System.out.println("Pass the scanner caching and maxversions as arguments."); + System.exit(-1); + } + String tableNameToIndex = tableName[0]; + IndexUtils.createIndexTable(tableNameToIndex, conf, cfs); + createMapReduceJob(tableNameToIndex, conf, caching, versions); + } + + private static void createMapReduceJob(String tableNameToIndex, Configuration conf, int caching, + int versions) throws IOException, InterruptedException, ClassNotFoundException { + // Set the details to TableInputFormat + Scan s = new Scan(); + s.setCaching(caching); + s.setMaxVersions(versions); + conf.set(TableInputFormat.INPUT_TABLE, tableNameToIndex); + + Set>> entrySet = cfs.entrySet(); + for (Entry> entry : entrySet) { + List quals = entry.getValue(); + addColumn(quals, Bytes.toBytes(entry.getKey()), s); + } + Job job = new Job(conf, "CreateIndex"); + String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); + + TableMapReduceUtil.initTableMapperJob(tableNameToIndex, // input table + s, // Scan instance to control CF and attribute selection + IndexCreationMapper.class, // mapper class + ImmutableBytesWritable.class, // mapper output key + Put.class, // mapper output value + job); + + TableMapReduceUtil.initTableReducerJob(IndexUtils.getIndexTableName(tableNameToIndex), // output + // table + null, // reducer class + job); + + if (hfileOutPath != null) { + HTable table = new HTable(conf, tableNameToIndex); + job.setReducerClass(KeyValueSortReducer.class); + Path outputDir = new Path(hfileOutPath); + FileOutputFormat.setOutputPath(job, outputDir); + HFileOutputFormat.configureIncrementalLoad(job, table); + } else { + job.setNumReduceTasks(0); + } + + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + com.google.common.base.Preconditions.class); + job.waitForCompletion(true); + assert job.isComplete() == true; + } + + private static void addColumn(List quals, byte[] cf, Scan s) { + for (String q : quals) { + s.addColumn(cf, Bytes.toBytes(q)); + } + } + +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/ByteArrayBuilder.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/ByteArrayBuilder.java new file mode 100644 index 0000000..bda6429 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/ByteArrayBuilder.java @@ -0,0 +1,67 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.util; + +// Custom ByteArrayBuilder class to avoid overhead. +public class ByteArrayBuilder { + private short pos; + private byte[] store; + + public ByteArrayBuilder(int size) { + store = new byte[size]; + pos = 0; + } + + public void put(byte[] src) { + System.arraycopy(src, 0, store, pos, src.length); + pos += src.length; + } + + public void put(byte[] src, int offset, int length) { + System.arraycopy(src, offset, store, pos, length); + pos += length; + } + + public static ByteArrayBuilder allocate(int size) { + return new ByteArrayBuilder(size); + } + + public short position() { + return pos; + } + + public void position(int newPosition) { + pos = (short) newPosition; + } + + // Be careful calling this method. This method exposes the underlying byte[] + // Any changes to this returned object will result in changes in the builder. + public byte[] array() { + return store; + } + + // This method creates a new byte[] and copy the bytes from the underlying byte[] + // Any changes to the returned object will not affect the builder. + public byte[] array(int offset, int length) { + byte[] subArray = new byte[length]; + System.arraycopy(store, offset, subArray, 0, length); + return subArray; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/IndexUtils.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/IndexUtils.java new file mode 100644 index 0000000..e169291 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/IndexUtils.java @@ -0,0 +1,650 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.util; + +import java.io.EOFException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.index.ColumnQualifier; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.ValuePartition; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; + +public class IndexUtils { + + private static final Log LOG = LogFactory.getLog(IndexUtils.class); + + private static final String DOT_TABLEINFO = ".tableinfo"; + + public static final String TABLE_INPUT_COLS = "table.columns.index"; + + /** + * Utility method to get the name of the index table when given the name of the actual table. + * @param tableName + * @return index table name + */ + public static String getIndexTableName(String tableName) { + // TODO The suffix for the index table is fixed now. Do we allow to make this configurable? + // We can handle things in byte[] way? + return tableName + Constants.INDEX_TABLE_SUFFIX; + } + + /** + * Utility method to get the name of the index table when given the name of the actual table. + * @param tableName + * @return index table name + */ + public static String getIndexTableName(byte[] tableName) { + return getIndexTableName(Bytes.toString(tableName)); + } + + /** + * Tells whether the passed table is a secondary index table or a normal table. + * @param tableName + * @return + */ + public static boolean isIndexTable(String tableName) { + return tableName.endsWith(Constants.INDEX_TABLE_SUFFIX); + } + + /** + * Tells whether the passed table is a secondary index table or a normal table. + * @param tableName + * @return + */ + public static boolean isIndexTable(byte[] tableName) { + return isIndexTable(Bytes.toString(tableName)); + } + + /** + * Checks whether the passed table is a catalog table or not + * @param tableName + * @return true when the passed table is a catalog table. + */ + public static boolean isCatalogTable(byte[] tableName) { + if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME) + || Bytes.equals(tableName, HConstants.META_TABLE_NAME)) { + return true; + } + return false; + } + + /** + * Returns the max length allowed for the index name. + * @return + */ + public static int getMaxIndexNameLength() { + // TODO we need to allow customers to configure this value. + return Constants.DEF_MAX_INDEX_NAME_LENGTH; + } + + /** + * Returns the main table name. + * @param index table name + * @return + */ + public static String extractActualTableName(String indexTableName) { + int endIndex = indexTableName.length() - Constants.INDEX_TABLE_SUFFIX.length(); + return indexTableName.substring(0, endIndex); + } + + public static byte[] changeValueAccToDataType(byte[] value, ValueType valueType) { + byte[] valueArr = new byte[value.length]; + System.arraycopy(value, 0, valueArr, 0, value.length); + + if (valueArr.length == 0) return valueArr; + switch (valueType) { + case String: + case Char: + break; + case Float: + float f = Bytes.toFloat(valueArr); + if (f > 0) { + valueArr[0] ^= (1 << 7); + } else { + valueArr[0] ^= 0xff; + valueArr[1] ^= 0xff; + valueArr[2] ^= 0xff; + valueArr[3] ^= 0xff; + } + break; + case Double: + double d = Bytes.toDouble(valueArr); + if (d > 0) { + valueArr[0] ^= (1 << 7); + } else { + for (int i = 0; i < 8; i++) { + valueArr[i] ^= 0xff; + } + } + break; + case Int: + case Long: + case Short: + case Byte: + valueArr[0] ^= (1 << 7); + break; + } + return valueArr; + } + + // TODO check this... Is this ok with all cases? + // No.. for -ve issues... Will see later.. + public static byte[] incrementValue(byte[] value, boolean copy) { + byte[] newValue = new byte[value.length]; + if (copy) { + System.arraycopy(value, 0, newValue, 0, newValue.length); + } else { + newValue = value; + } + for (int i = newValue.length - 1; i >= 0; i--) { + byte b = newValue[i]; + b = (byte) (b + 1); + if (b == 0) { + newValue[i] = 0; + } else { + newValue[i] = b; + break; + } + } + return newValue; + } + + public static String getActualTableNameFromIndexTableName(String indexTableName) { + String split[] = indexTableName.split(Constants.INDEX_TABLE_SUFFIX); + return split[0]; + } + + public static IndexedHTableDescriptor getIndexedHTableDescriptor(byte[] tableName, + Configuration conf) throws IOException { + IndexedHTableDescriptor tableDescriptor = null; + FSDataInputStream fsDataInputStream = null; + try { + FileSystem fs = FSUtils.getCurrentFileSystem(conf); + Path path = getTableInfoFilePath(conf, tableName, fs); + if (null != path) { + fsDataInputStream = fs.open(path); + tableDescriptor = new IndexedHTableDescriptor(); + tableDescriptor.readFields(fsDataInputStream); + } + } catch (IOException e) { + if (e instanceof EOFException) { + if (LOG.isDebugEnabled()) { + LOG.debug("Error reading data from the table descriptor . Got " + e + " exception"); + } + tableDescriptor = null; + } else { + throw e; + } + } finally { + if (fsDataInputStream != null) { + try { + fsDataInputStream.close(); + } catch (IOException e) { + LOG.error("IOException closing the input stream "); + } + } + } + return tableDescriptor; + } + + private static Path getTableInfoFilePath(Configuration conf, byte[] tableName, FileSystem fs) + throws IOException { + Path path = FSUtils.getTablePath(FSUtils.getRootDir(conf), tableName); + FileStatus[] status = FSUtils.listStatus(fs, path, new PathFilter() { + @Override + public boolean accept(Path p) { + // Accept any file that starts with TABLEINFO_NAME + return p.getName().startsWith(DOT_TABLEINFO); + } + }); + if (status == null || status.length < 1) { + return null; + } + Arrays.sort(status, new FileStatusFileNameComparator()); + if (status.length > 1) { + // Clean away old versions of .tableinfo + for (int i = 1; i < status.length; i++) { + Path p = status[i].getPath(); + // Clean up old versions + if (!fs.delete(p, false)) { + LOG.warn("Failed cleanup of " + p); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Cleaned up old tableinfo file " + p); + } + } + } + } + if (null != status[0]) { + path = status[0].getPath(); + } + return path; + } + + /** + * Compare {@link FileStatus} instances by {@link Path#getName()}. Returns in reverse order. + */ + static class FileStatusFileNameComparator implements Comparator { + @Override + public int compare(FileStatus left, FileStatus right) { + return -left.compareTo(right); + } + } + + public static Put prepareIndexPut(Put userPut, IndexSpecification index, HRegion indexRegion) + throws IOException { + byte[] indexRegionStartKey = indexRegion.getStartKey(); + return prepareIndexPut(userPut, index, indexRegionStartKey); + } + + public static Delete prepareIndexDelete(Delete userDelete, IndexSpecification index, + byte[] indexRegionStartKey) throws IOException { + ByteArrayBuilder indexRow = + IndexUtils.getIndexRowKeyHeader(index, indexRegionStartKey, userDelete.getRow()); + boolean update = false; + for (ColumnQualifier cq : index.getIndexColumns()) { + KeyValue kvFound = null; + for (Entry> entry : userDelete.getFamilyMap().entrySet()) { + for (KeyValue kv : entry.getValue()) { + if (Bytes.equals(cq.getColumnFamily(), kv.getFamily()) + && Bytes.equals(cq.getQualifier(), kv.getQualifier())) { + kvFound = kv; + update = true; + break; + } + } + } + if (kvFound == null) { + indexRow.position(indexRow.position() + cq.getMaxValueLength()); + } else { + IndexUtils.updateRowKeyForKV(cq, kvFound, indexRow); + } + } + if (update) { + // Append the actual row key at the end of the index row key. + indexRow.put(userDelete.getRow()); + Delete idxDelete = new Delete(indexRow.array()); + idxDelete.deleteColumn(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, + userDelete.getTimeStamp()); + + idxDelete.setWriteToWAL(false); + return idxDelete; + } + return null; + } + + // Default access specifier for the UT + public static Put prepareIndexPut(Put userPut, IndexSpecification index, + byte[] indexRegionStartKey) throws IOException { + long tsForIndexTabPut = 0; + + boolean bypass = true; + for (ColumnQualifier c : index.getIndexColumns()) { + List values = userPut.get(c.getColumnFamily(), c.getQualifier()); + if (null != values && values.size() > 0) { + bypass = false; + break; + } + } + if (bypass) { + // When this Put having no values for all the column in this index just skip this Put + // from adding corresponding entry in the index table. + return null; + } + byte[] primaryRowKey = userPut.getRow(); + ByteArrayBuilder indexRowKey = getIndexRowKeyHeader(index, indexRegionStartKey, primaryRowKey); + + // STEP 3 : Adding the column value + padding for each of the columns in + // the index. + for (ColumnQualifier indexCQ : index.getIndexColumns()) { + List values = userPut.get(indexCQ.getColumnFamily(), indexCQ.getQualifier()); + if (values == null || values.isEmpty()) { + // There is no value provided for the column. Going with the padding + // All the bytes in the byte[] 'indexRowKey' will be 0s already. + // No need to put a 0 padding bytes. Just need to advance the position by col max value + // length. + indexRowKey.position(indexRowKey.position() + indexCQ.getMaxValueLength()); + } else { + // A put can contains diff version values for the same column. + // We can consider the latest value only for the indexing. This needs to be documented. + // TODO + KeyValue kv = selectKVForIndexing(values); + updateRowKeyForKV(indexCQ, kv, indexRowKey); + if (tsForIndexTabPut < kv.getTimestamp()) { + tsForIndexTabPut = kv.getTimestamp(); + } + } + } + // Remember the offset of rowkey and store it as value + short rowKeyOffset = indexRowKey.position(); + + // STEP 4 : Adding the user table rowkey. + indexRowKey.put(primaryRowKey); + + // Creating the value to be put into the index column + // Last portion of index row key = [region start key length (2 bytes), offset of primary rowkey + // in index rowkey (2 bytes)] + ByteArrayBuilder indexColVal = ByteArrayBuilder.allocate(4); + indexColVal.put(Bytes.toBytes((short) indexRegionStartKey.length)); + indexColVal.put(Bytes.toBytes(rowKeyOffset)); + Put idxPut = new Put(indexRowKey.array()); + idxPut.add(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, tsForIndexTabPut, + indexColVal.array()); + idxPut.setWriteToWAL(false); + return idxPut; + } + + private static KeyValue selectKVForIndexing(List values) { + KeyValue kv = null; + long ts = HConstants.OLDEST_TIMESTAMP; + for (KeyValue value : values) { + // When the TS is same, then we need to consider the last KV + // appearing in the KVList + // as this will be added to the memstore with highest memstore TS. + if (value.getTimestamp() >= ts) { + kv = value; + ts = value.getTimestamp(); + } + } + return kv; + } + + public static ByteArrayBuilder getIndexRowKeyHeader(IndexSpecification index, + byte[] indexRegionStartKey, byte[] primaryRowKey) { + /* + * Format for the rowkey for index table [Startkey for the index region] + [one 0 byte] + [Index + * name] + [Padding for the max index name] + [[index col value]+[padding for the max col value] + * for each of the index col] + [user table row key] To know the reason for adding empty byte + * array refert to HDP-1666 + */ + byte[] indexName = Bytes.toBytes(index.getName()); + int totalValueLength = index.getTotalValueLength(); + int maxIndexNameLength = IndexUtils.getMaxIndexNameLength(); + int rowLength = + indexRegionStartKey.length + maxIndexNameLength + totalValueLength + primaryRowKey.length + + 1; + ByteArrayBuilder row = ByteArrayBuilder.allocate(rowLength); + + // STEP 1 : Adding the startkey for the index region and single empty Byte. + row.put(indexRegionStartKey); + // one byte [0] to be added after the index region startkey. This is for the case of + // entries added to the 1st region.Here the startkey of the region will be empty byte[] + // So the 1st byte(s) which comes will be the index name and it might not fit into the + // 1st region [As per the end key of that region] + // Well all the bytes in the byte[] 'row' will be 0s already. No need to put a 0 byte + // Just need to advance the position by 1 + row.position(row.position() + 1); + + // STEP 2 : Adding the index name and the padding needed + row.put(indexName); + int padLength = maxIndexNameLength - indexName.length; + // Well all the bytes in the byte[] 'row' will be 0s already. No need to put a 0 padding bytes + // Just need to advance the position by padLength + row.position(row.position() + padLength); + return row; + } + + public static void updateRowKeyForKV(ColumnQualifier indexCQ, KeyValue kv, + ByteArrayBuilder indexRowKey) throws IOException { + byte[] value = getValueFromKV(kv, indexCQ); + int valuePadLength = indexCQ.getMaxValueLength() - value.length; + if (valuePadLength < 0) { + String errMsg = + "The value length for the column " + indexCQ.getColumnFamilyString() + ":" + + indexCQ.getQualifierString() + " is greater than the cofigured max value length : " + + indexCQ.getMaxValueLength(); + LOG.warn(errMsg); + throw new IOException(errMsg); + } + indexRowKey.put(value); + indexRowKey.position(indexRowKey.position() + valuePadLength); + } + + private static byte[] getValueFromKV(KeyValue kv, ColumnQualifier indexCQ) { + ValuePartition vp = indexCQ.getValuePartition(); + byte value[] = null; + if (vp != null) { + value = vp.getPartOfValue(kv.getValue()); + if (value != null) { + value = IndexUtils.changeValueAccToDataType(value, indexCQ.getType()); + } + } else { + LOG.trace("No offset or separator is mentioned. So just returning the value fetched from kv"); + value = kv.getValue(); + value = IndexUtils.changeValueAccToDataType(value, indexCQ.getType()); + } + return value; + } + + public static byte[] getRowKeyFromKV(KeyValue kv) { + byte[] row = kv.getRow(); + // Row key of the index table entry = region startkey + index name + column value(s) + // + actual table rowkey. + // Every row in the index table will have exactly one KV in that. The value will be + // 4 bytes. First 2 bytes specify length of the region start key bytes part in the + // rowkey. Last 2 bytes specify the offset to the actual table rowkey part within the + // index table rowkey. + byte[] value = kv.getValue(); + short actualRowKeyOffset = Bytes.toShort(value, 2); + byte[] actualTableRowKey = new byte[row.length - actualRowKeyOffset]; + System.arraycopy(row, actualRowKeyOffset, actualTableRowKey, 0, actualTableRowKey.length); + return actualTableRowKey; + } + + public static void createIndexTable(String userTable, Configuration conf, + Map> indexColumnFamily) throws IOException, InterruptedException, + ClassNotFoundException { + HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); + + try { + HTableDescriptor tableDescriptor = hbaseAdmin.getTableDescriptor(Bytes.toBytes(userTable)); + + Collection existingColumnFamilies = tableDescriptor.getFamilies(); + + String input = conf.get(TABLE_INPUT_COLS); + + IndexedHTableDescriptor ihtd = + parse(userTable, existingColumnFamilies, input, indexColumnFamily); + + // disable the table + hbaseAdmin.disableTable(userTable); + + // This will create the index table. Also modifies the existing table htable descriptor. + hbaseAdmin.modifyTable(Bytes.toBytes(userTable), ihtd); + boolean found = false; + while (!found) { + try { + hbaseAdmin.getTableDescriptor(Bytes.toBytes(IndexUtils.getIndexTableName(userTable))); + } catch (TableNotFoundException tnfe) { + Thread.sleep(1000); + continue; + } + found = true; + } + + hbaseAdmin.enableTable(Bytes.toBytes(userTable)); + } finally { + if (hbaseAdmin != null) { + hbaseAdmin.close(); + } + } + } + + // This can be a comma seperated list + // We can pass like + // IDX1=>cf1:[q1->datatype& + // length],[q2],[q3];cf2:[q1->datatype&length],[q2->datatype&length],[q3->datatype& + // lenght]#IDX2=>cf1:q5,q5 + private static IndexedHTableDescriptor parse(String tableNameToIndex, + Collection existingColumnFamilies, String input, + Map> cfs) { + IndexedHTableDescriptor indexHTableDesc = new IndexedHTableDescriptor(tableNameToIndex); + List colFamilyList = new ArrayList(); + for (HColumnDescriptor hColumnDescriptor : existingColumnFamilies) { + indexHTableDesc.addFamily(hColumnDescriptor); + colFamilyList.add(hColumnDescriptor.getNameAsString()); + + } + if (input != null) { + String[] indexSplits = input.split("#"); + for (String index : indexSplits) { + String[] indexName = index.split("=>"); + if (indexName.length < 2) { + System.out.println("Invalid entry."); + System.exit(-1); + } + IndexSpecification iSpec = new IndexSpecification(indexName[0]); + + String[] cfSplits = indexName[1].split(";"); + if (cfSplits.length < 1) { + System.exit(-1); + } else { + for (String cf : cfSplits) { + String[] qualSplits = cf.split(":"); + if (qualSplits.length < 2) { + System.out.println("The qualifiers are not given"); + System.exit(-1); + } + if (!colFamilyList.contains(qualSplits[0])) { + System.out.println("Valid CF not found"); + System.exit(-1); + } + String[] qualDetails = qualSplits[1].split(","); + for (String details : qualDetails) { + String substring = details.substring(1, details.lastIndexOf("]")); + if (substring != null) { + String[] splitQual = substring.split("->"); + if (splitQual.length < 2) { + System.out.println("Default value length and data type will be take"); + iSpec.addIndexColumn(new HColumnDescriptor(qualSplits[0]), splitQual[0], + ValueType.String, Constants.DEF_MAX_INDEX_NAME_LENGTH); + } else { + String[] valueType = splitQual[1].split("&"); + iSpec.addIndexColumn(new HColumnDescriptor(qualSplits[0]), splitQual[0], + ValueType.valueOf(valueType[0]), Integer.parseInt(valueType[1])); + } + if (cfs != null) { + addToMap(cfs, qualSplits, splitQual); + } + } + } + } + } + indexHTableDesc.addIndex(iSpec); + } + } + return indexHTableDesc; + } + + private static void addToMap(Map> cfs, String[] qualSplits, + String[] splitQual) { + if (cfs.get(qualSplits[0]) == null) { + List qual = new ArrayList(); + qual.add(splitQual[0]); + cfs.put(qualSplits[0], qual); + } else { + List list = cfs.get(qualSplits[0]); + list.add(splitQual[0]); + } + } + + /** + * Reads the indexed table description directly from the file. + * @param tableName Table name + * @param conf HBase Configuration + * @return HTableDescriptor + * @throws IOException + */ + public static HTableDescriptor readIndexedHTableDescriptor(String tableName, Configuration conf) + throws IOException { + IndexedHTableDescriptor indexedHTabDescriptor = new IndexedHTableDescriptor(); + FSDataInputStream fsDataInputStream = null; + try { + FileSystem fs = FSUtils.getCurrentFileSystem(conf); + Path rootPath = FSUtils.getRootDir(conf); + Path path = FSUtils.getTablePath(rootPath, tableName); + FileStatus status = IndexMasterObserver.getTableInfoPath(fs, path); + if (null == status) { + throw new IOException(tableName + " status is null"); + } + fsDataInputStream = fs.open(status.getPath()); + indexedHTabDescriptor.readFields(fsDataInputStream); + return indexedHTabDescriptor; + } catch (EOFException e) { + return new HTableDescriptor(indexedHTabDescriptor); + } catch (IOException i) { + throw i; + } finally { + if (fsDataInputStream != null) { + fsDataInputStream.close(); + } + } + } + + public static byte[][] getSplitKeys(HRegionInfo[] regions) { + byte[][] splitKeys = null; + if (null != regions && regions.length > 1) { + // for the 1st region always the start key will be empty. We no need to + // pass this as a start key item for the index table because this will + // be added by HBase any way. So if we pass empty, HBase will create one + // extra region with start and end key as empty byte[]. + splitKeys = new byte[regions.length - 1][]; + int i = 0; + for (HRegionInfo region : regions) { + byte[] startKey = region.getStartKey(); + if (startKey.length > 0) { + splitKeys[i++] = startKey; + } + } + } + return splitKeys; + } +} diff --git a/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/SecondaryIndexColocator.java b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/SecondaryIndexColocator.java new file mode 100644 index 0000000..3489562 --- /dev/null +++ b/secondaryindex/src/main/java/org/apache/hadoop/hbase/index/util/SecondaryIndexColocator.java @@ -0,0 +1,515 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.UnknownRegionException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable; +import org.apache.hadoop.hbase.client.MetaScanner; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.HBaseFsckRepair; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState; +import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +public class SecondaryIndexColocator { + + public static boolean testingEnabled = false; + + public static final Log LOG = LogFactory.getLog(SecondaryIndexColocator.class); + + public Configuration conf = null; + + private Map> tableMap = new HashMap>(); + + private List regionsToMove = new ArrayList(); + + private Map> rsToRegionMap = + new HashMap>(); + + private Map disabledandDisablingTables = new TreeMap( + Bytes.BYTES_COMPARATOR); + + private Map enabledOrEnablingTables = new TreeMap( + Bytes.BYTES_COMPARATOR); + + // private Set staleMetaEntries = new HashSet(); + + private List> tablesToBeSetInZK = + new ArrayList>(); + + // List rit = new ArrayList(); + + private HBaseAdmin admin; + private ClusterStatus status; + private HConnection connection; + + public SecondaryIndexColocator(Configuration conf) { + this.conf = conf; + } + + public void setUp() throws IOException, ZooKeeperConnectionException { + admin = new HBaseAdmin(conf); + status = admin.getClusterStatus(); + connection = admin.getConnection(); + } + + public static void main(String args[]) throws Exception { + Configuration config = HBaseConfiguration.create(); + SecondaryIndexColocator secHbck = new SecondaryIndexColocator(config); + secHbck.setUp(); + secHbck.admin.setBalancerRunning(false, true); + boolean inconsistent = secHbck.checkForCoLocationInconsistency(); + if (inconsistent) { + secHbck.fixCoLocationInconsistency(); + } + secHbck.admin.setBalancerRunning(true, true); + } + + public boolean checkForCoLocationInconsistency() throws IOException, KeeperException, + InterruptedException { + getMetaInfo(); + // Do we need to complete the partial disable table + loadDisabledTables(); + // handleRIT(); + checkMetaInfoCosistency(); + setTablesInZK(); + // in the former steps there may have been movement of regions. So need to update + // in memory map of tables. + getMetaInfo(); + checkCoLocationAndGetRegionsToBeMoved(); + if (regionsToMove == null || regionsToMove.isEmpty()) { + return false; + } + return true; + } + + /* + * private void handleRIT() { if(rit != null && !rit.isEmpty()){ for(String s : rit){ + * RegionTransitionData data = ZKAssign.getDataNoWatch(zkw, pathOrRegionName, stat) } } } + */ + + private void checkMetaInfoCosistency() throws IOException, KeeperException, InterruptedException { + if (status == null) { + throw new IOException("Cluster status is not available."); + } + Collection regionServers = status.getServers(); + for (ServerName serverName : regionServers) { + HRegionInterface server = + connection.getHRegionConnection(serverName.getHostname(), serverName.getPort()); + Set onlineRegions = new HashSet(); + List regions = server.getOnlineRegions(); + if (regions == null) continue; + onlineRegions.addAll(regions); + if (rsToRegionMap == null) { + rsToRegionMap = new HashMap>(); + } + rsToRegionMap.put(serverName, onlineRegions); + } + if (tableMap != null && !tableMap.isEmpty()) { + for (Map.Entry> e : tableMap.entrySet()) { + if (isDisabledOrDisablingTable(Bytes.toBytes(e.getKey()))) { + // It should be disabled...But we can check this + if (disabledandDisablingTables.get(Bytes.toBytes(e.getKey())) == TableState.DISABLED) { + continue; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Table " + e.getKey() + + " is in DISABLING state. Trying to close all the regions for this table."); + } + // If the table is in DISABLING state , then there might be some regions which are + // still online. Close the regions and set the table as DISABLED. + for (MetaInfo metaInfo : e.getValue()) { + List sn = new ArrayList(); + for (Map.Entry> entry : rsToRegionMap.entrySet()) { + if (entry.getValue().contains(metaInfo.getRegionInfo())) { + sn.add(entry.getKey()); + } + } + if (sn.isEmpty()) { + // region is not assigned anywhere ,so continue. + continue; + } else { + HBaseFsckRepair.fixMultiAssignment(this.admin, metaInfo.getRegionInfo(), sn); + } + } + Pair p = new Pair(); + p.setFirst(e.getKey()); + p.setSecond(TableState.DISABLED); + tablesToBeSetInZK.add(p); + } + } else { + // first we are checking here for the tables to be enabled which + // we left in disabled stage in the previous step. + if (!disabledandDisablingTables.containsKey(Bytes.toBytes(e.getKey())) + && !enabledOrEnablingTables.containsKey(Bytes.toBytes(e.getKey()))) { + // if reached here then this table, which is disabled + // and still not present in our in-memory map of disabledTables , + // should + // be enabled. + this.admin.enableTable(e.getKey()); + this.enabledOrEnablingTables.put(Bytes.toBytes(e.getKey()), TableState.ENABLED); + continue; + } + boolean movedRegions = false; + for (MetaInfo metaInfo : e.getValue()) { + List sn = new ArrayList(); + for (Map.Entry> entry : rsToRegionMap.entrySet()) { + if (entry.getValue().contains(metaInfo.getRegionInfo())) { + sn.add(entry.getKey()); + } + } + if (sn.size() == 1 && sn.get(0).equals(metaInfo.getServerName())) { + // this means region is deployed on correct rs according to META. + if (LOG.isDebugEnabled()) { + LOG.debug("Info in META for region " + + metaInfo.getRegionInfo().getRegionNameAsString() + " is correct."); + } + continue; + } + // if it reaches here , it means that the region is deployed and + // in some other rs. Need to find it and call unassign. + if (sn.isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Region " + metaInfo.getRegionInfo().getRegionNameAsString() + + " not deployed on any rs.Trying to assign"); + } + HBaseFsckRepair.fixUnassigned(this.admin, metaInfo.getRegionInfo()); + HBaseFsckRepair.waitUntilAssigned(this.admin, metaInfo.getRegionInfo()); + movedRegions = true; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Region " + metaInfo.getRegionInfo().getRegionNameAsString() + + " is not deployed on the rs as mentioned in META. Re-assigning."); + } + HBaseFsckRepair.fixMultiAssignment(this.admin, metaInfo.getRegionInfo(), sn); + HBaseFsckRepair.waitUntilAssigned(this.admin, metaInfo.getRegionInfo()); + movedRegions = true; + } + } + if (movedRegions) { + Pair p = new Pair(); + p.setFirst(e.getKey()); + p.setSecond(TableState.ENABLED); + tablesToBeSetInZK.add(p); + } + } + } + } + } + + private void setTablesInZK() throws IOException { + if (tablesToBeSetInZK != null && !tablesToBeSetInZK.isEmpty()) { + for (Pair p : tablesToBeSetInZK) { + setStateInZK(p.getFirst(), p.getSecond()); + } + } + } + + private void setStateInZK(String tableName, TableState state) throws IOException { + if (state == TableState.ENABLED) { + admin.setEnableTable(tableName); + } + if (state == TableState.DISABLED) { + admin.setDisableTable(tableName); + } + } + + private boolean isDisabledOrDisablingTable(byte[] tableName) { + if (disabledandDisablingTables != null && !disabledandDisablingTables.isEmpty()) { + if (disabledandDisablingTables.containsKey(tableName)) return true; + } + return false; + } + + public void fixCoLocationInconsistency() { + if (regionsToMove != null && !regionsToMove.isEmpty()) { + Iterator itr = regionsToMove.iterator(); + while (itr.hasNext()) { + HRegionInfo hri = itr.next(); + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Moving region " + hri.getRegionNameAsString() + " to server "); + } + admin.move(hri.getEncodedNameAsBytes(), null); + itr.remove(); + } catch (UnknownRegionException e) { + LOG.error("Unnkown region exception.", e); + } catch (MasterNotRunningException e) { + LOG.error("Master not running.", e); + } catch (ZooKeeperConnectionException e) { + LOG.error("Zookeeper connection exception.", e); + } + } + } + } + + private void checkCoLocationAndGetRegionsToBeMoved() { + if (tableMap != null && !tableMap.isEmpty()) { + Iterator>> itr = tableMap.entrySet().iterator(); + while (itr.hasNext()) { + Map.Entry> e = itr.next(); + if (!IndexUtils.isIndexTable(e.getKey()) + && !IndexUtils.isCatalogTable(Bytes.toBytes(e.getKey()))) { + if (isDisabledOrDisablingTable(Bytes.toBytes(e.getKey()))) continue; + String indexTableName = IndexUtils.getIndexTableName(e.getKey()); + List idxRegionList = tableMap.get(indexTableName); + if (idxRegionList == null || idxRegionList.isEmpty()) { + itr.remove(); + continue; + } else { + getRegionsToMove(e.getValue(), idxRegionList); + itr.remove(); + } + } + } + } + } + + private void getRegionsToMove(List userRegions, List idxRegionList) { + Iterator userRegionItr = userRegions.iterator(); + while (userRegionItr.hasNext()) { + MetaInfo userRegionMetaInfo = userRegionItr.next(); + for (MetaInfo indexRegionMetaInfo : idxRegionList) { + if (Bytes.equals(userRegionMetaInfo.getRegionInfo().getStartKey(), indexRegionMetaInfo + .getRegionInfo().getStartKey())) { + if (!userRegionMetaInfo.getServerName().equals(indexRegionMetaInfo.getServerName())) { + if (regionsToMove == null) { + regionsToMove = new ArrayList(); + } + regionsToMove.add(userRegionMetaInfo.getRegionInfo()); + if (LOG.isDebugEnabled()) { + LOG.debug("Adding region " + + userRegionMetaInfo.getRegionInfo().getRegionNameAsString() + + " to regions to be moved list."); + } + } + break; + } + } + } + } + + private void getMetaInfo() throws IOException { + + MetaScannerVisitor visitor = new MetaScannerVisitorBase() { + + // comparator to sort KeyValues with latest timestamp + final Comparator comp = new Comparator() { + public int compare(KeyValue k1, KeyValue k2) { + return (int) (k1.getTimestamp() - k2.getTimestamp()); + } + }; + + public boolean processRow(Result result) throws IOException { + try { + + long ts = Collections.max(result.list(), comp).getTimestamp(); + // record the latest modification of this META record + Pair pair = MetaReader.parseCatalogResult(result); + if (pair != null) { + String tableName = pair.getFirst().getTableNameAsString(); + if (tableMap == null) { + tableMap = new HashMap>(); + } + List regionsOfTable = tableMap.get(tableName); + if (regionsOfTable == null) { + regionsOfTable = new ArrayList(); + tableMap.put(tableName, regionsOfTable); + } + Iterator itr = regionsOfTable.iterator(); + while (itr.hasNext()) { + MetaInfo m = itr.next(); + if (m.getRegionInfo().equals(pair.getFirst())) { + itr.remove(); + break; + } + } + MetaInfo m = new MetaInfo(pair.getFirst(), pair.getSecond(), ts); + regionsOfTable.add(m); + } + return true; + } catch (RuntimeException e) { + LOG.error("Result=" + result); + throw e; + } + } + }; + // Scan -ROOT- to pick up META regions + MetaScanner.metaScan(conf, visitor, null, null, Integer.MAX_VALUE, HConstants.ROOT_TABLE_NAME); + + // Scan .META. to pick up user regions + MetaScanner.metaScan(conf, visitor); + + } + + /** + * Stores the regioninfo entries scanned from META + */ + static class MetaInfo { + private HRegionInfo hri; + private ServerName regionServer; + private long timeStamp; + + public MetaInfo(HRegionInfo hri, ServerName regionServer, long modTime) { + this.hri = hri; + this.regionServer = regionServer; + this.timeStamp = modTime; + } + + public HRegionInfo getRegionInfo() { + return this.hri; + } + + public ServerName getServerName() { + return this.regionServer; + } + + public long getTimeStamp() { + return this.timeStamp; + } + } + + public void setAdmin(HBaseAdmin admin, HConnection conn, ClusterStatus status) { + if (testingEnabled) { + this.admin = admin; + this.connection = conn; + this.status = status; + } + } + + private void loadDisabledTables() throws ZooKeeperConnectionException, IOException, + KeeperException { + HConnectionManager.execute(new HConnectable(conf) { + @Override + public Void connect(HConnection connection) throws IOException { + ZooKeeperWatcher zkw = connection.getZooKeeperWatcher(); + try { + for (Entry> e : ZKTableReadOnly.getDisabledOrDisablingTables(zkw) + .entrySet()) { + for (String tableName : e.getValue()) { + disabledandDisablingTables.put(Bytes.toBytes(tableName), e.getKey()); + } + } + for (Entry> e : ZKTableReadOnly.getEnabledOrEnablingTables(zkw) + .entrySet()) { + for (String tableName : e.getValue()) { + enabledOrEnablingTables.put(Bytes.toBytes(tableName), e.getKey()); + } + } + // rit = ZKUtil.listChildrenNoWatch(zkw, zkw.assignmentZNode); + } catch (KeeperException ke) { + throw new IOException(ke); + } + return null; + } + }); + checkDisabledAndEnabledTables(); + } + + private void checkDisabledAndEnabledTables() throws IOException, KeeperException { + if (disabledandDisablingTables != null && !disabledandDisablingTables.isEmpty()) { + Map disabledHere = + new TreeMap(Bytes.BYTES_COMPARATOR); + Iterator> itr = disabledandDisablingTables.entrySet().iterator(); + while (itr.hasNext()) { + Entry tableEntry = itr.next(); + if (!IndexUtils.isIndexTable(tableEntry.getKey())) { + byte[] indexTableName = Bytes.toBytes(IndexUtils.getIndexTableName(tableEntry.getKey())); + if (null == tableMap.get(Bytes.toString(indexTableName))) { + continue; + } + boolean present = disabledandDisablingTables.containsKey(indexTableName); + if (!present && (enabledOrEnablingTables.get(indexTableName) == TableState.ENABLED)) { + // TODO How to handle ENABLING state(if it could happen). If try to disable ENABLING + // table + // it throws. + if (LOG.isDebugEnabled()) { + LOG.debug("Table " + Bytes.toString(tableEntry.getKey()) + + " is disabled but corresponding index table is " + "enabled. So disabling " + + Bytes.toString(indexTableName)); + } + this.admin.disableTable(indexTableName); + disabledHere.put(indexTableName, TableState.DISABLED); + } + } else { + if (tableEntry.getValue() != TableState.DISABLED) { + continue; + } + byte[] userTableName = + Bytes.toBytes(IndexUtils.getActualTableNameFromIndexTableName(Bytes + .toString(tableEntry.getKey()))); + if (!disabledandDisablingTables.containsKey(userTableName)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Index Table " + Bytes.toString(tableEntry.getKey()) + + " is disabled but corresponding user table is enabled. So Enabling " + + Bytes.toString(tableEntry.getKey())); + } + // Here we are not enabling the table. We will do it in the next step + // checkMetaInfoCosistency(). + // Because if we do here, META will be updated and our in-memory map will have old + // entries. + // So it will surely cause unnecessary unassignments and assignments in the next step. + // In the next + // step anyway we are moving regions. So no problem doing it there. + // this.admin.enableTable(tableName); + itr.remove(); + } + } + } + disabledandDisablingTables.putAll(disabledHere); + } + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestColumnQualifier.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestColumnQualifier.java new file mode 100644 index 0000000..43eb8ee --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestColumnQualifier.java @@ -0,0 +1,103 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; + +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test setting values in ColumnQuafilier. + */ +@Category(MediumTests.class) +public class TestColumnQualifier { + + @Test + public void testColumnQualifier() throws Exception { + ColumnQualifier cq = new ColumnQualifier("cf", "cq"); + assertEquals("Column family not match with acutal value.", "cf", cq.getColumnFamilyString()); + assertEquals("Column qualifier not match with actual value.", "cq", cq.getQualifierString()); + assertEquals("Column family bytes not match with actual value.", 0, + Bytes.compareTo(Bytes.toBytes("cf"), cq.getColumnFamily())); + assertEquals("Column qualifier bytes not match with actual value.", 0, + Bytes.compareTo(Bytes.toBytes("cq"), cq.getQualifier())); + } + + public void testColumQualifierEquals() { + ColumnQualifier cq = new ColumnQualifier(Bytes.toBytes("cf"), Bytes.toBytes("cq")); + assertTrue("ColumnQualifier state mismatch.", + cq.equals(new ColumnQualifier(Bytes.toBytes("cf"), Bytes.toBytes("cq")))); + assertTrue("ColumnQualifier state mismatch.", cq.equals(new ColumnQualifier("cf", "cq"))); + } + + @Test + public void testColumnQualifierSerialization() throws Exception { + ByteArrayOutputStream bos = null; + DataOutputStream dos = null; + ByteArrayInputStream bis = null; + DataInputStream dis = null; + try { + bos = new ByteArrayOutputStream(); + dos = new DataOutputStream(bos); + ColumnQualifier cq = + new ColumnQualifier("cf", "cq", ValueType.String, 10, new SeparatorPartition("--", 10)); + cq.write(dos); + dos.flush(); + byte[] byteArray = bos.toByteArray(); + bis = new ByteArrayInputStream(byteArray); + dis = new DataInputStream(bis); + ColumnQualifier c = new ColumnQualifier(); + c.readFields(dis); + ValuePartition vp = c.getValuePartition(); + if (vp instanceof SeparatorPartition) { + + } else { + fail("value partition details no read properly."); + } + assertTrue("ColumnQualifier state mismatch.", c.equals(cq)); + } finally { + if (null != bos) { + bos.close(); + } + if (null != dos) { + dos.close(); + } + if (null != bis) { + bis.close(); + } + if (null != dis) { + dis.close(); + } + } + + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestIndexSpecification.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestIndexSpecification.java new file mode 100644 index 0000000..ddc9aab --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestIndexSpecification.java @@ -0,0 +1,285 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.util.Set; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test whether {@link ColumnQualifier} details provided for {@link IndexSpecification} are valid or + * not. + */ +@Category(MediumTests.class) +public class TestIndexSpecification { + + @Test + public void testInvalidIndexSpecName() throws Exception { + // Contains => + try { + new IndexSpecification("index=>name"); + fail("IllegalArgexception should be thrown."); + } catch (IllegalArgumentException e) { + } + // Contains , + try { + new IndexSpecification("index,name"); + fail("IllegalArgexception should be thrown."); + } catch (IllegalArgumentException e) { + } + // Contains ? + try { + new IndexSpecification("index?name"); + fail("IllegalArgexception should be thrown."); + } catch (IllegalArgumentException e) { + } + // Contains numbers + try { + new IndexSpecification("index0name"); + } catch (IllegalArgumentException e) { + fail("Illegal Arg should not be thrown."); + } + // Contains numbers at beginning and end + try { + new IndexSpecification("0index0name0"); + } catch (IllegalArgumentException e) { + fail("Illegal Arg should not be thrown."); + } + // Contains '.' at the beginning + try { + new IndexSpecification(".indexname"); + fail("IllegalArgexception should be thrown."); + } catch (IllegalArgumentException e) { + } + // Contains "-" at the beginnning + try { + new IndexSpecification("-indexname"); + fail("IllegalArgexception should be thrown."); + } catch (IllegalArgumentException e) { + } + // Contains '.' in the middle + try { + new IndexSpecification("index.name"); + } catch (IllegalArgumentException e) { + fail("Illegal Arg should not be thrown."); + } + // Contains '-' in the middle + try { + new IndexSpecification("index-name"); + } catch (IllegalArgumentException e) { + fail("Illegal Arg should not be thrown."); + } + } + + @Test + public void testGetName() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + assertEquals("Index name not match with acutal index name.", "index_name", iSpec.getName()); + iSpec = new IndexSpecification("index_name"); + assertEquals("Index name not match with acutal index name.", "index_name", iSpec.getName()); + + } + + @Test + public void testAddIndexColumnWithNotNull() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + Set indexColumns = iSpec.getIndexColumns(); + assertEquals("Size of column qualifiers list should be zero.", 0, indexColumns.size()); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + indexColumns = iSpec.getIndexColumns(); + assertTrue("ColumnQualifier state mismatch.", + indexColumns.contains(new ColumnQualifier("cf", "cq", ValueType.String, 10))); + } + + @Test + public void testAddIndexColumnWithNullCF() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(null, "cq", ValueType.String, 10); + fail("Column family name should not be null in index specification."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexColumnWithNullQualifier() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(new HColumnDescriptor("cf"), null, ValueType.String, 10); + fail("Column qualifier name should not be null in index specification."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexColumnWithBlankCForQualifier() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(new HColumnDescriptor(" "), " ", ValueType.String, 10); + fail("Column family name and qualifier should not be blank."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexColumnWithBlankCF() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(new HColumnDescriptor(" "), "cq", ValueType.String, 10); + fail("Column family name should not be blank."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexColumnWithBlankQualifier() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(new HColumnDescriptor("cf"), " ", ValueType.String, 10); + fail("Column qualifier name should not be blank."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexColumnWithControlCharactersOrColonsInCF() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(new HColumnDescriptor("cf:"), "cq", ValueType.String, 10); + fail("Family names should not contain control characters or colons."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testAddIndexColumnWithDuplicaCFandQualifier() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + try { + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + fail("Column familily and qualifier combination should not be" + + " repeated in Index Specification."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testMinTTLCalculation() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec + .addIndexColumn(new HColumnDescriptor("cf").setTimeToLive(100), "cq", ValueType.String, 10); + iSpec + .addIndexColumn(new HColumnDescriptor("cf1").setTimeToLive(50), "cq", ValueType.String, 10); + assertEquals("The ttl should be 50", 50, iSpec.getTTL()); + } + + @Test + public void testMaxVersionsCalculation() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf").setMaxVersions(100), "cq", ValueType.String, + 10); + iSpec.addIndexColumn(new HColumnDescriptor("cf1").setMaxVersions(50), "cq", ValueType.String, + 10); + assertEquals("The max versions should be 50", 50, iSpec.getMaxVersions()); + } + + @Test + public void testMinTTLWhenDefault() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iSpec.addIndexColumn(new HColumnDescriptor("cf1"), "cq", ValueType.String, 10); + assertEquals("The ttl should be " + Integer.MAX_VALUE, HConstants.FOREVER, iSpec.getTTL()); + } + + @Test + public void testMinTTLWhenCombinationOfDefaultAndGivenValue() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iSpec + .addIndexColumn(new HColumnDescriptor("cf1").setTimeToLive(50), "cq", ValueType.String, 10); + assertEquals("The ttl should be 50", 50, iSpec.getTTL()); + } + + @Test + public void testMaxVersionsWhenDefault() throws Exception { + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iSpec.addIndexColumn(new HColumnDescriptor("cf1"), "cq", ValueType.String, 10); + assertEquals("The ttl should be 3", 3, iSpec.getMaxVersions()); + } + + @Test + public void testIndexSpecificationSerialization() throws Exception { + ByteArrayOutputStream bos = null; + DataOutputStream dos = null; + ByteArrayInputStream bis = null; + DataInputStream dis = null; + try { + bos = new ByteArrayOutputStream(); + dos = new DataOutputStream(bos); + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iSpec.write(dos); + dos.flush(); + byte[] byteArray = bos.toByteArray(); + bis = new ByteArrayInputStream(byteArray); + dis = new DataInputStream(bis); + IndexSpecification indexSpecification = new IndexSpecification(); + indexSpecification.readFields(dis); + assertEquals("Index name mismatch.", indexSpecification.getName(), iSpec.getName()); + assertTrue("ColumnQualifier state mismatch.", + iSpec.getIndexColumns().contains(new ColumnQualifier("cf", "cq", ValueType.String, 10))); + } finally { + if (null != bos) { + bos.close(); + } + if (null != dos) { + dos.close(); + } + if (null != bis) { + bis.close(); + } + if (null != dis) { + dis.close(); + } + } + + } +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestIndexedHTableDescriptor.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestIndexedHTableDescriptor.java new file mode 100644 index 0000000..cc6fdb7 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestIndexedHTableDescriptor.java @@ -0,0 +1,187 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.util.List; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test whether index specification details valid or not + */ +@Category(MediumTests.class) +public class TestIndexedHTableDescriptor { + + @Test + public void testAddIndex() throws Exception { + IndexedHTableDescriptor iHtd = new IndexedHTableDescriptor("testAddIndex"); + List indices = null; + assertEquals("Table name is not equal to actual value.", "testAddIndex", iHtd.getNameAsString()); + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iHtd.addIndex(iSpec); + indices = iHtd.getIndices(); + assertEquals("Index name should be equal with actual value.", "index_name", indices.get(0) + .getName()); + assertTrue( + "Column qualifier state mismatch.", + indices.get(0).getIndexColumns() + .contains(new ColumnQualifier("cf", "cq", ValueType.String, 10))); + + } + + @Test + public void testAddIndexWithDuplicaIndexNames() throws Exception { + IndexedHTableDescriptor iHtd = new IndexedHTableDescriptor("testAddIndexWithDuplicaIndexNames"); + assertEquals("Table name is not equal to actual value.", "testAddIndexWithDuplicaIndexNames", + iHtd.getNameAsString()); + IndexSpecification iSpec = null; + try { + iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iHtd.addIndex(iSpec); + iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iHtd.addIndex(iSpec); + fail("Duplicate index names should not present for same table."); + } catch (IllegalArgumentException e) { + + } + + } + + @Test + public void testAddIndexWithIndexNameLengthGreaterThanMaxLength() throws Exception { + IndexedHTableDescriptor iHtd = + new IndexedHTableDescriptor("testAddIndexWithIndexNameLengthGreaterThanMaxLength"); + assertEquals("Table name is not equal to actual value.", + "testAddIndexWithIndexNameLengthGreaterThanMaxLength", iHtd.getNameAsString()); + + IndexSpecification iSpec = null; + try { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 7; i++) { + sb.append("index_name"); + } + iSpec = new IndexSpecification(new String(sb)); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", ValueType.String, 10); + iHtd.addIndex(iSpec); + fail("Index name length should not be more than maximum length " + + Constants.DEF_MAX_INDEX_NAME_LENGTH + '.'); + } catch (IllegalArgumentException e) { + + } + + } + + @Test + public void testIndexedHTableDescriptorSerialization() throws Exception { + ByteArrayOutputStream bos = null; + DataOutputStream dos = null; + ByteArrayInputStream bis = null; + DataInputStream dis = null; + try { + bos = new ByteArrayOutputStream(); + dos = new DataOutputStream(bos); + IndexedHTableDescriptor iHtd = + new IndexedHTableDescriptor("testIndexedHTableDescriptorSerialization"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + iHtd.addFamily(hcd); + HColumnDescriptor hcd1 = new HColumnDescriptor("cf1"); + iHtd.addFamily(hcd1); + IndexSpecification iSpec = new IndexSpecification("index_name"); + iSpec.addIndexColumn(hcd, "cq", ValueType.String, 10); + iHtd.addIndex(iSpec); + IndexSpecification iSpec1 = new IndexSpecification(Bytes.toBytes("index_name1")); + iSpec1.addIndexColumn(hcd1, "cq1", ValueType.String, 10); + iHtd.addIndex(iSpec1); + iHtd.write(dos); + dos.flush(); + byte[] byteArray = bos.toByteArray(); + bis = new ByteArrayInputStream(byteArray); + dis = new DataInputStream(bis); + IndexedHTableDescriptor iHtdRead = new IndexedHTableDescriptor(); + iHtdRead.readFields(dis); + List indices = iHtdRead.getIndices(); + + IndexSpecification indexSpecification = indices.get(0); + IndexSpecification indexSpecification1 = indices.get(1); + assertEquals("Table name should match with actual value.", iHtdRead.getNameAsString(), + "testIndexedHTableDescriptorSerialization"); + assertTrue("HColumnDescriptor state mismatch.", + hcd.equals(iHtdRead.getFamily(Bytes.toBytes("cf")))); + assertEquals("Index name mismatch.", indexSpecification.getName(), iSpec.getName()); + assertTrue( + "ColumnQualifier state mismatch.", + indexSpecification.getIndexColumns().contains( + new ColumnQualifier("cf", "cq", ValueType.String, 10))); + assertTrue( + "ColumnQualifier state mismatch.", + indexSpecification1.getIndexColumns().contains( + new ColumnQualifier("cf1", "cq1", ValueType.String, 10))); + // add finally to close + } finally { + if (null != bos) { + bos.close(); + } + if (null != dos) { + dos.close(); + } + if (null != bis) { + bis.close(); + } + if (null != dis) { + dis.close(); + } + } + } + + @Test + public void testAddIndexWithBlankIndexName() throws Exception { + IndexedHTableDescriptor iHtd = new IndexedHTableDescriptor("testAddIndexWithBlankIndexName"); + assertEquals("Table name is not equal to actual value ", "testAddIndexWithBlankIndexName", + iHtd.getNameAsString()); + + IndexSpecification iSpec = null; + try { + iSpec = new IndexSpecification(" "); + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", null, 10); + iHtd.addIndex(iSpec); + fail("Index name should not be blank in Index Specification"); + } catch (IllegalArgumentException e) { + + } + + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestSecIndexLoadBalancer.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestSecIndexLoadBalancer.java new file mode 100644 index 0000000..c34eeb3 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestSecIndexLoadBalancer.java @@ -0,0 +1,569 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestSecIndexLoadBalancer { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + final int NUM_MASTERS = 1; + final int NUM_RS = 4; + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testRoundRobinAssignmentDuringIndexTableCreation() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + IndexedHTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testRoundRobinAssignmentDuringIndexTableCreation", + "cf", "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + String tableName = "testRoundRobinAssignmentDuringIndexTableCreation"; + String indexTableName = + "testRoundRobinAssignmentDuringIndexTableCreation" + Constants.INDEX_TABLE_SUFFIX; + boolean isRegionColocated = TestUtils.checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + } + + @Test(timeout = 180000) + public void testRoundRobinAssignmentDuringIndexTableEnable() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); + + IndexedHTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testRoundRobinAssignmentDuringIndexTableEnable", + "cf", "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + String tableName = "testRoundRobinAssignmentDuringIndexTableEnable"; + String indexTableName = + "testRoundRobinAssignmentDuringIndexTableEnable" + Constants.INDEX_TABLE_SUFFIX; + admin.disableTable(tableName); + admin.enableTable(tableName); + boolean isRegionColocated = TestUtils.checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + + } + + @Test(timeout = 180000) + public void testColocationAfterSplit() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testCompactionOnIndexTableShouldNotRetrieveTTLExpiredData"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + + HColumnDescriptor hcd = new HColumnDescriptor("col").setMaxVersions(Integer.MAX_VALUE); + + HColumnDescriptor hcd1 = new HColumnDescriptor("col1").setMaxVersions(Integer.MAX_VALUE); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec.addIndexColumn(hcd1, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addFamily(hcd1); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + + Put p = new Put("row11".getBytes()); + p.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p); + + Put p1 = new Put("row21".getBytes()); + p1.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p1.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p1); + + Put p2 = new Put("row31".getBytes()); + p2.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p2.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p2); + + Put p3 = new Put("row41".getBytes()); + p3.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p3.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p3); + + Put p4 = new Put("row51".getBytes()); + p4.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p4.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p4); + + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + + admin.split(userTableName, "row31"); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + List regionsOfUserTable = + master.getAssignmentManager().getRegionsOfTable(Bytes.toBytes(userTableName)); + + while (regionsOfUserTable.size() != 2) { + regionsOfUserTable = + master.getAssignmentManager().getRegionsOfTable(Bytes.toBytes(userTableName)); + } + + List regionsOfIndexTable = + master.getAssignmentManager().getRegionsOfTable(Bytes.toBytes(userTableName + "_idx")); + + while (regionsOfIndexTable.size() != 2) { + regionsOfIndexTable = + master.getAssignmentManager().getRegionsOfTable(Bytes.toBytes(userTableName + "_idx")); + } + + for (HRegionInfo hRegionInfo : regionsOfUserTable) { + for (HRegionInfo indexRegionInfo : regionsOfIndexTable) { + if (Bytes.equals(hRegionInfo.getStartKey(), indexRegionInfo.getStartKey())) { + assertEquals("The regions should be colocated", master.getAssignmentManager() + .getRegionServerOfRegion(hRegionInfo), master.getAssignmentManager() + .getRegionServerOfRegion(indexRegionInfo)); + } + } + } + + } + + @Test(timeout = 180000) + public void testRandomAssignmentDuringIndexTableEnable() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", false); + IndexedHTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testRandomAssignmentDuringIndexTableEnable", "cf", + "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[3][]; + for (int i = 0; i < 3; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + + String tableName = "testRandomAssignmentDuringIndexTableEnable"; + String indexTableName = + "testRandomAssignmentDuringIndexTableEnable" + Constants.INDEX_TABLE_SUFFIX; + admin.disableTable(tableName); + admin.enableTable(tableName); + boolean isRegionColocated = checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + + } + + @Test(timeout = 180000) + public void testBalanceClusterFromAdminBalancer() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", false); + master.getConfiguration().setBoolean("hbase.master.startup.retainassign", false); + master.getConfiguration().setBoolean("hbase.master.loadbalance.bytable", false); + + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor("testBalanceClusterFromAdminBalancer", "cf", "index_name", + "cf", "cq"); + HTableDescriptor htd = new HTableDescriptor("testBalanceClusterFromAdminBalancer1"); + htd.addFamily(new HColumnDescriptor("fam1")); + + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + c = 'A'; + byte[][] split1 = new byte[12][]; + for (int i = 0; i < 12; i++) { + byte[] b = { (byte) c }; + split1[i] = b; + c++; + } + admin.balanceSwitch(false); + admin.createTable(htd, split1); + String tableName = "testBalanceClusterFromAdminBalancer"; + String indexTableName = "testBalanceClusterFromAdminBalancer" + Constants.INDEX_TABLE_SUFFIX; + waitUntilIndexTableCreated(master, indexTableName); + admin.disableTable(tableName); + admin.enableTable(tableName); + admin.balanceSwitch(true); + admin.balancer(); + boolean isRegionsColocated = checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionsColocated); + } + + @Test(timeout = 180000) + public void testByTableBalanceClusterFromAdminBalancer() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = UTIL.getZooKeeperWatcher(UTIL); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", false); + master.getConfiguration().setBoolean("hbase.master.startup.retainassign", false); + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor("testByTableBalanceClusterFromAdminBalancer", "cf", + "index_name", "cf", "cq"); + HTableDescriptor htd = new HTableDescriptor("testByTableBalanceClusterFromAdminBalancer1"); + htd.addFamily(new HColumnDescriptor("fam1")); + + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + c = 'A'; + byte[][] split1 = new byte[12][]; + for (int i = 0; i < 12; i++) { + byte[] b = { (byte) c }; + split1[i] = b; + c++; + } + admin.balanceSwitch(false); + admin.createTable(htd, split1); + String tableName = "testByTableBalanceClusterFromAdminBalancer"; + String indexTableName = + "testByTableBalanceClusterFromAdminBalancer" + Constants.INDEX_TABLE_SUFFIX; + waitUntilIndexTableCreated(master, indexTableName); + admin.disableTable(tableName); + admin.enableTable(tableName); + admin.balanceSwitch(true); + admin.balancer(); + int totalNumRegions = 21; + Thread.sleep(10000); + List> iTableStartKeysAndLocations = null; + do { + iTableStartKeysAndLocations = getStartKeysAndLocations(master, indexTableName); + System.out.println("wait until all regions comeup " + iTableStartKeysAndLocations.size()); + Thread.sleep(1000); + } while (totalNumRegions > iTableStartKeysAndLocations.size()); + ZKAssign.blockUntilNoRIT(zkw); + boolean isRegionColocated = checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + } + + @Test(timeout = 180000) + public void testRandomAssignmentAfterRegionServerDown() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = UTIL.getZooKeeperWatcher(UTIL); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor("testRandomAssignmentAfterRegionServerDown", "cf", + "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + + String tableName = "testRandomAssignmentAfterRegionServerDown"; + String indexTableName = + "testRandomAssignmentAfterRegionServerDown" + Constants.INDEX_TABLE_SUFFIX; + HRegionServer regionServer = cluster.getRegionServer(1); + regionServer.abort("Aborting to test random assignment after region server down"); + while (master.getServerManager().areDeadServersInProgress()) { + Thread.sleep(1000); + } + int totalNumRegions = 21; + List> iTableStartKeysAndLocations = null; + do { + iTableStartKeysAndLocations = getStartKeysAndLocations(master, indexTableName); + System.out.println("wait until all regions comeup " + iTableStartKeysAndLocations.size()); + Thread.sleep(1000); + } while (totalNumRegions > iTableStartKeysAndLocations.size()); + ZKAssign.blockUntilNoRIT(zkw); + boolean isRegionColocated = checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + + } + + @Test(timeout = 180000) + public void testPostAssign() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + ZooKeeperWatcher zkw = UTIL.getZooKeeperWatcher(UTIL); + ZKAssign.deleteAllNodes(zkw); + IndexedHTableDescriptor iHtd = new IndexedHTableDescriptor("testPostAssign"); + char c = 'A'; + byte[][] split = new byte[4][]; + for (int i = 0; i < 4; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + List tableRegions = + MetaReader.getTableRegions(master.getCatalogTracker(), + Bytes.toBytes(iHtd.getNameAsString())); + for (HRegionInfo hRegionInfo : tableRegions) { + admin.assign(hRegionInfo.getRegionName()); + } + ZKAssign.blockUntilNoRIT(zkw); + boolean isRegionColocated = + checkForColocation(master, "testPostAssign", "testPostAssign" + + Constants.INDEX_TABLE_SUFFIX); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + + } + + @Test(timeout = 180000) + public void testRegionMove() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + ZooKeeperWatcher zkw = UTIL.getZooKeeperWatcher(UTIL); + + IndexedHTableDescriptor iHtd = new IndexedHTableDescriptor("testRegionMove"); + char c = 'A'; + byte[][] split = new byte[4][]; + for (int i = 0; i < 4; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + List tableRegions = + MetaReader.getTableRegions(master.getCatalogTracker(), + Bytes.toBytes(iHtd.getNameAsString())); + int numRegions = cluster.getRegionServerThreads().size(); + cluster.getRegionServer(1).getServerName(); + Random random = new Random(); + for (HRegionInfo hRegionInfo : tableRegions) { + int regionNumber = random.nextInt(numRegions); + ServerName serverName = cluster.getRegionServer(regionNumber).getServerName(); + admin.move(hRegionInfo.getEncodedNameAsBytes(), Bytes.toBytes(serverName.getServerName())); + } + ZKAssign.blockUntilNoRIT(zkw); + boolean isRegionColocated = + checkForColocation(master, "testRegionMove", "testRegionMove" + + Constants.INDEX_TABLE_SUFFIX); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + + } + + @Test(timeout = 180000) + public void testRetainAssignmentDuringMasterStartUp() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getConfiguration().setBoolean("hbase.master.startup.retainassign", true); + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor("testRetainAssignmentDuringMasterStartUp", "cf", + "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + String tableName = "testRetainAssignmentDuringMasterStartUp"; + String indexTableName = + "testRetainAssignmentDuringMasterStartUp" + Constants.INDEX_TABLE_SUFFIX; + waitUntilIndexTableCreated(master, indexTableName); + UTIL.shutdownMiniHBaseCluster(); + UTIL.startMiniHBaseCluster(1, 4); + cluster = UTIL.getHBaseCluster(); + master = cluster.getMaster(); + + boolean isRegionColocated = checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + + } + + @Test(timeout = 180000) + public void testRoundRobinAssignmentDuringMasterStartUp() throws Exception { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getConfiguration().setBoolean("hbase.master.startup.retainassign", false); + + IndexedHTableDescriptor iHtd = + TestUtils.createIndexedHTableDescriptor("testRoundRobinAssignmentDuringMasterStartUp", + "cf", "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + String tableName = "testRoundRobinAssignmentDuringMasterStartUp"; + String indexTableName = + "testRoundRobinAssignmentDuringMasterStartUp" + Constants.INDEX_TABLE_SUFFIX; + waitUntilIndexTableCreated(master, indexTableName); + UTIL.shutdownMiniHBaseCluster(); + UTIL.startMiniHBaseCluster(1, 4); + cluster = UTIL.getHBaseCluster(); + master = cluster.getMaster(); + boolean isRegionColocated = checkForColocation(master, tableName, indexTableName); + assertTrue("User regions and index regions should colocate.", isRegionColocated); + + } + + private IndexedHTableDescriptor createIndexedHTableDescriptor(String tableName, + String columnFamily, String indexName, String indexColumnFamily, String indexColumnQualifier) { + IndexedHTableDescriptor htd = new IndexedHTableDescriptor(tableName); + IndexSpecification iSpec = new IndexSpecification(indexName); + HColumnDescriptor hcd = new HColumnDescriptor(columnFamily); + iSpec.addIndexColumn(hcd, indexColumnQualifier, ValueType.String, 10); + htd.addFamily(hcd); + htd.addIndex(iSpec); + return htd; + } + + private void waitUntilIndexTableCreated(HMaster master, String tableName) throws IOException, + InterruptedException { + boolean isEnabled = false; + boolean isExist = false; + do { + isExist = MetaReader.tableExists(master.getCatalogTracker(), tableName); + isEnabled = master.getAssignmentManager().getZKTable().isEnabledTable(tableName); + Thread.sleep(1000); + } while ((false == isExist) && (false == isEnabled)); + } + + private List> getStartKeysAndLocations(HMaster master, String tableName) + throws IOException, InterruptedException { + + List> tableRegionsAndLocations = + MetaReader.getTableRegionsAndLocations(master.getCatalogTracker(), tableName); + List> startKeyAndLocationPairs = + new ArrayList>(tableRegionsAndLocations.size()); + Pair startKeyAndLocation = null; + for (Pair regionAndLocation : tableRegionsAndLocations) { + startKeyAndLocation = + new Pair(regionAndLocation.getFirst().getStartKey(), + regionAndLocation.getSecond()); + startKeyAndLocationPairs.add(startKeyAndLocation); + } + return startKeyAndLocationPairs; + + } + + private boolean checkForColocation(HMaster master, String tableName, String indexTableName) + throws IOException, InterruptedException { + List> uTableStartKeysAndLocations = + getStartKeysAndLocations(master, tableName); + List> iTableStartKeysAndLocations = + getStartKeysAndLocations(master, indexTableName); + + boolean regionsColocated = true; + if (uTableStartKeysAndLocations.size() != iTableStartKeysAndLocations.size()) { + regionsColocated = false; + } else { + for (int i = 0; i < uTableStartKeysAndLocations.size(); i++) { + Pair uStartKeyAndLocation = uTableStartKeysAndLocations.get(i); + Pair iStartKeyAndLocation = iTableStartKeysAndLocations.get(i); + + if (Bytes.compareTo(uStartKeyAndLocation.getFirst(), iStartKeyAndLocation.getFirst()) == 0) { + if (uStartKeyAndLocation.getSecond().equals(iStartKeyAndLocation.getSecond())) { + continue; + } + } + regionsColocated = false; + } + } + return regionsColocated; + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestUtils.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestUtils.java new file mode 100644 index 0000000..c2a9101 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestUtils.java @@ -0,0 +1,106 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestUtils { + + public static IndexedHTableDescriptor createIndexedHTableDescriptor(String tableName, + String columnFamily, String indexName, String indexColumnFamily, String indexColumnQualifier) { + IndexedHTableDescriptor htd = new IndexedHTableDescriptor(tableName); + IndexSpecification iSpec = new IndexSpecification(indexName); + HColumnDescriptor hcd = new HColumnDescriptor(columnFamily); + iSpec.addIndexColumn(hcd, indexColumnQualifier, ValueType.String, 10); + htd.addFamily(hcd); + htd.addIndex(iSpec); + return htd; + } + + public static void waitUntilIndexTableCreated(HMaster master, String tableName) + throws IOException, InterruptedException { + boolean isEnabled = false; + boolean isExist = false; + do { + isExist = MetaReader.tableExists(master.getCatalogTracker(), tableName); + isEnabled = master.getAssignmentManager().getZKTable().isEnabledTable(tableName); + Thread.sleep(1000); + } while ((false == isExist) && (false == isEnabled)); + } + + public static List> getStartKeysAndLocations(HMaster master, + String tableName) throws IOException, InterruptedException { + + List> tableRegionsAndLocations = + MetaReader.getTableRegionsAndLocations(master.getCatalogTracker(), tableName); + List> startKeyAndLocationPairs = + new ArrayList>(tableRegionsAndLocations.size()); + Pair startKeyAndLocation = null; + for (Pair regionAndLocation : tableRegionsAndLocations) { + startKeyAndLocation = + new Pair(regionAndLocation.getFirst().getStartKey(), + regionAndLocation.getSecond()); + startKeyAndLocationPairs.add(startKeyAndLocation); + } + return startKeyAndLocationPairs; + + } + + public static boolean checkForColocation(HMaster master, String tableName, String indexTableName) + throws IOException, InterruptedException { + List> uTableStartKeysAndLocations = + getStartKeysAndLocations(master, tableName); + List> iTableStartKeysAndLocations = + getStartKeysAndLocations(master, indexTableName); + + boolean regionsColocated = true; + if (uTableStartKeysAndLocations.size() != iTableStartKeysAndLocations.size()) { + regionsColocated = false; + } else { + for (int i = 0; i < uTableStartKeysAndLocations.size(); i++) { + Pair uStartKeyAndLocation = uTableStartKeysAndLocations.get(i); + Pair iStartKeyAndLocation = iTableStartKeysAndLocations.get(i); + + if (Bytes.compareTo(uStartKeyAndLocation.getFirst(), iStartKeyAndLocation.getFirst()) == 0) { + if (uStartKeyAndLocation.getSecond().equals(iStartKeyAndLocation.getSecond())) { + continue; + } + } + regionsColocated = false; + } + } + return regionsColocated; + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestValuePartitionInScan.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestValuePartitionInScan.java new file mode 100644 index 0000000..4d3df69 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/TestValuePartitionInScan.java @@ -0,0 +1,586 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index; + +import java.util.ArrayList; +import java.util.List; + +import junit.framework.Assert; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.client.EqualsExpression; +import org.apache.hadoop.hbase.index.client.IndexUtils; +import org.apache.hadoop.hbase.index.client.SingleIndexExpression; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.filter.SingleColumnValuePartitionFilter; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestValuePartitionInScan { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setInt("hbase.regionserver.lease.period", 10 * 60 * 1000); + conf.setInt("hbase.rpc.timeout", 10 * 60 * 1000); + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @After + public void tearDown() throws Exception { + IndexRegionObserver.setIsTestingEnabled(false); + } + + @Before + public void setUp() throws Exception { + IndexRegionObserver.setIndexedFlowUsed(false); + IndexRegionObserver.setSeekpointAdded(false); + IndexRegionObserver.setSeekPoints(null); + IndexRegionObserver.setIsTestingEnabled(true); + IndexRegionObserver.addSeekPoints(null); + } + + @Test(timeout = 180000) + public void testSeparatorPartition() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSeparatorPartition"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SeparatorPartition("_", 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + HTable table = new HTable(conf, "testSeparatorPartition"); + byte[] value1 = "2ndFloor_solitaire_huawei_bangalore_karnataka".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), + "7thFloor_solitaire_huawei_bangalore_karnataka".getBytes()); + table.put(p); + + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "rrr_sss_hhh_bangalore_karnataka".getBytes()); + table.put(p); + + Scan scan = new Scan(); + scan.setFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "huawei".getBytes(), vp)); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + } + + @Test(timeout = 180000) + public void testSpatialPartition() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSpatialPartition"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SpatialPartition(2, 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + HTable table = new HTable(conf, "testSpatialPartition"); + byte[] value1 = "helloworld".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "spatial".getBytes()); + table.put(p); + + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "partition".getBytes()); + table.put(p); + + Scan scan = new Scan(); + scan.setFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.LESS_OR_EQUAL, "rti".getBytes(), vp)); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 3, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 1 rows", testRes.size() == 3); + } + + @Test(timeout = 180000) + public void testSpatialPartitionIfMulitplePartsOfValueAreIndexedByDifferentIndicesOnSameColumn() + throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testSpatialPartitionIfMulitplePartsOfValueAreIndexedByDifferentIndicesOnSameColumn"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SpatialPartition(2, 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); + ihtd.addIndex(iSpec); + ValuePartition vp2 = new SpatialPartition(5, 2); + iSpec = new IndexSpecification("idx2"); + iSpec.addIndexColumn(hcd, "cq", vp2, ValueType.String, 200); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[] value1 = "helloworldmultiple".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "spatialmultiple".getBytes()); + table.put(p); + + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "partitionmultiple".getBytes()); + table.put(p); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "rti".getBytes(), vp)); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "ti".getBytes(), vp2)); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 1 rows", testRes.size() == 1); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.LESS, "rti".getBytes(), vp)); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER, "ti".getBytes(), vp2)); + scan = new Scan(); + scan.setFilter(masterFilter); + i = 0; + scanner = table.getScanner(scan); + testRes = new ArrayList(); + result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 3, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + + } + + @Test(timeout = 180000) + public void + testSeparatorPartitionIfMulitplePartsOfValueAreIndexedByDifferentIndicesOnSameColumn() + throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testSeparatorPartitionIfMulitplePartsOfValueAreIndexedByDifferentIndicesOnSameColumn"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SeparatorPartition("--", 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); + ihtd.addIndex(iSpec); + ValuePartition vp2 = new SeparatorPartition("--", 2); + iSpec = new IndexSpecification("idx2"); + iSpec.addIndexColumn(hcd, "cq", vp2, ValueType.String, 200); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[] value1 = "hello--world--multiple--1".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "spatial--partition--multiple".getBytes()); + table.put(p); + + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "partition--by--separator--multiple".getBytes()); + table.put(p); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp)); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "by".getBytes(), vp2)); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 1 rows", testRes.size() == 2); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "person".getBytes(), vp)); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.LESS, "multiple".getBytes(), vp2)); + scan = new Scan(); + scan.setFilter(masterFilter); + i = 0; + scanner = table.getScanner(scan); + testRes = new ArrayList(); + result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 3, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 1 rows", testRes.size() == 1); + + } + + @Test(timeout = 180000) + public void testCombinationOfPartitionFiltersWithSCVF() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCombinationOfPartitionFiltersWithSCVF"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SeparatorPartition("--", 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); + ihtd.addIndex(iSpec); + ValuePartition vp2 = new SeparatorPartition("--", 2); + iSpec = new IndexSpecification("idx2"); + iSpec.addIndexColumn(hcd, "cq", vp2, ValueType.String, 200); + ihtd.addIndex(iSpec); + iSpec = new IndexSpecification("idx3"); + iSpec.addIndexColumn(hcd, "cq", ValueType.String, 200); + ihtd.addIndex(iSpec); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[] value1 = "hello--world--multiple--1".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "spatial--partition--multiple".getBytes()); + table.put(p); + + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "partition--by--separator--multiple".getBytes()); + table.put(p); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp)); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "by".getBytes(), vp2)); + masterFilter.addFilter(new SingleColumnValueFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "spatial--partition--multiple".getBytes())); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 1 rows", testRes.size() == 1); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + masterFilter.addFilter(new SingleColumnValueFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "partition--by--separator--multiple".getBytes())); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "person".getBytes(), vp)); + masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.LESS, "multiple".getBytes(), vp2)); + scan = new Scan(); + scan.setFilter(masterFilter); + i = 0; + scanner = table.getScanner(scan); + testRes = new ArrayList(); + result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 3, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 1 rows", testRes.size() == 2); + + } + + @Test(timeout = 180000) + public void testCombinationOfPartitionFiltersWithSCVFPart2() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCombinationOfPartitionFiltersWithSCVFPart2"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SeparatorPartition("--", 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 100); + iSpec.addIndexColumn(hcd, "cq1", ValueType.String, 100); + ihtd.addIndex(iSpec); + ValuePartition vp2 = new SeparatorPartition("--", 2); + iSpec = new IndexSpecification("idx2"); + iSpec.addIndexColumn(hcd, "cq", vp2, ValueType.String, 100); + ihtd.addIndex(iSpec); + iSpec = new IndexSpecification("idx3"); + iSpec.addIndexColumn(hcd, "cq1", ValueType.String, 100); + ihtd.addIndex(iSpec); + iSpec = new IndexSpecification("idx4"); + iSpec.addIndexColumn(hcd, "cq", ValueType.String, 100); + iSpec.addIndexColumn(hcd, "cq1", ValueType.String, 100); + ihtd.addIndex(iSpec); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[] value1 = "hello--world--multiple--1".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "spatial--partition--multiple".getBytes()); + p.add("cf1".getBytes(), "cq1".getBytes(), "spatialPartition".getBytes()); + table.put(p); + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "partition--by--multiple--multiple".getBytes()); + p.add("cf1".getBytes(), "cq1".getBytes(), "partitionValue".getBytes()); + table.put(p); + p = new Put("row4".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "partition--multiple--multiple--multiple".getBytes()); + p.add("cf1".getBytes(), "cq1".getBytes(), "multiple".getBytes()); + table.put(p); + p = new Put("row5".getBytes()); + p.add("cf1".getBytes(), "cq1".getBytes(), "abcd".getBytes()); + table.put(p); + p = new Put("row6".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "1234".getBytes()); + table.put(p); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + filter1.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp)); + filter1.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "by".getBytes(), vp2)); + filter1.addFilter(new SingleColumnValueFilter(hcd.getName(), "cq".getBytes(), CompareOp.EQUAL, + "partition--multiple--multiple--multiple".getBytes())); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ONE); + filter2.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp)); + filter2.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp2)); + + FilterList filter3 = new FilterList(Operator.MUST_PASS_ALL); + filter3.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp)); + filter3.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "multiple".getBytes(), vp2)); + + FilterList filter4 = new FilterList(Operator.MUST_PASS_ALL); + filter3.addFilter(new SingleColumnValueFilter(hcd.getName(), "cq1".getBytes(), + CompareOp.GREATER_OR_EQUAL, "1234".getBytes())); + filter3.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.GREATER_OR_EQUAL, "multiple".getBytes(), vp2)); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + masterFilter.addFilter(filter3); + masterFilter.addFilter(filter4); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 1 rows", testRes.size() == 1); + } + + @Test(timeout = 180000) + public void testSingleColumnValuePartitionFilterBySettingAsAttributeToScan() throws Exception { + + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSingleColumnValuePartitionFilterBySettingAsAttributeToScan"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ValuePartition vp = new SeparatorPartition("_", 3); + IndexSpecification iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[] value1 = "2ndFloor_solitaire_huawei_bangalore_karnataka".getBytes(); + Put p = new Put("row".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), value1); + table.put(p); + p = new Put("row2".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), + "7thFloor_solitaire_huawei_bangalore_karnataka".getBytes()); + table.put(p); + + p = new Put("row3".getBytes()); + p.add("cf1".getBytes(), "cq".getBytes(), "rrr_sss_hhh_bangalore_karnataka".getBytes()); + table.put(p); + + Scan scan = new Scan(); + SingleIndexExpression singleIndexExpression = new SingleIndexExpression("idx1"); + byte[] value = "huawei".getBytes(); + Column column = new Column("cf1".getBytes(), "cq".getBytes(), vp); + EqualsExpression equalsExpression = new EqualsExpression(column, value); + singleIndexExpression.addEqualsExpression(equalsExpression); + scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(singleIndexExpression)); + scan.setFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), + CompareOp.EQUAL, "huawei".getBytes(), vp)); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/client/TestIndexUtils.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/client/TestIndexUtils.java new file mode 100644 index 0000000..2a8e040 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/client/TestIndexUtils.java @@ -0,0 +1,67 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ObjectInputStream; + +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestIndexUtils { + + private static final byte[] FAMILY1 = Bytes.toBytes("cf1"); + private static final byte[] QUALIFIER1 = Bytes.toBytes("c1"); + + @Test + public void testConvertingSimpleIndexExpressionToByteArray() throws Exception { + SingleIndexExpression singleIndexExpression = new SingleIndexExpression("idx1"); + Column column = new Column(FAMILY1, QUALIFIER1); + byte[] value = "1".getBytes(); + EqualsExpression equalsExpression = new EqualsExpression(column, value); + singleIndexExpression.addEqualsExpression(equalsExpression); + + byte[] bytes = IndexUtils.toBytes(singleIndexExpression); + ByteArrayInputStream bis = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bis); + SingleIndexExpression readExp = (SingleIndexExpression) ois.readObject(); + assertEquals("idx1", readExp.getIndexName()); + assertEquals(1, readExp.getEqualsExpressions().size()); + assertTrue(Bytes.equals(value, readExp.getEqualsExpressions().get(0).getValue())); + assertEquals(column, readExp.getEqualsExpressions().get(0).getColumn()); + } + + @Test + public void testConvertingBytesIntoIndexExpression() throws Exception { + SingleIndexExpression singleIndexExpression = new SingleIndexExpression("idx1"); + Column column = new Column(FAMILY1, QUALIFIER1); + byte[] value = "1".getBytes(); + EqualsExpression equalsExpression = new EqualsExpression(column, value); + singleIndexExpression.addEqualsExpression(equalsExpression); + + } +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/master/TestIndexMasterObserver.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/master/TestIndexMasterObserver.java new file mode 100644 index 0000000..58471d1 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/master/TestIndexMasterObserver.java @@ -0,0 +1,794 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Tests invocation of the {@link MasterObserverImpl} coprocessor hooks at all appropriate times + * during normal HMaster operations. + */ + +@Category(LargeTests.class) +public class TestIndexMasterObserver { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set("index.data.block.encoding.algo", "PREFIX"); + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testCreateTableWithIndexTableSuffix() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + IndexedHTableDescriptor htd = + createIndexedHTableDescriptor("testCreateTableWithIndexTableSuffix" + + Constants.INDEX_TABLE_SUFFIX, "cf", "index_name", "cf", "cq"); + try { + admin.createTable(htd); + fail("User table should not ends with " + Constants.INDEX_TABLE_SUFFIX); + } catch (IOException e) { + + } + } + + private IndexedHTableDescriptor createIndexedHTableDescriptor(String tableName, + String columnFamily, String indexName, String indexColumnFamily, String indexColumnQualifier) { + IndexedHTableDescriptor htd = new IndexedHTableDescriptor(tableName); + IndexSpecification iSpec = new IndexSpecification(indexName); + HColumnDescriptor hcd = new HColumnDescriptor(columnFamily); + iSpec.addIndexColumn(hcd, indexColumnQualifier, ValueType.String, 10); + htd.addFamily(hcd); + htd.addIndex(iSpec); + return htd; + } + + /* + * @Test public void testCreateIndexTableWhenUserTableAlreadyExist() throws Exception { HBaseAdmin + * admin = UTIL.getHBaseAdmin(); MiniHBaseCluster cluster = UTIL.getHBaseCluster(); HMaster master + * = cluster.getMaster(); HTableDescriptor htd = new HTableDescriptor( + * "testCreateIndexTableWhenUserTableAlreadyExist"); admin.createTable(htd); + * IndexedHTableDescriptor iHtd = createIndexedHTableDescriptor( + * "testCreateIndexTableWhenUserTableAlreadyExist", "cf", "index_name", "cf", "cq"); char c = 'A'; + * byte[][] split = new byte[1][]; for (int i = 0; i < 1; i++) { byte[] b = { (byte) c }; split[i] + * = b; c++; } admin.createTable(iHtd,split); assertTrue("Table is not created.",admin + * .isTableAvailable("testCreateIndexTableWhenUserTableAlreadyExist")); String tableName = + * "testCreateIndexTableWhenUserTableAlreadyExist" + Constants.INDEX_TABLE_SUFFIX; + * waitUntilIndexTableCreated(master, tableName); assertTrue( "Index table is not created.", + * admin.isTableAvailable(tableName)); } + */ + + @Test(timeout = 180000) + public void testCreateIndexTableWhenIndexTableAlreadyExist() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor("testCreateIndexTableWhenIndexTableAlreadyExist", "cf", + "index_name", "cf", "cq"); + admin.createTable(iHtd); + + admin.disableTable("testCreateIndexTableWhenIndexTableAlreadyExist"); + admin.deleteTable("testCreateIndexTableWhenIndexTableAlreadyExist"); + + admin.createTable(iHtd); + + assertTrue("Table is not created.", + admin.isTableAvailable("testCreateIndexTableWhenIndexTableAlreadyExist")); + + String indexTableName = + "testCreateIndexTableWhenIndexTableAlreadyExist" + Constants.INDEX_TABLE_SUFFIX; + + assertTrue("Index table is not created.", admin.isTableAvailable(indexTableName)); + + } + + @Test(timeout = 180000) + public void testCreateIndexTableWhenBothIndexAndUserTableExist() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor("testCreateIndexTableWhenBothIndexAndUserTableExist", "cf", + "index_name", "cf", "cq"); + admin.createTable(iHtd); + + try { + admin.createTable(iHtd); + fail("Should throw TableExistsException " + "if both user table and index table exist."); + } catch (TableExistsException t) { + + } + + } + + @Test(timeout = 180000) + public void testCreateIndexTableWithOutIndexDetails() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + + IndexedHTableDescriptor iHtd = + new IndexedHTableDescriptor("testCreateIndexTableWithOutIndexDetails"); + admin.createTable(iHtd); + assertTrue("Table is not created.", + admin.isTableAvailable("testCreateIndexTableWithOutIndexDetails")); + + String indexTableName = + "testCreateIndexTableWithOutIndexDetails" + Constants.INDEX_TABLE_SUFFIX; + + assertTrue("Index tables is not created.", admin.isTableAvailable(indexTableName)); + } + + @Test(timeout = 180000) + public void testCreateIndexTableWhenExistedIndexTableDisabled() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor("testCreateIndexTableWhenExistedIndexTableDisabled", "cf", + "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + admin.disableTable("testCreateIndexTableWhenExistedIndexTableDisabled"); + admin.deleteTable("testCreateIndexTableWhenExistedIndexTableDisabled"); + + iHtd = + createIndexedHTableDescriptor("testCreateIndexTableWhenExistedIndexTableDisabled", "cf", + "index_name", "cf", "cq"); + admin.createTable(iHtd); + assertTrue("Table is not created.", + admin.isTableAvailable("testCreateIndexTableWhenExistedIndexTableDisabled")); + + String indexTableName = + "testCreateIndexTableWhenExistedIndexTableDisabled" + Constants.INDEX_TABLE_SUFFIX; + + assertTrue("Index tables is not created.", admin.isTableAvailable(indexTableName)); + + } + + @Test(timeout = 180000) + public void testCreateIndexTableWhenExistedTableDisableFailed() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + MiniHBaseCluster cluster = UTIL.getMiniHBaseCluster(); + HMaster master = cluster.getMaster(); + HTableDescriptor htd = + new HTableDescriptor("testCreateIndexTableWhenExistedTableDisableFailed"); + admin.createTable(htd); + + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor("testCreateIndexTableWhenExistedTableDisableFailed", "cf", + "index_name", "cf", "cq"); + IndexMasterObserver ms = Mockito.mock(IndexMasterObserver.class); + Mockito + .doThrow(new RuntimeException()) + .when(ms) + .preCreateTable((ObserverContext) Mockito.anyObject(), + (HTableDescriptor) Mockito.anyObject(), (HRegionInfo[]) Mockito.anyObject()); + + try { + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + assertFalse(master.getAssignmentManager().getZKTable() + .isDisabledTable("testCreateIndexTableWhenExistedTableDisableFailed")); + // fail("Should throw RegionException."); + } catch (IOException r) { + + } finally { + } + } + + @Test(timeout = 180000) + public void testCreateIndexTableWhenExistedTableDeleteFailed() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor("testCreateIndexTableWhenExistedTableDeleteFailed", "cf", + "index_name", "cf", "cq"); + char c = 'A'; + byte[][] split = new byte[20][]; + for (int i = 0; i < 20; i++) { + byte[] b = { (byte) c }; + split[i] = b; + c++; + } + admin.createTable(iHtd, split); + admin.disableTable("testCreateIndexTableWhenExistedTableDeleteFailed"); + admin.deleteTable("testCreateIndexTableWhenExistedTableDeleteFailed"); + IndexMasterObserver ms = Mockito.mock(IndexMasterObserver.class); + Mockito + .doThrow(new RuntimeException()) + .when(ms) + .preCreateTable((ObserverContext) Mockito.anyObject(), + (HTableDescriptor) Mockito.anyObject(), (HRegionInfo[]) Mockito.anyObject()); + try { + admin.createTable(iHtd); + } catch (IOException e) { + + } + } + + @Test(timeout = 180000) + public void testIndexTableCreationAfterMasterRestart() throws Exception { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor("testIndexTableCreationAfterMasterRestart", "cf", + "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable("testIndexTableCreationAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX); + admin.deleteTable("testIndexTableCreationAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + cluster.abortMaster(0); + cluster.waitOnMaster(0); + HMaster master = cluster.startMaster().getMaster(); + cluster.waitForActiveAndReadyMaster(); + String indexTableName = + "testIndexTableCreationAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX; + + assertTrue("Index tables is not created.", admin.isTableAvailable(indexTableName)); + } + + @Test(timeout = 180000) + public void testIndexTableCreationAlongWithNormalTablesAfterMasterRestart() throws Exception { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + + HTableDescriptor htd = + new HTableDescriptor("testIndexTableCreationAlongWithNormalTablesAfterMasterRestart"); + admin.createTable(htd); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + cluster.abortMaster(0); + cluster.waitOnMaster(0); + HMaster master = cluster.startMaster().getMaster(); + cluster.waitForActiveAndReadyMaster(); + + boolean tableExist = + MetaReader.tableExists(master.getCatalogTracker(), + "testIndexTableCreationAlongWithNormalTablesAfterMasterRestart" + + Constants.INDEX_TABLE_SUFFIX); + assertFalse("Index table should be not created after master start up.", tableExist); + } + + @Test(timeout = 180000) + public void testPreCreateShouldNotBeSuccessfulIfIndicesAreNotSame() throws IOException, + KeeperException, InterruptedException { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testNotConsisIndex1"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec1); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec2.addIndexColumn(hcd, "q1", ValueType.Int, 10); + ihtd.addIndex(iSpec2); + + boolean returnVal = false; + try { + admin.createTable(ihtd); + fail("Exception should be thrown"); + } catch (IOException e) { + + returnVal = true; + } + Assert.assertTrue(returnVal); + ZKAssign.blockUntilNoRIT(zkw); + } + + @Test(timeout = 180000) + public void testPreCreateShouldNotBeSuccessfulIfIndicesAreNotSameAtLength() throws IOException, + KeeperException, InterruptedException { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testNotConsisIndex2"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec1.addIndexColumn(hcd, "q2", ValueType.String, 4); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec1); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec2.addIndexColumn(hcd, "q3", ValueType.String, 10); + iSpec2.addIndexColumn(hcd, "q2", ValueType.String, 10); + ihtd.addIndex(iSpec2); + + boolean returnVal = false; + try { + admin.createTable(ihtd); + fail("Exception should be thrown"); + } catch (IOException e) { + returnVal = true; + } + Assert.assertTrue(returnVal); + ZKAssign.blockUntilNoRIT(zkw); + } + + @Test(timeout = 180000) + public void testPreCreateShouldNotBeSuccessfulIfIndicesAreNotSameAtType() throws IOException, + KeeperException, InterruptedException { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testNotConsisIndex3"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec1.addIndexColumn(hcd, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec1); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec2.addIndexColumn(hcd, "q1", ValueType.Int, 10); + iSpec2.addIndexColumn(hcd, "q3", ValueType.String, 10); + ihtd.addIndex(iSpec2); + boolean returnVal = false; + try { + admin.createTable(ihtd); + fail("Exception should be thrown"); + } catch (IOException e) { + returnVal = true; + } + Assert.assertTrue(returnVal); + ZKAssign.blockUntilNoRIT(zkw); + } + + @Test(timeout = 180000) + public void testPreCreateShouldNotBeSuccessfulIfIndicesAreNotSameAtBothTypeAndLength() + throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testNotConsisIndex4"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec1.addIndexColumn(hcd, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec1); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec2.addIndexColumn(hcd, "q1", ValueType.Int, 10); + iSpec2.addIndexColumn(hcd, "q2", ValueType.String, 7); + ihtd.addIndex(iSpec2); + + boolean returnVal = false; + try { + admin.createTable(ihtd); + fail("IOException should be thrown"); + } catch (IOException e) { + returnVal = true; + } + Assert.assertTrue(returnVal); + ZKAssign.blockUntilNoRIT(zkw); + } + + @Test(timeout = 180000) + public void testPreCreateShouldBeSuccessfulIfIndicesAreSame() throws IOException, + KeeperException, InterruptedException { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testConsisIndex"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec1); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec2.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addIndex(iSpec2); + + try { + admin.createTable(ihtd); + } catch (IOException e) { + fail("Exception should not be thrown"); + } + ZKAssign.blockUntilNoRIT(zkw); + } + + @Test(timeout = 180000) + public void testIndexTableShouldBeDisabledIfUserTableDisabled() throws Exception { + String tableName = "testIndexTableDisabledIfUserTableDisabled"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable(tableName); + assertTrue("User table should be disabled.", admin.isTableDisabled(tableName)); + assertTrue("Index table should be disabled.", admin.isTableDisabled(indexTableName)); + } + + @Test(timeout = 180000) + public void testIndexTableShouldBeEnabledIfUserTableEnabled() throws Exception { + String tableName = "testIndexTableEnabledIfUserTableEnabled"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable(tableName); + admin.enableTable(tableName); + assertTrue("User table should be enabled.", admin.isTableEnabled(tableName)); + assertTrue("Index table should be enabled.", admin.isTableEnabled(indexTableName)); + } + + @Test(timeout = 180000) + public void testDisabledIndexTableShouldBeEnabledIfUserTableEnabledAndMasterRestarted() + throws Exception { + String tableName = "testDisabledIndexTableEnabledIfUserTableEnabledAndMasterRestarted"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable(indexTableName); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + cluster.abortMaster(0); + cluster.startMaster(); + cluster.waitOnMaster(0); + cluster.waitForActiveAndReadyMaster(); + Thread.sleep(1000); + assertTrue("User table should be enabled.", admin.isTableEnabled(tableName)); + assertTrue("Index table should be enabled.", admin.isTableEnabled(indexTableName)); + } + + @Test(timeout = 180000) + public void testEnabledIndexTableShouldBeDisabledIfUserTableDisabledAndMasterRestarted() + throws Exception { + String tableName = "testEnabledIndexTableDisabledIfUserTableDisabledAndMasterRestarted"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable(tableName); + admin.enableTable(indexTableName); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + cluster.abortMaster(0); + cluster.startMaster(); + cluster.waitOnMaster(0); + cluster.waitForActiveAndReadyMaster(); + Thread.sleep(1000); + assertTrue("User table should be disabled.", admin.isTableDisabled(tableName)); + assertTrue("Index table should be disabled.", admin.isTableDisabled(indexTableName)); + } + + @Test(timeout = 180000) + public void testDisabledIndexTableShouldBeEnabledIfUserTableInEnablingAndMasterRestarted() + throws Exception { + String tableName = "testDisabledIndexTableEnabledIfUserTableInEnablingAndMasterRestarted"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable(indexTableName); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getAssignmentManager().getZKTable().setEnablingTable(tableName); + cluster.abortMaster(0); + cluster.startMaster(); + cluster.waitOnMaster(0); + cluster.waitForActiveAndReadyMaster(); + Thread.sleep(1000); + assertTrue("User table should be enabled.", admin.isTableEnabled(tableName)); + assertTrue("Index table should be enabled.", admin.isTableEnabled(indexTableName)); + } + + @Test(timeout = 180000) + public void testEnabledIndexTableShouldBeDisabledIfUserTableInDisablingAndMasterRestarted() + throws Exception { + String tableName = "testEnabledIndexTableDisabledIfUserTableInDisablingAndMasterRestarted"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + master.getAssignmentManager().getZKTable().setDisablingTable(tableName); + cluster.abortMaster(0); + cluster.startMaster(); + cluster.waitOnMaster(0); + cluster.waitForActiveAndReadyMaster(); + Thread.sleep(1000); + assertTrue("User table should be disabled.", admin.isTableDisabled(tableName)); + assertTrue("Index table should be disabled.", admin.isTableDisabled(indexTableName)); + } + + @Test(timeout = 180000) + public void testIndexTableShouldBeDeletedIfUserTableDeleted() throws Exception { + String tableName = "testIndexTableDeletedIfUserTableDeleted"; + String indexTableName = IndexUtils.getIndexTableName(tableName); + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + IndexedHTableDescriptor iHtd = + createIndexedHTableDescriptor(tableName, "cf", "index_name", "cf", "cq"); + admin.createTable(iHtd); + admin.disableTable(tableName); + admin.deleteTable(tableName); + assertFalse("User table should not be available after deletion.", + admin.isTableAvailable(tableName)); + assertFalse("Index table should not be available after deletion.", + admin.isTableAvailable(indexTableName)); + } + + @Test(timeout = 180000) + public void testShouldModifyTableWithIndexDetails() throws Exception { + String tableName = "testShouldModifyTableWithIndexDetails"; + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f1"))); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f2"))); + admin.createTable(htd); + + admin.disableTable(tableName); + IndexedHTableDescriptor ihtd = + createIndexedHTableDescriptor(tableName, "f1", "idx1", "f1", "q1"); + admin.modifyTable(Bytes.toBytes(tableName), ihtd); + List regionsOfTable = + UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(Bytes.toBytes(tableName + "_idx")); + while (regionsOfTable.size() != 1) { + regionsOfTable = + UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(Bytes.toBytes(tableName + "_idx")); + } + admin.enableTable(tableName); + assertTrue(admin.isTableEnabled(Bytes.toBytes(tableName + "_idx"))); + } + + @Test(timeout = 180000) + public void testCreateIndexTableFromExistingTable() throws Exception { + String tableName = "testCreateIndexTableFromExistingTable"; + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f1"))); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f2"))); + byte[][] split = new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B") }; + admin.createTable(htd, split); + HTable table = new HTable(admin.getConfiguration(), tableName); + Put p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("f1"), Bytes.toBytes("q1"), Bytes.toBytes("2")); + p.add(Bytes.toBytes("f2"), Bytes.toBytes("q2"), Bytes.toBytes("3")); + table.put(p); + table.flushCommits(); + admin.flush(tableName); + UTIL.getConfiguration().set("table.columns.index", + "IDX1=>f1:[q1->Int&10],[q2],[q3];f2:[q1->String&15],[q2->Int&15]#IDX2=>f1:[q5]"); + IndexUtils.createIndexTable(tableName, UTIL.getConfiguration(), null); + List regionsOfTable = + UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(Bytes.toBytes(tableName + "_idx")); + while (regionsOfTable.size() != 3) { + Thread.sleep(500); + regionsOfTable = + UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(Bytes.toBytes(tableName + "_idx")); + } + MasterFileSystem masterFileSystem = UTIL.getHBaseCluster().getMaster().getMasterFileSystem(); + Path path = FSUtils.getTablePath(masterFileSystem.getRootDir(), htd.getName()); + FileSystem fs = masterFileSystem.getFileSystem(); + FileStatus status = getTableInfoPath(fs, path); + if (null == status) { + fail("Status should not be null"); + } + FSDataInputStream fsDataInputStream = fs.open(status.getPath()); + HTableDescriptor iHtd = new IndexedHTableDescriptor(); + iHtd.readFields(fsDataInputStream); + assertEquals(((IndexedHTableDescriptor) iHtd).getIndices().size(), 2); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(10); + List cf1 = next[0].getColumn(Bytes.toBytes("f1"), Bytes.toBytes("q1")); + List cf2 = next[0].getColumn(Bytes.toBytes("f2"), Bytes.toBytes("q2")); + assertTrue(cf1.size() > 0 && cf2.size() > 0); + } + + @Test(timeout = 180000) + public void testShouldRetainTheExistingCFsInHTD() throws Exception { + String tableName = "testShouldRetainTheExistingCFsInHTD"; + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f1"))); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f2"))); + admin.createTable(htd); + HTable table = new HTable(admin.getConfiguration(), tableName); + Put p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("f1"), Bytes.toBytes("q1"), Bytes.toBytes("2")); + p.add(Bytes.toBytes("f2"), Bytes.toBytes("q2"), Bytes.toBytes("3")); + table.put(p); + table.flushCommits(); + admin.flush(tableName); + UTIL.getConfiguration().set("table.columns.index", "IDX1=>f1:[q1->Int&10],[q2],[q3];"); + IndexUtils.createIndexTable(tableName, UTIL.getConfiguration(), null); + List regionsOfTable = + UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(Bytes.toBytes(tableName + "_idx")); + while (regionsOfTable.size() != 1) { + Thread.sleep(500); + regionsOfTable = + UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(Bytes.toBytes(tableName + "_idx")); + } + MasterFileSystem masterFileSystem = UTIL.getHBaseCluster().getMaster().getMasterFileSystem(); + Path path = FSUtils.getTablePath(masterFileSystem.getRootDir(), htd.getName()); + FileSystem fs = masterFileSystem.getFileSystem(); + FileStatus status = getTableInfoPath(fs, path); + if (null == status) { + fail("Status should not be null"); + } + FSDataInputStream fsDataInputStream = fs.open(status.getPath()); + HTableDescriptor iHtd = new IndexedHTableDescriptor(); + iHtd.readFields(fsDataInputStream); + assertEquals(((IndexedHTableDescriptor) iHtd).getIndices().size(), 1); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(10); + List cf1 = next[0].getColumn(Bytes.toBytes("f1"), Bytes.toBytes("q1")); + List cf2 = next[0].getColumn(Bytes.toBytes("f2"), Bytes.toBytes("q2")); + assertTrue(cf1.size() > 0 && cf2.size() > 0); + } + + @Test(timeout = 180000) + public void testBlockEncoding() throws Exception { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + Configuration conf = admin.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testBlockEncoding"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col1"); + + IndexSpecification iSpec = new IndexSpecification("Index1"); + + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName + "_idx"); + HTableDescriptor tableDescriptor = + admin.getTableDescriptor(Bytes.toBytes(userTableName + "_idx")); + assertEquals(DataBlockEncoding.PREFIX, + tableDescriptor.getColumnFamilies()[0].getDataBlockEncoding()); + } + + private static FileStatus getTableInfoPath(final FileSystem fs, final Path tabledir) + throws IOException { + FileStatus[] status = FSUtils.listStatus(fs, tabledir, new PathFilter() { + @Override + public boolean accept(Path p) { + // Accept any file that starts with TABLEINFO_NAME + return p.getName().startsWith(FSTableDescriptors.TABLEINFO_NAME); + } + }); + if (status == null || status.length < 1) return null; + Arrays.sort(status, new FileStatusFileNameComparator()); + if (status.length > 1) { + // Clean away old versions of .tableinfo + for (int i = 1; i < status.length; i++) { + Path p = status[i].getPath(); + // Clean up old versions + if (!fs.delete(p, false)) { + } else { + } + } + } + return status[0]; + } + + /** + * Compare {@link FileStatus} instances by {@link Path#getName()}. Returns in reverse order. + */ + private static class FileStatusFileNameComparator implements Comparator { + @Override + public int compare(FileStatus left, FileStatus right) { + return -left.compareTo(right); + } + } +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/MultithreadedTestUtil.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/MultithreadedTestUtil.java new file mode 100644 index 0000000..65e27ae --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/MultithreadedTestUtil.java @@ -0,0 +1,150 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; + +public abstract class MultithreadedTestUtil { + + public static final Log LOG = LogFactory.getLog(MultithreadedTestUtil.class); + + public static class TestContext { + private final Configuration conf; + private Throwable err = null; + private boolean stopped = false; + private int threadDoneCount = 0; + private Set testThreads = new HashSet(); + + public TestContext(Configuration configuration) { + this.conf = configuration; + } + + protected Configuration getConf() { + return conf; + } + + public synchronized boolean shouldRun() { + return !stopped && err == null; + } + + public void addThread(TestThread t) { + testThreads.add(t); + } + + public void startThreads() { + for (TestThread t : testThreads) { + t.start(); + } + } + + public void waitFor(long millis) throws Exception { + long endTime = System.currentTimeMillis() + millis; + while (!stopped) { + long left = endTime - System.currentTimeMillis(); + if (left <= 0) break; + synchronized (this) { + checkException(); + wait(left); + } + } + } + + private synchronized void checkException() throws Exception { + if (err != null) { + throw new RuntimeException("Deferred", err); + } + } + + public synchronized void threadFailed(Throwable t) { + if (err == null) err = t; + LOG.error("Failed!", err); + notify(); + } + + public synchronized void threadDone() { + threadDoneCount++; + } + + public void setStopFlag(boolean s) throws Exception { + synchronized (this) { + stopped = s; + } + } + + public void stop() throws Exception { + synchronized (this) { + stopped = true; + } + for (TestThread t : testThreads) { + t.join(); + } + checkException(); + } + } + + /** + * A thread that can be added to a test context, and properly passes exceptions through. + */ + public static abstract class TestThread extends Thread { + protected final TestContext ctx; + protected boolean stopped; + + public TestThread(TestContext ctx) { + this.ctx = ctx; + } + + public void run() { + try { + doWork(); + } catch (Throwable t) { + ctx.threadFailed(t); + } + ctx.threadDone(); + } + + public abstract void doWork() throws Exception; + + protected void stopTestThread() { + this.stopped = true; + } + } + + /** + * A test thread that performs a repeating operation. + */ + public static abstract class RepeatingTestThread extends TestThread { + public RepeatingTestThread(TestContext ctx) { + super(ctx); + } + + public final void doWork() throws Exception { + while (ctx.shouldRun() && !stopped) { + doAnAction(); + } + } + + public abstract void doAnAction() throws Exception; + } +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestAcidGuaranteesForIndex.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestAcidGuaranteesForIndex.java new file mode 100644 index 0000000..de8bde4 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestAcidGuaranteesForIndex.java @@ -0,0 +1,359 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.MultithreadedTestUtil.RepeatingTestThread; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.MultithreadedTestUtil.TestContext; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + +/** + * Test case that uses multiple threads to read and write multifamily rows into a table, verifying + * that reads never see partially-complete writes. This can run as a junit test, or with a main() + * function which runs against a real cluster (eg for testing with failures, region movement, etc) + */ +@Category(MediumTests.class) +public class TestAcidGuaranteesForIndex { + protected static final Log LOG = LogFactory.getLog(TestAcidGuaranteesForIndex.class); + public static final byte[] TABLE_NAME = Bytes.toBytes("TestAcidGuaranteesForIndex"); + public static final byte[] FAMILY_A = Bytes.toBytes("A"); + public static final byte[] FAMILY_B = Bytes.toBytes("B"); + public static final byte[] FAMILY_C = Bytes.toBytes("C"); + public static final byte[] QUALIFIER_NAME = Bytes.toBytes("data"); + + public static final byte[][] FAMILIES = new byte[][] { FAMILY_A, FAMILY_B, FAMILY_C }; + + private HBaseTestingUtility util; + + public static int NUM_COLS_TO_CHECK = 10; + + private void createTableIfMissing() throws IOException { + try { + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(TABLE_NAME); + for (byte[] family : FAMILIES) { + ihtd.addFamily(new HColumnDescriptor(family)); + } + IndexSpecification iSpec = new IndexSpecification("ScanIndex"); + iSpec.addIndexColumn(new HColumnDescriptor(Bytes.toString(FAMILY_A)), + Bytes.toString(QUALIFIER_NAME) + "1", ValueType.String, 10); + ihtd.addIndex(iSpec); + util.getHBaseAdmin().createTable(ihtd); + + } catch (TableExistsException tee) { + } + } + + public TestAcidGuaranteesForIndex() { + // Set small flush size for minicluster so we exercise reseeking scanners + Configuration conf = HBaseConfiguration.create(); + conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128 * 1024)); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + util = new HBaseTestingUtility(conf); + } + + /** + * Thread that does random full-row writes into a table. + */ + public static class AtomicityWriter extends RepeatingTestThread { + Random rand = new Random(); + byte data[] = "value".getBytes(); + byte targetRows[][]; + byte targetFamilies[][]; + HTable table; + AtomicLong numWritten = new AtomicLong(); + + public AtomicityWriter(TestContext ctx, byte targetRows[][], byte targetFamilies[][]) + throws IOException { + super(ctx); + this.targetRows = targetRows; + this.targetFamilies = targetFamilies; + table = new HTable(ctx.getConf(), TABLE_NAME); + } + + public void doAnAction() throws Exception { + // Pick a random row to write into + byte[] targetRow = targetRows[rand.nextInt(targetRows.length)]; + Put p = new Put(targetRow); + for (byte[] family : targetFamilies) { + for (int i = 0; i < NUM_COLS_TO_CHECK; i++) { + byte qualifier[] = Bytes.toBytes("data" + i); + p.add(family, qualifier, data); + } + } + table.put(p); + numWritten.getAndIncrement(); + } + } + + /** + * Thread that does single-row reads in a table, looking for partially completed rows. + */ + public static class AtomicGetReader extends RepeatingTestThread { + byte targetRow[]; + byte targetFamilies[][]; + HTable table; + int numVerified = 0; + AtomicLong numRead = new AtomicLong(); + + public AtomicGetReader(TestContext ctx, byte targetRow[], byte targetFamilies[][]) + throws IOException { + super(ctx); + this.targetRow = targetRow; + this.targetFamilies = targetFamilies; + table = new HTable(ctx.getConf(), TABLE_NAME); + } + + public void doAnAction() throws Exception { + Get g = new Get(targetRow); + Result res = table.get(g); + byte[] gotValue = null; + if (res.getRow() == null) { + // Trying to verify but we didn't find the row - the writing + // thread probably just hasn't started writing yet, so we can + // ignore this action + return; + } + + for (byte[] family : targetFamilies) { + for (int i = 0; i < NUM_COLS_TO_CHECK; i++) { + byte qualifier[] = Bytes.toBytes("data" + i); + byte thisValue[] = res.getValue(family, qualifier); + if (gotValue != null && !Bytes.equals(gotValue, thisValue)) { + gotFailure(gotValue, res); + } + numVerified++; + gotValue = thisValue; + } + } + numRead.getAndIncrement(); + } + + private void gotFailure(byte[] expected, Result res) { + StringBuilder msg = new StringBuilder(); + msg.append("Failed after ").append(numVerified).append("!"); + msg.append("Expected=").append(Bytes.toStringBinary(expected)); + msg.append("Got:\n"); + for (KeyValue kv : res.list()) { + msg.append(kv.toString()); + msg.append(" val= "); + msg.append(Bytes.toStringBinary(kv.getValue())); + msg.append("\n"); + } + throw new RuntimeException(msg.toString()); + } + } + + /** + * Thread that does full scans of the table looking for any partially completed rows. + */ + public static class AtomicScanReader extends RepeatingTestThread { + byte targetFamilies[][]; + HTable table; + AtomicLong numScans = new AtomicLong(); + AtomicLong numRowsScanned = new AtomicLong(); + Random rand = new Random(); + byte data[] = "value".getBytes(); + + public AtomicScanReader(TestContext ctx, byte targetFamilies[][]) throws IOException { + super(ctx); + this.targetFamilies = targetFamilies; + table = new HTable(ctx.getConf(), TABLE_NAME); + } + + public void doAnAction() throws Exception { + Scan s = new Scan(); + for (byte[] family : targetFamilies) { + s.addFamily(family); + } + Filter filter = + new SingleColumnValueFilter(FAMILY_A, (Bytes.toString(QUALIFIER_NAME) + "1").getBytes(), + CompareOp.EQUAL, data); + s.setFilter(filter); + ResultScanner scanner = table.getScanner(s); + + for (Result res : scanner) { + byte[] gotValue = null; + + for (byte[] family : targetFamilies) { + for (int i = 0; i < NUM_COLS_TO_CHECK; i++) { + byte qualifier[] = Bytes.toBytes("data" + i); + byte thisValue[] = res.getValue(family, qualifier); + if (gotValue != null && !Bytes.equals(gotValue, thisValue)) { + gotFailure(gotValue, res); + } + gotValue = thisValue; + } + } + numRowsScanned.getAndIncrement(); + } + numScans.getAndIncrement(); + } + + private void gotFailure(byte[] expected, Result res) { + StringBuilder msg = new StringBuilder(); + msg.append("Failed after ").append(numRowsScanned).append("!"); + msg.append("Expected=").append(Bytes.toStringBinary(expected)); + msg.append("Got:\n"); + for (KeyValue kv : res.list()) { + msg.append(kv.toString()); + msg.append(" val= "); + msg.append(Bytes.toStringBinary(kv.getValue())); + msg.append("\n"); + } + throw new RuntimeException(msg.toString()); + } + } + + public void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners, + int numUniqueRows) throws Exception { + createTableIfMissing(); + TestContext ctx = new TestContext(util.getConfiguration()); + + byte rows[][] = new byte[numUniqueRows][]; + for (int i = 0; i < numUniqueRows; i++) { + rows[i] = Bytes.toBytes("test_row_" + i); + } + + List writers = Lists.newArrayList(); + for (int i = 0; i < numWriters; i++) { + AtomicityWriter writer = new AtomicityWriter(ctx, rows, FAMILIES); + writers.add(writer); + ctx.addThread(writer); + } + // Add a flusher + ctx.addThread(new RepeatingTestThread(ctx) { + public void doAnAction() throws Exception { + util.flush(); + } + }); + + List getters = Lists.newArrayList(); + for (int i = 0; i < numGetters; i++) { + AtomicGetReader getter = new AtomicGetReader(ctx, rows[i % numUniqueRows], FAMILIES); + getters.add(getter); + ctx.addThread(getter); + } + + List scanners = Lists.newArrayList(); + for (int i = 0; i < numScanners; i++) { + AtomicScanReader scanner = new AtomicScanReader(ctx, FAMILIES); + scanners.add(scanner); + ctx.addThread(scanner); + } + + ctx.startThreads(); + ctx.waitFor(millisToRun); + ctx.stop(); + + LOG.info("Finished test. Writers:"); + for (AtomicityWriter writer : writers) { + LOG.info(" wrote " + writer.numWritten.get()); + } + LOG.info("Readers:"); + for (AtomicGetReader reader : getters) { + LOG.info(" read " + reader.numRead.get()); + } + LOG.info("Scanners:"); + for (AtomicScanReader scanner : scanners) { + LOG.info(" scanned " + scanner.numScans.get()); + LOG.info(" verified " + scanner.numRowsScanned.get() + " rows"); + } + } + + @Test + public void testGetAtomicity() throws Exception { + util.startMiniCluster(1); + try { + runTestAtomicity(20000, 5, 5, 0, 3); + } finally { + util.shutdownMiniCluster(); + } + } + + @Test + public void testScanAtomicity() throws Exception { + util.startMiniCluster(1); + try { + runTestAtomicity(20000, 5, 0, 5, 3); + } finally { + util.shutdownMiniCluster(); + } + } + + @Test + public void testMixedAtomicity() throws Exception { + util.startMiniCluster(1); + try { + runTestAtomicity(20000, 5, 2, 2, 3); + } finally { + util.shutdownMiniCluster(); + } + } + + public static void main(String args[]) throws Exception { + Configuration c = HBaseConfiguration.create(); + TestAcidGuaranteesForIndex test = new TestAcidGuaranteesForIndex(); + test.setConf(c); + test.runTestAtomicity(5000, 50, 2, 2, 3); + } + + private void setConf(Configuration c) { + util = new HBaseTestingUtility(c); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestDelete.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestDelete.java new file mode 100644 index 0000000..5c0d6cb --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestDelete.java @@ -0,0 +1,256 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeSet; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestDelete { + private static final String CF_EMP = "emp"; + private static final String CF_DEPT = "dept"; + private static final String CQ_ENAME = "ename"; + private static final String CQ_SAL = "salary"; + private static final String CQ_DNO = "dno"; + private static final String CQ_DNAME = "dname"; + private static final String START_KEY = "100"; + private static final String END_KEY = "200"; + + private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private String DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString(); + + private Path basedir; + private HRegion userRegion; + private HRegion indexRegion; + private Map indexMap; + private IndexRegionObserver indexer; + private Collection indexPuts; + private Collection indexDeletes; + + @Before + public void setup() throws Exception { + prepare(); + + indexMap = new HashMap(); + index("idx_ename", CF_EMP, CQ_ENAME, ValueType.String, 10); + index("idx_sal", CF_EMP, CQ_SAL, ValueType.String, 10); + index("idx_dname", CF_DEPT, CQ_DNAME, ValueType.String, 10); + index("idx_dno_ename", CF_DEPT, CQ_DNO, ValueType.String, 10); + index("idx_dno_ename", CF_EMP, CQ_ENAME, ValueType.String, 10); + indexer = new IndexRegionObserver(); + + indexPuts = new TreeSet(); + indexDeletes = new TreeSet(); + } + + @After + public void teardown() throws IOException { + HRegion.deleteRegion(basedir.getFileSystem(TEST_UTIL.getConfiguration()), basedir, + userRegion.getRegionInfo()); + } + + @Test + public void testDeleteVersion() throws IOException { + // prepare test data + put(101, 1, 1230); + put(101, 1, 1240); + + // Delete version. Boundary scenario + deleteVersion(101, CF_EMP, CQ_SAL, 1230); + + // should delete only one cell - one version + assertTrue("Should delete exactly one nearest version of index entry (salary)", + indexDeletes.size() == 1); + // verify deletes against puts + assertTrue("Index-deletes should be a subset of index puts", + indexPuts.containsAll(indexDeletes)); + } + + @Test + public void testDeleteCells() throws IOException { + // prepare test data + put(101, 0, 1230); + put(101, 0, 1240); + put(102, 2, 1240); + + // Delete cell - all versions of a qualifier + deleteColumn(101, CF_EMP, CQ_SAL); + + assertTrue("Should delete all versions of a cell index entry (salary)", + indexDeletes.size() == 2); + // verify deletes against puts + assertTrue("Index-deletes should be a subset of index puts", + indexPuts.containsAll(indexDeletes)); + } + + @Test + public void testDeleteFamily() throws IOException { + // prepare test data + put(101, 1, 1230); + put(101, 2, 1240); + put(102, 1, 1230); + put(102, 0, 1240); + + // Delete family - All cells of the family + deleteFamily(101, CF_EMP); + + // verify deletes against puts + assertTrue("Index-deletes should be a subset of index puts", + indexPuts.size() > indexDeletes.size()); + assertTrue("Index-deletes should be a subset of index puts", + indexPuts.containsAll(indexDeletes)); + } + + @Test + public void testDeleteRow() throws IOException { + // prepare test data + put(101, 1, 1230); + put(101, 2, 1240); + put(102, 1, 1250); + + // Delete row - all families & all cells + deleteRow(101); + deleteRow(102); + + // verify deletes against puts + assertEquals("Puts and deletes are not same", indexPuts, indexDeletes); + } + + private void prepare() throws IOException { + basedir = new Path(DIR + "TestIndexDelete"); + Configuration conf = TEST_UTIL.getConfiguration(); + + // Prepare the 'employee' table region + HTableDescriptor desc = new HTableDescriptor("employee"); + desc.addFamily(new HColumnDescriptor(CF_EMP)); + desc.addFamily(new HColumnDescriptor(CF_DEPT)); + HRegionInfo info = + new HRegionInfo(desc.getName(), START_KEY.getBytes(), END_KEY.getBytes(), false); + userRegion = HRegion.createHRegion(info, basedir, conf, desc); + + // Prepare the 'employee_idx' index table region + HTableDescriptor idxDesc = new HTableDescriptor("employee_idx"); + idxDesc.addFamily(new HColumnDescriptor(Constants.IDX_COL_FAMILY)); + HRegionInfo idxInfo = + new HRegionInfo(idxDesc.getName(), START_KEY.getBytes(), END_KEY.getBytes(), false); + indexRegion = HRegion.createHRegion(idxInfo, basedir, conf, idxDesc); + } + + private void index(String name, String cf, String cq, ValueType type, int maxSize) { + IndexSpecification index = indexMap.get(name); + if (index == null) { + index = new IndexSpecification(name); + } + index.addIndexColumn(new HColumnDescriptor(cf), cq, type, maxSize); + indexMap.put(name, index); + } + + // For simplicity try to derive all details from eno and dno + // Don't add department details when dno is 0 - Equivalent to adding EMP column family only + private void put(int eno, int dno, long ts) throws IOException { + Put put = new Put(toBytes("" + eno)); + + put.add(toBytes(CF_EMP), toBytes(CQ_ENAME), ts, toBytes("emp_" + eno)); + put.add(toBytes(CF_EMP), toBytes(CQ_SAL), ts, toBytes("" + eno * 100)); + // Don't add department details when dno is 0 + // Equivalent to adding EMP column family only + if (dno != 0) { + put.add(toBytes(CF_DEPT), toBytes(CQ_DNO), ts, toBytes("" + dno)); + put.add(toBytes(CF_DEPT), toBytes(CQ_DNAME), ts, toBytes("dept_" + dno)); + } + put.setWriteToWAL(false); + userRegion.put(put); + for (IndexSpecification spec : indexMap.values()) { + Put idxPut = IndexUtils.prepareIndexPut(put, spec, indexRegion); + if (idxPut != null) { + KeyValue kv = idxPut.get(Constants.IDX_COL_FAMILY, new byte[0]).get(0); + indexPuts.add(Bytes.toString(idxPut.getRow()) + "_" + kv.getTimestamp()); + } + } + } + + private void deleteRow(int eno) throws IOException { + Delete delete = new Delete(toBytes("" + eno)); + + // Make the delete ready-to-eat by indexer + for (byte[] family : userRegion.getTableDesc().getFamiliesKeys()) { + delete.deleteFamily(family, delete.getTimeStamp()); + } + delete(delete); + } + + private void deleteFamily(int eno, String cf) throws IOException { + Delete delete = new Delete(toBytes("" + eno)); + delete.deleteFamily(toBytes(cf)); + delete(delete); + } + + private void deleteColumn(int eno, String cf, String cq) throws IOException { + Delete delete = new Delete(toBytes("" + eno)); + delete.deleteColumns(toBytes(cf), toBytes(cq)); + delete(delete); + } + + private void deleteVersion(int eno, String cf, String cq, long ts) throws IOException { + Delete delete = new Delete(toBytes("" + eno)); + delete.deleteColumn(toBytes(cf), toBytes(cq), ts); + delete(delete); + } + + private void delete(Delete delete) throws IOException { + Collection deletes = + indexer.prepareIndexDeletes(delete, userRegion, + new ArrayList(indexMap.values()), indexRegion); + for (Mutation idxdelete : deletes) { + KeyValue kv = idxdelete.getFamilyMap().get(Constants.IDX_COL_FAMILY).get(0); + indexDeletes.add(Bytes.toString(idxdelete.getRow()) + "_" + kv.getTimestamp()); + } + } +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestExtendedPutOps.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestExtendedPutOps.java new file mode 100644 index 0000000..ba48b68 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestExtendedPutOps.java @@ -0,0 +1,777 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.index.ColumnQualifier; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.SeparatorPartition; +import org.apache.hadoop.hbase.index.SpatialPartition; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestExtendedPutOps { + private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final String DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString(); + + @Test(timeout = 180000) + public void testPutWithOneUnitLengthSeparator() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testPutWithOneUnitLengthSeparator"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("_", 4), + ValueType.String, 10); + + byte[] value1 = "2ndFloor_solitaire_huawei_bangalore_karnataka".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[10]; + System.arraycopy("bangalore".getBytes(), 0, expectedResult, 0, "bangalore".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "2ndFloor_solitaire_huawei_bangal".getBytes(); + p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + System.arraycopy("bangal".getBytes(), 0, expectedResult, 0, "bangal".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "2ndFloor_solitaire_huawei_".getBytes(); + p = new Put("row2".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testPutWithOneAsSplit() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testPutWithOneUnitLengthSeparator"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("---", 1), + ValueType.String, 10); + + byte[] value1 = "AB---CD---EF---GH---IJ---KL---MN---OP---".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[10]; + System.arraycopy("AB".getBytes(), 0, expectedResult, 0, "AB".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "---CD---EF---GH---IJ---KL---MN---OP---".getBytes(); + p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "AB".getBytes(); + p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + System.arraycopy("AB".getBytes(), 0, expectedResult, 0, "AB".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "".getBytes(); + p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testPutWithOneUnitLengthSeparatorWithoutValue() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testPutWithOneUnitLengthSeparatorWithoutValue"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("_", 4), + ValueType.String, 10); + byte[] value1 = "2ndFloor_solitaire_huawei__karnataka".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[10]; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithMultipleUnitLengthSeparator() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithMultipleUnitLengthSeparator"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("---", 6), + ValueType.String, 10); + + byte[] value1 = "AB---CD---EF---GH---IJ---KL---MN---OP---".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[10]; + System.arraycopy("KL".getBytes(), 0, expectedResult, 0, "KL".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "AB---CD---EF---GH---IJ---K".getBytes(); + p = new Put("row2".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + System.arraycopy("K".getBytes(), 0, expectedResult, 0, "K".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "AB---CD---EF---GH---".getBytes(); + p = new Put("row2".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithMultipleUnitLengthWithSimilarStringPattern() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithMultipleUnitLengthSeparator"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("---", 6), + ValueType.String, 10); + + byte[] value1 = "AB---CD---EF---GH---IJ---K-L---MN---OP---".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[10]; + System.arraycopy("K-L".getBytes(), 0, expectedResult, 0, "K-L".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "AB---CD---EF---GH---IJ---K--L".getBytes(); + p = new Put("row2".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + System.arraycopy("K--L".getBytes(), 0, expectedResult, 0, "K--L".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "AB---CD---EF---GH---IJ----".getBytes(); + p = new Put("row2".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[10]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[10]; + expectedResult[0] = '-'; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithOffsetAndLength() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithOffsetAndLength"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SpatialPartition(20, 2), + ValueType.String, 18); + + byte[] value1 = "AB---CD---EF---GH---IJ---KL---MN---OP---".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[2]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[2]; + System.arraycopy("IJ".getBytes(), 0, expectedResult, 0, "IJ".getBytes().length); + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithOffsetAndLengthWhenPutIsSmallerThanOffset() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = + new HTableDescriptor("testIndexPutWithOffsetAndLengthWhenPutIsSmallerThanOffset"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SpatialPartition(20, 2), + ValueType.String, 18); + + byte[] value1 = "AB---CD---EF---GH".getBytes(); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] indexRowKey = indexPut.getRow(); + byte[] actualResult = new byte[2]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + byte[] expectedResult = new byte[2]; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + + value1 = "AB---CD---EF---GH---I".getBytes(); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + indexRowKey = indexPut.getRow(); + actualResult = new byte[2]; + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + expectedResult = new byte[2]; + expectedResult[0] = 'I'; + Assert.assertTrue(Bytes.equals(actualResult, expectedResult)); + } + + @Test(timeout = 180000) + public void testExtentedParametersValidityFailScenarios() throws IOException { + IndexSpecification spec = new IndexSpecification("index"); + + // When separator length is zero + try { + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("", 4), + ValueType.String, 10); + Assert.fail("Testcase should fail if separator length is zero."); + } catch (IllegalArgumentException e) { + } + + // when the valuePosition is zero with separator + try { + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("--", 0), + ValueType.String, 10); + Assert + .fail("the testcase should fail if the valuePosition with the separator is passed as zero."); + } catch (IllegalArgumentException e) { + } + + } + + public void testExtentedParametersValidityPassScenarios() throws IOException { + IndexSpecification spec = new IndexSpecification("index"); + + // When the provided arguments are correct + try { + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SpatialPartition(4, 10), + ValueType.String, 10); + } catch (IllegalArgumentException e) { + Assert.fail("the testcase should not throw exception as the arguments passed are correct."); + } + + // When the provided arguments are correct + try { + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("--", 1), + ValueType.String, 10); + } catch (IllegalArgumentException e) { + Assert.fail("the testcase should not throw exception as the arguments passed are correct."); + } + + try { + spec.addIndexColumn(new HColumnDescriptor("col"), "ql2", ValueType.String, 10); + } catch (IllegalArgumentException e) { + Assert.fail("the testcase should not throw exception as the arguments passed are correct."); + } + } + + @Test(timeout = 180000) + public void testColumnQualifierSerialization() throws Exception { + ByteArrayOutputStream bos = null; + DataOutputStream dos = null; + ByteArrayInputStream bis = null; + DataInputStream dis = null; + try { + bos = new ByteArrayOutputStream(); + dos = new DataOutputStream(bos); + ColumnQualifier cq = + new ColumnQualifier("cf", "cq", ValueType.String, 10, new SpatialPartition(0, 5)); + cq.write(dos); + dos.flush(); + byte[] byteArray = bos.toByteArray(); + bis = new ByteArrayInputStream(byteArray); + dis = new DataInputStream(bis); + ColumnQualifier c = new ColumnQualifier(); + c.readFields(dis); + assertTrue("ColumnQualifier state mismatch.", c.equals(cq)); + } finally { + if (null != bos) { + bos.close(); + } + if (null != dos) { + dos.close(); + } + if (null != bis) { + bis.close(); + } + if (null != dis) { + dis.close(); + } + } + + } + + @Test(timeout = 180000) + public void testIndexPutwithPositiveIntDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutwithPositiveIntDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Int, 4); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql2", ValueType.Float, 4); + + byte[] value1 = Bytes.toBytes(1000); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut1 = IndexUtils.prepareIndexPut(p, spec, region); + int a = 1000; + byte[] expectedResult = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult = new byte[4]; + byte[] indexRowKey = indexPut1.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithNegativeIntDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithNegativeIntDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Int, 4); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql2", ValueType.Float, 4); + + byte[] value1 = Bytes.toBytes(-2562351); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + int a = -2562351; + byte[] expectedResult = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult = new byte[4]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithLongDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithNegativeIntDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Long, 4); + + byte[] value1 = Bytes.toBytes(-2562351L); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + long a = -2562351L; + byte[] expectedResult = Bytes.toBytes(a ^ (1L << 63)); + byte[] actualResult = new byte[8]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithShortDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithNegativeIntDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Short, 4); + + short s = 1000; + byte[] value1 = Bytes.toBytes(s); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] expectedResult = Bytes.toBytes(s); + expectedResult[0] ^= 1 << 7; + byte[] actualResult = new byte[2]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithByteDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithNegativeIntDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Short, 4); + + byte b = 100; + byte[] value1 = Bytes.toBytes(b); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] expectedResult = Bytes.toBytes(b); + expectedResult[0] ^= 1 << 7; + byte[] actualResult = new byte[2]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithCharDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithNegativeIntDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Char, 4); + + char c = 'A'; + byte[] value1 = new byte[2]; + value1[1] = (byte) c; + c >>= 8; + value1[0] = (byte) c; + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] actualResult = new byte[2]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(value1, actualResult)); + } + + @Test(timeout = 180000) + public void testIndexPutWithDoubleDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithNegativeIntDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + // HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Double, 8); + + byte[] value1 = Bytes.toBytes(109.4548957D); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + double d = 109.4548957D; + byte[] expectedResult = Bytes.toBytes(d); + expectedResult[0] ^= 1 << 7; + byte[] actualResult = new byte[8]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + + value1 = Bytes.toBytes(-109.4548957D); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + indexPut = IndexUtils.prepareIndexPut(p, spec, region); + d = -109.4548957D; + expectedResult = Bytes.toBytes(d); + for (int i = 0; i < 8; i++) { + expectedResult[i] ^= 0xff; + } + actualResult = new byte[8]; + indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + + @Test(timeout = 180000) + public void testSequenceOfIndexPutsWithNegativeInteger() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testSequenceOfIndexPutsWithDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Int, 4); + + byte[] value1 = Bytes.toBytes(-1000); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + int a = -1000; + byte[] expectedResult = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult = new byte[4]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + + value1 = Bytes.toBytes(-1500); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut1 = IndexUtils.prepareIndexPut(p, spec, region); + a = -1500; + byte[] expectedResult1 = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult1 = new byte[4]; + byte[] indexRowKey1 = indexPut1.getRow(); + System.arraycopy(indexRowKey1, 22, actualResult1, 0, actualResult1.length); + Assert.assertTrue(Bytes.equals(expectedResult1, actualResult1)); + + Assert.assertTrue(Bytes.compareTo(indexPut.getRow(), indexPut1.getRow()) > 0); + + value1 = Bytes.toBytes(1500); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut2 = IndexUtils.prepareIndexPut(p, spec, region); + a = 1500; + byte[] expectedResult2 = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult2 = new byte[4]; + byte[] indexRowKey2 = indexPut2.getRow(); + System.arraycopy(indexRowKey2, 22, actualResult2, 0, actualResult2.length); + Assert.assertTrue(Bytes.equals(expectedResult2, actualResult2)); + + Assert.assertTrue(Bytes.compareTo(indexPut2.getRow(), indexPut.getRow()) > 0); + + value1 = Bytes.toBytes(2000); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut3 = IndexUtils.prepareIndexPut(p, spec, region); + a = 2000; + byte[] expectedResult3 = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult3 = new byte[4]; + byte[] indexRowKey3 = indexPut3.getRow(); + System.arraycopy(indexRowKey3, 22, actualResult3, 0, actualResult3.length); + Assert.assertTrue(Bytes.equals(expectedResult3, actualResult3)); + + Assert.assertTrue(Bytes.compareTo(indexPut3.getRow(), indexPut2.getRow()) > 0); + } + + @Test(timeout = 180000) + public void testSequenceOfIndexPutsWithNegativeFloat() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testSequenceOfIndexPutsWithDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Float, 4); + + byte[] value1 = Bytes.toBytes(-10.40f); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + byte[] expectedResult = Bytes.toBytes(-10.40f); + expectedResult[0] ^= 0xff; + expectedResult[1] ^= 0xff; + expectedResult[2] ^= 0xff; + expectedResult[3] ^= 0xff; + byte[] actualResult = new byte[4]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + + value1 = Bytes.toBytes(-15.20f); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut1 = IndexUtils.prepareIndexPut(p, spec, region); + byte[] expectedResult1 = Bytes.toBytes(-15.20f); + expectedResult1[0] ^= 0xff; + expectedResult1[1] ^= 0xff; + expectedResult1[2] ^= 0xff; + expectedResult1[3] ^= 0xff; + byte[] actualResult1 = new byte[4]; + byte[] indexRowKey1 = indexPut1.getRow(); + System.arraycopy(indexRowKey1, 22, actualResult1, 0, actualResult1.length); + Assert.assertTrue(Bytes.equals(expectedResult1, actualResult1)); + + Assert.assertTrue(Bytes.compareTo(indexPut.getRow(), indexPut1.getRow()) > 0); + + value1 = Bytes.toBytes(30.50f); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut2 = IndexUtils.prepareIndexPut(p, spec, region); + byte[] expectedResult2 = Bytes.toBytes(30.50f); + expectedResult2[0] ^= 1 << 7; + byte[] actualResult2 = new byte[4]; + byte[] indexRowKey2 = indexPut2.getRow(); + System.arraycopy(indexRowKey2, 22, actualResult2, 0, actualResult2.length); + Assert.assertTrue(Bytes.equals(expectedResult2, actualResult2)); + + Assert.assertTrue(Bytes.compareTo(indexPut2.getRow(), indexPut.getRow()) > 0); + + value1 = Bytes.toBytes(40.54f); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut3 = IndexUtils.prepareIndexPut(p, spec, region); + byte[] expectedResult3 = Bytes.toBytes(40.54f); + expectedResult3[0] ^= 1 << 7; + byte[] actualResult3 = new byte[4]; + byte[] indexRowKey3 = indexPut3.getRow(); + System.arraycopy(indexRowKey3, 22, actualResult3, 0, actualResult3.length); + Assert.assertTrue(Bytes.equals(expectedResult3, actualResult3)); + + Assert.assertTrue(Bytes.compareTo(indexPut3.getRow(), indexPut2.getRow()) > 0); + } + + @Test(timeout = 180000) + public void testSequenceOfIndexPutsWithDataTypes() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testSequenceOfIndexPutsWithDataTypes"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", ValueType.Int, 4); + + byte[] value1 = Bytes.toBytes(1000); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + int a = 1000; + byte[] expectedResult = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult = new byte[4]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + + value1 = Bytes.toBytes(-2562351); + p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), value1); + Put indexPut1 = IndexUtils.prepareIndexPut(p, spec, region); + a = -2562351; + byte[] expectedResult1 = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult1 = new byte[4]; + byte[] indexRowKey1 = indexPut1.getRow(); + System.arraycopy(indexRowKey1, 22, actualResult1, 0, actualResult1.length); + Assert.assertTrue(Bytes.equals(expectedResult1, actualResult1)); + + Assert.assertTrue(Bytes.compareTo(indexPut.getRow(), indexPut1.getRow()) > 0); + } + + @Test(timeout = 180000) + public void testIndexPutWithSeparatorAndDataType() throws IOException { + Path basedir = new Path(DIR + "TestIndexPut"); + Configuration conf = TEST_UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("testIndexPutWithSeparatorAndDataType"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("col"), "ql1", new SeparatorPartition("---", 4), + ValueType.Int, 4); + + byte[] putValue = new byte[19]; + byte[] value1 = "AB---CD---EF---".getBytes(); + byte[] value2 = Bytes.toBytes(100000); + System.arraycopy(value1, 0, putValue, 0, value1.length); + System.arraycopy(value2, 0, putValue, value1.length, value2.length); + Put p = new Put("row".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), putValue); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + int a = 100000; + byte[] expectedResult = Bytes.toBytes(a ^ (1 << 31)); + byte[] actualResult = new byte[4]; + byte[] indexRowKey = indexPut.getRow(); + System.arraycopy(indexRowKey, 22, actualResult, 0, actualResult.length); + Assert.assertTrue(Bytes.equals(expectedResult, actualResult)); + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestForComplexIssues.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestForComplexIssues.java new file mode 100644 index 0000000..6cb76fd --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestForComplexIssues.java @@ -0,0 +1,529 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.UnknownRegionException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestForComplexIssues { + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + public static CountDownLatch latch = new CountDownLatch(1); + public static CountDownLatch latchForCompact = new CountDownLatch(1); + private static boolean compactionCalled = false; + private static volatile boolean closeCalled = false; + private static volatile boolean delayPostBatchMutate = false; + private static int openCount = 0; + private static int closeCount = 0; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, LocalIndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + + } + + @Before + public void setUp() throws Exception { + UTIL.startMiniCluster(2); + } + + @After + public void tearDown() throws Exception { + compactionCalled = false; + closeCalled = false; + delayPostBatchMutate = false; + UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testHDP2989() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.client.retries.number", 1); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setInt("hbase.regionserver.lease.period", 90000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testHDP2989"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + admin.flush(userTableName); + } + + MiniHBaseCluster hBaseCluster = UTIL.getHBaseCluster(); + List regions = hBaseCluster.getRegions(Bytes.toBytes(userTableName)); + HRegion hRegion = regions.get(0); + byte[] encodedNameAsBytes = hRegion.getRegionInfo().getEncodedNameAsBytes(); + int serverWith = hBaseCluster.getServerWith(hRegion.getRegionName()); + int destServerNo = (serverWith == 1 ? 0 : 1); + HRegionServer destServer = hBaseCluster.getRegionServer(destServerNo); + + admin.compact(userTableName); + + while (!compactionCalled) { + Thread.sleep(1000); + } + + byte[] dstRSName = Bytes.toBytes(destServer.getServerName().getServerName()); + // Move the main region + MoveThread move = new MoveThread(admin, encodedNameAsBytes, dstRSName); + move.start(); + + // Move the indexRegion + regions = hBaseCluster.getRegions(Bytes.toBytes(userTableName + "_idx")); + HRegion hRegion1 = regions.get(0); + encodedNameAsBytes = hRegion1.getRegionInfo().getEncodedNameAsBytes(); + serverWith = hBaseCluster.getServerWith(hRegion1.getRegionName()); + destServerNo = (serverWith == 1 ? 0 : 1); + destServer = hBaseCluster.getRegionServer(destServerNo); + + dstRSName = Bytes.toBytes(destServer.getServerName().getServerName()); + move = new MoveThread(admin, encodedNameAsBytes, dstRSName); + move.start(); + while (!closeCalled) { + Thread.sleep(200); + } + + String row = "row" + 46; + Put p = new Put(row.getBytes()); + String val = "Val" + 46; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + try { + table.put(p); + } catch (Exception e) { + e.printStackTrace(); + } + latchForCompact.countDown(); + closeCount++; + latch.countDown(); + List onlineRegions = destServer.getOnlineRegions(Bytes.toBytes(userTableName)); + List onlineIdxRegions = + destServer.getOnlineRegions(Bytes.toBytes(userTableName + "_idx")); + while (onlineRegions.size() == 0 || onlineIdxRegions.size() == 0) { + Thread.sleep(1000); + onlineRegions = destServer.getOnlineRegions(Bytes.toBytes(userTableName)); + onlineIdxRegions = destServer.getOnlineRegions(Bytes.toBytes(userTableName + "_idx")); + } + + /* + * closeCount++; latch.countDown(); + */ + + Scan s = new Scan(); + conf.setInt("hbase.client.retries.number", 10); + table = new HTable(conf, userTableName); + ResultScanner scanner = table.getScanner(s); + int i = 0; + for (Result result : scanner) { + i++; + } + HTable indextable = new HTable(conf, userTableName + "_idx"); + s = new Scan(); + scanner = indextable.getScanner(s); + int j = 0; + for (Result result : scanner) { + j++; + } + assertEquals("", i, j); + + } + + @Test(timeout = 180000) + // test for HDP-2983 + public + void testCompactionOfIndexRegionBeforeMainRegionOpens() throws Exception { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + Configuration conf = admin.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testCompactionOfIndexRegionBeforeMainRegionOpens"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col1"); + ihtd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + IndexSpecification iSpec1 = new IndexSpecification("Index2"); + iSpec1.addIndexColumn(hcd, "q2", ValueType.String, 10); + ihtd.addIndex(iSpec); + ihtd.addIndex(iSpec1); + admin.createTable(ihtd); + + ZKAssign.blockUntilNoRIT(zkw); + openCount++; + HTable table = new HTable(conf, userTableName); + Put p = null; + for (int i = 0; i < 25; i++) { + p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql"), Bytes.toBytes("test_val")); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("q2"), Bytes.toBytes("test_val")); + table.put(p); + if (i == 4 || i == 11 || i == 24) { + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + } + } + + HTable indextable = new HTable(conf, userTableName + "_idx"); + Scan s = new Scan(); + ResultScanner scanner = indextable.getScanner(s); + int j = 0; + for (Result result : scanner) { + j++; + } + assertEquals("", 50, j); + + MiniHBaseCluster hBaseCluster = UTIL.getHBaseCluster(); + List regions = hBaseCluster.getRegions(Bytes.toBytes(userTableName)); + HRegion hRegion = regions.get(0); + + List indexRegions = hBaseCluster.getRegions(Bytes.toBytes(userTableName + "_idx")); + HRegion indexHRegion = indexRegions.get(0); + int srcServer = hBaseCluster.getServerWith(hRegion.getRegionName()); + int destServer = (srcServer == 1 ? 0 : 1); + HRegionServer regionServer = hBaseCluster.getRegionServer(destServer); + + // Abort the regionserver + hBaseCluster.abortRegionServer(srcServer); + hBaseCluster.waitOnRegionServer(srcServer); + + boolean regionOnline = + hBaseCluster.getMaster().getAssignmentManager() + .isRegionAssigned(indexHRegion.getRegionInfo()); + while (!regionOnline) { + regionOnline = + hBaseCluster.getMaster().getAssignmentManager() + .isRegionAssigned(indexHRegion.getRegionInfo()); + } + admin.compact(userTableName + "_idx"); + List onlineRegions = + regionServer.getOnlineRegions(Bytes.toBytes(userTableName + "_idx")); + List storeFileList = + regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + while (storeFileList.size() != 1) { + Thread.sleep(1000); + storeFileList = regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + } + assertEquals("The total store files for the index table should be 1", 1, storeFileList.size()); + + indextable = new HTable(conf, userTableName + "_idx"); + s = new Scan(); + scanner = indextable.getScanner(s); + j = 0; + for (Result result : scanner) { + j++; + } + assertEquals("", 50, j); + // Decrement the latch + latch.countDown(); + ZKAssign.blockUntilNoRIT(zkw); + openCount = 0; + // Check after dropping the index + storeFileList = regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + String storeFileName = storeFileList.get(0); + // admin.dropIndex("Index2", userTableName); + ZKAssign.blockUntilNoRIT(zkw); + while (storeFileList.get(0).equals(storeFileName)) { + Thread.sleep(1000); + storeFileList = regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + } + + indextable = new HTable(conf, userTableName + "_idx"); + s = new Scan(); + scanner = indextable.getScanner(s); + j = 0; + for (Result result : scanner) { + j++; + } + assertEquals("", 25, j); + + } + + @Test(timeout = 180000) + public void testHDP3015() throws Exception { + final HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setInt("hbase.regionserver.lease.period", 90000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + final String userTableName = "testHDP3015"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i < 4; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + if (i == 3) { + delayPostBatchMutate = true; + } + table.put(p); + if (i == 2) { + new Thread() { + public void run() { + try { + admin.flush(userTableName + "_idx"); + } catch (Exception e) { + + } + } + }.start(); + } + } + MiniHBaseCluster hBaseCluster = UTIL.getHBaseCluster(); + List userRegions = hBaseCluster.getRegions(Bytes.toBytes(userTableName)); + + List indexRegions = hBaseCluster.getRegions(Bytes.toBytes(userTableName + "_idx")); + HRegion indexHRegion = indexRegions.get(0); + + int serverWith = hBaseCluster.getServerWith((userRegions.get(0).getRegionName())); + HRegionServer regionServer = hBaseCluster.getRegionServer(serverWith); + List storeFileList = regionServer.getStoreFileList(indexHRegion.getRegionName()); + while (storeFileList.size() == 0) { + storeFileList = regionServer.getStoreFileList(indexHRegion.getRegionName()); + } + + table = new HTable(conf, userTableName); + ResultScanner scanner = table.getScanner(new Scan()); + int i = 0; + for (Result result : scanner) { + i++; + } + table = new HTable(conf, userTableName + "_idx"); + scanner = table.getScanner(new Scan()); + int j = 0; + for (Result result : scanner) { + j++; + } + assertEquals("", i, j); + + hBaseCluster.abortRegionServer(serverWith); + hBaseCluster.waitOnRegionServer(serverWith); + + boolean regionOnline = + hBaseCluster.getMaster().getAssignmentManager() + .isRegionAssigned(indexHRegion.getRegionInfo()); + while (!regionOnline) { + regionOnline = + hBaseCluster.getMaster().getAssignmentManager() + .isRegionAssigned(indexHRegion.getRegionInfo()); + } + + table = new HTable(conf, userTableName); + scanner = table.getScanner(new Scan()); + i = 0; + for (Result result : scanner) { + i++; + } + table = new HTable(conf, userTableName + "_idx"); + scanner = table.getScanner(new Scan()); + j = 0; + for (Result result : scanner) { + j++; + } + assertEquals("", i, j); + + } + + public static class LocalIndexRegionObserver extends IndexRegionObserver { + + @Override + public InternalScanner preCompactScannerOpen(ObserverContext c, + Store store, List scanners, ScanType scanType, + long earliestPutTs, InternalScanner s) throws IOException { + if (store.getTableName().equals("testHDP2989")) { + try { + compactionCalled = true; + latchForCompact.await(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + return super.preCompactScannerOpen(c, store, scanners, scanType, earliestPutTs, s); + } + + @Override + public void postOpen(ObserverContext e) { + if (e.getEnvironment().getRegion().getRegionInfo().getTableNameAsString() + .equals("testCompactionOfIndexRegionBeforeMainRegionOpens") + && openCount > 0) { + try { + latch.await(); + } catch (InterruptedException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + } + + super.postOpen(e); + } + + @Override + public void postClose(ObserverContext e, boolean abortRequested) { + if (e.getEnvironment().getRegion().getRegionInfo().getTableNameAsString() + .equals("testHDP2989_idx") + && closeCount == 0) { + try { + closeCalled = true; + latch.await(); + } catch (InterruptedException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + } + super.postClose(e, abortRequested); + } + + @Override + public void postBatchMutate(ObserverContext ctx, + List mutations, WALEdit walEdit) { + if (ctx.getEnvironment().getRegion().getRegionInfo().getTableNameAsString() + .contains("testHDP3015")) { + if (delayPostBatchMutate) { + try { + latch.await(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + super.postBatchMutate(ctx, mutations, walEdit); + } else { + super.postBatchMutate(ctx, mutations, walEdit); + } + } + + @Override + public void preFlush(ObserverContext e) throws IOException { + if (e.getEnvironment().getRegion().getRegionInfo().getTableNameAsString() + .contains("testHDP3015")) { + while (!delayPostBatchMutate) { + try { + Thread.sleep(200); + } catch (InterruptedException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + } + + super.preFlush(e); + latch.countDown(); + } else { + super.preFlush(e); + } + } + } + + public static class MoveThread extends Thread { + private HBaseAdmin admin; + byte[] regionName; + byte[] dstRSName; + + public MoveThread(HBaseAdmin admin, byte[] regionName, byte[] dstRSName) { + this.admin = admin; + this.regionName = regionName; + this.dstRSName = dstRSName; + } + + @Override + public void run() { + try { + this.admin.move(regionName, dstRSName); + } catch (UnknownRegionException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (MasterNotRunningException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ZooKeeperConnectionException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexPutsWithRegionServerRestart.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexPutsWithRegionServerRestart.java new file mode 100644 index 0000000..eb34c82 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexPutsWithRegionServerRestart.java @@ -0,0 +1,145 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.TestIndexRegionObserver.MockIndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestIndexPutsWithRegionServerRestart { + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MockIndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testShouldRetrieveIndexPutsOnRSRestart() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testPutContainingTheIndexedColumn"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "myValue".getBytes()); + table.put(p); + + // Thread.sleep(2000); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(1, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(1, i); + + HRegionServer regionServer = UTIL.getHBaseCluster().getRegionServer(0); + HMaster master = UTIL.getHBaseCluster().getMaster(); + regionServer.abort("Aborting region server"); + while (master.getServerManager().areDeadServersInProgress()) { + Thread.sleep(1000); + } + UTIL.getHBaseCluster().startRegionServer(); + ZKAssign.blockUntilNoRIT(zkw); + i = countNumberOfRows(userTableName); + Assert.assertEquals(1, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(1, i); + } + + public int countNumberOfRows(String tableName) throws IOException { + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, tableName); + Scan s = new Scan(); + int i = 0; + ResultScanner scanner = table.getScanner(s); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + i++; + result = scanner.next(1); + } + return i; + } + + public Result[] getTheLastRow(String tableName) throws IOException { + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, tableName); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + Result[] result = scanner.next(1); + Result[] result1 = result; + while (result1 != null && result1.length > 0) { + result1 = scanner.next(1); + if (null == result1 || result1.length <= 0) break; + else result = result1; + } + return result; + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexRegionObserver.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexRegionObserver.java new file mode 100644 index 0000000..211d3a5 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexRegionObserver.java @@ -0,0 +1,2059 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.client.RowMutations; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.SplitTransaction; +import org.apache.hadoop.hbase.regionserver.SplitTransaction.SplitInfo; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestIndexRegionObserver { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final int TTL_SECONDS = 2; + private static final int TTL_MS = TTL_SECONDS * 1000; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MockIndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() { + MockIndexRegionObserver.count = 0; + MockIndexRegionObserver.isnullIndexRegion = false; + } + + @Test(timeout = 180000) + public void testPutWithAndWithoutTheIndexedColumn() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testPutContainingTheIndexedColumn"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "myValue".getBytes()); + table.put(p); + + // Thread.sleep(2000); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(1, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(1, i); + + // Test put without the indexed column + Put p1 = new Put("row2".getBytes()); + p1.add("col".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p1); + + i = countNumberOfRows(userTableName); + Assert.assertEquals(2, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(1, i); + } + + public int countNumberOfRows(String tableName) throws IOException { + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, tableName); + Scan s = new Scan(); + int i = 0; + ResultScanner scanner = table.getScanner(s); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + i++; + result = scanner.next(1); + } + return i; + } + + public Result[] getTheLastRow(String tableName) throws IOException { + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, tableName); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + Result[] result = scanner.next(1); + Result[] result1 = result; + while (result1 != null && result1.length > 0) { + result1 = scanner.next(1); + if (null == result1 || result1.length <= 0) break; + else result = result1; + } + return result; + } + + @Test(timeout = 180000) + public void testPostOpenCoprocessor() throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testPostOpenCoprocessor"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + // Check the number of indices + List list = IndexManager.getInstance().getIndicesForTable(userTableName); + Assert.assertEquals(1, list.size()); + + // Check the index name + boolean bool = false; + for (IndexSpecification e : list) { + if (e.getName().equals("Index1")) bool = true; + } + Assert.assertTrue(bool); + } + + @Test(timeout = 180000) + public void testMultipleIndicesOnUniqueColumns() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testMultipleIndicesOnUniqueColumns"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec1.addIndexColumn(hcd, "ql1", ValueType.String, 10); + iSpec2.addIndexColumn(hcd, "ql2", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec1); + ihtd.addIndex(iSpec2); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql3".getBytes(), "myValue".getBytes()); + p.add("col".getBytes(), "ql4".getBytes(), "myValue".getBytes()); + table.put(p); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(1, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(0, i); + + p = new Put("row2".getBytes()); + p.add("col".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + p.add("col".getBytes(), "ql2".getBytes(), "myValue".getBytes()); + table.put(p); + i = countNumberOfRows(userTableName); + Assert.assertEquals(2, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(2, i); + + } + + @Test(timeout = 180000) + public void testIndexOnMultipleCols() throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSingleIndexOnMultipleCols"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + + // Creating and adding the column families + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + HColumnDescriptor hcd3 = new HColumnDescriptor("col3"); + ihtd.addFamily(hcd1); + ihtd.addFamily(hcd2); + ihtd.addFamily(hcd3); + + // Create and add indices + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec1.addIndexColumn(hcd1, "ql1", ValueType.String, 10); + iSpec2.addIndexColumn(hcd2, "ql1", ValueType.String, 10); + iSpec2.addIndexColumn(hcd3, "ql1", ValueType.String, 10); + ihtd.addIndex(iSpec1); + ihtd.addIndex(iSpec2); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + Put p = new Put("row1".getBytes()); + p.add("col1".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + p.add("col2".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + p.add("col3".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(1, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(2, i); + + p = new Put("row2".getBytes()); + p.add("col1".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + p.add("col2".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + i = countNumberOfRows(userTableName); + Assert.assertEquals(2, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(4, i); + + p = new Put("row3".getBytes()); + p.add("col1".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + p.add("col3".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + i = countNumberOfRows(userTableName); + Assert.assertEquals(3, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(6, i); + + p = new Put("row4".getBytes()); + p.add("col2".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + p.add("col3".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + i = countNumberOfRows(userTableName); + Assert.assertEquals(4, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(7, i); + } + + @Test(timeout = 180000) + public void testPutsWithPadding() throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testPutsWithPadding"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + + // Creating and adding the column families + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + HColumnDescriptor hcd3 = new HColumnDescriptor("col3"); + HColumnDescriptor hcd4 = new HColumnDescriptor("col4"); + + ihtd.addFamily(hcd2); + ihtd.addFamily(hcd3); + ihtd.addFamily(hcd4); + + // Create and add indices + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec2.addIndexColumn(hcd2, "ql1", ValueType.String, 10); + iSpec2.addIndexColumn(hcd3, "ql1", ValueType.String, 10); + iSpec2.addIndexColumn(hcd4, "ql1", ValueType.String, 10); + ihtd.addIndex(iSpec2); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + Put p = new Put("row1".getBytes()); + p.add("col2".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(1, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(1, i); + + Result[] result = getTheLastRow(userTableName + Constants.INDEX_TABLE_SUFFIX); + byte[] rowKey1 = result[0].getRow(); + + p = new Put("row2".getBytes()); + p.add("col3".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + i = countNumberOfRows(userTableName); + Assert.assertEquals(2, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(2, i); + + result = getTheLastRow(userTableName + Constants.INDEX_TABLE_SUFFIX); + byte[] rowKey2 = result[0].getRow(); + + Assert.assertEquals(rowKey1.length, rowKey2.length); + + p = new Put("row3".getBytes()); + p.add("col3".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + table.put(p); + i = countNumberOfRows(userTableName); + Assert.assertEquals(3, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(3, i); + + result = getTheLastRow(userTableName + Constants.INDEX_TABLE_SUFFIX); + byte[] rowKey3 = result[0].getRow(); + Assert.assertEquals(rowKey2.length, rowKey3.length); + + /* + * p = new Put("row4".getBytes()); p.add("col3".getBytes(), "ql1".getBytes(), + * "myValuefgacfgn".getBytes()); p.add("col2".getBytes(), "ql1".getBytes(), + * "myValue".getBytes()); p.add("col4".getBytes(), "ql1".getBytes(), "myValue".getBytes()); + * table.put(p); i = countNumberOfRows(userTableName); Assert.assertEquals(4, i); i = + * countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); Assert.assertEquals(4, i); + * result = getTheLastRow(userTableName + Constants.INDEX_TABLE_SUFFIX); byte[] rowKey4 = + * result[0].getRow(); Assert.assertEquals(rowKey3.length, rowKey4.length); + */ + } + + @Test(timeout = 180000) + public void testBulkPut() throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testBulkPut"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + + // Creating and adding the column families + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + HColumnDescriptor hcd3 = new HColumnDescriptor("col3"); + ihtd.addFamily(hcd1); + ihtd.addFamily(hcd2); + ihtd.addFamily(hcd3); + + // Create and add indices + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + IndexSpecification iSpec2 = new IndexSpecification("Index2"); + iSpec1.addIndexColumn(hcd1, "ql1", ValueType.String, 10); + iSpec2.addIndexColumn(hcd3, "ql1", ValueType.String, 10); + ihtd.addIndex(iSpec1); + ihtd.addIndex(iSpec2); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + Thread[] t = new Thread[10]; + for (int i = 0; i < 10; i++) { + t[i] = new Testthread(conf, userTableName); + } + for (int i = 0; i < 10; i++) { + t[i].start(); + } + + for (int i = 0; i < 10; i++) { + t[i].join(); + } + + // System.out.println("Woke up"); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(5000, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(10000, i); + /* + * HLog log = UTIL.getHBaseCluster().getRegionServer(0).getWAL(); log.getHLogDirectoryName + * (UTIL.getHBaseCluster().getRegionServer(0).toString()); log.getReader( + * UTIL.getMiniHBaseCluster().getRegionServer(0).getFileSystem(), path, + * UTIL.getMiniHBaseCluster().getConfiguration()); + */ + } + + class Testthread extends Thread { + Configuration conf; + String userTableName; + HTable table; + + public Testthread(Configuration conf, String userTableName) throws IOException { + this.conf = conf; + this.userTableName = userTableName; + this.table = new HTable(conf, userTableName); + } + + public void run() { + for (int j = 0; j < 500; j++) { + double d = Math.random(); + byte[] rowKey = Bytes.toBytes(d); + Put p = new Put(rowKey); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + p.add(Bytes.toBytes("col2"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + p.add(Bytes.toBytes("col3"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + try { + table.put(p); + } catch (IOException e) { + } + } + } + } + + @Test(timeout = 180000) + public void testBulkPutWithRepeatedRows() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + final Configuration conf = UTIL.getConfiguration(); + final String userTableName = "TestBulkPutWithRepeatedRows"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + + // Creating and adding the column families + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + + ihtd.addFamily(hcd1); + + // Create and add indices + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + + iSpec1.addIndexColumn(hcd1, "ql1", ValueType.String, 10); + + ihtd.addIndex(iSpec1); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + new Thread() { + @Override + public void run() { + try { + HTable table = new HTable(conf, userTableName); + List puts = new ArrayList(5); + Put p = new Put("row1".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row2".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row3".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row4".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row5".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + table.put(puts); + } catch (IOException e) { + } + } + }.start(); + + new Thread() { + @Override + public void run() { + try { + HTable table = new HTable(conf, userTableName); + List puts = new ArrayList(5); + Put p = new Put("row6".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row7".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row3".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row4".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + p = new Put("row10".getBytes()); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql1"), Bytes.toBytes("myValue")); + puts.add(p); + table.put(puts); + } catch (IOException e) { + } + } + }.start(); + Thread.sleep(2000); + int i = countNumberOfRows(userTableName); + Assert.assertEquals(8, i); + i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); + Assert.assertEquals(8, i); + + } + + @Test(timeout = 180000) + public void testIndexPutRowkeyWithAllTheValues() throws IOException { + String DIR = UTIL.getDataTestDir("TestStore").toString(); + Path basedir = new Path(DIR + "TestIndexPut"); + // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); + FileSystem fs = UTIL.getTestFileSystem(); + Configuration conf = UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("TestIndexPut"); + HRegionInfo info = new HRegionInfo(htd.getName(), "A".getBytes(), "B".getBytes(), false); + HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); + IndexSpecification spec = new IndexSpecification("testSpec"); + spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); + spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); + + // Scenario where both the indexed cols are there in the put + byte[] rowKey = "Arow1".getBytes(); + Put p = new Put(rowKey); + long time = 1234567; + p.add("cf1".getBytes(), "ql1".getBytes(), time, "testvalue1".getBytes()); + p.add("cf2".getBytes(), "ql1".getBytes(), time + 10, "testvalue1".getBytes()); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + Assert.assertEquals(region.getStartKey().length + 1 + Constants.DEF_MAX_INDEX_NAME_LENGTH + 2 + * 10 + rowKey.length, indexPut.getRow().length); + Assert.assertEquals(time + 10, indexPut.get(Constants.IDX_COL_FAMILY, "".getBytes()).get(0) + .getTimestamp()); + + } + + @Test(timeout = 180000) + public void testIndexPutWithOnlyOneValue() throws IOException { + String DIR = UTIL.getDataTestDir("TestStore").toString(); + Path basedir = new Path(DIR + "TestIndexPut"); + // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); + FileSystem fs = UTIL.getTestFileSystem(); + Configuration conf = UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("TestIndexPut"); + HRegionInfo info = new HRegionInfo(htd.getName(), "A".getBytes(), "B".getBytes(), false); + HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); + IndexSpecification spec = new IndexSpecification("testSpec"); + spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); + spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); + + byte[] rowKey = "Arow1".getBytes(); + Put p = new Put(rowKey); + long time = 1234567; + p.add("cf1".getBytes(), "ql1".getBytes(), time, "testvalue1".getBytes()); + Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); + Assert.assertEquals(region.getStartKey().length + 1 + Constants.DEF_MAX_INDEX_NAME_LENGTH + 2 + * 10 + rowKey.length, indexPut.getRow().length); + Assert.assertEquals(time, indexPut.get(Constants.IDX_COL_FAMILY, "".getBytes()).get(0) + .getTimestamp()); + // asserting pad........this has to be hardcoded. + byte[] pad = new byte[10]; + System.arraycopy(indexPut.getRow(), region.getStartKey().length + 1 + + Constants.DEF_MAX_INDEX_NAME_LENGTH + 10, pad, 0, 10); + Assert.assertTrue(Bytes.equals(pad, new byte[10])); + } + + @Test(timeout = 180000) + public void testIndexPutWithValueGreaterThanLength() throws IOException { + String DIR = UTIL.getDataTestDir("TestStore").toString(); + Path basedir = new Path(DIR + "TestIndexPut"); + // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); + FileSystem fs = UTIL.getTestFileSystem(); + Configuration conf = UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("TestIndexPut"); + HRegionInfo info = new HRegionInfo(htd.getName(), "A".getBytes(), "B".getBytes(), false); + HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); + IndexSpecification spec = new IndexSpecification("testSpec"); + spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); + spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); + + // assert IOException when value length goes beyond the limit. + byte[] rowKey = "Arow1".getBytes(); + Put p = new Put(rowKey); + long time = 1234567; + boolean returnVal = false; + try { + p.add("cf1".getBytes(), "ql1".getBytes(), time, "testvalue11".getBytes()); + IndexUtils.prepareIndexPut(p, spec, region); + } catch (IOException e) { + returnVal = true; + } + Assert.assertTrue(returnVal); + } + + @Test(timeout = 180000) + public void testIndexPutSequence() throws IOException { + String DIR = UTIL.getDataTestDir("TestStore").toString(); + Path basedir = new Path(DIR + "TestIndexPut"); + // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); + FileSystem fs = UTIL.getTestFileSystem(); + Configuration conf = UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("TestIndexPut"); + HRegionInfo info = new HRegionInfo(htd.getName(), "A".getBytes(), "B".getBytes(), false); + HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); + spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); + + // scenario where same indexName but diff colvalues can disturb the order if + // used without pad + byte[] rowKey = "Arow1".getBytes(); + Put p1 = new Put(rowKey); + long time = 1234567; + p1.add("cf1".getBytes(), "ql1".getBytes(), time, "testcase".getBytes()); + p1.add("cf2".getBytes(), "ql1".getBytes(), time, "value".getBytes()); + Put indexPut1 = IndexUtils.prepareIndexPut(p1, spec, region); + Put p2 = new Put(rowKey); + p2.add("cf1".getBytes(), "ql1".getBytes(), time, "test".getBytes()); + p2.add("cf2".getBytes(), "ql1".getBytes(), time, "value".getBytes()); + Put indexPut2 = IndexUtils.prepareIndexPut(p2, spec, region); + // (spaces just for easier reading...not present in actual) + // Index Row key For p1 = "A index testcase value Arow1" + // Index Row key For p2 = "A index test value Arow1" + // NOW acc. to the lexographical ordering p2 should come second but we need + // it to come 1st datz where pad is needed. + byte[] rowKey1 = indexPut1.getRow(); + byte[] rowKey2 = indexPut2.getRow(); + + int result = Bytes.compareTo(rowKey1, rowKey2); + Assert.assertTrue(result > 0); + + // scenario where the index names are diff and padding is needed. + IndexSpecification spec1 = new IndexSpecification("ind"); + spec1.addIndexColumn(new HColumnDescriptor("cf3"), "ql1", ValueType.String, 10); + p1 = new Put(rowKey); + p1.add("cf3".getBytes(), "ql1".getBytes(), time, "testcase".getBytes()); + indexPut1 = IndexUtils.prepareIndexPut(p1, spec1, region); + // (spaces just for easier reading...not present in actual) + // Index Row key For p1 = "A ind testcase value Arow1" + // Index Row key For p2 = "A index test value Arow1" + // NOW acc. to the lexographical ordering p1 should come second but we need + // it to come 1st datz where pad is needed. + rowKey1 = indexPut1.getRow(); + result = Bytes.compareTo(rowKey1, rowKey2); + Assert.assertTrue(result < 0); + + } + + @Test(timeout = 180000) + public void testIndexTableValue() throws IOException { + String DIR = UTIL.getDataTestDir("TestStore").toString(); + Path basedir = new Path(DIR + "TestIndexPut"); + // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); + FileSystem fs = UTIL.getTestFileSystem(); + Configuration conf = UTIL.getConfiguration(); + HTableDescriptor htd = new HTableDescriptor("TestIndexPut"); + HRegionInfo info = new HRegionInfo(htd.getName(), "ABC".getBytes(), "BBB".getBytes(), false); + HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); + HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); + IndexSpecification spec = new IndexSpecification("index"); + spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); + spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); + + byte[] rowKey = "Arow1".getBytes(); + Put p1 = new Put(rowKey); + long time = 1234567; + p1.add("cf1".getBytes(), "ql1".getBytes(), time, "testcase".getBytes()); + p1.add("cf2".getBytes(), "ql1".getBytes(), time, "value".getBytes()); + Put indexPut1 = IndexUtils.prepareIndexPut(p1, spec, region); + + List kvs = indexPut1.get(Constants.IDX_COL_FAMILY, "".getBytes()); + KeyValue kv = null; + if (null != kvs) { + kv = kvs.get(0); + } + byte[] val = kv.getValue(); + byte[] startKeyLengthInBytes = new byte[2]; + System.arraycopy(val, 0, startKeyLengthInBytes, 0, startKeyLengthInBytes.length); + int startkeylen = (int) (Bytes.toShort(startKeyLengthInBytes)); + Assert.assertEquals(3, startkeylen); + + byte[] rowKeyOffset = new byte[2]; + System.arraycopy(val, startKeyLengthInBytes.length, rowKeyOffset, 0, rowKeyOffset.length); + int rowKeyOffsetInt = Bytes.toShort(rowKeyOffset); + Assert.assertEquals(42, rowKeyOffsetInt); + } + + @Test(timeout = 180000) + public void testIndexedRegionAfterSplitShouldReplaceStartKeyAndValue() throws IOException, + KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setInt("hbase.regionserver.lease.period", 90000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testIndexRegionSplit"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(new HColumnDescriptor("col"), "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + hRegionInfo.getRegionName(); + HTable tableidx = new HTable(conf, userTableName + "_idx"); + Scan s = new Scan(); + int i = 0; + ResultScanner scanner = table.getScanner(s); + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + i++; + } + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + Scan s1 = new Scan(); + int i1 = 0; + String indexTable = userTableName + "_idx"; + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + ResultScanner scanner1 = tableidx.getScanner(s1); + for (Result rr = scanner1.next(); rr != null; rr = scanner1.next()) { + i1++; + } + Assert.assertEquals("count should be equal", i, i1); + } + + @Test(timeout = 180000) + public void + testIndexedRegionAfterSplitShouldNotThrowExceptionIfThereAreNoSplitFilesForIndexedTable() + throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setInt("hbase.regionserver.lease.period", 90000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testIndexRegionSplit1"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addFamily(hcd2); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col2".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + HTable tableidx = new HTable(conf, userTableName + "_idx"); + Scan s = new Scan(); + int i = 0; + ResultScanner scanner = table.getScanner(s); + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + i++; + } + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + Scan s1 = new Scan(); + int i1 = 0; + String indexTable = userTableName + "_idx"; + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + Assert.assertEquals("Main table count shud be 10", 10, i); + ResultScanner scanner1 = tableidx.getScanner(s1); + for (Result rr = scanner1.next(); rr != null; rr = scanner1.next()) { + i1++; + } + Assert.assertEquals("Index table count shud be 0", 0, i1); + + // Trying to put data for indexed col + for (int k = 10; k < 20; k++) { + String row = "row" + k; + Put p = new Put(row.getBytes()); + String val = "Val" + k; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + int z = 0; + Scan s2 = new Scan(); + ResultScanner scanner2 = tableidx.getScanner(s2); + + admin.flush(userTableName + "_idx"); + + for (Result rr = scanner2.next(); rr != null; rr = scanner2.next()) { + z++; + } + Assert.assertEquals("Index table count shud be now 10", 10, z); + + List regionsOfTable1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + HRegionInfo hRegionInfo1 = regionsOfTable1.get(0); + + admin.split(hRegionInfo1.getRegionName(), "row3".getBytes()); + List mainTableRegions1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + + List indexTableRegions1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + while (mainTableRegions1.size() != 3) { + Thread.sleep(2000); + mainTableRegions1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + } + while (indexTableRegions1.size() != 3) { + Thread.sleep(2000); + indexTableRegions1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + } + // It shud be 3 after one more split + Assert.assertEquals(3, mainTableRegions1.size()); + Assert.assertEquals(3, indexTableRegions1.size()); + Scan s3 = new Scan(); + int a = 0; + ResultScanner scanner3 = tableidx.getScanner(s3); + for (Result rr = scanner3.next(); rr != null; rr = scanner3.next()) { + a++; + } + int b = 0; + Scan s4 = new Scan(); + ResultScanner scanner4 = table.getScanner(s4); + for (Result rr = scanner4.next(); rr != null; rr = scanner4.next()) { + b++; + } + // subracting 10 coz addtional 10 is for other col family which is not indexed + Assert.assertEquals("count should be equal", a, b - 10); + + } + + @Test(timeout = 180000) + public void testSeekToInIndexHalfStoreFileReaderShouldRetreiveClosestRowCorrectly() + throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + conf.setBoolean("hbase.use.secondary.index", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testIndexSplitWithClosestRow"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(new HColumnDescriptor("col"), "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i <= 10; i++) { + // Don't put row4 alone so that when getRowOrBefore for row4 is specifiesd row3 shud be + // returned + if (i != 4) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + hRegionInfo.getRegionName(); + HTable tableidx = new HTable(conf, userTableName + "_idx"); + admin.split(hRegionInfo.getRegionName(), "row6".getBytes()); + String indexTable = userTableName + "_idx"; + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + Scan s1 = new Scan(); + ResultScanner scanner1 = tableidx.getScanner(s1); + Result next = null; + for (int i = 0; i < 6; i++) { + next = scanner1.next(); + } + String row1 = "row4"; + Put row4Put = new Put(row1.getBytes()); + String val = "Val4"; + row4Put.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + + byte[] row4 = row4Put.getRow(); + byte[] nextRow = next.getRow(); + byte[] replacedRow = new byte[nextRow.length]; + System.arraycopy(nextRow, 0, replacedRow, 0, nextRow.length); + System.arraycopy(row4, 0, replacedRow, replacedRow.length - row4.length, row4.length); + Result rowOrBefore = tableidx.getRowOrBefore(replacedRow, "d".getBytes()); + + String expectedRow = "row3"; + Put p1 = new Put(expectedRow.getBytes()); + String actualStr = Bytes.toString(rowOrBefore.getRow()); + int lastIndexOf = actualStr.lastIndexOf("row3"); + Assert.assertEquals("SeekTo should return row3 as closestRowBefore", + Bytes.toString(p1.getRow()), actualStr.substring(lastIndexOf, actualStr.length())); + } + + @Test(timeout = 180000) + public + void + testSeekToInIndexHalfStoreFileReaderShouldRetreiveClosestRowCorrectlyWhenRowIsNotFoundInMainTable() + throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testIndexSplitWithClosestRowNotInMainTable"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(new HColumnDescriptor("col"), "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i <= 10; i++) { + // Don't put row8 alone so that when getRowOrBefore for row8 is specifiesd row7 shud be + // returned + if (i != 8) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + hRegionInfo.getRegionName(); + HTable tableidx = new HTable(conf, userTableName + "_idx"); + admin.split(hRegionInfo.getRegionName(), "row4".getBytes()); + + String indexTable = userTableName + "_idx"; + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + + Scan s1 = new Scan(); + ResultScanner scanner1 = tableidx.getScanner(s1); + Result next1 = null; + Thread.sleep(3000); + for (int i = 0; i < 10; i++) { + next1 = scanner1.next(); + } + String row1 = "row8"; + Put row4Put = new Put(row1.getBytes()); + String val = "Val8"; + row4Put.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + byte[] row4 = row4Put.getRow(); + byte[] nextRow = next1.getRow(); + byte[] replacedRow = new byte[nextRow.length]; + System.arraycopy(nextRow, 0, replacedRow, 0, nextRow.length); + System.arraycopy(row4, 0, replacedRow, replacedRow.length - row4.length, row4.length); + Result rowOrBefore = tableidx.getRowOrBefore(replacedRow, "d".getBytes()); + + String expectedRow = "row7"; + Put p1 = new Put(expectedRow.getBytes()); + String actualStr = Bytes.toString(rowOrBefore.getRow()); + int lastIndexOf = actualStr.lastIndexOf("row7"); + Assert.assertTrue("Expected row should have the start key replaced to split key ", Bytes + .toString(rowOrBefore.getRow()).startsWith("row4")); + Assert.assertEquals("SeekTo should return row7 as closestRowBefore and split is completed ", + Bytes.toString(p1.getRow()), actualStr.substring(lastIndexOf, actualStr.length())); + } + + @Test(timeout = 180000) + public void testPutWithValueLengthMoreThanMaxValueLength() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testPutWithValueLengthMoreThanMaxValueLength"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec1 = new IndexSpecification("Index1"); + iSpec1.addIndexColumn(hcd, "ql1", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec1); + admin.createTable(ihtd); + + HTable table = new HTable(conf, userTableName); + table.setAutoFlush(false); + List putList = new ArrayList(3); + putList.add(new Put("row1".getBytes()).add("col".getBytes(), "ql1".getBytes(), + "valueLengthMoreThanMaxValueLength".getBytes())); + putList.add(new Put("row2".getBytes()).add("col".getBytes(), "ql1".getBytes(), + "myValue".getBytes())); + putList.add(new Put("row3".getBytes()).add("col".getBytes(), "ql1".getBytes(), + "myValue".getBytes())); + table.put(putList); + try { + table.flushCommits(); + } catch (RetriesExhaustedWithDetailsException e) { + // nothing to do. + } + Assert.assertEquals(1, table.getWriteBuffer().size()); + } + + @Test(timeout = 180000) + public void testIfPrepareFailsFor2ndSplitShouldFailTheMainTableSplitAlso() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 90000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testPrepareFailForIdx"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + MockIndexRegionObserver.count++; + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + String indexTable = userTableName + "_idx"; + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + + MockIndexRegionObserver.count++; + List regionsOfTable2 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + HRegionInfo hRegionInfo1 = regionsOfTable2.get(0); + admin.split(hRegionInfo1.getRegionName(), "row3".getBytes()); + List mainTableRegions1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + List indexTableRegions1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + Assert.assertEquals(2, mainTableRegions1.size()); + Assert.assertEquals(2, indexTableRegions1.size()); + } + + public static class MockIndexRegionObserver extends IndexRegionObserver { + static int count = 0; + static boolean isnullIndexRegion = false; + + public SplitInfo preSplitBeforePONR(ObserverContext e, + byte[] splitKey) throws IOException { + if (e.getEnvironment().getRegion().getRegionInfo().getTableNameAsString() + .equals("testIndexManagerWithFailedSplitOfIndexRegion")) { + throw new IOException(); + } + if (isnullIndexRegion) { + return null; + } + if (count == 2) { + splitThreadLocal.remove(); + return null; + } else { + return super.preSplitBeforePONR(e, splitKey); + } + } + + @Override + public void preSplit(ObserverContext e) throws IOException { + super.preSplit(e); + } + } + + @Test(timeout = 180000) + public void testShouldNotSplitIfIndexRegionIsNullForIndexTable() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + conf.setInt("hbase.regionserver.lease.period", 90000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testIndexRegionSplit12"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + MockIndexRegionObserver.isnullIndexRegion = true; + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + String indexTable = userTableName + "_idx"; + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + Assert.assertEquals(1, mainTableRegions.size()); + Assert.assertEquals(1, indexTableRegions.size()); + } + + @Test(timeout = 180000) + public void testCheckAndPutFor1PutShouldHav2PutsInIndexTableAndShouldReplaceWithNewValue() + throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCheckAndPutContainingTheIndexedColumn"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + String idxTableName = userTableName + Constants.INDEX_TABLE_SUFFIX; + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "q1".getBytes(), "myValue".getBytes()); + table.put(p); + + int usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + int idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(1, idxtableCount); + + // Test check and put + Put p1 = new Put("row1".getBytes()); + p1.add("col".getBytes(), "q1".getBytes(), "myNewValue".getBytes()); + Assert.assertTrue(table.checkAndPut("row1".getBytes(), "col".getBytes(), "q1".getBytes(), + "myValue".getBytes(), p1)); + usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(2, idxtableCount); + + Get get = new Get("row1".getBytes()); + get.addColumn(Bytes.toBytes("col"), Bytes.toBytes("q1")); + Result result = table.get(get); + byte[] val = result.getValue(Bytes.toBytes("col"), Bytes.toBytes("q1")); + Assert.assertEquals("myNewValue", Bytes.toString(val)); + } + + @Test(timeout = 180000) + public void testCheckAndPutAndNormalPutInParallel() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCheckAndPutAndNormalPutInParallel"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + String idxTableName = userTableName + Constants.INDEX_TABLE_SUFFIX; + final HTable table = new HTable(conf, userTableName); + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "q1".getBytes(), "myValue".getBytes()); + table.put(p); + + int usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + int idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(1, idxtableCount); + + // Test check and put + Put p1 = new Put("row1".getBytes()); + p1.add("col".getBytes(), "q1".getBytes(), "myNewValue".getBytes()); + Assert.assertTrue(table.checkAndPut("row1".getBytes(), "col".getBytes(), "q1".getBytes(), + "myValue".getBytes(), p1)); + new Thread() { + public void run() { + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "q1".getBytes(), "myValue1".getBytes()); + try { + table.put(p); + } catch (IOException e) { + e.printStackTrace(); + } + } + }.start(); + Thread.sleep(3000); + usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(3, idxtableCount); + + } + + @Test(timeout = 180000) + public void testCheckAndDeleteShudDeleteTheRowSuccessfullyInBothIndexAndMainTable() + throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCheckAndDeleteContainingTheIndexedColumn"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + String idxTableName = userTableName + Constants.INDEX_TABLE_SUFFIX; + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "q1".getBytes(), "myValue".getBytes()); + table.put(p); + + int usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + int idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(1, idxtableCount); + + Delete delete = new Delete("row1".getBytes()); + delete.deleteFamily("col".getBytes()); + // CheckandDelete + Assert.assertTrue(table.checkAndDelete("row1".getBytes(), "col".getBytes(), "q1".getBytes(), + "myValue".getBytes(), delete)); + + Get get = new Get("row1".getBytes()); + get.addFamily("col".getBytes()); + Result r = table.get(get); + Assert.assertEquals(0, r.size()); + + usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(0, usertableCount); + idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(0, idxtableCount); + } + + @Test(timeout = 180000) + public void testRowMutations() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testMutateRows"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec.addIndexColumn(hcd, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + String idxTableName = userTableName + Constants.INDEX_TABLE_SUFFIX; + HTable table = new HTable(conf, userTableName); + HTable idxtable = new HTable(conf, idxTableName); + byte[] row = Bytes.toBytes("rowA"); + + Put put = new Put(row); + put.add("col".getBytes(), "q2".getBytes(), "delValue".getBytes()); + table.put(put); + + int usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + int idxtableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(1, idxtableCount); + + RowMutations rm = new RowMutations(row); + Put p = new Put(row); + p.add("col".getBytes(), "q1".getBytes(), "q1value".getBytes()); + rm.add(p); + Delete d = new Delete(row); + d.deleteColumns("col".getBytes(), "q2".getBytes()); + rm.add(d); + + // Mutate rows + table.mutateRow(rm); + + admin.flush(userTableName); + admin.majorCompact(userTableName); + // Now after one put and one delete it shud be 1 + usertableCount = countNumberOfRows(userTableName); + Assert.assertEquals(1, usertableCount); + + int idxTableCount = countNumberOfRows(idxTableName); + Assert.assertEquals(1, idxTableCount); + + Get get = new Get(row); + get.addFamily("col".getBytes()); + Result r = table.get(get); + Assert.assertEquals(1, r.size()); + + String del = Bytes.toString(r.getValue("col".getBytes(), "q2".getBytes())); + Assert.assertNull(del); + String putval = Bytes.toString(r.getValue("col".getBytes(), "q1".getBytes())); + Assert.assertEquals("q1value", putval); + + // Result of index table shud contain one keyvalue for "q1value" put only + Scan s = new Scan(); + ResultScanner scanner = idxtable.getScanner(s); + Result result = scanner.next(); + Assert.assertEquals(1, result.size()); + + String idxRow = Bytes.toString(result.getRow()); + int len = IndexUtils.getMaxIndexNameLength() + "q1value".length(); + String value = idxRow.substring(IndexUtils.getMaxIndexNameLength() + 1, len + 1); + // asserting value in idx table is for the put which has the value "q1value" + Assert.assertEquals("q1value", value); + } + + @Test(timeout = 180000) + public void testWithPutsAndDeletesAfterSplitShouldRetreiveTheRowsCorrectly() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + conf.setBoolean("hbase.use.secondary.index", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testPutsAndDeletes"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(new HColumnDescriptor("col"), "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i <= 5; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + + if (i < 3) { + Delete d = new Delete(row.getBytes()); + // Do a normal delete + table.delete(d); + } else { + if (i > 4) { + Delete d = new Delete(row.getBytes()); + // Do column family delete + d.deleteFamily("col".getBytes()); + table.delete(d); + } else { + Delete d = new Delete(row.getBytes()); + // Do delete column + d.deleteColumn("col".getBytes(), "ql".getBytes()); + table.delete(d); + } + } + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + hRegionInfo.getRegionName(); + HTable tableidx = new HTable(conf, userTableName + "_idx"); + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + String indexTable = userTableName + "_idx"; + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + int mainTableCount = 0; + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + mainTableCount++; + } + Assert.assertEquals(0, mainTableCount); + Scan s1 = new Scan(); + ResultScanner scanner1 = tableidx.getScanner(s1); + + int indexTableCount = 0; + for (Result rr = scanner1.next(); rr != null; rr = scanner1.next()) { + indexTableCount++; + } + Assert.assertEquals(0, indexTableCount); + + Put p = new Put("row7".getBytes()); + String val = "Val" + "7"; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + Scan s2 = new Scan(); + ResultScanner scanner2 = table.getScanner(s2); + for (Result rr = scanner2.next(); rr != null; rr = scanner2.next()) { + mainTableCount++; + } + Scan s3 = new Scan(); + ResultScanner scanner3 = tableidx.getScanner(s3); + for (Result rr = scanner3.next(); rr != null; rr = scanner3.next()) { + indexTableCount++; + } + Assert.assertEquals(1, mainTableCount); + Assert.assertEquals(1, indexTableCount); + + } + + @Test(timeout = 180000) + public void test6PutsAnd3DeletesAfterSplitShouldRetreiveRowsSuccessfully() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + conf.setBoolean("hbase.use.secondary.index", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "test6Puts3Deletes"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(new HColumnDescriptor("col"), "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + for (int i = 0; i <= 5; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + if (i < 3) { + Delete d = new Delete(row.getBytes()); + // Do a normal delete + table.delete(d); + } + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + hRegionInfo.getRegionName(); + + HTable tableidx = new HTable(conf, userTableName + "_idx"); + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + String indexTable = userTableName + "_idx"; + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + + List indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + } + while (indexTableRegions.size() != 2) { + Thread.sleep(2000); + indexTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTable.getBytes()); + } + Assert.assertEquals(2, mainTableRegions.size()); + Assert.assertEquals(2, indexTableRegions.size()); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + int mainTableCount = 0; + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + mainTableCount++; + } + Assert.assertEquals(3, mainTableCount); + Scan s1 = new Scan(); + ResultScanner scanner1 = tableidx.getScanner(s1); + + int indexTableCount = 0; + for (Result rr = scanner1.next(); rr != null; rr = scanner1.next()) { + indexTableCount++; + } + Assert.assertEquals(3, indexTableCount); + + } + + @Test(timeout = 180000) + public void testSplitForMainTableWithoutIndexShouldBeSuccessfUl() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + conf.setBoolean("hbase.use.secondary.index", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "test6Puts3Deletes34534"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + ihtd.addFamily(hcd); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + HRegionInfo hRegionInfo = regionsOfTable.get(0); + hRegionInfo.getRegionName(); + admin.split(hRegionInfo.getRegionName(), "row5".getBytes()); + + List mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + + while (mainTableRegions.size() != 2) { + Thread.sleep(2000); + mainTableRegions = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + } + for (int i = 20; i < 30; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + int mainTableCount = 0; + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + mainTableCount++; + } + Assert.assertEquals(20, mainTableCount); + + } + + @Test(timeout = 180000) + public void testSplittingIndexRegionExplicitly() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + conf.setBoolean("hbase.use.secondary.index", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testSplitTransaction"; + String indexTableName = "testSplitTransaction_idx"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(new HColumnDescriptor("col"), "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + + for (int i = 0; i < 10; i++) { + String row = "row" + i; + Put p = new Put(row.getBytes()); + String val = "Val" + i; + p.add("col".getBytes(), "ql".getBytes(), val.getBytes()); + table.put(p); + } + + List regionsOfUserTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + + List regionsOfIndexTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTableName.getBytes()); + + // try splitting index. + admin.split(indexTableName.getBytes()); + Thread.sleep(2000); + regionsOfIndexTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTableName.getBytes()); + Assert.assertEquals("Index table should not get splited", 1, regionsOfIndexTable.size()); + + // try splitting the user region. + admin.split(userTableName.getBytes(), "row5".getBytes()); + while (regionsOfUserTable.size() != 2) { + Thread.sleep(2000); + regionsOfUserTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + } + while (regionsOfIndexTable.size() != 2) { + Thread.sleep(2000); + regionsOfIndexTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(indexTableName.getBytes()); + } + Assert.assertEquals(2, regionsOfUserTable.size()); + Assert.assertEquals(2, regionsOfIndexTable.size()); + } + + @Test(timeout = 180000) + public void testIndexManagerCleanUp() throws Exception { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + Configuration conf = admin.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testIndexManagerCleanUp"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col1"); + ihtd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addIndex(iSpec); + + byte[][] splits = new byte[10][]; + char c = 'A'; + for (int i = 0; i < 10; i++) { + byte[] b = { (byte) c }; + splits[i] = b; + c++; + } + admin.createTable(ihtd, splits); + ZKAssign.blockUntilNoRIT(zkw); + IndexManager instance = IndexManager.getInstance(); + int regionCount = instance.getTableRegionCount(userTableName); + Assert.assertEquals(11, regionCount); + + admin.disableTable(Bytes.toBytes(userTableName)); + ZKAssign.blockUntilNoRIT(zkw); + regionCount = instance.getTableRegionCount(userTableName); + Assert.assertEquals(0, regionCount); + + admin.enableTable(userTableName); + ZKAssign.blockUntilNoRIT(zkw); + regionCount = instance.getTableRegionCount(userTableName); + Assert.assertEquals(11, regionCount); + } + + @Test(timeout = 180000) + public void testHDP2938() throws Exception { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testHDP2938"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + + HColumnDescriptor hcd1 = new HColumnDescriptor("col1").setMaxVersions(Integer.MAX_VALUE); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd1, "q2", ValueType.String, 10); + ihtd.addFamily(hcd1); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + admin.disableTable(userTableName); + admin.deleteTable(userTableName); + + ihtd = new IndexedHTableDescriptor(userTableName); + hcd1 = + new HColumnDescriptor("col1").setMaxVersions(Integer.MAX_VALUE).setTimeToLive( + TTL_SECONDS - 1); + iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd1, "q2", ValueType.String, 10); + ihtd.addFamily(hcd1); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + + Put p = new Put("row1".getBytes()); + p.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p); + admin.flush(userTableName + "_idx"); + + Put p1 = new Put("row01".getBytes()); + p1.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p1); + admin.flush(userTableName + "_idx"); + + Put p2 = new Put("row010".getBytes()); + p2.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p2); + admin.flush(userTableName + "_idx"); + + Put p3 = new Put("row001".getBytes()); + p3.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p3); + + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + + HRegionServer regionServer = UTIL.getHBaseCluster().getRegionServer(0); + List onlineRegions = regionServer.getOnlineRegions(Bytes.toBytes(userTableName)); + List storeFileList = + regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + onlineRegions = regionServer.getOnlineRegions(Bytes.toBytes(userTableName + "_idx")); + storeFileList = regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + while (storeFileList.size() < 4) { + Thread.sleep(1000); + storeFileList = regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + } + int prevSize = storeFileList.size(); + Assert.assertEquals("The total store files for the index table should be 4", 4, prevSize); + Scan s = new Scan(); + HTable indexTable = new HTable(conf, userTableName + "_idx"); + ResultScanner scanner = indexTable.getScanner(s); + // Result res = scanner.next(); + for (Result result : scanner) { + System.out.println(result); + } + for (String store : storeFileList) { + Threads.sleepWithoutInterrupt(TTL_MS); + } + admin.compact(userTableName + "_idx"); + + onlineRegions = regionServer.getOnlineRegions(Bytes.toBytes(userTableName + "_idx")); + storeFileList = regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + while (storeFileList.size() != 1) { + Thread.sleep(1000); + storeFileList = regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + } + Assert.assertEquals("The total store files for the index table should be 1", 1, + storeFileList.size()); + s = new Scan(); + indexTable = new HTable(conf, userTableName + "_idx"); + scanner = indexTable.getScanner(s); + // Result res = scanner.next(); + boolean dataAvailable = false; + for (Result result : scanner) { + dataAvailable = true; + System.out.println(result); + } + Assert.assertFalse("dataShould not be retrieved", dataAvailable); + } + + @Test(timeout = 180000) + public void testIndexManagerWithSplitTransactions() throws Exception { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + Configuration conf = admin.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testIndexManagerWithSplitTransactions"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col1"); + ihtd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + IndexManager manager = IndexManager.getInstance(); + int count = manager.getTableRegionCount(userTableName); + Assert.assertEquals(1, count); + + HTable table = new HTable(conf, userTableName); + Put p = null; + for (int i = 0; i < 10; i++) { + p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql"), Bytes.toBytes("test_val")); + table.put(p); + } + + admin.split(userTableName, "row5"); + ZKAssign.blockUntilNoRIT(zkw); + + count = manager.getTableRegionCount(userTableName); + Assert.assertEquals(2, count); + } + + @Test(timeout = 180000) + public void testIndexManagerWithFailedSplitOfIndexRegion() throws Exception { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + Configuration conf = admin.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testIndexManagerWithFailedSplitOfIndexRegion"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col1"); + ihtd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + IndexManager manager = IndexManager.getInstance(); + int count = manager.getTableRegionCount(userTableName); + Assert.assertEquals(1, count); + + HTable table = new HTable(conf, userTableName); + Put p = null; + for (int i = 0; i < 10; i++) { + p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql"), Bytes.toBytes("test_val")); + table.put(p); + } + + admin.split(userTableName); + ZKAssign.blockUntilNoRIT(zkw); + + count = manager.getTableRegionCount(userTableName); + Assert.assertEquals(1, count); + } + + @Test(timeout = 180000) + public void testIndexManagerWithFailedSplitTransaction() throws Exception { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + Configuration conf = admin.getConfiguration(); + conf.setBoolean("hbase.use.secondary.index", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testIndexManagerWithFailedSplitTransaction"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col1"); + ihtd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("Index1"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + IndexManager manager = IndexManager.getInstance(); + int count = manager.getTableRegionCount(userTableName); + Assert.assertEquals(1, count); + + HTable table = new HTable(conf, userTableName); + Put p = null; + for (int i = 0; i < 10; i++) { + p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql"), Bytes.toBytes("test_val")); + table.put(p); + } + List regions = UTIL.getMiniHBaseCluster().getRegions(Bytes.toBytes(userTableName)); + HRegionServer rs = UTIL.getMiniHBaseCluster().getRegionServer(0); + SplitTransaction st = null; + + st = new MockedSplitTransaction(regions.get(0), null) { + @Override + protected void splitStoreFiles(final Path splitdir, final List hstoreFilesToSplit) + throws IOException { + throw new IOException(); + } + }; + + try { + st.execute(rs, rs); + } catch (IOException e) { + st.rollback(rs, rs); + } + + count = manager.getTableRegionCount(userTableName); + Assert.assertEquals(1, count); + } + + public static class MockedSplitTransaction extends SplitTransaction { + + public MockedSplitTransaction(HRegion r, byte[] splitrow) { + super(r, splitrow); + } + } +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexRegionObserverForScan.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexRegionObserverForScan.java new file mode 100644 index 0000000..c3aeee8 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestIndexRegionObserverForScan.java @@ -0,0 +1,1642 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.UnknownRegionException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestIndexRegionObserverForScan { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + IndexRegionObserver.setIndexedFlowUsed(false); + IndexRegionObserver.setSeekpointAdded(false); + IndexRegionObserver.setSeekPoints(null); + IndexRegionObserver.setIsTestingEnabled(true); + } + + @After + public void tearDown() throws Exception { + IndexRegionObserver.setIsTestingEnabled(false); + } + + @Test(timeout = 180000) + public void testScanIndexedColumnWithOnePutShouldRetreiveOneRowSuccessfully() throws IOException, + KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testPutOnIndexedScanColumnWithOnePut"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p); + int i = countNumberOfRowsWithFilter(userTableName, "Val", true, false, 0); + Assert.assertEquals("Should match for 1 row successfully ", 1, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + + } + + @Test(timeout = 180000) + public void testScanIndexedColumnWithOnePutAndSplitKeyatBorderShouldRetreiveOneRowSuccessfully() + throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testScanOnIndexedScanSplitColumnWithOnePut"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + byte[][] split = + new byte[][] { "row1".getBytes(), "row21".getBytes(), "row41".getBytes(), + "row61".getBytes(), "row81".getBytes(), "row101".getBytes(), "row121".getBytes(), + "row141".getBytes(), }; + + // create table with splits this will create 9 regions + admin.createTable(ihtd, split); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p); + + Put p1 = new Put("row01".getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p1); + + Put p2 = new Put("row010".getBytes()); + p2.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p2); + + Put p3 = new Put("row001".getBytes()); + p3.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p3); + + validateCountOfMainTableIndIndexedTable(conf, userTableName, table); + + int i = countNumberOfRowsWithFilter(userTableName, "Val", true, false, 0); + Assert.assertEquals("Should match for 1 row successfully ", 4, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + + } + + @Test(timeout = 180000) + public + void + testScanIndexedColumnWithOnePutAndSplitKeyAndStartKeyRegionEmptyShouldRetreiveOneRowSuccessfully() + throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testScanOnIndexedSplitStRegExmptyColumnWithOnePut"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + + byte[][] split = new byte[][] { "A".getBytes(), "B".getBytes(), "C".getBytes() }; + + // create table with splits this will create 9 regions + admin.createTable(ihtd, split); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + Put p = new Put("00row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p); + + Put p1 = new Put("0row1".getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p1); + + Put p2 = new Put("000row1".getBytes()); + p2.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p2); + + Put p3 = new Put("0000row1".getBytes()); + p3.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p3); + + // Check for verification of number of rows in main table and user table + validateCountOfMainTableIndIndexedTable(conf, userTableName, table); + + // test put with the indexed column + + int i = countNumberOfRowsWithFilter(userTableName, "Val", true, false, 0); + Assert.assertEquals("Should match for 1 row successfully ", 4, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + + } + + @Test(timeout = 180000) + public void + testScanIndexedColumnWithOnePutAndSplitKeyHavingSpaceShouldRetreiveOneRowSuccessfully() + throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testScanOnIndexedSplitSpacedColumnWithOnePut"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + + byte[][] split = + new byte[][] { " row1".getBytes(), "row21".getBytes(), "row41".getBytes(), + "row61".getBytes(), "row81".getBytes(), "row101 ".getBytes(), "row121".getBytes(), + "row141".getBytes(), }; + + // create table with splits this will create 9 regions + admin.createTable(ihtd, split); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p); + validateCountOfMainTableIndIndexedTable(conf, userTableName, table); + int i = countNumberOfRowsWithFilter(userTableName, "Val", true, false, 0); + Assert.assertEquals("Should match for 1 row successfully ", 1, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + + } + + @Test(timeout = 180000) + public void testScanIndexedColumnShouldNotRetreiveRowIfThereIsNoMatch() throws IOException, + KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + String userTableName = "testPutOnIndexedScanColumnWith2Puts"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexq"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p); + + Put p1 = new Put("row2".getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "Val1".getBytes()); + table.put(p1); + int i = countNumberOfRowsWithFilter(userTableName, "unmatch", true, false, 0); + Assert.assertEquals("Should not match any rows ", 0, i); + Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testScanIndexedColumnWith5PutsAnd3EqualPutValuesShouldRetreive3RowsSuccessfully() + throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + String userTableName = "test5PutsWithIndexedScanColumn"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexe"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + Put p1 = new Put("row1".getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p1); + + Put p4 = new Put("row3".getBytes()); + p4.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p4); + + Put p5 = new Put("row2".getBytes()); + p5.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p5); + + Put p2 = new Put("row4".getBytes()); + p2.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p2); + + Put p3 = new Put("row5".getBytes()); + p3.add("col".getBytes(), "ql".getBytes(), "dogs".getBytes()); + table.put(p3); + + int i = countNumberOfRowsWithFilter(userTableName, "cat", true, false, 0); + + Assert.assertEquals("Should match for exactly 3 rows ", 3, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testPerformanceOfScanOnIndexedColumnShouldBeMoreWith1LakhRowsIfFlushIsNotMade() + throws Exception { + + long withIndex = doPerformanceTest(true, "tableWithIndexAndWithoutFlush", false); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + IndexRegionObserver.setSeekpointAdded(false); + IndexRegionObserver.setIndexedFlowUsed(false); + long withoutIndex = doPerformanceTest(false, "tableWithoutIndexAndWithoutFlush", false); + Assert.assertTrue( + "Without flush time taken for Indexed scan should be less than without index ", + withIndex < withoutIndex); + + Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertFalse("Indexed table should not be used ", + IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testShouldSuccessfullyReturn3RowsIfOnly3RowsMatchesIn1LakhRowsWithParallelPuts() + throws Exception { + String userTableName = "tableToCheck3Rows"; + doBulkParallelPuts(true, userTableName, false); + int i = countNumberOfRowsWithFilter(userTableName, "cat", true, false, 0); + Assert.assertEquals("Should match for exactly 3 rows in 1 lakh rows ", 3, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getSeekpointAdded()); + } + + @Test(timeout = 180000) + public void testPerformanceOfScanOnIndexedColumnShouldBeMoreWith1LakhRowsIfFlushIsMade() + throws Exception { + long withIndex = doPerformanceTest(true, "tableWithIndexAndWithFlush", true); + + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + IndexRegionObserver.setSeekpointAdded(false); + IndexRegionObserver.setIndexedFlowUsed(false); + long withoutIndex = doPerformanceTest(false, "tableWithoutIndexAndWithFlush", true); + Assert.assertTrue("With flush time taken for Indexed scan should be less than without index ", + withIndex < withoutIndex); + Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertFalse("Indexed table should not be used ", + IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testParallelScansShouldRetreiveRowsCorrectlyForIndexedColumn() throws Exception { + String userTableName = "testParallelScansOnIndexedColumn"; + doParallelScanPuts(userTableName); + ParallelScanThread p1 = new ParallelScanThread(userTableName, "cat", false, 0); + p1.start(); + + ParallelScanThread p2 = new ParallelScanThread(userTableName, "dog", false, 0); + p2.start(); + + ParallelScanThread p3 = new ParallelScanThread(userTableName, "pup", false, 0); + p3.start(); + + // wait for scan to complete + p1.join(); + p2.join(); + p3.join(); + Assert.assertEquals("Should match for exactly 700 cats ", 700, p1.count); + Assert.assertEquals("Should match for exactly 500 dogs ", 500, p2.count); + Assert.assertEquals("Should match for exactly 300 pups ", 300, p3.count); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testParallelScansWithCacheShouldRetreiveRowsCorrectlyForIndexedColumn() + throws Exception { + String userTableName = "testParallelScansWithCacheOnIndexedColumn"; + doParallelScanPuts(userTableName); + // In parallel scan setting the cache + ParallelScanThread p1 = new ParallelScanThread(userTableName, "cat", true, 200); + p1.start(); + + ParallelScanThread p2 = new ParallelScanThread(userTableName, "dog", true, 200); + p2.start(); + + ParallelScanThread p3 = new ParallelScanThread(userTableName, "pup", true, 200); + p3.start(); + + // wait for scan to complete + p1.join(); + p2.join(); + p3.join(); + Assert.assertEquals("Should match for exactly 700 cats ", 700, p1.count); + Assert.assertEquals("Should match for exactly 500 dogs ", 500, p2.count); + Assert.assertEquals("Should match for exactly 300 pups ", 300, p3.count); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testScanShouldBeSuccessfulEvenIfExceptionIsThrownFromPostScannerOpen() + throws Exception { + + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testgenerateExceptionInPostScannerOpen"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add("col".getBytes(), "ql".getBytes(), "Val".getBytes()); + table.put(p); + Scan s = new Scan(); + Filter filter = new ExceptionFilter(); + s.setFilter(filter); + int i = 0; + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + + Assert.assertEquals("Should match for 1 row successfully ", 1, i); + Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertFalse("Indexed table should not be used ", + IndexRegionObserver.getIndexedFlowUsed()); + + } + + private String doParallelScanPuts(String userTableName) throws IOException, + ZooKeeperConnectionException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndex"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + List puts = new ArrayList(); + + for (int i = 1; i <= 500; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + + for (int i = 501; i <= 1000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + puts.add(p1); + } + table.put(puts); + + for (int i = 1001; i <= 1300; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "pup".getBytes()); + puts.add(p1); + } + table.put(puts); + + for (int i = 1301; i <= 1500; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + return userTableName; + } + + @Test(timeout = 180000) + public void testScanShouldNotRetreiveRowsIfRowsArePresentOnlyInIndexedTableAndNotInMainTable() + throws Exception { + + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + final String userTableName = "testScanOnIndexedColumnForFalsePositve"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndex"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + HTable table = new HTable(conf, userTableName); + + List puts = new ArrayList(); + + for (int i = 1; i <= 100; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 101; i <= 200; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 201; i <= 300; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "pup".getBytes()); + puts.add(p1); + } + table.put(puts); + + // Doing one extra put explicilty into indexed table + HTable indexTable = new HTable(conf, userTableName + Constants.INDEX_TABLE_SUFFIX); + + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable((userTableName + Constants.INDEX_TABLE_SUFFIX).getBytes()); + + byte[] startRow = generateStartKey(regionsOfTable); + + Put p1 = new Put(startRow); + p1.add("d".getBytes(), "ql".getBytes(), "idxCat".getBytes()); + indexTable.put(p1); + + int i = countNumberOfRowsWithFilter(userTableName, "idxCat", true, false, 0); + Assert.assertEquals("Should not match any rows in main table ", 0, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); + + } + + @Test(timeout = 180000) + public void testScanWithPutsAndCacheSetShouldRetreiveMatchingRows() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + String userTableName = "testcachedColumn"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexe"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + int i = singleIndexPutAndCache(conf, userTableName); + Assert.assertEquals("Should match for exactly 5 rows ", 5, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertEquals("Remaining rows in cache should be 2 ", 2, IndexRegionObserver + .getSeekpoints().size()); + + } + + @Test(timeout = 180000) + public void testScanMultipleIdxWithSameColFamilyAndDifferentQualifierShouldBeSuccessful() + throws Exception { + + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + String userTableName = "testScanWithMultIndexedSameColFamilyColumn"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + ihtd.addFamily(hcd1); + IndexSpecification idx1 = new IndexSpecification("ScanMulIndex"); + idx1.addIndexColumn(hcd1, "ql", ValueType.String, 10); + idx1.addIndexColumn(hcd1, "q2", ValueType.String, 10); + ihtd.addIndex(idx1); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + Put p1 = new Put("row1".getBytes()); + p1.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p1.add("col1".getBytes(), "q2".getBytes(), "dog".getBytes()); + table.put(p1); + + Put p2 = new Put("row2".getBytes()); + p2.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p2.add("col1".getBytes(), "q2".getBytes(), "cat".getBytes()); + table.put(p2); + + Put p3 = new Put("row3".getBytes()); + p3.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p3.add("col1".getBytes(), "q2".getBytes(), "dog".getBytes()); + table.put(p3); + + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q2 + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("col1".getBytes(), "q2".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter2.setFilterIfMissing(true); + filterList.addFilter(filter1); + + filterList.addFilter(filter2); + s.setFilter(filterList); + + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + + Assert.assertEquals("Should match for 2 rows in multiple index successfully ", 2, i); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + + } + + @Test(timeout = 180000) + public void testScanMultipleIdxWithDifferentColFamilyShouldBeSuccessful() throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanWithMultIndexedDiffColFamilyColumn"; + putMulIndex(userTableName); + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q1 + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("col2".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter2.setFilterIfMissing(true); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + s.setFilter(filterList); + HTable table = new HTable(conf, userTableName); + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + Assert.assertEquals( + "Should match for 5 rows in multiple index with diff column family successfully ", 5, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testScanMultipleIdxWithDifferentColFamilyAndCacheShouldBeSuccessful() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanWithMultIndexedCacheDiffColFamilyColumn"; + putMulIndex(userTableName); + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q1 + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("col2".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter2.setFilterIfMissing(true); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + s.setCaching(4); + s.setFilter(filterList); + HTable table = new HTable(conf, userTableName); + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + Assert.assertEquals( + "Should match for 5 rows in multiple index with diff column family successfully ", 5, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertEquals("Remaining rows in cache should be 1 ", 1, IndexRegionObserver + .getSeekpoints().size()); + } + + @Test(timeout = 180000) + public void + testScanMultipleIdxWithDifferentFiltersShouldBeSuccessfulAndShouldNotGoWithIndexedFlow() + throws Exception { + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanWithMultIndexedDiffFilters"; + putMulIndex(userTableName); + HTable table = new HTable(conf, userTableName); + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q1 + Filter filter1 = + new RowFilter(CompareOp.LESS_OR_EQUAL, new BinaryComparator("row5".getBytes())); + Filter filter2 = new FirstKeyOnlyFilter(); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + s.setFilter(filterList); + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + Assert.assertEquals( + "Should match for 5 rows in multiple index with diff column family successfully ", 5, i); + Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertFalse("Indexed table should not be used ", + IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testScanWithIndexOn2ColumnsAndFiltersOn2ColumnsInReverseWayShouldBeSuccessful() + throws Exception { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScan2Indexed2ReversedFilters"; + putMulIndex(userTableName); + HTable table = new HTable(conf, userTableName); + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q1 + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("col2".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + filter2.setFilterIfMissing(true); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + s.setFilter(filterList); + + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + Assert.assertEquals( + "Should match for 5 rows in multiple index with diff column family successfully ", 5, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public + void + testScanMultipleIdxWithDifferentColumnsInFiltersShouldBeSuccessfulAndShouldNotGoWithIndexedFlow() + throws Exception { + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "test11ScanWithMultIndexedDiff11Filters"; + HBaseAdmin admin = UTIL.getHBaseAdmin(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + HColumnDescriptor hcd3 = new HColumnDescriptor("col3"); + ihtd.addFamily(hcd1); + ihtd.addFamily(hcd2); + ihtd.addFamily(hcd3); + IndexSpecification idx1 = new IndexSpecification("ScanMulIndex"); + idx1.addIndexColumn(hcd1, "ql", ValueType.String, 10); + idx1.addIndexColumn(hcd2, "ql", ValueType.String, 10); + ihtd.addIndex(idx1); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + + // test put with the multiple indexed column in diffrent column families + Put p1 = new Put("row1".getBytes()); + p1.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p1.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p1); + + Put p2 = new Put("row2".getBytes()); + p2.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p2.add("col2".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p2); + + Put p3 = new Put("row3".getBytes()); + p3.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p3.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p3); + + Put p4 = new Put("row4".getBytes()); + p4.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p4.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p4); + + Put p5 = new Put("row5".getBytes()); + p5.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p5.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p5); + + Put p6 = new Put("row6".getBytes()); + p6.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p6.add("col2".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p6); + + Put p7 = new Put("row7".getBytes()); + p7.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p7.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p7); + + Put p9 = new Put("row8".getBytes()); + p9.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p9.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p9); + + Put p8 = new Put("row9".getBytes()); + p8.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p8.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p8); + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q1 + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("col3".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter2.setFilterIfMissing(true); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + s.setFilter(filterList); + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + Assert.assertEquals( + "Should match for 1 rows in multiple index with diff column family successfully ", 1, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + + // Different values in column family should not retreive the rows.. Below + // ensures the same + Scan s1 = new Scan(); + FilterList filterList1 = new FilterList(); + // check for combination of cat in q1 and dog in q1 + SingleColumnValueFilter filter11 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + filter11.setFilterIfMissing(true); + SingleColumnValueFilter filter12 = + new SingleColumnValueFilter("col3".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "dog1".getBytes()); + filter12.setFilterIfMissing(true); + filterList1.addFilter(filter11); + filterList1.addFilter(filter12); + s1.setFilter(filterList1); + i = 0; + ResultScanner scanner1 = table.getScanner(s1); + for (Result result : scanner1) { + i++; + } + Assert.assertEquals( + "Should match for 0 rows in multiple index with diff column family successfully ", 0, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + + } + + @Test(timeout = 180000) + public void testScanWith4IdxAnd2ColumnsInFiltersShouldBeSuccessful() throws Exception { + HTable table = put4ColumnIndex(); + int i = 0; + Scan s = new Scan(); + FilterList filterList = new FilterList(); + // check for combination of cat in q1 and dog in q1 + Filter filter1 = + new SingleColumnValueFilter("col1".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + Filter filter2 = + new SingleColumnValueFilter("col2".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filterList.addFilter(filter1); + filterList.addFilter(filter2); + s.setFilter(filterList); + + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + Assert.assertEquals( + "Should match for 5 rows in multiple index with diff column family successfully ", 5, i); + Assert.assertTrue("Seek points should be added ", IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Indexed table should be used ", IndexRegionObserver.getIndexedFlowUsed()); + } + + // @Test(timeout = 180000) + public void testShouldBESuccessfulEvenOfSeparatorIsNegative() throws Exception { + + UTIL.getMiniHBaseCluster().startRegionServer(); + // UTIL.getMiniHBaseCluster().startRegionServer(); + List liveRegionServerThreads = + UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + Assert.assertEquals("2 Region Servers should be started ", 2, liveRegionServerThreads.size()); + + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + final String userTableName = "testCollocationExplicitScansOnIndexedColumn"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndex"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + + admin.createTable(ihtd); + + String anotherTable = "newTable"; + HTableDescriptor ihtd1 = new HTableDescriptor(anotherTable); + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + ihtd1.addFamily(hcd1); + + admin.createTable(ihtd1); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + + List regionsOfTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable(userTableName.getBytes()); + List regionsOfIndexTable = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable((userTableName + "_idx").getBytes()); + + List rs1 = new ArrayList(); + List rs2 = new ArrayList(); + + rs1.add(regionsOfIndexTable.get(0)); + rs1.add(regionsOfTable.get(0)); + + HRegionServer regionServer1 = UTIL.getMiniHBaseCluster().getRegionServer(0); + HRegionServer regionServer2 = UTIL.getMiniHBaseCluster().getRegionServer(1); + + for (HRegionInfo hRegionInfo : rs1) { + admin.move(hRegionInfo.getEncodedNameAsBytes(), + Bytes.toBytes(regionServer2.getServerName().getServerName())); + } + + // for (HRegionInfo hRegionInfo : rs2) { + // admin.move(hRegionInfo.getEncodedNameAsBytes(), Bytes + // .toBytes(regionServer2.getServerName().getServerName())); + // } + + UTIL.getMiniHBaseCluster().getMaster().balanceSwitch(false); + HTable table1 = new HTable(conf, anotherTable); + List puts = new ArrayList(); + + for (int i = 1; i <= 10; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + puts = new ArrayList(); + + for (int i = 11; i <= 20; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + puts.add(p1); + } + table1.put(puts); + + puts = new ArrayList(); + for (int i = 21; i <= 30; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "pup".getBytes()); + puts.add(p1); + } + table.put(puts); + + List regionsOfIndexTable1 = + UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionsOfTable((userTableName + "_idx").getBytes()); + + for (HRegionInfo hRegionInfo : regionsOfIndexTable1) { + + int originServerNum = UTIL.getMiniHBaseCluster().getServerWith(hRegionInfo.getRegionName()); + int targetServerNum = 2 - 1 - originServerNum; + HRegionServer targetServer = UTIL.getMiniHBaseCluster().getRegionServer(targetServerNum); + admin.move(hRegionInfo.getEncodedNameAsBytes(), + Bytes.toBytes(targetServer.getServerName().getServerName())); + + } + + // Scan should be successful + int i = countNumberOfRowsWithFilter(userTableName, "cat", true, false, 0); + Assert.assertEquals("Should match for exactly 10 rows ", 10, i); + + } + + @Test(timeout = 180000) + public void testScanShouldBeSuccessfulEvenIfUserRegionAndIndexRegionAreNotCollocated() + throws Exception { + + // starting 3 RS + UTIL.getMiniHBaseCluster().startRegionServer(); + UTIL.getMiniHBaseCluster().startRegionServer(); + List liveRegionServerThreads = + UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + Assert.assertEquals("3 Region Servers should be started ", 3, liveRegionServerThreads.size()); + + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + final String userTableName = "testCollocatedScansOnIndexedColumn"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndex"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + + byte[][] split = + new byte[][] { "row1".getBytes(), "row21".getBytes(), "row41".getBytes(), + "row61".getBytes(), "row81".getBytes(), "row101".getBytes(), "row121".getBytes(), + "row141".getBytes(), }; + + // create table with splits this will create 9 regions + admin.createTable(ihtd, split); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + + UTIL.getMiniHBaseCluster().getMaster().balanceSwitch(false); + + // Now collocation between the RS's is done so put the rows + doPuts(conf, userTableName); + + // Now move the 3 rows that has the string "cat" to different RS + + collocateRowToDifferentRS(admin, table, "row1001"); + collocateRowToDifferentRS(admin, table, "row11004"); + collocateRowToDifferentRS(admin, table, "row11007"); + + // Scan should be successful + int i = countNumberOfRowsWithFilter(userTableName, "cat", true, false, 0); + Assert.assertEquals("Should match for exactly 6000 rows ", 6000, i); + + UTIL.getMiniHBaseCluster().abortRegionServer(1); + UTIL.getMiniHBaseCluster().abortRegionServer(2); + UTIL.getMiniHBaseCluster().abortRegionServer(0); + } + + private void putMulIndex(String userTableName) throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + ihtd.addFamily(hcd1); + ihtd.addFamily(hcd2); + IndexSpecification idx1 = new IndexSpecification("ScanMulIndex"); + idx1.addIndexColumn(hcd1, "ql", ValueType.String, 10); + idx1.addIndexColumn(hcd2, "ql", ValueType.String, 10); + ihtd.addIndex(idx1); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + + // test put with the multiple indexed column in diffrent column families + Put p1 = new Put("row1".getBytes()); + p1.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p1.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p1); + + Put p2 = new Put("row2".getBytes()); + p2.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p2.add("col2".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p2); + + Put p3 = new Put("row3".getBytes()); + p3.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p3.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p3); + + Put p4 = new Put("row4".getBytes()); + p4.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p4.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p4); + + Put p5 = new Put("row5".getBytes()); + p5.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p5.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p5); + + Put p6 = new Put("row6".getBytes()); + p6.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p6.add("col2".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p6); + + Put p7 = new Put("row7".getBytes()); + p7.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p7.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p7); + + Put p8 = new Put("row8".getBytes()); + p8.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p8.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p8); + + } + + private void collocateRowToDifferentRS(HBaseAdmin admin, HTable table, String rowKey) + throws IOException, UnknownRegionException, MasterNotRunningException, + ZooKeeperConnectionException { + + HRegionInfo regionInfo = table.getRegionLocation(rowKey).getRegionInfo(); + int originServerNum = UTIL.getMiniHBaseCluster().getServerWith(regionInfo.getRegionName()); + int targetServerNum = 3 - 1 - originServerNum; + HRegionServer targetServer = UTIL.getMiniHBaseCluster().getRegionServer(targetServerNum); + admin.move(regionInfo.getEncodedNameAsBytes(), + Bytes.toBytes(targetServer.getServerName().getServerName())); + } + + private void doPuts(Configuration conf, String userTableName) throws IOException { + + HTable table = new HTable(conf, userTableName); + List puts = new ArrayList(); + + for (int i = 1; i <= 2000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + puts = new ArrayList(); + + for (int i = 2001; i <= 4000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 4001; i <= 6000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "pup".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 6001; i <= 8000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cats".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 8001; i <= 10000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "dogs".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 10001; i <= 12000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 12001; i <= 14000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "pup".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 14001; i <= 16000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + puts.add(p1); + } + table.put(puts); + + puts = new ArrayList(); + for (int i = 16001; i <= 18000; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + puts.add(p1); + } + table.put(puts); + puts = new ArrayList(); + + } + + class ParallelScanThread extends Thread { + + String filterString; + String userTableName; + int count; + int cacheNumber; + boolean iscached; + + ParallelScanThread(String userTableName, String filterString, boolean iscached, int cacheNumber) + throws IOException { + this.filterString = filterString; + this.userTableName = userTableName; + this.iscached = iscached; + this.cacheNumber = cacheNumber; + } + + public void run() { + try { + count = + countNumberOfRowsWithFilter(userTableName, filterString, true, iscached, cacheNumber); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + class ParallelPutsValueThread extends Thread { + + int start; + int end; + String userTableName; + Configuration conf; + HTable table; + String value; + List puts; + + ParallelPutsValueThread(int start, Configuration conf, String userTableName, int end, + String value) throws IOException { + this.start = start; + this.end = end; + this.conf = conf; + this.userTableName = userTableName; + this.table = new HTable(this.conf, this.userTableName); + this.value = value; + puts = new ArrayList(); + } + + public void run() { + + for (int i = this.start; i <= this.end; i++) { + Put p1 = new Put(("row" + i).getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), value.getBytes()); + puts.add(p1); + } + try { + this.table.put(puts); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + private void doBulkParallelPuts(boolean toIndex, final String userTableName, boolean toFlush) + throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + final Configuration conf = UTIL.getConfiguration(); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + + if (toIndex) { + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + IndexSpecification iSpec = new IndexSpecification("ScanIndexe"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + } else { + HTableDescriptor htd = new HTableDescriptor(userTableName); + htd.addFamily(hcd); + admin.createTable(htd); + } + + ZKAssign.blockUntilNoRIT(zkw); + ParallelPutsValueThread p1 = new ParallelPutsValueThread(1, conf, userTableName, 10000, "dogs"); + p1.start(); + + Put a1 = new Put(("row" + 10001).getBytes()); + a1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + HTable table2 = new HTable(conf, userTableName); + table2.put(a1); + ParallelPutsValueThread p2 = + new ParallelPutsValueThread(10002, conf, userTableName, 30000, "dogs"); + p2.start(); + Put a2 = new Put(("row" + 30001).getBytes()); + a2.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + HTable table4 = new HTable(conf, userTableName); + table4.put(a2); + + ParallelPutsValueThread p3 = + new ParallelPutsValueThread(30002, conf, userTableName, 60000, "dogs"); + p3.start(); + + Put a3 = new Put(("row" + 60001).getBytes()); + a3.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + HTable table6 = new HTable(conf, userTableName); + table6.put(a3); + + ParallelPutsValueThread p4 = + new ParallelPutsValueThread(60002, conf, userTableName, 100000, "dogs"); + p4.start(); + + p1.join(); + p2.join(); + p3.join(); + p4.join(); + + if (toFlush) { + if (toIndex) { + admin.flush(userTableName + Constants.INDEX_TABLE_SUFFIX); + admin.flush(userTableName); + } else { + admin.flush(userTableName); + } + } + } + + private long doPerformanceTest(boolean toIndex, final String userTableName, boolean toFlush) + throws IOException, KeeperException, InterruptedException { + doBulkParallelPuts(toIndex, userTableName, toFlush); + long before = System.currentTimeMillis(); + int i = countNumberOfRowsWithFilter(userTableName, "cat", toIndex, false, 0); + long after = System.currentTimeMillis(); + return (after - before); + + } + + private int countNumberOfRowsWithFilter(String tableName, String filterVal, boolean isIndexed, + boolean isCached, int cacheNumber) throws IOException { + Configuration conf = UTIL.getConfiguration(); + HTable table = new HTable(conf, tableName); + Scan s = new Scan(); + Filter filter = null; + if (isIndexed) { + filter = + new SingleColumnValueFilter("col".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + filterVal.getBytes()); + } else { + filter = + new SingleColumnValueFilter("col".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + } + s.setFilter(filter); + if (isCached) { + s.setCaching(cacheNumber); + } + int i = 0; + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + return i; + } + + private HTable put4ColumnIndex() throws IOException, ZooKeeperConnectionException, + KeeperException, InterruptedException { + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testcan4hMultIndexed2DiffFilters"; + HBaseAdmin admin = UTIL.getHBaseAdmin(); + conf.setInt("hbase.regionserver.lease.period", 900000000); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd1 = new HColumnDescriptor("col1"); + HColumnDescriptor hcd2 = new HColumnDescriptor("col2"); + HColumnDescriptor hcd3 = new HColumnDescriptor("col3"); + HColumnDescriptor hcd4 = new HColumnDescriptor("col4"); + ihtd.addFamily(hcd1); + ihtd.addFamily(hcd2); + ihtd.addFamily(hcd3); + ihtd.addFamily(hcd4); + IndexSpecification idx1 = new IndexSpecification("ScanMulIndex"); + idx1.addIndexColumn(hcd1, "ql", ValueType.String, 10); + idx1.addIndexColumn(hcd2, "ql", ValueType.String, 10); + idx1.addIndexColumn(hcd3, "ql", ValueType.String, 10); + idx1.addIndexColumn(hcd4, "ql", ValueType.String, 10); + ihtd.addIndex(idx1); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, userTableName); + + // test put with the multiple indexed column in diffrent column families + Put p1 = new Put("row1".getBytes()); + p1.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p1.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + p1.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p1.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p1); + + Put p2 = new Put("row2".getBytes()); + p2.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p2.add("col2".getBytes(), "ql".getBytes(), "cat".getBytes()); + p2.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p2.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p2); + + Put p3 = new Put("row3".getBytes()); + p3.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p3.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + p3.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p3.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p3); + + Put p4 = new Put("row4".getBytes()); + p4.add("col1".getBytes(), "ql".getBytes(), "dog".getBytes()); + p4.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + p4.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p4.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p4); + + Put p5 = new Put("row5".getBytes()); + p5.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p5.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + p5.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p5.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p5); + + Put p6 = new Put("row8".getBytes()); + p6.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p6.add("col2".getBytes(), "ql".getBytes(), "cat".getBytes()); + p6.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p6.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p6); + + Put p8 = new Put("row6".getBytes()); + p8.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p8.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + p8.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p8.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p8); + + Put p7 = new Put("row7".getBytes()); + p7.add("col1".getBytes(), "ql".getBytes(), "cat".getBytes()); + p7.add("col2".getBytes(), "ql".getBytes(), "dog".getBytes()); + p7.add("col3".getBytes(), "ql".getBytes(), "dog".getBytes()); + p7.add("col4".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p7); + return table; + } + + private byte[] generateStartKey(List regionsOfTable) { + byte[] startKey = regionsOfTable.get(0).getStartKey(); + + byte[] startRow = + new byte[startKey.length + Constants.DEF_MAX_INDEX_NAME_LENGTH + 10 + + "row999".getBytes().length]; + + System.arraycopy(startKey, 0, startRow, 0, startKey.length); + System.arraycopy("ScanIndex".getBytes(), 0, startRow, startKey.length, "ScanIndex".length()); + + byte[] arr = new byte[18 - "ScanIndex".length()]; + byte e[] = new byte[10]; + + System.arraycopy(arr, 0, startRow, startKey.length + "ScanIndex".length(), arr.length); + System.arraycopy(e, 0, startRow, startKey.length + Constants.DEF_MAX_INDEX_NAME_LENGTH, 10); + + System.arraycopy("idxCat".getBytes(), 0, startRow, startKey.length + + Constants.DEF_MAX_INDEX_NAME_LENGTH, "idxCat".getBytes().length); + + System.arraycopy("row99".getBytes(), 0, startRow, startKey.length + + Constants.DEF_MAX_INDEX_NAME_LENGTH + 10, "row99".getBytes().length); + + System.out.println("constructed rowkey for indexed table " + Bytes.toString(startRow)); + return startRow; + } + + private int singleIndexPutAndCache(Configuration conf, String userTableName) throws IOException { + HTable table = new HTable(conf, userTableName); + Put p1 = new Put("row1".getBytes()); + p1.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p1); + + Put p4 = new Put("row3".getBytes()); + p4.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p4); + + Put p5 = new Put("row2".getBytes()); + p5.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p5); + + Put p2 = new Put("row4".getBytes()); + p2.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p2); + + Put p3 = new Put("row5".getBytes()); + p3.add("col".getBytes(), "ql".getBytes(), "dogs".getBytes()); + table.put(p3); + + Put p6 = new Put("row6".getBytes()); + p6.add("col".getBytes(), "ql".getBytes(), "dog".getBytes()); + table.put(p6); + + Put p7 = new Put("row7".getBytes()); + p7.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p7); + + Put p8 = new Put("row8".getBytes()); + p8.add("col".getBytes(), "ql".getBytes(), "cat".getBytes()); + table.put(p8); + + int i = 0; + Scan s = new Scan(); + Filter filter = null; + filter = + new SingleColumnValueFilter("col".getBytes(), "ql".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + + s.setFilter(filter); + s.setCaching(3); + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + return i; + } + + private void validateCountOfMainTableIndIndexedTable(Configuration conf, String userTableName, + HTable table) throws IOException { + Scan s = new Scan(); + int j = 0; + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + j++; + } + + HTable table1 = new HTable(conf, userTableName + Constants.INDEX_TABLE_SUFFIX); + Scan s1 = new Scan(); + int k = 0; + ResultScanner scanner1 = table1.getScanner(s1); + for (Result result : scanner1) { + k++; + } + + Assert.assertEquals("COunt of rows in main tbale and indexed table shoudl be same ", j, k); + } + +} + +class ExceptionFilter extends FilterList { + public List getFilters() { + throw new RuntimeException("Exception thrwn from ExceptionFilter "); + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestMultipleIndicesInScan.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestMultipleIndicesInScan.java new file mode 100644 index 0000000..ac1d848 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestMultipleIndicesInScan.java @@ -0,0 +1,3493 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import junit.framework.Assert; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.DoubleComparator; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.FloatComparator; +import org.apache.hadoop.hbase.filter.IntComparator; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestMultipleIndicesInScan { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setInt("hbase.regionserver.lease.period", 10 * 60 * 1000); + conf.setInt("hbase.rpc.timeout", 10 * 60 * 1000); + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @After + public void tearDown() throws Exception { + IndexRegionObserver.setIsTestingEnabled(false); + } + + @Before + public void setUp() throws Exception { + IndexRegionObserver.setIndexedFlowUsed(false); + IndexRegionObserver.setSeekpointAdded(false); + IndexRegionObserver.setSeekPoints(null); + IndexRegionObserver.setIsTestingEnabled(true); + IndexRegionObserver.addSeekPoints(null); + } + + @Test(timeout = 180000) + public void testAndOrCombinationWithMultipleIndices() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSimpleScenarioForMultipleIndices"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", "c1" }, "idx4"); + ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, "testSimpleScenarioForMultipleIndices"); + + putforIDX1(Bytes.toBytes("row0"), table); + putforIDX1(Bytes.toBytes("row1"), table); + putforIDX2(Bytes.toBytes("row2"), table); + putforIDX3(Bytes.toBytes("row3"), table); + + putforIDX1(Bytes.toBytes("row4"), table); + putforIDX2(Bytes.toBytes("row4"), table); + putforIDX3(Bytes.toBytes("row4"), table); + + putforIDX1(Bytes.toBytes("row5"), table); + putforIDX1(Bytes.toBytes("row6"), table); + putforIDX2(Bytes.toBytes("row7"), table); + putforIDX3(Bytes.toBytes("row8"), table); + + putforIDX1(Bytes.toBytes("row9"), table); + putforIDX2(Bytes.toBytes("row9"), table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ONE); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "ele".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.EQUAL, + "fan".getBytes()); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + } + + @Test(timeout = 180000) + public void testReseekWhenSomeScannerAlreadyScannedGreaterValueThanSeekPoint() + throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testReseekWhenSomeScannerAlreadyScannedGreaterValueThanSeekPoint"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", "c1" }, "idx4"); + ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = + new HTable(conf, "testReseekWhenSomeScannerAlreadyScannedGreaterValueThanSeekPoint"); + + putforIDX1(Bytes.toBytes("row0"), table); + putforIDX1(Bytes.toBytes("row3"), table); + putforIDX1(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row6"), table); + putforIDX3(Bytes.toBytes("row1"), table); + putforIDX3(Bytes.toBytes("row3"), table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ONE); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "ele".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.EQUAL, + "fan".getBytes()); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + } + + private void putforIDX3(byte[] row, HTable htable) throws IOException { + Put p = new Put(row); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("bat")); + htable.put(p); + } + + private void putforIDX2(byte[] row, HTable htable) throws IOException { + Put p = new Put(row); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("apple")); + htable.put(p); + } + + private void putforIDX1(byte[] row, HTable htable) throws IOException { + Put p = new Put(row); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("cat")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c4"), Bytes.toBytes("dog")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c5"), Bytes.toBytes("ele")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c6"), Bytes.toBytes("fan")); + htable.put(p); + } + + private IndexSpecification createIndexSpecification(HColumnDescriptor hcd, ValueType type, + int maxValueLength, String[] qualifiers, String name) { + IndexSpecification index = new IndexSpecification(name.getBytes()); + for (String qualifier : qualifiers) { + index.addIndexColumn(hcd, qualifier, type, maxValueLength); + } + return index; + } + + @Test(timeout = 180000) + public void testWhenAppliedFilterGetsNoScanScheme() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testWhenAppliedFilterGetsNoScanScheme"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", "c1" }, "idx4"); + ihtd.addIndex(indexSpecification); + + SingleColumnValueFilter filter = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "ele".getBytes()); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, "testWhenAppliedFilterGetsNoScanScheme"); + + putforIDX1(Bytes.toBytes("row1"), table); + putforIDX1(Bytes.toBytes("row2"), table); + putforIDX1(Bytes.toBytes("row3"), table); + putforIDX1(Bytes.toBytes("row4"), table); + + Scan scan = new Scan(); + scan.setFilter(filter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + Assert.assertFalse("Index table should not be used", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertFalse("No seek points should get added from index flow", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue(testRes.size() == 4); + + } + + @Test(timeout = 180000) + public void testTheOROperation() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testTheOROperation"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", "c1" }, "idx4"); + ihtd.addIndex(indexSpecification); + + SingleColumnValueFilter filter = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, "testTheOROperation"); + + putforIDX2(Bytes.toBytes("row1"), table); + putforIDX3(Bytes.toBytes("row1"), table); + putforIDX2(Bytes.toBytes("row2"), table); + putforIDX2(Bytes.toBytes("row3"), table); + putforIDX3(Bytes.toBytes("row4"), table); + putforIDX3(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row6"), table); + putforIDX3(Bytes.toBytes("row7"), table); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + Assert.assertTrue("Index flow should be used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Index should fetch some seek points.", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("Index should fetch 7 seek points", 7, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertEquals("Final result should have 7 rows.", 7, testRes.size()); + } + + @Test(timeout = 180000) + public void testTheANDOpeartion() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testTheANDOpeartion"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + + SingleColumnValueFilter filter = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, "testTheANDOpeartion"); + + putforIDX2(Bytes.toBytes("row1"), table); + putforIDX3(Bytes.toBytes("row1"), table); + putforIDX2(Bytes.toBytes("row2"), table); + putforIDX2(Bytes.toBytes("row3"), table); + putforIDX3(Bytes.toBytes("row4"), table); + putforIDX3(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row6"), table); + putforIDX3(Bytes.toBytes("row7"), table); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + Assert.assertTrue("Index flow should be used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Index should fetch some seek points.", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("Index should fetch 2 seek points", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertEquals("Final result should have 2 rows.", 2, testRes.size()); + } + + @Test(timeout = 180000) + public void testAndOperationWithProperStartAndStopRow() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testAndOperationWithProperStartAndStopRow"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("5")); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 1); + + } + + @Test(timeout = 180000) + public void testAndOperationWithSimilarValuePattern() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testAndOperationWithSimilarValuePattern"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("aaa")); + table.put(p); + p = new Put(Bytes.toBytes("row9")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("aaa1")); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("aaa3")); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("aaa4")); + table.put(p); + p = new Put(Bytes.toBytes("row7")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("aaa5")); + table.put(p); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("aaa")); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 1); + Assert.assertTrue("Index should fetch some seek points.", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("Index should fetch 2 seek points", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + } + + @Test(timeout = 180000) + public void testScanWithMutlipleIndicesOnTheSameColAndSimilarPattern() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanWithMutlipleIndicesOnTheSameColAndSimilarPattern"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[][] val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row1"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row2"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row3"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog1"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row4"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1"), Bytes.toBytes("ant") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row5"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elefe"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row6"), val, table); + + table.flushCommits(); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("cat")); + SingleColumnValueFilter scvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("dog")); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("elef")); + masterFilter.addFilter(scvf); + masterFilter.addFilter(scvf1); + masterFilter.addFilter(scvf2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 2); + } + + @Test(timeout = 180000) + public void testScanWithMutlipleIndicesWithGreaterthanEqualCondOnTheSameColAndSimilarPattern() + throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testScanWithMutlipleIndicesWithGreaterthanEqualCondOnTheSameColAndSimilarPattern"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[][] val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row1"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row2"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row3"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog1"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row4"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1"), Bytes.toBytes("ant") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row5"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elefe"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row6"), val, table); + + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row7"), val, table); + + table.flushCommits(); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("cat")); + SingleColumnValueFilter scvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("dog")); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("elef")); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.GREATER_OR_EQUAL, + Bytes.toBytes("goat")); + masterFilter.addFilter(scvf); + masterFilter.addFilter(scvf1); + masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf3); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertEquals(testRes.size(), 3); + } + + @Test(timeout = 180000) + public void testScanWithMutlipleIndicesWithGreaterCondOnTheSameColAndSimilarPattern() + throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testScanWithMutlipleIndicesWithGreaterCondOnTheSameColAndSimilarPattern"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[][] val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row1"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row2"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row3"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog1"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row4"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1"), Bytes.toBytes("ant") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row5"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elefe"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row6"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row7"), val, table); + + table.flushCommits(); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("cat")); + SingleColumnValueFilter scvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("dog")); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("elef")); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.GREATER, + Bytes.toBytes("goat")); + + masterFilter.addFilter(scvf); + masterFilter.addFilter(scvf1); + masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf3); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertEquals(testRes.size(), 2); + } + + @Test(timeout = 180000) + public void testScanWithMutlipleIndicesWithLesserCondOnTheSameColAndSimilarPattern() + throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testScanWithMutlipleIndicesWithLesserCondOnTheSameColAndSimilarPattern"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[][] val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row1"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row2"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row3"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog1"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row4"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1"), Bytes.toBytes("ant") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row5"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elefe"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row6"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row7"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("gda") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row8"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goa") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row9"), val, table); + + table.flushCommits(); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("cat")); + SingleColumnValueFilter scvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("dog")); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("elef")); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.LESS, + Bytes.toBytes("goat")); + + masterFilter.addFilter(scvf); + masterFilter.addFilter(scvf1); + masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf3); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertEquals(testRes.size(), 2); + } + + @Test(timeout = 180000) + public void testScanWithMutlipleIndicesWithLesserEqualCondOnTheSameColAndSimilarPattern() + throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testScanWithMutlipleIndicesWithLesserEqualCondOnTheSameColAndSimilarPattern"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + byte[][] val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row1"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("ele"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row2"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row3"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog1"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row4"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat1"), Bytes.toBytes("ant") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row5"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elefe"), + Bytes.toBytes("goat1") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row6"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goat") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row7"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("gda") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row8"), val, table); + val = + new byte[][] { Bytes.toBytes("cat"), Bytes.toBytes("dog"), Bytes.toBytes("elef"), + Bytes.toBytes("goa") }; + putsForIdx1WithDiffValues(Bytes.toBytes("row9"), val, table); + + table.flushCommits(); + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("cat")); + SingleColumnValueFilter scvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("dog")); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("elef")); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.LESS_OR_EQUAL, + Bytes.toBytes("goat")); + + masterFilter.addFilter(scvf); + masterFilter.addFilter(scvf1); + masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf3); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertEquals(testRes.size(), 3); + } + + private void putsForIdx1WithDiffValues(byte[] row, byte[][] valList, HTable table) + throws IOException { + Put p = new Put(row); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), (valList[0])); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c4"), valList[1]); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c5"), valList[2]); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c6"), valList[3]); + table.put(p); + } + + @Test(timeout = 180000) + public void testWhenSomePointsAreFetchedFromIndexButMainScanStillHasSomeFiltersToApply() + throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "MainScanStillHasSomeFiltersToApply"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + scvf.setFilterIfMissing(true); + SingleColumnValueFilter scvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + scvf1.setFilterIfMissing(true); + FilterList orFilter = new FilterList(Operator.MUST_PASS_ONE); + orFilter.addFilter(scvf); + orFilter.addFilter(scvf1); + + FilterList andFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + scvf2.setFilterIfMissing(true); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + scvf3.setFilterIfMissing(true); + SingleColumnValueFilter scvf4 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "ele".getBytes()); + scvf4.setFilterIfMissing(true); + SingleColumnValueFilter scvf5 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.EQUAL, + "fan".getBytes()); + scvf5.setFilterIfMissing(true); + andFilter.addFilter(scvf5); + andFilter.addFilter(scvf4); + andFilter.addFilter(scvf3); + andFilter.addFilter(scvf2); + + FilterList master = new FilterList(Operator.MUST_PASS_ALL); + master.addFilter(andFilter); + master.addFilter(orFilter); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, "MainScanStillHasSomeFiltersToApply"); + + putforIDX1(Bytes.toBytes("row0"), table); + putforIDX1(Bytes.toBytes("row1"), table); + putforIDX2(Bytes.toBytes("row2"), table); + putforIDX3(Bytes.toBytes("row3"), table); + + putforIDX1(Bytes.toBytes("row4"), table); + putforIDX2(Bytes.toBytes("row4"), table); + putforIDX3(Bytes.toBytes("row4"), table); + + putforIDX1(Bytes.toBytes("row5"), table); + putforIDX1(Bytes.toBytes("row6"), table); + putforIDX2(Bytes.toBytes("row7"), table); + putforIDX3(Bytes.toBytes("row8"), table); + + putforIDX1(Bytes.toBytes("row9"), table); + putforIDX2(Bytes.toBytes("row9"), table); + + Scan scan = new Scan(); + scan.setFilter(master); + // scan.setCaching(10); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + Assert.assertTrue("Index flow should be used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Index should fetch some seek points.", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("Index should fetch 6 seek points", 3, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertEquals("Final result should have 2 rows.", 2, testRes.size()); + } + + @Test(timeout = 180000) + public void testWhenThereIsNoDataInIndexRegion() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testWhenThereIsNoDataInIndexRegion"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + + SingleColumnValueFilter filter = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, "testWhenThereIsNoDataInIndexRegion"); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + Assert.assertTrue("Index flow should be used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertFalse("Index should fetch some seek points.", + IndexRegionObserver.getSeekpointAdded()); + } + + @Test(timeout = 180000) + public void testMultipleScansOnTheIndexRegion() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + final Configuration conf = UTIL.getConfiguration(); + String userTableName = "testMultipleScansOnTheIndexRegion"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", "c1" }, "idx4"); + ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, "testMultipleScansOnTheIndexRegion"); + + putforIDX1(Bytes.toBytes("row0"), table); + putforIDX1(Bytes.toBytes("row1"), table); + putforIDX2(Bytes.toBytes("row2"), table); + putforIDX3(Bytes.toBytes("row3"), table); + + putforIDX1(Bytes.toBytes("row4"), table); + putforIDX2(Bytes.toBytes("row4"), table); + putforIDX3(Bytes.toBytes("row4"), table); + + putforIDX1(Bytes.toBytes("row5"), table); + putforIDX1(Bytes.toBytes("row6"), table); + putforIDX2(Bytes.toBytes("row7"), table); + putforIDX3(Bytes.toBytes("row8"), table); + + putforIDX1(Bytes.toBytes("row9"), table); + putforIDX2(Bytes.toBytes("row9"), table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ONE); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "cat".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "dog".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "ele".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.EQUAL, + "fan".getBytes()); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + Thread test = new testThread(masterFilter, "testMultipleScansOnTheIndexRegion"); + test.start(); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + test.join(); + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 4, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + } + + private class testThread extends Thread { + Filter filter = null; + String tableName = null; + + public testThread(Filter filter, String tableName) { + this.filter = filter; + this.tableName = tableName; + } + + @Override + public synchronized void start() { + try { + HTable table = new HTable(UTIL.getConfiguration(), tableName); + Scan scan = new Scan(); + scan.setFilter(filter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + } catch (IOException e) { + } + } + } + + @Test(timeout = 180000) + public void testFalsePositiveCases() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + final Configuration conf = UTIL.getConfiguration(); + String userTableName = "testFalsePositiveCases"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, "testFalsePositiveCases"); + + HTable idx_table = new HTable(conf, "testFalsePositiveCases_idx"); + + ByteArrayBuilder byteArray = new ByteArrayBuilder(33); + byteArray.put(new byte[1]); + byteArray.put(Bytes.toBytes("idx2")); + byteArray.put(new byte[14]); + byteArray.put(Bytes.toBytes("apple")); + byteArray.put(new byte[5]); + int offset = byteArray.position(); + byteArray.put(Bytes.toBytes("row1")); + + ByteArrayBuilder value = new ByteArrayBuilder(4); + value.put(Bytes.toBytes((short) byteArray.array().length)); + value.put(Bytes.toBytes((short) offset)); + Put p = new Put(byteArray.array()); + p.add(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, value.array()); + idx_table.put(p); + SingleColumnValueFilter filter = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertEquals("Overall result should have only 2 rows", 0, testRes.size()); + } + + @Test(timeout = 180000) + public void testSingleLevelRangeScanForAND() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSingleLevelRangeScanForAND"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + // SingleColumnValueFilter scvfsub = new SingleColumnValueFilter("cf1".getBytes(), + // "c1".getBytes(), + // CompareOp.EQUAL, "5".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + // filterList1.addFilter(scvfsub); + // filterList1.addFilter(scvf2sub); + // filterList1.addFilter(scvf3sub); + + // SingleColumnValueFilter scvf = new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), + // CompareOp.GREATER_OR_EQUAL, "5".getBytes()); + // SingleColumnValueFilter scvf2 = new SingleColumnValueFilter("cf1".getBytes(), + // "c2".getBytes(), + // CompareOp.LESS_OR_EQUAL, "5".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + + // masterFilter.addFilter(scvf); + // masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(scvf3); + // masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 2); + } + + @Test(timeout = 180000) + public void testSingleLevelRangeScanForOR() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testSingleLevelRangeScanForOR"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + // SingleColumnValueFilter scvfsub = new SingleColumnValueFilter("cf1".getBytes(), + // "c1".getBytes(), + // CompareOp.EQUAL, "5".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + // filterList1.addFilter(scvfsub); + // filterList1.addFilter(scvf2sub); + // filterList1.addFilter(scvf3sub); + + // SingleColumnValueFilter scvf = new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), + // CompareOp.GREATER_OR_EQUAL, "5".getBytes()); + // SingleColumnValueFilter scvf2 = new SingleColumnValueFilter("cf1".getBytes(), + // "c2".getBytes(), + // CompareOp.LESS_OR_EQUAL, "5".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + + // masterFilter.addFilter(scvf); + // masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(scvf3); + // masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 6, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 6 rows", testRes.size() == 6); + } + + @Test(timeout = 180000) + public void testEqualAndRangeCombinationWithMultipleIndices() throws IOException, + KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testEqualAndRangeCombinationWithMultipleIndices"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.GREATER_OR_EQUAL, + "4".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + + masterFilter.addFilter(scvfsub); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 1); + } + + @Test(timeout = 180000) + public void testEqualAndRangeCombinationWithMultipleIndicesPart2() throws IOException, + KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testEqualAndRangeCombinationWithMultipleIndicesPart2"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ALL); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.GREATER_OR_EQUAL, + "1".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "6".getBytes()); + + masterFilter.addFilter(scvfsub); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 4, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 4); + } + + @Test(timeout = 180000) + public void testOREvaluatorWithMultipleOperatorsInEachLevel() throws IOException, + KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testOREvaluatorWithMultipleOperatorsInEachLevel"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "5".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "5".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + filterList1.addFilter(scvfsub); + filterList1.addFilter(scvf2sub); + filterList1.addFilter(scvf3sub); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "4".getBytes()); + + masterFilter.addFilter(scvf); + masterFilter.addFilter(scvf2); + masterFilter.addFilter(scvf3); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 6, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 6 rows", testRes.size() == 6); + } + + @Test(timeout = 180000) + public void testIfAllScannersAreRangeInAllLevels() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testIfAllScannersAreRangeInAllLevels"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ALL); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.GREATER_OR_EQUAL, + "1".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + + masterFilter.addFilter(scvfsub); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 6, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 6); + } + + @Test(timeout = 180000) + public void testANDWithORbranchesWhereEachBranchHavingAtleastOneFilterOtherThanSCVF() + throws IOException, KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = + "testANDWithORbranchesWhereEachBranchHavingAtleastOneFilterOtherThanSCVF"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "1".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + RowFilter rowFilter = + new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row1"))); + FilterList filterList2 = new FilterList(Operator.MUST_PASS_ONE); + filterList2.addFilter(scvfsub); + filterList2.addFilter(scvf2sub); + filterList2.addFilter(scvf3sub); + filterList2.addFilter(rowFilter); + + SingleColumnValueFilter scvfsub1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + RowFilter rowFilter2 = + new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row1"))); + FilterList subFilterList = new FilterList(Operator.MUST_PASS_ONE); + subFilterList.addFilter(scvfsub1); + subFilterList.addFilter(scvf2sub2); + subFilterList.addFilter(scvf3sub3); + subFilterList.addFilter(rowFilter2); + + filterList1.addFilter(subFilterList); + masterFilter.addFilter(filterList1); + masterFilter.addFilter(filterList2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertFalse("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testORIfEachBranchHavingAtleastOneOtherFilterThanSCVF() throws IOException, + KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testORIfEachBranchHavingAtleastOneOtherFilterThanSCVF"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "1".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + RowFilter rowFilter = + new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row1"))); + FilterList filterList2 = new FilterList(Operator.MUST_PASS_ONE); + filterList2.addFilter(scvfsub); + filterList2.addFilter(scvf2sub); + filterList2.addFilter(scvf3sub); + filterList2.addFilter(rowFilter); + + SingleColumnValueFilter scvfsub1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + RowFilter rowFilter2 = + new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row1"))); + FilterList subFilterList = new FilterList(Operator.MUST_PASS_ONE); + subFilterList.addFilter(scvfsub1); + subFilterList.addFilter(scvf2sub2); + subFilterList.addFilter(scvf3sub3); + subFilterList.addFilter(rowFilter2); + + filterList1.addFilter(subFilterList); + masterFilter.addFilter(filterList1); + masterFilter.addFilter(filterList2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertFalse("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testORBranchesInWhichOneBranchHavingOtherFiltersThanSCVF() throws IOException, + KeeperException, InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testORBranchesInWhichOneBranchHavingOtherFiltersThanSCVF"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "1".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + RowFilter rowFilter = + new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row1"))); + FilterList filterList2 = new FilterList(Operator.MUST_PASS_ONE); + filterList2.addFilter(scvfsub); + filterList2.addFilter(scvf2sub); + filterList2.addFilter(scvf3sub); + filterList2.addFilter(rowFilter); + + masterFilter.addFilter(filterList1); + masterFilter.addFilter(filterList2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertFalse("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + } + + @Test(timeout = 180000) + public void testANDhavingORbranchWithOtherFilterThanSCVF() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testANDhavingORbranchWithOtherFilterThanSCVF"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "1".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + RowFilter rowFilter = + new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row1"))); + FilterList filterList2 = new FilterList(Operator.MUST_PASS_ONE); + filterList2.addFilter(scvfsub); + filterList2.addFilter(scvf2sub); + filterList2.addFilter(scvf3sub); + filterList2.addFilter(rowFilter); + + masterFilter.addFilter(filterList1); + masterFilter.addFilter(filterList2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 6, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 6); + } + + @Test(timeout = 180000) + public void testIfAllScannersAreRangeInAllLevelsPart2() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testIfAllScannersAreRangeInAllLevelsPart2"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "1".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "4".getBytes()); + + masterFilter.addFilter(scvfsub); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 1); + } + + @Test(timeout = 180000) + public void testIfAllScannersAreRangeInAllLevelsPart3() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testIfAllScannersAreRangeInAllLevelsPart3"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ALL); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "4".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "1".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + + masterFilter.addFilter(scvfsub); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 1); + } + + @Test(timeout = 180000) + public void testOREvaluationFromMultipleLevels() throws IOException, KeeperException, + InterruptedException { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testOREvaluationFromMultipleLevels"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "1".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "4".getBytes()); + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + + SingleColumnValueFilter scvfsub = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2sub = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.GREATER_OR_EQUAL, + "6".getBytes()); + SingleColumnValueFilter scvf3sub = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "3".getBytes()); + + masterFilter.addFilter(scvfsub); + masterFilter.addFilter(scvf2sub); + masterFilter.addFilter(scvf3sub); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 6, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 6); + } + + private void rangePutForIdx2(HTable table) throws IOException { + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("1")); + table.put(p); + p = new Put(Bytes.toBytes("row9")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("2")); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("3")); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("4")); + table.put(p); + p = new Put(Bytes.toBytes("row7")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("5")); + table.put(p); + p = new Put(Bytes.toBytes("row15")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("6")); + table.put(p); + } + + private void rangePutForIdx3(HTable table) throws IOException { + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("1")); + table.put(p); + p = new Put(Bytes.toBytes("row9")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("2")); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("3")); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("4")); + table.put(p); + p = new Put(Bytes.toBytes("row7")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("5")); + table.put(p); + p = new Put(Bytes.toBytes("row15")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c2"), Bytes.toBytes("6")); + table.put(p); + } + + private void rangePutForIdx4(HTable table) throws IOException { + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("1")); + table.put(p); + p = new Put(Bytes.toBytes("row9")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("2")); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("3")); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("4")); + table.put(p); + p = new Put(Bytes.toBytes("row7")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("5")); + table.put(p); + p = new Put(Bytes.toBytes("row15")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c3"), Bytes.toBytes("6")); + table.put(p); + } + + @Test(timeout = 180000) + public void testCombinationOfLESSorGREATERwithEQUAL() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCombinationOfLESSorGREATERwithEQUAL"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, + new String[] { "c3", "c4", "c5", "c6" }, "idx1"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c3" }, "idx4"); + ihtd.addIndex(indexSpecification); + + // indexSpecification = createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2", + // "c1" }, "idx4"); + // ihtd.addIndex(indexSpecification); + + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2(table); + rangePutForIdx3(table); + rangePutForIdx4(table); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + + FilterList filterList1 = new FilterList(Operator.MUST_PASS_ONE); + + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "2".getBytes()); + SingleColumnValueFilter scvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + SingleColumnValueFilter scvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + SingleColumnValueFilter scvf4 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "5".getBytes()); + + filterList1.addFilter(scvf); + filterList1.addFilter(scvf2); + filterList1.addFilter(scvf3); + filterList1.addFilter(scvf4); + masterFilter.addFilter(filterList1); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + i++; + result = scanner.next(1); + } + System.out.println("************* result count......*********** " + testRes.size() + + " ###### " + testRes); + Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Seekpoints should get added by index scanner", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("It should get two seek points from index scanner.", 4, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 4); + } + + @Test(timeout = 180000) + public void testIndexScanWithCaching() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testIndexScanWithCaching"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx2"); + ihtd.addIndex(indexSpecification); + indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c2" }, "idx3"); + ihtd.addIndex(indexSpecification); + + SingleColumnValueFilter filter = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "apple".getBytes()); + + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "bat".getBytes()); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter); + + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + HTable table = new HTable(conf, "testIndexScanWithCaching"); + + putforIDX2(Bytes.toBytes("row1"), table); + putforIDX3(Bytes.toBytes("row1"), table); + putforIDX2(Bytes.toBytes("row2"), table); + putforIDX2(Bytes.toBytes("row3"), table); + putforIDX3(Bytes.toBytes("row4"), table); + putforIDX3(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row5"), table); + putforIDX2(Bytes.toBytes("row6"), table); + putforIDX3(Bytes.toBytes("row7"), table); + + Scan scan = new Scan(); + scan.setFilter(masterFilter); + scan.setCaching(10); + int i = 0; + ResultScanner scanner = table.getScanner(scan); + int nextCalled = 0; + List testRes = new ArrayList(); + Result[] result = scanner.next(10); + nextCalled++; + while (result != null && result.length > 1) { + for (int j = 0; j < result.length; j++) { + testRes.add(result[j]); + } + i++; + result = scanner.next(10); + nextCalled++; + } + + Assert.assertTrue("Index flow should be used.", IndexRegionObserver.getIndexedFlowUsed()); + Assert.assertTrue("Index should fetch some seek points.", + IndexRegionObserver.getSeekpointAdded()); + Assert.assertEquals("Index should fetch 7 seek points", 7, IndexRegionObserver + .getMultipleSeekPoints().size()); + Assert.assertEquals("Final result should have 7 rows.", 7, testRes.size()); + Assert.assertEquals("All rows should be fetched in single next call.", 2, nextCalled); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegtiveIntValueWithEqualCondition() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testOtherDataTypes"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Int, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithInteger(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(-4)); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 1); + assertTrue(testRes.toString().contains("row3")); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeIntValueWithLessCondition() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeIntValueWithLessCondition"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Int, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithInteger(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + (Bytes.toBytes(-4))); + + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 2); + } + + @Test(timeout = 180000) + public void testShouldRetriveNegativeIntValueWithGreaterCondition() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetriveNegativeIntValueWithGreaterCondition"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Int, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithInteger(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + new IntComparator(Bytes.toBytes(-6))); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 5); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeIntValue() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeIntValue"; + HTableDescriptor ihtd = new HTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithInteger(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + new IntComparator(Bytes.toBytes(-6))); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 5); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeFloatValueWithGreaterCondition() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeFloatValueWithGreaterCondition"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Float, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithFloat(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + new FloatComparator(Bytes.toBytes(-5f))); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 4); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeFloatValueWithLessCondition() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeFloatValueWithLessCondition"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Float, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithFloat(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + new FloatComparator(Bytes.toBytes(-5f))); + + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 2); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeFloatValueWithEqualsCondition() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeFloatValueWithEqualsCondition"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Float, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithFloat(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + new FloatComparator(Bytes.toBytes(-5.3f))); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 1); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeDoubleValueWithLesserThanEqualsCondition() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeDoubleValueWithLesserThanEqualsCondition"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Double, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithDouble(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + new DoubleComparator(Bytes.toBytes(-5.3d))); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 2); + } + + @Test(timeout = 180000) + public void testShouldRetrieveNegativeDoubleValueWithGreaterThanEqualsCondition() + throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testShouldRetrieveNegativeDoubleValueWithGreaterThanEqualsCondition"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.Double, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + admin.createTable(ihtd); + HTable table = new HTable(conf, userTableName); + rangePutForIdx2WithDouble(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + new DoubleComparator(Bytes.toBytes(-5.3d))); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + testRes.add(result[0]); + result = scanner.next(1); + } + assertTrue(testRes.size() == 5); + } + + @Test(timeout = 180000) + public void testCachingWithValuesDistributedAmongMulitpleRegions() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCachingWithValuesDistributedAmongMulitpleRegions"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + byte[][] split = + new byte[][] { Bytes.toBytes("row1"), Bytes.toBytes("row2"), Bytes.toBytes("row3"), + Bytes.toBytes("row4") }; + admin.createTable(ihtd, split); + HTable table = new HTable(conf, userTableName); + insert100Rows(table); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("5")); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setCaching(5); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + int i = 0; + while (result != null && result.length > 0) { + System.out.println(Bytes.toString(result[0].getRow())); + testRes.add(result[0]); + result = scanner.next(1); + i++; + } + assertEquals(8, i); + } + + @Test(timeout = 180000) + public void testCachingWithValuesWhereSomeRegionsDontHaveAnyData() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCachingWithValuesWhereSomeRegionsDontHaveAnyData"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + byte[][] split = + new byte[][] { Bytes.toBytes("row1"), Bytes.toBytes("row3"), Bytes.toBytes("row5"), + Bytes.toBytes("row7") }; + admin.createTable(ihtd, split); + HTable table = new HTable(conf, userTableName); + for (int i = 0; i < 10; i++) { + if (i > 4 && i < 8) { + continue; + } + Put p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("5")); + table.put(p); + } + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("5")); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setCaching(5); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + int i = 0; + while (result != null && result.length > 0) { + System.out.println(Bytes.toString(result[0].getRow())); + testRes.add(result[0]); + result = scanner.next(1); + i++; + } + assertEquals(7, i); + } + + @Test(timeout = 180000) + public void testCachingWithLessNumberOfRowsThanCaching() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String userTableName = "testCachingWithLessNumberOfRowsThanCaching"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification indexSpecification = + createIndexSpecification(hcd, ValueType.String, 10, new String[] { "c1" }, "idx1"); + ihtd.addIndex(indexSpecification); + byte[][] split = + new byte[][] { Bytes.toBytes("row1"), Bytes.toBytes("row3"), Bytes.toBytes("row5"), + Bytes.toBytes("row7") }; + admin.createTable(ihtd, split); + HTable table = new HTable(conf, userTableName); + for (int i = 0; i < 10; i++) { + if (i > 4 && i < 8) { + continue; + } + Put p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("5")); + table.put(p); + } + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes("5")); + masterFilter.addFilter(scvf); + Scan scan = new Scan(); + scan.setCaching(10); + scan.setFilter(masterFilter); + ResultScanner scanner = table.getScanner(scan); + List testRes = new ArrayList(); + Result[] result = scanner.next(1); + int i = 0; + while (result != null && result.length > 0) { + System.out.println(Bytes.toString(result[0].getRow())); + testRes.add(result[0]); + result = scanner.next(1); + i++; + } + assertEquals(7, i); + } + + private void insert100Rows(HTable table) throws IOException { + for (int i = 0; i < 8; i++) { + Put p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes("5")); + + table.put(p); + } + } + + private void rangePutForIdx2WithInteger(HTable table) throws IOException { + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(1)); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(2)); + table.put(p); + p = new Put(Bytes.toBytes("row2")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(3)); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-4)); + table.put(p); + p = new Put(Bytes.toBytes("row4")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-5)); + table.put(p); + p = new Put(Bytes.toBytes("row5")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-6)); + table.put(p); + } + + private void rangePutForIdx2WithFloat(HTable table) throws IOException { + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(1.5f)); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(2.89f)); + table.put(p); + p = new Put(Bytes.toBytes("row2")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(3.9f)); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-4.7f)); + table.put(p); + p = new Put(Bytes.toBytes("row4")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-5.3f)); + table.put(p); + p = new Put(Bytes.toBytes("row5")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-6.456f)); + table.put(p); + } + + private void rangePutForIdx2WithDouble(HTable table) throws IOException { + Put p = new Put(Bytes.toBytes("row0")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(1.5d)); + table.put(p); + p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(2.89d)); + table.put(p); + p = new Put(Bytes.toBytes("row2")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(3.9d)); + table.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-4.7d)); + table.put(p); + p = new Put(Bytes.toBytes("row4")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-5.3d)); + table.put(p); + p = new Put(Bytes.toBytes("row5")); + p.add(Bytes.toBytes("cf1"), Bytes.toBytes("c1"), Bytes.toBytes(-6.456d)); + table.put(p); + } + + @Test(timeout = 180000) + public void testComplexRangeScan() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String tableName = "testComplexRangeScan"; + IndexSpecification spec1 = new IndexSpecification("idx1"); + IndexSpecification spec2 = new IndexSpecification("idx2"); + IndexSpecification spec3 = new IndexSpecification("idx3"); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor(tableName); + // HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + spec1.addIndexColumn(hcd, "detail", ValueType.String, 10); + spec2.addIndexColumn(hcd, "info", ValueType.String, 10); + spec3.addIndexColumn(hcd, "value", ValueType.String, 10); + htd.addFamily(hcd); + htd.addIndex(spec1); + htd.addIndex(spec2); + htd.addIndex(spec3); + String[] splitkeys = new String[9]; + + for (int i = 100, j = 0; i <= 900; i += 100, j++) { + splitkeys[j] = new Integer(i).toString(); + } + admin.createTable(htd, Bytes.toByteArrays(splitkeys)); + String rowname = "row"; + String startrow = ""; + int keys = 0; + List put = new ArrayList(); + for (int i = 1, j = 999; i < 1000; i++, j--) { + if (i % 100 == 0) { + startrow = splitkeys[keys++]; + } + Put p = new Put(Bytes.toBytes(startrow + rowname + i)); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("detail"), Bytes.toBytes(new Integer(i).toString())); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("info"), Bytes.toBytes(new Integer(j).toString())); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("value"), + Bytes.toBytes(new Integer(i % 100).toString())); + System.out.println(p); + put.add(p); + } + HTable table = new HTable(conf, tableName); + table.put(put); + + Scan s = new Scan(); + s.setCacheBlocks(true); + FilterList master = new FilterList(Operator.MUST_PASS_ONE); + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf".getBytes(), "detail".getBytes(), CompareOp.LESS_OR_EQUAL, + "6".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("cf".getBytes(), "info".getBytes(), CompareOp.GREATER_OR_EQUAL, + "992".getBytes()); + filter2.setFilterIfMissing(true); + SingleColumnValueFilter filter3 = + new SingleColumnValueFilter("cf".getBytes(), "value".getBytes(), CompareOp.EQUAL, + "9".getBytes()); + filter3.setFilterIfMissing(true); + master.addFilter(filter1); + master.addFilter(filter2); + master.addFilter(filter3); + s.setFilter(master); + // scanOperation(s, conf, tableName); + assertEquals("data consistency is missed ", 563, scanOperation(s, conf, tableName)); + System.out.println("Done ************"); + s = new Scan(); + s.setFilter(master); + s.setCaching(5); + // scanOperation(s, conf, tableName); + assertEquals("data consistency is missed ", 563, scanOperation(s, conf, tableName)); + } + + @Test(timeout = 180000) + public void testComplexRangeScanWithAnd() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String tableName = "RangeScanMetrix_2_new_id"; + IndexSpecification spec1 = new IndexSpecification("idx1"); + IndexSpecification spec2 = new IndexSpecification("idx2"); + IndexSpecification spec3 = new IndexSpecification("idx3"); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor(tableName); + // HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + spec1.addIndexColumn(hcd, "detail", ValueType.String, 10); + spec2.addIndexColumn(hcd, "info", ValueType.String, 10); + spec3.addIndexColumn(hcd, "value", ValueType.String, 10); + htd.addFamily(hcd); + htd.addIndex(spec1); + htd.addIndex(spec2); + htd.addIndex(spec3); + String[] splitkeys = new String[9]; + + for (int i = 100, j = 0; i <= 900; i += 100, j++) { + splitkeys[j] = new Integer(i).toString(); + } + admin.createTable(htd, Bytes.toByteArrays(splitkeys)); + String rowname = "row"; + String startrow = ""; + int keys = 0; + List put = new ArrayList(); + for (int i = 1, j = 999; i < 1000; i++, j--) { + if (i % 100 == 0) { + startrow = splitkeys[keys++]; + } + Put p = new Put(Bytes.toBytes(startrow + rowname + i)); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("detail"), Bytes.toBytes(new Integer(i).toString())); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("info"), Bytes.toBytes(new Integer(j).toString())); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("value"), + Bytes.toBytes(new Integer(i % 100).toString())); + System.out.println(p); + put.add(p); + } + HTable table = new HTable(conf, tableName); + table.put(put); + + Scan s = new Scan(); + s.setCacheBlocks(true); + s.setCaching(1); + FilterList master = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf".getBytes(), "detail".getBytes(), CompareOp.LESS_OR_EQUAL, + "65".getBytes()); + filter1.setFilterIfMissing(true); + SingleColumnValueFilter filter2 = + new SingleColumnValueFilter("cf".getBytes(), "info".getBytes(), CompareOp.GREATER, + "900".getBytes()); + filter2.setFilterIfMissing(true); + SingleColumnValueFilter filter3 = + new SingleColumnValueFilter("cf".getBytes(), "value".getBytes(), + CompareOp.GREATER_OR_EQUAL, "5".getBytes()); + filter3.setFilterIfMissing(true); + master.addFilter(filter1); + master.addFilter(filter2); + master.addFilter(filter3); + s.setFilter(master); + // scanOperation(s, conf, tableName); + assertEquals("data consistency is missed ", 18, scanOperation(s, conf, tableName)); + System.out.println("Done ************"); + s = new Scan(); + s.setFilter(master); + s.setCaching(5); + // scanOperation(s, conf, tableName); + assertEquals("data consistency is missed ", 18, scanOperation(s, conf, tableName)); + } + + @Test(timeout = 180000) + public void testVerifyTheStopRowIsCorrectInCaseOfGreaterOperatorsInSCVF() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + HBaseTestingUtility.getZooKeeperWatcher(UTIL); + Configuration conf = UTIL.getConfiguration(); + String tableName = "testDeleteIncosistent"; + IndexSpecification spec1 = new IndexSpecification("idx1"); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor(tableName); + // HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + spec1.addIndexColumn(hcd, "detail", ValueType.String, 10); + htd.addFamily(hcd); + htd.addIndex(spec1); + admin.createTable(htd); + HTable table = new HTable(conf, "testDeleteIncosistent"); + HTable table2 = new HTable(conf, "testDeleteIncosistent_idx"); + Put p = new Put(Bytes.toBytes("row5")); + p.add("cf".getBytes(), "detail".getBytes(), "5".getBytes()); + table2.put(IndexUtils.prepareIndexPut(p, spec1, HConstants.EMPTY_START_ROW)); + p = new Put(Bytes.toBytes("row6")); + p.add("cf".getBytes(), "detail".getBytes(), "6".getBytes()); + table.put(p); + Scan s = new Scan(); + SingleColumnValueFilter filter1 = + new SingleColumnValueFilter("cf".getBytes(), "detail".getBytes(), CompareOp.GREATER, + "5".getBytes()); + s.setFilter(filter1); + ResultScanner scanner = table.getScanner(s); + int i = 0; + for (Result result : scanner) { + i++; + } + assertEquals(1, i); + } + + private int scanOperation(Scan s, Configuration conf, String tableName) { + ResultScanner scanner = null; + int i = 0; + try { + HTable table = new HTable(conf, tableName); + scanner = table.getScanner(s); + + for (Result result : scanner) { + System.out.println(Bytes.toString(result.getRow())); + i++; + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (scanner != null) { + scanner.close(); + } + } + System.out.println("******* Return value " + i); + return i; + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanFilterEvaluator.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanFilterEvaluator.java new file mode 100644 index 0000000..cc78f8a --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanFilterEvaluator.java @@ -0,0 +1,1596 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.filter.SingleColumnRangeFilter; +import org.apache.hadoop.hbase.index.manager.IndexManager; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestScanFilterEvaluator extends TestCase { + +/* public void testCircularList() throws Exception{ + ScanFilterEvaluator.CirularDoublyLinkedList cir = new ScanFilterEvaluator.CirularDoublyLinkedList(); + cir.add(10); + cir.add(23); + cir.add(45); + Node head = cir.getHead(); + //System.out.println(head); + Node next = head; + for(int i = 0; i< 100; i++){ + next = cir.next(next); + System.out.println(next); + } + + }*/ + +/* public void testName1() throws Exception { + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + List indices = new ArrayList(); + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c1", "c2" }, "idx1")); + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c1" }, "idx2")); + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2" }, "idx3")); + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c1" }, "idx4")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.EQUAL, "a".getBytes()); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c2".getBytes(), CompareOp.EQUAL, "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c3".getBytes(), CompareOp.EQUAL, "a".getBytes()); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c4".getBytes(), CompareOp.EQUAL, "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c5".getBytes(), CompareOp.EQUAL, "a".getBytes()); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c6".getBytes(), CompareOp.GREATER, "K".getBytes()); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + // mapper.evaluate(masterFilter, indices); + }*/ + private static final String COLUMN_FAMILY = "MyCF"; + + static HRegion region = null; + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String DIR = TEST_UTIL.getDataTestDir("TestScanFilterEvaluator").toString(); + + static String method = "TestScanFilterEvaluator"; + static byte[] tableName = Bytes.toBytes(method); + static byte[] family = Bytes.toBytes("family"); + static byte[] qual = Bytes.toBytes("q"); + private final int MAX_VERSIONS = 2; + + private static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf, + byte[]... families) throws IOException { + return initHRegion(tableName, null, null, callingMethod, conf, families); + } + + /** + * @param tableName + * @param startKey + * @param stopKey + * @param callingMethod + * @param conf + * @param families + * @throws IOException + * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done. + */ + private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey, + String callingMethod, Configuration conf, byte[]... families) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + for (byte[] family : families) { + htd.addFamily(new HColumnDescriptor(family)); + } + HRegionInfo info = new HRegionInfo(htd.getName(), startKey, stopKey, false); + Path path = new Path(DIR + callingMethod); + FileSystem fs = FileSystem.get(conf); + if (fs.exists(path)) { + if (!fs.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + return HRegion.createHRegion(info, path, conf, htd); + } + + public void testName2() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, method, conf, family); + ArrayList arrayList = new ArrayList(); + List indices = arrayList; + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c1", "c2", + "c3", "c4", "c5", "c6", "c7" }, "idx1")); + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c1" }, "idx2")); + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2" }, "idx3")); + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c1" }, + "idx4")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + SingleColumnValueFilter iscvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c7".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + // filter2.addFilter(iscvf3); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + // Will throw null pointer here. + IndexSpecification indexSpec = new IndexSpecification("a"); + indexSpec + .addIndexColumn(new HColumnDescriptor(family), Bytes.toString(qual), ValueType.Int, 10); + boolean add = arrayList.add(indexSpec); + IndexManager.getInstance().addIndexForTable(this.region.getTableDesc().getNameAsString(), + arrayList); + mapper.evaluate(scan, indices, new byte[0], this.region, this.region.getTableDesc() + .getNameAsString()); + } + + public void testDiffCombinations() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, "testDiffCombinations", conf, family); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + List indices = new ArrayList(); + + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c3", + "c4", "c5", "c6" }, "idx1")); + + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c1", + "c3", "c4" }, "idx4")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + filter2.addFilter(iscvf1); + + // filter2.addFilter(iscvf3); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + + IndexManager.getInstance().addIndexForTable(this.region.getTableDesc().getNameAsString(), + indices); + mapper.evaluate(scan, indices, new byte[0], this.region, this.region.getTableDesc() + .getNameAsString()); + } + + public void testDiffCombinations1() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, "testDiffCombinations1", conf, family); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + List indices = new ArrayList(); + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c3", + "c4", }, "idx1")); + + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c1" }, + "idx2")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c5" }, "idx3")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + filter2.addFilter(iscvf1); + + // filter2.addFilter(iscvf3); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + IndexManager.getInstance().addIndexForTable(this.region.getTableDesc().getNameAsString(), + indices); + mapper.evaluate(scan, indices, new byte[0], this.region, this.region.getTableDesc() + .getNameAsString()); + } + + public void testDiffCombinations2() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, "testDiffCombinations2", conf, family); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + List indices = new ArrayList(); + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c3" }, + "idx1")); + + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c1" }, + "idx2")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c5" }, "idx3")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + filter2.addFilter(iscvf1); + + // filter2.addFilter(iscvf3); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + + IndexManager.getInstance().addIndexForTable(this.region.getTableDesc().getNameAsString(), + indices); + mapper.evaluate(scan, indices, new byte[0], this.region, this.region.getTableDesc() + .getNameAsString()); + } + + public void testDiffCombinations3() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, "testDiffCombinations3", conf, family); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + List indices = new ArrayList(); + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c3" }, + "idx1")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c1" }, "idx2")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c5" }, "idx3")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c4" }, "idx4")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c6" }, "idx5")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + filter2.addFilter(iscvf1); + + // filter2.addFilter(iscvf3); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + + IndexManager.getInstance().addIndexForTable(this.region.getTableDesc().getNameAsString(), + indices); + mapper.evaluate(scan, indices, new byte[0], this.region, this.region.getTableDesc() + .getNameAsString()); + } + + public void testWhenORWithSameColumnAppearsinDiffChild() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, "tesWhenORWithSameColumnAppearsinDiffChild", conf, family); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + List indices = new ArrayList(); + // create the indices. + indices.add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c2", "c3" }, + "idx1")); + + indices + .add(createIndexSpecification("cf1", ValueType.String, 10, new String[] { "c1" }, "idx2")); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + SingleColumnValueFilter iscvf3 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "d".getBytes()); + // filter2.addFilter(iscvf3); + + masterFilter.addFilter(filter); + masterFilter.addFilter(iscvf3); + Scan scan = new Scan(); + scan.setFilter(masterFilter); + + IndexManager.getInstance().addIndexForTable(this.region.getTableDesc().getNameAsString(), + indices); + mapper.evaluate(scan, indices, new byte[0], this.region, this.region.getTableDesc() + .getNameAsString()); + } + + public void testWhenORConditionAppears() throws Exception { + Configuration conf = HBaseConfiguration.create(); + region = initHRegion(tableName, "testWhenORConditionAppears", conf, family); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c2".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + "a".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c4".getBytes(), CompareOp.EQUAL, + "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c5".getBytes(), CompareOp.GREATER, + Bytes.toBytes(10)); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c6".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(100)); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + // System.out.println(doFiltersRestruct); + /* + * if (doFiltersRestruct instanceof FilterList) { FilterList list = ((FilterList) + * doFiltersRestruct); assertEquals(3, list.getFilters().size()); } + */ + } + + public void testORFiltersGrouping() throws Exception { + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ONE); + SingleColumnValueFilter iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + SingleColumnValueFilter iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + Filter resultFilter = mapper.doFiltersRestruct(masterFilter); + List filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnValueFilter); + assertTrue(((SingleColumnValueFilter) filterList.get(0)).getOperator().equals(CompareOp.EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + assertTrue(((SingleColumnRangeFilter) filterList.get(1)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + assertTrue(((SingleColumnRangeFilter) filterList.get(1)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + assertTrue(((SingleColumnRangeFilter) filterList.get(1)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + assertTrue(((SingleColumnRangeFilter) filterList.get(1)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "10".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "10".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(filterList.get(1) instanceof SingleColumnValueFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + assertTrue(((SingleColumnValueFilter) filterList.get(1)).getOperator().equals(CompareOp.EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "9".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(filterList.get(1) instanceof SingleColumnValueFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + assertTrue(((SingleColumnValueFilter) filterList.get(1)).getOperator().equals(CompareOp.EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "9".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 2); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(filterList.get(1) instanceof SingleColumnValueFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + assertTrue(((SingleColumnValueFilter) filterList.get(1)).getOperator().equals(CompareOp.EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "5".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "7".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "7".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "2".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "7".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + "7".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "3".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "3".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "3".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getUpperBoundOp().equals( + CompareOp.LESS_OR_EQUAL)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "3".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + "3".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER)); + + masterFilter = new FilterList(Operator.MUST_PASS_ONE); + iscvf1 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "3".getBytes()); + iscvf2 = + new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER_OR_EQUAL, + "5".getBytes()); + masterFilter.addFilter(iscvf1); + masterFilter.addFilter(iscvf2); + resultFilter = mapper.doFiltersRestruct(masterFilter); + filterList = ((FilterList) resultFilter).getFilters(); + assertTrue(filterList.size() == 1); + assertTrue(filterList.get(0) instanceof SingleColumnRangeFilter); + assertTrue(((SingleColumnRangeFilter) filterList.get(0)).getLowerBoundOp().equals( + CompareOp.GREATER_OR_EQUAL)); + + } + + /*public void testWhenColNamesAreRepeated() throws Exception { + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.EQUAL, "a".getBytes()); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.EQUAL, "K".getBytes()); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c3".getBytes(), CompareOp.EQUAL, "a".getBytes()); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c4".getBytes(), CompareOp.EQUAL, "K".getBytes()); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(3, list.getFilters().size()); + + List filters = list.getFilters(); + for (Filter filter2 : filters) { + if(filter2 instanceof SingleColumnValueFilter){ + SingleColumnValueFilter scvf = (SingleColumnValueFilter)filter2; + if(Bytes.equals(scvf.getQualifier(), "c1".getBytes())){ + assertTrue(Bytes.equals("K".getBytes(), scvf.getComparator().getValue())); + } + } + } + } + } + + public void testWhenSameColsConditionsComeMoreThanOnce() throws Exception{ + + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.GREATER, Bytes.toBytes(20)); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.LESS, Bytes.toBytes(20)); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c3".getBytes(), CompareOp.EQUAL, Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c4".getBytes(), CompareOp.EQUAL, Bytes.toBytes(100)); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + FilterList filter2 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.GREATER, Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c4".getBytes(), CompareOp.EQUAL, Bytes.toBytes(100)); + filter2.addFilter(iscvf1); + filter2.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + masterFilter.addFilter(filter2); + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(2, list.getFilters().size()); + } + + } + + public void testShouldTakeOnlyTheEqualConditionWhenGreaterAlsoComes() throws Exception { + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.GREATER, Bytes.toBytes(20)); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c2".getBytes(), CompareOp.EQUAL, Bytes.toBytes(20)); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(100)); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(3, list.getFilters().size()); + } + } + + public void testShouldTakeOnlyTheEqualConditionWhenLesserAlsoComes() throws Exception{ + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.EQUAL, Bytes.toBytes(20)); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c2".getBytes(), CompareOp.EQUAL, Bytes.toBytes(20)); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + Bytes.toBytes(100)); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(3, list.getFilters().size()); + List filters = list.getFilters(); + for (Filter filter2 : filters) { + if (filter2 instanceof SingleColumnValueFilter) { + SingleColumnValueFilter scvf = (SingleColumnValueFilter) filter2; + if (Bytes.equals(scvf.getQualifier(), "c1".getBytes())) { + assertEquals(CompareOp.EQUAL, scvf.getOperator()); + } + } + } + } + } + + public void testShouldNotIncludeFilterIfTheRangeConditionIsWrong() throws Exception{ + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.LESS, Bytes.toBytes(10)); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c2".getBytes(), CompareOp.EQUAL, Bytes.toBytes(20)); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + Bytes.toBytes(100)); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(2, list.getFilters().size()); + } + } + + public void testShouldTakeOnlyTheHighestFilterWhenTwoGreaterConditonsAreFound() throws Exception{ + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.GREATER, Bytes.toBytes(10)); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c2".getBytes(), CompareOp.EQUAL, Bytes.toBytes(20)); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, + Bytes.toBytes(100)); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(3, list.getFilters().size()); + List filters = list.getFilters(); + for (Filter filter2 : filters) { + if (filter2 instanceof SingleColumnValueFilter) { + SingleColumnValueFilter scvf = (SingleColumnValueFilter) filter2; + if (Bytes.equals(scvf.getQualifier(), "c1".getBytes())) { + assertTrue(Bytes.equals(Bytes.toBytes(100), scvf.getComparator().getValue())); + } + } + } + } + } + + public void testShouldTakeOnlyTheLowestFilterWhenTwoLesserConditonsAreFound() throws Exception{ + ScanFilterEvaluator mapper = new ScanFilterEvaluator(); + FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); + // create the filter + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + IndexedSingleColumnValueFilter iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c1".getBytes(), CompareOp.LESS, Bytes.toBytes(10)); + IndexedSingleColumnValueFilter iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), + "c2".getBytes(), CompareOp.EQUAL, Bytes.toBytes(20)); + filter.addFilter(iscvf1); + filter.addFilter(iscvf2); + + FilterList filter1 = new FilterList(Operator.MUST_PASS_ALL); + iscvf1 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c3".getBytes(), CompareOp.EQUAL, + Bytes.toBytes(10)); + iscvf2 = new IndexedSingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.LESS, + Bytes.toBytes(1)); + filter1.addFilter(iscvf1); + filter1.addFilter(iscvf2); + + masterFilter.addFilter(filter); + masterFilter.addFilter(filter1); + + Filter doFiltersRestruct = mapper.doFiltersRestruct(masterFilter); + if (doFiltersRestruct instanceof FilterList) { + FilterList list = ((FilterList) doFiltersRestruct); + assertEquals(3, list.getFilters().size()); + List filters = list.getFilters(); + for (Filter filter2 : filters) { + if (filter2 instanceof SingleColumnValueFilter) { + SingleColumnValueFilter scvf = (SingleColumnValueFilter) filter2; + if (Bytes.equals(scvf.getQualifier(), "c1".getBytes())) { + assertTrue(Bytes.equals(Bytes.toBytes(1), scvf.getComparator().getValue())); + } + } + } + } + } +*/ + private IndexSpecification createIndexSpecification(String cf, ValueType type, + int maxValueLength, String[] qualifiers, String name) { + IndexSpecification index = new IndexSpecification(name.getBytes()); + for (String qualifier : qualifiers) { + index.addIndexColumn(new HColumnDescriptor(cf), qualifier, type, maxValueLength); + } + return index; + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanFilterEvaluatorForIndexInScan.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanFilterEvaluatorForIndexInScan.java new file mode 100644 index 0000000..63659ff --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanFilterEvaluatorForIndexInScan.java @@ -0,0 +1,259 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import static org.junit.Assert.assertNull; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.Column; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.GroupingCondition; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.client.EqualsExpression; +import org.apache.hadoop.hbase.index.client.IndexExpression; +import org.apache.hadoop.hbase.index.client.IndexUtils; +import org.apache.hadoop.hbase.index.client.MultiIndexExpression; +import org.apache.hadoop.hbase.index.client.NoIndexExpression; +import org.apache.hadoop.hbase.index.client.RangeExpression; +import org.apache.hadoop.hbase.index.client.SingleIndexExpression; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestScanFilterEvaluatorForIndexInScan { + + private static final byte[] FAMILY1 = Bytes.toBytes("cf1"); + private static final byte[] FAMILY2 = Bytes.toBytes("cf2"); + private static final String COL1 = "c1"; + private static final String COL2 = "c2"; + private static final String COL3 = "c3"; + private static final byte[] QUALIFIER1 = Bytes.toBytes(COL1); + private static final byte[] QUALIFIER2 = Bytes.toBytes(COL2); + private static final byte[] QUALIFIER3 = Bytes.toBytes(COL3); + private static final String tableName = "tab1"; + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String DIR = TEST_UTIL.getDataTestDir( + "TestScanFilterEvaluatorForIndexInScan").toString(); + + @Test + public void testSingleIndexExpressionWithOneEqualsExpression() throws Exception { + String indexName = "idx1"; + SingleIndexExpression singleIndexExpression = new SingleIndexExpression(indexName); + byte[] value = "1".getBytes(); + Column column = new Column(FAMILY1, QUALIFIER1); + EqualsExpression equalsExpression = new EqualsExpression(column, value); + singleIndexExpression.addEqualsExpression(equalsExpression); + + Scan scan = new Scan(); + scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(singleIndexExpression)); + Filter filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER1, CompareOp.EQUAL, value); + scan.setFilter(filter); + ScanFilterEvaluator evaluator = new ScanFilterEvaluator(); + List indices = new ArrayList(); + IndexSpecification index = new IndexSpecification(indexName); + HColumnDescriptor colDesc = new HColumnDescriptor(FAMILY1); + index.addIndexColumn(colDesc, COL1, ValueType.String, 10); + indices.add(index); + HRegion region = + initHRegion(tableName.getBytes(), null, null, + "testSingleIndexExpressionWithOneEqualsExpression", TEST_UTIL.getConfiguration(), FAMILY1); + IndexRegionScanner scanner = evaluator.evaluate(scan, indices, new byte[0], region, tableName); + // TODO add assertions + } + + @Test + public void testSingleIndexExpressionWithMoreEqualsExpsAndOneRangeExp() throws Exception { + String indexName = "idx1"; + SingleIndexExpression singleIndexExpression = new SingleIndexExpression(indexName); + byte[] value1 = "1".getBytes(); + byte[] value2 = Bytes.toBytes(1234); + Column column = new Column(FAMILY1, QUALIFIER1); + EqualsExpression equalsExpression = new EqualsExpression(column, value1); + singleIndexExpression.addEqualsExpression(equalsExpression); + column = new Column(FAMILY1, QUALIFIER2); + equalsExpression = new EqualsExpression(column, value2); + singleIndexExpression.addEqualsExpression(equalsExpression); + column = new Column(FAMILY1, QUALIFIER3); + byte[] value3_1 = Bytes.toBytes(10.4F); + byte[] value3_2 = Bytes.toBytes(16.91F); + RangeExpression re = new RangeExpression(column, value3_1, value3_2, true, false); + singleIndexExpression.setRangeExpression(re); + + Scan scan = new Scan(); + scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(singleIndexExpression)); + FilterList fl = new FilterList(Operator.MUST_PASS_ALL); + Filter filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER1, CompareOp.EQUAL, value1); + fl.addFilter(filter); + filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER2, CompareOp.EQUAL, value2); + fl.addFilter(filter); + filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER3, CompareOp.GREATER_OR_EQUAL, value3_1); + fl.addFilter(filter); + filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER3, CompareOp.LESS, value3_2); + fl.addFilter(filter); + scan.setFilter(fl); + + ScanFilterEvaluator evaluator = new ScanFilterEvaluator(); + List indices = new ArrayList(); + IndexSpecification index = new IndexSpecification(indexName); + HColumnDescriptor colDesc = new HColumnDescriptor(FAMILY1); + index.addIndexColumn(colDesc, COL1, ValueType.String, 10); + index.addIndexColumn(colDesc, COL2, ValueType.Int, 4); + index.addIndexColumn(colDesc, COL3, ValueType.Float, 4); + indices.add(index); + + HRegion region = + initHRegion(tableName.getBytes(), null, null, + "testSingleIndexExpressionWithMoreEqualsExpsAndOneRangeExp", + TEST_UTIL.getConfiguration(), FAMILY1); + IndexRegionScanner scanner = evaluator.evaluate(scan, indices, new byte[0], region, tableName); + // TODO add assertions + } + + @Test + public void testMultiIndexExpression() throws Exception { + MultiIndexExpression multiIndexExpression = new MultiIndexExpression(GroupingCondition.AND); + String index1 = "idx1"; + SingleIndexExpression singleIndexExpression = new SingleIndexExpression(index1); + byte[] value2 = Bytes.toBytes(1234); + Column column = new Column(FAMILY1, QUALIFIER2); + EqualsExpression equalsExpression = new EqualsExpression(column, value2); + singleIndexExpression.addEqualsExpression(equalsExpression); + column = new Column(FAMILY1, QUALIFIER3); + byte[] value3_1 = Bytes.toBytes(10.4F); + byte[] value3_2 = Bytes.toBytes(16.91F); + RangeExpression re = new RangeExpression(column, value3_1, value3_2, true, false); + singleIndexExpression.setRangeExpression(re); + multiIndexExpression.addIndexExpression(singleIndexExpression); + + MultiIndexExpression multiIndexExpression2 = new MultiIndexExpression(GroupingCondition.OR); + String index2 = "idx2"; + singleIndexExpression = new SingleIndexExpression(index2); + byte[] value1 = Bytes.toBytes("asdf"); + column = new Column(FAMILY1, QUALIFIER1); + equalsExpression = new EqualsExpression(column, value1); + singleIndexExpression.addEqualsExpression(equalsExpression); + multiIndexExpression2.addIndexExpression(singleIndexExpression); + + String index3 = "idx3"; + singleIndexExpression = new SingleIndexExpression(index3); + byte[] value4 = Bytes.toBytes(567.009D); + column = new Column(FAMILY2, QUALIFIER1); + equalsExpression = new EqualsExpression(column, value4); + singleIndexExpression.addEqualsExpression(equalsExpression); + multiIndexExpression2.addIndexExpression(singleIndexExpression); + + multiIndexExpression.addIndexExpression(multiIndexExpression2); + + Scan scan = new Scan(); + scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(multiIndexExpression)); + FilterList outerFL = new FilterList(Operator.MUST_PASS_ALL); + FilterList fl = new FilterList(Operator.MUST_PASS_ALL); + Filter filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER2, CompareOp.EQUAL, value2); + fl.addFilter(filter); + filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER3, CompareOp.GREATER_OR_EQUAL, value3_1); + fl.addFilter(filter); + filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER3, CompareOp.LESS, value3_2); + fl.addFilter(filter); + outerFL.addFilter(fl); + FilterList innerFL = new FilterList(Operator.MUST_PASS_ONE); + innerFL.addFilter(new SingleColumnValueFilter(FAMILY1, QUALIFIER1, CompareOp.EQUAL, value1)); + innerFL.addFilter(new SingleColumnValueFilter(FAMILY2, QUALIFIER1, CompareOp.EQUAL, value4)); + outerFL.addFilter(innerFL); + scan.setFilter(outerFL); + + ScanFilterEvaluator evaluator = new ScanFilterEvaluator(); + List indices = new ArrayList(); + IndexSpecification is1 = new IndexSpecification(index1); + HColumnDescriptor colDesc = new HColumnDescriptor(FAMILY1); + is1.addIndexColumn(colDesc, COL2, ValueType.Int, 4); + is1.addIndexColumn(colDesc, COL3, ValueType.Float, 4); + indices.add(is1); + IndexSpecification is2 = new IndexSpecification(index2); + is2.addIndexColumn(colDesc, COL1, ValueType.String, 15); + indices.add(is2); + IndexSpecification is3 = new IndexSpecification(index3); + colDesc = new HColumnDescriptor(FAMILY2); + is3.addIndexColumn(colDesc, COL1, ValueType.Double, 8); + indices.add(is3); + + HRegion region = + initHRegion(tableName.getBytes(), null, null, "testMultiIndexExpression", + TEST_UTIL.getConfiguration(), FAMILY1); + IndexRegionScanner scanner = evaluator.evaluate(scan, indices, new byte[0], region, tableName); + // TODO add assertions + } + + @Test + public void testNoIndexExpression() throws Exception { + IndexExpression exp = new NoIndexExpression(); + Scan scan = new Scan(); + scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(exp)); + byte[] value1 = Bytes.toBytes("asdf"); + scan.setFilter(new SingleColumnValueFilter(FAMILY1, QUALIFIER1, CompareOp.EQUAL, value1)); + List indices = new ArrayList(); + IndexSpecification is1 = new IndexSpecification("idx1"); + HColumnDescriptor colDesc = new HColumnDescriptor(FAMILY1); + is1.addIndexColumn(colDesc, COL1, ValueType.String, 15); + indices.add(is1); + ScanFilterEvaluator evaluator = new ScanFilterEvaluator(); + HRegion region = + initHRegion(tableName.getBytes(), null, null, "testNoIndexExpression", + TEST_UTIL.getConfiguration(), FAMILY1); + IndexRegionScanner scanner = evaluator.evaluate(scan, indices, new byte[0], region, tableName); + assertNull(scanner); + } + + private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey, + String callingMethod, Configuration conf, byte[]... families) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + for (byte[] family : families) { + htd.addFamily(new HColumnDescriptor(family)); + } + HRegionInfo info = new HRegionInfo(htd.getName(), startKey, stopKey, false); + Path path = new Path(DIR + callingMethod); + FileSystem fs = FileSystem.get(conf); + if (fs.exists(path)) { + if (!fs.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + return HRegion.createHRegion(info, path, conf, htd); + } +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanWhenTTLExpired.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanWhenTTLExpired.java new file mode 100644 index 0000000..771e2b0 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/TestScanWhenTTLExpired.java @@ -0,0 +1,332 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.coprocessor.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestScanWhenTTLExpired { + + private HBaseAdmin admin = null; + private MiniHBaseCluster cluster = null; + private static final int NB_SERVERS = 1; + private static final int TTL_SECONDS = 2; + private static final int TTL_MS = TTL_SECONDS * 1000; + + private static final HBaseTestingUtility TESTING_UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void before() throws Exception { + Configuration conf = TESTING_UTIL.getConfiguration(); + conf.setInt("hbase.balancer.period", 60000); + // Needed because some tests have splits happening on RS that are killed + // We don't want to wait 3min for the master to figure it out + conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 4000); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + TESTING_UTIL.startMiniCluster(NB_SERVERS); + } + + @AfterClass + public static void after() throws Exception { + TESTING_UTIL.shutdownMiniCluster(); + } + + @Before + public void setup() throws IOException { + TESTING_UTIL.ensureSomeRegionServersAvailable(NB_SERVERS); + this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration()); + this.cluster = TESTING_UTIL.getMiniHBaseCluster(); + } + + @Test(timeout = 180000) + public void testScannerSelectionWhenPutHasOneColumn() throws IOException, KeeperException, + InterruptedException { + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL); + String userTableName = "testScannerSelectionWhenPutHasOneColumn"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + + HColumnDescriptor hcd = + new HColumnDescriptor("col").setMaxVersions(Integer.MAX_VALUE).setTimeToLive(TTL_SECONDS); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + Configuration conf = TESTING_UTIL.getConfiguration(); + HTable table = new HTable(conf, userTableName); + // test put with the indexed column + Put p = new Put("row1".getBytes()); + p.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + table.put(p); + + Put p1 = new Put("row01".getBytes()); + p1.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + table.put(p1); + + Put p2 = new Put("row010".getBytes()); + p2.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + table.put(p2); + + Put p3 = new Put("row001".getBytes()); + p3.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + table.put(p3); + + admin.flush(userTableName); + + HRegionServer regionServer = TESTING_UTIL.getHBaseCluster().getRegionServer(0); + List onlineRegions = regionServer.getOnlineRegions(Bytes.toBytes(userTableName)); + List storeFileList = + regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + + for (String store : storeFileList) { + Threads.sleepWithoutInterrupt(TTL_MS); + } + int i = countNumberOfRowsWithFilter(userTableName, "Val", true, false, 0); + assertEquals("No rows should be retrieved", 0, i); + } + + @Test(timeout = 180000) + public void testScannerSelectionWhenThereAreMutlipleCFs() throws IOException, KeeperException, + InterruptedException { + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL); + String userTableName = "testScannerSelectionWhenThereAreMutlipleCFs"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + + HColumnDescriptor hcd = + new HColumnDescriptor("col").setMaxVersions(Integer.MAX_VALUE).setTimeToLive( + Integer.MAX_VALUE); + HColumnDescriptor hcd1 = + new HColumnDescriptor("col1").setMaxVersions(Integer.MAX_VALUE).setTimeToLive( + TTL_SECONDS - 1); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec.addIndexColumn(hcd1, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addFamily(hcd1); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + Configuration conf = TESTING_UTIL.getConfiguration(); + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + + Put p = new Put("row1".getBytes()); + p.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p); + + Put p1 = new Put("row01".getBytes()); + p1.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p1.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p1); + + Put p2 = new Put("row010".getBytes()); + p2.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p2.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p2); + + Put p3 = new Put("row001".getBytes()); + p3.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p3.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p3); + + admin.flush(userTableName); + + HRegionServer regionServer = TESTING_UTIL.getHBaseCluster().getRegionServer(0); + List onlineRegions = regionServer.getOnlineRegions(Bytes.toBytes(userTableName)); + List storeFileList = + regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + + for (String store : storeFileList) { + Threads.sleepWithoutInterrupt(TTL_MS); + } + int i = countNumberOfRowsWithFilter(userTableName, "Val", true, false, 0); + assertEquals("No rows should be retrieved", 0, i); + + } + + @Test(timeout = 180000) + public void testCompactionOnIndexTableShouldNotRetrieveTTLExpiredData() throws Exception { + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL); + String userTableName = "testCompactionOnIndexTableShouldNotRetrieveTTLExpiredData"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + + HColumnDescriptor hcd = + new HColumnDescriptor("col").setMaxVersions(Integer.MAX_VALUE).setTimeToLive( + TTL_SECONDS - 1); + HColumnDescriptor hcd1 = + new HColumnDescriptor("col1").setMaxVersions(Integer.MAX_VALUE).setTimeToLive( + TTL_SECONDS - 1); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec.addIndexColumn(hcd1, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addFamily(hcd1); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + + Configuration conf = TESTING_UTIL.getConfiguration(); + HTable table = new HTable(conf, userTableName); + + // test put with the indexed column + + Put p = new Put("row1".getBytes()); + p.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p); + admin.flush(userTableName + "_idx"); + + Put p1 = new Put("row01".getBytes()); + p1.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p1.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p1); + admin.flush(userTableName + "_idx"); + + Put p2 = new Put("row010".getBytes()); + p2.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p2.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p2); + admin.flush(userTableName + "_idx"); + + Put p3 = new Put("row001".getBytes()); + p3.add(Bytes.toBytes("col"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); + p3.add("col1".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); + table.put(p3); + + admin.flush(userTableName); + admin.flush(userTableName + "_idx"); + + HRegionServer regionServer = TESTING_UTIL.getHBaseCluster().getRegionServer(0); + List onlineRegions = regionServer.getOnlineRegions(Bytes.toBytes(userTableName)); + List storeFileList = + regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + onlineRegions = regionServer.getOnlineRegions(Bytes.toBytes(userTableName + "_idx")); + storeFileList = regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + while (storeFileList.size() < 4) { + Thread.sleep(1000); + storeFileList = regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + } + int prevSize = storeFileList.size(); + assertEquals("The total store files for the index table should be 4", 4, prevSize); + Scan s = new Scan(); + HTable indexTable = new HTable(conf, userTableName + "_idx"); + ResultScanner scanner = indexTable.getScanner(s); + // Result res = scanner.next(); + for (Result result : scanner) { + System.out.println(result); + } + for (String store : storeFileList) { + Threads.sleepWithoutInterrupt(TTL_MS); + } + admin.compact(userTableName + "_idx"); + + onlineRegions = regionServer.getOnlineRegions(Bytes.toBytes(userTableName + "_idx")); + storeFileList = regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + while (storeFileList.size() != 1) { + Thread.sleep(1000); + storeFileList = regionServer.getStoreFileList(onlineRegions.get(0).getRegionName()); + } + assertEquals("The total store files for the index table should be 1", 1, storeFileList.size()); + s = new Scan(); + indexTable = new HTable(conf, userTableName + "_idx"); + scanner = indexTable.getScanner(s); + // Result res = scanner.next(); + boolean dataAvailable = false; + for (Result result : scanner) { + dataAvailable = true; + System.out.println(result); + } + assertFalse("dataShould not be retrieved", dataAvailable); + + } + + private int countNumberOfRowsWithFilter(String tableName, String filterVal, boolean isIndexed, + boolean isCached, int cacheNumber) throws IOException { + Configuration conf = TESTING_UTIL.getConfiguration(); + HTable table = new HTable(conf, tableName); + Scan s = new Scan(); + Filter filter = null; + if (isIndexed) { + filter = + new SingleColumnValueFilter(Bytes.toBytes("col"), Bytes.toBytes("q1"), CompareOp.EQUAL, + filterVal.getBytes()); + } else { + filter = + new SingleColumnValueFilter(Bytes.toBytes("col"), Bytes.toBytes("q1"), CompareOp.EQUAL, + "cat".getBytes()); + } + s.setFilter(filter); + if (isCached) { + s.setCaching(cacheNumber); + } + int i = 0; + ResultScanner scanner = table.getScanner(s); + for (Result result : scanner) { + i++; + } + return i; + } +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/io/TestIndexHalfStoreFileReader.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/io/TestIndexHalfStoreFileReader.java new file mode 100644 index 0000000..3c474a5 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/io/TestIndexHalfStoreFileReader.java @@ -0,0 +1,235 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.io; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestIndexHalfStoreFileReader { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private KeyValue seekToKeyVal; + private byte[] expectedRow; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test + public void testIndexHalfStoreFileReaderWithSeekTo() throws Exception { + HBaseTestingUtility test_util = new HBaseTestingUtility(); + String root_dir = test_util.getDataTestDir("TestIndexHalfStoreFile").toString(); + Path p = new Path(root_dir, "test"); + Configuration conf = test_util.getConfiguration(); + FileSystem fs = FileSystem.get(conf); + CacheConfig cacheConf = new CacheConfig(conf); + HFile.Writer w = + HFile.getWriterFactory(conf, cacheConf).withPath(fs, p).withBlockSize(1024) + .withComparator(KeyValue.KEY_COMPARATOR).create(); + String usertableName = "testIndexHalfStore"; + List items = genSomeKeys(usertableName); + for (KeyValue kv : items) { + w.append(kv); + } + w.close(); + HFile.Reader r = HFile.createReader(fs, p, cacheConf); + r.loadFileInfo(); + byte[] midkey = "005".getBytes(); + Reference top = new Reference(midkey, Reference.Range.top); + doTestOfScanAndReseek(p, fs, top, cacheConf); + r.close(); + } + + private void + doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, CacheConfig cacheConf) + throws IOException { + final IndexHalfStoreFileReader halfreader = + new IndexHalfStoreFileReader(fs, p, cacheConf, bottom, DataBlockEncoding.NONE); + halfreader.loadFileInfo(); + final HFileScanner scanner = halfreader.getScanner(false, false); + KeyValue getseekTorowKey3 = getSeekToRowKey(); + scanner.seekTo(getseekTorowKey3.getBuffer(), 8, 17); + boolean next = scanner.next(); + KeyValue keyValue = null; + if (next) { + keyValue = scanner.getKeyValue(); + } + byte[] expectedRow = getExpected(); + byte[] actualRow = keyValue.getRow(); + Assert.assertArrayEquals(expectedRow, actualRow); + halfreader.close(true); + } + + private List genSomeKeys(String userTableName) throws Exception { + List ret = new ArrayList(4); + HBaseAdmin admin = UTIL.getHBaseAdmin(); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd1 = new HColumnDescriptor("column1"); + HColumnDescriptor hcd2 = new HColumnDescriptor("column2"); + IndexSpecification iSpec1 = new IndexSpecification("Index"); + iSpec1.addIndexColumn(hcd1, "q", ValueType.String, 10); + iSpec1.addIndexColumn(hcd2, "q", ValueType.String, 10); + ihtd.addFamily(hcd1); + ihtd.addFamily(hcd2); + ihtd.addIndex(iSpec1); + admin.createTable(ihtd); + ZKAssign.blockUntilNoRIT(zkw); + ByteArrayBuilder indexColVal = ByteArrayBuilder.allocate(4); + indexColVal.put(Bytes.toBytes((short) 3)); + indexColVal.put(Bytes.toBytes((short) 32)); + + Put p1 = generatePuts("006".getBytes(), "05".getBytes()); + Put p2 = generatePuts("003".getBytes(), "06".getBytes()); + Put p3 = generatePuts("004".getBytes(), "06".getBytes()); + Put p4 = generatePuts("007".getBytes(), "06".getBytes()); + + byte[] seekToPut = + new byte[3 + 1 + IndexUtils.getMaxIndexNameLength() + 10 + "006".getBytes().length]; + System.arraycopy(p1.getRow(), 0, seekToPut, 0, p1.getRow().length); + byte[] seekToRow = "007".getBytes(); + System.arraycopy(seekToRow, 0, seekToPut, p1.getRow().length - 3, seekToRow.length); + System.arraycopy("005".getBytes(), 0, seekToPut, 0, 3); + setSeekToRowKey(seekToPut, indexColVal); + + byte[] expectedPut = + new byte[3 + 1 + IndexUtils.getMaxIndexNameLength() + 10 + "006".getBytes().length]; + System.arraycopy(p4.getRow(), 0, expectedPut, 0, p4.getRow().length); + // Copy first 3 bytes to splitKey since getKeyValue will replace the start key with splitKey. + // Just for assertion this is been added + System.arraycopy("005".getBytes(), 0, expectedPut, 0, 3); + setExpected(expectedPut); + + KeyValue kv = + new KeyValue(p1.getRow(), Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, 0, + indexColVal.array()); + ret.add(kv); + KeyValue kv1 = + new KeyValue(p2.getRow(), Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, 0, + indexColVal.array()); + ret.add(kv1); + KeyValue kv2 = + new KeyValue(p3.getRow(), Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, 0, + indexColVal.array()); + ret.add(kv2); + KeyValue kv3 = + new KeyValue(p4.getRow(), Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, 0, + indexColVal.array()); + ret.add(kv3); + return ret; + } + + private void setSeekToRowKey(byte[] seekTorowKey3, ByteArrayBuilder indexColVal) { + KeyValue kv = + new KeyValue(seekTorowKey3, Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, 0, + indexColVal.array()); + this.seekToKeyVal = kv; + } + + private KeyValue getSeekToRowKey() { + return this.seekToKeyVal; + } + + private void setExpected(byte[] expected) { + this.expectedRow = expected; + } + + private byte[] getExpected() { + return this.expectedRow; + } + + private Put generatePuts(byte[] rowKey, byte[] val) throws Exception { + Configuration conf = UTIL.getConfiguration(); + String usertableName = "testIndexHalfStore"; + HTable table = new HTable(conf, usertableName + "_idx"); + byte[] onebyte = new byte[1]; + String indexName = "Indexname"; + byte[] remIndex = new byte[IndexUtils.getMaxIndexNameLength() - indexName.length()]; + byte[] valPad = new byte[8]; + ByteArrayBuilder indexColVal = ByteArrayBuilder.allocate(4); + indexColVal.put(Bytes.toBytes((short) 3)); + indexColVal.put(Bytes.toBytes((short) 32)); + byte[] put = new byte[3 + 1 + IndexUtils.getMaxIndexNameLength() + 10 + rowKey.length]; + System.arraycopy("000".getBytes(), 0, put, 0, 3); + System.arraycopy(onebyte, 0, put, 3, onebyte.length); + System.arraycopy(indexName.getBytes(), 0, put, 3 + onebyte.length, indexName.length()); + System.arraycopy(remIndex, 0, put, 3 + onebyte.length + indexName.length(), remIndex.length); + System.arraycopy(val, 0, put, 3 + onebyte.length + indexName.length() + remIndex.length, + val.length); + System.arraycopy(valPad, 0, put, 3 + onebyte.length + indexName.length() + remIndex.length + + val.length, valPad.length); + System.arraycopy(rowKey, 0, put, 3 + onebyte.length + indexName.length() + remIndex.length + + val.length + valPad.length, rowKey.length); + Put p = new Put(put); + p.add(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, 0, indexColVal.array()); + table.put(p); + return p; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/io/TestIndexHalfStoreFileReaderWithEncoding.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/io/TestIndexHalfStoreFileReaderWithEncoding.java new file mode 100644 index 0000000..2cba2a8 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/io/TestIndexHalfStoreFileReaderWithEncoding.java @@ -0,0 +1,131 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.io; + +import static org.junit.Assert.assertEquals; + +import java.util.Map.Entry; +import java.util.NavigableMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestIndexHalfStoreFileReaderWithEncoding { + private static final String COL1 = "c1"; + private static final String CF1 = "cf1"; + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + conf.set("index.data.block.encoding.algo", "PREFIX"); + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testWithFastDiffEncoding() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + String tableName = "testWithFastDiffEncoding"; + String idxTableName = "testWithFastDiffEncoding_idx"; + IndexedHTableDescriptor htd = createIndexedHTableDescriptor(tableName, CF1, "idx1", CF1, COL1); + admin.createTable(htd); + HTable ht = new HTable(UTIL.getConfiguration(), tableName); + HTable hti = new HTable(UTIL.getConfiguration(), idxTableName); + Put put = new Put("a".getBytes()); + put.add(CF1.getBytes(), COL1.getBytes(), "1".getBytes()); + ht.put(put); + put = new Put("d".getBytes()); + put.add(CF1.getBytes(), COL1.getBytes(), "1".getBytes()); + ht.put(put); + put = new Put("k".getBytes()); + put.add(CF1.getBytes(), COL1.getBytes(), "1".getBytes()); + ht.put(put); + put = new Put("z".getBytes()); + put.add(CF1.getBytes(), COL1.getBytes(), "1".getBytes()); + ht.put(put); + Delete delete = new Delete("z".getBytes()); + ht.delete(delete); + admin.flush(tableName); + admin.flush(idxTableName); + NavigableMap regionLocations = ht.getRegionLocations(); + byte[] regionName = null; + for (Entry e : regionLocations.entrySet()) { + regionName = e.getKey().getRegionName(); + break; + } + // Splitting the single region. + admin.split(regionName, "e".getBytes()); + // Sleeping so that the compaction can complete. + // Split will initiate a compaction. + Thread.sleep(5 * 1000); + Scan scan = new Scan(); + ResultScanner scanner = hti.getScanner(scan); + Result res = scanner.next(); + int count = 0; + while (res != null) { + count++; + res = scanner.next(); + } + assertEquals(3, count); + } + + private IndexedHTableDescriptor createIndexedHTableDescriptor(String tableName, + String columnFamily, String indexName, String indexColumnFamily, String indexColumnQualifier) { + IndexedHTableDescriptor htd = new IndexedHTableDescriptor(tableName); + IndexSpecification iSpec = new IndexSpecification(indexName); + HColumnDescriptor hcd = new HColumnDescriptor(columnFamily); + iSpec.addIndexColumn(hcd, indexColumnQualifier, ValueType.String, 10); + htd.addFamily(hcd); + htd.addIndex(iSpec); + return htd; + } +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/manager/TestIndexManager.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/manager/TestIndexManager.java new file mode 100644 index 0000000..42a503b --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/manager/TestIndexManager.java @@ -0,0 +1,107 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.manager; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import junit.framework.TestCase; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.index.ColumnQualifier; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.junit.experimental.categories.Category; + +/** + * Test index manager functionality. + */ + +@Category(MediumTests.class) +public class TestIndexManager extends TestCase { + public void testAddIndexForTable() throws Exception { + + IndexManager im = IndexManager.getInstance(); + assertNotNull("Index Manager should not be null.", im); + + List indexList = new ArrayList(1); + IndexSpecification iSpec = new IndexSpecification("index_name"); + + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", null, 10); + indexList.add(iSpec); + im.addIndexForTable("index_name", indexList); + indexList = im.getIndicesForTable("index_name"); + assertEquals("Index name should be equal with actual value.", "index_name", indexList.get(0) + .getName()); + assertTrue("Column qualifier state mismatch.", + indexList.get(0).getIndexColumns().contains(new ColumnQualifier("cf", "cq", null, 10))); + + } + + public void testShouldNotThrowNPEIfValueTypeIsNull() throws Exception { + IndexManager im = IndexManager.getInstance(); + assertNotNull("Index Manager should not be null.", im); + + List indexList = new ArrayList(1); + IndexSpecification iSpec = new IndexSpecification("index_name"); + + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", null, 5); + indexList.add(iSpec); + im.addIndexForTable("index_name", indexList); + indexList = im.getIndicesForTable("index_name"); + + Set indexColumns = indexList.get(0).getIndexColumns(); + for (ColumnQualifier columnQualifier : indexColumns) { + assertNotNull(columnQualifier.getType()); + } + } + + public void testAddIndexForTableWhenStringAndValLengthIsZero() throws Exception { + IndexManager im = IndexManager.getInstance(); + assertNotNull("Index Manager should not be null.", im); + + List indexList = new ArrayList(1); + IndexSpecification iSpec = new IndexSpecification("index_name"); + + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", null, 0); + indexList.add(iSpec); + im.addIndexForTable("index_name", indexList); + indexList = im.getIndicesForTable("index_name"); + assertEquals("the total value length should be 2", 2, indexList.get(0).getTotalValueLength()); + } + + public void testRemoveIndicesForTable() throws Exception { + + IndexManager im = IndexManager.getInstance(); + assertNotNull("Index Manager should not be null.", im); + + List indexList = new ArrayList(1); + IndexSpecification iSpec = new IndexSpecification("index_name"); + + iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", null, 10); + indexList.add(iSpec); + im.removeIndices("index_name"); + indexList = im.getIndicesForTable("index_name"); + assertNull("Index specification List should be null.", indexList); + + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/mapreduce/TestIndexImportTsv.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/mapreduce/TestIndexImportTsv.java new file mode 100644 index 0000000..8df4356 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/mapreduce/TestIndexImportTsv.java @@ -0,0 +1,352 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.index.mapreduce.IndexImportTsv.TsvParser; +import org.apache.hadoop.hbase.index.mapreduce.IndexImportTsv.TsvParser.BadTsvLineException; +import org.apache.hadoop.hbase.index.mapreduce.IndexImportTsv.TsvParser.ParsedLine; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.util.GenericOptionsParser; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; + +@Category(MediumTests.class) +public class TestIndexImportTsv { + + @Test + public void testTsvParserSpecParsing() { + TsvParser parser; + + parser = new TsvParser("HBASE_ROW_KEY", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertFalse(parser.hasTimestamp()); + + parser = new TsvParser("HBASE_ROW_KEY,col1:scol1", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1)); + assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertFalse(parser.hasTimestamp()); + + parser = new TsvParser("HBASE_ROW_KEY,col1:scol1,col1:scol2", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1)); + assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(2)); + assertBytesEquals(Bytes.toBytes("scol2"), parser.getQualifier(2)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertFalse(parser.hasTimestamp()); + + parser = new TsvParser("HBASE_ROW_KEY,col1:scol1,HBASE_TS_KEY,col1:scol2", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1)); + assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(3)); + assertBytesEquals(Bytes.toBytes("scol2"), parser.getQualifier(3)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertTrue(parser.hasTimestamp()); + assertEquals(2, parser.getTimestampKeyColumnIndex()); + } + + @Test + public void testTsvParser() throws BadTsvLineException { + TsvParser parser = new TsvParser("col_a,col_b:qual,HBASE_ROW_KEY,col_d", "\t"); + assertBytesEquals(Bytes.toBytes("col_a"), parser.getFamily(0)); + assertBytesEquals(HConstants.EMPTY_BYTE_ARRAY, parser.getQualifier(0)); + assertBytesEquals(Bytes.toBytes("col_b"), parser.getFamily(1)); + assertBytesEquals(Bytes.toBytes("qual"), parser.getQualifier(1)); + assertNull(parser.getFamily(2)); + assertNull(parser.getQualifier(2)); + assertEquals(2, parser.getRowKeyColumnIndex()); + + assertEquals(TsvParser.DEFAULT_TIMESTAMP_COLUMN_INDEX, parser.getTimestampKeyColumnIndex()); + + byte[] line = Bytes.toBytes("val_a\tval_b\tval_c\tval_d"); + ParsedLine parsed = parser.parse(line, line.length); + checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line))); + } + + @Test + public void testTsvParserWithTimestamp() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,HBASE_TS_KEY,col_a,", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertNull(parser.getFamily(1)); + assertNull(parser.getQualifier(1)); + assertBytesEquals(Bytes.toBytes("col_a"), parser.getFamily(2)); + assertBytesEquals(HConstants.EMPTY_BYTE_ARRAY, parser.getQualifier(2)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertEquals(1, parser.getTimestampKeyColumnIndex()); + + byte[] line = Bytes.toBytes("rowkey\t1234\tval_a"); + ParsedLine parsed = parser.parse(line, line.length); + assertEquals(1234l, parsed.getTimestamp(-1)); + checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line))); + } + + private void checkParsing(ParsedLine parsed, Iterable expected) { + ArrayList parsedCols = new ArrayList(); + for (int i = 0; i < parsed.getColumnCount(); i++) { + parsedCols.add(Bytes.toString(parsed.getLineBytes(), parsed.getColumnOffset(i), + parsed.getColumnLength(i))); + } + if (!Iterables.elementsEqual(parsedCols, expected)) { + fail("Expected: " + Joiner.on(",").join(expected) + "\n" + "Got:" + + Joiner.on(",").join(parsedCols)); + } + } + + private void assertBytesEquals(byte[] a, byte[] b) { + assertEquals(Bytes.toStringBinary(a), Bytes.toStringBinary(b)); + } + + /** + * Test cases that throw BadTsvLineException + */ + @Test(expected = BadTsvLineException.class) + public void testTsvParserBadTsvLineExcessiveColumns() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t"); + byte[] line = Bytes.toBytes("val_a\tval_b\tval_c"); + parser.parse(line, line.length); + } + + @Test(expected = BadTsvLineException.class) + public void testTsvParserBadTsvLineZeroColumn() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t"); + byte[] line = Bytes.toBytes(""); + parser.parse(line, line.length); + } + + @Test(expected = BadTsvLineException.class) + public void testTsvParserBadTsvLineOnlyKey() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t"); + byte[] line = Bytes.toBytes("key_only"); + parser.parse(line, line.length); + } + + @Test(expected = BadTsvLineException.class) + public void testTsvParserBadTsvLineNoRowKey() throws BadTsvLineException { + TsvParser parser = new TsvParser("col_a,HBASE_ROW_KEY", "\t"); + byte[] line = Bytes.toBytes("only_cola_data_and_no_row_key"); + parser.parse(line, line.length); + } + + @Test(expected = BadTsvLineException.class) + public void testTsvParserInvalidTimestamp() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,HBASE_TS_KEY,col_a,", "\t"); + assertEquals(1, parser.getTimestampKeyColumnIndex()); + byte[] line = Bytes.toBytes("rowkey\ttimestamp\tval_a"); + ParsedLine parsed = parser.parse(line, line.length); + assertEquals(-1, parsed.getTimestamp(-1)); + checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line))); + } + + @Test(expected = BadTsvLineException.class) + public void testTsvParserNoTimestampValue() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a,HBASE_TS_KEY", "\t"); + assertEquals(2, parser.getTimestampKeyColumnIndex()); + byte[] line = Bytes.toBytes("rowkey\tval_a"); + parser.parse(line, line.length); + } + + @Test + public void testMROnTable() throws Exception { + String TABLE_NAME = "TestTable"; + String FAMILY = "FAM"; + String INPUT_FILE = "InputFile.esv"; + + // Prepare the arguments required for the test. + String[] args = + new String[] { "-D" + IndexImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,FAM:A,FAM:B", + "-D" + IndexImportTsv.SEPARATOR_CONF_KEY + "=\u001b", TABLE_NAME, INPUT_FILE }; + + doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, null, args, 1); + } + + @Test + public void testMROnTableWithTimestamp() throws Exception { + String TABLE_NAME = "TestTable"; + String FAMILY = "FAM"; + String INPUT_FILE = "InputFile1.csv"; + + // Prepare the arguments required for the test. + String[] args = + new String[] { + "-D" + IndexImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,HBASE_TS_KEY,FAM:A,FAM:B", + "-D" + IndexImportTsv.SEPARATOR_CONF_KEY + "=,", TABLE_NAME, INPUT_FILE }; + + String data = "KEY,1234,VALUE1,VALUE2\n"; + doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, data, args, 1); + } + + @Test + public void testMROnTableWithCustomMapper() throws Exception { + String TABLE_NAME = "TestTable"; + String FAMILY = "FAM"; + String INPUT_FILE = "InputFile2.esv"; + + // Prepare the arguments required for the test. + String[] args = + new String[] { + "-D" + IndexImportTsv.MAPPER_CONF_KEY + + "=org.apache.hadoop.hbase.mapreduce.TsvImporterCustomTestMapper", TABLE_NAME, + INPUT_FILE }; + + doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, null, args, 3); + } + + private void doMROnTableTest(String inputFile, String family, String tableName, String data, + String[] args, int valueMultiplier) throws Exception { + + // Cluster + HBaseTestingUtility htu1 = new HBaseTestingUtility(); + + htu1.startMiniCluster(); + htu1.startMiniMapReduceCluster(); + + GenericOptionsParser opts = new GenericOptionsParser(htu1.getConfiguration(), args); + Configuration conf = opts.getConfiguration(); + args = opts.getRemainingArgs(); + + try { + + FileSystem fs = FileSystem.get(conf); + FSDataOutputStream op = fs.create(new Path(inputFile), true); + if (data == null) { + data = "KEY\u001bVALUE1\u001bVALUE2\n"; + } + op.write(Bytes.toBytes(data)); + op.close(); + + final byte[] FAM = Bytes.toBytes(family); + final byte[] TAB = Bytes.toBytes(tableName); + final byte[] QA = Bytes.toBytes("A"); + final byte[] QB = Bytes.toBytes("B"); + + if (conf.get(IndexImportTsv.BULK_OUTPUT_CONF_KEY) == null) { + HTableDescriptor desc = new HTableDescriptor(TAB); + desc.addFamily(new HColumnDescriptor(FAM)); + new HBaseAdmin(conf).createTable(desc); + } + + IndexImportTsv.createHbaseAdmin(conf); + + Job job = IndexImportTsv.createSubmittableJob(conf, args); + job.waitForCompletion(false); + assertTrue(job.isSuccessful()); + + HTable table = new HTable(new Configuration(conf), TAB); + boolean verified = false; + long pause = conf.getLong("hbase.client.pause", 5 * 1000); + int numRetries = conf.getInt("hbase.client.retries.number", 5); + for (int i = 0; i < numRetries; i++) { + try { + Scan scan = new Scan(); + // Scan entire family. + scan.addFamily(FAM); + ResultScanner resScanner = table.getScanner(scan); + for (Result res : resScanner) { + assertTrue(res.size() == 2); + List kvs = res.list(); + assertEquals(toU8Str(kvs.get(0).getRow()), toU8Str(Bytes.toBytes("KEY"))); + assertEquals(toU8Str(kvs.get(1).getRow()), toU8Str(Bytes.toBytes("KEY"))); + assertEquals(toU8Str(kvs.get(0).getValue()), + toU8Str(Bytes.toBytes("VALUE" + valueMultiplier))); + assertEquals(toU8Str(kvs.get(1).getValue()), + toU8Str(Bytes.toBytes("VALUE" + 2 * valueMultiplier))); + // Only one result set is expected, so let it loop. + } + verified = true; + break; + } catch (NullPointerException e) { + // If here, a cell was empty. Presume its because updates came in + // after the scanner had been opened. Wait a while and retry. + } + try { + Thread.sleep(pause); + } catch (InterruptedException e) { + // continue + } + } + assertTrue(verified); + } finally { + htu1.shutdownMiniMapReduceCluster(); + htu1.shutdownMiniCluster(); + } + } + + @Test + public void testBulkOutputWithoutAnExistingTable() throws Exception { + String TABLE_NAME = "TestTable"; + String FAMILY = "FAM"; + String INPUT_FILE = "InputFile2.esv"; + + // Prepare the arguments required for the test. + String[] args = + new String[] { "-D" + IndexImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,FAM:A,FAM:B", + "-D" + IndexImportTsv.SEPARATOR_CONF_KEY + "=\u001b", + "-D" + IndexImportTsv.BULK_OUTPUT_CONF_KEY + "=output", TABLE_NAME, INPUT_FILE }; + doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, null, args, 3); + } + + public static String toU8Str(byte[] bytes) throws UnsupportedEncodingException { + return Bytes.toString(bytes); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/mapreduce/TestIndexMapReduceUtil.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/mapreduce/TestIndexMapReduceUtil.java new file mode 100644 index 0000000..724345a --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/mapreduce/TestIndexMapReduceUtil.java @@ -0,0 +1,203 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.mapreduce; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import junit.framework.Assert; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.mapreduce.TableInputFormat; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestIndexMapReduceUtil { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private HBaseAdmin admin; + private Configuration conf; + private String tableName; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + admin = UTIL.getHBaseAdmin(); + conf = UTIL.getConfiguration(); + } + + @After + public void tearDown() throws Exception { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + + @Test(timeout = 180000) + public void testShouldAbleReturnTrueForIndexedTable() throws Exception { + tableName = "testShouldAbleReturnTrueForIndexedTable"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.createTable(ihtd); + Assert.assertTrue(IndexMapReduceUtil.isIndexedTable(tableName, conf)); + } + + @Test(timeout = 180000) + public void testShouldAbleReturnFalseForNonIndexedTable() throws Exception { + tableName = "testShouldAbleReturnFalseForNonIndexedTable"; + HTableDescriptor ihtd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + ihtd.addFamily(hcd); + admin.createTable(ihtd); + Assert.assertFalse(IndexMapReduceUtil.isIndexedTable(tableName, conf)); + } + + @Test(timeout = 180000) + public void testShouldReturnStartKeyBesedOnTheRowKeyFromPreSplitRegion() throws Exception { + + tableName = "testShouldReturnStartKeyBesedOnTheRowKeyFromPreSplitRegion"; + HTable table = + UTIL.createTable(tableName.getBytes(), new byte[][] { "families".getBytes() }, 3, + "0".getBytes(), "9".getBytes(), 5); + + assertStartKey(conf, tableName, table, "3"); + assertStartKey(conf, tableName, table, "0"); + assertStartKey(conf, tableName, table, "25"); + assertStartKey(conf, tableName, table, "AAAAA123"); + assertStartKey(conf, tableName, table, "63"); + assertStartKey(conf, tableName, table, ""); + assertStartKey(conf, tableName, table, "9222"); + } + + @Test(timeout = 180000) + public void testShouldReturnStartKeyBesedOnTheRowKey() throws Exception { + + tableName = "testShouldReturnStartKeyBesedOnTheRowKey"; + HTable table = UTIL.createTable(tableName.getBytes(), new byte[][] { "families".getBytes() }); + + assertStartKey(conf, tableName, table, "3"); + assertStartKey(conf, tableName, table, "0"); + assertStartKey(conf, tableName, table, "25"); + assertStartKey(conf, tableName, table, "AAAAA123"); + assertStartKey(conf, tableName, table, ""); + } + + @Test(timeout = 180000) + public void testShouldFormIndexPutsAndIndexDeletes() throws Exception { + tableName = "testShouldFormIndexPutsAndIndexDeletes"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + IndexSpecification iSpec = new IndexSpecification("ScanIndexf"); + iSpec.addIndexColumn(hcd, "q1", ValueType.String, 10); + iSpec.addIndexColumn(hcd, "q2", ValueType.String, 10); + ihtd.addFamily(hcd); + ihtd.addIndex(iSpec); + admin.getConfiguration().set(TableInputFormat.INPUT_TABLE, tableName); + admin.createTable(ihtd); + HTable mainTable = new HTable(conf, Bytes.toBytes(tableName)); + Put put = new Put(Bytes.toBytes("r1")); + put.add(hcd.getName(), Bytes.toBytes("q1"), Bytes.toBytes("v1")); + mainTable.put(put); + put = new Put(Bytes.toBytes("r2")); + put.add(hcd.getName(), Bytes.toBytes("q1"), Bytes.toBytes("v1")); + mainTable.put(put); + put = new Put(Bytes.toBytes("r3")); + put.add(hcd.getName(), Bytes.toBytes("q1"), Bytes.toBytes("v1")); + put.add(hcd.getName(), Bytes.toBytes("q2"), Bytes.toBytes("v2")); + mainTable.put(put); + put = new Put(Bytes.toBytes("r4")); + put.add(hcd.getName(), Bytes.toBytes("q1"), Bytes.toBytes("v1")); + mainTable.put(put); + put = new Put(Bytes.toBytes("r5")); + put.add(hcd.getName(), Bytes.toBytes("q1"), Bytes.toBytes("v1")); + mainTable.put(put); + mainTable.flushCommits(); + admin.flush(tableName); + Delete del = new Delete(Bytes.toBytes("r3")); + del.deleteFamily(hcd.getName()); + mainTable.delete(del); + List indexDelete = IndexMapReduceUtil.getIndexDelete(del, admin.getConfiguration()); + assertTrue(indexDelete.size() == 0); + admin.flush(tableName); + del = new Delete(Bytes.toBytes("r5")); + del.deleteColumns(hcd.getName(), Bytes.toBytes("q1")); + mainTable.delete(del); + indexDelete = IndexMapReduceUtil.getIndexDelete(del, admin.getConfiguration()); + Map> familyMap = ((Delete) indexDelete.get(0)).getFamilyMap(); + Set>> entrySet = familyMap.entrySet(); + for (Entry> entry : entrySet) { + List value = entry.getValue(); + assertTrue(!value.get(0).isDeleteFamily()); + } + + } + + private void assertStartKey(Configuration conf, String tableName, HTable table, String rowKey) + throws IOException { + byte[] startKey = IndexMapReduceUtil.getStartKey(conf, tableName, Bytes.toBytes(rowKey)); + Assert.assertEquals("Fetching wrong start key for " + rowKey, + Bytes.toString(table.getRegionLocation(rowKey).getRegionInfo().getStartKey()), + Bytes.toString(startKey)); + } +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/utils/TestIndexUtils.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/utils/TestIndexUtils.java new file mode 100644 index 0000000..75c9270 --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/utils/TestIndexUtils.java @@ -0,0 +1,68 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.utils; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.index.util.IndexUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestIndexUtils { + + @Test + public void testIncrementValue1() throws Exception { + byte[] val = new byte[] { 97, 97, 97, 97, 127 }; + byte[] incrementValue = IndexUtils.incrementValue(val, true); + assertEquals(Bytes.compareTo(incrementValue, val), 1); + } + + @Test + public void testIncrementValue2() throws Exception { + byte[] val = new byte[] { 1, 127, 127, 127, 127, 127 }; + byte[] incrementValue = IndexUtils.incrementValue(val, true); + assertEquals(Bytes.compareTo(incrementValue, val), 1); + } + + @Test + public void testIncrementValue3() throws Exception { + byte[] val = new byte[] { 127, 127, 127, 127, -128 }; + byte[] incrementValue = IndexUtils.incrementValue(val, true); + assertEquals(Bytes.compareTo(incrementValue, val), 1); + } + + @Test + public void testIncrementValue4() throws Exception { + byte[] val = new byte[] { -1, -1, -1, -1, -1 }; + byte[] incrementValue = IndexUtils.incrementValue(val, true); + assertEquals(Bytes.compareTo(incrementValue, new byte[] { 0, 0, 0, 0, 0 }), 0); + } + + @Test + public void testIncrementValue5() throws Exception { + byte[] val = new byte[] { 56, 57, 58, -1, 127 }; + byte[] incrementValue = IndexUtils.incrementValue(val, true); + assertEquals(Bytes.compareTo(incrementValue, new byte[] { 56, 57, 58, -1, -128 }), 0); + } + +} diff --git a/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/utils/TestSecIndexColocator.java b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/utils/TestSecIndexColocator.java new file mode 100644 index 0000000..988a01b --- /dev/null +++ b/secondaryindex/src/test/java/org/apache/hadoop/hbase/index/utils/TestSecIndexColocator.java @@ -0,0 +1,489 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.index.utils; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import junit.framework.Assert; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.catalog.MetaEditor; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.index.util.SecondaryIndexColocator; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestSecIndexColocator { + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setBoolean("hbase.use.secondary.index", true); + UTIL.startMiniCluster(2); + } + + @AfterClass + public static void finishAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testCoLocationFixing() throws Exception { + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + HBaseAdmin admin = UTIL.getHBaseAdmin(); + Configuration config = UTIL.getConfiguration(); + + String userTableName = "testCoLocationFixing"; + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); + HColumnDescriptor hcd = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("idx"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + ihtd.addIndex(iSpec); + + char c = 'A'; + byte[][] splits = new byte[5][]; + for (int i = 0; i < 5; i++) { + byte[] b = { (byte) c }; + c++; + splits[i] = b; + } + admin.createTable(ihtd, splits); + + String userTableName1 = "testCoLocationFixing1"; + ihtd = new IndexedHTableDescriptor(userTableName1); + ihtd.addFamily(hcd); + iSpec = new IndexSpecification("idx1"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + ihtd.addIndex(iSpec); + admin.createTable(ihtd, splits); + + String userTableName2 = "testCoLocationFixing2"; + ihtd = new IndexedHTableDescriptor(userTableName2); + ihtd.addFamily(hcd); + iSpec = new IndexSpecification("idx2"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + ihtd.addIndex(iSpec); + admin.createTable(ihtd, splits); + + List regions = UTIL.getMetaTableRows(Bytes.toBytes(userTableName + "_idx")); + List regionsEncod = getEncodedNames(regions); + + List regions1 = UTIL.getMetaTableRows(Bytes.toBytes(userTableName1 + "_idx")); + List regionsEncod1 = getEncodedNames(regions1); + + List regions2 = UTIL.getMetaTableRows(Bytes.toBytes(userTableName2 + "_idx")); + List regionsEncod2 = getEncodedNames(regions2); + + for (int i = 0; i < 2; i++) { + admin.move(regionsEncod.get(i), null); + admin.move(regionsEncod1.get(i), null); + admin.move(regionsEncod2.get(i), null); + } + + ZKAssign.blockUntilNoRIT(zkw); + SecondaryIndexColocator colocator = new SecondaryIndexColocator(config); + colocator.setUp(); + boolean inconsistent = colocator.checkForCoLocationInconsistency(); + Assert.assertTrue("Inconsistency should be there before running the tool.", inconsistent); + colocator.fixCoLocationInconsistency(); + + ZKAssign.blockUntilNoRIT(zkw); + + colocator = new SecondaryIndexColocator(config); + colocator.setUp(); + inconsistent = colocator.checkForCoLocationInconsistency(); + Assert.assertFalse("No inconsistency should be there after running the tool", inconsistent); + } + + private List getEncodedNames(List regions) { + List regionsEncod = new ArrayList(); + for (byte[] r : regions) { + String rs = Bytes.toString(r); + int firstOcc = rs.indexOf('.'); + int lastOcc = rs.lastIndexOf('.'); + regionsEncod.add(Bytes.toBytes(rs.substring(firstOcc + 1, lastOcc))); + } + return regionsEncod; + } + + @Test(timeout = 180000) + public void testWhenUserTableIsDisabledButIndexTableIsInDisablingState() throws Exception { + String table = "testWhenUserTableIsDisabledButIndexTableIsInDisablingState"; + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + HTableDescriptor htd = new HTableDescriptor(table); + htd.addFamily(new HColumnDescriptor(new String("cf"))); + byte[][] splits = new byte[10][]; + char c = 'A'; + for (int i = 0; i < 10; i++) { + byte[] b = { (byte) c }; + splits[i] = b; + c++; + } + admin.createTable(htd, splits); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + admin.disableTable(table); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + byte[][] splits2 = new byte[11][]; + c = 'A'; + splits2[0] = new byte[0]; + for (int i = 1; i < 11; i++) { + byte[] b = { (byte) c }; + splits2[i] = b; + c++; + } + + FileSystem filesystem = FileSystem.get(UTIL.getConfiguration()); + Path rootdir = + filesystem.makeQualified(new Path(UTIL.getConfiguration().get(HConstants.HBASE_DIR))); + HMaster master = UTIL.getMiniHBaseCluster().getMasterThreads().get(0).getMaster(); + String table2 = "testWhenUserTableIsDisabledButIndexTableIsInDisablingState_idx"; + HTableDescriptor htd2 = new HTableDescriptor(table2); + htd2.addFamily(new HColumnDescriptor(new String("cf1"))); + FSTableDescriptors.createTableDescriptor(htd2, UTIL.getConfiguration()); + List regionsInMeta = + UTIL.createMultiRegionsInMeta(UTIL.getConfiguration(), htd2, splits2); + List newRegions = new ArrayList(); + for (HRegionInfo regionInfo : regionsInMeta) { + HRegion r = HRegion.createHRegion(regionInfo, rootdir, UTIL.getConfiguration(), htd2); + newRegions.add(r.getRegionInfo()); + } + + for (int i = 0; i < newRegions.size() / 2; i++) { + admin.assign(newRegions.get(i).getRegionName()); + } + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + master.getAssignmentManager().getZKTable().setDisablingTable(table2); + + SecondaryIndexColocator colocator = new SecondaryIndexColocator(UTIL.getConfiguration()); + colocator.setUp(); + boolean inconsistent = colocator.checkForCoLocationInconsistency(); + List serverThreads = + UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + List rs = new ArrayList(); + for (RegionServerThread regionServerThread : serverThreads) { + rs.add(regionServerThread.getRegionServer()); + } + + List onlineregions = new ArrayList(); + for (HRegionServer hrs : rs) { + onlineregions.addAll(hrs.getOnlineRegions()); + } + + boolean regionOnline = false; + for (HRegionInfo hri : onlineregions) { + for (HRegionInfo disabledregion : newRegions) { + if (hri.equals(disabledregion)) { + regionOnline = true; + break; + } + } + } + Assert.assertFalse("NO region from the disabledTable should be online.", regionOnline); + Assert.assertTrue("The disabling table should be now disabled", + ZKTableReadOnly.isDisabledTable(HBaseTestingUtility.getZooKeeperWatcher(UTIL), table2)); + } + + @Test(timeout = 180000) + public void testWhenUserTableIsDisabledButIndexTableIsInEnabledState() throws Exception { + String table = "testWhenUserTableIsDisabledButIndexTableIsInEnabledState"; + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + HTableDescriptor htd = new HTableDescriptor(table); + htd.addFamily(new HColumnDescriptor(new String("cf"))); + byte[][] splits = new byte[10][]; + char c = 'A'; + for (int i = 0; i < 10; i++) { + byte[] b = { (byte) c }; + splits[i] = b; + c++; + } + admin.createTable(htd, splits); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + admin.disableTable(table); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + byte[][] splits2 = new byte[11][]; + c = 'A'; + splits2[0] = new byte[0]; + for (int i = 1; i < 11; i++) { + byte[] b = { (byte) c }; + splits2[i] = b; + c++; + } + + FileSystem filesystem = FileSystem.get(UTIL.getConfiguration()); + Path rootdir = + filesystem.makeQualified(new Path(UTIL.getConfiguration().get(HConstants.HBASE_DIR))); + HMaster master = UTIL.getMiniHBaseCluster().getMasterThreads().get(0).getMaster(); + String table2 = "testWhenUserTableIsDisabledButIndexTableIsInEnabledState_idx"; + HTableDescriptor htd2 = new HTableDescriptor(table2); + htd2.addFamily(new HColumnDescriptor(new String("cf1"))); + FSTableDescriptors.createTableDescriptor(htd2, UTIL.getConfiguration()); + List regionsInMeta = + UTIL.createMultiRegionsInMeta(UTIL.getConfiguration(), htd2, splits2); + List newRegions = new ArrayList(); + for (HRegionInfo regionInfo : regionsInMeta) { + HRegion r = HRegion.createHRegion(regionInfo, rootdir, UTIL.getConfiguration(), htd2); + newRegions.add(r.getRegionInfo()); + } + + for (int i = 0; i < newRegions.size() / 2; i++) { + admin.assign(newRegions.get(i).getRegionName()); + } + master.getAssignmentManager().getZKTable().setEnabledTable(table2); + + SecondaryIndexColocator colocator = new SecondaryIndexColocator(UTIL.getConfiguration()); + colocator.setUp(); + colocator.checkForCoLocationInconsistency(); + List serverThreads = + UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + List rs = new ArrayList(); + for (RegionServerThread regionServerThread : serverThreads) { + rs.add(regionServerThread.getRegionServer()); + } + + List onlineregions = new ArrayList(); + for (HRegionServer hrs : rs) { + onlineregions.addAll(hrs.getOnlineRegions()); + } + + boolean regionOnline = false; + for (HRegionInfo hri : onlineregions) { + for (HRegionInfo disabledregion : newRegions) { + if (hri.equals(disabledregion)) { + regionOnline = true; + } + } + } + Assert.assertFalse("NO region from the disabledTable should be online.", regionOnline); + Assert.assertTrue("The enabled table should be now disabled", + ZKTableReadOnly.isDisabledTable(HBaseTestingUtility.getZooKeeperWatcher(UTIL), table2)); + } + + @Test(timeout = 180000) + public void testWhenAllUSerRegionsAreAssignedButNotSameForIndex() throws Exception { + String table = "testWhenAllUSerRegionsAreAssignedButNotSameForIndex"; + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + HTableDescriptor htd = new HTableDescriptor(table); + htd.addFamily(new HColumnDescriptor(new String("cf"))); + byte[][] splits = new byte[10][]; + char c = 'A'; + for (int i = 0; i < 10; i++) { + byte[] b = { (byte) c }; + splits[i] = b; + c++; + } + admin.createTable(htd, splits); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + + byte[][] splits2 = new byte[11][]; + c = 'A'; + splits2[0] = new byte[0]; + for (int i = 1; i < 11; i++) { + byte[] b = { (byte) c }; + splits2[i] = b; + c++; + } + + FileSystem filesystem = FileSystem.get(UTIL.getConfiguration()); + Path rootdir = + filesystem.makeQualified(new Path(UTIL.getConfiguration().get(HConstants.HBASE_DIR))); + HMaster master = UTIL.getMiniHBaseCluster().getMasterThreads().get(0).getMaster(); + String table2 = "testWhenAllUSerRegionsAreAssignedButNotSameForIndex_idx"; + HTableDescriptor htd2 = new HTableDescriptor(table2); + htd2.addFamily(new HColumnDescriptor(new String("cf1"))); + FSTableDescriptors.createTableDescriptor(htd2, UTIL.getConfiguration()); + List regionsInMeta = + UTIL.createMultiRegionsInMeta(UTIL.getConfiguration(), htd2, splits2); + List newRegions = new ArrayList(); + for (HRegionInfo regionInfo : regionsInMeta) { + HRegion r = HRegion.createHRegion(regionInfo, rootdir, UTIL.getConfiguration(), htd2); + newRegions.add(r.getRegionInfo()); + } + + for (int i = 0; i < newRegions.size() / 2; i++) { + admin.assign(newRegions.get(i).getRegionName()); + } + master.getAssignmentManager().getZKTable().setEnabledTable(table2); + + SecondaryIndexColocator colocator = new SecondaryIndexColocator(UTIL.getConfiguration()); + colocator.setUp(); + colocator.checkForCoLocationInconsistency(); + List serverThreads = + UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + List rs = new ArrayList(); + for (RegionServerThread regionServerThread : serverThreads) { + rs.add(regionServerThread.getRegionServer()); + } + + Set onlineregions = new HashSet(); + for (HRegionServer hrs : rs) { + onlineregions.addAll(hrs.getOnlineRegions()); + } + + boolean regionOffline = false; + for (HRegionInfo hri : newRegions) { + if (!onlineregions.contains(hri)) { + regionOffline = true; + break; + } + } + Assert.assertFalse("All region from the index Table should be online.", regionOffline); + } + + @Test(timeout = 180000) + public void testWhenUserTableIsEabledButIndexTableIsDisabled() throws Exception { + String table = "testWhenUserTableIsEabledButIndexTableIsDisabled"; + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + HTableDescriptor htd = new HTableDescriptor(table); + htd.addFamily(new HColumnDescriptor(new String("cf"))); + byte[][] splits = new byte[10][]; + char c = 'A'; + for (int i = 0; i < 10; i++) { + byte[] b = { (byte) c }; + splits[i] = b; + c++; + } + admin.createTable(htd, splits); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + + String table2 = "testWhenUserTableIsEabledButIndexTableIsDisabled_idx"; + HTableDescriptor htd2 = new HTableDescriptor(table2); + htd2.addFamily(new HColumnDescriptor(new String("cf"))); + admin.createTable(htd2, splits); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + admin.disableTable(table2); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + + List tableRegions = admin.getTableRegions(Bytes.toBytes(table2)); + SecondaryIndexColocator colocator = new SecondaryIndexColocator(UTIL.getConfiguration()); + colocator.setUp(); + boolean inconsistent = colocator.checkForCoLocationInconsistency(); + List serverThreads = + UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + + List rs = new ArrayList(); + for (RegionServerThread regionServerThread : serverThreads) { + rs.add(regionServerThread.getRegionServer()); + } + + Set onlineregions = new HashSet(); + for (HRegionServer hrs : rs) { + onlineregions.addAll(hrs.getOnlineRegions()); + } + + boolean regionOffline = false; + for (HRegionInfo hri : tableRegions) { + if (!onlineregions.contains(hri)) { + regionOffline = true; + break; + } + } + Assert.assertFalse("All region from the disabledTable should be online.", regionOffline); + } + + @Test(timeout = 180000) + public void testWhenRegionsAreNotAssignedAccordingToMeta() throws Exception { + String table = "testWhenRegionsAreNotAssignedAccordingToMeta"; + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + HTableDescriptor htd = new HTableDescriptor(table); + htd.addFamily(new HColumnDescriptor(new String("cf"))); + byte[][] splits = new byte[10][]; + char c = 'A'; + for (int i = 0; i < 10; i++) { + byte[] b = { (byte) c }; + splits[i] = b; + c++; + } + admin.createTable(htd, splits); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(UTIL)); + + ServerName sn = new ServerName("example.org", 1234, 5678); + HMaster master = UTIL.getMiniHBaseCluster().getMaster(0); + + List tableRegions = admin.getTableRegions(Bytes.toBytes(table)); + + for (int i = 0; i < 5; i++) { + MetaEditor.updateRegionLocation(master.getCatalogTracker(), tableRegions.get(i), sn); + } + + SecondaryIndexColocator colocator = new SecondaryIndexColocator(UTIL.getConfiguration()); + colocator.setUp(); + colocator.checkForCoLocationInconsistency(); + + List serverThreads = + UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + List rs = new ArrayList(); + for (RegionServerThread regionServerThread : serverThreads) { + rs.add(regionServerThread.getRegionServer()); + } + + Set onlineregions = new HashSet(); + for (HRegionServer hrs : rs) { + onlineregions.addAll(hrs.getOnlineRegions()); + } + + boolean regionOffline = false; + for (HRegionInfo hri : tableRegions) { + if (!onlineregions.contains(hri)) { + regionOffline = true; + break; + } + } + Assert.assertFalse( + "All the regions with wrong META info should be assiged to some online server.", + regionOffline); + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java new file mode 100644 index 0000000..d115928 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureClient.java @@ -0,0 +1,483 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.ipc; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.security.HBaseSaslRpcClient; +import org.apache.hadoop.hbase.security.HBaseSaslRpcServer.AuthMethod; +import org.apache.hadoop.hbase.security.KerberosInfo; +import org.apache.hadoop.hbase.security.TokenInfo; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier; +import org.apache.hadoop.hbase.security.token.AuthenticationTokenSelector; +import org.apache.hadoop.hbase.util.PoolMap; +import org.apache.hadoop.io.*; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenSelector; +import org.apache.hadoop.util.ReflectionUtils; + +import javax.net.SocketFactory; +import javax.security.sasl.SaslException; +import java.io.*; +import java.net.*; +import java.security.PrivilegedExceptionAction; +import java.util.HashMap; +import java.util.Hashtable; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A client for an IPC service, which support SASL authentication of connections + * using either GSSAPI for Kerberos authentication or DIGEST-MD5 for + * authentication using signed tokens. + * + *

+ * This is a copy of org.apache.hadoop.ipc.Client from secure Hadoop, + * reworked to remove code duplicated with + * {@link org.apache.hadoop.hbase.HBaseClient}. This is part of the loadable + * {@link SecureRpcEngine}, and only functions in connection with a + * {@link SecureServer} instance. + *

+ */ +public class SecureClient extends HBaseClient { + + private static final Log LOG = + LogFactory.getLog("org.apache.hadoop.ipc.SecureClient"); + + protected static Map> tokenHandlers = + new HashMap>(); + static { + tokenHandlers.put(AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE.toString(), + new AuthenticationTokenSelector()); + } + + /** Thread that reads responses and notifies callers. Each connection owns a + * socket connected to a remote address. Calls are multiplexed through this + * socket: responses may be delivered out of order. */ + protected class SecureConnection extends Connection { + private InetSocketAddress server; // server ip:port + private String serverPrincipal; // server's krb5 principal name + private SecureConnectionHeader header; // connection header + private AuthMethod authMethod; // authentication method + private boolean useSasl; + private Token token; + private HBaseSaslRpcClient saslRpcClient; + private int reloginMaxBackoff; // max pause before relogin on sasl failure + + public SecureConnection(ConnectionId remoteId) throws IOException { + super(remoteId); + this.server = remoteId.getAddress(); + + User ticket = remoteId.getTicket(); + Class protocol = remoteId.getProtocol(); + this.useSasl = User.isHBaseSecurityEnabled(conf); + if (useSasl && protocol != null) { + TokenInfo tokenInfo = protocol.getAnnotation(TokenInfo.class); + if (tokenInfo != null) { + TokenSelector tokenSelector = + tokenHandlers.get(tokenInfo.value()); + if (tokenSelector != null) { + token = tokenSelector.selectToken(new Text(clusterId), + ticket.getUGI().getTokens()); + } else if (LOG.isDebugEnabled()) { + LOG.debug("No token selector found for type "+tokenInfo.value()); + } + } + KerberosInfo krbInfo = protocol.getAnnotation(KerberosInfo.class); + if (krbInfo != null) { + String serverKey = krbInfo.serverPrincipal(); + if (serverKey == null) { + throw new IOException( + "Can't obtain server Kerberos config key from KerberosInfo"); + } + serverPrincipal = SecurityUtil.getServerPrincipal( + conf.get(serverKey), server.getAddress().getCanonicalHostName().toLowerCase()); + if (LOG.isDebugEnabled()) { + LOG.debug("RPC Server Kerberos principal name for protocol=" + + protocol.getCanonicalName() + " is " + serverPrincipal); + } + } + } + + if (!useSasl) { + authMethod = AuthMethod.SIMPLE; + } else if (token != null) { + authMethod = AuthMethod.DIGEST; + } else { + authMethod = AuthMethod.KERBEROS; + } + + header = new SecureConnectionHeader( + protocol == null ? null : protocol.getName(), ticket, authMethod); + + if (LOG.isDebugEnabled()) + LOG.debug("Use " + authMethod + " authentication for protocol " + + protocol.getSimpleName()); + + reloginMaxBackoff = conf.getInt("hbase.security.relogin.maxbackoff", 5000); + } + + private synchronized void disposeSasl() { + if (saslRpcClient != null) { + try { + saslRpcClient.dispose(); + saslRpcClient = null; + } catch (IOException ioe) { + LOG.info("Error disposing of SASL client", ioe); + } + } + } + + private synchronized boolean shouldAuthenticateOverKrb() throws IOException { + UserGroupInformation loginUser = UserGroupInformation.getLoginUser(); + UserGroupInformation currentUser = + UserGroupInformation.getCurrentUser(); + UserGroupInformation realUser = currentUser.getRealUser(); + return authMethod == AuthMethod.KERBEROS && + loginUser != null && + //Make sure user logged in using Kerberos either keytab or TGT + loginUser.hasKerberosCredentials() && + // relogin only in case it is the login user (e.g. JT) + // or superuser (like oozie). + (loginUser.equals(currentUser) || loginUser.equals(realUser)); + } + + private synchronized boolean setupSaslConnection(final InputStream in2, + final OutputStream out2) + throws IOException { + saslRpcClient = new HBaseSaslRpcClient(authMethod, token, serverPrincipal); + return saslRpcClient.saslConnect(in2, out2); + } + + /** + * If multiple clients with the same principal try to connect + * to the same server at the same time, the server assumes a + * replay attack is in progress. This is a feature of kerberos. + * In order to work around this, what is done is that the client + * backs off randomly and tries to initiate the connection + * again. + * The other problem is to do with ticket expiry. To handle that, + * a relogin is attempted. + *

+ * The retry logic is governed by the {@link #shouldAuthenticateOverKrb} + * method. In case when the user doesn't have valid credentials, we don't + * need to retry (from cache or ticket). In such cases, it is prudent to + * throw a runtime exception when we receive a SaslException from the + * underlying authentication implementation, so there is no retry from + * other high level (for eg, HCM or HBaseAdmin). + *

+ */ + private synchronized void handleSaslConnectionFailure( + final int currRetries, + final int maxRetries, final Exception ex, final Random rand, + final User user) + throws IOException, InterruptedException{ + user.runAs(new PrivilegedExceptionAction() { + public Object run() throws IOException, InterruptedException { + closeConnection(); + if (shouldAuthenticateOverKrb()) { + if (currRetries < maxRetries) { + LOG.debug("Exception encountered while connecting to " + + "the server : " + ex); + //try re-login + if (UserGroupInformation.isLoginKeytabBased()) { + UserGroupInformation.getLoginUser().reloginFromKeytab(); + } else { + UserGroupInformation.getLoginUser().reloginFromTicketCache(); + } + disposeSasl(); + //have granularity of milliseconds + //we are sleeping with the Connection lock held but since this + //connection instance is being used for connecting to the server + //in question, it is okay + Thread.sleep((rand.nextInt(reloginMaxBackoff) + 1)); + return null; + } else { + String msg = "Couldn't setup connection for " + + UserGroupInformation.getLoginUser().getUserName() + + " to " + serverPrincipal; + LOG.warn(msg); + throw (IOException) new IOException(msg).initCause(ex); + } + } else { + LOG.warn("Exception encountered while connecting to " + + "the server : " + ex); + } + if (ex instanceof RemoteException) { + throw (RemoteException)ex; + } + if (ex instanceof SaslException) { + String msg = "SASL authentication failed." + + " The most likely cause is missing or invalid credentials." + + " Consider 'kinit'."; + LOG.fatal(msg, ex); + throw new RuntimeException(msg, ex); + } + throw new IOException(ex); + } + }); + } + + @Override + protected synchronized void setupIOstreams() + throws IOException, InterruptedException { + if (socket != null || shouldCloseConnection.get()) { + return; + } + + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Connecting to "+server); + } + short numRetries = 0; + final short MAX_RETRIES = 5; + Random rand = null; + while (true) { + setupConnection(); + InputStream inStream = NetUtils.getInputStream(socket); + OutputStream outStream = NetUtils.getOutputStream(socket); + writeRpcHeader(outStream); + if (useSasl) { + final InputStream in2 = inStream; + final OutputStream out2 = outStream; + User ticket = remoteId.getTicket(); + if (authMethod == AuthMethod.KERBEROS) { + UserGroupInformation ugi = ticket.getUGI(); + if (ugi != null && ugi.getRealUser() != null) { + ticket = User.create(ugi.getRealUser()); + } + } + boolean continueSasl = false; + try { + continueSasl = + ticket.runAs(new PrivilegedExceptionAction() { + @Override + public Boolean run() throws IOException { + return setupSaslConnection(in2, out2); + } + }); + } catch (Exception ex) { + if (rand == null) { + rand = new Random(); + } + handleSaslConnectionFailure(numRetries++, MAX_RETRIES, ex, rand, + ticket); + continue; + } + if (continueSasl) { + // Sasl connect is successful. Let's set up Sasl i/o streams. + inStream = saslRpcClient.getInputStream(inStream); + outStream = saslRpcClient.getOutputStream(outStream); + } else { + // fall back to simple auth because server told us so. + authMethod = AuthMethod.SIMPLE; + header = new SecureConnectionHeader(header.getProtocol(), + header.getUser(), authMethod); + useSasl = false; + } + } + this.in = new DataInputStream(new BufferedInputStream + (new PingInputStream(inStream))); + this.out = new DataOutputStream + (new BufferedOutputStream(outStream)); + writeHeader(); + + // update last activity time + touch(); + + // start the receiver thread after the socket connection has been set up + start(); + return; + } + } catch (IOException e) { + markClosed(e); + close(); + + throw e; + } + } + + /* Write the RPC header */ + private void writeRpcHeader(OutputStream outStream) throws IOException { + DataOutputStream out = new DataOutputStream(new BufferedOutputStream(outStream)); + // Write out the header, version and authentication method + out.write(SecureServer.HEADER.array()); + out.write(SecureServer.CURRENT_VERSION); + authMethod.write(out); + out.flush(); + } + + /** + * Write the protocol header for each connection + * Out is not synchronized because only the first thread does this. + */ + private void writeHeader() throws IOException { + // Write out the ConnectionHeader + DataOutputBuffer buf = new DataOutputBuffer(); + header.write(buf); + + // Write out the payload length + int bufLen = buf.getLength(); + out.writeInt(bufLen); + out.write(buf.getData(), 0, bufLen); + } + + @Override + protected void receiveResponse() { + if (shouldCloseConnection.get()) { + return; + } + touch(); + + try { + int id = in.readInt(); // try to read an id + + if (LOG.isDebugEnabled()) + LOG.debug(getName() + " got value #" + id); + + Call call = calls.remove(id); + + int state = in.readInt(); // read call status + if (LOG.isDebugEnabled()) { + LOG.debug("call #"+id+" state is " + state); + } + if (state == Status.SUCCESS.state) { + Writable value = ReflectionUtils.newInstance(valueClass, conf); + value.readFields(in); // read value + if (LOG.isDebugEnabled()) { + LOG.debug("call #"+id+", response is:\n"+value.toString()); + } + // it's possible that this call may have been cleaned up due to a RPC + // timeout, so check if it still exists before setting the value. + if (call != null) { + call.setValue(value); + } + } else if (state == Status.ERROR.state) { + if (call != null) { + call.setException(new RemoteException(WritableUtils.readString(in), WritableUtils + .readString(in))); + } + } else if (state == Status.FATAL.state) { + // Close the connection + markClosed(new RemoteException(WritableUtils.readString(in), + WritableUtils.readString(in))); + } + } catch (IOException e) { + if (e instanceof SocketTimeoutException && remoteId.rpcTimeout > 0) { + // Clean up open calls but don't treat this as a fatal condition, + // since we expect certain responses to not make it by the specified + // {@link ConnectionId#rpcTimeout}. + closeException = e; + } else { + // Since the server did not respond within the default ping interval + // time, treat this as a fatal condition and close this connection + markClosed(e); + } + } finally { + if (remoteId.rpcTimeout > 0) { + cleanupCalls(remoteId.rpcTimeout); + } + } + } + + /** Close the connection. */ + protected synchronized void close() { + if (!shouldCloseConnection.get()) { + LOG.error("The connection is not in the closed state"); + return; + } + + // release the resources + // first thing to do;take the connection out of the connection list + synchronized (connections) { + connections.remove(remoteId, this); + } + + // close the streams and therefore the socket + IOUtils.closeStream(out); + IOUtils.closeStream(in); + disposeSasl(); + + // clean up all calls + if (closeException == null) { + if (!calls.isEmpty()) { + LOG.warn( + "A connection is closed for no cause and calls are not empty"); + + // clean up calls anyway + closeException = new IOException("Unexpected closed connection"); + cleanupCalls(); + } + } else { + // log the info + if (LOG.isDebugEnabled()) { + LOG.debug("closing ipc connection to " + server + ": " + + closeException.getMessage(),closeException); + } + + // cleanup calls + cleanupCalls(); + } + if (LOG.isDebugEnabled()) + LOG.debug(getName() + ": closed"); + } + } + + /** + * Construct an IPC client whose values are of the given {@link org.apache.hadoop.io.Writable} + * class. + * @param valueClass value class + * @param conf configuration + * @param factory socket factory + */ + public SecureClient(Class valueClass, Configuration conf, + SocketFactory factory) { + super(valueClass, conf, factory); + } + + /** + * Construct an IPC client with the default SocketFactory + * @param valueClass value class + * @param conf configuration + */ + public SecureClient(Class valueClass, Configuration conf) { + this(valueClass, conf, NetUtils.getDefaultSocketFactory(conf)); + } + + /** + * Creates a SecureConnection. Can be overridden by a subclass for testing. + * @param remoteId - the ConnectionId to use for the connection creation. + */ + @Override + protected SecureConnection createConnection(ConnectionId remoteId) throws IOException { + return new SecureConnection(remoteId); + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureConnectionHeader.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureConnectionHeader.java new file mode 100644 index 0000000..5060821 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureConnectionHeader.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.ipc; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.security.HBaseSaslRpcServer.AuthMethod; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.security.UserGroupInformation; + +/** + * The IPC connection header sent by the client to the server + * on connection establishment. Part of the {@link SecureRpcEngine} + * implementation. + */ +class SecureConnectionHeader extends ConnectionHeader { + private User user = null; + private AuthMethod authMethod; + + public SecureConnectionHeader() {} + + /** + * Create a new {@link org.apache.hadoop.hbase.ipc.SecureConnectionHeader} with the given protocol + * and {@link org.apache.hadoop.security.UserGroupInformation}. + * @param protocol protocol used for communication between the IPC client + * and the server + * @param ugi {@link org.apache.hadoop.security.UserGroupInformation} of the client communicating with + * the server + */ + public SecureConnectionHeader(String protocol, User user, AuthMethod authMethod) { + this.protocol = protocol; + this.user = user; + this.authMethod = authMethod; + } + + @Override + public void readFields(DataInput in) throws IOException { + protocol = Text.readString(in); + if (protocol.isEmpty()) { + protocol = null; + } + boolean ugiUsernamePresent = in.readBoolean(); + if (ugiUsernamePresent) { + String username = in.readUTF(); + boolean realUserNamePresent = in.readBoolean(); + if (realUserNamePresent) { + String realUserName = in.readUTF(); + UserGroupInformation realUserUgi = + UserGroupInformation.createRemoteUser(realUserName); + user = User.create( + UserGroupInformation.createProxyUser(username, realUserUgi)); + } else { + user = User.create(UserGroupInformation.createRemoteUser(username)); + } + } else { + user = null; + } + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, (protocol == null) ? "" : protocol); + if (user != null) { + UserGroupInformation ugi = user.getUGI(); + if (authMethod == AuthMethod.KERBEROS) { + // Send effective user for Kerberos auth + out.writeBoolean(true); + out.writeUTF(ugi.getUserName()); + out.writeBoolean(false); + } else if (authMethod == AuthMethod.DIGEST) { + // Don't send user for token auth + out.writeBoolean(false); + } else { + //Send both effective user and real user for simple auth + out.writeBoolean(true); + out.writeUTF(ugi.getUserName()); + if (ugi.getRealUser() != null) { + out.writeBoolean(true); + out.writeUTF(ugi.getRealUser().getUserName()); + } else { + out.writeBoolean(false); + } + } + } else { + out.writeBoolean(false); + } + } + + public String getProtocol() { + return protocol; + } + + public User getUser() { + return user; + } + + public String toString() { + return protocol + "-" + user; + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java new file mode 100644 index 0000000..fd505d7 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureRpcEngine.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.ipc; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; +import org.apache.hadoop.hbase.security.HBasePolicyProvider; +import org.apache.hadoop.hbase.security.HBaseSaslRpcServer; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.token.AuthenticationTokenSecretManager; +import org.apache.hadoop.hbase.util.Objects; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; + +import java.io.IOException; +import java.lang.reflect.*; +import java.net.InetSocketAddress; + +/** + * A loadable RPC engine supporting SASL authentication of connections, using + * GSSAPI for Kerberos authentication or DIGEST-MD5 for authentication via + * signed tokens. + * + *

+ * This is a fork of the {@code org.apache.hadoop.ipc.WriteableRpcEngine} from + * secure Hadoop, reworked to eliminate code duplication with the existing + * HBase {@link WritableRpcEngine}. + *

+ * + * @see SecureClient + * @see SecureServer + */ +public class SecureRpcEngine implements RpcEngine { + // Leave this out in the hadoop ipc package but keep class name. Do this + // so that we do not get the logging of this class' invocations by doing our + // blanket enabling DEBUG on the o.a.h.h. package. + protected static final Log LOG = + LogFactory.getLog("org.apache.hadoop.ipc.SecureRpcEngine"); + + private Configuration conf; + private SecureClient client; + + @Override + public void setConf(Configuration config) { + this.conf = config; + if (User.isHBaseSecurityEnabled(conf)) { + HBaseSaslRpcServer.init(conf); + } + // check for an already created client + if (this.client != null) { + this.client.stop(); + } + this.client = new SecureClient(HbaseObjectWritable.class, conf); + } + + @Override + public Configuration getConf() { + return this.conf; + } + + private static class Invoker implements InvocationHandler { + private Class protocol; + private InetSocketAddress address; + private User ticket; + private SecureClient client; + final private int rpcTimeout; + + public Invoker(SecureClient client, + Class protocol, + InetSocketAddress address, User ticket, int rpcTimeout) { + this.protocol = protocol; + this.address = address; + this.ticket = ticket; + this.client = client; + this.rpcTimeout = rpcTimeout; + } + + public Object invoke(Object proxy, Method method, Object[] args) + throws Throwable { + final boolean logDebug = LOG.isDebugEnabled(); + long startTime = 0; + if (logDebug) { + startTime = System.currentTimeMillis(); + } + HbaseObjectWritable value = (HbaseObjectWritable) + client.call(new Invocation(method, protocol, args), address, + protocol, ticket, rpcTimeout); + if (logDebug) { + long callTime = System.currentTimeMillis() - startTime; + LOG.debug("Call: " + method.getName() + " " + callTime); + } + return value.get(); + } + } + + /** + * Construct a client-side proxy object that implements the named protocol, + * talking to a server at the named address. + * + * @param protocol interface + * @param clientVersion version we are expecting + * @param addr remote address + * @param conf configuration + * @return proxy + * @throws java.io.IOException e + */ + @Override + public T getProxy( + Class protocol, long clientVersion, + InetSocketAddress addr, + Configuration conf, int rpcTimeout) + throws IOException { + if (this.client == null) { + throw new IOException("Client must be initialized by calling setConf(Configuration)"); + } + + T proxy = + (T) Proxy.newProxyInstance( + protocol.getClassLoader(), new Class[] { protocol }, + new Invoker(this.client, protocol, addr, User.getCurrent(), + HBaseRPC.getRpcTimeout(rpcTimeout))); + /* + * TODO: checking protocol version only needs to be done once when we setup a new + * SecureClient.Connection. Doing it every time we retrieve a proxy instance is resulting + * in unnecessary RPC traffic. + */ + long serverVersion = proxy.getProtocolVersion(protocol.getName(), + clientVersion); + if (serverVersion != clientVersion) { + throw new HBaseRPC.VersionMismatch(protocol.getName(), clientVersion, + serverVersion); + } + return proxy; + } + + /** Expert: Make multiple, parallel calls to a set of servers. */ + @Override + public Object[] call(Method method, Object[][] params, + InetSocketAddress[] addrs, + Class protocol, + User ticket, Configuration conf) + throws IOException, InterruptedException { + if (this.client == null) { + throw new IOException("Client must be initialized by calling setConf(Configuration)"); + } + + Invocation[] invocations = new Invocation[params.length]; + for (int i = 0; i < params.length; i++) { + invocations[i] = new Invocation(method, protocol, params[i]); + } + + Writable[] wrappedValues = + client.call(invocations, addrs, protocol, ticket); + + if (method.getReturnType() == Void.TYPE) { + return null; + } + + Object[] values = + (Object[])Array.newInstance(method.getReturnType(), wrappedValues.length); + for (int i = 0; i < values.length; i++) + if (wrappedValues[i] != null) + values[i] = ((HbaseObjectWritable)wrappedValues[i]).get(); + + return values; + } + + @Override + public void close() { + if (this.client != null) { + this.client.stop(); + } + } + + /** Construct a server for a protocol implementation instance listening on a + * port and address, with a secret manager. */ + @Override + public Server getServer(Class protocol, + final Object instance, + Class[] ifaces, + final String bindAddress, final int port, + final int numHandlers, + int metaHandlerCount, final boolean verbose, Configuration conf, + int highPriorityLevel) + throws IOException { + Server server = new Server(instance, ifaces, conf, bindAddress, port, + numHandlers, metaHandlerCount, verbose, + highPriorityLevel); + return server; + } + + /** An RPC Server. */ + public static class Server extends SecureServer { + private Object instance; + private Class implementation; + private Class[] ifaces; + private boolean verbose; + + private static String classNameBase(String className) { + String[] names = className.split("\\.", -1); + if (names == null || names.length == 0) { + return className; + } + return names[names.length-1]; + } + + /** Construct an RPC server. + * @param instance the instance whose methods will be called + * @param conf the configuration to use + * @param bindAddress the address to bind on to listen for connection + * @param port the port to listen for connections on + * @param numHandlers the number of method handler threads to run + * @param verbose whether each call should be logged + * @throws java.io.IOException e + */ + public Server(Object instance, final Class[] ifaces, + Configuration conf, String bindAddress, int port, + int numHandlers, int metaHandlerCount, boolean verbose, + int highPriorityLevel) + throws IOException { + super(bindAddress, port, Invocation.class, numHandlers, metaHandlerCount, conf, + classNameBase(instance.getClass().getName()), highPriorityLevel); + this.instance = instance; + this.implementation = instance.getClass(); + this.verbose = verbose; + + this.ifaces = ifaces; + + // create metrics for the advertised interfaces this server implements. + this.rpcMetrics.createMetrics(this.ifaces); + } + + public AuthenticationTokenSecretManager createSecretManager(){ + if (instance instanceof org.apache.hadoop.hbase.Server) { + org.apache.hadoop.hbase.Server server = + (org.apache.hadoop.hbase.Server)instance; + Configuration conf = server.getConfiguration(); + long keyUpdateInterval = + conf.getLong("hbase.auth.key.update.interval", 24*60*60*1000); + long maxAge = + conf.getLong("hbase.auth.token.max.lifetime", 7*24*60*60*1000); + return new AuthenticationTokenSecretManager(conf, server.getZooKeeper(), + server.getServerName().toString(), keyUpdateInterval, maxAge); + } + return null; + } + + @Override + public void startThreads() { + AuthenticationTokenSecretManager mgr = createSecretManager(); + if (mgr != null) { + setSecretManager(mgr); + mgr.start(); + } + this.authManager = new ServiceAuthorizationManager(); + HBasePolicyProvider.init(conf, authManager); + + // continue with base startup + super.startThreads(); + } + + @Override + public Writable call(Class protocol, + Writable param, long receivedTime, MonitoredRPCHandler status) + throws IOException { + try { + Invocation call = (Invocation)param; + if(call.getMethodName() == null) { + throw new IOException("Could not find requested method, the usual " + + "cause is a version mismatch between client and server."); + } + if (verbose) log("Call: " + call); + + Method method = + protocol.getMethod(call.getMethodName(), + call.getParameterClasses()); + method.setAccessible(true); + + Object impl = null; + if (protocol.isAssignableFrom(this.implementation)) { + impl = this.instance; + } + else { + throw new HBaseRPC.UnknownProtocolException(protocol); + } + + long startTime = System.currentTimeMillis(); + Object[] params = call.getParameters(); + Object value = method.invoke(impl, params); + int processingTime = (int) (System.currentTimeMillis() - startTime); + int qTime = (int) (startTime-receivedTime); + if (TRACELOG.isDebugEnabled()) { + TRACELOG.debug("Call #" + CurCall.get().id + + "; Served: " + protocol.getSimpleName()+"#"+call.getMethodName() + + " queueTime=" + qTime + + " processingTime=" + processingTime + + " contents=" + Objects.describeQuantity(params)); + } + rpcMetrics.rpcQueueTime.inc(qTime); + rpcMetrics.rpcProcessingTime.inc(processingTime); + rpcMetrics.inc(call.getMethodName(), processingTime); + if (verbose) log("Return: "+value); + + return new HbaseObjectWritable(method.getReturnType(), value); + } catch (InvocationTargetException e) { + Throwable target = e.getTargetException(); + if (target instanceof IOException) { + throw (IOException)target; + } + IOException ioe = new IOException(target.toString()); + ioe.setStackTrace(target.getStackTrace()); + throw ioe; + } catch (Throwable e) { + if (!(e instanceof IOException)) { + LOG.error("Unexpected throwable object ", e); + } + IOException ioe = new IOException(e.toString()); + ioe.setStackTrace(e.getStackTrace()); + throw ioe; + } + } + } + + protected static void log(String value) { + String v = value; + if (v != null && v.length() > 55) + v = v.substring(0, 55)+"..."; + LOG.info(v); + } +} \ No newline at end of file diff --git a/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java new file mode 100644 index 0000000..3448d99 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/ipc/SecureServer.java @@ -0,0 +1,756 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.ipc; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.io.WritableWithSize; +import org.apache.hadoop.hbase.security.HBaseSaslRpcServer; +import org.apache.hadoop.hbase.security.HBaseSaslRpcServer.AuthMethod; +import org.apache.hadoop.hbase.security.HBaseSaslRpcServer.SaslDigestCallbackHandler; +import org.apache.hadoop.hbase.security.HBaseSaslRpcServer.SaslGssCallbackHandler; +import org.apache.hadoop.hbase.security.HBaseSaslRpcServer.SaslStatus; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.ByteBufferOutputStream; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; +import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.collect.ImmutableSet; + +import javax.security.sasl.Sasl; +import javax.security.sasl.SaslException; +import javax.security.sasl.SaslServer; +import java.io.*; +import java.net.*; +import java.nio.ByteBuffer; +import java.nio.channels.*; +import java.security.PrivilegedExceptionAction; +import java.util.*; + +import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION; + +/** + * An abstract IPC service, supporting SASL authentication of connections, + * using GSSAPI for Kerberos authentication or DIGEST-MD5 for authentication + * via signed tokens. + * + *

+ * This is part of the {@link SecureRpcEngine} implementation. + *

+ * + * @see org.apache.hadoop.hbase.ipc.SecureClient + */ +public abstract class SecureServer extends HBaseServer { + private final boolean authorize; + private boolean isSecurityEnabled; + + /** + * The first four bytes of secure RPC connections + */ + public static final ByteBuffer HEADER = ByteBuffer.wrap("srpc".getBytes()); + + // 1 : Introduce ping and server does not throw away RPCs + // 3 : Introduce the protocol into the RPC connection header + // 4 : Introduced SASL security layer + public static final byte CURRENT_VERSION = 4; + public static final Set INSECURE_VERSIONS = ImmutableSet.of((byte) 3); + + public static final Log LOG = LogFactory.getLog(SecureServer.class); + private static final Log AUDITLOG = LogFactory.getLog("SecurityLogger." + + SecureServer.class.getName()); + + private static final String AUTH_FAILED_FOR = "Auth failed for "; + private static final String AUTH_SUCCESSFUL_FOR = "Auth successful for "; + + protected SecretManager secretManager; + protected ServiceAuthorizationManager authManager; + + protected class SecureCall extends HBaseServer.Call { + public SecureCall(int id, Writable param, Connection connection, + Responder responder, long size) { + super(id, param, connection, responder, size); + } + + @Override + protected synchronized void setResponse(Object value, Status status, + String errorClass, String error) { + Writable result = null; + if (value instanceof Writable) { + result = (Writable) value; + } else { + /* We might have a null value and errors. Avoid creating a + * HbaseObjectWritable, because the constructor fails on null. */ + if (value != null) { + result = new HbaseObjectWritable(value); + } + } + + int size = BUFFER_INITIAL_SIZE; + if (result instanceof WritableWithSize) { + // get the size hint. + WritableWithSize ohint = (WritableWithSize) result; + long hint = ohint.getWritableSize() + Bytes.SIZEOF_INT + Bytes.SIZEOF_INT; + if (hint > Integer.MAX_VALUE) { + // oops, new problem. + IOException ioe = + new IOException("Result buffer size too large: " + hint); + errorClass = ioe.getClass().getName(); + error = StringUtils.stringifyException(ioe); + } else { + size = (int)hint; + } + } + + ByteBufferOutputStream buf = new ByteBufferOutputStream(size); + DataOutputStream out = new DataOutputStream(buf); + try { + out.writeInt(this.id); // write call id + out.writeInt(status.state); // write status + } catch (IOException e) { + errorClass = e.getClass().getName(); + error = StringUtils.stringifyException(e); + } + + try { + if (status == Status.SUCCESS) { + result.write(out); + } else { + WritableUtils.writeString(out, errorClass); + WritableUtils.writeString(out, error); + } + if (((SecureConnection)connection).useWrap) { + wrapWithSasl(buf); + } + } catch (IOException e) { + LOG.warn("Error sending response to call: ", e); + } + + this.response = buf.getByteBuffer(); + } + + private void wrapWithSasl(ByteBufferOutputStream response) + throws IOException { + if (((SecureConnection)connection).useSasl) { + // getByteBuffer calls flip() + ByteBuffer buf = response.getByteBuffer(); + byte[] token; + // synchronization may be needed since there can be multiple Handler + // threads using saslServer to wrap responses. + synchronized (((SecureConnection)connection).saslServer) { + token = ((SecureConnection)connection).saslServer.wrap(buf.array(), + buf.arrayOffset(), buf.remaining()); + } + if (LOG.isTraceEnabled()) { + LOG.trace("Adding saslServer wrapped token of size " + token.length + + " as call response."); + } + buf.clear(); + DataOutputStream saslOut = new DataOutputStream(response); + saslOut.writeInt(token.length); + saslOut.write(token, 0, token.length); + } + } + } + + /** Reads calls from a connection and queues them for handling. */ + public class SecureConnection extends HBaseServer.Connection { + private boolean rpcHeaderRead = false; // if initial rpc header is read + private boolean headerRead = false; //if the connection header that + //follows version is read. + private ByteBuffer data; + private ByteBuffer dataLengthBuffer; + protected final LinkedList responseQueue; + private int dataLength; + private InetAddress addr; + + boolean useSasl; + SaslServer saslServer; + private AuthMethod authMethod; + private boolean saslContextEstablished; + private boolean skipInitialSaslHandshake; + private ByteBuffer rpcHeaderBuffer; + private ByteBuffer unwrappedData; + private ByteBuffer unwrappedDataLengthBuffer; + + public UserGroupInformation attemptingUser = null; // user name before auth + + // Fake 'call' for failed authorization response + private final int AUTHORIZATION_FAILED_CALLID = -1; + // Fake 'call' for SASL context setup + private static final int SASL_CALLID = -33; + private final SecureCall saslCall = new SecureCall(SASL_CALLID, null, this, null, 0); + + private boolean useWrap = false; + + public SecureConnection(SocketChannel channel, long lastContact) { + super(channel, lastContact); + this.header = new SecureConnectionHeader(); + this.channel = channel; + this.data = null; + this.dataLengthBuffer = ByteBuffer.allocate(4); + this.unwrappedData = null; + this.unwrappedDataLengthBuffer = ByteBuffer.allocate(4); + this.socket = channel.socket(); + this.addr = socket.getInetAddress(); + this.responseQueue = new LinkedList(); + } + + @Override + public String toString() { + return getHostAddress() + ":" + remotePort; + } + + public String getHostAddress() { + return hostAddress; + } + + public InetAddress getHostInetAddress() { + return addr; + } + + private User getAuthorizedUgi(String authorizedId) + throws IOException { + if (authMethod == AuthMethod.DIGEST) { + TokenIdentifier tokenId = HBaseSaslRpcServer.getIdentifier(authorizedId, + secretManager); + UserGroupInformation ugi = tokenId.getUser(); + if (ugi == null) { + throw new AccessControlException( + "Can't retrieve username from tokenIdentifier."); + } + ugi.addTokenIdentifier(tokenId); + return User.create(ugi); + } else { + return User.create(UserGroupInformation.createRemoteUser(authorizedId)); + } + } + + private void saslReadAndProcess(byte[] saslToken) throws IOException, + InterruptedException { + if (!saslContextEstablished) { + byte[] replyToken = null; + try { + if (saslServer == null) { + switch (authMethod) { + case DIGEST: + if (secretManager == null) { + throw new AccessControlException( + "Server is not configured to do DIGEST authentication."); + } + saslServer = Sasl.createSaslServer(AuthMethod.DIGEST + .getMechanismName(), null, HBaseSaslRpcServer.SASL_DEFAULT_REALM, + HBaseSaslRpcServer.SASL_PROPS, new SaslDigestCallbackHandler( + secretManager, this)); + break; + default: + UserGroupInformation current = UserGroupInformation + .getCurrentUser(); + String fullName = current.getUserName(); + if (LOG.isTraceEnabled()) { + LOG.trace("Kerberos principal name is " + fullName); + } + final String names[] = HBaseSaslRpcServer.splitKerberosName(fullName); + if (names.length != 3) { + throw new AccessControlException( + "Kerberos principal name does NOT have the expected " + + "hostname part: " + fullName); + } + current.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws SaslException { + saslServer = Sasl.createSaslServer(AuthMethod.KERBEROS + .getMechanismName(), names[0], names[1], + HBaseSaslRpcServer.SASL_PROPS, new SaslGssCallbackHandler()); + return null; + } + }); + } + if (saslServer == null) + throw new AccessControlException( + "Unable to find SASL server implementation for " + + authMethod.getMechanismName()); + if (LOG.isTraceEnabled()) { + LOG.trace("Created SASL server with mechanism = " + + authMethod.getMechanismName()); + } + } + if (LOG.isTraceEnabled()) { + LOG.trace("Have read input token of size " + saslToken.length + + " for processing by saslServer.evaluateResponse()"); + } + replyToken = saslServer.evaluateResponse(saslToken); + } catch (IOException e) { + IOException sendToClient = e; + Throwable cause = e; + while (cause != null) { + if (cause instanceof InvalidToken) { + sendToClient = (InvalidToken) cause; + break; + } + cause = cause.getCause(); + } + doSaslReply(SaslStatus.ERROR, null, sendToClient.getClass().getName(), + sendToClient.getLocalizedMessage()); + rpcMetrics.authenticationFailures.inc(); + String clientIP = this.toString(); + // attempting user could be null + AUDITLOG.warn(AUTH_FAILED_FOR + clientIP + ":" + attemptingUser); + throw e; + } + if (replyToken != null) { + if (LOG.isTraceEnabled()) { + LOG.trace("Will send token of size " + replyToken.length + + " from saslServer."); + } + doSaslReply(SaslStatus.SUCCESS, new BytesWritable(replyToken), null, + null); + } + if (saslServer.isComplete()) { + if (LOG.isDebugEnabled()) { + LOG.debug("SASL server context established. Negotiated QoP is " + + saslServer.getNegotiatedProperty(Sasl.QOP)); + } + String qop = (String) saslServer.getNegotiatedProperty(Sasl.QOP); + useWrap = qop != null && !"auth".equalsIgnoreCase(qop); + ticket = getAuthorizedUgi(saslServer.getAuthorizationID()); + if (LOG.isDebugEnabled()) { + LOG.debug("SASL server successfully authenticated client: " + ticket); + } + rpcMetrics.authenticationSuccesses.inc(); + AUDITLOG.info(AUTH_SUCCESSFUL_FOR + ticket); + saslContextEstablished = true; + } + } else { + if (LOG.isTraceEnabled()) { + LOG.trace("Have read input token of size " + saslToken.length + + " for processing by saslServer.unwrap()"); + } + if (!useWrap) { + processOneRpc(saslToken); + } else { + byte[] plaintextData = saslServer.unwrap(saslToken, 0, + saslToken.length); + processUnwrappedData(plaintextData); + } + } + } + + private void doSaslReply(SaslStatus status, Writable rv, + String errorClass, String error) throws IOException { + saslCall.setResponse(rv, + status == SaslStatus.SUCCESS ? Status.SUCCESS : Status.ERROR, + errorClass, error); + saslCall.responder = responder; + saslCall.sendResponseIfReady(); + } + + private void disposeSasl() { + if (saslServer != null) { + try { + saslServer.dispose(); + } catch (SaslException ignored) { + } + } + } + + public int readAndProcess() throws IOException, InterruptedException { + while (true) { + /* Read at most one RPC. If the header is not read completely yet + * then iterate until we read first RPC or until there is no data left. + */ + int count = -1; + if (dataLengthBuffer.remaining() > 0) { + count = channelRead(channel, dataLengthBuffer); + if (count < 0 || dataLengthBuffer.remaining() > 0) + return count; + } + + if (!rpcHeaderRead) { + //Every connection is expected to send the header. + if (rpcHeaderBuffer == null) { + rpcHeaderBuffer = ByteBuffer.allocate(2); + } + count = channelRead(channel, rpcHeaderBuffer); + if (count < 0 || rpcHeaderBuffer.remaining() > 0) { + return count; + } + int version = rpcHeaderBuffer.get(0); + byte[] method = new byte[] {rpcHeaderBuffer.get(1)}; + authMethod = AuthMethod.read(new DataInputStream( + new ByteArrayInputStream(method))); + dataLengthBuffer.flip(); + if (!HEADER.equals(dataLengthBuffer) || version != CURRENT_VERSION) { + //Warning is ok since this is not supposed to happen. + if (INSECURE_VERSIONS.contains(version)) { + LOG.warn("An insecure client (version '" + version + "') is attempting to connect " + + " to this version '" + CURRENT_VERSION + "' secure server from " + + hostAddress + ":" + remotePort); + } else { + LOG.warn("Incorrect header or version mismatch from " + + hostAddress + ":" + remotePort + + " got version " + version + + " expected version " + CURRENT_VERSION); + } + + return -1; + } + dataLengthBuffer.clear(); + if (authMethod == null) { + throw new IOException("Unable to read authentication method"); + } + if (isSecurityEnabled && authMethod == AuthMethod.SIMPLE) { + AccessControlException ae = new AccessControlException( + "Authentication is required"); + SecureCall failedCall = new SecureCall(AUTHORIZATION_FAILED_CALLID, null, this, + null, 0); + failedCall.setResponse(null, Status.FATAL, ae.getClass().getName(), + ae.getMessage()); + responder.doRespond(failedCall); + throw ae; + } + if (!isSecurityEnabled && authMethod != AuthMethod.SIMPLE) { + doSaslReply(SaslStatus.SUCCESS, new IntWritable( + HBaseSaslRpcServer.SWITCH_TO_SIMPLE_AUTH), null, null); + authMethod = AuthMethod.SIMPLE; + // client has already sent the initial Sasl message and we + // should ignore it. Both client and server should fall back + // to simple auth from now on. + skipInitialSaslHandshake = true; + } + if (authMethod != AuthMethod.SIMPLE) { + useSasl = true; + } + + rpcHeaderBuffer = null; + rpcHeaderRead = true; + continue; + } + + if (data == null) { + dataLengthBuffer.flip(); + dataLength = dataLengthBuffer.getInt(); + + if (dataLength == HBaseClient.PING_CALL_ID) { + if(!useWrap) { //covers the !useSasl too + dataLengthBuffer.clear(); + return 0; //ping message + } + } + if (dataLength < 0) { + LOG.warn("Unexpected data length " + dataLength + "!! from " + + getHostAddress()); + } + data = ByteBuffer.allocate(dataLength); + incRpcCount(); // Increment the rpc count + } + + count = channelRead(channel, data); + + if (data.remaining() == 0) { + dataLengthBuffer.clear(); + data.flip(); + if (skipInitialSaslHandshake) { + data = null; + skipInitialSaslHandshake = false; + continue; + } + boolean isHeaderRead = headerRead; + if (useSasl) { + saslReadAndProcess(data.array()); + } else { + processOneRpc(data.array()); + } + data = null; + if (!isHeaderRead) { + continue; + } + } + return count; + } + } + + /// Reads the connection header following version + private void processHeader(byte[] buf) throws IOException { + DataInputStream in = + new DataInputStream(new ByteArrayInputStream(buf)); + header.readFields(in); + try { + String protocolClassName = header.getProtocol(); + if (protocolClassName != null) { + protocol = getProtocolClass(header.getProtocol(), conf); + } + } catch (ClassNotFoundException cnfe) { + throw new IOException("Unknown protocol: " + header.getProtocol()); + } + + User protocolUser = header.getUser(); + if (!useSasl) { + ticket = protocolUser; + if (ticket != null) { + ticket.getUGI().setAuthenticationMethod(AuthMethod.SIMPLE.authenticationMethod); + } + } else { + // user is authenticated + ticket.getUGI().setAuthenticationMethod(authMethod.authenticationMethod); + //Now we check if this is a proxy user case. If the protocol user is + //different from the 'user', it is a proxy user scenario. However, + //this is not allowed if user authenticated with DIGEST. + if ((protocolUser != null) + && (!protocolUser.getName().equals(ticket.getName()))) { + if (authMethod == AuthMethod.DIGEST) { + // Not allowed to doAs if token authentication is used + throw new AccessControlException("Authenticated user (" + ticket + + ") doesn't match what the client claims to be (" + + protocolUser + ")"); + } else { + // Effective user can be different from authenticated user + // for simple auth or kerberos auth + // The user is the real user. Now we create a proxy user + UserGroupInformation realUser = ticket.getUGI(); + ticket = User.create( + UserGroupInformation.createProxyUser(protocolUser.getName(), + realUser)); + // Now the user is a proxy user, set Authentication method Proxy. + ticket.getUGI().setAuthenticationMethod(AuthenticationMethod.PROXY); + } + } + } + } + + private void processUnwrappedData(byte[] inBuf) throws IOException, + InterruptedException { + ReadableByteChannel ch = Channels.newChannel(new ByteArrayInputStream( + inBuf)); + // Read all RPCs contained in the inBuf, even partial ones + while (true) { + int count = -1; + if (unwrappedDataLengthBuffer.remaining() > 0) { + count = channelRead(ch, unwrappedDataLengthBuffer); + if (count <= 0 || unwrappedDataLengthBuffer.remaining() > 0) + return; + } + + if (unwrappedData == null) { + unwrappedDataLengthBuffer.flip(); + int unwrappedDataLength = unwrappedDataLengthBuffer.getInt(); + + if (unwrappedDataLength == HBaseClient.PING_CALL_ID) { + if (LOG.isTraceEnabled()) { + LOG.trace("Received ping message"); + } + unwrappedDataLengthBuffer.clear(); + continue; // ping message + } + unwrappedData = ByteBuffer.allocate(unwrappedDataLength); + } + + count = channelRead(ch, unwrappedData); + if (count <= 0 || unwrappedData.remaining() > 0) + return; + + if (unwrappedData.remaining() == 0) { + unwrappedDataLengthBuffer.clear(); + unwrappedData.flip(); + processOneRpc(unwrappedData.array()); + unwrappedData = null; + } + } + } + + private void processOneRpc(byte[] buf) throws IOException, + InterruptedException { + if (headerRead) { + processData(buf); + } else { + processHeader(buf); + headerRead = true; + if (!authorizeConnection()) { + throw new AccessControlException("Connection from " + this + + " for protocol " + header.getProtocol() + + " is unauthorized for user " + ticket); + } + } + } + + protected void processData(byte[] buf) throws IOException, InterruptedException { + DataInputStream dis = + new DataInputStream(new ByteArrayInputStream(buf)); + int id = dis.readInt(); // try to read an id + + if (LOG.isTraceEnabled()) { + LOG.trace(" got #" + id); + } + + Writable param = ReflectionUtils.newInstance(paramClass, conf); // read param + param.readFields(dis); + + SecureCall call = new SecureCall(id, param, this, responder, buf.length); + + if (priorityCallQueue != null && getQosLevel(param) > highPriorityLevel) { + priorityCallQueue.put(call); + updateCallQueueLenMetrics(priorityCallQueue); + } else if (replicationQueue != null && getQosLevel(param) == HConstants.REPLICATION_QOS) { + replicationQueue.put(call); + updateCallQueueLenMetrics(replicationQueue); + } else { + callQueue.put(call); // queue the call; maybe blocked here + updateCallQueueLenMetrics(callQueue); + } + } + + private boolean authorizeConnection() throws IOException { + try { + // If auth method is DIGEST, the token was obtained by the + // real user for the effective user, therefore not required to + // authorize real user. doAs is allowed only for simple or kerberos + // authentication + if (ticket != null && ticket.getUGI().getRealUser() != null + && (authMethod != AuthMethod.DIGEST)) { + ProxyUsers.authorize(ticket.getUGI(), this.getHostAddress(), conf); + } + authorize(ticket, header, getHostInetAddress()); + if (LOG.isDebugEnabled()) { + LOG.debug("Successfully authorized " + header); + } + rpcMetrics.authorizationSuccesses.inc(); + } catch (AuthorizationException ae) { + if (LOG.isDebugEnabled()) { + LOG.debug("Connection authorization failed: "+ae.getMessage(), ae); + } + rpcMetrics.authorizationFailures.inc(); + SecureCall failedCall = new SecureCall(AUTHORIZATION_FAILED_CALLID, null, this, + null, 0); + failedCall.setResponse(null, Status.FATAL, ae.getClass().getName(), + ae.getMessage()); + responder.doRespond(failedCall); + return false; + } + return true; + } + + protected synchronized void close() { + disposeSasl(); + data = null; + dataLengthBuffer = null; + if (!channel.isOpen()) + return; + try {socket.shutdownOutput();} catch(Exception ignored) {} // FindBugs DE_MIGHT_IGNORE + if (channel.isOpen()) { + try {channel.close();} catch(Exception ignored) {} + } + try {socket.close();} catch(Exception ignored) {} + } + } + + /** Constructs a server listening on the named port and address. Parameters passed must + * be of the named class. The handlerCount determines + * the number of handler threads that will be used to process calls. + * + */ + @SuppressWarnings("unchecked") + protected SecureServer(String bindAddress, int port, + Class paramClass, int handlerCount, + int priorityHandlerCount, Configuration conf, String serverName, + int highPriorityLevel) + throws IOException { + super(bindAddress, port, paramClass, handlerCount, priorityHandlerCount, + conf, serverName, highPriorityLevel); + this.authorize = + conf.getBoolean(HADOOP_SECURITY_AUTHORIZATION, false); + this.isSecurityEnabled = User.isHBaseSecurityEnabled(this.conf); + + if (isSecurityEnabled) { + HBaseSaslRpcServer.init(conf); + } + } + + @Override + protected Connection getConnection(SocketChannel channel, long time) { + return new SecureConnection(channel, time); + } + + Configuration getConf() { + return conf; + } + + /** for unit testing only, should be called before server is started */ + void disableSecurity() { + this.isSecurityEnabled = false; + } + + /** for unit testing only, should be called before server is started */ + void enableSecurity() { + this.isSecurityEnabled = true; + } + + /** Stops the service. No new calls will be handled after this is called. */ + public synchronized void stop() { + super.stop(); + } + + public SecretManager getSecretManager() { + return this.secretManager; + } + + public void setSecretManager(SecretManager secretManager) { + this.secretManager = (SecretManager) secretManager; + } + + /** + * Authorize the incoming client connection. + * + * @param user client user + * @param connection incoming connection + * @param addr InetAddress of incoming connection + * @throws org.apache.hadoop.security.authorize.AuthorizationException when the client isn't authorized to talk the protocol + */ + public void authorize(User user, + ConnectionHeader connection, + InetAddress addr + ) throws AuthorizationException { + if (authorize) { + Class protocol = null; + try { + protocol = getProtocolClass(connection.getProtocol(), getConf()); + } catch (ClassNotFoundException cfne) { + throw new AuthorizationException("Unknown protocol: " + + connection.getProtocol()); + } + authManager.authorize(user != null ? user.getUGI() : null, + protocol, getConf(), addr); + } + } +} \ No newline at end of file diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/AccessDeniedException.java b/security/src/main/java/org/apache/hadoop/hbase/security/AccessDeniedException.java new file mode 100644 index 0000000..b8c5d3b --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/AccessDeniedException.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security; + +import org.apache.hadoop.hbase.DoNotRetryIOException; + +/** + * Exception thrown by access-related methods. + */ +public class AccessDeniedException extends DoNotRetryIOException { + private static final long serialVersionUID = 1913879564363001780L; + + public AccessDeniedException() { + super(); + } + + public AccessDeniedException(Class clazz, String s) { + super( "AccessDenied [" + clazz.getName() + "]: " + s); + } + + public AccessDeniedException(String s) { + super(s); + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/HBasePolicyProvider.java b/security/src/main/java/org/apache/hadoop/hbase/security/HBasePolicyProvider.java new file mode 100644 index 0000000..cf1d3f1 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/HBasePolicyProvider.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ipc.HMasterInterface; +import org.apache.hadoop.hbase.ipc.HMasterRegionInterface; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.security.authorize.PolicyProvider; +import org.apache.hadoop.security.authorize.Service; +import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; + +/** + * Implementation of secure Hadoop policy provider for mapping + * protocol interfaces to hbase-policy.xml entries. + */ +public class HBasePolicyProvider extends PolicyProvider { + protected static Service[] services = { + new Service("security.client.protocol.acl", HRegionInterface.class), + new Service("security.admin.protocol.acl", HMasterInterface.class), + new Service("security.masterregion.protocol.acl", HMasterRegionInterface.class) + }; + + @Override + public Service[] getServices() { + return services; + } + + public static void init(Configuration conf, + ServiceAuthorizationManager authManager) { + // set service-level authorization security policy + System.setProperty("hadoop.policy.file", "hbase-policy.xml"); + if (conf.getBoolean( + ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, false)) { + authManager.refresh(conf, new HBasePolicyProvider()); + } + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java b/security/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java new file mode 100644 index 0000000..8090973 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.sasl.RealmCallback; +import javax.security.sasl.RealmChoiceCallback; +import javax.security.sasl.Sasl; +import javax.security.sasl.SaslException; +import javax.security.sasl.SaslClient; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.hbase.security.HBaseSaslRpcServer.AuthMethod; +import org.apache.hadoop.hbase.security.HBaseSaslRpcServer.SaslStatus; +import org.apache.hadoop.security.SaslInputStream; +import org.apache.hadoop.security.SaslOutputStream; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; + +/** + * A utility class that encapsulates SASL logic for RPC client. + * Copied from org.apache.hadoop.security + */ +public class HBaseSaslRpcClient { + public static final Log LOG = LogFactory.getLog(HBaseSaslRpcClient.class); + + private final SaslClient saslClient; + + /** + * Create a HBaseSaslRpcClient for an authentication method + * + * @param method + * the requested authentication method + * @param token + * token to use if needed by the authentication method + */ + public HBaseSaslRpcClient(AuthMethod method, + Token token, String serverPrincipal) + throws IOException { + switch (method) { + case DIGEST: + if (LOG.isDebugEnabled()) + LOG.debug("Creating SASL " + AuthMethod.DIGEST.getMechanismName() + + " client to authenticate to service at " + token.getService()); + saslClient = Sasl.createSaslClient(new String[] { AuthMethod.DIGEST + .getMechanismName() }, null, null, HBaseSaslRpcServer.SASL_DEFAULT_REALM, + HBaseSaslRpcServer.SASL_PROPS, new SaslClientCallbackHandler(token)); + break; + case KERBEROS: + if (LOG.isDebugEnabled()) { + LOG + .debug("Creating SASL " + AuthMethod.KERBEROS.getMechanismName() + + " client. Server's Kerberos principal name is " + + serverPrincipal); + } + if (serverPrincipal == null || serverPrincipal.length() == 0) { + throw new IOException( + "Failed to specify server's Kerberos principal name"); + } + String names[] = HBaseSaslRpcServer.splitKerberosName(serverPrincipal); + if (names.length != 3) { + throw new IOException( + "Kerberos principal name does NOT have the expected hostname part: " + + serverPrincipal); + } + saslClient = Sasl.createSaslClient(new String[] { AuthMethod.KERBEROS + .getMechanismName() }, null, names[0], names[1], + HBaseSaslRpcServer.SASL_PROPS, null); + break; + default: + throw new IOException("Unknown authentication method " + method); + } + if (saslClient == null) + throw new IOException("Unable to find SASL client implementation"); + } + + private static void readStatus(DataInputStream inStream) throws IOException { + int id = inStream.readInt(); // read and discard dummy id + int status = inStream.readInt(); // read status + if (status != SaslStatus.SUCCESS.state) { + throw new RemoteException(WritableUtils.readString(inStream), + WritableUtils.readString(inStream)); + } + } + + /** + * Do client side SASL authentication with server via the given InputStream + * and OutputStream + * + * @param inS + * InputStream to use + * @param outS + * OutputStream to use + * @return true if connection is set up, or false if needs to switch + * to simple Auth. + * @throws IOException + */ + public boolean saslConnect(InputStream inS, OutputStream outS) + throws IOException { + DataInputStream inStream = new DataInputStream(new BufferedInputStream(inS)); + DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream( + outS)); + + try { + byte[] saslToken = new byte[0]; + if (saslClient.hasInitialResponse()) + saslToken = saslClient.evaluateChallenge(saslToken); + if (saslToken != null) { + outStream.writeInt(saslToken.length); + outStream.write(saslToken, 0, saslToken.length); + outStream.flush(); + if (LOG.isDebugEnabled()) + LOG.debug("Have sent token of size " + saslToken.length + + " from initSASLContext."); + } + if (!saslClient.isComplete()) { + readStatus(inStream); + int len = inStream.readInt(); + if (len == HBaseSaslRpcServer.SWITCH_TO_SIMPLE_AUTH) { + if (LOG.isDebugEnabled()) + LOG.debug("Server asks us to fall back to simple auth."); + saslClient.dispose(); + return false; + } + saslToken = new byte[len]; + if (LOG.isDebugEnabled()) + LOG.debug("Will read input token of size " + saslToken.length + + " for processing by initSASLContext"); + inStream.readFully(saslToken); + } + + while (!saslClient.isComplete()) { + saslToken = saslClient.evaluateChallenge(saslToken); + if (saslToken != null) { + if (LOG.isDebugEnabled()) + LOG.debug("Will send token of size " + saslToken.length + + " from initSASLContext."); + outStream.writeInt(saslToken.length); + outStream.write(saslToken, 0, saslToken.length); + outStream.flush(); + } + if (!saslClient.isComplete()) { + readStatus(inStream); + saslToken = new byte[inStream.readInt()]; + if (LOG.isDebugEnabled()) + LOG.debug("Will read input token of size " + saslToken.length + + " for processing by initSASLContext"); + inStream.readFully(saslToken); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("SASL client context established. Negotiated QoP: " + + saslClient.getNegotiatedProperty(Sasl.QOP)); + } + return true; + } catch (IOException e) { + try { + saslClient.dispose(); + } catch (SaslException ignored) { + // ignore further exceptions during cleanup + } + throw e; + } + } + + /** + * Get a SASL wrapped InputStream. Can be called only after saslConnect() has + * been called. + * + * @param in + * the InputStream to wrap + * @return a SASL wrapped InputStream + * @throws IOException + */ + public InputStream getInputStream(InputStream in) throws IOException { + if (!saslClient.isComplete()) { + throw new IOException("Sasl authentication exchange hasn't completed yet"); + } + return new SaslInputStream(in, saslClient); + } + + /** + * Get a SASL wrapped OutputStream. Can be called only after saslConnect() has + * been called. + * + * @param out + * the OutputStream to wrap + * @return a SASL wrapped OutputStream + * @throws IOException + */ + public OutputStream getOutputStream(OutputStream out) throws IOException { + if (!saslClient.isComplete()) { + throw new IOException("Sasl authentication exchange hasn't completed yet"); + } + return new SaslOutputStream(out, saslClient); + } + + /** Release resources used by wrapped saslClient */ + public void dispose() throws SaslException { + saslClient.dispose(); + } + + private static class SaslClientCallbackHandler implements CallbackHandler { + private final String userName; + private final char[] userPassword; + + public SaslClientCallbackHandler(Token token) { + this.userName = HBaseSaslRpcServer.encodeIdentifier(token.getIdentifier()); + this.userPassword = HBaseSaslRpcServer.encodePassword(token.getPassword()); + } + + public void handle(Callback[] callbacks) + throws UnsupportedCallbackException { + NameCallback nc = null; + PasswordCallback pc = null; + RealmCallback rc = null; + for (Callback callback : callbacks) { + if (callback instanceof RealmChoiceCallback) { + continue; + } else if (callback instanceof NameCallback) { + nc = (NameCallback) callback; + } else if (callback instanceof PasswordCallback) { + pc = (PasswordCallback) callback; + } else if (callback instanceof RealmCallback) { + rc = (RealmCallback) callback; + } else { + throw new UnsupportedCallbackException(callback, + "Unrecognized SASL client callback"); + } + } + if (nc != null) { + if (LOG.isDebugEnabled()) + LOG.debug("SASL client callback: setting username: " + userName); + nc.setName(userName); + } + if (pc != null) { + if (LOG.isDebugEnabled()) + LOG.debug("SASL client callback: setting userPassword"); + pc.setPassword(userPassword); + } + if (rc != null) { + if (LOG.isDebugEnabled()) + LOG.debug("SASL client callback: setting realm: " + + rc.getDefaultText()); + rc.setText(rc.getDefaultText()); + } + } + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcServer.java b/security/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcServer.java new file mode 100644 index 0000000..5ffbb4f --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcServer.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security; + +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Map; +import java.util.TreeMap; + +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.sasl.AuthorizeCallback; +import javax.security.sasl.RealmCallback; +import javax.security.sasl.Sasl; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ipc.HBaseServer; +import org.apache.hadoop.hbase.ipc.SecureServer; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; + +/** + * A utility class for dealing with SASL on RPC server + */ +public class HBaseSaslRpcServer { + public static final Log LOG = LogFactory.getLog(HBaseSaslRpcServer.class); + public static final String SASL_DEFAULT_REALM = "default"; + public static final Map SASL_PROPS = + new TreeMap(); + + public static final int SWITCH_TO_SIMPLE_AUTH = -88; + + public static enum QualityOfProtection { + AUTHENTICATION("auth"), + INTEGRITY("auth-int"), + PRIVACY("auth-conf"); + + public final String saslQop; + + private QualityOfProtection(String saslQop) { + this.saslQop = saslQop; + } + + public String getSaslQop() { + return saslQop; + } + } + + public static void init(Configuration conf) { + QualityOfProtection saslQOP = QualityOfProtection.AUTHENTICATION; + String rpcProtection = conf.get("hbase.rpc.protection", + QualityOfProtection.AUTHENTICATION.name().toLowerCase()); + if (QualityOfProtection.INTEGRITY.name().toLowerCase() + .equals(rpcProtection)) { + saslQOP = QualityOfProtection.INTEGRITY; + } else if (QualityOfProtection.PRIVACY.name().toLowerCase().equals( + rpcProtection)) { + saslQOP = QualityOfProtection.PRIVACY; + } + + SASL_PROPS.put(Sasl.QOP, saslQOP.getSaslQop()); + SASL_PROPS.put(Sasl.SERVER_AUTH, "true"); + } + + static String encodeIdentifier(byte[] identifier) { + return new String(Base64.encodeBase64(identifier)); + } + + static byte[] decodeIdentifier(String identifier) { + return Base64.decodeBase64(identifier.getBytes()); + } + + public static T getIdentifier(String id, + SecretManager secretManager) throws InvalidToken { + byte[] tokenId = decodeIdentifier(id); + T tokenIdentifier = secretManager.createIdentifier(); + try { + tokenIdentifier.readFields(new DataInputStream(new ByteArrayInputStream( + tokenId))); + } catch (IOException e) { + throw (InvalidToken) new InvalidToken( + "Can't de-serialize tokenIdentifier").initCause(e); + } + return tokenIdentifier; + } + + static char[] encodePassword(byte[] password) { + return new String(Base64.encodeBase64(password)).toCharArray(); + } + + /** Splitting fully qualified Kerberos name into parts */ + public static String[] splitKerberosName(String fullName) { + return fullName.split("[/@]"); + } + + public enum SaslStatus { + SUCCESS (0), + ERROR (1); + + public final int state; + private SaslStatus(int state) { + this.state = state; + } + } + + /** Authentication method */ + public static enum AuthMethod { + SIMPLE((byte) 80, "", AuthenticationMethod.SIMPLE), + KERBEROS((byte) 81, "GSSAPI", AuthenticationMethod.KERBEROS), + DIGEST((byte) 82, "DIGEST-MD5", AuthenticationMethod.TOKEN); + + /** The code for this method. */ + public final byte code; + public final String mechanismName; + public final AuthenticationMethod authenticationMethod; + + private AuthMethod(byte code, String mechanismName, + AuthenticationMethod authMethod) { + this.code = code; + this.mechanismName = mechanismName; + this.authenticationMethod = authMethod; + } + + private static final int FIRST_CODE = values()[0].code; + + /** Return the object represented by the code. */ + private static AuthMethod valueOf(byte code) { + final int i = (code & 0xff) - FIRST_CODE; + return i < 0 || i >= values().length ? null : values()[i]; + } + + /** Return the SASL mechanism name */ + public String getMechanismName() { + return mechanismName; + } + + /** Read from in */ + public static AuthMethod read(DataInput in) throws IOException { + return valueOf(in.readByte()); + } + + /** Write to out */ + public void write(DataOutput out) throws IOException { + out.write(code); + } + }; + + /** CallbackHandler for SASL DIGEST-MD5 mechanism */ + public static class SaslDigestCallbackHandler implements CallbackHandler { + private SecretManager secretManager; + private SecureServer.SecureConnection connection; + + public SaslDigestCallbackHandler( + SecretManager secretManager, + SecureServer.SecureConnection connection) { + this.secretManager = secretManager; + this.connection = connection; + } + + private char[] getPassword(TokenIdentifier tokenid) throws InvalidToken { + return encodePassword(secretManager.retrievePassword(tokenid)); + } + + /** {@inheritDoc} */ + @Override + public void handle(Callback[] callbacks) throws InvalidToken, + UnsupportedCallbackException { + NameCallback nc = null; + PasswordCallback pc = null; + AuthorizeCallback ac = null; + for (Callback callback : callbacks) { + if (callback instanceof AuthorizeCallback) { + ac = (AuthorizeCallback) callback; + } else if (callback instanceof NameCallback) { + nc = (NameCallback) callback; + } else if (callback instanceof PasswordCallback) { + pc = (PasswordCallback) callback; + } else if (callback instanceof RealmCallback) { + continue; // realm is ignored + } else { + throw new UnsupportedCallbackException(callback, + "Unrecognized SASL DIGEST-MD5 Callback"); + } + } + if (pc != null) { + TokenIdentifier tokenIdentifier = getIdentifier(nc.getDefaultName(), secretManager); + char[] password = getPassword(tokenIdentifier); + UserGroupInformation user = null; + user = tokenIdentifier.getUser(); // may throw exception + connection.attemptingUser = user; + if (LOG.isDebugEnabled()) { + LOG.debug("SASL server DIGEST-MD5 callback: setting password " + + "for client: " + tokenIdentifier.getUser()); + } + pc.setPassword(password); + } + if (ac != null) { + String authid = ac.getAuthenticationID(); + String authzid = ac.getAuthorizationID(); + if (authid.equals(authzid)) { + ac.setAuthorized(true); + } else { + ac.setAuthorized(false); + } + if (ac.isAuthorized()) { + if (LOG.isDebugEnabled()) { + String username = + getIdentifier(authzid, secretManager).getUser().getUserName(); + LOG.debug("SASL server DIGEST-MD5 callback: setting " + + "canonicalized client ID: " + username); + } + ac.setAuthorizedID(authzid); + } + } + } + } + + /** CallbackHandler for SASL GSSAPI Kerberos mechanism */ + public static class SaslGssCallbackHandler implements CallbackHandler { + + /** {@inheritDoc} */ + @Override + public void handle(Callback[] callbacks) throws + UnsupportedCallbackException { + AuthorizeCallback ac = null; + for (Callback callback : callbacks) { + if (callback instanceof AuthorizeCallback) { + ac = (AuthorizeCallback) callback; + } else { + throw new UnsupportedCallbackException(callback, + "Unrecognized SASL GSSAPI Callback"); + } + } + if (ac != null) { + String authid = ac.getAuthenticationID(); + String authzid = ac.getAuthorizationID(); + if (authid.equals(authzid)) { + ac.setAuthorized(true); + } else { + ac.setAuthorized(false); + } + if (ac.isAuthorized()) { + if (LOG.isDebugEnabled()) + LOG.debug("SASL server GSSAPI callback: setting " + + "canonicalized client ID: " + authzid); + ac.setAuthorizedID(authzid); + } + } + } + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java new file mode 100644 index 0000000..cd32df9 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.filter.FilterBase; +import org.apache.hadoop.hbase.security.User; + +/** + * NOTE: for internal use only by AccessController implementation + * + *

+ * TODO: There is room for further performance optimization here. + * Calling TableAuthManager.authorize() per KeyValue imposes a fair amount of + * overhead. A more optimized solution might look at the qualifiers where + * permissions are actually granted and explicitly limit the scan to those. + *

+ *

+ * We should aim to use this _only_ when access to the requested column families + * is not granted at the column family levels. If table or column family + * access succeeds, then there is no need to impose the overhead of this filter. + *

+ */ +class AccessControlFilter extends FilterBase { + + private TableAuthManager authManager; + private byte[] table; + private User user; + + /** + * For Writable + */ + AccessControlFilter() { + } + + AccessControlFilter(TableAuthManager mgr, User ugi, + byte[] tableName) { + authManager = mgr; + table = tableName; + user = ugi; + } + + @Override + public ReturnCode filterKeyValue(KeyValue kv) { + if (authManager.authorize(user, table, kv, TablePermission.Action.READ)) { + return ReturnCode.INCLUDE; + } + return ReturnCode.NEXT_COL; + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + // no implementation, server-side use only + throw new UnsupportedOperationException( + "Serialization not supported. Intended for server-side use only."); + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + // no implementation, server-side use only + throw new UnsupportedOperationException( + "Serialization not supported. Intended for server-side use only."); + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java new file mode 100644 index 0000000..5b4b53d --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java @@ -0,0 +1,579 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.RegexStringComparator; +import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.io.Text; + +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.*; + +/** + * Maintains lists of permission grants to users and groups to allow for + * authorization checks by {@link AccessController}. + * + *

+ * Access control lists are stored in an "internal" metadata table named + * {@code _acl_}. Each table's permission grants are stored as a separate row, + * keyed by the table name. KeyValues for permissions assignments are stored + * in one of the formats: + * 

+ * Key                      Desc
+ * --------                 --------
+ * user                     table level permissions for a user [R=read, W=write]
+ * @group                   table level permissions for a group
+ * user,family              column family level permissions for a user
+ * @group,family            column family level permissions for a group
+ * user,family,qualifier    column qualifier level permissions for a user
+ * @group,family,qualifier  column qualifier level permissions for a group
+ *
+ * All values are encoded as byte arrays containing the codes from the + * {@link org.apache.hadoop.hbase.security.access.TablePermission.Action} enum. + *

+ */ +public class AccessControlLists { + /** Internal storage table for access control lists */ + public static final String ACL_TABLE_NAME_STR = "_acl_"; + public static final byte[] ACL_TABLE_NAME = Bytes.toBytes(ACL_TABLE_NAME_STR); + public static final byte[] ACL_GLOBAL_NAME = ACL_TABLE_NAME; + /** Column family used to store ACL grants */ + public static final String ACL_LIST_FAMILY_STR = "l"; + public static final byte[] ACL_LIST_FAMILY = Bytes.toBytes(ACL_LIST_FAMILY_STR); + + /** Table descriptor for ACL internal table */ + public static final HTableDescriptor ACL_TABLEDESC = new HTableDescriptor( + ACL_TABLE_NAME); + static { + ACL_TABLEDESC.addFamily( + new HColumnDescriptor(ACL_LIST_FAMILY, + 10, // Ten is arbitrary number. Keep versions to help debugging. + Compression.Algorithm.NONE.getName(), true, true, 8 * 1024, + HConstants.FOREVER, StoreFile.BloomType.NONE.toString(), + HConstants.REPLICATION_SCOPE_LOCAL)); + } + + /** + * Delimiter to separate user, column family, and qualifier in + * _acl_ table info: column keys */ + public static final char ACL_KEY_DELIMITER = ','; + /** Prefix character to denote group names */ + public static final String GROUP_PREFIX = "@"; + /** Configuration key for superusers */ + public static final String SUPERUSER_CONF_KEY = "hbase.superuser"; + + private static Log LOG = LogFactory.getLog(AccessControlLists.class); + + /** + * Check for existence of {@code _acl_} table and create it if it does not exist + * @param master reference to HMaster + */ + static void init(MasterServices master) throws IOException { + if (!MetaReader.tableExists(master.getCatalogTracker(), ACL_TABLE_NAME_STR)) { + master.createTable(ACL_TABLEDESC, null); + } + } + + /** + * Stores a new user permission grant in the access control lists table. + * @param conf the configuration + * @param userPerm the details of the permission to be granted + * @throws IOException in the case of an error accessing the metadata table + */ + static void addUserPermission(Configuration conf, UserPermission userPerm) + throws IOException { + Permission.Action[] actions = userPerm.getActions(); + + Put p = new Put(userPerm.isGlobal() ? ACL_GLOBAL_NAME : userPerm.getTable()); + byte[] key = userPermissionKey(userPerm); + + if ((actions == null) || (actions.length == 0)) { + String msg = "No actions associated with user '" + Bytes.toString(userPerm.getUser()) + "'"; + LOG.warn(msg); + throw new IOException(msg); + } + + byte[] value = new byte[actions.length]; + for (int i = 0; i < actions.length; i++) { + value[i] = actions[i].code(); + } + p.add(ACL_LIST_FAMILY, key, value); + if (LOG.isDebugEnabled()) { + LOG.debug("Writing permission for table "+ + Bytes.toString(userPerm.getTable())+" "+ + Bytes.toString(key)+": "+Bytes.toStringBinary(value) + ); + } + HTable acls = null; + try { + acls = new HTable(conf, ACL_TABLE_NAME); + acls.put(p); + } finally { + if (acls != null) acls.close(); + } + } + + /** + * Removes a previously granted permission from the stored access control + * lists. The {@link TablePermission} being removed must exactly match what + * is stored -- no wildcard matching is attempted. Ie, if user "bob" has + * been granted "READ" access to the "data" table, but only to column family + * plus qualifier "info:colA", then trying to call this method with only + * user "bob" and the table name "data" (but without specifying the + * column qualifier "info:colA") will have no effect. + * + * @param conf the configuration + * @param userPerm the details of the permission to be revoked + * @throws IOException if there is an error accessing the metadata table + */ + static void removeUserPermission(Configuration conf, UserPermission userPerm) + throws IOException { + + Delete d = new Delete(userPerm.isGlobal() ? ACL_GLOBAL_NAME : userPerm.getTable()); + byte[] key = userPermissionKey(userPerm); + + if (LOG.isDebugEnabled()) { + LOG.debug("Removing permission "+ userPerm.toString()); + } + d.deleteColumns(ACL_LIST_FAMILY, key); + HTable acls = null; + try { + acls = new HTable(conf, ACL_TABLE_NAME); + acls.delete(d); + } finally { + if (acls != null) acls.close(); + } + } + + /** + * Remove specified table from the _acl_ table. + */ + static void removeTablePermissions(Configuration conf, byte[] tableName) + throws IOException{ + Delete d = new Delete(tableName); + + if (LOG.isDebugEnabled()) { + LOG.debug("Removing permissions of removed table "+ Bytes.toString(tableName)); + } + + HTable acls = null; + try { + acls = new HTable(conf, ACL_TABLE_NAME); + acls.delete(d); + } finally { + if (acls != null) acls.close(); + } + } + + /** + * Remove specified table column from the _acl_ table. + */ + static void removeTablePermissions(Configuration conf, byte[] tableName, byte[] column) + throws IOException{ + + if (LOG.isDebugEnabled()) { + LOG.debug("Removing permissions of removed column " + Bytes.toString(column) + + " from table "+ Bytes.toString(tableName)); + } + + HTable acls = null; + try { + acls = new HTable(conf, ACL_TABLE_NAME); + + Scan scan = new Scan(); + scan.addFamily(ACL_LIST_FAMILY); + + String columnName = Bytes.toString(column); + scan.setFilter(new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator( + String.format("(%s%s%s)|(%s%s)$", + ACL_KEY_DELIMITER, columnName, ACL_KEY_DELIMITER, + ACL_KEY_DELIMITER, columnName)))); + + Set qualifierSet = new TreeSet(Bytes.BYTES_COMPARATOR); + ResultScanner scanner = acls.getScanner(scan); + try { + for (Result res : scanner) { + for (byte[] q : res.getFamilyMap(ACL_LIST_FAMILY).navigableKeySet()) { + qualifierSet.add(q); + } + } + } finally { + scanner.close(); + } + + if (qualifierSet.size() > 0) { + Delete d = new Delete(tableName); + for (byte[] qualifier : qualifierSet) { + d.deleteColumns(ACL_LIST_FAMILY, qualifier); + } + acls.delete(d); + } + } finally { + if (acls != null) acls.close(); + } + } + + /** + * Build qualifier key from user permission: + * username + * username,family + * username,family,qualifier + */ + static byte[] userPermissionKey(UserPermission userPerm) { + byte[] qualifier = userPerm.getQualifier(); + byte[] family = userPerm.getFamily(); + byte[] key = userPerm.getUser(); + + if (family != null && family.length > 0) { + key = Bytes.add(key, Bytes.add(new byte[]{ACL_KEY_DELIMITER}, family)); + if (qualifier != null && qualifier.length > 0) { + key = Bytes.add(key, Bytes.add(new byte[]{ACL_KEY_DELIMITER}, qualifier)); + } + } + + return key; + } + + /** + * Returns {@code true} if the given region is part of the {@code _acl_} + * metadata table. + */ + static boolean isAclRegion(HRegion region) { + return Bytes.equals(ACL_TABLE_NAME, region.getTableDesc().getName()); + } + + /** + * Returns {@code true} if the given table is {@code _acl_} metadata table. + */ + static boolean isAclTable(HTableDescriptor desc) { + return Bytes.equals(ACL_TABLE_NAME, desc.getName()); + } + + /** + * Loads all of the permission grants stored in a region of the {@code _acl_} + * table. + * + * @param aclRegion + * @return + * @throws IOException + */ + static Map> loadAll( + HRegion aclRegion) + throws IOException { + + if (!isAclRegion(aclRegion)) { + throw new IOException("Can only load permissions from "+ACL_TABLE_NAME_STR); + } + + Map> allPerms = + new TreeMap>(Bytes.BYTES_COMPARATOR); + + // do a full scan of _acl_ table + + Scan scan = new Scan(); + scan.addFamily(ACL_LIST_FAMILY); + + InternalScanner iScanner = null; + try { + iScanner = aclRegion.getScanner(scan); + + while (true) { + List row = new ArrayList(); + + boolean hasNext = iScanner.next(row); + ListMultimap perms = ArrayListMultimap.create(); + byte[] table = null; + for (KeyValue kv : row) { + if (table == null) { + table = kv.getRow(); + } + Pair permissionsOfUserOnTable = + parseTablePermissionRecord(table, kv); + if (permissionsOfUserOnTable != null) { + String username = permissionsOfUserOnTable.getFirst(); + TablePermission permissions = permissionsOfUserOnTable.getSecond(); + perms.put(username, permissions); + } + } + if (table != null) { + allPerms.put(table, perms); + } + if (!hasNext) { + break; + } + } + } finally { + if (iScanner != null) { + iScanner.close(); + } + } + + return allPerms; + } + + /** + * Load all permissions from the region server holding {@code _acl_}, + * primarily intended for testing purposes. + */ + static Map> loadAll( + Configuration conf) throws IOException { + Map> allPerms = + new TreeMap>(Bytes.BYTES_COMPARATOR); + + // do a full scan of _acl_, filtering on only first table region rows + + Scan scan = new Scan(); + scan.addFamily(ACL_LIST_FAMILY); + + HTable acls = null; + ResultScanner scanner = null; + try { + acls = new HTable(conf, ACL_TABLE_NAME); + scanner = acls.getScanner(scan); + for (Result row : scanner) { + ListMultimap resultPerms = + parseTablePermissions(row.getRow(), row); + allPerms.put(row.getRow(), resultPerms); + } + } finally { + if (scanner != null) scanner.close(); + if (acls != null) acls.close(); + } + + return allPerms; + } + + /** + * Reads user permission assignments stored in the l: column + * family of the first table row in _acl_. + * + *

+ * See {@link AccessControlLists class documentation} for the key structure + * used for storage. + *

+ */ + static ListMultimap getTablePermissions(Configuration conf, + byte[] tableName) throws IOException { + if (tableName == null) tableName = ACL_TABLE_NAME; + + // for normal user tables, we just read the table row from _acl_ + ListMultimap perms = ArrayListMultimap.create(); + HTable acls = null; + try { + acls = new HTable(conf, ACL_TABLE_NAME); + Get get = new Get(tableName); + get.addFamily(ACL_LIST_FAMILY); + Result row = acls.get(get); + if (!row.isEmpty()) { + perms = parseTablePermissions(tableName, row); + } else { + LOG.info("No permissions found in " + ACL_TABLE_NAME_STR + " for table " + + Bytes.toString(tableName)); + } + } finally { + if (acls != null) acls.close(); + } + + return perms; + } + + /** + * Returns the currently granted permissions for a given table as a list of + * user plus associated permissions. + */ + static List getUserPermissions( + Configuration conf, byte[] tableName) + throws IOException { + ListMultimap allPerms = getTablePermissions( + conf, tableName); + + List perms = new ArrayList(); + + for (Map.Entry entry : allPerms.entries()) { + UserPermission up = new UserPermission(Bytes.toBytes(entry.getKey()), + entry.getValue().getTable(), entry.getValue().getFamily(), + entry.getValue().getQualifier(), entry.getValue().getActions()); + perms.add(up); + } + return perms; + } + + private static ListMultimap parseTablePermissions( + byte[] table, Result result) { + ListMultimap perms = ArrayListMultimap.create(); + if (result != null && result.size() > 0) { + for (KeyValue kv : result.raw()) { + + Pair permissionsOfUserOnTable = + parseTablePermissionRecord(table, kv); + + if (permissionsOfUserOnTable != null) { + String username = permissionsOfUserOnTable.getFirst(); + TablePermission permissions = permissionsOfUserOnTable.getSecond(); + perms.put(username, permissions); + } + } + } + return perms; + } + + private static Pair parseTablePermissionRecord( + byte[] table, KeyValue kv) { + // return X given a set of permissions encoded in the permissionRecord kv. + byte[] family = kv.getFamily(); + + if (!Bytes.equals(family, ACL_LIST_FAMILY)) { + return null; + } + + byte[] key = kv.getQualifier(); + byte[] value = kv.getValue(); + if (LOG.isDebugEnabled()) { + LOG.debug("Read acl: kv ["+ + Bytes.toStringBinary(key)+": "+ + Bytes.toStringBinary(value)+"]"); + } + + // check for a column family appended to the key + // TODO: avoid the string conversion to make this more efficient + String username = Bytes.toString(key); + int idx = username.indexOf(ACL_KEY_DELIMITER); + byte[] permFamily = null; + byte[] permQualifier = null; + if (idx > 0 && idx < username.length()-1) { + String remainder = username.substring(idx+1); + username = username.substring(0, idx); + idx = remainder.indexOf(ACL_KEY_DELIMITER); + if (idx > 0 && idx < remainder.length()-1) { + permFamily = Bytes.toBytes(remainder.substring(0, idx)); + permQualifier = Bytes.toBytes(remainder.substring(idx+1)); + } else { + permFamily = Bytes.toBytes(remainder); + } + } + + return new Pair( + username, new TablePermission(table, permFamily, permQualifier, value)); + } + + /** + * Writes a set of permissions as {@link org.apache.hadoop.io.Writable} instances + * to the given output stream. + * @param out + * @param perms + * @param conf + * @throws IOException + */ + public static void writePermissions(DataOutput out, + ListMultimap perms, Configuration conf) + throws IOException { + Set keys = perms.keySet(); + out.writeInt(keys.size()); + for (String key : keys) { + Text.writeString(out, key); + HbaseObjectWritable.writeObject(out, perms.get(key), List.class, conf); + } + } + + /** + * Writes a set of permissions as {@link org.apache.hadoop.io.Writable} instances + * and returns the resulting byte array. + */ + public static byte[] writePermissionsAsBytes( + ListMultimap perms, Configuration conf) { + try { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + writePermissions(new DataOutputStream(bos), perms, conf); + return bos.toByteArray(); + } catch (IOException ioe) { + // shouldn't happen here + LOG.error("Error serializing permissions", ioe); + } + return null; + } + + /** + * Reads a set of permissions as {@link org.apache.hadoop.io.Writable} instances + * from the input stream. + */ + public static ListMultimap readPermissions( + DataInput in, Configuration conf) throws IOException { + ListMultimap perms = ArrayListMultimap.create(); + int length = in.readInt(); + for (int i=0; i userPerms = + (List)HbaseObjectWritable.readObject(in, conf); + perms.putAll(user, userPerms); + } + + return perms; + } + + /** + * Returns whether or not the given name should be interpreted as a group + * principal. Currently this simply checks if the name starts with the + * special group prefix character ("@"). + */ + public static boolean isGroupPrincipal(String name) { + return name != null && name.startsWith(GROUP_PREFIX); + } + + /** + * Returns the actual name for a group principal (stripped of the + * group prefix). + */ + public static String getGroupName(String aclKey) { + if (!isGroupPrincipal(aclKey)) { + return aclKey; + } + + return aclKey.substring(GROUP_PREFIX.length()); + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java new file mode 100644 index 0000000..aaff069 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -0,0 +1,1431 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.InetAddress; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.CoprocessorException; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.MasterObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.RegionObserver; +import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.RegionServerObserver; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; +import org.apache.hadoop.hbase.ipc.HBaseRPC; +import org.apache.hadoop.hbase.ipc.ProtocolSignature; +import org.apache.hadoop.hbase.ipc.RequestContext; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.security.AccessDeniedException; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.access.Permission.Action; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hbase.util.Pair; + +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.MapMaker; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +/** + * Provides basic authorization checks for data access and administrative + * operations. + * + *

+ * {@code AccessController} performs authorization checks for HBase operations + * based on: + *

    + *
  • the identity of the user performing the operation
    • + *
  • the scope over which the operation is performed, in increasing + * specificity: global, table, column family, or qualifier
    • + *
  • the type of action being performed (as mapped to + * {@link Permission.Action} values)
    • + *
+ * If the authorization check fails, an {@link AccessDeniedException} + * will be thrown for the operation. + *

+ * + *

+ * To perform authorization checks, {@code AccessController} relies on the + * {@link org.apache.hadoop.hbase.ipc.SecureRpcEngine} being loaded to provide + * the user identities for remote requests. + *

+ * + *

+ * The access control lists used for authorization can be manipulated via the + * exposed {@link AccessControllerProtocol} implementation, and the associated + * {@code grant}, {@code revoke}, and {@code user_permission} HBase shell + * commands. + *

+ */ +public class AccessController extends BaseRegionObserver + implements MasterObserver, RegionServerObserver, AccessControllerProtocol { + /** + * Represents the result of an authorization check for logging and error + * reporting. + */ + private static class AuthResult { + private final boolean allowed; + private final byte[] table; + private final byte[] family; + private final byte[] qualifier; + private final Permission.Action action; + private final String request; + private final String reason; + private final User user; + + public AuthResult(boolean allowed, String request, String reason, User user, + Permission.Action action, byte[] table, byte[] family, byte[] qualifier) { + this.allowed = allowed; + this.request = request; + this.reason = reason; + this.user = user; + this.table = table; + this.family = family; + this.qualifier = qualifier; + this.action = action; + } + + public boolean isAllowed() { return allowed; } + + public User getUser() { return user; } + + public String getReason() { return reason; } + + public String getRequest() { return request; } + + public String toContextString() { + return "(user=" + (user != null ? user.getName() : "UNKNOWN") + ", " + + "scope=" + (table == null ? "GLOBAL" : Bytes.toString(table)) + ", " + + "family=" + (family != null ? Bytes.toString(family) : "") + ", " + + "qualifer=" + (qualifier != null ? Bytes.toString(qualifier) : "") + ", " + + "action=" + (action != null ? action.toString() : "") + ")"; + } + + public String toString() { + return new StringBuilder("AuthResult") + .append(toContextString()).toString(); + } + + public static AuthResult allow(String request, String reason, User user, Permission.Action action, + byte[] table, byte[] family, byte[] qualifier) { + return new AuthResult(true, request, reason, user, action, table, family, qualifier); + } + + public static AuthResult allow(String request, String reason, User user, Permission.Action action, byte[] table) { + return new AuthResult(true, request, reason, user, action, table, null, null); + } + + public static AuthResult deny(String request, String reason, User user, + Permission.Action action, byte[] table) { + return new AuthResult(false, request, reason, user, action, table, null, null); + } + + public static AuthResult deny(String request, String reason, User user, + Permission.Action action, byte[] table, byte[] family, byte[] qualifier) { + return new AuthResult(false, request, reason, user, action, table, family, qualifier); + } + } + + public static final Log LOG = LogFactory.getLog(AccessController.class); + + private static final Log AUDITLOG = + LogFactory.getLog("SecurityLogger."+AccessController.class.getName()); + + /** + * Version number for AccessControllerProtocol + */ + private static final long PROTOCOL_VERSION = 1L; + + TableAuthManager authManager = null; + + // flags if we are running on a region of the _acl_ table + boolean aclRegion = false; + + // defined only for Endpoint implementation, so it can have way to + // access region services. + private RegionCoprocessorEnvironment regionEnv; + + /** Mapping of scanner instances to the user who created them */ + private Map scannerOwners = + new MapMaker().weakKeys().makeMap(); + + void initialize(RegionCoprocessorEnvironment e) throws IOException { + final HRegion region = e.getRegion(); + + Map> tables = + AccessControlLists.loadAll(region); + // For each table, write out the table's permissions to the respective + // znode for that table. + for (Map.Entry> t: + tables.entrySet()) { + byte[] table = t.getKey(); + ListMultimap perms = t.getValue(); + byte[] serialized = AccessControlLists.writePermissionsAsBytes(perms, + regionEnv.getConfiguration()); + this.authManager.getZKPermissionWatcher().writeToZookeeper(table, serialized); + } + } + + /** + * Writes all table ACLs for the tables in the given Map up into ZooKeeper + * znodes. This is called to synchronize ACL changes following {@code _acl_} + * table updates. + */ + void updateACL(RegionCoprocessorEnvironment e, + final Map> familyMap) { + Set tableSet = new TreeSet(Bytes.BYTES_COMPARATOR); + for (Map.Entry> f : familyMap.entrySet()) { + List kvs = f.getValue(); + for (KeyValue kv: kvs) { + if (Bytes.equals(kv.getBuffer(), kv.getFamilyOffset(), + kv.getFamilyLength(), AccessControlLists.ACL_LIST_FAMILY, 0, + AccessControlLists.ACL_LIST_FAMILY.length)) { + tableSet.add(kv.getRow()); + } + } + } + + ZKPermissionWatcher zkw = this.authManager.getZKPermissionWatcher(); + Configuration conf = regionEnv.getConfiguration(); + for (byte[] tableName: tableSet) { + try { + ListMultimap perms = + AccessControlLists.getTablePermissions(conf, tableName); + byte[] serialized = AccessControlLists.writePermissionsAsBytes(perms, conf); + zkw.writeToZookeeper(tableName, serialized); + } catch (IOException ex) { + LOG.error("Failed updating permissions mirror for '" + tableName + "'", ex); + } + } + } + + /** + * Check the current user for authorization to perform a specific action + * against the given set of row data. + * + *

Note: Ordering of the authorization checks + * has been carefully optimized to short-circuit the most common requests + * and minimize the amount of processing required.

+ * + * @param permRequest the action being requested + * @param e the coprocessor environment + * @param families the map of column families to qualifiers present in + * the request + * @return + */ + AuthResult permissionGranted(String request, User user, TablePermission.Action permRequest, + RegionCoprocessorEnvironment e, + Map> families) { + HRegionInfo hri = e.getRegion().getRegionInfo(); + byte[] tableName = hri.getTableName(); + + // 1. All users need read access to .META. and -ROOT- tables. + // this is a very common operation, so deal with it quickly. + if (hri.isRootRegion() || hri.isMetaRegion()) { + if (permRequest == TablePermission.Action.READ) { + return AuthResult.allow(request, "All users allowed", user, permRequest, tableName); + } + } + + if (user == null) { + return AuthResult.deny(request, "No user associated with request!", null, permRequest, tableName); + } + + // Users with CREATE/ADMIN rights need to modify .META. and _acl_ table + // e.g. When a new table is created a new entry in .META. is added, + // so the user need to be allowed to write on it. + // e.g. When a table is removed an entry is removed from .META. and _acl_ + // and the user need to be allowed to write on both tables. + if (permRequest == TablePermission.Action.WRITE && + (hri.isRootRegion() || hri.isMetaRegion() || + Bytes.equals(tableName, AccessControlLists.ACL_GLOBAL_NAME)) && + (authManager.authorize(user, Permission.Action.CREATE) || + authManager.authorize(user, Permission.Action.ADMIN))) + { + return AuthResult.allow(request, "Table permission granted", user, permRequest, tableName); + } + + // 2. check for the table-level, if successful we can short-circuit + if (authManager.authorize(user, tableName, (byte[])null, permRequest)) { + return AuthResult.allow(request, "Table permission granted", user, permRequest, tableName); + } + + // 3. check permissions against the requested families + if (families != null && families.size() > 0) { + // all families must pass + for (Map.Entry> family : families.entrySet()) { + // a) check for family level access + if (authManager.authorize(user, tableName, family.getKey(), + permRequest)) { + continue; // family-level permission overrides per-qualifier + } + + // b) qualifier level access can still succeed + if ((family.getValue() != null) && (family.getValue().size() > 0)) { + if (family.getValue() instanceof Set) { + // for each qualifier of the family + Set familySet = (Set)family.getValue(); + for (byte[] qualifier : familySet) { + if (!authManager.authorize(user, tableName, family.getKey(), + qualifier, permRequest)) { + return AuthResult.deny(request, "Failed qualifier check", user, + permRequest, tableName, family.getKey(), qualifier); + } + } + } else if (family.getValue() instanceof List) { // List + List kvList = (List)family.getValue(); + for (KeyValue kv : kvList) { + if (!authManager.authorize(user, tableName, family.getKey(), + kv.getQualifier(), permRequest)) { + return AuthResult.deny(request, "Failed qualifier check", user, + permRequest, tableName, family.getKey(), kv.getQualifier()); + } + } + } + } else { + // no qualifiers and family-level check already failed + return AuthResult.deny(request, "Failed family check", user, permRequest, + tableName, family.getKey(), null); + } + } + + // all family checks passed + return AuthResult.allow(request, "All family checks passed", user, permRequest, + tableName); + } + + // 4. no families to check and table level access failed + return AuthResult.deny(request, "No families to check and table permission failed", + user, permRequest, tableName); + } + + private void logResult(AuthResult result) { + if (AUDITLOG.isTraceEnabled()) { + InetAddress remoteAddr = null; + RequestContext ctx = RequestContext.get(); + if (ctx != null) { + remoteAddr = ctx.getRemoteAddress(); + } + AUDITLOG.trace("Access " + (result.isAllowed() ? "allowed" : "denied") + + " for user " + (result.getUser() != null ? result.getUser().getShortName() : "UNKNOWN") + + "; reason: " + result.getReason() + + "; remote address: " + (remoteAddr != null ? remoteAddr : "") + + "; request: " + result.getRequest() + + "; context: " + result.toContextString()); + } + } + + /** + * Returns the active user to which authorization checks should be applied. + * If we are in the context of an RPC call, the remote user is used, + * otherwise the currently logged in user is used. + */ + private User getActiveUser() throws IOException { + User user = RequestContext.getRequestUser(); + if (!RequestContext.isInRequestContext()) { + // for non-rpc handling, fallback to system user + user = User.getCurrent(); + } + + return user; + } + + /** + * Authorizes that the current user has any of the given permissions for the + * given table, column family and column qualifier. + * @param tableName Table requested + * @param family Column family requested + * @param qualifier Column qualifier requested + * @throws IOException if obtaining the current user fails + * @throws AccessDeniedException if user has no authorization + */ + private void requirePermission(String request, byte[] tableName, byte[] family, byte[] qualifier, + Action... permissions) throws IOException { + User user = getActiveUser(); + AuthResult result = null; + + for (Action permission : permissions) { + if (authManager.authorize(user, tableName, family, qualifier, permission)) { + result = AuthResult.allow(request, "Table permission granted", user, + permission, tableName, family, qualifier); + break; + } else { + // rest of the world + result = AuthResult.deny(request, "Insufficient permissions", user, + permission, tableName, family, qualifier); + } + } + logResult(result); + if (!result.isAllowed()) { + throw new AccessDeniedException("Insufficient permissions " + result.toContextString()); + } + } + + /** + * Authorizes that the current user has global privileges for the given action. + * @param perm The action being requested + * @throws IOException if obtaining the current user fails + * @throws AccessDeniedException if authorization is denied + */ + private void requirePermission(String request, Permission.Action perm) throws IOException { + User user = getActiveUser(); + if (authManager.authorize(user, perm)) { + logResult(AuthResult.allow(request, "Global check allowed", user, perm, null)); + } else { + logResult(AuthResult.deny(request, "Global check failed", user, perm, null)); + throw new AccessDeniedException("Insufficient permissions for user '" + + (user != null ? user.getShortName() : "null") +"' (global, action=" + + perm.toString() + ")"); + } + } + + /** + * Authorizes that the current user has permission to perform the given + * action on the set of table column families. + * @param perm Action that is required + * @param env The current coprocessor environment + * @param families The set of column families present/required in the request + * @throws AccessDeniedException if the authorization check failed + */ + private void requirePermission(String request, Permission.Action perm, + RegionCoprocessorEnvironment env, Collection families) + throws IOException { + // create a map of family-qualifier + HashMap> familyMap = new HashMap>(); + for (byte[] family : families) { + familyMap.put(family, null); + } + requirePermission(request, perm, env, familyMap); + } + + /** + * Authorizes that the current user has permission to perform the given + * action on the set of table column families. + * @param perm Action that is required + * @param env The current coprocessor environment + * @param families The map of column families-qualifiers. + * @throws AccessDeniedException if the authorization check failed + */ + public void requirePermission(String request, Permission.Action perm, + RegionCoprocessorEnvironment env, + Map> families) + throws IOException { + User user = getActiveUser(); + AuthResult result = permissionGranted(request, user, perm, env, families); + logResult(result); + + if (!result.isAllowed()) { + StringBuffer sb = new StringBuffer(""); + if ((families != null && families.size() > 0)) { + for (byte[] familyName : families.keySet()) { + if (sb.length() != 0) { + sb.append(", "); + } + sb.append(Bytes.toString(familyName)); + } + } + throw new AccessDeniedException("Insufficient permissions (table=" + + env.getRegion().getTableDesc().getNameAsString()+ + ((families != null && families.size() > 0) ? ", family: " + + sb.toString() : "") + ", action=" + + perm.toString() + ")"); + } + } + + /** + * Returns true if the current user is allowed the given action + * over at least one of the column qualifiers in the given column families. + */ + private boolean hasFamilyQualifierPermission(User user, + TablePermission.Action perm, + RegionCoprocessorEnvironment env, + Map> familyMap) + throws IOException { + HRegionInfo hri = env.getRegion().getRegionInfo(); + byte[] tableName = hri.getTableName(); + + if (user == null) { + return false; + } + + if (familyMap != null && familyMap.size() > 0) { + // at least one family must be allowed + for (Map.Entry> family : + familyMap.entrySet()) { + if (family.getValue() != null && !family.getValue().isEmpty()) { + for (byte[] qualifier : family.getValue()) { + if (authManager.matchPermission(user, tableName, + family.getKey(), qualifier, perm)) { + return true; + } + } + } else { + if (authManager.matchPermission(user, tableName, family.getKey(), + perm)) { + return true; + } + } + } + } else if (LOG.isDebugEnabled()) { + LOG.debug("Empty family map passed for permission check"); + } + + return false; + } + + /* ---- MasterObserver implementation ---- */ + public void start(CoprocessorEnvironment env) throws IOException { + + ZooKeeperWatcher zk = null; + if (env instanceof MasterCoprocessorEnvironment) { + // if running on HMaster + MasterCoprocessorEnvironment mEnv = (MasterCoprocessorEnvironment) env; + zk = mEnv.getMasterServices().getZooKeeper(); + } else if (env instanceof RegionServerCoprocessorEnvironment) { + RegionServerCoprocessorEnvironment rsEnv = (RegionServerCoprocessorEnvironment) env; + zk = rsEnv.getRegionServerServices().getZooKeeper(); + } else if (env instanceof RegionCoprocessorEnvironment) { + // if running at region + regionEnv = (RegionCoprocessorEnvironment) env; + zk = regionEnv.getRegionServerServices().getZooKeeper(); + } + + // If zk is null or IOException while obtaining auth manager, + // throw RuntimeException so that the coprocessor is unloaded. + if (zk != null) { + try { + this.authManager = TableAuthManager.get(zk, env.getConfiguration()); + } catch (IOException ioe) { + throw new RuntimeException("Error obtaining TableAuthManager", ioe); + } + } else { + throw new RuntimeException("Error obtaining TableAuthManager, zk found null."); + } + } + + public void stop(CoprocessorEnvironment env) { + + } + + @Override + public void preCreateTable(ObserverContext c, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + requirePermission("createTable", Permission.Action.CREATE); + } + + @Override + public void postCreateTable(ObserverContext c, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + if (!AccessControlLists.isAclTable(desc)) { + String owner = desc.getOwnerString(); + // default the table owner to current user, if not specified. + if (owner == null) owner = getActiveUser().getShortName(); + UserPermission userperm = new UserPermission(Bytes.toBytes(owner), desc.getName(), null, + Action.values()); + AccessControlLists.addUserPermission(c.getEnvironment().getConfiguration(), userperm); + } + } + + @Override + public void preDeleteTable(ObserverContext c, byte[] tableName) + throws IOException { + requirePermission("deleteTable", tableName, null, null, Action.ADMIN, Action.CREATE); + } + + @Override + public void postDeleteTable(ObserverContext c, + byte[] tableName) throws IOException { + AccessControlLists.removeTablePermissions(c.getEnvironment().getConfiguration(), tableName); + } + + @Override + public void preModifyTable(ObserverContext c, byte[] tableName, + HTableDescriptor htd) throws IOException { + requirePermission("modifyTable", tableName, null, null, Action.ADMIN, Action.CREATE); + } + + @Override + public void postModifyTable(ObserverContext c, + byte[] tableName, HTableDescriptor htd) throws IOException { + String owner = htd.getOwnerString(); + // default the table owner to current user, if not specified. + if (owner == null) owner = getActiveUser().getShortName(); + UserPermission userperm = new UserPermission(Bytes.toBytes(owner), htd.getName(), null, + Action.values()); + AccessControlLists.addUserPermission(c.getEnvironment().getConfiguration(), userperm); + } + + @Override + public void preAddColumn(ObserverContext c, byte[] tableName, + HColumnDescriptor column) throws IOException { + requirePermission("addColumn", tableName, null, null, Action.ADMIN, Action.CREATE); + } + + @Override + public void postAddColumn(ObserverContext c, + byte[] tableName, HColumnDescriptor column) throws IOException {} + + @Override + public void preModifyColumn(ObserverContext c, byte[] tableName, + HColumnDescriptor descriptor) throws IOException { + requirePermission("modifyColumn", tableName, null, null, Action.ADMIN, Action.CREATE); + } + + @Override + public void postModifyColumn(ObserverContext c, + byte[] tableName, HColumnDescriptor descriptor) throws IOException {} + + @Override + public void preDeleteColumn(ObserverContext c, byte[] tableName, + byte[] col) throws IOException { + requirePermission("deleteColumn", tableName, null, null, Action.ADMIN, Action.CREATE); + } + + @Override + public void postDeleteColumn(ObserverContext c, + byte[] tableName, byte[] col) throws IOException { + AccessControlLists.removeTablePermissions(c.getEnvironment().getConfiguration(), + tableName, col); + } + + @Override + public void preEnableTable(ObserverContext c, byte[] tableName) + throws IOException { + requirePermission("enableTable", tableName, null, null, Action.ADMIN, Action.CREATE); + } + + @Override + public void postEnableTable(ObserverContext c, + byte[] tableName) throws IOException {} + + @Override + public void preDisableTable(ObserverContext c, byte[] tableName) + throws IOException { + if (Bytes.equals(tableName, AccessControlLists.ACL_GLOBAL_NAME)) { + throw new AccessDeniedException("Not allowed to disable " + + AccessControlLists.ACL_TABLE_NAME_STR + " table."); + } + requirePermission("disableTable", tableName, null, null, Action.ADMIN, Action.CREATE); + } + + @Override + public void postDisableTable(ObserverContext c, + byte[] tableName) throws IOException {} + + @Override + public void preMove(ObserverContext c, HRegionInfo region, + ServerName srcServer, ServerName destServer) throws IOException { + requirePermission("move", region.getTableName(), null, null, Action.ADMIN); + } + + @Override + public void postMove(ObserverContext c, + HRegionInfo region, ServerName srcServer, ServerName destServer) + throws IOException {} + + @Override + public void preAssign(ObserverContext c, HRegionInfo regionInfo) + throws IOException { + requirePermission("assign", regionInfo.getTableName(), null, null, Action.ADMIN); + } + + @Override + public void postAssign(ObserverContext c, + HRegionInfo regionInfo) throws IOException {} + + @Override + public void preUnassign(ObserverContext c, HRegionInfo regionInfo, + boolean force) throws IOException { + requirePermission("unassign", regionInfo.getTableName(), null, null, Action.ADMIN); + } + + @Override + public void postUnassign(ObserverContext c, + HRegionInfo regionInfo, boolean force) throws IOException {} + + @Override + public void preBalance(ObserverContext c) + throws IOException { + requirePermission("balance", Permission.Action.ADMIN); + } + @Override + public void postBalance(ObserverContext c) + throws IOException {} + + @Override + public boolean preBalanceSwitch(ObserverContext c, + boolean newValue) throws IOException { + requirePermission("balanceSwitch", Permission.Action.ADMIN); + return newValue; + } + @Override + public void postBalanceSwitch(ObserverContext c, + boolean oldValue, boolean newValue) throws IOException {} + + @Override + public void preShutdown(ObserverContext c) + throws IOException { + requirePermission("shutdown", Permission.Action.ADMIN); + } + + @Override + public void preStopMaster(ObserverContext c) + throws IOException { + requirePermission("stopMaster", Permission.Action.ADMIN); + } + + @Override + public void postStartMaster(ObserverContext ctx) + throws IOException { + // initialize the ACL storage table + AccessControlLists.init(ctx.getEnvironment().getMasterServices()); + } + + @Override + public void preSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + requirePermission("snapshot", Permission.Action.ADMIN); + } + + @Override + public void postSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + } + + @Override + public void preCloneSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + requirePermission("cloneSnapshot", Permission.Action.ADMIN); + } + + @Override + public void postCloneSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + } + + @Override + public void preRestoreSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + requirePermission("restoreSnapshot", Permission.Action.ADMIN); + } + + @Override + public void postRestoreSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + } + + @Override + public void preDeleteSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot) throws IOException { + requirePermission("deleteSnapshot", Permission.Action.ADMIN); + } + + @Override + public void postDeleteSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot) throws IOException { + } + + /* ---- RegionObserver implementation ---- */ + + @Override + public void preOpen(ObserverContext e) throws IOException { + RegionCoprocessorEnvironment env = e.getEnvironment(); + final HRegion region = env.getRegion(); + if (region == null) { + LOG.error("NULL region from RegionCoprocessorEnvironment in preOpen()"); + return; + } else { + HRegionInfo regionInfo = region.getRegionInfo(); + if (isSpecialTable(regionInfo)) { + isSystemOrSuperUser(regionEnv.getConfiguration()); + } else { + requirePermission("open", Action.ADMIN); + } + } + } + + @Override + public void postOpen(ObserverContext c) { + RegionCoprocessorEnvironment env = c.getEnvironment(); + final HRegion region = env.getRegion(); + if (region == null) { + LOG.error("NULL region from RegionCoprocessorEnvironment in postOpen()"); + return; + } + if (AccessControlLists.isAclRegion(region)) { + aclRegion = true; + try { + initialize(env); + } catch (IOException ex) { + // if we can't obtain permissions, it's better to fail + // than perform checks incorrectly + throw new RuntimeException("Failed to initialize permissions cache", ex); + } + } + } + + @Override + public void preFlush(ObserverContext e) throws IOException { + requirePermission("flush", getTableName(e.getEnvironment()), null, null, Action.ADMIN); + } + + @Override + public void preSplit(ObserverContext e) throws IOException { + requirePermission("split", getTableName(e.getEnvironment()), null, null, Action.ADMIN); + } + + @Override + public InternalScanner preCompact(ObserverContext e, + final Store store, final InternalScanner scanner) throws IOException { + requirePermission("compact", getTableName(e.getEnvironment()), null, null, Action.ADMIN); + return scanner; + } + + @Override + public void preCompactSelection(final ObserverContext e, + final Store store, final List candidates) throws IOException { + requirePermission("compactSelection", getTableName(e.getEnvironment()), null, null, Action.ADMIN); + } + + @Override + public void preGetClosestRowBefore(final ObserverContext c, + final byte [] row, final byte [] family, final Result result) + throws IOException { + requirePermission("getClosestRowBefore", TablePermission.Action.READ, c.getEnvironment(), + (family != null ? Lists.newArrayList(family) : null)); + } + + @Override + public void preGet(final ObserverContext c, + final Get get, final List result) throws IOException { + /* + if column family level checks fail, check for a qualifier level permission + in one of the families. If it is present, then continue with the AccessControlFilter. + */ + RegionCoprocessorEnvironment e = c.getEnvironment(); + User requestUser = getActiveUser(); + AuthResult authResult = permissionGranted("get", requestUser, + TablePermission.Action.READ, e, get.getFamilyMap()); + if (!authResult.isAllowed()) { + if (hasFamilyQualifierPermission(requestUser, + TablePermission.Action.READ, e, get.getFamilyMap())) { + byte[] table = getTableName(e); + AccessControlFilter filter = new AccessControlFilter(authManager, + requestUser, table); + + // wrap any existing filter + if (get.getFilter() != null) { + FilterList wrapper = new FilterList(FilterList.Operator.MUST_PASS_ALL, + Lists.newArrayList(filter, get.getFilter())); + get.setFilter(wrapper); + } else { + get.setFilter(filter); + } + logResult(AuthResult.allow("get", "Access allowed with filter", requestUser, + TablePermission.Action.READ, authResult.table)); + } else { + logResult(authResult); + throw new AccessDeniedException("Insufficient permissions (table=" + + e.getRegion().getTableDesc().getNameAsString() + ", action=READ)"); + } + } else { + // log auth success + logResult(authResult); + } + } + + @Override + public boolean preExists(final ObserverContext c, + final Get get, final boolean exists) throws IOException { + requirePermission("exists", TablePermission.Action.READ, c.getEnvironment(), + get.familySet()); + return exists; + } + + @Override + public void prePut(final ObserverContext c, + final Put put, final WALEdit edit, final boolean writeToWAL) + throws IOException { + HRegion region = c.getEnvironment().getRegion(); + // TODO later when HBase supports making some tables as system table we can convert the index + // table as a system table. Then the core code itself will handle not contacting ACL for + // our index table. + boolean isIndexTable = region.getTableDesc().getNameAsString().endsWith("_idx"); + if (!isIndexTable) { + // When it is index table no need to do ACL checks + requirePermission("put", TablePermission.Action.WRITE, c.getEnvironment(), put.getFamilyMap()); + } + } + + @Override + public void postPut(final ObserverContext c, + final Put put, final WALEdit edit, final boolean writeToWAL) { + if (aclRegion) { + updateACL(c.getEnvironment(), put.getFamilyMap()); + } + } + + @Override + public void preDelete(final ObserverContext c, + final Delete delete, final WALEdit edit, final boolean writeToWAL) + throws IOException { + HRegion region = c.getEnvironment().getRegion(); + // TODO later when HBase supports making some tables as system table we can convert the index + // table as a system table. Then the core code itself will handle not contacting ACL for + // our index table. + boolean isIndexTable = region.getTableDesc().getNameAsString().endsWith("_idx"); + if (!isIndexTable) { + // When it is index table no need to do ACL checks + requirePermission("delete", TablePermission.Action.WRITE, c.getEnvironment(), + delete.getFamilyMap()); + } + } + + @Override + public void postDelete(final ObserverContext c, + final Delete delete, final WALEdit edit, final boolean writeToWAL) + throws IOException { + if (aclRegion) { + updateACL(c.getEnvironment(), delete.getFamilyMap()); + } + } + + @Override + public boolean preCheckAndPut(final ObserverContext c, + final byte [] row, final byte [] family, final byte [] qualifier, + final CompareFilter.CompareOp compareOp, + final WritableByteArrayComparable comparator, final Put put, + final boolean result) throws IOException { + Collection familyMap = Arrays.asList(new byte[][]{family}); + requirePermission("checkAndPut", TablePermission.Action.READ, c.getEnvironment(), familyMap); + requirePermission("checkAndPut", TablePermission.Action.WRITE, c.getEnvironment(), familyMap); + return result; + } + + @Override + public boolean preCheckAndDelete(final ObserverContext c, + final byte [] row, final byte [] family, final byte [] qualifier, + final CompareFilter.CompareOp compareOp, + final WritableByteArrayComparable comparator, final Delete delete, + final boolean result) throws IOException { + Collection familyMap = Arrays.asList(new byte[][]{family}); + requirePermission("checkAndDelete", TablePermission.Action.READ, c.getEnvironment(), familyMap); + requirePermission("checkAndDelete", TablePermission.Action.WRITE, c.getEnvironment(), familyMap); + return result; + } + + @Override + public long preIncrementColumnValue(final ObserverContext c, + final byte [] row, final byte [] family, final byte [] qualifier, + final long amount, final boolean writeToWAL) + throws IOException { + requirePermission("incrementColumnValue", TablePermission.Action.WRITE, c.getEnvironment(), + Arrays.asList(new byte[][]{family})); + return -1; + } + + @Override + public Result preAppend(ObserverContext c, Append append) + throws IOException { + requirePermission("append", TablePermission.Action.WRITE, c.getEnvironment(), append.getFamilyMap()); + return null; + } + + @Override + public Result preIncrement(final ObserverContext c, + final Increment increment) + throws IOException { + requirePermission("increment", TablePermission.Action.WRITE, c.getEnvironment(), + increment.getFamilyMap().keySet()); + return null; + } + + @Override + public RegionScanner preScannerOpen(final ObserverContext c, + final Scan scan, final RegionScanner s) throws IOException { + /* + if column family level checks fail, check for a qualifier level permission + in one of the families. If it is present, then continue with the AccessControlFilter. + */ + RegionCoprocessorEnvironment e = c.getEnvironment(); + User user = getActiveUser(); + AuthResult authResult = permissionGranted("scannerOpen", user, TablePermission.Action.READ, e, + scan.getFamilyMap()); + if (!authResult.isAllowed()) { + if (hasFamilyQualifierPermission(user, TablePermission.Action.READ, e, + scan.getFamilyMap())) { + byte[] table = getTableName(e); + AccessControlFilter filter = new AccessControlFilter(authManager, + user, table); + + // wrap any existing filter + if (scan.hasFilter()) { + FilterList wrapper = new FilterList(FilterList.Operator.MUST_PASS_ALL, + Lists.newArrayList(filter, scan.getFilter())); + scan.setFilter(wrapper); + } else { + scan.setFilter(filter); + } + logResult(AuthResult.allow("scannerOpen", "Access allowed with filter", user, + TablePermission.Action.READ, authResult.table)); + } else { + // no table/family level perms and no qualifier level perms, reject + logResult(authResult); + throw new AccessDeniedException("Insufficient permissions for user '"+ + (user != null ? user.getShortName() : "null")+"' "+ + "for scanner open on table " + Bytes.toString(getTableName(e))); + } + } else { + // log success + logResult(authResult); + } + return s; + } + + @Override + public RegionScanner postScannerOpen(final ObserverContext c, + final Scan scan, final RegionScanner s) throws IOException { + User user = getActiveUser(); + if (user != null && user.getShortName() != null) { // store reference to scanner owner for later checks + scannerOwners.put(s, user.getShortName()); + } + return s; + } + + @Override + public boolean preScannerNext(final ObserverContext c, + final InternalScanner s, final List result, + final int limit, final boolean hasNext) throws IOException { + requireScannerOwner(s); + return hasNext; + } + + @Override + public void preScannerClose(final ObserverContext c, + final InternalScanner s) throws IOException { + requireScannerOwner(s); + } + + @Override + public void postScannerClose(final ObserverContext c, + final InternalScanner s) throws IOException { + // clean up any associated owner mapping + scannerOwners.remove(s); + } + + /** + * Verify, when servicing an RPC, that the caller is the scanner owner. + * If so, we assume that access control is correctly enforced based on + * the checks performed in preScannerOpen() + */ + private void requireScannerOwner(InternalScanner s) + throws AccessDeniedException { + if (RequestContext.isInRequestContext()) { + String requestUserName = RequestContext.getRequestUserName(); + String owner = scannerOwners.get(s); + if (owner != null && !owner.equals(requestUserName)) { + throw new AccessDeniedException("User '"+ requestUserName +"' is not the scanner owner!"); + } + } + } + + /** + * Verifies user has WRITE privileges on + * the Column Families involved in the bulkLoadHFile + * request. Specific Column Write privileges are presently + * ignored. + */ + @Override + public void preBulkLoadHFile(ObserverContext ctx, + List> familyPaths) throws IOException { + List cfs = new LinkedList(); + for(Pair el : familyPaths) { + cfs.add(el.getFirst()); + } + requirePermission("bulkLoadHFile", Permission.Action.WRITE, ctx.getEnvironment(), cfs); + } + + private AuthResult hasSomeAccess(RegionCoprocessorEnvironment e, String request, Action action) throws IOException { + User requestUser = getActiveUser(); + byte[] tableName = e.getRegion().getTableDesc().getName(); + AuthResult authResult = permissionGranted(request, requestUser, + action, e, Collections.EMPTY_MAP); + if (!authResult.isAllowed()) { + for(UserPermission userPerm: + AccessControlLists.getUserPermissions(regionEnv.getConfiguration(), tableName)) { + for(Permission.Action userAction: userPerm.getActions()) { + if(userAction.equals(action)) { + return AuthResult.allow(request, "Access allowed", requestUser, + action, tableName); + } + } + } + } + return authResult; + } + + /** + * Authorization check for + * SecureBulkLoadProtocol.prepareBulkLoad() + * @param e + * @throws IOException + */ + public void prePrepareBulkLoad(RegionCoprocessorEnvironment e) throws IOException { + AuthResult authResult = hasSomeAccess(e, "prepareBulkLoad", Action.WRITE); + logResult(authResult); + if (!authResult.isAllowed()) { + throw new AccessDeniedException("Insufficient permissions (table=" + + e.getRegion().getTableDesc().getNameAsString() + ", action=WRITE)"); + } + } + + /** + * Authorization security check for + * SecureBulkLoadProtocol.cleanupBulkLoad() + * @param e + * @throws IOException + */ + //TODO this should end up as a coprocessor hook + public void preCleanupBulkLoad(RegionCoprocessorEnvironment e) throws IOException { + AuthResult authResult = hasSomeAccess(e, "cleanupBulkLoad", Action.WRITE); + logResult(authResult); + if (!authResult.isAllowed()) { + throw new AccessDeniedException("Insufficient permissions (table=" + + e.getRegion().getTableDesc().getNameAsString() + ", action=WRITE)"); + } + } + + /* ---- AccessControllerProtocol implementation ---- */ + /* + * These methods are only allowed to be called against the _acl_ region(s). + * This will be restricted by both client side and endpoint implementations. + */ + @Override + public void grant(UserPermission perm) throws IOException { + // verify it's only running at .acl. + if (aclRegion) { + if (LOG.isDebugEnabled()) { + LOG.debug("Received request to grant access permission " + perm.toString()); + } + + requirePermission("grant", perm.getTable(), perm.getFamily(), perm.getQualifier(), Action.ADMIN); + + AccessControlLists.addUserPermission(regionEnv.getConfiguration(), perm); + if (AUDITLOG.isTraceEnabled()) { + // audit log should store permission changes in addition to auth results + AUDITLOG.trace("Granted permission " + perm.toString()); + } + } else { + throw new CoprocessorException(AccessController.class, "This method " + + "can only execute at " + Bytes.toString(AccessControlLists.ACL_TABLE_NAME) + " table."); + } + } + + @Override + @Deprecated + public void grant(byte[] user, TablePermission permission) + throws IOException { + grant(new UserPermission(user, permission.getTable(), + permission.getFamily(), permission.getQualifier(), + permission.getActions())); + } + + @Override + public void revoke(UserPermission perm) throws IOException { + // only allowed to be called on _acl_ region + if (aclRegion) { + if (LOG.isDebugEnabled()) { + LOG.debug("Received request to revoke access permission " + perm.toString()); + } + + requirePermission("revoke", perm.getTable(), perm.getFamily(), + perm.getQualifier(), Action.ADMIN); + + AccessControlLists.removeUserPermission(regionEnv.getConfiguration(), perm); + if (AUDITLOG.isTraceEnabled()) { + // audit log should record all permission changes + AUDITLOG.trace("Revoked permission " + perm.toString()); + } + } else { + throw new CoprocessorException(AccessController.class, "This method " + + "can only execute at " + Bytes.toString(AccessControlLists.ACL_TABLE_NAME) + " table."); + } + } + + @Override + @Deprecated + public void revoke(byte[] user, TablePermission permission) + throws IOException { + revoke(new UserPermission(user, permission.getTable(), + permission.getFamily(), permission.getQualifier(), + permission.getActions())); + } + + @Override + public List getUserPermissions(final byte[] tableName) throws IOException { + // only allowed to be called on _acl_ region + if (aclRegion) { + requirePermission("userPermissions", tableName, null, null, Action.ADMIN); + + List perms = AccessControlLists.getUserPermissions( + regionEnv.getConfiguration(), tableName); + return perms; + } else { + throw new CoprocessorException(AccessController.class, "This method " + + "can only execute at " + Bytes.toString(AccessControlLists.ACL_TABLE_NAME) + " table."); + } + } + + @Override + public void checkPermissions(Permission[] permissions) throws IOException { + byte[] tableName = regionEnv.getRegion().getTableDesc().getName(); + for (Permission permission : permissions) { + if (permission instanceof TablePermission) { + TablePermission tperm = (TablePermission) permission; + for (Permission.Action action : permission.getActions()) { + if (!Arrays.equals(tperm.getTable(), tableName)) { + throw new CoprocessorException(AccessController.class, String.format("This method " + + "can only execute at the table specified in TablePermission. " + + "Table of the region:%s , requested table:%s", Bytes.toString(tableName), + Bytes.toString(tperm.getTable()))); + } + + HashMap> familyMap = Maps.newHashMapWithExpectedSize(1); + if (tperm.getFamily() != null) { + if (tperm.getQualifier() != null) { + familyMap.put(tperm.getFamily(), Sets.newHashSet(tperm.getQualifier())); + } else { + familyMap.put(tperm.getFamily(), null); + } + } + + requirePermission("checkPermissions", action, regionEnv, familyMap); + } + + } else { + for (Permission.Action action : permission.getActions()) { + requirePermission("checkPermissions", action); + } + } + } + } + + @Override + public long getProtocolVersion(String protocol, long clientVersion) throws IOException { + return PROTOCOL_VERSION; + } + + @Override + public ProtocolSignature getProtocolSignature(String protocol, + long clientVersion, int clientMethodsHash) throws IOException { + if (AccessControllerProtocol.class.getName().equals(protocol)) { + return new ProtocolSignature(PROTOCOL_VERSION, null); + } + throw new HBaseRPC.UnknownProtocolException( + "Unexpected protocol requested: "+protocol); + } + + private byte[] getTableName(RegionCoprocessorEnvironment e) { + HRegion region = e.getRegion(); + byte[] tableName = null; + + if (region != null) { + HRegionInfo regionInfo = region.getRegionInfo(); + if (regionInfo != null) { + tableName = regionInfo.getTableName(); + } + } + return tableName; + } + + + @Override + public void preClose(ObserverContext e, boolean abortRequested) + throws IOException { + requirePermission("close", Permission.Action.ADMIN); + } + + @Override + public void preLockRow(ObserverContext ctx, byte[] regionName, + byte[] row) throws IOException { + requirePermission("lockRow", getTableName(ctx.getEnvironment()), null, null, + Permission.Action.WRITE, Permission.Action.CREATE); + } + + @Override + public void preUnlockRow(ObserverContext ctx, byte[] regionName, + long lockId) throws IOException { + requirePermission("unlockRow", getTableName(ctx.getEnvironment()), null, null, + Permission.Action.WRITE, Permission.Action.CREATE); + } + + private void isSystemOrSuperUser(Configuration conf) throws IOException { + User user = User.getCurrent(); + if (user == null) { + throw new IOException("Unable to obtain the current user, " + + "authorization checks for internal operations will not work correctly!"); + } + + String currentUser = user.getShortName(); + List superusers = Lists.asList(currentUser, + conf.getStrings(AccessControlLists.SUPERUSER_CONF_KEY, new String[0])); + + User activeUser = getActiveUser(); + if (!(superusers.contains(activeUser.getShortName()))) { + throw new AccessDeniedException("User '" + (user != null ? user.getShortName() : "null") + + "is not system or super user."); + } + } + + private boolean isSpecialTable(HRegionInfo regionInfo) { + byte[] tableName = regionInfo.getTableName(); + return tableName.equals(AccessControlLists.ACL_TABLE_NAME) + || tableName.equals(Bytes.toBytes("-ROOT-")) + || tableName.equals(Bytes.toBytes(".META.")); + } + + @Override + public void preStopRegionServer(ObserverContext env) + throws IOException { + requirePermission("stop", Permission.Action.ADMIN); + } + + @Override + public void preCreateTableHandler(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + } + + @Override + public void postCreateTableHandler(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + } + + @Override + public void preDeleteTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void postDeleteTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void preModifyTableHandler(ObserverContext ctx, + byte[] tableName, HTableDescriptor htd) throws IOException { + } + + @Override + public void postModifyTableHandler(ObserverContext ctx, + byte[] tableName, HTableDescriptor htd) throws IOException { + } + + @Override + public void preAddColumnHandler(ObserverContext ctx, + byte[] tableName, HColumnDescriptor column) throws IOException { + } + + @Override + public void postAddColumnHandler(ObserverContext ctx, + byte[] tableName, HColumnDescriptor column) throws IOException { + } + + @Override + public void preModifyColumnHandler(ObserverContext ctx, + byte[] tableName, HColumnDescriptor descriptor) throws IOException { + } + + @Override + public void postModifyColumnHandler(ObserverContext ctx, + byte[] tableName, HColumnDescriptor descriptor) throws IOException { + } + + @Override + public void preDeleteColumnHandler(ObserverContext ctx, + byte[] tableName, byte[] c) throws IOException { + } + + @Override + public void postDeleteColumnHandler(ObserverContext ctx, + byte[] tableName, byte[] c) throws IOException { + } + + @Override + public void preEnableTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void postEnableTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void preDisableTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void postDisableTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControllerProtocol.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControllerProtocol.java new file mode 100644 index 0000000..2ecb60a --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/AccessControllerProtocol.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; + +/** + * A custom protocol defined for maintaining and querying access control lists. + */ +public interface AccessControllerProtocol extends CoprocessorProtocol { + + public static final long VERSION = 1L; + + /** + * Grants the given user or group the privilege to perform the given actions + * @param userPermission the details of the provided user permissions + * @throws IOException if the grant could not be applied + */ + public void grant(UserPermission userPermission) + throws IOException; + + /** + * Grants the given user or group the privilege to perform the given actions + * over the specified scope contained in {@link TablePermission} + * @param user the user name, or, if prefixed with "@", group name receiving + * the grant + * @param permission the details of the provided permissions + * @throws IOException if the grant could not be applied + * @deprecated Use {@link #revoke(UserPermission userPermission)} instead + */ + @Deprecated + public void grant(byte[] user, TablePermission permission) + throws IOException; + + /** + * Revokes a previously granted privilege from a user or group. + * Note that the provided {@link TablePermission} details must exactly match + * a stored grant. For example, if user "bob" has been granted "READ" access + * to table "data", over column family and qualifer "info:colA", then the + * table, column family and column qualifier must all be specified. + * Attempting to revoke permissions over just the "data" table will have + * no effect. + * @param permission the details of the previously granted permission to revoke + * @throws IOException if the revocation could not be performed + */ + public void revoke(UserPermission userPermission) + throws IOException; + + /** + * Revokes a previously granted privilege from a user or group. + * Note that the provided {@link TablePermission} details must exactly match + * a stored grant. For example, if user "bob" has been granted "READ" access + * to table "data", over column family and qualifer "info:colA", then the + * table, column family and column qualifier must all be specified. + * Attempting to revoke permissions over just the "data" table will have + * no effect. + * @param user the user name, or, if prefixed with "@", group name whose + * privileges are being revoked + * @param permission the details of the previously granted permission to revoke + * @throws IOException if the revocation could not be performed + * @deprecated Use {@link #revoke(UserPermission userPermission)} instead + */ + @Deprecated + public void revoke(byte[] user, TablePermission permission) + throws IOException; + + /** + * Queries the permissions currently stored for the given table, returning + * a list of currently granted permissions, along with the user or group + * each is associated with. + * @param tableName the table of the permission grants to return + * @return a list of the currently granted permissions, with associated user + * or group names + * @throws IOException if there is an error querying the permissions + */ + public List getUserPermissions(byte[] tableName) + throws IOException; + + /** + * Checks whether the given Permissions will pass the access checks for the + * current user. Global permissions can be checked from the -acl- table + * or any other table, however TablePermissions can only be checked by + * the table's regions. If access control checks fail this method throws + * AccessDeniedException. + * @param permissions to check for. Permission subclasses can be used + * to do more specific checks at the table/family/column level. + * @throws IOException if there is an error checking the permissions + */ + public void checkPermissions(Permission[] permissions) + throws IOException; +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/Permission.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/Permission.java new file mode 100644 index 0000000..a7a3bc9 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/Permission.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import com.google.common.collect.Maps; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.VersionedWritable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; + +/** + * Base permissions instance representing the ability to perform a given set + * of actions. + * + * @see TablePermission + */ +public class Permission extends VersionedWritable { + protected static final byte VERSION = 0; + public enum Action { + READ('R'), WRITE('W'), EXEC('X'), CREATE('C'), ADMIN('A'); + + private byte code; + Action(char code) { + this.code = (byte)code; + } + + public byte code() { return code; } + } + + private static Log LOG = LogFactory.getLog(Permission.class); + protected static Map ACTION_BY_CODE = Maps.newHashMap(); + + protected Action[] actions; + + static { + for (Action a : Action.values()) { + ACTION_BY_CODE.put(a.code(), a); + } + } + + /** Empty constructor for Writable implementation. Do not use. */ + public Permission() { + super(); + } + + public Permission(Action... assigned) { + if (assigned != null && assigned.length > 0) { + actions = Arrays.copyOf(assigned, assigned.length); + } + } + + public Permission(byte[] actionCodes) { + if (actionCodes != null) { + Action acts[] = new Action[actionCodes.length]; + int j = 0; + for (int i=0; i 0) { + actions = new Action[length]; + for (int i = 0; i < length; i++) { + byte b = in.readByte(); + Action a = ACTION_BY_CODE.get(b); + if (a == null) { + throw new IOException("Unknown action code '"+ + Bytes.toStringBinary(new byte[]{b})+"' in input"); + } + this.actions[i] = a; + } + } else { + actions = new Action[0]; + } + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeByte(actions != null ? actions.length : 0); + if (actions != null) { + for (Action a: actions) { + out.writeByte(a.code()); + } + } + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java new file mode 100644 index 0000000..e422710 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java @@ -0,0 +1,317 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.access; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.ipc.RequestContext; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Methods; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; + +import java.io.IOException; +import java.math.BigInteger; +import java.security.PrivilegedAction; +import java.security.SecureRandom; +import java.util.List; + +/** + * Coprocessor service for bulk loads in secure mode. + * This coprocessor has to be installed as part of enabling + * security in HBase. + * + * This service addresses two issues: + * + * 1. Moving files in a secure filesystem wherein the HBase Client + * and HBase Server are different filesystem users. + * 2. Does moving in a secure manner. Assuming that the filesystem + * is POSIX compliant. + * + * The algorithm is as follows: + * + * 1. Create an hbase owned staging directory which is + * world traversable (711): /hbase/staging + * 2. A user writes out data to his secure output directory: /user/foo/data + * 3. A call is made to hbase to create a secret staging directory + * which globally rwx (777): /user/staging/averylongandrandomdirectoryname + * 4. The user makes the data world readable and writable, then moves it + * into the random staging directory, then calls bulkLoadHFiles() + * + * Like delegation tokens the strength of the security lies in the length + * and randomness of the secret directory. + * + */ +@InterfaceAudience.Private +public class SecureBulkLoadEndpoint extends BaseEndpointCoprocessor + implements SecureBulkLoadProtocol { + + public static final long VERSION = 0L; + + //Random number is 320 bits wide + private static final int RANDOM_WIDTH = 320; + //We picked 32 as the radix, so the character set + //will only contain alpha numeric values + //320/5 = 64 characters + private static final int RANDOM_RADIX = 32; + + private static Log LOG = LogFactory.getLog(SecureBulkLoadEndpoint.class); + + private final static FsPermission PERM_ALL_ACCESS = FsPermission.valueOf("-rwxrwxrwx"); + private final static FsPermission PERM_HIDDEN = FsPermission.valueOf("-rwx--x--x"); + private final static String BULKLOAD_STAGING_DIR = "hbase.bulkload.staging.dir"; + + private SecureRandom random; + private FileSystem fs; + private Configuration conf; + + //two levels so it doesn't get deleted accidentally + //no sticky bit in Hadoop 1.0 + private Path baseStagingDir; + + private RegionCoprocessorEnvironment env; + + + @Override + public void start(CoprocessorEnvironment env) { + super.start(env); + + this.env = (RegionCoprocessorEnvironment)env; + random = new SecureRandom(); + conf = env.getConfiguration(); + baseStagingDir = getBaseStagingDir(conf); + + try { + fs = FileSystem.get(conf); + fs.mkdirs(baseStagingDir, PERM_HIDDEN); + fs.setPermission(baseStagingDir, PERM_HIDDEN); + //no sticky bit in hadoop-1.0, making directory nonempty so it never gets erased + fs.mkdirs(new Path(baseStagingDir,"DONOTERASE"), PERM_HIDDEN); + FileStatus status = fs.getFileStatus(baseStagingDir); + if(status == null) { + throw new IllegalStateException("Failed to create staging directory"); + } + if(!status.getPermission().equals(PERM_HIDDEN)) { + throw new IllegalStateException("Directory already exists but permissions aren't set to '-rwx--x--x' "); + } + } catch (IOException e) { + throw new IllegalStateException("Failed to get FileSystem instance",e); + } + } + + @Override + public String prepareBulkLoad(byte[] tableName) throws IOException { + getAccessController().prePrepareBulkLoad(env); + return createStagingDir(baseStagingDir, getActiveUser(), tableName).toString(); + } + + @Override + public void cleanupBulkLoad(String bulkToken) throws IOException { + getAccessController().preCleanupBulkLoad(env); + fs.delete(createStagingDir(baseStagingDir, + getActiveUser(), + env.getRegion().getTableDesc().getName(), + new Path(bulkToken).getName()), + true); + } + + @Override + public boolean bulkLoadHFiles(final List> familyPaths, + final Token userToken, final String bulkToken) throws IOException { + User user = getActiveUser(); + final UserGroupInformation ugi = user.getUGI(); + if(userToken != null) { + ugi.addToken(userToken); + } else if(User.isSecurityEnabled()) { + //we allow this to pass through in "simple" security mode + //for mini cluster testing + throw new DoNotRetryIOException("User token cannot be null"); + } + + HRegion region = env.getRegion(); + boolean bypass = false; + if (region.getCoprocessorHost() != null) { + bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths); + } + boolean loaded = false; + if (!bypass) { + loaded = ugi.doAs(new PrivilegedAction() { + @Override + public Boolean run() { + FileSystem fs = null; + try { + Configuration conf = env.getConfiguration(); + fs = FileSystem.get(conf); + for(Pair el: familyPaths) { + Path p = new Path(el.getSecond()); + LOG.debug("Setting permission for: " + p); + fs.setPermission(p, PERM_ALL_ACCESS); + Path stageFamily = new Path(bulkToken, Bytes.toString(el.getFirst())); + if(!fs.exists(stageFamily)) { + fs.mkdirs(stageFamily); + fs.setPermission(stageFamily, PERM_ALL_ACCESS); + } + } + //We call bulkLoadHFiles as requesting user + //To enable access prior to staging + return env.getRegion().bulkLoadHFiles(familyPaths, + new SecureBulkLoadListener(fs, bulkToken)); + } catch (Exception e) { + LOG.error("Failed to complete bulk load", e); + } + return false; + } + }); + } + if (region.getCoprocessorHost() != null) { + loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded); + } + return loaded; + } + + @Override + public long getProtocolVersion(String protocol, long clientVersion) + throws IOException { + if (SecureBulkLoadProtocol.class.getName().equals(protocol)) { + return SecureBulkLoadEndpoint.VERSION; + } + LOG.warn("Unknown protocol requested: " + protocol); + return -1; + } + + private AccessController getAccessController() { + return (AccessController) this.env.getRegion() + .getCoprocessorHost().findCoprocessor(AccessController.class.getName()); + } + + private Path createStagingDir(Path baseDir, User user, byte[] tableName) throws IOException { + String randomDir = user.getShortName()+"__"+Bytes.toString(tableName)+"__"+ + (new BigInteger(RANDOM_WIDTH, random).toString(RANDOM_RADIX)); + return createStagingDir(baseDir, user, tableName, randomDir); + } + + private Path createStagingDir(Path baseDir, + User user, + byte[] tableName, + String randomDir) throws IOException { + Path p = new Path(baseDir, randomDir); + fs.mkdirs(p, PERM_ALL_ACCESS); + fs.setPermission(p, PERM_ALL_ACCESS); + return p; + } + + private User getActiveUser() throws IOException { + User user = RequestContext.getRequestUser(); + if (!RequestContext.isInRequestContext()) { + throw new DoNotRetryIOException("Failed to get requesting user"); + } + + //this is for testing + if("simple".equalsIgnoreCase(conf.get(User.HBASE_SECURITY_CONF_KEY))) { + return User.createUserForTesting(conf, user.getShortName(), new String[]{}); + } + + return user; + } + + /** + * This returns the staging path for a given column family. + * This is needed for clean recovery and called reflectively in LoadIncrementalHFiles + */ + public static Path getStagingPath(Configuration conf, String bulkToken, byte[] family) { + Path stageP = new Path(getBaseStagingDir(conf), bulkToken); + return new Path(stageP, Bytes.toString(family)); + } + + private static Path getBaseStagingDir(Configuration conf) { + return new Path(conf.get(BULKLOAD_STAGING_DIR, "/tmp/hbase-staging")); + } + + private static class SecureBulkLoadListener implements HRegion.BulkLoadListener { + private FileSystem fs; + private String stagingDir; + + public SecureBulkLoadListener(FileSystem fs, String stagingDir) { + this.fs = fs; + this.stagingDir = stagingDir; + } + + @Override + public String prepareBulkLoad(final byte[] family, final String srcPath) throws IOException { + Path p = new Path(srcPath); + Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName())); + + if(!isFile(p)) { + throw new IOException("Path does not reference a file: " + p); + } + + LOG.debug("Moving " + p + " to " + stageP); + if(!fs.rename(p, stageP)) { + throw new IOException("Failed to move HFile: " + p + " to " + stageP); + } + return stageP.toString(); + } + + @Override + public void doneBulkLoad(byte[] family, String srcPath) throws IOException { + LOG.debug("Bulk Load done for: " + srcPath); + } + + @Override + public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException { + Path p = new Path(srcPath); + Path stageP = new Path(stagingDir, + new Path(Bytes.toString(family), p.getName())); + LOG.debug("Moving " + stageP + " back to " + p); + if(!fs.rename(stageP, p)) + throw new IOException("Failed to move HFile: " + stageP + " to " + p); + } + + /** + * Check if the path is referencing a file. + * This is mainly needed to avoid symlinks. + * @param p + * @return true if the p is a file + * @throws IOException + */ + private boolean isFile(Path p) throws IOException { + FileStatus status = fs.getFileStatus(p); + boolean isFile = !status.isDir(); + try { + isFile = isFile && !(Boolean)Methods.call(FileStatus.class, status, "isSymlink", null, null); + } catch (Exception e) { + } + return isFile; + } + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java new file mode 100644 index 0000000..e381d3c --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadProtocol.java @@ -0,0 +1,67 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.access; + +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.security.TokenInfo; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.security.token.Token; + +import java.io.IOException; +import java.util.List; + +/** + * Provides a secure way to bulk load data onto HBase + * These are internal API. Bulk load should be initiated + * via {@link org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles} + * with security enabled. + */ +@TokenInfo("HBASE_AUTH_TOKEN") +public interface SecureBulkLoadProtocol extends CoprocessorProtocol { + + /** + * Prepare for bulk load. + * Will be called before bulkLoadHFiles() + * @param tableName + * @return a bulkToken which uniquely identifies the bulk session + * @throws IOException + */ + String prepareBulkLoad(byte[] tableName) throws IOException; + + /** + * Cleanup after bulk load. + * Will be called after bulkLoadHFiles(). + * @param bulkToken + * @throws IOException + */ + void cleanupBulkLoad(String bulkToken) throws IOException; + + /** + * Secure version of HRegionServer.bulkLoadHFiles(). + * @param familyPaths column family to HFile path pairs + * @param userToken requesting user's HDFS delegation token + * @param bulkToken + * @return + * @throws IOException + */ + boolean bulkLoadHFiles(List> familyPaths, + Token userToken, String bulkToken) throws IOException; + +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java new file mode 100644 index 0000000..ba73f4e --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java @@ -0,0 +1,526 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +import java.io.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * Performs authorization checks for a given user's assigned permissions + */ +public class TableAuthManager { + private static class PermissionCache { + /** Cache of user permissions */ + private ListMultimap userCache = ArrayListMultimap.create(); + /** Cache of group permissions */ + private ListMultimap groupCache = ArrayListMultimap.create(); + + public List getUser(String user) { + return userCache.get(user); + } + + public void putUser(String user, T perm) { + userCache.put(user, perm); + } + + public List replaceUser(String user, Iterable perms) { + return userCache.replaceValues(user, perms); + } + + public List getGroup(String group) { + return groupCache.get(group); + } + + public void putGroup(String group, T perm) { + groupCache.put(group, perm); + } + + public List replaceGroup(String group, Iterable perms) { + return groupCache.replaceValues(group, perms); + } + + /** + * Returns a combined map of user and group permissions, with group names prefixed by + * {@link AccessControlLists#GROUP_PREFIX}. + */ + public ListMultimap getAllPermissions() { + ListMultimap tmp = ArrayListMultimap.create(); + tmp.putAll(userCache); + for (String group : groupCache.keySet()) { + tmp.putAll(AccessControlLists.GROUP_PREFIX + group, groupCache.get(group)); + } + return tmp; + } + } + + private static Log LOG = LogFactory.getLog(TableAuthManager.class); + + private static TableAuthManager instance; + + /** Cache of global permissions */ + private volatile PermissionCache globalCache; + + private ConcurrentSkipListMap> tableCache = + new ConcurrentSkipListMap>(Bytes.BYTES_COMPARATOR); + + private Configuration conf; + private ZKPermissionWatcher zkperms; + + private TableAuthManager(ZooKeeperWatcher watcher, Configuration conf) + throws IOException { + this.conf = conf; + + // initialize global permissions based on configuration + globalCache = initGlobal(conf); + + this.zkperms = new ZKPermissionWatcher(watcher, this, conf); + try { + this.zkperms.start(); + } catch (KeeperException ke) { + LOG.error("ZooKeeper initialization failed", ke); + } + } + + /** + * Returns a new {@code PermissionCache} initialized with permission assignments + * from the {@code hbase.superuser} configuration key. + */ + private PermissionCache initGlobal(Configuration conf) throws IOException { + User user = User.getCurrent(); + if (user == null) { + throw new IOException("Unable to obtain the current user, " + + "authorization checks for internal operations will not work correctly!"); + } + PermissionCache newCache = new PermissionCache(); + String currentUser = user.getShortName(); + + // the system user is always included + List superusers = Lists.asList(currentUser, conf.getStrings( + AccessControlLists.SUPERUSER_CONF_KEY, new String[0])); + if (superusers != null) { + for (String name : superusers) { + if (AccessControlLists.isGroupPrincipal(name)) { + newCache.putGroup(AccessControlLists.getGroupName(name), + new Permission(Permission.Action.values())); + } else { + newCache.putUser(name, new Permission(Permission.Action.values())); + } + } + } + return newCache; + } + + public ZKPermissionWatcher getZKPermissionWatcher() { + return this.zkperms; + } + + public void refreshCacheFromWritable(byte[] table, byte[] data) throws IOException { + if (data != null && data.length > 0) { + DataInput in = new DataInputStream(new ByteArrayInputStream(data)); + ListMultimap perms = AccessControlLists.readPermissions(in, conf); + if (perms != null) { + if (Bytes.equals(table, AccessControlLists.ACL_GLOBAL_NAME)) { + updateGlobalCache(perms); + } else { + updateTableCache(table, perms); + } + } + } else { + LOG.debug("Skipping permission cache refresh because writable data is empty"); + } + } + + /** + * Updates the internal global permissions cache + * + * @param userPerms + */ + private void updateGlobalCache(ListMultimap userPerms) { + PermissionCache newCache = null; + try { + newCache = initGlobal(conf); + for (Map.Entry entry : userPerms.entries()) { + if (AccessControlLists.isGroupPrincipal(entry.getKey())) { + newCache.putGroup(AccessControlLists.getGroupName(entry.getKey()), + new Permission(entry.getValue().getActions())); + } else { + newCache.putUser(entry.getKey(), new Permission(entry.getValue().getActions())); + } + } + globalCache = newCache; + } catch (IOException e) { + // Never happens + LOG.error("Error occured while updating the global cache", e); + } + } + + /** + * Updates the internal permissions cache for a single table, splitting + * the permissions listed into separate caches for users and groups to optimize + * group lookups. + * + * @param table + * @param tablePerms + */ + private void updateTableCache(byte[] table, ListMultimap tablePerms) { + PermissionCache newTablePerms = new PermissionCache(); + + for (Map.Entry entry : tablePerms.entries()) { + if (AccessControlLists.isGroupPrincipal(entry.getKey())) { + newTablePerms.putGroup(AccessControlLists.getGroupName(entry.getKey()), entry.getValue()); + } else { + newTablePerms.putUser(entry.getKey(), entry.getValue()); + } + } + + tableCache.put(table, newTablePerms); + } + + private PermissionCache getTablePermissions(byte[] table) { + if (!tableCache.containsKey(table)) { + tableCache.putIfAbsent(table, new PermissionCache()); + } + return tableCache.get(table); + } + + /** + * Authorizes a global permission + * @param perms + * @param action + * @return + */ + private boolean authorize(List perms, Permission.Action action) { + if (perms != null) { + for (Permission p : perms) { + if (p.implies(action)) { + return true; + } + } + } else if (LOG.isDebugEnabled()) { + LOG.debug("No permissions found"); + } + + return false; + } + + /** + * Authorize a global permission based on ACLs for the given user and the + * user's groups. + * @param user + * @param action + * @return + */ + public boolean authorize(User user, Permission.Action action) { + if (user == null) { + return false; + } + + if (authorize(globalCache.getUser(user.getShortName()), action)) { + return true; + } + + String[] groups = user.getGroupNames(); + if (groups != null) { + for (String group : groups) { + if (authorize(globalCache.getGroup(group), action)) { + return true; + } + } + } + return false; + } + + private boolean authorize(List perms, byte[] table, byte[] family, + Permission.Action action) { + return authorize(perms, table, family, null, action); + } + + private boolean authorize(List perms, byte[] table, byte[] family, + byte[] qualifier, Permission.Action action) { + if (perms != null) { + for (TablePermission p : perms) { + if (p.implies(table, family, qualifier, action)) { + return true; + } + } + } else if (LOG.isDebugEnabled()) { + LOG.debug("No permissions found for table="+Bytes.toStringBinary(table)); + } + return false; + } + + public boolean authorize(User user, byte[] table, KeyValue kv, + TablePermission.Action action) { + PermissionCache tablePerms = tableCache.get(table); + if (tablePerms != null) { + List userPerms = tablePerms.getUser(user.getShortName()); + if (authorize(userPerms, table, kv, action)) { + return true; + } + + String[] groupNames = user.getGroupNames(); + if (groupNames != null) { + for (String group : groupNames) { + List groupPerms = tablePerms.getGroup(group); + if (authorize(groupPerms, table, kv, action)) { + return true; + } + } + } + } + + return false; + } + + private boolean authorize(List perms, byte[] table, KeyValue kv, + TablePermission.Action action) { + if (perms != null) { + for (TablePermission p : perms) { + if (p.implies(table, kv, action)) { + return true; + } + } + } else if (LOG.isDebugEnabled()) { + LOG.debug("No permissions for authorize() check, table=" + + Bytes.toStringBinary(table)); + } + + return false; + } + + /** + * Checks global authorization for a specific action for a user, based on the + * stored user permissions. + */ + public boolean authorizeUser(String username, Permission.Action action) { + return authorize(globalCache.getUser(username), action); + } + + /** + * Checks authorization to a given table and column family for a user, based on the + * stored user permissions. + * + * @param username + * @param table + * @param family + * @param action + * @return + */ + public boolean authorizeUser(String username, byte[] table, byte[] family, + Permission.Action action) { + return authorizeUser(username, table, family, null, action); + } + + public boolean authorizeUser(String username, byte[] table, byte[] family, + byte[] qualifier, Permission.Action action) { + // global authorization supercedes table level + if (authorizeUser(username, action)) { + return true; + } + return authorize(getTablePermissions(table).getUser(username), table, family, + qualifier, action); + } + + + /** + * Checks authorization for a given action for a group, based on the stored + * permissions. + */ + public boolean authorizeGroup(String groupName, Permission.Action action) { + return authorize(globalCache.getGroup(groupName), action); + } + + /** + * Checks authorization to a given table and column family for a group, based + * on the stored permissions. + * @param groupName + * @param table + * @param family + * @param action + * @return + */ + public boolean authorizeGroup(String groupName, byte[] table, byte[] family, + Permission.Action action) { + // global authorization supercedes table level + if (authorizeGroup(groupName, action)) { + return true; + } + return authorize(getTablePermissions(table).getGroup(groupName), table, family, action); + } + + public boolean authorize(User user, byte[] table, byte[] family, + byte[] qualifier, Permission.Action action) { + if (authorizeUser(user.getShortName(), table, family, qualifier, action)) { + return true; + } + + String[] groups = user.getGroupNames(); + if (groups != null) { + for (String group : groups) { + if (authorizeGroup(group, table, family, action)) { + return true; + } + } + } + return false; + } + + public boolean authorize(User user, byte[] table, byte[] family, + Permission.Action action) { + return authorize(user, table, family, null, action); + } + + /** + * Returns true if the given user has a {@link TablePermission} matching up + * to the column family portion of a permission. Note that this permission + * may be scoped to a given column qualifier and does not guarantee that + * authorize() on the same column family would return true. + */ + public boolean matchPermission(User user, + byte[] table, byte[] family, TablePermission.Action action) { + PermissionCache tablePerms = tableCache.get(table); + if (tablePerms != null) { + List userPerms = tablePerms.getUser(user.getShortName()); + if (userPerms != null) { + for (TablePermission p : userPerms) { + if (p.matchesFamily(table, family, action)) { + return true; + } + } + } + + String[] groups = user.getGroupNames(); + if (groups != null) { + for (String group : groups) { + List groupPerms = tablePerms.getGroup(group); + if (groupPerms != null) { + for (TablePermission p : groupPerms) { + if (p.matchesFamily(table, family, action)) { + return true; + } + } + } + } + } + } + + return false; + } + + public boolean matchPermission(User user, + byte[] table, byte[] family, byte[] qualifier, + TablePermission.Action action) { + PermissionCache tablePerms = tableCache.get(table); + if (tablePerms != null) { + List userPerms = tablePerms.getUser(user.getShortName()); + if (userPerms != null) { + for (TablePermission p : userPerms) { + if (p.matchesFamilyQualifier(table, family, qualifier, action)) { + return true; + } + } + } + + String[] groups = user.getGroupNames(); + if (groups != null) { + for (String group : groups) { + List groupPerms = tablePerms.getGroup(group); + if (groupPerms != null) { + for (TablePermission p : groupPerms) { + if (p.matchesFamilyQualifier(table, family, qualifier, action)) { + return true; + } + } + } + } + } + } + + return false; + } + + public void remove(byte[] table) { + tableCache.remove(table); + } + + /** + * Overwrites the existing permission set for a given user for a table, and + * triggers an update for zookeeper synchronization. + * @param username + * @param table + * @param perms + */ + public void setUserPermissions(String username, byte[] table, + List perms) { + PermissionCache tablePerms = getTablePermissions(table); + tablePerms.replaceUser(username, perms); + writeToZooKeeper(table, tablePerms); + } + + /** + * Overwrites the existing permission set for a group and triggers an update + * for zookeeper synchronization. + * @param group + * @param table + * @param perms + */ + public void setGroupPermissions(String group, byte[] table, + List perms) { + PermissionCache tablePerms = getTablePermissions(table); + tablePerms.replaceGroup(group, perms); + writeToZooKeeper(table, tablePerms); + } + + public void writeToZooKeeper(byte[] table, + PermissionCache tablePerms) { + byte[] serialized = new byte[0]; + if (tablePerms != null) { + serialized = AccessControlLists.writePermissionsAsBytes(tablePerms.getAllPermissions(), conf); + } + zkperms.writeToZookeeper(table, serialized); + } + + static Map managerMap = + new HashMap(); + + public synchronized static TableAuthManager get( + ZooKeeperWatcher watcher, Configuration conf) throws IOException { + instance = managerMap.get(watcher); + if (instance == null) { + instance = new TableAuthManager(watcher, conf); + managerMap.put(watcher, instance); + } + return instance; + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/TablePermission.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/TablePermission.java new file mode 100644 index 0000000..d881201 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/TablePermission.java @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * Represents an authorization for access for the given actions, optionally + * restricted to the given column family or column qualifier, over the + * given table. If the family property is null, it implies + * full table access. + */ +public class TablePermission extends Permission { + private static Log LOG = LogFactory.getLog(TablePermission.class); + + private byte[] table; + private byte[] family; + private byte[] qualifier; + + /** Nullary constructor for Writable, do not use */ + public TablePermission() { + super(); + } + + /** + * Create a new permission for the given table and (optionally) column family, + * allowing the given actions. + * @param table the table + * @param family the family, can be null if a global permission on the table + * @param assigned the list of allowed actions + */ + public TablePermission(byte[] table, byte[] family, Action... assigned) { + this(table, family, null, assigned); + } + + /** + * Creates a new permission for the given table, restricted to the given + * column family and qualifer, allowing the assigned actions to be performed. + * @param table the table + * @param family the family, can be null if a global permission on the table + * @param assigned the list of allowed actions + */ + public TablePermission(byte[] table, byte[] family, byte[] qualifier, + Action... assigned) { + super(assigned); + this.table = table; + this.family = family; + this.qualifier = qualifier; + } + + /** + * Creates a new permission for the given table, family and column qualifier, + * allowing the actions matching the provided byte codes to be performed. + * @param table the table + * @param family the family, can be null if a global permission on the table + * @param actionCodes the list of allowed action codes + */ + public TablePermission(byte[] table, byte[] family, byte[] qualifier, + byte[] actionCodes) { + super(actionCodes); + this.table = table; + this.family = family; + this.qualifier = qualifier; + } + + public byte[] getTable() { + return table; + } + + public byte[] getFamily() { + return family; + } + + public byte[] getQualifier() { + return qualifier; + } + + /** + * Checks that a given table operation is authorized by this permission + * instance. + * + * @param table the table where the operation is being performed + * @param family the column family to which the operation is restricted, + * if null implies "all" + * @param qualifier the column qualifier to which the action is restricted, + * if null implies "all" + * @param action the action being requested + * @return true if the action within the given scope is allowed + * by this permission, false + */ + public boolean implies(byte[] table, byte[] family, byte[] qualifier, + Action action) { + if (!Bytes.equals(this.table, table)) { + return false; + } + + if (this.family != null && + (family == null || + !Bytes.equals(this.family, family))) { + return false; + } + + if (this.qualifier != null && + (qualifier == null || + !Bytes.equals(this.qualifier, qualifier))) { + return false; + } + + // check actions + return super.implies(action); + } + + /** + * Checks if this permission grants access to perform the given action on + * the given table and key value. + * @param table the table on which the operation is being performed + * @param kv the KeyValue on which the operation is being requested + * @param action the action requested + * @return true if the action is allowed over the given scope + * by this permission, otherwise false + */ + public boolean implies(byte[] table, KeyValue kv, Action action) { + if (!Bytes.equals(this.table, table)) { + return false; + } + + if (family != null && + (Bytes.compareTo(family, 0, family.length, + kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength()) != 0)) { + return false; + } + + if (qualifier != null && + (Bytes.compareTo(qualifier, 0, qualifier.length, + kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength()) != 0)) { + return false; + } + + // check actions + return super.implies(action); + } + + /** + * Returns true if this permission matches the given column + * family at least. This only indicates a partial match against the table + * and column family, however, and does not guarantee that implies() for the + * column same family would return true. In the case of a + * column-qualifier specific permission, for example, implies() would still + * return false. + */ + public boolean matchesFamily(byte[] table, byte[] family, Action action) { + if (!Bytes.equals(this.table, table)) { + return false; + } + + if (this.family != null && + (family == null || + !Bytes.equals(this.family, family))) { + return false; + } + + // ignore qualifier + // check actions + return super.implies(action); + } + + /** + * Returns if the given permission matches the given qualifier. + * @param table the table name to match + * @param family the column family to match + * @param qualifier the qualifier name to match + * @param action the action requested + * @return true if the table, family and qualifier match, + * otherwise false + */ + public boolean matchesFamilyQualifier(byte[] table, byte[] family, byte[] qualifier, + Action action) { + if (!matchesFamily(table, family, action)) { + return false; + } else { + if (this.qualifier != null && + (qualifier == null || + !Bytes.equals(this.qualifier, qualifier))) { + return false; + } + } + return super.implies(action); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof TablePermission)) { + return false; + } + TablePermission other = (TablePermission)obj; + + if (!(Bytes.equals(table, other.getTable()) && + ((family == null && other.getFamily() == null) || + Bytes.equals(family, other.getFamily())) && + ((qualifier == null && other.getQualifier() == null) || + Bytes.equals(qualifier, other.getQualifier())) + )) { + return false; + } + + // check actions + return super.equals(other); + } + + @Override + public int hashCode() { + final int prime = 37; + int result = super.hashCode(); + if (table != null) { + result = prime * result + Bytes.hashCode(table); + } + if (family != null) { + result = prime * result + Bytes.hashCode(family); + } + if (qualifier != null) { + result = prime * result + Bytes.hashCode(qualifier); + } + return result; + } + + public String toString() { + StringBuilder str = new StringBuilder("[TablePermission: ") + .append("table=").append(Bytes.toString(table)) + .append(", family=").append(Bytes.toString(family)) + .append(", qualifier=").append(Bytes.toString(qualifier)) + .append(", actions="); + if (actions != null) { + for (int i=0; i 0) + str.append(","); + if (actions[i] != null) + str.append(actions[i].toString()); + else + str.append("NULL"); + } + } + str.append("]"); + + return str.toString(); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + table = Bytes.readByteArray(in); + if (in.readBoolean()) { + family = Bytes.readByteArray(in); + } + if (in.readBoolean()) { + qualifier = Bytes.readByteArray(in); + } + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + Bytes.writeByteArray(out, table); + out.writeBoolean(family != null); + if (family != null) { + Bytes.writeByteArray(out, family); + } + out.writeBoolean(qualifier != null); + if (qualifier != null) { + Bytes.writeByteArray(out, qualifier); + } + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/UserPermission.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/UserPermission.java new file mode 100644 index 0000000..fd5b755 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/UserPermission.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.util.Bytes; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * Represents an authorization for access over the given table, column family + * plus qualifier, for the given user. + */ +public class UserPermission extends TablePermission { + private static Log LOG = LogFactory.getLog(UserPermission.class); + + private byte[] user; + + /** Nullary constructor for Writable, do not use */ + public UserPermission() { + super(); + } + + /** + * Creates a new instance for the given user. + * @param user the user + * @param assigned the list of allowed actions + */ + public UserPermission(byte[] user, Action... assigned) { + super(null, null, null, assigned); + this.user = user; + } + + /** + * Creates a new instance for the given user, + * matching the actions with the given codes. + * @param user the user + * @param actionCodes the list of allowed action codes + */ + public UserPermission(byte[] user, byte[] actionCodes) { + super(null, null, null, actionCodes); + this.user = user; + } + + /** + * Creates a new instance for the given user, table and column family. + * @param user the user + * @param table the table + * @param family the family, can be null if action is allowed over the entire + * table + * @param assigned the list of allowed actions + */ + public UserPermission(byte[] user, byte[] table, byte[] family, + Action... assigned) { + super(table, family, assigned); + this.user = user; + } + + /** + * Creates a new permission for the given user, table, column family and + * column qualifier. + * @param user the user + * @param table the table + * @param family the family, can be null if action is allowed over the entire + * table + * @param qualifier the column qualifier, can be null if action is allowed + * over the entire column family + * @param assigned the list of allowed actions + */ + public UserPermission(byte[] user, byte[] table, byte[] family, + byte[] qualifier, Action... assigned) { + super(table, family, qualifier, assigned); + this.user = user; + } + + /** + * Creates a new instance for the given user, table, column family and + * qualifier, matching the actions with the given codes. + * @param user the user + * @param table the table + * @param family the family, can be null if action is allowed over the entire + * table + * @param qualifier the column qualifier, can be null if action is allowed + * over the entire column family + * @param actionCodes the list of allowed action codes + */ + public UserPermission(byte[] user, byte[] table, byte[] family, + byte[] qualifier, byte[] actionCodes) { + super(table, family, qualifier, actionCodes); + this.user = user; + } + + public byte[] getUser() { + return user; + } + + /** + * Returns true if this permission describes a global user permission. + */ + public boolean isGlobal() { + byte[] tableName = getTable(); + return(tableName == null || tableName.length == 0); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof UserPermission)) { + return false; + } + UserPermission other = (UserPermission)obj; + + if ((Bytes.equals(user, other.getUser()) && + super.equals(obj))) { + return true; + } else { + return false; + } + } + + @Override + public int hashCode() { + final int prime = 37; + int result = super.hashCode(); + if (user != null) { + result = prime * result + Bytes.hashCode(user); + } + return result; + } + + public String toString() { + StringBuilder str = new StringBuilder("UserPermission: ") + .append("user=").append(Bytes.toString(user)) + .append(", ").append(super.toString()); + return str.toString(); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + user = Bytes.readByteArray(in); + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + Bytes.writeByteArray(out, user); + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java b/security/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java new file mode 100644 index 0000000..4870bb1 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +import java.io.IOException; +import java.util.List; + +/** + * Handles synchronization of access control list entries and updates + * throughout all nodes in the cluster. The {@link AccessController} instance + * on the {@code _acl_} table regions, creates a znode for each table as + * {@code /hbase/acl/tablename}, with the znode data containing a serialized + * list of the permissions granted for the table. The {@code AccessController} + * instances on all other cluster hosts watch the znodes for updates, which + * trigger updates in the {@link TableAuthManager} permission cache. + */ +public class ZKPermissionWatcher extends ZooKeeperListener { + private static Log LOG = LogFactory.getLog(ZKPermissionWatcher.class); + // parent node for permissions lists + static final String ACL_NODE = "acl"; + TableAuthManager authManager; + String aclZNode; + + public ZKPermissionWatcher(ZooKeeperWatcher watcher, + TableAuthManager authManager, Configuration conf) { + super(watcher); + this.authManager = authManager; + String aclZnodeParent = conf.get("zookeeper.znode.acl.parent", ACL_NODE); + this.aclZNode = ZKUtil.joinZNode(watcher.baseZNode, aclZnodeParent); + } + + public void start() throws KeeperException { + watcher.registerListener(this); + if (ZKUtil.watchAndCheckExists(watcher, aclZNode)) { + List existing = + ZKUtil.getChildDataAndWatchForNewChildren(watcher, aclZNode); + if (existing != null) { + refreshNodes(existing); + } + } + } + + @Override + public void nodeCreated(String path) { + if (path.equals(aclZNode)) { + try { + List nodes = + ZKUtil.getChildDataAndWatchForNewChildren(watcher, aclZNode); + refreshNodes(nodes); + } catch (KeeperException ke) { + LOG.error("Error reading data from zookeeper", ke); + // only option is to abort + watcher.abort("Zookeeper error obtaining acl node children", ke); + } + } + } + + @Override + public void nodeDeleted(String path) { + if (aclZNode.equals(ZKUtil.getParent(path))) { + String table = ZKUtil.getNodeName(path); + authManager.remove(Bytes.toBytes(table)); + } + } + + @Override + public void nodeDataChanged(String path) { + if (aclZNode.equals(ZKUtil.getParent(path))) { + // update cache on an existing table node + String table = ZKUtil.getNodeName(path); + try { + byte[] data = ZKUtil.getDataAndWatch(watcher, path); + authManager.refreshCacheFromWritable(Bytes.toBytes(table), data); + } catch (KeeperException ke) { + LOG.error("Error reading data from zookeeper for node "+table, ke); + // only option is to abort + watcher.abort("Zookeeper error getting data for node " + table, ke); + } catch (IOException ioe) { + LOG.error("Error reading permissions writables", ioe); + } + } + } + + @Override + public void nodeChildrenChanged(String path) { + if (path.equals(aclZNode)) { + // table permissions changed + try { + List nodes = + ZKUtil.getChildDataAndWatchForNewChildren(watcher, aclZNode); + refreshNodes(nodes); + } catch (KeeperException ke) { + LOG.error("Error reading data from zookeeper for path "+path, ke); + watcher.abort("Zookeeper error get node children for path "+path, ke); + } + } + } + + private void refreshNodes(List nodes) { + for (ZKUtil.NodeAndData n : nodes) { + if (n.isEmpty()) continue; + String path = n.getNode(); + String table = ZKUtil.getNodeName(path); + try { + byte[] nodeData = n.getData(); + if (LOG.isDebugEnabled()) { + LOG.debug("Updating permissions cache from node "+table+" with data: "+ + Bytes.toStringBinary(nodeData)); + } + authManager.refreshCacheFromWritable(Bytes.toBytes(table), + nodeData); + } catch (IOException ioe) { + LOG.error("Failed parsing permissions for table '" + table + + "' from zk", ioe); + } + } + } + + /*** + * Write a table's access controls to the permissions mirror in zookeeper + * @param tableName + * @param permsData + */ + public void writeToZookeeper(byte[] tableName, byte[] parmsData) { + String zkNode = ZKUtil.joinZNode(watcher.baseZNode, ACL_NODE); + zkNode = ZKUtil.joinZNode(zkNode, Bytes.toString(tableName)); + + try { + ZKUtil.createWithParents(watcher, zkNode); + ZKUtil.updateExistingNodeData(watcher, zkNode, parmsData, -1); + } catch (KeeperException e) { + LOG.error("Failed updating permissions for table '" + + Bytes.toString(tableName) + "'", e); + watcher.abort("Failed writing node "+zkNode+" to zookeeper", e); + } + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationKey.java b/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationKey.java new file mode 100644 index 0000000..c68af4e --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationKey.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.token; + +import javax.crypto.SecretKey; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; + +/** + * Represents a secret key used for signing and verifying authentication tokens + * by {@link AuthenticationTokenSecretManager}. + */ +public class AuthenticationKey implements Writable { + private int id; + private long expirationDate; + private SecretKey secret; + + public AuthenticationKey() { + // for Writable + } + + public AuthenticationKey(int keyId, long expirationDate, SecretKey key) { + this.id = keyId; + this.expirationDate = expirationDate; + this.secret = key; + } + + public int getKeyId() { + return id; + } + + public long getExpiration() { + return expirationDate; + } + + public void setExpiration(long timestamp) { + expirationDate = timestamp; + } + + SecretKey getKey() { + return secret; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof AuthenticationKey)) { + return false; + } + AuthenticationKey other = (AuthenticationKey)obj; + return id == other.getKeyId() && + expirationDate == other.getExpiration() && + (secret == null ? other.getKey() == null : + other.getKey() != null && + Bytes.equals(secret.getEncoded(), other.getKey().getEncoded())); + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(); + buf.append("AuthenticationKey[ ") + .append("id=").append(id) + .append(", expiration=").append(expirationDate) + .append(" ]"); + return buf.toString(); + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, id); + WritableUtils.writeVLong(out, expirationDate); + if (secret == null) { + WritableUtils.writeVInt(out, -1); + } else { + byte[] keyBytes = secret.getEncoded(); + WritableUtils.writeVInt(out, keyBytes.length); + out.write(keyBytes); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + id = WritableUtils.readVInt(in); + expirationDate = WritableUtils.readVLong(in); + int keyLength = WritableUtils.readVInt(in); + if (keyLength < 0) { + secret = null; + } else { + byte[] keyBytes = new byte[keyLength]; + in.readFully(keyBytes); + secret = AuthenticationTokenSecretManager.createSecretKey(keyBytes); + } + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationProtocol.java b/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationProtocol.java new file mode 100644 index 0000000..2eb9e15 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationProtocol.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.token; + +import java.io.IOException; + +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.security.token.Token; + +/** + * Defines a custom RPC protocol for obtaining authentication tokens + */ +public interface AuthenticationProtocol extends CoprocessorProtocol { + /** + * Obtains a token capable of authenticating as the current user for future + * connections. + * @return an authentication token for the current user + * @throws IOException If obtaining a token is denied or encounters an error + */ + public Token getAuthenticationToken() + throws IOException; + + /** + * Returns the currently authenticated username. + */ + public String whoami(); +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenIdentifier.java b/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenIdentifier.java new file mode 100644 index 0000000..62d2c96 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenIdentifier.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.token; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.TokenIdentifier; + +/** + * Represents the identity information stored in an HBase authentication token. + */ +public class AuthenticationTokenIdentifier extends TokenIdentifier { + public static final byte VERSION = 1; + public static final Text AUTH_TOKEN_TYPE = new Text("HBASE_AUTH_TOKEN"); + + protected String username; + protected int keyId; + protected long issueDate; + protected long expirationDate; + protected long sequenceNumber; + + public AuthenticationTokenIdentifier() { + } + + public AuthenticationTokenIdentifier(String username) { + this.username = username; + } + + public AuthenticationTokenIdentifier(String username, int keyId, + long issueDate, long expirationDate) { + this.username = username; + this.keyId = keyId; + this.issueDate = issueDate; + this.expirationDate = expirationDate; + } + + @Override + public Text getKind() { + return AUTH_TOKEN_TYPE; + } + + @Override + public UserGroupInformation getUser() { + if (username == null || "".equals(username)) { + return null; + } + return UserGroupInformation.createRemoteUser(username); + } + + public String getUsername() { + return username; + } + + void setUsername(String name) { + this.username = name; + } + + public int getKeyId() { + return keyId; + } + + void setKeyId(int id) { + this.keyId = id; + } + + public long getIssueDate() { + return issueDate; + } + + void setIssueDate(long timestamp) { + this.issueDate = timestamp; + } + + public long getExpirationDate() { + return expirationDate; + } + + void setExpirationDate(long timestamp) { + this.expirationDate = timestamp; + } + + public long getSequenceNumber() { + return sequenceNumber; + } + + void setSequenceNumber(long seq) { + this.sequenceNumber = seq; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeByte(VERSION); + WritableUtils.writeString(out, username); + WritableUtils.writeVInt(out, keyId); + WritableUtils.writeVLong(out, issueDate); + WritableUtils.writeVLong(out, expirationDate); + WritableUtils.writeVLong(out, sequenceNumber); + } + + @Override + public void readFields(DataInput in) throws IOException { + byte version = in.readByte(); + if (version != VERSION) { + throw new IOException("Version mismatch in deserialization: " + + "expected="+VERSION+", got="+version); + } + username = WritableUtils.readString(in); + keyId = WritableUtils.readVInt(in); + issueDate = WritableUtils.readVLong(in); + expirationDate = WritableUtils.readVLong(in); + sequenceNumber = WritableUtils.readVLong(in); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other instanceof AuthenticationTokenIdentifier) { + AuthenticationTokenIdentifier ident = (AuthenticationTokenIdentifier)other; + return sequenceNumber == ident.getSequenceNumber() + && keyId == ident.getKeyId() + && issueDate == ident.getIssueDate() + && expirationDate == ident.getExpirationDate() + && (username == null ? ident.getUsername() == null : + username.equals(ident.getUsername())); + } + return false; + } + + @Override + public int hashCode() { + return (int)sequenceNumber; + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java b/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java new file mode 100644 index 0000000..7218fc5 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.token; + +import javax.crypto.SecretKey; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.zookeeper.ClusterId; +import org.apache.hadoop.hbase.zookeeper.ZKLeaderManager; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.security.token.Token; +import org.apache.zookeeper.KeeperException; + +/** + * Manages an internal list of secret keys used to sign new authentication + * tokens as they are generated, and to valid existing tokens used for + * authentication. + * + *

+ * A single instance of {@code AuthenticationTokenSecretManager} will be + * running as the "leader" in a given HBase cluster. The leader is responsible + * for periodically generating new secret keys, which are then distributed to + * followers via ZooKeeper, and for expiring previously used secret keys that + * are no longer needed (as any tokens using them have expired). + *

+ */ +public class AuthenticationTokenSecretManager + extends SecretManager { + + static final String NAME_PREFIX = "SecretManager-"; + + private static Log LOG = LogFactory.getLog( + AuthenticationTokenSecretManager.class); + + private long lastKeyUpdate; + private long keyUpdateInterval; + private long tokenMaxLifetime; + private ZKSecretWatcher zkWatcher; + private LeaderElector leaderElector; + private ClusterId clusterId; + + private Map allKeys = + new ConcurrentHashMap(); + private AuthenticationKey currentKey; + + private int idSeq; + private AtomicLong tokenSeq = new AtomicLong(); + private String name; + + /** + * Create a new secret manager instance for generating keys. + * @param conf Configuration to use + * @param zk Connection to zookeeper for handling leader elections + * @param keyUpdateInterval Time (in milliseconds) between rolling a new master key for token signing + * @param tokenMaxLifetime Maximum age (in milliseconds) before a token expires and is no longer valid + */ + /* TODO: Restrict access to this constructor to make rogues instances more difficult. + * For the moment this class is instantiated from + * org.apache.hadoop.hbase.ipc.SecureServer so public access is needed. + */ + public AuthenticationTokenSecretManager(Configuration conf, + ZooKeeperWatcher zk, String serverName, + long keyUpdateInterval, long tokenMaxLifetime) { + this.zkWatcher = new ZKSecretWatcher(conf, zk, this); + this.keyUpdateInterval = keyUpdateInterval; + this.tokenMaxLifetime = tokenMaxLifetime; + this.leaderElector = new LeaderElector(zk, serverName); + this.name = NAME_PREFIX+serverName; + this.clusterId = new ClusterId(zk, zk); + } + + public void start() { + try { + // populate any existing keys + this.zkWatcher.start(); + // try to become leader + this.leaderElector.start(); + } catch (KeeperException ke) { + LOG.error("Zookeeper initialization failed", ke); + } + } + + public void stop() { + this.leaderElector.stop("SecretManager stopping"); + } + + public boolean isMaster() { + return leaderElector.isMaster(); + } + + public String getName() { + return name; + } + + @Override + protected byte[] createPassword(AuthenticationTokenIdentifier identifier) { + long now = EnvironmentEdgeManager.currentTimeMillis(); + AuthenticationKey secretKey = currentKey; + identifier.setKeyId(secretKey.getKeyId()); + identifier.setIssueDate(now); + identifier.setExpirationDate(now + tokenMaxLifetime); + identifier.setSequenceNumber(tokenSeq.getAndIncrement()); + return createPassword(WritableUtils.toByteArray(identifier), + secretKey.getKey()); + } + + @Override + public byte[] retrievePassword(AuthenticationTokenIdentifier identifier) + throws InvalidToken { + long now = EnvironmentEdgeManager.currentTimeMillis(); + if (identifier.getExpirationDate() < now) { + throw new InvalidToken("Token has expired"); + } + AuthenticationKey masterKey = allKeys.get(identifier.getKeyId()); + if (masterKey == null) { + throw new InvalidToken("Unknown master key for token (id="+ + identifier.getKeyId()+")"); + } + // regenerate the password + return createPassword(WritableUtils.toByteArray(identifier), + masterKey.getKey()); + } + + @Override + public AuthenticationTokenIdentifier createIdentifier() { + return new AuthenticationTokenIdentifier(); + } + + public Token generateToken(String username) { + AuthenticationTokenIdentifier ident = + new AuthenticationTokenIdentifier(username); + Token token = + new Token(ident, this); + if (clusterId.hasId()) { + token.setService(new Text(clusterId.getId())); + } + return token; + } + + public synchronized void addKey(AuthenticationKey key) throws IOException { + // ignore zk changes when running as master + if (leaderElector.isMaster()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Running as master, ignoring new key "+key.getKeyId()); + } + return; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Adding key "+key.getKeyId()); + } + + allKeys.put(key.getKeyId(), key); + if (currentKey == null || key.getKeyId() > currentKey.getKeyId()) { + currentKey = key; + } + // update current sequence + if (key.getKeyId() > idSeq) { + idSeq = key.getKeyId(); + } + } + + synchronized void removeKey(Integer keyId) { + // ignore zk changes when running as master + if (leaderElector.isMaster()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Running as master, ignoring removed key "+keyId); + } + return; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Removing key "+keyId); + } + + allKeys.remove(keyId); + } + + AuthenticationKey getCurrentKey() { + return currentKey; + } + + AuthenticationKey getKey(int keyId) { + return allKeys.get(keyId); + } + + synchronized void removeExpiredKeys() { + if (!leaderElector.isMaster()) { + LOG.info("Skipping removeExpiredKeys() because not running as master."); + return; + } + + long now = EnvironmentEdgeManager.currentTimeMillis(); + Iterator iter = allKeys.values().iterator(); + while (iter.hasNext()) { + AuthenticationKey key = iter.next(); + if (key.getExpiration() < now) { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing expired key "+key.getKeyId()); + } + iter.remove(); + zkWatcher.removeKeyFromZK(key); + } + } + } + + synchronized void rollCurrentKey() { + if (!leaderElector.isMaster()) { + LOG.info("Skipping rollCurrentKey() because not running as master."); + return; + } + + long now = EnvironmentEdgeManager.currentTimeMillis(); + AuthenticationKey prev = currentKey; + AuthenticationKey newKey = new AuthenticationKey(++idSeq, + Long.MAX_VALUE, // don't allow to expire until it's replaced by a new key + generateSecret()); + allKeys.put(newKey.getKeyId(), newKey); + currentKey = newKey; + zkWatcher.addKeyToZK(newKey); + lastKeyUpdate = now; + + if (prev != null) { + // make sure previous key is still stored + prev.setExpiration(now + tokenMaxLifetime); + allKeys.put(prev.getKeyId(), prev); + zkWatcher.updateKeyInZK(prev); + } + } + + public static SecretKey createSecretKey(byte[] raw) { + return SecretManager.createSecretKey(raw); + } + + private class LeaderElector extends Thread implements Stoppable { + private boolean stopped = false; + /** Flag indicating whether we're in charge of rolling/expiring keys */ + private boolean isMaster = false; + private ZKLeaderManager zkLeader; + + public LeaderElector(ZooKeeperWatcher watcher, String serverName) { + setDaemon(true); + setName("ZKSecretWatcher-leaderElector"); + zkLeader = new ZKLeaderManager(watcher, + ZKUtil.joinZNode(zkWatcher.getRootKeyZNode(), "keymaster"), + Bytes.toBytes(serverName), this); + } + + public boolean isMaster() { + return isMaster; + } + + @Override + public boolean isStopped() { + return stopped; + } + + @Override + public void stop(String reason) { + if (stopped) { + return; + } + + stopped = true; + // prevent further key generation when stopping + if (isMaster) { + zkLeader.stepDownAsLeader(); + } + isMaster = false; + LOG.info("Stopping leader election, because: "+reason); + interrupt(); + } + + public void run() { + zkLeader.start(); + zkLeader.waitToBecomeLeader(); + isMaster = true; + + while (!stopped) { + long now = EnvironmentEdgeManager.currentTimeMillis(); + + // clear any expired + removeExpiredKeys(); + + if (lastKeyUpdate + keyUpdateInterval < now) { + // roll a new master key + rollCurrentKey(); + } + + try { + Thread.sleep(5000); + } catch (InterruptedException ie) { + if (LOG.isDebugEnabled()) { + LOG.debug("Interrupted waiting for next update", ie); + } + } + } + } + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSelector.java b/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSelector.java new file mode 100644 index 0000000..943a89d --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSelector.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.token; + +import java.util.Collection; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenSelector; + +public class AuthenticationTokenSelector + implements TokenSelector { + + public AuthenticationTokenSelector() { + } + + @Override + public Token selectToken(Text serviceName, + Collection> tokens) { + if (serviceName != null) { + for (Token ident : tokens) { + if (serviceName.equals(ident.getService()) && + AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE.equals(ident.getKind())) { + return (Token)ident; + } + } + } + return null; + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/token/TokenProvider.java b/security/src/main/java/org/apache/hadoop/hbase/security/token/TokenProvider.java new file mode 100644 index 0000000..0a3a3a6 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/token/TokenProvider.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.token; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.ipc.RequestContext; +import org.apache.hadoop.hbase.ipc.RpcServer; +import org.apache.hadoop.hbase.ipc.SecureServer; +import org.apache.hadoop.hbase.security.AccessDeniedException; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.security.token.Token; + +/** + * Provides a service for obtaining authentication tokens via the + * {@link AuthenticationProtocol} coprocessor protocol. + */ +public class TokenProvider extends BaseEndpointCoprocessor + implements AuthenticationProtocol { + + public static final long VERSION = 0L; + private static Log LOG = LogFactory.getLog(TokenProvider.class); + + private AuthenticationTokenSecretManager secretManager; + + + @Override + public void start(CoprocessorEnvironment env) { + super.start(env); + + // if running at region + if (env instanceof RegionCoprocessorEnvironment) { + RegionCoprocessorEnvironment regionEnv = + (RegionCoprocessorEnvironment)env; + RpcServer server = regionEnv.getRegionServerServices().getRpcServer(); + if (server instanceof SecureServer) { + SecretManager mgr = ((SecureServer)server).getSecretManager(); + if (mgr instanceof AuthenticationTokenSecretManager) { + secretManager = (AuthenticationTokenSecretManager)mgr; + } + } + } + } + + @Override + public Token getAuthenticationToken() + throws IOException { + if (secretManager == null) { + throw new IOException( + "No secret manager configured for token authentication"); + } + + User currentUser = RequestContext.getRequestUser(); + UserGroupInformation ugi = null; + if (currentUser != null) { + ugi = currentUser.getUGI(); + } + if (currentUser == null) { + throw new AccessDeniedException("No authenticated user for request!"); + } else if (!isAllowedDelegationTokenOp(ugi)) { + LOG.warn("Token generation denied for user="+currentUser.getName() + +", authMethod="+ugi.getAuthenticationMethod()); + throw new AccessDeniedException( + "Token generation only allowed for Kerberos authenticated clients"); + } + + return secretManager.generateToken(currentUser.getName()); + } + + /** + * @param ugi + * @return true if delegation token operation is allowed + */ + private boolean isAllowedDelegationTokenOp(UserGroupInformation ugi) throws IOException { + AuthenticationMethod authMethod = ugi.getAuthenticationMethod(); + if (authMethod == AuthenticationMethod.PROXY) { + authMethod = ugi.getRealUser().getAuthenticationMethod(); + } + if (authMethod != AuthenticationMethod.KERBEROS + && authMethod != AuthenticationMethod.KERBEROS_SSL + && authMethod != AuthenticationMethod.CERTIFICATE) { + return false; + } + return true; + } + + @Override + public String whoami() { + return RequestContext.getRequestUserName(); + } + + @Override + public long getProtocolVersion(String protocol, long clientVersion) + throws IOException { + if (AuthenticationProtocol.class.getName().equals(protocol)) { + return TokenProvider.VERSION; + } + LOG.warn("Unknown protocol requested: "+protocol); + return -1; + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/token/TokenUtil.java b/security/src/main/java/org/apache/hadoop/hbase/security/token/TokenUtil.java new file mode 100644 index 0000000..49feeac --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/token/TokenUtil.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.token; + +import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.security.PrivilegedExceptionAction; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; + +/** + * Utility methods for obtaining authentication tokens. + */ +public class TokenUtil { + private static Log LOG = LogFactory.getLog(TokenUtil.class); + + /** + * Obtain and return an authentication token for the current user. + * @param conf The configuration for connecting to the cluster + * @return the authentication token instance + */ + public static Token obtainToken( + Configuration conf) throws IOException { + HTable meta = null; + try { + meta = new HTable(conf, ".META."); + AuthenticationProtocol prot = meta.coprocessorProxy( + AuthenticationProtocol.class, HConstants.EMPTY_START_ROW); + return prot.getAuthenticationToken(); + } finally { + if (meta != null) { + meta.close(); + } + } + } + + private static Text getClusterId(Token token) + throws IOException { + return token.getService() != null + ? token.getService() : new Text("default"); + } + + /** + * Obtain an authentication token for the given user and add it to the + * user's credentials. + * @param conf The configuration for connecting to the cluster + * @param user The user for whom to obtain the token + * @throws IOException If making a remote call to the {@link TokenProvider} fails + * @throws InterruptedException If executing as the given user is interrupted + */ + public static void obtainAndCacheToken(final Configuration conf, + UserGroupInformation user) + throws IOException, InterruptedException { + try { + Token token = + user.doAs(new PrivilegedExceptionAction>() { + public Token run() throws Exception { + return obtainToken(conf); + } + }); + + if (token == null) { + throw new IOException("No token returned for user "+user.getUserName()); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Obtained token "+token.getKind().toString()+" for user "+ + user.getUserName()); + } + user.addToken(token); + } catch (IOException ioe) { + throw ioe; + } catch (InterruptedException ie) { + throw ie; + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected exception obtaining token for user "+user.getUserName()); + } + } + + /** + * Obtain an authentication token on behalf of the given user and add it to + * the credentials for the given map reduce job. + * @param conf The configuration for connecting to the cluster + * @param user The user for whom to obtain the token + * @param job The job instance in which the token should be stored + * @throws IOException If making a remote call to the {@link TokenProvider} fails + * @throws InterruptedException If executing as the given user is interrupted + */ + public static void obtainTokenForJob(final Configuration conf, + UserGroupInformation user, Job job) + throws IOException, InterruptedException { + try { + Token token = + user.doAs(new PrivilegedExceptionAction>() { + public Token run() throws Exception { + return obtainToken(conf); + } + }); + + if (token == null) { + throw new IOException("No token returned for user "+user.getUserName()); + } + Text clusterId = getClusterId(token); + LOG.info("Obtained token "+token.getKind().toString()+" for user "+ + user.getUserName() + " on cluster "+clusterId.toString()); + job.getCredentials().addToken(clusterId, token); + } catch (IOException ioe) { + throw ioe; + } catch (InterruptedException ie) { + throw ie; + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected exception obtaining token for user "+user.getUserName()); + } + } + + /** + * Obtain an authentication token on behalf of the given user and add it to + * the credentials for the given map reduce job. + * @param user The user for whom to obtain the token + * @param job The job configuration in which the token should be stored + * @throws IOException If making a remote call to the {@link TokenProvider} fails + * @throws InterruptedException If executing as the given user is interrupted + */ + public static void obtainTokenForJob(final JobConf job, + UserGroupInformation user) + throws IOException, InterruptedException { + try { + Token token = + user.doAs(new PrivilegedExceptionAction>() { + public Token run() throws Exception { + return obtainToken(job); + } + }); + + if (token == null) { + throw new IOException("No token returned for user "+user.getUserName()); + } + Text clusterId = getClusterId(token); + LOG.info("Obtained token "+token.getKind().toString()+" for user "+ + user.getUserName()+" on cluster "+clusterId.toString()); + job.getCredentials().addToken(clusterId, token); + } catch (IOException ioe) { + throw ioe; + } catch (InterruptedException ie) { + throw ie; + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected exception obtaining token for user "+user.getUserName()); + } + } +} diff --git a/security/src/main/java/org/apache/hadoop/hbase/security/token/ZKSecretWatcher.java b/security/src/main/java/org/apache/hadoop/hbase/security/token/ZKSecretWatcher.java new file mode 100644 index 0000000..d780d50 --- /dev/null +++ b/security/src/main/java/org/apache/hadoop/hbase/security/token/ZKSecretWatcher.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.token; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +/** + * Synchronizes token encryption keys across cluster nodes. + */ +public class ZKSecretWatcher extends ZooKeeperListener { + private static final String DEFAULT_ROOT_NODE = "tokenauth"; + private static final String DEFAULT_KEYS_PARENT = "keys"; + private static Log LOG = LogFactory.getLog(ZKSecretWatcher.class); + + private AuthenticationTokenSecretManager secretManager; + private String baseKeyZNode; + private String keysParentZNode; + + public ZKSecretWatcher(Configuration conf, + ZooKeeperWatcher watcher, + AuthenticationTokenSecretManager secretManager) { + super(watcher); + this.secretManager = secretManager; + String keyZNodeParent = conf.get("zookeeper.znode.tokenauth.parent", DEFAULT_ROOT_NODE); + this.baseKeyZNode = ZKUtil.joinZNode(watcher.baseZNode, keyZNodeParent); + this.keysParentZNode = ZKUtil.joinZNode(baseKeyZNode, DEFAULT_KEYS_PARENT); + } + + public void start() throws KeeperException { + watcher.registerListener(this); + // make sure the base node exists + ZKUtil.createWithParents(watcher, keysParentZNode); + + if (ZKUtil.watchAndCheckExists(watcher, keysParentZNode)) { + List nodes = + ZKUtil.getChildDataAndWatchForNewChildren(watcher, keysParentZNode); + refreshNodes(nodes); + } + } + + @Override + public void nodeCreated(String path) { + if (path.equals(keysParentZNode)) { + try { + List nodes = + ZKUtil.getChildDataAndWatchForNewChildren(watcher, keysParentZNode); + refreshNodes(nodes); + } catch (KeeperException ke) { + LOG.fatal("Error reading data from zookeeper", ke); + watcher.abort("Error reading new key znode "+path, ke); + } + } + } + + @Override + public void nodeDeleted(String path) { + if (keysParentZNode.equals(ZKUtil.getParent(path))) { + String keyId = ZKUtil.getNodeName(path); + try { + Integer id = new Integer(keyId); + secretManager.removeKey(id); + } catch (NumberFormatException nfe) { + LOG.error("Invalid znode name for key ID '"+keyId+"'", nfe); + } + } + } + + @Override + public void nodeDataChanged(String path) { + if (keysParentZNode.equals(ZKUtil.getParent(path))) { + try { + byte[] data = ZKUtil.getDataAndWatch(watcher, path); + if (data == null || data.length == 0) { + LOG.debug("Ignoring empty node "+path); + return; + } + + AuthenticationKey key = (AuthenticationKey)Writables.getWritable(data, + new AuthenticationKey()); + secretManager.addKey(key); + } catch (KeeperException ke) { + LOG.fatal("Error reading data from zookeeper", ke); + watcher.abort("Error reading updated key znode "+path, ke); + } catch (IOException ioe) { + LOG.fatal("Error reading key writables", ioe); + watcher.abort("Error reading key writables from znode "+path, ioe); + } + } + } + + @Override + public void nodeChildrenChanged(String path) { + if (path.equals(keysParentZNode)) { + // keys changed + try { + List nodes = + ZKUtil.getChildDataAndWatchForNewChildren(watcher, keysParentZNode); + refreshNodes(nodes); + } catch (KeeperException ke) { + LOG.fatal("Error reading data from zookeeper", ke); + watcher.abort("Error reading changed keys from zookeeper", ke); + } + } + } + + public String getRootKeyZNode() { + return baseKeyZNode; + } + + private void refreshNodes(List nodes) { + for (ZKUtil.NodeAndData n : nodes) { + String path = n.getNode(); + String keyId = ZKUtil.getNodeName(path); + try { + byte[] data = n.getData(); + if (data == null || data.length == 0) { + LOG.debug("Ignoring empty node "+path); + continue; + } + AuthenticationKey key = (AuthenticationKey)Writables.getWritable( + data, new AuthenticationKey()); + secretManager.addKey(key); + } catch (IOException ioe) { + LOG.fatal("Failed reading new secret key for id '" + keyId + + "' from zk", ioe); + watcher.abort("Error deserializing key from znode "+path, ioe); + } + } + } + + private String getKeyNode(int keyId) { + return ZKUtil.joinZNode(keysParentZNode, Integer.toString(keyId)); + } + + public void removeKeyFromZK(AuthenticationKey key) { + String keyZNode = getKeyNode(key.getKeyId()); + try { + ZKUtil.deleteNode(watcher, keyZNode); + } catch (KeeperException.NoNodeException nne) { + LOG.error("Non-existent znode "+keyZNode+" for key "+key.getKeyId(), nne); + } catch (KeeperException ke) { + LOG.fatal("Failed removing znode "+keyZNode+" for key "+key.getKeyId(), + ke); + watcher.abort("Unhandled zookeeper error removing znode "+keyZNode+ + " for key "+key.getKeyId(), ke); + } + } + + public void addKeyToZK(AuthenticationKey key) { + String keyZNode = getKeyNode(key.getKeyId()); + try { + byte[] keyData = Writables.getBytes(key); + // TODO: is there any point in retrying beyond what ZK client does? + ZKUtil.createSetData(watcher, keyZNode, keyData); + } catch (KeeperException ke) { + LOG.fatal("Unable to synchronize master key "+key.getKeyId()+ + " to znode "+keyZNode, ke); + watcher.abort("Unable to synchronize secret key "+ + key.getKeyId()+" in zookeeper", ke); + } catch (IOException ioe) { + // this can only happen from an error serializing the key + watcher.abort("Failed serializing key "+key.getKeyId(), ioe); + } + } + + public void updateKeyInZK(AuthenticationKey key) { + String keyZNode = getKeyNode(key.getKeyId()); + try { + byte[] keyData = Writables.getBytes(key); + try { + ZKUtil.updateExistingNodeData(watcher, keyZNode, keyData, -1); + } catch (KeeperException.NoNodeException ne) { + // node was somehow removed, try adding it back + ZKUtil.createSetData(watcher, keyZNode, keyData); + } + } catch (KeeperException ke) { + LOG.fatal("Unable to update master key "+key.getKeyId()+ + " in znode "+keyZNode); + watcher.abort("Unable to synchronize secret key "+ + key.getKeyId()+" in zookeeper", ke); + } catch (IOException ioe) { + // this can only happen from an error serializing the key + watcher.abort("Failed serializing key "+key.getKeyId(), ioe); + } + } +} diff --git a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java new file mode 100644 index 0000000..03ec4c5 --- /dev/null +++ b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFiles.java @@ -0,0 +1,56 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.security.access.AccessControlLists; +import org.apache.hadoop.hbase.security.access.SecureTestUtil; + +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +/** + * Reruns TestLoadIncrementalHFiles using LoadIncrementalHFiles using secure mode. + * This suite is unable to verify the security handoff/turnover + * as miniCluster is running as system user thus has root privileges + * and delegation tokens don't seem to work on miniDFS. + * + * Thus SecureBulkload can only be completely verified by running + * integration tests against a secure cluster. This suite is still + * invaluable as it verifies the other mechanisms that need to be + * supported as part of a LoadIncrementalFiles call. + */ +@Category(LargeTests.class) +public class TestSecureLoadIncrementalHFiles extends TestLoadIncrementalHFiles{ + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + useSecure = true; + // setup configuration + SecureTestUtil.enableSecurity(util.getConfiguration()); + + util.startMiniCluster(); + + // Wait for the ACL table to become available + util.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000); + } + +} + diff --git a/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java new file mode 100644 index 0000000..e8593b2 --- /dev/null +++ b/security/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSecureLoadIncrementalHFilesSplitRecovery.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.security.access.AccessControlLists; +import org.apache.hadoop.hbase.security.access.SecureTestUtil; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + + +/** + * Reruns TestSecureLoadIncrementalHFilesSplitRecovery + * using LoadIncrementalHFiles in secure mode. + * This suite is unable to verify the security handoff/turnover + * as miniCluster is running as system user thus has root privileges + * and delegation tokens don't seem to work on miniDFS. + * + * Thus SecureBulkload can only be completely verified by running + * integration tests against a secure cluster. This suite is still + * invaluable as it verifies the other mechanisms that need to be + * supported as part of a LoadIncrementalFiles call. + */ +@Category(LargeTests.class) +public class TestSecureLoadIncrementalHFilesSplitRecovery extends TestLoadIncrementalHFilesSplitRecovery { + + //This "overrides" the parent static method + //make sure they are in sync + @BeforeClass + public static void setupCluster() throws Exception { + useSecure = true; + util = new HBaseTestingUtility(); + // setup configuration + SecureTestUtil.enableSecurity(util.getConfiguration()); + + util.startMiniCluster(); + + // Wait for the ACL table to become available + util.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000); + } + + //Disabling this test as it does not work in secure mode + @Test + @Override + public void testBulkLoadPhaseFailure() { + } +} + diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java new file mode 100644 index 0000000..5d55760 --- /dev/null +++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.ipc.SecureRpcEngine; +import org.apache.hadoop.hbase.security.User; + +/** + * Utility methods for testing security + */ +public class SecureTestUtil { + public static void enableSecurity(Configuration conf) throws IOException { + conf.set("hadoop.security.authorization", "false"); + conf.set("hadoop.security.authentication", "simple"); + conf.set("hbase.rpc.engine", SecureRpcEngine.class.getName()); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, AccessController.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, AccessController.class.getName()+ + ","+SecureBulkLoadEndpoint.class.getName()); + conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, AccessController.class.getName()); + // add the process running user to superusers + String currentUser = User.getCurrent().getName(); + conf.set("hbase.superuser", "admin,"+currentUser); + } +} diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestACLWithIndexTable.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestACLWithIndexTable.java new file mode 100644 index 0000000..ebec487 --- /dev/null +++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestACLWithIndexTable.java @@ -0,0 +1,894 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.access; + +import java.security.PrivilegedExceptionAction; +import java.util.List; + +import junit.framework.Assert; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType; +import org.apache.hadoop.hbase.index.Constants; +import org.apache.hadoop.hbase.index.IndexSpecification; +import org.apache.hadoop.hbase.index.IndexedHTableDescriptor; +import org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver; +import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionObserver; +import org.apache.hadoop.hbase.index.coprocessor.wal.IndexWALObserver; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestACLWithIndexTable { + + private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static Configuration conf = TEST_UTIL.getConfiguration(); + // user with all permissions + private static User SUPERUSER; + private static User TEST_USER; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + SecureTestUtil.enableSecurity(conf); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, AccessController.class.getName() + "," + + IndexMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, AccessController.class.getName() + "," + + SecureBulkLoadEndpoint.class.getName() + "," + IndexRegionObserver.class.getName()); + conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, AccessController.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); + conf.setInt("hbase.regionserver.lease.period", 10 * 60 * 1000); + conf.setInt("hbase.rpc.timeout", 10 * 60 * 1000); + conf.setBoolean("hbase.use.secondary.index", true); + TEST_UTIL.startMiniCluster(2); + + TEST_UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000); + // create a set of test users + SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); + TEST_USER = User.createUserForTesting(conf, "testUser", new String[0]); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testCreateTable() throws Exception { + // initilize access control + HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol protocol = + meta.coprocessorProxy(AccessControllerProtocol.class, Bytes.toBytes("testCreateTable")); + protocol.grant(new UserPermission(Bytes.toBytes(TEST_USER.getShortName()), Bytes + .toBytes("testCreateTable"), null, Permission.Action.ADMIN)); + + PrivilegedExceptionAction createTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testCreateTable"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + return null; + } + }; + + PrivilegedExceptionAction createIndexTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testCreateTable_idx"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + return null; + } + }; + + try { + TEST_USER.runAs(createIndexTable); + Assert.fail("Should throw exception"); + } catch (Exception e) { + + } + + try { + TEST_USER.runAs(createTable); + Assert.fail("Should throw exception"); + } catch (Exception e) { + + } + + try { + SUPERUSER.runAs(createTable); + } catch (Exception e) { + Assert.fail("Should not throw any exception.Super user should be allowed to create tables"); + } + + } + + @Test(timeout = 180000) + public void testDisableEnableTable() throws Exception { + // initilize access control + HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol protocol = + meta.coprocessorProxy(AccessControllerProtocol.class, + Bytes.toBytes("testDisableEnableTable")); + protocol.grant(new UserPermission(Bytes.toBytes(TEST_USER.getShortName()), Bytes + .toBytes("testDisableEnableTable"), null, Permission.Action.ADMIN)); + + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testDisableEnableTable"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + + // Creating the operation to be performed by the user + PrivilegedExceptionAction disableTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + admin.disableTable(Bytes.toBytes("testDisableEnableTable")); + return null; + } + }; + + PrivilegedExceptionAction disableIndexTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + admin.disableTable("testDisableEnableTable_idx"); + return null; + } + }; + PrivilegedExceptionAction enableTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + admin.enableTable(Bytes.toBytes("testDisableEnableTable")); + return null; + } + }; + PrivilegedExceptionAction enableIndexTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + admin.enableTable(Bytes.toBytes("testDisableEnableTable_idx")); + return null; + } + }; + + // Execute all operations one by one + try { + TEST_USER.runAs(disableIndexTable); + Assert.fail("Should throw exception"); + } catch (Exception e) { + + } + Assert.assertTrue("Index table should be enabled", + admin.isTableEnabled("testDisableEnableTable_idx")); + + try { + TEST_USER.runAs(disableTable); + } catch (Exception e) { + Assert.fail("Should not throw any exception"); + } + while (!admin.isTableDisabled("testDisableEnableTable")) { + Thread.sleep(10); + } + while (!admin.isTableDisabled("testDisableEnableTable_idx")) { + Thread.sleep(10); + } + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + Assert.assertTrue("Main table should be disabled", + admin.isTableDisabled("testDisableEnableTable")); + Assert.assertTrue("Index table should be disabled", + admin.isTableDisabled("testDisableEnableTable_idx")); + + try { + TEST_USER.runAs(enableIndexTable); + Assert.fail("Index table should not get enabled. It should throw exception"); + } catch (Exception e) { + + } + Assert.assertTrue("Index table should be disabled", + admin.isTableDisabled("testDisableEnableTable_idx")); + try { + TEST_USER.runAs(enableTable); + } catch (Exception e) { + Assert.fail("Should not throw any exception."); + } + while (!admin.isTableEnabled("testDisableEnableTable")) { + Thread.sleep(10); + } + while (!admin.isTableEnabled("testDisableEnableTable_idx")) { + Thread.sleep(10); + } + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + + Assert.assertTrue("Main table should be enabled", + admin.isTableEnabled("testDisableEnableTable")); + Assert.assertTrue("Index table should be enabled", + admin.isTableEnabled("testDisableEnableTable_idx")); + + } + + @Test(timeout = 180000) + public void testScanOperation() throws Exception { + // initilize access control + HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol protocol = + meta.coprocessorProxy(AccessControllerProtocol.class, Bytes.toBytes("testScanOperation")); + protocol.grant(new UserPermission(Bytes.toBytes(TEST_USER.getShortName()), Bytes + .toBytes("testScanOperation"), null, Permission.Action.READ)); + + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testScanOperation"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + + HTable table = new HTable(conf, "testScanOperation"); + Put p = null; + for (int i = 0; i < 10; i++) { + p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("q"), Bytes.toBytes("Test_val")); + table.put(p); + } + + PrivilegedExceptionAction scanTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HTable table = new HTable(conf, "testScanOperation"); + SingleColumnValueFilter svf = + new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("q"), CompareOp.EQUAL, + Bytes.toBytes("Test_val")); + Scan s = new Scan(); + s.setFilter(svf); + ResultScanner scanner = null; + try { + scanner = table.getScanner(s); + Result result = scanner.next(); + while (result != null) { + result = scanner.next(); + } + } finally { + if (scanner != null) { + scanner.close(); + } + } + return null; + } + }; + + PrivilegedExceptionAction scanIndexTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HTable table = new HTable(conf, "testScanOperation_idx"); + Scan s = new Scan(); + ResultScanner scanner = null; + try { + scanner = table.getScanner(s); + Result result = scanner.next(); + while (result != null) { + result = scanner.next(); + } + } finally { + if (scanner != null) { + scanner.close(); + } + } + return null; + } + }; + + try { + TEST_USER.runAs(scanIndexTable); + Assert.fail("Should throw exception"); + } catch (Exception e) { + + } + + try { + TEST_USER.runAs(scanTable); + } catch (Exception e) { + Assert.fail("Should not throw any exception."); + } + } + + @Test(timeout = 180000) + public void testCheckAndPut() throws Exception { + HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol protocol = + meta.coprocessorProxy(AccessControllerProtocol.class, Bytes.toBytes("testCheckAndPut")); + protocol.grant(new UserPermission(Bytes.toBytes(TEST_USER.getShortName()), Bytes + .toBytes("testCheckAndPut"), null, Permission.Action.WRITE, Permission.Action.READ)); + + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testCheckAndPut"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + final IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + + HTable table = new HTable(conf, "testCheckAndPut"); + Put p = null; + for (int i = 0; i < 10; i++) { + p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("q"), Bytes.toBytes("Test_val")); + table.put(p); + } + + HTable indexTable = new HTable(conf, "testCheckAndPut_idx"); + p = new Put(Bytes.toBytes("row" + 0)); + p.add(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, Bytes.toBytes("old_val")); + indexTable.put(p); + + PrivilegedExceptionAction putInIndexTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HTable indexTable2 = new HTable(conf, "testCheckAndPut_idx"); + Put put = new Put(Bytes.toBytes("row" + 0)); + put.add(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, Bytes.toBytes("latest")); + indexTable2.checkAndPut(Bytes.toBytes("row" + 0), Constants.IDX_COL_FAMILY, + Constants.IDX_COL_QUAL, Bytes.toBytes("old_val"), put); + return null; + } + }; + + PrivilegedExceptionAction putInMainTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HTable table2 = new HTable(conf, "testCheckAndPut"); + Put put = new Put(Bytes.toBytes("row" + 0)); + put.add(Bytes.toBytes("cf"), Bytes.toBytes("q"), Bytes.toBytes("latest")); + table2.checkAndPut(Bytes.toBytes("row" + 0), Bytes.toBytes("cf"), Bytes.toBytes("q"), + Bytes.toBytes("Test_val"), put); + return null; + } + }; + + try { + TEST_USER.runAs(putInIndexTable); + Assert.fail("Should throw exception."); + } catch (Exception e) { + + } + + try { + TEST_USER.runAs(putInMainTable); + } catch (Exception e) { + Assert.fail("Should not throw any exception."); + } + } + + @Test(timeout = 180000) + public void testMoveRegionOp() throws Exception { + HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol protocol = + meta.coprocessorProxy(AccessControllerProtocol.class, Bytes.toBytes("testMoveRegionOp")); + protocol.grant(new UserPermission(Bytes.toBytes(TEST_USER.getShortName()), Bytes + .toBytes("testMoveRegionOp"), null, Permission.Action.ADMIN)); + + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testMoveRegionOp"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + + PrivilegedExceptionAction moveMainTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + List tableRegions = admin.getTableRegions(Bytes.toBytes("testMoveRegionOp")); + if (tableRegions.size() > 0) { + HRegionInfo regionMoved = tableRegions.get(0); + admin.move(regionMoved.getEncodedNameAsBytes(), null); + } + return null; + } + }; + + PrivilegedExceptionAction moveIndexTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + List tableRegions = + admin.getTableRegions(Bytes.toBytes("testMoveRegionOp_idx")); + if (tableRegions.size() > 0) { + HRegionInfo regionMoved = tableRegions.get(0); + admin.move(regionMoved.getEncodedNameAsBytes(), null); + } + return null; + } + }; + + try { + TEST_USER.runAs(moveIndexTable); + Assert.fail("Should throw exception."); + } catch (Exception e) { + + } + + try { + TEST_USER.runAs(moveMainTable); + } catch (Exception e) { + Assert.fail("should not throw any exception"); + } + } + + @Test(timeout = 180000) + public void testUnassignOperation() throws Exception { + HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol protocol = + meta.coprocessorProxy(AccessControllerProtocol.class, + Bytes.toBytes("testUnassignOperation")); + protocol.grant(new UserPermission(Bytes.toBytes(TEST_USER.getShortName()), Bytes + .toBytes("testUnassignOperation"), null, Permission.Action.ADMIN)); + + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testUnassignOperation"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + + PrivilegedExceptionAction uassignMainRegion = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + List tableRegions = + admin.getTableRegions(Bytes.toBytes("testUnassignOperation")); + if (tableRegions.size() > 0) { + HRegionInfo regionToBeUassigned = tableRegions.get(0); + admin.unassign(regionToBeUassigned.getRegionName(), false); + } + return null; + } + }; + + PrivilegedExceptionAction uassignIndexRegion = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + List tableRegions = + admin.getTableRegions(Bytes.toBytes("testUnassignOperation_idx")); + if (tableRegions.size() > 0) { + HRegionInfo regionToBeUassigned = tableRegions.get(0); + admin.unassign(regionToBeUassigned.getRegionName(), false); + } + return null; + } + }; + + try { + TEST_USER.runAs(uassignIndexRegion); + Assert.fail("Should throw exception"); + } catch (Exception e) { + + } + + try { + TEST_USER.runAs(uassignMainRegion); + } catch (Exception e) { + Assert.fail("Should not throw any exception"); + } + } + + @Test(timeout = 180000) + public void testSplitOp() throws Exception { + HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol protocol = + meta.coprocessorProxy(AccessControllerProtocol.class, Bytes.toBytes("testSplitOp")); + protocol.grant(new UserPermission(Bytes.toBytes(TEST_USER.getShortName()), Bytes + .toBytes("testSplitOp"), null, Permission.Action.ADMIN)); + + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testSplitOp"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + + HTable table = new HTable(conf, "testSplitOp"); + Put p = null; + for (int i = 0; i < 10; i++) { + p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("q"), Bytes.toBytes("test_val")); + table.put(p); + } + + PrivilegedExceptionAction splitIndexRegion = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + admin.split("testSplitOp_idx"); + return null; + } + }; + + PrivilegedExceptionAction splitMainRegion = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + admin.split("testSplitOp"); + return null; + } + }; + + try { + TEST_USER.runAs(splitIndexRegion); + Assert.fail("Should throw exception."); + } catch (Exception e) { + + } + + try { + TEST_USER.runAs(splitMainRegion); + } catch (Exception e) { + Assert.fail("Should not throw any exception"); + } + } + + @Test(timeout = 180000) + public void testCompactionOp() throws Exception { + HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol protocol = + meta.coprocessorProxy(AccessControllerProtocol.class, Bytes.toBytes("testCompactionOp")); + protocol.grant(new UserPermission(Bytes.toBytes(TEST_USER.getShortName()), Bytes + .toBytes("testCompactionOp"), null, Permission.Action.ADMIN)); + + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testCompactionOp"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + + HTable table = new HTable(conf, "testCompactionOp"); + Put p = null; + for (int i = 0; i < 100; i++) { + p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("q"), Bytes.toBytes("test_val")); + table.put(p); + if (i % 10 == 0) { + admin.flush("testCompactionOp"); + admin.flush("testCompactionOp_idx"); + } + } + + PrivilegedExceptionAction compactIndexTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + admin.compact("testCompactionOp_idx"); + return null; + } + }; + + PrivilegedExceptionAction compactMainTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + admin.compact("testCompactionOp"); + return null; + } + }; + + try { + TEST_USER.runAs(compactIndexTable); + Assert.fail("Should throw exception"); + } catch (Exception e) { + } + + try { + TEST_USER.runAs(compactMainTable); + } catch (Exception e) { + Assert.fail("Should not throw any exception"); + } + } + + @Test(timeout = 180000) + public void testDeleteTable() throws Exception { + HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol protocol = + meta.coprocessorProxy(AccessControllerProtocol.class, Bytes.toBytes("testDeleteTable")); + protocol.grant(new UserPermission(Bytes.toBytes(TEST_USER.getShortName()), Bytes + .toBytes("testDeleteTable"), null, Permission.Action.ADMIN, Permission.Action.CREATE)); + + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testDeleteTable"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + + admin.disableTable("testDeleteTable"); + while (!admin.isTableDisabled("testDeleteTable")) { + Thread.sleep(10); + } + while (!admin.isTableDisabled("testDeleteTable_idx")) { + Thread.sleep(10); + } + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + + PrivilegedExceptionAction deleteIndexTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin2 = new HBaseAdmin(conf); + admin2.deleteTable(Bytes.toBytes("testDeleteTable_idx")); + return null; + } + }; + + PrivilegedExceptionAction deleteMainTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin2 = new HBaseAdmin(conf); + admin2.deleteTable(Bytes.toBytes("testDeleteTable")); + return null; + } + }; + + try { + TEST_USER.runAs(deleteIndexTable); + Assert.fail("Should throw exception"); + } catch (Exception e) { + + } + + try { + TEST_USER.runAs(deleteMainTable); + } catch (Exception e) { + Assert.fail("Should not throw any exception"); + } + } + + @Test(timeout = 180000) + public void testflushOp() throws Exception { + HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol protocol = + meta.coprocessorProxy(AccessControllerProtocol.class, Bytes.toBytes("testflushOp")); + protocol.grant(new UserPermission(Bytes.toBytes(TEST_USER.getShortName()), Bytes + .toBytes("testflushOp"), null, Permission.Action.ADMIN, Permission.Action.CREATE)); + + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testflushOp"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + + HTable table = new HTable(conf, "testflushOp"); + Put p = null; + for (int i = 0; i < 100; i++) { + p = new Put(Bytes.toBytes("row" + i)); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("q"), Bytes.toBytes("test_val")); + table.put(p); + } + + PrivilegedExceptionAction flushIndexRegion = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + admin.flush("testflushOp_idx"); + return null; + } + }; + + PrivilegedExceptionAction flushMainRegion = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + admin.flush("testflushOp"); + return null; + } + }; + + try { + TEST_USER.runAs(flushIndexRegion); + Assert.fail("Should throw exception."); + } catch (Exception e) { + + } + + try { + TEST_USER.runAs(flushMainRegion); + } catch (Exception e) { + Assert.fail("Should not throw any exception."); + } + } + + @Test(timeout = 180000) + public void testModifyTable() throws Exception { + HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol protocol = + meta.coprocessorProxy(AccessControllerProtocol.class, Bytes.toBytes("testModifyTable")); + protocol.grant(new UserPermission(Bytes.toBytes(TEST_USER.getShortName()), Bytes + .toBytes("testModifyTable"), null, Permission.Action.ADMIN, Permission.Action.CREATE)); + + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testModifyTable"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + admin.disableTable("testModifyTable"); + + PrivilegedExceptionAction modifyIndexTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + HTableDescriptor htd = new HTableDescriptor("testModifyTable_idx"); + htd.addFamily(new HColumnDescriptor("d")); + htd.addFamily(new HColumnDescriptor("d1")); + admin.modifyTable(Bytes.toBytes("testModifyTable_idx"), htd); + return null; + } + }; + + PrivilegedExceptionAction modifyMainTable = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor("testModifyTable"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + HColumnDescriptor hcd1 = new HColumnDescriptor("cf1"); + ihtd.addFamily(hcd); + ihtd.addFamily(hcd1); + IndexSpecification iSpec = new IndexSpecification("spec"); + IndexSpecification iSpec1 = new IndexSpecification("spec1"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + iSpec1.addIndexColumn(hcd1, "q", ValueType.String, 10); + ihtd.addIndex(iSpec); + ihtd.addIndex(iSpec1); + admin.modifyTable(Bytes.toBytes("testModifyTable"), ihtd); + return null; + } + }; + + try { + TEST_USER.runAs(modifyIndexTable); + Assert.fail("Should throw exception."); + } catch (Exception e) { + + } + + try { + TEST_USER.runAs(modifyMainTable); + } catch (Exception e) { + Assert.fail("Should not throw any exception"); + } + } + + @Test(timeout = 180000) + public void testModifyFamily() throws Exception { + HTable meta = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol protocol = + meta.coprocessorProxy(AccessControllerProtocol.class, Bytes.toBytes("testModifyFamily")); + protocol.grant(new UserPermission(Bytes.toBytes(TEST_USER.getShortName()), Bytes + .toBytes("testModifyFamily"), null, Permission.Action.ADMIN, Permission.Action.CREATE)); + + HBaseAdmin admin = new HBaseAdmin(conf); + IndexedHTableDescriptor htd = new IndexedHTableDescriptor("testModifyFamily"); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + htd.addFamily(hcd); + IndexSpecification iSpec = new IndexSpecification("spec"); + iSpec.addIndexColumn(hcd, "q", ValueType.String, 10); + htd.addIndex(iSpec); + admin.createTable(htd); + ZKAssign.blockUntilNoRIT(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL)); + admin.disableTable("testModifyFamily"); + + PrivilegedExceptionAction modifyIndexFamily = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + HColumnDescriptor hcd = new HColumnDescriptor("d"); + hcd.setMaxVersions(4); + admin.modifyColumn("testModifyFamily_idx", hcd); + return null; + } + }; + + PrivilegedExceptionAction modifyMainFamily = new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + HColumnDescriptor hcd = new HColumnDescriptor("cf"); + hcd.setMaxVersions(4); + admin.modifyColumn("testModifyFamily", hcd); + return null; + } + }; + + try { + TEST_USER.runAs(modifyIndexFamily); + Assert.fail("Should throw exception"); + } catch (Exception e) { + + } + + try { + TEST_USER.runAs(modifyMainFamily); + } catch (Exception e) { + Assert.fail("Should not throw any exception"); + } + } + +} diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java new file mode 100644 index 0000000..bd423f1 --- /dev/null +++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.security.AccessDeniedException; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestAccessControlFilter { + private static Log LOG = LogFactory.getLog(TestAccessControlFilter.class); + private static HBaseTestingUtility TEST_UTIL; + + private static User ADMIN; + private static User READER; + private static User LIMITED; + private static User DENIED; + + private static byte[] TABLE = Bytes.toBytes("testtable"); + private static byte[] FAMILY = Bytes.toBytes("f1"); + private static byte[] PRIVATE_COL = Bytes.toBytes("private"); + private static byte[] PUBLIC_COL = Bytes.toBytes("public"); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + TEST_UTIL = new HBaseTestingUtility(); + Configuration conf = TEST_UTIL.getConfiguration(); + SecureTestUtil.enableSecurity(conf); + String baseuser = User.getCurrent().getShortName(); + conf.set("hbase.superuser", conf.get("hbase.superuser", "") + + String.format(",%s.hfs.0,%s.hfs.1,%s.hfs.2", baseuser, baseuser, baseuser)); + TEST_UTIL.startMiniCluster(); + TEST_UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000); + + ADMIN = User.createUserForTesting(conf, "admin", new String[]{"supergroup"}); + READER = User.createUserForTesting(conf, "reader", new String[0]); + LIMITED = User.createUserForTesting(conf, "limited", new String[0]); + DENIED = User.createUserForTesting(conf, "denied", new String[0]); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testQualifierAccess() throws Exception { + final HTable table = TEST_UTIL.createTable(TABLE, FAMILY); + + // set permissions + ADMIN.runAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HTable aclmeta = new HTable(TEST_UTIL.getConfiguration(), + AccessControlLists.ACL_TABLE_NAME); + AccessControllerProtocol acls = aclmeta.coprocessorProxy( + AccessControllerProtocol.class, Bytes.toBytes("testtable")); + UserPermission perm = new UserPermission(Bytes.toBytes(READER.getShortName()), + TABLE, null, Permission.Action.READ); + acls.grant(perm); + perm = new UserPermission(Bytes.toBytes(LIMITED.getShortName()), + TABLE, FAMILY, PUBLIC_COL, Permission.Action.READ); + acls.grant(perm); + return null; + } + }); + + // put some test data + List puts = new ArrayList(100); + for (int i=0; i<100; i++) { + Put p = new Put(Bytes.toBytes(i)); + p.add(FAMILY, PRIVATE_COL, Bytes.toBytes("secret "+i)); + p.add(FAMILY, PUBLIC_COL, Bytes.toBytes("info "+i)); + puts.add(p); + } + table.put(puts); + + // test read + READER.runAs(new PrivilegedExceptionAction() { + public Object run() throws Exception { + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + // force a new RS connection + conf.set("testkey", UUID.randomUUID().toString()); + HTable t = new HTable(conf, TABLE); + ResultScanner rs = t.getScanner(new Scan()); + int rowcnt = 0; + for (Result r : rs) { + rowcnt++; + int rownum = Bytes.toInt(r.getRow()); + assertTrue(r.containsColumn(FAMILY, PRIVATE_COL)); + assertEquals("secret "+rownum, Bytes.toString(r.getValue(FAMILY, PRIVATE_COL))); + assertTrue(r.containsColumn(FAMILY, PUBLIC_COL)); + assertEquals("info "+rownum, Bytes.toString(r.getValue(FAMILY, PUBLIC_COL))); + } + assertEquals("Expected 100 rows returned", 100, rowcnt); + return null; + } + }); + + // test read with qualifier filter + LIMITED.runAs(new PrivilegedExceptionAction() { + public Object run() throws Exception { + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + // force a new RS connection + conf.set("testkey", UUID.randomUUID().toString()); + HTable t = new HTable(conf, TABLE); + ResultScanner rs = t.getScanner(new Scan()); + int rowcnt = 0; + for (Result r : rs) { + rowcnt++; + int rownum = Bytes.toInt(r.getRow()); + assertFalse(r.containsColumn(FAMILY, PRIVATE_COL)); + assertTrue(r.containsColumn(FAMILY, PUBLIC_COL)); + assertEquals("info " + rownum, Bytes.toString(r.getValue(FAMILY, PUBLIC_COL))); + } + assertEquals("Expected 100 rows returned", 100, rowcnt); + return null; + } + }); + + // test as user with no permission + DENIED.runAs(new PrivilegedExceptionAction(){ + public Object run() throws Exception { + try { + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + // force a new RS connection + conf.set("testkey", UUID.randomUUID().toString()); + HTable t = new HTable(conf, TABLE); + ResultScanner rs = t.getScanner(new Scan()); + fail("Attempt to open scanner should have been denied"); + } catch (AccessDeniedException ade) { + // expected + } + return null; + } + }); + } +} diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java new file mode 100644 index 0000000..212c14f --- /dev/null +++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java @@ -0,0 +1,1872 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.UnknownRowLockException; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorException; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; +import org.apache.hadoop.hbase.master.MasterCoprocessorHost; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; +import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost; +import org.apache.hadoop.hbase.security.AccessDeniedException; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.access.AccessControlLists; +import org.apache.hadoop.hbase.security.access.AccessControllerProtocol; +import org.apache.hadoop.hbase.security.access.Permission; +import org.apache.hadoop.hbase.security.access.UserPermission; +import org.apache.hadoop.hbase.security.access.Permission.Action; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Performs authorization checks for common operations, according to different + * levels of authorized users. + */ +@Category(LargeTests.class) +@SuppressWarnings("rawtypes") +public class TestAccessController { + private static final Log LOG = LogFactory.getLog(TestAccessController.class); + private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static Configuration conf; + + // user with all permissions + private static User SUPERUSER; + // user granted with all global permission + private static User USER_ADMIN; + // user with rw permissions + private static User USER_RW; + // user with rw permissions on table. + private static User USER_RW_ON_TABLE; + // user with read-only permissions + private static User USER_RO; + // user is table owner. will have all permissions on table + private static User USER_OWNER; + // user with create table permissions alone + private static User USER_CREATE; + // user with no permissions + private static User USER_NONE; + + private static byte[] TEST_TABLE = Bytes.toBytes("testtable"); + private static byte[] TEST_TABLE2 = Bytes.toBytes("testtable2"); + private static byte[] TEST_FAMILY = Bytes.toBytes("f1"); + + private static MasterCoprocessorEnvironment CP_ENV; + private static RegionCoprocessorEnvironment RCP_ENV; + private static RegionServerCoprocessorEnvironment RSCP_ENV; + private static AccessController ACCESS_CONTROLLER; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // setup configuration + conf = TEST_UTIL.getConfiguration(); + conf.set("hbase.master.hfilecleaner.plugins", + "org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner," + + "org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner"); + conf.set("hbase.master.logcleaner.plugins", + "org.apache.hadoop.hbase.master.snapshot.SnapshotLogCleaner"); + SecureTestUtil.enableSecurity(conf); + + TEST_UTIL.startMiniCluster(); + MasterCoprocessorHost cpHost = TEST_UTIL.getMiniHBaseCluster().getMaster().getCoprocessorHost(); + cpHost.load(AccessController.class, Coprocessor.PRIORITY_HIGHEST, conf); + ACCESS_CONTROLLER = (AccessController) cpHost.findCoprocessor(AccessController.class.getName()); + CP_ENV = cpHost.createEnvironment(AccessController.class, ACCESS_CONTROLLER, + Coprocessor.PRIORITY_HIGHEST, 1, conf); + RegionServerCoprocessorHost rsHost = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0) + .getCoprocessorHost(); + RSCP_ENV = rsHost.createEnvironment(AccessController.class, ACCESS_CONTROLLER, + Coprocessor.PRIORITY_HIGHEST, 1, conf); + + // Wait for the ACL table to become available + TEST_UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000); + + // create a set of test users + SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); + USER_ADMIN = User.createUserForTesting(conf, "admin2", new String[0]); + USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]); + USER_RO = User.createUserForTesting(conf, "rouser", new String[0]); + USER_RW_ON_TABLE = User.createUserForTesting(conf, "rwuser_1", new String[0]); + USER_OWNER = User.createUserForTesting(conf, "owner", new String[0]); + USER_CREATE = User.createUserForTesting(conf, "tbl_create", new String[0]); + USER_NONE = User.createUserForTesting(conf, "nouser", new String[0]); + + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + HTableDescriptor htd = new HTableDescriptor(TEST_TABLE); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + htd.setOwner(USER_OWNER); + admin.createTable(htd); + TEST_UTIL.waitTableEnabled(TEST_TABLE, 5000); + + HRegion region = TEST_UTIL.getHBaseCluster().getRegions(TEST_TABLE).get(0); + RegionCoprocessorHost rcpHost = region.getCoprocessorHost(); + RCP_ENV = rcpHost.createEnvironment(AccessController.class, ACCESS_CONTROLLER, + Coprocessor.PRIORITY_HIGHEST, 1, conf); + + // initilize access control + HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + TEST_TABLE); + + protocol.grant(new UserPermission(Bytes.toBytes(USER_ADMIN.getShortName()), + Permission.Action.ADMIN, Permission.Action.CREATE, Permission.Action.READ, + Permission.Action.WRITE)); + + protocol.grant(new UserPermission(Bytes.toBytes(USER_RW.getShortName()), TEST_TABLE, + TEST_FAMILY, Permission.Action.READ, Permission.Action.WRITE)); + + protocol.grant(new UserPermission(Bytes.toBytes(USER_RO.getShortName()), TEST_TABLE, + TEST_FAMILY, Permission.Action.READ)); + + protocol.grant(new UserPermission(Bytes.toBytes(USER_CREATE.getShortName()), TEST_TABLE, null, + Permission.Action.CREATE)); + + protocol.grant(new UserPermission(Bytes.toBytes(USER_RW_ON_TABLE.getShortName()), TEST_TABLE, + null, Permission.Action.READ, Permission.Action.WRITE)); + } finally { + acl.close(); + } + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + public void verifyAllowed(User user, PrivilegedExceptionAction... actions) throws Exception { + for (PrivilegedExceptionAction action : actions) { + try { + user.runAs(action); + } catch (AccessDeniedException ade) { + fail("Expected action to pass for user '" + user.getShortName() + "' but was denied"); + } catch (UnknownRowLockException exp){ + //expected + } + } + } + + public void verifyAllowed(PrivilegedExceptionAction action, User... users) throws Exception { + for (User user : users) { + verifyAllowed(user, action); + } + } + + public void verifyDenied(User user, PrivilegedExceptionAction... actions) throws Exception { + for (PrivilegedExceptionAction action : actions) { + try { + user.runAs(action); + fail("Expected AccessDeniedException for user '" + user.getShortName() + "'"); + } catch (AccessDeniedException ade) { + // expected result + } catch (IOException e) { + boolean isAccessDeniedException = false; + if(e instanceof RetriesExhaustedWithDetailsException) { + // in case of batch operations, and put, the client assembles a + // RetriesExhaustedWithDetailsException instead of throwing an + // AccessDeniedException + for(Throwable ex : ((RetriesExhaustedWithDetailsException) e).getCauses()) { + if (ex instanceof AccessDeniedException) { + isAccessDeniedException = true; + break; + } + } + } + else { + // For doBulkLoad calls AccessDeniedException + // is buried in the stack trace + Throwable ex = e; + do { + if (ex instanceof AccessDeniedException) { + isAccessDeniedException = true; + break; + } + } while((ex = ex.getCause()) != null); + } + if (!isAccessDeniedException) { + fail("Not receiving AccessDeniedException for user '" + user.getShortName() + "'"); + } + } + } + } + + public void verifyDenied(PrivilegedExceptionAction action, User... users) throws Exception { + for (User user : users) { + verifyDenied(user, action); + } + } + + @Test + public void testTableCreate() throws Exception { + PrivilegedExceptionAction createTable = new PrivilegedExceptionAction() { + public Object run() throws Exception { + HTableDescriptor htd = new HTableDescriptor("testnewtable"); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + ACCESS_CONTROLLER.preCreateTable(ObserverContext.createAndPrepare(CP_ENV, null), htd, null); + return null; + } + }; + + // verify that superuser can create tables + verifyAllowed(createTable, SUPERUSER, USER_ADMIN); + + // all others should be denied + verifyDenied(createTable, USER_CREATE, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testTableModify() throws Exception { + PrivilegedExceptionAction modifyTable = new PrivilegedExceptionAction() { + public Object run() throws Exception { + HTableDescriptor htd = new HTableDescriptor(TEST_TABLE); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + htd.addFamily(new HColumnDescriptor("fam_" + User.getCurrent().getShortName())); + ACCESS_CONTROLLER.preModifyTable(ObserverContext.createAndPrepare(CP_ENV, null), + TEST_TABLE, htd); + return null; + } + }; + + verifyAllowed(modifyTable, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER); + verifyDenied(modifyTable, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testTableDelete() throws Exception { + PrivilegedExceptionAction deleteTable = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER + .preDeleteTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE); + return null; + } + }; + + verifyAllowed(deleteTable, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER); + verifyDenied(deleteTable, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testAddColumn() throws Exception { + final HColumnDescriptor hcd = new HColumnDescriptor("fam_new"); + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preAddColumn(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE, + hcd); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER); + verifyDenied(action, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testModifyColumn() throws Exception { + final HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY); + hcd.setMaxVersions(10); + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preModifyColumn(ObserverContext.createAndPrepare(CP_ENV, null), + TEST_TABLE, hcd); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER); + verifyDenied(action, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testDeleteColumn() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preDeleteColumn(ObserverContext.createAndPrepare(CP_ENV, null), + TEST_TABLE, TEST_FAMILY); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER); + verifyDenied(action, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testTableDisable() throws Exception { + PrivilegedExceptionAction disableTable = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preDisableTable(ObserverContext.createAndPrepare(CP_ENV, null), + TEST_TABLE); + return null; + } + }; + + PrivilegedExceptionAction disableAclTable = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preDisableTable(ObserverContext.createAndPrepare(CP_ENV, null), + AccessControlLists.ACL_TABLE_NAME); + return null; + } + }; + + verifyAllowed(disableTable, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER); + verifyDenied(disableTable, USER_RW, USER_RO, USER_NONE); + + // No user should be allowed to disable _acl_ table + verifyDenied(disableAclTable, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER, USER_RW, USER_RO); + } + + @Test + public void testTableEnable() throws Exception { + PrivilegedExceptionAction enableTable = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER + .preEnableTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE); + return null; + } + }; + + verifyAllowed(enableTable, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER); + verifyDenied(enableTable, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testMove() throws Exception { + Map regions; + HTable table = new HTable(TEST_UTIL.getConfiguration(), TEST_TABLE); + try { + regions = table.getRegionsInfo(); + } finally { + table.close(); + } + final Map.Entry firstRegion = regions.entrySet().iterator().next(); + final ServerName server = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName(); + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preMove(ObserverContext.createAndPrepare(CP_ENV, null), + firstRegion.getKey(), server, server); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testAssign() throws Exception { + Map regions; + HTable table = new HTable(TEST_UTIL.getConfiguration(), TEST_TABLE); + try { + regions = table.getRegionsInfo(); + } finally { + table.close(); + } + final Map.Entry firstRegion = regions.entrySet().iterator().next(); + + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preAssign(ObserverContext.createAndPrepare(CP_ENV, null), + firstRegion.getKey()); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testUnassign() throws Exception { + Map regions; + HTable table = new HTable(TEST_UTIL.getConfiguration(), TEST_TABLE); + try { + regions = table.getRegionsInfo(); + } finally { + table.close(); + } + final Map.Entry firstRegion = regions.entrySet().iterator().next(); + + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preUnassign(ObserverContext.createAndPrepare(CP_ENV, null), + firstRegion.getKey(), false); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testBalance() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preBalance(ObserverContext.createAndPrepare(CP_ENV, null)); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN); + verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testBalanceSwitch() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preBalanceSwitch(ObserverContext.createAndPrepare(CP_ENV, null), true); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN); + verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testShutdown() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preShutdown(ObserverContext.createAndPrepare(CP_ENV, null)); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN); + verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testStopMaster() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preStopMaster(ObserverContext.createAndPrepare(CP_ENV, null)); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN); + verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE); + } + + private void verifyWrite(PrivilegedExceptionAction action) throws Exception { + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_RW); + verifyDenied(action, USER_NONE, USER_CREATE, USER_RO); + } + + @Test + public void testSplit() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preSplit(ObserverContext.createAndPrepare(RCP_ENV, null)); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testFlush() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preFlush(ObserverContext.createAndPrepare(RCP_ENV, null)); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testCompact() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preCompact(ObserverContext.createAndPrepare(RCP_ENV, null), null, null); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testPreCompactSelection() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preCompactSelection(ObserverContext.createAndPrepare(RCP_ENV, null), null, null); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); + } + + private void verifyRead(PrivilegedExceptionAction action) throws Exception { + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_RW, USER_RO); + verifyDenied(action, USER_NONE, USER_CREATE); + } + + private void verifyReadWrite(PrivilegedExceptionAction action) throws Exception { + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_RW); + verifyDenied(action, USER_NONE, USER_CREATE, USER_RO); + } + + @Test + public void testRead() throws Exception { + // get action + PrivilegedExceptionAction getAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Get g = new Get(Bytes.toBytes("random_row")); + g.addFamily(TEST_FAMILY); + HTable t = new HTable(conf, TEST_TABLE); + try { + t.get(g); + } finally { + t.close(); + } + return null; + } + }; + verifyRead(getAction); + + // action for scanning + PrivilegedExceptionAction scanAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Scan s = new Scan(); + s.addFamily(TEST_FAMILY); + + HTable table = new HTable(conf, TEST_TABLE); + try { + ResultScanner scanner = table.getScanner(s); + try { + for (Result r = scanner.next(); r != null; r = scanner.next()) { + // do nothing + } + } catch (IOException e) { + } finally { + scanner.close(); + } + } finally { + table.close(); + } + return null; + } + }; + verifyRead(scanAction); + } + + @Test + // test put, delete, increment + public void testWrite() throws Exception { + // put action + PrivilegedExceptionAction putAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Put p = new Put(Bytes.toBytes("random_row")); + p.add(TEST_FAMILY, Bytes.toBytes("Qualifier"), Bytes.toBytes(1)); + HTable t = new HTable(conf, TEST_TABLE); + try { + t.put(p); + } finally { + t.close(); + } + return null; + } + }; + verifyWrite(putAction); + + // delete action + PrivilegedExceptionAction deleteAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Delete d = new Delete(Bytes.toBytes("random_row")); + d.deleteFamily(TEST_FAMILY); + HTable t = new HTable(conf, TEST_TABLE); + try { + t.delete(d); + } finally { + t.close(); + } + return null; + } + }; + verifyWrite(deleteAction); + + // increment action + PrivilegedExceptionAction incrementAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Increment inc = new Increment(Bytes.toBytes("random_row")); + inc.addColumn(TEST_FAMILY, Bytes.toBytes("Qualifier"), 1); + HTable t = new HTable(conf, TEST_TABLE); + try { + t.increment(inc); + } finally { + t.close(); + } + return null; + } + }; + verifyWrite(incrementAction); + } + + @Test + public void testReadWrite() throws Exception { + // action for checkAndDelete + PrivilegedExceptionAction checkAndDeleteAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Delete d = new Delete(Bytes.toBytes("random_row")); + d.deleteFamily(TEST_FAMILY); + HTable t = new HTable(conf, TEST_TABLE); + try { + t.checkAndDelete(Bytes.toBytes("random_row"), TEST_FAMILY, Bytes.toBytes("q"), + Bytes.toBytes("test_value"), d); + } finally { + t.close(); + } + return null; + } + }; + verifyReadWrite(checkAndDeleteAction); + + // action for checkAndPut() + PrivilegedExceptionAction checkAndPut = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Put p = new Put(Bytes.toBytes("random_row")); + p.add(TEST_FAMILY, Bytes.toBytes("Qualifier"), Bytes.toBytes(1)); + HTable t = new HTable(conf, TEST_TABLE); + try { + t.checkAndPut(Bytes.toBytes("random_row"), TEST_FAMILY, Bytes.toBytes("q"), + Bytes.toBytes("test_value"), p); + } finally { + t.close(); + } + return null; + } + }; + verifyReadWrite(checkAndPut); + } + + @Test + public void testBulkLoad() throws Exception { + FileSystem fs = TEST_UTIL.getTestFileSystem(); + final Path dir = TEST_UTIL.getDataTestDir("testBulkLoad"); + fs.mkdirs(dir); + //need to make it globally writable + //so users creating HFiles have write permissions + fs.setPermission(dir, FsPermission.valueOf("-rwxrwxrwx")); + + PrivilegedExceptionAction bulkLoadAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + int numRows = 3; + + //Making the assumption that the test table won't split between the range + byte[][][] hfileRanges = {{{(byte)0}, {(byte)9}}}; + + Path bulkLoadBasePath = new Path(dir, new Path(User.getCurrent().getName())); + new BulkLoadHelper(bulkLoadBasePath) + .bulkLoadHFile(TEST_TABLE, TEST_FAMILY, Bytes.toBytes("q"), hfileRanges, numRows); + + return null; + } + }; + verifyWrite(bulkLoadAction); + + // Reinit after the bulk upload + TEST_UTIL.getHBaseAdmin().disableTable(TEST_TABLE); + TEST_UTIL.getHBaseAdmin().enableTable(TEST_TABLE); + } + + public class BulkLoadHelper { + private final FileSystem fs; + private final Path loadPath; + private final Configuration conf; + + public BulkLoadHelper(Path loadPath) throws IOException { + fs = TEST_UTIL.getTestFileSystem(); + conf = TEST_UTIL.getConfiguration(); + loadPath = loadPath.makeQualified(fs); + this.loadPath = loadPath; + } + + private void createHFile(Path path, + byte[] family, byte[] qualifier, + byte[] startKey, byte[] endKey, int numRows) throws IOException { + + HFile.Writer writer = null; + long now = System.currentTimeMillis(); + try { + writer = HFile.getWriterFactory(conf, new CacheConfig(conf)) + .withPath(fs, path) + .withComparator(KeyValue.KEY_COMPARATOR) + .create(); + // subtract 2 since numRows doesn't include boundary keys + for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, true, numRows-2)) { + KeyValue kv = new KeyValue(key, family, qualifier, now, key); + writer.append(kv); + } + } finally { + if(writer != null) + writer.close(); + } + } + + private void bulkLoadHFile( + byte[] tableName, + byte[] family, + byte[] qualifier, + byte[][][] hfileRanges, + int numRowsPerRange) throws Exception { + + Path familyDir = new Path(loadPath, Bytes.toString(family)); + fs.mkdirs(familyDir); + int hfileIdx = 0; + for (byte[][] range : hfileRanges) { + byte[] from = range[0]; + byte[] to = range[1]; + createHFile(new Path(familyDir, "hfile_"+(hfileIdx++)), + family, qualifier, from, to, numRowsPerRange); + } + //set global read so RegionServer can move it + setPermission(loadPath, FsPermission.valueOf("-rwxrwxrwx")); + + HTable table = new HTable(conf, tableName); + try { + TEST_UTIL.waitTableAvailable(tableName, 30000); + LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf); + loader.doBulkLoad(loadPath, table); + } finally { + table.close(); + } + } + + public void setPermission(Path dir, FsPermission perm) throws IOException { + if(!fs.getFileStatus(dir).isDir()) { + fs.setPermission(dir,perm); + } + else { + for(FileStatus el : fs.listStatus(dir)) { + fs.setPermission(el.getPath(), perm); + setPermission(el.getPath() , perm); + } + } + } + } + + @Test + public void testAppend() throws Exception { + + PrivilegedExceptionAction appendAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + byte[] row = Bytes.toBytes("random_row"); + byte[] qualifier = Bytes.toBytes("q"); + Put put = new Put(row); + put.add(TEST_FAMILY, qualifier, Bytes.toBytes(1)); + Append append = new Append(row); + append.add(TEST_FAMILY, qualifier, Bytes.toBytes(2)); + HTable t = new HTable(conf, TEST_TABLE); + try { + t.put(put); + t.append(append); + } finally { + t.close(); + } + return null; + } + }; + + verifyAllowed(appendAction, SUPERUSER, USER_ADMIN, USER_OWNER, USER_RW); + verifyDenied(appendAction, USER_CREATE, USER_RO, USER_NONE); + } + + @Test + public void testGrantRevoke() throws Exception { + + PrivilegedExceptionAction grantAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + TEST_TABLE); + protocol.grant(new UserPermission(Bytes.toBytes(USER_RO.getShortName()), TEST_TABLE, + TEST_FAMILY, (byte[]) null, Action.READ)); + } finally { + acl.close(); + } + return null; + } + }; + + PrivilegedExceptionAction revokeAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + TEST_TABLE); + protocol.revoke(new UserPermission(Bytes.toBytes(USER_RO.getShortName()), TEST_TABLE, + TEST_FAMILY, (byte[]) null, Action.READ)); + } finally { + acl.close(); + } + return null; + } + }; + + PrivilegedExceptionAction getPermissionsAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + TEST_TABLE); + protocol.getUserPermissions(TEST_TABLE); + } finally { + acl.close(); + } + return null; + } + }; + + verifyAllowed(grantAction, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(grantAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); + + verifyAllowed(revokeAction, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(revokeAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); + + verifyAllowed(getPermissionsAction, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(getPermissionsAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testPostGrantRevoke() throws Exception { + final byte[] tableName = Bytes.toBytes("TempTable"); + final byte[] family1 = Bytes.toBytes("f1"); + final byte[] family2 = Bytes.toBytes("f2"); + final byte[] qualifier = Bytes.toBytes("q"); + + // create table + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(tableName)) { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(family1)); + htd.addFamily(new HColumnDescriptor(family2)); + admin.createTable(htd); + + // create temp users + User tblUser = User + .createUserForTesting(TEST_UTIL.getConfiguration(), "tbluser", new String[0]); + User gblUser = User + .createUserForTesting(TEST_UTIL.getConfiguration(), "gbluser", new String[0]); + + // prepare actions: + PrivilegedExceptionAction putActionAll = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Put p = new Put(Bytes.toBytes("a")); + p.add(family1, qualifier, Bytes.toBytes("v1")); + p.add(family2, qualifier, Bytes.toBytes("v2")); + HTable t = new HTable(conf, tableName); + try { + t.put(p); + } finally { + t.close(); + } + return null; + } + }; + PrivilegedExceptionAction putAction1 = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Put p = new Put(Bytes.toBytes("a")); + p.add(family1, qualifier, Bytes.toBytes("v1")); + HTable t = new HTable(conf, tableName); + try { + t.put(p); + } finally { + t.close(); + } + return null; + } + }; + PrivilegedExceptionAction putAction2 = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Put p = new Put(Bytes.toBytes("a")); + p.add(family2, qualifier, Bytes.toBytes("v2")); + HTable t = new HTable(conf, tableName); + try { + t.put(p); + } finally { + t.close(); + } + return null; + } + }; + PrivilegedExceptionAction getActionAll = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Get g = new Get(Bytes.toBytes("random_row")); + g.addFamily(family1); + g.addFamily(family2); + HTable t = new HTable(conf, tableName); + try { + t.get(g); + } finally { + t.close(); + } + return null; + } + }; + PrivilegedExceptionAction getAction1 = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Get g = new Get(Bytes.toBytes("random_row")); + g.addFamily(family1); + HTable t = new HTable(conf, tableName); + try { + t.get(g); + } finally { + t.close(); + } + return null; + } + }; + PrivilegedExceptionAction getAction2 = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Get g = new Get(Bytes.toBytes("random_row")); + g.addFamily(family2); + HTable t = new HTable(conf, tableName); + try { + t.get(g); + } finally { + t.close(); + } + return null; + } + }; + PrivilegedExceptionAction deleteActionAll = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Delete d = new Delete(Bytes.toBytes("random_row")); + d.deleteFamily(family1); + d.deleteFamily(family2); + HTable t = new HTable(conf, tableName); + try { + t.delete(d); + } finally { + t.close(); + } + return null; + } + }; + PrivilegedExceptionAction deleteAction1 = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Delete d = new Delete(Bytes.toBytes("random_row")); + d.deleteFamily(family1); + HTable t = new HTable(conf, tableName); + try { + t.delete(d); + } finally { + t.close(); + } + return null; + } + }; + PrivilegedExceptionAction deleteAction2 = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Delete d = new Delete(Bytes.toBytes("random_row")); + d.deleteFamily(family2); + HTable t = new HTable(conf, tableName); + try { + t.delete(d); + } finally { + t.close(); + } + return null; + } + }; + + // initial check: + verifyDenied(tblUser, getActionAll, getAction1, getAction2); + verifyDenied(tblUser, putActionAll, putAction1, putAction2); + verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); + + verifyDenied(gblUser, getActionAll, getAction1, getAction2); + verifyDenied(gblUser, putActionAll, putAction1, putAction2); + verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); + + // grant table read permission + HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null, + Permission.Action.READ)); + protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()), + Permission.Action.READ)); + } finally { + acl.close(); + } + + Thread.sleep(100); + + // check + verifyAllowed(tblUser, getActionAll, getAction1, getAction2); + verifyDenied(tblUser, putActionAll, putAction1, putAction2); + verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); + + verifyAllowed(gblUser, getActionAll, getAction1, getAction2); + verifyDenied(gblUser, putActionAll, putAction1, putAction2); + verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); + + // grant table write permission + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null, + Permission.Action.WRITE)); + protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()), + Permission.Action.WRITE)); + } finally { + acl.close(); + } + + Thread.sleep(100); + + verifyDenied(tblUser, getActionAll, getAction1, getAction2); + verifyAllowed(tblUser, putActionAll, putAction1, putAction2); + verifyAllowed(tblUser, deleteActionAll, deleteAction1, deleteAction2); + + verifyDenied(gblUser, getActionAll, getAction1, getAction2); + verifyAllowed(gblUser, putActionAll, putAction1, putAction2); + verifyAllowed(gblUser, deleteActionAll, deleteAction1, deleteAction2); + + // revoke table permission + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null, + Permission.Action.READ, Permission.Action.WRITE)); + protocol.revoke(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, null)); + protocol.revoke(new UserPermission(Bytes.toBytes(gblUser.getShortName()))); + } finally { + acl.close(); + } + + Thread.sleep(100); + + verifyDenied(tblUser, getActionAll, getAction1, getAction2); + verifyDenied(tblUser, putActionAll, putAction1, putAction2); + verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); + + verifyDenied(gblUser, getActionAll, getAction1, getAction2); + verifyDenied(gblUser, putActionAll, putAction1, putAction2); + verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); + + // grant column family read permission + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family1, + Permission.Action.READ)); + protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()), + Permission.Action.READ)); + } finally { + acl.close(); + } + + Thread.sleep(100); + + // Access should be denied for family2 + verifyAllowed(tblUser, getActionAll, getAction1); + verifyDenied(tblUser, getAction2); + verifyDenied(tblUser, putActionAll, putAction1, putAction2); + verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); + + verifyAllowed(gblUser, getActionAll, getAction1, getAction2); + verifyDenied(gblUser, putActionAll, putAction1, putAction2); + verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); + + // grant column family write permission + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.grant(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family2, + Permission.Action.WRITE)); + protocol.grant(new UserPermission(Bytes.toBytes(gblUser.getShortName()), + Permission.Action.WRITE)); + } finally { + acl.close(); + } + + Thread.sleep(100); + + // READ from family1, WRITE to family2 are allowed + verifyAllowed(tblUser, getActionAll, getAction1); + verifyAllowed(tblUser, putAction2, deleteAction2); + verifyDenied(tblUser, getAction2); + verifyDenied(tblUser, putActionAll, putAction1); + verifyDenied(tblUser, deleteActionAll, deleteAction1); + + verifyDenied(gblUser, getActionAll, getAction1, getAction2); + verifyAllowed(gblUser, putActionAll, putAction1, putAction2); + verifyAllowed(gblUser, deleteActionAll, deleteAction1, deleteAction2); + + // revoke column family permission + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.revoke(new UserPermission(Bytes.toBytes(tblUser.getShortName()), tableName, family2)); + protocol.revoke(new UserPermission(Bytes.toBytes(gblUser.getShortName()))); + } finally { + acl.close(); + } + + Thread.sleep(100); + + // Revoke on family2 should not have impact on family1 permissions + verifyAllowed(tblUser, getActionAll, getAction1); + verifyDenied(tblUser, getAction2); + verifyDenied(tblUser, putActionAll, putAction1, putAction2); + verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); + + // Should not have access as global permissions are completely revoked + verifyDenied(gblUser, getActionAll, getAction1, getAction2); + verifyDenied(gblUser, putActionAll, putAction1, putAction2); + verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); + + // delete table + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + + private boolean hasFoundUserPermission(UserPermission userPermission, List perms) { + return perms.contains(userPermission); + } + + @Test + public void testPostGrantRevokeAtQualifierLevel() throws Exception { + final byte[] tableName = Bytes.toBytes("testGrantRevokeAtQualifierLevel"); + final byte[] family1 = Bytes.toBytes("f1"); + final byte[] family2 = Bytes.toBytes("f2"); + final byte[] qualifier = Bytes.toBytes("q"); + + // create table + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(tableName)) { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(family1)); + htd.addFamily(new HColumnDescriptor(family2)); + admin.createTable(htd); + + // create temp users + User user = User.createUserForTesting(TEST_UTIL.getConfiguration(), "user", new String[0]); + + PrivilegedExceptionAction getQualifierAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Get g = new Get(Bytes.toBytes("random_row")); + g.addColumn(family1, qualifier); + HTable t = new HTable(conf, tableName); + try { + t.get(g); + } finally { + t.close(); + } + return null; + } + }; + PrivilegedExceptionAction putQualifierAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Put p = new Put(Bytes.toBytes("random_row")); + p.add(family1, qualifier, Bytes.toBytes("v1")); + HTable t = new HTable(conf, tableName); + try { + t.put(p); + } finally { + t.close(); + } + return null; + } + }; + PrivilegedExceptionAction deleteQualifierAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Delete d = new Delete(Bytes.toBytes("random_row")); + d.deleteColumn(family1, qualifier); + // d.deleteFamily(family1); + HTable t = new HTable(conf, tableName); + try { + t.delete(d); + } finally { + t.close(); + } + return null; + } + }; + + HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1)); + } finally { + acl.close(); + } + + verifyDenied(user, getQualifierAction); + verifyDenied(user, putQualifierAction); + verifyDenied(user, deleteQualifierAction); + + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1, + qualifier, Permission.Action.READ)); + } finally { + acl.close(); + } + + Thread.sleep(100); + + verifyAllowed(user, getQualifierAction); + verifyDenied(user, putQualifierAction); + verifyDenied(user, deleteQualifierAction); + + // only grant write permission + // TODO: comment this portion after HBASE-3583 + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1, + qualifier, Permission.Action.WRITE)); + } finally { + acl.close(); + } + + Thread.sleep(100); + + verifyDenied(user, getQualifierAction); + verifyAllowed(user, putQualifierAction); + verifyAllowed(user, deleteQualifierAction); + + // grant both read and write permission. + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1, + qualifier, Permission.Action.READ, Permission.Action.WRITE)); + } finally { + acl.close(); + } + + Thread.sleep(100); + + verifyAllowed(user, getQualifierAction); + verifyAllowed(user, putQualifierAction); + verifyAllowed(user, deleteQualifierAction); + + // revoke family level permission won't impact column level. + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.revoke(new UserPermission(Bytes.toBytes(user.getShortName()), tableName, family1, + qualifier)); + } finally { + acl.close(); + } + + Thread.sleep(100); + + verifyDenied(user, getQualifierAction); + verifyDenied(user, putQualifierAction); + verifyDenied(user, deleteQualifierAction); + + // delete table + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + + @Test + public void testPermissionList() throws Exception { + final byte[] tableName = Bytes.toBytes("testPermissionList"); + final byte[] family1 = Bytes.toBytes("f1"); + final byte[] family2 = Bytes.toBytes("f2"); + final byte[] qualifier = Bytes.toBytes("q"); + final byte[] user = Bytes.toBytes("user"); + + // create table + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(tableName)) { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(family1)); + htd.addFamily(new HColumnDescriptor(family2)); + htd.setOwner(USER_OWNER); + admin.createTable(htd); + + List perms; + HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + perms = protocol.getUserPermissions(tableName); + } finally { + acl.close(); + } + + UserPermission ownerperm = new UserPermission(Bytes.toBytes(USER_OWNER.getName()), tableName, + null, Action.values()); + assertTrue("Owner should have all permissions on table", + hasFoundUserPermission(ownerperm, perms)); + + UserPermission up = new UserPermission(user, tableName, family1, qualifier, + Permission.Action.READ); + assertFalse("User should not be granted permission: " + up.toString(), + hasFoundUserPermission(up, perms)); + + // grant read permission + UserPermission upToSet = new UserPermission(user, tableName, family1, qualifier, + Permission.Action.READ); + + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.grant(upToSet); + perms = protocol.getUserPermissions(tableName); + } finally { + acl.close(); + } + + UserPermission upToVerify = new UserPermission(user, tableName, family1, qualifier, + Permission.Action.READ); + assertTrue("User should be granted permission: " + upToVerify.toString(), + hasFoundUserPermission(upToVerify, perms)); + + upToVerify = new UserPermission(user, tableName, family1, qualifier, Permission.Action.WRITE); + assertFalse("User should not be granted permission: " + upToVerify.toString(), + hasFoundUserPermission(upToVerify, perms)); + + // grant read+write + upToSet = new UserPermission(user, tableName, family1, qualifier, Permission.Action.WRITE, + Permission.Action.READ); + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.grant(upToSet); + perms = protocol.getUserPermissions(tableName); + } finally { + acl.close(); + } + + upToVerify = new UserPermission(user, tableName, family1, qualifier, Permission.Action.WRITE, + Permission.Action.READ); + assertTrue("User should be granted permission: " + upToVerify.toString(), + hasFoundUserPermission(upToVerify, perms)); + + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + protocol.revoke(upToSet); + perms = protocol.getUserPermissions(tableName); + } finally { + acl.close(); + } + + assertFalse("User should not be granted permission: " + upToVerify.toString(), + hasFoundUserPermission(upToVerify, perms)); + + // disable table before modification + admin.disableTable(tableName); + + User newOwner = User.createUserForTesting(conf, "new_owner", new String[] {}); + htd.setOwner(newOwner); + admin.modifyTable(tableName, htd); + + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + tableName); + perms = protocol.getUserPermissions(tableName); + } finally { + acl.close(); + } + + UserPermission newOwnerperm = new UserPermission(Bytes.toBytes(newOwner.getName()), tableName, + null, Action.values()); + assertTrue("New owner should have all permissions on table", + hasFoundUserPermission(newOwnerperm, perms)); + + // delete table + admin.deleteTable(tableName); + } + + /** global operations */ + private void verifyGlobal(PrivilegedExceptionAction action) throws Exception { + verifyAllowed(action, SUPERUSER); + + verifyDenied(action, USER_CREATE, USER_RW, USER_NONE, USER_RO); + } + + public void checkGlobalPerms(Permission.Action... actions) throws IOException { + Permission[] perms = new Permission[actions.length]; + for (int i = 0; i < actions.length; i++) { + perms[i] = new Permission(actions[i]); + } + HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + new byte[0]); + protocol.checkPermissions(perms); + } finally { + acl.close(); + } + } + + public void checkTablePerms(byte[] table, byte[] family, byte[] column, + Permission.Action... actions) throws IOException { + Permission[] perms = new Permission[actions.length]; + for (int i = 0; i < actions.length; i++) { + perms[i] = new TablePermission(table, family, column, actions[i]); + } + + checkTablePerms(table, perms); + } + + public void checkTablePerms(byte[] table, Permission... perms) throws IOException { + HTable acl = new HTable(conf, table); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + new byte[0]); + protocol.checkPermissions(perms); + } finally { + acl.close(); + } + } + + public void grant(AccessControllerProtocol protocol, User user, byte[] t, byte[] f, byte[] q, + Permission.Action... actions) throws IOException { + protocol.grant(new UserPermission(Bytes.toBytes(user.getShortName()), t, f, q, actions)); + } + + @Test + public void testCheckPermissions() throws Exception { + // -------------------------------------- + // test global permissions + PrivilegedExceptionAction globalAdmin = new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkGlobalPerms(Permission.Action.ADMIN); + return null; + } + }; + // verify that only superuser can admin + verifyGlobal(globalAdmin); + + // -------------------------------------- + // test multiple permissions + PrivilegedExceptionAction globalReadWrite = new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkGlobalPerms(Permission.Action.READ, Permission.Action.WRITE); + return null; + } + }; + + verifyGlobal(globalReadWrite); + + // -------------------------------------- + // table/column/qualifier level permissions + final byte[] TEST_Q1 = Bytes.toBytes("q1"); + final byte[] TEST_Q2 = Bytes.toBytes("q2"); + + User userTable = User.createUserForTesting(conf, "user_check_perms_table", new String[0]); + User userColumn = User.createUserForTesting(conf, "user_check_perms_family", new String[0]); + User userQualifier = User.createUserForTesting(conf, "user_check_perms_q", new String[0]); + + HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + TEST_TABLE); + grant(protocol, userTable, TEST_TABLE, null, null, Permission.Action.READ); + grant(protocol, userColumn, TEST_TABLE, TEST_FAMILY, null, Permission.Action.READ); + grant(protocol, userQualifier, TEST_TABLE, TEST_FAMILY, TEST_Q1, Permission.Action.READ); + } finally { + acl.close(); + } + + PrivilegedExceptionAction tableRead = new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_TABLE, null, null, Permission.Action.READ); + return null; + } + }; + + PrivilegedExceptionAction columnRead = new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_TABLE, TEST_FAMILY, null, Permission.Action.READ); + return null; + } + }; + + PrivilegedExceptionAction qualifierRead = new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_TABLE, TEST_FAMILY, TEST_Q1, Permission.Action.READ); + return null; + } + }; + + PrivilegedExceptionAction multiQualifierRead = new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_TABLE, new Permission[] { + new TablePermission(TEST_TABLE, TEST_FAMILY, TEST_Q1, Permission.Action.READ), + new TablePermission(TEST_TABLE, TEST_FAMILY, TEST_Q2, Permission.Action.READ), }); + return null; + } + }; + + PrivilegedExceptionAction globalAndTableRead = new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_TABLE, new Permission[] { new Permission(Permission.Action.READ), + new TablePermission(TEST_TABLE, null, (byte[]) null, Permission.Action.READ), }); + return null; + } + }; + + PrivilegedExceptionAction noCheck = new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_TABLE, new Permission[0]); + return null; + } + }; + + verifyAllowed(tableRead, SUPERUSER, userTable); + verifyDenied(tableRead, userColumn, userQualifier); + + verifyAllowed(columnRead, SUPERUSER, userTable, userColumn); + verifyDenied(columnRead, userQualifier); + + verifyAllowed(qualifierRead, SUPERUSER, userTable, userColumn, userQualifier); + + verifyAllowed(multiQualifierRead, SUPERUSER, userTable, userColumn); + verifyDenied(multiQualifierRead, userQualifier); + + verifyAllowed(globalAndTableRead, SUPERUSER); + verifyDenied(globalAndTableRead, userTable, userColumn, userQualifier); + + verifyAllowed(noCheck, SUPERUSER, userTable, userColumn, userQualifier); + + // -------------------------------------- + // test family level multiple permissions + PrivilegedExceptionAction familyReadWrite = new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_TABLE, TEST_FAMILY, null, Permission.Action.READ, + Permission.Action.WRITE); + return null; + } + }; + + verifyAllowed(familyReadWrite, SUPERUSER, USER_OWNER, USER_RW); + verifyDenied(familyReadWrite, USER_NONE, USER_CREATE, USER_RO); + + // -------------------------------------- + // check for wrong table region + acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy(AccessControllerProtocol.class, + TEST_TABLE); + try { + // but ask for TablePermissions for TEST_TABLE + protocol.checkPermissions(new Permission[] { (Permission) new TablePermission(TEST_TABLE, + null, (byte[]) null, Permission.Action.CREATE) }); + fail("this should have thrown CoprocessorException"); + } catch (CoprocessorException ex) { + // expected + } + } finally { + acl.close(); + } + } + + @Test + public void testLockAction() throws Exception { + PrivilegedExceptionAction lockAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preLockRow(ObserverContext.createAndPrepare(RCP_ENV, null), null, + Bytes.toBytes("random_row")); + return null; + } + }; + verifyAllowed(lockAction, SUPERUSER, USER_ADMIN, USER_OWNER, USER_CREATE, USER_RW_ON_TABLE); + verifyDenied(lockAction, USER_RO, USER_RW, USER_NONE); + } + + @Test + public void testUnLockAction() throws Exception { + PrivilegedExceptionAction unLockAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preUnlockRow(ObserverContext.createAndPrepare(RCP_ENV, null), null, + 123456); + return null; + } + }; + verifyAllowed(unLockAction, SUPERUSER, USER_ADMIN, USER_OWNER, USER_RW_ON_TABLE); + verifyDenied(unLockAction, USER_NONE, USER_RO, USER_RW); + } + + @Test + public void testStopRegionServer() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preStopRegionServer(ObserverContext.createAndPrepare(RSCP_ENV, null)); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN); + verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE); + } + + @Test + public void testOpenRegion() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preOpen(ObserverContext.createAndPrepare(RCP_ENV, null)); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } + + @Test + public void testCloseRegion() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preClose(ObserverContext.createAndPrepare(RCP_ENV, null), false); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } + + + @Test + public void testSnapshot() throws Exception { + PrivilegedExceptionAction snapshotAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preSnapshot(ObserverContext.createAndPrepare(CP_ENV, null), + null, null); + return null; + } + }; + + PrivilegedExceptionAction deleteAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preDeleteSnapshot(ObserverContext.createAndPrepare(CP_ENV, null), + null); + return null; + } + }; + + PrivilegedExceptionAction restoreAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preRestoreSnapshot(ObserverContext.createAndPrepare(CP_ENV, null), + null, null); + return null; + } + }; + + PrivilegedExceptionAction cloneAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + ACCESS_CONTROLLER.preCloneSnapshot(ObserverContext.createAndPrepare(CP_ENV, null), + null, null); + return null; + } + }; + + verifyAllowed(snapshotAction, SUPERUSER, USER_ADMIN); + verifyDenied(snapshotAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + + verifyAllowed(cloneAction, SUPERUSER, USER_ADMIN); + verifyDenied(deleteAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + + verifyAllowed(restoreAction, SUPERUSER, USER_ADMIN); + verifyDenied(restoreAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + + verifyAllowed(deleteAction, SUPERUSER, USER_ADMIN); + verifyDenied(cloneAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } + + @Test + public void testGlobalAuthorizationForNewRegisteredRS() throws Exception { + LOG.debug("Test for global authorization for a new registered RegionServer."); + MiniHBaseCluster hbaseCluster = TEST_UTIL.getHBaseCluster(); + final HRegionServer oldRs = hbaseCluster.getRegionServer(0); + + // Since each RegionServer running on different user, add global + // permissions for the new user. + HTable acl = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); + try { + AccessControllerProtocol protocol = acl.coprocessorProxy( + AccessControllerProtocol.class, TEST_TABLE); + String currentUser = User.getCurrent().getShortName(); + // User name for the new RegionServer we plan to add. + String activeUserForNewRs = currentUser + ".hfs." + + hbaseCluster.getLiveRegionServerThreads().size(); + + protocol.grant(new UserPermission(Bytes.toBytes(activeUserForNewRs), + Permission.Action.ADMIN, Permission.Action.CREATE, + Permission.Action.READ, Permission.Action.WRITE)); + + } finally { + acl.close(); + } + final HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + HTableDescriptor htd = new HTableDescriptor(TEST_TABLE2); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + htd.setOwner(USER_OWNER); + admin.createTable(htd); + + // Starting a new RegionServer. + JVMClusterUtil.RegionServerThread newRsThread = hbaseCluster + .startRegionServer(); + final HRegionServer newRs = newRsThread.getRegionServer(); + + // Move region to the new RegionServer. + final HTable table = new HTable(TEST_UTIL.getConfiguration(), TEST_TABLE2); + try { + NavigableMap regions = table + .getRegionLocations(); + final Map.Entry firstRegion = regions.entrySet() + .iterator().next(); + + PrivilegedExceptionAction moveAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + admin.move(firstRegion.getKey().getEncodedNameAsBytes(), + Bytes.toBytes(newRs.getServerName().getServerName())); + return null; + } + }; + SUPERUSER.runAs(moveAction); + + final int RETRIES_LIMIT = 10; + int retries = 0; + while (newRs.getOnlineRegions().size() < 1 && retries < RETRIES_LIMIT) { + LOG.debug("Waiting for region to be opened. Already retried " + retries + + " times."); + try { + Thread.sleep(200); + } catch (InterruptedException e) { + } + retries++; + if (retries == RETRIES_LIMIT - 1) { + fail("Retry exhaust for waiting region to be opened."); + } + } + // Verify write permission for user "admin2" who has the global + // permissions. + PrivilegedExceptionAction putAction = new PrivilegedExceptionAction() { + public Object run() throws Exception { + Put put = new Put(Bytes.toBytes("test")); + put.add(TEST_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value")); + table.put(put); + return null; + } + }; + USER_ADMIN.runAs(putAction); + } finally { + table.close(); + } + } + +} diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java new file mode 100644 index 0000000..0410e97 --- /dev/null +++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; + +/** + * Test the reading and writing of access permissions on {@code _acl_} table. + */ +@Category(LargeTests.class) +public class TestTablePermissions { + private static final Log LOG = LogFactory.getLog(TestTablePermissions.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static ZooKeeperWatcher ZKW; + private final static Abortable ABORTABLE = new Abortable() { + private final AtomicBoolean abort = new AtomicBoolean(false); + + @Override + public void abort(String why, Throwable e) { + LOG.info(why, e); + abort.set(true); + } + + @Override + public boolean isAborted() { + return abort.get(); + } + }; + + private static byte[] TEST_TABLE = Bytes.toBytes("perms_test"); + private static byte[] TEST_TABLE2 = Bytes.toBytes("perms_test2"); + private static byte[] TEST_FAMILY = Bytes.toBytes("f1"); + private static byte[] TEST_QUALIFIER = Bytes.toBytes("col1"); + + @BeforeClass + public static void beforeClass() throws Exception { + // setup configuration + Configuration conf = UTIL.getConfiguration(); + SecureTestUtil.enableSecurity(conf); + + UTIL.startMiniCluster(); + + // Wait for the ACL table to become available + UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000); + + ZKW = new ZooKeeperWatcher(UTIL.getConfiguration(), + "TestTablePermissions", ABORTABLE); + + UTIL.createTable(TEST_TABLE, TEST_FAMILY); + UTIL.createTable(TEST_TABLE2, TEST_FAMILY); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @After + public void tearDown() throws Exception { + Configuration conf = UTIL.getConfiguration(); + AccessControlLists.removeTablePermissions(conf, TEST_TABLE); + AccessControlLists.removeTablePermissions(conf, TEST_TABLE2); + AccessControlLists.removeTablePermissions(conf, AccessControlLists.ACL_TABLE_NAME); + } + + @Test + public void testBasicWrite() throws Exception { + Configuration conf = UTIL.getConfiguration(); + // add some permissions + AccessControlLists.addUserPermission(conf, + new UserPermission(Bytes.toBytes("george"), TEST_TABLE, null, (byte[])null, + UserPermission.Action.READ, UserPermission.Action.WRITE)); + AccessControlLists.addUserPermission(conf, + new UserPermission(Bytes.toBytes("hubert"), TEST_TABLE, null, (byte[])null, + UserPermission.Action.READ)); + AccessControlLists.addUserPermission(conf, + new UserPermission(Bytes.toBytes("humphrey"), + TEST_TABLE, TEST_FAMILY, TEST_QUALIFIER, + UserPermission.Action.READ)); + + // retrieve the same + ListMultimap perms = + AccessControlLists.getTablePermissions(conf, TEST_TABLE); + List userPerms = perms.get("george"); + assertNotNull("Should have permissions for george", userPerms); + assertEquals("Should have 1 permission for george", 1, userPerms.size()); + TablePermission permission = userPerms.get(0); + assertTrue("Permission should be for " + TEST_TABLE, + Bytes.equals(TEST_TABLE, permission.getTable())); + assertNull("Column family should be empty", permission.getFamily()); + + // check actions + assertNotNull(permission.getActions()); + assertEquals(2, permission.getActions().length); + List actions = Arrays.asList(permission.getActions()); + assertTrue(actions.contains(TablePermission.Action.READ)); + assertTrue(actions.contains(TablePermission.Action.WRITE)); + + userPerms = perms.get("hubert"); + assertNotNull("Should have permissions for hubert", userPerms); + assertEquals("Should have 1 permission for hubert", 1, userPerms.size()); + permission = userPerms.get(0); + assertTrue("Permission should be for " + TEST_TABLE, + Bytes.equals(TEST_TABLE, permission.getTable())); + assertNull("Column family should be empty", permission.getFamily()); + + // check actions + assertNotNull(permission.getActions()); + assertEquals(1, permission.getActions().length); + actions = Arrays.asList(permission.getActions()); + assertTrue(actions.contains(TablePermission.Action.READ)); + assertFalse(actions.contains(TablePermission.Action.WRITE)); + + userPerms = perms.get("humphrey"); + assertNotNull("Should have permissions for humphrey", userPerms); + assertEquals("Should have 1 permission for humphrey", 1, userPerms.size()); + permission = userPerms.get(0); + assertTrue("Permission should be for " + TEST_TABLE, + Bytes.equals(TEST_TABLE, permission.getTable())); + assertTrue("Permission should be for family " + TEST_FAMILY, + Bytes.equals(TEST_FAMILY, permission.getFamily())); + assertTrue("Permission should be for qualifier " + TEST_QUALIFIER, + Bytes.equals(TEST_QUALIFIER, permission.getQualifier())); + + // check actions + assertNotNull(permission.getActions()); + assertEquals(1, permission.getActions().length); + actions = Arrays.asList(permission.getActions()); + assertTrue(actions.contains(TablePermission.Action.READ)); + assertFalse(actions.contains(TablePermission.Action.WRITE)); + + // table 2 permissions + AccessControlLists.addUserPermission(conf, + new UserPermission(Bytes.toBytes("hubert"), TEST_TABLE2, null, (byte[])null, + TablePermission.Action.READ, TablePermission.Action.WRITE)); + + // check full load + Map> allPerms = + AccessControlLists.loadAll(conf); + assertEquals("Full permission map should have entries for both test tables", + 2, allPerms.size()); + + userPerms = allPerms.get(TEST_TABLE).get("hubert"); + assertNotNull(userPerms); + assertEquals(1, userPerms.size()); + permission = userPerms.get(0); + assertTrue(Bytes.equals(TEST_TABLE, permission.getTable())); + assertEquals(1, permission.getActions().length); + assertEquals(TablePermission.Action.READ, permission.getActions()[0]); + + userPerms = allPerms.get(TEST_TABLE2).get("hubert"); + assertNotNull(userPerms); + assertEquals(1, userPerms.size()); + permission = userPerms.get(0); + assertTrue(Bytes.equals(TEST_TABLE2, permission.getTable())); + assertEquals(2, permission.getActions().length); + actions = Arrays.asList(permission.getActions()); + assertTrue(actions.contains(TablePermission.Action.READ)); + assertTrue(actions.contains(TablePermission.Action.WRITE)); + } + + @Test + public void testPersistence() throws Exception { + Configuration conf = UTIL.getConfiguration(); + AccessControlLists.addUserPermission(conf, + new UserPermission(Bytes.toBytes("albert"), TEST_TABLE, null, + (byte[])null, TablePermission.Action.READ)); + AccessControlLists.addUserPermission(conf, + new UserPermission(Bytes.toBytes("betty"), TEST_TABLE, null, + (byte[])null, TablePermission.Action.READ, + TablePermission.Action.WRITE)); + AccessControlLists.addUserPermission(conf, + new UserPermission(Bytes.toBytes("clark"), + TEST_TABLE, TEST_FAMILY, + TablePermission.Action.READ)); + AccessControlLists.addUserPermission(conf, + new UserPermission(Bytes.toBytes("dwight"), + TEST_TABLE, TEST_FAMILY, TEST_QUALIFIER, + TablePermission.Action.WRITE)); + + // verify permissions survive changes in table metadata + ListMultimap preperms = + AccessControlLists.getTablePermissions(conf, TEST_TABLE); + + HTable table = new HTable(conf, TEST_TABLE); + table.put(new Put(Bytes.toBytes("row1")) + .add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes("v1"))); + table.put(new Put(Bytes.toBytes("row2")) + .add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes("v2"))); + HBaseAdmin admin = UTIL.getHBaseAdmin(); + admin.split(TEST_TABLE); + + // wait for split + Thread.sleep(10000); + + ListMultimap postperms = + AccessControlLists.getTablePermissions(conf, TEST_TABLE); + + checkMultimapEqual(preperms, postperms); + } + + @Test + public void testSerialization() throws Exception { + Configuration conf = UTIL.getConfiguration(); + ListMultimap permissions = ArrayListMultimap.create(); + permissions.put("george", new TablePermission(TEST_TABLE, null, + TablePermission.Action.READ)); + permissions.put("george", new TablePermission(TEST_TABLE, TEST_FAMILY, + TablePermission.Action.WRITE)); + permissions.put("george", new TablePermission(TEST_TABLE2, null, + TablePermission.Action.READ)); + permissions.put("hubert", new TablePermission(TEST_TABLE2, null, + TablePermission.Action.READ, TablePermission.Action.WRITE)); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + AccessControlLists.writePermissions(new DataOutputStream(bos), + permissions, conf); + + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + ListMultimap copy = + AccessControlLists.readPermissions(new DataInputStream(bis), conf); + + checkMultimapEqual(permissions, copy); + } + + public void checkMultimapEqual(ListMultimap first, + ListMultimap second) { + assertEquals(first.size(), second.size()); + for (String key : first.keySet()) { + List firstPerms = first.get(key); + List secondPerms = second.get(key); + assertNotNull(secondPerms); + assertEquals(firstPerms.size(), secondPerms.size()); + LOG.info("First permissions: "+firstPerms.toString()); + LOG.info("Second permissions: "+secondPerms.toString()); + for (TablePermission p : firstPerms) { + assertTrue("Permission "+p.toString()+" not found", secondPerms.contains(p)); + } + } + } + + @Test + public void testEquals() throws Exception { + TablePermission p1 = new TablePermission(TEST_TABLE, null, TablePermission.Action.READ); + TablePermission p2 = new TablePermission(TEST_TABLE, null, TablePermission.Action.READ); + assertTrue(p1.equals(p2)); + assertTrue(p2.equals(p1)); + + p1 = new TablePermission(TEST_TABLE, null, TablePermission.Action.READ, TablePermission.Action.WRITE); + p2 = new TablePermission(TEST_TABLE, null, TablePermission.Action.WRITE, TablePermission.Action.READ); + assertTrue(p1.equals(p2)); + assertTrue(p2.equals(p1)); + + p1 = new TablePermission(TEST_TABLE, TEST_FAMILY, TablePermission.Action.READ, TablePermission.Action.WRITE); + p2 = new TablePermission(TEST_TABLE, TEST_FAMILY, TablePermission.Action.WRITE, TablePermission.Action.READ); + assertTrue(p1.equals(p2)); + assertTrue(p2.equals(p1)); + + p1 = new TablePermission(TEST_TABLE, TEST_FAMILY, TEST_QUALIFIER, TablePermission.Action.READ, TablePermission.Action.WRITE); + p2 = new TablePermission(TEST_TABLE, TEST_FAMILY, TEST_QUALIFIER, TablePermission.Action.WRITE, TablePermission.Action.READ); + assertTrue(p1.equals(p2)); + assertTrue(p2.equals(p1)); + + p1 = new TablePermission(TEST_TABLE, null, TablePermission.Action.READ); + p2 = new TablePermission(TEST_TABLE, TEST_FAMILY, TablePermission.Action.READ); + assertFalse(p1.equals(p2)); + assertFalse(p2.equals(p1)); + + p1 = new TablePermission(TEST_TABLE, null, TablePermission.Action.READ); + p2 = new TablePermission(TEST_TABLE, null, TablePermission.Action.WRITE); + assertFalse(p1.equals(p2)); + assertFalse(p2.equals(p1)); + p2 = new TablePermission(TEST_TABLE, null, TablePermission.Action.READ, TablePermission.Action.WRITE); + assertFalse(p1.equals(p2)); + assertFalse(p2.equals(p1)); + + p1 = new TablePermission(TEST_TABLE, null, TablePermission.Action.READ); + p2 = new TablePermission(TEST_TABLE2, null, TablePermission.Action.READ); + assertFalse(p1.equals(p2)); + assertFalse(p2.equals(p1)); + + p2 = new TablePermission(TEST_TABLE, null); + assertFalse(p1.equals(p2)); + assertFalse(p2.equals(p1)); + } + + @Test + public void testGlobalPermission() throws Exception { + Configuration conf = UTIL.getConfiguration(); + + // add some permissions + AccessControlLists.addUserPermission(conf, + new UserPermission(Bytes.toBytes("user1"), + Permission.Action.READ, Permission.Action.WRITE)); + AccessControlLists.addUserPermission(conf, + new UserPermission(Bytes.toBytes("user2"), + Permission.Action.CREATE)); + AccessControlLists.addUserPermission(conf, + new UserPermission(Bytes.toBytes("user3"), + Permission.Action.ADMIN, Permission.Action.READ, Permission.Action.CREATE)); + + ListMultimap perms = AccessControlLists.getTablePermissions(conf, null); + List user1Perms = perms.get("user1"); + assertEquals("Should have 1 permission for user1", 1, user1Perms.size()); + assertEquals("user1 should have WRITE permission", + new Permission.Action[] { Permission.Action.READ, Permission.Action.WRITE }, + user1Perms.get(0).getActions()); + + List user2Perms = perms.get("user2"); + assertEquals("Should have 1 permission for user2", 1, user2Perms.size()); + assertEquals("user2 should have CREATE permission", + new Permission.Action[] { Permission.Action.CREATE }, + user2Perms.get(0).getActions()); + + List user3Perms = perms.get("user3"); + assertEquals("Should have 1 permission for user3", 1, user3Perms.size()); + assertEquals("user3 should have ADMIN, READ, CREATE permission", + new Permission.Action[] { + Permission.Action.ADMIN, Permission.Action.READ, Permission.Action.CREATE + }, + user3Perms.get(0).getActions()); + } + + @Test + public void testAuthManager() throws Exception { + Configuration conf = UTIL.getConfiguration(); + /* test a race condition causing TableAuthManager to sometimes fail global permissions checks + * when the global cache is being updated + */ + TableAuthManager authManager = TableAuthManager.get(ZKW, conf); + // currently running user is the system user and should have global admin perms + User currentUser = User.getCurrent(); + assertTrue(authManager.authorize(currentUser, Permission.Action.ADMIN)); + for (int i=1; i<=50; i++) { + AccessControlLists.addUserPermission(conf, new UserPermission(Bytes.toBytes("testauth"+i), + Permission.Action.ADMIN, Permission.Action.READ, Permission.Action.WRITE)); + // make sure the system user still shows as authorized + assertTrue("Failed current user auth check on iter "+i, + authManager.authorize(currentUser, Permission.Action.ADMIN)); + } + } +} diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/access/TestZKPermissionsWatcher.java b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestZKPermissionsWatcher.java new file mode 100644 index 0000000..0a17e4e --- /dev/null +++ b/security/src/test/java/org/apache/hadoop/hbase/security/access/TestZKPermissionsWatcher.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test the reading and writing of access permissions to and from zookeeper. + */ +@Category(LargeTests.class) +public class TestZKPermissionsWatcher { + private static final Log LOG = LogFactory.getLog(TestZKPermissionsWatcher.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static TableAuthManager AUTH_A; + private static TableAuthManager AUTH_B; + private final static Abortable ABORTABLE = new Abortable() { + private final AtomicBoolean abort = new AtomicBoolean(false); + + @Override + public void abort(String why, Throwable e) { + LOG.info(why, e); + abort.set(true); + } + + @Override + public boolean isAborted() { + return abort.get(); + } + }; + + private static byte[] TEST_TABLE = Bytes.toBytes("perms_test"); + + @BeforeClass + public static void beforeClass() throws Exception { + // setup configuration + Configuration conf = UTIL.getConfiguration(); + SecureTestUtil.enableSecurity(conf); + + // start minicluster + UTIL.startMiniCluster(); + AUTH_A = TableAuthManager.get(new ZooKeeperWatcher(conf, + "TestZKPermissionsWatcher_1", ABORTABLE), conf); + AUTH_B = TableAuthManager.get(new ZooKeeperWatcher(conf, + "TestZKPermissionsWatcher_2", ABORTABLE), conf); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test + public void testPermissionsWatcher() throws Exception { + assertFalse(AUTH_A.authorizeUser("george", TEST_TABLE, null, + TablePermission.Action.READ)); + assertFalse(AUTH_A.authorizeUser("george", TEST_TABLE, null, + TablePermission.Action.WRITE)); + assertFalse(AUTH_A.authorizeUser("hubert", TEST_TABLE, null, + TablePermission.Action.READ)); + assertFalse(AUTH_A.authorizeUser("hubert", TEST_TABLE, null, + TablePermission.Action.WRITE)); + + assertFalse(AUTH_B.authorizeUser("george", TEST_TABLE, null, + TablePermission.Action.READ)); + assertFalse(AUTH_B.authorizeUser("george", TEST_TABLE, null, + TablePermission.Action.WRITE)); + assertFalse(AUTH_B.authorizeUser("hubert", TEST_TABLE, null, + TablePermission.Action.READ)); + assertFalse(AUTH_B.authorizeUser("hubert", TEST_TABLE, null, + TablePermission.Action.WRITE)); + + // update ACL: george RW + List acl = new ArrayList(); + acl.add(new TablePermission(TEST_TABLE, null, TablePermission.Action.READ, + TablePermission.Action.WRITE)); + AUTH_A.setUserPermissions("george", TEST_TABLE, acl); + Thread.sleep(100); + + // check it + assertTrue(AUTH_A.authorizeUser("george", TEST_TABLE, null, + TablePermission.Action.READ)); + assertTrue(AUTH_A.authorizeUser("george", TEST_TABLE, null, + TablePermission.Action.WRITE)); + assertTrue(AUTH_B.authorizeUser("george", TEST_TABLE, null, + TablePermission.Action.READ)); + assertTrue(AUTH_B.authorizeUser("george", TEST_TABLE, null, + TablePermission.Action.WRITE)); + assertFalse(AUTH_A.authorizeUser("hubert", TEST_TABLE, null, + TablePermission.Action.READ)); + assertFalse(AUTH_A.authorizeUser("hubert", TEST_TABLE, null, + TablePermission.Action.WRITE)); + assertFalse(AUTH_B.authorizeUser("hubert", TEST_TABLE, null, + TablePermission.Action.READ)); + assertFalse(AUTH_B.authorizeUser("hubert", TEST_TABLE, null, + TablePermission.Action.WRITE)); + + // update ACL: hubert R + acl = new ArrayList(); + acl.add(new TablePermission(TEST_TABLE, null, TablePermission.Action.READ)); + AUTH_B.setUserPermissions("hubert", TEST_TABLE, acl); + Thread.sleep(100); + + // check it + assertTrue(AUTH_A.authorizeUser("george", TEST_TABLE, null, + TablePermission.Action.READ)); + assertTrue(AUTH_A.authorizeUser("george", TEST_TABLE, null, + TablePermission.Action.WRITE)); + assertTrue(AUTH_B.authorizeUser("george", TEST_TABLE, null, + TablePermission.Action.READ)); + assertTrue(AUTH_B.authorizeUser("george", TEST_TABLE, null, + TablePermission.Action.WRITE)); + assertTrue(AUTH_A.authorizeUser("hubert", TEST_TABLE, null, + TablePermission.Action.READ)); + assertFalse(AUTH_A.authorizeUser("hubert", TEST_TABLE, null, + TablePermission.Action.WRITE)); + assertTrue(AUTH_B.authorizeUser("hubert", TEST_TABLE, null, + TablePermission.Action.READ)); + assertFalse(AUTH_B.authorizeUser("hubert", TEST_TABLE, null, + TablePermission.Action.WRITE)); + } +} diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java b/security/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java new file mode 100644 index 0000000..64b53e9 --- /dev/null +++ b/security/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.token; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.security.PrivilegedExceptionAction; +import java.util.UUID; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.ipc.HBaseRPC; +import org.apache.hadoop.hbase.ipc.RequestContext; +import org.apache.hadoop.hbase.ipc.RpcServer; +import org.apache.hadoop.hbase.ipc.SecureRpcEngine; +import org.apache.hadoop.hbase.ipc.SecureServer; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.security.token.Token; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests for authentication token creation and usage + */ +@Category(LargeTests.class) +public class TestTokenAuthentication { + public static interface IdentityProtocol extends CoprocessorProtocol { + public String whoami(); + public String getAuthMethod(); + } + + public static class IdentityCoprocessor extends BaseEndpointCoprocessor + implements IdentityProtocol { + public String whoami() { + return RequestContext.getRequestUserName(); + } + + public String getAuthMethod() { + UserGroupInformation ugi = null; + User user = RequestContext.getRequestUser(); + if (user != null) { + ugi = user.getUGI(); + } + if (ugi != null) { + return ugi.getAuthenticationMethod().toString(); + } + return null; + } + } + + private static HBaseTestingUtility TEST_UTIL; + private static AuthenticationTokenSecretManager secretManager; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + TEST_UTIL = new HBaseTestingUtility(); + Configuration conf = TEST_UTIL.getConfiguration(); + conf.set(HBaseRPC.RPC_ENGINE_PROP, SecureRpcEngine.class.getName()); + conf.set("hbase.coprocessor.region.classes", + IdentityCoprocessor.class.getName()); + TEST_UTIL.startMiniCluster(); + HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); + RpcServer server = rs.getRpcServer(); + assertTrue(server instanceof SecureServer); + SecretManager mgr = + ((SecureServer)server).getSecretManager(); + assertTrue(mgr instanceof AuthenticationTokenSecretManager); + secretManager = (AuthenticationTokenSecretManager)mgr; + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testTokenCreation() throws Exception { + Token token = + secretManager.generateToken("testuser"); + + AuthenticationTokenIdentifier ident = new AuthenticationTokenIdentifier(); + Writables.getWritable(token.getIdentifier(), ident); + assertEquals("Token username should match", "testuser", + ident.getUsername()); + byte[] passwd = secretManager.retrievePassword(ident); + assertTrue("Token password and password from secret manager should match", + Bytes.equals(token.getPassword(), passwd)); + } + + // @Test - Disable due to kerberos requirement + public void testTokenAuthentication() throws Exception { + UserGroupInformation testuser = + UserGroupInformation.createUserForTesting("testuser", new String[]{"testgroup"}); + + testuser.setAuthenticationMethod( + UserGroupInformation.AuthenticationMethod.TOKEN); + final Configuration conf = TEST_UTIL.getConfiguration(); + conf.set("hadoop.security.authentication", "kerberos"); + conf.set("randomkey", UUID.randomUUID().toString()); + testuser.setConfiguration(conf); + Token token = + secretManager.generateToken("testuser"); + testuser.addToken(token); + + // verify the server authenticates us as this token user + testuser.doAs(new PrivilegedExceptionAction() { + public Object run() throws Exception { + HTable table = new HTable(conf, ".META."); + IdentityProtocol prot = table.coprocessorProxy( + IdentityProtocol.class, HConstants.EMPTY_START_ROW); + String myname = prot.whoami(); + assertEquals("testuser", myname); + String authMethod = prot.getAuthMethod(); + assertEquals("TOKEN", authMethod); + return null; + } + }); + } +} diff --git a/security/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcher.java b/security/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcher.java new file mode 100644 index 0000000..bad817c --- /dev/null +++ b/security/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcher.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.token; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test the synchronization of token authentication master keys through + * ZKSecretWatcher + */ +@Category(LargeTests.class) +public class TestZKSecretWatcher { + private static Log LOG = LogFactory.getLog(TestZKSecretWatcher.class); + private static HBaseTestingUtility TEST_UTIL; + private static AuthenticationTokenSecretManager KEY_MASTER; + private static AuthenticationTokenSecretManager KEY_SLAVE; + private static AuthenticationTokenSecretManager KEY_SLAVE2; + private static AuthenticationTokenSecretManager KEY_SLAVE3; + + private static class MockAbortable implements Abortable { + private boolean abort; + public void abort(String reason, Throwable e) { + LOG.info("Aborting: "+reason, e); + abort = true; + } + + public boolean isAborted() { + return abort; + } + } + + @BeforeClass + public static void setupBeforeClass() throws Exception { + TEST_UTIL = new HBaseTestingUtility(); + TEST_UTIL.startMiniZKCluster(); + Configuration conf = TEST_UTIL.getConfiguration(); + + ZooKeeperWatcher zk = newZK(conf, "server1", new MockAbortable()); + AuthenticationTokenSecretManager[] tmp = new AuthenticationTokenSecretManager[2]; + tmp[0] = new AuthenticationTokenSecretManager( + conf, zk, "server1", 60*60*1000, 60*1000); + tmp[0].start(); + + zk = newZK(conf, "server2", new MockAbortable()); + tmp[1] = new AuthenticationTokenSecretManager( + conf, zk, "server2", 60*60*1000, 60*1000); + tmp[1].start(); + + while (KEY_MASTER == null) { + for (int i=0; i<2; i++) { + if (tmp[i].isMaster()) { + KEY_MASTER = tmp[i]; + KEY_SLAVE = tmp[ i+1 % 2 ]; + break; + } + } + Thread.sleep(500); + } + LOG.info("Master is "+KEY_MASTER.getName()+ + ", slave is "+KEY_SLAVE.getName()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + @Test + public void testKeyUpdate() throws Exception { + // sanity check + assertTrue(KEY_MASTER.isMaster()); + assertFalse(KEY_SLAVE.isMaster()); + int maxKeyId = 0; + + KEY_MASTER.rollCurrentKey(); + AuthenticationKey key1 = KEY_MASTER.getCurrentKey(); + assertNotNull(key1); + LOG.debug("Master current key: "+key1.getKeyId()); + + // wait for slave to update + Thread.sleep(1000); + AuthenticationKey slaveCurrent = KEY_SLAVE.getCurrentKey(); + assertNotNull(slaveCurrent); + assertEquals(key1, slaveCurrent); + LOG.debug("Slave current key: "+slaveCurrent.getKeyId()); + + // generate two more keys then expire the original + KEY_MASTER.rollCurrentKey(); + AuthenticationKey key2 = KEY_MASTER.getCurrentKey(); + LOG.debug("Master new current key: "+key2.getKeyId()); + KEY_MASTER.rollCurrentKey(); + AuthenticationKey key3 = KEY_MASTER.getCurrentKey(); + LOG.debug("Master new current key: "+key3.getKeyId()); + + // force expire the original key + key1.setExpiration(EnvironmentEdgeManager.currentTimeMillis() - 1000); + KEY_MASTER.removeExpiredKeys(); + // verify removed from master + assertNull(KEY_MASTER.getKey(key1.getKeyId())); + + // wait for slave to catch up + Thread.sleep(1000); + // make sure the slave has both new keys + AuthenticationKey slave2 = KEY_SLAVE.getKey(key2.getKeyId()); + assertNotNull(slave2); + assertEquals(key2, slave2); + AuthenticationKey slave3 = KEY_SLAVE.getKey(key3.getKeyId()); + assertNotNull(slave3); + assertEquals(key3, slave3); + slaveCurrent = KEY_SLAVE.getCurrentKey(); + assertEquals(key3, slaveCurrent); + LOG.debug("Slave current key: "+slaveCurrent.getKeyId()); + + // verify that the expired key has been removed + assertNull(KEY_SLAVE.getKey(key1.getKeyId())); + + // bring up a new slave + Configuration conf = TEST_UTIL.getConfiguration(); + ZooKeeperWatcher zk = newZK(conf, "server3", new MockAbortable()); + KEY_SLAVE2 = new AuthenticationTokenSecretManager( + conf, zk, "server3", 60*60*1000, 60*1000); + KEY_SLAVE2.start(); + + Thread.sleep(1000); + // verify the new slave has current keys (and not expired) + slave2 = KEY_SLAVE2.getKey(key2.getKeyId()); + assertNotNull(slave2); + assertEquals(key2, slave2); + slave3 = KEY_SLAVE2.getKey(key3.getKeyId()); + assertNotNull(slave3); + assertEquals(key3, slave3); + slaveCurrent = KEY_SLAVE2.getCurrentKey(); + assertEquals(key3, slaveCurrent); + assertNull(KEY_SLAVE2.getKey(key1.getKeyId())); + + // test leader failover + KEY_MASTER.stop(); + + // wait for master to stop + Thread.sleep(1000); + assertFalse(KEY_MASTER.isMaster()); + + // check for a new master + AuthenticationTokenSecretManager[] mgrs = + new AuthenticationTokenSecretManager[]{ KEY_SLAVE, KEY_SLAVE2 }; + AuthenticationTokenSecretManager newMaster = null; + int tries = 0; + while (newMaster == null && tries++ < 5) { + for (AuthenticationTokenSecretManager mgr : mgrs) { + if (mgr.isMaster()) { + newMaster = mgr; + break; + } + } + if (newMaster == null) { + Thread.sleep(500); + } + } + assertNotNull(newMaster); + + AuthenticationKey current = newMaster.getCurrentKey(); + // new master will immediately roll the current key, so it's current may be greater + assertTrue(current.getKeyId() >= slaveCurrent.getKeyId()); + LOG.debug("New master, current key: "+current.getKeyId()); + + // roll the current key again on new master and verify the key ID increments + newMaster.rollCurrentKey(); + AuthenticationKey newCurrent = newMaster.getCurrentKey(); + LOG.debug("New master, rolled new current key: "+newCurrent.getKeyId()); + assertTrue(newCurrent.getKeyId() > current.getKeyId()); + + // add another slave + ZooKeeperWatcher zk3 = newZK(conf, "server4", new MockAbortable()); + KEY_SLAVE3 = new AuthenticationTokenSecretManager( + conf, zk3, "server4", 60*60*1000, 60*1000); + KEY_SLAVE3.start(); + Thread.sleep(5000); + + // check master failover again + newMaster.stop(); + + // wait for master to stop + Thread.sleep(5000); + assertFalse(newMaster.isMaster()); + + // check for a new master + mgrs = new AuthenticationTokenSecretManager[]{ KEY_SLAVE, KEY_SLAVE2, KEY_SLAVE3 }; + newMaster = null; + tries = 0; + while (newMaster == null && tries++ < 5) { + for (AuthenticationTokenSecretManager mgr : mgrs) { + if (mgr.isMaster()) { + newMaster = mgr; + break; + } + } + if (newMaster == null) { + Thread.sleep(500); + } + } + assertNotNull(newMaster); + + AuthenticationKey current2 = newMaster.getCurrentKey(); + // new master will immediately roll the current key, so it's current may be greater + assertTrue(current2.getKeyId() >= newCurrent.getKeyId()); + LOG.debug("New master 2, current key: "+current2.getKeyId()); + + // roll the current key again on new master and verify the key ID increments + newMaster.rollCurrentKey(); + AuthenticationKey newCurrent2 = newMaster.getCurrentKey(); + LOG.debug("New master 2, rolled new current key: "+newCurrent2.getKeyId()); + assertTrue(newCurrent2.getKeyId() > current2.getKeyId()); + } + + private static ZooKeeperWatcher newZK(Configuration conf, String name, + Abortable abort) throws Exception { + Configuration copy = HBaseConfiguration.create(conf); + ZooKeeperWatcher zk = new ZooKeeperWatcher(copy, name, abort); + return zk; + } +} diff --git a/security/src/test/resources/hbase-site.xml b/security/src/test/resources/hbase-site.xml new file mode 100644 index 0000000..cca8832 --- /dev/null +++ b/security/src/test/resources/hbase-site.xml @@ -0,0 +1,145 @@ + + + + + + hbase.regionserver.msginterval + 1000 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.client.pause + 1000 + General client pause value. Used mostly as value to wait + before running a retry of a failed get, region lookup, etc. + + + hbase.client.retries.number + 10 + Maximum retries. Used as maximum for all retryable + operations such as fetching of the root region from root region + server, getting a cell's value, starting a row update, etc. + Default: 10. + + + + hbase.server.thread.wakefrequency + 1000 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as META scanner and log roller. + + + + hbase.master.event.waiting.time + 50 + Time to sleep between checks to see if a table event took place. + + + + hbase.regionserver.handler.count + 5 + Count of RPC Server instances spun up on RegionServers + Same property is used by the HMaster for count of master handlers. + Default is 10. + + + + hbase.master.info.port + -1 + The port for the hbase master web UI + Set to -1 if you do not want the info server to run. + + + + hbase.regionserver.info.port + -1 + The port for the hbase regionserver web UI + Set to -1 if you do not want the info server to run. + + + + hbase.regionserver.info.port.auto + true + Info server auto port bind. Enables automatic port + search if hbase.regionserver.info.port is already in use. + Enabled for testing to run multiple tests on one machine. + + + + hbase.master.lease.thread.wakefrequency + 3000 + The interval between checks for expired region server leases. + This value has been reduced due to the other reduced values above so that + the master will notice a dead region server sooner. The default is 15 seconds. + + + + hbase.regionserver.safemode + false + + Turn on/off safe mode in region server. Always on for production, always off + for tests. + + + + hbase.hregion.max.filesize + 67108864 + + Maximum desired file size for an HRegion. If filesize exceeds + value + (value / 2), the HRegion is split in two. Default: 256M. + + Keep the maximum filesize small so we split more often in tests. + + + + hadoop.log.dir + ${user.dir}/../logs + + + hbase.zookeeper.property.clientPort + 21818 + Property from ZooKeeper's config zoo.cfg. + The port at which the clients will connect. + + + + hbase.defaults.for.version.skip + true + + Set to true to skip the 'hbase.defaults.for.version'. + Setting this to true can be useful in contexts other than + the other side of a maven generation; i.e. running in an + ide. You'll want to set this boolean to true to avoid + seeing the RuntimException complaint: "hbase-default.xml file + seems to be for and old version of HBase (@@@VERSION@@@), this + version is X.X.X-SNAPSHOT" + + + + hbase.rpc.engine + org.apache.hadoop.hbase.ipc.SecureRpcEngine + + diff --git a/src/assembly/all.xml b/src/assembly/all.xml new file mode 100644 index 0000000..be61f37 --- /dev/null +++ b/src/assembly/all.xml @@ -0,0 +1,116 @@ + + + + + all + + dir + + + + + ${basedir}/*.txt + + + + + pom.xml + + + + src + 0644 + 0755 + + + security + 0644 + 0755 + + + secondaryindex + 0644 + 0755 + + + conf + 0644 + 0755 + + + bin + 0755 + 0755 + + + src/main/ruby + lib/ruby + 0644 + 0755 + + + target + / + + ${project.build.finalName}.jar + ${project.build.finalName}-tests.jar + + 0644 + 0755 + + + target/hbase-webapps + hbase-webapps + 0644 + 0755 + + + target/site + docs + 0644 + 0644 + + + src/packages + sbin + 755 + + update-hbase-env.sh + + + + + + /lib + false + runtime + + org.apache.hbase:hbase + + 0644 + 0644 + + + diff --git a/src/docbkx/book.xml b/src/docbkx/book.xml new file mode 100644 index 0000000..4e19d91 --- /dev/null +++ b/src/docbkx/book.xml @@ -0,0 +1,3739 @@ + + + + + + <link xlink:href="http://www.hbase.org"> + The Apache HBase™ Reference Guide + </link> + + + + + + + + + 2012Apache Software Foundation. + All Rights Reserved. Apache Hadoop, Hadoop, MapReduce, HDFS, Zookeeper, HBase, and the HBase project logo are trademarks of the Apache Software Foundation. + + + + This is the official reference guide of + Apache HBase (TM), + a distributed, versioned, column-oriented database built on top of + Apache Hadoop and + Apache ZooKeeper. + + + + + + + + + + + + + + + + + + + + + + + + Data Model + In short, applications store data into an HBase table. + Tables are made of rows and columns. + All columns in HBase belong to a particular column family. + Table cells -- the intersection of row and column + coordinates -- are versioned. + A cell’s content is an uninterpreted array of bytes. + + Table row keys are also byte arrays so almost anything can + serve as a row key from strings to binary representations of longs or + even serialized data structures. Rows in HBase tables + are sorted by row key. The sort is byte-ordered. All table accesses are + via the table row key -- its primary key. + + +
Conceptual View + + The following example is a slightly modified form of the one on page + 2 of the BigTable paper. + There is a table called webtable that contains two column families named + contents and anchor. + In this example, anchor contains two + columns (anchor:cssnsi.com, anchor:my.look.ca) + and contents contains one column (contents:html). + + Column Names + + By convention, a column name is made of its column family prefix and a + qualifier. For example, the + column + contents:html is of the column family contents + The colon character (:) delimits the column family from the + column family qualifier. + + + Table <varname>webtable</varname> + + + + + + + Row KeyTime StampColumnFamily contentsColumnFamily anchor + + + "com.cnn.www"t9anchor:cnnsi.com = "CNN" + "com.cnn.www"t8anchor:my.look.ca = "CNN.com" + "com.cnn.www"t6contents:html = "<html>..." + "com.cnn.www"t5contents:html = "<html>..." + "com.cnn.www"t3contents:html = "<html>..." + + +
+ +
+
Physical View + + Although at a conceptual level tables may be viewed as a sparse set of rows. + Physically they are stored on a per-column family basis. New columns + (i.e., columnfamily:column) can be added to any + column family without pre-announcing them. + ColumnFamily <varname>anchor</varname> + + + + + + Row KeyTime StampColumn Family anchor + + + "com.cnn.www"t9anchor:cnnsi.com = "CNN" + "com.cnn.www"t8anchor:my.look.ca = "CNN.com" + + +
+ ColumnFamily <varname>contents</varname> + + + + + + Row KeyTime StampColumnFamily "contents:" + + + "com.cnn.www"t6contents:html = "<html>..." + "com.cnn.www"t5contents:html = "<html>..." + "com.cnn.www"t3contents:html = "<html>..." + + +
+ It is important to note in the diagram above that the empty cells shown in the + conceptual view are not stored since they need not be in a column-oriented + storage format. Thus a request for the value of the contents:html + column at time stamp t8 would return no value. Similarly, a + request for an anchor:my.look.ca value at time stamp + t9 would return no value. However, if no timestamp is + supplied, the most recent value for a particular column would be returned + and would also be the first one found since timestamps are stored in + descending order. Thus a request for the values of all columns in the row + com.cnn.www if no timestamp is specified would be: + the value of contents:html from time stamp + t6, the value of anchor:cnnsi.com + from time stamp t9, the value of + anchor:my.look.ca from time stamp t8. + + For more information about the internals of how Apache HBase stores data, see . + +
+ +
+ Table + + Tables are declared up front at schema definition time. + +
+ +
+ Row + Row keys are uninterrpreted bytes. Rows are + lexicographically sorted with the lowest order appearing first + in a table. The empty byte array is used to denote both the + start and end of a tables' namespace. +
+ +
+ Column Family<indexterm><primary>Column Family</primary></indexterm> + + Columns in Apache HBase are grouped into column families. + All column members of a column family have the same prefix. For example, the + columns courses:history and + courses:math are both members of the + courses column family. + The colon character (:) delimits the column family from the + column family qualifierColumn Family Qualifier. + The column family prefix must be composed of + printable characters. The qualifying tail, the + column family qualifier, can be made of any + arbitrary bytes. Column families must be declared up front + at schema definition time whereas columns do not need to be + defined at schema time but can be conjured on the fly while + the table is up an running. + Physically, all column family members are stored together on the + filesystem. Because tunings and + storage specifications are done at the column family level, it is + advised that all column family members have the same general access + pattern and size characteristics. + + +
+
+ Cells<indexterm><primary>Cells</primary></indexterm> + A {row, column, version} tuple exactly + specifies a cell in HBase. + Cell content is uninterrpreted bytes +
+
+ Data Model Operations + The four primary data model operations are Get, Put, Scan, and Delete. Operations are applied via + HTable instances. + +
+ Get + Get returns + attributes for a specified row. Gets are executed via + + HTable.get. + +
+
+ Put + Put either + adds new rows to a table (if the key is new) or can update existing rows (if the key already exists). Puts are executed via + + HTable.put (writeBuffer) or + HTable.batch (non-writeBuffer). + +
+
+ Scans + Scan allow + iteration over multiple rows for specified attributes. + + The following is an example of a + on an HTable table instance. Assume that a table is populated with rows with keys "row1", "row2", "row3", + and then another set of rows with the keys "abc1", "abc2", and "abc3". The following example shows how startRow and stopRow + can be applied to a Scan instance to return the rows beginning with "row". + +HTable htable = ... // instantiate HTable + +Scan scan = new Scan(); +scan.addColumn(Bytes.toBytes("cf"),Bytes.toBytes("attr")); +scan.setStartRow( Bytes.toBytes("row")); // start key is inclusive +scan.setStopRow( Bytes.toBytes("row" + (char)0)); // stop key is exclusive +ResultScanner rs = htable.getScanner(scan); +try { + for (Result r = rs.next(); r != null; r = rs.next()) { + // process result... +} finally { + rs.close(); // always close the ResultScanner! +} + + +
+
+ Delete + Delete removes + a row from a table. Deletes are executed via + + HTable.delete. + + HBase does not modify data in place, and so deletes are handled by creating new markers called tombstones. + These tombstones, along with the dead values, are cleaned up on major compactions. + + See for more information on deleting versions of columns, and see + for more information on compactions. + + +
+ +
+ + +
+ Versions<indexterm><primary>Versions</primary></indexterm> + + A {row, column, version} tuple exactly + specifies a cell in HBase. It's possible to have an + unbounded number of cells where the row and column are the same but the + cell address differs only in its version dimension. + + While rows and column keys are expressed as bytes, the version is + specified using a long integer. Typically this long contains time + instances such as those returned by + java.util.Date.getTime() or + System.currentTimeMillis(), that is: the difference, + measured in milliseconds, between the current time and midnight, January + 1, 1970 UTC. + + The HBase version dimension is stored in decreasing order, so that + when reading from a store file, the most recent values are found + first. + + There is a lot of confusion over the semantics of + cell versions, in HBase. In particular, a couple + questions that often come up are: + + If multiple writes to a cell have the same version, are all + versions maintained or just the last? + Currently, only the last written is fetchable. + + + + + Is it OK to write cells in a non-increasing version + order? + Yes + + + + + Below we describe how the version dimension in HBase currently + works + See HBASE-2406 + for discussion of HBase versions. Bending time + in HBase makes for a good read on the version, or time, + dimension in HBase. It has more detail on versioning than is + provided here. As of this writing, the limiitation + Overwriting values at existing timestamps + mentioned in the article no longer holds in HBase. This section is + basically a synopsis of this article by Bruno Dumon. + . + +
+ Versions and HBase Operations + + In this section we look at the behavior of the version dimension + for each of the core HBase operations. + +
+ Get/Scan + + Gets are implemented on top of Scans. The below discussion of + Get applies equally to Scans. + + By default, i.e. if you specify no explicit version, when + doing a get, the cell whose version has the + largest value is returned (which may or may not be the latest one + written, see later). The default behavior can be modified in the + following ways: + + + + to return more than one version, see Get.setMaxVersions() + + + + to return versions other than the latest, see Get.setTimeRange() + + To retrieve the latest version that is less than or equal + to a given value, thus giving the 'latest' state of the record + at a certain point in time, just use a range from 0 to the + desired version and set the max versions to 1. + + + +
+
+ Default Get Example + The following Get will only retrieve the current version of the row + +Get get = new Get(Bytes.toBytes("row1")); +Result r = htable.get(get); +byte[] b = r.getValue(Bytes.toBytes("cf"), Bytes.toBytes("attr")); // returns current version of value + + +
+
+ Versioned Get Example + The following Get will return the last 3 versions of the row. + +Get get = new Get(Bytes.toBytes("row1")); +get.setMaxVersions(3); // will return last 3 versions of row +Result r = htable.get(get); +byte[] b = r.getValue(Bytes.toBytes("cf"), Bytes.toBytes("attr")); // returns current version of value +List<KeyValue> kv = r.getColumn(Bytes.toBytes("cf"), Bytes.toBytes("attr")); // returns all versions of this column + + +
+ +
+ Put + + Doing a put always creates a new version of a + cell, at a certain timestamp. By default the + system uses the server's currentTimeMillis, but + you can specify the version (= the long integer) yourself, on a + per-column level. This means you could assign a time in the past or + the future, or use the long value for non-time purposes. + + To overwrite an existing value, do a put at exactly the same + row, column, and version as that of the cell you would + overshadow. +
+ Implicit Version Example + The following Put will be implicitly versioned by HBase with the current time. + +Put put = new Put(Bytes.toBytes(row)); +put.add(Bytes.toBytes("cf"), Bytes.toBytes("attr1"), Bytes.toBytes( data)); +htable.put(put); + + +
+
+ Explicit Version Example + The following Put has the version timestamp explicitly set. + +Put put = new Put( Bytes.toBytes(row)); +long explicitTimeInMs = 555; // just an example +put.add(Bytes.toBytes("cf"), Bytes.toBytes("attr1"), explicitTimeInMs, Bytes.toBytes(data)); +htable.put(put); + + Caution: the version timestamp is internally by HBase for things like time-to-live calculations. + It's usually best to avoid setting this timestamp yourself. Prefer using a separate + timestamp attribute of the row, or have the timestamp a part of the rowkey, or both. + +
+ +
+ +
+ Delete + + There are three different types of internal delete markers + See Lars Hofhansl's blog for discussion of his attempt + adding another, Scanning in HBase: Prefix Delete Marker: + + Delete: for a specific version of a column. + + Delete column: for all versions of a column. + + Delete family: for all columns of a particular ColumnFamily + + + When deleting an entire row, HBase will internally create a tombstone for each ColumnFamily (i.e., not each individual column). + + Deletes work by creating tombstone + markers. For example, let's suppose we want to delete a row. For + this you can specify a version, or else by default the + currentTimeMillis is used. What this means is + delete all cells where the version is less than or equal to + this version. HBase never modifies data in place, so for + example a delete will not immediately delete (or mark as deleted) + the entries in the storage file that correspond to the delete + condition. Rather, a so-called tombstone is + written, which will mask the deleted values + When HBase does a major compaction, the tombstones are + processed to actually remove the dead values, together with the + tombstones themselves. + . If the version you specified when deleting a row is + larger than the version of any value in the row, then you can + consider the complete row to be deleted. + For an informative discussion on how deletes and versioning interact, see + the thread Put w/ timestamp -> Deleteall -> Put w/ timestamp fails + up on the user mailing list. + Also see for more information on the internal KeyValue format. + +
+
+ +
+ Current Limitations + +
+ Deletes mask Puts + + Deletes mask puts, even puts that happened after the delete + was entered + HBASE-2256 + . Remember that a delete writes a tombstone, which only + disappears after then next major compaction has run. Suppose you do + a delete of everything <= T. After this you do a new put with a + timestamp <= T. This put, even if it happened after the delete, + will be masked by the delete tombstone. Performing the put will not + fail, but when you do a get you will notice the put did have no + effect. It will start working again after the major compaction has + run. These issues should not be a problem if you use + always-increasing versions for new puts to a row. But they can occur + even if you do not care about time: just do delete and put + immediately after each other, and there is some chance they happen + within the same millisecond. +
+ +
+ Major compactions change query results + + ...create three cell versions at t1, t2 and t3, with a + maximum-versions setting of 2. So when getting all versions, only + the values at t2 and t3 will be returned. But if you delete the + version at t2 or t3, the one at t1 will appear again. Obviously, + once a major compaction has run, such behavior will not be the case + anymore... + See Garbage Collection in Bending + time in HBase + +
+
+
+
+ Sort Order + All data model operations HBase return data in sorted order. First by row, + then by ColumnFamily, followed by column qualifier, and finally timestamp (sorted + in reverse, so newest records are returned first). + +
+
+ Column Metadata + There is no store of column metadata outside of the internal KeyValue instances for a ColumnFamily. + Thus, while HBase can support not only a wide number of columns per row, but a heterogenous set of columns + between rows as well, it is your responsibility to keep track of the column names. + + The only way to get a complete set of columns that exist for a ColumnFamily is to process all the rows. + For more information about how HBase stores data internally, see . + +
+
Joins + Whether HBase supports joins is a common question on the dist-list, and there is a simple answer: it doesn't, + at not least in the way that RDBMS' support them (e.g., with equi-joins or outer-joins in SQL). As has been illustrated + in this chapter, the read data model operations in HBase are Get and Scan. + + However, that doesn't mean that equivalent join functionality can't be supported in your application, but + you have to do it yourself. The two primary strategies are either denormalizing the data upon writing to HBase, + or to have lookup tables and do the join between HBase tables in your application or MapReduce code (and as RDBMS' + demonstrate, there are several strategies for this depending on the size of the tables, e.g., nested loops vs. + hash-joins). So which is the best approach? It depends on what you are trying to do, and as such there isn't a single + answer that works for every use case. + +
+
ACID + 
See ACID Semantics.
+            Lars Hofhansl has also written a note on
+            ACID in HBase.
+
+ + + + HBase and Schema Design + A good general introduction on the strength and weaknesses modelling on + the various non-rdbms datastores is Ian Varley's Master thesis, + No Relation: The Mixed Blessings of Non-Relational Databases. + Recommended. Also, read for how HBase stores data internally. + +
+ + Schema Creation + + HBase schemas can be created or updated with + or by using HBaseAdmin in the Java API. + + Tables must be disabled when making ColumnFamily modifications, for example.. + +Configuration config = HBaseConfiguration.create(); +HBaseAdmin admin = new HBaseAdmin(conf); +String table = "myTable"; + +admin.disableTable(table); + +HColumnDescriptor cf1 = ...; +admin.addColumn(table, cf1); // adding new ColumnFamily +HColumnDescriptor cf2 = ...; +admin.modifyColumn(table, cf2); // modifying existing ColumnFamily + +admin.enableTable(table); + + See for more information about configuring client connections. + Note: online schema changes are supported in the 0.92.x codebase, but the 0.90.x codebase requires the table + to be disabled. + +
Schema Updates + When changes are made to either Tables or ColumnFamilies (e.g., region size, block size), these changes + take effect the next time there is a major compaction and the StoreFiles get re-written. + + See for more information on StoreFiles. + +
+
+
+ + On the number of column families + + + HBase currently does not do well with anything above two or three column families so keep the number + of column families in your schema low. Currently, flushing and compactions are done on a per Region basis so + if one column family is carrying the bulk of the data bringing on flushes, the adjacent families + will also be flushed though the amount of data they carry is small. When many column families the + flushing and compaction interaction can make for a bunch of needless i/o loading (To be addressed by + changing flushing and compaction to work on a per column family basis). For more information + on compactions, see . + + Try to make do with one column family if you can in your schemas. Only introduce a + second and third column family in the case where data access is usually column scoped; + i.e. you query one column family or the other but usually not both at the one time. + +
Cardinality of ColumnFamilies + Where multiple ColumnFamilies exist in a single table, be aware of the cardinality (i.e., number of rows). + If ColumnFamilyA has 1 million rows and ColumnFamilyB has 1 billion rows, ColumnFamilyA's data will likely be spread + across many, many regions (and RegionServers). This makes mass scans for ColumnFamilyA less efficient. + +
+
+
Rowkey Design +
+ + Monotonically Increasing Row Keys/Timeseries Data + + + In the HBase chapter of Tom White's book Hadoop: The Definitive Guide (O'Reilly) there is a an optimization note on watching out for a phenomenon where an import process walks in lock-step with all clients in concert pounding one of the table's regions (and thus, a single node), then moving onto the next region, etc. With monotonically increasing row-keys (i.e., using a timestamp), this will happen. See this comic by IKai Lan on why monotonically increasing row keys are problematic in BigTable-like datastores: + monotonically increasing values are bad. The pile-up on a single region brought on + by monotonically increasing keys can be mitigated by randomizing the input records to not be in sorted order, but in general it's best to avoid using a timestamp or a sequence (e.g. 1, 2, 3) as the row-key. + + + + If you do need to upload time series data into HBase, you should + study OpenTSDB as a + successful example. It has a page describing the schema it uses in + HBase. The key format in OpenTSDB is effectively [metric_type][event_timestamp], which would appear at first glance to contradict the previous advice about not using a timestamp as the key. However, the difference is that the timestamp is not in the lead position of the key, and the design assumption is that there are dozens or hundreds (or more) of different metric types. Thus, even with a continual stream of input data with a mix of metric types, the Puts are distributed across various points of regions in the table. + +
+
+ Try to minimize row and column sizes + Or why are my StoreFile indices large? + In HBase, values are always freighted with their coordinates; as a + cell value passes through the system, it'll be accompanied by its + row, column name, and timestamp - always. If your rows and column names + are large, especially compared to the size of the cell value, then + you may run up against some interesting scenarios. One such is + the case described by Marc Limotte at the tail of + HBASE-3551 + (recommended!). + Therein, the indices that are kept on HBase storefiles () + to facilitate random access may end up occupyng large chunks of the HBase + allotted RAM because the cell value coordinates are large. + Mark in the above cited comment suggests upping the block size so + entries in the store file index happen at a larger interval or + modify the table schema so it makes for smaller rows and column + names. + Compression will also make for larger indices. See + the thread a question storefileIndexSize + up on the user mailing list. + + Most of the time small inefficiencies don't matter all that much. Unfortunately, + this is a case where they do. Whatever patterns are selected for ColumnFamilies, attributes, and rowkeys they could be repeated + several billion times in your data. + See for more information on HBase stores data internally to see why this is important. +
Column Families + Try to keep the ColumnFamily names as small as possible, preferably one character (e.g. "d" for data/default). + + See for more information on HBase stores data internally to see why this is important. +
+
Attributes + Although verbose attribute names (e.g., "myVeryImportantAttribute") are easier to read, prefer shorter attribute names (e.g., "via") + to store in HBase. + + See for more information on HBase stores data internally to see why this is important. +
+
Rowkey Length + Keep them as short as is reasonable such that they can still be useful for required data access (e.g., Get vs. Scan). + A short key that is useless for data access is not better than a longer key with better get/scan properties. Expect tradeoffs + when designing rowkeys. + +
+
Byte Patterns + A long is 8 bytes. You can store an unsigned number up to 18,446,744,073,709,551,615 in those eight bytes. + If you stored this number as a String -- presuming a byte per character -- you need nearly 3x the bytes. + + Not convinced? Below is some sample code that you can run on your own. + +// long +// +long l = 1234567890L; +byte[] lb = Bytes.toBytes(l); +System.out.println("long bytes length: " + lb.length); // returns 8 + +String s = "" + l; +byte[] sb = Bytes.toBytes(s); +System.out.println("long as string length: " + sb.length); // returns 10 + +// hash +// +MessageDigest md = MessageDigest.getInstance("MD5"); +byte[] digest = md.digest(Bytes.toBytes(s)); +System.out.println("md5 digest bytes length: " + digest.length); // returns 16 + +String sDigest = new String(digest); +byte[] sbDigest = Bytes.toBytes(sDigest); +System.out.println("md5 digest as string length: " + sbDigest.length); // returns 26 + + +
+ +
+
Reverse Timestamps + A common problem in database processing is quickly finding the most recent version of a value. A technique using reverse timestamps + as a part of the key can help greatly with a special case of this problem. Also found in the HBase chapter of Tom White's book Hadoop: The Definitive Guide (O'Reilly), + the technique involves appending (Long.MAX_VALUE - timestamp) to the end of any key, e.g., [key][reverse_timestamp]. + + The most recent value for [key] in a table can be found by performing a Scan for [key] and obtaining the first record. Since HBase keys + are in sorted order, this key sorts before any older row-keys for [key] and thus is first. + + This technique would be used instead of using HBase Versioning where the intent is to hold onto all versions + "forever" (or a very long time) and at the same time quickly obtain access to any other version by using the same Scan technique. + +
+
+ Rowkeys and ColumnFamilies + Rowkeys are scoped to ColumnFamilies. Thus, the same rowkey could exist in each ColumnFamily that exists in a table without collision. + +
+
Immutability of Rowkeys + Rowkeys cannot be changed. The only way they can be "changed" in a table is if the row is deleted and then re-inserted. + This is a fairly common question on the HBase dist-list so it pays to get the rowkeys right the first time (and/or before you've + inserted a lot of data). + +
+
Relationship Between RowKeys and Region Splits + If you pre-split your table, it is critical to understand how your rowkey will be distributed across + the region boundaries. As an example of why this is important, consider the example of using displayable hex characters as the + lead position of the key (e.g., ""0000000000000000" to "ffffffffffffffff"). Running those key ranges through Bytes.split + (which is the split strategy used when creating regions in HBaseAdmin.createTable(byte[] startKey, byte[] endKey, numRegions) + for 10 regions will generate the following splits... + + + +48 48 48 48 48 48 48 48 48 48 48 48 48 48 48 48 // 0 +54 -10 -10 -10 -10 -10 -10 -10 -10 -10 -10 -10 -10 -10 -10 -10 // 6 +61 -67 -67 -67 -67 -67 -67 -67 -67 -67 -67 -67 -67 -67 -67 -68 // = +68 -124 -124 -124 -124 -124 -124 -124 -124 -124 -124 -124 -124 -124 -124 -126 // D +75 75 75 75 75 75 75 75 75 75 75 75 75 75 75 72 // K +82 18 18 18 18 18 18 18 18 18 18 18 18 18 18 14 // R +88 -40 -40 -40 -40 -40 -40 -40 -40 -40 -40 -40 -40 -40 -40 -44 // X +95 -97 -97 -97 -97 -97 -97 -97 -97 -97 -97 -97 -97 -97 -97 -102 // _ +102 102 102 102 102 102 102 102 102 102 102 102 102 102 102 102 // f + + ... (note: the lead byte is listed to the right as a comment.) Given that the first split is a '0' and the last split is an 'f', + everything is great, right? Not so fast. + + The problem is that all the data is going to pile up in the first 2 regions and the last region thus creating a "lumpy" (and + possibly "hot") region problem. To understand why, refer to an ASCII Table. + '0' is byte 48, and 'f' is byte 102, but there is a huge gap in byte values (bytes 58 to 96) that will never appear in this + keyspace because the only values are [0-9] and [a-f]. Thus, the middle regions regions will + never be used. To make pre-spliting work with this example keyspace, a custom definition of splits (i.e., and not relying on the + built-in split method) is required. + + Lesson #1: Pre-splitting tables is generally a best practice, but you need to pre-split them in such a way that all the + regions are accessible in the keyspace. While this example demonstrated the problem with a hex-key keyspace, the same problem can happen + with any keyspace. Know your data. + + Lesson #2: While generally not advisable, using hex-keys (and more generally, displayable data) can still work with pre-split + tables as long as all the created regions are accessible in the keyspace. + + To conclude this example, the following is an example of how appropriate splits can be pre-created for hex-keys:. + +public static boolean createTable(HBaseAdmin admin, HTableDescriptor table, byte[][] splits) +throws IOException { + try { + admin.createTable( table, splits ); + return true; + } catch (TableExistsException e) { + logger.info("table " + table.getNameAsString() + " already exists"); + // the table already exists... + return false; + } +} + +public static byte[][] getHexSplits(String startKey, String endKey, int numRegions) { + byte[][] splits = new byte[numRegions-1][]; + BigInteger lowestKey = new BigInteger(startKey, 16); + BigInteger highestKey = new BigInteger(endKey, 16); + BigInteger range = highestKey.subtract(lowestKey); + BigInteger regionIncrement = range.divide(BigInteger.valueOf(numRegions)); + lowestKey = lowestKey.add(regionIncrement); + for(int i=0; i < numRegions-1;i++) { + BigInteger key = lowestKey.add(regionIncrement.multiply(BigInteger.valueOf(i))); + byte[] b = String.format("%016x", key).getBytes(); + splits[i] = b; + } + return splits; +} +
+
+
+ + Number of Versions + +
Maximum Number of Versions + The maximum number of row versions to store is configured per column + family via HColumnDescriptor. + The default for max versions is 3. + This is an important parameter because as described in + section HBase does not overwrite row values, but rather + stores different values per row by time (and qualifier). Excess versions are removed during major + compactions. The number of max versions may need to be increased or decreased depending on application needs. + + It is not recommended setting the number of max versions to an exceedingly high level (e.g., hundreds or more) unless those old values are + very dear to you because this will greatly increase StoreFile size. + +
+
+ + Minimum Number of Versions + + Like maximum number of row versions, the minimum number of row versions to keep is configured per column + family via HColumnDescriptor. + The default for min versions is 0, which means the feature is disabled. + The minimum number of row versions parameter is used together with the time-to-live parameter and can be combined with the + number of row versions parameter to allow configurations such as + "keep the last T minutes worth of data, at most N versions, but keep at least M versions around" + (where M is the value for minimum number of row versions, M<N). + This parameter should only be set when time-to-live is enabled for a column family and must be less than the + number of row versions. + +
+
+
+ + Supported Datatypes + + HBase supports a "bytes-in/bytes-out" interface via Put and + Result, so anything that can be + converted to an array of bytes can be stored as a value. Input could be strings, numbers, complex objects, or even images as long as they can rendered as bytes. + + There are practical limits to the size of values (e.g., storing 10-50MB objects in HBase would probably be too much to ask); + search the mailling list for conversations on this topic. All rows in HBase conform to the datamodel, and + that includes versioning. Take that into consideration when making your design, as well as block size for the ColumnFamily. + +
+ Counters + + One supported datatype that deserves special mention are "counters" (i.e., the ability to do atomic increments of numbers). See + Increment in HTable. + + Synchronization on counters are done on the RegionServer, not in the client. + +
+
+
Joins + If you have multiple tables, don't forget to factor in the potential for into the schema design. + +
+
+ Time To Live (TTL) + ColumnFamilies can set a TTL length in seconds, and HBase will automatically delete rows once the expiration time is reached. + This applies to all versions of a row - even the current one. The TTL time encoded in the HBase for the row is specified in UTC. + + See HColumnDescriptor for more information. + +
+
+ + Keeping Deleted Cells + + ColumnFamilies can optionally keep deleted cells. That means deleted cells can still be retrieved with + Get or + Scan operations, + as long these operations have a time range specified that ends before the timestamp of any delete that would affect the cells. + This allows for point in time queries even in the presence of deletes. + + + Deleted cells are still subject to TTL and there will never be more than "maximum number of versions" deleted cells. + A new "raw" scan options returns all deleted rows and the delete markers. + + See HColumnDescriptor for more information. + +
+
+ + Secondary Indexes and Alternate Query Paths + + This section could also be titled "what if my table rowkey looks like this but I also want to query my table like that." + A common example on the dist-list is where a row-key is of the format "user-timestamp" but there are reporting requirements on activity across users for certain + time ranges. Thus, selecting by user is easy because it is in the lead position of the key, but time is not. + + There is no single answer on the best way to handle this because it depends on... + + Number of users + Data size and data arrival rate + Flexibility of reporting requirements (e.g., completely ad-hoc date selection vs. pre-configured ranges) + Desired execution speed of query (e.g., 90 seconds may be reasonable to some for an ad-hoc report, whereas it may be too long for others) + + ... and solutions are also influenced by the size of the cluster and how much processing power you have to throw at the solution. + Common techniques are in sub-sections below. This is a comprehensive, but not exhaustive, list of approaches. + + It should not be a surprise that secondary indexes require additional cluster space and processing. + This is precisely what happens in an RDBMS because the act of creating an alternate index requires both space and processing cycles to update. RBDMS products + are more advanced in this regard to handle alternative index management out of the box. However, HBase scales better at larger data volumes, so this is a feature trade-off. + + Pay attention to when implementing any of these approaches. + Additionally, see the David Butler response in this dist-list thread HBase, mail # user - Stargate+hbase + +
+ + Filter Query + + Depending on the case, it may be appropriate to use . In this case, no secondary index is created. + However, don't try a full-scan on a large table like this from an application (i.e., single-threaded client). + +
+
+ + Periodic-Update Secondary Index + + A secondary index could be created in an other table which is periodically updated via a MapReduce job. The job could be executed intra-day, but depending on + load-strategy it could still potentially be out of sync with the main data table. + See for more information. +
+
+ + Dual-Write Secondary Index + + Another strategy is to build the secondary index while publishing data to the cluster (e.g., write to data table, write to index table). + If this is approach is taken after a data table already exists, then bootstrapping will be needed for the secondary index with a MapReduce job (see ). +
+
+ + Summary Tables + + Where time-ranges are very wide (e.g., year-long report) and where the data is voluminous, summary tables are a common approach. + These would be generated with MapReduce jobs into another table. + See for more information. +
+
+ + Coprocessor Secondary Index + + Coprocessors act like RDBMS triggers. These were added in 0.92. For more information, see + +
+
+
Schema Design Smackdown + This section will describe common schema design questions that appear on the dist-list. These are + general guidelines and not laws - each application must consider its own needs. + +
Rows vs. Versions + A common question is whether one should prefer rows or HBase's built-in-versioning. The context is typically where there are + "a lot" of versions of a row to be retained (e.g., where it is significantly above the HBase default of 3 max versions). The + rows-approach would require storing a timstamp in some portion of the rowkey so that they would not overwite with each successive update. + + Preference: Rows (generally speaking). + +
+
Rows vs. Columns + Another common question is whether one should prefer rows or columns. The context is typically in extreme cases of wide + tables, such as having 1 row with 1 million attributes, or 1 million rows with 1 columns apiece. + + Preference: Rows (generally speaking). To be clear, this guideline is in the context is in extremely wide cases, not in the + standard use-case where one needs to store a few dozen or hundred columns. But there is also a middle path between these two + options, and that is "Rows as Columns." + +
+
Rows as Columns + The middle path between Rows vs. Columns is packing data that would be a separate row into columns, for certain rows. + OpenTSDB is the best example of this case where a single row represents a defined time-range, and then discrete events are treated as + columns. This approach is often more complex, and may require the additional complexity of re-writing your data, but has the + advantage of being I/O efficient. For an overview of this approach, see + Lessons Learned from OpenTSDB + from HBaseCon2012. + +
+ +
+
Operational and Performance Configuration Options + See the Performance section for more information operational and performance + schema design options, such as Bloom Filters, Table-configured regionsizes, compression, and blocksizes. + +
+ +
Constraints + HBase currently supports 'constraints' in traditional (SQL) database parlance. The advised usage for Constraints is in enforcing business rules for attributes in the table (eg. make sure values are in the range 1-10). + Constraints could also be used to enforce referential integrity, but this is strongly discouraged as it will dramatically decrease the write throughput of the tables where integrity checking is enabled. + Extensive documentation on using Constraints can be found at: Constraint since version 0.94. + +
+ + + + + HBase and MapReduce + See + HBase and MapReduce up in javadocs. + Start there. Below is some additional help. + For more information about MapReduce (i.e., the framework in general), see the + Hadoop MapReduce Tutorial. +
+ Map-Task Spitting +
+ The Default HBase MapReduce Splitter + When TableInputFormat + is used to source an HBase table in a MapReduce job, + its splitter will make a map task for each region of the table. + Thus, if there are 100 regions in the table, there will be + 100 map-tasks for the job - regardless of how many column families are selected in the Scan. +
+
+ Custom Splitters + For those interested in implementing custom splitters, see the method getSplits in + TableInputFormatBase. + That is where the logic for map-task assignment resides. + +
+
+
+ HBase MapReduce Examples +
+ HBase MapReduce Read Example + The following is an example of using HBase as a MapReduce source in read-only manner. Specifically, + there is a Mapper instance but no Reducer, and nothing is being emitted from the Mapper. There job would be defined + as follows... + +Configuration config = HBaseConfiguration.create(); +Job job = new Job(config, "ExampleRead"); +job.setJarByClass(MyReadJob.class); // class that contains mapper + +Scan scan = new Scan(); +scan.setCaching(500); // 1 is the default in Scan, which will be bad for MapReduce jobs +scan.setCacheBlocks(false); // don't set to true for MR jobs +// set other scan attrs +... + +TableMapReduceUtil.initTableMapperJob( + tableName, // input HBase table name + scan, // Scan instance to control CF and attribute selection + MyMapper.class, // mapper + null, // mapper output key + null, // mapper output value + job); +job.setOutputFormatClass(NullOutputFormat.class); // because we aren't emitting anything from mapper + +boolean b = job.waitForCompletion(true); +if (!b) { + throw new IOException("error with job!"); +} + + ...and the mapper instance would extend TableMapper... + +public static class MyMapper extends TableMapper<Text, Text> { + + public void map(ImmutableBytesWritable row, Result value, Context context) throws InterruptedException, IOException { + // process data for the row from the Result instance. + } +} + + +
+
+ HBase MapReduce Read/Write Example + The following is an example of using HBase both as a source and as a sink with MapReduce. + This example will simply copy data from one table to another. + +Configuration config = HBaseConfiguration.create(); +Job job = new Job(config,"ExampleReadWrite"); +job.setJarByClass(MyReadWriteJob.class); // class that contains mapper + +Scan scan = new Scan(); +scan.setCaching(500); // 1 is the default in Scan, which will be bad for MapReduce jobs +scan.setCacheBlocks(false); // don't set to true for MR jobs +// set other scan attrs + +TableMapReduceUtil.initTableMapperJob( + sourceTable, // input table + scan, // Scan instance to control CF and attribute selection + MyMapper.class, // mapper class + null, // mapper output key + null, // mapper output value + job); +TableMapReduceUtil.initTableReducerJob( + targetTable, // output table + null, // reducer class + job); +job.setNumReduceTasks(0); + +boolean b = job.waitForCompletion(true); +if (!b) { + throw new IOException("error with job!"); +} + + An explanation is required of what TableMapReduceUtil is doing, especially with the reducer. + TableOutputFormat is being used + as the outputFormat class, and several parameters are being set on the config (e.g., TableOutputFormat.OUTPUT_TABLE), as + well as setting the reducer output key to ImmutableBytesWritable and reducer value to Writable. + These could be set by the programmer on the job and conf, but TableMapReduceUtil tries to make things easier. + The following is the example mapper, which will create a Put and matching the input Result + and emit it. Note: this is what the CopyTable utility does. + + +public static class MyMapper extends TableMapper<ImmutableBytesWritable, Put> { + + public void map(ImmutableBytesWritable row, Result value, Context context) throws IOException, InterruptedException { + // this example is just copying the data from the source table... + context.write(row, resultToPut(row,value)); + } + + private static Put resultToPut(ImmutableBytesWritable key, Result result) throws IOException { + Put put = new Put(key.get()); + for (KeyValue kv : result.raw()) { + put.add(kv); + } + return put; + } +} + + There isn't actually a reducer step, so TableOutputFormat takes care of sending the Put + to the target table. + + This is just an example, developers could choose not to use TableOutputFormat and connect to the + target table themselves. + + +
+
+ HBase MapReduce Read/Write Example With Multi-Table Output + TODO: example for MultiTableOutputFormat. + +
+
+ HBase MapReduce Summary to HBase Example + The following example uses HBase as a MapReduce source and sink with a summarization step. This example will + count the number of distinct instances of a value in a table and write those summarized counts in another table. + +Configuration config = HBaseConfiguration.create(); +Job job = new Job(config,"ExampleSummary"); +job.setJarByClass(MySummaryJob.class); // class that contains mapper and reducer + +Scan scan = new Scan(); +scan.setCaching(500); // 1 is the default in Scan, which will be bad for MapReduce jobs +scan.setCacheBlocks(false); // don't set to true for MR jobs +// set other scan attrs + +TableMapReduceUtil.initTableMapperJob( + sourceTable, // input table + scan, // Scan instance to control CF and attribute selection + MyMapper.class, // mapper class + Text.class, // mapper output key + IntWritable.class, // mapper output value + job); +TableMapReduceUtil.initTableReducerJob( + targetTable, // output table + MyTableReducer.class, // reducer class + job); +job.setNumReduceTasks(1); // at least one, adjust as required + +boolean b = job.waitForCompletion(true); +if (!b) { + throw new IOException("error with job!"); +} + + In this example mapper a column with a String-value is chosen as the value to summarize upon. + This value is used as the key to emit from the mapper, and an IntWritable represents an instance counter. + +public static class MyMapper extends TableMapper<Text, IntWritable> { + + private final IntWritable ONE = new IntWritable(1); + private Text text = new Text(); + + public void map(ImmutableBytesWritable row, Result value, Context context) throws IOException, InterruptedException { + String val = new String(value.getValue(Bytes.toBytes("cf"), Bytes.toBytes("attr1"))); + text.set(val); // we can only emit Writables... + + context.write(text, ONE); + } +} + + In the reducer, the "ones" are counted (just like any other MR example that does this), and then emits a Put. + +public static class MyTableReducer extends TableReducer<Text, IntWritable, ImmutableBytesWritable> { + + public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { + int i = 0; + for (IntWritable val : values) { + i += val.get(); + } + Put put = new Put(Bytes.toBytes(key.toString())); + put.add(Bytes.toBytes("cf"), Bytes.toBytes("count"), Bytes.toBytes(i)); + + context.write(null, put); + } +} + + +
+
+ HBase MapReduce Summary to File Example + This very similar to the summary example above, with exception that this is using HBase as a MapReduce source + but HDFS as the sink. The differences are in the job setup and in the reducer. The mapper remains the same. + + +Configuration config = HBaseConfiguration.create(); +Job job = new Job(config,"ExampleSummaryToFile"); +job.setJarByClass(MySummaryFileJob.class); // class that contains mapper and reducer + +Scan scan = new Scan(); +scan.setCaching(500); // 1 is the default in Scan, which will be bad for MapReduce jobs +scan.setCacheBlocks(false); // don't set to true for MR jobs +// set other scan attrs + +TableMapReduceUtil.initTableMapperJob( + sourceTable, // input table + scan, // Scan instance to control CF and attribute selection + MyMapper.class, // mapper class + Text.class, // mapper output key + IntWritable.class, // mapper output value + job); +job.setReducerClass(MyReducer.class); // reducer class +job.setNumReduceTasks(1); // at least one, adjust as required +FileOutputFormat.setOutputPath(job, new Path("/tmp/mr/mySummaryFile")); // adjust directories as required + +boolean b = job.waitForCompletion(true); +if (!b) { + throw new IOException("error with job!"); +} + + As stated above, the previous Mapper can run unchanged with this example. + As for the Reducer, it is a "generic" Reducer instead of extending TableMapper and emitting Puts. + + public static class MyReducer extends Reducer<Text, IntWritable, Text, IntWritable> { + + public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { + int i = 0; + for (IntWritable val : values) { + i += val.get(); + } + context.write(key, new IntWritable(i)); + } +} + +
+
+ HBase MapReduce Summary to HBase Without Reducer + It is also possible to perform summaries without a reducer - if you use HBase as the reducer. + + An HBase target table would need to exist for the job summary. The HTable method incrementColumnValue + would be used to atomically increment values. From a performance perspective, it might make sense to keep a Map + of values with their values to be incremeneted for each map-task, and make one update per key at during the + cleanup method of the mapper. However, your milage may vary depending on the number of rows to be processed and + unique keys. + + In the end, the summary results are in HBase. + +
+
+ HBase MapReduce Summary to RDBMS + Sometimes it is more appropriate to generate summaries to an RDBMS. For these cases, it is possible + to generate summaries directly to an RDBMS via a custom reducer. The setup method + can connect to an RDBMS (the connection information can be passed via custom parameters in the context) and the + cleanup method can close the connection. + + It is critical to understand that number of reducers for the job affects the summarization implementation, and + you'll have to design this into your reducer. Specifically, whether it is designed to run as a singleton (one reducer) + or multiple reducers. Neither is right or wrong, it depends on your use-case. Recognize that the more reducers that + are assigned to the job, the more simultaneous connections to the RDBMS will be created - this will scale, but only to a point. + + + public static class MyRdbmsReducer extends Reducer<Text, IntWritable, Text, IntWritable> { + + private Connection c = null; + + public void setup(Context context) { + // create DB connection... + } + + public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { + // do summarization + // in this example the keys are Text, but this is just an example + } + + public void cleanup(Context context) { + // close db connection + } + +} + + In the end, the summary results are written to your RDBMS table/s. + +
+ +
+
+ Accessing Other HBase Tables in a MapReduce Job + Although the framework currently allows one HBase table as input to a + MapReduce job, other HBase tables can + be accessed as lookup tables, etc., in a + MapReduce job via creating an HTable instance in the setup method of the Mapper. + public class MyMapper extends TableMapper<Text, LongWritable> { + private HTable myOtherTable; + + public void setup(Context context) { + myOtherTable = new HTable("myOtherTable"); + } + + public void map(ImmutableBytesWritable row, Result value, Context context) throws IOException, InterruptedException { + // process Result... + // use 'myOtherTable' for lookups + } + + + +
+
+ Speculative Execution + It is generally advisable to turn off speculative execution for + MapReduce jobs that use HBase as a source. This can either be done on a + per-Job basis through properties, on on the entire cluster. Especially + for longer running jobs, speculative execution will create duplicate + map-tasks which will double-write your data to HBase; this is probably + not what you want. + + See for more information. + +
+ + + + + + Architecture +
+ Overview +
+ NoSQL? + HBase is a type of "NoSQL" database. "NoSQL" is a general term meaning that the database isn't an RDBMS which + supports SQL as its primary access language, but there are many types of NoSQL databases: BerkeleyDB is an + example of a local NoSQL database, whereas HBase is very much a distributed database. Technically speaking, + HBase is really more a "Data Store" than "Data Base" because it lacks many of the features you find in an RDBMS, + such as typed columns, secondary indexes, triggers, and advanced query languages, etc. + + However, HBase has many features which supports both linear and modular scaling. HBase clusters expand + by adding RegionServers that are hosted on commodity class servers. If a cluster expands from 10 to 20 + RegionServers, for example, it doubles both in terms of storage and as well as processing capacity. + RDBMS can scale well, but only up to a point - specifically, the size of a single database server - and for the best + performance requires specialized hardware and storage devices. HBase features of note are: + + Strongly consistent reads/writes: HBase is not an "eventually consistent" DataStore. This + makes it very suitable for tasks such as high-speed counter aggregation. + Automatic sharding: HBase tables are distributed on the cluster via regions, and regions are + automatically split and re-distributed as your data grows. + Automatic RegionServer failover + Hadoop/HDFS Integration: HBase supports HDFS out of the box as its distributed file system. + MapReduce: HBase supports massively parallelized processing via MapReduce for using HBase as both + source and sink. + Java Client API: HBase supports an easy to use Java API for programmatic access. + Thrift/REST API: HBase also supports Thrift and REST for non-Java front-ends. + Block Cache and Bloom Filters: HBase supports a Block Cache and Bloom Filters for high volume query optimization. + Operational Management: HBase provides build-in web-pages for operational insight as well as JMX metrics. + + +
+ +
+ When Should I Use HBase? + HBase isn't suitable for every problem. + First, make sure you have enough data. If you have hundreds of millions or billions of rows, then + HBase is a good candidate. If you only have a few thousand/million rows, then using a traditional RDBMS + might be a better choice due to the fact that all of your data might wind up on a single node (or two) and + the rest of the cluster may be sitting idle. + + Second, make sure you can live without all the extra features that an RDBMS provides (e.g., typed columns, + secondary indexes, transactions, advanced query languages, etc.) An application built against an RDBMS cannot be + "ported" to HBase by simply changing a JDBC driver, for example. Consider moving from an RDBMS to HBase as a + complete redesign as opposed to a port. + + Third, make sure you have enough hardware. Even HDFS doesn't do well with anything less than + 5 DataNodes (due to things such as HDFS block replication which has a default of 3), plus a NameNode. + + HBase can run quite well stand-alone on a laptop - but this should be considered a development + configuration only. + +
+
+ What Is The Difference Between HBase and Hadoop/HDFS? + HDFS is a distributed file system that is well suited for the storage of large files. + It's documentation states that it is not, however, a general purpose file system, and does not provide fast individual record lookups in files. + HBase, on the other hand, is built on top of HDFS and provides fast record lookups (and updates) for large tables. + This can sometimes be a point of conceptual confusion. HBase internally puts your data in indexed "StoreFiles" that exist + on HDFS for high-speed lookups. See the and the rest of this chapter for more information on how HBase achieves its goals. + +
+
+ +
+ Catalog Tables + The catalog tables -ROOT- and .META. exist as HBase tables. They are filtered out + of the HBase shell's list command, but they are in fact tables just like any other. + +
+ ROOT + -ROOT- keeps track of where the .META. table is. The -ROOT- table structure is as follows: + + Key: + + .META. region key (.META.,,1) + + + Values: + + info:regioninfo (serialized HRegionInfo + instance of .META.) + info:server (server:port of the RegionServer holding .META.) + info:serverstartcode (start-time of the RegionServer process holding .META.) + + +
+
+ META + The .META. table keeps a list of all regions in the system. The .META. table structure is as follows: + + Key: + + Region key of the format ([table],[region start key],[region id]) + + + Values: + + info:regioninfo (serialized + HRegionInfo instance for this region) + + info:server (server:port of the RegionServer containing this region) + info:serverstartcode (start-time of the RegionServer process containing this region) + + + When a table is in the process of splitting two other columns will be created, info:splitA and info:splitB + which represent the two daughter regions. The values for these columns are also serialized HRegionInfo instances. + After the region has been split eventually this row will be deleted. + + Notes on HRegionInfo: the empty key is used to denote table start and table end. A region with an empty start key + is the first region in a table. If region has both an empty start and an empty end key, it's the only region in the table + + In the (hopefully unlikely) event that programmatic processing of catalog metadata is required, see the + Writables utility. + +
+
+ Startup Sequencing + The META location is set in ROOT first. Then META is updated with server and startcode values. + + For information on region-RegionServer assignment, see . + +
+
+ +
+ Client + The HBase client + HTable + is responsible for finding RegionServers that are serving the + particular row range of interest. It does this by querying + the .META. and -ROOT- catalog tables + (TODO: Explain). After locating the required + region(s), the client directly contacts + the RegionServer serving that region (i.e., it does not go + through the master) and issues the read or write request. + This information is cached in the client so that subsequent requests + need not go through the lookup process. Should a region be reassigned + either by the master load balancer or because a RegionServer has died, + the client will requery the catalog tables to determine the new + location of the user region. + + See for more information about the impact of the Master on HBase Client + communication. + + Administrative functions are handled through HBaseAdmin + +
Connections + For connection configuration information, see . + + HTable + instances are not thread-safe. Only one thread use an instance of HTable at any given + time. When creating HTable instances, it is advisable to use the same HBaseConfiguration +instance. This will ensure sharing of ZooKeeper and socket instances to the RegionServers +which is usually what you want. For example, this is preferred: + HBaseConfiguration conf = HBaseConfiguration.create(); +HTable table1 = new HTable(conf, "myTable"); +HTable table2 = new HTable(conf, "myTable"); + as opposed to this: + HBaseConfiguration conf1 = HBaseConfiguration.create(); +HTable table1 = new HTable(conf1, "myTable"); +HBaseConfiguration conf2 = HBaseConfiguration.create(); +HTable table2 = new HTable(conf2, "myTable"); + For more information about how connections are handled in the HBase client, + see HConnectionManager. + +
Connection Pooling + For applications which require high-end multithreaded access (e.g., web-servers or application servers that may serve many application threads + in a single JVM), one solution is HTablePool. + But as written currently, it is difficult to control client resource consumption when using HTablePool. + + + Another solution is to precreate an HConnection using + HConnectionManager.createConnection(Configuration) as + well as an ExecutorService; then use the + HTable(byte[], HConnection, ExecutorService) + constructor to create HTable instances on demand. + This construction is very lightweight and resources are controlled/shared if you go this route. + +
+
+
WriteBuffer and Batch Methods + If is turned off on + HTable, + Puts are sent to RegionServers when the writebuffer + is filled. The writebuffer is 2MB by default. Before an HTable instance is + discarded, either close() or + flushCommits() should be invoked so Puts + will not be lost. + + Note: htable.delete(Delete); does not go in the writebuffer! This only applies to Puts. + + For additional information on write durability, review the ACID semantics page. + + For fine-grained control of batching of + Puts or Deletes, + see the batch methods on HTable. + +
+
External Clients + Information on non-Java clients and custom protocols is covered in + +
+
RowLocks + RowLocks are still + in the client API however they are discouraged because if not managed properly these can + lock up the RegionServers. + + There is an oustanding ticket HBASE-2332 to + remove this feature from the client. + +
+
+ +
Client Request Filters + Get and Scan instances can be + optionally configured with filters which are applied on the RegionServer. + + Filters can be confusing because there are many different types, and it is best to approach them by understanding the groups + of Filter functionality. + +
Structural + Structural Filters contain other Filters. +
FilterList + FilterList + represents a list of Filters with a relationship of FilterList.Operator.MUST_PASS_ALL or + FilterList.Operator.MUST_PASS_ONE between the Filters. The following example shows an 'or' between two + Filters (checking for either 'my value' or 'my other value' on the same attribute). + +FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ONE); +SingleColumnValueFilter filter1 = new SingleColumnValueFilter( + cf, + column, + CompareOp.EQUAL, + Bytes.toBytes("my value") + ); +list.add(filter1); +SingleColumnValueFilter filter2 = new SingleColumnValueFilter( + cf, + column, + CompareOp.EQUAL, + Bytes.toBytes("my other value") + ); +list.add(filter2); +scan.setFilter(list); + + +
+
+
Column Value +
SingleColumnValueFilter + SingleColumnValueFilter + can be used to test column values for equivalence (CompareOp.EQUAL + ), inequality (CompareOp.NOT_EQUAL), or ranges + (e.g., CompareOp.GREATER). The folowing is example of testing equivalence a column to a String value "my value"... + +SingleColumnValueFilter filter = new SingleColumnValueFilter( + cf, + column, + CompareOp.EQUAL, + Bytes.toBytes("my value") + ); +scan.setFilter(filter); + + +
+
+
Column Value Comparators + There are several Comparator classes in the Filter package that deserve special mention. + These Comparators are used in concert with other Filters, such as . + +
RegexStringComparator + RegexStringComparator + supports regular expressions for value comparisons. + +RegexStringComparator comp = new RegexStringComparator("my."); // any value that starts with 'my' +SingleColumnValueFilter filter = new SingleColumnValueFilter( + cf, + column, + CompareOp.EQUAL, + comp + ); +scan.setFilter(filter); + + See the Oracle JavaDoc for supported RegEx patterns in Java. + +
+
SubstringComparator + SubstringComparator + can be used to determine if a given substring exists in a value. The comparison is case-insensitive. + + +SubstringComparator comp = new SubstringComparator("y val"); // looking for 'my value' +SingleColumnValueFilter filter = new SingleColumnValueFilter( + cf, + column, + CompareOp.EQUAL, + comp + ); +scan.setFilter(filter); + +
+
BinaryPrefixComparator + See BinaryPrefixComparator. +
+
BinaryComparator + See BinaryComparator. +
+
+
KeyValue Metadata + As HBase stores data internally as KeyValue pairs, KeyValue Metadata Filters evaluate the existence of keys (i.e., ColumnFamily:Column qualifiers) + for a row, as opposed to values the previous section. + +
FamilyFilter + FamilyFilter can be used + to filter on the ColumnFamily. It is generally a better idea to select ColumnFamilies in the Scan than to do it with a Filter. +
+
QualifierFilter + QualifierFilter can be used + to filter based on Column (aka Qualifier) name. + +
+
ColumnPrefixFilter + ColumnPrefixFilter can be used + to filter based on the lead portion of Column (aka Qualifier) names. + + A ColumnPrefixFilter seeks ahead to the first column matching the prefix in each row and for each involved column family. It can be used to efficiently + get a subset of the columns in very wide rows. + + Note: The same column qualifier can be used in different column families. This filter returns all matching columns. + + Example: Find all columns in a row and family that start with "abc" + +HTableInterface t = ...; +byte[] row = ...; +byte[] family = ...; +byte[] prefix = Bytes.toBytes("abc"); +Scan scan = new Scan(row, row); // (optional) limit to one row +scan.addFamily(family); // (optional) limit to one family +Filter f = new ColumnPrefixFilter(prefix); +scan.setFilter(f); +scan.setBatch(10); // set this if there could be many columns returned +ResultScanner rs = t.getScanner(scan); +for (Result r = rs.next(); r != null; r = rs.next()) { + for (KeyValue kv : r.raw()) { + // each kv represents a column + } +} +rs.close(); + + +
+
MultipleColumnPrefixFilter + MultipleColumnPrefixFilter behaves like ColumnPrefixFilter + but allows specifying multiple prefixes. + + Like ColumnPrefixFilter, MultipleColumnPrefixFilter efficiently seeks ahead to the first column matching the lowest prefix and also seeks past ranges of columns between prefixes. + It can be used to efficiently get discontinuous sets of columns from very wide rows. + + Example: Find all columns in a row and family that start with "abc" or "xyz" + +HTableInterface t = ...; +byte[] row = ...; +byte[] family = ...; +byte[][] prefixes = new byte[][] {Bytes.toBytes("abc"), Bytes.toBytes("xyz")}; +Scan scan = new Scan(row, row); // (optional) limit to one row +scan.addFamily(family); // (optional) limit to one family +Filter f = new MultipleColumnPrefixFilter(prefixes); +scan.setFilter(f); +scan.setBatch(10); // set this if there could be many columns returned +ResultScanner rs = t.getScanner(scan); +for (Result r = rs.next(); r != null; r = rs.next()) { + for (KeyValue kv : r.raw()) { + // each kv represents a column + } +} +rs.close(); + + +
+
ColumnRangeFilter + A ColumnRangeFilter allows efficient intra row scanning. + + A ColumnRangeFilter can seek ahead to the first matching column for each involved column family. It can be used to efficiently + get a 'slice' of the columns of a very wide row. + i.e. you have a million columns in a row but you only want to look at columns bbbb-bbdd. + + Note: The same column qualifier can be used in different column families. This filter returns all matching columns. + + Example: Find all columns in a row and family between "bbbb" (inclusive) and "bbdd" (inclusive) + +HTableInterface t = ...; +byte[] row = ...; +byte[] family = ...; +byte[] startColumn = Bytes.toBytes("bbbb"); +byte[] endColumn = Bytes.toBytes("bbdd"); +Scan scan = new Scan(row, row); // (optional) limit to one row +scan.addFamily(family); // (optional) limit to one family +Filter f = new ColumnRangeFilter(startColumn, true, endColumn, true); +scan.setFilter(f); +scan.setBatch(10); // set this if there could be many columns returned +ResultScanner rs = t.getScanner(scan); +for (Result r = rs.next(); r != null; r = rs.next()) { + for (KeyValue kv : r.raw()) { + // each kv represents a column + } +} +rs.close(); + + + Note: Introduced in HBase 0.92 +
+
+
RowKey +
RowFilter + It is generally a better idea to use the startRow/stopRow methods on Scan for row selection, however + RowFilter can also be used. +
+
+
Utility +
FirstKeyOnlyFilter + This is primarily used for rowcount jobs. + See FirstKeyOnlyFilter. +
+
+
+ +
Master + HMaster is the implementation of the Master Server. The Master server + is responsible for monitoring all RegionServer instances in the cluster, and is + the interface for all metadata changes. In a distributed cluster, the Master typically runs on the + J Mohamed Zahoor goes into some more detail on the Master Architecture in this blog posting, HBase HMaster Architecture + . + + +
Startup Behavior + If run in a multi-Master environment, all Masters compete to run the cluster. If the active + Master loses its lease in ZooKeeper (or the Master shuts down), then then the remaining Masters jostle to + take over the Master role. + +
+
Runtime Impact + A common dist-list question is what happens to an HBase cluster when the Master goes down. Because the + HBase client talks directly to the RegionServers, the cluster can still function in a "steady + state." Additionally, per ROOT and META exist as HBase tables (i.e., are + not resident in the Master). However, the Master controls critical functions such as RegionServer failover and + completing region splits. So while the cluster can still run for a time without the Master, + the Master should be restarted as soon as possible. + +
+
Interface + The methods exposed by HMasterInterface are primarily metadata-oriented methods: + + Table (createTable, modifyTable, removeTable, enable, disable) + + ColumnFamily (addColumn, modifyColumn, removeColumn) + + Region (move, assign, unassign) + + + For example, when the HBaseAdmin method disableTable is invoked, it is serviced by the Master server. + +
+
Processes + The Master runs several background threads: + +
LoadBalancer + Periodically, and when there are no regions in transition, + a load balancer will run and move regions around to balance the cluster's load. + See for configuring this property. + See for more information on region assignment. + +
+
CatalogJanitor + Periodically checks and cleans up the .META. table. See for more information on META. +
+
+ +
+
RegionServer + HRegionServer is the RegionServer implementation. It is responsible for serving and managing regions. + In a distributed cluster, a RegionServer runs on a . + +
Interface + The methods exposed by HRegionRegionInterface contain both data-oriented and region-maintenance methods: + + Data (get, put, delete, next, etc.) + + Region (splitRegion, compactRegion, etc.) + + + For example, when the HBaseAdmin method majorCompact is invoked on a table, the client is actually iterating through + all regions for the specified table and requesting a major compaction directly to each region. + +
+
Processes + The RegionServer runs a variety of background threads: +
CompactSplitThread + Checks for splits and handle minor compactions. +
+
MajorCompactionChecker + Checks for major compactions. +
+
MemStoreFlusher + Periodically flushes in-memory writes in the MemStore to StoreFiles. +
+
LogRoller + Periodically checks the RegionServer's HLog. +
+
+ +
Coprocessors + Coprocessors were added in 0.92. There is a thorough Blog Overview of CoProcessors + posted. Documentation will eventually move to this reference guide, but the blog is the most current information available at this time. + +
+ +
+ Block Cache +
+ Design + The Block Cache is an LRU cache that contains three levels of block priority to allow for scan-resistance and in-memory ColumnFamilies: + + + Single access priority: The first time a block is loaded from HDFS it normally has this priority and it will be part of the first group to be considered + during evictions. The advantage is that scanned blocks are more likely to get evicted than blocks that are getting more usage. + + Mutli access priority: If a block in the previous priority group is accessed again, it upgrades to this priority. It is thus part of the second group + considered during evictions. + + In-memory access priority: If the block's family was configured to be "in-memory", it will be part of this priority disregarding the number of times it + was accessed. Catalog tables are configured like this. This group is the last one considered during evictions. + + + + For more information, see the LruBlockCache source + +
+
+ Usage + Block caching is enabled by default for all the user tables which means that any read operation will load the LRU cache. This might be good for a large number of use cases, + but further tunings are usually required in order to achieve better performance. An important concept is the + working set size, or WSS, which is: "the amount of memory needed to compute the answer to a problem". + For a website, this would be the data that's needed to answer the queries over a short amount of time. + + The way to calculate how much memory is available in HBase for caching is: + + + number of region servers * heap size * hfile.block.cache.size * 0.85 + + The default value for the block cache is 0.25 which represents 25% of the available heap. The last value (85%) is the default acceptable loading factor in the LRU cache after + which eviction is started. The reason it is included in this equation is that it would be unrealistic to say that it is possible to use 100% of the available memory since this would + make the process blocking from the point where it loads new blocks. Here are some examples: + + + One region server with the default heap size (1GB) and the default block cache size will have 217MB of block cache available. + + 20 region servers with the heap size set to 8GB and a default block cache size will have 34GB of block cache. + + 100 region servers with the heap size set to 24GB and a block cache size of 0.5 will have about 1TB of block cache. + + + Your data isn't the only resident of the block cache, here are others that you may have to take into account: + + + Catalog tables: The -ROOT- and .META. tables are forced into the block cache and have the in-memory priority which means that they are harder to evict. The former never uses + more than a few hundreds of bytes while the latter can occupy a few MBs (depending on the number of regions). + + HFiles indexes: HFile is the file format that HBase uses to store data in HDFS and it contains a multi-layered index in order seek to the data without having to read the whole file. + The size of those indexes is a factor of the block size (64KB by default), the size of your keys and the amount of data you are storing. For big data sets it's not unusual to see numbers around + 1GB per region server, although not all of it will be in cache because the LRU will evict indexes that aren't used. + + Keys: Taking into account only the values that are being stored is missing half the picture since every value is stored along with its keys + (row key, family, qualifier, and timestamp). See . + + Bloom filters: Just like the HFile indexes, those data structures (when enabled) are stored in the LRU. + + + Currently the recommended way to measure HFile indexes and bloom filters sizes is to look at the region server web UI and checkout the relevant metrics. For keys, + sampling can be done by using the HFile command line tool and look for the average key size metric. + + It's generally bad to use block caching when the WSS doesn't fit in memory. This is the case when you have for example 40GB available across all your region servers' block caches + but you need to process 1TB of data. One of the reasons is that the churn generated by the evictions will trigger more garbage collections unnecessarily. Here are two use cases: + + + Fully random reading pattern: This is a case where you almost never access the same row twice within a short amount of time such that the chance of hitting a cached block is close + to 0. Setting block caching on such a table is a waste of memory and CPU cycles, more so that it will generate more garbage to pick up by the JVM. For more information on monitoring GC, + see . + + Mapping a table: In a typical MapReduce job that takes a table in input, every row will be read only once so there's no need to put them into the block cache. The Scan object has + the option of turning this off via the setCaching method (set it to false). You can still keep block caching turned on on this table if you need fast random read access. An example would be + counting the number of rows in a table that serves live traffic, caching every block of that table would create massive churn and would surely evict data that's currently in use. + + +
+
+ +
+ Write Ahead Log (WAL) + +
+ Purpose + + Each RegionServer adds updates (Puts, Deletes) to its write-ahead log (WAL) + first, and then to the for the affected . + This ensures that HBase has durable writes. Without WAL, there is the possibility of data loss in the case of a RegionServer failure + before each MemStore is flushed and new StoreFiles are written. HLog + is the HBase WAL implementation, and there is one HLog instance per RegionServer. + The WAL is in HDFS in /hbase/.logs/ with subdirectories per region. + + For more general information about the concept of write ahead logs, see the Wikipedia + Write-Ahead Log article. + +
+
+ WAL Flushing + TODO (describe). + +
+ +
+ WAL Splitting + +
How edits are recovered from a crashed RegionServer + When a RegionServer crashes, it will lose its ephemeral lease in + ZooKeeper...TODO +
+
+ <varname>hbase.hlog.split.skip.errors</varname> + + When set to true, any error + encountered splitting will be logged, the problematic WAL will be + moved into the .corrupt directory under the hbase + rootdir, and processing will continue. If set to + false, the default, the exception will be propagated and the + split logged as failed. + See HBASE-2958 + When hbase.hlog.split.skip.errors is set to false, we fail the + split but thats it. We need to do more than just fail split + if this flag is set. + +
+ +
+ How EOFExceptions are treated when splitting a crashed + RegionServers' WALs + + If we get an EOF while splitting logs, we proceed with the split + even when hbase.hlog.split.skip.errors == + false. An EOF while reading the last log in the + set of files to split is near-guaranteed since the RegionServer likely + crashed mid-write of a record. But we'll continue even if we got an + EOF reading other than the last file in the set. + For background, see HBASE-2643 + Figure how to deal with eof splitting logs + +
+
+
+ +
+ +
+ Regions + Regions are the basic element of availability and + distribution for tables, and are comprised of a Store per Column Family. The heirarchy of objects + is as follows: + +Table (HBase table) + Region (Regions for the table) + Store (Store per ColumnFamily for each Region for the table) + MemStore (MemStore for each Store for each Region for the table) + StoreFile (StoreFiles for each Store for each Region for the table) + Block (Blocks within a StoreFile within a Store for each Region for the table) + + For a description of what HBase files look like when written to HDFS, see . + + +
+ Region Size + + Determining the "right" region size can be tricky, and there are a few factors + to consider: + + + + HBase scales by having regions across many servers. Thus if + you have 2 regions for 16GB data, on a 20 node machine your data + will be concentrated on just a few machines - nearly the entire + cluster will be idle. This really cant be stressed enough, since a + common problem is loading 200MB data into HBase then wondering why + your awesome 10 node cluster isn't doing anything. + + + + On the other hand, high region count has been known to make things slow. + This is getting better with each release of HBase, but it is probably better to have + 700 regions than 3000 for the same amount of data. + + + + There is not much memory footprint difference between 1 region + and 10 in terms of indexes, etc, held by the RegionServer. + + + + When starting off, it's probably best to stick to the default region-size, perhaps going + smaller for hot tables (or manually split hot regions to spread the load over + the cluster), or go with larger region sizes if your cell sizes tend to be + largish (100k and up). + See for more information on configuration. + +
+ +
+ Region-RegionServer Assignment + This section describes how Regions are assigned to RegionServers. + + +
+ Startup + When HBase starts regions are assigned as follows (short version): + + The Master invokes the AssignmentManager upon startup. + + The AssignmentManager looks at the existing region assignments in META. + + If the region assignment is still valid (i.e., if the RegionServer is still online) + then the assignment is kept. + + If the assignment is invalid, then the LoadBalancerFactory is invoked to assign the + region. The DefaultLoadBalancer will randomly assign the region to a RegionServer. + + META is updated with the RegionServer assignment (if needed) and the RegionServer start codes + (start time of the RegionServer process) upon region opening by the RegionServer. + + + +
+ +
+ Failover + When a RegionServer fails (short version): + + The regions immediately become unavailable because the RegionServer is down. + + The Master will detect that the RegionServer has failed. + + The region assignments will be considered invalid and will be re-assigned just + like the startup sequence. + + + +
+ +
+ Region Load Balancing + + Regions can be periodically moved by the . + +
+ +
+ +
+ Region-RegionServer Locality + Over time, Region-RegionServer locality is achieved via HDFS block replication. + The HDFS client does the following by default when choosing locations to write replicas: + + First replica is written to local node + + Second replica is written to another node in same rack + + Third replica is written to a node in another rack (if sufficient nodes) + + + Thus, HBase eventually achieves locality for a region after a flush or a compaction. + In a RegionServer failover situation a RegionServer may be assigned regions with non-local + StoreFiles (because none of the replicas are local), however as new data is written + in the region, or the table is compacted and StoreFiles are re-written, they will become "local" + to the RegionServer. + + For more information, see HDFS Design on Replica Placement + and also Lars George's blog on HBase and HDFS locality. + +
+ +
+ Region Splits + + Splits run unaided on the RegionServer; i.e. the Master does not + participate. The RegionServer splits a region, offlines the split + region and then adds the daughter regions to META, opens daughters on + the parent's hosting RegionServer and then reports the split to the + Master. See for how to manually manage + splits (and for why you might do this) +
+ Custom Split Policies + The default split policy can be overwritten using a custom RegionSplitPolicy (HBase 0.94+). + Typically a custom split policy should extend HBase's default split policy: ConstantSizeRegionSplitPolicy. + + The policy can set globally through the HBaseConfiguration used or on a per table basis: + +HTableDescriptor myHtd = ...; +myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName()); + + +
+
+ +
+ Store + A Store hosts a MemStore and 0 or more StoreFiles (HFiles). A Store corresponds to a column family for a table for a given region. + +
+ MemStore + The MemStore holds in-memory modifications to the Store. Modifications are KeyValues. + When asked to flush, current memstore is moved to snapshot and is cleared. + HBase continues to serve edits out of new memstore and backing snapshot until flusher reports in that the + flush succeeded. At this point the snapshot is let go. +
+
+ StoreFile (HFile) + StoreFiles are where your data lives. + +
HFile Format + The hfile file format is based on + the SSTable file described in the BigTable [2006] paper and on + Hadoop's tfile + (The unit test suite and the compression harness were taken directly from tfile). + Schubert Zhang's blog post on HFile: A Block-Indexed File Format to Store Sorted Key-Value Pairs makes for a thorough introduction to HBase's hfile. Matteo Bertozzi has also put up a + helpful description, HBase I/O: HFile. + + For more information, see the HFile source code. + Also see for information about the HFile v2 format that was included in 0.92. + +
+
+ HFile Tool + + To view a textualized version of hfile content, you can do use + the org.apache.hadoop.hbase.io.hfile.HFile + tool. Type the following to see usage:$ ${HBASE_HOME}/bin/hbase org.apache.hadoop.hbase.io.hfile.HFile For + example, to view the content of the file + hdfs://10.81.47.41:8020/hbase/TEST/1418428042/DSMP/4759508618286845475, + type the following: $ ${HBASE_HOME}/bin/hbase org.apache.hadoop.hbase.io.hfile.HFile -v -f hdfs://10.81.47.41:8020/hbase/TEST/1418428042/DSMP/4759508618286845475 If + you leave off the option -v to see just a summary on the hfile. See + usage for other things to do with the HFile + tool. +
+
+ StoreFile Directory Structure on HDFS + For more information of what StoreFiles look like on HDFS with respect to the directory structure, see . + +
+
+ +
+ Blocks + StoreFiles are composed of blocks. The blocksize is configured on a per-ColumnFamily basis. + + Compression happens at the block level within StoreFiles. For more information on compression, see . + + For more information on blocks, see the HFileBlock source code. + +
+
+ KeyValue + The KeyValue class is the heart of data storage in HBase. KeyValue wraps a byte array and takes offsets and lengths into passed array + at where to start interpreting the content as KeyValue. + + The KeyValue format inside a byte array is: + + keylength + valuelength + key + value + + + The Key is further decomposed as: + + rowlength + row (i.e., the rowkey) + columnfamilylength + columnfamily + columnqualifier + timestamp + keytype (e.g., Put, Delete, DeleteColumn, DeleteFamily) + + + KeyValue instances are not split across blocks. + For example, if there is an 8 MB KeyValue, even if the block-size is 64kb this KeyValue will be read + in as a coherent block. For more information, see the KeyValue source code. + +
Example + To emphasize the points above, examine what happens with two Puts for two different columns for the same row: + + Put #1: rowkey=row1, cf:attr1=value1 + Put #2: rowkey=row1, cf:attr2=value2 + + Even though these are for the same row, a KeyValue is created for each column: + Key portion for Put #1: + + rowlength ------------> 4 + row -----------------> row1 + columnfamilylength ---> 2 + columnfamily --------> cf + columnqualifier ------> attr1 + timestamp -----------> server time of Put + keytype -------------> Put + + + Key portion for Put #2: + + rowlength ------------> 4 + row -----------------> row1 + columnfamilylength ---> 2 + columnfamily --------> cf + columnqualifier ------> attr2 + timestamp -----------> server time of Put + keytype -------------> Put + + + +
+ It is critical to understand that the rowkey, ColumnFamily, and column (aka columnqualifier) are embedded within + the KeyValue instance. The longer these identifiers are, the bigger the KeyValue is. +
+
+ Compaction + There are two types of compactions: minor and major. Minor compactions will usually pick up a couple of the smaller adjacent + StoreFiles and rewrite them as one. Minors do not drop deletes or expired cells, only major compactions do this. Sometimes a minor compaction + will pick up all the StoreFiles in the Store and in this case it actually promotes itself to being a major compaction. + + After a major compaction runs there will be a single StoreFile per Store, and this will help performance usually. Caution: major compactions rewrite all of the Stores data and on a loaded system, this may not be tenable; + major compactions will usually have to be done manually on large systems. See . + + Compactions will not perform region merges. See for more information on region merging. + +
+ Compaction File Selection + To understand the core algorithm for StoreFile selection, there is some ASCII-art in the Store source code that + will serve as useful reference. It has been copied below: + +/* normal skew: + * + * older ----> newer + * _ + * | | _ + * | | | | _ + * --|-|- |-|- |-|---_-------_------- minCompactSize + * | | | | | | | | _ | | + * | | | | | | | | | | | | + * | | | | | | | | | | | | + */ + + Important knobs: + + hbase.store.compaction.ratio Ratio used in compaction + file selection algorithm (default 1.2f). + hbase.hstore.compaction.min (.90 hbase.hstore.compactionThreshold) (files) Minimum number + of StoreFiles per Store to be selected for a compaction to occur (default 2). + hbase.hstore.compaction.max (files) Maximum number of StoreFiles to compact per minor compaction (default 10). + hbase.hstore.compaction.min.size (bytes) + Any StoreFile smaller than this setting with automatically be a candidate for compaction. Defaults to + hbase.hregion.memstore.flush.size (128 mb). + hbase.hstore.compaction.max.size (.92) (bytes) + Any StoreFile larger than this setting with automatically be excluded from compaction (default Long.MAX_VALUE). + + + The minor compaction StoreFile selection logic is size based, and selects a file for compaction when the file + <= sum(smaller_files) * hbase.hstore.compaction.ratio. + +
+
+ Minor Compaction File Selection - Example #1 (Basic Example) + This example mirrors an example from the unit test TestCompactSelection. + + hbase.store.compaction.ratio = 1.0f + hbase.hstore.compaction.min = 3 (files) > + hbase.hstore.compaction.max = 5 (files) > + hbase.hstore.compaction.min.size = 10 (bytes) > + hbase.hstore.compaction.max.size = 1000 (bytes) > + + The following StoreFiles exist: 100, 50, 23, 12, and 12 bytes apiece (oldest to newest). + With the above parameters, the files that would be selected for minor compaction are 23, 12, and 12. + + Why? + + 100 --> No, because sum(50, 23, 12, 12) * 1.0 = 97. + 50 --> No, because sum(23, 12, 12) * 1.0 = 47. + 23 --> Yes, because sum(12, 12) * 1.0 = 24. + 12 --> Yes, because the previous file has been included, and because this + does not exceed the the max-file limit of 5 + 12 --> Yes, because the previous file had been included, and because this + does not exceed the the max-file limit of 5. + + +
+
+ Minor Compaction File Selection - Example #2 (Not Enough Files To Compact) + This example mirrors an example from the unit test TestCompactSelection. + + hbase.store.compaction.ratio = 1.0f + hbase.hstore.compaction.min = 3 (files) > + hbase.hstore.compaction.max = 5 (files) > + hbase.hstore.compaction.min.size = 10 (bytes) > + hbase.hstore.compaction.max.size = 1000 (bytes) > + + + The following StoreFiles exist: 100, 25, 12, and 12 bytes apiece (oldest to newest). + With the above parameters, no compaction will be started. + + Why? + + 100 --> No, because sum(25, 12, 12) * 1.0 = 47 + 25 --> No, because sum(12, 12) * 1.0 = 24 + 12 --> No. Candidate because sum(12) * 1.0 = 12, there are only 2 files to compact and that is less than the threshold of 3 + 12 --> No. Candidate because the previous StoreFile was, but there are not enough files to compact + + +
+
+ Minor Compaction File Selection - Example #3 (Limiting Files To Compact) + This example mirrors an example from the unit test TestCompactSelection. + + hbase.store.compaction.ratio = 1.0f + hbase.hstore.compaction.min = 3 (files) > + hbase.hstore.compaction.max = 5 (files) > + hbase.hstore.compaction.min.size = 10 (bytes) > + hbase.hstore.compaction.max.size = 1000 (bytes) > + + The following StoreFiles exist: 7, 6, 5, 4, 3, 2, and 1 bytes apiece (oldest to newest). + With the above parameters, the files that would be selected for minor compaction are 7, 6, 5, 4, 3. + + Why? + + 7 --> Yes, because sum(6, 5, 4, 3, 2, 1) * 1.0 = 21. Also, 7 is less than the min-size + 6 --> Yes, because sum(5, 4, 3, 2, 1) * 1.0 = 15. Also, 6 is less than the min-size. + 5 --> Yes, because sum(4, 3, 2, 1) * 1.0 = 10. Also, 5 is less than the min-size. + 4 --> Yes, because sum(3, 2, 1) * 1.0 = 6. Also, 4 is less than the min-size. + 3 --> Yes, because sum(2, 1) * 1.0 = 3. Also, 3 is less than the min-size. + 2 --> No. Candidate because previous file was selected and 2 is less than the min-size, but the max-number of files to compact has been reached. + 1 --> No. Candidate because previous file was selected and 1 is less than the min-size, but max-number of files to compact has been reached. + + +
+
+ Impact of Key Configuration Options + hbase.store.compaction.ratio. A large ratio (e.g., 10) will produce a single giant file. Conversely, a value of .25 will + produce behavior similar to the BigTable compaction algorithm - resulting in 4 StoreFiles. + + hbase.hstore.compaction.min.size. Because + this limit represents the "automatic include" limit for all StoreFiles smaller than this value, this value may need to + be adjusted downwards in write-heavy environments where many 1 or 2 mb StoreFiles are being flushed, because every file + will be targeted for compaction and the resulting files may still be under the min-size and require further compaction, etc. + +
+
+ +
+ +
+ +
Bulk Loading +
Overview + + HBase includes several methods of loading data into tables. + The most straightforward method is to either use the TableOutputFormat + class from a MapReduce job, or use the normal client APIs; however, + these are not always the most efficient methods. + + + The bulk load feature uses a MapReduce job to output table data in HBase's internal + data format, and then directly loads the generated StoreFiles into a running + cluster. Using bulk load will use less CPU and network resources than + simply using the HBase API. + +
+
Bulk Load Architecture + + The HBase bulk load process consists of two main steps. + +
Preparing data via a MapReduce job + + The first step of a bulk load is to generate HBase data files (StoreFiles) from + a MapReduce job using HFileOutputFormat. This output format writes + out data in HBase's internal storage format so that they can be + later loaded very efficiently into the cluster. + + + In order to function efficiently, HFileOutputFormat must be + configured such that each output HFile fits within a single region. + In order to do this, jobs whose output will be bulk loaded into HBase + use Hadoop's TotalOrderPartitioner class to partition the map output + into disjoint ranges of the key space, corresponding to the key + ranges of the regions in the table. + + + HFileOutputFormat includes a convenience function, + configureIncrementalLoad(), which automatically sets up + a TotalOrderPartitioner based on the current region boundaries of a + table. + +
+
Completing the data load + + After the data has been prepared using + HFileOutputFormat, it is loaded into the cluster using + completebulkload. This command line tool iterates + through the prepared data files, and for each one determines the + region the file belongs to. It then contacts the appropriate Region + Server which adopts the HFile, moving it into its storage directory + and making the data available to clients. + + + If the region boundaries have changed during the course of bulk load + preparation, or between the preparation and completion steps, the + completebulkloads utility will automatically split the + data files into pieces corresponding to the new boundaries. This + process is not optimally efficient, so users should take care to + minimize the delay between preparing a bulk load and importing it + into the cluster, especially if other clients are simultaneously + loading data through other means. + +
+
+
Importing the prepared data using the completebulkload tool + + After a data import has been prepared, either by using the + importtsv tool with the + "importtsv.bulk.output" option or by some other MapReduce + job using the HFileOutputFormat, the + completebulkload tool is used to import the data into the + running cluster. + + + The completebulkload tool simply takes the output path + where importtsv or your MapReduce job put its results, and + the table name to import into. For example: + + $ hadoop jar hbase-VERSION.jar completebulkload [-c /path/to/hbase/config/hbase-site.xml] /user/todd/myoutput mytable + + The -c config-file option can be used to specify a file + containing the appropriate hbase parameters (e.g., hbase-site.xml) if + not supplied already on the CLASSPATH (In addition, the CLASSPATH must + contain the directory that has the zookeeper configuration file if + zookeeper is NOT managed by HBase). + + + Note: If the target table does not already exist in HBase, this + tool will create the table automatically. + + This tool will run quickly, after which point the new data will be visible in + the cluster. + +
+
See Also + For more information about the referenced utilities, see and . + +
+
Advanced Usage + + Although the importtsv tool is useful in many cases, advanced users may + want to generate data programatically, or import data from other formats. To get + started doing so, dig into ImportTsv.java and check the JavaDoc for + HFileOutputFormat. + + + The import step of the bulk load can also be done programatically. See the + LoadIncrementalHFiles class for more information. + +
+
+ +
HDFS + As HBase runs on HDFS (and each StoreFile is written as a file on HDFS), + it is important to have an understanding of the HDFS Architecture + especially in terms of how it stores files, handles failovers, and replicates blocks. + + See the Hadoop documentation on HDFS Architecture + for more information. + +
NameNode + The NameNode is responsible for maintaining the filesystem metadata. See the above HDFS Architecture link + for more information. + +
+
DataNode + The DataNodes are responsible for storing HDFS blocks. See the above HDFS Architecture link + for more information. + +
+
+ + + + + + + + + + + + + + FAQ + + General + + When should I use HBase? + + See the in the Architecture chapter. + + + + + Are there other HBase FAQs? + + + See the FAQ that is up on the wiki, HBase Wiki FAQ. + + + + + Does HBase support SQL? + + + Not really. SQL-ish support for HBase via Hive is in development, however Hive is based on MapReduce which is not generally suitable for low-latency requests. + See the section for examples on the HBase client. + + + + + How can I find examples of NoSQL/HBase? + + See the link to the BigTable paper in in the appendix, as + well as the other papers. + + + + + What is the history of HBase? + + See . + + + + + Architecture + + How does HBase handle Region-RegionServer assignment and locality? + + + See . + + + + + Configuration + + How can I get started with my first cluster? + + + See . + + + + + Where can I learn about the rest of the configuration options? + + + See . + + + + + Schema Design / Data Access + + How should I design my schema in HBase? + + + See and + + + + + + How can I store (fill in the blank) in HBase? + + + + See . + + + + + + How can I handle secondary indexes in HBase? + + + + See + + + + + Can I change a table's rowkeys? + + + This is a very common quesiton. You can't. See . + + + + + What APIs does HBase support? + + + See , and . + + + + + MapReduce + + How can I use MapReduce with HBase? + + + See + + + + + Performance and Troubleshooting + + + How can I improve HBase cluster performance? + + + + See . + + + + + + How can I troubleshoot my HBase cluster? + + + + See . + + + + + Amazon EC2 + + + I am running HBase on Amazon EC2 and... + + + + EC2 issues are a special case. See Troubleshooting and Performance sections. + + + + + Operations + + + How do I manage my HBase cluster? + + + + See + + + + + + How do I back up my HBase cluster? + + + + See + + + + + HBase in Action + + Where can I find interesting videos and presentations on HBase? + + + See + + + + + + + + + hbck In Depth + HBaseFsck (hbck) is a tool for checking for region consistency and table integrity problems +and repairing a corrupted HBase. It works in two basic modes -- a read-only inconsistency +identifying mode and a multi-phase read-write repair mode. + +
+ Running hbck to identify inconsistencies +To check to see if your HBase cluster has corruptions, run hbck against your HBase cluster: + +$ ./bin/hbase hbck + + +At the end of the commands output it prints OK or tells you the number of INCONSISTENCIES +present. You may also want to run run hbck a few times because some inconsistencies can be +transient (e.g. cluster is starting up or a region is splitting). Operationally you may want to run +hbck regularly and setup alert (e.g. via nagios) if it repeatedly reports inconsistencies . +A run of hbck will report a list of inconsistencies along with a brief description of the regions and +tables affected. The using the -details option will report more details including a representative +listing of all the splits present in all the tables. + + +$ ./bin/hbase hbck -details + +If you just want to know if some tables are corrupted, you can limit hbck to identify inconsistencies +in only specific tables. For example the following command would only attempt to check table +TableFoo and TableBar. The benefit is that hbck will run in less time. + +$ ./bin/hbase hbck TableFoo TableBar + +
+
Inconsistencies + + If after several runs, inconsistencies continue to be reported, you may have encountered a +corruption. These should be rare, but in the event they occur newer versions of HBase include +the hbck tool enabled with automatic repair options. + + + There are two invariants that when violated create inconsistencies in HBase: + + + HBase’s region consistency invariant is satisfied if every region is assigned and +deployed on exactly one region server, and all places where this state kept is in +accordance. + + HBase’s table integrity invariant is satisfied if for each table, every possible row key +resolves to exactly one region. + + + +Repairs generally work in three phases -- a read-only information gathering phase that identifies +inconsistencies, a table integrity repair phase that restores the table integrity invariant, and then +finally a region consistency repair phase that restores the region consistency invariant. +Starting from version 0.90.0, hbck could detect region consistency problems report on a subset +of possible table integrity problems. It also included the ability to automatically fix the most +common inconsistency, region assignment and deployment consistency problems. This repair +could be done by using the -fix command line option. These problems close regions if they are +open on the wrong server or on multiple region servers and also assigns regions to region +servers if they are not open. + + +Starting from HBase versions 0.90.7, 0.92.2 and 0.94.0, several new command line options are +introduced to aid repairing a corrupted HBase. This hbck sometimes goes by the nickname +“uberhbck”. Each particular version of uber hbck is compatible with the HBase’s of the same +major version (0.90.7 uberhbck can repair a 0.90.4). However, versions <=0.90.6 and versions +<=0.92.1 may require restarting the master or failing over to a backup master. + +
+
Localized repairs + + When repairing a corrupted HBase, it is best to repair the lowest risk inconsistencies first. +These are generally region consistency repairs -- localized single region repairs, that only modify +in-memory data, ephemeral zookeeper data, or patch holes in the META table. +Region consistency requires that the HBase instance has the state of the region’s data in HDFS +(.regioninfo files), the region’s row in the .META. table., and region’s deployment/assignments on +region servers and the master in accordance. Options for repairing region consistency include: + + -fixAssignments (equivalent to the 0.90 -fix option) repairs unassigned, incorrectly +assigned or multiply assigned regions. + + -fixMeta which removes meta rows when corresponding regions are not present in +HDFS and adds new meta rows if they regions are present in HDFS while not in META. + + + To fix deployment and assignment problems you can run this command: + + +$ ./bin/hbase hbck -fixAssignments + +To fix deployment and assignment problems as well as repairing incorrect meta rows you can +run this command:. + +$ ./bin/hbase hbck -fixAssignments -fixMeta + +There are a few classes of table integrity problems that are low risk repairs. The first two are +degenerate (startkey == endkey) regions and backwards regions (startkey > endkey). These are +automatically handled by sidelining the data to a temporary directory (/hbck/xxxx). +The third low-risk class is hdfs region holes. This can be repaired by using the: + + -fixHdfsHoles option for fabricating new empty regions on the file system. +If holes are detected you can use -fixHdfsHoles and should include -fixMeta and -fixAssignments to make the new region consistent. + + + +$ ./bin/hbase hbck -fixAssignments -fixMeta -fixHdfsHoles + +Since this is a common operation, we’ve added a the -repairHoles flag that is equivalent to the +previous command: + +$ ./bin/hbase hbck -repairHoles + +If inconsistencies still remain after these steps, you most likely have table integrity problems +related to orphaned or overlapping regions. +
+
Region Overlap Repairs +Table integrity problems can require repairs that deal with overlaps. This is a riskier operation +because it requires modifications to the file system, requires some decision making, and may +require some manual steps. For these repairs it is best to analyze the output of a hbck -details +run so that you isolate repairs attempts only upon problems the checks identify. Because this is +riskier, there are safeguard that should be used to limit the scope of the repairs. +WARNING: This is a relatively new and have only been tested on online but idle HBase instances +(no reads/writes). Use at your own risk in an active production environment! +The options for repairing table integrity violations include: + + -fixHdfsOrphans option for “adopting” a region directory that is missing a region +metadata file (the .regioninfo file). + + -fixHdfsOverlaps ability for fixing overlapping regions + + +When repairing overlapping regions, a region’s data can be modified on the file system in two +ways: 1) by merging regions into a larger region or 2) by sidelining regions by moving data to +“sideline” directory where data could be restored later. Merging a large number of regions is +technically correct but could result in an extremely large region that requires series of costly +compactions and splitting operations. In these cases, it is probably better to sideline the regions +that overlap with the most other regions (likely the largest ranges) so that merges can happen on +a more reasonable scale. Since these sidelined regions are already laid out in HBase’s native +directory and HFile format, they can be restored by using HBase’s bulk load mechanism. +The default safeguard thresholds are conservative. These options let you override the default +thresholds and to enable the large region sidelining feature. + + -maxMerge <n> maximum number of overlapping regions to merge + + -sidelineBigOverlaps if more than maxMerge regions are overlapping, sideline attempt +to sideline the regions overlapping with the most other regions. + + -maxOverlapsToSideline <n> if sidelining large overlapping regions, sideline at most n +regions. + + + +Since often times you would just want to get the tables repaired, you can use this option to turn +on all repair options: + + -repair includes all the region consistency options and only the hole repairing table +integrity options. + + +Finally, there are safeguards to limit repairs to only specific tables. For example the following +command would only attempt to check and repair table TableFoo and TableBar. + +$ ./bin/hbase hbck -repair TableFoo TableBar + +
Special cases: Meta is not properly assigned +There are a few special cases that hbck can handle as well. +Sometimes the meta table’s only region is inconsistently assigned or deployed. In this case +there is a special -fixMetaOnly option that can try to fix meta assignments. + +$ ./bin/hbase hbck -fixMetaOnly -fixAssignments + +
+
Special cases: HBase version file is missing +HBase’s data on the file system requires a version file in order to start. If this flie is missing, you +can use the -fixVersionFile option to fabricating a new HBase version file. This assumes that +the version of hbck you are running is the appropriate version for the HBase cluster. +
+
Special case: Root and META are corrupt. +The most drastic corruption scenario is the case where the ROOT or META is corrupted and +HBase will not start. In this case you can use the OfflineMetaRepair tool create new ROOT +and META regions and tables. +This tool assumes that HBase is offline. It then marches through the existing HBase home +directory, loads as much information from region metadata files (.regioninfo files) as possible +from the file system. If the region metadata has proper table integrity, it sidelines the original root +and meta table directories, and builds new ones with pointers to the region directories and their +data. + +$ ./bin/hbase org.apache.hadoop.hbase.util.hbck.OfflineMetaRepair + +NOTE: This tool is not as clever as uberhbck but can be used to bootstrap repairs that uberhbck +can complete. +If the tool succeeds you should be able to start hbase and run online repairs if necessary. +
+
Special cases: Offline split parent + +Once a region is split, the offline parent will be cleaned up automatically. Sometimes, daughter regions +are split again before their parents are cleaned up. HBase can clean up parents in the right order. However, +there could be some lingering offline split parents sometimes. They are in META, in HDFS, and not deployed. +But HBase can't clean them up. In this case, you can use the -fixSplitParents option to reset +them in META to be online and not split. Therefore, hbck can merge them with other regions if fixing +overlapping regions option is used. + + +This option should not normally be used, and it is not in -fixAll. + +
+
+ + + + + Compression In HBase<indexterm><primary>Compression</primary></indexterm> + +
+ CompressionTest Tool + + HBase includes a tool to test compression is set up properly. + To run it, type /bin/hbase org.apache.hadoop.hbase.util.CompressionTest. + This will emit usage on how to run the tool. + + You need to restart regionserver for it to pick up fixed codecs! + Be aware that the regionserver caches the result of the compression check it runs + ahead of each region open. This means + that you will have to restart the regionserver for it to notice that you have fixed + any codec issues. + +
+ +
+ + <varname> + hbase.regionserver.codecs + </varname> + + + To have a RegionServer test a set of codecs and fail-to-start if any + code is missing or misinstalled, add the configuration + + hbase.regionserver.codecs + + to your hbase-site.xml with a value of + codecs to test on startup. For example if the + + hbase.regionserver.codecs + value is lzo,gz and if lzo is not present + or improperly installed, the misconfigured RegionServer will fail + to start. + + + Administrators might make use of this facility to guard against + the case where a new server is added to cluster but the cluster + requires install of a particular coded. + +
+ +
+ + LZO + + Unfortunately, HBase cannot ship with LZO because of + the licensing issues; HBase is Apache-licensed, LZO is GPL. + Therefore LZO install is to be done post-HBase install. + See the Using LZO Compression + wiki page for how to make LZO work with HBase. + + A common problem users run into when using LZO is that while initial + setup of the cluster runs smooth, a month goes by and some sysadmin goes to + add a machine to the cluster only they'll have forgotten to do the LZO + fixup on the new machine. In versions since HBase 0.90.0, we should + fail in a way that makes it plain what the problem is, but maybe not. + See + for a feature to help protect against failed LZO install. +
+ +
+ + GZIP + + + GZIP will generally compress better than LZO though slower. + For some setups, better compression may be preferred. + Java will use java's GZIP unless the native Hadoop libs are + available on the CLASSPATH; in this case it will use native + compressors instead (If the native libs are NOT present, + you will see lots of Got brand-new compressor + reports in your logs; see ). + +
+
+ + SNAPPY + + + If snappy is installed, HBase can make use of it (courtesy of + hadoop-snappy + See Alejandro's note up on the list on difference between Snappy in Hadoop + and Snappy in HBase). + + + + + Build and install snappy on all nodes + of your cluster (see below) + + + + + Use CompressionTest to verify snappy support is enabled and the libs can be loaded ON ALL NODES of your cluster: + $ hbase org.apache.hadoop.hbase.util.CompressionTest hdfs://host/path/to/hbase snappy + + + + + Create a column family with snappy compression and verify it in the hbase shell: + $ hbase> create 't1', { NAME => 'cf1', COMPRESSION => 'SNAPPY' } +hbase> describe 't1' + In the output of the "describe" command, you need to ensure it lists "COMPRESSION => 'SNAPPY'" + + + + + + +
+ + Installation + + + You will find the snappy library file under the .libs directory from your Snappy build (For example + /home/hbase/snappy-1.0.5/.libs/). The file is called libsnappy.so.1.x.x where 1.x.x is the version of the snappy + code you are building. You can either copy this file into your hbase directory under libsnappy.so name, or simply + create a symbolic link to it. + + + + The second file you need is the hadoop native library. You will find this file in your hadoop installation directory + under lib/native/Linux-amd64-64/ or lib/native/Linux-i386-32/. The file you are looking for is libhadoop.so.1.x.x. + Again, you can simply copy this file or link to it, under the name libhadoop.so. + + + + At the end of the installation, you should have both libsnappy.so and libhadoop.so links or files present into + lib/native/Linux-amd64-64 or into lib/native/Linux-i386-32 + + To point hbase at snappy support, in hbase-env.sh set + export HBASE_LIBRARY_PATH=/pathtoyourhadoop/lib/native/Linux-amd64-64 + In /pathtoyourhadoop/lib/native/Linux-amd64-64 you should have something like: + + libsnappy.a + libsnappy.so + libsnappy.so.1 + libsnappy.so.1.1.2 + + +
+
+
+ Changing Compression Schemes + A frequent question on the dist-list is how to change compression schemes for ColumnFamilies. This is actually quite simple, + and can be done via an alter command. Because the compression scheme is encoded at the block-level in StoreFiles, the table does + not need to be re-created and the data does not copied somewhere else. Just make sure + the old codec is still available until you are sure that all of the old StoreFiles have been compacted. + +
+ + + + <link xlink:href="https://github.com/brianfrankcooper/YCSB/">YCSB: The Yahoo! Cloud Serving Benchmark</link> and HBase + TODO: Describe how YCSB is poor for putting up a decent cluster load. + TODO: Describe setup of YCSB for HBase + Ted Dunning redid YCSB so it's mavenized and added facility for verifying workloads. See Ted Dunning's YCSB. + + + + + HFile format version 2 + +
Motivation + Note: this feature was introduced in HBase 0.92 + We found it necessary to revise the HFile format after encountering high memory usage and slow startup times caused by large Bloom filters and block indexes in the region server. Bloom filters can get as large as 100 MB per HFile, which adds up to 2 GB when aggregated over 20 regions. Block indexes can grow as large as 6 GB in aggregate size over the same set of regions. A region is not considered opened until all of its block index data is loaded. Large Bloom filters produce a different performance problem: the first get request that requires a Bloom filter lookup will incur the latency of loading the entire Bloom filter bit array. + To speed up region server startup we break Bloom filters and block indexes into multiple blocks and write those blocks out as they fill up, which also reduces the HFile writer’s memory footprint. In the Bloom filter case, “filling up a block” means accumulating enough keys to efficiently utilize a fixed-size bit array, and in the block index case we accumulate an “index block” of the desired size. Bloom filter blocks and index blocks (we call these “inline blocks”) become interspersed with data blocks, and as a side effect we can no longer rely on the difference between block offsets to determine data block length, as it was done in version 1. + HFile is a low-level file format by design, and it should not deal with application-specific details such as Bloom filters, which are handled at StoreFile level. Therefore, we call Bloom filter blocks in an HFile "inline" blocks. We also supply HFile with an interface to write those inline blocks. + Another format modification aimed at reducing the region server startup time is to use a contiguous “load-on-open” section that has to be loaded in memory at the time an HFile is being opened. Currently, as an HFile opens, there are separate seek operations to read the trailer, data/meta indexes, and file info. To read the Bloom filter, there are two more seek operations for its “data” and “meta” portions. In version 2, we seek once to read the trailer and seek again to read everything else we need to open the file from a contiguous block.
+
HFile format version 1 overview As we will be discussing the changes we are making to the HFile format, it is useful to give a short overview of the previous (HFile version 1) format. An HFile in the existing format is structured as follows: + + + + + + HFile Version 1 + + + HFile Version 1 + + + + Image courtesy of Lars George, hbase-architecture-101-storage.html. + +
Block index format in version 1 + The block index in version 1 is very straightforward. For each entry, it contains: + + + Offset (long) + + + Uncompressed size (int) + + + Key (a serialized byte array written using Bytes.writeByteArray) + + + Key length as a variable-length integer (VInt) + + + + + Key bytes + + + + + + The number of entries in the block index is stored in the fixed file trailer, and has to be passed in to the method that reads the block index. One of the limitations of the block index in version 1 is that it does not provide the compressed size of a block, which turns out to be necessary for decompression. Therefore, the HFile reader has to infer this compressed size from the offset difference between blocks. We fix this limitation in version 2, where we store on-disk block size instead of uncompressed size, and get uncompressed size from the block header.
+ HBase file format with inline blocks (version 2) + +
Overview + The version of HBase introducing the above features reads both version 1 and 2 HFiles, but only writes version 2 HFiles. A version 2 HFile is structured as follows: + + + + + + HFile Version 2 + + + HFile Version 2 + + + + + +
+
Unified version 2 block format + In the version 2 every block in the data section contains the following fields: + + + 8 bytes: Block type, a sequence of bytes equivalent to version 1's "magic records". Supported block types are: + + + DATA – data blocks + + + + + LEAF_INDEX – leaf-level index blocks in a multi-level-block-index + + + + + BLOOM_CHUNK – Bloom filter chunks + + + + + META – meta blocks (not used for Bloom filters in version 2 anymore) + + + + + INTERMEDIATE_INDEX – intermediate-level index blocks in a multi-level blockindex + + + + + ROOT_INDEX – root>level index blocks in a multi>level block index + + + + + FILE_INFO – the “file info” block, a small key>value map of metadata + + + + + BLOOM_META – a Bloom filter metadata block in the load>on>open section + + + + + TRAILER – a fixed>size file trailer. As opposed to the above, this is not an + HFile v2 block but a fixed>size (for each HFile version) data structure + + + + + INDEX_V1 – this block type is only used for legacy HFile v1 block + + + + + + Compressed size of the block's data, not including the header (int). + + +Can be used for skipping the current data block when scanning HFile data. + + + + Uncompressed size of the block's data, not including the header (int) + + This is equal to the compressed size if the compression algorithm is NON + + + + File offset of the previous block of the same type (long) + + Can be used for seeking to the previous data/index block + + + + Compressed data (or uncompressed data if the compression algorithm is NONE). + + + The above format of blocks is used in the following HFile sections: + + + Scanned block section. The section is named so because it contains all data blocks that need to be read when an HFile is scanned sequentially.  Also contains leaf block index and Bloom chunk blocks. + + + Non-scanned block section. This section still contains unified-format v2 blocks but it does not have to be read when doing a sequential scan. This section contains “meta” blocks and intermediate-level index blocks. + + + + We are supporting “meta” blocks in version 2 the same way they were supported in version 1, even though we do not store Bloom filter data in these blocks anymore.
+ +
Block index in version 2 + There are three types of block indexes in HFile version 2, stored in two different formats (root and non-root): + + + Data index — version 2 multi-level block index, consisting of: + + + + Version 2 root index, stored in the data block index section of the file + + + + +Optionally, version 2 intermediate levels, stored in the non%root format in the data index section of the file. Intermediate levels can only be present if leaf level blocks are present + + + + +Optionally, version 2 leaf levels, stored in the non%root format inline with data blocks + + + + + + Meta index — version 2 root index format only, stored in the meta index section of the file + + + Bloom index — version 2 root index format only, stored in the “load-on-open” section as part of Bloom filter metadata. + +
+
+ Root block index format in version 2 + This format applies to: + + + Root level of the version 2 data index + + + Entire meta and Bloom indexes in version 2, which are always single-level. + + + A version 2 root index block is a sequence of entries of the following format, similar to entries of a version 1 block index, but storing on-disk size instead of uncompressed size. + + + Offset (long) + +This offset may point to a data block or to a deeper>level index block. + + + + On-disk size (int) + + + Key (a serialized byte array stored using Bytes.writeByteArray) + + + Key (VInt) + + + + Key bytes + + + + + + A single-level version 2 block index consists of just a single root index block. To read a root index block of version 2, one needs to know the number of entries. For the data index and the meta index the number of entries is stored in the trailer, and for the Bloom index it is stored in the compound Bloom filter metadata. + + For a multi-level block index we also store the following fields in the root index block in the load-on-open section of the HFile, in addition to the data structure described above: + + + Middle leaf index block offset + + + Middle leaf block on-disk size (meaning the leaf index block containing the reference to the “middle” data block of the file) + + + The index of the mid-key (defined below) in the middle leaf-level block. + + + + These additional fields are used to efficiently retrieve the mid-key of the HFile used in HFile splits, which we define as the first key of the block with a zero-based index of (n – 1) / 2, if the total number of blocks in the HFile is n. This definition is consistent with how the mid-key was determined in HFile version 1, and is reasonable in general, because blocks are likely to be the same size on average, but we don’t have any estimates on individual key/value pair sizes. + + When writing a version 2 HFile, the total number of data blocks pointed to by every leaf-level index block is kept track of. When we finish writing and the total number of leaf-level blocks is determined, it is clear which leaf-level block contains the mid-key, and the fields listed above are computed.  When reading the HFile and the mid-key is requested, we retrieve the middle leaf index block (potentially from the block cache) and get the mid-key value from the appropriate position inside that leaf block.
+
+ Non-root block index format in version 2 + This format applies to intermediate-level and leaf index blocks of a version 2 multi-level data block index. Every non-root index block is structured as follows. + + + numEntries: the number of entries (int). + + + entryOffsets: the “secondary index” of offsets of entries in the block, to facilitate a quick binary search on the key (numEntries + 1 int values). The last value is the total length of all entries in this index block. For example, in a non-root index block with entry sizes 60, 80, 50 the “secondary index” will contain the following int array: {0, 60, 140, 190}. + + + Entries. Each entry contains: + + + +Offset of the block referenced by this entry in the file (long) + + + + +On>disk size of the referenced block (int) + + + + +Key. The length can be calculated from entryOffsets. + + + + + +
+ Bloom filters in version 2 + In contrast with version 1, in a version 2 HFile Bloom filter metadata is stored in the load-on-open section of the HFile for quick startup. + + + A compound Bloom filter. + + + + Bloom filter version = 3 (int). There used to be a DynamicByteBloomFilter class that had the Bloom filter version number 2 + + + + +The total byte size of all compound Bloom filter chunks (long) + + + + + Number of hash functions (int + + + + +Type of hash functions (int) + + + + +The total key count inserted into the Bloom filter (long) + + + + +The maximum total number of keys in the Bloom filter (long) + + + + +The number of chunks (int) + + + + +Comparator class used for Bloom filter keys, a UTF>8 encoded string stored using Bytes.writeByteArray + + + + + Bloom block index in the version 2 root block index format + + + + +
File Info format in versions 1 and 2 + The file info block is a serialized HbaseMapWritable (essentially a map from byte arrays to byte arrays) with the following keys, among others. StoreFile-level logic adds more keys to this. + + + + hfile.LASTKEY + + + The last key of the file (byte array) + + + + + hfile.AVG_KEY_LEN + + + The average key length in the file (int) + + + + + hfile.AVG_VALUE_LEN + + + The average value length in the file (int) + + + + File info format did not change in version 2. However, we moved the file info to the final section of the file, which can be loaded as one block at the time the HFile is being opened. Also, we do not store comparator in the version 2 file info anymore. Instead, we store it in the fixed file trailer. This is because we need to know the comparator at the time of parsing the load-on-open section of the HFile.
+ Fixed file trailer format differences between versions 1 and 2 + The following table shows common and different fields between fixed file trailers in versions 1 and 2. Note that the size of the trailer is different depending on the version, so it is “fixed” only within one version. However, the version is always stored as the last four-byte integer in the file. + + + + + + + + + Version 1 + + + Version 2 + + + + + File info offset (long) + + + + + Data index offset (long) + + + loadOnOpenOffset (long) + The offset of the section that we need toload when opening the file. + + + + + Number of data index entries (int) + + + + + metaIndexOffset (long) + This field is not being used by the version 1 reader, so we removed it from version 2. + + + uncompressedDataIndexSize (long) + The total uncompressed size of the whole data block index, including root-level, intermediate-level, and leaf-level blocks. + + + + + Number of meta index entries (int) + + + + + Total uncompressed bytes (long) + + + + + numEntries (int) + + + numEntries (long) + + + + + Compression codec: 0 = LZO, 1 = GZ, 2 = NONE (int) + + + + + + + + The number of levels in the data block index (int) + + + + + + + + firstDataBlockOffset (long) + The offset of the first first data block. Used when scanning. + + + + + + + + lastDataBlockEnd (long) + The offset of the first byte after the last key/value data block. We don't need to go beyond this offset when scanning. + + + + + Version: 1 (int) + + + Version: 2 (int) + + + +
+ + + Other Information About HBase +
HBase Videos + Introduction to HBase + + Introduction to HBase by Todd Lipcon (Chicago Data Summit 2011). + + Introduction to HBase by Todd Lipcon (2010). + + + + Building Real Time Services at Facebook with HBase by Jonathan Gray (Hadoop World 2011). + + HBase and Hadoop, Mixing Real-Time and Batch Processing at StumbleUpon by JD Cryans (Hadoop World 2010). + +
+
HBase Presentations (Slides) + Advanced HBase Schema Design by Lars George (Hadoop World 2011). + + Introduction to HBase by Todd Lipcon (Chicago Data Summit 2011). + + Getting The Most From Your HBase Install by Ryan Rawson, Jonathan Gray (Hadoop World 2009). + +
+
HBase Papers + BigTable by Google (2006). + + HBase and HDFS Locality by Lars George (2010). + + No Relation: The Mixed Blessings of Non-Relational Databases by Ian Varley (2009). + +
+
HBase Sites + Cloudera's HBase Blog has a lot of links to useful HBase information. + + CAP Confusion is a relevant entry for background information on + distributed storage systems. + + + + HBase Wiki has a page with a number of presentations. + + HBase RefCard from DZone. + +
+
HBase Books + HBase: The Definitive Guide by Lars George. + +
+
Hadoop Books + Hadoop: The Definitive Guide by Tom White. + +
+ + + + HBase History + + 2006: BigTable paper published by Google. + + 2006 (end of year): HBase development starts. + + 2008: HBase becomes Hadoop sub-project. + + 2010: HBase becomes Apache top-level project. + + + + + HBase and the Apache Software Foundation + HBase is a project in the Apache Software Foundation and as such there are responsibilities to the ASF to ensure + a healthy project. +
ASF Development Process + See the Apache Development Process page + for all sorts of information on how the ASF is structured (e.g., PMC, committers, contributors), to tips on contributing + and getting involved, and how open-source works at ASF. + +
+
ASF Board Reporting + Once a quarter, each project in the ASF portfolio submits a report to the ASF board. This is done by the HBase project + lead and the committers. See ASF board reporting for more information. + +
+ + + Enabling Dapper-like Tracing in HBase +HBASE-6449 added support +for tracing requests through HBase, using the open source tracing library, +HTrace. Setting up tracing is quite simple, +however it currently requires some very minor changes to your client code (it would not be very difficult to remove this requirement). + +
SpanReceivers +The tracing system works by collecting information in structs called ‘Spans’. +It is up to you to choose how you want to receive this information by implementing the +SpanReceiver interface, which defines one method: +public void receiveSpan(Span span); +This method serves as a callback whenever a span is completed. HTrace allows you to use +as many SpanReceivers as you want so you can easily send trace information to multiple destinations. + + +Configure what SpanReceivers you’d like to use by putting a comma separated list of the +fully-qualified class name of classes implementing SpanReceiver in +hbase-site.xml property: hbase.trace.spanreceiver.classes. + + +HBase includes a HBaseLocalFileSpanReceiver that writes all span +information to local files in a JSON-based format. The HBaseLocalFileSpanReceiver +looks in hbase-site.xml for a hbase.trace.spanreceiver.localfilespanreceiver.filename +property with a value describing the name of the file to which nodes should write their span information. + + +If you do not want to use the included HBaseLocalFileSpanReceiver, +you are encouraged to write your own receiver (take a look at HBaseLocalFileSpanReceiver +for an example). If you think others would benefit from your receiver, file a JIRA or send a pull request to +HTrace. + +
+
+Client Modifications +Currently, you must turn on tracing in your client code. To do this, you simply turn on tracing for +requests you think are interesting, and turn it off when the request is done. + + +For example, if you wanted to trace all of your get operations, you change this: +HTable table = new HTable(...); +Get get = new Get(...); + +into: + +Span getSpan = Trace.startSpan(“doing get”, Sampler.ALWAYS); +try { + HTable table = new HTable(...); + Get get = new Get(...); +... +} finally { + getSpan.stop(); +} + +If you wanted to trace half of your ‘get’ operations, you would pass in: +new ProbabilitySampler(0.5) in lieu of Sampler.ALWAYS to Trace.startSpan(). +See the HTrace README for more information on Samplers. + +
+ + + + + Index + + diff --git a/src/docbkx/case_studies.xml b/src/docbkx/case_studies.xml new file mode 100644 index 0000000..2e3bba0 --- /dev/null +++ b/src/docbkx/case_studies.xml @@ -0,0 +1,324 @@ + + + + Apache HBase (TM) Case Studies +
+ Overview + This chapter will describe a variety of performance and troubleshooting case studies that can + provide a useful blueprint on diagnosing Apache HBase (TM) cluster issues. + For more information on Performance and Troubleshooting, see and . + +
+ +
+ Schema Design + +
+ List Data + The following is an exchange from the user dist-list regarding a fairly common question: + how to handle per-user list data in Apache HBase. + + *** QUESTION *** + + We're looking at how to store a large amount of (per-user) list data in +HBase, and we were trying to figure out what kind of access pattern made +the most sense. One option is store the majority of the data in a key, so +we could have something like: + + + +<FixedWidthUserName><FixedWidthValueId1>:"" (no value) +<FixedWidthUserName><FixedWidthValueId2>:"" (no value) +<FixedWidthUserName><FixedWidthValueId3>:"" (no value) + + +The other option we had was to do this entirely using: + +<FixedWidthUserName><FixedWidthPageNum0>:<FixedWidthLength><FixedIdNextPageNum><ValueId1><ValueId2><ValueId3>... +<FixedWidthUserName><FixedWidthPageNum1>:<FixedWidthLength><FixedIdNextPageNum><ValueId1><ValueId2><ValueId3>... + + +where each row would contain multiple values. +So in one case reading the first thirty values would be: + + +scan { STARTROW => 'FixedWidthUsername' LIMIT => 30} + +And in the second case it would be + +get 'FixedWidthUserName\x00\x00\x00\x00' + + +The general usage pattern would be to read only the first 30 values of +these lists, with infrequent access reading deeper into the lists. Some +users would have <= 30 total values in these lists, and some users would +have millions (i.e. power-law distribution) + + + The single-value format seems like it would take up more space on HBase, +but would offer some improved retrieval / pagination flexibility. Would +there be any significant performance advantages to be able to paginate via +gets vs paginating with scans? + + + My initial understanding was that doing a scan should be faster if our +paging size is unknown (and caching is set appropriately), but that gets +should be faster if we'll always need the same page size. I've ended up +hearing different people tell me opposite things about performance. I +assume the page sizes would be relatively consistent, so for most use cases +we could guarantee that we only wanted one page of data in the +fixed-page-length case. I would also assume that we would have infrequent +updates, but may have inserts into the middle of these lists (meaning we'd +need to update all subsequent rows). + + +Thanks for help / suggestions / follow-up questions. + + *** ANSWER *** + +If I understand you correctly, you're ultimately trying to store +triples in the form "user, valueid, value", right? E.g., something +like: + + +"user123, firstname, Paul", +"user234, lastname, Smith" + + +(But the usernames are fixed width, and the valueids are fixed width). + + +And, your access pattern is along the lines of: "for user X, list the +next 30 values, starting with valueid Y". Is that right? And these +values should be returned sorted by valueid? + + +The tl;dr version is that you should probably go with one row per +user+value, and not build a complicated intra-row pagination scheme on +your own unless you're really sure it is needed. + + +Your two options mirror a common question people have when designing +HBase schemas: should I go "tall" or "wide"? Your first schema is +"tall": each row represents one value for one user, and so there are +many rows in the table for each user; the row key is user + valueid, +and there would be (presumably) a single column qualifier that means +"the value". This is great if you want to scan over rows in sorted +order by row key (thus my question above, about whether these ids are +sorted correctly). You can start a scan at any user+valueid, read the +next 30, and be done. What you're giving up is the ability to have +transactional guarantees around all the rows for one user, but it +doesn't sound like you need that. Doing it this way is generally +recommended (see +here http://hbase.apache.org/book.html#schema.smackdown). + + +Your second option is "wide": you store a bunch of values in one row, +using different qualifiers (where the qualifier is the valueid). The +simple way to do that would be to just store ALL values for one user +in a single row. I'm guessing you jumped to the "paginated" version +because you're assuming that storing millions of columns in a single +row would be bad for performance, which may or may not be true; as +long as you're not trying to do too much in a single request, or do +things like scanning over and returning all of the cells in the row, +it shouldn't be fundamentally worse. The client has methods that allow +you to get specific slices of columns. + + +Note that neither case fundamentally uses more disk space than the +other; you're just "shifting" part of the identifying information for +a value either to the left (into the row key, in option one) or to the +right (into the column qualifiers in option 2). Under the covers, +every key/value still stores the whole row key, and column family +name. (If this is a bit confusing, take an hour and watch Lars +George's excellent video about understanding HBase schema design: +http://www.youtube.com/watch?v=_HLoH_PgrLk). + + +A manually paginated version has lots more complexities, as you note, +like having to keep track of how many things are in each page, +re-shuffling if new values are inserted, etc. That seems significantly +more complex. It might have some slight speed advantages (or +disadvantages!) at extremely high throughput, and the only way to +really know that would be to try it out. If you don't have time to +build it both ways and compare, my advice would be to start with the +simplest option (one row per user+value). Start simple and iterate! :) + + +
+ + +
+ +
+ Performance/Troubleshooting + +
+ Case Study #1 (Performance Issue On A Single Node) +
Scenario + Following a scheduled reboot, one data node began exhibiting unusual behavior. Routine MapReduce + jobs run against HBase tables which regularly completed in five or six minutes began taking 30 or 40 minutes + to finish. These jobs were consistently found to be waiting on map and reduce tasks assigned to the troubled data node + (e.g., the slow map tasks all had the same Input Split). + The situation came to a head during a distributed copy, when the copy was severely prolonged by the lagging node. + +
+
Hardware + Datanodes: + + Two 12-core processors + Six Enerprise SATA disks + 24GB of RAM + Two bonded gigabit NICs + + + Network: + + 10 Gigabit top-of-rack switches + 20 Gigabit bonded interconnects between racks. + + +
+
Hypotheses +
HBase "Hot Spot" Region + We hypothesized that we were experiencing a familiar point of pain: a "hot spot" region in an HBase table, + where uneven key-space distribution can funnel a huge number of requests to a single HBase region, bombarding the RegionServer + process and cause slow response time. Examination of the HBase Master status page showed that the number of HBase requests to the + troubled node was almost zero. Further, examination of the HBase logs showed that there were no region splits, compactions, or other region transitions + in progress. This effectively ruled out a "hot spot" as the root cause of the observed slowness. + +
+
HBase Region With Non-Local Data + Our next hypothesis was that one of the MapReduce tasks was requesting data from HBase that was not local to the datanode, thus + forcing HDFS to request data blocks from other servers over the network. Examination of the datanode logs showed that there were very + few blocks being requested over the network, indicating that the HBase region was correctly assigned, and that the majority of the necessary + data was located on the node. This ruled out the possibility of non-local data causing a slowdown. + +
+
Excessive I/O Wait Due To Swapping Or An Over-Worked Or Failing Hard Disk + After concluding that the Hadoop and HBase were not likely to be the culprits, we moved on to troubleshooting the datanode's hardware. + Java, by design, will periodically scan its entire memory space to do garbage collection. If system memory is heavily overcommitted, the Linux + kernel may enter a vicious cycle, using up all of its resources swapping Java heap back and forth from disk to RAM as Java tries to run garbage + collection. Further, a failing hard disk will often retry reads and/or writes many times before giving up and returning an error. This can manifest + as high iowait, as running processes wait for reads and writes to complete. Finally, a disk nearing the upper edge of its performance envelope will + begin to cause iowait as it informs the kernel that it cannot accept any more data, and the kernel queues incoming data into the dirty write pool in memory. + However, using vmstat(1) and free(1), we could see that no swap was being used, and the amount of disk IO was only a few kilobytes per second. + +
+
Slowness Due To High Processor Usage + Next, we checked to see whether the system was performing slowly simply due to very high computational load. top(1) showed that the system load + was higher than normal, but vmstat(1) and mpstat(1) showed that the amount of processor being used for actual computation was low. + +
+
Network Saturation (The Winner) + Since neither the disks nor the processors were being utilized heavily, we moved on to the performance of the network interfaces. The datanode had two + gigabit ethernet adapters, bonded to form an active-standby interface. ifconfig(8) showed some unusual anomalies, namely interface errors, overruns, framing errors. + While not unheard of, these kinds of errors are exceedingly rare on modern hardware which is operating as it should: + +$ /sbin/ifconfig bond0 +bond0 Link encap:Ethernet HWaddr 00:00:00:00:00:00 +inet addr:10.x.x.x Bcast:10.x.x.255 Mask:255.255.255.0 +UP BROADCAST RUNNING MASTER MULTICAST MTU:1500 Metric:1 +RX packets:2990700159 errors:12 dropped:0 overruns:1 frame:6 <--- Look Here! Errors! +TX packets:3443518196 errors:0 dropped:0 overruns:0 carrier:0 +collisions:0 txqueuelen:0 +RX bytes:2416328868676 (2.4 TB) TX bytes:3464991094001 (3.4 TB) + + + These errors immediately lead us to suspect that one or more of the ethernet interfaces might have negotiated the wrong line speed. This was confirmed both by running an ICMP ping + from an external host and observing round-trip-time in excess of 700ms, and by running ethtool(8) on the members of the bond interface and discovering that the active interface + was operating at 100Mbs/, full duplex. + +$ sudo ethtool eth0 +Settings for eth0: +Supported ports: [ TP ] +Supported link modes: 10baseT/Half 10baseT/Full + 100baseT/Half 100baseT/Full + 1000baseT/Full +Supports auto-negotiation: Yes +Advertised link modes: 10baseT/Half 10baseT/Full + 100baseT/Half 100baseT/Full + 1000baseT/Full +Advertised pause frame use: No +Advertised auto-negotiation: Yes +Link partner advertised link modes: Not reported +Link partner advertised pause frame use: No +Link partner advertised auto-negotiation: No +Speed: 100Mb/s <--- Look Here! Should say 1000Mb/s! +Duplex: Full +Port: Twisted Pair +PHYAD: 1 +Transceiver: internal +Auto-negotiation: on +MDI-X: Unknown +Supports Wake-on: umbg +Wake-on: g +Current message level: 0x00000003 (3) +Link detected: yes + + + In normal operation, the ICMP ping round trip time should be around 20ms, and the interface speed and duplex should read, "1000MB/s", and, "Full", respectively. + +
+
+
Resolution + After determining that the active ethernet adapter was at the incorrect speed, we used the ifenslave(8) command to make the standby interface + the active interface, which yielded an immediate improvement in MapReduce performance, and a 10 times improvement in network throughput: + + On the next trip to the datacenter, we determined that the line speed issue was ultimately caused by a bad network cable, which was replaced. + +
+
+
+ Case Study #2 (Performance Research 2012) + Investigation results of a self-described "we're not sure what's wrong, but it seems slow" problem. + http://gbif.blogspot.com/2012/03/hbase-performance-evaluation-continued.html + +
+ +
+ Case Study #3 (Performance Research 2010)) + + Investigation results of general cluster performance from 2010. Although this research is on an older version of the codebase, this writeup + is still very useful in terms of approach. + http://hstack.org/hbase-performance-testing/ + +
+ +
+ Case Study #4 (xcievers Config) + Case study of configuring xceivers, and diagnosing errors from mis-configurations. + http://www.larsgeorge.com/2012/03/hadoop-hbase-and-xceivers.html + + See also . + +
+ +
+ + diff --git a/src/docbkx/community.xml b/src/docbkx/community.xml new file mode 100644 index 0000000..2c09908 --- /dev/null +++ b/src/docbkx/community.xml @@ -0,0 +1,109 @@ + + + + Community +
+ Decisions +
+ Feature Branches + Feature Branches are easy to make. You do not have to be a committer to make one. Just request the name of your branch be added to JIRA up on the + developer's mailing list and a committer will add it for you. Thereafter you can file issues against your feature branch in Apache HBase (TM) JIRA. Your code you + keep elsewhere -- it should be public so it can be observed -- and you can update dev mailing list on progress. When the feature is ready for commit, + 3 +1s from committers will get your feature mergedSee HBase, mail # dev - Thoughts about large feature dev branches + +
+
+ Patch +1 Policy + +The below policy is something we put in place 09/2012. It is a +suggested policy rather than a hard requirement. We want to try it +first to see if it works before we cast it in stone. + + +Apache HBase is made of +components. +Components have one or more s. See the 'Description' field on the +components +JIRA page for who the current owners are by component. + + +Patches that fit within the scope of a single Apache HBase component require, +at least, a +1 by one of the component's owners before commit. If +owners are absent -- busy or otherwise -- two +1s by non-owners will +suffice. + + +Patches that span components need at least two +1s before they can be +committed, preferably +1s by owners of components touched by the +x-component patch (TODO: This needs tightening up but I think fine for +first pass). + + +Any -1 on a patch by anyone vetos a patch; it cannot be committed +until the justification for the -1 is addressed. + +
+
+
+ Community Roles +
+ Component Owner + +Component owners are listed in the description field on this Apache HBase JIRA components +page. The owners are listed in the 'Description' field rather than in the 'Component +Lead' field because the latter only allows us list one individual +whereas it is encouraged that components have multiple owners. + + +Owners are volunteers who are (usually, but not necessarily) expert in +their component domain and may have an agenda on how they think their +Apache HBase component should evolve. + + +Duties include: + + + +Owners will try and review patches that land within their component's scope. + + + + +If applicable, if an owner has an agenda, they will publish their +goals or the design toward which they are driving their component + + + + + +If you would like to be volunteer as a component owner, just write the +dev list and we'll sign you up. Owners do not need to be committers. + +
+
+ diff --git a/src/docbkx/configuration.xml b/src/docbkx/configuration.xml new file mode 100644 index 0000000..2d182fa --- /dev/null +++ b/src/docbkx/configuration.xml @@ -0,0 +1,1175 @@ + + + + Apache HBase (TM) Configuration + This chapter is the Not-So-Quick start guide to Apache HBase (TM) configuration. It goes + over system requirements, Hadoop setup, the different Apache HBase run modes, and the + various configurations in HBase. Please read this chapter carefully. At a mimimum + ensure that all have + been satisfied. Failure to do so will cause you (and us) grief debugging strange errors + and/or data loss. + + + Apache HBase uses the same configuration system as Apache Hadoop. + To configure a deploy, edit a file of environment variables + in conf/hbase-env.sh -- this configuration + is used mostly by the launcher shell scripts getting the cluster + off the ground -- and then add configuration to an XML file to + do things like override HBase defaults, tell HBase what Filesystem to + use, and the location of the ZooKeeper ensemble + + +Be careful editing XML. Make sure you close all elements. +Run your file through xmllint or similar +to ensure well-formedness of your document after an edit session. + + + . + + + When running in distributed mode, after you make + an edit to an HBase configuration, make sure you copy the + content of the conf directory to + all nodes of the cluster. HBase will not do this for you. + Use rsync. + +
+ Basic Prerequisites + This section lists required services and some required system configuration. + + +
+ Java + Just like Hadoop, HBase requires at least java 6 from + Oracle. +
+ +
+ Operating System +
+ ssh + + ssh must be installed and + sshd must be running to use Hadoop's scripts to + manage remote Hadoop and HBase daemons. You must be able to ssh to all + nodes, including your local node, using passwordless login (Google + "ssh passwordless login"). If on mac osx, see the section, + SSH: Setting up Remote Desktop and Enabling Self-Login + on the hadoop wiki. +
+ +
+ DNS + + HBase uses the local hostname to self-report its IP address. + Both forward and reverse DNS resolving must work in versions of + HBase previous to 0.92.0 + The hadoop-dns-checker tool can be used to verify + DNS is working correctly on the cluster. The project README file provides detailed instructions on usage. +. + + If your machine has multiple interfaces, HBase will use the + interface that the primary hostname resolves to. + + If this is insufficient, you can set + hbase.regionserver.dns.interface to indicate the + primary interface. This only works if your cluster configuration is + consistent and every host has the same network interface + configuration. + + Another alternative is setting + hbase.regionserver.dns.nameserver to choose a + different nameserver than the system wide default. +
+
+ Loopback IP + HBase expects the loopback IP address to be 127.0.0.1. See +
+ +
+ NTP + + The clocks on cluster members should be in basic alignments. + Some skew is tolerable but wild skew could generate odd behaviors. Run + NTP + on your cluster, or an equivalent. + + If you are having problems querying data, or "weird" cluster + operations, check system time! +
+ +
+ + <varname>ulimit</varname><indexterm> + <primary>ulimit</primary> + </indexterm> + and + <varname>nproc</varname><indexterm> + <primary>nproc</primary> + </indexterm> + + + Apache HBase is a database. It uses a lot of files all at the same time. + The default ulimit -n -- i.e. user file limit -- of 1024 on most *nix systems + is insufficient (On mac os x its 256). Any significant amount of loading will + lead you to . + You may also notice errors such as... + 2010-04-06 03:04:37,542 INFO org.apache.hadoop.hdfs.DFSClient: Exception increateBlockOutputStream java.io.EOFException + 2010-04-06 03:04:37,542 INFO org.apache.hadoop.hdfs.DFSClient: Abandoning block blk_-6935524980745310745_1391901 + Do yourself a favor and change the upper bound on the + number of file descriptors. Set it to north of 10k. The math runs roughly as follows: per ColumnFamily + there is at least one StoreFile and possibly up to 5 or 6 if the region is under load. Multiply the + average number of StoreFiles per ColumnFamily times the number of regions per RegionServer. For example, assuming + that a schema had 3 ColumnFamilies per region with an average of 3 StoreFiles per ColumnFamily, + and there are 100 regions per RegionServer, the JVM will open 3 * 3 * 100 = 900 file descriptors + (not counting open jar files, config files, etc.) + + You should also up the hbase users' + nproc setting; under load, a low-nproc + setting could manifest as OutOfMemoryError + See Jack Levin's major hdfs issues + note up on the user list. + The requirement that a database requires upping of system limits + is not peculiar to Apache HBase. See for example the section + Setting Shell Limits for the Oracle User in + + Short Guide to install Oracle 10 on Linux.. + + + To be clear, upping the file descriptors and nproc for the user who is + running the HBase process is an operating system configuration, not an + HBase configuration. Also, a common mistake is that administrators + will up the file descriptors for a particular user but for whatever + reason, HBase will be running as some one else. HBase prints in its + logs as the first line the ulimit its seeing. Ensure its correct. + + A useful read setting config on you hadoop cluster is Aaron + Kimballs' Configuration + Parameters: What can you just ignore? + + +
+ <varname>ulimit</varname> on Ubuntu + + If you are on Ubuntu you will need to make the following + changes: + + In the file /etc/security/limits.conf add + a line like: hadoop - nofile 32768 + Replace hadoop with whatever user is running + Hadoop and HBase. If you have separate users, you will need 2 + entries, one for each user. In the same file set nproc hard and soft + limits. For example: hadoop soft/hard nproc 32000. + + In the file /etc/pam.d/common-session add + as the last line in the file: session required pam_limits.so + Otherwise the changes in /etc/security/limits.conf won't be + applied. + + Don't forget to log out and back in again for the changes to + take effect! +
+
+ +
+ Windows + + Apache HBase has been little tested running on Windows. Running a + production install of HBase on top of Windows is not + recommended. + + If you are running HBase on Windows, you must install Cygwin to have a *nix-like + environment for the shell scripts. The full details are explained in + the Windows + Installation guide. Also + search our user mailing list to pick + up latest fixes figured by Windows users. +
+ +
+ +
+ <link + xlink:href="http://hadoop.apache.org">Hadoop</link><indexterm> + <primary>Hadoop</primary> + </indexterm> + Selecting a Hadoop version is critical for your HBase deployment. Below table shows some information about what versions of Hadoop are supported by various HBase versions. Based on the version of HBase, you should select the most appropriate version of Hadoop. We are not in the Hadoop distro selection business. You can use Hadoop distributions from Apache, or learn about vendor distributions of Hadoop at + + + Hadoop version support matrix + + + HBase-0.92.xHBase-0.94.xHBase-0.96 + + Hadoop-0.20.205S X X + Hadoop-0.22.x S X X + Hadoop-1.0.x S S S + Hadoop-1.1.x NT S S + Hadoop-0.23.x X S NT + Hadoop-2.x X S S +
+ + Where + + S = supported and tested, + X = not supported, + NT = it should run, but not tested enough. + + + + Because HBase depends on Hadoop, it bundles an instance of the Hadoop jar under its lib directory. The bundled jar is ONLY for use in standalone mode. In distributed mode, it is critical that the version of Hadoop that is out on your cluster match what is under HBase. Replace the hadoop jar found in the HBase lib directory with the hadoop jar you are running on your cluster to avoid version mismatch issues. Make sure you replace the jar in HBase everywhere on your cluster. Hadoop version mismatch issues have various manifestations but often all looks like its hung up. + +
+ Apache HBase 0.92 and 0.94 + HBase 0.92 and 0.94 versions can work with Hadoop versions, 0.20.205, 0.22.x, 1.0.x, and 1.1.x. HBase-0.94 can additionally work with Hadoop-0.23.x and 2.x, but you may have to recompile the code using the specific maven profile (see top level pom.xml) +
+ +
+ Apache HBase 0.96 + Apache HBase 0.96.0 requires Apache Hadoop 1.x at a minimum, and it can run equally well on hadoop-2.0. + As of Apache HBase 0.96.x, Apache Hadoop 1.0.x at least is required. We will no longer run properly on older Hadoops such as 0.20.205 or branch-0.20-append. Do not move to Apache HBase 0.96.x if you cannot upgrade your HadoopSee HBase, mail # dev - DISCUSS: Have hbase require at least hadoop 1.0.0 in hbase 0.96.0?. +
+ +
+ Hadoop versions 0.20.x - 1.x + + HBase will lose data unless it is running on an HDFS that has a durable + sync implementation. DO NOT use Hadoop 0.20.2, Hadoop 0.20.203.0, and Hadoop 0.20.204.0 which DO NOT have this attribute. Currently only Hadoop versions 0.20.205.x or any release in excess of this version -- this includes hadoop-1.0.0 -- have a working, durable sync + + The Cloudera blog post An update on Apache Hadoop 1.0 + by Charles Zedlweski has a nice exposition on how all the Hadoop versions relate. + Its worth checking out if you are having trouble making sense of the + Hadoop version morass. + + . Sync has to be explicitly enabled by setting + dfs.support.append equal + to true on both the client side -- in hbase-site.xml + -- and on the serverside in hdfs-site.xml (The sync + facility HBase needs is a subset of the append code path). + + <property> + <name>dfs.support.append</name> + <value>true</value> + </property> + + You will have to restart your cluster after making this edit. Ignore the chicken-little + comment you'll find in the hdfs-default.xml in the + description for the dfs.support.append configuration. + +
+
+ Apache HBase on Secure Hadoop + Apache HBase will run on any Hadoop 0.20.x that incorporates Hadoop + security features as long as you do as + suggested above and replace the Hadoop jar that ships with HBase + with the secure version. If you want to read more about how to setup + Secure HBase, see . +
+ +
+ <varname>dfs.datanode.max.xcievers</varname><indexterm> + <primary>xcievers</primary> + </indexterm> + + An Hadoop HDFS datanode has an upper bound on the number of + files that it will serve at any one time. The upper bound parameter is + called xcievers (yes, this is misspelled). Again, + before doing any loading, make sure you have configured Hadoop's + conf/hdfs-site.xml setting the + xceivers value to at least the following: + + <property> + <name>dfs.datanode.max.xcievers</name> + <value>4096</value> + </property> + + + Be sure to restart your HDFS after making the above + configuration. + + Not having this configuration in place makes for strange looking + failures. Eventually you'll see a complain in the datanode logs + complaining about the xcievers exceeded, but on the run up to this one + manifestation is complaint about missing blocks. For example: + 10/12/08 20:10:31 INFO hdfs.DFSClient: Could not obtain block + blk_XXXXXXXXXXXXXXXXXXXXXX_YYYYYYYY from any node: + java.io.IOException: No live nodes contain current block. Will get new + block locations from namenode and retry... + See Hadoop HDFS: Deceived by Xciever for an informative rant on xceivering. + See also + +
+ +
+
+ +
+ HBase run modes: Standalone and Distributed + + HBase has two run modes: and . Out of the box, HBase runs in + standalone mode. To set up a distributed deploy, you will need to + configure HBase by editing files in the HBase conf + directory. + + Whatever your mode, you will need to edit + conf/hbase-env.sh to tell HBase which + java to use. In this file you set HBase environment + variables such as the heapsize and other options for the + JVM, the preferred location for log files, + etc. Set JAVA_HOME to point at the root of your + java install. + +
+ Standalone HBase + + This is the default mode. Standalone mode is what is described + in the section. In + standalone mode, HBase does not use HDFS -- it uses the local + filesystem instead -- and it runs all HBase daemons and a local + ZooKeeper all up in the same JVM. Zookeeper binds to a well known port + so clients may talk to HBase. +
+ +
+ Distributed + + Distributed mode can be subdivided into distributed but all + daemons run on a single node -- a.k.a + pseudo-distributed-- and + fully-distributed where the daemons are spread + across all nodes in the cluster + The pseudo-distributed vs fully-distributed nomenclature + comes from Hadoop. + . + + Distributed modes require an instance of the Hadoop + Distributed File System (HDFS). See the Hadoop + requirements and instructions for how to set up a HDFS. Before + proceeding, ensure you have an appropriate, working HDFS. + + Below we describe the different distributed setups. Starting, + verification and exploration of your install, whether a + pseudo-distributed or + fully-distributed configuration is described in a + section that follows, . The same verification script applies to both + deploy types. + +
+ Pseudo-distributed + + A pseudo-distributed mode is simply a distributed mode run on + a single host. Use this configuration testing and prototyping on + HBase. Do not use this configuration for production nor for + evaluating HBase performance. + + First, setup your HDFS in pseudo-distributed mode. + + Next, configure HBase. Below is an example conf/hbase-site.xml. + This is the file into + which you add local customizations and overrides for + and . + Note that the hbase.rootdir property points to the + local HDFS instance. + + + Now skip to for how to start and verify your + pseudo-distributed install. + See Pseudo-distributed + mode extras for notes on how to start extra Masters and + RegionServers when running pseudo-distributed. + + + + Let HBase create the hbase.rootdir + directory. If you don't, you'll get warning saying HBase needs a + migration run because the directory is missing files expected by + HBase (it'll create them if you let it). + + +
+ Pseudo-distributed Configuration File + Below is a sample pseudo-distributed file for the node h-24-30.example.com. +hbase-site.xml + +<configuration> + ... + <property> + <name>hbase.rootdir</name> + <value>hdfs://h-24-30.sfo.stumble.net:8020/hbase</value> + </property> + <property> + <name>hbase.cluster.distributed</name> + <value>true</value> + </property> + <property> + <name>hbase.zookeeper.quorum</name> + <value>h-24-30.sfo.stumble.net</value> + </property> + ... +</configuration> + + + +
+ +
+ Pseudo-distributed Extras + +
+ Startup + To start up the initial HBase cluster... + % bin/start-hbase.sh + + To start up an extra backup master(s) on the same server run... + % bin/local-master-backup.sh start 1 + ... the '1' means use ports 60001 & 60011, and this backup master's logfile will be at logs/hbase-${USER}-1-master-${HOSTNAME}.log. + + To startup multiple backup masters run... % bin/local-master-backup.sh start 2 3 You can start up to 9 backup masters (10 total). + + To start up more regionservers... + % bin/local-regionservers.sh start 1 + where '1' means use ports 60201 & 60301 and its logfile will be at logs/hbase-${USER}-1-regionserver-${HOSTNAME}.log. + + To add 4 more regionservers in addition to the one you just started by running... % bin/local-regionservers.sh start 2 3 4 5 + This supports up to 99 extra regionservers (100 total). + +
+
+ Stop + Assuming you want to stop master backup # 1, run... + % cat /tmp/hbase-${USER}-1-master.pid |xargs kill -9 + Note that bin/local-master-backup.sh stop 1 will try to stop the cluster along with the master. + + To stop an individual regionserver, run... + % bin/local-regionservers.sh stop 1 + + +
+ +
+ +
+ +
+ Fully-distributed + + For running a fully-distributed operation on more than one + host, make the following configurations. In + hbase-site.xml, add the property + hbase.cluster.distributed and set it to + true and point the HBase + hbase.rootdir at the appropriate HDFS NameNode + and location in HDFS where you would like HBase to write data. For + example, if you namenode were running at namenode.example.org on + port 8020 and you wanted to home your HBase in HDFS at + /hbase, make the following + configuration. + + +<configuration> + ... + <property> + <name>hbase.rootdir</name> + <value>hdfs://namenode.example.org:8020/hbase</value> + <description>The directory shared by RegionServers. + </description> + </property> + <property> + <name>hbase.cluster.distributed</name> + <value>true</value> + <description>The mode the cluster will be in. Possible values are + false: standalone and pseudo-distributed setups with managed Zookeeper + true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh) + </description> + </property> + ... +</configuration> + + +
+ <filename>regionservers</filename> + + In addition, a fully-distributed mode requires that you + modify conf/regionservers. The + file + lists all hosts that you would have running + HRegionServers, one host per line (This + file in HBase is like the Hadoop slaves + file). All servers listed in this file will be started and stopped + when HBase cluster start or stop is run. +
+ +
+ ZooKeeper and HBase + See section for ZooKeeper setup for HBase. +
+ +
+ HDFS Client Configuration + + Of note, if you have made HDFS client + configuration on your Hadoop cluster -- i.e. + configuration you want HDFS clients to use as opposed to + server-side configurations -- HBase will not see this + configuration unless you do one of the following: + + + + Add a pointer to your HADOOP_CONF_DIR + to the HBASE_CLASSPATH environment variable + in hbase-env.sh. + + + + Add a copy of hdfs-site.xml (or + hadoop-site.xml) or, better, symlinks, + under ${HBASE_HOME}/conf, or + + + + if only a small set of HDFS client configurations, add + them to hbase-site.xml. + + + + An example of such an HDFS client configuration is + dfs.replication. If for example, you want to + run with a replication factor of 5, hbase will create files with + the default of 3 unless you do the above to make the configuration + available to HBase. +
+
+
+ +
+ Running and Confirming Your Installation + + + + Make sure HDFS is running first. Start and stop the Hadoop HDFS + daemons by running bin/start-hdfs.sh over in the + HADOOP_HOME directory. You can ensure it started + properly by testing the put and + get of files into the Hadoop filesystem. HBase does + not normally use the mapreduce daemons. These do not need to be + started. + + + + If you are managing your own ZooKeeper, + start it and confirm its running else, HBase will start up ZooKeeper + for you as part of its start process. + + + + Start HBase with the following command: + + + + bin/start-hbase.sh + + Run the above from the + + HBASE_HOME + + directory. + + You should now have a running HBase instance. HBase logs can be + found in the logs subdirectory. Check them out + especially if HBase had trouble starting. + + + + HBase also puts up a UI listing vital attributes. By default its + deployed on the Master host at port 60010 (HBase RegionServers listen + on port 60020 by default and put up an informational http server at + 60030). If the Master were running on a host named + master.example.org on the default port, to see the + Master's homepage you'd point your browser at + http://master.example.org:60010. + + + + Once HBase has started, see the for how to + create tables, add data, scan your insertions, and finally disable and + drop your tables. + + + + To stop HBase after exiting the HBase shell enter + $ ./bin/stop-hbase.sh +stopping hbase............... Shutdown can take a moment to + complete. It can take longer if your cluster is comprised of many + machines. If you are running a distributed operation, be sure to wait + until HBase has shut down completely before stopping the Hadoop + daemons. + + +
+
+ + + +
+ Configuration Files + +
+ <filename>hbase-site.xml</filename> and <filename>hbase-default.xml</filename> + Just as in Hadoop where you add site-specific HDFS configuration + to the hdfs-site.xml file, + for HBase, site specific customizations go into + the file conf/hbase-site.xml. + For the list of configurable properties, see + + below or view the raw hbase-default.xml + source file in the HBase source code at + src/main/resources. + + + Not all configuration options make it out to + hbase-default.xml. Configuration + that it is thought rare anyone would change can exist only + in code; the only way to turn up such configurations is + via a reading of the source code itself. + + + Currently, changes here will require a cluster restart for HBase to notice the change. + + + +
+ +
+ <filename>hbase-env.sh</filename> + Set HBase environment variables in this file. + Examples include options to pass the JVM on start of + an HBase daemon such as heap size and garbarge collector configs. + You can also set configurations for HBase configuration, log directories, + niceness, ssh options, where to locate process pid files, + etc. Open the file at + conf/hbase-env.sh and peruse its content. + Each option is fairly well documented. Add your own environment + variables here if you want them read by HBase daemons on startup. + + Changes here will require a cluster restart for HBase to notice the change. + +
+ +
+ <filename>log4j.properties</filename> + Edit this file to change rate at which HBase files + are rolled and to change the level at which HBase logs messages. + + + Changes here will require a cluster restart for HBase to notice the change + though log levels can be changed for particular daemons via the HBase UI. + +
+ +
Client configuration and dependencies connecting to an HBase cluster + + Since the HBase Master may move around, clients bootstrap by looking to ZooKeeper for + current critical locations. ZooKeeper is where all these values are kept. Thus clients + require the location of the ZooKeeper ensemble information before they can do anything else. + Usually this the ensemble location is kept out in the hbase-site.xml and + is picked up by the client from the CLASSPATH. + + If you are configuring an IDE to run a HBase client, you should + include the conf/ directory on your classpath so + hbase-site.xml settings can be found (or + add src/test/resources to pick up the hbase-site.xml + used by tests). + + + Minimally, a client of HBase needs several libraries in its CLASSPATH when connecting to a cluster, including: + +commons-configuration (commons-configuration-1.6.jar) +commons-lang (commons-lang-2.5.jar) +commons-logging (commons-logging-1.1.1.jar) +hadoop-core (hadoop-core-1.0.0.jar) +hbase (hbase-0.92.0.jar) +log4j (log4j-1.2.16.jar) +slf4j-api (slf4j-api-1.5.8.jar) +slf4j-log4j (slf4j-log4j12-1.5.8.jar) +zookeeper (zookeeper-3.4.2.jar) + + + An example basic hbase-site.xml for client only + might look as follows: + + + + + hbase.zookeeper.quorum + example1,example2,example3 + The directory shared by region servers. + + + +]]> + + +
+ Java client configuration + The configuration used by a Java client is kept + in an HBaseConfiguration instance. + The factory method on HBaseConfiguration, HBaseConfiguration.create();, + on invocation, will read in the content of the first hbase-site.xml found on + the client's CLASSPATH, if one is present + (Invocation will also factor in any hbase-default.xml found; + an hbase-default.xml ships inside the hbase.X.X.X.jar). + It is also possible to specify configuration directly without having to read from a + hbase-site.xml. For example, to set the ZooKeeper + ensemble for the cluster programmatically do as follows: + Configuration config = HBaseConfiguration.create(); +config.set("hbase.zookeeper.quorum", "localhost"); // Here we are running zookeeper locally + If multiple ZooKeeper instances make up your ZooKeeper ensemble, + they may be specified in a comma-separated list (just as in the hbase-site.xml file). + This populated Configuration instance can then be passed to an + HTable, + and so on. + +
+
+ +
+ +
+ Example Configurations + +
+ Basic Distributed HBase Install + + Here is an example basic configuration for a distributed ten + node cluster. The nodes are named example0, + example1, etc., through node + example9 in this example. The HBase Master and the + HDFS namenode are running on the node example0. + RegionServers run on nodes + example1-example9. A 3-node + ZooKeeper ensemble runs on example1, + example2, and example3 on the + default ports. ZooKeeper data is persisted to the directory + /export/zookeeper. Below we show what the main + configuration files -- hbase-site.xml, + regionservers, and + hbase-env.sh -- found in the HBase + conf directory might look like. + +
+ <filename>hbase-site.xml</filename> + + + +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration> + <property> + <name>hbase.zookeeper.quorum</name> + <value>example1,example2,example3</value> + <description>The directory shared by RegionServers. + </description> + </property> + <property> + <name>hbase.zookeeper.property.dataDir</name> + <value>/export/zookeeper</value> + <description>Property from ZooKeeper's config zoo.cfg. + The directory where the snapshot is stored. + </description> + </property> + <property> + <name>hbase.rootdir</name> + <value>hdfs://example0:8020/hbase</value> + <description>The directory shared by RegionServers. + </description> + </property> + <property> + <name>hbase.cluster.distributed</name> + <value>true</value> + <description>The mode the cluster will be in. Possible values are + false: standalone and pseudo-distributed setups with managed Zookeeper + true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh) + </description> + </property> +</configuration> + + +
+ +
+ <filename>regionservers</filename> + + In this file you list the nodes that will run RegionServers. + In our case we run RegionServers on all but the head node + example1 which is carrying the HBase Master and + the HDFS namenode + + + example1 + example3 + example4 + example5 + example6 + example7 + example8 + example9 + +
+ +
+ <filename>hbase-env.sh</filename> + + Below we use a diff to show the differences + from default in the hbase-env.sh file. Here we + are setting the HBase heap to be 4G instead of the default + 1G. + + + +$ git diff hbase-env.sh +diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh +index e70ebc6..96f8c27 100644 +--- a/conf/hbase-env.sh ++++ b/conf/hbase-env.sh +@@ -31,7 +31,7 @@ export JAVA_HOME=/usr/lib//jvm/java-6-sun/ + # export HBASE_CLASSPATH= + + # The maximum amount of heap to use, in MB. Default is 1000. +-# export HBASE_HEAPSIZE=1000 ++export HBASE_HEAPSIZE=4096 + + # Extra Java runtime options. + # Below are what we set by default. May only work with SUN JVM. + + + + Use rsync to copy the content of the + conf directory to all nodes of the + cluster. +
+
+
+ + +
+ The Important Configurations + Below we list what the important + Configurations. We've divided this section into + required configuration and worth-a-look recommended configs. + + + +
Required Configurations + Review the and sections. + +
Big Cluster Configurations + If a cluster with a lot of regions, it is possible if an eager beaver + regionserver checks in soon after master start while all the rest in the + cluster are laggardly, this first server to checkin will be assigned all + regions. If lots of regions, this first server could buckle under the + load. To prevent the above scenario happening up the + hbase.master.wait.on.regionservers.mintostart from its + default value of 1. See + HBASE-6389 Modify the conditions to ensure that Master waits for sufficient number of Region Servers before starting region assignments + for more detail. + +
+
+ +
Recommended Configurations +
+ ZooKeeper Configuration +
<varname>zookeeper.session.timeout</varname> + The default timeout is three minutes (specified in milliseconds). This means + that if a server crashes, it will be three minutes before the Master notices + the crash and starts recovery. You might like to tune the timeout down to + a minute or even less so the Master notices failures the sooner. + Before changing this value, be sure you have your JVM garbage collection + configuration under control otherwise, a long garbage collection that lasts + beyond the ZooKeeper session timeout will take out + your RegionServer (You might be fine with this -- you probably want recovery to start + on the server if a RegionServer has been in GC for a long period of time). + + To change this configuration, edit hbase-site.xml, + copy the changed file around the cluster and restart. + + We set this value high to save our having to field noob questions up on the mailing lists asking + why a RegionServer went down during a massive import. The usual cause is that their JVM is untuned and + they are running into long GC pauses. Our thinking is that + while users are getting familiar with HBase, we'd save them having to know all of its + intricacies. Later when they've built some confidence, then they can play + with configuration such as this. + +
+
Number of ZooKeeper Instances + See . + +
+
+
+ HDFS Configurations +
+ dfs.datanode.failed.volumes.tolerated + This is the "...number of volumes that are allowed to fail before a datanode stops offering service. By default + any volume failure will cause a datanode to shutdown" from the hdfs-default.xml + description. If you have > three or four disks, you might want to set this to 1 or if you have many disks, + two or more. + +
+
+
<varname>hbase.regionserver.handler.count</varname> + + This setting defines the number of threads that are kept open to answer + incoming requests to user tables. The default of 10 is rather low in order to + prevent users from killing their region servers when using large write buffers + with a high number of concurrent clients. The rule of thumb is to keep this + number low when the payload per request approaches the MB (big puts, scans using + a large cache) and high when the payload is small (gets, small puts, ICVs, deletes). + + + It is safe to set that number to the + maximum number of incoming clients if their payload is small, the typical example + being a cluster that serves a website since puts aren't typically buffered + and most of the operations are gets. + + + The reason why it is dangerous to keep this setting high is that the aggregate + size of all the puts that are currently happening in a region server may impose + too much pressure on its memory, or even trigger an OutOfMemoryError. A region server + running on low memory will trigger its JVM's garbage collector to run more frequently + up to a point where GC pauses become noticeable (the reason being that all the memory + used to keep all the requests' payloads cannot be trashed, no matter how hard the + garbage collector tries). After some time, the overall cluster + throughput is affected since every request that hits that region server will take longer, + which exacerbates the problem even more. + + You can get a sense of whether you have too little or too many handlers by + + on an individual RegionServer then tailing its logs (Queued requests + consume memory). + +
+
+ Configuration for large memory machines + + HBase ships with a reasonable, conservative configuration that will + work on nearly all + machine types that people might want to test with. If you have larger + machines -- HBase has 8G and larger heap -- you might the following configuration options helpful. + TODO. + + +
+ +
+ Compression + You should consider enabling ColumnFamily compression. There are several options that are near-frictionless and in most all cases boost + performance by reducing the size of StoreFiles and thus reducing I/O. + + See for more information. +
+
+ Bigger Regions + + Consider going to larger regions to cut down on the total number of regions + on your cluster. Generally less Regions to manage makes for a smoother running + cluster (You can always later manually split the big Regions should one prove + hot and you want to spread the request load over the cluster). A lower number of regions is + preferred, generally in the range of 20 to low-hundreds + per RegionServer. Adjust the regionsize as appropriate to achieve this number. + + For the 0.90.x codebase, the upper-bound of regionsize is about 4Gb, with a default of 256Mb. + For 0.92.x codebase, due to the HFile v2 change much larger regionsizes can be supported (e.g., 20Gb). + + You may need to experiment with this setting based on your hardware configuration and application needs. + + Adjust hbase.hregion.max.filesize in your hbase-site.xml. + RegionSize can also be set on a per-table basis via + HTableDescriptor. + +
+ How many regions per RegionServer? + + Typically you want to keep your region count low on HBase for numerous reasons. + Usually right around 100 regions per RegionServer has yielded the best results. + Here are some of the reasons below for keeping region count low: + + + MSLAB requires 2mb per memstore (that's 2mb per family per region). + 1000 regions that have 2 families each is 3.9GB of heap used, and it's not even storing data yet. NB: the 2MB value is configurable. + + If you fill all the regions at somewhat the same rate, the global memory usage makes it that it forces tiny + flushes when you have too many regions which in turn generates compactions. + Rewriting the same data tens of times is the last thing you want. + An example is filling 1000 regions (with one family) equally and let's consider a lower bound for global memstore + usage of 5GB (the region server would have a big heap). + Once it reaches 5GB it will force flush the biggest region, + at that point they should almost all have about 5MB of data so + it would flush that amount. 5MB inserted later, it would flush another + region that will now have a bit over 5MB of data, and so on. + A basic formula for the amount of regions to have per region server would + look like this: + Heap * upper global memstore limit = amount of heap devoted to memstore + then the amount of heap devoted to memstore / (Number of regions per RS * CFs). + This will give you the rough memstore size if everything is being written to. + A more accurate formula is + Heap * upper global memstore limit = amount of heap devoted to memstore then the + amount of heap devoted to memstore / (Number of actively written regions per RS * CFs). + This can allot you a higher region count from the write perspective if you know how many + regions you will be writing to at one time. + + The master as is is allergic to tons of regions, and will + take a lot of time assigning them and moving them around in batches. + The reason is that it's heavy on ZK usage, and it's not very async + at the moment (could really be improved -- and has been imporoved a bunch + in 0.96 hbase). + + + In older versions of HBase (pre-v2 hfile, 0.90 and previous), tons of regions + on a few RS can cause the store file index to rise raising heap usage and can + create memory pressure or OOME on the RSs + + + + Another issue is the effect of the number of regions on mapreduce jobs. + Keeping 5 regions per RS would be too low for a job, whereas 1000 will generate too many maps. + +
+ +
+
+ Managed Splitting + + Rather than let HBase auto-split your Regions, manage the splitting manually + What follows is taken from the javadoc at the head of + the org.apache.hadoop.hbase.util.RegionSplitter tool + added to HBase post-0.90.0 release. + + . + With growing amounts of data, splits will continually be needed. Since + you always know exactly what regions you have, long-term debugging and + profiling is much easier with manual splits. It is hard to trace the logs to + understand region level problems if it keeps splitting and getting renamed. + Data offlining bugs + unknown number of split regions == oh crap! If an + HLog or StoreFile + was mistakenly unprocessed by HBase due to a weird bug and + you notice it a day or so later, you can be assured that the regions + specified in these files are the same as the current regions and you have + less headaches trying to restore/replay your data. + You can finely tune your compaction algorithm. With roughly uniform data + growth, it's easy to cause split / compaction storms as the regions all + roughly hit the same data size at the same time. With manual splits, you can + let staggered, time-based major compactions spread out your network IO load. + + + How do I turn off automatic splitting? Automatic splitting is determined by the configuration value + hbase.hregion.max.filesize. It is not recommended that you set this + to Long.MAX_VALUE in case you forget about manual splits. A suggested setting + is 100GB, which would result in > 1hr major compactions if reached. + + What's the optimal number of pre-split regions to create? + Mileage will vary depending upon your application. + You could start low with 10 pre-split regions / server and watch as data grows + over time. It's better to err on the side of too little regions and rolling split later. + A more complicated answer is that this depends upon the largest storefile + in your region. With a growing data size, this will get larger over time. You + want the largest region to be just big enough that the Store compact + selection algorithm only compacts it due to a timed major. If you don't, your + cluster can be prone to compaction storms as the algorithm decides to run + major compactions on a large series of regions all at once. Note that + compaction storms are due to the uniform data growth, not the manual split + decision. + + If you pre-split your regions too thin, you can increase the major compaction +interval by configuring HConstants.MAJOR_COMPACTION_PERIOD. If your data size +grows too large, use the (post-0.90.0 HBase) org.apache.hadoop.hbase.util.RegionSplitter +script to perform a network IO safe rolling split +of all regions. + +
+
Managed Compactions + A common administrative technique is to manage major compactions manually, rather than letting + HBase do it. By default, HConstants.MAJOR_COMPACTION_PERIOD is one day and major compactions + may kick in when you least desire it - especially on a busy system. To turn off automatic major compactions set + the value to 0. + + It is important to stress that major compactions are absolutely necessary for StoreFile cleanup, the only variant is when + they occur. They can be administered through the HBase shell, or via + HBaseAdmin. + + For more information about compactions and the compaction file selection process, see +
+ +
Speculative Execution + Speculative Execution of MapReduce tasks is on by default, and for HBase clusters it is generally advised to turn off + Speculative Execution at a system-level unless you need it for a specific case, where it can be configured per-job. + Set the properties mapred.map.tasks.speculative.execution and + mapred.reduce.tasks.speculative.execution to false. + +
+
+ +
Other Configurations +
Balancer + The balancer is a periodic operation which is run on the master to redistribute regions on the cluster. It is configured via + hbase.balancer.period and defaults to 300000 (5 minutes). + See for more information on the LoadBalancer. + +
+
Disabling Blockcache + Do not turn off block cache (You'd do it by setting hbase.block.cache.size to zero). + Currently we do not do well if you do this because the regionserver will spend all its time loading hfile + indices over and over again. If your working set it such that block cache does you no good, at least + size the block cache such that hfile indices will stay up in the cache (you can get a rough idea + on the size you need by surveying regionserver UIs; you'll see index block size accounted near the + top of the webpage). +
+
+ <link xlink:href="http://en.wikipedia.org/wiki/Nagle's_algorithm">Nagle's</link> or the small package problem + If a big 40ms or so occasional delay is seen in operations against HBase, + try the Nagles' setting. For example, see the user mailing list thread, + Inconsistent scan performance with caching set to 1 + and the issue cited therein where setting notcpdelay improved scan speeds. You might also + see the graphs on the tail of HBASE-7008 Set scanner caching to a better default + where our Lars Hofhansl tries various data sizes w/ Nagle's on and off measuring the effect. +
+ +
+ +
+ + diff --git a/src/docbkx/customization.xsl b/src/docbkx/customization.xsl new file mode 100644 index 0000000..a5065a4 --- /dev/null +++ b/src/docbkx/customization.xsl @@ -0,0 +1,48 @@ + + + + + + + + + + +
+ + +comments powered by Disqus + + + diff --git a/src/docbkx/developer.xml b/src/docbkx/developer.xml new file mode 100644 index 0000000..854b6f6 --- /dev/null +++ b/src/docbkx/developer.xml @@ -0,0 +1,1188 @@ + + + + Building and Developing Apache HBase (TM) + This chapter will be of interest only to those building and developing Apache HBase (TM) (i.e., as opposed to + just downloading the latest distribution). + +
+ Apache HBase Repositories + There are two different repositories for Apache HBase: Subversion (SVN) and Git. The former is the system of record for committers, but the latter is easier to work with to build and contribute. SVN updates get automatically propagated to the Git repo. +
+ SVN + +svn co http://svn.apache.org/repos/asf/hbase/trunk hbase-core-trunk + +
+
+ Git + +git clone git://git.apache.org/hbase.git + +
+
+ +
+ IDEs +
+ Eclipse +
+ Code Formatting + Under the dev-support folder, you will find hbase_eclipse_formatter.xml. + We encourage you to have this formatter in place in eclipse when editing HBase code. To load it into eclipse: + +Go to Eclipse->Preferences... +In Preferences, Go to Java->Code Style->Formatter +Import... hbase_eclipse_formatter.xml +Click Apply +Still in Preferences, Go to Java->Editor->Save Actions +Check the following: + +Perform the selected actions on save +Format source code +Format edited lines + + +Click Apply + + + In addition to the automatic formatting, make sure you follow the style guidelines explained in + Also, no @author tags - that's a rule. Quality Javadoc comments are appreciated. And include the Apache license. +
+
+ Subversive Plugin + Download and install the Subversive plugin. + Set up an SVN Repository target from , then check out the code. +
+
+ Git Plugin + If you cloned the project via git, download and install the Git plugin (EGit). Attach to your local git repo (via the Git Repositories window) and you'll be able to see file revision history, generate patches, etc. +
+
+ HBase Project Setup in Eclipse + The easiest way is to use the m2eclipse plugin for Eclipse. Eclipse Indigo or newer has m2eclipse built-in, or it can be found here:http://www.eclipse.org/m2e/. M2Eclipse provides Maven integration for Eclipse - it even lets you use the direct Maven commands from within Eclipse to compile and test your project. + To import the project, you merely need to go to File->Import...Maven->Existing Maven Projects and then point Eclipse at the HBase root directory; m2eclipse will automatically find all the hbase modules for you. + If you install m2eclipse and import HBase in your workspace, you will have to fix your eclipse Build Path. + Remove target folder, add target/generated-jamon + and target/generated-sources/java folders. You may also remove from your Build Path + the exclusions on the src/main/resources and src/test/resources + to avoid error message in the console 'Failed to execute goal org.apache.maven.plugins:maven-antrun-plugin:1.6:run (default) on project hbase: + 'An Ant BuildException has occured: Replace: source file .../target/classes/hbase-default.xml doesn't exist'. This will also + reduce the eclipse build cycles and make your life easier when developing. +
+
+ Import into eclipse with the command line + For those not inclined to use m2eclipse, you can generate the Eclipse files from the command line. First, run (you should only have to do this once): + mvn clean install -DskipTests + and then close Eclipse and execute... + mvn eclipse:eclipse + ... from your local HBase project directory in your workspace to generate some new .project + and .classpathfiles. Then reopen Eclipse, or refresh your eclipse project (F5), and import + the .project file in the HBase directory to a workspace. + +
+
+ Maven Classpath Variable + The M2_REPO classpath variable needs to be set up for the project. This needs to be set to + your local Maven repository, which is usually ~/.m2/repository + If this classpath variable is not configured, you will see compile errors in Eclipse like this... + +Description Resource Path Location Type +The project cannot be built until build path errors are resolved hbase Unknown Java Problem +Unbound classpath variable: 'M2_REPO/asm/asm/3.1/asm-3.1.jar' in project 'hbase' hbase Build path Build Path Problem +Unbound classpath variable: 'M2_REPO/com/github/stephenc/high-scale-lib/high-scale-lib/1.1.1/high-scale-lib-1.1.1.jar' in project 'hbase' hbase Build path Build Path Problem +Unbound classpath variable: 'M2_REPO/com/google/guava/guava/r09/guava-r09.jar' in project 'hbase' hbase Build path Build Path Problem +Unbound classpath variable: 'M2_REPO/com/google/protobuf/protobuf-java/2.3.0/protobuf-java-2.3.0.jar' in project 'hbase' hbase Build path Build Path Problem Unbound classpath variable: + +
+
+ Eclipse Known Issues + Eclipse will currently complain about Bytes.java. It is not possible to turn these errors off. + +Description Resource Path Location Type +Access restriction: The method arrayBaseOffset(Class) from the type Unsafe is not accessible due to restriction on required library /System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/classes.jar Bytes.java /hbase/src/main/java/org/apache/hadoop/hbase/util line 1061 Java Problem +Access restriction: The method arrayIndexScale(Class) from the type Unsafe is not accessible due to restriction on required library /System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/classes.jar Bytes.java /hbase/src/main/java/org/apache/hadoop/hbase/util line 1064 Java Problem +Access restriction: The method getLong(Object, long) from the type Unsafe is not accessible due to restriction on required library /System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/classes.jar Bytes.java /hbase/src/main/java/org/apache/hadoop/hbase/util line 1111 Java Problem + +
+
+ Eclipse - More Information + For additional information on setting up Eclipse for HBase development on Windows, see + Michael Morello's blog on the topic. + +
+
+
+ +
+ Building Apache HBase +
+ Basic Compile + Thanks to maven, building HBase is pretty easy. You can read about the various maven commands in , but the simplest command to compile HBase from its java source code is: + +mvn package -DskipTests + + Or, to clean up before compiling: + +mvn clean package -DskipTests + + With Eclipse set up as explained above in , you can also simply use the build command in Eclipse. To create the full installable HBase package takes a little bit more work, so read on. + +
+
+ Building in snappy compression support + Pass -Dsnappy to trigger the snappy maven profile for building + snappy native libs into hbase. See also +
+ +
+ Building the HBase tarball + Do the following to build the HBase tarball. + Passing the -Prelease will generate javadoc and run the RAT plugin to verify licenses on source. + % MAVEN_OPTS="-Xmx2g" mvn clean site install assembly:assembly -DskipTests -Prelease + +
+ + +
Build Gotchas + If you see Unable to find resource 'VM_global_library.vm', ignore it. + Its not an error. It is officially ugly though. + +
+
+
+ Adding an Apache HBase release to Apache's Maven Repository + Follow the instructions at + Publishing Maven Artifacts after + reading the below miscellaney. + + You must use maven 3.0.x (Check by running mvn -version). + + Let me list out the commands I used first. The sections that follow dig in more + on what is going on. In this example, we are releasing the 0.92.2 jar to the apache + maven repository. + + # First make a copy of the tag we want to release; presumes the release has been tagged already + # We do this because we need to make some commits for the mvn release plugin to work. + 853 svn copy -m "Publishing 0.92.2 to mvn" https://svn.apache.org/repos/asf/hbase/tags/0.92.2 https://svn.apache.org/repos/asf/hbase/tags/0.92.2mvn + 857 svn checkout https://svn.apache.org/repos/asf/hbase/tags/0.92.2mvn + 858 cd 0.92.2mvn/ + # Edit the version making it release version with a '-SNAPSHOT' suffix (See below for more on this) + 860 vi pom.xml + 861 svn commit -m "Add SNAPSHOT to the version" pom.xml + 862 ~/bin/mvn/bin/mvn release:clean + 865 ~/bin/mvn/bin/mvn release:prepare + 866 # Answer questions and then ^C to kill the build after the last question. See below for more on this. + 867 vi release.properties + # Change the references to trunk svn to be 0.92.2mvn; the release plugin presumes trunk + # Then restart the release:prepare -- it won't ask questions + # because the properties file exists. + 868 ~/bin/mvn/bin/mvn release:prepare + # The apache-release profile comes from the apache parent pom and does signing of artifacts published + 869 ~/bin/mvn/bin/mvn release:perform -Papache-release + # When done copying up to apache staging repository, + # browse to repository.apache.org, login and finish + # the release as according to the above + # "Publishing Maven Artifacts. + + + Below is more detail on the commmands listed above. + At the mvn release:perform step, before starting, if you are for example + releasing hbase 0.92.2, you need to make sure the pom.xml version is 0.92.2-SNAPSHOT. This needs + to be checked in. Since we do the maven release after actual release, I've been doing this + checkin into a copy of the release tag rather than into the actual release tag itself (presumes the release has been properly tagged in svn). + So, say we released hbase 0.92.2 and now we want to do the release to the maven repository, in svn, the 0.92.2 + release will be tagged 0.92.2. Making the maven release, copy the 0.92.2 tag to 0.92.2mvn. + Check out this tag and change the version therein and commit. + + + Currently, the mvn release wants to go against trunk. I haven't figured how to tell it to do otherwise + so I do the below hack. The hack comprises answering the questions put to you by the mvn release plugin properly, + then immediately control-C'ing the build after the last question asked as the build release step starts to run. + After control-C'ing it, You'll notice a release.properties in your build dir. Review it. + Make sure it is using the proper branch -- it tends to use trunk rather than the 0.92.2mvn or whatever + that you want it to use -- so hand edit the release.properties file that was put under ${HBASE_HOME} + by the release:perform invocation. When done, resstart the + release:perform. + + Here is how I'd answer the questions at release:prepare time: + What is the release version for "HBase"? (org.apache.hbase:hbase) 0.92.2: : +What is SCM release tag or label for "HBase"? (org.apache.hbase:hbase) hbase-0.92.2: : 0.92.2mvn +What is the new development version for "HBase"? (org.apache.hbase:hbase) 0.92.3-SNAPSHOT: : +[INFO] Transforming 'HBase'... + + When you run release:perform, pass -Papache-release + else it will not 'sign' the artifacts it uploads. + + A strange issue I ran into was the one where the upload into the apache + repository was being sprayed across multiple apache machines making it so I could + not release. See INFRA-4482 Why is my upload to mvn spread across multiple repositories?. + + Here is my ~/.m2/settings.xml. + This is read by the release plugin. The apache-release profile will pick up your + gpg key setup from here if you've specified it into the file. The password + can be maven encrypted as suggested in the "Publishing Maven Artifacts" but plain + text password works too (just don't let anyone see your local settings.xml). + <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 + http://maven.apache.org/xsd/settings-1.0.0.xsd"> + <servers> + <!- To publish a snapshot of some part of Maven --> + <server> + <id>apache.snapshots.https</id> + <username>YOUR_APACHE_ID + </username> + <password>YOUR_APACHE_PASSWORD + </password> + </server> + <!-- To publish a website using Maven --> + <!-- To stage a release of some part of Maven --> + <server> + <id>apache.releases.https</id> + <username>YOUR_APACHE_ID + </username> + <password>YOUR_APACHE_PASSWORD + </password> + </server> + </servers> + <profiles> + <profile> + <id>apache-release</id> + <properties> + <gpg.keyname>YOUR_KEYNAME</gpg.keyname> + <!--Keyname is something like this ... 00A5F21E... do gpg --list-keys to find it--> + <gpg.passphrase>YOUR_KEY_PASSWORD + </gpg.passphrase> + </properties> + </profile> + </profiles> +</settings> + + + + If you see run into the below, its because you need to edit version in the pom.xml and add + -SNAPSHOT to the version (and commit). + [INFO] Scanning for projects... +[INFO] Searching repository for plugin with prefix: 'release'. +[INFO] ------------------------------------------------------------------------ +[INFO] Building HBase +[INFO] task-segment: [release:prepare] (aggregator-style) +[INFO] ------------------------------------------------------------------------ +[INFO] [release:prepare {execution: default-cli}] +[INFO] ------------------------------------------------------------------------ +[ERROR] BUILD FAILURE +[INFO] ------------------------------------------------------------------------ +[INFO] You don't have a SNAPSHOT project in the reactor projects list. +[INFO] ------------------------------------------------------------------------ +[INFO] For more information, run Maven with the -e switch +[INFO] ------------------------------------------------------------------------ +[INFO] Total time: 3 seconds +[INFO] Finished at: Sat Mar 26 18:11:07 PDT 2011 +[INFO] Final Memory: 35M/423M +[INFO] ----------------------------------------------------------------------- + +
+
+ Generating the HBase Reference Guide + The manual is marked up using docbook. + We then use the docbkx maven plugin + to transform the markup to html. This plugin is run when you specify the site + goal as in when you run mvn site or you can call the plugin explicitly to + just generate the manual by doing mvn docbkx:generate-html + (TODO: It looks like you have to run mvn site first because docbkx wants to + include a transformed hbase-default.xml. Fix). + When you run mvn site, we do the document generation twice, once to generate the multipage + manual and then again for the single page manual (the single page version is easier to search). + +
+
+ Updating hbase.apache.org +
+ Contributing to hbase.apache.org + The Apache HBase apache web site (including this reference guide) is maintained as part of the main Apache HBase source tree, under /src/docbkx and /src/site. The former is this reference guide; the latter, in most cases, are legacy pages that are in the process of being merged into the docbkx tree. + To contribute to the reference guide, edit these files and submit them as a patch (see ). Your Jira should contain a summary of the changes in each section (see HBASE-6081 for an example). + To generate the site locally while you're working on it, run: + mvn site + Then you can load up the generated HTML files in your browser (file are under /target/site). +
+
+ Publishing hbase.apache.org + As of INFRA-5680 Migrate apache hbase website, + to publish the website, build it, and then deploy it over a checkout of https://svn.apache.org/repos/asf/hbase/hbase.apache.org/trunk, + and then check it in. For example, if trunk is checked out out at /Users/stack/checkouts/trunk + and hbase.apache.org is checked out at /Users/stack/checkouts/hbase.apache.org/trunk, to update + the site, do the following: + + # Build the site and deploy it to the checked out directory + # Getting the javadoc into site is a little tricky. You have to build it independent, then + # 'aggregate' it at top-level so the pre-site site lifecycle step can find it; that is + # what the javadoc:javadoc and javadoc:aggregate is about. + $ MAVEN_OPTS=" -Xmx3g" mvn clean -DskipTests javadoc:javadoc javadoc:aggregate site site:stage -DstagingDirectory=/Users/stack/checkouts/hbase.apache.org/trunk + # Check the deployed site by viewing in a brower. + # If all is good, commit it and it will show up at http://hbase.apache.org + # + $ cd /Users/stack/checkouts/hbase.apache.org/trunk + $ svn commit -m 'Committing latest version of website...' + + +
+
+
+ Tests + + Developers, at a minimum, should familiarize themselves with the unit test detail; unit tests in +HBase have a character not usually seen in other projects. + +
+Apache HBase Modules +As of 0.96, Apache HBase is split into multiple modules which creates "interesting" rules for +how and where tests are written. If you are writting code for hbase-server, see + for how to write your tests; these tests can spin +up a minicluster and will need to be categorized. For any other module, for example +hbase-common, the tests must be strict unit tests and just test the class +under test - no use of the HBaseTestingUtility or minicluster is allowed (or even possible +given the dependency tree). +
+ Running Tests in other Modules + If the module you are developing in has no other dependencies on other HBase modules, then + you can cd into that module and just run: + mvn test + which will just run the tests IN THAT MODULE. If there are other dependencies on other modules, + then you will have run the command from the ROOT HBASE DIRECTORY. This will run the tests in the other + modules, unless you specify to skip the tests in that module. For instance, to skip the tests in the hbase-server module, + you would run: + mvn clean test -PskipServerTests + from the top level directory to run all the tests in modules other than hbase-server. Note that you + can specify to skip tests in multiple modules as well as just for a single module. For example, to skip + the tests in hbase-server and hbase-common, you would run: + mvn clean test -PskipServerTests -PskipCommonTests + Also, keep in mind that if you are running tests in the hbase-server module you will need to + apply the maven profiles discussed in to get the tests to run properly. +
+
+ +
+Unit Tests +Apache HBase unit tests are subdivided into four categories: small, medium, large, and +integration with corresponding JUnit categories: +SmallTests, MediumTests, +LargeTests, IntegrationTests. +JUnit categories are denoted using java annotations and look like this in your unit test code. +... +@Category(SmallTests.class) +public class TestHRegionInfo { + @Test + public void testCreateHRegionInfoName() throws Exception { + // ... + } +} +The above example shows how to mark a unit test as belonging to the small category. +All unit tests in HBase have a categorization. + + +The first three categories, small, medium, and large are for tests run when +you type $ mvn test; i.e. these three categorizations are for +HBase unit tests. The integration category is for not for unit tests but for integration +tests. These are run when you invoke $ mvn verify. Integration tests +are described in integration tests section and will not be discussed further +in this section on HBase unit tests. + +Apache HBase uses a patched maven surefire plugin and maven profiles to implement +its unit test characterizations. + +Read the below to figure which annotation of the set small, medium, and large to +put on your new HBase unit test. + + +
+Small Tests<indexterm><primary>SmallTests</primary></indexterm> + +Small tests are executed in a shared JVM. We put in this category all the tests that can +be executed quickly in a shared JVM. The maximum execution time for a small test is 15 seconds, +and small tests should not use a (mini)cluster. +
+ +
+Medium Tests<indexterm><primary>MediumTests</primary></indexterm> +Medium tests represent tests that must be executed +before proposing a patch. They are designed to run in less than 30 minutes altogether, +and are quite stable in their results. They are designed to last less than 50 seconds +individually. They can use a cluster, and each of them is executed in a separate JVM. + +
+ +
+Large Tests<indexterm><primary>LargeTests</primary></indexterm> +Large tests are everything else. They are typically large-scale +tests, regression tests for specific bugs, timeout tests, performance tests. +They are executed before a commit on the pre-integration machines. They can be run on +the developer machine as well. + +
+
+Integration Tests<indexterm><primary>IntegrationTests</primary></indexterm> +Integration tests are system level tests. See +integration tests section for more info. + +
+
+ +
+Running tests +Below we describe how to run the Apache HBase junit categories. + +
+Default: small and medium category tests + +Running mvn test will execute all small tests in a single JVM +(no fork) and then medium tests in a separate JVM for each test instance. +Medium tests are NOT executed if there is an error in a small test. +Large tests are NOT executed. There is one report for small tests, and one report for +medium tests if they are executed. + +
+ +
+Running all tests +Running mvn test -P runAllTests +will execute small tests in a single JVM then medium and large tests in a separate JVM for each test. +Medium and large tests are NOT executed if there is an error in a small test. +Large tests are NOT executed if there is an error in a small or medium test. +There is one report for small tests, and one report for medium and large tests if they are executed. + +
+ +
+Running a single test or all tests in a package +To run an individual test, e.g. MyTest, do +mvn test -Dtest=MyTest You can also +pass multiple, individual tests as a comma-delimited list: +mvn test -Dtest=MyTest1,MyTest2,MyTest3 +You can also pass a package, which will run all tests under the package: +mvn test -Dtest=org.apache.hadoop.hbase.client.* + + + +When -Dtest is specified, localTests profile will be used. It will use the official release +of maven surefire, rather than our custom surefire plugin, and the old connector (The HBase build uses a patched +version of the maven surefire plugin). Each junit tests is executed in a separate JVM (A fork per test class). +There is no parallelization when tests are running in this mode. You will see a new message at the end of the +-report: "[INFO] Tests are skipped". It's harmless. While you need to make sure the sum of Tests run: in +the Results : section of test reports matching the number of tests you specified because no +error will be reported when a non-existent test case is specified. + +
+ +
+Other test invocation permutations +Running mvn test -P runSmallTests will execute "small" tests only, using a single JVM. + +Running mvn test -P runMediumTests will execute "medium" tests only, launching a new JVM for each test-class. + +Running mvn test -P runLargeTests will execute "large" tests only, launching a new JVM for each test-class. + +For convenience, you can run mvn test -P runDevTests to execute both small and medium tests, using a single JVM. + +
+ +
+Running tests faster + +By default, $ mvn test -P runAllTests runs 5 tests in parallel. +It can be increased on a developer's machine. Allowing that you can have 2 +tests in parallel per core, and you need about 2Gb of memory per test (at the +extreme), if you have an 8 core, 24Gb box, you can have 16 tests in parallel. +but the memory available limits it to 12 (24/2), To run all tests with 12 tests +in parallell, do this: +mvn test -P runAllTests -Dsurefire.secondPartThreadCount=12. +To increase the speed, you can as well use a ramdisk. You will need 2Gb of memory +to run all tests. You will also need to delete the files between two test run. +The typical way to configure a ramdisk on Linux is: +$ sudo mkdir /ram2G +sudo mount -t tmpfs -o size=2048M tmpfs /ram2G +You can then use it to run all HBase tests with the command: +mvn test -P runAllTests -Dsurefire.secondPartThreadCount=12 -Dtest.build.data.basedirectory=/ram2G + +
+ +
+<command>hbasetests.sh</command> +It's also possible to use the script hbasetests.sh. This script runs the medium and +large tests in parallel with two maven instances, and provides a single report. This script does not use +the hbase version of surefire so no parallelization is being done other than the two maven instances the +script sets up. +It must be executed from the directory which contains the pom.xml. +For example running +./dev-support/hbasetests.sh will execute small and medium tests. +Running ./dev-support/hbasetests.sh runAllTests will execute all tests. +Running ./dev-support/hbasetests.sh replayFailed will rerun the failed tests a +second time, in a separate jvm and without parallelisation. + +
+
+Test Resource Checker<indexterm><primary>Test Resource Checker</primary></indexterm> + +A custom Maven SureFire plugin listener checks a number of resources before +and after each HBase unit test runs and logs its findings at the end of the test +output files which can be found in target/surefire-reports +per Maven module (Tests write test reports named for the test class into this directory. +Check the *-out.txt files). The resources counted are the number +of threads, the number of file descriptors, etc. If the number has increased, it adds +a LEAK? comment in the logs. As you can have an HBase instance +running in the background, some threads can be deleted/created without any specific +action in the test. However, if the test does not work as expected, or if the test +should not impact these resources, it's worth checking these log lines +...hbase.ResourceChecker(157): before... and +...hbase.ResourceChecker(157): after.... For example: + +2012-09-26 09:22:15,315 INFO [pool-1-thread-1] hbase.ResourceChecker(157): after: regionserver.TestColumnSeeking#testReseeking Thread=65 (was 65), OpenFileDescriptor=107 (was 107), MaxFileDescriptor=10240 (was 10240), ConnectionCount=1 (was 1) + + +
+
+ +
+Writing Tests +
+General rules + + +As much as possible, tests should be written as category small tests. + + +All tests must be written to support parallel execution on the same machine, hence they should not use shared resources as fixed ports or fixed file names. + + +Tests should not overlog. More than 100 lines/second makes the logs complex to read and use i/o that are hence not available for the other tests. + + +Tests can be written with HBaseTestingUtility. +This class offers helper functions to create a temp directory and do the cleanup, or to start a cluster. + + +
+
+Categories and execution time + + +All tests must be categorized, if not they could be skipped. + + +All tests should be written to be as fast as possible. + + +Small category tests should last less than 15 seconds, and must not have any side effect. + + +Medium category tests should last less than 50 seconds. + + +Large category tests should last less than 3 minutes. This should ensure a good parallelization for people using it, and ease the analysis when the test fails. + + +
+
+Sleeps in tests +Whenever possible, tests should not use Thread.sleep, but rather waiting for the real event they need. This is faster and clearer for the reader. +Tests should not do a Thread.sleep without testing an ending condition. This allows understanding what the test is waiting for. Moreover, the test will work whatever the machine performance is. +Sleep should be minimal to be as fast as possible. Waiting for a variable should be done in a 40ms sleep loop. Waiting for a socket operation should be done in a 200 ms sleep loop. + +
+ +
+Tests using a cluster + + +Tests using a HRegion do not have to start a cluster: A region can use the local file system. +Start/stopping a cluster cost around 10 seconds. They should not be started per test method but per test class. +Started cluster must be shutdown using HBaseTestingUtility#shutdownMiniCluster, which cleans the directories. +As most as possible, tests should use the default settings for the cluster. When they don't, they should document it. This will allow to share the cluster later. + +
+
+ +
+Integration Tests +HBase integration/system tests are tests that are beyond HBase unit tests. They +are generally long-lasting, sizeable (the test can be asked to 1M rows or 1B rows), +targetable (they can take configuration that will point them at the ready-made cluster +they are to run against; integration tests do not include cluster start/stop code), +and verifying success, integration tests rely on public APIs only; they do not +attempt to examine server internals asserting success/fail. Integration tests +are what you would run when you need to more elaborate proofing of a release candidate +beyond what unit tests can do. They are not generally run on the Apache Continuous Integration +build server, however, some sites opt to run integration tests as a part of their +continuous testing on an actual cluster. + + +Integration tests currently live under the src/test directory +in the hbase-it submodule and will match the regex: **/IntegrationTest*.java. +All integration tests are also annotated with @Category(IntegrationTests.class). + + + +Integration tests can be run in two modes: using a mini cluster, or against an actual distributed cluster. +Maven failsafe is used to run the tests using the mini cluster. IntegrationTestsDriver class is used for +executing the tests against a distributed cluster. Integration tests SHOULD NOT assume that they are running against a +mini cluster, and SHOULD NOT use private API's to access cluster state. To interact with the distributed or mini +cluster uniformly, IntegrationTestingUtility, and HBaseCluster classes, +and public client API's can be used. + + + +On a distributed cluster, integration tests that use ChaosMonkey or otherwise manipulate services thru cluster manager (e.g. restart regionservers) use SSH to do it. +To run these, test process should be able to run commands on remote end, so ssh should be configured accordingly (for example, if HBase runs under hbase +user in your cluster, you can set up passwordless ssh for that user and run the test also under it). To facilitate that, hbase.it.clustermanager.ssh.user, +hbase.it.clustermanager.ssh.opts and hbase.it.clustermanager.ssh.cmd configuration settings can be used. "User" is the remote user that cluster manager should use to perform ssh commands. +"Opts" contains additional options that are passed to SSH (for example, "-i /tmp/my-key"). +Finally, if you have some custom environment setup, "cmd" is the override format for the entire tunnel (ssh) command. The default string is {/usr/bin/ssh %1$s %2$s%3$s%4$s "%5$s"} and is a good starting point. This is a standard Java format string with 5 arguments that is used to execute the remote command. The argument 1 (%1$s) is SSH options set the via opts setting or via environment variable, 2 is SSH user name, 3 is "@" if username is set or "" otherwise, 4 is the target host name, and 5 is the logical command to execute (that may include single quotes, so don't use them). For example, if you run the tests under non-hbase user and want to ssh as that user and change to hbase on remote machine, you can use {/usr/bin/ssh %1$s %2$s%3$s%4$s "su hbase - -c \"%5$s\""}. That way, to kill RS (for example) integration tests may run {/usr/bin/ssh some-hostname "su hbase - -c \"ps aux | ... | kill ...\""}. +The command is logged in the test logs, so you can verify it is correct for your environment. + + +
+Running integration tests against mini cluster +HBase 0.92 added a verify maven target. +Invoking it, for example by doing mvn verify, will +run all the phases up to and including the verify phase via the +maven failsafe plugin, +running all the above mentioned HBase unit tests as well as tests that are in the HBase integration test group. +After you have completed + mvn install -DskipTests +You can run just the integration tests by invoking: + +cd hbase-it +mvn verify + +If you just want to run the integration tests in top-level, you need to run two commands. First: + mvn failsafe:integration-test +This actually runs ALL the integration tests. + This command will always output BUILD SUCCESS even if there are test failures. + + At this point, you could grep the output by hand looking for failed tests. However, maven will do this for us; just use: + mvn failsafe:verify + The above command basically looks at all the test results (so don't remove the 'target' directory) for test failures and reports the results. + +
+ Running a subset of Integration tests + This is very similar to how you specify running a subset of unit tests (see above), but use the property + it.test instead of test. +To just run IntegrationTestClassXYZ.java, use: + mvn failsafe:integration-test -Dit.test=IntegrationTestClassXYZ + The next thing you might want to do is run groups of integration tests, say all integration tests that are named IntegrationTestClassX*.java: + mvn failsafe:integration-test -Dit.test=*ClassX* + This runs everything that is an integration test that matches *ClassX*. This means anything matching: "**/IntegrationTest*ClassX*". + You can also run multiple groups of integration tests using comma-delimited lists (similar to unit tests). Using a list of matches still supports full regex matching for each of the groups.This would look something like: + mvn failsafe:integration-test -Dit.test=*ClassX*, *ClassY + +
+
+
+Running integration tests against distributed cluster + +If you have an already-setup HBase cluster, you can launch the integration tests by invoking the class IntegrationTestsDriver. You may have to +run test-compile first. The configuration will be picked by the bin/hbase script. +mvn test-compile +Then launch the tests with: +bin/hbase [--config config_dir] org.apache.hadoop.hbase.IntegrationTestsDriver [-test=class_regex] + +This execution will launch the tests under hbase-it/src/test, having @Category(IntegrationTests.class) annotation, +and a name starting with IntegrationTests. If specified, class_regex will be used to filter test classes. The regex is checked against full class name; so, part of class name can be used. +IntegrationTestsDriver uses Junit to run the tests. Currently there is no support for running integration tests against a distributed cluster using maven (see HBASE-6201). + + + +The tests interact with the distributed cluster by using the methods in the DistributedHBaseCluster (implementing HBaseCluster) class, which in turn uses a pluggable ClusterManager. Concrete implementations provide actual functionality for carrying out deployment-specific and environment-dependent tasks (SSH, etc). The default ClusterManager is HBaseClusterManager, which uses SSH to remotely execute start/stop/kill/signal commands, and assumes some posix commands (ps, etc). Also assumes the user running the test has enough "power" to start/stop servers on the remote machines. By default, it picks up HBASE_SSH_OPTS, HBASE_HOME, HBASE_CONF_DIR from the env, and uses bin/hbase-daemon.sh to carry out the actions. Currently tarball deployments, deployments which uses hbase-daemons.sh, and Apache Ambari deployments are supported. /etc/init.d/ scripts are not supported for now, but it can be easily added. For other deployment options, a ClusterManager can be implemented and plugged in. + +
+ +
+Destructive integration / system tests + + In 0.96, a tool named ChaosMonkey has been introduced. It is modeled after the same-named tool by Netflix. +Some of the tests use ChaosMonkey to simulate faults in the running cluster in the way of killing random servers, +disconnecting servers, etc. ChaosMonkey can also be used as a stand-alone tool to run a (misbehaving) policy while you +are running other tests. + + + +ChaosMonkey defines Action's and Policy's. Actions are sequences of events. We have at least the following actions: + +Restart active master (sleep 5 sec) +Restart random regionserver (sleep 5 sec) +Restart random regionserver (sleep 60 sec) +Restart META regionserver (sleep 5 sec) +Restart ROOT regionserver (sleep 5 sec) +Batch restart of 50% of regionservers (sleep 5 sec) +Rolling restart of 100% of regionservers (sleep 5 sec) + + +Policies on the other hand are responsible for executing the actions based on a strategy. +The default policy is to execute a random action every minute based on predefined action +weights. ChaosMonkey executes predefined named policies until it is stopped. More than one +policy can be active at any time. + + + + To run ChaosMonkey as a standalone tool deploy your HBase cluster as usual. ChaosMonkey uses the configuration +from the bin/hbase script, thus no extra configuration needs to be done. You can invoke the ChaosMonkey by running: +bin/hbase org.apache.hadoop.hbase.util.ChaosMonkey + +This will output smt like: + +12/11/19 23:21:57 INFO util.ChaosMonkey: Using ChaosMonkey Policy: class org.apache.hadoop.hbase.util.ChaosMonkey$PeriodicRandomActionPolicy, period:60000 +12/11/19 23:21:57 INFO util.ChaosMonkey: Sleeping for 26953 to add jitter +12/11/19 23:22:24 INFO util.ChaosMonkey: Performing action: Restart active master +12/11/19 23:22:24 INFO util.ChaosMonkey: Killing master:master.example.com,60000,1353367210440 +12/11/19 23:22:24 INFO hbase.HBaseCluster: Aborting Master: master.example.com,60000,1353367210440 +12/11/19 23:22:24 INFO hbase.ClusterManager: Executing remote command: ps aux | grep master | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:master.example.com +12/11/19 23:22:25 INFO hbase.ClusterManager: Executed remote command, exit code:0 , output: +12/11/19 23:22:25 INFO hbase.HBaseCluster: Waiting service:master to stop: master.example.com,60000,1353367210440 +12/11/19 23:22:25 INFO hbase.ClusterManager: Executing remote command: ps aux | grep master | grep -v grep | tr -s ' ' | cut -d ' ' -f2 , hostname:master.example.com +12/11/19 23:22:25 INFO hbase.ClusterManager: Executed remote command, exit code:0 , output: +12/11/19 23:22:25 INFO util.ChaosMonkey: Killed master server:master.example.com,60000,1353367210440 +12/11/19 23:22:25 INFO util.ChaosMonkey: Sleeping for:5000 +12/11/19 23:22:30 INFO util.ChaosMonkey: Starting master:master.example.com +12/11/19 23:22:30 INFO hbase.HBaseCluster: Starting Master on: master.example.com +12/11/19 23:22:30 INFO hbase.ClusterManager: Executing remote command: /homes/enis/code/hbase-0.94/bin/../bin/hbase-daemon.sh --config /homes/enis/code/hbase-0.94/bin/../conf start master , hostname:master.example.com +12/11/19 23:22:31 INFO hbase.ClusterManager: Executed remote command, exit code:0 , output:starting master, logging to /homes/enis/code/hbase-0.94/bin/../logs/hbase-enis-master-master.example.com.out +.... +12/11/19 23:22:33 INFO util.ChaosMonkey: Started master: master.example.com,60000,1353367210440 +12/11/19 23:22:33 INFO util.ChaosMonkey: Sleeping for:51321 +12/11/19 23:23:24 INFO util.ChaosMonkey: Performing action: Restart random region server +12/11/19 23:23:24 INFO util.ChaosMonkey: Killing region server:rs3.example.com,60020,1353367027826 +12/11/19 23:23:24 INFO hbase.HBaseCluster: Aborting RS: rs3.example.com,60020,1353367027826 +12/11/19 23:23:24 INFO hbase.ClusterManager: Executing remote command: ps aux | grep regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:rs3.example.com +12/11/19 23:23:25 INFO hbase.ClusterManager: Executed remote command, exit code:0 , output: +12/11/19 23:23:25 INFO hbase.HBaseCluster: Waiting service:regionserver to stop: rs3.example.com,60020,1353367027826 +12/11/19 23:23:25 INFO hbase.ClusterManager: Executing remote command: ps aux | grep regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 , hostname:rs3.example.com +12/11/19 23:23:25 INFO hbase.ClusterManager: Executed remote command, exit code:0 , output: +12/11/19 23:23:25 INFO util.ChaosMonkey: Killed region server:rs3.example.com,60020,1353367027826. Reported num of rs:6 +12/11/19 23:23:25 INFO util.ChaosMonkey: Sleeping for:60000 +12/11/19 23:24:25 INFO util.ChaosMonkey: Starting region server:rs3.example.com +12/11/19 23:24:25 INFO hbase.HBaseCluster: Starting RS on: rs3.example.com +12/11/19 23:24:25 INFO hbase.ClusterManager: Executing remote command: /homes/enis/code/hbase-0.94/bin/../bin/hbase-daemon.sh --config /homes/enis/code/hbase-0.94/bin/../conf start regionserver , hostname:rs3.example.com +12/11/19 23:24:26 INFO hbase.ClusterManager: Executed remote command, exit code:0 , output:starting regionserver, logging to /homes/enis/code/hbase-0.94/bin/../logs/hbase-enis-regionserver-rs3.example.com.out + +12/11/19 23:24:27 INFO util.ChaosMonkey: Started region server:rs3.example.com,60020,1353367027826. Reported num of rs:6 + + +As you can see from the log, ChaosMonkey started the default PeriodicRandomActionPolicy, which is configured with all the available actions, and ran RestartActiveMaster and RestartRandomRs actions. ChaosMonkey tool, if run from command line, will keep on running until the process is killed. + +
+
+
+ +
+ Maven Build Commands + All commands executed from the local HBase project directory. + + Note: use Maven 3 (Maven 2 may work but we suggest you use Maven 3). + +
+ Compile + +mvn compile + +
+ +
+ Running all or individual Unit Tests + See the section + above in +
+ +
+ Building against various hadoop versions. + As of 0.96, Apache HBase supports building against Apache Hadoop versions: 1.0.3, 2.0.0-alpha and 3.0.0-SNAPSHOT. + By default, we will build with Hadoop-1.0.3. To change the version to run with Hadoop-2.0.0-alpha, you would run: + mvn -Dhadoop.profile=2.0 ... + + That is, designate build with hadoop.profile 2.0. Pass 2.0 for hadoop.profile to build against hadoop 2.0. + Tests may not all pass as of this writing so you may need to pass -DskipTests unless you are inclined + to fix the failing tests. + + Similarly, for 3.0, you would just replace the profile value. Note that Hadoop-3.0.0-SNAPSHOT does not currently have a deployed maven artificat - you will need to build and install your own in your local maven repository if you want to run against this profile. + + + In earilier verions of Apache HBase, you can build against older versions of Apache Hadoop, notably, Hadoop 0.22.x and 0.23.x. + If you are running, for example HBase-0.94 and wanted to build against Hadoop 0.23.x, you would run with: + mvn -Dhadoop.profile=22 ... +
+
+ +
+ Getting Involved + Apache HBase gets better only when people contribute! + + As Apache HBase is an Apache Software Foundation project, see for more information about how the ASF functions. + +
+ Mailing Lists + Sign up for the dev-list and the user-list. See the + mailing lists page. + Posing questions - and helping to answer other people's questions - is encouraged! + There are varying levels of experience on both lists so patience and politeness are encouraged (and please + stay on topic.) + +
+
+ Jira + Check for existing issues in Jira. + If it's either a new feature request, enhancement, or a bug, file a ticket. + +
Jira Priorities + The following is a guideline on setting Jira issue priorities: + + Blocker: Should only be used if the issue WILL cause data loss or cluster instability reliably. + Critical: The issue described can cause data loss or cluster instability in some cases. + Major: Important but not tragic issues, like updates to the client API that will add a lot of much-needed functionality or significant + bugs that need to be fixed but that don't cause data loss. + Minor: Useful enhancements and annoying but not damaging bugs. + Trivial: Useful enhancements but generally cosmetic. + + +
+
+ Code Blocks in Jira Comments + A commonly used macro in Jira is {code}. If you do this in a Jira comment... + +{code} + code snippet +{code} + + ... Jira will format the code snippet like code, instead of a regular comment. It improves readability. + +
+
+
+ +
+ Developing +
Codelines + Most development is done on TRUNK. However, there are branches for minor releases (e.g., 0.90.1, 0.90.2, and 0.90.3 are on the 0.90 branch). + If you have any questions on this just send an email to the dev dist-list. +
+ +
+ Unit Tests + In HBase we use JUnit 4. + If you need to run miniclusters of HDFS, ZooKeeper, HBase, or MapReduce testing, + be sure to checkout the HBaseTestingUtility. + Alex Baranau of Sematext describes how it can be used in + HBase Case-Study: Using HBaseTestingUtility for Local Testing and Development (2010). + +
+ Mockito + Sometimes you don't need a full running server + unit testing. For example, some methods can make do with a + a org.apache.hadoop.hbase.Server instance + or a org.apache.hadoop.hbase.master.MasterServices + Interface reference rather than a full-blown + org.apache.hadoop.hbase.master.HMaster. + In these cases, you maybe able to get away with a mocked + Server instance. For example: + + TODO... + + +
+
+ +
+ Code Standards + See and . + + Also, please pay attention to the interface stability/audience classifications that you + will see all over our code base. They look like this at the head of the class: + @InterfaceAudience.Public +@InterfaceStability.Stable + + If the InterfaceAudience is Private, + we can change the class (and we do not need to include a InterfaceStability mark). + If a class is marked Public but its InterfaceStability + is marked Unstable, we can change it. If it's + marked Public/Evolving, we're allowed to change it + but should try not to. If it's Public and Stable + we can't change it without a deprecation path or with a really GREAT reason. + When you add new classes, mark them with the annotations above if publically accessible. + If you are not cleared on how to mark your additions, ask up on the dev list. + + This convention comes from our parent project Hadoop. +
+ +
+ Invariants + We don't have many but what we have we list below. All are subject to challenge of + course but until then, please hold to the rules of the road. + +
+ No permanent state in ZooKeeper + ZooKeeper state should transient (treat it like memory). If deleted, hbase + should be able to recover and essentially be in the same stateThere are currently + a few exceptions that we need to fix around whether a table is enabled or disabled. + +
+ +
+ +
+ Running In-Situ + If you are developing Apache HBase, frequently it is useful to test your changes against a more-real cluster than what you find in unit tests. In this case, HBase can be run directly from the source in local-mode. + All you need to do is run: + + ${HBASE_HOME}/bin/start-hbase.sh + + This will spin up a full local-cluster, just as if you had packaged up HBase and installed it on your machine. + + Keep in mind that you will need to have installed HBase into your local maven repository for the in-situ cluster to work properly. That is, you will need to run: + mvn clean install -DskipTests + to ensure that maven can find the correct classpath and dependencies. Generally, the above command + is just a good thing to try running first, if maven is acting oddly. +
+ +
+ +
+ Submitting Patches + If you are new to submitting patches to open source or new to submitting patches to Apache, + I'd suggest you start by reading the On Contributing Patches + page from Apache Commons Project. Its a nice overview that + applies equally to the Apache HBase Project. +
+ Create Patch + See the aforementioned Apache Commons link for how to make patches against a checked out subversion + repository. Patch files can also be easily generated from Eclipse, for example by selecting "Team -> Create Patch". + Patches can also be created by git diff and svn diff. + + Please submit one patch-file per Jira. For example, if multiple files are changed make sure the + selected resource when generating the patch is a directory. Patch files can reflect changes in multiple files. + Make sure you review for code style. +
+
+ Patch File Naming + The patch file should have the Apache HBase Jira ticket in the name. For example, if a patch was submitted for Foo.java, then + a patch file called Foo_HBASE_XXXX.patch would be acceptable where XXXX is the Apache HBase Jira number. + + If you generating from a branch, then including the target branch in the filename is advised, e.g., HBASE-XXXX-0.90.patch. + +
+
+ Unit Tests + Yes, please. Please try to include unit tests with every code patch (and especially new classes and large changes). + Make sure unit tests pass locally before submitting the patch. + Also, see . + If you are creating a new unit test class, notice how other unit test classes have classification/sizing + annotations at the top and a static method on the end. Be sure to include these in any new unit test files + you generate. See for more on how the annotations work. + +
+
+ Attach Patch to Jira + The patch should be attached to the associated Jira ticket "More Actions -> Attach Files". Make sure you click the + ASF license inclusion, otherwise the patch can't be considered for inclusion. + + Once attached to the ticket, click "Submit Patch" and + the status of the ticket will change. Committers will review submitted patches for inclusion into the codebase. Please + understand that not every patch may get committed, and that feedback will likely be provided on the patch. Fear not, though, + because the Apache HBase community is helpful! + +
+ +
+ Common Patch Feedback + The following items are representative of common patch feedback. Your patch process will go faster if these are + taken into account before submission. + + + See the Java coding standards + for more information on coding conventions in Java. + +
+ Space Invaders + Rather than do this... + +if ( foo.equals( bar ) ) { // don't do this + + ... do this instead... + +if (foo.equals(bar)) { + + + Also, rather than do this... + +foo = barArray[ i ]; // don't do this + + ... do this instead... + +foo = barArray[i]; + + +
+
+ Auto Generated Code + Auto-generated code in Eclipse often looks like this... + + public void readFields(DataInput arg0) throws IOException { // don't do this + foo = arg0.readUTF(); // don't do this + + ... do this instead ... + + public void readFields(DataInput di) throws IOException { + foo = di.readUTF(); + + See the difference? 'arg0' is what Eclipse uses for arguments by default. + +
+
+ Long Lines + + Keep lines less than 100 characters. + +Bar bar = foo.veryLongMethodWithManyArguments(argument1, argument2, argument3, argument4, argument5, argument6, argument7, argument8, argument9); // don't do this + + ... do something like this instead ... + +Bar bar = foo.veryLongMethodWithManyArguments( + argument1, argument2, argument3,argument4, argument5, argument6, argument7, argument8, argument9); + + +
+
+ Trailing Spaces + + This happens more than people would imagine. + +Bar bar = foo.getBar(); <--- imagine there's an extra space(s) after the semicolon instead of a line break. + + Make sure there's a line-break after the end of your code, and also avoid lines that have nothing + but whitespace. + +
+
+ Implementing Writable + + Applies pre-0.96 only + In 0.96, HBase moved to protobufs. The below section on Writables + applies to 0.94.x and previous, not to 0.96 and beyond. + + + Every class returned by RegionServers must implement Writable. If you + are creating a new class that needs to implement this interface, don't forget the default constructor. + +
+
+ Javadoc + This is also a very common feedback item. Don't forget Javadoc! + Javadoc warnings are checked during precommit. If the precommit tool gives you a '-1', + please fix the javadoc issue. Your patch won't be committed if it adds such warnings. + + +
+
+ Findbugs + + Findbugs is used to detect common bugs pattern. As Javadoc, it is checked during + the precommit build up on Apache's Jenkins, and as with Javadoc, please fix them. + You can run findbugs locally with 'mvn findbugs:findbugs': it will generate the + findbugs files locally. Sometimes, you may have to write code smarter than + Findbugs. You can annotate your code to tell Findbugs you know what you're + doing, by annotating your class with: + @edu.umd.cs.findbugs.annotations.SuppressWarnings( + value="HE_EQUALS_USE_HASHCODE", + justification="I know what I'm doing") + + + Note that we're using the apache licensed version of the annotations. + +
+ +
+ Javadoc - Useless Defaults + Don't just leave the @param arguments the way your IDE generated them. Don't do this... + + /** + * + * @param bar <---- don't do this!!!! + * @return <---- or this!!!! + */ + public Foo getFoo(Bar bar); + + ... either add something descriptive to the @param and @return lines, or just remove them. + But the preference is to add something descriptive and useful. + +
+
+ One Thing At A Time, Folks + If you submit a patch for one thing, don't do auto-reformatting or unrelated reformatting of code on a completely + different area of code. + + Likewise, don't add unrelated cleanup or refactorings outside the scope of your Jira. + +
+
+ Ambigious Unit Tests + Make sure that you're clear about what you are testing in your unit tests and why. + +
+ +
+ +
+ ReviewBoard + Larger patches should go through ReviewBoard. + + For more information on how to use ReviewBoard, see + the ReviewBoard documentation. + +
+
+ Committing Patches + + Committers do this. See How To Commit in the Apache HBase wiki. + + Commiters will also resolve the Jira, typically after the patch passes a build. + +
+ Committers are responsible for making sure commits do not break the build or tests + + If a committer commits a patch it is their responsibility + to make sure it passes the test suite. It is helpful + if contributors keep an eye out that their patch + does not break the hbase build and/or tests but ultimately, + a contributor cannot be expected to be up on the + particular vagaries and interconnections that occur + in a project like hbase. A committer should. + +
+
+ +
+ + + diff --git a/src/docbkx/external_apis.xml b/src/docbkx/external_apis.xml new file mode 100644 index 0000000..6380b6e --- /dev/null +++ b/src/docbkx/external_apis.xml @@ -0,0 +1,424 @@ + + + + Apache HBase (TM) External APIs + This chapter will cover access to Apache HBase (TM) either through non-Java languages, or through custom protocols. + +
+ Non-Java Languages Talking to the JVM + Currently the documentation on this topic in the + Apache HBase Wiki. + See also the Thrift API Javadoc. + +
+ +
+ REST + Currently most of the documentation on REST exists in the + Apache HBase Wiki on REST (The REST gateway used to be + called 'Stargate'). There are also a nice set of blogs on How-to: Use the Apache HBase REST Interface + by Jesse Anderson. + +
+ +
+ Thrift + Currently most of the documentation on Thrift exists in the + Apache HBase Wiki on Thrift. + +
Filter Language +
Use Case + Note: this feature was introduced in Apache HBase 0.92 + This allows the user to perform server-side filtering when accessing HBase over Thrift. The user specifies a filter via a string. The string is parsed on the server to construct the filter +
+ +
General Filter String Syntax + A simple filter expression is expressed as: “FilterName (argument, argument, ... , argument)” + You must specify the name of the filter followed by the argument list in parenthesis. Commas separate the individual arguments + If the argument represents a string, it should be enclosed in single quotes. + If it represents a boolean, an integer or a comparison operator like <, + >, != etc. it should not be enclosed in quotes + The filter name must be one word. All ASCII characters are allowed except for whitespace, single quotes and parenthesis. + The filter’s arguments can contain any ASCII character. If single quotes are present in the argument, they must be escaped by a + preceding single quote +
+ +
Compound Filters and Operators + Currently, two binary operators – AND/OR and two unary operators – WHILE/SKIP are supported. + Note: the operators are all in uppercase + AND – as the name suggests, if this + operator is used, the key-value must pass both the filters + OR – as the name suggests, if this operator + is used, the key-value must pass at least one of the filters + SKIP – For a particular row, if any of the + key-values don’t pass the filter condition, the entire row is skipped + WHILE - For a particular row, it continues + to emit key-values until a key-value is reached that fails the filter condition + Compound Filters: Using these operators, a + hierarchy of filters can be created. For example: “(Filter1 AND Filter2) OR (Filter3 AND Filter4)” +
+ +
Order of Evaluation + Parenthesis have the highest precedence. The SKIP and WHILE operators are next and have the same precedence.The AND operator has the next highest precedence followed by the OR operator. + For example: + A filter string of the form:“Filter1 AND Filter2 OR Filter3” + will be evaluated as:“(Filter1 AND Filter2) OR Filter3” + A filter string of the form:“Filter1 AND SKIP Filter2 OR Filter3” + will be evaluated as:“(Filter1 AND (SKIP Filter2)) OR Filter3” +
+ +
Compare Operator + A compare operator can be any of the following: + + + LESS (<) + + + LESS_OR_EQUAL (<=) + + + EQUAL (=) + + + NOT_EQUAL (!=) + + + GREATER_OR_EQUAL (>=) + + + GREATER (>) + + + NO_OP (no operation) + + + The client should use the symbols (<, <=, =, !=, >, >=) to express + compare operators. +
+ +
Comparator + A comparator can be any of the following: + + + BinaryComparator - This + lexicographically compares against the specified byte array using + Bytes.compareTo(byte[], byte[]) + + + BinaryPrefixComparator - This + lexicographically compares against a specified byte array. It only compares up to + the length of this byte array. + + + RegexStringComparator - This compares + against the specified byte array using the given regular expression. Only EQUAL + and NOT_EQUAL comparisons are valid with this comparator + + + SubStringComparator - This tests if + the given substring appears in a specified byte array. The comparison is case + insensitive. Only EQUAL and NOT_EQUAL comparisons are valid with this + comparator + + + The general syntax of a comparator is: ComparatorType:ComparatorValue + The ComparatorType for the various comparators is as follows: + + + BinaryComparator - binary + + + BinaryPrefixComparator - binaryprefix + + + RegexStringComparator - regexstring + + + SubStringComparator - substring + + + The ComparatorValue can be any value. + Example1: >, 'binary:abc' will match everything that is lexicographically greater than "abc" + Example2: =, 'binaryprefix:abc' will match everything whose first 3 characters are lexicographically equal to "abc" + Example3: !=, 'regexstring:ab*yz' will match everything that doesn't begin with "ab" and ends with "yz" + Example4: =, 'substring:abc123' will match everything that begins with the substring "abc123" +
+ +
Example PHP Client Program that uses the Filter Language + +<? $_SERVER['PHP_ROOT'] = realpath(dirname(__FILE__).'/..'); + require_once $_SERVER['PHP_ROOT'].'/flib/__flib.php'; + flib_init(FLIB_CONTEXT_SCRIPT); + require_module('storage/hbase'); + $hbase = new HBase('<server_name_running_thrift_server>', <port on which thrift server is running>); + $hbase->open(); + $client = $hbase->getClient(); + $result = $client->scannerOpenWithFilterString('table_name', "(PrefixFilter ('row2') AND (QualifierFilter (>=, 'binary:xyz'))) AND (TimestampsFilter ( 123, 456))"); + $to_print = $client->scannerGetList($result,1); + while ($to_print) { + print_r($to_print); + $to_print = $client->scannerGetList($result,1); + } + $client->scannerClose($result); +?> + +
+ +
Example Filter Strings + + + + “PrefixFilter (‘Row’) AND PageFilter (1) AND FirstKeyOnlyFilter ()” will return all key-value pairs that match the following conditions: + 1) The row containing the key-value should have prefix “Row” + 2) The key-value must be located in the first row of the table + 3) The key-value pair must be the first key-value in the row + + + + + + + + + “(RowFilter (=, ‘binary:Row 1’) AND TimeStampsFilter (74689, 89734)) OR + ColumnRangeFilter (‘abc’, true, ‘xyz’, false))” will return all key-value pairs that match both the following conditions: + 1) The key-value is in a row having row key “Row 1” + 2) The key-value must have a timestamp of either 74689 or 89734. + Or it must match the following condition: + 1) The key-value pair must be in a column that is lexicographically >= abc and < xyz  + + + + + + + + + “SKIP ValueFilter (0)” will skip the entire row if any of the values in the row is not 0 + + + +
+ +
Individual Filter Syntax + + + KeyOnlyFilter + Description: This filter doesn’t take any + arguments. It returns only the key component of each key-value. + Syntax: KeyOnlyFilter () + Example: "KeyOnlyFilter ()" + + + + FirstKeyOnlyFilter + Description: This filter doesn’t take any + arguments. It returns only the first key-value from each row. + Syntax: FirstKeyOnlyFilter () + Example: "FirstKeyOnlyFilter ()" + + + + PrefixFilter + Description: This filter takes one argument – a prefix of a + row key. It returns only those key-values present in a row that starts with the + specified row prefix + Syntax: PrefixFilter (‘<row_prefix>’) + Example: "PrefixFilter (‘Row’)" + + + + + ColumnPrefixFilter + Description: This filter takes one argument + – a column prefix. It returns only those key-values present in a column that starts + with the specified column prefix. The column prefix must be of the form: “qualifier” + Syntax:ColumnPrefixFilter(‘<column_prefix>’) + Example: "ColumnPrefixFilter(‘Col’)" + + + + MultipleColumnPrefixFilter + Description: This filter takes a list of + column prefixes. It returns key-values that are present in a column that starts with + any of the specified column prefixes. Each of the column prefixes must be of the form: “qualifier” + Syntax:MultipleColumnPrefixFilter(‘<column_prefix>’, ‘<column_prefix>’, …, ‘<column_prefix>’) + Example: "MultipleColumnPrefixFilter(‘Col1’, ‘Col2’)" + + + + ColumnCountGetFilter + Description: This filter takes one argument + – a limit. It returns the first limit number of columns in the table + Syntax: ColumnCountGetFilter (‘<limit>’) + Example: "ColumnCountGetFilter (4)" + + + + PageFilter + Description: This filter takes one argument + – a page size. It returns page size number of rows from the table. + Syntax: PageFilter (‘<page_size>’) + Example: "PageFilter (2)" + + + + ColumnPaginationFilter + Description: This filter takes two + arguments – a limit and offset. It returns limit number of columns after offset number + of columns. It does this for all the rows + Syntax: ColumnPaginationFilter(‘<limit>’, ‘<offest>’) + Example: "ColumnPaginationFilter (3, 5)" + + + + InclusiveStopFilter + Description: This filter takes one argument + – a row key on which to stop scanning. It returns all key-values present in rows up to + and including the specified row + Syntax: InclusiveStopFilter(‘<stop_row_key>’) + Example: "InclusiveStopFilter ('Row2')" + + + + TimeStampsFilter + Description: This filter takes a list of + timestamps. It returns those key-values whose timestamps matches any of the specified + timestamps + Syntax: TimeStampsFilter (<timestamp>, <timestamp>, ... ,<timestamp>) + Example: "TimeStampsFilter (5985489, 48895495, 58489845945)" + + + + RowFilter + Description: This filter takes a compare + operator and a comparator. It compares each row key with the comparator using the + compare operator and if the comparison returns true, it returns all the key-values in + that row + Syntax: RowFilter (<compareOp>, ‘<row_comparator>’) + Example: "RowFilter (<=, ‘xyz)" + + + + Family Filter + Description: This filter takes a compare + operator and a comparator. It compares each qualifier name with the comparator using + the compare operator and if the comparison returns true, it returns all the key-values + in that column + Syntax: QualifierFilter (<compareOp>, ‘<qualifier_comparator>’) + Example: "QualifierFilter (=, ‘Column1’)" + + + + QualifierFilter + Description: This filter takes a compare + operator and a comparator. It compares each qualifier name with the comparator using + the compare operator and if the comparison returns true, it returns all the key-values + in that column + Syntax: QualifierFilter (<compareOp>,‘<qualifier_comparator>’) + Example: "QualifierFilter (=,‘Column1’)" + + + + ValueFilter + Description: This filter takes a compare operator and a + comparator. It compares each value with the comparator using the compare operator and + if the comparison returns true, it returns that key-value + Syntax: ValueFilter (<compareOp>,‘<value_comparator>’) + Example: "ValueFilter (!=, ‘Value’)" + + + + DependentColumnFilter + Description: This filter takes two arguments – a family + and a qualifier. It tries to locate this column in each row and returns all key-values + in that row that have the same timestamp. If the row doesn’t contain the specified + column – none of the key-values in that row will be returned. + The filter can also take an optional boolean argument – dropDependentColumn. If set to true, the column we were depending on doesn’t get returned. + The filter can also take two more additional optional arguments – a compare operator and a value comparator, which are further checks in addition to the family and qualifier. If the dependent column is found, its value should also pass the value check and then only is its timestamp taken into consideration + Syntax: DependentColumnFilter (‘<family>’, ‘<qualifier>’, <boolean>, <compare operator>, ‘<value comparator’) + Syntax: DependentColumnFilter (‘<family>’, ‘<qualifier>’, <boolean>) + Syntax: DependentColumnFilter (‘<family>’, ‘<qualifier>’) + Example: "DependentColumnFilter (‘conf’, ‘blacklist’, false, >=, ‘zebra’)" + Example: "DependentColumnFilter (‘conf’, 'blacklist', true)" + Example: "DependentColumnFilter (‘conf’, 'blacklist')" + + + + SingleColumnValueFilter + Description: This filter takes a column family, a + qualifier, a compare operator and a comparator. If the specified column is not found – + all the columns of that row will be emitted. If the column is found and the comparison + with the comparator returns true, all the columns of the row will be emitted. If the + condition fails, the row will not be emitted. + This filter also takes two additional optional boolean arguments – filterIfColumnMissing and setLatestVersionOnly + If the filterIfColumnMissing flag is set to true the columns of the row will not be emitted if the specified column to check is not found in the row. The default value is false. + If the setLatestVersionOnly flag is set to false, it will test previous versions (timestamps) too. The default value is true. + These flags are optional and if you must set neither or both + Syntax: SingleColumnValueFilter(<compare operator>, ‘<comparator>’, ‘<family>’, ‘<qualifier>’,<filterIfColumnMissing_boolean>, <latest_version_boolean>) + Syntax: SingleColumnValueFilter(<compare operator>, ‘<comparator>’, ‘<family>’, ‘<qualifier>) + Example: "SingleColumnValueFilter (<=, ‘abc’,‘FamilyA’, ‘Column1’, true, false)" + Example: "SingleColumnValueFilter (<=, ‘abc’,‘FamilyA’, ‘Column1’)" + + + + SingleColumnValueExcludeFilter + Description: This filter takes the same arguments and + behaves same as SingleColumnValueFilter – however, if the column is found and the + condition passes, all the columns of the row will be emitted except for the tested + column value. + Syntax: SingleColumnValueExcludeFilter(<compare operator>, '<comparator>', '<family>', '<qualifier>',<latest_version_boolean>, <filterIfColumnMissing_boolean>) + Syntax: SingleColumnValueExcludeFilter(<compare operator>, '<comparator>', '<family>', '<qualifier>') + Example: "SingleColumnValueExcludeFilter (‘<=’, ‘abc’,‘FamilyA’, ‘Column1’, ‘false’, ‘true’)" + Example: "SingleColumnValueExcludeFilter (‘<=’, ‘abc’, ‘FamilyA’, ‘Column1’)" + + + + ColumnRangeFilter + Description: This filter is used for selecting only those + keys with columns that are between minColumn and maxColumn. It also takes two boolean + variables to indicate whether to include the minColumn and maxColumn or not. + If you don’t want to set the minColumn or the maxColumn – you can pass in an empty argument. + Syntax: ColumnRangeFilter (‘<minColumn>’, <minColumnInclusive_bool>, ‘<maxColumn>’, <maxColumnInclusive_bool>) + Example: "ColumnRangeFilter (‘abc’, true, ‘xyz’, false)" + + + +
+ +
+ +
+ +
+ C/C++ Apache HBase Client + FB's Chip Turner wrote a pure C/C++ client. Check it out. + +
+ + diff --git a/src/docbkx/getting_started.xml b/src/docbkx/getting_started.xml new file mode 100644 index 0000000..e1c4344 --- /dev/null +++ b/src/docbkx/getting_started.xml @@ -0,0 +1,221 @@ + + + + Getting Started + +
+ Introduction + + will get you up and + running on a single-node instance of HBase using the local filesystem. + +
+ +
+ Quick Start + + This guide describes setup of a standalone HBase instance that uses + the local filesystem. It leads you through creating a table, inserting + rows via the HBase shell, and then cleaning + up and shutting down your standalone HBase instance. The below exercise + should take no more than ten minutes (not including download time). + Before we proceed, make sure you are good on the below loopback prerequisite. + + Loopback IP + HBase expects the loopback IP address to be 127.0.0.1. Ubuntu and some other distributions, + for example, will default to 127.0.1.1 and this will cause problems for you. + + /etc/hosts should look something like this: + + 127.0.0.1 localhost + 127.0.0.1 ubuntu.ubuntu-domain ubuntu + + + + + +
+ Download and unpack the latest stable release. + + Choose a download site from this list of Apache Download + Mirrors. Click on the suggested top link. This will take you to a + mirror of HBase Releases. Click on the folder named + stable and then download the file that ends in + .tar.gz to your local filesystem; e.g. + hbase-0.94.2.tar.gz. + + Decompress and untar your download and then change into the + unpacked directory. + + $ tar xfz hbase-.tar.gz +$ cd hbase- + + + At this point, you are ready to start HBase. But before starting + it, edit conf/hbase-site.xml, the file you write + your site-specific configurations into. Set + hbase.rootdir, the directory HBase writes data to, + and hbase.zookeeper.property.dataDir, the director + ZooKeeper writes its data too: +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration> + <property> + <name>hbase.rootdir</name> + <value>file:///DIRECTORY/hbase</value> + </property> + <property> + <name>hbase.zookeeper.property.dataDir</name> + <value>/DIRECTORY/zookeeper</value> + </property> +</configuration> Replace DIRECTORY in the above with the + path to the directory you would have HBase and ZooKeeper write their data. By default, + hbase.rootdir is set to /tmp/hbase-${user.name} + and similarly so for the default ZooKeeper data location which means you'll lose all + your data whenever your server reboots unless you change it (Most operating systems clear + /tmp on restart). +
+ +
+ Start HBase + + Now start HBase:$ ./bin/start-hbase.sh +starting Master, logging to logs/hbase-user-master-example.org.out + + You should now have a running standalone HBase instance. In + standalone mode, HBase runs all daemons in the the one JVM; i.e. both + the HBase and ZooKeeper daemons. HBase logs can be found in the + logs subdirectory. Check them out especially if + it seems HBase had trouble starting. + + + Is <application>java</application> installed? + + All of the above presumes a 1.6 version of Oracle + java is installed on your machine and + available on your path (See ); i.e. when you type + java, you see output that describes the + options the java program takes (HBase requires java 6). If this is not + the case, HBase will not start. Install java, edit + conf/hbase-env.sh, uncommenting the + JAVA_HOME line pointing it to your java install, then, + retry the steps above. + +
+ +
+ Shell Exercises + + Connect to your running HBase via the shell. + + $ ./bin/hbase shell +HBase Shell; enter 'help<RETURN>' for list of supported commands. +Type "exit<RETURN>" to leave the HBase Shell +Version: 0.90.0, r1001068, Fri Sep 24 13:55:42 PDT 2010 + +hbase(main):001:0> + + Type help and then + <RETURN> to see a listing of shell commands and + options. Browse at least the paragraphs at the end of the help emission + for the gist of how variables and command arguments are entered into the + HBase shell; in particular note how table names, rows, and columns, + etc., must be quoted. + + Create a table named test with a single column family named cf. + Verify its creation by listing all tables and then insert some + values. + + hbase(main):003:0> create 'test', 'cf' +0 row(s) in 1.2200 seconds +hbase(main):003:0> list 'test' +.. +1 row(s) in 0.0550 seconds +hbase(main):004:0> put 'test', 'row1', 'cf:a', 'value1' +0 row(s) in 0.0560 seconds +hbase(main):005:0> put 'test', 'row2', 'cf:b', 'value2' +0 row(s) in 0.0370 seconds +hbase(main):006:0> put 'test', 'row3', 'cf:c', 'value3' +0 row(s) in 0.0450 seconds + + Above we inserted 3 values, one at a time. The first insert is at + row1, column cf:a with a value of + value1. Columns in HBase are comprised of a column family prefix -- + cf in this example -- followed by a colon and then a + column qualifier suffix (a in this case). + + Verify the data insert by running a scan of the table as follows + + hbase(main):007:0> scan 'test' +ROW COLUMN+CELL +row1 column=cf:a, timestamp=1288380727188, value=value1 +row2 column=cf:b, timestamp=1288380738440, value=value2 +row3 column=cf:c, timestamp=1288380747365, value=value3 +3 row(s) in 0.0590 seconds + + Get a single row + + hbase(main):008:0> get 'test', 'row1' +COLUMN CELL +cf:a timestamp=1288380727188, value=value1 +1 row(s) in 0.0400 seconds + + Now, disable and drop your table. This will clean up all done + above. + + hbase(main):012:0> disable 'test' +0 row(s) in 1.0930 seconds +hbase(main):013:0> drop 'test' +0 row(s) in 0.0770 seconds + + Exit the shell by typing exit. + + hbase(main):014:0> exit +
+ +
+ Stopping HBase + + Stop your hbase instance by running the stop script. + + $ ./bin/stop-hbase.sh +stopping hbase............... +
+ +
+ Where to go next + + The above described standalone setup is good for testing and + experiments only. In the next chapter, , + we'll go into depth on the different HBase run modes, system requirements + running HBase, and critical configurations setting up a distributed HBase deploy. +
+
+ + diff --git a/src/docbkx/ops_mgt.xml b/src/docbkx/ops_mgt.xml new file mode 100644 index 0000000..0009ab4 --- /dev/null +++ b/src/docbkx/ops_mgt.xml @@ -0,0 +1,839 @@ + + + + Apache HBase (TM) Operational Management + This chapter will cover operational tools and practices required of a running Apache HBase cluster. + The subject of operations is related to the topics of , , + and but is a distinct topic in itself. + +
+ HBase Tools and Utilities + + Here we list HBase tools for administration, analysis, fixup, and + debugging. +
Driver + There is a Driver class that is executed by the HBase jar can be used to invoke frequently accessed utilities. For example, +HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/hbase-VERSION.jar + +... will return... + +An example program must be given as the first argument. +Valid program names are: + completebulkload: Complete a bulk data load. + copytable: Export a table from local cluster to peer cluster + export: Write table data to HDFS. + import: Import data written by Export. + importtsv: Import data in TSV format. + rowcounter: Count rows in HBase table + verifyrep: Compare the data from tables in two different clusters. WARNING: It doesn't work for incrementColumnValues'd cells since the timestamp is chan + +... for allowable program names. + +
+
+ HBase <application>hbck</application> + An fsck for your HBase install + To run hbck against your HBase cluster run + $ ./bin/hbase hbck + At the end of the commands output it prints OK + or INCONSISTENCY. If your cluster reports + inconsistencies, pass -details to see more detail emitted. + If inconsistencies, run hbck a few times because the + inconsistency may be transient (e.g. cluster is starting up or a region is + splitting). + Passing -fix may correct the inconsistency (This latter + is an experimental feature). + + For more information, see . + +
+
HFile Tool + See . +
+
+ WAL Tools + +
+ <classname>HLog</classname> tool + + The main method on HLog offers manual + split and dump facilities. Pass it WALs or the product of a split, the + content of the recovered.edits. directory. + + You can get a textual dump of a WAL file content by doing the + following: $ ./bin/hbase org.apache.hadoop.hbase.regionserver.wal.HLog --dump hdfs://example.org:8020/hbase/.logs/example.org,60020,1283516293161/10.10.21.10%3A60020.1283973724012 The + return code will be non-zero if issues with the file so you can test + wholesomeness of file by redirecting STDOUT to + /dev/null and testing the program return. + + Similarly you can force a split of a log file directory by + doing: $ ./bin/hbase org.apache.hadoop.hbase.regionserver.wal.HLog --split hdfs://example.org:8020/hbase/.logs/example.org,60020,1283516293161/ + +
+ <classname>HLogPrettyPrinter</classname> + HLogPrettyPrinter is a tool with configurable options to print the contents of an HLog. + +
+ +
+
+
Compression Tool + See . +
+
+ CopyTable + + CopyTable is a utility that can copy part or of all of a table, either to the same cluster or another cluster. The usage is as follows: +$ bin/hbase org.apache.hadoop.hbase.mapreduce.CopyTable [--starttime=X] [--endtime=Y] [--new.name=NEW] [--peer.adr=ADR] tablename + + + + Options: + + starttime Beginning of the time range. Without endtime means starttime to forever. + endtime End of the time range. Without endtime means starttime to forever. + versions Number of cell versions to copy. + new.name New table's name. + peer.adr Address of the peer cluster given in the format hbase.zookeeper.quorum:hbase.zookeeper.client.port:zookeeper.znode.parent + families Comma-separated list of ColumnFamilies to copy. + all.cells Also copy delete markers and uncollected deleted cells (advanced option). + + Args: + + tablename Name of table to copy. + + + Example of copying 'TestTable' to a cluster that uses replication for a 1 hour window: +$ bin/hbase org.apache.hadoop.hbase.mapreduce.CopyTable +--starttime=1265875194289 --endtime=1265878794289 +--peer.adr=server1,server2,server3:2181:/hbase TestTable + + Scanner Caching + Caching for the input Scan is configured via hbase.client.scanner.caching in the job configuration. + + + + See Jonathan Hsieh's Online HBase Backups with CopyTable blog post for more on CopyTable. + +
+
+ Export + Export is a utility that will dump the contents of table to HDFS in a sequence file. Invoke via: +$ bin/hbase org.apache.hadoop.hbase.mapreduce.Export <tablename> <outputdir> [<versions> [<starttime> [<endtime>]]] + + + Note: caching for the input Scan is configured via hbase.client.scanner.caching in the job configuration. + +
+
+ Import + Import is a utility that will load data that has been exported back into HBase. Invoke via: +$ bin/hbase org.apache.hadoop.hbase.mapreduce.Import <tablename> <inputdir> + + +
+
+ ImportTsv + ImportTsv is a utility that will load data in TSV format into HBase. It has two distinct usages: loading data from TSV format in HDFS + into HBase via Puts, and preparing StoreFiles to be loaded via the completebulkload. + + To load data via Puts (i.e., non-bulk loading): +$ bin/hbase org.apache.hadoop.hbase.mapreduce.ImportTsv -Dimporttsv.columns=a,b,c <tablename> <hdfs-inputdir> + + + To generate StoreFiles for bulk-loading: +$ bin/hbase org.apache.hadoop.hbase.mapreduce.ImportTsv -Dimporttsv.columns=a,b,c -Dimporttsv.bulk.output=hdfs://storefile-outputdir <tablename> <hdfs-data-inputdir> + + + These generated StoreFiles can be loaded into HBase via . + +
ImportTsv Options + Running ImportTsv with no arguments prints brief usage information: + +Usage: importtsv -Dimporttsv.columns=a,b,c <tablename> <inputdir> + +Imports the given input directory of TSV data into the specified table. + +The column names of the TSV data must be specified using the -Dimporttsv.columns +option. This option takes the form of comma-separated column names, where each +column name is either a simple column family, or a columnfamily:qualifier. The special +column name HBASE_ROW_KEY is used to designate that this column should be used +as the row key for each imported record. You must specify exactly one column +to be the row key, and you must specify a column name for every column that exists in the +input data. + +By default importtsv will load data directly into HBase. To instead generate +HFiles of data to prepare for a bulk data load, pass the option: + -Dimporttsv.bulk.output=/path/for/output + Note: the target table will be created with default column family descriptors if it does not already exist. + +Other options that may be specified with -D include: + -Dimporttsv.skip.bad.lines=false - fail if encountering an invalid line + '-Dimporttsv.separator=|' - eg separate on pipes instead of tabs + -Dimporttsv.timestamp=currentTimeAsLong - use the specified timestamp for the import + -Dimporttsv.mapper.class=my.Mapper - A user-defined Mapper to use instead of org.apache.hadoop.hbase.mapreduce.TsvImporterMapper + +
+
ImportTsv Example + For example, assume that we are loading data into a table called 'datatsv' with a ColumnFamily called 'd' with two columns "c1" and "c2". + + Assume that an input file exists as follows: + +row1 c1 c2 +row2 c1 c2 +row3 c1 c2 +row4 c1 c2 +row5 c1 c2 +row6 c1 c2 +row7 c1 c2 +row8 c1 c2 +row9 c1 c2 +row10 c1 c2 + + + For ImportTsv to use this imput file, the command line needs to look like this: + + HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/hbase-VERSION.jar importtsv -Dimporttsv.columns=HBASE_ROW_KEY,d:c1,d:c2 -Dimporttsv.bulk.output=hdfs://storefileoutput datatsv hdfs://inputfile + + ... and in this example the first column is the rowkey, which is why the HBASE_ROW_KEY is used. The second and third columns in the file will be imported as "d:c1" and "d:c2", respectively. + +
+
ImportTsv Warning + If you have preparing a lot of data for bulk loading, make sure the target HBase table is pre-split appropriately. + +
+
See Also + For more information about bulk-loading HFiles into HBase, see +
+
+ +
+ CompleteBulkLoad + The completebulkload utility will move generated StoreFiles into an HBase table. This utility is often used + in conjunction with output from . + + There are two ways to invoke this utility, with explicit classname and via the driver: +$ bin/hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles <hdfs://storefileoutput> <tablename> + +.. and via the Driver.. +HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/hbase-VERSION.jar completebulkload <hdfs://storefileoutput> <tablename> + + +
CompleteBulkLoad Warning + Data generated via MapReduce is often created with file permissions that are not compatible with the running HBase process. Assuming you're running HDFS with permissions enabled, those permissions will need to be updated before you run CompleteBulkLoad. + +
+ For more information about bulk-loading HFiles into HBase, see . + +
+
+ WALPlayer + WALPlayer is a utility to replay WAL files into HBase. + + The WAL can be replayed for a set of tables or all tables, and a + timerange can be provided (in milliseconds). The WAL is filtered to + this set of tables. The output can optionally be mapped to another set of tables. + + WALPlayer can also generate HFiles for later bulk importing, in that case + only a single table and no mapping can be specified. + + Invoke via: +$ bin/hbase org.apache.hadoop.hbase.mapreduce.WALPlayer [options] <wal inputdir> <tables> [<tableMappings>]> + + + For example: +$ bin/hbase org.apache.hadoop.hbase.mapreduce.WALPlayer /backuplogdir oldTable1,oldTable2 newTable1,newTable2 + + + + WALPlayer, by default, runs as a mapreduce job. To NOT run WALPlayer as a mapreduce job on your cluster, + force it to run all in the local process by adding the flags -Dmapred.job.tracker=local on the command line. + +
+
+ RowCounter and CellCounter + RowCounter is a + mapreduce job to count all the rows of a table. This is a good utility to use as a sanity check to ensure that HBase can read + all the blocks of a table if there are any concerns of metadata inconsistency. It will run the mapreduce all in a single + process but it will run faster if you have a MapReduce cluster in place for it to exploit. +$ bin/hbase org.apache.hadoop.hbase.mapreduce.RowCounter <tablename> [<column1> <column2>...] + + + Note: caching for the input Scan is configured via hbase.client.scanner.caching in the job configuration. + + HBase ships another diagnostic mapreduce job called + CellCounter. Like + RowCounter, it gathers more fine-grained statistics about your table. The statistics gathered by RowCounter are more fine-grained + and include: + + Total number of rows in the table. + Total number of CFs across all rows. + Total qualifiers across all rows. + Total occurrence of each CF. + Total occurrence of each qualifier. + Total number of versions of each qualifier. + + + The program allows you to limit the scope of the run. Provide a row regex or prefix to limit the rows to analyze. Use + hbase.mapreduce.scan.column.family to specify scanning a single column family. + $ bin/hbase org.apache.hadoop.hbase.mapreduce.CellCounter <tablename> <outputDir> [regex or prefix] + + Note: just like RowCounter, caching for the input Scan is configured via hbase.client.scanner.caching in the + job configuration. +
+ +
+ +
+ Region Management +
+ Major Compaction + Major compactions can be requested via the HBase shell or HBaseAdmin.majorCompact. + + Note: major compactions do NOT do region merges. See for more information about compactions. + + +
+
+ Merge + Merge is a utility that can merge adjoining regions in the same table (see org.apache.hadoop.hbase.util.Merge). +$ bin/hbase org.apache.hbase.util.Merge <tablename> <region1> <region2> + + If you feel you have too many regions and want to consolidate them, Merge is the utility you need. Merge must + run be done when the cluster is down. + See the O'Reilly HBase Book for + an example of usage. + + Additionally, there is a Ruby script attached to HBASE-1621 + for region merging. + +
+
+ +
Node Management +
Node Decommission + You can stop an individual RegionServer by running the following + script in the HBase directory on the particular node: + $ ./bin/hbase-daemon.sh stop regionserver + The RegionServer will first close all regions and then shut itself down. + On shutdown, the RegionServer's ephemeral node in ZooKeeper will expire. + The master will notice the RegionServer gone and will treat it as + a 'crashed' server; it will reassign the nodes the RegionServer was carrying. + Disable the Load Balancer before Decommissioning a node + If the load balancer runs while a node is shutting down, then + there could be contention between the Load Balancer and the + Master's recovery of the just decommissioned RegionServer. + Avoid any problems by disabling the balancer first. + See below. + + + + + A downside to the above stop of a RegionServer is that regions could be offline for + a good period of time. Regions are closed in order. If many regions on the server, the + first region to close may not be back online until all regions close and after the master + notices the RegionServer's znode gone. In Apache HBase 0.90.2, we added facility for having + a node gradually shed its load and then shutdown itself down. Apache HBase 0.90.2 added the + graceful_stop.sh script. Here is its usage: + $ ./bin/graceful_stop.sh +Usage: graceful_stop.sh [--config &conf-dir>] [--restart] [--reload] [--thrift] [--rest] &hostname> + thrift If we should stop/start thrift before/after the hbase stop/start + rest If we should stop/start rest before/after the hbase stop/start + restart If we should restart after graceful stop + reload Move offloaded regions back on to the stopped server + debug Move offloaded regions back on to the stopped server + hostname Hostname of server we are to stop + + + To decommission a loaded RegionServer, run the following: + $ ./bin/graceful_stop.sh HOSTNAME + where HOSTNAME is the host carrying the RegionServer + you would decommission. + On <varname>HOSTNAME</varname> + The HOSTNAME passed to graceful_stop.sh + must match the hostname that hbase is using to identify RegionServers. + Check the list of RegionServers in the master UI for how HBase is + referring to servers. Its usually hostname but can also be FQDN. + Whatever HBase is using, this is what you should pass the + graceful_stop.sh decommission + script. If you pass IPs, the script is not yet smart enough to make + a hostname (or FQDN) of it and so it will fail when it checks if server is + currently running; the graceful unloading of regions will not run. + + The graceful_stop.sh script will move the regions off the + decommissioned RegionServer one at a time to minimize region churn. + It will verify the region deployed in the new location before it + will moves the next region and so on until the decommissioned server + is carrying zero regions. At this point, the graceful_stop.sh + tells the RegionServer stop. The master will at this point notice the + RegionServer gone but all regions will have already been redeployed + and because the RegionServer went down cleanly, there will be no + WAL logs to split. + Load Balancer + + It is assumed that the Region Load Balancer is disabled while the + graceful_stop script runs (otherwise the balancer + and the decommission script will end up fighting over region deployments). + Use the shell to disable the balancer: + hbase(main):001:0> balance_switch false +true +0 row(s) in 0.3590 seconds +This turns the balancer OFF. To reenable, do: + hbase(main):001:0> balance_switch true +false +0 row(s) in 0.3590 seconds + + + +
+ Bad or Failing Disk + It is good having set if you have a decent number of disks + per machine for the case where a disk plain dies. But usually disks do the "John Wayne" -- i.e. take a while + to go down spewing errors in dmesg -- or for some reason, run much slower than their + companions. In this case you want to decommission the disk. You have two options. You can + decommission the datanode + or, less disruptive in that only the bad disks data will be rereplicated, can stop the datanode, + unmount the bad volume (You can't umount a volume while the datanode is using it), and then restart the + datanode (presuming you have set dfs.datanode.failed.volumes.tolerated > 0). The regionserver will + throw some errors in its logs as it recalibrates where to get its data from -- it will likely + roll its WAL log too -- but in general but for some latency spikes, it should keep on chugging. + + If you are doing short-circuit reads, you will have to move the regions off the regionserver + before you stop the datanode; when short-circuiting reading, though chmod'd so regionserver cannot + have access, because it already has the files open, it will be able to keep reading the file blocks + from the bad disk even though the datanode is down. Move the regions back after you restart the + datanode. + + +
+
+
+ Rolling Restart + + You can also ask this script to restart a RegionServer after the shutdown + AND move its old regions back into place. The latter you might do to + retain data locality. A primitive rolling restart might be effected by + running something like the following: + $ for i in `cat conf/regionservers|sort`; do ./bin/graceful_stop.sh --restart --reload --debug $i; done &> /tmp/log.txt & + + Tail the output of /tmp/log.txt to follow the scripts + progress. The above does RegionServers only. Be sure to disable the + load balancer before doing the above. You'd need to do the master + update separately. Do it before you run the above script. + Here is a pseudo-script for how you might craft a rolling restart script: + + Untar your release, make sure of its configuration and + then rsync it across the cluster. If this is 0.90.2, patch it + with HBASE-3744 and HBASE-3756. + + + + Run hbck to ensure the cluster consistent + $ ./bin/hbase hbck + Effect repairs if inconsistent. + + + + Restart the Master: $ ./bin/hbase-daemon.sh stop master; ./bin/hbase-daemon.sh start master + + + + + Disable the region balancer:$ echo "balance_switch false" | ./bin/hbase shell + + + + Run the graceful_stop.sh script per RegionServer. For example: + $ for i in `cat conf/regionservers|sort`; do ./bin/graceful_stop.sh --restart --reload --debug $i; done &> /tmp/log.txt & + + If you are running thrift or rest servers on the RegionServer, pass --thrift or --rest options (See usage + for graceful_stop.sh script). + + + + Restart the Master again. This will clear out dead servers list and reenable the balancer. + + + + Run hbck to ensure the cluster is consistent. + + + + +
+
+ +
+ HBase Metrics +
+ Metric Setup + See Metrics for + an introduction and how to enable Metrics emission. + +
+
+ RegionServer Metrics +
<varname>hbase.regionserver.blockCacheCount</varname> + Block cache item count in memory. This is the number of blocks of StoreFiles (HFiles) in the cache. +
+
<varname>hbase.regionserver.blockCacheEvictedCount</varname> + Number of blocks that had to be evicted from the block cache due to heap size constraints. +
+
<varname>hbase.regionserver.blockCacheFree</varname> + Block cache memory available (bytes). +
+
<varname>hbase.regionserver.blockCacheHitCachingRatio</varname> + Block cache hit caching ratio (0 to 100). The cache-hit ratio for reads configured to look in the cache (i.e., cacheBlocks=true). +
+
<varname>hbase.regionserver.blockCacheHitCount</varname> + Number of blocks of StoreFiles (HFiles) read from the cache. +
+
<varname>hbase.regionserver.blockCacheHitRatio</varname> + Block cache hit ratio (0 to 100). Includes all read requests, although those with cacheBlocks=false + will always read from disk and be counted as a "cache miss". +
+
<varname>hbase.regionserver.blockCacheMissCount</varname> + Number of blocks of StoreFiles (HFiles) requested but not read from the cache. +
+
<varname>hbase.regionserver.blockCacheSize</varname> + Block cache size in memory (bytes). i.e., memory in use by the BlockCache +
+
<varname>hbase.regionserver.compactionQueueSize</varname> + Size of the compaction queue. This is the number of Stores in the RegionServer that have been targeted for compaction. +
+
<varname>hbase.regionserver.flushQueueSize</varname> + Number of enqueued regions in the MemStore awaiting flush. +
+
<varname>hbase.regionserver.fsReadLatency_avg_time</varname> + Filesystem read latency (ms). This is the average time to read from HDFS. +
+
<varname>hbase.regionserver.fsReadLatency_num_ops</varname> + Filesystem read operations. +
+
<varname>hbase.regionserver.fsSyncLatency_avg_time</varname> + Filesystem sync latency (ms). Latency to sync the write-ahead log records to the filesystem. +
+
<varname>hbase.regionserver.fsSyncLatency_num_ops</varname> + Number of operations to sync the write-ahead log records to the filesystem. +
+
<varname>hbase.regionserver.fsWriteLatency_avg_time</varname> + Filesystem write latency (ms). Total latency for all writers, including StoreFiles and write-head log. +
+
<varname>hbase.regionserver.fsWriteLatency_num_ops</varname> + Number of filesystem write operations, including StoreFiles and write-ahead log. +
+
<varname>hbase.regionserver.memstoreSizeMB</varname> + Sum of all the memstore sizes in this RegionServer (MB) +
+
<varname>hbase.regionserver.regions</varname> + Number of regions served by the RegionServer +
+
<varname>hbase.regionserver.requests</varname> + Total number of read and write requests. Requests correspond to RegionServer RPC calls, thus a single Get will result in 1 request, but a Scan with caching set to 1000 will result in 1 request for each 'next' call (i.e., not each row). A bulk-load request will constitute 1 request per HFile. +
+
<varname>hbase.regionserver.storeFileIndexSizeMB</varname> + Sum of all the StoreFile index sizes in this RegionServer (MB) +
+
<varname>hbase.regionserver.stores</varname> + Number of Stores open on the RegionServer. A Store corresponds to a ColumnFamily. For example, if a table (which contains the column family) has 3 regions on a RegionServer, there will be 3 stores open for that column family. +
+
<varname>hbase.regionserver.storeFiles</varname> + Number of StoreFiles open on the RegionServer. A store may have more than one StoreFile (HFile). +
+
+
+ +
+ HBase Monitoring +
+ Overview + The following metrics are arguably the most important to monitor for each RegionServer for + "macro monitoring", preferably with a system like OpenTSDB. + If your cluster is having performance issues it's likely that you'll see something unusual with + this group. + + HBase: + + Requests + Compactions queue + + + OS: + + IO Wait + User CPU + + + Java: + + GC + + + + + + For more information on HBase metrics, see . + +
+ +
+ Slow Query Log +The HBase slow query log consists of parseable JSON structures describing the properties of those client operations (Gets, Puts, Deletes, etc.) that either took too long to run, or produced too much output. The thresholds for "too long to run" and "too much output" are configurable, as described below. The output is produced inline in the main region server logs so that it is easy to discover further details from context with other logged events. It is also prepended with identifying tags (responseTooSlow), (responseTooLarge), (operationTooSlow), and (operationTooLarge) in order to enable easy filtering with grep, in case the user desires to see only slow queries. + + +
Configuration +There are two configuration knobs that can be used to adjust the thresholds for when queries are logged. + + + + +hbase.ipc.warn.response.time Maximum number of milliseconds that a query can be run without being logged. Defaults to 10000, or 10 seconds. Can be set to -1 to disable logging by time. + +hbase.ipc.warn.response.size Maximum byte size of response that a query can return without being logged. Defaults to 100 megabytes. Can be set to -1 to disable logging by size. + + +
+ +
Metrics +The slow query log exposes to metrics to JMX. +hadoop.regionserver_rpc_slowResponse a global metric reflecting the durations of all responses that triggered logging. +hadoop.regionserver_rpc_methodName.aboveOneSec A metric reflecting the durations of all responses that lasted for more than one second. + + +
+ +
Output +The output is tagged with operation e.g. (operationTooSlow) if the call was a client operation, such as a Put, Get, or Delete, which we expose detailed fingerprint information for. If not, it is tagged (responseTooSlow) and still produces parseable JSON output, but with less verbose information solely regarding its duration and size in the RPC itself. TooLarge is substituted for TooSlow if the response size triggered the logging, with TooLarge appearing even in the case that both size and duration triggered logging. + +
+
Example + +2011-09-08 10:01:25,824 WARN org.apache.hadoop.ipc.HBaseServer: (operationTooSlow): {"tables":{"riley2":{"puts":[{"totalColumns":11,"families":{"actions":[{"timestamp":1315501284459,"qualifier":"0","vlen":9667580},{"timestamp":1315501284459,"qualifier":"1","vlen":10122412},{"timestamp":1315501284459,"qualifier":"2","vlen":11104617},{"timestamp":1315501284459,"qualifier":"3","vlen":13430635}]},"row":"cfcd208495d565ef66e7dff9f98764da:0"}],"families":["actions"]}},"processingtimems":956,"client":"10.47.34.63:33623","starttimems":1315501284456,"queuetimems":0,"totalPuts":1,"class":"HRegionServer","responsesize":0,"method":"multiPut"} + + +Note that everything inside the "tables" structure is output produced by MultiPut's fingerprint, while the rest of the information is RPC-specific, such as processing time and client IP/port. Other client operations follow the same pattern and the same general structure, with necessary differences due to the nature of the individual operations. In the case that the call is not a client operation, that detailed fingerprint information will be completely absent. + + +This particular example, for example, would indicate that the likely cause of slowness is simply a very large (on the order of 100MB) multiput, as we can tell by the "vlen," or value length, fields of each put in the multiPut. + +
+
+ + + +
+ +
+ Cluster Replication + See Cluster Replication. + +
+
+ HBase Backup + There are two broad strategies for performing HBase backups: backing up with a full cluster shutdown, and backing up on a live cluster. + Each approach has pros and cons. + + For additional information, see HBase Backup Options over on the Sematext Blog. + +
Full Shutdown Backup + Some environments can tolerate a periodic full shutdown of their HBase cluster, for example if it is being used a back-end analytic capacity + and not serving front-end web-pages. The benefits are that the NameNode/Master are RegionServers are down, so there is no chance of missing + any in-flight changes to either StoreFiles or metadata. The obvious con is that the cluster is down. The steps include: + +
Stop HBase + + +
+
Distcp + Distcp could be used to either copy the contents of the HBase directory in HDFS to either the same cluster in another directory, or + to a different cluster. + + Note: Distcp works in this situation because the cluster is down and there are no in-flight edits to files. + Distcp-ing of files in the HBase directory is not generally recommended on a live cluster. + +
+
Restore (if needed) + The backup of the hbase directory from HDFS is copied onto the 'real' hbase directory via distcp. The act of copying these files + creates new HDFS metadata, which is why a restore of the NameNode edits from the time of the HBase backup isn't required for this kind of + restore, because it's a restore (via distcp) of a specific HDFS directory (i.e., the HBase part) not the entire HDFS file-system. + +
+
+
Live Cluster Backup - Replication + This approach assumes that there is a second cluster. + See the HBase page on replication for more information. + +
+
Live Cluster Backup - CopyTable + The utility could either be used to copy data from one table to another on the + same cluster, or to copy data to another table on another cluster. + + Since the cluster is up, there is a risk that edits could be missed in the copy process. + +
+
Live Cluster Backup - Export + The approach dumps the content of a table to HDFS on the same cluster. To restore the data, the + utility would be used. + + Since the cluster is up, there is a risk that edits could be missed in the export process. + +
+
+ +
+ HBase Snapshots + HBase Snapshots allow you to take a snapshot of a table without too much impact on Region Servers. + Snapshot, Clone and restore operations don't involve data copying. + Also, Exporting the snapshot to another cluster doesn't have impact on the Region Servers. + + Prior to version 0.94.6, the only way to backup or to clone a table is to use CopyTable/ExportTable, + or to copy all the hfiles in HDFS after disabling the table. + The disadvantages of these methods are that you can degrade region server performance + (Copy/Export Table) or you need to disable the table, that means no reads or writes; + and this is usually unacceptable. + +
Configuration + To turn on the snapshot support just set the + hbase.snapshot.enabled property to true. + (Snapshots are enabled by default in 0.95+ and off by default in 0.94.6+) + + <property> + <name>hbase.snapshot.enabled</name> + <value>true</value> + </property> + + +
+
Take a Snapshot + You can take a snapshot of a table regardless of whether it is enabled or disabled. + The snapshot operation doesn't involve any data copying. + + $ ./bin/hbase shell + hbase> snapshot 'myTable', 'myTableSnapshot-122112' + + +
+
Listing Snapshots + List all snapshots taken (by printing the names and relative information). + + $ ./bin/hbase shell + hbase> list_snapshots + + +
+
Deleting Snapshots + You can remove a snapshot, and the files retained for that snapshot will be removed + if no longer needed. + + $ ./bin/hbase shell + hbase> delete_snapshot 'myTableSnapshot-122112' + + +
+
Clone a table from snapshot + From a snapshot you can create a new table (clone operation) with the same data + that you had when the snapshot was taken. + The clone operation, doesn't involve data copies, and a change to the cloned table + doesn't impact the snapshot or the original table. + + $ ./bin/hbase shell + hbase> clone_snapshot 'myTableSnapshot-122112', 'myNewTestTable' + + +
+
Restore a snapshot + The restore operation requires the table to be disabled, and the table will be + restored to the state at the time when the snapshot was taken, + changing both data and schema if required. + + $ ./bin/hbase shell + hbase> disable 'myTable' + hbase> restore_snapshot 'myTableSnapshot-122112' + + + + Since Replication works at log level and snapshots at file-system level, + after a restore, the replicas will be in a different state from the master. + If you want to use restore, you need to stop replication and redo the bootstrap. + + + In case of partial data-loss due to misbehaving client, instead of a full restore + that requires the table to be disabled, you can clone the table from the snapshot + and use a Map-Reduce job to copy the data that you need, from the clone to the main one. + +
+
Snapshots operations and ACLs + If you are using security with the AccessController Coprocessor (See ), + only a global administrator can take, clone, or restore a snapshot, and these actions do not capture the ACL rights. + This means that restoring a table preserves the ACL rights of the existing table, + while cloning a table creates a new table that has no ACL rights until the administrator adds them. +
+
Export to another cluster + The ExportSnapshot tool copies all the data related to a snapshot (hfiles, logs, snapshot metadata) to another cluster. + The tool executes a Map-Reduce job, similar to distcp, to copy files between the two clusters, + and since it works at file-system level the hbase cluster does not have to be online. + To copy a snapshot called MySnapshot to an HBase cluster srv2 (hdfs:///srv2:8082/hbase) using 16 mappers: +$ bin/hbase class org.apache.hadoop.hbase.snapshot.ExportSnapshot -snapshot MySnapshot -copy-to hdfs:///srv2:8082/hbase -mappers 16 + + +
+
+ +
Capacity Planning +
Storage + A common question for HBase administrators is estimating how much storage will be required for an HBase cluster. + There are several apsects to consider, the most important of which is what data load into the cluster. Start + with a solid understanding of how HBase handles data internally (KeyValue). + +
KeyValue + HBase storage will be dominated by KeyValues. See and for + how HBase stores data internally. + + It is critical to understand that there is a KeyValue instance for every attribute stored in a row, and the + rowkey-length, ColumnFamily name-length and attribute lengths will drive the size of the database more than any other + factor. + +
+
StoreFiles and Blocks + KeyValue instances are aggregated into blocks, and the blocksize is configurable on a per-ColumnFamily basis. + Blocks are aggregated into StoreFile's. See . + +
+
HDFS Block Replication + Because HBase runs on top of HDFS, factor in HDFS block replication into storage calculations. + +
+
+
Regions + Another common question for HBase administrators is determining the right number of regions per + RegionServer. This affects both storage and hardware planning. See . + +
+
+ + diff --git a/src/docbkx/performance.xml b/src/docbkx/performance.xml new file mode 100644 index 0000000..b65dab0 --- /dev/null +++ b/src/docbkx/performance.xml @@ -0,0 +1,682 @@ + + + + Apache HBase (TM) Performance Tuning + +
+ Operating System +
+ Memory + RAM, RAM, RAM. Don't starve HBase. +
+
+ 64-bit + Use a 64-bit platform (and 64-bit JVM). +
+
+ Swapping + Watch out for swapping. Set swappiness to 0. +
+
+
+ Network + + Perhaps the most important factor in avoiding network issues degrading Hadoop and HBbase performance is the switching hardware + that is used, decisions made early in the scope of the project can cause major problems when you double or triple the size of your cluster (or more). + + + Important items to consider: + + Switching capacity of the device + Number of systems connected + Uplink capacity + + +
+ Single Switch + The single most important factor in this configuration is that the switching capacity of the hardware is capable of + handling the traffic which can be generated by all systems connected to the switch. Some lower priced commodity hardware + can have a slower switching capacity than could be utilized by a full switch. + +
+
+ Multiple Switches + Multiple switches are a potential pitfall in the architecture. The most common configuration of lower priced hardware is a + simple 1Gbps uplink from one switch to another. This often overlooked pinch point can easily become a bottleneck for cluster communication. + Especially with MapReduce jobs that are both reading and writing a lot of data the communication across this uplink could be saturated. + + Mitigation of this issue is fairly simple and can be accomplished in multiple ways: + + Use appropriate hardware for the scale of the cluster which you're attempting to build. + Use larger single switch configurations i.e. single 48 port as opposed to 2x 24 port + Configure port trunking for uplinks to utilize multiple interfaces to increase cross switch bandwidth. + + +
+
+ Multiple Racks + Multiple rack configurations carry the same potential issues as multiple switches, and can suffer performance degradation from two main areas: + + Poor switch capacity performance + Insufficient uplink to another rack + + If the the switches in your rack have appropriate switching capacity to handle all the hosts at full speed, the next most likely issue will be caused by homing + more of your cluster across racks. The easiest way to avoid issues when spanning multiple racks is to use port trunking to create a bonded uplink to other racks. + The downside of this method however, is in the overhead of ports that could potentially be used. An example of this is, creating an 8Gbps port channel from rack + A to rack B, using 8 of your 24 ports to communicate between racks gives you a poor ROI, using too few however can mean you're not getting the most out of your cluster. + + Using 10Gbe links between racks will greatly increase performance, and assuming your switches support a 10Gbe uplink or allow for an expansion card will allow you to + save your ports for machines as opposed to uplinks. + +
+
+ Network Interfaces + Are all the network interfaces functioning correctly? Are you sure? See the Troubleshooting Case Study in . + +
+
+ +
+ Java + +
+ The Garbage Collector and Apache HBase + +
+ Long GC pauses + + In his presentation, Avoiding + Full GCs with MemStore-Local Allocation Buffers, Todd Lipcon + describes two cases of stop-the-world garbage collections common in + HBase, especially during loading; CMS failure modes and old generation + heap fragmentation brought. To address the first, start the CMS + earlier than default by adding + -XX:CMSInitiatingOccupancyFraction and setting it down + from defaults. Start at 60 or 70 percent (The lower you bring down the + threshold, the more GCing is done, the more CPU used). To address the + second fragmentation issue, Todd added an experimental facility, + MSLAB, that + must be explicitly enabled in Apache HBase 0.90.x (Its defaulted to be on in + Apache 0.92.x HBase). See hbase.hregion.memstore.mslab.enabled + to true in your Configuration. See the cited + slides for background and detailThe latest jvms do better + regards fragmentation so make sure you are running a recent release. + Read down in the message, + Identifying concurrent mode failures caused by fragmentation.. + Be aware that when enabled, each MemStore instance will occupy at least + an MSLAB instance of memory. If you have thousands of regions or lots + of regions each with many column families, this allocation of MSLAB + may be responsible for a good portion of your heap allocation and in + an extreme case cause you to OOME. Disable MSLAB in this case, or + lower the amount of memory it uses or float less regions per server. + + For more information about GC logs, see . + +
+
+
+ +
+ HBase Configurations + + See . + + +
+ Number of Regions + + The number of regions for an HBase table is driven by the . Also, see the architecture + section on +
+ +
+ Managing Compactions + + For larger systems, managing compactions and splits may be + something you want to consider. +
+ +
+ <varname>hbase.regionserver.handler.count</varname> + See . + +
+
+ <varname>hfile.block.cache.size</varname> + See . + A memory setting for the RegionServer process. + +
+
+ <varname>hbase.regionserver.global.memstore.upperLimit</varname> + See . + This memory setting is often adjusted for the RegionServer process depending on needs. + +
+
+ <varname>hbase.regionserver.global.memstore.lowerLimit</varname> + See . + This memory setting is often adjusted for the RegionServer process depending on needs. + +
+
+ <varname>hbase.hstore.blockingStoreFiles</varname> + See . + If there is blocking in the RegionServer logs, increasing this can help. + +
+
+ <varname>hbase.hregion.memstore.block.multiplier</varname> + See . + If there is enough RAM, increasing this can help. + +
+
+ <varname>hbase.regionserver.checksum.verify</varname> + Have HBase write the checksum into the datablock and save + having to do the checksum seek whenever you read. See the + release note on HBASE-5074 support checksums in HBase block cache. + +
+ +
+ + + + +
+ ZooKeeper + See for information on configuring ZooKeeper, and see the part + about having a dedicated disk. + +
+
+ Schema Design + +
+ Number of Column Families + See . +
+
+ Key and Attribute Lengths + See . See also for + compression caveats. +
+
Table RegionSize + The regionsize can be set on a per-table basis via setFileSize on + HTableDescriptor in the + event where certain tables require different regionsizes than the configured default regionsize. + + See for more information. + +
+
+ Bloom Filters + Bloom Filters can be enabled per-ColumnFamily. + Use HColumnDescriptor.setBloomFilterType(NONE | ROW | + ROWCOL) to enable blooms per Column Family. Default = + NONE for no bloom filters. If + ROW, the hash of the row will be added to the bloom + on each insert. If ROWCOL, the hash of the row + + column family + column family qualifier will be added to the bloom on + each key insert. + See HColumnDescriptor and + for more information or this answer up in quora, +How are bloom filters used in HBase?. + +
+
ColumnFamily BlockSize + The blocksize can be configured for each ColumnFamily in a table, and this defaults to 64k. Larger cell values require larger blocksizes. + There is an inverse relationship between blocksize and the resulting StoreFile indexes (i.e., if the blocksize is doubled then the resulting + indexes should be roughly halved). + + See HColumnDescriptor + and for more information. + +
+
+ In-Memory ColumnFamilies + ColumnFamilies can optionally be defined as in-memory. Data is still persisted to disk, just like any other ColumnFamily. + In-memory blocks have the highest priority in the , but it is not a guarantee that the entire table + will be in memory. + + See HColumnDescriptor for more information. + +
+
+ Compression + Production systems should use compression with their ColumnFamily definitions. See for more information. + +
However... + Compression deflates data on disk. When it's in-memory (e.g., in the + MemStore) or on the wire (e.g., transferring between RegionServer and Client) it's inflated. + So while using ColumnFamily compression is a best practice, but it's not going to completely eliminate + the impact of over-sized Keys, over-sized ColumnFamily names, or over-sized Column names. + + See on for schema design tips, and for more information on HBase stores data internally. + +
+
+
+ +
+ Writing to HBase + +
+ Batch Loading + Use the bulk load tool if you can. See + . + Otherwise, pay attention to the below. + +
+ +
+ + Table Creation: Pre-Creating Regions + + +Tables in HBase are initially created with one region by default. For bulk imports, this means that all clients will write to the same region +until it is large enough to split and become distributed across the cluster. A useful pattern to speed up the bulk import process is to pre-create empty regions. + Be somewhat conservative in this, because too-many regions can actually degrade performance. + + There are two different approaches to pre-creating splits. The first approach is to rely on the default HBaseAdmin strategy + (which is implemented in Bytes.split)... + + +byte[] startKey = ...; // your lowest keuy +byte[] endKey = ...; // your highest key +int numberOfRegions = ...; // # of regions to create +admin.createTable(table, startKey, endKey, numberOfRegions); + + And the other approach is to define the splits yourself... + + +byte[][] splits = ...; // create your own splits +admin.createTable(table, splits); + + + See for issues related to understanding your keyspace and pre-creating regions. + +
+
+ + Table Creation: Deferred Log Flush + + +The default behavior for Puts using the Write Ahead Log (WAL) is that HLog edits will be written immediately. If deferred log flush is used, +WAL edits are kept in memory until the flush period. The benefit is aggregated and asynchronous HLog- writes, but the potential downside is that if + the RegionServer goes down the yet-to-be-flushed edits are lost. This is safer, however, than not using WAL at all with Puts. + + +Deferred log flush can be configured on tables via HTableDescriptor. The default value of hbase.regionserver.optionallogflushinterval is 1000ms. + +
+ +
+ HBase Client: AutoFlush + + When performing a lot of Puts, make sure that setAutoFlush is set + to false on your HTable + instance. Otherwise, the Puts will be sent one at a time to the + RegionServer. Puts added via htable.add(Put) and htable.add( <List> Put) + wind up in the same write buffer. If autoFlush = false, + these messages are not sent until the write-buffer is filled. To + explicitly flush the messages, call flushCommits. + Calling close on the HTable + instance will invoke flushCommits. +
+
+ HBase Client: Turn off WAL on Puts + A frequently discussed option for increasing throughput on Puts is to call writeToWAL(false). Turning this off means + that the RegionServer will not write the Put to the Write Ahead Log, + only into the memstore, HOWEVER the consequence is that if there + is a RegionServer failure there will be data loss. + If writeToWAL(false) is used, do so with extreme caution. You may find in actuality that + it makes little difference if your load is well distributed across the cluster. + + In general, it is best to use WAL for Puts, and where loading throughput + is a concern to use bulk loading techniques instead. + +
+
+ HBase Client: Group Puts by RegionServer + In addition to using the writeBuffer, grouping Puts by RegionServer can reduce the number of client RPC calls per writeBuffer flush. + There is a utility HTableUtil currently on TRUNK that does this, but you can either copy that or implement your own verison for + those still on 0.90.x or earlier. + +
+
+ MapReduce: Skip The Reducer + When writing a lot of data to an HBase table from a MR job (e.g., with TableOutputFormat), and specifically where Puts are being emitted + from the Mapper, skip the Reducer step. When a Reducer step is used, all of the output (Puts) from the Mapper will get spooled to disk, then sorted/shuffled to other + Reducers that will most likely be off-node. It's far more efficient to just write directly to HBase. + + For summary jobs where HBase is used as a source and a sink, then writes will be coming from the Reducer step (e.g., summarize values then write out result). + This is a different processing problem than from the the above case. + +
+ +
+ Anti-Pattern: One Hot Region + If all your data is being written to one region at a time, then re-read the + section on processing timeseries data. + Also, if you are pre-splitting regions and all your data is still winding up in a single region even though + your keys aren't monotonically increasing, confirm that your keyspace actually works with the split strategy. There are a + variety of reasons that regions may appear "well split" but won't work with your data. As + the HBase client communicates directly with the RegionServers, this can be obtained via + HTable.getRegionLocation. + + See , as well as +
+ +
+ +
+ Reading from HBase + +
+ Scan Caching + + If HBase is used as an input source for a MapReduce job, for + example, make sure that the input Scan + instance to the MapReduce job has setCaching set to something greater + than the default (which is 1). Using the default value means that the + map-task will make call back to the region-server for every record + processed. Setting this value to 500, for example, will transfer 500 + rows at a time to the client to be processed. There is a cost/benefit to + have the cache value be large because it costs more in memory for both + client and RegionServer, so bigger isn't always better. +
+ Scan Caching in MapReduce Jobs + Scan settings in MapReduce jobs deserve special attention. Timeouts can result (e.g., UnknownScannerException) + in Map tasks if it takes longer to process a batch of records before the client goes back to the RegionServer for the + next set of data. This problem can occur because there is non-trivial processing occuring per row. If you process + rows quickly, set caching higher. If you process rows more slowly (e.g., lots of transformations per row, writes), + then set caching lower. + + Timeouts can also happen in a non-MapReduce use case (i.e., single threaded HBase client doing a Scan), but the + processing that is often performed in MapReduce jobs tends to exacerbate this issue. + +
+
+
+ Scan Attribute Selection + + Whenever a Scan is used to process large numbers of rows (and especially when used + as a MapReduce source), be aware of which attributes are selected. If scan.addFamily is called + then all of the attributes in the specified ColumnFamily will be returned to the client. + If only a small number of the available attributes are to be processed, then only those attributes should be specified + in the input scan because attribute over-selection is a non-trivial performance penalty over large datasets. + +
+
+ MapReduce - Input Splits + For MapReduce jobs that use HBase tables as a source, if there a pattern where the "slow" map tasks seem to + have the same Input Split (i.e., the RegionServer serving the data), see the + Troubleshooting Case Study in . + +
+ +
+ Close ResultScanners + + This isn't so much about improving performance but rather + avoiding performance problems. If you forget to + close ResultScanners + you can cause problems on the RegionServers. Always have ResultScanner + processing enclosed in try/catch blocks... +Scan scan = new Scan(); +// set attrs... +ResultScanner rs = htable.getScanner(scan); +try { + for (Result r = rs.next(); r != null; r = rs.next()) { + // process result... +} finally { + rs.close(); // always close the ResultScanner! +} +htable.close(); +
+ +
+ Block Cache + + Scan + instances can be set to use the block cache in the RegionServer via the + setCacheBlocks method. For input Scans to MapReduce jobs, this should be + false. For frequently accessed rows, it is advisable to use the block + cache. +
+
+ Optimal Loading of Row Keys + When performing a table scan + where only the row keys are needed (no families, qualifiers, values or timestamps), add a FilterList with a + MUST_PASS_ALL operator to the scanner using setFilter. The filter list + should include both a FirstKeyOnlyFilter + and a KeyOnlyFilter. + Using this filter combination will result in a worst case scenario of a RegionServer reading a single value from disk + and minimal network traffic to the client for a single row. + +
+
+ Concurrency: Monitor Data Spread + When performing a high number of concurrent reads, monitor the data spread of the target tables. If the target table(s) have + too few regions then the reads could likely be served from too few nodes. + See , as well as +
+
+ Bloom Filters + Enabling Bloom Filters can save your having to go to disk and + can help improve read latencys. + Bloom filters were developed over in HBase-1200 + Add bloomfilters. + For description of the development process -- why static blooms + rather than dynamic -- and for an overview of the unique properties + that pertain to blooms in HBase, as well as possible future + directions, see the Development Process section + of the document BloomFilters + in HBase attached to HBase-1200. + + The bloom filters described here are actually version two of + blooms in HBase. In versions up to 0.19.x, HBase had a dynamic bloom + option based on work done by the European Commission One-Lab + Project 034819. The core of the HBase bloom work was later + pulled up into Hadoop to implement org.apache.hadoop.io.BloomMapFile. + Version 1 of HBase blooms never worked that well. Version 2 is a + rewrite from scratch though again it starts with the one-lab + work. + + See also . + + +
+ Bloom StoreFile footprint + + Bloom filters add an entry to the StoreFile + general FileInfo data structure and then two + extra entries to the StoreFile metadata + section. + +
+ BloomFilter in the <classname>StoreFile</classname> + <classname>FileInfo</classname> data structure + + FileInfo has a + BLOOM_FILTER_TYPE entry which is set to + NONE, ROW or + ROWCOL. +
+ +
+ BloomFilter entries in <classname>StoreFile</classname> + metadata + + BLOOM_FILTER_META holds Bloom Size, Hash + Function used, etc. Its small in size and is cached on + StoreFile.Reader load + BLOOM_FILTER_DATA is the actual bloomfilter + data. Obtained on-demand. Stored in the LRU cache, if it is enabled + (Its enabled by default). +
+
+
+ Bloom Filter Configuration +
+ <varname>io.hfile.bloom.enabled</varname> global kill + switch + + io.hfile.bloom.enabled in + Configuration serves as the kill switch in case + something goes wrong. Default = true. +
+ +
+ <varname>io.hfile.bloom.error.rate</varname> + + io.hfile.bloom.error.rate = average false + positive rate. Default = 1%. Decrease rate by ½ (e.g. to .5%) == +1 + bit per bloom entry. +
+ +
+ <varname>io.hfile.bloom.max.fold</varname> + + io.hfile.bloom.max.fold = guaranteed minimum + fold rate. Most people should leave this alone. Default = 7, or can + collapse to at least 1/128th of original size. See the + Development Process section of the document BloomFilters + in HBase for more on what this option means. +
+
+
+ +
+ +
+ Deleting from HBase +
+ Using HBase Tables as Queues + HBase tables are sometimes used as queues. In this case, special care must be taken to regularly perform major compactions on tables used in + this manner. As is documented in , marking rows as deleted creates additional StoreFiles which then need to be processed + on reads. Tombstones only get cleaned up with major compactions. + + See also and HBaseAdmin.majorCompact. + +
+
+ Delete RPC Behavior + Be aware that htable.delete(Delete) doesn't use the writeBuffer. It will execute an RegionServer RPC with each invocation. + For a large number of deletes, consider htable.delete(List). + + See + +
+
+ +
HDFS + Because HBase runs on it is important to understand how it works and how it affects + HBase. + +
Current Issues With Low-Latency Reads + The original use-case for HDFS was batch processing. As such, there low-latency reads were historically not a priority. + With the increased adoption of Apache HBase this is changing, and several improvements are already in development. + See the + Umbrella Jira Ticket for HDFS Improvements for HBase. + +
+
+ Leveraging local data +Since Hadoop 1.0.0 (also 0.22.1, 0.23.1, CDH3u3 and HDP 1.0) via +HDFS-2246, +it is possible for the DFSClient to take a "short circuit" and +read directly from disk instead of going through the DataNode when the +data is local. What this means for HBase is that the RegionServers can +read directly off their machine's disks instead of having to open a +socket to talk to the DataNode, the former being generally much +fasterSee JD's Performance Talk. +Also see HBase, mail # dev - read short circuit thread for +more discussion around short circuit reads. + +To enable "short circuit" reads, you must set two configurations. +First, the hdfs-site.xml needs to be amended. Set +the property dfs.block.local-path-access.user +to be the only user that can use the shortcut. +This has to be the user that started HBase. Then in hbase-site.xml, +set dfs.client.read.shortcircuit to be true + + + For optimal performance when short-circuit reads are enabled, it is recommended that HDFS checksums are disabled. + To maintain data integrity with HDFS checksums disabled, HBase can be configured to write its own checksums into + its datablocks and verify against these. See . + + +The DataNodes need to be restarted in order to pick up the new +configuration. Be aware that if a process started under another +username than the one configured here also has the shortcircuit +enabled, it will get an Exception regarding an unauthorized access but +the data will still be read. + +
+
Performance Comparisons of HBase vs. HDFS + A fairly common question on the dist-list is why HBase isn't as performant as HDFS files in a batch context (e.g., as + a MapReduce source or sink). The short answer is that HBase is doing a lot more than HDFS (e.g., reading the KeyValues, + returning the most current row or specified timestamps, etc.), and as such HBase is 4-5 times slower than HDFS in this + processing context. Not that there isn't room for improvement (and this gap will, over time, be reduced), but HDFS + will always be faster in this use-case. + +
+
+ +
Amazon EC2 + Performance questions are common on Amazon EC2 environments because it is a shared environment. You will + not see the same throughput as a dedicated server. In terms of running tests on EC2, run them several times for the same + reason (i.e., it's a shared environment and you don't know what else is happening on the server). + + If you are running on EC2 and post performance questions on the dist-list, please state this fact up-front that + because EC2 issues are practically a separate class of performance issues. + +
+ +
Case Studies + For Performance and Troubleshooting Case Studies, see . + +
+ diff --git a/src/docbkx/preface.xml b/src/docbkx/preface.xml new file mode 100644 index 0000000..af54aa2 --- /dev/null +++ b/src/docbkx/preface.xml @@ -0,0 +1,65 @@ + + + + Preface + + This is the official reference guide for the HBase version it ships with. + This document describes HBase version . + Herein you will find either the definitive documentation on an HBase topic + as of its standing when the referenced HBase version shipped, or it + will point to the location in javadoc, + JIRA + or wiki where + the pertinent information can be found. + + This reference guide is a work in progress. Feel free to add content by adding + a patch to an issue up in the HBase JIRA. + + + Heads-up + + If this is your first foray into the wonderful world of + Distributed Computing, then you are in for + some interesting times. First off, distributed systems are + hard; making a distributed system hum requires a disparate + skillset that spans systems (hardware and software) and + networking. Your cluster' operation can hiccup because of any + of a myriad set of reasons from bugs in HBase itself through misconfigurations + -- misconfiguration of HBase but also operating system misconfigurations -- + through to hardware problems whether it be a bug in your network card + drivers or an underprovisioned RAM bus (to mention two recent + examples of hardware issues that manifested as "HBase is slow"). + You will also need to do a recalibration if up to this your + computing has been bound to a single box. Here is one good + starting point: + Fallacies of Distributed Computing. + + + diff --git a/src/docbkx/security.xml b/src/docbkx/security.xml new file mode 100644 index 0000000..ed4a0c2 --- /dev/null +++ b/src/docbkx/security.xml @@ -0,0 +1,532 @@ + + + +Secure Apache HBase (TM) +
+ Secure Client Access to Apache HBase + Newer releases of Apache HBase (TM) (>= 0.92) support optional SASL authentication of clientsSee + also Matteo Bertozzi's article on Understanding User Authentication and Authorization in Apache HBase.. + This describes how to set up Apache HBase and clients for connection to secure HBase resources. + +
Prerequisites + + You need to have a working Kerberos KDC. + + + A HBase configured for secure client access is expected to be running + on top of a secured HDFS cluster. HBase must be able to authenticate + to HDFS services. HBase needs Kerberos credentials to interact with + the Kerberos-enabled HDFS daemons. Authenticating a service should be + done using a keytab file. The procedure for creating keytabs for HBase + service is the same as for creating keytabs for Hadoop. Those steps + are omitted here. Copy the resulting keytab files to wherever HBase + Master and RegionServer processes are deployed and make them readable + only to the user account under which the HBase daemons will run. + + + A Kerberos principal has three parts, with the form + username/fully.qualified.domain.name@YOUR-REALM.COM. We + recommend using hbase as the username portion. + + + The following is an example of the configuration properties for + Kerberos operation that must be added to the + hbase-site.xml file on every server machine in the + cluster. Required for even the most basic interactions with a + secure Hadoop configuration, independent of HBase security. + + + hbase.regionserver.kerberos.principal + hbase/_HOST@YOUR-REALM.COM + + + hbase.regionserver.keytab.file + /etc/hbase/conf/keytab.krb5 + + + hbase.master.kerberos.principal + hbase/_HOST@YOUR-REALM.COM + + + hbase.master.keytab.file + /etc/hbase/conf/keytab.krb5 + + ]]> + + Each HBase client user should also be given a Kerberos principal. This + principal should have a password assigned to it (as opposed to a + keytab file). The client principal's maxrenewlife should + be set so that it can be renewed enough times for the HBase client + process to complete. For example, if a user runs a long-running HBase + client process that takes at most 3 days, we might create this user's + principal within kadmin with: addprinc -maxrenewlife + 3days + + + Long running daemons with indefinite lifetimes that require client + access to HBase can instead be configured to log in from a keytab. For + each host running such daemons, create a keytab with + kadmin or kadmin.local. The procedure for + creating keytabs for HBase service is the same as for creating + keytabs for Hadoop. Those steps are omitted here. Copy the resulting + keytab files to where the client daemon will execute and make them + readable only to the user account under which the daemon will run. + +
+ +
Server-side Configuration for Secure Operation + + Add the following to the hbase-site.xml file on every server machine in the cluster: + + + hbase.security.authentication + kerberos + + + hbase.security.authorization + true + + + hbase.coprocessor.region.classes + org.apache.hadoop.hbase.security.token.TokenProvider + + ]]> + + A full shutdown and restart of HBase service is required when deploying + these configuration changes. + +
+ +
Client-side Configuration for Secure Operation + + Add the following to the hbase-site.xml file on every client: + + + hbase.security.authentication + kerberos + + ]]> + + The client environment must be logged in to Kerberos from KDC or + keytab via the kinit command before communication with + the HBase cluster will be possible. + + + Be advised that if the hbase.security.authentication + in the client- and server-side site files do not match, the client will + not be able to communicate with the cluster. + + + Once HBase is configured for secure RPC it is possible to optionally + configure encrypted communication. To do so, add the following to the + hbase-site.xml file on every client: + + + hbase.rpc.protection + privacy + + ]]> + + This configuration property can also be set on a per connection basis. + Set it in the Configuration supplied to + HTable: + + + Configuration conf = HBaseConfiguration.create(); + conf.set("hbase.rpc.protection", "privacy"); + HTable table = new HTable(conf, tablename); + + + Expect a ~10% performance penalty for encrypted communication. + +
+ +
Client-side Configuration for Secure Operation - Thrift Gateway + + Add the following to the hbase-site.xml file for every Thrift gateway: + + hbase.thrift.keytab.file + /etc/hbase/conf/hbase.keytab + + + hbase.thrift.kerberos.principal + $USER/_HOST@HADOOP.LOCALDOMAIN + + ]]> + + + Substitute the appropriate credential and keytab for $USER and $KEYTAB + respectively. + + + The Thrift gateway will authenticate with HBase using the supplied + credential. No authentication will be performed by the Thrift gateway + itself. All client access via the Thrift gateway will use the Thrift + gateway's credential and have its privilege. + +
+ +
Client-side Configuration for Secure Operation - REST Gateway + + Add the following to the hbase-site.xml file for every REST gateway: + + hbase.rest.keytab.file + $KEYTAB + + + hbase.rest.kerberos.principal + $USER/_HOST@HADOOP.LOCALDOMAIN + + ]]> + + + Substitute the appropriate credential and keytab for $USER and $KEYTAB + respectively. + + + The REST gateway will authenticate with HBase using the supplied + credential. No authentication will be performed by the REST gateway + itself. All client access via the REST gateway will use the REST + gateway's credential and have its privilege. + + + It should be possible for clients to authenticate with the HBase + cluster through the REST gateway in a pass-through manner via SPEGNO + HTTP authentication. This is future work. + +
+ +
+ + +
+ Access Control + + Newer releases of Apache HBase (>= 0.92) support optional access control + list (ACL-) based protection of resources on a column family and/or + table basis. + + + This describes how to set up Secure HBase for access control, with an + example of granting and revoking user permission on table resources + provided. + + +
Prerequisites + + You must configure HBase for secure operation. Refer to the section + "Secure Client Access to HBase" and complete all of the steps described + there. + + + You must also configure ZooKeeper for secure operation. Changes to ACLs + are synchronized throughout the cluster using ZooKeeper. Secure + authentication to ZooKeeper must be enabled or otherwise it will be + possible to subvert HBase access control via direct client access to + ZooKeeper. Refer to the section on secure ZooKeeper configuration and + complete all of the steps described there. + +
+ +
Overview + + With Secure RPC and Access Control enabled, client access to HBase is + authenticated and user data is private unless access has been + explicitly granted. Access to data can be granted at a table or per + column family basis. + + + However, the following items have been left out of the initial + implementation for simplicity: + + + + Row-level or per value (cell): This would require broader changes for storing the ACLs inline with rows. It is a future goal. + + + Push down of file ownership to HDFS: HBase is not designed for the case where files may have different permissions than the HBase system principal. Pushing file ownership down into HDFS would necessitate changes to core code. Also, while HDFS file ownership would make applying quotas easy, and possibly make bulk imports more straightforward, it is not clear that it would offer a more secure setup. + + + HBase managed "roles" as collections of permissions: We will not model "roles" internally in HBase to begin with. We instead allow group names to be granted permissions, which allows external modeling of roles via group membership. Groups are created and manipulated externally to HBase, via the Hadoop group mapping service. + + + +Access control mechanisms are mature and fairly standardized in the relational database world. The HBase implementation approximates current convention, but HBase has a simpler feature set than relational databases, especially in terms of client operations. We don't distinguish between an insert (new record) and update (of existing record), for example, as both collapse down into a Put. Accordingly, the important operations condense to four permissions: READ, WRITE, CREATE, and ADMIN. + + + Operation To Permission Mapping + + + Permission + Operation + + + + + + Read + Get + + + + Exists + + + + Scan + + + + Write + Put + + + + Delete + + + + Lock/UnlockRow + + + + IncrementColumnValue + + + + CheckAndDelete/Put + + + + Flush + + + + Compact + + + + Create + Create + + + + Alter + + + + Drop + + + + Admin + Enable/Disable + + + + Split + + + + Major Compact + + + + Grant + + + + Revoke + + + + Shutdown + + +
+ + Permissions can be granted in any of the following scopes, though + CREATE and ADMIN permissions are effective only at table scope. + + + + + Table + + + Read: User can read from any column family in table + Write: User can write to any column family in table + Create: User can alter table attributes; add, alter, or drop column families; and drop the table. + Admin: User can alter table attributes; add, alter, or drop column families; and enable, disable, or drop the table. User can also trigger region (re)assignments or relocation. + + + + + Column Family + + + Read: User can read from the column family + Write: User can write to the column family + + + + + + + There is also an implicit global scope for the superuser. + + + The superuser is a principal, specified in the HBase site configuration + file, that has equivalent access to HBase as the 'root' user would on a + UNIX derived system. Normally this is the principal that the HBase + processes themselves authenticate as. Although future versions of HBase + Access Control may support multiple superusers, the superuser privilege + will always include the principal used to run the HMaster process. Only + the superuser is allowed to create tables, switch the balancer on or + off, or take other actions with global consequence. Furthermore, the + superuser has an implicit grant of all permissions to all resources. + + + Tables have a new metadata attribute: OWNER, the user principal who owns + the table. By default this will be set to the user principal who creates + the table, though it may be changed at table creation time or during an + alter operation by setting or changing the OWNER table attribute. Only a + single user principal can own a table at a given time. A table owner will + have all permissions over a given table. + +
+ +
Server-side Configuration for Access Control + + Enable the AccessController coprocessor in the cluster configuration + and restart HBase. The restart can be a rolling one. Complete the + restart of all Master and RegionServer processes before setting up + ACLs. + + + To enable the AccessController, modify the hbase-site.xml file on every server machine in the cluster to look like: + + + hbase.coprocessor.master.classes + org.apache.hadoop.hbase.security.access.AccessController + + + hbase.coprocessor.region.classes + org.apache.hadoop.hbase.security.token.TokenProvider, + org.apache.hadoop.hbase.security.access.AccessController + + ]]> +
+ +
Shell Enhancements for Access Control + +The HBase shell has been extended to provide simple commands for editing and updating user permissions. The following commands have been added for access control list management: + + Grant + + + grant <user> <permissions> <table> [ <column family> [ <column qualifier> ] ] + + + + <permissions> is zero or more letters from the set "RWCA": READ('R'), WRITE('W'), CREATE('C'), ADMIN('A'). + + + Note: Grants and revocations of individual permissions on a resource are both accomplished using the grant command. A separate revoke command is also provided by the shell, but this is for fast revocation of all of a user's access rights to a given resource only. + + + Revoke + + + + revoke <user> <table> [ <column family> [ <column qualifier> ] ] + + + + Alter + + + The alter command has been extended to allow ownership assignment: + + alter 'tablename', {OWNER => 'username'} + + + + User Permission + + + The user_permission command shows all access permissions for the current user for a given table: + + user_permission <table> + + +
+ +
+ +
+ Secure Bulk Load + + Bulk loading in secure mode is a bit more involved than normal setup, since the client has to transfer the ownership of the files generated from the mapreduce job to HBase. Secure bulk loading is implemented by a coprocessor, named SecureBulkLoadEndpoint. SecureBulkLoadEndpoint uses a staging directory "hbase.bulkload.staging.dir", which defaults to /tmp/hbase-staging/. The algorithm is as follows. + + Create an hbase owned staging directory which is world traversable (-rwx--x--x, 711) /tmp/hbase-staging. + A user writes out data to his secure output directory: /user/foo/data + A call is made to hbase to create a secret staging directory + which is globally readable/writable (-rwxrwxrwx, 777): /tmp/hbase-staging/averylongandrandomdirectoryname + The user makes the data world readable and writable, then moves it + into the random staging directory, then calls bulkLoadHFiles() + + + + Like delegation tokens the strength of the security lies in the length + and randomness of the secret directory. + + + + You have to enable the secure bulk load to work properly. You can modify the hbase-site.xml file on every server machine in the cluster and add the SecureBulkLoadEndpoint class to the list of regionserver coprocessors: + + + hbase.bulkload.staging.dir + /tmp/hbase-staging + + + hbase.coprocessor.region.classes + org.apache.hadoop.hbase.security.token.TokenProvider, + org.apache.hadoop.hbase.security.access.AccessController,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint + + ]]> +
+ diff --git a/src/docbkx/shell.xml b/src/docbkx/shell.xml new file mode 100644 index 0000000..2a15353 --- /dev/null +++ b/src/docbkx/shell.xml @@ -0,0 +1,119 @@ + + + + The Apache HBase Shell + + + The Apache HBase (TM) Shell is (J)Ruby's + IRB with some HBase particular commands added. Anything you can do in + IRB, you should be able to do in the HBase Shell. + To run the HBase shell, + do as follows: + $ ./bin/hbase shell + + Type help and then <RETURN> + to see a listing of shell + commands and options. Browse at least the paragraphs at the end of + the help emission for the gist of how variables and command + arguments are entered into the + HBase shell; in particular note how table names, rows, and + columns, etc., must be quoted. + See + for example basic shell operation. + +
Scripting + For examples scripting Apache HBase, look in the + HBase bin directory. Look at the files + that end in *.rb. To run one of these + files, do as follows: + $ ./bin/hbase org.jruby.Main PATH_TO_SCRIPT + +
+ +
Shell Tricks +
<filename>irbrc</filename> + Create an .irbrc file for yourself in your + home directory. Add customizations. A useful one is + command history so commands are save across Shell invocations: + + $ more .irbrc + require 'irb/ext/save-history' + IRB.conf[:SAVE_HISTORY] = 100 + IRB.conf[:HISTORY_FILE] = "#{ENV['HOME']}/.irb-save-history" + See the ruby documentation of + .irbrc to learn about other possible + confiurations. + +
+
LOG data to timestamp + + To convert the date '08/08/16 20:56:29' from an hbase log into a timestamp, do: + + hbase(main):021:0> import java.text.SimpleDateFormat + hbase(main):022:0> import java.text.ParsePosition + hbase(main):023:0> SimpleDateFormat.new("yy/MM/dd HH:mm:ss").parse("08/08/16 20:56:29", ParsePosition.new(0)).getTime() => 1218920189000 + + + To go the other direction: + + hbase(main):021:0> import java.util.Date + hbase(main):022:0> Date.new(1218920189000).toString() => "Sat Aug 16 20:56:29 UTC 2008" + + + To output in a format that is exactly like that of the HBase log format will take a little messing with + SimpleDateFormat. + +
+
Debug +
Shell debug switch + You can set a debug switch in the shell to see more output + -- e.g. more of the stack trace on exception -- + when you run a command: + hbase> debug <RETURN> + +
+
DEBUG log level + To enable DEBUG level logging in the shell, + launch it with the -d option. + $ ./bin/hbase shell -d + +
+
+
Commands +
count + Count command returns the number of rows in a table. + It's quite fast when configured with the right CACHE + hbase> count '<tablename>', CACHE => 1000 + The above count fetches 1000 rows at a time. Set CACHE lower if your rows are big. + Default is to fetch one row at a time. + +
+
+ +
+ diff --git a/src/docbkx/troubleshooting.xml b/src/docbkx/troubleshooting.xml new file mode 100644 index 0000000..5967b03 --- /dev/null +++ b/src/docbkx/troubleshooting.xml @@ -0,0 +1,1090 @@ + + + + Troubleshooting and Debugging Apache HBase (TM) +
+ General Guidelines + + Always start with the master log (TODO: Which lines?). + Normally it’s just printing the same lines over and over again. + If not, then there’s an issue. + Google or search-hadoop.com + should return some hits for those exceptions you’re seeing. + + + An error rarely comes alone in Apache HBase (TM), usually when something gets screwed up what will + follow may be hundreds of exceptions and stack traces coming from all over the place. + The best way to approach this type of problem is to walk the log up to where it all + began, for example one trick with RegionServers is that they will print some + metrics when aborting so grepping for Dump + should get you around the start of the problem. + + + RegionServer suicides are “normal”, as this is what they do when something goes wrong. + For example, if ulimit and xcievers (the two most important initial settings, see ) + aren’t changed, it will make it impossible at some point for DataNodes to create new threads + that from the HBase point of view is seen as if HDFS was gone. Think about what would happen if your + MySQL database was suddenly unable to access files on your local file system, well it’s the same with + HBase and HDFS. Another very common reason to see RegionServers committing seppuku is when they enter + prolonged garbage collection pauses that last longer than the default ZooKeeper session timeout. + For more information on GC pauses, see the + 3 part blog post by Todd Lipcon + and above. + +
+
+ Logs + + The key process logs are as follows... (replace <user> with the user that started the service, and <hostname> for the machine name) + + + NameNode: $HADOOP_HOME/logs/hadoop-<user>-namenode-<hostname>.log + + + DataNode: $HADOOP_HOME/logs/hadoop-<user>-datanode-<hostname>.log + + + JobTracker: $HADOOP_HOME/logs/hadoop-<user>-jobtracker-<hostname>.log + + + TaskTracker: $HADOOP_HOME/logs/hadoop-<user>-tasktracker-<hostname>.log + + + HMaster: $HBASE_HOME/logs/hbase-<user>-master-<hostname>.log + + + RegionServer: $HBASE_HOME/logs/hbase-<user>-regionserver-<hostname>.log + + + ZooKeeper: TODO + +
+ Log Locations + For stand-alone deployments the logs are obviously going to be on a single machine, however this is a development configuration only. + Production deployments need to run on a cluster. +
+ NameNode + The NameNode log is on the NameNode server. The HBase Master is typically run on the NameNode server, and well as ZooKeeper. + For smaller clusters the JobTracker is typically run on the NameNode server as well. +
+
+ DataNode + Each DataNode server will have a DataNode log for HDFS, as well as a RegionServer log for HBase. + Additionally, each DataNode server will also have a TaskTracker log for MapReduce task execution. +
+
+
+ Log Levels +
Enabling RPC-level logging + Enabling the RPC-level logging on a RegionServer can often given + insight on timings at the server. Once enabled, the amount of log + spewed is voluminous. It is not recommended that you leave this + logging on for more than short bursts of time. To enable RPC-level + logging, browse to the RegionServer UI and click on + Log Level. Set the log level to DEBUG for the package + org.apache.hadoop.ipc (Thats right, for + hadoop.ipc, NOT, hbase.ipc). Then tail the RegionServers log. Analyze. + To disable, set the logging level back to INFO level. + +
+
+
+ JVM Garbage Collection Logs + HBase is memory intensive, and using the default GC you can see long pauses in all threads including the Juliet Pause aka "GC of Death". + To help debug this or confirm this is happening GC logging can be turned on in the Java virtual machine. + + + To enable, in hbase-env.sh add: + +export HBASE_OPTS="-XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/home/hadoop/hbase/logs/gc-hbase.log" + + Adjust the log directory to wherever you log. Note: The GC log does NOT roll automatically, so you'll have to keep an eye on it so it doesn't fill up the disk. + + + At this point you should see logs like so: + +64898.952: [GC [1 CMS-initial-mark: 2811538K(3055704K)] 2812179K(3061272K), 0.0007360 secs] [Times: user=0.00 sys=0.00, real=0.00 secs] +64898.953: [CMS-concurrent-mark-start] +64898.971: [GC 64898.971: [ParNew: 5567K->576K(5568K), 0.0101110 secs] 2817105K->2812715K(3061272K), 0.0102200 secs] [Times: user=0.07 sys=0.00, real=0.01 secs] + + + + In this section, the first line indicates a 0.0007360 second pause for the CMS to initially mark. This pauses the entire VM, all threads for that period of time. + + + The third line indicates a "minor GC", which pauses the VM for 0.0101110 seconds - aka 10 milliseconds. It has reduced the "ParNew" from about 5.5m to 576k. + Later on in this cycle we see: + +64901.445: [CMS-concurrent-mark: 1.542/2.492 secs] [Times: user=10.49 sys=0.33, real=2.49 secs] +64901.445: [CMS-concurrent-preclean-start] +64901.453: [GC 64901.453: [ParNew: 5505K->573K(5568K), 0.0062440 secs] 2868746K->2864292K(3061272K), 0.0063360 secs] [Times: user=0.05 sys=0.00, real=0.01 secs] +64901.476: [GC 64901.476: [ParNew: 5563K->575K(5568K), 0.0072510 secs] 2869283K->2864837K(3061272K), 0.0073320 secs] [Times: user=0.05 sys=0.01, real=0.01 secs] +64901.500: [GC 64901.500: [ParNew: 5517K->573K(5568K), 0.0120390 secs] 2869780K->2865267K(3061272K), 0.0121150 secs] [Times: user=0.09 sys=0.00, real=0.01 secs] +64901.529: [GC 64901.529: [ParNew: 5507K->569K(5568K), 0.0086240 secs] 2870200K->2865742K(3061272K), 0.0087180 secs] [Times: user=0.05 sys=0.00, real=0.01 secs] +64901.554: [GC 64901.555: [ParNew: 5516K->575K(5568K), 0.0107130 secs] 2870689K->2866291K(3061272K), 0.0107820 secs] [Times: user=0.06 sys=0.00, real=0.01 secs] +64901.578: [CMS-concurrent-preclean: 0.070/0.133 secs] [Times: user=0.48 sys=0.01, real=0.14 secs] +64901.578: [CMS-concurrent-abortable-preclean-start] +64901.584: [GC 64901.584: [ParNew: 5504K->571K(5568K), 0.0087270 secs] 2871220K->2866830K(3061272K), 0.0088220 secs] [Times: user=0.05 sys=0.00, real=0.01 secs] +64901.609: [GC 64901.609: [ParNew: 5512K->569K(5568K), 0.0063370 secs] 2871771K->2867322K(3061272K), 0.0064230 secs] [Times: user=0.06 sys=0.00, real=0.01 secs] +64901.615: [CMS-concurrent-abortable-preclean: 0.007/0.037 secs] [Times: user=0.13 sys=0.00, real=0.03 secs] +64901.616: [GC[YG occupancy: 645 K (5568 K)]64901.616: [Rescan (parallel) , 0.0020210 secs]64901.618: [weak refs processing, 0.0027950 secs] [1 CMS-remark: 2866753K(3055704K)] 2867399K(3061272K), 0.0049380 secs] [Times: user=0.00 sys=0.01, real=0.01 secs] +64901.621: [CMS-concurrent-sweep-start] + + + + The first line indicates that the CMS concurrent mark (finding garbage) has taken 2.4 seconds. But this is a _concurrent_ 2.4 seconds, Java has not been paused at any point in time. + + + There are a few more minor GCs, then there is a pause at the 2nd last line: + +64901.616: [GC[YG occupancy: 645 K (5568 K)]64901.616: [Rescan (parallel) , 0.0020210 secs]64901.618: [weak refs processing, 0.0027950 secs] [1 CMS-remark: 2866753K(3055704K)] 2867399K(3061272K), 0.0049380 secs] [Times: user=0.00 sys=0.01, real=0.01 secs] + + + + The pause here is 0.0049380 seconds (aka 4.9 milliseconds) to 'remark' the heap. + + + At this point the sweep starts, and you can watch the heap size go down: + +64901.637: [GC 64901.637: [ParNew: 5501K->569K(5568K), 0.0097350 secs] 2871958K->2867441K(3061272K), 0.0098370 secs] [Times: user=0.05 sys=0.00, real=0.01 secs] +... lines removed ... +64904.936: [GC 64904.936: [ParNew: 5532K->568K(5568K), 0.0070720 secs] 1365024K->1360689K(3061272K), 0.0071930 secs] [Times: user=0.05 sys=0.00, real=0.01 secs] +64904.953: [CMS-concurrent-sweep: 2.030/3.332 secs] [Times: user=9.57 sys=0.26, real=3.33 secs] + + At this point, the CMS sweep took 3.332 seconds, and heap went from about ~ 2.8 GB to 1.3 GB (approximate). + + + The key points here is to keep all these pauses low. CMS pauses are always low, but if your ParNew starts growing, you can see minor GC pauses approach 100ms, exceed 100ms and hit as high at 400ms. + + + This can be due to the size of the ParNew, which should be relatively small. If your ParNew is very large after running HBase for a while, in one example a ParNew was about 150MB, then you might have to constrain the size of ParNew (The larger it is, the longer the collections take but if its too small, objects are promoted to old gen too quickly). In the below we constrain new gen size to 64m. + + + Add this to HBASE_OPTS: + +export HBASE_OPTS="-XX:NewSize=64m -XX:MaxNewSize=64m <cms options from above> <gc logging options from above>" + + + + For more information on GC pauses, see the 3 part blog post by Todd Lipcon + and above. + +
+
+
+ Resources +
+ search-hadoop.com + + search-hadoop.com indexes all the mailing lists and is great for historical searches. + Search here first when you have an issue as its more than likely someone has already had your problem. + +
+
+ Mailing Lists + Ask a question on the Apache HBase mailing lists. + The 'dev' mailing list is aimed at the community of developers actually building Apache HBase and for features currently under development, and 'user' + is generally used for questions on released versions of Apache HBase. Before going to the mailing list, make sure your + question has not already been answered by searching the mailing list archives first. Use + . + Take some time crafting your questionSee Getting Answers; a quality question that includes all context and + exhibits evidence the author has tried to find answers in the manual and out on lists + is more likely to get a prompt response. + +
+
+ IRC + #hbase on irc.freenode.net +
+
+ JIRA + + JIRA is also really helpful when looking for Hadoop/HBase-specific issues. + +
+
+
+ Tools +
+ Builtin Tools +
+ Master Web Interface + The Master starts a web-interface on port 60010 by default. + + The Master web UI lists created tables and their definition (e.g., ColumnFamilies, blocksize, etc.). Additionally, + the available RegionServers in the cluster are listed along with selected high-level metrics (requests, number of regions, usedHeap, maxHeap). + The Master web UI allows navigation to each RegionServer's web UI. + +
+
+ RegionServer Web Interface + RegionServers starts a web-interface on port 60030 by default. + + The RegionServer web UI lists online regions and their start/end keys, as well as point-in-time RegionServer metrics (requests, regions, storeFileIndexSize, compactionQueueSize, etc.). + + See for more information in metric definitions. + +
+
+ zkcli + zkcli is a very useful tool for investigating ZooKeeper-related issues. To invoke: + +./hbase zkcli -server host:port <cmd> <args> + + The commands (and arguments) are: + + connect host:port + get path [watch] + ls path [watch] + set path data [version] + delquota [-n|-b] path + quit + printwatches on|off + create [-s] [-e] path data acl + stat path [watch] + close + ls2 path [watch] + history + listquota path + setAcl path acl + getAcl path + sync path + redo cmdno + addauth scheme auth + delete path [version] + setquota -n|-b val path + + +
+
+
+ External Tools +
+ tail + + tail is the command line tool that lets you look at the end of a file. Add the “-f” option and it will refresh when new data is available. It’s useful when you are wondering what’s happening, for example, when a cluster is taking a long time to shutdown or startup as you can just fire a new terminal and tail the master log (and maybe a few RegionServers). + +
+
+ top + + top is probably one of the most important tool when first trying to see what’s running on a machine and how the resources are consumed. Here’s an example from production system: + +top - 14:46:59 up 39 days, 11:55, 1 user, load average: 3.75, 3.57, 3.84 +Tasks: 309 total, 1 running, 308 sleeping, 0 stopped, 0 zombie +Cpu(s): 4.5%us, 1.6%sy, 0.0%ni, 91.7%id, 1.4%wa, 0.1%hi, 0.6%si, 0.0%st +Mem: 24414432k total, 24296956k used, 117476k free, 7196k buffers +Swap: 16008732k total, 14348k used, 15994384k free, 11106908k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +15558 hadoop 18 -2 3292m 2.4g 3556 S 79 10.4 6523:52 java +13268 hadoop 18 -2 8967m 8.2g 4104 S 21 35.1 5170:30 java + 8895 hadoop 18 -2 1581m 497m 3420 S 11 2.1 4002:32 java +… + + + + Here we can see that the system load average during the last five minutes is 3.75, which very roughly means that on average 3.75 threads were waiting for CPU time during these 5 minutes. In general, the “perfect” utilization equals to the number of cores, under that number the machine is under utilized and over that the machine is over utilized. This is an important concept, see this article to understand it more: http://www.linuxjournal.com/article/9001. + + + Apart from load, we can see that the system is using almost all its available RAM but most of it is used for the OS cache (which is good). The swap only has a few KBs in it and this is wanted, high numbers would indicate swapping activity which is the nemesis of performance of Java systems. Another way to detect swapping is when the load average goes through the roof (although this could also be caused by things like a dying disk, among others). + + + The list of processes isn’t super useful by default, all we know is that 3 java processes are using about 111% of the CPUs. To know which is which, simply type “c” and each line will be expanded. Typing “1” will give you the detail of how each CPU is used instead of the average for all of them like shown here. + +
+
+ jps + + jps is shipped with every JDK and gives the java process ids for the current user (if root, then it gives the ids for all users). Example: + +hadoop@sv4borg12:~$ jps +1322 TaskTracker +17789 HRegionServer +27862 Child +1158 DataNode +25115 HQuorumPeer +2950 Jps +19750 ThriftServer +18776 jmx + + In order, we see a: + + Hadoop TaskTracker, manages the local Childs + HBase RegionServer, serves regions + Child, its MapReduce task, cannot tell which type exactly + Hadoop TaskTracker, manages the local Childs + Hadoop DataNode, serves blocks + HQuorumPeer, a ZooKeeper ensemble member + Jps, well… it’s the current process + ThriftServer, it’s a special one will be running only if thrift was started + jmx, this is a local process that’s part of our monitoring platform ( poorly named maybe). You probably don’t have that. + + + + You can then do stuff like checking out the full command line that started the process: + +hadoop@sv4borg12:~$ ps aux | grep HRegionServer +hadoop 17789 155 35.2 9067824 8604364 ? S<l Mar04 9855:48 /usr/java/jdk1.6.0_14/bin/java -Xmx8000m -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -XX:+UseConcMarkSweepGC -XX:NewSize=64m -XX:MaxNewSize=64m -XX:CMSInitiatingOccupancyFraction=88 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/export1/hadoop/logs/gc-hbase.log -Dcom.sun.management.jmxremote.port=10102 -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.password.file=/home/hadoop/hbase/conf/jmxremote.password -Dcom.sun.management.jmxremote -Dhbase.log.dir=/export1/hadoop/logs -Dhbase.log.file=hbase-hadoop-regionserver-sv4borg12.log -Dhbase.home.dir=/home/hadoop/hbase -Dhbase.id.str=hadoop -Dhbase.root.logger=INFO,DRFA -Djava.library.path=/home/hadoop/hbase/lib/native/Linux-amd64-64 -classpath /home/hadoop/hbase/bin/../conf:[many jars]:/home/hadoop/hadoop/conf org.apache.hadoop.hbase.regionserver.HRegionServer start + + +
+
+ jstack + + jstack is one of the most important tools when trying to figure out what a java process is doing apart from looking at the logs. It has to be used in conjunction with jps in order to give it a process id. It shows a list of threads, each one has a name, and they appear in the order that they were created (so the top ones are the most recent threads). Here’s a few example: + + + The main thread of a RegionServer that’s waiting for something to do from the master: + + "regionserver60020" prio=10 tid=0x0000000040ab4000 nid=0x45cf waiting on condition [0x00007f16b6a96000..0x00007f16b6a96a70] + java.lang.Thread.State: TIMED_WAITING (parking) + at sun.misc.Unsafe.park(Native Method) + - parking to wait for <0x00007f16cd5c2f30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) + at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) + at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) + at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:395) + at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:647) + at java.lang.Thread.run(Thread.java:619) + + The MemStore flusher thread that is currently flushing to a file: +"regionserver60020.cacheFlusher" daemon prio=10 tid=0x0000000040f4e000 nid=0x45eb in Object.wait() [0x00007f16b5b86000..0x00007f16b5b87af0] + java.lang.Thread.State: WAITING (on object monitor) + at java.lang.Object.wait(Native Method) + at java.lang.Object.wait(Object.java:485) + at org.apache.hadoop.ipc.Client.call(Client.java:803) + - locked <0x00007f16cb14b3a8> (a org.apache.hadoop.ipc.Client$Call) + at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:221) + at $Proxy1.complete(Unknown Source) + at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) + at java.lang.reflect.Method.invoke(Method.java:597) + at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82) + at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59) + at $Proxy1.complete(Unknown Source) + at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.closeInternal(DFSClient.java:3390) + - locked <0x00007f16cb14b470> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream) + at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.close(DFSClient.java:3304) + at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:61) + at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:86) + at org.apache.hadoop.hbase.io.hfile.HFile$Writer.close(HFile.java:650) + at org.apache.hadoop.hbase.regionserver.StoreFile$Writer.close(StoreFile.java:853) + at org.apache.hadoop.hbase.regionserver.Store.internalFlushCache(Store.java:467) + - locked <0x00007f16d00e6f08> (a java.lang.Object) + at org.apache.hadoop.hbase.regionserver.Store.flushCache(Store.java:427) + at org.apache.hadoop.hbase.regionserver.Store.access$100(Store.java:80) + at org.apache.hadoop.hbase.regionserver.Store$StoreFlusherImpl.flushCache(Store.java:1359) + at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:907) + at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:834) + at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:786) + at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:250) + at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:224) + at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:146) + + + + A handler thread that’s waiting for stuff to do (like put, delete, scan, etc): + +"IPC Server handler 16 on 60020" daemon prio=10 tid=0x00007f16b011d800 nid=0x4a5e waiting on condition [0x00007f16afefd000..0x00007f16afefd9f0] + java.lang.Thread.State: WAITING (parking) + at sun.misc.Unsafe.park(Native Method) + - parking to wait for <0x00007f16cd3f8dd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) + at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) + at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) + at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) + at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013) + + + + And one that’s busy doing an increment of a counter (it’s in the phase where it’s trying to create a scanner in order to read the last value): + +"IPC Server handler 66 on 60020" daemon prio=10 tid=0x00007f16b006e800 nid=0x4a90 runnable [0x00007f16acb77000..0x00007f16acb77cf0] + java.lang.Thread.State: RUNNABLE + at org.apache.hadoop.hbase.regionserver.KeyValueHeap.<init>(KeyValueHeap.java:56) + at org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:79) + at org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:1202) + at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.<init>(HRegion.java:2209) + at org.apache.hadoop.hbase.regionserver.HRegion.instantiateInternalScanner(HRegion.java:1063) + at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1055) + at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1039) + at org.apache.hadoop.hbase.regionserver.HRegion.getLastIncrement(HRegion.java:2875) + at org.apache.hadoop.hbase.regionserver.HRegion.incrementColumnValue(HRegion.java:2978) + at org.apache.hadoop.hbase.regionserver.HRegionServer.incrementColumnValue(HRegionServer.java:2433) + at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) + at java.lang.reflect.Method.invoke(Method.java:597) + at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:560) + at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1027) + + + + A thread that receives data from HDFS: + +"IPC Client (47) connection to sv4borg9/10.4.24.40:9000 from hadoop" daemon prio=10 tid=0x00007f16a02d0000 nid=0x4fa3 runnable [0x00007f16b517d000..0x00007f16b517dbf0] + java.lang.Thread.State: RUNNABLE + at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) + at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:215) + at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) + at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) + - locked <0x00007f17d5b68c00> (a sun.nio.ch.Util$1) + - locked <0x00007f17d5b68be8> (a java.util.Collections$UnmodifiableSet) + - locked <0x00007f1877959b50> (a sun.nio.ch.EPollSelectorImpl) + at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) + at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332) + at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) + at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155) + at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128) + at java.io.FilterInputStream.read(FilterInputStream.java:116) + at org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:304) + at java.io.BufferedInputStream.fill(BufferedInputStream.java:218) + at java.io.BufferedInputStream.read(BufferedInputStream.java:237) + - locked <0x00007f1808539178> (a java.io.BufferedInputStream) + at java.io.DataInputStream.readInt(DataInputStream.java:370) + at org.apache.hadoop.ipc.Client$Connection.receiveResponse(Client.java:569) + at org.apache.hadoop.ipc.Client$Connection.run(Client.java:477) + + + + And here is a master trying to recover a lease after a RegionServer died: + +"LeaseChecker" daemon prio=10 tid=0x00000000407ef800 nid=0x76cd waiting on condition [0x00007f6d0eae2000..0x00007f6d0eae2a70] +-- + java.lang.Thread.State: WAITING (on object monitor) + at java.lang.Object.wait(Native Method) + at java.lang.Object.wait(Object.java:485) + at org.apache.hadoop.ipc.Client.call(Client.java:726) + - locked <0x00007f6d1cd28f80> (a org.apache.hadoop.ipc.Client$Call) + at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220) + at $Proxy1.recoverBlock(Unknown Source) + at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2636) + at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.<init>(DFSClient.java:2832) + at org.apache.hadoop.hdfs.DFSClient.append(DFSClient.java:529) + at org.apache.hadoop.hdfs.DistributedFileSystem.append(DistributedFileSystem.java:186) + at org.apache.hadoop.fs.FileSystem.append(FileSystem.java:530) + at org.apache.hadoop.hbase.util.FSUtils.recoverFileLease(FSUtils.java:619) + at org.apache.hadoop.hbase.regionserver.wal.HLog.splitLog(HLog.java:1322) + at org.apache.hadoop.hbase.regionserver.wal.HLog.splitLog(HLog.java:1210) + at org.apache.hadoop.hbase.master.HMaster.splitLogAfterStartup(HMaster.java:648) + at org.apache.hadoop.hbase.master.HMaster.joinCluster(HMaster.java:572) + at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:503) + + +
+
+ OpenTSDB + + OpenTSDB is an excellent alternative to Ganglia as it uses Apache HBase to store all the time series and doesn’t have to downsample. Monitoring your own HBase cluster that hosts OpenTSDB is a good exercise. + + + Here’s an example of a cluster that’s suffering from hundreds of compactions launched almost all around the same time, which severely affects the IO performance: (TODO: insert graph plotting compactionQueueSize) + + + It’s a good practice to build dashboards with all the important graphs per machine and per cluster so that debugging issues can be done with a single quick look. For example, at StumbleUpon there’s one dashboard per cluster with the most important metrics from both the OS and Apache HBase. You can then go down at the machine level and get even more detailed metrics. + +
+
+ clusterssh+top + + clusterssh+top, it’s like a poor man’s monitoring system and it can be quite useful when you have only a few machines as it’s very easy to setup. Starting clusterssh will give you one terminal per machine and another terminal in which whatever you type will be retyped in every window. This means that you can type “top” once and it will start it for all of your machines at the same time giving you full view of the current state of your cluster. You can also tail all the logs at the same time, edit files, etc. + +
+
+
+ +
+ Client + For more information on the HBase client, see . + +
+ ScannerTimeoutException or UnknownScannerException + This is thrown if the time between RPC calls from the client to RegionServer exceeds the scan timeout. + For example, if Scan.setCaching is set to 500, then there will be an RPC call to fetch the next batch of rows every 500 .next() calls on the ResultScanner + because data is being transferred in blocks of 500 rows to the client. Reducing the setCaching value may be an option, but setting this value too low makes for inefficient + processing on numbers of rows. + + See . + +
+
+ <classname>LeaseException</classname> when calling <classname>Scanner.next</classname> + +In some situations clients that fetch data from a RegionServer get a LeaseException instead of the usual +. Usually the source of the exception is +org.apache.hadoop.hbase.regionserver.Leases.removeLease(Leases.java:230) (line number may vary). +It tends to happen in the context of a slow/freezing RegionServer#next call. +It can be prevented by having hbase.rpc.timeout > hbase.regionserver.lease.period. +Harsh J investigated the issue as part of the mailing list thread +HBase, mail # user - Lease does not exist exceptions + +
+
+ Shell or client application throws lots of scary exceptions during normal operation + Since 0.20.0 the default log level for org.apache.hadoop.hbase.*is DEBUG. + + On your clients, edit $HBASE_HOME/conf/log4j.properties and change this: log4j.logger.org.apache.hadoop.hbase=DEBUG to this: log4j.logger.org.apache.hadoop.hbase=INFO, or even log4j.logger.org.apache.hadoop.hbase=WARN. + +
+
+ Long Client Pauses With Compression + This is a fairly frequent question on the Apache HBase dist-list. The scenario is that a client is typically inserting a lot of data into a + relatively un-optimized HBase cluster. Compression can exacerbate the pauses, although it is not the source of the problem. + See on the pattern for pre-creating regions and confirm that the table isn't starting with a single region. + See for cluster configuration, particularly hbase.hstore.blockingStoreFiles, hbase.hregion.memstore.block.multiplier, + MAX_FILESIZE (region size), and MEMSTORE_FLUSHSIZE. + A slightly longer explanation of why pauses can happen is as follows: Puts are sometimes blocked on the MemStores which are blocked by the flusher thread which is blocked because there are + too many files to compact because the compactor is given too many small files to compact and has to compact the same data repeatedly. This situation can occur even with minor compactions. + Compounding this situation, Apache HBase doesn't compress data in memory. Thus, the 64MB that lives in the MemStore could become a 6MB file after compression - which results in a smaller StoreFile. The upside is that + more data is packed into the same region, but performance is achieved by being able to write larger files - which is why HBase waits until the flushize before writing a new StoreFile. And smaller StoreFiles + become targets for compaction. Without compression the files are much bigger and don't need as much compaction, however this is at the expense of I/O. + + + For additional information, see this thread on Long client pauses with compression. + + +
+
+ ZooKeeper Client Connection Errors + Errors like this... + +11/07/05 11:26:41 WARN zookeeper.ClientCnxn: Session 0x0 for server null, + unexpected error, closing socket connection and attempting reconnect + java.net.ConnectException: Connection refused: no further information + at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) + at sun.nio.ch.SocketChannelImpl.finishConnect(Unknown Source) + at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1078) + 11/07/05 11:26:43 INFO zookeeper.ClientCnxn: Opening socket connection to + server localhost/127.0.0.1:2181 + 11/07/05 11:26:44 WARN zookeeper.ClientCnxn: Session 0x0 for server null, + unexpected error, closing socket connection and attempting reconnect + java.net.ConnectException: Connection refused: no further information + at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) + at sun.nio.ch.SocketChannelImpl.finishConnect(Unknown Source) + at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1078) + 11/07/05 11:26:45 INFO zookeeper.ClientCnxn: Opening socket connection to + server localhost/127.0.0.1:2181 + + ... are either due to ZooKeeper being down, or unreachable due to network issues. + + The utility may help investigate ZooKeeper issues. + +
+
+ Client running out of memory though heap size seems to be stable (but the off-heap/direct heap keeps growing) + +You are likely running into the issue that is described and worked through in +the mail thread HBase, mail # user - Suspected memory leak +and continued over in HBase, mail # dev - FeedbackRe: Suspected memory leak. +A workaround is passing your client-side JVM a reasonable value for -XX:MaxDirectMemorySize. By default, +the MaxDirectMemorySize is equal to your -Xmx max heapsize setting (if -Xmx is set). +Try seting it to something smaller (for example, one user had success setting it to 1g when +they had a client-side heap of 12g). If you set it too small, it will bring on FullGCs so keep +it a bit hefty. You want to make this setting client-side only especially if you are running the new experiemental +server-side off-heap cache since this feature depends on being able to use big direct buffers (You may have to keep +separate client-side and server-side config dirs). + +
+
+ Client Slowdown When Calling Admin Methods (flush, compact, etc.) + +This is a client issue fixed by HBASE-5073 in 0.90.6. +There was a ZooKeeper leak in the client and the client was getting pummeled by ZooKeeper events with each additional +invocation of the admin API. + +
+ +
+ Secure Client Cannot Connect ([Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]) + +There can be several causes that produce this symptom. + + +First, check that you have a valid Kerberos ticket. One is required in order to set up communication with a secure Apache HBase cluster. Examine the ticket currently in the credential cache, if any, by running the klist command line utility. If no ticket is listed, you must obtain a ticket by running the kinit command with either a keytab specified, or by interactively entering a password for the desired principal. + + +Then, consult the Java Security Guide troubleshooting section. The most common problem addressed there is resolved by setting javax.security.auth.useSubjectCredsOnly system property value to false. + + +Because of a change in the format in which MIT Kerberos writes its credentials cache, there is a bug in the Oracle JDK 6 Update 26 and earlier that causes Java to be unable to read the Kerberos credentials cache created by versions of MIT Kerberos 1.8.1 or higher. If you have this problematic combination of components in your environment, to work around this problem, first log in with kinit and then immediately refresh the credential cache with kinit -R. The refresh will rewrite the credential cache without the problematic formatting. + + +Finally, depending on your Kerberos configuration, you may need to install the Java Cryptography Extension, or JCE. Insure the JCE jars are on the classpath on both server and client systems. + + +You may also need to download the unlimited strength JCE policy files. Uncompress and extract the downloaded file, and install the policy jars into <java-home>/lib/security. + +
+ +
+ +
+ MapReduce +
+ You Think You're On The Cluster, But You're Actually Local + This following stacktrace happened using ImportTsv, but things like this + can happen on any job with a mis-configuration. + + WARN mapred.LocalJobRunner: job_local_0001 +java.lang.IllegalArgumentException: Can't read partitions file + at org.apache.hadoop.hbase.mapreduce.hadoopbackport.TotalOrderPartitioner.setConf(TotalOrderPartitioner.java:111) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:62) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117) + at org.apache.hadoop.mapred.MapTask$NewOutputCollector.<init>(MapTask.java:560) + at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:639) + at org.apache.hadoop.mapred.MapTask.run(MapTask.java:323) + at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:210) +Caused by: java.io.FileNotFoundException: File _partition.lst does not exist. + at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:383) + at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:251) + at org.apache.hadoop.fs.FileSystem.getLength(FileSystem.java:776) + at org.apache.hadoop.io.SequenceFile$Reader.<init>(SequenceFile.java:1424) + at org.apache.hadoop.io.SequenceFile$Reader.<init>(SequenceFile.java:1419) + at org.apache.hadoop.hbase.mapreduce.hadoopbackport.TotalOrderPartitioner.readPartitions(TotalOrderPartitioner.java:296) + + .. see the critical portion of the stack? It's... + + at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:210) + + LocalJobRunner means the job is running locally, not on the cluster. + + See + + http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/package-summary.html#classpath for more + information on HBase MapReduce jobs and classpaths. + +
+
+ +
+ NameNode + For more information on the NameNode, see . + +
+ HDFS Utilization of Tables and Regions + To determine how much space HBase is using on HDFS use the hadoop shell commands from the NameNode. For example... + hadoop fs -dus /hbase/ ...returns the summarized disk utilization for all HBase objects. + hadoop fs -dus /hbase/myTable ...returns the summarized disk utilization for the HBase table 'myTable'. + hadoop fs -du /hbase/myTable ...returns a list of the regions under the HBase table 'myTable' and their disk utilization. + For more information on HDFS shell commands, see the HDFS FileSystem Shell documentation. + +
+
+ Browsing HDFS for HBase Objects + Somtimes it will be necessary to explore the HBase objects that exist on HDFS. These objects could include the WALs (Write Ahead Logs), tables, regions, StoreFiles, etc. + The easiest way to do this is with the NameNode web application that runs on port 50070. The NameNode web application will provide links to the all the DataNodes in the cluster so that + they can be browsed seamlessly. + The HDFS directory structure of HBase tables in the cluster is... + +/hbase + /<Table> (Tables in the cluster) + /<Region> (Regions for the table) + /<ColumnFamiy> (ColumnFamilies for the Region for the table) + /<StoreFile> (StoreFiles for the ColumnFamily for the Regions for the table) + + + The HDFS directory structure of HBase WAL is.. + +/hbase + /.logs + /<RegionServer> (RegionServers) + /<HLog> (WAL HLog files for the RegionServer) + + + See the HDFS User Guide for other non-shell diagnostic + utilities like fsck. + +
+ Use Cases + Two common use-cases for querying HDFS for HBase objects is research the degree of uncompaction of a table. If there are a large number of StoreFiles for each ColumnFamily it could + indicate the need for a major compaction. Additionally, after a major compaction if the resulting StoreFile is "small" it could indicate the need for a reduction of ColumnFamilies for + the table. + +
+ +
+
+ +
+ Network +
+ Network Spikes + If you are seeing periodic network spikes you might want to check the compactionQueues to see if major + compactions are happening. + + See for more information on managing compactions. + +
+
+ Loopback IP + HBase expects the loopback IP Address to be 127.0.0.1. See the Getting Started section on . + +
+
+ Network Interfaces + Are all the network interfaces functioning correctly? Are you sure? See the Troubleshooting Case Study in . + +
+ +
+ +
+ RegionServer + For more information on the RegionServers, see . + +
+ Startup Errors +
+ Master Starts, But RegionServers Do Not + The Master believes the RegionServers have the IP of 127.0.0.1 - which is localhost and resolves to the master's own localhost. + + The RegionServers are erroneously informing the Master that their IP addresses are 127.0.0.1. + + Modify /etc/hosts on the region servers, from... + +# Do not remove the following line, or various programs +# that require network functionality will fail. +127.0.0.1 fully.qualified.regionservername regionservername localhost.localdomain localhost +::1 localhost6.localdomain6 localhost6 + + ... to (removing the master node's name from localhost)... + +# Do not remove the following line, or various programs +# that require network functionality will fail. +127.0.0.1 localhost.localdomain localhost +::1 localhost6.localdomain6 localhost6 + + +
+ +
+ Compression Link Errors + + Since compression algorithms such as LZO need to be installed and configured on each cluster this is a frequent source of startup error. If you see messages like this... + +11/02/20 01:32:15 ERROR lzo.GPLNativeCodeLoader: Could not load native gpl library +java.lang.UnsatisfiedLinkError: no gplcompression in java.library.path + at java.lang.ClassLoader.loadLibrary(ClassLoader.java:1734) + at java.lang.Runtime.loadLibrary0(Runtime.java:823) + at java.lang.System.loadLibrary(System.java:1028) + + .. then there is a path issue with the compression libraries. See the Configuration section on LZO compression configuration. + +
+
+
+ Runtime Errors + +
+ RegionServer Hanging + + Are you running an old JVM (< 1.6.0_u21?)? When you look at a thread dump, + does it look like threads are BLOCKED but no one holds the lock all are + blocked on? See HBASE 3622 Deadlock in HBaseServer (JVM bug?). + Adding -XX:+UseMembar to the HBase HBASE_OPTS in conf/hbase-env.sh + may fix it. + + Also, are you using ? These are discouraged because they can lock up the + RegionServers if not managed properly. + +
+
+ java.io.IOException...(Too many open files) + + If you see log messages like this... + +2010-09-13 01:24:17,336 WARN org.apache.hadoop.hdfs.server.datanode.DataNode: +Disk-related IOException in BlockReceiver constructor. Cause is java.io.IOException: Too many open files + at java.io.UnixFileSystem.createFileExclusively(Native Method) + at java.io.File.createNewFile(File.java:883) + + ... see the Getting Started section on ulimit and nproc configuration. + +
+
+ xceiverCount 258 exceeds the limit of concurrent xcievers 256 + + This typically shows up in the DataNode logs. + + + See the Getting Started section on xceivers configuration. + +
+
+ System instability, and the presence of "java.lang.OutOfMemoryError: unable to create new native thread in exceptions" HDFS DataNode logs or that of any system daemon + + See the Getting Started section on ulimit and nproc configuration. The default on recent Linux + distributions is 1024 - which is far too low for HBase. + +
+
+ DFS instability and/or RegionServer lease timeouts + + If you see warning messages like this... + +2009-02-24 10:01:33,516 WARN org.apache.hadoop.hbase.util.Sleeper: We slept xxx ms, ten times longer than scheduled: 10000 +2009-02-24 10:01:33,516 WARN org.apache.hadoop.hbase.util.Sleeper: We slept xxx ms, ten times longer than scheduled: 15000 +2009-02-24 10:01:36,472 WARN org.apache.hadoop.hbase.regionserver.HRegionServer: unable to report to master for xxx milliseconds - retrying + + ... or see full GC compactions then you may be experiencing full GC's. + +
+
+ "No live nodes contain current block" and/or YouAreDeadException + + These errors can happen either when running out of OS file handles or in periods of severe network problems where the nodes are unreachable. + + + See the Getting Started section on ulimit and nproc configuration and check your network. + +
+
+ ZooKeeper SessionExpired events + Master or RegionServers shutting down with messages like those in the logs: + +WARN org.apache.zookeeper.ClientCnxn: Exception +closing session 0x278bd16a96000f to sun.nio.ch.SelectionKeyImpl@355811ec +java.io.IOException: TIMED OUT + at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:906) +WARN org.apache.hadoop.hbase.util.Sleeper: We slept 79410ms, ten times longer than scheduled: 5000 +INFO org.apache.zookeeper.ClientCnxn: Attempting connection to server hostname/IP:PORT +INFO org.apache.zookeeper.ClientCnxn: Priming connection to java.nio.channels.SocketChannel[connected local=/IP:PORT remote=hostname/IP:PORT] +INFO org.apache.zookeeper.ClientCnxn: Server connection successful +WARN org.apache.zookeeper.ClientCnxn: Exception closing session 0x278bd16a96000d to sun.nio.ch.SelectionKeyImpl@3544d65e +java.io.IOException: Session Expired + at org.apache.zookeeper.ClientCnxn$SendThread.readConnectResult(ClientCnxn.java:589) + at org.apache.zookeeper.ClientCnxn$SendThread.doIO(ClientCnxn.java:709) + at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:945) +ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: ZooKeeper session expired + + + The JVM is doing a long running garbage collecting which is pausing every threads (aka "stop the world"). + Since the RegionServer's local ZooKeeper client cannot send heartbeats, the session times out. + By design, we shut down any node that isn't able to contact the ZooKeeper ensemble after getting a timeout so that it stops serving data that may already be assigned elsewhere. + + + + Make sure you give plenty of RAM (in hbase-env.sh), the default of 1GB won't be able to sustain long running imports. + Make sure you don't swap, the JVM never behaves well under swapping. + Make sure you are not CPU starving the RegionServer thread. For example, if you are running a MapReduce job using 6 CPU-intensive tasks on a machine with 4 cores, you are probably starving the RegionServer enough to create longer garbage collection pauses. + Increase the ZooKeeper session timeout + + If you wish to increase the session timeout, add the following to your hbase-site.xml to increase the timeout from the default of 60 seconds to 120 seconds. + +<property> + <name>zookeeper.session.timeout</name> + <value>1200000</value> +</property> +<property> + <name>hbase.zookeeper.property.tickTime</name> + <value>6000</value> +</property> + + + + Be aware that setting a higher timeout means that the regions served by a failed RegionServer will take at least + that amount of time to be transfered to another RegionServer. For a production system serving live requests, we would instead + recommend setting it lower than 1 minute and over-provision your cluster in order the lower the memory load on each machines (hence having + less garbage to collect per machine). + + + If this is happening during an upload which only happens once (like initially loading all your data into HBase), consider bulk loading. + + See for other general information about ZooKeeper troubleshooting. +
+
+ NotServingRegionException + This exception is "normal" when found in the RegionServer logs at DEBUG level. This exception is returned back to the client + and then the client goes back to .META. to find the new location of the moved region. + However, if the NotServingRegionException is logged ERROR, then the client ran out of retries and something probably wrong. +
+
+ Regions listed by domain name, then IP + + Fix your DNS. In versions of Apache HBase before 0.92.x, reverse DNS needs to give same answer + as forward lookup. See HBASE 3431 + RegionServer is not using the name given it by the master; double entry in master listing of servers for gorey details. + +
+
+ Logs flooded with '2011-01-10 12:40:48,407 INFO org.apache.hadoop.io.compress.CodecPool: Got + brand-new compressor' messages + We are not using the native versions of compression + libraries. See HBASE-1900 Put back native support when hadoop 0.21 is released. + Copy the native libs from hadoop under hbase lib dir or + symlink them into place and the message should go away. + +
+
+ Server handler X on 60020 caught: java.nio.channels.ClosedChannelException + + If you see this type of message it means that the region server was trying to read/send data from/to a client but + it already went away. Typical causes for this are if the client was killed (you see a storm of messages like this when a MapReduce + job is killed or fails) or if the client receives a SocketTimeoutException. It's harmless, but you should consider digging in + a bit more if you aren't doing something to trigger them. + +
+ +
+
+ Shutdown Errors + +
+ +
+ +
+ Master + For more information on the Master, see . + +
+ Startup Errors +
+ Master says that you need to run the hbase migrations script + Upon running that, the hbase migrations script says no files in root directory. + HBase expects the root directory to either not exist, or to have already been initialized by hbase running a previous time. If you create a new directory for HBase using Hadoop DFS, this error will occur. + Make sure the HBase root directory does not currently exist or has been initialized by a previous run of HBase. Sure fire solution is to just use Hadoop dfs to delete the HBase root and let HBase create and initialize the directory itself. + +
+
+ Packet len6080218 is out of range! + If you have many regions on your cluster and you see an error + like that reported above in this sections title in your logs, see + HBASE-4246 Cluster with too many regions cannot withstand some master failover scenarios. +
+ +
+
+ Shutdown Errors + +
+ +
+ +
+ ZooKeeper +
+ Startup Errors +
+ Could not find my address: xyz in list of ZooKeeper quorum servers + A ZooKeeper server wasn't able to start, throws that error. xyz is the name of your server. + This is a name lookup problem. HBase tries to start a ZooKeeper server on some machine but that machine isn't able to find itself in the hbase.zookeeper.quorum configuration. + + Use the hostname presented in the error message instead of the value you used. If you have a DNS server, you can set hbase.zookeeper.dns.interface and hbase.zookeeper.dns.nameserver in hbase-site.xml to make sure it resolves to the correct FQDN. + +
+ +
+
+ ZooKeeper, The Cluster Canary + ZooKeeper is the cluster's "canary in the mineshaft". It'll be the first to notice issues if any so making sure its happy is the short-cut to a humming cluster. + + + See the ZooKeeper Operating Environment Troubleshooting page. It has suggestions and tools for checking disk and networking performance; i.e. the operating environment your ZooKeeper and HBase are running in. + + Additionally, the utility may help investigate ZooKeeper issues. + +
+ +
+ +
+ Amazon EC2 +
+ ZooKeeper does not seem to work on Amazon EC2 + HBase does not start when deployed as Amazon EC2 instances. Exceptions like the below appear in the Master and/or RegionServer logs: + + 2009-10-19 11:52:27,030 INFO org.apache.zookeeper.ClientCnxn: Attempting + connection to server ec2-174-129-15-236.compute-1.amazonaws.com/10.244.9.171:2181 + 2009-10-19 11:52:27,032 WARN org.apache.zookeeper.ClientCnxn: Exception + closing session 0x0 to sun.nio.ch.SelectionKeyImpl@656dc861 + java.net.ConnectException: Connection refused + + + Security group policy is blocking the ZooKeeper port on a public address. + Use the internal EC2 host names when configuring the ZooKeeper quorum peer list. + +
+
+ Instability on Amazon EC2 + Questions on HBase and Amazon EC2 come up frequently on the HBase dist-list. Search for old threads using Search Hadoop + +
+
+ Remote Java Connection into EC2 Cluster Not Working + + See Andrew's answer here, up on the user list: Remote Java client connection into EC2 instance. + +
+ +
+ +
+ HBase and Hadoop version issues +
+ <code>NoClassDefFoundError</code> when trying to run 0.90.x on hadoop-0.20.205.x (or hadoop-1.0.x) + Apache HBase 0.90.x does not ship with hadoop-0.20.205.x, etc. To make it run, you need to replace the hadoop + jars that Apache HBase shipped with in its lib directory with those of the Hadoop you want to + run HBase on. If even after replacing Hadoop jars you get the below exception: + +sv4r6s38: Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/configuration/Configuration +sv4r6s38: at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.<init>(DefaultMetricsSystem.java:37) +sv4r6s38: at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.<clinit>(DefaultMetricsSystem.java:34) +sv4r6s38: at org.apache.hadoop.security.UgiInstrumentation.create(UgiInstrumentation.java:51) +sv4r6s38: at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:209) +sv4r6s38: at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:177) +sv4r6s38: at org.apache.hadoop.security.UserGroupInformation.isSecurityEnabled(UserGroupInformation.java:229) +sv4r6s38: at org.apache.hadoop.security.KerberosName.<clinit>(KerberosName.java:83) +sv4r6s38: at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:202) +sv4r6s38: at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:177) + +you need to copy under hbase/lib, the commons-configuration-X.jar you find +in your Hadoop's lib directory. That should fix the above complaint. + +
+ +
+ ...cannot communicate with client version... +If you see something like the following in your logs +... +2012-09-24 10:20:52,168 FATAL org.apache.hadoop.hbase.master.HMaster: Unhandled exception. Starting shutdown. +org.apache.hadoop.ipc.RemoteException: Server IPC version 7 cannot communicate with client version 4 +... +...are you trying to talk to an Hadoop 2.0.x from an HBase that has an Hadoop 1.0.x client? +Use the HBase built against Hadoop 2.0 or rebuild your HBase passing the -Dhadoop.profile=2.0 +attribute to Maven (See for more). + + +
+
+ +
+ Case Studies + For Performance and Troubleshooting Case Studies, see . + +
+ + diff --git a/src/docbkx/upgrading.xml b/src/docbkx/upgrading.xml new file mode 100644 index 0000000..d1dcdd8 --- /dev/null +++ b/src/docbkx/upgrading.xml @@ -0,0 +1,221 @@ + + + + Upgrading + You cannot skip major verisons upgrading. If you are upgrading from + version 0.20.x to 0.92.x, you must first go from 0.20.x to 0.90.x and then go + from 0.90.x to 0.92.x. + + Review , in particular the section on Hadoop version. + +
+ Upgrading from 0.94.x to 0.96.x + The Singularity + You will have to stop your old 0.94 cluster completely to upgrade. If you are replicating + between clusters, both clusters will have to go down to upgrade. Make sure it is a clean shutdown + so there are no WAL files laying around (TODO: Can 0.96 read 0.94 WAL files?). Make sure + zookeeper is cleared of state. All clients must be upgraded to 0.96 too. + + The API has changed in a few areas; in particular how you use coprocessors (TODO: MapReduce too?) + + TODO: Write about 3.4 zk ensemble and multi support +
+
+ Upgrading from 0.92.x to 0.94.x + 0.92 and 0.94 are interface compatible. You can do a rolling upgrade between these versions. + +
+
+ Upgrading from 0.90.x to 0.92.x + Upgrade Guide +You will find that 0.92.0 runs a little differently to 0.90.x releases. Here are a few things to watch out for upgrading from 0.90.x to 0.92.0. +tl;dr + +If you've not patience, here are the important things to know upgrading. + +Once you upgrade, you can’t go back. + + +MSLAB is on by default. Watch that heap usage if you have a lot of regions. + + +Distributed splitting is on by defaul. It should make region server failover faster. + + +There’s a separate tarball for security. + + +If -XX:MaxDirectMemorySize is set in your hbase-env.sh, it’s going to enable the experimental off-heap cache (You may not want this). + + + + + + +
+You can’t go back! + +To move to 0.92.0, all you need to do is shutdown your cluster, replace your hbase 0.90.x with hbase 0.92.0 binaries (be sure you clear out all 0.90.x instances) and restart (You cannot do a rolling restart from 0.90.x to 0.92.x -- you must restart). +On startup, the .META. table content is rewritten removing the table schema from the info:regioninfo column. +Also, any flushes done post first startup will write out data in the new 0.92.0 file format, HFile V2. +This means you cannot go back to 0.90.x once you’ve started HBase 0.92.0 over your HBase data directory. + +
+ +
+MSLAB is ON by default + +In 0.92.0, the hbase.hregion.memstore.mslab.enabled flag is set to true +(See ). In 0.90.x it was false. When it is enabled, memstores will step allocate memory in MSLAB 2MB chunks even if the +memstore has zero or just a few small elements. This is fine usually but if you had lots of regions per regionserver in a 0.90.x cluster (and MSLAB was off), +you may find yourself OOME'ing on upgrade because the thousands of regions * number of column families * 2MB MSLAB (at a minimum) +puts your heap over the top. Set hbase.hregion.memstore.mslab.enabled to +false or set the MSLAB size down from 2MB by setting hbase.hregion.memstore.mslab.chunksize to something less. + +
+ +
Distributed splitting is on by default + +Previous, WAL logs on crash were split by the Master alone. In 0.92.0, log splitting is done by the cluster (See See “HBASE-1364 [performance] Distributed splitting of regionserver commit logs”). This should cut down significantly on the amount of time it takes splitting logs and getting regions back online again. + +
+ +
Memory accounting is different now + +In 0.92.0, indices and bloom filters take up residence in the same LRU used caching blocks that come from the filesystem. +In 0.90.x, the HFile v1 indices lived outside of the LRU so they took up space even if the index was on a ‘cold’ file, one that wasn’t being actively used. With the indices now in the LRU, you may find you +have less space for block caching. Adjust your block cache accordingly. See the for more detail. +The block size default size has been changed in 0.92.0 from 0.2 (20 percent of heap) to 0.25. + +
+ + +
On the Hadoop version to use + +Run 0.92.0 on Hadoop 1.0.x (or CDH3u3 when it ships). The performance benefits are worth making the move. Otherwise, our Hadoop prescription is as it has been; you need an Hadoop that supports a working sync. See . + + +If running on Hadoop 1.0.x (or CDH3u3), enable local read. See Practical Caching presentation for ruminations on the performance benefits ‘going local’ (and for how to enable local reads). + +
+
HBase 0.92.0 ships with ZooKeeper 3.4.2 + +If you can, upgrade your zookeeper. If you can’t, 3.4.2 clients should work against 3.3.X ensembles (HBase makes use of 3.4.2 API). + +
+
+Online alter is off by default + +In 0.92.0, we’ve added an experimental online schema alter facility (See ). Its off by default. Enable it at your own risk. Online alter and splitting tables do not play well together so be sure your cluster quiescent using this feature (for now). + +
+
+WebUI + +The webui has had a few additions made in 0.92.0. It now shows a list of the regions currently transitioning, recent compactions/flushes, and a process list of running processes (usually empty if all is well and requests are being handled promptly). Other additions including requests by region, a debugging servlet dump, etc. + +
+
+Security tarball + +We now ship with two tarballs; secure and insecure HBase. Documentation on how to setup a secure HBase is on the way. + +
+ +
Experimental off-heap cache + + +A new cache was contributed to 0.92.0 to act as a solution between using the “on-heap” cache which is the current LRU cache the region servers have and the operating system cache which is out of our control. +To enable, set “-XX:MaxDirectMemorySize” in hbase-env.sh to the value for maximum direct memory size and specify hbase.offheapcache.percentage in hbase-site.xml with the percentage that you want to dedicate to off-heap cache. This should only be set for servers and not for clients. Use at your own risk. +See this blog post for additional information on this new experimental feature: http://www.cloudera.com/blog/2012/01/caching-in-hbase-slabcache/ + +
+ +
Changes in HBase replication + +0.92.0 adds two new features: multi-slave and multi-master replication. The way to enable this is the same as adding a new peer, so in order to have multi-master you would just run add_peer for each cluster that acts as a master to the other slave clusters. Collisions are handled at the timestamp level which may or may not be what you want, this needs to be evaluated on a per use case basis. Replication is still experimental in 0.92 and is disabled by default, run it at your own risk. + +
+ + +
RegionServer now aborts if OOME + +If an OOME, we now have the JVM kill -9 the regionserver process so it goes down fast. Previous, a RegionServer might stick around after incurring an OOME limping along in some wounded state. To disable this facility, and recommend you leave it in place, you’d need to edit the bin/hbase file. Look for the addition of the -XX:OnOutOfMemoryError="kill -9 %p" arguments (See [HBASE-4769] - ‘Abort RegionServer Immediately on OOME’) + +
+ + +
HFile V2 and the “Bigger, Fewer” Tendency + +0.92.0 stores data in a new format, . As HBase runs, it will move all your data from HFile v1 to HFile v2 format. This auto-migration will run in the background as flushes and compactions run. +HFile V2 allows HBase run with larger regions/files. In fact, we encourage that all HBasers going forward tend toward Facebook axiom #1, run with larger, fewer regions. +If you have lots of regions now -- more than 100s per host -- you should look into setting your region size up after you move to 0.92.0 (In 0.92.0, default size is now 1G, up from 256M), and then running online merge tool (See “HBASE-1621 merge tool should work on online cluster, but disabled table”). + +
+
+
+ Upgrading to HBase 0.90.x from 0.20.x or 0.89.x + This version of 0.90.x HBase can be started on data written by + HBase 0.20.x or HBase 0.89.x. There is no need of a migration step. + HBase 0.89.x and 0.90.x does write out the name of region directories + differently -- it names them with a md5 hash of the region name rather + than a jenkins hash -- so this means that once started, there is no + going back to HBase 0.20.x. + + + Be sure to remove the hbase-default.xml from + your conf + directory on upgrade. A 0.20.x version of this file will have + sub-optimal configurations for 0.90.x HBase. The + hbase-default.xml file is now bundled into the + HBase jar and read from there. If you would like to review + the content of this file, see it in the src tree at + src/main/resources/hbase-default.xml or + see . + + + Finally, if upgrading from 0.20.x, check your + .META. schema in the shell. In the past we would + recommend that users run with a 16kb + MEMSTORE_FLUSHSIZE. + Run hbase> scan '-ROOT-' in the shell. This will output + the current .META. schema. Check + MEMSTORE_FLUSHSIZE size. Is it 16kb (16384)? If so, you will + need to change this (The 'normal'/default value is 64MB (67108864)). + Run the script bin/set_meta_memstore_size.rb. + This will make the necessary edit to your .META. schema. + Failure to run this change will make for a slow cluster + + See HBASE-3499 Users upgrading to 0.90.0 need to have their .META. table updated with the right MEMSTORE_SIZE + + + . + + +
+ diff --git a/src/docbkx/zookeeper.xml b/src/docbkx/zookeeper.xml new file mode 100644 index 0000000..d6301e2 --- /dev/null +++ b/src/docbkx/zookeeper.xml @@ -0,0 +1,595 @@ + + + + + ZooKeeper<indexterm> + <primary>ZooKeeper</primary> + </indexterm> + + A distributed Apache HBase (TM) installation depends on a running ZooKeeper cluster. + All participating nodes and clients need to be able to access the + running ZooKeeper ensemble. Apache HBase by default manages a ZooKeeper + "cluster" for you. It will start and stop the ZooKeeper ensemble + as part of the HBase start/stop process. You can also manage the + ZooKeeper ensemble independent of HBase and just point HBase at + the cluster it should use. To toggle HBase management of + ZooKeeper, use the HBASE_MANAGES_ZK variable in + conf/hbase-env.sh. This variable, which + defaults to true, tells HBase whether to + start/stop the ZooKeeper ensemble servers as part of HBase + start/stop. + + When HBase manages the ZooKeeper ensemble, you can specify + ZooKeeper configuration using its native + zoo.cfg file, or, the easier option is to + just specify ZooKeeper options directly in + conf/hbase-site.xml. A ZooKeeper + configuration option can be set as a property in the HBase + hbase-site.xml XML configuration file by + prefacing the ZooKeeper option name with + hbase.zookeeper.property. For example, the + clientPort setting in ZooKeeper can be changed + by setting the + hbase.zookeeper.property.clientPort property. + For all default values used by HBase, including ZooKeeper + configuration, see . Look for the + hbase.zookeeper.property prefix + For the full list of ZooKeeper configurations, see + ZooKeeper's zoo.cfg. HBase does not ship + with a zoo.cfg so you will need to browse + the conf directory in an appropriate + ZooKeeper download. + + + You must at least list the ensemble servers in + hbase-site.xml using the + hbase.zookeeper.quorum property. This property + defaults to a single ensemble member at + localhost which is not suitable for a fully + distributed HBase. (It binds to the local machine only and remote + clients will not be able to connect). + How many ZooKeepers should I run? + + You can run a ZooKeeper ensemble that comprises 1 node + only but in production it is recommended that you run a + ZooKeeper ensemble of 3, 5 or 7 machines; the more members an + ensemble has, the more tolerant the ensemble is of host + failures. Also, run an odd number of machines. In ZooKeeper, + an even number of peers is supported, but it is normally not used + because an even sized ensemble requires, proportionally, more peers + to form a quorum than an odd sized ensemble requires. For example, an + ensemble with 4 peers requires 3 to form a quorum, while an ensemble with + 5 also requires 3 to form a quorum. Thus, an ensemble of 5 allows 2 peers to + fail, and thus is more fault tolerant than the ensemble of 4, which allows + only 1 down peer. + + Give each ZooKeeper server around 1GB of RAM, and if possible, its own + dedicated disk (A dedicated disk is the best thing you can do + to ensure a performant ZooKeeper ensemble). For very heavily + loaded clusters, run ZooKeeper servers on separate machines + from RegionServers (DataNodes and TaskTrackers). + + + For example, to have HBase manage a ZooKeeper quorum on + nodes rs{1,2,3,4,5}.example.com, bound to + port 2222 (the default is 2181) ensure + HBASE_MANAGE_ZK is commented out or set to + true in conf/hbase-env.sh + and then edit conf/hbase-site.xml and set + hbase.zookeeper.property.clientPort and + hbase.zookeeper.quorum. You should also set + hbase.zookeeper.property.dataDir to other than + the default as the default has ZooKeeper persist data under + /tmp which is often cleared on system + restart. In the example below we have ZooKeeper persist to + /user/local/zookeeper. + <configuration> + ... + <property> + <name>hbase.zookeeper.property.clientPort</name> + <value>2222</value> + <description>Property from ZooKeeper's config zoo.cfg. + The port at which the clients will connect. + </description> + </property> + <property> + <name>hbase.zookeeper.quorum</name> + <value>rs1.example.com,rs2.example.com,rs3.example.com,rs4.example.com,rs5.example.com</value> + <description>Comma separated list of servers in the ZooKeeper Quorum. + For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com". + By default this is set to localhost for local and pseudo-distributed modes + of operation. For a fully-distributed setup, this should be set to a full + list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh + this is the list of servers which we will start/stop ZooKeeper on. + </description> + </property> + <property> + <name>hbase.zookeeper.property.dataDir</name> + <value>/usr/local/zookeeper</value> + <description>Property from ZooKeeper's config zoo.cfg. + The directory where the snapshot is stored. + </description> + </property> + ... + </configuration> + + ZooKeeper Maintenance + Be sure to set up the data dir cleaner described under + Zookeeper Maintenance else you could + have 'interesting' problems a couple of months in; i.e. zookeeper could start + dropping sessions if it has to run through a directory of hundreds of thousands of + logs which is wont to do around leader reelection time -- a process rare but run on + occasion whether because a machine is dropped or happens to hiccup. + + +
+ Using existing ZooKeeper ensemble + + To point HBase at an existing ZooKeeper cluster, one that + is not managed by HBase, set HBASE_MANAGES_ZK + in conf/hbase-env.sh to false + + ... + # Tell HBase whether it should manage its own instance of Zookeeper or not. + export HBASE_MANAGES_ZK=false Next set ensemble locations + and client port, if non-standard, in + hbase-site.xml, or add a suitably + configured zoo.cfg to HBase's + CLASSPATH. HBase will prefer the + configuration found in zoo.cfg over any + settings in hbase-site.xml. + + When HBase manages ZooKeeper, it will start/stop the + ZooKeeper servers as a part of the regular start/stop scripts. + If you would like to run ZooKeeper yourself, independent of + HBase start/stop, you would do the following + + +${HBASE_HOME}/bin/hbase-daemons.sh {start,stop} zookeeper + + + Note that you can use HBase in this manner to spin up a + ZooKeeper cluster, unrelated to HBase. Just make sure to set + HBASE_MANAGES_ZK to false + if you want it to stay up across HBase restarts so that when + HBase shuts down, it doesn't take ZooKeeper down with it. + + For more information about running a distinct ZooKeeper + cluster, see the ZooKeeper Getting + Started Guide. Additionally, see the ZooKeeper Wiki or the + ZooKeeper documentation + for more information on ZooKeeper sizing. + +
+ + +
+ SASL Authentication with ZooKeeper + Newer releases of Apache HBase (>= 0.92) will + support connecting to a ZooKeeper Quorum that supports + SASL authentication (which is available in Zookeeper + versions 3.4.0 or later). + + This describes how to set up HBase to mutually + authenticate with a ZooKeeper Quorum. ZooKeeper/HBase + mutual authentication (HBASE-2418) + is required as part of a complete secure HBase configuration + (HBASE-3025). + + For simplicity of explication, this section ignores + additional configuration required (Secure HDFS and Coprocessor + configuration). It's recommended to begin with an + HBase-managed Zookeeper configuration (as opposed to a + standalone Zookeeper quorum) for ease of learning. + + +
Operating System Prerequisites
+ + + You need to have a working Kerberos KDC setup. For + each $HOST that will run a ZooKeeper + server, you should have a principle + zookeeper/$HOST. For each such host, + add a service key (using the kadmin or + kadmin.local tool's ktadd + command) for zookeeper/$HOST and copy + this file to $HOST, and make it + readable only to the user that will run zookeeper on + $HOST. Note the location of this file, + which we will use below as + $PATH_TO_ZOOKEEPER_KEYTAB. + + + + Similarly, for each $HOST that will run + an HBase server (master or regionserver), you should + have a principle: hbase/$HOST. For each + host, add a keytab file called + hbase.keytab containing a service + key for hbase/$HOST, copy this file to + $HOST, and make it readable only to the + user that will run an HBase service on + $HOST. Note the location of this file, + which we will use below as + $PATH_TO_HBASE_KEYTAB. + + + + Each user who will be an HBase client should also be + given a Kerberos principal. This principal should + usually have a password assigned to it (as opposed to, + as with the HBase servers, a keytab file) which only + this user knows. The client's principal's + maxrenewlife should be set so that it can + be renewed enough so that the user can complete their + HBase client processes. For example, if a user runs a + long-running HBase client process that takes at most 3 + days, we might create this user's principal within + kadmin with: addprinc -maxrenewlife + 3days. The Zookeeper client and server + libraries manage their own ticket refreshment by + running threads that wake up periodically to do the + refreshment. + + + On each host that will run an HBase client + (e.g. hbase shell), add the following + file to the HBase home directory's conf + directory: + + + Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=false + useTicketCache=true; + }; + + + We'll refer to this JAAS configuration file as + $CLIENT_CONF below. + +
+ HBase-managed Zookeeper Configuration + + On each node that will run a zookeeper, a + master, or a regionserver, create a JAAS + configuration file in the conf directory of the node's + HBASE_HOME directory that looks like the + following: + + + Server { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="$PATH_TO_ZOOKEEPER_KEYTAB" + storeKey=true + useTicketCache=false + principal="zookeeper/$HOST"; + }; + Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + useTicketCache=false + keyTab="$PATH_TO_HBASE_KEYTAB" + principal="hbase/$HOST"; + }; + + + where the $PATH_TO_HBASE_KEYTAB and + $PATH_TO_ZOOKEEPER_KEYTAB files are what + you created above, and $HOST is the hostname for that + node. + + The Server section will be used by + the Zookeeper quorum server, while the + Client section will be used by the HBase + master and regionservers. The path to this file should + be substituted for the text $HBASE_SERVER_CONF + in the hbase-env.sh + listing below. + + + The path to this file should be substituted for the + text $CLIENT_CONF in the + hbase-env.sh listing below. + + + Modify your hbase-env.sh to include the + following: + + + export HBASE_OPTS="-Djava.security.auth.login.config=$CLIENT_CONF" + export HBASE_MANAGES_ZK=true + export HBASE_ZOOKEEPER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF" + export HBASE_MASTER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF" + export HBASE_REGIONSERVER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF" + + + where $HBASE_SERVER_CONF and + $CLIENT_CONF are the full paths to the + JAAS configuration files created above. + + Modify your hbase-site.xml on each node + that will run zookeeper, master or regionserver to contain: + + + + hbase.zookeeper.quorum + $ZK_NODES + + + hbase.cluster.distributed + true + + + hbase.zookeeper.property.authProvider.1 + org.apache.zookeeper.server.auth.SASLAuthenticationProvider + + + hbase.zookeeper.property.kerberos.removeHostFromPrincipal + true + + + hbase.zookeeper.property.kerberos.removeRealmFromPrincipal + true + + + ]]> + + where $ZK_NODES is the + comma-separated list of hostnames of the Zookeeper + Quorum hosts. + + Start your hbase cluster by running one or more + of the following set of commands on the appropriate + hosts: + + + + bin/hbase zookeeper start + bin/hbase master start + bin/hbase regionserver start + + +
+ +
External Zookeeper Configuration + Add a JAAS configuration file that looks like: + + + Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + useTicketCache=false + keyTab="$PATH_TO_HBASE_KEYTAB" + principal="hbase/$HOST"; + }; + + + where the $PATH_TO_HBASE_KEYTAB is the keytab + created above for HBase services to run on this host, and $HOST is the + hostname for that node. Put this in the HBase home's + configuration directory. We'll refer to this file's + full pathname as $HBASE_SERVER_CONF below. + + Modify your hbase-env.sh to include the following: + + + export HBASE_OPTS="-Djava.security.auth.login.config=$CLIENT_CONF" + export HBASE_MANAGES_ZK=false + export HBASE_MASTER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF" + export HBASE_REGIONSERVER_OPTS="-Djava.security.auth.login.config=$HBASE_SERVER_CONF" + + + + Modify your hbase-site.xml on each node + that will run a master or regionserver to contain: + + + + hbase.zookeeper.quorum + $ZK_NODES + + + hbase.cluster.distributed + true + + + ]]> + + + where $ZK_NODES is the + comma-separated list of hostnames of the Zookeeper + Quorum hosts. + + + Add a zoo.cfg for each Zookeeper Quorum host containing: + + authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider + kerberos.removeHostFromPrincipal=true + kerberos.removeRealmFromPrincipal=true + + + Also on each of these hosts, create a JAAS configuration file containing: + + + Server { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="$PATH_TO_ZOOKEEPER_KEYTAB" + storeKey=true + useTicketCache=false + principal="zookeeper/$HOST"; + }; + + + where $HOST is the hostname of each + Quorum host. We will refer to the full pathname of + this file as $ZK_SERVER_CONF below. + + + + + Start your Zookeepers on each Zookeeper Quorum host with: + + + SERVER_JVMFLAGS="-Djava.security.auth.login.config=$ZK_SERVER_CONF" bin/zkServer start + + + + + + Start your HBase cluster by running one or more of the following set of commands on the appropriate nodes: + + + + bin/hbase master start + bin/hbase regionserver start + + + +
+ +
+ Zookeeper Server Authentication Log Output + If the configuration above is successful, + you should see something similar to the following in + your Zookeeper server logs: + +11/12/05 22:43:39 INFO zookeeper.Login: successfully logged in. +11/12/05 22:43:39 INFO server.NIOServerCnxnFactory: binding to port 0.0.0.0/0.0.0.0:2181 +11/12/05 22:43:39 INFO zookeeper.Login: TGT refresh thread started. +11/12/05 22:43:39 INFO zookeeper.Login: TGT valid starting at: Mon Dec 05 22:43:39 UTC 2011 +11/12/05 22:43:39 INFO zookeeper.Login: TGT expires: Tue Dec 06 22:43:39 UTC 2011 +11/12/05 22:43:39 INFO zookeeper.Login: TGT refresh sleeping until: Tue Dec 06 18:36:42 UTC 2011 +.. +11/12/05 22:43:59 INFO auth.SaslServerCallbackHandler: + Successfully authenticated client: authenticationID=hbase/ip-10-166-175-249.us-west-1.compute.internal@HADOOP.LOCALDOMAIN; + authorizationID=hbase/ip-10-166-175-249.us-west-1.compute.internal@HADOOP.LOCALDOMAIN. +11/12/05 22:43:59 INFO auth.SaslServerCallbackHandler: Setting authorizedID: hbase +11/12/05 22:43:59 INFO server.ZooKeeperServer: adding SASL authorization for authorizationID: hbase + + + + +
+ +
+ Zookeeper Client Authentication Log Output + On the Zookeeper client side (HBase master or regionserver), + you should see something similar to the following: + + +11/12/05 22:43:59 INFO zookeeper.ZooKeeper: Initiating client connection, connectString=ip-10-166-175-249.us-west-1.compute.internal:2181 sessionTimeout=180000 watcher=master:60000 +11/12/05 22:43:59 INFO zookeeper.ClientCnxn: Opening socket connection to server /10.166.175.249:2181 +11/12/05 22:43:59 INFO zookeeper.RecoverableZooKeeper: The identifier of this process is 14851@ip-10-166-175-249 +11/12/05 22:43:59 INFO zookeeper.Login: successfully logged in. +11/12/05 22:43:59 INFO client.ZooKeeperSaslClient: Client will use GSSAPI as SASL mechanism. +11/12/05 22:43:59 INFO zookeeper.Login: TGT refresh thread started. +11/12/05 22:43:59 INFO zookeeper.ClientCnxn: Socket connection established to ip-10-166-175-249.us-west-1.compute.internal/10.166.175.249:2181, initiating session +11/12/05 22:43:59 INFO zookeeper.Login: TGT valid starting at: Mon Dec 05 22:43:59 UTC 2011 +11/12/05 22:43:59 INFO zookeeper.Login: TGT expires: Tue Dec 06 22:43:59 UTC 2011 +11/12/05 22:43:59 INFO zookeeper.Login: TGT refresh sleeping until: Tue Dec 06 18:30:37 UTC 2011 +11/12/05 22:43:59 INFO zookeeper.ClientCnxn: Session establishment complete on server ip-10-166-175-249.us-west-1.compute.internal/10.166.175.249:2181, sessionid = 0x134106594320000, negotiated timeout = 180000 + + +
+ +
+ Configuration from Scratch + + This has been tested on the current standard Amazon + Linux AMI. First setup KDC and principals as + described above. Next checkout code and run a sanity + check. + + + git clone git://git.apache.org/hbase.git + cd hbase + mvn clean test -Dtest=TestZooKeeperACL + + + Then configure HBase as described above. + Manually edit target/cached_classpath.txt (see below).. + + + bin/hbase zookeeper & + bin/hbase master & + bin/hbase regionserver & + +
+ + +
+ Future improvements + +
Fix target/cached_classpath.txt + + You must override the standard hadoop-core jar file from the + target/cached_classpath.txt + file with the version containing the HADOOP-7070 fix. You can use the following script to do this: + + + echo `find ~/.m2 -name "*hadoop-core*7070*SNAPSHOT.jar"` ':' `cat target/cached_classpath.txt` | sed 's/ //g' > target/tmp.txt + mv target/tmp.txt target/cached_classpath.txt + + + + +
+ +
+ Set JAAS configuration + programmatically + + + This would avoid the need for a separate Hadoop jar + that fixes HADOOP-7070. +
+ +
+ Elimination of + <code>kerberos.removeHostFromPrincipal</code> and + <code>kerberos.removeRealmFromPrincipal</code> +
+ +
+ + +
+ + + + + diff --git a/src/examples/README.txt b/src/examples/README.txt new file mode 100644 index 0000000..009c3d8 --- /dev/null +++ b/src/examples/README.txt @@ -0,0 +1,11 @@ +Example code. + +* src/examples/thrift + Examples for interacting with HBase via Thrift from C++, PHP, Python and Ruby. +* org.apache.hadoop.hbase.mapreduce.SampleUploader + Demonstrates uploading data from text files (presumably stored in HDFS) to HBase. +* org.apache.hadoop.hbase.mapreduce.IndexBuilder + Demonstrates map/reduce with a table as the source and other tables as the sink. + +As of 0.20 there is no ant target for building the examples. You can easily build +the Java examples by copying them to the right location in the main source hierarchy. \ No newline at end of file diff --git a/src/examples/healthcheck/healthcheck.sh b/src/examples/healthcheck/healthcheck.sh new file mode 100644 index 0000000..5846360 --- /dev/null +++ b/src/examples/healthcheck/healthcheck.sh @@ -0,0 +1,84 @@ +#!/bin/bash + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # This is an example script for checking health of a node ( master or region server). + # The health chore script should essentially output an message containing "ERROR" at an undesirable + # outcome of the checks in the script. + +err=0; + +function check_disks { + +for m in `awk '$3~/ext3/ {printf" %s ",$2}' /etc/fstab` ; do + fsdev="" + fsdev=`awk -v m=$m '$2==m {print $1}' /proc/mounts`; + if [ -z "$fsdev" ] ; then + msg_="$msg_ $m(u)" + else + msg_="$msg_`awk -v m=$m '$2==m { if ( $4 ~ /^ro,/ ) {printf"%s(ro)",$2 } ; }' /proc/mounts`" + fi + done + + if [ -z "$msg_" ] ; then + echo "disks ok" ; exit 0 + else + echo "$msg_" ; exit 2 + fi + +} + +function check_link { + /usr/bin/snmpwalk -t 5 -Oe -Oq -Os -v 1 -c public localhost if | \ + awk ' { + split($1,a,".") ; + if ( a[1] == "ifIndex" ) { ifIndex[a[2]] = $2 } + if ( a[1] == "ifDescr" ) { ifDescr[a[2]] = $2 } + if ( a[1] == "ifType" ) { ifType[a[2]] = $2 } + if ( a[1] == "ifSpeed" ) { ifSpeed[a[2]] = $2 } + if ( a[1] == "ifAdminStatus" ) { ifAdminStatus[a[2]] = $2 } + if ( a[1] == "ifOperStatus" ) { ifOperStatus[a[2]] = $2 } + } + END { + up=0; + for (i in ifIndex ) { + if ( ifType[i] == 6 && ifAdminStatus[i] == 1 && ifOperStatus[i] == 1 && ifSpeed[i] == 1000000000 ) { + up=i; + } + } + if ( up == 0 ) { print "check link" ; exit 2 } + else { print ifDescr[up],"ok" } + }' + exit $? ; +} + +for check in disks link ; do + msg=`check_${check}` ; + if [ $? -eq 0 ] ; then + ok_msg="$ok_msg$msg," + else + err_msg="$err_msg$msg," + fi +done + +if [ ! -z "$err_msg" ] ; then + echo -n "ERROR $err_msg " +fi +if [ ! -z "$ok_msg" ] ; then + echo -n "OK: $ok_msg" +fi +echo +exit 0 diff --git a/src/examples/mapreduce/index-builder-setup.rb b/src/examples/mapreduce/index-builder-setup.rb new file mode 100644 index 0000000..cda7fdd --- /dev/null +++ b/src/examples/mapreduce/index-builder-setup.rb @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set up sample data for IndexBuilder example +create "people", "attributes" +create "people-email", "INDEX" +create "people-phone", "INDEX" +create "people-name", "INDEX" + +[["1", "jenny", "jenny@example.com", "867-5309"], + ["2", "alice", "alice@example.com", "555-1234"], + ["3", "kevin", "kevinpet@example.com", "555-1212"]].each do |fields| + (id, name, email, phone) = *fields + put "people", id, "attributes:name", name + put "people", id, "attributes:email", email + put "people", id, "attributes:phone", phone +end + diff --git a/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/IndexBuilder.java b/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/IndexBuilder.java new file mode 100644 index 0000000..31c1b38 --- /dev/null +++ b/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/IndexBuilder.java @@ -0,0 +1,154 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.IOException; +import java.util.HashMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.util.GenericOptionsParser; + +/** + * Example map/reduce job to construct index tables that can be used to quickly + * find a row based on the value of a column. It demonstrates: + *
    + *
  • Using TableInputFormat and TableMapReduceUtil to use an HTable as input + * to a map/reduce job.
    • + *
  • Passing values from main method to children via the configuration.
    • + *
  • Using MultiTableOutputFormat to output to multiple tables from a + * map/reduce job.
    • + *
  • A real use case of building a secondary index over a table.
    • + *
+ * + *

Usage

+ * + *

+ * Modify ${HADOOP_HOME}/conf/hadoop-env.sh to include the hbase jar, the + * zookeeper jar, the examples output directory, and the hbase conf directory in + * HADOOP_CLASSPATH, and then run + * bin/hadoop org.apache.hadoop.hbase.mapreduce.IndexBuilder TABLE_NAME COLUMN_FAMILY ATTR [ATTR ...] + *

+ * + *

+ * To run with the sample data provided in index-builder-setup.rb, use the + * arguments people attributes name email phone. + *

+ * + *

+ * This code was written against HBase 0.21 trunk. + *

+ */ +public class IndexBuilder { + /** the column family containing the indexed row key */ + public static final byte[] INDEX_COLUMN = Bytes.toBytes("INDEX"); + /** the qualifier containing the indexed row key */ + public static final byte[] INDEX_QUALIFIER = Bytes.toBytes("ROW"); + + /** + * Internal Mapper to be run by Hadoop. + */ + public static class Map extends + Mapper { + private byte[] family; + private HashMap indexes; + + @Override + protected void map(ImmutableBytesWritable rowKey, Result result, Context context) + throws IOException, InterruptedException { + for(java.util.Map.Entry index : indexes.entrySet()) { + byte[] qualifier = index.getKey(); + ImmutableBytesWritable tableName = index.getValue(); + byte[] value = result.getValue(family, qualifier); + if (value != null) { + // original: row 123 attribute:phone 555-1212 + // index: row 555-1212 INDEX:ROW 123 + Put put = new Put(value); + put.add(INDEX_COLUMN, INDEX_QUALIFIER, rowKey.get()); + context.write(tableName, put); + } + } + } + + @Override + protected void setup(Context context) throws IOException, + InterruptedException { + Configuration configuration = context.getConfiguration(); + String tableName = configuration.get("index.tablename"); + String[] fields = configuration.getStrings("index.fields"); + String familyName = configuration.get("index.familyname"); + family = Bytes.toBytes(familyName); + indexes = new HashMap(); + for(String field : fields) { + // if the table is "people" and the field to index is "email", then the + // index table will be called "people-email" + indexes.put(Bytes.toBytes(field), + new ImmutableBytesWritable(Bytes.toBytes(tableName + "-" + field))); + } + } + } + + /** + * Job configuration. + */ + public static Job configureJob(Configuration conf, String [] args) + throws IOException { + String tableName = args[0]; + String columnFamily = args[1]; + System.out.println("****" + tableName); + conf.set(TableInputFormat.SCAN, TableMapReduceUtil.convertScanToString(new Scan())); + conf.set(TableInputFormat.INPUT_TABLE, tableName); + conf.set("index.tablename", tableName); + conf.set("index.familyname", columnFamily); + String[] fields = new String[args.length - 2]; + for(int i = 0; i < fields.length; i++) { + fields[i] = args[i + 2]; + } + conf.setStrings("index.fields", fields); + conf.set("index.familyname", "attributes"); + Job job = new Job(conf, tableName); + job.setJarByClass(IndexBuilder.class); + job.setMapperClass(Map.class); + job.setNumReduceTasks(0); + job.setInputFormatClass(TableInputFormat.class); + job.setOutputFormatClass(MultiTableOutputFormat.class); + return job; + } + + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); + if(otherArgs.length < 3) { + System.err.println("Only " + otherArgs.length + " arguments supplied, required: 3"); + System.err.println("Usage: IndexBuilder [ ...]"); + System.exit(-1); + } + Job job = configureJob(conf, otherArgs); + System.exit(job.waitForCompletion(true) ? 0 : 1); + } +} diff --git a/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/SampleUploader.java b/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/SampleUploader.java new file mode 100644 index 0000000..5629cca --- /dev/null +++ b/src/examples/mapreduce/org/apache/hadoop/hbase/mapreduce/SampleUploader.java @@ -0,0 +1,148 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; +import org.apache.hadoop.util.GenericOptionsParser; + +/** + * Sample Uploader MapReduce + *

+ * This is EXAMPLE code. You will need to change it to work for your context. + *

+ * Uses {@link TableReducer} to put the data into HBase. Change the InputFormat + * to suit your data. In this example, we are importing a CSV file. + *

+ * 

row,family,qualifier,value
+ *

+ * The table and columnfamily we're to insert into must preexist. + *

+ * There is no reducer in this example as it is not necessary and adds + * significant overhead. If you need to do any massaging of data before + * inserting into HBase, you can do this in the map as well. + *

Do the following to start the MR job: + * 

+ * ./bin/hadoop org.apache.hadoop.hbase.mapreduce.SampleUploader /tmp/input.csv TABLE_NAME
+ *
+ *

+ * This code was written against HBase 0.21 trunk. + */ +public class SampleUploader { + + private static final String NAME = "SampleUploader"; + + static class Uploader + extends Mapper { + + private long checkpoint = 100; + private long count = 0; + + @Override + public void map(LongWritable key, Text line, Context context) + throws IOException { + + // Input is a CSV file + // Each map() is a single line, where the key is the line number + // Each line is comma-delimited; row,family,qualifier,value + + // Split CSV line + String [] values = line.toString().split(","); + if(values.length != 4) { + return; + } + + // Extract each value + byte [] row = Bytes.toBytes(values[0]); + byte [] family = Bytes.toBytes(values[1]); + byte [] qualifier = Bytes.toBytes(values[2]); + byte [] value = Bytes.toBytes(values[3]); + + // Create Put + Put put = new Put(row); + put.add(family, qualifier, value); + + // Uncomment below to disable WAL. This will improve performance but means + // you will experience data loss in the case of a RegionServer crash. + // put.setWriteToWAL(false); + + try { + context.write(new ImmutableBytesWritable(row), put); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + // Set status every checkpoint lines + if(++count % checkpoint == 0) { + context.setStatus("Emitting Put " + count); + } + } + } + + /** + * Job configuration. + */ + public static Job configureJob(Configuration conf, String [] args) + throws IOException { + Path inputPath = new Path(args[0]); + String tableName = args[1]; + Job job = new Job(conf, NAME + "_" + tableName); + job.setJarByClass(Uploader.class); + FileInputFormat.setInputPaths(job, inputPath); + job.setInputFormatClass(SequenceFileInputFormat.class); + job.setMapperClass(Uploader.class); + // No reducers. Just write straight to table. Call initTableReducerJob + // because it sets up the TableOutputFormat. + TableMapReduceUtil.initTableReducerJob(tableName, null, job); + job.setNumReduceTasks(0); + return job; + } + + /** + * Main entry point. + * + * @param args The command line parameters. + * @throws Exception When running the job fails. + */ + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); + if(otherArgs.length != 2) { + System.err.println("Wrong number of arguments: " + otherArgs.length); + System.err.println("Usage: " + NAME + " "); + System.exit(-1); + } + Job job = configureJob(conf, otherArgs); + System.exit(job.waitForCompletion(true) ? 0 : 1); + } +} diff --git a/src/examples/thrift/DemoClient.cpp b/src/examples/thrift/DemoClient.cpp new file mode 100644 index 0000000..06cbc44 --- /dev/null +++ b/src/examples/thrift/DemoClient.cpp @@ -0,0 +1,315 @@ +/** + * Copyright 2008 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Instructions: + * 1. Run Thrift to generate the cpp module HBase + * thrift --gen cpp ../../../src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift + * 2. Execute {make}. + * 3. Execute {./DemoClient}. + */ + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include "Hbase.h" + +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; + +using namespace apache::hadoop::hbase::thrift; + +namespace { + +typedef std::vector StrVec; +typedef std::map StrMap; +typedef std::vector ColVec; +typedef std::map ColMap; +typedef std::vector CellVec; +typedef std::map CellMap; + + +static void +printRow(const std::vector &rowResult) +{ + for (size_t i = 0; i < rowResult.size(); i++) { + std::cout << "row: " << rowResult[i].row << ", cols: "; + for (CellMap::const_iterator it = rowResult[i].columns.begin(); + it != rowResult[i].columns.end(); ++it) { + std::cout << it->first << " => " << it->second.value << "; "; + } + std::cout << std::endl; + } +} + +static void +printVersions(const std::string &row, const CellVec &versions) +{ + std::cout << "row: " << row << ", values: "; + for (CellVec::const_iterator it = versions.begin(); it != versions.end(); ++it) { + std::cout << (*it).value << "; "; + } + std::cout << std::endl; +} + +} + +int +main(int argc, char** argv) +{ + if (argc < 3) { + std::cerr << "Invalid arguments!\n" << "Usage: DemoClient host port" << std::endl; + return -1; + } + + boost::shared_ptr socket(new TSocket("localhost", boost::lexical_cast(argv[2]))); + boost::shared_ptr transport(new TBufferedTransport(socket)); + boost::shared_ptr protocol(new TBinaryProtocol(transport)); + HbaseClient client(protocol); + + try { + transport->open(); + + std::string t("demo_table"); + + // + // Scan all tables, look for the demo table and delete it. + // + std::cout << "scanning tables..." << std::endl; + StrVec tables; + client.getTableNames(tables); + for (StrVec::const_iterator it = tables.begin(); it != tables.end(); ++it) { + std::cout << " found: " << *it << std::endl; + if (t == *it) { + if (client.isTableEnabled(*it)) { + std::cout << " disabling table: " << *it << std::endl; + client.disableTable(*it); + } + std::cout << " deleting table: " << *it << std::endl; + client.deleteTable(*it); + } + } + + // + // Create the demo table with two column families, entry: and unused: + // + ColVec columns; + columns.push_back(ColumnDescriptor()); + columns.back().name = "entry:"; + columns.back().maxVersions = 10; + columns.push_back(ColumnDescriptor()); + columns.back().name = "unused:"; + + std::cout << "creating table: " << t << std::endl; + try { + client.createTable(t, columns); + } catch (const AlreadyExists &ae) { + std::cerr << "WARN: " << ae.message << std::endl; + } + + ColMap columnMap; + client.getColumnDescriptors(columnMap, t); + std::cout << "column families in " << t << ": " << std::endl; + for (ColMap::const_iterator it = columnMap.begin(); it != columnMap.end(); ++it) { + std::cout << " column: " << it->second.name << ", maxVer: " << it->second.maxVersions << std::endl; + } + + // + // Test UTF-8 handling + // + std::string invalid("foo-\xfc\xa1\xa1\xa1\xa1\xa1"); + std::string valid("foo-\xE7\x94\x9F\xE3\x83\x93\xE3\x83\xBC\xE3\x83\xAB"); + + // non-utf8 is fine for data + std::vector mutations; + mutations.push_back(Mutation()); + mutations.back().column = "entry:foo"; + mutations.back().value = invalid; + client.mutateRow(t, "foo", mutations); + + // try empty strings + mutations.clear(); + mutations.push_back(Mutation()); + mutations.back().column = "entry:"; + mutations.back().value = ""; + client.mutateRow(t, "", mutations); + + // this row name is valid utf8 + mutations.clear(); + mutations.push_back(Mutation()); + mutations.back().column = "entry:foo"; + mutations.back().value = valid; + client.mutateRow(t, valid, mutations); + + // non-utf8 is now allowed in row names because HBase stores values as binary + mutations.clear(); + mutations.push_back(Mutation()); + mutations.back().column = "entry:foo"; + mutations.back().value = invalid; + client.mutateRow(t, invalid, mutations); + + // Run a scanner on the rows we just created + StrVec columnNames; + columnNames.push_back("entry:"); + + std::cout << "Starting scanner..." << std::endl; + int scanner = client.scannerOpen(t, "", columnNames); + try { + while (true) { + std::vector value; + client.scannerGet(value, scanner); + if (value.size() == 0) + break; + printRow(value); + } + } catch (const IOError &ioe) { + std::cerr << "FATAL: Scanner raised IOError" << std::endl; + } + + client.scannerClose(scanner); + std::cout << "Scanner finished" << std::endl; + + // + // Run some operations on a bunch of rows. + // + for (int i = 100; i >= 0; --i) { + // format row keys as "00000" to "00100" + char buf[32]; + sprintf(buf, "%05d", i); + std::string row(buf); + std::vector rowResult; + + mutations.clear(); + mutations.push_back(Mutation()); + mutations.back().column = "unused:"; + mutations.back().value = "DELETE_ME"; + client.mutateRow(t, row, mutations); + client.getRow(rowResult, t, row); + printRow(rowResult); + client.deleteAllRow(t, row); + + mutations.clear(); + mutations.push_back(Mutation()); + mutations.back().column = "entry:num"; + mutations.back().value = "0"; + mutations.push_back(Mutation()); + mutations.back().column = "entry:foo"; + mutations.back().value = "FOO"; + client.mutateRow(t, row, mutations); + client.getRow(rowResult, t, row); + printRow(rowResult); + + // sleep to force later timestamp + poll(0, 0, 50); + + mutations.clear(); + mutations.push_back(Mutation()); + mutations.back().column = "entry:foo"; + mutations.back().isDelete = true; + mutations.push_back(Mutation()); + mutations.back().column = "entry:num"; + mutations.back().value = "-1"; + client.mutateRow(t, row, mutations); + client.getRow(rowResult, t, row); + printRow(rowResult); + + mutations.clear(); + mutations.push_back(Mutation()); + mutations.back().column = "entry:num"; + mutations.back().value = boost::lexical_cast(i); + mutations.push_back(Mutation()); + mutations.back().column = "entry:sqr"; + mutations.back().value = boost::lexical_cast(i*i); + client.mutateRow(t, row, mutations); + client.getRow(rowResult, t, row); + printRow(rowResult); + + mutations.clear(); + mutations.push_back(Mutation()); + mutations.back().column = "entry:num"; + mutations.back().value = "-999"; + mutations.push_back(Mutation()); + mutations.back().column = "entry:sqr"; + mutations.back().isDelete = true; + client.mutateRowTs(t, row, mutations, 1); // shouldn't override latest + client.getRow(rowResult, t, row); + printRow(rowResult); + + CellVec versions; + client.getVer(versions, t, row, "entry:num", 10); + printVersions(row, versions); + assert(versions.size()); + std::cout << std::endl; + + try { + std::vector value; + client.get(value, t, row, "entry:foo"); + if (value.size()) { + std::cerr << "FATAL: shouldn't get here!" << std::endl; + return -1; + } + } catch (const IOError &ioe) { + // blank + } + } + + // scan all rows/columns + + columnNames.clear(); + client.getColumnDescriptors(columnMap, t); + std::cout << "The number of columns: " << columnMap.size() << std::endl; + for (ColMap::const_iterator it = columnMap.begin(); it != columnMap.end(); ++it) { + std::cout << " column with name: " + it->second.name << std::endl; + columnNames.push_back(it->second.name); + } + std::cout << std::endl; + + std::cout << "Starting scanner..." << std::endl; + scanner = client.scannerOpenWithStop(t, "00020", "00040", columnNames); + try { + while (true) { + std::vector value; + client.scannerGet(value, scanner); + if (value.size() == 0) + break; + printRow(value); + } + } catch (const IOError &ioe) { + std::cerr << "FATAL: Scanner raised IOError" << std::endl; + } + + client.scannerClose(scanner); + std::cout << "Scanner finished" << std::endl; + + transport->close(); + } catch (const TException &tx) { + std::cerr << "ERROR: " << tx.what() << std::endl; + } +} diff --git a/src/examples/thrift/DemoClient.java b/src/examples/thrift/DemoClient.java new file mode 100644 index 0000000..bb03fcc --- /dev/null +++ b/src/examples/thrift/DemoClient.java @@ -0,0 +1,346 @@ +/** + * Copyright 2008 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.thrift; + +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.SortedMap; + +import org.apache.hadoop.hbase.thrift.generated.AlreadyExists; +import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor; +import org.apache.hadoop.hbase.thrift.generated.Hbase; +import org.apache.hadoop.hbase.thrift.generated.IOError; +import org.apache.hadoop.hbase.thrift.generated.IllegalArgument; +import org.apache.hadoop.hbase.thrift.generated.Mutation; +import org.apache.hadoop.hbase.thrift.generated.TCell; +import org.apache.hadoop.hbase.thrift.generated.TRowResult; + +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; + +/* + * Instructions: + * 1. Run Thrift to generate the java module HBase + * thrift --gen java ../../../src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift + * 2. Acquire a jar of compiled Thrift java classes. As of this writing, HBase ships + * with this jar (libthrift-[VERSION].jar). If this jar is not present, or it is + * out-of-date with your current version of thrift, you can compile the jar + * yourself by executing {ant} in {$THRIFT_HOME}/lib/java. + * 3. Compile and execute this file with both the libthrift jar and the gen-java/ + * directory in the classpath. This can be done on the command-line with the + * following lines: (from the directory containing this file and gen-java/) + * + * javac -cp /path/to/libthrift/jar.jar:gen-java/ DemoClient.java + * mv DemoClient.class gen-java/org/apache/hadoop/hbase/thrift/ + * java -cp /path/to/libthrift/jar.jar:gen-java/ org.apache.hadoop.hbase.thrift.DemoClient + * + */ +public class DemoClient { + + static protected int port; + static protected String host; + CharsetDecoder decoder = null; + + public static void main(String[] args) + throws IOError, TException, UnsupportedEncodingException, IllegalArgument, AlreadyExists { + + if (args.length != 2) { + + System.out.println("Invalid arguments!"); + System.out.println("Usage: DemoClient host port"); + + System.exit(-1); + } + + port = Integer.parseInt(args[1]); + host = args[0]; + + + DemoClient client = new DemoClient(); + client.run(); + } + + DemoClient() { + decoder = Charset.forName("UTF-8").newDecoder(); + } + + // Helper to translate byte[]'s to UTF8 strings + private String utf8(byte[] buf) { + try { + return decoder.decode(ByteBuffer.wrap(buf)).toString(); + } catch (CharacterCodingException e) { + return "[INVALID UTF-8]"; + } + } + + // Helper to translate strings to UTF8 bytes + private byte[] bytes(String s) { + try { + return s.getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + return null; + } + } + + private void run() throws IOError, TException, IllegalArgument, + AlreadyExists { + + TTransport transport = new TSocket(host, port); + TProtocol protocol = new TBinaryProtocol(transport, true, true); + Hbase.Client client = new Hbase.Client(protocol); + + transport.open(); + + byte[] t = bytes("demo_table"); + + // + // Scan all tables, look for the demo table and delete it. + // + System.out.println("scanning tables..."); + for (ByteBuffer name : client.getTableNames()) { + System.out.println(" found: " + utf8(name.array())); + if (utf8(name.array()).equals(utf8(t))) { + if (client.isTableEnabled(name)) { + System.out.println(" disabling table: " + utf8(name.array())); + client.disableTable(name); + } + System.out.println(" deleting table: " + utf8(name.array())); + client.deleteTable(name); + } + } + + // + // Create the demo table with two column families, entry: and unused: + // + ArrayList columns = new ArrayList(); + ColumnDescriptor col = null; + col = new ColumnDescriptor(); + col.name = ByteBuffer.wrap(bytes("entry:")); + col.maxVersions = 10; + columns.add(col); + col = new ColumnDescriptor(); + col.name = ByteBuffer.wrap(bytes("unused:")); + columns.add(col); + + System.out.println("creating table: " + utf8(t)); + try { + client.createTable(ByteBuffer.wrap(t), columns); + } catch (AlreadyExists ae) { + System.out.println("WARN: " + ae.message); + } + + System.out.println("column families in " + utf8(t) + ": "); + Map columnMap = client.getColumnDescriptors(ByteBuffer.wrap(t)); + for (ColumnDescriptor col2 : columnMap.values()) { + System.out.println(" column: " + utf8(col2.name.array()) + ", maxVer: " + Integer.toString(col2.maxVersions)); + } + + // + // Test UTF-8 handling + // + byte[] invalid = {(byte) 'f', (byte) 'o', (byte) 'o', (byte) '-', (byte) 0xfc, (byte) 0xa1, (byte) 0xa1, (byte) 0xa1, (byte) 0xa1}; + byte[] valid = {(byte) 'f', (byte) 'o', (byte) 'o', (byte) '-', (byte) 0xE7, (byte) 0x94, (byte) 0x9F, (byte) 0xE3, (byte) 0x83, (byte) 0x93, (byte) 0xE3, (byte) 0x83, (byte) 0xBC, (byte) 0xE3, (byte) 0x83, (byte) 0xAB}; + + ArrayList mutations; + // non-utf8 is fine for data + mutations = new ArrayList(); + mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:foo")), ByteBuffer.wrap(invalid))); + client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(bytes("foo")), mutations); + + // try empty strings + mutations = new ArrayList(); + mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:")), ByteBuffer.wrap(bytes("")))); + client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(bytes("")), mutations); + + // this row name is valid utf8 + mutations = new ArrayList(); + mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:foo")), ByteBuffer.wrap(valid))); + client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(valid), mutations); + + // non-utf8 is now allowed in row names because HBase stores values as binary + ByteBuffer bf = ByteBuffer.wrap(invalid); + + mutations = new ArrayList(); + mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:foo")), ByteBuffer.wrap(invalid))); + client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(invalid), mutations); + + + // Run a scanner on the rows we just created + ArrayList columnNames = new ArrayList(); + columnNames.add(ByteBuffer.wrap(bytes("entry:"))); + + System.out.println("Starting scanner..."); + int scanner = client.scannerOpen(ByteBuffer.wrap(t), ByteBuffer.wrap(bytes("")), columnNames); + + while (true) { + List entry = client.scannerGet(scanner); + if (entry.isEmpty()) { + break; + } + printRow(entry); + } + + // + // Run some operations on a bunch of rows + // + for (int i = 100; i >= 0; --i) { + // format row keys as "00000" to "00100" + NumberFormat nf = NumberFormat.getInstance(); + nf.setMinimumIntegerDigits(5); + nf.setGroupingUsed(false); + byte[] row = bytes(nf.format(i)); + + mutations = new ArrayList(); + mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("unused:")), ByteBuffer.wrap(bytes("DELETE_ME")))); + client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), mutations); + printRow(client.getRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row))); + client.deleteAllRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row)); + + mutations = new ArrayList(); + mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:num")), ByteBuffer.wrap(bytes("0")))); + mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:foo")), ByteBuffer.wrap(bytes("FOO")))); + client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), mutations); + printRow(client.getRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row))); + + Mutation m = null; + mutations = new ArrayList(); + m = new Mutation(); + m.column = ByteBuffer.wrap(bytes("entry:foo")); + m.isDelete = true; + mutations.add(m); + m = new Mutation(); + m.column = ByteBuffer.wrap(bytes("entry:num")); + m.value = ByteBuffer.wrap(bytes("-1")); + mutations.add(m); + client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), mutations); + printRow(client.getRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row))); + + mutations = new ArrayList(); + mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:num")), ByteBuffer.wrap(bytes(Integer.toString(i))))); + mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:sqr")), ByteBuffer.wrap(bytes(Integer.toString(i * i))))); + client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), mutations); + printRow(client.getRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row))); + + // sleep to force later timestamp + try { + Thread.sleep(50); + } catch (InterruptedException e) { + // no-op + } + + mutations.clear(); + m = new Mutation(); + m.column = ByteBuffer.wrap(bytes("entry:num")); + m.value= ByteBuffer.wrap(bytes("-999")); + mutations.add(m); + m = new Mutation(); + m.column = ByteBuffer.wrap(bytes("entry:sqr")); + m.isDelete = true; + client.mutateRowTs(ByteBuffer.wrap(t), ByteBuffer.wrap(row), mutations, 1); // shouldn't override latest + printRow(client.getRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row))); + + List versions = client.getVer(ByteBuffer.wrap(t), ByteBuffer.wrap(row), ByteBuffer.wrap(bytes("entry:num")), 10); + printVersions(ByteBuffer.wrap(row), versions); + if (versions.isEmpty()) { + System.out.println("FATAL: wrong # of versions"); + System.exit(-1); + } + + + List result = client.get(ByteBuffer.wrap(t), ByteBuffer.wrap(row), ByteBuffer.wrap(bytes("entry:foo"))); + if (result.isEmpty() == false) { + System.out.println("FATAL: shouldn't get here"); + System.exit(-1); + } + + System.out.println(""); + } + + // scan all rows/columnNames + + columnNames.clear(); + for (ColumnDescriptor col2 : client.getColumnDescriptors(ByteBuffer.wrap(t)).values()) { + System.out.println("column with name: " + new String(col2.name.array())); + System.out.println(col2.toString()); + + columnNames.add(col2.name); + } + + System.out.println("Starting scanner..."); + scanner = client.scannerOpenWithStop(ByteBuffer.wrap(t), ByteBuffer.wrap(bytes("00020")), ByteBuffer.wrap(bytes("00040")), + columnNames); + + while (true) { + List entry = client.scannerGet(scanner); + if (entry.isEmpty()) { + System.out.println("Scanner finished"); + break; + } + printRow(entry); + } + + transport.close(); + } + + private final void printVersions(ByteBuffer row, List versions) { + StringBuilder rowStr = new StringBuilder(); + for (TCell cell : versions) { + rowStr.append(utf8(cell.value.array())); + rowStr.append("; "); + } + System.out.println("row: " + utf8(row.array()) + ", values: " + rowStr); + } + + private final void printRow(TRowResult rowResult) { + // copy values into a TreeMap to get them in sorted order + + TreeMap sorted = new TreeMap(); + for (Map.Entry column : rowResult.columns.entrySet()) { + sorted.put(utf8(column.getKey().array()), column.getValue()); + } + + StringBuilder rowStr = new StringBuilder(); + for (SortedMap.Entry entry : sorted.entrySet()) { + rowStr.append(entry.getKey()); + rowStr.append(" => "); + rowStr.append(utf8(entry.getValue().value.array())); + rowStr.append("; "); + } + System.out.println("row: " + utf8(rowResult.row.array()) + ", cols: " + rowStr); + } + + private void printRow(List rows) { + for (TRowResult rowResult : rows) { + printRow(rowResult); + } + } +} diff --git a/src/examples/thrift/DemoClient.php b/src/examples/thrift/DemoClient.php new file mode 100644 index 0000000..669f2b6 --- /dev/null +++ b/src/examples/thrift/DemoClient.php @@ -0,0 +1,277 @@ +row}, cols: \n" ); + $values = $rowresult->columns; + asort( $values ); + foreach ( $values as $k=>$v ) { + echo( " {$k} => {$v->value}\n" ); + } +} + +$socket = new TSocket( 'localhost', 9090 ); +$socket->setSendTimeout( 10000 ); // Ten seconds (too long for production, but this is just a demo ;) +$socket->setRecvTimeout( 20000 ); // Twenty seconds +$transport = new TBufferedTransport( $socket ); +$protocol = new TBinaryProtocol( $transport ); +$client = new HbaseClient( $protocol ); + +$transport->open(); + +$t = 'demo_table'; + +?> + +DemoClient + + +

+getTableNames();
+sort( $tables );
+foreach ( $tables as $name ) {
+  echo( "  found: {$name}\n" );
+  if ( $name == $t ) {
+    if ($client->isTableEnabled( $name )) {
+      echo( "    disabling table: {$name}\n");
+      $client->disableTable( $name );
+    }
+    echo( "    deleting table: {$name}\n" );
+    $client->deleteTable( $name );
+  }
+}
+
+#
+# Create the demo table with two column families, entry: and unused:
+#
+$columns = array(
+  new ColumnDescriptor( array(
+    'name' => 'entry:',
+    'maxVersions' => 10
+  ) ),
+  new ColumnDescriptor( array(
+    'name' => 'unused:'
+  ) )
+);
+
+echo( "creating table: {$t}\n" );
+try {
+  $client->createTable( $t, $columns );
+} catch ( AlreadyExists $ae ) {
+  echo( "WARN: {$ae->message}\n" );
+}
+
+echo( "column families in {$t}:\n" );
+$descriptors = $client->getColumnDescriptors( $t );
+asort( $descriptors );
+foreach ( $descriptors as $col ) {
+  echo( "  column: {$col->name}, maxVer: {$col->maxVersions}\n" );
+}
+
+#
+# Test UTF-8 handling
+#
+$invalid = "foo-\xfc\xa1\xa1\xa1\xa1\xa1";
+$valid = "foo-\xE7\x94\x9F\xE3\x83\x93\xE3\x83\xBC\xE3\x83\xAB";
+
+# non-utf8 is fine for data
+$mutations = array(
+  new Mutation( array(
+    'column' => 'entry:foo',
+    'value' => $invalid
+  ) ),
+);
+$client->mutateRow( $t, "foo", $mutations );
+
+# try empty strings
+$mutations = array(
+  new Mutation( array(
+    'column' => 'entry:',
+    'value' => ""
+  ) ),
+);
+$client->mutateRow( $t, "", $mutations );
+
+# this row name is valid utf8
+$mutations = array(
+  new Mutation( array(
+    'column' => 'entry:foo',
+    'value' => $valid
+  ) ),
+);
+$client->mutateRow( $t, $valid, $mutations );
+
+# non-utf8 is not allowed in row names
+try {
+  $mutations = array(
+    new Mutation( array(
+      'column' => 'entry:foo',
+      'value' => $invalid
+    ) ),
+  );
+  $client->mutateRow( $t, $invalid, $mutations );
+  throw new Exception( "shouldn't get here!" );
+} catch ( IOError $e ) {
+  echo( "expected error: {$e->message}\n" );
+}
+
+# Run a scanner on the rows we just created
+echo( "Starting scanner...\n" );
+$scanner = $client->scannerOpen( $t, "", array( "entry:" ) );
+try {
+  while (true) printRow( $client->scannerGet( $scanner ) );
+} catch ( NotFound $nf ) {
+  $client->scannerClose( $scanner );
+  echo( "Scanner finished\n" );
+}
+
+#
+# Run some operations on a bunch of rows.
+#
+for ($e=100; $e>=0; $e--) {
+
+  # format row keys as "00000" to "00100"
+  $row = str_pad( $e, 5, '0', STR_PAD_LEFT );
+
+  $mutations = array(
+    new Mutation( array(
+      'column' => 'unused:',
+      'value' => "DELETE_ME"
+    ) ),
+  );
+  $client->mutateRow( $t, $row, $mutations);
+  printRow( $client->getRow( $t, $row ));
+  $client->deleteAllRow( $t, $row );
+
+  $mutations = array(
+    new Mutation( array(
+      'column' => 'entry:num',
+      'value' => "0"
+    ) ),
+    new Mutation( array(
+      'column' => 'entry:foo',
+      'value' => "FOO"
+    ) ),
+  );
+  $client->mutateRow( $t, $row, $mutations );
+  printRow( $client->getRow( $t, $row ));
+
+  $mutations = array(
+    new Mutation( array(
+      'column' => 'entry:foo',
+      'isDelete' => 1
+    ) ),
+    new Mutation( array(
+      'column' => 'entry:num',
+      'value' => '-1'
+    ) ),
+  );
+  $client->mutateRow( $t, $row, $mutations );
+  printRow( $client->getRow( $t, $row ) );
+
+  $mutations = array(
+    new Mutation( array(
+      'column' => "entry:num",
+      'value' => $e
+    ) ),
+    new Mutation( array(
+      'column' => "entry:sqr",
+      'value' => $e * $e
+    ) ),
+  );
+  $client->mutateRow( $t, $row, $mutations );
+  printRow( $client->getRow( $t, $row ));
+  
+  $mutations = array(
+    new Mutation( array(
+      'column' => 'entry:num',
+      'value' => '-999'
+    ) ),
+    new Mutation( array(
+      'column' => 'entry:sqr',
+      'isDelete' => 1
+    ) ),
+  );
+  $client->mutateRowTs( $t, $row, $mutations, 1 ); # shouldn't override latest
+  printRow( $client->getRow( $t, $row ) );
+
+  $versions = $client->getVer( $t, $row, "entry:num", 10 );
+  echo( "row: {$row}, values: \n" );
+  foreach ( $versions as $v ) echo( "  {$v->value};\n" );
+  
+  try {
+    $client->get( $t, $row, "entry:foo");
+    throw new Exception ( "shouldn't get here! " );
+  } catch ( NotFound $nf ) {
+    # blank
+  }
+
+}
+
+$columns = array();
+foreach ( $client->getColumnDescriptors($t) as $col=>$desc ) {
+  echo("column with name: {$desc->name}\n");
+  $columns[] = $desc->name.":";
+}
+
+echo( "Starting scanner...\n" );
+$scanner = $client->scannerOpenWithStop( $t, "00020", "00040", $columns );
+try {
+  while (true) printRow( $client->scannerGet( $scanner ) );
+} catch ( NotFound $nf ) {
+  $client->scannerClose( $scanner );
+  echo( "Scanner finished\n" );
+}
+  
+$transport->close();
+
+?>
+
+ + + diff --git a/src/examples/thrift/DemoClient.pl b/src/examples/thrift/DemoClient.pl new file mode 100644 index 0000000..17d92fa --- /dev/null +++ b/src/examples/thrift/DemoClient.pl @@ -0,0 +1,293 @@ +#!/usr/bin/perl +# Copyright 2011 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Instructions: +# 1. Run Thrift to generate the perl module HBase +# thrift --gen perl ../../../src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift +# 2. Add the gen-perl/Hbase directory to your perl's @INC path: +# a. This is one of the paths listed in: perl -e 'print join ("\n", @INC) . "\n"' +# -OR- +# b. set PERL5LIB to include the gen-perl/ directory +# 3. Execute perl DemoClient.pl. + +use strict; +use warnings; + +use Thrift::Socket; +use Thrift::BufferedTransport; +use Thrift::BinaryProtocol; +use Hbase::Hbase; +use Data::Dumper; + +sub printRow($) +{ + my $rowresult = shift; + + return if (!$rowresult || @{$rowresult} < 1); + # rowresult is presummed to be a Hbase::TRowResult object + + printf ("row: {%s}, cols: \n", $rowresult->[0]->{row}); + my $values = $rowresult->[0]->{columns}; + foreach my $key ( sort ( keys %{$values} ) ) + { + printf ("{%s} => {%s}\n", $key, $values->{$key}->{value}); + } +} + +my $host = $ARGV[0] || "localhost"; +my $port = $ARGV[1] || 9090; + +my $socket = Thrift::Socket->new ($host, $port); +$socket->setSendTimeout (10000); # Ten seconds (value is in millisec) +$socket->setRecvTimeout (20000); # Twenty seconds (value is in millisec) + +my $transport = Thrift::BufferedTransport->new ($socket); +my $protocol = Thrift::BinaryProtocol->new ($transport); +my $client = Hbase::HbaseClient->new ($protocol); + +eval { + $transport->open (); +}; +if ($@) +{ + print "Unable to connect: $@->{message}\n"; + exit 1; +} + +my $demo_table = "demo_table"; + +print "scanning tables...\n"; + +# +# Search for all the tables in the HBase DB, return value is an arrayref +# +my $tables = $client->getTableNames(); +foreach my $table (sort @{$tables}) +{ + print " found {$table}\n"; + # This client will re-create the $demo_table, so we need to drop it first + if ($table eq $demo_table) + { + # Before we can drop a table, it has to be disabled first + if ($client->isTableEnabled ($table)) + { + print " disabling table: {$table}\n"; + $client->disableTable ($table); + } + # We assume the table has been disabled at this point + print " deleting table: {$table}\n"; + $client->deleteTable ($table); + } +} + +# +# Create the demo table with two column families, entry: and unused: +# +my $columns = [ + Hbase::ColumnDescriptor->new ( { name => "entry:", maxVersions => 10 } ), + Hbase::ColumnDescriptor->new ( { name => "unused:" } ), + ]; + +print "creating table: {$demo_table}\n"; +eval { + # This can throw Hbase::IllegalArgument (HASH) + $client->createTable ( $demo_table, $columns ); +}; +if ($@) +{ + die "ERROR: Unable to create table {$demo_table}: $@->{message}\n"; +} + +print "column families in {$demo_table}:\n"; +my $descriptors = $client->getColumnDescriptors ($demo_table); +foreach my $col (sort keys %{$descriptors}) +{ + printf (" column: {%s}, maxVer: {%s}\n", $descriptors->{$col}->{name}, $descriptors->{$col}->{maxVersions} ); +} + +# +# Test UTF-8 handling +# +my $invalid = "foo-\xfc\xa1\xa1\xa1\xa1\xa1"; +my $valid = "foo-\xE7\x94\x9F\xE3\x83\x93\xE3\x83\xBC\xE3\x83\xAB"; + +# non-utf8 is fine for data +my $key = "foo"; +my $mutations = [ Hbase::Mutation->new ( { column => "entry:$key", value => $invalid } ) ]; +$client->mutateRow ( $demo_table, $key, $mutations ); + +# try emptry strings +$key = ""; +$mutations = [ Hbase::Mutation->new ( { column => "entry:$key", value => "" } ) ]; +$client->mutateRow ( $demo_table, $key, $mutations ); + +# this row name is valid utf8 +$key = "foo"; +# This is another way to use the Mutation class +my $mutation = Hbase::Mutation->new (); +$mutation->{column} = "entry:$key"; +$mutation->{value} = $valid; +$mutations = [ $mutation ]; +$client->mutateRow ( $demo_table, $key, $mutations ); + +# non-utf8 is not allowed in row names +eval { + $mutations = [ Hbase::Mutation->new ( { column => "entry:$key", value => $invalid } ) ]; + # this can throw a TApplicationException (HASH) error + $client->mutateRow ($demo_table, $key, $mutations); + die ("shouldn't get here!"); +}; +if ($@) +{ + print "expected error: $@->{message}\n"; +} + +# +# Run a scanner on the rows we just created +# +print "Starting scanner...\n"; +$key = ""; +# scannerOpen expects ( table, key, ) +# if key is empty, it searches for all entries in the table +# if column descriptors is empty, it searches for all column descriptors within the table +my $scanner = $client->scannerOpen ( $demo_table, $key, [ "entry:" ] ); +eval { + + # scannerGet returns an empty arrayref (instead of an undef) to indicate no results + my $result = $client->scannerGet ( $scanner ); + while ( $result && @{$result} > 0 ) + { + printRow ( $result ); + $result = $client->scannerGet ( $scanner ); + } + + $client->scannerClose ( $scanner ); + print "Scanner finished\n"; +}; +if ($@) +{ + $client->scannerClose ( $scanner ); + print "Scanner finished\n"; +} + +# +# Run some operations on a bunch of rows +# +for (my $e = 100; $e > 0; $e--) +{ + # format row keys as "00000" to "00100"; + my $row = sprintf ("%05d", $e); + + $mutations = [ Hbase::Mutation->new ( { column => "unused:", value => "DELETE_ME" } ) ]; + $client->mutateRow ( $demo_table, $row, $mutations ); + printRow ( $client->getRow ( $demo_table, $row ) ); + $client->deleteAllRow ( $demo_table, $row ); + + $mutations = [ + Hbase::Mutation->new ( { column => "entry:num", value => "0" } ), + Hbase::Mutation->new ( { column => "entry:foo", value => "FOO" } ), + ]; + $client->mutateRow ( $demo_table, $row, $mutations ); + printRow ( $client->getRow ( $demo_table, $row ) ); + + $mutations = [ + Hbase::Mutation->new ( { column => "entry:foo", isDelete => 1 } ), + Hbase::Mutation->new ( { column => "entry:num", value => -1 } ), + ]; + $client->mutateRow ( $demo_table, $row, $mutations ); + printRow ( $client->getRow ( $demo_table, $row ) ); + + $mutations = [ + Hbase::Mutation->new ( { column => "entry:num", value => $e } ), + Hbase::Mutation->new ( { column => "entry:sqr", value => $e * $e } ), + ]; + $client->mutateRow ( $demo_table, $row, $mutations ); + printRow ( $client->getRow ( $demo_table, $row ) ); + + $mutations = [ + Hbase::Mutation->new ( { column => "entry:num", value => -999 } ), + Hbase::Mutation->new ( { column => "entry:sqr", isDelete => 1 } ), + ]; + + # mutateRowTs => modify the row entry at the specified timestamp (ts) + $client->mutateRowTs ( $demo_table, $row, $mutations, 1 ); # shouldn't override latest + printRow ( $client->getRow ( $demo_table, $row ) ); + + my $versions = $client->getVer ( $demo_table, $row, "entry:num", 10 ); + printf ( "row: {%s}, values: \n", $row ); + foreach my $v ( @{$versions} ) + { + printf ( " {%s} @ {%s}\n", $v->{value}, $v->{timestamp} ); + } + + eval { + + my $result = $client->get ( $demo_table, $row, "entry:foo" ); + + # Unfortunately, the API returns an empty arrayref instead of undef + # to signify a "not found", which makes it slightly inconvenient. + die "shouldn't get here!" if ($result && @{$result} > 0); + + if (!$result || ($result && @{$result} < 1)) + { + print "expected: {$row} not found in {$demo_table}\n"; + } + }; + if ($@) + { + print "expected error: $@\n"; + } +} + +my $column_descriptor = $client->getColumnDescriptors ( $demo_table ); +$columns = []; +foreach my $col ( keys %{$column_descriptor} ) +{ + my $colname = $column_descriptor->{$col}->{name}; + print "column with name: {$colname}\n"; + push ( @{$columns}, $colname); +} + +print "Starting scanner...\n"; +$scanner = $client->scannerOpenWithStop ( $demo_table, "00020", "00040", $columns ); +eval { + + # scannerGet returns an empty arrayref (instead of an undef) to indicate no results + my $result = $client->scannerGet ( $scanner ); + while ( $result && @$result > 0 ) + { + printRow ( $result ); + $result = $client->scannerGet ( $scanner ); + } + + $client->scannerClose ( $scanner ); + print "Scanner finished\n"; +}; +if ($@) +{ + $client->scannerClose ( $scanner ); + print "Scanner finished\n"; +} + +$transport->close (); + +exit 0; + diff --git a/src/examples/thrift/DemoClient.py b/src/examples/thrift/DemoClient.py new file mode 100644 index 0000000..eabbbe8 --- /dev/null +++ b/src/examples/thrift/DemoClient.py @@ -0,0 +1,202 @@ +#!/usr/bin/python +'''Copyright 2008 The Apache Software Foundation + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +''' +# Instructions: +# 1. Run Thrift to generate the python module HBase +# thrift --gen py ../../../src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift +# 2. Create a directory of your choosing that contains: +# a. This file (DemoClient.py). +# b. The directory gen-py/hbase (generated by instruction step 1). +# c. The directory {$THRIFT_HOME}/lib/py/build/lib.{YOUR_SYSTEM}/thrift. +# Or, modify the import statements below such that this file can access the +# directories from steps 3b and 3c. +# 3. Execute {python DemoClient.py}. + +import sys +import time + +from thrift import Thrift +from thrift.transport import TSocket, TTransport +from thrift.protocol import TBinaryProtocol +from hbase import ttypes +from hbase.Hbase import Client, ColumnDescriptor, Mutation + +def printVersions(row, versions): + print "row: " + row + ", values: ", + for cell in versions: + print cell.value + "; ", + print + +def printRow(entry): + print "row: " + entry.row + ", cols:", + for k in sorted(entry.columns): + print k + " => " + entry.columns[k].value, + print + +# Make socket +transport = TSocket.TSocket('localhost', 9090) + +# Buffering is critical. Raw sockets are very slow +transport = TTransport.TBufferedTransport(transport) + +# Wrap in a protocol +protocol = TBinaryProtocol.TBinaryProtocol(transport) + +# Create a client to use the protocol encoder +client = Client(protocol) + +# Connect! +transport.open() + +t = "demo_table" + +# +# Scan all tables, look for the demo table and delete it. +# +print "scanning tables..." +for table in client.getTableNames(): + print " found: %s" %(table) + if table == t: + if client.isTableEnabled(table): + print " disabling table: %s" %(t) + client.disableTable(table) + print " deleting table: %s" %(t) + client.deleteTable(table) + +columns = [] +col = ColumnDescriptor() +col.name = 'entry:' +col.maxVersions = 10 +columns.append(col) +col = ColumnDescriptor() +col.name = 'unused:' +columns.append(col) + +try: + print "creating table: %s" %(t) + client.createTable(t, columns) +except AlreadyExists, ae: + print "WARN: " + ae.message + +cols = client.getColumnDescriptors(t) +print "column families in %s" %(t) +for col_name in cols.keys(): + col = cols[col_name] + print " column: %s, maxVer: %d" % (col.name, col.maxVersions) +# +# Test UTF-8 handling +# +invalid = "foo-\xfc\xa1\xa1\xa1\xa1\xa1" +valid = "foo-\xE7\x94\x9F\xE3\x83\x93\xE3\x83\xBC\xE3\x83\xAB"; + +# non-utf8 is fine for data +mutations = [Mutation(column="entry:foo",value=invalid)] +print str(mutations) +client.mutateRow(t, "foo", mutations) + +# try empty strings +mutations = [Mutation(column="entry:", value="")] +client.mutateRow(t, "", mutations) + +# this row name is valid utf8 +mutations = [Mutation(column="entry:foo", value=valid)] +client.mutateRow(t, valid, mutations) + +# non-utf8 is not allowed in row names +try: + mutations = [Mutation(column="entry:foo", value=invalid)] + client.mutateRow(t, invalid, mutations) +except ttypes.IOError, e: + print 'expected exception: %s' %(e.message) + +# Run a scanner on the rows we just created +print "Starting scanner..." +scanner = client.scannerOpen(t, "", ["entry:"]) + +r = client.scannerGet(scanner) +while r: + printRow(r[0]) + r = client.scannerGet(scanner) +print "Scanner finished" + +# +# Run some operations on a bunch of rows. +# +for e in range(100, 0, -1): + # format row keys as "00000" to "00100" + row = "%0.5d" % (e) + + mutations = [Mutation(column="unused:", value="DELETE_ME")] + client.mutateRow(t, row, mutations) + printRow(client.getRow(t, row)[0]) + client.deleteAllRow(t, row) + + mutations = [Mutation(column="entry:num", value="0"), + Mutation(column="entry:foo", value="FOO")] + client.mutateRow(t, row, mutations) + printRow(client.getRow(t, row)[0]); + + mutations = [Mutation(column="entry:foo",isDelete=True), + Mutation(column="entry:num",value="-1")] + client.mutateRow(t, row, mutations) + printRow(client.getRow(t, row)[0]) + + mutations = [Mutation(column="entry:num", value=str(e)), + Mutation(column="entry:sqr", value=str(e*e))] + client.mutateRow(t, row, mutations) + printRow(client.getRow(t, row)[0]) + + time.sleep(0.05) + + mutations = [Mutation(column="entry:num",value="-999"), + Mutation(column="entry:sqr",isDelete=True)] + client.mutateRowTs(t, row, mutations, 1) # shouldn't override latest + printRow(client.getRow(t, row)[0]) + + versions = client.getVer(t, row, "entry:num", 10) + printVersions(row, versions) + if len(versions) != 4: + print("FATAL: wrong # of versions") + sys.exit(-1) + + r = client.get(t, row, "entry:foo") + if not r: + print "yup, we didn't find entry:foo" + # just to be explicit, we get lists back, if it's empty there was no matching row. + if len(r) > 0: + raise "shouldn't get here!" + +columnNames = [] +for (col, desc) in client.getColumnDescriptors(t).items(): + print "column with name: "+desc.name + print desc + columnNames.append(desc.name+":") + +print "Starting scanner..." +scanner = client.scannerOpenWithStop(t, "00020", "00040", columnNames) + +r = client.scannerGet(scanner) +while r: + printRow(r[0]) + r = client.scannerGet(scanner) + +client.scannerClose(scanner) +print "Scanner finished" + +transport.close() diff --git a/src/examples/thrift/DemoClient.rb b/src/examples/thrift/DemoClient.rb new file mode 100644 index 0000000..2b7b5e7 --- /dev/null +++ b/src/examples/thrift/DemoClient.rb @@ -0,0 +1,245 @@ +#!/usr/bin/ruby + +# Copyright 2008 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Instructions: +# 1. Run Thrift to generate the ruby module HBase +# thrift --gen rb ../../../src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift +# 2. Modify the import string below to point to {$THRIFT_HOME}/lib/rb/lib. +# 3. Execute {ruby DemoClient.rb}. + +# You will need to modify this import string: +$:.push('~/Thrift/thrift-20080411p1/lib/rb/lib') +$:.push('./gen-rb') + +require 'thrift/transport/tsocket' +require 'thrift/protocol/tbinaryprotocol' + +require 'Hbase' + +def printRow(rowresult) + print "row: #{rowresult.row}, cols: " + rowresult.columns.sort.each do |k,v| + print "#{k} => #{v.value}; " + end + puts "" +end + +transport = TBufferedTransport.new(TSocket.new("localhost", 9090)) +protocol = TBinaryProtocol.new(transport) +client = Apache::Hadoop::Hbase::Thrift::Hbase::Client.new(protocol) + +transport.open() + +t = "demo_table" + +# +# Scan all tables, look for the demo table and delete it. +# +puts "scanning tables..." +client.getTableNames().sort.each do |name| + puts " found: #{name}" + if (name == t) + if (client.isTableEnabled(name)) + puts " disabling table: #{name}" + client.disableTable(name) + end + puts " deleting table: #{name}" + client.deleteTable(name) + end +end + +# +# Create the demo table with two column families, entry: and unused: +# +columns = [] +col = Apache::Hadoop::Hbase::Thrift::ColumnDescriptor.new +col.name = "entry:" +col.maxVersions = 10 +columns << col; +col = Apache::Hadoop::Hbase::Thrift::ColumnDescriptor.new +col.name = "unused:" +columns << col; + +puts "creating table: #{t}" +begin + client.createTable(t, columns) +rescue Apache::Hadoop::Hbase::Thrift::AlreadyExists => ae + puts "WARN: #{ae.message}" +end + +puts "column families in #{t}: " +client.getColumnDescriptors(t).sort.each do |key, col| + puts " column: #{col.name}, maxVer: #{col.maxVersions}" +end + +# +# Test UTF-8 handling +# +invalid = "foo-\xfc\xa1\xa1\xa1\xa1\xa1" +valid = "foo-\xE7\x94\x9F\xE3\x83\x93\xE3\x83\xBC\xE3\x83\xAB"; + +# non-utf8 is fine for data +mutations = [] +m = Apache::Hadoop::Hbase::Thrift::Mutation.new +m.column = "entry:foo" +m.value = invalid +mutations << m +client.mutateRow(t, "foo", mutations) + +# try empty strings +mutations = [] +m = Apache::Hadoop::Hbase::Thrift::Mutation.new +m.column = "entry:" +m.value = "" +mutations << m +client.mutateRow(t, "", mutations) + +# this row name is valid utf8 +mutations = [] +m = Apache::Hadoop::Hbase::Thrift::Mutation.new +m.column = "entry:foo" +m.value = valid +mutations << m +client.mutateRow(t, valid, mutations) + +# non-utf8 is not allowed in row names +begin + mutations = [] + m = Apache::Hadoop::Hbase::Thrift::Mutation.new + m.column = "entry:foo" + m.value = invalid + mutations << m + client.mutateRow(t, invalid, mutations) + raise "shouldn't get here!" +rescue Apache::Hadoop::Hbase::Thrift::IOError => e + puts "expected error: #{e.message}" +end + +# Run a scanner on the rows we just created +puts "Starting scanner..." +scanner = client.scannerOpen(t, "", ["entry:"]) +begin + while (true) + printRow(client.scannerGet(scanner)) + end +rescue Apache::Hadoop::Hbase::Thrift::NotFound => nf + client.scannerClose(scanner) + puts "Scanner finished" +end + +# +# Run some operations on a bunch of rows. +# +(0..100).to_a.reverse.each do |e| + # format row keys as "00000" to "00100" + row = format("%0.5d", e) + + mutations = [] + m = Apache::Hadoop::Hbase::Thrift::Mutation.new + m.column = "unused:" + m.value = "DELETE_ME" + mutations << m + client.mutateRow(t, row, mutations) + printRow(client.getRow(t, row)) + client.deleteAllRow(t, row) + + mutations = [] + m = Apache::Hadoop::Hbase::Thrift::Mutation.new + m.column = "entry:num" + m.value = "0" + mutations << m + m = Apache::Hadoop::Hbase::Thrift::Mutation.new + m.column = "entry:foo" + m.value = "FOO" + mutations << m + client.mutateRow(t, row, mutations) + printRow(client.getRow(t, row)) + + mutations = [] + m = Apache::Hadoop::Hbase::Thrift::Mutation.new + m.column = "entry:foo" + m.isDelete = 1 + mutations << m + m = Apache::Hadoop::Hbase::Thrift::Mutation.new + m.column = "entry:num" + m.value = "-1" + mutations << m + client.mutateRow(t, row, mutations) + printRow(client.getRow(t, row)); + + mutations = [] + m = Apache::Hadoop::Hbase::Thrift::Mutation.new + m.column = "entry:num" + m.value = e.to_s + mutations << m + m = Apache::Hadoop::Hbase::Thrift::Mutation.new + m.column = "entry:sqr" + m.value = (e*e).to_s + mutations << m + client.mutateRow(t, row, mutations) + printRow(client.getRow(t, row)) + + mutations = [] + m = Apache::Hadoop::Hbase::Thrift::Mutation.new + m.column = "entry:num" + m.value = "-999" + mutations << m + m = Apache::Hadoop::Hbase::Thrift::Mutation.new + m.column = "entry:sqr" + m.isDelete = 1 + mutations << m + client.mutateRowTs(t, row, mutations, 1) # shouldn't override latest + printRow(client.getRow(t, row)); + + versions = client.getVer(t, row, "entry:num", 10) + print "row: #{row}, values: " + versions.each do |v| + print "#{v.value}; " + end + puts "" + + begin + client.get(t, row, "entry:foo") + raise "shouldn't get here!" + rescue Apache::Hadoop::Hbase::Thrift::NotFound => nf + # blank + end + + puts "" +end + +columns = [] +client.getColumnDescriptors(t).each do |col, desc| + puts "column with name: #{desc.name}" + columns << desc.name + ":" +end + +puts "Starting scanner..." +scanner = client.scannerOpenWithStop(t, "00020", "00040", columns) +begin + while (true) + printRow(client.scannerGet(scanner)) + end +rescue Apache::Hadoop::Hbase::Thrift::NotFound => nf + client.scannerClose(scanner) + puts "Scanner finished" +end + +transport.close() diff --git a/src/examples/thrift/Makefile b/src/examples/thrift/Makefile new file mode 100644 index 0000000..691a1e9 --- /dev/null +++ b/src/examples/thrift/Makefile @@ -0,0 +1,35 @@ +# Copyright 2008 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Makefile for C++ Hbase Thrift DemoClient +# NOTE: run 'thrift -cpp Hbase.thrift' first + +THRIFT_DIR = /usr/local/include/thrift +LIB_DIR = /usr/local/lib + +GEN_SRC = ./gen-cpp/Hbase.cpp \ + ./gen-cpp/Hbase_types.cpp \ + ./gen-cpp/Hbase_constants.cpp + +default: DemoClient + +DemoClient: DemoClient.cpp + g++ -o DemoClient -I${THRIFT_DIR} -I./gen-cpp -L${LIB_DIR} -lthrift DemoClient.cpp ${GEN_SRC} + +clean: + rm -rf DemoClient diff --git a/src/examples/thrift/README.txt b/src/examples/thrift/README.txt new file mode 100644 index 0000000..c742f8d --- /dev/null +++ b/src/examples/thrift/README.txt @@ -0,0 +1,16 @@ +Hbase Thrift Client Examples +============================ + +Included in this directory are sample clients of the HBase ThriftServer. They +all perform the same actions but are implemented in C++, Java, Ruby, PHP, and +Python respectively. + +To run/compile this clients, you will first need to install the thrift package +(from http://developers.facebook.com/thrift/) and then run thrift to generate +the language files: + +thrift --gen cpp --gen java --gen rb --gen py -php \ + ../../../src/java/org/apache/hadoop/hbase/thrift/Hbase.thrift + +See the individual DemoClient test files for more specific instructions on +running each test. diff --git a/src/examples/thrift2/DemoClient.java b/src/examples/thrift2/DemoClient.java new file mode 100644 index 0000000..d5b805c --- /dev/null +++ b/src/examples/thrift2/DemoClient.java @@ -0,0 +1,90 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.thrift2; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.thrift2.generated.TColumnValue; +import org.apache.hadoop.hbase.thrift2.generated.TGet; +import org.apache.hadoop.hbase.thrift2.generated.THBaseService; +import org.apache.hadoop.hbase.thrift2.generated.TIOError; +import org.apache.hadoop.hbase.thrift2.generated.TPut; +import org.apache.hadoop.hbase.thrift2.generated.TResult; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; + +public class DemoClient { + public static void main(String[] args) throws TIOError, TException { + System.out.println("Thrift2 Demo"); + System.out.println("This demo assumes you have a table called \"example\" with a column family called \"family1\""); + + String host = "localhost"; + int port = 9090; + int timeout = 10000; + boolean framed = false; + + TTransport transport = new TSocket(host, port, timeout); + if (framed) { + transport = new TFramedTransport(transport); + } + TProtocol protocol = new TBinaryProtocol(transport); + // This is our thrift client. + THBaseService.Iface client = new THBaseService.Client(protocol); + + // open the transport + transport.open(); + + ByteBuffer table = ByteBuffer.wrap("example".getBytes()); + + TPut put = new TPut(); + put.setRow("row1".getBytes()); + + TColumnValue columnValue = new TColumnValue(); + columnValue.setFamily("family1".getBytes()); + columnValue.setQualifier("qualifier1".getBytes()); + columnValue.setValue("value1".getBytes()); + List columnValues = new ArrayList(); + columnValues.add(columnValue); + put.setColumnValues(columnValues); + + client.put(table, put); + + TGet get = new TGet(); + get.setRow("row1".getBytes()); + + TResult result = client.get(table, get); + + System.out.print("row = " + new String(result.getRow())); + for (TColumnValue resultColumnValue : result.getColumnValues()) { + System.out.print("family = " + new String(resultColumnValue.getFamily())); + System.out.print("qualifier = " + new String(resultColumnValue.getFamily())); + System.out.print("value = " + new String(resultColumnValue.getValue())); + System.out.print("timestamp = " + resultColumnValue.getTimestamp()); + } + + transport.close(); + } +} \ No newline at end of file diff --git a/src/examples/thrift2/DemoClient.py b/src/examples/thrift2/DemoClient.py new file mode 100644 index 0000000..67abc5b --- /dev/null +++ b/src/examples/thrift2/DemoClient.py @@ -0,0 +1,69 @@ +""" + Copyright 2011 The Apache Software Foundation + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +# Instructions: +# 1. Run Thrift to generate the python module hbase +# thrift --gen py ../../../src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift +# 2. Create a directory of your choosing that contains: +# a. This file (DemoClient.py). +# b. The directory gen-py/hbase (generated by instruction step 1). +# 3. pip install thrift==0.7.0 +# 4. Create a table call "example", with a family called "family1" using the hbase shell. +# 5. Start the hbase thrift2 server +# bin/hbase thrift2 start +# 6. Execute {python DemoClient.py}. + +from thrift.transport import TTransport +from thrift.transport import TSocket +from thrift.transport import THttpClient +from thrift.protocol import TBinaryProtocol + +from hbase import THBaseService +from hbase.ttypes import * + +print "Thrift2 Demo" +print "This demo assumes you have a table called \"example\" with a column family called \"family1\"" + +host = "localhost" +port = 9090 +framed = False + +socket = TSocket.TSocket(host, port) +if framed: + transport = TTransport.TFramedTransport(socket) +else: + transport = TTransport.TBufferedTransport(socket) +protocol = TBinaryProtocol.TBinaryProtocol(transport) +client = THBaseService.Client(protocol) + +transport.open() + +table = "example" + +put = TPut(row="row1", columnValues=[TColumnValue(family="family1",qualifier="qualifier1",value="value1")]) +print "Putting:", put +client.put(table, put) + +get = TGet(row="row1") +print "Getting:", get +result = client.get(table, get) + +print "Result:", result + +transport.close() \ No newline at end of file diff --git a/src/main/avro/hbase.avpr b/src/main/avro/hbase.avpr new file mode 100644 index 0000000..61208fd --- /dev/null +++ b/src/main/avro/hbase.avpr @@ -0,0 +1,609 @@ +{ + "protocol" : "HBase", + "namespace" : "org.apache.hadoop.hbase.avro.generated", + "types" : [ { + "type" : "record", + "name" : "AServerAddress", + "fields" : [ { + "name" : "hostname", + "type" : "string" + }, { + "name" : "inetSocketAddress", + "type" : "string" + }, { + "name" : "port", + "type" : "int" + } ] + }, { + "type" : "record", + "name" : "ARegionLoad", + "fields" : [ { + "name" : "memStoreSizeMB", + "type" : "int" + }, { + "name" : "name", + "type" : "bytes" + }, { + "name" : "storefileIndexSizeMB", + "type" : "int" + }, { + "name" : "storefiles", + "type" : "int" + }, { + "name" : "storefileSizeMB", + "type" : "int" + }, { + "name" : "stores", + "type" : "int" + } ] + }, { + "type" : "record", + "name" : "AServerLoad", + "fields" : [ { + "name" : "load", + "type" : "int" + }, { + "name" : "maxHeapMB", + "type" : "int" + }, { + "name" : "memStoreSizeInMB", + "type" : "int" + }, { + "name" : "numberOfRegions", + "type" : "int" + }, { + "name" : "numberOfRequests", + "type" : "int" + }, { + "name" : "regionsLoad", + "type" : { + "type" : "array", + "items" : "ARegionLoad" + } + }, { + "name" : "storefileIndexSizeInMB", + "type" : "int" + }, { + "name" : "storefiles", + "type" : "int" + }, { + "name" : "storefileSizeInMB", + "type" : "int" + }, { + "name" : "usedHeapMB", + "type" : "int" + } ] + }, { + "type" : "record", + "name" : "AServerInfo", + "fields" : [ { + "name" : "infoPort", + "type" : "int" + }, { + "name" : "load", + "type" : "AServerLoad" + }, { + "name" : "serverAddress", + "type" : "AServerAddress" + }, { + "name" : "serverName", + "type" : "string" + }, { + "name" : "startCode", + "type" : "long" + } ] + }, { + "type" : "record", + "name" : "AClusterStatus", + "fields" : [ { + "name" : "averageLoad", + "type" : "double" + }, { + "name" : "deadServerNames", + "type" : { + "type" : "array", + "items" : "string" + } + }, { + "name" : "deadServers", + "type" : "int" + }, { + "name" : "hbaseVersion", + "type" : "string" + }, { + "name" : "regionsCount", + "type" : "int" + }, { + "name" : "requestsCount", + "type" : "int" + }, { + "name" : "serverInfos", + "type" : { + "type" : "array", + "items" : "AServerInfo" + } + }, { + "name" : "servers", + "type" : "int" + } ] + }, { + "type" : "enum", + "name" : "ACompressionAlgorithm", + "symbols" : [ "LZO", "GZ", "NONE" ] + }, { + "type" : "record", + "name" : "AFamilyDescriptor", + "fields" : [ { + "name" : "name", + "type" : "bytes" + }, { + "name" : "compression", + "type" : [ "ACompressionAlgorithm", "null" ] + }, { + "name" : "maxVersions", + "type" : [ "int", "null" ] + }, { + "name" : "blocksize", + "type" : [ "int", "null" ] + }, { + "name" : "inMemory", + "type" : [ "boolean", "null" ] + }, { + "name" : "timeToLive", + "type" : [ "int", "null" ] + }, { + "name" : "blockCacheEnabled", + "type" : [ "boolean", "null" ] + } ] + }, { + "type" : "record", + "name" : "ATableDescriptor", + "fields" : [ { + "name" : "name", + "type" : "bytes" + }, { + "name" : "families", + "type" : [ { + "type" : "array", + "items" : "AFamilyDescriptor" + }, "null" ] + }, { + "name" : "maxFileSize", + "type" : [ "long", "null" ] + }, { + "name" : "memStoreFlushSize", + "type" : [ "long", "null" ] + }, { + "name" : "rootRegion", + "type" : [ "boolean", "null" ] + }, { + "name" : "metaRegion", + "type" : [ "boolean", "null" ] + }, { + "name" : "metaTable", + "type" : [ "boolean", "null" ] + }, { + "name" : "readOnly", + "type" : [ "boolean", "null" ] + }, { + "name" : "deferredLogFlush", + "type" : [ "boolean", "null" ] + } ] + }, { + "type" : "record", + "name" : "AColumn", + "fields" : [ { + "name" : "family", + "type" : "bytes" + }, { + "name" : "qualifier", + "type" : [ "bytes", "null" ] + } ] + }, { + "type" : "record", + "name" : "ATimeRange", + "fields" : [ { + "name" : "minStamp", + "type" : "long" + }, { + "name" : "maxStamp", + "type" : "long" + } ] + }, { + "type" : "record", + "name" : "AGet", + "fields" : [ { + "name" : "row", + "type" : "bytes" + }, { + "name" : "columns", + "type" : [ { + "type" : "array", + "items" : "AColumn" + }, "null" ] + }, { + "name" : "timestamp", + "type" : [ "long", "null" ] + }, { + "name" : "timerange", + "type" : [ "ATimeRange", "null" ] + }, { + "name" : "maxVersions", + "type" : [ "int", "null" ] + } ] + }, { + "type" : "record", + "name" : "AResultEntry", + "fields" : [ { + "name" : "family", + "type" : "bytes" + }, { + "name" : "qualifier", + "type" : "bytes" + }, { + "name" : "value", + "type" : "bytes" + }, { + "name" : "timestamp", + "type" : "long" + } ] + }, { + "type" : "record", + "name" : "AResult", + "fields" : [ { + "name" : "row", + "type" : "bytes" + }, { + "name" : "entries", + "type" : { + "type" : "array", + "items" : "AResultEntry" + } + } ] + }, { + "type" : "record", + "name" : "AColumnValue", + "fields" : [ { + "name" : "family", + "type" : "bytes" + }, { + "name" : "qualifier", + "type" : "bytes" + }, { + "name" : "value", + "type" : "bytes" + }, { + "name" : "timestamp", + "type" : [ "long", "null" ] + } ] + }, { + "type" : "record", + "name" : "APut", + "fields" : [ { + "name" : "row", + "type" : "bytes" + }, { + "name" : "columnValues", + "type" : { + "type" : "array", + "items" : "AColumnValue" + } + } ] + }, { + "type" : "record", + "name" : "ADelete", + "fields" : [ { + "name" : "row", + "type" : "bytes" + }, { + "name" : "columns", + "type" : [ { + "type" : "array", + "items" : "AColumn" + }, "null" ] + } ] + }, { + "type" : "record", + "name" : "AScan", + "fields" : [ { + "name" : "startRow", + "type" : [ "bytes", "null" ] + }, { + "name" : "stopRow", + "type" : [ "bytes", "null" ] + }, { + "name" : "columns", + "type" : [ { + "type" : "array", + "items" : "AColumn" + }, "null" ] + }, { + "name" : "timestamp", + "type" : [ "long", "null" ] + }, { + "name" : "timerange", + "type" : [ "ATimeRange", "null" ] + }, { + "name" : "maxVersions", + "type" : [ "int", "null" ] + } ] + }, { + "type" : "error", + "name" : "AIOError", + "fields" : [ { + "name" : "message", + "type" : "string" + } ] + }, { + "type" : "error", + "name" : "AIllegalArgument", + "fields" : [ { + "name" : "message", + "type" : "string" + } ] + }, { + "type" : "error", + "name" : "ATableExists", + "fields" : [ { + "name" : "message", + "type" : "string" + } ] + }, { + "type" : "error", + "name" : "AMasterNotRunning", + "fields" : [ { + "name" : "message", + "type" : "string" + } ] + } ], + "messages" : { + "getHBaseVersion" : { + "request" : [ ], + "response" : "string", + "errors" : [ "AIOError" ] + }, + "getClusterStatus" : { + "request" : [ ], + "response" : "AClusterStatus", + "errors" : [ "AIOError" ] + }, + "listTables" : { + "request" : [ ], + "response" : { + "type" : "array", + "items" : "ATableDescriptor" + }, + "errors" : [ "AIOError" ] + }, + "describeTable" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + } ], + "response" : "ATableDescriptor", + "errors" : [ "AIOError" ] + }, + "isTableEnabled" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + } ], + "response" : "boolean", + "errors" : [ "AIOError" ] + }, + "tableExists" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + } ], + "response" : "boolean", + "errors" : [ "AIOError" ] + }, + "describeFamily" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + }, { + "name" : "family", + "type" : "bytes" + } ], + "response" : "AFamilyDescriptor", + "errors" : [ "AIOError" ] + }, + "createTable" : { + "request" : [ { + "name" : "table", + "type" : "ATableDescriptor" + } ], + "response" : "null", + "errors" : [ "AIOError", "AIllegalArgument", "ATableExists", "AMasterNotRunning" ] + }, + "deleteTable" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + } ], + "response" : "null", + "errors" : [ "AIOError" ] + }, + "modifyTable" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + }, { + "name" : "tableDescriptor", + "type" : "ATableDescriptor" + } ], + "response" : "null", + "errors" : [ "AIOError" ] + }, + "enableTable" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + } ], + "response" : "null", + "errors" : [ "AIOError" ] + }, + "disableTable" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + } ], + "response" : "null", + "errors" : [ "AIOError" ] + }, + "flush" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + } ], + "response" : "null", + "errors" : [ "AIOError" ] + }, + "split" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + } ], + "response" : "null", + "errors" : [ "AIOError" ] + }, + "addFamily" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + }, { + "name" : "family", + "type" : "AFamilyDescriptor" + } ], + "response" : "null", + "errors" : [ "AIOError" ] + }, + "deleteFamily" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + }, { + "name" : "family", + "type" : "bytes" + } ], + "response" : "null", + "errors" : [ "AIOError" ] + }, + "modifyFamily" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + }, { + "name" : "familyName", + "type" : "bytes" + }, { + "name" : "familyDescriptor", + "type" : "AFamilyDescriptor" + } ], + "response" : "null", + "errors" : [ "AIOError" ] + }, + "get" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + }, { + "name" : "get", + "type" : "AGet" + } ], + "response" : "AResult", + "errors" : [ "AIOError" ] + }, + "exists" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + }, { + "name" : "get", + "type" : "AGet" + } ], + "response" : "boolean", + "errors" : [ "AIOError" ] + }, + "put" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + }, { + "name" : "put", + "type" : "APut" + } ], + "response" : "null", + "errors" : [ "AIOError" ] + }, + "delete" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + }, { + "name" : "delete", + "type" : "ADelete" + } ], + "response" : "null", + "errors" : [ "AIOError" ] + }, + "incrementColumnValue" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + }, { + "name" : "row", + "type" : "bytes" + }, { + "name" : "family", + "type" : "bytes" + }, { + "name" : "qualifier", + "type" : "bytes" + }, { + "name" : "amount", + "type" : "long" + }, { + "name" : "writeToWAL", + "type" : "boolean" + } ], + "response" : "long", + "errors" : [ "AIOError" ] + }, + "scannerOpen" : { + "request" : [ { + "name" : "table", + "type" : "bytes" + }, { + "name" : "scan", + "type" : "AScan" + } ], + "response" : "int", + "errors" : [ "AIOError" ] + }, + "scannerClose" : { + "request" : [ { + "name" : "scannerId", + "type" : "int" + } ], + "response" : "null", + "errors" : [ "AIOError", "AIllegalArgument" ] + }, + "scannerGetRows" : { + "request" : [ { + "name" : "scannerId", + "type" : "int" + }, { + "name" : "numberOfRows", + "type" : "int" + } ], + "response" : { + "type" : "array", + "items" : "AResult" + }, + "errors" : [ "AIOError", "AIllegalArgument" ] + } + } +} diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/common/TaskMonitorTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/common/TaskMonitorTmpl.jamon new file mode 100644 index 0000000..4379ef5 --- /dev/null +++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/common/TaskMonitorTmpl.jamon @@ -0,0 +1,93 @@ +<%doc> +Copyright 2011 The Apache Software Foundation + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +<%import> +java.util.*; +org.apache.hadoop.hbase.monitoring.*; +org.apache.hadoop.util.StringUtils; + +<%args> +TaskMonitor taskMonitor = TaskMonitor.get(); +String filter = "general"; +String format = "html"; + +<%java> +List tasks = taskMonitor.getTasks(); +Iterator iter = tasks.iterator(); +// apply requested filter +while (iter.hasNext()) { + MonitoredTask t = iter.next(); + if (filter.equals("general")) { + if (t instanceof MonitoredRPCHandler) + iter.remove(); + } else if (filter.equals("handler")) { + if (!(t instanceof MonitoredRPCHandler)) + iter.remove(); + } else if (filter.equals("rpc")) { + if (!(t instanceof MonitoredRPCHandler) || + !((MonitoredRPCHandler) t).isRPCRunning()) + iter.remove(); + } else if (filter.equals("operation")) { + if (!(t instanceof MonitoredRPCHandler) || + !((MonitoredRPCHandler) t).isOperationRunning()) + iter.remove(); + } +} +long now = System.currentTimeMillis(); +Collections.reverse(tasks); +boolean first = true; + +<%if format.equals("json")%> +[<%for MonitoredTask task : tasks%><%if first%><%java>first = false;<%else>,<% task.toJSON() %>] +<%else> +

Tasks

+ + <%if tasks.isEmpty()%> + No tasks currently running on this node. + <%else> + + + + + + + + <%for MonitoredTask task : tasks %> + + + + + + + +
Start TimeDescriptionStateStatus
<% new Date(task.getStartTime()) %><% task.getDescription() %><% task.getState() %> + (since <% StringUtils.formatTimeDiff(now, task.getStateTime()) %> ago) + <% task.getStatus() %> + (since <% StringUtils.formatTimeDiff(now, task.getStatusTime()) %> + ago)
+ + + diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon new file mode 100644 index 0000000..0dc0691 --- /dev/null +++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon @@ -0,0 +1,70 @@ +<%doc> +Copyright 2011 The Apache Software Foundation + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +<%import> +org.apache.hadoop.hbase.HRegionInfo; +org.apache.hadoop.hbase.master.AssignmentManager; +org.apache.hadoop.hbase.master.AssignmentManager.RegionState; +java.util.Iterator; +java.util.Map; + +<%args> +AssignmentManager assignmentManager; +int limit = 100; + +<%java> +Map rit = assignmentManager.getRegionsInTransition(); + +int toRemove = rit.size() - limit; +int removed = 0; +if (toRemove > 0) { + // getRegionsInTransition returned a copy, so we can mutate it + for (Iterator> it = rit.entrySet().iterator(); + it.hasNext() && toRemove > 0; + ) { + Map.Entry e = it.next(); + if (HRegionInfo.FIRST_META_REGIONINFO.getEncodedName().equals( + e.getKey()) || + HRegionInfo.ROOT_REGIONINFO.getEncodedName().equals( + e.getKey())) { + // don't remove the meta regions, they're too interesting! + continue; + } + it.remove(); + toRemove--; + removed++; + } +} + + + +

Regions in Transition

+<%if rit.isEmpty() %> +No regions in transition. +<%else> + + + <%for Map.Entry entry : rit.entrySet() %> + + +
RegionState
<% entry.getKey() %><% entry.getValue().toDescriptiveString() %>
+ <%if removed > 0 %> + (<% removed %> more regions in transition not shown) + + \ No newline at end of file diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon new file mode 100644 index 0000000..097934e --- /dev/null +++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon @@ -0,0 +1,287 @@ +<%doc> +Copyright 2011 The Apache Software Foundation + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +<%args> +HMaster master; +HBaseAdmin admin; +Map frags = null; +ServerName rootLocation = null; +ServerName metaLocation = null; +List servers = null; +Set deadServers = null; +boolean showAppendWarning = false; +String filter = "general"; +String format = "html"; + +<%import> +java.util.*; +org.apache.hadoop.util.StringUtils; +org.apache.hadoop.hbase.util.Bytes; +org.apache.hadoop.hbase.util.JvmVersion; +org.apache.hadoop.hbase.util.FSUtils; +org.apache.hadoop.hbase.master.HMaster; +org.apache.hadoop.hbase.HConstants; +org.apache.hadoop.hbase.HServerLoad; +org.apache.hadoop.hbase.ServerName; +org.apache.hadoop.hbase.client.HBaseAdmin; +org.apache.hadoop.hbase.client.HConnectionManager; +org.apache.hadoop.hbase.HTableDescriptor; +org.apache.hadoop.hbase.HBaseConfiguration; +org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; + +<%if format.equals("json") %> + <& ../common/TaskMonitorTmpl; filter = filter; format = "json" &> + <%java return; %> + + + + + +HBase Master: <% master.getServerName() %> + + + + +

Master: <% master.getServerName().getHostname() %>:<% master.getServerName().getPort() %>

+ + + +<%if JvmVersion.isBadJvmVersion() %> +
+ Your current JVM version <% System.getProperty("java.version") %> is known to be + unstable with HBase. Please see the + HBase wiki + for details. +
+ +<%if showAppendWarning %> +
+ You are currently running the HMaster without HDFS append support enabled. + This may result in data loss. + Please see the HBase wiki + for details. +
+ + +
+

Attributes

+ + + + + + + + + +<%if frags != null %> + + + + + + + + + +
Attribute NameValueDescription
HBase Version<% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %>HBase version and revision
HBase Compiled<% org.apache.hadoop.hbase.util.VersionInfo.getDate() %>, <% org.apache.hadoop.hbase.util.VersionInfo.getUser() %>When HBase version was compiled and by whom
Hadoop Version<% org.apache.hadoop.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.util.VersionInfo.getRevision() %>Hadoop version and revision
Hadoop Compiled<% org.apache.hadoop.util.VersionInfo.getDate() %>, <% org.apache.hadoop.util.VersionInfo.getUser() %>When Hadoop version was compiled and by whom
HBase Root Directory<% FSUtils.getRootDir(master.getConfiguration()).toString() %>Location of HBase home directory
HBase Cluster ID<% master.getClusterId() != null ? master.getClusterId() : "Not set" %>Unique identifier generated for each HBase cluster
Load average<% StringUtils.limitDecimalTo2(master.getServerManager().getAverageLoad()) %>Average number of regions per regionserver. Naive computation.
Fragmentation<% frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %>Overall fragmentation of all tables, including .META. and -ROOT-.
Zookeeper Quorum<% master.getZooKeeperWatcher().getQuorum() %>Addresses of all registered ZK servers. For more, see zk dump.
+ Coprocessors<% java.util.Arrays.toString(master.getCoprocessors()) %> + Coprocessors currently loaded loaded by the master
HMaster Start Time<% new Date(master.getMasterStartTime()) %>Date stamp of when this HMaster was started
HMaster Active Time<% new Date(master.getMasterActiveTime()) %>Date stamp of when this HMaster became active
+ +<& ../common/TaskMonitorTmpl; filter = filter &> + +<%if (rootLocation != null) %> +<& catalogTables &> + +<%if (metaLocation != null) %> +<& userTables &> + +<& userSnapshots &> +<%if (servers != null) %> +<& regionServers &> + +<%if (deadServers != null) %> +<& deadRegionServers &> + + +<& AssignmentManagerStatusTmpl; assignmentManager=master.getAssignmentManager()&> + + + + + +<%def catalogTables> +

Tables

+ + + + <%if (frags != null) %> + + + + + + + <%if (frags != null)%> + + + + + <%if (metaLocation != null) %> + + + <%if (frags != null)%> + + + + + + +
Catalog TableFrag.Description
<% Bytes.toString(HConstants.ROOT_TABLE_NAME) %><% frags.get("-ROOT-") != null ? frags.get("-ROOT-").intValue() + "%" : "n/a" %>The -ROOT- table holds references to all .META. regions.
<% Bytes.toString(HConstants.META_TABLE_NAME) %><% frags.get(".META.") != null ? frags.get(".META.").intValue() + "%" : "n/a" %>The .META. table holds references to all User Table regions
+ + +<%def userTables> +<%java> + HTableDescriptor[] tables = admin.listTables(); + +<%if (tables != null && tables.length > 0)%> + + + +<%if (frags != null) %> + + + + + +<%for HTableDescriptor htDesc : tables%> + + + <%if (frags != null) %> + + + + + + +

<% tables.length %> table(s) in set. [Details]

+
User TableFrag.Online RegionsDescription
><% htDesc.getNameAsString() %> <% frags.get(htDesc.getNameAsString()) != null ? frags.get(htDesc.getNameAsString()).intValue() + "%" : "n/a" %><% master.getAssignmentManager().getRegionsOfTable(htDesc.getName()).size() %> + <% htDesc.toStringCustomizedValues() %>
+ + + +<%def userSnapshots> +<%java> + List snapshots = admin.listSnapshots(); + +<%if (snapshots != null && snapshots.size() > 0)%> + + + + + + + +<%for SnapshotDescription snapshotDesc : snapshots%> + + + + + + + + +

<% snapshots.size() %> snapshot(s) in set.

+
SnapshotTableCreation TimeType
<% snapshotDesc.getName() %><% snapshotDesc.getTable() %><% new Date(snapshotDesc.getCreationTime()) %><% snapshotDesc.getType() %>
+ + + +<%def regionServers> +

Region Servers

+<%if (servers != null && servers.size() > 0)%> +<%java> + int totalRegions = 0; + int totalRequests = 0; + + + + +<%java> + ServerName [] serverNames = servers.toArray(new ServerName[servers.size()]); + Arrays.sort(serverNames); + for (ServerName serverName: serverNames) { + // TODO: this is incorrect since this conf might differ from RS to RS + // or be set to 0 to get ephemeral ports + int infoPort = master.getConfiguration().getInt("hbase.regionserver.info.port", 60030); + String url = "http://" + serverName.getHostname() + ":" + infoPort + "/"; + HServerLoad hsl = master.getServerManager().getLoad(serverName); + String loadStr = hsl == null? "-": hsl.toString(); + if (hsl != null) { + totalRegions += hsl.getNumberOfRegions(); + totalRequests += hsl.getNumberOfRequests(); + } + long startcode = serverName.getStartcode(); + + +<%java> + } + + +
ServerNameStart timeLoad
<% serverName %><% new Date(startcode) %><% loadStr %>
Total: servers: <% servers.size() %>requestsPerSecond=<% totalRequests %>, numberOfOnlineRegions=<% totalRegions %>
+ +

Load is requests per second and count of regions loaded

+ + + +<%def deadRegionServers> +

Dead Region Servers

+<%if (deadServers != null && deadServers.size() > 0)%> + + + +<%java> + ServerName [] deadServerNames = deadServers.toArray(new ServerName[deadServers.size()]); + Arrays.sort(deadServerNames); + for (ServerName deadServerName: deadServerNames) { + int infoPort = master.getConfiguration().getInt("hbase.regionserver.info.port", 60030); + + +<%java> + } + + +
ServerName
<% deadServerName %>
Total: servers: <% deadServers.size() %>
+ + + +<%java> + HConnectionManager.deleteConnection(admin.getConfiguration(), false); + diff --git a/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon new file mode 100644 index 0000000..ae76204 --- /dev/null +++ b/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon @@ -0,0 +1,150 @@ +<%doc> +Copyright 2011 The Apache Software Foundation + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +<%args> +HRegionServer regionServer; +String filter = "general"; +String format = "html"; + +<%import> +java.util.*; +java.io.IOException; +org.apache.hadoop.io.Text; +org.apache.hadoop.hbase.regionserver.HRegionServer; +org.apache.hadoop.hbase.regionserver.HRegion; +org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics; +org.apache.hadoop.hbase.util.Bytes; +org.apache.hadoop.hbase.HConstants; +org.apache.hadoop.hbase.HServerInfo; +org.apache.hadoop.hbase.HServerLoad; +org.apache.hadoop.hbase.HRegionInfo; +org.apache.hadoop.hbase.ServerName; +org.apache.hadoop.hbase.HBaseConfiguration; + +<%if format.equals("json") %> + <& ../common/TaskMonitorTmpl; filter = filter; format = "json" &> + <%java return; %> + +<%java> + HServerInfo serverInfo = null; + ServerName serverName = null; + try { + serverInfo = regionServer.getHServerInfo(); + serverName = regionServer.getServerName(); + } catch (IOException e) { + e.printStackTrace(); + } + RegionServerMetrics metrics = regionServer.getMetrics(); + List onlineRegions = regionServer.getOnlineRegions(); + int interval = regionServer.getConfiguration().getInt("hbase.regionserver.msginterval", 3000)/1000; + int masterInfoPort = regionServer.getConfiguration().getInt("hbase.master.info.port", 60010); + + + + + +HBase Region Server: <% serverName %>:<% serverInfo.getServerAddress().getPort() %> + + + + + +

RegionServer: <% serverName %>

+ +
+ +

Attributes

+ ++++ + + + + + + + + + + + +
Attribute NameValueDescription
HBase Version<% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %>HBase version and revision
HBase Compiled<% org.apache.hadoop.hbase.util.VersionInfo.getDate() %>, <% org.apache.hadoop.hbase.util.VersionInfo.getUser() %>When HBase version was compiled and by whom
Metrics<% metrics.toString() %>RegionServer Metrics; file and heap sizes are in megabytes
Zookeeper Quorum<% regionServer.getZooKeeper().getQuorum() %>Addresses of all registered ZK servers
Coprocessors + <% java.util.Arrays.toString(regionServer.getCoprocessors()) %> + Coprocessors currently loaded by this regionserver
RS Start Time<% new Date(regionServer.getStartcode()) %>Date stamp of when this region server was started
HBase Master +<%if (masterInfoPort < 0) %> +No hbase.master.info.port found +<%else> +<%java> +String host = regionServer.getMasterAddressManager().getMasterAddress().getHostname() + ":" + masterInfoPort; +String url = "http://" + host + "/"; + +<% host %> + +Address of HBase Master
+ +<& ../common/TaskMonitorTmpl; filter = filter &> + +

Regions

+<%if (onlineRegions != null && onlineRegions.size() > 0) %> + + +<%java> + Collections.sort(onlineRegions); + +<%for HRegionInfo r: onlineRegions %> +<%java> + HServerLoad.RegionLoad load = regionServer.createRegionLoad(r.getEncodedName()); + + + + + + +
Region NameStart KeyEnd KeyMetrics
<% r.getRegionNameAsString() %><% Bytes.toStringBinary(r.getStartKey()) %><% Bytes.toStringBinary(r.getEndKey()) %><% load == null? "null": load.toString() %>
+

Region names are made of the containing table's name, a comma, +the start key, a comma, and a randomly generated region id. To illustrate, +the region named +domains,apache.org,5464829424211263407 is party to the table +domains, has an id of 5464829424211263407 and the first key +in the region is apache.org. The -ROOT- +and .META. 'tables' are internal sytem tables (or 'catalog' tables in db-speak). +The -ROOT- keeps a list of all regions in the .META. table. The .META. table +keeps a list of all regions in the system. The empty key is used to denote +table start and table end. A region with an empty start key is the first region in a table. +If region has both an empty start and an empty end key, its the only region in the table. See +HBase Home for further explication.

+<%else> +

Not serving regions

+ + + diff --git a/src/main/java/org/apache/hadoop/hbase/Abortable.java b/src/main/java/org/apache/hadoop/hbase/Abortable.java new file mode 100644 index 0000000..03249c1 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/Abortable.java @@ -0,0 +1,43 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +/** + * Interface to support the aborting of a given server or client. + *

+ * This is used primarily for ZooKeeper usage when we could get an unexpected + * and fatal exception, requiring an abort. + *

+ * Implemented by the Master, RegionServer, and TableServers (client). + */ +public interface Abortable { + /** + * Abort the server or client. + * @param why Why we're aborting. + * @param e Throwable that caused abort. Can be null. + */ + public void abort(String why, Throwable e); + + /** + * Check if the server or client was aborted. + * @return true if the server or client was aborted, false otherwise + */ + public boolean isAborted(); +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/BaseConfigurable.java b/src/main/java/org/apache/hadoop/hbase/BaseConfigurable.java new file mode 100644 index 0000000..36efb50 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/BaseConfigurable.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; + +/** + * HBase version of Hadoop's Configured class that doesn't initialize the + * configuration via {@link #setConf(Configuration)} in the constructor, but + * only sets the configuration through the {@link #setConf(Configuration)} + * method + */ +public class BaseConfigurable implements Configurable { + + private Configuration conf; + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return this.conf; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/Chore.java b/src/main/java/org/apache/hadoop/hbase/Chore.java new file mode 100644 index 0000000..1581b0c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/Chore.java @@ -0,0 +1,129 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.util.HasThread; +import org.apache.hadoop.hbase.util.Sleeper; + +/** + * Chore is a task performed on a period in hbase. The chore is run in its own + * thread. This base abstract class provides while loop and sleeping facility. + * If an unhandled exception, the threads exit is logged. + * Implementers just need to add checking if there is work to be done and if + * so, do it. Its the base of most of the chore threads in hbase. + * + *

Don't subclass Chore if the task relies on being woken up for something to + * do, such as an entry being added to a queue, etc. + */ +public abstract class Chore extends HasThread { + private final Log LOG = LogFactory.getLog(this.getClass()); + private final Sleeper sleeper; + protected final Stoppable stopper; + + /** + * @param p Period at which we should run. Will be adjusted appropriately + * should we find work and it takes time to complete. + * @param stopper When {@link Stoppable#isStopped()} is true, this thread will + * cleanup and exit cleanly. + */ + public Chore(String name, final int p, final Stoppable stopper) { + super(name); + this.sleeper = new Sleeper(p, stopper); + this.stopper = stopper; + } + + /** + * @see java.lang.Thread#run() + */ + @Override + public void run() { + try { + boolean initialChoreComplete = false; + while (!this.stopper.isStopped()) { + long startTime = System.currentTimeMillis(); + try { + if (!initialChoreComplete) { + initialChoreComplete = initialChore(); + } else { + chore(); + } + } catch (Exception e) { + LOG.error("Caught exception", e); + if (this.stopper.isStopped()) { + continue; + } + } + this.sleeper.sleep(startTime); + } + } catch (Throwable t) { + LOG.fatal(getName() + "error", t); + } finally { + LOG.info(getName() + " exiting"); + cleanup(); + } + } + + /** + * If the thread is currently sleeping, trigger the core to happen immediately. + * If it's in the middle of its operation, will begin another operation + * immediately after finishing this one. + */ + public void triggerNow() { + this.sleeper.skipSleepCycle(); + } + + /* + * Exposed for TESTING! + * calls directly the chore method, from the current thread. + */ + public void choreForTesting() { + chore(); + } + + /** + * Override to run a task before we start looping. + * @return true if initial chore was successful + */ + protected boolean initialChore() { + // Default does nothing. + return true; + } + + /** + * Look for chores. If any found, do them else just return. + */ + protected abstract void chore(); + + /** + * Sleep for period. + */ + protected void sleep() { + this.sleeper.sleep(); + } + + /** + * Called when the chore has completed, allowing subclasses to cleanup any + * extra overhead + */ + protected void cleanup() { + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java b/src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java new file mode 100644 index 0000000..5c51e4b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java @@ -0,0 +1,33 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +/** + * This exception is thrown by the master when a region server clock skew is + * too high. + */ +@SuppressWarnings("serial") +public class ClockOutOfSyncException extends IOException { + public ClockOutOfSyncException(String message) { + super(message); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java b/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java new file mode 100644 index 0000000..d962594 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java @@ -0,0 +1,347 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.VersionMismatchException; +import org.apache.hadoop.io.VersionedWritable; + +/** + * Status information on the HBase cluster. + *

+ * ClusterStatus provides clients with information such as: + *

    + *
  • The count and names of region servers in the cluster.
    • + *
  • The count and names of dead region servers in the cluster.
    • + *
  • The name of the active master for the cluster.
    • + *
  • The name(s) of the backup master(s) for the cluster, if they exist.
    • + *
  • The average cluster load.
    • + *
  • The number of regions deployed on the cluster.
    • + *
  • The number of requests since last report.
    • + *
  • Detailed region server loading and resource usage information, + * per server and per region.
    • + *
  • Regions in transition at master
    • + *
  • The unique cluster ID
    • + *
+ */ +public class ClusterStatus extends VersionedWritable { + /** + * Version for object serialization. Incremented for changes in serialized + * representation. + *
+ *
0
Initial version
+ *
1
Added cluster ID
+ *
2
Added Map of ServerName to ServerLoad
+ *
3
Added master and backupMasters
+ *
+ */ + private static final byte VERSION_MASTER_BACKUPMASTERS = 2; + private static final byte VERSION = 2; + + private String hbaseVersion; + private Map liveServers; + private Collection deadServers; + private ServerName master; + private Collection backupMasters; + private Map intransition; + private String clusterId; + private String[] masterCoprocessors; + + /** + * Constructor, for Writable + */ + public ClusterStatus() { + super(); + } + + public ClusterStatus(final String hbaseVersion, final String clusterid, + final Map servers, + final Collection deadServers, + final ServerName master, + final Collection backupMasters, + final Map rit, + final String[] masterCoprocessors) { + this.hbaseVersion = hbaseVersion; + this.liveServers = servers; + this.deadServers = deadServers; + this.master = master; + this.backupMasters = backupMasters; + this.intransition = rit; + this.clusterId = clusterid; + this.masterCoprocessors = masterCoprocessors; + } + + /** + * @return the names of region servers on the dead list + */ + public Collection getDeadServerNames() { + return Collections.unmodifiableCollection(deadServers); + } + + /** + * @return the number of region servers in the cluster + */ + public int getServersSize() { + return liveServers.size(); + } + + /** + * @return the number of dead region servers in the cluster + */ + public int getDeadServers() { + return deadServers.size(); + } + + /** + * @return the average cluster load + */ + public double getAverageLoad() { + int load = getRegionsCount(); + return (double)load / (double)getServersSize(); + } + + /** + * @return the number of regions deployed on the cluster + */ + public int getRegionsCount() { + int count = 0; + for (Map.Entry e: this.liveServers.entrySet()) { + count += e.getValue().getNumberOfRegions(); + } + return count; + } + + /** + * @return the number of requests since last report + */ + public int getRequestsCount() { + int count = 0; + for (Map.Entry e: this.liveServers.entrySet()) { + count += e.getValue().getNumberOfRequests(); + } + return count; + } + + /** + * @return the HBase version string as reported by the HMaster + */ + public String getHBaseVersion() { + return hbaseVersion; + } + + /** + * @see java.lang.Object#equals(java.lang.Object) + */ + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ClusterStatus)) { + return false; + } + return (getVersion() == ((ClusterStatus)o).getVersion()) && + getHBaseVersion().equals(((ClusterStatus)o).getHBaseVersion()) && + this.liveServers.equals(((ClusterStatus)o).liveServers) && + this.deadServers.containsAll(((ClusterStatus)o).deadServers) && + Arrays.equals(this.masterCoprocessors, + ((ClusterStatus)o).masterCoprocessors) && + this.master.equals(((ClusterStatus)o).master) && + this.backupMasters.containsAll(((ClusterStatus)o).backupMasters); + } + + /** + * @see java.lang.Object#hashCode() + */ + public int hashCode() { + return VERSION + hbaseVersion.hashCode() + this.liveServers.hashCode() + + this.deadServers.hashCode() + this.master.hashCode() + + this.backupMasters.hashCode(); + } + + /** @return the object version number */ + public byte getVersion() { + return VERSION; + } + + // + // Getters + // + + /** + * Returns detailed region server information: A list of + * {@link ServerName}. + * @return region server information + * @deprecated Use {@link #getServers()} + */ + public Collection getServerInfo() { + return getServers(); + } + + public Collection getServers() { + return Collections.unmodifiableCollection(this.liveServers.keySet()); + } + + /** + * Returns detailed information about the current master {@link ServerName}. + * @return current master information if it exists + */ + public ServerName getMaster() { + return this.master; + } + + /** + * @return the number of backup masters in the cluster + */ + public int getBackupMastersSize() { + return this.backupMasters.size(); + } + + /** + * @return the names of backup masters + */ + public Collection getBackupMasters() { + return Collections.unmodifiableCollection(this.backupMasters); + } + + /** + * @param sn + * @return Server's load or null if not found. + */ + public HServerLoad getLoad(final ServerName sn) { + return this.liveServers.get(sn); + } + + public Map getRegionsInTransition() { + return this.intransition; + } + + public String getClusterId() { + return clusterId; + } + + public String[] getMasterCoprocessors() { + return masterCoprocessors; + } + + // + // Writable + // + + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeUTF(hbaseVersion); + out.writeInt(getServersSize()); + for (Map.Entry e: this.liveServers.entrySet()) { + Bytes.writeByteArray(out, e.getKey().getVersionedBytes()); + e.getValue().write(out); + } + out.writeInt(deadServers.size()); + for (ServerName server: deadServers) { + Bytes.writeByteArray(out, server.getVersionedBytes()); + } + out.writeInt(this.intransition.size()); + for (Map.Entry e: this.intransition.entrySet()) { + out.writeUTF(e.getKey()); + e.getValue().write(out); + } + out.writeUTF(clusterId); + out.writeInt(masterCoprocessors.length); + for(String masterCoprocessor: masterCoprocessors) { + out.writeUTF(masterCoprocessor); + } + Bytes.writeByteArray(out, this.master.getVersionedBytes()); + out.writeInt(this.backupMasters.size()); + for (ServerName backupMaster: this.backupMasters) { + Bytes.writeByteArray(out, backupMaster.getVersionedBytes()); + } + } + + public void readFields(DataInput in) throws IOException { + int version = getVersion(); + try { + super.readFields(in); + } catch (VersionMismatchException e) { + /* + * No API in VersionMismatchException to get the expected and found + * versions. We use the only tool available to us: toString(), whose + * output has a dependency on hadoop-common. Boo. + */ + int startIndex = e.toString().lastIndexOf('v') + 1; + version = Integer.parseInt(e.toString().substring(startIndex)); + } + hbaseVersion = in.readUTF(); + int count = in.readInt(); + this.liveServers = new HashMap(count); + for (int i = 0; i < count; i++) { + byte [] versionedBytes = Bytes.readByteArray(in); + HServerLoad hsl = new HServerLoad(); + hsl.readFields(in); + this.liveServers.put(ServerName.parseVersionedServerName(versionedBytes), hsl); + } + count = in.readInt(); + deadServers = new ArrayList(count); + for (int i = 0; i < count; i++) { + deadServers.add(ServerName.parseVersionedServerName(Bytes.readByteArray(in))); + } + count = in.readInt(); + this.intransition = new TreeMap(); + for (int i = 0; i < count; i++) { + String key = in.readUTF(); + RegionState regionState = new RegionState(); + regionState.readFields(in); + this.intransition.put(key, regionState); + } + this.clusterId = in.readUTF(); + int masterCoprocessorsLength = in.readInt(); + masterCoprocessors = new String[masterCoprocessorsLength]; + for(int i = 0; i < masterCoprocessorsLength; i++) { + masterCoprocessors[i] = in.readUTF(); + } + // Only read extra fields for master and backup masters if + // version indicates that we should do so, else use defaults + if (version >= VERSION_MASTER_BACKUPMASTERS) { + this.master = ServerName.parseVersionedServerName( + Bytes.readByteArray(in)); + count = in.readInt(); + this.backupMasters = new ArrayList(count); + for (int i = 0; i < count; i++) { + this.backupMasters.add(ServerName.parseVersionedServerName( + Bytes.readByteArray(in))); + } + } else { + this.master = new ServerName(ServerName.UNKNOWN_SERVERNAME, -1, + ServerName.NON_STARTCODE); + this.backupMasters = new ArrayList(0); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/Coprocessor.java b/src/main/java/org/apache/hadoop/hbase/Coprocessor.java new file mode 100644 index 0000000..c0cb463 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/Coprocessor.java @@ -0,0 +1,52 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.IOException; + +/** + * Coprocess interface. + */ +public interface Coprocessor { + static final int VERSION = 1; + + /** Highest installation priority */ + static final int PRIORITY_HIGHEST = 0; + /** High (system) installation priority */ + static final int PRIORITY_SYSTEM = Integer.MAX_VALUE / 4; + /** Default installation priority for user coprocessors */ + static final int PRIORITY_USER = Integer.MAX_VALUE / 2; + /** Lowest installation priority */ + static final int PRIORITY_LOWEST = Integer.MAX_VALUE; + + /** + * Lifecycle state of a given coprocessor instance. + */ + public enum State { + UNINSTALLED, + INSTALLED, + STARTING, + ACTIVE, + STOPPING, + STOPPED + } + + // Interface + void start(CoprocessorEnvironment env) throws IOException; + + void stop(CoprocessorEnvironment env) throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java b/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java new file mode 100644 index 0000000..dbb5cc9 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java @@ -0,0 +1,52 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.HTableInterface; + +/** + * Coprocessor environment state. + */ +public interface CoprocessorEnvironment { + + /** @return the Coprocessor interface version */ + public int getVersion(); + + /** @return the HBase version as a string (e.g. "0.21.0") */ + public String getHBaseVersion(); + + /** @return the loaded coprocessor instance */ + public Coprocessor getInstance(); + + /** @return the priority assigned to the loaded coprocessor */ + public int getPriority(); + + /** @return the load sequence number */ + public int getLoadSequence(); + + /** @return the configuration */ + public Configuration getConfiguration(); + + /** + * @return an interface for accessing the given table + * @throws IOException + */ + public HTableInterface getTable(byte[] tableName) throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/DaemonThreadFactory.java b/src/main/java/org/apache/hadoop/hbase/DaemonThreadFactory.java new file mode 100644 index 0000000..d621cbf --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/DaemonThreadFactory.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Thread factory that creates daemon threads + */ +public class DaemonThreadFactory implements ThreadFactory { + static final AtomicInteger poolNumber = new AtomicInteger(1); + final ThreadGroup group; + final AtomicInteger threadNumber = new AtomicInteger(1); + final String namePrefix; + + public DaemonThreadFactory(String name) { + SecurityManager s = System.getSecurityManager(); + group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup(); + namePrefix = name + poolNumber.getAndIncrement() + "-thread-"; + } + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); + if (!t.isDaemon()) { + t.setDaemon(true); + } + if (t.getPriority() != Thread.NORM_PRIORITY) { + t.setPriority(Thread.NORM_PRIORITY); + } + return t; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java b/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java new file mode 100644 index 0000000..5a0c6d2 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java @@ -0,0 +1,52 @@ +/** + * Copyright 2008 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + + +/** + * Subclass if exception is not meant to be retried: e.g. + * {@link UnknownScannerException} + */ +public class DoNotRetryIOException extends HBaseIOException { + + private static final long serialVersionUID = 1197446454511704139L; + + /** + * default constructor + */ + public DoNotRetryIOException() { + super(); + } + + /** + * @param message + */ + public DoNotRetryIOException(String message) { + super(message); + } + + /** + * @param message + * @param cause + */ + public DoNotRetryIOException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java new file mode 100644 index 0000000..9b1d021 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java @@ -0,0 +1,41 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; +import java.io.IOException; + + +/** + * Thrown during flush if the possibility snapshot content was not properly + * persisted into store files. Response should include replay of hlog content. + */ +public class DroppedSnapshotException extends IOException { + + private static final long serialVersionUID = -5463156580831677374L; + + /** + * @param msg + */ + public DroppedSnapshotException(String msg) { + super(msg); + } + + /** + * default constructor + */ + public DroppedSnapshotException() { + super(); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/EmptyWatcher.java b/src/main/java/org/apache/hadoop/hbase/EmptyWatcher.java new file mode 100644 index 0000000..e0e0a28 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/EmptyWatcher.java @@ -0,0 +1,33 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.WatchedEvent; + +/** + * An empty ZooKeeper watcher + */ +public class EmptyWatcher implements Watcher { + public static EmptyWatcher instance = new EmptyWatcher(); + private EmptyWatcher() {} + + public void process(WatchedEvent event) {} +} diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java new file mode 100644 index 0000000..590e774 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java @@ -0,0 +1,160 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.VersionInfo; + +/** + * Adds HBase configuration files to a Configuration + */ +public class HBaseConfiguration extends Configuration { + + private static final Log LOG = LogFactory.getLog(HBaseConfiguration.class); + + // a constant to convert a fraction to a percentage + private static final int CONVERT_TO_PERCENTAGE = 100; + + /** + * Instantinating HBaseConfiguration() is deprecated. Please use + * HBaseConfiguration#create() to construct a plain Configuration + */ + @Deprecated + public HBaseConfiguration() { + //TODO:replace with private constructor, HBaseConfiguration should not extend Configuration + super(); + addHbaseResources(this); + LOG.warn("instantiating HBaseConfiguration() is deprecated. Please use" + + " HBaseConfiguration#create() to construct a plain Configuration"); + } + + /** + * Instantiating HBaseConfiguration() is deprecated. Please use + * HBaseConfiguration#create(conf) to construct a plain Configuration + */ + @Deprecated + public HBaseConfiguration(final Configuration c) { + //TODO:replace with private constructor + this(); + merge(this, c); + } + + private static void checkDefaultsVersion(Configuration conf) { + if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return; + String defaultsVersion = conf.get("hbase.defaults.for.version"); + String thisVersion = VersionInfo.getVersion(); + if (!thisVersion.equals(defaultsVersion)) { + throw new RuntimeException( + "hbase-default.xml file seems to be for and old version of HBase (" + + defaultsVersion + "), this version is " + thisVersion); + } + } + + private static void checkForClusterFreeMemoryLimit(Configuration conf) { + float globalMemstoreLimit = conf.getFloat("hbase.regionserver.global.memstore.upperLimit", 0.4f); + int gml = (int)(globalMemstoreLimit * CONVERT_TO_PERCENTAGE); + float blockCacheUpperLimit = + conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, + HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT); + int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE); + if (CONVERT_TO_PERCENTAGE - (gml + bcul) + < (int)(CONVERT_TO_PERCENTAGE * + HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD)) { + throw new RuntimeException( + "Current heap configuration for MemStore and BlockCache exceeds " + + "the threshold required for successful cluster operation. " + + "The combined value cannot exceed 0.8. Please check " + + "the settings for hbase.regionserver.global.memstore.upperLimit and " + + "hfile.block.cache.size in your configuration. " + + "hbase.regionserver.global.memstore.upperLimit is " + + globalMemstoreLimit + + " hfile.block.cache.size is " + blockCacheUpperLimit); + } + } + + public static Configuration addHbaseResources(Configuration conf) { + conf.addResource("hbase-default.xml"); + conf.addResource("hbase-site.xml"); + + checkDefaultsVersion(conf); + checkForClusterFreeMemoryLimit(conf); + return conf; + } + + /** + * Creates a Configuration with HBase resources + * @return a Configuration with HBase resources + */ + public static Configuration create() { + Configuration conf = new Configuration(); + return addHbaseResources(conf); + } + + /** + * @param that Configuration to clone. + * @return a Configuration created with the hbase-*.xml files plus + * the given configuration. + */ + public static Configuration create(final Configuration that) { + Configuration conf = create(); + merge(conf, that); + return conf; + } + + /** + * Merge two configurations. + * @param destConf the configuration that will be overwritten with items + * from the srcConf + * @param srcConf the source configuration + **/ + public static void merge(Configuration destConf, Configuration srcConf) { + for (Entry e : srcConf) { + destConf.set(e.getKey(), e.getValue()); + } + } + + /** + * + * @return whether to show HBase Configuration in servlet + */ + public static boolean isShowConfInServlet() { + boolean isShowConf = false; + try { + if (Class.forName("org.apache.hadoop.conf.ConfServlet") != null) { + isShowConf = true; + } + } catch (Exception e) { + + } + return isShowConf; + } + + /** For debugging. Dump configurations to system output as xml format. + * Master and RS configurations can also be dumped using + * http services. e.g. "curl http://master:60010/dump" + */ + public static void main(String[] args) throws Exception { + HBaseConfiguration.create().writeXml(System.out); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java new file mode 100644 index 0000000..1a864c2 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HBaseFileSystem.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.regionserver.wal.HLogFileSystem; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.Threads; + +/** + * An abstraction of the underlying filesystem. This is used by other entities such as + * {@link HLogFileSystem}, to make calls to the underlying filesystem. + * + */ +public abstract class HBaseFileSystem { + + public static final Log LOG = LogFactory.getLog(HBaseFileSystem.class); + + /** + * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the + * client level. + */ + protected static int hdfsClientRetriesNumber; + private static int baseSleepBeforeRetries; + private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10; + private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000; + // This static block is added for performance reasons. This is to ensure we are not checking + // in the method calls whether retry properties are set or not. Refer to HBase-8288 for more + // context. + static { + setRetryCounts(HBaseConfiguration.create()); + } + + /** + * Deletes a file. Assumes the user has already checked for this directory existence. + * @param fs + * @param dir + * @return true if the directory is deleted. + * @throws IOException + */ + public static boolean deleteFileFromFileSystem(FileSystem fs, Path dir) + throws IOException { + IOException lastIOE = null; + int i = 0; + do { + try { + return fs.delete(dir, false); + } catch (IOException ioe) { + lastIOE = ioe; + if (!fs.exists(dir)) return true; + // dir is there, retry deleting after some time. + sleepBeforeRetry("Delete File", i + 1); + } + } while (++i <= hdfsClientRetriesNumber); + throw new IOException("Exception in deleteFileFromFileSystem", lastIOE); + } + + + /** + * Deletes a directory. Assumes the user has already checked for this directory existence. + * @param fs + * @param dir + * @return true if the directory is deleted. + * @throws IOException + */ + public static boolean deleteDirFromFileSystem(FileSystem fs, Path dir) + throws IOException { + IOException lastIOE = null; + int i = 0; + do { + try { + return fs.delete(dir, true); + } catch (IOException ioe) { + lastIOE = ioe; + if (!fs.exists(dir)) return true; + // dir is there, retry deleting after some time. + sleepBeforeRetry("Delete Dir", i + 1); + } + } while (++i <= hdfsClientRetriesNumber); + throw new IOException("Exception in deleteDirFromFileSystem", lastIOE); + } + + protected static void setRetryCounts(Configuration conf) { + hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number", + DEFAULT_HDFS_CLIENT_RETRIES_NUMBER); + baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries", + DEFAULT_BASE_SLEEP_BEFORE_RETRIES); + } + + /** + * Creates a directory for a filesystem and configuration object. Assumes the user has already + * checked for this directory existence. + * @param fs + * @param dir + * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks + * whether the directory exists or not, and returns true if it exists. + * @throws IOException + */ + public static boolean makeDirOnFileSystem(FileSystem fs, Path dir) + throws IOException { + int i = 0; + IOException lastIOE = null; + do { + try { + return fs.mkdirs(dir); + } catch (IOException ioe) { + lastIOE = ioe; + if (fs.exists(dir)) return true; // directory is present + sleepBeforeRetry("Create Directory", i+1); + } + } while (++i <= hdfsClientRetriesNumber); + throw new IOException("Exception in makeDirOnFileSystem", lastIOE); + } + + /** + * Renames a directory. Assumes the user has already checked for this directory existence. + * @param fs + * @param src + * @param dst + * @return true if the directory is renamed. + * @throws IOException + */ + public static boolean renameDirForFileSystem(FileSystem fs, Path src, Path dst) + throws IOException { + IOException lastIOE = null; + int i = 0; + do { + try { + return fs.rename(src, dst); + } catch (IOException ioe) { + lastIOE = ioe; + if (!fs.exists(src) && fs.exists(dst)) return true; + // src is there, retry renaming after some time. + sleepBeforeRetry("Rename Directory", i + 1); + } + } while (++i <= hdfsClientRetriesNumber); + throw new IOException("Exception in renameDirForFileSystem", lastIOE); + } + + /** + * Creates a path on the file system. Checks whether the path exists already or not, and use it + * for retrying in case underlying fs throws an exception. + * If the dir already exists and overwrite flag is false, the underlying FileSystem throws + * an IOE. It is not retried and the IOE is re-thrown to the caller. + * @param fs + * @param dir + * @param overwrite + * @return + * @throws IOException + */ + public static FSDataOutputStream createPathOnFileSystem(FileSystem fs, Path dir, + boolean overwrite) throws IOException { + int i = 0; + boolean existsBefore = fs.exists(dir); + IOException lastIOE = null; + do { + try { + return fs.create(dir, overwrite); + } catch (IOException ioe) { + lastIOE = ioe; + if (existsBefore && !overwrite) throw ioe;// a legitimate exception + sleepBeforeRetry("Create Path", i + 1); + } + } while (++i <= hdfsClientRetriesNumber); + throw new IOException("Exception in createPathOnFileSystem", lastIOE); + } + + /** + * Creates the specified file with the given permission. + * If the dir already exists and the overwrite flag is false, underlying FileSystem throws + * an IOE. It is not retried and the IOE is re-thrown to the caller. + * @param fs + * @param path + * @param perm + * @param overwrite + * @return + * @throws IOException + */ + public static FSDataOutputStream createPathWithPermsOnFileSystem(FileSystem fs, + Path path, FsPermission perm, boolean overwrite) throws IOException { + int i = 0; + IOException lastIOE = null; + boolean existsBefore = fs.exists(path); + do { + try { + return fs.create(path, perm, overwrite, FSUtils.getDefaultBufferSize(fs), + FSUtils.getDefaultReplication(fs, path), FSUtils.getDefaultBlockSize(fs, path), null); + } catch (IOException ioe) { + lastIOE = ioe; + if (existsBefore && !overwrite) throw ioe;// a legitimate exception + sleepBeforeRetry("Create Path with Perms", i + 1); + } + } while (++i <= hdfsClientRetriesNumber); + throw new IOException("Exception in createPathWithPermsOnFileSystem", lastIOE); + } + +/** + * Creates the file. Assumes the user has already checked for this file existence. + * @param fs + * @param dir + * @return result true if the file is created with this call, false otherwise. + * @throws IOException + */ + public static boolean createNewFileOnFileSystem(FileSystem fs, Path file) + throws IOException { + int i = 0; + IOException lastIOE = null; + do { + try { + return fs.createNewFile(file); + } catch (IOException ioe) { + lastIOE = ioe; + if (fs.exists(file)) return true; // file exists now, return true. + sleepBeforeRetry("Create NewFile", i + 1); + } + } while (++i <= hdfsClientRetriesNumber); + throw new IOException("Exception in createNewFileOnFileSystem", lastIOE); + } + + /** + * sleeping logic for static methods; handles the interrupt exception. Keeping a static version + * for this to avoid re-looking for the integer values. + */ + protected static void sleepBeforeRetry(String msg, int sleepMultiplier) { + if (sleepMultiplier > hdfsClientRetriesNumber) { + LOG.warn(msg + ", retries exhausted"); + return; + } + LOG.info(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier); + Threads.sleep(baseSleepBeforeRetries * sleepMultiplier); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/HBaseIOException.java b/src/main/java/org/apache/hadoop/hbase/HBaseIOException.java new file mode 100644 index 0000000..fb1a17d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HBaseIOException.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +/** + * All hbase specific IOExceptions should be subclasses of HBaseIOException + */ +public class HBaseIOException extends IOException { + + private static final long serialVersionUID = 1L; + + public HBaseIOException() { + super(); + } + + /** + * {@inheritDoc} + */ + public HBaseIOException(String message) { + super(message); + } + + /** + * {@inheritDoc} + **/ + public HBaseIOException(String message, Throwable cause) { + super(message, cause); + } + + /** + * {@inheritDoc} + */ + public HBaseIOException(Throwable cause) { + super(cause); + }} diff --git a/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java new file mode 100644 index 0000000..dca442d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java @@ -0,0 +1,1071 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; + +/** + * An HColumnDescriptor contains information about a column family such as the + * number of versions, compression settings, etc. + * + * It is used as input when creating a table or adding a column. Once set, the + * parameters that specify a column cannot be changed without deleting the + * column and recreating it. If there is data stored in the column, it will be + * deleted when the column is deleted. + */ +public class HColumnDescriptor implements WritableComparable { + // For future backward compatibility + + // Version 3 was when column names become byte arrays and when we picked up + // Time-to-live feature. Version 4 was when we moved to byte arrays, HBASE-82. + // Version 5 was when bloom filter descriptors were removed. + // Version 6 adds metadata as a map where keys and values are byte[]. + // Version 7 -- add new compression and hfile blocksize to HColumnDescriptor (HBASE-1217) + // Version 8 -- reintroduction of bloom filters, changed from boolean to enum + // Version 9 -- add data block encoding + private static final byte COLUMN_DESCRIPTOR_VERSION = (byte) 9; + + // These constants are used as FileInfo keys + public static final String COMPRESSION = "COMPRESSION"; + public static final String COMPRESSION_COMPACT = "COMPRESSION_COMPACT"; + public static final String ENCODE_ON_DISK = + "ENCODE_ON_DISK"; + public static final String DATA_BLOCK_ENCODING = + "DATA_BLOCK_ENCODING"; + public static final String BLOCKCACHE = "BLOCKCACHE"; + public static final String CACHE_DATA_ON_WRITE = "CACHE_DATA_ON_WRITE"; + public static final String CACHE_INDEX_ON_WRITE = "CACHE_INDEX_ON_WRITE"; + public static final String CACHE_BLOOMS_ON_WRITE = "CACHE_BLOOMS_ON_WRITE"; + public static final String EVICT_BLOCKS_ON_CLOSE = "EVICT_BLOCKS_ON_CLOSE"; + + /** + * Size of storefile/hfile 'blocks'. Default is {@link #DEFAULT_BLOCKSIZE}. + * Use smaller block sizes for faster random-access at expense of larger + * indices (more memory consumption). + */ + public static final String BLOCKSIZE = "BLOCKSIZE"; + + public static final String LENGTH = "LENGTH"; + public static final String TTL = "TTL"; + public static final String BLOOMFILTER = "BLOOMFILTER"; + public static final String FOREVER = "FOREVER"; + public static final String REPLICATION_SCOPE = "REPLICATION_SCOPE"; + public static final String MIN_VERSIONS = "MIN_VERSIONS"; + public static final String KEEP_DELETED_CELLS = "KEEP_DELETED_CELLS"; + + /** + * Default compression type. + */ + public static final String DEFAULT_COMPRESSION = + Compression.Algorithm.NONE.getName(); + + /** + * Default value of the flag that enables data block encoding on disk, as + * opposed to encoding in cache only. We encode blocks everywhere by default, + * as long as {@link #DATA_BLOCK_ENCODING} is not NONE. + */ + public static final boolean DEFAULT_ENCODE_ON_DISK = true; + + /** Default data block encoding algorithm. */ + public static final String DEFAULT_DATA_BLOCK_ENCODING = + DataBlockEncoding.NONE.toString(); + + /** + * Default number of versions of a record to keep. + */ + public static final int DEFAULT_VERSIONS = 3; + + /** + * Default is not to keep a minimum of versions. + */ + public static final int DEFAULT_MIN_VERSIONS = 0; + + /* + * Cache here the HCD value. + * Question: its OK to cache since when we're reenable, we create a new HCD? + */ + private volatile Integer blocksize = null; + + /** + * Default setting for whether to serve from memory or not. + */ + public static final boolean DEFAULT_IN_MEMORY = false; + + /** + * Default setting for preventing deleted from being collected immediately. + */ + public static final boolean DEFAULT_KEEP_DELETED = false; + + /** + * Default setting for whether to use a block cache or not. + */ + public static final boolean DEFAULT_BLOCKCACHE = true; + + /** + * Default setting for whether to cache data blocks on write if block caching + * is enabled. + */ + public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false; + + /** + * Default setting for whether to cache index blocks on write if block + * caching is enabled. + */ + public static final boolean DEFAULT_CACHE_INDEX_ON_WRITE = false; + + /** + * Default size of blocks in files stored to the filesytem (hfiles). + */ + public static final int DEFAULT_BLOCKSIZE = HFile.DEFAULT_BLOCKSIZE; + + /** + * Default setting for whether or not to use bloomfilters. + */ + public static final String DEFAULT_BLOOMFILTER = StoreFile.BloomType.NONE.toString(); + + /** + * Default setting for whether to cache bloom filter blocks on write if block + * caching is enabled. + */ + public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false; + + /** + * Default time to live of cell contents. + */ + public static final int DEFAULT_TTL = HConstants.FOREVER; + + /** + * Default scope. + */ + public static final int DEFAULT_REPLICATION_SCOPE = HConstants.REPLICATION_SCOPE_LOCAL; + + /** + * Default setting for whether to evict cached blocks from the blockcache on + * close. + */ + public static final boolean DEFAULT_EVICT_BLOCKS_ON_CLOSE = false; + + private final static Map DEFAULT_VALUES = new HashMap(); + private final static Set RESERVED_KEYWORDS + = new HashSet(); + static { + DEFAULT_VALUES.put(BLOOMFILTER, DEFAULT_BLOOMFILTER); + DEFAULT_VALUES.put(REPLICATION_SCOPE, String.valueOf(DEFAULT_REPLICATION_SCOPE)); + DEFAULT_VALUES.put(HConstants.VERSIONS, String.valueOf(DEFAULT_VERSIONS)); + DEFAULT_VALUES.put(MIN_VERSIONS, String.valueOf(DEFAULT_MIN_VERSIONS)); + DEFAULT_VALUES.put(COMPRESSION, DEFAULT_COMPRESSION); + DEFAULT_VALUES.put(TTL, String.valueOf(DEFAULT_TTL)); + DEFAULT_VALUES.put(BLOCKSIZE, String.valueOf(DEFAULT_BLOCKSIZE)); + DEFAULT_VALUES.put(HConstants.IN_MEMORY, String.valueOf(DEFAULT_IN_MEMORY)); + DEFAULT_VALUES.put(BLOCKCACHE, String.valueOf(DEFAULT_BLOCKCACHE)); + DEFAULT_VALUES.put(KEEP_DELETED_CELLS, String.valueOf(DEFAULT_KEEP_DELETED)); + DEFAULT_VALUES.put(ENCODE_ON_DISK, + String.valueOf(DEFAULT_ENCODE_ON_DISK)); + DEFAULT_VALUES.put(DATA_BLOCK_ENCODING, + String.valueOf(DEFAULT_DATA_BLOCK_ENCODING)); + DEFAULT_VALUES.put(CACHE_DATA_ON_WRITE, + String.valueOf(DEFAULT_CACHE_DATA_ON_WRITE)); + DEFAULT_VALUES.put(CACHE_INDEX_ON_WRITE, + String.valueOf(DEFAULT_CACHE_INDEX_ON_WRITE)); + DEFAULT_VALUES.put(CACHE_BLOOMS_ON_WRITE, + String.valueOf(DEFAULT_CACHE_BLOOMS_ON_WRITE)); + DEFAULT_VALUES.put(EVICT_BLOCKS_ON_CLOSE, + String.valueOf(DEFAULT_EVICT_BLOCKS_ON_CLOSE)); + for (String s : DEFAULT_VALUES.keySet()) { + RESERVED_KEYWORDS.add(new ImmutableBytesWritable(Bytes.toBytes(s))); + } + } + + // Column family name + private byte [] name; + + // Column metadata + protected final Map values = + new HashMap(); + + /* + * Cache the max versions rather than calculate it every time. + */ + private int cachedMaxVersions = -1; + + /** + * Default constructor. Must be present for Writable. + */ + public HColumnDescriptor() { + this.name = null; + } + + /** + * Construct a column descriptor specifying only the family name + * The other attributes are defaulted. + * + * @param familyName Column family name. Must be 'printable' -- digit or + * letter -- and may not contain a : + */ + public HColumnDescriptor(final String familyName) { + this(Bytes.toBytes(familyName)); + } + + /** + * Construct a column descriptor specifying only the family name + * The other attributes are defaulted. + * + * @param familyName Column family name. Must be 'printable' -- digit or + * letter -- and may not contain a : + */ + public HColumnDescriptor(final byte [] familyName) { + this (familyName == null || familyName.length <= 0? + HConstants.EMPTY_BYTE_ARRAY: familyName, DEFAULT_VERSIONS, + DEFAULT_COMPRESSION, DEFAULT_IN_MEMORY, DEFAULT_BLOCKCACHE, + DEFAULT_TTL, DEFAULT_BLOOMFILTER); + } + + /** + * Constructor. + * Makes a deep copy of the supplied descriptor. + * Can make a modifiable descriptor from an UnmodifyableHColumnDescriptor. + * @param desc The descriptor. + */ + public HColumnDescriptor(HColumnDescriptor desc) { + super(); + this.name = desc.name.clone(); + for (Map.Entry e: + desc.values.entrySet()) { + this.values.put(e.getKey(), e.getValue()); + } + setMaxVersions(desc.getMaxVersions()); + } + + /** + * Constructor + * @param familyName Column family name. Must be 'printable' -- digit or + * letter -- and may not contain a : + * @param maxVersions Maximum number of versions to keep + * @param compression Compression type + * @param inMemory If true, column data should be kept in an HRegionServer's + * cache + * @param blockCacheEnabled If true, MapFile blocks should be cached + * @param timeToLive Time-to-live of cell contents, in seconds + * (use HConstants.FOREVER for unlimited TTL) + * @param bloomFilter Bloom filter type for this column + * + * @throws IllegalArgumentException if passed a family name that is made of + * other than 'word' characters: i.e. [a-zA-Z_0-9] or contains + * a : + * @throws IllegalArgumentException if the number of versions is <= 0 + * @deprecated use {@link #HColumnDescriptor(String)} and setters + */ + @Deprecated + public HColumnDescriptor(final byte [] familyName, final int maxVersions, + final String compression, final boolean inMemory, + final boolean blockCacheEnabled, + final int timeToLive, final String bloomFilter) { + this(familyName, maxVersions, compression, inMemory, blockCacheEnabled, + DEFAULT_BLOCKSIZE, timeToLive, bloomFilter, DEFAULT_REPLICATION_SCOPE); + } + + /** + * Constructor + * @param familyName Column family name. Must be 'printable' -- digit or + * letter -- and may not contain a : + * @param maxVersions Maximum number of versions to keep + * @param compression Compression type + * @param inMemory If true, column data should be kept in an HRegionServer's + * cache + * @param blockCacheEnabled If true, MapFile blocks should be cached + * @param blocksize Block size to use when writing out storefiles. Use + * smaller block sizes for faster random-access at expense of larger indices + * (more memory consumption). Default is usually 64k. + * @param timeToLive Time-to-live of cell contents, in seconds + * (use HConstants.FOREVER for unlimited TTL) + * @param bloomFilter Bloom filter type for this column + * @param scope The scope tag for this column + * + * @throws IllegalArgumentException if passed a family name that is made of + * other than 'word' characters: i.e. [a-zA-Z_0-9] or contains + * a : + * @throws IllegalArgumentException if the number of versions is <= 0 + * @deprecated use {@link #HColumnDescriptor(String)} and setters + */ + @Deprecated + public HColumnDescriptor(final byte [] familyName, final int maxVersions, + final String compression, final boolean inMemory, + final boolean blockCacheEnabled, final int blocksize, + final int timeToLive, final String bloomFilter, final int scope) { + this(familyName, DEFAULT_MIN_VERSIONS, maxVersions, DEFAULT_KEEP_DELETED, + compression, DEFAULT_ENCODE_ON_DISK, DEFAULT_DATA_BLOCK_ENCODING, + inMemory, blockCacheEnabled, blocksize, timeToLive, bloomFilter, + scope); + } + + /** + * Constructor + * @param familyName Column family name. Must be 'printable' -- digit or + * letter -- and may not contain a : + * @param minVersions Minimum number of versions to keep + * @param maxVersions Maximum number of versions to keep + * @param keepDeletedCells Whether to retain deleted cells until they expire + * up to maxVersions versions. + * @param compression Compression type + * @param encodeOnDisk whether to use the specified data block encoding + * on disk. If false, the encoding will be used in cache only. + * @param dataBlockEncoding data block encoding + * @param inMemory If true, column data should be kept in an HRegionServer's + * cache + * @param blockCacheEnabled If true, MapFile blocks should be cached + * @param blocksize Block size to use when writing out storefiles. Use + * smaller blocksizes for faster random-access at expense of larger indices + * (more memory consumption). Default is usually 64k. + * @param timeToLive Time-to-live of cell contents, in seconds + * (use HConstants.FOREVER for unlimited TTL) + * @param bloomFilter Bloom filter type for this column + * @param scope The scope tag for this column + * + * @throws IllegalArgumentException if passed a family name that is made of + * other than 'word' characters: i.e. [a-zA-Z_0-9] or contains + * a : + * @throws IllegalArgumentException if the number of versions is <= 0 + * @deprecated use {@link #HColumnDescriptor(String)} and setters + */ + @Deprecated + public HColumnDescriptor(final byte[] familyName, final int minVersions, + final int maxVersions, final boolean keepDeletedCells, + final String compression, final boolean encodeOnDisk, + final String dataBlockEncoding, final boolean inMemory, + final boolean blockCacheEnabled, final int blocksize, + final int timeToLive, final String bloomFilter, final int scope) { + isLegalFamilyName(familyName); + this.name = familyName; + + if (maxVersions <= 0) { + // TODO: Allow maxVersion of 0 to be the way you say "Keep all versions". + // Until there is support, consider 0 or < 0 -- a configuration error. + throw new IllegalArgumentException("Maximum versions must be positive"); + } + + if (minVersions > 0) { + if (timeToLive == HConstants.FOREVER) { + throw new IllegalArgumentException("Minimum versions requires TTL."); + } + if (minVersions >= maxVersions) { + throw new IllegalArgumentException("Minimum versions must be < " + + "maximum versions."); + } + } + + setMaxVersions(maxVersions); + setMinVersions(minVersions); + setKeepDeletedCells(keepDeletedCells); + setInMemory(inMemory); + setBlockCacheEnabled(blockCacheEnabled); + setTimeToLive(timeToLive); + setCompressionType(Compression.Algorithm. + valueOf(compression.toUpperCase())); + setEncodeOnDisk(encodeOnDisk); + setDataBlockEncoding(DataBlockEncoding. + valueOf(dataBlockEncoding.toUpperCase())); + setBloomFilterType(StoreFile.BloomType. + valueOf(bloomFilter.toUpperCase())); + setBlocksize(blocksize); + setScope(scope); + } + + /** + * @param b Family name. + * @return b + * @throws IllegalArgumentException If not null and not a legitimate family + * name: i.e. 'printable' and ends in a ':' (Null passes are allowed because + * b can be null when deserializing). Cannot start with a '.' + * either. Also Family can not be an empty value or equal "recovered.edits". + */ + public static byte [] isLegalFamilyName(final byte [] b) { + if (b == null) { + return b; + } + if (b[0] == '.') { + throw new IllegalArgumentException("Family names cannot start with a " + + "period: " + Bytes.toString(b)); + } + for (int i = 0; i < b.length; i++) { + if (Character.isISOControl(b[i]) || b[i] == ':' || b[i] == '\\' || b[i] == '/') { + throw new IllegalArgumentException("Illegal character <" + b[i] + + ">. Family names cannot contain control characters or colons: " + + Bytes.toString(b)); + } + } + byte[] recoveredEdit = Bytes.toBytes(HLog.RECOVERED_EDITS_DIR); + if (Bytes.equals(recoveredEdit, b)) { + throw new IllegalArgumentException("Family name cannot be: " + + HLog.RECOVERED_EDITS_DIR); + } + return b; + } + + /** + * @return Name of this column family + */ + public byte [] getName() { + return name; + } + + /** + * @return Name of this column family + */ + public String getNameAsString() { + return Bytes.toString(this.name); + } + + /** + * @param key The key. + * @return The value. + */ + public byte[] getValue(byte[] key) { + ImmutableBytesWritable ibw = values.get(new ImmutableBytesWritable(key)); + if (ibw == null) + return null; + return ibw.get(); + } + + /** + * @param key The key. + * @return The value as a string. + */ + public String getValue(String key) { + byte[] value = getValue(Bytes.toBytes(key)); + if (value == null) + return null; + return Bytes.toString(value); + } + + /** + * @return All values. + */ + public Map getValues() { + // shallow pointer copy + return Collections.unmodifiableMap(values); + } + + /** + * @param key The key. + * @param value The value. + * @return this (for chained invocation) + */ + public HColumnDescriptor setValue(byte[] key, byte[] value) { + values.put(new ImmutableBytesWritable(key), + new ImmutableBytesWritable(value)); + return this; + } + + /** + * @param key Key whose key and value we're to remove from HCD parameters. + */ + public void remove(final byte [] key) { + values.remove(new ImmutableBytesWritable(key)); + } + + /** + * @param key The key. + * @param value The value. + * @return this (for chained invocation) + */ + public HColumnDescriptor setValue(String key, String value) { + if (value == null) { + remove(Bytes.toBytes(key)); + } else { + setValue(Bytes.toBytes(key), Bytes.toBytes(value)); + } + return this; + } + + /** @return compression type being used for the column family */ + public Compression.Algorithm getCompression() { + String n = getValue(COMPRESSION); + if (n == null) { + return Compression.Algorithm.NONE; + } + return Compression.Algorithm.valueOf(n.toUpperCase()); + } + + /** @return compression type being used for the column family for major + compression */ + public Compression.Algorithm getCompactionCompression() { + String n = getValue(COMPRESSION_COMPACT); + if (n == null) { + return getCompression(); + } + return Compression.Algorithm.valueOf(n.toUpperCase()); + } + + /** @return maximum number of versions */ + public int getMaxVersions() { + return this.cachedMaxVersions; + } + + /** + * @param maxVersions maximum number of versions + * @return this (for chained invocation) + */ + public HColumnDescriptor setMaxVersions(int maxVersions) { + setValue(HConstants.VERSIONS, Integer.toString(maxVersions)); + cachedMaxVersions = maxVersions; + return this; + } + + /** + * @return The storefile/hfile blocksize for this column family. + */ + public synchronized int getBlocksize() { + if (this.blocksize == null) { + String value = getValue(BLOCKSIZE); + this.blocksize = (value != null)? + Integer.decode(value): Integer.valueOf(DEFAULT_BLOCKSIZE); + } + return this.blocksize.intValue(); + } + + /** + * @param s Blocksize to use when writing out storefiles/hfiles on this + * column family. + * @return this (for chained invocation) + */ + public HColumnDescriptor setBlocksize(int s) { + setValue(BLOCKSIZE, Integer.toString(s)); + this.blocksize = null; + return this; + } + + /** + * @return Compression type setting. + */ + public Compression.Algorithm getCompressionType() { + return getCompression(); + } + + /** + * Compression types supported in hbase. + * LZO is not bundled as part of the hbase distribution. + * See LZO Compression + * for how to enable it. + * @param type Compression type setting. + * @return this (for chained invocation) + */ + public HColumnDescriptor setCompressionType(Compression.Algorithm type) { + return setValue(COMPRESSION, type.getName().toUpperCase()); + } + + /** @return data block encoding algorithm used on disk */ + public DataBlockEncoding getDataBlockEncodingOnDisk() { + String encodeOnDiskStr = getValue(ENCODE_ON_DISK); + boolean encodeOnDisk; + if (encodeOnDiskStr == null) { + encodeOnDisk = DEFAULT_ENCODE_ON_DISK; + } else { + encodeOnDisk = Boolean.valueOf(encodeOnDiskStr); + } + + if (!encodeOnDisk) { + // No encoding on disk. + return DataBlockEncoding.NONE; + } + return getDataBlockEncoding(); + } + + /** + * Set the flag indicating that we only want to encode data block in cache + * but not on disk. + * @return this (for chained invocation) + */ + public HColumnDescriptor setEncodeOnDisk(boolean encodeOnDisk) { + return setValue(ENCODE_ON_DISK, String.valueOf(encodeOnDisk)); + } + + /** + * @return the data block encoding algorithm used in block cache and + * optionally on disk + */ + public DataBlockEncoding getDataBlockEncoding() { + String type = getValue(DATA_BLOCK_ENCODING); + if (type == null) { + type = DEFAULT_DATA_BLOCK_ENCODING; + } + return DataBlockEncoding.valueOf(type); + } + + /** + * Set data block encoding algorithm used in block cache. + * @param type What kind of data block encoding will be used. + * @return this (for chained invocation) + */ + public HColumnDescriptor setDataBlockEncoding(DataBlockEncoding type) { + String name; + if (type != null) { + name = type.toString(); + } else { + name = DataBlockEncoding.NONE.toString(); + } + return setValue(DATA_BLOCK_ENCODING, name); + } + + /** + * @return Compression type setting. + */ + public Compression.Algorithm getCompactionCompressionType() { + return getCompactionCompression(); + } + + /** + * Compression types supported in hbase. + * LZO is not bundled as part of the hbase distribution. + * See LZO Compression + * for how to enable it. + * @param type Compression type setting. + * @return this (for chained invocation) + */ + public HColumnDescriptor setCompactionCompressionType( + Compression.Algorithm type) { + return setValue(COMPRESSION_COMPACT, type.getName().toUpperCase()); + } + + /** + * @return True if we are to keep all in use HRegionServer cache. + */ + public boolean isInMemory() { + String value = getValue(HConstants.IN_MEMORY); + if (value != null) + return Boolean.valueOf(value).booleanValue(); + return DEFAULT_IN_MEMORY; + } + + /** + * @param inMemory True if we are to keep all values in the HRegionServer + * cache + * @return this (for chained invocation) + */ + public HColumnDescriptor setInMemory(boolean inMemory) { + return setValue(HConstants.IN_MEMORY, Boolean.toString(inMemory)); + } + + public boolean getKeepDeletedCells() { + String value = getValue(KEEP_DELETED_CELLS); + if (value != null) { + return Boolean.valueOf(value).booleanValue(); + } + return DEFAULT_KEEP_DELETED; + } + + /** + * @param keepDeletedCells True if deleted rows should not be collected + * immediately. + * @return this (for chained invocation) + */ + public HColumnDescriptor setKeepDeletedCells(boolean keepDeletedCells) { + return setValue(KEEP_DELETED_CELLS, Boolean.toString(keepDeletedCells)); + } + + /** + * @return Time-to-live of cell contents, in seconds. + */ + public int getTimeToLive() { + String value = getValue(TTL); + return (value != null)? Integer.valueOf(value).intValue(): DEFAULT_TTL; + } + + /** + * @param timeToLive Time-to-live of cell contents, in seconds. + * @return this (for chained invocation) + */ + public HColumnDescriptor setTimeToLive(int timeToLive) { + return setValue(TTL, Integer.toString(timeToLive)); + } + + /** + * @return The minimum number of versions to keep. + */ + public int getMinVersions() { + String value = getValue(MIN_VERSIONS); + return (value != null)? Integer.valueOf(value).intValue(): 0; + } + + /** + * @param minVersions The minimum number of versions to keep. + * (used when timeToLive is set) + * @return this (for chained invocation) + */ + public HColumnDescriptor setMinVersions(int minVersions) { + return setValue(MIN_VERSIONS, Integer.toString(minVersions)); + } + + /** + * @return True if MapFile blocks should be cached. + */ + public boolean isBlockCacheEnabled() { + String value = getValue(BLOCKCACHE); + if (value != null) + return Boolean.valueOf(value).booleanValue(); + return DEFAULT_BLOCKCACHE; + } + + /** + * @param blockCacheEnabled True if MapFile blocks should be cached. + * @return this (for chained invocation) + */ + public HColumnDescriptor setBlockCacheEnabled(boolean blockCacheEnabled) { + return setValue(BLOCKCACHE, Boolean.toString(blockCacheEnabled)); + } + + /** + * @return bloom filter type used for new StoreFiles in ColumnFamily + */ + public StoreFile.BloomType getBloomFilterType() { + String n = getValue(BLOOMFILTER); + if (n == null) { + n = DEFAULT_BLOOMFILTER; + } + return StoreFile.BloomType.valueOf(n.toUpperCase()); + } + + /** + * @param bt bloom filter type + * @return this (for chained invocation) + */ + public HColumnDescriptor setBloomFilterType(final StoreFile.BloomType bt) { + return setValue(BLOOMFILTER, bt.toString()); + } + + /** + * @return the scope tag + */ + public int getScope() { + String value = getValue(REPLICATION_SCOPE); + if (value != null) { + return Integer.valueOf(value).intValue(); + } + return DEFAULT_REPLICATION_SCOPE; + } + + /** + * @param scope the scope tag + * @return this (for chained invocation) + */ + public HColumnDescriptor setScope(int scope) { + return setValue(REPLICATION_SCOPE, Integer.toString(scope)); + } + + /** + * @return true if we should cache data blocks on write + */ + public boolean shouldCacheDataOnWrite() { + String value = getValue(CACHE_DATA_ON_WRITE); + if (value != null) { + return Boolean.valueOf(value).booleanValue(); + } + return DEFAULT_CACHE_DATA_ON_WRITE; + } + + /** + * @param value true if we should cache data blocks on write + * @return this (for chained invocation) + */ + public HColumnDescriptor setCacheDataOnWrite(boolean value) { + return setValue(CACHE_DATA_ON_WRITE, Boolean.toString(value)); + } + + /** + * @return true if we should cache index blocks on write + */ + public boolean shouldCacheIndexesOnWrite() { + String value = getValue(CACHE_INDEX_ON_WRITE); + if (value != null) { + return Boolean.valueOf(value).booleanValue(); + } + return DEFAULT_CACHE_INDEX_ON_WRITE; + } + + /** + * @param value true if we should cache index blocks on write + * @return this (for chained invocation) + */ + public HColumnDescriptor setCacheIndexesOnWrite(boolean value) { + return setValue(CACHE_INDEX_ON_WRITE, Boolean.toString(value)); + } + + /** + * @return true if we should cache bloomfilter blocks on write + */ + public boolean shouldCacheBloomsOnWrite() { + String value = getValue(CACHE_BLOOMS_ON_WRITE); + if (value != null) { + return Boolean.valueOf(value).booleanValue(); + } + return DEFAULT_CACHE_BLOOMS_ON_WRITE; + } + + /** + * @param value true if we should cache bloomfilter blocks on write + * @return this (for chained invocation) + */ + public HColumnDescriptor setCacheBloomsOnWrite(boolean value) { + return setValue(CACHE_BLOOMS_ON_WRITE, Boolean.toString(value)); + } + + /** + * @return true if we should evict cached blocks from the blockcache on + * close + */ + public boolean shouldEvictBlocksOnClose() { + String value = getValue(EVICT_BLOCKS_ON_CLOSE); + if (value != null) { + return Boolean.valueOf(value).booleanValue(); + } + return DEFAULT_EVICT_BLOCKS_ON_CLOSE; + } + + /** + * @param value true if we should evict cached blocks from the blockcache on + * close + * @return this (for chained invocation) + */ + public HColumnDescriptor setEvictBlocksOnClose(boolean value) { + return setValue(EVICT_BLOCKS_ON_CLOSE, Boolean.toString(value)); + } + + /** + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder s = new StringBuilder(); + s.append('{'); + s.append(HConstants.NAME); + s.append(" => '"); + s.append(Bytes.toString(name)); + s.append("'"); + s.append(getValues(true)); + s.append('}'); + return s.toString(); + } + + /** + * @return Column family descriptor with only the customized attributes. + */ + public String toStringCustomizedValues() { + StringBuilder s = new StringBuilder(); + s.append('{'); + s.append(HConstants.NAME); + s.append(" => '"); + s.append(Bytes.toString(name)); + s.append("'"); + s.append(getValues(false)); + s.append('}'); + return s.toString(); + } + + private StringBuilder getValues(boolean printDefaults) { + StringBuilder s = new StringBuilder(); + + boolean hasConfigKeys = false; + + // print all reserved keys first + for (ImmutableBytesWritable k : values.keySet()) { + if (!RESERVED_KEYWORDS.contains(k)) { + hasConfigKeys = true; + continue; + } + String key = Bytes.toString(k.get()); + String value = Bytes.toString(values.get(k).get()); + if (printDefaults + || !DEFAULT_VALUES.containsKey(key) + || !DEFAULT_VALUES.get(key).equalsIgnoreCase(value)) { + s.append(", "); + s.append(key); + + s.append(" => "); + s.append('\'').append(value).append('\''); + } + } + + // print all non-reserved, advanced config keys as a separate subset + if (hasConfigKeys) { + s.append(", "); + s.append(HConstants.CONFIG).append(" => "); + s.append('{'); + boolean printComma = false; + for (ImmutableBytesWritable k : values.keySet()) { + if (RESERVED_KEYWORDS.contains(k)) { + continue; + } + String key = Bytes.toString(k.get()); + String value = Bytes.toString(values.get(k).get()); + if (printComma) { + s.append(", "); + } + printComma = true; + s.append('\'').append(key).append('\''); + s.append(" => "); + s.append('\'').append(value).append('\''); + } + s.append('}'); + } + return s; + } + + public static Map getDefaultValues() { + return Collections.unmodifiableMap(DEFAULT_VALUES); + } + + /** + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof HColumnDescriptor)) { + return false; + } + return compareTo((HColumnDescriptor)obj) == 0; + } + + /** + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + int result = Bytes.hashCode(this.name); + result ^= Byte.valueOf(COLUMN_DESCRIPTOR_VERSION).hashCode(); + result ^= values.hashCode(); + return result; + } + + // Writable + + public void readFields(DataInput in) throws IOException { + int version = in.readByte(); + if (version < 6) { + if (version <= 2) { + Text t = new Text(); + t.readFields(in); + this.name = t.getBytes(); +// if(KeyValue.getFamilyDelimiterIndex(this.name, 0, this.name.length) +// > 0) { +// this.name = stripColon(this.name); +// } + } else { + this.name = Bytes.readByteArray(in); + } + this.values.clear(); + setMaxVersions(in.readInt()); + int ordinal = in.readInt(); + setCompressionType(Compression.Algorithm.values()[ordinal]); + setInMemory(in.readBoolean()); + setBloomFilterType(in.readBoolean() ? BloomType.ROW : BloomType.NONE); + if (getBloomFilterType() != BloomType.NONE && version < 5) { + // If a bloomFilter is enabled and the column descriptor is less than + // version 5, we need to skip over it to read the rest of the column + // descriptor. There are no BloomFilterDescriptors written to disk for + // column descriptors with a version number >= 5 + throw new UnsupportedClassVersionError(this.getClass().getName() + + " does not support backward compatibility with versions older " + + "than version 5"); + } + if (version > 1) { + setBlockCacheEnabled(in.readBoolean()); + } + if (version > 2) { + setTimeToLive(in.readInt()); + } + } else { + // version 6+ + this.name = Bytes.readByteArray(in); + this.values.clear(); + int numValues = in.readInt(); + for (int i = 0; i < numValues; i++) { + ImmutableBytesWritable key = new ImmutableBytesWritable(); + ImmutableBytesWritable value = new ImmutableBytesWritable(); + key.readFields(in); + value.readFields(in); + + // in version 8, the BloomFilter setting changed from bool to enum + if (version < 8 && Bytes.toString(key.get()).equals(BLOOMFILTER)) { + value.set(Bytes.toBytes( + Boolean.getBoolean(Bytes.toString(value.get())) + ? BloomType.ROW.toString() + : BloomType.NONE.toString())); + } + + values.put(key, value); + } + if (version == 6) { + // Convert old values. + setValue(COMPRESSION, Compression.Algorithm.NONE.getName()); + } + String value = getValue(HConstants.VERSIONS); + this.cachedMaxVersions = (value != null)? + Integer.valueOf(value).intValue(): DEFAULT_VERSIONS; + } + } + + public void write(DataOutput out) throws IOException { + out.writeByte(COLUMN_DESCRIPTOR_VERSION); + Bytes.writeByteArray(out, this.name); + out.writeInt(values.size()); + for (Map.Entry e: + values.entrySet()) { + e.getKey().write(out); + e.getValue().write(out); + } + } + + // Comparable + + public int compareTo(HColumnDescriptor o) { + int result = Bytes.compareTo(this.name, o.getName()); + if (result == 0) { + // punt on comparison for ordering, just calculate difference + result = this.values.hashCode() - o.values.hashCode(); + if (result < 0) + result = -1; + else if (result > 0) + result = 1; + } + return result; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java new file mode 100644 index 0000000..ac91454 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -0,0 +1,728 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.regex.Pattern; + +import org.apache.commons.lang.ArrayUtils; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * HConstants holds a bunch of HBase-related constants + */ +public final class HConstants { + /** + * Status codes used for return values of bulk operations. + */ + public enum OperationStatusCode { + NOT_RUN, + SUCCESS, + BAD_FAMILY, + SANITY_CHECK_FAILURE, + FAILURE; + } + + /** long constant for zero */ + public static final Long ZERO_L = Long.valueOf(0L); + public static final String NINES = "99999999999999"; + public static final String ZEROES = "00000000000000"; + + // For migration + + /** name of version file */ + public static final String VERSION_FILE_NAME = "hbase.version"; + + /** + * Current version of file system. + * Version 4 supports only one kind of bloom filter. + * Version 5 changes versions in catalog table regions. + * Version 6 enables blockcaching on catalog tables. + * Version 7 introduces hfile -- hbase 0.19 to 0.20.. + */ + // public static final String FILE_SYSTEM_VERSION = "6"; + public static final String FILE_SYSTEM_VERSION = "7"; + + // Configuration parameters + + //TODO: Is having HBase homed on port 60k OK? + + /** Cluster is in distributed mode or not */ + public static final String CLUSTER_DISTRIBUTED = "hbase.cluster.distributed"; + + /** Config for pluggable load balancers */ + public static final String HBASE_MASTER_LOADBALANCER_CLASS = "hbase.master.loadbalancer.class"; + + /** Config for pluggable hbase cluster manager */ + public static final String HBASE_CLUSTER_MANAGER_CLASS = "hbase.it.clustermanager.class"; + + /** Cluster is standalone or pseudo-distributed */ + public static final boolean CLUSTER_IS_LOCAL = false; + + /** Cluster is fully-distributed */ + public static final boolean CLUSTER_IS_DISTRIBUTED = true; + + /** Default value for cluster distributed mode */ + public static final boolean DEFAULT_CLUSTER_DISTRIBUTED = CLUSTER_IS_LOCAL; + + /** default host address */ + public static final String DEFAULT_HOST = "0.0.0.0"; + + /** Parameter name for port master listens on. */ + public static final String MASTER_PORT = "hbase.master.port"; + + /** default port that the master listens on */ + public static final int DEFAULT_MASTER_PORT = 60000; + + /** default port for master web api */ + public static final int DEFAULT_MASTER_INFOPORT = 60010; + + /** Parameter name for the master type being backup (waits for primary to go inactive). */ + public static final String MASTER_TYPE_BACKUP = "hbase.master.backup"; + + /** by default every master is a possible primary master unless the conf explicitly overrides it */ + public static final boolean DEFAULT_MASTER_TYPE_BACKUP = false; + + /** Name of ZooKeeper quorum configuration parameter. */ + public static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; + + /** Name of ZooKeeper config file in conf/ directory. */ + public static final String ZOOKEEPER_CONFIG_NAME = "zoo.cfg"; + + /** Common prefix of ZooKeeper configuration properties */ + public static final String ZK_CFG_PROPERTY_PREFIX = + "hbase.zookeeper.property."; + + public static final int ZK_CFG_PROPERTY_PREFIX_LEN = + ZK_CFG_PROPERTY_PREFIX.length(); + + /** + * The ZK client port key in the ZK properties map. The name reflects the + * fact that this is not an HBase configuration key. + */ + public static final String CLIENT_PORT_STR = "clientPort"; + + /** Parameter name for the client port that the zookeeper listens on */ + public static final String ZOOKEEPER_CLIENT_PORT = + ZK_CFG_PROPERTY_PREFIX + CLIENT_PORT_STR; + + /** Default client port that the zookeeper listens on */ + public static final int DEFAULT_ZOOKEPER_CLIENT_PORT = 2181; + + /** Parameter name for the wait time for the recoverable zookeeper */ + public static final String ZOOKEEPER_RECOVERABLE_WAITTIME = "hbase.zookeeper.recoverable.waittime"; + + /** Default wait time for the recoverable zookeeper */ + public static final long DEFAULT_ZOOKEPER_RECOVERABLE_WAITIME = 10000; + + /** Parameter name for the root dir in ZK for this cluster */ + public static final String ZOOKEEPER_ZNODE_PARENT = "zookeeper.znode.parent"; + + public static final String DEFAULT_ZOOKEEPER_ZNODE_PARENT = "/hbase"; + + /** + * Parameter name for the limit on concurrent client-side zookeeper + * connections + */ + public static final String ZOOKEEPER_MAX_CLIENT_CNXNS = + ZK_CFG_PROPERTY_PREFIX + "maxClientCnxns"; + + /** Parameter name for the ZK data directory */ + public static final String ZOOKEEPER_DATA_DIR = + ZK_CFG_PROPERTY_PREFIX + "dataDir"; + + /** Default limit on concurrent client-side zookeeper connections */ + public static final int DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS = 300; + + /** Configuration key for ZooKeeper session timeout */ + public static final String ZK_SESSION_TIMEOUT = "zookeeper.session.timeout"; + + /** Default value for ZooKeeper session timeout */ + public static final int DEFAULT_ZK_SESSION_TIMEOUT = 180 * 1000; + + /** Configuration key for whether to use ZK.multi */ + public static final String ZOOKEEPER_USEMULTI = "hbase.zookeeper.useMulti"; + + /** Parameter name for port region server listens on. */ + public static final String REGIONSERVER_PORT = "hbase.regionserver.port"; + + /** Default port region server listens on. */ + public static final int DEFAULT_REGIONSERVER_PORT = 60020; + + /** default port for region server web api */ + public static final int DEFAULT_REGIONSERVER_INFOPORT = 60030; + + /** A flag that enables automatic selection of regionserver info port */ + public static final String REGIONSERVER_INFO_PORT_AUTO = + "hbase.regionserver.info.port.auto"; + + /** Parameter name for what region server interface to use. */ + public static final String REGION_SERVER_CLASS = "hbase.regionserver.class"; + + /** Parameter name for what region server implementation to use. */ + public static final String REGION_SERVER_IMPL= "hbase.regionserver.impl"; + + /** Default region server interface class name. */ + public static final String DEFAULT_REGION_SERVER_CLASS = HRegionInterface.class.getName(); + + /** Parameter name for what master implementation to use. */ + public static final String MASTER_IMPL= "hbase.master.impl"; + + /** Parameter name for how often threads should wake up */ + public static final String THREAD_WAKE_FREQUENCY = "hbase.server.thread.wakefrequency"; + + /** Default value for thread wake frequency */ + public static final int DEFAULT_THREAD_WAKE_FREQUENCY = 10 * 1000; + + /** Parameter name for how often we should try to write a version file, before failing */ + public static final String VERSION_FILE_WRITE_ATTEMPTS = "hbase.server.versionfile.writeattempts"; + + /** Parameter name for how often we should try to write a version file, before failing */ + public static final int DEFAULT_VERSION_FILE_WRITE_ATTEMPTS = 3; + + /** Parameter name for how often a region should should perform a major compaction */ + public static final String MAJOR_COMPACTION_PERIOD = "hbase.hregion.majorcompaction"; + + /** Parameter name for the maximum batch of KVs to be used in flushes and compactions */ + public static final String COMPACTION_KV_MAX = "hbase.hstore.compaction.kv.max"; + + /** Parameter name for HBase instance root directory */ + public static final String HBASE_DIR = "hbase.rootdir"; + + /** Parameter name for HBase client IPC pool type */ + public static final String HBASE_CLIENT_IPC_POOL_TYPE = "hbase.client.ipc.pool.type"; + + /** Parameter name for HBase client IPC pool size */ + public static final String HBASE_CLIENT_IPC_POOL_SIZE = "hbase.client.ipc.pool.size"; + + /** Parameter name for HBase client operation timeout, which overrides RPC timeout */ + public static final String HBASE_CLIENT_OPERATION_TIMEOUT = "hbase.client.operation.timeout"; + + /** Default HBase client operation timeout, which is tantamount to a blocking call */ + public static final int DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT = Integer.MAX_VALUE; + + /** Used to construct the name of the log directory for a region server + * Use '.' as a special character to seperate the log files from table data */ + public static final String HREGION_LOGDIR_NAME = ".logs"; + + /** Used to construct the name of the splitlog directory for a region server */ + public static final String SPLIT_LOGDIR_NAME = "splitlog"; + + public static final String CORRUPT_DIR_NAME = ".corrupt"; + + /** Like the previous, but for old logs that are about to be deleted */ + public static final String HREGION_OLDLOGDIR_NAME = ".oldlogs"; + + /** Used by HBCK to sideline backup data */ + public static final String HBCK_SIDELINEDIR_NAME = ".hbck"; + + /** Used to construct the name of the compaction directory during compaction */ + public static final String HREGION_COMPACTIONDIR_NAME = "compaction.dir"; + + /** Conf key for the max file size after which we split the region */ + public static final String HREGION_MAX_FILESIZE = + "hbase.hregion.max.filesize"; + + /** Default maximum file size */ + public static final long DEFAULT_MAX_FILE_SIZE = 10 * 1024 * 1024 * 1024L; + + /** + * The max number of threads used for opening and closing stores or store + * files in parallel + */ + public static final String HSTORE_OPEN_AND_CLOSE_THREADS_MAX = + "hbase.hstore.open.and.close.threads.max"; + + /** + * The default number for the max number of threads used for opening and + * closing stores or store files in parallel + */ + public static final int DEFAULT_HSTORE_OPEN_AND_CLOSE_THREADS_MAX = 1; + + + /** Conf key for the memstore size at which we flush the memstore */ + public static final String HREGION_MEMSTORE_FLUSH_SIZE = + "hbase.hregion.memstore.flush.size"; + + /** Default size of a reservation block */ + public static final int DEFAULT_SIZE_RESERVATION_BLOCK = 1024 * 1024 * 5; + + /** Maximum value length, enforced on KeyValue construction */ + public static final int MAXIMUM_VALUE_LENGTH = Integer.MAX_VALUE; + + /** name of the file for unique cluster ID */ + public static final String CLUSTER_ID_FILE_NAME = "hbase.id"; + + /** Configuration key storing the cluster ID */ + public static final String CLUSTER_ID = "hbase.cluster.id"; + + // Always store the location of the root table's HRegion. + // This HRegion is never split. + + // region name = table + startkey + regionid. This is the row key. + // each row in the root and meta tables describes exactly 1 region + // Do we ever need to know all the information that we are storing? + + // Note that the name of the root table starts with "-" and the name of the + // meta table starts with "." Why? it's a trick. It turns out that when we + // store region names in memory, we use a SortedMap. Since "-" sorts before + // "." (and since no other table name can start with either of these + // characters, the root region will always be the first entry in such a Map, + // followed by all the meta regions (which will be ordered by their starting + // row key as well), followed by all user tables. So when the Master is + // choosing regions to assign, it will always choose the root region first, + // followed by the meta regions, followed by user regions. Since the root + // and meta regions always need to be on-line, this ensures that they will + // be the first to be reassigned if the server(s) they are being served by + // should go down. + + /** The root table's name.*/ + public static final byte [] ROOT_TABLE_NAME = Bytes.toBytes("-ROOT-"); + + /** The META table's name. */ + public static final byte [] META_TABLE_NAME = Bytes.toBytes(".META."); + + /** delimiter used between portions of a region name */ + public static final int META_ROW_DELIMITER = ','; + + /** The catalog family as a string*/ + public static final String CATALOG_FAMILY_STR = "info"; + + /** The catalog family */ + public static final byte [] CATALOG_FAMILY = Bytes.toBytes(CATALOG_FAMILY_STR); + + /** The regioninfo column qualifier */ + public static final byte [] REGIONINFO_QUALIFIER = Bytes.toBytes("regioninfo"); + + /** The server column qualifier */ + public static final byte [] SERVER_QUALIFIER = Bytes.toBytes("server"); + + /** The startcode column qualifier */ + public static final byte [] STARTCODE_QUALIFIER = Bytes.toBytes("serverstartcode"); + + /** The lower-half split region column qualifier */ + public static final byte [] SPLITA_QUALIFIER = Bytes.toBytes("splitA"); + + /** The upper-half split region column qualifier */ + public static final byte [] SPLITB_QUALIFIER = Bytes.toBytes("splitB"); + + /** + * The meta table version column qualifier. + * We keep current version of the meta table in this column in -ROOT- + * table: i.e. in the 'info:v' column. + */ + public static final byte [] META_VERSION_QUALIFIER = Bytes.toBytes("v"); + + /** + * The current version of the meta table. + * Before this the meta had HTableDescriptor serialized into the HRegionInfo; + * i.e. pre-hbase 0.92. There was no META_VERSION column in the root table + * in this case. The presence of a version and its value being zero indicates + * meta is up-to-date. + */ + public static final short META_VERSION = 0; + + // Other constants + + /** + * An empty instance. + */ + public static final byte [] EMPTY_BYTE_ARRAY = new byte [0]; + + /** + * Used by scanners, etc when they want to start at the beginning of a region + */ + public static final byte [] EMPTY_START_ROW = EMPTY_BYTE_ARRAY; + + /** + * Last row in a table. + */ + public static final byte [] EMPTY_END_ROW = EMPTY_START_ROW; + + /** + * Used by scanners and others when they're trying to detect the end of a + * table + */ + public static final byte [] LAST_ROW = EMPTY_BYTE_ARRAY; + + /** + * Max length a row can have because of the limitation in TFile. + */ + public static final int MAX_ROW_LENGTH = Short.MAX_VALUE; + + /** When we encode strings, we always specify UTF8 encoding */ + public static final String UTF8_ENCODING = "UTF-8"; + + /** + * Timestamp to use when we want to refer to the latest cell. + * This is the timestamp sent by clients when no timestamp is specified on + * commit. + */ + public static final long LATEST_TIMESTAMP = Long.MAX_VALUE; + + /** + * Timestamp to use when we want to refer to the oldest cell. + */ + public static final long OLDEST_TIMESTAMP = Long.MIN_VALUE; + + /** + * LATEST_TIMESTAMP in bytes form + */ + public static final byte [] LATEST_TIMESTAMP_BYTES = Bytes.toBytes(LATEST_TIMESTAMP); + + /** + * Define for 'return-all-versions'. + */ + public static final int ALL_VERSIONS = Integer.MAX_VALUE; + + /** + * Unlimited time-to-live. + */ +// public static final int FOREVER = -1; + public static final int FOREVER = Integer.MAX_VALUE; + + /** + * Seconds in a week + */ + public static final int WEEK_IN_SECONDS = 7 * 24 * 3600; + + //TODO: although the following are referenced widely to format strings for + // the shell. They really aren't a part of the public API. It would be + // nice if we could put them somewhere where they did not need to be + // public. They could have package visibility + public static final String NAME = "NAME"; + public static final String VERSIONS = "VERSIONS"; + public static final String IN_MEMORY = "IN_MEMORY"; + public static final String CONFIG = "CONFIG"; + + /** + * This is a retry backoff multiplier table similar to the BSD TCP syn + * backoff table, a bit more aggressive than simple exponential backoff. + */ + public static int RETRY_BACKOFF[] = { 1, 1, 1, 2, 2, 4, 4, 8, 16, 32 }; + + public static final String REGION_IMPL = "hbase.hregion.impl"; + + /** modifyTable op for replacing the table descriptor */ + public static enum Modify { + CLOSE_REGION, + TABLE_COMPACT, + TABLE_FLUSH, + TABLE_MAJOR_COMPACT, + TABLE_SET_HTD, + TABLE_SPLIT + } + + /** + * Scope tag for locally scoped data. + * This data will not be replicated. + */ + public static final int REPLICATION_SCOPE_LOCAL = 0; + + /** + * Scope tag for globally scoped data. + * This data will be replicated to all peers. + */ + public static final int REPLICATION_SCOPE_GLOBAL = 1; + + /** + * Default cluster ID, cannot be used to identify a cluster so a key with + * this value means it wasn't meant for replication. + */ + public static final UUID DEFAULT_CLUSTER_ID = new UUID(0L,0L); + + /** + * Parameter name for maximum number of bytes returned when calling a + * scanner's next method. + */ + public static String HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY = "hbase.client.scanner.max.result.size"; + + /** + * Maximum number of bytes returned when calling a scanner's next method. + * Note that when a single row is larger than this limit the row is still + * returned completely. + * + * The default value is unlimited. + */ + public static long DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE = Long.MAX_VALUE; + + /** + * Parameter name for client pause value, used mostly as value to wait + * before running a retry of a failed get, region lookup, etc. + */ + public static String HBASE_CLIENT_PAUSE = "hbase.client.pause"; + + /** + * Default value of {@link #HBASE_CLIENT_PAUSE}. + */ + public static long DEFAULT_HBASE_CLIENT_PAUSE = 1000; + + /** + * Parameter name for server pause value, used mostly as value to wait before + * running a retry of a failed operation. + */ + public static String HBASE_SERVER_PAUSE = "hbase.server.pause"; + + /** + * Default value of {@link #HBASE_SERVER_PAUSE}. + */ + public static int DEFAULT_HBASE_SERVER_PAUSE = 1000; + + /** + * Parameter name for maximum retries, used as maximum for all retryable + * operations such as fetching of the root region from root region server, + * getting a cell's value, starting a row update, etc. + */ + public static String HBASE_CLIENT_RETRIES_NUMBER = "hbase.client.retries.number"; + + /** + * Default value of {@link #HBASE_CLIENT_RETRIES_NUMBER}. + */ + public static int DEFAULT_HBASE_CLIENT_RETRIES_NUMBER = 10; + + /** + * Parameter name for maximum attempts, used to limit the number of times the + * client will try to obtain the proxy for a given region server. + */ + public static String HBASE_CLIENT_RPC_MAXATTEMPTS = "hbase.client.rpc.maxattempts"; + + /** + * Default value of {@link #HBASE_CLIENT_RPC_MAXATTEMPTS}. + */ + public static int DEFAULT_HBASE_CLIENT_RPC_MAXATTEMPTS = 1; + + /** + * Parameter name for client prefetch limit, used as the maximum number of regions + * info that will be prefetched. + */ + public static String HBASE_CLIENT_PREFETCH_LIMIT = "hbase.client.prefetch.limit"; + + /** + * Default value of {@link #HBASE_CLIENT_PREFETCH_LIMIT}. + */ + public static int DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT = 10; + + /** + * Parameter name for number of rows that will be fetched when calling next on + * a scanner if it is not served from memory. Higher caching values will + * enable faster scanners but will eat up more memory and some calls of next + * may take longer and longer times when the cache is empty. + */ + public static String HBASE_META_SCANNER_CACHING = "hbase.meta.scanner.caching"; + + /** + * Default value of {@link #HBASE_META_SCANNER_CACHING}. + */ + public static int DEFAULT_HBASE_META_SCANNER_CACHING = 100; + + /** + * Parameter name for unique identifier for this {@link org.apache.hadoop.conf.Configuration} + * instance. If there are two or more {@link org.apache.hadoop.conf.Configuration} instances that, + * for all intents and purposes, are the same except for their instance ids, + * then they will not be able to share the same {@link org.apache.hadoop.hbase.client.HConnection} instance. + * On the other hand, even if the instance ids are the same, it could result + * in non-shared {@link org.apache.hadoop.hbase.client.HConnection} + * instances if some of the other connection parameters differ. + */ + public static String HBASE_CLIENT_INSTANCE_ID = "hbase.client.instance.id"; + + /** + * HRegion server lease period in milliseconds. Clients must report in within this period + * else they are considered dead. Unit measured in ms (milliseconds). + */ + public static String HBASE_REGIONSERVER_LEASE_PERIOD_KEY = + "hbase.regionserver.lease.period"; + + /** + * Default value of {@link #HBASE_REGIONSERVER_LEASE_PERIOD_KEY}. + */ + public static long DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD = 60000; + + /** + * timeout for each RPC + */ + public static String HBASE_RPC_TIMEOUT_KEY = "hbase.rpc.timeout"; + + /** + * Default value of {@link #HBASE_RPC_TIMEOUT_KEY} + */ + public static int DEFAULT_HBASE_RPC_TIMEOUT = 60000; + + /* + * cluster replication constants. + */ + public static final String + REPLICATION_ENABLE_KEY = "hbase.replication"; + public static final String + REPLICATION_SOURCE_SERVICE_CLASSNAME = "hbase.replication.source.service"; + public static final String + REPLICATION_SINK_SERVICE_CLASSNAME = "hbase.replication.sink.service"; + public static final String REPLICATION_SERVICE_CLASSNAME_DEFAULT = + "org.apache.hadoop.hbase.replication.regionserver.Replication"; + + /** HBCK special code name used as server name when manipulating ZK nodes */ + public static final String HBCK_CODE_NAME = "HBCKServerName"; + + public static final ServerName HBCK_CODE_SERVERNAME = + new ServerName(HBCK_CODE_NAME, -1, -1L); + + public static final String KEY_FOR_HOSTNAME_SEEN_BY_MASTER = + "hbase.regionserver.hostname.seen.by.master"; + + public static final String HBASE_MASTER_LOGCLEANER_PLUGINS = + "hbase.master.logcleaner.plugins"; + + public static final String HBASE_REGION_SPLIT_POLICY_KEY = + "hbase.regionserver.region.split.policy"; + + /** + * Configuration key for the size of the block cache + */ + public static final String HFILE_BLOCK_CACHE_SIZE_KEY = + "hfile.block.cache.size"; + + public static final float HFILE_BLOCK_CACHE_SIZE_DEFAULT = 0.25f; + + /* + * Minimum percentage of free heap necessary for a successful cluster startup. + */ + public static final float HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD = 0.2f; + + public static final Pattern CP_HTD_ATTR_KEY_PATTERN = Pattern.compile + ("^coprocessor\\$([0-9]+)$", Pattern.CASE_INSENSITIVE); + public static final Pattern CP_HTD_ATTR_VALUE_PATTERN = + Pattern.compile("(^[^\\|]*)\\|([^\\|]+)\\|[\\s]*([\\d]*)[\\s]*(\\|.*)?$"); + + public static final String CP_HTD_ATTR_VALUE_PARAM_KEY_PATTERN = "[^=,]+"; + public static final String CP_HTD_ATTR_VALUE_PARAM_VALUE_PATTERN = "[^,]+"; + public static final Pattern CP_HTD_ATTR_VALUE_PARAM_PATTERN = Pattern.compile( + "(" + CP_HTD_ATTR_VALUE_PARAM_KEY_PATTERN + ")=(" + + CP_HTD_ATTR_VALUE_PARAM_VALUE_PATTERN + "),?"); + + /** The delay when re-trying a socket operation in a loop (HBASE-4712) */ + public static final int SOCKET_RETRY_WAIT_MS = 200; + + /** Host name of the local machine */ + public static final String LOCALHOST = "localhost"; + + /** Enable file permission modification from standard hbase */ + public static final String ENABLE_DATA_FILE_UMASK = "hbase.data.umask.enable"; + /** File permission umask to use when creating hbase data files */ + public static final String DATA_FILE_UMASK_KEY = "hbase.data.umask"; + + /** + * If this parameter is set to true, then hbase will read + * data and then verify checksums. Checksum verification + * inside hdfs will be switched off. However, if the hbase-checksum + * verification fails, then it will switch back to using + * hdfs checksums for verifiying data that is being read from storage. + * + * If this parameter is set to false, then hbase will not + * verify any checksums, instead it will depend on checksum verification + * being done in the hdfs client. + */ + public static final String HBASE_CHECKSUM_VERIFICATION = + "hbase.regionserver.checksum.verify"; + + /** + * The name of the configuration parameter that specifies + * the number of bytes in a newly created checksum chunk. + */ + public static final String BYTES_PER_CHECKSUM = + "hbase.hstore.bytes.per.checksum"; + + /** + * The name of the configuration parameter that specifies + * the name of an algorithm that is used to compute checksums + * for newly created blocks. + */ + public static final String CHECKSUM_TYPE_NAME = + "hbase.hstore.checksum.algorithm"; + + /** Configuration name of HLog Compression */ + public static final String ENABLE_WAL_COMPRESSION = + "hbase.regionserver.wal.enablecompression"; + + /** + * QOS attributes: these attributes are used to demarcate RPC call processing + * by different set of handlers. For example, HIGH_QOS tagged methods are + * handled by high priority handlers. + */ + public static final int NORMAL_QOS = 0; + public static final int QOS_THRESHOLD = 10; + public static final int HIGH_QOS = 100; + public static final int REPLICATION_QOS = 5; // normal_QOS < replication_QOS < high_QOS + + /** + * The byte array represents for NO_NEXT_INDEXED_KEY; + * The actual value is irrelevant because this is always compared by reference. + */ + public static final byte [] NO_NEXT_INDEXED_KEY = Bytes.toBytes("NO_NEXT_INDEXED_KEY"); + + /** Directory under /hbase where archived hfiles are stored */ + public static final String HFILE_ARCHIVE_DIRECTORY = ".archive"; + + /** + * Name of the directory to store all snapshots. See SnapshotDescriptionUtils for + * remaining snapshot constants; this is here to keep HConstants dependencies at a minimum and + * uni-directional. + */ + public static final String SNAPSHOT_DIR_NAME = ".hbase-snapshot"; + + /* Name of old snapshot directory. See HBASE-8352 for details on why it needs to be renamed */ + public static final String OLD_SNAPSHOT_DIR_NAME = ".snapshot"; + + /** Temporary directory used for table creation and deletion */ + public static final String HBASE_TEMP_DIRECTORY = ".tmp"; + + /** Directories that are not HBase table directories */ + public static final List HBASE_NON_TABLE_DIRS = + Collections.unmodifiableList(Arrays.asList(new String[] { HREGION_LOGDIR_NAME, + HREGION_OLDLOGDIR_NAME, CORRUPT_DIR_NAME, SPLIT_LOGDIR_NAME, + HBCK_SIDELINEDIR_NAME, HFILE_ARCHIVE_DIRECTORY, SNAPSHOT_DIR_NAME, HBASE_TEMP_DIRECTORY, + OLD_SNAPSHOT_DIR_NAME })); + + /** Directories that are not HBase user table directories */ + public static final List HBASE_NON_USER_TABLE_DIRS = + Collections.unmodifiableList(Arrays.asList((String[])ArrayUtils.addAll( + new String[] { Bytes.toString(META_TABLE_NAME), Bytes.toString(ROOT_TABLE_NAME) }, + HBASE_NON_TABLE_DIRS.toArray()))); + + /** Health script related settings. */ + public static final String HEALTH_SCRIPT_LOC = "hbase.node.health.script.location"; + public static final String HEALTH_SCRIPT_TIMEOUT = "hbase.node.health.script.timeout"; + public static final String HEALTH_CHORE_WAKE_FREQ = + "hbase.node.health.script.frequency"; + public static final long DEFAULT_HEALTH_SCRIPT_TIMEOUT = 60000; + /** + * The maximum number of health check failures a server can encounter consecutively. + */ + public static final String HEALTH_FAILURE_THRESHOLD = + "hbase.node.health.failure.threshold"; + public static final int DEFAULT_HEALTH_FAILURE_THRESHOLD = 3; + + private HConstants() { + // Can't be instantiated with this ctor. + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java b/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java new file mode 100644 index 0000000..6682d17 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java @@ -0,0 +1,239 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.TreeMap; +import java.util.TreeSet; + + +/** + * Data structure to describe the distribution of HDFS blocks amount hosts. + * + * Adding erroneous data will be ignored silently. + */ +public class HDFSBlocksDistribution { + private Map hostAndWeights = null; + private long uniqueBlocksTotalWeight = 0; + + /** + * Stores the hostname and weight for that hostname. + * + * This is used when determining the physical locations of the blocks making + * up a region. + * + * To make a prioritized list of the hosts holding the most data of a region, + * this class is used to count the total weight for each host. The weight is + * currently just the size of the file. + */ + public static class HostAndWeight { + + private String host; + private long weight; + + /** + * Constructor + * @param host the host name + * @param weight the weight + */ + public HostAndWeight(String host, long weight) { + this.host = host; + this.weight = weight; + } + + /** + * add weight + * @param weight the weight + */ + public void addWeight(long weight) { + this.weight += weight; + } + + /** + * @return the host name + */ + public String getHost() { + return host; + } + + /** + * @return the weight + */ + public long getWeight() { + return weight; + } + + /** + * comparator used to sort hosts based on weight + */ + public static class WeightComparator implements Comparator { + @Override + public int compare(HostAndWeight l, HostAndWeight r) { + if(l.getWeight() == r.getWeight()) { + return l.getHost().compareTo(r.getHost()); + } + return l.getWeight() < r.getWeight() ? -1 : 1; + } + } + } + + /** + * Constructor + */ + public HDFSBlocksDistribution() { + this.hostAndWeights = + new TreeMap(); + } + + /** + * @see java.lang.Object#toString() + */ + @Override + public synchronized String toString() { + return "number of unique hosts in the disribution=" + + this.hostAndWeights.size(); + } + + /** + * add some weight to a list of hosts, update the value of unique block weight + * @param hosts the list of the host + * @param weight the weight + */ + public void addHostsAndBlockWeight(String[] hosts, long weight) { + if (hosts == null || hosts.length == 0) { + // erroneous data + return; + } + + addUniqueWeight(weight); + for (String hostname : hosts) { + addHostAndBlockWeight(hostname, weight); + } + } + + /** + * add some weight to the total unique weight + * @param weight the weight + */ + private void addUniqueWeight(long weight) { + uniqueBlocksTotalWeight += weight; + } + + + /** + * add some weight to a specific host + * @param host the host name + * @param weight the weight + */ + private void addHostAndBlockWeight(String host, long weight) { + if (host == null) { + // erroneous data + return; + } + + HostAndWeight hostAndWeight = this.hostAndWeights.get(host); + if(hostAndWeight == null) { + hostAndWeight = new HostAndWeight(host, weight); + this.hostAndWeights.put(host, hostAndWeight); + } else { + hostAndWeight.addWeight(weight); + } + } + + /** + * @return the hosts and their weights + */ + public Map getHostAndWeights() { + return this.hostAndWeights; + } + + /** + * return the weight for a specific host, that will be the total bytes of all + * blocks on the host + * @param host the host name + * @return the weight of the given host + */ + public long getWeight(String host) { + long weight = 0; + if (host != null) { + HostAndWeight hostAndWeight = this.hostAndWeights.get(host); + if(hostAndWeight != null) { + weight = hostAndWeight.getWeight(); + } + } + return weight; + } + + /** + * @return the sum of all unique blocks' weight + */ + public long getUniqueBlocksTotalWeight() { + return uniqueBlocksTotalWeight; + } + + /** + * return the locality index of a given host + * @param host the host name + * @return the locality index of the given host + */ + public float getBlockLocalityIndex(String host) { + float localityIndex = 0; + HostAndWeight hostAndWeight = this.hostAndWeights.get(host); + if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) { + localityIndex=(float)hostAndWeight.weight/(float)uniqueBlocksTotalWeight; + } + return localityIndex; + } + + + /** + * This will add the distribution from input to this object + * @param otherBlocksDistribution the other hdfs blocks distribution + */ + public void add(HDFSBlocksDistribution otherBlocksDistribution) { + Map otherHostAndWeights = + otherBlocksDistribution.getHostAndWeights(); + for (Map.Entry otherHostAndWeight: + otherHostAndWeights.entrySet()) { + addHostAndBlockWeight(otherHostAndWeight.getValue().host, + otherHostAndWeight.getValue().weight); + } + addUniqueWeight(otherBlocksDistribution.getUniqueBlocksTotalWeight()); + } + + /** + * return the sorted list of hosts in terms of their weights + */ + public List getTopHosts() { + NavigableSet orderedHosts = new TreeSet( + new HostAndWeight.WeightComparator()); + orderedHosts.addAll(this.hostAndWeights.values()); + List topHosts = new ArrayList(orderedHosts.size()); + for(HostAndWeight haw : orderedHosts.descendingSet()) { + topHosts.add(haw.getHost()); + } + return topHosts; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java new file mode 100644 index 0000000..53f6720 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -0,0 +1,827 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.EOFException; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.KeyValue.KVComparator; +import org.apache.hadoop.hbase.migration.HRegionInfo090x; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.JenkinsHash; +import org.apache.hadoop.hbase.util.MD5Hash; +import org.apache.hadoop.io.VersionedWritable; +import org.apache.hadoop.io.WritableComparable; + +/** + * HRegion information. + * Contains HRegion id, start and end keys, a reference to this + * HRegions' table descriptor, etc. + */ +public class HRegionInfo extends VersionedWritable +implements WritableComparable { + // VERSION == 0 when HRegionInfo had an HTableDescriptor inside it. + public static final byte VERSION_PRE_092 = 0; + public static final byte VERSION = 1; + private static final Log LOG = LogFactory.getLog(HRegionInfo.class); + + /** + * The new format for a region name contains its encodedName at the end. + * The encoded name also serves as the directory name for the region + * in the filesystem. + * + * New region name format: + * <tablename>,,<startkey>,<regionIdTimestamp>.<encodedName>. + * where, + * <encodedName> is a hex version of the MD5 hash of + * <tablename>,<startkey>,<regionIdTimestamp> + * + * The old region name format: + * <tablename>,<startkey>,<regionIdTimestamp> + * For region names in the old format, the encoded name is a 32-bit + * JenkinsHash integer value (in its decimal notation, string form). + *

+ * **NOTE** + * + * ROOT, the first META region, and regions created by an older + * version of HBase (0.20 or prior) will continue to use the + * old region name format. + */ + + /** Separator used to demarcate the encodedName in a region name + * in the new format. See description on new format above. + */ + private static final int ENC_SEPARATOR = '.'; + public static final int MD5_HEX_LENGTH = 32; + + /** A non-capture group so that this can be embedded. */ + public static final String ENCODED_REGION_NAME_REGEX = "(?:[a-f0-9]+)"; + + /** + * Does region name contain its encoded name? + * @param regionName region name + * @return boolean indicating if this a new format region + * name which contains its encoded name. + */ + private static boolean hasEncodedName(final byte[] regionName) { + // check if region name ends in ENC_SEPARATOR + if ((regionName.length >= 1) + && (regionName[regionName.length - 1] == ENC_SEPARATOR)) { + // region name is new format. it contains the encoded name. + return true; + } + return false; + } + + /** + * @param regionName + * @return the encodedName + */ + public static String encodeRegionName(final byte [] regionName) { + String encodedName; + if (hasEncodedName(regionName)) { + // region is in new format: + // ,,/encodedName/ + encodedName = Bytes.toString(regionName, + regionName.length - MD5_HEX_LENGTH - 1, + MD5_HEX_LENGTH); + } else { + // old format region name. ROOT and first META region also + // use this format.EncodedName is the JenkinsHash value. + int hashVal = Math.abs(JenkinsHash.getInstance().hash(regionName, + regionName.length, 0)); + encodedName = String.valueOf(hashVal); + } + return encodedName; + } + + /** + * Use logging. + * @param encodedRegionName The encoded regionname. + * @return -ROOT- if passed 70236052 or + * .META. if passed 1028785192 else returns + * encodedRegionName + */ + public static String prettyPrint(final String encodedRegionName) { + if (encodedRegionName.equals("70236052")) { + return encodedRegionName + "/-ROOT-"; + } else if (encodedRegionName.equals("1028785192")) { + return encodedRegionName + "/.META."; + } + return encodedRegionName; + } + + /** delimiter used between portions of a region name */ + public static final int DELIMITER = ','; + + /** HRegionInfo for root region */ + public static final HRegionInfo ROOT_REGIONINFO = + new HRegionInfo(0L, Bytes.toBytes("-ROOT-")); + + /** HRegionInfo for first meta region */ + public static final HRegionInfo FIRST_META_REGIONINFO = + new HRegionInfo(1L, Bytes.toBytes(".META.")); + + private byte [] endKey = HConstants.EMPTY_BYTE_ARRAY; + // This flag is in the parent of a split while the parent is still referenced + // by daughter regions. We USED to set this flag when we disabled a table + // but now table state is kept up in zookeeper as of 0.90.0 HBase. + private boolean offLine = false; + private long regionId = -1; + private transient byte [] regionName = HConstants.EMPTY_BYTE_ARRAY; + private String regionNameStr = ""; + private boolean split = false; + private byte [] startKey = HConstants.EMPTY_BYTE_ARRAY; + private int hashCode = -1; + //TODO: Move NO_HASH to HStoreFile which is really the only place it is used. + public static final String NO_HASH = null; + private volatile String encodedName = NO_HASH; + private byte [] encodedNameAsBytes = null; + + // Current TableName + private byte[] tableName = null; + + private void setHashCode() { + int result = Arrays.hashCode(this.regionName); + result ^= this.regionId; + result ^= Arrays.hashCode(this.startKey); + result ^= Arrays.hashCode(this.endKey); + result ^= Boolean.valueOf(this.offLine).hashCode(); + result ^= Arrays.hashCode(this.tableName); + this.hashCode = result; + } + + + /** + * Private constructor used constructing HRegionInfo for the catalog root and + * first meta regions + */ + private HRegionInfo(long regionId, byte[] tableName) { + super(); + this.regionId = regionId; + this.tableName = tableName.clone(); + // Note: Root & First Meta regions names are still in old format + this.regionName = createRegionName(tableName, null, + regionId, false); + this.regionNameStr = Bytes.toStringBinary(this.regionName); + setHashCode(); + } + + /** Default constructor - creates empty object */ + public HRegionInfo() { + super(); + } + + /** + * Used only for migration + * @param other HRegionInfoForMigration + */ + public HRegionInfo(HRegionInfo090x other) { + super(); + this.endKey = other.getEndKey(); + this.offLine = other.isOffline(); + this.regionId = other.getRegionId(); + this.regionName = other.getRegionName(); + this.regionNameStr = Bytes.toStringBinary(this.regionName); + this.split = other.isSplit(); + this.startKey = other.getStartKey(); + this.hashCode = other.hashCode(); + this.encodedName = other.getEncodedName(); + this.tableName = other.getTableDesc().getName(); + } + + public HRegionInfo(final byte[] tableName) { + this(tableName, null, null); + } + + /** + * Construct HRegionInfo with explicit parameters + * + * @param tableName the table name + * @param startKey first key in region + * @param endKey end of key range + * @throws IllegalArgumentException + */ + public HRegionInfo(final byte[] tableName, final byte[] startKey, + final byte[] endKey) + throws IllegalArgumentException { + this(tableName, startKey, endKey, false); + } + + + /** + * Construct HRegionInfo with explicit parameters + * + * @param tableName the table descriptor + * @param startKey first key in region + * @param endKey end of key range + * @param split true if this region has split and we have daughter regions + * regions that may or may not hold references to this region. + * @throws IllegalArgumentException + */ + public HRegionInfo(final byte[] tableName, final byte[] startKey, + final byte[] endKey, final boolean split) + throws IllegalArgumentException { + this(tableName, startKey, endKey, split, System.currentTimeMillis()); + } + + + /** + * Construct HRegionInfo with explicit parameters + * + * @param tableName the table descriptor + * @param startKey first key in region + * @param endKey end of key range + * @param split true if this region has split and we have daughter regions + * regions that may or may not hold references to this region. + * @param regionid Region id to use. + * @throws IllegalArgumentException + */ + public HRegionInfo(final byte[] tableName, final byte[] startKey, + final byte[] endKey, final boolean split, final long regionid) + throws IllegalArgumentException { + + super(); + if (tableName == null) { + throw new IllegalArgumentException("tableName cannot be null"); + } + this.tableName = tableName.clone(); + this.offLine = false; + this.regionId = regionid; + + this.regionName = createRegionName(this.tableName, startKey, regionId, true); + + this.regionNameStr = Bytes.toStringBinary(this.regionName); + this.split = split; + this.endKey = endKey == null? HConstants.EMPTY_END_ROW: endKey.clone(); + this.startKey = startKey == null? + HConstants.EMPTY_START_ROW: startKey.clone(); + this.tableName = tableName.clone(); + setHashCode(); + } + + /** + * Costruct a copy of another HRegionInfo + * + * @param other + */ + public HRegionInfo(HRegionInfo other) { + super(); + this.endKey = other.getEndKey(); + this.offLine = other.isOffline(); + this.regionId = other.getRegionId(); + this.regionName = other.getRegionName(); + this.regionNameStr = Bytes.toStringBinary(this.regionName); + this.split = other.isSplit(); + this.startKey = other.getStartKey(); + this.hashCode = other.hashCode(); + this.encodedName = other.getEncodedName(); + this.tableName = other.tableName; + } + + + /** + * Make a region name of passed parameters. + * @param tableName + * @param startKey Can be null + * @param regionid Region id (Usually timestamp from when region was created). + * @param newFormat should we create the region name in the new format + * (such that it contains its encoded name?). + * @return Region name made of passed tableName, startKey and id + */ + public static byte [] createRegionName(final byte [] tableName, + final byte [] startKey, final long regionid, boolean newFormat) { + return createRegionName(tableName, startKey, Long.toString(regionid), newFormat); + } + + /** + * Make a region name of passed parameters. + * @param tableName + * @param startKey Can be null + * @param id Region id (Usually timestamp from when region was created). + * @param newFormat should we create the region name in the new format + * (such that it contains its encoded name?). + * @return Region name made of passed tableName, startKey and id + */ + public static byte [] createRegionName(final byte [] tableName, + final byte [] startKey, final String id, boolean newFormat) { + return createRegionName(tableName, startKey, Bytes.toBytes(id), newFormat); + } + + /** + * Make a region name of passed parameters. + * @param tableName + * @param startKey Can be null + * @param id Region id (Usually timestamp from when region was created). + * @param newFormat should we create the region name in the new format + * (such that it contains its encoded name?). + * @return Region name made of passed tableName, startKey and id + */ + public static byte [] createRegionName(final byte [] tableName, + final byte [] startKey, final byte [] id, boolean newFormat) { + byte [] b = new byte [tableName.length + 2 + id.length + + (startKey == null? 0: startKey.length) + + (newFormat ? (MD5_HEX_LENGTH + 2) : 0)]; + + int offset = tableName.length; + System.arraycopy(tableName, 0, b, 0, offset); + b[offset++] = DELIMITER; + if (startKey != null && startKey.length > 0) { + System.arraycopy(startKey, 0, b, offset, startKey.length); + offset += startKey.length; + } + b[offset++] = DELIMITER; + System.arraycopy(id, 0, b, offset, id.length); + offset += id.length; + + if (newFormat) { + // + // Encoded name should be built into the region name. + // + // Use the region name thus far (namely, ,,) + // to compute a MD5 hash to be used as the encoded name, and append + // it to the byte buffer. + // + String md5Hash = MD5Hash.getMD5AsHex(b, 0, offset); + byte [] md5HashBytes = Bytes.toBytes(md5Hash); + + if (md5HashBytes.length != MD5_HEX_LENGTH) { + LOG.error("MD5-hash length mismatch: Expected=" + MD5_HEX_LENGTH + + "; Got=" + md5HashBytes.length); + } + + // now append the bytes '..' to the end + b[offset++] = ENC_SEPARATOR; + System.arraycopy(md5HashBytes, 0, b, offset, MD5_HEX_LENGTH); + offset += MD5_HEX_LENGTH; + b[offset++] = ENC_SEPARATOR; + } + + return b; + } + + /** + * Gets the table name from the specified region name. + * @param regionName + * @return Table name. + */ + public static byte [] getTableName(byte [] regionName) { + int offset = -1; + for (int i = 0; i < regionName.length; i++) { + if (regionName[i] == DELIMITER) { + offset = i; + break; + } + } + byte [] tableName = new byte[offset]; + System.arraycopy(regionName, 0, tableName, 0, offset); + return tableName; + } + + /** + * Gets the start key from the specified region name. + * @param regionName + * @return Start key. + */ + public static byte[] getStartKey(final byte[] regionName) throws IOException { + return parseRegionName(regionName)[1]; + } + + /** + * Separate elements of a regionName. + * @param regionName + * @return Array of byte[] containing tableName, startKey and id + * @throws IOException + */ + public static byte [][] parseRegionName(final byte [] regionName) + throws IOException { + int offset = -1; + for (int i = 0; i < regionName.length; i++) { + if (regionName[i] == DELIMITER) { + offset = i; + break; + } + } + if(offset == -1) throw new IOException("Invalid regionName format"); + byte [] tableName = new byte[offset]; + System.arraycopy(regionName, 0, tableName, 0, offset); + offset = -1; + for (int i = regionName.length - 1; i > 0; i--) { + if(regionName[i] == DELIMITER) { + offset = i; + break; + } + } + if(offset == -1) throw new IOException("Invalid regionName format"); + byte [] startKey = HConstants.EMPTY_BYTE_ARRAY; + if(offset != tableName.length + 1) { + startKey = new byte[offset - tableName.length - 1]; + System.arraycopy(regionName, tableName.length + 1, startKey, 0, + offset - tableName.length - 1); + } + byte [] id = new byte[regionName.length - offset - 1]; + System.arraycopy(regionName, offset + 1, id, 0, + regionName.length - offset - 1); + byte [][] elements = new byte[3][]; + elements[0] = tableName; + elements[1] = startKey; + elements[2] = id; + return elements; + } + + /** @return the regionId */ + public long getRegionId(){ + return regionId; + } + + /** + * @return the regionName as an array of bytes. + * @see #getRegionNameAsString() + */ + public byte [] getRegionName(){ + return regionName; + } + + /** + * @return Region name as a String for use in logging, etc. + */ + public String getRegionNameAsString() { + if (hasEncodedName(this.regionName)) { + // new format region names already have their encoded name. + return this.regionNameStr; + } + + // old format. regionNameStr doesn't have the region name. + // + // + return this.regionNameStr + "." + this.getEncodedName(); + } + + /** @return the encoded region name */ + public synchronized String getEncodedName() { + if (this.encodedName == NO_HASH) { + this.encodedName = encodeRegionName(this.regionName); + } + return this.encodedName; + } + + public synchronized byte [] getEncodedNameAsBytes() { + if (this.encodedNameAsBytes == null) { + this.encodedNameAsBytes = Bytes.toBytes(getEncodedName()); + } + return this.encodedNameAsBytes; + } + + /** @return the startKey */ + public byte [] getStartKey(){ + return startKey; + } + + /** @return the endKey */ + public byte [] getEndKey(){ + return endKey; + } + + /** + * Get current table name of the region + * @return byte array of table name + */ + public byte[] getTableName() { + if (tableName == null || tableName.length == 0) { + tableName = getTableName(getRegionName()); + } + return tableName; + } + + /** + * Get current table name as string + * @return string representation of current table + */ + public String getTableNameAsString() { + return Bytes.toString(tableName); + } + + /** + * Returns true if the given inclusive range of rows is fully contained + * by this region. For example, if the region is foo,a,g and this is + * passed ["b","c"] or ["a","c"] it will return true, but if this is passed + * ["b","z"] it will return false. + * @throws IllegalArgumentException if the range passed is invalid (ie end < start) + */ + public boolean containsRange(byte[] rangeStartKey, byte[] rangeEndKey) { + if (Bytes.compareTo(rangeStartKey, rangeEndKey) > 0) { + throw new IllegalArgumentException( + "Invalid range: " + Bytes.toStringBinary(rangeStartKey) + + " > " + Bytes.toStringBinary(rangeEndKey)); + } + + boolean firstKeyInRange = Bytes.compareTo(rangeStartKey, startKey) >= 0; + boolean lastKeyInRange = + Bytes.compareTo(rangeEndKey, endKey) < 0 || + Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY); + return firstKeyInRange && lastKeyInRange; + } + + /** + * Return true if the given row falls in this region. + */ + public boolean containsRow(byte[] row) { + return Bytes.compareTo(row, startKey) >= 0 && + (Bytes.compareTo(row, endKey) < 0 || + Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)); + } + + /** + * @return the tableDesc + * @deprecated Do not use; expensive call + * use HRegionInfo.getTableNameAsString() in place of + * HRegionInfo.getTableDesc().getNameAsString() + */ + @Deprecated + public HTableDescriptor getTableDesc() { + Configuration c = HBaseConfiguration.create(); + c.set("fs.defaultFS", c.get(HConstants.HBASE_DIR)); + c.set("fs.default.name", c.get(HConstants.HBASE_DIR)); + FileSystem fs; + try { + fs = FileSystem.get(c); + } catch (IOException e) { + throw new RuntimeException(e); + } + FSTableDescriptors fstd = + new FSTableDescriptors(fs, new Path(c.get(HConstants.HBASE_DIR))); + try { + return fstd.get(this.tableName); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * @param newDesc new table descriptor to use + * @deprecated Do not use; expensive call + */ + @Deprecated + public void setTableDesc(HTableDescriptor newDesc) { + Configuration c = HBaseConfiguration.create(); + FileSystem fs; + try { + fs = FileSystem.get(c); + } catch (IOException e) { + throw new RuntimeException(e); + } + FSTableDescriptors fstd = + new FSTableDescriptors(fs, new Path(c.get(HConstants.HBASE_DIR))); + try { + fstd.add(newDesc); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** @return true if this is the root region */ + public boolean isRootRegion() { + return Bytes.equals(tableName, HRegionInfo.ROOT_REGIONINFO.getTableName()); + } + + /** @return true if this region is from a table that is a meta table, + * either .META. or -ROOT- + */ + public boolean isMetaTable() { + return isRootRegion() || isMetaRegion(); + } + + /** @return true if this region is a meta region */ + public boolean isMetaRegion() { + return Bytes.equals(tableName, HRegionInfo.FIRST_META_REGIONINFO.getTableName()); + } + + /** + * @return True if has been split and has daughters. + */ + public boolean isSplit() { + return this.split; + } + + /** + * @param split set split status + */ + public void setSplit(boolean split) { + this.split = split; + } + + /** + * @return True if this region is offline. + */ + public boolean isOffline() { + return this.offLine; + } + + /** + * The parent of a region split is offline while split daughters hold + * references to the parent. Offlined regions are closed. + * @param offLine Set online/offline status. + */ + public void setOffline(boolean offLine) { + this.offLine = offLine; + } + + + /** + * @return True if this is a split parent region. + */ + public boolean isSplitParent() { + if (!isSplit()) return false; + if (!isOffline()) { + LOG.warn("Region is split but NOT offline: " + getRegionNameAsString()); + } + return true; + } + + /** + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "{" + HConstants.NAME + " => '" + + this.regionNameStr + + "', STARTKEY => '" + + Bytes.toStringBinary(this.startKey) + "', ENDKEY => '" + + Bytes.toStringBinary(this.endKey) + + "', ENCODED => " + getEncodedName() + "," + + (isOffline()? " OFFLINE => true,": "") + + (isSplit()? " SPLIT => true,": "") + "}"; + } + + /** + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null) { + return false; + } + if (!(o instanceof HRegionInfo)) { + return false; + } + return this.compareTo((HRegionInfo)o) == 0; + } + + /** + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + return this.hashCode; + } + + /** @return the object version number */ + @Override + public byte getVersion() { + return VERSION; + } + + // + // Writable + // + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + Bytes.writeByteArray(out, endKey); + out.writeBoolean(offLine); + out.writeLong(regionId); + Bytes.writeByteArray(out, regionName); + out.writeBoolean(split); + Bytes.writeByteArray(out, startKey); + Bytes.writeByteArray(out, tableName); + out.writeInt(hashCode); + } + + @Override + public void readFields(DataInput in) throws IOException { + // Read the single version byte. We don't ask the super class do it + // because freaks out if its not the current classes' version. This method + // can deserialize version 0 and version 1 of HRI. + byte version = in.readByte(); + if (version == 0) { + // This is the old HRI that carried an HTD. Migrate it. The below + // was copied from the old 0.90 HRI readFields. + this.endKey = Bytes.readByteArray(in); + this.offLine = in.readBoolean(); + this.regionId = in.readLong(); + this.regionName = Bytes.readByteArray(in); + this.regionNameStr = Bytes.toStringBinary(this.regionName); + this.split = in.readBoolean(); + this.startKey = Bytes.readByteArray(in); + try { + HTableDescriptor htd = new HTableDescriptor(); + htd.readFields(in); + this.tableName = htd.getName(); + } catch(EOFException eofe) { + throw new IOException("HTD not found in input buffer", eofe); + } + this.hashCode = in.readInt(); + } else if (getVersion() == version) { + this.endKey = Bytes.readByteArray(in); + this.offLine = in.readBoolean(); + this.regionId = in.readLong(); + this.regionName = Bytes.readByteArray(in); + this.regionNameStr = Bytes.toStringBinary(this.regionName); + this.split = in.readBoolean(); + this.startKey = Bytes.readByteArray(in); + this.tableName = Bytes.readByteArray(in); + this.hashCode = in.readInt(); + } else { + throw new IOException("Non-migratable/unknown version=" + getVersion()); + } + } + + // + // Comparable + // + + public int compareTo(HRegionInfo o) { + if (o == null) { + return 1; + } + + // Are regions of same table? + int result = Bytes.compareTo(this.tableName, o.tableName); + if (result != 0) { + return result; + } + + // Compare start keys. + result = Bytes.compareTo(this.startKey, o.startKey); + if (result != 0) { + return result; + } + + // Compare end keys. + result = Bytes.compareTo(this.endKey, o.endKey); + + if (result != 0) { + if (this.getStartKey().length != 0 + && this.getEndKey().length == 0) { + return 1; // this is last region + } + if (o.getStartKey().length != 0 + && o.getEndKey().length == 0) { + return -1; // o is the last region + } + return result; + } + + // regionId is usually milli timestamp -- this defines older stamps + // to be "smaller" than newer stamps in sort order. + if (this.regionId > o.regionId) { + return 1; + } else if (this.regionId < o.regionId) { + return -1; + } + + if (this.offLine == o.offLine) + return 0; + if (this.offLine == true) return -1; + + return 1; + } + + /** + * @return Comparator to use comparing {@link KeyValue}s. + */ + public KVComparator getComparator() { + return isRootRegion()? KeyValue.ROOT_COMPARATOR: isMetaRegion()? + KeyValue.META_COMPARATOR: KeyValue.COMPARATOR; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java b/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java new file mode 100644 index 0000000..85fb91d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java @@ -0,0 +1,134 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.util.Addressing; + +/** + * Data structure to hold HRegionInfo and the address for the hosting + * HRegionServer. Immutable. Comparable, but we compare the 'location' only: + * i.e. the hostname and port, and *not* the regioninfo. This means two + * instances are the same if they refer to the same 'location' (the same + * hostname and port), though they may be carrying different regions. + */ +public class HRegionLocation implements Comparable { + private final HRegionInfo regionInfo; + private final String hostname; + private final int port; + // Cache of the 'toString' result. + private String cachedString = null; + // Cache of the hostname + port + private String cachedHostnamePort; + + /** + * Constructor + * @param regionInfo the HRegionInfo for the region + * @param hostname Hostname + * @param port port + */ + public HRegionLocation(HRegionInfo regionInfo, final String hostname, + final int port) { + this.regionInfo = regionInfo; + this.hostname = hostname; + this.port = port; + } + + /** + * @see java.lang.Object#toString() + */ + @Override + public synchronized String toString() { + if (this.cachedString == null) { + this.cachedString = "region=" + this.regionInfo.getRegionNameAsString() + + ", hostname=" + this.hostname + ", port=" + this.port; + } + return this.cachedString; + } + + /** + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null) { + return false; + } + if (!(o instanceof HRegionLocation)) { + return false; + } + return this.compareTo((HRegionLocation)o) == 0; + } + + /** + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + int result = this.hostname.hashCode(); + result ^= this.port; + return result; + } + + /** @return HRegionInfo */ + public HRegionInfo getRegionInfo(){ + return regionInfo; + } + + /** + * Do not use!!! Creates a HServerAddress instance which will do a resolve. + * @return HServerAddress + * @deprecated Use {@link #getHostnamePort} + */ + public HServerAddress getServerAddress() { + return new HServerAddress(this.hostname, this.port); + } + + public String getHostname() { + return this.hostname; + } + + public int getPort() { + return this.port; + } + + /** + * @return String made of hostname and port formatted as per {@link Addressing#createHostAndPortStr(String, int)} + */ + public synchronized String getHostnamePort() { + if (this.cachedHostnamePort == null) { + this.cachedHostnamePort = + Addressing.createHostAndPortStr(this.hostname, this.port); + } + return this.cachedHostnamePort; + } + + // + // Comparable + // + + public int compareTo(HRegionLocation o) { + int result = this.hostname.compareTo(o.getHostname()); + if (result != 0) return result; + return this.port - o.getPort(); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/HServerAddress.java b/src/main/java/org/apache/hadoop/hbase/HServerAddress.java new file mode 100644 index 0000000..e189aaf --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HServerAddress.java @@ -0,0 +1,200 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; + +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.WritableComparable; + +/** + * HServerAddress hosts a {@link InetSocketAddress} and makes it + * {@link WritableComparable}. Resolves on construction AND on + * deserialization -- since we're internally creating an InetSocketAddress -- + * so could end up with different results if the two ends of serialization have + * different resolvers. Be careful where you use it. Should only be used when + * you need to pass an InetSocketAddress across an RPC. Even then its a bad + * idea because of the above resolve issue. + * @deprecated Use {@link InetSocketAddress} or {@link ServerName} or + * a hostname String and port. + */ +public class HServerAddress implements WritableComparable { + // Hard to deprecate this class. Its in the API as internal class, + // in particular as an inner class of HRegionLocation. Besides, sometimes + // we do want to serialize a InetSocketAddress; this class can be used then. + private InetSocketAddress address = null; + private String cachedToString = ""; + + /** + * Constructor for deserialization use only. + */ + public HServerAddress() { + super(); + } + + /** + * Construct an instance from an {@link InetSocketAddress}. + * @param address InetSocketAddress of server + */ + public HServerAddress(InetSocketAddress address) { + this.address = address; + checkBindAddressCanBeResolved(); + this.cachedToString = createCachedToString(); + } + + private String createCachedToString() { + return this.address.toString(); + } + + /** + * @param hostname Hostname + * @param port Port number + */ + public HServerAddress(final String hostname, final int port) { + this(getResolvedAddress(new InetSocketAddress(hostname, port))); + } + + /** + * Copy-constructor. + * @param other HServerAddress to copy from + */ + public HServerAddress(HServerAddress other) { + this(getResolvedAddress(new InetSocketAddress(other.getHostname(), other.getPort()))); + } + + private static InetSocketAddress getResolvedAddress(InetSocketAddress address) { + String bindAddress = getBindAddressInternal(address); + int port = address.getPort(); + return new InetSocketAddress(bindAddress, port); + } + + /** @return Bind address -- the raw IP, the result of a call to + * InetSocketAddress#getAddress()#getHostAddress() -- + * or null if cannot resolve */ + public String getBindAddress() { + return getBindAddressInternal(address); + } + + private static String getBindAddressInternal(InetSocketAddress address) { + final InetAddress addr = address.getAddress(); + if (addr != null) { + return addr.getHostAddress(); + } else { + LogFactory.getLog(HServerAddress.class).error("Could not resolve the" + + " DNS name of " + address.getHostName()); + return null; + } + } + + private void checkBindAddressCanBeResolved() { + if (getBindAddress() == null) { + throw new IllegalArgumentException("Could not resolve the" + + " DNS name of " + this.address.toString()); + } + } + + /** @return Port number */ + public int getPort() { + return this.address.getPort(); + } + + /** @return Hostname */ + public String getHostname() { + // Kerberos is case-sensitive, and dictates that, where hostnames are + // case-insensitive (as in DNS), the lowercase version must be used + // So here we lowercase to properly interact with kerberos auth + return this.address.getHostName().toLowerCase(); + } + + /** + * @return Returns ':' + */ + public String getHostnameAndPort() { + return getHostname() + ":" + getPort(); + } + + /** @return The InetSocketAddress */ + public InetSocketAddress getInetSocketAddress() { + return this.address; + } + + /** + * @return String formatted as <bind address> ':' <port> + */ + @Override + public String toString() { + return this.cachedToString; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null) return false; + if (getClass() != o.getClass()) return false; + return compareTo((HServerAddress)o) == 0; + } + + @Override + public int hashCode() { + int result = address == null? 0: address.hashCode(); + result ^= toString().hashCode(); + return result; + } + + // + // Writable + // + + public void readFields(DataInput in) throws IOException { + String hostname = in.readUTF(); + int port = in.readInt(); + if (hostname != null && hostname.length() > 0) { + this.address = getResolvedAddress(new InetSocketAddress(hostname, port)); + checkBindAddressCanBeResolved(); + createCachedToString(); + } + } + + public void write(DataOutput out) throws IOException { + if (this.address == null) { + out.writeUTF(""); + out.writeInt(0); + } else { + out.writeUTF(this.address.getAddress().getHostName()); + out.writeInt(this.address.getPort()); + } + } + + // + // Comparable + // + + public int compareTo(HServerAddress o) { + // Addresses as Strings may not compare though address is for the one + // server with only difference being that one address has hostname + // resolved whereas other only has IP. + if (this.address.equals(o.address)) return 0; + return toString().compareTo(o.toString()); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/HServerInfo.java b/src/main/java/org/apache/hadoop/hbase/HServerInfo.java new file mode 100644 index 0000000..5934246 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HServerInfo.java @@ -0,0 +1,151 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.io.VersionedWritable; +import org.apache.hadoop.io.WritableComparable; + + +/** + * HServerInfo is meta info about an {@link HRegionServer}. It hosts the + * {@link HServerAddress}, its webui port, and its server startcode. It was + * used to pass meta info about a server across an RPC but we've since made + * it so regionserver info is up in ZooKeeper and so this class is on its + * way out. It used to carry {@link HServerLoad} but as off HBase 0.92.0, the + * HServerLoad is passed independent of this class. Also, we now no longer pass + * the webui from regionserver to master (TODO: Fix). + * @deprecated Use {@link InetSocketAddress} and or {@link ServerName} and or + * {@link HServerLoad} + */ +public class HServerInfo extends VersionedWritable +implements WritableComparable { + private static final byte VERSION = 1; + private HServerAddress serverAddress = new HServerAddress(); + private long startCode; + private int webuiport; + + public HServerInfo() { + super(); + } + + /** + * Constructor that creates a HServerInfo with a generated startcode + * @param serverAddress + * @param webuiport Port the webui runs on. + */ + public HServerInfo(final HServerAddress serverAddress, final int webuiport) { + this(serverAddress, System.currentTimeMillis(), webuiport); + } + + public HServerInfo(HServerAddress serverAddress, long startCode, + final int webuiport) { + this.serverAddress = serverAddress; + this.startCode = startCode; + this.webuiport = webuiport; + } + + /** + * Copy-constructor + * @param other + */ + public HServerInfo(HServerInfo other) { + this.serverAddress = new HServerAddress(other.getServerAddress()); + this.startCode = other.getStartCode(); + this.webuiport = other.getInfoPort(); + } + + /** @return the object version number */ + public byte getVersion() { + return VERSION; + } + + public synchronized HServerAddress getServerAddress() { + return new HServerAddress(serverAddress); + } + + public synchronized long getStartCode() { + return startCode; + } + + public int getInfoPort() { + return getWebuiPort(); + } + + public int getWebuiPort() { + return this.webuiport; + } + + public String getHostname() { + return this.serverAddress.getHostname(); + } + + /** + * @return ServerName and load concatenated. + */ + @Override + public synchronized String toString() { + return ServerName.getServerName(this.serverAddress.getHostnameAndPort(), + this.startCode); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + return compareTo((HServerInfo)obj) == 0; + } + + @Override + public int hashCode() { + int code = this.serverAddress.hashCode(); + code ^= this.webuiport; + code ^= this.startCode; + return code; + } + + public void readFields(DataInput in) throws IOException { + super.readFields(in); + this.serverAddress.readFields(in); + this.startCode = in.readLong(); + this.webuiport = in.readInt(); + } + + public void write(DataOutput out) throws IOException { + super.write(out); + this.serverAddress.write(out); + out.writeLong(this.startCode); + out.writeInt(this.webuiport); + } + + public int compareTo(HServerInfo o) { + int compare = this.serverAddress.compareTo(o.getServerAddress()); + if (compare != 0) return compare; + if (this.webuiport != o.getInfoPort()) return this.webuiport - o.getInfoPort(); + if (this.startCode != o.getStartCode()) return (int)(this.startCode - o.getStartCode()); + return 0; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java new file mode 100644 index 0000000..ffdbc6b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java @@ -0,0 +1,717 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Strings; +import org.apache.hadoop.io.VersionedWritable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableUtils; + +/** + * This class is used exporting current state of load on a RegionServer. + */ +public class HServerLoad extends VersionedWritable +implements WritableComparable { + private static final byte VERSION = 2; + // Empty load instance. + public static final HServerLoad EMPTY_HSERVERLOAD = new HServerLoad(); + + /** Number of requests per second since last report. + */ + // TODO: Instead build this up out of region counters. + private int numberOfRequests = 0; + + /** Total Number of requests from the start of the region server. + */ + private int totalNumberOfRequests = 0; + + /** the amount of used heap, in MB */ + private int usedHeapMB = 0; + + /** the maximum allowable size of the heap, in MB */ + private int maxHeapMB = 0; + + // Regionserver-level coprocessors, e.g., WALObserver implementations. + private Set coprocessors = new TreeSet(); + + /** + * HBASE-4070: Improve region server metrics to report loaded coprocessors. + * @return the set of all the server-wide coprocessors on this regionserver + */ + public String[] getRsCoprocessors() { + return coprocessors.toArray(new String[0]); + } + + /** per-region load metrics */ + private Map regionLoad = + new TreeMap(Bytes.BYTES_COMPARATOR); + + /** @return the object version number */ + public byte getVersion() { + return VERSION; + } + + /** + * Encapsulates per-region loading metrics. + */ + public static class RegionLoad extends VersionedWritable { + private static final byte VERSION = 2; + + /** @return the object version number */ + public byte getVersion() { + return VERSION; + } + + /** the region name */ + private byte[] name; + /** the number of stores for the region */ + private int stores; + /** the number of storefiles for the region */ + private int storefiles; + /** the total size of the store files for the region, uncompressed, in MB */ + private int storeUncompressedSizeMB; + /** the current total size of the store files for the region, in MB */ + private int storefileSizeMB; + /** the current size of the memstore for the region, in MB */ + private int memstoreSizeMB; + + /** + * The current total size of root-level store file indexes for the region, + * in MB. The same as {@link #rootIndexSizeKB} but in MB. + */ + private int storefileIndexSizeMB; + /** the current total read requests made to region */ + private long readRequestsCount; + /** the current total write requests made to region */ + private long writeRequestsCount; + /** the total compacting key values in currently running compaction */ + private long totalCompactingKVs; + /** the completed count of key values in currently running compaction */ + private long currentCompactedKVs; + + /** The current total size of root-level indexes for the region, in KB. */ + private int rootIndexSizeKB; + + /** The total size of all index blocks, not just the root level, in KB. */ + private int totalStaticIndexSizeKB; + + /** + * The total size of all Bloom filter blocks, not just loaded into the + * block cache, in KB. + */ + private int totalStaticBloomSizeKB; + + /** + * Constructor, for Writable + */ + public RegionLoad() { + super(); + } + + /** + * @param name + * @param stores + * @param storefiles + * @param storeUncompressedSizeMB + * @param storefileSizeMB + * @param memstoreSizeMB + * @param storefileIndexSizeMB + * @param readRequestsCount + * @param writeRequestsCount + * @param totalCompactingKVs + * @param currentCompactedKVs + */ + public RegionLoad(final byte[] name, final int stores, + final int storefiles, final int storeUncompressedSizeMB, + final int storefileSizeMB, + final int memstoreSizeMB, final int storefileIndexSizeMB, + final int rootIndexSizeKB, final int totalStaticIndexSizeKB, + final int totalStaticBloomSizeKB, + final long readRequestsCount, final long writeRequestsCount, + final long totalCompactingKVs, final long currentCompactedKVs) { + this.name = name; + this.stores = stores; + this.storefiles = storefiles; + this.storeUncompressedSizeMB = storeUncompressedSizeMB; + this.storefileSizeMB = storefileSizeMB; + this.memstoreSizeMB = memstoreSizeMB; + this.storefileIndexSizeMB = storefileIndexSizeMB; + this.rootIndexSizeKB = rootIndexSizeKB; + this.totalStaticIndexSizeKB = totalStaticIndexSizeKB; + this.totalStaticBloomSizeKB = totalStaticBloomSizeKB; + this.readRequestsCount = readRequestsCount; + this.writeRequestsCount = writeRequestsCount; + this.totalCompactingKVs = totalCompactingKVs; + this.currentCompactedKVs = currentCompactedKVs; + } + + /** + * @return the region name + */ + public byte[] getName() { + return name; + } + + /** + * @return the region name as a string + */ + public String getNameAsString() { + return Bytes.toString(name); + } + + /** + * @return the number of stores + */ + public int getStores() { + return stores; + } + + /** + * @return the number of storefiles + */ + public int getStorefiles() { + return storefiles; + } + + /** + * @return the total size of the storefiles, in MB + */ + public int getStorefileSizeMB() { + return storefileSizeMB; + } + + /** + * @return the memstore size, in MB + */ + public int getMemStoreSizeMB() { + return memstoreSizeMB; + } + + /** + * @return the approximate size of storefile indexes on the heap, in MB + */ + public int getStorefileIndexSizeMB() { + return storefileIndexSizeMB; + } + + /** + * @return the number of requests made to region + */ + public long getRequestsCount() { + return readRequestsCount + writeRequestsCount; + } + + /** + * @return the number of read requests made to region + */ + public long getReadRequestsCount() { + return readRequestsCount; + } + + /** + * @return the number of read requests made to region + */ + public long getWriteRequestsCount() { + return writeRequestsCount; + } + + /** + * @return The current total size of root-level indexes for the region, in KB. + */ + public int getRootIndexSizeKB() { + return rootIndexSizeKB; + } + + /** + * @return The total size of all index blocks, not just the root level, in KB. + */ + public int getTotalStaticIndexSizeKB() { + return totalStaticIndexSizeKB; + } + + /** + * @return The total size of all Bloom filter blocks, not just loaded into the + * block cache, in KB. + */ + public int getTotalStaticBloomSizeKB() { + return totalStaticBloomSizeKB; + } + + /** + * @return the total number of kvs in current compaction + */ + public long getTotalCompactingKVs() { + return totalCompactingKVs; + } + + /** + * @return the number of already compacted kvs in current compaction + */ + public long getCurrentCompactedKVs() { + return currentCompactedKVs; + } + + // Setters + + /** + * @param name the region name + */ + public void setName(byte[] name) { + this.name = name; + } + + /** + * @param stores the number of stores + */ + public void setStores(int stores) { + this.stores = stores; + } + + /** + * @param storefiles the number of storefiles + */ + public void setStorefiles(int storefiles) { + this.storefiles = storefiles; + } + + /** + * @param memstoreSizeMB the memstore size, in MB + */ + public void setMemStoreSizeMB(int memstoreSizeMB) { + this.memstoreSizeMB = memstoreSizeMB; + } + + /** + * @param storefileIndexSizeMB the approximate size of storefile indexes + * on the heap, in MB + */ + public void setStorefileIndexSizeMB(int storefileIndexSizeMB) { + this.storefileIndexSizeMB = storefileIndexSizeMB; + } + + /** + * @param requestsCount the number of read requests to region + */ + public void setReadRequestsCount(int requestsCount) { + this.readRequestsCount = requestsCount; + } + + /** + * @param requestsCount the number of write requests to region + */ + public void setWriteRequestsCount(int requestsCount) { + this.writeRequestsCount = requestsCount; + } + + /** + * @param totalCompactingKVs the number of kvs total in current compaction + */ + public void setTotalCompactingKVs(long totalCompactingKVs) { + this.totalCompactingKVs = totalCompactingKVs; + } + + /** + * @param currentCompactedKVs the number of kvs already compacted in + * current compaction + */ + public void setCurrentCompactedKVs(long currentCompactedKVs) { + this.currentCompactedKVs = currentCompactedKVs; + } + + /** + * HBASE-5256 and HBASE-5283 introduced incompatible serialization changes + * This method reads the fields in 0.92 serialization format, ex-version field + * @param in + * @throws IOException + */ + private void readFields92(DataInput in) throws IOException { + // in 0.92, the version was actually written twice, consume the second copy + int version = in.readByte(); + int namelen = in.readInt(); + this.name = new byte[namelen]; + in.readFully(this.name); + this.stores = in.readInt(); + this.storefiles = in.readInt(); + this.storeUncompressedSizeMB = in.readInt(); + this.storefileSizeMB = in.readInt(); + this.memstoreSizeMB = in.readInt(); + this.storefileIndexSizeMB = in.readInt(); + this.readRequestsCount = in.readInt(); + this.writeRequestsCount = in.readInt(); + this.rootIndexSizeKB = in.readInt(); + this.totalStaticIndexSizeKB = in.readInt(); + this.totalStaticBloomSizeKB = in.readInt(); + this.totalCompactingKVs = in.readLong(); + this.currentCompactedKVs = in.readLong(); + int coprocessorsSize = in.readInt(); + // Backward compatibility - there may be coprocessors in the region load, ignore them. + for (int i = 0; i < coprocessorsSize; i++) { + in.readUTF(); + } + } + + // Writable + public void readFields(DataInput in) throws IOException { + int version = in.readByte(); + if (version > VERSION) throw new IOException("Version mismatch; " + version); + if (version == 1) { + readFields92(in); + return; + } + int namelen = WritableUtils.readVInt(in); + this.name = new byte[namelen]; + in.readFully(this.name); + this.stores = WritableUtils.readVInt(in); + this.storefiles = WritableUtils.readVInt(in); + this.storeUncompressedSizeMB = WritableUtils.readVInt(in); + this.storefileSizeMB = WritableUtils.readVInt(in); + this.memstoreSizeMB = WritableUtils.readVInt(in); + this.storefileIndexSizeMB = WritableUtils.readVInt(in); + this.readRequestsCount = WritableUtils.readVLong(in); + this.writeRequestsCount = WritableUtils.readVLong(in); + this.rootIndexSizeKB = WritableUtils.readVInt(in); + this.totalStaticIndexSizeKB = WritableUtils.readVInt(in); + this.totalStaticBloomSizeKB = WritableUtils.readVInt(in); + this.totalCompactingKVs = WritableUtils.readVLong(in); + this.currentCompactedKVs = WritableUtils.readVLong(in); + int coprocessorsSize = WritableUtils.readVInt(in); + // Backward compatibility - there may be coprocessors in the region load, ignore them. + for (int i = 0; i < coprocessorsSize; i++) { + in.readUTF(); + } + } + + public void write(DataOutput out) throws IOException { + super.write(out); + WritableUtils.writeVInt(out, name.length); + out.write(name); + WritableUtils.writeVInt(out, stores); + WritableUtils.writeVInt(out, storefiles); + WritableUtils.writeVInt(out, storeUncompressedSizeMB); + WritableUtils.writeVInt(out, storefileSizeMB); + WritableUtils.writeVInt(out, memstoreSizeMB); + WritableUtils.writeVInt(out, storefileIndexSizeMB); + WritableUtils.writeVLong(out, readRequestsCount); + WritableUtils.writeVLong(out, writeRequestsCount); + WritableUtils.writeVInt(out, rootIndexSizeKB); + WritableUtils.writeVInt(out, totalStaticIndexSizeKB); + WritableUtils.writeVInt(out, totalStaticBloomSizeKB); + WritableUtils.writeVLong(out, totalCompactingKVs); + WritableUtils.writeVLong(out, currentCompactedKVs); + // Backward compatibility - write out 0 as coprocessor count, + // we don't report region-level coprocessors anymore. + WritableUtils.writeVInt(out, 0); + } + + /** + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "numberOfStores", + Integer.valueOf(this.stores)); + sb = Strings.appendKeyValue(sb, "numberOfStorefiles", + Integer.valueOf(this.storefiles)); + sb = Strings.appendKeyValue(sb, "storefileUncompressedSizeMB", + Integer.valueOf(this.storeUncompressedSizeMB)); + sb = Strings.appendKeyValue(sb, "storefileSizeMB", + Integer.valueOf(this.storefileSizeMB)); + if (this.storeUncompressedSizeMB != 0) { + sb = Strings.appendKeyValue(sb, "compressionRatio", + String.format("%.4f", (float)this.storefileSizeMB/ + (float)this.storeUncompressedSizeMB)); + } + sb = Strings.appendKeyValue(sb, "memstoreSizeMB", + Integer.valueOf(this.memstoreSizeMB)); + sb = Strings.appendKeyValue(sb, "storefileIndexSizeMB", + Integer.valueOf(this.storefileIndexSizeMB)); + sb = Strings.appendKeyValue(sb, "readRequestsCount", + Long.valueOf(this.readRequestsCount)); + sb = Strings.appendKeyValue(sb, "writeRequestsCount", + Long.valueOf(this.writeRequestsCount)); + sb = Strings.appendKeyValue(sb, "rootIndexSizeKB", + Integer.valueOf(this.rootIndexSizeKB)); + sb = Strings.appendKeyValue(sb, "totalStaticIndexSizeKB", + Integer.valueOf(this.totalStaticIndexSizeKB)); + sb = Strings.appendKeyValue(sb, "totalStaticBloomSizeKB", + Integer.valueOf(this.totalStaticBloomSizeKB)); + sb = Strings.appendKeyValue(sb, "totalCompactingKVs", + Long.valueOf(this.totalCompactingKVs)); + sb = Strings.appendKeyValue(sb, "currentCompactedKVs", + Long.valueOf(this.currentCompactedKVs)); + float compactionProgressPct = Float.NaN; + if( this.totalCompactingKVs > 0 ) { + compactionProgressPct = Float.valueOf( + this.currentCompactedKVs / this.totalCompactingKVs); + } + sb = Strings.appendKeyValue(sb, "compactionProgressPct", + compactionProgressPct); + return sb.toString(); + } + } + + /* + * TODO: Other metrics that might be considered when the master is actually + * doing load balancing instead of merely trying to decide where to assign + * a region: + *

    + *
  • # of CPUs, heap size (to determine the "class" of machine). For + * now, we consider them to be homogeneous.
    • + *
  • #requests per region (Map<{String|HRegionInfo}, Integer>)
    • + *
  • #compactions and/or #splits (churn)
    • + *
  • server death rate (maybe there is something wrong with this server)
    • + *
+ */ + + /** default constructor (used by Writable) */ + public HServerLoad() { + super(); + } + + /** + * Constructor + * @param numberOfRequests + * @param usedHeapMB + * @param maxHeapMB + * @param coprocessors : coprocessors loaded at the regionserver-level + */ + public HServerLoad(final int totalNumberOfRequests, + final int numberOfRequests, final int usedHeapMB, final int maxHeapMB, + final Map regionLoad, + final Set coprocessors) { + this.numberOfRequests = numberOfRequests; + this.usedHeapMB = usedHeapMB; + this.maxHeapMB = maxHeapMB; + this.regionLoad = regionLoad; + this.totalNumberOfRequests = totalNumberOfRequests; + this.coprocessors = coprocessors; + } + + /** + * Constructor + * @param hsl the template HServerLoad + */ + public HServerLoad(final HServerLoad hsl) { + this(hsl.totalNumberOfRequests, hsl.numberOfRequests, hsl.usedHeapMB, + hsl.maxHeapMB, hsl.getRegionsLoad(), hsl.coprocessors); + for (Map.Entry e : hsl.regionLoad.entrySet()) { + this.regionLoad.put(e.getKey(), e.getValue()); + } + } + + /** + * Originally, this method factored in the effect of requests going to the + * server as well. However, this does not interact very well with the current + * region rebalancing code, which only factors number of regions. For the + * interim, until we can figure out how to make rebalancing use all the info + * available, we're just going to make load purely the number of regions. + * + * @return load factor for this server + */ + public int getLoad() { + // int load = numberOfRequests == 0 ? 1 : numberOfRequests; + // load *= numberOfRegions == 0 ? 1 : numberOfRegions; + // return load; + return this.regionLoad.size(); + } + + /** + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return toString(1); + } + + /** + * Returns toString() with the number of requests divided by the message + * interval in seconds + * @param msgInterval + * @return The load as a String + */ + public String toString(int msgInterval) { + int numberOfRegions = this.regionLoad.size(); + StringBuilder sb = new StringBuilder(); + sb = Strings.appendKeyValue(sb, "requestsPerSecond", + Integer.valueOf(numberOfRequests/msgInterval)); + sb = Strings.appendKeyValue(sb, "numberOfOnlineRegions", + Integer.valueOf(numberOfRegions)); + sb = Strings.appendKeyValue(sb, "usedHeapMB", + Integer.valueOf(this.usedHeapMB)); + sb = Strings.appendKeyValue(sb, "maxHeapMB", Integer.valueOf(maxHeapMB)); + return sb.toString(); + } + + /** + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null) { + return false; + } + if (getClass() != o.getClass()) { + return false; + } + return compareTo((HServerLoad)o) == 0; + } + + // Getters + + /** + * @return the numberOfRegions + */ + public int getNumberOfRegions() { + return this.regionLoad.size(); + } + + /** + * @return the numberOfRequests per second. + */ + public int getNumberOfRequests() { + return numberOfRequests; + } + + /** + * @return the numberOfRequests + */ + public int getTotalNumberOfRequests() { + return totalNumberOfRequests; + } + + /** + * @return the amount of heap in use, in MB + */ + public int getUsedHeapMB() { + return usedHeapMB; + } + + /** + * @return the maximum allowable heap size, in MB + */ + public int getMaxHeapMB() { + return maxHeapMB; + } + + /** + * @return region load metrics + */ + public Map getRegionsLoad() { + return Collections.unmodifiableMap(regionLoad); + } + + /** + * @return Count of storefiles on this regionserver + */ + public int getStorefiles() { + int count = 0; + for (RegionLoad info: regionLoad.values()) + count += info.getStorefiles(); + return count; + } + + /** + * @return Total size of store files in MB + */ + public int getStorefileSizeInMB() { + int count = 0; + for (RegionLoad info: regionLoad.values()) + count += info.getStorefileSizeMB(); + return count; + } + + /** + * @return Size of memstores in MB + */ + public int getMemStoreSizeInMB() { + int count = 0; + for (RegionLoad info: regionLoad.values()) + count += info.getMemStoreSizeMB(); + return count; + } + + /** + * @return Size of store file indexes in MB + */ + public int getStorefileIndexSizeInMB() { + int count = 0; + for (RegionLoad info: regionLoad.values()) + count += info.getStorefileIndexSizeMB(); + return count; + } + + // Writable + + public void readFields(DataInput in) throws IOException { + super.readFields(in); + int version = in.readByte(); + if (version > VERSION) throw new IOException("Version mismatch; " + version); + numberOfRequests = in.readInt(); + usedHeapMB = in.readInt(); + maxHeapMB = in.readInt(); + int numberOfRegions = in.readInt(); + for (int i = 0; i < numberOfRegions; i++) { + RegionLoad rl = new RegionLoad(); + rl.readFields(in); + regionLoad.put(rl.getName(), rl); + } + totalNumberOfRequests = in.readInt(); + int coprocessorsSize = in.readInt(); + for(int i = 0; i < coprocessorsSize; i++) { + coprocessors.add(in.readUTF()); + } + } + + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeByte(VERSION); + out.writeInt(numberOfRequests); + out.writeInt(usedHeapMB); + out.writeInt(maxHeapMB); + out.writeInt(this.regionLoad.size()); + for (RegionLoad rl: regionLoad.values()) + rl.write(out); + out.writeInt(totalNumberOfRequests); + out.writeInt(coprocessors.size()); + for (String coprocessor: coprocessors) { + out.writeUTF(coprocessor); + } + } + + // Comparable + + public int compareTo(HServerLoad o) { + return this.getLoad() - o.getLoad(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java new file mode 100644 index 0000000..e9c17b1 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -0,0 +1,1237 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.HashSet; +import java.util.TreeSet; +import java.util.TreeMap; +import java.util.regex.Matcher; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableComparable; + +/** + * HTableDescriptor contains the details about an HBase table such as the descriptors of + * all the column families, is the table a catalog table, -ROOT- or + * .META. , is the table is read only, the maximum size of the memstore, + * when the region split should occur, coprocessors associated with it etc... + */ +public class HTableDescriptor implements WritableComparable { + + /** + * Changes prior to version 3 were not recorded here. + * Version 3 adds metadata as a map where keys and values are byte[]. + * Version 4 adds indexes + * Version 5 removed transactional pollution -- e.g. indexes + */ + private static final byte TABLE_DESCRIPTOR_VERSION = 5; + + private byte [] name = HConstants.EMPTY_BYTE_ARRAY; + + private String nameAsString = ""; + + /** + * A map which holds the metadata information of the table. This metadata + * includes values like IS_ROOT, IS_META, DEFERRED_LOG_FLUSH, SPLIT_POLICY, + * MAX_FILE_SIZE, READONLY, MEMSTORE_FLUSHSIZE etc... + */ + protected final Map values = + new HashMap(); + + private static final String FAMILIES = "FAMILIES"; + + public static final String SPLIT_POLICY = "SPLIT_POLICY"; + + /** + * INTERNAL Used by HBase Shell interface to access this metadata + * attribute which denotes the maximum size of the store file after which + * a region split occurs + * + * @see #getMaxFileSize() + */ + public static final String MAX_FILESIZE = "MAX_FILESIZE"; + private static final ImmutableBytesWritable MAX_FILESIZE_KEY = + new ImmutableBytesWritable(Bytes.toBytes(MAX_FILESIZE)); + + public static final String OWNER = "OWNER"; + public static final ImmutableBytesWritable OWNER_KEY = + new ImmutableBytesWritable(Bytes.toBytes(OWNER)); + + /** + * INTERNAL Used by rest interface to access this metadata + * attribute which denotes if the table is Read Only + * + * @see #isReadOnly() + */ + public static final String READONLY = "READONLY"; + private static final ImmutableBytesWritable READONLY_KEY = + new ImmutableBytesWritable(Bytes.toBytes(READONLY)); + + /** + * INTERNAL Used by HBase Shell interface to access this metadata + * attribute which represents the maximum size of the memstore after which + * its contents are flushed onto the disk + * + * @see #getMemStoreFlushSize() + */ + public static final String MEMSTORE_FLUSHSIZE = "MEMSTORE_FLUSHSIZE"; + private static final ImmutableBytesWritable MEMSTORE_FLUSHSIZE_KEY = + new ImmutableBytesWritable(Bytes.toBytes(MEMSTORE_FLUSHSIZE)); + + /** + * INTERNAL Used by rest interface to access this metadata + * attribute which denotes if the table is a -ROOT- region or not + * + * @see #isRootRegion() + */ + public static final String IS_ROOT = "IS_ROOT"; + private static final ImmutableBytesWritable IS_ROOT_KEY = + new ImmutableBytesWritable(Bytes.toBytes(IS_ROOT)); + + /** + * INTERNAL Used by rest interface to access this metadata + * attribute which denotes if it is a catalog table, either + * .META. or -ROOT- + * + * @see #isMetaRegion() + */ + public static final String IS_META = "IS_META"; + private static final ImmutableBytesWritable IS_META_KEY = + new ImmutableBytesWritable(Bytes.toBytes(IS_META)); + + /** + * INTERNAL Used by HBase Shell interface to access this metadata + * attribute which denotes if the deferred log flush option is enabled + */ + public static final String DEFERRED_LOG_FLUSH = "DEFERRED_LOG_FLUSH"; + private static final ImmutableBytesWritable DEFERRED_LOG_FLUSH_KEY = + new ImmutableBytesWritable(Bytes.toBytes(DEFERRED_LOG_FLUSH)); + + /* + * The below are ugly but better than creating them each time till we + * replace booleans being saved as Strings with plain booleans. Need a + * migration script to do this. TODO. + */ + private static final ImmutableBytesWritable FALSE = + new ImmutableBytesWritable(Bytes.toBytes(Boolean.FALSE.toString())); + + private static final ImmutableBytesWritable TRUE = + new ImmutableBytesWritable(Bytes.toBytes(Boolean.TRUE.toString())); + + private static final boolean DEFAULT_DEFERRED_LOG_FLUSH = false; + + /** + * Constant that denotes whether the table is READONLY by default and is false + */ + public static final boolean DEFAULT_READONLY = false; + + /** + * Constant that denotes the maximum default size of the memstore after which + * the contents are flushed to the store files + */ + public static final long DEFAULT_MEMSTORE_FLUSH_SIZE = 1024*1024*128L; + private final static Map DEFAULT_VALUES + = new HashMap(); + private final static Set RESERVED_KEYWORDS + = new HashSet(); + static { + DEFAULT_VALUES.put(MAX_FILESIZE, + String.valueOf(HConstants.DEFAULT_MAX_FILE_SIZE)); + DEFAULT_VALUES.put(READONLY, String.valueOf(DEFAULT_READONLY)); + DEFAULT_VALUES.put(MEMSTORE_FLUSHSIZE, + String.valueOf(DEFAULT_MEMSTORE_FLUSH_SIZE)); + DEFAULT_VALUES.put(DEFERRED_LOG_FLUSH, + String.valueOf(DEFAULT_DEFERRED_LOG_FLUSH)); + for (String s : DEFAULT_VALUES.keySet()) { + RESERVED_KEYWORDS.add(new ImmutableBytesWritable(Bytes.toBytes(s))); + } + RESERVED_KEYWORDS.add(IS_ROOT_KEY); + RESERVED_KEYWORDS.add(IS_META_KEY); + } + + + + private volatile Boolean meta = null; + private volatile Boolean root = null; + private Boolean isDeferredLog = null; + + /** + * Maps column family name to the respective HColumnDescriptors + */ + private final Map families = + new TreeMap(Bytes.BYTES_RAWCOMPARATOR); + + /** + * INTERNAL Private constructor used internally creating table descriptors for + * catalog tables, .META. and -ROOT-. + */ + protected HTableDescriptor(final byte [] name, HColumnDescriptor[] families) { + this.name = name.clone(); + this.nameAsString = Bytes.toString(this.name); + setMetaFlags(name); + for(HColumnDescriptor descriptor : families) { + this.families.put(descriptor.getName(), descriptor); + } + } + + /** + * INTERNAL Private constructor used internally creating table descriptors for + * catalog tables, .META. and -ROOT-. + */ + protected HTableDescriptor(final byte [] name, HColumnDescriptor[] families, + Map values) { + this.name = name.clone(); + this.nameAsString = Bytes.toString(this.name); + setMetaFlags(name); + for(HColumnDescriptor descriptor : families) { + this.families.put(descriptor.getName(), descriptor); + } + for (Map.Entry entry: + values.entrySet()) { + this.values.put(entry.getKey(), entry.getValue()); + } + } + + /** + * Default constructor which constructs an empty object. + * For deserializing an HTableDescriptor instance only. + * @see #HTableDescriptor(byte[]) + */ + public HTableDescriptor() { + super(); + } + + /** + * Construct a table descriptor specifying table name. + * @param name Table name. + * @throws IllegalArgumentException if passed a table name + * that is made of other than 'word' characters, underscore or period: i.e. + * [a-zA-Z_0-9.]. + * @see HADOOP-1581 HBASE: Un-openable tablename bug + */ + public HTableDescriptor(final String name) { + this(Bytes.toBytes(name)); + } + + /** + * Construct a table descriptor specifying a byte array table name + * @param name - Table name as a byte array. + * @throws IllegalArgumentException if passed a table name + * that is made of other than 'word' characters, underscore or period: i.e. + * [a-zA-Z_0-9-.]. + * @see HADOOP-1581 HBASE: Un-openable tablename bug + */ + public HTableDescriptor(final byte [] name) { + super(); + setMetaFlags(this.name); + this.name = this.isMetaRegion()? name: isLegalTableName(name); + this.nameAsString = Bytes.toString(this.name); + } + + /** + * Construct a table descriptor by cloning the descriptor passed as a parameter. + *

+ * Makes a deep copy of the supplied descriptor. + * Can make a modifiable descriptor from an UnmodifyableHTableDescriptor. + * @param desc The descriptor. + */ + public HTableDescriptor(final HTableDescriptor desc) { + super(); + this.name = desc.name.clone(); + this.nameAsString = Bytes.toString(this.name); + setMetaFlags(this.name); + for (HColumnDescriptor c: desc.families.values()) { + this.families.put(c.getName(), new HColumnDescriptor(c)); + } + for (Map.Entry e: + desc.values.entrySet()) { + this.values.put(e.getKey(), e.getValue()); + } + } + + /* + * Set meta flags on this table. + * IS_ROOT_KEY is set if its a -ROOT- table + * IS_META_KEY is set either if its a -ROOT- or a .META. table + * Called by constructors. + * @param name + */ + private void setMetaFlags(final byte [] name) { + setRootRegion(Bytes.equals(name, HConstants.ROOT_TABLE_NAME)); + setMetaRegion(isRootRegion() || + Bytes.equals(name, HConstants.META_TABLE_NAME)); + } + + /** + * Check if the descriptor represents a -ROOT- region. + * + * @return true if this is a -ROOT- region + */ + public boolean isRootRegion() { + if (this.root == null) { + this.root = isSomething(IS_ROOT_KEY, false)? Boolean.TRUE: Boolean.FALSE; + } + return this.root.booleanValue(); + } + + /** + * INTERNAL Used to denote if the current table represents + * -ROOT- region. This is used internally by the + * HTableDescriptor constructors + * + * @param isRoot true if this is the -ROOT- region + */ + protected void setRootRegion(boolean isRoot) { + // TODO: Make the value a boolean rather than String of boolean. + values.put(IS_ROOT_KEY, isRoot? TRUE: FALSE); + } + + /** + * Checks if this table is either -ROOT- or .META. + * region. + * + * @return true if this is either a -ROOT- or .META. + * region + */ + public boolean isMetaRegion() { + if (this.meta == null) { + this.meta = calculateIsMetaRegion(); + } + return this.meta.booleanValue(); + } + + private synchronized Boolean calculateIsMetaRegion() { + byte [] value = getValue(IS_META_KEY); + return (value != null)? Boolean.valueOf(Bytes.toString(value)): Boolean.FALSE; + } + + private boolean isSomething(final ImmutableBytesWritable key, + final boolean valueIfNull) { + byte [] value = getValue(key); + if (value != null) { + // TODO: Make value be a boolean rather than String of boolean. + return Boolean.valueOf(Bytes.toString(value)).booleanValue(); + } + return valueIfNull; + } + + /** + * INTERNAL Used to denote if the current table represents + * -ROOT- or .META. region. This is used + * internally by the HTableDescriptor constructors + * + * @param isMeta true if its either -ROOT- or + * .META. region + */ + protected void setMetaRegion(boolean isMeta) { + values.put(IS_META_KEY, isMeta? TRUE: FALSE); + } + + /** + * Checks if the table is a .META. table + * + * @return true if table is .META. region. + */ + public boolean isMetaTable() { + return isMetaRegion() && !isRootRegion(); + } + + /** + * Checks of the tableName being passed represents either + * -ROOT- or .META. + * + * @return true if a tablesName is either -ROOT- + * or .META. + */ + public static boolean isMetaTable(final byte [] tableName) { + return Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME) || + Bytes.equals(tableName, HConstants.META_TABLE_NAME); + } + + // A non-capture group so that this can be embedded. + public static final String VALID_USER_TABLE_REGEX = "(?:[a-zA-Z_0-9][a-zA-Z_0-9.-]*)"; + + /** + * Check passed byte buffer, "tableName", is legal user-space table name. + * @return Returns passed tableName param + * @throws NullPointerException If passed tableName is null + * @throws IllegalArgumentException if passed a tableName + * that is made of other than 'word' characters or underscores: i.e. + * [a-zA-Z_0-9]. + */ + public static byte [] isLegalTableName(final byte [] tableName) { + if (tableName == null || tableName.length <= 0) { + throw new IllegalArgumentException("Name is null or empty"); + } + if (tableName[0] == '.' || tableName[0] == '-') { + throw new IllegalArgumentException("Illegal first character <" + tableName[0] + + "> at 0. User-space table names can only start with 'word " + + "characters': i.e. [a-zA-Z_0-9]: " + Bytes.toString(tableName)); + } + for (int i = 0; i < tableName.length; i++) { + if (Character.isLetterOrDigit(tableName[i]) || tableName[i] == '_' || + tableName[i] == '-' || tableName[i] == '.') { + continue; + } + throw new IllegalArgumentException("Illegal character <" + tableName[i] + + "> at " + i + ". User-space table names can only contain " + + "'word characters': i.e. [a-zA-Z_0-9-.]: " + Bytes.toString(tableName)); + } + return tableName; + } + + /** + * Getter for accessing the metadata associated with the key + * + * @param key The key. + * @return The value. + * @see #values + */ + public byte[] getValue(byte[] key) { + return getValue(new ImmutableBytesWritable(key)); + } + + private byte[] getValue(final ImmutableBytesWritable key) { + ImmutableBytesWritable ibw = values.get(key); + if (ibw == null) + return null; + return ibw.get(); + } + + /** + * Getter for accessing the metadata associated with the key + * + * @param key The key. + * @return The value. + * @see #values + */ + public String getValue(String key) { + byte[] value = getValue(Bytes.toBytes(key)); + if (value == null) + return null; + return Bytes.toString(value); + } + + /** + * Getter for fetching an unmodifiable {@link #values} map. + * + * @return unmodifiable map {@link #values}. + * @see #values + */ + public Map getValues() { + // shallow pointer copy + return Collections.unmodifiableMap(values); + } + + /** + * Setter for storing metadata as a (key, value) pair in {@link #values} map + * + * @param key The key. + * @param value The value. + * @see #values + */ + public void setValue(byte[] key, byte[] value) { + setValue(new ImmutableBytesWritable(key), value); + } + + /* + * @param key The key. + * @param value The value. + */ + private void setValue(final ImmutableBytesWritable key, + final byte[] value) { + values.put(key, new ImmutableBytesWritable(value)); + } + + /* + * @param key The key. + * @param value The value. + */ + private void setValue(final ImmutableBytesWritable key, + final ImmutableBytesWritable value) { + values.put(key, value); + } + + /** + * Setter for storing metadata as a (key, value) pair in {@link #values} map + * + * @param key The key. + * @param value The value. + * @see #values + */ + public void setValue(String key, String value) { + if (value == null) { + remove(Bytes.toBytes(key)); + } else { + setValue(Bytes.toBytes(key), Bytes.toBytes(value)); + } + } + + /** + * Remove metadata represented by the key from the {@link #values} map + * + * @param key Key whose key and value we're to remove from HTableDescriptor + * parameters. + */ + public void remove(final byte [] key) { + values.remove(new ImmutableBytesWritable(key)); + } + + /** + * Remove metadata represented by the key from the {@link #values} map + * + * @param key Key whose key and value we're to remove from HTableDescriptor + * parameters. + */ + public void remove(final String key) { + remove(Bytes.toBytes(key)); + } + + /** + * Check if the readOnly flag of the table is set. If the readOnly flag is + * set then the contents of the table can only be read from but not modified. + * + * @return true if all columns in the table should be read only + */ + public boolean isReadOnly() { + return isSomething(READONLY_KEY, DEFAULT_READONLY); + } + + /** + * Setting the table as read only sets all the columns in the table as read + * only. By default all tables are modifiable, but if the readOnly flag is + * set to true then the contents of the table can only be read but not modified. + * + * @param readOnly True if all of the columns in the table should be read + * only. + */ + public void setReadOnly(final boolean readOnly) { + setValue(READONLY_KEY, readOnly? TRUE: FALSE); + } + + /** + * Check if deferred log edits are enabled on the table. + * + * @return true if that deferred log flush is enabled on the table + * + * @see #setDeferredLogFlush(boolean) + */ + public synchronized boolean isDeferredLogFlush() { + if(this.isDeferredLog == null) { + this.isDeferredLog = + isSomething(DEFERRED_LOG_FLUSH_KEY, DEFAULT_DEFERRED_LOG_FLUSH); + } + return this.isDeferredLog; + } + + /** + * This is used to defer the log edits syncing to the file system. Everytime + * an edit is sent to the server it is first sync'd to the file system by the + * log writer. This sync is an expensive operation and thus can be deferred so + * that the edits are kept in memory for a specified period of time as represented + * by hbase.regionserver.optionallogflushinterval and not flushed + * for every edit. + *

+ * NOTE:- This option might result in data loss if the region server crashes + * before these deferred edits in memory are flushed onto the filesystem. + *

+ * + * @param isDeferredLogFlush + */ + public void setDeferredLogFlush(final boolean isDeferredLogFlush) { + setValue(DEFERRED_LOG_FLUSH_KEY, isDeferredLogFlush? TRUE: FALSE); + this.isDeferredLog = isDeferredLogFlush; + } + + /** + * Get the name of the table as a byte array. + * + * @return name of table + */ + public byte [] getName() { + return name; + } + + /** + * Get the name of the table as a String + * + * @return name of table as a String + */ + public String getNameAsString() { + return this.nameAsString; + } + + /** + * This get the class associated with the region split policy which + * determines when a region split should occur. The class used by + * default is {@link org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy} + * which split the region base on a constant {@link #getMaxFileSize()} + * + * @return the class name of the region split policy for this table. + * If this returns null, the default constant size based split policy + * is used. + */ + public String getRegionSplitPolicyClassName() { + return getValue(SPLIT_POLICY); + } + + /** + * Set the name of the table. + * + * @param name name of table + */ + public void setName(byte[] name) { + this.name = name; + this.nameAsString = Bytes.toString(this.name); + setMetaFlags(this.name); + } + + /** + * Returns the maximum size upto which a region can grow to after which a region + * split is triggered. The region size is represented by the size of the biggest + * store file in that region. + * + * @return max hregion size for table, -1 if not set. + * + * @see #setMaxFileSize(long) + */ + public long getMaxFileSize() { + byte [] value = getValue(MAX_FILESIZE_KEY); + if (value != null) { + return Long.parseLong(Bytes.toString(value)); + } + return -1; + } + + /** + * Sets the maximum size upto which a region can grow to after which a region + * split is triggered. The region size is represented by the size of the biggest + * store file in that region, i.e. If the biggest store file grows beyond the + * maxFileSize, then the region split is triggered. This defaults to a value of + * 256 MB. + *

+ * This is not an absolute value and might vary. Assume that a single row exceeds + * the maxFileSize then the storeFileSize will be greater than maxFileSize since + * a single row cannot be split across multiple regions + *

+ * + * @param maxFileSize The maximum file size that a store file can grow to + * before a split is triggered. + */ + public void setMaxFileSize(long maxFileSize) { + setValue(MAX_FILESIZE_KEY, Bytes.toBytes(Long.toString(maxFileSize))); + } + + /** + * Returns the size of the memstore after which a flush to filesystem is triggered. + * + * @return memory cache flush size for each hregion, -1 if not set. + * + * @see #setMemStoreFlushSize(long) + */ + public long getMemStoreFlushSize() { + byte [] value = getValue(MEMSTORE_FLUSHSIZE_KEY); + if (value != null) { + return Long.parseLong(Bytes.toString(value)); + } + return -1; + } + + /** + * Represents the maximum size of the memstore after which the contents of the + * memstore are flushed to the filesystem. This defaults to a size of 64 MB. + * + * @param memstoreFlushSize memory cache flush size for each hregion + */ + public void setMemStoreFlushSize(long memstoreFlushSize) { + setValue(MEMSTORE_FLUSHSIZE_KEY, + Bytes.toBytes(Long.toString(memstoreFlushSize))); + } + + /** + * Adds a column family. + * @param family HColumnDescriptor of family to add. + */ + public void addFamily(final HColumnDescriptor family) { + if (family.getName() == null || family.getName().length <= 0) { + throw new NullPointerException("Family name cannot be null or empty"); + } + this.families.put(family.getName(), family); + } + + /** + * Checks to see if this table contains the given column family + * @param familyName Family name or column name. + * @return true if the table contains the specified family name + */ + public boolean hasFamily(final byte [] familyName) { + return families.containsKey(familyName); + } + + /** + * @return Name of this table and then a map of all of the column family + * descriptors. + * @see #getNameAsString() + */ + @Override + public String toString() { + StringBuilder s = new StringBuilder(); + s.append('\'').append(Bytes.toString(name)).append('\''); + s.append(getValues(true)); + for (HColumnDescriptor f : families.values()) { + s.append(", ").append(f); + } + return s.toString(); + } + + /** + * @return Name of this table and then a map of all of the column family + * descriptors (with only the non-default column family attributes) + */ + public String toStringCustomizedValues() { + StringBuilder s = new StringBuilder(); + s.append('\'').append(Bytes.toString(name)).append('\''); + s.append(getValues(false)); + for(HColumnDescriptor hcd : families.values()) { + s.append(", ").append(hcd.toStringCustomizedValues()); + } + return s.toString(); + } + + private StringBuilder getValues(boolean printDefaults) { + StringBuilder s = new StringBuilder(); + + // step 1: set partitioning and pruning + Set reservedKeys = new TreeSet(); + Set configKeys = new TreeSet(); + for (ImmutableBytesWritable k : values.keySet()) { + if (k == null || k.get() == null) continue; + String key = Bytes.toString(k.get()); + // in this section, print out reserved keywords + coprocessor info + if (!RESERVED_KEYWORDS.contains(k) && !key.startsWith("coprocessor$")) { + configKeys.add(k); + continue; + } + // only print out IS_ROOT/IS_META if true + String value = Bytes.toString(values.get(k).get()); + if (key.equalsIgnoreCase(IS_ROOT) || key.equalsIgnoreCase(IS_META)) { + // Skip. Don't bother printing out read-only values if false. + if (value.toLowerCase().equals(Boolean.FALSE.toString())) { + continue; + } + } + + // see if a reserved key is a default value. may not want to print it out + if (printDefaults + || !DEFAULT_VALUES.containsKey(key) + || !DEFAULT_VALUES.get(key).equalsIgnoreCase(value)) { + reservedKeys.add(k); + } + } + + + + // early exit optimization + if (reservedKeys.isEmpty() && configKeys.isEmpty()) return s; + + // step 2: printing + s.append(", {METHOD => 'table_att'"); + + // print all reserved keys first + for (ImmutableBytesWritable k : reservedKeys) { + String key = Bytes.toString(k.get()); + String value = Bytes.toString(values.get(k).get()); + + s.append(", "); + s.append(key); + s.append(" => "); + s.append('\'').append(value).append('\''); + } + if (!configKeys.isEmpty()) { + // print all non-reserved, advanced config keys as a separate subset + s.append(", "); + s.append(HConstants.CONFIG).append(" => "); + s.append("{"); + boolean printComma = false; + for (ImmutableBytesWritable k : configKeys) { + String key = Bytes.toString(k.get()); + String value = Bytes.toString(values.get(k).get()); + if (printComma) s.append(", "); + printComma = true; + s.append('\'').append(key).append('\''); + s.append(" => "); + s.append('\'').append(value).append('\''); + } + s.append("}"); + } + + s.append('}'); // end METHOD + return s; + } + + public static Map getDefaultValues() { + return Collections.unmodifiableMap(DEFAULT_VALUES); + } + + /** + * Compare the contents of the descriptor with another one passed as a parameter. + * Checks if the obj passed is an instance of HTableDescriptor, if yes then the + * contents of the descriptors are compared. + * + * @return true if the contents of the the two descriptors exactly match + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof HTableDescriptor)) { + return false; + } + return compareTo((HTableDescriptor)obj) == 0; + } + + /** + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + int result = Bytes.hashCode(this.name); + result ^= Byte.valueOf(TABLE_DESCRIPTOR_VERSION).hashCode(); + if (this.families != null && this.families.size() > 0) { + for (HColumnDescriptor e: this.families.values()) { + result ^= e.hashCode(); + } + } + result ^= values.hashCode(); + return result; + } + + // Writable + /** + * INTERNAL This method is a part of {@link WritableComparable} interface + * and is used for de-serialization of the HTableDescriptor over RPC + */ + @Override + public void readFields(DataInput in) throws IOException { + int version = in.readInt(); + if (version < 3) + throw new IOException("versions < 3 are not supported (and never existed!?)"); + // version 3+ + name = Bytes.readByteArray(in); + nameAsString = Bytes.toString(this.name); + setRootRegion(in.readBoolean()); + setMetaRegion(in.readBoolean()); + values.clear(); + int numVals = in.readInt(); + for (int i = 0; i < numVals; i++) { + ImmutableBytesWritable key = new ImmutableBytesWritable(); + ImmutableBytesWritable value = new ImmutableBytesWritable(); + key.readFields(in); + value.readFields(in); + values.put(key, value); + } + families.clear(); + int numFamilies = in.readInt(); + for (int i = 0; i < numFamilies; i++) { + HColumnDescriptor c = new HColumnDescriptor(); + c.readFields(in); + families.put(c.getName(), c); + } + if (version < 4) { + return; + } + } + + /** + * INTERNAL This method is a part of {@link WritableComparable} interface + * and is used for serialization of the HTableDescriptor over RPC + */ + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(TABLE_DESCRIPTOR_VERSION); + Bytes.writeByteArray(out, name); + out.writeBoolean(isRootRegion()); + out.writeBoolean(isMetaRegion()); + out.writeInt(values.size()); + for (Map.Entry e: + values.entrySet()) { + e.getKey().write(out); + e.getValue().write(out); + } + out.writeInt(families.size()); + for(Iterator it = families.values().iterator(); + it.hasNext(); ) { + HColumnDescriptor family = it.next(); + family.write(out); + } + } + + // Comparable + + /** + * Compares the descriptor with another descriptor which is passed as a parameter. + * This compares the content of the two descriptors and not the reference. + * + * @return 0 if the contents of the descriptors are exactly matching, + * 1 if there is a mismatch in the contents + */ + @Override + public int compareTo(final HTableDescriptor other) { + int result = Bytes.compareTo(this.name, other.name); + if (result == 0) { + result = families.size() - other.families.size(); + } + if (result == 0 && families.size() != other.families.size()) { + result = Integer.valueOf(families.size()).compareTo( + Integer.valueOf(other.families.size())); + } + if (result == 0) { + for (Iterator it = families.values().iterator(), + it2 = other.families.values().iterator(); it.hasNext(); ) { + result = it.next().compareTo(it2.next()); + if (result != 0) { + break; + } + } + } + if (result == 0) { + // punt on comparison for ordering, just calculate difference + result = this.values.hashCode() - other.values.hashCode(); + if (result < 0) + result = -1; + else if (result > 0) + result = 1; + } + return result; + } + + /** + * Returns an unmodifiable collection of all the {@link HColumnDescriptor} + * of all the column families of the table. + * + * @return Immutable collection of {@link HColumnDescriptor} of all the + * column families. + */ + public Collection getFamilies() { + return Collections.unmodifiableCollection(this.families.values()); + } + + /** + * Returns all the column family names of the current table. The map of + * HTableDescriptor contains mapping of family name to HColumnDescriptors. + * This returns all the keys of the family map which represents the column + * family names of the table. + * + * @return Immutable sorted set of the keys of the families. + */ + public Set getFamiliesKeys() { + return Collections.unmodifiableSet(this.families.keySet()); + } + + /** + * Returns an array all the {@link HColumnDescriptor} of the column families + * of the table. + * + * @return Array of all the HColumnDescriptors of the current table + * + * @see #getFamilies() + */ + public HColumnDescriptor[] getColumnFamilies() { + return getFamilies().toArray(new HColumnDescriptor[0]); + } + + + /** + * Returns the HColumnDescriptor for a specific column family with name as + * specified by the parameter column. + * + * @param column Column family name + * @return Column descriptor for the passed family name or the family on + * passed in column. + */ + public HColumnDescriptor getFamily(final byte [] column) { + return this.families.get(column); + } + + + /** + * Removes the HColumnDescriptor with name specified by the parameter column + * from the table descriptor + * + * @param column Name of the column family to be removed. + * @return Column descriptor for the passed family name or the family on + * passed in column. + */ + public HColumnDescriptor removeFamily(final byte [] column) { + return this.families.remove(column); + } + + + /** + * Add a table coprocessor to this table. The coprocessor + * type must be {@link org.apache.hadoop.hbase.coprocessor.RegionObserver} + * or Endpoint. + * It won't check if the class can be loaded or not. + * Whether a coprocessor is loadable or not will be determined when + * a region is opened. + * @param className Full class name. + * @throws IOException + */ + public void addCoprocessor(String className) throws IOException { + addCoprocessor(className, null, Coprocessor.PRIORITY_USER, null); + } + + + /** + * Add a table coprocessor to this table. The coprocessor + * type must be {@link org.apache.hadoop.hbase.coprocessor.RegionObserver} + * or Endpoint. + * It won't check if the class can be loaded or not. + * Whether a coprocessor is loadable or not will be determined when + * a region is opened. + * @param jarFilePath Path of the jar file. If it's null, the class will be + * loaded from default classloader. + * @param className Full class name. + * @param priority Priority + * @param kvs Arbitrary key-value parameter pairs passed into the coprocessor. + * @throws IOException + */ + public void addCoprocessor(String className, Path jarFilePath, + int priority, final Map kvs) + throws IOException { + if (hasCoprocessor(className)) { + throw new IOException("Coprocessor " + className + " already exists."); + } + // validate parameter kvs + StringBuilder kvString = new StringBuilder(); + if (kvs != null) { + for (Map.Entry e: kvs.entrySet()) { + if (!e.getKey().matches(HConstants.CP_HTD_ATTR_VALUE_PARAM_KEY_PATTERN)) { + throw new IOException("Illegal parameter key = " + e.getKey()); + } + if (!e.getValue().matches(HConstants.CP_HTD_ATTR_VALUE_PARAM_VALUE_PATTERN)) { + throw new IOException("Illegal parameter (" + e.getKey() + + ") value = " + e.getValue()); + } + if (kvString.length() != 0) { + kvString.append(','); + } + kvString.append(e.getKey()); + kvString.append('='); + kvString.append(e.getValue()); + } + } + + // generate a coprocessor key + int maxCoprocessorNumber = 0; + Matcher keyMatcher; + for (Map.Entry e: + this.values.entrySet()) { + keyMatcher = + HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher( + Bytes.toString(e.getKey().get())); + if (!keyMatcher.matches()) { + continue; + } + maxCoprocessorNumber = Math.max(Integer.parseInt(keyMatcher.group(1)), + maxCoprocessorNumber); + } + maxCoprocessorNumber++; + + String key = "coprocessor$" + Integer.toString(maxCoprocessorNumber); + String value = ((jarFilePath == null)? "" : jarFilePath.toString()) + + "|" + className + "|" + Integer.toString(priority) + "|" + + kvString.toString(); + setValue(key, value); + } + + + /** + * Check if the table has an attached co-processor represented by the name className + * + * @param className - Class name of the co-processor + * @return true of the table has a co-processor className + */ + public boolean hasCoprocessor(String className) { + Matcher keyMatcher; + Matcher valueMatcher; + for (Map.Entry e: + this.values.entrySet()) { + keyMatcher = + HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher( + Bytes.toString(e.getKey().get())); + if (!keyMatcher.matches()) { + continue; + } + valueMatcher = + HConstants.CP_HTD_ATTR_VALUE_PATTERN.matcher( + Bytes.toString(e.getValue().get())); + if (!valueMatcher.matches()) { + continue; + } + // get className and compare + String clazz = valueMatcher.group(2).trim(); // classname is the 2nd field + if (clazz.equals(className.trim())) { + return true; + } + } + return false; + } + + /** + * Return the list of attached co-processor represented by their name className + * + * @return The list of co-processors classNames + */ + public List getCoprocessors() { + List result = new ArrayList(); + Matcher keyMatcher; + Matcher valueMatcher; + for (Map.Entry e : this.values.entrySet()) { + keyMatcher = HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher(Bytes.toString(e.getKey().get())); + if (!keyMatcher.matches()) { + continue; + } + valueMatcher = HConstants.CP_HTD_ATTR_VALUE_PATTERN.matcher(Bytes + .toString(e.getValue().get())); + if (!valueMatcher.matches()) { + continue; + } + result.add(valueMatcher.group(2).trim()); // classname is the 2nd field + } + return result; + } + + /** + * Remove a coprocessor from those set on the table + * @param className Class name of the co-processor + */ + public void removeCoprocessor(String className) { + ImmutableBytesWritable match = null; + Matcher keyMatcher; + Matcher valueMatcher; + for (Map.Entry e : this.values + .entrySet()) { + keyMatcher = HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher(Bytes.toString(e + .getKey().get())); + if (!keyMatcher.matches()) { + continue; + } + valueMatcher = HConstants.CP_HTD_ATTR_VALUE_PATTERN.matcher(Bytes + .toString(e.getValue().get())); + if (!valueMatcher.matches()) { + continue; + } + // get className and compare + String clazz = valueMatcher.group(2).trim(); // classname is the 2nd field + // remove the CP if it is present + if (clazz.equals(className.trim())) { + match = e.getKey(); + break; + } + } + // if we found a match, remove it + if (match != null) + this.values.remove(match); + } + + /** + * Returns the {@link Path} object representing the table directory under + * path rootdir + * + * @param rootdir qualified path of HBase root directory + * @param tableName name of table + * @return {@link Path} for table + */ + public static Path getTableDir(Path rootdir, final byte [] tableName) { + return new Path(rootdir, Bytes.toString(tableName)); + } + + /** Table descriptor for -ROOT- catalog table */ + public static final HTableDescriptor ROOT_TABLEDESC = new HTableDescriptor( + HConstants.ROOT_TABLE_NAME, + new HColumnDescriptor[] { + new HColumnDescriptor(HConstants.CATALOG_FAMILY) + // Ten is arbitrary number. Keep versions to help debugging. + .setMaxVersions(10) + .setInMemory(true) + .setBlocksize(8 * 1024) + .setTimeToLive(HConstants.FOREVER) + .setScope(HConstants.REPLICATION_SCOPE_LOCAL) + }); + + /** Table descriptor for .META. catalog table */ + public static final HTableDescriptor META_TABLEDESC = new HTableDescriptor( + HConstants.META_TABLE_NAME, new HColumnDescriptor[] { + new HColumnDescriptor(HConstants.CATALOG_FAMILY) + // Ten is arbitrary number. Keep versions to help debugging. + .setMaxVersions(10) + .setInMemory(true) + .setBlocksize(8 * 1024) + .setScope(HConstants.REPLICATION_SCOPE_LOCAL) + }); + + @Deprecated + public void setOwner(User owner) { + setOwnerString(owner != null ? owner.getShortName() : null); + } + + // used by admin.rb:alter(table_name,*args) to update owner. + @Deprecated + public void setOwnerString(String ownerString) { + if (ownerString != null) { + setValue(OWNER_KEY, Bytes.toBytes(ownerString)); + } else { + values.remove(OWNER_KEY); + } + } + + @Deprecated + public String getOwnerString() { + if (getValue(OWNER_KEY) != null) { + return Bytes.toString(getValue(OWNER_KEY)); + } + // Note that every table should have an owner (i.e. should have OWNER_KEY set). + // .META. and -ROOT- should return system user as owner, not null (see + // MasterFileSystem.java:bootstrap()). + return null; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java b/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java new file mode 100644 index 0000000..bae41f6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java @@ -0,0 +1,95 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HealthChecker.HealthCheckerExitStatus; +import org.apache.hadoop.util.StringUtils; + +/** + * The Class HealthCheckChore for running health checker regularly. + */ + public class HealthCheckChore extends Chore { + private static Log LOG = LogFactory.getLog(HealthCheckChore.class); + private HealthChecker healthChecker; + private Configuration config; + private int threshold; + private int numTimesUnhealthy = 0; + private long failureWindow; + private long startWindow; + + public HealthCheckChore(int sleepTime, Stoppable stopper, Configuration conf) { + super("HealthChecker", sleepTime, stopper); + LOG.info("Health Check Chore runs every " + StringUtils.formatTime(sleepTime)); + this.config = conf; + String healthCheckScript = this.config.get(HConstants.HEALTH_SCRIPT_LOC); + long scriptTimeout = this.config.getLong(HConstants.HEALTH_SCRIPT_TIMEOUT, + HConstants.DEFAULT_HEALTH_SCRIPT_TIMEOUT); + healthChecker = new HealthChecker(); + healthChecker.init(healthCheckScript, scriptTimeout); + this.threshold = config.getInt(HConstants.HEALTH_FAILURE_THRESHOLD, + HConstants.DEFAULT_HEALTH_FAILURE_THRESHOLD); + this.failureWindow = this.threshold * sleepTime; + } + + @Override + protected void chore() { + HealthReport report = healthChecker.checkHealth(); + boolean isHealthy = (report.getStatus() == HealthCheckerExitStatus.SUCCESS); + if (!isHealthy) { + boolean needToStop = decideToStop(); + if (needToStop) { + this.stopper.stop("The region server reported unhealthy " + threshold + + " number of times consecutively."); + } + // Always log health report. + LOG.info("Health status at " + StringUtils.formatTime(System.currentTimeMillis()) + " : " + + report.getHealthReport()); + } + } + + private boolean decideToStop() { + boolean stop = false; + if (numTimesUnhealthy == 0) { + // First time we are seeing a failure. No need to stop, just + // record the time. + numTimesUnhealthy++; + startWindow = System.currentTimeMillis(); + } else { + if ((System.currentTimeMillis() - startWindow) < failureWindow) { + numTimesUnhealthy++; + if (numTimesUnhealthy == threshold) { + stop = true; + } else { + stop = false; + } + } else { + // Outside of failure window, so we reset to 1. + numTimesUnhealthy = 1; + startWindow = System.currentTimeMillis(); + stop = false; + } + } + return stop; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/HealthChecker.java b/src/main/java/org/apache/hadoop/hbase/HealthChecker.java new file mode 100644 index 0000000..e77317c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HealthChecker.java @@ -0,0 +1,127 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.util.ArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.Shell.ExitCodeException; +import org.apache.hadoop.util.Shell.ShellCommandExecutor; + +/** + * A utility for executing an external script that checks the health of + * the node. An example script can be found at + * src/examples/healthcheck/healthcheck.sh + */ +class HealthChecker { + + private static Log LOG = LogFactory.getLog(HealthChecker.class); + private ShellCommandExecutor shexec = null; + private String exceptionStackTrace; + + /** Pattern used for searching in the output of the node health script */ + static private final String ERROR_PATTERN = "ERROR"; + + private String healthCheckScript; + private long scriptTimeout; + + enum HealthCheckerExitStatus { + SUCCESS, + TIMED_OUT, + FAILED_WITH_EXIT_CODE, + FAILED_WITH_EXCEPTION, + FAILED + } + + /** + * Initialize. + * + * @param configuration + */ + public void init(String location, long timeout) { + this.healthCheckScript = location; + this.scriptTimeout = timeout; + ArrayList execScript = new ArrayList(); + execScript.add(healthCheckScript); + this.shexec = new ShellCommandExecutor(execScript.toArray(new String[execScript.size()]), null, + null, scriptTimeout); + LOG.info("HealthChecker initialized."); + } + + public HealthReport checkHealth() { + HealthCheckerExitStatus status = HealthCheckerExitStatus.SUCCESS; + try { + shexec.execute(); + } catch (ExitCodeException e) { + // ignore the exit code of the script + LOG.warn("Caught exception : " + e); + status = HealthCheckerExitStatus.FAILED_WITH_EXIT_CODE; + } catch (IOException e) { + LOG.warn("Caught exception : " + e); + if (!shexec.isTimedOut()) { + status = HealthCheckerExitStatus.FAILED_WITH_EXCEPTION; + exceptionStackTrace = org.apache.hadoop.util.StringUtils.stringifyException(e); + } else { + status = HealthCheckerExitStatus.TIMED_OUT; + } + } finally { + if (status == HealthCheckerExitStatus.SUCCESS) { + if (hasErrors(shexec.getOutput())) { + status = HealthCheckerExitStatus.FAILED; + } + } + } + return new HealthReport(status, getHealthReport(status)); + } + + private boolean hasErrors(String output) { + String[] splits = output.split("\n"); + for (String split : splits) { + if (split.startsWith(ERROR_PATTERN)) { + return true; + } + } + return false; + } + + private String getHealthReport(HealthCheckerExitStatus status){ + String healthReport = null; + switch (status) { + case SUCCESS: + healthReport = "Server is healthy."; + break; + case TIMED_OUT: + healthReport = "Health script timed out"; + break; + case FAILED_WITH_EXCEPTION: + healthReport = exceptionStackTrace; + break; + case FAILED_WITH_EXIT_CODE: + healthReport = "Health script failed with exit code."; + break; + case FAILED: + healthReport = shexec.getOutput(); + break; + } + return healthReport; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/HealthReport.java b/src/main/java/org/apache/hadoop/hbase/HealthReport.java new file mode 100644 index 0000000..5292929 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/HealthReport.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.HealthChecker.HealthCheckerExitStatus; + +/** + * The Class RegionServerHealthReport containing information about + * health of the region server. + */ +class HealthReport { + + private HealthCheckerExitStatus status; + private String healthReport; + + HealthReport(HealthCheckerExitStatus status, String healthReport) { + super(); + this.status = status; + this.healthReport = healthReport; + } + + /** + * Gets the status of the region server. + * + * @return HealthCheckerExitStatus + */ + HealthCheckerExitStatus getStatus() { + return status; + } + + /** + * Gets the health report of the region server. + * + * @return String + */ + String getHealthReport() { + return healthReport; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((healthReport == null) ? 0 : healthReport.hashCode()); + result = prime * result + ((status == null) ? 0 : status.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof HealthReport)) { + return false; + } + HealthReport other = (HealthReport) obj; + if (healthReport == null) { + if (other.healthReport != null) { + return false; + } + } else if (!healthReport.equals(other.healthReport)) { + return false; + } + if (status != other.status) { + return false; + } + return true; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java b/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java new file mode 100644 index 0000000..bb2b666 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java @@ -0,0 +1,50 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +/** + * Thrown if a request is table schema modification is requested but + * made for an invalid family name. + */ +public class InvalidFamilyOperationException extends IOException { + private static final long serialVersionUID = 1L << 22 - 1L; + /** default constructor */ + public InvalidFamilyOperationException() { + super(); + } + + /** + * Constructor + * @param s message + */ + public InvalidFamilyOperationException(String s) { + super(s); + } + + /** + * Constructor taking another exception. + * @param e Exception to grab data from. + */ + public InvalidFamilyOperationException(Exception e) { + super(e); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java new file mode 100644 index 0000000..39d1f09 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -0,0 +1,2289 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; +import org.apache.hadoop.io.RawComparator; +import org.apache.hadoop.io.Writable; + +import com.google.common.primitives.Longs; + +/** + * An HBase Key/Value. This is the fundamental HBase Type. + * + *

If being used client-side, the primary methods to access individual fields + * are {@link #getRow()}, {@link #getFamily()}, {@link #getQualifier()}, + * {@link #getTimestamp()}, and {@link #getValue()}. These methods allocate new + * byte arrays and return copies. Avoid their use server-side. + * + *

Instances of this class are immutable. They do not implement Comparable + * but Comparators are provided. Comparators change with context, + * whether user table or a catalog table comparison. Its critical you use the + * appropriate comparator. There are Comparators for KeyValue instances and + * then for just the Key portion of a KeyValue used mostly by {@link HFile}. + * + *

KeyValue wraps a byte array and takes offsets and lengths into passed + * array at where to start interpreting the content as KeyValue. The KeyValue + * format inside a byte array is: + * <keylength> <valuelength> <key> <value> + * Key is further decomposed as: + * <rowlength> <row> <columnfamilylength> <columnfamily> <columnqualifier> <timestamp> <keytype> + * The rowlength maximum is Short.MAX_SIZE, + * column family length maximum is + * Byte.MAX_SIZE, and column qualifier + key length must + * be < Integer.MAX_SIZE. + * The column does not contain the family/qualifier delimiter, {@link #COLUMN_FAMILY_DELIMITER} + */ +public class KeyValue implements Writable, HeapSize { + static final Log LOG = LogFactory.getLog(KeyValue.class); + // TODO: Group Key-only comparators and operations into a Key class, just + // for neatness sake, if can figure what to call it. + + /** + * Colon character in UTF-8 + */ + public static final char COLUMN_FAMILY_DELIMITER = ':'; + + public static final byte[] COLUMN_FAMILY_DELIM_ARRAY = + new byte[]{COLUMN_FAMILY_DELIMITER}; + + /** + * Comparator for plain key/values; i.e. non-catalog table key/values. + */ + public static KVComparator COMPARATOR = new KVComparator(); + + /** + * Comparator for plain key; i.e. non-catalog table key. Works on Key portion + * of KeyValue only. + */ + public static KeyComparator KEY_COMPARATOR = new KeyComparator(); + + /** + * A {@link KVComparator} for .META. catalog table + * {@link KeyValue}s. + */ + public static KVComparator META_COMPARATOR = new MetaComparator(); + + /** + * A {@link KVComparator} for .META. catalog table + * {@link KeyValue} keys. + */ + public static KeyComparator META_KEY_COMPARATOR = new MetaKeyComparator(); + + /** + * A {@link KVComparator} for -ROOT- catalog table + * {@link KeyValue}s. + */ + public static KVComparator ROOT_COMPARATOR = new RootComparator(); + + /** + * A {@link KVComparator} for -ROOT- catalog table + * {@link KeyValue} keys. + */ + public static KeyComparator ROOT_KEY_COMPARATOR = new RootKeyComparator(); + + /** + * Get the appropriate row comparator for the specified table. + * + * Hopefully we can get rid of this, I added this here because it's replacing + * something in HSK. We should move completely off of that. + * + * @param tableName The table name. + * @return The comparator. + */ + public static KeyComparator getRowComparator(byte [] tableName) { + if(Bytes.equals(HTableDescriptor.ROOT_TABLEDESC.getName(),tableName)) { + return ROOT_COMPARATOR.getRawComparator(); + } + if(Bytes.equals(HTableDescriptor.META_TABLEDESC.getName(), tableName)) { + return META_COMPARATOR.getRawComparator(); + } + return COMPARATOR.getRawComparator(); + } + + /** Size of the key length field in bytes*/ + public static final int KEY_LENGTH_SIZE = Bytes.SIZEOF_INT; + + /** Size of the key type field in bytes */ + public static final int TYPE_SIZE = Bytes.SIZEOF_BYTE; + + /** Size of the row length field in bytes */ + public static final int ROW_LENGTH_SIZE = Bytes.SIZEOF_SHORT; + + /** Size of the family length field in bytes */ + public static final int FAMILY_LENGTH_SIZE = Bytes.SIZEOF_BYTE; + + /** Size of the timestamp field in bytes */ + public static final int TIMESTAMP_SIZE = Bytes.SIZEOF_LONG; + + // Size of the timestamp and type byte on end of a key -- a long + a byte. + public static final int TIMESTAMP_TYPE_SIZE = TIMESTAMP_SIZE + TYPE_SIZE; + + // Size of the length shorts and bytes in key. + public static final int KEY_INFRASTRUCTURE_SIZE = ROW_LENGTH_SIZE + + FAMILY_LENGTH_SIZE + TIMESTAMP_TYPE_SIZE; + + // How far into the key the row starts at. First thing to read is the short + // that says how long the row is. + public static final int ROW_OFFSET = + Bytes.SIZEOF_INT /*keylength*/ + + Bytes.SIZEOF_INT /*valuelength*/; + + // Size of the length ints in a KeyValue datastructure. + public static final int KEYVALUE_INFRASTRUCTURE_SIZE = ROW_OFFSET; + + /** + * Key type. + * Has space for other key types to be added later. Cannot rely on + * enum ordinals . They change if item is removed or moved. Do our own codes. + */ + public static enum Type { + Minimum((byte)0), + Put((byte)4), + + Delete((byte)8), + DeleteColumn((byte)12), + DeleteFamily((byte)14), + + // Maximum is used when searching; you look from maximum on down. + Maximum((byte)255); + + private final byte code; + + Type(final byte c) { + this.code = c; + } + + public byte getCode() { + return this.code; + } + + /** + * Cannot rely on enum ordinals . They change if item is removed or moved. + * Do our own codes. + * @param b + * @return Type associated with passed code. + */ + public static Type codeToType(final byte b) { + for (Type t : Type.values()) { + if (t.getCode() == b) { + return t; + } + } + throw new RuntimeException("Unknown code " + b); + } + } + + /** + * Lowest possible key. + * Makes a Key with highest possible Timestamp, empty row and column. No + * key can be equal or lower than this one in memstore or in store file. + */ + public static final KeyValue LOWESTKEY = + new KeyValue(HConstants.EMPTY_BYTE_ARRAY, HConstants.LATEST_TIMESTAMP); + + private byte [] bytes = null; + private int offset = 0; + private int length = 0; + + /** + * @return True if a delete type, a {@link KeyValue.Type#Delete} or + * a {KeyValue.Type#DeleteFamily} or a {@link KeyValue.Type#DeleteColumn} + * KeyValue type. + */ + public static boolean isDelete(byte t) { + return Type.Delete.getCode() <= t && t <= Type.DeleteFamily.getCode(); + } + + /** Here be dragons **/ + + // used to achieve atomic operations in the memstore. + public long getMemstoreTS() { + return memstoreTS; + } + + public void setMemstoreTS(long memstoreTS) { + this.memstoreTS = memstoreTS; + } + + // default value is 0, aka DNC + private long memstoreTS = 0; + + /** Dragon time over, return to normal business */ + + + /** Writable Constructor -- DO NOT USE */ + public KeyValue() {} + + /** + * Creates a KeyValue from the start of the specified byte array. + * Presumes bytes content is formatted as a KeyValue blob. + * @param bytes byte array + */ + public KeyValue(final byte [] bytes) { + this(bytes, 0); + } + + /** + * Creates a KeyValue from the specified byte array and offset. + * Presumes bytes content starting at offset is + * formatted as a KeyValue blob. + * @param bytes byte array + * @param offset offset to start of KeyValue + */ + public KeyValue(final byte [] bytes, final int offset) { + this(bytes, offset, getLength(bytes, offset)); + } + + /** + * Creates a KeyValue from the specified byte array, starting at offset, and + * for length length. + * @param bytes byte array + * @param offset offset to start of the KeyValue + * @param length length of the KeyValue + */ + public KeyValue(final byte [] bytes, final int offset, final int length) { + this.bytes = bytes; + this.offset = offset; + this.length = length; + } + + /** + * Creates a KeyValue from the specified byte array, starting at offset, + * for length length, and a known keyLength. + * @param bytes byte array + * @param offset offset to start of the KeyValue + * @param length length of the KeyValue + * @param keyLength length of the key portion of the KeyValue + */ + public KeyValue(final byte [] bytes, final int offset, final int length, final int keyLength) { + this.bytes = bytes; + this.offset = offset; + this.length = length; + this.keyLength = keyLength; + } + + /** Constructors that build a new backing byte array from fields */ + + /** + * Constructs KeyValue structure filled with null value. + * Sets type to {@link KeyValue.Type#Maximum} + * @param row - row key (arbitrary byte array) + * @param timestamp + */ + public KeyValue(final byte [] row, final long timestamp) { + this(row, timestamp, Type.Maximum); + } + + /** + * Constructs KeyValue structure filled with null value. + * @param row - row key (arbitrary byte array) + * @param timestamp + */ + public KeyValue(final byte [] row, final long timestamp, Type type) { + this(row, null, null, timestamp, type, null); + } + + /** + * Constructs KeyValue structure filled with null value. + * Sets type to {@link KeyValue.Type#Maximum} + * @param row - row key (arbitrary byte array) + * @param family family name + * @param qualifier column qualifier + */ + public KeyValue(final byte [] row, final byte [] family, + final byte [] qualifier) { + this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum); + } + + /** + * Constructs KeyValue structure filled with null value. + * @param row - row key (arbitrary byte array) + * @param family family name + * @param qualifier column qualifier + */ + public KeyValue(final byte [] row, final byte [] family, + final byte [] qualifier, final byte [] value) { + this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Put, value); + } + + /** + * Constructs KeyValue structure filled with specified values. + * @param row row key + * @param family family name + * @param qualifier column qualifier + * @param timestamp version timestamp + * @param type key type + * @throws IllegalArgumentException + */ + public KeyValue(final byte[] row, final byte[] family, + final byte[] qualifier, final long timestamp, Type type) { + this(row, family, qualifier, timestamp, type, null); + } + + /** + * Constructs KeyValue structure filled with specified values. + * @param row row key + * @param family family name + * @param qualifier column qualifier + * @param timestamp version timestamp + * @param value column value + * @throws IllegalArgumentException + */ + public KeyValue(final byte[] row, final byte[] family, + final byte[] qualifier, final long timestamp, final byte[] value) { + this(row, family, qualifier, timestamp, Type.Put, value); + } + + /** + * Constructs KeyValue structure filled with specified values. + * @param row row key + * @param family family name + * @param qualifier column qualifier + * @param timestamp version timestamp + * @param type key type + * @param value column value + * @throws IllegalArgumentException + */ + public KeyValue(final byte[] row, final byte[] family, + final byte[] qualifier, final long timestamp, Type type, + final byte[] value) { + this(row, family, qualifier, 0, qualifier==null ? 0 : qualifier.length, + timestamp, type, value, 0, value==null ? 0 : value.length); + } + + /** + * Constructs KeyValue structure filled with specified values. + * @param row row key + * @param family family name + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * @param timestamp version timestamp + * @param type key type + * @param value column value + * @param voffset value offset + * @param vlength value length + * @throws IllegalArgumentException + */ + public KeyValue(byte [] row, byte [] family, + byte [] qualifier, int qoffset, int qlength, long timestamp, Type type, + byte [] value, int voffset, int vlength) { + this(row, 0, row==null ? 0 : row.length, + family, 0, family==null ? 0 : family.length, + qualifier, qoffset, qlength, timestamp, type, + value, voffset, vlength); + } + + /** + * Constructs KeyValue structure filled with specified values. + *

+ * Column is split into two fields, family and qualifier. + * @param row row key + * @param roffset row offset + * @param rlength row length + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * @param timestamp version timestamp + * @param type key type + * @param value column value + * @param voffset value offset + * @param vlength value length + * @throws IllegalArgumentException + */ + public KeyValue(final byte [] row, final int roffset, final int rlength, + final byte [] family, final int foffset, final int flength, + final byte [] qualifier, final int qoffset, final int qlength, + final long timestamp, final Type type, + final byte [] value, final int voffset, final int vlength) { + this.bytes = createByteArray(row, roffset, rlength, + family, foffset, flength, qualifier, qoffset, qlength, + timestamp, type, value, voffset, vlength); + this.length = bytes.length; + this.offset = 0; + } + + /** + * Constructs an empty KeyValue structure, with specified sizes. + * This can be used to partially fill up KeyValues. + *

+ * Column is split into two fields, family and qualifier. + * @param rlength row length + * @param flength family length + * @param qlength qualifier length + * @param timestamp version timestamp + * @param type key type + * @param vlength value length + * @throws IllegalArgumentException + */ + public KeyValue(final int rlength, + final int flength, + final int qlength, + final long timestamp, final Type type, + final int vlength) { + this.bytes = createEmptyByteArray(rlength, + flength, qlength, + timestamp, type, vlength); + this.length = bytes.length; + this.offset = 0; + } + + /** + * Create an empty byte[] representing a KeyValue + * All lengths are preset and can be filled in later. + * @param rlength + * @param flength + * @param qlength + * @param timestamp + * @param type + * @param vlength + * @return The newly created byte array. + */ + static byte[] createEmptyByteArray(final int rlength, int flength, + int qlength, final long timestamp, final Type type, int vlength) { + if (rlength > Short.MAX_VALUE) { + throw new IllegalArgumentException("Row > " + Short.MAX_VALUE); + } + if (flength > Byte.MAX_VALUE) { + throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE); + } + // Qualifier length + if (qlength > Integer.MAX_VALUE - rlength - flength) { + throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE); + } + // Key length + long longkeylength = KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength; + if (longkeylength > Integer.MAX_VALUE) { + throw new IllegalArgumentException("keylength " + longkeylength + " > " + + Integer.MAX_VALUE); + } + int keylength = (int)longkeylength; + // Value length + if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON + throw new IllegalArgumentException("Valuer > " + + HConstants.MAXIMUM_VALUE_LENGTH); + } + + // Allocate right-sized byte array. + byte [] bytes = new byte[KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength]; + // Write the correct size markers + int pos = 0; + pos = Bytes.putInt(bytes, pos, keylength); + pos = Bytes.putInt(bytes, pos, vlength); + pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff)); + pos += rlength; + pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff)); + pos += flength + qlength; + pos = Bytes.putLong(bytes, pos, timestamp); + pos = Bytes.putByte(bytes, pos, type.getCode()); + return bytes; + } + + /** + * Write KeyValue format into a byte array. + * + * @param row row key + * @param roffset row offset + * @param rlength row length + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * @param timestamp version timestamp + * @param type key type + * @param value column value + * @param voffset value offset + * @param vlength value length + * @return The newly created byte array. + */ + static byte [] createByteArray(final byte [] row, final int roffset, + final int rlength, final byte [] family, final int foffset, int flength, + final byte [] qualifier, final int qoffset, int qlength, + final long timestamp, final Type type, + final byte [] value, final int voffset, int vlength) { + if (rlength > Short.MAX_VALUE) { + throw new IllegalArgumentException("Row > " + Short.MAX_VALUE); + } + if (row == null) { + throw new IllegalArgumentException("Row is null"); + } + // Family length + flength = family == null ? 0 : flength; + if (flength > Byte.MAX_VALUE) { + throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE); + } + // Qualifier length + qlength = qualifier == null ? 0 : qlength; + if (qlength > Integer.MAX_VALUE - rlength - flength) { + throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE); + } + // Key length + long longkeylength = KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength; + if (longkeylength > Integer.MAX_VALUE) { + throw new IllegalArgumentException("keylength " + longkeylength + " > " + + Integer.MAX_VALUE); + } + int keylength = (int)longkeylength; + // Value length + vlength = value == null? 0 : vlength; + if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON + throw new IllegalArgumentException("Valuer > " + + HConstants.MAXIMUM_VALUE_LENGTH); + } + + // Allocate right-sized byte array. + byte [] bytes = new byte[KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength]; + // Write key, value and key row length. + int pos = 0; + pos = Bytes.putInt(bytes, pos, keylength); + pos = Bytes.putInt(bytes, pos, vlength); + pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff)); + pos = Bytes.putBytes(bytes, pos, row, roffset, rlength); + pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff)); + if(flength != 0) { + pos = Bytes.putBytes(bytes, pos, family, foffset, flength); + } + if(qlength != 0) { + pos = Bytes.putBytes(bytes, pos, qualifier, qoffset, qlength); + } + pos = Bytes.putLong(bytes, pos, timestamp); + pos = Bytes.putByte(bytes, pos, type.getCode()); + if (value != null && value.length > 0) { + pos = Bytes.putBytes(bytes, pos, value, voffset, vlength); + } + return bytes; + } + + /** + * Write KeyValue format into a byte array. + *

+ * Takes column in the form family:qualifier + * @param row - row key (arbitrary byte array) + * @param roffset + * @param rlength + * @param column + * @param coffset + * @param clength + * @param timestamp + * @param type + * @param value + * @param voffset + * @param vlength + * @return The newly created byte array. + */ + static byte [] createByteArray(final byte [] row, final int roffset, + final int rlength, + final byte [] column, final int coffset, int clength, + final long timestamp, final Type type, + final byte [] value, final int voffset, int vlength) { + // If column is non-null, figure where the delimiter is at. + int delimiteroffset = 0; + if (column != null && column.length > 0) { + delimiteroffset = getFamilyDelimiterIndex(column, coffset, clength); + if (delimiteroffset > Byte.MAX_VALUE) { + throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE); + } + } else { + return createByteArray(row,roffset,rlength,null,0,0,null,0,0,timestamp, + type,value,voffset,vlength); + } + int flength = delimiteroffset-coffset; + int qlength = clength - flength - 1; + return createByteArray(row, roffset, rlength, column, coffset, + flength, column, delimiteroffset+1, qlength, timestamp, type, + value, voffset, vlength); + } + + // Needed doing 'contains' on List. Only compares the key portion, not the + // value. + public boolean equals(Object other) { + if (!(other instanceof KeyValue)) { + return false; + } + KeyValue kv = (KeyValue)other; + // Comparing bytes should be fine doing equals test. Shouldn't have to + // worry about special .META. comparators doing straight equals. + return Bytes.equals(getBuffer(), getKeyOffset(), getKeyLength(), + kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength()); + } + + public int hashCode() { + byte[] b = getBuffer(); + int start = getOffset(), end = getOffset() + getLength(); + int h = b[start++]; + for (int i = start; i < end; i++) { + h = (h * 13) ^ b[i]; + } + return h; + } + + //--------------------------------------------------------------------------- + // + // KeyValue cloning + // + //--------------------------------------------------------------------------- + + /** + * Clones a KeyValue. This creates a copy, re-allocating the buffer. + * @return Fully copied clone of this KeyValue + */ + public KeyValue clone() { + byte [] b = new byte[this.length]; + System.arraycopy(this.bytes, this.offset, b, 0, this.length); + KeyValue ret = new KeyValue(b, 0, b.length); + // Important to clone the memstoreTS as well - otherwise memstore's + // update-in-place methods (eg increment) will end up creating + // new entries + ret.setMemstoreTS(memstoreTS); + return ret; + } + + /** + * Creates a deep copy of this KeyValue, re-allocating the buffer. + * Same function as {@link #clone()}. Added for clarity vs shallowCopy() + * @return Deep copy of this KeyValue + */ + public KeyValue deepCopy() { + return clone(); + } + + /** + * Creates a shallow copy of this KeyValue, reusing the data byte buffer. + * http://en.wikipedia.org/wiki/Object_copy + * @return Shallow copy of this KeyValue + */ + public KeyValue shallowCopy() { + KeyValue shallowCopy = new KeyValue(this.bytes, this.offset, this.length); + shallowCopy.setMemstoreTS(this.memstoreTS); + return shallowCopy; + } + + //--------------------------------------------------------------------------- + // + // String representation + // + //--------------------------------------------------------------------------- + + public String toString() { + if (this.bytes == null || this.bytes.length == 0) { + return "empty"; + } + return keyToString(this.bytes, this.offset + ROW_OFFSET, getKeyLength()) + + "/vlen=" + getValueLength() + "/ts=" + memstoreTS; + } + + /** + * @param k Key portion of a KeyValue. + * @return Key as a String. + */ + public static String keyToString(final byte [] k) { + return keyToString(k, 0, k.length); + } + + /** + * Use for logging. + * @param b Key portion of a KeyValue. + * @param o Offset to start of key + * @param l Length of key. + * @return Key as a String. + */ + /** + * Produces a string map for this key/value pair. Useful for programmatic use + * and manipulation of the data stored in an HLogKey, for example, printing + * as JSON. Values are left out due to their tendency to be large. If needed, + * they can be added manually. + * + * @return the Map containing data from this key + */ + public Map toStringMap() { + Map stringMap = new HashMap(); + stringMap.put("row", Bytes.toStringBinary(getRow())); + stringMap.put("family", Bytes.toStringBinary(getFamily())); + stringMap.put("qualifier", Bytes.toStringBinary(getQualifier())); + stringMap.put("timestamp", getTimestamp()); + stringMap.put("vlen", getValueLength()); + return stringMap; + } + + public static String keyToString(final byte [] b, final int o, final int l) { + if (b == null) return ""; + int rowlength = Bytes.toShort(b, o); + String row = Bytes.toStringBinary(b, o + Bytes.SIZEOF_SHORT, rowlength); + int columnoffset = o + Bytes.SIZEOF_SHORT + 1 + rowlength; + int familylength = b[columnoffset - 1]; + int columnlength = l - ((columnoffset - o) + TIMESTAMP_TYPE_SIZE); + String family = familylength == 0? "": + Bytes.toStringBinary(b, columnoffset, familylength); + String qualifier = columnlength == 0? "": + Bytes.toStringBinary(b, columnoffset + familylength, + columnlength - familylength); + long timestamp = Bytes.toLong(b, o + (l - TIMESTAMP_TYPE_SIZE)); + String timestampStr = humanReadableTimestamp(timestamp); + byte type = b[o + l - 1]; + return row + "/" + family + + (family != null && family.length() > 0? ":" :"") + + qualifier + "/" + timestampStr + "/" + Type.codeToType(type); + } + + public static String humanReadableTimestamp(final long timestamp) { + if (timestamp == HConstants.LATEST_TIMESTAMP) { + return "LATEST_TIMESTAMP"; + } + if (timestamp == HConstants.OLDEST_TIMESTAMP) { + return "OLDEST_TIMESTAMP"; + } + return String.valueOf(timestamp); + } + + //--------------------------------------------------------------------------- + // + // Public Member Accessors + // + //--------------------------------------------------------------------------- + + /** + * @return The byte array backing this KeyValue. + */ + public byte [] getBuffer() { + return this.bytes; + } + + /** + * @return Offset into {@link #getBuffer()} at which this KeyValue starts. + */ + public int getOffset() { + return this.offset; + } + + /** + * @return Length of bytes this KeyValue occupies in {@link #getBuffer()}. + */ + public int getLength() { + return length; + } + + //--------------------------------------------------------------------------- + // + // Length and Offset Calculators + // + //--------------------------------------------------------------------------- + + /** + * Determines the total length of the KeyValue stored in the specified + * byte array and offset. Includes all headers. + * @param bytes byte array + * @param offset offset to start of the KeyValue + * @return length of entire KeyValue, in bytes + */ + private static int getLength(byte [] bytes, int offset) { + return ROW_OFFSET + + Bytes.toInt(bytes, offset) + + Bytes.toInt(bytes, offset + Bytes.SIZEOF_INT); + } + + /** + * @return Key offset in backing buffer.. + */ + public int getKeyOffset() { + return this.offset + ROW_OFFSET; + } + + public String getKeyString() { + return Bytes.toStringBinary(getBuffer(), getKeyOffset(), getKeyLength()); + } + + /** + * @return Length of key portion. + */ + private int keyLength = 0; + + public int getKeyLength() { + if (keyLength == 0) { + keyLength = Bytes.toInt(this.bytes, this.offset); + } + return keyLength; + } + + /** + * @return Value offset + */ + public int getValueOffset() { + return getKeyOffset() + getKeyLength(); + } + + /** + * @return Value length + */ + public int getValueLength() { + return Bytes.toInt(this.bytes, this.offset + Bytes.SIZEOF_INT); + } + + /** + * @return Row offset + */ + public int getRowOffset() { + return getKeyOffset() + Bytes.SIZEOF_SHORT; + } + + /** + * @return Row length + */ + public short getRowLength() { + return Bytes.toShort(this.bytes, getKeyOffset()); + } + + /** + * @return Family offset + */ + public int getFamilyOffset() { + return getFamilyOffset(getRowLength()); + } + + /** + * @return Family offset + */ + public int getFamilyOffset(int rlength) { + return this.offset + ROW_OFFSET + Bytes.SIZEOF_SHORT + rlength + Bytes.SIZEOF_BYTE; + } + + /** + * @return Family length + */ + public byte getFamilyLength() { + return getFamilyLength(getFamilyOffset()); + } + + /** + * @return Family length + */ + public byte getFamilyLength(int foffset) { + return this.bytes[foffset-1]; + } + + /** + * @return Qualifier offset + */ + public int getQualifierOffset() { + return getQualifierOffset(getFamilyOffset()); + } + + /** + * @return Qualifier offset + */ + public int getQualifierOffset(int foffset) { + return foffset + getFamilyLength(foffset); + } + + /** + * @return Qualifier length + */ + public int getQualifierLength() { + return getQualifierLength(getRowLength(),getFamilyLength()); + } + + /** + * @return Qualifier length + */ + public int getQualifierLength(int rlength, int flength) { + return getKeyLength() - + (KEY_INFRASTRUCTURE_SIZE + rlength + flength); + } + + /** + * @return Column (family + qualifier) length + */ + public int getTotalColumnLength() { + int rlength = getRowLength(); + int foffset = getFamilyOffset(rlength); + return getTotalColumnLength(rlength,foffset); + } + + /** + * @return Column (family + qualifier) length + */ + public int getTotalColumnLength(int rlength, int foffset) { + int flength = getFamilyLength(foffset); + int qlength = getQualifierLength(rlength,flength); + return flength + qlength; + } + + /** + * @return Timestamp offset + */ + public int getTimestampOffset() { + return getTimestampOffset(getKeyLength()); + } + + /** + * @param keylength Pass if you have it to save on a int creation. + * @return Timestamp offset + */ + public int getTimestampOffset(final int keylength) { + return getKeyOffset() + keylength - TIMESTAMP_TYPE_SIZE; + } + + /** + * @return True if this KeyValue has a LATEST_TIMESTAMP timestamp. + */ + public boolean isLatestTimestamp() { + return Bytes.equals(getBuffer(), getTimestampOffset(), Bytes.SIZEOF_LONG, + HConstants.LATEST_TIMESTAMP_BYTES, 0, Bytes.SIZEOF_LONG); + } + + /** + * @return True if this is a "fake" KV created for internal seeking purposes, + * which should not be seen by user code + */ + public boolean isInternal() { + byte type = getType(); + return type == Type.Minimum.code || type == Type.Maximum.code; + } + /** + * @param now Time to set into this IFF timestamp == + * {@link HConstants#LATEST_TIMESTAMP} (else, its a noop). + * @return True is we modified this. + */ + public boolean updateLatestStamp(final byte [] now) { + if (this.isLatestTimestamp()) { + int tsOffset = getTimestampOffset(); + System.arraycopy(now, 0, this.bytes, tsOffset, Bytes.SIZEOF_LONG); + // clear cache or else getTimestamp() possibly returns an old value + return true; + } + return false; + } + + //--------------------------------------------------------------------------- + // + // Methods that return copies of fields + // + //--------------------------------------------------------------------------- + + /** + * Do not use unless you have to. Used internally for compacting and testing. + * + * Use {@link #getRow()}, {@link #getFamily()}, {@link #getQualifier()}, and + * {@link #getValue()} if accessing a KeyValue client-side. + * @return Copy of the key portion only. + */ + public byte [] getKey() { + int keylength = getKeyLength(); + byte [] key = new byte[keylength]; + System.arraycopy(getBuffer(), getKeyOffset(), key, 0, keylength); + return key; + } + + /** + * Returns value in a new byte array. + * Primarily for use client-side. If server-side, use + * {@link #getBuffer()} with appropriate offsets and lengths instead to + * save on allocations. + * @return Value in a new byte array. + */ + public byte [] getValue() { + int o = getValueOffset(); + int l = getValueLength(); + byte [] result = new byte[l]; + System.arraycopy(getBuffer(), o, result, 0, l); + return result; + } + + /** + * Primarily for use client-side. Returns the row of this KeyValue in a new + * byte array.

+ * + * If server-side, use {@link #getBuffer()} with appropriate offsets and + * lengths instead. + * @return Row in a new byte array. + */ + public byte [] getRow() { + int o = getRowOffset(); + short l = getRowLength(); + byte result[] = new byte[l]; + System.arraycopy(getBuffer(), o, result, 0, l); + return result; + } + + /** + * + * @return Timestamp + */ + public long getTimestamp() { + return getTimestamp(getKeyLength()); + } + + /** + * @param keylength Pass if you have it to save on a int creation. + * @return Timestamp + */ + long getTimestamp(final int keylength) { + int tsOffset = getTimestampOffset(keylength); + return Bytes.toLong(this.bytes, tsOffset); + } + + /** + * @return Type of this KeyValue. + */ + public byte getType() { + return getType(getKeyLength()); + } + + /** + * @param keylength Pass if you have it to save on a int creation. + * @return Type of this KeyValue. + */ + byte getType(final int keylength) { + return this.bytes[this.offset + keylength - 1 + ROW_OFFSET]; + } + + /** + * @return True if a delete type, a {@link KeyValue.Type#Delete} or + * a {KeyValue.Type#DeleteFamily} or a {@link KeyValue.Type#DeleteColumn} + * KeyValue type. + */ + public boolean isDelete() { + return KeyValue.isDelete(getType()); + } + + /** + * @return True if this KV is a {@link KeyValue.Type#Delete} type. + */ + public boolean isDeleteType() { + // TODO: Fix this method name vis-a-vis isDelete! + return getType() == Type.Delete.getCode(); + } + + /** + * @return True if this KV is a delete family type. + */ + public boolean isDeleteFamily() { + return getType() == Type.DeleteFamily.getCode(); + } + + /** + * + * @return True if this KV is a delete family or column type. + */ + public boolean isDeleteColumnOrFamily() { + int t = getType(); + return t == Type.DeleteColumn.getCode() || t == Type.DeleteFamily.getCode(); + } + + /** + * Primarily for use client-side. Returns the family of this KeyValue in a + * new byte array.

+ * + * If server-side, use {@link #getBuffer()} with appropriate offsets and + * lengths instead. + * @return Returns family. Makes a copy. + */ + public byte [] getFamily() { + int o = getFamilyOffset(); + int l = getFamilyLength(o); + byte [] result = new byte[l]; + System.arraycopy(this.bytes, o, result, 0, l); + return result; + } + + /** + * Primarily for use client-side. Returns the column qualifier of this + * KeyValue in a new byte array.

+ * + * If server-side, use {@link #getBuffer()} with appropriate offsets and + * lengths instead. + * Use {@link #getBuffer()} with appropriate offsets and lengths instead. + * @return Returns qualifier. Makes a copy. + */ + public byte [] getQualifier() { + int o = getQualifierOffset(); + int l = getQualifierLength(); + byte [] result = new byte[l]; + System.arraycopy(this.bytes, o, result, 0, l); + return result; + } + + //--------------------------------------------------------------------------- + // + // KeyValue splitter + // + //--------------------------------------------------------------------------- + + /** + * Utility class that splits a KeyValue buffer into separate byte arrays. + *

+ * Should get rid of this if we can, but is very useful for debugging. + */ + public static class SplitKeyValue { + private byte [][] split; + SplitKeyValue() { + this.split = new byte[6][]; + } + public void setRow(byte [] value) { this.split[0] = value; } + public void setFamily(byte [] value) { this.split[1] = value; } + public void setQualifier(byte [] value) { this.split[2] = value; } + public void setTimestamp(byte [] value) { this.split[3] = value; } + public void setType(byte [] value) { this.split[4] = value; } + public void setValue(byte [] value) { this.split[5] = value; } + public byte [] getRow() { return this.split[0]; } + public byte [] getFamily() { return this.split[1]; } + public byte [] getQualifier() { return this.split[2]; } + public byte [] getTimestamp() { return this.split[3]; } + public byte [] getType() { return this.split[4]; } + public byte [] getValue() { return this.split[5]; } + } + + public SplitKeyValue split() { + SplitKeyValue split = new SplitKeyValue(); + int splitOffset = this.offset; + int keyLen = Bytes.toInt(bytes, splitOffset); + splitOffset += Bytes.SIZEOF_INT; + int valLen = Bytes.toInt(bytes, splitOffset); + splitOffset += Bytes.SIZEOF_INT; + short rowLen = Bytes.toShort(bytes, splitOffset); + splitOffset += Bytes.SIZEOF_SHORT; + byte [] row = new byte[rowLen]; + System.arraycopy(bytes, splitOffset, row, 0, rowLen); + splitOffset += rowLen; + split.setRow(row); + byte famLen = bytes[splitOffset]; + splitOffset += Bytes.SIZEOF_BYTE; + byte [] family = new byte[famLen]; + System.arraycopy(bytes, splitOffset, family, 0, famLen); + splitOffset += famLen; + split.setFamily(family); + int colLen = keyLen - + (rowLen + famLen + Bytes.SIZEOF_SHORT + Bytes.SIZEOF_BYTE + + Bytes.SIZEOF_LONG + Bytes.SIZEOF_BYTE); + byte [] qualifier = new byte[colLen]; + System.arraycopy(bytes, splitOffset, qualifier, 0, colLen); + splitOffset += colLen; + split.setQualifier(qualifier); + byte [] timestamp = new byte[Bytes.SIZEOF_LONG]; + System.arraycopy(bytes, splitOffset, timestamp, 0, Bytes.SIZEOF_LONG); + splitOffset += Bytes.SIZEOF_LONG; + split.setTimestamp(timestamp); + byte [] type = new byte[1]; + type[0] = bytes[splitOffset]; + splitOffset += Bytes.SIZEOF_BYTE; + split.setType(type); + byte [] value = new byte[valLen]; + System.arraycopy(bytes, splitOffset, value, 0, valLen); + split.setValue(value); + return split; + } + + //--------------------------------------------------------------------------- + // + // Compare specified fields against those contained in this KeyValue + // + //--------------------------------------------------------------------------- + + /** + * @param family + * @return True if matching families. + */ + public boolean matchingFamily(final byte [] family) { + return matchingFamily(family, 0, family.length); + } + + public boolean matchingFamily(final byte[] family, int offset, int length) { + if (this.length == 0 || this.bytes.length == 0) { + return false; + } + return Bytes.equals(family, offset, length, + this.bytes, getFamilyOffset(), getFamilyLength()); + } + + public boolean matchingFamily(final KeyValue other) { + return matchingFamily(other.getBuffer(), other.getFamilyOffset(), + other.getFamilyLength()); + } + + /** + * @param qualifier + * @return True if matching qualifiers. + */ + public boolean matchingQualifier(final byte [] qualifier) { + return matchingQualifier(qualifier, 0, qualifier.length); + } + + public boolean matchingQualifier(final byte [] qualifier, int offset, int length) { + return Bytes.equals(qualifier, offset, length, + this.bytes, getQualifierOffset(), getQualifierLength()); + } + + public boolean matchingQualifier(final KeyValue other) { + return matchingQualifier(other.getBuffer(), other.getQualifierOffset(), + other.getQualifierLength()); + } + + public boolean matchingRow(final byte [] row) { + return matchingRow(row, 0, row.length); + } + + public boolean matchingRow(final byte[] row, int offset, int length) { + return Bytes.equals(row, offset, length, + this.bytes, getRowOffset(), getRowLength()); + } + + public boolean matchingRow(KeyValue other) { + return matchingRow(other.getBuffer(), other.getRowOffset(), + other.getRowLength()); + } + + /** + * @param column Column minus its delimiter + * @return True if column matches. + */ + public boolean matchingColumnNoDelimiter(final byte [] column) { + int rl = getRowLength(); + int o = getFamilyOffset(rl); + int fl = getFamilyLength(o); + int l = fl + getQualifierLength(rl,fl); + return Bytes.equals(column, 0, column.length, this.bytes, o, l); + } + + /** + * + * @param family column family + * @param qualifier column qualifier + * @return True if column matches + */ + public boolean matchingColumn(final byte[] family, final byte[] qualifier) { + int rl = getRowLength(); + int o = getFamilyOffset(rl); + int fl = getFamilyLength(o); + int ql = getQualifierLength(rl,fl); + if (!Bytes.equals(family, 0, family.length, this.bytes, o, fl)) { + return false; + } + if (qualifier == null || qualifier.length == 0) { + if (ql == 0) { + return true; + } + return false; + } + return Bytes.equals(qualifier, 0, qualifier.length, + this.bytes, o + fl, ql); + } + + /** + * @param left + * @param loffset + * @param llength + * @param lfamilylength Offset of family delimiter in left column. + * @param right + * @param roffset + * @param rlength + * @param rfamilylength Offset of family delimiter in right column. + * @return The result of the comparison. + */ + static int compareColumns(final byte [] left, final int loffset, + final int llength, final int lfamilylength, + final byte [] right, final int roffset, final int rlength, + final int rfamilylength) { + // Compare family portion first. + int diff = Bytes.compareTo(left, loffset, lfamilylength, + right, roffset, rfamilylength); + if (diff != 0) { + return diff; + } + // Compare qualifier portion + return Bytes.compareTo(left, loffset + lfamilylength, + llength - lfamilylength, + right, roffset + rfamilylength, rlength - rfamilylength); + } + + /** + * @return True if non-null row and column. + */ + public boolean nonNullRowAndColumn() { + return getRowLength() > 0 && !isEmptyColumn(); + } + + /** + * @return True if column is empty. + */ + public boolean isEmptyColumn() { + return getQualifierLength() == 0; + } + + /** + * Creates a new KeyValue that only contains the key portion (the value is + * set to be null). + * @param lenAsVal replace value with the actual value length (false=empty) + */ + public KeyValue createKeyOnly(boolean lenAsVal) { + // KV format: + // Rebuild as: <0:4> + int dataLen = lenAsVal? Bytes.SIZEOF_INT : 0; + byte [] newBuffer = new byte[getKeyLength() + ROW_OFFSET + dataLen]; + System.arraycopy(this.bytes, this.offset, newBuffer, 0, + Math.min(newBuffer.length,this.length)); + Bytes.putInt(newBuffer, Bytes.SIZEOF_INT, dataLen); + if (lenAsVal) { + Bytes.putInt(newBuffer, newBuffer.length - dataLen, this.getValueLength()); + } + return new KeyValue(newBuffer); + } + + /** + * Splits a column in family:qualifier form into separate byte arrays. + *

+ * Not recommend to be used as this is old-style API. + * @param c The column. + * @return The parsed column. + */ + public static byte [][] parseColumn(byte [] c) { + final int index = getDelimiter(c, 0, c.length, COLUMN_FAMILY_DELIMITER); + if (index == -1) { + // If no delimiter, return array of size 1 + return new byte [][] { c }; + } else if(index == c.length - 1) { + // Only a family, return array size 1 + byte [] family = new byte[c.length-1]; + System.arraycopy(c, 0, family, 0, family.length); + return new byte [][] { family }; + } + // Family and column, return array size 2 + final byte [][] result = new byte [2][]; + result[0] = new byte [index]; + System.arraycopy(c, 0, result[0], 0, index); + final int len = c.length - (index + 1); + result[1] = new byte[len]; + System.arraycopy(c, index + 1 /*Skip delimiter*/, result[1], 0, + len); + return result; + } + + /** + * Makes a column in family:qualifier form from separate byte arrays. + *

+ * Not recommended for usage as this is old-style API. + * @param family + * @param qualifier + * @return family:qualifier + */ + public static byte [] makeColumn(byte [] family, byte [] qualifier) { + return Bytes.add(family, COLUMN_FAMILY_DELIM_ARRAY, qualifier); + } + + /** + * @param b + * @return Index of the family-qualifier colon delimiter character in passed + * buffer. + */ + public static int getFamilyDelimiterIndex(final byte [] b, final int offset, + final int length) { + return getRequiredDelimiter(b, offset, length, COLUMN_FAMILY_DELIMITER); + } + + private static int getRequiredDelimiter(final byte [] b, + final int offset, final int length, final int delimiter) { + int index = getDelimiter(b, offset, length, delimiter); + if (index < 0) { + throw new IllegalArgumentException("No " + (char)delimiter + " in <" + + Bytes.toString(b) + ">" + ", length=" + length + ", offset=" + offset); + } + return index; + } + + /** + * This function is only used in Meta key comparisons so its error message + * is specific for meta key errors. + */ + static int getRequiredDelimiterInReverse(final byte [] b, + final int offset, final int length, final int delimiter) { + int index = getDelimiterInReverse(b, offset, length, delimiter); + if (index < 0) { + throw new IllegalArgumentException(".META. key must have two '" + (char)delimiter + "' " + + "delimiters and have the following format: ',,'"); + } + return index; + } + + /** + * @param b + * @param delimiter + * @return Index of delimiter having started from start of b + * moving rightward. + */ + public static int getDelimiter(final byte [] b, int offset, final int length, + final int delimiter) { + if (b == null) { + throw new IllegalArgumentException("Passed buffer is null"); + } + int result = -1; + for (int i = offset; i < length + offset; i++) { + if (b[i] == delimiter) { + result = i; + break; + } + } + return result; + } + + /** + * Find index of passed delimiter walking from end of buffer backwards. + * @param b + * @param delimiter + * @return Index of delimiter + */ + public static int getDelimiterInReverse(final byte [] b, final int offset, + final int length, final int delimiter) { + if (b == null) { + throw new IllegalArgumentException("Passed buffer is null"); + } + int result = -1; + for (int i = (offset + length) - 1; i >= offset; i--) { + if (b[i] == delimiter) { + result = i; + break; + } + } + return result; + } + + /** + * A {@link KVComparator} for -ROOT- catalog table + * {@link KeyValue}s. + */ + public static class RootComparator extends MetaComparator { + private final KeyComparator rawcomparator = new RootKeyComparator(); + + public KeyComparator getRawComparator() { + return this.rawcomparator; + } + + @Override + protected Object clone() throws CloneNotSupportedException { + return new RootComparator(); + } + } + + /** + * A {@link KVComparator} for .META. catalog table + * {@link KeyValue}s. + */ + public static class MetaComparator extends KVComparator { + private final KeyComparator rawcomparator = new MetaKeyComparator(); + + public KeyComparator getRawComparator() { + return this.rawcomparator; + } + + @Override + protected Object clone() throws CloneNotSupportedException { + return new MetaComparator(); + } + } + + /** + * Compare KeyValues. When we compare KeyValues, we only compare the Key + * portion. This means two KeyValues with same Key but different Values are + * considered the same as far as this Comparator is concerned. + * Hosts a {@link KeyComparator}. + */ + public static class KVComparator implements java.util.Comparator { + private final KeyComparator rawcomparator = new KeyComparator(); + + /** + * @return RawComparator that can compare the Key portion of a KeyValue. + * Used in hfile where indices are the Key portion of a KeyValue. + */ + public KeyComparator getRawComparator() { + return this.rawcomparator; + } + + public int compare(final KeyValue left, final KeyValue right) { + int ret = getRawComparator().compare(left.getBuffer(), + left.getOffset() + ROW_OFFSET, left.getKeyLength(), + right.getBuffer(), right.getOffset() + ROW_OFFSET, + right.getKeyLength()); + if (ret != 0) return ret; + // Negate this comparison so later edits show up first + return -Longs.compare(left.getMemstoreTS(), right.getMemstoreTS()); + } + + public int compareTimestamps(final KeyValue left, final KeyValue right) { + return compareTimestamps(left, left.getKeyLength(), right, + right.getKeyLength()); + } + + int compareTimestamps(final KeyValue left, final int lkeylength, + final KeyValue right, final int rkeylength) { + // Compare timestamps + long ltimestamp = left.getTimestamp(lkeylength); + long rtimestamp = right.getTimestamp(rkeylength); + return getRawComparator().compareTimestamps(ltimestamp, rtimestamp); + } + + /** + * @param left + * @param right + * @return Result comparing rows. + */ + public int compareRows(final KeyValue left, final KeyValue right) { + return compareRows(left, left.getRowLength(), right, + right.getRowLength()); + } + + /** + * @param left + * @param lrowlength Length of left row. + * @param right + * @param rrowlength Length of right row. + * @return Result comparing rows. + */ + public int compareRows(final KeyValue left, final short lrowlength, + final KeyValue right, final short rrowlength) { + return getRawComparator().compareRows(left.getBuffer(), + left.getRowOffset(), lrowlength, + right.getBuffer(), right.getRowOffset(), rrowlength); + } + + /** + * @param left + * @param row - row key (arbitrary byte array) + * @return RawComparator + */ + public int compareRows(final KeyValue left, final byte [] row) { + return getRawComparator().compareRows(left.getBuffer(), + left.getRowOffset(), left.getRowLength(), row, 0, row.length); + } + + public int compareRows(byte [] left, int loffset, int llength, + byte [] right, int roffset, int rlength) { + return getRawComparator().compareRows(left, loffset, llength, + right, roffset, rlength); + } + + public int compareColumns(final KeyValue left, final byte [] right, + final int roffset, final int rlength, final int rfamilyoffset) { + int offset = left.getFamilyOffset(); + int length = left.getFamilyLength() + left.getQualifierLength(); + return getRawComparator().compareColumns(left.getBuffer(), offset, length, + left.getFamilyLength(offset), + right, roffset, rlength, rfamilyoffset); + } + + int compareColumns(final KeyValue left, final short lrowlength, + final KeyValue right, final short rrowlength) { + int lfoffset = left.getFamilyOffset(lrowlength); + int rfoffset = right.getFamilyOffset(rrowlength); + int lclength = left.getTotalColumnLength(lrowlength,lfoffset); + int rclength = right.getTotalColumnLength(rrowlength, rfoffset); + int lfamilylength = left.getFamilyLength(lfoffset); + int rfamilylength = right.getFamilyLength(rfoffset); + return getRawComparator().compareColumns(left.getBuffer(), lfoffset, + lclength, lfamilylength, + right.getBuffer(), rfoffset, rclength, rfamilylength); + } + + /** + * Compares the row and column of two keyvalues for equality + * @param left + * @param right + * @return True if same row and column. + */ + public boolean matchingRowColumn(final KeyValue left, + final KeyValue right) { + short lrowlength = left.getRowLength(); + short rrowlength = right.getRowLength(); + // TsOffset = end of column data. just comparing Row+CF length of each + return ((left.getTimestampOffset() - left.getOffset()) == + (right.getTimestampOffset() - right.getOffset())) && + matchingRows(left, lrowlength, right, rrowlength) && + compareColumns(left, lrowlength, right, rrowlength) == 0; + } + + /** + * @param left + * @param right + * @return True if rows match. + */ + public boolean matchingRows(final KeyValue left, final byte [] right) { + return Bytes.equals(left.getBuffer(), left.getRowOffset(), left.getRowLength(), + right, 0, right.length); + } + + /** + * Compares the row of two keyvalues for equality + * @param left + * @param right + * @return True if rows match. + */ + public boolean matchingRows(final KeyValue left, final KeyValue right) { + short lrowlength = left.getRowLength(); + short rrowlength = right.getRowLength(); + return matchingRows(left, lrowlength, right, rrowlength); + } + + /** + * @param left + * @param lrowlength + * @param right + * @param rrowlength + * @return True if rows match. + */ + public boolean matchingRows(final KeyValue left, final short lrowlength, + final KeyValue right, final short rrowlength) { + return lrowlength == rrowlength && + Bytes.equals(left.getBuffer(), left.getRowOffset(), lrowlength, + right.getBuffer(), right.getRowOffset(), rrowlength); + } + + public boolean matchingRows(final byte [] left, final int loffset, + final int llength, + final byte [] right, final int roffset, final int rlength) { + return Bytes.equals(left, loffset, llength, + right, roffset, rlength); + } + + /** + * Compares the row and timestamp of two keys + * Was called matchesWithoutColumn in HStoreKey. + * @param right Key to compare against. + * @return True if same row and timestamp is greater than the timestamp in + * right + */ + public boolean matchingRowsGreaterTimestamp(final KeyValue left, + final KeyValue right) { + short lrowlength = left.getRowLength(); + short rrowlength = right.getRowLength(); + if (!matchingRows(left, lrowlength, right, rrowlength)) { + return false; + } + return left.getTimestamp() >= right.getTimestamp(); + } + + @Override + protected Object clone() throws CloneNotSupportedException { + return new KVComparator(); + } + + /** + * @return Comparator that ignores timestamps; useful counting versions. + */ + public KVComparator getComparatorIgnoringTimestamps() { + KVComparator c = null; + try { + c = (KVComparator)this.clone(); + c.getRawComparator().ignoreTimestamp = true; + } catch (CloneNotSupportedException e) { + LOG.error("Not supported", e); + } + return c; + } + + /** + * @return Comparator that ignores key type; useful checking deletes + */ + public KVComparator getComparatorIgnoringType() { + KVComparator c = null; + try { + c = (KVComparator)this.clone(); + c.getRawComparator().ignoreType = true; + } catch (CloneNotSupportedException e) { + LOG.error("Not supported", e); + } + return c; + } + } + + /** + * Creates a KeyValue that is last on the specified row id. That is, + * every other possible KeyValue for the given row would compareTo() + * less than the result of this call. + * @param row row key + * @return Last possible KeyValue on passed row + */ + public static KeyValue createLastOnRow(final byte[] row) { + return new KeyValue(row, null, null, HConstants.LATEST_TIMESTAMP, Type.Minimum); + } + + /** + * Create a KeyValue that is smaller than all other possible KeyValues + * for the given row. That is any (valid) KeyValue on 'row' would sort + * _after_ the result. + * + * @param row - row key (arbitrary byte array) + * @return First possible KeyValue on passed row + */ + public static KeyValue createFirstOnRow(final byte [] row) { + return createFirstOnRow(row, HConstants.LATEST_TIMESTAMP); + } + + /** + * Create a KeyValue that is smaller than all other possible KeyValues + * for the given row. That is any (valid) KeyValue on 'row' would sort + * _after_ the result. + * + * @param row - row key (arbitrary byte array) + * @return First possible KeyValue on passed row + */ + public static KeyValue createFirstOnRow(final byte [] row, int roffset, short rlength) { + return new KeyValue(row, roffset, rlength, + null, 0, 0, null, 0, 0, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0); + } + + /** + * Creates a KeyValue that is smaller than all other KeyValues that + * are older than the passed timestamp. + * @param row - row key (arbitrary byte array) + * @param ts - timestamp + * @return First possible key on passed row and timestamp. + */ + public static KeyValue createFirstOnRow(final byte [] row, + final long ts) { + return new KeyValue(row, null, null, ts, Type.Maximum); + } + + /** + * Create a KeyValue for the specified row, family and qualifier that would be + * smaller than all other possible KeyValues that have the same row,family,qualifier. + * Used for seeking. + * @param row - row key (arbitrary byte array) + * @param family - family name + * @param qualifier - column qualifier + * @return First possible key on passed row, and column. + */ + public static KeyValue createFirstOnRow(final byte [] row, final byte [] family, + final byte [] qualifier) { + return new KeyValue(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum); + } + + /** + * Create a Delete Family KeyValue for the specified row and family that would + * be smaller than all other possible Delete Family KeyValues that have the + * same row and family. + * Used for seeking. + * @param row - row key (arbitrary byte array) + * @param family - family name + * @return First Delete Family possible key on passed row. + */ + public static KeyValue createFirstDeleteFamilyOnRow(final byte [] row, + final byte [] family) { + return new KeyValue(row, family, null, HConstants.LATEST_TIMESTAMP, + Type.DeleteFamily); + } + + /** + * @param row - row key (arbitrary byte array) + * @param f - family name + * @param q - column qualifier + * @param ts - timestamp + * @return First possible key on passed row, column and timestamp + */ + public static KeyValue createFirstOnRow(final byte [] row, final byte [] f, + final byte [] q, final long ts) { + return new KeyValue(row, f, q, ts, Type.Maximum); + } + + /** + * Create a KeyValue for the specified row, family and qualifier that would be + * smaller than all other possible KeyValues that have the same row, + * family, qualifier. + * Used for seeking. + * @param row row key + * @param roffset row offset + * @param rlength row length + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * @return First possible key on passed Row, Family, Qualifier. + */ + public static KeyValue createFirstOnRow(final byte [] row, + final int roffset, final int rlength, final byte [] family, + final int foffset, final int flength, final byte [] qualifier, + final int qoffset, final int qlength) { + return new KeyValue(row, roffset, rlength, family, + foffset, flength, qualifier, qoffset, qlength, + HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0); + } + + /** + * Create a KeyValue for the specified row, family and qualifier that would be + * larger than or equal to all other possible KeyValues that have the same + * row, family, qualifier. + * Used for reseeking. + * @param row row key + * @param roffset row offset + * @param rlength row length + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * @return Last possible key on passed row, family, qualifier. + */ + public static KeyValue createLastOnRow(final byte [] row, + final int roffset, final int rlength, final byte [] family, + final int foffset, final int flength, final byte [] qualifier, + final int qoffset, final int qlength) { + return new KeyValue(row, roffset, rlength, family, + foffset, flength, qualifier, qoffset, qlength, + HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0); + } + + /** + * Similar to {@link #createLastOnRow(byte[], int, int, byte[], int, int, + * byte[], int, int)} but creates the last key on the row/column of this KV + * (the value part of the returned KV is always empty). Used in creating + * "fake keys" for the multi-column Bloom filter optimization to skip the + * row/column we already know is not in the file. + * @return the last key on the row/column of the given key-value pair + */ + public KeyValue createLastOnRowCol() { + return new KeyValue( + bytes, getRowOffset(), getRowLength(), + bytes, getFamilyOffset(), getFamilyLength(), + bytes, getQualifierOffset(), getQualifierLength(), + HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0); + } + + /** + * Creates the first KV with the row/family/qualifier of this KV and the + * given timestamp. Uses the "maximum" KV type that guarantees that the new + * KV is the lowest possible for this combination of row, family, qualifier, + * and timestamp. This KV's own timestamp is ignored. While this function + * copies the value from this KV, it is normally used on key-only KVs. + */ + public KeyValue createFirstOnRowColTS(long ts) { + return new KeyValue( + bytes, getRowOffset(), getRowLength(), + bytes, getFamilyOffset(), getFamilyLength(), + bytes, getQualifierOffset(), getQualifierLength(), + ts, Type.Maximum, bytes, getValueOffset(), getValueLength()); + } + + /** + * @param b + * @return A KeyValue made of a byte array that holds the key-only part. + * Needed to convert hfile index members to KeyValues. + */ + public static KeyValue createKeyValueFromKey(final byte [] b) { + return createKeyValueFromKey(b, 0, b.length); + } + + /** + * @param bb + * @return A KeyValue made of a byte buffer that holds the key-only part. + * Needed to convert hfile index members to KeyValues. + */ + public static KeyValue createKeyValueFromKey(final ByteBuffer bb) { + return createKeyValueFromKey(bb.array(), bb.arrayOffset(), bb.limit()); + } + + /** + * @param b + * @param o + * @param l + * @return A KeyValue made of a byte array that holds the key-only part. + * Needed to convert hfile index members to KeyValues. + */ + public static KeyValue createKeyValueFromKey(final byte [] b, final int o, + final int l) { + byte [] newb = new byte[l + ROW_OFFSET]; + System.arraycopy(b, o, newb, ROW_OFFSET, l); + Bytes.putInt(newb, 0, l); + Bytes.putInt(newb, Bytes.SIZEOF_INT, 0); + return new KeyValue(newb); + } + + /** + * Compare key portion of a {@link KeyValue} for keys in -ROOT- + * table. + */ + public static class RootKeyComparator extends MetaKeyComparator { + public int compareRows(byte [] left, int loffset, int llength, + byte [] right, int roffset, int rlength) { + // Rows look like this: .META.,ROW_FROM_META,RID + // LOG.info("ROOT " + Bytes.toString(left, loffset, llength) + + // "---" + Bytes.toString(right, roffset, rlength)); + final int metalength = 7; // '.META.' length + int lmetaOffsetPlusDelimiter = loffset + metalength; + int leftFarDelimiter = getDelimiterInReverse(left, + lmetaOffsetPlusDelimiter, + llength - metalength, HRegionInfo.DELIMITER); + int rmetaOffsetPlusDelimiter = roffset + metalength; + int rightFarDelimiter = getDelimiterInReverse(right, + rmetaOffsetPlusDelimiter, rlength - metalength, + HRegionInfo.DELIMITER); + if (leftFarDelimiter < 0 && rightFarDelimiter >= 0) { + // Nothing between .META. and regionid. Its first key. + return -1; + } else if (rightFarDelimiter < 0 && leftFarDelimiter >= 0) { + return 1; + } else if (leftFarDelimiter < 0 && rightFarDelimiter < 0) { + return 0; + } + int result = super.compareRows(left, lmetaOffsetPlusDelimiter, + leftFarDelimiter - lmetaOffsetPlusDelimiter, + right, rmetaOffsetPlusDelimiter, + rightFarDelimiter - rmetaOffsetPlusDelimiter); + if (result != 0) { + return result; + } + // Compare last part of row, the rowid. + leftFarDelimiter++; + rightFarDelimiter++; + result = compareRowid(left, leftFarDelimiter, + llength - (leftFarDelimiter - loffset), + right, rightFarDelimiter, rlength - (rightFarDelimiter - roffset)); + return result; + } + } + + /** + * Comparator that compares row component only of a KeyValue. + */ + public static class RowComparator implements Comparator { + final KVComparator comparator; + + public RowComparator(final KVComparator c) { + this.comparator = c; + } + + public int compare(KeyValue left, KeyValue right) { + return comparator.compareRows(left, right); + } + } + + /** + * Compare key portion of a {@link KeyValue} for keys in .META. + * table. + */ + public static class MetaKeyComparator extends KeyComparator { + public int compareRows(byte [] left, int loffset, int llength, + byte [] right, int roffset, int rlength) { + // LOG.info("META " + Bytes.toString(left, loffset, llength) + + // "---" + Bytes.toString(right, roffset, rlength)); + int leftDelimiter = getDelimiter(left, loffset, llength, + HRegionInfo.DELIMITER); + int rightDelimiter = getDelimiter(right, roffset, rlength, + HRegionInfo.DELIMITER); + if (leftDelimiter < 0 && rightDelimiter >= 0) { + // Nothing between .META. and regionid. Its first key. + return -1; + } else if (rightDelimiter < 0 && leftDelimiter >= 0) { + return 1; + } else if (leftDelimiter < 0 && rightDelimiter < 0) { + return 0; + } + // Compare up to the delimiter + int result = Bytes.compareTo(left, loffset, leftDelimiter - loffset, + right, roffset, rightDelimiter - roffset); + if (result != 0) { + return result; + } + // Compare middle bit of the row. + // Move past delimiter + leftDelimiter++; + rightDelimiter++; + int leftFarDelimiter = getRequiredDelimiterInReverse(left, leftDelimiter, + llength - (leftDelimiter - loffset), HRegionInfo.DELIMITER); + int rightFarDelimiter = getRequiredDelimiterInReverse(right, + rightDelimiter, rlength - (rightDelimiter - roffset), + HRegionInfo.DELIMITER); + // Now compare middlesection of row. + result = super.compareRows(left, leftDelimiter, + leftFarDelimiter - leftDelimiter, right, rightDelimiter, + rightFarDelimiter - rightDelimiter); + if (result != 0) { + return result; + } + // Compare last part of row, the rowid. + leftFarDelimiter++; + rightFarDelimiter++; + result = compareRowid(left, leftFarDelimiter, + llength - (leftFarDelimiter - loffset), + right, rightFarDelimiter, rlength - (rightFarDelimiter - roffset)); + return result; + } + + protected int compareRowid(byte[] left, int loffset, int llength, + byte[] right, int roffset, int rlength) { + return Bytes.compareTo(left, loffset, llength, right, roffset, rlength); + } + } + + /** + * Avoids redundant comparisons for better performance. + */ + public static interface SamePrefixComparator { + /** + * Compare two keys assuming that the first n bytes are the same. + * @param commonPrefix How many bytes are the same. + */ + public int compareIgnoringPrefix(int commonPrefix, + T left, int loffset, int llength, + T right, int roffset, int rlength); + } + + /** + * Compare key portion of a {@link KeyValue}. + */ + public static class KeyComparator + implements RawComparator, SamePrefixComparator { + volatile boolean ignoreTimestamp = false; + volatile boolean ignoreType = false; + + public int compare(byte[] left, int loffset, int llength, byte[] right, + int roffset, int rlength) { + // Compare row + short lrowlength = Bytes.toShort(left, loffset); + short rrowlength = Bytes.toShort(right, roffset); + int compare = compareRows(left, loffset + Bytes.SIZEOF_SHORT, + lrowlength, right, roffset + Bytes.SIZEOF_SHORT, rrowlength); + if (compare != 0) { + return compare; + } + + // Compare the rest of the two KVs without making any assumptions about + // the common prefix. This function will not compare rows anyway, so we + // don't need to tell it that the common prefix includes the row. + return compareWithoutRow(0, left, loffset, llength, right, roffset, + rlength, rrowlength); + } + + /** + * Compare the two key-values, ignoring the prefix of the given length + * that is known to be the same between the two. + * @param commonPrefix the prefix length to ignore + */ + @Override + public int compareIgnoringPrefix(int commonPrefix, byte[] left, + int loffset, int llength, byte[] right, int roffset, int rlength) { + // Compare row + short lrowlength = Bytes.toShort(left, loffset); + short rrowlength; + + int comparisonResult = 0; + if (commonPrefix < ROW_LENGTH_SIZE) { + // almost nothing in common + rrowlength = Bytes.toShort(right, roffset); + comparisonResult = compareRows(left, loffset + ROW_LENGTH_SIZE, + lrowlength, right, roffset + ROW_LENGTH_SIZE, rrowlength); + } else { // the row length is the same + rrowlength = lrowlength; + if (commonPrefix < ROW_LENGTH_SIZE + rrowlength) { + // The rows are not the same. Exclude the common prefix and compare + // the rest of the two rows. + int common = commonPrefix - ROW_LENGTH_SIZE; + comparisonResult = compareRows( + left, loffset + common + ROW_LENGTH_SIZE, lrowlength - common, + right, roffset + common + ROW_LENGTH_SIZE, rrowlength - common); + } + } + if (comparisonResult != 0) { + return comparisonResult; + } + + assert lrowlength == rrowlength; + + return compareWithoutRow(commonPrefix, left, loffset, llength, right, + roffset, rlength, lrowlength); + } + + /** + * Compare columnFamily, qualifier, timestamp, and key type (everything + * except the row). This method is used both in the normal comparator and + * the "same-prefix" comparator. Note that we are assuming that row portions + * of both KVs have already been parsed and found identical, and we don't + * validate that assumption here. + * @param commonPrefix + * the length of the common prefix of the two key-values being + * compared, including row length and row + */ + private int compareWithoutRow(int commonPrefix, byte[] left, int loffset, + int llength, byte[] right, int roffset, int rlength, short rowlength) { + /*** + * KeyValue Format and commonLength: + * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|.... + * ------------------|-------commonLength--------|-------------- + */ + int commonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + rowlength; + + // commonLength + TIMESTAMP_TYPE_SIZE + int commonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + commonLength; + // ColumnFamily + Qualifier length. + int lcolumnlength = llength - commonLengthWithTSAndType; + int rcolumnlength = rlength - commonLengthWithTSAndType; + + byte ltype = left[loffset + (llength - 1)]; + byte rtype = right[roffset + (rlength - 1)]; + + // If the column is not specified, the "minimum" key type appears the + // latest in the sorted order, regardless of the timestamp. This is used + // for specifying the last key/value in a given row, because there is no + // "lexicographically last column" (it would be infinitely long). The + // "maximum" key type does not need this behavior. + if (lcolumnlength == 0 && ltype == Type.Minimum.getCode()) { + // left is "bigger", i.e. it appears later in the sorted order + return 1; + } + if (rcolumnlength == 0 && rtype == Type.Minimum.getCode()) { + return -1; + } + + int lfamilyoffset = commonLength + loffset; + int rfamilyoffset = commonLength + roffset; + + // Column family length. + int lfamilylength = left[lfamilyoffset - 1]; + int rfamilylength = right[rfamilyoffset - 1]; + // If left family size is not equal to right family size, we need not + // compare the qualifiers. + boolean sameFamilySize = (lfamilylength == rfamilylength); + int common = 0; + if (commonPrefix > 0) { + common = Math.max(0, commonPrefix - commonLength); + if (!sameFamilySize) { + // Common should not be larger than Math.min(lfamilylength, + // rfamilylength). + common = Math.min(common, Math.min(lfamilylength, rfamilylength)); + } else { + common = Math.min(common, Math.min(lcolumnlength, rcolumnlength)); + } + } + if (!sameFamilySize) { + // comparing column family is enough. + return Bytes.compareTo(left, lfamilyoffset + common, lfamilylength + - common, right, rfamilyoffset + common, rfamilylength - common); + } + // Compare family & qualifier together. + final int comparison = Bytes.compareTo(left, lfamilyoffset + common, + lcolumnlength - common, right, rfamilyoffset + common, + rcolumnlength - common); + if (comparison != 0) { + return comparison; + } + return compareTimestampAndType(left, loffset, llength, right, roffset, + rlength, ltype, rtype); + } + + private int compareTimestampAndType(byte[] left, int loffset, int llength, + byte[] right, int roffset, int rlength, byte ltype, byte rtype) { + int compare; + if (!this.ignoreTimestamp) { + // Get timestamps. + long ltimestamp = Bytes.toLong(left, + loffset + (llength - TIMESTAMP_TYPE_SIZE)); + long rtimestamp = Bytes.toLong(right, + roffset + (rlength - TIMESTAMP_TYPE_SIZE)); + compare = compareTimestamps(ltimestamp, rtimestamp); + if (compare != 0) { + return compare; + } + } + + if (!this.ignoreType) { + // Compare types. Let the delete types sort ahead of puts; i.e. types + // of higher numbers sort before those of lesser numbers. Maximum (255) + // appears ahead of everything, and minimum (0) appears after + // everything. + return (0xff & rtype) - (0xff & ltype); + } + return 0; + } + + public int compare(byte[] left, byte[] right) { + return compare(left, 0, left.length, right, 0, right.length); + } + + public int compareRows(byte [] left, int loffset, int llength, + byte [] right, int roffset, int rlength) { + return Bytes.compareTo(left, loffset, llength, right, roffset, rlength); + } + + protected int compareColumns( + byte [] left, int loffset, int llength, final int lfamilylength, + byte [] right, int roffset, int rlength, final int rfamilylength) { + return KeyValue.compareColumns(left, loffset, llength, lfamilylength, + right, roffset, rlength, rfamilylength); + } + + int compareTimestamps(final long ltimestamp, final long rtimestamp) { + // The below older timestamps sorting ahead of newer timestamps looks + // wrong but it is intentional. This way, newer timestamps are first + // found when we iterate over a memstore and newer versions are the + // first we trip over when reading from a store file. + if (ltimestamp < rtimestamp) { + return 1; + } else if (ltimestamp > rtimestamp) { + return -1; + } + return 0; + } + } + + // HeapSize + public long heapSize() { + return ClassSize.align(ClassSize.OBJECT + ClassSize.REFERENCE + + ClassSize.align(ClassSize.ARRAY) + ClassSize.align(length) + + (3 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_LONG); + } + + // this overload assumes that the length bytes have already been read, + // and it expects the length of the KeyValue to be explicitly passed + // to it. + public void readFields(int length, final DataInput in) throws IOException { + this.length = length; + this.offset = 0; + this.keyLength = 0; + this.bytes = new byte[this.length]; + in.readFully(this.bytes, 0, this.length); + } + + // Writable + public void readFields(final DataInput in) throws IOException { + int length = in.readInt(); + readFields(length, in); + } + + public void write(final DataOutput out) throws IOException { + out.writeInt(this.length); + out.write(this.bytes, this.offset, this.length); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java new file mode 100644 index 0000000..fe4897a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java @@ -0,0 +1,454 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.util.Threads; + +import java.util.concurrent.CopyOnWriteArrayList; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.util.JVMClusterUtil; + +/** + * This class creates a single process HBase cluster. One thread is created for + * a master and one per region server. + * + * Call {@link #startup()} to start the cluster running and {@link #shutdown()} + * to close it all down. {@link #join} the cluster is you want to wait on + * shutdown completion. + * + *

Runs master on port 60000 by default. Because we can't just kill the + * process -- not till HADOOP-1700 gets fixed and even then.... -- we need to + * be able to find the master with a remote client to run shutdown. To use a + * port other than 60000, set the hbase.master to a value of 'local:PORT': + * that is 'local', not 'localhost', and the port number the master should use + * instead of 60000. + * + */ +public class LocalHBaseCluster { + static final Log LOG = LogFactory.getLog(LocalHBaseCluster.class); + private final List masterThreads = + new CopyOnWriteArrayList(); + private final List regionThreads = + new CopyOnWriteArrayList(); + private final static int DEFAULT_NO = 1; + /** local mode */ + public static final String LOCAL = "local"; + /** 'local:' */ + public static final String LOCAL_COLON = LOCAL + ":"; + private final Configuration conf; + private final Class masterClass; + private final Class regionServerClass; + + /** + * Constructor. + * @param conf + * @throws IOException + */ + public LocalHBaseCluster(final Configuration conf) + throws IOException { + this(conf, DEFAULT_NO); + } + + /** + * Constructor. + * @param conf Configuration to use. Post construction has the master's + * address. + * @param noRegionServers Count of regionservers to start. + * @throws IOException + */ + public LocalHBaseCluster(final Configuration conf, final int noRegionServers) + throws IOException { + this(conf, 1, noRegionServers, getMasterImplementation(conf), + getRegionServerImplementation(conf)); + } + + /** + * Constructor. + * @param conf Configuration to use. Post construction has the active master + * address. + * @param noMasters Count of masters to start. + * @param noRegionServers Count of regionservers to start. + * @throws IOException + */ + public LocalHBaseCluster(final Configuration conf, final int noMasters, + final int noRegionServers) + throws IOException { + this(conf, noMasters, noRegionServers, getMasterImplementation(conf), + getRegionServerImplementation(conf)); + } + + @SuppressWarnings("unchecked") + private static Class getRegionServerImplementation(final Configuration conf) { + return (Class)conf.getClass(HConstants.REGION_SERVER_IMPL, + HRegionServer.class); + } + + @SuppressWarnings("unchecked") + private static Class getMasterImplementation(final Configuration conf) { + return (Class)conf.getClass(HConstants.MASTER_IMPL, + HMaster.class); + } + + /** + * Constructor. + * @param conf Configuration to use. Post construction has the master's + * address. + * @param noMasters Count of masters to start. + * @param noRegionServers Count of regionservers to start. + * @param masterClass + * @param regionServerClass + * @throws IOException + */ + @SuppressWarnings("unchecked") + public LocalHBaseCluster(final Configuration conf, final int noMasters, + final int noRegionServers, final Class masterClass, + final Class regionServerClass) + throws IOException { + this.conf = conf; + // Always have masters and regionservers come up on port '0' so we don't + // clash over default ports. + conf.set(HConstants.MASTER_PORT, "0"); + conf.set(HConstants.REGIONSERVER_PORT, "0"); + this.masterClass = (Class) + conf.getClass(HConstants.MASTER_IMPL, masterClass); + // Start the HMasters. + for (int i = 0; i < noMasters; i++) { + addMaster(new Configuration(conf), i); + } + // Start the HRegionServers. + this.regionServerClass = + (Class)conf.getClass(HConstants.REGION_SERVER_IMPL, + regionServerClass); + + for (int i = 0; i < noRegionServers; i++) { + addRegionServer(new Configuration(conf), i); + } + } + + public JVMClusterUtil.RegionServerThread addRegionServer() + throws IOException { + return addRegionServer(new Configuration(conf), this.regionThreads.size()); + } + + public JVMClusterUtil.RegionServerThread addRegionServer( + Configuration config, final int index) + throws IOException { + // Create each regionserver with its own Configuration instance so each has + // its HConnection instance rather than share (see HBASE_INSTANCES down in + // the guts of HConnectionManager. + JVMClusterUtil.RegionServerThread rst = + JVMClusterUtil.createRegionServerThread(config, + this.regionServerClass, index); + this.regionThreads.add(rst); + return rst; + } + + public JVMClusterUtil.RegionServerThread addRegionServer( + final Configuration config, final int index, User user) + throws IOException, InterruptedException { + return user.runAs( + new PrivilegedExceptionAction() { + public JVMClusterUtil.RegionServerThread run() throws Exception { + return addRegionServer(config, index); + } + }); + } + + public JVMClusterUtil.MasterThread addMaster() throws IOException { + return addMaster(new Configuration(conf), this.masterThreads.size()); + } + + public JVMClusterUtil.MasterThread addMaster(Configuration c, final int index) + throws IOException { + // Create each master with its own Configuration instance so each has + // its HConnection instance rather than share (see HBASE_INSTANCES down in + // the guts of HConnectionManager. + JVMClusterUtil.MasterThread mt = JVMClusterUtil.createMasterThread(c, + (Class) conf.getClass(HConstants.MASTER_IMPL, this.masterClass), index); + this.masterThreads.add(mt); + return mt; + } + + public JVMClusterUtil.MasterThread addMaster( + final Configuration c, final int index, User user) + throws IOException, InterruptedException { + return user.runAs( + new PrivilegedExceptionAction() { + public JVMClusterUtil.MasterThread run() throws Exception { + return addMaster(c, index); + } + }); + } + + /** + * @param serverNumber + * @return region server + */ + public HRegionServer getRegionServer(int serverNumber) { + return regionThreads.get(serverNumber).getRegionServer(); + } + + /** + * @return Read-only list of region server threads. + */ + public List getRegionServers() { + return Collections.unmodifiableList(this.regionThreads); + } + + /** + * @return List of running servers (Some servers may have been killed or + * aborted during lifetime of cluster; these servers are not included in this + * list). + */ + public List getLiveRegionServers() { + List liveServers = + new ArrayList(); + List list = getRegionServers(); + for (JVMClusterUtil.RegionServerThread rst: list) { + if (rst.isAlive()) liveServers.add(rst); + else LOG.info("Not alive " + rst.getName()); + } + return liveServers; + } + + /** + * Wait for the specified region server to stop + * Removes this thread from list of running threads. + * @param serverNumber + * @return Name of region server that just went down. + */ + public String waitOnRegionServer(int serverNumber) { + JVMClusterUtil.RegionServerThread regionServerThread = + this.regionThreads.remove(serverNumber); + while (regionServerThread.isAlive()) { + try { + LOG.info("Waiting on " + + regionServerThread.getRegionServer().toString()); + regionServerThread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + return regionServerThread.getName(); + } + + /** + * Wait for the specified region server to stop + * Removes this thread from list of running threads. + * @param rst + * @return Name of region server that just went down. + */ + public String waitOnRegionServer(JVMClusterUtil.RegionServerThread rst) { + while (rst.isAlive()) { + try { + LOG.info("Waiting on " + + rst.getRegionServer().toString()); + rst.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + for (int i=0;i getMasters() { + return Collections.unmodifiableList(this.masterThreads); + } + + /** + * @return List of running master servers (Some servers may have been killed + * or aborted during lifetime of cluster; these servers are not included in + * this list). + */ + public List getLiveMasters() { + List liveServers = + new ArrayList(); + List list = getMasters(); + for (JVMClusterUtil.MasterThread mt: list) { + if (mt.isAlive()) { + liveServers.add(mt); + } + } + return liveServers; + } + + /** + * Wait for the specified master to stop + * Removes this thread from list of running threads. + * @param serverNumber + * @return Name of master that just went down. + */ + public String waitOnMaster(int serverNumber) { + JVMClusterUtil.MasterThread masterThread = + this.masterThreads.remove(serverNumber); + while (masterThread.isAlive()) { + try { + LOG.info("Waiting on " + + masterThread.getMaster().getServerName().toString()); + masterThread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + return masterThread.getName(); + } + + /** + * Wait for the specified master to stop + * Removes this thread from list of running threads. + * @param masterThread + * @return Name of master that just went down. + */ + public String waitOnMaster(JVMClusterUtil.MasterThread masterThread) { + while (masterThread.isAlive()) { + try { + LOG.info("Waiting on " + + masterThread.getMaster().getServerName().toString()); + masterThread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + for (int i=0;i + * Listens for ZooKeeper events related to the master address. The node + * /master will contain the address of the current master. + * This listener is interested in + * NodeDeleted and NodeCreated events on + * /master. + *

+ * Utilizes {@link ZooKeeperNodeTracker} for zk interactions. + *

+ * You can get the current master via {@link #getMasterAddress()} + */ +public class MasterAddressTracker extends ZooKeeperNodeTracker { + /** + * Construct a master address listener with the specified + * zookeeper reference. + *

+ * This constructor does not trigger any actions, you must call methods + * explicitly. Normally you will just want to execute {@link #start()} to + * begin tracking of the master address. + * + * @param watcher zk reference and watcher + * @param abortable abortable in case of fatal error + */ + public MasterAddressTracker(ZooKeeperWatcher watcher, Abortable abortable) { + super(watcher, watcher.masterAddressZNode, abortable); + } + + /** + * Get the address of the current master if one is available. Returns null + * if no current master. + * @return Server name or null if timed out. + */ + public ServerName getMasterAddress() { + return bytesToServerName(super.getData(false)); + } + + /** + * Check if there is a master available. + * @return true if there is a master set, false if not. + */ + public boolean hasMaster() { + return super.getData(false) != null; + } + + /** + * @param bytes Byte array of {@link ServerName#toString()} + * @return A {@link ServerName} instance. + */ + private ServerName bytesToServerName(final byte [] bytes) { + return bytes == null ? null: ServerName.parseVersionedServerName(bytes); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/MasterNotRunningException.java b/src/main/java/org/apache/hadoop/hbase/MasterNotRunningException.java new file mode 100644 index 0000000..6cf564c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/MasterNotRunningException.java @@ -0,0 +1,49 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +/** + * Thrown if the master is not running + */ +public class MasterNotRunningException extends IOException { + private static final long serialVersionUID = 1L << 23 - 1L; + /** default constructor */ + public MasterNotRunningException() { + super(); + } + + /** + * Constructor + * @param s message + */ + public MasterNotRunningException(String s) { + super(s); + } + + /** + * Constructor taking another exception. + * @param e Exception to grab data from. + */ + public MasterNotRunningException(Exception e) { + super(e); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/NotAllMetaRegionsOnlineException.java b/src/main/java/org/apache/hadoop/hbase/NotAllMetaRegionsOnlineException.java new file mode 100644 index 0000000..2c275e3 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/NotAllMetaRegionsOnlineException.java @@ -0,0 +1,43 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.DoNotRetryIOException; + +/** + * Thrown when an operation requires the root and all meta regions to be online + */ +public class NotAllMetaRegionsOnlineException extends DoNotRetryIOException { + private static final long serialVersionUID = 6439786157874827523L; + /** + * default constructor + */ + public NotAllMetaRegionsOnlineException() { + super(); + } + + /** + * @param message + */ + public NotAllMetaRegionsOnlineException(String message) { + super(message); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/NotServingRegionException.java b/src/main/java/org/apache/hadoop/hbase/NotServingRegionException.java new file mode 100644 index 0000000..32da8cb --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/NotServingRegionException.java @@ -0,0 +1,53 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Thrown by a region server if it is sent a request for a region it is not + * serving. + */ +public class NotServingRegionException extends IOException { + private static final long serialVersionUID = 1L << 17 - 1L; + + /** default constructor */ + public NotServingRegionException() { + super(); + } + + /** + * Constructor + * @param s message + */ + public NotServingRegionException(String s) { + super(s); + } + + /** + * Constructor + * @param s message + */ + public NotServingRegionException(final byte [] s) { + super(Bytes.toString(s)); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java b/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java new file mode 100644 index 0000000..309dca4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java @@ -0,0 +1,35 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +/** + * This exception is thrown by the master when a region server was shut down and + * restarted so fast that the master still hasn't processed the server shutdown + * of the first instance, or when master is initializing and client call admin + * operations + */ +@SuppressWarnings("serial") +public class PleaseHoldException extends IOException { + public PleaseHoldException(String message) { + super(message); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/RegionException.java b/src/main/java/org/apache/hadoop/hbase/RegionException.java new file mode 100644 index 0000000..63063a5 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/RegionException.java @@ -0,0 +1,43 @@ +/** + * Copyright 2008 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +/** + * Thrown when something happens related to region handling. + * Subclasses have to be more specific. + */ +public class RegionException extends IOException { + private static final long serialVersionUID = 1473510258071111371L; + + /** default constructor */ + public RegionException() { + super(); + } + + /** + * Constructor + * @param s message + */ + public RegionException(String s) { + super(s); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java b/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java new file mode 100644 index 0000000..f5217bc --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Thrown by a region server if it will block and wait to serve a request. + * For example, the client wants to insert something to a region while the + * region is compacting. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class RegionTooBusyException extends IOException { + private static final long serialVersionUID = 1728345723728342L; + + /** default constructor */ + public RegionTooBusyException() { + super(); + } + + /** + * Constructor + * @param msg message + */ + public RegionTooBusyException(final String msg) { + super(msg); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/RemoteExceptionHandler.java b/src/main/java/org/apache/hadoop/hbase/RemoteExceptionHandler.java new file mode 100644 index 0000000..485c254 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/RemoteExceptionHandler.java @@ -0,0 +1,120 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +import org.apache.hadoop.ipc.RemoteException; + +/** + * An immutable class which contains a static method for handling + * org.apache.hadoop.ipc.RemoteException exceptions. + */ +public class RemoteExceptionHandler { + /* Not instantiable */ + private RemoteExceptionHandler() {super();} + + /** + * Examine passed Throwable. See if its carrying a RemoteException. If so, + * run {@link #decodeRemoteException(RemoteException)} on it. Otherwise, + * pass back t unaltered. + * @param t Throwable to examine. + * @return Decoded RemoteException carried by t or + * t unaltered. + */ + public static Throwable checkThrowable(final Throwable t) { + Throwable result = t; + if (t instanceof RemoteException) { + try { + result = + RemoteExceptionHandler.decodeRemoteException((RemoteException)t); + } catch (Throwable tt) { + result = tt; + } + } + return result; + } + + /** + * Examine passed IOException. See if its carrying a RemoteException. If so, + * run {@link #decodeRemoteException(RemoteException)} on it. Otherwise, + * pass back e unaltered. + * @param e Exception to examine. + * @return Decoded RemoteException carried by e or + * e unaltered. + */ + public static IOException checkIOException(final IOException e) { + Throwable t = checkThrowable(e); + return t instanceof IOException? (IOException)t: new IOException(t); + } + + /** + * Converts org.apache.hadoop.ipc.RemoteException into original exception, + * if possible. If the original exception is an Error or a RuntimeException, + * throws the original exception. + * + * @param re original exception + * @return decoded RemoteException if it is an instance of or a subclass of + * IOException, or the original RemoteException if it cannot be decoded. + * + * @throws IOException indicating a server error ocurred if the decoded + * exception is not an IOException. The decoded exception is set as + * the cause. + * @deprecated Use {@link RemoteException#unwrapRemoteException()} instead. + * In fact we should look into deprecating this whole class - St.Ack 2010929 + */ + public static IOException decodeRemoteException(final RemoteException re) + throws IOException { + IOException i = re; + + try { + Class c = Class.forName(re.getClassName()); + + Class[] parameterTypes = { String.class }; + Constructor ctor = c.getConstructor(parameterTypes); + + Object[] arguments = { re.getMessage() }; + Throwable t = (Throwable) ctor.newInstance(arguments); + + if (t instanceof IOException) { + i = (IOException) t; + + } else { + i = new IOException("server error"); + i.initCause(t); + throw i; + } + + } catch (ClassNotFoundException x) { + // continue + } catch (NoSuchMethodException x) { + // continue + } catch (IllegalAccessException x) { + // continue + } catch (InvocationTargetException x) { + // continue + } catch (InstantiationException x) { + // continue + } + return i; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/Server.java b/src/main/java/org/apache/hadoop/hbase/Server.java new file mode 100644 index 0000000..de19e2c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/Server.java @@ -0,0 +1,50 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; + +/** + * Defines the set of shared functions implemented by HBase servers (Masters + * and RegionServers). + */ +public interface Server extends Abortable, Stoppable { + /** + * Gets the configuration object for this server. + */ + public Configuration getConfiguration(); + + /** + * Gets the ZooKeeper instance for this server. + */ + public ZooKeeperWatcher getZooKeeper(); + + /** + * @return Master's instance of {@link CatalogTracker} + */ + public CatalogTracker getCatalogTracker(); + + /** + * @return The unique server name for this server. + */ + public ServerName getServerName(); +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/ServerName.java b/src/main/java/org/apache/hadoop/hbase/ServerName.java new file mode 100644 index 0000000..2d6fe4e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/ServerName.java @@ -0,0 +1,295 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.Collection; +import java.util.regex.Pattern; + +import org.apache.hadoop.hbase.util.Addressing; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Instance of an HBase ServerName. + * A server name is used uniquely identifying a server instance and is made + * of the combination of hostname, port, and startcode. The startcode + * distingushes restarted servers on same hostname and port (startcode is + * usually timestamp of server startup). The {@link #toString()} format of + * ServerName is safe to use in the filesystem and as znode name up in + * ZooKeeper. Its format is: + * <hostname> '{@link #SERVERNAME_SEPARATOR}' <port> '{@link #SERVERNAME_SEPARATOR}' <startcode>. + * For example, if hostname is example.org, port is 1234, + * and the startcode for the regionserver is 1212121212, then + * the {@link #toString()} would be example.org,1234,1212121212. + * + *

You can obtain a versioned serialized form of this class by calling + * {@link #getVersionedBytes()}. To deserialize, call {@link #parseVersionedServerName(byte[])} + * + *

Immutable. + */ +public class ServerName implements Comparable { + /** + * Version for this class. + * Its a short rather than a byte so I can for sure distinguish between this + * version of this class and the version previous to this which did not have + * a version. + */ + private static final short VERSION = 0; + static final byte [] VERSION_BYTES = Bytes.toBytes(VERSION); + + /** + * What to use if no startcode supplied. + */ + public static final int NON_STARTCODE = -1; + + /** + * This character is used as separator between server hostname, port and + * startcode. + */ + public static final String SERVERNAME_SEPARATOR = ","; + + public static Pattern SERVERNAME_PATTERN = + Pattern.compile("[^" + SERVERNAME_SEPARATOR + "]+" + + SERVERNAME_SEPARATOR + Addressing.VALID_PORT_REGEX + + SERVERNAME_SEPARATOR + Addressing.VALID_PORT_REGEX + "$"); + + /** + * What to use if server name is unknown. + */ + public static final String UNKNOWN_SERVERNAME = "#unknown#"; + + private final String servername; + private final String hostname; + private final int port; + private final long startcode; + + /** + * Cached versioned bytes of this ServerName instance. + * @see #getVersionedBytes() + */ + private byte [] bytes; + + public ServerName(final String hostname, final int port, final long startcode) { + this.hostname = hostname; + this.port = port; + this.startcode = startcode; + this.servername = getServerName(hostname, port, startcode); + } + + public ServerName(final String serverName) { + this(parseHostname(serverName), parsePort(serverName), + parseStartcode(serverName)); + } + + public ServerName(final String hostAndPort, final long startCode) { + this(Addressing.parseHostname(hostAndPort), + Addressing.parsePort(hostAndPort), startCode); + } + + public static String parseHostname(final String serverName) { + if (serverName == null || serverName.length() <= 0) { + throw new IllegalArgumentException("Passed hostname is null or empty"); + } + int index = serverName.indexOf(SERVERNAME_SEPARATOR); + return serverName.substring(0, index); + } + + public static int parsePort(final String serverName) { + String [] split = serverName.split(SERVERNAME_SEPARATOR); + return Integer.parseInt(split[1]); + } + + public static long parseStartcode(final String serverName) { + int index = serverName.lastIndexOf(SERVERNAME_SEPARATOR); + return Long.parseLong(serverName.substring(index + 1)); + } + + @Override + public String toString() { + return getServerName(); + } + + /** + * @return {@link #getServerName()} as bytes with a short-sized prefix with + * the ServerName#VERSION of this class. + */ + public synchronized byte [] getVersionedBytes() { + if (this.bytes == null) { + this.bytes = Bytes.add(VERSION_BYTES, Bytes.toBytes(getServerName())); + } + return this.bytes; + } + + public String getServerName() { + return servername; + } + + public String getHostname() { + return hostname; + } + + public int getPort() { + return port; + } + + public long getStartcode() { + return startcode; + } + + /** + * @param hostName + * @param port + * @param startcode + * @return Server name made of the concatenation of hostname, port and + * startcode formatted as <hostname> ',' <port> ',' <startcode> + */ + public static String getServerName(String hostName, int port, long startcode) { + final StringBuilder name = new StringBuilder(hostName.length() + 1 + 5 + 1 + 13); + name.append(hostName); + name.append(SERVERNAME_SEPARATOR); + name.append(port); + name.append(SERVERNAME_SEPARATOR); + name.append(startcode); + return name.toString(); + } + + /** + * @param hostAndPort String in form of <hostname> ':' <port> + * @param startcode + * @return Server name made of the concatenation of hostname, port and + * startcode formatted as <hostname> ',' <port> ',' <startcode> + */ + public static String getServerName(final String hostAndPort, + final long startcode) { + int index = hostAndPort.indexOf(":"); + if (index <= 0) throw new IllegalArgumentException("Expected ':' "); + return getServerName(hostAndPort.substring(0, index), + Integer.parseInt(hostAndPort.substring(index + 1)), startcode); + } + + /** + * @return Hostname and port formatted as described at + * {@link Addressing#createHostAndPortStr(String, int)} + */ + public String getHostAndPort() { + return Addressing.createHostAndPortStr(this.hostname, this.port); + } + + /** + * @param serverName ServerName in form specified by {@link #getServerName()} + * @return The server start code parsed from servername + */ + public static long getServerStartcodeFromServerName(final String serverName) { + int index = serverName.lastIndexOf(SERVERNAME_SEPARATOR); + return Long.parseLong(serverName.substring(index + 1)); + } + + /** + * Utility method to excise the start code from a server name + * @param inServerName full server name + * @return server name less its start code + */ + public static String getServerNameLessStartCode(String inServerName) { + if (inServerName != null && inServerName.length() > 0) { + int index = inServerName.lastIndexOf(SERVERNAME_SEPARATOR); + if (index > 0) { + return inServerName.substring(0, index); + } + } + return inServerName; + } + + @Override + public int compareTo(ServerName other) { + int compare = this.getHostname().toLowerCase(). + compareTo(other.getHostname().toLowerCase()); + if (compare != 0) return compare; + compare = this.getPort() - other.getPort(); + if (compare != 0) return compare; + return (int)(this.getStartcode() - other.getStartcode()); + } + + @Override + public int hashCode() { + return getServerName().hashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null) return false; + if (!(o instanceof ServerName)) return false; + return this.compareTo((ServerName)o) == 0; + } + + + /** + * @return ServerName with matching hostname and port. + */ + public static ServerName findServerWithSameHostnamePort(final Collection names, + final ServerName serverName) { + for (ServerName sn: names) { + if (isSameHostnameAndPort(serverName, sn)) return sn; + } + return null; + } + + /** + * @param left + * @param right + * @return True if other has same hostname and port. + */ + public static boolean isSameHostnameAndPort(final ServerName left, + final ServerName right) { + if (left == null) return false; + if (right == null) return false; + return left.getHostname().equals(right.getHostname()) && + left.getPort() == right.getPort(); + } + + /** + * Use this method instantiating a {@link ServerName} from bytes + * gotten from a call to {@link #getVersionedBytes()}. Will take care of the + * case where bytes were written by an earlier version of hbase. + * @param versionedBytes Pass bytes gotten from a call to {@link #getVersionedBytes()} + * @return A ServerName instance. + * @see #getVersionedBytes() + */ + public static ServerName parseVersionedServerName(final byte [] versionedBytes) { + // Version is a short. + short version = Bytes.toShort(versionedBytes); + if (version == VERSION) { + int length = versionedBytes.length - Bytes.SIZEOF_SHORT; + return new ServerName(Bytes.toString(versionedBytes, Bytes.SIZEOF_SHORT, length)); + } + // Presume the bytes were written with an old version of hbase and that the + // bytes are actually a String of the form "'' ':' ''". + return new ServerName(Bytes.toString(versionedBytes), NON_STARTCODE); + } + + /** + * @param str Either an instance of {@link ServerName#toString()} or a + * "'' ':' ''". + * @return A ServerName instance. + */ + public static ServerName parseServerName(final String str) { + return SERVERNAME_PATTERN.matcher(str).matches()? new ServerName(str): + new ServerName(str, NON_STARTCODE); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/Stoppable.java b/src/main/java/org/apache/hadoop/hbase/Stoppable.java new file mode 100644 index 0000000..74d4f4a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/Stoppable.java @@ -0,0 +1,36 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +/** + * Implementers are Stoppable. + */ +public interface Stoppable { + /** + * Stop this service. + * @param why Why we're stopping. + */ + public void stop(String why); + + /** + * @return True if {@link #stop(String)} has been closed. + */ + public boolean isStopped(); +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java b/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java new file mode 100644 index 0000000..58c211b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.util.Map; + +/** + * Get, remove and modify table descriptors. + * Used by servers to host descriptors. + */ +public interface TableDescriptors { + /** + * @param tablename + * @return HTableDescriptor for tablename + * @throws IOException + */ + public HTableDescriptor get(final String tablename) + throws IOException; + + /** + * @param tablename + * @return HTableDescriptor for tablename + * @throws IOException + */ + public HTableDescriptor get(final byte[] tablename) + throws IOException; + + /** + * Get Map of all HTableDescriptors. Populates the descriptor cache as a + * side effect. + * @return Map of all descriptors. + * @throws IOException + */ + public Map getAll() + throws IOException; + + /** + * Add or update descriptor + * @param htd Descriptor to set into TableDescriptors + * @throws IOException + */ + public void add(final HTableDescriptor htd) + throws IOException; + + /** + * @param tablename + * @return Instance of table descriptor or null if none found. + * @throws IOException + */ + public HTableDescriptor remove(final String tablename) + throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/TableExistsException.java b/src/main/java/org/apache/hadoop/hbase/TableExistsException.java new file mode 100644 index 0000000..5fde219 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/TableExistsException.java @@ -0,0 +1,38 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +/** + * Thrown when a table exists but should not + */ +public class TableExistsException extends IOException { + private static final long serialVersionUID = 1L << 7 - 1L; + /** default constructor */ + public TableExistsException() { + super(); + } + + /** + * Constructor + * + * @param s message + */ + public TableExistsException(String s) { + super(s); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/TableInfoMissingException.java b/src/main/java/org/apache/hadoop/hbase/TableInfoMissingException.java new file mode 100644 index 0000000..2faf198 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/TableInfoMissingException.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +/** + * + * Failed to find .tableinfo file under table dir + * + */ +@SuppressWarnings("serial") +public class TableInfoMissingException extends HBaseIOException { + + public TableInfoMissingException() { + super(); + } + + public TableInfoMissingException( String message ) { + super(message); + } + + public TableInfoMissingException( String message, Throwable t ) { + super(message, t); + } + + public TableInfoMissingException( Throwable t ) { + super(t); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/TableNotDisabledException.java b/src/main/java/org/apache/hadoop/hbase/TableNotDisabledException.java new file mode 100644 index 0000000..4287800 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/TableNotDisabledException.java @@ -0,0 +1,50 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Thrown if a table should be offline but is not + */ +public class TableNotDisabledException extends IOException { + private static final long serialVersionUID = 1L << 19 - 1L; + /** default constructor */ + public TableNotDisabledException() { + super(); + } + + /** + * Constructor + * @param s message + */ + public TableNotDisabledException(String s) { + super(s); + } + + /** + * @param tableName Name of table that is not disabled + */ + public TableNotDisabledException(byte[] tableName) { + this(Bytes.toString(tableName)); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/TableNotEnabledException.java b/src/main/java/org/apache/hadoop/hbase/TableNotEnabledException.java new file mode 100644 index 0000000..d0392d5 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/TableNotEnabledException.java @@ -0,0 +1,50 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Thrown if a table should be enabled but is not + */ +public class TableNotEnabledException extends IOException { + private static final long serialVersionUID = 262144L; + /** default constructor */ + public TableNotEnabledException() { + super(); + } + + /** + * Constructor + * @param s message + */ + public TableNotEnabledException(String s) { + super(s); + } + + /** + * @param tableName Name of table that is not enabled + */ + public TableNotEnabledException(byte[] tableName) { + this(Bytes.toString(tableName)); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/TableNotFoundException.java b/src/main/java/org/apache/hadoop/hbase/TableNotFoundException.java new file mode 100644 index 0000000..dc6da43 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/TableNotFoundException.java @@ -0,0 +1,35 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +/** Thrown when a table can not be located */ +public class TableNotFoundException extends RegionException { + private static final long serialVersionUID = 993179627856392526L; + + /** default constructor */ + public TableNotFoundException() { + super(); + } + + /** @param s message */ + public TableNotFoundException(String s) { + super(s); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/UnknownRegionException.java b/src/main/java/org/apache/hadoop/hbase/UnknownRegionException.java new file mode 100644 index 0000000..e87f42a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/UnknownRegionException.java @@ -0,0 +1,33 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +/** + * Thrown when we are asked to operate on a region we know nothing about. + */ +public class UnknownRegionException extends IOException { + private static final long serialVersionUID = 1968858760475205392L; + + public UnknownRegionException(String regionName) { + super(regionName); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java b/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java new file mode 100644 index 0000000..675e6e2 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/UnknownRowLockException.java @@ -0,0 +1,42 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + + +/** + * Thrown if a region server is passed an unknown row lock id + * @deprecated row locks are deprecated (and thus so our associated exceptions). + */ +public class UnknownRowLockException extends DoNotRetryIOException { + private static final long serialVersionUID = 993179627856392526L; + + /** constructor */ + public UnknownRowLockException() { + super(); + } + + /** + * Constructor + * @param s message + */ + public UnknownRowLockException(String s) { + super(s); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/UnknownScannerException.java b/src/main/java/org/apache/hadoop/hbase/UnknownScannerException.java new file mode 100644 index 0000000..13f2f6c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/UnknownScannerException.java @@ -0,0 +1,44 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + + +/** + * Thrown if a region server is passed an unknown scanner id. + * Usually means the client has take too long between checkins and so the + * scanner lease on the serverside has expired OR the serverside is closing + * down and has cancelled all leases. + */ +public class UnknownScannerException extends DoNotRetryIOException { + private static final long serialVersionUID = 993179627856392526L; + + /** constructor */ + public UnknownScannerException() { + super(); + } + + /** + * Constructor + * @param s message + */ + public UnknownScannerException(String s) { + super(s); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/VersionAnnotation.java b/src/main/java/org/apache/hadoop/hbase/VersionAnnotation.java new file mode 100644 index 0000000..ecea580 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/VersionAnnotation.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.lang.annotation.*; + +/** + * A package attribute that captures the version of hbase that was compiled. + * Copied down from hadoop. All is same except name of interface. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.PACKAGE) +public @interface VersionAnnotation { + + /** + * Get the Hadoop version + * @return the version string "0.6.3-dev" + */ + String version(); + + /** + * Get the username that compiled Hadoop. + */ + String user(); + + /** + * Get the date when Hadoop was compiled. + * @return the date in unix 'date' format + */ + String date(); + + /** + * Get the url for the subversion repository. + */ + String url(); + + /** + * Get the subversion revision. + * @return the revision number as a string (eg. "451451") + */ + String revision(); +} diff --git a/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java b/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java new file mode 100644 index 0000000..fcd2ccd --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java @@ -0,0 +1,34 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +/** + * This exception is thrown by the master when a region server reports and is + * already being processed as dead. This can happen when a region server loses + * its session but didn't figure it yet. + */ +@SuppressWarnings("serial") +public class YouAreDeadException extends IOException { + public YouAreDeadException(String message) { + super(message); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java b/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java new file mode 100644 index 0000000..ad48b25 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java @@ -0,0 +1,49 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +/** + * Thrown if the client can't connect to zookeeper + */ +public class ZooKeeperConnectionException extends IOException { + private static final long serialVersionUID = 1L << 23 - 1L; + /** default constructor */ + public ZooKeeperConnectionException() { + super(); + } + + /** + * Constructor + * @param s message + */ + public ZooKeeperConnectionException(String s) { + super(s); + } + + /** + * Constructor taking another exception. + * @param e Exception to grab data from. + */ + public ZooKeeperConnectionException(String message, Exception e) { + super(message, e); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java b/src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java new file mode 100644 index 0000000..f45bbbd --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java @@ -0,0 +1,615 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.avro; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericArray; +import org.apache.avro.generic.GenericData; +import org.apache.avro.ipc.HttpServer; +import org.apache.avro.ipc.specific.SpecificResponder; +import org.apache.avro.util.Utf8; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.avro.generated.AClusterStatus; +import org.apache.hadoop.hbase.avro.generated.ADelete; +import org.apache.hadoop.hbase.avro.generated.AFamilyDescriptor; +import org.apache.hadoop.hbase.avro.generated.AGet; +import org.apache.hadoop.hbase.avro.generated.AIOError; +import org.apache.hadoop.hbase.avro.generated.AIllegalArgument; +import org.apache.hadoop.hbase.avro.generated.AMasterNotRunning; +import org.apache.hadoop.hbase.avro.generated.APut; +import org.apache.hadoop.hbase.avro.generated.AResult; +import org.apache.hadoop.hbase.avro.generated.AScan; +import org.apache.hadoop.hbase.avro.generated.ATableDescriptor; +import org.apache.hadoop.hbase.avro.generated.ATableExists; +import org.apache.hadoop.hbase.avro.generated.HBase; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.HTablePool; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Start an Avro server + */ +public class AvroServer { + + /** + * The HBaseImpl is a glue object that connects Avro RPC calls to the + * HBase client API primarily defined in the HBaseAdmin and HTable objects. + */ + public static class HBaseImpl implements HBase { + // + // PROPERTIES + // + protected Configuration conf = null; + protected HBaseAdmin admin = null; + protected HTablePool htablePool = null; + protected final Log LOG = LogFactory.getLog(this.getClass().getName()); + + // nextScannerId and scannerMap are used to manage scanner state + protected int nextScannerId = 0; + protected HashMap scannerMap = null; + + // + // UTILITY METHODS + // + + /** + * Assigns a unique ID to the scanner and adds the mapping to an internal + * hash-map. + * + * @param scanner + * @return integer scanner id + */ + protected synchronized int addScanner(ResultScanner scanner) { + int id = nextScannerId++; + scannerMap.put(id, scanner); + return id; + } + + /** + * Returns the scanner associated with the specified ID. + * + * @param id + * @return a Scanner, or null if ID was invalid. + */ + protected synchronized ResultScanner getScanner(int id) { + return scannerMap.get(id); + } + + /** + * Removes the scanner associated with the specified ID from the internal + * id->scanner hash-map. + * + * @param id + * @return a Scanner, or null if ID was invalid. + */ + protected synchronized ResultScanner removeScanner(int id) { + return scannerMap.remove(id); + } + + // + // CTOR METHODS + // + + // TODO(hammer): figure out appropriate setting of maxSize for htablePool + /** + * Constructs an HBaseImpl object. + * @throws IOException + */ + HBaseImpl() throws IOException { + this(HBaseConfiguration.create()); + } + + HBaseImpl(final Configuration c) throws IOException { + conf = c; + admin = new HBaseAdmin(conf); + htablePool = new HTablePool(conf, 10); + scannerMap = new HashMap(); + } + + // + // SERVICE METHODS + // + + // TODO(hammer): Investigate use of the Command design pattern + + // + // Cluster metadata + // + + public Utf8 getHBaseVersion() throws AIOError { + try { + return new Utf8(admin.getClusterStatus().getHBaseVersion()); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + public AClusterStatus getClusterStatus() throws AIOError { + try { + return AvroUtil.csToACS(admin.getClusterStatus()); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + public GenericArray listTables() throws AIOError { + try { + HTableDescriptor[] tables = admin.listTables(); + Schema atdSchema = Schema.createArray(ATableDescriptor.SCHEMA$); + GenericData.Array result = null; + result = new GenericData.Array(tables.length, atdSchema); + for (HTableDescriptor table : tables) { + result.add(AvroUtil.htdToATD(table)); + } + return result; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + // + // Table metadata + // + + // TODO(hammer): Handle the case where the table does not exist explicitly? + public ATableDescriptor describeTable(ByteBuffer table) throws AIOError { + try { + return AvroUtil.htdToATD(admin.getTableDescriptor(Bytes.toBytes(table))); + } catch (TableNotFoundException e) { + return null; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + public boolean isTableEnabled(ByteBuffer table) throws AIOError { + try { + return admin.isTableEnabled(Bytes.toBytes(table)); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + public boolean tableExists(ByteBuffer table) throws AIOError { + try { + return admin.tableExists(Bytes.toBytes(table)); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + // + // Family metadata + // + + // TODO(hammer): Handle the case where the family does not exist explicitly? + public AFamilyDescriptor describeFamily(ByteBuffer table, ByteBuffer family) throws AIOError { + try { + HTableDescriptor htd = admin.getTableDescriptor(Bytes.toBytes(table)); + return AvroUtil.hcdToAFD(htd.getFamily(Bytes.toBytes(family))); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + // + // Table admin + // + + public Void createTable(ATableDescriptor table) throws AIOError, + AIllegalArgument, + ATableExists, + AMasterNotRunning { + try { + admin.createTable(AvroUtil.atdToHTD(table)); + return null; + } catch (IllegalArgumentException e) { + AIllegalArgument iae = new AIllegalArgument(); + iae.message = new Utf8(e.getMessage()); + throw iae; + } catch (TableExistsException e) { + ATableExists tee = new ATableExists(); + tee.message = new Utf8(e.getMessage()); + throw tee; + } catch (MasterNotRunningException e) { + AMasterNotRunning mnre = new AMasterNotRunning(); + mnre.message = new Utf8(e.getMessage()); + throw mnre; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + // Note that disable, flush and major compaction of .META. needed in client + // TODO(hammer): more selective cache dirtying than flush? + public Void deleteTable(ByteBuffer table) throws AIOError { + try { + admin.deleteTable(Bytes.toBytes(table)); + return null; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + // NB: Asynchronous operation + public Void modifyTable(ByteBuffer tableName, ATableDescriptor tableDescriptor) throws AIOError { + try { + admin.modifyTable(Bytes.toBytes(tableName), + AvroUtil.atdToHTD(tableDescriptor)); + return null; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + public Void enableTable(ByteBuffer table) throws AIOError { + try { + admin.enableTable(Bytes.toBytes(table)); + return null; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + public Void disableTable(ByteBuffer table) throws AIOError { + try { + admin.disableTable(Bytes.toBytes(table)); + return null; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + // NB: Asynchronous operation + public Void flush(ByteBuffer table) throws AIOError { + try { + admin.flush(Bytes.toBytes(table)); + return null; + } catch (InterruptedException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + // NB: Asynchronous operation + public Void split(ByteBuffer table) throws AIOError { + try { + admin.split(Bytes.toBytes(table)); + return null; + } catch (InterruptedException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + // + // Family admin + // + + public Void addFamily(ByteBuffer table, AFamilyDescriptor family) throws AIOError { + try { + admin.addColumn(Bytes.toBytes(table), + AvroUtil.afdToHCD(family)); + return null; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + // NB: Asynchronous operation + public Void deleteFamily(ByteBuffer table, ByteBuffer family) throws AIOError { + try { + admin.deleteColumn(Bytes.toBytes(table), Bytes.toBytes(family)); + return null; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + // NB: Asynchronous operation + public Void modifyFamily(ByteBuffer table, ByteBuffer familyName, AFamilyDescriptor familyDescriptor) throws AIOError { + try { + admin.modifyColumn(Bytes.toBytes(table), AvroUtil.afdToHCD(familyDescriptor)); + return null; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + // + // Single-row DML + // + + // TODO(hammer): Java with statement for htablepool concision? + // TODO(hammer): Can Get have timestamp and timerange simultaneously? + // TODO(hammer): Do I need to catch the RuntimeException of getTable? + // TODO(hammer): Handle gets with no results + // TODO(hammer): Uses exists(Get) to ensure columns exist + public AResult get(ByteBuffer table, AGet aget) throws AIOError { + HTableInterface htable = htablePool.getTable(Bytes.toBytes(table)); + try { + return AvroUtil.resultToAResult(htable.get(AvroUtil.agetToGet(aget))); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } finally { + try { + htablePool.putTable(htable); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + } + + public boolean exists(ByteBuffer table, AGet aget) throws AIOError { + HTableInterface htable = htablePool.getTable(Bytes.toBytes(table)); + try { + return htable.exists(AvroUtil.agetToGet(aget)); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } finally { + try { + htablePool.putTable(htable); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + } + + public Void put(ByteBuffer table, APut aput) throws AIOError { + HTableInterface htable = htablePool.getTable(Bytes.toBytes(table)); + try { + htable.put(AvroUtil.aputToPut(aput)); + return null; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } finally { + try { + htablePool.putTable(htable); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + } + + public Void delete(ByteBuffer table, ADelete adelete) throws AIOError { + HTableInterface htable = htablePool.getTable(Bytes.toBytes(table)); + try { + htable.delete(AvroUtil.adeleteToDelete(adelete)); + return null; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } finally { + try { + htablePool.putTable(htable); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + } + + public long incrementColumnValue(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, long amount, boolean writeToWAL) throws AIOError { + HTableInterface htable = htablePool.getTable(Bytes.toBytes(table)); + try { + return htable.incrementColumnValue(Bytes.toBytes(row), Bytes.toBytes(family), Bytes.toBytes(qualifier), amount, writeToWAL); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } finally { + try { + htablePool.putTable(htable); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + } + + // + // Multi-row DML + // + + public int scannerOpen(ByteBuffer table, AScan ascan) throws AIOError { + HTableInterface htable = htablePool.getTable(Bytes.toBytes(table)); + try { + Scan scan = AvroUtil.ascanToScan(ascan); + return addScanner(htable.getScanner(scan)); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } finally { + try { + htablePool.putTable(htable); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + } + + public Void scannerClose(int scannerId) throws AIOError, AIllegalArgument { + try { + ResultScanner scanner = getScanner(scannerId); + if (scanner == null) { + AIllegalArgument aie = new AIllegalArgument(); + aie.message = new Utf8("scanner ID is invalid: " + scannerId); + throw aie; + } + scanner.close(); + removeScanner(scannerId); + return null; + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + + public GenericArray scannerGetRows(int scannerId, int numberOfRows) throws AIOError, AIllegalArgument { + try { + ResultScanner scanner = getScanner(scannerId); + if (scanner == null) { + AIllegalArgument aie = new AIllegalArgument(); + aie.message = new Utf8("scanner ID is invalid: " + scannerId); + throw aie; + } + return AvroUtil.resultsToAResults(scanner.next(numberOfRows)); + } catch (IOException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; + } + } + } + + // + // MAIN PROGRAM + // + + private static void printUsageAndExit() { + printUsageAndExit(null); + } + + private static void printUsageAndExit(final String message) { + if (message != null) { + System.err.println(message); + } + System.out.println("Usage: java org.apache.hadoop.hbase.avro.AvroServer " + + "--help | [--port=PORT] start"); + System.out.println("Arguments:"); + System.out.println(" start Start Avro server"); + System.out.println(" stop Stop Avro server"); + System.out.println("Options:"); + System.out.println(" port Port to listen on. Default: 9090"); + System.out.println(" help Print this message and exit"); + System.exit(0); + } + + protected static void doMain(final String[] args) throws Exception { + if (args.length < 1) { + printUsageAndExit(); + } + int port = 9090; + final String portArgKey = "--port="; + for (String cmd: args) { + if (cmd.startsWith(portArgKey)) { + port = Integer.parseInt(cmd.substring(portArgKey.length())); + continue; + } else if (cmd.equals("--help") || cmd.equals("-h")) { + printUsageAndExit(); + } else if (cmd.equals("start")) { + continue; + } else if (cmd.equals("stop")) { + printUsageAndExit("To shutdown the Avro server run " + + "bin/hbase-daemon.sh stop avro or send a kill signal to " + + "the Avro server pid"); + } + + // Print out usage if we get to here. + printUsageAndExit(); + } + Log LOG = LogFactory.getLog("AvroServer"); + LOG.info("starting HBase Avro server on port " + Integer.toString(port)); + SpecificResponder r = new SpecificResponder(HBase.class, new HBaseImpl()); + HttpServer server = new HttpServer(r, port); + server.start(); + server.join(); + } + + // TODO(hammer): Look at Cassandra's daemonization and integration with JSVC + // TODO(hammer): Don't eat it after a single exception + // TODO(hammer): Figure out why we do doMain() + // TODO(hammer): Figure out if we want String[] or String [] syntax + public static void main(String[] args) throws Exception { + doMain(args); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/avro/AvroUtil.java b/src/main/java/org/apache/hadoop/hbase/avro/AvroUtil.java new file mode 100644 index 0000000..abd2ae6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/avro/AvroUtil.java @@ -0,0 +1,413 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.avro; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.List; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericArray; +import org.apache.avro.generic.GenericData; +import org.apache.avro.util.Utf8; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HServerLoad; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.avro.generated.AClusterStatus; +import org.apache.hadoop.hbase.avro.generated.AColumn; +import org.apache.hadoop.hbase.avro.generated.AColumnValue; +import org.apache.hadoop.hbase.avro.generated.ACompressionAlgorithm; +import org.apache.hadoop.hbase.avro.generated.ADelete; +import org.apache.hadoop.hbase.avro.generated.AFamilyDescriptor; +import org.apache.hadoop.hbase.avro.generated.AGet; +import org.apache.hadoop.hbase.avro.generated.AIllegalArgument; +import org.apache.hadoop.hbase.avro.generated.APut; +import org.apache.hadoop.hbase.avro.generated.ARegionLoad; +import org.apache.hadoop.hbase.avro.generated.AResult; +import org.apache.hadoop.hbase.avro.generated.AResultEntry; +import org.apache.hadoop.hbase.avro.generated.AScan; +import org.apache.hadoop.hbase.avro.generated.AServerAddress; +import org.apache.hadoop.hbase.avro.generated.AServerInfo; +import org.apache.hadoop.hbase.avro.generated.AServerLoad; +import org.apache.hadoop.hbase.avro.generated.ATableDescriptor; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.util.Bytes; + +public class AvroUtil { + + // + // Cluster metadata + // + + static public AServerAddress hsaToASA(HServerAddress hsa) throws IOException { + AServerAddress asa = new AServerAddress(); + asa.hostname = new Utf8(hsa.getHostname()); + asa.inetSocketAddress = new Utf8(hsa.getInetSocketAddress().toString()); + asa.port = hsa.getPort(); + return asa; + } + + static public ARegionLoad hrlToARL(HServerLoad.RegionLoad rl) throws IOException { + ARegionLoad arl = new ARegionLoad(); + arl.memStoreSizeMB = rl.getMemStoreSizeMB(); + arl.name = ByteBuffer.wrap(rl.getName()); + arl.storefileIndexSizeMB = rl.getStorefileIndexSizeMB(); + arl.storefiles = rl.getStorefiles(); + arl.storefileSizeMB = rl.getStorefileSizeMB(); + arl.stores = rl.getStores(); + return arl; + } + + static public AServerLoad hslToASL(HServerLoad hsl) throws IOException { + AServerLoad asl = new AServerLoad(); + asl.load = hsl.getLoad(); + asl.maxHeapMB = hsl.getMaxHeapMB(); + asl.memStoreSizeInMB = hsl.getMemStoreSizeInMB(); + asl.numberOfRegions = hsl.getNumberOfRegions(); + asl.numberOfRequests = hsl.getNumberOfRequests(); + + Collection regionLoads = hsl.getRegionsLoad().values(); + Schema s = Schema.createArray(ARegionLoad.SCHEMA$); + GenericData.Array aregionLoads = null; + if (regionLoads != null) { + aregionLoads = new GenericData.Array(regionLoads.size(), s); + for (HServerLoad.RegionLoad rl : regionLoads) { + aregionLoads.add(hrlToARL(rl)); + } + } else { + aregionLoads = new GenericData.Array(0, s); + } + asl.regionsLoad = aregionLoads; + + asl.storefileIndexSizeInMB = hsl.getStorefileIndexSizeInMB(); + asl.storefiles = hsl.getStorefiles(); + asl.storefileSizeInMB = hsl.getStorefileSizeInMB(); + asl.usedHeapMB = hsl.getUsedHeapMB(); + return asl; + } + + static public AServerInfo hsiToASI(ServerName sn, HServerLoad hsl) throws IOException { + AServerInfo asi = new AServerInfo(); + asi.infoPort = -1; + asi.load = hslToASL(hsl); + asi.serverAddress = hsaToASA(new HServerAddress(sn.getHostname(), sn.getPort())); + asi.serverName = new Utf8(sn.toString()); + asi.startCode = sn.getStartcode(); + return asi; + } + + static public AClusterStatus csToACS(ClusterStatus cs) throws IOException { + AClusterStatus acs = new AClusterStatus(); + acs.averageLoad = cs.getAverageLoad(); + Collection deadServerNames = cs.getDeadServerNames(); + Schema stringArraySchema = Schema.createArray(Schema.create(Schema.Type.STRING)); + GenericData.Array adeadServerNames = null; + if (deadServerNames != null) { + adeadServerNames = new GenericData.Array(deadServerNames.size(), stringArraySchema); + for (ServerName deadServerName : deadServerNames) { + adeadServerNames.add(new Utf8(deadServerName.toString())); + } + } else { + adeadServerNames = new GenericData.Array(0, stringArraySchema); + } + acs.deadServerNames = adeadServerNames; + acs.deadServers = cs.getDeadServers(); + acs.hbaseVersion = new Utf8(cs.getHBaseVersion()); + acs.regionsCount = cs.getRegionsCount(); + acs.requestsCount = cs.getRequestsCount(); + Collection hserverInfos = cs.getServers(); + Schema s = Schema.createArray(AServerInfo.SCHEMA$); + GenericData.Array aserverInfos = null; + if (hserverInfos != null) { + aserverInfos = new GenericData.Array(hserverInfos.size(), s); + for (ServerName hsi : hserverInfos) { + aserverInfos.add(hsiToASI(hsi, cs.getLoad(hsi))); + } + } else { + aserverInfos = new GenericData.Array(0, s); + } + acs.serverInfos = aserverInfos; + acs.servers = cs.getServers().size(); + return acs; + } + + // + // Table metadata + // + + static public ATableDescriptor htdToATD(HTableDescriptor table) throws IOException { + ATableDescriptor atd = new ATableDescriptor(); + atd.name = ByteBuffer.wrap(table.getName()); + Collection families = table.getFamilies(); + Schema afdSchema = Schema.createArray(AFamilyDescriptor.SCHEMA$); + GenericData.Array afamilies = null; + if (families.size() > 0) { + afamilies = new GenericData.Array(families.size(), afdSchema); + for (HColumnDescriptor hcd : families) { + AFamilyDescriptor afamily = hcdToAFD(hcd); + afamilies.add(afamily); + } + } else { + afamilies = new GenericData.Array(0, afdSchema); + } + atd.families = afamilies; + atd.maxFileSize = table.getMaxFileSize(); + atd.memStoreFlushSize = table.getMemStoreFlushSize(); + atd.rootRegion = table.isRootRegion(); + atd.metaRegion = table.isMetaRegion(); + atd.metaTable = table.isMetaTable(); + atd.readOnly = table.isReadOnly(); + atd.deferredLogFlush = table.isDeferredLogFlush(); + return atd; + } + + static public HTableDescriptor atdToHTD(ATableDescriptor atd) throws IOException, AIllegalArgument { + HTableDescriptor htd = new HTableDescriptor(Bytes.toBytes(atd.name)); + if (atd.families != null && atd.families.size() > 0) { + for (AFamilyDescriptor afd : atd.families) { + htd.addFamily(afdToHCD(afd)); + } + } + if (atd.maxFileSize != null) { + htd.setMaxFileSize(atd.maxFileSize); + } + if (atd.memStoreFlushSize != null) { + htd.setMemStoreFlushSize(atd.memStoreFlushSize); + } + if (atd.readOnly != null) { + htd.setReadOnly(atd.readOnly); + } + if (atd.deferredLogFlush != null) { + htd.setDeferredLogFlush(atd.deferredLogFlush); + } + if (atd.rootRegion != null || atd.metaRegion != null || atd.metaTable != null) { + AIllegalArgument aie = new AIllegalArgument(); + aie.message = new Utf8("Can't set root or meta flag on create table."); + throw aie; + } + return htd; + } + + // + // Family metadata + // + + static public AFamilyDescriptor hcdToAFD(HColumnDescriptor hcd) throws IOException { + AFamilyDescriptor afamily = new AFamilyDescriptor(); + afamily.name = ByteBuffer.wrap(hcd.getName()); + String compressionAlgorithm = hcd.getCompressionType().getName(); + if (compressionAlgorithm == "LZO") { + afamily.compression = ACompressionAlgorithm.LZO; + } else if (compressionAlgorithm == "GZ") { + afamily.compression = ACompressionAlgorithm.GZ; + } else { + afamily.compression = ACompressionAlgorithm.NONE; + } + afamily.maxVersions = hcd.getMaxVersions(); + afamily.blocksize = hcd.getBlocksize(); + afamily.inMemory = hcd.isInMemory(); + afamily.timeToLive = hcd.getTimeToLive(); + afamily.blockCacheEnabled = hcd.isBlockCacheEnabled(); + return afamily; + } + + static public HColumnDescriptor afdToHCD(AFamilyDescriptor afd) throws IOException { + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes(afd.name)); + + ACompressionAlgorithm compressionAlgorithm = afd.compression; + if (compressionAlgorithm == ACompressionAlgorithm.LZO) { + hcd.setCompressionType(Compression.Algorithm.LZO); + } else if (compressionAlgorithm == ACompressionAlgorithm.GZ) { + hcd.setCompressionType(Compression.Algorithm.GZ); + } else { + hcd.setCompressionType(Compression.Algorithm.NONE); + } + + if (afd.maxVersions != null) { + hcd.setMaxVersions(afd.maxVersions); + } + + if (afd.blocksize != null) { + hcd.setBlocksize(afd.blocksize); + } + + if (afd.inMemory != null) { + hcd.setInMemory(afd.inMemory); + } + + if (afd.timeToLive != null) { + hcd.setTimeToLive(afd.timeToLive); + } + + if (afd.blockCacheEnabled != null) { + hcd.setBlockCacheEnabled(afd.blockCacheEnabled); + } + return hcd; + } + + // + // Single-Row DML (Get) + // + + // TODO(hammer): More concise idiom than if not null assign? + static public Get agetToGet(AGet aget) throws IOException { + Get get = new Get(Bytes.toBytes(aget.row)); + if (aget.columns != null) { + for (AColumn acolumn : aget.columns) { + if (acolumn.qualifier != null) { + get.addColumn(Bytes.toBytes(acolumn.family), Bytes.toBytes(acolumn.qualifier)); + } else { + get.addFamily(Bytes.toBytes(acolumn.family)); + } + } + } + if (aget.timestamp != null) { + get.setTimeStamp(aget.timestamp); + } + if (aget.timerange != null) { + get.setTimeRange(aget.timerange.minStamp, aget.timerange.maxStamp); + } + if (aget.maxVersions != null) { + get.setMaxVersions(aget.maxVersions); + } + return get; + } + + // TODO(hammer): Pick one: Timestamp or TimeStamp + static public AResult resultToAResult(Result result) { + AResult aresult = new AResult(); + byte[] row = result.getRow(); + aresult.row = ByteBuffer.wrap(row != null ? row : new byte[1]); + Schema s = Schema.createArray(AResultEntry.SCHEMA$); + GenericData.Array entries = null; + List resultKeyValues = result.list(); + if (resultKeyValues != null && resultKeyValues.size() > 0) { + entries = new GenericData.Array(resultKeyValues.size(), s); + for (KeyValue resultKeyValue : resultKeyValues) { + AResultEntry entry = new AResultEntry(); + entry.family = ByteBuffer.wrap(resultKeyValue.getFamily()); + entry.qualifier = ByteBuffer.wrap(resultKeyValue.getQualifier()); + entry.value = ByteBuffer.wrap(resultKeyValue.getValue()); + entry.timestamp = resultKeyValue.getTimestamp(); + entries.add(entry); + } + } else { + entries = new GenericData.Array(0, s); + } + aresult.entries = entries; + return aresult; + } + + // + // Single-Row DML (Put) + // + + static public Put aputToPut(APut aput) throws IOException { + Put put = new Put(Bytes.toBytes(aput.row)); + for (AColumnValue acv : aput.columnValues) { + if (acv.timestamp != null) { + put.add(Bytes.toBytes(acv.family), + Bytes.toBytes(acv.qualifier), + acv.timestamp, + Bytes.toBytes(acv.value)); + } else { + put.add(Bytes.toBytes(acv.family), + Bytes.toBytes(acv.qualifier), + Bytes.toBytes(acv.value)); + } + } + return put; + } + + // + // Single-Row DML (Delete) + // + + static public Delete adeleteToDelete(ADelete adelete) throws IOException { + Delete delete = new Delete(Bytes.toBytes(adelete.row)); + if (adelete.columns != null) { + for (AColumn acolumn : adelete.columns) { + if (acolumn.qualifier != null) { + delete.deleteColumns(Bytes.toBytes(acolumn.family), Bytes.toBytes(acolumn.qualifier)); + } else { + delete.deleteFamily(Bytes.toBytes(acolumn.family)); + } + } + } + return delete; + } + + // + // Multi-row DML (Scan) + // + + static public Scan ascanToScan(AScan ascan) throws IOException { + Scan scan = new Scan(); + if (ascan.startRow != null) { + scan.setStartRow(Bytes.toBytes(ascan.startRow)); + } + if (ascan.stopRow != null) { + scan.setStopRow(Bytes.toBytes(ascan.stopRow)); + } + if (ascan.columns != null) { + for (AColumn acolumn : ascan.columns) { + if (acolumn.qualifier != null) { + scan.addColumn(Bytes.toBytes(acolumn.family), Bytes.toBytes(acolumn.qualifier)); + } else { + scan.addFamily(Bytes.toBytes(acolumn.family)); + } + } + } + if (ascan.timestamp != null) { + scan.setTimeStamp(ascan.timestamp); + } + if (ascan.timerange != null) { + scan.setTimeRange(ascan.timerange.minStamp, ascan.timerange.maxStamp); + } + if (ascan.maxVersions != null) { + scan.setMaxVersions(ascan.maxVersions); + } + return scan; + } + + // TODO(hammer): Better to return null or empty array? + static public GenericArray resultsToAResults(Result[] results) { + Schema s = Schema.createArray(AResult.SCHEMA$); + GenericData.Array aresults = null; + if (results != null && results.length > 0) { + aresults = new GenericData.Array(results.length, s); + for (Result result : results) { + aresults.add(resultToAResult(result)); + } + } else { + aresults = new GenericData.Array(0, s); + } + return aresults; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/avro/package.html b/src/main/java/org/apache/hadoop/hbase/avro/package.html new file mode 100644 index 0000000..0588095 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/avro/package.html @@ -0,0 +1,70 @@ + + + + + + + +Provides an HBase Avro service. + +This directory contains an Avro interface definition file for an HBase RPC +service and a Java server implementation. + +

What is Avro?

+ +

Avro is a data serialization and RPC system. For more, see the +current specification. +

+ +

Description

+ +

The HBase API is defined in the +file hbase.genavro. A server-side implementation of the API is in +org.apache.hadoop.hbase.avro.AvroServer. The generated interfaces, +types, and RPC utility files are checked into SVN under the +org.apache.hadoop.hbase.avro.generated directory. + +

+ +

The files were generated by running the commands: +

+  java -jar avro-tools-1.4.1.jar idl hbase.avdl hbase.avpr
+  java -jar avro-tools-1.4.1.jar compile protocol hbase.avpr $HBASE_HOME/src/main/java
+
+

+ +

The 'avro-tools-x.y.z.jar' jarfile is an Avro utility, and it is +distributed as a part of the Avro package. Additionally, specific +language runtime libraries are apart of the Avro package. A version of the +Java runtime is listed as a dendency in Maven. +

+ +

To start AvroServer, use: +

+  ./bin/hbase avro start [--port=PORT]
+
+The default port is 9090. +

+ +

To stop, use: +

+  ./bin/hbase-daemon.sh stop avro
+
+

+ + diff --git a/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java new file mode 100644 index 0000000..8a14b11 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java @@ -0,0 +1,718 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.HBaseFileSystem; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; +import org.apache.hadoop.io.MultipleIOException; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.Collections2; +import com.google.common.collect.Lists; + +/** + * Utility class to handle the removal of HFiles (or the respective {@link StoreFile StoreFiles}) + * for a HRegion from the {@link FileSystem}. The hfiles will be archived or deleted, depending on + * the state of the system. + */ +public class HFileArchiver { + private static final Log LOG = LogFactory.getLog(HFileArchiver.class); + private static final String SEPARATOR = "."; + + /** Number of retries in case of fs operation failure */ + private static final int DEFAULT_RETRIES_NUMBER = 6; + + private HFileArchiver() { + // hidden ctor since this is just a util + } + + /** + * Cleans up all the files for a HRegion by archiving the HFiles to the + * archive directory + * @param conf the configuration to use + * @param fs the file system object + * @param info HRegionInfo for region to be deleted + * @throws IOException + */ + public static void archiveRegion(Configuration conf, FileSystem fs, HRegionInfo info) + throws IOException { + Path rootDir = FSUtils.getRootDir(conf); + archiveRegion(fs, rootDir, HTableDescriptor.getTableDir(rootDir, info.getTableName()), + HRegion.getRegionDir(rootDir, info)); + } + + /** + * Remove an entire region from the table directory via archiving the region's hfiles. + * @param fs {@link FileSystem} from which to remove the region + * @param rootdir {@link Path} to the root directory where hbase files are stored (for building + * the archive path) + * @param tableDir {@link Path} to where the table is being stored (for building the archive path) + * @param regionDir {@link Path} to where a region is being stored (for building the archive path) + * @return true if the region was sucessfully deleted. false if the filesystem + * operations could not complete. + * @throws IOException if the request cannot be completed + */ + public static boolean archiveRegion(FileSystem fs, Path rootdir, Path tableDir, Path regionDir) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("ARCHIVING region " + regionDir.toString()); + } + + // otherwise, we archive the files + // make sure we can archive + if (tableDir == null || regionDir == null) { + LOG.error("No archive directory could be found because tabledir (" + tableDir + + ") or regiondir (" + regionDir + "was null. Deleting files instead."); + deleteRegionWithoutArchiving(fs, regionDir); + // we should have archived, but failed to. Doesn't matter if we deleted + // the archived files correctly or not. + return false; + } + + // make sure the regiondir lives under the tabledir + Preconditions.checkArgument(regionDir.toString().startsWith(tableDir.toString())); + Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(rootdir, tableDir, regionDir); + + LOG.debug("Have an archive directory, preparing to move files"); + FileStatusConverter getAsFile = new FileStatusConverter(fs); + // otherwise, we attempt to archive the store files + + // build collection of just the store directories to archive + Collection toArchive = new ArrayList(); + final PathFilter dirFilter = new FSUtils.DirFilter(fs); + PathFilter nonHidden = new PathFilter() { + @Override + public boolean accept(Path file) { + return dirFilter.accept(file) && !file.getName().toString().startsWith("."); + } + }; + FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden); + // if there no files, we can just delete the directory and return; + if (storeDirs == null) { + LOG.debug("Region directory (" + regionDir + ") was empty, just deleting and returning!"); + return deleteRegionWithoutArchiving(fs, regionDir); + } + + // convert the files in the region to a File + toArchive.addAll(Lists.transform(Arrays.asList(storeDirs), getAsFile)); + LOG.debug("Archiving:" + toArchive); + boolean success = false; + try { + success = resolveAndArchive(fs, regionArchiveDir, toArchive); + } catch (IOException e) { + LOG.error("Failed to archive: " + toArchive, e); + success = false; + } + + // if that was successful, then we delete the region + if (success) { + LOG.debug("Successfully resolved and archived, now can just delete region."); + return deleteRegionWithoutArchiving(fs, regionDir); + } + + throw new IOException("Received error when attempting to archive files (" + toArchive + + "), cannot delete region directory."); + } + + /** + * Remove from the specified region the store files of the specified column family, + * either by archiving them or outright deletion + * @param fs the filesystem where the store files live + * @param conf {@link Configuration} to examine to determine the archive directory + * @param parent Parent region hosting the store files + * @param tableDir {@link Path} to where the table is being stored (for building the archive path) + * @param family the family hosting the store files + * @throws IOException if the files could not be correctly disposed. + */ + public static void archiveFamily(FileSystem fs, Configuration conf, + HRegionInfo parent, Path tableDir, byte[] family) throws IOException { + Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family))); + FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir, null); + if (storeFiles == null) { + LOG.debug("No store files to dispose for region=" + parent.getRegionNameAsString() + + ", family=" + Bytes.toString(family)); + return; + } + + FileStatusConverter getAsFile = new FileStatusConverter(fs); + Collection toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile); + Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, tableDir, family); + + // do the actual archive + if (!resolveAndArchive(fs, storeArchiveDir, toArchive)) { + throw new IOException("Failed to archive/delete all the files for region:" + + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family) + + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); + } + } + + /** + * Remove the store files, either by archiving them or outright deletion + * @param conf {@link Configuration} to examine to determine the archive directory + * @param fs the filesystem where the store files live + * @param parent Parent region hosting the store files + * @param family the family hosting the store files + * @param compactedFiles files to be disposed of. No further reading of these files should be + * attempted; otherwise likely to cause an {@link IOException} + * @throws IOException if the files could not be correctly disposed. + */ + public static void archiveStoreFiles(Configuration conf, FileSystem fs, HRegion parent, + byte[] family, Collection compactedFiles) throws IOException { + + // sometimes in testing, we don't have rss, so we need to check for that + if (fs == null) { + LOG.warn("Passed filesystem is null, so just deleting the files without archiving for region:" + + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family)); + deleteStoreFilesWithoutArchiving(compactedFiles); + return; + } + + // short circuit if we don't have any files to delete + if (compactedFiles.size() == 0) { + LOG.debug("No store files to dispose, done!"); + return; + } + + // build the archive path + if (parent == null || family == null) throw new IOException( + "Need to have a parent region and a family to archive from."); + + Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, family); + + // make sure we don't archive if we can't and that the archive dir exists + if (!HBaseFileSystem.makeDirOnFileSystem(fs, storeArchiveDir)) { + throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" + + Bytes.toString(family) + ", deleting compacted files instead."); + } + + // otherwise we attempt to archive the store files + LOG.debug("Archiving compacted store files."); + + // wrap the storefile into a File + StoreToFile getStorePath = new StoreToFile(fs); + Collection storeFiles = Collections2.transform(compactedFiles, getStorePath); + + // do the actual archive + if (!resolveAndArchive(fs, storeArchiveDir, storeFiles)) { + throw new IOException("Failed to archive/delete all the files for region:" + + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family) + + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); + } + } + + /** + * Archive the store file + * @param fs the filesystem where the store files live + * @param regionInfo region hosting the store files + * @param conf {@link Configuration} to examine to determine the archive directory + * @param tableDir {@link Path} to where the table is being stored (for building the archive path) + * @param family the family hosting the store files + * @param storeFile file to be archived + * @throws IOException if the files could not be correctly disposed. + */ + public static void archiveStoreFile(FileSystem fs, HRegionInfo regionInfo, + Configuration conf, Path tableDir, byte[] family, Path storeFile) throws IOException { + Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family); + // make sure we don't archive if we can't and that the archive dir exists + if (!HBaseFileSystem.makeDirOnFileSystem(fs, storeArchiveDir)) { + throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" + + Bytes.toString(family) + ", deleting compacted files instead."); + } + + // do the actual archive + long start = EnvironmentEdgeManager.currentTimeMillis(); + File file = new FileablePath(fs, storeFile); + if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) { + throw new IOException("Failed to archive/delete the file for region:" + + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family) + + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); + } + } + + /** + * Archive the given files and resolve any conflicts with existing files via appending the time + * archiving started (so all conflicts in the same group have the same timestamp appended). + *

+ * If any of the passed files to archive are directories, archives the all files under that + * directory. Archive directory structure for children is the base archive directory name + the + * parent directory and is built recursively is passed files are directories themselves. + * @param fs {@link FileSystem} on which to archive the files + * @param baseArchiveDir base archive directory to archive the given files + * @param toArchive files to be archived + * @return true on success, false otherwise + * @throws IOException on unexpected failure + */ + private static boolean resolveAndArchive(FileSystem fs, Path baseArchiveDir, + Collection toArchive) throws IOException { + LOG.debug("Starting to archive files:" + toArchive); + long start = EnvironmentEdgeManager.currentTimeMillis(); + List failures = resolveAndArchive(fs, baseArchiveDir, toArchive, start); + + // notify that some files were not archived. + // We can't delete the files otherwise snapshots or other backup system + // that relies on the archiver end up with data loss. + if (failures.size() > 0) { + LOG.warn("Failed to complete archive of: " + failures + + ". Those files are still in the original location, and they may slow down reads."); + return false; + } + return true; + } + + /** + * Resolve any conflict with an existing archive file via timestamp-append + * renaming of the existing file and then archive the passed in files. + * @param fs {@link FileSystem} on which to archive the files + * @param baseArchiveDir base archive directory to store the files. If any of + * the files to archive are directories, will append the name of the + * directory to the base archive directory name, creating a parallel + * structure. + * @param toArchive files/directories that need to be archvied + * @param start time the archiving started - used for resolving archive + * conflicts. + * @return the list of failed to archive files. + * @throws IOException if an unexpected file operation exception occured + */ + private static List resolveAndArchive(FileSystem fs, Path baseArchiveDir, + Collection toArchive, long start) throws IOException { + // short circuit if no files to move + if (toArchive.size() == 0) return Collections.emptyList(); + + LOG.debug("moving files to the archive directory: " + baseArchiveDir); + + // make sure the archive directory exists + if (!fs.exists(baseArchiveDir)) { + if (!HBaseFileSystem.makeDirOnFileSystem(fs, baseArchiveDir)) { + throw new IOException("Failed to create the archive directory:" + baseArchiveDir + + ", quitting archive attempt."); + } + LOG.debug("Created archive directory:" + baseArchiveDir); + } + + List failures = new ArrayList(); + String startTime = Long.toString(start); + for (File file : toArchive) { + // if its a file archive it + try { + LOG.debug("Archiving:" + file); + if (file.isFile()) { + // attempt to archive the file + if (!resolveAndArchiveFile(baseArchiveDir, file, startTime)) { + LOG.warn("Couldn't archive " + file + " into backup directory: " + baseArchiveDir); + failures.add(file); + } + } else { + // otherwise its a directory and we need to archive all files + LOG.debug(file + " is a directory, archiving children files"); + // so we add the directory name to the one base archive + Path parentArchiveDir = new Path(baseArchiveDir, file.getName()); + // and then get all the files from that directory and attempt to + // archive those too + Collection children = file.getChildren(); + failures.addAll(resolveAndArchive(fs, parentArchiveDir, children, start)); + } + } catch (IOException e) { + LOG.warn("Failed to archive file: " + file, e); + failures.add(file); + } + } + return failures; + } + + /** + * Attempt to archive the passed in file to the archive directory. + *

+ * If the same file already exists in the archive, it is moved to a timestamped directory under + * the archive directory and the new file is put in its place. + * @param archiveDir {@link Path} to the directory that stores the archives of the hfiles + * @param currentFile {@link Path} to the original HFile that will be archived + * @param archiveStartTime time the archiving started, to resolve naming conflicts + * @return true if the file is successfully archived. false if there was a + * problem, but the operation still completed. + * @throws IOException on failure to complete {@link FileSystem} operations. + */ + private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile, + String archiveStartTime) throws IOException { + // build path as it should be in the archive + String filename = currentFile.getName(); + Path archiveFile = new Path(archiveDir, filename); + FileSystem fs = currentFile.getFileSystem(); + + // if the file already exists in the archive, move that one to a timestamped backup. This is a + // really, really unlikely situtation, where we get the same name for the existing file, but + // is included just for that 1 in trillion chance. + if (fs.exists(archiveFile)) { + if (LOG.isDebugEnabled()) { + LOG.debug("File:" + archiveFile + " already exists in archive, moving to " + + "timestamped backup and overwriting current."); + } + + // move the archive file to the stamped backup + Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime); + if (!HBaseFileSystem.renameDirForFileSystem(fs, archiveFile, backedupArchiveFile)) { + LOG.error("Could not rename archive file to backup: " + backedupArchiveFile + + ", deleting existing file in favor of newer."); + // try to delete the exisiting file, if we can't rename it + if (!HBaseFileSystem.deleteFileFromFileSystem(fs, archiveFile)) { + throw new IOException("Couldn't delete existing archive file (" + archiveFile + + ") or rename it to the backup file (" + backedupArchiveFile + + ") to make room for similarly named file."); + } + } + LOG.debug("Backed up archive file from: " + archiveFile); + } + + LOG.debug("No existing file in archive for:" + archiveFile + + ", free to archive original file."); + + // at this point, we should have a free spot for the archive file + boolean success = false; + for (int i = 0; !success && i < DEFAULT_RETRIES_NUMBER; ++i) { + if (i > 0) { + // Ensure that the archive directory exists. + // The previous "move to archive" operation has failed probably because + // the cleaner has removed our archive directory (HBASE-7643). + // (we're in a retry loop, so don't worry too much about the exception) + try { + if (!fs.exists(archiveDir) + && HBaseFileSystem.makeDirOnFileSystem(fs, archiveDir)) { + LOG.debug("Created archive directory:" + archiveDir); + } + } catch (IOException e) { + LOG.warn("Failed to create the archive directory: " + archiveDir, e); + } + } + + try { + success = currentFile.moveAndClose(archiveFile); + } catch (IOException e) { + LOG.warn("Failed to archive file: " + currentFile + " on try #" + i, e); + success = false; + } + } + + if (!success) { + LOG.error("Failed to archive file:" + currentFile); + return false; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Finished archiving file from: " + currentFile + ", to: " + archiveFile); + } + return true; + } + + /** + * Simple delete of regular files from the {@link FileSystem}. + *

+ * This method is a more generic implementation that the other deleteXXX + * methods in this class, allowing more code reuse at the cost of a couple + * more, short-lived objects (which should have minimum impact on the jvm). + * @param fs {@link FileSystem} where the files live + * @param files {@link Collection} of files to be deleted + * @throws IOException if a file cannot be deleted. All files will be + * attempted to deleted before throwing the exception, rather than + * failing at the first file. + */ + private static void deleteFilesWithoutArchiving(Collection files) throws IOException { + List errors = new ArrayList(0); + for (File file : files) { + try { + LOG.debug("Deleting region file:" + file); + file.delete(); + } catch (IOException e) { + LOG.error("Failed to delete file:" + file); + errors.add(e); + } + } + if (errors.size() > 0) { + throw MultipleIOException.createIOException(errors); + } + } + + /** + * Without regard for backup, delete a region. Should be used with caution. + * @param regionDir {@link Path} to the region to be deleted. + * @param fs FileSystem from which to delete the region + * @return true on successful deletion, false otherwise + * @throws IOException on filesystem operation failure + */ + private static boolean deleteRegionWithoutArchiving(FileSystem fs, Path regionDir) + throws IOException { + if (HBaseFileSystem.deleteDirFromFileSystem(fs, regionDir)) { + LOG.debug("Deleted all region files in: " + regionDir); + return true; + } + LOG.debug("Failed to delete region directory:" + regionDir); + return false; + } + + /** + * Just do a simple delete of the given store files + *

+ * A best effort is made to delete each of the files, rather than bailing on the first failure. + *

+ * This method is preferable to {@link #deleteFilesWithoutArchiving(Collection)} since it consumes + * less resources, but is limited in terms of usefulness + * @param compactedFiles store files to delete from the file system. + * @throws IOException if a file cannot be deleted. All files will be attempted to deleted before + * throwing the exception, rather than failing at the first file. + */ + private static void deleteStoreFilesWithoutArchiving(Collection compactedFiles) + throws IOException { + LOG.debug("Deleting store files without archiving."); + List errors = new ArrayList(0); + for (StoreFile hsf : compactedFiles) { + try { + hsf.deleteReader(); + } catch (IOException e) { + LOG.error("Failed to delete store file:" + hsf.getPath()); + errors.add(e); + } + } + if (errors.size() > 0) { + throw MultipleIOException.createIOException(errors); + } + } + + /** + * Adapt a type to match the {@link File} interface, which is used internally for handling + * archival/removal of files + * @param type to adapt to the {@link File} interface + */ + private static abstract class FileConverter implements Function { + protected final FileSystem fs; + + public FileConverter(FileSystem fs) { + this.fs = fs; + } + } + + /** + * Convert a FileStatus to something we can manage in the archiving + */ + private static class FileStatusConverter extends FileConverter { + public FileStatusConverter(FileSystem fs) { + super(fs); + } + + @Override + public File apply(FileStatus input) { + return new FileablePath(fs, input.getPath()); + } + } + + /** + * Convert the {@link StoreFile} into something we can manage in the archive + * methods + */ + private static class StoreToFile extends FileConverter { + public StoreToFile(FileSystem fs) { + super(fs); + } + + @Override + public File apply(StoreFile input) { + return new FileableStoreFile(fs, input); + } + } + + /** + * Wrapper to handle file operations uniformly + */ + private static abstract class File { + protected final FileSystem fs; + + public File(FileSystem fs) { + this.fs = fs; + } + + /** + * Delete the file + * @throws IOException on failure + */ + abstract void delete() throws IOException; + + /** + * Check to see if this is a file or a directory + * @return true if it is a file, false otherwise + * @throws IOException on {@link FileSystem} connection error + */ + abstract boolean isFile() throws IOException; + + /** + * @return if this is a directory, returns all the children in the + * directory, otherwise returns an empty list + * @throws IOException + */ + abstract Collection getChildren() throws IOException; + + /** + * close any outside readers of the file + * @throws IOException + */ + abstract void close() throws IOException; + + /** + * @return the name of the file (not the full fs path, just the individual + * file name) + */ + abstract String getName(); + + /** + * @return the path to this file + */ + abstract Path getPath(); + + /** + * Move the file to the given destination + * @param dest + * @return true on success + * @throws IOException + */ + public boolean moveAndClose(Path dest) throws IOException { + this.close(); + Path p = this.getPath(); + return HBaseFileSystem.renameDirForFileSystem(fs, p, dest); + } + + /** + * @return the {@link FileSystem} on which this file resides + */ + public FileSystem getFileSystem() { + return this.fs; + } + + @Override + public String toString() { + return this.getClass() + ", file:" + getPath().toString(); + } + } + + /** + * A {@link File} that wraps a simple {@link Path} on a {@link FileSystem}. + */ + private static class FileablePath extends File { + private final Path file; + private final FileStatusConverter getAsFile; + + public FileablePath(FileSystem fs, Path file) { + super(fs); + this.file = file; + this.getAsFile = new FileStatusConverter(fs); + } + + @Override + public void delete() throws IOException { + if (!HBaseFileSystem.deleteDirFromFileSystem(fs, file)) + throw new IOException("Failed to delete:" + this.file); + } + + @Override + public String getName() { + return file.getName(); + } + + @Override + public Collection getChildren() throws IOException { + if (fs.isFile(file)) return Collections.emptyList(); + return Collections2.transform(Arrays.asList(fs.listStatus(file)), getAsFile); + } + + @Override + public boolean isFile() throws IOException { + return fs.isFile(file); + } + + @Override + public void close() throws IOException { + // NOOP - files are implicitly closed on removal + } + + @Override + Path getPath() { + return file; + } + } + + /** + * {@link File} adapter for a {@link StoreFile} living on a {@link FileSystem} + * . + */ + private static class FileableStoreFile extends File { + StoreFile file; + + public FileableStoreFile(FileSystem fs, StoreFile store) { + super(fs); + this.file = store; + } + + @Override + public void delete() throws IOException { + file.deleteReader(); + } + + @Override + public String getName() { + return file.getPath().getName(); + } + + @Override + public boolean isFile() { + return true; + } + + @Override + public Collection getChildren() throws IOException { + // storefiles don't have children + return Collections.emptyList(); + } + + @Override + public void close() throws IOException { + file.closeReader(true); + } + + @Override + Path getPath() { + return file.getPath(); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java new file mode 100644 index 0000000..58c61ee --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java @@ -0,0 +1,646 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.catalog; + +import java.io.EOFException; +import java.io.IOException; +import java.net.ConnectException; +import java.net.NoRouteToHostException; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.RetriesExhaustedException; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.MetaNodeTracker; +import org.apache.hadoop.hbase.zookeeper.RootRegionTracker; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.ipc.RemoteException; + +/** + * Tracks the availability of the catalog tables -ROOT- and + * .META.. + * + * This class is "read-only" in that the locations of the catalog tables cannot + * be explicitly set. Instead, ZooKeeper is used to learn of the availability + * and location of -ROOT-. -ROOT- is used to learn of + * the location of .META. If not available in -ROOT-, + * ZooKeeper is used to monitor for a new location of .META.. + * + *

Call {@link #start()} to start up operation. Call {@link #stop()}} to + * interrupt waits and close up shop. + */ +public class CatalogTracker { + // TODO: This class needs a rethink. The original intent was that it would be + // the one-stop-shop for root and meta locations and that it would get this + // info from reading and watching zk state. The class was to be used by + // servers when they needed to know of root and meta movement but also by + // client-side (inside in HTable) so rather than figure root and meta + // locations on fault, the client would instead get notifications out of zk. + // + // But this original intent is frustrated by the fact that this class has to + // read an hbase table, the -ROOT- table, to figure out the .META. region + // location which means we depend on an HConnection. HConnection will do + // retrying but also, it has its own mechanism for finding root and meta + // locations (and for 'verifying'; it tries the location and if it fails, does + // new lookup, etc.). So, at least for now, HConnection (or HTable) can't + // have a CT since CT needs a HConnection (Even then, do want HT to have a CT? + // For HT keep up a session with ZK? Rather, shouldn't we do like asynchbase + // where we'd open a connection to zk, read what we need then let the + // connection go?). The 'fix' is make it so both root and meta addresses + // are wholey up in zk -- not in zk (root) -- and in an hbase table (meta). + // + // But even then, this class does 'verification' of the location and it does + // this by making a call over an HConnection (which will do its own root + // and meta lookups). Isn't this verification 'useless' since when we + // return, whatever is dependent on the result of this call then needs to + // use HConnection; what we have verified may change in meantime (HConnection + // uses the CT primitives, the root and meta trackers finding root locations). + // + // When meta is moved to zk, this class may make more sense. In the + // meantime, it does not cohere. It should just watch meta and root and not + // NOT do verification -- let that be out in HConnection since its going to + // be done there ultimately anyways. + // + // This class has spread throughout the codebase. It needs to be reigned in. + // This class should be used server-side only, even if we move meta location + // up into zk. Currently its used over in the client package. Its used in + // MetaReader and MetaEditor classes usually just to get the Configuration + // its using (It does this indirectly by asking its HConnection for its + // Configuration and even then this is just used to get an HConnection out on + // the other end). I made https://issues.apache.org/jira/browse/HBASE-4495 for + // doing CT fixup. St.Ack 09/30/2011. + // + + // TODO: Timeouts have never been as advertised in here and its worse now + // with retries; i.e. the HConnection retries and pause goes ahead whatever + // the passed timeout is. Fix. + private static final Log LOG = LogFactory.getLog(CatalogTracker.class); + private final HConnection connection; + private final ZooKeeperWatcher zookeeper; + private final RootRegionTracker rootRegionTracker; + private final MetaNodeTracker metaNodeTracker; + private final AtomicBoolean metaAvailable = new AtomicBoolean(false); + private boolean instantiatedzkw = false; + private Abortable abortable; + + /* + * Do not clear this address once set. Its needed when we do + * server shutdown processing -- we need to know who had .META. last. If you + * want to know if the address is good, rely on {@link #metaAvailable} value. + */ + private ServerName metaLocation; + + private volatile boolean stopped = false; + + static final byte [] ROOT_REGION_NAME = + HRegionInfo.ROOT_REGIONINFO.getRegionName(); + static final byte [] META_REGION_NAME = + HRegionInfo.FIRST_META_REGIONINFO.getRegionName(); + + /** + * Constructs a catalog tracker. Find current state of catalog tables. + * Begin active tracking by executing {@link #start()} post construction. Does + * not timeout. + * + * @param conf + * the {@link Configuration} from which a {@link HConnection} will be + * obtained; if problem, this connections + * {@link HConnection#abort(String, Throwable)} will be called. + * @throws IOException + */ + public CatalogTracker(final Configuration conf) throws IOException { + this(null, conf, null); + } + + /** + * Constructs the catalog tracker. Find current state of catalog tables. + * Begin active tracking by executing {@link #start()} post construction. + * Does not timeout. + * @param zk If zk is null, we'll create an instance (and shut it down + * when {@link #stop()} is called) else we'll use what is passed. + * @param conf + * @param abortable If fatal exception we'll call abort on this. May be null. + * If it is we'll use the Connection associated with the passed + * {@link Configuration} as our Abortable. + * @throws IOException + */ + public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf, + Abortable abortable) + throws IOException { + this(zk, conf, HConnectionManager.getConnection(conf), abortable); + } + + CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf, + HConnection connection, Abortable abortable) + throws IOException { + this.connection = connection; + if (abortable == null) { + // A connection is abortable. + this.abortable = this.connection; + } + Abortable throwableAborter = new Abortable() { + + @Override + public void abort(String why, Throwable e) { + throw new RuntimeException(why, e); + } + + @Override + public boolean isAborted() { + return true; + } + + }; + if (zk == null) { + // Create our own. Set flag so we tear it down on stop. + this.zookeeper = + new ZooKeeperWatcher(conf, "catalogtracker-on-" + connection.toString(), + abortable); + instantiatedzkw = true; + } else { + this.zookeeper = zk; + } + this.rootRegionTracker = new RootRegionTracker(zookeeper, throwableAborter); + final CatalogTracker ct = this; + // Override nodeDeleted so we get notified when meta node deleted + this.metaNodeTracker = new MetaNodeTracker(zookeeper, throwableAborter) { + public void nodeDeleted(String path) { + if (!path.equals(node)) return; + ct.resetMetaLocation(); + } + }; + } + + /** + * Starts the catalog tracker. + * Determines current availability of catalog tables and ensures all further + * transitions of either region are tracked. + * @throws IOException + * @throws InterruptedException + */ + public void start() throws IOException, InterruptedException { + LOG.debug("Starting catalog tracker " + this); + try { + this.rootRegionTracker.start(); + this.metaNodeTracker.start(); + } catch (RuntimeException e) { + Throwable t = e.getCause(); + this.abortable.abort(e.getMessage(), t); + throw new IOException("Attempt to start root/meta tracker failed.", t); + } + } + + /** + * Stop working. + * Interrupts any ongoing waits. + */ + public void stop() { + if (!this.stopped) { + LOG.debug("Stopping catalog tracker " + this); + this.stopped = true; + this.rootRegionTracker.stop(); + this.metaNodeTracker.stop(); + try { + if (this.connection != null) { + this.connection.close(); + } + } catch (IOException e) { + // Although the {@link Closeable} interface throws an {@link + // IOException}, in reality, the implementation would never do that. + LOG.error("Attempt to close catalog tracker's connection failed.", e); + } + if (this.instantiatedzkw) { + this.zookeeper.close(); + } + // Call this and it will interrupt any ongoing waits on meta. + synchronized (this.metaAvailable) { + this.metaAvailable.notifyAll(); + } + } + } + + /** + * Gets the current location for -ROOT- or null if location is + * not currently available. + * @return {@link ServerName} for server hosting -ROOT- or null + * if none available + * @throws InterruptedException + */ + public ServerName getRootLocation() throws InterruptedException { + return this.rootRegionTracker.getRootRegionLocation(); + } + + /** + * @return {@link ServerName} for server hosting .META. or null + * if none available + */ + public ServerName getMetaLocation() { + return this.metaLocation; + } + + /** + * Method used by master on startup trying to figure state of cluster. + * Returns the current meta location unless its null. In this latter case, + * it has not yet been set so go check whats up in -ROOT- and + * return that. + * @return {@link ServerName} for server hosting .META. or if null, + * we'll read the location that is up in -ROOT- table (which + * could be null or just plain stale). + * @throws IOException + */ + public ServerName getMetaLocationOrReadLocationFromRoot() throws IOException { + ServerName sn = getMetaLocation(); + return sn != null? sn: MetaReader.getMetaRegionLocation(this); + } + + /** + * Waits indefinitely for availability of -ROOT-. Used during + * cluster startup. + * @throws InterruptedException if interrupted while waiting + */ + public void waitForRoot() + throws InterruptedException { + this.rootRegionTracker.blockUntilAvailable(); + } + + /** + * Gets the current location for -ROOT- if available and waits + * for up to the specified timeout if not immediately available. Returns null + * if the timeout elapses before root is available. + * @param timeout maximum time to wait for root availability, in milliseconds + * @return {@link ServerName} for server hosting -ROOT- or null + * if none available + * @throws InterruptedException if interrupted while waiting + * @throws NotAllMetaRegionsOnlineException if root not available before + * timeout + */ + ServerName waitForRoot(final long timeout) + throws InterruptedException, NotAllMetaRegionsOnlineException { + ServerName sn = rootRegionTracker.waitRootRegionLocation(timeout); + if (sn == null) { + throw new NotAllMetaRegionsOnlineException("Timed out; " + timeout + "ms"); + } + return sn; + } + + /** + * Gets a connection to the server hosting root, as reported by ZooKeeper, + * waiting up to the specified timeout for availability. + * @param timeout How long to wait on root location + * @see #waitForRoot(long) for additional information + * @return connection to server hosting root + * @throws InterruptedException + * @throws NotAllMetaRegionsOnlineException if timed out waiting + * @throws IOException + * @deprecated Use #getRootServerConnection(long) + */ + public HRegionInterface waitForRootServerConnection(long timeout) + throws InterruptedException, NotAllMetaRegionsOnlineException, IOException { + return getRootServerConnection(timeout); + } + + /** + * Gets a connection to the server hosting root, as reported by ZooKeeper, + * waiting up to the specified timeout for availability. + *

WARNING: Does not retry. Use an {@link HTable} instead. + * @param timeout How long to wait on root location + * @see #waitForRoot(long) for additional information + * @return connection to server hosting root + * @throws InterruptedException + * @throws NotAllMetaRegionsOnlineException if timed out waiting + * @throws IOException + */ + HRegionInterface getRootServerConnection(long timeout) + throws InterruptedException, NotAllMetaRegionsOnlineException, IOException { + return getCachedConnection(waitForRoot(timeout)); + } + + /** + * Gets a connection to the server currently hosting .META. or + * null if location is not currently available. + *

+ * If a location is known, a connection to the cached location is returned. + * If refresh is true, the cached connection is verified first before + * returning. If the connection is not valid, it is reset and rechecked. + *

+ * If no location for meta is currently known, method checks ROOT for a new + * location, verifies META is currently there, and returns a cached connection + * to the server hosting META. + * + * @return connection to server hosting meta, null if location not available + * @throws IOException + * @throws InterruptedException + */ + private HRegionInterface getMetaServerConnection() + throws IOException, InterruptedException { + synchronized (metaAvailable) { + if (metaAvailable.get()) { + HRegionInterface current = getCachedConnection(this.metaLocation); + // If we are to refresh, verify we have a good connection by making + // an invocation on it. + if (verifyRegionLocation(current, this.metaLocation, META_REGION_NAME)) { + return current; + } + resetMetaLocation(); + } + // We got here because there is no meta available or because whats + // available is bad. + + // Now read the current .META. content from -ROOT-. Note: This goes via + // an HConnection. It has its own way of figuring root and meta locations + // which we have to wait on. + ServerName newLocation = MetaReader.getMetaRegionLocation(this); + if (newLocation == null) return null; + + HRegionInterface newConnection = getCachedConnection(newLocation); + if (verifyRegionLocation(newConnection, newLocation, META_REGION_NAME)) { + setMetaLocation(newLocation); + return newConnection; + } else { + if (LOG.isTraceEnabled()) { + LOG.trace("New .META. server: " + newLocation + " isn't valid." + + " Cached .META. server: " + this.metaLocation); + } + } + return null; + } + } + + /** + * Waits indefinitely for availability of .META.. Used during + * cluster startup. Does not verify meta, just that something has been + * set up in zk. + * @see #waitForMeta(long) + * @throws InterruptedException if interrupted while waiting + */ + public void waitForMeta() throws InterruptedException { + while (!this.stopped) { + try { + if (waitForMeta(100) != null) break; + } catch (NotAllMetaRegionsOnlineException e) { + if (LOG.isTraceEnabled()) { + LOG.info(".META. still not available, sleeping and retrying." + + " Reason: " + e.getMessage()); + } + } catch (IOException e) { + LOG.info("Retrying", e); + } + } + } + + /** + * Gets the current location for .META. if available and waits + * for up to the specified timeout if not immediately available. Throws an + * exception if timed out waiting. This method differs from {@link #waitForMeta()} + * in that it will go ahead and verify the location gotten from ZooKeeper and + * -ROOT- region by trying to use returned connection. + * @param timeout maximum time to wait for meta availability, in milliseconds + * @return {@link ServerName} for server hosting .META. or null + * if none available + * @throws InterruptedException if interrupted while waiting + * @throws IOException unexpected exception connecting to meta server + * @throws NotAllMetaRegionsOnlineException if meta not available before + * timeout + */ + public ServerName waitForMeta(long timeout) + throws InterruptedException, IOException, NotAllMetaRegionsOnlineException { + long stop = timeout == 0 ? Long.MAX_VALUE : System.currentTimeMillis() + timeout; + long waitTime = Math.min(50, timeout); + synchronized (metaAvailable) { + while(!stopped && System.currentTimeMillis() < stop) { + if (getMetaServerConnection() != null) { + return metaLocation; + } + // perhaps -ROOT- region isn't available, let us wait a bit and retry. + metaAvailable.wait(waitTime); + } + if (getMetaServerConnection() == null) { + throw new NotAllMetaRegionsOnlineException("Timed out (" + timeout + "ms)"); + } + return metaLocation; + } + } + + /** + * Gets a connection to the server hosting meta, as reported by ZooKeeper, + * waiting up to the specified timeout for availability. + * @see #waitForMeta(long) for additional information + * @return connection to server hosting meta + * @throws InterruptedException + * @throws NotAllMetaRegionsOnlineException if timed out waiting + * @throws IOException + * @deprecated Does not retry; use an HTable instance instead. + */ + public HRegionInterface waitForMetaServerConnection(long timeout) + throws InterruptedException, NotAllMetaRegionsOnlineException, IOException { + return getCachedConnection(waitForMeta(timeout)); + } + + /** + * Called when we figure current meta is off (called from zk callback). + */ + public void resetMetaLocation() { + LOG.debug("Current cached META location, " + metaLocation + + ", is not valid, resetting"); + synchronized(this.metaAvailable) { + this.metaAvailable.set(false); + this.metaAvailable.notifyAll(); + } + } + + /** + * @param metaLocation + */ + void setMetaLocation(final ServerName metaLocation) { + LOG.debug("Set new cached META location: " + metaLocation); + synchronized (this.metaAvailable) { + this.metaLocation = metaLocation; + this.metaAvailable.set(true); + // no synchronization because these are private and already under lock + this.metaAvailable.notifyAll(); + } + } + + /** + * @param sn ServerName to get a connection against. + * @return The HRegionInterface we got when we connected to sn + * May have come from cache, may not be good, may have been setup by this + * invocation, or may be null. + * @throws IOException + */ + private HRegionInterface getCachedConnection(ServerName sn) + throws IOException { + if (sn == null) { + return null; + } + HRegionInterface protocol = null; + try { + protocol = connection.getHRegionConnection(sn.getHostname(), sn.getPort()); + } catch (RetriesExhaustedException e) { + if (e.getCause() != null && e.getCause() instanceof ConnectException) { + // Catch this; presume it means the cached connection has gone bad. + } else { + throw e; + } + } catch (SocketTimeoutException e) { + LOG.debug("Timed out connecting to " + sn); + } catch (NoRouteToHostException e) { + LOG.debug("Connecting to " + sn, e); + } catch (SocketException e) { + LOG.debug("Exception connecting to " + sn); + } catch (UnknownHostException e) { + LOG.debug("Unknown host exception connecting to " + sn); + } catch (IOException ioe) { + Throwable cause = ioe.getCause(); + if (ioe instanceof ConnectException) { + // Catch. Connect refused. + } else if (cause != null && cause instanceof EOFException) { + // Catch. Other end disconnected us. + } else if (cause != null && cause.getMessage() != null && + cause.getMessage().toLowerCase().contains("connection reset")) { + // Catch. Connection reset. + } else { + throw ioe; + } + + } + return protocol; + } + + /** + * Verify we can connect to hostingServer and that its carrying + * regionName. + * @param hostingServer Interface to the server hosting regionName + * @param serverName The servername that goes with the metaServer + * Interface. Used logging. + * @param regionName The regionname we are interested in. + * @return True if we were able to verify the region located at other side of + * the Interface. + * @throws IOException + */ + // TODO: We should be able to get the ServerName from the HRegionInterface + // rather than have to pass it in. Its made awkward by the fact that the + // HRI is likely a proxy against remote server so the getServerName needs + // to be fixed to go to a local method or to a cache before we can do this. + private boolean verifyRegionLocation(HRegionInterface hostingServer, + final ServerName address, final byte [] regionName) + throws IOException { + if (hostingServer == null) { + LOG.info("Passed hostingServer is null"); + return false; + } + Throwable t = null; + try { + // Try and get regioninfo from the hosting server. + return hostingServer.getRegionInfo(regionName) != null; + } catch (ConnectException e) { + t = e; + } catch (RetriesExhaustedException e) { + t = e; + } catch (RemoteException e) { + IOException ioe = e.unwrapRemoteException(); + t = ioe; + } catch (IOException e) { + Throwable cause = e.getCause(); + if (cause != null && cause instanceof EOFException) { + t = cause; + } else if (cause != null && cause.getMessage() != null + && cause.getMessage().contains("Connection reset")) { + t = cause; + } else { + t = e; + } + } + LOG.info("Failed verification of " + Bytes.toStringBinary(regionName) + + " at address=" + address + "; " + t); + return false; + } + + /** + * Verify -ROOT- is deployed and accessible. + * @param timeout How long to wait on zk for root address (passed through to + * the internal call to {@link #waitForRootServerConnection(long)}. + * @return True if the -ROOT- location is healthy. + * @throws IOException + * @throws InterruptedException + */ + public boolean verifyRootRegionLocation(final long timeout) + throws InterruptedException, IOException { + HRegionInterface connection = null; + try { + connection = waitForRootServerConnection(timeout); + } catch (NotAllMetaRegionsOnlineException e) { + // Pass + } catch (ServerNotRunningYetException e) { + // Pass -- remote server is not up so can't be carrying root + } catch (UnknownHostException e) { + // Pass -- server name doesn't resolve so it can't be assigned anything. + } + return (connection == null)? false: + verifyRegionLocation(connection, + this.rootRegionTracker.getRootRegionLocation(), ROOT_REGION_NAME); + } + + /** + * Verify .META. is deployed and accessible. + * @param timeout How long to wait on zk for .META. address + * (passed through to the internal call to {@link #waitForMetaServerConnection(long)}. + * @return True if the .META. location is healthy. + * @throws IOException Some unexpected IOE. + * @throws InterruptedException + */ + public boolean verifyMetaRegionLocation(final long timeout) + throws InterruptedException, IOException { + HRegionInterface connection = null; + try { + connection = waitForMetaServerConnection(timeout); + } catch (NotAllMetaRegionsOnlineException e) { + // Pass + } catch (ServerNotRunningYetException e) { + // Pass -- remote server is not up so can't be carrying .META. + } catch (UnknownHostException e) { + // Pass -- server name doesn't resolve so it can't be assigned anything. + } catch (RetriesExhaustedException e) { + // Pass -- failed after bunch of retries. + LOG.debug("Failed verify meta region location after retries", e); + } + return connection != null; + } + + // Used by tests. + MetaNodeTracker getMetaNodeTracker() { + return this.metaNodeTracker; + } + + public HConnection getConnection() { + return this.connection; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java new file mode 100644 index 0000000..94ffeb6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java @@ -0,0 +1,396 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.catalog; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.ConnectException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.PairOfSameType; +import org.apache.hadoop.hbase.util.Writables; + +/** + * Writes region and assignment information to .META.. + * TODO: Put MetaReader and MetaEditor together; doesn't make sense having + * them distinct. + */ +public class MetaEditor { + // TODO: Strip CatalogTracker from this class. Its all over and in the end + // its only used to get its Configuration so we can get associated + // Connection. + private static final Log LOG = LogFactory.getLog(MetaEditor.class); + + private static Put makePutFromRegionInfo(HRegionInfo regionInfo) + throws IOException { + Put put = new Put(regionInfo.getRegionName()); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(regionInfo)); + return put; + } + + /** + * Put the passed p to the .META. table. + * @param ct CatalogTracker on whose back we will ride the edit. + * @param p Put to add to .META. + * @throws IOException + */ + static void putToMetaTable(final CatalogTracker ct, final Put p) + throws IOException { + put(MetaReader.getMetaHTable(ct), p); + } + + /** + * Put the passed p to the .META. table. + * @param ct CatalogTracker on whose back we will ride the edit. + * @param p Put to add to .META. + * @throws IOException + */ + static void putToRootTable(final CatalogTracker ct, final Put p) + throws IOException { + put(MetaReader.getRootHTable(ct), p); + } + + /** + * Put the passed p to a catalog table. + * @param ct CatalogTracker on whose back we will ride the edit. + * @param p Put to add + * @throws IOException + */ + static void putToCatalogTable(final CatalogTracker ct, final Put p) + throws IOException { + HTable t = MetaReader.getCatalogHTable(ct, p.getRow()); + put(t, p); + } + + /** + * @param t Table to use (will be closed when done). + * @param p + * @throws IOException + */ + private static void put(final HTable t, final Put p) throws IOException { + try { + t.put(p); + } finally { + t.close(); + } + } + + /** + * Put the passed ps to the .META. table. + * @param ct CatalogTracker on whose back we will ride the edit. + * @param ps Put to add to .META. + * @throws IOException + */ + static void putsToMetaTable(final CatalogTracker ct, final List ps) + throws IOException { + HTable t = MetaReader.getMetaHTable(ct); + try { + t.put(ps); + } finally { + t.close(); + } + } + + /** + * Delete the passed d from the .META. table. + * @param ct CatalogTracker on whose back we will ride the edit. + * @param d Delete to add to .META. + * @throws IOException + */ + static void deleteFromMetaTable(final CatalogTracker ct, final Delete d) + throws IOException { + List dels = new ArrayList(1); + dels.add(d); + deleteFromMetaTable(ct, dels); + } + + /** + * Delete the passed deletes from the .META. table. + * @param ct CatalogTracker on whose back we will ride the edit. + * @param deletes Deletes to add to .META. This list should support #remove. + * @throws IOException + */ + public static void deleteFromMetaTable(final CatalogTracker ct, final List deletes) + throws IOException { + HTable t = MetaReader.getMetaHTable(ct); + try { + t.delete(deletes); + } finally { + t.close(); + } + } + + /** + * Execute the passed mutations against .META. table. + * @param ct CatalogTracker on whose back we will ride the edit. + * @param mutations Puts and Deletes to execute on .META. + * @throws IOException + */ + static void mutateMetaTable(final CatalogTracker ct, final List mutations) + throws IOException { + HTable t = MetaReader.getMetaHTable(ct); + try { + t.batch(mutations); + } catch (InterruptedException e) { + InterruptedIOException ie = new InterruptedIOException(e.getMessage()); + ie.initCause(e); + throw ie; + } finally { + t.close(); + } + } + + /** + * Adds a META row for the specified new region. + * @param regionInfo region information + * @throws IOException if problem connecting or updating meta + */ + public static void addRegionToMeta(CatalogTracker catalogTracker, + HRegionInfo regionInfo) + throws IOException { + putToMetaTable(catalogTracker, makePutFromRegionInfo(regionInfo)); + LOG.info("Added region " + regionInfo.getRegionNameAsString() + " to META"); + } + + /** + * Adds a META row for each of the specified new regions. + * @param catalogTracker CatalogTracker + * @param regionInfos region information list + * @throws IOException if problem connecting or updating meta + */ + public static void addRegionsToMeta(CatalogTracker catalogTracker, + List regionInfos) + throws IOException { + List puts = new ArrayList(); + for (HRegionInfo regionInfo : regionInfos) { + puts.add(makePutFromRegionInfo(regionInfo)); + } + putsToMetaTable(catalogTracker, puts); + LOG.info("Added " + puts.size() + " regions in META"); + } + + /** + * Offline parent in meta. + * Used when splitting. + * @param catalogTracker + * @param parent + * @param a Split daughter region A + * @param b Split daughter region B + * @throws NotAllMetaRegionsOnlineException + * @throws IOException + */ + public static void offlineParentInMeta(CatalogTracker catalogTracker, + HRegionInfo parent, final HRegionInfo a, final HRegionInfo b) + throws NotAllMetaRegionsOnlineException, IOException { + HRegionInfo copyOfParent = new HRegionInfo(parent); + copyOfParent.setOffline(true); + copyOfParent.setSplit(true); + Put put = new Put(copyOfParent.getRegionName()); + addRegionInfo(put, copyOfParent); + put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER, + Writables.getBytes(a)); + put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER, + Writables.getBytes(b)); + putToMetaTable(catalogTracker, put); + LOG.info("Offlined parent region " + parent.getRegionNameAsString() + + " in META"); + } + + public static void addDaughter(final CatalogTracker catalogTracker, + final HRegionInfo regionInfo, final ServerName sn) + throws NotAllMetaRegionsOnlineException, IOException { + Put put = new Put(regionInfo.getRegionName()); + addRegionInfo(put, regionInfo); + if (sn != null) addLocation(put, sn); + putToMetaTable(catalogTracker, put); + LOG.info("Added daughter " + regionInfo.getRegionNameAsString() + + (sn == null? ", serverName=null": ", serverName=" + sn.toString())); + } + + /** + * Updates the location of the specified META region in ROOT to be the + * specified server hostname and startcode. + *

+ * Uses passed catalog tracker to get a connection to the server hosting + * ROOT and makes edits to that region. + * + * @param catalogTracker catalog tracker + * @param regionInfo region to update location of + * @param sn Server name + * @throws IOException + * @throws ConnectException Usually because the regionserver carrying .META. + * is down. + * @throws NullPointerException Because no -ROOT- server connection + */ + public static void updateMetaLocation(CatalogTracker catalogTracker, + HRegionInfo regionInfo, ServerName sn) + throws IOException, ConnectException { + updateLocation(catalogTracker, regionInfo, sn); + } + + /** + * Updates the location of the specified region in META to be the specified + * server hostname and startcode. + *

+ * Uses passed catalog tracker to get a connection to the server hosting + * META and makes edits to that region. + * + * @param catalogTracker catalog tracker + * @param regionInfo region to update location of + * @param sn Server name + * @throws IOException + */ + public static void updateRegionLocation(CatalogTracker catalogTracker, + HRegionInfo regionInfo, ServerName sn) + throws IOException { + updateLocation(catalogTracker, regionInfo, sn); + } + + /** + * Updates the location of the specified region to be the specified server. + *

+ * Connects to the specified server which should be hosting the specified + * catalog region name to perform the edit. + * + * @param catalogTracker + * @param regionInfo region to update location of + * @param sn Server name + * @throws IOException In particular could throw {@link java.net.ConnectException} + * if the server is down on other end. + */ + private static void updateLocation(final CatalogTracker catalogTracker, + HRegionInfo regionInfo, ServerName sn) + throws IOException { + Put put = new Put(regionInfo.getRegionName()); + addLocation(put, sn); + putToCatalogTable(catalogTracker, put); + LOG.info("Updated row " + regionInfo.getRegionNameAsString() + + " with server=" + sn); + } + + /** + * Deletes the specified region from META. + * @param catalogTracker + * @param regionInfo region to be deleted from META + * @throws IOException + */ + public static void deleteRegion(CatalogTracker catalogTracker, + HRegionInfo regionInfo) + throws IOException { + Delete delete = new Delete(regionInfo.getRegionName()); + deleteFromMetaTable(catalogTracker, delete); + LOG.info("Deleted region " + regionInfo.getRegionNameAsString() + " from META"); + } + + /** + * Deletes the specified regions from META. + * @param catalogTracker + * @param regionsInfo list of regions to be deleted from META + * @throws IOException + */ + public static void deleteRegions(CatalogTracker catalogTracker, + List regionsInfo) throws IOException { + List deletes = new ArrayList(regionsInfo.size()); + for (HRegionInfo hri: regionsInfo) { + deletes.add(new Delete(hri.getRegionName())); + } + deleteFromMetaTable(catalogTracker, deletes); + LOG.info("Deleted from META, regions: " + regionsInfo); + } + + /** + * Adds and Removes the specified regions from .META. + * @param catalogTracker + * @param regionsToRemove list of regions to be deleted from META + * @param regionsToAdd list of regions to be added to META + * @throws IOException + */ + public static void mutateRegions(CatalogTracker catalogTracker, + final List regionsToRemove, final List regionsToAdd) + throws IOException { + List mutation = new ArrayList(); + if (regionsToRemove != null) { + for (HRegionInfo hri: regionsToRemove) { + mutation.add(new Delete(hri.getRegionName())); + } + } + if (regionsToAdd != null) { + for (HRegionInfo hri: regionsToAdd) { + mutation.add(makePutFromRegionInfo(hri)); + } + } + mutateMetaTable(catalogTracker, mutation); + if (regionsToRemove != null && regionsToRemove.size() > 0) { + LOG.debug("Deleted from META, regions: " + regionsToRemove); + } + if (regionsToAdd != null && regionsToAdd.size() > 0) { + LOG.debug("Add to META, regions: " + regionsToAdd); + } + } + + public static HRegionInfo getHRegionInfo( + Result data) throws IOException { + byte [] bytes = + data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + if (bytes == null) return null; + HRegionInfo info = Writables.getHRegionInfo(bytes); + LOG.info("Current INFO from scan results = " + info); + return info; + } + + /** + * Returns the daughter regions by reading from the corresponding columns of the .META. table + * Result. If the region is not a split parent region, it returns PairOfSameType(null, null). + */ + public static PairOfSameType getDaughterRegions(Result data) throws IOException { + HRegionInfo splitA = Writables.getHRegionInfoOrNull( + data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER)); + HRegionInfo splitB = Writables.getHRegionInfoOrNull( + data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER)); + return new PairOfSameType(splitA, splitB); + } + + private static Put addRegionInfo(final Put p, final HRegionInfo hri) + throws IOException { + p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + return p; + } + + private static Put addLocation(final Put p, final ServerName sn) { + p.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, + Bytes.toBytes(sn.getHostAndPort())); + p.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, + Bytes.toBytes(sn.getStartcode())); + return p; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java new file mode 100644 index 0000000..01aa515 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java @@ -0,0 +1,275 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.catalog; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.catalog.MetaReader.Visitor; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.migration.HRegionInfo090x; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; + +/** + * Tools to help with migration of meta tables so they no longer host + * instances of HTableDescriptor. + * @deprecated Used migration from 0.90 to 0.92 so will be going away in next + * release + */ +public class MetaMigrationRemovingHTD { + private static final Log LOG = LogFactory.getLog(MetaMigrationRemovingHTD.class); + + /** + * Update legacy META rows, removing HTD from HRI. + * @param masterServices + * @return List of table descriptors. + * @throws IOException + */ + public static Set updateMetaWithNewRegionInfo( + final MasterServices masterServices) + throws IOException { + MigratingVisitor v = new MigratingVisitor(masterServices); + MetaReader.fullScan(masterServices.getCatalogTracker(), v); + updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker()); + return v.htds; + } + + /** + * Update the ROOT with new HRI. (HRI with no HTD) + * @param masterServices + * @return List of table descriptors + * @throws IOException + */ + static Set updateRootWithNewRegionInfo( + final MasterServices masterServices) + throws IOException { + MigratingVisitor v = new MigratingVisitor(masterServices); + MetaReader.fullScan(masterServices.getCatalogTracker(), v, null, true); + return v.htds; + } + + /** + * Meta visitor that migrates the info:regioninfo as it visits. + */ + static class MigratingVisitor implements Visitor { + private final MasterServices services; + final Set htds = new HashSet(); + + MigratingVisitor(final MasterServices services) { + this.services = services; + } + + @Override + public boolean visit(Result r) throws IOException { + if (r == null || r.isEmpty()) return true; + // Check info:regioninfo, info:splitA, and info:splitB. Make sure all + // have migrated HRegionInfos... that there are no leftover 090 version + // HRegionInfos. + byte [] hriBytes = getBytes(r, HConstants.REGIONINFO_QUALIFIER); + // Presumes that an edit updating all three cells either succeeds or + // doesn't -- that we don't have case of info:regioninfo migrated but not + // info:splitA. + if (isMigrated(hriBytes)) return true; + // OK. Need to migrate this row in meta. + HRegionInfo090x hri090 = getHRegionInfo090x(hriBytes); + HTableDescriptor htd = hri090.getTableDesc(); + if (htd == null) { + LOG.warn("A 090 HRI has null HTD? Continuing; " + hri090.toString()); + return true; + } + if (!this.htds.contains(htd)) { + // If first time we are adding a table, then write it out to fs. + // Presumes that first region in table has THE table's schema which + // might not be too bad of a presumption since it'll be first region + // 'altered' + this.services.getMasterFileSystem().createTableDescriptor(htd); + this.htds.add(htd); + } + // This will 'migrate' the hregioninfo from 090 version to 092. + HRegionInfo hri = new HRegionInfo(hri090); + // Now make a put to write back to meta. + Put p = new Put(hri.getRegionName()); + p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + // Now check info:splitA and info:splitB if present. Migrate these too. + checkSplit(r, p, HConstants.SPLITA_QUALIFIER); + checkSplit(r, p, HConstants.SPLITB_QUALIFIER); + // Below we fake out putToCatalogTable + MetaEditor.putToCatalogTable(this.services.getCatalogTracker(), p); + LOG.info("Migrated " + Bytes.toString(p.getRow())); + return true; + } + } + + static void checkSplit(final Result r, final Put p, final byte [] which) + throws IOException { + byte [] hriSplitBytes = getBytes(r, which); + if (!isMigrated(hriSplitBytes)) { + // This will convert the HRI from 090 to 092 HRI. + HRegionInfo hri = Writables.getHRegionInfo(hriSplitBytes); + p.add(HConstants.CATALOG_FAMILY, which, Writables.getBytes(hri)); + } + } + + /** + * @param r Result to dig in. + * @param qualifier Qualifier to look at in the passed r. + * @return Bytes for an HRegionInfo or null if no bytes or empty bytes found. + */ + static byte [] getBytes(final Result r, final byte [] qualifier) { + byte [] hriBytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier); + if (hriBytes == null || hriBytes.length <= 0) return null; + return hriBytes; + } + + /** + * @param r Result to look in. + * @param qualifier What to look at in the passed result. + * @return Either a 090 vintage HRegionInfo OR null if no HRegionInfo or + * the HRegionInfo is up to date and not in need of migration. + * @throws IOException + */ + static HRegionInfo090x get090HRI(final Result r, final byte [] qualifier) + throws IOException { + byte [] hriBytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier); + if (hriBytes == null || hriBytes.length <= 0) return null; + if (isMigrated(hriBytes)) return null; + return getHRegionInfo090x(hriBytes); + } + + static boolean isMigrated(final byte [] hriBytes) { + if (hriBytes == null || hriBytes.length <= 0) return true; + // Else, what version this HRegionInfo instance is at. The first byte + // is the version byte in a serialized HRegionInfo. If its same as our + // current HRI, then nothing to do. + if (hriBytes[0] == HRegionInfo.VERSION) return true; + if (hriBytes[0] == HRegionInfo.VERSION_PRE_092) return false; + // Unknown version. Return true that its 'migrated' but log warning. + // Should 'never' happen. + assert false: "Unexpected version; bytes=" + Bytes.toStringBinary(hriBytes); + return true; + } + + /** + * Migrate root and meta to newer version. This updates the META and ROOT + * and removes the HTD from HRI. + * @param masterServices + * @throws IOException + */ + public static void migrateRootAndMeta(final MasterServices masterServices) + throws IOException { + updateRootWithNewRegionInfo(masterServices); + updateMetaWithNewRegionInfo(masterServices); + } + + /** + * Update the version flag in -ROOT-. + * @param catalogTracker + * @throws IOException + */ + public static void updateRootWithMetaMigrationStatus(final CatalogTracker catalogTracker) + throws IOException { + Put p = new Put(HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); + MetaEditor.putToRootTable(catalogTracker, setMetaVersion(p)); + LOG.info("Updated -ROOT- meta version=" + HConstants.META_VERSION); + } + + static Put setMetaVersion(final Put p) { + p.add(HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER, + Bytes.toBytes(HConstants.META_VERSION)); + return p; + } + + /** + * @return True if the meta table has been migrated. + * @throws IOException + */ + // Public because used in tests + public static boolean isMetaHRIUpdated(final MasterServices services) + throws IOException { + List results = MetaReader.fullScanOfRoot(services.getCatalogTracker()); + if (results == null || results.isEmpty()) { + LOG.info("Not migrated"); + return false; + } + // Presume only the one result because we only support on meta region. + Result r = results.get(0); + short version = getMetaVersion(r); + boolean migrated = version >= HConstants.META_VERSION; + LOG.info("Meta version=" + version + "; migrated=" + migrated); + return migrated; + } + + /** + * @param r Result to look at + * @return Current meta table version or -1 if no version found. + */ + static short getMetaVersion(final Result r) { + byte [] value = r.getValue(HConstants.CATALOG_FAMILY, + HConstants.META_VERSION_QUALIFIER); + return value == null || value.length <= 0? -1: Bytes.toShort(value); + } + + /** + * @return True if migrated. + * @throws IOException + */ + public static boolean updateMetaWithNewHRI(final MasterServices services) + throws IOException { + if (isMetaHRIUpdated(services)) { + LOG.info("ROOT/Meta already up-to date with new HRI."); + return true; + } + LOG.info("Meta has HRI with HTDs. Updating meta now."); + try { + migrateRootAndMeta(services); + LOG.info("ROOT and Meta updated with new HRI."); + return true; + } catch (IOException e) { + throw new RuntimeException("Update ROOT/Meta with new HRI failed." + + "Master startup aborted."); + } + } + + /** + * Get HREgionInfoForMigration serialized from bytes. + * @param bytes serialized bytes + * @return An instance of a 090 HRI or null if we failed deserialize + */ + public static HRegionInfo090x getHRegionInfo090x(final byte [] bytes) { + if (bytes == null || bytes.length == 0) return null; + HRegionInfo090x hri = null; + try { + hri = (HRegionInfo090x)Writables.getWritable(bytes, new HRegionInfo090x()); + } catch (IOException ioe) { + LOG.warn("Failed deserialize as a 090 HRegionInfo); bytes=" + + Bytes.toStringBinary(bytes), ioe); + } + return hri; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java new file mode 100644 index 0000000..cbb6d48 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java @@ -0,0 +1,784 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.catalog; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.NotServingRegionException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.PairOfSameType; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.ipc.RemoteException; + +/** + * Reads region and assignment information from .META.. + */ +public class MetaReader { + // TODO: Strip CatalogTracker from this class. Its all over and in the end + // its only used to get its Configuration so we can get associated + // Connection. + private static final Log LOG = LogFactory.getLog(MetaReader.class); + + static final byte [] META_REGION_PREFIX; + static { + // Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX. + // FIRST_META_REGIONINFO == '.META.,,1'. META_REGION_PREFIX == '.META.,' + int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2; + META_REGION_PREFIX = new byte [len]; + System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0, + META_REGION_PREFIX, 0, len); + } + + /** + * @param row + * @return True if row is row of -ROOT- table. + */ + private static boolean isRootTableRow(final byte [] row) { + if (row.length < META_REGION_PREFIX.length + 2 /* ',', + '1' */) { + // Can't be meta table region. + return false; + } + // Compare the prefix of row. If it matches META_REGION_PREFIX prefix, + // then this is row from -ROOT_ table. + return Bytes.equals(row, 0, META_REGION_PREFIX.length, + META_REGION_PREFIX, 0, META_REGION_PREFIX.length); + } + + /** + * Performs a full scan of .META., skipping regions from any + * tables in the specified set of disabled tables. + * @param catalogTracker + * @param disabledTables set of disabled tables that will not be returned + * @return Returns a map of every region to it's currently assigned server, + * according to META. If the region does not have an assignment it will have + * a null value in the map. + * @throws IOException + */ + public static Map fullScan( + CatalogTracker catalogTracker, final Set disabledTables) + throws IOException { + return fullScan(catalogTracker, disabledTables, false); + } + + /** + * Performs a full scan of .META., skipping regions from any + * tables in the specified set of disabled tables. + * @param catalogTracker + * @param disabledTables set of disabled tables that will not be returned + * @param excludeOfflinedSplitParents If true, do not include offlined split + * parents in the return. + * @return Returns a map of every region to it's currently assigned server, + * according to META. If the region does not have an assignment it will have + * a null value in the map. + * @throws IOException + */ + public static Map fullScan( + CatalogTracker catalogTracker, final Set disabledTables, + final boolean excludeOfflinedSplitParents) + throws IOException { + final Map regions = + new TreeMap(); + Visitor v = new Visitor() { + @Override + public boolean visit(Result r) throws IOException { + if (r == null || r.isEmpty()) return true; + Pair region = parseCatalogResult(r); + if (region == null) return true; + HRegionInfo hri = region.getFirst(); + if (hri == null) return true; + if (hri.getTableNameAsString() == null) return true; + if (disabledTables.contains( + hri.getTableNameAsString())) return true; + // Are we to include split parents in the list? + if (excludeOfflinedSplitParents && hri.isSplitParent()) return true; + regions.put(hri, region.getSecond()); + return true; + } + }; + fullScan(catalogTracker, v); + return regions; + } + + /** + * Performs a full scan of .META.. + * @return List of {@link Result} + * @throws IOException + */ + public static List fullScan(CatalogTracker catalogTracker) + throws IOException { + CollectAllVisitor v = new CollectAllVisitor(); + fullScan(catalogTracker, v, null); + return v.getResults(); + } + + /** + * Performs a full scan of a -ROOT- table. + * @return List of {@link Result} + * @throws IOException + */ + public static List fullScanOfRoot(CatalogTracker catalogTracker) + throws IOException { + CollectAllVisitor v = new CollectAllVisitor(); + fullScan(catalogTracker, v, null, true); + return v.getResults(); + } + + /** + * Performs a full scan of .META.. + * @param catalogTracker + * @param visitor Visitor invoked against each row. + * @throws IOException + */ + public static void fullScan(CatalogTracker catalogTracker, + final Visitor visitor) + throws IOException { + fullScan(catalogTracker, visitor, null); + } + + /** + * Performs a full scan of .META.. + * @param catalogTracker + * @param visitor Visitor invoked against each row. + * @param startrow Where to start the scan. Pass null if want to begin scan + * at first row (The visitor will stop the Scan when its done so no need to + * pass a stoprow). + * @throws IOException + */ + public static void fullScan(CatalogTracker catalogTracker, + final Visitor visitor, final byte [] startrow) + throws IOException { + fullScan(catalogTracker, visitor, startrow, false); + } + + /** + * Callers should call close on the returned {@link HTable} instance. + * @param catalogTracker We'll use this catalogtracker's connection + * @param tableName Table to get an {@link HTable} against. + * @return An {@link HTable} for tableName + * @throws IOException + */ + private static HTable getHTable(final CatalogTracker catalogTracker, + final byte [] tableName) + throws IOException { + // Passing the CatalogTracker's connection configuration ensures this + // HTable instance uses the CatalogTracker's connection. + org.apache.hadoop.hbase.client.HConnection c = catalogTracker.getConnection(); + if (c == null) throw new NullPointerException("No connection"); + return new HTable(catalogTracker.getConnection().getConfiguration(), tableName); + } + + /** + * Callers should call close on the returned {@link HTable} instance. + * @param catalogTracker + * @param row Row we are putting + * @return + * @throws IOException + */ + static HTable getCatalogHTable(final CatalogTracker catalogTracker, + final byte [] row) + throws IOException { + return isRootTableRow(row)? + getRootHTable(catalogTracker): + getMetaHTable(catalogTracker); + } + + /** + * Callers should call close on the returned {@link HTable} instance. + * @param ct + * @return An {@link HTable} for .META. + * @throws IOException + */ + static HTable getMetaHTable(final CatalogTracker ct) + throws IOException { + return getHTable(ct, HConstants.META_TABLE_NAME); + } + + /** + * Callers should call close on the returned {@link HTable} instance. + * @param ct + * @return An {@link HTable} for -ROOT- + * @throws IOException + */ + static HTable getRootHTable(final CatalogTracker ct) + throws IOException { + return getHTable(ct, HConstants.ROOT_TABLE_NAME); + } + + /** + * @param t Table to use (will be closed when done). + * @param g Get to run + * @throws IOException + */ + private static Result get(final HTable t, final Get g) throws IOException { + try { + return t.get(g); + } finally { + t.close(); + } + } + + /** + * Reads the location of META from ROOT. + * @param metaServer connection to server hosting ROOT + * @return location of META in ROOT where location, or null if not available + * @throws IOException + * @deprecated Does not retry; use #getMetaRegionLocation(CatalogTracker) + */ + public static ServerName readMetaLocation(HRegionInterface metaServer) + throws IOException { + return readLocation(metaServer, CatalogTracker.ROOT_REGION_NAME, + CatalogTracker.META_REGION_NAME); + } + + /** + * Gets the location of .META. region by reading content of + * -ROOT-. + * @param ct + * @return location of .META. region as a {@link ServerName} or + * null if not found + * @throws IOException + */ + static ServerName getMetaRegionLocation(final CatalogTracker ct) + throws IOException { + return MetaReader.readRegionLocation(ct, CatalogTracker.META_REGION_NAME); + } + + /** + * Reads the location of the specified region + * @param catalogTracker + * @param regionName region whose location we are after + * @return location of region as a {@link ServerName} or null if not found + * @throws IOException + */ + static ServerName readRegionLocation(CatalogTracker catalogTracker, + byte [] regionName) + throws IOException { + Pair pair = getRegion(catalogTracker, regionName); + return (pair == null || pair.getSecond() == null)? null: pair.getSecond(); + } + + // TODO: Remove when deprecated dependencies are removed. + private static ServerName readLocation(HRegionInterface metaServer, + byte [] catalogRegionName, byte [] regionName) + throws IOException { + Result r = null; + try { + r = metaServer.get(catalogRegionName, + new Get(regionName). + addColumn(HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER). + addColumn(HConstants.CATALOG_FAMILY, + HConstants.STARTCODE_QUALIFIER)); + } catch (java.net.SocketTimeoutException e) { + // Treat this exception + message as unavailable catalog table. Catch it + // and fall through to return a null + } catch (java.net.SocketException e) { + // Treat this exception + message as unavailable catalog table. Catch it + // and fall through to return a null + } catch (RemoteException re) { + IOException ioe = re.unwrapRemoteException(); + if (ioe instanceof NotServingRegionException) { + // Treat this NSRE as unavailable table. Catch and fall through to + // return null below + } else if (ioe.getMessage().contains("Server not running")) { + // Treat as unavailable table. + } else { + throw re; + } + } catch (IOException e) { + if (e.getCause() != null && e.getCause() instanceof IOException && + e.getCause().getMessage() != null && + e.getCause().getMessage().contains("Connection reset by peer")) { + // Treat this exception + message as unavailable catalog table. Catch it + // and fall through to return a null + } else { + throw e; + } + } + if (r == null || r.isEmpty()) { + return null; + } + return getServerNameFromCatalogResult(r); + } + + /** + * Gets the region info and assignment for the specified region. + * @param catalogTracker + * @param regionName Region to lookup. + * @return Location and HRegionInfo for regionName + * @throws IOException + */ + public static Pair getRegion( + CatalogTracker catalogTracker, byte [] regionName) + throws IOException { + Get get = new Get(regionName); + get.addFamily(HConstants.CATALOG_FAMILY); + Result r = get(getCatalogHTable(catalogTracker, regionName), get); + return (r == null || r.isEmpty())? null: parseCatalogResult(r); + } + + /** + * Extract a {@link ServerName} + * For use on catalog table {@link Result}. + * @param r Result to pull from + * @return A ServerName instance or null if necessary fields not found or empty. + */ + public static ServerName getServerNameFromCatalogResult(final Result r) { + byte[] value = r.getValue(HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER); + if (value == null || value.length == 0) return null; + String hostAndPort = Bytes.toString(value); + value = r.getValue(HConstants.CATALOG_FAMILY, + HConstants.STARTCODE_QUALIFIER); + if (value == null || value.length == 0) return null; + return new ServerName(hostAndPort, Bytes.toLong(value)); + } + + /** + * Extract a HRegionInfo and ServerName. + * For use on catalog table {@link Result}. + * @param r Result to pull from + * @return A pair of the {@link HRegionInfo} and the {@link ServerName} + * (or null for server address if no address set in .META.). + * @throws IOException + */ + public static Pair parseCatalogResult(final Result r) + throws IOException { + HRegionInfo info = + parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER); + ServerName sn = getServerNameFromCatalogResult(r); + return new Pair(info, sn); + } + + /** + * Parse the content of the cell at {@link HConstants#CATALOG_FAMILY} and + * qualifier as an HRegionInfo and return it, or null. + * For use on catalog table {@link Result}. + * @param r Result instance to pull from. + * @param qualifier Column family qualifier -- either + * {@link HConstants#SPLITA_QUALIFIER}, {@link HConstants#SPLITB_QUALIFIER} or + * {@link HConstants#REGIONINFO_QUALIFIER}. + * @return An HRegionInfo instance or null. + * @throws IOException + */ + public static HRegionInfo parseHRegionInfoFromCatalogResult(final Result r, + byte [] qualifier) + throws IOException { + byte [] bytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier); + if (bytes == null || bytes.length <= 0) return null; + return Writables.getHRegionInfoOrNull(bytes); + } + + + + /** + * Checks if the specified table exists. Looks at the META table hosted on + * the specified server. + * @param catalogTracker + * @param tableName table to check + * @return true if the table exists in meta, false if not + * @throws IOException + */ + public static boolean tableExists(CatalogTracker catalogTracker, + String tableName) + throws IOException { + if (tableName.equals(HTableDescriptor.ROOT_TABLEDESC.getNameAsString()) || + tableName.equals(HTableDescriptor.META_TABLEDESC.getNameAsString())) { + // Catalog tables always exist. + return true; + } + final byte [] tableNameBytes = Bytes.toBytes(tableName); + // Make a version of ResultCollectingVisitor that only collects the first + CollectingVisitor visitor = new CollectingVisitor() { + private HRegionInfo current = null; + + @Override + public boolean visit(Result r) throws IOException { + this.current = + parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER); + if (this.current == null) { + LOG.warn("No serialized HRegionInfo in " + r); + return true; + } + if (!isInsideTable(this.current, tableNameBytes)) return false; + // Else call super and add this Result to the collection. + super.visit(r); + // Stop collecting regions from table after we get one. + return false; + } + + @Override + void add(Result r) { + // Add the current HRI. + this.results.add(this.current); + } + }; + fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableNameBytes)); + // If visitor has results >= 1 then table exists. + return visitor.getResults().size() >= 1; + } + + /** + * Returns the daughter regions by reading the corresponding columns of the catalog table + * Result. + * @param data a Result object from the catalog table scan + * @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split + * parent + */ + public static PairOfSameType getDaughterRegions(Result data) throws IOException { + HRegionInfo splitA = Writables.getHRegionInfoOrNull(data.getValue(HConstants.CATALOG_FAMILY, + HConstants.SPLITA_QUALIFIER)); + HRegionInfo splitB = Writables.getHRegionInfoOrNull(data.getValue(HConstants.CATALOG_FAMILY, + HConstants.SPLITB_QUALIFIER)); + return new PairOfSameType(splitA, splitB); + } + + /** + * Gets all of the regions of the specified table. + * @param catalogTracker + * @param tableName + * @return Ordered list of {@link HRegionInfo}. + * @throws IOException + */ + public static List getTableRegions(CatalogTracker catalogTracker, + byte [] tableName) + throws IOException { + return getTableRegions(catalogTracker, tableName, false); + } + + /** + * Gets all of the regions of the specified table. + * @param catalogTracker + * @param tableName + * @param excludeOfflinedSplitParents If true, do not include offlined split + * parents in the return. + * @return Ordered list of {@link HRegionInfo}. + * @throws IOException + */ + public static List getTableRegions(CatalogTracker catalogTracker, + byte [] tableName, final boolean excludeOfflinedSplitParents) + throws IOException { + List> result = null; + try { + result = getTableRegionsAndLocations(catalogTracker, tableName, + excludeOfflinedSplitParents); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return getListOfHRegionInfos(result); + } + + static List getListOfHRegionInfos(final List> pairs) { + if (pairs == null || pairs.isEmpty()) return null; + List result = new ArrayList(pairs.size()); + for (Pair pair: pairs) { + result.add(pair.getFirst()); + } + return result; + } + + /** + * @param current + * @param tableName + * @return True if current tablename is equal to + * tableName + */ + static boolean isInsideTable(final HRegionInfo current, final byte [] tableName) { + return Bytes.equals(tableName, current.getTableName()); + } + + /** + * @param tableName + * @return Place to start Scan in .META. when passed a + * tableName; returns <tableName&rt; <,&rt; <,&rt; + */ + static byte [] getTableStartRowForMeta(final byte [] tableName) { + byte [] startRow = new byte[tableName.length + 2]; + System.arraycopy(tableName, 0, startRow, 0, tableName.length); + startRow[startRow.length - 2] = HRegionInfo.DELIMITER; + startRow[startRow.length - 1] = HRegionInfo.DELIMITER; + return startRow; + } + + /** + * This method creates a Scan object that will only scan catalog rows that + * belong to the specified table. It doesn't specify any columns. + * This is a better alternative to just using a start row and scan until + * it hits a new table since that requires parsing the HRI to get the table + * name. + * @param tableName bytes of table's name + * @return configured Scan object + */ + public static Scan getScanForTableName(byte[] tableName) { + String strName = Bytes.toString(tableName); + // Start key is just the table name with delimiters + byte[] startKey = Bytes.toBytes(strName + ",,"); + // Stop key appends the smallest possible char to the table name + byte[] stopKey = Bytes.toBytes(strName + " ,,"); + + Scan scan = new Scan(startKey); + scan.setStopRow(stopKey); + return scan; + } + + /** + * @param catalogTracker + * @param tableName + * @return Return list of regioninfos and server. + * @throws IOException + * @throws InterruptedException + */ + public static List> + getTableRegionsAndLocations(CatalogTracker catalogTracker, String tableName) + throws IOException, InterruptedException { + return getTableRegionsAndLocations(catalogTracker, Bytes.toBytes(tableName), + true); + } + + /** + * @param catalogTracker + * @param tableName + * @return Return list of regioninfos and server addresses. + * @throws IOException + * @throws InterruptedException + */ + public static List> + getTableRegionsAndLocations(final CatalogTracker catalogTracker, + final byte [] tableName, final boolean excludeOfflinedSplitParents) + throws IOException, InterruptedException { + if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { + // If root, do a bit of special handling. + ServerName serverName = catalogTracker.getRootLocation(); + List> list = + new ArrayList>(); + list.add(new Pair(HRegionInfo.ROOT_REGIONINFO, + serverName)); + return list; + } + // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress + CollectingVisitor> visitor = + new CollectingVisitor>() { + private Pair current = null; + + @Override + public boolean visit(Result r) throws IOException { + HRegionInfo hri = + parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER); + if (hri == null) { + LOG.warn("No serialized HRegionInfo in " + r); + return true; + } + if (!isInsideTable(hri, tableName)) return false; + if (excludeOfflinedSplitParents && hri.isSplitParent()) return true; + ServerName sn = getServerNameFromCatalogResult(r); + // Populate this.current so available when we call #add + this.current = new Pair(hri, sn); + // Else call super and add this Result to the collection. + return super.visit(r); + } + + @Override + void add(Result r) { + this.results.add(this.current); + } + }; + fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName), + Bytes.equals(tableName, HConstants.META_TABLE_NAME)); + return visitor.getResults(); + } + + /** + * @param catalogTracker + * @param serverName + * @return List of user regions installed on this server (does not include + * catalog regions). + * @throws IOException + */ + public static NavigableMap + getServerUserRegions(CatalogTracker catalogTracker, final ServerName serverName) + throws IOException { + final NavigableMap hris = new TreeMap(); + // Fill the above hris map with entries from .META. that have the passed + // servername. + CollectingVisitor v = new CollectingVisitor() { + @Override + void add(Result r) { + if (r == null || r.isEmpty()) return; + ServerName sn = getServerNameFromCatalogResult(r); + if (sn != null && sn.equals(serverName)) this.results.add(r); + } + }; + fullScan(catalogTracker, v); + List results = v.getResults(); + if (results != null && !results.isEmpty()) { + // Convert results to Map keyed by HRI + for (Result r: results) { + Pair p = parseCatalogResult(r); + if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r); + } + } + return hris; + } + + public static void fullScanMetaAndPrint(final CatalogTracker catalogTracker) + throws IOException { + Visitor v = new Visitor() { + @Override + public boolean visit(Result r) throws IOException { + if (r == null || r.isEmpty()) return true; + LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r); + HRegionInfo hrim = MetaEditor.getHRegionInfo(r); + LOG.info("fullScanMetaAndPrint.HRI Print= " + hrim); + return true; + } + }; + fullScan(catalogTracker, v); + } + + /** + * Fully scan a given region, on a given server starting with given row. + * @param hRegionInterface region server + * @param visitor visitor + * @param regionName name of region + * @param startrow start row + * @throws IOException + * @deprecated Does not retry; use fullScan xxx instead. + x + */ + public static void fullScan(HRegionInterface hRegionInterface, + Visitor visitor, final byte[] regionName, + byte[] startrow) throws IOException { + if (hRegionInterface == null) return; + Scan scan = new Scan(); + if (startrow != null) scan.setStartRow(startrow); + scan.addFamily(HConstants.CATALOG_FAMILY); + long scannerid = hRegionInterface.openScanner(regionName, scan); + try { + Result data; + while((data = hRegionInterface.next(scannerid)) != null) { + if (!data.isEmpty()) visitor.visit(data); + } + } finally { + hRegionInterface.close(scannerid); + } + return; + } + + /** + * Performs a full scan of a catalog table. + * @param catalogTracker + * @param visitor Visitor invoked against each row. + * @param startrow Where to start the scan. Pass null if want to begin scan + * at first row. + * @param scanRoot True if we are to scan -ROOT- rather than + * .META., the default (pass false to scan .META.) + * @throws IOException + */ + static void fullScan(CatalogTracker catalogTracker, + final Visitor visitor, final byte [] startrow, final boolean scanRoot) + throws IOException { + Scan scan = new Scan(); + if (startrow != null) scan.setStartRow(startrow); + if (startrow == null && !scanRoot) { + int caching = catalogTracker.getConnection().getConfiguration() + .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100); + scan.setCaching(caching); + } + scan.addFamily(HConstants.CATALOG_FAMILY); + HTable metaTable = scanRoot? + getRootHTable(catalogTracker): getMetaHTable(catalogTracker); + ResultScanner scanner = metaTable.getScanner(scan); + try { + Result data; + while((data = scanner.next()) != null) { + if (data.isEmpty()) continue; + // Break if visit returns false. + if (!visitor.visit(data)) break; + } + } finally { + scanner.close(); + metaTable.close(); + } + return; + } + + /** + * Implementations 'visit' a catalog table row. + */ + public interface Visitor { + /** + * Visit the catalog table row. + * @param r A row from catalog table + * @return True if we are to proceed scanning the table, else false if + * we are to stop now. + */ + public boolean visit(final Result r) throws IOException; + } + + /** + * A {@link Visitor} that collects content out of passed {@link Result}. + */ + static abstract class CollectingVisitor implements Visitor { + final List results = new ArrayList(); + @Override + public boolean visit(Result r) throws IOException { + if (r == null || r.isEmpty()) return true; + add(r); + return true; + } + + abstract void add(Result r); + + /** + * @return Collected results; wait till visits complete to collect all + * possible results + */ + List getResults() { + return this.results; + } + } + + /** + * Collects all returned. + */ + static class CollectAllVisitor extends CollectingVisitor { + @Override + void add(Result r) { + this.results.add(r); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java new file mode 100644 index 0000000..1cbf1b6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java @@ -0,0 +1,72 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.catalog; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +/** + * Makes changes to the location of -ROOT- in ZooKeeper. + */ +public class RootLocationEditor { + private static final Log LOG = LogFactory.getLog(RootLocationEditor.class); + + /** + * Deletes the location of -ROOT- in ZooKeeper. + * @param zookeeper zookeeper reference + * @throws KeeperException unexpected zookeeper exception + */ + public static void deleteRootLocation(ZooKeeperWatcher zookeeper) + throws KeeperException { + LOG.info("Unsetting ROOT region location in ZooKeeper"); + try { + // Just delete the node. Don't need any watches, only we will create it. + ZKUtil.deleteNode(zookeeper, zookeeper.rootServerZNode); + } catch(KeeperException.NoNodeException nne) { + // Has already been deleted + } + } + + /** + * Sets the location of -ROOT- in ZooKeeper to the + * specified server address. + * @param zookeeper zookeeper reference + * @param location The server hosting -ROOT- + * @throws KeeperException unexpected zookeeper exception + */ + public static void setRootLocation(ZooKeeperWatcher zookeeper, + final ServerName location) + throws KeeperException { + LOG.info("Setting ROOT region location in ZooKeeper as " + location); + try { + ZKUtil.createAndWatch(zookeeper, zookeeper.rootServerZNode, + Bytes.toBytes(location.toString())); + } catch(KeeperException.NodeExistsException nee) { + LOG.debug("ROOT region location already existed, updated location"); + ZKUtil.setData(zookeeper, zookeeper.rootServerZNode, + Bytes.toBytes(location.toString())); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/client/AbstractClientScanner.java b/src/main/java/org/apache/hadoop/hbase/client/AbstractClientScanner.java new file mode 100644 index 0000000..0473047 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/AbstractClientScanner.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.util.Iterator; + +/** + * Helper class for custom client scanners. + */ +public abstract class AbstractClientScanner implements ResultScanner { + + @Override + public Iterator iterator() { + return new Iterator() { + // The next RowResult, possibly pre-read + Result next = null; + + // return true if there is another item pending, false if there isn't. + // this method is where the actual advancing takes place, but you need + // to call next() to consume it. hasNext() will only advance if there + // isn't a pending next(). + public boolean hasNext() { + if (next == null) { + try { + next = AbstractClientScanner.this.next(); + return next != null; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return true; + } + + // get the pending next item and advance the iterator. returns null if + // there is no next item. + public Result next() { + // since hasNext() does the real advancing, we call this to determine + // if there is a next before proceeding. + if (!hasNext()) { + return null; + } + + // if we get to here, then hasNext() has given us an item to return. + // we want to return the item and then null out the next pointer, so + // we use a temporary variable. + Result temp = next; + next = null; + return temp; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/Action.java b/src/main/java/org/apache/hadoop/hbase/client/Action.java new file mode 100644 index 0000000..40b0f2e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/Action.java @@ -0,0 +1,106 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Writable; + +/* + * A Get, Put or Delete associated with it's region. Used internally by + * {@link HTable::batch} to associate the action with it's region and maintain + * the index from the original request. + */ +public class Action implements Writable, Comparable { + + private Row action; + private int originalIndex; + private R result; + + public Action() { + super(); + } + + /* + * This constructor is replaced by {@link #Action(Row, int)} + */ + @Deprecated + public Action(byte[] regionName, Row action, int originalIndex) { + this(action, originalIndex); + } + + public Action(Row action, int originalIndex) { + super(); + this.action = action; + this.originalIndex = originalIndex; + } + + @Deprecated + public byte[] getRegionName() { + return null; + } + + @Deprecated + public void setRegionName(byte[] regionName) { + } + + public R getResult() { + return result; + } + + public void setResult(R result) { + this.result = result; + } + + public Row getAction() { + return action; + } + + public int getOriginalIndex() { + return originalIndex; + } + + @Override + public int compareTo(Object o) { + return action.compareTo(((Action) o).getAction()); + } + + // /////////////////////////////////////////////////////////////////////////// + // Writable + // /////////////////////////////////////////////////////////////////////////// + + public void write(final DataOutput out) throws IOException { + HbaseObjectWritable.writeObject(out, action, Row.class, null); + out.writeInt(originalIndex); + HbaseObjectWritable.writeObject(out, result, + result != null ? result.getClass() : Writable.class, null); + } + + public void readFields(final DataInput in) throws IOException { + this.action = (Row) HbaseObjectWritable.readObject(in, null); + this.originalIndex = in.readInt(); + this.result = (R) HbaseObjectWritable.readObject(in, null); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/Append.java b/src/main/java/org/apache/hadoop/hbase/client/Append.java new file mode 100644 index 0000000..ce435c8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/Append.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Writable; + +/** + * Performs Append operations on a single row. + *

+ * Note that this operation does not appear atomic to readers. Appends are done + * under a single row lock, so write operations to a row are synchronized, but + * readers do not take row locks so get and scan operations can see this + * operation partially completed. + *

+ * To append to a set of columns of a row, instantiate an Append object with the + * row to append to. At least one column to append must be specified using the + * {@link #add(byte[], byte[], byte[])} method. + */ +public class Append extends Mutation { + private static final String RETURN_RESULTS = "_rr_"; + private static final byte APPEND_VERSION = (byte)1; + + /** + * @param returnResults + * True (default) if the append operation should return the results. + * A client that is not interested in the result can save network + * bandwidth setting this to false. + */ + public void setReturnResults(boolean returnResults) { + setAttribute(RETURN_RESULTS, Bytes.toBytes(returnResults)); + } + + /** + * @return current setting for returnResults + */ + public boolean isReturnResults() { + byte[] v = getAttribute(RETURN_RESULTS); + return v == null ? true : Bytes.toBoolean(v); + } + + /** Constructor for Writable. DO NOT USE */ + public Append() {} + + /** + * Create a Append operation for the specified row. + *

+ * At least one column must be appended to. + * @param row row key + */ + public Append(byte[] row) { + this.row = Arrays.copyOf(row, row.length); + } + + /** + * Add the specified column and value to this Append operation. + * @param family family name + * @param qualifier column qualifier + * @param value value to append to specified column + * @return this + */ + public Append add(byte [] family, byte [] qualifier, byte [] value) { + List list = familyMap.get(family); + if(list == null) { + list = new ArrayList(); + } + list.add(new KeyValue( + this.row, family, qualifier, this.ts, KeyValue.Type.Put, value)); + familyMap.put(family, list); + return this; + } + + @Override + public void readFields(final DataInput in) + throws IOException { + int version = in.readByte(); + if (version > APPEND_VERSION) { + throw new IOException("version not supported: "+version); + } + this.row = Bytes.readByteArray(in); + this.ts = in.readLong(); + this.lockId = in.readLong(); + this.writeToWAL = in.readBoolean(); + int numFamilies = in.readInt(); + if (!this.familyMap.isEmpty()) this.familyMap.clear(); + for(int i=0;i keys = new ArrayList(numKeys); + int totalLen = in.readInt(); + byte [] buf = new byte[totalLen]; + int offset = 0; + for (int j = 0; j < numKeys; j++) { + int keyLength = in.readInt(); + in.readFully(buf, offset, keyLength); + keys.add(new KeyValue(buf, offset, keyLength)); + offset += keyLength; + } + this.familyMap.put(family, keys); + } + readAttributes(in); + } + + @Override + public void write(final DataOutput out) + throws IOException { + out.writeByte(APPEND_VERSION); + Bytes.writeByteArray(out, this.row); + out.writeLong(this.ts); + out.writeLong(this.lockId); + out.writeBoolean(this.writeToWAL); + out.writeInt(familyMap.size()); + for (Map.Entry> entry : familyMap.entrySet()) { + Bytes.writeByteArray(out, entry.getKey()); + List keys = entry.getValue(); + out.writeInt(keys.size()); + int totalLen = 0; + for(KeyValue kv : keys) { + totalLen += kv.getLength(); + } + out.writeInt(totalLen); + for(KeyValue kv : keys) { + out.writeInt(kv.getLength()); + out.write(kv.getBuffer(), kv.getOffset(), kv.getLength()); + } + } + writeAttributes(out); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/Attributes.java b/src/main/java/org/apache/hadoop/hbase/client/Attributes.java new file mode 100644 index 0000000..5999e3a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/Attributes.java @@ -0,0 +1,47 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.util.Map; + +public interface Attributes { + /** + * Sets an attribute. + * In case value = null attribute is removed from the attributes map. + * Attribute names starting with _ indicate system attributes. + * @param name attribute name + * @param value attribute value + */ + public void setAttribute(String name, byte[] value); + + /** + * Gets an attribute + * @param name attribute name + * @return attribute value if attribute is set, null otherwise + */ + public byte[] getAttribute(String name); + + /** + * Gets all attributes + * @return unmodifiable map of all attributes + */ + public Map getAttributesMap(); +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java new file mode 100644 index 0000000..0fbabe5 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java @@ -0,0 +1,381 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.NotServingRegionException; +import org.apache.hadoop.hbase.UnknownScannerException; +import org.apache.hadoop.hbase.client.metrics.ScanMetrics; +import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.DataOutputBuffer; + +/** + * Implements the scanner interface for the HBase client. + * If there are multiple regions in a table, this scanner will iterate + * through them all. + */ +public class ClientScanner extends AbstractClientScanner { + private final Log LOG = LogFactory.getLog(this.getClass()); + private Scan scan; + private boolean closed = false; + // Current region scanner is against. Gets cleared if current region goes + // wonky: e.g. if it splits on us. + private HRegionInfo currentRegion = null; + private ScannerCallable callable = null; + private final LinkedList cache = new LinkedList(); + private final int caching; + private long lastNext; + // Keep lastResult returned successfully in case we have to reset scanner. + private Result lastResult = null; + private ScanMetrics scanMetrics = null; + private final long maxScannerResultSize; + private final HConnection connection; + private final byte[] tableName; + private final int scannerTimeout; + + /** + * Create a new ClientScanner for the specified table. An HConnection will be + * retrieved using the passed Configuration. + * Note that the passed {@link Scan}'s start row maybe changed changed. + * + * @param conf The {@link Configuration} to use. + * @param scan {@link Scan} to use in this scanner + * @param tableName The table that we wish to scan + * @throws IOException + */ + public ClientScanner(final Configuration conf, final Scan scan, + final byte[] tableName) throws IOException { + this(conf, scan, tableName, HConnectionManager.getConnection(conf)); + } + + /** + * Create a new ClientScanner for the specified table + * Note that the passed {@link Scan}'s start row maybe changed changed. + * + * @param conf The {@link Configuration} to use. + * @param scan {@link Scan} to use in this scanner + * @param tableName The table that we wish to scan + * @param connection Connection identifying the cluster + * @throws IOException + */ + public ClientScanner(final Configuration conf, final Scan scan, + final byte[] tableName, HConnection connection) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Creating scanner over " + + Bytes.toString(tableName) + + " starting at key '" + Bytes.toStringBinary(scan.getStartRow()) + "'"); + } + this.scan = scan; + this.tableName = tableName; + this.lastNext = System.currentTimeMillis(); + this.connection = connection; + this.maxScannerResultSize = conf.getLong( + HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, + HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE); + this.scannerTimeout = (int) conf.getLong( + HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, + HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD); + + // check if application wants to collect scan metrics + byte[] enableMetrics = scan.getAttribute( + Scan.SCAN_ATTRIBUTES_METRICS_ENABLE); + if (enableMetrics != null && Bytes.toBoolean(enableMetrics)) { + scanMetrics = new ScanMetrics(); + } + + // Use the caching from the Scan. If not set, use the default cache setting for this table. + if (this.scan.getCaching() > 0) { + this.caching = this.scan.getCaching(); + } else { + this.caching = conf.getInt("hbase.client.scanner.caching", 1); + } + + // initialize the scanner + nextScanner(this.caching, false); + } + + protected HConnection getConnection() { + return this.connection; + } + + protected byte[] getTableName() { + return this.tableName; + } + + protected Scan getScan() { + return scan; + } + + protected long getTimestamp() { + return lastNext; + } + + // returns true if the passed region endKey + private boolean checkScanStopRow(final byte [] endKey) { + if (this.scan.getStopRow().length > 0) { + // there is a stop row, check to see if we are past it. + byte [] stopRow = scan.getStopRow(); + int cmp = Bytes.compareTo(stopRow, 0, stopRow.length, + endKey, 0, endKey.length); + if (cmp <= 0) { + // stopRow <= endKey (endKey is equals to or larger than stopRow) + // This is a stop. + return true; + } + } + return false; //unlikely. + } + + /* + * Gets a scanner for the next region. If this.currentRegion != null, then + * we will move to the endrow of this.currentRegion. Else we will get + * scanner at the scan.getStartRow(). We will go no further, just tidy + * up outstanding scanners, if currentRegion != null and + * done is true. + * @param nbRows + * @param done Server-side says we're done scanning. + */ + private boolean nextScanner(int nbRows, final boolean done) + throws IOException { + // Close the previous scanner if it's open + if (this.callable != null) { + this.callable.setClose(); + callable.withRetries(); + this.callable = null; + } + + // Where to start the next scanner + byte [] localStartKey; + + // if we're at end of table, close and return false to stop iterating + if (this.currentRegion != null) { + byte [] endKey = this.currentRegion.getEndKey(); + if (endKey == null || + Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY) || + checkScanStopRow(endKey) || + done) { + close(); + if (LOG.isDebugEnabled()) { + LOG.debug("Finished with scanning at " + this.currentRegion); + } + return false; + } + localStartKey = endKey; + if (LOG.isDebugEnabled()) { + LOG.debug("Finished with region " + this.currentRegion); + } + } else { + localStartKey = this.scan.getStartRow(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Advancing internal scanner to startKey at '" + + Bytes.toStringBinary(localStartKey) + "'"); + } + try { + callable = getScannerCallable(localStartKey, nbRows); + // Open a scanner on the region server starting at the + // beginning of the region + callable.withRetries(); + this.currentRegion = callable.getHRegionInfo(); + if (this.scanMetrics != null) { + this.scanMetrics.countOfRegions.inc(); + } + } catch (IOException e) { + close(); + throw e; + } + return true; + } + + protected ScannerCallable getScannerCallable(byte [] localStartKey, + int nbRows) { + scan.setStartRow(localStartKey); + ScannerCallable s = new ScannerCallable(getConnection(), + getTableName(), scan, this.scanMetrics); + s.setCaching(nbRows); + return s; + } + + /** + * Publish the scan metrics. For now, we use scan.setAttribute to pass the metrics back to the + * application or TableInputFormat.Later, we could push it to other systems. We don't use metrics + * framework because it doesn't support multi-instances of the same metrics on the same machine; + * for scan/map reduce scenarios, we will have multiple scans running at the same time. + * + * By default, scan metrics are disabled; if the application wants to collect them, this behavior + * can be turned on by calling calling: + * + * scan.setAttribute(SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE)) + */ + private void writeScanMetrics() throws IOException { + if (this.scanMetrics == null) { + return; + } + final DataOutputBuffer d = new DataOutputBuffer(); + scanMetrics.write(d); + scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA, d.getData()); + } + + public Result next() throws IOException { + // If the scanner is closed and there's nothing left in the cache, next is a no-op. + if (cache.size() == 0 && this.closed) { + return null; + } + if (cache.size() == 0) { + Result [] values = null; + long remainingResultSize = maxScannerResultSize; + int countdown = this.caching; + // We need to reset it if it's a new callable that was created + // with a countdown in nextScanner + callable.setCaching(this.caching); + // This flag is set when we want to skip the result returned. We do + // this when we reset scanner because it split under us. + boolean skipFirst = false; + do { + try { + if (skipFirst) { + // Skip only the first row (which was the last row of the last + // already-processed batch). + callable.setCaching(1); + values = callable.withRetries(); + callable.setCaching(this.caching); + skipFirst = false; + } + // Server returns a null values if scanning is to stop. Else, + // returns an empty array if scanning is to go on and we've just + // exhausted current region. + values = callable.withRetries(); + } catch (DoNotRetryIOException e) { + if (e instanceof UnknownScannerException) { + long timeout = lastNext + scannerTimeout; + // If we are over the timeout, throw this exception to the client + // Else, it's because the region moved and we used the old id + // against the new region server; reset the scanner. + if (timeout < System.currentTimeMillis()) { + long elapsed = System.currentTimeMillis() - lastNext; + ScannerTimeoutException ex = new ScannerTimeoutException( + elapsed + "ms passed since the last invocation, " + + "timeout is currently set to " + scannerTimeout); + ex.initCause(e); + throw ex; + } + } else { + Throwable cause = e.getCause(); + if (cause == null || (!(cause instanceof NotServingRegionException) + && !(cause instanceof RegionServerStoppedException))) { + throw e; + } + } + // Else, its signal from depths of ScannerCallable that we got an + // NSRE on a next and that we need to reset the scanner. + if (this.lastResult != null) { + this.scan.setStartRow(this.lastResult.getRow()); + // Skip first row returned. We already let it out on previous + // invocation. + skipFirst = true; + } + // Clear region + this.currentRegion = null; + continue; + } + long currentTime = System.currentTimeMillis(); + if (this.scanMetrics != null ) { + this.scanMetrics.sumOfMillisSecBetweenNexts.inc(currentTime-lastNext); + } + lastNext = currentTime; + if (values != null && values.length > 0) { + for (Result rs : values) { + cache.add(rs); + for (KeyValue kv : rs.raw()) { + remainingResultSize -= kv.heapSize(); + } + countdown--; + this.lastResult = rs; + } + } + // Values == null means server-side filter has determined we must STOP + } while (remainingResultSize > 0 && countdown > 0 && nextScanner(countdown, values == null)); + } + + if (cache.size() > 0) { + return cache.poll(); + } + + // if we exhausted this scanner before calling close, write out the scan metrics + writeScanMetrics(); + return null; + } + + /** + * Get nbRows rows. + * How many RPCs are made is determined by the {@link Scan#setCaching(int)} + * setting (or hbase.client.scanner.caching in hbase-site.xml). + * @param nbRows number of rows to return + * @return Between zero and nbRows RowResults. Scan is done + * if returned array is of zero-length (We never return null). + * @throws IOException + */ + public Result [] next(int nbRows) throws IOException { + // Collect values to be returned here + ArrayList resultSets = new ArrayList(nbRows); + for(int i = 0; i < nbRows; i++) { + Result next = next(); + if (next != null) { + resultSets.add(next); + } else { + break; + } + } + return resultSets.toArray(new Result[resultSets.size()]); + } + + public void close() { + if (callable != null) { + callable.setClose(); + try { + callable.withRetries(); + } catch (IOException e) { + // We used to catch this error, interpret, and rethrow. However, we + // have since decided that it's not nice for a scanner's close to + // throw exceptions. Chances are it was just an UnknownScanner + // exception due to lease time out. + } finally { + // we want to output the scan metrics even if an error occurred on close + try { + writeScanMetrics(); + } catch (IOException e) { + // As above, we still don't want the scanner close() method to throw. + } + } + callable = null; + } + closed = true; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java b/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java new file mode 100644 index 0000000..0ffae89 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HConstants; + +import java.util.Random; + +/** + * Utility used by client connections such as {@link HConnection} and + * {@link ServerCallable} + */ +public class ConnectionUtils { + + private static final Random RANDOM = new Random(); + /** + * Calculate pause time. + * Built on {@link HConstants#RETRY_BACKOFF}. + * @param pause + * @param tries + * @return How long to wait after tries retries + */ + public static long getPauseTime(final long pause, final int tries) { + int ntries = tries; + if (ntries >= HConstants.RETRY_BACKOFF.length) { + ntries = HConstants.RETRY_BACKOFF.length - 1; + } + + long normalPause = pause * HConstants.RETRY_BACKOFF[ntries]; + long jitter = (long)(normalPause * RANDOM.nextFloat() * 0.01f); // 1% possible jitter + return normalPause + jitter; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/client/Delete.java b/src/main/java/org/apache/hadoop/hbase/client/Delete.java new file mode 100644 index 0000000..4472dcb --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/Delete.java @@ -0,0 +1,330 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Writable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Used to perform Delete operations on a single row. + *

+ * To delete an entire row, instantiate a Delete object with the row + * to delete. To further define the scope of what to delete, perform + * additional methods as outlined below. + *

+ * To delete specific families, execute {@link #deleteFamily(byte[]) deleteFamily} + * for each family to delete. + *

+ * To delete multiple versions of specific columns, execute + * {@link #deleteColumns(byte[], byte[]) deleteColumns} + * for each column to delete. + *

+ * To delete specific versions of specific columns, execute + * {@link #deleteColumn(byte[], byte[], long) deleteColumn} + * for each column version to delete. + *

+ * Specifying timestamps, deleteFamily and deleteColumns will delete all + * versions with a timestamp less than or equal to that passed. If no + * timestamp is specified, an entry is added with a timestamp of 'now' + * where 'now' is the servers's System.currentTimeMillis(). + * Specifying a timestamp to the deleteColumn method will + * delete versions only with a timestamp equal to that specified. + * If no timestamp is passed to deleteColumn, internally, it figures the + * most recent cell's timestamp and adds a delete at that timestamp; i.e. + * it deletes the most recently added cell. + *

The timestamp passed to the constructor is used ONLY for delete of + * rows. For anything less -- a deleteColumn, deleteColumns or + * deleteFamily -- then you need to use the method overrides that take a + * timestamp. The constructor timestamp is not referenced. + */ +public class Delete extends Mutation + implements Writable, Comparable { + private static final byte DELETE_VERSION = (byte)3; + + /** Constructor for Writable. DO NOT USE */ + public Delete() { + this((byte [])null); + } + + /** + * Create a Delete operation for the specified row. + *

+ * If no further operations are done, this will delete everything + * associated with the specified row (all versions of all columns in all + * families). + * @param row row key + */ + public Delete(byte [] row) { + this(row, HConstants.LATEST_TIMESTAMP); + } + + /** + * Create a Delete operation for the specified row and timestamp.

+ * + * If no further operations are done, this will delete all columns in all + * families of the specified row with a timestamp less than or equal to the + * specified timestamp.

+ * + * This timestamp is ONLY used for a delete row operation. If specifying + * families or columns, you must specify each timestamp individually. + * @param row row key + * @param timestamp maximum version timestamp (only for delete row) + */ + public Delete(byte [] row, long timestamp) { + this.row = row; + this.ts = timestamp; + } + + /** + * Create a Delete operation for the specified row and timestamp, using + * an optional row lock.

+ * + * If no further operations are done, this will delete all columns in all + * families of the specified row with a timestamp less than or equal to the + * specified timestamp.

+ * + * This timestamp is ONLY used for a delete row operation. If specifying + * families or columns, you must specify each timestamp individually. + * @param row row key + * @param timestamp maximum version timestamp (only for delete row) + * @param rowLock previously acquired row lock, or null + * @deprecated {@link RowLock} is deprecated, use {@link #Delete(byte[], long)}. + */ + public Delete(byte [] row, long timestamp, RowLock rowLock) { + this.row = row; + this.ts = timestamp; + if (rowLock != null) { + this.lockId = rowLock.getLockId(); + } + } + + /** + * @param d Delete to clone. + */ + public Delete(final Delete d) { + this.row = d.getRow(); + this.ts = d.getTimeStamp(); + this.lockId = d.getLockId(); + this.familyMap.putAll(d.getFamilyMap()); + this.writeToWAL = d.writeToWAL; + } + + /** + * Advanced use only. + * Add an existing delete marker to this Delete object. + * @param kv An existing KeyValue of type "delete". + * @return this for invocation chaining + * @throws IOException + */ + public Delete addDeleteMarker(KeyValue kv) throws IOException { + if (!kv.isDelete()) { + throw new IOException("The recently added KeyValue is not of type " + + "delete. Rowkey: " + Bytes.toStringBinary(this.row)); + } + if (Bytes.compareTo(this.row, 0, row.length, kv.getBuffer(), + kv.getRowOffset(), kv.getRowLength()) != 0) { + throw new IOException("The row in the recently added KeyValue " + + Bytes.toStringBinary(kv.getBuffer(), kv.getRowOffset(), + kv.getRowLength()) + " doesn't match the original one " + + Bytes.toStringBinary(this.row)); + } + byte [] family = kv.getFamily(); + List list = familyMap.get(family); + if (list == null) { + list = new ArrayList(); + } + list.add(kv); + familyMap.put(family, list); + return this; + } + + /** + * Delete all versions of all columns of the specified family. + *

+ * Overrides previous calls to deleteColumn and deleteColumns for the + * specified family. + * @param family family name + * @return this for invocation chaining + */ + public Delete deleteFamily(byte [] family) { + this.deleteFamily(family, HConstants.LATEST_TIMESTAMP); + return this; + } + + /** + * Delete all columns of the specified family with a timestamp less than + * or equal to the specified timestamp. + *

+ * Overrides previous calls to deleteColumn and deleteColumns for the + * specified family. + * @param family family name + * @param timestamp maximum version timestamp + * @return this for invocation chaining + */ + public Delete deleteFamily(byte [] family, long timestamp) { + List list = familyMap.get(family); + if(list == null) { + list = new ArrayList(); + } else if(!list.isEmpty()) { + list.clear(); + } + list.add(new KeyValue(row, family, null, timestamp, KeyValue.Type.DeleteFamily)); + familyMap.put(family, list); + return this; + } + + /** + * Delete all versions of the specified column. + * @param family family name + * @param qualifier column qualifier + * @return this for invocation chaining + */ + public Delete deleteColumns(byte [] family, byte [] qualifier) { + this.deleteColumns(family, qualifier, HConstants.LATEST_TIMESTAMP); + return this; + } + + /** + * Delete all versions of the specified column with a timestamp less than + * or equal to the specified timestamp. + * @param family family name + * @param qualifier column qualifier + * @param timestamp maximum version timestamp + * @return this for invocation chaining + */ + public Delete deleteColumns(byte [] family, byte [] qualifier, long timestamp) { + List list = familyMap.get(family); + if (list == null) { + list = new ArrayList(); + } + list.add(new KeyValue(this.row, family, qualifier, timestamp, + KeyValue.Type.DeleteColumn)); + familyMap.put(family, list); + return this; + } + + /** + * Delete the latest version of the specified column. + * This is an expensive call in that on the server-side, it first does a + * get to find the latest versions timestamp. Then it adds a delete using + * the fetched cells timestamp. + * @param family family name + * @param qualifier column qualifier + * @return this for invocation chaining + */ + public Delete deleteColumn(byte [] family, byte [] qualifier) { + this.deleteColumn(family, qualifier, HConstants.LATEST_TIMESTAMP); + return this; + } + + /** + * Delete the specified version of the specified column. + * @param family family name + * @param qualifier column qualifier + * @param timestamp version timestamp + * @return this for invocation chaining + */ + public Delete deleteColumn(byte [] family, byte [] qualifier, long timestamp) { + List list = familyMap.get(family); + if(list == null) { + list = new ArrayList(); + } + list.add(new KeyValue( + this.row, family, qualifier, timestamp, KeyValue.Type.Delete)); + familyMap.put(family, list); + return this; + } + + /** + * Set the timestamp of the delete. + * + * @param timestamp + */ + public void setTimestamp(long timestamp) { + this.ts = timestamp; + } + + @Override + public Map toMap(int maxCols) { + // we start with the fingerprint map and build on top of it. + Map map = super.toMap(maxCols); + // why is put not doing this? + map.put("ts", this.ts); + return map; + } + + //Writable + public void readFields(final DataInput in) throws IOException { + int version = in.readByte(); + if (version > DELETE_VERSION) { + throw new IOException("version not supported"); + } + this.row = Bytes.readByteArray(in); + this.ts = in.readLong(); + this.lockId = in.readLong(); + if (version > 2) { + this.writeToWAL = in.readBoolean(); + } + this.familyMap.clear(); + int numFamilies = in.readInt(); + for(int i=0;i list = new ArrayList(numColumns); + for(int j=0;j 1) { + readAttributes(in); + } + } + + public void write(final DataOutput out) throws IOException { + out.writeByte(DELETE_VERSION); + Bytes.writeByteArray(out, this.row); + out.writeLong(this.ts); + out.writeLong(this.lockId); + out.writeBoolean(this.writeToWAL); + out.writeInt(familyMap.size()); + for(Map.Entry> entry : familyMap.entrySet()) { + Bytes.writeByteArray(out, entry.getKey()); + List list = entry.getValue(); + out.writeInt(list.size()); + for(KeyValue kv : list) { + kv.write(out); + } + } + writeAttributes(out); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/Durability.java b/src/main/java/org/apache/hadoop/hbase/client/Durability.java new file mode 100644 index 0000000..82995a6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/Durability.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +/** + * Enum describing the durability guarantees for {@link Mutation} + * Note that the items must be sorted in order of increasing durability + */ +public enum Durability { + /** + * Use the column family's default setting to determine durability. + * This must remain the first option. + */ + USE_DEFAULT, + /** + * Do not write the Mutation to the WAL + */ + SKIP_WAL, + /** + * Write the Mutation to the WAL asynchronously + */ + ASYNC_WAL, + /** + * Write the Mutation to the WAL synchronously. + * The data is flushed to the filesystem implementation, but not necessarily to disk. + * For HDFS this will flush the data to the designated number of DataNodes. + * See HADOOP-6313 + */ + SYNC_WAL, + /** + * Write the Mutation to the WAL synchronously and force the entries to disk. + * (Note: this is currently not supported and will behave identical to {@link #SYNC_WAL}) + * See HADOOP-6313 + */ + FSYNC_WAL; + + // efficiently translate ordinal back to items of this Enum + // (Enum.values()[ordinal] generates too much garbage) + public static Durability valueOf(int ordinal) { + switch (ordinal) { + case 0: return USE_DEFAULT; + case 1: return SKIP_WAL; + case 2: return ASYNC_WAL; + case 3: return SYNC_WAL; + case 4: return FSYNC_WAL; + default: throw new IllegalArgumentException("Unknown Durability Ordinal:"+ordinal); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/Get.java b/src/main/java/org/apache/hadoop/hbase/client/Get.java new file mode 100644 index 0000000..ad9a058 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/Get.java @@ -0,0 +1,462 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.io.TimeRange; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Classes; +import org.apache.hadoop.io.Writable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * Used to perform Get operations on a single row. + *

+ * To get everything for a row, instantiate a Get object with the row to get. + * To further define the scope of what to get, perform additional methods as + * outlined below. + *

+ * To get all columns from specific families, execute {@link #addFamily(byte[]) addFamily} + * for each family to retrieve. + *

+ * To get specific columns, execute {@link #addColumn(byte[], byte[]) addColumn} + * for each column to retrieve. + *

+ * To only retrieve columns within a specific range of version timestamps, + * execute {@link #setTimeRange(long, long) setTimeRange}. + *

+ * To only retrieve columns with a specific timestamp, execute + * {@link #setTimeStamp(long) setTimestamp}. + *

+ * To limit the number of versions of each column to be returned, execute + * {@link #setMaxVersions(int) setMaxVersions}. + *

+ * To add a filter, execute {@link #setFilter(Filter) setFilter}. + */ +public class Get extends OperationWithAttributes + implements Writable, Row, Comparable { + private static final byte GET_VERSION = (byte)2; + + private byte [] row = null; + private long lockId = -1L; + private int maxVersions = 1; + private boolean cacheBlocks = true; + private Filter filter = null; + private TimeRange tr = new TimeRange(); + private Map> familyMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); + + /** Constructor for Writable. DO NOT USE */ + public Get() {} + + /** + * Create a Get operation for the specified row. + *

+ * If no further operations are done, this will get the latest version of + * all columns in all families of the specified row. + * @param row row key + */ + public Get(byte [] row) { + this(row, null); + } + + /** + * Create a Get operation for the specified row, using an existing row lock. + *

+ * If no further operations are done, this will get the latest version of + * all columns in all families of the specified row. + * @param row row key + * @param rowLock previously acquired row lock, or null + * @deprecated {@link RowLock} is deprecated, use {@link #Get(byte[])}. + */ + public Get(byte [] row, RowLock rowLock) { + this.row = row; + if(rowLock != null) { + this.lockId = rowLock.getLockId(); + } + } + + /** + * Get all columns from the specified family. + *

+ * Overrides previous calls to addColumn for this family. + * @param family family name + * @return the Get object + */ + public Get addFamily(byte [] family) { + familyMap.remove(family); + familyMap.put(family, null); + return this; + } + + /** + * Get the column from the specific family with the specified qualifier. + *

+ * Overrides previous calls to addFamily for this family. + * @param family family name + * @param qualifier column qualifier + * @return the Get objec + */ + public Get addColumn(byte [] family, byte [] qualifier) { + NavigableSet set = familyMap.get(family); + if(set == null) { + set = new TreeSet(Bytes.BYTES_COMPARATOR); + } + if (qualifier == null) { + qualifier = HConstants.EMPTY_BYTE_ARRAY; + } + set.add(qualifier); + familyMap.put(family, set); + return this; + } + + /** + * Get versions of columns only within the specified timestamp range, + * [minStamp, maxStamp). + * @param minStamp minimum timestamp value, inclusive + * @param maxStamp maximum timestamp value, exclusive + * @throws IOException if invalid time range + * @return this for invocation chaining + */ + public Get setTimeRange(long minStamp, long maxStamp) + throws IOException { + tr = new TimeRange(minStamp, maxStamp); + return this; + } + + /** + * Get versions of columns with the specified timestamp. + * @param timestamp version timestamp + * @return this for invocation chaining + */ + public Get setTimeStamp(long timestamp) { + try { + tr = new TimeRange(timestamp, timestamp+1); + } catch(IOException e) { + // Will never happen + } + return this; + } + + /** + * Get all available versions. + * @return this for invocation chaining + */ + public Get setMaxVersions() { + this.maxVersions = Integer.MAX_VALUE; + return this; + } + + /** + * Get up to the specified number of versions of each column. + * @param maxVersions maximum versions for each column + * @throws IOException if invalid number of versions + * @return this for invocation chaining + */ + public Get setMaxVersions(int maxVersions) throws IOException { + if(maxVersions <= 0) { + throw new IOException("maxVersions must be positive"); + } + this.maxVersions = maxVersions; + return this; + } + + /** + * Apply the specified server-side filter when performing the Get. + * Only {@link Filter#filterKeyValue(KeyValue)} is called AFTER all tests + * for ttl, column match, deletes and max versions have been run. + * @param filter filter to run on the server + * @return this for invocation chaining + */ + public Get setFilter(Filter filter) { + this.filter = filter; + return this; + } + + /* Accessors */ + + /** + * @return Filter + */ + public Filter getFilter() { + return this.filter; + } + + /** + * Set whether blocks should be cached for this Get. + *

+ * This is true by default. When true, default settings of the table and + * family are used (this will never override caching blocks if the block + * cache is disabled for that family or entirely). + * + * @param cacheBlocks if false, default settings are overridden and blocks + * will not be cached + */ + public void setCacheBlocks(boolean cacheBlocks) { + this.cacheBlocks = cacheBlocks; + } + + /** + * Get whether blocks should be cached for this Get. + * @return true if default caching should be used, false if blocks should not + * be cached + */ + public boolean getCacheBlocks() { + return cacheBlocks; + } + + /** + * Method for retrieving the get's row + * @return row + */ + public byte [] getRow() { + return this.row; + } + + /** + * Method for retrieving the get's RowLock + * @return RowLock + */ + @SuppressWarnings("deprecation") + public RowLock getRowLock() { + return new RowLock(this.row, this.lockId); + } + + /** + * Method for retrieving the get's lockId + * @return lockId + */ + public long getLockId() { + return this.lockId; + } + + /** + * Method for retrieving the get's maximum number of version + * @return the maximum number of version to fetch for this get + */ + public int getMaxVersions() { + return this.maxVersions; + } + + /** + * Method for retrieving the get's TimeRange + * @return timeRange + */ + public TimeRange getTimeRange() { + return this.tr; + } + + /** + * Method for retrieving the keys in the familyMap + * @return keys in the current familyMap + */ + public Set familySet() { + return this.familyMap.keySet(); + } + + /** + * Method for retrieving the number of families to get from + * @return number of families + */ + public int numFamilies() { + return this.familyMap.size(); + } + + /** + * Method for checking if any families have been inserted into this Get + * @return true if familyMap is non empty false otherwise + */ + public boolean hasFamilies() { + return !this.familyMap.isEmpty(); + } + + /** + * Method for retrieving the get's familyMap + * @return familyMap + */ + public Map> getFamilyMap() { + return this.familyMap; + } + + /** + * Compile the table and column family (i.e. schema) information + * into a String. Useful for parsing and aggregation by debugging, + * logging, and administration tools. + * @return Map + */ + @Override + public Map getFingerprint() { + Map map = new HashMap(); + List families = new ArrayList(); + map.put("families", families); + for (Map.Entry> entry : + this.familyMap.entrySet()) { + families.add(Bytes.toStringBinary(entry.getKey())); + } + return map; + } + + /** + * Compile the details beyond the scope of getFingerprint (row, columns, + * timestamps, etc.) into a Map along with the fingerprinted information. + * Useful for debugging, logging, and administration tools. + * @param maxCols a limit on the number of columns output prior to truncation + * @return Map + */ + @Override + public Map toMap(int maxCols) { + // we start with the fingerprint map and build on top of it. + Map map = getFingerprint(); + // replace the fingerprint's simple list of families with a + // map from column families to lists of qualifiers and kv details + Map> columns = new HashMap>(); + map.put("families", columns); + // add scalar information first + map.put("row", Bytes.toStringBinary(this.row)); + map.put("maxVersions", this.maxVersions); + map.put("cacheBlocks", this.cacheBlocks); + List timeRange = new ArrayList(); + timeRange.add(this.tr.getMin()); + timeRange.add(this.tr.getMax()); + map.put("timeRange", timeRange); + int colCount = 0; + // iterate through affected families and add details + for (Map.Entry> entry : + this.familyMap.entrySet()) { + List familyList = new ArrayList(); + columns.put(Bytes.toStringBinary(entry.getKey()), familyList); + if(entry.getValue() == null) { + colCount++; + --maxCols; + familyList.add("ALL"); + } else { + colCount += entry.getValue().size(); + if (maxCols <= 0) { + continue; + } + for (byte [] column : entry.getValue()) { + if (--maxCols <= 0) { + continue; + } + familyList.add(Bytes.toStringBinary(column)); + } + } + } + map.put("totalColumns", colCount); + if (this.filter != null) { + map.put("filter", this.filter.toString()); + } + // add the id if set + if (getId() != null) { + map.put("id", getId()); + } + return map; + } + + //Row + public int compareTo(Row other) { + return Bytes.compareTo(this.getRow(), other.getRow()); + } + + //Writable + public void readFields(final DataInput in) + throws IOException { + int version = in.readByte(); + if (version > GET_VERSION) { + throw new IOException("unsupported version"); + } + this.row = Bytes.readByteArray(in); + this.lockId = in.readLong(); + this.maxVersions = in.readInt(); + boolean hasFilter = in.readBoolean(); + if (hasFilter) { + this.filter = Classes.createWritableForName( + Bytes.toString(Bytes.readByteArray(in))); + this.filter.readFields(in); + } + this.cacheBlocks = in.readBoolean(); + this.tr = new TimeRange(); + tr.readFields(in); + int numFamilies = in.readInt(); + this.familyMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); + for(int i=0; i set = null; + if(hasColumns) { + int numColumns = in.readInt(); + set = new TreeSet(Bytes.BYTES_COMPARATOR); + for(int j=0; j> entry : + familyMap.entrySet()) { + Bytes.writeByteArray(out, entry.getKey()); + NavigableSet columnSet = entry.getValue(); + if(columnSet == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeInt(columnSet.size()); + for(byte [] qualifier : columnSet) { + Bytes.writeByteArray(out, qualifier); + } + } + } + writeAttributes(out); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java new file mode 100644 index 0000000..5f78c6e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -0,0 +1,2428 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.lang.reflect.Proxy; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.SocketTimeoutException; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.NotServingRegionException; +import org.apache.hadoop.hbase.RegionException; +import org.apache.hadoop.hbase.RemoteExceptionHandler; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.TableNotEnabledException; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.UnknownRegionException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.ipc.HMasterInterface; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.ipc.MasterExecRPCInvoker; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.CompactionState; +import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException; +import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException; +import org.apache.hadoop.hbase.snapshot.HSnapshotDescription; +import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; +import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException; +import org.apache.hadoop.hbase.util.Addressing; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.util.StringUtils; + +/** + * Provides an interface to manage HBase database table metadata + general + * administrative functions. Use HBaseAdmin to create, drop, list, enable and + * disable tables. Use it also to add and drop table column families. + * + *

See {@link HTable} to add, update, and delete data from an individual table. + *

Currently HBaseAdmin instances are not expected to be long-lived. For + * example, an HBaseAdmin instance will not ride over a Master restart. + */ +public class HBaseAdmin implements Abortable, Closeable { + private final Log LOG = LogFactory.getLog(this.getClass().getName()); +// private final HConnection connection; + private HConnection connection; + private volatile Configuration conf; + private final long pause; + private final int numRetries; + // Some operations can take a long time such as disable of big table. + // numRetries is for 'normal' stuff... Mutliply by this factor when + // want to wait a long time. + private final int retryLongerMultiplier; + private boolean aborted; + + static final String INDEX_TABLE_SUFFIX = "_idx"; + private static volatile boolean synchronousBalanceSwitchSupported = true; + + /** + * Constructor + * + * @param c Configuration object + * @throws MasterNotRunningException if the master is not running + * @throws ZooKeeperConnectionException if unable to connect to zookeeper + */ + public HBaseAdmin(Configuration c) + throws MasterNotRunningException, ZooKeeperConnectionException { + this.conf = HBaseConfiguration.create(c); + this.connection = HConnectionManager.getConnection(this.conf); + this.pause = this.conf.getLong("hbase.client.pause", 1000); + this.numRetries = this.conf.getInt("hbase.client.retries.number", 10); + this.retryLongerMultiplier = this.conf.getInt( + "hbase.client.retries.longer.multiplier", 10); + + int tries = 0; + while ( true ){ + try { + + this.connection.getMaster(); + return; + + } catch (MasterNotRunningException mnre) { + HConnectionManager.deleteStaleConnection(this.connection); + this.connection = HConnectionManager.getConnection(this.conf); + } + + tries++; + if (tries >= numRetries) { + // we should delete connection between client and zookeeper + HConnectionManager.deleteStaleConnection(this.connection); + throw new MasterNotRunningException("Retried " + numRetries + " times"); + } + + try { + Thread.sleep(getPauseTime(tries)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + // we should delete connection between client and zookeeper + HConnectionManager.deleteStaleConnection(this.connection); + throw new MasterNotRunningException( + "Interrupted after "+tries+" tries"); + } + } + } + + /** + * Constructor for externally managed HConnections. + * This constructor fails fast if the HMaster is not running. + * The HConnection can be re-used again in another attempt. + * This constructor fails fast. + * + * @param connection The HConnection instance to use + * @throws MasterNotRunningException if the master is not running + * @throws ZooKeeperConnectionException if unable to connect to zookeeper + */ + public HBaseAdmin(HConnection connection) + throws MasterNotRunningException, ZooKeeperConnectionException { + this.conf = connection.getConfiguration(); + this.connection = connection; + + this.pause = this.conf.getLong("hbase.client.pause", 1000); + this.numRetries = this.conf.getInt("hbase.client.retries.number", 10); + this.retryLongerMultiplier = this.conf.getInt( + "hbase.client.retries.longer.multiplier", 10); + + this.connection.getMaster(); + } + + /** + * @return A new CatalogTracker instance; call {@link #cleanupCatalogTracker(CatalogTracker)} + * to cleanup the returned catalog tracker. + * @throws ZooKeeperConnectionException + * @throws IOException + * @see #cleanupCatalogTracker(CatalogTracker) + */ + private synchronized CatalogTracker getCatalogTracker() + throws ZooKeeperConnectionException, IOException { + CatalogTracker ct = null; + try { + ct = new CatalogTracker(this.conf); + ct.start(); + } catch (InterruptedException e) { + // Let it out as an IOE for now until we redo all so tolerate IEs + Thread.currentThread().interrupt(); + throw new IOException("Interrupted", e); + } + return ct; + } + + private void cleanupCatalogTracker(final CatalogTracker ct) { + ct.stop(); + } + + @Override + public void abort(String why, Throwable e) { + // Currently does nothing but throw the passed message and exception + this.aborted = true; + throw new RuntimeException(why, e); + } + + @Override + public boolean isAborted(){ + return this.aborted; + } + + /** @return HConnection used by this object. */ + public HConnection getConnection() { + return connection; + } + + /** + * Get a connection to the currently set master. + * @return proxy connection to master server for this instance + * @throws MasterNotRunningException if the master is not running + * @throws ZooKeeperConnectionException if unable to connect to zookeeper + * @deprecated Master is an implementation detail for HBaseAdmin. + * Deprecated in HBase 0.94 + */ + @Deprecated + public HMasterInterface getMaster() + throws MasterNotRunningException, ZooKeeperConnectionException { + return this.connection.getMaster(); + } + + /** @return - true if the master server is running + * @throws ZooKeeperConnectionException + * @throws MasterNotRunningException */ + public boolean isMasterRunning() + throws MasterNotRunningException, ZooKeeperConnectionException { + return this.connection.isMasterRunning(); + } + + /** + * @param tableName Table to check. + * @return True if table exists already. + * @throws IOException + */ + public boolean tableExists(final String tableName) + throws IOException { + boolean b = false; + CatalogTracker ct = getCatalogTracker(); + try { + b = MetaReader.tableExists(ct, tableName); + } finally { + cleanupCatalogTracker(ct); + } + return b; + } + + /** + * @param tableName Table to check. + * @return True if table exists already. + * @throws IOException + */ + public boolean tableExists(final byte [] tableName) + throws IOException { + return tableExists(Bytes.toString(tableName)); + } + + /** + * List all the userspace tables. In other words, scan the META table. + * + * If we wanted this to be really fast, we could implement a special + * catalog table that just contains table names and their descriptors. + * Right now, it only exists as part of the META table's region info. + * + * @return - returns an array of HTableDescriptors + * @throws IOException if a remote or network exception occurs + */ + public HTableDescriptor[] listTables() throws IOException { + return this.connection.listTables(); + } + + /** + * List all the userspace tables matching the given pattern. + * + * @param pattern The compiled regular expression to match against + * @return - returns an array of HTableDescriptors + * @throws IOException if a remote or network exception occurs + * @see #listTables() + */ + public HTableDescriptor[] listTables(Pattern pattern) throws IOException { + List matched = new LinkedList(); + HTableDescriptor[] tables = listTables(); + for (HTableDescriptor table : tables) { + if (pattern.matcher(table.getNameAsString()).matches()) { + matched.add(table); + } + } + return matched.toArray(new HTableDescriptor[matched.size()]); + } + + /** + * List all the userspace tables matching the given regular expression. + * + * @param regex The regular expression to match against + * @return - returns an array of HTableDescriptors + * @throws IOException if a remote or network exception occurs + * @see #listTables(java.util.regex.Pattern) + */ + public HTableDescriptor[] listTables(String regex) throws IOException { + return listTables(Pattern.compile(regex)); + } + + + /** + * Method for getting the tableDescriptor + * @param tableName as a byte [] + * @return the tableDescriptor + * @throws TableNotFoundException + * @throws IOException if a remote or network exception occurs + */ + public HTableDescriptor getTableDescriptor(final byte [] tableName) + throws TableNotFoundException, IOException { + return this.connection.getHTableDescriptor(tableName); + } + + private long getPauseTime(int tries) { + int triesCount = tries; + if (triesCount >= HConstants.RETRY_BACKOFF.length) { + triesCount = HConstants.RETRY_BACKOFF.length - 1; + } + return this.pause * HConstants.RETRY_BACKOFF[triesCount]; + } + + /** + * Creates a new table. + * Synchronous operation. + * + * @param desc table descriptor for table + * + * @throws IllegalArgumentException if the table name is reserved + * @throws MasterNotRunningException if master is not running + * @throws TableExistsException if table already exists (If concurrent + * threads, the table may have been created between test-for-existence + * and attempt-at-creation). + * @throws IOException if a remote or network exception occurs + */ + public void createTable(HTableDescriptor desc) + throws IOException { + createTable(desc, null); + } + + /** + * Creates a new table with the specified number of regions. The start key + * specified will become the end key of the first region of the table, and + * the end key specified will become the start key of the last region of the + * table (the first region has a null start key and the last region has a + * null end key). + * + * BigInteger math will be used to divide the key range specified into + * enough segments to make the required number of total regions. + * + * Synchronous operation. + * + * @param desc table descriptor for table + * @param startKey beginning of key range + * @param endKey end of key range + * @param numRegions the total number of regions to create + * + * @throws IllegalArgumentException if the table name is reserved + * @throws MasterNotRunningException if master is not running + * @throws TableExistsException if table already exists (If concurrent + * threads, the table may have been created between test-for-existence + * and attempt-at-creation). + * @throws IOException + */ + public void createTable(HTableDescriptor desc, byte [] startKey, + byte [] endKey, int numRegions) + throws IOException { + HTableDescriptor.isLegalTableName(desc.getName()); + if(numRegions < 3) { + throw new IllegalArgumentException("Must create at least three regions"); + } else if(Bytes.compareTo(startKey, endKey) >= 0) { + throw new IllegalArgumentException("Start key must be smaller than end key"); + } + if (numRegions == 3) { + createTable(desc, new byte[][] { startKey, endKey }); + return; + } + byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3); + if(splitKeys == null || splitKeys.length != numRegions - 1) { + throw new IllegalArgumentException("Unable to split key range into enough regions"); + } + createTable(desc, splitKeys); + } + + /** + * Creates a new table with an initial set of empty regions defined by the + * specified split keys. The total number of regions created will be the + * number of split keys plus one. Synchronous operation. + * Note : Avoid passing empty split key. + * + * @param desc table descriptor for table + * @param splitKeys array of split keys for the initial regions of the table + * + * @throws IllegalArgumentException if the table name is reserved, if the split keys + * are repeated and if the split key has empty byte array. + * @throws MasterNotRunningException if master is not running + * @throws TableExistsException if table already exists (If concurrent + * threads, the table may have been created between test-for-existence + * and attempt-at-creation). + * @throws IOException + */ + public void createTable(final HTableDescriptor desc, byte [][] splitKeys) + throws IOException { + HTableDescriptor.isLegalTableName(desc.getName()); + try { + createTableAsync(desc, splitKeys); + } catch (SocketTimeoutException ste) { + LOG.warn("Creating " + desc.getNameAsString() + " took too long", ste); + } + int numRegs = splitKeys == null ? 1 : splitKeys.length + 1; + int prevRegCount = 0; + boolean doneWithMetaScan = false; + + MetaScannerVisitorBaseWithTableName userTableVisitor = null; + MetaScannerVisitorBaseWithTableName indexTableVisitor = null; + boolean indexedHTD = isIndexedHTD(desc); + + for (int tries = 0; tries < this.numRetries * this.retryLongerMultiplier; ++tries) { + + AtomicInteger actualRegCount = null; + // Wait for new table to come on-line + if (userTableVisitor == null) { + userTableVisitor = new MetaScannerVisitorBaseWithTableName(desc.getNameAsString()); + } + actualRegCount = userTableVisitor.getActualRgnCnt(); + actualRegCount.set(0); + MetaScanner.metaScan(conf, userTableVisitor, desc.getName()); + if (actualRegCount.get() != numRegs) { + if (tries == this.numRetries * this.retryLongerMultiplier - 1) { + throw new RegionOfflineException("Only " + actualRegCount.get() + " of " + numRegs + + " regions are online; retries exhausted."); + } + try { // Sleep + Thread.sleep(getPauseTime(tries)); + } catch (InterruptedException e) { + throw new InterruptedIOException("Interrupted when opening" + " regions; " + + actualRegCount.get() + " of " + numRegs + " regions processed so far"); + } + if (actualRegCount.get() > prevRegCount) { // Making progress + prevRegCount = actualRegCount.get(); + tries = -1; + } + } else { + if (indexedHTD) { + String indexTableName = desc.getNameAsString() + INDEX_TABLE_SUFFIX; + if (indexTableVisitor == null) { + indexTableVisitor = new MetaScannerVisitorBaseWithTableName(indexTableName); + } + actualRegCount = indexTableVisitor.getActualRgnCnt(); + actualRegCount.set(0); + MetaScanner.metaScan(conf, indexTableVisitor, Bytes.toBytes(indexTableName)); + if (actualRegCount.get() != numRegs) { + if (tries == this.numRetries * this.retryLongerMultiplier - 1) { + throw new RegionOfflineException("Only " + actualRegCount.get() + " of " + numRegs + + " regions are online; retries exhausted."); + } + try { // Sleep + Thread.sleep(getPauseTime(tries)); + } catch (InterruptedException e) { + throw new InterruptedIOException("Interrupted when opening" + " regions; " + + actualRegCount.get() + " of " + numRegs + " regions processed so far"); + } + if (actualRegCount.get() > prevRegCount) { // Making progress + prevRegCount = actualRegCount.get(); + tries = -1; + } + } else if (isTableEnabled(indexTableName)) { + return; + } + } else if (isTableEnabled(desc.getName())) { + return; + } + } + } + throw new TableNotEnabledException( + "Retries exhausted while still waiting for table: " + + desc.getNameAsString() + " to be enabled"); + } + + private boolean isIndexedHTD(final HTableDescriptor desc) { + try { + Class IndexedHTDKlass = HTableDescriptor.class; + IndexedHTDKlass = Class.forName("org.apache.hadoop.hbase.index.IndexedHTableDescriptor"); + return IndexedHTDKlass.isInstance(desc); + } catch (ClassNotFoundException e) { + // Nothing to do. + } + return false; + } + + public class MetaScannerVisitorBaseWithTableName implements MetaScannerVisitor { + byte[] tableName = null; + AtomicInteger actualRegCount = new AtomicInteger(0); + + MetaScannerVisitorBaseWithTableName(String tableName) { + this.tableName = Bytes.toBytes(tableName); + } + + AtomicInteger getActualRgnCnt() { + return actualRegCount; + } + + @Override + public void close() throws IOException { + } + + @Override + public boolean processRow(Result rowResult) throws IOException { + HRegionInfo info = + Writables.getHRegionInfoOrNull(rowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER)); + // If regioninfo is null, skip this row + if (null == info) { + return true; + } + if (!(Bytes.equals(info.getTableName(), tableName))) { + return false; + } + String hostAndPort = null; + byte[] value = rowResult.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); + // Make sure that regions are assigned to server + if (value != null && value.length > 0) { + hostAndPort = Bytes.toString(value); + } + if (!(info.isOffline() || info.isSplit()) && hostAndPort != null) { + actualRegCount.incrementAndGet(); + } + return true; + } + } + + /** + * Creates a new table but does not block and wait for it to come online. + * Asynchronous operation. To check if the table exists, use + * {@link: #isTableAvailable} -- it is not safe to create an HTable + * instance to this table before it is available. + * Note : Avoid passing empty split key. + * @param desc table descriptor for table + * + * @throws IllegalArgumentException Bad table name, if the split keys + * are repeated and if the split key has empty byte array. + * @throws MasterNotRunningException if master is not running + * @throws TableExistsException if table already exists (If concurrent + * threads, the table may have been created between test-for-existence + * and attempt-at-creation). + * @throws IOException + */ + public void createTableAsync(HTableDescriptor desc, byte [][] splitKeys) + throws IOException { + HTableDescriptor.isLegalTableName(desc.getName()); + if(splitKeys != null && splitKeys.length > 0) { + Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR); + // Verify there are no duplicate split keys + byte [] lastKey = null; + for(byte [] splitKey : splitKeys) { + if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) { + throw new IllegalArgumentException( + "Empty split key must not be passed in the split keys."); + } + if(lastKey != null && Bytes.equals(splitKey, lastKey)) { + throw new IllegalArgumentException("All split keys must be unique, " + + "found duplicate: " + Bytes.toStringBinary(splitKey) + + ", " + Bytes.toStringBinary(lastKey)); + } + lastKey = splitKey; + } + } + try { + getMaster().createTable(desc, splitKeys); + } catch (RemoteException e) { + throw e.unwrapRemoteException(); + } + } + + /** + * Deletes a table. + * Synchronous operation. + * + * @param tableName name of table to delete + * @throws IOException if a remote or network exception occurs + */ + public void deleteTable(final String tableName) throws IOException { + deleteTable(Bytes.toBytes(tableName)); + } + + /** + * Deletes a table. + * Synchronous operation. + * + * @param tableName name of table to delete + * @throws IOException if a remote or network exception occurs + */ + public void deleteTable(final byte [] tableName) throws IOException { + isMasterRunning(); + HTableDescriptor.isLegalTableName(tableName); + HRegionLocation firstMetaServer = getFirstMetaServerForTable(tableName); + boolean tableExists = true; + try { + getMaster().deleteTable(tableName); + } catch (RemoteException e) { + throw RemoteExceptionHandler.decodeRemoteException(e); + } + // Wait until all regions deleted + HRegionInterface server = + connection.getHRegionConnection(firstMetaServer.getHostname(), firstMetaServer.getPort()); + for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) { + long scannerId = -1L; + try { + + Scan scan = MetaReader.getScanForTableName(tableName); + scan.addColumn(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + scannerId = server.openScanner( + firstMetaServer.getRegionInfo().getRegionName(), scan); + // Get a batch at a time. + Result values = server.next(scannerId); + + // let us wait until .META. table is updated and + // HMaster removes the table from its HTableDescriptors + if (values == null) { + tableExists = false; + HTableDescriptor[] htds = getMaster().getHTableDescriptors(); + if (htds != null && htds.length > 0) { + for (HTableDescriptor htd: htds) { + if (Bytes.equals(tableName, htd.getName())) { + tableExists = true; + break; + } + } + } + if (!tableExists) { + break; + } + } + } catch (IOException ex) { + if(tries == numRetries - 1) { // no more tries left + if (ex instanceof RemoteException) { + ex = RemoteExceptionHandler.decodeRemoteException((RemoteException) ex); + } + throw ex; + } + } finally { + if (scannerId != -1L) { + try { + server.close(scannerId); + } catch (Exception ex) { + LOG.warn(ex); + } + } + } + try { + Thread.sleep(getPauseTime(tries)); + } catch (InterruptedException e) { + // continue + } + } + + if (tableExists) { + throw new IOException("Retries exhausted, it took too long to wait"+ + " for the table " + Bytes.toString(tableName) + " to be deleted."); + } + // Delete cached information to prevent clients from using old locations + this.connection.clearRegionCache(tableName); + LOG.info("Deleted " + Bytes.toString(tableName)); + } + + /** + * Deletes tables matching the passed in pattern and wait on completion. + * + * Warning: Use this method carefully, there is no prompting and the effect is + * immediate. Consider using {@link #listTables(java.lang.String)} and + * {@link #deleteTable(byte[])} + * + * @param regex The regular expression to match table names against + * @return Table descriptors for tables that couldn't be deleted + * @throws IOException + * @see #deleteTables(java.util.regex.Pattern) + * @see #deleteTable(java.lang.String) + */ + public HTableDescriptor[] deleteTables(String regex) throws IOException { + return deleteTables(Pattern.compile(regex)); + } + + /** + * Delete tables matching the passed in pattern and wait on completion. + * + * Warning: Use this method carefully, there is no prompting and the effect is + * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and + * {@link #deleteTable(byte[])} + * + * @param pattern The pattern to match table names against + * @return Table descriptors for tables that couldn't be deleted + * @throws IOException + */ + public HTableDescriptor[] deleteTables(Pattern pattern) throws IOException { + List failed = new LinkedList(); + for (HTableDescriptor table : listTables(pattern)) { + try { + deleteTable(table.getName()); + } catch (IOException ex) { + LOG.info("Failed to delete table " + table.getNameAsString(), ex); + failed.add(table); + } + } + return failed.toArray(new HTableDescriptor[failed.size()]); + } + + + public void enableTable(final String tableName) + throws IOException { + enableTable(Bytes.toBytes(tableName)); + } + + /** + * Enable a table. May timeout. Use {@link #enableTableAsync(byte[])} + * and {@link #isTableEnabled(byte[])} instead. + * The table has to be in disabled state for it to be enabled. + * @param tableName name of the table + * @throws IOException if a remote or network exception occurs + * There could be couple types of IOException + * TableNotFoundException means the table doesn't exist. + * TableNotDisabledException means the table isn't in disabled state. + * @see #isTableEnabled(byte[]) + * @see #disableTable(byte[]) + * @see #enableTableAsync(byte[]) + */ + public void enableTable(final byte [] tableName) + throws IOException { + enableTableAsync(tableName); + + // Wait until all regions are enabled + waitUntilTableIsEnabled(tableName); + + LOG.info("Enabled table " + Bytes.toString(tableName)); + } + + /** + * Wait for the table to be enabled and available + * If enabling the table exceeds the retry period, an exception is thrown. + * @param tableName name of the table + * @throws IOException if a remote or network exception occurs or + * table is not enabled after the retries period. + */ + private void waitUntilTableIsEnabled(final byte[] tableName) throws IOException { + boolean enabled = false; + long start = EnvironmentEdgeManager.currentTimeMillis(); + for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) { + enabled = isTableEnabled(tableName) && isTableAvailable(tableName); + if (enabled) { + break; + } + long sleep = getPauseTime(tries); + if (LOG.isDebugEnabled()) { + LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " + + "enabled in " + Bytes.toString(tableName)); + } + try { + Thread.sleep(sleep); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + // Do this conversion rather than let it out because do not want to + // change the method signature. + throw new IOException("Interrupted", e); + } + } + if (!enabled) { + long msec = EnvironmentEdgeManager.currentTimeMillis() - start; + throw new IOException("Table '" + Bytes.toString(tableName) + + "' not yet enabled, after " + msec + "ms."); + } + } + + public void enableTableAsync(final String tableName) + throws IOException { + enableTableAsync(Bytes.toBytes(tableName)); + } + + /** + * Brings a table on-line (enables it). Method returns immediately though + * enable of table may take some time to complete, especially if the table + * is large (All regions are opened as part of enabling process). Check + * {@link #isTableEnabled(byte[])} to learn when table is fully online. If + * table is taking too long to online, check server logs. + * @param tableName + * @throws IOException + * @since 0.90.0 + */ + public void enableTableAsync(final byte [] tableName) + throws IOException { + HTableDescriptor.isLegalTableName(tableName); + isMasterRunning(); + try { + getMaster().enableTable(tableName); + } catch (RemoteException e) { + throw e.unwrapRemoteException(); + } + LOG.info("Started enable of " + Bytes.toString(tableName)); + } + + /** + * Enable tables matching the passed in pattern and wait on completion. + * + * Warning: Use this method carefully, there is no prompting and the effect is + * immediate. Consider using {@link #listTables(java.lang.String)} and + * {@link #enableTable(byte[])} + * + * @param regex The regular expression to match table names against + * @throws IOException + * @see #enableTables(java.util.regex.Pattern) + * @see #enableTable(java.lang.String) + */ + public HTableDescriptor[] enableTables(String regex) throws IOException { + return enableTables(Pattern.compile(regex)); + } + + /** + * Enable tables matching the passed in pattern and wait on completion. + * + * Warning: Use this method carefully, there is no prompting and the effect is + * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and + * {@link #enableTable(byte[])} + * + * @param pattern The pattern to match table names against + * @throws IOException + */ + public HTableDescriptor[] enableTables(Pattern pattern) throws IOException { + List failed = new LinkedList(); + for (HTableDescriptor table : listTables(pattern)) { + if (isTableDisabled(table.getName())) { + try { + enableTable(table.getName()); + } catch (IOException ex) { + LOG.info("Failed to enable table " + table.getNameAsString(), ex); + failed.add(table); + } + } + } + return failed.toArray(new HTableDescriptor[failed.size()]); + } + + public void disableTableAsync(final String tableName) throws IOException { + disableTableAsync(Bytes.toBytes(tableName)); + } + + /** + * Starts the disable of a table. If it is being served, the master + * will tell the servers to stop serving it. This method returns immediately. + * The disable of a table can take some time if the table is large (all + * regions are closed as part of table disable operation). + * Call {@link #isTableDisabled(byte[])} to check for when disable completes. + * If table is taking too long to online, check server logs. + * @param tableName name of table + * @throws IOException if a remote or network exception occurs + * @see #isTableDisabled(byte[]) + * @see #isTableEnabled(byte[]) + * @since 0.90.0 + */ + public void disableTableAsync(final byte [] tableName) throws IOException { + HTableDescriptor.isLegalTableName(tableName); + isMasterRunning(); + try { + getMaster().disableTable(tableName); + } catch (RemoteException e) { + throw e.unwrapRemoteException(); + } + LOG.info("Started disable of " + Bytes.toString(tableName)); + } + + public void disableTable(final String tableName) + throws IOException { + disableTable(Bytes.toBytes(tableName)); + } + + /** + * Disable table and wait on completion. May timeout eventually. Use + * {@link #disableTableAsync(byte[])} and {@link #isTableDisabled(String)} + * instead. + * The table has to be in enabled state for it to be disabled. + * @param tableName + * @throws IOException + * There could be couple types of IOException + * TableNotFoundException means the table doesn't exist. + * TableNotEnabledException means the table isn't in enabled state. + */ + public void disableTable(final byte [] tableName) + throws IOException { + disableTableAsync(tableName); + // Wait until table is disabled + boolean disabled = false; + for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) { + disabled = isTableDisabled(tableName); + if (disabled) { + break; + } + long sleep = getPauseTime(tries); + if (LOG.isDebugEnabled()) { + LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " + + "disabled in " + Bytes.toString(tableName)); + } + try { + Thread.sleep(sleep); + } catch (InterruptedException e) { + // Do this conversion rather than let it out because do not want to + // change the method signature. + Thread.currentThread().interrupt(); + throw new IOException("Interrupted", e); + } + } + if (!disabled) { + throw new RegionException("Retries exhausted, it took too long to wait"+ + " for the table " + Bytes.toString(tableName) + " to be disabled."); + } + LOG.info("Disabled " + Bytes.toString(tableName)); + } + + /** + * Disable tables matching the passed in pattern and wait on completion. + * + * Warning: Use this method carefully, there is no prompting and the effect is + * immediate. Consider using {@link #listTables(java.lang.String)} and + * {@link #disableTable(byte[])} + * + * @param regex The regular expression to match table names against + * @return Table descriptors for tables that couldn't be disabled + * @throws IOException + * @see #disableTables(java.util.regex.Pattern) + * @see #disableTable(java.lang.String) + */ + public HTableDescriptor[] disableTables(String regex) throws IOException { + return disableTables(Pattern.compile(regex)); + } + + /** + * Disable tables matching the passed in pattern and wait on completion. + * + * Warning: Use this method carefully, there is no prompting and the effect is + * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and + * {@link #disableTable(byte[])} + * + * @param pattern The pattern to match table names against + * @return Table descriptors for tables that couldn't be disabled + * @throws IOException + */ + public HTableDescriptor[] disableTables(Pattern pattern) throws IOException { + List failed = new LinkedList(); + for (HTableDescriptor table : listTables(pattern)) { + if (isTableEnabled(table.getName())) { + try { + disableTable(table.getName()); + } catch (IOException ex) { + LOG.info("Failed to disable table " + table.getNameAsString(), ex); + failed.add(table); + } + } + } + return failed.toArray(new HTableDescriptor[failed.size()]); + } + + /** + * @param tableName name of table to check + * @return true if table is on-line + * @throws IOException if a remote or network exception occurs + */ + public boolean isTableEnabled(String tableName) throws IOException { + return isTableEnabled(Bytes.toBytes(tableName)); + } + /** + * @param tableName name of table to check + * @return true if table is on-line + * @throws IOException if a remote or network exception occurs + */ + public boolean isTableEnabled(byte[] tableName) throws IOException { + if (!HTableDescriptor.isMetaTable(tableName)) { + HTableDescriptor.isLegalTableName(tableName); + } + if(!tableExists(tableName)){ + throw new TableNotFoundException(Bytes.toString(tableName)); + } + + boolean indexEnabled = this.conf.getBoolean("hbase.use.secondary.index", false); + if (!indexEnabled) { + return connection.isTableEnabled(tableName); + } else { + boolean isTableEnabled = connection.isTableEnabled(tableName); + if (isTableEnabled && !isIndexTable(Bytes.toString(tableName))) { + String indexTableName = getIndexTableName(Bytes.toString(tableName)); + if (connection.isTableAvailable(Bytes.toBytes(indexTableName))) { + return connection.isTableEnabled(Bytes.toBytes(indexTableName)); + } + return true; + } + return isTableEnabled; + } + } + + /** + * @param tableName name of table to check + * @return true if table is off-line + * @throws IOException if a remote or network exception occurs + */ + public boolean isTableDisabled(final String tableName) throws IOException { + return isTableDisabled(Bytes.toBytes(tableName)); + } + + public static boolean isIndexTable(String tableName) { + return tableName.endsWith(INDEX_TABLE_SUFFIX); + } + + public static String getIndexTableName(String tableName) { + // TODO The suffix for the index table is fixed now. Do we allow to make this configurable? + // We can handle things in byte[] way? + return tableName + INDEX_TABLE_SUFFIX; + } + + /** + * @param tableName name of table to check + * @return true if table is off-line + * @throws IOException if a remote or network exception occurs + */ + public boolean isTableDisabled(byte[] tableName) throws IOException { + if (!HTableDescriptor.isMetaTable(tableName)) { + HTableDescriptor.isLegalTableName(tableName); + } + boolean indexEnabled = this.conf.getBoolean("hbase.use.secondary.index", false); + if (!indexEnabled) { + return connection.isTableDisabled(tableName); + } else { + boolean isTableDisabled = connection.isTableDisabled(tableName); + if (isTableDisabled && !isIndexTable(Bytes.toString(tableName))) { + String indexTableName = getIndexTableName(Bytes.toString(tableName)); + if (connection.isTableAvailable(Bytes.toBytes(indexTableName))) { + return connection.isTableDisabled(Bytes.toBytes(indexTableName)); + } + return true; + } + return isTableDisabled; + } + } + + /** + * @param tableName name of table to check + * @return true if all regions of the table are available + * @throws IOException if a remote or network exception occurs + */ + public boolean isTableAvailable(byte[] tableName) throws IOException { + return connection.isTableAvailable(tableName); + } + + /** + * @param tableName name of table to check + * @return true if all regions of the table are available + * @throws IOException if a remote or network exception occurs + */ + public boolean isTableAvailable(String tableName) throws IOException { + return connection.isTableAvailable(Bytes.toBytes(tableName)); + } + + /** + * Get the status of alter command - indicates how many regions have received + * the updated schema Asynchronous operation. + * + * @param tableName + * name of the table to get the status of + * @return Pair indicating the number of regions updated Pair.getFirst() is the + * regions that are yet to be updated Pair.getSecond() is the total number + * of regions of the table + * @throws IOException + * if a remote or network exception occurs + */ + public Pair getAlterStatus(final byte[] tableName) + throws IOException { + HTableDescriptor.isLegalTableName(tableName); + try { + return getMaster().getAlterStatus(tableName); + } catch (RemoteException e) { + throw RemoteExceptionHandler.decodeRemoteException(e); + } + } + + /** + * Add a column to an existing table. + * Asynchronous operation. + * + * @param tableName name of the table to add column to + * @param column column descriptor of column to be added + * @throws IOException if a remote or network exception occurs + */ + public void addColumn(final String tableName, HColumnDescriptor column) + throws IOException { + addColumn(Bytes.toBytes(tableName), column); + } + + /** + * Add a column to an existing table. + * Asynchronous operation. + * + * @param tableName name of the table to add column to + * @param column column descriptor of column to be added + * @throws IOException if a remote or network exception occurs + */ + public void addColumn(final byte [] tableName, HColumnDescriptor column) + throws IOException { + HTableDescriptor.isLegalTableName(tableName); + try { + getMaster().addColumn(tableName, column); + } catch (RemoteException e) { + throw RemoteExceptionHandler.decodeRemoteException(e); + } + } + + /** + * Delete a column from a table. + * Asynchronous operation. + * + * @param tableName name of table + * @param columnName name of column to be deleted + * @throws IOException if a remote or network exception occurs + */ + public void deleteColumn(final String tableName, final String columnName) + throws IOException { + deleteColumn(Bytes.toBytes(tableName), Bytes.toBytes(columnName)); + } + + /** + * Delete a column from a table. + * Asynchronous operation. + * + * @param tableName name of table + * @param columnName name of column to be deleted + * @throws IOException if a remote or network exception occurs + */ + public void deleteColumn(final byte [] tableName, final byte [] columnName) + throws IOException { + try { + getMaster().deleteColumn(tableName, columnName); + } catch (RemoteException e) { + throw RemoteExceptionHandler.decodeRemoteException(e); + } + } + + /** + * Modify an existing column family on a table. + * Asynchronous operation. + * + * @param tableName name of table + * @param descriptor new column descriptor to use + * @throws IOException if a remote or network exception occurs + */ + public void modifyColumn(final String tableName, HColumnDescriptor descriptor) + throws IOException { + modifyColumn(Bytes.toBytes(tableName), descriptor); + } + + /** + * Modify an existing column family on a table. + * Asynchronous operation. + * + * @param tableName name of table + * @param descriptor new column descriptor to use + * @throws IOException if a remote or network exception occurs + */ + public void modifyColumn(final byte [] tableName, HColumnDescriptor descriptor) + throws IOException { + try { + getMaster().modifyColumn(tableName, descriptor); + } catch (RemoteException re) { + // Convert RE exceptions in here; client shouldn't have to deal with them, + // at least w/ the type of exceptions that come out of this method: + // TableNotFoundException, etc. + throw RemoteExceptionHandler.decodeRemoteException(re); + } + } + + /** + * Close a region. For expert-admins. Runs close on the regionserver. The + * master will not be informed of the close. + * @param regionname region name to close + * @param serverName If supplied, we'll use this location rather than + * the one currently in .META. + * @throws IOException if a remote or network exception occurs + */ + public void closeRegion(final String regionname, final String serverName) + throws IOException { + closeRegion(Bytes.toBytes(regionname), serverName); + } + + /** + * Close a region. For expert-admins Runs close on the regionserver. The + * master will not be informed of the close. + * @param regionname region name to close + * @param serverName The servername of the regionserver. If passed null we + * will use servername found in the .META. table. A server name + * is made of host, port and startcode. Here is an example: + * host187.example.com,60020,1289493121758 + * @throws IOException if a remote or network exception occurs + */ + public void closeRegion(final byte [] regionname, final String serverName) + throws IOException { + CatalogTracker ct = getCatalogTracker(); + try { + if (serverName != null) { + Pair pair = MetaReader.getRegion(ct, regionname); + if (pair == null || pair.getFirst() == null) { + throw new UnknownRegionException(Bytes.toStringBinary(regionname)); + } else { + closeRegion(new ServerName(serverName), pair.getFirst()); + } + } else { + Pair pair = MetaReader.getRegion(ct, regionname); + if (pair == null) { + throw new UnknownRegionException(Bytes.toStringBinary(regionname)); + } else if (pair.getSecond() == null) { + throw new NoServerForRegionException(Bytes.toStringBinary(regionname)); + } else { + closeRegion(pair.getSecond(), pair.getFirst()); + } + } + } finally { + cleanupCatalogTracker(ct); + } + } + + /** + * For expert-admins. Runs close on the regionserver. Closes a region based on + * the encoded region name. The region server name is mandatory. If the + * servername is provided then based on the online regions in the specified + * regionserver the specified region will be closed. The master will not be + * informed of the close. Note that the regionname is the encoded regionname. + * + * @param encodedRegionName + * The encoded region name; i.e. the hash that makes up the region + * name suffix: e.g. if regionname is + * TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. + * , then the encoded region name is: + * 527db22f95c8a9e0116f0cc13c680396. + * @param serverName + * The servername of the regionserver. A server name is made of host, + * port and startcode. This is mandatory. Here is an example: + * host187.example.com,60020,1289493121758 + * @return true if the region was closed, false if not. + * @throws IOException + * if a remote or network exception occurs + */ + public boolean closeRegionWithEncodedRegionName(final String encodedRegionName, + final String serverName) throws IOException { + byte[] encodedRegionNameInBytes = Bytes.toBytes(encodedRegionName); + if (null == serverName || ("").equals(serverName.trim())) { + throw new IllegalArgumentException( + "The servername cannot be null or empty."); + } + ServerName sn = new ServerName(serverName); + HRegionInterface rs = this.connection.getHRegionConnection( + sn.getHostname(), sn.getPort()); + // Close the region without updating zk state. + boolean isRegionClosed = rs.closeRegion(encodedRegionNameInBytes, false); + if (false == isRegionClosed) { + LOG.error("Not able to close the region " + encodedRegionName + "."); + } + return isRegionClosed; + } + + /** + * Close a region. For expert-admins Runs close on the regionserver. The + * master will not be informed of the close. + * @param sn + * @param hri + * @throws IOException + */ + public void closeRegion(final ServerName sn, final HRegionInfo hri) + throws IOException { + HRegionInterface rs = + this.connection.getHRegionConnection(sn.getHostname(), sn.getPort()); + // Close the region without updating zk state. + rs.closeRegion(hri, false); + } + + /** + * Flush a table or an individual region. + * Asynchronous operation. + * + * @param tableNameOrRegionName table or region to flush + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + public void flush(final String tableNameOrRegionName) + throws IOException, InterruptedException { + flush(Bytes.toBytes(tableNameOrRegionName)); + } + + /** + * Flush a table or an individual region. + * Asynchronous operation. + * + * @param tableNameOrRegionName table or region to flush + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + public void flush(final byte [] tableNameOrRegionName) + throws IOException, InterruptedException { + CatalogTracker ct = getCatalogTracker(); + try { + Pair regionServerPair + = getRegion(tableNameOrRegionName, ct); + if (regionServerPair != null) { + if (regionServerPair.getSecond() == null) { + throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName)); + } else { + flush(regionServerPair.getSecond(), regionServerPair.getFirst()); + } + } else { + final String tableName = tableNameString(tableNameOrRegionName, ct); + List> pairs = + MetaReader.getTableRegionsAndLocations(ct, + tableName); + for (Pair pair: pairs) { + if (pair.getFirst().isOffline()) continue; + if (pair.getSecond() == null) continue; + try { + flush(pair.getSecond(), pair.getFirst()); + } catch (NotServingRegionException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to flush " + pair.getFirst() + ": " + + StringUtils.stringifyException(e)); + } + } + } + } + } finally { + cleanupCatalogTracker(ct); + } + } + + private void flush(final ServerName sn, final HRegionInfo hri) + throws IOException { + HRegionInterface rs = + this.connection.getHRegionConnection(sn.getHostname(), sn.getPort()); + rs.flushRegion(hri); + } + + /** + * Compact a table or an individual region. + * Asynchronous operation. + * + * @param tableNameOrRegionName table or region to compact + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + public void compact(final String tableNameOrRegionName) + throws IOException, InterruptedException { + compact(Bytes.toBytes(tableNameOrRegionName)); + } + + /** + * Compact a table or an individual region. + * Asynchronous operation. + * + * @param tableNameOrRegionName table or region to compact + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + public void compact(final byte [] tableNameOrRegionName) + throws IOException, InterruptedException { + compact(tableNameOrRegionName, null, false); + } + + /** + * Compact a column family within a table or region. + * Asynchronous operation. + * + * @param tableOrRegionName table or region to compact + * @param columnFamily column family within a table or region + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + public void compact(String tableOrRegionName, String columnFamily) + throws IOException, InterruptedException { + compact(Bytes.toBytes(tableOrRegionName), Bytes.toBytes(columnFamily)); + } + + /** + * Compact a column family within a table or region. + * Asynchronous operation. + * + * @param tableNameOrRegionName table or region to compact + * @param columnFamily column family within a table or region + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + public void compact(final byte [] tableNameOrRegionName, final byte[] columnFamily) + throws IOException, InterruptedException { + compact(tableNameOrRegionName, columnFamily, false); + } + + /** + * Major compact a table or an individual region. + * Asynchronous operation. + * + * @param tableNameOrRegionName table or region to major compact + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + public void majorCompact(final String tableNameOrRegionName) + throws IOException, InterruptedException { + majorCompact(Bytes.toBytes(tableNameOrRegionName)); + } + + /** + * Major compact a table or an individual region. + * Asynchronous operation. + * + * @param tableNameOrRegionName table or region to major compact + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + public void majorCompact(final byte [] tableNameOrRegionName) + throws IOException, InterruptedException { + compact(tableNameOrRegionName, null, true); + } + + /** + * Major compact a column family within a table or region. + * Asynchronous operation. + * + * @param tableNameOrRegionName table or region to major compact + * @param columnFamily column family within a table or region + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + public void majorCompact(final String tableNameOrRegionName, + final String columnFamily) throws IOException, InterruptedException { + majorCompact(Bytes.toBytes(tableNameOrRegionName), + Bytes.toBytes(columnFamily)); + } + + /** + * Major compact a column family within a table or region. + * Asynchronous operation. + * + * @param tableNameOrRegionName table or region to major compact + * @param columnFamily column family within a table or region + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + public void majorCompact(final byte [] tableNameOrRegionName, + final byte[] columnFamily) throws IOException, InterruptedException { + compact(tableNameOrRegionName, columnFamily, true); + } + + /** + * Compact a table or an individual region. + * Asynchronous operation. + * + * @param tableNameOrRegionName table or region to compact + * @param columnFamily column family within a table or region + * @param major True if we are to do a major compaction. + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + private void compact(final byte [] tableNameOrRegionName, + final byte[] columnFamily, final boolean major) + throws IOException, InterruptedException { + CatalogTracker ct = getCatalogTracker(); + try { + Pair regionServerPair + = getRegion(tableNameOrRegionName, ct); + if (regionServerPair != null) { + if (regionServerPair.getSecond() == null) { + throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName)); + } else { + compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major, columnFamily); + } + } else { + final String tableName = tableNameString(tableNameOrRegionName, ct); + List> pairs = + MetaReader.getTableRegionsAndLocations(ct, + tableName); + for (Pair pair: pairs) { + if (pair.getFirst().isOffline()) continue; + if (pair.getSecond() == null) continue; + try { + compact(pair.getSecond(), pair.getFirst(), major, columnFamily); + } catch (NotServingRegionException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to" + (major ? " major" : "") + " compact " + + pair.getFirst() + ": " + + StringUtils.stringifyException(e)); + } + } + } + } + } finally { + cleanupCatalogTracker(ct); + } + } + + private void compact(final ServerName sn, final HRegionInfo hri, + final boolean major, final byte [] family) + throws IOException { + HRegionInterface rs = + this.connection.getHRegionConnection(sn.getHostname(), sn.getPort()); + if (family != null) { + try { + rs.compactRegion(hri, major, family); + } catch (IOException ioe) { + String notFoundMsg = "java.lang.NoSuchMethodException: org.apache.hadoop.hbase.ipc.HRegionInterface." + + "compactRegion(org.apache.hadoop.hbase.HRegionInfo, boolean, [B)"; + if (ioe.getMessage().contains(notFoundMsg)) { + throw new IOException("per-column family compaction not supported on this version " + + "of the HBase server. You may still compact at the table or region level by " + + "omitting the column family name. Alternatively, you can upgrade the HBase server"); + } + throw ioe; + } + } else { + rs.compactRegion(hri, major); + } + } + + /** + * Move the region r to dest. + * @param encodedRegionName The encoded region name; i.e. the hash that makes + * up the region name suffix: e.g. if regionname is + * TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396., + * then the encoded region name is: 527db22f95c8a9e0116f0cc13c680396. + * @param destServerName The servername of the destination regionserver. If + * passed the empty byte array we'll assign to a random server. A server name + * is made of host, port and startcode. Here is an example: + * host187.example.com,60020,1289493121758 + * @throws UnknownRegionException Thrown if we can't find a region named + * encodedRegionName + * @throws ZooKeeperConnectionException + * @throws MasterNotRunningException + */ + public void move(final byte [] encodedRegionName, final byte [] destServerName) + throws UnknownRegionException, MasterNotRunningException, ZooKeeperConnectionException { + getMaster().move(encodedRegionName, destServerName); + } + + /** + * @param regionName + * Region name to assign. + * @throws MasterNotRunningException + * @throws ZooKeeperConnectionException + * @throws IOException + */ + public void assign(final byte[] regionName) throws MasterNotRunningException, + ZooKeeperConnectionException, IOException { + getMaster().assign(regionName); + } + + /** + * Unassign a region from current hosting regionserver. Region will then be + * assigned to a regionserver chosen at random. Region could be reassigned + * back to the same server. Use {@link #move(byte[], byte[])} if you want + * to control the region movement. + * @param regionName Region to unassign. Will clear any existing RegionPlan + * if one found. + * @param force If true, force unassign (Will remove region from + * regions-in-transition too if present. If results in double assignment + * use hbck -fix to resolve. To be used by experts). + * @throws MasterNotRunningException + * @throws ZooKeeperConnectionException + * @throws IOException + */ + public void unassign(final byte [] regionName, final boolean force) + throws MasterNotRunningException, ZooKeeperConnectionException, IOException { + getMaster().unassign(regionName, force); + } + + /** + * Turn the load balancer on or off. + * @param b If true, enable balancer. If false, disable balancer. + * @return Previous balancer value + * @deprecated use setBalancerRunning(boolean, boolean) instead + */ + @Deprecated + public boolean balanceSwitch(final boolean b) + throws MasterNotRunningException, ZooKeeperConnectionException { + return getMaster().balanceSwitch(b); + } + + /** + * Turn the load balancer on or off. + * @param on If true, enable balancer. If false, disable balancer. + * @param synchronous If true, it waits until current balance() call, if outstanding, to return. + * @return Previous balancer value + */ + public boolean setBalancerRunning(final boolean on, final boolean synchronous) + throws MasterNotRunningException, ZooKeeperConnectionException { + if (synchronous && synchronousBalanceSwitchSupported) { + try { + return getMaster().synchronousBalanceSwitch(on); + } catch (UndeclaredThrowableException ute) { + String error = ute.getCause().getMessage(); + if (error != null && error.matches( + "(?s).+NoSuchMethodException:.+synchronousBalanceSwitch.+")) { + LOG.info("HMaster doesn't support synchronousBalanceSwitch"); + synchronousBalanceSwitchSupported = false; + } else { + throw ute; + } + } + } + return balanceSwitch(on); + } + + /** + * Invoke the balancer. Will run the balancer and if regions to move, it will + * go ahead and do the reassignments. Can NOT run for various reasons. Check + * logs. + * @return True if balancer ran, false otherwise. + */ + public boolean balancer() + throws MasterNotRunningException, ZooKeeperConnectionException { + return getMaster().balance(); + } + + /** + * Split a table or an individual region. + * Asynchronous operation. + * + * @param tableNameOrRegionName table or region to split + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + public void split(final String tableNameOrRegionName) + throws IOException, InterruptedException { + split(Bytes.toBytes(tableNameOrRegionName)); + } + + /** + * Split a table or an individual region. Implicitly finds an optimal split + * point. Asynchronous operation. + * + * @param tableNameOrRegionName table to region to split + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + */ + public void split(final byte [] tableNameOrRegionName) + throws IOException, InterruptedException { + split(tableNameOrRegionName, null); + } + + public void split(final String tableNameOrRegionName, + final String splitPoint) throws IOException, InterruptedException { + split(Bytes.toBytes(tableNameOrRegionName), Bytes.toBytes(splitPoint)); + } + + /** + * Split a table or an individual region. + * Asynchronous operation. + * + * @param tableNameOrRegionName table to region to split + * @param splitPoint the explicit position to split on + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException interrupt exception occurred + */ + public void split(final byte [] tableNameOrRegionName, + final byte [] splitPoint) throws IOException, InterruptedException { + CatalogTracker ct = getCatalogTracker(); + try { + Pair regionServerPair + = getRegion(tableNameOrRegionName, ct); + if (regionServerPair != null) { + if (regionServerPair.getSecond() == null) { + throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName)); + } else { + split(regionServerPair.getSecond(), regionServerPair.getFirst(), splitPoint); + } + } else { + final String tableName = tableNameString(tableNameOrRegionName, ct); + List> pairs = + MetaReader.getTableRegionsAndLocations(ct, + tableName); + for (Pair pair: pairs) { + // May not be a server for a particular row + if (pair.getSecond() == null) continue; + HRegionInfo r = pair.getFirst(); + // check for parents + if (r.isSplitParent()) continue; + // if a split point given, only split that particular region + if (splitPoint != null && !r.containsRow(splitPoint)) continue; + // call out to region server to do split now + split(pair.getSecond(), pair.getFirst(), splitPoint); + } + } + } finally { + cleanupCatalogTracker(ct); + } + } + + private void split(final ServerName sn, final HRegionInfo hri, + byte[] splitPoint) throws IOException { + HRegionInterface rs = + this.connection.getHRegionConnection(sn.getHostname(), sn.getPort()); + rs.splitRegion(hri, splitPoint); + } + + /** + * Modify an existing table, more IRB friendly version. + * Asynchronous operation. This means that it may be a while before your + * schema change is updated across all of the table. + * + * @param tableName name of table. + * @param htd modified description of the table + * @throws IOException if a remote or network exception occurs + */ + public void modifyTable(final byte [] tableName, HTableDescriptor htd) + throws IOException { + try { + getMaster().modifyTable(tableName, htd); + } catch (RemoteException re) { + // Convert RE exceptions in here; client shouldn't have to deal with them, + // at least w/ the type of exceptions that come out of this method: + // TableNotFoundException, etc. + throw RemoteExceptionHandler.decodeRemoteException(re); + } + } + + /** + * @param tableNameOrRegionName Name of a table or name of a region. + * @param ct A {@link CatalogTracker} instance (caller of this method usually has one). + * @return a pair of HRegionInfo and ServerName if tableNameOrRegionName is + * a verified region name (we call {@link MetaReader#getRegion( CatalogTracker, byte[])} + * else null. + * Throw an exception if tableNameOrRegionName is null. + * @throws IOException + */ + Pair getRegion(final byte[] tableNameOrRegionName, + final CatalogTracker ct) throws IOException { + if (tableNameOrRegionName == null) { + throw new IllegalArgumentException("Pass a table name or region name"); + } + Pair pair = MetaReader.getRegion(ct, tableNameOrRegionName); + if (pair == null) { + final AtomicReference> result = + new AtomicReference>(null); + final String encodedName = Bytes.toString(tableNameOrRegionName); + MetaScannerVisitor visitor = new MetaScannerVisitorBase() { + @Override + public boolean processRow(Result data) throws IOException { + if (data == null || data.size() <= 0) { + return true; + } + HRegionInfo info = MetaReader.parseHRegionInfoFromCatalogResult( + data, HConstants.REGIONINFO_QUALIFIER); + if (info == null) { + LOG.warn("No serialized HRegionInfo in " + data); + return true; + } + if (!encodedName.equals(info.getEncodedName())) return true; + ServerName sn = MetaReader.getServerNameFromCatalogResult(data); + result.set(new Pair(info, sn)); + return false; // found the region, stop + } + }; + + MetaScanner.metaScan(conf, visitor); + pair = result.get(); + } + return pair; + } + + /** + * Convert the table name byte array into a table name string and check if table + * exists or not. + * @param tableNameBytes Name of a table. + * @param ct A {@link #CatalogTracker} instance (caller of this method usually has one). + * @return tableName in string form. + * @throws IOException if a remote or network exception occurs. + * @throws TableNotFoundException if table does not exist. + */ + private String tableNameString(final byte[] tableNameBytes, CatalogTracker ct) + throws IOException { + String tableNameString = Bytes.toString(tableNameBytes); + if (!MetaReader.tableExists(ct, tableNameString)) { + throw new TableNotFoundException(tableNameString); + } + return tableNameString; + } + + /** + * Shuts down the HBase cluster + * @throws IOException if a remote or network exception occurs + */ + public synchronized void shutdown() throws IOException { + isMasterRunning(); + try { + getMaster().shutdown(); + } catch (RemoteException e) { + throw RemoteExceptionHandler.decodeRemoteException(e); + } + } + + /** + * Shuts down the current HBase master only. + * Does not shutdown the cluster. + * @see #shutdown() + * @throws IOException if a remote or network exception occurs + */ + public synchronized void stopMaster() throws IOException { + isMasterRunning(); + try { + getMaster().stopMaster(); + } catch (RemoteException e) { + throw RemoteExceptionHandler.decodeRemoteException(e); + } + } + + /** + * Stop the designated regionserver + * @param hostnamePort Hostname and port delimited by a : as in + * example.org:1234 + * @throws IOException if a remote or network exception occurs + */ + public synchronized void stopRegionServer(final String hostnamePort) + throws IOException { + String hostname = Addressing.parseHostname(hostnamePort); + int port = Addressing.parsePort(hostnamePort); + HRegionInterface rs = + this.connection.getHRegionConnection(hostname, port); + rs.stop("Called by admin client " + this.connection.toString()); + } + + /** + * @return cluster status + * @throws IOException if a remote or network exception occurs + */ + public ClusterStatus getClusterStatus() throws IOException { + return getMaster().getClusterStatus(); + } + + private HRegionLocation getFirstMetaServerForTable(final byte [] tableName) + throws IOException { + return connection.locateRegion(HConstants.META_TABLE_NAME, + HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false)); + } + + /** + * @return Configuration used by the instance. + */ + public Configuration getConfiguration() { + return this.conf; + } + + /** + * Check to see if HBase is running. Throw an exception if not. + * + * @param conf system configuration + * @throws MasterNotRunningException if the master is not running + * @throws ZooKeeperConnectionException if unable to connect to zookeeper + */ + public static void checkHBaseAvailable(Configuration conf) + throws MasterNotRunningException, ZooKeeperConnectionException { + Configuration copyOfConf = HBaseConfiguration.create(conf); + copyOfConf.setInt("hbase.client.retries.number", 1); + HBaseAdmin admin = new HBaseAdmin(copyOfConf); + try { + admin.close(); + } catch (IOException ioe) { + admin.LOG.info("Failed to close connection", ioe); + } + } + + /** + * get the regions of a given table. + * + * @param tableName the name of the table + * @return Ordered list of {@link HRegionInfo}. + * @throws IOException + */ + public List getTableRegions(final byte[] tableName) + throws IOException { + CatalogTracker ct = getCatalogTracker(); + List Regions = null; + try { + Regions = MetaReader.getTableRegions(ct, tableName, true); + } finally { + cleanupCatalogTracker(ct); + } + return Regions; + } + + public void close() throws IOException { + if (this.connection != null) { + this.connection.close(); + } + } + + /** + * Get tableDescriptors + * @param tableNames List of table names + * @return HTD[] the tableDescriptor + * @throws IOException if a remote or network exception occurs + */ + public HTableDescriptor[] getTableDescriptors(List tableNames) + throws IOException { + return this.connection.getHTableDescriptors(tableNames); + } + + /** + * Roll the log writer. That is, start writing log messages to a new file. + * + * @param serverName + * The servername of the regionserver. A server name is made of host, + * port and startcode. This is mandatory. Here is an example: + * host187.example.com,60020,1289493121758 + * @return If lots of logs, flush the returned regions so next time through + * we can clean logs. Returns null if nothing to flush. Names are actual + * region names as returned by {@link HRegionInfo#getEncodedName()} + * @throws IOException if a remote or network exception occurs + * @throws FailedLogCloseException + */ + public synchronized byte[][] rollHLogWriter(String serverName) + throws IOException, FailedLogCloseException { + ServerName sn = new ServerName(serverName); + HRegionInterface rs = this.connection.getHRegionConnection( + sn.getHostname(), sn.getPort()); + return rs.rollHLogWriter(); + } + + public String[] getMasterCoprocessors() { + try { + return getClusterStatus().getMasterCoprocessors(); + } catch (IOException e) { + LOG.error("Could not getClusterStatus()",e); + return null; + } + } + + /** + * Get the current compaction state of a table or region. + * It could be in a major compaction, a minor compaction, both, or none. + * + * @param tableNameOrRegionName table or region to major compact + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + * @return the current compaction state + */ + public CompactionState getCompactionState(final String tableNameOrRegionName) + throws IOException, InterruptedException { + return getCompactionState(Bytes.toBytes(tableNameOrRegionName)); + } + + /** + * Get the current compaction state of a table or region. + * It could be in a major compaction, a minor compaction, both, or none. + * + * @param tableNameOrRegionName table or region to major compact + * @throws IOException if a remote or network exception occurs + * @throws InterruptedException + * @return the current compaction state + */ + public CompactionState getCompactionState(final byte [] tableNameOrRegionName) + throws IOException, InterruptedException { + CompactionState state = CompactionState.NONE; + CatalogTracker ct = getCatalogTracker(); + try { + Pair regionServerPair + = getRegion(tableNameOrRegionName, ct); + if (regionServerPair != null) { + if (regionServerPair.getSecond() == null) { + throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName)); + } else { + ServerName sn = regionServerPair.getSecond(); + HRegionInterface rs = + this.connection.getHRegionConnection(sn.getHostname(), sn.getPort()); + return CompactionState.valueOf( + rs.getCompactionState(regionServerPair.getFirst().getRegionName())); + } + } else { + final String tableName = tableNameString(tableNameOrRegionName, ct); + List> pairs = + MetaReader.getTableRegionsAndLocations(ct, tableName); + for (Pair pair: pairs) { + if (pair.getFirst().isOffline()) continue; + if (pair.getSecond() == null) continue; + try { + ServerName sn = pair.getSecond(); + HRegionInterface rs = + this.connection.getHRegionConnection(sn.getHostname(), sn.getPort()); + switch (CompactionState.valueOf( + rs.getCompactionState(pair.getFirst().getRegionName()))) { + case MAJOR_AND_MINOR: + return CompactionState.MAJOR_AND_MINOR; + case MAJOR: + if (state == CompactionState.MINOR) { + return CompactionState.MAJOR_AND_MINOR; + } + state = CompactionState.MAJOR; + break; + case MINOR: + if (state == CompactionState.MAJOR) { + return CompactionState.MAJOR_AND_MINOR; + } + state = CompactionState.MINOR; + break; + case NONE: + default: // nothing, continue + } + } catch (NotServingRegionException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to get compaction state of " + + pair.getFirst() + ": " + + StringUtils.stringifyException(e)); + } + } + } + } + } finally { + cleanupCatalogTracker(ct); + } + return state; + } + + /** + * Creates and returns a proxy to the CoprocessorProtocol instance running in the + * master. + * + * @param protocol The class or interface defining the remote protocol + * @return A CoprocessorProtocol instance + */ + public T coprocessorProxy( + Class protocol) { + return (T) Proxy.newProxyInstance(this.getClass().getClassLoader(), + new Class[]{protocol}, + new MasterExecRPCInvoker(conf, + connection, + protocol)); + } + + + /** + * Create a timestamp consistent snapshot for the given table. + *

+ * Snapshots are considered unique based on the name of the snapshot. Attempts to take a + * snapshot with the same name (even a different type or with different parameters) will fail with + * a {@link SnapshotCreationException} indicating the duplicate naming. + *

+ * Snapshot names follow the same naming constraints as tables in HBase. See + * {@link HTableDescriptor#isLegalTableName(byte[])}. + * @param snapshotName name of the snapshot to be created + * @param tableName name of the table for which snapshot is created + * @throws IOException if a remote or network exception occurs + * @throws SnapshotCreationException if snapshot creation failed + * @throws IllegalArgumentException if the snapshot request is formatted incorrectly + */ + public void snapshot(final String snapshotName, final String tableName) throws IOException, + SnapshotCreationException, IllegalArgumentException { + snapshot(snapshotName, tableName, SnapshotDescription.Type.FLUSH); + } + + /** + * Take a snapshot for the given table. If the table is enabled, a FLUSH-type snapshot will be + * taken. If the table is disabled, an offline snapshot is taken. + *

+ * Snapshots are considered unique based on the name of the snapshot. Attempts to take a + * snapshot with the same name (even a different type or with different parameters) will fail with + * a {@link SnapshotCreationException} indicating the duplicate naming. + *

+ * Snapshot names follow the same naming constraints as tables in HBase. See + * {@link HTableDescriptor#isLegalTableName(byte[])}. + * @param snapshotName name of the snapshot to be created + * @param tableName name of the table for which snapshot is created + * @throws IOException if a remote or network exception occurs + * @throws SnapshotCreationException if snapshot creation failed + * @throws IllegalArgumentException if the snapshot request is formatted incorrectly + */ + public void snapshot(final byte[] snapshotName, final byte[] tableName) throws IOException, + SnapshotCreationException, IllegalArgumentException { + snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName)); + } + + /** + * Create typed snapshot of the table. + *

+ * Snapshots are considered unique based on the name of the snapshot. Attempts to take a + * snapshot with the same name (even a different type or with different parameters) will fail with + * a {@link SnapshotCreationException} indicating the duplicate naming. + *

+ * Snapshot names follow the same naming constraints as tables in HBase. See + * {@link HTableDescriptor#isLegalTableName(byte[])}. + *

+ * @param snapshotName name to give the snapshot on the filesystem. Must be unique from all other + * snapshots stored on the cluster + * @param tableName name of the table to snapshot + * @param type type of snapshot to take + * @throws IOException we fail to reach the master + * @throws SnapshotCreationException if snapshot creation failed + * @throws IllegalArgumentException if the snapshot request is formatted incorrectly + */ + public void snapshot(final String snapshotName, final String tableName, + SnapshotDescription.Type type) throws IOException, SnapshotCreationException, + IllegalArgumentException { + SnapshotDescription.Builder builder = SnapshotDescription.newBuilder(); + builder.setTable(tableName); + builder.setName(snapshotName); + builder.setType(type); + snapshot(builder.build()); + } + + /** + * Take a snapshot and wait for the server to complete that snapshot (blocking). + *

+ * Only a single snapshot should be taken at a time for an instance of HBase, or results may be + * undefined (you can tell multiple HBase clusters to snapshot at the same time, but only one at a + * time for a single cluster). + *

+ * Snapshots are considered unique based on the name of the snapshot. Attempts to take a + * snapshot with the same name (even a different type or with different parameters) will fail with + * a {@link SnapshotCreationException} indicating the duplicate naming. + *

+ * Snapshot names follow the same naming constraints as tables in HBase. See + * {@link HTableDescriptor#isLegalTableName(byte[])}. + *

+ * You should probably use {@link #snapshot(String, String)} or {@link #snapshot(byte[], byte[])} + * unless you are sure about the type of snapshot that you want to take. + * @param snapshot snapshot to take + * @throws IOException or we lose contact with the master. + * @throws SnapshotCreationException if snapshot failed to be taken + * @throws IllegalArgumentException if the snapshot request is formatted incorrectly + */ + public void snapshot(SnapshotDescription snapshot) throws IOException, SnapshotCreationException, + IllegalArgumentException { + HSnapshotDescription snapshotWritable = new HSnapshotDescription(snapshot); + + try { + // actually take the snapshot + long max = takeSnapshotAsync(snapshot); + long start = EnvironmentEdgeManager.currentTimeMillis(); + long maxPauseTime = max / this.numRetries; + boolean done = false; + int tries = 0; + LOG.debug("Waiting a max of " + max + " ms for snapshot '" + + SnapshotDescriptionUtils.toString(snapshot) + "' to complete. (max " + + maxPauseTime + " ms per retry)"); + while (tries == 0 || (EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done) { + try { + // sleep a backoff <= pauseTime amount + long sleep = getPauseTime(tries++); + sleep = sleep > maxPauseTime ? maxPauseTime : sleep; + LOG.debug("(#" + tries + ") Sleeping: " + sleep + + "ms while waiting for snapshot completion."); + Thread.sleep(sleep); + + } catch (InterruptedException e) { + LOG.debug("Interrupted while waiting for snapshot " + snapshot + " to complete"); + Thread.currentThread().interrupt(); + } + LOG.debug("Getting current status of snapshot from master..."); + done = getMaster().isSnapshotDone(snapshotWritable); + } + + if (!done) { + throw new SnapshotCreationException("Snapshot '" + snapshot.getName() + + "' wasn't completed in expectedTime:" + max + " ms", snapshot); + } + } catch (RemoteException e) { + throw RemoteExceptionHandler.decodeRemoteException(e); + } + } + + /** + * Take a snapshot without waiting for the server to complete that snapshot (asynchronous) + *

+ * Only a single snapshot should be taken at a time, or results may be undefined. + * @param snapshot snapshot to take + * @return the max time in millis to wait for the snapshot + * @throws IOException if the snapshot did not succeed or we lose contact with the master. + * @throws SnapshotCreationException if snapshot creation failed + * @throws IllegalArgumentException if the snapshot request is formatted incorrectly + */ + public long takeSnapshotAsync(SnapshotDescription snapshot) throws IOException, + SnapshotCreationException { + SnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot); + HSnapshotDescription snapshotWritable = new HSnapshotDescription(snapshot); + return getMaster().snapshot(snapshotWritable); + } + + /** + * Check the current state of the passed snapshot. + *

+ * There are three possible states: + *

    + *
  1. running - returns false
    2. + *
  2. finished - returns true
    3. + *
  3. finished with error - throws the exception that caused the snapshot to fail
    4. + *
+ *

+ * The cluster only knows about the most recent snapshot. Therefore, if another snapshot has been + * run/started since the snapshot your are checking, you will recieve an + * {@link UnknownSnapshotException}. + * @param snapshot description of the snapshot to check + * @return true if the snapshot is completed, false if the snapshot is still + * running + * @throws IOException if we have a network issue + * @throws HBaseSnapshotException if the snapshot failed + * @throws UnknownSnapshotException if the requested snapshot is unknown + */ + public boolean isSnapshotFinished(final SnapshotDescription snapshot) + throws IOException, HBaseSnapshotException, UnknownSnapshotException { + try { + return getMaster().isSnapshotDone(new HSnapshotDescription(snapshot)); + } catch (RemoteException e) { + throw RemoteExceptionHandler.decodeRemoteException(e); + } + } + + /** + * Restore the specified snapshot on the original table. (The table must be disabled) + * Before restoring the table, a new snapshot with the current table state is created. + * In case of failure, the table will be rolled back to the its original state. + * + * @param snapshotName name of the snapshot to restore + * @throws IOException if a remote or network exception occurs + * @throws RestoreSnapshotException if snapshot failed to be restored + * @throws IllegalArgumentException if the restore request is formatted incorrectly + */ + public void restoreSnapshot(final byte[] snapshotName) + throws IOException, RestoreSnapshotException { + restoreSnapshot(Bytes.toString(snapshotName)); + } + + /** + * Restore the specified snapshot on the original table. (The table must be disabled) + * Before restoring the table, a new snapshot with the current table state is created. + * In case of failure, the table will be rolled back to its original state. + * + * @param snapshotName name of the snapshot to restore + * @throws IOException if a remote or network exception occurs + * @throws RestoreSnapshotException if snapshot failed to be restored + * @throws IllegalArgumentException if the restore request is formatted incorrectly + */ + public void restoreSnapshot(final String snapshotName) + throws IOException, RestoreSnapshotException { + String rollbackSnapshot = snapshotName + "-" + EnvironmentEdgeManager.currentTimeMillis(); + + String tableName = null; + for (SnapshotDescription snapshotInfo: listSnapshots()) { + if (snapshotInfo.getName().equals(snapshotName)) { + tableName = snapshotInfo.getTable(); + break; + } + } + + if (tableName == null) { + throw new RestoreSnapshotException( + "Unable to find the table name for snapshot=" + snapshotName); + } + + // Take a snapshot of the current state + snapshot(rollbackSnapshot, tableName); + + // Restore snapshot + try { + internalRestoreSnapshot(snapshotName, tableName); + } catch (IOException e) { + // Try to rollback + try { + String msg = "Restore snapshot=" + snapshotName + + " failed. Rollback to snapshot=" + rollbackSnapshot + " succeeded."; + LOG.error(msg, e); + internalRestoreSnapshot(rollbackSnapshot, tableName); + throw new RestoreSnapshotException(msg, e); + } catch (IOException ex) { + String msg = "Failed to restore and rollback to snapshot=" + rollbackSnapshot; + LOG.error(msg, ex); + throw new RestoreSnapshotException(msg, ex); + } + } + } + + /** + * Create a new table by cloning the snapshot content. + * + * @param snapshotName name of the snapshot to be cloned + * @param tableName name of the table where the snapshot will be restored + * @throws IOException if a remote or network exception occurs + * @throws TableExistsException if table to be created already exists + * @throws RestoreSnapshotException if snapshot failed to be cloned + * @throws IllegalArgumentException if the specified table has not a valid name + */ + public void cloneSnapshot(final byte[] snapshotName, final byte[] tableName) + throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException { + cloneSnapshot(Bytes.toString(snapshotName), Bytes.toString(tableName)); + } + + /** + * Create a new table by cloning the snapshot content. + * + * @param snapshotName name of the snapshot to be cloned + * @param tableName name of the table where the snapshot will be restored + * @throws IOException if a remote or network exception occurs + * @throws TableExistsException if table to be created already exists + * @throws RestoreSnapshotException if snapshot failed to be cloned + * @throws IllegalArgumentException if the specified table has not a valid name + */ + public void cloneSnapshot(final String snapshotName, final String tableName) + throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException { + if (tableExists(tableName)) { + throw new TableExistsException("Table '" + tableName + " already exists"); + } + internalRestoreSnapshot(snapshotName, tableName); + waitUntilTableIsEnabled(Bytes.toBytes(tableName)); + } + + /** + * Execute Restore/Clone snapshot and wait for the server to complete (blocking). + * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to + * create an HTable instance to this table before it is available. + * @param snapshot snapshot to restore + * @param tableName table name to restore the snapshot on + * @throws IOException if a remote or network exception occurs + * @throws RestoreSnapshotException if snapshot failed to be restored + * @throws IllegalArgumentException if the restore request is formatted incorrectly + */ + private void internalRestoreSnapshot(final String snapshotName, final String tableName) + throws IOException, RestoreSnapshotException { + HSnapshotDescription snapshot = new HSnapshotDescription( + SnapshotDescription.newBuilder().setName(snapshotName).setTable(tableName).build()); + + try { + // actually restore the snapshot + getMaster().restoreSnapshot(snapshot); + + final long maxPauseTime = 5000; + boolean done = false; + int tries = 0; + while (!done) { + try { + // sleep a backoff <= pauseTime amount + long sleep = getPauseTime(tries++); + sleep = sleep > maxPauseTime ? maxPauseTime : sleep; + LOG.debug(tries + ") Sleeping: " + sleep + " ms while we wait for snapshot restore to complete."); + Thread.sleep(sleep); + } catch (InterruptedException e) { + LOG.debug("Interrupted while waiting for snapshot " + snapshot + " restore to complete"); + Thread.currentThread().interrupt(); + } + LOG.debug("Getting current status of snapshot restore from master..."); + done = getMaster().isRestoreSnapshotDone(snapshot); + } + if (!done) { + throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored."); + } + } catch (RemoteException e) { + throw RemoteExceptionHandler.decodeRemoteException(e); + } + } + + /** + * List completed snapshots. + * @return a list of snapshot descriptors for completed snapshots + * @throws IOException if a network error occurs + */ + public List listSnapshots() throws IOException { + List snapshots = new LinkedList(); + try { + for (HSnapshotDescription snapshot: getMaster().getCompletedSnapshots()) { + snapshots.add(snapshot.getProto()); + } + } catch (RemoteException e) { + throw RemoteExceptionHandler.decodeRemoteException(e); + } + return snapshots; + } + + /** + * Delete an existing snapshot. + * @param snapshotName name of the snapshot + * @throws IOException if a remote or network exception occurs + */ + public void deleteSnapshot(final byte[] snapshotName) throws IOException { + // make sure the snapshot is possibly valid + HTableDescriptor.isLegalTableName(snapshotName); + // do the delete + SnapshotDescription snapshot = SnapshotDescription.newBuilder() + .setName(Bytes.toString(snapshotName)).build(); + try { + getMaster().deleteSnapshot(new HSnapshotDescription(snapshot)); + } catch (RemoteException e) { + throw RemoteExceptionHandler.decodeRemoteException(e); + } + } + + /** + * Delete an existing snapshot. + * @param snapshotName name of the snapshot + * @throws IOException if a remote or network exception occurs + */ + public void deleteSnapshot(final String snapshotName) throws IOException { + deleteSnapshot(Bytes.toBytes(snapshotName)); + } + + /** + * Forcefully sets the table state as DISABLED in ZK + * @param tableName + */ + public void setDisableTable(String tableName) throws IOException { + setTableState(tableName, "DISABLED"); + } + + /** + * Forcefully sets the table state as ENABLED in ZK + * @param tablename + */ + public void setEnableTable(String tableName) throws IOException { + setTableState(tableName, "ENABLED"); + } + + private void setTableState(String tableName, String state) throws IOException { + try { + getMaster().setTableState(tableName, state); + } catch (RemoteException e) { + throw e.unwrapRemoteException(); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnection.java b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java new file mode 100644 index 0000000..1774a89 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/HConnection.java @@ -0,0 +1,410 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.ipc.HMasterInterface; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; + +/** + * Cluster connection. Hosts a connection to the ZooKeeper ensemble and + * thereafter into the HBase cluster. Knows how to locate regions out on the cluster, + * keeps a cache of locations and then knows how to recalibrate after they move. + * {@link HConnectionManager} manages instances of this class. + * + *

HConnections are used by {@link HTable} mostly but also by + * {@link HBaseAdmin}, {@link CatalogTracker}, + * and {@link ZooKeeperWatcher}. HConnection instances can be shared. Sharing + * is usually what you want because rather than each HConnection instance + * having to do its own discovery of regions out on the cluster, instead, all + * clients get to share the one cache of locations. Sharing makes cleanup of + * HConnections awkward. See {@link HConnectionManager} for cleanup + * discussion. + * + * @see HConnectionManager + */ +public interface HConnection extends Abortable, Closeable { + /** + * @return Configuration instance being used by this HConnection instance. + */ + public Configuration getConfiguration(); + + /** + * Retrieve ZooKeeperWatcher used by this connection. + * @return ZooKeeperWatcher handle being used by the connection. + * @throws IOException if a remote or network exception occurs + * @deprecated Removed because it was a mistake exposing zookeeper in this + * interface (ZooKeeper is an implementation detail). + * Deprecated in HBase 0.94 + */ + @Deprecated + public ZooKeeperWatcher getZooKeeperWatcher() throws IOException; + + /** + * @return proxy connection to master server for this instance + * @throws MasterNotRunningException if the master is not running + * @throws ZooKeeperConnectionException if unable to connect to zookeeper + * @deprecated Removed because it was a mistake exposing master in this + * interface (master is an implementation detail). Master functions are + * available from HConnection or HBaseAdmin, without having to use + * directly the master. + * Deprecated in HBase 0.94 + */ + @Deprecated + public HMasterInterface getMaster() + throws MasterNotRunningException, ZooKeeperConnectionException; + + /** @return - true if the master server is running */ + public boolean isMasterRunning() + throws MasterNotRunningException, ZooKeeperConnectionException; + + /** + * A table that isTableEnabled == false and isTableDisabled == false + * is possible. This happens when a table has a lot of regions + * that must be processed. + * @param tableName table name + * @return true if the table is enabled, false otherwise + * @throws IOException if a remote or network exception occurs + */ + public boolean isTableEnabled(byte[] tableName) throws IOException; + + /** + * @param tableName table name + * @return true if the table is disabled, false otherwise + * @throws IOException if a remote or network exception occurs + */ + public boolean isTableDisabled(byte[] tableName) throws IOException; + + /** + * @param tableName table name + * @return true if all regions of the table are available, false otherwise + * @throws IOException if a remote or network exception occurs + */ + public boolean isTableAvailable(byte[] tableName) throws IOException; + + /** + * List all the userspace tables. In other words, scan the META table. + * + * If we wanted this to be really fast, we could implement a special + * catalog table that just contains table names and their descriptors. + * Right now, it only exists as part of the META table's region info. + * + * @return - returns an array of HTableDescriptors + * @throws IOException if a remote or network exception occurs + */ + public HTableDescriptor[] listTables() throws IOException; + + /** + * @param tableName table name + * @return table metadata + * @throws IOException if a remote or network exception occurs + */ + public HTableDescriptor getHTableDescriptor(byte[] tableName) + throws IOException; + + /** + * Find the location of the region of tableName that row + * lives in. + * @param tableName name of the table row is in + * @param row row key you're trying to find the region of + * @return HRegionLocation that describes where to find the region in + * question + * @throws IOException if a remote or network exception occurs + */ + public HRegionLocation locateRegion(final byte [] tableName, + final byte [] row) + throws IOException; + + /** + * Allows flushing the region cache. + */ + public void clearRegionCache(); + + /** + * Allows flushing the region cache of all locations that pertain to + * tableName + * @param tableName Name of the table whose regions we are to remove from + * cache. + */ + public void clearRegionCache(final byte [] tableName); + + /** + * Deletes cached locations for the specific region. + * @param location The location object for the region, to be purged from cache. + */ + public void deleteCachedRegionLocation(final HRegionLocation location); + + /** + * Find the location of the region of tableName that row + * lives in, ignoring any value that might be in the cache. + * @param tableName name of the table row is in + * @param row row key you're trying to find the region of + * @return HRegionLocation that describes where to find the region in + * question + * @throws IOException if a remote or network exception occurs + */ + public HRegionLocation relocateRegion(final byte [] tableName, + final byte [] row) + throws IOException; + + /** + * Gets the location of the region of regionName. + * @param regionName name of the region to locate + * @return HRegionLocation that describes where to find the region in + * question + * @throws IOException if a remote or network exception occurs + */ + public HRegionLocation locateRegion(final byte [] regionName) + throws IOException; + + /** + * Gets the locations of all regions in the specified table, tableName. + * @param tableName table to get regions of + * @return list of region locations for all regions of table + * @throws IOException + */ + public List locateRegions(final byte[] tableName) + throws IOException; + + /** + * Gets the locations of all regions in the specified table, tableName. + * @param tableName table to get regions of + * @param useCache Should we use the cache to retrieve the region information. + * @param offlined True if we are to include offlined regions, false and we'll leave out offlined + * regions from returned list. + * @return list of region locations for all regions of table + * @throws IOException + */ + public List locateRegions(final byte[] tableName, final boolean useCache, + final boolean offlined) throws IOException; + + /** + * Establishes a connection to the region server at the specified address. + * @param regionServer - the server to connect to + * @return proxy for HRegionServer + * @throws IOException if a remote or network exception occurs + * @deprecated Use {@link #getHRegionConnection(String, int)} + */ + public HRegionInterface getHRegionConnection(HServerAddress regionServer) + throws IOException; + + /** + * Establishes a connection to the region server at the specified address. + * @param hostname RegionServer hostname + * @param port RegionServer port + * @return proxy for HRegionServer + * @throws IOException if a remote or network exception occurs + * + */ + public HRegionInterface getHRegionConnection(final String hostname, final int port) + throws IOException; + + /** + * Establishes a connection to the region server at the specified address. + * @param regionServer - the server to connect to + * @param getMaster - do we check if master is alive + * @return proxy for HRegionServer + * @throws IOException if a remote or network exception occurs + * @deprecated Use {@link #getHRegionConnection(HServerAddress, boolean)} + */ + public HRegionInterface getHRegionConnection(HServerAddress regionServer, + boolean getMaster) + throws IOException; + + /** + * Establishes a connection to the region server at the specified address. + * @param hostname RegionServer hostname + * @param port RegionServer port + * @param getMaster - do we check if master is alive + * @return proxy for HRegionServer + * @throws IOException if a remote or network exception occurs + */ + public HRegionInterface getHRegionConnection(final String hostname, + final int port, boolean getMaster) + throws IOException; + + /** + * Find region location hosting passed row + * @param tableName table name + * @param row Row to find. + * @param reload If true do not use cache, otherwise bypass. + * @return Location of row. + * @throws IOException if a remote or network exception occurs + */ + HRegionLocation getRegionLocation(byte [] tableName, byte [] row, + boolean reload) + throws IOException; + + /** + * Pass in a ServerCallable with your particular bit of logic defined and + * this method will manage the process of doing retries with timed waits + * and refinds of missing regions. + * + * @param the type of the return value + * @param callable callable to run + * @return an object of type T + * @throws IOException if a remote or network exception occurs + * @throws RuntimeException other unspecified error + * @deprecated Use {@link HConnectionManager#withoutRetries(ServerCallable)} + */ + public T getRegionServerWithRetries(ServerCallable callable) + throws IOException, RuntimeException; + + /** + * Pass in a ServerCallable with your particular bit of logic defined and + * this method will pass it to the defined region server. + * @param the type of the return value + * @param callable callable to run + * @return an object of type T + * @throws IOException if a remote or network exception occurs + * @throws RuntimeException other unspecified error + * @deprecated Use {@link HConnectionManager#withoutRetries(ServerCallable)} + */ + public T getRegionServerWithoutRetries(ServerCallable callable) + throws IOException, RuntimeException; + + /** + * Process a mixed batch of Get, Put and Delete actions. All actions for a + * RegionServer are forwarded in one RPC call. + * + * + * @param actions The collection of actions. + * @param tableName Name of the hbase table + * @param pool thread pool for parallel execution + * @param results An empty array, same size as list. If an exception is thrown, + * you can test here for partial results, and to determine which actions + * processed successfully. + * @throws IOException if there are problems talking to META. Per-item + * exceptions are stored in the results array. + */ + public void processBatch(List actions, final byte[] tableName, + ExecutorService pool, Object[] results) + throws IOException, InterruptedException; + + /** + * Parameterized batch processing, allowing varying return types for different + * {@link Row} implementations. + */ + public void processBatchCallback(List list, + byte[] tableName, + ExecutorService pool, + Object[] results, + Batch.Callback callback) throws IOException, InterruptedException; + + + /** + * Executes the given + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call} + * callable for each row in the given list and invokes + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Callback#update(byte[], byte[], Object)} + * for each result returned. + * + * @param protocol the protocol interface being called + * @param rows a list of row keys for which the callable should be invoked + * @param tableName table name for the coprocessor invoked + * @param pool ExecutorService used to submit the calls per row + * @param call instance on which to invoke + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call#call(Object)} + * for each row + * @param callback instance on which to invoke + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Callback#update(byte[], byte[], Object)} + * for each result + * @param the protocol interface type + * @param the callable's return type + * @throws IOException + */ + public void processExecs( + final Class protocol, + List rows, + final byte[] tableName, + ExecutorService pool, + final Batch.Call call, + final Batch.Callback callback) throws IOException, Throwable; + + /** + * Enable or disable region cache prefetch for the table. It will be + * applied for the given table's all HTable instances within this + * connection. By default, the cache prefetch is enabled. + * @param tableName name of table to configure. + * @param enable Set to true to enable region cache prefetch. + */ + public void setRegionCachePrefetch(final byte[] tableName, + final boolean enable); + + /** + * Check whether region cache prefetch is enabled or not. + * @param tableName name of table to check + * @return true if table's region cache prefetch is enabled. Otherwise + * it is disabled. + */ + public boolean getRegionCachePrefetch(final byte[] tableName); + + /** + * Load the region map and warm up the global region cache for the table. + * @param tableName name of the table to perform region cache prewarm. + * @param regions a region map. + */ + public void prewarmRegionCache(final byte[] tableName, + final Map regions); + + /** + * Scan zookeeper to get the number of region servers + * @return the number of region servers that are currently running + * @throws IOException if a remote or network exception occurs + * @deprecated This method will be changed from public to package protected. + */ + public int getCurrentNrHRS() throws IOException; + + /** + * @param tableNames List of table names + * @return HTD[] table metadata + * @throws IOException if a remote or network exception occurs + */ + public HTableDescriptor[] getHTableDescriptors(List tableNames) + throws IOException; + + /** + * @return true if this connection is closed + */ + public boolean isClosed(); + + /** + * Clear any caches that pertain to server name sn + * @param sn A server name as hostname:port + */ + public void clearCaches(final String sn); +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java new file mode 100644 index 0000000..e813f7a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java @@ -0,0 +1,1924 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.Closeable; +import java.io.IOException; +import java.lang.reflect.Proxy; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArraySet; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MasterAddressTracker; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.RemoteExceptionHandler; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.ipc.ExecRPCInvoker; +import org.apache.hadoop.hbase.ipc.HBaseRPC; +import org.apache.hadoop.hbase.ipc.HMasterInterface; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.ipc.RpcEngine; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Addressing; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.SoftValueSortedMap; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.hbase.zookeeper.ClusterId; +import org.apache.hadoop.hbase.zookeeper.RootRegionTracker; +import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.zookeeper.KeeperException; + +/** + * A non-instantiable class that manages {@link HConnection}s. + * This class has a static Map of {@link HConnection} instances keyed by + * {@link Configuration}; all invocations of {@link #getConnection(Configuration)} + * that pass the same {@link Configuration} instance will be returned the same + * {@link HConnection} instance (Adding properties to a Configuration + * instance does not change its object identity). Sharing {@link HConnection} + * instances is usually what you want; all clients of the {@link HConnection} + * instances share the HConnections' cache of Region locations rather than each + * having to discover for itself the location of meta, root, etc. It makes + * sense for the likes of the pool of HTables class {@link HTablePool}, for + * instance (If concerned that a single {@link HConnection} is insufficient + * for sharing amongst clients in say an heavily-multithreaded environment, + * in practise its not proven to be an issue. Besides, {@link HConnection} is + * implemented atop Hadoop RPC and as of this writing, Hadoop RPC does a + * connection per cluster-member, exclusively). + * + *

But sharing connections + * makes clean up of {@link HConnection} instances a little awkward. Currently, + * clients cleanup by calling + * {@link #deleteConnection(Configuration)}. This will shutdown the + * zookeeper connection the HConnection was using and clean up all + * HConnection resources as well as stopping proxies to servers out on the + * cluster. Not running the cleanup will not end the world; it'll + * just stall the closeup some and spew some zookeeper connection failed + * messages into the log. Running the cleanup on a {@link HConnection} that is + * subsequently used by another will cause breakage so be careful running + * cleanup. + *

To create a {@link HConnection} that is not shared by others, you can + * create a new {@link Configuration} instance, pass this new instance to + * {@link #getConnection(Configuration)}, and then when done, close it up by + * doing something like the following: + * 

+ * {@code
+ * Configuration newConfig = new Configuration(originalConf);
+ * HConnection connection = HConnectionManager.getConnection(newConfig);
+ * // Use the connection to your hearts' delight and then when done...
+ * HConnectionManager.deleteConnection(newConfig, true);
+ * }
+ *
+ *

Cleanup used to be done inside in a shutdown hook. On startup we'd + * register a shutdown hook that called {@link #deleteAllConnections()} + * on its way out but the order in which shutdown hooks run is not defined so + * were problematic for clients of HConnection that wanted to register their + * own shutdown hooks so we removed ours though this shifts the onus for + * cleanup to the client. + */ +@SuppressWarnings("serial") +public class HConnectionManager { + // An LRU Map of HConnectionKey -> HConnection (TableServer). All + // access must be synchronized. This map is not private because tests + // need to be able to tinker with it. + static final Map HBASE_INSTANCES; + + public static final int MAX_CACHED_HBASE_INSTANCES; + + private static Log LOG = LogFactory.getLog(HConnectionManager.class); + + static { + // We set instances to one more than the value specified for {@link + // HConstants#ZOOKEEPER_MAX_CLIENT_CNXNS}. By default, the zk default max + // connections to the ensemble from the one client is 30, so in that case we + // should run into zk issues before the LRU hit this value of 31. + MAX_CACHED_HBASE_INSTANCES = HBaseConfiguration.create().getInt( + HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, + HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS) + 1; + HBASE_INSTANCES = new LinkedHashMap( + (int) (MAX_CACHED_HBASE_INSTANCES / 0.75F) + 1, 0.75F, true) { + @Override + protected boolean removeEldestEntry( + Map.Entry eldest) { + return size() > MAX_CACHED_HBASE_INSTANCES; + } + }; + } + + /* + * Non-instantiable. + */ + protected HConnectionManager() { + super(); + } + + /** + * Get the connection that goes with the passed conf + * configuration instance. + * If no current connection exists, method creates a new connection for the + * passed conf instance. + * @param conf configuration + * @return HConnection object for conf + * @throws ZooKeeperConnectionException + */ + public static HConnection getConnection(Configuration conf) + throws ZooKeeperConnectionException { + HConnectionKey connectionKey = new HConnectionKey(conf); + synchronized (HBASE_INSTANCES) { + HConnectionImplementation connection = HBASE_INSTANCES.get(connectionKey); + if (connection == null) { + connection = new HConnectionImplementation(conf, true); + HBASE_INSTANCES.put(connectionKey, connection); + } else if (connection.isClosed()) { + HConnectionManager.deleteConnection(connectionKey, true); + connection = new HConnectionImplementation(conf, true); + HBASE_INSTANCES.put(connectionKey, connection); + } + connection.incCount(); + return connection; + } + } + + /** + * Create a new HConnection instance using the passed conf + * instance. + * Note: This bypasses the usual HConnection life cycle management! + * Use this with caution, the caller is responsible for closing the + * created connection. + * @param conf configuration + * @return HConnection object for conf + * @throws ZooKeeperConnectionException + */ + public static HConnection createConnection(Configuration conf) + throws ZooKeeperConnectionException { + return new HConnectionImplementation(conf, false); + } + + /** + * Delete connection information for the instance specified by configuration. + * If there are no more references to it, this will then close connection to + * the zookeeper ensemble and let go of all resources. + * + * @param conf + * configuration whose identity is used to find {@link HConnection} + * instance. + * @param stopProxy + * No longer used. This parameter is ignored. + * @deprecated use {@link #createConnection(org.apache.hadoop.conf.Configuration)} instead + */ + @Deprecated + public static void deleteConnection(Configuration conf, boolean stopProxy) { + deleteConnection(conf); + } + + /** + * Delete connection information for the instance specified by configuration. + * If there are no more references to it, this will then close connection to + * the zookeeper ensemble and let go of all resources. + * + * @param conf + * configuration whose identity is used to find {@link HConnection} + * instance. + */ + public static void deleteConnection(Configuration conf) { + deleteConnection(new HConnectionKey(conf), false); + } + + /** + * Delete stale connection information for the instance specified by configuration. + * This will then close connection to + * the zookeeper ensemble and let go of all resources. + * + * @param connection + */ + public static void deleteStaleConnection(HConnection connection) { + deleteConnection(connection, true); + } + + /** + * Delete information for all connections. + * @param stopProxy No longer used. This parameter is ignored. + * @deprecated use {@link #deleteAllConnections()} instead + */ + @Deprecated + public static void deleteAllConnections(boolean stopProxy) { + deleteAllConnections(); + } + + /** + * Delete information for all connections. + * @throws IOException + */ + public static void deleteAllConnections() { + synchronized (HBASE_INSTANCES) { + Set connectionKeys = new HashSet(); + connectionKeys.addAll(HBASE_INSTANCES.keySet()); + for (HConnectionKey connectionKey : connectionKeys) { + deleteConnection(connectionKey, false); + } + HBASE_INSTANCES.clear(); + } + } + + private static void deleteConnection(HConnection connection, boolean staleConnection) { + synchronized (HBASE_INSTANCES) { + for (Entry connectionEntry : HBASE_INSTANCES + .entrySet()) { + if (connectionEntry.getValue() == connection) { + deleteConnection(connectionEntry.getKey(), staleConnection); + break; + } + } + } + } + + private static void deleteConnection(HConnectionKey connectionKey, + boolean staleConnection) { + synchronized (HBASE_INSTANCES) { + HConnectionImplementation connection = HBASE_INSTANCES + .get(connectionKey); + if (connection != null) { + connection.decCount(); + if (connection.isZeroReference() || staleConnection) { + HBASE_INSTANCES.remove(connectionKey); + connection.internalClose(); + } + }else { + LOG.error("Connection not found in the list, can't delete it "+ + "(connection key="+connectionKey+"). May be the key was modified?"); + } + } + } + + /** + * It is provided for unit test cases which verify the behavior of region + * location cache prefetch. + * @return Number of cached regions for the table. + * @throws ZooKeeperConnectionException + */ + static int getCachedRegionCount(Configuration conf, + final byte[] tableName) + throws IOException { + return execute(new HConnectable(conf) { + @Override + public Integer connect(HConnection connection) { + return ((HConnectionImplementation) connection) + .getNumberOfCachedRegionLocations(tableName); + } + }); + } + + /** + * It's provided for unit test cases which verify the behavior of region + * location cache prefetch. + * @return true if the region where the table and row reside is cached. + * @throws ZooKeeperConnectionException + */ + static boolean isRegionCached(Configuration conf, + final byte[] tableName, final byte[] row) throws IOException { + return execute(new HConnectable(conf) { + @Override + public Boolean connect(HConnection connection) { + return ((HConnectionImplementation) connection).isRegionCached(tableName, row); + } + }); + } + + /** + * This class makes it convenient for one to execute a command in the context + * of a {@link HConnection} instance based on the given {@link Configuration}. + * + *

+ * If you find yourself wanting to use a {@link HConnection} for a relatively + * short duration of time, and do not want to deal with the hassle of creating + * and cleaning up that resource, then you should consider using this + * convenience class. + * + * @param + * the return type of the {@link HConnectable#connect(HConnection)} + * method. + */ + public static abstract class HConnectable { + public Configuration conf; + + public HConnectable(Configuration conf) { + this.conf = conf; + } + + public abstract T connect(HConnection connection) throws IOException; + } + + /** + * This convenience method invokes the given {@link HConnectable#connect} + * implementation using a {@link HConnection} instance that lasts just for the + * duration of that invocation. + * + * @param the return type of the connect method + * @param connectable the {@link HConnectable} instance + * @return the value returned by the connect method + * @throws IOException + */ + public static T execute(HConnectable connectable) throws IOException { + if (connectable == null || connectable.conf == null) { + return null; + } + Configuration conf = connectable.conf; + HConnection connection = HConnectionManager.getConnection(conf); + boolean connectSucceeded = false; + try { + T returnValue = connectable.connect(connection); + connectSucceeded = true; + return returnValue; + } finally { + try { + connection.close(); + } catch (Exception e) { + if (connectSucceeded) { + throw new IOException("The connection to " + connection + + " could not be deleted.", e); + } + } + } + } + + /** + * Denotes a unique key to a {@link HConnection} instance. + * + * In essence, this class captures the properties in {@link Configuration} + * that may be used in the process of establishing a connection. In light of + * that, if any new such properties are introduced into the mix, they must be + * added to the {@link HConnectionKey#properties} list. + * + */ + public static class HConnectionKey { + public static String[] CONNECTION_PROPERTIES = new String[] { + HConstants.ZOOKEEPER_QUORUM, HConstants.ZOOKEEPER_ZNODE_PARENT, + HConstants.ZOOKEEPER_CLIENT_PORT, + HConstants.ZOOKEEPER_RECOVERABLE_WAITTIME, + HConstants.HBASE_CLIENT_PAUSE, HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS, + HConstants.HBASE_RPC_TIMEOUT_KEY, + HConstants.HBASE_CLIENT_PREFETCH_LIMIT, + HConstants.HBASE_META_SCANNER_CACHING, + HConstants.HBASE_CLIENT_INSTANCE_ID }; + + private Map properties; + private String username; + + public HConnectionKey(Configuration conf) { + Map m = new HashMap(); + if (conf != null) { + for (String property : CONNECTION_PROPERTIES) { + String value = conf.get(property); + if (value != null) { + m.put(property, value); + } + } + } + this.properties = Collections.unmodifiableMap(m); + + try { + User currentUser = User.getCurrent(); + if (currentUser != null) { + username = currentUser.getName(); + } + } catch (IOException ioe) { + LOG.warn("Error obtaining current user, skipping username in HConnectionKey", + ioe); + } + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + if (username != null) { + result = username.hashCode(); + } + for (String property : CONNECTION_PROPERTIES) { + String value = properties.get(property); + if (value != null) { + result = prime * result + value.hashCode(); + } + } + + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + HConnectionKey that = (HConnectionKey) obj; + if (this.username != null && !this.username.equals(that.username)) { + return false; + } else if (this.username == null && that.username != null) { + return false; + } + if (this.properties == null) { + if (that.properties != null) { + return false; + } + } else { + if (that.properties == null) { + return false; + } + for (String property : CONNECTION_PROPERTIES) { + String thisValue = this.properties.get(property); + String thatValue = that.properties.get(property); + if (thisValue == thatValue) { + continue; + } + if (thisValue == null || !thisValue.equals(thatValue)) { + return false; + } + } + } + return true; + } + + @Override + public String toString() { + return "HConnectionKey{" + + "properties=" + properties + + ", username='" + username + '\'' + + '}'; + } + } + + /* Encapsulates connection to zookeeper and regionservers.*/ + static class HConnectionImplementation implements HConnection, Closeable { + static final Log LOG = LogFactory.getLog(HConnectionImplementation.class); + private final Class serverInterfaceClass; + private final long pause; + private final int numRetries; + private final int maxRPCAttempts; + private final int rpcTimeout; + private final int prefetchRegionLimit; + + private final Object masterLock = new Object(); + private volatile boolean closed; + private volatile boolean aborted; + private volatile boolean resetting; + private volatile HMasterInterface master; + // ZooKeeper reference + private volatile ZooKeeperWatcher zooKeeper; + // ZooKeeper-based master address tracker + private volatile MasterAddressTracker masterAddressTracker; + private volatile RootRegionTracker rootRegionTracker; + private volatile ClusterId clusterId; + + private final Object metaRegionLock = new Object(); + + private final Object userRegionLock = new Object(); + + private final Object resetLock = new Object(); + + private final Configuration conf; + + private RpcEngine rpcEngine; + + // Known region HServerAddress.toString() -> HRegionInterface + + private final Map servers = + new ConcurrentHashMap(); + private final ConcurrentHashMap connectionLock = + new ConcurrentHashMap(); + + /** + * Map of table to table {@link HRegionLocation}s. The table key is made + * by doing a {@link Bytes#mapKey(byte[])} of the table's name. + */ + private final Map> + cachedRegionLocations = + new HashMap>(); + + // The presence of a server in the map implies it's likely that there is an + // entry in cachedRegionLocations that map to this server; but the absence + // of a server in this map guarentees that there is no entry in cache that + // maps to the absent server. + private final Set cachedServers = + new HashSet(); + + // region cache prefetch is enabled by default. this set contains all + // tables whose region cache prefetch are disabled. + private final Set regionCachePrefetchDisabledTables = + new CopyOnWriteArraySet(); + + private int refCount; + + // indicates whether this connection's life cycle is managed + private final boolean managed; + /** + * constructor + * @param conf Configuration object + */ + @SuppressWarnings("unchecked") + public HConnectionImplementation(Configuration conf, boolean managed) + throws ZooKeeperConnectionException { + this.conf = conf; + this.managed = managed; + String serverClassName = conf.get(HConstants.REGION_SERVER_CLASS, + HConstants.DEFAULT_REGION_SERVER_CLASS); + this.closed = false; + try { + this.serverInterfaceClass = + (Class) Class.forName(serverClassName); + } catch (ClassNotFoundException e) { + throw new UnsupportedOperationException( + "Unable to find region server interface " + serverClassName, e); + } + this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE, + HConstants.DEFAULT_HBASE_CLIENT_PAUSE); + this.numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + this.maxRPCAttempts = conf.getInt( + HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS, + HConstants.DEFAULT_HBASE_CLIENT_RPC_MAXATTEMPTS); + this.rpcTimeout = conf.getInt( + HConstants.HBASE_RPC_TIMEOUT_KEY, + HConstants.DEFAULT_HBASE_RPC_TIMEOUT); + this.prefetchRegionLimit = conf.getInt( + HConstants.HBASE_CLIENT_PREFETCH_LIMIT, + HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT); + + this.master = null; + this.resetting = false; + } + + private synchronized void ensureZookeeperTrackers() + throws ZooKeeperConnectionException { + // initialize zookeeper and master address manager + if (zooKeeper == null) { + zooKeeper = getZooKeeperWatcher(); + } + if (clusterId == null) { + clusterId = new ClusterId(zooKeeper, this); + if (clusterId.hasId()) { + conf.set(HConstants.CLUSTER_ID, clusterId.getId()); + } + } + if (masterAddressTracker == null) { + masterAddressTracker = new MasterAddressTracker(zooKeeper, this); + masterAddressTracker.start(); + } + if (rootRegionTracker == null) { + rootRegionTracker = new RootRegionTracker(zooKeeper, this); + rootRegionTracker.start(); + } + // RpcEngine needs access to zookeeper data, like cluster ID + if (rpcEngine == null) { + this.rpcEngine = HBaseRPC.getProtocolEngine(conf); + } + } + + private synchronized void resetZooKeeperTrackers() { + if (masterAddressTracker != null) { + masterAddressTracker.stop(); + masterAddressTracker = null; + } + if (rootRegionTracker != null) { + rootRegionTracker.stop(); + rootRegionTracker = null; + } + clusterId = null; + if (zooKeeper != null) { + zooKeeper.close(); + zooKeeper = null; + } + } + + public Configuration getConfiguration() { + return this.conf; + } + + /** + * Log failure of getMaster attempt + * @return true if should retry + */ + private boolean shouldRetryGetMaster(int tries, Exception e) { + if (tries == numRetries - 1) { + // This was our last chance - don't bother sleeping + LOG.info("getMaster attempt " + tries + " of " + numRetries + + " failed; no more retrying.", e); + return false; + } + LOG.info("getMaster attempt " + tries + " of " + numRetries + + " failed; retrying after sleep of " + + ConnectionUtils.getPauseTime(this.pause, tries), e); + return true; + } + + public HMasterInterface getMaster() + throws MasterNotRunningException, ZooKeeperConnectionException { + // TODO: REMOVE. MOVE TO HBaseAdmin and redo as a Callable!!! + + // Check if we already have a good master connection + try { + if (master != null && master.isMasterRunning()) { + return master; + } + } catch (UndeclaredThrowableException ute) { + // log, but ignore, the loop below will attempt to reconnect + LOG.info("Exception contacting master. Retrying...", ute.getCause()); + } + + ensureZookeeperTrackers(); + checkIfBaseNodeAvailable(); + ServerName sn = null; + synchronized (this.masterLock) { + try { + if (master != null && master.isMasterRunning()) { + return master; + } + } catch (UndeclaredThrowableException ute) { + // log, but ignore, the loop below will attempt to reconnect + LOG.info("Exception contacting master. Retrying...", ute.getCause()); + } + this.master = null; + + for (int tries = 0; + !this.closed && this.master == null && tries < numRetries; + tries++) { + + try { + sn = masterAddressTracker.getMasterAddress(); + if (sn == null) { + LOG.info("ZooKeeper available but no active master location found"); + throw new MasterNotRunningException(); + } + + InetSocketAddress isa = + new InetSocketAddress(sn.getHostname(), sn.getPort()); + HMasterInterface tryMaster = rpcEngine.getProxy( + HMasterInterface.class, HMasterInterface.VERSION, isa, this.conf, + this.rpcTimeout); + + if (tryMaster.isMasterRunning()) { + this.master = tryMaster; + this.masterLock.notifyAll(); + break; + } + + } catch (IOException e) { + if (!shouldRetryGetMaster(tries, e)) break; + } catch (UndeclaredThrowableException ute) { + if (!shouldRetryGetMaster(tries, ute)) break; + } + + // Cannot connect to master or it is not running. Sleep & retry + try { + this.masterLock.wait(ConnectionUtils.getPauseTime(this.pause, tries)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Thread was interrupted while trying to connect to master."); + } + } + + if (this.master == null) { + if (sn == null) { + throw new MasterNotRunningException(); + } + throw new MasterNotRunningException(sn.toString()); + } + return this.master; + } + } + + private void checkIfBaseNodeAvailable() throws MasterNotRunningException { + if (false == masterAddressTracker.checkIfBaseNodeAvailable()) { + String errorMsg = "Check the value configured in 'zookeeper.znode.parent'. " + + "There could be a mismatch with the one configured in the master."; + LOG.error(errorMsg); + throw new MasterNotRunningException(errorMsg); + } + } + + public boolean isMasterRunning() + throws MasterNotRunningException, ZooKeeperConnectionException { + if (this.master == null) { + getMaster(); + } + boolean isRunning = master.isMasterRunning(); + if(isRunning) { + return true; + } + throw new MasterNotRunningException(); + } + + public HRegionLocation getRegionLocation(final byte [] name, + final byte [] row, boolean reload) + throws IOException { + return reload? relocateRegion(name, row): locateRegion(name, row); + } + + public boolean isTableEnabled(byte[] tableName) throws IOException { + return testTableOnlineState(tableName, true); + } + + public boolean isTableDisabled(byte[] tableName) throws IOException { + return testTableOnlineState(tableName, false); + } + + public boolean isTableAvailable(final byte[] tableName) throws IOException { + final AtomicBoolean available = new AtomicBoolean(true); + final AtomicInteger regionCount = new AtomicInteger(0); + MetaScannerVisitor visitor = new MetaScannerVisitorBase() { + @Override + public boolean processRow(Result row) throws IOException { + byte[] value = row.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + HRegionInfo info = Writables.getHRegionInfoOrNull(value); + if (info != null) { + if (Bytes.equals(tableName, info.getTableName())) { + value = row.getValue(HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER); + if (value == null) { + available.set(false); + return false; + } + regionCount.incrementAndGet(); + } + } + return true; + } + }; + MetaScanner.metaScan(conf, visitor); + return available.get() && (regionCount.get() > 0); + } + + /* + * @param True if table is online + */ + private boolean testTableOnlineState(byte [] tableName, boolean online) + throws IOException { + if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { + // The root region is always enabled + return online; + } + ZooKeeperWatcher zkw = getZooKeeperWatcher(); + String tableNameStr = Bytes.toString(tableName); + try { + if (online) { + return ZKTableReadOnly.isEnabledTable(zkw, tableNameStr); + } + return ZKTableReadOnly.isDisabledTable(zkw, tableNameStr); + } catch (KeeperException e) { + throw new IOException("Enable/Disable failed", e); + } + } + + @Override + public HRegionLocation locateRegion(final byte[] regionName) throws IOException { + return locateRegion(HRegionInfo.getTableName(regionName), + HRegionInfo.getStartKey(regionName), false, true); + } + + @Override + public List locateRegions(final byte[] tableName) + throws IOException { + return locateRegions(tableName, false, true); + } + + @Override + public List locateRegions(final byte[] tableName, final boolean useCache, + final boolean offlined) throws IOException { + NavigableMap regions = MetaScanner.allTableRegions(conf, tableName, + offlined); + final List locations = new ArrayList(); + for (HRegionInfo regionInfo : regions.keySet()) { + locations.add(locateRegion(tableName, regionInfo.getStartKey(), useCache, true)); + } + return locations; + } + + public HRegionLocation locateRegion(final byte [] tableName, + final byte [] row) + throws IOException{ + return locateRegion(tableName, row, true, true); + } + + public HRegionLocation relocateRegion(final byte [] tableName, + final byte [] row) + throws IOException{ + + // Since this is an explicit request not to use any caching, finding + // disabled tables should not be desirable. This will ensure that an exception is thrown when + // the first time a disabled table is interacted with. + if (isTableDisabled(tableName)) { + throw new DoNotRetryIOException(Bytes.toString(tableName) + " is disabled."); + } + + return locateRegion(tableName, row, false, true); + } + + private HRegionLocation locateRegion(final byte [] tableName, + final byte [] row, boolean useCache, boolean retry) + throws IOException { + if (this.closed) throw new IOException(toString() + " closed"); + if (tableName == null || tableName.length == 0) { + throw new IllegalArgumentException( + "table name cannot be null or zero length"); + } + ensureZookeeperTrackers(); + if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { + try { + ServerName servername = this.rootRegionTracker.waitRootRegionLocation(this.rpcTimeout); + LOG.debug("Looked up root region location, connection=" + this + + "; serverName=" + ((servername == null)? "": servername.toString())); + if (servername == null) return null; + return new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, + servername.getHostname(), servername.getPort()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } + } else if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) { + return locateRegionInMeta(HConstants.ROOT_TABLE_NAME, tableName, row, + useCache, metaRegionLock, retry); + } else { + // Region not in the cache - have to go to the meta RS + return locateRegionInMeta(HConstants.META_TABLE_NAME, tableName, row, + useCache, userRegionLock, retry); + } + } + + /* + * Search .META. for the HRegionLocation info that contains the table and + * row we're seeking. It will prefetch certain number of regions info and + * save them to the global region cache. + */ + private void prefetchRegionCache(final byte[] tableName, + final byte[] row) { + // Implement a new visitor for MetaScanner, and use it to walk through + // the .META. + MetaScannerVisitor visitor = new MetaScannerVisitorBase() { + public boolean processRow(Result result) throws IOException { + try { + byte[] value = result.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + HRegionInfo regionInfo = null; + + if (value != null) { + // convert the row result into the HRegionLocation we need! + regionInfo = Writables.getHRegionInfo(value); + + // possible we got a region of a different table... + if (!Bytes.equals(regionInfo.getTableName(), + tableName)) { + return false; // stop scanning + } + if (regionInfo.isOffline()) { + // don't cache offline regions + return true; + } + value = result.getValue(HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER); + if (value == null) { + return true; // don't cache it + } + final String hostAndPort = Bytes.toString(value); + String hostname = Addressing.parseHostname(hostAndPort); + int port = Addressing.parsePort(hostAndPort); + value = result.getValue(HConstants.CATALOG_FAMILY, + HConstants.STARTCODE_QUALIFIER); + // instantiate the location + HRegionLocation loc = + new HRegionLocation(regionInfo, hostname, port); + // cache this meta entry + cacheLocation(tableName, loc); + } + return true; + } catch (RuntimeException e) { + throw new IOException(e); + } + } + }; + try { + // pre-fetch certain number of regions info at region cache. + MetaScanner.metaScan(conf, visitor, tableName, row, + this.prefetchRegionLimit); + } catch (IOException e) { + LOG.warn("Encountered problems when prefetch META table: ", e); + } + } + + /* + * Search one of the meta tables (-ROOT- or .META.) for the HRegionLocation + * info that contains the table and row we're seeking. + */ + private HRegionLocation locateRegionInMeta(final byte [] parentTable, + final byte [] tableName, final byte [] row, boolean useCache, + Object regionLockObject, boolean retry) + throws IOException { + HRegionLocation location; + // If we are supposed to be using the cache, look in the cache to see if + // we already have the region. + if (useCache) { + location = getCachedLocation(tableName, row); + if (location != null) { + return location; + } + } + + int localNumRetries = retry ? numRetries : 1; + // build the key of the meta region we should be looking for. + // the extra 9's on the end are necessary to allow "exact" matches + // without knowing the precise region names. + byte [] metaKey = HRegionInfo.createRegionName(tableName, row, + HConstants.NINES, false); + for (int tries = 0; true; tries++) { + if (tries >= localNumRetries) { + throw new NoServerForRegionException("Unable to find region for " + + Bytes.toStringBinary(row) + " after " + numRetries + " tries."); + } + + HRegionLocation metaLocation = null; + try { + // locate the root or meta region + metaLocation = locateRegion(parentTable, metaKey, true, false); + // If null still, go around again. + if (metaLocation == null) continue; + HRegionInterface server = + getHRegionConnection(metaLocation.getHostname(), metaLocation.getPort()); + + Result regionInfoRow = null; + // This block guards against two threads trying to load the meta + // region at the same time. The first will load the meta region and + // the second will use the value that the first one found. + synchronized (regionLockObject) { + // If the parent table is META, we may want to pre-fetch some + // region info into the global region cache for this table. + if (Bytes.equals(parentTable, HConstants.META_TABLE_NAME) && + (getRegionCachePrefetch(tableName)) ) { + prefetchRegionCache(tableName, row); + } + + // Check the cache again for a hit in case some other thread made the + // same query while we were waiting on the lock. If not supposed to + // be using the cache, delete any existing cached location so it won't + // interfere. + if (useCache) { + location = getCachedLocation(tableName, row); + if (location != null) { + return location; + } + } else { + deleteCachedLocation(tableName, row); + } + + // Query the root or meta region for the location of the meta region + regionInfoRow = server.getClosestRowBefore( + metaLocation.getRegionInfo().getRegionName(), metaKey, + HConstants.CATALOG_FAMILY); + } + if (regionInfoRow == null) { + throw new TableNotFoundException(Bytes.toString(tableName)); + } + byte [] value = regionInfoRow.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + if (value == null || value.length == 0) { + throw new IOException("HRegionInfo was null or empty in " + + Bytes.toString(parentTable) + ", row=" + regionInfoRow); + } + // convert the row result into the HRegionLocation we need! + HRegionInfo regionInfo = (HRegionInfo) Writables.getWritable( + value, new HRegionInfo()); + // possible we got a region of a different table... + if (!Bytes.equals(regionInfo.getTableName(), tableName)) { + throw new TableNotFoundException( + "Table '" + Bytes.toString(tableName) + "' was not found, got: " + + Bytes.toString(regionInfo.getTableName()) + "."); + } + if (regionInfo.isSplit()) { + throw new RegionOfflineException("the only available region for" + + " the required row is a split parent," + + " the daughters should be online soon: " + + regionInfo.getRegionNameAsString()); + } + if (regionInfo.isOffline()) { + throw new RegionOfflineException("the region is offline, could" + + " be caused by a disable table call: " + + regionInfo.getRegionNameAsString()); + } + + value = regionInfoRow.getValue(HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER); + String hostAndPort = ""; + if (value != null) { + hostAndPort = Bytes.toString(value); + } + if (hostAndPort.equals("")) { + throw new NoServerForRegionException("No server address listed " + + "in " + Bytes.toString(parentTable) + " for region " + + regionInfo.getRegionNameAsString() + " containing row " + + Bytes.toStringBinary(row)); + } + + // Instantiate the location + String hostname = Addressing.parseHostname(hostAndPort); + int port = Addressing.parsePort(hostAndPort); + location = new HRegionLocation(regionInfo, hostname, port); + cacheLocation(tableName, location); + return location; + } catch (TableNotFoundException e) { + // if we got this error, probably means the table just plain doesn't + // exist. rethrow the error immediately. this should always be coming + // from the HTable constructor. + throw e; + } catch (IOException e) { + if (e instanceof RemoteException) { + e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e); + } + if (tries < numRetries - 1) { + if (LOG.isDebugEnabled()) { + LOG.debug("locateRegionInMeta parentTable=" + + Bytes.toString(parentTable) + ", metaLocation=" + + ((metaLocation == null)? "null": "{" + metaLocation + "}") + + ", attempt=" + tries + " of " + + this.numRetries + " failed; retrying after sleep of " + + ConnectionUtils.getPauseTime(this.pause, tries) + " because: " + e.getMessage()); + } + } else { + throw e; + } + // Only relocate the parent region if necessary + if(!(e instanceof RegionOfflineException || + e instanceof NoServerForRegionException)) { + relocateRegion(parentTable, metaKey); + } + } + try{ + Thread.sleep(ConnectionUtils.getPauseTime(this.pause, tries)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Giving up trying to location region in " + + "meta: thread is interrupted."); + } + } + } + + /* + * Search the cache for a location that fits our table and row key. + * Return null if no suitable region is located. TODO: synchronization note + * + *

TODO: This method during writing consumes 15% of CPU doing lookup + * into the Soft Reference SortedMap. Improve. + * + * @param tableName + * @param row + * @return Null or region location found in cache. + */ + HRegionLocation getCachedLocation(final byte [] tableName, + final byte [] row) { + SoftValueSortedMap tableLocations = + getTableLocations(tableName); + + // start to examine the cache. we can only do cache actions + // if there's something in the cache for this table. + if (tableLocations.isEmpty()) { + return null; + } + + HRegionLocation possibleRegion = tableLocations.get(row); + if (possibleRegion != null) { + return possibleRegion; + } + + possibleRegion = tableLocations.lowerValueByKey(row); + if (possibleRegion == null) { + return null; + } + + // make sure that the end key is greater than the row we're looking + // for, otherwise the row actually belongs in the next region, not + // this one. the exception case is when the endkey is + // HConstants.EMPTY_END_ROW, signifying that the region we're + // checking is actually the last region in the table. + byte[] endKey = possibleRegion.getRegionInfo().getEndKey(); + if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) || + KeyValue.getRowComparator(tableName).compareRows( + endKey, 0, endKey.length, row, 0, row.length) > 0) { + return possibleRegion; + } + + // Passed all the way through, so we got nothin - complete cache miss + return null; + } + + /** + * Delete a cached location + * @param tableName tableName + * @param row + */ + void deleteCachedLocation(final byte [] tableName, final byte [] row) { + synchronized (this.cachedRegionLocations) { + Map tableLocations = getTableLocations(tableName); + if (!tableLocations.isEmpty()) { + // start to examine the cache. we can only do cache actions + // if there's something in the cache for this table. + HRegionLocation rl = getCachedLocation(tableName, row); + if (rl != null) { + tableLocations.remove(rl.getRegionInfo().getStartKey()); + if (LOG.isDebugEnabled()) { + LOG.debug("Removed " + + rl.getRegionInfo().getRegionNameAsString() + + " for tableName=" + Bytes.toString(tableName) + + " from cache " + "because of " + Bytes.toStringBinary(row)); + } + } + } + } + } + + @Override + public void deleteCachedRegionLocation(final HRegionLocation location) { + if (location == null) { + return; + } + synchronized (this.cachedRegionLocations) { + byte[] tableName = location.getRegionInfo().getTableName(); + Map tableLocations = getTableLocations(tableName); + if (!tableLocations.isEmpty()) { + // Delete if there's something in the cache for this region. + HRegionLocation removedLocation = + tableLocations.remove(location.getRegionInfo().getStartKey()); + if (LOG.isDebugEnabled() && removedLocation != null) { + LOG.debug("Removed " + + location.getRegionInfo().getRegionNameAsString() + + " for tableName=" + Bytes.toString(tableName) + + " from cache"); + } + } + } + } + + @Override + public void clearCaches(String sn) { + clearCachedLocationForServer(sn); + } + + /* + * Delete all cached entries of a table that maps to a specific location. + * + * @param tablename + * @param server + */ + private void clearCachedLocationForServer(final String server) { + boolean deletedSomething = false; + synchronized (this.cachedRegionLocations) { + if (!cachedServers.contains(server)) { + return; + } + for (Map tableLocations : + cachedRegionLocations.values()) { + for (Entry e : tableLocations.entrySet()) { + if (e.getValue().getHostnamePort().equals(server)) { + tableLocations.remove(e.getKey()); + deletedSomething = true; + } + } + } + cachedServers.remove(server); + } + if (deletedSomething && LOG.isDebugEnabled()) { + LOG.debug("Removed all cached region locations that map to " + server); + } + } + + /* + * @param tableName + * @return Map of cached locations for passed tableName + */ + private SoftValueSortedMap getTableLocations( + final byte [] tableName) { + // find the map of cached locations for this table + Integer key = Bytes.mapKey(tableName); + SoftValueSortedMap result; + synchronized (this.cachedRegionLocations) { + result = this.cachedRegionLocations.get(key); + // if tableLocations for this table isn't built yet, make one + if (result == null) { + result = new SoftValueSortedMap( + Bytes.BYTES_COMPARATOR); + this.cachedRegionLocations.put(key, result); + } + } + return result; + } + + @Override + public void clearRegionCache() { + synchronized(this.cachedRegionLocations) { + this.cachedRegionLocations.clear(); + this.cachedServers.clear(); + } + } + + @Override + public void clearRegionCache(final byte [] tableName) { + synchronized (this.cachedRegionLocations) { + this.cachedRegionLocations.remove(Bytes.mapKey(tableName)); + } + } + + /* + * Put a newly discovered HRegionLocation into the cache. + */ + private void cacheLocation(final byte [] tableName, + final HRegionLocation location) { + byte [] startKey = location.getRegionInfo().getStartKey(); + Map tableLocations = + getTableLocations(tableName); + boolean hasNewCache = false; + synchronized (this.cachedRegionLocations) { + cachedServers.add(location.getHostnamePort()); + hasNewCache = (tableLocations.put(startKey, location) == null); + } + if (hasNewCache) { + LOG.debug("Cached location for " + + location.getRegionInfo().getRegionNameAsString() + + " is " + location.getHostnamePort()); + } + } + + public HRegionInterface getHRegionConnection(HServerAddress hsa) + throws IOException { + return getHRegionConnection(hsa, false); + } + + @Override + public HRegionInterface getHRegionConnection(final String hostname, + final int port) + throws IOException { + return getHRegionConnection(hostname, port, false); + } + + public HRegionInterface getHRegionConnection(HServerAddress hsa, + boolean master) + throws IOException { + return getHRegionConnection(null, -1, hsa.getInetSocketAddress(), master); + } + + @Override + public HRegionInterface getHRegionConnection(final String hostname, + final int port, final boolean master) + throws IOException { + return getHRegionConnection(hostname, port, null, master); + } + + /** + * Either the passed isa is null or hostname + * can be but not both. + * @param hostname + * @param port + * @param isa + * @param master + * @return Proxy. + * @throws IOException + */ + HRegionInterface getHRegionConnection(final String hostname, final int port, + final InetSocketAddress isa, final boolean master) + throws IOException { + if (master) getMaster(); + HRegionInterface server; + String rsName = null; + if (isa != null) { + rsName = Addressing.createHostAndPortStr(isa.getHostName(), + isa.getPort()); + } else { + rsName = Addressing.createHostAndPortStr(hostname, port); + } + ensureZookeeperTrackers(); + // See if we already have a connection (common case) + server = this.servers.get(rsName); + if (server == null) { + // create a unique lock for this RS (if necessary) + this.connectionLock.putIfAbsent(rsName, rsName); + // get the RS lock + synchronized (this.connectionLock.get(rsName)) { + // do one more lookup in case we were stalled above + server = this.servers.get(rsName); + if (server == null) { + try { + // Only create isa when we need to. + InetSocketAddress address = isa != null? isa: + new InetSocketAddress(hostname, port); + // definitely a cache miss. establish an RPC for this RS + server = HBaseRPC.waitForProxy(this.rpcEngine, + serverInterfaceClass, HRegionInterface.VERSION, + address, this.conf, + this.maxRPCAttempts, this.rpcTimeout, this.rpcTimeout); + this.servers.put(Addressing.createHostAndPortStr( + address.getHostName(), address.getPort()), server); + } catch (RemoteException e) { + LOG.warn("RemoteException connecting to RS", e); + // Throw what the RemoteException was carrying. + throw e.unwrapRemoteException(); + } + } + } + } + return server; + } + + /** + * Get the ZooKeeper instance for this TableServers instance. + * + * If ZK has not been initialized yet, this will connect to ZK. + * @returns zookeeper reference + * @throws ZooKeeperConnectionException if there's a problem connecting to zk + */ + public synchronized ZooKeeperWatcher getZooKeeperWatcher() + throws ZooKeeperConnectionException { + if(zooKeeper == null) { + try { + if (this.closed) { + throw new IOException(toString() + " closed"); + } + this.zooKeeper = new ZooKeeperWatcher(conf, "hconnection", this); + } catch(ZooKeeperConnectionException zce) { + throw zce; + } catch (IOException e) { + throw new ZooKeeperConnectionException("An error is preventing" + + " HBase from connecting to ZooKeeper", e); + } + } + return zooKeeper; + } + + public T getRegionServerWithRetries(ServerCallable callable) + throws IOException, RuntimeException { + return callable.withRetries(); + } + + public T getRegionServerWithoutRetries(ServerCallable callable) + throws IOException, RuntimeException { + return callable.withoutRetries(); + } + + private Callable createCallable(final HRegionLocation loc, + final MultiAction multi, final byte [] tableName) { + // TODO: This does not belong in here!!! St.Ack HConnections should + // not be dealing in Callables; Callables have HConnections, not other + // way around. + final HConnection connection = this; + return new Callable() { + public MultiResponse call() throws IOException { + ServerCallable callable = + new ServerCallable(connection, tableName, null) { + public MultiResponse call() throws IOException { + return server.multi(multi); + } + @Override + public void connect(boolean reload) throws IOException { + server = connection.getHRegionConnection(loc.getHostname(), loc.getPort()); + } + }; + return callable.withoutRetries(); + } + }; + } + + public void processBatch(List list, + final byte[] tableName, + ExecutorService pool, + Object[] results) throws IOException, InterruptedException { + // This belongs in HTable!!! Not in here. St.Ack + + // results must be the same size as list + if (results.length != list.size()) { + throw new IllegalArgumentException("argument results must be the same size as argument list"); + } + + processBatchCallback(list, tableName, pool, results, null); + } + + /** + * Executes the given + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call} + * callable for each row in the + * given list and invokes + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Callback#update(byte[], byte[], Object)} + * for each result returned. + * + * @param protocol the protocol interface being called + * @param rows a list of row keys for which the callable should be invoked + * @param tableName table name for the coprocessor invoked + * @param pool ExecutorService used to submit the calls per row + * @param callable instance on which to invoke + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call#call(Object)} + * for each row + * @param callback instance on which to invoke + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Callback#update(byte[], byte[], Object)} + * for each result + * @param the protocol interface type + * @param the callable's return type + * @throws IOException + */ + public void processExecs( + final Class protocol, + List rows, + final byte[] tableName, + ExecutorService pool, + final Batch.Call callable, + final Batch.Callback callback) + throws IOException, Throwable { + + Map> futures = + new TreeMap>(Bytes.BYTES_COMPARATOR); + for (final byte[] r : rows) { + final ExecRPCInvoker invoker = + new ExecRPCInvoker(conf, this, protocol, tableName, r); + Future future = pool.submit( + new Callable() { + public R call() throws Exception { + T instance = (T)Proxy.newProxyInstance(conf.getClassLoader(), + new Class[]{protocol}, + invoker); + R result = callable.call(instance); + byte[] region = invoker.getRegionName(); + if (callback != null) { + callback.update(region, r, result); + } + return result; + } + }); + futures.put(r, future); + } + for (Map.Entry> e : futures.entrySet()) { + try { + e.getValue().get(); + } catch (ExecutionException ee) { + LOG.warn("Error executing for row "+Bytes.toStringBinary(e.getKey()), ee); + throw ee.getCause(); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted executing for row " + + Bytes.toStringBinary(e.getKey()), ie); + } + } + } + + /** + * Parameterized batch processing, allowing varying return types for + * different {@link Row} implementations. + */ + public void processBatchCallback( + List list, + byte[] tableName, + ExecutorService pool, + Object[] results, + Batch.Callback callback) + throws IOException, InterruptedException { + // This belongs in HTable!!! Not in here. St.Ack + + // results must be the same size as list + if (results.length != list.size()) { + throw new IllegalArgumentException( + "argument results must be the same size as argument list"); + } + if (list.isEmpty()) { + return; + } + + // Keep track of the most recent servers for any given item for better + // exceptional reporting. We keep HRegionLocation to save on parsing. + // Later below when we use lastServers, we'll pull what we need from + // lastServers. + HRegionLocation [] lastServers = new HRegionLocation[results.length]; + List workingList = new ArrayList(list); + boolean retry = true; + // count that helps presize actions array + int actionCount = 0; + + for (int tries = 0; tries < numRetries && retry; ++tries) { + + // sleep first, if this is a retry + if (tries >= 1) { + long sleepTime = ConnectionUtils.getPauseTime(this.pause, tries); + LOG.debug("Retry " +tries+ ", sleep for " +sleepTime+ "ms!"); + Thread.sleep(sleepTime); + } + // step 1: break up into regionserver-sized chunks and build the data structs + Map> actionsByServer = + new HashMap>(); + for (int i = 0; i < workingList.size(); i++) { + Row row = workingList.get(i); + if (row != null) { + HRegionLocation loc = locateRegion(tableName, row.getRow()); + byte[] regionName = loc.getRegionInfo().getRegionName(); + + MultiAction actions = actionsByServer.get(loc); + if (actions == null) { + actions = new MultiAction(); + actionsByServer.put(loc, actions); + } + + Action action = new Action(row, i); + lastServers[i] = loc; + actions.add(regionName, action); + } + } + + // step 2: make the requests + + Map> futures = + new HashMap>( + actionsByServer.size()); + + for (Entry> e: actionsByServer.entrySet()) { + futures.put(e.getKey(), pool.submit(createCallable(e.getKey(), e.getValue(), tableName))); + } + + // step 3: collect the failures and successes and prepare for retry + + for (Entry> responsePerServer + : futures.entrySet()) { + HRegionLocation loc = responsePerServer.getKey(); + + try { + Future future = responsePerServer.getValue(); + MultiResponse resp = future.get(); + + if (resp == null) { + // Entire server failed + LOG.debug("Failed all for server: " + loc.getHostnamePort() + + ", removing from cache"); + continue; + } + + for (Entry>> e : resp.getResults().entrySet()) { + byte[] regionName = e.getKey(); + List> regionResults = e.getValue(); + for (Pair regionResult : regionResults) { + if (regionResult == null) { + // if the first/only record is 'null' the entire region failed. + LOG.debug("Failures for region: " + + Bytes.toStringBinary(regionName) + + ", removing from cache"); + } else { + // Result might be an Exception, including DNRIOE + results[regionResult.getFirst()] = regionResult.getSecond(); + if (callback != null && !(regionResult.getSecond() instanceof Throwable)) { + callback.update(e.getKey(), + list.get(regionResult.getFirst()).getRow(), + (R)regionResult.getSecond()); + } + } + } + } + } catch (ExecutionException e) { + LOG.warn("Failed all from " + loc, e); + } + } + + // step 4: identify failures and prep for a retry (if applicable). + + // Find failures (i.e. null Result), and add them to the workingList (in + // order), so they can be retried. + retry = false; + workingList.clear(); + actionCount = 0; + for (int i = 0; i < results.length; i++) { + // if null (fail) or instanceof Throwable && not instanceof DNRIOE + // then retry that row. else dont. + if (results[i] == null || + (results[i] instanceof Throwable && + !(results[i] instanceof DoNotRetryIOException))) { + + retry = true; + actionCount++; + Row row = list.get(i); + workingList.add(row); + deleteCachedLocation(tableName, row.getRow()); + } else { + if (results[i] != null && results[i] instanceof Throwable) { + actionCount++; + } + // add null to workingList, so the order remains consistent with the original list argument. + workingList.add(null); + } + } + } + + List exceptions = new ArrayList(actionCount); + List actions = new ArrayList(actionCount); + List addresses = new ArrayList(actionCount); + + for (int i = 0 ; i < results.length; i++) { + if (results[i] == null || results[i] instanceof Throwable) { + exceptions.add((Throwable)results[i]); + actions.add(list.get(i)); + addresses.add(lastServers[i].getHostnamePort()); + } + } + + if (!exceptions.isEmpty()) { + throw new RetriesExhaustedWithDetailsException(exceptions, + actions, + addresses); + } + } + + /* + * Return the number of cached region for a table. It will only be called + * from a unit test. + */ + int getNumberOfCachedRegionLocations(final byte[] tableName) { + Integer key = Bytes.mapKey(tableName); + synchronized (this.cachedRegionLocations) { + Map tableLocs = + this.cachedRegionLocations.get(key); + + if (tableLocs == null) { + return 0; + } + return tableLocs.values().size(); + } + } + + /** + * Check the region cache to see whether a region is cached yet or not. + * Called by unit tests. + * @param tableName tableName + * @param row row + * @return Region cached or not. + */ + boolean isRegionCached(final byte[] tableName, final byte[] row) { + HRegionLocation location = getCachedLocation(tableName, row); + return location != null; + } + + public void setRegionCachePrefetch(final byte[] tableName, + final boolean enable) { + if (!enable) { + regionCachePrefetchDisabledTables.add(Bytes.mapKey(tableName)); + } + else { + regionCachePrefetchDisabledTables.remove(Bytes.mapKey(tableName)); + } + } + + public boolean getRegionCachePrefetch(final byte[] tableName) { + return !regionCachePrefetchDisabledTables.contains(Bytes.mapKey(tableName)); + } + + @Override + public void prewarmRegionCache(byte[] tableName, + Map regions) { + for (Map.Entry e : regions.entrySet()) { + HServerAddress hsa = e.getValue(); + if (hsa == null || hsa.getInetSocketAddress() == null) continue; + cacheLocation(tableName, + new HRegionLocation(e.getKey(), hsa.getHostname(), hsa.getPort())); + } + } + + @Override + public void abort(final String msg, Throwable t) { + if (t instanceof KeeperException) { + LOG.info("This client just lost it's session with ZooKeeper, will" + + " automatically reconnect when needed."); + if (t instanceof KeeperException.SessionExpiredException) { + LOG.info("ZK session expired. This disconnect could have been" + + " caused by a network partition or a long-running GC pause," + + " either way it's recommended that you verify your environment."); + synchronized (resetLock) { + if (resetting) return; + this.resetting = true; + } + resetZooKeeperTrackers(); + this.resetting = false; + } + return; + } + if (t != null) LOG.fatal(msg, t); + else LOG.fatal(msg); + this.aborted = true; + close(); + } + + @Override + public boolean isClosed() { + return this.closed; + } + + @Override + public boolean isAborted(){ + return this.aborted; + } + + public int getCurrentNrHRS() throws IOException { + try { + ZooKeeperWatcher zkw = getZooKeeperWatcher(); + // We go to zk rather than to master to get count of regions to avoid + // HTable having a Master dependency. See HBase-2828 + return ZKUtil.getNumberOfChildren(zkw, + zkw.rsZNode); + } catch (KeeperException ke) { + throw new IOException("Unexpected ZooKeeper exception", ke); + } + } + + /** + * Increment this client's reference count. + */ + void incCount() { + ++refCount; + } + + /** + * Decrement this client's reference count. + */ + void decCount() { + if (refCount > 0) { + --refCount; + } + } + + /** + * Return if this client has no reference + * + * @return true if this client has no reference; false otherwise + */ + boolean isZeroReference() { + return refCount == 0; + } + + void internalClose() { + if (this.closed) { + return; + } + master = null; + + this.servers.clear(); + if (this.rpcEngine != null) { + this.rpcEngine.close(); + } + + synchronized (this) { + if (this.zooKeeper != null) { + LOG.info("Closed zookeeper sessionid=0x" + + Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId())); + this.zooKeeper.close(); + this.zooKeeper = null; + } + this.closed = true; + } + } + + public void close() { + if (managed) { + if (aborted) { + HConnectionManager.deleteStaleConnection(this); + } else { + HConnectionManager.deleteConnection(this, false); + } + } else { + internalClose(); + } + if (LOG.isTraceEnabled()) LOG.debug("" + this.zooKeeper + " closed."); + } + + /** + * Close the connection for good, regardless of what the current value of + * {@link #refCount} is. Ideally, {@link #refCount} should be zero at this + * point, which would be the case if all of its consumers close the + * connection. However, on the off chance that someone is unable to close + * the connection, perhaps because it bailed out prematurely, the method + * below will ensure that this {@link HConnection} instance is cleaned up. + * Caveat: The JVM may take an unknown amount of time to call finalize on an + * unreachable object, so our hope is that every consumer cleans up after + * itself, like any good citizen. + */ + @Override + protected void finalize() throws Throwable { + // Pretend as if we are about to release the last remaining reference + refCount = 1; + close(); + LOG.debug("The connection to " + this.zooKeeper + + " was closed by the finalize method."); + } + + public HTableDescriptor[] listTables() throws IOException { + if (this.master == null) { + this.master = getMaster(); + } + HTableDescriptor[] htd = master.getHTableDescriptors(); + return htd; + } + + public HTableDescriptor[] getHTableDescriptors(List tableNames) throws IOException { + if (tableNames == null || tableNames.isEmpty()) return new HTableDescriptor[0]; + if (tableNames == null || tableNames.size() == 0) return null; + if (this.master == null) { + this.master = getMaster(); + } + return master.getHTableDescriptors(tableNames); + } + + public HTableDescriptor getHTableDescriptor(final byte[] tableName) + throws IOException { + if (tableName == null || tableName.length == 0) return null; + if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { + return new UnmodifyableHTableDescriptor(HTableDescriptor.ROOT_TABLEDESC); + } + if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) { + return HTableDescriptor.META_TABLEDESC; + } + if (this.master == null) { + this.master = getMaster(); + } + + HTableDescriptor[] htds = master.getHTableDescriptors(); + if (htds != null && htds.length > 0) { + for (HTableDescriptor htd: htds) { + if (Bytes.equals(tableName, htd.getName())) { + return htd; + } + } + } + throw new TableNotFoundException(Bytes.toString(tableName)); + } + } + + /** + * Set the number of retries to use serverside when trying to communicate + * with another server over {@link HConnection}. Used updating catalog + * tables, etc. Call this method before we create any Connections. + * @param c The Configuration instance to set the retries into. + * @param log Used to log what we set in here. + */ + public static void setServerSideHConnectionRetries(final Configuration c, + final Log log) { + int hcRetries = c.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + // Go big. Multiply by 10. If we can't get to meta after this many retries + // then something seriously wrong. + int serversideMultiplier = + c.getInt("hbase.client.serverside.retries.multiplier", 10); + int retries = hcRetries * serversideMultiplier; + c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, retries); + log.debug("Set serverside HConnection retries=" + retries); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java new file mode 100644 index 0000000..6431ecc --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java @@ -0,0 +1,1290 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.Closeable; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Collections; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.ipc.ExecRPCInvoker; +import org.apache.hadoop.hbase.util.Addressing; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.Threads; + +/** + *

Used to communicate with a single HBase table. + * + *

This class is not thread safe for reads nor write. + * + *

In case of writes (Put, Delete), the underlying write buffer can + * be corrupted if multiple threads contend over a single HTable instance. + * + *

In case of reads, some fields used by a Scan are shared among all threads. + * The HTable implementation can either not contract to be safe in case of a Get + * + *

To access a table in a multi threaded environment, please consider + * using the {@link HTablePool} class to create your HTable instances. + * + *

Instances of HTable passed the same {@link Configuration} instance will + * share connections to servers out on the cluster and to the zookeeper ensemble + * as well as caches of region locations. This is usually a *good* thing and it + * is recommended to reuse the same configuration object for all your tables. + * This happens because they will all share the same underlying + * {@link HConnection} instance. See {@link HConnectionManager} for more on + * how this mechanism works. + * + *

{@link HConnection} will read most of the + * configuration it needs from the passed {@link Configuration} on initial + * construction. Thereafter, for settings such as + * hbase.client.pause, hbase.client.retries.number, + * and hbase.client.rpc.maxattempts updating their values in the + * passed {@link Configuration} subsequent to {@link HConnection} construction + * will go unnoticed. To run with changed values, make a new + * {@link HTable} passing a new {@link Configuration} instance that has the + * new configuration. + * + *

Note that this class implements the {@link Closeable} interface. When a + * HTable instance is no longer required, it *should* be closed in order to ensure + * that the underlying resources are promptly released. Please note that the close + * method can throw java.io.IOException that must be handled. + * + * @see HBaseAdmin for create, drop, list, enable and disable of tables. + * @see HConnection + * @see HConnectionManager + */ +public class HTable implements HTableInterface { + private static final Log LOG = LogFactory.getLog(HTable.class); + private HConnection connection; + private final byte [] tableName; + private volatile Configuration configuration; + private final ArrayList writeBuffer = new ArrayList(); + private long writeBufferSize; + private boolean clearBufferOnFail; + private boolean autoFlush; + private long currentWriteBufferSize; + protected int scannerCaching; + private int maxKeyValueSize; + private ExecutorService pool; // For Multi + private boolean closed; + private int operationTimeout; + private final boolean cleanupPoolOnClose; // shutdown the pool in close() + private final boolean cleanupConnectionOnClose; // close the connection in close() + + /** + * Creates an object to access a HBase table. + * Shares zookeeper connection and other resources with other HTable instances + * created with the same conf instance. Uses already-populated + * region cache if one is available, populated by any other HTable instances + * sharing this conf instance. Recommended. + * @param conf Configuration object to use. + * @param tableName Name of the table. + * @throws IOException if a remote or network exception occurs + */ + public HTable(Configuration conf, final String tableName) + throws IOException { + this(conf, Bytes.toBytes(tableName)); + } + + + /** + * Creates an object to access a HBase table. + * Shares zookeeper connection and other resources with other HTable instances + * created with the same conf instance. Uses already-populated + * region cache if one is available, populated by any other HTable instances + * sharing this conf instance. Recommended. + * @param conf Configuration object to use. + * @param tableName Name of the table. + * @throws IOException if a remote or network exception occurs + */ + public HTable(Configuration conf, final byte [] tableName) + throws IOException { + this.tableName = tableName; + this.cleanupPoolOnClose = this.cleanupConnectionOnClose = true; + if (conf == null) { + this.connection = null; + return; + } + this.connection = HConnectionManager.getConnection(conf); + this.configuration = conf; + + int maxThreads = conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE); + if (maxThreads == 0) { + maxThreads = 1; // is there a better default? + } + long keepAliveTime = conf.getLong("hbase.htable.threads.keepalivetime", 60); + + // Using the "direct handoff" approach, new threads will only be created + // if it is necessary and will grow unbounded. This could be bad but in HCM + // we only create as many Runnables as there are region servers. It means + // it also scales when new region servers are added. + this.pool = new ThreadPoolExecutor(1, maxThreads, + keepAliveTime, TimeUnit.SECONDS, + new SynchronousQueue(), + Threads.newDaemonThreadFactory("hbase-table")); + ((ThreadPoolExecutor)this.pool).allowCoreThreadTimeOut(true); + + this.finishSetup(); + } + + /** + * Creates an object to access a HBase table. + * Shares zookeeper connection and other resources with other HTable instances + * created with the same conf instance. Uses already-populated + * region cache if one is available, populated by any other HTable instances + * sharing this conf instance. + * Use this constructor when the ExecutorService is externally managed. + * @param conf Configuration object to use. + * @param tableName Name of the table. + * @param pool ExecutorService to be used. + * @throws IOException if a remote or network exception occurs + */ + public HTable(Configuration conf, final byte[] tableName, final ExecutorService pool) + throws IOException { + this.connection = HConnectionManager.getConnection(conf); + this.configuration = conf; + this.pool = pool; + this.tableName = tableName; + this.cleanupPoolOnClose = false; + this.cleanupConnectionOnClose = true; + + this.finishSetup(); + } + + /** + * Creates an object to access a HBase table. + * Shares zookeeper connection and other resources with other HTable instances + * created with the same connection instance. + * Use this constructor when the ExecutorService and HConnection instance are + * externally managed. + * @param tableName Name of the table. + * @param connection HConnection to be used. + * @param pool ExecutorService to be used. + * @throws IOException if a remote or network exception occurs + */ + public HTable(final byte[] tableName, final HConnection connection, + final ExecutorService pool) throws IOException { + if (pool == null || pool.isShutdown()) { + throw new IllegalArgumentException("Pool is null or shut down."); + } + if (connection == null || connection.isClosed()) { + throw new IllegalArgumentException("Connection is null or closed."); + } + this.tableName = tableName; + this.cleanupPoolOnClose = this.cleanupConnectionOnClose = false; + this.connection = connection; + this.configuration = connection.getConfiguration(); + this.pool = pool; + + this.finishSetup(); + } + + /** + * setup this HTable's parameter based on the passed configuration + * @param conf + */ + private void finishSetup() throws IOException { + this.connection.locateRegion(tableName, HConstants.EMPTY_START_ROW); + this.operationTimeout = HTableDescriptor.isMetaTable(tableName) ? HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT + : this.configuration.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, + HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); + this.writeBufferSize = this.configuration.getLong( + "hbase.client.write.buffer", 2097152); + this.clearBufferOnFail = true; + this.autoFlush = true; + this.currentWriteBufferSize = 0; + this.scannerCaching = this.configuration.getInt( + "hbase.client.scanner.caching", 1); + + this.maxKeyValueSize = this.configuration.getInt( + "hbase.client.keyvalue.maxsize", -1); + this.closed = false; + } + + /** + * {@inheritDoc} + */ + @Override + public Configuration getConfiguration() { + return configuration; + } + + /** + * Tells whether or not a table is enabled or not. This method creates a + * new HBase configuration, so it might make your unit tests fail due to + * incorrect ZK client port. + * @param tableName Name of table to check. + * @return {@code true} if table is online. + * @throws IOException if a remote or network exception occurs + * @deprecated use {@link HBaseAdmin#isTableEnabled(byte[])} + */ + @Deprecated + public static boolean isTableEnabled(String tableName) throws IOException { + return isTableEnabled(Bytes.toBytes(tableName)); + } + + /** + * Tells whether or not a table is enabled or not. This method creates a + * new HBase configuration, so it might make your unit tests fail due to + * incorrect ZK client port. + * @param tableName Name of table to check. + * @return {@code true} if table is online. + * @throws IOException if a remote or network exception occurs + * @deprecated use {@link HBaseAdmin#isTableEnabled(byte[])} + */ + @Deprecated + public static boolean isTableEnabled(byte[] tableName) throws IOException { + return isTableEnabled(HBaseConfiguration.create(), tableName); + } + + /** + * Tells whether or not a table is enabled or not. + * @param conf The Configuration object to use. + * @param tableName Name of table to check. + * @return {@code true} if table is online. + * @throws IOException if a remote or network exception occurs + * @deprecated use {@link HBaseAdmin#isTableEnabled(byte[])} + */ + @Deprecated + public static boolean isTableEnabled(Configuration conf, String tableName) + throws IOException { + return isTableEnabled(conf, Bytes.toBytes(tableName)); + } + + /** + * Tells whether or not a table is enabled or not. + * @param conf The Configuration object to use. + * @param tableName Name of table to check. + * @return {@code true} if table is online. + * @throws IOException if a remote or network exception occurs + */ + public static boolean isTableEnabled(Configuration conf, + final byte[] tableName) throws IOException { + return HConnectionManager.execute(new HConnectable(conf) { + @Override + public Boolean connect(HConnection connection) throws IOException { + return connection.isTableEnabled(tableName); + } + }); + } + + /** + * Find region location hosting passed row using cached info + * @param row Row to find. + * @return The location of the given row. + * @throws IOException if a remote or network exception occurs + */ + public HRegionLocation getRegionLocation(final String row) + throws IOException { + return connection.getRegionLocation(tableName, Bytes.toBytes(row), false); + } + + /** + * Finds the region on which the given row is being served. + * @param row Row to find. + * @return Location of the row. + * @throws IOException if a remote or network exception occurs + * @deprecated use {@link #getRegionLocation(byte [], boolean)} instead + */ + public HRegionLocation getRegionLocation(final byte [] row) + throws IOException { + return connection.getRegionLocation(tableName, row, false); + } + + /** + * Finds the region on which the given row is being served. + * @param row Row to find. + * @param reload whether or not to reload information or just use cached + * information + * @return Location of the row. + * @throws IOException if a remote or network exception occurs + */ + public HRegionLocation getRegionLocation(final byte [] row, boolean reload) + throws IOException { + return connection.getRegionLocation(tableName, row, reload); + } + + /** + * {@inheritDoc} + */ + @Override + public byte [] getTableName() { + return this.tableName; + } + + /** + * INTERNAL Used by unit tests and tools to do low-level + * manipulations. + * @return An HConnection instance. + * @deprecated This method will be changed from public to package protected. + */ + // TODO(tsuna): Remove this. Unit tests shouldn't require public helpers. + public HConnection getConnection() { + return this.connection; + } + + /** + * Gets the number of rows that a scanner will fetch at once. + *

+ * The default value comes from {@code hbase.client.scanner.caching}. + * @deprecated Use {@link Scan#setCaching(int)} and {@link Scan#getCaching()} + */ + public int getScannerCaching() { + return scannerCaching; + } + + /** + * Sets the number of rows that a scanner will fetch at once. + *

+ * This will override the value specified by + * {@code hbase.client.scanner.caching}. + * Increasing this value will reduce the amount of work needed each time + * {@code next()} is called on a scanner, at the expense of memory use + * (since more rows will need to be maintained in memory by the scanners). + * @param scannerCaching the number of rows a scanner will fetch at once. + * @deprecated Use {@link Scan#setCaching(int)} + */ + public void setScannerCaching(int scannerCaching) { + this.scannerCaching = scannerCaching; + } + + /** + * {@inheritDoc} + */ + @Override + public HTableDescriptor getTableDescriptor() throws IOException { + return new UnmodifyableHTableDescriptor( + this.connection.getHTableDescriptor(this.tableName)); + } + + /** + * Gets the starting row key for every region in the currently open table. + *

+ * This is mainly useful for the MapReduce integration. + * @return Array of region starting row keys + * @throws IOException if a remote or network exception occurs + */ + public byte [][] getStartKeys() throws IOException { + return getStartEndKeys().getFirst(); + } + + /** + * Gets the ending row key for every region in the currently open table. + *

+ * This is mainly useful for the MapReduce integration. + * @return Array of region ending row keys + * @throws IOException if a remote or network exception occurs + */ + public byte[][] getEndKeys() throws IOException { + return getStartEndKeys().getSecond(); + } + + /** + * Get the split keys of the currently open table. + * @return Array of split keys + * @throws IOException + */ + public byte[][] getSplitKeys() throws IOException { + List splitKeysList = new ArrayList(); + for (HRegionInfo regionInfo : getRegionLocations().keySet()) { + if (Bytes.equals(regionInfo.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) { + continue; + } + splitKeysList.add(regionInfo.getStartKey()); + } + return splitKeysList.toArray(new byte[splitKeysList.size()][]); + } + + /** + * Gets the starting and ending row keys for every region in the currently + * open table. + *

+ * This is mainly useful for the MapReduce integration. + * @return Pair of arrays of region starting and ending row keys + * @throws IOException if a remote or network exception occurs + */ + public Pair getStartEndKeys() throws IOException { + NavigableMap regions = getRegionLocations(); + final List startKeyList = new ArrayList(regions.size()); + final List endKeyList = new ArrayList(regions.size()); + + for (HRegionInfo region : regions.keySet()) { + startKeyList.add(region.getStartKey()); + endKeyList.add(region.getEndKey()); + } + + return new Pair( + startKeyList.toArray(new byte[startKeyList.size()][]), + endKeyList.toArray(new byte[endKeyList.size()][])); + } + + /** + * Gets all the regions and their address for this table. + * @return A map of HRegionInfo with it's server address + * @throws IOException if a remote or network exception occurs + * @deprecated Use {@link #getRegionLocations()} or {@link #getStartEndKeys()} + */ + public Map getRegionsInfo() throws IOException { + final Map regionMap = + new TreeMap(); + + final Map regionLocations = getRegionLocations(); + + for (Map.Entry entry : regionLocations.entrySet()) { + HServerAddress server = new HServerAddress(); + ServerName serverName = entry.getValue(); + if (serverName != null && serverName.getHostAndPort() != null) { + server = new HServerAddress(Addressing.createInetSocketAddressFromHostAndPortStr( + serverName.getHostAndPort())); + } + regionMap.put(entry.getKey(), server); + } + + return regionMap; + } + + /** + * Gets all the regions and their address for this table. + *

+ * This is mainly useful for the MapReduce integration. + * @return A map of HRegionInfo with it's server address + * @throws IOException if a remote or network exception occurs + */ + public NavigableMap getRegionLocations() throws IOException { + return MetaScanner.allTableRegions(getConfiguration(), getTableName(), false); + } + + /** + * Get the corresponding regions for an arbitrary range of keys. + *

+ * @param startRow Starting row in range, inclusive + * @param endRow Ending row in range, exclusive + * @return A list of HRegionLocations corresponding to the regions that + * contain the specified range + * @throws IOException if a remote or network exception occurs + */ + public List getRegionsInRange(final byte [] startKey, + final byte [] endKey) throws IOException { + return getKeysAndRegionsInRange(startKey, endKey, false).getSecond(); + } + + /** + * Get the corresponding start keys and regions for an arbitrary range of + * keys. + *

+ * @param startKey Starting row in range, inclusive + * @param endKey Ending row in range + * @param includeEndKey true if endRow is inclusive, false if exclusive + * @return A pair of list of start keys and list of HRegionLocations that + * contain the specified range + * @throws IOException if a remote or network exception occurs + */ + private Pair, List> getKeysAndRegionsInRange( + final byte[] startKey, final byte[] endKey, final boolean includeEndKey) + throws IOException { + final boolean endKeyIsEndOfTable = Bytes.equals(endKey,HConstants.EMPTY_END_ROW); + if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) { + throw new IllegalArgumentException( + "Invalid range: " + Bytes.toStringBinary(startKey) + + " > " + Bytes.toStringBinary(endKey)); + } + List keysInRange = new ArrayList(); + List regionsInRange = new ArrayList(); + byte[] currentKey = startKey; + do { + HRegionLocation regionLocation = getRegionLocation(currentKey, false); + keysInRange.add(currentKey); + regionsInRange.add(regionLocation); + currentKey = regionLocation.getRegionInfo().getEndKey(); + } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW) + && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0 + || (includeEndKey && Bytes.compareTo(currentKey, endKey) == 0))); + return new Pair, List>(keysInRange, + regionsInRange); + } + + /** + * Save the passed region information and the table's regions + * cache. + *

+ * This is mainly useful for the MapReduce integration. You can call + * {@link #deserializeRegionInfo deserializeRegionInfo} + * to deserialize regions information from a + * {@link DataInput}, then call this method to load them to cache. + * + * 

+   * {@code
+   * HTable t1 = new HTable("foo");
+   * FileInputStream fis = new FileInputStream("regions.dat");
+   * DataInputStream dis = new DataInputStream(fis);
+   *
+   * Map hm = t1.deserializeRegionInfo(dis);
+   * t1.prewarmRegionCache(hm);
+   * }
+   *
+ * @param regionMap This piece of regions information will be loaded + * to region cache. + */ + public void prewarmRegionCache(Map regionMap) { + this.connection.prewarmRegionCache(this.getTableName(), regionMap); + } + + /** + * Serialize the regions information of this table and output + * to out. + *

+ * This is mainly useful for the MapReduce integration. A client could + * perform a large scan for all the regions for the table, serialize the + * region info to a file. MR job can ship a copy of the meta for the table in + * the DistributedCache. + * 

+   * {@code
+   * FileOutputStream fos = new FileOutputStream("regions.dat");
+   * DataOutputStream dos = new DataOutputStream(fos);
+   * table.serializeRegionInfo(dos);
+   * dos.flush();
+   * dos.close();
+   * }
+   *
+ * @param out {@link DataOutput} to serialize this object into. + * @throws IOException if a remote or network exception occurs + */ + public void serializeRegionInfo(DataOutput out) throws IOException { + Map allRegions = this.getRegionsInfo(); + // first, write number of regions + out.writeInt(allRegions.size()); + for (Map.Entry es : allRegions.entrySet()) { + es.getKey().write(out); + es.getValue().write(out); + } + } + + /** + * Read from in and deserialize the regions information. + * + *

It behaves similarly as {@link #getRegionsInfo getRegionsInfo}, except + * that it loads the region map from a {@link DataInput} object. + * + *

It is supposed to be followed immediately by {@link + * #prewarmRegionCache prewarmRegionCache}. + * + *

+ * Please refer to {@link #prewarmRegionCache prewarmRegionCache} for usage. + * + * @param in {@link DataInput} object. + * @return A map of HRegionInfo with its server address. + * @throws IOException if an I/O exception occurs. + */ + public Map deserializeRegionInfo(DataInput in) + throws IOException { + final Map allRegions = + new TreeMap(); + + // the first integer is expected to be the size of records + int regionsCount = in.readInt(); + for (int i = 0; i < regionsCount; ++i) { + HRegionInfo hri = new HRegionInfo(); + hri.readFields(in); + HServerAddress hsa = new HServerAddress(); + hsa.readFields(in); + allRegions.put(hri, hsa); + } + return allRegions; + } + + /** + * {@inheritDoc} + */ + @Override + public Result getRowOrBefore(final byte[] row, final byte[] family) + throws IOException { + return new ServerCallable(connection, tableName, row, operationTimeout) { + public Result call() throws IOException { + return server.getClosestRowBefore(location.getRegionInfo().getRegionName(), + row, family); + } + }.withRetries(); + } + + /** + * {@inheritDoc} + */ + @Override + public ResultScanner getScanner(final Scan scan) throws IOException { + if (scan.getCaching() <= 0) { + scan.setCaching(getScannerCaching()); + } + return new ClientScanner(getConfiguration(), scan, getTableName(), + this.connection); + } + + /** + * {@inheritDoc} + */ + @Override + public ResultScanner getScanner(byte [] family) throws IOException { + Scan scan = new Scan(); + scan.addFamily(family); + return getScanner(scan); + } + + /** + * {@inheritDoc} + */ + @Override + public ResultScanner getScanner(byte [] family, byte [] qualifier) + throws IOException { + Scan scan = new Scan(); + scan.addColumn(family, qualifier); + return getScanner(scan); + } + + /** + * {@inheritDoc} + */ + @Override + public Result get(final Get get) throws IOException { + return new ServerCallable(connection, tableName, get.getRow(), operationTimeout) { + public Result call() throws IOException { + return server.get(location.getRegionInfo().getRegionName(), get); + } + }.withRetries(); + } + + /** + * {@inheritDoc} + */ + @Override + public Result[] get(List gets) throws IOException { + try { + Object [] r1 = batch((List)gets); + + // translate. + Result [] results = new Result[r1.length]; + int i=0; + for (Object o : r1) { + // batch ensures if there is a failure we get an exception instead + results[i++] = (Result) o; + } + + return results; + } catch (InterruptedException e) { + throw new IOException(e); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void batch(final List actions, final Object[] results) + throws InterruptedException, IOException { + connection.processBatch(actions, tableName, pool, results); + } + + /** + * {@inheritDoc} + */ + @Override + public Object[] batch(final List actions) throws InterruptedException, IOException { + Object[] results = new Object[actions.size()]; + connection.processBatch(actions, tableName, pool, results); + return results; + } + + /** + * {@inheritDoc} + */ + @Override + public void delete(final Delete delete) throws IOException { + List deletes = new ArrayList(1); + deletes.add(delete); + delete(deletes); + } + + /** + * {@inheritDoc} + */ + @Override + public void delete(final List deletes) + throws IOException { + Object[] results = new Object[deletes.size()]; + try { + connection.processBatch((List) deletes, tableName, pool, results); + } catch (InterruptedException e) { + throw new IOException(e); + } finally { + // mutate list so that it is empty for complete success, or contains only failed records + // results are returned in the same order as the requests in list + // walk the list backwards, so we can remove from list without impacting the indexes of earlier members + for (int i = results.length - 1; i>=0; i--) { + // if result is not null, it succeeded + if (results[i] instanceof Result) { + deletes.remove(i); + } + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public void put(final Put put) throws IOException { + doPut(put); + if (autoFlush) { + flushCommits(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void put(final List puts) throws IOException { + for (Put put : puts) { + doPut(put); + } + if (autoFlush) { + flushCommits(); + } + } + + private void doPut(Put put) throws IOException{ + validatePut(put); + writeBuffer.add(put); + currentWriteBufferSize += put.heapSize(); + if (currentWriteBufferSize > writeBufferSize) { + flushCommits(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void mutateRow(final RowMutations rm) throws IOException { + new ServerCallable(connection, tableName, rm.getRow(), + operationTimeout) { + public Void call() throws IOException { + server.mutateRow(location.getRegionInfo().getRegionName(), rm); + return null; + } + }.withRetries(); + } + + /** + * {@inheritDoc} + */ + @Override + public Result append(final Append append) throws IOException { + if (append.numFamilies() == 0) { + throw new IOException( + "Invalid arguments to append, no columns specified"); + } + return new ServerCallable(connection, tableName, append.getRow(), operationTimeout) { + public Result call() throws IOException { + return server.append( + location.getRegionInfo().getRegionName(), append); + } + }.withRetries(); + } + + /** + * {@inheritDoc} + */ + @Override + public Result increment(final Increment increment) throws IOException { + if (!increment.hasFamilies()) { + throw new IOException( + "Invalid arguments to increment, no columns specified"); + } + return new ServerCallable(connection, tableName, increment.getRow(), operationTimeout) { + public Result call() throws IOException { + return server.increment( + location.getRegionInfo().getRegionName(), increment); + } + }.withRetries(); + } + + /** + * {@inheritDoc} + */ + @Override + public long incrementColumnValue(final byte [] row, final byte [] family, + final byte [] qualifier, final long amount) + throws IOException { + return incrementColumnValue(row, family, qualifier, amount, true); + } + + /** + * {@inheritDoc} + */ + @Override + public long incrementColumnValue(final byte [] row, final byte [] family, + final byte [] qualifier, final long amount, final boolean writeToWAL) + throws IOException { + NullPointerException npe = null; + if (row == null) { + npe = new NullPointerException("row is null"); + } else if (family == null) { + npe = new NullPointerException("column is null"); + } + if (npe != null) { + throw new IOException( + "Invalid arguments to incrementColumnValue", npe); + } + return new ServerCallable(connection, tableName, row, operationTimeout) { + public Long call() throws IOException { + return server.incrementColumnValue( + location.getRegionInfo().getRegionName(), row, family, + qualifier, amount, writeToWAL); + } + }.withRetries(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean checkAndPut(final byte [] row, + final byte [] family, final byte [] qualifier, final byte [] value, + final Put put) + throws IOException { + return new ServerCallable(connection, tableName, row, operationTimeout) { + public Boolean call() throws IOException { + return server.checkAndPut(location.getRegionInfo().getRegionName(), + row, family, qualifier, value, put) ? Boolean.TRUE : Boolean.FALSE; + } + }.withRetries(); + } + + + /** + * {@inheritDoc} + */ + @Override + public boolean checkAndDelete(final byte [] row, + final byte [] family, final byte [] qualifier, final byte [] value, + final Delete delete) + throws IOException { + return new ServerCallable(connection, tableName, row, operationTimeout) { + public Boolean call() throws IOException { + return server.checkAndDelete( + location.getRegionInfo().getRegionName(), + row, family, qualifier, value, delete) + ? Boolean.TRUE : Boolean.FALSE; + } + }.withRetries(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean exists(final Get get) throws IOException { + return new ServerCallable(connection, tableName, get.getRow(), operationTimeout) { + public Boolean call() throws IOException { + return server. + exists(location.getRegionInfo().getRegionName(), get); + } + }.withRetries(); + } + + /** + * {@inheritDoc} + */ + @Override + public void flushCommits() throws IOException { + try { + Object[] results = new Object[writeBuffer.size()]; + try { + this.connection.processBatch(writeBuffer, tableName, pool, results); + } catch (InterruptedException e) { + throw new IOException(e); + } finally { + // mutate list so that it is empty for complete success, or contains + // only failed records results are returned in the same order as the + // requests in list walk the list backwards, so we can remove from list + // without impacting the indexes of earlier members + for (int i = results.length - 1; i>=0; i--) { + if (results[i] instanceof Result) { + // successful Puts are removed from the list here. + writeBuffer.remove(i); + } + } + } + } finally { + if (clearBufferOnFail) { + writeBuffer.clear(); + currentWriteBufferSize = 0; + } else { + // the write buffer was adjusted by processBatchOfPuts + currentWriteBufferSize = 0; + for (Put aPut : writeBuffer) { + currentWriteBufferSize += aPut.heapSize(); + } + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public void close() throws IOException { + if (this.closed) { + return; + } + flushCommits(); + if (cleanupPoolOnClose) { + this.pool.shutdown(); + } + if (cleanupConnectionOnClose) { + if (this.connection != null) { + this.connection.close(); + } + } + this.closed = true; + } + + // validate for well-formedness + private void validatePut(final Put put) throws IllegalArgumentException{ + if (put.isEmpty()) { + throw new IllegalArgumentException("No columns to insert"); + } + if (maxKeyValueSize > 0) { + for (List list : put.getFamilyMap().values()) { + for (KeyValue kv : list) { + if (kv.getLength() > maxKeyValueSize) { + throw new IllegalArgumentException("KeyValue size too large"); + } + } + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public RowLock lockRow(final byte [] row) + throws IOException { + return new ServerCallable(connection, tableName, row, operationTimeout) { + public RowLock call() throws IOException { + long lockId = + server.lockRow(location.getRegionInfo().getRegionName(), row); + return new RowLock(row,lockId); + } + }.withRetries(); + } + + /** + * {@inheritDoc} + */ + @Override + public void unlockRow(final RowLock rl) + throws IOException { + new ServerCallable(connection, tableName, rl.getRow(), operationTimeout) { + public Boolean call() throws IOException { + server.unlockRow(location.getRegionInfo().getRegionName(), + rl.getLockId()); + return null; // FindBugs NP_BOOLEAN_RETURN_NULL + } + }.withRetries(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isAutoFlush() { + return autoFlush; + } + + /** + * See {@link #setAutoFlush(boolean, boolean)} + * + * @param autoFlush + * Whether or not to enable 'auto-flush'. + */ + public void setAutoFlush(boolean autoFlush) { + setAutoFlush(autoFlush, autoFlush); + } + + /** + * Turns 'auto-flush' on or off. + *

+ * When enabled (default), {@link Put} operations don't get buffered/delayed + * and are immediately executed. Failed operations are not retried. This is + * slower but safer. + *

+ * Turning off {@link #autoFlush} means that multiple {@link Put}s will be + * accepted before any RPC is actually sent to do the write operations. If the + * application dies before pending writes get flushed to HBase, data will be + * lost. + *

+ * When you turn {@link #autoFlush} off, you should also consider the + * {@link #clearBufferOnFail} option. By default, asynchronous {@link Put} + * requests will be retried on failure until successful. However, this can + * pollute the writeBuffer and slow down batching performance. Additionally, + * you may want to issue a number of Put requests and call + * {@link #flushCommits()} as a barrier. In both use cases, consider setting + * clearBufferOnFail to true to erase the buffer after {@link #flushCommits()} + * has been called, regardless of success. + * + * @param autoFlush + * Whether or not to enable 'auto-flush'. + * @param clearBufferOnFail + * Whether to keep Put failures in the writeBuffer + * @see #flushCommits + */ + public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) { + this.autoFlush = autoFlush; + this.clearBufferOnFail = autoFlush || clearBufferOnFail; + } + + /** + * Returns the maximum size in bytes of the write buffer for this HTable. + *

+ * The default value comes from the configuration parameter + * {@code hbase.client.write.buffer}. + * @return The size of the write buffer in bytes. + */ + public long getWriteBufferSize() { + return writeBufferSize; + } + + /** + * Sets the size of the buffer in bytes. + *

+ * If the new size is less than the current amount of data in the + * write buffer, the buffer gets flushed. + * @param writeBufferSize The new write buffer size, in bytes. + * @throws IOException if a remote or network exception occurs. + */ + public void setWriteBufferSize(long writeBufferSize) throws IOException { + this.writeBufferSize = writeBufferSize; + if(currentWriteBufferSize > writeBufferSize) { + flushCommits(); + } + } + + /** + * Returns the write buffer. + * @return The current write buffer. + */ + public ArrayList getWriteBuffer() { + return writeBuffer; + } + + /** + * The pool is used for mutli requests for this HTable + * @return the pool used for mutli + */ + ExecutorService getPool() { + return this.pool; + } + + /** + * Enable or disable region cache prefetch for the table. It will be + * applied for the given table's all HTable instances who share the same + * connection. By default, the cache prefetch is enabled. + * @param tableName name of table to configure. + * @param enable Set to true to enable region cache prefetch. Or set to + * false to disable it. + * @throws IOException + */ + public static void setRegionCachePrefetch(final byte[] tableName, + final boolean enable) throws IOException { + HConnectionManager.execute(new HConnectable(HBaseConfiguration + .create()) { + @Override + public Void connect(HConnection connection) throws IOException { + connection.setRegionCachePrefetch(tableName, enable); + return null; + } + }); + } + + /** + * Enable or disable region cache prefetch for the table. It will be + * applied for the given table's all HTable instances who share the same + * connection. By default, the cache prefetch is enabled. + * @param conf The Configuration object to use. + * @param tableName name of table to configure. + * @param enable Set to true to enable region cache prefetch. Or set to + * false to disable it. + * @throws IOException + */ + public static void setRegionCachePrefetch(final Configuration conf, + final byte[] tableName, final boolean enable) throws IOException { + HConnectionManager.execute(new HConnectable(conf) { + @Override + public Void connect(HConnection connection) throws IOException { + connection.setRegionCachePrefetch(tableName, enable); + return null; + } + }); + } + + /** + * Check whether region cache prefetch is enabled or not for the table. + * @param conf The Configuration object to use. + * @param tableName name of table to check + * @return true if table's region cache prefecth is enabled. Otherwise + * it is disabled. + * @throws IOException + */ + public static boolean getRegionCachePrefetch(final Configuration conf, + final byte[] tableName) throws IOException { + return HConnectionManager.execute(new HConnectable(conf) { + @Override + public Boolean connect(HConnection connection) throws IOException { + return connection.getRegionCachePrefetch(tableName); + } + }); + } + + /** + * Check whether region cache prefetch is enabled or not for the table. + * @param tableName name of table to check + * @return true if table's region cache prefecth is enabled. Otherwise + * it is disabled. + * @throws IOException + */ + public static boolean getRegionCachePrefetch(final byte[] tableName) throws IOException { + return HConnectionManager.execute(new HConnectable( + HBaseConfiguration.create()) { + @Override + public Boolean connect(HConnection connection) throws IOException { + return connection.getRegionCachePrefetch(tableName); + } + }); + } + + /** + * Explicitly clears the region cache to fetch the latest value from META. + * This is a power user function: avoid unless you know the ramifications. + */ + public void clearRegionCache() { + this.connection.clearRegionCache(); + } + + /** + * {@inheritDoc} + */ + @Override + public T coprocessorProxy( + Class protocol, byte[] row) { + return (T)Proxy.newProxyInstance(this.getClass().getClassLoader(), + new Class[]{protocol}, + new ExecRPCInvoker(configuration, + connection, + protocol, + tableName, + row)); + } + + /** + * {@inheritDoc} + */ + @Override + public Map coprocessorExec( + Class protocol, byte[] startKey, byte[] endKey, + Batch.Call callable) + throws IOException, Throwable { + + final Map results = Collections.synchronizedMap(new TreeMap( + Bytes.BYTES_COMPARATOR)); + coprocessorExec(protocol, startKey, endKey, callable, + new Batch.Callback(){ + public void update(byte[] region, byte[] row, R value) { + results.put(region, value); + } + }); + return results; + } + + /** + * {@inheritDoc} + */ + @Override + public void coprocessorExec( + Class protocol, byte[] startKey, byte[] endKey, + Batch.Call callable, Batch.Callback callback) + throws IOException, Throwable { + + // get regions covered by the row range + List keys = getStartKeysInRange(startKey, endKey); + connection.processExecs(protocol, keys, tableName, pool, callable, + callback); + } + + private List getStartKeysInRange(byte[] start, byte[] end) + throws IOException { + if (start == null) { + start = HConstants.EMPTY_START_ROW; + } + if (end == null) { + end = HConstants.EMPTY_END_ROW; + } + return getKeysAndRegionsInRange(start, end, true).getFirst(); + } + + public void setOperationTimeout(int operationTimeout) { + this.operationTimeout = operationTimeout; + } + + public int getOperationTimeout() { + return operationTimeout; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableFactory.java b/src/main/java/org/apache/hadoop/hbase/client/HTableFactory.java new file mode 100644 index 0000000..90f6cb9 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/HTableFactory.java @@ -0,0 +1,46 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; + +/** + * Factory for creating HTable instances. + * + * @since 0.21.0 + */ +public class HTableFactory implements HTableInterfaceFactory { + @Override + public HTableInterface createHTableInterface(Configuration config, + byte[] tableName) { + try { + return new HTable(config, tableName); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + + @Override + public void releaseHTableInterface(HTableInterface table) throws IOException { + table.close(); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java b/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java new file mode 100644 index 0000000..e7e296c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java @@ -0,0 +1,532 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; + +/** + * Used to communicate with a single HBase table. + * + * @since 0.21.0 + */ +public interface HTableInterface extends Closeable { + + /** + * Gets the name of this table. + * + * @return the table name. + */ + byte[] getTableName(); + + /** + * Returns the {@link Configuration} object used by this instance. + *

+ * The reference returned is not a copy, so any change made to it will + * affect this instance. + */ + Configuration getConfiguration(); + + /** + * Gets the {@link HTableDescriptor table descriptor} for this table. + * @throws IOException if a remote or network exception occurs. + */ + HTableDescriptor getTableDescriptor() throws IOException; + + /** + * Test for the existence of columns in the table, as specified in the Get. + *

+ * + * This will return true if the Get matches one or more keys, false if not. + *

+ * + * This is a server-side call so it prevents any data from being transfered to + * the client. + * + * @param get the Get + * @return true if the specified Get matches one or more keys, false if not + * @throws IOException e + */ + boolean exists(Get get) throws IOException; + + /** + * Method that does a batch call on Deletes, Gets, Puts, Increments, Appends and RowMutations. + * The execution ordering of the actions is not defined. Meaning if you do a Put and a + * Get in the same {@link #batch} call, you will not necessarily be + * guaranteed that the Get returns what the Put had put. + * + * @param actions list of Get, Put, Delete, Increment, Append, RowMutations objects + * @param results Empty Object[], same size as actions. Provides access to partial + * results, in case an exception is thrown. A null in the result array means that + * the call for that action failed, even after retries + * @throws IOException + * @since 0.90.0 + */ + void batch(final List actions, final Object[] results) throws IOException, InterruptedException; + + /** + * Same as {@link #batch(List, Object[])}, but returns an array of + * results instead of using a results parameter reference. + * + * @param actions list of Get, Put, Delete, Increment, Append, RowMutations objects + * @return the results from the actions. A null in the return array means that + * the call for that action failed, even after retries + * @throws IOException + * @since 0.90.0 + */ + Object[] batch(final List actions) throws IOException, InterruptedException; + + /** + * Extracts certain cells from a given row. + * @param get The object that specifies what data to fetch and from which row. + * @return The data coming from the specified row, if it exists. If the row + * specified doesn't exist, the {@link Result} instance returned won't + * contain any {@link KeyValue}, as indicated by {@link Result#isEmpty()}. + * @throws IOException if a remote or network exception occurs. + * @since 0.20.0 + */ + Result get(Get get) throws IOException; + + /** + * Extracts certain cells from the given rows, in batch. + * + * @param gets The objects that specify what data to fetch and from which rows. + * + * @return The data coming from the specified rows, if it exists. If the row + * specified doesn't exist, the {@link Result} instance returned won't + * contain any {@link KeyValue}, as indicated by {@link Result#isEmpty()}. + * If there are any failures even after retries, there will be a null in + * the results array for those Gets, AND an exception will be thrown. + * @throws IOException if a remote or network exception occurs. + * + * @since 0.90.0 + */ + Result[] get(List gets) throws IOException; + + /** + * Return the row that matches row exactly, + * or the one that immediately precedes it. + * + * @param row A row key. + * @param family Column family to include in the {@link Result}. + * @throws IOException if a remote or network exception occurs. + * @since 0.20.0 + * + * @deprecated As of version 0.92 this method is deprecated without + * replacement. + * getRowOrBefore is used internally to find entries in .META. and makes + * various assumptions about the table (which are true for .META. but not + * in general) to be efficient. + */ + Result getRowOrBefore(byte[] row, byte[] family) throws IOException; + + /** + * Returns a scanner on the current table as specified by the {@link Scan} + * object. + * Note that the passed {@link Scan}'s start row and caching properties + * maybe changed. + * + * @param scan A configured {@link Scan} object. + * @return A scanner. + * @throws IOException if a remote or network exception occurs. + * @since 0.20.0 + */ + ResultScanner getScanner(Scan scan) throws IOException; + + /** + * Gets a scanner on the current table for the given family. + * + * @param family The column family to scan. + * @return A scanner. + * @throws IOException if a remote or network exception occurs. + * @since 0.20.0 + */ + ResultScanner getScanner(byte[] family) throws IOException; + + /** + * Gets a scanner on the current table for the given family and qualifier. + * + * @param family The column family to scan. + * @param qualifier The column qualifier to scan. + * @return A scanner. + * @throws IOException if a remote or network exception occurs. + * @since 0.20.0 + */ + ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException; + + + /** + * Puts some data in the table. + *

+ * If {@link #isAutoFlush isAutoFlush} is false, the update is buffered + * until the internal buffer is full. + * @param put The data to put. + * @throws IOException if a remote or network exception occurs. + * @since 0.20.0 + */ + void put(Put put) throws IOException; + + /** + * Puts some data in the table, in batch. + *

+ * If {@link #isAutoFlush isAutoFlush} is false, the update is buffered + * until the internal buffer is full. + *

+ * This can be used for group commit, or for submitting user defined + * batches. The writeBuffer will be periodically inspected while the List + * is processed, so depending on the List size the writeBuffer may flush + * not at all, or more than once. + * @param puts The list of mutations to apply. The batch put is done by + * aggregating the iteration of the Puts over the write buffer + * at the client-side for a single RPC call. + * @throws IOException if a remote or network exception occurs. + * @since 0.20.0 + */ + void put(List puts) throws IOException; + + /** + * Atomically checks if a row/family/qualifier value matches the expected + * value. If it does, it adds the put. If the passed value is null, the check + * is for the lack of column (ie: non-existance) + * + * @param row to check + * @param family column family to check + * @param qualifier column qualifier to check + * @param value the expected value + * @param put data to put if check succeeds + * @throws IOException e + * @return true if the new put was executed, false otherwise + */ + boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, + byte[] value, Put put) throws IOException; + + /** + * Deletes the specified cells/row. + * + * @param delete The object that specifies what to delete. + * @throws IOException if a remote or network exception occurs. + * @since 0.20.0 + */ + void delete(Delete delete) throws IOException; + + /** + * Deletes the specified cells/rows in bulk. + * @param deletes List of things to delete. List gets modified by this + * method (in particular it gets re-ordered, so the order in which the elements + * are inserted in the list gives no guarantee as to the order in which the + * {@link Delete}s are executed). + * @throws IOException if a remote or network exception occurs. In that case + * the {@code deletes} argument will contain the {@link Delete} instances + * that have not be successfully applied. + * @since 0.20.1 + */ + void delete(List deletes) throws IOException; + + /** + * Atomically checks if a row/family/qualifier value matches the expected + * value. If it does, it adds the delete. If the passed value is null, the + * check is for the lack of column (ie: non-existance) + * + * @param row to check + * @param family column family to check + * @param qualifier column qualifier to check + * @param value the expected value + * @param delete data to delete if check succeeds + * @throws IOException e + * @return true if the new delete was executed, false otherwise + */ + boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, + byte[] value, Delete delete) throws IOException; + + /** + * Performs multiple mutations atomically on a single row. Currently + * {@link Put} and {@link Delete} are supported. + * + * @param arm object that specifies the set of mutations to perform + * atomically + * @throws IOException + */ + public void mutateRow(final RowMutations rm) throws IOException; + + /** + * Appends values to one or more columns within a single row. + *

+ * This operation does not appear atomic to readers. Appends are done + * under a single row lock, so write operations to a row are synchronized, but + * readers do not take row locks so get and scan operations can see this + * operation partially completed. + * + * @param append object that specifies the columns and amounts to be used + * for the increment operations + * @throws IOException e + * @return values of columns after the append operation (maybe null) + */ + public Result append(final Append append) throws IOException; + + /** + * Increments one or more columns within a single row. + *

+ * This operation does not appear atomic to readers. Increments are done + * under a single row lock, so write operations to a row are synchronized, but + * readers do not take row locks so get and scan operations can see this + * operation partially completed. + * + * @param increment object that specifies the columns and amounts to be used + * for the increment operations + * @throws IOException e + * @return values of columns after the increment + */ + public Result increment(final Increment increment) throws IOException; + + /** + * Atomically increments a column value. + *

+ * Equivalent to {@link #incrementColumnValue(byte[], byte[], byte[], + * long, boolean) incrementColumnValue}(row, family, qualifier, amount, + * true)} + * @param row The row that contains the cell to increment. + * @param family The column family of the cell to increment. + * @param qualifier The column qualifier of the cell to increment. + * @param amount The amount to increment the cell with (or decrement, if the + * amount is negative). + * @return The new value, post increment. + * @throws IOException if a remote or network exception occurs. + */ + long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, + long amount) throws IOException; + + /** + * Atomically increments a column value. If the column value already exists + * and is not a big-endian long, this could throw an exception. If the column + * value does not yet exist it is initialized to amount and + * written to the specified column. + * + *

Setting writeToWAL to false means that in a fail scenario, you will lose + * any increments that have not been flushed. + * @param row The row that contains the cell to increment. + * @param family The column family of the cell to increment. + * @param qualifier The column qualifier of the cell to increment. + * @param amount The amount to increment the cell with (or decrement, if the + * amount is negative). + * @param writeToWAL if {@code true}, the operation will be applied to the + * Write Ahead Log (WAL). This makes the operation slower but safer, as if + * the call returns successfully, it is guaranteed that the increment will + * be safely persisted. When set to {@code false}, the call may return + * successfully before the increment is safely persisted, so it's possible + * that the increment be lost in the event of a failure happening before the + * operation gets persisted. + * @return The new value, post increment. + * @throws IOException if a remote or network exception occurs. + */ + long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, + long amount, boolean writeToWAL) throws IOException; + + /** + * Tells whether or not 'auto-flush' is turned on. + * + * @return {@code true} if 'auto-flush' is enabled (default), meaning + * {@link Put} operations don't get buffered/delayed and are immediately + * executed. + */ + boolean isAutoFlush(); + + /** + * Executes all the buffered {@link Put} operations. + *

+ * This method gets called once automatically for every {@link Put} or batch + * of {@link Put}s (when put(List) is used) when + * {@link #isAutoFlush} is {@code true}. + * @throws IOException if a remote or network exception occurs. + */ + void flushCommits() throws IOException; + + /** + * Releases any resources help or pending changes in internal buffers. + * + * @throws IOException if a remote or network exception occurs. + */ + void close() throws IOException; + + /** + * Obtains a lock on a row. + * + * @param row The row to lock. + * @return A {@link RowLock} containing the row and lock id. + * @throws IOException if a remote or network exception occurs. + * @see RowLock + * @see #unlockRow + * @deprecated {@link RowLock} and associated operations are deprecated + */ + RowLock lockRow(byte[] row) throws IOException; + + /** + * Releases a row lock. + * + * @param rl The row lock to release. + * @throws IOException if a remote or network exception occurs. + * @see RowLock + * @see #unlockRow + * @deprecated {@link RowLock} and associated operations are deprecated + */ + void unlockRow(RowLock rl) throws IOException; + + /** + * Creates and returns a proxy to the CoprocessorProtocol instance running in the + * region containing the specified row. The row given does not actually have + * to exist. Whichever region would contain the row based on start and end keys will + * be used. Note that the {@code row} parameter is also not passed to the + * coprocessor handler registered for this protocol, unless the {@code row} + * is separately passed as an argument in a proxy method call. The parameter + * here is just used to locate the region used to handle the call. + * + * @param protocol The class or interface defining the remote protocol + * @param row The row key used to identify the remote region location + * @return A CoprocessorProtocol instance + */ + T coprocessorProxy(Class protocol, byte[] row); + + /** + * Invoke the passed + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call} against + * the {@link CoprocessorProtocol} instances running in the selected regions. + * All regions beginning with the region containing the startKey + * row, through to the region containing the endKey row (inclusive) + * will be used. If startKey or endKey is + * null, the first and last regions in the table, respectively, + * will be used in the range selection. + * + * @param protocol the CoprocessorProtocol implementation to call + * @param startKey start region selection with region containing this row + * @param endKey select regions up to and including the region containing + * this row + * @param callable wraps the CoprocessorProtocol implementation method calls + * made per-region + * @param CoprocessorProtocol subclass for the remote invocation + * @param Return type for the + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call#call(Object)} + * method + * @return a Map of region names to + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call#call(Object)} return values + */ + Map coprocessorExec( + Class protocol, byte[] startKey, byte[] endKey, Batch.Call callable) + throws IOException, Throwable; + + /** + * Invoke the passed + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call} against + * the {@link CoprocessorProtocol} instances running in the selected regions. + * All regions beginning with the region containing the startKey + * row, through to the region containing the endKey row + * (inclusive) + * will be used. If startKey or endKey is + * null, the first and last regions in the table, respectively, + * will be used in the range selection. + * + *

+ * For each result, the given + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Callback#update(byte[], byte[], Object)} + * method will be called. + *

+ * + * @param protocol the CoprocessorProtocol implementation to call + * @param startKey start region selection with region containing this row + * @param endKey select regions up to and including the region containing + * this row + * @param callable wraps the CoprocessorProtocol implementation method calls + * made per-region + * @param callback an instance upon which + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Callback#update(byte[], byte[], Object)} with the + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call#call(Object)} + * return value for each region + * @param CoprocessorProtocol subclass for the remote invocation + * @param Return type for the + * {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call#call(Object)} + * method + */ + void coprocessorExec( + Class protocol, byte[] startKey, byte[] endKey, + Batch.Call callable, Batch.Callback callback) + throws IOException, Throwable; + + /** + * See {@link #setAutoFlush(boolean, boolean)} + * + * @param autoFlush + * Whether or not to enable 'auto-flush'. + */ + public void setAutoFlush(boolean autoFlush); + + /** + * Turns 'auto-flush' on or off. + *

+ * When enabled (default), {@link Put} operations don't get buffered/delayed + * and are immediately executed. Failed operations are not retried. This is + * slower but safer. + *

+ * Turning off {@link #autoFlush} means that multiple {@link Put}s will be + * accepted before any RPC is actually sent to do the write operations. If the + * application dies before pending writes get flushed to HBase, data will be + * lost. + *

+ * When you turn {@link #autoFlush} off, you should also consider the + * {@link #clearBufferOnFail} option. By default, asynchronous {@link Put} + * requests will be retried on failure until successful. However, this can + * pollute the writeBuffer and slow down batching performance. Additionally, + * you may want to issue a number of Put requests and call + * {@link #flushCommits()} as a barrier. In both use cases, consider setting + * clearBufferOnFail to true to erase the buffer after {@link #flushCommits()} + * has been called, regardless of success. + * + * @param autoFlush + * Whether or not to enable 'auto-flush'. + * @param clearBufferOnFail + * Whether to keep Put failures in the writeBuffer + * @see #flushCommits + */ + public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail); + + /** + * Returns the maximum size in bytes of the write buffer for this HTable. + *

+ * The default value comes from the configuration parameter + * {@code hbase.client.write.buffer}. + * @return The size of the write buffer in bytes. + */ + public long getWriteBufferSize(); + + /** + * Sets the size of the buffer in bytes. + *

+ * If the new size is less than the current amount of data in the + * write buffer, the buffer gets flushed. + * @param writeBufferSize The new write buffer size, in bytes. + * @throws IOException if a remote or network exception occurs. + */ + public void setWriteBufferSize(long writeBufferSize) throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableInterfaceFactory.java b/src/main/java/org/apache/hadoop/hbase/client/HTableInterfaceFactory.java new file mode 100644 index 0000000..ab7efcc --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/HTableInterfaceFactory.java @@ -0,0 +1,49 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; + + +/** + * Defines methods to create new HTableInterface. + * + * @since 0.21.0 + */ +public interface HTableInterfaceFactory { + + /** + * Creates a new HTableInterface. + * + * @param config HBaseConfiguration instance. + * @param tableName name of the HBase table. + * @return HTableInterface instance. + */ + HTableInterface createHTableInterface(Configuration config, byte[] tableName); + + + /** + * Release the HTable resource represented by the table. + * @param table + */ + void releaseHTableInterface(final HTableInterface table) throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java new file mode 100644 index 0000000..d9615ac --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java @@ -0,0 +1,542 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.PoolMap; +import org.apache.hadoop.hbase.util.PoolMap.PoolType; + +/** + * A simple pool of HTable instances. + * + * Each HTablePool acts as a pool for all tables. To use, instantiate an + * HTablePool and use {@link #getTable(String)} to get an HTable from the pool. + * + * This method is not needed anymore, clients should call + * HTableInterface.close() rather than returning the tables to the pool + * + * Once you are done with it, close your instance of {@link HTableInterface} + * by calling {@link HTableInterface#close()} rather than returning the tables + * to the pool with (deprecated) {@link #putTable(HTableInterface)}. + * + *

+ * A pool can be created with a maxSize which defines the most HTable + * references that will ever be retained for each table. Otherwise the default + * is {@link Integer#MAX_VALUE}. + * + *

+ * Pool will manage its own connections to the cluster. See + * {@link HConnectionManager}. + */ +public class HTablePool implements Closeable { + private final PoolMap tables; + private final int maxSize; + private final PoolType poolType; + private final Configuration config; + private final HTableInterfaceFactory tableFactory; + + /** + * Default Constructor. Default HBaseConfiguration and no limit on pool size. + */ + public HTablePool() { + this(HBaseConfiguration.create(), Integer.MAX_VALUE); + } + + /** + * Constructor to set maximum versions and use the specified configuration. + * + * @param config + * configuration + * @param maxSize + * maximum number of references to keep for each table + */ + public HTablePool(final Configuration config, final int maxSize) { + this(config, maxSize, null, null); + } + + /** + * Constructor to set maximum versions and use the specified configuration and + * table factory. + * + * @param config + * configuration + * @param maxSize + * maximum number of references to keep for each table + * @param tableFactory + * table factory + */ + public HTablePool(final Configuration config, final int maxSize, + final HTableInterfaceFactory tableFactory) { + this(config, maxSize, tableFactory, PoolType.Reusable); + } + + /** + * Constructor to set maximum versions and use the specified configuration and + * pool type. + * + * @param config + * configuration + * @param maxSize + * maximum number of references to keep for each table + * @param poolType + * pool type which is one of {@link PoolType#Reusable} or + * {@link PoolType#ThreadLocal} + */ + public HTablePool(final Configuration config, final int maxSize, + final PoolType poolType) { + this(config, maxSize, null, poolType); + } + + /** + * Constructor to set maximum versions and use the specified configuration, + * table factory and pool type. The HTablePool supports the + * {@link PoolType#Reusable} and {@link PoolType#ThreadLocal}. If the pool + * type is null or not one of those two values, then it will default to + * {@link PoolType#Reusable}. + * + * @param config + * configuration + * @param maxSize + * maximum number of references to keep for each table + * @param tableFactory + * table factory + * @param poolType + * pool type which is one of {@link PoolType#Reusable} or + * {@link PoolType#ThreadLocal} + */ + public HTablePool(final Configuration config, final int maxSize, + final HTableInterfaceFactory tableFactory, PoolType poolType) { + // Make a new configuration instance so I can safely cleanup when + // done with the pool. + this.config = config == null ? HBaseConfiguration.create() : config; + this.maxSize = maxSize; + this.tableFactory = tableFactory == null ? new HTableFactory() + : tableFactory; + if (poolType == null) { + this.poolType = PoolType.Reusable; + } else { + switch (poolType) { + case Reusable: + case ThreadLocal: + this.poolType = poolType; + break; + default: + this.poolType = PoolType.Reusable; + break; + } + } + this.tables = new PoolMap(this.poolType, + this.maxSize); + } + + /** + * Get a reference to the specified table from the pool. + *

+ *

+ * + * @param tableName + * table name + * @return a reference to the specified table + * @throws RuntimeException + * if there is a problem instantiating the HTable + */ + public HTableInterface getTable(String tableName) { + // call the old getTable implementation renamed to findOrCreateTable + HTableInterface table = findOrCreateTable(tableName); + // return a proxy table so when user closes the proxy, the actual table + // will be returned to the pool + return new PooledHTable(table); + } + + /** + * Get a reference to the specified table from the pool. + *

+ * + * Create a new one if one is not available. + * + * @param tableName + * table name + * @return a reference to the specified table + * @throws RuntimeException + * if there is a problem instantiating the HTable + */ + private HTableInterface findOrCreateTable(String tableName) { + HTableInterface table = tables.get(tableName); + if (table == null) { + table = createHTable(tableName); + } + return table; + } + + /** + * Get a reference to the specified table from the pool. + *

+ * + * Create a new one if one is not available. + * + * @param tableName + * table name + * @return a reference to the specified table + * @throws RuntimeException + * if there is a problem instantiating the HTable + */ + public HTableInterface getTable(byte[] tableName) { + return getTable(Bytes.toString(tableName)); + } + + /** + * This method is not needed anymore, clients should call + * HTableInterface.close() rather than returning the tables to the pool + * + * @param table + * the proxy table user got from pool + * @deprecated + */ + public void putTable(HTableInterface table) throws IOException { + // we need to be sure nobody puts a proxy implementation in the pool + // but if the client code is not updated + // and it will continue to call putTable() instead of calling close() + // then we need to return the wrapped table to the pool instead of the + // proxy + // table + if (table instanceof PooledHTable) { + returnTable(((PooledHTable) table).getWrappedTable()); + } else { + // normally this should not happen if clients pass back the same + // table + // object they got from the pool + // but if it happens then it's better to reject it + throw new IllegalArgumentException("not a pooled table: " + table); + } + } + + /** + * Puts the specified HTable back into the pool. + *

+ * + * If the pool already contains maxSize references to the table, then + * the table instance gets closed after flushing buffered edits. + * + * @param table + * table + */ + private void returnTable(HTableInterface table) throws IOException { + // this is the old putTable method renamed and made private + String tableName = Bytes.toString(table.getTableName()); + if (tables.size(tableName) >= maxSize) { + // release table instance since we're not reusing it + this.tables.remove(tableName, table); + this.tableFactory.releaseHTableInterface(table); + return; + } + tables.put(tableName, table); + } + + protected HTableInterface createHTable(String tableName) { + return this.tableFactory.createHTableInterface(config, + Bytes.toBytes(tableName)); + } + + /** + * Closes all the HTable instances , belonging to the given table, in the + * table pool. + *

+ * Note: this is a 'shutdown' of the given table pool and different from + * {@link #putTable(HTableInterface)}, that is used to return the table + * instance to the pool for future re-use. + * + * @param tableName + */ + public void closeTablePool(final String tableName) throws IOException { + Collection tables = this.tables.values(tableName); + if (tables != null) { + for (HTableInterface table : tables) { + this.tableFactory.releaseHTableInterface(table); + } + } + this.tables.remove(tableName); + } + + /** + * See {@link #closeTablePool(String)}. + * + * @param tableName + */ + public void closeTablePool(final byte[] tableName) throws IOException { + closeTablePool(Bytes.toString(tableName)); + } + + /** + * Closes all the HTable instances , belonging to all tables in the table + * pool. + *

+ * Note: this is a 'shutdown' of all the table pools. + */ + public void close() throws IOException { + for (String tableName : tables.keySet()) { + closeTablePool(tableName); + } + this.tables.clear(); + } + + public int getCurrentPoolSize(String tableName) { + return tables.size(tableName); + } + + /** + * A proxy class that implements HTableInterface.close method to return the + * wrapped table back to the table pool + * + */ + class PooledHTable implements HTableInterface { + + private HTableInterface table; // actual table implementation + + public PooledHTable(HTableInterface table) { + this.table = table; + } + + @Override + public byte[] getTableName() { + return table.getTableName(); + } + + @Override + public Configuration getConfiguration() { + return table.getConfiguration(); + } + + @Override + public HTableDescriptor getTableDescriptor() throws IOException { + return table.getTableDescriptor(); + } + + @Override + public boolean exists(Get get) throws IOException { + return table.exists(get); + } + + @Override + public void batch(List actions, Object[] results) throws IOException, + InterruptedException { + table.batch(actions, results); + } + + @Override + public Object[] batch(List actions) throws IOException, + InterruptedException { + return table.batch(actions); + } + + @Override + public Result get(Get get) throws IOException { + return table.get(get); + } + + @Override + public Result[] get(List gets) throws IOException { + return table.get(gets); + } + + @Override + @SuppressWarnings("deprecation") + public Result getRowOrBefore(byte[] row, byte[] family) throws IOException { + return table.getRowOrBefore(row, family); + } + + @Override + public ResultScanner getScanner(Scan scan) throws IOException { + return table.getScanner(scan); + } + + @Override + public ResultScanner getScanner(byte[] family) throws IOException { + return table.getScanner(family); + } + + @Override + public ResultScanner getScanner(byte[] family, byte[] qualifier) + throws IOException { + return table.getScanner(family, qualifier); + } + + @Override + public void put(Put put) throws IOException { + table.put(put); + } + + @Override + public void put(List puts) throws IOException { + table.put(puts); + } + + @Override + public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, + byte[] value, Put put) throws IOException { + return table.checkAndPut(row, family, qualifier, value, put); + } + + @Override + public void delete(Delete delete) throws IOException { + table.delete(delete); + } + + @Override + public void delete(List deletes) throws IOException { + table.delete(deletes); + } + + @Override + public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, + byte[] value, Delete delete) throws IOException { + return table.checkAndDelete(row, family, qualifier, value, delete); + } + + @Override + public Result increment(Increment increment) throws IOException { + return table.increment(increment); + } + + @Override + public long incrementColumnValue(byte[] row, byte[] family, + byte[] qualifier, long amount) throws IOException { + return table.incrementColumnValue(row, family, qualifier, amount); + } + + @Override + public long incrementColumnValue(byte[] row, byte[] family, + byte[] qualifier, long amount, boolean writeToWAL) throws IOException { + return table.incrementColumnValue(row, family, qualifier, amount, + writeToWAL); + } + + @Override + public boolean isAutoFlush() { + return table.isAutoFlush(); + } + + @Override + public void flushCommits() throws IOException { + table.flushCommits(); + } + + /** + * Returns the actual table back to the pool + * + * @throws IOException + */ + public void close() throws IOException { + returnTable(table); + } + + /** + * @deprecated {@link RowLock} and associated operations are deprecated + */ + @Override + public RowLock lockRow(byte[] row) throws IOException { + return table.lockRow(row); + } + + /** + * @deprecated {@link RowLock} and associated operations are deprecated + */ + @Override + public void unlockRow(RowLock rl) throws IOException { + table.unlockRow(rl); + } + + @Override + public T coprocessorProxy( + Class protocol, byte[] row) { + return table.coprocessorProxy(protocol, row); + } + + @Override + public Map coprocessorExec( + Class protocol, byte[] startKey, byte[] endKey, + Batch.Call callable) throws IOException, Throwable { + return table.coprocessorExec(protocol, startKey, endKey, callable); + } + + @Override + public void coprocessorExec( + Class protocol, byte[] startKey, byte[] endKey, + Batch.Call callable, Batch.Callback callback) + throws IOException, Throwable { + table.coprocessorExec(protocol, startKey, endKey, callable, callback); + } + + @Override + public String toString() { + return "PooledHTable{" + ", table=" + table + '}'; + } + + /** + * Expose the wrapped HTable to tests in the same package + * + * @return wrapped htable + */ + HTableInterface getWrappedTable() { + return table; + } + + @Override + public void mutateRow(RowMutations rm) throws IOException { + table.mutateRow(rm); + } + + @Override + public Result append(Append append) throws IOException { + return table.append(append); + } + + @Override + public void setAutoFlush(boolean autoFlush) { + table.setAutoFlush(autoFlush); + } + + @Override + public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) { + table.setAutoFlush(autoFlush, clearBufferOnFail); + } + + @Override + public long getWriteBufferSize() { + return table.getWriteBufferSize(); + } + + @Override + public void setWriteBufferSize(long writeBufferSize) throws IOException { + table.setWriteBufferSize(writeBufferSize); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableUtil.java b/src/main/java/org/apache/hadoop/hbase/client/HTableUtil.java new file mode 100644 index 0000000..bc0872a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/HTableUtil.java @@ -0,0 +1,137 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.lang.InterruptedException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Row; + +/** + * Utility class for HTable. + * + * + */ +public class HTableUtil { + + private static final int INITIAL_LIST_SIZE = 250; + + /** + * Processes a List of Puts and writes them to an HTable instance in RegionServer buckets via the htable.put method. + * This will utilize the writeBuffer, thus the writeBuffer flush frequency may be tuned accordingly via htable.setWriteBufferSize. + *

+ * The benefit of submitting Puts in this manner is to minimize the number of RegionServer RPCs in each flush. + *

+ * Assumption #1: Regions have been pre-created for the table. If they haven't, then all of the Puts will go to the same region, + * defeating the purpose of this utility method. See the Apache HBase book for an explanation of how to do this. + *
+ * Assumption #2: Row-keys are not monotonically increasing. See the Apache HBase book for an explanation of this problem. + *
+ * Assumption #3: That the input list of Puts is big enough to be useful (in the thousands or more). The intent of this + * method is to process larger chunks of data. + *
+ * Assumption #4: htable.setAutoFlush(false) has been set. This is a requirement to use the writeBuffer. + *

+ * @param htable HTable instance for target HBase table + * @param puts List of Put instances + * @throws IOException if a remote or network exception occurs + * + */ + public static void bucketRsPut(HTable htable, List puts) throws IOException { + + Map> putMap = createRsPutMap(htable, puts); + for (List rsPuts: putMap.values()) { + htable.put( rsPuts ); + } + htable.flushCommits(); + } + + /** + * Processes a List of Rows (Put, Delete) and writes them to an HTable instance in RegionServer buckets via the htable.batch method. + *

+ * The benefit of submitting Puts in this manner is to minimize the number of RegionServer RPCs, thus this will + * produce one RPC of Puts per RegionServer. + *

+ * Assumption #1: Regions have been pre-created for the table. If they haven't, then all of the Puts will go to the same region, + * defeating the purpose of this utility method. See the Apache HBase book for an explanation of how to do this. + *
+ * Assumption #2: Row-keys are not monotonically increasing. See the Apache HBase book for an explanation of this problem. + *
+ * Assumption #3: That the input list of Rows is big enough to be useful (in the thousands or more). The intent of this + * method is to process larger chunks of data. + *

+ * This method accepts a list of Row objects because the underlying .batch method accepts a list of Row objects. + *

+ * @param htable HTable instance for target HBase table + * @param rows List of Row instances + * @throws IOException if a remote or network exception occurs + */ + public static void bucketRsBatch(HTable htable, List rows) throws IOException { + + try { + Map> rowMap = createRsRowMap(htable, rows); + for (List rsRows: rowMap.values()) { + htable.batch( rsRows ); + } + } catch (InterruptedException e) { + throw new IOException(e); + } + + } + + private static Map> createRsPutMap(HTable htable, List puts) throws IOException { + + Map> putMap = new HashMap>(); + for (Put put: puts) { + HRegionLocation rl = htable.getRegionLocation( put.getRow() ); + String hostname = rl.getHostname(); + List recs = putMap.get( hostname); + if (recs == null) { + recs = new ArrayList(INITIAL_LIST_SIZE); + putMap.put( hostname, recs); + } + recs.add(put); + } + return putMap; + } + + private static Map> createRsRowMap(HTable htable, List rows) throws IOException { + + Map> rowMap = new HashMap>(); + for (Row row: rows) { + HRegionLocation rl = htable.getRegionLocation( row.getRow() ); + String hostname = rl.getHostname(); + List recs = rowMap.get( hostname); + if (recs == null) { + recs = new ArrayList(INITIAL_LIST_SIZE); + rowMap.put( hostname, recs); + } + recs.add(row); + } + return rowMap; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/Increment.java b/src/main/java/org/apache/hadoop/hbase/client/Increment.java new file mode 100644 index 0000000..52fac04 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/Increment.java @@ -0,0 +1,340 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.hadoop.hbase.io.TimeRange; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Writable; + +/** + * Used to perform Increment operations on a single row. + *

+ * This operation does not appear atomic to readers. Increments are done + * under a single row lock, so write operations to a row are synchronized, but + * readers do not take row locks so get and scan operations can see this + * operation partially completed. + *

+ * To increment columns of a row, instantiate an Increment object with the row + * to increment. At least one column to increment must be specified using the + * {@link #addColumn(byte[], byte[], long)} method. + */ +public class Increment implements Row { + private static final byte INCREMENT_VERSION = (byte)2; + + private byte [] row = null; + private long lockId = -1L; + private boolean writeToWAL = true; + private TimeRange tr = new TimeRange(); + private Map> familyMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); + + /** Constructor for Writable. DO NOT USE */ + public Increment() {} + + /** + * Create a Increment operation for the specified row. + *

+ * At least one column must be incremented. + * @param row row key + */ + public Increment(byte [] row) { + this(row, null); + } + + /** + * Create a Increment operation for the specified row, using an existing row + * lock. + *

+ * At least one column must be incremented. + * @param row row key + * @param rowLock previously acquired row lock, or null + * @deprecated {@link RowLock} and associated operations are deprecated, + * use {@link #Increment(byte[])} + */ + public Increment(byte [] row, RowLock rowLock) { + this.row = row; + if(rowLock != null) { + this.lockId = rowLock.getLockId(); + } + } + + /** + * Increment the column from the specific family with the specified qualifier + * by the specified amount. + *

+ * Overrides previous calls to addColumn for this family and qualifier. + * @param family family name + * @param qualifier column qualifier + * @param amount amount to increment by + * @return the Increment object + */ + public Increment addColumn(byte [] family, byte [] qualifier, long amount) { + NavigableMap set = familyMap.get(family); + if(set == null) { + set = new TreeMap(Bytes.BYTES_COMPARATOR); + } + set.put(qualifier, amount); + familyMap.put(family, set); + return this; + } + + /* Accessors */ + + /** + * Method for retrieving the increment's row + * @return row + */ + public byte [] getRow() { + return this.row; + } + + /** + * Method for retrieving the increment's RowLock + * @return RowLock + * @deprecated {@link RowLock} and associated operations are deprecated + */ + public RowLock getRowLock() { + return new RowLock(this.row, this.lockId); + } + + /** + * Method for retrieving the increment's lockId + * @return lockId + * @deprecated {@link RowLock} and associated operations are deprecated + */ + public long getLockId() { + return this.lockId; + } + + /** + * Method for retrieving whether WAL will be written to or not + * @return true if WAL should be used, false if not + */ + public boolean getWriteToWAL() { + return this.writeToWAL; + } + + /** + * Sets whether this operation should write to the WAL or not. + * @param writeToWAL true if WAL should be used, false if not + * @return this increment operation + */ + public Increment setWriteToWAL(boolean writeToWAL) { + this.writeToWAL = writeToWAL; + return this; + } + + /** + * Gets the TimeRange used for this increment. + * @return TimeRange + */ + public TimeRange getTimeRange() { + return this.tr; + } + + /** + * Sets the TimeRange to be used on the Get for this increment. + *

+ * This is useful for when you have counters that only last for specific + * periods of time (ie. counters that are partitioned by time). By setting + * the range of valid times for this increment, you can potentially gain + * some performance with a more optimal Get operation. + *

+ * This range is used as [minStamp, maxStamp). + * @param minStamp minimum timestamp value, inclusive + * @param maxStamp maximum timestamp value, exclusive + * @throws IOException if invalid time range + * @return this + */ + public Increment setTimeRange(long minStamp, long maxStamp) + throws IOException { + tr = new TimeRange(minStamp, maxStamp); + return this; + } + + /** + * Method for retrieving the keys in the familyMap + * @return keys in the current familyMap + */ + public Set familySet() { + return this.familyMap.keySet(); + } + + /** + * Method for retrieving the number of families to increment from + * @return number of families + */ + public int numFamilies() { + return this.familyMap.size(); + } + + /** + * Method for retrieving the number of columns to increment + * @return number of columns across all families + */ + public int numColumns() { + if (!hasFamilies()) return 0; + int num = 0; + for (NavigableMap family : familyMap.values()) { + num += family.size(); + } + return num; + } + + /** + * Method for checking if any families have been inserted into this Increment + * @return true if familyMap is non empty false otherwise + */ + public boolean hasFamilies() { + return !this.familyMap.isEmpty(); + } + + /** + * Method for retrieving the increment's familyMap + * @return familyMap + */ + public Map> getFamilyMap() { + return this.familyMap; + } + + /** + * @return String + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("row="); + sb.append(Bytes.toStringBinary(this.row)); + if(this.familyMap.size() == 0) { + sb.append(", no columns set to be incremented"); + return sb.toString(); + } + sb.append(", families="); + boolean moreThanOne = false; + for(Map.Entry> entry : + this.familyMap.entrySet()) { + if(moreThanOne) { + sb.append("), "); + } else { + moreThanOne = true; + sb.append("{"); + } + sb.append("(family="); + sb.append(Bytes.toString(entry.getKey())); + sb.append(", columns="); + if(entry.getValue() == null) { + sb.append("NONE"); + } else { + sb.append("{"); + boolean moreThanOneB = false; + for(Map.Entry column : entry.getValue().entrySet()) { + if(moreThanOneB) { + sb.append(", "); + } else { + moreThanOneB = true; + } + sb.append(Bytes.toStringBinary(column.getKey()) + "+=" + column.getValue()); + } + sb.append("}"); + } + } + sb.append("}"); + return sb.toString(); + } + + //Writable + public void readFields(final DataInput in) + throws IOException { + int version = in.readByte(); + if (version > INCREMENT_VERSION) { + throw new IOException("unsupported version"); + } + this.row = Bytes.readByteArray(in); + this.tr = new TimeRange(); + tr.readFields(in); + this.lockId = in.readLong(); + int numFamilies = in.readInt(); + if (numFamilies == 0) { + throw new IOException("At least one column required"); + } + this.familyMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); + for(int i=0; i set = null; + if(hasColumns) { + int numColumns = in.readInt(); + set = new TreeMap(Bytes.BYTES_COMPARATOR); + for(int j=0; j 1) { + this.writeToWAL = in.readBoolean(); + } + } + + public void write(final DataOutput out) + throws IOException { + out.writeByte(INCREMENT_VERSION); + Bytes.writeByteArray(out, this.row); + tr.write(out); + out.writeLong(this.lockId); + if (familyMap.size() == 0) { + throw new IOException("At least one column required"); + } + out.writeInt(familyMap.size()); + for(Map.Entry> entry : + familyMap.entrySet()) { + Bytes.writeByteArray(out, entry.getKey()); + NavigableMap columnSet = entry.getValue(); + if(columnSet == null) { + throw new IOException("At least one column required per family"); + } else { + out.writeBoolean(true); + out.writeInt(columnSet.size()); + for(Map.Entry qualifier : columnSet.entrySet()) { + Bytes.writeByteArray(out, qualifier.getKey()); + out.writeLong(qualifier.getValue()); + } + } + } + out.writeBoolean(writeToWAL); + } + + @Override + public int compareTo(Row i) { + return Bytes.compareTo(this.getRow(), i.getRow()); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/IsolationLevel.java b/src/main/java/org/apache/hadoop/hbase/client/IsolationLevel.java new file mode 100644 index 0000000..2631b46 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/IsolationLevel.java @@ -0,0 +1,54 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +/** + * Specify Isolation levels in Scan operations. + *

+ * There are two isolation levels. A READ_COMMITTED isolation level + * indicates that only data that is committed be returned in a scan. + * An isolation level of READ_UNCOMMITTED indicates that a scan + * should return data that is being modified by transactions that might + * not have been committed yet. + */ +public enum IsolationLevel { + + READ_COMMITTED(1), + READ_UNCOMMITTED(2); + + IsolationLevel(int value) {} + + public byte [] toBytes() { + return new byte [] { toByte() }; + } + + public byte toByte() { + return (byte)this.ordinal(); + } + + public static IsolationLevel fromBytes(byte [] bytes) { + return IsolationLevel.fromByte(bytes[0]); + } + + public static IsolationLevel fromByte(byte vbyte) { + return IsolationLevel.values()[vbyte]; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java new file mode 100644 index 0000000..360de34 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java @@ -0,0 +1,527 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable; +import org.apache.hadoop.hbase.errorhandling.TimeoutException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; + +/** + * Scanner class that contains the .META. table scanning logic + * and uses a Retryable scanner. Provided visitors will be called + * for each row. + * + * Although public visibility, this is not a public-facing API and may evolve in + * minor releases. + * + *

Note that during concurrent region splits, the scanner might not see + * META changes across rows (for parent and daughter entries) consistently. + * see HBASE-5986, and {@link BlockingMetaScannerVisitor} for details.

+ */ +public class MetaScanner { + private static final Log LOG = LogFactory.getLog(MetaScanner.class); + /** + * Scans the meta table and calls a visitor on each RowResult and uses a empty + * start row value as table name. + * + * @param configuration conf + * @param visitor A custom visitor + * @throws IOException e + */ + public static void metaScan(Configuration configuration, + MetaScannerVisitor visitor) + throws IOException { + metaScan(configuration, visitor, null); + } + + /** + * Scans the meta table and calls a visitor on each RowResult. Uses a table + * name to locate meta regions. + * + * @param configuration config + * @param visitor visitor object + * @param userTableName User table name in meta table to start scan at. Pass + * null if not interested in a particular table. + * @throws IOException e + */ + public static void metaScan(Configuration configuration, + MetaScannerVisitor visitor, byte [] userTableName) + throws IOException { + metaScan(configuration, visitor, userTableName, null, Integer.MAX_VALUE); + } + + /** + * Scans the meta table and calls a visitor on each RowResult. Uses a table + * name and a row name to locate meta regions. And it only scans at most + * rowLimit of rows. + * + * @param configuration HBase configuration. + * @param visitor Visitor object. + * @param userTableName User table name in meta table to start scan at. Pass + * null if not interested in a particular table. + * @param row Name of the row at the user table. The scan will start from + * the region row where the row resides. + * @param rowLimit Max of processed rows. If it is less than 0, it + * will be set to default value Integer.MAX_VALUE. + * @throws IOException e + */ + public static void metaScan(Configuration configuration, + MetaScannerVisitor visitor, byte [] userTableName, byte[] row, + int rowLimit) + throws IOException { + metaScan(configuration, visitor, userTableName, row, rowLimit, + HConstants.META_TABLE_NAME); + } + + /** + * Scans the meta table and calls a visitor on each RowResult. Uses a table + * name and a row name to locate meta regions. And it only scans at most + * rowLimit of rows. + * + * @param configuration HBase configuration. + * @param visitor Visitor object. Closes the visitor before returning. + * @param tableName User table name in meta table to start scan at. Pass + * null if not interested in a particular table. + * @param row Name of the row at the user table. The scan will start from + * the region row where the row resides. + * @param rowLimit Max of processed rows. If it is less than 0, it + * will be set to default value Integer.MAX_VALUE. + * @param metaTableName Meta table to scan, root or meta. + * @throws IOException e + */ + public static void metaScan(Configuration configuration, + final MetaScannerVisitor visitor, final byte[] tableName, + final byte[] row, final int rowLimit, final byte[] metaTableName) + throws IOException { + try { + HConnectionManager.execute(new HConnectable(configuration) { + @Override + public Void connect(HConnection connection) throws IOException { + metaScan(conf, connection, visitor, tableName, row, rowLimit, + metaTableName); + return null; + } + }); + } finally { + visitor.close(); + } + } + + private static void metaScan(Configuration configuration, HConnection connection, + MetaScannerVisitor visitor, byte [] tableName, byte[] row, + int rowLimit, final byte [] metaTableName) + throws IOException { + int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE; + + // if row is not null, we want to use the startKey of the row's region as + // the startRow for the meta scan. + byte[] startRow; + if (row != null) { + // Scan starting at a particular row in a particular table + assert tableName != null; + byte[] searchRow = + HRegionInfo.createRegionName(tableName, row, HConstants.NINES, + false); + HTable metaTable = null; + try { + metaTable = new HTable(configuration, HConstants.META_TABLE_NAME); + Result startRowResult = metaTable.getRowOrBefore(searchRow, + HConstants.CATALOG_FAMILY); + if (startRowResult == null) { + throw new TableNotFoundException("Cannot find row in .META. for table: " + + Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow)); + } + byte[] value = startRowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + if (value == null || value.length == 0) { + throw new IOException("HRegionInfo was null or empty in Meta for " + + Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow)); + } + HRegionInfo regionInfo = Writables.getHRegionInfo(value); + + byte[] rowBefore = regionInfo.getStartKey(); + startRow = HRegionInfo.createRegionName(tableName, rowBefore, + HConstants.ZEROES, false); + } finally { + if (metaTable != null) { + metaTable.close(); + } + } + } else if (tableName == null || tableName.length == 0) { + // Full META scan + startRow = HConstants.EMPTY_START_ROW; + } else { + // Scan META for an entire table + startRow = HRegionInfo.createRegionName( + tableName, HConstants.EMPTY_START_ROW, HConstants.ZEROES, false); + } + + // Scan over each meta region + ScannerCallable callable; + int rows = Math.min(rowLimit, configuration.getInt( + HConstants.HBASE_META_SCANNER_CACHING, + HConstants.DEFAULT_HBASE_META_SCANNER_CACHING)); + do { + final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY); + if (LOG.isDebugEnabled()) { + LOG.debug("Scanning " + Bytes.toString(metaTableName) + + " starting at row=" + Bytes.toStringBinary(startRow) + " for max=" + + rowUpperLimit + " rows using " + connection.toString()); + } + callable = new ScannerCallable(connection, metaTableName, scan, null); + // Open scanner + callable.withRetries(); + + int processedRows = 0; + try { + callable.setCaching(rows); + done: do { + if (processedRows >= rowUpperLimit) { + break; + } + //we have all the rows here + Result [] rrs = callable.withRetries(); + if (rrs == null || rrs.length == 0 || rrs[0].size() == 0) { + break; //exit completely + } + for (Result rr : rrs) { + if (processedRows >= rowUpperLimit) { + break done; + } + if (!visitor.processRow(rr)) + break done; //exit completely + processedRows++; + } + //here, we didn't break anywhere. Check if we have more rows + } while(true); + // Advance the startRow to the end key of the current region + startRow = callable.getHRegionInfo().getEndKey(); + } finally { + // Close scanner + callable.setClose(); + callable.withRetries(); + } + } while (Bytes.compareTo(startRow, HConstants.LAST_ROW) != 0); + } + + /** + * Lists all of the regions currently in META. + * @param conf + * @return List of all user-space regions. + * @throws IOException + */ + public static List listAllRegions(Configuration conf) + throws IOException { + return listAllRegions(conf, true); + } + + /** + * Lists all of the regions currently in META. + * @param conf + * @param offlined True if we are to include offlined regions, false and we'll + * leave out offlined regions from returned list. + * @return List of all user-space regions. + * @throws IOException + */ + public static List listAllRegions(Configuration conf, final boolean offlined) + throws IOException { + final List regions = new ArrayList(); + MetaScannerVisitor visitor = new BlockingMetaScannerVisitor(conf) { + @Override + public boolean processRowInternal(Result result) throws IOException { + if (result == null || result.isEmpty()) { + return true; + } + byte [] bytes = result.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + if (bytes == null) { + LOG.warn("Null REGIONINFO_QUALIFIER: " + result); + return true; + } + HRegionInfo regionInfo = Writables.getHRegionInfo(bytes); + // If region offline AND we are not to include offlined regions, return. + if (regionInfo.isOffline() && !offlined) return true; + regions.add(regionInfo); + return true; + } + }; + metaScan(conf, visitor); + return regions; + } + + /** + * Lists all of the table regions currently in META. + * @param conf + * @param offlined True if we are to include offlined regions, false and we'll + * leave out offlined regions from returned list. + * @return Map of all user-space regions to servers + * @throws IOException + */ + public static NavigableMap allTableRegions(Configuration conf, + final byte [] tablename, final boolean offlined) throws IOException { + final NavigableMap regions = + new TreeMap(); + MetaScannerVisitor visitor = new TableMetaScannerVisitor(conf, tablename) { + @Override + public boolean processRowInternal(Result rowResult) throws IOException { + HRegionInfo info = Writables.getHRegionInfo( + rowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER)); + byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER); + String hostAndPort = null; + if (value != null && value.length > 0) { + hostAndPort = Bytes.toString(value); + } + value = rowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.STARTCODE_QUALIFIER); + long startcode = -1L; + if (value != null && value.length > 0) startcode = Bytes.toLong(value); + if (!(info.isOffline() || info.isSplit())) { + ServerName sn = null; + if (hostAndPort != null && hostAndPort.length() > 0) { + sn = new ServerName(hostAndPort, startcode); + } + regions.put(new UnmodifyableHRegionInfo(info), sn); + } + return true; + } + }; + metaScan(conf, visitor, tablename); + return regions; + } + + /** + * Visitor class called to process each row of the .META. table + */ + public interface MetaScannerVisitor extends Closeable { + /** + * Visitor method that accepts a RowResult and the meta region location. + * Implementations can return false to stop the region's loop if it becomes + * unnecessary for some reason. + * + * @param rowResult result + * @return A boolean to know if it should continue to loop in the region + * @throws IOException e + */ + public boolean processRow(Result rowResult) throws IOException; + } + + public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor { + @Override + public void close() throws IOException { + } + } + + /** + * A MetaScannerVisitor that provides a consistent view of the table's + * META entries during concurrent splits (see HBASE-5986 for details). This class + * does not guarantee ordered traversal of meta entries, and can block until the + * META entries for daughters are available during splits. + */ + public static abstract class BlockingMetaScannerVisitor + extends MetaScannerVisitorBase { + + private static final int DEFAULT_BLOCKING_TIMEOUT = 10000; + private Configuration conf; + private TreeSet daughterRegions = new TreeSet(Bytes.BYTES_COMPARATOR); + private int blockingTimeout; + private HTable metaTable; + + public BlockingMetaScannerVisitor(Configuration conf) { + this.conf = conf; + this.blockingTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, + DEFAULT_BLOCKING_TIMEOUT); + } + + public abstract boolean processRowInternal(Result rowResult) throws IOException; + + @Override + public void close() throws IOException { + super.close(); + if (metaTable != null) { + metaTable.close(); + metaTable = null; + } + } + + public HTable getMetaTable() throws IOException { + if (metaTable == null) { + metaTable = new HTable(conf, HConstants.META_TABLE_NAME); + } + return metaTable; + } + + @Override + public boolean processRow(Result rowResult) throws IOException { + HRegionInfo info = Writables.getHRegionInfoOrNull( + rowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER)); + if (info == null) { + return true; + } + + if (daughterRegions.remove(info.getRegionName())) { + return true; //we have already processed this row + } + + if (info.isSplitParent()) { + /* we have found a parent region which was split. We have to ensure that it's daughters are + * seen by this scanner as well, so we block until they are added to the META table. Even + * though we are waiting for META entries, ACID semantics in HBase indicates that this + * scanner might not see the new rows. So we manually query the daughter rows */ + HRegionInfo splitA = Writables.getHRegionInfoOrNull(rowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.SPLITA_QUALIFIER)); + HRegionInfo splitB = Writables.getHRegionInfoOrNull(rowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.SPLITB_QUALIFIER)); + + HTable metaTable = getMetaTable(); + long start = System.currentTimeMillis(); + if (splitA != null) { + try { + Result resultA = getRegionResultBlocking(metaTable, blockingTimeout, + info.getRegionName(), splitA.getRegionName()); + if (resultA != null) { + processRow(resultA); + daughterRegions.add(splitA.getRegionName()); + } + // else parent is gone, so skip this daughter + } catch (TimeoutException e) { + throw new RegionOfflineException("Split daughter region " + + splitA.getRegionNameAsString() + " cannot be found in META. Parent:" + + info.getRegionNameAsString()); + } + } + long rem = blockingTimeout - (System.currentTimeMillis() - start); + + if (splitB != null) { + try { + Result resultB = getRegionResultBlocking(metaTable, rem, + info.getRegionName(), splitB.getRegionName()); + if (resultB != null) { + processRow(resultB); + daughterRegions.add(splitB.getRegionName()); + } + // else parent is gone, so skip this daughter + } catch (TimeoutException e) { + throw new RegionOfflineException("Split daughter region " + + splitB.getRegionNameAsString() + " cannot be found in META. Parent:" + + info.getRegionNameAsString()); + } + } + } + + return processRowInternal(rowResult); + } + + /** + * Returns region Result by querying the META table for regionName. It will block until + * the region is found in META. It will also check for parent in META to make sure that + * if parent is deleted, we no longer have to wait, and should continue (HBASE-8590) + * @return Result object is daughter is found, or null if parent is gone from META + * @throws TimeoutException if timeout is reached + */ + private Result getRegionResultBlocking(HTable metaTable, long timeout, byte[] parentRegionName, byte[] regionName) + throws IOException, TimeoutException { + boolean logged = false; + long start = System.currentTimeMillis(); + while (System.currentTimeMillis() - start < timeout) { + Get get = new Get(regionName); + Result result = metaTable.get(get); + HRegionInfo info = Writables.getHRegionInfoOrNull( + result.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER)); + if (info != null) { + return result; + } + + // check whether parent is still there, if not it means we do not need to wait + Get parentGet = new Get(parentRegionName); + Result parentResult = metaTable.get(parentGet); + HRegionInfo parentInfo = Writables.getHRegionInfoOrNull( + parentResult.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER)); + if (parentInfo == null) { + // this means that parent is no more (catalog janitor or somebody else deleted it) + return null; + } + + try { + if (!logged) { + if (LOG.isDebugEnabled()) { + LOG.debug("blocking until region is in META: " + Bytes.toStringBinary(regionName)); + } + logged = true; + } + Thread.sleep(10); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + break; + } + } + throw new TimeoutException("getRegionResultBlocking", start, System.currentTimeMillis(), + timeout); + } + } + + /** + * A MetaScannerVisitor for a table. Provides a consistent view of the table's + * META entries during concurrent splits (see HBASE-5986 for details). This class + * does not guarantee ordered traversal of meta entries, and can block until the + * META entries for daughters are available during splits. + */ + public static abstract class TableMetaScannerVisitor extends BlockingMetaScannerVisitor { + private byte[] tableName; + + public TableMetaScannerVisitor(Configuration conf, byte[] tableName) { + super(conf); + this.tableName = tableName; + } + + @Override + public final boolean processRow(Result rowResult) throws IOException { + HRegionInfo info = Writables.getHRegionInfoOrNull( + rowResult.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER)); + if (info == null) { + return true; + } + if (!(Bytes.equals(info.getTableName(), tableName))) { + return false; + } + return super.processRow(rowResult); + } + + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java b/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java new file mode 100644 index 0000000..6a864c8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java @@ -0,0 +1,122 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.util.Bytes; + +import java.io.DataOutput; +import java.io.IOException; +import java.io.DataInput; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * Container for Actions (i.e. Get, Delete, or Put), which are grouped by + * regionName. Intended to be used with HConnectionManager.processBatch() + */ +public final class MultiAction implements Writable { + + // map of regions to lists of puts/gets/deletes for that region. + public Map>> actions = + new TreeMap>>( + Bytes.BYTES_COMPARATOR); + + public MultiAction() { + } + + /** + * Get the total number of Actions + * + * @return total number of Actions for all groups in this container. + */ + public int size() { + int size = 0; + for (List l : actions.values()) { + size += l.size(); + } + return size; + } + + /** + * Add an Action to this container based on it's regionName. If the regionName + * is wrong, the initial execution will fail, but will be automatically + * retried after looking up the correct region. + * + * @param regionName + * @param a + */ + public void add(byte[] regionName, Action a) { + List> rsActions = actions.get(regionName); + if (rsActions == null) { + rsActions = new ArrayList>(); + actions.put(regionName, rsActions); + } + rsActions.add(a); + } + + public Set getRegions() { + return actions.keySet(); + } + + /** + * @return All actions from all regions in this container + */ + public List> allActions() { + List> res = new ArrayList>(); + for (List> lst : actions.values()) { + res.addAll(lst); + } + return res; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(actions.size()); + for (Map.Entry>> e : actions.entrySet()) { + Bytes.writeByteArray(out, e.getKey()); + List> lst = e.getValue(); + out.writeInt(lst.size()); + for (Action a : lst) { + HbaseObjectWritable.writeObject(out, a, a.getClass(), null); + } + } + } + + @Override + public void readFields(DataInput in) throws IOException { + actions.clear(); + int mapSize = in.readInt(); + for (int i = 0; i < mapSize; i++) { + byte[] key = Bytes.readByteArray(in); + int listSize = in.readInt(); + List> lst = new ArrayList>(listSize); + for (int j = 0; j < listSize; j++) { + lst.add((Action) HbaseObjectWritable.readObject(in, null)); + } + actions.put(key, lst); + } + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/MultiPut.java b/src/main/java/org/apache/hadoop/hbase/client/MultiPut.java new file mode 100644 index 0000000..9235e2d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/MultiPut.java @@ -0,0 +1,236 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Writable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * @deprecated Use MultiAction instead + * Data type class for putting multiple regions worth of puts in one RPC. + */ +public class MultiPut extends Operation implements Writable { + public HServerAddress address; // client code ONLY + + // TODO make this configurable + public static final int DEFAULT_MAX_PUT_OUTPUT = 10; + + // map of regions to lists of puts for that region. + public Map > puts = new TreeMap>(Bytes.BYTES_COMPARATOR); + + /** + * Writable constructor only. + */ + public MultiPut() {} + + /** + * MultiPut for putting multiple regions worth of puts in one RPC. + * @param a address + */ + public MultiPut(HServerAddress a) { + address = a; + } + + public int size() { + int size = 0; + for( List l : puts.values()) { + size += l.size(); + } + return size; + } + + public void add(byte[] regionName, Put aPut) { + List rsput = puts.get(regionName); + if (rsput == null) { + rsput = new ArrayList(); + puts.put(regionName, rsput); + } + rsput.add(aPut); + } + + public Collection allPuts() { + List res = new ArrayList(); + for ( List pp : puts.values() ) { + res.addAll(pp); + } + return res; + } + + /** + * Compile the table and column family (i.e. schema) information + * into a String. Useful for parsing and aggregation by debugging, + * logging, and administration tools. + * @return Map + */ + @Override + public Map getFingerprint() { + Map map = new HashMap(); + // for extensibility, we have a map of table information that we will + // populate with only family information for each table + Map tableInfo = + new HashMap(); + map.put("tables", tableInfo); + for (Map.Entry> entry : puts.entrySet()) { + // our fingerprint only concerns itself with which families are touched, + // not how many Puts touch them, so we use this Set to do just that. + Set familySet; + try { + // since the puts are stored by region, we may have already + // recorded families for this region. if that is the case, + // we want to add to the existing Set. if not, we make a new Set. + String tableName = Bytes.toStringBinary( + HRegionInfo.parseRegionName(entry.getKey())[0]); + if (tableInfo.get(tableName) == null) { + Map table = new HashMap(); + familySet = new TreeSet(); + table.put("families", familySet); + tableInfo.put(tableName, table); + } else { + familySet = (Set) tableInfo.get(tableName).get("families"); + } + } catch (IOException ioe) { + // in the case of parse error, default to labeling by region + Map table = new HashMap(); + familySet = new TreeSet(); + table.put("families", familySet); + tableInfo.put(Bytes.toStringBinary(entry.getKey()), table); + } + // we now iterate through each Put and keep track of which families + // are affected in this table. + for (Put p : entry.getValue()) { + for (byte[] fam : p.getFamilyMap().keySet()) { + familySet.add(Bytes.toStringBinary(fam)); + } + } + } + return map; + } + + /** + * Compile the details beyond the scope of getFingerprint (mostly + * toMap from the Puts) into a Map along with the fingerprinted + * information. Useful for debugging, logging, and administration tools. + * @param maxCols a limit on the number of columns output prior to truncation + * @return Map + */ + @Override + public Map toMap(int maxCols) { + Map map = getFingerprint(); + Map tableInfo = (Map) map.get("tables"); + int putCount = 0; + for (Map.Entry> entry : puts.entrySet()) { + // If the limit has been hit for put output, just adjust our counter + if (putCount >= DEFAULT_MAX_PUT_OUTPUT) { + putCount += entry.getValue().size(); + continue; + } + List regionPuts = entry.getValue(); + List> putSummaries = + new ArrayList>(); + // find out how many of this region's puts we can add without busting + // the maximum + int regionPutsToAdd = regionPuts.size(); + putCount += regionPutsToAdd; + if (putCount > DEFAULT_MAX_PUT_OUTPUT) { + regionPutsToAdd -= putCount - DEFAULT_MAX_PUT_OUTPUT; + } + for (Iterator iter = regionPuts.iterator(); regionPutsToAdd-- > 0;) { + putSummaries.add(iter.next().toMap(maxCols)); + } + // attempt to extract the table name from the region name + String tableName = ""; + try { + tableName = Bytes.toStringBinary( + HRegionInfo.parseRegionName(entry.getKey())[0]); + } catch (IOException ioe) { + // in the case of parse error, default to labeling by region + tableName = Bytes.toStringBinary(entry.getKey()); + } + // since the puts are stored by region, we may have already + // recorded puts for this table. if that is the case, + // we want to add to the existing List. if not, we place a new list + // in the map + Map table = + (Map) tableInfo.get(tableName); + if (table == null) { + // in case the Put has changed since getFingerprint's map was built + table = new HashMap(); + tableInfo.put(tableName, table); + table.put("puts", putSummaries); + } else if (table.get("puts") == null) { + table.put("puts", putSummaries); + } else { + ((List>) table.get("puts")).addAll(putSummaries); + } + } + map.put("totalPuts", putCount); + return map; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(puts.size()); + for( Map.Entry> e : puts.entrySet()) { + Bytes.writeByteArray(out, e.getKey()); + + List ps = e.getValue(); + out.writeInt(ps.size()); + for( Put p : ps ) { + p.write(out); + } + } + } + + @Override + public void readFields(DataInput in) throws IOException { + puts.clear(); + + int mapSize = in.readInt(); + + for (int i = 0 ; i < mapSize; i++) { + byte[] key = Bytes.readByteArray(in); + + int listSize = in.readInt(); + List ps = new ArrayList(listSize); + for ( int j = 0 ; j < listSize; j++ ) { + Put put = new Put(); + put.readFields(in); + ps.add(put); + } + puts.put(key, ps); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/MultiPutResponse.java b/src/main/java/org/apache/hadoop/hbase/client/MultiPutResponse.java new file mode 100644 index 0000000..7e0311a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/MultiPutResponse.java @@ -0,0 +1,73 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Writable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Map; +import java.util.TreeMap; + +/** + * @deprecated Replaced by MultiResponse + * Response class for MultiPut. + */ +public class MultiPutResponse implements Writable { + + protected MultiPut request; // used in client code ONLY + + protected Map answers = new TreeMap(Bytes.BYTES_COMPARATOR); + + public MultiPutResponse() {} + + public void addResult(byte[] regionName, int result) { + answers.put(regionName, result); + } + + public Integer getAnswer(byte[] region) { + return answers.get(region); + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(answers.size()); + for( Map.Entry e : answers.entrySet()) { + Bytes.writeByteArray(out, e.getKey()); + out.writeInt(e.getValue()); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + answers.clear(); + + int mapSize = in.readInt(); + for( int i = 0 ; i < mapSize ; i++ ) { + byte[] key = Bytes.readByteArray(in); + int value = in.readInt(); + + answers.put(key, value); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/MultiResponse.java b/src/main/java/org/apache/hadoop/hbase/client/MultiResponse.java new file mode 100644 index 0000000..290e4c7 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/MultiResponse.java @@ -0,0 +1,167 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.util.StringUtils; + +import java.io.DataOutput; +import java.io.IOException; +import java.io.DataInput; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.ArrayList; +import java.util.TreeMap; + +/** + * A container for Result objects, grouped by regionName. + */ +public class MultiResponse implements Writable { + + // map of regionName to list of (Results paired to the original index for that + // Result) + private Map>> results = + new TreeMap>>(Bytes.BYTES_COMPARATOR); + + public MultiResponse() { + } + + /** + * @return Number of pairs in this container + */ + public int size() { + int size = 0; + for (Collection c : results.values()) { + size += c.size(); + } + return size; + } + + /** + * Add the pair to the container, grouped by the regionName + * + * @param regionName + * @param r + * First item in the pair is the original index of the Action + * (request). Second item is the Result. Result will be empty for + * successful Put and Delete actions. + */ + public void add(byte[] regionName, Pair r) { + List> rs = results.get(regionName); + if (rs == null) { + rs = new ArrayList>(); + results.put(regionName, rs); + } + rs.add(r); + } + + public void add(byte []regionName, int originalIndex, Object resOrEx) { + add(regionName, new Pair(originalIndex, resOrEx)); + } + + public Map>> getResults() { + return results; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(results.size()); + for (Map.Entry>> e : results.entrySet()) { + Bytes.writeByteArray(out, e.getKey()); + List> lst = e.getValue(); + out.writeInt(lst.size()); + for (Pair r : lst) { + if (r == null) { + out.writeInt(-1); // Cant have index -1; on other side we recognize -1 as 'null' + } else { + out.writeInt(r.getFirst()); // Can this can npe!?! + Object obj = r.getSecond(); + if (obj instanceof Throwable) { + out.writeBoolean(true); // true, Throwable/exception. + + Throwable t = (Throwable) obj; + // serialize exception + WritableUtils.writeString(out, t.getClass().getName()); + WritableUtils.writeString(out, + StringUtils.stringifyException(t)); + + } else { + out.writeBoolean(false); // no exception + + if (! (obj instanceof Writable)) + obj = null; // squash all non-writables to null. + HbaseObjectWritable.writeObject(out, r.getSecond(), + obj != null ? obj.getClass() : Writable.class, null); + } + } + } + } + } + + @Override + public void readFields(DataInput in) throws IOException { + results.clear(); + int mapSize = in.readInt(); + for (int i = 0; i < mapSize; i++) { + byte[] key = Bytes.readByteArray(in); + int listSize = in.readInt(); + List> lst = new ArrayList>( + listSize); + for (int j = 0; j < listSize; j++) { + Integer idx = in.readInt(); + if (idx == -1) { + lst.add(null); + } else { + boolean isException = in.readBoolean(); + Object o = null; + if (isException) { + String klass = WritableUtils.readString(in); + String desc = WritableUtils.readString(in); + try { + // the type-unsafe insertion, but since we control what klass is.. + Class c = (Class) Class.forName(klass); + Constructor cn = c.getDeclaredConstructor(String.class); + o = cn.newInstance(desc); + } catch (ClassNotFoundException ignored) { + } catch (NoSuchMethodException ignored) { + } catch (InvocationTargetException ignored) { + } catch (InstantiationException ignored) { + } catch (IllegalAccessException ignored) { + } + } else { + o = HbaseObjectWritable.readObject(in, null); + } + lst.add(new Pair(idx, o)); + } + } + results.put(key, lst); + } + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java new file mode 100644 index 0000000..730dee5 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/Mutation.java @@ -0,0 +1,262 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + +public abstract class Mutation extends OperationWithAttributes implements Row { + private static final Log LOG = LogFactory.getLog(Mutation.class); + // Attribute used in Mutations to indicate the originating cluster. + private static final String CLUSTER_ID_ATTR = "_c.id_"; + private static final String DURABILITY_ID_ATTR = "_dur_"; + + protected byte [] row = null; + protected long ts = HConstants.LATEST_TIMESTAMP; + protected long lockId = -1L; + protected boolean writeToWAL = true; + protected Map> familyMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); + + /** + * Compile the column family (i.e. schema) information + * into a Map. Useful for parsing and aggregation by debugging, + * logging, and administration tools. + * @return Map + */ + @Override + public Map getFingerprint() { + Map map = new HashMap(); + List families = new ArrayList(); + // ideally, we would also include table information, but that information + // is not stored in each Operation instance. + map.put("families", families); + for (Map.Entry> entry : this.familyMap.entrySet()) { + families.add(Bytes.toStringBinary(entry.getKey())); + } + return map; + } + + /** + * Compile the details beyond the scope of getFingerprint (row, columns, + * timestamps, etc.) into a Map along with the fingerprinted information. + * Useful for debugging, logging, and administration tools. + * @param maxCols a limit on the number of columns output prior to truncation + * @return Map + */ + @Override + public Map toMap(int maxCols) { + // we start with the fingerprint map and build on top of it. + Map map = getFingerprint(); + // replace the fingerprint's simple list of families with a + // map from column families to lists of qualifiers and kv details + Map>> columns = + new HashMap>>(); + map.put("families", columns); + map.put("row", Bytes.toStringBinary(this.row)); + int colCount = 0; + // iterate through all column families affected + for (Map.Entry> entry : this.familyMap.entrySet()) { + // map from this family to details for each kv affected within the family + List> qualifierDetails = + new ArrayList>(); + columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails); + colCount += entry.getValue().size(); + if (maxCols <= 0) { + continue; + } + // add details for each kv + for (KeyValue kv : entry.getValue()) { + if (--maxCols <= 0 ) { + continue; + } + Map kvMap = kv.toStringMap(); + // row and family information are already available in the bigger map + kvMap.remove("row"); + kvMap.remove("family"); + qualifierDetails.add(kvMap); + } + } + map.put("totalColumns", colCount); + // add the id if set + if (getId() != null) { + map.put("id", getId()); + } + return map; + } + + /** + * @deprecated Use {@link #getDurability()} instead. + * @return true if edits should be applied to WAL, false if not + */ + public boolean getWriteToWAL() { + return this.writeToWAL; + } + + /** + * Set whether this Delete should be written to the WAL or not. + * Not writing the WAL means you may lose edits on server crash. + * This method will reset any changes made via {@link #setDurability(Durability)} + * @param write true if edits should be written to WAL, false if not + * @deprecated Use {@link #setDurability(Durability)} instead. + */ + public void setWriteToWAL(boolean write) { + setDurability(write ? Durability.USE_DEFAULT : Durability.SKIP_WAL); + } + + /** + * Set the durability for this mutation. + * Note that RegionServers prior to 0.94.7 will only honor {@link Durability#SKIP_WAL}. + * This method will reset any changes made via {@link #setWriteToWAL(boolean)} + * @param d + */ + public void setDurability(Durability d) { + setAttribute(DURABILITY_ID_ATTR, Bytes.toBytes(d.ordinal())); + this.writeToWAL = d != Durability.SKIP_WAL; + } + + /** Get the current durability */ + public Durability getDurability() { + byte[] attr = getAttribute(DURABILITY_ID_ATTR); + if (attr != null) { + try { + return Durability.valueOf(Bytes.toInt(attr)); + } catch (IllegalArgumentException iax) { + LOG.warn("Invalid or unknown durability settting", iax); + } + } + return writeToWAL ? Durability.USE_DEFAULT : Durability.SKIP_WAL; + } + + /** + * Method for retrieving the put's familyMap + * @return familyMap + */ + public Map> getFamilyMap() { + return this.familyMap; + } + + /** + * Method for setting the put's familyMap + */ + public void setFamilyMap(Map> map) { + this.familyMap = map; + } + + /** + * Method to check if the familyMap is empty + * @return true if empty, false otherwise + */ + public boolean isEmpty() { + return familyMap.isEmpty(); + } + + /** + * Method for retrieving the delete's row + * @return row + */ + @Override + public byte [] getRow() { + return this.row; + } + + public int compareTo(final Row d) { + return Bytes.compareTo(this.getRow(), d.getRow()); + } + + /** + * Method for retrieving the delete's RowLock + * @return RowLock + * @deprecated {@link RowLock} and associated operations are deprecated + */ + public RowLock getRowLock() { + return new RowLock(this.row, this.lockId); + } + + /** + * Method for retrieving the delete's lock ID. + * + * @return The lock ID. + * @deprecated {@link RowLock} and associated operations are deprecated + */ + public long getLockId() { + return this.lockId; + } + + /** + * Method for retrieving the timestamp + * @return timestamp + */ + public long getTimeStamp() { + return this.ts; + } + + /** + * Set the replication custer id. + * @param clusterId + */ + public void setClusterId(UUID clusterId) { + if (clusterId == null) return; + byte[] val = new byte[2*Bytes.SIZEOF_LONG]; + Bytes.putLong(val, 0, clusterId.getMostSignificantBits()); + Bytes.putLong(val, Bytes.SIZEOF_LONG, clusterId.getLeastSignificantBits()); + setAttribute(CLUSTER_ID_ATTR, val); + } + + /** + * @return The replication cluster id. + */ + public UUID getClusterId() { + byte[] attr = getAttribute(CLUSTER_ID_ATTR); + if (attr == null) { + return HConstants.DEFAULT_CLUSTER_ID; + } + return new UUID(Bytes.toLong(attr,0), Bytes.toLong(attr, Bytes.SIZEOF_LONG)); + } + + /** + * @return the total number of KeyValues + */ + public int size() { + int size = 0; + for(List kvList : this.familyMap.values()) { + size += kvList.size(); + } + return size; + } + + /** + * @return the number of different families + */ + public int numFamilies() { + return familyMap.size(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/NoServerForRegionException.java b/src/main/java/org/apache/hadoop/hbase/client/NoServerForRegionException.java new file mode 100644 index 0000000..4f33914 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/NoServerForRegionException.java @@ -0,0 +1,42 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.RegionException; + +/** + * Thrown when no region server can be found for a region + */ +public class NoServerForRegionException extends RegionException { + private static final long serialVersionUID = 1L << 11 - 1L; + + /** default constructor */ + public NoServerForRegionException() { + super(); + } + + /** + * Constructor + * @param s message + */ + public NoServerForRegionException(String s) { + super(s); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/Operation.java b/src/main/java/org/apache/hadoop/hbase/client/Operation.java new file mode 100644 index 0000000..dedd2e2 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/Operation.java @@ -0,0 +1,110 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.util.Map; + +import org.codehaus.jackson.map.ObjectMapper; + +/** + * Superclass for any type that maps to a potentially application-level query. + * (e.g. Put, Get, Delete, Scan, Next, etc.) + * Contains methods for exposure to logging and debugging tools. + */ +public abstract class Operation { + // TODO make this configurable + private static final int DEFAULT_MAX_COLS = 5; + + /** + * Produces a Map containing a fingerprint which identifies the type and + * the static schema components of a query (i.e. column families) + * @return a map containing fingerprint information (i.e. column families) + */ + public abstract Map getFingerprint(); + + /** + * Produces a Map containing a summary of the details of a query + * beyond the scope of the fingerprint (i.e. columns, rows...) + * @param maxCols a limit on the number of columns output prior to truncation + * @return a map containing parameters of a query (i.e. rows, columns...) + */ + public abstract Map toMap(int maxCols); + + /** + * Produces a Map containing a full summary of a query. + * @return a map containing parameters of a query (i.e. rows, columns...) + */ + public Map toMap() { + return toMap(DEFAULT_MAX_COLS); + } + + /** + * Produces a JSON object for fingerprint and details exposure in a + * parseable format. + * @param maxCols a limit on the number of columns to include in the JSON + * @return a JSONObject containing this Operation's information, as a string + */ + public String toJSON(int maxCols) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(toMap(maxCols)); + } + + /** + * Produces a JSON object sufficient for description of a query + * in a debugging or logging context. + * @return the produced JSON object, as a string + */ + public String toJSON() throws IOException { + return toJSON(DEFAULT_MAX_COLS); + } + + /** + * Produces a string representation of this Operation. It defaults to a JSON + * representation, but falls back to a string representation of the + * fingerprint and details in the case of a JSON encoding failure. + * @param maxCols a limit on the number of columns output in the summary + * prior to truncation + * @return a JSON-parseable String + */ + public String toString(int maxCols) { + /* for now this is merely a wrapper from producing a JSON string, but + * toJSON is kept separate in case this is changed to be a less parsable + * pretty printed representation. + */ + try { + return toJSON(maxCols); + } catch (IOException ioe) { + return toMap(maxCols).toString(); + } + } + + /** + * Produces a string representation of this Operation. It defaults to a JSON + * representation, but falls back to a string representation of the + * fingerprint and details in the case of a JSON encoding failure. + * @return String + */ + @Override + public String toString() { + return toString(DEFAULT_MAX_COLS); + } +} + diff --git a/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java b/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java new file mode 100644 index 0000000..97e19b9 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java @@ -0,0 +1,133 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; +import org.apache.hadoop.io.WritableUtils; + +public abstract class OperationWithAttributes extends Operation implements Attributes { + // a opaque blob of attributes + private Map attributes; + + // used for uniquely identifying an operation + static public String ID_ATRIBUTE = "_operation.attributes.id"; + + public void setAttribute(String name, byte[] value) { + if (attributes == null && value == null) { + return; + } + + if (attributes == null) { + attributes = new HashMap(); + } + + if (value == null) { + attributes.remove(name); + if (attributes.isEmpty()) { + this.attributes = null; + } + } else { + attributes.put(name, value); + } + } + + public byte[] getAttribute(String name) { + if (attributes == null) { + return null; + } + + return attributes.get(name); + } + + public Map getAttributesMap() { + if (attributes == null) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(attributes); + } + + protected long getAttributeSize() { + long size = 0; + if (attributes != null) { + size += ClassSize.align(this.attributes.size() * ClassSize.MAP_ENTRY); + for(Map.Entry entry : this.attributes.entrySet()) { + size += ClassSize.align(ClassSize.STRING + entry.getKey().length()); + size += ClassSize.align(ClassSize.ARRAY + entry.getValue().length); + } + } + return size; + } + + protected void writeAttributes(final DataOutput out) throws IOException { + if (this.attributes == null) { + out.writeInt(0); + } else { + out.writeInt(this.attributes.size()); + for (Map.Entry attr : this.attributes.entrySet()) { + WritableUtils.writeString(out, attr.getKey()); + Bytes.writeByteArray(out, attr.getValue()); + } + } + } + + protected void readAttributes(final DataInput in) throws IOException { + int numAttributes = in.readInt(); + if (numAttributes > 0) { + this.attributes = new HashMap(numAttributes); + for(int i=0; i + * To perform a Put, instantiate a Put object with the row to insert to and + * for each column to be inserted, execute {@link #add(byte[], byte[], byte[]) add} or + * {@link #add(byte[], byte[], long, byte[]) add} if setting the timestamp. + */ +public class Put extends Mutation + implements HeapSize, Writable, Comparable { + private static final byte PUT_VERSION = (byte)2; + + private static final long OVERHEAD = ClassSize.align( + ClassSize.OBJECT + 2 * ClassSize.REFERENCE + + 2 * Bytes.SIZEOF_LONG + Bytes.SIZEOF_BOOLEAN + + ClassSize.REFERENCE + ClassSize.TREEMAP); + + /** Constructor for Writable. DO NOT USE */ + public Put() {} + + /** + * Create a Put operation for the specified row. + * @param row row key + */ + public Put(byte [] row) { + this(row, null); + } + + /** + * Create a Put operation for the specified row, using an existing row lock. + * @param row row key + * @param rowLock previously acquired row lock, or null + * @deprecated {@link RowLock} and associated operations are deprecated, use {@link #Put(byte[])} + */ + public Put(byte [] row, RowLock rowLock) { + this(row, HConstants.LATEST_TIMESTAMP, rowLock); + } + + /** + * Create a Put operation for the specified row, using a given timestamp. + * + * @param row row key + * @param ts timestamp + */ + public Put(byte[] row, long ts) { + this(row, ts, null); + } + + /** + * Create a Put operation for the specified row, using a given timestamp, and an existing row lock. + * @param row row key + * @param ts timestamp + * @param rowLock previously acquired row lock, or null + * @deprecated {@link RowLock} and associated operations are deprecated, + * use {@link #Put(byte[], long)} + */ + public Put(byte [] row, long ts, RowLock rowLock) { + if(row == null || row.length > HConstants.MAX_ROW_LENGTH) { + throw new IllegalArgumentException("Row key is invalid"); + } + this.row = Arrays.copyOf(row, row.length); + this.ts = ts; + if(rowLock != null) { + this.lockId = rowLock.getLockId(); + } + } + + /** + * Copy constructor. Creates a Put operation cloned from the specified Put. + * @param putToCopy put to copy + */ + public Put(Put putToCopy) { + this(putToCopy.getRow(), putToCopy.ts, putToCopy.getRowLock()); + this.familyMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); + for(Map.Entry> entry : + putToCopy.getFamilyMap().entrySet()) { + this.familyMap.put(entry.getKey(), entry.getValue()); + } + this.writeToWAL = putToCopy.writeToWAL; + } + + /** + * Add the specified column and value to this Put operation. + * @param family family name + * @param qualifier column qualifier + * @param value column value + * @return this + */ + public Put add(byte [] family, byte [] qualifier, byte [] value) { + return add(family, qualifier, this.ts, value); + } + + /** + * Add the specified column and value, with the specified timestamp as + * its version to this Put operation. + * @param family family name + * @param qualifier column qualifier + * @param ts version timestamp + * @param value column value + * @return this + */ + public Put add(byte [] family, byte [] qualifier, long ts, byte [] value) { + List list = getKeyValueList(family); + KeyValue kv = createPutKeyValue(family, qualifier, ts, value); + list.add(kv); + familyMap.put(kv.getFamily(), list); + return this; + } + + /** + * Add the specified KeyValue to this Put operation. Operation assumes that + * the passed KeyValue is immutable and its backing array will not be modified + * for the duration of this Put. + * @param kv individual KeyValue + * @return this + * @throws java.io.IOException e + */ + public Put add(KeyValue kv) throws IOException{ + byte [] family = kv.getFamily(); + List list = getKeyValueList(family); + //Checking that the row of the kv is the same as the put + int res = Bytes.compareTo(this.row, 0, row.length, + kv.getBuffer(), kv.getRowOffset(), kv.getRowLength()); + if(res != 0) { + throw new IOException("The row in the recently added KeyValue " + + Bytes.toStringBinary(kv.getBuffer(), kv.getRowOffset(), + kv.getRowLength()) + " doesn't match the original one " + + Bytes.toStringBinary(this.row)); + } + list.add(kv); + familyMap.put(family, list); + return this; + } + + /* + * Create a KeyValue with this objects row key and the Put identifier. + * + * @return a KeyValue with this objects row key and the Put identifier. + */ + private KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, + byte[] value) { + return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, + value); + } + + /** + * A convenience method to determine if this object's familyMap contains + * a value assigned to the given family & qualifier. + * Both given arguments must match the KeyValue object to return true. + * + * @param family column family + * @param qualifier column qualifier + * @return returns true if the given family and qualifier already has an + * existing KeyValue object in the family map. + */ + public boolean has(byte [] family, byte [] qualifier) { + return has(family, qualifier, this.ts, new byte[0], true, true); + } + + /** + * A convenience method to determine if this object's familyMap contains + * a value assigned to the given family, qualifier and timestamp. + * All 3 given arguments must match the KeyValue object to return true. + * + * @param family column family + * @param qualifier column qualifier + * @param ts timestamp + * @return returns true if the given family, qualifier and timestamp already has an + * existing KeyValue object in the family map. + */ + public boolean has(byte [] family, byte [] qualifier, long ts) { + return has(family, qualifier, ts, new byte[0], false, true); + } + + /** + * A convenience method to determine if this object's familyMap contains + * a value assigned to the given family, qualifier and timestamp. + * All 3 given arguments must match the KeyValue object to return true. + * + * @param family column family + * @param qualifier column qualifier + * @param value value to check + * @return returns true if the given family, qualifier and value already has an + * existing KeyValue object in the family map. + */ + public boolean has(byte [] family, byte [] qualifier, byte [] value) { + return has(family, qualifier, this.ts, value, true, false); + } + + /** + * A convenience method to determine if this object's familyMap contains + * the given value assigned to the given family, qualifier and timestamp. + * All 4 given arguments must match the KeyValue object to return true. + * + * @param family column family + * @param qualifier column qualifier + * @param ts timestamp + * @param value value to check + * @return returns true if the given family, qualifier timestamp and value + * already has an existing KeyValue object in the family map. + */ + public boolean has(byte [] family, byte [] qualifier, long ts, byte [] value) { + return has(family, qualifier, ts, value, false, false); + } + + /* + * Private method to determine if this object's familyMap contains + * the given value assigned to the given family, qualifier and timestamp + * respecting the 2 boolean arguments + * + * @param family + * @param qualifier + * @param ts + * @param value + * @param ignoreTS + * @param ignoreValue + * @return returns true if the given family, qualifier timestamp and value + * already has an existing KeyValue object in the family map. + */ + private boolean has(byte[] family, byte[] qualifier, long ts, byte[] value, + boolean ignoreTS, boolean ignoreValue) { + List list = getKeyValueList(family); + if (list.size() == 0) { + return false; + } + // Boolean analysis of ignoreTS/ignoreValue. + // T T => 2 + // T F => 3 (first is always true) + // F T => 2 + // F F => 1 + if (!ignoreTS && !ignoreValue) { + for (KeyValue kv : list) { + if (Arrays.equals(kv.getFamily(), family) && + Arrays.equals(kv.getQualifier(), qualifier) && + Arrays.equals(kv.getValue(), value) && + kv.getTimestamp() == ts) { + return true; + } + } + } else if (ignoreValue && !ignoreTS) { + for (KeyValue kv : list) { + if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier) + && kv.getTimestamp() == ts) { + return true; + } + } + } else if (!ignoreValue && ignoreTS) { + for (KeyValue kv : list) { + if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier) + && Arrays.equals(kv.getValue(), value)) { + return true; + } + } + } else { + for (KeyValue kv : list) { + if (Arrays.equals(kv.getFamily(), family) && + Arrays.equals(kv.getQualifier(), qualifier)) { + return true; + } + } + } + return false; + } + + /** + * Returns a list of all KeyValue objects with matching column family and qualifier. + * + * @param family column family + * @param qualifier column qualifier + * @return a list of KeyValue objects with the matching family and qualifier, + * returns an empty list if one doesnt exist for the given family. + */ + public List get(byte[] family, byte[] qualifier) { + List filteredList = new ArrayList(); + for (KeyValue kv: getKeyValueList(family)) { + if (Arrays.equals(kv.getQualifier(), qualifier)) { + filteredList.add(kv); + } + } + return filteredList; + } + + /** + * Creates an empty list if one doesnt exist for the given column family + * or else it returns the associated list of KeyValue objects. + * + * @param family column family + * @return a list of KeyValue objects, returns an empty list if one doesnt exist. + */ + private List getKeyValueList(byte[] family) { + List list = familyMap.get(family); + if(list == null) { + list = new ArrayList(0); + } + return list; + } + + //HeapSize + public long heapSize() { + long heapsize = OVERHEAD; + //Adding row + heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length); + + //Adding map overhead + heapsize += + ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY); + for(Map.Entry> entry : this.familyMap.entrySet()) { + //Adding key overhead + heapsize += + ClassSize.align(ClassSize.ARRAY + entry.getKey().length); + + //This part is kinds tricky since the JVM can reuse references if you + //store the same value, but have a good match with SizeOf at the moment + //Adding value overhead + heapsize += ClassSize.align(ClassSize.ARRAYLIST); + int size = entry.getValue().size(); + heapsize += ClassSize.align(ClassSize.ARRAY + + size * ClassSize.REFERENCE); + + for(KeyValue kv : entry.getValue()) { + heapsize += kv.heapSize(); + } + } + heapsize += getAttributeSize(); + + return ClassSize.align((int)heapsize); + } + + //Writable + public void readFields(final DataInput in) + throws IOException { + int version = in.readByte(); + if (version > PUT_VERSION) { + throw new IOException("version not supported"); + } + this.row = Bytes.readByteArray(in); + this.ts = in.readLong(); + this.lockId = in.readLong(); + this.writeToWAL = in.readBoolean(); + int numFamilies = in.readInt(); + if (!this.familyMap.isEmpty()) this.familyMap.clear(); + for(int i=0;i keys = new ArrayList(numKeys); + int totalLen = in.readInt(); + byte [] buf = new byte[totalLen]; + int offset = 0; + for (int j = 0; j < numKeys; j++) { + int keyLength = in.readInt(); + in.readFully(buf, offset, keyLength); + keys.add(new KeyValue(buf, offset, keyLength)); + offset += keyLength; + } + this.familyMap.put(family, keys); + } + if (version > 1) { + readAttributes(in); + } + } + + public void write(final DataOutput out) + throws IOException { + out.writeByte(PUT_VERSION); + Bytes.writeByteArray(out, this.row); + out.writeLong(this.ts); + out.writeLong(this.lockId); + out.writeBoolean(this.writeToWAL); + out.writeInt(familyMap.size()); + for (Map.Entry> entry : familyMap.entrySet()) { + Bytes.writeByteArray(out, entry.getKey()); + List keys = entry.getValue(); + out.writeInt(keys.size()); + int totalLen = 0; + for(KeyValue kv : keys) { + totalLen += kv.getLength(); + } + out.writeInt(totalLen); + for(KeyValue kv : keys) { + out.writeInt(kv.getLength()); + out.write(kv.getBuffer(), kv.getOffset(), kv.getLength()); + } + } + writeAttributes(out); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/RegionOfflineException.java b/src/main/java/org/apache/hadoop/hbase/client/RegionOfflineException.java new file mode 100644 index 0000000..d223860 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/RegionOfflineException.java @@ -0,0 +1,36 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.RegionException; + +/** Thrown when a table can not be located */ +public class RegionOfflineException extends RegionException { + private static final long serialVersionUID = 466008402L; + /** default constructor */ + public RegionOfflineException() { + super(); + } + + /** @param s message */ + public RegionOfflineException(String s) { + super(s); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/Result.java b/src/main/java/org/apache/hadoop/hbase/client/Result.java new file mode 100644 index 0000000..bf8bda3 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/Result.java @@ -0,0 +1,663 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.SplitKeyValue; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.io.WritableWithSize; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Writable; + +/** + * Single row result of a {@link Get} or {@link Scan} query.

+ * + * This class is NOT THREAD SAFE.

+ * + * Convenience methods are available that return various {@link Map} + * structures and values directly.

+ * + * To get a complete mapping of all cells in the Result, which can include + * multiple families and multiple versions, use {@link #getMap()}.

+ * + * To get a mapping of each family to its columns (qualifiers and values), + * including only the latest version of each, use {@link #getNoVersionMap()}. + * + * To get a mapping of qualifiers to latest values for an individual family use + * {@link #getFamilyMap(byte[])}.

+ * + * To get the latest value for a specific family and qualifier use {@link #getValue(byte[], byte[])}. + * + * A Result is backed by an array of {@link KeyValue} objects, each representing + * an HBase cell defined by the row, family, qualifier, timestamp, and value.

+ * + * The underlying {@link KeyValue} objects can be accessed through the method {@link #list()}. + * Each KeyValue can then be accessed + * through {@link KeyValue#getRow()}, {@link KeyValue#getFamily()}, {@link KeyValue#getQualifier()}, + * {@link KeyValue#getTimestamp()}, and {@link KeyValue#getValue()}. + */ +public class Result implements Writable, WritableWithSize { + private static final byte RESULT_VERSION = (byte)1; + + private KeyValue [] kvs = null; + private NavigableMap>> familyMap = null; + // We're not using java serialization. Transient here is just a marker to say + // that this is where we cache row if we're ever asked for it. + private transient byte [] row = null; + private ImmutableBytesWritable bytes = null; + + /** + * Constructor used for Writable. + */ + public Result() {} + + /** + * Instantiate a Result with the specified array of KeyValues. + * @param kvs array of KeyValues + */ + public Result(KeyValue [] kvs) { + if(kvs != null && kvs.length > 0) { + this.kvs = kvs; + } + } + + /** + * Instantiate a Result with the specified List of KeyValues. + * @param kvs List of KeyValues + */ + public Result(List kvs) { + this(kvs.toArray(new KeyValue[kvs.size()])); + } + + /** + * Instantiate a Result from the specified raw binary format. + * @param bytes raw binary format of Result + */ + public Result(ImmutableBytesWritable bytes) { + this.bytes = bytes; + } + + /** + * Method for retrieving the row key that corresponds to + * the row from which this Result was created. + * @return row + */ + public byte [] getRow() { + if (this.row == null) { + if(this.kvs == null) { + readFields(); + } + this.row = this.kvs.length == 0? null: this.kvs[0].getRow(); + } + return this.row; + } + + /** + * Return the array of KeyValues backing this Result instance. + * + * The array is sorted from smallest -> largest using the + * {@link KeyValue#COMPARATOR}. + * + * The array only contains what your Get or Scan specifies and no more. + * For example if you request column "A" 1 version you will have at most 1 + * KeyValue in the array. If you request column "A" with 2 version you will + * have at most 2 KeyValues, with the first one being the newer timestamp and + * the second being the older timestamp (this is the sort order defined by + * {@link KeyValue#COMPARATOR}). If columns don't exist, they won't be + * present in the result. Therefore if you ask for 1 version all columns, + * it is safe to iterate over this array and expect to see 1 KeyValue for + * each column and no more. + * + * This API is faster than using getFamilyMap() and getMap() + * + * @return array of KeyValues + */ + public KeyValue[] raw() { + if(this.kvs == null) { + readFields(); + } + return kvs; + } + + /** + * Create a sorted list of the KeyValue's in this result. + * + * Since HBase 0.20.5 this is equivalent to raw(). + * + * @return The sorted list of KeyValue's. + */ + public List list() { + if(this.kvs == null) { + readFields(); + } + return isEmpty()? null: Arrays.asList(raw()); + } + + /** + * Return the KeyValues for the specific column. The KeyValues are sorted in + * the {@link KeyValue#COMPARATOR} order. That implies the first entry in + * the list is the most recent column. If the query (Scan or Get) only + * requested 1 version the list will contain at most 1 entry. If the column + * did not exist in the result set (either the column does not exist + * or the column was not selected in the query) the list will be empty. + * + * Also see getColumnLatest which returns just a KeyValue + * + * @param family the family + * @param qualifier + * @return a list of KeyValues for this column or empty list if the column + * did not exist in the result set + */ + public List getColumn(byte [] family, byte [] qualifier) { + List result = new ArrayList(); + + KeyValue [] kvs = raw(); + + if (kvs == null || kvs.length == 0) { + return result; + } + int pos = binarySearch(kvs, family, qualifier); + if (pos == -1) { + return result; // cant find it + } + + for (int i = pos ; i < kvs.length ; i++ ) { + KeyValue kv = kvs[i]; + if (kv.matchingColumn(family,qualifier)) { + result.add(kv); + } else { + break; + } + } + + return result; + } + + protected int binarySearch(final KeyValue [] kvs, + final byte [] family, + final byte [] qualifier) { + KeyValue searchTerm = + KeyValue.createFirstOnRow(kvs[0].getRow(), + family, qualifier); + + // pos === ( -(insertion point) - 1) + int pos = Arrays.binarySearch(kvs, searchTerm, KeyValue.COMPARATOR); + // never will exact match + if (pos < 0) { + pos = (pos+1) * -1; + // pos is now insertion point + } + if (pos == kvs.length) { + return -1; // doesn't exist + } + return pos; + } + + /** + * The KeyValue for the most recent for a given column. If the column does + * not exist in the result set - if it wasn't selected in the query (Get/Scan) + * or just does not exist in the row the return value is null. + * + * @param family + * @param qualifier + * @return KeyValue for the column or null + */ + public KeyValue getColumnLatest(byte [] family, byte [] qualifier) { + KeyValue [] kvs = raw(); // side effect possibly. + if (kvs == null || kvs.length == 0) { + return null; + } + int pos = binarySearch(kvs, family, qualifier); + if (pos == -1) { + return null; + } + KeyValue kv = kvs[pos]; + if (kv.matchingColumn(family, qualifier)) { + return kv; + } + return null; + } + + /** + * Get the latest version of the specified column. + * @param family family name + * @param qualifier column qualifier + * @return value of latest version of column, null if none found + */ + public byte[] getValue(byte [] family, byte [] qualifier) { + KeyValue kv = getColumnLatest(family, qualifier); + if (kv == null) { + return null; + } + return kv.getValue(); + } + + /** + * Checks for existence of the specified column. + * @param family family name + * @param qualifier column qualifier + * @return true if at least one value exists in the result, false if not + */ + public boolean containsColumn(byte [] family, byte [] qualifier) { + KeyValue kv = getColumnLatest(family, qualifier); + return kv != null; + } + + /** + * Map of families to all versions of its qualifiers and values. + *

+ * Returns a three level Map of the form: + * Map&family,Map<qualifier,Map<timestamp,value>>> + *

+ * Note: All other map returning methods make use of this map internally. + * @return map from families to qualifiers to versions + */ + public NavigableMap>> getMap() { + if(this.familyMap != null) { + return this.familyMap; + } + if(isEmpty()) { + return null; + } + this.familyMap = + new TreeMap>> + (Bytes.BYTES_COMPARATOR); + for(KeyValue kv : this.kvs) { + SplitKeyValue splitKV = kv.split(); + byte [] family = splitKV.getFamily(); + NavigableMap> columnMap = + familyMap.get(family); + if(columnMap == null) { + columnMap = new TreeMap> + (Bytes.BYTES_COMPARATOR); + familyMap.put(family, columnMap); + } + byte [] qualifier = splitKV.getQualifier(); + NavigableMap versionMap = columnMap.get(qualifier); + if(versionMap == null) { + versionMap = new TreeMap(new Comparator() { + public int compare(Long l1, Long l2) { + return l2.compareTo(l1); + } + }); + columnMap.put(qualifier, versionMap); + } + Long timestamp = Bytes.toLong(splitKV.getTimestamp()); + byte [] value = splitKV.getValue(); + versionMap.put(timestamp, value); + } + return this.familyMap; + } + + /** + * Map of families to their most recent qualifiers and values. + *

+ * Returns a two level Map of the form: Map&family,Map<qualifier,value>> + *

+ * The most recent version of each qualifier will be used. + * @return map from families to qualifiers and value + */ + public NavigableMap> getNoVersionMap() { + if(this.familyMap == null) { + getMap(); + } + if(isEmpty()) { + return null; + } + NavigableMap> returnMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); + for(Map.Entry>> + familyEntry : familyMap.entrySet()) { + NavigableMap qualifierMap = + new TreeMap(Bytes.BYTES_COMPARATOR); + for(Map.Entry> qualifierEntry : + familyEntry.getValue().entrySet()) { + byte [] value = + qualifierEntry.getValue().get(qualifierEntry.getValue().firstKey()); + qualifierMap.put(qualifierEntry.getKey(), value); + } + returnMap.put(familyEntry.getKey(), qualifierMap); + } + return returnMap; + } + + /** + * Map of qualifiers to values. + *

+ * Returns a Map of the form: Map<qualifier,value> + * @param family column family to get + * @return map of qualifiers to values + */ + public NavigableMap getFamilyMap(byte [] family) { + if(this.familyMap == null) { + getMap(); + } + if(isEmpty()) { + return null; + } + NavigableMap returnMap = + new TreeMap(Bytes.BYTES_COMPARATOR); + NavigableMap> qualifierMap = + familyMap.get(family); + if(qualifierMap == null) { + return returnMap; + } + for(Map.Entry> entry : + qualifierMap.entrySet()) { + byte [] value = + entry.getValue().get(entry.getValue().firstKey()); + returnMap.put(entry.getKey(), value); + } + return returnMap; + } + + /** + * Returns the value of the first column in the Result. + * @return value of the first column + */ + public byte [] value() { + if (isEmpty()) { + return null; + } + return kvs[0].getValue(); + } + + /** + * Returns the raw binary encoding of this Result.

+ * + * Please note, there may be an offset into the underlying byte array of the + * returned ImmutableBytesWritable. Be sure to use both + * {@link ImmutableBytesWritable#get()} and {@link ImmutableBytesWritable#getOffset()} + * @return pointer to raw binary of Result + */ + public ImmutableBytesWritable getBytes() { + return this.bytes; + } + + /** + * Check if the underlying KeyValue [] is empty or not + * @return true if empty + */ + public boolean isEmpty() { + if(this.kvs == null) { + readFields(); + } + return this.kvs == null || this.kvs.length == 0; + } + + /** + * @return the size of the underlying KeyValue [] + */ + public int size() { + if(this.kvs == null) { + readFields(); + } + return this.kvs == null? 0: this.kvs.length; + } + + /** + * @return String + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("keyvalues="); + if(isEmpty()) { + sb.append("NONE"); + return sb.toString(); + } + sb.append("{"); + boolean moreThanOne = false; + for(KeyValue kv : this.kvs) { + if(moreThanOne) { + sb.append(", "); + } else { + moreThanOne = true; + } + sb.append(kv.toString()); + } + sb.append("}"); + return sb.toString(); + } + + //Writable + public void readFields(final DataInput in) + throws IOException { + familyMap = null; + row = null; + kvs = null; + int totalBuffer = in.readInt(); + if(totalBuffer == 0) { + bytes = null; + return; + } + byte [] raw = new byte[totalBuffer]; + readChunked(in, raw, 0, totalBuffer); + bytes = new ImmutableBytesWritable(raw, 0, totalBuffer); + } + + private void readChunked(final DataInput in, byte[] dest, int ofs, int len) + throws IOException { + int maxRead = 8192; + + for (; ofs < len; ofs += maxRead) + in.readFully(dest, ofs, Math.min(len - ofs, maxRead)); + } + + //Create KeyValue[] when needed + private void readFields() { + if (bytes == null) { + this.kvs = new KeyValue[0]; + return; + } + byte [] buf = bytes.get(); + int offset = bytes.getOffset(); + int finalOffset = bytes.getSize() + offset; + List kvs = new ArrayList(); + while(offset < finalOffset) { + int keyLength = Bytes.toInt(buf, offset); + offset += Bytes.SIZEOF_INT; + kvs.add(new KeyValue(buf, offset, keyLength)); + offset += keyLength; + } + this.kvs = kvs.toArray(new KeyValue[kvs.size()]); + } + + public long getWritableSize() { + if (isEmpty()) + return Bytes.SIZEOF_INT; // int size = 0 + + long size = Bytes.SIZEOF_INT; // totalLen + + for (KeyValue kv : kvs) { + size += kv.getLength(); + size += Bytes.SIZEOF_INT; // kv.getLength + } + + return size; + } + + public void write(final DataOutput out) + throws IOException { + if(isEmpty()) { + out.writeInt(0); + } else { + int totalLen = 0; + for(KeyValue kv : kvs) { + totalLen += kv.getLength() + Bytes.SIZEOF_INT; + } + out.writeInt(totalLen); + for(KeyValue kv : kvs) { + out.writeInt(kv.getLength()); + out.write(kv.getBuffer(), kv.getOffset(), kv.getLength()); + } + } + } + + public static long getWriteArraySize(Result [] results) { + long size = Bytes.SIZEOF_BYTE; // RESULT_VERSION + if (results == null || results.length == 0) { + size += Bytes.SIZEOF_INT; + return size; + } + + size += Bytes.SIZEOF_INT; // results.length + size += Bytes.SIZEOF_INT; // bufLen + for (Result result : results) { + size += Bytes.SIZEOF_INT; // either 0 or result.size() + if (result == null || result.isEmpty()) + continue; + + for (KeyValue kv : result.raw()) { + size += Bytes.SIZEOF_INT; // kv.getLength(); + size += kv.getLength(); + } + } + + return size; + } + + public static void writeArray(final DataOutput out, Result [] results) + throws IOException { + // Write version when writing array form. + // This assumes that results are sent to the client as Result[], so we + // have an opportunity to handle version differences without affecting + // efficiency. + out.writeByte(RESULT_VERSION); + if(results == null || results.length == 0) { + out.writeInt(0); + return; + } + out.writeInt(results.length); + int bufLen = 0; + for(Result result : results) { + bufLen += Bytes.SIZEOF_INT; + if(result == null || result.isEmpty()) { + continue; + } + for(KeyValue key : result.raw()) { + bufLen += key.getLength() + Bytes.SIZEOF_INT; + } + } + out.writeInt(bufLen); + for(Result result : results) { + if(result == null || result.isEmpty()) { + out.writeInt(0); + continue; + } + out.writeInt(result.size()); + for(KeyValue kv : result.raw()) { + out.writeInt(kv.getLength()); + out.write(kv.getBuffer(), kv.getOffset(), kv.getLength()); + } + } + } + + public static Result [] readArray(final DataInput in) + throws IOException { + // Read version for array form. + // This assumes that results are sent to the client as Result[], so we + // have an opportunity to handle version differences without affecting + // efficiency. + int version = in.readByte(); + if (version > RESULT_VERSION) { + throw new IOException("version not supported"); + } + int numResults = in.readInt(); + if(numResults == 0) { + return new Result[0]; + } + Result [] results = new Result[numResults]; + int bufSize = in.readInt(); + byte [] buf = new byte[bufSize]; + int offset = 0; + for(int i=0;i { + + /** + * Grab the next row's worth of values. The scanner will return a Result. + * @return Result object if there is another row, null if the scanner is + * exhausted. + * @throws IOException e + */ + public Result next() throws IOException; + + /** + * @param nbRows number of rows to return + * @return Between zero and nbRows Results + * @throws IOException e + */ + public Result [] next(int nbRows) throws IOException; + + /** + * Closes the scanner and releases any resources it has allocated + */ + public void close(); +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java new file mode 100644 index 0000000..b9042f6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java @@ -0,0 +1,105 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.util.Date; +import java.util.List; + +/** + * Exception thrown by HTable methods when an attempt to do something (like + * commit changes) fails after a bunch of retries. + */ +public class RetriesExhaustedException extends IOException { + private static final long serialVersionUID = 1876775844L; + + public RetriesExhaustedException(final String msg) { + super(msg); + } + + public RetriesExhaustedException(final String msg, final IOException e) { + super(msg, e); + } + + /** + * Datastructure that allows adding more info around Throwable incident. + */ + public static class ThrowableWithExtraContext { + private final Throwable t; + private final long when; + private final String extras; + + public ThrowableWithExtraContext(final Throwable t, final long when, + final String extras) { + this.t = t; + this.when = when; + this.extras = extras; + } + + @Override + public String toString() { + return new Date(this.when).toString() + ", " + extras + ", " + t.toString(); + } + } + + /** + * Create a new RetriesExhaustedException from the list of prior failures. + * @param callableVitals Details from the {@link ServerCallable} we were using + * when we got this exception. + * @param numTries The number of tries we made + * @param exceptions List of exceptions that failed before giving up + */ + public RetriesExhaustedException(final String callableVitals, int numTries, + List exceptions) { + super(getMessage(callableVitals, numTries, exceptions)); + } + + /** + * Create a new RetriesExhaustedException from the list of prior failures. + * @param numTries + * @param exceptions List of exceptions that failed before giving up + */ + public RetriesExhaustedException(final int numTries, + final List exceptions) { + super(getMessage(numTries, exceptions)); + } + + private static String getMessage(String callableVitals, int numTries, + List exceptions) { + StringBuilder buffer = new StringBuilder("Failed contacting "); + buffer.append(callableVitals); + buffer.append(" after "); + buffer.append(numTries + 1); + buffer.append(" attempts.\nExceptions:\n"); + for (Throwable t : exceptions) { + buffer.append(t.toString()); + buffer.append("\n"); + } + return buffer.toString(); + } + + private static String getMessage(final int numTries, + final List exceptions) { + StringBuilder buffer = new StringBuilder("Failed after attempts="); + buffer.append(numTries + 1); + buffer.append(", exceptions:\n"); + for (ThrowableWithExtraContext t : exceptions) { + buffer.append(t.toString()); + buffer.append("\n"); + } + return buffer.toString(); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java new file mode 100644 index 0000000..ea14c06 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java @@ -0,0 +1,150 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; +import org.apache.hadoop.hbase.util.Addressing; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * This subclass of {@link org.apache.hadoop.hbase.client.RetriesExhaustedException} + * is thrown when we have more information about which rows were causing which + * exceptions on what servers. You can call {@link #mayHaveClusterIssues()} + * and if the result is false, you have input error problems, otherwise you + * may have cluster issues. You can iterate over the causes, rows and last + * known server addresses via {@link #getNumExceptions()} and + * {@link #getCause(int)}, {@link #getRow(int)} and {@link #getAddress(int)}. + */ +@SuppressWarnings("serial") +public class RetriesExhaustedWithDetailsException +extends RetriesExhaustedException { + List exceptions; + List actions; + List hostnameAndPort; + + public RetriesExhaustedWithDetailsException(List exceptions, + List actions, + List hostnameAndPort) { + super("Failed " + exceptions.size() + " action" + + pluralize(exceptions) + ": " + + getDesc(exceptions, actions, hostnameAndPort)); + + this.exceptions = exceptions; + this.actions = actions; + this.hostnameAndPort = hostnameAndPort; + } + + public List getCauses() { + return exceptions; + } + + public int getNumExceptions() { + return exceptions.size(); + } + + public Throwable getCause(int i) { + return exceptions.get(i); + } + + public Row getRow(int i) { + return actions.get(i); + } + + public HServerAddress getAddress(int i) { + return new HServerAddress(Addressing.createInetSocketAddressFromHostAndPortStr(getHostnamePort(i))); + } + + public String getHostnamePort(final int i) { + return this.hostnameAndPort.get(i); + } + + public boolean mayHaveClusterIssues() { + boolean res = false; + + // If all of the exceptions are DNRIOE not exception + for (Throwable t : exceptions) { + if ( !(t instanceof DoNotRetryIOException)) { + res = true; + } + } + return res; + } + + + public static String pluralize(Collection c) { + return pluralize(c.size()); + } + + public static String pluralize(int c) { + return c > 1 ? "s" : ""; + } + + public static String getDesc(List exceptions, + List actions, + List hostnamePort) { + String s = getDesc(classifyExs(exceptions)); + s += "servers with issues: "; + Set uniqAddr = new HashSet(); + uniqAddr.addAll(hostnamePort); + for(String addr : uniqAddr) { + s += addr + ", "; + } + return s; + } + + public static Map classifyExs(List ths) { + Map cls = new HashMap(); + for (Throwable t : ths) { + if (t == null) continue; + String name = ""; + if (t instanceof DoNotRetryIOException) { + name = t.getMessage(); + } else { + name = t.getClass().getSimpleName(); + } + Integer i = cls.get(name); + if (i == null) { + i = 0; + } + i += 1; + cls.put(name, i); + } + return cls; + } + + public static String getDesc(Map classificaton) { + String s = ""; + for (Map.Entry e : classificaton.entrySet()) { + s += e.getKey() + ": " + e.getValue() + " time" + + pluralize(e.getValue()) + ", "; + } + return s; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/Row.java b/src/main/java/org/apache/hadoop/hbase/client/Row.java new file mode 100644 index 0000000..cd332bd --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/Row.java @@ -0,0 +1,32 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.io.WritableComparable; + +/** + * Has a row. + */ +public interface Row extends WritableComparable { + /** + * @return The row. + */ + public byte [] getRow(); +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/RowLock.java b/src/main/java/org/apache/hadoop/hbase/client/RowLock.java new file mode 100644 index 0000000..5888ec8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/RowLock.java @@ -0,0 +1,63 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +/** + * Holds row name and lock id. + * @deprecated {@link RowLock} and associated operations are deprecated. + */ +public class RowLock { + private byte [] row = null; + private long lockId = -1L; + + /** + * Creates a RowLock from a row and lock id + * @param row row to lock on + * @param lockId the lock id + */ + public RowLock(final byte [] row, final long lockId) { + this.row = row; + this.lockId = lockId; + } + + /** + * Creates a RowLock with only a lock id + * @param lockId lock id + */ + public RowLock(final long lockId) { + this.lockId = lockId; + } + + /** + * Get the row for this RowLock + * @return the row + */ + public byte [] getRow() { + return row; + } + + /** + * Get the lock id from this RowLock + * @return the lock id + */ + public long getLockId() { + return lockId; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/RowMutations.java b/src/main/java/org/apache/hadoop/hbase/client/RowMutations.java new file mode 100644 index 0000000..228c798 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/RowMutations.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Performs multiple mutations atomically on a single row. + * Currently {@link Put} and {@link Delete} are supported. + * + * The mutations are performed in the order in which they + * were added. + */ +public class RowMutations implements Row { + private List mutations = new ArrayList(); + private byte [] row; + private static final byte VERSION = (byte)0; + + /** Constructor for Writable. DO NOT USE */ + public RowMutations() {} + + /** + * Create an atomic mutation for the specified row. + * @param row row key + */ + public RowMutations(byte [] row) { + if(row == null || row.length > HConstants.MAX_ROW_LENGTH) { + throw new IllegalArgumentException("Row key is invalid"); + } + this.row = Arrays.copyOf(row, row.length); + } + + /** + * Add a {@link Put} operation to the list of mutations + * @param p The {@link Put} to add + * @throws IOException + */ + public void add(Put p) throws IOException { + internalAdd(p); + } + + /** + * Add a {@link Delete} operation to the list of mutations + * @param d The {@link Delete} to add + * @throws IOException + */ + public void add(Delete d) throws IOException { + internalAdd(d); + } + + private void internalAdd(Mutation m) throws IOException { + int res = Bytes.compareTo(this.row, m.getRow()); + if(res != 0) { + throw new IOException("The row in the recently added Put/Delete " + + Bytes.toStringBinary(m.getRow()) + " doesn't match the original one " + + Bytes.toStringBinary(this.row)); + } + mutations.add(m); + } + + @Override + public void readFields(final DataInput in) throws IOException { + int version = in.readByte(); + if (version > VERSION) { + throw new IOException("version not supported"); + } + this.row = Bytes.readByteArray(in); + int numMutations = in.readInt(); + mutations.clear(); + for(int i = 0; i < numMutations; i++) { + mutations.add((Mutation) HbaseObjectWritable.readObject(in, null)); + } + } + + @Override + public void write(final DataOutput out) throws IOException { + out.writeByte(VERSION); + Bytes.writeByteArray(out, this.row); + out.writeInt(mutations.size()); + for (Mutation m : mutations) { + HbaseObjectWritable.writeObject(out, m, m.getClass(), null); + } + } + + @Override + public int compareTo(Row i) { + return Bytes.compareTo(this.getRow(), i.getRow()); + } + + @Override + public byte[] getRow() { + return row; + } + + /** + * @return An unmodifiable list of the current mutations. + */ + public List getMutations() { + return Collections.unmodifiableList(mutations); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java new file mode 100644 index 0000000..553bde3 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java @@ -0,0 +1,695 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.IncompatibleFilterException; +import org.apache.hadoop.hbase.io.TimeRange; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Classes; +import org.apache.hadoop.io.Writable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * Used to perform Scan operations. + *

+ * All operations are identical to {@link Get} with the exception of + * instantiation. Rather than specifying a single row, an optional startRow + * and stopRow may be defined. If rows are not specified, the Scanner will + * iterate over all rows. + *

+ * To scan everything for each row, instantiate a Scan object. + *

+ * To modify scanner caching for just this scan, use {@link #setCaching(int) setCaching}. + * If caching is NOT set, we will use the caching value of the hosting + * {@link HTable}. See {@link HTable#setScannerCaching(int)}. + *

+ * To further define the scope of what to get when scanning, perform additional + * methods as outlined below. + *

+ * To get all columns from specific families, execute {@link #addFamily(byte[]) addFamily} + * for each family to retrieve. + *

+ * To get specific columns, execute {@link #addColumn(byte[], byte[]) addColumn} + * for each column to retrieve. + *

+ * To only retrieve columns within a specific range of version timestamps, + * execute {@link #setTimeRange(long, long) setTimeRange}. + *

+ * To only retrieve columns with a specific timestamp, execute + * {@link #setTimeStamp(long) setTimestamp}. + *

+ * To limit the number of versions of each column to be returned, execute + * {@link #setMaxVersions(int) setMaxVersions}. + *

+ * To limit the maximum number of values returned for each call to next(), + * execute {@link #setBatch(int) setBatch}. + *

+ * To add a filter, execute {@link #setFilter(org.apache.hadoop.hbase.filter.Filter) setFilter}. + *

+ * Expert: To explicitly disable server-side block caching for this scan, + * execute {@link #setCacheBlocks(boolean)}. + */ +public class Scan extends OperationWithAttributes implements Writable { + private static final String RAW_ATTR = "_raw_"; + private static final String ONDEMAND_ATTR = "_ondemand_"; + private static final String ISOLATION_LEVEL = "_isolationlevel_"; + + private static final byte SCAN_VERSION = (byte)2; + private byte [] startRow = HConstants.EMPTY_START_ROW; + private byte [] stopRow = HConstants.EMPTY_END_ROW; + private int maxVersions = 1; + private int batch = -1; + // If application wants to collect scan metrics, it needs to + // call scan.setAttribute(SCAN_ATTRIBUTES_ENABLE, Bytes.toBytes(Boolean.TRUE)) + static public String SCAN_ATTRIBUTES_METRICS_ENABLE = "scan.attributes.metrics.enable"; + static public String SCAN_ATTRIBUTES_METRICS_DATA = "scan.attributes.metrics.data"; + + // If an application wants to use multiple scans over different tables each scan must + // define this attribute with the appropriate table name by calling + // scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName)) + static public final String SCAN_ATTRIBUTES_TABLE_NAME = "scan.attributes.table.name"; + + /* + * -1 means no caching + */ + private int caching = -1; + private boolean cacheBlocks = true; + private Filter filter = null; + private TimeRange tr = new TimeRange(); + private Map> familyMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); + + /** + * Create a Scan operation across all rows. + */ + public Scan() {} + + public Scan(byte [] startRow, Filter filter) { + this(startRow); + this.filter = filter; + } + + /** + * Create a Scan operation starting at the specified row. + *

+ * If the specified row does not exist, the Scanner will start from the + * next closest row after the specified row. + * @param startRow row to start scanner at or after + */ + public Scan(byte [] startRow) { + this.startRow = startRow; + } + + /** + * Create a Scan operation for the range of rows specified. + * @param startRow row to start scanner at or after (inclusive) + * @param stopRow row to stop scanner before (exclusive) + */ + public Scan(byte [] startRow, byte [] stopRow) { + this.startRow = startRow; + this.stopRow = stopRow; + } + + /** + * Creates a new instance of this class while copying all values. + * + * @param scan The scan instance to copy from. + * @throws IOException When copying the values fails. + */ + public Scan(Scan scan) throws IOException { + startRow = scan.getStartRow(); + stopRow = scan.getStopRow(); + maxVersions = scan.getMaxVersions(); + batch = scan.getBatch(); + caching = scan.getCaching(); + cacheBlocks = scan.getCacheBlocks(); + filter = scan.getFilter(); // clone? + TimeRange ctr = scan.getTimeRange(); + tr = new TimeRange(ctr.getMin(), ctr.getMax()); + Map> fams = scan.getFamilyMap(); + for (Map.Entry> entry : fams.entrySet()) { + byte [] fam = entry.getKey(); + NavigableSet cols = entry.getValue(); + if (cols != null && cols.size() > 0) { + for (byte[] col : cols) { + addColumn(fam, col); + } + } else { + addFamily(fam); + } + } + for (Map.Entry attr : scan.getAttributesMap().entrySet()) { + setAttribute(attr.getKey(), attr.getValue()); + } + } + + /** + * Builds a scan object with the same specs as get. + * @param get get to model scan after + */ + public Scan(Get get) { + this.startRow = get.getRow(); + this.stopRow = get.getRow(); + this.filter = get.getFilter(); + this.cacheBlocks = get.getCacheBlocks(); + this.maxVersions = get.getMaxVersions(); + this.tr = get.getTimeRange(); + this.familyMap = get.getFamilyMap(); + } + + public boolean isGetScan() { + return this.startRow != null && this.startRow.length > 0 && + Bytes.equals(this.startRow, this.stopRow); + } + + /** + * Get all columns from the specified family. + *

+ * Overrides previous calls to addColumn for this family. + * @param family family name + * @return this + */ + public Scan addFamily(byte [] family) { + familyMap.remove(family); + familyMap.put(family, null); + return this; + } + + /** + * Get the column from the specified family with the specified qualifier. + *

+ * Overrides previous calls to addFamily for this family. + * @param family family name + * @param qualifier column qualifier + * @return this + */ + public Scan addColumn(byte [] family, byte [] qualifier) { + NavigableSet set = familyMap.get(family); + if(set == null) { + set = new TreeSet(Bytes.BYTES_COMPARATOR); + } + if (qualifier == null) { + qualifier = HConstants.EMPTY_BYTE_ARRAY; + } + set.add(qualifier); + familyMap.put(family, set); + + return this; + } + + /** + * Get versions of columns only within the specified timestamp range, + * [minStamp, maxStamp). Note, default maximum versions to return is 1. If + * your time range spans more than one version and you want all versions + * returned, up the number of versions beyond the defaut. + * @param minStamp minimum timestamp value, inclusive + * @param maxStamp maximum timestamp value, exclusive + * @throws IOException if invalid time range + * @see #setMaxVersions() + * @see #setMaxVersions(int) + * @return this + */ + public Scan setTimeRange(long minStamp, long maxStamp) + throws IOException { + tr = new TimeRange(minStamp, maxStamp); + return this; + } + + /** + * Get versions of columns with the specified timestamp. Note, default maximum + * versions to return is 1. If your time range spans more than one version + * and you want all versions returned, up the number of versions beyond the + * defaut. + * @param timestamp version timestamp + * @see #setMaxVersions() + * @see #setMaxVersions(int) + * @return this + */ + public Scan setTimeStamp(long timestamp) { + try { + tr = new TimeRange(timestamp, timestamp+1); + } catch(IOException e) { + // Will never happen + } + return this; + } + + /** + * Set the start row of the scan. + * @param startRow row to start scan on (inclusive) + * Note: In order to make startRow exclusive add a trailing 0 byte + * @return this + */ + public Scan setStartRow(byte [] startRow) { + this.startRow = startRow; + return this; + } + + /** + * Set the stop row. + * @param stopRow row to end at (exclusive) + * Note: In order to make stopRow inclusive add a trailing 0 byte + * @return this + */ + public Scan setStopRow(byte [] stopRow) { + this.stopRow = stopRow; + return this; + } + + /** + * Get all available versions. + * @return this + */ + public Scan setMaxVersions() { + this.maxVersions = Integer.MAX_VALUE; + return this; + } + + /** + * Get up to the specified number of versions of each column. + * @param maxVersions maximum versions for each column + * @return this + */ + public Scan setMaxVersions(int maxVersions) { + this.maxVersions = maxVersions; + return this; + } + + /** + * Set the maximum number of values to return for each call to next() + * @param batch the maximum number of values + */ + public void setBatch(int batch) { + if (this.hasFilter() && this.filter.hasFilterRow()) { + throw new IncompatibleFilterException( + "Cannot set batch on a scan using a filter" + + " that returns true for filter.hasFilterRow"); + } + this.batch = batch; + } + + /** + * Set the number of rows for caching that will be passed to scanners. + * If not set, the default setting from {@link HTable#getScannerCaching()} will apply. + * Higher caching values will enable faster scanners but will use more memory. + * @param caching the number of rows for caching + */ + public void setCaching(int caching) { + this.caching = caching; + } + + /** + * Apply the specified server-side filter when performing the Scan. + * @param filter filter to run on the server + * @return this + */ + public Scan setFilter(Filter filter) { + this.filter = filter; + return this; + } + + /** + * Setting the familyMap + * @param familyMap map of family to qualifier + * @return this + */ + public Scan setFamilyMap(Map> familyMap) { + this.familyMap = familyMap; + return this; + } + + /** + * Getting the familyMap + * @return familyMap + */ + public Map> getFamilyMap() { + return this.familyMap; + } + + /** + * @return the number of families in familyMap + */ + public int numFamilies() { + if(hasFamilies()) { + return this.familyMap.size(); + } + return 0; + } + + /** + * @return true if familyMap is non empty, false otherwise + */ + public boolean hasFamilies() { + return !this.familyMap.isEmpty(); + } + + /** + * @return the keys of the familyMap + */ + public byte[][] getFamilies() { + if(hasFamilies()) { + return this.familyMap.keySet().toArray(new byte[0][0]); + } + return null; + } + + /** + * @return the startrow + */ + public byte [] getStartRow() { + return this.startRow; + } + + /** + * @return the stoprow + */ + public byte [] getStopRow() { + return this.stopRow; + } + + /** + * @return the max number of versions to fetch + */ + public int getMaxVersions() { + return this.maxVersions; + } + + /** + * @return maximum number of values to return for a single call to next() + */ + public int getBatch() { + return this.batch; + } + + /** + * @return caching the number of rows fetched when calling next on a scanner + */ + public int getCaching() { + return this.caching; + } + + /** + * @return TimeRange + */ + public TimeRange getTimeRange() { + return this.tr; + } + + /** + * @return RowFilter + */ + public Filter getFilter() { + return filter; + } + + /** + * @return true is a filter has been specified, false if not + */ + public boolean hasFilter() { + return filter != null; + } + + /** + * Set whether blocks should be cached for this Scan. + *

+ * This is true by default. When true, default settings of the table and + * family are used (this will never override caching blocks if the block + * cache is disabled for that family or entirely). + * + * @param cacheBlocks if false, default settings are overridden and blocks + * will not be cached + */ + public void setCacheBlocks(boolean cacheBlocks) { + this.cacheBlocks = cacheBlocks; + } + + /** + * Get whether blocks should be cached for this Scan. + * @return true if default caching should be used, false if blocks should not + * be cached + */ + public boolean getCacheBlocks() { + return cacheBlocks; + } + + /** + * Set the value indicating whether loading CFs on demand should be allowed (cluster + * default is false). On-demand CF loading doesn't load column families until necessary, e.g. + * if you filter on one column, the other column family data will be loaded only for the rows + * that are included in result, not all rows like in normal case. + * With column-specific filters, like SingleColumnValueFilter w/filterIfMissing == true, + * this can deliver huge perf gains when there's a cf with lots of data; however, it can + * also lead to some inconsistent results, as follows: + * - if someone does a concurrent update to both column families in question you may get a row + * that never existed, e.g. for { rowKey = 5, { cat_videos => 1 }, { video => "my cat" } } + * someone puts rowKey 5 with { cat_videos => 0 }, { video => "my dog" }, concurrent scan + * filtering on "cat_videos == 1" can get { rowKey = 5, { cat_videos => 1 }, + * { video => "my dog" } }. + * - if there's a concurrent split and you have more than 2 column families, some rows may be + * missing some column families. + */ + public void setLoadColumnFamiliesOnDemand(boolean value) { + setAttribute(ONDEMAND_ATTR, Bytes.toBytes(value)); + } + + /** + * Get the logical value indicating whether on-demand CF loading should be allowed. + */ + public boolean doLoadColumnFamiliesOnDemand() { + byte[] attr = getAttribute(ONDEMAND_ATTR); + return attr == null ? false : Bytes.toBoolean(attr); + } + + /** + * Compile the table and column family (i.e. schema) information + * into a String. Useful for parsing and aggregation by debugging, + * logging, and administration tools. + * @return Map + */ + @Override + public Map getFingerprint() { + Map map = new HashMap(); + List families = new ArrayList(); + if(this.familyMap.size() == 0) { + map.put("families", "ALL"); + return map; + } else { + map.put("families", families); + } + for (Map.Entry> entry : + this.familyMap.entrySet()) { + families.add(Bytes.toStringBinary(entry.getKey())); + } + return map; + } + + /** + * Compile the details beyond the scope of getFingerprint (row, columns, + * timestamps, etc.) into a Map along with the fingerprinted information. + * Useful for debugging, logging, and administration tools. + * @param maxCols a limit on the number of columns output prior to truncation + * @return Map + */ + @Override + public Map toMap(int maxCols) { + // start with the fingerpring map and build on top of it + Map map = getFingerprint(); + // map from families to column list replaces fingerprint's list of families + Map> familyColumns = + new HashMap>(); + map.put("families", familyColumns); + // add scalar information first + map.put("startRow", Bytes.toStringBinary(this.startRow)); + map.put("stopRow", Bytes.toStringBinary(this.stopRow)); + map.put("maxVersions", this.maxVersions); + map.put("batch", this.batch); + map.put("caching", this.caching); + map.put("cacheBlocks", this.cacheBlocks); + List timeRange = new ArrayList(); + timeRange.add(this.tr.getMin()); + timeRange.add(this.tr.getMax()); + map.put("timeRange", timeRange); + int colCount = 0; + // iterate through affected families and list out up to maxCols columns + for (Map.Entry> entry : + this.familyMap.entrySet()) { + List columns = new ArrayList(); + familyColumns.put(Bytes.toStringBinary(entry.getKey()), columns); + if(entry.getValue() == null) { + colCount++; + --maxCols; + columns.add("ALL"); + } else { + colCount += entry.getValue().size(); + if (maxCols <= 0) { + continue; + } + for (byte [] column : entry.getValue()) { + if (--maxCols <= 0) { + continue; + } + columns.add(Bytes.toStringBinary(column)); + } + } + } + map.put("totalColumns", colCount); + if (this.filter != null) { + map.put("filter", this.filter.toString()); + } + // add the id if set + if (getId() != null) { + map.put("id", getId()); + } + return map; + } + + //Writable + public void readFields(final DataInput in) + throws IOException { + int version = in.readByte(); + if (version > (int)SCAN_VERSION) { + throw new IOException("version not supported"); + } + this.startRow = Bytes.readByteArray(in); + this.stopRow = Bytes.readByteArray(in); + this.maxVersions = in.readInt(); + this.batch = in.readInt(); + this.caching = in.readInt(); + this.cacheBlocks = in.readBoolean(); + if(in.readBoolean()) { + this.filter = Classes.createWritableForName( + Bytes.toString(Bytes.readByteArray(in))); + this.filter.readFields(in); + } + this.tr = new TimeRange(); + tr.readFields(in); + int numFamilies = in.readInt(); + this.familyMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); + for(int i=0; i set = new TreeSet(Bytes.BYTES_COMPARATOR); + for(int j=0; j 1) { + readAttributes(in); + } + } + + public void write(final DataOutput out) + throws IOException { + out.writeByte(SCAN_VERSION); + Bytes.writeByteArray(out, this.startRow); + Bytes.writeByteArray(out, this.stopRow); + out.writeInt(this.maxVersions); + out.writeInt(this.batch); + out.writeInt(this.caching); + out.writeBoolean(this.cacheBlocks); + if(this.filter == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + Bytes.writeByteArray(out, Bytes.toBytes(filter.getClass().getName())); + filter.write(out); + } + tr.write(out); + out.writeInt(familyMap.size()); + for(Map.Entry> entry : familyMap.entrySet()) { + Bytes.writeByteArray(out, entry.getKey()); + NavigableSet columnSet = entry.getValue(); + if(columnSet != null){ + out.writeInt(columnSet.size()); + for(byte [] qualifier : columnSet) { + Bytes.writeByteArray(out, qualifier); + } + } else { + out.writeInt(0); + } + } + writeAttributes(out); + } + + /** + * Enable/disable "raw" mode for this scan. + * If "raw" is enabled the scan will return all + * delete marker and deleted rows that have not + * been collected, yet. + * This is mostly useful for Scan on column families + * that have KEEP_DELETED_ROWS enabled. + * It is an error to specify any column when "raw" is set. + * @param raw True/False to enable/disable "raw" mode. + */ + public void setRaw(boolean raw) { + setAttribute(RAW_ATTR, Bytes.toBytes(raw)); + } + + /** + * @return True if this Scan is in "raw" mode. + */ + public boolean isRaw() { + byte[] attr = getAttribute(RAW_ATTR); + return attr == null ? false : Bytes.toBoolean(attr); + } + + /* + * Set the isolation level for this scan. If the + * isolation level is set to READ_UNCOMMITTED, then + * this scan will return data from committed and + * uncommitted transactions. If the isolation level + * is set to READ_COMMITTED, then this scan will return + * data from committed transactions only. If a isolation + * level is not explicitly set on a Scan, then it + * is assumed to be READ_COMMITTED. + * @param level IsolationLevel for this scan + */ + public void setIsolationLevel(IsolationLevel level) { + setAttribute(ISOLATION_LEVEL, level.toBytes()); + } + /* + * @return The isolation level of this scan. + * If no isolation level was set for this scan object, + * then it returns READ_COMMITTED. + * @return The IsolationLevel for this scan + */ + public IsolationLevel getIsolationLevel() { + byte[] attr = getAttribute(ISOLATION_LEVEL); + return attr == null ? IsolationLevel.READ_COMMITTED : + IsolationLevel.fromBytes(attr); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java b/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java new file mode 100644 index 0000000..2710150 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java @@ -0,0 +1,264 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.net.UnknownHostException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.metrics.ScanMetrics; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.NotServingRegionException; +import org.apache.hadoop.hbase.RemoteExceptionHandler; +import org.apache.hadoop.hbase.UnknownScannerException; +import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.net.DNS; + +/** + * Retries scanner operations such as create, next, etc. + * Used by {@link ResultScanner}s made by {@link HTable}. + */ +public class ScannerCallable extends ServerCallable { + public static final String LOG_SCANNER_LATENCY_CUTOFF + = "hbase.client.log.scanner.latency.cutoff"; + public static final String LOG_SCANNER_ACTIVITY = "hbase.client.log.scanner.activity"; + private static final Log LOG = LogFactory.getLog(ScannerCallable.class); + private long scannerId = -1L; + private boolean instantiated = false; + private boolean closed = false; + private Scan scan; + private int caching = 1; + private ScanMetrics scanMetrics; + private boolean logScannerActivity = false; + private int logCutOffLatency = 1000; + + // indicate if it is a remote server call + private boolean isRegionServerRemote = true; + + /** + * @param connection which connection + * @param tableName table callable is on + * @param scan the scan to execute + * @param scanMetrics the ScanMetrics to used, if it is null, ScannerCallable + * won't collect metrics + */ + public ScannerCallable (HConnection connection, byte [] tableName, Scan scan, + ScanMetrics scanMetrics) { + super(connection, tableName, scan.getStartRow()); + this.scan = scan; + this.scanMetrics = scanMetrics; + Configuration conf = connection.getConfiguration(); + logScannerActivity = conf.getBoolean(LOG_SCANNER_ACTIVITY, false); + logCutOffLatency = conf.getInt(LOG_SCANNER_LATENCY_CUTOFF, 1000); + } + + /** + * @param reload force reload of server location + * @throws IOException + */ + @Override + public void connect(boolean reload) throws IOException { + if (!instantiated || reload) { + super.connect(reload); + checkIfRegionServerIsRemote(); + instantiated = true; + } + + // check how often we retry. + // HConnectionManager will call instantiateServer with reload==true + // if and only if for retries. + if (reload && this.scanMetrics != null) { + this.scanMetrics.countOfRPCRetries.inc(); + if (isRegionServerRemote) { + this.scanMetrics.countOfRemoteRPCRetries.inc(); + } + } + } + + /** + * compare the local machine hostname with region server's hostname + * to decide if hbase client connects to a remote region server + * @throws UnknownHostException. + */ + private void checkIfRegionServerIsRemote() throws UnknownHostException { + String myAddress = DNS.getDefaultHost("default", "default"); + if (this.location.getHostname().equalsIgnoreCase(myAddress)) { + isRegionServerRemote = false; + } else { + isRegionServerRemote = true; + } + } + + /** + * @see java.util.concurrent.Callable#call() + */ + public Result [] call() throws IOException { + if (scannerId != -1L && closed) { + close(); + } else if (scannerId == -1L && !closed) { + this.scannerId = openScanner(); + } else { + Result [] rrs = null; + try { + incRPCcallsMetrics(); + long timestamp = System.currentTimeMillis(); + rrs = server.next(scannerId, caching); + if (logScannerActivity) { + long now = System.currentTimeMillis(); + if (now - timestamp > logCutOffLatency) { + int rows = rrs == null ? 0 : rrs.length; + LOG.info("Took " + (now-timestamp) + "ms to fetch " + + rows + " rows from scanner=" + scannerId); + } + } + updateResultsMetrics(rrs); + } catch (IOException e) { + if (logScannerActivity) { + LOG.info("Got exception in fetching from scanner=" + + scannerId, e); + } + IOException ioe = null; + if (e instanceof RemoteException) { + ioe = RemoteExceptionHandler.decodeRemoteException((RemoteException)e); + } + if (ioe == null) throw new IOException(e); + if (logScannerActivity && (ioe instanceof UnknownScannerException)) { + try { + HRegionLocation location = + connection.relocateRegion(tableName, scan.getStartRow()); + LOG.info("Scanner=" + scannerId + + " expired, current region location is " + location.toString() + + " ip:" + location.getServerAddress().getBindAddress()); + } catch (Throwable t) { + LOG.info("Failed to relocate region", t); + } + } + if (ioe instanceof NotServingRegionException) { + // Throw a DNRE so that we break out of cycle of calling NSRE + // when what we need is to open scanner against new location. + // Attach NSRE to signal client that it needs to resetup scanner. + if (this.scanMetrics != null) { + this.scanMetrics.countOfNSRE.inc(); + } + throw new DoNotRetryIOException("Reset scanner", ioe); + } else if (ioe instanceof RegionServerStoppedException) { + // Throw a DNRE so that we break out of cycle of calling RSSE + // when what we need is to open scanner against new location. + // Attach RSSE to signal client that it needs to resetup scanner. + throw new DoNotRetryIOException("Reset scanner", ioe); + } else { + // The outer layers will retry + throw ioe; + } + } + return rrs; + } + return null; + } + + private void incRPCcallsMetrics() { + if (this.scanMetrics == null) { + return; + } + this.scanMetrics.countOfRPCcalls.inc(); + if (isRegionServerRemote) { + this.scanMetrics.countOfRemoteRPCcalls.inc(); + } + } + + private void updateResultsMetrics(Result[] rrs) { + if (this.scanMetrics == null || rrs == null) { + return; + } + for (Result rr : rrs) { + this.scanMetrics.countOfBytesInResults.inc(rr.getBytes().getLength()); + if (isRegionServerRemote) { + this.scanMetrics.countOfBytesInRemoteResults.inc( + rr.getBytes().getLength()); + } + } + } + + private void close() { + if (this.scannerId == -1L) { + return; + } + try { + incRPCcallsMetrics(); + this.server.close(this.scannerId); + } catch (IOException e) { + LOG.warn("Ignore, probably already closed", e); + } + this.scannerId = -1L; + } + + protected long openScanner() throws IOException { + incRPCcallsMetrics(); + long id = this.server.openScanner(this.location.getRegionInfo().getRegionName(), + this.scan); + if (logScannerActivity) { + LOG.info("Open scanner=" + id + " for scan=" + scan.toString() + + " on region " + this.location.toString() + " ip:" + + this.location.getServerAddress().getBindAddress()); + } + return id; + } + + protected Scan getScan() { + return scan; + } + + /** + * Call this when the next invocation of call should close the scanner + */ + public void setClose() { + this.closed = true; + } + + /** + * @return the HRegionInfo for the current region + */ + public HRegionInfo getHRegionInfo() { + if (!instantiated) { + return null; + } + return location.getRegionInfo(); + } + + /** + * Get the number of rows that will be fetched on next + * @return the number of rows for caching + */ + public int getCaching() { + return caching; + } + + /** + * Set the number of rows that will be fetched on next + * @param caching the number of rows for caching + */ + public void setCaching(int caching) { + this.caching = caching; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/ScannerTimeoutException.java b/src/main/java/org/apache/hadoop/hbase/client/ScannerTimeoutException.java new file mode 100644 index 0000000..5a10b0e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/ScannerTimeoutException.java @@ -0,0 +1,41 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.DoNotRetryIOException; + +/** + * Thrown when a scanner has timed out. + */ +public class ScannerTimeoutException extends DoNotRetryIOException { + + private static final long serialVersionUID = 8788838690290688313L; + + /** default constructor */ + ScannerTimeoutException() { + super(); + } + + /** @param s */ + ScannerTimeoutException(String s) { + super(s); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java b/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java new file mode 100644 index 0000000..f238eb0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java @@ -0,0 +1,240 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.ConnectException; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.NotServingRegionException; +import org.apache.hadoop.hbase.ipc.HBaseRPC; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.ipc.RemoteException; + +/** + * Abstract class that implements {@link Callable}. Implementation stipulates + * return type and method we actually invoke on remote Server. Usually + * used inside a try/catch that fields usual connection failures all wrapped + * up in a retry loop. + *

Call {@link #connect(boolean)} to connect to server hosting region + * that contains the passed row in the passed table before invoking + * {@link #call()}. + * @see HConnection#getRegionServerWithoutRetries(ServerCallable) + * @param the class that the ServerCallable handles + */ +public abstract class ServerCallable implements Callable { + protected final HConnection connection; + protected final byte [] tableName; + protected final byte [] row; + protected HRegionLocation location; + protected HRegionInterface server; + protected int callTimeout; + protected long startTime, endTime; + + /** + * @param connection Connection to use. + * @param tableName Table name to which row belongs. + * @param row The row we want in tableName. + */ + public ServerCallable(HConnection connection, byte [] tableName, byte [] row) { + this(connection, tableName, row, HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); + } + + public ServerCallable(HConnection connection, byte [] tableName, byte [] row, int callTimeout) { + this.connection = connection; + this.tableName = tableName; + this.row = row; + this.callTimeout = callTimeout; + } + + /** + * Connect to the server hosting region with row from tablename. + * @param reload Set this to true if connection should re-find the region + * @throws IOException e + */ + public void connect(final boolean reload) throws IOException { + this.location = connection.getRegionLocation(tableName, row, reload); + this.server = connection.getHRegionConnection(location.getHostname(), + location.getPort()); + } + + /** @return the server name + * @deprecated Just use {@link #toString()} instead. + */ + public String getServerName() { + if (location == null) return null; + return location.getHostnamePort(); + } + + /** @return the region name + * @deprecated Just use {@link #toString()} instead. + */ + public byte[] getRegionName() { + if (location == null) return null; + return location.getRegionInfo().getRegionName(); + } + + /** @return the row + * @deprecated Just use {@link #toString()} instead. + */ + public byte [] getRow() { + return row; + } + + public void beforeCall() { + HBaseRPC.setRpcTimeout(this.callTimeout); + this.startTime = System.currentTimeMillis(); + } + + public void afterCall() { + HBaseRPC.resetRpcTimeout(); + this.endTime = System.currentTimeMillis(); + } + + public void shouldRetry(Throwable throwable) throws IOException { + if (this.callTimeout != HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT) + if (throwable instanceof SocketTimeoutException + || (this.endTime - this.startTime > this.callTimeout)) { + throw (SocketTimeoutException) (SocketTimeoutException) new SocketTimeoutException( + "Call to access row '" + Bytes.toString(row) + "' on table '" + + Bytes.toString(tableName) + + "' failed on socket timeout exception: " + throwable) + .initCause(throwable); + } else { + this.callTimeout = ((int) (this.endTime - this.startTime)); + } + } + + /** + * @return {@link HConnection} instance used by this Callable. + */ + HConnection getConnection() { + return this.connection; + } + + /** + * Run this instance with retries, timed waits, + * and refinds of missing regions. + * + * @param the type of the return value + * @return an object of type T + * @throws IOException if a remote or network exception occurs + * @throws RuntimeException other unspecified error + */ + public T withRetries() + throws IOException, RuntimeException { + Configuration c = getConnection().getConfiguration(); + final long pause = c.getLong(HConstants.HBASE_CLIENT_PAUSE, + HConstants.DEFAULT_HBASE_CLIENT_PAUSE); + final int numRetries = c.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + List exceptions = + new ArrayList(); + for (int tries = 0; tries < numRetries; tries++) { + try { + beforeCall(); + connect(tries != 0); + return call(); + } catch (Throwable t) { + shouldRetry(t); + t = translateException(t); + if (t instanceof SocketTimeoutException || + t instanceof ConnectException || + t instanceof RetriesExhaustedException) { + // if thrown these exceptions, we clear all the cache entries that + // map to that slow/dead server; otherwise, let cache miss and ask + // .META. again to find the new location + HRegionLocation hrl = location; + if (hrl != null) { + getConnection().clearCaches(hrl.getHostnamePort()); + } + } else if (t instanceof NotServingRegionException && numRetries == 1) { + // Purge cache entries for this specific region from META cache + // since we don't call connect(true) when number of retries is 1. + getConnection().deleteCachedRegionLocation(location); + } + RetriesExhaustedException.ThrowableWithExtraContext qt = + new RetriesExhaustedException.ThrowableWithExtraContext(t, + System.currentTimeMillis(), toString()); + exceptions.add(qt); + if (tries == numRetries - 1) { + throw new RetriesExhaustedException(tries, exceptions); + } + } finally { + afterCall(); + } + try { + Thread.sleep(ConnectionUtils.getPauseTime(pause, tries)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Giving up after tries=" + tries, e); + } + } + return null; + } + + /** + * Run this instance against the server once. + * @param the type of the return value + * @return an object of type T + * @throws IOException if a remote or network exception occurs + * @throws RuntimeException other unspecified error + */ + public T withoutRetries() + throws IOException, RuntimeException { + try { + beforeCall(); + connect(false); + return call(); + } catch (Throwable t) { + Throwable t2 = translateException(t); + if (t2 instanceof IOException) { + throw (IOException)t2; + } else { + throw new RuntimeException(t2); + } + } finally { + afterCall(); + } + } + + private static Throwable translateException(Throwable t) throws IOException { + if (t instanceof UndeclaredThrowableException) { + t = t.getCause(); + } + if (t instanceof RemoteException) { + t = ((RemoteException)t).unwrapRemoteException(); + } + if (t instanceof DoNotRetryIOException) { + throw (DoNotRetryIOException)t; + } + return t; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHColumnDescriptor.java b/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHColumnDescriptor.java new file mode 100644 index 0000000..301ea12 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHColumnDescriptor.java @@ -0,0 +1,93 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.io.hfile.Compression; + +/** + * Immutable HColumnDescriptor + */ +public class UnmodifyableHColumnDescriptor extends HColumnDescriptor { + + /** + * @param desc wrapped + */ + public UnmodifyableHColumnDescriptor (final HColumnDescriptor desc) { + super(desc); + } + + /** + * @see org.apache.hadoop.hbase.HColumnDescriptor#setValue(byte[], byte[]) + */ + @Override + public HColumnDescriptor setValue(byte[] key, byte[] value) { + throw new UnsupportedOperationException("HColumnDescriptor is read-only"); + } + + /** + * @see org.apache.hadoop.hbase.HColumnDescriptor#setValue(java.lang.String, java.lang.String) + */ + @Override + public HColumnDescriptor setValue(String key, String value) { + throw new UnsupportedOperationException("HColumnDescriptor is read-only"); + } + + /** + * @see org.apache.hadoop.hbase.HColumnDescriptor#setMaxVersions(int) + */ + @Override + public HColumnDescriptor setMaxVersions(int maxVersions) { + throw new UnsupportedOperationException("HColumnDescriptor is read-only"); + } + + /** + * @see org.apache.hadoop.hbase.HColumnDescriptor#setInMemory(boolean) + */ + @Override + public HColumnDescriptor setInMemory(boolean inMemory) { + throw new UnsupportedOperationException("HColumnDescriptor is read-only"); + } + + /** + * @see org.apache.hadoop.hbase.HColumnDescriptor#setBlockCacheEnabled(boolean) + */ + @Override + public HColumnDescriptor setBlockCacheEnabled(boolean blockCacheEnabled) { + throw new UnsupportedOperationException("HColumnDescriptor is read-only"); + } + + /** + * @see org.apache.hadoop.hbase.HColumnDescriptor#setTimeToLive(int) + */ + @Override + public HColumnDescriptor setTimeToLive(int timeToLive) { + throw new UnsupportedOperationException("HColumnDescriptor is read-only"); + } + + /** + * @see org.apache.hadoop.hbase.HColumnDescriptor#setCompressionType(org.apache.hadoop.hbase.io.hfile.Compression.Algorithm) + */ + @Override + public HColumnDescriptor setCompressionType(Compression.Algorithm type) { + throw new UnsupportedOperationException("HColumnDescriptor is read-only"); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHRegionInfo.java new file mode 100644 index 0000000..412f770 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHRegionInfo.java @@ -0,0 +1,50 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HRegionInfo; + +class UnmodifyableHRegionInfo extends HRegionInfo { + /* + * Creates an unmodifyable copy of an HRegionInfo + * + * @param info + */ + UnmodifyableHRegionInfo(HRegionInfo info) { + super(info); + } + + /** + * @param split set split status + */ + @Override + public void setSplit(boolean split) { + throw new UnsupportedOperationException("HRegionInfo is read-only"); + } + + /** + * @param offLine set online - offline status + */ + @Override + public void setOffline(boolean offLine) { + throw new UnsupportedOperationException("HRegionInfo is read-only"); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHTableDescriptor.java new file mode 100644 index 0000000..27d1faa --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHTableDescriptor.java @@ -0,0 +1,124 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; + +/** + * Read-only table descriptor. + */ +public class UnmodifyableHTableDescriptor extends HTableDescriptor { + /** Default constructor */ + public UnmodifyableHTableDescriptor() { + super(); + } + + /* + * Create an unmodifyable copy of an HTableDescriptor + * @param desc + */ + UnmodifyableHTableDescriptor(final HTableDescriptor desc) { + super(desc.getName(), getUnmodifyableFamilies(desc), desc.getValues()); + } + + + /* + * @param desc + * @return Families as unmodifiable array. + */ + private static HColumnDescriptor[] getUnmodifyableFamilies( + final HTableDescriptor desc) { + HColumnDescriptor [] f = new HColumnDescriptor[desc.getFamilies().size()]; + int i = 0; + for (HColumnDescriptor c: desc.getFamilies()) { + f[i++] = c; + } + return f; + } + + /** + * Does NOT add a column family. This object is immutable + * @param family HColumnDescriptor of familyto add. + */ + @Override + public void addFamily(final HColumnDescriptor family) { + throw new UnsupportedOperationException("HTableDescriptor is read-only"); + } + + /** + * @param column + * @return Column descriptor for the passed family name or the family on + * passed in column. + */ + @Override + public HColumnDescriptor removeFamily(final byte [] column) { + throw new UnsupportedOperationException("HTableDescriptor is read-only"); + } + + /** + * @see org.apache.hadoop.hbase.HTableDescriptor#setReadOnly(boolean) + */ + @Override + public void setReadOnly(boolean readOnly) { + throw new UnsupportedOperationException("HTableDescriptor is read-only"); + } + + /** + * @see org.apache.hadoop.hbase.HTableDescriptor#setValue(byte[], byte[]) + */ + @Override + public void setValue(byte[] key, byte[] value) { + throw new UnsupportedOperationException("HTableDescriptor is read-only"); + } + + /** + * @see org.apache.hadoop.hbase.HTableDescriptor#setValue(java.lang.String, java.lang.String) + */ + @Override + public void setValue(String key, String value) { + throw new UnsupportedOperationException("HTableDescriptor is read-only"); + } + + /** + * @see org.apache.hadoop.hbase.HTableDescriptor#setMaxFileSize(long) + */ + @Override + public void setMaxFileSize(long maxFileSize) { + throw new UnsupportedOperationException("HTableDescriptor is read-only"); + } + + /** + * @see org.apache.hadoop.hbase.HTableDescriptor#setMemStoreFlushSize(long) + */ + @Override + public void setMemStoreFlushSize(long memstoreFlushSize) { + throw new UnsupportedOperationException("HTableDescriptor is read-only"); + } + +// /** +// * @see org.apache.hadoop.hbase.HTableDescriptor#addIndex(org.apache.hadoop.hbase.client.tableindexed.IndexSpecification) +// */ +// @Override +// public void addIndex(IndexSpecification index) { +// throw new UnsupportedOperationException("HTableDescriptor is read-only"); +// } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java new file mode 100644 index 0000000..4974a29 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java @@ -0,0 +1,557 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client.coprocessor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.NavigableSet; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.AggregateProtocol; +import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +/** + * This client class is for invoking the aggregate functions deployed on the + * Region Server side via the AggregateProtocol. This class will implement the + * supporting functionality for summing/processing the individual results + * obtained from the AggregateProtocol for each region. + *

+ * This will serve as the client side handler for invoking the aggregate + * functions. + *

    + * For all aggregate functions, + *
  • start row < end row is an essential condition (if they are not + * {@link HConstants#EMPTY_BYTE_ARRAY}) + *
  • Column family can't be null. In case where multiple families are + * provided, an IOException will be thrown. An optional column qualifier can + * also be defined. + *
  • For methods to find maximum, minimum, sum, rowcount, it returns the + * parameter type. For average and std, it returns a double value. For row + * count, it returns a long value. + */ +public class AggregationClient { + + private static final Log log = LogFactory.getLog(AggregationClient.class); + Configuration conf; + + /** + * Constructor with Conf object + * @param cfg + */ + public AggregationClient(Configuration cfg) { + this.conf = cfg; + } + + /** + * It gives the maximum value of a column for a given column family for the + * given range. In case qualifier is null, a max of all values for the given + * family is returned. + * @param tableName + * @param ci + * @param scan + * @return max val + * @throws Throwable + * The caller is supposed to handle the exception as they are thrown + * & propagated to it. + */ + public R max(final byte[] tableName, final ColumnInterpreter ci, + final Scan scan) throws Throwable { + validateParameters(scan); + class MaxCallBack implements Batch.Callback { + R max = null; + + R getMax() { + return max; + } + + @Override + public synchronized void update(byte[] region, byte[] row, R result) { + max = (max == null || (result != null && ci.compare(max, result) < 0)) ? result : max; + } + } + MaxCallBack aMaxCallBack = new MaxCallBack(); + HTable table = null; + try { + table = new HTable(conf, tableName); + table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), + scan.getStopRow(), new Batch.Call() { + @Override + public R call(AggregateProtocol instance) throws IOException { + return instance.getMax(ci, scan); + } + }, aMaxCallBack); + } finally { + if (table != null) { + table.close(); + } + } + return aMaxCallBack.getMax(); + } + + private void validateParameters(Scan scan) throws IOException { + if (scan == null + || (Bytes.equals(scan.getStartRow(), scan.getStopRow()) && !Bytes + .equals(scan.getStartRow(), HConstants.EMPTY_START_ROW)) + || ((Bytes.compareTo(scan.getStartRow(), scan.getStopRow()) > 0) && + !Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW))) { + throw new IOException( + "Agg client Exception: Startrow should be smaller than Stoprow"); + } else if (scan.getFamilyMap().size() != 1) { + throw new IOException("There must be only one family."); + } + } + + /** + * It gives the minimum value of a column for a given column family for the + * given range. In case qualifier is null, a min of all values for the given + * family is returned. + * @param tableName + * @param ci + * @param scan + * @return min val + * @throws Throwable + */ + public R min(final byte[] tableName, final ColumnInterpreter ci, + final Scan scan) throws Throwable { + validateParameters(scan); + class MinCallBack implements Batch.Callback { + + private R min = null; + + public R getMinimum() { + return min; + } + + @Override + public synchronized void update(byte[] region, byte[] row, R result) { + min = (min == null || (result != null && ci.compare(result, min) < 0)) ? result : min; + } + } + MinCallBack minCallBack = new MinCallBack(); + HTable table = null; + try { + table = new HTable(conf, tableName); + table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), + scan.getStopRow(), new Batch.Call() { + + @Override + public R call(AggregateProtocol instance) throws IOException { + return instance.getMin(ci, scan); + } + }, minCallBack); + } finally { + if (table != null) { + table.close(); + } + } + log.debug("Min fom all regions is: " + minCallBack.getMinimum()); + return minCallBack.getMinimum(); + } + + /** + * It gives the row count, by summing up the individual results obtained from + * regions. In case the qualifier is null, FirstKEyValueFilter is used to + * optimised the operation. In case qualifier is provided, I can't use the + * filter as it may set the flag to skip to next row, but the value read is + * not of the given filter: in this case, this particular row will not be + * counted ==> an error. + * @param tableName + * @param ci + * @param scan + * @return + * @throws Throwable + */ + public long rowCount(final byte[] tableName, + final ColumnInterpreter ci, final Scan scan) throws Throwable { + validateParameters(scan); + class RowNumCallback implements Batch.Callback { + private final AtomicLong rowCountL = new AtomicLong(0); + + public long getRowNumCount() { + return rowCountL.get(); + } + + @Override + public void update(byte[] region, byte[] row, Long result) { + rowCountL.addAndGet(result.longValue()); + } + } + RowNumCallback rowNum = new RowNumCallback(); + HTable table = null; + try { + table = new HTable(conf, tableName); + table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), + scan.getStopRow(), new Batch.Call() { + @Override + public Long call(AggregateProtocol instance) throws IOException { + return instance.getRowNum(ci, scan); + } + }, rowNum); + } finally { + if (table != null) { + table.close(); + } + } + return rowNum.getRowNumCount(); + } + + /** + * It sums up the value returned from various regions. In case qualifier is + * null, summation of all the column qualifiers in the given family is done. + * @param tableName + * @param ci + * @param scan + * @return sum + * @throws Throwable + */ + public S sum(final byte[] tableName, final ColumnInterpreter ci, + final Scan scan) throws Throwable { + validateParameters(scan); + class SumCallBack implements Batch.Callback { + S sumVal = null; + + public S getSumResult() { + return sumVal; + } + + @Override + public synchronized void update(byte[] region, byte[] row, S result) { + sumVal = ci.add(sumVal, result); + } + } + SumCallBack sumCallBack = new SumCallBack(); + HTable table = null; + try { + table = new HTable(conf, tableName); + table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), + scan.getStopRow(), new Batch.Call() { + @Override + public S call(AggregateProtocol instance) throws IOException { + return instance.getSum(ci, scan); + } + }, sumCallBack); + } finally { + if (table != null) { + table.close(); + } + } + return sumCallBack.getSumResult(); + } + + /** + * It computes average while fetching sum and row count from all the + * corresponding regions. Approach is to compute a global sum of region level + * sum and rowcount and then compute the average. + * @param tableName + * @param scan + * @throws Throwable + */ + private Pair getAvgArgs(final byte[] tableName, + final ColumnInterpreter ci, final Scan scan) throws Throwable { + validateParameters(scan); + class AvgCallBack implements Batch.Callback> { + S sum = null; + Long rowCount = 0l; + + public Pair getAvgArgs() { + return new Pair(sum, rowCount); + } + + @Override + public synchronized void update(byte[] region, byte[] row, Pair result) { + sum = ci.add(sum, result.getFirst()); + rowCount += result.getSecond(); + } + } + AvgCallBack avgCallBack = new AvgCallBack(); + HTable table = null; + try { + table = new HTable(conf, tableName); + table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), + scan.getStopRow(), + new Batch.Call>() { + @Override + public Pair call(AggregateProtocol instance) + throws IOException { + return instance.getAvg(ci, scan); + } + }, avgCallBack); + } finally { + if (table != null) { + table.close(); + } + } + return avgCallBack.getAvgArgs(); + } + + /** + * This is the client side interface/handle for calling the average method for + * a given cf-cq combination. It was necessary to add one more call stack as + * its return type should be a decimal value, irrespective of what + * columninterpreter says. So, this methods collects the necessary parameters + * to compute the average and returs the double value. + * @param tableName + * @param ci + * @param scan + * @return + * @throws Throwable + */ + public double avg(final byte[] tableName, + final ColumnInterpreter ci, Scan scan) throws Throwable { + Pair p = getAvgArgs(tableName, ci, scan); + return ci.divideForAvg(p.getFirst(), p.getSecond()); + } + + /** + * It computes a global standard deviation for a given column and its value. + * Standard deviation is square root of (average of squares - + * average*average). From individual regions, it obtains sum, square sum and + * number of rows. With these, the above values are computed to get the global + * std. + * @param tableName + * @param scan + * @return + * @throws Throwable + */ + private Pair, Long> getStdArgs(final byte[] tableName, + final ColumnInterpreter ci, final Scan scan) throws Throwable { + validateParameters(scan); + class StdCallback implements Batch.Callback, Long>> { + long rowCountVal = 0l; + S sumVal = null, sumSqVal = null; + + public Pair, Long> getStdParams() { + List l = new ArrayList(); + l.add(sumVal); + l.add(sumSqVal); + Pair, Long> p = new Pair, Long>(l, rowCountVal); + return p; + } + + @Override + public synchronized void update(byte[] region, byte[] row, Pair, Long> result) { + sumVal = ci.add(sumVal, result.getFirst().get(0)); + sumSqVal = ci.add(sumSqVal, result.getFirst().get(1)); + rowCountVal += result.getSecond(); + } + } + StdCallback stdCallback = new StdCallback(); + HTable table = null; + try { + table = new HTable(conf, tableName); + table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), + scan.getStopRow(), + new Batch.Call, Long>>() { + @Override + public Pair, Long> call(AggregateProtocol instance) + throws IOException { + return instance.getStd(ci, scan); + } + + }, stdCallback); + } finally { + if (table != null) { + table.close(); + } + } + return stdCallback.getStdParams(); + } + + /** + * This is the client side interface/handle for calling the std method for a + * given cf-cq combination. It was necessary to add one more call stack as its + * return type should be a decimal value, irrespective of what + * columninterpreter says. So, this methods collects the necessary parameters + * to compute the std and returns the double value. + * @param tableName + * @param ci + * @param scan + * @return + * @throws Throwable + */ + public double std(final byte[] tableName, ColumnInterpreter ci, + Scan scan) throws Throwable { + Pair, Long> p = getStdArgs(tableName, ci, scan); + double res = 0d; + double avg = ci.divideForAvg(p.getFirst().get(0), p.getSecond()); + double avgOfSumSq = ci.divideForAvg(p.getFirst().get(1), p.getSecond()); + res = avgOfSumSq - (avg) * (avg); // variance + res = Math.pow(res, 0.5); + return res; + } + + /** + * It helps locate the region with median for a given column whose weight + * is specified in an optional column. + * From individual regions, it obtains sum of values and sum of weights. + * @param tableName + * @param ci + * @param scan + * @return pair whose first element is a map between start row of the region + * and (sum of values, sum of weights) for the region, the second element is + * (sum of values, sum of weights) for all the regions chosen + * @throws Throwable + */ + private Pair>, List> + getMedianArgs(final byte[] tableName, + final ColumnInterpreter ci, final Scan scan) throws Throwable { + validateParameters(scan); + final NavigableMap> map = + new TreeMap>(Bytes.BYTES_COMPARATOR); + class StdCallback implements Batch.Callback> { + S sumVal = null, sumWeights = null; + + public Pair>, List> getMedianParams() { + List l = new ArrayList(); + l.add(sumVal); + l.add(sumWeights); + Pair>, List> p = + new Pair>, List>(map, l); + return p; + } + + @Override + public synchronized void update(byte[] region, byte[] row, List result) { + map.put(row, result); + sumVal = ci.add(sumVal, result.get(0)); + sumWeights = ci.add(sumWeights, result.get(1)); + } + } + StdCallback stdCallback = new StdCallback(); + HTable table = null; + try { + table = new HTable(conf, tableName); + table.coprocessorExec(AggregateProtocol.class, scan.getStartRow(), + scan.getStopRow(), new Batch.Call>() { + @Override + public List call(AggregateProtocol instance) throws IOException { + return instance.getMedian(ci, scan); + } + + }, stdCallback); + } finally { + if (table != null) { + table.close(); + } + } + return stdCallback.getMedianParams(); + } + + /** + * This is the client side interface/handler for calling the median method for a + * given cf-cq combination. This method collects the necessary parameters + * to compute the median and returns the median. + * @param tableName + * @param ci + * @param scan + * @return R the median + * @throws Throwable + */ + public R median(final byte[] tableName, ColumnInterpreter ci, + Scan scan) throws Throwable { + Pair>, List> p = getMedianArgs(tableName, ci, scan); + byte[] startRow = null; + byte[] colFamily = scan.getFamilies()[0]; + NavigableSet quals = scan.getFamilyMap().get(colFamily); + NavigableMap> map = p.getFirst(); + S sumVal = p.getSecond().get(0); + S sumWeights = p.getSecond().get(1); + double halfSumVal = ci.divideForAvg(sumVal, 2L); + double movingSumVal = 0; + boolean weighted = false; + if (quals.size() > 1) { + weighted = true; + halfSumVal = ci.divideForAvg(sumWeights, 2L); + } + + for (Map.Entry> entry : map.entrySet()) { + S s = weighted ? entry.getValue().get(1) : entry.getValue().get(0); + double newSumVal = movingSumVal + ci.divideForAvg(s, 1L); + if (newSumVal > halfSumVal) break; // we found the region with the median + movingSumVal = newSumVal; + startRow = entry.getKey(); + } + // scan the region with median and find it + Scan scan2 = new Scan(scan); + // inherit stop row from method parameter + if (startRow != null) scan2.setStartRow(startRow); + HTable table = null; + ResultScanner scanner = null; + try { + table = new HTable(conf, tableName); + int cacheSize = scan2.getCaching(); + if (!scan2.getCacheBlocks() || scan2.getCaching() < 2) { + scan2.setCacheBlocks(true); + cacheSize = 5; + scan2.setCaching(cacheSize); + } + scanner = table.getScanner(scan2); + Result[] results = null; + byte[] qualifier = quals.pollFirst(); + // qualifier for the weight column + byte[] weightQualifier = weighted ? quals.pollLast() : qualifier; + R value = null; + do { + results = scanner.next(cacheSize); + if (results != null && results.length > 0) { + for (int i = 0; i < results.length; i++) { + Result r = results[i]; + // retrieve weight + KeyValue kv = r.getColumnLatest(colFamily, weightQualifier); + R newValue = ci.getValue(colFamily, weightQualifier, kv); + S s = ci.castToReturnType(newValue); + double newSumVal = movingSumVal + ci.divideForAvg(s, 1L); + // see if we have moved past the median + if (newSumVal > halfSumVal) { + return value; + } + movingSumVal = newSumVal; + kv = r.getColumnLatest(colFamily, qualifier); + value = ci.getValue(colFamily, qualifier, kv); + } + } + } while (results != null && results.length > 0); + } finally { + if (scanner != null) { + scanner.close(); + } + if (table != null) { + table.close(); + } + } + return null; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Batch.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Batch.java new file mode 100644 index 0000000..d430a38 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Batch.java @@ -0,0 +1,166 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client.coprocessor; + +import org.apache.commons.lang.reflect.MethodUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; + +import java.io.IOException; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; + + +/** + * A collection of interfaces and utilities used for interacting with custom RPC + * interfaces exposed by Coprocessors. + */ +public abstract class Batch { + private static Log LOG = LogFactory.getLog(Batch.class); + + /** + * Creates a new {@link Batch.Call} instance that invokes a method + * with the given parameters and returns the result. + * + *

    + * Note that currently the method is naively looked up using the method name + * and class types of the passed arguments, which means that + * none of the arguments can be null. + * For more flexibility, see + * {@link Batch#forMethod(java.lang.reflect.Method, Object...)}. + *

    + * + * @param protocol the protocol class being called + * @param method the method name + * @param args zero or more arguments to be passed to the method + * (individual args cannot be null!) + * @param the class type of the protocol implementation being invoked + * @param the return type for the method call + * @return a {@code Callable} instance that will invoke the given method + * and return the results + * @throws NoSuchMethodException if the method named, with the given argument + * types, cannot be found in the protocol class + * @see Batch#forMethod(java.lang.reflect.Method, Object...) + * @see org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call, org.apache.hadoop.hbase.client.coprocessor.Batch.Callback) + */ + public static Call forMethod( + final Class protocol, final String method, final Object... args) + throws NoSuchMethodException { + Class[] types = new Class[args.length]; + for (int i=0; i the class type of the protocol implementation being invoked + * @param the return type for the method call + * @return a {@code Callable} instance that will invoke the given method and + * return the results + * @see org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call, org.apache.hadoop.hbase.client.coprocessor.Batch.Callback) + */ + public static Call forMethod( + final Method method, final Object... args) { + return new Call() { + public R call(T instance) throws IOException { + try { + if (Proxy.isProxyClass(instance.getClass())) { + InvocationHandler invoker = Proxy.getInvocationHandler(instance); + return (R)invoker.invoke(instance, method, args); + } else { + LOG.warn("Non proxied invocation of method '"+method.getName()+"'!"); + return (R)method.invoke(instance, args); + } + } + catch (IllegalAccessException iae) { + throw new IOException("Unable to invoke method '"+ + method.getName()+"'", iae); + } + catch (InvocationTargetException ite) { + throw new IOException(ite.toString(), ite); + } + catch (Throwable t) { + throw new IOException(t.toString(), t); + } + } + }; + } + + /** + * Defines a unit of work to be executed. + * + *

    + * When used with + * {@link org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call, org.apache.hadoop.hbase.client.coprocessor.Batch.Callback)} + * the implementations {@link Batch.Call#call(Object)} method will be invoked + * with a proxy to the + * {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol} + * sub-type instance. + *

    + * @see org.apache.hadoop.hbase.client.coprocessor + * @see org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call) + * @see org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call, org.apache.hadoop.hbase.client.coprocessor.Batch.Callback) + * @param the instance type to be passed to + * {@link Batch.Call#call(Object)} + * @param the return type from {@link Batch.Call#call(Object)} + */ + public static interface Call { + public R call(T instance) throws IOException; + } + + /** + * Defines a generic callback to be triggered for each {@link Batch.Call#call(Object)} + * result. + * + *

    + * When used with + * {@link org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call, org.apache.hadoop.hbase.client.coprocessor.Batch.Callback)}, + * the implementation's {@link Batch.Callback#update(byte[], byte[], Object)} + * method will be called with the {@link Batch.Call#call(Object)} return value + * from each region in the selected range. + *

    + * @param the return type from the associated {@link Batch.Call#call(Object)} + * @see org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call, org.apache.hadoop.hbase.client.coprocessor.Batch.Callback) + */ + public static interface Callback { + public void update(byte[] region, byte[] row, R result); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java new file mode 100644 index 0000000..592e792 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java @@ -0,0 +1,104 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client.coprocessor; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.RoundingMode; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * ColumnInterpreter for doing Aggregation's with BigDecimal columns. + * This class is required at the RegionServer also. + * + */ +public class BigDecimalColumnInterpreter implements ColumnInterpreter { + private static final Log log = LogFactory.getLog(BigDecimalColumnInterpreter.class); + + @Override + public void readFields(DataInput arg0) throws IOException { + } + + @Override + public void write(DataOutput arg0) throws IOException { + } + + @Override + public BigDecimal getValue(byte[] family, byte[] qualifier, KeyValue kv) + throws IOException { + if ((kv == null || kv.getValue() == null)) return null; + return Bytes.toBigDecimal(kv.getValue()).setScale(2, RoundingMode.HALF_EVEN); + } + + @Override + public BigDecimal add(BigDecimal val1, BigDecimal val2) { + if ((((val1 == null) ? 1 : 0) ^ ((val2 == null) ? 1 : 0)) != 0) { + return ((val1 == null) ? val2 : val1); + } + if (val1 == null) return null; + return val1.add(val2).setScale(2, RoundingMode.HALF_EVEN); + } + + @Override + public BigDecimal getMaxValue() { + return BigDecimal.valueOf(Double.MAX_VALUE); + } + + @Override + public BigDecimal getMinValue() { + return BigDecimal.valueOf(Double.MIN_VALUE); + } + + @Override + public BigDecimal multiply(BigDecimal val1, BigDecimal val2) { + return (((val1 == null) || (val2 == null)) ? null : val1.multiply(val2).setScale(2, + RoundingMode.HALF_EVEN)); + } + + @Override + public BigDecimal increment(BigDecimal val) { + return ((val == null) ? null : val.add(BigDecimal.ONE)); + } + + @Override + public BigDecimal castToReturnType(BigDecimal val) { + return val; + } + + @Override + public int compare(BigDecimal val1, BigDecimal val2) { + if ((((val1 == null) ? 1 : 0) ^ ((val2 == null) ? 1 : 0)) != 0) { + return ((val1 == null) ? -1 : 1); + } + if (val1 == null) return 0; + return val1.compareTo(val2); + } + + @Override + public double divideForAvg(BigDecimal val1, Long paramLong) { + return (((paramLong == null) || (val1 == null)) ? (Double.NaN) : + val1.doubleValue() / paramLong.doubleValue()); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java new file mode 100644 index 0000000..cafa8b0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Exec.java @@ -0,0 +1,135 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client.coprocessor; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.Row; +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.ipc.Invocation; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Classes; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.lang.reflect.Method; + +/** + * Represents an arbitrary method invocation against a Coprocessor + * instance. In order for a coprocessor implementation to be remotely callable + * by clients, it must define and implement a {@link CoprocessorProtocol} + * subclass. Only methods defined in the {@code CoprocessorProtocol} interface + * will be callable by clients. + * + *

    + * This class is used internally by + * {@link org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call, org.apache.hadoop.hbase.client.coprocessor.Batch.Callback)} + * to wrap the {@code CoprocessorProtocol} method invocations requested in + * RPC calls. It should not be used directly by HBase clients. + *

    + * + * @see ExecResult + * @see org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call) + * @see org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call, org.apache.hadoop.hbase.client.coprocessor.Batch.Callback) + */ +public class Exec extends Invocation implements Row { + /** Row key used as a reference for any region lookups */ + private byte[] referenceRow; + private Class protocol; + private String protocolName; + + public Exec() { + } + + public Exec(Configuration configuration, + Class protocol, + Method method, Object[] parameters) { + this(configuration, HConstants.EMPTY_BYTE_ARRAY, + protocol, method, parameters); + } + + public Exec(Configuration configuration, + byte[] row, + Class protocol, + Method method, Object[] parameters) { + super(method, protocol, parameters); + this.conf = configuration; + this.referenceRow = row; + this.protocol = protocol; + this.protocolName = protocol.getName(); + } + + public String getProtocolName() { + return protocolName; + } + + public Class getProtocol() { + return protocol; + } + + public byte[] getRow() { + return referenceRow; + } + + public int compareTo(Row row) { + return Bytes.compareTo(referenceRow, row.getRow()); + } + + @Override + public void write(DataOutput out) throws IOException { + // fields for Invocation + out.writeUTF(this.methodName); + out.writeInt(parameterClasses.length); + for (int i = 0; i < parameterClasses.length; i++) { + HbaseObjectWritable.writeObject(out, parameters[i], + parameters[i] != null ? parameters[i].getClass() : parameterClasses[i], + conf); + out.writeUTF(parameterClasses[i].getName()); + } + // fields for Exec + Bytes.writeByteArray(out, referenceRow); + out.writeUTF(protocol.getName()); + } + + @Override + public void readFields(DataInput in) throws IOException { + // fields for Invocation + methodName = in.readUTF(); + parameters = new Object[in.readInt()]; + parameterClasses = new Class[parameters.length]; + HbaseObjectWritable objectWritable = new HbaseObjectWritable(); + for (int i = 0; i < parameters.length; i++) { + parameters[i] = HbaseObjectWritable.readObject(in, objectWritable, + this.conf); + String parameterClassName = in.readUTF(); + try { + parameterClasses[i] = Classes.extendedForName(parameterClassName); + } catch (ClassNotFoundException e) { + throw new IOException("Couldn't find class: " + parameterClassName); + } + } + // fields for Exec + referenceRow = Bytes.readByteArray(in); + protocolName = in.readUTF(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/ExecResult.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/ExecResult.java new file mode 100644 index 0000000..e4812af --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/ExecResult.java @@ -0,0 +1,85 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client.coprocessor; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Classes; +import org.apache.hadoop.io.Writable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.Serializable; + +/** + * Represents the return value from a + * {@link org.apache.hadoop.hbase.client.coprocessor.Exec} invocation. + * This simply wraps the value for easier + * {@link org.apache.hadoop.hbase.io.HbaseObjectWritable} + * serialization. + * + *

    + * This class is used internally by the HBase client code to properly serialize + * responses from {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol} + * method invocations. It should not be used directly by clients. + *

    + * + * @see Exec + * @see org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call) + * @see org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call, org.apache.hadoop.hbase.client.coprocessor.Batch.Callback) + */ +public class ExecResult implements Writable { + private byte[] regionName; + private Object value; + + public ExecResult() { + } + + public ExecResult(Object value) { + this(HConstants.EMPTY_BYTE_ARRAY, value); + } + public ExecResult(byte[] region, Object value) { + this.regionName = region; + this.value = value; + } + + public byte[] getRegionName() { + return regionName; + } + + public Object getValue() { + return value; + } + + @Override + public void write(DataOutput out) throws IOException { + Bytes.writeByteArray(out, regionName); + HbaseObjectWritable.writeObject(out, value, + value != null ? value.getClass() : Writable.class, null); + } + + @Override + public void readFields(DataInput in) throws IOException { + regionName = Bytes.readByteArray(in); + value = HbaseObjectWritable.readObject(in, null); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java new file mode 100644 index 0000000..c37b5fd --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java @@ -0,0 +1,106 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client.coprocessor; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * a concrete column interpreter implementation. The cell value is a Long value + * and its promoted data type is also a Long value. For computing aggregation + * function, this class is used to find the datatype of the cell value. Client + * is supposed to instantiate it and passed along as a parameter. See + * TestAggregateProtocol methods for its sample usage. + * Its methods handle null arguments gracefully. + */ +public class LongColumnInterpreter implements ColumnInterpreter { + + public Long getValue(byte[] colFamily, byte[] colQualifier, KeyValue kv) + throws IOException { + if (kv == null || kv.getValueLength() != Bytes.SIZEOF_LONG) + return null; + return Bytes.toLong(kv.getBuffer(), kv.getValueOffset()); + } + + @Override + public Long add(Long l1, Long l2) { + if (l1 == null ^ l2 == null) { + return (l1 == null) ? l2 : l1; // either of one is null. + } else if (l1 == null) // both are null + return null; + return l1 + l2; + } + + @Override + public int compare(final Long l1, final Long l2) { + if (l1 == null ^ l2 == null) { + return l1 == null ? -1 : 1; // either of one is null. + } else if (l1 == null) + return 0; // both are null + return l1.compareTo(l2); // natural ordering. + } + + @Override + public Long getMaxValue() { + return Long.MAX_VALUE; + } + + @Override + public Long increment(Long o) { + return o == null ? null : (o + 1l); + } + + @Override + public Long multiply(Long l1, Long l2) { + return (l1 == null || l2 == null) ? null : l1 * l2; + } + + @Override + public Long getMinValue() { + return Long.MIN_VALUE; + } + + @Override + public void readFields(DataInput arg0) throws IOException { + // nothing to serialize + } + + @Override + public void write(DataOutput arg0) throws IOException { + // nothing to serialize + } + + @Override + public double divideForAvg(Long l1, Long l2) { + return (l2 == null || l1 == null) ? Double.NaN : (l1.doubleValue() / l2 + .doubleValue()); + } + + @Override + public Long castToReturnType(Long o) { + return o; + } + +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/client/coprocessor/package-info.java b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/package-info.java new file mode 100644 index 0000000..8b0fbdf --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/coprocessor/package-info.java @@ -0,0 +1,200 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** +Provides client classes for invoking Coprocessor RPC protocols + +

    +

     +

    + +

    Overview

    +

    +The coprocessor framework provides a way for custom code to run in place on the +HBase region servers with each of a table's regions. These client classes +enable applications to communicate with coprocessor instances via custom RPC +protocols. +

    + +

    +In order to provide a custom RPC protocol to clients, a coprocessor implementation +defines an interface that extends {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol}. +The interface can define any methods that the coprocessor wishes to expose. +Using this protocol, you can communicate with the coprocessor instances via +the {@link org.apache.hadoop.hbase.client.HTable#coprocessorProxy(Class, byte[])} and +{@link org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call, org.apache.hadoop.hbase.client.coprocessor.Batch.Callback)} +methods. +

    + +

    +Since {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol} instances are +associated with individual regions within the table, the client RPC calls +must ultimately identify which regions should be used in the CoprocessorProtocol +method invocations. Since regions are seldom handled directly in client code +and the region names may change over time, the coprocessor RPC calls use row keys +to identify which regions should be used for the method invocations. Clients +can call CoprocessorProtocol methods against either: +

      +
    • a single region - calling + {@link org.apache.hadoop.hbase.client.HTable#coprocessorProxy(Class, byte[])} + with a single row key. This returns a dynamic proxy of the CoprocessorProtocol + interface which uses the region containing the given row key (even if the + row does not exist) as the RPC endpoint.
      • +
    • a range of regions - calling + {@link org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call, org.apache.hadoop.hbase.client.coprocessor.Batch.Callback)} + with a starting row key and an ending row key. All regions in the table + from the region containing the start row key to the region containing the end + row key (inclusive), will we used as the RPC endpoints.
      • +
    +

    + +

    Note that the row keys passed as parameters to the HTable +methods are not passed to the CoprocessorProtocol implementations. +They are only used to identify the regions for endpoints of the remote calls. +

    + +

    +The {@link org.apache.hadoop.hbase.client.coprocessor.Batch} class defines two +interfaces used for CoprocessorProtocol invocations against +multiple regions. Clients implement {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call} to +call methods of the actual CoprocessorProtocol instance. The interface's +call() method will be called once per selected region, passing the +CoprocessorProtocol instance for the region as a parameter. Clients +can optionally implement {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Callback} +to be notified of the results from each region invocation as they complete. +The instance's {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Callback#update(byte[], byte[], Object)} +method will be called with the {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call#call(Object)} +return value from each region. +

    + +

    Example usage

    +

    +To start with, let's use a fictitious coprocessor, RowCountCoprocessor +that counts the number of rows and key-values in each region where it is running. +For clients to query this information, the coprocessor defines and implements +the following {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol} extension +interface: +

    + +
    +
    +public interface RowCountProtocol extends CoprocessorProtocol {
+  long getRowCount();
+  long getRowCount(Filter filt);
+  long getKeyValueCount();
+}
+
    + +

    +Now we need a way to access the results that RowCountCoprocessor +is making available. If we want to find the row count for all regions, we could +use: +

    + +
    +
    +HTable table = new HTable("mytable");
+// find row count keyed by region name
+Map results = table.coprocessorExec(
+    RowCountProtocol.class, // the protocol interface we're invoking
+    null, null,             // start and end row keys
+    new Batch.Call() {
+       public Long call(RowCountProtocol counter) {
+         return counter.getRowCount();
+       }
+     });
+
    + +

    +This will return a java.util.Map of the counter.getRowCount() +result for the RowCountCoprocessor instance running in each region +of mytable, keyed by the region name. +

    + +

    +By implementing {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call} +as an anonymous class, we can invoke RowCountProtocol methods +directly against the {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call#call(Object)} +method's argument. Calling {@link org.apache.hadoop.hbase.client.HTable#coprocessorExec(Class, byte[], byte[], org.apache.hadoop.hbase.client.coprocessor.Batch.Call)} +will take care of invoking Batch.Call.call() against our anonymous class +with the RowCountCoprocessor instance for each table region. +

    + +

    +For this simple case, where we only want to obtain the result from a single +CoprocessorProtocol method, there's also a bit of syntactic sugar +we can use to cut down on the amount of code required: +

    + +
    +
    +HTable table = new HTable("mytable");
+Batch.Call call = Batch.forMethod(RowCountProtocol.class, "getRowCount");
+Map results = table.coprocessorExec(RowCountProtocol.class, null, null, call);
+
    + +

    +{@link org.apache.hadoop.hbase.client.coprocessor.Batch#forMethod(Class, String, Object...)} +is a simple factory method that will return a {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call} +instance that will call RowCountProtocol.getRowCount() for us +using reflection. +

    + +

    +However, if you want to perform additional processing on the results, +implementing {@link org.apache.hadoop.hbase.client.coprocessor.Batch.Call} +directly will provide more power and flexibility. For example, if you would +like to combine row count and key-value count for each region: +

    + +
    +
    +HTable table = new HTable("mytable");
+// combine row count and kv count for region
+Map> results = table.coprocessorExec(
+    RowCountProtocol.class,
+    null, null,
+    new Batch.Call>() {
+        public Pair call(RowCountProtocol counter) {
+          return new Pair(counter.getRowCount(), counter.getKeyValueCount());
+        }
+    });
+
    + +

    +Similarly, you could average the number of key-values per row for each region: +

    + +
    +
    +Map results = table.coprocessorExec(
+    RowCountProtocol.class,
+    null, null,
+    new Batch.Call() {
+        public Double call(RowCountProtocol counter) {
+          return ((double)counter.getKeyValueCount()) / ((double)counter.getRowCount());
+        }
+    });
+
    +*/ +package org.apache.hadoop.hbase.client.coprocessor; \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/client/metrics/ScanMetrics.java b/src/main/java/org/apache/hadoop/hbase/client/metrics/ScanMetrics.java new file mode 100644 index 0000000..c5faafb --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/metrics/ScanMetrics.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client.metrics; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.metrics.util.MetricsBase; +import org.apache.hadoop.metrics.util.MetricsRegistry; +import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong; + + +/** + * Provides client-side metrics related to scan operations + * The data can be passed to mapreduce framework or other systems. + * Currently metrics framework won't be able to support the scenario + * where multiple scan instances run on the same machine trying to + * update the same metric. We use metrics objects in the class, + * so that it can be easily switched to metrics framework later when it support + * this scenario. + * Some of these metrics are general for any client operation such as put + * However, there is no need for this. So they are defined under scan operation + * for now. + */ +public class ScanMetrics implements Writable { + + private static final byte SCANMETRICS_VERSION = (byte)1; + private static final Log LOG = LogFactory.getLog(ScanMetrics.class); + private MetricsRegistry registry = new MetricsRegistry(); + + /** + * number of RPC calls + */ + public final MetricsTimeVaryingLong countOfRPCcalls = + new MetricsTimeVaryingLong("RPC_CALLS", registry); + + /** + * number of remote RPC calls + */ + public final MetricsTimeVaryingLong countOfRemoteRPCcalls = + new MetricsTimeVaryingLong("REMOTE_RPC_CALLS", registry); + + /** + * sum of milliseconds between sequential next calls + */ + public final MetricsTimeVaryingLong sumOfMillisSecBetweenNexts = + new MetricsTimeVaryingLong("MILLIS_BETWEEN_NEXTS", registry); + + /** + * number of NotServingRegionException caught + */ + public final MetricsTimeVaryingLong countOfNSRE = + new MetricsTimeVaryingLong("NOT_SERVING_REGION_EXCEPTION", registry); + + /** + * number of bytes in Result objects from region servers + */ + public final MetricsTimeVaryingLong countOfBytesInResults = + new MetricsTimeVaryingLong("BYTES_IN_RESULTS", registry); + + /** + * number of bytes in Result objects from remote region servers + */ + public final MetricsTimeVaryingLong countOfBytesInRemoteResults = + new MetricsTimeVaryingLong("BYTES_IN_REMOTE_RESULTS", registry); + + /** + * number of regions + */ + public final MetricsTimeVaryingLong countOfRegions = + new MetricsTimeVaryingLong("REGIONS_SCANNED", registry); + + /** + * number of RPC retries + */ + public final MetricsTimeVaryingLong countOfRPCRetries = + new MetricsTimeVaryingLong("RPC_RETRIES", registry); + + /** + * number of remote RPC retries + */ + public final MetricsTimeVaryingLong countOfRemoteRPCRetries = + new MetricsTimeVaryingLong("REMOTE_RPC_RETRIES", registry); + + /** + * constructor + */ + public ScanMetrics () { + } + + /** + * serialize all the MetricsTimeVaryingLong + */ + public void write(DataOutput out) throws IOException { + out.writeByte(SCANMETRICS_VERSION); + Collection mbs = registry.getMetricsList(); + + // we only handle MetricsTimeVaryingLong for now. + int metricsCount = 0; + for (MetricsBase mb : mbs) { + if ( mb instanceof MetricsTimeVaryingLong) { + metricsCount++; + } else { + throw new IOException("unsupported metrics type. metrics name: " + + mb.getName() + ", metrics description: " + mb.getDescription()); + } + } + + out.writeInt(metricsCount); + for (MetricsBase mb : mbs) { + out.writeUTF(mb.getName()); + out.writeLong(((MetricsTimeVaryingLong) mb).getCurrentIntervalValue()); + } + } + + public void readFields(DataInput in) throws IOException { + int version = in.readByte(); + if (version > (int)SCANMETRICS_VERSION) { + throw new IOException("version " + version + " not supported"); + } + + int metricsCount = in.readInt(); + for (int i=0; i mbs = registry.getMetricsList(); + ArrayList mlv = + new ArrayList(); + for (MetricsBase mb : mbs) { + if ( mb instanceof MetricsTimeVaryingLong) { + mlv.add((MetricsTimeVaryingLong) mb); + } + } + return mlv.toArray(new MetricsTimeVaryingLong[mlv.size()]); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/client/package-info.java b/src/main/java/org/apache/hadoop/hbase/client/package-info.java new file mode 100644 index 0000000..b00a556 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/package-info.java @@ -0,0 +1,186 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** +Provides HBase Client + +

    Table of Contents

    + + +

    Overview

    +

    To administer HBase, create and drop tables, list and alter tables, + use {@link org.apache.hadoop.hbase.client.HBaseAdmin}. Once created, table access is via an instance + of {@link org.apache.hadoop.hbase.client.HTable}. You add content to a table a row at a time. To insert, + create an instance of a {@link org.apache.hadoop.hbase.client.Put} object. Specify value, target column + and optionally a timestamp. Commit your update using {@link org.apache.hadoop.hbase.client.HTable#put(Put)}. + To fetch your inserted value, use {@link org.apache.hadoop.hbase.client.Get}. The Get can be specified to be broad -- get all + on a particular row -- or narrow; i.e. return only a single cell value. After creating an instance of + Get, invoke {@link org.apache.hadoop.hbase.client.HTable#get(Get)}. Use + {@link org.apache.hadoop.hbase.client.Scan} to set up a scanner -- a Cursor- like access. After + creating and configuring your Scan instance, call {@link org.apache.hadoop.hbase.client.HTable#getScanner(Scan)} and then + invoke next on the returned object. Both {@link org.apache.hadoop.hbase.client.HTable#get(Get)} and + {@link org.apache.hadoop.hbase.client.HTable#getScanner(Scan)} return a +{@link org.apache.hadoop.hbase.client.Result}. +A Result is a List of {@link org.apache.hadoop.hbase.KeyValue}s. It has facility for packaging the return +in different formats. + Use {@link org.apache.hadoop.hbase.client.Delete} to remove content. + You can remove individual cells or entire families, etc. Pass it to + {@link org.apache.hadoop.hbase.client.HTable#delete(Delete)} to execute. +

    +

    Puts, Gets and Deletes take out a lock on the target row for the duration of their operation. + Concurrent modifications to a single row are serialized. Gets and scans run concurrently without + interference of the row locks and are guaranteed to not to return half written rows. +

    +

    Client code accessing a cluster finds the cluster by querying ZooKeeper. + This means that the ZooKeeper quorum to use must be on the client CLASSPATH. + Usually this means make sure the client can find your hbase-site.xml. +

    + +

    Example API Usage

    + +

    Once you have a running HBase, you probably want a way to hook your application up to it. + If your application is in Java, then you should use the Java API. Here's an example of what + a simple client might look like. This example assumes that you've created a table called + "myTable" with a column family called "myColumnFamily". +

    + +
    +
    +import java.io.IOException;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+
+
+// Class that has nothing but a main.
+// Does a Put, Get and a Scan against an hbase table.
+public class MyLittleHBaseClient {
+  public static void main(String[] args) throws IOException {
+    // You need a configuration object to tell the client where to connect.
+    // When you create a HBaseConfiguration, it reads in whatever you've set
+    // into your hbase-site.xml and in hbase-default.xml, as long as these can
+    // be found on the CLASSPATH
+    Configuration config = HBaseConfiguration.create();
+
+    // This instantiates an HTable object that connects you to
+    // the "myLittleHBaseTable" table.
+    HTable table = new HTable(config, "myLittleHBaseTable");
+
+    // To add to a row, use Put.  A Put constructor takes the name of the row
+    // you want to insert into as a byte array.  In HBase, the Bytes class has
+    // utility for converting all kinds of java types to byte arrays.  In the
+    // below, we are converting the String "myLittleRow" into a byte array to
+    // use as a row key for our update. Once you have a Put instance, you can
+    // adorn it by setting the names of columns you want to update on the row,
+    // the timestamp to use in your update, etc.If no timestamp, the server
+    // applies current time to the edits.
+    Put p = new Put(Bytes.toBytes("myLittleRow"));
+
+    // To set the value you'd like to update in the row 'myLittleRow', specify
+    // the column family, column qualifier, and value of the table cell you'd
+    // like to update.  The column family must already exist in your table
+    // schema.  The qualifier can be anything.  All must be specified as byte
+    // arrays as hbase is all about byte arrays.  Lets pretend the table
+    // 'myLittleHBaseTable' was created with a family 'myLittleFamily'.
+    p.add(Bytes.toBytes("myLittleFamily"), Bytes.toBytes("someQualifier"),
+      Bytes.toBytes("Some Value"));
+
+    // Once you've adorned your Put instance with all the updates you want to
+    // make, to commit it do the following (The HTable#put method takes the
+    // Put instance you've been building and pushes the changes you made into
+    // hbase)
+    table.put(p);
+
+    // Now, to retrieve the data we just wrote. The values that come back are
+    // Result instances. Generally, a Result is an object that will package up
+    // the hbase return into the form you find most palatable.
+    Get g = new Get(Bytes.toBytes("myLittleRow"));
+    Result r = table.get(g);
+    byte [] value = r.getValue(Bytes.toBytes("myLittleFamily"),
+      Bytes.toBytes("someQualifier"));
+    // If we convert the value bytes, we should get back 'Some Value', the
+    // value we inserted at this location.
+    String valueStr = Bytes.toString(value);
+    System.out.println("GET: " + valueStr);
+
+    // Sometimes, you won't know the row you're looking for. In this case, you
+    // use a Scanner. This will give you cursor-like interface to the contents
+    // of the table.  To set up a Scanner, do like you did above making a Put
+    // and a Get, create a Scan.  Adorn it with column names, etc.
+    Scan s = new Scan();
+    s.addColumn(Bytes.toBytes("myLittleFamily"), Bytes.toBytes("someQualifier"));
+    ResultScanner scanner = table.getScanner(s);
+    try {
+      // Scanners return Result instances.
+      // Now, for the actual iteration. One way is to use a while loop like so:
+      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
+        // print out the row we found and the columns we were looking for
+        System.out.println("Found row: " + rr);
+      }
+
+      // The other approach is to use a foreach loop. Scanners are iterable!
+      // for (Result rr : scanner) {
+      //   System.out.println("Found row: " + rr);
+      // }
+    } finally {
+      // Make sure you close your scanners when you are done!
+      // Thats why we have it inside a try/finally clause
+      scanner.close();
+    }
+  }
+}
+
    +
    + +

    There are many other methods for putting data into and getting data out of + HBase, but these examples should get you started. See the HTable javadoc for + more methods. Additionally, there are methods for managing tables in the + HBaseAdmin class.

    + +

    If your client is NOT Java, then you should consider the Thrift or REST + libraries.

    + +

    Related Documentation

    + +     + + +

    There are many other methods for putting data into and getting data out of + HBase, but these examples should get you started. See the HTable javadoc for + more methods. Additionally, there are methods for managing tables in the + HBaseAdmin class.

    + + + +*/ +package org.apache.hadoop.hbase.client; diff --git a/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java new file mode 100644 index 0000000..b4efc3f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java @@ -0,0 +1,202 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client.replication; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.replication.ReplicationZookeeper; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +/** + *

    + * This class provides the administrative interface to HBase cluster + * replication. In order to use it, the cluster and the client using + * ReplicationAdmin must be configured with hbase.replication + * set to true. + *

    + *

    + * Adding a new peer results in creating new outbound connections from every + * region server to a subset of region servers on the slave cluster. Each + * new stream of replication will start replicating from the beginning of the + * current HLog, meaning that edits from that past will be replicated. + *

    + *

    + * Removing a peer is a destructive and irreversible operation that stops + * all the replication streams for the given cluster and deletes the metadata + * used to keep track of the replication state. + *

    + *

    + * Enabling and disabling peers is currently not supported. + *

    + *

    + * As cluster replication is still experimental, a kill switch is provided + * in order to stop all replication-related operations, see + * {@link #setReplicating(boolean)}. When setting it back to true, the new + * state of all the replication streams will be unknown and may have holes. + * Use at your own risk. + *

    + *

    + * To see which commands are available in the shell, type + * replication. + *

    + */ +public class ReplicationAdmin implements Closeable { + + private final ReplicationZookeeper replicationZk; + private final HConnection connection; + + /** + * Constructor that creates a connection to the local ZooKeeper ensemble. + * @param conf Configuration to use + * @throws IOException if the connection to ZK cannot be made + * @throws RuntimeException if replication isn't enabled. + */ + public ReplicationAdmin(Configuration conf) throws IOException { + if (!conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false)) { + throw new RuntimeException("hbase.replication isn't true, please " + + "enable it in order to use replication"); + } + this.connection = HConnectionManager.getConnection(conf); + ZooKeeperWatcher zkw = this.connection.getZooKeeperWatcher(); + try { + this.replicationZk = new ReplicationZookeeper(this.connection, conf, zkw); + } catch (KeeperException e) { + throw new IOException("Unable setup the ZooKeeper connection", e); + } + } + + /** + * Add a new peer cluster to replicate to. + * @param id a short that identifies the cluster + * @param clusterKey the concatenation of the slave cluster's + * hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent + * @throws IllegalStateException if there's already one slave since + * multi-slave isn't supported yet. + */ + public void addPeer(String id, String clusterKey) throws IOException { + this.replicationZk.addPeer(id, clusterKey); + } + + /** + * Removes a peer cluster and stops the replication to it. + * @param id a short that identifies the cluster + */ + public void removePeer(String id) throws IOException { + this.replicationZk.removePeer(id); + } + + /** + * Restart the replication stream to the specified peer. + * @param id a short that identifies the cluster + */ + public void enablePeer(String id) throws IOException { + this.replicationZk.enablePeer(id); + } + + /** + * Stop the replication stream to the specified peer. + * @param id a short that identifies the cluster + */ + public void disablePeer(String id) throws IOException { + this.replicationZk.disablePeer(id); + } + + /** + * Get the number of slave clusters the local cluster has. + * @return number of slave clusters + */ + public int getPeersCount() { + return this.replicationZk.listPeersIdsAndWatch().size(); + } + + /** + * Map of this cluster's peers for display. + * @return A map of peer ids to peer cluster keys + */ + public Map listPeers() { + return this.replicationZk.listPeers(); + } + + /** + * Get state of the peer + * + * @param id peer's identifier + * @return current state of the peer + */ + public String getPeerState(String id) throws IOException { + try { + return this.replicationZk.getPeerState(id).name(); + } catch (KeeperException e) { + throw new IOException("Couldn't get the state of the peer " + id, e); + } + } + + /** + * Get the current status of the kill switch, if the cluster is replicating + * or not. + * @return true if the cluster is replicated, otherwise false + */ + public boolean getReplicating() throws IOException { + try { + return this.replicationZk.getReplication(); + } catch (KeeperException e) { + throw new IOException("Couldn't get the replication status"); + } + } + + /** + * Kill switch for all replication-related features + * @param newState true to start replication, false to stop it. + * completely + * @return the previous state + */ + public boolean setReplicating(boolean newState) throws IOException { + boolean prev = true; + try { + prev = getReplicating(); + this.replicationZk.setReplicating(newState); + } catch (KeeperException e) { + throw new IOException("Unable to set the replication state", e); + } + return prev; + } + + /** + * Get the ZK-support tool created and used by this object for replication. + * @return the ZK-support tool + */ + ReplicationZookeeper getReplicationZk() { + return replicationZk; + } + + @Override + public void close() throws IOException { + if (this.connection != null) { + this.connection.close(); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/BaseConstraint.java b/src/main/java/org/apache/hadoop/hbase/constraint/BaseConstraint.java new file mode 100644 index 0000000..974738c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/BaseConstraint.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.conf.Configured; + +/** + * Base class to use when actually implementing a {@link Constraint}. It takes + * care of getting and setting of configuration for the constraint. + */ +public abstract class BaseConstraint extends Configured implements Constraint { + +} diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java b/src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java new file mode 100644 index 0000000..5a55d1a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Put; + +/** + * Apply a {@link Constraint} (in traditional database terminology) to a HTable. + * Any number of {@link Constraint Constraints} can be added to the table, in + * any order. + *

    + * A {@link Constraint} must be added to a table before the table is loaded via + * {@link Constraints#add(HTableDescriptor, Class...)} or + * {@link Constraints#add(HTableDescriptor, org.apache.hadoop.hbase.util.Pair...)} + * (if you want to add a configuration with the {@link Constraint}). Constraints + * will be run in the order that they are added. Further, a Constraint will be + * configured before it is run (on load). + *

    + * See {@link Constraints#enableConstraint(HTableDescriptor, Class)} and + * {@link Constraints#disableConstraint(HTableDescriptor, Class)} for + * enabling/disabling of a given {@link Constraint} after it has been added. + *

    + * If a {@link Put} is invalid, the Constraint should throw some sort of + * {@link ConstraintException}, indicating that the {@link Put} has failed. When + * this exception is thrown, not further retries of the {@link Put} are + * attempted nor are any other {@link Constraint Constraints} attempted (the + * {@link Put} is clearly not valid). Therefore, there are performance + * implications in the order in which {@link BaseConstraint Constraints} are + * specified. + *

    + * If a {@link Constraint} fails to fail the {@link Put} via a + * {@link ConstraintException}, but instead throws a {@link RuntimeException}, + * the entire constraint processing mechanism ({@link ConstraintProcessor}) will + * be unloaded from the table. This ensures that the region server is still + * functional, but no more {@link Put Puts} will be checked via + * {@link Constraint Constraints}. + *

    + * Further, {@link Constraint Constraints} should probably not be used to + * enforce cross-table references as it will cause tremendous write slowdowns, + * but it is possible. + *

    + * NOTE: Implementing classes must have a nullary (no-args) constructor + * + * @see BaseConstraint + * @see Constraints + */ +public interface Constraint extends Configurable { + + /** + * Check a {@link Put} to ensure it is valid for the table. If the {@link Put} + * is valid, then just return from the method. Otherwise, throw an + * {@link Exception} specifying what happened. This {@link Exception} is + * propagated back to the client so you can see what caused the {@link Put} to + * fail. + * @param p {@link Put} to check + * @throws ConstraintException when the {@link Put} does not match the + * constraint. + */ + public void check(Put p) throws ConstraintException; + +} diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/ConstraintException.java b/src/main/java/org/apache/hadoop/hbase/constraint/ConstraintException.java new file mode 100644 index 0000000..1831421 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/ConstraintException.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.client.Put; + +/** + * Exception that a user defined constraint throws on failure of a {@link Put}. + *

    + * Does NOT attempt the {@link Put} multiple times, since the constraint + * should fail every time for the same {@link Put} (it should be + * idempotent). + */ +public class ConstraintException extends DoNotRetryIOException { + private static final long serialVersionUID = 1197446454511704140L; + + public ConstraintException() { + super(); + } + + public ConstraintException(String msg) + { + super(msg); + } + + + public ConstraintException(String msg, Throwable cause) { + super(msg, cause); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/ConstraintProcessor.java b/src/main/java/org/apache/hadoop/hbase/constraint/ConstraintProcessor.java new file mode 100644 index 0000000..522369c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/ConstraintProcessor.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; + +/*** + * Processes multiple {@link Constraint Constraints} on a given table. + *

    + * This is an ease of use mechanism - all the functionality here could be + * implemented on any given system by a coprocessor. + */ +public class ConstraintProcessor extends BaseRegionObserver { + + private static final Log LOG = LogFactory.getLog(ConstraintProcessor.class); + + private final ClassLoader classloader; + + private List constraints = new ArrayList(); + + /** + * Create the constraint processor. + *

    + * Stores the current classloader. + */ + public ConstraintProcessor() { + classloader = this.getClass().getClassLoader(); + } + + @Override + public void start(CoprocessorEnvironment environment) { + // make sure we are on a region server + if (!(environment instanceof RegionCoprocessorEnvironment)) { + throw new IllegalArgumentException( + "Constraints only act on regions - started in an environment that was not a region"); + } + RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) environment; + HTableDescriptor desc = env.getRegion().getTableDesc(); + // load all the constraints from the HTD + try { + this.constraints = Constraints.getConstraints(desc, classloader); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + + if (LOG.isInfoEnabled()) { + LOG.info("Finished loading " + constraints.size() + + " user Constraints on table: " + new String(desc.getName())); + } + + } + + @Override + public void prePut(ObserverContext e, Put put, + WALEdit edit, boolean writeToWAL) throws IOException { + // check the put against the stored constraints + for (Constraint c : constraints) { + c.check(put); + } + // if we made it here, then the Put is valid + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/Constraints.java b/src/main/java/org/apache/hadoop/hbase/constraint/Constraints.java new file mode 100644 index 0000000..c78d894 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/Constraints.java @@ -0,0 +1,622 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +/** + * Utilities for adding/removing constraints from a table. + *

    + * Constraints can be added on table load time, via the {@link HTableDescriptor}. + *

    + * NOTE: this class is NOT thread safe. Concurrent setting/enabling/disabling of + * constraints can cause constraints to be run at incorrect times or not at all. + */ +public final class Constraints { + private static final int DEFAULT_PRIORITY = -1; + + private Constraints() { + } + + private static final Log LOG = LogFactory.getLog(Constraints.class); + private static final String CONSTRAINT_HTD_KEY_PREFIX = "constraint $"; + private static final Pattern CONSTRAINT_HTD_ATTR_KEY_PATTERN = Pattern + .compile(CONSTRAINT_HTD_KEY_PREFIX, Pattern.LITERAL); + + // configuration key for if the constraint is enabled + private static final String ENABLED_KEY = "_ENABLED"; + // configuration key for the priority + private static final String PRIORITY_KEY = "_PRIORITY"; + + // smallest priority a constraiNt can have + private static final long MIN_PRIORITY = 0L; + // ensure a priority less than the smallest we could intentionally set + private static final long UNSET_PRIORITY = MIN_PRIORITY - 1; + + private static String COUNTER_KEY = "hbase.constraint.counter"; + + /** + * Enable constraints on a table. + *

    + * Currently, if you attempt to add a constraint to the table, then + * Constraints will automatically be turned on. + * + * @param desc + * table description to add the processor + * @throws IOException + * If the {@link ConstraintProcessor} CP couldn't be added to the + * table. + */ + public static void enable(HTableDescriptor desc) throws IOException { + // if the CP has already been loaded, do nothing + String clazz = ConstraintProcessor.class.getName(); + if (desc.hasCoprocessor(clazz)) { + return; + } + + // add the constrain processor CP to the table + desc.addCoprocessor(clazz); + } + + /** + * Turn off processing constraints for a given table, even if constraints have + * been turned on or added. + * + * @param desc + * {@link HTableDescriptor} where to disable {@link Constraint + * Constraints}. + */ + public static void disable(HTableDescriptor desc) { + desc.removeCoprocessor(ConstraintProcessor.class.getName()); + } + + /** + * Remove all {@link Constraint Constraints} that have been added to the table + * and turn off the constraint processing. + *

    + * All {@link Configuration Configurations} and their associated + * {@link Constraint} are removed. + * + * @param desc + * {@link HTableDescriptor} to remove {@link Constraint Constraints} + * from. + */ + public static void remove(HTableDescriptor desc) { + // disable constraints + disable(desc); + + // remove all the constraint settings + List keys = new ArrayList(); + // loop through all the key, values looking for constraints + for (Map.Entry e : desc + .getValues().entrySet()) { + String key = Bytes.toString((e.getKey().get())); + String[] className = CONSTRAINT_HTD_ATTR_KEY_PATTERN.split(key); + if (className.length == 2) { + keys.add(e.getKey()); + } + } + // now remove all the keys we found + for (ImmutableBytesWritable key : keys) { + desc.remove(key.get()); + } + } + + /** + * Check to see if the Constraint is currently set. + * + * @param desc + * {@link HTableDescriptor} to check + * @param clazz + * {@link Constraint} class to check for. + * @return true if the {@link Constraint} is present, even if it is + * disabled. false otherwise. + */ + public static boolean has(HTableDescriptor desc, + Class clazz) { + return getKeyValueForClass(desc, clazz) != null; + } + + /** + * Get the kv {@link Entry} in the descriptor for the specified class + * + * @param desc + * {@link HTableDescriptor} to read + * @param clazz + * to search for + * @return the {@link Pair} of in the table, if that class is + * present. null otherwise. + */ + private static Pair getKeyValueForClass( + HTableDescriptor desc, Class clazz) { + // get the serialized version of the constraint + String key = serializeConstraintClass(clazz); + String value = desc.getValue(key); + + return value == null ? null : new Pair(key, value); + } + + /** + * Add configuration-less constraints to the table. + *

    + * This will overwrite any configuration associated with the previous + * constraint of the same class. + *

    + * Each constraint, when added to the table, will have a specific priority, + * dictating the order in which the {@link Constraint} will be run. A + * {@link Constraint} earlier in the list will be run before those later in + * the list. The same logic applies between two Constraints over time (earlier + * added is run first on the regionserver). + * + * @param desc + * {@link HTableDescriptor} to add {@link Constraint Constraints} + * @param constraints + * {@link Constraint Constraints} to add. All constraints are + * considered automatically enabled on add + * @throws IOException + * If constraint could not be serialized/added to table + */ + public static void add(HTableDescriptor desc, + Class... constraints) throws IOException { + // make sure constraints are enabled + enable(desc); + long priority = getNextPriority(desc); + + // store each constraint + for (Class clazz : constraints) { + addConstraint(desc, clazz, null, priority++); + } + updateLatestPriority(desc, priority); + } + + /** + * Add constraints and their associated configurations to the table. + *

    + * Adding the same constraint class twice will overwrite the first + * constraint's configuration + *

    + * Each constraint, when added to the table, will have a specific priority, + * dictating the order in which the {@link Constraint} will be run. A + * {@link Constraint} earlier in the list will be run before those later in + * the list. The same logic applies between two Constraints over time (earlier + * added is run first on the regionserver). + * + * @param desc + * {@link HTableDescriptor} to add a {@link Constraint} + * @param constraints + * {@link Pair} of a {@link Constraint} and its associated + * {@link Configuration}. The Constraint will be configured on load + * with the specified configuration.All constraints are considered + * automatically enabled on add + * @throws IOException + * if any constraint could not be deserialized. Assumes if 1 + * constraint is not loaded properly, something has gone terribly + * wrong and that all constraints need to be enforced. + */ + public static void add(HTableDescriptor desc, + Pair, Configuration>... constraints) + throws IOException { + enable(desc); + long priority = getNextPriority(desc); + for (Pair, Configuration> pair : constraints) { + addConstraint(desc, pair.getFirst(), pair.getSecond(), priority++); + } + updateLatestPriority(desc, priority); + } + + /** + * Add a {@link Constraint} to the table with the given configuration + *

    + * Each constraint, when added to the table, will have a specific priority, + * dictating the order in which the {@link Constraint} will be run. A + * {@link Constraint} added will run on the regionserver before those added to + * the {@link HTableDescriptor} later. + * + * @param desc + * table descriptor to the constraint to + * @param constraint + * to be added + * @param conf + * configuration associated with the constraint + * @throws IOException + * if any constraint could not be deserialized. Assumes if 1 + * constraint is not loaded properly, something has gone terribly + * wrong and that all constraints need to be enforced. + */ + public static void add(HTableDescriptor desc, + Class constraint, Configuration conf) + throws IOException { + enable(desc); + long priority = getNextPriority(desc); + addConstraint(desc, constraint, conf, priority++); + + updateLatestPriority(desc, priority); + } + + /** + * Write the raw constraint and configuration to the descriptor. + *

    + * This method takes care of creating a new configuration based on the passed + * in configuration and then updating that with enabled and priority of the + * constraint. + *

    + * When a constraint is added, it is automatically enabled. + */ + private static void addConstraint(HTableDescriptor desc, + Class clazz, Configuration conf, long priority) + throws IOException { + writeConstraint(desc, serializeConstraintClass(clazz), + configure(conf, true, priority)); + } + + /** + * Setup the configuration for a constraint as to whether it is enabled and + * its priority + * + * @param conf + * on which to base the new configuration + * @param enabled + * true if it should be run + * @param priority + * relative to other constraints + * @returns a new configuration, storable in the {@link HTableDescriptor} + */ + private static Configuration configure(Configuration conf, boolean enabled, + long priority) { + // create the configuration to actually be stored + // clone if possible, but otherwise just create an empty configuration + Configuration toWrite = conf == null ? new Configuration() + : new Configuration(conf); + + // update internal properties + toWrite.setBooleanIfUnset(ENABLED_KEY, enabled); + + // set if unset long + if (toWrite.getLong(PRIORITY_KEY, UNSET_PRIORITY) == UNSET_PRIORITY) { + toWrite.setLong(PRIORITY_KEY, priority); + } + + return toWrite; + } + + /** + * Just write the class to a String representation of the class as a key for + * the {@link HTableDescriptor} + * + * @param clazz + * Constraint class to convert to a {@link HTableDescriptor} key + * @return key to store in the {@link HTableDescriptor} + */ + private static String serializeConstraintClass( + Class clazz) { + String constraintClazz = clazz.getName(); + return CONSTRAINT_HTD_KEY_PREFIX + constraintClazz; + } + + /** + * Write the given key and associated configuration to the + * {@link HTableDescriptor} + */ + private static void writeConstraint(HTableDescriptor desc, String key, + Configuration conf) throws IOException { + // store the key and conf in the descriptor + desc.setValue(key, serializeConfiguration(conf)); + } + + /** + * Write the configuration to a String + * + * @param conf + * to write + * @return String representation of that configuration + * @throws IOException + */ + private static String serializeConfiguration(Configuration conf) + throws IOException { + // write the configuration out to the data stream + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(bos); + conf.writeXml(dos); + dos.flush(); + byte[] data = bos.toByteArray(); + return Bytes.toString(data); + } + + /** + * Read the {@link Configuration} stored in the byte stream. + * + * @param bytes + * to read from + * @return A valid configuration + */ + private static Configuration readConfiguration(byte[] bytes) + throws IOException { + ByteArrayInputStream is = new ByteArrayInputStream(bytes); + Configuration conf = new Configuration(false); + conf.addResource(is); + return conf; + } + + /** + * Read in the configuration from the String encoded configuration + * + * @param bytes + * to read from + * @return A valid configuration + * @throws IOException + * if the configuration could not be read + */ + private static Configuration readConfiguration(String bytes) + throws IOException { + return readConfiguration(Bytes.toBytes(bytes)); + } + + private static long getNextPriority(HTableDescriptor desc) { + String value = desc.getValue(COUNTER_KEY); + + long priority; + // get the current priority + if (value == null) { + priority = MIN_PRIORITY; + } else { + priority = Long.parseLong(value) + 1; + } + + return priority; + } + + private static void updateLatestPriority(HTableDescriptor desc, long priority) { + // update the max priority + desc.setValue(COUNTER_KEY, Long.toString(priority)); + } + + /** + * Update the configuration for the {@link Constraint}; does not change the + * order in which the constraint is run. + * + * @param desc + * {@link HTableDescriptor} to update + * @param clazz + * {@link Constraint} to update + * @param configuration + * to update the {@link Constraint} with. + * @throws IOException + * if the Constraint was not stored correctly + * @throws IllegalArgumentException + * if the Constraint was not present on this table. + */ + public static void setConfiguration(HTableDescriptor desc, + Class clazz, Configuration configuration) + throws IOException, IllegalArgumentException { + // get the entry for this class + Pair e = getKeyValueForClass(desc, clazz); + + if (e == null) { + throw new IllegalArgumentException("Constraint: " + clazz.getName() + + " is not associated with this table."); + } + + // clone over the configuration elements + Configuration conf = new Configuration(configuration); + + // read in the previous info about the constraint + Configuration internal = readConfiguration(e.getSecond()); + + // update the fields based on the previous settings + conf.setIfUnset(ENABLED_KEY, internal.get(ENABLED_KEY)); + conf.setIfUnset(PRIORITY_KEY, internal.get(PRIORITY_KEY)); + + // update the current value + writeConstraint(desc, e.getFirst(), conf); + } + + /** + * Remove the constraint (and associated information) for the table + * descriptor. + * + * @param desc + * {@link HTableDescriptor} to modify + * @param clazz + * {@link Constraint} class to remove + */ + public static void remove(HTableDescriptor desc, + Class clazz) { + String key = serializeConstraintClass(clazz); + desc.remove(key); + } + + /** + * Enable the given {@link Constraint}. Retains all the information (e.g. + * Configuration) for the {@link Constraint}, but makes sure that it gets + * loaded on the table. + * + * @param desc + * {@link HTableDescriptor} to modify + * @param clazz + * {@link Constraint} to enable + * @throws IOException + * If the constraint cannot be properly deserialized + */ + public static void enableConstraint(HTableDescriptor desc, + Class clazz) throws IOException { + changeConstraintEnabled(desc, clazz, true); + } + + /** + * Disable the given {@link Constraint}. Retains all the information (e.g. + * Configuration) for the {@link Constraint}, but it just doesn't load the + * {@link Constraint} on the table. + * + * @param desc + * {@link HTableDescriptor} to modify + * @param clazz + * {@link Constraint} to disable. + * @throws IOException + * if the constraint cannot be found + */ + public static void disableConstraint(HTableDescriptor desc, + Class clazz) throws IOException { + changeConstraintEnabled(desc, clazz, false); + } + + /** + * Change the whether the constraint (if it is already present) is enabled or + * disabled. + */ + private static void changeConstraintEnabled(HTableDescriptor desc, + Class clazz, boolean enabled) throws IOException { + // get the original constraint + Pair entry = getKeyValueForClass(desc, clazz); + if (entry == null) { + throw new IllegalArgumentException("Constraint: " + clazz.getName() + + " is not associated with this table. You can't enable it!"); + } + + // create a new configuration from that conf + Configuration conf = readConfiguration(entry.getSecond()); + + // set that it is enabled + conf.setBoolean(ENABLED_KEY, enabled); + + // write it back out + writeConstraint(desc, entry.getFirst(), conf); + } + + /** + * Check to see if the given constraint is enabled. + * + * @param desc + * {@link HTableDescriptor} to check. + * @param clazz + * {@link Constraint} to check for + * @return true if the {@link Constraint} is present and enabled. + * false otherwise. + * @throws IOException + * If the constraint has improperly stored in the table + */ + public static boolean enabled(HTableDescriptor desc, + Class clazz) throws IOException { + // get the kv + Pair entry = getKeyValueForClass(desc, clazz); + // its not enabled so just return false. In fact, its not even present! + if (entry == null) { + return false; + } + + // get the info about the constraint + Configuration conf = readConfiguration(entry.getSecond()); + + return conf.getBoolean(ENABLED_KEY, false); + } + + /** + * Get the constraints stored in the table descriptor + * + * @param desc + * To read from + * @param classloader + * To use when loading classes. If a special classloader is used on a + * region, for instance, then that should be the classloader used to + * load the constraints. This could also apply to unit-testing + * situation, where want to ensure that class is reloaded or not. + * @return List of configured {@link Constraint Constraints} + * @throws IOException + * if any part of reading/arguments fails + */ + static List getConstraints(HTableDescriptor desc, + ClassLoader classloader) throws IOException { + List constraints = new ArrayList(); + // loop through all the key, values looking for constraints + for (Map.Entry e : desc + .getValues().entrySet()) { + // read out the constraint + String key = Bytes.toString(e.getKey().get()).trim(); + String[] className = CONSTRAINT_HTD_ATTR_KEY_PATTERN.split(key); + if (className.length == 2) { + key = className[1]; + if (LOG.isDebugEnabled()) { + LOG.debug("Loading constraint:" + key); + } + + // read in the rest of the constraint + Configuration conf; + try { + conf = readConfiguration(e.getValue().get()); + } catch (IOException e1) { + // long that we don't have a valid configuration stored, and move on. + LOG.warn("Corrupted configuration found for key:" + key + + ", skipping it."); + continue; + } + // if it is not enabled, skip it + if (!conf.getBoolean(ENABLED_KEY, false)) { + if (LOG.isDebugEnabled()) + LOG.debug("Constraint: " + key + " is DISABLED - skipping it"); + // go to the next constraint + continue; + } + + try { + // add the constraint, now that we expect it to be valid. + Class clazz = classloader.loadClass(key) + .asSubclass(Constraint.class); + Constraint constraint = clazz.newInstance(); + constraint.setConf(conf); + constraints.add(constraint); + } catch (ClassNotFoundException e1) { + throw new IOException(e1); + } catch (InstantiationException e1) { + throw new IOException(e1); + } catch (IllegalAccessException e1) { + throw new IOException(e1); + } + } + } + // sort them, based on the priorities + Collections.sort(constraints, constraintComparator); + return constraints; + } + + private static final Comparator constraintComparator = new Comparator() { + @Override + public int compare(Constraint c1, Constraint c2) { + // compare the priorities of the constraints stored in their configuration + return Long.valueOf(c1.getConf().getLong(PRIORITY_KEY, DEFAULT_PRIORITY)) + .compareTo(c2.getConf().getLong(PRIORITY_KEY, DEFAULT_PRIORITY)); + } + }; + +} diff --git a/src/main/java/org/apache/hadoop/hbase/constraint/package-info.java b/src/main/java/org/apache/hadoop/hbase/constraint/package-info.java new file mode 100644 index 0000000..8ded75e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/constraint/package-info.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Restrict the domain of a data attribute, often times to fulfill business rules/requirements. + * +

    +

    Table of Contents

    + +

    + +

    Overview

    + Constraints are used to enforce business rules in a database. + By checking all {@link org.apache.hadoop.hbase.client.Put Puts} on a given table, you can enforce very specific data policies. + For instance, you can ensure that a certain column family-column qualifier pair always has a value between 1 and 10. + Otherwise, the {@link org.apache.hadoop.hbase.client.Put} is rejected and the data integrity is maintained. +

    + Constraints are designed to be configurable, so a constraint can be used across different tables, but implement different + behavior depending on the specific configuration given to that constraint. +

    + By adding a constraint to a table (see Example Usage), constraints will automatically enabled. + You also then have the option of to disable (just 'turn off') or remove (delete all associated information) all constraints on a table. + If you remove all constraints + (see {@link org.apache.hadoop.hbase.constraint.Constraints#remove(org.apache.hadoop.hbase.HTableDescriptor)}, + you must re-add any {@link org.apache.hadoop.hbase.constraint.Constraint} you want on that table. + However, if they are just disabled (see {@link org.apache.hadoop.hbase.constraint.Constraints#disable(org.apache.hadoop.hbase.HTableDescriptor)}, + all you need to do is enable constraints again, and everything will be turned back on as it was configured. + Individual constraints can also be individually enabled, disabled or removed without affecting other constraints. +

    + By default, constraints are disabled on a table. + This means you will not see any slow down on a table if constraints are not enabled. +

    + +

    Concurrency and Atomicity

    + Currently, no attempts at enforcing correctness in a multi-threaded scenario when modifying a constraint, via + {@link org.apache.hadoop.hbase.constraint.Constraints}, to the the {@link org.apache.hadoop.hbase.HTableDescriptor}. + This is particularly important when adding a constraint(s) to the {@link org.apache.hadoop.hbase.HTableDescriptor} + as it first retrieves the next priority from a custom value set in the descriptor, + adds each constraint (with increasing priority) to the descriptor, and then the next available priority is re-stored + back in the {@link org.apache.hadoop.hbase.HTableDescriptor}. +

    + Locking is recommended around each of Constraints add methods: + {@link org.apache.hadoop.hbase.constraint.Constraints#add(org.apache.hadoop.hbase.HTableDescriptor, Class...)}, + {@link org.apache.hadoop.hbase.constraint.Constraints#add(org.apache.hadoop.hbase.HTableDescriptor, org.apache.hadoop.hbase.util.Pair...)}, + and {@link org.apache.hadoop.hbase.constraint.Constraints#add(org.apache.hadoop.hbase.HTableDescriptor, Class, org.apache.hadoop.conf.Configuration)}. + Any changes on a single HTableDescriptor should be serialized, either within a single thread or via external mechanisms. +

    + Note that having a higher priority means that a constraint will run later; e.g. a constraint with priority 1 will run before a + constraint with priority 2. +

    + Since Constraints currently are designed to just implement simple checks (e.g. is the value in the right range), there will + be no atomicity conflicts. + Even if one of the puts finishes the constraint first, the single row will not be corrupted and the 'fastest' write will win; + the underlying region takes care of breaking the tie and ensuring that writes get serialized to the table. + So yes, this doesn't ensure that we are going to get specific ordering or even a fully consistent view of the underlying data. +

    + Each constraint should only use local/instance variables, unless doing more advanced usage. Static variables could cause difficulties + when checking concurrent writes to the same region, leading to either highly locked situations (decreasing through-put) or higher probability of errors. + However, as long as each constraint just uses local variables, each thread interacting with the constraint will execute correctly and efficiently. + +

    Caveats

    + In traditional (SQL) databases, Constraints are often used to enforce referential integrity. + However, in HBase, this will likely cause significant overhead and dramatically decrease the number of + {@link org.apache.hadoop.hbase.client.Put Puts}/second possible on a table. This is because to check the referential integrity + when making a {@link org.apache.hadoop.hbase.client.Put}, one must block on a scan for the 'remote' table, checking for the valid reference. + For millions of {@link org.apache.hadoop.hbase.client.Put Puts} a second, this will breakdown very quickly. + There are several options around the blocking behavior including, but not limited to: +
      +
    • Create a 'pre-join' table where the keys are already denormalized
      • +
    • Designing for 'incorrect' references
      • +
    • Using an external enforcement mechanism
      • +
    + There are also several general considerations that must be taken into account, when using Constraints: +
      +
    1. All changes made via {@link org.apache.hadoop.hbase.constraint.Constraints} will make modifications to the + {@link org.apache.hadoop.hbase.HTableDescriptor} for a given table. As such, the usual renabling of tables should be used for + propagating changes to the table. When at all possible, Constraints should be added to the table before the table is created.
      2. +
    2. Constraints are run in the order that they are added to a table. This has implications for what order constraints should + be added to a table.
      3. +
    3. Whenever new Constraint jars are added to a region server, those region servers need to go through a rolling restart to + make sure that they pick up the new jars and can enable the new constraints.
      4. +
    4. There are certain keys that are reserved for the Configuration namespace: +
        +
      • _ENABLED - used server-side to determine if a constraint should be run
        • +
      • _PRIORITY - used server-side to determine what order a constraint should be run
        • +
      + If these items are set, they will be respected in the constraint configuration, but they are taken care of by default in when + adding constraints to an {@link org.apache.hadoop.hbase.HTableDescriptor} via the usual method.
      5. +
    +

    + Under the hood, constraints are implemented as a Coprocessor (see {@link org.apache.hadoop.hbase.constraint.ConstraintProcessor} + if you are interested). + + +

    Example usage

    + First, you must define a {@link org.apache.hadoop.hbase.constraint.Constraint}. + The best way to do this is to extend {@link org.apache.hadoop.hbase.constraint.BaseConstraint}, which takes care of some of the more + mundane details of using a {@link org.apache.hadoop.hbase.constraint.Constraint}. +

    + Let's look at one possible implementation of a constraint - an IntegerConstraint(there are also several simple examples in the tests). + The IntegerConstraint checks to make sure that the value is a String-encoded int. + It is really simple to implement this kind of constraint, the only method needs to be implemented is + {@link org.apache.hadoop.hbase.constraint.Constraint#check(org.apache.hadoop.hbase.client.Put)}: + +

    + 
    + public class IntegerConstraint extends BaseConstraint {
+ public void check(Put p) throws ConstraintException {
+
+ Map<byte[], List<KeyValue>> familyMap = p.getFamilyMap();
+
+ for (List <KeyValue> kvs : familyMap.values()) {
+ for (KeyValue kv : kvs) {
+
+ // just make sure that we can actually pull out an int
+ // this will automatically throw a NumberFormatException if we try to
+ // store something that isn't an Integer.
+
+ try {
+ Integer.parseInt(new String(kv.getValue()));
+ } catch (NumberFormatException e) {
+ throw new ConstraintException("Value in Put (" + p
+ + ") was not a String-encoded integer", e);
+ } } } 
+
    +
    +

    + Note that all exceptions that you expect to be thrown must be caught and then rethrown as a + {@link org.apache.hadoop.hbase.constraint.ConstraintException}. This way, you can be sure that a + {@link org.apache.hadoop.hbase.client.Put} fails for an expected reason, rather than for any reason. + For example, an {@link java.lang.OutOfMemoryError} is probably indicative of an inherent problem in + the {@link org.apache.hadoop.hbase.constraint.Constraint}, rather than a failed {@link org.apache.hadoop.hbase.client.Put}. +

    + If an unexpected exception is thrown (for example, any kind of uncaught {@link java.lang.RuntimeException}), + constraint-checking will be 'unloaded' from the regionserver where that error occurred. + This means no further {@link org.apache.hadoop.hbase.constraint.Constraint Constraints} will be checked on that server + until it is reloaded. This is done to ensure the system remains as available as possible. + Therefore, be careful when writing your own Constraint. +

    + So now that we have a Constraint, we want to add it to a table. It's as easy as: + +

    + 
    + HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
+ ...
+ Constraints.add(desc, IntegerConstraint.class);
+
    +

    + Once we added the IntegerConstraint, constraints will be enabled on the table (once it is created) and + we will always check to make sure that the value is an String-encoded integer. +

    + However, suppose we also write our own constraint, MyConstraint.java. + First, you need to make sure this class-files are in the classpath (in a jar) on the regionserver where + that constraint will be run (this could require a rolling restart on the region server - see Caveats above) +

    + Suppose that MyConstraint also uses a Configuration (see {@link org.apache.hadoop.hbase.constraint.Constraint#getConf()}). + Then adding MyConstraint looks like this: + +

    + 
    + HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
+ Configuration conf = new Configuration(false);
+ ...
+ (add values to the conf)
+ (modify the table descriptor)
+ ...
+ Constraints.add(desc, new Pair(MyConstraint.class, conf));
+
    +

    + At this point we added both the IntegerConstraint and MyConstraint to the table, the IntegerConstraint + will be run first, followed by MyConstraint. +

    + Suppose we realize that the {@link org.apache.hadoop.conf.Configuration} for MyConstraint is actually wrong + when it was added to the table. Note, when it is added to the table, it is not added by reference, + but is instead copied into the {@link org.apache.hadoop.hbase.HTableDescriptor}. + Thus, to change the {@link org.apache.hadoop.conf.Configuration} we are using for MyConstraint, we need to do this: + +

    + 
    + (add/modify the conf)
+ ...
+ Constraints.setConfiguration(desc, MyConstraint.class, conf);
+
    +

    + This will overwrite the previous configuration for MyConstraint, but not change the order of the + constraint nor if it is enabled/disabled. +

    + Note that the same constraint class can be added multiple times to a table without repercussion. + A use case for this is the same constraint working differently based on its configuration. + +

    + Suppose then we want to disable just MyConstraint. Its as easy as: +

    + 
    + Constraints.disable(desc, MyConstraint.class);
+
    +

    + This just turns off MyConstraint, but retains the position and the configuration associated with MyConstraint. + Now, if we want to re-enable the constraint, its just another one-liner: +

    + 
    + Constraints.enable(desc, MyConstraint.class);
+
    +

    + Similarly, constraints on the entire table are disabled via: +

    + 
    + Constraints.disable(desc);
+
    +

    + Or enabled via: + +

    + 
    + Constraints.enable(desc);
+
    +

    + Lastly, suppose you want to remove MyConstraint from the table, including with position it should be run at and its configuration. + This is similarly simple: +

    + 
    + Constraints.remove(desc, MyConstraint.class);
+
    +

    + Also, removing all constraints from a table is similarly simple: +

    + 
    + Constraints.remove(desc);
+
    + This will remove all constraints (and associated information) from the table and turn off the constraint processing. +

    NOTE

    + It is important to note the use above of +

    + 
    + Configuration conf = new Configuration(false);
+
    + If you just use new Configuration(), then the Configuration will be loaded with the default + properties. While in the simple case, this is not going to be an issue, it will cause pain down the road. + First, these extra properties are going to cause serious bloat in your {@link org.apache.hadoop.hbase.HTableDescriptor}, + meaning you are keeping around a ton of redundant information. Second, it is going to make examining + your table in the shell, via describe 'table', a huge pain as you will have to dig through + a ton of irrelevant config values to find the ones you set. In short, just do it the right way. + */ +package org.apache.hadoop.hbase.constraint; diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateImplementation.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateImplementation.java new file mode 100644 index 0000000..f6f4b42 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateImplementation.java @@ -0,0 +1,277 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.ipc.ProtocolSignature; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.util.Pair; + +/** + * A concrete AggregateProtocol implementation. Its system level coprocessor + * that computes the aggregate function at a region level. + */ +public class AggregateImplementation extends BaseEndpointCoprocessor implements + AggregateProtocol { + protected static Log log = LogFactory.getLog(AggregateImplementation.class); + + @Override + public ProtocolSignature getProtocolSignature( + String protocol, long version, int clientMethodsHashCode) + throws IOException { + if (AggregateProtocol.class.getName().equals(protocol)) { + return new ProtocolSignature(AggregateProtocol.VERSION, null); + } + throw new IOException("Unknown protocol: " + protocol); + } + + @Override + public T getMax(ColumnInterpreter ci, Scan scan) + throws IOException { + T temp; + T max = null; + InternalScanner scanner = ((RegionCoprocessorEnvironment) getEnvironment()) + .getRegion().getScanner(scan); + List results = new ArrayList(); + byte[] colFamily = scan.getFamilies()[0]; + byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst(); + // qualifier can be null. + try { + boolean hasMoreRows = false; + do { + hasMoreRows = scanner.next(results); + for (KeyValue kv : results) { + temp = ci.getValue(colFamily, qualifier, kv); + max = (max == null || (temp != null && ci.compare(temp, max) > 0)) ? temp : max; + } + results.clear(); + } while (hasMoreRows); + } finally { + scanner.close(); + } + log.info("Maximum from this region is " + + ((RegionCoprocessorEnvironment) getEnvironment()).getRegion() + .getRegionNameAsString() + ": " + max); + return max; + } + + @Override + public T getMin(ColumnInterpreter ci, Scan scan) + throws IOException { + T min = null; + T temp; + InternalScanner scanner = ((RegionCoprocessorEnvironment) getEnvironment()) + .getRegion().getScanner(scan); + List results = new ArrayList(); + byte[] colFamily = scan.getFamilies()[0]; + byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst(); + try { + boolean hasMoreRows = false; + do { + hasMoreRows = scanner.next(results); + for (KeyValue kv : results) { + temp = ci.getValue(colFamily, qualifier, kv); + min = (min == null || (temp != null && ci.compare(temp, min) < 0)) ? temp : min; + } + results.clear(); + } while (hasMoreRows); + } finally { + scanner.close(); + } + log.info("Minimum from this region is " + + ((RegionCoprocessorEnvironment) getEnvironment()).getRegion() + .getRegionNameAsString() + ": " + min); + return min; + } + + @Override + public S getSum(ColumnInterpreter ci, Scan scan) + throws IOException { + long sum = 0l; + S sumVal = null; + T temp; + InternalScanner scanner = ((RegionCoprocessorEnvironment) getEnvironment()) + .getRegion().getScanner(scan); + byte[] colFamily = scan.getFamilies()[0]; + byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst(); + List results = new ArrayList(); + try { + boolean hasMoreRows = false; + do { + hasMoreRows = scanner.next(results); + for (KeyValue kv : results) { + temp = ci.getValue(colFamily, qualifier, kv); + if (temp != null) + sumVal = ci.add(sumVal, ci.castToReturnType(temp)); + } + results.clear(); + } while (hasMoreRows); + } finally { + scanner.close(); + } + log.debug("Sum from this region is " + + ((RegionCoprocessorEnvironment) getEnvironment()).getRegion() + .getRegionNameAsString() + ": " + sum); + return sumVal; + } + + @Override + public long getRowNum(ColumnInterpreter ci, Scan scan) + throws IOException { + long counter = 0l; + List results = new ArrayList(); + byte[] colFamily = scan.getFamilies()[0]; + byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst(); + if (scan.getFilter() == null && qualifier == null) + scan.setFilter(new FirstKeyOnlyFilter()); + InternalScanner scanner = ((RegionCoprocessorEnvironment) getEnvironment()) + .getRegion().getScanner(scan); + try { + boolean hasMoreRows = false; + do { + hasMoreRows = scanner.next(results); + if (results.size() > 0) { + counter++; + } + results.clear(); + } while (hasMoreRows); + } finally { + scanner.close(); + } + log.info("Row counter from this region is " + + ((RegionCoprocessorEnvironment) getEnvironment()).getRegion() + .getRegionNameAsString() + ": " + counter); + return counter; + } + + @Override + public Pair getAvg(ColumnInterpreter ci, Scan scan) + throws IOException { + S sumVal = null; + Long rowCountVal = 0l; + InternalScanner scanner = ((RegionCoprocessorEnvironment) getEnvironment()) + .getRegion().getScanner(scan); + byte[] colFamily = scan.getFamilies()[0]; + byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst(); + List results = new ArrayList(); + boolean hasMoreRows = false; + try { + do { + results.clear(); + hasMoreRows = scanner.next(results); + for (KeyValue kv : results) { + sumVal = ci.add(sumVal, ci.castToReturnType(ci.getValue(colFamily, + qualifier, kv))); + } + rowCountVal++; + } while (hasMoreRows); + } finally { + scanner.close(); + } + Pair pair = new Pair(sumVal, rowCountVal); + return pair; + } + + @Override + public Pair, Long> getStd(ColumnInterpreter ci, Scan scan) + throws IOException { + S sumVal = null, sumSqVal = null, tempVal = null; + long rowCountVal = 0l; + InternalScanner scanner = ((RegionCoprocessorEnvironment) getEnvironment()) + .getRegion().getScanner(scan); + byte[] colFamily = scan.getFamilies()[0]; + byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst(); + List results = new ArrayList(); + + boolean hasMoreRows = false; + try { + do { + tempVal = null; + hasMoreRows = scanner.next(results); + for (KeyValue kv : results) { + tempVal = ci.add(tempVal, ci.castToReturnType(ci.getValue(colFamily, + qualifier, kv))); + } + results.clear(); + sumVal = ci.add(sumVal, tempVal); + sumSqVal = ci.add(sumSqVal, ci.multiply(tempVal, tempVal)); + rowCountVal++; + } while (hasMoreRows); + } finally { + scanner.close(); + } + List l = new ArrayList(); + l.add(sumVal); + l.add(sumSqVal); + Pair, Long> p = new Pair, Long>(l, rowCountVal); + return p; + } + + @Override + public List getMedian(ColumnInterpreter ci, Scan scan) + throws IOException { + S sumVal = null, sumWeights = null, tempVal = null, tempWeight = null; + + InternalScanner scanner = ((RegionCoprocessorEnvironment) getEnvironment()) + .getRegion().getScanner(scan); + byte[] colFamily = scan.getFamilies()[0]; + NavigableSet quals = scan.getFamilyMap().get(colFamily); + byte[] valQualifier = quals.pollFirst(); + // if weighted median is requested, get qualifier for the weight column + byte[] weightQualifier = quals.size() > 1 ? quals.pollLast() : null; + List results = new ArrayList(); + + boolean hasMoreRows = false; + try { + do { + tempVal = null; + tempWeight = null; + hasMoreRows = scanner.next(results); + for (KeyValue kv : results) { + tempVal = ci.add(tempVal, ci.castToReturnType(ci.getValue(colFamily, + valQualifier, kv))); + if (weightQualifier != null) { + tempWeight = ci.add(tempWeight, + ci.castToReturnType(ci.getValue(colFamily, weightQualifier, kv))); + } + } + results.clear(); + sumVal = ci.add(sumVal, tempVal); + sumWeights = ci.add(sumWeights, tempWeight); + } while (hasMoreRows); + } finally { + scanner.close(); + } + List l = new ArrayList(); + l.add(sumVal); + l.add(sumWeights == null ? ci.castToReturnType(ci.getMinValue()) : sumWeights); + return l; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateProtocol.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateProtocol.java new file mode 100644 index 0000000..e654c0c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/AggregateProtocol.java @@ -0,0 +1,144 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.coprocessor.AggregationClient; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.util.Pair; + +/** + * Defines the aggregation functions that are to be supported in this + * Coprocessor. For each method, it takes a Scan object and a columnInterpreter. + * The scan object should have a column family (else an exception will be + * thrown), and an optional column qualifier. In the current implementation + * {@link AggregateImplementation}, only one column family and column qualifier + * combination is served. In case there are more than one, only first one will + * be picked. Refer to {@link AggregationClient} for some general conditions on + * input parameters. + */ +public interface AggregateProtocol extends CoprocessorProtocol { + public static final long VERSION = 1L; + + /** + * Gives the maximum for a given combination of column qualifier and column + * family, in the given row range as defined in the Scan object. In its + * current implementation, it takes one column family and one column qualifier + * (if provided). In case of null column qualifier, maximum value for the + * entire column family will be returned. + * @param ci + * @param scan + * @return max value as mentioned above + * @throws IOException + */ + T getMax(ColumnInterpreter ci, Scan scan) throws IOException; + + /** + * Gives the minimum for a given combination of column qualifier and column + * family, in the given row range as defined in the Scan object. In its + * current implementation, it takes one column family and one column qualifier + * (if provided). In case of null column qualifier, minimum value for the + * entire column family will be returned. + * @param ci + * @param scan + * @return min as mentioned above + * @throws IOException + */ + T getMin(ColumnInterpreter ci, Scan scan) throws IOException; + + /** + * Gives the sum for a given combination of column qualifier and column + * family, in the given row range as defined in the Scan object. In its + * current implementation, it takes one column family and one column qualifier + * (if provided). In case of null column qualifier, sum for the entire column + * family will be returned. + * @param ci + * @param scan + * @return sum of values as defined by the column interpreter + * @throws IOException + */ + S getSum(ColumnInterpreter ci, Scan scan) throws IOException; + + /** + * @param ci + * @param scan + * @return Row count for the given column family and column qualifier, in + * the given row range as defined in the Scan object. + * @throws IOException + */ + long getRowNum(ColumnInterpreter ci, Scan scan) + throws IOException; + + /** + * Gives a Pair with first object as Sum and second object as row count, + * computed for a given combination of column qualifier and column family in + * the given row range as defined in the Scan object. In its current + * implementation, it takes one column family and one column qualifier (if + * provided). In case of null column qualifier, an aggregate sum over all the + * entire column family will be returned. + *

    + * The average is computed in + * {@link AggregationClient#avg(byte[], ColumnInterpreter, Scan)} by + * processing results from all regions, so its "ok" to pass sum and a Long + * type. + * @param ci + * @param scan + * @return Average + * @throws IOException + */ + Pair getAvg(ColumnInterpreter ci, Scan scan) + throws IOException; + + /** + * Gives a Pair with first object a List containing Sum and sum of squares, + * and the second object as row count. It is computed for a given combination of + * column qualifier and column family in the given row range as defined in the + * Scan object. In its current implementation, it takes one column family and + * one column qualifier (if provided). The idea is get the value of variance first: + * the average of the squares less the square of the average a standard + * deviation is square root of variance. + * @param ci + * @param scan + * @return STD + * @throws IOException + */ + Pair, Long> getStd(ColumnInterpreter ci, Scan scan) + throws IOException; + + /** + * Gives a List containing sum of values and sum of weights. + * It is computed for the combination of column + * family and column qualifier(s) in the given row range as defined in the + * Scan object. In its current implementation, it takes one column family and + * two column qualifiers. The first qualifier is for values column and + * the second qualifier (optional) is for weight column. + * @param ci + * @param scan + * @return Pair + * @throws IOException + */ + List getMedian(ColumnInterpreter ci, Scan scan) + throws IOException; + +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseEndpointCoprocessor.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseEndpointCoprocessor.java new file mode 100644 index 0000000..3a787fd --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseEndpointCoprocessor.java @@ -0,0 +1,77 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; + +import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.ipc.ProtocolSignature; +import org.apache.hadoop.hbase.ipc.VersionedProtocol; + +/** + * This abstract class provides default implementation of an Endpoint. + * It also maintains a CoprocessorEnvironment object which can be + * used to access region resource. + * + * It's recommended to use this abstract class to implement your Endpoint. + * However you still can just implement the interface CoprocessorProtocol + * and Coprocessor to develop an Endpoint. But you won't be able to access + * the region related resource, i.e., CoprocessorEnvironment. + */ +public abstract class BaseEndpointCoprocessor implements Coprocessor, + CoprocessorProtocol, VersionedProtocol { + /** + * This Interfaces' version. Version changes when the Interface changes. + */ + // All HBase Interfaces used derive from HBaseRPCProtocolVersion. It + // maintained a single global version number on all HBase Interfaces. This + // meant all HBase RPC was broke though only one of the three RPC Interfaces + // had changed. This has since been undone. + public static final long VERSION = 28L; + + private CoprocessorEnvironment env; + + /** + * @return env Coprocessor environment. + */ + public CoprocessorEnvironment getEnvironment() { + return env; + } + + @Override + public void start(CoprocessorEnvironment env) { + this.env = env; + } + + @Override + public void stop(CoprocessorEnvironment env) { } + + @Override + public ProtocolSignature getProtocolSignature( + String protocol, long version, int clientMethodsHashCode) + throws IOException { + return new ProtocolSignature(VERSION, null); + } + + @Override + public long getProtocolVersion(String protocol, long clientVersion) + throws IOException { + return VERSION; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java new file mode 100644 index 0000000..07c537e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java @@ -0,0 +1,315 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.UnknownRegionException; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; + +import java.io.IOException; + +public class BaseMasterObserver implements MasterObserver { + @Override + public void preCreateTable(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + } + + @Override + public void postCreateTable(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + } + + @Override + public void preCreateTableHandler(final ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + } + + @Override + public void postCreateTableHandler(final ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + } + + @Override + public void preDeleteTable(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void postDeleteTable(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void preDeleteTableHandler(final ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void postDeleteTableHandler(final ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void preModifyTable(ObserverContext ctx, + byte[] tableName, HTableDescriptor htd) throws IOException { + } + + @Override + public void postModifyTableHandler(ObserverContext ctx, + byte[] tableName, HTableDescriptor htd) throws IOException { + } + + @Override + public void preModifyTableHandler(ObserverContext ctx, + byte[] tableName, HTableDescriptor htd) throws IOException { + } + + @Override + public void postModifyTable(ObserverContext ctx, + byte[] tableName, HTableDescriptor htd) throws IOException { + } + + @Override + public void preAddColumn(ObserverContext ctx, + byte[] tableName, HColumnDescriptor column) throws IOException { + } + + @Override + public void postAddColumn(ObserverContext ctx, + byte[] tableName, HColumnDescriptor column) throws IOException { + } + + @Override + public void preAddColumnHandler(ObserverContext ctx, + byte[] tableName, HColumnDescriptor column) throws IOException { + } + + @Override + public void postAddColumnHandler(ObserverContext ctx, + byte[] tableName, HColumnDescriptor column) throws IOException { + } + + @Override + public void preModifyColumn(ObserverContext ctx, + byte[] tableName, HColumnDescriptor descriptor) throws IOException { + } + + @Override + public void postModifyColumn(ObserverContext ctx, + byte[] tableName, HColumnDescriptor descriptor) throws IOException { + } + + @Override + public void preModifyColumnHandler(ObserverContext ctx, + byte[] tableName, HColumnDescriptor descriptor) throws IOException { + } + + @Override + public void postModifyColumnHandler(ObserverContext ctx, + byte[] tableName, HColumnDescriptor descriptor) throws IOException { + } + + @Override + public void preDeleteColumn(ObserverContext ctx, + byte[] tableName, byte[] c) throws IOException { + } + + @Override + public void postDeleteColumn(ObserverContext ctx, + byte[] tableName, byte[] c) throws IOException { + } + + @Override + public void preDeleteColumnHandler(ObserverContext ctx, + byte[] tableName, byte[] c) throws IOException { + } + + @Override + public void postDeleteColumnHandler(ObserverContext ctx, + byte[] tableName, byte[] c) throws IOException { + } + + @Override + public void preEnableTable(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void postEnableTable(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void preEnableTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void postEnableTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void preDisableTable(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void postDisableTable(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void preDisableTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void postDisableTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + } + + @Override + public void preAssign(ObserverContext ctx, + HRegionInfo regionInfo) throws IOException { + } + + @Override + public void postAssign(ObserverContext ctx, + HRegionInfo regionInfo) throws IOException { + } + + @Override + public void preUnassign(ObserverContext ctx, + HRegionInfo regionInfo, boolean force) throws IOException { + } + + @Override + public void postUnassign(ObserverContext ctx, + HRegionInfo regionInfo, boolean force) throws IOException { + } + + @Override + public void preBalance(ObserverContext ctx) + throws IOException { + } + + @Override + public void postBalance(ObserverContext ctx) + throws IOException { + } + + @Override + public boolean preBalanceSwitch(ObserverContext ctx, + boolean b) throws IOException { + return b; + } + + @Override + public void postBalanceSwitch(ObserverContext ctx, + boolean oldValue, boolean newValue) throws IOException { + } + + @Override + public void preShutdown(ObserverContext ctx) + throws IOException { + } + + @Override + public void preStopMaster(ObserverContext ctx) + throws IOException { + } + + @Override + public void postStartMaster(ObserverContext ctx) + throws IOException { + } + + @Override + public void start(CoprocessorEnvironment ctx) throws IOException { + } + + @Override + public void stop(CoprocessorEnvironment ctx) throws IOException { + } + + @Override + public void preMove(ObserverContext ctx, + HRegionInfo region, ServerName srcServer, ServerName destServer) + throws IOException { + } + + @Override + public void postMove(ObserverContext ctx, + HRegionInfo region, ServerName srcServer, ServerName destServer) + throws IOException { + } + + @Override + public void preSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + } + + @Override + public void postSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + } + + @Override + public void preCloneSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + } + + @Override + public void postCloneSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + } + + @Override + public void preRestoreSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + } + + @Override + public void postRestoreSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + } + + @Override + public void preDeleteSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot) throws IOException { + } + + @Override + public void postDeleteSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot) throws IOException { + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java new file mode 100644 index 0000000..b98bba9 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java @@ -0,0 +1,393 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.util.List; +import java.util.NavigableSet; + +import com.google.common.collect.ImmutableList; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.compactions.CompactSelection; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Pair; + +import java.io.IOException; + +/** + * An abstract class that implements RegionObserver. + * By extending it, you can create your own region observer without + * overriding all abstract methods of RegionObserver. + */ +public abstract class BaseRegionObserver implements RegionObserver { + @Override + public void start(CoprocessorEnvironment e) throws IOException { } + + @Override + public void stop(CoprocessorEnvironment e) throws IOException { } + + @Override + public void preOpen(ObserverContext e) throws IOException { } + + @Override + public void postOpen(ObserverContext e) { } + + @Override + public void preClose(ObserverContext c, boolean abortRequested) + throws IOException { } + + @Override + public void postClose(ObserverContext e, + boolean abortRequested) { } + + @Override + public InternalScanner preFlushScannerOpen(final ObserverContext c, + final Store store, final KeyValueScanner memstoreScanner, final InternalScanner s) + throws IOException { + return s; + } + + @Override + public void preFlush(ObserverContext e) throws IOException { + } + + @Override + public void postFlush(ObserverContext e) throws IOException { + } + + @Override + public InternalScanner preFlush(ObserverContext e, Store store, + InternalScanner scanner) throws IOException { + return scanner; + } + + @Override + public void postFlush(ObserverContext e, Store store, + StoreFile resultFile) throws IOException { + } + + @Override + public void preSplit(ObserverContext e) throws IOException { + } + + @Override + public void postSplit(ObserverContext e, HRegion l, HRegion r) + throws IOException { + } + + @Override + public void preCompactSelection(final ObserverContext c, + final Store store, final List candidates, final CompactionRequest request) + throws IOException { + preCompactSelection(c, store, candidates); + } + + @Override + public void preCompactSelection(final ObserverContext c, + final Store store, final List candidates) throws IOException { } + + @Override + public void postCompactSelection(final ObserverContext c, + final Store store, final ImmutableList selected, CompactionRequest request) { + postCompactSelection(c, store, selected); + } + + @Override + public void postCompactSelection(final ObserverContext c, + final Store store, final ImmutableList selected) { } + + @Override + public InternalScanner preCompact(ObserverContext e, + final Store store, final InternalScanner scanner) throws IOException { + return scanner; + } + + @Override + public InternalScanner preCompact(ObserverContext e, + final Store store, final InternalScanner scanner, CompactionRequest request) + throws IOException { + return preCompact(e, store, scanner); + } + + @Override + public InternalScanner preCompactScannerOpen(final ObserverContext c, + final Store store, List scanners, final ScanType scanType, + final long earliestPutTs, final InternalScanner s) throws IOException { + return s; + } + + @Override + public InternalScanner preCompactScannerOpen( + final ObserverContext c, final Store store, + List scanners, final ScanType scanType, final long earliestPutTs, + final InternalScanner s, CompactionRequest request) throws IOException { + return preCompactScannerOpen(c, store, scanners, scanType, earliestPutTs, s); + } + + @Override + public void postCompact(ObserverContext e, final Store store, + final StoreFile resultFile) throws IOException { + } + + @Override + public void postCompact(ObserverContext e, final Store store, + final StoreFile resultFile, CompactionRequest request) throws IOException { + postCompact(e, store, resultFile); + } + + @Override + public void preGetClosestRowBefore(final ObserverContext e, + final byte [] row, final byte [] family, final Result result) + throws IOException { + } + + @Override + public void postGetClosestRowBefore(final ObserverContext e, + final byte [] row, final byte [] family, final Result result) + throws IOException { + } + + @Override + public void preGet(final ObserverContext e, + final Get get, final List results) throws IOException { + } + + @Override + public void postGet(final ObserverContext e, + final Get get, final List results) throws IOException { + } + + @Override + public boolean preExists(final ObserverContext e, + final Get get, final boolean exists) throws IOException { + return exists; + } + + @Override + public boolean postExists(final ObserverContext e, + final Get get, boolean exists) throws IOException { + return exists; + } + + @Override + public void prePut(final ObserverContext e, + final Put put, final WALEdit edit, final boolean writeToWAL) throws IOException { + } + + @Override + public void postPut(final ObserverContext e, + final Put put, final WALEdit edit, final boolean writeToWAL) throws IOException { + } + + @Override + public void preDelete(final ObserverContext e, + final Delete delete, final WALEdit edit, final boolean writeToWAL) throws IOException { + } + + @Override + public void postDelete(final ObserverContext e, + final Delete delete, final WALEdit edit, final boolean writeToWAL) throws IOException { + } + + @Override + public void preBatchMutate(final ObserverContext c, + final MiniBatchOperationInProgress> miniBatchOp) throws IOException { + } + + @Override + public void postBatchMutate(final ObserverContext c, + final MiniBatchOperationInProgress> miniBatchOp) throws IOException { + } + + @Override + public boolean preCheckAndPut(final ObserverContext e, + final byte [] row, final byte [] family, final byte [] qualifier, + final CompareOp compareOp, final WritableByteArrayComparable comparator, + final Put put, final boolean result) throws IOException { + return result; + } + + @Override + public boolean postCheckAndPut(final ObserverContext e, + final byte [] row, final byte [] family, final byte [] qualifier, + final CompareOp compareOp, final WritableByteArrayComparable comparator, + final Put put, final boolean result) throws IOException { + return result; + } + + @Override + public boolean preCheckAndDelete(final ObserverContext e, + final byte [] row, final byte [] family, final byte [] qualifier, + final CompareOp compareOp, final WritableByteArrayComparable comparator, + final Delete delete, final boolean result) throws IOException { + return result; + } + + @Override + public boolean postCheckAndDelete(final ObserverContext e, + final byte [] row, final byte [] family, final byte [] qualifier, + final CompareOp compareOp, final WritableByteArrayComparable comparator, + final Delete delete, final boolean result) throws IOException { + return result; + } + + @Override + public Result preAppend(final ObserverContext e, + final Append append) throws IOException { + return null; + } + + @Override + public Result postAppend(final ObserverContext e, + final Append append, final Result result) throws IOException { + return result; + } + + @Override + public long preIncrementColumnValue(final ObserverContext e, + final byte [] row, final byte [] family, final byte [] qualifier, + final long amount, final boolean writeToWAL) throws IOException { + return amount; + } + + @Override + public long postIncrementColumnValue(final ObserverContext e, + final byte [] row, final byte [] family, final byte [] qualifier, + final long amount, final boolean writeToWAL, long result) + throws IOException { + return result; + } + + @Override + public Result preIncrement(final ObserverContext e, + final Increment increment) throws IOException { + return null; + } + + @Override + public Result postIncrement(final ObserverContext e, + final Increment increment, final Result result) throws IOException { + return result; + } + + @Override + public RegionScanner preScannerOpen(final ObserverContext e, + final Scan scan, final RegionScanner s) throws IOException { + return s; + } + + @Override + public KeyValueScanner preStoreScannerOpen(final ObserverContext c, + final Store store, final Scan scan, final NavigableSet targetCols, + final KeyValueScanner s) throws IOException { + return s; + } + + @Override + public RegionScanner postScannerOpen(final ObserverContext e, + final Scan scan, final RegionScanner s) throws IOException { + return s; + } + + @Override + public boolean preScannerNext(final ObserverContext e, + final InternalScanner s, final List results, + final int limit, final boolean hasMore) throws IOException { + return hasMore; + } + + @Override + public boolean postScannerNext(final ObserverContext e, + final InternalScanner s, final List results, final int limit, + final boolean hasMore) throws IOException { + return hasMore; + } + + @Override + public boolean postScannerFilterRow(final ObserverContext e, + final InternalScanner s, final byte[] currentRow, final boolean hasMore) throws IOException { + return hasMore; + } + + @Override + public void preScannerClose(final ObserverContext e, + final InternalScanner s) throws IOException { + } + + @Override + public void postScannerClose(final ObserverContext e, + final InternalScanner s) throws IOException { + } + + @Override + public void preWALRestore(ObserverContext env, HRegionInfo info, + HLogKey logKey, WALEdit logEdit) throws IOException { + } + + @Override + public void postWALRestore(ObserverContext env, + HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { + } + + @Override + public void preBulkLoadHFile(final ObserverContext ctx, + List> familyPaths) throws IOException { + } + + @Override + public boolean postBulkLoadHFile(ObserverContext ctx, + List> familyPaths, boolean hasLoaded) throws IOException { + return hasLoaded; + } + + @Override + public void preLockRow(ObserverContext ctx, byte[] regionName, + byte[] row) throws IOException { } + + @Override + public void preUnlockRow(ObserverContext ctx, byte[] regionName, + long lockId) throws IOException { } + + @Override + public void postLockRow(ObserverContext ctx, byte[] regionName, + byte[] row) throws IOException { } + + @Override + public void postUnlockRow(ObserverContext ctx, byte[] regionName, + long lockId) throws IOException { } +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java new file mode 100644 index 0000000..149fe21 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java @@ -0,0 +1,114 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.coprocessor.LongColumnInterpreter; +import org.apache.hadoop.io.Writable; + +/** + * Defines how value for specific column is interpreted and provides utility + * methods like compare, add, multiply etc for them. Takes column family, column + * qualifier and return the cell value. Its concrete implementation should + * handle null case gracefully. Refer to {@link LongColumnInterpreter} for an + * example. + *

    + * Takes two generic parameters. The cell value type of the interpreter is . + * During some computations like sum, average, the return type can be different + * than the cell value data type, for eg, sum of int cell values might overflow + * in case of a int result, we should use Long for its result. Therefore, this + * class mandates to use a different (promoted) data type for result of these + * computations . All computations are performed on the promoted data type + * . There is a conversion method + * {@link ColumnInterpreter#castToReturnType(Object)} which takes a type and + * returns a type. + * @param Cell value data type + * @param Promoted data type + */ +public interface ColumnInterpreter extends Writable { + + /** + * @param colFamily + * @param colQualifier + * @param kv + * @return value of type T + * @throws IOException + */ + T getValue(byte[] colFamily, byte[] colQualifier, KeyValue kv) + throws IOException; + + /** + * @param l1 + * @param l2 + * @return sum or non null value among (if either of them is null); otherwise + * returns a null. + */ + public S add(S l1, S l2); + + /** + * returns the maximum value for this type T + * @return max + */ + + T getMaxValue(); + + T getMinValue(); + + /** + * @param o1 + * @param o2 + * @return multiplication + */ + S multiply(S o1, S o2); + + /** + * @param o + * @return increment + */ + S increment(S o); + + /** + * provides casting opportunity between the data types. + * @param o + * @return cast + */ + S castToReturnType(T o); + + /** + * This takes care if either of arguments are null. returns 0 if they are + * equal or both are null; + *

      + *
    • >0 if l1 > l2 or l1 is not null and l2 is null. + *
    • < 0 if l1 < l2 or l1 is null and l2 is not null. + */ + int compare(final T l1, final T l2); + + /** + * used for computing average of data values. Not providing the divide + * method that takes two values as it is not needed as of now. + * @param o + * @param l + * @return Average + */ + double divideForAvg(S o, Long l); +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorException.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorException.java new file mode 100644 index 0000000..d4344d0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorException.java @@ -0,0 +1,52 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.DoNotRetryIOException; + +/** + * Thrown if a coprocessor encounters any exception. + */ +public class CoprocessorException extends DoNotRetryIOException { + private static final long serialVersionUID = 4357922136679804887L; + + /** Default Constructor */ + public CoprocessorException() { + super(); + } + + /** + * Constructor with a Class object and exception message. + * @param clazz + * @param s + */ + public CoprocessorException(Class clazz, String s) { + super( "Coprocessor [" + clazz.getName() + "]: " + s); + } + + /** + * Constructs the exception and supplies a string as the message + * @param s - message + */ + public CoprocessorException(String s) { + super(s); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java new file mode 100644 index 0000000..838d056 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java @@ -0,0 +1,726 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.UUID; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Row; +import org.apache.hadoop.hbase.client.RowLock; +import org.apache.hadoop.hbase.client.RowMutations; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CoprocessorClassLoader; +import org.apache.hadoop.hbase.util.SortedCopyOnWriteSet; +import org.apache.hadoop.hbase.util.VersionInfo; + +/** + * Provides the common setup framework and runtime services for coprocessor + * invocation from HBase services. + * @param the specific environment extension that a concrete implementation + * provides + */ +public abstract class CoprocessorHost { + public static final String REGION_COPROCESSOR_CONF_KEY = + "hbase.coprocessor.region.classes"; + public static final String REGIONSERVER_COPROCESSOR_CONF_KEY = + "hbase.coprocessor.regionserver.classes"; + public static final String USER_REGION_COPROCESSOR_CONF_KEY = + "hbase.coprocessor.user.region.classes"; + public static final String MASTER_COPROCESSOR_CONF_KEY = + "hbase.coprocessor.master.classes"; + public static final String WAL_COPROCESSOR_CONF_KEY = + "hbase.coprocessor.wal.classes"; + + private static final Log LOG = LogFactory.getLog(CoprocessorHost.class); + /** Ordered set of loaded coprocessors with lock */ + protected SortedSet coprocessors = + new SortedCopyOnWriteSet(new EnvironmentPriorityComparator()); + protected Configuration conf; + // unique file prefix to use for local copies of jars when classloading + protected String pathPrefix; + protected volatile int loadSequence; + + public CoprocessorHost() { + pathPrefix = UUID.randomUUID().toString(); + } + + /** + * Not to be confused with the per-object _coprocessors_ (above), + * coprocessorNames is static and stores the set of all coprocessors ever + * loaded by any thread in this JVM. It is strictly additive: coprocessors are + * added to coprocessorNames, by loadInstance() but are never removed, since + * the intention is to preserve a history of all loaded coprocessors for + * diagnosis in case of server crash (HBASE-4014). + */ + private static Set coprocessorNames = + Collections.synchronizedSet(new HashSet()); + public static Set getLoadedCoprocessors() { + return coprocessorNames; + } + + /** + * Used to create a parameter to the HServerLoad constructor so that + * HServerLoad can provide information about the coprocessors loaded by this + * regionserver. + * (HBASE-4070: Improve region server metrics to report loaded coprocessors + * to master). + */ + public Set getCoprocessors() { + Set returnValue = new TreeSet(); + for(CoprocessorEnvironment e: coprocessors) { + returnValue.add(e.getInstance().getClass().getSimpleName()); + } + return returnValue; + } + + /** + * Load system coprocessors. Read the class names from configuration. + * Called by constructor. + */ + protected void loadSystemCoprocessors(Configuration conf, String confKey) { + Class implClass = null; + + // load default coprocessors from configure file + String[] defaultCPClasses = conf.getStrings(confKey); + if (defaultCPClasses == null || defaultCPClasses.length == 0) + return; + + int priority = Coprocessor.PRIORITY_SYSTEM; + List configured = new ArrayList(); + for (String className : defaultCPClasses) { + className = className.trim(); + if (findCoprocessor(className) != null) { + continue; + } + ClassLoader cl = this.getClass().getClassLoader(); + Thread.currentThread().setContextClassLoader(cl); + try { + implClass = cl.loadClass(className); + configured.add(loadInstance(implClass, Coprocessor.PRIORITY_SYSTEM, conf)); + LOG.info("System coprocessor " + className + " was loaded " + + "successfully with priority (" + priority++ + ")."); + } catch (ClassNotFoundException e) { + LOG.warn("Class " + className + " cannot be found. " + + e.getMessage()); + } catch (IOException e) { + LOG.warn("Load coprocessor " + className + " failed. " + + e.getMessage()); + } + } + + // add entire set to the collection for COW efficiency + coprocessors.addAll(configured); + } + + /** + * Load a coprocessor implementation into the host + * @param path path to implementation jar + * @param className the main class name + * @param priority chaining priority + * @param conf configuration for coprocessor + * @throws java.io.IOException Exception + */ + @SuppressWarnings("deprecation") + public E load(Path path, String className, int priority, + Configuration conf) throws IOException { + Class implClass = null; + LOG.debug("Loading coprocessor class " + className + " with path " + + path + " and priority " + priority); + + ClassLoader cl = null; + if (path == null) { + try { + implClass = getClass().getClassLoader().loadClass(className); + } catch (ClassNotFoundException e) { + throw new IOException("No jar path specified for " + className); + } + } else { + cl = CoprocessorClassLoader.getClassLoader( + path, getClass().getClassLoader(), pathPrefix, conf); + try { + implClass = cl.loadClass(className); + } catch (ClassNotFoundException e) { + throw new IOException("Cannot load external coprocessor class " + className, e); + } + } + + //load custom code for coprocessor + Thread currentThread = Thread.currentThread(); + ClassLoader hostClassLoader = currentThread.getContextClassLoader(); + try{ + // switch temporarily to the thread classloader for custom CP + currentThread.setContextClassLoader(cl); + E cpInstance = loadInstance(implClass, priority, conf); + return cpInstance; + } finally { + // restore the fresh (host) classloader + currentThread.setContextClassLoader(hostClassLoader); + } + } + + /** + * @param implClass Implementation class + * @param priority priority + * @param conf configuration + * @throws java.io.IOException Exception + */ + public void load(Class implClass, int priority, Configuration conf) + throws IOException { + E env = loadInstance(implClass, priority, conf); + coprocessors.add(env); + } + + /** + * @param implClass Implementation class + * @param priority priority + * @param conf configuration + * @throws java.io.IOException Exception + */ + public E loadInstance(Class implClass, int priority, Configuration conf) + throws IOException { + // create the instance + Coprocessor impl; + Object o = null; + try { + o = implClass.newInstance(); + impl = (Coprocessor)o; + } catch (InstantiationException e) { + throw new IOException(e); + } catch (IllegalAccessException e) { + throw new IOException(e); + } + // create the environment + E env = createEnvironment(implClass, impl, priority, ++loadSequence, conf); + if (env instanceof Environment) { + ((Environment)env).startup(); + } + // HBASE-4014: maintain list of loaded coprocessors for later crash analysis + // if server (master or regionserver) aborts. + coprocessorNames.add(implClass.getName()); + return env; + } + + /** + * Called when a new Coprocessor class is loaded + */ + public abstract E createEnvironment(Class implClass, Coprocessor instance, + int priority, int sequence, Configuration conf); + + public void shutdown(CoprocessorEnvironment e) { + if (e instanceof Environment) { + ((Environment)e).shutdown(); + } else { + LOG.warn("Shutdown called on unknown environment: "+ + e.getClass().getName()); + } + } + + /** + * Find a coprocessor implementation by class name + * @param className the class name + * @return the coprocessor, or null if not found + */ + public Coprocessor findCoprocessor(String className) { + // initialize the coprocessors + for (E env: coprocessors) { + if (env.getInstance().getClass().getName().equals(className) || + env.getInstance().getClass().getSimpleName().equals(className)) { + return env.getInstance(); + } + } + return null; + } + + /** + * Retrieves the set of classloaders used to instantiate Coprocessor classes defined in external + * jar files. + * @return A set of ClassLoader instances + */ + Set getExternalClassLoaders() { + Set externalClassLoaders = new HashSet(); + final ClassLoader systemClassLoader = this.getClass().getClassLoader(); + for (E env : coprocessors) { + ClassLoader cl = env.getInstance().getClass().getClassLoader(); + if (cl != systemClassLoader ){ + //do not include system classloader + externalClassLoaders.add(cl); + } + } + return externalClassLoaders; + } + + /** + * Find a coprocessor environment by class name + * @param className the class name + * @return the coprocessor, or null if not found + */ + public CoprocessorEnvironment findCoprocessorEnvironment(String className) { + // initialize the coprocessors + for (E env: coprocessors) { + if (env.getInstance().getClass().getName().equals(className) || + env.getInstance().getClass().getSimpleName().equals(className)) { + return env; + } + } + return null; + } + + /** + * Environment priority comparator. + * Coprocessors are chained in sorted order. + */ + static class EnvironmentPriorityComparator + implements Comparator { + public int compare(final CoprocessorEnvironment env1, + final CoprocessorEnvironment env2) { + if (env1.getPriority() < env2.getPriority()) { + return -1; + } else if (env1.getPriority() > env2.getPriority()) { + return 1; + } + if (env1.getLoadSequence() < env2.getLoadSequence()) { + return -1; + } else if (env1.getLoadSequence() > env2.getLoadSequence()) { + return 1; + } + return 0; + } + } + + /** + * Encapsulation of the environment of each coprocessor + */ + public static class Environment implements CoprocessorEnvironment { + + /** + * A wrapper for HTable. Can be used to restrict privilege. + * + * Currently it just helps to track tables opened by a Coprocessor and + * facilitate close of them if it is aborted. + * + * We also disallow row locking. + * + * There is nothing now that will stop a coprocessor from using HTable + * objects directly instead of this API, but in the future we intend to + * analyze coprocessor implementations as they are loaded and reject those + * which attempt to use objects and methods outside the Environment + * sandbox. + */ + class HTableWrapper implements HTableInterface { + + private byte[] tableName; + private HTable table; + + public HTableWrapper(byte[] tableName) throws IOException { + this.tableName = tableName; + this.table = new HTable(conf, tableName); + openTables.add(this); + } + + void internalClose() throws IOException { + table.close(); + } + + public Configuration getConfiguration() { + return table.getConfiguration(); + } + + public void close() throws IOException { + try { + internalClose(); + } finally { + openTables.remove(this); + } + } + + public Result getRowOrBefore(byte[] row, byte[] family) + throws IOException { + return table.getRowOrBefore(row, family); + } + + public Result get(Get get) throws IOException { + return table.get(get); + } + + public boolean exists(Get get) throws IOException { + return table.exists(get); + } + + public void put(Put put) throws IOException { + table.put(put); + } + + public void put(List puts) throws IOException { + table.put(puts); + } + + public void delete(Delete delete) throws IOException { + table.delete(delete); + } + + public void delete(List deletes) throws IOException { + table.delete(deletes); + } + + public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, + byte[] value, Put put) throws IOException { + return table.checkAndPut(row, family, qualifier, value, put); + } + + public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, + byte[] value, Delete delete) throws IOException { + return table.checkAndDelete(row, family, qualifier, value, delete); + } + + public long incrementColumnValue(byte[] row, byte[] family, + byte[] qualifier, long amount) throws IOException { + return table.incrementColumnValue(row, family, qualifier, amount); + } + + public long incrementColumnValue(byte[] row, byte[] family, + byte[] qualifier, long amount, boolean writeToWAL) + throws IOException { + return table.incrementColumnValue(row, family, qualifier, amount, + writeToWAL); + } + + @Override + public Result append(Append append) throws IOException { + return table.append(append); + } + + @Override + public Result increment(Increment increment) throws IOException { + return table.increment(increment); + } + + public void flushCommits() throws IOException { + table.flushCommits(); + } + + public boolean isAutoFlush() { + return table.isAutoFlush(); + } + + public ResultScanner getScanner(Scan scan) throws IOException { + return table.getScanner(scan); + } + + public ResultScanner getScanner(byte[] family) throws IOException { + return table.getScanner(family); + } + + public ResultScanner getScanner(byte[] family, byte[] qualifier) + throws IOException { + return table.getScanner(family, qualifier); + } + + public HTableDescriptor getTableDescriptor() throws IOException { + return table.getTableDescriptor(); + } + + public byte[] getTableName() { + return tableName; + } + + /** + * @deprecated {@link RowLock} and associated operations are deprecated. + */ + public RowLock lockRow(byte[] row) throws IOException { + throw new RuntimeException( + "row locking is not allowed within the coprocessor environment"); + } + + /** + * @deprecated {@link RowLock} and associated operations are deprecated. + */ + public void unlockRow(RowLock rl) throws IOException { + throw new RuntimeException( + "row locking is not allowed within the coprocessor environment"); + } + + @Override + public void batch(List actions, Object[] results) + throws IOException, InterruptedException { + table.batch(actions, results); + } + + @Override + public Object[] batch(List actions) + throws IOException, InterruptedException { + return table.batch(actions); + } + + @Override + public Result[] get(List gets) throws IOException { + return table.get(gets); + } + + @Override + public void coprocessorExec(Class protocol, + byte[] startKey, byte[] endKey, Batch.Call callable, + Batch.Callback callback) throws IOException, Throwable { + table.coprocessorExec(protocol, startKey, endKey, callable, callback); + } + + @Override + public Map coprocessorExec( + Class protocol, byte[] startKey, byte[] endKey, Batch.Call callable) + throws IOException, Throwable { + return table.coprocessorExec(protocol, startKey, endKey, callable); + } + + @Override + public T coprocessorProxy(Class protocol, + byte[] row) { + return table.coprocessorProxy(protocol, row); + } + + @Override + public void mutateRow(RowMutations rm) throws IOException { + table.mutateRow(rm); + } + + @Override + public void setAutoFlush(boolean autoFlush) { + table.setAutoFlush(autoFlush); + } + + @Override + public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) { + table.setAutoFlush(autoFlush, clearBufferOnFail); + } + + @Override + public long getWriteBufferSize() { + return table.getWriteBufferSize(); + } + + @Override + public void setWriteBufferSize(long writeBufferSize) throws IOException { + table.setWriteBufferSize(writeBufferSize); + } + } + + /** The coprocessor */ + public Coprocessor impl; + /** Chaining priority */ + protected int priority = Coprocessor.PRIORITY_USER; + /** Current coprocessor state */ + Coprocessor.State state = Coprocessor.State.UNINSTALLED; + /** Accounting for tables opened by the coprocessor */ + protected List openTables = + Collections.synchronizedList(new ArrayList()); + private int seq; + private Configuration conf; + + /** + * Constructor + * @param impl the coprocessor instance + * @param priority chaining priority + */ + public Environment(final Coprocessor impl, final int priority, + final int seq, final Configuration conf) { + this.impl = impl; + this.priority = priority; + this.state = Coprocessor.State.INSTALLED; + this.seq = seq; + this.conf = conf; + } + + /** Initialize the environment */ + public void startup() { + if (state == Coprocessor.State.INSTALLED || + state == Coprocessor.State.STOPPED) { + state = Coprocessor.State.STARTING; + try { + impl.start(this); + state = Coprocessor.State.ACTIVE; + } catch (IOException ioe) { + LOG.error("Error starting coprocessor "+impl.getClass().getName(), ioe); + } + } else { + LOG.warn("Not starting coprocessor "+impl.getClass().getName()+ + " because not inactive (state="+state.toString()+")"); + } + } + + /** Clean up the environment */ + protected void shutdown() { + if (state == Coprocessor.State.ACTIVE) { + state = Coprocessor.State.STOPPING; + try { + impl.stop(this); + state = Coprocessor.State.STOPPED; + } catch (IOException ioe) { + LOG.error("Error stopping coprocessor "+impl.getClass().getName(), ioe); + } + } else { + LOG.warn("Not stopping coprocessor "+impl.getClass().getName()+ + " because not active (state="+state.toString()+")"); + } + // clean up any table references + for (HTableInterface table: openTables) { + try { + ((HTableWrapper)table).internalClose(); + } catch (IOException e) { + // nothing can be done here + LOG.warn("Failed to close " + + Bytes.toStringBinary(table.getTableName()), e); + } + } + } + + @Override + public Coprocessor getInstance() { + return impl; + } + + @Override + public int getPriority() { + return priority; + } + + @Override + public int getLoadSequence() { + return seq; + } + + /** @return the coprocessor environment version */ + @Override + public int getVersion() { + return Coprocessor.VERSION; + } + + /** @return the HBase release */ + @Override + public String getHBaseVersion() { + return VersionInfo.getVersion(); + } + + @Override + public Configuration getConfiguration() { + return conf; + } + + /** + * Open a table from within the Coprocessor environment + * @param tableName the table name + * @return an interface for manipulating the table + * @exception java.io.IOException Exception + */ + @Override + public HTableInterface getTable(byte[] tableName) throws IOException { + return new HTableWrapper(tableName); + } + } + + protected void abortServer(final String service, + final Server server, + final CoprocessorEnvironment environment, + final Throwable e) { + String coprocessorName = (environment.getInstance()).toString(); + server.abort("Aborting service: " + service + " running on : " + + server.getServerName() + " because coprocessor: " + + coprocessorName + " threw an exception.", e); + } + + protected void abortServer(final CoprocessorEnvironment environment, + final Throwable e) { + String coprocessorName = (environment.getInstance()).toString(); + LOG.error("The coprocessor: " + coprocessorName + " threw an unexpected " + + "exception: " + e + ", but there's no specific implementation of " + + " abortServer() for this coprocessor's environment."); + } + + + /** + * This is used by coprocessor hooks which are declared to throw IOException + * (or its subtypes). For such hooks, we should handle throwable objects + * depending on the Throwable's type. Those which are instances of + * IOException should be passed on to the client. This is in conformance with + * the HBase idiom regarding IOException: that it represents a circumstance + * that should be passed along to the client for its own handling. For + * example, a coprocessor that implements access controls would throw a + * subclass of IOException, such as AccessDeniedException, in its preGet() + * method to prevent an unauthorized client's performing a Get on a particular + * table. + * @param env Coprocessor Environment + * @param e Throwable object thrown by coprocessor. + * @exception IOException Exception + */ + protected void handleCoprocessorThrowable(final CoprocessorEnvironment env, + final Throwable e) + throws IOException { + if (e instanceof IOException) { + throw (IOException)e; + } + // If we got here, e is not an IOException. A loaded coprocessor has a + // fatal bug, and the server (master or regionserver) should remove the + // faulty coprocessor from its set of active coprocessors. Setting + // 'hbase.coprocessor.abortonerror' to true will cause abortServer(), + // which may be useful in development and testing environments where + // 'failing fast' for error analysis is desired. + if (env.getConfiguration().getBoolean("hbase.coprocessor.abortonerror",false)) { + // server is configured to abort. + abortServer(env, e); + } else { + LOG.error("Removing coprocessor '" + env.toString() + "' from " + + "environment because it threw: " + e,e); + coprocessors.remove(env); + throw new DoNotRetryIOException("Coprocessor: '" + env.toString() + + "' threw: '" + e + "' and has been removed" + "from the active " + + "coprocessor set.", e); + } + } +} + + diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java new file mode 100644 index 0000000..75f0653 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java @@ -0,0 +1,29 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.master.MasterServices; + +public interface MasterCoprocessorEnvironment extends CoprocessorEnvironment { + /** @return reference to the HMaster services */ + MasterServices getMasterServices(); +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java new file mode 100644 index 0000000..319112c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java @@ -0,0 +1,582 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; + +import java.io.IOException; + +/** + * Defines coprocessor hooks for interacting with operations on the + * {@link org.apache.hadoop.hbase.master.HMaster} process. + */ +public interface MasterObserver extends Coprocessor { + + /** + * Called before a new table is created by + * {@link org.apache.hadoop.hbase.master.HMaster}. Called as part of create + * table RPC call + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param desc the HTableDescriptor for the table + * @param regions the initial regions created for the table + * @throws IOException + */ + void preCreateTable(final ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException; + + /** + * Called after the createTable operation has been requested. Called as part + * of create table RPC call. + * @param ctx the environment to interact with the framework and master + * @param desc the HTableDescriptor for the table + * @param regions the initial regions created for the table + * @throws IOException + */ + void postCreateTable(final ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException; + + /** + * Called before a new table is created by {@link org.apache.hadoop.hbase.master.HMaster}. Called + * as part of create table handler and it is async to the create RPC call. It can't bypass the + * default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param desc the HTableDescriptor for the table + * @param regions the initial regions created for the table + * @throws IOException + */ + void preCreateTableHandler(final ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException; + + /** + * Called after the createTable operation has been requested. Called as part of create table RPC + * call. Called as part of create table handler and it is async to the create RPC call. + * @param ctx the environment to interact with the framework and master + * @param desc the HTableDescriptor for the table + * @param regions the initial regions created for the table + * @throws IOException + */ + void postCreateTableHandler(final ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException; + + /** + * Called before {@link org.apache.hadoop.hbase.master.HMaster} deletes a + * table. Called as part of delete table RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + */ + void preDeleteTable(final ObserverContext ctx, + byte[] tableName) throws IOException; + + /** + * Called after the deleteTable operation has been requested. Called as part + * of delete table RPC call. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + */ + void postDeleteTable(final ObserverContext ctx, + byte[] tableName) throws IOException; + + /** + * Called before {@link org.apache.hadoop.hbase.master.HMaster} deletes a + * table. Called as part of delete table handler and + * it is async to the delete RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + */ + void preDeleteTableHandler( + final ObserverContext ctx, byte[] tableName) + throws IOException; + + /** + * Called after {@link org.apache.hadoop.hbase.master.HMaster} deletes a + * table. Called as part of delete table handler and it is async to the + * delete RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + */ + void postDeleteTableHandler( + final ObserverContext ctx, byte[] tableName) + throws IOException; + + /** + * Called prior to modifying a table's properties. Called as part of modify + * table RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param htd the HTableDescriptor + */ + void preModifyTable(final ObserverContext ctx, + final byte[] tableName, HTableDescriptor htd) throws IOException; + + /** + * Called after the modifyTable operation has been requested. Called as part + * of modify table RPC call. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param htd the HTableDescriptor + */ + void postModifyTable(final ObserverContext ctx, + final byte[] tableName, HTableDescriptor htd) throws IOException; + + /** + * Called prior to modifying a table's properties. Called as part of modify + * table handler and it is async to the modify table RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param htd the HTableDescriptor + */ + void preModifyTableHandler( + final ObserverContext ctx, + final byte[] tableName, HTableDescriptor htd) throws IOException; + + /** + * Called after to modifying a table's properties. Called as part of modify + * table handler and it is async to the modify table RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param htd the HTableDescriptor + */ + void postModifyTableHandler( + final ObserverContext ctx, + final byte[] tableName, HTableDescriptor htd) throws IOException; + + /** + * Called prior to adding a new column family to the table. Called as part of + * add column RPC call. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param column the HColumnDescriptor + */ + void preAddColumn(final ObserverContext ctx, + byte[] tableName, HColumnDescriptor column) throws IOException; + + /** + * Called after the new column family has been created. Called as part of + * add column RPC call. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param column the HColumnDescriptor + */ + void postAddColumn(final ObserverContext ctx, + byte[] tableName, HColumnDescriptor column) throws IOException; + + /** + * Called prior to adding a new column family to the table. Called as part of + * add column handler. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param column the HColumnDescriptor + */ + void preAddColumnHandler( + final ObserverContext ctx, + byte[] tableName, HColumnDescriptor column) throws IOException; + + /** + * Called after the new column family has been created. Called as part of + * add column handler. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param column the HColumnDescriptor + */ + void postAddColumnHandler( + final ObserverContext ctx, + byte[] tableName, HColumnDescriptor column) throws IOException; + + /** + * Called prior to modifying a column family's attributes. Called as part of + * modify column RPC call. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param descriptor the HColumnDescriptor + */ + void preModifyColumn(final ObserverContext ctx, + byte [] tableName, HColumnDescriptor descriptor) throws IOException; + + /** + * Called after the column family has been updated. Called as part of modify + * column RPC call. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param descriptor the HColumnDescriptor + */ + void postModifyColumn(final ObserverContext ctx, + byte[] tableName, HColumnDescriptor descriptor) throws IOException; + + /** + * Called prior to modifying a column family's attributes. Called as part of + * modify column handler. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param descriptor the HColumnDescriptor + */ + void preModifyColumnHandler( + final ObserverContext ctx, + byte[] tableName, HColumnDescriptor descriptor) throws IOException; + + /** + * Called after the column family has been updated. Called as part of modify + * column handler. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param descriptor the HColumnDescriptor + */ + void postModifyColumnHandler( + final ObserverContext ctx, + byte[] tableName, HColumnDescriptor descriptor) throws IOException; + + /** + * Called prior to deleting the entire column family. Called as part of + * delete column RPC call. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param c the column + */ + void preDeleteColumn(final ObserverContext ctx, + final byte [] tableName, final byte[] c) throws IOException; + + /** + * Called after the column family has been deleted. Called as part of delete + * column RPC call. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param c the column + */ + void postDeleteColumn(final ObserverContext ctx, + final byte [] tableName, final byte[] c) throws IOException; + + /** + * Called prior to deleting the entire column family. Called as part of + * delete column handler. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param c the column + */ + void preDeleteColumnHandler( + final ObserverContext ctx, + final byte[] tableName, final byte[] c) throws IOException; + + /** + * Called after the column family has been deleted. Called as part of + * delete column handler. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + * @param c the column + */ + void postDeleteColumnHandler( + final ObserverContext ctx, + final byte[] tableName, final byte[] c) throws IOException; + + /** + * Called prior to enabling a table. Called as part of enable table RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + */ + void preEnableTable(final ObserverContext ctx, + final byte[] tableName) throws IOException; + + /** + * Called after the enableTable operation has been requested. Called as part + * of enable table RPC call. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + */ + void postEnableTable(final ObserverContext ctx, + final byte[] tableName) throws IOException; + + /** + * Called prior to enabling a table. Called as part of enable table handler + * and it is async to the enable table RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + */ + void preEnableTableHandler( + final ObserverContext ctx, + final byte[] tableName) throws IOException; + + /** + * Called after the enableTable operation has been requested. Called as part + * of enable table handler and it is async to the enable table RPC call. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + */ + void postEnableTableHandler( + final ObserverContext ctx, + final byte[] tableName) throws IOException; + + /** + * Called prior to disabling a table. Called as part of disable table RPC + * call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + */ + void preDisableTable(final ObserverContext ctx, + final byte[] tableName) throws IOException; + + /** + * Called after the disableTable operation has been requested. Called as part + * of disable table RPC call. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + */ + void postDisableTable(final ObserverContext ctx, + final byte[] tableName) throws IOException; + + /** + * Called prior to disabling a table. Called as part of disable table handler + * and it is asyn to the disable table RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + */ + void preDisableTableHandler( + final ObserverContext ctx, + final byte[] tableName) throws IOException; + + /** + * Called after the disableTable operation has been requested. Called as part + * of disable table handler and it is asyn to the disable table RPC call. + * @param ctx the environment to interact with the framework and master + * @param tableName the name of the table + */ + void postDisableTableHandler( + final ObserverContext ctx, + final byte[] tableName) throws IOException; + + /** + * Called prior to moving a given region from one region server to another. + * @param ctx the environment to interact with the framework and master + * @param region the HRegionInfo + * @param srcServer the source ServerName + * @param destServer the destination ServerName + */ + void preMove(final ObserverContext ctx, + final HRegionInfo region, final ServerName srcServer, + final ServerName destServer) + throws IOException; + + /** + * Called after the region move has been requested. + * @param ctx the environment to interact with the framework and master + * @param region the HRegionInfo + * @param srcServer the source ServerName + * @param destServer the destination ServerName + */ + void postMove(final ObserverContext ctx, + final HRegionInfo region, final ServerName srcServer, + final ServerName destServer) + throws IOException; + + /** + * Called prior to assigning a specific region. + * @param ctx the environment to interact with the framework and master + * @param regionInfo the regionInfo of the region + */ + void preAssign(final ObserverContext ctx, + final HRegionInfo regionInfo) throws IOException; + + /** + * Called after the region assignment has been requested. + * @param ctx the environment to interact with the framework and master + * @param regionInfo the regionInfo of the region + */ + void postAssign(final ObserverContext ctx, + final HRegionInfo regionInfo) throws IOException; + + /** + * Called prior to unassigning a given region. + * @param ctx the environment to interact with the framework and master + * @param regionInfo + * @param force whether to force unassignment or not + */ + void preUnassign(final ObserverContext ctx, + final HRegionInfo regionInfo, final boolean force) throws IOException; + + /** + * Called after the region unassignment has been requested. + * @param ctx the environment to interact with the framework and master + * @param regionInfo + * @param force whether to force unassignment or not + */ + void postUnassign(final ObserverContext ctx, + final HRegionInfo regionInfo, final boolean force) throws IOException; + + /** + * Called prior to requesting rebalancing of the cluster regions, though after + * the initial checks for regions in transition and the balance switch flag. + * @param ctx the environment to interact with the framework and master + */ + void preBalance(final ObserverContext ctx) + throws IOException; + + /** + * Called after the balancing plan has been submitted. + * @param ctx the environment to interact with the framework and master + */ + void postBalance(final ObserverContext ctx) + throws IOException; + + /** + * Called prior to modifying the flag used to enable/disable region balancing. + * @param ctx the coprocessor instance's environment + * @param newValue the new flag value submitted in the call + */ + boolean preBalanceSwitch(final ObserverContext ctx, + final boolean newValue) throws IOException; + + /** + * Called after the flag to enable/disable balancing has changed. + * @param ctx the coprocessor instance's environment + * @param oldValue the previously set balanceSwitch value + * @param newValue the newly set balanceSwitch value + */ + void postBalanceSwitch(final ObserverContext ctx, + final boolean oldValue, final boolean newValue) throws IOException; + + /** + * Called prior to shutting down the full HBase cluster, including this + * {@link org.apache.hadoop.hbase.master.HMaster} process. + */ + void preShutdown(final ObserverContext ctx) + throws IOException; + + + /** + * Called immediately prior to stopping this + * {@link org.apache.hadoop.hbase.master.HMaster} process. + */ + void preStopMaster(final ObserverContext ctx) + throws IOException; + + /** + * Called immediately after an active master instance has completed + * initialization. Will not be called on standby master instances unless + * they take over the active role. + */ + void postStartMaster(final ObserverContext ctx) + throws IOException; + + /** + * Called before a new snapshot is taken. + * Called as part of snapshot RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param snapshot the SnapshotDescriptor for the snapshot + * @param hTableDescriptor the hTableDescriptor of the table to snapshot + * @throws IOException + */ + void preSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException; + + /** + * Called after the snapshot operation has been requested. + * Called as part of snapshot RPC call. + * @param ctx the environment to interact with the framework and master + * @param snapshot the SnapshotDescriptor for the snapshot + * @param hTableDescriptor the hTableDescriptor of the table to snapshot + * @throws IOException + */ + void postSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException; + + /** + * Called before a snapshot is cloned. + * Called as part of restoreSnapshot RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param snapshot the SnapshotDescriptor for the snapshot + * @param hTableDescriptor the hTableDescriptor of the table to create + * @throws IOException + */ + void preCloneSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException; + + /** + * Called after a snapshot clone operation has been requested. + * Called as part of restoreSnapshot RPC call. + * @param ctx the environment to interact with the framework and master + * @param snapshot the SnapshotDescriptor for the snapshot + * @param hTableDescriptor the hTableDescriptor of the table to create + * @throws IOException + */ + void postCloneSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException; + + /** + * Called before a snapshot is restored. + * Called as part of restoreSnapshot RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param snapshot the SnapshotDescriptor for the snapshot + * @param hTableDescriptor the hTableDescriptor of the table to restore + * @throws IOException + */ + void preRestoreSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException; + + /** + * Called after a snapshot restore operation has been requested. + * Called as part of restoreSnapshot RPC call. + * @param ctx the environment to interact with the framework and master + * @param snapshot the SnapshotDescriptor for the snapshot + * @param hTableDescriptor the hTableDescriptor of the table to restore + * @throws IOException + */ + void postRestoreSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException; + + /** + * Called before a snapshot is deleted. + * Called as part of deleteSnapshot RPC call. + * It can't bypass the default action, e.g., ctx.bypass() won't have effect. + * @param ctx the environment to interact with the framework and master + * @param snapshot the SnapshotDescriptor of the snapshot to delete + * @throws IOException + */ + void preDeleteSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot) throws IOException; + + /** + * Called after the delete snapshot operation has been requested. + * Called as part of deleteSnapshot RPC call. + * @param ctx the environment to interact with the framework and master + * @param snapshot the SnapshotDescriptor of the snapshot to delete + * @throws IOException + */ + void postDeleteSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot) throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserverExt.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserverExt.java new file mode 100644 index 0000000..622363b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserverExt.java @@ -0,0 +1,37 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; + +/** + * This is an extension for the MasterObserver interface. The APIs added into this interface are not + * exposed by HBase. This is internally being used by CMWH HBase. Customer should not make use of + * this interface points.
      + * Note : The APIs in this interface is subject to change at any time. + */ +public interface MasterObserverExt { + + /** + * Call before the master initialization is set to true. + */ + void preMasterInitialization(final ObserverContext ctx) + throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/MultiRowMutationEndpoint.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/MultiRowMutationEndpoint.java new file mode 100644 index 0000000..c640eae --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/MultiRowMutationEndpoint.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.WrongRegionException; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * This class demonstrates how to implement atomic multi row transactions using + * {@link HRegion#mutateRowsWithLocks(java.util.Collection, java.util.Collection)} + * and Coprocessor endpoints. + */ +public class MultiRowMutationEndpoint extends BaseEndpointCoprocessor implements + MultiRowMutationProtocol { + + @Override + public void mutateRows(List mutations) throws IOException { + // get the coprocessor environment + RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) getEnvironment(); + + // set of rows to lock, sorted to avoid deadlocks + SortedSet rowsToLock = new TreeSet(Bytes.BYTES_COMPARATOR); + + HRegionInfo regionInfo = env.getRegion().getRegionInfo(); + for (Mutation m : mutations) { + // check whether rows are in range for this region + if (!HRegion.rowIsInRange(regionInfo, m.getRow())) { + String msg = "Requested row out of range '" + + Bytes.toStringBinary(m.getRow()) + "'"; + if (rowsToLock.isEmpty()) { + // if this is the first row, region might have moved, + // allow client to retry + throw new WrongRegionException(msg); + } else { + // rows are split between regions, do not retry + throw new DoNotRetryIOException(msg); + } + } + rowsToLock.add(m.getRow()); + } + // call utility method on region + env.getRegion().mutateRowsWithLocks(mutations, rowsToLock); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/MultiRowMutationProtocol.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/MultiRowMutationProtocol.java new file mode 100644 index 0000000..e8eea9f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/MultiRowMutationProtocol.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.regionserver.HRegion; + +/** + * Defines a protocol to perform multi row transactions. + * See {@link MultiRowMutationEndpoint} for the implementation. + *
      + * See + * {@link HRegion#mutateRowsWithLocks(java.util.Collection, java.util.Collection)} + * for details and limitations. + *
      + * Example: + * 
      + * List mutations = ...;
+ * Put p1 = new Put(row1);
+ * Put p2 = new Put(row2);
+ * ...
+ * mutations.add(p1);
+ * mutations.add(p2);
+ * MultiRowMutationProtocol mrOp = t.coprocessorProxy(
+ *   MultiRowMutationProtocol.class, row1);
+ * mrOp.mutateRows(mutations);
+ *
      + */ +public interface MultiRowMutationProtocol extends CoprocessorProtocol { + public void mutateRows(List mutations) throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/ObserverContext.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/ObserverContext.java new file mode 100644 index 0000000..d5cf6aa --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/ObserverContext.java @@ -0,0 +1,110 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.CoprocessorEnvironment; + +/** + * Carries the execution state for a given invocation of an Observer coprocessor + * ({@link RegionObserver}, {@link MasterObserver}, or {@link WALObserver}) + * method. The same ObserverContext instance is passed sequentially to all loaded + * coprocessors for a given Observer method trigger, with the + * CoprocessorEnvironment reference swapped out for each + * coprocessor. + * @param The {@link CoprocessorEnvironment} subclass applicable to the + * revelant Observer interface. + */ +public class ObserverContext { + private E env; + private boolean bypass; + private boolean complete; + + public ObserverContext() { + } + + public E getEnvironment() { + return env; + } + + public void prepare(E env) { + this.env = env; + } + + /** + * Call to indicate that the current coprocessor's return value should be + * used in place of the normal HBase obtained value. + */ + public void bypass() { + bypass = true; + } + + /** + * Call to indicate that additional coprocessors further down the execution + * chain do not need to be invoked. Implies that this coprocessor's response + * is definitive. + */ + public void complete() { + complete = true; + } + + /** + * For use by the coprocessor framework. + * @return true if {@link ObserverContext#bypass()} + * was called by one of the loaded coprocessors, false otherwise. + */ + public boolean shouldBypass() { + boolean current = bypass; + bypass = false; + return current; + } + + /** + * For use by the coprocessor framework. + * @return true if {@link ObserverContext#complete()} + * was called by one of the loaded coprocessors, false otherwise. + */ + public boolean shouldComplete() { + boolean current = complete; + complete = false; + return current; + } + + /** + * Instantiates a new ObserverContext instance if the passed reference is + * null and sets the environment in the new or existing instance. + * This allows deferring the instantiation of a ObserverContext until it is + * actually needed. + * + * @param env The coprocessor environment to set + * @param context An existing ObserverContext instance to use, or null + * to create a new instance + * @param The environment type for the context + * @return An instance of ObserverContext with the environment set + */ + public static ObserverContext createAndPrepare( + T env, ObserverContext context) { + if (context == null) { + context = new ObserverContext(); + } + context.prepare(env); + return context; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java new file mode 100644 index 0000000..35a5c19 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java @@ -0,0 +1,39 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.util.concurrent.ConcurrentMap; + +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; + +public interface RegionCoprocessorEnvironment extends CoprocessorEnvironment { + /** @return the region associated with this coprocessor */ + public HRegion getRegion(); + + /** @return reference to the region server services */ + public RegionServerServices getRegionServerServices(); + + /** @return shared data between all instances of this coprocessor */ + public ConcurrentMap getSharedData(); + +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java new file mode 100644 index 0000000..93ff252 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java @@ -0,0 +1,957 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.List; +import java.util.NavigableSet; + +import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFileScanner; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; + +import com.google.common.collect.ImmutableList; +import org.apache.hadoop.hbase.util.Pair; + +/** + * Coprocessors implement this interface to observe and mediate client actions + * on the region. + */ +public interface RegionObserver extends Coprocessor { + + /** + * Called before the region is reported as open to the master. + * @param c the environment provided by the region server + * @throws IOException if an error occurred on the coprocessor + */ + void preOpen(final ObserverContext c) throws IOException; + + /** + * Called after the region is reported as open to the master. + * @param c the environment provided by the region server + */ + void postOpen(final ObserverContext c); + + /** + * Called before a memstore is flushed to disk and prior to creating the scanner to read from + * the memstore. To override or modify how a memstore is flushed, + * implementing classes can return a new scanner to provide the KeyValues to be + * stored into the new {@code StoreFile} or null to perform the default processing. + * Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no + * effect in this hook. + * @param c the environment provided by the region server + * @param store the store being flushed + * @param memstoreScanner the scanner for the memstore that is flushed + * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain + * @return the scanner to use during the flush. {@code null} if the default implementation + * is to be used. + * @throws IOException if an error occurred on the coprocessor + */ + InternalScanner preFlushScannerOpen(final ObserverContext c, + final Store store, final KeyValueScanner memstoreScanner, final InternalScanner s) + throws IOException; + + /** + * Called before the memstore is flushed to disk. + * @param c the environment provided by the region server + * @throws IOException if an error occurred on the coprocessor + * @deprecated use {@link #preFlush(ObserverContext, Store, InternalScanner)} instead + */ + void preFlush(final ObserverContext c) throws IOException; + + /** + * Called before a Store's memstore is flushed to disk. + * @param c the environment provided by the region server + * @param store the store where compaction is being requested + * @param scanner the scanner over existing data used in the store file + * @return the scanner to use during compaction. Should not be {@code null} + * unless the implementation is writing new store files on its own. + * @throws IOException if an error occurred on the coprocessor + */ + InternalScanner preFlush(final ObserverContext c, final Store store, + final InternalScanner scanner) throws IOException; + + /** + * Called after the memstore is flushed to disk. + * @param c the environment provided by the region server + * @throws IOException if an error occurred on the coprocessor + * @deprecated use {@link #preFlush(ObserverContext, Store, InternalScanner)} instead. + */ + void postFlush(final ObserverContext c) throws IOException; + + /** + * Called after a Store's memstore is flushed to disk. + * @param c the environment provided by the region server + * @param store the store being flushed + * @param resultFile the new store file written out during compaction + * @throws IOException if an error occurred on the coprocessor + */ + void postFlush(final ObserverContext c, final Store store, + final StoreFile resultFile) throws IOException; + + /** + * Called prior to selecting the {@link StoreFile StoreFiles} to compact from the list of + * available candidates. To alter the files used for compaction, you may mutate the passed in list + * of candidates. + * @param c the environment provided by the region server + * @param store the store where compaction is being requested + * @param candidates the store files currently available for compaction + * @param request custom compaction request + * @throws IOException if an error occurred on the coprocessor + */ + void preCompactSelection(final ObserverContext c, + final Store store, final List candidates, final CompactionRequest request) + throws IOException; + + /** + * Called prior to selecting the {@link StoreFile}s to compact from the list of available + * candidates. To alter the files used for compaction, you may mutate the passed in list of + * candidates. + * @param c the environment provided by the region server + * @param store the store where compaction is being requested + * @param candidates the store files currently available for compaction + * @throws IOException if an error occurred on the coprocessor + */ + void preCompactSelection(final ObserverContext c, + final Store store, final List candidates) throws IOException; + + /** + * Called after the {@link StoreFile}s to compact have been selected from the + * available candidates. + * @param c the environment provided by the region server + * @param store the store being compacted + * @param selected the store files selected to compact + */ + void postCompactSelection(final ObserverContext c, + final Store store, final ImmutableList selected); + + /** + * Called after the {@link StoreFile}s to compact have been selected from the available + * candidates. + * @param c the environment provided by the region server + * @param store the store being compacted + * @param selected the store files selected to compact + * @param request custom compaction request + */ + void postCompactSelection(final ObserverContext c, + final Store store, final ImmutableList selected, CompactionRequest request); + + /** + * Called prior to writing the {@link StoreFile}s selected for compaction into a new + * {@code StoreFile}. To override or modify the compaction process, implementing classes have two + * options: + *
        + *
      • Wrap the provided {@link InternalScanner} with a custom + * implementation that is returned from this method. The custom scanner + * can then inspect {@link KeyValue}s from the wrapped scanner, applying + * its own policy to what gets written.
        • + *
      • Call {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} + * and provide a custom implementation for writing of new + * {@link StoreFile}s. Note: any implementations bypassing + * core compaction using this approach must write out new store files + * themselves or the existing data will no longer be available after + * compaction.
        • + *
      + * @param c the environment provided by the region server + * @param store the store being compacted + * @param scanner the scanner over existing data used in the store file rewriting + * @return the scanner to use during compaction. Should not be {@code null} unless the + * implementation is writing new store files on its own. + * @throws IOException if an error occurred on the coprocessor + */ + InternalScanner preCompact(final ObserverContext c, + final Store store, final InternalScanner scanner) throws IOException; + + /** + * Called prior to writing the {@link StoreFile}s selected for compaction into a new + * {@code StoreFile}. To override or modify the compaction process, implementing classes have two + * options: + *
        + *
      • Wrap the provided {@link InternalScanner} with a custom implementation that is returned + * from this method. The custom scanner can then inspect {@link KeyValue}s from the wrapped + * scanner, applying its own policy to what gets written.
        • + *
      • Call {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} and provide a + * custom implementation for writing of new {@link StoreFile}s. Note: any implementations + * bypassing core compaction using this approach must write out new store files themselves or the + * existing data will no longer be available after compaction.
        • + *
      + * @param c the environment provided by the region server + * @param store the store being compacted + * @param scanner the scanner over existing data used in the store file rewriting + * @param request the requested compaction + * @return the scanner to use during compaction. Should not be {@code null} unless the + * implementation is writing new store files on its own. + * @throws IOException if an error occurred on the coprocessor + */ + InternalScanner preCompact(final ObserverContext c, + final Store store, final InternalScanner scanner, CompactionRequest request) + throws IOException; + + /** + * Called prior to writing the {@link StoreFile}s selected for compaction into a new + * {@code StoreFile} and prior to creating the scanner used to read the input files. To override + * or modify the compaction process, implementing classes can return a new scanner to provide the + * KeyValues to be stored into the new {@code StoreFile} or null to perform the default + * processing. Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no + * effect in this hook. + * @param c the environment provided by the region server + * @param store the store being compacted + * @param scanners the list {@link StoreFileScanner}s to be read from + * @param scantype the {@link ScanType} indicating whether this is a major or minor compaction + * @param earliestPutTs timestamp of the earliest put that was found in any of the involved store + * files + * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain + * @return the scanner to use during compaction. {@code null} if the default implementation is to + * be used. + * @throws IOException if an error occurred on the coprocessor + */ + InternalScanner preCompactScannerOpen(final ObserverContext c, + final Store store, List scanners, final ScanType scanType, + final long earliestPutTs, final InternalScanner s) throws IOException; + + /** + * Called prior to writing the {@link StoreFile}s selected for compaction into a new + * {@code StoreFile} and prior to creating the scanner used to read the input files. To override + * or modify the compaction process, implementing classes can return a new scanner to provide the + * KeyValues to be stored into the new {@code StoreFile} or null to perform the default + * processing. Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no + * effect in this hook. + * @param c the environment provided by the region server + * @param store the store being compacted + * @param scanners the list {@link StoreFileScanner}s to be read from + * @param scanType the {@link ScanType} indicating whether this is a major or minor compaction + * @param earliestPutTs timestamp of the earliest put that was found in any of the involved store + * files + * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain + * @param request the requested compaction + * @return the scanner to use during compaction. {@code null} if the default implementation is to + * be used. + * @throws IOException if an error occurred on the coprocessor + */ + InternalScanner preCompactScannerOpen(final ObserverContext c, + final Store store, List scanners, final ScanType scanType, + final long earliestPutTs, final InternalScanner s, CompactionRequest request) + throws IOException; + + /** + * Called after compaction has completed and the new store file has been + * moved in to place. + * @param c the environment provided by the region server + * @param store the store being compacted + * @param resultFile the new store file written out during compaction + * @throws IOException if an error occurred on the coprocessor + */ + void postCompact(final ObserverContext c, final Store store, + StoreFile resultFile) throws IOException; + + /** + * Called after compaction has completed and the new store file has been moved in to place. + * @param c the environment provided by the region server + * @param store the store being compacted + * @param resultFile the new store file written out during compaction + * @param request the requested compaction + * @throws IOException if an error occurred on the coprocessor + */ + void postCompact(final ObserverContext c, final Store store, + StoreFile resultFile, CompactionRequest request) throws IOException; + + /** + * Called before the region is split. + * @param c the environment provided by the region server (e.getRegion() returns the parent + * region) + * @throws IOException if an error occurred on the coprocessor + */ + void preSplit(final ObserverContext c) throws IOException; + + /** + * Called after the region is split. + * @param c the environment provided by the region server + * (e.getRegion() returns the parent region) + * @param l the left daughter region + * @param r the right daughter region + * @throws IOException if an error occurred on the coprocessor + */ + void postSplit(final ObserverContext c, final HRegion l, + final HRegion r) throws IOException; + + /** + * Called before the region is reported as closed to the master. + * @param c the environment provided by the region server + * @param abortRequested true if the region server is aborting + * @throws IOException + */ + void preClose(final ObserverContext c, + boolean abortRequested) throws IOException; + + /** + * Called after the region is reported as closed to the master. + * @param c the environment provided by the region server + * @param abortRequested true if the region server is aborting + */ + void postClose(final ObserverContext c, + boolean abortRequested); + + /** + * Called before a client makes a GetClosestRowBefore request. + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param row the row + * @param family the family + * @param result The result to return to the client if default processing + * is bypassed. Can be modified. Will not be used if default processing + * is not bypassed. + * @throws IOException if an error occurred on the coprocessor + */ + void preGetClosestRowBefore(final ObserverContext c, + final byte [] row, final byte [] family, final Result result) + throws IOException; + + /** + * Called after a client makes a GetClosestRowBefore request. + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param row the row + * @param family the desired family + * @param result the result to return to the client, modify as necessary + * @throws IOException if an error occurred on the coprocessor + */ + void postGetClosestRowBefore(final ObserverContext c, + final byte [] row, final byte [] family, final Result result) + throws IOException; + + /** + * Called before the client performs a Get + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param get the Get request + * @param result The result to return to the client if default processing + * is bypassed. Can be modified. Will not be used if default processing + * is not bypassed. + * @throws IOException if an error occurred on the coprocessor + */ + void preGet(final ObserverContext c, final Get get, + final List result) + throws IOException; + + /** + * Called after the client performs a Get + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param get the Get request + * @param result the result to return to the client, modify as necessary + * @throws IOException if an error occurred on the coprocessor + */ + void postGet(final ObserverContext c, final Get get, + final List result) + throws IOException; + + /** + * Called before the client tests for existence using a Get. + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param get the Get request + * @param exists + * @return the value to return to the client if bypassing default processing + * @throws IOException if an error occurred on the coprocessor + */ + boolean preExists(final ObserverContext c, final Get get, + final boolean exists) + throws IOException; + + /** + * Called after the client tests for existence using a Get. + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param get the Get request + * @param exists the result returned by the region server + * @return the result to return to the client + * @throws IOException if an error occurred on the coprocessor + */ + boolean postExists(final ObserverContext c, final Get get, + final boolean exists) + throws IOException; + + /** + * Called before the client stores a value. + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param put The Put object + * @param edit The WALEdit object that will be written to the wal + * @param writeToWAL true if the change should be written to the WAL + * @throws IOException if an error occurred on the coprocessor + */ + void prePut(final ObserverContext c, + final Put put, final WALEdit edit, final boolean writeToWAL) + throws IOException; + + /** + * Called after the client stores a value. + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param put The Put object + * @param edit The WALEdit object for the wal + * @param writeToWAL true if the change should be written to the WAL + * @throws IOException if an error occurred on the coprocessor + */ + void postPut(final ObserverContext c, + final Put put, final WALEdit edit, final boolean writeToWAL) + throws IOException; + + /** + * Called before the client deletes a value. + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param delete The Delete object + * @param edit The WALEdit object for the wal + * @param writeToWAL true if the change should be written to the WAL + * @throws IOException if an error occurred on the coprocessor + */ + void preDelete(final ObserverContext c, + final Delete delete, final WALEdit edit, final boolean writeToWAL) + throws IOException; + + /** + * Called after the client deletes a value. + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param delete The Delete object + * @param edit The WALEdit object for the wal + * @param writeToWAL true if the change should be written to the WAL + * @throws IOException if an error occurred on the coprocessor + */ + void postDelete(final ObserverContext c, + final Delete delete, final WALEdit edit, final boolean writeToWAL) + throws IOException; + + /** + * This will be called for every batch mutation operation happening at the server. This will be + * called after acquiring the locks on the mutating rows and after applying the proper timestamp + * for each Mutation at the server. The batch may contain Put/Delete. By setting OperationStatus + * of Mutations ({@link MiniBatchOperationInProgress#setOperationStatus(int, OperationStatus)}), + * {@link RegionObserver} can make HRegion to skip these Mutations. + * @param c the environment provided by the region server + * @param miniBatchOp batch of Mutations getting applied to region. + * @throws IOException if an error occurred on the coprocessor + */ + void preBatchMutate(final ObserverContext c, + final MiniBatchOperationInProgress> miniBatchOp) throws IOException; + + /** + * This will be called after applying a batch of Mutations on a region. The Mutations are added to + * memstore and WAL. + * @param c the environment provided by the region server + * @param miniBatchOp batch of Mutations applied to region. + * @throws IOException if an error occurred on the coprocessor + */ + void postBatchMutate(final ObserverContext c, + final MiniBatchOperationInProgress> miniBatchOp) throws IOException; + + /** + * Called before checkAndPut + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param row row to check + * @param family column family + * @param qualifier column qualifier + * @param compareOp the comparison operation + * @param comparator the comparator + * @param put data to put if check succeeds + * @param result + * @return the return value to return to client if bypassing default + * processing + * @throws IOException if an error occurred on the coprocessor + */ + boolean preCheckAndPut(final ObserverContext c, + final byte [] row, final byte [] family, final byte [] qualifier, + final CompareOp compareOp, final WritableByteArrayComparable comparator, + final Put put, final boolean result) + throws IOException; + + /** + * Called after checkAndPut + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param row row to check + * @param family column family + * @param qualifier column qualifier + * @param compareOp the comparison operation + * @param comparator the comparator + * @param put data to put if check succeeds + * @param result from the checkAndPut + * @return the possibly transformed return value to return to client + * @throws IOException if an error occurred on the coprocessor + */ + boolean postCheckAndPut(final ObserverContext c, + final byte [] row, final byte [] family, final byte [] qualifier, + final CompareOp compareOp, final WritableByteArrayComparable comparator, + final Put put, final boolean result) + throws IOException; + + /** + * Called before checkAndDelete + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param row row to check + * @param family column family + * @param qualifier column qualifier + * @param compareOp the comparison operation + * @param comparator the comparator + * @param delete delete to commit if check succeeds + * @param result + * @return the value to return to client if bypassing default processing + * @throws IOException if an error occurred on the coprocessor + */ + boolean preCheckAndDelete(final ObserverContext c, + final byte [] row, final byte [] family, final byte [] qualifier, + final CompareOp compareOp, final WritableByteArrayComparable comparator, + final Delete delete, final boolean result) + throws IOException; + + /** + * Called after checkAndDelete + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param row row to check + * @param family column family + * @param qualifier column qualifier + * @param compareOp the comparison operation + * @param comparator the comparator + * @param delete delete to commit if check succeeds + * @param result from the CheckAndDelete + * @return the possibly transformed returned value to return to client + * @throws IOException if an error occurred on the coprocessor + */ + boolean postCheckAndDelete(final ObserverContext c, + final byte [] row, final byte [] family, final byte [] qualifier, + final CompareOp compareOp, final WritableByteArrayComparable comparator, + final Delete delete, final boolean result) + throws IOException; + + /** + * Called before incrementColumnValue + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param row row to check + * @param family column family + * @param qualifier column qualifier + * @param amount long amount to increment + * @param writeToWAL true if the change should be written to the WAL + * @return value to return to the client if bypassing default processing + * @throws IOException if an error occurred on the coprocessor + */ + long preIncrementColumnValue(final ObserverContext c, + final byte [] row, final byte [] family, final byte [] qualifier, + final long amount, final boolean writeToWAL) + throws IOException; + + /** + * Called after incrementColumnValue + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param row row to check + * @param family column family + * @param qualifier column qualifier + * @param amount long amount to increment + * @param writeToWAL true if the change should be written to the WAL + * @param result the result returned by incrementColumnValue + * @return the result to return to the client + * @throws IOException if an error occurred on the coprocessor + */ + long postIncrementColumnValue(final ObserverContext c, + final byte [] row, final byte [] family, final byte [] qualifier, + final long amount, final boolean writeToWAL, final long result) + throws IOException; + + /** + * Called before Append + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param append Append object + * @return result to return to the client if bypassing default processing + * @throws IOException if an error occurred on the coprocessor + */ + Result preAppend(final ObserverContext c, + final Append append) + throws IOException; + + /** + * Called after Append + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param append Append object + * @param result the result returned by increment + * @return the result to return to the client + * @throws IOException if an error occurred on the coprocessor + */ + Result postAppend(final ObserverContext c, + final Append append, final Result result) + throws IOException; + + /** + * Called before Increment + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param increment increment object + * @return result to return to the client if bypassing default processing + * @throws IOException if an error occurred on the coprocessor + */ + Result preIncrement(final ObserverContext c, + final Increment increment) + throws IOException; + + /** + * Called after increment + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param increment increment object + * @param result the result returned by increment + * @return the result to return to the client + * @throws IOException if an error occurred on the coprocessor + */ + Result postIncrement(final ObserverContext c, + final Increment increment, final Result result) + throws IOException; + + /** + * Called before the client opens a new scanner. + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param scan the Scan specification + * @param s if not null, the base scanner + * @return an RegionScanner instance to use instead of the base scanner if + * overriding default behavior, null otherwise + * @throws IOException if an error occurred on the coprocessor + */ + RegionScanner preScannerOpen(final ObserverContext c, + final Scan scan, final RegionScanner s) + throws IOException; + + /** + * Called before a store opens a new scanner. + * This hook is called when a "user" scanner is opened. + *

      + * See {@link #preFlushScannerOpen(ObserverContext, Store, KeyValueScanner, InternalScanner)} + * and {@link #preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, InternalScanner)} + * to override scanners created for flushes or compactions, resp. + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors. + * Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no + * effect in this hook. + * @param c the environment provided by the region server + * @param store the store being scanned + * @param scan the Scan specification + * @param targetCols columns to be used in the scanner + * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain + * @return a KeyValueScanner instance to use or {@code null} to use the default implementation + * @throws IOException if an error occurred on the coprocessor + */ + KeyValueScanner preStoreScannerOpen(final ObserverContext c, + final Store store, final Scan scan, final NavigableSet targetCols, + final KeyValueScanner s) throws IOException; + + /** + * Called after the client opens a new scanner. + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param scan the Scan specification + * @param s if not null, the base scanner + * @return the scanner instance to use + * @throws IOException if an error occurred on the coprocessor + */ + RegionScanner postScannerOpen(final ObserverContext c, + final Scan scan, final RegionScanner s) + throws IOException; + + /** + * Called before the client asks for the next row on a scanner. + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param s the scanner + * @param result The result to return to the client if default processing + * is bypassed. Can be modified. Will not be returned if default processing + * is not bypassed. + * @param limit the maximum number of results to return + * @param hasNext the 'has more' indication + * @return 'has more' indication that should be sent to client + * @throws IOException if an error occurred on the coprocessor + */ + boolean preScannerNext(final ObserverContext c, + final InternalScanner s, final List result, + final int limit, final boolean hasNext) + throws IOException; + + /** + * Called after the client asks for the next row on a scanner. + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param s the scanner + * @param result the result to return to the client, can be modified + * @param limit the maximum number of results to return + * @param hasNext the 'has more' indication + * @return 'has more' indication that should be sent to client + * @throws IOException if an error occurred on the coprocessor + */ + boolean postScannerNext(final ObserverContext c, + final InternalScanner s, final List result, final int limit, + final boolean hasNext) + throws IOException; + + /** + * This will be called by the scan flow when the current scanned row is being filtered out by the + * filter. The filter may be filtering out the row via any of the below scenarios + *

        + *
      1. + * boolean filterRowKey(byte [] buffer, int offset, int length) returning true
        2. + *
      2. + * boolean filterRow() returning true
        3. + *
      3. + * void filterRow(List kvs) removing all the kvs from the passed List
        4. + *
      + * @param c the environment provided by the region server + * @param s the scanner + * @param currentRow The current rowkey which got filtered out + * @param hasMore the 'has more' indication + * @return whether more rows are available for the scanner or not + * @throws IOException + */ + boolean postScannerFilterRow(final ObserverContext c, + final InternalScanner s, final byte[] currentRow, final boolean hasMore) throws IOException; + + /** + * Called before the client closes a scanner. + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param s the scanner + * @throws IOException if an error occurred on the coprocessor + */ + void preScannerClose(final ObserverContext c, + final InternalScanner s) + throws IOException; + + /** + * Called after the client closes a scanner. + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained + * coprocessors + * @param c the environment provided by the region server + * @param s the scanner + * @throws IOException if an error occurred on the coprocessor + */ + void postScannerClose(final ObserverContext c, + final InternalScanner s) + throws IOException; + + /** + * Called before a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} + * replayed for this region. + * + * @param ctx + * @param info + * @param logKey + * @param logEdit + * @throws IOException + */ + void preWALRestore(final ObserverContext ctx, + HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException; + + /** + * Called after a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} + * replayed for this region. + * + * @param ctx + * @param info + * @param logKey + * @param logEdit + * @throws IOException + */ + void postWALRestore(final ObserverContext ctx, + HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException; + + /** + * Called before bulkLoadHFile. Users can create a StoreFile instance to + * access the contents of a HFile. + * + * @param ctx + * @param familyPaths pairs of { CF, HFile path } submitted for bulk load. Adding + * or removing from this list will add or remove HFiles to be bulk loaded. + * @throws IOException + */ + void preBulkLoadHFile(final ObserverContext ctx, + List> familyPaths) throws IOException; + + /** + * Called after bulkLoadHFile. + * + * @param ctx + * @param familyPaths pairs of { CF, HFile path } submitted for bulk load + * @param hasLoaded whether the bulkLoad was successful + * @return the new value of hasLoaded + * @throws IOException + */ + boolean postBulkLoadHFile(final ObserverContext ctx, + List> familyPaths, boolean hasLoaded) throws IOException; + + /** + * Called before locking a row. + * + * @param ctx + * @param regionName + * @param row + * @throws IOException Signals that an I/O exception has occurred. + * @deprecated Will be removed in 0.96 + */ + @Deprecated + void preLockRow(final ObserverContext ctx, + final byte[] regionName, final byte[] row) throws IOException; + + /** + * Called after locking a row. + * + * @param ctx + * @param regionName the region name + * @param row + * @throws IOException Signals that an I/O exception has occurred. + * @deprecated Will be removed in 0.96 + */ + @Deprecated + void postLockRow(final ObserverContext ctx, + final byte[] regionName, final byte[] row) throws IOException; + + /** + * Called before unlocking a row. + * + * @param ctx + * @param regionName + * @param lockId the lock id + * @throws IOException Signals that an I/O exception has occurred. + * @deprecated Will be removed in 0.96 + */ + @Deprecated + void preUnlockRow(final ObserverContext ctx, + final byte[] regionName, final long lockId) throws IOException; + + /** + * Called after unlocking a row. + * @param ctx + * @param regionName the region name + * @param lockId the lock id + * @throws IOException Signals that an I/O exception has occurred. + * @deprecated Will be removed in 0.96 + */ + @Deprecated + void postUnlockRow(final ObserverContext ctx, + final byte[] regionName, final long lockId) throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserverExt.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserverExt.java new file mode 100644 index 0000000..8c92d56 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserverExt.java @@ -0,0 +1,134 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.OperationStatus; +import org.apache.hadoop.hbase.regionserver.SplitTransaction.SplitInfo; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Pair; + +/** + * This is an extension for the RegionObserver interface. The APIs added into this interface are not + * exposed by HBase. This is internally being used by CMWH HBase. Customer should not make use of + * this interface points.
      + * Note : The APIs in this interface is subject to change at any time. + */ +public interface RegionObserverExt { + + /** + * Internally the Put/Delete are handled as a batch Called before actual put operation starts in + * the region. + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained coprocessors + * @param ctx the environment provided by the region server + * @param mutations list of mutations + * @param edit The WALEdit object that will be written to the wal + * @throws IOException if an error occurred on the coprocessor + */ + void preBatchMutate(final ObserverContext ctx, + final List> mutationVsBatchOp, final WALEdit edit) + throws IOException; + + /** + * Internally the Put/Delete are handled as a batch. Called after actual batch put operation + * completes in the region. + *

      + * Call CoprocessorEnvironment#bypass to skip default actions + *

      + * Call CoprocessorEnvironment#complete to skip any subsequent chained coprocessors + * @param ctx the environment provided by the region server + * @param mutations list of mutations + * @param walEdit The WALEdit object that will be written to the wal + */ + void postBatchMutate(final ObserverContext ctx, + final List mutations, WALEdit walEdit) throws IOException; + + /** + * Called after the completion of batch put/delete and will be called even if the batch operation + * fails + * @param ctx + * @param mutations list of mutations + * @throws IOException + */ + void postCompleteBatchMutate(final ObserverContext ctx, + List mutations) throws IOException; + + /** + * This will be called by the scan flow when the current scanned row is being filtered out by the + * filter. The filter may be filtering out the row via any of the below scenarios + *

        + *
      1. + * boolean filterRowKey(byte [] buffer, int offset, int length) returning true
        2. + *
      2. + * boolean filterRow() returning true
        3. + *
      3. + * void filterRow(List kvs) removing all the kvs from the passed List
        4. + *
      + * @param ctx the environment provided by the region server + * @param s the scanner + * @param currentRow The current rowkey which got filtered out. + * @return Returns whether more rows are available for the scanner or not. + * @throws IOException + */ + boolean postFilterRow(final ObserverContext ctx, + final InternalScanner s, final byte[] currentRow) throws IOException; + + /** + * This will be called before PONR step as part of split transaction + * @param ctx + * @param splitKey + * @return + * @throws IOException + */ + SplitInfo preSplitBeforePONR(final ObserverContext ctx, + byte[] splitKey) throws IOException; + + /** + * This is used to roll back the split related transactions. + * @param ctx + * @return + * @throws IOException + */ + void preRollBack(final ObserverContext ctx) throws IOException; + + /** + * Used after closeRegionOperation in the batchMutate() + * @param ctx + * @throws IOException + */ + void postCloseRegionOperation(final ObserverContext ctx) + throws IOException; + + /** + * Used after startRegionOperation in the batchMutate() + * @param ctx + * @throws IOException + */ + void postStartRegionOperation(final ObserverContext ctx) + throws IOException; + +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java new file mode 100644 index 0000000..6a07b8b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; + +public interface RegionServerCoprocessorEnvironment extends CoprocessorEnvironment { + + /** @return reference to the HMaster services */ + RegionServerServices getRegionServerServices(); + +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java new file mode 100644 index 0000000..044d086 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; + +import org.apache.hadoop.hbase.Coprocessor; + +public interface RegionServerObserver extends Coprocessor { + + /** + * Called before stopping region server. + * @param env An instance of RegionServerCoprocessorEnvironment + * @throws IOException Signals that an I/O exception has occurred. + */ + void preStopRegionServer(final ObserverContext env) + throws IOException; + +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java new file mode 100644 index 0000000..a521833 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/SecureBulkLoadClient.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.util.Methods; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.security.token.Token; + +import java.io.IOException; +import java.util.List; + +public class SecureBulkLoadClient { + private static Class protocolClazz; + private static Class endpointClazz; + private Object proxy; + private HTable table; + + public SecureBulkLoadClient(HTable table) throws IOException { + this(table, HConstants.EMPTY_START_ROW); + } + + public SecureBulkLoadClient(HTable table, byte[] startRow) throws IOException { + try { + protocolClazz = protocolClazz!=null?protocolClazz: + Class.forName("org.apache.hadoop.hbase.security.access.SecureBulkLoadProtocol"); + endpointClazz = endpointClazz!=null?endpointClazz: + Class.forName("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint"); + proxy = table.coprocessorProxy(protocolClazz, startRow); + this.table = table; + } catch (ClassNotFoundException e) { + throw new IOException("Failed to initialize SecureBulkLoad", e); + } + } + + public String prepareBulkLoad(byte[] tableName) throws IOException { + try { + String bulkToken = (String) Methods.call(protocolClazz, proxy, + "prepareBulkLoad", new Class[]{byte[].class}, new Object[]{tableName}); + return bulkToken; + } catch (Exception e) { + throw new IOException("Failed to prepareBulkLoad", e); + } + } + + public void cleanupBulkLoad(String bulkToken) throws IOException { + try { + Methods.call(protocolClazz, proxy, + "cleanupBulkLoad", new Class[]{String.class},new Object[]{bulkToken}); + } catch (Exception e) { + throw new IOException("Failed to prepareBulkLoad", e); + } + } + + public boolean bulkLoadHFiles(List> familyPaths, + Token userToken, String bulkToken) throws IOException { + try { + return (Boolean)Methods.call(protocolClazz, proxy, "bulkLoadHFiles", + new Class[]{List.class, Token.class, String.class},new Object[]{familyPaths, userToken, bulkToken}); + } catch (Exception e) { + throw new IOException("Failed to bulkLoadHFiles", e); + } + } + + public Path getStagingPath(String bulkToken, byte[] family) throws IOException { + try { + return (Path)Methods.call(endpointClazz, null, "getStagingPath", + new Class[]{Configuration.class, String.class, byte[].class}, + new Object[]{table.getConfiguration(), bulkToken, family}); + } catch (Exception e) { + throw new IOException("Failed to getStagingPath", e); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java new file mode 100644 index 0000000..2790abd --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java @@ -0,0 +1,29 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.wal.HLog; + +public interface WALCoprocessorEnvironment extends CoprocessorEnvironment { + /** @return reference to the region server services */ + public HLog getWAL(); +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java new file mode 100644 index 0000000..c90189d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java @@ -0,0 +1,68 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; + +import java.io.IOException; + +/** + * It's provided to have a way for coprocessors to observe, rewrite, + * or skip WALEdits as they are being written to the WAL. + * + * {@link org.apache.hadoop.hbase.coprocessor.RegionObserver} provides + * hooks for adding logic for WALEdits in the region context during reconstruction, + * + * Defines coprocessor hooks for interacting with operations on the + * {@link org.apache.hadoop.hbase.regionserver.wal.HLog}. + */ +public interface WALObserver extends Coprocessor { + + /** + * Called before a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} + * is writen to WAL. + * + * @param ctx + * @param info + * @param logKey + * @param logEdit + * @return true if default behavior should be bypassed, false otherwise + * @throws IOException + */ + boolean preWALWrite(ObserverContext ctx, + HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException; + + /** + * Called after a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} + * is writen to WAL. + * + * @param ctx + * @param info + * @param logKey + * @param logEdit + * @throws IOException + */ + void postWALWrite(ObserverContext ctx, + HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteEndpoint.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteEndpoint.java new file mode 100644 index 0000000..d259f9f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteEndpoint.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor.example; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HConstants.OperationStatusCode; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.OperationStatus; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +public class BulkDeleteEndpoint extends BaseEndpointCoprocessor implements BulkDeleteProtocol { + private static final String NO_OF_VERSIONS_TO_DELETE = "noOfVersionsToDelete"; + private static final Log LOG = LogFactory.getLog(BulkDeleteEndpoint.class); + + @Override + public BulkDeleteResponse delete(Scan scan, byte deleteType, Long timestamp, + int rowBatchSize) { + long totalRowsDeleted = 0L; + long totalVersionsDeleted = 0L; + BulkDeleteResponse response = new BulkDeleteResponse(); + HRegion region = ((RegionCoprocessorEnvironment) getEnvironment()).getRegion(); + boolean hasMore = true; + RegionScanner scanner = null; + if (scan.getFilter() == null && deleteType == DeleteType.ROW) { + // What we need is just the rowkeys. So only 1st KV from any row is enough. + // Only when it is a row delete, we can apply this filter + // In other types we rely on the scan to know which all columns to be deleted. + scan.setFilter(new FirstKeyOnlyFilter()); + } + // When the delete is based on some conditions so that Filters are available in the scan, + // we assume that the scan is perfect having necessary column(s) only. + try { + scanner = region.getScanner(scan); + while (hasMore) { + List> deleteRows = new ArrayList>(rowBatchSize); + for (int i = 0; i < rowBatchSize; i++) { + List results = new ArrayList(); + hasMore = scanner.next(results); + if (results.size() > 0) { + deleteRows.add(results); + } + if (!hasMore) { + // There are no more rows. + break; + } + } + if (deleteRows.size() > 0) { + Pair[] deleteWithLockArr = new Pair[deleteRows.size()]; + int i = 0; + for (List deleteRow : deleteRows) { + Delete delete = createDeleteMutation(deleteRow, deleteType, timestamp); + deleteWithLockArr[i++] = new Pair(delete, null); + } + OperationStatus[] opStatus = region.batchMutate(deleteWithLockArr); + for (i = 0; i < opStatus.length; i++) { + if (opStatus[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) { + break; + } + totalRowsDeleted++; + if (deleteType == DeleteType.VERSION) { + byte[] versionsDeleted = deleteWithLockArr[i].getFirst().getAttribute( + NO_OF_VERSIONS_TO_DELETE); + if (versionsDeleted != null) { + totalVersionsDeleted += Bytes.toInt(versionsDeleted); + } + } + } + } + } + } catch (IOException ioe) { + LOG.error(ioe); + response.setIoException(ioe); + } finally { + if (scanner != null) { + try { + scanner.close(); + } catch (IOException ioe) { + LOG.error(ioe); + } + } + } + response.setRowsDeleted(totalRowsDeleted); + response.setVersionsDeleted(totalVersionsDeleted); + return response; + } + + private Delete createDeleteMutation(List deleteRow, byte deleteType, Long timestamp) { + long ts; + if (timestamp == null) { + ts = HConstants.LATEST_TIMESTAMP; + } else { + ts = timestamp; + } + // We just need the rowkey. Get it from 1st KV. + byte[] row = deleteRow.get(0).getRow(); + Delete delete = new Delete(row, ts, null); + if (deleteType != DeleteType.ROW) { + switch (deleteType) { + case DeleteType.FAMILY: + Set families = new TreeSet(Bytes.BYTES_COMPARATOR); + for (KeyValue kv : deleteRow) { + if (families.add(kv.getFamily())) { + delete.deleteFamily(kv.getFamily(), ts); + } + } + break; + + case DeleteType.COLUMN: + Set columns = new HashSet(); + for (KeyValue kv : deleteRow) { + Column column = new Column(kv.getFamily(), kv.getQualifier()); + if (columns.add(column)) { + // Making deleteColumns() calls more than once for the same cf:qualifier is not correct + // Every call to deleteColumns() will add a new KV to the familymap which will finally + // get written to the memstore as part of delete(). + delete.deleteColumns(column.family, column.qualifier, ts); + } + } + break; + + case DeleteType.VERSION: + // When some timestamp was passed to the delete() call only one version of the column (with + // given timestamp) will be deleted. If no timestamp passed, it will delete N versions. + // How many versions will get deleted depends on the Scan being passed. All the KVs that + // the scan fetched will get deleted. + int noOfVersionsToDelete = 0; + if (timestamp == null) { + for (KeyValue kv : deleteRow) { + delete.deleteColumn(kv.getFamily(), kv.getQualifier(), kv.getTimestamp()); + noOfVersionsToDelete++; + } + } else { + columns = new HashSet(); + for (KeyValue kv : deleteRow) { + Column column = new Column(kv.getFamily(), kv.getQualifier()); + // Only one version of particular column getting deleted. + if (columns.add(column)) { + delete.deleteColumn(column.family, column.qualifier, ts); + noOfVersionsToDelete++; + } + } + } + delete.setAttribute(NO_OF_VERSIONS_TO_DELETE, Bytes.toBytes(noOfVersionsToDelete)); + } + } + return delete; + } + + private static class Column { + private byte[] family; + private byte[] qualifier; + + public Column(byte[] family, byte[] qualifier) { + this.family = family; + this.qualifier = qualifier; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Column)) { + return false; + } + Column column = (Column) other; + return Bytes.equals(this.family, column.family) + && Bytes.equals(this.qualifier, column.qualifier); + } + + @Override + public int hashCode() { + int h = 31; + h = h + 13 * Bytes.hashCode(this.family); + h = h + 13 * Bytes.hashCode(this.qualifier); + return h; + } + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteProtocol.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteProtocol.java new file mode 100644 index 0000000..0a3d2ca --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteProtocol.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor.example; + +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; + +/** + * Defines a protocol to delete data in bulk based on a scan. The scan can be range scan or with + * conditions(filters) etc. + *
      Example: 
      + * Scan scan = new Scan();
+ * // set scan properties(rowkey range, filters, timerange etc).
+ * HTable ht = ...;
+ * long noOfDeletedRows = 0L;
+ * Batch.Call<BulkDeleteProtocol, BulkDeleteResponse> callable = 
+ *     new Batch.Call<BulkDeleteProtocol, BulkDeleteResponse>() {
+ *   public BulkDeleteResponse call(BulkDeleteProtocol instance) throws IOException {
+ *     return instance.deleteRows(scan, BulkDeleteProtocol.DeleteType, timestamp, rowBatchSize);
+ *   }
+ * };
+ * Map<byte[], BulkDeleteResponse> result = ht.coprocessorExec(BulkDeleteProtocol.class,
+ *      scan.getStartRow(), scan.getStopRow(), callable);
+ *  for (BulkDeleteResponse response : result.values()) {
+ *    noOfDeletedRows = response.getRowsDeleted();
+ *  }
+ *
      + */ +public interface BulkDeleteProtocol extends CoprocessorProtocol { + + public interface DeleteType { + /** + * Delete full row + */ + byte ROW = 0; + /** + * Delete full family(s). + * Which family(s) to be deleted will be determined by the Scan. + * Scan need to select all the families which need to be deleted. + */ + byte FAMILY = 1; + /** + * Delete full column(s). + * Which column(s) to be deleted will be determined by the Scan. + * Scan need to select all the qualifiers which need to be deleted. + */ + byte COLUMN = 2; + /** + * Delete one or more version(s) of column(s). + * Which column(s) and version(s) to be deleted will be determined by the Scan. + * Scan need to select all the qualifiers and its versions which need to be deleted. + * When a timestamp is passed only one version at that timestamp will be deleted(even if scan + * fetches many versions) + */ + byte VERSION = 3; + } + + /** + * + * @param scan + * @param deleteType + * @param timestamp + * @param rowBatchSize + * The number of rows which need to be accumulated by scan and delete as one batch + * @return + */ + BulkDeleteResponse delete(Scan scan, byte deleteType, Long timestamp, int rowBatchSize); +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteResponse.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteResponse.java new file mode 100644 index 0000000..0ccabb8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/BulkDeleteResponse.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor.example; + +import java.io.IOException; +import java.io.Serializable; + +/** + * Wrapper class which returns the result of the bulk deletion operation happened at the server for + * a region. This includes the total number of rows deleted and/or any {@link IOException} which is + * happened while doing the operation. It will also include total number of versions deleted, when + * the delete type is VERSION. + */ +public class BulkDeleteResponse implements Serializable { + private static final long serialVersionUID = -8192337710525997237L; + private long rowsDeleted; + private IOException ioException; + private long versionsDeleted; + + public BulkDeleteResponse() { + + } + + public void setRowsDeleted(long rowsDeleted) { + this.rowsDeleted = rowsDeleted; + } + + public long getRowsDeleted() { + return rowsDeleted; + } + + public void setIoException(IOException ioException) { + this.ioException = ioException; + } + + public IOException getIoException() { + return ioException; + } + + public long getVersionsDeleted() { + return versionsDeleted; + } + + public void setVersionsDeleted(long versionsDeleted) { + this.versionsDeleted = versionsDeleted; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java new file mode 100644 index 0000000..f2bfe11 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java @@ -0,0 +1,230 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor.example; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.NavigableSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.RegionObserver; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreScanner; +import org.apache.hadoop.hbase.regionserver.Store.ScanInfo; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooKeeper; + +/** + * This is an example showing how a RegionObserver could configured + * via ZooKeeper in order to control a Region compaction, flush, and scan policy. + * + * This also demonstrated the use of shared {@link RegionObserver} state. + * See {@link RegionCoprocessorEnvironment#getSharedData()}. + * + * This would be useful for an incremental backup tool, which would indicate the last + * time of a successful backup via ZK and instruct HBase to not delete data that was + * inserted since (based on wall clock time). + * + * This implements org.apache.zookeeper.Watcher directly instead of using + * {@link ZooKeeperWatcher}, because RegionObservers come and go and currently + * listeners registered with ZooKeeperWatcher cannot be removed. + */ +public class ZooKeeperScanPolicyObserver extends BaseRegionObserver { + public static String node = "/backup/example/lastbackup"; + public static String zkkey = "ZK"; + private static final Log LOG = LogFactory.getLog(ZooKeeperScanPolicyObserver.class); + + /** + * Internal watcher that keep "data" up to date asynchronously. + */ + private static class ZKWatcher implements Watcher { + private byte[] data = null; + private ZooKeeper zk; + private volatile boolean needSetup = true; + private volatile long lastSetupTry = 0; + + public ZKWatcher(ZooKeeper zk) { + this.zk = zk; + // trigger the listening + getData(); + } + + /** + * Get the maintained data. In case of any ZK exceptions this will retry + * establishing the connection (but not more than twice/minute). + * + * getData is on the critical path, so make sure it is fast unless there is + * a problem (network partion, ZK ensemble down, etc) + * Make sure at most one (unlucky) thread retries and other threads don't pile up + * while that threads tries to recreate the connection. + * + * @return the last know version of the data + */ + public byte[] getData() { + // try at most twice/minute + if (needSetup && EnvironmentEdgeManager.currentTimeMillis() > lastSetupTry + 30000) { + synchronized (this) { + // make sure only one thread tries to reconnect + if (needSetup) { + needSetup = false; + } else { + return data; + } + } + // do this without the lock held to avoid threads piling up on this lock, + // as it can take a while + try { + LOG.debug("Connecting to ZK"); + // record this attempt + lastSetupTry = EnvironmentEdgeManager.currentTimeMillis(); + if (zk.exists(node, false) != null) { + data = zk.getData(node, this, null); + LOG.debug("Read synchronously: "+(data == null ? "null" : Bytes.toLong(data))); + } else { + zk.exists(node, this); + } + } catch (Exception x) { + // try again if this fails + needSetup = true; + } + } + return data; + } + + @Override + public void process(WatchedEvent event) { + switch(event.getType()) { + case NodeDataChanged: + case NodeCreated: + try { + // get data and re-watch + data = zk.getData(node, this, null); + LOG.debug("Read asynchronously: "+(data == null ? "null" : Bytes.toLong(data))); + } catch (InterruptedException ix) { + } catch (KeeperException kx) { + needSetup = true; + } + break; + + case NodeDeleted: + try { + // just re-watch + zk.exists(node, this); + data = null; + } catch (InterruptedException ix) { + } catch (KeeperException kx) { + needSetup = true; + } + break; + + default: + // ignore + } + } + } + + @Override + public void start(CoprocessorEnvironment e) throws IOException { + RegionCoprocessorEnvironment re = (RegionCoprocessorEnvironment) e; + if (!re.getSharedData().containsKey(zkkey)) { + // there is a short race here + // in the worst case we create a watcher that will be notified once + re.getSharedData().putIfAbsent( + zkkey, + new ZKWatcher(re.getRegionServerServices().getZooKeeper() + .getRecoverableZooKeeper().getZooKeeper())); + } + } + + @Override + public void stop(CoprocessorEnvironment e) throws IOException { + // nothing to do here + } + + protected ScanInfo getScanInfo(Store store, RegionCoprocessorEnvironment e) { + byte[] data = ((ZKWatcher)e.getSharedData().get(zkkey)).getData(); + if (data == null) { + return null; + } + ScanInfo oldSI = store.getScanInfo(); + if (oldSI.getTtl() == Long.MAX_VALUE) { + return null; + } + long ttl = Math.max(EnvironmentEdgeManager.currentTimeMillis() - Bytes.toLong(data), oldSI.getTtl()); + return new ScanInfo(store.getFamily(), ttl, + oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + } + + @Override + public InternalScanner preFlushScannerOpen(final ObserverContext c, + Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException { + Store.ScanInfo scanInfo = getScanInfo(store, c.getEnvironment()); + if (scanInfo == null) { + // take default action + return null; + } + Scan scan = new Scan(); + scan.setMaxVersions(scanInfo.getMaxVersions()); + return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner), + ScanType.MINOR_COMPACT, store.getHRegion().getSmallestReadPoint(), + HConstants.OLDEST_TIMESTAMP); + } + + @Override + public InternalScanner preCompactScannerOpen(final ObserverContext c, + Store store, List scanners, ScanType scanType, long earliestPutTs, + InternalScanner s) throws IOException { + Store.ScanInfo scanInfo = getScanInfo(store, c.getEnvironment()); + if (scanInfo == null) { + // take default action + return null; + } + Scan scan = new Scan(); + scan.setMaxVersions(scanInfo.getMaxVersions()); + return new StoreScanner(store, scanInfo, scan, scanners, scanType, store.getHRegion() + .getSmallestReadPoint(), earliestPutTs); + } + + @Override + public KeyValueScanner preStoreScannerOpen(final ObserverContext c, + final Store store, final Scan scan, final NavigableSet targetCols, + final KeyValueScanner s) throws IOException { + Store.ScanInfo scanInfo = getScanInfo(store, c.getEnvironment()); + if (scanInfo == null) { + // take default action + return null; + } + return new StoreScanner(store, scanInfo, scan, targetCols); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/coprocessor/package-info.java b/src/main/java/org/apache/hadoop/hbase/coprocessor/package-info.java new file mode 100644 index 0000000..f92a245 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/coprocessor/package-info.java @@ -0,0 +1,367 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + +

      Table of Contents

      + + +

      Overview

      +Coprocessors are code that runs in-process on each region server. Regions +contain references to the coprocessor implementation classes associated +with them. Coprocessor classes can be loaded either from local +jars on the region server's classpath or via the HDFS classloader. +

      +Multiple types of coprocessors are provided to provide sufficient flexibility +for potential use cases. Right now there are: +

      +

        +
      • Coprocessor: provides region lifecycle management hooks, e.g., region +open/close/split/flush/compact operations.
        • +
      • RegionObserver: provides hook for monitor table operations from +client side, such as table get/put/scan/delete, etc.
        • +
      • Endpoint: provides on demand triggers for any arbitrary function +executed at a region. One use case is column aggregation at region +server.
        • +
      + +

      Coprocessor

      +A coprocessor is required to +implement Coprocessor interface so that coprocessor framework +can manage it internally. +

      +Another design goal of this interface is to provide simple features for +making coprocessors useful, while exposing no more internal state or +control actions of the region server than necessary and not exposing them +directly. +

      +Over the lifecycle of a region, the methods of this interface are invoked +when the corresponding events happen. The master transitions regions +through the following states: +

      +    +unassigned -> pendingOpen -> open -> pendingClose -> closed. +

      +Coprocessors have opportunity to intercept and handle events in +pendingOpen, open, and pendingClose states. +

      + +

      PendingOpen

      +

      +The region server is opening a region to bring it online. Coprocessors +can piggyback or fail this process. +

      +

        +
      • preOpen, postOpen: Called before and after the region is reported as + online to the master.

        • +

      +

      +

      Open

      +The region is open on the region server and is processing both client +requests (get, put, scan, etc.) and administrative actions (flush, compact, +split, etc.). Coprocessors can piggyback administrative actions via: +

      +

        +
      • preFlush, postFlush: Called before and after the memstore is flushed + into a new store file.

        • +

      • preCompact, postCompact: Called before and after compaction.

        • +

      • preSplit, postSplit: Called after the region is split.

        • +

      +

      +

      PendingClose

      +The region server is closing the region. This can happen as part of normal +operations or may happen when the region server is aborting due to fatal +conditions such as OOME, health check failure, or fatal filesystem +problems. Coprocessors can piggyback this event. If the server is aborting +an indication to this effect will be passed as an argument. +

      +

        +
      • preClose and postClose: Called before and after the region is + reported as closed to the master.

        • +

      +

      + +

      RegionObserver

      +If the coprocessor implements the RegionObserver interface it can +observe and mediate client actions on the region: +

      +

        +
      • preGet, postGet: Called before and after a client makes a Get + request.

        • +

      • preExists, postExists: Called before and after the client tests + for existence using a Get.

        • +

      • prePut and postPut: Called before and after the client stores a value. +

        • +

      • preDelete and postDelete: Called before and after the client + deletes a value.

        • +

      • preScannerOpen postScannerOpen: Called before and after the client + opens a new scanner.

        • +

      • preScannerNext, postScannerNext: Called before and after the client + asks for the next row on a scanner.

        • +

      • preScannerClose, postScannerClose: Called before and after the client + closes a scanner.

        • +

      • preCheckAndPut, postCheckAndPut: Called before and after the client + calls checkAndPut().

        • +

      • preCheckAndDelete, postCheckAndDelete: Called before and after the client + calls checkAndDelete().

        • +

      +You can also extend abstract class BaseRegionObserverCoprocessor +which +implements both Coprocessor and RegionObserver. +In addition, it overrides all methods with default behaviors so you don't +have to override all of them. +

      +Here's an example of what a simple RegionObserver might look like. This +example shows how to implement access control for HBase. This +coprocessor checks user information for a given client request, e.g., +Get/Put/Delete/Scan by injecting code at certain +RegionObserver +preXXX hooks. If the user is not allowed to access the resource, a +CoprocessorException will be thrown. And the client request will be +denied by receiving this exception. +

      +
      +package org.apache.hadoop.hbase.coprocessor;
+
+import java.util.List;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Get;
+
+// Sample access-control coprocessor. It utilizes RegionObserver
+// and intercept preXXX() method to check user privilege for the given table
+// and column family.
+public class AccessControlCoprocessor extends BaseRegionObserverCoprocessor {
+  // @Override
+  public Get preGet(CoprocessorEnvironment e, Get get)
+      throws CoprocessorException {
+
+    // check permissions..
+    if (access_not_allowed)  {
+      throw new AccessDeniedException("User is not allowed to access.");
+    }
+    return get;
+  }
+
+  // override prePut(), preDelete(), etc.
+}
+
      +
      + +

      Endpoint

      +Coprocessor and RegionObserver provide certain hooks +for injecting user code running at each region. The user code will be triggerd +by existing HTable and HBaseAdmin operations at +the certain hook points. +

      +Through Endpoint and dynamic RPC protocol, you can define your own +interface communicated between client and region server, +i.e., you can create a new method, specify passed parameters and return types +for this new method. +And the new Endpoint methods can be triggered by +calling client side dynamic RPC functions -- HTable.coprocessorExec(...) +. +

      +To implement a Endpoint, you need to: +

        +
      • Extend CoprocessorProtocol: the interface defines +communication protocol for the new Endpoint, and will be +served as the RPC protocol between client and region server.
        • +
      • Extend both BaseEndpointCoprocessor abstract class, +and the above extended CoprocessorProtocol interface: +the actually implemented class running at region server.
        • +
      +

      +Here's an example of performing column aggregation at region server: +

      +
      +// A sample protocol for performing aggregation at regions.
+public static interface ColumnAggregationProtocol
+extends CoprocessorProtocol {
+  // Perform aggregation for a given column at the region. The aggregation
+  // will include all the rows inside the region. It can be extended to
+  // allow passing start and end rows for a fine-grained aggregation.
+  public int sum(byte[] family, byte[] qualifier) throws IOException;
+}
+// Aggregation implementation at a region.
+public static class ColumnAggregationEndpoint extends BaseEndpointCoprocessor
+implements ColumnAggregationProtocol {
+  // @Override
+  // Scan the region by the given family and qualifier. Return the aggregation
+  // result.
+  public int sum(byte[] family, byte[] qualifier)
+  throws IOException {
+    // aggregate at each region
+    Scan scan = new Scan();
+    scan.addColumn(family, qualifier);
+    int sumResult = 0;
+    // use an internal scanner to perform scanning.
+    InternalScanner scanner = getEnvironment().getRegion().getScanner(scan);
+    try {
+      List<KeyValue> curVals = new ArrayList<KeyValue>();
+      boolean done = false;
+      do {
+        curVals.clear();
+        done = scanner.next(curVals);
+        KeyValue kv = curVals.get(0);
+        sumResult += Bytes.toInt(kv.getValue());
+      } while (done);
+    } finally {
+      scanner.close();
+    }
+    return sumResult;
+  }
+}
+
      +
      +

      +Client invocations are performed through HTable, +which has the following methods added by dynamic RPC protocol: + +

      +
      +public <T extends CoprocessorProtocol> T coprocessorProxy(Class<T> protocol, Row row)
+
+public <T extends CoprocessorProtocol, R> void coprocessorExec(
+    Class<T> protocol, List<? extends Row> rows,
+    BatchCall<T,R> callable, BatchCallback<R> callback)
+
+public <T extends CoprocessorProtocol, R> void coprocessorExec(
+    Class<T> protocol, RowRange range,
+    BatchCall<T,R> callable, BatchCallback<R> callback)
+
      +
      + +

      +Here is a client side example of invoking +ColumnAggregationEndpoint: +

      +
      +HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
+Scan scan;
+Map<byte[], Integer> results;
+
+// scan: for all regions
+scan = new Scan();
+results = table.coprocessorExec(ColumnAggregationProtocol.class, scan,
+    new BatchCall<ColumnAggregationProtocol,Integer>() {
+      public Integer call(ColumnAggregationProtocol instance) throws IOException{
+        return instance.sum(TEST_FAMILY, TEST_QUALIFIER);
+      }
+    });
+int sumResult = 0;
+int expectedResult = 0;
+for (Map.Entry<byte[], Integer> e : results.entrySet()) {
+  sumResult += e.getValue();
+}
+
      +
      +

      Coprocess loading

      +A customized coprocessor can be loaded by two different ways, by configuration, +or by HTableDescriptor for a newly created table. +

      +(Currently we don't really have an on demand coprocessor loading machanism for +opened regions.) +

      Load from configuration

      +Whenever a region is opened, it will read coprocessor class names from +hbase.coprocessor.region.classes from Configuration. +Coprocessor framework will automatically load the configured classes as +default coprocessors. The classes must be included in the classpath already. + +

      +

      +
      +  <property>
+    <name>hbase.coprocessor.region.classes</name>
+    <value>org.apache.hadoop.hbase.coprocessor.AccessControllCoprocessor, org.apache.hadoop.hbase.coprocessor.ColumnAggregationProtocol</value>
+    <description>A comma-separated list of Coprocessors that are loaded by
+    default. For any override coprocessor method from RegionObservor or
+    Coprocessor, these classes' implementation will be called
+    in order. After implement your own
+    Coprocessor, just put it in HBase's classpath and add the fully
+    qualified class name here.
+    </description>
+  </property>
+
      +
      +

      +The first defined coprocessor will be assigned +Coprocessor.Priority.SYSTEM as priority. And each following +coprocessor's priority will be incremented by one. Coprocessors are executed +in order according to the natural ordering of the int. + +

      Load from table attribute

      +Coprocessor classes can also be configured at table attribute. The +attribute key must start with "Coprocessor" and values of the form is +<path>:<class>:<priority>, so that the framework can +recognize and load it. +

      +

      +
      +'COPROCESSOR$1' => 'hdfs://localhost:8020/hbase/coprocessors/test.jar:Test:1000'
+'COPROCESSOR$2' => '/hbase/coprocessors/test2.jar:AnotherTest:1001'
+
      +
      +

      +<class> is the coprocessor implementation class. A jar can contain +more than one coprocessor implementation, but only one can be specified +at a time in each table attribute. +

      +<priority> is an integer. Coprocessors are executed in order according +to the natural ordering of the int. Coprocessors can optionally abort +actions. So typically one would want to put authoritative CPs (security +policy implementations, perhaps) ahead of observers. +

      +

      +
      +  Path path = new Path(fs.getUri() + Path.SEPARATOR +
+    "TestClassloading.jar");
+
+  // create a table that references the jar
+  HTableDescriptor htd = new HTableDescriptor(getClass().getName());
+  htd.addFamily(new HColumnDescriptor("test"));
+  htd.setValue("Coprocessor$1",
+    path.toString() +
+    ":" + classFullName +
+    ":" + Coprocessor.Priority.USER);
+  HBaseAdmin admin = new HBaseAdmin(this.conf);
+  admin.createTable(htd);
+
+
      Chain of RegionObservers
      
+As described above, multiple coprocessors can be loaded at one region at the
+same time. In case of RegionObserver, you can have more than one
+RegionObservers register to one same hook point, i.e, preGet(), etc.
+When a region reach the
+hook point, the framework will invoke each registered RegionObserver by the
+order of assigned priority.
+
+
      +
      + +*/ +package org.apache.hadoop.hbase.coprocessor; diff --git a/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignException.java b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignException.java new file mode 100644 index 0000000..ee340cb --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignException.java @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.errorhandling; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage; +import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage; +import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage; + +import com.google.protobuf.InvalidProtocolBufferException; + +/** + * A ForeignException is an exception from another thread or process. + *

      + * ForeignExceptions are sent to 'remote' peers to signal an abort in the face of failures. + * When serialized for transmission we encode using Protobufs to ensure version compatibility. + *

      + * Foreign exceptions contain a Throwable as its cause. This can be a "regular" exception + * generated locally or a ProxyThrowable that is a representation of the original exception + * created on original 'remote' source. These ProxyThrowables have their their stacks traces and + * messages overridden to reflect the original 'remote' exception. The only way these + * ProxyThrowables are generated are by this class's {@link #deserialize(byte[])} method. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +@SuppressWarnings("serial") +public class ForeignException extends IOException { + + /** + * Name of the throwable's source such as a host or thread name. Must be non-null. + */ + private final String source; + + /** + * Create a new ForeignException that can be serialized. It is assumed that this came form a + * local source. + * @param source + * @param cause + */ + public ForeignException(String source, Throwable cause) { + super(cause); + assert source != null; + assert cause != null; + this.source = source; + } + + /** + * Create a new ForeignException that can be serialized. It is assumed that this is locally + * generated. + * @param source + * @param msg + */ + public ForeignException(String source, String msg) { + super(new IllegalArgumentException(msg)); + this.source = source; + } + + public String getSource() { + return source; + } + + /** + * The cause of a ForeignException can be an exception that was generated on a local in process + * thread, or a thread from a 'remote' separate process. + * + * If the cause is a ProxyThrowable, we know it came from deserialization which usually means + * it came from not only another thread, but also from a remote thread. + * + * @return true if went through deserialization, false if locally generated + */ + public boolean isRemote() { + return getCause() instanceof ProxyThrowable; + } + + @Override + public String toString() { + String className = getCause().getClass().getName() ; + return className + " via " + getSource() + ":" + getLocalizedMessage(); + } + + /** + * Convert a stack trace to list of {@link StackTraceElement}. + * @param trace the stack trace to convert to protobuf message + * @return null if the passed stack is null. + */ + private static List toStackTraceElementMessages( + StackTraceElement[] trace) { + // if there is no stack trace, ignore it and just return the message + if (trace == null) return null; + // build the stack trace for the message + List pbTrace = + new ArrayList(trace.length); + for (StackTraceElement elem : trace) { + StackTraceElementMessage.Builder stackBuilder = StackTraceElementMessage.newBuilder(); + stackBuilder.setDeclaringClass(elem.getClassName()); + stackBuilder.setFileName(elem.getFileName()); + stackBuilder.setLineNumber(elem.getLineNumber()); + stackBuilder.setMethodName(elem.getMethodName()); + pbTrace.add(stackBuilder.build()); + } + return pbTrace; + } + + /** + * This is a Proxy Throwable that contains the information of the original remote exception + */ + private static class ProxyThrowable extends Throwable { + ProxyThrowable(String msg, StackTraceElement[] trace) { + super(msg); + this.setStackTrace(trace); + } + } + + /** + * Converts a ForeignException to an array of bytes. + * @param source the name of the external exception source + * @param t the "local" external exception (local) + * @return protobuf serialized version of ForeignException + */ + public static byte[] serialize(String source, Throwable t) { + GenericExceptionMessage.Builder gemBuilder = GenericExceptionMessage.newBuilder(); + gemBuilder.setClassName(t.getClass().getName()); + if (t.getMessage() != null) { + gemBuilder.setMessage(t.getMessage()); + } + // set the stack trace, if there is one + List stack = + ForeignException.toStackTraceElementMessages(t.getStackTrace()); + if (stack != null) { + gemBuilder.addAllTrace(stack); + } + GenericExceptionMessage payload = gemBuilder.build(); + ForeignExceptionMessage.Builder exception = ForeignExceptionMessage.newBuilder(); + exception.setGenericException(payload).setSource(source); + ForeignExceptionMessage eem = exception.build(); + return eem.toByteArray(); + } + + /** + * Takes a series of bytes and tries to generate an ForeignException instance for it. + * @param bytes + * @return the ForeignExcpetion instance + * @throws InvalidProtocolBufferException if there was deserialization problem this is thrown. + */ + public static ForeignException deserialize(byte[] bytes) throws InvalidProtocolBufferException { + // figure out the data we need to pass + ForeignExceptionMessage eem = ForeignExceptionMessage.parseFrom(bytes); + GenericExceptionMessage gem = eem.getGenericException(); + StackTraceElement [] trace = ForeignException.toStackTrace(gem.getTraceList()); + ProxyThrowable dfe = new ProxyThrowable(gem.getMessage(), trace); + ForeignException e = new ForeignException(eem.getSource(), dfe); + return e; + } + + /** + * Unwind a serialized array of {@link StackTraceElementMessage}s to a + * {@link StackTraceElement}s. + * @param traceList list that was serialized + * @return the deserialized list or null if it couldn't be unwound (e.g. wasn't set on + * the sender). + */ + private static StackTraceElement[] toStackTrace(List traceList) { + if (traceList == null || traceList.size() == 0) { + return new StackTraceElement[0]; // empty array + } + StackTraceElement[] trace = new StackTraceElement[traceList.size()]; + for (int i = 0; i < traceList.size(); i++) { + StackTraceElementMessage elem = traceList.get(i); + trace[i] = new StackTraceElement( + elem.getDeclaringClass(), elem.getMethodName(), elem.getFileName(), elem.getLineNumber()); + } + return trace; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionDispatcher.java b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionDispatcher.java new file mode 100644 index 0000000..ceb3b84 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionDispatcher.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.errorhandling; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * The dispatcher acts as the state holding entity for foreign error handling. The first + * exception received by the dispatcher get passed directly to the listeners. Subsequent + * exceptions are dropped. + *

      + * If there are multiple dispatchers that are all in the same foreign exception monitoring group, + * ideally all these monitors are "peers" -- any error on one dispatcher should get propagated to + * all others (via rpc, or some other mechanism). Due to racing error conditions the exact reason + * for failure may be different on different peers, but the fact that they are in error state + * should eventually hold on all. + *

      + * This is thread-safe and must be because this is expected to be used to propagate exceptions + * from foreign threads. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class ForeignExceptionDispatcher implements ForeignExceptionListener, ForeignExceptionSnare { + public static final Log LOG = LogFactory.getLog(ForeignExceptionDispatcher.class); + protected final String name; + protected final List listeners = + new ArrayList(); + private ForeignException exception; + + public ForeignExceptionDispatcher(String name) { + this.name = name; + } + + public ForeignExceptionDispatcher() { + this(""); + } + + public String getName() { + return name; + } + + @Override + public synchronized void receive(ForeignException e) { + // if we already have an exception, then ignore it + if (exception != null) return; + + LOG.debug(name + " accepting received exception" , e); + // mark that we got the error + if (e != null) { + exception = e; + } else { + exception = new ForeignException(name, ""); + } + + // notify all the listeners + dispatch(e); + } + + @Override + public synchronized void rethrowException() throws ForeignException { + if (exception != null) { + // This gets the stack where this is caused, (instead of where it was deserialized). + // This is much more useful for debugging + throw new ForeignException(exception.getSource(), exception.getCause()); + } + } + + @Override + public synchronized boolean hasException() { + return exception != null; + } + + @Override + synchronized public ForeignException getException() { + return exception; + } + + /** + * Sends an exception to all listeners. + * @param message human readable message passed to the listener + * @param e {@link ForeignException} containing the cause. Can be null. + */ + private void dispatch(ForeignException e) { + // update all the listeners with the passed error + for (ForeignExceptionListener l: listeners) { + l.receive(e); + } + } + + /** + * Listen for failures to a given process. This method should only be used during + * initialization and not added to after exceptions are accepted. + * @param errorable listener for the errors. may be null. + */ + public synchronized void addListener(ForeignExceptionListener errorable) { + this.listeners.add(errorable); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionListener.java b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionListener.java new file mode 100644 index 0000000..014da53 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionListener.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.errorhandling; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * The ForeignExceptionListener is an interface for objects that can receive a ForeignException. + *

      + * Implementations must be thread-safe, because this is expected to be used to propagate exceptions + * from foreign threads. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface ForeignExceptionListener { + + /** + * Receive a ForeignException. + *

      + * Implementers must ensure that this method is thread-safe. + * @param e exception causing the error. Implementations must accept and handle null here. + */ + public void receive(ForeignException e); +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionSnare.java b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionSnare.java new file mode 100644 index 0000000..47586dd --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignExceptionSnare.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.errorhandling; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * This is an interface for a cooperative exception throwing mechanism. Implementations are + * containers that holds an exception from a separate thread. This can be used to receive + * exceptions from 'foreign' threads or from separate 'foreign' processes. + *

      + * To use, one would pass an implementation of this object to a long running method and + * periodically check by calling {@link #rethrowException()}. If any foreign exceptions have + * been received, the calling thread is then responsible for handling the rethrown exception. + *

      + * One could use the boolean {@link #hasException()} to determine if there is an exceptoin as well. + *

      + * NOTE: This is very similar to the InterruptedException/interrupt/interrupted pattern. There, + * the notification state is bound to a Thread. Using this, applications receive Exceptions in + * the snare. The snare is referenced and checked by multiple threads which enables exception + * notification in all the involved threads/processes. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public interface ForeignExceptionSnare { + + /** + * Rethrow an exception currently held by the {@link ForeignExceptionSnare}. If there is + * no exception this is a no-op + * + * @throws ForeignException + * all exceptions from remote sources are procedure exceptions + */ + public void rethrowException() throws ForeignException; + + /** + * Non-exceptional form of {@link #rethrowException()}. Checks to see if any + * process to which the exception checkers is bound has created an error that + * would cause a failure. + * + * @return true if there has been an error,false otherwise + */ + public boolean hasException(); + + /** + * Get the value of the captured exception. + * + * @return the captured foreign exception or null if no exception captured. + */ + public ForeignException getException(); +} diff --git a/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutException.java b/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutException.java new file mode 100644 index 0000000..b67d7d4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutException.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.errorhandling; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Exception for timeout of a task. + * @see TimeoutExceptionInjector + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +@SuppressWarnings("serial") +public class TimeoutException extends Exception { + + private final String sourceName; + private final long start; + private final long end; + private final long expected; + + /** + * Exception indicating that an operation attempt has timed out + * @param start time the operation started (ms since epoch) + * @param end time the timeout was triggered (ms since epoch) + * @param expected expected amount of time for the operation to complete (ms) (ideally, expected <= end-start) + */ + public TimeoutException(String sourceName, long start, long end, long expected) { + super("Timeout elapsed! Source:" + sourceName + " Start:" + start + ", End:" + end + + ", diff:" + (end - start) + ", max:" + expected + " ms"); + this.sourceName = sourceName; + this.start = start; + this.end = end; + this.expected = expected; + } + + public long getStart() { + return start; + } + + public long getEnd() { + return end; + } + + public long getMaxAllowedOperationTime() { + return expected; + } + + public String getSourceName() { + return sourceName; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutExceptionInjector.java b/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutExceptionInjector.java new file mode 100644 index 0000000..9f40cbf --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/errorhandling/TimeoutExceptionInjector.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.errorhandling; + +import java.util.Timer; +import java.util.TimerTask; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; + +/** + * Time a given process/operation and report a failure if the elapsed time exceeds the max allowed + * time. + *

      + * The timer won't start tracking time until calling {@link #start()}. If {@link #complete()} or + * {@link #trigger()} is called before {@link #start()}, calls to {@link #start()} will fail. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class TimeoutExceptionInjector { + + private static final Log LOG = LogFactory.getLog(TimeoutExceptionInjector.class); + + private final long maxTime; + private volatile boolean complete; + private final Timer timer; + private final TimerTask timerTask; + private long start = -1; + + /** + * Create a generic timer for a task/process. + * @param listener listener to notify if the process times out + * @param maxTime max allowed running time for the process. Timer starts on calls to + * {@link #start()} + */ + public TimeoutExceptionInjector(final ForeignExceptionListener listener, final long maxTime) { + this.maxTime = maxTime; + timer = new Timer(); + timerTask = new TimerTask() { + @Override + public void run() { + // ensure we don't run this task multiple times + synchronized (this) { + // quick exit if we already marked the task complete + if (TimeoutExceptionInjector.this.complete) return; + // mark the task is run, to avoid repeats + TimeoutExceptionInjector.this.complete = true; + } + long end = EnvironmentEdgeManager.currentTimeMillis(); + TimeoutException tee = new TimeoutException( + "Timeout caused Foreign Exception", start, end, maxTime); + String source = "timer-" + timer; + listener.receive(new ForeignException(source, tee)); + } + }; + } + + public long getMaxTime() { + return maxTime; + } + + /** + * For all time forward, do not throw an error because the process has completed. + */ + public void complete() { + // warn if the timer is already marked complete. This isn't going to be thread-safe, but should + // be good enough and its not worth locking just for a warning. + if (this.complete) { + LOG.warn("Timer already marked completed, ignoring!"); + return; + } + LOG.debug("Marking timer as complete - no error notifications will be received for this timer."); + synchronized (this.timerTask) { + this.complete = true; + } + this.timer.cancel(); + } + + /** + * Start a timer to fail a process if it takes longer than the expected time to complete. + *

      + * Non-blocking. + * @throws IllegalStateException if the timer has already been marked done via {@link #complete()} + * or {@link #trigger()} + */ + public synchronized void start() throws IllegalStateException { + if (this.start >= 0) { + LOG.warn("Timer already started, can't be started again. Ignoring second request."); + return; + } + LOG.debug("Scheduling process timer to run in: " + maxTime + " ms"); + timer.schedule(timerTask, maxTime); + this.start = EnvironmentEdgeManager.currentTimeMillis(); + } + + /** + * Trigger the timer immediately. + *

      + * Exposed for testing. + */ + public void trigger() { + synchronized (timerTask) { + if (this.complete) { + LOG.warn("Timer already completed, not triggering."); + return; + } + LOG.debug("Triggering timer immediately!"); + this.timer.cancel(); + this.timerTask.run(); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java new file mode 100644 index 0000000..46dafe0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java @@ -0,0 +1,265 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.executor; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Server; + + +/** + * Abstract base class for all HBase event handlers. Subclasses should + * implement the {@link #process()} method. Subclasses should also do all + * necessary checks up in their constructor if possible -- check table exists, + * is disabled, etc. -- so they fail fast rather than later when process is + * running. Do it this way because process be invoked directly but event + * handlers are also + * run in an executor context -- i.e. asynchronously -- and in this case, + * exceptions thrown at process time will not be seen by the invoker, not till + * we implement a call-back mechanism so the client can pick them up later. + *

      + * Event handlers have an {@link EventType}. + * {@link EventType} is a list of ALL handler event types. We need to keep + * a full list in one place -- and as enums is a good shorthand for an + * implemenations -- because event handlers can be passed to executors when + * they are to be run asynchronously. The + * hbase executor, see {@link ExecutorService}, has a switch for passing + * event type to executor. + *

      + * Event listeners can be installed and will be called pre- and post- process if + * this EventHandler is run in a Thread (its a Runnable so if its {@link #run()} + * method gets called). Implement + * {@link EventHandlerListener}s, and registering using + * {@link #setListener(EventHandlerListener)}. + * @see ExecutorService + */ +public abstract class EventHandler implements Runnable, Comparable { + private static final Log LOG = LogFactory.getLog(EventHandler.class); + + // type of event this object represents + protected EventType eventType; + + protected Server server; + + // sequence id generator for default FIFO ordering of events + protected static AtomicLong seqids = new AtomicLong(0); + + // sequence id for this event + private final long seqid; + + // Listener to call pre- and post- processing. May be null. + private EventHandlerListener listener; + + // Time to wait for events to happen, should be kept short + protected final int waitingTimeForEvents; + + /** + * This interface provides pre- and post-process hooks for events. + */ + public interface EventHandlerListener { + /** + * Called before any event is processed + * @param event The event handler whose process method is about to be called. + */ + public void beforeProcess(EventHandler event); + /** + * Called after any event is processed + * @param event The event handler whose process method is about to be called. + */ + public void afterProcess(EventHandler event); + } + + /** + * List of all HBase event handler types. Event types are named by a + * convention: event type names specify the component from which the event + * originated and then where its destined -- e.g. RS2ZK_ prefix means the + * event came from a regionserver destined for zookeeper -- and then what + * the even is; e.g. REGION_OPENING. + * + *

      WARNING: Please do not insert, remove or swap any line in this enum + * Doing so would change or shift all the codes used to serialize + * events, which makes backwards compatibility very hard for clients. + */ + public enum EventType { + // Messages originating from RS (NOTE: there is NO direct communication from + // RS to Master). These are a result of RS updates into ZK. + //RS_ZK_REGION_CLOSING (1), // It is replaced by M_ZK_REGION_CLOSING(HBASE-4739) + RS_ZK_REGION_CLOSED (2), // RS has finished closing a region + RS_ZK_REGION_OPENING (3), // RS is in process of opening a region + RS_ZK_REGION_OPENED (4), // RS has finished opening a region + RS_ZK_REGION_SPLITTING (5), // RS has started a region split + RS_ZK_REGION_SPLIT (6), // RS split has completed. + RS_ZK_REGION_FAILED_OPEN (7), // RS failed to open a region + + // Messages originating from Master to RS + M_RS_OPEN_REGION (20), // Master asking RS to open a region + M_RS_OPEN_ROOT (21), // Master asking RS to open root + M_RS_OPEN_META (22), // Master asking RS to open meta + M_RS_CLOSE_REGION (23), // Master asking RS to close a region + M_RS_CLOSE_ROOT (24), // Master asking RS to close root + M_RS_CLOSE_META (25), // Master asking RS to close meta + + // Messages originating from Client to Master + C_M_DELETE_TABLE (40), // Client asking Master to delete a table + C_M_DISABLE_TABLE (41), // Client asking Master to disable a table + C_M_ENABLE_TABLE (42), // Client asking Master to enable a table + C_M_MODIFY_TABLE (43), // Client asking Master to modify a table + C_M_ADD_FAMILY (44), // Client asking Master to add family to table + C_M_DELETE_FAMILY (45), // Client asking Master to delete family of table + C_M_MODIFY_FAMILY (46), // Client asking Master to modify family of table + C_M_CREATE_TABLE (47), // Client asking Master to create a table + + // Updates from master to ZK. This is done by the master and there is + // nothing to process by either Master or RS + M_ZK_REGION_OFFLINE (50), // Master adds this region as offline in ZK + M_ZK_REGION_CLOSING (51), // Master adds this region as closing in ZK + + // Master controlled events to be executed on the master + M_SERVER_SHUTDOWN (70), // Master is processing shutdown of a RS + M_META_SERVER_SHUTDOWN (72), // Master is processing shutdown of RS hosting a meta region (-ROOT- or .META.). + + // WARNING: Please do not insert, remove or swap any line in this enum. + // RegionTransitionData.write() uses eventType.ordinal() that is the enum index + // and not the value specified in the enum definition. so we can't add stuff in the middle. + C_M_SNAPSHOT_TABLE (48), // Client asking Master to snapshot an offline table + C_M_RESTORE_SNAPSHOT (49); // Client asking Master to snapshot an offline table + + /** + * Constructor + */ + EventType(int value) {} + public boolean isOnlineSchemaChangeSupported() { + return ( + this.equals(EventType.C_M_ADD_FAMILY) || + this.equals(EventType.C_M_DELETE_FAMILY) || + this.equals(EventType.C_M_MODIFY_FAMILY) || + this.equals(EventType.C_M_MODIFY_TABLE) + ); + } + } + + /** + * Default base class constructor. + */ + public EventHandler(Server server, EventType eventType) { + this.server = server; + this.eventType = eventType; + seqid = seqids.incrementAndGet(); + this.waitingTimeForEvents = server.getConfiguration(). + getInt("hbase.master.event.waiting.time", 1000); + } + + public void run() { + try { + if (getListener() != null) getListener().beforeProcess(this); + process(); + if (getListener() != null) getListener().afterProcess(this); + } catch(Throwable t) { + LOG.error("Caught throwable while processing event " + eventType, t); + } + } + + /** + * This method is the main processing loop to be implemented by the various + * subclasses. + * @throws IOException + */ + public abstract void process() throws IOException; + + /** + * Return the event type + * @return The event type. + */ + public EventType getEventType() { + return this.eventType; + } + + /** + * Get the priority level for this handler instance. This uses natural + * ordering so lower numbers are higher priority. + *

      + * Lowest priority is Integer.MAX_VALUE. Highest priority is 0. + *

      + * Subclasses should override this method to allow prioritizing handlers. + *

      + * Handlers with the same priority are handled in FIFO order. + *

      + * @return Integer.MAX_VALUE by default, override to set higher priorities + */ + public int getPriority() { + return Integer.MAX_VALUE; + } + + /** + * @return This events' sequence id. + */ + public long getSeqid() { + return this.seqid; + } + + /** + * Default prioritized runnable comparator which implements a FIFO ordering. + *

      + * Subclasses should not override this. Instead, if they want to implement + * priority beyond FIFO, they should override {@link #getPriority()}. + */ + @Override + public int compareTo(Runnable o) { + EventHandler eh = (EventHandler)o; + if(getPriority() != eh.getPriority()) { + return (getPriority() < eh.getPriority()) ? -1 : 1; + } + return (this.seqid < eh.seqid) ? -1 : 1; + } + + /** + * @return Current listener or null if none set. + */ + public synchronized EventHandlerListener getListener() { + return listener; + } + + /** + * @param listener Listener to call pre- and post- {@link #process()}. + */ + public synchronized void setListener(EventHandlerListener listener) { + this.listener = listener; + } + + @Override + public String toString() { + return "Event #" + getSeqid() + + " of type " + eventType + + " (" + getInformativeName() + ")"; + } + + /** + * Event implementations should override thie class to provide an + * informative name about what event they are handling. For example, + * event-specific information such as which region or server is + * being processed should be included if possible. + */ + public String getInformativeName() { + return this.getClass().toString(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java b/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java new file mode 100644 index 0000000..a34dd69 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java @@ -0,0 +1,452 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.executor; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Writer; +import java.lang.management.ThreadInfo; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.monitoring.ThreadMonitoring; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * This is a generic executor service. This component abstracts a + * threadpool, a queue to which {@link EventHandler.EventType}s can be submitted, + * and a Runnable that handles the object that is added to the queue. + * + *

      In order to create a new service, create an instance of this class and + * then do: instance.startExecutorService("myService");. When done + * call {@link #shutdown()}. + * + *

      In order to use the service created above, call + * {@link #submit(EventHandler)}. Register pre- and post- processing listeners + * by registering your implementation of {@link EventHandler.EventHandlerListener} + * with {@link #registerListener(EventHandler.EventType, EventHandler.EventHandlerListener)}. Be sure + * to deregister your listener when done via {@link #unregisterListener(EventHandler.EventType)}. + */ +public class ExecutorService { + private static final Log LOG = LogFactory.getLog(ExecutorService.class); + + // hold the all the executors created in a map addressable by their names + private final ConcurrentHashMap executorMap = + new ConcurrentHashMap(); + + // listeners that are called before and after an event is processed + private ConcurrentHashMap eventHandlerListeners = + new ConcurrentHashMap(); + + // Name of the server hosting this executor service. + private final String servername; + + /** + * The following is a list of all executor types, both those that run in the + * master and those that run in the regionserver. + */ + public enum ExecutorType { + + // Master executor services + MASTER_CLOSE_REGION (1), + MASTER_OPEN_REGION (2), + MASTER_SERVER_OPERATIONS (3), + MASTER_TABLE_OPERATIONS (4), + MASTER_RS_SHUTDOWN (5), + MASTER_META_SERVER_OPERATIONS (6), + + // RegionServer executor services + RS_OPEN_REGION (20), + RS_OPEN_ROOT (21), + RS_OPEN_META (22), + RS_CLOSE_REGION (23), + RS_CLOSE_ROOT (24), + RS_CLOSE_META (25); + + ExecutorType(int value) {} + + /** + * @param serverName + * @return Conflation of the executor type and the passed servername. + */ + String getExecutorName(String serverName) { + return this.toString() + "-" + serverName.replace("%", "%%"); + } + } + + /** + * Returns the executor service type (the thread pool instance) for the + * passed event handler type. + * @param type EventHandler type. + */ + public ExecutorType getExecutorServiceType(final EventHandler.EventType type) { + switch(type) { + // Master executor services + + case RS_ZK_REGION_CLOSED: + case RS_ZK_REGION_FAILED_OPEN: + return ExecutorType.MASTER_CLOSE_REGION; + + case RS_ZK_REGION_OPENED: + return ExecutorType.MASTER_OPEN_REGION; + + case RS_ZK_REGION_SPLIT: + case M_SERVER_SHUTDOWN: + return ExecutorType.MASTER_SERVER_OPERATIONS; + + case M_META_SERVER_SHUTDOWN: + return ExecutorType.MASTER_META_SERVER_OPERATIONS; + + case C_M_DELETE_TABLE: + case C_M_DISABLE_TABLE: + case C_M_ENABLE_TABLE: + case C_M_MODIFY_TABLE: + case C_M_CREATE_TABLE: + case C_M_SNAPSHOT_TABLE: + case C_M_RESTORE_SNAPSHOT: + return ExecutorType.MASTER_TABLE_OPERATIONS; + + // RegionServer executor services + + case M_RS_OPEN_REGION: + return ExecutorType.RS_OPEN_REGION; + + case M_RS_OPEN_ROOT: + return ExecutorType.RS_OPEN_ROOT; + + case M_RS_OPEN_META: + return ExecutorType.RS_OPEN_META; + + case M_RS_CLOSE_REGION: + return ExecutorType.RS_CLOSE_REGION; + + case M_RS_CLOSE_ROOT: + return ExecutorType.RS_CLOSE_ROOT; + + case M_RS_CLOSE_META: + return ExecutorType.RS_CLOSE_META; + + default: + throw new RuntimeException("Unhandled event type " + type); + } + } + + /** + * Default constructor. + * @param servername Name of the hosting server. + */ + public ExecutorService(final String servername) { + super(); + this.servername = servername; + } + + /** + * Start an executor service with a given name. If there was a service already + * started with the same name, this throws a RuntimeException. + * @param name Name of the service to start. + */ + void startExecutorService(String name, int maxThreads) { + if (this.executorMap.get(name) != null) { + throw new RuntimeException("An executor service with the name " + name + + " is already running!"); + } + Executor hbes = new Executor(name, maxThreads, this.eventHandlerListeners); + if (this.executorMap.putIfAbsent(name, hbes) != null) { + throw new RuntimeException("An executor service with the name " + name + + " is already running (2)!"); + } + LOG.debug("Starting executor service name=" + name + + ", corePoolSize=" + hbes.threadPoolExecutor.getCorePoolSize() + + ", maxPoolSize=" + hbes.threadPoolExecutor.getMaximumPoolSize()); + } + + boolean isExecutorServiceRunning(String name) { + return this.executorMap.containsKey(name); + } + + public void shutdown() { + for(Entry entry: this.executorMap.entrySet()) { + List wasRunning = + entry.getValue().threadPoolExecutor.shutdownNow(); + if (!wasRunning.isEmpty()) { + LOG.info(entry.getValue() + " had " + wasRunning + " on shutdown"); + } + } + this.executorMap.clear(); + } + + Executor getExecutor(final ExecutorType type) { + return getExecutor(type.getExecutorName(this.servername)); + } + + Executor getExecutor(String name) { + Executor executor = this.executorMap.get(name); + return executor; + } + + + public void startExecutorService(final ExecutorType type, final int maxThreads) { + String name = type.getExecutorName(this.servername); + if (isExecutorServiceRunning(name)) { + LOG.debug("Executor service " + toString() + " already running on " + + this.servername); + return; + } + startExecutorService(name, maxThreads); + } + + public void submit(final EventHandler eh) { + Executor executor = getExecutor(getExecutorServiceType(eh.getEventType())); + if (executor == null) { + // This happens only when events are submitted after shutdown() was + // called, so dropping them should be "ok" since it means we're + // shutting down. + LOG.error("Cannot submit [" + eh + "] because the executor is missing." + + " Is this process shutting down?"); + } else { + executor.submit(eh); + } + } + + /** + * Subscribe to updates before and after processing instances of + * {@link EventHandler.EventType}. Currently only one listener per + * event type. + * @param type Type of event we're registering listener for + * @param listener The listener to run. + */ + public void registerListener(final EventHandler.EventType type, + final EventHandlerListener listener) { + this.eventHandlerListeners.put(type, listener); + } + + /** + * Stop receiving updates before and after processing instances of + * {@link EventHandler.EventType} + * @param type Type of event we're registering listener for + * @return The listener we removed or null if we did not remove it. + */ + public EventHandlerListener unregisterListener(final EventHandler.EventType type) { + return this.eventHandlerListeners.remove(type); + } + + public Map getAllExecutorStatuses() { + Map ret = Maps.newHashMap(); + for (Map.Entry e : executorMap.entrySet()) { + ret.put(e.getKey(), e.getValue().getStatus()); + } + return ret; + } + + /** + * Executor instance. + */ + static class Executor { + // how long to retain excess threads + final long keepAliveTimeInMillis = 1000; + // the thread pool executor that services the requests + final TrackingThreadPoolExecutor threadPoolExecutor; + // work queue to use - unbounded queue + final BlockingQueue q = new LinkedBlockingQueue(); + private final String name; + private final Map eventHandlerListeners; + private static final AtomicLong seqids = new AtomicLong(0); + private final long id; + + protected Executor(String name, int maxThreads, + final Map eventHandlerListeners) { + this.id = seqids.incrementAndGet(); + this.name = name; + this.eventHandlerListeners = eventHandlerListeners; + // create the thread pool executor + this.threadPoolExecutor = new TrackingThreadPoolExecutor( + maxThreads, maxThreads, + keepAliveTimeInMillis, TimeUnit.MILLISECONDS, q); + // name the threads for this threadpool + ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); + tfb.setNameFormat(this.name + "-%d"); + this.threadPoolExecutor.setThreadFactory(tfb.build()); + } + + /** + * Submit the event to the queue for handling. + * @param event + */ + void submit(final EventHandler event) { + // If there is a listener for this type, make sure we call the before + // and after process methods. + EventHandlerListener listener = + this.eventHandlerListeners.get(event.getEventType()); + if (listener != null) { + event.setListener(listener); + } + this.threadPoolExecutor.execute(event); + } + + public String toString() { + return getClass().getSimpleName() + "-" + id + "-" + name; + } + + public ExecutorStatus getStatus() { + List queuedEvents = Lists.newArrayList(); + for (Runnable r : q) { + if (!(r instanceof EventHandler)) { + LOG.warn("Non-EventHandler " + r + " queued in " + name); + continue; + } + queuedEvents.add((EventHandler)r); + } + + List running = Lists.newArrayList(); + for (Map.Entry e : + threadPoolExecutor.getRunningTasks().entrySet()) { + Runnable r = e.getValue(); + if (!(r instanceof EventHandler)) { + LOG.warn("Non-EventHandler " + r + " running in " + name); + continue; + } + running.add(new RunningEventStatus(e.getKey(), (EventHandler)r)); + } + + return new ExecutorStatus(this, queuedEvents, running); + } + } + + /** + * A subclass of ThreadPoolExecutor that keeps track of the Runnables that + * are executing at any given point in time. + */ + static class TrackingThreadPoolExecutor extends ThreadPoolExecutor { + private ConcurrentMap running = Maps.newConcurrentMap(); + + public TrackingThreadPoolExecutor(int corePoolSize, int maximumPoolSize, + long keepAliveTime, TimeUnit unit, BlockingQueue workQueue) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); + } + + @Override + protected void afterExecute(Runnable r, Throwable t) { + super.afterExecute(r, t); + running.remove(Thread.currentThread()); + } + + @Override + protected void beforeExecute(Thread t, Runnable r) { + Runnable oldPut = running.put(t, r); + assert oldPut == null : "inconsistency for thread " + t; + super.beforeExecute(t, r); + } + + /** + * @return a map of the threads currently running tasks + * inside this executor. Each key is an active thread, + * and the value is the task that is currently running. + * Note that this is not a stable snapshot of the map. + */ + public ConcurrentMap getRunningTasks() { + return running; + } + } + + /** + * A snapshot of the status of a particular executor. This includes + * the contents of the executor's pending queue, as well as the + * threads and events currently being processed. + * + * This is a consistent snapshot that is immutable once constructed. + */ + public static class ExecutorStatus { + final Executor executor; + final List queuedEvents; + final List running; + + ExecutorStatus(Executor executor, + List queuedEvents, + List running) { + this.executor = executor; + this.queuedEvents = queuedEvents; + this.running = running; + } + + /** + * Dump a textual representation of the executor's status + * to the given writer. + * + * @param out the stream to write to + * @param indent a string prefix for each line, used for indentation + */ + public void dumpTo(Writer out, String indent) throws IOException { + out.write(indent + "Status for executor: " + executor + "\n"); + out.write(indent + "=======================================\n"); + out.write(indent + queuedEvents.size() + " events queued, " + + running.size() + " running\n"); + if (!queuedEvents.isEmpty()) { + out.write(indent + "Queued:\n"); + for (EventHandler e : queuedEvents) { + out.write(indent + " " + e + "\n"); + } + out.write("\n"); + } + if (!running.isEmpty()) { + out.write(indent + "Running:\n"); + for (RunningEventStatus stat : running) { + out.write(indent + " Running on thread '" + + stat.threadInfo.getThreadName() + + "': " + stat.event + "\n"); + out.write(ThreadMonitoring.formatThreadInfo( + stat.threadInfo, indent + " ")); + out.write("\n"); + } + } + out.flush(); + } + } + + /** + * The status of a particular event that is in the middle of being + * handled by an executor. + */ + public static class RunningEventStatus { + final ThreadInfo threadInfo; + final EventHandler event; + + public RunningEventStatus(Thread t, EventHandler event) { + this.threadInfo = ThreadMonitoring.getThreadInfo(t); + this.event = event; + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java b/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java new file mode 100644 index 0000000..2f5f092 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java @@ -0,0 +1,250 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.executor; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.io.Writable; + +/** + * Data serialized into ZooKeeper for region transitions. + */ +public class RegionTransitionData implements Writable { + /** + * Type of transition event (offline, opening, opened, closing, closed). + * Required. + */ + private EventType eventType; + + /** Region being transitioned. Required. */ + private byte [] regionName; + + /** Server event originated from. Optional. */ + private ServerName origin; + + /** Time the event was created. Required but automatically set. */ + private long stamp; + + private byte [] payload; + + /** + * Writable constructor. Do not use directly. + */ + public RegionTransitionData() {} + + /** + * Construct data for a new region transition event with the specified event + * type and region name. + * + *

      Used when the server name is not known (the master is setting it). This + * happens during cluster startup or during failure scenarios. When + * processing a failed regionserver, the master assigns the regions from that + * server to other servers though the region was never 'closed'. During + * master failover, the new master may have regions stuck in transition + * without a destination so may have to set regions offline and generate a new + * assignment. + * + *

      Since only the master uses this constructor, the type should always be + * {@link EventType#M_ZK_REGION_OFFLINE}. + * + * @param eventType type of event + * @param regionName name of region as per HRegionInfo#getRegionName() + */ + public RegionTransitionData(EventType eventType, byte [] regionName) { + this(eventType, regionName, null); + } + + /** + * Construct data for a new region transition event with the specified event + * type, region name, and server name. + * + *

      Used when the server name is known (a regionserver is setting it). + * + *

      Valid types for this constructor are {@link EventType#M_ZK_REGION_CLOSING}, + * {@link EventType#RS_ZK_REGION_CLOSED}, {@link EventType#RS_ZK_REGION_OPENING}, + * {@link EventType#RS_ZK_REGION_SPLITTING}, + * and {@link EventType#RS_ZK_REGION_OPENED}. + * + * @param eventType type of event + * @param regionName name of region as per HRegionInfo#getRegionName() + * @param origin Originating {@link ServerName} + */ + public RegionTransitionData(EventType eventType, byte [] regionName, + final ServerName origin) { + this(eventType, regionName, origin, null); + } + + /** + * Construct data for a new region transition event with the specified event + * type, region name, and server name. + * + *

      Used when the server name is known (a regionserver is setting it). + * + *

      Valid types for this constructor are {@link EventType#RS_ZK_REGION_SPLIT} + * since SPLIT is only type that currently carries a payload. + * + * @param eventType type of event + * @param regionName name of region as per HRegionInfo#getRegionName() + * @param serverName Originating {@link ServerName} + * @param payload Payload examples include the daughters involved in a + * {@link EventType#RS_ZK_REGION_SPLIT}. Can be null + */ + public RegionTransitionData(EventType eventType, byte [] regionName, + final ServerName serverName, final byte [] payload) { + this.eventType = eventType; + this.stamp = System.currentTimeMillis(); + this.regionName = regionName; + this.origin = serverName; + this.payload = payload; + } + + /** + * Gets the type of region transition event. + * + *

      One of: + *

        + *
      • {@link EventType#M_ZK_REGION_OFFLINE} + *
      • {@link EventType#M_ZK_REGION_CLOSING} + *
      • {@link EventType#RS_ZK_REGION_CLOSED} + *
      • {@link EventType#RS_ZK_REGION_OPENING} + *
      • {@link EventType#RS_ZK_REGION_OPENED} + *
      • {@link EventType#RS_ZK_REGION_SPLITTING} + *
      • {@link EventType#RS_ZK_REGION_SPLIT} + *
      + * @return type of region transition event + */ + public EventType getEventType() { + return eventType; + } + + /** + * Gets the name of the region being transitioned. + * + *

      Region name is required so this never returns null. + * @return region name, the result of a call to HRegionInfo#getRegionName() + */ + public byte [] getRegionName() { + return regionName; + } + + /** + * Gets the server the event originated from. If null, this event originated + * from the master. + * + * @return server name of originating regionserver, or null if from master + */ + public ServerName getOrigin() { + return origin; + } + + /** + * Gets the timestamp when this event was created. + * + * @return stamp event was created + */ + public long getStamp() { + return stamp; + } + + /** + * @return Payload if any. + */ + public byte [] getPayload() { + return this.payload; + } + + @Override + public void readFields(DataInput in) throws IOException { + // the event type byte + eventType = EventType.values()[in.readShort()]; + // the timestamp + stamp = in.readLong(); + // the encoded name of the region being transitioned + regionName = Bytes.readByteArray(in); + // remaining fields are optional so prefixed with boolean + // the name of the regionserver sending the data + if (in.readBoolean()) { + byte [] versionedBytes = Bytes.readByteArray(in); + this.origin = ServerName.parseVersionedServerName(versionedBytes); + } + if (in.readBoolean()) { + this.payload = Bytes.readByteArray(in); + } + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeShort(eventType.ordinal()); + out.writeLong(System.currentTimeMillis()); + Bytes.writeByteArray(out, regionName); + // remaining fields are optional so prefixed with boolean + out.writeBoolean(this.origin != null); + if (this.origin != null) { + Bytes.writeByteArray(out, this.origin.getVersionedBytes()); + } + out.writeBoolean(this.payload != null); + if (this.payload != null) { + Bytes.writeByteArray(out, this.payload); + } + } + + /** + * Get the bytes for this instance. Throws a {@link RuntimeException} if + * there is an error deserializing this instance because it represents a code + * bug. + * @return binary representation of this instance + */ + public byte [] getBytes() { + try { + return Writables.getBytes(this); + } catch(IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Get an instance from bytes. Throws a {@link RuntimeException} if + * there is an error serializing this instance from bytes because it + * represents a code bug. + * @param bytes binary representation of this instance + * @return instance of this class + */ + public static RegionTransitionData fromBytes(byte [] bytes) { + try { + RegionTransitionData data = new RegionTransitionData(); + Writables.getWritable(bytes, data); + return data; + } catch(IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public String toString() { + return "region=" + Bytes.toStringBinary(regionName) + ", origin=" + this.origin + + ", state=" + eventType; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/BinaryComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/BinaryComparator.java new file mode 100644 index 0000000..1e56948 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/BinaryComparator.java @@ -0,0 +1,46 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.util.Bytes; + +/** + * A binary comparator which lexicographically compares against the specified + * byte array using {@link org.apache.hadoop.hbase.util.Bytes#compareTo(byte[], byte[])}. + */ +public class BinaryComparator extends WritableByteArrayComparable { + + /** Nullary constructor for Writable, do not use */ + public BinaryComparator() { } + + /** + * Constructor + * @param value value + */ + public BinaryComparator(byte[] value) { + super(value); + } + + @Override + public int compareTo(byte [] value, int offset, int length) { + return Bytes.compareTo(this.value, 0, this.value.length, value, offset, length); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/BinaryPrefixComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/BinaryPrefixComparator.java new file mode 100644 index 0000000..030341a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/BinaryPrefixComparator.java @@ -0,0 +1,48 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.util.Bytes; + +/** + * A comparator which compares against a specified byte array, but only compares + * up to the length of this byte array. For the rest it is similar to + * {@link BinaryComparator}. + */ +public class BinaryPrefixComparator extends WritableByteArrayComparable { + + /** Nullary constructor for Writable, do not use */ + public BinaryPrefixComparator() { } + + /** + * Constructor + * @param value value + */ + public BinaryPrefixComparator(byte[] value) { + super(value); + } + + @Override + public int compareTo(byte [] value, int offset, int length) { + return Bytes.compareTo(this.value, 0, this.value.length, value, offset, + this.value.length <= length ? this.value.length : length); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java new file mode 100644 index 0000000..088a099 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java @@ -0,0 +1,99 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * A bit comparator which performs the specified bitwise operation on each of the bytes + * with the specified byte array. Then returns whether the result is non-zero. + */ +public class BitComparator extends WritableByteArrayComparable { + + /** Nullary constructor for Writable, do not use */ + public BitComparator() {} + + /** Bit operators. */ + public enum BitwiseOp { + /** and */ + AND, + /** or */ + OR, + /** xor */ + XOR + } + protected BitwiseOp bitOperator; + + /** + * Constructor + * @param value value + * @param bitOperator operator to use on the bit comparison + */ + public BitComparator(byte[] value, BitwiseOp bitOperator) { + super(value); + this.bitOperator = bitOperator; + } + + /** + * @return the bitwise operator + */ + public BitwiseOp getOperator() { + return bitOperator; + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + bitOperator = BitwiseOp.valueOf(in.readUTF()); + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeUTF(bitOperator.name()); + } + + @Override + public int compareTo(byte[] value, int offset, int length) { + if (length != this.value.length) { + return 1; + } + int b = 0; + //Iterating backwards is faster because we can quit after one non-zero byte. + for (int i = length - 1; i >= 0 && b == 0; i--) { + switch (bitOperator) { + case AND: + b = (this.value[i] & value[i+offset]) & 0xff; + break; + case OR: + b = (this.value[i] | value[i+offset]) & 0xff; + break; + case XOR: + b = (this.value[i] ^ value[i+offset]) & 0xff; + break; + } + } + return b == 0 ? 1 : 0; + } +} + diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java new file mode 100644 index 0000000..2bca1c4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java @@ -0,0 +1,97 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; + +import com.google.common.base.Preconditions; + +/** + * Simple filter that returns first N columns on row only. + * This filter was written to test filters in Get and as soon as it gets + * its quota of columns, {@link #filterAllRemaining()} returns true. This + * makes this filter unsuitable as a Scan filter. + */ +public class ColumnCountGetFilter extends FilterBase { + private int limit = 0; + private int count = 0; + + /** + * Used during serialization. + * Do not use. + */ + public ColumnCountGetFilter() { + super(); + } + + public ColumnCountGetFilter(final int n) { + Preconditions.checkArgument(n >= 0, "limit be positive %s", n); + this.limit = n; + } + + public int getLimit() { + return limit; + } + + @Override + public boolean filterAllRemaining() { + return this.count > this.limit; + } + + @Override + public ReturnCode filterKeyValue(KeyValue v) { + this.count++; + return filterAllRemaining() ? ReturnCode.NEXT_COL: + ReturnCode.INCLUDE_AND_NEXT_COL; + } + + @Override + public void reset() { + this.count = 0; + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + Preconditions.checkArgument(filterArguments.size() == 1, + "Expected 1 but got: %s", filterArguments.size()); + int limit = ParseFilter.convertByteArrayToInt(filterArguments.get(0)); + return new ColumnCountGetFilter(limit); + } + + @Override + public void readFields(DataInput in) throws IOException { + this.limit = in.readInt(); + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(this.limit); + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + " " + this.limit; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java new file mode 100644 index 0000000..1214a1a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java @@ -0,0 +1,117 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; + +import org.apache.hadoop.hbase.KeyValue; +import com.google.common.base.Preconditions; + +/** + * A filter, based on the ColumnCountGetFilter, takes two arguments: limit and offset. + * This filter can be used for row-based indexing, where references to other tables are stored across many columns, + * in order to efficient lookups and paginated results for end users. Only most recent versions are considered + * for pagination. + */ +public class ColumnPaginationFilter extends FilterBase +{ + private int limit = 0; + private int offset = 0; + private int count = 0; + + /** + * Used during serialization. Do not use. + */ + public ColumnPaginationFilter() + { + super(); + } + + public ColumnPaginationFilter(final int limit, final int offset) + { + Preconditions.checkArgument(limit >= 0, "limit must be positive %s", limit); + Preconditions.checkArgument(offset >= 0, "offset must be positive %s", offset); + this.limit = limit; + this.offset = offset; + } + + /** + * @return limit + */ + public int getLimit() { + return limit; + } + + /** + * @return offset + */ + public int getOffset() { + return offset; + } + + @Override + public ReturnCode filterKeyValue(KeyValue v) + { + if(count >= offset + limit) + { + return ReturnCode.NEXT_ROW; + } + + ReturnCode code = count < offset ? ReturnCode.NEXT_COL : + ReturnCode.INCLUDE_AND_NEXT_COL; + count++; + return code; + } + + @Override + public void reset() + { + this.count = 0; + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + Preconditions.checkArgument(filterArguments.size() == 2, + "Expected 2 but got: %s", filterArguments.size()); + int limit = ParseFilter.convertByteArrayToInt(filterArguments.get(0)); + int offset = ParseFilter.convertByteArrayToInt(filterArguments.get(1)); + return new ColumnPaginationFilter(limit, offset); + } + + public void readFields(DataInput in) throws IOException + { + this.limit = in.readInt(); + this.offset = in.readInt(); + } + + public void write(DataOutput out) throws IOException + { + out.writeInt(this.limit); + out.writeInt(this.offset); + } + + @Override + public String toString() { + return String.format("%s (%d, %d)", this.getClass().getSimpleName(), + this.limit, this.offset); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java new file mode 100644 index 0000000..cb89b7d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java @@ -0,0 +1,109 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + +import java.io.DataOutput; +import java.io.IOException; +import java.io.DataInput; +import java.util.ArrayList; + +import com.google.common.base.Preconditions; + +/** + * This filter is used for selecting only those keys with columns that matches + * a particular prefix. For example, if prefix is 'an', it will pass keys with + * columns like 'and', 'anti' but not keys with columns like 'ball', 'act'. + */ +public class ColumnPrefixFilter extends FilterBase { + protected byte [] prefix = null; + + public ColumnPrefixFilter() { + super(); + } + + public ColumnPrefixFilter(final byte [] prefix) { + this.prefix = prefix; + } + + public byte[] getPrefix() { + return prefix; + } + + @Override + public ReturnCode filterKeyValue(KeyValue kv) { + if (this.prefix == null || kv.getBuffer() == null) { + return ReturnCode.INCLUDE; + } else { + return filterColumn(kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength()); + } + } + + public ReturnCode filterColumn(byte[] buffer, int qualifierOffset, int qualifierLength) { + if (qualifierLength < prefix.length) { + int cmp = Bytes.compareTo(buffer, qualifierOffset, qualifierLength, this.prefix, 0, + qualifierLength); + if (cmp <= 0) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } else { + return ReturnCode.NEXT_ROW; + } + } else { + int cmp = Bytes.compareTo(buffer, qualifierOffset, this.prefix.length, this.prefix, 0, + this.prefix.length); + if (cmp < 0) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } else if (cmp > 0) { + return ReturnCode.NEXT_ROW; + } else { + return ReturnCode.INCLUDE; + } + } + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + Preconditions.checkArgument(filterArguments.size() == 1, + "Expected 1 but got: %s", filterArguments.size()); + byte [] columnPrefix = ParseFilter.removeQuotesFromByteArray(filterArguments.get(0)); + return new ColumnPrefixFilter(columnPrefix); + } + + public void write(DataOutput out) throws IOException { + Bytes.writeByteArray(out, this.prefix); + } + + public void readFields(DataInput in) throws IOException { + this.prefix = Bytes.readByteArray(in); + } + + public KeyValue getNextKeyHint(KeyValue kv) { + return KeyValue.createFirstOnRow( + kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(), + kv.getFamilyOffset(), kv.getFamilyLength(), prefix, 0, prefix.length); + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + " " + Bytes.toStringBinary(this.prefix); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java new file mode 100644 index 0000000..bd5c573 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java @@ -0,0 +1,213 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + +import java.io.DataOutput; +import java.io.IOException; +import java.io.DataInput; +import java.util.ArrayList; + +import com.google.common.base.Preconditions; + +/** + * This filter is used for selecting only those keys with columns that are + * between minColumn to maxColumn. For example, if minColumn is 'an', and + * maxColumn is 'be', it will pass keys with columns like 'ana', 'bad', but not + * keys with columns like 'bed', 'eye' + * + * If minColumn is null, there is no lower bound. If maxColumn is null, there is + * no upper bound. + * + * minColumnInclusive and maxColumnInclusive specify if the ranges are inclusive + * or not. + */ +public class ColumnRangeFilter extends FilterBase { + protected byte[] minColumn = null; + protected boolean minColumnInclusive = true; + protected byte[] maxColumn = null; + protected boolean maxColumnInclusive = false; + + public ColumnRangeFilter() { + super(); + } + /** + * Create a filter to select those keys with columns that are between minColumn + * and maxColumn. + * @param minColumn minimum value for the column range. If if it's null, + * there is no lower bound. + * @param minColumnInclusive if true, include minColumn in the range. + * @param maxColumn maximum value for the column range. If it's null, + * @param maxColumnInclusive if true, include maxColumn in the range. + * there is no upper bound. + */ + public ColumnRangeFilter(final byte[] minColumn, boolean minColumnInclusive, + final byte[] maxColumn, boolean maxColumnInclusive) { + this.minColumn = minColumn; + this.minColumnInclusive = minColumnInclusive; + this.maxColumn = maxColumn; + this.maxColumnInclusive = maxColumnInclusive; + } + + /** + * @return if min column range is inclusive. + */ + public boolean isMinColumnInclusive() { + return minColumnInclusive; + } + + /** + * @return if max column range is inclusive. + */ + public boolean isMaxColumnInclusive() { + return maxColumnInclusive; + } + + /** + * @return the min column range for the filter + */ + public byte[] getMinColumn() { + return this.minColumn; + } + + /** + * @return true if min column is inclusive, false otherwise + */ + public boolean getMinColumnInclusive() { + return this.minColumnInclusive; + } + + /** + * @return the max column range for the filter + */ + public byte[] getMaxColumn() { + return this.maxColumn; + } + + /** + * @return true if max column is inclusive, false otherwise + */ + public boolean getMaxColumnInclusive() { + return this.maxColumnInclusive; + } + + @Override + public ReturnCode filterKeyValue(KeyValue kv) { + byte[] buffer = kv.getBuffer(); + int qualifierOffset = kv.getQualifierOffset(); + int qualifierLength = kv.getQualifierLength(); + int cmpMin = 1; + + if (this.minColumn != null) { + cmpMin = Bytes.compareTo(buffer, qualifierOffset, qualifierLength, + this.minColumn, 0, this.minColumn.length); + } + + if (cmpMin < 0) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } + + if (!this.minColumnInclusive && cmpMin == 0) { + return ReturnCode.SKIP; + } + + if (this.maxColumn == null) { + return ReturnCode.INCLUDE; + } + + int cmpMax = Bytes.compareTo(buffer, qualifierOffset, qualifierLength, + this.maxColumn, 0, this.maxColumn.length); + + if (this.maxColumnInclusive && cmpMax <= 0 || + !this.maxColumnInclusive && cmpMax < 0) { + return ReturnCode.INCLUDE; + } + + return ReturnCode.NEXT_ROW; + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + Preconditions.checkArgument(filterArguments.size() == 4, + "Expected 4 but got: %s", filterArguments.size()); + byte [] minColumn = ParseFilter.removeQuotesFromByteArray(filterArguments.get(0)); + boolean minColumnInclusive = ParseFilter.convertByteArrayToBoolean(filterArguments.get(1)); + byte [] maxColumn = ParseFilter.removeQuotesFromByteArray(filterArguments.get(2)); + boolean maxColumnInclusive = ParseFilter.convertByteArrayToBoolean(filterArguments.get(3)); + + if (minColumn.length == 0) + minColumn = null; + if (maxColumn.length == 0) + maxColumn = null; + return new ColumnRangeFilter(minColumn, minColumnInclusive, + maxColumn, maxColumnInclusive); + } + + @Override + public void write(DataOutput out) throws IOException { + // need to write out a flag for null value separately. Otherwise, + // we will not be able to differentiate empty string and null + out.writeBoolean(this.minColumn == null); + Bytes.writeByteArray(out, this.minColumn); + out.writeBoolean(this.minColumnInclusive); + + out.writeBoolean(this.maxColumn == null); + Bytes.writeByteArray(out, this.maxColumn); + out.writeBoolean(this.maxColumnInclusive); + } + + @Override + public void readFields(DataInput in) throws IOException { + boolean isMinColumnNull = in.readBoolean(); + this.minColumn = Bytes.readByteArray(in); + + if (isMinColumnNull) { + this.minColumn = null; + } + + this.minColumnInclusive = in.readBoolean(); + + boolean isMaxColumnNull = in.readBoolean(); + this.maxColumn = Bytes.readByteArray(in); + if (isMaxColumnNull) { + this.maxColumn = null; + } + this.maxColumnInclusive = in.readBoolean(); + } + + + @Override + public KeyValue getNextKeyHint(KeyValue kv) { + return KeyValue.createFirstOnRow(kv.getBuffer(), kv.getRowOffset(), kv + .getRowLength(), kv.getBuffer(), kv.getFamilyOffset(), kv + .getFamilyLength(), this.minColumn, 0, this.minColumn == null ? 0 + : this.minColumn.length); + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + " " + + (this.minColumnInclusive ? "[" : "(") + Bytes.toStringBinary(this.minColumn) + + ", " + Bytes.toStringBinary(this.maxColumn) + + (this.maxColumnInclusive ? "]" : ")"); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java new file mode 100644 index 0000000..9089f4b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java @@ -0,0 +1,167 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.util.Bytes; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; + +import com.google.common.base.Preconditions; +/** + * This is a generic filter to be used to filter by comparison. It takes an + * operator (equal, greater, not equal, etc) and a byte [] comparator. + *

      + * To filter by row key, use {@link RowFilter}. + *

      + * To filter by column qualifier, use {@link QualifierFilter}. + *

      + * To filter by value, use {@link SingleColumnValueFilter}. + *

      + * These filters can be wrapped with {@link SkipFilter} and {@link WhileMatchFilter} + * to add more control. + *

      + * Multiple filters can be combined using {@link FilterList}. + */ +public abstract class CompareFilter extends FilterBase { + + /** Comparison operators. */ + public enum CompareOp { + /** less than */ + LESS, + /** less than or equal to */ + LESS_OR_EQUAL, + /** equals */ + EQUAL, + /** not equal */ + NOT_EQUAL, + /** greater than or equal to */ + GREATER_OR_EQUAL, + /** greater than */ + GREATER, + /** no operation */ + NO_OP, + } + + protected CompareOp compareOp; + protected WritableByteArrayComparable comparator; + + /** + * Writable constructor, do not use. + */ + public CompareFilter() { + } + + /** + * Constructor. + * @param compareOp the compare op for row matching + * @param comparator the comparator for row matching + */ + public CompareFilter(final CompareOp compareOp, + final WritableByteArrayComparable comparator) { + this.compareOp = compareOp; + this.comparator = comparator; + } + + /** + * @return operator + */ + public CompareOp getOperator() { + return compareOp; + } + + /** + * @return the comparator + */ + public WritableByteArrayComparable getComparator() { + return comparator; + } + + protected boolean doCompare(final CompareOp compareOp, + final WritableByteArrayComparable comparator, final byte [] data, + final int offset, final int length) { + if (compareOp == CompareOp.NO_OP) { + return true; + } + int compareResult = comparator.compareTo(data, offset, length); + switch (compareOp) { + case LESS: + return compareResult <= 0; + case LESS_OR_EQUAL: + return compareResult < 0; + case EQUAL: + return compareResult != 0; + case NOT_EQUAL: + return compareResult == 0; + case GREATER_OR_EQUAL: + return compareResult > 0; + case GREATER: + return compareResult >= 0; + default: + throw new RuntimeException("Unknown Compare op " + + compareOp.name()); + } + } + + public static ArrayList extractArguments(ArrayList filterArguments) { + Preconditions.checkArgument(filterArguments.size() == 2, + "Expected 2 but got: %s", filterArguments.size()); + CompareOp compareOp = ParseFilter.createCompareOp(filterArguments.get(0)); + WritableByteArrayComparable comparator = ParseFilter.createComparator( + ParseFilter.removeQuotesFromByteArray(filterArguments.get(1))); + + if (comparator instanceof RegexStringComparator || + comparator instanceof SubstringComparator) { + if (compareOp != CompareOp.EQUAL && + compareOp != CompareOp.NOT_EQUAL) { + throw new IllegalArgumentException ("A regexstring comparator and substring comparator" + + " can only be used with EQUAL and NOT_EQUAL"); + } + } + ArrayList arguments = new ArrayList(); + arguments.add(compareOp); + arguments.add(comparator); + return arguments; + } + + public void readFields(DataInput in) throws IOException { + compareOp = CompareOp.valueOf(in.readUTF()); + comparator = (WritableByteArrayComparable) + HbaseObjectWritable.readObject(in, null); + } + + public void write(DataOutput out) throws IOException { + out.writeUTF(compareOp.name()); + HbaseObjectWritable.writeObject(out, comparator, + WritableByteArrayComparable.class, null); + } + + @Override + public String toString() { + return String.format("%s (%s, %s)", + this.getClass().getSimpleName(), + this.compareOp.name(), + Bytes.toStringBinary(this.comparator.getValue())); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/DecimalComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/DecimalComparator.java new file mode 100644 index 0000000..3c6d2e4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/DecimalComparator.java @@ -0,0 +1,69 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.util.Bytes; + +public class DecimalComparator extends BinaryComparator { + + protected byte[] msb = new byte[1]; + protected byte[] temp; + + public DecimalComparator() { + + } + + /** + * Constructor + * @param value value + */ + public DecimalComparator(byte[] value) { + super(value); + byte b = value[0]; + msb[0] = (byte) ((b >> 7) & 1); + temp = new byte[value.length]; + System.arraycopy(value, 0, temp, 0, value.length); + } + + @Override + public int compareTo(byte[] value, int offset, int length) { + return super.compareTo(value, offset, length); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + msb = Bytes.readByteArray(in); + temp = Bytes.readByteArray(in); + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + Bytes.writeByteArray(out, msb); + Bytes.writeByteArray(out, temp); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java new file mode 100644 index 0000000..c50c821 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java @@ -0,0 +1,250 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.ArrayList; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + +import com.google.common.base.Preconditions; + +/** + * A filter for adding inter-column timestamp matching + * Only cells with a correspondingly timestamped entry in + * the target column will be retained + * Not compatible with Scan.setBatch as operations need + * full rows for correct filtering + */ +public class DependentColumnFilter extends CompareFilter { + + protected byte[] columnFamily; + protected byte[] columnQualifier; + protected boolean dropDependentColumn; + + protected Set stampSet = new HashSet(); + + /** + * Should only be used for writable + */ + public DependentColumnFilter() { + } + + /** + * Build a dependent column filter with value checking + * dependent column varies will be compared using the supplied + * compareOp and comparator, for usage of which + * refer to {@link CompareFilter} + * + * @param family dependent column family + * @param qualifier dependent column qualifier + * @param dropDependentColumn whether the column should be discarded after + * @param valueCompareOp comparison op + * @param valueComparator comparator + */ + public DependentColumnFilter(final byte [] family, final byte[] qualifier, + final boolean dropDependentColumn, final CompareOp valueCompareOp, + final WritableByteArrayComparable valueComparator) { + // set up the comparator + super(valueCompareOp, valueComparator); + this.columnFamily = family; + this.columnQualifier = qualifier; + this.dropDependentColumn = dropDependentColumn; + } + + /** + * Constructor for DependentColumn filter. + * Keyvalues where a keyvalue from target column + * with the same timestamp do not exist will be dropped. + * + * @param family name of target column family + * @param qualifier name of column qualifier + */ + public DependentColumnFilter(final byte [] family, final byte [] qualifier) { + this(family, qualifier, false); + } + + /** + * Constructor for DependentColumn filter. + * Keyvalues where a keyvalue from target column + * with the same timestamp do not exist will be dropped. + * + * @param family name of dependent column family + * @param qualifier name of dependent qualifier + * @param dropDependentColumn whether the dependent columns keyvalues should be discarded + */ + public DependentColumnFilter(final byte [] family, final byte [] qualifier, + final boolean dropDependentColumn) { + this(family, qualifier, dropDependentColumn, CompareOp.NO_OP, null); + } + + /** + * @return the column family + */ + public byte[] getFamily() { + return this.columnFamily; + } + + /** + * @return the column qualifier + */ + public byte[] getQualifier() { + return this.columnQualifier; + } + + /** + * @return true if we should drop the dependent column, false otherwise + */ + public boolean dropDependentColumn() { + return this.dropDependentColumn; + } + + public boolean getDropDependentColumn() { + return this.dropDependentColumn; + } + + @Override + public boolean filterAllRemaining() { + return false; + } + + @Override + public ReturnCode filterKeyValue(KeyValue v) { + // Check if the column and qualifier match + if (!v.matchingColumn(this.columnFamily, this.columnQualifier)) { + // include non-matches for the time being, they'll be discarded afterwards + return ReturnCode.INCLUDE; + } + // If it doesn't pass the op, skip it + if (comparator != null + && doCompare(compareOp, comparator, v.getBuffer(), v.getValueOffset(), + v.getValueLength())) + return ReturnCode.SKIP; + + stampSet.add(v.getTimestamp()); + if(dropDependentColumn) { + return ReturnCode.SKIP; + } + return ReturnCode.INCLUDE; + } + + @Override + public void filterRow(List kvs) { + Iterator it = kvs.iterator(); + KeyValue kv; + while(it.hasNext()) { + kv = it.next(); + if(!stampSet.contains(kv.getTimestamp())) { + it.remove(); + } + } + } + + @Override + public boolean hasFilterRow() { + return true; + } + + @Override + public boolean filterRow() { + return false; + } + + @Override + public boolean filterRowKey(byte[] buffer, int offset, int length) { + return false; + } + @Override + public void reset() { + stampSet.clear(); + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + Preconditions.checkArgument(filterArguments.size() == 2 || + filterArguments.size() == 3 || + filterArguments.size() == 5, + "Expected 2, 3 or 5 but got: %s", filterArguments.size()); + if (filterArguments.size() == 2) { + byte [] family = ParseFilter.removeQuotesFromByteArray(filterArguments.get(0)); + byte [] qualifier = ParseFilter.removeQuotesFromByteArray(filterArguments.get(1)); + return new DependentColumnFilter(family, qualifier); + + } else if (filterArguments.size() == 3) { + byte [] family = ParseFilter.removeQuotesFromByteArray(filterArguments.get(0)); + byte [] qualifier = ParseFilter.removeQuotesFromByteArray(filterArguments.get(1)); + boolean dropDependentColumn = ParseFilter.convertByteArrayToBoolean(filterArguments.get(2)); + return new DependentColumnFilter(family, qualifier, dropDependentColumn); + + } else if (filterArguments.size() == 5) { + byte [] family = ParseFilter.removeQuotesFromByteArray(filterArguments.get(0)); + byte [] qualifier = ParseFilter.removeQuotesFromByteArray(filterArguments.get(1)); + boolean dropDependentColumn = ParseFilter.convertByteArrayToBoolean(filterArguments.get(2)); + CompareOp compareOp = ParseFilter.createCompareOp(filterArguments.get(3)); + WritableByteArrayComparable comparator = ParseFilter.createComparator( + ParseFilter.removeQuotesFromByteArray(filterArguments.get(4))); + return new DependentColumnFilter(family, qualifier, dropDependentColumn, + compareOp, comparator); + } else { + throw new IllegalArgumentException("Expected 2, 3 or 5 but got: " + filterArguments.size()); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + this.columnFamily = Bytes.readByteArray(in); + if(this.columnFamily.length == 0) { + this.columnFamily = null; + } + + this.columnQualifier = Bytes.readByteArray(in); + if(this.columnQualifier.length == 0) { + this.columnQualifier = null; + } + + this.dropDependentColumn = in.readBoolean(); + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + Bytes.writeByteArray(out, this.columnFamily); + Bytes.writeByteArray(out, this.columnQualifier); + out.writeBoolean(this.dropDependentColumn); + } + + @Override + public String toString() { + return String.format("%s (%s, %s, %s, %s, %s)", + this.getClass().getSimpleName(), + Bytes.toStringBinary(this.columnFamily), + Bytes.toStringBinary(this.columnQualifier), + this.dropDependentColumn, + this.compareOp.name(), + Bytes.toStringBinary(this.comparator.getValue())); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/DoubleComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/DoubleComparator.java new file mode 100644 index 0000000..4b6b02d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/DoubleComparator.java @@ -0,0 +1,53 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; + +public class DoubleComparator extends DecimalComparator { + public DoubleComparator() { + } + + public DoubleComparator(byte[] value) { + super(value); + } + + @Override + public int compareTo(byte[] actualValue, int offset, int length) { + ByteArrayBuilder val = new ByteArrayBuilder(length); + val.put(actualValue, offset, length); + byte[] array = val.array(); + if (msb[0] == 0) { + value[0] ^= (1 << 7); + array[0] ^= (1 << 7); + } else { + for (int i = 0; i < 8; i++) { + value[i] ^= 0xff; + } + + for (int i = 0; i < 8; i++) { + array[i] ^= 0xff; + } + } + int compareTo = super.compareTo(array, 0, length); + System.arraycopy(temp, 0, value, 0, value.length); + return compareTo; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FamilyFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/FamilyFilter.java new file mode 100644 index 0000000..63ec44a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/FamilyFilter.java @@ -0,0 +1,76 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; + +import java.util.ArrayList; + +/** + * This filter is used to filter based on the column family. It takes an + * operator (equal, greater, not equal, etc) and a byte [] comparator for the + * column family portion of a key. + *

      + * This filter can be wrapped with {@link org.apache.hadoop.hbase.filter.WhileMatchFilter} and {@link org.apache.hadoop.hbase.filter.SkipFilter} + * to add more control. + *

      + * Multiple filters can be combined using {@link org.apache.hadoop.hbase.filter.FilterList}. + *

      + * If an already known column family is looked for, use {@link org.apache.hadoop.hbase.client.Get#addFamily(byte[])} + * directly rather than a filter. + */ +public class FamilyFilter extends CompareFilter { + /** + * Writable constructor, do not use. + */ + public FamilyFilter() { + } + + /** + * Constructor. + * + * @param familyCompareOp the compare op for column family matching + * @param familyComparator the comparator for column family matching + */ + public FamilyFilter(final CompareOp familyCompareOp, + final WritableByteArrayComparable familyComparator) { + super(familyCompareOp, familyComparator); + } + + @Override + public ReturnCode filterKeyValue(KeyValue v) { + int familyLength = v.getFamilyLength(); + if (familyLength > 0) { + if (doCompare(this.compareOp, this.comparator, v.getBuffer(), + v.getFamilyOffset(), familyLength)) { + return ReturnCode.SKIP; + } + } + return ReturnCode.INCLUDE; + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + ArrayList arguments = CompareFilter.extractArguments(filterArguments); + CompareOp compareOp = (CompareOp)arguments.get(0); + WritableByteArrayComparable comparator = (WritableByteArrayComparable)arguments.get(1); + return new FamilyFilter(compareOp, comparator); +} +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/Filter.java b/src/main/java/org/apache/hadoop/hbase/filter/Filter.java new file mode 100644 index 0000000..ac1079f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/Filter.java @@ -0,0 +1,170 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.io.Writable; + +import java.util.List; + +/** + * Interface for row and column filters directly applied within the regionserver. + * A filter can expect the following call sequence: + *

        + *
      • {@link #reset()}
        • + *
      • {@link #filterAllRemaining()} -> true indicates scan is over, false, keep going on.
        • + *
      • {@link #filterRowKey(byte[],int,int)} -> true to drop this row, + * if false, we will also call
        • + *
      • {@link #filterKeyValue(KeyValue)} -> true to drop this key/value
        • + *
      • {@link #filterRow(List)} -> allows directmodification of the final list to be submitted + *
      • {@link #filterRow()} -> last chance to drop entire row based on the sequence of + * filterValue() calls. Eg: filter a row if it doesn't contain a specified column. + *
        • + *
      + * + * Filter instances are created one per region/scan. This interface replaces + * the old RowFilterInterface. + * + * When implementing your own filters, consider inheriting {@link FilterBase} to help + * you reduce boilerplate. + * + * @see FilterBase + */ +public interface Filter extends Writable { + /** + * Reset the state of the filter between rows. + */ + public void reset(); + + /** + * Filters a row based on the row key. If this returns true, the entire + * row will be excluded. If false, each KeyValue in the row will be + * passed to {@link #filterKeyValue(KeyValue)} below. + * + * @param buffer buffer containing row key + * @param offset offset into buffer where row key starts + * @param length length of the row key + * @return true, remove entire row, false, include the row (maybe). + */ + public boolean filterRowKey(byte [] buffer, int offset, int length); + + /** + * If this returns true, the scan will terminate. + * + * @return true to end scan, false to continue. + */ + public boolean filterAllRemaining(); + + /** + * A way to filter based on the column family, column qualifier and/or the + * column value. Return code is described below. This allows filters to + * filter only certain number of columns, then terminate without matching ever + * column. + * + * If your filter returns ReturnCode.NEXT_ROW, it should return + * ReturnCode.NEXT_ROW until {@link #reset()} is called + * just in case the caller calls for the next row. + * + * @param v the KeyValue in question + * @return code as described below + * @see Filter.ReturnCode + */ + public ReturnCode filterKeyValue(final KeyValue v); + + /** + * Give the filter a chance to transform the passed KeyValue. + * If the KeyValue is changed a new KeyValue object must be returned. + * @see org.apache.hadoop.hbase.KeyValue#shallowCopy() + * + * The transformed KeyValue is what is eventually returned to the + * client. Most filters will return the passed KeyValue unchanged. + * @see org.apache.hadoop.hbase.filter.KeyOnlyFilter#transform(KeyValue) + * for an example of a transformation. + * + * @param v the KeyValue in question + * @return the changed KeyValue + */ + public KeyValue transform(final KeyValue v); + + /** + * Return codes for filterValue(). + */ + public enum ReturnCode { + /** + * Include the KeyValue + */ + INCLUDE, + /** + * Include the KeyValue and seek to the next column skipping older versions. + */ + INCLUDE_AND_NEXT_COL, + /** + * Skip this KeyValue + */ + SKIP, + /** + * Skip this column. Go to the next column in this row. + */ + NEXT_COL, + /** + * Done with columns, skip to next row. Note that filterRow() will + * still be called. + */ + NEXT_ROW, + /** + * Seek to next key which is given as hint by the filter. + */ + SEEK_NEXT_USING_HINT, +} + + /** + * Chance to alter the list of keyvalues to be submitted. + * Modifications to the list will carry on + * @param kvs the list of keyvalues to be filtered + */ + public void filterRow(List kvs); + + /** + * @return True if this filter actively uses filterRow(List). + * Primarily used to check for conflicts with scans(such as scans + * that do not read a full row at a time) + */ + public boolean hasFilterRow(); + + /** + * Last chance to veto row based on previous {@link #filterKeyValue(KeyValue)} + * calls. The filter needs to retain state then return a particular value for + * this call if they wish to exclude a row if a certain column is missing + * (for example). + * @return true to exclude row, false to include row. + */ + public boolean filterRow(); + + /** + * If the filter returns the match code SEEK_NEXT_USING_HINT, then + * it should also tell which is the next key it must seek to. + * After receiving the match code SEEK_NEXT_USING_HINT, the QueryMatcher would + * call this function to find out which key it must next seek to. + * @return KeyValue which must be next seeked. return null if the filter is + * not sure which key to seek to next. + */ + public KeyValue getNextKeyHint(final KeyValue currentKV); +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java new file mode 100644 index 0000000..291b5dd --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java @@ -0,0 +1,175 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; + +import java.util.List; +import java.util.ArrayList; + +/** + * Abstract base class to help you implement new Filters. Common "ignore" or NOOP type + * methods can go here, helping to reduce boiler plate in an ever-expanding filter + * library. + * + * If you could instantiate FilterBase, it would end up being a "null" filter - + * that is one that never filters anything. + */ +public abstract class FilterBase implements Filter { + + /** + * Filters that are purely stateless and do nothing in their reset() methods can inherit + * this null/empty implementation. + * + * @inheritDoc + */ + @Override + public void reset() { + } + + /** + * Filters that do not filter by row key can inherit this implementation that + * never filters anything. (ie: returns false). + * + * @inheritDoc + */ + @Override + public boolean filterRowKey(byte [] buffer, int offset, int length) { + return false; + } + + /** + * Filters that never filter all remaining can inherit this implementation that + * never stops the filter early. + * + * @inheritDoc + */ + @Override + public boolean filterAllRemaining() { + return false; + } + + /** + * Filters that dont filter by key value can inherit this implementation that + * includes all KeyValues. + * + * @inheritDoc + */ + @Override + public ReturnCode filterKeyValue(KeyValue ignored) { + return ReturnCode.INCLUDE; + } + + /** + * By default no transformation takes place + * + * @inheritDoc + */ + @Override + public KeyValue transform(KeyValue v) { + return v; + } + + /** + * Filters that never filter by modifying the returned List of KeyValues can + * inherit this implementation that does nothing. + * + * @inheritDoc + */ + @Override + public void filterRow(List ignored) { + } + + /** + * Fitlers that never filter by modifying the returned List of KeyValues can + * inherit this implementation that does nothing. + * + * @inheritDoc + */ + @Override + public boolean hasFilterRow() { + return false; + } + + /** + * Filters that never filter by rows based on previously gathered state from + * {@link #filterKeyValue(KeyValue)} can inherit this implementation that + * never filters a row. + * + * @inheritDoc + */ + @Override + public boolean filterRow() { + return false; + } + + /** + * Filters that are not sure which key must be next seeked to, can inherit + * this implementation that, by default, returns a null KeyValue. + * + * @inheritDoc + */ + public KeyValue getNextKeyHint(KeyValue currentKV) { + return null; + } + + /** + * Check that given column family is essential for filter to check row. Most + * filters always return true here. But some could have more sophisticated + * logic which could significantly reduce scanning process by not even + * touching columns until we are 100% sure that it's data is needed in result. + * + * By default, we require all scan's column families to be present. Our + * subclasses may be more precise. + */ + public boolean isFamilyEssential(byte[] name) { + return true; + } + + /** + * Check that given column family is essential for filter to check row. + * This accommodates Filter implementation which didn't have this capability + * + * @param filter + * @param name column family name + * @return whether column family is essential + */ + public static boolean isFamilyEssential(Filter filter, byte[] name) { + return (!(filter instanceof FilterBase) || ((FilterBase) filter).isFamilyEssential(name)) && + (!(filter instanceof FilterList) || ((FilterList) filter).isFamilyEssential(name)); + } + + /** + * Given the filter's arguments it constructs the filter + *

      + * @param filterArguments the filter's arguments + * @return constructed filter object + */ + public static Filter createFilterFromArguments(ArrayList filterArguments) { + throw new IllegalArgumentException("This method has not been implemented"); + } + + /** + * Return filter's info for debugging and logging purpose. + */ + public String toString() { + return this.getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java new file mode 100644 index 0000000..f65065c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java @@ -0,0 +1,343 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.io.Writable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Implementation of {@link Filter} that represents an ordered List of Filters + * which will be evaluated with a specified boolean operator {@link Operator#MUST_PASS_ALL} + * (!AND) or {@link Operator#MUST_PASS_ONE} (!OR). + * Since you can use Filter Lists as children of Filter Lists, you can create a + * hierarchy of filters to be evaluated. + * Defaults to {@link Operator#MUST_PASS_ALL}. + *

      TODO: Fix creation of Configuration on serialization and deserialization. + */ +public class FilterList implements Filter { + /** set operator */ + public static enum Operator { + /** !AND */ + MUST_PASS_ALL, + /** !OR */ + MUST_PASS_ONE + } + + private static final Configuration conf = HBaseConfiguration.create(); + private static final int MAX_LOG_FILTERS = 5; + private Operator operator = Operator.MUST_PASS_ALL; + private List filters = new ArrayList(); + + /** + * Default constructor, filters nothing. Required though for RPC + * deserialization. + */ + public FilterList() { + super(); + } + + /** + * Constructor that takes a set of {@link Filter}s. The default operator + * MUST_PASS_ALL is assumed. + * + * @param rowFilters list of filters + */ + public FilterList(final List rowFilters) { + this.filters = rowFilters; + } + + /** + * Constructor that takes a var arg number of {@link Filter}s. The fefault operator + * MUST_PASS_ALL is assumed. + * @param rowFilters + */ + public FilterList(final Filter... rowFilters) { + this.filters = Arrays.asList(rowFilters); + } + + /** + * Constructor that takes an operator. + * + * @param operator Operator to process filter set with. + */ + public FilterList(final Operator operator) { + this.operator = operator; + } + + /** + * Constructor that takes a set of {@link Filter}s and an operator. + * + * @param operator Operator to process filter set with. + * @param rowFilters Set of row filters. + */ + public FilterList(final Operator operator, final List rowFilters) { + this.filters = rowFilters; + this.operator = operator; + } + + /** + * Constructor that takes a var arg number of {@link Filter}s and an operator. + * + * @param operator Operator to process filter set with. + * @param rowFilters Filters to use + */ + public FilterList(final Operator operator, final Filter... rowFilters) { + this.filters = Arrays.asList(rowFilters); + this.operator = operator; + } + + /** + * Get the operator. + * + * @return operator + */ + public Operator getOperator() { + return operator; + } + + /** + * Get the filters. + * + * @return filters + */ + public List getFilters() { + return filters; + } + + /** + * Add a filter. + * + * @param filter another filter + */ + public void addFilter(Filter filter) { + this.filters.add(filter); + } + + @Override + public void reset() { + for (Filter filter : filters) { + filter.reset(); + } + } + + @Override + public boolean filterRowKey(byte[] rowKey, int offset, int length) { + for (Filter filter : filters) { + if (this.operator == Operator.MUST_PASS_ALL) { + if (filter.filterAllRemaining() || + filter.filterRowKey(rowKey, offset, length)) { + return true; + } + } else if (this.operator == Operator.MUST_PASS_ONE) { + if (!filter.filterAllRemaining() && + !filter.filterRowKey(rowKey, offset, length)) { + return false; + } + } + } + return this.operator == Operator.MUST_PASS_ONE; + } + + @Override + public boolean filterAllRemaining() { + for (Filter filter : filters) { + if (filter.filterAllRemaining()) { + if (operator == Operator.MUST_PASS_ALL) { + return true; + } + } else { + if (operator == Operator.MUST_PASS_ONE) { + return false; + } + } + } + return operator == Operator.MUST_PASS_ONE; + } + + @Override + public KeyValue transform(KeyValue v) { + KeyValue current = v; + for (Filter filter : filters) { + current = filter.transform(current); + } + return current; + } + + @Override + public ReturnCode filterKeyValue(KeyValue v) { + ReturnCode rc = operator == Operator.MUST_PASS_ONE? + ReturnCode.SKIP: ReturnCode.INCLUDE; + for (Filter filter : filters) { + if (operator == Operator.MUST_PASS_ALL) { + if (filter.filterAllRemaining()) { + return ReturnCode.NEXT_ROW; + } + ReturnCode code = filter.filterKeyValue(v); + switch (code) { + // Override INCLUDE and continue to evaluate. + case INCLUDE_AND_NEXT_COL: + rc = ReturnCode.INCLUDE_AND_NEXT_COL; + case INCLUDE: + continue; + default: + return code; + } + } else if (operator == Operator.MUST_PASS_ONE) { + if (filter.filterAllRemaining()) { + continue; + } + + switch (filter.filterKeyValue(v)) { + case INCLUDE: + if (rc != ReturnCode.INCLUDE_AND_NEXT_COL) { + rc = ReturnCode.INCLUDE; + } + break; + case INCLUDE_AND_NEXT_COL: + rc = ReturnCode.INCLUDE_AND_NEXT_COL; + // must continue here to evaluate all filters + case NEXT_ROW: + case SKIP: + // continue; + } + } + } + return rc; + } + + @Override + public void filterRow(List kvs) { + for (Filter filter : filters) { + filter.filterRow(kvs); + } + } + + @Override + public boolean hasFilterRow() { + for (Filter filter : filters) { + if(filter.hasFilterRow()) { + return true; + } + } + return false; + } + + @Override + public boolean filterRow() { + for (Filter filter : filters) { + if (operator == Operator.MUST_PASS_ALL) { + if (filter.filterRow()) { + return true; + } + } else if (operator == Operator.MUST_PASS_ONE) { + if (!filter.filterRow()) { + return false; + } + } + } + return operator == Operator.MUST_PASS_ONE; + } + + public void readFields(final DataInput in) throws IOException { + byte opByte = in.readByte(); + operator = Operator.values()[opByte]; + int size = in.readInt(); + if (size > 0) { + filters = new ArrayList(size); + for (int i = 0; i < size; i++) { + Filter filter = (Filter)HbaseObjectWritable.readObject(in, conf); + filters.add(filter); + } + } + } + + public void write(final DataOutput out) throws IOException { + out.writeByte(operator.ordinal()); + out.writeInt(filters.size()); + for (Filter filter : filters) { + HbaseObjectWritable.writeObject(out, filter, Writable.class, conf); + } + } + + @Override + public KeyValue getNextKeyHint(KeyValue currentKV) { + KeyValue keyHint = null; + for (Filter filter : filters) { + KeyValue curKeyHint = filter.getNextKeyHint(currentKV); + if (curKeyHint == null && operator == Operator.MUST_PASS_ONE) { + // If we ever don't have a hint and this is must-pass-one, then no hint + return null; + } + if (curKeyHint != null) { + // If this is the first hint we find, set it + if (keyHint == null) { + keyHint = curKeyHint; + continue; + } + // There is an existing hint + if (operator == Operator.MUST_PASS_ALL && + KeyValue.COMPARATOR.compare(keyHint, curKeyHint) < 0) { + // If all conditions must pass, we can keep the max hint + keyHint = curKeyHint; + } else if (operator == Operator.MUST_PASS_ONE && + KeyValue.COMPARATOR.compare(keyHint, curKeyHint) > 0) { + // If any condition can pass, we need to keep the min hint + keyHint = curKeyHint; + } + } + } + return keyHint; + } + + public boolean isFamilyEssential(byte[] name) { + for (Filter filter : filters) { + if (FilterBase.isFamilyEssential(filter, name)) { + return true; + } + } + return false; + } + + @Override + public String toString() { + return toString(MAX_LOG_FILTERS); + } + + protected String toString(int maxFilters) { + int endIndex = this.filters.size() < maxFilters + ? this.filters.size() : maxFilters; + return String.format("%s %s (%d/%d): %s", + this.getClass().getSimpleName(), + this.operator == Operator.MUST_PASS_ALL ? "AND" : "OR", + endIndex, + this.filters.size(), + this.filters.subList(0, endIndex).toString()); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.java new file mode 100644 index 0000000..7a068b4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.java @@ -0,0 +1,64 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; + +import java.io.DataOutput; +import java.io.IOException; +import java.io.DataInput; +import java.util.List; +import java.util.ArrayList; + +import com.google.common.base.Preconditions; + +/** + * A filter that will only return the first KV from each row. + *

      + * This filter can be used to more efficiently perform row count operations. + */ +public class FirstKeyOnlyFilter extends FilterBase { + private boolean foundKV = false; + + public FirstKeyOnlyFilter() { + } + + public void reset() { + foundKV = false; + } + + public ReturnCode filterKeyValue(KeyValue v) { + if(foundKV) return ReturnCode.NEXT_ROW; + foundKV = true; + return ReturnCode.INCLUDE; + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + Preconditions.checkArgument(filterArguments.size() == 0, + "Expected 0 but got: %s", filterArguments.size()); + return new FirstKeyOnlyFilter(); + } + + public void write(DataOutput out) throws IOException { + } + + public void readFields(DataInput in) throws IOException { + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FloatComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/FloatComparator.java new file mode 100644 index 0000000..36b6601 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/FloatComparator.java @@ -0,0 +1,58 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; + +public class FloatComparator extends DecimalComparator { + + public FloatComparator() { + + } + + public FloatComparator(byte[] value) { + super(value); + } + + @Override + public int compareTo(byte[] actualValue, int offset, int length) { + ByteArrayBuilder val = new ByteArrayBuilder(length); + val.put(actualValue, offset, length); + byte[] array = val.array(); + if (msb[0] == 0) { + value[0] ^= (1 << 7); + array[0] ^= (1 << 7); + } else { + value[0] ^= 0xff; + value[1] ^= 0xff; + value[2] ^= 0xff; + value[3] ^= 0xff; + + array[0] ^= 0xff; + array[1] ^= 0xff; + array[2] ^= 0xff; + array[3] ^= 0xff; + } + int compareTo = super.compareTo(array, 0, length); + System.arraycopy(temp, 0, value, 0, value.length); + return compareTo; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java new file mode 100644 index 0000000..388e091 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java @@ -0,0 +1,289 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +/** + * Filters data based on fuzzy row key. Performs fast-forwards during scanning. + * It takes pairs (row key, fuzzy info) to match row keys. Where fuzzy info is + * a byte array with 0 or 1 as its values: + *

        + *
      • + * 0 - means that this byte in provided row key is fixed, i.e. row key's byte at same position + * must match + *
        • + *
      • + * 1 - means that this byte in provided row key is NOT fixed, i.e. row key's byte at this + * position can be different from the one in provided row key + *
        • + *
      + * + * + * Example: + * Let's assume row key format is userId_actionId_year_month. Length of userId is fixed + * and is 4, length of actionId is 2 and year and month are 4 and 2 bytes long respectively. + * + * Let's assume that we need to fetch all users that performed certain action (encoded as "99") + * in Jan of any year. Then the pair (row key, fuzzy info) would be the following: + * row key = "????_99_????_01" (one can use any value instead of "?") + * fuzzy info = "\x01\x01\x01\x01\x00\x00\x00\x00\x01\x01\x01\x01\x00\x00\x00" + * + * I.e. fuzzy info tells the matching mask is "????_99_????_01", where at ? can be any value. + * + */ +public class FuzzyRowFilter extends FilterBase { + private List> fuzzyKeysData; + private boolean done = false; + + /** + * Used internally for reflection, do NOT use it directly + */ + public FuzzyRowFilter() { + } + + public FuzzyRowFilter(List> fuzzyKeysData) { + this.fuzzyKeysData = fuzzyKeysData; + } + + // TODO: possible improvement: save which fuzzy row key to use when providing a hint + @Override + public ReturnCode filterKeyValue(KeyValue kv) { + byte[] rowKey = kv.getRow(); + // assigning "worst" result first and looking for better options + SatisfiesCode bestOption = SatisfiesCode.NO_NEXT; + for (Pair fuzzyData : fuzzyKeysData) { + SatisfiesCode satisfiesCode = + satisfies(rowKey, fuzzyData.getFirst(), fuzzyData.getSecond()); + if (satisfiesCode == SatisfiesCode.YES) { + return ReturnCode.INCLUDE; + } + + if (satisfiesCode == SatisfiesCode.NEXT_EXISTS) { + bestOption = SatisfiesCode.NEXT_EXISTS; + } + } + + if (bestOption == SatisfiesCode.NEXT_EXISTS) { + return ReturnCode.SEEK_NEXT_USING_HINT; + } + + // the only unhandled SatisfiesCode is NO_NEXT, i.e. we are done + done = true; + return ReturnCode.NEXT_ROW; + } + + @Override + public KeyValue getNextKeyHint(KeyValue currentKV) { + byte[] rowKey = currentKV.getRow(); + byte[] nextRowKey = null; + // Searching for the "smallest" row key that satisfies at least one fuzzy row key + for (Pair fuzzyData : fuzzyKeysData) { + byte[] nextRowKeyCandidate = getNextForFuzzyRule(rowKey, + fuzzyData.getFirst(), fuzzyData.getSecond()); + if (nextRowKeyCandidate == null) { + continue; + } + if (nextRowKey == null || Bytes.compareTo(nextRowKeyCandidate, nextRowKey) < 0) { + nextRowKey = nextRowKeyCandidate; + } + } + + if (nextRowKey == null) { + // SHOULD NEVER happen + // TODO: is there a better way than throw exception? (stop the scanner?) + throw new IllegalStateException("No next row key that satisfies fuzzy exists when" + + " getNextKeyHint() is invoked." + + " Filter: " + this.toString() + + " currentKV: " + currentKV.toString()); + } + + return KeyValue.createFirstOnRow(nextRowKey); + } + + @Override + public boolean filterAllRemaining() { + return done; + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + dataOutput.writeInt(this.fuzzyKeysData.size()); + for (Pair fuzzyData : fuzzyKeysData) { + Bytes.writeByteArray(dataOutput, fuzzyData.getFirst()); + Bytes.writeByteArray(dataOutput, fuzzyData.getSecond()); + } + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + int count = dataInput.readInt(); + this.fuzzyKeysData = new ArrayList>(count); + for (int i = 0; i < count; i++) { + byte[] keyBytes = Bytes.readByteArray(dataInput); + byte[] keyMeta = Bytes.readByteArray(dataInput); + this.fuzzyKeysData.add(new Pair(keyBytes, keyMeta)); + } + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("FuzzyRowFilter"); + sb.append("{fuzzyKeysData="); + for (Pair fuzzyData : fuzzyKeysData) { + sb.append('{').append(Bytes.toStringBinary(fuzzyData.getFirst())).append(":"); + sb.append(Bytes.toStringBinary(fuzzyData.getSecond())).append('}'); + } + sb.append("}, "); + return sb.toString(); + } + + // Utility methods + + static enum SatisfiesCode { + // row satisfies fuzzy rule + YES, + // row doesn't satisfy fuzzy rule, but there's possible greater row that does + NEXT_EXISTS, + // row doesn't satisfy fuzzy rule and there's no greater row that does + NO_NEXT + } + + static SatisfiesCode satisfies(byte[] row, + byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) { + return satisfies(row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta); + } + + private static SatisfiesCode satisfies(byte[] row, int offset, int length, + byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) { + if (row == null) { + // do nothing, let scan to proceed + return SatisfiesCode.YES; + } + + boolean nextRowKeyCandidateExists = false; + + for (int i = 0; i < fuzzyKeyMeta.length && i < length; i++) { + // First, checking if this position is fixed and not equals the given one + boolean byteAtPositionFixed = fuzzyKeyMeta[i] == 0; + boolean fixedByteIncorrect = byteAtPositionFixed && fuzzyKeyBytes[i] != row[i + offset]; + if (fixedByteIncorrect) { + // in this case there's another row that satisfies fuzzy rule and bigger than this row + if (nextRowKeyCandidateExists) { + return SatisfiesCode.NEXT_EXISTS; + } + + // If this row byte is less than fixed then there's a byte array bigger than + // this row and which satisfies the fuzzy rule. Otherwise there's no such byte array: + // this row is simply bigger than any byte array that satisfies the fuzzy rule + boolean rowByteLessThanFixed = (row[i + offset] & 0xFF) < (fuzzyKeyBytes[i] & 0xFF); + return rowByteLessThanFixed ? SatisfiesCode.NEXT_EXISTS : SatisfiesCode.NO_NEXT; + } + + // Second, checking if this position is not fixed and byte value is not the biggest. In this + // case there's a byte array bigger than this row and which satisfies the fuzzy rule. To get + // bigger byte array that satisfies the rule we need to just increase this byte + // (see the code of getNextForFuzzyRule below) by one. + // Note: if non-fixed byte is already at biggest value, this doesn't allow us to say there's + // bigger one that satisfies the rule as it can't be increased. + if (fuzzyKeyMeta[i] == 1 && !isMax(fuzzyKeyBytes[i])) { + nextRowKeyCandidateExists = true; + } + } + + return SatisfiesCode.YES; + } + + private static boolean isMax(byte fuzzyKeyByte) { + return (fuzzyKeyByte & 0xFF) == 255; + } + + static byte[] getNextForFuzzyRule(byte[] row, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) { + return getNextForFuzzyRule(row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta); + } + + /** + * @return greater byte array than given (row) which satisfies the fuzzy rule if it exists, + * null otherwise + */ + private static byte[] getNextForFuzzyRule(byte[] row, int offset, int length, + byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) { + // To find out the next "smallest" byte array that satisfies fuzzy rule and "greater" than + // the given one we do the following: + // 1. setting values on all "fixed" positions to the values from fuzzyKeyBytes + // 2. if during the first step given row did not increase, then we increase the value at + // the first "non-fixed" position (where it is not maximum already) + + // It is easier to perform this by using fuzzyKeyBytes copy and setting "non-fixed" position + // values than otherwise. + byte[] result = Arrays.copyOf(fuzzyKeyBytes, + length > fuzzyKeyBytes.length ? length : fuzzyKeyBytes.length); + int toInc = -1; + + boolean increased = false; + for (int i = 0; i < result.length; i++) { + if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 1) { + result[i] = row[offset + i]; + if (!isMax(row[i])) { + // this is "non-fixed" position and is not at max value, hence we can increase it + toInc = i; + } + } else if (i < fuzzyKeyMeta.length && fuzzyKeyMeta[i] == 0) { + if ((row[i + offset] & 0xFF) < (fuzzyKeyBytes[i] & 0xFF)) { + // if setting value for any fixed position increased the original array, + // we are OK + increased = true; + break; + } + if ((row[i + offset] & 0xFF) > (fuzzyKeyBytes[i] & 0xFF)) { + // if setting value for any fixed position makes array "smaller", then just stop: + // in case we found some non-fixed position to increase we will do it, otherwise + // there's no "next" row key that satisfies fuzzy rule and "greater" than given row + break; + } + } + } + + if (!increased) { + if (toInc < 0) { + return null; + } + result[toInc]++; + + // Setting all "non-fixed" positions to zeroes to the right of the one we increased so + // that found "next" row key is the smallest possible + for (int i = toInc + 1; i < result.length; i++) { + if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 1) { + result[i] = 0; + } + } + } + + return result; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java new file mode 100644 index 0000000..afa31c5 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java @@ -0,0 +1,97 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; +import java.util.ArrayList; + +import com.google.common.base.Preconditions; + +/** + * A Filter that stops after the given row. There is no "RowStopFilter" because + * the Scan spec allows you to specify a stop row. + * + * Use this filter to include the stop row, eg: [A,Z]. + */ +public class InclusiveStopFilter extends FilterBase { + private byte [] stopRowKey; + private boolean done = false; + + public InclusiveStopFilter() { + super(); + } + + public InclusiveStopFilter(final byte [] stopRowKey) { + this.stopRowKey = stopRowKey; + } + + public byte[] getStopRowKey() { + return this.stopRowKey; + } + + public boolean filterRowKey(byte[] buffer, int offset, int length) { + if (buffer == null) { + //noinspection RedundantIfStatement + if (this.stopRowKey == null) { + return true; //filter... + } + return false; + } + // if stopRowKey is <= buffer, then true, filter row. + int cmp = Bytes.compareTo(stopRowKey, 0, stopRowKey.length, + buffer, offset, length); + + if(cmp < 0) { + done = true; + } + return done; + } + + public boolean filterAllRemaining() { + return done; + } + + public static Filter createFilterFromArguments (ArrayList filterArguments) { + Preconditions.checkArgument(filterArguments.size() == 1, + "Expected 1 but got: %s", filterArguments.size()); + byte [] stopRowKey = ParseFilter.removeQuotesFromByteArray(filterArguments.get(0)); + return new InclusiveStopFilter(stopRowKey); + } + + public void write(DataOutput out) throws IOException { + Bytes.writeByteArray(out, this.stopRowKey); + } + + public void readFields(DataInput in) throws IOException { + this.stopRowKey = Bytes.readByteArray(in); + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + " " + Bytes.toStringBinary(this.stopRowKey); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/IncompatibleFilterException.java b/src/main/java/org/apache/hadoop/hbase/filter/IncompatibleFilterException.java new file mode 100644 index 0000000..75edf19 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/IncompatibleFilterException.java @@ -0,0 +1,40 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +/** + * Used to indicate a filter incompatibility + */ +public class IncompatibleFilterException extends RuntimeException { + private static final long serialVersionUID = 3236763276623198231L; + +/** constructor */ + public IncompatibleFilterException() { + super(); + } + + /** + * constructor + * @param s message + */ + public IncompatibleFilterException(String s) { + super(s); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/IntComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/IntComparator.java new file mode 100644 index 0000000..79034b8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/IntComparator.java @@ -0,0 +1,46 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.index.util.ByteArrayBuilder; + +public class IntComparator extends DecimalComparator { + + public IntComparator() { + + } + + public IntComparator(byte[] value) { + super(value); + } + + @Override + public int compareTo(byte[] actualValue, int offset, int length) { + ByteArrayBuilder val = new ByteArrayBuilder(length); + val.put(actualValue, offset, length); + value[0] ^= (1 << 7); + byte[] array = val.array(); + array[0] ^= (1 << 7); + int compareTo = super.compareTo(array, 0, length); + System.arraycopy(temp, 0, value, 0, value.length); + return compareTo; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/InvalidRowFilterException.java b/src/main/java/org/apache/hadoop/hbase/filter/InvalidRowFilterException.java new file mode 100644 index 0000000..14b8e31 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/InvalidRowFilterException.java @@ -0,0 +1,41 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +/** + * Used to indicate an invalid RowFilter. + */ +public class InvalidRowFilterException extends RuntimeException { + private static final long serialVersionUID = 2667894046345657865L; + + + /** constructor */ + public InvalidRowFilterException() { + super(); + } + + /** + * constructor + * @param s message + */ + public InvalidRowFilterException(String s) { + super(s); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java new file mode 100644 index 0000000..5cf659c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java @@ -0,0 +1,67 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + +import java.util.ArrayList; + +import com.google.common.base.Preconditions; + +/** + * A filter that will only return the key component of each KV (the value will + * be rewritten as empty). + *

      + * This filter can be used to grab all of the keys without having to also grab + * the values. + */ +public class KeyOnlyFilter extends FilterBase { + + boolean lenAsVal; + public KeyOnlyFilter() { this(false); } + public KeyOnlyFilter(boolean lenAsVal) { this.lenAsVal = lenAsVal; } + + @Override + public KeyValue transform(KeyValue kv) { + return kv.createKeyOnly(this.lenAsVal); + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + Preconditions.checkArgument((filterArguments.size() == 0 || filterArguments.size() == 1), + "Expected: 0 or 1 but got: %s", filterArguments.size()); + KeyOnlyFilter filter = new KeyOnlyFilter(); + if (filterArguments.size() == 1) { + filter.lenAsVal = ParseFilter.convertByteArrayToBoolean(filterArguments.get(0)); + } + return filter; + } + + public void write(DataOutput out) throws IOException { + out.writeBoolean(this.lenAsVal); + } + + public void readFields(DataInput in) throws IOException { + this.lenAsVal = in.readBoolean(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/MultipleColumnPrefixFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/MultipleColumnPrefixFilter.java new file mode 100644 index 0000000..11fe6e3 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/MultipleColumnPrefixFilter.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + +import java.io.DataOutput; +import java.io.IOException; +import java.io.DataInput; +import java.util.Arrays; +import java.util.Comparator; +import java.util.TreeSet; +import java.util.ArrayList; + +/** + * This filter is used for selecting only those keys with columns that matches + * a particular prefix. For example, if prefix is 'an', it will pass keys will + * columns like 'and', 'anti' but not keys with columns like 'ball', 'act'. + */ +public class MultipleColumnPrefixFilter extends FilterBase { + protected byte [] hint = null; + protected TreeSet sortedPrefixes = createTreeSet(); + private final static int MAX_LOG_PREFIXES = 5; + + public MultipleColumnPrefixFilter() { + super(); + } + + public MultipleColumnPrefixFilter(final byte [][] prefixes) { + if (prefixes != null) { + for (int i = 0; i < prefixes.length; i++) { + if (!sortedPrefixes.add(prefixes[i])) + throw new IllegalArgumentException ("prefixes must be distinct"); + } + } + } + + public byte [][] getPrefix() { + int count = 0; + byte [][] temp = new byte [sortedPrefixes.size()][]; + for (byte [] prefixes : sortedPrefixes) { + temp [count++] = prefixes; + } + return temp; + } + + @Override + public ReturnCode filterKeyValue(KeyValue kv) { + if (sortedPrefixes.size() == 0 || kv.getBuffer() == null) { + return ReturnCode.INCLUDE; + } else { + return filterColumn(kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength()); + } + } + + public ReturnCode filterColumn(byte[] buffer, int qualifierOffset, int qualifierLength) { + byte [] qualifier = Arrays.copyOfRange(buffer, qualifierOffset, + qualifierLength + qualifierOffset); + TreeSet lesserOrEqualPrefixes = + (TreeSet) sortedPrefixes.headSet(qualifier, true); + + if (lesserOrEqualPrefixes.size() != 0) { + byte [] largestPrefixSmallerThanQualifier = lesserOrEqualPrefixes.last(); + + if (Bytes.startsWith(qualifier, largestPrefixSmallerThanQualifier)) { + return ReturnCode.INCLUDE; + } + + if (lesserOrEqualPrefixes.size() == sortedPrefixes.size()) { + return ReturnCode.NEXT_ROW; + } else { + hint = sortedPrefixes.higher(largestPrefixSmallerThanQualifier); + return ReturnCode.SEEK_NEXT_USING_HINT; + } + } else { + hint = sortedPrefixes.first(); + return ReturnCode.SEEK_NEXT_USING_HINT; + } + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + byte [][] prefixes = new byte [filterArguments.size()][]; + for (int i = 0 ; i < filterArguments.size(); i++) { + byte [] columnPrefix = ParseFilter.removeQuotesFromByteArray(filterArguments.get(i)); + prefixes[i] = columnPrefix; + } + return new MultipleColumnPrefixFilter(prefixes); + } + + public void write(DataOutput out) throws IOException { + out.writeInt(sortedPrefixes.size()); + for (byte [] element : sortedPrefixes) { + Bytes.writeByteArray(out, element); + } + } + + public void readFields(DataInput in) throws IOException { + int x = in.readInt(); + this.sortedPrefixes = createTreeSet(); + for (int j = 0; j < x; j++) { + sortedPrefixes.add(Bytes.readByteArray(in)); + } + } + + public KeyValue getNextKeyHint(KeyValue kv) { + return KeyValue.createFirstOnRow( + kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(), + kv.getFamilyOffset(), kv.getFamilyLength(), hint, 0, hint.length); + } + + public TreeSet createTreeSet() { + return new TreeSet(new Comparator() { + @Override + public int compare (Object o1, Object o2) { + if (o1 == null || o2 == null) + throw new IllegalArgumentException ("prefixes can't be null"); + + byte [] b1 = (byte []) o1; + byte [] b2 = (byte []) o2; + return Bytes.compareTo (b1, 0, b1.length, b2, 0, b2.length); + } + }); + } + + @Override + public String toString() { + return toString(MAX_LOG_PREFIXES); + } + + protected String toString(int maxPrefixes) { + StringBuilder prefixes = new StringBuilder(); + + int count = 0; + for (byte[] ba : this.sortedPrefixes) { + if (count >= maxPrefixes) { + break; + } + ++count; + prefixes.append(Bytes.toStringBinary(ba)); + if (count < this.sortedPrefixes.size() && count < maxPrefixes) { + prefixes.append(", "); + } + } + + return String.format("%s (%d/%d): [%s]", this.getClass().getSimpleName(), + count, this.sortedPrefixes.size(), prefixes.toString()); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/NullComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/NullComparator.java new file mode 100644 index 0000000..45eb477 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/NullComparator.java @@ -0,0 +1,43 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +/** + * A binary comparator which lexicographically compares against the specified + * byte array using {@link org.apache.hadoop.hbase.util.Bytes#compareTo(byte[], byte[])}. + */ +public class NullComparator extends WritableByteArrayComparable { + + /** Nullary constructor for Writable, do not use */ + public NullComparator() { + value = new byte[0]; + } + + @Override + public int compareTo(byte[] value) { + return value != null ? 1 : 0; + } + + @Override + public int compareTo(byte[] value, int offset, int length) { + throw new UnsupportedOperationException(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java new file mode 100644 index 0000000..b00205a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java @@ -0,0 +1,96 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; +import java.util.ArrayList; + +import com.google.common.base.Preconditions; +/** + * Implementation of Filter interface that limits results to a specific page + * size. It terminates scanning once the number of filter-passed rows is > + * the given page size. + *

      + * Note that this filter cannot guarantee that the number of results returned + * to a client are <= page size. This is because the filter is applied + * separately on different region servers. It does however optimize the scan of + * individual HRegions by making sure that the page size is never exceeded + * locally. + */ +public class PageFilter extends FilterBase { + private long pageSize = Long.MAX_VALUE; + private int rowsAccepted = 0; + + /** + * Default constructor, filters nothing. Required though for RPC + * deserialization. + */ + public PageFilter() { + super(); + } + + /** + * Constructor that takes a maximum page size. + * + * @param pageSize Maximum result size. + */ + public PageFilter(final long pageSize) { + Preconditions.checkArgument(pageSize >= 0, "must be positive %s", pageSize); + this.pageSize = pageSize; + } + + public long getPageSize() { + return pageSize; + } + + public boolean filterAllRemaining() { + return this.rowsAccepted >= this.pageSize; + } + + public boolean filterRow() { + this.rowsAccepted++; + return this.rowsAccepted > this.pageSize; + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + Preconditions.checkArgument(filterArguments.size() == 1, + "Expected 1 but got: %s", filterArguments.size()); + long pageSize = ParseFilter.convertByteArrayToLong(filterArguments.get(0)); + return new PageFilter(pageSize); + } + + public void readFields(final DataInput in) throws IOException { + this.pageSize = in.readLong(); + } + + public void write(final DataOutput out) throws IOException { + out.writeLong(pageSize); + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + " " + this.pageSize; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ParseConstants.java b/src/main/java/org/apache/hadoop/hbase/filter/ParseConstants.java new file mode 100644 index 0000000..373d7a6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/ParseConstants.java @@ -0,0 +1,263 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.Bytes; +import java.nio.ByteBuffer; +import java.util.HashMap; +import org.apache.hadoop.hbase.filter.*; + +/** + * ParseConstants holds a bunch of constants related to parsing Filter Strings + * Used by {@link ParseFilter} + */ +public final class ParseConstants { + + /** + * ASCII code for LPAREN + */ + public static final int LPAREN = '('; + + /** + * ASCII code for RPAREN + */ + public static final int RPAREN = ')'; + + /** + * ASCII code for whitespace + */ + public static final int WHITESPACE = ' '; + + /** + * ASCII code for tab + */ + public static final int TAB = '\t'; + + /** + * ASCII code for 'A' + */ + public static final int A = 'A'; + + /** + * ASCII code for 'N' + */ + public static final int N = 'N'; + + /** + * ASCII code for 'D' + */ + public static final int D = 'D'; + + /** + * ASCII code for 'O' + */ + public static final int O = 'O'; + + /** + * ASCII code for 'R' + */ + public static final int R = 'R'; + + /** + * ASCII code for 'S' + */ + public static final int S = 'S'; + + /** + * ASCII code for 'K' + */ + public static final int K = 'K'; + + /** + * ASCII code for 'I' + */ + public static final int I = 'I'; + + /** + * ASCII code for 'P' + */ + public static final int P = 'P'; + + /** + * SKIP Array + */ + public static final byte [] SKIP_ARRAY = new byte [ ] {'S', 'K', 'I', 'P'}; + public static final ByteBuffer SKIP_BUFFER = ByteBuffer.wrap(SKIP_ARRAY); + + /** + * ASCII code for 'W' + */ + public static final int W = 'W'; + + /** + * ASCII code for 'H' + */ + public static final int H = 'H'; + + /** + * ASCII code for 'L' + */ + public static final int L = 'L'; + + /** + * ASCII code for 'E' + */ + public static final int E = 'E'; + + /** + * WHILE Array + */ + public static final byte [] WHILE_ARRAY = new byte [] {'W', 'H', 'I', 'L', 'E'}; + public static final ByteBuffer WHILE_BUFFER = ByteBuffer.wrap(WHILE_ARRAY); + + /** + * OR Array + */ + public static final byte [] OR_ARRAY = new byte [] {'O','R'}; + public static final ByteBuffer OR_BUFFER = ByteBuffer.wrap(OR_ARRAY); + + /** + * AND Array + */ + public static final byte [] AND_ARRAY = new byte [] {'A','N', 'D'}; + public static final ByteBuffer AND_BUFFER = ByteBuffer.wrap(AND_ARRAY); + + /** + * ASCII code for Backslash + */ + public static final int BACKSLASH = '\\'; + + /** + * ASCII code for a single quote + */ + public static final int SINGLE_QUOTE = '\''; + + /** + * ASCII code for a comma + */ + public static final int COMMA = ','; + + /** + * LESS_THAN Array + */ + public static final byte [] LESS_THAN_ARRAY = new byte [] {'<'}; + public static final ByteBuffer LESS_THAN_BUFFER = ByteBuffer.wrap(LESS_THAN_ARRAY); + + /** + * LESS_THAN_OR_EQUAL_TO Array + */ + public static final byte [] LESS_THAN_OR_EQUAL_TO_ARRAY = new byte [] {'<', '='}; + public static final ByteBuffer LESS_THAN_OR_EQUAL_TO_BUFFER = + ByteBuffer.wrap(LESS_THAN_OR_EQUAL_TO_ARRAY); + + /** + * GREATER_THAN Array + */ + public static final byte [] GREATER_THAN_ARRAY = new byte [] {'>'}; + public static final ByteBuffer GREATER_THAN_BUFFER = ByteBuffer.wrap(GREATER_THAN_ARRAY); + + /** + * GREATER_THAN_OR_EQUAL_TO Array + */ + public static final byte [] GREATER_THAN_OR_EQUAL_TO_ARRAY = new byte [] {'>', '='}; + public static final ByteBuffer GREATER_THAN_OR_EQUAL_TO_BUFFER = + ByteBuffer.wrap(GREATER_THAN_OR_EQUAL_TO_ARRAY); + + /** + * EQUAL_TO Array + */ + public static final byte [] EQUAL_TO_ARRAY = new byte [] {'='}; + public static final ByteBuffer EQUAL_TO_BUFFER = ByteBuffer.wrap(EQUAL_TO_ARRAY); + + /** + * NOT_EQUAL_TO Array + */ + public static final byte [] NOT_EQUAL_TO_ARRAY = new byte [] {'!', '='}; + public static final ByteBuffer NOT_EQUAL_TO_BUFFER = ByteBuffer.wrap(NOT_EQUAL_TO_ARRAY); + + /** + * ASCII code for equal to (=) + */ + public static final int EQUAL_TO = '='; + + /** + * AND Byte Array + */ + public static final byte [] AND = new byte [] {'A','N','D'}; + + /** + * OR Byte Array + */ + public static final byte [] OR = new byte [] {'O', 'R'}; + + /** + * LPAREN Array + */ + public static final byte [] LPAREN_ARRAY = new byte [] {'('}; + public static final ByteBuffer LPAREN_BUFFER = ByteBuffer.wrap(LPAREN_ARRAY); + + /** + * ASCII code for colon (:) + */ + public static final int COLON = ':'; + + /** + * ASCII code for Zero + */ + public static final int ZERO = '0'; + + /** + * ASCII code foe Nine + */ + public static final int NINE = '9'; + + /** + * BinaryType byte array + */ + public static final byte [] binaryType = new byte [] {'b','i','n','a','r','y'}; + + /** + * BinaryPrefixType byte array + */ + public static final byte [] binaryPrefixType = new byte [] {'b','i','n','a','r','y', + 'p','r','e','f','i','x'}; + + /** + * RegexStringType byte array + */ + public static final byte [] regexStringType = new byte [] {'r','e','g','e', 'x', + 's','t','r','i','n','g'}; + + /** + * SubstringType byte array + */ + public static final byte [] substringType = new byte [] {'s','u','b','s','t','r','i','n','g'}; + + /** + * ASCII for Minus Sign + */ + public static final int MINUS_SIGN = '-'; + + /** + * Package containing filters + */ + public static final String FILTER_PACKAGE = "org.apache.hadoop.hbase.filter"; +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java new file mode 100644 index 0000000..409d41e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java @@ -0,0 +1,856 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.util.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * This class allows a user to specify a filter via a string + * The string is parsed using the methods of this class and + * a filter object is constructed. This filter object is then wrapped + * in a scanner object which is then returned + *

      + * This class addresses the HBASE-4168 JIRA. More documentaton on this + * Filter Language can be found at: https://issues.apache.org/jira/browse/HBASE-4176 + */ +public class ParseFilter { + private static final Log LOG = LogFactory.getLog(ParseFilter.class); + + private static HashMap operatorPrecedenceHashMap; + private static HashMap filterHashMap; + + static { + // Registers all the filter supported by the Filter Language + filterHashMap = new HashMap(); + filterHashMap.put("KeyOnlyFilter", ParseConstants.FILTER_PACKAGE + "." + + "KeyOnlyFilter"); + filterHashMap.put("FirstKeyOnlyFilter", ParseConstants.FILTER_PACKAGE + "." + + "FirstKeyOnlyFilter"); + filterHashMap.put("PrefixFilter", ParseConstants.FILTER_PACKAGE + "." + + "PrefixFilter"); + filterHashMap.put("ColumnPrefixFilter", ParseConstants.FILTER_PACKAGE + "." + + "ColumnPrefixFilter"); + filterHashMap.put("MultipleColumnPrefixFilter", ParseConstants.FILTER_PACKAGE + "." + + "MultipleColumnPrefixFilter"); + filterHashMap.put("ColumnCountGetFilter", ParseConstants.FILTER_PACKAGE + "." + + "ColumnCountGetFilter"); + filterHashMap.put("PageFilter", ParseConstants.FILTER_PACKAGE + "." + + "PageFilter"); + filterHashMap.put("ColumnPaginationFilter", ParseConstants.FILTER_PACKAGE + "." + + "ColumnPaginationFilter"); + filterHashMap.put("InclusiveStopFilter", ParseConstants.FILTER_PACKAGE + "." + + "InclusiveStopFilter"); + filterHashMap.put("TimestampsFilter", ParseConstants.FILTER_PACKAGE + "." + + "TimestampsFilter"); + filterHashMap.put("RowFilter", ParseConstants.FILTER_PACKAGE + "." + + "RowFilter"); + filterHashMap.put("FamilyFilter", ParseConstants.FILTER_PACKAGE + "." + + "FamilyFilter"); + filterHashMap.put("QualifierFilter", ParseConstants.FILTER_PACKAGE + "." + + "QualifierFilter"); + filterHashMap.put("ValueFilter", ParseConstants.FILTER_PACKAGE + "." + + "ValueFilter"); + filterHashMap.put("ColumnRangeFilter", ParseConstants.FILTER_PACKAGE + "." + + "ColumnRangeFilter"); + filterHashMap.put("SingleColumnValueFilter", ParseConstants.FILTER_PACKAGE + "." + + "SingleColumnValueFilter"); + filterHashMap.put("SingleColumnValueExcludeFilter", ParseConstants.FILTER_PACKAGE + "." + + "SingleColumnValueExcludeFilter"); + filterHashMap.put("DependentColumnFilter", ParseConstants.FILTER_PACKAGE + "." + + "DependentColumnFilter"); + + // Creates the operatorPrecedenceHashMap + operatorPrecedenceHashMap = new HashMap(); + operatorPrecedenceHashMap.put(ParseConstants.SKIP_BUFFER, 1); + operatorPrecedenceHashMap.put(ParseConstants.WHILE_BUFFER, 1); + operatorPrecedenceHashMap.put(ParseConstants.AND_BUFFER, 2); + operatorPrecedenceHashMap.put(ParseConstants.OR_BUFFER, 3); + } + + /** + * Parses the filterString and constructs a filter using it + *

      + * @param filterString filter string given by the user + * @return filter object we constructed + */ + public Filter parseFilterString (String filterString) + throws CharacterCodingException { + return parseFilterString(Bytes.toBytes(filterString)); + } + + /** + * Parses the filterString and constructs a filter using it + *

      + * @param filterStringAsByteArray filter string given by the user + * @return filter object we constructed + */ + public Filter parseFilterString (byte [] filterStringAsByteArray) + throws CharacterCodingException { + // stack for the operators and parenthesis + Stack operatorStack = new Stack(); + // stack for the filter objects + Stack filterStack = new Stack(); + + Filter filter = null; + for (int i=0; i + * A simpleFilterExpression is of the form: FilterName('arg', 'arg', 'arg') + * The user given filter string can have many simpleFilterExpressions combined + * using operators. + *

      + * This function extracts a simpleFilterExpression from the + * larger filterString given the start offset of the simpler expression + *

      + * @param filterStringAsByteArray filter string given by the user + * @param filterExpressionStartOffset start index of the simple filter expression + * @return byte array containing the simple filter expression + */ + public byte [] extractFilterSimpleExpression (byte [] filterStringAsByteArray, + int filterExpressionStartOffset) + throws CharacterCodingException { + int quoteCount = 0; + for (int i=filterExpressionStartOffset; i + * @param filterStringAsByteArray filter string given by the user + * @return filter object we constructed + */ + public Filter parseSimpleFilterExpression (byte [] filterStringAsByteArray) + throws CharacterCodingException { + + String filterName = Bytes.toString(getFilterName(filterStringAsByteArray)); + ArrayList filterArguments = getFilterArguments(filterStringAsByteArray); + if (!filterHashMap.containsKey(filterName)) { + throw new IllegalArgumentException("Filter Name " + filterName + " not supported"); + } + try { + filterName = filterHashMap.get(filterName); + Class c = Class.forName(filterName); + Class[] argTypes = new Class [] {ArrayList.class}; + Method m = c.getDeclaredMethod("createFilterFromArguments", argTypes); + return (Filter) m.invoke(null,filterArguments); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + throw new IllegalArgumentException("Incorrect filter string " + + new String(filterStringAsByteArray)); + } + +/** + * Returns the filter name given a simple filter expression + *

      + * @param filterStringAsByteArray a simple filter expression + * @return name of filter in the simple filter expression + */ + public static byte [] getFilterName (byte [] filterStringAsByteArray) { + int filterNameStartIndex = 0; + int filterNameEndIndex = 0; + + for (int i=filterNameStartIndex; i + * @param filterStringAsByteArray filter string given by the user + * @return an ArrayList containing the arguments of the filter in the filter string + */ + public static ArrayList getFilterArguments (byte [] filterStringAsByteArray) { + int argumentListStartIndex = KeyValue.getDelimiter(filterStringAsByteArray, 0, + filterStringAsByteArray.length, + ParseConstants.LPAREN); + if (argumentListStartIndex == -1) { + throw new IllegalArgumentException("Incorrect argument list"); + } + + int argumentStartIndex = 0; + int argumentEndIndex = 0; + ArrayList filterArguments = new ArrayList(); + + for (int i = argumentListStartIndex + 1; i, != etc + argumentStartIndex = i; + for (int j = argumentStartIndex; j < filterStringAsByteArray.length; j++) { + if (filterStringAsByteArray[j] == ParseConstants.WHITESPACE || + filterStringAsByteArray[j] == ParseConstants.COMMA || + filterStringAsByteArray[j] == ParseConstants.RPAREN) { + argumentEndIndex = j - 1; + i = j; + byte [] filterArgument = new byte [argumentEndIndex - argumentStartIndex + 1]; + Bytes.putBytes(filterArgument, 0, filterStringAsByteArray, + argumentStartIndex, argumentEndIndex - argumentStartIndex + 1); + filterArguments.add(filterArgument); + break; + } else if (j == filterStringAsByteArray.length - 1) { + throw new IllegalArgumentException("Incorrect argument list"); + } + } + } + } + return filterArguments; + } + +/** + * This function is called while parsing the filterString and an operator is parsed + *

      + * @param operatorStack the stack containing the operators and parenthesis + * @param filterStack the stack containing the filters + * @param operator the operator found while parsing the filterString + */ + public void reduce(Stack operatorStack, + Stack filterStack, + ByteBuffer operator) { + while (!operatorStack.empty() && + !(ParseConstants.LPAREN_BUFFER.equals(operatorStack.peek())) && + hasHigherPriority(operatorStack.peek(), operator)) { + filterStack.push(popArguments(operatorStack, filterStack)); + } + } + + /** + * Pops an argument from the operator stack and the number of arguments required by the operator + * from the filterStack and evaluates them + *

      + * @param operatorStack the stack containing the operators + * @param filterStack the stack containing the filters + * @return the evaluated filter + */ + public static Filter popArguments (Stack operatorStack, Stack filterStack) { + ByteBuffer argumentOnTopOfStack = operatorStack.peek(); + + if (argumentOnTopOfStack.equals(ParseConstants.OR_BUFFER)) { + // The top of the stack is an OR + try { + ArrayList listOfFilters = new ArrayList(); + while (!operatorStack.empty() && operatorStack.peek().equals(ParseConstants.OR_BUFFER)) { + Filter filter = filterStack.pop(); + listOfFilters.add(0, filter); + operatorStack.pop(); + } + Filter filter = filterStack.pop(); + listOfFilters.add(0, filter); + Filter orFilter = new FilterList(FilterList.Operator.MUST_PASS_ONE, listOfFilters); + return orFilter; + } catch (EmptyStackException e) { + throw new IllegalArgumentException("Incorrect input string - an OR needs two filters"); + } + + } else if (argumentOnTopOfStack.equals(ParseConstants.AND_BUFFER)) { + // The top of the stack is an AND + try { + ArrayList listOfFilters = new ArrayList(); + while (!operatorStack.empty() && operatorStack.peek().equals(ParseConstants.AND_BUFFER)) { + Filter filter = filterStack.pop(); + listOfFilters.add(0, filter); + operatorStack.pop(); + } + Filter filter = filterStack.pop(); + listOfFilters.add(0, filter); + Filter andFilter = new FilterList(FilterList.Operator.MUST_PASS_ALL, listOfFilters); + return andFilter; + } catch (EmptyStackException e) { + throw new IllegalArgumentException("Incorrect input string - an AND needs two filters"); + } + + } else if (argumentOnTopOfStack.equals(ParseConstants.SKIP_BUFFER)) { + // The top of the stack is a SKIP + try { + Filter wrappedFilter = filterStack.pop(); + Filter skipFilter = new SkipFilter(wrappedFilter); + operatorStack.pop(); + return skipFilter; + } catch (EmptyStackException e) { + throw new IllegalArgumentException("Incorrect input string - a SKIP wraps a filter"); + } + + } else if (argumentOnTopOfStack.equals(ParseConstants.WHILE_BUFFER)) { + // The top of the stack is a WHILE + try { + Filter wrappedFilter = filterStack.pop(); + Filter whileMatchFilter = new WhileMatchFilter(wrappedFilter); + operatorStack.pop(); + return whileMatchFilter; + } catch (EmptyStackException e) { + throw new IllegalArgumentException("Incorrect input string - a WHILE wraps a filter"); + } + + } else if (argumentOnTopOfStack.equals(ParseConstants.LPAREN_BUFFER)) { + // The top of the stack is a LPAREN + try { + Filter filter = filterStack.pop(); + operatorStack.pop(); + return filter; + } catch (EmptyStackException e) { + throw new IllegalArgumentException("Incorrect Filter String"); + } + + } else { + throw new IllegalArgumentException("Incorrect arguments on operatorStack"); + } + } + +/** + * Returns which operator has higher precedence + *

      + * If a has higher precedence than b, it returns true + * If they have the same precedence, it returns false + */ + public boolean hasHigherPriority(ByteBuffer a, ByteBuffer b) { + if ((operatorPrecedenceHashMap.get(a) - operatorPrecedenceHashMap.get(b)) < 0) { + return true; + } + return false; + } + +/** + * Removes the single quote escaping a single quote - thus it returns an unescaped argument + *

      + * @param filterStringAsByteArray filter string given by user + * @param argumentStartIndex start index of the argument + * @param argumentEndIndex end index of the argument + * @return returns an unescaped argument + */ + public static byte [] createUnescapdArgument (byte [] filterStringAsByteArray, + int argumentStartIndex, int argumentEndIndex) { + int unescapedArgumentLength = 2; + for (int i = argumentStartIndex + 1; i <= argumentEndIndex - 1; i++) { + unescapedArgumentLength ++; + if (filterStringAsByteArray[i] == ParseConstants.SINGLE_QUOTE && + i != (argumentEndIndex - 1) && + filterStringAsByteArray[i+1] == ParseConstants.SINGLE_QUOTE) { + i++; + continue; + } + } + + byte [] unescapedArgument = new byte [unescapedArgumentLength]; + int count = 1; + unescapedArgument[0] = '\''; + for (int i = argumentStartIndex + 1; i <= argumentEndIndex - 1; i++) { + if (filterStringAsByteArray [i] == ParseConstants.SINGLE_QUOTE && + i != (argumentEndIndex - 1) && + filterStringAsByteArray [i+1] == ParseConstants.SINGLE_QUOTE) { + unescapedArgument[count++] = filterStringAsByteArray [i+1]; + i++; + } + else { + unescapedArgument[count++] = filterStringAsByteArray [i]; + } + } + unescapedArgument[unescapedArgumentLength - 1] = '\''; + return unescapedArgument; + } + +/** + * Checks if the current index of filter string we are on is the beginning of the keyword 'OR' + *

      + * @param filterStringAsByteArray filter string given by the user + * @param indexOfOr index at which an 'O' was read + * @return true if the keyword 'OR' is at the current index + */ + public static boolean checkForOr (byte [] filterStringAsByteArray, int indexOfOr) + throws CharacterCodingException, ArrayIndexOutOfBoundsException { + + try { + if (filterStringAsByteArray[indexOfOr] == ParseConstants.O && + filterStringAsByteArray[indexOfOr+1] == ParseConstants.R && + (filterStringAsByteArray[indexOfOr-1] == ParseConstants.WHITESPACE || + filterStringAsByteArray[indexOfOr-1] == ParseConstants.RPAREN) && + (filterStringAsByteArray[indexOfOr+2] == ParseConstants.WHITESPACE || + filterStringAsByteArray[indexOfOr+2] == ParseConstants.LPAREN)) { + return true; + } else { + return false; + } + } catch (ArrayIndexOutOfBoundsException e) { + return false; + } + } + +/** + * Checks if the current index of filter string we are on is the beginning of the keyword 'AND' + *

      + * @param filterStringAsByteArray filter string given by the user + * @param indexOfAnd index at which an 'A' was read + * @return true if the keyword 'AND' is at the current index + */ + public static boolean checkForAnd (byte [] filterStringAsByteArray, int indexOfAnd) + throws CharacterCodingException { + + try { + if (filterStringAsByteArray[indexOfAnd] == ParseConstants.A && + filterStringAsByteArray[indexOfAnd+1] == ParseConstants.N && + filterStringAsByteArray[indexOfAnd+2] == ParseConstants.D && + (filterStringAsByteArray[indexOfAnd-1] == ParseConstants.WHITESPACE || + filterStringAsByteArray[indexOfAnd-1] == ParseConstants.RPAREN) && + (filterStringAsByteArray[indexOfAnd+3] == ParseConstants.WHITESPACE || + filterStringAsByteArray[indexOfAnd+3] == ParseConstants.LPAREN)) { + return true; + } else { + return false; + } + } catch (ArrayIndexOutOfBoundsException e) { + return false; + } + } + +/** + * Checks if the current index of filter string we are on is the beginning of the keyword 'SKIP' + *

      + * @param filterStringAsByteArray filter string given by the user + * @param indexOfSkip index at which an 'S' was read + * @return true if the keyword 'SKIP' is at the current index + */ + public static boolean checkForSkip (byte [] filterStringAsByteArray, int indexOfSkip) + throws CharacterCodingException { + + try { + if (filterStringAsByteArray[indexOfSkip] == ParseConstants.S && + filterStringAsByteArray[indexOfSkip+1] == ParseConstants.K && + filterStringAsByteArray[indexOfSkip+2] == ParseConstants.I && + filterStringAsByteArray[indexOfSkip+3] == ParseConstants.P && + (indexOfSkip == 0 || + filterStringAsByteArray[indexOfSkip-1] == ParseConstants.WHITESPACE || + filterStringAsByteArray[indexOfSkip-1] == ParseConstants.RPAREN || + filterStringAsByteArray[indexOfSkip-1] == ParseConstants.LPAREN) && + (filterStringAsByteArray[indexOfSkip+4] == ParseConstants.WHITESPACE || + filterStringAsByteArray[indexOfSkip+4] == ParseConstants.LPAREN)) { + return true; + } else { + return false; + } + } catch (ArrayIndexOutOfBoundsException e) { + return false; + } + } + +/** + * Checks if the current index of filter string we are on is the beginning of the keyword 'WHILE' + *

      + * @param filterStringAsByteArray filter string given by the user + * @param indexOfWhile index at which an 'W' was read + * @return true if the keyword 'WHILE' is at the current index + */ + public static boolean checkForWhile (byte [] filterStringAsByteArray, int indexOfWhile) + throws CharacterCodingException { + + try { + if (filterStringAsByteArray[indexOfWhile] == ParseConstants.W && + filterStringAsByteArray[indexOfWhile+1] == ParseConstants.H && + filterStringAsByteArray[indexOfWhile+2] == ParseConstants.I && + filterStringAsByteArray[indexOfWhile+3] == ParseConstants.L && + filterStringAsByteArray[indexOfWhile+4] == ParseConstants.E && + (indexOfWhile == 0 || filterStringAsByteArray[indexOfWhile-1] == ParseConstants.WHITESPACE + || filterStringAsByteArray[indexOfWhile-1] == ParseConstants.RPAREN || + filterStringAsByteArray[indexOfWhile-1] == ParseConstants.LPAREN) && + (filterStringAsByteArray[indexOfWhile+5] == ParseConstants.WHITESPACE || + filterStringAsByteArray[indexOfWhile+5] == ParseConstants.LPAREN)) { + return true; + } else { + return false; + } + } catch (ArrayIndexOutOfBoundsException e) { + return false; + } + } + +/** + * Returns a boolean indicating whether the quote was escaped or not + *

      + * @param array byte array in which the quote was found + * @param quoteIndex index of the single quote + * @return returns true if the quote was unescaped + */ + public static boolean isQuoteUnescaped (byte [] array, int quoteIndex) { + if (array == null) { + throw new IllegalArgumentException("isQuoteUnescaped called with a null array"); + } + + if (quoteIndex == array.length - 1 || array[quoteIndex+1] != ParseConstants.SINGLE_QUOTE) { + return true; + } + else { + return false; + } + } + +/** + * Takes a quoted byte array and converts it into an unquoted byte array + * For example: given a byte array representing 'abc', it returns a + * byte array representing abc + *

      + * @param quotedByteArray the quoted byte array + * @return Unquoted byte array + */ + public static byte [] removeQuotesFromByteArray (byte [] quotedByteArray) { + if (quotedByteArray == null || + quotedByteArray.length < 2 || + quotedByteArray[0] != ParseConstants.SINGLE_QUOTE || + quotedByteArray[quotedByteArray.length - 1] != ParseConstants.SINGLE_QUOTE) { + throw new IllegalArgumentException("removeQuotesFromByteArray needs a quoted byte array"); + } else { + byte [] targetString = new byte [quotedByteArray.length - 2]; + Bytes.putBytes(targetString, 0, quotedByteArray, 1, quotedByteArray.length - 2); + return targetString; + } + } + +/** + * Converts an int expressed in a byte array to an actual int + *

      + * This doesn't use Bytes.toInt because that assumes + * that there will be {@link Bytes#SIZEOF_INT} bytes available. + *

      + * @param numberAsByteArray the int value expressed as a byte array + * @return the int value + */ + public static int convertByteArrayToInt (byte [] numberAsByteArray) { + + long tempResult = ParseFilter.convertByteArrayToLong(numberAsByteArray); + + if (tempResult > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Integer Argument too large"); + } else if (tempResult < Integer.MIN_VALUE) { + throw new IllegalArgumentException("Integer Argument too small"); + } + + int result = (int) tempResult; + return result; + } + +/** + * Converts a long expressed in a byte array to an actual long + *

      + * This doesn't use Bytes.toLong because that assumes + * that there will be {@link Bytes#SIZEOF_INT} bytes available. + *

      + * @param numberAsByteArray the long value expressed as a byte array + * @return the long value + */ + public static long convertByteArrayToLong (byte [] numberAsByteArray) { + if (numberAsByteArray == null) { + throw new IllegalArgumentException("convertByteArrayToLong called with a null array"); + } + + int i = 0; + long result = 0; + boolean isNegative = false; + + if (numberAsByteArray[i] == ParseConstants.MINUS_SIGN) { + i++; + isNegative = true; + } + + while (i != numberAsByteArray.length) { + if (numberAsByteArray[i] < ParseConstants.ZERO || + numberAsByteArray[i] > ParseConstants.NINE) { + throw new IllegalArgumentException("Byte Array should only contain digits"); + } + result = result*10 + (numberAsByteArray[i] - ParseConstants.ZERO); + if (result < 0) { + throw new IllegalArgumentException("Long Argument too large"); + } + i++; + } + + if (isNegative) { + return -result; + } else { + return result; + } + } + +/** + * Converts a boolean expressed in a byte array to an actual boolean + *

      + * This doesn't used Bytes.toBoolean because Bytes.toBoolean(byte []) + * assumes that 1 stands for true and 0 for false. + * Here, the byte array representing "true" and "false" is parsed + *

      + * @param booleanAsByteArray the boolean value expressed as a byte array + * @return the boolean value + */ + public static boolean convertByteArrayToBoolean (byte [] booleanAsByteArray) { + if (booleanAsByteArray == null) { + throw new IllegalArgumentException("convertByteArrayToBoolean called with a null array"); + } + + if (booleanAsByteArray.length == 4 && + (booleanAsByteArray[0] == 't' || booleanAsByteArray[0] == 'T') && + (booleanAsByteArray[1] == 'r' || booleanAsByteArray[1] == 'R') && + (booleanAsByteArray[2] == 'u' || booleanAsByteArray[2] == 'U') && + (booleanAsByteArray[3] == 'e' || booleanAsByteArray[3] == 'E')) { + return true; + } + else if (booleanAsByteArray.length == 5 && + (booleanAsByteArray[0] == 'f' || booleanAsByteArray[0] == 'F') && + (booleanAsByteArray[1] == 'a' || booleanAsByteArray[1] == 'A') && + (booleanAsByteArray[2] == 'l' || booleanAsByteArray[2] == 'L') && + (booleanAsByteArray[3] == 's' || booleanAsByteArray[3] == 'S') && + (booleanAsByteArray[4] == 'e' || booleanAsByteArray[4] == 'E')) { + return false; + } + else { + throw new IllegalArgumentException("Incorrect Boolean Expression"); + } + } + +/** + * Takes a compareOperator symbol as a byte array and returns the corresponding CompareOperator + *

      + * @param compareOpAsByteArray the comparatorOperator symbol as a byte array + * @return the Compare Operator + */ + public static CompareFilter.CompareOp createCompareOp (byte [] compareOpAsByteArray) { + ByteBuffer compareOp = ByteBuffer.wrap(compareOpAsByteArray); + if (compareOp.equals(ParseConstants.LESS_THAN_BUFFER)) + return CompareOp.LESS; + else if (compareOp.equals(ParseConstants.LESS_THAN_OR_EQUAL_TO_BUFFER)) + return CompareOp.LESS_OR_EQUAL; + else if (compareOp.equals(ParseConstants.GREATER_THAN_BUFFER)) + return CompareOp.GREATER; + else if (compareOp.equals(ParseConstants.GREATER_THAN_OR_EQUAL_TO_BUFFER)) + return CompareOp.GREATER_OR_EQUAL; + else if (compareOp.equals(ParseConstants.NOT_EQUAL_TO_BUFFER)) + return CompareOp.NOT_EQUAL; + else if (compareOp.equals(ParseConstants.EQUAL_TO_BUFFER)) + return CompareOp.EQUAL; + else + throw new IllegalArgumentException("Invalid compare operator"); + } + +/** + * Parses a comparator of the form comparatorType:comparatorValue form and returns a comparator + *

      + * @param comparator the comparator in the form comparatorType:comparatorValue + * @return the parsed comparator + */ + public static WritableByteArrayComparable createComparator (byte [] comparator) { + if (comparator == null) + throw new IllegalArgumentException("Incorrect Comparator"); + byte [][] parsedComparator = ParseFilter.parseComparator(comparator); + byte [] comparatorType = parsedComparator[0]; + byte [] comparatorValue = parsedComparator[1]; + + + if (Bytes.equals(comparatorType, ParseConstants.binaryType)) + return new BinaryComparator(comparatorValue); + else if (Bytes.equals(comparatorType, ParseConstants.binaryPrefixType)) + return new BinaryPrefixComparator(comparatorValue); + else if (Bytes.equals(comparatorType, ParseConstants.regexStringType)) + return new RegexStringComparator(new String(comparatorValue)); + else if (Bytes.equals(comparatorType, ParseConstants.substringType)) + return new SubstringComparator(new String(comparatorValue)); + else + throw new IllegalArgumentException("Incorrect comparatorType"); + } + +/** + * Splits a column in comparatorType:comparatorValue form into separate byte arrays + *

      + * @param comparator the comparator + * @return the parsed arguments of the comparator as a 2D byte array + */ + public static byte [][] parseComparator (byte [] comparator) { + final int index = KeyValue.getDelimiter(comparator, 0, comparator.length, ParseConstants.COLON); + if (index == -1) { + throw new IllegalArgumentException("Incorrect comparator"); + } + + byte [][] result = new byte [2][0]; + result[0] = new byte [index]; + System.arraycopy(comparator, 0, result[0], 0, index); + + final int len = comparator.length - (index + 1); + result[1] = new byte[len]; + System.arraycopy(comparator, index + 1, result[1], 0, len); + + return result; + } + +/** + * Return a Set of filters supported by the Filter Language + */ + public Set getSupportedFilters () { + return filterHashMap.keySet(); + } + + /** + * Returns all known filters + * @return an unmodifiable map of filters + */ + public static Map getAllFilters() { + return Collections.unmodifiableMap(filterHashMap); + } + + /** + * Register a new filter with the parser. If the filter is already registered, + * an IllegalArgumentException will be thrown. + * + * @param name a name for the filter + * @param filterClass fully qualified class name + */ + public static void registerFilter(String name, String filterClass) { + if(LOG.isInfoEnabled()) + LOG.info("Registering new filter " + name); + + filterHashMap.put(name, filterClass); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java new file mode 100644 index 0000000..e3c3d39 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java @@ -0,0 +1,92 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + +import java.io.DataOutput; +import java.io.IOException; +import java.io.DataInput; +import java.util.List; +import java.util.ArrayList; + +import com.google.common.base.Preconditions; + +/** + * Pass results that have same row prefix. + */ +public class PrefixFilter extends FilterBase { + protected byte [] prefix = null; + protected boolean passedPrefix = false; + + public PrefixFilter(final byte [] prefix) { + this.prefix = prefix; + } + + public PrefixFilter() { + super(); + } + + public byte[] getPrefix() { + return prefix; + } + + public boolean filterRowKey(byte[] buffer, int offset, int length) { + if (buffer == null || this.prefix == null) + return true; + if (length < prefix.length) + return true; + // if they are equal, return false => pass row + // else return true, filter row + // if we are passed the prefix, set flag + int cmp = Bytes.compareTo(buffer, offset, this.prefix.length, this.prefix, 0, + this.prefix.length); + if(cmp > 0) { + passedPrefix = true; + } + return cmp != 0; + } + + public boolean filterAllRemaining() { + return passedPrefix; + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + Preconditions.checkArgument(filterArguments.size() == 1, + "Expected 1 but got: %s", filterArguments.size()); + byte [] prefix = ParseFilter.removeQuotesFromByteArray(filterArguments.get(0)); + return new PrefixFilter(prefix); + } + + public void write(DataOutput out) throws IOException { + Bytes.writeByteArray(out, this.prefix); + } + + public void readFields(DataInput in) throws IOException { + this.prefix = Bytes.readByteArray(in); + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + " " + Bytes.toStringBinary(this.prefix); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/QualifierFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/QualifierFilter.java new file mode 100644 index 0000000..cd69277 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/QualifierFilter.java @@ -0,0 +1,77 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Get; + +import java.util.ArrayList; + +/** + * This filter is used to filter based on the column qualifier. It takes an + * operator (equal, greater, not equal, etc) and a byte [] comparator for the + * column qualifier portion of a key. + *

      + * This filter can be wrapped with {@link WhileMatchFilter} and {@link SkipFilter} + * to add more control. + *

      + * Multiple filters can be combined using {@link FilterList}. + *

      + * If an already known column qualifier is looked for, use {@link Get#addColumn} + * directly rather than a filter. + */ +public class QualifierFilter extends CompareFilter { + + /** + * Writable constructor, do not use. + */ + public QualifierFilter() { + } + + /** + * Constructor. + * @param op the compare op for column qualifier matching + * @param qualifierComparator the comparator for column qualifier matching + */ + public QualifierFilter(final CompareOp op, + final WritableByteArrayComparable qualifierComparator) { + super(op, qualifierComparator); + } + + @Override + public ReturnCode filterKeyValue(KeyValue v) { + int qualifierLength = v.getQualifierLength(); + if (qualifierLength > 0) { + if (doCompare(this.compareOp, this.comparator, v.getBuffer(), + v.getQualifierOffset(), qualifierLength)) { + return ReturnCode.SKIP; + } + } + return ReturnCode.INCLUDE; + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + ArrayList arguments = CompareFilter.extractArguments(filterArguments); + CompareOp compareOp = (CompareOp)arguments.get(0); + WritableByteArrayComparable comparator = (WritableByteArrayComparable)arguments.get(1); + return new QualifierFilter(compareOp, comparator); +} +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/RandomRowFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/RandomRowFilter.java new file mode 100644 index 0000000..c23ac9b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/RandomRowFilter.java @@ -0,0 +1,118 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Random; + +import org.apache.hadoop.hbase.KeyValue; + +/** + * A filter that includes rows based on a chance. + * + */ +public class RandomRowFilter extends FilterBase { + protected static final Random random = new Random(); + + protected float chance; + protected boolean filterOutRow; + + /** + * Writable constructor, do not use. + */ + public RandomRowFilter() { + } + + /** + * Create a new filter with a specified chance for a row to be included. + * + * @param chance + */ + public RandomRowFilter(float chance) { + this.chance = chance; + } + + /** + * @return The chance that a row gets included. + */ + public float getChance() { + return chance; + } + + /** + * Set the chance that a row is included. + * + * @param chance + */ + public void setChance(float chance) { + this.chance = chance; + } + + @Override + public boolean filterAllRemaining() { + return false; + } + + @Override + public ReturnCode filterKeyValue(KeyValue v) { + if (filterOutRow) { + return ReturnCode.NEXT_ROW; + } + return ReturnCode.INCLUDE; + } + + @Override + public boolean filterRow() { + return filterOutRow; + } + + @Override + public boolean filterRowKey(byte[] buffer, int offset, int length) { + if (chance < 0) { + // with a zero chance, the rows is always excluded + filterOutRow = true; + } else if (chance > 1) { + // always included + filterOutRow = false; + } else { + // roll the dice + filterOutRow = !(random.nextFloat() < chance); + } + return filterOutRow; + } + + @Override + public void reset() { + filterOutRow = false; + } + + @Override + public void readFields(DataInput in) throws IOException { + chance = in.readFloat(); + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeFloat(chance); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java new file mode 100644 index 0000000..210de0d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java @@ -0,0 +1,121 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.util.Bytes; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.IllegalCharsetNameException; +import java.util.regex.Pattern; + +/** + * This comparator is for use with {@link CompareFilter} implementations, such + * as {@link RowFilter}, {@link QualifierFilter}, and {@link ValueFilter}, for + * filtering based on the value of a given column. Use it to test if a given + * regular expression matches a cell value in the column. + *

      + * Only EQUAL or NOT_EQUAL comparisons are valid with this comparator. + *

      + * For example: + *

      + * 

      + * ValueFilter vf = new ValueFilter(CompareOp.EQUAL,
+ *     new RegexStringComparator(
+ *       // v4 IP address
+ *       "(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3,3}" +
+ *         "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))(\\/[0-9]+)?" +
+ *         "|" +
+ *       // v6 IP address
+ *       "((([\\dA-Fa-f]{1,4}:){7}[\\dA-Fa-f]{1,4})(:([\\d]{1,3}.)" +
+ *         "{3}[\\d]{1,3})?)(\\/[0-9]+)?"));
+ *
      + */ +public class RegexStringComparator extends WritableByteArrayComparable { + + private static final Log LOG = LogFactory.getLog(RegexStringComparator.class); + + private Charset charset = Charset.forName(HConstants.UTF8_ENCODING); + + private Pattern pattern; + + /** Nullary constructor for Writable, do not use */ + public RegexStringComparator() { } + + /** + * Constructor + * @param expr a valid regular expression + */ + public RegexStringComparator(String expr) { + super(Bytes.toBytes(expr)); + this.pattern = Pattern.compile(expr, Pattern.DOTALL); + } + + /** + * Specifies the {@link Charset} to use to convert the row key to a String. + *

      + * The row key needs to be converted to a String in order to be matched + * against the regular expression. This method controls which charset is + * used to do this conversion. + *

      + * If the row key is made of arbitrary bytes, the charset {@code ISO-8859-1} + * is recommended. + * @param charset The charset to use. + */ + public void setCharset(final Charset charset) { + this.charset = charset; + } + + @Override + public int compareTo(byte[] value, int offset, int length) { + // Use find() for subsequence match instead of matches() (full sequence + // match) to adhere to the principle of least surprise. + return pattern.matcher(new String(value, offset, length, charset)).find() ? 0 + : 1; + } + + @Override + public void readFields(DataInput in) throws IOException { + final String expr = in.readUTF(); + this.value = Bytes.toBytes(expr); + this.pattern = Pattern.compile(expr); + final String charset = in.readUTF(); + if (charset.length() > 0) { + try { + this.charset = Charset.forName(charset); + } catch (IllegalCharsetNameException e) { + LOG.error("invalid charset", e); + } + } + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeUTF(pattern.toString()); + out.writeUTF(charset.name()); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/RowFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/RowFilter.java new file mode 100644 index 0000000..3b00efd --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/RowFilter.java @@ -0,0 +1,93 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import java.util.ArrayList; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Scan; + +/** + * This filter is used to filter based on the key. It takes an operator + * (equal, greater, not equal, etc) and a byte [] comparator for the row, + * and column qualifier portions of a key. + *

      + * This filter can be wrapped with {@link WhileMatchFilter} to add more control. + *

      + * Multiple filters can be combined using {@link FilterList}. + *

      + * If an already known row range needs to be scanned, use {@link Scan} start + * and stop rows directly rather than a filter. + */ +public class RowFilter extends CompareFilter { + + private boolean filterOutRow = false; + + /** + * Writable constructor, do not use. + */ + public RowFilter() { + super(); + } + + /** + * Constructor. + * @param rowCompareOp the compare op for row matching + * @param rowComparator the comparator for row matching + */ + public RowFilter(final CompareOp rowCompareOp, + final WritableByteArrayComparable rowComparator) { + super(rowCompareOp, rowComparator); + } + + @Override + public void reset() { + this.filterOutRow = false; + } + + @Override + public ReturnCode filterKeyValue(KeyValue v) { + if(this.filterOutRow) { + return ReturnCode.NEXT_ROW; + } + return ReturnCode.INCLUDE; + } + + @Override + public boolean filterRowKey(byte[] data, int offset, int length) { + if(doCompare(this.compareOp, this.comparator, data, offset, length)) { + this.filterOutRow = true; + } + return this.filterOutRow; + } + + @Override + public boolean filterRow() { + return this.filterOutRow; + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + ArrayList arguments = CompareFilter.extractArguments(filterArguments); + CompareOp compareOp = (CompareOp)arguments.get(0); + WritableByteArrayComparable comparator = (WritableByteArrayComparable)arguments.get(1); + return new RowFilter(compareOp, comparator); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java new file mode 100644 index 0000000..638629c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java @@ -0,0 +1,112 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; + +import java.util.ArrayList; +import java.util.List; +import java.util.Iterator; + +/** + * A {@link Filter} that checks a single column value, but does not emit the + * tested column. This will enable a performance boost over + * {@link SingleColumnValueFilter}, if the tested column value is not actually + * needed as input (besides for the filtering itself). + */ +public class SingleColumnValueExcludeFilter extends SingleColumnValueFilter { + + /** + * Writable constructor, do not use. + */ + public SingleColumnValueExcludeFilter() { + super(); + } + + /** + * Constructor for binary compare of the value of a single column. If the + * column is found and the condition passes, all columns of the row will be + * emitted; except for the tested column value. If the column is not found or + * the condition fails, the row will not be emitted. + * + * @param family name of column family + * @param qualifier name of column qualifier + * @param compareOp operator + * @param value value to compare column values against + */ + public SingleColumnValueExcludeFilter(byte[] family, byte[] qualifier, + CompareOp compareOp, byte[] value) { + super(family, qualifier, compareOp, value); + } + + /** + * Constructor for binary compare of the value of a single column. If the + * column is found and the condition passes, all columns of the row will be + * emitted; except for the tested column value. If the condition fails, the + * row will not be emitted. + *

      + * Use the filterIfColumnMissing flag to set whether the rest of the columns + * in a row will be emitted if the specified column to check is not found in + * the row. + * + * @param family name of column family + * @param qualifier name of column qualifier + * @param compareOp operator + * @param comparator Comparator to use. + */ + public SingleColumnValueExcludeFilter(byte[] family, byte[] qualifier, + CompareOp compareOp, WritableByteArrayComparable comparator) { + super(family, qualifier, compareOp, comparator); + } + + // We cleaned result row in FilterRow to be consistent with scanning process. + public boolean hasFilterRow() { + return true; + } + + // Here we remove from row all key values from testing column + public void filterRow(List kvs) { + Iterator it = kvs.iterator(); + while (it.hasNext()) { + KeyValue kv = (KeyValue)it.next(); + // If the current column is actually the tested column, + // we will skip it instead. + if (kv.matchingColumn(this.columnFamily, this.columnQualifier)) { + it.remove(); + } + } + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + SingleColumnValueFilter tempFilter = (SingleColumnValueFilter) + SingleColumnValueFilter.createFilterFromArguments(filterArguments); + SingleColumnValueExcludeFilter filter = new SingleColumnValueExcludeFilter ( + tempFilter.getFamily(), tempFilter.getQualifier(), + tempFilter.getOperator(), tempFilter.getComparator()); + + if (filterArguments.size() == 6) { + filter.setFilterIfMissing(tempFilter.getFilterIfMissing()); + filter.setLatestVersionOnly(tempFilter.getLatestVersionOnly()); +} + return filter; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java new file mode 100644 index 0000000..4d2d504 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java @@ -0,0 +1,326 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.util.Bytes; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.ArrayList; + +import com.google.common.base.Preconditions; + +/** + * This filter is used to filter cells based on value. It takes a {@link CompareFilter.CompareOp} + * operator (equal, greater, not equal, etc), and either a byte [] value or + * a WritableByteArrayComparable. + *

      + * If we have a byte [] value then we just do a lexicographic compare. For + * example, if passed value is 'b' and cell has 'a' and the compare operator + * is LESS, then we will filter out this cell (return true). If this is not + * sufficient (eg you want to deserialize a long and then compare it to a fixed + * long value), then you can pass in your own comparator instead. + *

      + * You must also specify a family and qualifier. Only the value of this column + * will be tested. When using this filter on a {@link Scan} with specified + * inputs, the column to be tested should also be added as input (otherwise + * the filter will regard the column as missing). + *

      + * To prevent the entire row from being emitted if the column is not found + * on a row, use {@link #setFilterIfMissing}. + * Otherwise, if the column is found, the entire row will be emitted only if + * the value passes. If the value fails, the row will be filtered out. + *

      + * In order to test values of previous versions (timestamps), set + * {@link #setLatestVersionOnly} to false. The default is true, meaning that + * only the latest version's value is tested and all previous versions are ignored. + *

      + * To filter based on the value of all scanned columns, use {@link ValueFilter}. + */ +public class SingleColumnValueFilter extends FilterBase { + static final Log LOG = LogFactory.getLog(SingleColumnValueFilter.class); + + protected byte [] columnFamily; + protected byte [] columnQualifier; + private CompareOp compareOp; + private WritableByteArrayComparable comparator; + private boolean foundColumn = false; + private boolean matchedColumn = false; + private boolean filterIfMissing = false; + private boolean latestVersionOnly = true; + + /** + * Writable constructor, do not use. + */ + public SingleColumnValueFilter() { + } + + /** + * Constructor for binary compare of the value of a single column. If the + * column is found and the condition passes, all columns of the row will be + * emitted. If the condition fails, the row will not be emitted. + *

      + * Use the filterIfColumnMissing flag to set whether the rest of the columns + * in a row will be emitted if the specified column to check is not found in + * the row. + * + * @param family name of column family + * @param qualifier name of column qualifier + * @param compareOp operator + * @param value value to compare column values against + */ + public SingleColumnValueFilter(final byte [] family, final byte [] qualifier, + final CompareOp compareOp, final byte[] value) { + this(family, qualifier, compareOp, new BinaryComparator(value)); + } + + /** + * Constructor for binary compare of the value of a single column. If the + * column is found and the condition passes, all columns of the row will be + * emitted. If the condition fails, the row will not be emitted. + *

      + * Use the filterIfColumnMissing flag to set whether the rest of the columns + * in a row will be emitted if the specified column to check is not found in + * the row. + * + * @param family name of column family + * @param qualifier name of column qualifier + * @param compareOp operator + * @param comparator Comparator to use. + */ + public SingleColumnValueFilter(final byte [] family, final byte [] qualifier, + final CompareOp compareOp, final WritableByteArrayComparable comparator) { + this.columnFamily = family; + this.columnQualifier = qualifier; + this.compareOp = compareOp; + this.comparator = comparator; + } + + /** + * @return operator + */ + public CompareOp getOperator() { + return compareOp; + } + + /** + * @return the comparator + */ + public WritableByteArrayComparable getComparator() { + return comparator; + } + + /** + * @return the family + */ + public byte[] getFamily() { + return columnFamily; + } + + /** + * @return the qualifier + */ + public byte[] getQualifier() { + return columnQualifier; + } + + public ReturnCode filterKeyValue(KeyValue keyValue) { + // System.out.println("REMOVE KEY=" + keyValue.toString() + ", value=" + Bytes.toString(keyValue.getValue())); + if (this.matchedColumn) { + // We already found and matched the single column, all keys now pass + return ReturnCode.INCLUDE; + } else if (this.latestVersionOnly && this.foundColumn) { + // We found but did not match the single column, skip to next row + return ReturnCode.NEXT_ROW; + } + if (!keyValue.matchingColumn(this.columnFamily, this.columnQualifier)) { + return ReturnCode.INCLUDE; + } + foundColumn = true; + if (filterColumnValue(keyValue.getBuffer(), + keyValue.getValueOffset(), keyValue.getValueLength())) { + return this.latestVersionOnly? ReturnCode.NEXT_ROW: ReturnCode.INCLUDE; + } + this.matchedColumn = true; + return ReturnCode.INCLUDE; + } + + private boolean filterColumnValue(final byte [] data, final int offset, + final int length) { + int compareResult = this.comparator.compareTo(data, offset, length); + switch (this.compareOp) { + case LESS: + return compareResult <= 0; + case LESS_OR_EQUAL: + return compareResult < 0; + case EQUAL: + return compareResult != 0; + case NOT_EQUAL: + return compareResult == 0; + case GREATER_OR_EQUAL: + return compareResult > 0; + case GREATER: + return compareResult >= 0; + default: + throw new RuntimeException("Unknown Compare op " + compareOp.name()); + } + } + + public boolean filterRow() { + // If column was found, return false if it was matched, true if it was not + // If column not found, return true if we filter if missing, false if not + return this.foundColumn? !this.matchedColumn: this.filterIfMissing; + } + + public void reset() { + foundColumn = false; + matchedColumn = false; + } + + /** + * Get whether entire row should be filtered if column is not found. + * @return true if row should be skipped if column not found, false if row + * should be let through anyways + */ + public boolean getFilterIfMissing() { + return filterIfMissing; + } + + /** + * Set whether entire row should be filtered if column is not found. + *

      + * If true, the entire row will be skipped if the column is not found. + *

      + * If false, the row will pass if the column is not found. This is default. + * @param filterIfMissing flag + */ + public void setFilterIfMissing(boolean filterIfMissing) { + this.filterIfMissing = filterIfMissing; + } + + /** + * Get whether only the latest version of the column value should be compared. + * If true, the row will be returned if only the latest version of the column + * value matches. If false, the row will be returned if any version of the + * column value matches. The default is true. + * @return return value + */ + public boolean getLatestVersionOnly() { + return latestVersionOnly; + } + + /** + * Set whether only the latest version of the column value should be compared. + * If true, the row will be returned if only the latest version of the column + * value matches. If false, the row will be returned if any version of the + * column value matches. The default is true. + * @param latestVersionOnly flag + */ + public void setLatestVersionOnly(boolean latestVersionOnly) { + this.latestVersionOnly = latestVersionOnly; + } + + public static Filter createFilterFromArguments(ArrayList filterArguments) { + Preconditions.checkArgument(filterArguments.size() == 4 || filterArguments.size() == 6, + "Expected 4 or 6 but got: %s", filterArguments.size()); + byte [] family = ParseFilter.removeQuotesFromByteArray(filterArguments.get(0)); + byte [] qualifier = ParseFilter.removeQuotesFromByteArray(filterArguments.get(1)); + CompareOp compareOp = ParseFilter.createCompareOp(filterArguments.get(2)); + WritableByteArrayComparable comparator = ParseFilter.createComparator( + ParseFilter.removeQuotesFromByteArray(filterArguments.get(3))); + + if (comparator instanceof RegexStringComparator || + comparator instanceof SubstringComparator) { + if (compareOp != CompareOp.EQUAL && + compareOp != CompareOp.NOT_EQUAL) { + throw new IllegalArgumentException ("A regexstring comparator and substring comparator " + + "can only be used with EQUAL and NOT_EQUAL"); + } + } + + SingleColumnValueFilter filter = new SingleColumnValueFilter(family, qualifier, + compareOp, comparator); + + if (filterArguments.size() == 6) { + boolean filterIfMissing = ParseFilter.convertByteArrayToBoolean(filterArguments.get(4)); + boolean latestVersionOnly = ParseFilter.convertByteArrayToBoolean(filterArguments.get(5)); + filter.setFilterIfMissing(filterIfMissing); + filter.setLatestVersionOnly(latestVersionOnly); + } + return filter; + } + + public void readFields(final DataInput in) throws IOException { + this.columnFamily = Bytes.readByteArray(in); + if(this.columnFamily.length == 0) { + this.columnFamily = null; + } + this.columnQualifier = Bytes.readByteArray(in); + if(this.columnQualifier.length == 0) { + this.columnQualifier = null; + } + this.compareOp = CompareOp.valueOf(in.readUTF()); + this.comparator = + (WritableByteArrayComparable)HbaseObjectWritable.readObject(in, null); + this.foundColumn = in.readBoolean(); + this.matchedColumn = in.readBoolean(); + this.filterIfMissing = in.readBoolean(); + this.latestVersionOnly = in.readBoolean(); + } + + public void write(final DataOutput out) throws IOException { + Bytes.writeByteArray(out, this.columnFamily); + Bytes.writeByteArray(out, this.columnQualifier); + out.writeUTF(compareOp.name()); + HbaseObjectWritable.writeObject(out, comparator, + WritableByteArrayComparable.class, null); + out.writeBoolean(foundColumn); + out.writeBoolean(matchedColumn); + out.writeBoolean(filterIfMissing); + out.writeBoolean(latestVersionOnly); + } + + /** + * The only CF this filter needs is given column family. So, it's the only essential + * column in whole scan. If filterIfMissing == false, all families are essential, + * because of possibility of skipping the rows without any data in filtered CF. + */ + public boolean isFamilyEssential(byte[] name) { + return !this.filterIfMissing || Bytes.equals(name, this.columnFamily); + } + + @Override + public String toString() { + return String.format("%s (%s, %s, %s, %s)", + this.getClass().getSimpleName(), Bytes.toStringBinary(this.columnFamily), + Bytes.toStringBinary(this.columnQualifier), this.compareOp.name(), + Bytes.toStringBinary(this.comparator.getValue())); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java new file mode 100644 index 0000000..5fe17df --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java @@ -0,0 +1,106 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Classes; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * A wrapper filter that filters an entire row if any of the KeyValue checks do + * not pass. + *

      + * For example, if all columns in a row represent weights of different things, + * with the values being the actual weights, and we want to filter out the + * entire row if any of its weights are zero. In this case, we want to prevent + * rows from being emitted if a single key is filtered. Combine this filter + * with a {@link ValueFilter}: + *

      + * 

      + * scan.setFilter(new SkipFilter(new ValueFilter(CompareOp.EQUAL,
+ *     new BinaryComparator(Bytes.toBytes(0))));
+ * 
+ * Any row which contained a column whose value was 0 will be filtered out.
+ * Without this filter, the other non-zero valued columns in the row would still
+ * be emitted.
+ */
+public class SkipFilter extends FilterBase {
+  private boolean filterRow = false;
+  private Filter filter;
+
+  public SkipFilter() {
+    super();
+  }
+
+  public SkipFilter(Filter filter) {
+    this.filter = filter;
+  }
+
+  public Filter getFilter() {
+    return filter;
+  }
+
+  public void reset() {
+    filter.reset();
+    filterRow = false;
+  }
+
+  private void changeFR(boolean value) {
+    filterRow = filterRow || value;
+  }
+
+  public ReturnCode filterKeyValue(KeyValue v) {
+    ReturnCode c = filter.filterKeyValue(v);
+    changeFR(c != ReturnCode.INCLUDE);
+    return c;
+  }
+
+  @Override
+  public KeyValue transform(KeyValue v) {
+    return filter.transform(v);
+  }
+
+  public boolean filterRow() {
+    return filterRow;
+  }
+
+  public void write(DataOutput out) throws IOException {
+    out.writeUTF(this.filter.getClass().getName());
+    this.filter.write(out);
+  }
+
+  public void readFields(DataInput in) throws IOException {
+    this.filter = Classes.createForName(in.readUTF());
+    this.filter.readFields(in);
+  }
+
+  public boolean isFamilyEssential(byte[] name) {
+    return FilterBase.isFamilyEssential(this.filter, name);
+  }
+
+  @Override
+  public String toString() {
+    return this.getClass().getSimpleName() + " " + this.filter.toString();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/SubstringComparator.java b/src/main/java/org/apache/hadoop/hbase/filter/SubstringComparator.java
new file mode 100644
index 0000000..90bc718
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/filter/SubstringComparator.java
@@ -0,0 +1,84 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.filter;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * This comparator is for use with SingleColumnValueFilter, for filtering based on
+ * the value of a given column. Use it to test if a given substring appears
+ * in a cell value in the column. The comparison is case insensitive.
+ * 
      
+ * Only EQUAL or NOT_EQUAL tests are valid with this comparator.
+ * 
      
+ * For example:
+ * 
      
+ * 
      + * SingleColumnValueFilter scvf =
+ *   new SingleColumnValueFilter("col", CompareOp.EQUAL,
+ *     new SubstringComparator("substr"));
+ * 
      
+ */
+public class SubstringComparator extends WritableByteArrayComparable {
+
+  private String substr;
+
+  /** Nullary constructor for Writable, do not use */
+  public SubstringComparator() {
+    super();
+  }
+
+  /**
+   * Constructor
+   * @param substr the substring
+   */
+  public SubstringComparator(String substr) {
+    super(Bytes.toBytes(substr.toLowerCase()));
+    this.substr = substr.toLowerCase();
+  }
+
+  @Override
+  public byte[] getValue() {
+    return Bytes.toBytes(substr);
+  }
+
+  @Override
+  public int compareTo(byte[] value, int offset, int length) {
+    return Bytes.toString(value, offset, length).toLowerCase().contains(substr) ? 0
+        : 1;
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    String substr = in.readUTF();
+    this.value = Bytes.toBytes(substr);
+    this.substr = substr;
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeUTF(substr);
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/TimestampsFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/TimestampsFilter.java
new file mode 100644
index 0000000..5e780b5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/filter/TimestampsFilter.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.filter;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.TreeSet;
+import java.util.ArrayList;
+
+import org.apache.hadoop.hbase.KeyValue;
+import com.google.common.base.Preconditions;
+
+/**
+ * Filter that returns only cells whose timestamp (version) is
+ * in the specified list of timestamps (versions).
+ * 
      
+ * Note: Use of this filter overrides any time range/time stamp
+ * options specified using {@link org.apache.hadoop.hbase.client.Get#setTimeRange(long, long)},
+ * {@link org.apache.hadoop.hbase.client.Scan#setTimeRange(long, long)}, {@link org.apache.hadoop.hbase.client.Get#setTimeStamp(long)},
+ * or {@link org.apache.hadoop.hbase.client.Scan#setTimeStamp(long)}.
+ */
+public class TimestampsFilter extends FilterBase {
+
+  TreeSet timestamps;
+  private static final int MAX_LOG_TIMESTAMPS = 5;
+
+  // Used during scans to hint the scan to stop early
+  // once the timestamps fall below the minTimeStamp.
+  long minTimeStamp = Long.MAX_VALUE;
+
+  /**
+   * Used during deserialization. Do not use otherwise.
+   */
+  public TimestampsFilter() {
+    super();
+  }
+
+  /**
+   * Constructor for filter that retains only those
+   * cells whose timestamp (version) is in the specified
+   * list of timestamps.
+   *
+   * @param timestamps
+   */
+  public TimestampsFilter(List timestamps) {
+    for (Long timestamp : timestamps) {
+      Preconditions.checkArgument(timestamp >= 0, "must be positive %s", timestamp);
+    }
+    this.timestamps = new TreeSet(timestamps);
+    init();
+  }
+
+  /**
+   * @return the list of timestamps
+   */
+  public List getTimestamps() {
+    List list = new ArrayList(timestamps.size());
+    list.addAll(timestamps);
+    return list;
+  }
+
+  private void init() {
+    if (this.timestamps.size() > 0) {
+      minTimeStamp = this.timestamps.first();
+    }
+  }
+
+  /**
+   * Gets the minimum timestamp requested by filter.
+   * @return  minimum timestamp requested by filter.
+   */
+  public long getMin() {
+    return minTimeStamp;
+  }
+
+  @Override
+  public ReturnCode filterKeyValue(KeyValue v) {
+    if (this.timestamps.contains(v.getTimestamp())) {
+      return ReturnCode.INCLUDE;
+    } else if (v.getTimestamp() < minTimeStamp) {
+      // The remaining versions of this column are guaranteed
+      // to be lesser than all of the other values.
+      return ReturnCode.NEXT_COL;
+    }
+    return ReturnCode.SKIP;
+  }
+
+  public static Filter createFilterFromArguments(ArrayList filterArguments) {
+    ArrayList timestamps = new ArrayList();
+    for (int i = 0; i();
+    for (int idx = 0; idx < numTimestamps; idx++) {
+      this.timestamps.add(in.readLong());
+    }
+    init();
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    int numTimestamps = this.timestamps.size();
+    out.writeInt(numTimestamps);
+    for (Long timestamp : this.timestamps) {
+      out.writeLong(timestamp);
+    }
+  }
+
+  @Override
+  public String toString() {
+    return toString(MAX_LOG_TIMESTAMPS);
+  }
+
+  protected String toString(int maxTimestamps) {
+    StringBuilder tsList = new StringBuilder();
+
+    int count = 0;
+    for (Long ts : this.timestamps) {
+      if (count >= maxTimestamps) {
+        break;
+      }
+      ++count;
+      tsList.append(ts.toString());
+      if (count < this.timestamps.size() && count < maxTimestamps) {
+        tsList.append(", ");
+      }
+    }
+
+    return String.format("%s (%d/%d): [%s]", this.getClass().getSimpleName(),
+        count, this.timestamps.size(), tsList.toString());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java
new file mode 100644
index 0000000..2452129
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java
@@ -0,0 +1,73 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+import java.util.ArrayList;
+
+/**
+ * This filter is used to filter based on column value. It takes an
+ * operator (equal, greater, not equal, etc) and a byte [] comparator for the
+ * cell value.
+ * 
      
+ * This filter can be wrapped with {@link WhileMatchFilter} and {@link SkipFilter}
+ * to add more control.
+ * 
      
+ * Multiple filters can be combined using {@link FilterList}.
+ * 
      
+ * To test the value of a single qualifier when scanning multiple qualifiers,
+ * use {@link SingleColumnValueFilter}.
+ */
+public class ValueFilter extends CompareFilter {
+
+  /**
+   * Writable constructor, do not use.
+   */
+  public ValueFilter() {
+  }
+
+  /**
+   * Constructor.
+   * @param valueCompareOp the compare op for value matching
+   * @param valueComparator the comparator for value matching
+   */
+  public ValueFilter(final CompareOp valueCompareOp,
+      final WritableByteArrayComparable valueComparator) {
+    super(valueCompareOp, valueComparator);
+  }
+
+  @Override
+  public ReturnCode filterKeyValue(KeyValue v) {
+    if (doCompare(this.compareOp, this.comparator, v.getBuffer(),
+        v.getValueOffset(), v.getValueLength())) {
+      return ReturnCode.SKIP;
+    }
+    return ReturnCode.INCLUDE;
+  }
+
+  public static Filter createFilterFromArguments(ArrayList filterArguments) {
+    ArrayList arguments = CompareFilter.extractArguments(filterArguments);
+    CompareOp compareOp = (CompareOp)arguments.get(0);
+    WritableByteArrayComparable comparator = (WritableByteArrayComparable)arguments.get(1);
+    return new ValueFilter(compareOp, comparator);
+}
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java b/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
new file mode 100644
index 0000000..bed8d58
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Classes;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * A wrapper filter that returns true from {@link #filterAllRemaining()} as soon
+ * as the wrapped filters {@link Filter#filterRowKey(byte[], int, int)},
+ * {@link Filter#filterKeyValue(org.apache.hadoop.hbase.KeyValue)},
+ * {@link org.apache.hadoop.hbase.filter.Filter#filterRow()} or
+ * {@link org.apache.hadoop.hbase.filter.Filter#filterAllRemaining()} methods
+ * returns true.
+ */
+public class WhileMatchFilter extends FilterBase {
+  private boolean filterAllRemaining = false;
+  private Filter filter;
+
+  public WhileMatchFilter() {
+    super();
+  }
+
+  public WhileMatchFilter(Filter filter) {
+    this.filter = filter;
+  }
+
+  public Filter getFilter() {
+    return filter;
+  }
+
+  public void reset() {
+    this.filter.reset();
+  }
+
+  private void changeFAR(boolean value) {
+    filterAllRemaining = filterAllRemaining || value;
+  }
+
+  public boolean filterAllRemaining() {
+    return this.filterAllRemaining || this.filter.filterAllRemaining();
+  }
+
+  public boolean filterRowKey(byte[] buffer, int offset, int length) {
+    boolean value = filter.filterRowKey(buffer, offset, length);
+    changeFAR(value);
+    return value;
+  }
+
+  public ReturnCode filterKeyValue(KeyValue v) {
+    ReturnCode c = filter.filterKeyValue(v);
+    changeFAR(c != ReturnCode.INCLUDE);
+    return c;
+  }
+
+  @Override
+  public KeyValue transform(KeyValue v) {
+    return filter.transform(v);
+  }
+
+  public boolean filterRow() {
+    boolean filterRow = this.filter.filterRow();
+    changeFAR(filterRow);
+    return filterRow;
+  }
+
+  public void write(DataOutput out) throws IOException {
+    out.writeUTF(this.filter.getClass().getName());
+    this.filter.write(out);
+  }
+
+  public void readFields(DataInput in) throws IOException {
+    this.filter = Classes.createForName(in.readUTF());
+    this.filter.readFields(in);
+  }
+
+  public boolean isFamilyEssential(byte[] name) {
+    return FilterBase.isFamilyEssential(this.filter, name);
+  }
+
+  @Override
+  public String toString() {
+    return this.getClass().getSimpleName() + " " + this.filter.toString();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/WritableByteArrayComparable.java b/src/main/java/org/apache/hadoop/hbase/filter/WritableByteArrayComparable.java
new file mode 100644
index 0000000..c7731e3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/filter/WritableByteArrayComparable.java
@@ -0,0 +1,76 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.filter;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/** Base class, combines Comparable and Writable. */
+public abstract class WritableByteArrayComparable implements Writable, Comparable {
+
+  byte[] value;
+
+  /**
+   * Nullary constructor, for Writable
+   */
+  public WritableByteArrayComparable() { }
+
+  /**
+   * Constructor.
+   * @param value the value to compare against
+   */
+  public WritableByteArrayComparable(byte [] value) {
+    this.value = value;
+  }
+
+  public byte[] getValue() {
+    return value;
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    value = Bytes.readByteArray(in);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    Bytes.writeByteArray(out, value);
+  }
+
+  @Override
+  public int compareTo(byte [] value) {
+    return compareTo(value, 0, value.length);
+  }
+
+  /**
+   * Special compareTo method for subclasses, to avoid
+   * copying byte[] unnecessarily.
+   * @param value byte[] to compare
+   * @param offset offset into value
+   * @param length number of bytes to compare
+   * @return a negative integer, zero, or a positive integer as this object
+   *         is less than, equal to, or greater than the specified object.
+   */
+  public abstract int compareTo(byte [] value, int offset, int length);
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/filter/package-info.java b/src/main/java/org/apache/hadoop/hbase/filter/package-info.java
new file mode 100644
index 0000000..73ccef8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/filter/package-info.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Provides row-level filters applied to HRegion scan results during calls to
+ * {@link org.apache.hadoop.hbase.client.ResultScanner#next()}.
+
+
      
+Filters run the extent of a table unless you wrap your filter in a
+{@link org.apache.hadoop.hbase.filter.WhileMatchFilter}.
+The latter returns as soon as the filter stops matching.
+
      
+
      Do not rely on filters carrying state across rows; its not reliable in current
+hbase as we have no handlers in place for when regions split, close or server
+crashes.
+
      
+*/
+package org.apache.hadoop.hbase.filter;
diff --git a/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java b/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
new file mode 100644
index 0000000..ef1cda0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
@@ -0,0 +1,196 @@
+/*
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FilterFileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.util.Methods;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Progressable;
+
+/**
+ * An encapsulation for the FileSystem object that hbase uses to access
+ * data. This class allows the flexibility of using  
+ * separate filesystem objects for reading and writing hfiles and hlogs.
+ * In future, if we want to make hlogs be in a different filesystem,
+ * this is the place to make it happen.
+ */
+public class HFileSystem extends FilterFileSystem {
+
+  private final FileSystem noChecksumFs;   // read hfile data from storage
+  private final boolean useHBaseChecksum;
+
+  /**
+   * Create a FileSystem object for HBase regionservers.
+   * @param conf The configuration to be used for the filesystem
+   * @param useHBaseChecksums if true, then use
+   *        checksum verfication in hbase, otherwise
+   *        delegate checksum verification to the FileSystem.
+   */
+  public HFileSystem(Configuration conf, boolean useHBaseChecksum)
+    throws IOException {
+
+    // Create the default filesystem with checksum verification switched on.
+    // By default, any operation to this FilterFileSystem occurs on
+    // the underlying filesystem that has checksums switched on.
+    this.fs = FileSystem.get(conf);
+    this.useHBaseChecksum = useHBaseChecksum;
+    
+    fs.initialize(getDefaultUri(conf), conf);
+
+    // If hbase checksum verification is switched on, then create a new
+    // filesystem object that has cksum verification turned off.
+    // We will avoid verifying checksums in the fs client, instead do it
+    // inside of hbase.
+    // If this is the local file system hadoop has a bug where seeks
+    // do not go to the correct location if setVerifyChecksum(false) is called.
+    // This manifests itself in that incorrect data is read and HFileBlocks won't be able to read
+    // their header magic numbers. See HBASE-5885
+    if (useHBaseChecksum && !(fs instanceof LocalFileSystem)) {
+      conf = new Configuration(conf);
+      conf.setBoolean("dfs.client.read.shortcircuit.skip.checksum", true);
+      this.noChecksumFs = newInstanceFileSystem(conf);
+      this.noChecksumFs.setVerifyChecksum(false);
+    } else {
+      this.noChecksumFs = fs;
+    }
+  }
+
+  /**
+   * Wrap a FileSystem object within a HFileSystem. The noChecksumFs and
+   * writefs are both set to be the same specified fs. 
+   * Do not verify hbase-checksums while reading data from filesystem.
+   * @param fs Set the noChecksumFs and writeFs to this specified filesystem.
+   */
+  public HFileSystem(FileSystem fs) {
+    this.fs = fs;
+    this.noChecksumFs = fs;
+    this.useHBaseChecksum = false;
+  }
+
+  /**
+   * Returns the filesystem that is specially setup for 
+   * doing reads from storage. This object avoids doing 
+   * checksum verifications for reads.
+   * @return The FileSystem object that can be used to read data
+   *         from files.
+   */
+  public FileSystem getNoChecksumFs() {
+    return noChecksumFs;
+  }
+
+  /**
+   * Returns the underlying filesystem
+   * @return The underlying FileSystem for this FilterFileSystem object.
+   */
+  public FileSystem getBackingFs() throws IOException {
+    return fs;
+  }
+
+  /**
+   * Are we verifying checksums in HBase?
+   * @return True, if hbase is configured to verify checksums,
+   *         otherwise false.
+   */
+  public boolean useHBaseChecksum() {
+    return useHBaseChecksum;
+  }
+
+  /**
+   * Close this filesystem object
+   */
+  @Override
+  public void close() throws IOException {
+    super.close();
+    if (this.noChecksumFs != fs) {
+      this.noChecksumFs.close();
+    }
+  }
+
+ /**
+   * Returns a brand new instance of the FileSystem. It does not use
+   * the FileSystem.Cache. In newer versions of HDFS, we can directly
+   * invoke FileSystem.newInstance(Configuration).
+   * 
+   * @param conf Configuration
+   * @return A new instance of the filesystem
+   */
+  private static FileSystem newInstanceFileSystem(Configuration conf)
+    throws IOException {
+    URI uri = FileSystem.getDefaultUri(conf);
+    FileSystem fs = null;
+    Class clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
+    if (clazz != null) {
+      // This will be true for Hadoop 1.0, or 0.20.
+      fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
+      fs.initialize(uri, conf);
+    } else {
+      // For Hadoop 2.0, we have to go through FileSystem for the filesystem
+      // implementation to be loaded by the service loader in case it has not
+      // been loaded yet.
+      Configuration clone = new Configuration(conf);
+      clone.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", true);
+      fs = FileSystem.get(uri, clone);
+    }
+    if (fs == null) {
+      throw new IOException("No FileSystem for scheme: " + uri.getScheme());
+    }
+    return fs;
+  }
+
+  /**
+   * Create a new HFileSystem object, similar to FileSystem.get().
+   * This returns a filesystem object that avoids checksum
+   * verification in the filesystem for hfileblock-reads.
+   * For these blocks, checksum verification is done by HBase.
+   */
+  static public FileSystem get(Configuration conf) throws IOException {
+    return new HFileSystem(conf, true);
+  }
+
+  /**
+   * Wrap a LocalFileSystem within a HFileSystem.
+   */
+  static public FileSystem getLocalFs(Configuration conf) throws IOException {
+    return new HFileSystem(FileSystem.getLocal(conf));
+  }
+
+  /**
+   * The org.apache.hadoop.fs.FilterFileSystem does not yet support 
+   * createNonRecursive. This is a hadoop bug and when it is fixed in Hadoop,
+   * this definition will go away.
+   */
+  public FSDataOutputStream createNonRecursive(Path f,
+      boolean overwrite,
+      int bufferSize, short replication, long blockSize,
+      Progressable progress) throws IOException {
+    return fs.createNonRecursive(f, overwrite, bufferSize, replication,
+                                 blockSize, progress);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/CodeToClassAndBack.java b/src/main/java/org/apache/hadoop/hbase/io/CodeToClassAndBack.java
new file mode 100644
index 0000000..61a1c5e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/CodeToClassAndBack.java
@@ -0,0 +1,71 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import java.util.*;
+
+/**
+ * A Static Interface.
+ * Instead of having this code in the the HbaseMapWritable code, where it
+ * blocks the possibility of altering the variables and changing their types,
+ * it is put here in this static interface where the static final Maps are
+ * loaded one time. Only byte[] and Cell are supported at this time.
+ */
+public interface CodeToClassAndBack {
+  /**
+   * Static map that contains mapping from code to class
+   */
+  public static final Map> CODE_TO_CLASS =
+    new HashMap>();
+
+  /**
+   * Static map that contains mapping from class to code
+   */
+  public static final Map, Byte> CLASS_TO_CODE =
+    new HashMap, Byte>();
+
+  /**
+   * Class list for supported classes
+   */
+  public Class[] classList = {byte[].class};
+
+  /**
+   * The static loader that is used instead of the static constructor in
+   * HbaseMapWritable.
+   */
+  public InternalStaticLoader sl =
+    new InternalStaticLoader(classList, CODE_TO_CLASS, CLASS_TO_CODE);
+
+  /**
+   * Class that loads the static maps with their values.
+   */
+  public class InternalStaticLoader{
+    InternalStaticLoader(Class[] classList,
+        Map> CODE_TO_CLASS, Map, Byte> CLASS_TO_CODE){
+      byte code = 1;
+      for(int i=0; iThe Problem:
+ * 
        
+ *  
      • 
+ *    HDFS doesn't have support for hardlinks, and this make impossible to referencing
+ *    the same data blocks using different names.
+ *  
        • 
+ *  
      • 
+ *    HBase store files in one location (e.g. table/region/family/) and when the file is not
+ *    needed anymore (e.g. compaction, region deletetion, ...) moves it to an archive directory.
+ *  
        • 
+ * 
      
+ * If we want to create a reference to a file, we need to remember that it can be in its
+ * original location or in the archive folder.
+ * The FileLink class tries to abstract this concept and given a set of locations
+ * it is able to switch between them making this operation transparent for the user.
+ * More concrete implementations of the FileLink are the {@link HFileLink} and the {@link HLogLink}.
+ *
+ * 
      Back-references:
+ * To help the {@link CleanerChore} to keep track of the links to a particular file,
+ * during the FileLink creation, a new file is placed inside a back-reference directory.
+ * There's one back-reference directory for each file that has links,
+ * and in the directory there's one file per link.
+ *
+ * 
      HFileLink Example
+ * 
        
+ *  
      • 
+ *      /hbase/table/region-x/cf/file-k
+ *      (Original File)
+ *  
        • 
+ *  
      • 
+ *      /hbase/table-cloned/region-y/cf/file-k.region-x.table
+ *     (HFileLink to the original file)
+ *  
        • 
+ *  
      • 
+ *      /hbase/table-2nd-cloned/region-z/cf/file-k.region-x.table
+ *      (HFileLink to the original file)
+ *  
        • 
+ *  
      • 
+ *      /hbase/.archive/table/region-x/.links-file-k/region-y.table-cloned
+ *      (Back-reference to the link in table-cloned)
+ *  
        • 
+ *  
      • 
+ *      /hbase/.archive/table/region-x/.links-file-k/region-z.table-cloned
+ *      (Back-reference to the link in table-2nd-cloned)
+ *  
        • 
+ * 
      
+ */
+@InterfaceAudience.Private
+public class FileLink {
+  private static final Log LOG = LogFactory.getLog(FileLink.class);
+
+  /** Define the Back-reference directory name prefix: .links-/ */
+  public static final String BACK_REFERENCES_DIRECTORY_PREFIX = ".links-";
+
+  /**
+   * FileLink InputStream that handles the switch between the original path
+   * and the alternative locations, when the file is moved.
+   */
+  private static class FileLinkInputStream extends InputStream
+      implements Seekable, PositionedReadable {
+    private FSDataInputStream in = null;
+    private Path currentPath = null;
+    private long pos = 0;
+
+    private final FileLink fileLink;
+    private final int bufferSize;
+    private final FileSystem fs;
+
+    public FileLinkInputStream(final FileSystem fs, final FileLink fileLink)
+        throws IOException {
+      this(fs, fileLink, FSUtils.getDefaultBufferSize(fs));
+    }
+
+    public FileLinkInputStream(final FileSystem fs, final FileLink fileLink, int bufferSize)
+        throws IOException {
+      this.bufferSize = bufferSize;
+      this.fileLink = fileLink;
+      this.fs = fs;
+
+      this.in = tryOpen();
+    }
+
+    @Override
+    public int read() throws IOException {
+      int res;
+      try {
+        res = in.read();
+      } catch (FileNotFoundException e) {
+        res = tryOpen().read();
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        res = tryOpen().read();
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        res = tryOpen().read();
+      }
+      if (res > 0) pos += 1;
+      return res;
+    }
+
+    @Override
+    public int read(byte b[]) throws IOException {
+       return read(b, 0, b.length);
+    }
+
+    @Override
+    public int read(byte b[], int off, int len) throws IOException {
+      int n;
+      try {
+        n = in.read(b, off, len);
+      } catch (FileNotFoundException e) {
+        n = tryOpen().read(b, off, len);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        n = tryOpen().read(b, off, len);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        n = tryOpen().read(b, off, len);
+      }
+      if (n > 0) pos += n;
+      assert(in.getPos() == pos);
+      return n;
+    }
+
+    @Override
+    public int read(long position, byte[] buffer, int offset, int length) throws IOException {
+      int n;
+      try {
+        n = in.read(position, buffer, offset, length);
+      } catch (FileNotFoundException e) {
+        n = tryOpen().read(position, buffer, offset, length);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        n = tryOpen().read(position, buffer, offset, length);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        n = tryOpen().read(position, buffer, offset, length);
+      }
+      return n;
+    }
+
+    @Override
+    public void readFully(long position, byte[] buffer) throws IOException {
+      readFully(position, buffer, 0, buffer.length);
+    }
+
+    @Override
+    public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
+      try {
+        in.readFully(position, buffer, offset, length);
+      } catch (FileNotFoundException e) {
+        tryOpen().readFully(position, buffer, offset, length);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        tryOpen().readFully(position, buffer, offset, length);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        tryOpen().readFully(position, buffer, offset, length);
+      }
+    }
+
+    @Override
+    public long skip(long n) throws IOException {
+      long skipped;
+
+      try {
+        skipped = in.skip(n);
+      } catch (FileNotFoundException e) {
+        skipped = tryOpen().skip(n);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        skipped = tryOpen().skip(n);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        skipped = tryOpen().skip(n);
+      }
+
+      if (skipped > 0) pos += skipped;
+      return skipped;
+    }
+
+    @Override
+    public int available() throws IOException {
+      try {
+        return in.available();
+      } catch (FileNotFoundException e) {
+        return tryOpen().available();
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        return tryOpen().available();
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        return tryOpen().available();
+      }
+    }
+
+    @Override
+    public void seek(long pos) throws IOException {
+      try {
+        in.seek(pos);
+      } catch (FileNotFoundException e) {
+        tryOpen().seek(pos);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        tryOpen().seek(pos);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        tryOpen().seek(pos);
+      }
+      this.pos = pos;
+    }
+
+    @Override
+    public long getPos() throws IOException {
+      return pos;
+    }
+
+    @Override
+    public boolean seekToNewSource(long targetPos) throws IOException {
+      boolean res;
+      try {
+        res = in.seekToNewSource(targetPos);
+      } catch (FileNotFoundException e) {
+        res = tryOpen().seekToNewSource(targetPos);
+      } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
+        res = tryOpen().seekToNewSource(targetPos);
+      } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
+        res = tryOpen().seekToNewSource(targetPos);
+      }
+      if (res) pos = targetPos;
+      return res;
+    }
+
+    @Override
+    public void close() throws IOException {
+      in.close();
+    }
+
+    @Override
+    public synchronized void mark(int readlimit) {
+    }
+
+    @Override
+    public synchronized void reset() throws IOException {
+      throw new IOException("mark/reset not supported");
+    }
+
+    @Override
+    public boolean markSupported() {
+      return false;
+    }
+
+    /**
+     * Try to open the file from one of the available locations.
+     *
+     * @return FSDataInputStream stream of the opened file link
+     * @throws IOException on unexpected error, or file not found.
+     */
+    private FSDataInputStream tryOpen() throws IOException {
+      for (Path path: fileLink.getLocations()) {
+        if (path.equals(currentPath)) continue;
+        try {
+          in = fs.open(path, bufferSize);
+          in.seek(pos);
+          assert(in.getPos() == pos) : "Link unable to seek to the right position=" + pos;
+          if (LOG.isTraceEnabled()) {
+            if (currentPath != null) {
+              LOG.debug("link open path=" + path);
+            } else {
+              LOG.trace("link switch from path=" + currentPath + " to path=" + path);
+            }
+          }
+          currentPath = path;
+          return(in);
+        } catch (FileNotFoundException e) {
+          // Try another file location
+        }
+      }
+      throw new FileNotFoundException("Unable to open link: " + fileLink);
+    }
+  }
+
+  private Path[] locations = null;
+
+  protected FileLink() {
+    this.locations = null;
+  }
+
+  /**
+   * @param originPath Original location of the file to link
+   * @param alternativePaths Alternative locations to look for the linked file
+   */
+  public FileLink(Path originPath, Path... alternativePaths) {
+    setLocations(originPath, alternativePaths);
+  }
+
+  /**
+   * @param locations locations to look for the linked file
+   */
+  public FileLink(final Collection locations) {
+    this.locations = locations.toArray(new Path[locations.size()]);
+  }
+
+  /**
+   * @return the locations to look for the linked file.
+   */
+  public Path[] getLocations() {
+    return locations;
+  }
+
+  public String toString() {
+    StringBuilder str = new StringBuilder(getClass().getName());
+    str.append(" locations=[");
+    int i = 0;
+    for (Path location: locations) {
+      if (i++ > 0) str.append(", ");
+      str.append(location.toString());
+    }
+    str.append("]");
+    return str.toString();
+  }
+
+  /**
+   * @return the path of the first available link.
+   */
+  public Path getAvailablePath(FileSystem fs) throws IOException {
+    for (Path path: locations) {
+      if (fs.exists(path)) {
+        return path;
+      }
+    }
+    throw new FileNotFoundException("Unable to open link: " + this);
+  }
+
+  /**
+   * Get the FileStatus of the referenced file.
+   *
+   * @param fs {@link FileSystem} on which to get the file status
+   * @return InputStream for the hfile link.
+   * @throws IOException on unexpected error.
+   */
+  public FileStatus getFileStatus(FileSystem fs) throws IOException {
+    for (Path path: locations) {
+      try {
+        return fs.getFileStatus(path);
+      } catch (FileNotFoundException e) {
+        // Try another file location
+      }
+    }
+    throw new FileNotFoundException("Unable to open link: " + this);
+  }
+
+  /**
+   * Open the FileLink for read.
+   * 
      
+   * It uses a wrapper of FSDataInputStream that is agnostic to the location
+   * of the file, even if the file switches between locations.
+   *
+   * @param fs {@link FileSystem} on which to open the FileLink
+   * @return InputStream for reading the file link.
+   * @throws IOException on unexpected error.
+   */
+  public FSDataInputStream open(final FileSystem fs) throws IOException {
+    return new FSDataInputStream(new FileLinkInputStream(fs, this));
+  }
+
+  /**
+   * Open the FileLink for read.
+   * 
      
+   * It uses a wrapper of FSDataInputStream that is agnostic to the location
+   * of the file, even if the file switches between locations.
+   *
+   * @param fs {@link FileSystem} on which to open the FileLink
+   * @param bufferSize the size of the buffer to be used.
+   * @return InputStream for reading the file link.
+   * @throws IOException on unexpected error.
+   */
+  public FSDataInputStream open(final FileSystem fs, int bufferSize) throws IOException {
+    return new FSDataInputStream(new FileLinkInputStream(fs, this, bufferSize));
+  }
+
+  /**
+   * NOTE: This method must be used only in the constructor!
+   * It creates a List with the specified locations for the link.
+   */
+  protected void setLocations(Path originPath, Path... alternativePaths) {
+    assert this.locations == null : "Link locations already set";
+    this.locations = new Path[1 + alternativePaths.length];
+    this.locations[0] = originPath;
+    for (int i = 0; i < alternativePaths.length; i++) {
+      this.locations[i + 1] = alternativePaths[i];
+    }
+  }
+
+  /**
+   * Get the directory to store the link back references
+   *
+   * 
      To simplify the reference count process, during the FileLink creation
+   * a back-reference is added to the back-reference directory of the specified file.
+   *
+   * @param storeDir Root directory for the link reference folder
+   * @param fileName File Name with links
+   * @return Path for the link back references.
+   */
+  public static Path getBackReferencesDir(final Path storeDir, final String fileName) {
+    return new Path(storeDir, BACK_REFERENCES_DIRECTORY_PREFIX + fileName);
+  }
+
+  /**
+   * Get the referenced file name from the reference link directory path.
+   *
+   * @param dirPath Link references directory path
+   * @return Name of the file referenced
+   */
+  public static String getBackReferenceFileName(final Path dirPath) {
+    return dirPath.getName().substring(BACK_REFERENCES_DIRECTORY_PREFIX.length());
+  }
+
+  /**
+   * Checks if the specified directory path is a back reference links folder.
+   *
+   * @param dirPath Directory path to verify
+   * @return True if the specified directory is a link references folder
+   */
+  public static boolean isBackReferencesDir(final Path dirPath) {
+    if (dirPath == null) return false;
+    return dirPath.getName().startsWith(BACK_REFERENCES_DIRECTORY_PREFIX);
+  }
+}
+
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
new file mode 100644
index 0000000..59bf578
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
@@ -0,0 +1,371 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+
+/**
+ * HFileLink describes a link to an hfile.
+ *
+ * An hfile can be served from a region or from the hfile archive directory (/hbase/.archive)
+ * HFileLink allows to access the referenced hfile regardless of the location where it is.
+ *
+ * 
      Searches for hfiles in the following order and locations:
+ * 
        
+ *  
      • /hbase/table/region/cf/hfile
        • 
+ *  
      • /hbase/.archive/table/region/cf/hfile
        • 
+ * 
      
+ *
+ * The link checks first in the original path if it is not present
+ * it fallbacks to the archived path.
+ */
+@InterfaceAudience.Private
+public class HFileLink extends FileLink {
+  private static final Log LOG = LogFactory.getLog(HFileLink.class);
+
+  /**
+   * A non-capture group, for HFileLink, so that this can be embedded.
+   * The HFileLink describe a link to an hfile in a different table/region
+   * and the name is in the form: table=region-hfile.
+   * 
      
+   * Table name is ([a-zA-Z_0-9][a-zA-Z_0-9.-]*), so '=' is an invalid character for the table name.
+   * Region name is ([a-f0-9]+), so '-' is an invalid character for the region name.
+   * HFile is ([0-9a-f]+(?:_SeqId_[0-9]+_)?) covering the plain hfiles (uuid)
+   * and the bulk loaded (_SeqId_[0-9]+_) hfiles.
+   */
+  public static final String LINK_NAME_REGEX =
+    String.format("%s=%s-%s", HTableDescriptor.VALID_USER_TABLE_REGEX,
+      HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFile.HFILE_NAME_REGEX);
+
+  /** Define the HFile Link name parser in the form of: table=region-hfile */
+  private static final Pattern LINK_NAME_PATTERN =
+    Pattern.compile(String.format("^(%s)=(%s)-(%s)$", HTableDescriptor.VALID_USER_TABLE_REGEX,
+      HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFile.HFILE_NAME_REGEX));
+
+  /**
+   * The pattern should be used for hfile and reference links
+   * that can be found in /hbase/table/region/family/
+   */
+  private static final Pattern REF_OR_HFILE_LINK_PATTERN =
+    Pattern.compile(String.format("^(%s)=(%s)-(.+)$", HTableDescriptor.VALID_USER_TABLE_REGEX,
+      HRegionInfo.ENCODED_REGION_NAME_REGEX));
+
+  private final Path archivePath;
+  private final Path originPath;
+  private final Path tempPath;
+
+  /**
+   * @param conf {@link Configuration} from which to extract specific archive locations
+   * @param path The path of the HFile Link.
+   * @throws IOException on unexpected error.
+   */
+  public HFileLink(Configuration conf, Path path) throws IOException {
+    this(FSUtils.getRootDir(conf), HFileArchiveUtil.getArchivePath(conf), path);
+  }
+
+  /**
+   * @param rootDir Path to the root directory where hbase files are stored
+   * @param archiveDir Path to the hbase archive directory
+   * @param path The path of the HFile Link.
+   */
+  public HFileLink(final Path rootDir, final Path archiveDir, final Path path) {
+    Path hfilePath = getRelativeTablePath(path);
+    this.tempPath = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY), hfilePath);
+    this.originPath = new Path(rootDir, hfilePath);
+    this.archivePath = new Path(archiveDir, hfilePath);
+    setLocations(originPath, tempPath, archivePath);
+  }
+
+  /**
+   * @return the origin path of the hfile.
+   */
+  public Path getOriginPath() {
+    return this.originPath;
+  }
+
+  /**
+   * @return the path of the archived hfile.
+   */
+  public Path getArchivePath() {
+    return this.archivePath;
+  }
+
+  /**
+   * @param path Path to check.
+   * @return True if the path is a HFileLink.
+   */
+  public static boolean isHFileLink(final Path path) {
+    return isHFileLink(path.getName());
+  }
+
+
+  /**
+   * @param fileName File name to check.
+   * @return True if the path is a HFileLink.
+   */
+  public static boolean isHFileLink(String fileName) {
+    Matcher m = LINK_NAME_PATTERN.matcher(fileName);
+    if (!m.matches()) return false;
+
+    return m.groupCount() > 2 && m.group(3) != null && m.group(2) != null && m.group(1) != null;
+  }
+
+  /**
+   * Convert a HFileLink path to a table relative path.
+   * e.g. the link: /hbase/test/0123/cf/testtb=4567-abcd
+   *      becomes: /hbase/testtb/4567/cf/abcd
+   *
+   * @param path HFileLink path
+   * @return Relative table path
+   * @throws IOException on unexpected error.
+   */
+  private static Path getRelativeTablePath(final Path path) {
+    // table=region-hfile
+    Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(path.getName());
+    if (!m.matches()) {
+      throw new IllegalArgumentException(path.getName() + " is not a valid HFileLink name!");
+    }
+
+    // Convert the HFileLink name into a real table/region/cf/hfile path.
+    String tableName = m.group(1);
+    String regionName = m.group(2);
+    String hfileName = m.group(3);
+    String familyName = path.getParent().getName();
+    return new Path(new Path(tableName, regionName), new Path(familyName, hfileName));
+  }
+
+  /**
+   * Get the HFile name of the referenced link
+   *
+   * @param fileName HFileLink file name
+   * @return the name of the referenced HFile
+   */
+  public static String getReferencedHFileName(final String fileName) {
+    Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!");
+    }
+    return(m.group(3));
+  }
+
+  /**
+   * Get the Region name of the referenced link
+   *
+   * @param fileName HFileLink file name
+   * @return the name of the referenced Region
+   */
+  public static String getReferencedRegionName(final String fileName) {
+    Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!");
+    }
+    return(m.group(2));
+  }
+
+  /**
+   * Get the Table name of the referenced link
+   *
+   * @param fileName HFileLink file name
+   * @return the name of the referenced Table
+   */
+  public static String getReferencedTableName(final String fileName) {
+    Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!");
+    }
+    return(m.group(1));
+  }
+
+  /**
+   * Create a new HFileLink name
+   *
+   * @param hfileRegionInfo - Linked HFile Region Info
+   * @param hfileName - Linked HFile name
+   * @return file name of the HFile Link
+   */
+  public static String createHFileLinkName(final HRegionInfo hfileRegionInfo,
+      final String hfileName) {
+    return createHFileLinkName(hfileRegionInfo.getTableNameAsString(),
+                      hfileRegionInfo.getEncodedName(), hfileName);
+  }
+
+  /**
+   * Create a new HFileLink name
+   *
+   * @param tableName - Linked HFile table name
+   * @param regionName - Linked HFile region name
+   * @param hfileName - Linked HFile name
+   * @return file name of the HFile Link
+   */
+  public static String createHFileLinkName(final String tableName,
+      final String regionName, final String hfileName) {
+    return String.format("%s=%s-%s", tableName, regionName, hfileName);
+  }
+
+  /**
+   * Create a new HFileLink
+   *
+   * 
      It also adds a back-reference to the hfile back-reference directory
+   * to simplify the reference-count and the cleaning process.
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param fs {@link FileSystem} on which to write the HFileLink
+   * @param dstFamilyPath - Destination path (table/region/cf/)
+   * @param hfileRegionInfo - Linked HFile Region Info
+   * @param hfileName - Linked HFile name
+   * @return true if the file is created, otherwise the file exists.
+   * @throws IOException on file or parent directory creation failure
+   */
+  public static boolean create(final Configuration conf, final FileSystem fs,
+      final Path dstFamilyPath, final HRegionInfo hfileRegionInfo,
+      final String hfileName) throws IOException {
+    String linkedTable = hfileRegionInfo.getTableNameAsString();
+    String linkedRegion = hfileRegionInfo.getEncodedName();
+    return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName);
+  }
+
+  /**
+   * Create a new HFileLink
+   *
+   * 
      It also adds a back-reference to the hfile back-reference directory
+   * to simplify the reference-count and the cleaning process.
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param fs {@link FileSystem} on which to write the HFileLink
+   * @param dstFamilyPath - Destination path (table/region/cf/)
+   * @param linkedTable - Linked Table Name
+   * @param linkedRegion - Linked Region Name
+   * @param hfileName - Linked HFile name
+   * @return true if the file is created, otherwise the file exists.
+   * @throws IOException on file or parent directory creation failure
+   */
+  public static boolean create(final Configuration conf, final FileSystem fs,
+      final Path dstFamilyPath, final String linkedTable, final String linkedRegion,
+      final String hfileName) throws IOException {
+    String familyName = dstFamilyPath.getName();
+    String regionName = dstFamilyPath.getParent().getName();
+    String tableName = dstFamilyPath.getParent().getParent().getName();
+
+    String name = createHFileLinkName(linkedTable, linkedRegion, hfileName);
+    String refName = createBackReferenceName(tableName, regionName);
+
+    // Make sure the destination directory exists
+    fs.mkdirs(dstFamilyPath);
+
+    // Make sure the FileLink reference directory exists
+    Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
+          linkedTable, linkedRegion, familyName);
+    Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName);
+    fs.mkdirs(backRefssDir);
+
+    // Create the reference for the link
+    Path backRefPath = new Path(backRefssDir, refName);
+    fs.createNewFile(backRefPath);
+    try {
+      // Create the link
+      return fs.createNewFile(new Path(dstFamilyPath, name));
+    } catch (IOException e) {
+      LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e);
+      // Revert the reference if the link creation failed
+      fs.delete(backRefPath, false);
+      throw e;
+    }
+  }
+
+  /**
+   * Create a new HFileLink starting from a hfileLink name
+   *
+   * 
      It also adds a back-reference to the hfile back-reference directory
+   * to simplify the reference-count and the cleaning process.
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param fs {@link FileSystem} on which to write the HFileLink
+   * @param dstFamilyPath - Destination path (table/region/cf/)
+   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
+   * @return true if the file is created, otherwise the file exists.
+   * @throws IOException on file or parent directory creation failure
+   */
+  public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs,
+      final Path dstFamilyPath, final String hfileLinkName) throws IOException {
+    Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!");
+    }
+    return create(conf, fs, dstFamilyPath, m.group(1), m.group(2), m.group(3));
+  }
+
+  /**
+   * Create the back reference name
+   */
+  private static String createBackReferenceName(final String tableName, final String regionName) {
+    return regionName + "." + tableName;
+  }
+
+  /**
+   * Get the full path of the HFile referenced by the back reference
+   *
+   * @param rootDir root hbase directory
+   * @param linkRefPath Link Back Reference path
+   * @return full path of the referenced hfile
+   * @throws IOException on unexpected error.
+   */
+  public static Path getHFileFromBackReference(final Path rootDir, final Path linkRefPath) {
+    int separatorIndex = linkRefPath.getName().indexOf('.');
+    String linkRegionName = linkRefPath.getName().substring(0, separatorIndex);
+    String linkTableName = linkRefPath.getName().substring(separatorIndex + 1);
+    String hfileName = getBackReferenceFileName(linkRefPath.getParent());
+    Path familyPath = linkRefPath.getParent().getParent();
+    Path regionPath = familyPath.getParent();
+    Path tablePath = regionPath.getParent();
+
+    String linkName = createHFileLinkName(tablePath.getName(), regionPath.getName(), hfileName);
+    Path linkTableDir = FSUtils.getTablePath(rootDir, linkTableName);
+    Path regionDir = HRegion.getRegionDir(linkTableDir, linkRegionName);
+    return new Path(new Path(regionDir, familyPath.getName()), linkName);
+  }
+
+  /**
+   * Get the full path of the HFile referenced by the back reference
+   *
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param linkRefPath Link Back Reference path
+   * @return full path of the referenced hfile
+   * @throws IOException on unexpected error.
+   */
+  public static Path getHFileFromBackReference(final Configuration conf, final Path linkRefPath)
+      throws IOException {
+    return getHFileFromBackReference(FSUtils.getRootDir(conf), linkRefPath);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HLogLink.java b/src/main/java/org/apache/hadoop/hbase/io/HLogLink.java
new file mode 100644
index 0000000..8feb8cb
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/HLogLink.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * HLogLink describes a link to a WAL.
+ *
+ * An hlog can be in /hbase/.logs//
+ * or it can be in /hbase/.oldlogs/
+ *
+ * The link checks first in the original path,
+ * if it is not present it fallbacks to the archived path.
+ */
+@InterfaceAudience.Private
+public class HLogLink extends FileLink {
+  /**
+   * @param conf {@link Configuration} from which to extract specific archive locations
+   * @param serverName Region Server owner of the log
+   * @param logName WAL file name
+   * @throws IOException on unexpected error.
+   */
+  public HLogLink(final Configuration conf,
+      final String serverName, final String logName) throws IOException {
+    this(FSUtils.getRootDir(conf), serverName, logName);
+  }
+
+  /**
+   * @param rootDir Path to the root directory where hbase files are stored
+   * @param serverName Region Server owner of the log
+   * @param logName WAL file name
+   */
+  public HLogLink(final Path rootDir, final String serverName, final String logName) {
+    final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+    final Path logDir = new Path(new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), serverName);
+    setLocations(new Path(logDir, logName), new Path(oldLogDir, logName));
+  }
+
+  /**
+   * @param originPath Path to the wal in the log directory
+   * @param archivePath Path to the wal in the archived log directory
+   */
+  public HLogLink(final Path originPath, final Path archivePath) {
+    setLocations(originPath, archivePath);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
new file mode 100644
index 0000000..e07f370
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
@@ -0,0 +1,329 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * A facade for a {@link org.apache.hadoop.hbase.io.hfile.HFile.Reader} that serves up
+ * either the top or bottom half of a HFile where 'bottom' is the first half
+ * of the file containing the keys that sort lowest and 'top' is the second half
+ * of the file with keys that sort greater than those of the bottom half.
+ * The top includes the split files midkey, of the key that follows if it does
+ * not exist in the file.
+ *
+ * 
      This type works in tandem with the {@link Reference} type.  This class
+ * is used reading while Reference is used writing.
+ *
+ * 
      This file is not splitable.  Calls to {@link #midkey()} return null.
+ */
+public class HalfStoreFileReader extends StoreFile.Reader {
+  final Log LOG = LogFactory.getLog(HalfStoreFileReader.class);
+  final boolean top;
+  // This is the key we split around.  Its the first possible entry on a row:
+  // i.e. empty column and a timestamp of LATEST_TIMESTAMP.
+  protected final byte [] splitkey;
+  
+  private byte[] firstKey = null;
+  
+  private boolean firstKeySeeked = false;
+
+  /**
+   * Creates a half file reader for a normal hfile.
+   * @param fs fileystem to read from
+   * @param p path to hfile
+   * @param cacheConf
+   * @param r original reference file (contains top or bottom)
+   * @param preferredEncodingInCache
+   * @throws IOException
+   */
+  public HalfStoreFileReader(final FileSystem fs, final Path p,
+      final CacheConfig cacheConf, final Reference r,
+      DataBlockEncoding preferredEncodingInCache) throws IOException {
+    super(fs, p, cacheConf, preferredEncodingInCache);
+    // This is not actual midkey for this half-file; its just border
+    // around which we split top and bottom.  Have to look in files to find
+    // actual last and first keys for bottom and top halves.  Half-files don't
+    // have an actual midkey themselves. No midkey is how we indicate file is
+    // not splittable.
+    this.splitkey = r.getSplitKey();
+    // Is it top or bottom half?
+    this.top = Reference.isTopFileRegion(r.getFileRegion());
+  }
+
+  /**
+   * Creates a half file reader for a hfile referred to by an hfilelink.
+   * @param fs fileystem to read from
+   * @param p path to hfile
+   * @param link
+   * @param cacheConf
+   * @param r original reference file (contains top or bottom)
+   * @param preferredEncodingInCache
+   * @throws IOException
+   */
+  public HalfStoreFileReader(final FileSystem fs, final Path p, final HFileLink link,
+      final CacheConfig cacheConf, final Reference r,
+      DataBlockEncoding preferredEncodingInCache) throws IOException {
+    super(fs, p, link, link.getFileStatus(fs).getLen(), cacheConf, preferredEncodingInCache, true);
+    // This is not actual midkey for this half-file; its just border
+    // around which we split top and bottom.  Have to look in files to find
+    // actual last and first keys for bottom and top halves.  Half-files don't
+    // have an actual midkey themselves. No midkey is how we indicate file is
+    // not splittable.
+    this.splitkey = r.getSplitKey();
+    // Is it top or bottom half?
+    this.top = Reference.isTopFileRegion(r.getFileRegion());
+  }
+
+  protected boolean isTop() {
+    return this.top;
+  }
+
+  @Override
+  public HFileScanner getScanner(final boolean cacheBlocks,
+      final boolean pread, final boolean isCompaction) {
+    final HFileScanner s = super.getScanner(cacheBlocks, pread, isCompaction);
+    return new HFileScanner() {
+      final HFileScanner delegate = s;
+      public boolean atEnd = false;
+
+      public ByteBuffer getKey() {
+        if (atEnd) return null;
+        return delegate.getKey();
+      }
+
+      public String getKeyString() {
+        if (atEnd) return null;
+
+        return delegate.getKeyString();
+      }
+
+      public ByteBuffer getValue() {
+        if (atEnd) return null;
+
+        return delegate.getValue();
+      }
+
+      public String getValueString() {
+        if (atEnd) return null;
+
+        return delegate.getValueString();
+      }
+
+      public KeyValue getKeyValue() {
+        if (atEnd) return null;
+
+        return delegate.getKeyValue();
+      }
+
+      public boolean next() throws IOException {
+        if (atEnd) return false;
+
+        boolean b = delegate.next();
+        if (!b) {
+          return b;
+        }
+        // constrain the bottom.
+        if (!top) {
+          ByteBuffer bb = getKey();
+          if (getComparator().compare(bb.array(), bb.arrayOffset(), bb.limit(),
+              splitkey, 0, splitkey.length) >= 0) {
+            atEnd = true;
+            return false;
+          }
+        }
+        return true;
+      }
+
+      public boolean seekBefore(byte[] key) throws IOException {
+        return seekBefore(key, 0, key.length);
+      }
+
+      public boolean seekBefore(byte [] key, int offset, int length)
+      throws IOException {
+        if (top) {
+          byte[] fk = getFirstKey();
+          // This will be null when the file is empty in which we can not seekBefore to any key
+          if (fk == null) return false;
+          if (getComparator().compare(key, offset, length, fk, 0,
+              fk.length) <= 0) {
+            return false;
+          }
+        } else {
+          // The equals sign isn't strictly necessary just here to be consistent with seekTo
+          if (getComparator().compare(key, offset, length, splitkey, 0,
+              splitkey.length) >= 0) {
+            return this.delegate.seekBefore(splitkey, 0, splitkey.length);
+          }
+        }
+        return this.delegate.seekBefore(key, offset, length);
+      }
+
+      public boolean seekTo() throws IOException {
+        if (top) {
+          int r = this.delegate.seekTo(splitkey);
+          if (r < 0) {
+            // midkey is < first key in file
+            return this.delegate.seekTo();
+          }
+          if (r > 0) {
+            return this.delegate.next();
+          }
+          return true;
+        }
+
+        boolean b = delegate.seekTo();
+        if (!b) {
+          return b;
+        }
+        // Check key.
+        ByteBuffer k = this.delegate.getKey();
+        return this.delegate.getReader().getComparator().
+          compare(k.array(), k.arrayOffset(), k.limit(),
+            splitkey, 0, splitkey.length) < 0;
+      }
+
+      public int seekTo(byte[] key) throws IOException {
+        return seekTo(key, 0, key.length);
+      }
+
+      public int seekTo(byte[] key, int offset, int length) throws IOException {
+        if (top) {
+          if (getComparator().compare(key, offset, length, splitkey, 0,
+              splitkey.length) < 0) {
+            return -1;
+          }
+        } else {
+          if (getComparator().compare(key, offset, length, splitkey, 0,
+              splitkey.length) >= 0) {
+            // we would place the scanner in the second half.
+            // it might be an error to return false here ever...
+            boolean res = delegate.seekBefore(splitkey, 0, splitkey.length);
+            if (!res) {
+              throw new IOException("Seeking for a key in bottom of file, but key exists in top of file, failed on seekBefore(midkey)");
+            }
+            return 1;
+          }
+        }
+        return delegate.seekTo(key, offset, length);
+      }
+
+      @Override
+      public int reseekTo(byte[] key) throws IOException {
+        return reseekTo(key, 0, key.length);
+      }
+
+      @Override
+      public int reseekTo(byte[] key, int offset, int length)
+      throws IOException {
+        //This function is identical to the corresponding seekTo function except
+        //that we call reseekTo (and not seekTo) on the delegate.
+        if (top) {
+          if (getComparator().compare(key, offset, length, splitkey, 0,
+              splitkey.length) < 0) {
+            return -1;
+          }
+        } else {
+          if (getComparator().compare(key, offset, length, splitkey, 0,
+              splitkey.length) >= 0) {
+            // we would place the scanner in the second half.
+            // it might be an error to return false here ever...
+            boolean res = delegate.seekBefore(splitkey, 0, splitkey.length);
+            if (!res) {
+              throw new IOException("Seeking for a key in bottom of file, but" +
+                  " key exists in top of file, failed on seekBefore(midkey)");
+            }
+            return 1;
+          }
+        }
+        if (atEnd) {
+          // skip the 'reseek' and just return 1.
+          return 1;
+        }
+        return delegate.reseekTo(key, offset, length);
+      }
+
+      public org.apache.hadoop.hbase.io.hfile.HFile.Reader getReader() {
+        return this.delegate.getReader();
+      }
+
+      public boolean isSeeked() {
+        return this.delegate.isSeeked();
+      }
+    };
+  }
+
+  @Override
+  public boolean passesKeyRangeFilter(Scan scan) {
+    return true;
+  }
+  
+  @Override
+  public byte[] getLastKey() {
+    if (top) {
+      return super.getLastKey();
+    }
+    // Get a scanner that caches the block and that uses pread.
+    HFileScanner scanner = getScanner(true, true);
+    try {
+      if (scanner.seekBefore(this.splitkey)) {
+        return Bytes.toBytes(scanner.getKey());
+      }
+    } catch (IOException e) {
+      LOG.warn("Failed seekBefore " + Bytes.toStringBinary(this.splitkey), e);
+    }
+    return null;
+  }
+
+  @Override
+  public byte[] midkey() throws IOException {
+    // Returns null to indicate file is not splitable.
+    return null;
+  }
+  
+  @Override
+  public byte[] getFirstKey() {
+    if (!firstKeySeeked) {
+      HFileScanner scanner = getScanner(true, true, false);
+      try {
+        if (scanner.seekTo()) {
+          this.firstKey = Bytes.toBytes(scanner.getKey());
+        }
+        firstKeySeeked = true;
+      } catch (IOException e) {
+        LOG.warn("Failed seekTo first KV in the file", e);
+      }
+    }
+    return this.firstKey;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HbaseMapWritable.java b/src/main/java/org/apache/hadoop/hbase/io/HbaseMapWritable.java
new file mode 100644
index 0000000..45eb495
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/HbaseMapWritable.java
@@ -0,0 +1,221 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * A Writable Map.
+ * Like {@link org.apache.hadoop.io.MapWritable} but dumb. It will fail
+ * if passed a value type that it has not already been told about. Its  been
+ * primed with hbase Writables and byte [].  Keys are always byte arrays.
+ *
+ * @param   key  TODO: Parameter K is never used, could be removed.
+ * @param  value Expects a Writable or byte [].
+ */
+public class HbaseMapWritable 
+implements SortedMap, Configurable, Writable, CodeToClassAndBack{
+  private AtomicReference conf = null;
+  protected SortedMap instance = null;
+
+  /**
+   * The default contructor where a TreeMap is used
+   **/
+   public HbaseMapWritable(){
+     this (new TreeMap(Bytes.BYTES_COMPARATOR));
+   }
+
+  /**
+   * Contructor where another SortedMap can be used
+   *
+   * @param map the SortedMap to be used
+   */
+  public HbaseMapWritable(SortedMap map){
+    conf = new AtomicReference();
+    instance = map;
+  }
+
+
+  /** @return the conf */
+  public Configuration getConf() {
+    return conf.get();
+  }
+
+  /** @param conf the conf to set */
+  public void setConf(Configuration conf) {
+    this.conf.set(conf);
+  }
+
+  public void clear() {
+    instance.clear();
+  }
+
+  public boolean containsKey(Object key) {
+    return instance.containsKey(key);
+  }
+
+  public boolean containsValue(Object value) {
+    return instance.containsValue(value);
+  }
+
+  public Set> entrySet() {
+    return instance.entrySet();
+  }
+
+  public V get(Object key) {
+    return instance.get(key);
+  }
+
+  public boolean isEmpty() {
+    return instance.isEmpty();
+  }
+
+  public Set keySet() {
+    return instance.keySet();
+  }
+
+  public int size() {
+    return instance.size();
+  }
+
+  public Collection values() {
+    return instance.values();
+  }
+
+  public void putAll(Map m) {
+    this.instance.putAll(m);
+  }
+
+  public V remove(Object key) {
+    return this.instance.remove(key);
+  }
+
+  public V put(byte [] key, V value) {
+    return this.instance.put(key, value);
+  }
+
+  public Comparator comparator() {
+    return this.instance.comparator();
+  }
+
+  public byte[] firstKey() {
+    return this.instance.firstKey();
+  }
+
+  public SortedMap headMap(byte[] toKey) {
+    return this.instance.headMap(toKey);
+  }
+
+  public byte[] lastKey() {
+    return this.instance.lastKey();
+  }
+
+  public SortedMap subMap(byte[] fromKey, byte[] toKey) {
+    return this.instance.subMap(fromKey, toKey);
+  }
+
+  public SortedMap tailMap(byte[] fromKey) {
+    return this.instance.tailMap(fromKey);
+  }
+
+  // Writable
+
+  /** @return the Class class for the specified id */
+  @SuppressWarnings("boxing")
+  protected Class getClass(byte id) {
+    return CODE_TO_CLASS.get(id);
+  }
+
+  /** @return the id for the specified Class */
+  @SuppressWarnings("boxing")
+  protected byte getId(Class clazz) {
+    Byte b = CLASS_TO_CODE.get(clazz);
+    if (b == null) {
+      throw new NullPointerException("Nothing for : " + clazz);
+    }
+    return b;
+  }
+
+  /**
+   * @see java.lang.Object#toString()
+   */
+  @Override
+  public String toString() {
+    return this.instance.toString();
+  }
+
+  public void write(DataOutput out) throws IOException {
+    // Write out the number of entries in the map
+    out.writeInt(this.instance.size());
+    // Then write out each key/value pair
+    for (Map.Entry e: instance.entrySet()) {
+      Bytes.writeByteArray(out, e.getKey());
+      Byte id = getId(e.getValue().getClass());
+      out.writeByte(id);
+      Object value = e.getValue();
+      if (value instanceof byte []) {
+        Bytes.writeByteArray(out, (byte [])value);
+      } else {
+        ((Writable)value).write(out);
+      }
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  public void readFields(DataInput in) throws IOException {
+    // First clear the map.  Otherwise we will just accumulate
+    // entries every time this method is called.
+    this.instance.clear();
+    // Read the number of entries in the map
+    int entries = in.readInt();
+    // Then read each key/value pair
+    for (int i = 0; i < entries; i++) {
+      byte [] key = Bytes.readByteArray(in);
+      byte id = in.readByte();
+      Class clazz = getClass(id);
+      V value = null;
+      if (clazz.equals(byte [].class)) {
+        byte [] bytes = Bytes.readByteArray(in);
+        value = (V)bytes;
+      } else {
+        Writable w = (Writable)ReflectionUtils.
+          newInstance(clazz, getConf());
+        w.readFields(in);
+        value = (V)w;
+      }
+      this.instance.put(key, value);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
new file mode 100644
index 0000000..317601e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
@@ -0,0 +1,813 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.InputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.lang.reflect.Array;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Action;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.MultiAction;
+import org.apache.hadoop.hbase.client.MultiResponse;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.Exec;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.BitComparator;
+import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
+import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
+import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.DependentColumnFilter;
+import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
+import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
+import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.hbase.filter.RandomRowFilter;
+import org.apache.hadoop.hbase.filter.RowFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.filter.SkipFilter;
+import org.apache.hadoop.hbase.filter.ValueFilter;
+import org.apache.hadoop.hbase.filter.WhileMatchFilter;
+import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ProtoUtil;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.ObjectWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableFactories;
+import org.apache.hadoop.io.WritableUtils;
+
+import com.google.protobuf.Message;
+
+/**
+ * This is a customized version of the polymorphic hadoop
+ * {@link ObjectWritable}.  It removes UTF8 (HADOOP-414).
+ * Using {@link Text} intead of UTF-8 saves ~2% CPU between reading and writing
+ * objects running a short sequentialWrite Performance Evaluation test just in
+ * ObjectWritable alone; more when we're doing randomRead-ing.  Other
+ * optimizations include our passing codes for classes instead of the
+ * actual class names themselves.  This makes it so this class needs amendment
+ * if non-Writable classes are introduced -- if passed a Writable for which we
+ * have no code, we just do the old-school passing of the class name, etc. --
+ * but passing codes the  savings are large particularly when cell
+ * data is small (If < a couple of kilobytes, the encoding/decoding of class
+ * name and reflection to instantiate class was costing in excess of the cell
+ * handling).
+ */
+public class HbaseObjectWritable implements Writable, WritableWithSize, Configurable {
+  protected final static Log LOG = LogFactory.getLog(HbaseObjectWritable.class);
+
+  // Here we maintain two static maps of classes to code and vice versa.
+  // Add new classes+codes as wanted or figure way to auto-generate these
+  // maps from the HMasterInterface.
+  static final Map> CODE_TO_CLASS =
+    new HashMap>();
+  static final Map, Integer> CLASS_TO_CODE =
+    new HashMap, Integer>();
+  // Special code that means 'not-encoded'; in this case we do old school
+  // sending of the class name using reflection, etc.
+  private static final byte NOT_ENCODED = 0;
+  //Generic array means that the array type is not one of the pre-defined arrays
+  //in the CLASS_TO_CODE map, but we have to still encode the array since it's
+  //elements are serializable by this class.
+  private static final int GENERIC_ARRAY_CODE;
+  private static final int NEXT_CLASS_CODE;
+  static {
+    ////////////////////////////////////////////////////////////////////////////
+    // WARNING: Please do not insert, remove or swap any line in this static  //
+    // block.  Doing so would change or shift all the codes used to serialize //
+    // objects, which makes backwards compatibility very hard for clients.    //
+    // New codes should always be added at the end. Code removal is           //
+    // discouraged because code is a short now.                               //
+    ////////////////////////////////////////////////////////////////////////////
+
+    int code = NOT_ENCODED + 1;
+    // Primitive types.
+    addToMap(Boolean.TYPE, code++);
+    addToMap(Byte.TYPE, code++);
+    addToMap(Character.TYPE, code++);
+    addToMap(Short.TYPE, code++);
+    addToMap(Integer.TYPE, code++);
+    addToMap(Long.TYPE, code++);
+    addToMap(Float.TYPE, code++);
+    addToMap(Double.TYPE, code++);
+    addToMap(Void.TYPE, code++);
+
+    // Other java types
+    addToMap(String.class, code++);
+    addToMap(byte [].class, code++);
+    addToMap(byte [][].class, code++);
+
+    // Hadoop types
+    addToMap(Text.class, code++);
+    addToMap(Writable.class, code++);
+    addToMap(Writable [].class, code++);
+    addToMap(HbaseMapWritable.class, code++);
+    addToMap(NullInstance.class, code++);
+
+    // Hbase types
+    addToMap(HColumnDescriptor.class, code++);
+    addToMap(HConstants.Modify.class, code++);
+
+    // We used to have a class named HMsg but its been removed.  Rather than
+    // just axe it, use following random Integer class -- we just chose any
+    // class from java.lang -- instead just so codes that follow stay
+    // in same relative place.
+    addToMap(Integer.class, code++);
+    addToMap(Integer[].class, code++);
+
+    addToMap(HRegion.class, code++);
+    addToMap(HRegion[].class, code++);
+    addToMap(HRegionInfo.class, code++);
+    addToMap(HRegionInfo[].class, code++);
+    addToMap(HServerAddress.class, code++);
+    addToMap(HServerInfo.class, code++);
+    addToMap(HTableDescriptor.class, code++);
+    addToMap(MapWritable.class, code++);
+
+    //
+    // HBASE-880
+    //
+    addToMap(ClusterStatus.class, code++);
+    addToMap(Delete.class, code++);
+    addToMap(Get.class, code++);
+    addToMap(KeyValue.class, code++);
+    addToMap(KeyValue[].class, code++);
+    addToMap(Put.class, code++);
+    addToMap(Put[].class, code++);
+    addToMap(Result.class, code++);
+    addToMap(Result[].class, code++);
+    addToMap(Scan.class, code++);
+
+    addToMap(WhileMatchFilter.class, code++);
+    addToMap(PrefixFilter.class, code++);
+    addToMap(PageFilter.class, code++);
+    addToMap(InclusiveStopFilter.class, code++);
+    addToMap(ColumnCountGetFilter.class, code++);
+    addToMap(SingleColumnValueFilter.class, code++);
+    addToMap(SingleColumnValueExcludeFilter.class, code++);
+    addToMap(BinaryComparator.class, code++);
+    addToMap(BitComparator.class, code++);
+    addToMap(CompareFilter.class, code++);
+    addToMap(RowFilter.class, code++);
+    addToMap(ValueFilter.class, code++);
+    addToMap(QualifierFilter.class, code++);
+    addToMap(SkipFilter.class, code++);
+    addToMap(WritableByteArrayComparable.class, code++);
+    addToMap(FirstKeyOnlyFilter.class, code++);
+    addToMap(DependentColumnFilter.class, code++);
+
+    addToMap(Delete [].class, code++);
+
+    addToMap(HLog.Entry.class, code++);
+    addToMap(HLog.Entry[].class, code++);
+    addToMap(HLogKey.class, code++);
+
+    addToMap(List.class, code++);
+
+    addToMap(NavigableSet.class, code++);
+    addToMap(ColumnPrefixFilter.class, code++);
+
+    // Multi
+    addToMap(Row.class, code++);
+    addToMap(Action.class, code++);
+    addToMap(MultiAction.class, code++);
+    addToMap(MultiResponse.class, code++);
+
+    // coprocessor execution
+    addToMap(Exec.class, code++);
+    addToMap(Increment.class, code++);
+
+    addToMap(KeyOnlyFilter.class, code++);
+
+    // serializable
+    addToMap(Serializable.class, code++);
+
+    addToMap(RandomRowFilter.class, code++);
+
+    addToMap(CompareOp.class, code++);
+
+    addToMap(ColumnRangeFilter.class, code++);
+
+    addToMap(HServerLoad.class, code++);
+
+    addToMap(RegionOpeningState.class, code++);
+
+    addToMap(HTableDescriptor[].class, code++);
+
+    addToMap(Append.class, code++);
+
+    addToMap(RowMutations.class, code++);
+
+    addToMap(Message.class, code++);
+
+    //java.lang.reflect.Array is a placeholder for arrays not defined above
+    GENERIC_ARRAY_CODE = code++;
+    addToMap(Array.class, GENERIC_ARRAY_CODE);
+    
+    addToMap(FuzzyRowFilter.class, code++);
+
+    // we aren't going to bump the rpc version number.
+    // we don't want to cause incompatiblity with older 0.94/0.92 clients.
+    addToMap(HSnapshotDescription.class, code);
+
+    // make sure that this is the last statement in this static block
+    NEXT_CLASS_CODE = code;
+  }
+
+  private Class declaredClass;
+  private Object instance;
+  private Configuration conf;
+
+  /** default constructor for writable */
+  public HbaseObjectWritable() {
+    super();
+  }
+
+  /**
+   * @param instance
+   */
+  public HbaseObjectWritable(Object instance) {
+    set(instance);
+  }
+
+  /**
+   * @param declaredClass
+   * @param instance
+   */
+  public HbaseObjectWritable(Class declaredClass, Object instance) {
+    this.declaredClass = declaredClass;
+    this.instance = instance;
+  }
+
+  /** @return the instance, or null if none. */
+  public Object get() { return instance; }
+
+  /** @return the class this is meant to be. */
+  public Class getDeclaredClass() { return declaredClass; }
+
+  /**
+   * Reset the instance.
+   * @param instance
+   */
+  public void set(Object instance) {
+    this.declaredClass = instance.getClass();
+    this.instance = instance;
+  }
+
+  /**
+   * @see java.lang.Object#toString()
+   */
+  @Override
+  public String toString() {
+    return "OW[class=" + declaredClass + ",value=" + instance + "]";
+  }
+
+
+  public void readFields(DataInput in) throws IOException {
+    readObject(in, this, this.conf);
+  }
+
+  public void write(DataOutput out) throws IOException {
+    writeObject(out, instance, declaredClass, conf);
+  }
+
+  public long getWritableSize() {
+    return getWritableSize(instance, declaredClass, conf);
+  }
+
+  private static class NullInstance extends Configured implements Writable {
+    Class declaredClass;
+    /** default constructor for writable */
+    @SuppressWarnings("unused")
+    public NullInstance() { super(null); }
+
+    /**
+     * @param declaredClass
+     * @param conf
+     */
+    public NullInstance(Class declaredClass, Configuration conf) {
+      super(conf);
+      this.declaredClass = declaredClass;
+    }
+
+    public void readFields(DataInput in) throws IOException {
+      this.declaredClass = CODE_TO_CLASS.get(WritableUtils.readVInt(in));
+    }
+
+    public void write(DataOutput out) throws IOException {
+      writeClassCode(out, this.declaredClass);
+    }
+  }
+
+  static Integer getClassCode(final Class c)
+  throws IOException {
+    Integer code = CLASS_TO_CODE.get(c);
+    if (code == null ) {
+      if (List.class.isAssignableFrom(c)) {
+        code = CLASS_TO_CODE.get(List.class);
+      } else if (Writable.class.isAssignableFrom(c)) {
+        code = CLASS_TO_CODE.get(Writable.class);
+      } else if (c.isArray()) {
+        code = CLASS_TO_CODE.get(Array.class);
+      } else if (Message.class.isAssignableFrom(c)) {
+        code = CLASS_TO_CODE.get(Message.class);
+      } else if (Serializable.class.isAssignableFrom(c)){
+        code = CLASS_TO_CODE.get(Serializable.class);
+      }
+    }
+    return code;
+  }
+
+  /**
+   * @return the next object code in the list.  Used in testing to verify that additional fields are not added 
+   */
+  static int getNextClassCode(){
+    return NEXT_CLASS_CODE;
+  }
+
+  /**
+   * Write out the code for passed Class.
+   * @param out
+   * @param c
+   * @throws IOException
+   */
+  static void writeClassCode(final DataOutput out, final Class c)
+      throws IOException {
+    Integer code = getClassCode(c);
+
+    if (code == null) {
+      LOG.error("Unsupported type " + c);
+      StackTraceElement[] els = new Exception().getStackTrace();
+      for(StackTraceElement elem : els) {
+        LOG.error(elem.getMethodName());
+      }
+      throw new UnsupportedOperationException("No code for unexpected " + c);
+    }
+    WritableUtils.writeVInt(out, code);
+  }
+
+  public static long getWritableSize(Object instance, Class declaredClass,
+                                     Configuration conf) {
+    long size = Bytes.SIZEOF_BYTE; // code
+    if (instance == null) {
+      return 0L;
+    }
+
+    if (declaredClass.isArray()) {
+      if (declaredClass.equals(Result[].class)) {
+
+        return size + Result.getWriteArraySize((Result[])instance);
+      }
+    }
+    if (declaredClass.equals(Result.class)) {
+      Result r = (Result) instance;
+      // one extra class code for writable instance.
+      return r.getWritableSize() + size + Bytes.SIZEOF_BYTE;
+    }
+    return 0L; // no hint is the default.
+  }
+  /**
+   * Write a {@link Writable}, {@link String}, primitive type, or an array of
+   * the preceding.
+   * @param out
+   * @param instance
+   * @param declaredClass
+   * @param conf
+   * @throws IOException
+   */
+  @SuppressWarnings("unchecked")
+  public static void writeObject(DataOutput out, Object instance,
+                                 Class declaredClass,
+                                 Configuration conf)
+  throws IOException {
+
+    Object instanceObj = instance;
+    Class declClass = declaredClass;
+
+    if (instanceObj == null) {                       // null
+      instanceObj = new NullInstance(declClass, conf);
+      declClass = Writable.class;
+    }
+    writeClassCode(out, declClass);
+    if (declClass.isArray()) {                // array
+      // If bytearray, just dump it out -- avoid the recursion and
+      // byte-at-a-time we were previously doing.
+      if (declClass.equals(byte [].class)) {
+        Bytes.writeByteArray(out, (byte [])instanceObj);
+      } else if(declClass.equals(Result [].class)) {
+        Result.writeArray(out, (Result [])instanceObj);
+      } else {
+        //if it is a Generic array, write the element's type
+        if (getClassCode(declaredClass) == GENERIC_ARRAY_CODE) {
+          Class componentType = declaredClass.getComponentType();
+          writeClass(out, componentType);
+        }
+
+        int length = Array.getLength(instanceObj);
+        out.writeInt(length);
+        for (int i = 0; i < length; i++) {
+          Object item = Array.get(instanceObj, i);
+          writeObject(out, item,
+                    item.getClass(), conf);
+        }
+      }
+    } else if (List.class.isAssignableFrom(declClass)) {
+      List list = (List)instanceObj;
+      int length = list.size();
+      out.writeInt(length);
+      for (int i = 0; i < length; i++) {
+        Object elem = list.get(i);
+        writeObject(out, elem,
+                  elem == null ? Writable.class : elem.getClass(), conf);
+      }
+    } else if (declClass == String.class) {   // String
+      Text.writeString(out, (String)instanceObj);
+    } else if (declClass.isPrimitive()) {     // primitive type
+      if (declClass == Boolean.TYPE) {        // boolean
+        out.writeBoolean(((Boolean)instanceObj).booleanValue());
+      } else if (declClass == Character.TYPE) { // char
+        out.writeChar(((Character)instanceObj).charValue());
+      } else if (declClass == Byte.TYPE) {    // byte
+        out.writeByte(((Byte)instanceObj).byteValue());
+      } else if (declClass == Short.TYPE) {   // short
+        out.writeShort(((Short)instanceObj).shortValue());
+      } else if (declClass == Integer.TYPE) { // int
+        out.writeInt(((Integer)instanceObj).intValue());
+      } else if (declClass == Long.TYPE) {    // long
+        out.writeLong(((Long)instanceObj).longValue());
+      } else if (declClass == Float.TYPE) {   // float
+        out.writeFloat(((Float)instanceObj).floatValue());
+      } else if (declClass == Double.TYPE) {  // double
+        out.writeDouble(((Double)instanceObj).doubleValue());
+      } else if (declClass == Void.TYPE) {    // void
+      } else {
+        throw new IllegalArgumentException("Not a primitive: "+declClass);
+      }
+    } else if (declClass.isEnum()) {         // enum
+      Text.writeString(out, ((Enum)instanceObj).name());
+    } else if (Message.class.isAssignableFrom(declaredClass)) {
+      Text.writeString(out, instanceObj.getClass().getName());
+      ((Message)instance).writeDelimitedTo(
+          DataOutputOutputStream.constructOutputStream(out));
+    } else if (Writable.class.isAssignableFrom(declClass)) { // Writable
+      Class  c = instanceObj.getClass();
+      Integer code = CLASS_TO_CODE.get(c);
+      if (code == null) {
+        out.writeByte(NOT_ENCODED);
+        Text.writeString(out, c.getName());
+      } else {
+        writeClassCode(out, c);
+      }
+      ((Writable)instanceObj).write(out);
+    } else if (Serializable.class.isAssignableFrom(declClass)) {
+      Class  c = instanceObj.getClass();
+      Integer code = CLASS_TO_CODE.get(c);
+      if (code == null) {
+        out.writeByte(NOT_ENCODED);
+        Text.writeString(out, c.getName());
+      } else {
+        writeClassCode(out, c);
+      }
+      ByteArrayOutputStream bos = null;
+      ObjectOutputStream oos = null;
+      try{
+        bos = new ByteArrayOutputStream();
+        oos = new ObjectOutputStream(bos);
+        oos.writeObject(instanceObj);
+        byte[] value = bos.toByteArray();
+        out.writeInt(value.length);
+        out.write(value);
+      } finally {
+        if(bos!=null) bos.close();
+        if(oos!=null) oos.close();
+      }
+    } else {
+      throw new IOException("Can't write: "+instanceObj+" as "+declClass);
+    }
+  }
+
+  /** Writes the encoded class code as defined in CLASS_TO_CODE, or
+   * the whole class name if not defined in the mapping.
+   */
+  static void writeClass(DataOutput out, Class c) throws IOException {
+    Integer code = CLASS_TO_CODE.get(c);
+    if (code == null) {
+      WritableUtils.writeVInt(out, NOT_ENCODED);
+      Text.writeString(out, c.getName());
+    } else {
+      WritableUtils.writeVInt(out, code);
+    }
+  }
+
+  /** Reads and returns the class as written by {@link #writeClass(DataOutput, Class)} */
+  static Class readClass(Configuration conf, DataInput in) throws IOException {
+    Class instanceClass = null;
+    int b = (byte)WritableUtils.readVInt(in);
+    if (b == NOT_ENCODED) {
+      String className = Text.readString(in);
+      try {
+        instanceClass = getClassByName(conf, className);
+      } catch (ClassNotFoundException e) {
+        LOG.error("Can't find class " + className, e);
+        throw new IOException("Can't find class " + className, e);
+      }
+    } else {
+      instanceClass = CODE_TO_CLASS.get(b);
+    }
+    return instanceClass;
+  }
+
+  /**
+   * Read a {@link Writable}, {@link String}, primitive type, or an array of
+   * the preceding.
+   * @param in
+   * @param conf
+   * @return the object
+   * @throws IOException
+   */
+  public static Object readObject(DataInput in, Configuration conf)
+    throws IOException {
+    return readObject(in, null, conf);
+  }
+
+  /**
+   * Read a {@link Writable}, {@link String}, primitive type, or an array of
+   * the preceding.
+   * @param in
+   * @param objectWritable
+   * @param conf
+   * @return the object
+   * @throws IOException
+   */
+  @SuppressWarnings("unchecked")
+  public static Object readObject(DataInput in,
+      HbaseObjectWritable objectWritable, Configuration conf)
+  throws IOException {
+    Class declaredClass = CODE_TO_CLASS.get(WritableUtils.readVInt(in));
+    Object instance;
+    if (declaredClass.isPrimitive()) {            // primitive types
+      if (declaredClass == Boolean.TYPE) {             // boolean
+        instance = Boolean.valueOf(in.readBoolean());
+      } else if (declaredClass == Character.TYPE) {    // char
+        instance = Character.valueOf(in.readChar());
+      } else if (declaredClass == Byte.TYPE) {         // byte
+        instance = Byte.valueOf(in.readByte());
+      } else if (declaredClass == Short.TYPE) {        // short
+        instance = Short.valueOf(in.readShort());
+      } else if (declaredClass == Integer.TYPE) {      // int
+        instance = Integer.valueOf(in.readInt());
+      } else if (declaredClass == Long.TYPE) {         // long
+        instance = Long.valueOf(in.readLong());
+      } else if (declaredClass == Float.TYPE) {        // float
+        instance = Float.valueOf(in.readFloat());
+      } else if (declaredClass == Double.TYPE) {       // double
+        instance = Double.valueOf(in.readDouble());
+      } else if (declaredClass == Void.TYPE) {         // void
+        instance = null;
+      } else {
+        throw new IllegalArgumentException("Not a primitive: "+declaredClass);
+      }
+    } else if (declaredClass.isArray()) {              // array
+      if (declaredClass.equals(byte [].class)) {
+        instance = Bytes.readByteArray(in);
+      } else if(declaredClass.equals(Result [].class)) {
+        instance = Result.readArray(in);
+      } else {
+        int length = in.readInt();
+        instance = Array.newInstance(declaredClass.getComponentType(), length);
+        for (int i = 0; i < length; i++) {
+          Array.set(instance, i, readObject(in, conf));
+        }
+      }
+    } else if (declaredClass.equals(Array.class)) { //an array not declared in CLASS_TO_CODE
+      Class componentType = readClass(conf, in);
+      int length = in.readInt();
+      instance = Array.newInstance(componentType, length);
+      for (int i = 0; i < length; i++) {
+        Array.set(instance, i, readObject(in, conf));
+      }
+    } else if (List.class.isAssignableFrom(declaredClass)) {            // List
+      int length = in.readInt();
+      instance = new ArrayList(length);
+      for (int i = 0; i < length; i++) {
+        ((ArrayList)instance).add(readObject(in, conf));
+      }
+    } else if (declaredClass == String.class) {        // String
+      instance = Text.readString(in);
+    } else if (declaredClass.isEnum()) {         // enum
+      instance = Enum.valueOf((Class) declaredClass,
+        Text.readString(in));
+    } else if (declaredClass == Message.class) {
+      String className = Text.readString(in);
+      try {
+        declaredClass = getClassByName(conf, className);
+        instance = tryInstantiateProtobuf(declaredClass, in);
+      } catch (ClassNotFoundException e) {
+        LOG.error("Can't find class " + className, e);
+        throw new IOException("Can't find class " + className, e);
+      }
+    } else {                                      // Writable or Serializable
+      Class instanceClass = null;
+      int b = (byte)WritableUtils.readVInt(in);
+      if (b == NOT_ENCODED) {
+        String className = Text.readString(in);
+        try {
+          instanceClass = getClassByName(conf, className);
+        } catch (ClassNotFoundException e) {
+          LOG.error("Can't find class " + className, e);
+          throw new IOException("Can't find class " + className, e);
+        }
+      } else {
+        instanceClass = CODE_TO_CLASS.get(b);
+      }
+      if(Writable.class.isAssignableFrom(instanceClass)){
+        Writable writable = WritableFactories.newInstance(instanceClass, conf);
+        try {
+          writable.readFields(in);
+        } catch (Exception e) {
+          LOG.error("Error in readFields", e);
+          throw new IOException("Error in readFields" , e);
+        }
+        instance = writable;
+        if (instanceClass == NullInstance.class) {  // null
+          declaredClass = ((NullInstance)instance).declaredClass;
+          instance = null;
+        }
+      } else {
+        int length = in.readInt();
+        byte[] objectBytes = new byte[length];
+        in.readFully(objectBytes);
+        ByteArrayInputStream bis = null;
+        ObjectInputStream ois = null;
+        try {
+          bis = new ByteArrayInputStream(objectBytes);
+          ois = new ObjectInputStream(bis);
+          instance = ois.readObject();
+        } catch (ClassNotFoundException e) {
+          LOG.error("Class not found when attempting to deserialize object", e);
+          throw new IOException("Class not found when attempting to " +
+              "deserialize object", e);
+        } finally {
+          if(bis!=null) bis.close();
+          if(ois!=null) ois.close();
+        }
+      }
+    }
+    if (objectWritable != null) {                 // store values
+      objectWritable.declaredClass = declaredClass;
+      objectWritable.instance = instance;
+    }
+    return instance;
+  }
+
+  /**
+   * Try to instantiate a protocol buffer of the given message class
+   * from the given input stream.
+   *
+   * @param protoClass the class of the generated protocol buffer
+   * @param dataIn the input stream to read from
+   * @return the instantiated Message instance
+   * @throws IOException if an IO problem occurs
+   */
+  private static Message tryInstantiateProtobuf(
+      Class protoClass,
+      DataInput dataIn) throws IOException {
+
+    try {
+      if (dataIn instanceof InputStream) {
+        // We can use the built-in parseDelimitedFrom and not have to re-copy
+        // the data
+        Method parseMethod = getStaticProtobufMethod(protoClass,
+            "parseDelimitedFrom", InputStream.class);
+        return (Message)parseMethod.invoke(null, (InputStream)dataIn);
+      } else {
+        // Have to read it into a buffer first, since protobuf doesn't deal
+        // with the DataInput interface directly.
+
+        // Read the size delimiter that writeDelimitedTo writes
+        int size = ProtoUtil.readRawVarint32(dataIn);
+        if (size < 0) {
+          throw new IOException("Invalid size: " + size);
+        }
+
+        byte[] data = new byte[size];
+        dataIn.readFully(data);
+        Method parseMethod = getStaticProtobufMethod(protoClass,
+            "parseFrom", byte[].class);
+        return (Message)parseMethod.invoke(null, data);
+      }
+    } catch (InvocationTargetException e) {
+
+      if (e.getCause() instanceof IOException) {
+        throw (IOException)e.getCause();
+      } else {
+        throw new IOException(e.getCause());
+      }
+    } catch (IllegalAccessException iae) {
+      throw new AssertionError("Could not access parse method in " +
+          protoClass);
+    }
+  }
+
+  static Method getStaticProtobufMethod(Class declaredClass, String method,
+      Class ... args) {
+
+    try {
+      return declaredClass.getMethod(method, args);
+    } catch (Exception e) {
+      // This is a bug in Hadoop - protobufs should all have this static method
+      throw new AssertionError("Protocol buffer class " + declaredClass +
+          " does not have an accessible parseFrom(InputStream) method!");
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  private static Class getClassByName(Configuration conf, String className)
+  throws ClassNotFoundException {
+    if(conf != null) {
+      return conf.getClassByName(className);
+    }
+    ClassLoader cl = Thread.currentThread().getContextClassLoader();
+    if(cl == null) {
+      cl = HbaseObjectWritable.class.getClassLoader();
+    }
+    return Class.forName(className, true, cl);
+  }
+
+  private static void addToMap(final Class clazz, final int code) {
+    CLASS_TO_CODE.put(clazz, code);
+    CODE_TO_CLASS.put(code, clazz);
+  }
+
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+
+  public Configuration getConf() {
+    return this.conf;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/HeapSize.java b/src/main/java/org/apache/hadoop/hbase/io/HeapSize.java
new file mode 100644
index 0000000..bd78846
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/HeapSize.java
@@ -0,0 +1,47 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io;
+
+/**
+ * Implementations can be asked for an estimate of their size in bytes.
+ * 
      
+ * Useful for sizing caches.  Its a given that implementation approximations
+ * do not account for 32 vs 64 bit nor for different VM implementations.
+ * 
      
+ * An Object's size is determined by the non-static data members in it,
+ * as well as the fixed {@link Object} overhead.
+ * 
      
+ * For example:
+ * 
      + * public class SampleObject implements HeapSize {
+ *
+ *   int [] numbers;
+ *   int x;
+ * }
+ * 
      
+ */
+public interface HeapSize {
+  /**
+   * @return Approximate 'exclusive deep size' of implementing object.  Includes
+   * count of payload and hosting object sizings.
+  */
+  public long heapSize();
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java b/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java
new file mode 100644
index 0000000..0cd5213
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java
@@ -0,0 +1,269 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import java.io.IOException;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+
+/**
+ * A byte sequence that is usable as a key or value.  Based on
+ * {@link org.apache.hadoop.io.BytesWritable} only this class is NOT resizable
+ * and DOES NOT distinguish between the size of the seqeunce and the current
+ * capacity as {@link org.apache.hadoop.io.BytesWritable} does. Hence its
+ * comparatively 'immutable'. When creating a new instance of this class,
+ * the underlying byte [] is not copied, just referenced.  The backing
+ * buffer is accessed when we go to serialize.
+ */
+public class ImmutableBytesWritable
+implements WritableComparable {
+  private byte[] bytes;
+  private int offset;
+  private int length;
+
+  /**
+   * Create a zero-size sequence.
+   */
+  public ImmutableBytesWritable() {
+    super();
+  }
+
+  /**
+   * Create a ImmutableBytesWritable using the byte array as the initial value.
+   * @param bytes This array becomes the backing storage for the object.
+   */
+  public ImmutableBytesWritable(byte[] bytes) {
+    this(bytes, 0, bytes.length);
+  }
+
+  /**
+   * Set the new ImmutableBytesWritable to the contents of the passed
+   * ibw.
+   * @param ibw the value to set this ImmutableBytesWritable to.
+   */
+  public ImmutableBytesWritable(final ImmutableBytesWritable ibw) {
+    this(ibw.get(), 0, ibw.getSize());
+  }
+
+  /**
+   * Set the value to a given byte range
+   * @param bytes the new byte range to set to
+   * @param offset the offset in newData to start at
+   * @param length the number of bytes in the range
+   */
+  public ImmutableBytesWritable(final byte[] bytes, final int offset,
+      final int length) {
+    this.bytes = bytes;
+    this.offset = offset;
+    this.length = length;
+  }
+
+  /**
+   * Get the data from the BytesWritable.
+   * @return The data is only valid between offset and offset+length.
+   */
+  public byte [] get() {
+    if (this.bytes == null) {
+      throw new IllegalStateException("Uninitialiized. Null constructor " +
+        "called w/o accompaying readFields invocation");
+    }
+    return this.bytes;
+  }
+
+  /**
+   * @param b Use passed bytes as backing array for this instance.
+   */
+  public void set(final byte [] b) {
+    set(b, 0, b.length);
+  }
+
+  /**
+   * @param b Use passed bytes as backing array for this instance.
+   * @param offset
+   * @param length
+   */
+  public void set(final byte [] b, final int offset, final int length) {
+    this.bytes = b;
+    this.offset = offset;
+    this.length = length;
+  }
+
+  /**
+   * @return the number of valid bytes in the buffer
+   */
+  public int getSize() {
+    if (this.bytes == null) {
+      throw new IllegalStateException("Uninitialiized. Null constructor " +
+        "called w/o accompaying readFields invocation");
+    }
+    return this.length;
+  }
+
+  /**
+   * @return the number of valid bytes in the buffer
+   */
+  //Should probably deprecate getSize() so that we keep the same calls for all
+  //byte []
+  public int getLength() {
+    if (this.bytes == null) {
+      throw new IllegalStateException("Uninitialiized. Null constructor " +
+        "called w/o accompaying readFields invocation");
+    }
+    return this.length;
+  }
+
+  /**
+   * @return offset
+   */
+  public int getOffset(){
+    return this.offset;
+  }
+
+  public void readFields(final DataInput in) throws IOException {
+    this.length = in.readInt();
+    this.bytes = new byte[this.length];
+    in.readFully(this.bytes, 0, this.length);
+    this.offset = 0;
+  }
+
+  public void write(final DataOutput out) throws IOException {
+    out.writeInt(this.length);
+    out.write(this.bytes, this.offset, this.length);
+  }
+
+  // Below methods copied from BytesWritable
+  @Override
+  public int hashCode() {
+    int hash = 1;
+    for (int i = offset; i < offset + length; i++)
+      hash = (31 * hash) + (int)bytes[i];
+    return hash;
+  }
+
+  /**
+   * Define the sort order of the BytesWritable.
+   * @param that The other bytes writable
+   * @return Positive if left is bigger than right, 0 if they are equal, and
+   *         negative if left is smaller than right.
+   */
+  public int compareTo(ImmutableBytesWritable that) {
+    return WritableComparator.compareBytes(
+      this.bytes, this.offset, this.length,
+      that.bytes, that.offset, that.length);
+  }
+
+  /**
+   * Compares the bytes in this object to the specified byte array
+   * @param that
+   * @return Positive if left is bigger than right, 0 if they are equal, and
+   *         negative if left is smaller than right.
+   */
+  public int compareTo(final byte [] that) {
+    return WritableComparator.compareBytes(
+      this.bytes, this.offset, this.length,
+      that, 0, that.length);
+  }
+
+  /**
+   * @see java.lang.Object#equals(java.lang.Object)
+   */
+  @Override
+  public boolean equals(Object right_obj) {
+    if (right_obj instanceof byte []) {
+      return compareTo((byte [])right_obj) == 0;
+    }
+    if (right_obj instanceof ImmutableBytesWritable) {
+      return compareTo((ImmutableBytesWritable)right_obj) == 0;
+    }
+    return false;
+  }
+
+  /**
+   * @see java.lang.Object#toString()
+   */
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder(3*this.bytes.length);
+    for (int idx = offset; idx < offset + length; idx++) {
+      // if not the first, put a blank separator in
+      if (idx != offset) {
+        sb.append(' ');
+      }
+      String num = Integer.toHexString(bytes[idx]);
+      // if it is only one digit, add a leading 0.
+      if (num.length() < 2) {
+        sb.append('0');
+      }
+      sb.append(num);
+    }
+    return sb.toString();
+  }
+
+  /** A Comparator optimized for ImmutableBytesWritable.
+   */
+  public static class Comparator extends WritableComparator {
+    private BytesWritable.Comparator comparator =
+      new BytesWritable.Comparator();
+
+    /** constructor */
+    public Comparator() {
+      super(ImmutableBytesWritable.class);
+    }
+
+    /**
+     * @see org.apache.hadoop.io.WritableComparator#compare(byte[], int, int, byte[], int, int)
+     */
+    @Override
+    public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+      return comparator.compare(b1, s1, l1, b2, s2, l2);
+    }
+  }
+
+  static { // register this comparator
+    WritableComparator.define(ImmutableBytesWritable.class, new Comparator());
+  }
+
+  /**
+   * @param array List of byte [].
+   * @return Array of byte [].
+   */
+  public static byte [][] toArray(final List array) {
+    // List#toArray doesn't work on lists of byte [].
+    byte[][] results = new byte[array.size()][];
+    for (int i = 0; i < array.size(); i++) {
+      results[i] = array.get(i);
+    }
+    return results;
+  }
+
+  /**
+   * Returns a copy of the bytes referred to by this writable
+   */
+  public byte[] copyBytes() {
+    return Arrays.copyOfRange(bytes, offset, offset+length);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/Reference.java b/src/main/java/org/apache/hadoop/hbase/io/Reference.java
new file mode 100644
index 0000000..06dc504
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/Reference.java
@@ -0,0 +1,156 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * A reference to the top or bottom half of a store file.  The file referenced
+ * lives under a different region.  References are made at region split time.
+ *
+ * 
      References work with a special half store file type.  References know how
+ * to write out the reference format in the file system and are whats juggled
+ * when references are mixed in with direct store files.  The half store file
+ * type is used reading the referred to file.
+ *
+ * 
      References to store files located over in some other region look like
+ * this in the file system
+ * 1278437856009925445.3323223323:
+ * i.e. an id followed by hash of the referenced region.
+ * Note, a region is itself not splitable if it has instances of store file
+ * references.  References are cleaned up by compactions.
+ */
+public class Reference implements Writable {
+  private byte [] splitkey;
+  private Range region;
+
+  /**
+   * For split HStoreFiles, it specifies if the file covers the lower half or
+   * the upper half of the key range
+   */
+  public static enum Range {
+    /** HStoreFile contains upper half of key range */
+    top,
+    /** HStoreFile contains lower half of key range */
+    bottom
+  }
+
+  /**
+   * Constructor
+   * @param splitRow This is row we are splitting around.
+   * @param fr
+   */
+  public Reference(final byte [] splitRow, final Range fr) {
+    this.splitkey = splitRow == null?
+      null: KeyValue.createFirstOnRow(splitRow).getKey();
+    this.region = fr;
+  }
+
+  /**
+   * Used by serializations.
+   */
+  public Reference() {
+    this(null, Range.bottom);
+  }
+
+  /**
+   *
+   * @return Range
+   */
+  public Range getFileRegion() {
+    return this.region;
+  }
+
+  /**
+   * @return splitKey
+   */
+  public byte [] getSplitKey() {
+    return splitkey;
+  }
+
+  /**
+   * @see java.lang.Object#toString()
+   */
+  @Override
+  public String toString() {
+    return "" + this.region;
+  }
+
+  // Make it serializable.
+
+  public void write(DataOutput out) throws IOException {
+    // Write true if we're doing top of the file.
+    out.writeBoolean(isTopFileRegion(this.region));
+    Bytes.writeByteArray(out, this.splitkey);
+  }
+
+  public void readFields(DataInput in) throws IOException {
+    boolean tmp = in.readBoolean();
+    // If true, set region to top.
+    this.region = tmp? Range.top: Range.bottom;
+    this.splitkey = Bytes.readByteArray(in);
+  }
+
+  public static boolean isTopFileRegion(final Range r) {
+    return r.equals(Range.top);
+  }
+
+  public Path write(final FileSystem fs, final Path p)
+  throws IOException {
+    FSDataOutputStream out = HBaseFileSystem.createPathOnFileSystem(fs, p, false);
+    try {
+      write(out);
+    } finally {
+      out.close();
+    }
+    return p;
+  }
+
+  /**
+   * Read a Reference from FileSystem.
+   * @param fs
+   * @param p
+   * @return New Reference made from passed p
+   * @throws IOException
+   */
+  public static Reference read(final FileSystem fs, final Path p)
+  throws IOException {
+    FSDataInputStream in = fs.open(p);
+    try {
+      Reference r = new Reference();
+      r.readFields(in);
+      return r;
+    } finally {
+      in.close();
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java b/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
new file mode 100644
index 0000000..12a9b68
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
@@ -0,0 +1,189 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Represents an interval of version timestamps.
+ * 
      
+ * Evaluated according to minStamp <= timestamp < maxStamp
+ * or [minStamp,maxStamp) in interval notation.
+ * 
      
+ * Only used internally; should not be accessed directly by clients.
+ */
+public class TimeRange implements Writable {
+  private long minStamp = 0L;
+  private long maxStamp = Long.MAX_VALUE;
+  private boolean allTime = false;
+
+  /**
+   * Default constructor.
+   * Represents interval [0, Long.MAX_VALUE) (allTime)
+   */
+  public TimeRange() {
+    allTime = true;
+  }
+
+  /**
+   * Represents interval [minStamp, Long.MAX_VALUE)
+   * @param minStamp the minimum timestamp value, inclusive
+   */
+  public TimeRange(long minStamp) {
+    this.minStamp = minStamp;
+  }
+
+  /**
+   * Represents interval [minStamp, Long.MAX_VALUE)
+   * @param minStamp the minimum timestamp value, inclusive
+   */
+  public TimeRange(byte [] minStamp) {
+  	this.minStamp = Bytes.toLong(minStamp);
+  }
+
+  /**
+   * Represents interval [minStamp, maxStamp)
+   * @param minStamp the minimum timestamp, inclusive
+   * @param maxStamp the maximum timestamp, exclusive
+   * @throws IOException
+   */
+  public TimeRange(long minStamp, long maxStamp)
+  throws IOException {
+    if(maxStamp < minStamp) {
+      throw new IOException("maxStamp is smaller than minStamp");
+    }
+    this.minStamp = minStamp;
+    this.maxStamp = maxStamp;
+  }
+
+  /**
+   * Represents interval [minStamp, maxStamp)
+   * @param minStamp the minimum timestamp, inclusive
+   * @param maxStamp the maximum timestamp, exclusive
+   * @throws IOException
+   */
+  public TimeRange(byte [] minStamp, byte [] maxStamp)
+  throws IOException {
+    this(Bytes.toLong(minStamp), Bytes.toLong(maxStamp));
+  }
+
+  /**
+   * @return the smallest timestamp that should be considered
+   */
+  public long getMin() {
+    return minStamp;
+  }
+
+  /**
+   * @return the biggest timestamp that should be considered
+   */
+  public long getMax() {
+    return maxStamp;
+  }
+
+  /**
+   * Check if the specified timestamp is within this TimeRange.
+   * 
      
+   * Returns true if within interval [minStamp, maxStamp), false
+   * if not.
+   * @param bytes timestamp to check
+   * @param offset offset into the bytes
+   * @return true if within TimeRange, false if not
+   */
+  public boolean withinTimeRange(byte [] bytes, int offset) {
+  	if(allTime) return true;
+  	return withinTimeRange(Bytes.toLong(bytes, offset));
+  }
+
+  /**
+   * Check if the specified timestamp is within this TimeRange.
+   * 
      
+   * Returns true if within interval [minStamp, maxStamp), false
+   * if not.
+   * @param timestamp timestamp to check
+   * @return true if within TimeRange, false if not
+   */
+  public boolean withinTimeRange(long timestamp) {
+  	if(allTime) return true;
+  	// check if >= minStamp
+  	return (minStamp <= timestamp && timestamp < maxStamp);
+  }
+
+  /**
+   * Check if the specified timestamp is within this TimeRange.
+   * 
      
+   * Returns true if within interval [minStamp, maxStamp), false
+   * if not.
+   * @param timestamp timestamp to check
+   * @return true if within TimeRange, false if not
+   */
+  public boolean withinOrAfterTimeRange(long timestamp) {
+    if(allTime) return true;
+    // check if >= minStamp
+    return (timestamp >= minStamp);
+  }
+
+  /**
+   * Compare the timestamp to timerange
+   * @param timestamp
+   * @return -1 if timestamp is less than timerange,
+   * 0 if timestamp is within timerange,
+   * 1 if timestamp is greater than timerange
+   */
+  public int compare(long timestamp) {
+    if (timestamp < minStamp) {
+      return -1;
+    } else if (timestamp >= maxStamp) {
+      return 1;
+    } else {
+      return 0;
+    }
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("maxStamp=");
+    sb.append(this.maxStamp);
+    sb.append(", minStamp=");
+    sb.append(this.minStamp);
+    return sb.toString();
+  }
+
+  //Writable
+  public void readFields(final DataInput in) throws IOException {
+    this.minStamp = in.readLong();
+    this.maxStamp = in.readLong();
+    this.allTime = in.readBoolean();
+  }
+
+  public void write(final DataOutput out) throws IOException {
+    out.writeLong(minStamp);
+    out.writeLong(maxStamp);
+    out.writeBoolean(this.allTime);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/WritableWithSize.java b/src/main/java/org/apache/hadoop/hbase/io/WritableWithSize.java
new file mode 100644
index 0000000..f8aefa1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/WritableWithSize.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io;
+
+/**
+ * An optional interface to 'size' writables.
+ */
+public interface WritableWithSize {
+  /**
+   * Provide a size hint to the caller. write() should ideally
+   * not go beyond this if at all possible.
+   *
+   * You can return 0 if there is no size hint.
+   *
+   * @return the size of the writable
+   */
+  public long getWritableSize();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
new file mode 100644
index 0000000..1dedbbd
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.encoding;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
+import org.apache.hadoop.hbase.util.ByteBufferUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * Base class for all data block encoders that use a buffer.
+ */
+abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
+
+  private static int INITIAL_KEY_BUFFER_SIZE = 512;
+
+  @Override
+  public ByteBuffer uncompressKeyValues(DataInputStream source,
+      boolean includesMemstoreTS) throws IOException {
+    return uncompressKeyValues(source, 0, 0, includesMemstoreTS);
+  }
+
+  protected static class SeekerState {
+    protected int valueOffset = -1;
+    protected int keyLength;
+    protected int valueLength;
+    protected int lastCommonPrefix;
+
+    /** We need to store a copy of the key. */
+    protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
+
+    protected long memstoreTS;
+    protected int nextKvOffset;
+
+    protected boolean isValid() {
+      return valueOffset != -1;
+    }
+
+    protected void invalidate() {
+      valueOffset = -1;
+    }
+
+    protected void ensureSpaceForKey() {
+      if (keyLength > keyBuffer.length) {
+        // rare case, but we need to handle arbitrary length of key
+        int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2;
+        while (keyLength > newKeyBufferLength) {
+          newKeyBufferLength *= 2;
+        }
+        byte[] newKeyBuffer = new byte[newKeyBufferLength];
+        System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length);
+        keyBuffer = newKeyBuffer;
+      }
+    }
+
+    /**
+     * Copy the state from the next one into this instance (the previous state
+     * placeholder). Used to save the previous state when we are advancing the
+     * seeker to the next key/value.
+     */
+    protected void copyFromNext(SeekerState nextState) {
+      if (keyBuffer.length != nextState.keyBuffer.length) {
+        keyBuffer = nextState.keyBuffer.clone();
+      } else if (!isValid()) {
+        // Note: we can only call isValid before we override our state, so this
+        // comes before all the assignments at the end of this method.
+        System.arraycopy(nextState.keyBuffer, 0, keyBuffer, 0,
+             nextState.keyLength);
+      } else {
+        // don't copy the common prefix between this key and the previous one
+        System.arraycopy(nextState.keyBuffer, nextState.lastCommonPrefix,
+            keyBuffer, nextState.lastCommonPrefix, nextState.keyLength
+                - nextState.lastCommonPrefix);
+      }
+
+      valueOffset = nextState.valueOffset;
+      keyLength = nextState.keyLength;
+      valueLength = nextState.valueLength;
+      lastCommonPrefix = nextState.lastCommonPrefix;
+      nextKvOffset = nextState.nextKvOffset;
+      memstoreTS = nextState.memstoreTS;
+    }
+
+  }
+
+  protected abstract static class
+      BufferedEncodedSeeker
+      implements EncodedSeeker {
+
+    protected final RawComparator comparator;
+    protected final SamePrefixComparator samePrefixComparator;
+    protected ByteBuffer currentBuffer;
+    protected STATE current = createSeekerState(); // always valid
+    protected STATE previous = createSeekerState(); // may not be valid
+
+    @SuppressWarnings("unchecked")
+    public BufferedEncodedSeeker(RawComparator comparator) {
+      this.comparator = comparator;
+      if (comparator instanceof SamePrefixComparator) {
+        this.samePrefixComparator = (SamePrefixComparator) comparator;
+      } else {
+        this.samePrefixComparator = null;
+      }
+    }
+
+    @Override
+    public void setCurrentBuffer(ByteBuffer buffer) {
+      currentBuffer = buffer;
+      decodeFirst();
+      previous.invalidate();
+    }
+
+    @Override
+    public ByteBuffer getKeyDeepCopy() {
+      ByteBuffer keyBuffer = ByteBuffer.allocate(current.keyLength);
+      keyBuffer.put(current.keyBuffer, 0, current.keyLength);
+      return keyBuffer;
+    }
+
+    @Override
+    public ByteBuffer getValueShallowCopy() {
+      return ByteBuffer.wrap(currentBuffer.array(),
+          currentBuffer.arrayOffset() + current.valueOffset,
+          current.valueLength);
+    }
+
+    @Override
+    public ByteBuffer getKeyValueBuffer() {
+      ByteBuffer kvBuffer = ByteBuffer.allocate(
+          2 * Bytes.SIZEOF_INT + current.keyLength + current.valueLength);
+      kvBuffer.putInt(current.keyLength);
+      kvBuffer.putInt(current.valueLength);
+      kvBuffer.put(current.keyBuffer, 0, current.keyLength);
+      kvBuffer.put(currentBuffer.array(),
+          currentBuffer.arrayOffset() + current.valueOffset,
+          current.valueLength);
+      return kvBuffer;
+    }
+
+    @Override
+    public KeyValue getKeyValue() {
+      ByteBuffer kvBuf = getKeyValueBuffer();
+      KeyValue kv = new KeyValue(kvBuf.array(), kvBuf.arrayOffset());
+      kv.setMemstoreTS(current.memstoreTS);
+      return kv;
+    }
+
+    @Override
+    public void rewind() {
+      currentBuffer.rewind();
+      decodeFirst();
+      previous.invalidate();
+    }
+
+    @Override
+    public boolean next() {
+      if (!currentBuffer.hasRemaining()) {
+        return false;
+      }
+      decodeNext();
+      previous.invalidate();
+      return true;
+    }
+
+    @Override
+    public int seekToKeyInBlock(byte[] key, int offset, int length,
+        boolean seekBefore) {
+      int commonPrefix = 0;
+      previous.invalidate();
+      do {
+        int comp;
+        if (samePrefixComparator != null) {
+          commonPrefix = Math.min(commonPrefix, current.lastCommonPrefix);
+
+          // extend commonPrefix
+          commonPrefix += ByteBufferUtils.findCommonPrefix(
+              key, offset + commonPrefix, length - commonPrefix,
+              current.keyBuffer, commonPrefix,
+              current.keyLength - commonPrefix);
+
+          comp = samePrefixComparator.compareIgnoringPrefix(commonPrefix, key,
+              offset, length, current.keyBuffer, 0, current.keyLength);
+        } else {
+          comp = comparator.compare(key, offset, length,
+              current.keyBuffer, 0, current.keyLength);
+        }
+
+        if (comp == 0) { // exact match
+          if (seekBefore) {
+            if (!previous.isValid()) {
+              // The caller (seekBefore) has to ensure that we are not at the
+              // first key in the block.
+              throw new IllegalStateException("Cannot seekBefore if " +
+                  "positioned at the first key in the block: key=" +
+                  Bytes.toStringBinary(key, offset, length));
+            }
+            moveToPrevious();
+            return 1;
+          }
+          return 0;
+        }
+
+        if (comp < 0) { // already too large, check previous
+          if (previous.isValid()) {
+            moveToPrevious();
+          }
+          return 1;
+        }
+
+        // move to next, if more data is available
+        if (currentBuffer.hasRemaining()) {
+          previous.copyFromNext(current);
+          decodeNext();
+        } else {
+          break;
+        }
+      } while (true);
+
+      // we hit the end of the block, not an exact match
+      return 1;
+    }
+
+    private void moveToPrevious() {
+      if (!previous.isValid()) {
+        throw new IllegalStateException(
+            "Can move back only once and not in first key in the block.");
+      }
+
+      STATE tmp = previous;
+      previous = current;
+      current = tmp;
+
+      // move after last key value
+      currentBuffer.position(current.nextKvOffset);
+
+      previous.invalidate();
+    }
+
+    @SuppressWarnings("unchecked")
+    protected STATE createSeekerState() {
+      // This will fail for non-default seeker state if the subclass does not
+      // override this method.
+      return (STATE) new SeekerState();
+    }
+
+    abstract protected void decodeFirst();
+    abstract protected void decodeNext();
+  }
+
+  protected final void afterEncodingKeyValue(ByteBuffer in,
+      DataOutputStream out, boolean includesMemstoreTS) {
+    if (includesMemstoreTS) {
+      // Copy memstore timestamp from the byte buffer to the output stream.
+      long memstoreTS = -1;
+      try {
+        memstoreTS = ByteBufferUtils.readVLong(in);
+        WritableUtils.writeVLong(out, memstoreTS);
+      } catch (IOException ex) {
+        throw new RuntimeException("Unable to copy memstore timestamp " +
+            memstoreTS + " after encoding a key/value");
+      }
+    }
+  }
+
+  protected final void afterDecodingKeyValue(DataInputStream source,
+      ByteBuffer dest, boolean includesMemstoreTS) {
+    if (includesMemstoreTS) {
+      long memstoreTS = -1;
+      try {
+        // Copy memstore timestamp from the data input stream to the byte
+        // buffer.
+        memstoreTS = WritableUtils.readVLong(source);
+        ByteBufferUtils.writeVLong(dest, memstoreTS);
+      } catch (IOException ex) {
+        throw new RuntimeException("Unable to copy memstore timestamp " +
+            memstoreTS + " after decoding a key/value");
+      }
+    }
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/CompressionState.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/CompressionState.java
new file mode 100644
index 0000000..319f782
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/CompressionState.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.encoding;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.ByteBufferUtils;
+
+/**
+ * Stores the state of data block encoder at the beginning of new key.
+ */
+class CompressionState {
+  int keyLength;
+  int valueLength;
+
+  short rowLength;
+  int prevOffset = FIRST_KEY;
+  byte familyLength;
+  int qualifierLength;
+  byte type;
+
+  private final static int FIRST_KEY = -1;
+
+  boolean isFirst() {
+    return prevOffset == FIRST_KEY;
+  }
+
+  /**
+   * Analyze the key and fill the state.
+   * Uses mark() and reset() in ByteBuffer.
+   * @param in Buffer at the position where key starts
+   * @param keyLength Length of key in bytes
+   * @param valueLength Length of values in bytes
+   */
+  void readKey(ByteBuffer in, int keyLength, int valueLength) {
+    readKey(in, keyLength, valueLength, 0, null);
+  }
+
+  /** 
+   * Analyze the key and fill the state assuming we know previous state.
+   * Uses mark() and reset() in ByteBuffer to avoid moving the position.
+   * 
      
+   * This method overrides all the fields of this instance, except
+   * {@link #prevOffset}, which is usually manipulated directly by encoders
+   * and decoders.
+   * @param in Buffer at the position where key starts
+   * @param keyLength Length of key in bytes
+   * @param valueLength Length of values in bytes
+   * @param commonPrefix how many first bytes are common with previous KeyValue
+   * @param previousState State from previous KeyValue
+   */
+  void readKey(ByteBuffer in, int keyLength, int valueLength,
+      int commonPrefix, CompressionState previousState) {
+    this.keyLength = keyLength;
+    this.valueLength = valueLength;
+
+    // fill the state
+    in.mark(); // mark beginning of key
+
+    if (commonPrefix < KeyValue.ROW_LENGTH_SIZE) {
+      rowLength = in.getShort();
+      ByteBufferUtils.skip(in, rowLength);
+
+      familyLength = in.get();
+
+      qualifierLength = keyLength - rowLength - familyLength -
+          KeyValue.KEY_INFRASTRUCTURE_SIZE;
+      ByteBufferUtils.skip(in, familyLength + qualifierLength);
+    } else {
+      rowLength = previousState.rowLength;
+      familyLength = previousState.familyLength;
+      qualifierLength = previousState.qualifierLength +
+          keyLength - previousState.keyLength;
+      ByteBufferUtils.skip(in, (KeyValue.ROW_LENGTH_SIZE +
+          KeyValue.FAMILY_LENGTH_SIZE) +
+          rowLength + familyLength + qualifierLength);
+    }
+
+    readTimestamp(in);
+
+    type = in.get();
+
+    in.reset();
+  }
+
+  protected void readTimestamp(ByteBuffer in) {
+    // used in subclasses to add timestamp to state
+    ByteBufferUtils.skip(in, KeyValue.TIMESTAMP_SIZE);
+  }
+
+  void copyFrom(CompressionState state) {
+    keyLength = state.keyLength;
+    valueLength = state.valueLength;
+
+    rowLength = state.rowLength;
+    prevOffset = state.prevOffset;
+    familyLength = state.familyLength;
+    qualifierLength = state.qualifierLength;
+    type = state.type;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
new file mode 100644
index 0000000..548985b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.encoding;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.util.ByteBufferUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.RawComparator;
+
+/**
+ * Just copy data, do not do any kind of compression. Use for comparison and
+ * benchmarking.
+ */
+public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder {
+  @Override
+  public void compressKeyValues(DataOutputStream out,
+      ByteBuffer in, boolean includesMemstoreTS) throws IOException {
+    in.rewind();
+    ByteBufferUtils.putInt(out, in.limit());
+    ByteBufferUtils.moveBufferToStream(out, in, in.limit());
+  }
+
+  @Override
+  public ByteBuffer uncompressKeyValues(DataInputStream source,
+      int preserveHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
+      throws IOException {
+    int decompressedSize = source.readInt();
+    ByteBuffer buffer = ByteBuffer.allocate(decompressedSize +
+        preserveHeaderLength);
+    buffer.position(preserveHeaderLength);
+    ByteBufferUtils.copyFromStreamToBuffer(buffer, source, decompressedSize);
+
+    return buffer;
+  }
+
+  @Override
+  public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
+    int keyLength = block.getInt(Bytes.SIZEOF_INT);
+    return ByteBuffer.wrap(block.array(),
+        block.arrayOffset() + 3 * Bytes.SIZEOF_INT, keyLength).slice();
+  }
+
+
+  @Override
+  public String toString() {
+    return CopyKeyDataBlockEncoder.class.getSimpleName();
+  }
+
+  @Override
+  public EncodedSeeker createSeeker(RawComparator comparator,
+      final boolean includesMemstoreTS) {
+    return new BufferedEncodedSeeker(comparator) {
+      @Override
+      protected void decodeNext() {
+        current.keyLength = currentBuffer.getInt();
+        current.valueLength = currentBuffer.getInt();
+        current.ensureSpaceForKey();
+        currentBuffer.get(current.keyBuffer, 0, current.keyLength);
+        current.valueOffset = currentBuffer.position();
+        ByteBufferUtils.skip(currentBuffer, current.valueLength);
+        if (includesMemstoreTS) {
+          current.memstoreTS = ByteBufferUtils.readVLong(currentBuffer);
+        } else {
+          current.memstoreTS = 0;
+        }
+        current.nextKvOffset = currentBuffer.position();
+      }
+
+      @Override
+      protected void decodeFirst() {
+        ByteBufferUtils.skip(currentBuffer, Bytes.SIZEOF_INT);
+        current.lastCommonPrefix = 0;
+        decodeNext();
+      }
+    };
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
new file mode 100644
index 0000000..b2f2319
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.encoding;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.io.RawComparator;
+
+/**
+ * Encoding of KeyValue. It aims to be fast and efficient using assumptions:
+ * 
        
+ * 
      • the KeyValues are stored sorted by key
        • 
+ * 
      • we know the structure of KeyValue
        • 
+ * 
      • the values are always iterated forward from beginning of block
        • 
+ * 
      • knowledge of Key Value format
        • 
+ * 
      
+ * It is designed to work fast enough to be feasible as in memory compression.
+ */
+public interface DataBlockEncoder {
+  /**
+   * Compress KeyValues and write them to output buffer.
+   * @param out Where to write compressed data.
+   * @param in Source of KeyValue for compression.
+   * @param includesMemstoreTS true if including memstore timestamp after every
+   *          key-value pair
+   * @throws IOException If there is an error writing to output stream.
+   */
+  public void compressKeyValues(DataOutputStream out,
+      ByteBuffer in, boolean includesMemstoreTS) throws IOException;
+
+  /**
+   * Uncompress.
+   * @param source Compressed stream of KeyValues.
+   * @param includesMemstoreTS true if including memstore timestamp after every
+   *          key-value pair
+   * @return Uncompressed block of KeyValues.
+   * @throws IOException If there is an error in source.
+   */
+  public ByteBuffer uncompressKeyValues(DataInputStream source,
+      boolean includesMemstoreTS) throws IOException;
+
+  /**
+   * Uncompress.
+   * @param source Compressed stream of KeyValues.
+   * @param allocateHeaderLength allocate this many bytes for the header.
+   * @param skipLastBytes Do not copy n last bytes.
+   * @param includesMemstoreTS true if including memstore timestamp after every
+   *          key-value pair
+   * @return Uncompressed block of KeyValues.
+   * @throws IOException If there is an error in source.
+   */
+  public ByteBuffer uncompressKeyValues(DataInputStream source,
+      int allocateHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
+      throws IOException;
+
+  /**
+   * Return first key in block. Useful for indexing. Typically does not make
+   * a deep copy but returns a buffer wrapping a segment of the actual block's
+   * byte array. This is because the first key in block is usually stored
+   * unencoded.
+   * @param block encoded block we want index, the position will not change
+   * @return First key in block.
+   */
+  public ByteBuffer getFirstKeyInBlock(ByteBuffer block);
+
+  /**
+   * Create a HFileBlock seeker which find KeyValues within a block.
+   * @param comparator what kind of comparison should be used
+   * @param includesMemstoreTS true if including memstore timestamp after every
+   *          key-value pair
+   * @return A newly created seeker.
+   */
+  public EncodedSeeker createSeeker(RawComparator comparator,
+      boolean includesMemstoreTS);
+
+  /**
+   * An interface which enable to seek while underlying data is encoded.
+   *
+   * It works on one HFileBlock, but it is reusable. See
+   * {@link #setCurrentBuffer(ByteBuffer)}.
+   */
+  public static interface EncodedSeeker {
+    /**
+     * Set on which buffer there will be done seeking.
+     * @param buffer Used for seeking.
+     */
+    public void setCurrentBuffer(ByteBuffer buffer);
+
+    /**
+     * Does a deep copy of the key at the current position. A deep copy is
+     * necessary because buffers are reused in the decoder.
+     * @return key at current position
+     */
+    public ByteBuffer getKeyDeepCopy();
+
+    /**
+     * Does a shallow copy of the value at the current position. A shallow
+     * copy is possible because the returned buffer refers to the backing array
+     * of the original encoded buffer.
+     * @return value at current position
+     */
+    public ByteBuffer getValueShallowCopy();
+
+    /** @return key value at current position. */
+    public ByteBuffer getKeyValueBuffer();
+
+    /**
+     * @return the KeyValue object at the current position. Includes memstore
+     *         timestamp.
+     */
+    public KeyValue getKeyValue();
+
+    /** Set position to beginning of given block */
+    public void rewind();
+
+    /**
+     * Move to next position
+     * @return true on success, false if there is no more positions.
+     */
+    public boolean next();
+
+    /**
+     * Moves the seeker position within the current block to:
+     * 
        
+     * 
      • the last key that that is less than or equal to the given key if
+     * seekBefore is false
        • 
+     * 
      • the last key that is strictly less than the given key if 
+     * seekBefore is true. The caller is responsible for loading the
+     * previous block if the requested key turns out to be the first key of the
+     * current block.
        • 
+     * 
      
+     * @param key byte array containing the key
+     * @param offset key position the array
+     * @param length key length in bytes
+     * @param seekBefore find the key strictly less than the given key in case
+     *          of an exact match. Does not matter in case of an inexact match.
+     * @return 0 on exact match, 1 on inexact match.
+     */
+    public int seekToKeyInBlock(byte[] key, int offset, int length,
+        boolean seekBefore);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoding.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoding.java
new file mode 100644
index 0000000..963b5da
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoding.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.encoding;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Provide access to all data block encoding algorithms. All of the algorithms
+ * are required to have unique id which should NEVER be changed. If you
+ * want to add a new algorithm/version, assign it a new id. Announce the new id
+ * in the HBase mailing list to prevent collisions.
+ */
+public enum DataBlockEncoding {
+
+  /** Disable data block encoding. */
+  NONE(0, null),
+  // id 1 is reserved for the BITSET algorithm to be added later
+  PREFIX(2, new PrefixKeyDeltaEncoder()),
+  DIFF(3, new DiffKeyDeltaEncoder()),
+  FAST_DIFF(4, new FastDiffDeltaEncoder());
+
+  private final short id;
+  private final byte[] idInBytes;
+  private final DataBlockEncoder encoder;
+
+  public static final int ID_SIZE = Bytes.SIZEOF_SHORT;
+
+  /** Maps data block encoding ids to enum instances. */
+  private static Map idToEncoding =
+      new HashMap();
+
+  static {
+    for (DataBlockEncoding algo : values()) {
+      if (idToEncoding.containsKey(algo.id)) {
+        throw new RuntimeException(String.format(
+            "Two data block encoder algorithms '%s' and '%s' have " +
+            "the same id %d",
+            idToEncoding.get(algo.id).toString(), algo.toString(),
+            (int) algo.id));
+      }
+      idToEncoding.put(algo.id, algo);
+    }
+  }
+
+  private DataBlockEncoding(int id, DataBlockEncoder encoder) {
+    if (id < Short.MIN_VALUE || id > Short.MAX_VALUE) {
+      throw new AssertionError(
+          "Data block encoding algorithm id is out of range: " + id);
+    }
+    this.id = (short) id;
+    this.idInBytes = Bytes.toBytes(this.id);
+    if (idInBytes.length != ID_SIZE) {
+      // White this may seem redundant, if we accidentally serialize
+      // the id as e.g. an int instead of a short, all encoders will break.
+      throw new RuntimeException("Unexpected length of encoder ID byte " +
+          "representation: " + Bytes.toStringBinary(idInBytes));
+    }
+    this.encoder = encoder;
+  }
+
+  /**
+   * @return name converted to bytes.
+   */
+  public byte[] getNameInBytes() {
+    return Bytes.toBytes(toString());
+  }
+
+  /**
+   * @return The id of a data block encoder.
+   */
+  public short getId() {
+    return id;
+  }
+
+  /**
+   * Writes id in bytes.
+   * @param stream where the id should be written.
+   */
+  public void writeIdInBytes(OutputStream stream) throws IOException {
+    stream.write(idInBytes);
+  }
+
+  /**
+   * Return new data block encoder for given algorithm type.
+   * @return data block encoder if algorithm is specified, null if none is
+   *         selected.
+   */
+  public DataBlockEncoder getEncoder() {
+    return encoder;
+  }
+
+  /**
+   * Provide access to all data block encoders, even those which are not
+   * exposed in the enum. Useful for testing and benchmarking.
+   * @return list of all data block encoders.
+   */
+  public static List getAllEncoders() {
+    ArrayList encoders = new ArrayList();
+    for (DataBlockEncoding algo : values()) {
+      DataBlockEncoder encoder = algo.getEncoder();
+      if (encoder != null) {
+        encoders.add(encoder);
+      }
+    }
+
+    // Add encoders that are only used in testing.
+    encoders.add(new CopyKeyDataBlockEncoder());
+    return encoders;
+  }
+
+  /**
+   * Find and create data block encoder for given id;
+   * @param encoderId id of data block encoder.
+   * @return Newly created data block encoder.
+   */
+  public static DataBlockEncoder getDataBlockEncoderById(short encoderId) {
+    if (!idToEncoding.containsKey(encoderId)) {
+      throw new IllegalArgumentException(String.format(
+          "There is no data block encoder for given id '%d'",
+          (int) encoderId));
+    }
+
+    return idToEncoding.get(encoderId).getEncoder();
+  }
+
+  /**
+   * Find and return the name of data block encoder for the given id.
+   * @param encoderId id of data block encoder
+   * @return name, same as used in options in column family
+   */
+  public static String getNameFromId(short encoderId) {
+    return idToEncoding.get(encoderId).toString();
+  }
+
+  /**
+   * Check if given encoder has this id.
+   * @param encoder encoder which id will be checked
+   * @param encoderId id which we except
+   * @return true if id is right for given encoder, false otherwise
+   * @exception IllegalArgumentException
+   *            thrown when there is no matching data block encoder
+   */
+  public static boolean isCorrectEncoder(DataBlockEncoder encoder,
+      short encoderId) {
+    if (!idToEncoding.containsKey(encoderId)) {
+      throw new IllegalArgumentException(String.format(
+          "There is no data block encoder for given id '%d'",
+          (int) encoderId));
+    }
+
+    DataBlockEncoding algorithm = idToEncoding.get(encoderId);
+    return algorithm.getClass().equals(encoder.getClass());
+  }
+
+  public static DataBlockEncoding getEncodingById(short dataBlockEncodingId) {
+    return idToEncoding.get(dataBlockEncodingId);
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
new file mode 100644
index 0000000..7e51818
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
@@ -0,0 +1,549 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.encoding;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.ByteBufferUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.RawComparator;
+
+/**
+ * Compress using:
+ * - store size of common prefix
+ * - save column family once, it is same within HFile
+ * - use integer compression for key, value and prefix (7-bit encoding)
+ * - use bits to avoid duplication key length, value length
+ *   and type if it same as previous
+ * - store in 3 bits length of timestamp field
+ * - allow diff in timestamp instead of actual value
+ *
+ * Format:
+ * - 1 byte:    flag
+ * - 1-5 bytes: key length (only if FLAG_SAME_KEY_LENGTH is not set in flag)
+ * - 1-5 bytes: value length (only if FLAG_SAME_VALUE_LENGTH is not set in flag)
+ * - 1-5 bytes: prefix length
+ * - ... bytes: rest of the row (if prefix length is small enough)
+ * - ... bytes: qualifier (or suffix depending on prefix length)
+ * - 1-8 bytes: timestamp or diff
+ * - 1 byte:    type (only if FLAG_SAME_TYPE is not set in the flag)
+ * - ... bytes: value
+ */
+public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
+  static final int FLAG_SAME_KEY_LENGTH = 1;
+  static final int FLAG_SAME_VALUE_LENGTH = 1 << 1;
+  static final int FLAG_SAME_TYPE = 1 << 2;
+  static final int FLAG_TIMESTAMP_IS_DIFF = 1 << 3;
+  static final int MASK_TIMESTAMP_LENGTH = (1 << 4) | (1 << 5) | (1 << 6);
+  static final int SHIFT_TIMESTAMP_LENGTH = 4;
+  static final int FLAG_TIMESTAMP_SIGN = 1 << 7;
+
+  protected static class DiffCompressionState extends CompressionState {
+    long timestamp;
+    byte[] familyNameWithSize;
+
+    @Override
+    protected void readTimestamp(ByteBuffer in) {
+      timestamp = in.getLong();
+    }
+
+    @Override
+    void copyFrom(CompressionState state) {
+      super.copyFrom(state);
+      DiffCompressionState state2 = (DiffCompressionState) state;
+      timestamp = state2.timestamp;
+    }
+  }
+
+  private void compressSingleKeyValue(DiffCompressionState previousState,
+      DiffCompressionState currentState, DataOutputStream out,
+      ByteBuffer in) throws IOException {
+    byte flag = 0;
+    int kvPos = in.position();
+    int keyLength = in.getInt();
+    int valueLength = in.getInt();
+
+    long timestamp;
+    long diffTimestamp = 0;
+    int diffTimestampFitsInBytes = 0;
+
+    int commonPrefix;
+
+    int timestampFitsInBytes;
+
+    if (previousState.isFirst()) {
+      currentState.readKey(in, keyLength, valueLength);
+      currentState.prevOffset = kvPos;
+      timestamp = currentState.timestamp;
+      if (timestamp < 0) {
+        flag |= FLAG_TIMESTAMP_SIGN;
+        timestamp = -timestamp;
+      }
+      timestampFitsInBytes = ByteBufferUtils.longFitsIn(timestamp);
+
+      flag |= (timestampFitsInBytes - 1) << SHIFT_TIMESTAMP_LENGTH;
+      commonPrefix = 0;
+
+      // put column family
+      in.mark();
+      ByteBufferUtils.skip(in, currentState.rowLength
+          + KeyValue.ROW_LENGTH_SIZE);
+      ByteBufferUtils.moveBufferToStream(out, in, currentState.familyLength
+          + KeyValue.FAMILY_LENGTH_SIZE);
+      in.reset();
+    } else {
+      // find a common prefix and skip it
+      commonPrefix =
+          ByteBufferUtils.findCommonPrefix(in, in.position(),
+              previousState.prevOffset + KeyValue.ROW_OFFSET, keyLength
+                  - KeyValue.TIMESTAMP_TYPE_SIZE);
+      // don't compress timestamp and type using prefix
+
+      currentState.readKey(in, keyLength, valueLength,
+          commonPrefix, previousState);
+      currentState.prevOffset = kvPos;
+      timestamp = currentState.timestamp;
+      boolean negativeTimestamp = timestamp < 0;
+      if (negativeTimestamp) {
+        timestamp = -timestamp;
+      }
+      timestampFitsInBytes = ByteBufferUtils.longFitsIn(timestamp);
+
+      if (keyLength == previousState.keyLength) {
+        flag |= FLAG_SAME_KEY_LENGTH;
+      }
+      if (valueLength == previousState.valueLength) {
+        flag |= FLAG_SAME_VALUE_LENGTH;
+      }
+      if (currentState.type == previousState.type) {
+        flag |= FLAG_SAME_TYPE;
+      }
+
+      // encode timestamp
+      diffTimestamp = previousState.timestamp - currentState.timestamp;
+      boolean minusDiffTimestamp = diffTimestamp < 0;
+      if (minusDiffTimestamp) {
+        diffTimestamp = -diffTimestamp;
+      }
+      diffTimestampFitsInBytes = ByteBufferUtils.longFitsIn(diffTimestamp);
+      if (diffTimestampFitsInBytes < timestampFitsInBytes) {
+        flag |= (diffTimestampFitsInBytes - 1) << SHIFT_TIMESTAMP_LENGTH;
+        flag |= FLAG_TIMESTAMP_IS_DIFF;
+        if (minusDiffTimestamp) {
+          flag |= FLAG_TIMESTAMP_SIGN;
+        }
+      } else {
+        flag |= (timestampFitsInBytes - 1) << SHIFT_TIMESTAMP_LENGTH;
+        if (negativeTimestamp) {
+          flag |= FLAG_TIMESTAMP_SIGN;
+        }
+      }
+    }
+
+    out.write(flag);
+
+    if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
+      ByteBufferUtils.putCompressedInt(out, keyLength);
+    }
+    if ((flag & FLAG_SAME_VALUE_LENGTH) == 0) {
+      ByteBufferUtils.putCompressedInt(out, valueLength);
+    }
+
+    ByteBufferUtils.putCompressedInt(out, commonPrefix);
+    ByteBufferUtils.skip(in, commonPrefix);
+
+    if (previousState.isFirst() ||
+        commonPrefix < currentState.rowLength + KeyValue.ROW_LENGTH_SIZE) {
+      int restRowLength =
+          currentState.rowLength + KeyValue.ROW_LENGTH_SIZE - commonPrefix;
+      ByteBufferUtils.moveBufferToStream(out, in, restRowLength);
+      ByteBufferUtils.skip(in, currentState.familyLength +
+          KeyValue.FAMILY_LENGTH_SIZE);
+      ByteBufferUtils.moveBufferToStream(out, in, currentState.qualifierLength);
+    } else {
+      ByteBufferUtils.moveBufferToStream(out, in,
+          keyLength - commonPrefix - KeyValue.TIMESTAMP_TYPE_SIZE);
+    }
+
+    if ((flag & FLAG_TIMESTAMP_IS_DIFF) == 0) {
+      ByteBufferUtils.putLong(out, timestamp, timestampFitsInBytes);
+    } else {
+      ByteBufferUtils.putLong(out, diffTimestamp, diffTimestampFitsInBytes);
+    }
+
+    if ((flag & FLAG_SAME_TYPE) == 0) {
+      out.write(currentState.type);
+    }
+    ByteBufferUtils.skip(in, KeyValue.TIMESTAMP_TYPE_SIZE);
+
+    ByteBufferUtils.moveBufferToStream(out, in, valueLength);
+  }
+
+  private void uncompressSingleKeyValue(DataInputStream source,
+      ByteBuffer buffer,
+      DiffCompressionState state)
+          throws IOException, EncoderBufferTooSmallException {
+    // read the column family at the beginning
+    if (state.isFirst()) {
+      state.familyLength = source.readByte();
+      state.familyNameWithSize =
+          new byte[(state.familyLength & 0xff) + KeyValue.FAMILY_LENGTH_SIZE];
+      state.familyNameWithSize[0] = state.familyLength;
+      source.read(state.familyNameWithSize, KeyValue.FAMILY_LENGTH_SIZE,
+          state.familyLength);
+    }
+
+    // read flag
+    byte flag = source.readByte();
+
+    // read key/value/common lengths
+    int keyLength;
+    int valueLength;
+    if ((flag & FLAG_SAME_KEY_LENGTH) != 0) {
+      keyLength = state.keyLength;
+    } else {
+      keyLength = ByteBufferUtils.readCompressedInt(source);
+    }
+    if ((flag & FLAG_SAME_VALUE_LENGTH) != 0) {
+      valueLength = state.valueLength;
+    } else {
+      valueLength = ByteBufferUtils.readCompressedInt(source);
+    }
+    int commonPrefix = ByteBufferUtils.readCompressedInt(source);
+
+    // create KeyValue buffer and fill it prefix
+    int keyOffset = buffer.position();
+    ByteBufferUtils.ensureSpace(buffer, keyLength + valueLength
+        + KeyValue.ROW_OFFSET);
+    buffer.putInt(keyLength);
+    buffer.putInt(valueLength);
+
+    // copy common from previous key
+    if (commonPrefix > 0) {
+      ByteBufferUtils.copyFromBufferToBuffer(buffer, buffer, state.prevOffset
+          + KeyValue.ROW_OFFSET, commonPrefix);
+    }
+
+    // copy the rest of the key from the buffer
+    int keyRestLength;
+    if (state.isFirst() || commonPrefix <
+        state.rowLength + KeyValue.ROW_LENGTH_SIZE) {
+      // omit the family part of the key, it is always the same
+      short rowLength;
+      int rowRestLength;
+
+      // check length of row
+      if (commonPrefix < KeyValue.ROW_LENGTH_SIZE) {
+        // not yet copied, do it now
+        ByteBufferUtils.copyFromStreamToBuffer(buffer, source,
+            KeyValue.ROW_LENGTH_SIZE - commonPrefix);
+        ByteBufferUtils.skip(buffer, -KeyValue.ROW_LENGTH_SIZE);
+        rowLength = buffer.getShort();
+        rowRestLength = rowLength;
+      } else {
+        // already in buffer, just read it
+        rowLength = buffer.getShort(keyOffset + KeyValue.ROW_OFFSET);
+        rowRestLength = rowLength + KeyValue.ROW_LENGTH_SIZE - commonPrefix;
+      }
+
+      // copy the rest of row
+      ByteBufferUtils.copyFromStreamToBuffer(buffer, source, rowRestLength);
+      state.rowLength = rowLength;
+
+      // copy the column family
+      buffer.put(state.familyNameWithSize);
+
+      keyRestLength = keyLength - rowLength -
+          state.familyNameWithSize.length -
+          (KeyValue.ROW_LENGTH_SIZE + KeyValue.TIMESTAMP_TYPE_SIZE);
+    } else {
+      // prevRowWithSizeLength is the same as on previous row
+      keyRestLength = keyLength - commonPrefix - KeyValue.TIMESTAMP_TYPE_SIZE;
+    }
+    // copy the rest of the key, after column family -> column qualifier
+    ByteBufferUtils.copyFromStreamToBuffer(buffer, source, keyRestLength);
+
+    // handle timestamp
+    int timestampFitsInBytes =
+        ((flag & MASK_TIMESTAMP_LENGTH) >>> SHIFT_TIMESTAMP_LENGTH) + 1;
+    long timestamp = ByteBufferUtils.readLong(source, timestampFitsInBytes);
+    if ((flag & FLAG_TIMESTAMP_SIGN) != 0) {
+      timestamp = -timestamp;
+    }
+    if ((flag & FLAG_TIMESTAMP_IS_DIFF) != 0) {
+      timestamp = state.timestamp - timestamp;
+    }
+    buffer.putLong(timestamp);
+
+    // copy the type field
+    byte type;
+    if ((flag & FLAG_SAME_TYPE) != 0) {
+      type = state.type;
+    } else {
+      type = source.readByte();
+    }
+    buffer.put(type);
+
+    // copy value part
+    ByteBufferUtils.copyFromStreamToBuffer(buffer, source, valueLength);
+
+    state.keyLength = keyLength;
+    state.valueLength = valueLength;
+    state.prevOffset = keyOffset;
+    state.timestamp = timestamp;
+    state.type = type;
+    // state.qualifier is unused
+  }
+
+  @Override
+  public void compressKeyValues(DataOutputStream out,
+      ByteBuffer in, boolean includesMemstoreTS) throws IOException {
+    in.rewind();
+    ByteBufferUtils.putInt(out, in.limit());
+    DiffCompressionState previousState = new DiffCompressionState();
+    DiffCompressionState currentState = new DiffCompressionState();
+    while (in.hasRemaining()) {
+      compressSingleKeyValue(previousState, currentState,
+          out, in);
+      afterEncodingKeyValue(in, out, includesMemstoreTS);
+
+      // swap previousState <-> currentState
+      DiffCompressionState tmp = previousState;
+      previousState = currentState;
+      currentState = tmp;
+    }
+  }
+
+  @Override
+  public ByteBuffer uncompressKeyValues(DataInputStream source,
+      int allocHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
+      throws IOException {
+    int decompressedSize = source.readInt();
+    ByteBuffer buffer = ByteBuffer.allocate(decompressedSize +
+        allocHeaderLength);
+    buffer.position(allocHeaderLength);
+    DiffCompressionState state = new DiffCompressionState();
+    while (source.available() > skipLastBytes) {
+      uncompressSingleKeyValue(source, buffer, state);
+      afterDecodingKeyValue(source, buffer, includesMemstoreTS);
+    }
+
+    if (source.available() != skipLastBytes) {
+      throw new IllegalStateException("Read too much bytes.");
+    }
+
+    return buffer;
+  }
+
+  @Override
+  public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
+    block.mark();
+    block.position(Bytes.SIZEOF_INT);
+    byte familyLength = block.get();
+    ByteBufferUtils.skip(block, familyLength);
+    byte flag = block.get();
+    int keyLength = ByteBufferUtils.readCompressedInt(block);
+    ByteBufferUtils.readCompressedInt(block); // valueLength
+    ByteBufferUtils.readCompressedInt(block); // commonLength
+    ByteBuffer result = ByteBuffer.allocate(keyLength);
+
+    // copy row
+    int pos = result.arrayOffset();
+    block.get(result.array(), pos, Bytes.SIZEOF_SHORT);
+    pos += Bytes.SIZEOF_SHORT;
+    short rowLength = result.getShort();
+    block.get(result.array(), pos, rowLength);
+    pos += rowLength;
+
+    // copy family
+    int savePosition = block.position();
+    block.position(Bytes.SIZEOF_INT);
+    block.get(result.array(), pos, familyLength + Bytes.SIZEOF_BYTE);
+    pos += familyLength + Bytes.SIZEOF_BYTE;
+
+    // copy qualifier
+    block.position(savePosition);
+    int qualifierLength =
+        keyLength - pos + result.arrayOffset() - KeyValue.TIMESTAMP_TYPE_SIZE;
+    block.get(result.array(), pos, qualifierLength);
+    pos += qualifierLength;
+
+    // copy the timestamp and type
+    int timestampFitInBytes =
+        ((flag & MASK_TIMESTAMP_LENGTH) >>> SHIFT_TIMESTAMP_LENGTH) + 1;
+    long timestamp = ByteBufferUtils.readLong(block, timestampFitInBytes);
+    if ((flag & FLAG_TIMESTAMP_SIGN) != 0) {
+      timestamp = -timestamp;
+    }
+    result.putLong(pos, timestamp);
+    pos += Bytes.SIZEOF_LONG;
+    block.get(result.array(), pos, Bytes.SIZEOF_BYTE);
+
+    block.reset();
+    return result;
+  }
+
+  @Override
+  public String toString() {
+    return DiffKeyDeltaEncoder.class.getSimpleName();
+  }
+
+  protected static class DiffSeekerState extends SeekerState {
+    private int rowLengthWithSize;
+    private long timestamp;
+
+    @Override
+    protected void copyFromNext(SeekerState that) {
+      super.copyFromNext(that);
+      DiffSeekerState other = (DiffSeekerState) that;
+      rowLengthWithSize = other.rowLengthWithSize;
+      timestamp = other.timestamp;
+    }
+  }
+
+  @Override
+  public EncodedSeeker createSeeker(RawComparator comparator,
+      final boolean includesMemstoreTS) {
+    return new BufferedEncodedSeeker(comparator) {
+      private byte[] familyNameWithSize;
+      private static final int TIMESTAMP_WITH_TYPE_LENGTH =
+          Bytes.SIZEOF_LONG + Bytes.SIZEOF_BYTE;
+
+      private void decode(boolean isFirst) {
+        byte flag = currentBuffer.get();
+        byte type = 0;
+        if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
+          if (!isFirst) {
+            type = current.keyBuffer[current.keyLength - Bytes.SIZEOF_BYTE];
+          }
+          current.keyLength = ByteBufferUtils.readCompressedInt(currentBuffer);
+        }
+        if ((flag & FLAG_SAME_VALUE_LENGTH) == 0) {
+          current.valueLength =
+              ByteBufferUtils.readCompressedInt(currentBuffer);
+        }
+        current.lastCommonPrefix =
+            ByteBufferUtils.readCompressedInt(currentBuffer);
+
+        current.ensureSpaceForKey();
+
+        if (current.lastCommonPrefix < Bytes.SIZEOF_SHORT) {
+          // length of row is different, copy everything except family
+
+          // copy the row size
+          currentBuffer.get(current.keyBuffer, current.lastCommonPrefix,
+              Bytes.SIZEOF_SHORT - current.lastCommonPrefix);
+          current.rowLengthWithSize = Bytes.toShort(current.keyBuffer, 0) +
+              Bytes.SIZEOF_SHORT;
+
+          // copy the rest of row
+          currentBuffer.get(current.keyBuffer, Bytes.SIZEOF_SHORT,
+              current.rowLengthWithSize - Bytes.SIZEOF_SHORT);
+
+          // copy the column family
+          System.arraycopy(familyNameWithSize, 0, current.keyBuffer,
+              current.rowLengthWithSize, familyNameWithSize.length);
+
+          // copy the qualifier
+          currentBuffer.get(current.keyBuffer,
+              current.rowLengthWithSize + familyNameWithSize.length,
+              current.keyLength - current.rowLengthWithSize -
+              familyNameWithSize.length - TIMESTAMP_WITH_TYPE_LENGTH);
+        } else if (current.lastCommonPrefix < current.rowLengthWithSize) {
+          // we have to copy part of row and qualifier,
+          // but column family is in right place
+
+          // before column family (rest of row)
+          currentBuffer.get(current.keyBuffer, current.lastCommonPrefix,
+              current.rowLengthWithSize - current.lastCommonPrefix);
+
+          // after column family (qualifier)
+          currentBuffer.get(current.keyBuffer,
+              current.rowLengthWithSize + familyNameWithSize.length,
+              current.keyLength - current.rowLengthWithSize -
+              familyNameWithSize.length - TIMESTAMP_WITH_TYPE_LENGTH);
+        } else {
+          // copy just the ending
+          currentBuffer.get(current.keyBuffer, current.lastCommonPrefix,
+              current.keyLength - TIMESTAMP_WITH_TYPE_LENGTH -
+              current.lastCommonPrefix);
+        }
+
+        // timestamp
+        int pos = current.keyLength - TIMESTAMP_WITH_TYPE_LENGTH;
+        int timestampFitInBytes = 1 +
+            ((flag & MASK_TIMESTAMP_LENGTH) >>> SHIFT_TIMESTAMP_LENGTH);
+        long timestampOrDiff =
+            ByteBufferUtils.readLong(currentBuffer, timestampFitInBytes);
+        if ((flag & FLAG_TIMESTAMP_SIGN) != 0) {
+          timestampOrDiff = -timestampOrDiff;
+        }
+        if ((flag & FLAG_TIMESTAMP_IS_DIFF) == 0) { // it is timestamp
+          current.timestamp = timestampOrDiff;
+        } else { // it is diff
+          current.timestamp = current.timestamp - timestampOrDiff;
+        }
+        Bytes.putLong(current.keyBuffer, pos, current.timestamp);
+        pos += Bytes.SIZEOF_LONG;
+
+        // type
+        if ((flag & FLAG_SAME_TYPE) == 0) {
+          currentBuffer.get(current.keyBuffer, pos, Bytes.SIZEOF_BYTE);
+        } else if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
+          current.keyBuffer[pos] = type;
+        }
+
+        current.valueOffset = currentBuffer.position();
+        ByteBufferUtils.skip(currentBuffer, current.valueLength);
+
+        if (includesMemstoreTS) {
+          current.memstoreTS = ByteBufferUtils.readVLong(currentBuffer);
+        } else {
+          current.memstoreTS = 0;
+        }
+        current.nextKvOffset = currentBuffer.position();
+      }
+
+      @Override
+      protected void decodeFirst() {
+        ByteBufferUtils.skip(currentBuffer, Bytes.SIZEOF_INT);
+
+        // read column family
+        byte familyNameLength = currentBuffer.get();
+        familyNameWithSize = new byte[familyNameLength + Bytes.SIZEOF_BYTE];
+        familyNameWithSize[0] = familyNameLength;
+        currentBuffer.get(familyNameWithSize, Bytes.SIZEOF_BYTE,
+            familyNameLength);
+        decode(true);
+      }
+
+      @Override
+      protected void decodeNext() {
+        decode(false);
+      }
+
+      @Override
+      protected DiffSeekerState createSeekerState() {
+        return new DiffSeekerState();
+      }
+    };
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
new file mode 100644
index 0000000..279a393
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.encoding;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.io.compress.Compressor;
+
+/**
+ * Encapsulates a data block compressed using a particular encoding algorithm.
+ * Useful for testing and benchmarking.
+ */
+public class EncodedDataBlock {
+  private static final int BUFFER_SIZE = 4 * 1024;
+  protected DataBlockEncoder dataBlockEncoder;
+  ByteArrayOutputStream uncompressedOutputStream;
+  ByteBuffer uncompressedBuffer;
+  private byte[] cacheCompressData;
+  private ByteArrayOutputStream compressedStream = new ByteArrayOutputStream();
+  private boolean includesMemstoreTS;
+
+  /**
+   * Create a buffer which will be encoded using dataBlockEncoder.
+   * @param dataBlockEncoder Algorithm used for compression.
+   */
+  public EncodedDataBlock(DataBlockEncoder dataBlockEncoder,
+      boolean includesMemstoreTS) {
+    this.dataBlockEncoder = dataBlockEncoder;
+    uncompressedOutputStream = new ByteArrayOutputStream(BUFFER_SIZE);
+  }
+
+  /**
+   * Add KeyValue and compress it.
+   * @param kv Item to be added and compressed.
+   */
+  public void addKv(KeyValue kv) {
+    cacheCompressData = null;
+    uncompressedOutputStream.write(
+        kv.getBuffer(), kv.getOffset(), kv.getLength());
+  }
+
+  /**
+   * Provides access to compressed value.
+   * @return Forwards sequential iterator.
+   */
+  public Iterator getIterator() {
+    final int uncompressedSize = uncompressedOutputStream.size();
+    final ByteArrayInputStream bais = new ByteArrayInputStream(
+        getCompressedData());
+    final DataInputStream dis = new DataInputStream(bais);
+
+
+    return new Iterator() {
+      private ByteBuffer decompressedData = null;
+
+      @Override
+      public boolean hasNext() {
+        if (decompressedData == null) {
+          return uncompressedSize > 0;
+        }
+        return decompressedData.hasRemaining();
+      }
+
+      @Override
+      public KeyValue next() {
+        if (decompressedData == null) {
+          try {
+            decompressedData = dataBlockEncoder.uncompressKeyValues(
+                dis, includesMemstoreTS);
+          } catch (IOException e) {
+            throw new RuntimeException("Problem with data block encoder, " +
+                "most likely it requested more bytes than are available.", e);
+          }
+          decompressedData.rewind();
+        }
+
+        int offset = decompressedData.position();
+        KeyValue kv = new KeyValue(decompressedData.array(), offset);
+        decompressedData.position(offset + kv.getLength());
+
+        return kv;
+      }
+
+      @Override
+      public void remove() {
+        throw new NotImplementedException("remove() is not supported!");
+      }
+
+      @Override
+      public String toString() {
+        return "Iterator of: " + dataBlockEncoder.getClass().getName();
+      }
+
+    };
+  }
+
+  /**
+   * Find the size of minimal buffer that could store compressed data.
+   * @return Size in bytes of compressed data.
+   */
+  public int getSize() {
+    return getCompressedData().length;
+  }
+
+  /**
+   * Find the size of compressed data assuming that buffer will be compressed
+   * using given algorithm.
+   * @param compressor Algorithm used for compression.
+   * @param buffer Array to be compressed.
+   * @param offset Offset to beginning of the data.
+   * @param length Length to be compressed.
+   * @return Size of compressed data in bytes.
+   */
+  public static int checkCompressedSize(Compressor compressor, byte[] buffer,
+      int offset, int length) {
+    byte[] compressedBuffer = new byte[buffer.length];
+    // in fact the buffer could be of any positive size
+    compressor.setInput(buffer, offset, length);
+    compressor.finish();
+    int currentPos = 0;
+    while (!compressor.finished()) {
+      try {
+        // we don't care about compressed data,
+        // we just want to callculate number of bytes
+        currentPos += compressor.compress(compressedBuffer, 0,
+            compressedBuffer.length);
+      } catch (IOException e) {
+        throw new RuntimeException(
+            "For some reason compressor couldn't read data. " +
+            "It is likely a problem with " +
+            compressor.getClass().getName(), e);
+      }
+    }
+    return currentPos;
+  }
+
+  /**
+   * Estimate size after second stage of compression (e.g. LZO).
+   * @param compressor Algorithm which will be used for compressions.
+   * @return Size after second stage of compression.
+   */
+  public int checkCompressedSize(Compressor compressor) {
+    // compress
+    byte[] compressedBytes = getCompressedData();
+    return checkCompressedSize(compressor, compressedBytes, 0,
+        compressedBytes.length);
+  }
+
+  private byte[] getCompressedData() {
+    // is cached
+    if (cacheCompressData != null) {
+      return cacheCompressData;
+    }
+    cacheCompressData = doCompressData();
+
+    return cacheCompressData;
+  }
+
+  private ByteBuffer getUncompressedBuffer() {
+    if (uncompressedBuffer == null ||
+        uncompressedBuffer.limit() < uncompressedOutputStream.size()) {
+      uncompressedBuffer = ByteBuffer.wrap(
+          uncompressedOutputStream.toByteArray());
+    }
+    return uncompressedBuffer;
+  }
+
+  /**
+   * Do the compression.
+   * @return Compressed byte buffer.
+   */
+  public byte[] doCompressData() {
+    compressedStream.reset();
+    DataOutputStream dataOut = new DataOutputStream(compressedStream);
+    try {
+      this.dataBlockEncoder.compressKeyValues(
+          dataOut, getUncompressedBuffer(), includesMemstoreTS);
+    } catch (IOException e) {
+      throw new RuntimeException(String.format(
+          "Bug in decoding part of algorithm %s. " +
+          "Probably it requested more bytes than are available.",
+          toString()), e);
+    }
+    return compressedStream.toByteArray();
+  }
+
+  @Override
+  public String toString() {
+    return dataBlockEncoder.toString();
+  }
+
+  /**
+   * Get uncompressed buffer.
+   * @return The buffer.
+   */
+  public byte[] getRawKeyValues() {
+    return uncompressedOutputStream.toByteArray();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/EncoderBufferTooSmallException.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/EncoderBufferTooSmallException.java
new file mode 100644
index 0000000..55195c1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/EncoderBufferTooSmallException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.encoding;
+
+/**
+ * Internal error which indicates a bug in a data block encoding algorithm.
+ */
+public class EncoderBufferTooSmallException extends RuntimeException {
+  private static final long serialVersionUID = 4767495176134878737L;
+
+  public EncoderBufferTooSmallException(String message) {
+    super(message);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
new file mode 100644
index 0000000..0ca02c8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
@@ -0,0 +1,546 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.encoding;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.ByteBufferUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.RawComparator;
+
+/**
+ * Encoder similar to {@link DiffKeyDeltaEncoder} but supposedly faster.
+ *
+ * Compress using:
+ * - store size of common prefix
+ * - save column family once in the first KeyValue
+ * - use integer compression for key, value and prefix (7-bit encoding)
+ * - use bits to avoid duplication key length, value length
+ *   and type if it same as previous
+ * - store in 3 bits length of prefix timestamp
+ *    with previous KeyValue's timestamp
+ * - one bit which allow to omit value if it is the same
+ *
+ * Format:
+ * - 1 byte:    flag
+ * - 1-5 bytes: key length (only if FLAG_SAME_KEY_LENGTH is not set in flag)
+ * - 1-5 bytes: value length (only if FLAG_SAME_VALUE_LENGTH is not set in flag)
+ * - 1-5 bytes: prefix length
+ * - ... bytes: rest of the row (if prefix length is small enough)
+ * - ... bytes: qualifier (or suffix depending on prefix length)
+ * - 1-8 bytes: timestamp suffix
+ * - 1 byte:    type (only if FLAG_SAME_TYPE is not set in the flag)
+ * - ... bytes: value (only if FLAG_SAME_VALUE is not set in the flag)
+ *
+ */
+public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
+  final int MASK_TIMESTAMP_LENGTH = (1 << 0) | (1 << 1) | (1 << 2);
+  final int SHIFT_TIMESTAMP_LENGTH = 0;
+  final int FLAG_SAME_KEY_LENGTH = 1 << 3;
+  final int FLAG_SAME_VALUE_LENGTH = 1 << 4;
+  final int FLAG_SAME_TYPE = 1 << 5;
+  final int FLAG_SAME_VALUE = 1 << 6;
+
+  private static class FastDiffCompressionState extends CompressionState {
+    byte[] timestamp = new byte[KeyValue.TIMESTAMP_SIZE];
+    int prevTimestampOffset;
+
+    @Override
+    protected void readTimestamp(ByteBuffer in) {
+      in.get(timestamp);
+    }
+
+    @Override
+    void copyFrom(CompressionState state) {
+      super.copyFrom(state);
+      FastDiffCompressionState state2 = (FastDiffCompressionState) state;
+      System.arraycopy(state2.timestamp, 0, timestamp, 0,
+          KeyValue.TIMESTAMP_SIZE);
+      prevTimestampOffset = state2.prevTimestampOffset;
+    }
+
+    /**
+     * Copies the first key/value from the given stream, and initializes
+     * decompression state based on it. Assumes that we have already read key
+     * and value lengths. Does not set {@link #qualifierLength} (not used by
+     * decompression) or {@link #prevOffset} (set by the calle afterwards).
+     */
+    private void decompressFirstKV(ByteBuffer out, DataInputStream in)
+        throws IOException {
+      int kvPos = out.position();
+      out.putInt(keyLength);
+      out.putInt(valueLength);
+      prevTimestampOffset = out.position() + keyLength -
+          KeyValue.TIMESTAMP_TYPE_SIZE;
+      ByteBufferUtils.copyFromStreamToBuffer(out, in, keyLength + valueLength);
+      rowLength = out.getShort(kvPos + KeyValue.ROW_OFFSET);
+      familyLength = out.get(kvPos + KeyValue.ROW_OFFSET +
+          KeyValue.ROW_LENGTH_SIZE + rowLength);
+      type = out.get(prevTimestampOffset + KeyValue.TIMESTAMP_SIZE);
+    }
+
+  }
+
+  private void compressSingleKeyValue(
+        FastDiffCompressionState previousState,
+        FastDiffCompressionState currentState,
+        OutputStream out, ByteBuffer in) throws IOException {
+    currentState.prevOffset = in.position();
+    int keyLength = in.getInt();
+    int valueOffset =
+        currentState.prevOffset + keyLength + KeyValue.ROW_OFFSET;
+    int valueLength = in.getInt();
+    byte flag = 0;
+
+    if (previousState.isFirst()) {
+      // copy the key, there is no common prefix with none
+      out.write(flag);
+      ByteBufferUtils.putCompressedInt(out, keyLength);
+      ByteBufferUtils.putCompressedInt(out, valueLength);
+      ByteBufferUtils.putCompressedInt(out, 0);
+
+      currentState.readKey(in, keyLength, valueLength);
+
+      ByteBufferUtils.moveBufferToStream(out, in, keyLength + valueLength);
+    } else {
+      // find a common prefix and skip it
+      int commonPrefix = ByteBufferUtils.findCommonPrefix(in, in.position(),
+          previousState.prevOffset + KeyValue.ROW_OFFSET,
+          Math.min(keyLength, previousState.keyLength) -
+          KeyValue.TIMESTAMP_TYPE_SIZE);
+
+      currentState.readKey(in, keyLength, valueLength,
+          commonPrefix, previousState);
+
+      if (keyLength == previousState.keyLength) {
+        flag |= FLAG_SAME_KEY_LENGTH;
+      }
+      if (valueLength == previousState.valueLength) {
+        flag |= FLAG_SAME_VALUE_LENGTH;
+      }
+      if (currentState.type == previousState.type) {
+        flag |= FLAG_SAME_TYPE;
+      }
+
+      int commonTimestampPrefix = findCommonTimestampPrefix(
+          currentState, previousState);
+      flag |= commonTimestampPrefix << SHIFT_TIMESTAMP_LENGTH;
+
+      // Check if current and previous values are the same. Compare value
+      // length first as an optimization.
+      if (valueLength == previousState.valueLength) {
+        int previousValueOffset = previousState.prevOffset
+            + previousState.keyLength + KeyValue.ROW_OFFSET;
+        if (ByteBufferUtils.arePartsEqual(in,
+                previousValueOffset, previousState.valueLength,
+                valueOffset, valueLength)) {
+          flag |= FLAG_SAME_VALUE;
+        }
+      }
+
+      out.write(flag);
+      if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
+        ByteBufferUtils.putCompressedInt(out, keyLength);
+      }
+      if ((flag & FLAG_SAME_VALUE_LENGTH) == 0) {
+        ByteBufferUtils.putCompressedInt(out, valueLength);
+      }
+      ByteBufferUtils.putCompressedInt(out, commonPrefix);
+
+      ByteBufferUtils.skip(in, commonPrefix);
+      if (commonPrefix < currentState.rowLength + KeyValue.ROW_LENGTH_SIZE) {
+        // Previous and current rows are different. Copy the differing part of
+        // the row, skip the column family, and copy the qualifier.
+        ByteBufferUtils.moveBufferToStream(out, in,
+            currentState.rowLength + KeyValue.ROW_LENGTH_SIZE - commonPrefix);
+        ByteBufferUtils.skip(in, currentState.familyLength +
+            KeyValue.FAMILY_LENGTH_SIZE);
+        ByteBufferUtils.moveBufferToStream(out, in,
+            currentState.qualifierLength);
+      } else {
+        // The common part includes the whole row. As the column family is the
+        // same across the whole file, it will automatically be included in the
+        // common prefix, so we need not special-case it here.
+        int restKeyLength = keyLength - commonPrefix -
+            KeyValue.TIMESTAMP_TYPE_SIZE;
+        ByteBufferUtils.moveBufferToStream(out, in, restKeyLength);
+      }
+      ByteBufferUtils.skip(in, commonTimestampPrefix);
+      ByteBufferUtils.moveBufferToStream(out, in,
+          KeyValue.TIMESTAMP_SIZE - commonTimestampPrefix);
+
+      // Write the type if it is not the same as before.
+      if ((flag & FLAG_SAME_TYPE) == 0) {
+        out.write(currentState.type);
+      }
+
+      // Write the value if it is not the same as before.
+      if ((flag & FLAG_SAME_VALUE) == 0) {
+        ByteBufferUtils.copyBufferToStream(out, in, valueOffset, valueLength);
+      }
+
+      // Skip key type and value in the input buffer.
+      ByteBufferUtils.skip(in, KeyValue.TYPE_SIZE + currentState.valueLength);
+    }
+  }
+
+  private int findCommonTimestampPrefix(FastDiffCompressionState left,
+      FastDiffCompressionState right) {
+    int prefixTimestamp = 0;
+    while (prefixTimestamp < (KeyValue.TIMESTAMP_SIZE - 1) &&
+        left.timestamp[prefixTimestamp]
+            == right.timestamp[prefixTimestamp]) {
+      prefixTimestamp++;
+    }
+    return prefixTimestamp; // has to be at most 7 bytes
+  }
+
+  private void uncompressSingleKeyValue(DataInputStream source,
+      ByteBuffer out, FastDiffCompressionState state)
+          throws IOException, EncoderBufferTooSmallException {
+    byte flag = source.readByte();
+    int prevKeyLength = state.keyLength;
+
+    if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
+      state.keyLength = ByteBufferUtils.readCompressedInt(source);
+    }
+    if ((flag & FLAG_SAME_VALUE_LENGTH) == 0) {
+      state.valueLength = ByteBufferUtils.readCompressedInt(source);
+    }
+    int commonLength = ByteBufferUtils.readCompressedInt(source);
+
+    ByteBufferUtils.ensureSpace(out, state.keyLength + state.valueLength +
+        KeyValue.ROW_OFFSET);
+
+    int kvPos = out.position();
+
+    if (!state.isFirst()) {
+      // copy the prefix
+      int common;
+      int prevOffset;
+
+      if ((flag & FLAG_SAME_VALUE_LENGTH) == 0) {
+        out.putInt(state.keyLength);
+        out.putInt(state.valueLength);
+        prevOffset = state.prevOffset + KeyValue.ROW_OFFSET;
+        common = commonLength;
+      } else {
+        if ((flag & FLAG_SAME_KEY_LENGTH) != 0) {
+          prevOffset = state.prevOffset;
+          common = commonLength + KeyValue.ROW_OFFSET;
+        } else {
+          out.putInt(state.keyLength);
+          prevOffset = state.prevOffset + KeyValue.KEY_LENGTH_SIZE;
+          common = commonLength + KeyValue.KEY_LENGTH_SIZE;
+        }
+      }
+
+      ByteBufferUtils.copyFromBufferToBuffer(out, out, prevOffset, common);
+
+      // copy the rest of the key from the buffer
+      int keyRestLength;
+      if (commonLength < state.rowLength + KeyValue.ROW_LENGTH_SIZE) {
+        // omit the family part of the key, it is always the same
+        int rowWithSizeLength;
+        int rowRestLength;
+
+        // check length of row
+        if (commonLength < KeyValue.ROW_LENGTH_SIZE) {
+          // not yet copied, do it now
+          ByteBufferUtils.copyFromStreamToBuffer(out, source,
+              KeyValue.ROW_LENGTH_SIZE - commonLength);
+
+          rowWithSizeLength = out.getShort(out.position() -
+              KeyValue.ROW_LENGTH_SIZE) + KeyValue.ROW_LENGTH_SIZE;
+          rowRestLength = rowWithSizeLength - KeyValue.ROW_LENGTH_SIZE;
+        } else {
+          // already in kvBuffer, just read it
+          rowWithSizeLength = out.getShort(kvPos + KeyValue.ROW_OFFSET) +
+              KeyValue.ROW_LENGTH_SIZE;
+          rowRestLength = rowWithSizeLength - commonLength;
+        }
+
+        // copy the rest of row
+        ByteBufferUtils.copyFromStreamToBuffer(out, source, rowRestLength);
+
+        // copy the column family
+        ByteBufferUtils.copyFromBufferToBuffer(out, out,
+            state.prevOffset + KeyValue.ROW_OFFSET + KeyValue.ROW_LENGTH_SIZE
+                + state.rowLength, state.familyLength
+                + KeyValue.FAMILY_LENGTH_SIZE);
+        state.rowLength = (short) (rowWithSizeLength -
+            KeyValue.ROW_LENGTH_SIZE);
+
+        keyRestLength = state.keyLength - rowWithSizeLength -
+            state.familyLength -
+            (KeyValue.FAMILY_LENGTH_SIZE + KeyValue.TIMESTAMP_TYPE_SIZE);
+      } else {
+        // prevRowWithSizeLength is the same as on previous row
+        keyRestLength = state.keyLength - commonLength -
+            KeyValue.TIMESTAMP_TYPE_SIZE;
+      }
+      // copy the rest of the key, after column family == column qualifier
+      ByteBufferUtils.copyFromStreamToBuffer(out, source, keyRestLength);
+
+      // copy timestamp
+      int prefixTimestamp =
+          (flag & MASK_TIMESTAMP_LENGTH) >>> SHIFT_TIMESTAMP_LENGTH;
+      ByteBufferUtils.copyFromBufferToBuffer(out, out,
+          state.prevTimestampOffset, prefixTimestamp);
+      state.prevTimestampOffset = out.position() - prefixTimestamp;
+      ByteBufferUtils.copyFromStreamToBuffer(out, source,
+          KeyValue.TIMESTAMP_SIZE - prefixTimestamp);
+
+      // copy the type and value
+      if ((flag & FLAG_SAME_TYPE) != 0) {
+        out.put(state.type);
+        if ((flag & FLAG_SAME_VALUE) != 0) {
+          ByteBufferUtils.copyFromBufferToBuffer(out, out, state.prevOffset +
+              KeyValue.ROW_OFFSET + prevKeyLength, state.valueLength);
+        } else {
+          ByteBufferUtils.copyFromStreamToBuffer(out, source,
+              state.valueLength);
+        }
+      } else {
+        if ((flag & FLAG_SAME_VALUE) != 0) {
+          ByteBufferUtils.copyFromStreamToBuffer(out, source,
+              KeyValue.TYPE_SIZE);
+          ByteBufferUtils.copyFromBufferToBuffer(out, out, state.prevOffset +
+              KeyValue.ROW_OFFSET + prevKeyLength, state.valueLength);
+        } else {
+          ByteBufferUtils.copyFromStreamToBuffer(out, source,
+              state.valueLength + KeyValue.TYPE_SIZE);
+        }
+        state.type = out.get(state.prevTimestampOffset +
+            KeyValue.TIMESTAMP_SIZE);
+      }
+    } else { // this is the first element
+      state.decompressFirstKV(out, source);
+    }
+
+    state.prevOffset = kvPos;
+  }
+
+  @Override
+  public void compressKeyValues(DataOutputStream out,
+      ByteBuffer in, boolean includesMemstoreTS) throws IOException {
+    in.rewind();
+    ByteBufferUtils.putInt(out, in.limit());
+    FastDiffCompressionState previousState = new FastDiffCompressionState();
+    FastDiffCompressionState currentState = new FastDiffCompressionState();
+    while (in.hasRemaining()) {
+      compressSingleKeyValue(previousState, currentState,
+          out, in);
+      afterEncodingKeyValue(in, out, includesMemstoreTS);
+
+      // swap previousState <-> currentState
+      FastDiffCompressionState tmp = previousState;
+      previousState = currentState;
+      currentState = tmp;
+    }
+  }
+
+  @Override
+  public ByteBuffer uncompressKeyValues(DataInputStream source,
+      int allocHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
+          throws IOException {
+    int decompressedSize = source.readInt();
+    ByteBuffer buffer = ByteBuffer.allocate(decompressedSize +
+        allocHeaderLength);
+    buffer.position(allocHeaderLength);
+    FastDiffCompressionState state = new FastDiffCompressionState();
+    while (source.available() > skipLastBytes) {
+      uncompressSingleKeyValue(source, buffer, state);
+      afterDecodingKeyValue(source, buffer, includesMemstoreTS);
+    }
+
+    if (source.available() != skipLastBytes) {
+      throw new IllegalStateException("Read too much bytes.");
+    }
+
+    return buffer;
+  }
+
+  @Override
+  public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
+    block.mark();
+    block.position(Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE);
+    int keyLength = ByteBufferUtils.readCompressedInt(block);
+    ByteBufferUtils.readCompressedInt(block); // valueLength
+    ByteBufferUtils.readCompressedInt(block); // commonLength
+    int pos = block.position();
+    block.reset();
+    return ByteBuffer.wrap(block.array(), pos, keyLength).slice();
+  }
+
+  @Override
+  public String toString() {
+    return FastDiffDeltaEncoder.class.getSimpleName();
+  }
+
+  protected static class FastDiffSeekerState extends SeekerState {
+    private byte[] prevTimestampAndType =
+        new byte[KeyValue.TIMESTAMP_TYPE_SIZE];
+    private int rowLengthWithSize;
+    private int familyLengthWithSize;
+
+    @Override
+    protected void copyFromNext(SeekerState that) {
+      super.copyFromNext(that);
+      FastDiffSeekerState other = (FastDiffSeekerState) that;
+      System.arraycopy(other.prevTimestampAndType, 0,
+          prevTimestampAndType, 0,
+          KeyValue.TIMESTAMP_TYPE_SIZE);
+      rowLengthWithSize = other.rowLengthWithSize;
+      familyLengthWithSize = other.familyLengthWithSize;
+    }
+  }
+
+  @Override
+  public EncodedSeeker createSeeker(RawComparator comparator,
+      final boolean includesMemstoreTS) {
+    return new BufferedEncodedSeeker(comparator) {
+      private void decode(boolean isFirst) {
+        byte flag = currentBuffer.get();
+        if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
+          if (!isFirst) {
+            System.arraycopy(current.keyBuffer,
+                current.keyLength - current.prevTimestampAndType.length,
+                current.prevTimestampAndType, 0,
+                current.prevTimestampAndType.length);
+          }
+          current.keyLength = ByteBufferUtils.readCompressedInt(currentBuffer);
+        }
+        if ((flag & FLAG_SAME_VALUE_LENGTH) == 0) {
+          current.valueLength =
+              ByteBufferUtils.readCompressedInt(currentBuffer);
+        }
+        current.lastCommonPrefix =
+            ByteBufferUtils.readCompressedInt(currentBuffer);
+
+        current.ensureSpaceForKey();
+
+        if (isFirst) {
+          // copy everything
+          currentBuffer.get(current.keyBuffer, current.lastCommonPrefix,
+              current.keyLength - current.prevTimestampAndType.length);
+          current.rowLengthWithSize = Bytes.toShort(current.keyBuffer, 0) +
+              Bytes.SIZEOF_SHORT;
+          current.familyLengthWithSize =
+              current.keyBuffer[current.rowLengthWithSize] + Bytes.SIZEOF_BYTE;
+        } else if (current.lastCommonPrefix < Bytes.SIZEOF_SHORT) {
+          // length of row is different, copy everything except family
+
+          // copy the row size
+          int oldRowLengthWithSize = current.rowLengthWithSize;
+          currentBuffer.get(current.keyBuffer, current.lastCommonPrefix,
+              Bytes.SIZEOF_SHORT - current.lastCommonPrefix);
+          current.rowLengthWithSize = Bytes.toShort(current.keyBuffer, 0) +
+              Bytes.SIZEOF_SHORT;
+
+          // move the column family
+          System.arraycopy(current.keyBuffer, oldRowLengthWithSize,
+              current.keyBuffer, current.rowLengthWithSize,
+              current.familyLengthWithSize);
+
+          // copy the rest of row
+          currentBuffer.get(current.keyBuffer, Bytes.SIZEOF_SHORT,
+              current.rowLengthWithSize - Bytes.SIZEOF_SHORT);
+
+          // copy the qualifier
+          currentBuffer.get(current.keyBuffer, current.rowLengthWithSize
+              + current.familyLengthWithSize, current.keyLength
+              - current.rowLengthWithSize - current.familyLengthWithSize
+              - current.prevTimestampAndType.length);
+        } else if (current.lastCommonPrefix < current.rowLengthWithSize) {
+          // We have to copy part of row and qualifier, but the column family
+          // is in the right place.
+
+          // before column family (rest of row)
+          currentBuffer.get(current.keyBuffer, current.lastCommonPrefix,
+              current.rowLengthWithSize - current.lastCommonPrefix);
+
+          // after column family (qualifier)
+          currentBuffer.get(current.keyBuffer, current.rowLengthWithSize
+              + current.familyLengthWithSize, current.keyLength
+              - current.rowLengthWithSize - current.familyLengthWithSize
+              - current.prevTimestampAndType.length);
+        } else {
+          // copy just the ending
+          currentBuffer.get(current.keyBuffer, current.lastCommonPrefix,
+              current.keyLength - current.prevTimestampAndType.length
+                  - current.lastCommonPrefix);
+        }
+
+        // timestamp
+        int pos = current.keyLength - current.prevTimestampAndType.length;
+        int commonTimestampPrefix = (flag & MASK_TIMESTAMP_LENGTH) >>>
+          SHIFT_TIMESTAMP_LENGTH;
+        if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
+          System.arraycopy(current.prevTimestampAndType, 0, current.keyBuffer,
+              pos, commonTimestampPrefix);
+        }
+        pos += commonTimestampPrefix;
+        currentBuffer.get(current.keyBuffer, pos,
+            Bytes.SIZEOF_LONG - commonTimestampPrefix);
+        pos += Bytes.SIZEOF_LONG - commonTimestampPrefix;
+
+        // type
+        if ((flag & FLAG_SAME_TYPE) == 0) {
+          currentBuffer.get(current.keyBuffer, pos, Bytes.SIZEOF_BYTE);
+        } else if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
+          current.keyBuffer[pos] =
+              current.prevTimestampAndType[Bytes.SIZEOF_LONG];
+        }
+
+        // handle value
+        if ((flag & FLAG_SAME_VALUE) == 0) {
+          current.valueOffset = currentBuffer.position();
+          ByteBufferUtils.skip(currentBuffer, current.valueLength);
+        }
+
+        if (includesMemstoreTS) {
+          current.memstoreTS = ByteBufferUtils.readVLong(currentBuffer);
+        } else {
+          current.memstoreTS = 0;
+        }
+        current.nextKvOffset = currentBuffer.position();
+      }
+
+      @Override
+      protected void decodeFirst() {
+        ByteBufferUtils.skip(currentBuffer, Bytes.SIZEOF_INT);
+        decode(true);
+      }
+
+      @Override
+      protected void decodeNext() {
+        decode(false);
+      }
+
+      @Override
+      protected FastDiffSeekerState createSeekerState() {
+        return new FastDiffSeekerState();
+      }
+    };
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
new file mode 100644
index 0000000..ac63ead
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.encoding;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.ByteBufferUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.RawComparator;
+
+/**
+ * Compress key by storing size of common prefix with previous KeyValue
+ * and storing raw size of rest.
+ *
+ * Format:
+ * 1-5 bytes: compressed key length minus prefix (7-bit encoding)
+ * 1-5 bytes: compressed value length (7-bit encoding)
+ * 1-3 bytes: compressed length of common key prefix
+ * ... bytes: rest of key (including timestamp)
+ * ... bytes: value
+ *
+ * In a worst case compressed KeyValue will be three bytes longer than original.
+ *
+ */
+public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder {
+
+  private int addKV(int prevKeyOffset, DataOutputStream out,
+      ByteBuffer in, int prevKeyLength) throws IOException {
+    int keyLength = in.getInt();
+    int valueLength = in.getInt();
+
+    if (prevKeyOffset == -1) {
+      // copy the key, there is no common prefix with none
+      ByteBufferUtils.putCompressedInt(out, keyLength);
+      ByteBufferUtils.putCompressedInt(out, valueLength);
+      ByteBufferUtils.putCompressedInt(out, 0);
+      ByteBufferUtils.moveBufferToStream(out, in, keyLength + valueLength);
+    } else {
+      // find a common prefix and skip it
+      int common = ByteBufferUtils.findCommonPrefix(
+          in, prevKeyOffset + KeyValue.ROW_OFFSET,
+          in.position(),
+          Math.min(prevKeyLength, keyLength));
+
+      ByteBufferUtils.putCompressedInt(out, keyLength - common);
+      ByteBufferUtils.putCompressedInt(out, valueLength);
+      ByteBufferUtils.putCompressedInt(out, common);
+
+      ByteBufferUtils.skip(in, common);
+      ByteBufferUtils.moveBufferToStream(out, in, keyLength - common
+          + valueLength);
+    }
+
+    return keyLength;
+  }
+
+  @Override
+  public void compressKeyValues(DataOutputStream writeHere,
+      ByteBuffer in, boolean includesMemstoreTS) throws IOException {
+    in.rewind();
+    ByteBufferUtils.putInt(writeHere, in.limit());
+    int prevOffset = -1;
+    int offset = 0;
+    int keyLength = 0;
+    while (in.hasRemaining()) {
+      offset = in.position();
+      keyLength = addKV(prevOffset, writeHere, in, keyLength);
+      afterEncodingKeyValue(in, writeHere, includesMemstoreTS);
+      prevOffset = offset;
+    }
+  }
+
+  @Override
+  public ByteBuffer uncompressKeyValues(DataInputStream source,
+      int allocHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
+          throws IOException {
+    int decompressedSize = source.readInt();
+    ByteBuffer buffer = ByteBuffer.allocate(decompressedSize +
+        allocHeaderLength);
+    buffer.position(allocHeaderLength);
+    int prevKeyOffset = 0;
+
+    while (source.available() > skipLastBytes) {
+      prevKeyOffset = uncompressKeyValue(source, buffer, prevKeyOffset);
+      afterDecodingKeyValue(source, buffer, includesMemstoreTS);
+    }
+
+    if (source.available() != skipLastBytes) {
+      throw new IllegalStateException("Read too many bytes.");
+    }
+
+    buffer.limit(buffer.position());
+    return buffer;
+  }
+
+  private int uncompressKeyValue(DataInputStream source, ByteBuffer buffer,
+      int prevKeyOffset)
+          throws IOException, EncoderBufferTooSmallException {
+    int keyLength = ByteBufferUtils.readCompressedInt(source);
+    int valueLength = ByteBufferUtils.readCompressedInt(source);
+    int commonLength = ByteBufferUtils.readCompressedInt(source);
+    int keyOffset;
+    keyLength += commonLength;
+
+    ByteBufferUtils.ensureSpace(buffer, keyLength + valueLength
+        + KeyValue.ROW_OFFSET);
+
+    buffer.putInt(keyLength);
+    buffer.putInt(valueLength);
+
+    // copy the prefix
+    if (commonLength > 0) {
+      keyOffset = buffer.position();
+      ByteBufferUtils.copyFromBufferToBuffer(buffer, buffer, prevKeyOffset,
+          commonLength);
+    } else {
+      keyOffset = buffer.position();
+    }
+
+    // copy rest of the key and value
+    int len = keyLength - commonLength + valueLength;
+    ByteBufferUtils.copyFromStreamToBuffer(buffer, source, len);
+    return keyOffset;
+  }
+
+  @Override
+  public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
+    block.mark();
+    block.position(Bytes.SIZEOF_INT);
+    int keyLength = ByteBufferUtils.readCompressedInt(block);
+    ByteBufferUtils.readCompressedInt(block);
+    int commonLength = ByteBufferUtils.readCompressedInt(block);
+    if (commonLength != 0) {
+      throw new AssertionError("Nonzero common length in the first key in "
+          + "block: " + commonLength);
+    }
+    int pos = block.position();
+    block.reset();
+    return ByteBuffer.wrap(block.array(), pos, keyLength).slice();
+  }
+
+  @Override
+  public String toString() {
+    return PrefixKeyDeltaEncoder.class.getSimpleName();
+  }
+
+  @Override
+  public EncodedSeeker createSeeker(RawComparator comparator,
+      final boolean includesMemstoreTS) {
+    return new BufferedEncodedSeeker(comparator) {
+      @Override
+      protected void decodeNext() {
+        current.keyLength = ByteBufferUtils.readCompressedInt(currentBuffer);
+        current.valueLength = ByteBufferUtils.readCompressedInt(currentBuffer);
+        current.lastCommonPrefix =
+            ByteBufferUtils.readCompressedInt(currentBuffer);
+        current.keyLength += current.lastCommonPrefix;
+        current.ensureSpaceForKey();
+        currentBuffer.get(current.keyBuffer, current.lastCommonPrefix,
+            current.keyLength - current.lastCommonPrefix);
+        current.valueOffset = currentBuffer.position();
+        ByteBufferUtils.skip(currentBuffer, current.valueLength);
+        if (includesMemstoreTS) {
+          current.memstoreTS = ByteBufferUtils.readVLong(currentBuffer);
+        } else {
+          current.memstoreTS = 0;
+        }
+        current.nextKvOffset = currentBuffer.position();
+      }
+
+      @Override
+      protected void decodeFirst() {
+        ByteBufferUtils.skip(currentBuffer, Bytes.SIZEOF_INT);
+        decodeNext();
+      }
+    };
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
new file mode 100644
index 0000000..5573b66
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
@@ -0,0 +1,357 @@
+  /*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
+import org.apache.hadoop.io.RawComparator;
+
+/**
+ * Common functionality needed by all versions of {@link HFile} readers.
+ */
+public abstract class AbstractHFileReader extends SchemaConfigured
+    implements HFile.Reader {
+
+  /** Filesystem-level block reader for this HFile format version. */
+  protected HFileBlock.FSReader fsBlockReader;
+
+  /** Stream to read from. Does checksum verifications in file system */
+  protected FSDataInputStream istream;
+
+  /** The file system stream of the underlying {@link HFile} that
+   * does not do checksum verification in the file system */
+  protected FSDataInputStream istreamNoFsChecksum;
+
+  /**
+   * True if we should close the input stream when done. We don't close it if we
+   * didn't open it.
+   */
+  protected final boolean closeIStream;
+
+  /** Data block index reader keeping the root data index in memory */
+  protected HFileBlockIndex.BlockIndexReader dataBlockIndexReader;
+
+  /** Meta block index reader -- always single level */
+  protected HFileBlockIndex.BlockIndexReader metaBlockIndexReader;
+
+  protected final FixedFileTrailer trailer;
+
+  /** Filled when we read in the trailer. */
+  protected final Compression.Algorithm compressAlgo;
+
+  /**
+   * What kind of data block encoding should be used while reading, writing,
+   * and handling cache.
+   */
+  protected HFileDataBlockEncoder dataBlockEncoder =
+      NoOpDataBlockEncoder.INSTANCE;
+
+  /** Last key in the file. Filled in when we read in the file info */
+  protected byte [] lastKey = null;
+
+  /** Average key length read from file info */
+  protected int avgKeyLen = -1;
+
+  /** Average value length read from file info */
+  protected int avgValueLen = -1;
+
+  /** Key comparator */
+  protected RawComparator comparator;
+
+  /** Size of this file. */
+  protected final long fileSize;
+
+  /** Block cache configuration. */
+  protected final CacheConfig cacheConf;
+
+  /** Path of file */
+  protected final Path path;
+
+  /** File name to be used for block names */
+  protected final String name;
+
+  protected FileInfo fileInfo;
+
+  /** The filesystem used for accesing data */
+  protected HFileSystem hfs;
+
+  protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
+      final FSDataInputStream fsdis, final long fileSize,
+      final boolean closeIStream,
+      final CacheConfig cacheConf) {
+    this(path, trailer, fsdis, fsdis, fileSize, closeIStream, cacheConf, null);
+  }
+
+  protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
+      final FSDataInputStream fsdis, final FSDataInputStream fsdisNoFsChecksum,
+      final long fileSize,
+      final boolean closeIStream,
+      final CacheConfig cacheConf, final HFileSystem hfs) {
+    super(null, path);
+    this.trailer = trailer;
+    this.compressAlgo = trailer.getCompressionCodec();
+    this.cacheConf = cacheConf;
+    this.fileSize = fileSize;
+    this.istream = fsdis;
+    this.closeIStream = closeIStream;
+    this.path = path;
+    this.name = path.getName();
+    this.hfs = hfs;
+    this.istreamNoFsChecksum = fsdisNoFsChecksum;
+  }
+
+  @SuppressWarnings("serial")
+  public static class BlockIndexNotLoadedException
+      extends IllegalStateException {
+    public BlockIndexNotLoadedException() {
+      // Add a message in case anyone relies on it as opposed to class name.
+      super("Block index not loaded");
+    }
+  }
+
+  protected String toStringFirstKey() {
+    return KeyValue.keyToString(getFirstKey());
+  }
+
+  protected String toStringLastKey() {
+    return KeyValue.keyToString(getLastKey());
+  }
+
+  public abstract boolean isFileInfoLoaded();
+
+  @Override
+  public String toString() {
+    return "reader=" + path.toString() +
+        (!isFileInfoLoaded()? "":
+          ", compression=" + compressAlgo.getName() +
+          ", cacheConf=" + cacheConf +
+          ", firstKey=" + toStringFirstKey() +
+          ", lastKey=" + toStringLastKey()) +
+          ", avgKeyLen=" + avgKeyLen +
+          ", avgValueLen=" + avgValueLen +
+          ", entries=" + trailer.getEntryCount() +
+          ", length=" + fileSize;
+  }
+
+  @Override
+  public long length() {
+    return fileSize;
+  }
+
+  /**
+   * Create a Scanner on this file. No seeks or reads are done on creation. Call
+   * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
+   * nothing to clean up in a Scanner. Letting go of your references to the
+   * scanner is sufficient. NOTE: Do not use this overload of getScanner for
+   * compactions.
+   *
+   * @param cacheBlocks True if we should cache blocks read in by this scanner.
+   * @param pread Use positional read rather than seek+read if true (pread is
+   *          better for random reads, seek+read is better scanning).
+   * @return Scanner on this file.
+   */
+  @Override
+  public HFileScanner getScanner(boolean cacheBlocks, final boolean pread) {
+    return getScanner(cacheBlocks, pread, false);
+  }
+
+  /**
+   * @return the first key in the file. May be null if file has no entries. Note
+   *         that this is not the first row key, but rather the byte form of the
+   *         first KeyValue.
+   */
+  @Override
+  public byte [] getFirstKey() {
+    if (dataBlockIndexReader == null) {
+      throw new BlockIndexNotLoadedException();
+    }
+    return dataBlockIndexReader.isEmpty() ? null
+        : dataBlockIndexReader.getRootBlockKey(0);
+  }
+
+  /**
+   * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's
+   * patch goes in to eliminate {@link KeyValue} here.
+   *
+   * @return the first row key, or null if the file is empty.
+   */
+  @Override
+  public byte[] getFirstRowKey() {
+    byte[] firstKey = getFirstKey();
+    if (firstKey == null)
+      return null;
+    return KeyValue.createKeyValueFromKey(firstKey).getRow();
+  }
+
+  /**
+   * TODO left from {@link HFile} version 1: move this to StoreFile after
+   * Ryan's patch goes in to eliminate {@link KeyValue} here.
+   *
+   * @return the last row key, or null if the file is empty.
+   */
+  @Override
+  public byte[] getLastRowKey() {
+    byte[] lastKey = getLastKey();
+    if (lastKey == null)
+      return null;
+    return KeyValue.createKeyValueFromKey(lastKey).getRow();
+  }
+
+  /** @return number of KV entries in this HFile */
+  @Override
+  public long getEntries() {
+    return trailer.getEntryCount();
+  }
+
+  /** @return comparator */
+  @Override
+  public RawComparator getComparator() {
+    return comparator;
+  }
+
+  /** @return compression algorithm */
+  @Override
+  public Compression.Algorithm getCompressionAlgorithm() {
+    return compressAlgo;
+  }
+
+  /**
+   * @return the total heap size of data and meta block indexes in bytes. Does
+   *         not take into account non-root blocks of a multilevel data index.
+   */
+  public long indexSize() {
+    return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0)
+        + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize()
+            : 0);
+  }
+
+  @Override
+  public String getName() {
+    return name;
+  }
+
+  @Override
+  public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() {
+    return dataBlockIndexReader;
+  }
+
+  @Override
+  public FixedFileTrailer getTrailer() {
+    return trailer;
+  }
+
+  @Override
+  public FileInfo loadFileInfo() throws IOException {
+    return fileInfo;
+  }
+
+  /**
+   * An exception thrown when an operation requiring a scanner to be seeked
+   * is invoked on a scanner that is not seeked.
+   */
+  @SuppressWarnings("serial")
+  public static class NotSeekedException extends IllegalStateException {
+    public NotSeekedException() {
+      super("Not seeked to a key/value");
+    }
+  }
+
+  protected static abstract class Scanner implements HFileScanner {
+    protected ByteBuffer blockBuffer;
+
+    protected boolean cacheBlocks;
+    protected final boolean pread;
+    protected final boolean isCompaction;
+
+    protected int currKeyLen;
+    protected int currValueLen;
+    protected int currMemstoreTSLen;
+    protected long currMemstoreTS;
+
+    protected int blockFetches;
+
+    protected final HFile.Reader reader;
+
+    public Scanner(final HFile.Reader reader, final boolean cacheBlocks,
+        final boolean pread, final boolean isCompaction) {
+      this.reader = reader;
+      this.cacheBlocks = cacheBlocks;
+      this.pread = pread;
+      this.isCompaction = isCompaction;
+    }
+
+    @Override
+    public boolean isSeeked(){
+      return blockBuffer != null;
+    }
+
+    @Override
+    public String toString() {
+      return "HFileScanner for reader " + String.valueOf(getReader());
+    }
+
+    protected void assertSeeked() {
+      if (!isSeeked())
+        throw new NotSeekedException();
+    }
+
+    @Override
+    public int seekTo(byte[] key) throws IOException {
+      return seekTo(key, 0, key.length);
+    }
+    
+    @Override
+    public boolean seekBefore(byte[] key) throws IOException {
+      return seekBefore(key, 0, key.length);
+    }
+    
+    @Override
+    public int reseekTo(byte[] key) throws IOException {
+      return reseekTo(key, 0, key.length);
+    }
+
+    @Override
+    public HFile.Reader getReader() {
+      return reader;
+    }
+  }
+
+  /** For testing */
+  HFileBlock.FSReader getUncachedBlockReader() {
+    return fsBlockReader;
+  }
+
+  public Path getPath() {
+    return path;
+  }
+
+  @Override
+  public DataBlockEncoding getEncodingOnDisk() {
+    return dataBlockEncoder.getEncodingOnDisk();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
new file mode 100644
index 0000000..32aa2ed
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue.KeyComparator;
+import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Common functionality needed by all versions of {@link HFile} writers.
+ */
+public abstract class AbstractHFileWriter extends SchemaConfigured
+    implements HFile.Writer {
+
+  /** Key previously appended. Becomes the last key in the file. */
+  protected byte[] lastKeyBuffer = null;
+
+  protected int lastKeyOffset = -1;
+  protected int lastKeyLength = -1;
+
+  /** FileSystem stream to write into. */
+  protected FSDataOutputStream outputStream;
+
+  /** True if we opened the outputStream (and so will close it). */
+  protected final boolean closeOutputStream;
+
+  /** A "file info" block: a key-value map of file-wide metadata. */
+  protected FileInfo fileInfo = new HFile.FileInfo();
+
+  /** Number of uncompressed bytes we allow per block. */
+  protected final int blockSize;
+
+  /** Total # of key/value entries, i.e. how many times add() was called. */
+  protected long entryCount = 0;
+
+  /** Used for calculating the average key length. */
+  protected long totalKeyLength = 0;
+
+  /** Used for calculating the average value length. */
+  protected long totalValueLength = 0;
+
+  /** Total uncompressed bytes, maybe calculate a compression ratio later. */
+  protected long totalUncompressedBytes = 0;
+
+  /** Key comparator. Used to ensure we write in order. */
+  protected final RawComparator comparator;
+
+  /** Meta block names. */
+  protected List metaNames = new ArrayList();
+
+  /** {@link Writable}s representing meta block data. */
+  protected List metaData = new ArrayList();
+
+  /** The compression algorithm used. NONE if no compression. */
+  protected final Compression.Algorithm compressAlgo;
+  
+  /**
+   * The data block encoding which will be used.
+   * {@link NoOpDataBlockEncoder#INSTANCE} if there is no encoding.
+   */
+  protected final HFileDataBlockEncoder blockEncoder;
+
+  /** First key in a block. */
+  protected byte[] firstKeyInBlock = null;
+
+  /** May be null if we were passed a stream. */
+  protected final Path path;
+
+
+  /** Cache configuration for caching data on write. */
+  protected final CacheConfig cacheConf;
+
+  /**
+   * Name for this object used when logging or in toString. Is either
+   * the result of a toString on stream or else name of passed file Path.
+   */
+  protected final String name;
+
+  public AbstractHFileWriter(CacheConfig cacheConf,
+      FSDataOutputStream outputStream, Path path, int blockSize,
+      Compression.Algorithm compressAlgo,
+      HFileDataBlockEncoder dataBlockEncoder,
+      KeyComparator comparator) {
+    super(null, path);
+    this.outputStream = outputStream;
+    this.path = path;
+    this.name = path != null ? path.getName() : outputStream.toString();
+    this.blockSize = blockSize;
+    this.compressAlgo = compressAlgo == null
+        ? HFile.DEFAULT_COMPRESSION_ALGORITHM : compressAlgo;
+    this.blockEncoder = dataBlockEncoder != null
+        ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
+    this.comparator = comparator != null ? comparator
+        : Bytes.BYTES_RAWCOMPARATOR;
+
+    closeOutputStream = path != null;
+    this.cacheConf = cacheConf;
+  }
+
+  /**
+   * Add last bits of metadata to file info before it is written out.
+   */
+  protected void finishFileInfo() throws IOException {
+    if (lastKeyBuffer != null) {
+      // Make a copy. The copy is stuffed into HMapWritable. Needs a clean
+      // byte buffer. Won't take a tuple.
+      fileInfo.append(FileInfo.LASTKEY, Arrays.copyOfRange(lastKeyBuffer,
+          lastKeyOffset, lastKeyOffset + lastKeyLength), false);
+    }
+
+    // Average key length.
+    int avgKeyLen =
+        entryCount == 0 ? 0 : (int) (totalKeyLength / entryCount);
+    fileInfo.append(FileInfo.AVG_KEY_LEN, Bytes.toBytes(avgKeyLen), false);
+
+    // Average value length.
+    int avgValueLen =
+        entryCount == 0 ? 0 : (int) (totalValueLength / entryCount);
+    fileInfo.append(FileInfo.AVG_VALUE_LEN, Bytes.toBytes(avgValueLen), false);
+  }
+
+  /**
+   * Add to the file info. All added key/value pairs can be obtained using
+   * {@link HFile.Reader#loadFileInfo()}.
+   *
+   * @param k Key
+   * @param v Value
+   * @throws IOException in case the key or the value are invalid
+   */
+  @Override
+  public void appendFileInfo(final byte[] k, final byte[] v)
+      throws IOException {
+    fileInfo.append(k, v, true);
+  }
+
+  /**
+   * Sets the file info offset in the trailer, finishes up populating fields in
+   * the file info, and writes the file info into the given data output. The
+   * reason the data output is not always {@link #outputStream} is that we store
+   * file info as a block in version 2.
+   *
+   * @param trailer fixed file trailer
+   * @param out the data output to write the file info to
+   * @throws IOException
+   */
+  protected final void writeFileInfo(FixedFileTrailer trailer, DataOutput out)
+      throws IOException {
+    trailer.setFileInfoOffset(outputStream.getPos());
+    finishFileInfo();
+    fileInfo.write(out);
+  }
+
+  /**
+   * Checks that the given key does not violate the key order.
+   *
+   * @param key Key to check.
+   * @return true if the key is duplicate
+   * @throws IOException if the key or the key order is wrong
+   */
+  protected boolean checkKey(final byte[] key, final int offset,
+      final int length) throws IOException {
+    boolean isDuplicateKey = false;
+
+    if (key == null || length <= 0) {
+      throw new IOException("Key cannot be null or empty");
+    }
+    if (length > HFile.MAXIMUM_KEY_LENGTH) {
+      throw new IOException("Key length " + length + " > "
+          + HFile.MAXIMUM_KEY_LENGTH);
+    }
+    if (lastKeyBuffer != null) {
+      int keyComp = comparator.compare(lastKeyBuffer, lastKeyOffset,
+          lastKeyLength, key, offset, length);
+      if (keyComp > 0) {
+        throw new IOException("Added a key not lexically larger than"
+            + " previous key="
+            + Bytes.toStringBinary(key, offset, length)
+            + ", lastkey="
+            + Bytes.toStringBinary(lastKeyBuffer, lastKeyOffset,
+                lastKeyLength));
+      } else if (keyComp == 0) {
+        isDuplicateKey = true;
+      }
+    }
+    return isDuplicateKey;
+  }
+
+  /** Checks the given value for validity. */
+  protected void checkValue(final byte[] value, final int offset,
+      final int length) throws IOException {
+    if (value == null) {
+      throw new IOException("Value cannot be null");
+    }
+  }
+
+  /**
+   * @return Path or null if we were passed a stream rather than a Path.
+   */
+  @Override
+  public Path getPath() {
+    return path;
+  }
+
+  @Override
+  public String toString() {
+    return "writer=" + (path != null ? path.toString() : null) + ", name="
+        + name + ", compression=" + compressAlgo.getName();
+  }
+
+  /**
+   * Sets remaining trailer fields, writes the trailer to disk, and optionally
+   * closes the output stream.
+   */
+  protected void finishClose(FixedFileTrailer trailer) throws IOException {
+    trailer.setMetaIndexCount(metaNames.size());
+    trailer.setTotalUncompressedBytes(totalUncompressedBytes+ trailer.getTrailerSize());
+    trailer.setEntryCount(entryCount);
+    trailer.setCompressionCodec(compressAlgo);
+
+    trailer.serialize(outputStream);
+
+    if (closeOutputStream) {
+      outputStream.close();
+      outputStream = null;
+    }
+  }
+
+  public static Compression.Algorithm compressionByName(String algoName) {
+    if (algoName == null)
+      return HFile.DEFAULT_COMPRESSION_ALGORITHM;
+    return Compression.getCompressionAlgorithmByName(algoName);
+  }
+
+  /** A helper method to create HFile output streams in constructors */
+  protected static FSDataOutputStream createOutputStream(Configuration conf,
+      FileSystem fs, Path path) throws IOException {
+    FsPermission perms = FSUtils.getFilePermissions(fs, conf,
+        HConstants.DATA_FILE_UMASK_KEY);
+    return FSUtils.create(fs, path, perms);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
new file mode 100644
index 0000000..e3908b3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
@@ -0,0 +1,126 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Block cache interface. Anything that implements the {@link Cacheable}
+ * interface can be put in the cache.
+ */
+public interface BlockCache {
+  /**
+   * Add block to cache.
+   * @param cacheKey The block's cache key.
+   * @param buf The block contents wrapped in a ByteBuffer.
+   * @param inMemory Whether block should be treated as in-memory
+   */
+  public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory);
+
+  /**
+   * Add block to cache (defaults to not in-memory).
+   * @param cacheKey The block's cache key.
+   * @param buf The object to cache.
+   */
+  public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf);
+
+  /**
+   * Fetch block from cache.
+   * @param cacheKey Block to fetch.
+   * @param caching Whether this request has caching enabled (used for stats)
+   * @param repeat Whether this is a repeat lookup for the same block
+   *        (used to avoid double counting cache misses when doing double-check locking)
+   *        {@see HFileReaderV2#readBlock(long, long, boolean, boolean, boolean, BlockType)}
+   * @return Block or null if block is not in 2 cache.
+   */
+  public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat);
+
+  /**
+   * Evict block from cache.
+   * @param cacheKey Block to evict
+   * @return true if block existed and was evicted, false if not
+   */
+  public boolean evictBlock(BlockCacheKey cacheKey);
+
+  /**
+   * Evicts all blocks for the given HFile.
+   *
+   * @return the number of blocks evicted
+   */
+  public int evictBlocksByHfileName(String hfileName);
+
+  /**
+   * Get the statistics for this block cache.
+   * @return Stats
+   */
+  public CacheStats getStats();
+
+  /**
+   * Shutdown the cache.
+   */
+  public void shutdown();
+
+  /**
+   * Returns the total size of the block cache, in bytes.
+   * @return size of cache, in bytes
+   */
+  public long size();
+
+  /**
+   * Returns the free size of the block cache, in bytes.
+   * @return free space in cache, in bytes
+   */
+  public long getFreeSize();
+
+  /**
+   * Returns the occupied size of the block cache, in bytes.
+   * @return occupied space in cache, in bytes
+   */
+  public long getCurrentSize();
+
+  /**
+   * Returns the number of evictions that have occurred.
+   * @return number of evictions
+   */
+  public long getEvictedCount();
+
+  /**
+   * Returns the number of blocks currently cached in the block cache.
+   * @return number of blocks in the cache
+   */
+  public long getBlockCount();
+
+  /**
+   * Performs a BlockCache summary and returns a List of BlockCacheColumnFamilySummary objects.
+   * This method could be fairly heavyweight in that it evaluates the entire HBase file-system
+   * against what is in the RegionServer BlockCache.
+   * 

      
+   * The contract of this interface is to return the List in sorted order by Table name, then
+   * ColumnFamily.
+   *
+   * @param conf HBaseConfiguration
+   * @return List of BlockCacheColumnFamilySummary
+   * @throws IOException exception
+   */
+  public List getBlockCacheColumnFamilySummaries(Configuration conf) throws IOException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheColumnFamilySummary.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheColumnFamilySummary.java
new file mode 100644
index 0000000..34513f1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheColumnFamilySummary.java
@@ -0,0 +1,246 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * BlockCacheColumnFamilySummary represents a summary of the blockCache usage 
+ * at Table/ColumnFamily granularity.
+ * 

      
+ * As ColumnFamilies are owned by Tables, a summary by ColumnFamily implies that
+ * the owning Table is included in the summarization.
+ *
+ */
+public class BlockCacheColumnFamilySummary implements Writable, Comparable {
+
+  private String table = "";
+  private String columnFamily = "";
+  private int blocks;
+  private long heapSize;
+
+  /**
+   * Default constructor for Writable
+   */
+  public BlockCacheColumnFamilySummary() {
+    
+  }
+  
+  /**
+   * 
+   * @param table table
+   * @param columnFamily columnFamily
+   */
+  public BlockCacheColumnFamilySummary(String table, String columnFamily) {
+    this.table = table;
+    this.columnFamily = columnFamily;
+  }
+  
+  /**
+   * 
+   * @return table
+   */
+  public String getTable() {
+    return table;
+  }
+  /**
+   * 
+   * @param table (table that owns the cached block)
+   */
+  public void setTable(String table) {
+    this.table = table;
+  }
+  /**
+   * 
+   * @return columnFamily
+   */
+  public String getColumnFamily() {
+    return columnFamily;
+  }
+  /**
+   * 
+   * @param columnFamily (columnFamily that owns the cached block)
+   */
+  public void setColumnFamily(String columnFamily) {
+    this.columnFamily = columnFamily;
+  }
+  
+  /**
+   * 
+   * @return blocks in the cache
+   */
+  public int getBlocks() {
+    return blocks;
+  }
+  /**
+   * 
+   * @param blocks in the cache
+   */
+  public void setBlocks(int blocks) {
+    this.blocks = blocks;
+  }
+  
+  /**
+   * 
+   * @return heapSize in the cache
+   */
+  public long getHeapSize() {
+    return heapSize;
+  }
+  
+  /**
+   * Increments the number of blocks in the cache for this entry
+   */
+  public void incrementBlocks() {
+    this.blocks++;
+  }
+
+  /**
+   * 
+   * @param heapSize to increment
+   */
+  public void incrementHeapSize(long heapSize) {
+    this.heapSize = this.heapSize + heapSize;
+  }
+
+  /**
+   * 
+   * @param heapSize (total heapSize for the table/CF)
+   */
+  public void setHeapSize(long heapSize) {
+    this.heapSize = heapSize;
+  }
+  
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    table = in.readUTF();
+    columnFamily = in.readUTF();
+    blocks = in.readInt();
+    heapSize = in.readLong();
+  }
+  
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeUTF(table);
+    out.writeUTF(columnFamily);
+    out.writeInt(blocks);
+    out.writeLong(heapSize);
+  }
+  
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result
+        + ((columnFamily == null) ? 0 : columnFamily.hashCode());
+    result = prime * result + ((table == null) ? 0 : table.hashCode());
+    return result;
+  }
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    BlockCacheColumnFamilySummary other = (BlockCacheColumnFamilySummary) obj;
+    if (columnFamily == null) {
+      if (other.columnFamily != null)
+        return false;
+    } else if (!columnFamily.equals(other.columnFamily))
+      return false;
+    if (table == null) {
+      if (other.table != null)
+        return false;
+    } else if (!table.equals(other.table))
+      return false;
+    return true;
+  }
+  
+  
+  
+  @Override
+  public String toString() {
+    return "BlockCacheSummaryEntry [table=" + table + ", columnFamily="
+        + columnFamily + ", blocks=" + blocks + ", heapSize=" + heapSize + "]";
+  }
+  
+  /**
+   * Construct a BlockCacheSummaryEntry from a full StoreFile Path
+   * 

      
+   * The path is expected to be in the format of...
+   * 
      +   * hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
+   * 
      
+   * ... where: 
      
+   *  '-ROOT-' = Table 
      
+   * '70236052' = Region 
      
+   * 'info' = ColumnFamily 
      
+   * '3944417774205889744' = StoreFile
+   * 
+   * @param path (full StoreFile Path)
+   * @return BlockCacheSummaryEntry
+   */
+  public static BlockCacheColumnFamilySummary createFromStoreFilePath(Path path) {
+       
+    // The full path will look something like this...
+    // hdfs://localhost:51169/user/doug.meil/-ROOT-/70236052/info/3944417774205889744
+    //                                        tbl    region   cf   sf
+    String sp = path.toString();
+    String s[] = sp.split("\\/");
+
+    BlockCacheColumnFamilySummary bcse = null;
+    if (s.length >= 4) {
+      // why 4?   StoreFile, CF, Region, Table
+      String table = s[s.length - 4];  // 4th from the end
+      String cf = s[s.length - 2];     // 2nd from the end
+      bcse = new BlockCacheColumnFamilySummary(table, cf);
+    } 
+    return bcse;
+  }
+
+  @Override
+  public int compareTo(BlockCacheColumnFamilySummary o) {
+    int i = table.compareTo(o.getTable());
+    if (i != 0) {
+      return i;
+    } 
+    return columnFamily.compareTo(o.getColumnFamily());
+  }
+
+  /**
+   * Creates a new BlockCacheSummaryEntry
+   * 
+   * @param e BlockCacheSummaryEntry
+   * @return new BlockCacheSummaryEntry
+   */
+  public static BlockCacheColumnFamilySummary create(BlockCacheColumnFamilySummary e) {
+    BlockCacheColumnFamilySummary e2 = new BlockCacheColumnFamilySummary();
+    e2.setTable(e.getTable());
+    e2.setColumnFamily(e.getColumnFamily());
+    return e2;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java
new file mode 100644
index 0000000..d1ab403
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+
+/**
+ * Cache Key for use with implementations of {@link BlockCache}
+ */
+public class BlockCacheKey implements HeapSize {
+  private final String hfileName;
+  private final long offset;
+  private final DataBlockEncoding encoding;
+
+  public BlockCacheKey(String file, long offset, DataBlockEncoding encoding,
+      BlockType blockType) {
+    this.hfileName = file;
+    this.offset = offset;
+    // We add encoding to the cache key only for data blocks. If the block type
+    // is unknown (this should never be the case in production), we just use
+    // the provided encoding, because it might be a data block.
+    this.encoding = (blockType == null || blockType.isData()) ? encoding :
+        DataBlockEncoding.NONE;
+  }
+
+  /**
+   * Construct a new BlockCacheKey
+   * @param file The name of the HFile this block belongs to.
+   * @param offset Offset of the block into the file
+   */
+  public BlockCacheKey(String file, long offset) {
+    this(file, offset, DataBlockEncoding.NONE, null);
+  }
+
+  @Override
+  public int hashCode() {
+    return hfileName.hashCode() * 127 + (int) (offset ^ (offset >>> 32)) +
+        encoding.ordinal() * 17;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (o instanceof BlockCacheKey) {
+      BlockCacheKey k = (BlockCacheKey) o;
+      return offset == k.offset
+          && (hfileName == null ? k.hfileName == null : hfileName
+              .equals(k.hfileName));
+    } else {
+      return false;
+    }
+  }
+
+  @Override
+  public String toString() {
+    return hfileName + "_" + offset
+        + (encoding == DataBlockEncoding.NONE ? "" : "_" + encoding);
+  }
+
+  /**
+   * Strings have two bytes per character due to default Java Unicode encoding
+   * (hence length times 2).
+   */
+  @Override
+  public long heapSize() {
+    return ClassSize.align(ClassSize.OBJECT + 2 * hfileName.length() +
+        Bytes.SIZEOF_LONG + 2 * ClassSize.REFERENCE);
+  }
+
+  // can't avoid this unfortunately
+  /**
+   * @return The hfileName portion of this cache key
+   */
+  public String getHfileName() {
+    return hfileName;
+  }
+
+  public DataBlockEncoding getDataBlockEncoding() {
+    return encoding;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
new file mode 100644
index 0000000..5797694
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
@@ -0,0 +1,218 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Various types of {@link HFile} blocks. Ordinal values of these enum constants
+ * must not be relied upon. The values in the enum appear in the order they
+ * appear in a version 2 {@link HFile}.
+ */
+public enum BlockType {
+
+  // Scanned block section
+
+  /** Data block, both versions */
+  DATA("DATABLK*", BlockCategory.DATA),
+
+  /** An encoded data block (e.g. with prefix compression), version 2 */
+  ENCODED_DATA("DATABLKE", BlockCategory.DATA) {
+    @Override
+    public int getId() {
+      return DATA.ordinal();
+    }
+  },
+
+  /** Version 2 leaf index block. Appears in the data block section */
+  LEAF_INDEX("IDXLEAF2", BlockCategory.INDEX),
+
+  /** Bloom filter block, version 2 */
+  BLOOM_CHUNK("BLMFBLK2", BlockCategory.BLOOM),
+
+  // Non-scanned block section
+
+  /** Meta blocks */
+  META("METABLKc", BlockCategory.META),
+
+  /** Intermediate-level version 2 index in the non-data block section */
+  INTERMEDIATE_INDEX("IDXINTE2", BlockCategory.INDEX),
+
+  // Load-on-open section.
+
+  /** Root index block, also used for the single-level meta index, version 2 */
+  ROOT_INDEX("IDXROOT2", BlockCategory.INDEX),
+
+  /** File info, version 2 */
+  FILE_INFO("FILEINF2", BlockCategory.META),
+
+  /** General Bloom filter metadata, version 2 */
+  GENERAL_BLOOM_META("BLMFMET2", BlockCategory.BLOOM),
+
+  /** Delete Family Bloom filter metadata, version 2 */
+  DELETE_FAMILY_BLOOM_META("DFBLMET2", BlockCategory.BLOOM),
+
+  // Trailer
+
+  /** Fixed file trailer, both versions (always just a magic string) */
+  TRAILER("TRABLK\"$", BlockCategory.META),
+
+  // Legacy blocks
+
+  /** Block index magic string in version 1 */
+  INDEX_V1("IDXBLK)+", BlockCategory.INDEX);
+
+  public enum BlockCategory {
+    DATA, META, INDEX, BLOOM, ALL_CATEGORIES, UNKNOWN;
+
+    /**
+     * Throws an exception if the block category passed is the special category
+     * meaning "all categories".
+     */
+    public void expectSpecific() {
+      if (this == ALL_CATEGORIES) {
+        throw new IllegalArgumentException("Expected a specific block " +
+            "category but got " + this);
+      }
+    }
+  }
+
+  public static final int MAGIC_LENGTH = 8;
+
+  private final byte[] magic;
+  private final BlockCategory metricCat;
+
+  private BlockType(String magicStr, BlockCategory metricCat) {
+    magic = Bytes.toBytes(magicStr);
+    this.metricCat = metricCat;
+    assert magic.length == MAGIC_LENGTH;
+  }
+
+  /**
+   * Use this instead of {@link #ordinal()}. They work exactly the same, except
+   * DATA and ENCODED_DATA get the same id using this method (overridden for
+   * {@link #ENCODED_DATA}).
+   * @return block type id from 0 to the number of block types - 1
+   */
+  public int getId() {
+    // Default implementation, can be overridden for individual enum members.
+    return ordinal();
+  }
+
+  public void writeToStream(OutputStream out) throws IOException {
+    out.write(magic);
+  }
+
+  public void write(DataOutput out) throws IOException {
+    out.write(magic);
+  }
+
+  public void write(ByteBuffer buf) {
+    buf.put(magic);
+  }
+
+  public BlockCategory getCategory() {
+    return metricCat;
+  }
+
+  public static BlockType parse(byte[] buf, int offset, int length)
+      throws IOException {
+    if (length != MAGIC_LENGTH) {
+      throw new IOException("Magic record of invalid length: "
+          + Bytes.toStringBinary(buf, offset, length));
+    }
+
+    for (BlockType blockType : values())
+      if (Bytes.compareTo(blockType.magic, 0, MAGIC_LENGTH, buf, offset,
+          MAGIC_LENGTH) == 0)
+        return blockType;
+
+    throw new IOException("Invalid HFile block magic: "
+        + Bytes.toStringBinary(buf, offset, MAGIC_LENGTH));
+  }
+
+  public static BlockType read(DataInputStream in) throws IOException {
+    byte[] buf = new byte[MAGIC_LENGTH];
+    in.readFully(buf);
+    return parse(buf, 0, buf.length);
+  }
+
+  public static BlockType read(ByteBuffer buf) throws IOException {
+    BlockType blockType = parse(buf.array(),
+        buf.arrayOffset() + buf.position(),
+        Math.min(buf.limit() - buf.position(), MAGIC_LENGTH));
+
+    // If we got here, we have read exactly MAGIC_LENGTH bytes.
+    buf.position(buf.position() + MAGIC_LENGTH);
+    return blockType;
+  }
+
+  /**
+   * Put the magic record out to the specified byte array position.
+   *
+   * @param bytes the byte array
+   * @param offset position in the array
+   * @return incremented offset
+   */
+  public int put(byte[] bytes, int offset) {
+    System.arraycopy(magic, 0, bytes, offset, MAGIC_LENGTH);
+    return offset + MAGIC_LENGTH;
+  }
+
+  /**
+   * Reads a magic record of the length {@link #MAGIC_LENGTH} from the given
+   * stream and expects it to match this block type.
+   */
+  public void readAndCheck(DataInputStream in) throws IOException {
+    byte[] buf = new byte[MAGIC_LENGTH];
+    in.readFully(buf);
+    if (Bytes.compareTo(buf, magic) != 0) {
+      throw new IOException("Invalid magic: expected "
+          + Bytes.toStringBinary(magic) + ", got " + Bytes.toStringBinary(buf));
+    }
+  }
+
+  /**
+   * Reads a magic record of the length {@link #MAGIC_LENGTH} from the given
+   * byte buffer and expects it to match this block type.
+   */
+  public void readAndCheck(ByteBuffer in) throws IOException {
+    byte[] buf = new byte[MAGIC_LENGTH];
+    in.get(buf);
+    if (Bytes.compareTo(buf, magic) != 0) {
+      throw new IOException("Invalid magic: expected "
+          + Bytes.toStringBinary(magic) + ", got " + Bytes.toStringBinary(buf));
+    }
+  }
+
+  /**
+   * @return whether this block type is encoded or unencoded data block
+   */
+  public final boolean isData() {
+    return this == DATA || this == ENCODED_DATA;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockWithScanInfo.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockWithScanInfo.java
new file mode 100644
index 0000000..ceb05e3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockWithScanInfo.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+/**
+ * BlockWithScanInfo is wrapper class for HFileBlock with other attributes. These attributes are
+ * supposed to be much cheaper to be maintained in each caller thread than in HFileBlock itself.
+ */
+public class BlockWithScanInfo {
+  private final HFileBlock hFileBlock;
+  /**
+   * The first key in the next block following this one in the HFile.
+   * If this key is unknown, this is reference-equal with HConstants.NO_NEXT_INDEXED_KEY
+   */
+  private final byte[] nextIndexedKey;
+
+  public BlockWithScanInfo(HFileBlock hFileBlock, byte[] nextIndexedKey) {
+    this.hFileBlock = hFileBlock;
+    this.nextIndexedKey = nextIndexedKey;
+  }
+
+  public HFileBlock getHFileBlock() {
+    return hFileBlock;
+  }
+
+  public byte[] getNextIndexedKey() {
+    return nextIndexedKey;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/BoundedRangeFileInputStream.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/BoundedRangeFileInputStream.java
new file mode 100644
index 0000000..f7da04d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/BoundedRangeFileInputStream.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+
+/**
+ * BoundedRangeFIleInputStream abstracts a contiguous region of a Hadoop
+ * FSDataInputStream as a regular input stream. One can create multiple
+ * BoundedRangeFileInputStream on top of the same FSDataInputStream and they
+ * would not interfere with each other.
+ * Copied from hadoop-335 tfile.
+ */
+class BoundedRangeFileInputStream  extends InputStream {
+
+  private FSDataInputStream in;
+  private long pos;
+  private long end;
+  private long mark;
+  private final byte[] oneByte = new byte[1];
+  private final boolean pread;
+
+  /**
+   * Constructor
+   *
+   * @param in
+   *          The FSDataInputStream we connect to.
+   * @param offset
+   *          Beginning offset of the region.
+   * @param length
+   *          Length of the region.
+   * @param pread If true, use Filesystem positional read rather than seek+read.
+   *
+   *          The actual length of the region may be smaller if (off_begin +
+   *          length) goes beyond the end of FS input stream.
+   */
+  public BoundedRangeFileInputStream(FSDataInputStream in, long offset,
+      long length, final boolean pread) {
+    if (offset < 0 || length < 0) {
+      throw new IndexOutOfBoundsException("Invalid offset/length: " + offset
+          + "/" + length);
+    }
+
+    this.in = in;
+    this.pos = offset;
+    this.end = offset + length;
+    this.mark = -1;
+    this.pread = pread;
+  }
+
+  @Override
+  public int available() throws IOException {
+    int avail = in.available();
+    if (pos + avail > end) {
+      avail = (int) (end - pos);
+    }
+
+    return avail;
+  }
+
+  @Override
+  public int read() throws IOException {
+    int ret = read(oneByte);
+    if (ret == 1) return oneByte[0] & 0xff;
+    return -1;
+  }
+
+  @Override
+  public int read(byte[] b) throws IOException {
+    return read(b, 0, b.length);
+  }
+
+  @Override
+  public int read(byte[] b, int off, int len) throws IOException {
+    if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
+      throw new IndexOutOfBoundsException();
+    }
+
+    int n = (int) Math.min(Integer.MAX_VALUE, Math.min(len, (end - pos)));
+    if (n == 0) return -1;
+    int ret = 0;
+    if (this.pread) {
+      ret = in.read(pos, b, off, n);
+    } else {
+      synchronized (in) {
+        in.seek(pos);
+        ret = in.read(b, off, n);
+      }
+    }
+    if (ret < 0) {
+      end = pos;
+      return -1;
+    }
+    pos += ret;
+    return ret;
+  }
+
+  @Override
+  /*
+   * We may skip beyond the end of the file.
+   */
+  public long skip(long n) throws IOException {
+    long len = Math.min(n, end - pos);
+    pos += len;
+    return len;
+  }
+
+  @Override
+  public void mark(int readlimit) {
+    mark = pos;
+  }
+
+  @Override
+  public void reset() throws IOException {
+    if (mark < 0) throw new IOException("Resetting to invalid mark");
+    pos = mark;
+  }
+
+  @Override
+  public boolean markSupported() {
+    return true;
+  }
+
+  @Override
+  public void close() {
+    // Invalidate the state of the stream.
+    in = null;
+    pos = end;
+    mark = -1;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
new file mode 100644
index 0000000..13585b9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -0,0 +1,359 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.DirectMemoryUtils;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Stores all of the cache objects and configuration for a single HFile.
+ */
+public class CacheConfig {
+  private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
+
+  /**
+   * Configuration key to cache data blocks on write. There are separate
+   * switches for bloom blocks and non-root index blocks.
+   */
+  public static final String CACHE_BLOCKS_ON_WRITE_KEY =
+      "hbase.rs.cacheblocksonwrite";
+
+  /**
+   * Configuration key to cache leaf and intermediate-level index blocks on
+   * write.
+   */
+  public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
+      "hfile.block.index.cacheonwrite";
+
+  /**
+   * Configuration key to cache compound bloom filter blocks on write.
+   */
+  public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
+      "hfile.block.bloom.cacheonwrite";
+
+  /**
+   * TODO: Implement this (jgray)
+   * Configuration key to cache data blocks in compressed format.
+   */
+  public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
+      "hbase.rs.blockcache.cachedatacompressed";
+
+  /**
+   * Configuration key to evict all blocks of a given file from the block cache
+   * when the file is closed.
+   */
+  public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
+      "hbase.rs.evictblocksonclose";
+
+  // Defaults
+
+  public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
+  public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
+  public static final boolean DEFAULT_IN_MEMORY = false;
+  public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
+  public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
+  public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
+  public static final boolean DEFAULT_COMPRESSED_CACHE = false;
+
+  /** Local reference to the block cache, null if completely disabled */
+  private final BlockCache blockCache;
+
+  /**
+   * Whether blocks should be cached on read (default is on if there is a
+   * cache but this can be turned off on a per-family or per-request basis)
+   */
+  private boolean cacheDataOnRead;
+
+  /** Whether blocks should be flagged as in-memory when being cached */
+  private final boolean inMemory;
+
+  /** Whether data blocks should be cached when new files are written */
+  private boolean cacheDataOnWrite;
+
+  /** Whether index blocks should be cached when new files are written */
+  private final boolean cacheIndexesOnWrite;
+
+  /** Whether compound bloom filter blocks should be cached on write */
+  private final boolean cacheBloomsOnWrite;
+
+  /** Whether blocks of a file should be evicted when the file is closed */
+  private boolean evictOnClose;
+
+  /** Whether data blocks should be stored in compressed form in the cache */
+  private final boolean cacheCompressed;
+
+  /**
+   * Create a cache configuration using the specified configuration object and
+   * family descriptor.
+   * @param conf hbase configuration
+   * @param family column family configuration
+   */
+  public CacheConfig(Configuration conf, HColumnDescriptor family) {
+    this(CacheConfig.instantiateBlockCache(conf),
+        family.isBlockCacheEnabled(),
+        family.isInMemory(),
+        // For the following flags we enable them regardless of per-schema settings
+        // if they are enabled in the global configuration.
+        conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
+            DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
+        conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
+            DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
+        conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
+            DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
+        conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
+            DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
+        conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE)
+     );
+  }
+
+  /**
+   * Create a cache configuration using the specified configuration object and
+   * defaults for family level settings.
+   * @param conf hbase configuration
+   */
+  public CacheConfig(Configuration conf) {
+    this(CacheConfig.instantiateBlockCache(conf),
+        DEFAULT_CACHE_DATA_ON_READ,
+        DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set
+                           // strictly from conf
+        conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
+        conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
+            DEFAULT_CACHE_INDEXES_ON_WRITE),
+            conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
+                DEFAULT_CACHE_BLOOMS_ON_WRITE),
+        conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
+        conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY,
+            DEFAULT_COMPRESSED_CACHE)
+     );
+  }
+
+  /**
+   * Create a block cache configuration with the specified cache and
+   * configuration parameters.
+   * @param blockCache reference to block cache, null if completely disabled
+   * @param cacheDataOnRead whether data blocks should be cached on read
+   * @param inMemory whether blocks should be flagged as in-memory
+   * @param cacheDataOnWrite whether data blocks should be cached on write
+   * @param cacheIndexesOnWrite whether index blocks should be cached on write
+   * @param cacheBloomsOnWrite whether blooms should be cached on write
+   * @param evictOnClose whether blocks should be evicted when HFile is closed
+   * @param cacheCompressed whether to store blocks as compressed in the cache
+   */
+  CacheConfig(final BlockCache blockCache,
+      final boolean cacheDataOnRead, final boolean inMemory,
+      final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
+      final boolean cacheBloomsOnWrite, final boolean evictOnClose,
+      final boolean cacheCompressed) {
+    this.blockCache = blockCache;
+    this.cacheDataOnRead = cacheDataOnRead;
+    this.inMemory = inMemory;
+    this.cacheDataOnWrite = cacheDataOnWrite;
+    this.cacheIndexesOnWrite = cacheIndexesOnWrite;
+    this.cacheBloomsOnWrite = cacheBloomsOnWrite;
+    this.evictOnClose = evictOnClose;
+    this.cacheCompressed = cacheCompressed;
+  }
+
+  /**
+   * Constructs a cache configuration copied from the specified configuration.
+   * @param cacheConf
+   */
+  public CacheConfig(CacheConfig cacheConf) {
+    this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
+        cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
+        cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
+        cacheConf.cacheCompressed);
+  }
+
+  /**
+   * Checks whether the block cache is enabled.
+   */
+  public boolean isBlockCacheEnabled() {
+    return this.blockCache != null;
+  }
+
+  /**
+   * Returns the block cache.
+   * @return the block cache, or null if caching is completely disabled
+   */
+  public BlockCache getBlockCache() {
+    return this.blockCache;
+  }
+
+  /**
+   * Returns whether the blocks of this HFile should be cached on read or not.
+   * @return true if blocks should be cached on read, false if not
+   */
+  public boolean shouldCacheDataOnRead() {
+    return isBlockCacheEnabled() && cacheDataOnRead;
+  }
+
+  /**
+   * Should we cache a block of a particular category? We always cache
+   * important blocks such as index blocks, as long as the block cache is
+   * available.
+   */
+  public boolean shouldCacheBlockOnRead(BlockCategory category) {
+    boolean shouldCache = isBlockCacheEnabled()
+        && (cacheDataOnRead ||
+            category == BlockCategory.INDEX ||
+            category == BlockCategory.BLOOM);
+    return shouldCache;
+  }
+
+  /**
+   * @return true if blocks in this file should be flagged as in-memory
+   */
+  public boolean isInMemory() {
+    return isBlockCacheEnabled() && this.inMemory;
+  }
+
+  /**
+   * @return true if data blocks should be written to the cache when an HFile is
+   *         written, false if not
+   */
+  public boolean shouldCacheDataOnWrite() {
+    return isBlockCacheEnabled() && this.cacheDataOnWrite;
+  }
+
+  /**
+   * Only used for testing.
+   * @param cacheDataOnWrite whether data blocks should be written to the cache
+   *                         when an HFile is written
+   */
+  public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
+    this.cacheDataOnWrite = cacheDataOnWrite;
+  }
+
+  /**
+   * @return true if index blocks should be written to the cache when an HFile
+   *         is written, false if not
+   */
+  public boolean shouldCacheIndexesOnWrite() {
+    return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
+  }
+
+  /**
+   * @return true if bloom blocks should be written to the cache when an HFile
+   *         is written, false if not
+   */
+  public boolean shouldCacheBloomsOnWrite() {
+    return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
+  }
+
+  /**
+   * @return true if blocks should be evicted from the cache when an HFile
+   *         reader is closed, false if not
+   */
+  public boolean shouldEvictOnClose() {
+    return isBlockCacheEnabled() && this.evictOnClose;
+  }
+
+  /**
+   * Only used for testing.
+   * @param evictOnClose whether blocks should be evicted from the cache when an
+   *                     HFile reader is closed
+   */
+  public void setEvictOnClose(boolean evictOnClose) {
+    this.evictOnClose = evictOnClose;
+  }
+
+  /**
+   * @return true if blocks should be compressed in the cache, false if not
+   */
+  public boolean shouldCacheCompressed() {
+    return isBlockCacheEnabled() && this.cacheCompressed;
+  }
+
+  @Override
+  public String toString() {
+    if (!isBlockCacheEnabled()) {
+      return "CacheConfig:disabled";
+    }
+    return "CacheConfig:enabled " +
+      "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
+      "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
+      "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
+      "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
+      "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
+      "[cacheCompressed=" + shouldCacheCompressed() + "]";
+  }
+
+  // Static block cache reference and methods
+
+  /**
+   * Static reference to the block cache, or null if no caching should be used
+   * at all.
+   */
+  private static BlockCache globalBlockCache;
+
+  /** Boolean whether we have disabled the block cache entirely. */
+  private static boolean blockCacheDisabled = false;
+
+  /**
+   * Returns the block cache or null in case none should be used.
+   *
+   * @param conf  The current configuration.
+   * @return The block cache or null.
+   */
+  private static synchronized BlockCache instantiateBlockCache(
+      Configuration conf) {
+    if (globalBlockCache != null) return globalBlockCache;
+    if (blockCacheDisabled) return null;
+
+    float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
+      HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
+    if (cachePercentage == 0L) {
+      blockCacheDisabled = true;
+      return null;
+    }
+    if (cachePercentage > 1.0) {
+      throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
+        " must be between 0.0 and 1.0, and not > 1.0");
+    }
+
+    // Calculate the amount of heap to give the heap.
+    MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+    long cacheSize = (long)(mu.getMax() * cachePercentage);
+    int blockSize = conf.getInt("hbase.offheapcache.minblocksize",
+        HFile.DEFAULT_BLOCKSIZE);
+    long offHeapCacheSize =
+      (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
+          DirectMemoryUtils.getDirectMemorySize());
+    LOG.info("Allocating LruBlockCache with maximum size " +
+      StringUtils.humanReadableInt(cacheSize));
+    if (offHeapCacheSize <= 0) {
+      globalBlockCache = new LruBlockCache(cacheSize,
+          StoreFile.DEFAULT_BLOCKSIZE_SMALL, conf);
+    } else {
+      globalBlockCache = new DoubleBlockCache(cacheSize, offHeapCacheSize,
+          StoreFile.DEFAULT_BLOCKSIZE_SMALL, blockSize, conf);
+    }
+    return globalBlockCache;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
new file mode 100644
index 0000000..439d431
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
@@ -0,0 +1,196 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Class that implements cache metrics.
+ */
+public class CacheStats {
+
+  /** Sliding window statistics. The number of metric periods to include in
+   * sliding window hit ratio calculations.
+   */
+  static final int DEFAULT_WINDOW_PERIODS = 5;
+
+  /** The number of getBlock requests that were cache hits */
+  private final AtomicLong hitCount = new AtomicLong(0);
+  /**
+   * The number of getBlock requests that were cache hits, but only from
+   * requests that were set to use the block cache.  This is because all reads
+   * attempt to read from the block cache even if they will not put new blocks
+   * into the block cache.  See HBASE-2253 for more information.
+   */
+  private final AtomicLong hitCachingCount = new AtomicLong(0);
+  /** The number of getBlock requests that were cache misses */
+  private final AtomicLong missCount = new AtomicLong(0);
+  /**
+   * The number of getBlock requests that were cache misses, but only from
+   * requests that were set to use the block cache.
+   */
+  private final AtomicLong missCachingCount = new AtomicLong(0);
+  /** The number of times an eviction has occurred */
+  private final AtomicLong evictionCount = new AtomicLong(0);
+  /** The total number of blocks that have been evicted */
+  private final AtomicLong evictedBlockCount = new AtomicLong(0);
+
+  /** The number of metrics periods to include in window */
+  private final int numPeriodsInWindow;
+  /** Hit counts for each period in window */
+  private final long [] hitCounts;
+  /** Caching hit counts for each period in window */
+  private final long [] hitCachingCounts;
+  /** Access counts for each period in window */
+  private final long [] requestCounts;
+  /** Caching access counts for each period in window */
+  private final long [] requestCachingCounts;
+  /** Last hit count read */
+  private long lastHitCount = 0;
+  /** Last hit caching count read */
+  private long lastHitCachingCount = 0;
+  /** Last request count read */
+  private long lastRequestCount = 0;
+  /** Last request caching count read */
+  private long lastRequestCachingCount = 0;
+  /** Current window index (next to be updated) */
+  private int windowIndex = 0;
+
+  public CacheStats() {
+    this(DEFAULT_WINDOW_PERIODS);
+  }
+
+  public CacheStats(int numPeriodsInWindow) {
+    this.numPeriodsInWindow = numPeriodsInWindow;
+    this.hitCounts = initializeZeros(numPeriodsInWindow);
+    this.hitCachingCounts = initializeZeros(numPeriodsInWindow);
+    this.requestCounts = initializeZeros(numPeriodsInWindow);
+    this.requestCachingCounts = initializeZeros(numPeriodsInWindow);
+  }
+
+  public void miss(boolean caching) {
+    missCount.incrementAndGet();
+    if (caching) missCachingCount.incrementAndGet();
+  }
+
+  public void hit(boolean caching) {
+    hitCount.incrementAndGet();
+    if (caching) hitCachingCount.incrementAndGet();
+  }
+
+  public void evict() {
+    evictionCount.incrementAndGet();
+  }
+
+  public void evicted() {
+    evictedBlockCount.incrementAndGet();
+  }
+
+  public long getRequestCount() {
+    return getHitCount() + getMissCount();
+  }
+
+  public long getRequestCachingCount() {
+    return getHitCachingCount() + getMissCachingCount();
+  }
+
+  public long getMissCount() {
+    return missCount.get();
+  }
+
+  public long getMissCachingCount() {
+    return missCachingCount.get();
+  }
+
+  public long getHitCount() {
+    return hitCount.get();
+  }
+
+  public long getHitCachingCount() {
+    return hitCachingCount.get();
+  }
+
+  public long getEvictionCount() {
+    return evictionCount.get();
+  }
+
+  public long getEvictedCount() {
+    return evictedBlockCount.get();
+  }
+
+  public double getHitRatio() {
+    return ((float)getHitCount()/(float)getRequestCount());
+  }
+
+  public double getHitCachingRatio() {
+    return ((float)getHitCachingCount()/(float)getRequestCachingCount());
+  }
+
+  public double getMissRatio() {
+    return ((float)getMissCount()/(float)getRequestCount());
+  }
+
+  public double getMissCachingRatio() {
+    return ((float)getMissCachingCount()/(float)getRequestCachingCount());
+  }
+
+  public double evictedPerEviction() {
+    return ((float)getEvictedCount()/(float)getEvictionCount());
+  }
+
+  public void rollMetricsPeriod() {
+    hitCounts[windowIndex] = getHitCount() - lastHitCount;
+    lastHitCount = getHitCount();
+    hitCachingCounts[windowIndex] =
+      getHitCachingCount() - lastHitCachingCount;
+    lastHitCachingCount = getHitCachingCount();
+    requestCounts[windowIndex] = getRequestCount() - lastRequestCount;
+    lastRequestCount = getRequestCount();
+    requestCachingCounts[windowIndex] =
+      getRequestCachingCount() - lastRequestCachingCount;
+    lastRequestCachingCount = getRequestCachingCount();
+    windowIndex = (windowIndex + 1) % numPeriodsInWindow;
+  }
+
+  public double getHitRatioPastNPeriods() {
+    double ratio = ((double)sum(hitCounts)/(double)sum(requestCounts));
+    return Double.isNaN(ratio) ? 0 : ratio;
+  }
+
+  public double getHitCachingRatioPastNPeriods() {
+    double ratio =
+      ((double)sum(hitCachingCounts)/(double)sum(requestCachingCounts));
+    return Double.isNaN(ratio) ? 0 : ratio;
+  }
+
+  private static long sum(long [] counts) {
+    long sum = 0;
+    for (long count : counts) sum += count;
+    return sum;
+  }
+
+  private static long [] initializeZeros(int n) {
+    long [] zeros = new long [n];
+    for (int i=0; i getDeserializer();
+
+  /**
+   * @return the block type of this cached HFile block
+   */
+  public BlockType getBlockType();
+
+  /**
+   * @return the metrics object identified by table and column family
+   */
+  public SchemaMetrics getSchemaMetrics();
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheableDeserializer.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheableDeserializer.java
new file mode 100644
index 0000000..6210c6a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheableDeserializer.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Interface for a deserializer. Throws an IOException if the serialized data is
+ * incomplete or wrong.
+ * */
+public interface CacheableDeserializer {
+  /**
+   * Returns the deserialized object.
+   *
+   * @return T the deserialized object.
+   */
+  public T deserialize(ByteBuffer b) throws IOException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
new file mode 100644
index 0000000..a66ab85
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
@@ -0,0 +1,114 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+
+/**
+ * Represents an entry in the {@link LruBlockCache}.
+ *
+ * 
      Makes the block memory-aware with {@link HeapSize} and Comparable
+ * to sort by access time for the LRU.  It also takes care of priority by
+ * either instantiating as in-memory or handling the transition from single
+ * to multiple access.
+ */
+public class CachedBlock implements HeapSize, Comparable {
+
+  public final static long PER_BLOCK_OVERHEAD = ClassSize.align(
+    ClassSize.OBJECT + (3 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG) +
+    ClassSize.STRING + ClassSize.BYTE_BUFFER);
+
+  static enum BlockPriority {
+    /**
+     * Accessed a single time (used for scan-resistance)
+     */
+    SINGLE,
+    /**
+     * Accessed multiple times
+     */
+    MULTI,
+    /**
+     * Block from in-memory store
+     */
+    MEMORY
+  };
+
+  private final BlockCacheKey cacheKey;
+  private final Cacheable buf;
+  private volatile long accessTime;
+  private long size;
+  private BlockPriority priority;
+
+  public CachedBlock(BlockCacheKey cacheKey, Cacheable buf, long accessTime) {
+    this(cacheKey, buf, accessTime, false);
+  }
+
+  public CachedBlock(BlockCacheKey cacheKey, Cacheable buf, long accessTime,
+      boolean inMemory) {
+    this.cacheKey = cacheKey;
+    this.buf = buf;
+    this.accessTime = accessTime;
+    // We approximate the size of this class by the size of its name string
+    // plus the size of its byte buffer plus the overhead associated with all
+    // the base classes. We also include the base class
+    // sizes in the PER_BLOCK_OVERHEAD variable rather than align()ing them with
+    // their buffer lengths. This variable is used elsewhere in unit tests.
+    this.size = ClassSize.align(cacheKey.heapSize())
+        + ClassSize.align(buf.heapSize()) + PER_BLOCK_OVERHEAD;
+    if(inMemory) {
+      this.priority = BlockPriority.MEMORY;
+    } else {
+      this.priority = BlockPriority.SINGLE;
+    }
+  }
+
+  /**
+   * Block has been accessed.  Update its local access time.
+   */
+  public void access(long accessTime) {
+    this.accessTime = accessTime;
+    if(this.priority == BlockPriority.SINGLE) {
+      this.priority = BlockPriority.MULTI;
+    }
+  }
+
+  public long heapSize() {
+    return size;
+  }
+
+  public int compareTo(CachedBlock that) {
+    if(this.accessTime == that.accessTime) return 0;
+    return this.accessTime < that.accessTime ? 1 : -1;
+  }
+
+  public Cacheable getBuffer() {
+    return this.buf;
+  }
+
+  public BlockCacheKey getCacheKey() {
+    return this.cacheKey;
+  }
+
+  public BlockPriority getPriority() {
+    return this.priority;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java
new file mode 100644
index 0000000..1637fbf
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlockQueue.java
@@ -0,0 +1,108 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import com.google.common.collect.MinMaxPriorityQueue;
+
+import org.apache.hadoop.hbase.io.HeapSize;
+
+/**
+ * A memory-bound queue that will grow until an element brings
+ * total size >= maxSize.  From then on, only entries that are sorted larger
+ * than the smallest current entry will be inserted/replaced.
+ *
+ * 
      Use this when you want to find the largest elements (according to their
+ * ordering, not their heap size) that consume as close to the specified
+ * maxSize as possible.  Default behavior is to grow just above rather than
+ * just below specified max.
+ *
+ * 
      Object used in this queue must implement {@link HeapSize} as well as
+ * {@link Comparable}.
+ */
+public class CachedBlockQueue implements HeapSize {
+
+  private MinMaxPriorityQueue queue;
+
+  private long heapSize;
+  private long maxSize;
+
+  /**
+   * @param maxSize the target size of elements in the queue
+   * @param blockSize expected average size of blocks
+   */
+  public CachedBlockQueue(long maxSize, long blockSize) {
+    int initialSize = (int)(maxSize / blockSize);
+    if(initialSize == 0) initialSize++;
+    queue = MinMaxPriorityQueue.expectedSize(initialSize).create();
+    heapSize = 0;
+    this.maxSize = maxSize;
+  }
+
+  /**
+   * Attempt to add the specified cached block to this queue.
+   *
+   * 
      If the queue is smaller than the max size, or if the specified element
+   * is ordered before the smallest element in the queue, the element will be
+   * added to the queue.  Otherwise, there is no side effect of this call.
+   * @param cb block to try to add to the queue
+   */
+  public void add(CachedBlock cb) {
+    if(heapSize < maxSize) {
+      queue.add(cb);
+      heapSize += cb.heapSize();
+    } else {
+      CachedBlock head = queue.peek();
+      if(cb.compareTo(head) > 0) {
+        heapSize += cb.heapSize();
+        heapSize -= head.heapSize();
+        if(heapSize > maxSize) {
+          queue.poll();
+        } else {
+          heapSize += head.heapSize();
+        }
+        queue.add(cb);
+      }
+    }
+  }
+
+  /**
+   * @return The next element in this queue, or {@code null} if the queue is
+   * empty.
+   */
+  public CachedBlock poll() {
+    return queue.poll();
+  }
+
+  /**
+   * @return The last element in this queue, or {@code null} if the queue is
+   * empty.
+   */
+  public CachedBlock pollLast() {
+    return queue.pollLast();
+  }
+
+  /**
+   * Total size of all elements in this queue.
+   * @return size of all elements currently in queue, in bytes
+   */
+  public long heapSize() {
+    return heapSize;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
new file mode 100644
index 0000000..4ef1be7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.zip.Checksum;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumFactory;
+import org.apache.hadoop.hbase.util.ChecksumType;
+
+/**
+ * Utility methods to compute and validate checksums.
+ */
+public class ChecksumUtil {
+
+  /** This is used to reserve space in a byte buffer */
+  private static byte[] DUMMY_VALUE = new byte[128 * HFileBlock.CHECKSUM_SIZE];
+
+  /** 
+   * This is used by unit tests to make checksum failures throw an 
+   * exception instead of returning null. Returning a null value from 
+   * checksum validation will cause the higher layer to retry that 
+   * read with hdfs-level checksums. Instead, we would like checksum 
+   * failures to cause the entire unit test to fail.
+   */
+  private static boolean generateExceptions = false;
+
+  /**
+   * Generates a checksum for all the data in indata. The checksum is
+   * written to outdata.
+   * @param indata input data stream
+   * @param startOffset starting offset in the indata stream from where to
+   *                    compute checkums from
+   * @param endOffset ending offset in the indata stream upto
+   *                   which checksums needs to be computed
+   * @param outData the output buffer where checksum values are written
+   * @param outOffset the starting offset in the outdata where the
+   *                  checksum values are written
+   * @param checksumType type of checksum
+   * @param bytesPerChecksum number of bytes per checksum value
+   */
+  static void generateChecksums(byte[] indata,
+    int startOffset, int endOffset, 
+    byte[] outdata, int outOffset,
+    ChecksumType checksumType,
+    int bytesPerChecksum) throws IOException {
+
+    if (checksumType == ChecksumType.NULL) {
+      return; // No checkums for this block.
+    }
+
+    Checksum checksum = checksumType.getChecksumObject();
+    int bytesLeft = endOffset - startOffset;
+    int chunkNum = 0;
+
+    while (bytesLeft > 0) {
+      // generate the checksum for one chunk
+      checksum.reset();
+      int count = Math.min(bytesLeft, bytesPerChecksum);
+      checksum.update(indata, startOffset, count);
+
+      // write the checksum value to the output buffer.
+      int cksumValue = (int)checksum.getValue();
+      outOffset = Bytes.putInt(outdata, outOffset, cksumValue);
+      chunkNum++;
+      startOffset += count;
+      bytesLeft -= count;
+    }
+  }
+
+  /**
+   * Validates that the data in the specified HFileBlock matches the
+   * checksum.  Generates the checksum for the data and
+   * then validate that it matches the value stored in the header.
+   * If there is a checksum mismatch, then return false. Otherwise
+   * return true.
+   * The header is extracted from the specified HFileBlock while the
+   * data-to-be-verified is extracted from 'data'.
+   */
+  static boolean validateBlockChecksum(Path path, HFileBlock block, 
+    byte[] data, int hdrSize) throws IOException {
+
+    // If this is an older version of the block that does not have
+    // checksums, then return false indicating that checksum verification
+    // did not succeed. Actually, this methiod should never be called
+    // when the minorVersion is 0, thus this is a defensive check for a
+    // cannot-happen case. Since this is a cannot-happen case, it is
+    // better to return false to indicate a checksum validation failure.
+    if (block.getMinorVersion() < HFileBlock.MINOR_VERSION_WITH_CHECKSUM) {
+      return false;
+    }
+
+    // Get a checksum object based on the type of checksum that is
+    // set in the HFileBlock header. A ChecksumType.NULL indicates that 
+    // the caller is not interested in validating checksums, so we
+    // always return true.
+    ChecksumType cktype = ChecksumType.codeToType(block.getChecksumType());
+    if (cktype == ChecksumType.NULL) {
+      return true; // No checkums validations needed for this block.
+    }
+    Checksum checksumObject = cktype.getChecksumObject();
+    checksumObject.reset();
+
+    // read in the stored value of the checksum size from the header.
+    int bytesPerChecksum = block.getBytesPerChecksum();
+
+    // bytesPerChecksum is always larger than the size of the header
+    if (bytesPerChecksum < hdrSize) {
+      String msg = "Unsupported value of bytesPerChecksum. " +
+                   " Minimum is " + hdrSize + 
+                   " but the configured value is " + bytesPerChecksum;
+      HFile.LOG.warn(msg);
+      return false;   // cannot happen case, unable to verify checksum
+    }
+    // Extract the header and compute checksum for the header.
+    ByteBuffer hdr = block.getBufferWithHeader();
+    checksumObject.update(hdr.array(), hdr.arrayOffset(), hdrSize);
+
+    int off = hdrSize;
+    int consumed = hdrSize;
+    int bytesLeft = block.getOnDiskDataSizeWithHeader() - off;
+    int cksumOffset = block.getOnDiskDataSizeWithHeader();
+    
+    // validate each chunk
+    while (bytesLeft > 0) {
+      int thisChunkSize = bytesPerChecksum - consumed;
+      int count = Math.min(bytesLeft, thisChunkSize);
+      checksumObject.update(data, off, count);
+
+      int storedChecksum = Bytes.toInt(data, cksumOffset);
+      if (storedChecksum != (int)checksumObject.getValue()) {
+        String msg = "File " + path +
+                     " Stored checksum value of " + storedChecksum +
+                     " at offset " + cksumOffset +
+                     " does not match computed checksum " +
+                     checksumObject.getValue() +
+                     ", total data size " + data.length +
+                     " Checksum data range offset " + off + " len " + count +
+                     HFileBlock.toStringHeader(block.getBufferReadOnly());
+        HFile.LOG.warn(msg);
+        if (generateExceptions) {
+          throw new IOException(msg); // this is only for unit tests
+        } else {
+          return false;               // checksum validation failure
+        }
+      }
+      cksumOffset += HFileBlock.CHECKSUM_SIZE;
+      bytesLeft -= count; 
+      off += count;
+      consumed = 0;
+      checksumObject.reset();
+    }
+    return true; // checksum is valid
+  }
+
+  /**
+   * Returns the number of bytes needed to store the checksums for
+   * a specified data size
+   * @param datasize number of bytes of data
+   * @param bytesPerChecksum number of bytes in a checksum chunk
+   * @return The number of bytes needed to store the checksum values
+   */
+  static long numBytes(long datasize, int bytesPerChecksum) {
+    return numChunks(datasize, bytesPerChecksum) * 
+                     HFileBlock.CHECKSUM_SIZE;
+  }
+
+  /**
+   * Returns the number of checksum chunks needed to store the checksums for
+   * a specified data size
+   * @param datasize number of bytes of data
+   * @param bytesPerChecksum number of bytes in a checksum chunk
+   * @return The number of checksum chunks
+   */
+  static long numChunks(long datasize, int bytesPerChecksum) {
+    long numChunks = datasize/bytesPerChecksum;
+    if (datasize % bytesPerChecksum != 0) {
+      numChunks++;
+    }
+    return numChunks;
+  }
+
+  /**
+   * Write dummy checksums to the end of the specified bytes array
+   * to reserve space for writing checksums later
+   * @param baos OutputStream to write dummy checkum values
+   * @param numBytes Number of bytes of data for which dummy checksums
+   *                 need to be generated
+   * @param bytesPerChecksum Number of bytes per checksum value
+   */
+  static void reserveSpaceForChecksums(ByteArrayOutputStream baos,
+    int numBytes, int bytesPerChecksum) throws IOException {
+    long numChunks = numChunks(numBytes, bytesPerChecksum);
+    long bytesLeft = numChunks * HFileBlock.CHECKSUM_SIZE;
+    while (bytesLeft > 0) {
+      long count = Math.min(bytesLeft, DUMMY_VALUE.length);
+      baos.write(DUMMY_VALUE, 0, (int)count);
+      bytesLeft -= count;
+    }
+  }
+
+  /**
+   * Mechanism to throw an exception in case of hbase checksum
+   * failure. This is used by unit tests only.
+   * @param value Setting this to true will cause hbase checksum
+   *              verification failures to generate exceptions.
+   */
+  public static void generateExceptionForChecksumFailureForTest(boolean value) {
+    generateExceptions = value;
+  }
+}
+
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
new file mode 100644
index 0000000..08bd903
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
@@ -0,0 +1,393 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.annotation.Annotation;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.CodecPool;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionInputStream;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.io.compress.Decompressor;
+import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.io.compress.DefaultCodec;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Compression related stuff.
+ * Copied from hadoop-3315 tfile.
+ */
+public final class Compression {
+  static final Log LOG = LogFactory.getLog(Compression.class);
+
+  /**
+   * Prevent the instantiation of class.
+   */
+  private Compression() {
+    super();
+  }
+
+  static class FinishOnFlushCompressionStream extends FilterOutputStream {
+    public FinishOnFlushCompressionStream(CompressionOutputStream cout) {
+      super(cout);
+    }
+
+    @Override
+    public void write(byte b[], int off, int len) throws IOException {
+      out.write(b, off, len);
+    }
+
+    @Override
+    public void flush() throws IOException {
+      CompressionOutputStream cout = (CompressionOutputStream) out;
+      cout.finish();
+      cout.flush();
+      cout.resetState();
+    }
+  }
+
+  /**
+   * Returns the classloader to load the Codec class from.
+   * @return
+   */
+  private static ClassLoader getClassLoaderForCodec() {
+    ClassLoader cl = Thread.currentThread().getContextClassLoader();
+    if (cl == null) {
+      cl = Compression.class.getClassLoader();
+    }
+    if (cl == null) {
+      cl = ClassLoader.getSystemClassLoader();
+    }
+    if (cl == null) {
+      throw new RuntimeException("A ClassLoader to load the Codec could not be determined");
+    }
+    return cl;
+  }
+
+  /**
+   * Compression algorithms. The ordinal of these cannot change or else you
+   * risk breaking all existing HFiles out there.  Even the ones that are
+   * not compressed! (They use the NONE algorithm)
+   */
+  public static enum Algorithm {
+    LZO("lzo") {
+      // Use base type to avoid compile-time dependencies.
+      private volatile transient CompressionCodec lzoCodec;
+      private transient Object lock = new Object();
+
+      @Override
+      CompressionCodec getCodec(Configuration conf) {
+        if (lzoCodec == null) {
+          synchronized (lock) {
+            if (lzoCodec == null) {
+              lzoCodec = buildCodec(conf);
+            }
+          }
+        }
+        return lzoCodec;
+      }
+
+      private CompressionCodec buildCodec(Configuration conf) {
+        try {
+          Class externalCodec =
+              ClassLoader.getSystemClassLoader()
+                  .loadClass("com.hadoop.compression.lzo.LzoCodec");
+          return (CompressionCodec) ReflectionUtils.newInstance(externalCodec,
+              new Configuration(conf));
+        } catch (ClassNotFoundException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    },
+    GZ("gz") {
+      private volatile transient GzipCodec codec;
+      private transient Object lock = new Object();
+
+      @Override
+      DefaultCodec getCodec(Configuration conf) {
+        if (codec == null) {
+          synchronized (lock) {
+            if (codec == null) {
+              codec = buildCodec(conf);
+            }
+          }
+        }
+
+        return codec;
+      }
+
+      private GzipCodec buildCodec(Configuration conf) {
+        GzipCodec gzcodec = new ReusableStreamGzipCodec();
+        gzcodec.setConf(new Configuration(conf));
+        return gzcodec;
+      }
+    },
+
+    NONE("none") {
+      @Override
+      DefaultCodec getCodec(Configuration conf) {
+        return null;
+      }
+
+      @Override
+      public synchronized InputStream createDecompressionStream(
+          InputStream downStream, Decompressor decompressor,
+          int downStreamBufferSize) throws IOException {
+        if (downStreamBufferSize > 0) {
+          return new BufferedInputStream(downStream, downStreamBufferSize);
+        }
+        // else {
+          // Make sure we bypass FSInputChecker buffer.
+        // return new BufferedInputStream(downStream, 1024);
+        // }
+        // }
+        return downStream;
+      }
+
+      @Override
+      public synchronized OutputStream createCompressionStream(
+          OutputStream downStream, Compressor compressor,
+          int downStreamBufferSize) throws IOException {
+        if (downStreamBufferSize > 0) {
+          return new BufferedOutputStream(downStream, downStreamBufferSize);
+        }
+
+        return downStream;
+      }
+    },
+    SNAPPY("snappy") {
+      // Use base type to avoid compile-time dependencies.
+      private volatile transient CompressionCodec snappyCodec;
+      private transient Object lock = new Object();
+
+      @Override
+      CompressionCodec getCodec(Configuration conf) {
+        if (snappyCodec == null) {
+          synchronized (lock) {
+            if (snappyCodec == null) {
+              snappyCodec = buildCodec(conf);
+            }
+          }
+        }
+        return snappyCodec;
+      }
+
+      private CompressionCodec buildCodec(Configuration conf) {
+        try {
+          Class externalCodec =
+              ClassLoader.getSystemClassLoader()
+                  .loadClass("org.apache.hadoop.io.compress.SnappyCodec");
+          return (CompressionCodec) ReflectionUtils.newInstance(externalCodec,
+              conf);
+        } catch (ClassNotFoundException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    },
+    LZ4("lz4") {
+      // Use base type to avoid compile-time dependencies.
+      private volatile transient CompressionCodec lz4Codec;
+      private transient Object lock = new Object();
+
+      @Override
+      CompressionCodec getCodec(Configuration conf) {
+        if (lz4Codec == null) {
+          synchronized (lock) {
+            if (lz4Codec == null) {
+              lz4Codec = buildCodec(conf);
+            }
+          }
+          buildCodec(conf);
+        }
+        return lz4Codec;
+      }
+
+      private CompressionCodec buildCodec(Configuration conf) {
+        try {
+          Class externalCodec =
+              getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.Lz4Codec");
+          return (CompressionCodec) ReflectionUtils.newInstance(externalCodec,
+              conf);
+        } catch (ClassNotFoundException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    };
+
+    private final Configuration conf;
+    private final String compressName;
+  // data input buffer size to absorb small reads from application.
+    private static final int DATA_IBUF_SIZE = 1 * 1024;
+  // data output buffer size to absorb small writes from application.
+    private static final int DATA_OBUF_SIZE = 4 * 1024;
+
+    Algorithm(String name) {
+      this.conf = new Configuration();
+      this.conf.setBoolean("hadoop.native.lib", true);
+      this.compressName = name;
+    }
+
+    abstract CompressionCodec getCodec(Configuration conf);
+
+    public InputStream createDecompressionStream(
+        InputStream downStream, Decompressor decompressor,
+        int downStreamBufferSize) throws IOException {
+      CompressionCodec codec = getCodec(conf);
+      // Set the internal buffer size to read from down stream.
+      if (downStreamBufferSize > 0) {
+        ((Configurable)codec).getConf().setInt("io.file.buffer.size",
+            downStreamBufferSize);
+      }
+      CompressionInputStream cis =
+          codec.createInputStream(downStream, decompressor);
+      BufferedInputStream bis2 = new BufferedInputStream(cis, DATA_IBUF_SIZE);
+      return bis2;
+
+    }
+
+    public OutputStream createCompressionStream(
+        OutputStream downStream, Compressor compressor, int downStreamBufferSize)
+        throws IOException {
+      OutputStream bos1 = null;
+      if (downStreamBufferSize > 0) {
+        bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
+      }
+      else {
+        bos1 = downStream;
+      }
+      CompressionOutputStream cos =
+          createPlainCompressionStream(bos1, compressor);
+      BufferedOutputStream bos2 =
+          new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
+              DATA_OBUF_SIZE);
+      return bos2;
+    }
+
+    /**
+     * Creates a compression stream without any additional wrapping into
+     * buffering streams.
+     */
+    CompressionOutputStream createPlainCompressionStream(
+        OutputStream downStream, Compressor compressor) throws IOException {
+      CompressionCodec codec = getCodec(conf);
+      ((Configurable)codec).getConf().setInt("io.file.buffer.size", 32 * 1024);
+      return codec.createOutputStream(downStream, compressor);
+    }
+
+    public Compressor getCompressor() {
+      CompressionCodec codec = getCodec(conf);
+      if (codec != null) {
+        Compressor compressor = CodecPool.getCompressor(codec);
+        if (compressor != null) {
+          if (compressor.finished()) {
+            // Somebody returns the compressor to CodecPool but is still using
+            // it.
+            LOG
+                .warn("Compressor obtained from CodecPool is already finished()");
+            // throw new AssertionError(
+            // "Compressor obtained from CodecPool is already finished()");
+          }
+          compressor.reset();
+        }
+        return compressor;
+      }
+      return null;
+    }
+
+    public void returnCompressor(Compressor compressor) {
+      if (compressor != null) {
+        CodecPool.returnCompressor(compressor);
+      }
+    }
+
+    public Decompressor getDecompressor() {
+      CompressionCodec codec = getCodec(conf);
+      if (codec != null) {
+        Decompressor decompressor = CodecPool.getDecompressor(codec);
+        if (decompressor != null) {
+          if (decompressor.finished()) {
+            // Somebody returns the decompressor to CodecPool but is still using
+            // it.
+            LOG
+                .warn("Deompressor obtained from CodecPool is already finished()");
+            // throw new AssertionError(
+            // "Decompressor obtained from CodecPool is already finished()");
+          }
+          decompressor.reset();
+        }
+        return decompressor;
+      }
+
+      return null;
+    }
+
+    public void returnDecompressor(Decompressor decompressor) {
+      if (decompressor != null) {
+        CodecPool.returnDecompressor(decompressor);
+        Annotation[] annotations = decompressor.getClass().getAnnotations();
+        if (annotations != null) {
+          for (Annotation annotation : annotations) {
+            String annoName = annotation.annotationType().getSimpleName();
+            if (annoName.equals("DoNotPool")) {
+              decompressor.end();              
+            }
+          }
+        }
+      }
+    }
+
+    public String getName() {
+      return compressName;
+    }
+  }
+
+  public static Algorithm getCompressionAlgorithmByName(String compressName) {
+    Algorithm[] algos = Algorithm.class.getEnumConstants();
+
+    for (Algorithm a : algos) {
+      if (a.getName().equals(compressName)) {
+        return a;
+      }
+    }
+
+    throw new IllegalArgumentException(
+        "Unsupported compression algorithm name: " + compressName);
+  }
+
+  static String[] getSupportedAlgorithms() {
+    Algorithm[] algos = Algorithm.class.getEnumConstants();
+
+    String[] ret = new String[algos.length];
+    int i = 0;
+    for (Algorithm a : algos) {
+      ret[i++] = a.getName();
+    }
+
+    return ret;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/CorruptHFileException.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/CorruptHFileException.java
new file mode 100644
index 0000000..03ca06c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/CorruptHFileException.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+
+/**
+ * This exception is thrown when attempts to read an HFile fail due to corruption or truncation
+ * issues.
+ */
+public class CorruptHFileException extends DoNotRetryIOException {
+  private static final long serialVersionUID = 1L;
+
+  CorruptHFileException(String m, Throwable t) {
+    super(m, t);
+  }
+
+  CorruptHFileException(String m) {
+    super(m);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
new file mode 100644
index 0000000..0a6c062
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
@@ -0,0 +1,174 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.slab.SlabCache;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * DoubleBlockCache is an abstraction layer that combines two caches, the
+ * smaller onHeapCache and the larger offHeapCache. CacheBlock attempts to cache
+ * the block in both caches, while readblock reads first from the faster on heap
+ * cache before looking for the block in the off heap cache. Metrics are the
+ * combined size and hits and misses of both caches.
+ *
+ **/
+public class DoubleBlockCache implements BlockCache, HeapSize {
+
+  static final Log LOG = LogFactory.getLog(DoubleBlockCache.class.getName());
+
+  private final LruBlockCache onHeapCache;
+  private final SlabCache offHeapCache;
+  private final CacheStats stats;
+
+  /**
+   * Default constructor. Specify maximum size and expected average block size
+   * (approximation is fine).
+   * 
      
+   * All other factors will be calculated based on defaults specified in this
+   * class.
+   *
+   * @param onHeapSize maximum size of the onHeapCache, in bytes.
+   * @param offHeapSize maximum size of the offHeapCache, in bytes.
+   * @param onHeapBlockSize average block size of the on heap cache.
+   * @param offHeapBlockSize average block size for the off heap cache
+   * @param conf configuration file. currently used only by the off heap cache.
+   */
+  public DoubleBlockCache(long onHeapSize, long offHeapSize,
+      long onHeapBlockSize, long offHeapBlockSize, Configuration conf) {
+
+    LOG.info("Creating on-heap cache of size "
+        + StringUtils.humanReadableInt(onHeapSize)
+        + "bytes with an average block size of "
+        + StringUtils.humanReadableInt(onHeapBlockSize) + " bytes.");
+    onHeapCache = new LruBlockCache(onHeapSize, onHeapBlockSize, conf);
+
+    LOG.info("Creating off-heap cache of size "
+        + StringUtils.humanReadableInt(offHeapSize)
+        + "bytes with an average block size of "
+        + StringUtils.humanReadableInt(offHeapBlockSize) + " bytes.");
+    offHeapCache = new SlabCache(offHeapSize, offHeapBlockSize);
+
+    offHeapCache.addSlabByConf(conf);
+    this.stats = new CacheStats();
+  }
+
+  @Override
+  public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
+    onHeapCache.cacheBlock(cacheKey, buf, inMemory);
+    offHeapCache.cacheBlock(cacheKey, buf);
+  }
+
+  @Override
+  public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
+    onHeapCache.cacheBlock(cacheKey, buf);
+    offHeapCache.cacheBlock(cacheKey, buf);
+  }
+
+  @Override
+  public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) {
+    Cacheable cachedBlock;
+
+    if ((cachedBlock = onHeapCache.getBlock(cacheKey, caching, repeat)) != null) {
+      stats.hit(caching);
+      return cachedBlock;
+
+    } else if ((cachedBlock = offHeapCache.getBlock(cacheKey, caching, repeat)) != null) {
+      if (caching) {
+        onHeapCache.cacheBlock(cacheKey, cachedBlock);
+      }
+      stats.hit(caching);
+      return cachedBlock;
+    }
+
+    if (!repeat) stats.miss(caching);
+    return null;
+  }
+
+  @Override
+  public boolean evictBlock(BlockCacheKey cacheKey) {
+    stats.evict();
+    boolean cacheA = onHeapCache.evictBlock(cacheKey);
+    boolean cacheB = offHeapCache.evictBlock(cacheKey);
+    boolean evicted = cacheA || cacheB;
+    if (evicted) {
+      stats.evicted();
+    }
+    return evicted;
+  }
+
+  @Override
+  public CacheStats getStats() {
+    return this.stats;
+  }
+
+  @Override
+  public void shutdown() {
+    onHeapCache.shutdown();
+    offHeapCache.shutdown();
+  }
+
+  @Override
+  public long heapSize() {
+    return onHeapCache.heapSize() + offHeapCache.heapSize();
+  }
+
+  public long size() {
+    return onHeapCache.size() + offHeapCache.size();
+  }
+
+  public long getFreeSize() {
+    return onHeapCache.getFreeSize() + offHeapCache.getFreeSize();
+  }
+
+  public long getCurrentSize() {
+    return onHeapCache.getCurrentSize() + offHeapCache.getCurrentSize();
+  }
+
+  public long getEvictedCount() {
+    return onHeapCache.getEvictedCount() + offHeapCache.getEvictedCount();
+  }
+
+  @Override
+  public int evictBlocksByHfileName(String hfileName) {
+    onHeapCache.evictBlocksByHfileName(hfileName);
+    offHeapCache.evictBlocksByHfileName(hfileName);
+    return 0;
+  }
+
+  @Override
+  public List getBlockCacheColumnFamilySummaries(
+      Configuration conf) throws IOException {
+    return onHeapCache.getBlockCacheColumnFamilySummaries(conf);
+  }
+
+  @Override
+  public long getBlockCount() {
+    return onHeapCache.getBlockCount() + offHeapCache.getBlockCount();
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
new file mode 100644
index 0000000..693643c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
@@ -0,0 +1,533 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static org.apache.hadoop.hbase.io.hfile.HFile.MAX_FORMAT_VERSION;
+import static org.apache.hadoop.hbase.io.hfile.HFile.MIN_FORMAT_VERSION;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.RawComparator;
+
+import com.google.common.io.NullOutputStream;
+
+/**
+ * The {@link HFile} has a fixed trailer which contains offsets to other
+ * variable parts of the file. Also includes basic metadata on this file. The
+ * trailer size is fixed within a given {@link HFile} format version only, but
+ * we always store the version number as the last four-byte integer of the file.
+ * The version number itself is split into two portions, a major 
+ * version and a minor version. 
+ * The last three bytes of a file is the major
+ * version and a single preceding byte is the minor number. The major version
+ * determines which readers/writers to use to read/write a hfile while a minor
+ * version determines smaller changes in hfile format that do not need a new
+ * reader/writer type.
+ */
+public class FixedFileTrailer {
+
+  private static final Log LOG = LogFactory.getLog(FixedFileTrailer.class);
+
+  /**
+   * We store the comparator class name as a fixed-length field in the trailer.
+   */
+  private static final int MAX_COMPARATOR_NAME_LENGTH = 128;
+
+  /**
+   * Offset to the fileinfo data, a small block of vitals. Necessary in v1 but
+   * only potentially useful for pretty-printing in v2.
+   */
+  private long fileInfoOffset;
+
+  /**
+   * In version 1, the offset to the data block index. Starting from version 2,
+   * the meaning of this field is the offset to the section of the file that
+   * should be loaded at the time the file is being opened, and as of the time
+   * of writing, this happens to be the offset of the file info section.
+   */
+  private long loadOnOpenDataOffset;
+
+  /** The number of entries in the root data index. */
+  private int dataIndexCount;
+
+  /** Total uncompressed size of all blocks of the data index */
+  private long uncompressedDataIndexSize;
+
+  /** The number of entries in the meta index */
+  private int metaIndexCount;
+
+  /** The total uncompressed size of keys/values stored in the file. */
+  private long totalUncompressedBytes;
+
+  /**
+   * The number of key/value pairs in the file. This field was int in version 1,
+   * but is now long.
+   */
+  private long entryCount;
+
+  /** The compression codec used for all blocks. */
+  private Compression.Algorithm compressionCodec = Compression.Algorithm.NONE;
+
+  /**
+   * The number of levels in the potentially multi-level data index. Used from
+   * version 2 onwards.
+   */
+  private int numDataIndexLevels;
+
+  /** The offset of the first data block. */
+  private long firstDataBlockOffset;
+
+  /**
+   * It is guaranteed that no key/value data blocks start after this offset in
+   * the file.
+   */
+  private long lastDataBlockOffset;
+
+  /** Raw key comparator class name in version 2 */
+  private String comparatorClassName = RawComparator.class.getName();
+
+  /** The {@link HFile} format major version. */
+  private final int majorVersion;
+
+  /** The {@link HFile} format minor version. */
+  private final int minorVersion;
+
+  FixedFileTrailer(int majorVersion, int minorVersion) {
+    this.majorVersion = majorVersion;
+    this.minorVersion = minorVersion;
+    HFile.checkFormatVersion(majorVersion);
+  }
+
+  private static int[] computeTrailerSizeByVersion() {
+    int versionToSize[] = new int[HFile.MAX_FORMAT_VERSION + 1];
+    for (int version = MIN_FORMAT_VERSION;
+         version <= MAX_FORMAT_VERSION;
+         ++version) {
+      FixedFileTrailer fft = new FixedFileTrailer(version, 
+                                   HFileBlock.MINOR_VERSION_NO_CHECKSUM);
+      DataOutputStream dos = new DataOutputStream(new NullOutputStream());
+      try {
+        fft.serialize(dos);
+      } catch (IOException ex) {
+        // The above has no reason to fail.
+        throw new RuntimeException(ex);
+      }
+      versionToSize[version] = dos.size();
+    }
+    return versionToSize;
+  }
+
+  private static int getMaxTrailerSize() {
+    int maxSize = 0;
+    for (int version = MIN_FORMAT_VERSION;
+         version <= MAX_FORMAT_VERSION;
+         ++version)
+      maxSize = Math.max(getTrailerSize(version), maxSize);
+    return maxSize;
+  }
+
+  private static final int TRAILER_SIZE[] = computeTrailerSizeByVersion();
+  private static final int MAX_TRAILER_SIZE = getMaxTrailerSize();
+
+  static int getTrailerSize(int version) {
+    return TRAILER_SIZE[version];
+  }
+
+  public int getTrailerSize() {
+    return getTrailerSize(majorVersion);
+  }
+
+  /**
+   * Write the trailer to a data stream. We support writing version 1 for
+   * testing and for determining version 1 trailer size. It is also easy to see
+   * what fields changed in version 2.
+   *
+   * @param outputStream
+   * @throws IOException
+   */
+  void serialize(DataOutputStream outputStream) throws IOException {
+    HFile.checkFormatVersion(majorVersion);
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    DataOutput baosDos = new DataOutputStream(baos);
+
+    BlockType.TRAILER.write(baosDos);
+    baosDos.writeLong(fileInfoOffset);
+    baosDos.writeLong(loadOnOpenDataOffset);
+    baosDos.writeInt(dataIndexCount);
+
+    if (majorVersion == 1) {
+      // This used to be metaIndexOffset, but it was not used in version 1.
+      baosDos.writeLong(0);
+    } else {
+      baosDos.writeLong(uncompressedDataIndexSize);
+    }
+
+    baosDos.writeInt(metaIndexCount);
+    baosDos.writeLong(totalUncompressedBytes);
+    if (majorVersion == 1) {
+      baosDos.writeInt((int) Math.min(Integer.MAX_VALUE, entryCount));
+    } else {
+      // This field is long from version 2 onwards.
+      baosDos.writeLong(entryCount);
+    }
+    baosDos.writeInt(compressionCodec.ordinal());
+
+    if (majorVersion > 1) {
+      baosDos.writeInt(numDataIndexLevels);
+      baosDos.writeLong(firstDataBlockOffset);
+      baosDos.writeLong(lastDataBlockOffset);
+      Bytes.writeStringFixedSize(baosDos, comparatorClassName,
+          MAX_COMPARATOR_NAME_LENGTH);
+    }
+
+    // serialize the major and minor versions
+    baosDos.writeInt(materializeVersion(majorVersion, minorVersion));
+
+    outputStream.write(baos.toByteArray());
+  }
+
+  /**
+   * Deserialize the fixed file trailer from the given stream. The version needs
+   * to already be specified. Make sure this is consistent with
+   * {@link #serialize(DataOutputStream)}.
+   *
+   * @param inputStream
+   * @param version
+   * @throws IOException
+   */
+  void deserialize(DataInputStream inputStream) throws IOException {
+    HFile.checkFormatVersion(majorVersion);
+
+    BlockType.TRAILER.readAndCheck(inputStream);
+
+    fileInfoOffset = inputStream.readLong();
+    loadOnOpenDataOffset = inputStream.readLong();
+    dataIndexCount = inputStream.readInt();
+
+    if (majorVersion == 1) {
+      inputStream.readLong(); // Read and skip metaIndexOffset.
+    } else {
+      uncompressedDataIndexSize = inputStream.readLong();
+    }
+    metaIndexCount = inputStream.readInt();
+
+    totalUncompressedBytes = inputStream.readLong();
+    entryCount = majorVersion == 1 ? inputStream.readInt() : inputStream.readLong();
+    compressionCodec = Compression.Algorithm.values()[inputStream.readInt()];
+    if (majorVersion > 1) {
+      numDataIndexLevels = inputStream.readInt();
+      firstDataBlockOffset = inputStream.readLong();
+      lastDataBlockOffset = inputStream.readLong();
+      comparatorClassName =
+          Bytes.readStringFixedSize(inputStream, MAX_COMPARATOR_NAME_LENGTH);
+    }
+
+    int version = inputStream.readInt();
+    expectMajorVersion(extractMajorVersion(version));
+    expectMinorVersion(extractMinorVersion(version));
+  }
+
+  private void append(StringBuilder sb, String s) {
+    if (sb.length() > 0)
+      sb.append(", ");
+    sb.append(s);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    append(sb, "fileinfoOffset=" + fileInfoOffset);
+    append(sb, "loadOnOpenDataOffset=" + loadOnOpenDataOffset);
+    append(sb, "dataIndexCount=" + dataIndexCount);
+    append(sb, "metaIndexCount=" + metaIndexCount);
+    append(sb, "totalUncomressedBytes=" + totalUncompressedBytes);
+    append(sb, "entryCount=" + entryCount);
+    append(sb, "compressionCodec=" + compressionCodec);
+    if (majorVersion == 2) {
+      append(sb, "uncompressedDataIndexSize=" + uncompressedDataIndexSize);
+      append(sb, "numDataIndexLevels=" + numDataIndexLevels);
+      append(sb, "firstDataBlockOffset=" + firstDataBlockOffset);
+      append(sb, "lastDataBlockOffset=" + lastDataBlockOffset);
+      append(sb, "comparatorClassName=" + comparatorClassName);
+    }
+    append(sb, "majorVersion=" + majorVersion);
+    append(sb, "minorVersion=" + minorVersion);
+
+    return sb.toString();
+  }
+
+  /**
+   * Reads a file trailer from the given file.
+   *
+   * @param istream the input stream with the ability to seek. Does not have to
+   *          be buffered, as only one read operation is made.
+   * @param fileSize the file size. Can be obtained using
+   *          {@link org.apache.hadoop.fs.FileSystem#getFileStatus(
+   *          org.apache.hadoop.fs.Path)}.
+   * @return the fixed file trailer read
+   * @throws IOException if failed to read from the underlying stream, or the
+   *           trailer is corrupted, or the version of the trailer is
+   *           unsupported
+   */
+  public static FixedFileTrailer readFromStream(FSDataInputStream istream,
+      long fileSize) throws IOException {
+    int bufferSize = MAX_TRAILER_SIZE;
+    long seekPoint = fileSize - bufferSize;
+    if (seekPoint < 0) {
+      // It is hard to imagine such a small HFile.
+      seekPoint = 0;
+      bufferSize = (int) fileSize;
+    }
+
+    istream.seek(seekPoint);
+    ByteBuffer buf = ByteBuffer.allocate(bufferSize);
+    istream.readFully(buf.array(), buf.arrayOffset(),
+        buf.arrayOffset() + buf.limit());
+
+    // Read the version from the last int of the file.
+    buf.position(buf.limit() - Bytes.SIZEOF_INT);
+    int version = buf.getInt();
+
+    // Extract the major and minor versions.
+    int majorVersion = extractMajorVersion(version);
+    int minorVersion = extractMinorVersion(version);
+
+    HFile.checkFormatVersion(majorVersion); // throws IAE if invalid
+
+    int trailerSize = getTrailerSize(majorVersion);
+
+    FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion);
+    fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(),
+        buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
+    return fft;
+  }
+
+  public void expectMajorVersion(int expected) {
+    if (majorVersion != expected) {
+      throw new IllegalArgumentException("Invalid HFile major version: "
+          + majorVersion 
+          + " (expected: " + expected + ")");
+    }
+  }
+
+  public void expectMinorVersion(int expected) {
+    if (minorVersion != expected) {
+      throw new IllegalArgumentException("Invalid HFile minor version: "
+          + minorVersion + " (expected: " + expected + ")");
+    }
+  }
+
+  public void expectAtLeastMajorVersion(int lowerBound) {
+    if (majorVersion < lowerBound) {
+      throw new IllegalArgumentException("Invalid HFile major version: "
+          + majorVersion
+          + " (expected: " + lowerBound + " or higher).");
+    }
+  }
+
+  public long getFileInfoOffset() {
+    return fileInfoOffset;
+  }
+
+  public void setFileInfoOffset(long fileInfoOffset) {
+    this.fileInfoOffset = fileInfoOffset;
+  }
+
+  public long getLoadOnOpenDataOffset() {
+    return loadOnOpenDataOffset;
+  }
+
+  public void setLoadOnOpenOffset(long loadOnOpenDataOffset) {
+    this.loadOnOpenDataOffset = loadOnOpenDataOffset;
+  }
+
+  public int getDataIndexCount() {
+    return dataIndexCount;
+  }
+
+  public void setDataIndexCount(int dataIndexCount) {
+    this.dataIndexCount = dataIndexCount;
+  }
+
+  public int getMetaIndexCount() {
+    return metaIndexCount;
+  }
+
+  public void setMetaIndexCount(int metaIndexCount) {
+    this.metaIndexCount = metaIndexCount;
+  }
+
+  public long getTotalUncompressedBytes() {
+    return totalUncompressedBytes;
+  }
+
+  public void setTotalUncompressedBytes(long totalUncompressedBytes) {
+    this.totalUncompressedBytes = totalUncompressedBytes;
+  }
+
+  public long getEntryCount() {
+    return entryCount;
+  }
+
+  public void setEntryCount(long newEntryCount) {
+    if (majorVersion == 1) {
+      int intEntryCount = (int) Math.min(Integer.MAX_VALUE, newEntryCount);
+      if (intEntryCount != newEntryCount) {
+        LOG.info("Warning: entry count is " + newEntryCount + " but writing "
+            + intEntryCount + " into the version " + majorVersion + " trailer");
+      }
+      entryCount = intEntryCount;
+      return;
+    }
+    entryCount = newEntryCount;
+  }
+
+  public Compression.Algorithm getCompressionCodec() {
+    return compressionCodec;
+  }
+
+  public void setCompressionCodec(Compression.Algorithm compressionCodec) {
+    this.compressionCodec = compressionCodec;
+  }
+
+  public int getNumDataIndexLevels() {
+    expectAtLeastMajorVersion(2);
+    return numDataIndexLevels;
+  }
+
+  public void setNumDataIndexLevels(int numDataIndexLevels) {
+    expectAtLeastMajorVersion(2);
+    this.numDataIndexLevels = numDataIndexLevels;
+  }
+
+  public long getLastDataBlockOffset() {
+    expectAtLeastMajorVersion(2);
+    return lastDataBlockOffset;
+  }
+
+  public void setLastDataBlockOffset(long lastDataBlockOffset) {
+    expectAtLeastMajorVersion(2);
+    this.lastDataBlockOffset = lastDataBlockOffset;
+  }
+
+  public long getFirstDataBlockOffset() {
+    expectAtLeastMajorVersion(2);
+    return firstDataBlockOffset;
+  }
+
+  public void setFirstDataBlockOffset(long firstDataBlockOffset) {
+    expectAtLeastMajorVersion(2);
+    this.firstDataBlockOffset = firstDataBlockOffset;
+  }
+
+  /**
+   * Returns the major version of this HFile format
+   */
+  public int getMajorVersion() {
+    return majorVersion;
+  }
+
+  /**
+   * Returns the minor version of this HFile format
+   */
+  int getMinorVersion() {
+    return minorVersion;
+  }
+
+  @SuppressWarnings("rawtypes")
+  public void setComparatorClass(Class klass) {
+    expectAtLeastMajorVersion(2);
+    comparatorClassName = klass.getName();
+  }
+
+  @SuppressWarnings("unchecked")
+  private static Class> getComparatorClass(
+      String comparatorClassName) throws IOException {
+    try {
+      return (Class>)
+          Class.forName(comparatorClassName);
+    } catch (ClassNotFoundException ex) {
+      throw new IOException(ex);
+    }
+  }
+
+  public static RawComparator createComparator(
+      String comparatorClassName) throws IOException {
+    try {
+      return getComparatorClass(comparatorClassName).newInstance();
+    } catch (InstantiationException e) {
+      throw new IOException(e);
+    } catch (IllegalAccessException e) {
+      throw new IOException(e);
+    }
+  }
+
+  RawComparator createComparator() throws IOException {
+    expectAtLeastMajorVersion(2);
+    return createComparator(comparatorClassName);
+  }
+
+  public long getUncompressedDataIndexSize() {
+    if (majorVersion == 1)
+      return 0;
+    return uncompressedDataIndexSize;
+  }
+
+  public void setUncompressedDataIndexSize(
+      long uncompressedDataIndexSize) {
+    expectAtLeastMajorVersion(2);
+    this.uncompressedDataIndexSize = uncompressedDataIndexSize;
+  }
+
+  /**
+   * Extracts the major version for a 4-byte serialized version data.
+   * The major version is the 3 least significant bytes
+   */
+  private static int extractMajorVersion(int serializedVersion) {
+    return (serializedVersion & 0x00ffffff);
+  }
+
+  /**
+   * Extracts the minor version for a 4-byte serialized version data.
+   * The major version are the 3 the most significant bytes
+   */
+  private static int extractMinorVersion(int serializedVersion) {
+    return (serializedVersion >>> 24);
+  }
+
+  /**
+   * Create a 4 byte serialized version number by combining the
+   * minor and major version numbers.
+   */
+  private static int materializeVersion(int majorVersion, int minorVersion) {
+    return ((majorVersion & 0x00ffffff) | (minorVersion << 24));
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
new file mode 100644
index 0000000..002a69a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -0,0 +1,777 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.KeyComparator;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.HbaseMapWritable;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.SchemaAware;
+import org.apache.hadoop.hbase.util.BloomFilterWriter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.Writable;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+/**
+ * File format for hbase.
+ * A file of sorted key/value pairs. Both keys and values are byte arrays.
+ * 
      
+ * The memory footprint of a HFile includes the following (below is taken from the
+ * TFile documentation
+ * but applies also to HFile):
+ * 
        
+ * 
      • Some constant overhead of reading or writing a compressed block.
+ * 
          
+ * 
        • Each compressed block requires one compression/decompression codec for
+ * I/O.
+ * 
        • Temporary space to buffer the key.
+ * 
        • Temporary space to buffer the value.
+ * 
        
+ * 
      • HFile index, which is proportional to the total number of Data Blocks.
+ * The total amount of memory needed to hold the index can be estimated as
+ * (56+AvgKeySize)*NumBlocks.
+ * 
      
+ * Suggestions on performance optimization.
+ * 
        
+ * 
      • Minimum block size. We recommend a setting of minimum block size between
+ * 8KB to 1MB for general usage. Larger block size is preferred if files are
+ * primarily for sequential access. However, it would lead to inefficient random
+ * access (because there are more data to decompress). Smaller blocks are good
+ * for random access, but require more memory to hold the block index, and may
+ * be slower to create (because we must flush the compressor stream at the
+ * conclusion of each data block, which leads to an FS I/O flush). Further, due
+ * to the internal caching in Compression codec, the smallest possible block
+ * size would be around 20KB-30KB.
+ * 
      • The current implementation does not offer true multi-threading for
+ * reading. The implementation uses FSDataInputStream seek()+read(), which is
+ * shown to be much faster than positioned-read call in single thread mode.
+ * However, it also means that if multiple threads attempt to access the same
+ * HFile (using multiple scanners) simultaneously, the actual I/O is carried out
+ * sequentially even if they access different DFS blocks (Reexamine! pread seems
+ * to be 10% faster than seek+read in my testing -- stack).
+ * 
      • Compression codec. Use "none" if the data is not very compressable (by
+ * compressable, I mean a compression ratio at least 2:1). Generally, use "lzo"
+ * as the starting point for experimenting. "gz" overs slightly better
+ * compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to
+ * decompress, comparing to "lzo".
+ * 
      
+ *
+ * For more on the background behind HFile, see HBASE-61.
+ * 
      
+ * File is made of data blocks followed by meta data blocks (if any), a fileinfo
+ * block, data block index, meta data block index, and a fixed size trailer
+ * which records the offsets at which file changes content type.
+ * 
      <data blocks><meta blocks><fileinfo><data index><meta index><trailer>
      
+ * Each block has a bit of magic at its start.  Block are comprised of
+ * key/values.  In data blocks, they are both byte arrays.  Metadata blocks are
+ * a String key and a byte array value.  An empty file looks like this:
+ * 
      <fileinfo><trailer>
      .  That is, there are not data nor meta
+ * blocks present.
+ * 
      
+ * TODO: Do scanners need to be able to take a start and end row?
+ * TODO: Should BlockIndex know the name of its file?  Should it have a Path
+ * that points at its file say for the case where an index lives apart from
+ * an HFile instance?
+ */
+public class HFile {
+  static final Log LOG = LogFactory.getLog(HFile.class);
+
+  /**
+   * Maximum length of key in HFile.
+   */
+  public final static int MAXIMUM_KEY_LENGTH = Integer.MAX_VALUE;
+
+  /**
+   * Default block size for an HFile.
+   */
+  public final static int DEFAULT_BLOCKSIZE = 64 * 1024;
+
+  /**
+   * Default compression: none.
+   */
+  public final static Compression.Algorithm DEFAULT_COMPRESSION_ALGORITHM =
+    Compression.Algorithm.NONE;
+
+  /** Minimum supported HFile format version */
+  public static final int MIN_FORMAT_VERSION = 1;
+
+  /** Maximum supported HFile format version */
+  public static final int MAX_FORMAT_VERSION = 2;
+
+  /** Default compression name: none. */
+  public final static String DEFAULT_COMPRESSION =
+    DEFAULT_COMPRESSION_ALGORITHM.getName();
+
+  /**
+   * We assume that HFile path ends with
+   * ROOT_DIR/TABLE_NAME/REGION_NAME/CF_NAME/HFILE, so it has at least this
+   * many levels of nesting. This is needed for identifying table and CF name
+   * from an HFile path.
+   */
+  public final static int MIN_NUM_HFILE_PATH_LEVELS = 5;
+
+  /**
+   * The number of bytes per checksum.
+   */
+  public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
+  public static final ChecksumType DEFAULT_CHECKSUM_TYPE = ChecksumType.CRC32;
+
+  // For measuring latency of "sequential" reads and writes
+  private static final AtomicInteger readOps = new AtomicInteger();
+  private static final AtomicLong readTimeNano = new AtomicLong();
+  private static final AtomicInteger writeOps = new AtomicInteger();
+  private static final AtomicLong writeTimeNano = new AtomicLong();
+
+  // For measuring latency of pread
+  private static final AtomicInteger preadOps = new AtomicInteger();
+  private static final AtomicLong preadTimeNano = new AtomicLong();
+
+  // For measuring number of checksum failures
+  static final AtomicLong checksumFailures = new AtomicLong();
+
+  // For getting more detailed stats on FS latencies
+  // If, for some reason, the metrics subsystem stops polling for latencies,
+  // I don't want data to pile up in a memory leak
+  // so, after LATENCY_BUFFER_SIZE items have been enqueued for processing,
+  // fs latency stats will be dropped (and this behavior will be logged)
+  private static final int LATENCY_BUFFER_SIZE = 5000;
+  private static final BlockingQueue fsReadLatenciesNanos =
+      new ArrayBlockingQueue(LATENCY_BUFFER_SIZE);
+  private static final BlockingQueue fsWriteLatenciesNanos =
+      new ArrayBlockingQueue(LATENCY_BUFFER_SIZE);
+  private static final BlockingQueue fsPreadLatenciesNanos =
+      new ArrayBlockingQueue(LATENCY_BUFFER_SIZE);
+
+  public static final void offerReadLatency(long latencyNanos, boolean pread) {
+    if (pread) {
+      fsPreadLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full
+      preadOps.incrementAndGet();
+      preadTimeNano.addAndGet(latencyNanos);
+    } else {
+      fsReadLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full
+      readTimeNano.addAndGet(latencyNanos);
+      readOps.incrementAndGet();
+    }
+  }
+
+  public static final void offerWriteLatency(long latencyNanos) {
+    fsWriteLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full
+
+    writeTimeNano.addAndGet(latencyNanos);
+    writeOps.incrementAndGet();
+  }
+
+  public static final Collection getReadLatenciesNanos() {
+    final List latencies =
+        Lists.newArrayListWithCapacity(fsReadLatenciesNanos.size());
+    fsReadLatenciesNanos.drainTo(latencies);
+    return latencies;
+  }
+
+  public static final Collection getPreadLatenciesNanos() {
+    final List latencies =
+        Lists.newArrayListWithCapacity(fsPreadLatenciesNanos.size());
+    fsPreadLatenciesNanos.drainTo(latencies);
+    return latencies;
+  }
+
+  public static final Collection getWriteLatenciesNanos() {
+    final List latencies =
+        Lists.newArrayListWithCapacity(fsWriteLatenciesNanos.size());
+    fsWriteLatenciesNanos.drainTo(latencies);
+    return latencies;
+  }
+
+  // for test purpose
+  public static volatile AtomicLong dataBlockReadCnt = new AtomicLong(0);
+
+  // number of sequential reads
+  public static final int getReadOps() {
+    return readOps.getAndSet(0);
+  }
+
+  public static final long getReadTimeMs() {
+    return readTimeNano.getAndSet(0) / 1000000;
+  }
+
+  // number of positional reads
+  public static final int getPreadOps() {
+    return preadOps.getAndSet(0);
+  }
+
+  public static final long getPreadTimeMs() {
+    return preadTimeNano.getAndSet(0) / 1000000;
+  }
+
+  public static final int getWriteOps() {
+    return writeOps.getAndSet(0);
+  }
+
+  public static final long getWriteTimeMs() {
+    return writeTimeNano.getAndSet(0) / 1000000;
+  }
+
+  /**
+   * Number of checksum verification failures. It also
+   * clears the counter.
+   */
+  public static final long getChecksumFailuresCount() {
+    return checksumFailures.getAndSet(0);
+  }
+
+  /** API required to write an {@link HFile} */
+  public interface Writer extends Closeable {
+
+    /** Add an element to the file info map. */
+    void appendFileInfo(byte[] key, byte[] value) throws IOException;
+
+    void append(KeyValue kv) throws IOException;
+
+    void append(byte[] key, byte[] value) throws IOException;
+
+    /** @return the path to this {@link HFile} */
+    Path getPath();
+
+    String getColumnFamilyName();
+
+    void appendMetaBlock(String bloomFilterMetaKey, Writable metaWriter);
+
+    /**
+     * Adds an inline block writer such as a multi-level block index writer or
+     * a compound Bloom filter writer.
+     */
+    void addInlineBlockWriter(InlineBlockWriter bloomWriter);
+
+    /**
+     * Store general Bloom filter in the file. This does not deal with Bloom filter
+     * internals but is necessary, since Bloom filters are stored differently
+     * in HFile version 1 and version 2.
+     */
+    void addGeneralBloomFilter(BloomFilterWriter bfw);
+
+    /**
+     * Store delete family Bloom filter in the file, which is only supported in
+     * HFile V2.
+     */
+    void addDeleteFamilyBloomFilter(BloomFilterWriter bfw) throws IOException;
+  }
+
+  /**
+   * This variety of ways to construct writers is used throughout the code, and
+   * we want to be able to swap writer implementations.
+   */
+  public static abstract class WriterFactory {
+    protected final Configuration conf;
+    protected final CacheConfig cacheConf;
+    protected FileSystem fs;
+    protected Path path;
+    protected FSDataOutputStream ostream;
+    protected int blockSize = HColumnDescriptor.DEFAULT_BLOCKSIZE;
+    protected Compression.Algorithm compression =
+        HFile.DEFAULT_COMPRESSION_ALGORITHM;
+    protected HFileDataBlockEncoder encoder = NoOpDataBlockEncoder.INSTANCE;
+    protected KeyComparator comparator;
+    protected ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE;
+    protected int bytesPerChecksum = DEFAULT_BYTES_PER_CHECKSUM;
+    protected boolean includeMVCCReadpoint = true;
+
+    WriterFactory(Configuration conf, CacheConfig cacheConf) {
+      this.conf = conf;
+      this.cacheConf = cacheConf;
+    }
+
+    public WriterFactory withPath(FileSystem fs, Path path) {
+      Preconditions.checkNotNull(fs);
+      Preconditions.checkNotNull(path);
+      this.fs = fs;
+      this.path = path;
+      return this;
+    }
+
+    public WriterFactory withOutputStream(FSDataOutputStream ostream) {
+      Preconditions.checkNotNull(ostream);
+      this.ostream = ostream;
+      return this;
+    }
+
+    public WriterFactory withBlockSize(int blockSize) {
+      this.blockSize = blockSize;
+      return this;
+    }
+
+    public WriterFactory withCompression(Compression.Algorithm compression) {
+      Preconditions.checkNotNull(compression);
+      this.compression = compression;
+      return this;
+    }
+
+    public WriterFactory withCompression(String compressAlgo) {
+      Preconditions.checkNotNull(compression);
+      this.compression = AbstractHFileWriter.compressionByName(compressAlgo);
+      return this;
+    }
+
+    public WriterFactory withDataBlockEncoder(HFileDataBlockEncoder encoder) {
+      Preconditions.checkNotNull(encoder);
+      this.encoder = encoder;
+      return this;
+    }
+
+    public WriterFactory withComparator(KeyComparator comparator) {
+      Preconditions.checkNotNull(comparator);
+      this.comparator = comparator;
+      return this;
+    }
+
+    public WriterFactory withChecksumType(ChecksumType checksumType) {
+      Preconditions.checkNotNull(checksumType);
+      this.checksumType = checksumType;
+      return this;
+    }
+
+    public WriterFactory withBytesPerChecksum(int bytesPerChecksum) {
+      this.bytesPerChecksum = bytesPerChecksum;
+      return this;
+    }
+
+    public WriterFactory includeMVCCReadpoint(boolean includeMVCCReadpoint) {
+      this.includeMVCCReadpoint = includeMVCCReadpoint;
+      return this;
+    }
+
+    public Writer create() throws IOException {
+      if ((path != null ? 1 : 0) + (ostream != null ? 1 : 0) != 1) {
+        throw new AssertionError("Please specify exactly one of " +
+            "filesystem/path or path");
+      }
+      if (path != null) {
+        ostream = AbstractHFileWriter.createOutputStream(conf, fs, path);
+      }
+      return createWriter(fs, path, ostream, blockSize, compression, encoder, comparator,
+          checksumType, bytesPerChecksum, includeMVCCReadpoint);
+    }
+
+    protected abstract Writer createWriter(FileSystem fs, Path path,
+        FSDataOutputStream ostream, int blockSize,
+        Compression.Algorithm compress,
+        HFileDataBlockEncoder dataBlockEncoder,
+        KeyComparator comparator, ChecksumType checksumType,
+        int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException;
+  }
+
+  /** The configuration key for HFile version to use for new files */
+  public static final String FORMAT_VERSION_KEY = "hfile.format.version";
+
+  public static int getFormatVersion(Configuration conf) {
+    int version = conf.getInt(FORMAT_VERSION_KEY, MAX_FORMAT_VERSION);
+    checkFormatVersion(version);
+    return version;
+  }
+
+  /**
+   * Returns the factory to be used to create {@link HFile} writers.
+   * Disables block cache access for all writers created through the
+   * returned factory.
+   */
+  public static final WriterFactory getWriterFactoryNoCache(Configuration
+       conf) {
+    Configuration tempConf = new Configuration(conf);
+    tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
+    return HFile.getWriterFactory(conf, new CacheConfig(tempConf));
+  }
+
+  /**
+   * Returns the factory to be used to create {@link HFile} writers
+   */
+  public static final WriterFactory getWriterFactory(Configuration conf,
+      CacheConfig cacheConf) {
+    SchemaMetrics.configureGlobally(conf);
+    int version = getFormatVersion(conf);
+    switch (version) {
+    case 1:
+      return new HFileWriterV1.WriterFactoryV1(conf, cacheConf);
+    case 2:
+      return new HFileWriterV2.WriterFactoryV2(conf, cacheConf);
+    default:
+      throw new IllegalArgumentException("Cannot create writer for HFile " +
+          "format version " + version);
+    }
+  }
+
+  /** An abstraction used by the block index */
+  public interface CachingBlockReader {
+    HFileBlock readBlock(long offset, long onDiskBlockSize,
+        boolean cacheBlock, final boolean pread, final boolean isCompaction,
+        BlockType expectedBlockType)
+        throws IOException;
+  }
+
+  /** An interface used by clients to open and iterate an {@link HFile}. */
+  public interface Reader extends Closeable, CachingBlockReader,
+      SchemaAware {
+    /**
+     * Returns this reader's "name". Usually the last component of the path.
+     * Needs to be constant as the file is being moved to support caching on
+     * write.
+     */
+    String getName();
+
+    String getColumnFamilyName();
+
+    RawComparator getComparator();
+
+    HFileScanner getScanner(boolean cacheBlocks,
+       final boolean pread, final boolean isCompaction);
+
+    ByteBuffer getMetaBlock(String metaBlockName,
+       boolean cacheBlock) throws IOException;
+
+    Map loadFileInfo() throws IOException;
+
+    byte[] getLastKey();
+
+    byte[] midkey() throws IOException;
+
+    long length();
+
+    long getEntries();
+
+    byte[] getFirstKey();
+
+    long indexSize();
+
+    byte[] getFirstRowKey();
+
+    byte[] getLastRowKey();
+
+    FixedFileTrailer getTrailer();
+
+    HFileBlockIndex.BlockIndexReader getDataBlockIndexReader();
+
+    HFileScanner getScanner(boolean cacheBlocks, boolean pread);
+
+    Compression.Algorithm getCompressionAlgorithm();
+
+    /**
+     * Retrieves general Bloom filter metadata as appropriate for each
+     * {@link HFile} version.
+     * Knows nothing about how that metadata is structured.
+     */
+    DataInput getGeneralBloomFilterMetadata() throws IOException;
+
+    /**
+     * Retrieves delete family Bloom filter metadata as appropriate for each
+     * {@link HFile}  version.
+     * Knows nothing about how that metadata is structured.
+     */
+    DataInput getDeleteBloomFilterMetadata() throws IOException;
+
+    Path getPath();
+
+    /** Close method with optional evictOnClose */
+    void close(boolean evictOnClose) throws IOException;
+
+    DataBlockEncoding getEncodingOnDisk();
+  }
+
+  /**
+   * Method returns the reader given the specified arguments.
+   * TODO This is a bad abstraction.  See HBASE-6635.
+   *
+   * @param path hfile's path
+   * @param fsdis an open checksummed stream of path's file
+   * @param fsdisNoFsChecksum an open unchecksummed stream of path's file
+   * @param size max size of the trailer.
+   * @param closeIStream boolean for closing file after the getting the reader version.
+   * @param cacheConf Cache configuation values, cannot be null.
+   * @param preferredEncodingInCache
+   * @param hfs
+   * @return an appropriate instance of HFileReader
+   * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
+   */
+  private static Reader pickReaderVersion(Path path, FSDataInputStream fsdis,
+      FSDataInputStream fsdisNoFsChecksum,
+      long size, boolean closeIStream, CacheConfig cacheConf,
+      DataBlockEncoding preferredEncodingInCache, HFileSystem hfs)
+      throws IOException {
+    FixedFileTrailer trailer = null;
+    try {
+      trailer = FixedFileTrailer.readFromStream(fsdis, size);
+    } catch (IllegalArgumentException iae) {
+      throw new CorruptHFileException("Problem reading HFile Trailer from file " + path, iae);
+    }
+    switch (trailer.getMajorVersion()) {
+    case 1:
+      return new HFileReaderV1(path, trailer, fsdis, size, closeIStream,
+          cacheConf);
+    case 2:
+      return new HFileReaderV2(path, trailer, fsdis, fsdisNoFsChecksum,
+          size, closeIStream,
+          cacheConf, preferredEncodingInCache, hfs);
+    default:
+      throw new CorruptHFileException("Invalid HFile version " + trailer.getMajorVersion());
+    }
+  }
+
+  /**
+   * @param fs A file system
+   * @param path Path to HFile
+   * @param cacheConf Cache configuration for hfile's contents
+   * @param preferredEncodingInCache Preferred in-cache data encoding algorithm.
+   * @return A version specific Hfile Reader
+   * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
+   */
+  public static Reader createReaderWithEncoding(
+      FileSystem fs, Path path, CacheConfig cacheConf,
+      DataBlockEncoding preferredEncodingInCache) throws IOException {
+    final boolean closeIStream = true;
+    HFileSystem hfs = null;
+    FSDataInputStream fsdis = fs.open(path);
+    FSDataInputStream fsdisNoFsChecksum = fsdis;
+    // If the fs is not an instance of HFileSystem, then create an
+    // instance of HFileSystem that wraps over the specified fs.
+    // In this case, we will not be able to avoid checksumming inside
+    // the filesystem.
+    if (!(fs instanceof HFileSystem)) {
+      hfs = new HFileSystem(fs);
+    } else {
+      hfs = (HFileSystem)fs;
+      // open a stream to read data without checksum verification in
+      // the filesystem
+      if (hfs != null) {
+        fsdisNoFsChecksum = hfs.getNoChecksumFs().open(path);
+      }
+    }
+    return pickReaderVersion(path, fsdis, fsdisNoFsChecksum,
+        fs.getFileStatus(path).getLen(), closeIStream, cacheConf,
+        preferredEncodingInCache, hfs);
+  }
+
+  /**
+   * @param fs A file system
+   * @param path Path to HFile
+   * @param fsdis an open checksummed stream of path's file
+   * @param fsdisNoFsChecksum an open unchecksummed stream of path's file
+   * @param size max size of the trailer.
+   * @param cacheConf Cache configuration for hfile's contents
+   * @param preferredEncodingInCache Preferred in-cache data encoding algorithm.
+   * @param closeIStream boolean for closing file after the getting the reader version.
+   * @return A version specific Hfile Reader
+   * @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
+   */
+  public static Reader createReaderWithEncoding(
+      FileSystem fs, Path path, FSDataInputStream fsdis,
+      FSDataInputStream fsdisNoFsChecksum, long size, CacheConfig cacheConf,
+      DataBlockEncoding preferredEncodingInCache, boolean closeIStream)
+      throws IOException {
+    HFileSystem hfs = null;
+
+    // If the fs is not an instance of HFileSystem, then create an
+    // instance of HFileSystem that wraps over the specified fs.
+    // In this case, we will not be able to avoid checksumming inside
+    // the filesystem.
+    if (!(fs instanceof HFileSystem)) {
+      hfs = new HFileSystem(fs);
+    } else {
+      hfs = (HFileSystem)fs;
+    }
+    return pickReaderVersion(path, fsdis, fsdisNoFsChecksum, size,
+                             closeIStream, cacheConf,
+                             preferredEncodingInCache, hfs);
+  }
+
+  /**
+   * @param fs filesystem
+   * @param path Path to file to read
+   * @param cacheConf This must not be null.  @see {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#CacheConfig(Configuration)}
+   * @return an active Reader instance
+   * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile is corrupt/invalid.
+   */
+  public static Reader createReader(
+      FileSystem fs, Path path, CacheConfig cacheConf) throws IOException {
+    return createReaderWithEncoding(fs, path, cacheConf,
+        DataBlockEncoding.NONE);
+  }
+
+  /**
+   * This factory method is used only by unit tests
+   */
+  static Reader createReaderFromStream(Path path,
+      FSDataInputStream fsdis, long size, CacheConfig cacheConf)
+      throws IOException {
+    final boolean closeIStream = false;
+    return pickReaderVersion(path, fsdis, fsdis, size, closeIStream, cacheConf,
+        DataBlockEncoding.NONE, null);
+  }
+
+  /*
+   * Metadata for this file.  Conjured by the writer.  Read in by the reader.
+   */
+  static class FileInfo extends HbaseMapWritable {
+    static final String RESERVED_PREFIX = "hfile.";
+    static final byte[] RESERVED_PREFIX_BYTES = Bytes.toBytes(RESERVED_PREFIX);
+    static final byte [] LASTKEY = Bytes.toBytes(RESERVED_PREFIX + "LASTKEY");
+    static final byte [] AVG_KEY_LEN =
+      Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN");
+    static final byte [] AVG_VALUE_LEN =
+      Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN");
+    static final byte [] COMPARATOR =
+      Bytes.toBytes(RESERVED_PREFIX + "COMPARATOR");
+
+    /**
+     * Append the given key/value pair to the file info, optionally checking the
+     * key prefix.
+     *
+     * @param k key to add
+     * @param v value to add
+     * @param checkPrefix whether to check that the provided key does not start
+     *          with the reserved prefix
+     * @return this file info object
+     * @throws IOException if the key or value is invalid
+     */
+    public FileInfo append(final byte[] k, final byte[] v,
+        final boolean checkPrefix) throws IOException {
+      if (k == null || v == null) {
+        throw new NullPointerException("Key nor value may be null");
+      }
+      if (checkPrefix && isReservedFileInfoKey(k)) {
+        throw new IOException("Keys with a " + FileInfo.RESERVED_PREFIX
+            + " are reserved");
+      }
+      put(k, v);
+      return this;
+    }
+
+  }
+
+  /** Return true if the given file info key is reserved for internal use. */
+  public static boolean isReservedFileInfoKey(byte[] key) {
+    return Bytes.startsWith(key, FileInfo.RESERVED_PREFIX_BYTES);
+  }
+
+  /**
+   * Get names of supported compression algorithms. The names are acceptable by
+   * HFile.Writer.
+   *
+   * @return Array of strings, each represents a supported compression
+   *         algorithm. Currently, the following compression algorithms are
+   *         supported.
+   *         
        
+   *         
      • "none" - No compression.
+   *         
      • "gz" - GZIP compression.
+   *         
      
+   */
+  public static String[] getSupportedCompressionAlgorithms() {
+    return Compression.getSupportedAlgorithms();
+  }
+
+  // Utility methods.
+  /*
+   * @param l Long to convert to an int.
+   * @return l cast as an int.
+   */
+  static int longToInt(final long l) {
+    // Expecting the size() of a block not exceeding 4GB. Assuming the
+    // size() will wrap to negative integer if it exceeds 2GB (From tfile).
+    return (int)(l & 0x00000000ffffffffL);
+  }
+
+  /**
+   * Returns all files belonging to the given region directory. Could return an
+   * empty list.
+   *
+   * @param fs  The file system reference.
+   * @param regionDir  The region directory to scan.
+   * @return The list of files found.
+   * @throws IOException When scanning the files fails.
+   */
+  static List getStoreFiles(FileSystem fs, Path regionDir)
+      throws IOException {
+    List res = new ArrayList();
+    PathFilter dirFilter = new FSUtils.DirFilter(fs);
+    FileStatus[] familyDirs = fs.listStatus(regionDir, dirFilter);
+    for(FileStatus dir : familyDirs) {
+      FileStatus[] files = fs.listStatus(dir.getPath());
+      for (FileStatus file : files) {
+        if (!file.isDir()) {
+          res.add(file.getPath());
+        }
+      }
+    }
+    return res;
+  }
+
+  public static void main(String[] args) throws IOException {
+    HFilePrettyPrinter prettyPrinter = new HFilePrettyPrinter();
+    System.exit(prettyPrinter.run(args));
+  }
+
+  /**
+   * Checks the given {@link HFile} format version, and throws an exception if
+   * invalid. Note that if the version number comes from an input file and has
+   * not been verified, the caller needs to re-throw an {@link IOException} to
+   * indicate that this is not a software error, but corrupted input.
+   *
+   * @param version an HFile version
+   * @throws IllegalArgumentException if the version is invalid
+   */
+  public static void checkFormatVersion(int version)
+      throws IllegalArgumentException {
+    if (version < MIN_FORMAT_VERSION || version > MAX_FORMAT_VERSION) {
+      throw new IllegalArgumentException("Invalid HFile version: " + version
+          + " (expected to be " + "between " + MIN_FORMAT_VERSION + " and "
+          + MAX_FORMAT_VERSION + ")");
+    }
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
new file mode 100644
index 0000000..6bf7437
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -0,0 +1,2192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static org.apache.hadoop.hbase.io.hfile.BlockType.MAGIC_LENGTH;
+import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.NONE;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
+import org.apache.hadoop.hbase.regionserver.MemStore;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.hbase.util.CompoundBloomFilter;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.io.compress.Decompressor;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Reading {@link HFile} version 1 and 2 blocks, and writing version 2 blocks.
+ * 
        
+ * 
      • In version 1 all blocks are always compressed or uncompressed, as
+ * specified by the {@link HFile}'s compression algorithm, with a type-specific
+ * magic record stored in the beginning of the compressed data (i.e. one needs
+ * to uncompress the compressed block to determine the block type). There is
+ * only a single compression algorithm setting for all blocks. Offset and size
+ * information from the block index are required to read a block.
+ * 
      • In version 2 a block is structured as follows:
+ * 
          
+ * 
        • Magic record identifying the block type (8 bytes)
+ * 
        • Compressed block size, header not included (4 bytes)
+ * 
        • Uncompressed block size, header not included (4 bytes)
+ * 
        • The offset of the previous block of the same type (8 bytes). This is
+ * used to be able to navigate to the previous block without going to the block
+ * 
        • For minorVersions >=1, there is an additional 4 byte field 
+ * bytesPerChecksum that records the number of bytes in a checksum chunk.
+ * 
        • For minorVersions >=1, there is a 4 byte value to store the size of
+ * data on disk (excluding the checksums)
+ * 
        • For minorVersions >=1, a series of 4 byte checksums, one each for
+ * the number of bytes specified by bytesPerChecksum.
+ * index.
+ * 
        • Compressed data (or uncompressed data if compression is disabled). The
+ * compression algorithm is the same for all the blocks in the {@link HFile},
+ * similarly to what was done in version 1.
+ * 
        
+ * 
      
+ * The version 2 block representation in the block cache is the same as above,
+ * except that the data section is always uncompressed in the cache.
+ */
+public class HFileBlock extends SchemaConfigured implements Cacheable {
+
+  /** Minor versions starting with this number have hbase checksums */
+  static final int MINOR_VERSION_WITH_CHECKSUM = 1;
+
+  /** minor version that does not support checksums */
+  static final int MINOR_VERSION_NO_CHECKSUM = 0;
+
+  /**
+   * On a checksum failure on a Reader, these many suceeding read
+   * requests switch back to using hdfs checksums before auto-reenabling
+   * hbase checksum verification.
+   */
+  static final int CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD = 3;
+
+  /** The size data structures with minor version is 0 */
+  static final int HEADER_SIZE_NO_CHECKSUM = MAGIC_LENGTH + 2 * Bytes.SIZEOF_INT
+      + Bytes.SIZEOF_LONG;
+
+  public static final boolean FILL_HEADER = true;
+  public static final boolean DONT_FILL_HEADER = false;
+
+  /** The size of a version 2 {@link HFile} block header, minor version 1.
+   * There is a 1 byte checksum type, followed by a 4 byte bytesPerChecksum
+   * followed by another 4 byte value to store sizeofDataOnDisk.
+   */
+  static final int HEADER_SIZE_WITH_CHECKSUMS = HEADER_SIZE_NO_CHECKSUM + Bytes.SIZEOF_BYTE +
+                                 2 * Bytes.SIZEOF_INT;
+
+  /**
+   * The size of block header when blockType is {@link BlockType#ENCODED_DATA}.
+   * This extends normal header by adding the id of encoder.
+   */
+  public static final int ENCODED_HEADER_SIZE = HEADER_SIZE_WITH_CHECKSUMS
+      + DataBlockEncoding.ID_SIZE;
+
+  /** Just an array of bytes of the right size. */
+  static final byte[] DUMMY_HEADER_WITH_CHECKSUM = new byte[HEADER_SIZE_WITH_CHECKSUMS];
+  static final byte[] DUMMY_HEADER_NO_CHECKSUM = 
+     new byte[HEADER_SIZE_NO_CHECKSUM];
+
+  public static final int BYTE_BUFFER_HEAP_SIZE = (int) ClassSize.estimateBase(
+      ByteBuffer.wrap(new byte[0], 0, 0).getClass(), false);
+
+  static final int EXTRA_SERIALIZATION_SPACE = Bytes.SIZEOF_LONG +
+      Bytes.SIZEOF_INT;
+
+  /**
+   * Each checksum value is an integer that can be stored in 4 bytes.
+   */
+  static final int CHECKSUM_SIZE = Bytes.SIZEOF_INT;
+
+  private static final CacheableDeserializer blockDeserializer =
+      new CacheableDeserializer() {
+        public HFileBlock deserialize(ByteBuffer buf) throws IOException{
+          ByteBuffer newByteBuffer = ByteBuffer.allocate(buf.limit()
+              - HFileBlock.EXTRA_SERIALIZATION_SPACE);
+          buf.limit(buf.limit()
+              - HFileBlock.EXTRA_SERIALIZATION_SPACE).rewind();
+          newByteBuffer.put(buf);
+          HFileBlock ourBuffer = new HFileBlock(newByteBuffer, 
+                                   MINOR_VERSION_NO_CHECKSUM);
+
+          buf.position(buf.limit());
+          buf.limit(buf.limit() + HFileBlock.EXTRA_SERIALIZATION_SPACE);
+          ourBuffer.offset = buf.getLong();
+          ourBuffer.nextBlockOnDiskSizeWithHeader = buf.getInt();
+          return ourBuffer;
+        }
+      };
+
+  private BlockType blockType;
+
+  /** Size on disk without the header. It includes checksum data too. */
+  private int onDiskSizeWithoutHeader;
+
+  /** Size of pure data. Does not include header or checksums */
+  private final int uncompressedSizeWithoutHeader;
+
+  /** The offset of the previous block on disk */
+  private final long prevBlockOffset;
+
+  /** The Type of checksum, better to store the byte than an object */
+  private final byte checksumType;
+
+  /** The number of bytes for which a checksum is computed */
+  private final int bytesPerChecksum;
+
+  /** Size on disk of header and data. Does not include checksum data */
+  private final int onDiskDataSizeWithHeader;
+
+  /** The minor version of the hfile. */
+  private final int minorVersion;
+
+  /** The in-memory representation of the hfile block */
+  private ByteBuffer buf;
+
+  /** Whether there is a memstore timestamp after every key/value */
+  private boolean includesMemstoreTS;
+
+  /**
+   * The offset of this block in the file. Populated by the reader for
+   * convenience of access. This offset is not part of the block header.
+   */
+  private long offset = -1;
+
+  /**
+   * The on-disk size of the next block, including the header, obtained by
+   * peeking into the first {@link HFileBlock#headerSize(int)} bytes of the next block's
+   * header, or -1 if unknown.
+   */
+  private int nextBlockOnDiskSizeWithHeader = -1;
+
+  /**
+   * Creates a new {@link HFile} block from the given fields. This constructor
+   * is mostly used when the block data has already been read and uncompressed,
+   * and is sitting in a byte buffer. 
+   *
+   * @param blockType the type of this block, see {@link BlockType}
+   * @param onDiskSizeWithoutHeader compressed size of the block if compression
+   *          is used, otherwise uncompressed size, header size not included
+   * @param uncompressedSizeWithoutHeader uncompressed size of the block,
+   *          header size not included. Equals onDiskSizeWithoutHeader if
+   *          compression is disabled.
+   * @param prevBlockOffset the offset of the previous block in the
+   *          {@link HFile}
+   * @param buf block header {@link HFileBlock#headerSize(int)} bytes) followed by
+   *          uncompressed data. This
+   * @param fillHeader true to fill in the first {@link HFileBlock#headerSize(int)} bytes of
+   *          the buffer based on the header fields provided
+   * @param offset the file offset the block was read from
+   * @param minorVersion the minor version of this block
+   * @param bytesPerChecksum the number of bytes per checksum chunk
+   * @param checksumType the checksum algorithm to use
+   * @param onDiskDataSizeWithHeader size of header and data on disk not
+   *        including checksum data
+   */
+  HFileBlock(BlockType blockType, int onDiskSizeWithoutHeader,
+      int uncompressedSizeWithoutHeader, long prevBlockOffset, ByteBuffer buf,
+      boolean fillHeader, long offset, boolean includesMemstoreTS, 
+      int minorVersion, int bytesPerChecksum, byte checksumType,
+      int onDiskDataSizeWithHeader) {
+    this.blockType = blockType;
+    this.onDiskSizeWithoutHeader = onDiskSizeWithoutHeader;
+    this.uncompressedSizeWithoutHeader = uncompressedSizeWithoutHeader;
+    this.prevBlockOffset = prevBlockOffset;
+    this.buf = buf;
+    if (fillHeader)
+      overwriteHeader();
+    this.offset = offset;
+    this.includesMemstoreTS = includesMemstoreTS;
+    this.minorVersion = minorVersion;
+    this.bytesPerChecksum = bytesPerChecksum;
+    this.checksumType = checksumType;
+    this.onDiskDataSizeWithHeader = onDiskDataSizeWithHeader;
+  }
+
+  /**
+   * Creates a block from an existing buffer starting with a header. Rewinds
+   * and takes ownership of the buffer. By definition of rewind, ignores the
+   * buffer position, but if you slice the buffer beforehand, it will rewind
+   * to that point. The reason this has a minorNumber and not a majorNumber is
+   * because majorNumbers indicate the format of a HFile whereas minorNumbers 
+   * indicate the format inside a HFileBlock.
+   */
+  HFileBlock(ByteBuffer b, int minorVersion) throws IOException {
+    b.rewind();
+    blockType = BlockType.read(b);
+    onDiskSizeWithoutHeader = b.getInt();
+    uncompressedSizeWithoutHeader = b.getInt();
+    prevBlockOffset = b.getLong();
+    this.minorVersion = minorVersion;
+    if (minorVersion >= MINOR_VERSION_WITH_CHECKSUM) {
+      this.checksumType = b.get();
+      this.bytesPerChecksum = b.getInt();
+      this.onDiskDataSizeWithHeader = b.getInt();
+    } else {
+      this.checksumType = ChecksumType.NULL.getCode();
+      this.bytesPerChecksum = 0;
+      this.onDiskDataSizeWithHeader = onDiskSizeWithoutHeader +
+                                       HEADER_SIZE_NO_CHECKSUM;
+    }
+    buf = b;
+    buf.rewind();
+  }
+
+  public BlockType getBlockType() {
+    return blockType;
+  }
+
+  /** @return get data block encoding id that was used to encode this block */
+  public short getDataBlockEncodingId() {
+    if (blockType != BlockType.ENCODED_DATA) {
+      throw new IllegalArgumentException("Querying encoder ID of a block " +
+          "of type other than " + BlockType.ENCODED_DATA + ": " + blockType);
+    }
+    return buf.getShort(headerSize());
+  }
+
+  /**
+   * @return the on-disk size of the block with header size included. This
+   * includes the header, the data and the checksum data.
+   */
+  public int getOnDiskSizeWithHeader() {
+    return onDiskSizeWithoutHeader + headerSize();
+  }
+
+  /**
+   * Returns the size of the compressed part of the block in case compression
+   * is used, or the uncompressed size of the data part otherwise. Header size
+   * and checksum data size is not included.
+   *
+   * @return the on-disk size of the data part of the block, header and
+   *         checksum not included. 
+   */
+  int getOnDiskSizeWithoutHeader() {
+    return onDiskSizeWithoutHeader;
+  }
+
+  /**
+   * @return the uncompressed size of the data part of the block, header not
+   *         included
+   */
+   public int getUncompressedSizeWithoutHeader() {
+    return uncompressedSizeWithoutHeader;
+  }
+
+  /**
+   * @return the offset of the previous block of the same type in the file, or
+   *         -1 if unknown
+   */
+  public long getPrevBlockOffset() {
+    return prevBlockOffset;
+  }
+
+  /**
+   * Writes header fields into the first {@link ©HEADER_SIZE_WITH_CHECKSUMS} bytes of the
+   * buffer. Resets the buffer position to the end of header as side effect.
+   */
+  private void overwriteHeader() {
+    buf.rewind();
+    blockType.write(buf);
+    buf.putInt(onDiskSizeWithoutHeader);
+    buf.putInt(uncompressedSizeWithoutHeader);
+    buf.putLong(prevBlockOffset);
+  }
+
+  /**
+   * Returns a buffer that does not include the header. The array offset points
+   * to the start of the block data right after the header. The underlying data
+   * array is not copied. Checksum data is not included in the returned buffer.
+   *
+   * @return the buffer with header skipped
+   */
+  ByteBuffer getBufferWithoutHeader() {
+    return ByteBuffer.wrap(buf.array(), buf.arrayOffset() + headerSize(),
+        buf.limit() - headerSize() - totalChecksumBytes()).slice();
+  }
+
+  /**
+   * Returns the buffer this block stores internally. The clients must not
+   * modify the buffer object. This method has to be public because it is
+   * used in {@link CompoundBloomFilter} to avoid object creation on every
+   * Bloom filter lookup, but has to be used with caution. Checksum data
+   * is not included in the returned buffer.
+   *
+   * @return the buffer of this block for read-only operations
+   */
+  public ByteBuffer getBufferReadOnly() {
+    return ByteBuffer.wrap(buf.array(), buf.arrayOffset(),
+        buf.limit() - totalChecksumBytes()).slice();
+  }
+
+  /**
+   * Returns a byte buffer of this block, including header data, positioned at
+   * the beginning of header. The underlying data array is not copied.
+   *
+   * @return the byte buffer with header included
+   */
+  ByteBuffer getBufferWithHeader() {
+    ByteBuffer dupBuf = buf.duplicate();
+    dupBuf.rewind();
+    return dupBuf;
+  }
+
+  /**
+   * Deserializes fields of the given writable using the data portion of this
+   * block. Does not check that all the block data has been read.
+   */
+  void readInto(Writable w) throws IOException {
+    Preconditions.checkNotNull(w);
+
+    if (Writables.getWritable(buf.array(), buf.arrayOffset() + headerSize(),
+        buf.limit() - headerSize(), w) == null) {
+      throw new IOException("Failed to deserialize block " + this + " into a "
+          + w.getClass().getSimpleName());
+    }
+  }
+
+  private void sanityCheckAssertion(long valueFromBuf, long valueFromField,
+      String fieldName) throws IOException {
+    if (valueFromBuf != valueFromField) {
+      throw new AssertionError(fieldName + " in the buffer (" + valueFromBuf
+          + ") is different from that in the field (" + valueFromField + ")");
+    }
+  }
+
+  /**
+   * Checks if the block is internally consistent, i.e. the first
+   * {@link HFileBlock#headerSize(int)} bytes of the buffer contain a valid header consistent
+   * with the fields. This function is primary for testing and debugging, and
+   * is not thread-safe, because it alters the internal buffer pointer.
+   */
+  void sanityCheck() throws IOException {
+    buf.rewind();
+
+    {
+      BlockType blockTypeFromBuf = BlockType.read(buf);
+      if (blockTypeFromBuf != blockType) {
+        throw new IOException("Block type stored in the buffer: " +
+            blockTypeFromBuf + ", block type field: " + blockType);
+      }
+    }
+
+    sanityCheckAssertion(buf.getInt(), onDiskSizeWithoutHeader,
+        "onDiskSizeWithoutHeader");
+
+    sanityCheckAssertion(buf.getInt(), uncompressedSizeWithoutHeader,
+        "uncompressedSizeWithoutHeader");
+
+    sanityCheckAssertion(buf.getLong(), prevBlockOffset, "prevBlocKOffset");
+    if (minorVersion >= MINOR_VERSION_WITH_CHECKSUM) {
+      sanityCheckAssertion(buf.get(), checksumType, "checksumType");
+      sanityCheckAssertion(buf.getInt(), bytesPerChecksum, "bytesPerChecksum");
+      sanityCheckAssertion(buf.getInt(), onDiskDataSizeWithHeader, 
+                           "onDiskDataSizeWithHeader");
+    }
+
+    int cksumBytes = totalChecksumBytes();
+    int hdrSize = headerSize();
+    int expectedBufLimit = uncompressedSizeWithoutHeader + headerSize() +
+                           cksumBytes;
+    if (buf.limit() != expectedBufLimit) {
+      throw new AssertionError("Expected buffer limit " + expectedBufLimit
+          + ", got " + buf.limit());
+    }
+
+    // We might optionally allocate HEADER_SIZE_WITH_CHECKSUMS more bytes to read the next
+    // block's, header, so there are two sensible values for buffer capacity.
+    int size = uncompressedSizeWithoutHeader + hdrSize + cksumBytes;
+    if (buf.capacity() != size &&
+        buf.capacity() != size + hdrSize) {
+      throw new AssertionError("Invalid buffer capacity: " + buf.capacity() +
+          ", expected " + size + " or " + (size + hdrSize));
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "blockType="
+        + blockType
+        + ", onDiskSizeWithoutHeader="
+        + onDiskSizeWithoutHeader
+        + ", uncompressedSizeWithoutHeader="
+        + uncompressedSizeWithoutHeader
+        + ", prevBlockOffset="
+        + prevBlockOffset
+        + ", dataBeginsWith="
+        + Bytes.toStringBinary(buf.array(), buf.arrayOffset() + headerSize(),
+            Math.min(32, buf.limit() - buf.arrayOffset() - headerSize()))
+        + ", fileOffset=" + offset;
+  }
+
+  private void validateOnDiskSizeWithoutHeader(
+      int expectedOnDiskSizeWithoutHeader) throws IOException {
+    if (onDiskSizeWithoutHeader != expectedOnDiskSizeWithoutHeader) {
+      String blockInfoMsg =
+        "Block offset: " + offset + ", data starts with: "
+          + Bytes.toStringBinary(buf.array(), buf.arrayOffset(),
+              buf.arrayOffset() + Math.min(32, buf.limit()));
+      throw new IOException("On-disk size without header provided is "
+          + expectedOnDiskSizeWithoutHeader + ", but block "
+          + "header contains " + onDiskSizeWithoutHeader + ". " +
+          blockInfoMsg);
+    }
+  }
+
+  /**
+   * Always allocates a new buffer of the correct size. Copies header bytes
+   * from the existing buffer. Does not change header fields. 
+   * Reserve room to keep checksum bytes too.
+   *
+   * @param extraBytes whether to reserve room in the buffer to read the next
+   *          block's header
+   */
+  private void allocateBuffer(boolean extraBytes) {
+    int cksumBytes = totalChecksumBytes();
+    int capacityNeeded = headerSize() + uncompressedSizeWithoutHeader +
+        cksumBytes +
+        (extraBytes ? headerSize() : 0);
+
+    ByteBuffer newBuf = ByteBuffer.allocate(capacityNeeded);
+
+    // Copy header bytes.
+    System.arraycopy(buf.array(), buf.arrayOffset(), newBuf.array(),
+        newBuf.arrayOffset(), headerSize());
+
+    buf = newBuf;
+    buf.limit(headerSize() + uncompressedSizeWithoutHeader + cksumBytes);
+  }
+
+  /** An additional sanity-check in case no compression is being used. */
+  public void assumeUncompressed() throws IOException {
+    if (onDiskSizeWithoutHeader != uncompressedSizeWithoutHeader + 
+        totalChecksumBytes()) {
+      throw new IOException("Using no compression but "
+          + "onDiskSizeWithoutHeader=" + onDiskSizeWithoutHeader + ", "
+          + "uncompressedSizeWithoutHeader=" + uncompressedSizeWithoutHeader
+          + ", numChecksumbytes=" + totalChecksumBytes());
+    }
+  }
+
+  /**
+   * @param expectedType the expected type of this block
+   * @throws IOException if this block's type is different than expected
+   */
+  public void expectType(BlockType expectedType) throws IOException {
+    if (blockType != expectedType) {
+      throw new IOException("Invalid block type: expected=" + expectedType
+          + ", actual=" + blockType);
+    }
+  }
+
+  /** @return the offset of this block in the file it was read from */
+  public long getOffset() {
+    if (offset < 0) {
+      throw new IllegalStateException(
+          "HFile block offset not initialized properly");
+    }
+    return offset;
+  }
+
+  /**
+   * @return a byte stream reading the data section of this block
+   */
+  public DataInputStream getByteStream() {
+    return new DataInputStream(new ByteArrayInputStream(buf.array(),
+        buf.arrayOffset() + headerSize(), buf.limit() - headerSize()));
+  }
+
+  @Override
+  public long heapSize() {
+    long size = ClassSize.align(
+        // Base class size, including object overhead.
+        SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
+        // Block type and byte buffer references
+        2 * ClassSize.REFERENCE +
+        // On-disk size, uncompressed size, and next block's on-disk size
+        // bytePerChecksum,  onDiskDataSize and minorVersion
+        6 * Bytes.SIZEOF_INT +
+        // Checksum type
+        1 * Bytes.SIZEOF_BYTE +
+        // This and previous block offset
+        2 * Bytes.SIZEOF_LONG +
+        // "Include memstore timestamp" flag
+        Bytes.SIZEOF_BOOLEAN
+    );
+
+    if (buf != null) {
+      // Deep overhead of the byte buffer. Needs to be aligned separately.
+      size += ClassSize.align(buf.capacity() + BYTE_BUFFER_HEAP_SIZE);
+    }
+
+    return ClassSize.align(size);
+  }
+
+  /**
+   * Read from an input stream. Analogous to
+   * {@link IOUtils#readFully(InputStream, byte[], int, int)}, but specifies a
+   * number of "extra" bytes that would be desirable but not absolutely
+   * necessary to read.
+   *
+   * @param in the input stream to read from
+   * @param buf the buffer to read into
+   * @param bufOffset the destination offset in the buffer
+   * @param necessaryLen the number of bytes that are absolutely necessary to
+   *          read
+   * @param extraLen the number of extra bytes that would be nice to read
+   * @return true if succeeded reading the extra bytes
+   * @throws IOException if failed to read the necessary bytes
+   */
+  public static boolean readWithExtra(InputStream in, byte buf[],
+      int bufOffset, int necessaryLen, int extraLen) throws IOException {
+    int bytesRemaining = necessaryLen + extraLen;
+    while (bytesRemaining > 0) {
+      int ret = in.read(buf, bufOffset, bytesRemaining);
+      if (ret == -1 && bytesRemaining <= extraLen) {
+        // We could not read the "extra data", but that is OK.
+        break;
+      }
+
+      if (ret < 0) {
+        throw new IOException("Premature EOF from inputStream (read "
+            + "returned " + ret + ", was trying to read " + necessaryLen
+            + " necessary bytes and " + extraLen + " extra bytes, "
+            + "successfully read "
+            + (necessaryLen + extraLen - bytesRemaining));
+      }
+      bufOffset += ret;
+      bytesRemaining -= ret;
+    }
+    return bytesRemaining <= 0;
+  }
+
+  /**
+   * @return the on-disk size of the next block (including the header size)
+   *         that was read by peeking into the next block's header
+   */
+  public int getNextBlockOnDiskSizeWithHeader() {
+    return nextBlockOnDiskSizeWithHeader;
+  }
+
+
+  /**
+   * Unified version 2 {@link HFile} block writer. The intended usage pattern
+   * is as follows:
+   * 
        
+   * 
      • Construct an {@link HFileBlock.Writer}, providing a compression
+   * algorithm
+   * 
      • Call {@link Writer#startWriting(BlockType, boolean)} and get a data stream to
+   * write to
+   * 
      • Write your data into the stream
+   * 
      • Call {@link Writer#writeHeaderAndData(FSDataOutputStream)} as many times as you need to
+   * store the serialized block into an external stream, or call
+   * {@link Writer#getHeaderAndData()} to get it as a byte array.
+   * 
      • Repeat to write more blocks
+   * 
      
+   * 
      
+   */
+  public static class Writer {
+
+    private enum State {
+      INIT,
+      WRITING,
+      BLOCK_READY
+    };
+
+    /** Writer state. Used to ensure the correct usage protocol. */
+    private State state = State.INIT;
+
+    /** Compression algorithm for all blocks this instance writes. */
+    private final Compression.Algorithm compressAlgo;
+
+    /** Data block encoder used for data blocks */
+    private final HFileDataBlockEncoder dataBlockEncoder;
+
+    /**
+     * The stream we use to accumulate data in uncompressed format for each
+     * block. We reset this stream at the end of each block and reuse it. The
+     * header is written as the first {@link HFileBlock#headerSize(int)}  bytes into this
+     * stream.
+     */
+    private ByteArrayOutputStream baosInMemory;
+
+    /** Compressor, which is also reused between consecutive blocks. */
+    private Compressor compressor;
+
+    /** Compression output stream */
+    private CompressionOutputStream compressionStream;
+    
+    /** Underlying stream to write compressed bytes to */
+    private ByteArrayOutputStream compressedByteStream;
+
+    /**
+     * Current block type. Set in {@link #startWriting(BlockType)}. Could be
+     * changed in {@link #encodeDataBlockForDisk()} from {@link BlockType#DATA}
+     * to {@link BlockType#ENCODED_DATA}.
+     */
+    private BlockType blockType;
+
+    /**
+     * A stream that we write uncompressed bytes to, which compresses them and
+     * writes them to {@link #baosInMemory}.
+     */
+    private DataOutputStream userDataStream;
+
+    /**
+     * Bytes to be written to the file system, including the header. Compressed
+     * if compression is turned on. It also includes the checksum data that 
+     * immediately follows the block data. (header + data + checksums)
+     */
+    private byte[] onDiskBytesWithHeader;
+
+    /**
+     * The size of the data on disk that does not include the checksums.
+     * (header + data)
+     */
+    private int onDiskDataSizeWithHeader;
+
+    /**
+     * The size of the checksum data on disk. It is used only if data is
+     * not compressed. If data is compressed, then the checksums are already
+     * part of onDiskBytesWithHeader. If data is uncompressed, then this
+     * variable stores the checksum data for this block.
+     */
+    private byte[] onDiskChecksum;
+
+    /**
+     * Valid in the READY state. Contains the header and the uncompressed (but
+     * potentially encoded, if this is a data block) bytes, so the length is
+     * {@link #uncompressedSizeWithoutHeader} + {@link HFileBlock#headerSize(int)}.
+     * Does not store checksums.
+     */
+    private byte[] uncompressedBytesWithHeader;
+
+    /**
+     * Current block's start offset in the {@link HFile}. Set in
+     * {@link #writeHeaderAndData(FSDataOutputStream)}.
+     */
+    private long startOffset;
+
+    /**
+     * Offset of previous block by block type. Updated when the next block is
+     * started.
+     */
+    private long[] prevOffsetByType;
+
+    /** The offset of the previous block of the same type */
+    private long prevOffset;
+
+    /** Whether we are including memstore timestamp after every key/value */
+    private boolean includesMemstoreTS;
+
+    /** Checksum settings */
+    private ChecksumType checksumType;
+    private int bytesPerChecksum;
+
+    private final int minorVersion;
+
+    /**
+     * @param compressionAlgorithm compression algorithm to use
+     * @param dataBlockEncoderAlgo data block encoding algorithm to use
+     * @param checksumType type of checksum
+     * @param bytesPerChecksum bytes per checksum
+     */
+    public Writer(Compression.Algorithm compressionAlgorithm,
+          HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS,
+          int minorVersion,
+          ChecksumType checksumType, int bytesPerChecksum) {
+      this.minorVersion = minorVersion;
+      compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm;
+      this.dataBlockEncoder = dataBlockEncoder != null
+          ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
+
+      baosInMemory = new ByteArrayOutputStream();
+      if (compressAlgo != NONE) {
+        compressor = compressionAlgorithm.getCompressor();
+        compressedByteStream = new ByteArrayOutputStream();
+        try {
+          compressionStream =
+              compressionAlgorithm.createPlainCompressionStream(
+                  compressedByteStream, compressor);
+        } catch (IOException e) {
+          throw new RuntimeException("Could not create compression stream " + 
+              "for algorithm " + compressionAlgorithm, e);
+        }
+      }
+      if (minorVersion > MINOR_VERSION_NO_CHECKSUM
+          && bytesPerChecksum < HEADER_SIZE_WITH_CHECKSUMS) {
+        throw new RuntimeException("Unsupported value of bytesPerChecksum. " +
+            " Minimum is " + HEADER_SIZE_WITH_CHECKSUMS + " but the configured value is " +
+            bytesPerChecksum);
+      }
+      
+      prevOffsetByType = new long[BlockType.values().length];
+      for (int i = 0; i < prevOffsetByType.length; ++i)
+        prevOffsetByType[i] = -1;
+
+      this.includesMemstoreTS = includesMemstoreTS;
+      this.checksumType = checksumType;
+      this.bytesPerChecksum = bytesPerChecksum;
+    }
+
+    /**
+     * Starts writing into the block. The previous block's data is discarded.
+     *
+     * @return the stream the user can write their data into
+     * @throws IOException
+     */
+    public DataOutputStream startWriting(BlockType newBlockType)
+        throws IOException {
+      if (state == State.BLOCK_READY && startOffset != -1) {
+        // We had a previous block that was written to a stream at a specific
+        // offset. Save that offset as the last offset of a block of that type.
+        prevOffsetByType[blockType.getId()] = startOffset;
+      }
+
+      startOffset = -1;
+      blockType = newBlockType;
+
+      baosInMemory.reset();
+      baosInMemory.write(getDummyHeaderForVersion(this.minorVersion));
+
+      state = State.WRITING;
+
+      // We will compress it later in finishBlock()
+      userDataStream = new DataOutputStream(baosInMemory);
+      return userDataStream;
+    }
+
+    /**
+     * Returns the stream for the user to write to. The block writer takes care
+     * of handling compression and buffering for caching on write. Can only be
+     * called in the "writing" state.
+     *
+     * @return the data output stream for the user to write to
+     */
+    DataOutputStream getUserDataStream() {
+      expectState(State.WRITING);
+      return userDataStream;
+    }
+
+    /**
+     * Transitions the block writer from the "writing" state to the "block
+     * ready" state.  Does nothing if a block is already finished.
+     */
+    private void ensureBlockReady() throws IOException {
+      Preconditions.checkState(state != State.INIT,
+          "Unexpected state: " + state);
+
+      if (state == State.BLOCK_READY)
+        return;
+
+      // This will set state to BLOCK_READY.
+      finishBlock();
+    }
+
+    /**
+     * An internal method that flushes the compressing stream (if using
+     * compression), serializes the header, and takes care of the separate
+     * uncompressed stream for caching on write, if applicable. Sets block
+     * write state to "block ready".
+     */
+    private void finishBlock() throws IOException {
+      userDataStream.flush();
+
+      // This does an array copy, so it is safe to cache this byte array.
+      uncompressedBytesWithHeader = baosInMemory.toByteArray();
+      prevOffset = prevOffsetByType[blockType.getId()];
+
+      // We need to set state before we can package the block up for
+      // cache-on-write. In a way, the block is ready, but not yet encoded or
+      // compressed.
+      state = State.BLOCK_READY;
+      encodeDataBlockForDisk();
+
+      doCompressionAndChecksumming();
+    }
+
+    /**
+     * Do compression if it is enabled, or re-use the uncompressed buffer if
+     * it is not. Fills in the compressed block's header if doing compression.
+     * Also, compute the checksums. In the case of no-compression, write the
+     * checksums to its own seperate data structure called onDiskChecksum. In
+     * the case when compression is enabled, the checksums are written to the
+     * outputbyte stream 'baos'.
+     */
+    private void doCompressionAndChecksumming() throws IOException {
+      if ( minorVersion <= MINOR_VERSION_NO_CHECKSUM) {
+        version20compression();
+      } else {
+        version21ChecksumAndCompression();
+      }
+    }
+
+    private void version20compression() throws IOException {
+      onDiskChecksum = HConstants.EMPTY_BYTE_ARRAY;
+
+      if (compressAlgo != NONE) {
+        compressedByteStream.reset();
+        compressedByteStream.write(DUMMY_HEADER_NO_CHECKSUM);
+
+        compressionStream.resetState();
+
+        compressionStream.write(uncompressedBytesWithHeader, headerSize(this.minorVersion),
+            uncompressedBytesWithHeader.length - headerSize(this.minorVersion));
+
+
+        compressionStream.flush();
+        compressionStream.finish();
+        onDiskDataSizeWithHeader = compressedByteStream.size(); // data size
+        onDiskBytesWithHeader = compressedByteStream.toByteArray();
+
+        put20Header(onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length,
+            uncompressedBytesWithHeader.length);
+
+
+        //set the header for the uncompressed bytes (for cache-on-write)
+        put20Header(uncompressedBytesWithHeader, 0,
+            onDiskBytesWithHeader.length + onDiskChecksum.length,
+            uncompressedBytesWithHeader.length);
+
+      } else {
+        onDiskBytesWithHeader = uncompressedBytesWithHeader;
+
+        onDiskDataSizeWithHeader = onDiskBytesWithHeader.length;
+
+        //set the header for the uncompressed bytes
+        put20Header(uncompressedBytesWithHeader, 0,
+            onDiskBytesWithHeader.length,
+            uncompressedBytesWithHeader.length);
+      }
+    }
+
+    private void version21ChecksumAndCompression() throws IOException {
+      // do the compression
+      if (compressAlgo != NONE) {
+        compressedByteStream.reset();
+        compressedByteStream.write(DUMMY_HEADER_WITH_CHECKSUM);
+
+        compressionStream.resetState();
+
+        compressionStream.write(uncompressedBytesWithHeader, headerSize(this.minorVersion),
+            uncompressedBytesWithHeader.length - headerSize(this.minorVersion));
+
+        compressionStream.flush();
+        compressionStream.finish();
+
+        // generate checksums
+        onDiskDataSizeWithHeader = compressedByteStream.size(); // data size
+
+        // reserve space for checksums in the output byte stream
+        ChecksumUtil.reserveSpaceForChecksums(compressedByteStream, 
+          onDiskDataSizeWithHeader, bytesPerChecksum);
+
+
+        onDiskBytesWithHeader = compressedByteStream.toByteArray();
+        put21Header(onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length,
+            uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
+
+       // generate checksums for header and data. The checksums are
+       // part of onDiskBytesWithHeader itself.
+       ChecksumUtil.generateChecksums(
+         onDiskBytesWithHeader, 0, onDiskDataSizeWithHeader,
+         onDiskBytesWithHeader, onDiskDataSizeWithHeader,
+         checksumType, bytesPerChecksum);
+
+        // Checksums are already part of onDiskBytesWithHeader
+        onDiskChecksum = HConstants.EMPTY_BYTE_ARRAY;
+
+        //set the header for the uncompressed bytes (for cache-on-write)
+        put21Header(uncompressedBytesWithHeader, 0,
+            onDiskBytesWithHeader.length + onDiskChecksum.length,
+            uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
+
+      } else {
+        // If we are not using any compression, then the
+        // checksums are written to its own array onDiskChecksum.
+        onDiskBytesWithHeader = uncompressedBytesWithHeader;
+
+        onDiskDataSizeWithHeader = onDiskBytesWithHeader.length;
+        int numBytes = (int)ChecksumUtil.numBytes(
+                          uncompressedBytesWithHeader.length,
+                          bytesPerChecksum);
+        onDiskChecksum = new byte[numBytes];
+
+        //set the header for the uncompressed bytes
+        put21Header(uncompressedBytesWithHeader, 0,
+            onDiskBytesWithHeader.length + onDiskChecksum.length,
+            uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
+
+        ChecksumUtil.generateChecksums(
+          uncompressedBytesWithHeader, 0, uncompressedBytesWithHeader.length,
+          onDiskChecksum, 0,
+          checksumType, bytesPerChecksum);
+      }
+    }
+
+    /**
+     * Encodes this block if it is a data block and encoding is turned on in
+     * {@link #dataBlockEncoder}.
+     */
+    private void encodeDataBlockForDisk() throws IOException {
+      if (blockType != BlockType.DATA) {
+        return; // skip any non-data block
+      }
+
+      // do data block encoding, if data block encoder is set
+      ByteBuffer rawKeyValues = ByteBuffer.wrap(uncompressedBytesWithHeader,
+          headerSize(this.minorVersion), uncompressedBytesWithHeader.length -
+          headerSize(this.minorVersion)).slice();
+      Pair encodingResult =
+          dataBlockEncoder.beforeWriteToDisk(rawKeyValues,
+              includesMemstoreTS, getDummyHeaderForVersion(this.minorVersion));
+
+      BlockType encodedBlockType = encodingResult.getSecond();
+      if (encodedBlockType == BlockType.ENCODED_DATA) {
+        uncompressedBytesWithHeader = encodingResult.getFirst().array();
+        blockType = BlockType.ENCODED_DATA;
+      } else {
+        // There is no encoding configured. Do some extra sanity-checking.
+        if (encodedBlockType != BlockType.DATA) {
+          throw new IOException("Unexpected block type coming out of data " +
+              "block encoder: " + encodedBlockType);
+        }
+        if (userDataStream.size() !=
+            uncompressedBytesWithHeader.length - headerSize(this.minorVersion)) {
+          throw new IOException("Uncompressed size mismatch: "
+              + userDataStream.size() + " vs. "
+              + (uncompressedBytesWithHeader.length - headerSize(this.minorVersion)));
+        }
+      }
+    }
+
+    /**
+     * Put the header into the given byte array at the given offset.
+     * @param onDiskSize size of the block on disk header + data + checksum
+     * @param uncompressedSize size of the block after decompression (but
+     *          before optional data block decoding) including header
+     * @param onDiskDataSize size of the block on disk with header
+     *        and data but not including the checksums
+     */
+    private void put21Header(byte[] dest, int offset, int onDiskSize,
+                             int uncompressedSize, int onDiskDataSize) {
+      offset = blockType.put(dest, offset);
+      offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE_WITH_CHECKSUMS);
+      offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE_WITH_CHECKSUMS);
+      offset = Bytes.putLong(dest, offset, prevOffset);
+      offset = Bytes.putByte(dest, offset, checksumType.getCode());
+      offset = Bytes.putInt(dest, offset, bytesPerChecksum);
+      offset = Bytes.putInt(dest, offset, onDiskDataSizeWithHeader);
+    }
+
+
+    private void put20Header(byte[] dest, int offset, int onDiskSize,
+                             int uncompressedSize) {
+      offset = blockType.put(dest, offset);
+      offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE_NO_CHECKSUM);
+      offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE_NO_CHECKSUM);
+      Bytes.putLong(dest, offset, prevOffset);
+    }
+    /**
+     * Similar to {@link #writeHeaderAndData(FSDataOutputStream)}, but records
+     * the offset of this block so that it can be referenced in the next block
+     * of the same type.
+     *
+     * @param out
+     * @throws IOException
+     */
+    public void writeHeaderAndData(FSDataOutputStream out) throws IOException {
+      long offset = out.getPos();
+      if (startOffset != -1 && offset != startOffset) {
+        throw new IOException("A " + blockType + " block written to a "
+            + "stream twice, first at offset " + startOffset + ", then at "
+            + offset);
+      }
+      startOffset = offset;
+
+      writeHeaderAndData((DataOutputStream) out);
+    }
+
+    /**
+     * Writes the header and the compressed data of this block (or uncompressed
+     * data when not using compression) into the given stream. Can be called in
+     * the "writing" state or in the "block ready" state. If called in the
+     * "writing" state, transitions the writer to the "block ready" state.
+     *
+     * @param out the output stream to write the
+     * @throws IOException
+     */
+    private void writeHeaderAndData(DataOutputStream out) throws IOException {
+      ensureBlockReady();
+      out.write(onDiskBytesWithHeader);
+      if (compressAlgo == NONE && minorVersion > MINOR_VERSION_NO_CHECKSUM) {
+        if (onDiskChecksum == HConstants.EMPTY_BYTE_ARRAY) {
+          throw new IOException("A " + blockType 
+              + " without compression should have checksums " 
+              + " stored separately.");
+        }
+        out.write(onDiskChecksum);
+      }
+    }
+
+    /**
+     * Returns the header or the compressed data (or uncompressed data when not
+     * using compression) as a byte array. Can be called in the "writing" state
+     * or in the "block ready" state. If called in the "writing" state,
+     * transitions the writer to the "block ready" state. This returns
+     * the header + data + checksums stored on disk.
+     *
+     * @return header and data as they would be stored on disk in a byte array
+     * @throws IOException
+     */
+    byte[] getHeaderAndDataForTest() throws IOException {
+      ensureBlockReady();
+      if (compressAlgo == NONE) {
+        if (onDiskChecksum == HConstants.EMPTY_BYTE_ARRAY) {
+          throw new IOException("A " + blockType 
+              + " without compression should have checksums " 
+              + " stored separately.");
+        }
+        // This is not very optimal, because we are doing an extra copy.
+        // But this method is used only by unit tests.
+        byte[] output = new byte[onDiskBytesWithHeader.length +
+                                 onDiskChecksum.length];
+        System.arraycopy(onDiskBytesWithHeader, 0,
+                         output, 0, onDiskBytesWithHeader.length);
+        System.arraycopy(onDiskChecksum, 0,
+                         output, onDiskBytesWithHeader.length,
+                         onDiskChecksum.length);
+        return output;
+      }
+      return onDiskBytesWithHeader;
+    }
+
+    /**
+     * Releases the compressor this writer uses to compress blocks into the
+     * compressor pool. Needs to be called before the writer is discarded.
+     */
+    public void releaseCompressor() {
+      if (compressor != null) {
+        compressAlgo.returnCompressor(compressor);
+        compressor = null;
+      }
+    }
+
+    /**
+     * Returns the on-disk size of the data portion of the block. This is the
+     * compressed size if compression is enabled. Can only be called in the
+     * "block ready" state. Header is not compressed, and its size is not
+     * included in the return value.
+     *
+     * @return the on-disk size of the block, not including the header.
+     */
+    int getOnDiskSizeWithoutHeader() {
+      expectState(State.BLOCK_READY);
+      return onDiskBytesWithHeader.length + onDiskChecksum.length - headerSize(this.minorVersion);
+    }
+
+    /**
+     * Returns the on-disk size of the block. Can only be called in the
+     * "block ready" state.
+     *
+     * @return the on-disk size of the block ready to be written, including the
+     *         header size, the data and the checksum data.
+     */
+    int getOnDiskSizeWithHeader() {
+      expectState(State.BLOCK_READY);
+      return onDiskBytesWithHeader.length + onDiskChecksum.length;
+    }
+
+    /**
+     * The uncompressed size of the block data. Does not include header size.
+     */
+    int getUncompressedSizeWithoutHeader() {
+      expectState(State.BLOCK_READY);
+      return uncompressedBytesWithHeader.length - headerSize(this.minorVersion);
+    }
+
+    /**
+     * The uncompressed size of the block data, including header size.
+     */
+    int getUncompressedSizeWithHeader() {
+      expectState(State.BLOCK_READY);
+      return uncompressedBytesWithHeader.length;
+    }
+
+    /** @return true if a block is being written  */
+    public boolean isWriting() {
+      return state == State.WRITING;
+    }
+
+    /**
+     * Returns the number of bytes written into the current block so far, or
+     * zero if not writing the block at the moment. Note that this will return
+     * zero in the "block ready" state as well.
+     *
+     * @return the number of bytes written
+     */
+    public int blockSizeWritten() {
+      if (state != State.WRITING)
+        return 0;
+      return userDataStream.size();
+    }
+
+    /**
+     * Returns the header followed by the uncompressed data, even if using
+     * compression. This is needed for storing uncompressed blocks in the block
+     * cache. Can be called in the "writing" state or the "block ready" state.
+     * Returns only the header and data, does not include checksum data.
+     *
+     * @return uncompressed block bytes for caching on write
+     */
+    ByteBuffer getUncompressedBufferWithHeader() {
+      expectState(State.BLOCK_READY);
+      return ByteBuffer.wrap(uncompressedBytesWithHeader);
+    }
+
+    private void expectState(State expectedState) {
+      if (state != expectedState) {
+        throw new IllegalStateException("Expected state: " + expectedState +
+            ", actual state: " + state);
+      }
+    }
+
+    /**
+     * Takes the given {@link BlockWritable} instance, creates a new block of
+     * its appropriate type, writes the writable into this block, and flushes
+     * the block into the output stream. The writer is instructed not to buffer
+     * uncompressed bytes for cache-on-write.
+     *
+     * @param bw the block-writable object to write as a block
+     * @param out the file system output stream
+     * @throws IOException
+     */
+    public void writeBlock(BlockWritable bw, FSDataOutputStream out)
+        throws IOException {
+      bw.writeToBlock(startWriting(bw.getBlockType()));
+      writeHeaderAndData(out);
+    }
+
+    /**
+     * Creates a new HFileBlock. Checksums have already been validated, so
+     * the byte buffer passed into the constructor of this newly created
+     * block does not have checksum data even though the header minor 
+     * version is MINOR_VERSION_WITH_CHECKSUM. This is indicated by setting a
+     * 0 value in bytesPerChecksum.
+     */
+    public HFileBlock getBlockForCaching() {
+      return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(),
+          getUncompressedSizeWithoutHeader(), prevOffset,
+          getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset,
+          includesMemstoreTS, this.minorVersion,
+          0, ChecksumType.NULL.getCode(),  // no checksums in cached data
+          onDiskBytesWithHeader.length + onDiskChecksum.length);
+    }
+  }
+
+  /** Something that can be written into a block. */
+  public interface BlockWritable {
+
+    /** The type of block this data should use. */
+    BlockType getBlockType();
+
+    /**
+     * Writes the block to the provided stream. Must not write any magic
+     * records.
+     *
+     * @param out a stream to write uncompressed data into
+     */
+    void writeToBlock(DataOutput out) throws IOException;
+  }
+
+  // Block readers and writers
+
+  /** An interface allowing to iterate {@link HFileBlock}s. */
+  public interface BlockIterator {
+
+    /**
+     * Get the next block, or null if there are no more blocks to iterate.
+     */
+    HFileBlock nextBlock() throws IOException;
+
+    /**
+     * Similar to {@link #nextBlock()} but checks block type, throws an
+     * exception if incorrect, and returns the HFile block
+     */
+    HFileBlock nextBlockWithBlockType(BlockType blockType) throws IOException;
+  }
+
+  /** A full-fledged reader with iteration ability. */
+  public interface FSReader {
+
+    /**
+     * Reads the block at the given offset in the file with the given on-disk
+     * size and uncompressed size.
+     *
+     * @param offset
+     * @param onDiskSize the on-disk size of the entire block, including all
+     *          applicable headers, or -1 if unknown
+     * @param uncompressedSize the uncompressed size of the compressed part of
+     *          the block, or -1 if unknown
+     * @return the newly read block
+     */
+    HFileBlock readBlockData(long offset, long onDiskSize,
+        int uncompressedSize, boolean pread) throws IOException;
+
+    /**
+     * Creates a block iterator over the given portion of the {@link HFile}.
+     * The iterator returns blocks starting with offset such that offset <=
+     * startOffset < endOffset.
+     *
+     * @param startOffset the offset of the block to start iteration with
+     * @param endOffset the offset to end iteration at (exclusive)
+     * @return an iterator of blocks between the two given offsets
+     */
+    BlockIterator blockRange(long startOffset, long endOffset);
+  }
+
+  /**
+   * A common implementation of some methods of {@link FSReader} and some
+   * tools for implementing HFile format version-specific block readers.
+   */
+  private abstract static class AbstractFSReader implements FSReader {
+
+    /** The file system stream of the underlying {@link HFile} that 
+     * does checksum validations in the filesystem */
+    protected final FSDataInputStream istream;
+
+    /** The file system stream of the underlying {@link HFile} that
+     * does not do checksum verification in the file system */
+    protected final FSDataInputStream istreamNoFsChecksum;
+
+    /** Compression algorithm used by the {@link HFile} */
+    protected Compression.Algorithm compressAlgo;
+
+    /** The size of the file we are reading from, or -1 if unknown. */
+    protected long fileSize;
+
+    /** The minor version of this reader */
+    private int minorVersion;
+
+    /** The size of the header */
+    protected int hdrSize;
+
+    /** The filesystem used to access data */
+    protected HFileSystem hfs;
+
+    /** The path (if any) where this data is coming from */
+    protected Path path;
+
+    private final Lock streamLock = new ReentrantLock();
+
+    /** The default buffer size for our buffered streams */
+    public static final int DEFAULT_BUFFER_SIZE = 1 << 20;
+
+    public AbstractFSReader(FSDataInputStream istream, 
+        FSDataInputStream istreamNoFsChecksum,
+        Algorithm compressAlgo,
+        long fileSize, int minorVersion, HFileSystem hfs, Path path) 
+        throws IOException {
+      this.istream = istream;
+      this.compressAlgo = compressAlgo;
+      this.fileSize = fileSize;
+      this.minorVersion = minorVersion;
+      this.hfs = hfs;
+      this.path = path;
+      this.hdrSize = headerSize(minorVersion);
+      this.istreamNoFsChecksum = istreamNoFsChecksum;
+    }
+
+    @Override
+    public BlockIterator blockRange(final long startOffset,
+        final long endOffset) {
+      return new BlockIterator() {
+        private long offset = startOffset;
+
+        @Override
+        public HFileBlock nextBlock() throws IOException {
+          if (offset >= endOffset)
+            return null;
+          HFileBlock b = readBlockData(offset, -1, -1, false);
+          offset += b.getOnDiskSizeWithHeader();
+          return b;
+        }
+
+        @Override
+        public HFileBlock nextBlockWithBlockType(BlockType blockType)
+            throws IOException {
+          HFileBlock blk = nextBlock();
+          if (blk.getBlockType() != blockType) {
+            throw new IOException("Expected block of type " + blockType
+                + " but found " + blk.getBlockType());
+          }
+          return blk;
+        }
+      };
+    }
+
+    /**
+     * Does a positional read or a seek and read into the given buffer. Returns
+     * the on-disk size of the next block, or -1 if it could not be determined.
+     *
+     * @param dest destination buffer
+     * @param destOffset offset in the destination buffer
+     * @param size size of the block to be read
+     * @param peekIntoNextBlock whether to read the next block's on-disk size
+     * @param fileOffset position in the stream to read at
+     * @param pread whether we should do a positional read
+     * @param istream The input source of data
+     * @return the on-disk size of the next block with header size included, or
+     *         -1 if it could not be determined
+     * @throws IOException
+     */
+    protected int readAtOffset(FSDataInputStream istream,
+        byte[] dest, int destOffset, int size,
+        boolean peekIntoNextBlock, long fileOffset, boolean pread)
+        throws IOException {
+      if (peekIntoNextBlock &&
+          destOffset + size + hdrSize > dest.length) {
+        // We are asked to read the next block's header as well, but there is
+        // not enough room in the array.
+        throw new IOException("Attempted to read " + size + " bytes and " +
+            hdrSize + " bytes of next header into a " + dest.length +
+            "-byte array at offset " + destOffset);
+      }
+
+      if (!pread && streamLock.tryLock()) {
+        // Seek + read. Better for scanning.
+        try {
+          istream.seek(fileOffset);
+
+          long realOffset = istream.getPos();
+          if (realOffset != fileOffset) {
+            throw new IOException("Tried to seek to " + fileOffset + " to "
+                + "read " + size + " bytes, but pos=" + realOffset
+                + " after seek");
+          }
+
+          if (!peekIntoNextBlock) {
+            IOUtils.readFully(istream, dest, destOffset, size);
+            return -1;
+          }
+
+          // Try to read the next block header.
+          if (!readWithExtra(istream, dest, destOffset, size, hdrSize))
+            return -1;
+        } finally {
+          streamLock.unlock();
+        }
+      } else {
+        // Positional read. Better for random reads; or when the streamLock is already locked.
+        int extraSize = peekIntoNextBlock ? hdrSize : 0;
+
+        int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
+        if (ret < size) {
+          throw new IOException("Positional read of " + size + " bytes " +
+              "failed at offset " + fileOffset + " (returned " + ret + ")");
+        }
+
+        if (ret == size || ret < size + extraSize) {
+          // Could not read the next block's header, or did not try.
+          return -1;
+        }
+      }
+
+      assert peekIntoNextBlock;
+      return Bytes.toInt(dest, destOffset + size + BlockType.MAGIC_LENGTH) +
+          hdrSize;
+    }
+
+    /**
+     * Decompresses data from the given stream using the configured compression
+     * algorithm.
+     * @param dest
+     * @param destOffset
+     * @param bufferedBoundedStream
+     *          a stream to read compressed data from, bounded to the exact
+     *          amount of compressed data
+     * @param uncompressedSize
+     *          uncompressed data size, header not included
+     * @throws IOException
+     */
+    protected void decompress(byte[] dest, int destOffset,
+        InputStream bufferedBoundedStream,
+        int uncompressedSize) throws IOException {
+      Decompressor decompressor = null;
+      try {
+        decompressor = compressAlgo.getDecompressor();
+        InputStream is = compressAlgo.createDecompressionStream(
+            bufferedBoundedStream, decompressor, 0);
+
+        IOUtils.readFully(is, dest, destOffset, uncompressedSize);
+        is.close();
+      } finally {
+        if (decompressor != null) {
+          compressAlgo.returnDecompressor(decompressor);
+        }
+      }
+    }
+
+    /**
+     * Creates a buffered stream reading a certain slice of the file system
+     * input stream. We need this because the decompression we use seems to
+     * expect the input stream to be bounded.
+     *
+     * @param offset the starting file offset the bounded stream reads from
+     * @param size the size of the segment of the file the stream should read
+     * @param pread whether to use position reads
+     * @return a stream restricted to the given portion of the file
+     */
+    protected InputStream createBufferedBoundedStream(long offset,
+        int size, boolean pread) {
+      return new BufferedInputStream(new BoundedRangeFileInputStream(istream,
+          offset, size, pread), Math.min(DEFAULT_BUFFER_SIZE, size));
+    }
+
+    /**
+     * @return The minorVersion of this HFile
+     */
+    protected int getMinorVersion() {
+      return minorVersion;
+    }
+  }
+
+  /**
+   * Reads version 1 blocks from the file system. In version 1 blocks,
+   * everything is compressed, including the magic record, if compression is
+   * enabled. Everything might be uncompressed if no compression is used. This
+   * reader returns blocks represented in the uniform version 2 format in
+   * memory.
+   */
+  static class FSReaderV1 extends AbstractFSReader {
+
+    /** Header size difference between version 1 and 2 */
+    private static final int HEADER_DELTA = HEADER_SIZE_NO_CHECKSUM - 
+                                            MAGIC_LENGTH;
+
+    public FSReaderV1(FSDataInputStream istream, Algorithm compressAlgo,
+        long fileSize) throws IOException {
+      super(istream, istream, compressAlgo, fileSize, 0, null, null);
+    }
+
+    /**
+     * Read a version 1 block. There is no uncompressed header, and the block
+     * type (the magic record) is part of the compressed data. This
+     * implementation assumes that the bounded range file input stream is
+     * needed to stop the decompressor reading into next block, because the
+     * decompressor just grabs a bunch of data without regard to whether it is
+     * coming to end of the compressed section.
+     *
+     * The block returned is still a version 2 block, and in particular, its
+     * first {@link #HEADER_SIZE_WITH_CHECKSUMS} bytes contain a valid version 2 header.
+     *
+     * @param offset the offset of the block to read in the file
+     * @param onDiskSizeWithMagic the on-disk size of the version 1 block,
+     *          including the magic record, which is the part of compressed
+     *          data if using compression
+     * @param uncompressedSizeWithMagic uncompressed size of the version 1
+     *          block, including the magic record
+     */
+    @Override
+    public HFileBlock readBlockData(long offset, long onDiskSizeWithMagic,
+        int uncompressedSizeWithMagic, boolean pread) throws IOException {
+      if (uncompressedSizeWithMagic <= 0) {
+        throw new IOException("Invalid uncompressedSize="
+            + uncompressedSizeWithMagic + " for a version 1 block");
+      }
+
+      if (onDiskSizeWithMagic <= 0 || onDiskSizeWithMagic >= Integer.MAX_VALUE)
+      {
+        throw new IOException("Invalid onDiskSize=" + onDiskSizeWithMagic
+            + " (maximum allowed: " + Integer.MAX_VALUE + ")");
+      }
+
+      int onDiskSize = (int) onDiskSizeWithMagic;
+
+      if (uncompressedSizeWithMagic < MAGIC_LENGTH) {
+        throw new IOException("Uncompressed size for a version 1 block is "
+            + uncompressedSizeWithMagic + " but must be at least "
+            + MAGIC_LENGTH);
+      }
+
+      // The existing size already includes magic size, and we are inserting
+      // a version 2 header.
+      ByteBuffer buf = ByteBuffer.allocate(uncompressedSizeWithMagic
+          + HEADER_DELTA);
+
+      int onDiskSizeWithoutHeader;
+      if (compressAlgo == Compression.Algorithm.NONE) {
+        // A special case when there is no compression.
+        if (onDiskSize != uncompressedSizeWithMagic) {
+          throw new IOException("onDiskSize=" + onDiskSize
+              + " and uncompressedSize=" + uncompressedSizeWithMagic
+              + " must be equal for version 1 with no compression");
+        }
+
+        // The first MAGIC_LENGTH bytes of what this will read will be
+        // overwritten.
+        readAtOffset(istream, buf.array(), buf.arrayOffset() + HEADER_DELTA,
+            onDiskSize, false, offset, pread);
+
+        onDiskSizeWithoutHeader = uncompressedSizeWithMagic - MAGIC_LENGTH;
+      } else {
+        InputStream bufferedBoundedStream = createBufferedBoundedStream(
+            offset, onDiskSize, pread);
+        decompress(buf.array(), buf.arrayOffset() + HEADER_DELTA,
+            bufferedBoundedStream, uncompressedSizeWithMagic);
+
+        // We don't really have a good way to exclude the "magic record" size
+        // from the compressed block's size, since it is compressed as well.
+        onDiskSizeWithoutHeader = onDiskSize;
+      }
+
+      BlockType newBlockType = BlockType.parse(buf.array(), buf.arrayOffset()
+          + HEADER_DELTA, MAGIC_LENGTH);
+
+      // We set the uncompressed size of the new HFile block we are creating
+      // to the size of the data portion of the block without the magic record,
+      // since the magic record gets moved to the header.
+      HFileBlock b = new HFileBlock(newBlockType, onDiskSizeWithoutHeader,
+          uncompressedSizeWithMagic - MAGIC_LENGTH, -1L, buf, FILL_HEADER,
+          offset, MemStore.NO_PERSISTENT_TS, 0, 0, ChecksumType.NULL.getCode(),
+          onDiskSizeWithoutHeader + HEADER_SIZE_NO_CHECKSUM);
+      return b;
+    }
+  }
+
+  /**
+   * We always prefetch the header of the next block, so that we know its
+   * on-disk size in advance and can read it in one operation.
+   */
+  private static class PrefetchedHeader {
+    long offset = -1;
+    byte[] header = new byte[HEADER_SIZE_WITH_CHECKSUMS];
+    ByteBuffer buf = ByteBuffer.wrap(header, 0, HEADER_SIZE_WITH_CHECKSUMS);
+  }
+
+  /** Reads version 2 blocks from the filesystem. */
+  static class FSReaderV2 extends AbstractFSReader {
+
+    // The configuration states that we should validate hbase checksums
+    private final boolean useHBaseChecksumConfigured;
+
+    // Record the current state of this reader with respect to
+    // validating checkums in HBase. This is originally set the same
+    // value as useHBaseChecksumConfigured, but can change state as and when
+    // we encounter checksum verification failures.
+    private volatile boolean useHBaseChecksum;
+
+    // In the case of a checksum failure, do these many succeeding
+    // reads without hbase checksum verification.
+    private volatile int checksumOffCount = -1;
+
+    /** Whether we include memstore timestamp in data blocks */
+    protected boolean includesMemstoreTS;
+
+    /** Data block encoding used to read from file */
+    protected HFileDataBlockEncoder dataBlockEncoder =
+        NoOpDataBlockEncoder.INSTANCE;
+
+    private ThreadLocal prefetchedHeaderForThread =
+        new ThreadLocal() {
+          @Override
+          public PrefetchedHeader initialValue() {
+            return new PrefetchedHeader();
+          }
+        };
+
+    public FSReaderV2(FSDataInputStream istream, 
+        FSDataInputStream istreamNoFsChecksum, Algorithm compressAlgo,
+        long fileSize, int minorVersion, HFileSystem hfs, Path path) 
+      throws IOException {
+      super(istream, istreamNoFsChecksum, compressAlgo, fileSize, 
+            minorVersion, hfs, path);
+
+      if (hfs != null) {
+        // Check the configuration to determine whether hbase-level
+        // checksum verification is needed or not.
+        useHBaseChecksum = hfs.useHBaseChecksum();
+      } else {
+        // The configuration does not specify anything about hbase checksum
+        // validations. Set it to true here assuming that we will verify
+        // hbase checksums for all reads. For older files that do not have 
+        // stored checksums, this flag will be reset later.
+        useHBaseChecksum = true;
+      }
+
+      // for older versions, hbase did not store checksums.
+      if (getMinorVersion() < MINOR_VERSION_WITH_CHECKSUM) {
+        useHBaseChecksum = false;
+      }
+      this.useHBaseChecksumConfigured = useHBaseChecksum;
+    }
+
+    /**
+     * A constructor that reads files with the latest minor version.
+     * This is used by unit tests only.
+     */
+    FSReaderV2(FSDataInputStream istream, Algorithm compressAlgo,
+        long fileSize) throws IOException {
+      this(istream, istream, compressAlgo, fileSize,
+           HFileReaderV2.MAX_MINOR_VERSION, null, null);
+    }
+
+    /**
+     * Reads a version 2 block. Tries to do as little memory allocation as
+     * possible, using the provided on-disk size.
+     *
+     * @param offset the offset in the stream to read at
+     * @param onDiskSizeWithHeaderL the on-disk size of the block, including
+     *          the header, or -1 if unknown
+     * @param uncompressedSize the uncompressed size of the the block. Always
+     *          expected to be -1. This parameter is only used in version 1.
+     * @param pread whether to use a positional read
+     */
+    @Override
+    public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL,
+        int uncompressedSize, boolean pread) throws IOException {
+
+      // It is ok to get a reference to the stream here without any
+      // locks because it is marked final.
+      FSDataInputStream is = this.istreamNoFsChecksum;
+
+      // get a copy of the current state of whether to validate
+      // hbase checksums or not for this read call. This is not 
+      // thread-safe but the one constaint is that if we decide 
+      // to skip hbase checksum verification then we are 
+      // guaranteed to use hdfs checksum verification.
+      boolean doVerificationThruHBaseChecksum = this.useHBaseChecksum;
+      if (!doVerificationThruHBaseChecksum) {
+        is = this.istream;
+      }
+                     
+      HFileBlock blk = readBlockDataInternal(is, offset, 
+                         onDiskSizeWithHeaderL, 
+                         uncompressedSize, pread,
+                         doVerificationThruHBaseChecksum);
+      if (blk == null) {
+        HFile.LOG.warn("HBase checksum verification failed for file " +
+                       path + " at offset " +
+                       offset + " filesize " + fileSize +
+                       ". Retrying read with HDFS checksums turned on...");
+
+        if (!doVerificationThruHBaseChecksum) {
+          String msg = "HBase checksum verification failed for file " +
+                       path + " at offset " +
+                       offset + " filesize " + fileSize + 
+                       " but this cannot happen because doVerify is " +
+                       doVerificationThruHBaseChecksum;
+          HFile.LOG.warn(msg);
+          throw new IOException(msg); // cannot happen case here
+        }
+        HFile.checksumFailures.incrementAndGet(); // update metrics
+ 
+        // If we have a checksum failure, we fall back into a mode where
+        // the next few reads use HDFS level checksums. We aim to make the
+        // next CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD reads avoid
+        // hbase checksum verification, but since this value is set without
+        // holding any locks, it can so happen that we might actually do
+        // a few more than precisely this number.
+        this.checksumOffCount = CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD;
+        this.useHBaseChecksum = false;
+        doVerificationThruHBaseChecksum = false;
+        is = this.istream;
+        blk = readBlockDataInternal(is, offset, onDiskSizeWithHeaderL,
+                                    uncompressedSize, pread,
+                                    doVerificationThruHBaseChecksum);
+        if (blk != null) {
+          HFile.LOG.warn("HDFS checksum verification suceeded for file " +
+                         path + " at offset " +
+                         offset + " filesize " + fileSize);
+        }
+      } 
+      if (blk == null && !doVerificationThruHBaseChecksum) {
+        String msg = "readBlockData failed, possibly due to " +
+                     "checksum verification failed for file " + path +
+                     " at offset " + offset + " filesize " + fileSize;
+        HFile.LOG.warn(msg);
+        throw new IOException(msg);
+      }
+
+      // If there is a checksum mismatch earlier, then retry with 
+      // HBase checksums switched off and use HDFS checksum verification.
+      // This triggers HDFS to detect and fix corrupt replicas. The
+      // next checksumOffCount read requests will use HDFS checksums.
+      // The decrementing of this.checksumOffCount is not thread-safe,
+      // but it is harmless because eventually checksumOffCount will be
+      // a negative number.
+      if (!this.useHBaseChecksum && this.useHBaseChecksumConfigured) {
+        if (this.checksumOffCount-- < 0) {
+          this.useHBaseChecksum = true; // auto re-enable hbase checksums
+        }
+      }
+      return blk;
+    }
+
+    /**
+     * Reads a version 2 block. 
+     *
+     * @param offset the offset in the stream to read at
+     * @param onDiskSizeWithHeaderL the on-disk size of the block, including
+     *          the header, or -1 if unknown
+     * @param uncompressedSize the uncompressed size of the the block. Always
+     *          expected to be -1. This parameter is only used in version 1.
+     * @param pread whether to use a positional read
+     * @param verifyChecksum Whether to use HBase checksums. 
+     *        If HBase checksum is switched off, then use HDFS checksum.
+     * @return the HFileBlock or null if there is a HBase checksum mismatch
+     */
+    private HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, 
+        long onDiskSizeWithHeaderL,
+        int uncompressedSize, boolean pread, boolean verifyChecksum) 
+        throws IOException {
+      if (offset < 0) {
+        throw new IOException("Invalid offset=" + offset + " trying to read "
+            + "block (onDiskSize=" + onDiskSizeWithHeaderL
+            + ", uncompressedSize=" + uncompressedSize + ")");
+      }
+      if (uncompressedSize != -1) {
+        throw new IOException("Version 2 block reader API does not need " +
+            "the uncompressed size parameter");
+      }
+
+      if ((onDiskSizeWithHeaderL < hdrSize && onDiskSizeWithHeaderL != -1)
+          || onDiskSizeWithHeaderL >= Integer.MAX_VALUE) {
+        throw new IOException("Invalid onDisksize=" + onDiskSizeWithHeaderL
+            + ": expected to be at least " + hdrSize
+            + " and at most " + Integer.MAX_VALUE + ", or -1 (offset="
+            + offset + ", uncompressedSize=" + uncompressedSize + ")");
+      }
+
+      int onDiskSizeWithHeader = (int) onDiskSizeWithHeaderL;
+
+      HFileBlock b;
+      if (onDiskSizeWithHeader > 0) {
+        // We know the total on-disk size but not the uncompressed size. Read
+        // the entire block into memory, then parse the header and decompress
+        // from memory if using compression. This code path is used when
+        // doing a random read operation relying on the block index, as well as
+        // when the client knows the on-disk size from peeking into the next
+        // block's header (e.g. this block's header) when reading the previous
+        // block. This is the faster and more preferable case.
+
+        int onDiskSizeWithoutHeader = onDiskSizeWithHeader - hdrSize;
+        assert onDiskSizeWithoutHeader >= 0;
+
+        // See if we can avoid reading the header. This is desirable, because
+        // we will not incur a seek operation to seek back if we have already
+        // read this block's header as part of the previous read's look-ahead.
+        PrefetchedHeader prefetchedHeader = prefetchedHeaderForThread.get();
+        byte[] header = prefetchedHeader.offset == offset
+            ? prefetchedHeader.header : null;
+
+        // Size that we have to skip in case we have already read the header.
+        int preReadHeaderSize = header == null ? 0 : hdrSize;
+
+        if (compressAlgo == Compression.Algorithm.NONE) {
+          // Just read the whole thing. Allocate enough space to read the
+          // next block's header too.
+
+          ByteBuffer headerAndData = ByteBuffer.allocate(onDiskSizeWithHeader
+              + hdrSize);
+          headerAndData.limit(onDiskSizeWithHeader);
+
+          if (header != null) {
+            System.arraycopy(header, 0, headerAndData.array(), 0,
+                hdrSize);
+          }
+
+          int nextBlockOnDiskSizeWithHeader = readAtOffset(is,
+              headerAndData.array(), headerAndData.arrayOffset()
+                  + preReadHeaderSize, onDiskSizeWithHeader
+                  - preReadHeaderSize, true, offset + preReadHeaderSize,
+                  pread);
+
+          b = new HFileBlock(headerAndData, getMinorVersion());
+          b.assumeUncompressed();
+          b.validateOnDiskSizeWithoutHeader(onDiskSizeWithoutHeader);
+          b.nextBlockOnDiskSizeWithHeader = nextBlockOnDiskSizeWithHeader;
+          if (verifyChecksum &&
+              !validateBlockChecksum(b, headerAndData.array(), hdrSize)) {
+            return null;             // checksum mismatch
+          }
+          if (b.nextBlockOnDiskSizeWithHeader > 0)
+            setNextBlockHeader(offset, b);
+        } else {
+          // Allocate enough space to fit the next block's header too.
+          byte[] onDiskBlock = new byte[onDiskSizeWithHeader + hdrSize];
+
+          int nextBlockOnDiskSize = readAtOffset(is, onDiskBlock,
+              preReadHeaderSize, onDiskSizeWithHeader - preReadHeaderSize,
+              true, offset + preReadHeaderSize, pread);
+
+          if (header == null)
+            header = onDiskBlock;
+
+          try {
+            b = new HFileBlock(ByteBuffer.wrap(header, 0, hdrSize), 
+                               getMinorVersion());
+          } catch (IOException ex) {
+            // Seen in load testing. Provide comprehensive debug info.
+            throw new IOException("Failed to read compressed block at "
+                + offset + ", onDiskSizeWithoutHeader=" + onDiskSizeWithHeader
+                + ", preReadHeaderSize=" + preReadHeaderSize
+                + ", header.length=" + header.length + ", header bytes: "
+                + Bytes.toStringBinary(header, 0, hdrSize), ex);
+          }
+          b.validateOnDiskSizeWithoutHeader(onDiskSizeWithoutHeader);
+          b.nextBlockOnDiskSizeWithHeader = nextBlockOnDiskSize;
+          if (verifyChecksum && 
+              !validateBlockChecksum(b, onDiskBlock, hdrSize)) {
+            return null;             // checksum mismatch
+          }
+
+          DataInputStream dis = new DataInputStream(new ByteArrayInputStream(
+              onDiskBlock, hdrSize, onDiskSizeWithoutHeader));
+
+          // This will allocate a new buffer but keep header bytes.
+          b.allocateBuffer(b.nextBlockOnDiskSizeWithHeader > 0);
+
+          decompress(b.buf.array(), b.buf.arrayOffset() + hdrSize, dis,
+              b.uncompressedSizeWithoutHeader);
+
+          // Copy next block's header bytes into the new block if we have them.
+          if (nextBlockOnDiskSize > 0) {
+            System.arraycopy(onDiskBlock, onDiskSizeWithHeader, b.buf.array(),
+                b.buf.arrayOffset() + hdrSize
+                    + b.uncompressedSizeWithoutHeader + b.totalChecksumBytes(), 
+                hdrSize);
+
+            setNextBlockHeader(offset, b);
+          }
+        }
+
+      } else {
+        // We don't know the on-disk size. Read the header first, determine the
+        // on-disk size from it, and read the remaining data, thereby incurring
+        // two read operations. This might happen when we are doing the first
+        // read in a series of reads or a random read, and we don't have access
+        // to the block index. This is costly and should happen very rarely.
+
+        // Check if we have read this block's header as part of reading the
+        // previous block. If so, don't read the header again.
+        PrefetchedHeader prefetchedHeader = prefetchedHeaderForThread.get();
+        ByteBuffer headerBuf = prefetchedHeader.offset == offset ?
+            prefetchedHeader.buf : null;
+
+        if (headerBuf == null) {
+          // Unfortunately, we still have to do a separate read operation to
+          // read the header.
+          headerBuf = ByteBuffer.allocate(hdrSize);
+          readAtOffset(is, headerBuf.array(), headerBuf.arrayOffset(), hdrSize,
+              false, offset, pread);
+        }
+
+        b = new HFileBlock(headerBuf, getMinorVersion());
+
+        // This will also allocate enough room for the next block's header.
+        b.allocateBuffer(true);
+
+        if (compressAlgo == Compression.Algorithm.NONE) {
+
+          // Avoid creating bounded streams and using a "codec" that does
+          // nothing.
+          b.assumeUncompressed();
+          b.nextBlockOnDiskSizeWithHeader = readAtOffset(is, b.buf.array(),
+              b.buf.arrayOffset() + hdrSize,
+              b.uncompressedSizeWithoutHeader + b.totalChecksumBytes(), 
+              true, offset + hdrSize,
+              pread);
+          if (verifyChecksum && 
+              !validateBlockChecksum(b, b.buf.array(), hdrSize)) {
+            return null;             // checksum mismatch
+          }
+
+          if (b.nextBlockOnDiskSizeWithHeader > 0) {
+            setNextBlockHeader(offset, b);
+          }
+        } else {
+          // Allocate enough space for the block's header and compressed data.
+          byte[] compressedBytes = new byte[b.getOnDiskSizeWithHeader()
+              + hdrSize];
+
+          b.nextBlockOnDiskSizeWithHeader = readAtOffset(is, compressedBytes,
+              hdrSize, b.onDiskSizeWithoutHeader, true, offset
+                  + hdrSize, pread);
+          if (verifyChecksum &&
+              !validateBlockChecksum(b, compressedBytes, hdrSize)) {
+            return null;             // checksum mismatch
+          }
+          DataInputStream dis = new DataInputStream(new ByteArrayInputStream(
+              compressedBytes, hdrSize, b.onDiskSizeWithoutHeader));
+
+          decompress(b.buf.array(), b.buf.arrayOffset() + hdrSize, dis,
+              b.uncompressedSizeWithoutHeader);
+
+          if (b.nextBlockOnDiskSizeWithHeader > 0) {
+            // Copy the next block's header into the new block.
+            int nextHeaderOffset = b.buf.arrayOffset() + hdrSize
+                + b.uncompressedSizeWithoutHeader + b.totalChecksumBytes();
+            System.arraycopy(compressedBytes,
+                compressedBytes.length - hdrSize,
+                b.buf.array(),
+                nextHeaderOffset,
+                hdrSize);
+
+            setNextBlockHeader(offset, b);
+          }
+        }
+      }
+
+      b.includesMemstoreTS = includesMemstoreTS;
+      b.offset = offset;
+      return b;
+    }
+
+    private void setNextBlockHeader(long offset, HFileBlock b) {
+      PrefetchedHeader prefetchedHeader = prefetchedHeaderForThread.get();
+      prefetchedHeader.offset = offset + b.getOnDiskSizeWithHeader();
+      int nextHeaderOffset = b.buf.arrayOffset() + hdrSize
+          + b.uncompressedSizeWithoutHeader + b.totalChecksumBytes();
+      System.arraycopy(b.buf.array(), nextHeaderOffset,
+          prefetchedHeader.header, 0, hdrSize);
+    }
+
+    void setIncludesMemstoreTS(boolean enabled) {
+      includesMemstoreTS = enabled;
+    }
+
+    void setDataBlockEncoder(HFileDataBlockEncoder encoder) {
+      this.dataBlockEncoder = encoder;
+    }
+
+    /**
+     * Generates the checksum for the header as well as the data and
+     * then validates that it matches the value stored in the header.
+     * If there is a checksum mismatch, then return false. Otherwise
+     * return true.
+     */
+    protected boolean validateBlockChecksum(HFileBlock block, 
+      byte[] data, int hdrSize) throws IOException {
+      return ChecksumUtil.validateBlockChecksum(path, block,
+                                                data, hdrSize);
+    }
+  }
+
+  @Override
+  public int getSerializedLength() {
+    if (buf != null) {
+      return this.buf.limit() + HFileBlock.EXTRA_SERIALIZATION_SPACE;
+    }
+    return 0;
+  }
+
+  @Override
+  public void serialize(ByteBuffer destination) {
+    destination.put(this.buf.duplicate());
+    destination.putLong(this.offset);
+    destination.putInt(this.nextBlockOnDiskSizeWithHeader);
+    destination.rewind();
+  }
+
+  @Override
+  public CacheableDeserializer getDeserializer() {
+    return HFileBlock.blockDeserializer;
+  }
+
+  @Override
+  public boolean equals(Object comparison) {
+    if (this == comparison) {
+      return true;
+    }
+    if (comparison == null) {
+      return false;
+    }
+    if (comparison.getClass() != this.getClass()) {
+      return false;
+    }
+
+    HFileBlock castedComparison = (HFileBlock) comparison;
+
+    if (castedComparison.blockType != this.blockType) {
+      return false;
+    }
+    if (castedComparison.nextBlockOnDiskSizeWithHeader != this.nextBlockOnDiskSizeWithHeader) {
+      return false;
+    }
+    if (castedComparison.offset != this.offset) {
+      return false;
+    }
+    if (castedComparison.onDiskSizeWithoutHeader != this.onDiskSizeWithoutHeader) {
+      return false;
+    }
+    if (castedComparison.prevBlockOffset != this.prevBlockOffset) {
+      return false;
+    }
+    if (castedComparison.uncompressedSizeWithoutHeader != this.uncompressedSizeWithoutHeader) {
+      return false;
+    }
+    if (this.buf.compareTo(castedComparison.buf) != 0) {
+      return false;
+    }
+    if (this.buf.position() != castedComparison.buf.position()){
+      return false;
+    }
+    if (this.buf.limit() != castedComparison.buf.limit()){
+      return false;
+    }
+    return true;
+  }
+
+  public boolean doesIncludeMemstoreTS() {
+    return includesMemstoreTS;
+  }
+
+  public DataBlockEncoding getDataBlockEncoding() {
+    if (blockType == BlockType.ENCODED_DATA) {
+      return DataBlockEncoding.getEncodingById(getDataBlockEncodingId());
+    }
+    return DataBlockEncoding.NONE;
+  }
+
+  byte getChecksumType() {
+    return this.checksumType;
+  }
+
+  int getBytesPerChecksum() {
+    return this.bytesPerChecksum;
+  }
+
+  int getOnDiskDataSizeWithHeader() {
+    return this.onDiskDataSizeWithHeader;
+  }
+
+  int getMinorVersion() {
+    return this.minorVersion;
+  }
+
+  /** 
+   * Calcuate the number of bytes required to store all the checksums
+   * for this block. Each checksum value is a 4 byte integer.
+   */
+  int totalChecksumBytes() {
+    // If the hfile block has minorVersion 0, then there are no checksum
+    // data to validate. Similarly, a zero value in this.bytesPerChecksum
+    // indicates that cached blocks do not have checksum data because
+    // checksums were already validated when the block was read from disk.
+    if (minorVersion < MINOR_VERSION_WITH_CHECKSUM || this.bytesPerChecksum == 0) {
+      return 0;
+    }
+    return (int)ChecksumUtil.numBytes(onDiskDataSizeWithHeader, bytesPerChecksum);
+  }
+
+  /**
+   * Returns the size of this block header.
+   */
+  public int headerSize() {
+    return headerSize(this.minorVersion);
+  }
+
+  /**
+   * Maps a minor version to the size of the header.
+   */
+  static private int headerSize(int minorVersion) {
+    if (minorVersion < MINOR_VERSION_WITH_CHECKSUM) {
+      return HEADER_SIZE_NO_CHECKSUM;
+    }
+    return HEADER_SIZE_WITH_CHECKSUMS;
+  }
+
+  /**
+   * Return the appropriate DUMMY_HEADER_WITH_CHECKSUM for the minor version
+   */
+  public byte[] getDummyHeaderForVersion() {
+    return getDummyHeaderForVersion(minorVersion);
+  }
+
+  /**
+   * Return the appropriate DUMMY_HEADER_WITH_CHECKSUM for the minor version
+   */
+  static private byte[] getDummyHeaderForVersion(int minorVersion) {
+    if (minorVersion < MINOR_VERSION_WITH_CHECKSUM) {
+      return DUMMY_HEADER_NO_CHECKSUM;
+    }
+    return DUMMY_HEADER_WITH_CHECKSUM;
+  }
+
+  /**
+   * Convert the contents of the block header into a human readable string.
+   * This is mostly helpful for debugging. This assumes that the block
+   * has minor version > 0.
+   */
+  static String toStringHeader(ByteBuffer buf) throws IOException {
+    int offset = buf.arrayOffset();
+    byte[] b = buf.array();
+    long magic = Bytes.toLong(b, offset); 
+    BlockType bt = BlockType.read(buf);
+    offset += Bytes.SIZEOF_LONG;
+    int compressedBlockSizeNoHeader = Bytes.toInt(b, offset);
+    offset += Bytes.SIZEOF_INT;
+    int uncompressedBlockSizeNoHeader = Bytes.toInt(b, offset);
+    offset += Bytes.SIZEOF_INT;
+    long prevBlockOffset = Bytes.toLong(b, offset); 
+    offset += Bytes.SIZEOF_LONG;
+    byte cksumtype = b[offset];
+    offset += Bytes.SIZEOF_BYTE;
+    long bytesPerChecksum = Bytes.toInt(b, offset); 
+    offset += Bytes.SIZEOF_INT;
+    long onDiskDataSizeWithHeader = Bytes.toInt(b, offset); 
+    offset += Bytes.SIZEOF_INT;
+    return " Header dump: magic: " + magic +
+                   " blockType " + bt +
+                   " compressedBlockSizeNoHeader " + 
+                   compressedBlockSizeNoHeader +
+                   " uncompressedBlockSizeNoHeader " + 
+                   uncompressedBlockSizeNoHeader +
+                   " prevBlockOffset " + prevBlockOffset +
+                   " checksumType " + ChecksumType.codeToType(cksumtype) +
+                   " bytesPerChecksum " + bytesPerChecksum +
+                   " onDiskDataSizeWithHeader " + onDiskDataSizeWithHeader;
+  }
+}
+
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
new file mode 100644
index 0000000..8611b11
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
@@ -0,0 +1,1432 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.HFile.CachingBlockReader;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.hbase.util.CompoundBloomFilterWriter;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Provides functionality to write ({@link BlockIndexWriter}) and read
+ * ({@link BlockIndexReader}) single-level and multi-level block indexes.
+ *
+ * Examples of how to use the block index writer can be found in
+ * {@link CompoundBloomFilterWriter} and {@link HFileWriterV2}. Examples of how
+ * to use the reader can be found in {@link HFileReaderV2} and
+ * TestHFileBlockIndex.
+ */
+public class HFileBlockIndex {
+
+  private static final Log LOG = LogFactory.getLog(HFileBlockIndex.class);
+
+  static final int DEFAULT_MAX_CHUNK_SIZE = 128 * 1024;
+
+  /**
+   * The maximum size guideline for index blocks (both leaf, intermediate, and
+   * root). If not specified, DEFAULT_MAX_CHUNK_SIZE is used.
+   */
+  public static final String MAX_CHUNK_SIZE_KEY = "hfile.index.block.max.size";
+
+  /**
+   * The number of bytes stored in each "secondary index" entry in addition to
+   * key bytes in the non-root index block format. The first long is the file
+   * offset of the deeper-level block the entry points to, and the int that
+   * follows is that block's on-disk size without including header.
+   */
+  static final int SECONDARY_INDEX_ENTRY_OVERHEAD = Bytes.SIZEOF_INT
+      + Bytes.SIZEOF_LONG;
+
+  /**
+   * Error message when trying to use inline block API in single-level mode.
+   */
+  private static final String INLINE_BLOCKS_NOT_ALLOWED =
+      "Inline blocks are not allowed in the single-level-only mode";
+
+  /**
+   * The size of a meta-data record used for finding the mid-key in a
+   * multi-level index. Consists of the middle leaf-level index block offset
+   * (long), its on-disk size without header included (int), and the mid-key
+   * entry's zero-based index in that leaf index block.
+   */
+  private static final int MID_KEY_METADATA_SIZE = Bytes.SIZEOF_LONG +
+      2 * Bytes.SIZEOF_INT;
+
+  /**
+   * The reader will always hold the root level index in the memory. Index
+   * blocks at all other levels will be cached in the LRU cache in practice,
+   * although this API does not enforce that.
+   *
+   * All non-root (leaf and intermediate) index blocks contain what we call a
+   * "secondary index": an array of offsets to the entries within the block.
+   * This allows us to do binary search for the entry corresponding to the
+   * given key without having to deserialize the block.
+   */
+  public static class BlockIndexReader implements HeapSize {
+    /** Needed doing lookup on blocks. */
+    private final RawComparator comparator;
+
+    // Root-level data.
+    private byte[][] blockKeys;
+    private long[] blockOffsets;
+    private int[] blockDataSizes;
+    private int rootByteSize = 0;
+    private int rootCount = 0;
+
+    // Mid-key metadata.
+    private long midLeafBlockOffset = -1;
+    private int midLeafBlockOnDiskSize = -1;
+    private int midKeyEntry = -1;
+
+    /** Pre-computed mid-key */
+    private AtomicReference midKey = new AtomicReference();
+
+    /**
+     * The number of levels in the block index tree. One if there is only root
+     * level, two for root and leaf levels, etc.
+     */
+    private int searchTreeLevel;
+
+    /** A way to read {@link HFile} blocks at a given offset */
+    private CachingBlockReader cachingBlockReader;
+
+    public BlockIndexReader(final RawComparator c, final int treeLevel,
+        final CachingBlockReader cachingBlockReader) {
+      this(c, treeLevel);
+      this.cachingBlockReader = cachingBlockReader;
+    }
+
+    public BlockIndexReader(final RawComparator c, final int treeLevel)
+    {
+      comparator = c;
+      searchTreeLevel = treeLevel;
+    }
+
+    /**
+     * @return true if the block index is empty.
+     */
+    public boolean isEmpty() {
+      return blockKeys.length == 0;
+    }
+
+    /**
+     * Verifies that the block index is non-empty and throws an
+     * {@link IllegalStateException} otherwise.
+     */
+    public void ensureNonEmpty() {
+      if (blockKeys.length == 0) {
+        throw new IllegalStateException("Block index is empty or not loaded");
+      }
+    }
+
+    /**
+     * Return the data block which contains this key. This function will only
+     * be called when the HFile version is larger than 1.
+     *
+     * @param key the key we are looking for
+     * @param keyOffset the offset of the key in its byte array
+     * @param keyLength the length of the key
+     * @param currentBlock the current block, to avoid re-reading the same
+     *          block
+     * @return reader a basic way to load blocks
+     * @throws IOException
+     */
+    public HFileBlock seekToDataBlock(final byte[] key, int keyOffset,
+        int keyLength, HFileBlock currentBlock, boolean cacheBlocks,
+        boolean pread, boolean isCompaction)
+        throws IOException {
+      BlockWithScanInfo blockWithScanInfo = loadDataBlockWithScanInfo(key, keyOffset, keyLength,
+          currentBlock, cacheBlocks, pread, isCompaction);
+      if (blockWithScanInfo == null) {
+        return null;
+      } else {
+        return blockWithScanInfo.getHFileBlock();
+      }
+    }
+
+    /**
+     * Return the BlockWithScanInfo which contains the DataBlock with other scan info
+     * such as nextIndexedKey.
+     * This function will only be called when the HFile version is larger than 1.
+     *
+     * @param key the key we are looking for
+     * @param keyOffset the offset of the key in its byte array
+     * @param keyLength the length of the key
+     * @param currentBlock the current block, to avoid re-reading the same
+     *          block
+     * @param cacheBlocks
+     * @param pread
+     * @param isCompaction
+     * @return the BlockWithScanInfo which contains the DataBlock with other scan info
+     *         such as nextIndexedKey.
+     * @throws IOException
+     */
+    public BlockWithScanInfo loadDataBlockWithScanInfo(final byte[] key, int keyOffset,
+        int keyLength, HFileBlock currentBlock, boolean cacheBlocks,
+        boolean pread, boolean isCompaction)
+        throws IOException {
+      int rootLevelIndex = rootBlockContainingKey(key, keyOffset, keyLength);
+      if (rootLevelIndex < 0 || rootLevelIndex >= blockOffsets.length) {
+        return null;
+      }
+
+      // the next indexed key
+      byte[] nextIndexedKey = null;
+
+      // Read the next-level (intermediate or leaf) index block.
+      long currentOffset = blockOffsets[rootLevelIndex];
+      int currentOnDiskSize = blockDataSizes[rootLevelIndex];
+
+      if (rootLevelIndex < blockKeys.length - 1) {
+        nextIndexedKey = blockKeys[rootLevelIndex + 1];
+      } else {
+        nextIndexedKey = HConstants.NO_NEXT_INDEXED_KEY;
+      }
+
+      int lookupLevel = 1; // How many levels deep we are in our lookup.
+      int index = -1;
+
+      HFileBlock block;
+      while (true) {
+
+        if (currentBlock != null && currentBlock.getOffset() == currentOffset)
+        {
+          // Avoid reading the same block again, even with caching turned off.
+          // This is crucial for compaction-type workload which might have
+          // caching turned off. This is like a one-block cache inside the
+          // scanner.
+          block = currentBlock;
+        } else {
+          // Call HFile's caching block reader API. We always cache index
+          // blocks, otherwise we might get terrible performance.
+          boolean shouldCache = cacheBlocks || (lookupLevel < searchTreeLevel);
+          BlockType expectedBlockType;
+          if (lookupLevel < searchTreeLevel - 1) {
+            expectedBlockType = BlockType.INTERMEDIATE_INDEX;
+          } else if (lookupLevel == searchTreeLevel - 1) {
+            expectedBlockType = BlockType.LEAF_INDEX;
+          } else {
+            // this also accounts for ENCODED_DATA
+            expectedBlockType = BlockType.DATA;
+          }
+          block = cachingBlockReader.readBlock(currentOffset,
+              currentOnDiskSize, shouldCache, pread, isCompaction,
+              expectedBlockType);
+        }
+
+        if (block == null) {
+          throw new IOException("Failed to read block at offset " +
+              currentOffset + ", onDiskSize=" + currentOnDiskSize);
+        }
+
+        // Found a data block, break the loop and check our level in the tree.
+        if (block.getBlockType().equals(BlockType.DATA) ||
+            block.getBlockType().equals(BlockType.ENCODED_DATA)) {
+          break;
+        }
+
+        // Not a data block. This must be a leaf-level or intermediate-level
+        // index block. We don't allow going deeper than searchTreeLevel.
+        if (++lookupLevel > searchTreeLevel) {
+          throw new IOException("Search Tree Level overflow: lookupLevel="+
+              lookupLevel + ", searchTreeLevel=" + searchTreeLevel);
+        }
+
+        // Locate the entry corresponding to the given key in the non-root
+        // (leaf or intermediate-level) index block.
+        ByteBuffer buffer = block.getBufferWithoutHeader();
+        index = locateNonRootIndexEntry(buffer, key, keyOffset, keyLength, comparator);
+        if (index == -1) {
+          throw new IOException("The key "
+              + Bytes.toStringBinary(key, keyOffset, keyLength)
+              + " is before the" + " first key of the non-root index block "
+              + block);
+        }
+
+        currentOffset = buffer.getLong();
+        currentOnDiskSize = buffer.getInt();
+
+        // Only update next indexed key if there is a next indexed key in the current level
+        byte[] tmpNextIndexedKey = getNonRootIndexedKey(buffer, index + 1);
+        if (tmpNextIndexedKey != null) {
+          nextIndexedKey = tmpNextIndexedKey;
+        }
+      }
+
+      if (lookupLevel != searchTreeLevel) {
+        throw new IOException("Reached a data block at level " + lookupLevel +
+            " but the number of levels is " + searchTreeLevel);
+      }
+
+      // set the next indexed key for the current block.
+      BlockWithScanInfo blockWithScanInfo = new BlockWithScanInfo(block, nextIndexedKey);
+      return blockWithScanInfo;
+    }
+
+    /**
+     * An approximation to the {@link HFile}'s mid-key. Operates on block
+     * boundaries, and does not go inside blocks. In other words, returns the
+     * first key of the middle block of the file.
+     *
+     * @return the first key of the middle block
+     */
+    public byte[] midkey() throws IOException {
+      if (rootCount == 0)
+        throw new IOException("HFile empty");
+
+      byte[] midKey = this.midKey.get();
+      if (midKey != null)
+        return midKey;
+
+      if (midLeafBlockOffset >= 0) {
+        if (cachingBlockReader == null) {
+          throw new IOException("Have to read the middle leaf block but " +
+              "no block reader available");
+        }
+
+        // Caching, using pread, assuming this is not a compaction.
+        HFileBlock midLeafBlock = cachingBlockReader.readBlock(
+            midLeafBlockOffset, midLeafBlockOnDiskSize, true, true, false,
+            BlockType.LEAF_INDEX);
+
+        ByteBuffer b = midLeafBlock.getBufferWithoutHeader();
+        int numDataBlocks = b.getInt();
+        int keyRelOffset = b.getInt(Bytes.SIZEOF_INT * (midKeyEntry + 1));
+        int keyLen = b.getInt(Bytes.SIZEOF_INT * (midKeyEntry + 2)) -
+            keyRelOffset;
+        int keyOffset = b.arrayOffset() +
+            Bytes.SIZEOF_INT * (numDataBlocks + 2) + keyRelOffset +
+            SECONDARY_INDEX_ENTRY_OVERHEAD;
+        midKey = Arrays.copyOfRange(b.array(), keyOffset, keyOffset + keyLen);
+      } else {
+        // The middle of the root-level index.
+        midKey = blockKeys[rootCount / 2];
+      }
+
+      this.midKey.set(midKey);
+      return midKey;
+    }
+
+    /**
+     * @param i from 0 to {@link #getRootBlockCount() - 1}
+     */
+    public byte[] getRootBlockKey(int i) {
+      return blockKeys[i];
+    }
+
+    /**
+     * @param i from 0 to {@link #getRootBlockCount() - 1}
+     */
+    public long getRootBlockOffset(int i) {
+      return blockOffsets[i];
+    }
+
+    /**
+     * @param i zero-based index of a root-level block
+     * @return the on-disk size of the root-level block for version 2, or the
+     *         uncompressed size for version 1
+     */
+    public int getRootBlockDataSize(int i) {
+      return blockDataSizes[i];
+    }
+
+    /**
+     * @return the number of root-level blocks in this block index
+     */
+    public int getRootBlockCount() {
+      return rootCount;
+    }
+
+    /**
+     * Finds the root-level index block containing the given key.
+     *
+     * @param key
+     *          Key to find
+     * @return Offset of block containing key (between 0 and the
+     *         number of blocks - 1) or -1 if this file does not contain the
+     *         request.
+     */
+    public int rootBlockContainingKey(final byte[] key, int offset,
+        int length) {
+      int pos = Bytes.binarySearch(blockKeys, key, offset, length,
+          comparator);
+      // pos is between -(blockKeys.length + 1) to blockKeys.length - 1, see
+      // binarySearch's javadoc.
+
+      if (pos >= 0) {
+        // This means this is an exact match with an element of blockKeys.
+        assert pos < blockKeys.length;
+        return pos;
+      }
+
+      // Otherwise, pos = -(i + 1), where blockKeys[i - 1] < key < blockKeys[i],
+      // and i is in [0, blockKeys.length]. We are returning j = i - 1 such that
+      // blockKeys[j] <= key < blockKeys[j + 1]. In particular, j = -1 if
+      // key < blockKeys[0], meaning the file does not contain the given key.
+
+      int i = -pos - 1;
+      assert 0 <= i && i <= blockKeys.length;
+      return i - 1;
+    }
+
+    /**
+     * Adds a new entry in the root block index. Only used when reading.
+     *
+     * @param key Last key in the block
+     * @param offset file offset where the block is stored
+     * @param dataSize the uncompressed data size
+     */
+    private void add(final byte[] key, final long offset, final int dataSize) {
+      blockOffsets[rootCount] = offset;
+      blockKeys[rootCount] = key;
+      blockDataSizes[rootCount] = dataSize;
+
+      rootCount++;
+      rootByteSize += SECONDARY_INDEX_ENTRY_OVERHEAD + key.length;
+    }
+
+    /**
+     * The indexed key at the ith position in the nonRootIndex. The position starts at 0.
+     * @param nonRootIndex
+     * @param i the ith position
+     * @return The indexed key at the ith position in the nonRootIndex.
+     */
+    private byte[] getNonRootIndexedKey(ByteBuffer nonRootIndex, int i) {
+      int numEntries = nonRootIndex.getInt(0);
+      if (i < 0 || i >= numEntries) {
+        return null;
+      }
+
+      // Entries start after the number of entries and the secondary index.
+      // The secondary index takes numEntries + 1 ints.
+      int entriesOffset = Bytes.SIZEOF_INT * (numEntries + 2);
+      // Targetkey's offset relative to the end of secondary index
+      int targetKeyRelOffset = nonRootIndex.getInt(
+          Bytes.SIZEOF_INT * (i + 1));
+
+      // The offset of the target key in the blockIndex buffer
+      int targetKeyOffset = entriesOffset     // Skip secondary index
+          + targetKeyRelOffset               // Skip all entries until mid
+          + SECONDARY_INDEX_ENTRY_OVERHEAD;  // Skip offset and on-disk-size
+
+      // We subtract the two consecutive secondary index elements, which
+      // gives us the size of the whole (offset, onDiskSize, key) tuple. We
+      // then need to subtract the overhead of offset and onDiskSize.
+      int targetKeyLength = nonRootIndex.getInt(Bytes.SIZEOF_INT * (i + 2)) -
+        targetKeyRelOffset - SECONDARY_INDEX_ENTRY_OVERHEAD;
+
+      int from = nonRootIndex.arrayOffset() + targetKeyOffset;
+      int to = from + targetKeyLength;
+      return Arrays.copyOfRange(nonRootIndex.array(), from, to);
+    }
+
+    /**
+     * Performs a binary search over a non-root level index block. Utilizes the
+     * secondary index, which records the offsets of (offset, onDiskSize,
+     * firstKey) tuples of all entries.
+     *
+     * @param key the key we are searching for offsets to individual entries in
+     *          the blockIndex buffer
+     * @param keyOffset the offset of the key in its byte array
+     * @param keyLength the length of the key
+     * @param nonRootIndex the non-root index block buffer, starting with the
+     *          secondary index. The position is ignored.
+     * @return the index i in [0, numEntries - 1] such that keys[i] <= key <
+     *         keys[i + 1], if keys is the array of all keys being searched, or
+     *         -1 otherwise
+     * @throws IOException
+     */
+    static int binarySearchNonRootIndex(byte[] key, int keyOffset,
+        int keyLength, ByteBuffer nonRootIndex,
+        RawComparator comparator) {
+
+      int numEntries = nonRootIndex.getInt(0);
+      int low = 0;
+      int high = numEntries - 1;
+      int mid = 0;
+
+      // Entries start after the number of entries and the secondary index.
+      // The secondary index takes numEntries + 1 ints.
+      int entriesOffset = Bytes.SIZEOF_INT * (numEntries + 2);
+
+      // If we imagine that keys[-1] = -Infinity and
+      // keys[numEntries] = Infinity, then we are maintaining an invariant that
+      // keys[low - 1] < key < keys[high + 1] while narrowing down the range.
+
+      while (low <= high) {
+        mid = (low + high) >>> 1;
+
+        // Midkey's offset relative to the end of secondary index
+        int midKeyRelOffset = nonRootIndex.getInt(
+            Bytes.SIZEOF_INT * (mid + 1));
+
+        // The offset of the middle key in the blockIndex buffer
+        int midKeyOffset = entriesOffset       // Skip secondary index
+            + midKeyRelOffset                  // Skip all entries until mid
+            + SECONDARY_INDEX_ENTRY_OVERHEAD;  // Skip offset and on-disk-size
+
+        // We subtract the two consecutive secondary index elements, which
+        // gives us the size of the whole (offset, onDiskSize, key) tuple. We
+        // then need to subtract the overhead of offset and onDiskSize.
+        int midLength = nonRootIndex.getInt(Bytes.SIZEOF_INT * (mid + 2)) -
+            midKeyRelOffset - SECONDARY_INDEX_ENTRY_OVERHEAD;
+
+        // we have to compare in this order, because the comparator order
+        // has special logic when the 'left side' is a special key.
+        int cmp = comparator.compare(key, keyOffset, keyLength,
+            nonRootIndex.array(), nonRootIndex.arrayOffset() + midKeyOffset,
+            midLength);
+
+        // key lives above the midpoint
+        if (cmp > 0)
+          low = mid + 1; // Maintain the invariant that keys[low - 1] < key
+        // key lives below the midpoint
+        else if (cmp < 0)
+          high = mid - 1; // Maintain the invariant that key < keys[high + 1]
+        else
+          return mid; // exact match
+      }
+
+      // As per our invariant, keys[low - 1] < key < keys[high + 1], meaning
+      // that low - 1 < high + 1 and (low - high) <= 1. As per the loop break
+      // condition, low >= high + 1. Therefore, low = high + 1.
+
+      if (low != high + 1) {
+        throw new IllegalStateException("Binary search broken: low=" + low
+            + " " + "instead of " + (high + 1));
+      }
+
+      // OK, our invariant says that keys[low - 1] < key < keys[low]. We need to
+      // return i such that keys[i] <= key < keys[i + 1]. Therefore i = low - 1.
+      int i = low - 1;
+
+      // Some extra validation on the result.
+      if (i < -1 || i >= numEntries) {
+        throw new IllegalStateException("Binary search broken: result is " +
+            i + " but expected to be between -1 and (numEntries - 1) = " +
+            (numEntries - 1));
+      }
+
+      return i;
+    }
+
+    /**
+     * Search for one key using the secondary index in a non-root block. In case
+     * of success, positions the provided buffer at the entry of interest, where
+     * the file offset and the on-disk-size can be read.
+     *
+     * @param nonRootBlock a non-root block without header. Initial position
+     *          does not matter.
+     * @param key the byte array containing the key
+     * @param keyOffset the offset of the key in its byte array
+     * @param keyLength the length of the key
+     * @return the index position where the given key was found,
+     *         otherwise return -1 in the case the given key is before the first key.
+     *
+     */
+    static int locateNonRootIndexEntry(ByteBuffer nonRootBlock, byte[] key,
+        int keyOffset, int keyLength, RawComparator comparator) {
+      int entryIndex = binarySearchNonRootIndex(key, keyOffset, keyLength,
+          nonRootBlock, comparator);
+
+      if (entryIndex != -1) {
+        int numEntries = nonRootBlock.getInt(0);
+
+        // The end of secondary index and the beginning of entries themselves.
+        int entriesOffset = Bytes.SIZEOF_INT * (numEntries + 2);
+
+        // The offset of the entry we are interested in relative to the end of
+        // the secondary index.
+        int entryRelOffset = nonRootBlock.getInt(Bytes.SIZEOF_INT
+            * (1 + entryIndex));
+
+        nonRootBlock.position(entriesOffset + entryRelOffset);
+      }
+
+      return entryIndex;
+    }
+
+    /**
+     * Read in the root-level index from the given input stream. Must match
+     * what was written into the root level by
+     * {@link BlockIndexWriter#writeIndexBlocks(FSDataOutputStream)} at the
+     * offset that function returned.
+     *
+     * @param in the buffered input stream or wrapped byte input stream
+     * @param numEntries the number of root-level index entries
+     * @throws IOException
+     */
+    public void readRootIndex(DataInput in, final int numEntries)
+        throws IOException {
+      blockOffsets = new long[numEntries];
+      blockKeys = new byte[numEntries][];
+      blockDataSizes = new int[numEntries];
+
+      // If index size is zero, no index was written.
+      if (numEntries > 0) {
+        for (int i = 0; i < numEntries; ++i) {
+          long offset = in.readLong();
+          int dataSize = in.readInt();
+          byte[] key = Bytes.readByteArray(in);
+          add(key, offset, dataSize);
+        }
+      }
+    }
+    
+    /**
+     * Read in the root-level index from the given input stream. Must match
+     * what was written into the root level by
+     * {@link BlockIndexWriter#writeIndexBlocks(FSDataOutputStream)} at the
+     * offset that function returned.
+     *
+     * @param blk the HFile block
+     * @param numEntries the number of root-level index entries
+     * @return the buffered input stream or wrapped byte input stream
+     * @throws IOException
+     */
+    public DataInputStream readRootIndex(HFileBlock blk, final int numEntries) throws IOException {
+      DataInputStream in = blk.getByteStream();
+      readRootIndex(in, numEntries);
+      return in;
+    }
+
+    /**
+     * Read the root-level metadata of a multi-level block index. Based on
+     * {@link #readRootIndex(DataInput, int)}, but also reads metadata
+     * necessary to compute the mid-key in a multi-level index.
+     *
+     * @param blk the HFile block
+     * @param numEntries the number of root-level index entries
+     * @throws IOException
+     */
+    public void readMultiLevelIndexRoot(HFileBlock blk,
+        final int numEntries) throws IOException {
+      DataInputStream in = readRootIndex(blk, numEntries);
+      // after reading the root index the checksum bytes have to
+      // be subtracted to know if the mid key exists.
+      int checkSumBytes = blk.totalChecksumBytes();
+      if ((in.available() - checkSumBytes) < MID_KEY_METADATA_SIZE) {
+        // No mid-key metadata available.
+        return;
+      }
+      midLeafBlockOffset = in.readLong();
+      midLeafBlockOnDiskSize = in.readInt();
+      midKeyEntry = in.readInt();
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder();
+      sb.append("size=" + rootCount).append("\n");
+      for (int i = 0; i < rootCount; i++) {
+        sb.append("key=").append(KeyValue.keyToString(blockKeys[i]))
+            .append("\n  offset=").append(blockOffsets[i])
+            .append(", dataSize=" + blockDataSizes[i]).append("\n");
+      }
+      return sb.toString();
+    }
+
+    @Override
+    public long heapSize() {
+      long heapSize = ClassSize.align(6 * ClassSize.REFERENCE +
+          3 * Bytes.SIZEOF_INT + ClassSize.OBJECT);
+
+      // Mid-key metadata.
+      heapSize += MID_KEY_METADATA_SIZE;
+
+      // Calculating the size of blockKeys
+      if (blockKeys != null) {
+        // Adding array + references overhead
+        heapSize += ClassSize.align(ClassSize.ARRAY + blockKeys.length
+            * ClassSize.REFERENCE);
+
+        // Adding bytes
+        for (byte[] key : blockKeys) {
+          heapSize += ClassSize.align(ClassSize.ARRAY + key.length);
+        }
+      }
+
+      if (blockOffsets != null) {
+        heapSize += ClassSize.align(ClassSize.ARRAY + blockOffsets.length
+            * Bytes.SIZEOF_LONG);
+      }
+
+      if (blockDataSizes != null) {
+        heapSize += ClassSize.align(ClassSize.ARRAY + blockDataSizes.length
+            * Bytes.SIZEOF_INT);
+      }
+
+      return ClassSize.align(heapSize);
+    }
+
+  }
+
+  /**
+   * Writes the block index into the output stream. Generate the tree from
+   * bottom up. The leaf level is written to disk as a sequence of inline
+   * blocks, if it is larger than a certain number of bytes. If the leaf level
+   * is not large enough, we write all entries to the root level instead.
+   *
+   * After all leaf blocks have been written, we end up with an index
+   * referencing the resulting leaf index blocks. If that index is larger than
+   * the allowed root index size, the writer will break it up into
+   * reasonable-size intermediate-level index block chunks write those chunks
+   * out, and create another index referencing those chunks. This will be
+   * repeated until the remaining index is small enough to become the root
+   * index. However, in most practical cases we will only have leaf-level
+   * blocks and the root index, or just the root index.
+   */
+  public static class BlockIndexWriter extends SchemaConfigured
+      implements InlineBlockWriter {
+    /**
+     * While the index is being written, this represents the current block
+     * index referencing all leaf blocks, with one exception. If the file is
+     * being closed and there are not enough blocks to complete even a single
+     * leaf block, no leaf blocks get written and this contains the entire
+     * block index. After all levels of the index were written by
+     * {@link #writeIndexBlocks(FSDataOutputStream)}, this contains the final
+     * root-level index.
+     */
+    private BlockIndexChunk rootChunk = new BlockIndexChunk();
+
+    /**
+     * Current leaf-level chunk. New entries referencing data blocks get added
+     * to this chunk until it grows large enough to be written to disk.
+     */
+    private BlockIndexChunk curInlineChunk = new BlockIndexChunk();
+
+    /**
+     * The number of block index levels. This is one if there is only root
+     * level (even empty), two if there a leaf level and root level, and is
+     * higher if there are intermediate levels. This is only final after
+     * {@link #writeIndexBlocks(FSDataOutputStream)} has been called. The
+     * initial value accounts for the root level, and will be increased to two
+     * as soon as we find out there is a leaf-level in
+     * {@link #blockWritten(long, int)}.
+     */
+    private int numLevels = 1;
+
+    private HFileBlock.Writer blockWriter;
+    private byte[] firstKey = null;
+
+    /**
+     * The total number of leaf-level entries, i.e. entries referenced by
+     * leaf-level blocks. For the data block index this is equal to the number
+     * of data blocks.
+     */
+    private long totalNumEntries;
+
+    /** Total compressed size of all index blocks. */
+    private long totalBlockOnDiskSize;
+
+    /** Total uncompressed size of all index blocks. */
+    private long totalBlockUncompressedSize;
+
+    /** The maximum size guideline of all multi-level index blocks. */
+    private int maxChunkSize;
+
+    /** Whether we require this block index to always be single-level. */
+    private boolean singleLevelOnly;
+
+    /** Block cache, or null if cache-on-write is disabled */
+    private BlockCache blockCache;
+
+    /** Name to use for computing cache keys */
+    private String nameForCaching;
+
+    /** Creates a single-level block index writer */
+    public BlockIndexWriter() {
+      this(null, null, null);
+      singleLevelOnly = true;
+    }
+
+    /**
+     * Creates a multi-level block index writer.
+     *
+     * @param blockWriter the block writer to use to write index blocks
+     * @param blockCache if this is not null, index blocks will be cached
+     *    on write into this block cache.
+     */
+    public BlockIndexWriter(HFileBlock.Writer blockWriter,
+        BlockCache blockCache, String nameForCaching) {
+      if ((blockCache == null) != (nameForCaching == null)) {
+        throw new IllegalArgumentException("Block cache and file name for " +
+            "caching must be both specified or both null");
+      }
+
+      this.blockWriter = blockWriter;
+      this.blockCache = blockCache;
+      this.nameForCaching = nameForCaching;
+      this.maxChunkSize = HFileBlockIndex.DEFAULT_MAX_CHUNK_SIZE;
+    }
+
+    public void setMaxChunkSize(int maxChunkSize) {
+      if (maxChunkSize <= 0) {
+        throw new IllegalArgumentException("Invald maximum index block size");
+      }
+      this.maxChunkSize = maxChunkSize;
+    }
+
+    /**
+     * Writes the root level and intermediate levels of the block index into
+     * the output stream, generating the tree from bottom up. Assumes that the
+     * leaf level has been inline-written to the disk if there is enough data
+     * for more than one leaf block. We iterate by breaking the current level
+     * of the block index, starting with the index of all leaf-level blocks,
+     * into chunks small enough to be written to disk, and generate its parent
+     * level, until we end up with a level small enough to become the root
+     * level.
+     *
+     * If the leaf level is not large enough, there is no inline block index
+     * anymore, so we only write that level of block index to disk as the root
+     * level.
+     *
+     * @param out FSDataOutputStream
+     * @return position at which we entered the root-level index.
+     * @throws IOException
+     */
+    public long writeIndexBlocks(FSDataOutputStream out) throws IOException {
+      if (curInlineChunk != null && curInlineChunk.getNumEntries() != 0) {
+        throw new IOException("Trying to write a multi-level block index, " +
+            "but are " + curInlineChunk.getNumEntries() + " entries in the " +
+            "last inline chunk.");
+      }
+
+      // We need to get mid-key metadata before we create intermediate
+      // indexes and overwrite the root chunk.
+      byte[] midKeyMetadata = numLevels > 1 ? rootChunk.getMidKeyMetadata()
+          : null;
+
+      if (curInlineChunk != null) {
+        while (rootChunk.getRootSize() > maxChunkSize) {
+          rootChunk = writeIntermediateLevel(out, rootChunk);
+          numLevels += 1;
+        }
+      }
+
+      // write the root level
+      long rootLevelIndexPos = out.getPos();
+
+      {
+        DataOutput blockStream =
+            blockWriter.startWriting(BlockType.ROOT_INDEX);
+        rootChunk.writeRoot(blockStream);
+        if (midKeyMetadata != null)
+          blockStream.write(midKeyMetadata);
+        blockWriter.writeHeaderAndData(out);
+      }
+
+      // Add root index block size
+      totalBlockOnDiskSize += blockWriter.getOnDiskSizeWithoutHeader();
+      totalBlockUncompressedSize +=
+          blockWriter.getUncompressedSizeWithoutHeader();
+
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Wrote a " + numLevels + "-level index with root level at pos "
+          + rootLevelIndexPos + ", " + rootChunk.getNumEntries()
+          + " root-level entries, " + totalNumEntries + " total entries, "
+          + StringUtils.humanReadableInt(this.totalBlockOnDiskSize) +
+          " on-disk size, "
+          + StringUtils.humanReadableInt(totalBlockUncompressedSize) +
+          " total uncompressed size.");
+      }
+      return rootLevelIndexPos;
+    }
+
+    /**
+     * Writes the block index data as a single level only. Does not do any
+     * block framing.
+     *
+     * @param out the buffered output stream to write the index to. Typically a
+     *          stream writing into an {@link HFile} block.
+     * @param description a short description of the index being written. Used
+     *          in a log message.
+     * @throws IOException
+     */
+    public void writeSingleLevelIndex(DataOutput out, String description)
+        throws IOException {
+      expectNumLevels(1);
+
+      if (!singleLevelOnly)
+        throw new IOException("Single-level mode is turned off");
+
+      if (rootChunk.getNumEntries() > 0)
+        throw new IOException("Root-level entries already added in " +
+            "single-level mode");
+
+      rootChunk = curInlineChunk;
+      curInlineChunk = new BlockIndexChunk();
+
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Wrote a single-level " + description + " index with "
+          + rootChunk.getNumEntries() + " entries, " + rootChunk.getRootSize()
+          + " bytes");
+      }
+      rootChunk.writeRoot(out);
+    }
+
+    /**
+     * Split the current level of the block index into intermediate index
+     * blocks of permitted size and write those blocks to disk. Return the next
+     * level of the block index referencing those intermediate-level blocks.
+     *
+     * @param out
+     * @param currentLevel the current level of the block index, such as the a
+     *          chunk referencing all leaf-level index blocks
+     * @return the parent level block index, which becomes the root index after
+     *         a few (usually zero) iterations
+     * @throws IOException
+     */
+    private BlockIndexChunk writeIntermediateLevel(FSDataOutputStream out,
+        BlockIndexChunk currentLevel) throws IOException {
+      // Entries referencing intermediate-level blocks we are about to create.
+      BlockIndexChunk parent = new BlockIndexChunk();
+
+      // The current intermediate-level block index chunk.
+      BlockIndexChunk curChunk = new BlockIndexChunk();
+
+      for (int i = 0; i < currentLevel.getNumEntries(); ++i) {
+        curChunk.add(currentLevel.getBlockKey(i),
+            currentLevel.getBlockOffset(i), currentLevel.getOnDiskDataSize(i));
+
+        if (curChunk.getRootSize() >= maxChunkSize)
+          writeIntermediateBlock(out, parent, curChunk);
+      }
+
+      if (curChunk.getNumEntries() > 0) {
+        writeIntermediateBlock(out, parent, curChunk);
+      }
+
+      return parent;
+    }
+
+    private void writeIntermediateBlock(FSDataOutputStream out,
+        BlockIndexChunk parent, BlockIndexChunk curChunk) throws IOException {
+      long beginOffset = out.getPos();
+      DataOutputStream dos = blockWriter.startWriting(
+          BlockType.INTERMEDIATE_INDEX);
+      curChunk.writeNonRoot(dos);
+      byte[] curFirstKey = curChunk.getBlockKey(0);
+      blockWriter.writeHeaderAndData(out);
+
+      if (blockCache != null) {
+        HFileBlock blockForCaching = blockWriter.getBlockForCaching();
+        passSchemaMetricsTo(blockForCaching);
+        blockCache.cacheBlock(new BlockCacheKey(nameForCaching,
+            beginOffset, DataBlockEncoding.NONE, 
+            blockForCaching.getBlockType()), blockForCaching);
+      }
+
+      // Add intermediate index block size
+      totalBlockOnDiskSize += blockWriter.getOnDiskSizeWithoutHeader();
+      totalBlockUncompressedSize +=
+          blockWriter.getUncompressedSizeWithoutHeader();
+
+      // OFFSET is the beginning offset the chunk of block index entries.
+      // SIZE is the total byte size of the chunk of block index entries
+      // + the secondary index size
+      // FIRST_KEY is the first key in the chunk of block index
+      // entries.
+      parent.add(curFirstKey, beginOffset,
+          blockWriter.getOnDiskSizeWithHeader());
+
+      // clear current block index chunk
+      curChunk.clear();
+      curFirstKey = null;
+    }
+
+    /**
+     * @return how many block index entries there are in the root level
+     */
+    public final int getNumRootEntries() {
+      return rootChunk.getNumEntries();
+    }
+
+    /**
+     * @return the number of levels in this block index.
+     */
+    public int getNumLevels() {
+      return numLevels;
+    }
+
+    private void expectNumLevels(int expectedNumLevels) {
+      if (numLevels != expectedNumLevels) {
+        throw new IllegalStateException("Number of block index levels is "
+            + numLevels + "but is expected to be " + expectedNumLevels);
+      }
+    }
+
+    /**
+     * Whether there is an inline block ready to be written. In general, we
+     * write an leaf-level index block as an inline block as soon as its size
+     * as serialized in the non-root format reaches a certain threshold.
+     */
+    @Override
+    public boolean shouldWriteBlock(boolean closing) {
+      if (singleLevelOnly) {
+        throw new UnsupportedOperationException(INLINE_BLOCKS_NOT_ALLOWED);
+      }
+
+      if (curInlineChunk == null) {
+        throw new IllegalStateException("curInlineChunk is null; has shouldWriteBlock been " +
+            "called with closing=true and then called again?");
+      }
+
+      if (curInlineChunk.getNumEntries() == 0) {
+        return false;
+      }
+
+      // We do have some entries in the current inline chunk.
+      if (closing) {
+        if (rootChunk.getNumEntries() == 0) {
+          // We did not add any leaf-level blocks yet. Instead of creating a
+          // leaf level with one block, move these entries to the root level.
+
+          expectNumLevels(1);
+          rootChunk = curInlineChunk;
+          curInlineChunk = null;  // Disallow adding any more index entries.
+          return false;
+        }
+
+        return true;
+      } else {
+        return curInlineChunk.getNonRootSize() >= maxChunkSize;
+      }
+    }
+
+    /**
+     * Write out the current inline index block. Inline blocks are non-root
+     * blocks, so the non-root index format is used.
+     *
+     * @param out
+     */
+    @Override
+    public void writeInlineBlock(DataOutput out) throws IOException {
+      if (singleLevelOnly)
+        throw new UnsupportedOperationException(INLINE_BLOCKS_NOT_ALLOWED);
+
+      // Write the inline block index to the output stream in the non-root
+      // index block format.
+      curInlineChunk.writeNonRoot(out);
+
+      // Save the first key of the inline block so that we can add it to the
+      // parent-level index.
+      firstKey = curInlineChunk.getBlockKey(0);
+
+      // Start a new inline index block
+      curInlineChunk.clear();
+    }
+
+    /**
+     * Called after an inline block has been written so that we can add an
+     * entry referring to that block to the parent-level index.
+     */
+    @Override
+    public void blockWritten(long offset, int onDiskSize, int uncompressedSize)
+    {
+      // Add leaf index block size
+      totalBlockOnDiskSize += onDiskSize;
+      totalBlockUncompressedSize += uncompressedSize;
+
+      if (singleLevelOnly)
+        throw new UnsupportedOperationException(INLINE_BLOCKS_NOT_ALLOWED);
+
+      if (firstKey == null) {
+        throw new IllegalStateException("Trying to add second-level index " +
+            "entry with offset=" + offset + " and onDiskSize=" + onDiskSize +
+            "but the first key was not set in writeInlineBlock");
+      }
+
+      if (rootChunk.getNumEntries() == 0) {
+        // We are writing the first leaf block, so increase index level.
+        expectNumLevels(1);
+        numLevels = 2;
+      }
+
+      // Add another entry to the second-level index. Include the number of
+      // entries in all previous leaf-level chunks for mid-key calculation.
+      rootChunk.add(firstKey, offset, onDiskSize, totalNumEntries);
+      firstKey = null;
+    }
+
+    @Override
+    public BlockType getInlineBlockType() {
+      return BlockType.LEAF_INDEX;
+    }
+
+    /**
+     * Add one index entry to the current leaf-level block. When the leaf-level
+     * block gets large enough, it will be flushed to disk as an inline block.
+     *
+     * @param firstKey the first key of the data block
+     * @param blockOffset the offset of the data block
+     * @param blockDataSize the on-disk size of the data block ({@link HFile}
+     *          format version 2), or the uncompressed size of the data block (
+     *          {@link HFile} format version 1).
+     */
+    public void addEntry(byte[] firstKey, long blockOffset, int blockDataSize)
+    {
+      curInlineChunk.add(firstKey, blockOffset, blockDataSize);
+      ++totalNumEntries;
+    }
+
+    /**
+     * @throws IOException if we happened to write a multi-level index.
+     */
+    public void ensureSingleLevel() throws IOException {
+      if (numLevels > 1) {
+        throw new IOException ("Wrote a " + numLevels + "-level index with " +
+            rootChunk.getNumEntries() + " root-level entries, but " +
+            "this is expected to be a single-level block index.");
+      }
+    }
+
+    /**
+     * @return true if we are using cache-on-write. This is configured by the
+     *         caller of the constructor by either passing a valid block cache
+     *         or null.
+     */
+    @Override
+    public boolean cacheOnWrite() {
+      return blockCache != null;
+    }
+
+    /**
+     * The total uncompressed size of the root index block, intermediate-level
+     * index blocks, and leaf-level index blocks.
+     *
+     * @return the total uncompressed size of all index blocks
+     */
+    public long getTotalUncompressedSize() {
+      return totalBlockUncompressedSize;
+    }
+
+  }
+
+  /**
+   * A single chunk of the block index in the process of writing. The data in
+   * this chunk can become a leaf-level, intermediate-level, or root index
+   * block.
+   */
+  static class BlockIndexChunk {
+
+    /** First keys of the key range corresponding to each index entry. */
+    private final List blockKeys = new ArrayList();
+
+    /** Block offset in backing stream. */
+    private final List blockOffsets = new ArrayList();
+
+    /** On-disk data sizes of lower-level data or index blocks. */
+    private final List onDiskDataSizes = new ArrayList();
+
+    /**
+     * The cumulative number of sub-entries, i.e. entries on deeper-level block
+     * index entries. numSubEntriesAt[i] is the number of sub-entries in the
+     * blocks corresponding to this chunk's entries #0 through #i inclusively.
+     */
+    private final List numSubEntriesAt = new ArrayList();
+
+    /**
+     * The offset of the next entry to be added, relative to the end of the
+     * "secondary index" in the "non-root" format representation of this index
+     * chunk. This is the next value to be added to the secondary index.
+     */
+    private int curTotalNonRootEntrySize = 0;
+
+    /**
+     * The accumulated size of this chunk if stored in the root index format.
+     */
+    private int curTotalRootSize = 0;
+
+    /**
+     * The "secondary index" used for binary search over variable-length
+     * records in a "non-root" format block. These offsets are relative to the
+     * end of this secondary index.
+     */
+    private final List secondaryIndexOffsetMarks =
+        new ArrayList();
+
+    /**
+     * Adds a new entry to this block index chunk.
+     *
+     * @param firstKey the first key in the block pointed to by this entry
+     * @param blockOffset the offset of the next-level block pointed to by this
+     *          entry
+     * @param onDiskDataSize the on-disk data of the block pointed to by this
+     *          entry, including header size
+     * @param curTotalNumSubEntries if this chunk is the root index chunk under
+     *          construction, this specifies the current total number of
+     *          sub-entries in all leaf-level chunks, including the one
+     *          corresponding to the second-level entry being added.
+     */
+    void add(byte[] firstKey, long blockOffset, int onDiskDataSize,
+        long curTotalNumSubEntries) {
+      // Record the offset for the secondary index
+      secondaryIndexOffsetMarks.add(curTotalNonRootEntrySize);
+      curTotalNonRootEntrySize += SECONDARY_INDEX_ENTRY_OVERHEAD
+          + firstKey.length;
+
+      curTotalRootSize += Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT
+          + WritableUtils.getVIntSize(firstKey.length) + firstKey.length;
+
+      blockKeys.add(firstKey);
+      blockOffsets.add(blockOffset);
+      onDiskDataSizes.add(onDiskDataSize);
+
+      if (curTotalNumSubEntries != -1) {
+        numSubEntriesAt.add(curTotalNumSubEntries);
+
+        // Make sure the parallel arrays are in sync.
+        if (numSubEntriesAt.size() != blockKeys.size()) {
+          throw new IllegalStateException("Only have key/value count " +
+              "stats for " + numSubEntriesAt.size() + " block index " +
+              "entries out of " + blockKeys.size());
+        }
+      }
+    }
+
+    /**
+     * The same as {@link #add(byte[], long, int, long)} but does not take the
+     * key/value into account. Used for single-level indexes.
+     *
+     * @see {@link #add(byte[], long, int, long)}
+     */
+    public void add(byte[] firstKey, long blockOffset, int onDiskDataSize) {
+      add(firstKey, blockOffset, onDiskDataSize, -1);
+    }
+
+    public void clear() {
+      blockKeys.clear();
+      blockOffsets.clear();
+      onDiskDataSizes.clear();
+      secondaryIndexOffsetMarks.clear();
+      numSubEntriesAt.clear();
+      curTotalNonRootEntrySize = 0;
+      curTotalRootSize = 0;
+    }
+
+    /**
+     * Finds the entry corresponding to the deeper-level index block containing
+     * the given deeper-level entry (a "sub-entry"), assuming a global 0-based
+     * ordering of sub-entries.
+     *
+     * 
      
+     *  Implementation note.  We are looking for i such that
+     * numSubEntriesAt[i - 1] <= k < numSubEntriesAt[i], because a deeper-level
+     * block #i (0-based) contains sub-entries # numSubEntriesAt[i - 1]'th
+     * through numSubEntriesAt[i] - 1, assuming a global 0-based ordering of
+     * sub-entries. i is by definition the insertion point of k in
+     * numSubEntriesAt.
+     *
+     * @param k sub-entry index, from 0 to the total number sub-entries - 1
+     * @return the 0-based index of the entry corresponding to the given
+     *         sub-entry
+     */
+    public int getEntryBySubEntry(long k) {
+      // We define mid-key as the key corresponding to k'th sub-entry
+      // (0-based).
+
+      int i = Collections.binarySearch(numSubEntriesAt, k);
+
+      // Exact match: cumulativeWeight[i] = k. This means chunks #0 through
+      // #i contain exactly k sub-entries, and the sub-entry #k (0-based)
+      // is in the (i + 1)'th chunk.
+      if (i >= 0)
+        return i + 1;
+
+      // Inexact match. Return the insertion point.
+      return -i - 1;
+    }
+
+    /**
+     * Used when writing the root block index of a multi-level block index.
+     * Serializes additional information allowing to efficiently identify the
+     * mid-key.
+     *
+     * @return a few serialized fields for finding the mid-key
+     * @throws IOException if could not create metadata for computing mid-key
+     */
+    public byte[] getMidKeyMetadata() throws IOException {
+      ByteArrayOutputStream baos = new ByteArrayOutputStream(
+          MID_KEY_METADATA_SIZE);
+      DataOutputStream baosDos = new DataOutputStream(baos);
+      long totalNumSubEntries = numSubEntriesAt.get(blockKeys.size() - 1);
+      if (totalNumSubEntries == 0) {
+        throw new IOException("No leaf-level entries, mid-key unavailable");
+      }
+      long midKeySubEntry = (totalNumSubEntries - 1) / 2;
+      int midKeyEntry = getEntryBySubEntry(midKeySubEntry);
+
+      baosDos.writeLong(blockOffsets.get(midKeyEntry));
+      baosDos.writeInt(onDiskDataSizes.get(midKeyEntry));
+
+      long numSubEntriesBefore = midKeyEntry > 0
+          ? numSubEntriesAt.get(midKeyEntry - 1) : 0;
+      long subEntryWithinEntry = midKeySubEntry - numSubEntriesBefore;
+      if (subEntryWithinEntry < 0 || subEntryWithinEntry > Integer.MAX_VALUE)
+      {
+        throw new IOException("Could not identify mid-key index within the "
+            + "leaf-level block containing mid-key: out of range ("
+            + subEntryWithinEntry + ", numSubEntriesBefore="
+            + numSubEntriesBefore + ", midKeySubEntry=" + midKeySubEntry
+            + ")");
+      }
+
+      baosDos.writeInt((int) subEntryWithinEntry);
+
+      if (baosDos.size() != MID_KEY_METADATA_SIZE) {
+        throw new IOException("Could not write mid-key metadata: size=" +
+            baosDos.size() + ", correct size: " + MID_KEY_METADATA_SIZE);
+      }
+
+      // Close just to be good citizens, although this has no effect.
+      baos.close();
+
+      return baos.toByteArray();
+    }
+
+    /**
+     * Writes the block index chunk in the non-root index block format. This
+     * format contains the number of entries, an index of integer offsets
+     * for quick binary search on variable-length records, and tuples of
+     * block offset, on-disk block size, and the first key for each entry.
+     *
+     * @param out
+     * @throws IOException
+     */
+    void writeNonRoot(DataOutput out) throws IOException {
+      // The number of entries in the block.
+      out.writeInt(blockKeys.size());
+
+      if (secondaryIndexOffsetMarks.size() != blockKeys.size()) {
+        throw new IOException("Corrupted block index chunk writer: " +
+            blockKeys.size() + " entries but " +
+            secondaryIndexOffsetMarks.size() + " secondary index items");
+      }
+
+      // For each entry, write a "secondary index" of relative offsets to the
+      // entries from the end of the secondary index. This works, because at
+      // read time we read the number of entries and know where the secondary
+      // index ends.
+      for (int currentSecondaryIndex : secondaryIndexOffsetMarks)
+        out.writeInt(currentSecondaryIndex);
+
+      // We include one other element in the secondary index to calculate the
+      // size of each entry more easily by subtracting secondary index elements.
+      out.writeInt(curTotalNonRootEntrySize);
+
+      for (int i = 0; i < blockKeys.size(); ++i) {
+        out.writeLong(blockOffsets.get(i));
+        out.writeInt(onDiskDataSizes.get(i));
+        out.write(blockKeys.get(i));
+      }
+    }
+
+    /**
+     * @return the size of this chunk if stored in the non-root index block
+     *         format
+     */
+    int getNonRootSize() {
+      return Bytes.SIZEOF_INT                          // Number of entries
+          + Bytes.SIZEOF_INT * (blockKeys.size() + 1)  // Secondary index
+          + curTotalNonRootEntrySize;                  // All entries
+    }
+
+    /**
+     * Writes this chunk into the given output stream in the root block index
+     * format. This format is similar to the {@link HFile} version 1 block
+     * index format, except that we store on-disk size of the block instead of
+     * its uncompressed size.
+     *
+     * @param out the data output stream to write the block index to. Typically
+     *          a stream writing into an {@link HFile} block.
+     * @throws IOException
+     */
+    void writeRoot(DataOutput out) throws IOException {
+      for (int i = 0; i < blockKeys.size(); ++i) {
+        out.writeLong(blockOffsets.get(i));
+        out.writeInt(onDiskDataSizes.get(i));
+        Bytes.writeByteArray(out, blockKeys.get(i));
+      }
+    }
+
+    /**
+     * @return the size of this chunk if stored in the root index block format
+     */
+    int getRootSize() {
+      return curTotalRootSize;
+    }
+
+    /**
+     * @return the number of entries in this block index chunk
+     */
+    public int getNumEntries() {
+      return blockKeys.size();
+    }
+
+    public byte[] getBlockKey(int i) {
+      return blockKeys.get(i);
+    }
+
+    public long getBlockOffset(int i) {
+      return blockOffsets.get(i);
+    }
+
+    public int getOnDiskDataSize(int i) {
+      return onDiskDataSizes.get(i);
+    }
+
+    public long getCumulativeNumKV(int i) {
+      if (i < 0)
+        return 0;
+      return numSubEntriesAt.get(i);
+    }
+
+  }
+
+  public static int getMaxChunkSize(Configuration conf) {
+    return conf.getInt(MAX_CHUNK_SIZE_KEY, DEFAULT_MAX_CHUNK_SIZE);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
new file mode 100644
index 0000000..7be4b1b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * Controls what kind of data block encoding is used. If data block encoding is
+ * not set or the given block is not a data block (encoded or not), methods
+ * should just return the unmodified block.
+ */
+public interface HFileDataBlockEncoder {
+  /** Type of encoding used for data blocks in HFile. Stored in file info. */
+  public static final byte[] DATA_BLOCK_ENCODING = Bytes.toBytes("DATA_BLOCK_ENCODING");
+  
+  /**
+   * Converts a block from the on-disk format to the in-cache format. Called in
+   * the following cases:
+   * 
        
+   * 
      • After an encoded or unencoded data block is read from disk, but before
+   * it is put into the cache.
        • 
+   * 
      • To convert brand-new blocks to the in-cache format when doing
+   * cache-on-write.
        • 
+   * 
      
+   * @param block a block in an on-disk format (read from HFile or freshly
+   *          generated).
+   * @return non null block which is coded according to the settings.
+   */
+  public HFileBlock diskToCacheFormat(HFileBlock block,
+      boolean isCompaction);
+
+  /**
+   * Should be called before an encoded or unencoded data block is written to
+   * disk.
+   * @param in KeyValues next to each other
+   * @param dummyHeader A dummy header to be written as a placeholder
+   * @return a non-null on-heap buffer containing the contents of the
+   *         HFileBlock with unfilled header and block type
+   */
+  public Pair beforeWriteToDisk(
+      ByteBuffer in, boolean includesMemstoreTS, byte[] dummyHeader);
+
+  /**
+   * Decides whether we should use a scanner over encoded blocks.
+   * @param isCompaction whether we are in a compaction.
+   * @return Whether to use encoded scanner.
+   */
+  public boolean useEncodedScanner(boolean isCompaction);
+
+  /**
+   * Save metadata in HFile which will be written to disk
+   * @param writer writer for a given HFile
+   * @exception IOException on disk problems
+   */
+  public void saveMetadata(HFile.Writer writer) throws IOException;
+
+  /** @return the on-disk data block encoding */
+  public DataBlockEncoding getEncodingOnDisk();
+
+  /** @return the preferred in-cache data block encoding for normal reads */
+  public DataBlockEncoding getEncodingInCache();
+
+  /**
+   * @return the effective in-cache data block encoding, taking into account
+   *         whether we are doing a compaction.
+   */
+  public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction);
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
new file mode 100644
index 0000000..c7b2272
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Do different kinds of data block encoding according to column family
+ * options.
+ */
+public class HFileDataBlockEncoderImpl implements HFileDataBlockEncoder {
+  private final DataBlockEncoding onDisk;
+  private final DataBlockEncoding inCache;
+
+  public HFileDataBlockEncoderImpl(DataBlockEncoding encoding) {
+    this(encoding, encoding);
+  }
+
+  /**
+   * Do data block encoding with specified options.
+   * @param onDisk What kind of data block encoding will be used before writing
+   *          HFileBlock to disk. This must be either the same as inCache or
+   *          {@link DataBlockEncoding#NONE}.
+   * @param inCache What kind of data block encoding will be used in block
+   *          cache.
+   */
+  public HFileDataBlockEncoderImpl(DataBlockEncoding onDisk,
+      DataBlockEncoding inCache) {
+    this.onDisk = onDisk != null ?
+        onDisk : DataBlockEncoding.NONE;
+    this.inCache = inCache != null ?
+        inCache : DataBlockEncoding.NONE;
+    Preconditions.checkArgument(onDisk == DataBlockEncoding.NONE ||
+        onDisk == inCache, "on-disk encoding (" + onDisk + ") must be " +
+        "either the same as in-cache encoding (" + inCache + ") or " +
+        DataBlockEncoding.NONE);
+  }
+
+  public static HFileDataBlockEncoder createFromFileInfo(
+      FileInfo fileInfo, DataBlockEncoding preferredEncodingInCache)
+      throws IOException {
+    
+    boolean hasPreferredCacheEncoding = preferredEncodingInCache != null
+        && preferredEncodingInCache != DataBlockEncoding.NONE;
+
+    byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING);
+    if (dataBlockEncodingType == null && !hasPreferredCacheEncoding) {
+      return NoOpDataBlockEncoder.INSTANCE;
+    }
+
+    DataBlockEncoding onDisk;
+    if (dataBlockEncodingType == null) {
+      onDisk = DataBlockEncoding.NONE;
+    }else {
+      String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType);
+      try {
+        onDisk = DataBlockEncoding.valueOf(dataBlockEncodingStr);
+      } catch (IllegalArgumentException ex) {
+        throw new IOException("Invalid data block encoding type in file info: "
+            + dataBlockEncodingStr, ex);
+      }
+    }
+
+    DataBlockEncoding inCache;
+    if (onDisk == DataBlockEncoding.NONE) {
+      // This is an "in-cache-only" encoding or fully-unencoded scenario.
+      // Either way, we use the given encoding (possibly NONE) specified by
+      // the column family in cache.
+      inCache = preferredEncodingInCache;
+    } else {
+      // Leave blocks in cache encoded the same way as they are on disk.
+      // If we switch encoding type for the CF or the in-cache-only encoding
+      // flag, old files will keep their encoding both on disk and in cache,
+      // but new files will be generated with the new encoding.
+      inCache = onDisk;
+    }
+    return new HFileDataBlockEncoderImpl(onDisk, inCache);
+  }
+
+  @Override
+  public void saveMetadata(HFile.Writer writer) throws IOException {
+    writer.appendFileInfo(DATA_BLOCK_ENCODING, onDisk.getNameInBytes());
+  }
+
+  @Override
+  public DataBlockEncoding getEncodingOnDisk() {
+    return onDisk;
+  }
+
+  @Override
+  public DataBlockEncoding getEncodingInCache() {
+    return inCache;
+  }
+
+  @Override
+  public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
+    if (!useEncodedScanner(isCompaction)) {
+      return DataBlockEncoding.NONE;
+    }
+    return inCache;
+  }
+
+  @Override
+  public HFileBlock diskToCacheFormat(HFileBlock block, boolean isCompaction) {
+    if (block.getBlockType() == BlockType.DATA) {
+      if (!useEncodedScanner(isCompaction)) {
+        // Unencoded block, and we don't want to encode in cache.
+        return block;
+      }
+      // Encode the unencoded block with the in-cache encoding.
+      return encodeDataBlock(block, inCache, block.doesIncludeMemstoreTS());
+    }
+
+    if (block.getBlockType() == BlockType.ENCODED_DATA) {
+      if (block.getDataBlockEncodingId() == onDisk.getId()) {
+        // The block is already in the desired in-cache encoding.
+        return block;
+      }
+      // We don't want to re-encode a block in a different encoding. The HFile
+      // reader should have been instantiated in such a way that we would not
+      // have to do this.
+      throw new AssertionError("Expected on-disk data block encoding " +
+          onDisk + ", got " + block.getDataBlockEncoding());
+    }
+    return block;
+  }
+
+  /**
+   * Precondition: a non-encoded buffer.
+   * Postcondition: on-disk encoding.
+   */
+  @Override
+  public Pair beforeWriteToDisk(ByteBuffer in,
+      boolean includesMemstoreTS, byte[] dummyHeader) {
+    if (onDisk == DataBlockEncoding.NONE) {
+      // there is no need to encode the block before writing it to disk
+      return new Pair(in, BlockType.DATA);
+    }
+
+    ByteBuffer encodedBuffer = encodeBufferToHFileBlockBuffer(in,
+        onDisk, includesMemstoreTS, dummyHeader);
+    return new Pair(encodedBuffer,
+        BlockType.ENCODED_DATA);
+  }
+
+  @Override
+  public boolean useEncodedScanner(boolean isCompaction) {
+    if (isCompaction && onDisk == DataBlockEncoding.NONE) {
+      return false;
+    }
+    return inCache != DataBlockEncoding.NONE;
+  }
+
+  private ByteBuffer encodeBufferToHFileBlockBuffer(ByteBuffer in,
+      DataBlockEncoding algo, boolean includesMemstoreTS,
+      byte[] dummyHeader) {
+    ByteArrayOutputStream encodedStream = new ByteArrayOutputStream();
+    DataOutputStream dataOut = new DataOutputStream(encodedStream);
+    DataBlockEncoder encoder = algo.getEncoder();
+    try {
+      encodedStream.write(dummyHeader);
+      algo.writeIdInBytes(dataOut);
+      encoder.compressKeyValues(dataOut, in,
+          includesMemstoreTS);
+    } catch (IOException e) {
+      throw new RuntimeException(String.format("Bug in data block encoder " +
+          "'%s', it probably requested too much data", algo.toString()), e);
+    }
+    return ByteBuffer.wrap(encodedStream.toByteArray());
+  }
+
+  private HFileBlock encodeDataBlock(HFileBlock block,
+      DataBlockEncoding algo, boolean includesMemstoreTS) {
+    ByteBuffer compressedBuffer = encodeBufferToHFileBlockBuffer(
+        block.getBufferWithoutHeader(), algo, includesMemstoreTS,
+        block.getDummyHeaderForVersion());
+    int sizeWithoutHeader = compressedBuffer.limit() - block.headerSize();
+    HFileBlock encodedBlock = new HFileBlock(BlockType.ENCODED_DATA,
+        block.getOnDiskSizeWithoutHeader(),
+        sizeWithoutHeader, block.getPrevBlockOffset(),
+        compressedBuffer, HFileBlock.FILL_HEADER, block.getOffset(),
+        includesMemstoreTS, block.getMinorVersion(),
+        block.getBytesPerChecksum(), block.getChecksumType(),
+        block.getOnDiskDataSizeWithHeader());
+    block.passSchemaMetricsTo(encodedBlock);
+    return encodedBlock;
+  }
+
+  @Override
+  public String toString() {
+    return getClass().getSimpleName() + "(onDisk=" + onDisk + ", inCache=" +
+        inCache + ")";
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
new file mode 100644
index 0000000..b68e2af
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
@@ -0,0 +1,451 @@
+
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
+import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.util.BloomFilter;
+import org.apache.hadoop.hbase.util.BloomFilterFactory;
+import org.apache.hadoop.hbase.util.ByteBloomFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Writables;
+
+/**
+ * Implements pretty-printing functionality for {@link HFile}s.
+ */
+public class HFilePrettyPrinter {
+
+  private static final Log LOG = LogFactory.getLog(HFilePrettyPrinter.class);
+
+  private Options options = new Options();
+
+  private boolean verbose;
+  private boolean printValue;
+  private boolean printKey;
+  private boolean shouldPrintMeta;
+  private boolean printBlocks;
+  private boolean printStats;
+  private boolean checkRow;
+  private boolean checkFamily;
+  private boolean isSeekToRow = false;
+
+  /**
+   * The row which the user wants to specify and print all the KeyValues for.
+   */
+  private byte[] row = null;
+  private Configuration conf;
+
+  private List files = new ArrayList();
+  private int count;
+
+  private static final String FOUR_SPACES = "    ";
+
+  public HFilePrettyPrinter() {
+    options.addOption("v", "verbose", false,
+        "Verbose output; emits file and meta data delimiters");
+    options.addOption("p", "printkv", false, "Print key/value pairs");
+    options.addOption("e", "printkey", false, "Print keys");
+    options.addOption("m", "printmeta", false, "Print meta data of file");
+    options.addOption("b", "printblocks", false, "Print block index meta data");
+    options.addOption("k", "checkrow", false,
+        "Enable row order check; looks for out-of-order keys");
+    options.addOption("a", "checkfamily", false, "Enable family check");
+    options.addOption("f", "file", true,
+        "File to scan. Pass full-path; e.g. hdfs://a:9000/hbase/.META./12/34");
+    options.addOption("w", "seekToRow", true,
+      "Seek to this row and print all the kvs for this row only");
+    options.addOption("r", "region", true,
+        "Region to scan. Pass region name; e.g. '.META.,,1'");
+    options.addOption("s", "stats", false, "Print statistics");
+  }
+
+  public boolean parseOptions(String args[]) throws ParseException,
+      IOException {
+    if (args.length == 0) {
+      HelpFormatter formatter = new HelpFormatter();
+      formatter.printHelp("HFile", options, true);
+      return false;
+    }
+    CommandLineParser parser = new PosixParser();
+    CommandLine cmd = parser.parse(options, args);
+
+    verbose = cmd.hasOption("v");
+    printValue = cmd.hasOption("p");
+    printKey = cmd.hasOption("e") || printValue;
+    shouldPrintMeta = cmd.hasOption("m");
+    printBlocks = cmd.hasOption("b");
+    printStats = cmd.hasOption("s");
+    checkRow = cmd.hasOption("k");
+    checkFamily = cmd.hasOption("a");
+
+    if (cmd.hasOption("f")) {
+      files.add(new Path(cmd.getOptionValue("f")));
+    }
+
+    if (cmd.hasOption("w")) {
+      String key = cmd.getOptionValue("w");
+      if (key != null && key.length() != 0) {
+        row = key.getBytes();
+        isSeekToRow = true;
+      } else {
+        System.err.println("Invalid row is specified.");
+        System.exit(-1);
+      }
+    }
+
+    if (cmd.hasOption("r")) {
+      String regionName = cmd.getOptionValue("r");
+      byte[] rn = Bytes.toBytes(regionName);
+      byte[][] hri = HRegionInfo.parseRegionName(rn);
+      Path rootDir = FSUtils.getRootDir(conf);
+      Path tableDir = new Path(rootDir, Bytes.toString(hri[0]));
+      String enc = HRegionInfo.encodeRegionName(rn);
+      Path regionDir = new Path(tableDir, enc);
+      if (verbose)
+        System.out.println("region dir -> " + regionDir);
+      List regionFiles = HFile.getStoreFiles(FileSystem.get(conf),
+          regionDir);
+      if (verbose)
+        System.out.println("Number of region files found -> "
+            + regionFiles.size());
+      if (verbose) {
+        int i = 1;
+        for (Path p : regionFiles) {
+          if (verbose)
+            System.out.println("Found file[" + i++ + "] -> " + p);
+        }
+      }
+      files.addAll(regionFiles);
+    }
+
+    return true;
+  }
+
+  /**
+   * Runs the command-line pretty-printer, and returns the desired command
+   * exit code (zero for success, non-zero for failure).
+   */
+  public int run(String[] args) {
+    conf = HBaseConfiguration.create();
+    conf.set("fs.defaultFS",
+        conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
+    conf.set("fs.default.name",
+        conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
+    SchemaMetrics.configureGlobally(conf);
+    try {
+      if (!parseOptions(args))
+        return 1;
+    } catch (IOException ex) {
+      LOG.error("Error parsing command-line options", ex);
+      return 1;
+    } catch (ParseException ex) {
+      LOG.error("Error parsing command-line options", ex);
+      return 1;
+    }
+
+    // iterate over all files found
+    for (Path fileName : files) {
+      try {
+        processFile(fileName);
+      } catch (IOException ex) {
+        LOG.error("Error reading " + fileName, ex);
+      }
+    }
+
+    if (verbose || printKey) {
+      System.out.println("Scanned kv count -> " + count);
+    }
+
+    return 0;
+  }
+
+  private void processFile(Path file) throws IOException {
+    if (verbose)
+      System.out.println("Scanning -> " + file);
+    FileSystem fs = file.getFileSystem(conf);
+    if (!fs.exists(file)) {
+      System.err.println("ERROR, file doesnt exist: " + file);
+    }
+
+    HFile.Reader reader = HFile.createReader(fs, file, new CacheConfig(conf));
+
+    Map fileInfo = reader.loadFileInfo();
+
+    KeyValueStatsCollector fileStats = null;
+
+    if (verbose || printKey || checkRow || checkFamily || printStats) {
+      // scan over file and read key/value's and check if requested
+      HFileScanner scanner = reader.getScanner(false, false, false);
+      fileStats = new KeyValueStatsCollector();
+      boolean shouldScanKeysValues = false;
+      if (this.isSeekToRow) {
+        // seek to the first kv on this row
+        shouldScanKeysValues = 
+          (scanner.seekTo(KeyValue.createFirstOnRow(this.row).getKey()) != -1);
+      } else {
+        shouldScanKeysValues = scanner.seekTo();
+      }
+      if (shouldScanKeysValues)
+        scanKeysValues(file, fileStats, scanner, row);
+    }
+
+    // print meta data
+    if (shouldPrintMeta) {
+      printMeta(reader, fileInfo);
+    }
+
+    if (printBlocks) {
+      System.out.println("Block Index:");
+      System.out.println(reader.getDataBlockIndexReader());
+    }
+
+    if (printStats) {
+      fileStats.finish();
+      System.out.println("Stats:\n" + fileStats);
+    }
+
+    reader.close();
+  }
+
+  private void scanKeysValues(Path file, KeyValueStatsCollector fileStats,
+      HFileScanner scanner,  byte[] row) throws IOException {
+    KeyValue pkv = null;
+    do {
+      KeyValue kv = scanner.getKeyValue();
+      if (row != null && row.length != 0) {
+        int result = Bytes.compareTo(kv.getRow(), row);
+        if (result > 0) {
+          break;
+        } else if (result < 0) {
+          continue;
+        }
+      }
+      // collect stats
+      if (printStats) {
+        fileStats.collect(kv);
+      }
+      // dump key value
+      if (printKey) {
+        System.out.print("K: " + kv);
+        if (printValue) {
+          System.out.print(" V: " + Bytes.toStringBinary(kv.getValue()));
+        }
+        System.out.println();
+      }
+      // check if rows are in order
+      if (checkRow && pkv != null) {
+        if (Bytes.compareTo(pkv.getRow(), kv.getRow()) > 0) {
+          System.err.println("WARNING, previous row is greater then"
+              + " current row\n\tfilename -> " + file + "\n\tprevious -> "
+              + Bytes.toStringBinary(pkv.getKey()) + "\n\tcurrent  -> "
+              + Bytes.toStringBinary(kv.getKey()));
+        }
+      }
+      // check if families are consistent
+      if (checkFamily) {
+        String fam = Bytes.toString(kv.getFamily());
+        if (!file.toString().contains(fam)) {
+          System.err.println("WARNING, filename does not match kv family,"
+              + "\n\tfilename -> " + file + "\n\tkeyvalue -> "
+              + Bytes.toStringBinary(kv.getKey()));
+        }
+        if (pkv != null
+            && !Bytes.equals(pkv.getFamily(), kv.getFamily())) {
+          System.err.println("WARNING, previous kv has different family"
+              + " compared to current key\n\tfilename -> " + file
+              + "\n\tprevious -> " + Bytes.toStringBinary(pkv.getKey())
+              + "\n\tcurrent  -> " + Bytes.toStringBinary(kv.getKey()));
+        }
+      }
+      pkv = kv;
+      ++count;
+    } while (scanner.next());
+  }
+
+  /**
+   * Format a string of the form "k1=v1, k2=v2, ..." into separate lines
+   * with a four-space indentation.
+   */
+  private static String asSeparateLines(String keyValueStr) {
+    return keyValueStr.replaceAll(", ([a-zA-Z]+=)",
+                                  ",\n" + FOUR_SPACES + "$1");
+  }
+
+  private void printMeta(HFile.Reader reader, Map fileInfo)
+      throws IOException {
+    System.out.println("Block index size as per heapsize: "
+        + reader.indexSize());
+    System.out.println(asSeparateLines(reader.toString()));
+    System.out.println("Trailer:\n    "
+        + asSeparateLines(reader.getTrailer().toString()));
+    System.out.println("Fileinfo:");
+    for (Map.Entry e : fileInfo.entrySet()) {
+      System.out.print(FOUR_SPACES + Bytes.toString(e.getKey()) + " = ");
+      if (Bytes.compareTo(e.getKey(), Bytes.toBytes("MAX_SEQ_ID_KEY")) == 0) {
+        long seqid = Bytes.toLong(e.getValue());
+        System.out.println(seqid);
+      } else if (Bytes.compareTo(e.getKey(), Bytes.toBytes("TIMERANGE")) == 0) {
+        TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
+        Writables.copyWritable(e.getValue(), timeRangeTracker);
+        System.out.println(timeRangeTracker.getMinimumTimestamp() + "...."
+            + timeRangeTracker.getMaximumTimestamp());
+      } else if (Bytes.compareTo(e.getKey(), FileInfo.AVG_KEY_LEN) == 0
+          || Bytes.compareTo(e.getKey(), FileInfo.AVG_VALUE_LEN) == 0) {
+        System.out.println(Bytes.toInt(e.getValue()));
+      } else {
+        System.out.println(Bytes.toStringBinary(e.getValue()));
+      }
+    }
+
+    System.out.println("Mid-key: " + Bytes.toStringBinary(reader.midkey()));
+
+    // Printing general bloom information
+    DataInput bloomMeta = reader.getGeneralBloomFilterMetadata();
+    BloomFilter bloomFilter = null;
+    if (bloomMeta != null)
+      bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);
+
+    System.out.println("Bloom filter:");
+    if (bloomFilter != null) {
+      System.out.println(FOUR_SPACES + bloomFilter.toString().replaceAll(
+          ByteBloomFilter.STATS_RECORD_SEP, "\n" + FOUR_SPACES));
+    } else {
+      System.out.println(FOUR_SPACES + "Not present");
+    }
+
+    // Printing delete bloom information
+    bloomMeta = reader.getDeleteBloomFilterMetadata();
+    bloomFilter = null;
+    if (bloomMeta != null)
+      bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);
+
+    System.out.println("Delete Family Bloom filter:");
+    if (bloomFilter != null) {
+      System.out.println(FOUR_SPACES
+          + bloomFilter.toString().replaceAll(ByteBloomFilter.STATS_RECORD_SEP,
+              "\n" + FOUR_SPACES));
+    } else {
+      System.out.println(FOUR_SPACES + "Not present");
+    }
+  }
+
+  private static class LongStats {
+    private long min = Long.MAX_VALUE;
+    private long max = Long.MIN_VALUE;
+    private long sum = 0;
+    private long count = 0;
+
+    void collect(long d) {
+      if (d < min) min = d;
+      if (d > max) max = d;
+      sum += d;
+      count++;
+    }
+
+    public String toString() {
+      return "count: " + count +
+        "\tmin: " + min +
+        "\tmax: " + max +
+        "\tmean: " + ((double)sum/count);
+    }
+  }
+
+  private static class KeyValueStatsCollector {
+    LongStats keyLen = new LongStats();
+    LongStats valLen = new LongStats();
+    LongStats rowSizeBytes = new LongStats();
+    LongStats rowSizeCols = new LongStats();
+
+    long curRowBytes = 0;
+    long curRowCols = 0;
+
+    byte[] biggestRow = null;
+
+    private KeyValue prevKV = null;
+    private long maxRowBytes = 0;
+
+    public void collect(KeyValue kv) {
+      keyLen.collect(kv.getKeyLength());
+      valLen.collect(kv.getValueLength());
+      if (prevKV != null &&
+          KeyValue.COMPARATOR.compareRows(prevKV, kv) != 0) {
+        // new row
+        collectRow();
+      }
+      curRowBytes += kv.getLength();
+      curRowCols++;
+      prevKV = kv;
+    }
+
+    private void collectRow() {
+      rowSizeBytes.collect(curRowBytes);
+      rowSizeCols.collect(curRowCols);
+
+      if (curRowBytes > maxRowBytes && prevKV != null) {
+        biggestRow = prevKV.getRow();
+      }
+
+      curRowBytes = 0;
+      curRowCols = 0;
+    }
+
+    public void finish() {
+      if (curRowCols > 0) {
+        collectRow();
+      }
+    }
+
+    @Override
+    public String toString() {
+      if (prevKV == null)
+        return "no data available for statistics";
+
+      return
+        "Key length: " + keyLen + "\n" +
+        "Val length: " + valLen + "\n" +
+        "Row size (bytes): " + rowSizeBytes + "\n" +
+        "Row size (columns): " + rowSizeCols + "\n" +
+        "Key of biggest row: " + Bytes.toStringBinary(biggestRow);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
new file mode 100644
index 0000000..2f40db9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
@@ -0,0 +1,702 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
+import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
+import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.RawComparator;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * {@link HFile} reader for version 1. Does not support data block encoding,
+ * even in cache only, i.e. HFile v1 blocks are always brought into cache
+ * unencoded.
+ */
+public class HFileReaderV1 extends AbstractHFileReader {
+  private static final Log LOG = LogFactory.getLog(HFileReaderV1.class);
+
+  private volatile boolean fileInfoLoaded = false;
+
+  /**
+   * Opens a HFile.  You must load the index before you can
+   * use it by calling {@link #loadFileInfo()}.
+   *
+   * @param fsdis input stream.  Caller is responsible for closing the passed
+   * stream.
+   * @param size Length of the stream.
+   * @param cacheConf cache references and configuration
+   */
+  public HFileReaderV1(Path path, FixedFileTrailer trailer,
+      final FSDataInputStream fsdis, final long size,
+      final boolean closeIStream,
+      final CacheConfig cacheConf) throws IOException {
+    super(path, trailer, fsdis, size, closeIStream, cacheConf);
+
+    trailer.expectMajorVersion(1);
+    fsBlockReader = new HFileBlock.FSReaderV1(fsdis, compressAlgo, fileSize);
+  }
+
+  private byte[] readAllIndex(final FSDataInputStream in,
+      final long indexOffset, final int indexSize) throws IOException {
+    byte[] allIndex = new byte[indexSize];
+    in.seek(indexOffset);
+    IOUtils.readFully(in, allIndex, 0, allIndex.length);
+
+    return allIndex;
+  }
+
+  /**
+   * Read in the index and file info.
+   *
+   * @return A map of fileinfo data.
+   * @see Writer#appendFileInfo(byte[], byte[])
+   * @throws IOException
+   */
+  @Override
+  public FileInfo loadFileInfo() throws IOException {
+    if (fileInfoLoaded)
+      return fileInfo;
+
+    // Read in the fileinfo and get what we need from it.
+    istream.seek(trailer.getFileInfoOffset());
+    fileInfo = new FileInfo();
+    fileInfo.readFields(istream);
+    lastKey = fileInfo.get(FileInfo.LASTKEY);
+    avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
+    avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
+
+    // Comparator is stored in the file info in version 1.
+    String clazzName = Bytes.toString(fileInfo.get(FileInfo.COMPARATOR));
+    comparator = getComparator(clazzName);
+
+    dataBlockIndexReader =
+        new HFileBlockIndex.BlockIndexReader(comparator, 1);
+    metaBlockIndexReader =
+        new HFileBlockIndex.BlockIndexReader(Bytes.BYTES_RAWCOMPARATOR, 1);
+
+    int sizeToLoadOnOpen = (int) (fileSize - trailer.getLoadOnOpenDataOffset() -
+        trailer.getTrailerSize());
+    byte[] dataAndMetaIndex = readAllIndex(istream,
+        trailer.getLoadOnOpenDataOffset(), sizeToLoadOnOpen);
+
+    ByteArrayInputStream bis = new ByteArrayInputStream(dataAndMetaIndex);
+    DataInputStream dis = new DataInputStream(bis);
+
+    // Read in the data index.
+    if (trailer.getDataIndexCount() > 0)
+      BlockType.INDEX_V1.readAndCheck(dis);
+    dataBlockIndexReader.readRootIndex(dis, trailer.getDataIndexCount());
+
+    // Read in the metadata index.
+    if (trailer.getMetaIndexCount() > 0)
+      BlockType.INDEX_V1.readAndCheck(dis);
+    metaBlockIndexReader.readRootIndex(dis, trailer.getMetaIndexCount());
+
+    fileInfoLoaded = true;
+    return fileInfo;
+  }
+
+  /**
+   * Creates comparator from the given class name.
+   *
+   * @param clazzName the comparator class name read from the trailer
+   * @return an instance of the comparator to use
+   * @throws IOException in case comparator class name is invalid
+   */
+  @SuppressWarnings("unchecked")
+  private RawComparator getComparator(final String clazzName)
+  throws IOException {
+    if (clazzName == null || clazzName.length() == 0) {
+      return null;
+    }
+    try {
+      return (RawComparator)Class.forName(clazzName).newInstance();
+    } catch (InstantiationException e) {
+      throw new IOException(e);
+    } catch (IllegalAccessException e) {
+      throw new IOException(e);
+    } catch (ClassNotFoundException e) {
+      throw new IOException(e);
+    }
+  }
+
+  /**
+   * Create a Scanner on this file. No seeks or reads are done on creation. Call
+   * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
+   * nothing to clean up in a Scanner. Letting go of your references to the
+   * scanner is sufficient.
+   *
+   * @param cacheBlocks True if we should cache blocks read in by this scanner.
+   * @param pread Use positional read rather than seek+read if true (pread is
+   *          better for random reads, seek+read is better scanning).
+   * @param isCompaction is scanner being used for a compaction?
+   * @return Scanner on this file.
+   */
+  @Override
+  public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
+                                final boolean isCompaction) {
+    return new ScannerV1(this, cacheBlocks, pread, isCompaction);
+  }
+
+  /**
+   * @param key Key to search.
+   * @return Block number of the block containing the key or -1 if not in this
+   * file.
+   */
+  protected int blockContainingKey(final byte[] key, int offset, int length) {
+    Preconditions.checkState(!dataBlockIndexReader.isEmpty(),
+        "Block index not loaded");
+    return dataBlockIndexReader.rootBlockContainingKey(key, offset, length);
+  }
+
+  /**
+   * @param metaBlockName
+   * @param cacheBlock Add block to cache, if found
+   * @return Block wrapped in a ByteBuffer
+   * @throws IOException
+   */
+  @Override
+  public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
+      throws IOException {
+    if (trailer.getMetaIndexCount() == 0) {
+      return null; // there are no meta blocks
+    }
+    if (metaBlockIndexReader == null) {
+      throw new IOException("Meta index not loaded");
+    }
+
+    byte[] nameBytes = Bytes.toBytes(metaBlockName);
+    int block = metaBlockIndexReader.rootBlockContainingKey(nameBytes, 0,
+        nameBytes.length);
+    if (block == -1)
+      return null;
+    long offset = metaBlockIndexReader.getRootBlockOffset(block);
+    long nextOffset;
+    if (block == metaBlockIndexReader.getRootBlockCount() - 1) {
+      nextOffset = trailer.getFileInfoOffset();
+    } else {
+      nextOffset = metaBlockIndexReader.getRootBlockOffset(block + 1);
+    }
+
+    long startTimeNs = System.nanoTime();
+
+    BlockCacheKey cacheKey = new BlockCacheKey(name, offset,
+        DataBlockEncoding.NONE, BlockType.META);
+
+    BlockCategory effectiveCategory = BlockCategory.META;
+    if (metaBlockName.equals(HFileWriterV1.BLOOM_FILTER_META_KEY) ||
+        metaBlockName.equals(HFileWriterV1.BLOOM_FILTER_DATA_KEY)) {
+      effectiveCategory = BlockCategory.BLOOM;
+    }
+
+    // Per meta key from any given file, synchronize reads for said block
+    synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
+      // Check cache for block.  If found return.
+      if (cacheConf.isBlockCacheEnabled()) {
+        HFileBlock cachedBlock =
+          (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey,
+              cacheConf.shouldCacheBlockOnRead(effectiveCategory), false);
+        if (cachedBlock != null) {
+          getSchemaMetrics().updateOnCacheHit(effectiveCategory,
+              SchemaMetrics.NO_COMPACTION);
+          return cachedBlock.getBufferWithoutHeader();
+        }
+        // Cache Miss, please load.
+      }
+
+      HFileBlock hfileBlock = fsBlockReader.readBlockData(offset,
+          nextOffset - offset, metaBlockIndexReader.getRootBlockDataSize(block),
+          true);
+      passSchemaMetricsTo(hfileBlock);
+      hfileBlock.expectType(BlockType.META);
+
+      final long delta = System.nanoTime() - startTimeNs;
+      HFile.offerReadLatency(delta, true);
+      getSchemaMetrics().updateOnCacheMiss(effectiveCategory,
+          SchemaMetrics.NO_COMPACTION, delta);
+
+      // Cache the block
+      if (cacheBlock && cacheConf.shouldCacheBlockOnRead(effectiveCategory)) {
+        cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock,
+            cacheConf.isInMemory());
+      }
+
+      return hfileBlock.getBufferWithoutHeader();
+    }
+  }
+
+  /**
+   * Read in a file block.
+   * @param block Index of block to read.
+   * @param pread Use positional read instead of seek+read (positional is
+   * better doing random reads whereas seek+read is better scanning).
+   * @param isCompaction is this block being read as part of a compaction
+   * @return Block wrapped in a ByteBuffer.
+   * @throws IOException
+   */
+  ByteBuffer readBlockBuffer(int block, boolean cacheBlock,
+      final boolean pread, final boolean isCompaction) throws IOException {
+    if (dataBlockIndexReader == null) {
+      throw new IOException("Block index not loaded");
+    }
+    if (block < 0 || block >= dataBlockIndexReader.getRootBlockCount()) {
+      throw new IOException("Requested block is out of range: " + block +
+        ", max: " + dataBlockIndexReader.getRootBlockCount());
+    }
+
+    long offset = dataBlockIndexReader.getRootBlockOffset(block);
+    BlockCacheKey cacheKey = new BlockCacheKey(name, offset);
+
+    // For any given block from any given file, synchronize reads for said
+    // block.
+    // Without a cache, this synchronizing is needless overhead, but really
+    // the other choice is to duplicate work (which the cache would prevent you
+    // from doing).
+    synchronized (dataBlockIndexReader.getRootBlockKey(block)) {
+      // Check cache for block.  If found return.
+      if (cacheConf.isBlockCacheEnabled()) {
+        HFileBlock cachedBlock =
+          (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey,
+              cacheConf.shouldCacheDataOnRead(), false);
+        if (cachedBlock != null) {
+          getSchemaMetrics().updateOnCacheHit(
+              cachedBlock.getBlockType().getCategory(), isCompaction);
+          return cachedBlock.getBufferWithoutHeader();
+        }
+        // Carry on, please load.
+      }
+
+      // Load block from filesystem.
+      long startTimeNs = System.nanoTime();
+      long nextOffset;
+
+      if (block == dataBlockIndexReader.getRootBlockCount() - 1) {
+        // last block!  The end of data block is first meta block if there is
+        // one or if there isn't, the fileinfo offset.
+        nextOffset = (metaBlockIndexReader.getRootBlockCount() == 0) ?
+            this.trailer.getFileInfoOffset() :
+            metaBlockIndexReader.getRootBlockOffset(0);
+      } else {
+        nextOffset = dataBlockIndexReader.getRootBlockOffset(block + 1);
+      }
+
+      HFileBlock hfileBlock = fsBlockReader.readBlockData(offset, nextOffset
+          - offset, dataBlockIndexReader.getRootBlockDataSize(block), pread);
+      passSchemaMetricsTo(hfileBlock);
+      hfileBlock.expectType(BlockType.DATA);
+
+      final long delta = System.nanoTime() - startTimeNs;
+      HFile.offerReadLatency(delta, pread);
+      getSchemaMetrics().updateOnCacheMiss(BlockCategory.DATA, isCompaction,
+          delta);
+
+      // Cache the block
+      if (cacheBlock && cacheConf.shouldCacheBlockOnRead(
+          hfileBlock.getBlockType().getCategory())) {
+        cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock,
+            cacheConf.isInMemory());
+      }
+      return hfileBlock.getBufferWithoutHeader();
+    }
+  }
+
+  /**
+   * @return Last key in the file.  May be null if file has no entries.
+   * Note that this is not the last rowkey, but rather the byte form of
+   * the last KeyValue.
+   */
+  public byte[] getLastKey() {
+    if (!fileInfoLoaded) {
+      throw new RuntimeException("Load file info first");
+    }
+    return dataBlockIndexReader.isEmpty() ? null : lastKey;
+  }
+
+  /**
+   * @return Midkey for this file. We work with block boundaries only so
+   *         returned midkey is an approximation only.
+   *
+   * @throws IOException
+   */
+  @Override
+  public byte[] midkey() throws IOException {
+    Preconditions.checkState(isFileInfoLoaded(), "File info is not loaded");
+    Preconditions.checkState(!dataBlockIndexReader.isEmpty(),
+        "Data block index is not loaded or is empty");
+    return dataBlockIndexReader.midkey();
+  }
+
+  @Override
+  public void close() throws IOException {
+    close(cacheConf.shouldEvictOnClose());
+  }
+
+  @Override
+  public void close(boolean evictOnClose) throws IOException {
+    if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
+      int numEvicted = 0;
+      for (int i = 0; i < dataBlockIndexReader.getRootBlockCount(); i++) {
+        if (cacheConf.getBlockCache().evictBlock(
+            new BlockCacheKey(name,
+                dataBlockIndexReader.getRootBlockOffset(i),
+                DataBlockEncoding.NONE, BlockType.DATA))) {
+          numEvicted++;
+        }
+      }
+      LOG.debug("On close of file " + name + " evicted " + numEvicted
+          + " block(s) of " + dataBlockIndexReader.getRootBlockCount()
+          + " total blocks");
+    }
+    if (this.closeIStream && this.istream != null) {
+      this.istream.close();
+      this.istream = null;
+    }
+
+    getSchemaMetrics().flushMetrics();
+  }
+
+  protected abstract static class AbstractScannerV1
+      extends AbstractHFileReader.Scanner {
+    protected int currBlock;
+
+    /**
+     * This masks a field with the same name in the superclass and saves us the
+     * runtime overhead of casting from abstract reader to reader V1.
+     */
+    protected HFileReaderV1 reader;
+
+    public AbstractScannerV1(HFileReaderV1 reader, boolean cacheBlocks,
+        final boolean pread, final boolean isCompaction) {
+      super(reader, cacheBlocks, pread, isCompaction);
+      this.reader = (HFileReaderV1) reader;
+    }
+
+    /**
+     * Within a loaded block, seek looking for the first key
+     * that is smaller than (or equal to?) the key we are interested in.
+     *
+     * A note on the seekBefore - if you have seekBefore = true, AND the
+     * first key in the block = key, then you'll get thrown exceptions.
+     * @param key to find
+     * @param seekBefore find the key before the exact match.
+     * @return
+     */
+    protected abstract int blockSeek(byte[] key, int offset, int length,
+        boolean seekBefore);
+
+    protected abstract void loadBlock(int bloc, boolean rewind)
+        throws IOException;
+
+    @Override
+    public int seekTo(byte[] key, int offset, int length) throws IOException {
+      int b = reader.blockContainingKey(key, offset, length);
+      if (b < 0) return -1; // falls before the beginning of the file! :-(
+      // Avoid re-reading the same block (that'd be dumb).
+      loadBlock(b, true);
+      return blockSeek(key, offset, length, false);
+    }
+
+    @Override
+    public int reseekTo(byte[] key, int offset, int length)
+        throws IOException {
+      if (blockBuffer != null && currKeyLen != 0) {
+        ByteBuffer bb = getKey();
+        int compared = reader.getComparator().compare(key, offset,
+            length, bb.array(), bb.arrayOffset(), bb.limit());
+        if (compared < 1) {
+          // If the required key is less than or equal to current key, then
+          // don't do anything.
+          return compared;
+        }
+      }
+
+      int b = reader.blockContainingKey(key, offset, length);
+      if (b < 0) {
+        return -1;
+      }
+      loadBlock(b, false);
+      return blockSeek(key, offset, length, false);
+    }
+
+    @Override
+    public boolean seekBefore(byte[] key, int offset, int length)
+        throws IOException {
+      int b = reader.blockContainingKey(key, offset, length);
+      if (b < 0)
+        return false; // key is before the start of the file.
+
+      // Question: does this block begin with 'key'?
+      byte[] firstkKey = reader.getDataBlockIndexReader().getRootBlockKey(b);
+      if (reader.getComparator().compare(firstkKey, 0, firstkKey.length,
+          key, offset, length) == 0) {
+        // Ok the key we're interested in is the first of the block, so go back
+        // by one.
+        if (b == 0) {
+          // we have a 'problem', the key we want is the first of the file.
+          return false;
+        }
+        b--;
+        // TODO shortcut: seek forward in this block to the last key of the
+        // block.
+      }
+      loadBlock(b, true);
+      blockSeek(key, offset, length, true);
+      return true;
+    }
+  }
+
+  /**
+   * Implementation of {@link HFileScanner} interface.
+   */
+
+  protected static class ScannerV1 extends AbstractScannerV1 {
+    private HFileReaderV1 reader;
+
+    public ScannerV1(HFileReaderV1 reader, boolean cacheBlocks,
+        final boolean pread, final boolean isCompaction) {
+      super(reader, cacheBlocks, pread, isCompaction);
+      this.reader = reader;
+    }
+
+    @Override
+    public KeyValue getKeyValue() {
+      if (blockBuffer == null) {
+        return null;
+      }
+      return new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
+          + blockBuffer.position() - 8);
+    }
+
+    @Override
+    public ByteBuffer getKey() {
+      Preconditions.checkState(blockBuffer != null && currKeyLen > 0,
+          "you need to seekTo() before calling getKey()");
+
+      ByteBuffer keyBuff = blockBuffer.slice();
+      keyBuff.limit(currKeyLen);
+      keyBuff.rewind();
+      // Do keyBuff.asReadOnly()?
+      return keyBuff;
+    }
+
+    @Override
+    public ByteBuffer getValue() {
+      if (blockBuffer == null || currKeyLen == 0) {
+        throw new RuntimeException(
+            "you need to seekTo() before calling getValue()");
+      }
+
+      // TODO: Could this be done with one ByteBuffer rather than create two?
+      ByteBuffer valueBuff = blockBuffer.slice();
+      valueBuff.position(currKeyLen);
+      valueBuff = valueBuff.slice();
+      valueBuff.limit(currValueLen);
+      valueBuff.rewind();
+      return valueBuff;
+    }
+
+    @Override
+    public boolean next() throws IOException {
+      if (blockBuffer == null) {
+        throw new IOException("Next called on non-seeked scanner");
+      }
+
+      try {
+        blockBuffer.position(blockBuffer.position() + currKeyLen
+            + currValueLen);
+      } catch (IllegalArgumentException e) {
+        LOG.error("Current pos = " + blockBuffer.position() +
+                  "; currKeyLen = " + currKeyLen +
+                  "; currValLen = " + currValueLen +
+                  "; block limit = " + blockBuffer.limit() +
+                  "; HFile name = " + reader.getName() +
+                  "; currBlock id = " + currBlock, e);
+        throw e;
+      }
+      if (blockBuffer.remaining() <= 0) {
+        currBlock++;
+        if (currBlock >= reader.getDataBlockIndexReader().getRootBlockCount()) {
+          // damn we are at the end
+          currBlock = 0;
+          blockBuffer = null;
+          return false;
+        }
+        blockBuffer = reader.readBlockBuffer(currBlock, cacheBlocks, pread,
+            isCompaction);
+        currKeyLen = blockBuffer.getInt();
+        currValueLen = blockBuffer.getInt();
+        blockFetches++;
+        return true;
+      }
+
+      currKeyLen = blockBuffer.getInt();
+      currValueLen = blockBuffer.getInt();
+      return true;
+    }
+
+    @Override
+    protected int blockSeek(byte[] key, int offset, int length,
+        boolean seekBefore) {
+      int klen, vlen;
+      int lastLen = 0;
+      do {
+        klen = blockBuffer.getInt();
+        vlen = blockBuffer.getInt();
+        int comp = reader.getComparator().compare(key, offset, length,
+            blockBuffer.array(),
+            blockBuffer.arrayOffset() + blockBuffer.position(), klen);
+        if (comp == 0) {
+          if (seekBefore) {
+            blockBuffer.position(blockBuffer.position() - lastLen - 16);
+            currKeyLen = blockBuffer.getInt();
+            currValueLen = blockBuffer.getInt();
+            return 1; // non exact match.
+          }
+          currKeyLen = klen;
+          currValueLen = vlen;
+          return 0; // indicate exact match
+        }
+        if (comp < 0) {
+          // go back one key:
+          blockBuffer.position(blockBuffer.position() - lastLen - 16);
+          currKeyLen = blockBuffer.getInt();
+          currValueLen = blockBuffer.getInt();
+          return 1;
+        }
+        blockBuffer.position(blockBuffer.position() + klen + vlen);
+        lastLen = klen + vlen;
+      } while (blockBuffer.remaining() > 0);
+
+      // ok we are at the end, so go back a littleeeeee....
+      // The 8 in the below is intentionally different to the 16s in the above
+      // Do the math you you'll figure it.
+      blockBuffer.position(blockBuffer.position() - lastLen - 8);
+      currKeyLen = blockBuffer.getInt();
+      currValueLen = blockBuffer.getInt();
+      return 1; // didn't exactly find it.
+    }
+
+    @Override
+    public String getKeyString() {
+      return Bytes.toStringBinary(blockBuffer.array(),
+          blockBuffer.arrayOffset() + blockBuffer.position(), currKeyLen);
+    }
+
+    @Override
+    public String getValueString() {
+      return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset() +
+        blockBuffer.position() + currKeyLen, currValueLen);
+    }
+
+    @Override
+    public boolean seekTo() throws IOException {
+      if (reader.getDataBlockIndexReader().isEmpty()) {
+        return false;
+      }
+      if (blockBuffer != null && currBlock == 0) {
+        blockBuffer.rewind();
+        currKeyLen = blockBuffer.getInt();
+        currValueLen = blockBuffer.getInt();
+        return true;
+      }
+      currBlock = 0;
+      blockBuffer = reader.readBlockBuffer(currBlock, cacheBlocks, pread,
+          isCompaction);
+      currKeyLen = blockBuffer.getInt();
+      currValueLen = blockBuffer.getInt();
+      blockFetches++;
+      return true;
+    }
+
+    @Override
+    protected void loadBlock(int bloc, boolean rewind) throws IOException {
+      if (blockBuffer == null) {
+        blockBuffer = reader.readBlockBuffer(bloc, cacheBlocks, pread,
+            isCompaction);
+        currBlock = bloc;
+        blockFetches++;
+      } else {
+        if (bloc != currBlock) {
+          blockBuffer = reader.readBlockBuffer(bloc, cacheBlocks, pread,
+              isCompaction);
+          currBlock = bloc;
+          blockFetches++;
+        } else {
+          // we are already in the same block, just rewind to seek again.
+          if (rewind) {
+            blockBuffer.rewind();
+          }
+          else {
+            // Go back by (size of rowlength + size of valuelength) = 8 bytes
+            blockBuffer.position(blockBuffer.position()-8);
+          }
+        }
+      }
+    }
+
+  }
+
+  @Override
+  public HFileBlock readBlock(long offset, long onDiskBlockSize,
+      boolean cacheBlock, boolean pread, boolean isCompaction,
+      BlockType expectedBlockType) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public DataInput getGeneralBloomFilterMetadata() throws IOException {
+    // Shouldn't cache Bloom filter blocks, otherwise server would abort when
+    // splitting, see HBASE-6479
+    ByteBuffer buf = getMetaBlock(HFileWriterV1.BLOOM_FILTER_META_KEY, false);
+    if (buf == null)
+      return null;
+    ByteArrayInputStream bais = new ByteArrayInputStream(buf.array(),
+        buf.arrayOffset(), buf.limit());
+    return new DataInputStream(bais);
+  }
+
+  @Override
+  public DataInput getDeleteBloomFilterMetadata() throws IOException {
+    return null;
+  }
+
+  @Override
+  public boolean isFileInfoLoaded() {
+    return fileInfoLoaded;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
new file mode 100644
index 0000000..f5f1f9b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -0,0 +1,1140 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
+import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.IdLock;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * {@link HFile} reader for version 2.
+ */
+public class HFileReaderV2 extends AbstractHFileReader {
+
+  private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
+
+  /**
+   * The size of a (key length, value length) tuple that prefixes each entry in
+   * a data block.
+   */
+  private static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
+
+  private boolean includesMemstoreTS = false;
+  private boolean decodeMemstoreTS = false;
+
+  private boolean shouldIncludeMemstoreTS() {
+    return includesMemstoreTS;
+  }
+
+  /**
+   * A "sparse lock" implementation allowing to lock on a particular block
+   * identified by offset. The purpose of this is to avoid two clients loading
+   * the same block, and have all but one client wait to get the block from the
+   * cache.
+   */
+  private IdLock offsetLock = new IdLock();
+
+  /**
+   * Blocks read from the load-on-open section, excluding data root index, meta
+   * index, and file info.
+   */
+  private List loadOnOpenBlocks = new ArrayList();
+
+  /** Minimum minor version supported by this HFile format */
+  static final int MIN_MINOR_VERSION = 0;
+
+  /** Maximum minor version supported by this HFile format */
+  static final int MAX_MINOR_VERSION = 1;
+
+  /**
+   * Opens a HFile. You must load the index before you can use it by calling
+   * {@link #loadFileInfo()}.
+   *
+   * @param path Path to HFile.
+   * @param trailer File trailer.
+   * @param fsdis input stream. Caller is responsible for closing the passed
+   *          stream.
+   * @param size Length of the stream.
+   * @param closeIStream Whether to close the stream.
+   * @param cacheConf Cache configuration.
+   * @param preferredEncodingInCache the encoding to use in cache in case we
+   *          have a choice. If the file is already encoded on disk, we will
+   *          still use its on-disk encoding in cache.
+   */
+  public HFileReaderV2(Path path, FixedFileTrailer trailer,
+      final FSDataInputStream fsdis, final FSDataInputStream fsdisNoFsChecksum,
+      final long size,
+      final boolean closeIStream, final CacheConfig cacheConf,
+      DataBlockEncoding preferredEncodingInCache, final HFileSystem hfs)
+      throws IOException {
+    super(path, trailer, fsdis, fsdisNoFsChecksum, size, 
+          closeIStream, cacheConf, hfs);
+    trailer.expectMajorVersion(2);
+    validateMinorVersion(path, trailer.getMinorVersion());
+    HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis,
+        fsdisNoFsChecksum,
+        compressAlgo, fileSize, trailer.getMinorVersion(), hfs, path);
+    this.fsBlockReader = fsBlockReaderV2; // upcast
+
+    // Comparator class name is stored in the trailer in version 2.
+    comparator = trailer.createComparator();
+    dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator,
+        trailer.getNumDataIndexLevels(), this);
+    metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(
+        Bytes.BYTES_RAWCOMPARATOR, 1);
+
+    // Parse load-on-open data.
+
+    HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(
+        trailer.getLoadOnOpenDataOffset(),
+        fileSize - trailer.getTrailerSize());
+
+    // Data index. We also read statistics about the block index written after
+    // the root level.
+    dataBlockIndexReader.readMultiLevelIndexRoot(
+        blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
+        trailer.getDataIndexCount());
+
+    // Meta index.
+    metaBlockIndexReader.readRootIndex(
+        blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
+        trailer.getMetaIndexCount());
+
+    // File info
+    fileInfo = new FileInfo();
+    fileInfo.readFields(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
+    lastKey = fileInfo.get(FileInfo.LASTKEY);
+    avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
+    avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
+    byte [] keyValueFormatVersion =
+        fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
+    includesMemstoreTS = keyValueFormatVersion != null &&
+        Bytes.toInt(keyValueFormatVersion) ==
+            HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
+    fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS);
+    if (includesMemstoreTS) {
+      decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0;
+    }
+
+    // Read data block encoding algorithm name from file info.
+    dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo,
+        preferredEncodingInCache);
+    fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
+
+    // Store all other load-on-open blocks for further consumption.
+    HFileBlock b;
+    while ((b = blockIter.nextBlock()) != null) {
+      loadOnOpenBlocks.add(b);
+    }
+  }
+
+  /**
+   * Create a Scanner on this file. No seeks or reads are done on creation. Call
+   * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
+   * nothing to clean up in a Scanner. Letting go of your references to the
+   * scanner is sufficient.
+   *
+   * @param cacheBlocks True if we should cache blocks read in by this scanner.
+   * @param pread Use positional read rather than seek+read if true (pread is
+   *          better for random reads, seek+read is better scanning).
+   * @param isCompaction is scanner being used for a compaction?
+   * @return Scanner on this file.
+   */
+   @Override
+   public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
+      final boolean isCompaction) {
+    // check if we want to use data block encoding in memory
+    if (dataBlockEncoder.useEncodedScanner(isCompaction)) {
+      return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
+          includesMemstoreTS);
+    }
+
+    return new ScannerV2(this, cacheBlocks, pread, isCompaction);
+  }
+
+  /**
+   * @param metaBlockName
+   * @param cacheBlock Add block to cache, if found
+   * @return block wrapped in a ByteBuffer, with header skipped
+   * @throws IOException
+   */
+  @Override
+  public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock)
+      throws IOException {
+    if (trailer.getMetaIndexCount() == 0) {
+      return null; // there are no meta blocks
+    }
+    if (metaBlockIndexReader == null) {
+      throw new IOException("Meta index not loaded");
+    }
+
+    byte[] mbname = Bytes.toBytes(metaBlockName);
+    int block = metaBlockIndexReader.rootBlockContainingKey(mbname, 0,
+        mbname.length);
+    if (block == -1)
+      return null;
+    long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
+    long startTimeNs = System.nanoTime();
+
+    // Per meta key from any given file, synchronize reads for said block. This
+    // is OK to do for meta blocks because the meta block index is always
+    // single-level.
+    synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
+      // Check cache for block. If found return.
+      long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block);
+      BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset,
+          DataBlockEncoding.NONE, BlockType.META);
+
+      cacheBlock &= cacheConf.shouldCacheDataOnRead();
+      if (cacheConf.isBlockCacheEnabled()) {
+        HFileBlock cachedBlock =
+          (HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock, false);
+        if (cachedBlock != null) {
+          // Return a distinct 'shallow copy' of the block,
+          // so pos does not get messed by the scanner
+          getSchemaMetrics().updateOnCacheHit(BlockCategory.META, false);
+          return cachedBlock.getBufferWithoutHeader();
+        }
+        // Cache Miss, please load.
+      }
+
+      HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset,
+          blockSize, -1, true);
+      passSchemaMetricsTo(metaBlock);
+
+      final long delta = System.nanoTime() - startTimeNs;
+      HFile.offerReadLatency(delta, true);
+      getSchemaMetrics().updateOnCacheMiss(BlockCategory.META, false, delta);
+
+      // Cache the block
+      if (cacheBlock) {
+        cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock,
+            cacheConf.isInMemory());
+      }
+
+      return metaBlock.getBufferWithoutHeader();
+    }
+  }
+
+  /**
+   * Read in a file block.
+   * @param dataBlockOffset offset to read.
+   * @param onDiskBlockSize size of the block
+   * @param cacheBlock
+   * @param pread Use positional read instead of seek+read (positional is
+   *          better doing random reads whereas seek+read is better scanning).
+   * @param isCompaction is this block being read as part of a compaction
+   * @param expectedBlockType the block type we are expecting to read with this
+   *          read operation, or null to read whatever block type is available
+   *          and avoid checking (that might reduce caching efficiency of
+   *          encoded data blocks)
+   * @return Block wrapped in a ByteBuffer.
+   * @throws IOException
+   */
+  @Override
+  public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
+      final boolean cacheBlock, boolean pread, final boolean isCompaction,
+      BlockType expectedBlockType)
+      throws IOException {
+    if (dataBlockIndexReader == null) {
+      throw new IOException("Block index not loaded");
+    }
+    if (dataBlockOffset < 0
+        || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) {
+      throw new IOException("Requested block is out of range: "
+          + dataBlockOffset + ", lastDataBlockOffset: "
+          + trailer.getLastDataBlockOffset());
+    }
+    // For any given block from any given file, synchronize reads for said
+    // block.
+    // Without a cache, this synchronizing is needless overhead, but really
+    // the other choice is to duplicate work (which the cache would prevent you
+    // from doing).
+
+    BlockCacheKey cacheKey =
+        new BlockCacheKey(name, dataBlockOffset,
+            dataBlockEncoder.getEffectiveEncodingInCache(isCompaction),
+            expectedBlockType);
+
+    boolean useLock = false;
+    IdLock.Entry lockEntry = null;
+
+    try {
+      while (true) {
+
+        if (useLock) {
+          lockEntry = offsetLock.getLockEntry(dataBlockOffset);
+        }
+
+        // Check cache for block. If found return.
+        if (cacheConf.isBlockCacheEnabled()) {
+          // Try and get the block from the block cache.  If the useLock variable is true then this
+          // is the second time through the loop and it should not be counted as a block cache miss.
+          HFileBlock cachedBlock = (HFileBlock)
+              cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock, useLock);
+          if (cachedBlock != null) {
+            BlockCategory blockCategory =
+                cachedBlock.getBlockType().getCategory();
+
+            getSchemaMetrics().updateOnCacheHit(blockCategory, isCompaction);
+
+            if (cachedBlock.getBlockType() == BlockType.DATA) {
+              HFile.dataBlockReadCnt.incrementAndGet();
+            }
+
+            validateBlockType(cachedBlock, expectedBlockType);
+
+            // Validate encoding type for encoded blocks. We include encoding
+            // type in the cache key, and we expect it to match on a cache hit.
+            if (cachedBlock.getBlockType() == BlockType.ENCODED_DATA &&
+                cachedBlock.getDataBlockEncoding() !=
+                    dataBlockEncoder.getEncodingInCache()) {
+              throw new IOException("Cached block under key " + cacheKey + " " +
+                  "has wrong encoding: " + cachedBlock.getDataBlockEncoding() +
+                  " (expected: " + dataBlockEncoder.getEncodingInCache() + ")");
+            }
+            return cachedBlock;
+          }
+          // Carry on, please load.
+        }
+        if (!useLock) {
+          // check cache again with lock
+          useLock = true;
+          continue;
+        }
+
+        // Load block from filesystem.
+        long startTimeNs = System.nanoTime();
+        HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset,
+            onDiskBlockSize, -1, pread);
+        hfileBlock = dataBlockEncoder.diskToCacheFormat(hfileBlock,
+            isCompaction);
+        validateBlockType(hfileBlock, expectedBlockType);
+        passSchemaMetricsTo(hfileBlock);
+        BlockCategory blockCategory = hfileBlock.getBlockType().getCategory();
+
+        final long delta = System.nanoTime() - startTimeNs;
+        HFile.offerReadLatency(delta, pread);
+        getSchemaMetrics().updateOnCacheMiss(blockCategory, isCompaction, delta);
+
+        // Cache the block if necessary
+        if (cacheBlock && cacheConf.shouldCacheBlockOnRead(
+            hfileBlock.getBlockType().getCategory())) {
+          cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock,
+              cacheConf.isInMemory());
+        }
+
+        if (hfileBlock.getBlockType() == BlockType.DATA) {
+          HFile.dataBlockReadCnt.incrementAndGet();
+        }
+
+        return hfileBlock;
+      }
+    } finally {
+      if (lockEntry != null) {
+        offsetLock.releaseLockEntry(lockEntry);
+      }
+    }
+  }
+
+  /**
+   * Compares the actual type of a block retrieved from cache or disk with its
+   * expected type and throws an exception in case of a mismatch. Expected
+   * block type of {@link BlockType#DATA} is considered to match the actual
+   * block type [@link {@link BlockType#ENCODED_DATA} as well.
+   * @param block a block retrieved from cache or disk
+   * @param expectedBlockType the expected block type, or null to skip the
+   *          check
+   */
+  private void validateBlockType(HFileBlock block,
+      BlockType expectedBlockType) throws IOException {
+    if (expectedBlockType == null) {
+      return;
+    }
+    BlockType actualBlockType = block.getBlockType();
+    if (actualBlockType == BlockType.ENCODED_DATA &&
+        expectedBlockType == BlockType.DATA) {
+      // We consider DATA to match ENCODED_DATA for the purpose of this
+      // verification.
+      return;
+    }
+    if (actualBlockType != expectedBlockType) {
+      throw new IOException("Expected block type " + expectedBlockType + ", " +
+          "but got " + actualBlockType + ": " + block);
+    }
+  }
+
+  /**
+   * @return Last key in the file. May be null if file has no entries. Note that
+   *         this is not the last row key, but rather the byte form of the last
+   *         KeyValue.
+   */
+  @Override
+  public byte[] getLastKey() {
+    return dataBlockIndexReader.isEmpty() ? null : lastKey;
+  }
+
+  /**
+   * @return Midkey for this file. We work with block boundaries only so
+   *         returned midkey is an approximation only.
+   * @throws IOException
+   */
+  @Override
+  public byte[] midkey() throws IOException {
+    return dataBlockIndexReader.midkey();
+  }
+
+  @Override
+  public void close() throws IOException {
+    close(cacheConf.shouldEvictOnClose());
+  }
+
+  public void close(boolean evictOnClose) throws IOException {
+    if (evictOnClose && cacheConf.isBlockCacheEnabled()) {
+      int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name);
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("On close, file=" + name + " evicted=" + numEvicted
+          + " block(s)");
+      }
+    }
+    if (closeIStream) {
+      if (istream != istreamNoFsChecksum && istreamNoFsChecksum != null) {
+        istreamNoFsChecksum.close();
+        istreamNoFsChecksum = null;
+      }
+      if (istream != null) {
+        istream.close();
+        istream = null;
+      }
+    }
+
+    getSchemaMetrics().flushMetrics();
+  }
+
+  protected abstract static class AbstractScannerV2
+      extends AbstractHFileReader.Scanner {
+    protected HFileBlock block;
+
+    /**
+     * The next indexed key is to keep track of the indexed key of the next data block.
+     * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the
+     * current data block is the last data block.
+     *
+     * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet.
+     */
+    protected byte[] nextIndexedKey;
+
+    public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
+        final boolean pread, final boolean isCompaction) {
+      super(r, cacheBlocks, pread, isCompaction);
+    }
+
+    /**
+     * An internal API function. Seek to the given key, optionally rewinding to
+     * the first key of the block before doing the seek.
+     *
+     * @param key key byte array
+     * @param offset key offset in the key byte array
+     * @param length key length
+     * @param rewind whether to rewind to the first key of the block before
+     *        doing the seek. If this is false, we are assuming we never go
+     *        back, otherwise the result is undefined.
+     * @return -1 if the key is earlier than the first key of the file,
+     *         0 if we are at the given key, and 1 if we are past the given key
+     * @throws IOException
+     */
+    protected int seekTo(byte[] key, int offset, int length, boolean rewind)
+        throws IOException {
+      HFileBlockIndex.BlockIndexReader indexReader =
+          reader.getDataBlockIndexReader();
+      BlockWithScanInfo blockWithScanInfo =
+        indexReader.loadDataBlockWithScanInfo(key, offset, length, block,
+            cacheBlocks, pread, isCompaction);
+      if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
+        // This happens if the key e.g. falls before the beginning of the file.
+        return -1;
+      }
+      return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
+          blockWithScanInfo.getNextIndexedKey(), rewind, key, offset, length, false);
+    }
+
+    protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
+
+    protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
+        boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
+        throws IOException;
+
+    @Override
+    public int seekTo(byte[] key, int offset, int length) throws IOException {
+      // Always rewind to the first key of the block, because the given key
+      // might be before or after the current key.
+      return seekTo(key, offset, length, true);
+    }
+
+    @Override
+    public int reseekTo(byte[] key, int offset, int length) throws IOException {
+      int compared;
+      if (isSeeked()) {
+        ByteBuffer bb = getKey();
+        compared = reader.getComparator().compare(key, offset,
+            length, bb.array(), bb.arrayOffset(), bb.limit());
+        if (compared < 1) {
+          // If the required key is less than or equal to current key, then
+          // don't do anything.
+          return compared;
+        } else {
+          if (this.nextIndexedKey != null &&
+              (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY ||
+               reader.getComparator().compare(key, offset, length,
+                   nextIndexedKey, 0, nextIndexedKey.length) < 0)) {
+            // The reader shall continue to scan the current data block instead of querying the
+            // block index as long as it knows the target key is strictly smaller than
+            // the next indexed key or the current data block is the last data block.
+            return loadBlockAndSeekToKey(this.block, this.nextIndexedKey,
+                false, key, offset, length, false);
+          }
+        }
+      }
+      // Don't rewind on a reseek operation, because reseek implies that we are
+      // always going forward in the file.
+      return seekTo(key, offset, length, false);
+    }
+
+    @Override
+    public boolean seekBefore(byte[] key, int offset, int length)
+        throws IOException {
+      HFileBlock seekToBlock =
+          reader.getDataBlockIndexReader().seekToDataBlock(key, offset, length,
+              block, cacheBlocks, pread, isCompaction);
+      if (seekToBlock == null) {
+        return false;
+      }
+      ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
+
+      if (reader.getComparator().compare(firstKey.array(),
+          firstKey.arrayOffset(), firstKey.limit(), key, offset, length) == 0)
+      {
+        long previousBlockOffset = seekToBlock.getPrevBlockOffset();
+        // The key we are interested in
+        if (previousBlockOffset == -1) {
+          // we have a 'problem', the key we want is the first of the file.
+          return false;
+        }
+
+        // It is important that we compute and pass onDiskSize to the block
+        // reader so that it does not have to read the header separately to
+        // figure out the size.
+        seekToBlock = reader.readBlock(previousBlockOffset,
+            seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
+            pread, isCompaction, BlockType.DATA);
+        // TODO shortcut: seek forward in this block to the last key of the
+        // block.
+      }
+      byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey);
+      loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, offset, length, true);
+      return true;
+    }
+
+
+    /**
+     * Scans blocks in the "scanned" section of the {@link HFile} until the next
+     * data block is found.
+     *
+     * @return the next block, or null if there are no more data blocks
+     * @throws IOException
+     */
+    protected HFileBlock readNextDataBlock() throws IOException {
+      long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
+      if (block == null)
+        return null;
+
+      HFileBlock curBlock = block;
+
+      do {
+        if (curBlock.getOffset() >= lastDataBlockOffset)
+          return null;
+
+        if (curBlock.getOffset() < 0) {
+          throw new IOException("Invalid block file offset: " + block);
+        }
+
+        // We are reading the next block without block type validation, because
+        // it might turn out to be a non-data block.
+        curBlock = reader.readBlock(curBlock.getOffset()
+            + curBlock.getOnDiskSizeWithHeader(),
+            curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
+            isCompaction, null);
+      } while (!(curBlock.getBlockType().equals(BlockType.DATA) ||
+          curBlock.getBlockType().equals(BlockType.ENCODED_DATA)));
+
+      return curBlock;
+    }
+  }
+
+  /**
+   * Implementation of {@link HFileScanner} interface.
+   */
+  protected static class ScannerV2 extends AbstractScannerV2 {
+    private HFileReaderV2 reader;
+
+    public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
+        final boolean pread, final boolean isCompaction) {
+      super(r, cacheBlocks, pread, isCompaction);
+      this.reader = r;
+    }
+
+    @Override
+    public KeyValue getKeyValue() {
+      if (!isSeeked())
+        return null;
+
+      KeyValue ret = new KeyValue(blockBuffer.array(),
+          blockBuffer.arrayOffset() + blockBuffer.position(),
+          KEY_VALUE_LEN_SIZE + currKeyLen + currValueLen,
+          currKeyLen);
+      if (this.reader.shouldIncludeMemstoreTS()) {
+        ret.setMemstoreTS(currMemstoreTS);
+      }
+      return ret;
+    }
+
+    @Override
+    public ByteBuffer getKey() {
+      assertSeeked();
+      return ByteBuffer.wrap(
+          blockBuffer.array(),
+          blockBuffer.arrayOffset() + blockBuffer.position()
+              + KEY_VALUE_LEN_SIZE, currKeyLen).slice();
+    }
+
+    @Override
+    public ByteBuffer getValue() {
+      assertSeeked();
+      return ByteBuffer.wrap(
+          blockBuffer.array(),
+          blockBuffer.arrayOffset() + blockBuffer.position()
+              + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen).slice();
+    }
+
+    private void setNonSeekedState() {
+      block = null;
+      blockBuffer = null;
+      currKeyLen = 0;
+      currValueLen = 0;
+      currMemstoreTS = 0;
+      currMemstoreTSLen = 0;
+    }
+
+    /**
+     * Go to the next key/value in the block section. Loads the next block if
+     * necessary. If successful, {@link #getKey()} and {@link #getValue()} can
+     * be called.
+     *
+     * @return true if successfully navigated to the next key/value
+     */
+    @Override
+    public boolean next() throws IOException {
+      assertSeeked();
+
+      try {
+        blockBuffer.position(blockBuffer.position() + KEY_VALUE_LEN_SIZE
+            + currKeyLen + currValueLen + currMemstoreTSLen);
+      } catch (IllegalArgumentException e) {
+        LOG.error("Current pos = " + blockBuffer.position()
+            + "; currKeyLen = " + currKeyLen + "; currValLen = "
+            + currValueLen + "; block limit = " + blockBuffer.limit()
+            + "; HFile name = " + reader.getName()
+            + "; currBlock currBlockOffset = " + block.getOffset());
+        throw e;
+      }
+
+      if (blockBuffer.remaining() <= 0) {
+        long lastDataBlockOffset =
+            reader.getTrailer().getLastDataBlockOffset();
+
+        if (block.getOffset() >= lastDataBlockOffset) {
+          setNonSeekedState();
+          return false;
+        }
+
+        // read the next block
+        HFileBlock nextBlock = readNextDataBlock();
+        if (nextBlock == null) {
+          setNonSeekedState();
+          return false;
+        }
+
+        updateCurrBlock(nextBlock);
+        return true;
+      }
+
+      // We are still in the same block.
+      readKeyValueLen();
+      return true;
+    }
+
+    /**
+     * Positions this scanner at the start of the file.
+     *
+     * @return false if empty file; i.e. a call to next would return false and
+     *         the current key and value are undefined.
+     * @throws IOException
+     */
+    @Override
+    public boolean seekTo() throws IOException {
+      if (reader == null) {
+        return false;
+      }
+
+      if (reader.getTrailer().getEntryCount() == 0) {
+        // No data blocks.
+        return false;
+      }
+
+      long firstDataBlockOffset =
+          reader.getTrailer().getFirstDataBlockOffset();
+      if (block != null && block.getOffset() == firstDataBlockOffset) {
+        blockBuffer.rewind();
+        readKeyValueLen();
+        return true;
+      }
+
+      block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
+          isCompaction, BlockType.DATA);
+      if (block.getOffset() < 0) {
+        throw new IOException("Invalid block offset: " + block.getOffset());
+      }
+      updateCurrBlock(block);
+      return true;
+    }
+
+    @Override
+    protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
+        boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
+        throws IOException {
+      if (block == null || block.getOffset() != seekToBlock.getOffset()) {
+        updateCurrBlock(seekToBlock);
+      } else if (rewind) {
+        blockBuffer.rewind();
+      }
+
+      // Update the nextIndexedKey
+      this.nextIndexedKey = nextIndexedKey;
+      return blockSeek(key, offset, length, seekBefore);
+    }
+
+    /**
+     * Updates the current block to be the given {@link HFileBlock}. Seeks to
+     * the the first key/value pair.
+     *
+     * @param newBlock the block to make current
+     */
+    private void updateCurrBlock(HFileBlock newBlock) {
+      block = newBlock;
+
+      // sanity check
+      if (block.getBlockType() != BlockType.DATA) {
+        throw new IllegalStateException("ScannerV2 works only on data " +
+            "blocks, got " + block.getBlockType() + "; " +
+            "fileName=" + reader.name + ", " +
+            "dataBlockEncoder=" + reader.dataBlockEncoder + ", " +
+            "isCompaction=" + isCompaction);
+      }
+
+      blockBuffer = block.getBufferWithoutHeader();
+      readKeyValueLen();
+      blockFetches++;
+
+      // Reset the next indexed key
+      this.nextIndexedKey = null;
+    }
+
+    private final void readKeyValueLen() {
+      blockBuffer.mark();
+      currKeyLen = blockBuffer.getInt();
+      currValueLen = blockBuffer.getInt();
+      blockBuffer.reset();
+      if (this.reader.shouldIncludeMemstoreTS()) {
+        if (this.reader.decodeMemstoreTS) {
+          try {
+            int memstoreTSOffset = blockBuffer.arrayOffset()
+                + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen
+                + currValueLen;
+            currMemstoreTS = Bytes.readVLong(blockBuffer.array(),
+                memstoreTSOffset);
+            currMemstoreTSLen = WritableUtils.getVIntSize(currMemstoreTS);
+          } catch (Exception e) {
+            throw new RuntimeException("Error reading memstore timestamp", e);
+          }
+        } else {
+          currMemstoreTS = 0;
+          currMemstoreTSLen = 1;
+        }
+      }
+
+      if (currKeyLen < 0 || currValueLen < 0
+          || currKeyLen > blockBuffer.limit()
+          || currValueLen > blockBuffer.limit()) {
+        throw new IllegalStateException("Invalid currKeyLen " + currKeyLen
+            + " or currValueLen " + currValueLen + ". Block offset: "
+            + block.getOffset() + ", block length: " + blockBuffer.limit()
+            + ", position: " + blockBuffer.position() + " (without header).");
+      }
+    }
+
+    /**
+     * Within a loaded block, seek looking for the last key that is smaller
+     * than (or equal to?) the key we are interested in.
+     *
+     * A note on the seekBefore: if you have seekBefore = true, AND the first
+     * key in the block = key, then you'll get thrown exceptions. The caller has
+     * to check for that case and load the previous block as appropriate.
+     *
+     * @param key the key to find
+     * @param seekBefore find the key before the given key in case of exact
+     *          match.
+     * @return 0 in case of an exact key match, 1 in case of an inexact match
+     */
+    private int blockSeek(byte[] key, int offset, int length,
+        boolean seekBefore) {
+      int klen, vlen;
+      long memstoreTS = 0;
+      int memstoreTSLen = 0;
+      int lastKeyValueSize = -1;
+      do {
+        blockBuffer.mark();
+        klen = blockBuffer.getInt();
+        vlen = blockBuffer.getInt();
+        blockBuffer.reset();
+        if (this.reader.shouldIncludeMemstoreTS()) {
+          if (this.reader.decodeMemstoreTS) {
+            try {
+              int memstoreTSOffset = blockBuffer.arrayOffset()
+                  + blockBuffer.position() + KEY_VALUE_LEN_SIZE + klen + vlen;
+              memstoreTS = Bytes.readVLong(blockBuffer.array(),
+                  memstoreTSOffset);
+              memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
+            } catch (Exception e) {
+              throw new RuntimeException("Error reading memstore timestamp", e);
+            }
+          } else {
+            memstoreTS = 0;
+            memstoreTSLen = 1;
+          }
+        }
+
+        int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position()
+            + KEY_VALUE_LEN_SIZE;
+        int comp = reader.getComparator().compare(key, offset, length,
+            blockBuffer.array(), keyOffset, klen);
+
+        if (comp == 0) {
+          if (seekBefore) {
+            if (lastKeyValueSize < 0) {
+              throw new IllegalStateException("blockSeek with seekBefore "
+                  + "at the first key of the block: key="
+                  + Bytes.toStringBinary(key) + ", blockOffset="
+                  + block.getOffset() + ", onDiskSize="
+                  + block.getOnDiskSizeWithHeader());
+            }
+            blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
+            readKeyValueLen();
+            return 1; // non exact match.
+          }
+          currKeyLen = klen;
+          currValueLen = vlen;
+          if (this.reader.shouldIncludeMemstoreTS()) {
+            currMemstoreTS = memstoreTS;
+            currMemstoreTSLen = memstoreTSLen;
+          }
+          return 0; // indicate exact match
+        }
+
+        if (comp < 0) {
+          if (lastKeyValueSize > 0)
+            blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
+          readKeyValueLen();
+          return 1;
+        }
+
+        // The size of this key/value tuple, including key/value length fields.
+        lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
+        blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
+      } while (blockBuffer.remaining() > 0);
+
+      // Seek to the last key we successfully read. This will happen if this is
+      // the last key/value pair in the file, in which case the following call
+      // to next() has to return false.
+      blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
+      readKeyValueLen();
+      return 1; // didn't exactly find it.
+    }
+
+    @Override
+    protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
+      ByteBuffer buffer = curBlock.getBufferWithoutHeader();
+      // It is safe to manipulate this buffer because we own the buffer object.
+      buffer.rewind();
+      int klen = buffer.getInt();
+      buffer.getInt();
+      ByteBuffer keyBuff = buffer.slice();
+      keyBuff.limit(klen);
+      keyBuff.rewind();
+      return keyBuff;
+    }
+
+    @Override
+    public String getKeyString() {
+      return Bytes.toStringBinary(blockBuffer.array(),
+          blockBuffer.arrayOffset() + blockBuffer.position()
+              + KEY_VALUE_LEN_SIZE, currKeyLen);
+    }
+
+    @Override
+    public String getValueString() {
+      return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
+          + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
+          currValueLen);
+    }
+  }
+
+  /**
+   * ScannerV2 that operates on encoded data blocks.
+   */
+  protected static class EncodedScannerV2 extends AbstractScannerV2 {
+    private DataBlockEncoder.EncodedSeeker seeker = null;
+    private DataBlockEncoder dataBlockEncoder = null;
+    private final boolean includesMemstoreTS;
+
+    public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
+        boolean pread, boolean isCompaction, boolean includesMemstoreTS) {
+      super(reader, cacheBlocks, pread, isCompaction);
+      this.includesMemstoreTS = includesMemstoreTS;
+    }
+
+    private void setDataBlockEncoder(DataBlockEncoder dataBlockEncoder) {
+      this.dataBlockEncoder = dataBlockEncoder;
+      seeker = dataBlockEncoder.createSeeker(reader.getComparator(),
+          includesMemstoreTS);
+    }
+
+    /**
+     * Updates the current block to be the given {@link HFileBlock}. Seeks to
+     * the the first key/value pair.
+     *
+     * @param newBlock the block to make current
+     */
+    private void updateCurrentBlock(HFileBlock newBlock) {
+      block = newBlock;
+
+      // sanity checks
+      if (block.getBlockType() != BlockType.ENCODED_DATA) {
+        throw new IllegalStateException(
+            "EncodedScannerV2 works only on encoded data blocks");
+      }
+
+      short dataBlockEncoderId = block.getDataBlockEncodingId();
+      if (dataBlockEncoder == null ||
+          !DataBlockEncoding.isCorrectEncoder(dataBlockEncoder,
+              dataBlockEncoderId)) {
+        DataBlockEncoder encoder =
+            DataBlockEncoding.getDataBlockEncoderById(dataBlockEncoderId);
+        setDataBlockEncoder(encoder);
+      }
+
+      seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
+      blockFetches++;
+    }
+
+    private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
+      ByteBuffer origBlock = newBlock.getBufferReadOnly();
+      ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
+          origBlock.arrayOffset() + newBlock.headerSize() +
+          DataBlockEncoding.ID_SIZE,
+          newBlock.getUncompressedSizeWithoutHeader() -
+          DataBlockEncoding.ID_SIZE).slice();
+      return encodedBlock;
+    }
+
+    @Override
+    public boolean seekTo() throws IOException {
+      if (reader == null) {
+        return false;
+      }
+
+      if (reader.getTrailer().getEntryCount() == 0) {
+        // No data blocks.
+        return false;
+      }
+
+      long firstDataBlockOffset =
+          reader.getTrailer().getFirstDataBlockOffset();
+      if (block != null && block.getOffset() == firstDataBlockOffset) {
+        seeker.rewind();
+        return true;
+      }
+
+      block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
+          isCompaction, BlockType.DATA);
+      if (block.getOffset() < 0) {
+        throw new IOException("Invalid block offset: " + block.getOffset());
+      }
+      updateCurrentBlock(block);
+      return true;
+    }
+
+    @Override
+    public boolean next() throws IOException {
+      boolean isValid = seeker.next();
+      if (!isValid) {
+        block = readNextDataBlock();
+        isValid = block != null;
+        if (isValid) {
+          updateCurrentBlock(block);
+        }
+      }
+      return isValid;
+    }
+
+    @Override
+    public ByteBuffer getKey() {
+      assertValidSeek();
+      return seeker.getKeyDeepCopy();
+    }
+
+    @Override
+    public ByteBuffer getValue() {
+      assertValidSeek();
+      return seeker.getValueShallowCopy();
+    }
+
+    @Override
+    public KeyValue getKeyValue() {
+      if (block == null) {
+        return null;
+      }
+      return seeker.getKeyValue();
+    }
+
+    @Override
+    public String getKeyString() {
+      ByteBuffer keyBuffer = getKey();
+      return Bytes.toStringBinary(keyBuffer.array(),
+          keyBuffer.arrayOffset(), keyBuffer.limit());
+    }
+
+    @Override
+    public String getValueString() {
+      ByteBuffer valueBuffer = getValue();
+      return Bytes.toStringBinary(valueBuffer.array(),
+          valueBuffer.arrayOffset(), valueBuffer.limit());
+    }
+
+    private void assertValidSeek() {
+      if (block == null) {
+        throw new NotSeekedException();
+      }
+    }
+
+    @Override
+    protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
+      return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
+    }
+
+    @Override
+    protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
+        boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
+        throws IOException  {
+      if (block == null || block.getOffset() != seekToBlock.getOffset()) {
+        updateCurrentBlock(seekToBlock);
+      } else if (rewind) {
+        seeker.rewind();
+      }
+      this.nextIndexedKey = nextIndexedKey;
+      return seeker.seekToKeyInBlock(key, offset, length, seekBefore);
+    }
+  }
+
+  /**
+   * Returns a buffer with the Bloom filter metadata. The caller takes
+   * ownership of the buffer.
+   */
+  @Override
+  public DataInput getGeneralBloomFilterMetadata() throws IOException {
+    return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
+  }
+
+  @Override
+  public DataInput getDeleteBloomFilterMetadata() throws IOException {
+    return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
+  }
+
+  private DataInput getBloomFilterMetadata(BlockType blockType)
+  throws IOException {
+    if (blockType != BlockType.GENERAL_BLOOM_META &&
+        blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
+      throw new RuntimeException("Block Type: " + blockType.toString() +
+          " is not supported") ;
+    }
+
+    for (HFileBlock b : loadOnOpenBlocks)
+      if (b.getBlockType() == blockType)
+        return b.getByteStream();
+    return null;
+  }
+
+  @Override
+  public boolean isFileInfoLoaded() {
+    return true; // We load file info in constructor in version 2.
+  }
+
+  /**
+   * Validates that the minor version is within acceptable limits.
+   * Otherwise throws an Runtime exception
+   */
+  private void validateMinorVersion(Path path, int minorVersion) {
+    if (minorVersion < MIN_MINOR_VERSION ||
+        minorVersion > MAX_MINOR_VERSION) {
+      String msg = "Minor version for path " + path + 
+                   " is expected to be between " +
+                   MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION +
+                   " but is found to be " + minorVersion;
+      LOG.error(msg);
+      throw new RuntimeException(msg);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
new file mode 100644
index 0000000..b06878f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
@@ -0,0 +1,146 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+/**
+ * A scanner allows you to position yourself within a HFile and
+ * scan through it.  It allows you to reposition yourself as well.
+ *
+ * 
      A scanner doesn't always have a key/value that it is pointing to
+ * when it is first created and before
+ * {@link #seekTo()}/{@link #seekTo(byte[])} are called.
+ * In this case, {@link #getKey()}/{@link #getValue()} returns null.  At most
+ * other times, a key and value will be available.  The general pattern is that
+ * you position the Scanner using the seekTo variants and then getKey and
+ * getValue.
+ */
+public interface HFileScanner {
+  /**
+   * SeekTo or just before the passed key.  Examine the return
+   * code to figure whether we found the key or not.
+   * Consider the key stream of all the keys in the file,
+   * k[0] .. k[n], where there are n keys in the file.
+   * @param key Key to find.
+   * @return -1, if key < k[0], no position;
+   * 0, such that k[i] = key and scanner is left in position i; and
+   * 1, such that k[i] < key, and scanner is left in position i.
+   * The scanner will position itself between k[i] and k[i+1] where
+   * k[i] < key <= k[i+1].
+   * If there is no key k[i+1] greater than or equal to the input key, then the
+   * scanner will position itself at the end of the file and next() will return
+   * false when it is called.
+   * @throws IOException
+   */
+  public int seekTo(byte[] key) throws IOException;
+  public int seekTo(byte[] key, int offset, int length) throws IOException;
+  /**
+   * Reseek to or just before the passed key. Similar to seekTo
+   * except that this can be called even if the scanner is not at the beginning
+   * of a file.
+   * This can be used to seek only to keys which come after the current position
+   * of the scanner.
+   * Consider the key stream of all the keys in the file,
+   * k[0] .. k[n], where there are n keys in the file after
+   * current position of HFileScanner.
+   * The scanner will position itself between k[i] and k[i+1] where
+   * k[i] < key <= k[i+1].
+   * If there is no key k[i+1] greater than or equal to the input key, then the
+   * scanner will position itself at the end of the file and next() will return
+   * false when it is called.
+   * @param key Key to find (should be non-null)
+   * @return -1, if key < k[0], no position;
+   * 0, such that k[i] = key and scanner is left in position i; and
+   * 1, such that k[i] < key, and scanner is left in position i.
+   * @throws IOException
+   */
+  public int reseekTo(byte[] key) throws IOException;
+  public int reseekTo(byte[] key, int offset, int length) throws IOException;
+  /**
+   * Consider the key stream of all the keys in the file,
+   * k[0] .. k[n], where there are n keys in the file.
+   * @param key Key to find
+   * @return false if key <= k[0] or true with scanner in position 'i' such
+   * that: k[i] < key.  Furthermore: there may be a k[i+1], such that
+   * k[i] < key <= k[i+1] but there may also NOT be a k[i+1], and next() will
+   * return false (EOF).
+   * @throws IOException
+   */
+  public boolean seekBefore(byte [] key) throws IOException;
+  public boolean seekBefore(byte []key, int offset, int length) throws IOException;
+  /**
+   * Positions this scanner at the start of the file.
+   * @return False if empty file; i.e. a call to next would return false and
+   * the current key and value are undefined.
+   * @throws IOException
+   */
+  public boolean seekTo() throws IOException;
+  /**
+   * Scans to the next entry in the file.
+   * @return Returns false if you are at the end otherwise true if more in file.
+   * @throws IOException
+   */
+  public boolean next() throws IOException;
+  /**
+   * Gets a buffer view to the current key. You must call
+   * {@link #seekTo(byte[])} before this method.
+   * @return byte buffer for the key. The limit is set to the key size, and the
+   * position is 0, the start of the buffer view.
+   */
+  public ByteBuffer getKey();
+  /**
+   * Gets a buffer view to the current value.  You must call
+   * {@link #seekTo(byte[])} before this method.
+   *
+   * @return byte buffer for the value. The limit is set to the value size, and
+   * the position is 0, the start of the buffer view.
+   */
+  public ByteBuffer getValue();
+  /**
+   * @return Instance of {@link KeyValue}.
+   */
+  public KeyValue getKeyValue();
+  /**
+   * Convenience method to get a copy of the key as a string - interpreting the
+   * bytes as UTF8. You must call {@link #seekTo(byte[])} before this method.
+   * @return key as a string
+   */
+  public String getKeyString();
+  /**
+   * Convenience method to get a copy of the value as a string - interpreting
+   * the bytes as UTF8. You must call {@link #seekTo(byte[])} before this method.
+   * @return value as a string
+   */
+  public String getValueString();
+  /**
+   * @return Reader that underlies this Scanner instance.
+   */
+  public HFile.Reader getReader();
+  /**
+   * @return True is scanner has had one of the seek calls invoked; i.e.
+   * {@link #seekBefore(byte[])} or {@link #seekTo()} or {@link #seekTo(byte[])}.
+   * Otherwise returns false.
+   */
+  public boolean isSeeked();
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
new file mode 100644
index 0000000..ed0a1ba
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
@@ -0,0 +1,451 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.KeyComparator;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
+import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
+import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
+import org.apache.hadoop.hbase.regionserver.MemStore;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.hbase.util.BloomFilterWriter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.compress.Compressor;
+
+/**
+ * Writes version 1 HFiles. Mainly used for testing backwards-compatibility.
+ */
+public class HFileWriterV1 extends AbstractHFileWriter {
+
+  /** Meta data block name for bloom filter parameters. */
+  static final String BLOOM_FILTER_META_KEY = "BLOOM_FILTER_META";
+
+  /** Meta data block name for bloom filter bits. */
+  public static final String BLOOM_FILTER_DATA_KEY = "BLOOM_FILTER_DATA";
+
+  private static final Log LOG = LogFactory.getLog(HFileWriterV1.class);
+
+  // A stream made per block written.
+  private DataOutputStream out;
+
+  // Offset where the current block began.
+  private long blockBegin;
+
+  // First keys of every block.
+  private ArrayList blockKeys = new ArrayList();
+
+  // Block offset in backing stream.
+  private ArrayList blockOffsets = new ArrayList();
+
+  // Raw (decompressed) data size.
+  private ArrayList blockDataSizes = new ArrayList();
+
+  private Compressor compressor;
+
+  // Additional byte array output stream used to fill block cache
+  private ByteArrayOutputStream baos;
+  private DataOutputStream baosDos;
+  private int blockNumber = 0;
+
+  static class WriterFactoryV1 extends HFile.WriterFactory {
+    WriterFactoryV1(Configuration conf, CacheConfig cacheConf) {
+      super(conf, cacheConf);
+    }
+
+    @Override
+    public Writer createWriter(FileSystem fs, Path path,
+        FSDataOutputStream ostream, int blockSize,
+        Algorithm compressAlgo, HFileDataBlockEncoder dataBlockEncoder,
+        KeyComparator comparator, final ChecksumType checksumType,
+        final int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException {
+      // version 1 does not implement checksums
+      return new HFileWriterV1(conf, cacheConf, fs, path, ostream, blockSize,
+          compressAlgo, dataBlockEncoder, comparator);
+    }
+  }
+
+  /** Constructor that takes a path, creates and closes the output stream. */
+  public HFileWriterV1(Configuration conf, CacheConfig cacheConf,
+      FileSystem fs, Path path, FSDataOutputStream ostream,
+      int blockSize, Compression.Algorithm compress,
+      HFileDataBlockEncoder blockEncoder,
+      final KeyComparator comparator) throws IOException {
+    super(cacheConf, ostream == null ? createOutputStream(conf, fs, path) : ostream, path,
+        blockSize, compress, blockEncoder, comparator);
+    SchemaMetrics.configureGlobally(conf);
+  }
+
+  /**
+   * If at block boundary, opens new block.
+   *
+   * @throws IOException
+   */
+  private void checkBlockBoundary() throws IOException {
+    if (this.out != null && this.out.size() < blockSize)
+      return;
+    finishBlock();
+    newBlock();
+  }
+
+  /**
+   * Do the cleanup if a current block.
+   *
+   * @throws IOException
+   */
+  private void finishBlock() throws IOException {
+    if (this.out == null)
+      return;
+    long startTimeNs = System.nanoTime();
+
+    int size = releaseCompressingStream(this.out);
+    this.out = null;
+    blockKeys.add(firstKeyInBlock);
+    blockOffsets.add(Long.valueOf(blockBegin));
+    blockDataSizes.add(Integer.valueOf(size));
+    this.totalUncompressedBytes += size;
+
+    HFile.offerWriteLatency(System.nanoTime() - startTimeNs);
+    
+    if (cacheConf.shouldCacheDataOnWrite()) {
+      baosDos.flush();
+      // we do not do data block encoding on disk for HFile v1
+      byte[] bytes = baos.toByteArray();
+      HFileBlock block = new HFileBlock(BlockType.DATA,
+          (int) (outputStream.getPos() - blockBegin), bytes.length, -1,
+          ByteBuffer.wrap(bytes, 0, bytes.length), HFileBlock.FILL_HEADER,
+          blockBegin, MemStore.NO_PERSISTENT_TS, 
+          HFileBlock.MINOR_VERSION_NO_CHECKSUM,        // minor version
+          0,                                         // bytesPerChecksum
+          ChecksumType.NULL.getCode(),               // checksum type
+          (int) (outputStream.getPos() - blockBegin) +
+          HFileBlock.HEADER_SIZE_NO_CHECKSUM);       // onDiskDataSizeWithHeader
+
+      block = blockEncoder.diskToCacheFormat(block, false);
+      passSchemaMetricsTo(block);
+      cacheConf.getBlockCache().cacheBlock(
+          new BlockCacheKey(name, blockBegin, DataBlockEncoding.NONE,
+              block.getBlockType()), block);
+      baosDos.close();
+    }
+    blockNumber++;
+  }
+
+  /**
+   * Ready a new block for writing.
+   *
+   * @throws IOException
+   */
+  private void newBlock() throws IOException {
+    // This is where the next block begins.
+    blockBegin = outputStream.getPos();
+    this.out = getCompressingStream();
+    BlockType.DATA.write(out);
+    firstKeyInBlock = null;
+    if (cacheConf.shouldCacheDataOnWrite()) {
+      this.baos = new ByteArrayOutputStream();
+      this.baosDos = new DataOutputStream(baos);
+      baosDos.write(HFileBlock.DUMMY_HEADER_NO_CHECKSUM);
+    }
+  }
+
+  /**
+   * Sets up a compressor and creates a compression stream on top of
+   * this.outputStream. Get one per block written.
+   *
+   * @return A compressing stream; if 'none' compression, returned stream does
+   * not compress.
+   *
+   * @throws IOException
+   *
+   * @see {@link #releaseCompressingStream(DataOutputStream)}
+   */
+  private DataOutputStream getCompressingStream() throws IOException {
+    this.compressor = compressAlgo.getCompressor();
+    // Get new DOS compression stream. In tfile, the DOS, is not closed,
+    // just finished, and that seems to be fine over there. TODO: Check
+    // no memory retention of the DOS. Should I disable the 'flush' on the
+    // DOS as the BCFile over in tfile does? It wants to make it so flushes
+    // don't go through to the underlying compressed stream. Flush on the
+    // compressed downstream should be only when done. I was going to but
+    // looks like when we call flush in here, its legitimate flush that
+    // should go through to the compressor.
+    OutputStream os = this.compressAlgo.createCompressionStream(
+        this.outputStream, this.compressor, 0);
+    return new DataOutputStream(os);
+  }
+
+  /**
+   * Let go of block compressor and compressing stream gotten in call {@link
+   * #getCompressingStream}.
+   *
+   * @param dos
+   *
+   * @return How much was written on this stream since it was taken out.
+   *
+   * @see #getCompressingStream()
+   *
+   * @throws IOException
+   */
+  private int releaseCompressingStream(final DataOutputStream dos)
+      throws IOException {
+    dos.flush();
+    this.compressAlgo.returnCompressor(this.compressor);
+    this.compressor = null;
+    return dos.size();
+  }
+
+  /**
+   * Add a meta block to the end of the file. Call before close(). Metadata
+   * blocks are expensive. Fill one with a bunch of serialized data rather than
+   * do a metadata block per metadata instance. If metadata is small, consider
+   * adding to file info using {@link #appendFileInfo(byte[], byte[])}
+   *
+   * @param metaBlockName
+   *          name of the block
+   * @param content
+   *          will call readFields to get data later (DO NOT REUSE)
+   */
+  public void appendMetaBlock(String metaBlockName, Writable content) {
+    byte[] key = Bytes.toBytes(metaBlockName);
+    int i;
+    for (i = 0; i < metaNames.size(); ++i) {
+      // stop when the current key is greater than our own
+      byte[] cur = metaNames.get(i);
+      if (Bytes.BYTES_RAWCOMPARATOR.compare(cur, 0, cur.length, key, 0,
+          key.length) > 0) {
+        break;
+      }
+    }
+    metaNames.add(i, key);
+    metaData.add(i, content);
+  }
+
+  /**
+   * Add key/value to file. Keys must be added in an order that agrees with the
+   * Comparator passed on construction.
+   *
+   * @param kv
+   *          KeyValue to add. Cannot be empty nor null.
+   * @throws IOException
+   */
+  public void append(final KeyValue kv) throws IOException {
+    append(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(),
+        kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
+  }
+
+  /**
+   * Add key/value to file. Keys must be added in an order that agrees with the
+   * Comparator passed on construction.
+   *
+   * @param key
+   *          Key to add. Cannot be empty nor null.
+   * @param value
+   *          Value to add. Cannot be empty nor null.
+   * @throws IOException
+   */
+  public void append(final byte[] key, final byte[] value) throws IOException {
+    append(key, 0, key.length, value, 0, value.length);
+  }
+
+  /**
+   * Add key/value to file. Keys must be added in an order that agrees with the
+   * Comparator passed on construction.
+   *
+   * @param key
+   * @param koffset
+   * @param klength
+   * @param value
+   * @param voffset
+   * @param vlength
+   * @throws IOException
+   */
+  private void append(final byte[] key, final int koffset, final int klength,
+      final byte[] value, final int voffset, final int vlength)
+      throws IOException {
+    boolean dupKey = checkKey(key, koffset, klength);
+    checkValue(value, voffset, vlength);
+    if (!dupKey) {
+      checkBlockBoundary();
+    }
+    // Write length of key and value and then actual key and value bytes.
+    this.out.writeInt(klength);
+    totalKeyLength += klength;
+    this.out.writeInt(vlength);
+    totalValueLength += vlength;
+    this.out.write(key, koffset, klength);
+    this.out.write(value, voffset, vlength);
+    // Are we the first key in this block?
+    if (this.firstKeyInBlock == null) {
+      // Copy the key.
+      this.firstKeyInBlock = new byte[klength];
+      System.arraycopy(key, koffset, this.firstKeyInBlock, 0, klength);
+    }
+    this.lastKeyBuffer = key;
+    this.lastKeyOffset = koffset;
+    this.lastKeyLength = klength;
+    this.entryCount++;
+    // If we are pre-caching blocks on write, fill byte array stream
+    if (cacheConf.shouldCacheDataOnWrite()) {
+      this.baosDos.writeInt(klength);
+      this.baosDos.writeInt(vlength);
+      this.baosDos.write(key, koffset, klength);
+      this.baosDos.write(value, voffset, vlength);
+    }
+  }
+
+  public void close() throws IOException {
+    if (this.outputStream == null) {
+      return;
+    }
+    // Save data block encoder metadata in the file info.
+    blockEncoder.saveMetadata(this);
+    // Write out the end of the data blocks, then write meta data blocks.
+    // followed by fileinfo, data block index and meta block index.
+
+    finishBlock();
+
+    FixedFileTrailer trailer = new FixedFileTrailer(1,
+                                 HFileBlock.MINOR_VERSION_NO_CHECKSUM);
+
+    // Write out the metadata blocks if any.
+    ArrayList metaOffsets = null;
+    ArrayList metaDataSizes = null;
+    if (metaNames.size() > 0) {
+      metaOffsets = new ArrayList(metaNames.size());
+      metaDataSizes = new ArrayList(metaNames.size());
+      for (int i = 0; i < metaNames.size(); ++i) {
+        // store the beginning offset
+        long curPos = outputStream.getPos();
+        metaOffsets.add(curPos);
+        // write the metadata content
+        DataOutputStream dos = getCompressingStream();
+        BlockType.META.write(dos);
+        metaData.get(i).write(dos);
+        int size = releaseCompressingStream(dos);
+        // store the metadata size
+        metaDataSizes.add(size);
+      }
+    }
+
+    writeFileInfo(trailer, outputStream);
+
+    // Write the data block index.
+    trailer.setLoadOnOpenOffset(writeBlockIndex(this.outputStream,
+        this.blockKeys, this.blockOffsets, this.blockDataSizes));
+    LOG.info("Wrote a version 1 block index with " + this.blockKeys.size()
+        + " keys");
+
+    if (metaNames.size() > 0) {
+      // Write the meta index.
+      writeBlockIndex(this.outputStream, metaNames, metaOffsets, metaDataSizes);
+    }
+
+    // Now finish off the trailer.
+    trailer.setDataIndexCount(blockKeys.size());
+
+    finishClose(trailer);
+  }
+
+  @Override
+  protected void finishFileInfo() throws IOException {
+    super.finishFileInfo();
+
+    // In version 1, we store comparator name in the file info.
+    fileInfo.append(FileInfo.COMPARATOR,
+        Bytes.toBytes(comparator.getClass().getName()), false);
+  }
+
+  @Override
+  public void addInlineBlockWriter(InlineBlockWriter bloomWriter) {
+    // Inline blocks only exist in HFile format version 2.
+    throw new UnsupportedOperationException();
+  }
+
+  /**
+   * Version 1 general Bloom filters are stored in two meta blocks with two different
+   * keys.
+   */
+  @Override
+  public void addGeneralBloomFilter(BloomFilterWriter bfw) {
+    appendMetaBlock(BLOOM_FILTER_META_KEY,
+        bfw.getMetaWriter());
+    Writable dataWriter = bfw.getDataWriter();
+    if (dataWriter != null) {
+      appendMetaBlock(BLOOM_FILTER_DATA_KEY, dataWriter);
+    }
+  }
+
+  @Override
+  public void addDeleteFamilyBloomFilter(BloomFilterWriter bfw)
+  throws IOException {
+    throw new IOException("Delete Bloom filter is not supported in HFile V1");
+  }
+
+  /**
+   * Write out the index in the version 1 format. This conforms to the legacy
+   * version 1 format, but can still be read by
+   * {@link HFileBlockIndex.BlockIndexReader#readRootIndex(java.io.DataInputStream,
+   * int)}.
+   *
+   * @param out the stream to write to
+   * @param keys
+   * @param offsets
+   * @param uncompressedSizes in contrast with a version 2 root index format,
+   *          the sizes stored in the version 1 are uncompressed sizes
+   * @return
+   * @throws IOException
+   */
+  private static long writeBlockIndex(final FSDataOutputStream out,
+      final List keys, final List offsets,
+      final List uncompressedSizes) throws IOException {
+    long pos = out.getPos();
+    // Don't write an index if nothing in the index.
+    if (keys.size() > 0) {
+      BlockType.INDEX_V1.write(out);
+      // Write the index.
+      for (int i = 0; i < keys.size(); ++i) {
+        out.writeLong(offsets.get(i).longValue());
+        out.writeInt(uncompressedSizes.get(i).intValue());
+        byte[] key = keys.get(i);
+        Bytes.writeByteArray(out, key);
+      }
+    }
+    return pos;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
new file mode 100644
index 0000000..ceb4745
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
@@ -0,0 +1,485 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.KeyComparator;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.hbase.util.BloomFilterWriter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * Writes HFile format version 2.
+ */
+public class HFileWriterV2 extends AbstractHFileWriter {
+  static final Log LOG = LogFactory.getLog(HFileWriterV2.class);
+
+  /** Max memstore (mvcc) timestamp in FileInfo */
+  public static final byte [] MAX_MEMSTORE_TS_KEY =
+      Bytes.toBytes("MAX_MEMSTORE_TS_KEY");
+
+  /** KeyValue version in FileInfo */
+  public static final byte [] KEY_VALUE_VERSION =
+      Bytes.toBytes("KEY_VALUE_VERSION");
+
+  /** Version for KeyValue which includes memstore timestamp */
+  public static final int KEY_VALUE_VER_WITH_MEMSTORE = 1;
+
+  /** Inline block writers for multi-level block index and compound Blooms. */
+  private List inlineBlockWriters =
+      new ArrayList();
+
+  /** Unified version 2 block writer */
+  private HFileBlock.Writer fsBlockWriter;
+
+  private HFileBlockIndex.BlockIndexWriter dataBlockIndexWriter;
+  private HFileBlockIndex.BlockIndexWriter metaBlockIndexWriter;
+
+  /** The offset of the first data block or -1 if the file is empty. */
+  private long firstDataBlockOffset = -1;
+
+  /** The offset of the last data block or 0 if the file is empty. */
+  private long lastDataBlockOffset;
+
+  /** Additional data items to be written to the "load-on-open" section. */
+  private List additionalLoadOnOpenData =
+    new ArrayList();
+
+  /** Checksum related settings */
+  private ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE;
+  private int bytesPerChecksum = HFile.DEFAULT_BYTES_PER_CHECKSUM;
+
+  private final boolean includeMemstoreTS;
+  private long maxMemstoreTS = 0;
+
+  private int minorVersion = HFileReaderV2.MAX_MINOR_VERSION;
+
+  static class WriterFactoryV2 extends HFile.WriterFactory {
+    WriterFactoryV2(Configuration conf, CacheConfig cacheConf) {
+      super(conf, cacheConf);
+    }
+
+    @Override
+    public Writer createWriter(FileSystem fs, Path path,
+        FSDataOutputStream ostream, int blockSize,
+        Compression.Algorithm compress, HFileDataBlockEncoder blockEncoder,
+        final KeyComparator comparator, final ChecksumType checksumType,
+        final int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException {
+      return new HFileWriterV2(conf, cacheConf, fs, path, ostream, blockSize, compress,
+          blockEncoder, comparator, checksumType, bytesPerChecksum, includeMVCCReadpoint);
+    }
+  }
+
+  /** Constructor that takes a path, creates and closes the output stream. */
+  public HFileWriterV2(Configuration conf, CacheConfig cacheConf,
+      FileSystem fs, Path path, FSDataOutputStream ostream, int blockSize,
+      Compression.Algorithm compressAlgo, HFileDataBlockEncoder blockEncoder,
+      final KeyComparator comparator, final ChecksumType checksumType,
+      final int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException {
+    super(cacheConf,
+        ostream == null ? createOutputStream(conf, fs, path) : ostream,
+        path, blockSize, compressAlgo, blockEncoder, comparator);
+    SchemaMetrics.configureGlobally(conf);
+    this.checksumType = checksumType;
+    this.bytesPerChecksum = bytesPerChecksum;
+    this.includeMemstoreTS = includeMVCCReadpoint;
+    if (!conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, false)) {
+      this.minorVersion = 0;
+    }
+    finishInit(conf);
+  }
+
+  /** Additional initialization steps */
+  private void finishInit(final Configuration conf) {
+    if (fsBlockWriter != null)
+      throw new IllegalStateException("finishInit called twice");
+
+    // HFile filesystem-level (non-caching) block writer
+    fsBlockWriter = new HFileBlock.Writer(compressAlgo, blockEncoder,
+        includeMemstoreTS, minorVersion, checksumType, bytesPerChecksum);
+
+    // Data block index writer
+    boolean cacheIndexesOnWrite = cacheConf.shouldCacheIndexesOnWrite();
+    dataBlockIndexWriter = new HFileBlockIndex.BlockIndexWriter(fsBlockWriter,
+        cacheIndexesOnWrite ? cacheConf.getBlockCache(): null,
+        cacheIndexesOnWrite ? name : null);
+    dataBlockIndexWriter.setMaxChunkSize(
+        HFileBlockIndex.getMaxChunkSize(conf));
+    inlineBlockWriters.add(dataBlockIndexWriter);
+
+    // Meta data block index writer
+    metaBlockIndexWriter = new HFileBlockIndex.BlockIndexWriter();
+    LOG.debug("Initialized with " + cacheConf);
+
+    if (isSchemaConfigured()) {
+      schemaConfigurationChanged();
+    }
+  }
+
+  @Override
+  protected void schemaConfigurationChanged() {
+    passSchemaMetricsTo(dataBlockIndexWriter);
+    passSchemaMetricsTo(metaBlockIndexWriter);
+  }
+
+  /**
+   * At a block boundary, write all the inline blocks and opens new block.
+   *
+   * @throws IOException
+   */
+  private void checkBlockBoundary() throws IOException {
+    if (fsBlockWriter.blockSizeWritten() < blockSize)
+      return;
+
+    finishBlock();
+    writeInlineBlocks(false);
+    newBlock();
+  }
+
+  /** Clean up the current block */
+  private void finishBlock() throws IOException {
+    if (!fsBlockWriter.isWriting() || fsBlockWriter.blockSizeWritten() == 0)
+      return;
+
+    long startTimeNs = System.nanoTime();
+
+    // Update the first data block offset for scanning.
+    if (firstDataBlockOffset == -1) {
+      firstDataBlockOffset = outputStream.getPos();
+    }
+
+    // Update the last data block offset
+    lastDataBlockOffset = outputStream.getPos();
+
+    fsBlockWriter.writeHeaderAndData(outputStream);
+
+    int onDiskSize = fsBlockWriter.getOnDiskSizeWithHeader();
+    dataBlockIndexWriter.addEntry(firstKeyInBlock, lastDataBlockOffset,
+        onDiskSize);
+    totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
+
+    HFile.offerWriteLatency(System.nanoTime() - startTimeNs);
+    
+    if (cacheConf.shouldCacheDataOnWrite()) {
+      doCacheOnWrite(lastDataBlockOffset);
+    }
+  }
+
+  /** Gives inline block writers an opportunity to contribute blocks. */
+  private void writeInlineBlocks(boolean closing) throws IOException {
+    for (InlineBlockWriter ibw : inlineBlockWriters) {
+      while (ibw.shouldWriteBlock(closing)) {
+        long offset = outputStream.getPos();
+        boolean cacheThisBlock = ibw.cacheOnWrite();
+        ibw.writeInlineBlock(fsBlockWriter.startWriting(
+            ibw.getInlineBlockType()));
+        fsBlockWriter.writeHeaderAndData(outputStream);
+        ibw.blockWritten(offset, fsBlockWriter.getOnDiskSizeWithHeader(),
+            fsBlockWriter.getUncompressedSizeWithoutHeader());
+        totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
+
+        if (cacheThisBlock) {
+          doCacheOnWrite(offset);
+        }
+      }
+    }
+  }
+
+  /**
+   * Caches the last written HFile block.
+   * @param offset the offset of the block we want to cache. Used to determine
+   *          the cache key.
+   */
+  private void doCacheOnWrite(long offset) {
+    // We don't cache-on-write data blocks on compaction, so assume this is not
+    // a compaction.
+    final boolean isCompaction = false;
+    HFileBlock cacheFormatBlock = blockEncoder.diskToCacheFormat(
+        fsBlockWriter.getBlockForCaching(), isCompaction);
+    passSchemaMetricsTo(cacheFormatBlock);
+    cacheConf.getBlockCache().cacheBlock(
+        new BlockCacheKey(name, offset, blockEncoder.getEncodingInCache(),
+            cacheFormatBlock.getBlockType()), cacheFormatBlock);
+  }
+
+  /**
+   * Ready a new block for writing.
+   *
+   * @throws IOException
+   */
+  private void newBlock() throws IOException {
+    // This is where the next block begins.
+    fsBlockWriter.startWriting(BlockType.DATA);
+    firstKeyInBlock = null;
+  }
+
+  /**
+   * Add a meta block to the end of the file. Call before close(). Metadata
+   * blocks are expensive. Fill one with a bunch of serialized data rather than
+   * do a metadata block per metadata instance. If metadata is small, consider
+   * adding to file info using {@link #appendFileInfo(byte[], byte[])}
+   *
+   * @param metaBlockName
+   *          name of the block
+   * @param content
+   *          will call readFields to get data later (DO NOT REUSE)
+   */
+  @Override
+  public void appendMetaBlock(String metaBlockName, Writable content) {
+    byte[] key = Bytes.toBytes(metaBlockName);
+    int i;
+    for (i = 0; i < metaNames.size(); ++i) {
+      // stop when the current key is greater than our own
+      byte[] cur = metaNames.get(i);
+      if (Bytes.BYTES_RAWCOMPARATOR.compare(cur, 0, cur.length, key, 0,
+          key.length) > 0) {
+        break;
+      }
+    }
+    metaNames.add(i, key);
+    metaData.add(i, content);
+  }
+
+  /**
+   * Add key/value to file. Keys must be added in an order that agrees with the
+   * Comparator passed on construction.
+   *
+   * @param kv
+   *          KeyValue to add. Cannot be empty nor null.
+   * @throws IOException
+   */
+  @Override
+  public void append(final KeyValue kv) throws IOException {
+    append(kv.getMemstoreTS(), kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(),
+        kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
+    this.maxMemstoreTS = Math.max(this.maxMemstoreTS, kv.getMemstoreTS());
+  }
+
+  /**
+   * Add key/value to file. Keys must be added in an order that agrees with the
+   * Comparator passed on construction.
+   *
+   * @param key
+   *          Key to add. Cannot be empty nor null.
+   * @param value
+   *          Value to add. Cannot be empty nor null.
+   * @throws IOException
+   */
+  @Override
+  public void append(final byte[] key, final byte[] value) throws IOException {
+    append(0, key, 0, key.length, value, 0, value.length);
+  }
+
+  /**
+   * Add key/value to file. Keys must be added in an order that agrees with the
+   * Comparator passed on construction.
+   *
+   * @param key
+   * @param koffset
+   * @param klength
+   * @param value
+   * @param voffset
+   * @param vlength
+   * @throws IOException
+   */
+  private void append(final long memstoreTS, final byte[] key, final int koffset, final int klength,
+      final byte[] value, final int voffset, final int vlength)
+      throws IOException {
+    boolean dupKey = checkKey(key, koffset, klength);
+    checkValue(value, voffset, vlength);
+    if (!dupKey) {
+      checkBlockBoundary();
+    }
+
+    if (!fsBlockWriter.isWriting())
+      newBlock();
+
+    // Write length of key and value and then actual key and value bytes.
+    // Additionally, we may also write down the memstoreTS.
+    {
+      DataOutputStream out = fsBlockWriter.getUserDataStream();
+      out.writeInt(klength);
+      totalKeyLength += klength;
+      out.writeInt(vlength);
+      totalValueLength += vlength;
+      out.write(key, koffset, klength);
+      out.write(value, voffset, vlength);
+      if (this.includeMemstoreTS) {
+        WritableUtils.writeVLong(out, memstoreTS);
+      }
+    }
+
+    // Are we the first key in this block?
+    if (firstKeyInBlock == null) {
+      // Copy the key.
+      firstKeyInBlock = new byte[klength];
+      System.arraycopy(key, koffset, firstKeyInBlock, 0, klength);
+    }
+
+    lastKeyBuffer = key;
+    lastKeyOffset = koffset;
+    lastKeyLength = klength;
+    entryCount++;
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (outputStream == null) {
+      return;
+    }
+    // Save data block encoder metadata in the file info.
+    blockEncoder.saveMetadata(this);
+    // Write out the end of the data blocks, then write meta data blocks.
+    // followed by fileinfo, data block index and meta block index.
+
+    finishBlock();
+    writeInlineBlocks(true);
+
+    FixedFileTrailer trailer = new FixedFileTrailer(2, minorVersion);
+
+    // Write out the metadata blocks if any.
+    if (!metaNames.isEmpty()) {
+      for (int i = 0; i < metaNames.size(); ++i) {
+        // store the beginning offset
+        long offset = outputStream.getPos();
+        // write the metadata content
+        DataOutputStream dos = fsBlockWriter.startWriting(BlockType.META);
+        metaData.get(i).write(dos);
+
+        fsBlockWriter.writeHeaderAndData(outputStream);
+        totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
+
+        // Add the new meta block to the meta index.
+        metaBlockIndexWriter.addEntry(metaNames.get(i), offset,
+            fsBlockWriter.getOnDiskSizeWithHeader());
+      }
+    }
+
+    // Load-on-open section.
+
+    // Data block index.
+    //
+    // In version 2, this section of the file starts with the root level data
+    // block index. We call a function that writes intermediate-level blocks
+    // first, then root level, and returns the offset of the root level block
+    // index.
+
+    long rootIndexOffset = dataBlockIndexWriter.writeIndexBlocks(outputStream);
+    trailer.setLoadOnOpenOffset(rootIndexOffset);
+
+    // Meta block index.
+    metaBlockIndexWriter.writeSingleLevelIndex(fsBlockWriter.startWriting(
+        BlockType.ROOT_INDEX), "meta");
+    fsBlockWriter.writeHeaderAndData(outputStream);
+    totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
+
+    if (this.includeMemstoreTS) {
+      appendFileInfo(MAX_MEMSTORE_TS_KEY, Bytes.toBytes(maxMemstoreTS));
+      appendFileInfo(KEY_VALUE_VERSION, Bytes.toBytes(KEY_VALUE_VER_WITH_MEMSTORE));
+    }
+
+    // File info
+    writeFileInfo(trailer, fsBlockWriter.startWriting(BlockType.FILE_INFO));
+    fsBlockWriter.writeHeaderAndData(outputStream);
+    totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
+
+    // Load-on-open data supplied by higher levels, e.g. Bloom filters.
+    for (BlockWritable w : additionalLoadOnOpenData){
+      fsBlockWriter.writeBlock(w, outputStream);
+      totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
+    }
+
+    // Now finish off the trailer.
+    trailer.setNumDataIndexLevels(dataBlockIndexWriter.getNumLevels());
+    trailer.setUncompressedDataIndexSize(
+        dataBlockIndexWriter.getTotalUncompressedSize());
+    trailer.setFirstDataBlockOffset(firstDataBlockOffset);
+    trailer.setLastDataBlockOffset(lastDataBlockOffset);
+    trailer.setComparatorClass(comparator.getClass());
+    trailer.setDataIndexCount(dataBlockIndexWriter.getNumRootEntries());
+
+
+    finishClose(trailer);
+
+    fsBlockWriter.releaseCompressor();
+  }
+
+  @Override
+  public void addInlineBlockWriter(InlineBlockWriter ibw) {
+    inlineBlockWriters.add(ibw);
+  }
+
+  @Override
+  public void addGeneralBloomFilter(final BloomFilterWriter bfw) {
+    this.addBloomFilter(bfw, BlockType.GENERAL_BLOOM_META);
+  }
+
+  @Override
+  public void addDeleteFamilyBloomFilter(final BloomFilterWriter bfw) {
+    this.addBloomFilter(bfw, BlockType.DELETE_FAMILY_BLOOM_META);
+  }
+
+  private void addBloomFilter(final BloomFilterWriter bfw,
+      final BlockType blockType) {
+    if (bfw.getKeyCount() <= 0)
+      return;
+
+    if (blockType != BlockType.GENERAL_BLOOM_META &&
+        blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
+      throw new RuntimeException("Block Type: " + blockType.toString() +
+          "is not supported");
+    }
+    additionalLoadOnOpenData.add(new BlockWritable() {
+      @Override
+      public BlockType getBlockType() {
+        return blockType;
+      }
+
+      @Override
+      public void writeToBlock(DataOutput out) throws IOException {
+        bfw.getMetaWriter().write(out);
+        Writable dataWriter = bfw.getDataWriter();
+        if (dataWriter != null)
+          dataWriter.write(out);
+      }
+    });
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/InlineBlockWriter.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/InlineBlockWriter.java
new file mode 100644
index 0000000..c384036
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/InlineBlockWriter.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * A way to write "inline" blocks into an {@link HFile}. Inline blocks are
+ * interspersed with data blocks. For example, Bloom filter chunks and
+ * leaf-level blocks of a multi-level block index are stored as inline blocks.
+ */
+public interface InlineBlockWriter {
+
+  /**
+   * Determines whether there is a new block to be written out.
+   *
+   * @param closing
+   *          whether the file is being closed, in which case we need to write
+   *          out all available data and not wait to accumulate another block
+   */
+  boolean shouldWriteBlock(boolean closing);
+
+  /**
+   * Writes the block to the provided stream. Must not write any magic records.
+   * Called only if {@link #shouldWriteBlock(boolean)} returned true.
+   *
+   * @param out
+   *          a stream (usually a compressing stream) to write the block to
+   */
+  void writeInlineBlock(DataOutput out) throws IOException;
+
+  /**
+   * Called after a block has been written, and its offset, raw size, and
+   * compressed size have been determined. Can be used to add an entry to a
+   * block index. If this type of inline blocks needs a block index, the inline
+   * block writer is responsible for maintaining it.
+   *
+   * @param offset the offset of the block in the stream
+   * @param onDiskSize the on-disk size of the block
+   * @param uncompressedSize the uncompressed size of the block
+   */
+  void blockWritten(long offset, int onDiskSize, int uncompressedSize);
+
+  /**
+   * The type of blocks this block writer produces.
+   */
+  BlockType getInlineBlockType();
+
+  /**
+   * @return true if inline blocks produced by this writer should be cached
+   */
+  boolean cacheOnWrite();
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/InvalidHFileException.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/InvalidHFileException.java
new file mode 100644
index 0000000..87c7e07
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/InvalidHFileException.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.IOException;
+
+/**
+ * Thrown when an invalid HFile format is detected
+ */
+public class InvalidHFileException extends IOException {
+  private static final long serialVersionUID = 4660352028739861249L;
+
+  /** constructor */
+  public InvalidHFileException() {
+    super();
+  }
+
+  /**
+   * Constructor
+   * @param s message
+   */
+  public InvalidHFileException(String s) {
+    super(s);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
new file mode 100644
index 0000000..6533c93
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -0,0 +1,829 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.IOException;
+import java.lang.ref.WeakReference;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HasThread;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * A block cache implementation that is memory-aware using {@link HeapSize},
+ * memory-bound using an LRU eviction algorithm, and concurrent: backed by a
+ * {@link ConcurrentHashMap} and with a non-blocking eviction thread giving
+ * constant-time {@link #cacheBlock} and {@link #getBlock} operations.
      
+ *
+ * Contains three levels of block priority to allow for
+ * scan-resistance and in-memory families.  A block is added with an inMemory
+ * flag if necessary, otherwise a block becomes a single access priority.  Once
+ * a blocked is accessed again, it changes to multiple access.  This is used
+ * to prevent scans from thrashing the cache, adding a least-frequently-used
+ * element to the eviction algorithm.
      
+ *
+ * Each priority is given its own chunk of the total cache to ensure
+ * fairness during eviction.  Each priority will retain close to its maximum
+ * size, however, if any priority is not using its entire chunk the others
+ * are able to grow beyond their chunk size.
      
+ *
+ * Instantiated at a minimum with the total size and average block size.
+ * All sizes are in bytes.  The block size is not especially important as this
+ * cache is fully dynamic in its sizing of blocks.  It is only used for
+ * pre-allocating data structures and in initial heap estimation of the map.
      
+ *
+ * The detailed constructor defines the sizes for the three priorities (they
+ * should total to the maximum size defined).  It also sets the levels that
+ * trigger and control the eviction thread.
      
+ *
+ * The acceptable size is the cache size level which triggers the eviction
+ * process to start.  It evicts enough blocks to get the size below the
+ * minimum size specified.
      
+ *
+ * Eviction happens in a separate thread and involves a single full-scan
+ * of the map.  It determines how many bytes must be freed to reach the minimum
+ * size, and then while scanning determines the fewest least-recently-used
+ * blocks necessary from each of the three priorities (would be 3 times bytes
+ * to free).  It then uses the priority chunk sizes to evict fairly according
+ * to the relative sizes and usage.
+ */
+public class LruBlockCache implements BlockCache, HeapSize {
+
+  static final Log LOG = LogFactory.getLog(LruBlockCache.class);
+
+  static final String LRU_MIN_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.min.factor";
+  static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.acceptable.factor";
+
+  /** Default Configuration Parameters*/
+
+  /** Backing Concurrent Map Configuration */
+  static final float DEFAULT_LOAD_FACTOR = 0.75f;
+  static final int DEFAULT_CONCURRENCY_LEVEL = 16;
+
+  /** Eviction thresholds */
+  static final float DEFAULT_MIN_FACTOR = 0.75f;
+  static final float DEFAULT_ACCEPTABLE_FACTOR = 0.85f;
+
+  /** Priority buckets */
+  static final float DEFAULT_SINGLE_FACTOR = 0.25f;
+  static final float DEFAULT_MULTI_FACTOR = 0.50f;
+  static final float DEFAULT_MEMORY_FACTOR = 0.25f;
+
+  /** Statistics thread */
+  static final int statThreadPeriod = 60 * 5;
+
+  /** Concurrent map (the cache) */
+  private final ConcurrentHashMap map;
+
+  /** Eviction lock (locked when eviction in process) */
+  private final ReentrantLock evictionLock = new ReentrantLock(true);
+
+  /** Volatile boolean to track if we are in an eviction process or not */
+  private volatile boolean evictionInProgress = false;
+
+  /** Eviction thread */
+  private final EvictionThread evictionThread;
+
+  /** Statistics thread schedule pool (for heavy debugging, could remove) */
+  private final ScheduledExecutorService scheduleThreadPool =
+    Executors.newScheduledThreadPool(1,
+      new ThreadFactoryBuilder()
+        .setNameFormat("LRU Statistics #%d")
+        .setDaemon(true)
+        .build());
+
+  /** Current size of cache */
+  private final AtomicLong size;
+
+  /** Current number of cached elements */
+  private final AtomicLong elements;
+
+  /** Cache access count (sequential ID) */
+  private final AtomicLong count;
+
+  /** Cache statistics */
+  private final CacheStats stats;
+
+  /** Maximum allowable size of cache (block put if size > max, evict) */
+  private long maxSize;
+
+  /** Approximate block size */
+  private long blockSize;
+
+  /** Acceptable size of cache (no evictions if size < acceptable) */
+  private float acceptableFactor;
+
+  /** Minimum threshold of cache (when evicting, evict until size < min) */
+  private float minFactor;
+
+  /** Single access bucket size */
+  private float singleFactor;
+
+  /** Multiple access bucket size */
+  private float multiFactor;
+
+  /** In-memory bucket size */
+  private float memoryFactor;
+
+  /** Overhead of the structure itself */
+  private long overhead;
+
+  /**
+   * Default constructor.  Specify maximum size and expected average block
+   * size (approximation is fine).
+   *
+   * 
      All other factors will be calculated based on defaults specified in
+   * this class.
+   * @param maxSize maximum size of cache, in bytes
+   * @param blockSize approximate size of each block, in bytes
+   * @param conf configuration
+   */
+  public LruBlockCache(long maxSize, long blockSize, Configuration conf) {
+    this(maxSize, blockSize, true, conf);
+  }
+
+  /**
+   * Constructor used for testing.  Allows disabling of the eviction thread.
+   */
+  public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, Configuration conf) {
+    this(maxSize, blockSize, evictionThread,
+        (int)Math.ceil(1.2*maxSize/blockSize),
+        DEFAULT_LOAD_FACTOR,
+        DEFAULT_CONCURRENCY_LEVEL,
+        conf.getFloat(LRU_MIN_FACTOR_CONFIG_NAME, DEFAULT_MIN_FACTOR),
+        conf.getFloat(LRU_ACCEPTABLE_FACTOR_CONFIG_NAME, DEFAULT_ACCEPTABLE_FACTOR),
+        DEFAULT_SINGLE_FACTOR,
+        DEFAULT_MULTI_FACTOR,
+        DEFAULT_MEMORY_FACTOR);
+  }
+
+
+  /**
+   * Configurable constructor.  Use this constructor if not using defaults.
+   * @param maxSize maximum size of this cache, in bytes
+   * @param blockSize expected average size of blocks, in bytes
+   * @param evictionThread whether to run evictions in a bg thread or not
+   * @param mapInitialSize initial size of backing ConcurrentHashMap
+   * @param mapLoadFactor initial load factor of backing ConcurrentHashMap
+   * @param mapConcurrencyLevel initial concurrency factor for backing CHM
+   * @param minFactor percentage of total size that eviction will evict until
+   * @param acceptableFactor percentage of total size that triggers eviction
+   * @param singleFactor percentage of total size for single-access blocks
+   * @param multiFactor percentage of total size for multiple-access blocks
+   * @param memoryFactor percentage of total size for in-memory blocks
+   */
+  public LruBlockCache(long maxSize, long blockSize, boolean evictionThread,
+      int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel,
+      float minFactor, float acceptableFactor,
+      float singleFactor, float multiFactor, float memoryFactor) {
+    if(singleFactor + multiFactor + memoryFactor != 1) {
+      throw new IllegalArgumentException("Single, multi, and memory factors " +
+          " should total 1.0");
+    }
+    if(minFactor >= acceptableFactor) {
+      throw new IllegalArgumentException("minFactor must be smaller than acceptableFactor");
+    }
+    if(minFactor >= 1.0f || acceptableFactor >= 1.0f) {
+      throw new IllegalArgumentException("all factors must be < 1");
+    }
+    this.maxSize = maxSize;
+    this.blockSize = blockSize;
+    map = new ConcurrentHashMap(mapInitialSize,
+        mapLoadFactor, mapConcurrencyLevel);
+    this.minFactor = minFactor;
+    this.acceptableFactor = acceptableFactor;
+    this.singleFactor = singleFactor;
+    this.multiFactor = multiFactor;
+    this.memoryFactor = memoryFactor;
+    this.stats = new CacheStats();
+    this.count = new AtomicLong(0);
+    this.elements = new AtomicLong(0);
+    this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel);
+    this.size = new AtomicLong(this.overhead);
+    if(evictionThread) {
+      this.evictionThread = new EvictionThread(this);
+      this.evictionThread.start(); // FindBugs SC_START_IN_CTOR
+    } else {
+      this.evictionThread = null;
+    }
+    this.scheduleThreadPool.scheduleAtFixedRate(new StatisticsThread(this),
+        statThreadPeriod, statThreadPeriod, TimeUnit.SECONDS);
+  }
+
+  public void setMaxSize(long maxSize) {
+    this.maxSize = maxSize;
+    if(this.size.get() > acceptableSize() && !evictionInProgress) {
+      runEviction();
+    }
+  }
+
+  // BlockCache implementation
+
+  /**
+   * Cache the block with the specified name and buffer.
+   * 
      
+   * It is assumed this will NOT be called on an already cached block. In rare cases (HBASE-8547)
+   * this can happen, for which we compare the buffer contents.
+   * @param cacheKey block's cache key
+   * @param buf block buffer
+   * @param inMemory if block is in-memory
+   */
+  @Override
+  public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
+    CachedBlock cb = map.get(cacheKey);
+    if(cb != null) {
+      // compare the contents, if they are not equal, we are in big trouble
+      if (compare(buf, cb.getBuffer()) != 0) {
+        throw new RuntimeException("Cached block contents differ, which should not have happened."
+          + "cacheKey:" + cacheKey);
+      }
+      String msg = "Cached an already cached block: " + cacheKey + " cb:" + cb.getCacheKey();
+      msg += ". This is harmless and can happen in rare cases (see HBASE-8547)";
+      LOG.warn(msg);
+      return;
+    }
+    cb = new CachedBlock(cacheKey, buf, count.incrementAndGet(), inMemory);
+    long newSize = updateSizeMetrics(cb, false);
+    map.put(cacheKey, cb);
+    elements.incrementAndGet();
+    if(newSize > acceptableSize() && !evictionInProgress) {
+      runEviction();
+    }
+  }
+
+  private int compare(Cacheable left, Cacheable right) {
+    ByteBuffer l = ByteBuffer.allocate(left.getSerializedLength());
+    left.serialize(l);
+    ByteBuffer r = ByteBuffer.allocate(right.getSerializedLength());
+    right.serialize(r);
+    return Bytes.compareTo(l.array(), l.arrayOffset(), l.limit(),
+      r.array(), r.arrayOffset(), r.limit());
+  }
+
+  /**
+   * Cache the block with the specified name and buffer.
+   * 
      
+   * It is assumed this will NEVER be called on an already cached block.  If
+   * that is done, it is assumed that you are reinserting the same exact
+   * block due to a race condition and will update the buffer but not modify
+   * the size of the cache.
+   * @param cacheKey block's cache key
+   * @param buf block buffer
+   */
+  public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
+    cacheBlock(cacheKey, buf, false);
+  }
+
+  /**
+   * Helper function that updates the local size counter and also updates any
+   * per-cf or per-blocktype metrics it can discern from given
+   * {@link CachedBlock}
+   *
+   * @param cb
+   * @param evict
+   */
+  protected long updateSizeMetrics(CachedBlock cb, boolean evict) {
+    long heapsize = cb.heapSize();
+    if (evict) {
+      heapsize *= -1;
+    }
+    Cacheable cachedBlock = cb.getBuffer();
+    SchemaMetrics schemaMetrics = cachedBlock.getSchemaMetrics();
+    if (schemaMetrics != null) {
+      schemaMetrics.updateOnCachePutOrEvict(
+          cachedBlock.getBlockType().getCategory(), heapsize, evict);
+    }
+    return size.addAndGet(heapsize);
+  }
+
+  /**
+   * Get the buffer of the block with the specified name.
+   * @param cacheKey block's cache key
+   * @param caching true if the caller caches blocks on cache misses
+   * @param repeat Whether this is a repeat lookup for the same block
+   *        (used to avoid double counting cache misses when doing double-check locking)
+   *        {@see HFileReaderV2#readBlock(long, long, boolean, boolean, boolean, BlockType)}
+   * @return buffer of specified cache key, or null if not in cache
+   */
+  @Override
+  public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) {
+    CachedBlock cb = map.get(cacheKey);
+    if(cb == null) {
+      if (!repeat) stats.miss(caching);
+      return null;
+    }
+    stats.hit(caching);
+    cb.access(count.incrementAndGet());
+    return cb.getBuffer();
+  }
+
+
+  @Override
+  public boolean evictBlock(BlockCacheKey cacheKey) {
+    CachedBlock cb = map.get(cacheKey);
+    if (cb == null) return false;
+    evictBlock(cb);
+    return true;
+  }
+
+  /**
+   * Evicts all blocks for a specific HFile. This is an
+   * expensive operation implemented as a linear-time search through all blocks
+   * in the cache. Ideally this should be a search in a log-access-time map.
+   *
+   * 
      
+   * This is used for evict-on-close to remove all blocks of a specific HFile.
+   *
+   * @return the number of blocks evicted
+   */
+  @Override
+  public int evictBlocksByHfileName(String hfileName) {
+    int numEvicted = 0;
+    for (BlockCacheKey key : map.keySet()) {
+      if (key.getHfileName().equals(hfileName)) {
+        if (evictBlock(key))
+          ++numEvicted;
+      }
+    }
+    return numEvicted;
+  }
+
+  protected long evictBlock(CachedBlock block) {
+    map.remove(block.getCacheKey());
+    updateSizeMetrics(block, true);
+    elements.decrementAndGet();
+    stats.evicted();
+    return block.heapSize();
+  }
+
+  /**
+   * Multi-threaded call to run the eviction process.
+   */
+  private void runEviction() {
+    if(evictionThread == null) {
+      evict();
+    } else {
+      evictionThread.evict();
+    }
+  }
+
+  /**
+   * Eviction method.
+   */
+  void evict() {
+
+    // Ensure only one eviction at a time
+    if(!evictionLock.tryLock()) return;
+
+    try {
+      evictionInProgress = true;
+      long currentSize = this.size.get();
+      long bytesToFree = currentSize - minSize();
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Block cache LRU eviction started; Attempting to free " +
+          StringUtils.byteDesc(bytesToFree) + " of total=" +
+          StringUtils.byteDesc(currentSize));
+      }
+
+      if(bytesToFree <= 0) return;
+
+      // Instantiate priority buckets
+      BlockBucket bucketSingle = new BlockBucket(bytesToFree, blockSize,
+          singleSize());
+      BlockBucket bucketMulti = new BlockBucket(bytesToFree, blockSize,
+          multiSize());
+      BlockBucket bucketMemory = new BlockBucket(bytesToFree, blockSize,
+          memorySize());
+
+      // Scan entire map putting into appropriate buckets
+      for(CachedBlock cachedBlock : map.values()) {
+        switch(cachedBlock.getPriority()) {
+          case SINGLE: {
+            bucketSingle.add(cachedBlock);
+            break;
+          }
+          case MULTI: {
+            bucketMulti.add(cachedBlock);
+            break;
+          }
+          case MEMORY: {
+            bucketMemory.add(cachedBlock);
+            break;
+          }
+        }
+      }
+
+      PriorityQueue bucketQueue =
+        new PriorityQueue(3);
+
+      bucketQueue.add(bucketSingle);
+      bucketQueue.add(bucketMulti);
+      bucketQueue.add(bucketMemory);
+
+      int remainingBuckets = 3;
+      long bytesFreed = 0;
+
+      BlockBucket bucket;
+      while((bucket = bucketQueue.poll()) != null) {
+        long overflow = bucket.overflow();
+        if(overflow > 0) {
+          long bucketBytesToFree = Math.min(overflow,
+            (bytesToFree - bytesFreed) / remainingBuckets);
+          bytesFreed += bucket.free(bucketBytesToFree);
+        }
+        remainingBuckets--;
+      }
+
+      if (LOG.isDebugEnabled()) {
+        long single = bucketSingle.totalSize();
+        long multi = bucketMulti.totalSize();
+        long memory = bucketMemory.totalSize();
+        LOG.debug("Block cache LRU eviction completed; " +
+          "freed=" + StringUtils.byteDesc(bytesFreed) + ", " +
+          "total=" + StringUtils.byteDesc(this.size.get()) + ", " +
+          "single=" + StringUtils.byteDesc(single) + ", " +
+          "multi=" + StringUtils.byteDesc(multi) + ", " +
+          "memory=" + StringUtils.byteDesc(memory));
+      }
+    } finally {
+      stats.evict();
+      evictionInProgress = false;
+      evictionLock.unlock();
+    }
+  }
+
+  /**
+   * Used to group blocks into priority buckets.  There will be a BlockBucket
+   * for each priority (single, multi, memory).  Once bucketed, the eviction
+   * algorithm takes the appropriate number of elements out of each according
+   * to configuration parameters and their relatives sizes.
+   */
+  private class BlockBucket implements Comparable {
+
+    private CachedBlockQueue queue;
+    private long totalSize = 0;
+    private long bucketSize;
+
+    public BlockBucket(long bytesToFree, long blockSize, long bucketSize) {
+      this.bucketSize = bucketSize;
+      queue = new CachedBlockQueue(bytesToFree, blockSize);
+      totalSize = 0;
+    }
+
+    public void add(CachedBlock block) {
+      totalSize += block.heapSize();
+      queue.add(block);
+    }
+
+    public long free(long toFree) {
+      CachedBlock cb;
+      long freedBytes = 0;
+      while ((cb = queue.pollLast()) != null) {
+        freedBytes += evictBlock(cb);
+        if (freedBytes >= toFree) {
+          return freedBytes;
+        }
+      }
+      return freedBytes;
+    }
+
+    public long overflow() {
+      return totalSize - bucketSize;
+    }
+
+    public long totalSize() {
+      return totalSize;
+    }
+
+    public int compareTo(BlockBucket that) {
+      if(this.overflow() == that.overflow()) return 0;
+      return this.overflow() > that.overflow() ? 1 : -1;
+    }
+  }
+
+  /**
+   * Get the maximum size of this cache.
+   * @return max size in bytes
+   */
+  public long getMaxSize() {
+    return this.maxSize;
+  }
+
+  /**
+   * Get the current size of this cache.
+   * @return current size in bytes
+   */
+  public long getCurrentSize() {
+    return this.size.get();
+  }
+
+  /**
+   * Get the current size of this cache.
+   * @return current size in bytes
+   */
+  public long getFreeSize() {
+    return getMaxSize() - getCurrentSize();
+  }
+
+  /**
+   * Get the size of this cache (number of cached blocks)
+   * @return number of cached blocks
+   */
+  public long size() {
+    return this.elements.get();
+  }
+
+  @Override
+  public long getBlockCount() {
+    return this.elements.get();
+  }
+
+  /**
+   * Get the number of eviction runs that have occurred
+   */
+  public long getEvictionCount() {
+    return this.stats.getEvictionCount();
+  }
+
+  /**
+   * Get the number of blocks that have been evicted during the lifetime
+   * of this cache.
+   */
+  public long getEvictedCount() {
+    return this.stats.getEvictedCount();
+  }
+
+  EvictionThread getEvictionThread() {
+    return this.evictionThread;
+  }
+
+  /*
+   * Eviction thread.  Sits in waiting state until an eviction is triggered
+   * when the cache size grows above the acceptable level.
      
+   *
+   * Thread is triggered into action by {@link LruBlockCache#runEviction()}
+   */
+  static class EvictionThread extends HasThread {
+    private WeakReference cache;
+    private boolean go = true;
+    // flag set after enter the run method, used for test
+    private boolean enteringRun = false;
+
+    public EvictionThread(LruBlockCache cache) {
+      super(Thread.currentThread().getName() + ".LruBlockCache.EvictionThread");
+      setDaemon(true);
+      this.cache = new WeakReference(cache);
+    }
+
+    @Override
+    public void run() {
+      enteringRun = true;
+      while (this.go) {
+        synchronized(this) {
+          try {
+            this.wait();
+          } catch(InterruptedException e) {}
+        }
+        LruBlockCache cache = this.cache.get();
+        if(cache == null) break;
+        cache.evict();
+      }
+    }
+
+    public void evict() {
+      synchronized(this) {
+        this.notify(); // FindBugs NN_NAKED_NOTIFY
+      }
+    }
+
+    void shutdown() {
+      this.go = false;
+      interrupt();
+    }
+
+    /**
+     * Used for the test.
+     */
+    boolean isEnteringRun() {
+      return this.enteringRun;
+    }
+  }
+
+  /*
+   * Statistics thread.  Periodically prints the cache statistics to the log.
+   */
+  static class StatisticsThread extends Thread {
+    LruBlockCache lru;
+
+    public StatisticsThread(LruBlockCache lru) {
+      super("LruBlockCache.StatisticsThread");
+      setDaemon(true);
+      this.lru = lru;
+    }
+    @Override
+    public void run() {
+      lru.logStats();
+    }
+  }
+
+  public void logStats() {
+    if (!LOG.isDebugEnabled()) return;
+    // Log size
+    long totalSize = heapSize();
+    long freeSize = maxSize - totalSize;
+    LruBlockCache.LOG.debug("Stats: " +
+        "total=" + StringUtils.byteDesc(totalSize) + ", " +
+        "free=" + StringUtils.byteDesc(freeSize) + ", " +
+        "max=" + StringUtils.byteDesc(this.maxSize) + ", " +
+        "blocks=" + size() +", " +
+        "accesses=" + stats.getRequestCount() + ", " +
+        "hits=" + stats.getHitCount() + ", " +
+        "hitRatio=" +
+          (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) + ", " +
+        "cachingAccesses=" + stats.getRequestCachingCount() + ", " +
+        "cachingHits=" + stats.getHitCachingCount() + ", " +
+        "cachingHitsRatio=" +
+          (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitCachingRatio(), 2)+ ", ")) + ", " +
+        "evictions=" + stats.getEvictionCount() + ", " +
+        "evicted=" + stats.getEvictedCount() + ", " +
+        "evictedPerRun=" + stats.evictedPerEviction());
+  }
+
+  /**
+   * Get counter statistics for this cache.
+   *
+   * 
      Includes: total accesses, hits, misses, evicted blocks, and runs
+   * of the eviction processes.
+   */
+  public CacheStats getStats() {
+    return this.stats;
+  }
+
+  public final static long CACHE_FIXED_OVERHEAD = ClassSize.align(
+      (3 * Bytes.SIZEOF_LONG) + (8 * ClassSize.REFERENCE) +
+      (5 * Bytes.SIZEOF_FLOAT) + Bytes.SIZEOF_BOOLEAN
+      + ClassSize.OBJECT);
+
+  // HeapSize implementation
+  public long heapSize() {
+    return getCurrentSize();
+  }
+
+  public static long calculateOverhead(long maxSize, long blockSize, int concurrency){
+    // FindBugs ICAST_INTEGER_MULTIPLY_CAST_TO_LONG
+    return CACHE_FIXED_OVERHEAD + ClassSize.CONCURRENT_HASHMAP +
+        ((long)Math.ceil(maxSize*1.2/blockSize)
+            * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
+        (concurrency * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
+  }
+
+  @Override
+  public List getBlockCacheColumnFamilySummaries(Configuration conf) throws IOException {
+
+    Map sfMap = FSUtils.getTableStoreFilePathMap(
+        FileSystem.get(conf),
+        FSUtils.getRootDir(conf));
+
+    // quirky, but it's a compound key and this is a shortcut taken instead of
+    // creating a class that would represent only a key.
+    Map bcs =
+      new HashMap();
+
+    for (CachedBlock cb : map.values()) {
+      String sf = cb.getCacheKey().getHfileName();
+      Path path = sfMap.get(sf);
+      if ( path != null) {
+        BlockCacheColumnFamilySummary lookup =
+          BlockCacheColumnFamilySummary.createFromStoreFilePath(path);
+        BlockCacheColumnFamilySummary bcse = bcs.get(lookup);
+        if (bcse == null) {
+          bcse = BlockCacheColumnFamilySummary.create(lookup);
+          bcs.put(lookup,bcse);
+        }
+        bcse.incrementBlocks();
+        bcse.incrementHeapSize(cb.heapSize());
+      }
+    }
+    List list =
+        new ArrayList(bcs.values());
+    Collections.sort( list );
+    return list;
+  }
+
+  // Simple calculators of sizes given factors and maxSize
+
+  private long acceptableSize() {
+    return (long)Math.floor(this.maxSize * this.acceptableFactor);
+  }
+  private long minSize() {
+    return (long)Math.floor(this.maxSize * this.minFactor);
+  }
+  private long singleSize() {
+    return (long)Math.floor(this.maxSize * this.singleFactor * this.minFactor);
+  }
+  private long multiSize() {
+    return (long)Math.floor(this.maxSize * this.multiFactor * this.minFactor);
+  }
+  private long memorySize() {
+    return (long)Math.floor(this.maxSize * this.memoryFactor * this.minFactor);
+  }
+
+  public void shutdown() {
+    this.scheduleThreadPool.shutdown();
+    for (int i = 0; i < 10; i++) {
+      if (!this.scheduleThreadPool.isShutdown()) Threads.sleep(10);
+    }
+    if (!this.scheduleThreadPool.isShutdown()) {
+      List runnables = this.scheduleThreadPool.shutdownNow();
+      LOG.debug("Still running " + runnables);
+    }
+    this.evictionThread.shutdown();
+  }
+
+  /** Clears the cache. Used in tests. */
+  public void clearCache() {
+    map.clear();
+  }
+
+  /**
+   * Used in testing. May be very inefficient.
+   * @return the set of cached file names
+   */
+  SortedSet getCachedFileNamesForTest() {
+    SortedSet fileNames = new TreeSet();
+    for (BlockCacheKey cacheKey : map.keySet()) {
+      fileNames.add(cacheKey.getHfileName());
+    }
+    return fileNames;
+  }
+
+  Map getBlockTypeCountsForTest() {
+    Map counts =
+        new EnumMap(BlockType.class);
+    for (CachedBlock cb : map.values()) {
+      BlockType blockType = ((HFileBlock) cb.getBuffer()).getBlockType();
+      Integer count = counts.get(blockType);
+      counts.put(blockType, (count == null ? 0 : count) + 1);
+    }
+    return counts;
+  }
+
+  public Map getEncodingCountsForTest() {
+    Map counts =
+        new EnumMap(DataBlockEncoding.class);
+    for (BlockCacheKey cacheKey : map.keySet()) {
+      DataBlockEncoding encoding = cacheKey.getDataBlockEncoding();
+      Integer count = counts.get(encoding);
+      counts.put(encoding, (count == null ? 0 : count) + 1);
+    }
+    return counts;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
new file mode 100644
index 0000000..c69d087
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * Does not perform any kind of encoding/decoding.
+ */
+public class NoOpDataBlockEncoder implements HFileDataBlockEncoder {
+
+  public static final NoOpDataBlockEncoder INSTANCE =
+      new NoOpDataBlockEncoder();
+
+  /** Cannot be instantiated. Use {@link #INSTANCE} instead. */
+  private NoOpDataBlockEncoder() {
+  }
+
+  @Override
+  public HFileBlock diskToCacheFormat(HFileBlock block, boolean isCompaction) {
+    if (block.getBlockType() == BlockType.ENCODED_DATA) {
+      throw new IllegalStateException("Unexpected encoded block");
+    }
+    return block;
+  }
+
+  @Override
+  public Pair beforeWriteToDisk(
+      ByteBuffer in, boolean includesMemstoreTS, byte[] dummyHeader) {
+    return new Pair(in, BlockType.DATA);
+  }
+
+  @Override
+  public boolean useEncodedScanner(boolean isCompaction) {
+    return false;
+  }
+
+  @Override
+  public void saveMetadata(HFile.Writer writer) {
+  }
+
+  @Override
+  public DataBlockEncoding getEncodingOnDisk() {
+    return DataBlockEncoding.NONE;
+  }
+
+  @Override
+  public DataBlockEncoding getEncodingInCache() {
+    return DataBlockEncoding.NONE;
+  }
+
+  @Override
+  public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
+    return DataBlockEncoding.NONE;
+  }
+
+  @Override
+  public String toString() {
+    return getClass().getSimpleName();
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/ReusableStreamGzipCodec.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/ReusableStreamGzipCodec.java
new file mode 100644
index 0000000..2b1d48b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/ReusableStreamGzipCodec.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.CompressorStream;
+import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.io.compress.zlib.ZlibFactory;
+
+/**
+ * Fixes an inefficiency in Hadoop's Gzip codec, allowing to reuse compression
+ * streams.
+ */
+public class ReusableStreamGzipCodec extends GzipCodec {
+
+  private static final Log LOG = LogFactory.getLog(Compression.class);
+
+  /**
+   * A bridge that wraps around a DeflaterOutputStream to make it a
+   * CompressionOutputStream.
+   */
+  protected static class ReusableGzipOutputStream extends CompressorStream {
+
+    private static final int GZIP_HEADER_LENGTH = 10;
+
+    /**
+     * Fixed ten-byte gzip header. See {@link GZIPOutputStream}'s source for
+     * details.
+     */
+    private static final byte[] GZIP_HEADER;
+
+    static {
+      // Capture the fixed ten-byte header hard-coded in GZIPOutputStream.
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      byte[] header = null;
+      GZIPOutputStream gzipStream = null;
+      try {
+        gzipStream  = new GZIPOutputStream(baos);
+        gzipStream.finish();
+        header = Arrays.copyOfRange(baos.toByteArray(), 0, GZIP_HEADER_LENGTH);
+      } catch (IOException e) {
+        throw new RuntimeException("Could not create gzip stream", e);
+      } finally {
+        if (gzipStream != null) {
+          try {
+            gzipStream.close();
+          } catch (IOException e) {
+            LOG.error(e);
+          }
+        }
+      }
+      GZIP_HEADER = header;
+    }
+
+    private static class ResetableGZIPOutputStream extends GZIPOutputStream {
+      public ResetableGZIPOutputStream(OutputStream out) throws IOException {
+        super(out);
+      }
+
+      public void resetState() throws IOException {
+        def.reset();
+        crc.reset();
+        out.write(GZIP_HEADER);
+      }
+    }
+
+    public ReusableGzipOutputStream(OutputStream out) throws IOException {
+      super(new ResetableGZIPOutputStream(out));
+    }
+
+    @Override
+    public void close() throws IOException {
+      out.close();
+    }
+
+    @Override
+    public void flush() throws IOException {
+      out.flush();
+    }
+
+    @Override
+    public void write(int b) throws IOException {
+      out.write(b);
+    }
+
+    @Override
+    public void write(byte[] data, int offset, int length) throws IOException {
+      out.write(data, offset, length);
+    }
+
+    @Override
+    public void finish() throws IOException {
+      ((GZIPOutputStream) out).finish();
+    }
+
+    @Override
+    public void resetState() throws IOException {
+      ((ResetableGZIPOutputStream) out).resetState();
+    }
+  }
+
+  @Override
+  public CompressionOutputStream createOutputStream(OutputStream out)
+      throws IOException {
+    if (ZlibFactory.isNativeZlibLoaded(getConf())) {
+      return super.createOutputStream(out);
+    }
+    return new ReusableGzipOutputStream(out);
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
new file mode 100644
index 0000000..a14909b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
@@ -0,0 +1,138 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.lang.ref.ReferenceQueue;
+import java.lang.ref.SoftReference;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+
+
+/**
+ * Simple one RFile soft reference cache.
+ */
+public class SimpleBlockCache implements BlockCache {
+  private static class Ref extends SoftReference {
+    public BlockCacheKey blockId;
+    public Ref(BlockCacheKey blockId, Cacheable block, ReferenceQueue q) {
+      super(block, q);
+      this.blockId = blockId;
+    }
+  }
+  private Map cache =
+    new HashMap();
+
+  private ReferenceQueue q = new ReferenceQueue();
+  public int dumps = 0;
+
+  /**
+   * Constructor
+   */
+  public SimpleBlockCache() {
+    super();
+  }
+
+  void processQueue() {
+    Ref r;
+    while ( (r = (Ref)q.poll()) != null) {
+      cache.remove(r.blockId);
+      dumps++;
+    }
+  }
+
+  /**
+   * @return the size
+   */
+  public synchronized long size() {
+    processQueue();
+    return cache.size();
+  }
+
+  public synchronized Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) {
+    processQueue(); // clear out some crap.
+    Ref ref = cache.get(cacheKey);
+    if (ref == null)
+      return null;
+    return ref.get();
+  }
+
+  public synchronized void cacheBlock(BlockCacheKey cacheKey, Cacheable block) {
+    cache.put(cacheKey, new Ref(cacheKey, block, q));
+  }
+
+  public synchronized void cacheBlock(BlockCacheKey cacheKey, Cacheable block,
+      boolean inMemory) {
+    cache.put(cacheKey, new Ref(cacheKey, block, q));
+  }
+
+  @Override
+  public boolean evictBlock(BlockCacheKey cacheKey) {
+    return cache.remove(cacheKey) != null;
+  }
+
+  public void shutdown() {
+    // noop
+  }
+
+  @Override
+  public CacheStats getStats() {
+    // TODO: implement this if we ever actually use this block cache
+    return null;
+  }
+
+  @Override
+  public long getFreeSize() {
+    // TODO: implement this if we ever actually use this block cache
+    return 0;
+  }
+
+  @Override
+  public long getCurrentSize() {
+    // TODO: implement this if we ever actually use this block cache
+    return 0;
+  }
+
+  @Override
+  public long getEvictedCount() {
+    // TODO: implement this if we ever actually use this block cache
+    return 0;
+  }
+
+  @Override
+  public int evictBlocksByHfileName(String string) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public List getBlockCacheColumnFamilySummaries(Configuration conf) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getBlockCount() {
+    // TODO: implement this if we ever actually use this block cache
+    return 0;
+  }
+
+}
+
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
new file mode 100644
index 0000000..d68c2d5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
@@ -0,0 +1,357 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile.slab;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
+import org.apache.hadoop.hbase.io.hfile.CacheStats;
+import org.apache.hadoop.hbase.io.hfile.Cacheable;
+import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+
+/**
+ * SingleSizeCache is a slab allocated cache that caches elements up to a single
+ * size. It uses a slab allocator (Slab.java) to divide a direct bytebuffer,
+ * into evenly sized blocks. Any cached data will take up exactly 1 block. An
+ * exception will be thrown if the cached data cannot fit into the blockSize of
+ * this SingleSizeCache.
+ *
+ * Eviction and LRUness is taken care of by Guava's MapMaker, which creates a
+ * ConcurrentLinkedHashMap.
+ *
+ **/
+public class SingleSizeCache implements BlockCache, HeapSize {
+  private final Slab backingStore;
+  private final ConcurrentMap backingMap;
+  private final int numBlocks;
+  private final int blockSize;
+  private final CacheStats stats;
+  private final SlabItemActionWatcher actionWatcher;
+  private final AtomicLong size;
+  private final AtomicLong timeSinceLastAccess;
+  public final static long CACHE_FIXED_OVERHEAD = ClassSize
+      .align((2 * Bytes.SIZEOF_INT) + (5 * ClassSize.REFERENCE)
+          + +ClassSize.OBJECT);
+
+  static final Log LOG = LogFactory.getLog(SingleSizeCache.class);
+
+  /**
+   * Default constructor. Specify the size of the blocks, number of blocks, and
+   * the SlabCache this cache will be assigned to.
+   *
+   *
+   * @param blockSize the size of each block, in bytes
+   *
+   * @param numBlocks the number of blocks of blockSize this cache will hold.
+   *
+   * @param master the SlabCache this SingleSlabCache is assigned to.
+   */
+  public SingleSizeCache(int blockSize, int numBlocks,
+      SlabItemActionWatcher master) {
+    this.blockSize = blockSize;
+    this.numBlocks = numBlocks;
+    backingStore = new Slab(blockSize, numBlocks);
+    this.stats = new CacheStats();
+    this.actionWatcher = master;
+    this.size = new AtomicLong(CACHE_FIXED_OVERHEAD + backingStore.heapSize());
+    this.timeSinceLastAccess = new AtomicLong();
+
+    // This evictionListener is called whenever the cache automatically
+    // evicts
+    // something.
+    RemovalListener listener =
+      new RemovalListener() {
+        @Override
+        public void onRemoval(
+            RemovalNotification notification) {
+          if (!notification.wasEvicted()) {
+            // Only process removals by eviction, not by replacement or
+            // explicit removal
+            return;
+          }
+          CacheablePair value = notification.getValue();
+          timeSinceLastAccess.set(System.nanoTime()
+              - value.recentlyAccessed.get());
+          stats.evict();
+          doEviction(notification.getKey(), value);
+        }
+      };
+
+    backingMap = CacheBuilder.newBuilder()
+        .maximumSize(numBlocks - 1)
+        .removalListener(listener)
+        .build()
+        .asMap();
+
+
+  }
+
+  @Override
+  public void cacheBlock(BlockCacheKey blockName, Cacheable toBeCached) {
+    ByteBuffer storedBlock;
+
+    try {
+      storedBlock = backingStore.alloc(toBeCached.getSerializedLength());
+    } catch (InterruptedException e) {
+      LOG.warn("SlabAllocator was interrupted while waiting for block to become available");
+      LOG.warn(e);
+      return;
+    }
+
+    CacheablePair newEntry = new CacheablePair(toBeCached.getDeserializer(),
+        storedBlock);
+    toBeCached.serialize(storedBlock);
+
+    synchronized (this) {
+      CacheablePair alreadyCached = backingMap.putIfAbsent(blockName, newEntry);
+    
+
+      if (alreadyCached != null) {
+        backingStore.free(storedBlock);
+        throw new RuntimeException("already cached " + blockName);
+      }
+      if (actionWatcher != null) {
+        actionWatcher.onInsertion(blockName, this);
+      }
+    }
+    newEntry.recentlyAccessed.set(System.nanoTime());
+    this.size.addAndGet(newEntry.heapSize());
+  }
+
+  @Override
+  public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat) {
+    CacheablePair contentBlock = backingMap.get(key);
+    if (contentBlock == null) {
+      if (!repeat) stats.miss(caching);
+      return null;
+    }
+
+    stats.hit(caching);
+    // If lock cannot be obtained, that means we're undergoing eviction.
+    try {
+      contentBlock.recentlyAccessed.set(System.nanoTime());
+      synchronized (contentBlock) {
+        if (contentBlock.serializedData == null) {
+          // concurrently evicted
+          LOG.warn("Concurrent eviction of " + key);
+          return null;
+        }
+        return contentBlock.deserializer
+            .deserialize(contentBlock.serializedData.asReadOnlyBuffer());
+      }
+    } catch (Throwable t) {
+      LOG.error("Deserializer threw an exception. This may indicate a bug.", t);
+      return null;
+    }
+  }
+
+  /**
+   * Evicts the block
+   *
+   * @param key the key of the entry we are going to evict
+   * @return the evicted ByteBuffer
+   */
+  public boolean evictBlock(BlockCacheKey key) {
+    stats.evict();
+    CacheablePair evictedBlock = backingMap.remove(key);
+
+    if (evictedBlock != null) {
+      doEviction(key, evictedBlock);
+    }
+    return evictedBlock != null;
+
+  }
+
+  private void doEviction(BlockCacheKey key, CacheablePair evictedBlock) {
+    long evictedHeap = 0;
+    synchronized (evictedBlock) {
+      if (evictedBlock.serializedData == null) {
+        // someone else already freed
+        return;
+      }
+      evictedHeap = evictedBlock.heapSize();
+      ByteBuffer bb = evictedBlock.serializedData;
+      evictedBlock.serializedData = null;
+      backingStore.free(bb);
+
+      // We have to do this callback inside the synchronization here.
+      // Otherwise we can have the following interleaving:
+      // Thread A calls getBlock():
+      // SlabCache directs call to this SingleSizeCache
+      // It gets the CacheablePair object
+      // Thread B runs eviction
+      // doEviction() is called and sets serializedData = null, here.
+      // Thread A sees the null serializedData, and returns null
+      // Thread A calls cacheBlock on the same block, and gets
+      // "already cached" since the block is still in backingStore
+
+      if (actionWatcher != null) {
+        actionWatcher.onEviction(key, this);
+      }
+    }
+    stats.evicted();
+    size.addAndGet(-1 * evictedHeap);
+  }
+
+  public void logStats() {
+
+    long milliseconds = this.timeSinceLastAccess.get() / 1000000;
+
+    LOG.info("For Slab of size " + this.blockSize + ": "
+        + this.getOccupiedSize() / this.blockSize
+        + " occupied, out of a capacity of " + this.numBlocks
+        + " blocks. HeapSize is "
+        + StringUtils.humanReadableInt(this.heapSize()) + " bytes." + ", "
+        + "churnTime=" + StringUtils.formatTime(milliseconds));
+
+    LOG.info("Slab Stats: " + "accesses="
+        + stats.getRequestCount()
+        + ", "
+        + "hits="
+        + stats.getHitCount()
+        + ", "
+        + "hitRatio="
+        + (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(
+            stats.getHitRatio(), 2) + "%, "))
+        + "cachingAccesses="
+        + stats.getRequestCachingCount()
+        + ", "
+        + "cachingHits="
+        + stats.getHitCachingCount()
+        + ", "
+        + "cachingHitsRatio="
+        + (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent(
+            stats.getHitCachingRatio(), 2) + "%, ")) + "evictions="
+        + stats.getEvictionCount() + ", " + "evicted="
+        + stats.getEvictedCount() + ", " + "evictedPerRun="
+        + stats.evictedPerEviction());
+
+  }
+
+  public void shutdown() {
+    backingStore.shutdown();
+  }
+
+  public long heapSize() {
+    return this.size.get() + backingStore.heapSize();
+  }
+
+  public long size() {
+    return (long) this.blockSize * (long) this.numBlocks;
+  }
+
+  public long getFreeSize() {
+    return (long) backingStore.getBlocksRemaining() * (long) blockSize;
+  }
+
+  public long getOccupiedSize() {
+    return (long) (numBlocks - backingStore.getBlocksRemaining()) * (long) blockSize;
+  }
+
+  public long getEvictedCount() {
+    return stats.getEvictedCount();
+  }
+
+  public CacheStats getStats() {
+    return this.stats;
+  }
+
+  @Override
+  public long getBlockCount() {
+    return numBlocks - backingStore.getBlocksRemaining();
+  }
+
+  /* Since its offheap, it doesn't matter if its in memory or not */
+  @Override
+  public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
+    this.cacheBlock(cacheKey, buf);
+  }
+
+  /*
+   * This is never called, as evictions are handled in the SlabCache layer,
+   * implemented in the event we want to use this as a standalone cache.
+   */
+  @Override
+  public int evictBlocksByHfileName(String hfileName) {
+    int evictedCount = 0;
+    for (BlockCacheKey e : backingMap.keySet()) {
+      if (e.getHfileName().equals(hfileName)) {
+        this.evictBlock(e);
+      }
+    }
+    return evictedCount;
+  }
+
+  @Override
+  public long getCurrentSize() {
+    return 0;
+  }
+
+  /*
+   * Not implemented. Extremely costly to do this from the off heap cache, you'd
+   * need to copy every object on heap once
+   */
+  @Override
+  public List getBlockCacheColumnFamilySummaries(
+      Configuration conf) {
+    throw new UnsupportedOperationException();
+  }
+
+  /* Just a pair class, holds a reference to the parent cacheable */
+  private class CacheablePair implements HeapSize {
+    final CacheableDeserializer deserializer;
+    ByteBuffer serializedData;
+    AtomicLong recentlyAccessed;
+
+    private CacheablePair(CacheableDeserializer deserializer,
+        ByteBuffer serializedData) {
+      this.recentlyAccessed = new AtomicLong();
+      this.deserializer = deserializer;
+      this.serializedData = serializedData;
+    }
+
+    /*
+     * Heapsize overhead of this is the default object overhead, the heapsize of
+     * the serialized object, and the cost of a reference to the bytebuffer,
+     * which is already accounted for in SingleSizeCache
+     */
+    @Override
+    public long heapSize() {
+      return ClassSize.align(ClassSize.OBJECT + ClassSize.REFERENCE * 3
+          + ClassSize.ATOMIC_LONG);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/Slab.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/Slab.java
new file mode 100644
index 0000000..ed32980
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/Slab.java
@@ -0,0 +1,134 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile.slab;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.hbase.util.DirectMemoryUtils;
+import com.google.common.base.Preconditions;
+
+/**
+ * Slab is a class which is designed to allocate blocks of a certain size.
+ * Constructor creates a number of DirectByteBuffers and slices them into the
+ * requisite size, then puts them all in a buffer.
+ **/
+
+class Slab implements org.apache.hadoop.hbase.io.HeapSize {
+  static final Log LOG = LogFactory.getLog(Slab.class);
+
+  /** This is where our items, or blocks of the slab, are stored. */
+  private LinkedBlockingQueue buffers;
+
+  /** This is where our Slabs are stored */
+  private ConcurrentLinkedQueue slabs;
+
+  private final int blockSize;
+  private final int numBlocks;
+  private long heapSize;
+
+  Slab(int blockSize, int numBlocks) {
+    buffers = new LinkedBlockingQueue();
+    slabs = new ConcurrentLinkedQueue();
+
+    this.blockSize = blockSize;
+    this.numBlocks = numBlocks;
+
+    this.heapSize = ClassSize.estimateBase(this.getClass(), false);
+
+    int maxBlocksPerSlab = Integer.MAX_VALUE / blockSize;
+    int maxSlabSize = maxBlocksPerSlab * blockSize;
+
+    int numFullSlabs = numBlocks / maxBlocksPerSlab;
+    int partialSlabSize = (numBlocks % maxBlocksPerSlab) * blockSize;
+    for (int i = 0; i < numFullSlabs; i++) {
+      allocateAndSlice(maxSlabSize, blockSize);
+    }
+
+    if (partialSlabSize > 0) {
+      allocateAndSlice(partialSlabSize, blockSize);
+    }
+  }
+
+  private void allocateAndSlice(int size, int sliceSize) {
+    ByteBuffer newSlab = ByteBuffer.allocateDirect(size);
+    slabs.add(newSlab);
+    for (int j = 0; j < newSlab.capacity(); j += sliceSize) {
+      newSlab.limit(j + sliceSize).position(j);
+      ByteBuffer aSlice = newSlab.slice();
+      buffers.add(aSlice);
+      heapSize += ClassSize.estimateBase(aSlice.getClass(), false);
+    }
+  }
+
+  /*
+   * Shutdown deallocates the memory for all the DirectByteBuffers. Each
+   * DirectByteBuffer has a "cleaner" method, which is similar to a
+   * deconstructor in C++.
+   */
+  void shutdown() {
+    for (ByteBuffer aSlab : slabs) {
+      try {
+        DirectMemoryUtils.destroyDirectByteBuffer(aSlab);
+      } catch (Exception e) {
+        LOG.warn("Unable to deallocate direct memory during shutdown", e);
+      }
+    }
+  }
+
+  int getBlockSize() {
+    return this.blockSize;
+  }
+
+  int getBlockCapacity() {
+    return this.numBlocks;
+  }
+
+  int getBlocksRemaining() {
+    return this.buffers.size();
+  }
+
+  /*
+   * Throws an exception if you try to allocate a
+   * bigger size than the allocator can handle. Alloc will block until a buffer is available.
+   */
+  ByteBuffer alloc(int bufferSize) throws InterruptedException {
+    int newCapacity = Preconditions.checkPositionIndex(bufferSize, blockSize);
+
+    ByteBuffer returnedBuffer = buffers.take();
+
+    returnedBuffer.clear().limit(newCapacity);
+    return returnedBuffer;
+  }
+
+  void free(ByteBuffer toBeFreed) {
+    Preconditions.checkArgument(toBeFreed.capacity() == blockSize);
+    buffers.add(toBeFreed);
+  }
+
+  @Override
+  public long heapSize() {
+    return heapSize;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
new file mode 100644
index 0000000..9fc6043
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
@@ -0,0 +1,426 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io.hfile.slab;
+
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
+import org.apache.hadoop.hbase.io.hfile.CacheStats;
+import org.apache.hadoop.hbase.io.hfile.Cacheable;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.hbase.util.HasThread;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * SlabCache is composed of multiple SingleSizeCaches. It uses a TreeMap in
+ * order to determine where a given element fits. Redirects gets and puts to the
+ * correct SingleSizeCache.
+ *
+ **/
+public class SlabCache implements SlabItemActionWatcher, BlockCache, HeapSize {
+
+  private final ConcurrentHashMap backingStore;
+  private final TreeMap sizer;
+  static final Log LOG = LogFactory.getLog(SlabCache.class);
+  static final int STAT_THREAD_PERIOD_SECS = 60 * 5;
+
+  private final ScheduledExecutorService scheduleThreadPool = Executors.newScheduledThreadPool(1,
+      new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Slab Statistics #%d").build());
+
+  long size;
+  private final CacheStats stats;
+  final SlabStats requestStats;
+  final SlabStats successfullyCachedStats;
+  private final long avgBlockSize;
+  private static final long CACHE_FIXED_OVERHEAD = ClassSize.estimateBase(
+      SlabCache.class, false);
+
+  /**
+   * Default constructor, creates an empty SlabCache.
+   *
+   * @param size Total size allocated to the SlabCache. (Bytes)
+   * @param avgBlockSize Average size of a block being cached.
+   **/
+
+  public SlabCache(long size, long avgBlockSize) {
+    this.avgBlockSize = avgBlockSize;
+    this.size = size;
+    this.stats = new CacheStats();
+    this.requestStats = new SlabStats();
+    this.successfullyCachedStats = new SlabStats();
+
+    backingStore = new ConcurrentHashMap();
+    sizer = new TreeMap();
+    this.scheduleThreadPool.scheduleAtFixedRate(new StatisticsThread(this),
+        STAT_THREAD_PERIOD_SECS, STAT_THREAD_PERIOD_SECS, TimeUnit.SECONDS);
+
+  }
+
+  /**
+   * A way of allocating the desired amount of Slabs of each particular size.
+   *
+   * This reads two lists from conf, hbase.offheap.slab.proportions and
+   * hbase.offheap.slab.sizes.
+   *
+   * The first list is the percentage of our total space we allocate to the
+   * slabs.
+   *
+   * The second list is blocksize of the slabs in bytes. (E.g. the slab holds
+   * blocks of this size).
+   *
+   * @param conf Configuration file.
+   */
+  public void addSlabByConf(Configuration conf) {
+    // Proportions we allocate to each slab of the total size.
+    String[] porportions = conf.getStrings(
+        "hbase.offheapcache.slab.proportions", "0.80", "0.20");
+    String[] sizes = conf.getStrings("hbase.offheapcache.slab.sizes",
+        Long.valueOf(avgBlockSize * 11 / 10).toString(),
+        Long.valueOf(avgBlockSize * 21 / 10).toString());
+
+    if (porportions.length != sizes.length) {
+      throw new IllegalArgumentException(
+          "SlabCache conf not "
+              + "initialized, error in configuration. hbase.offheap.slab.proportions specifies "
+              + porportions.length
+              + " slabs while hbase.offheap.slab.sizes specifies "
+              + sizes.length + " slabs "
+              + "offheapslabporportions and offheapslabsizes");
+    }
+    /*
+     * We use BigDecimals instead of floats because float rounding is annoying
+     */
+
+    BigDecimal[] parsedProportions = stringArrayToBigDecimalArray(porportions);
+    BigDecimal[] parsedSizes = stringArrayToBigDecimalArray(sizes);
+
+    BigDecimal sumProportions = new BigDecimal(0);
+    for (BigDecimal b : parsedProportions) {
+      /* Make sure all proportions are greater than 0 */
+      Preconditions
+          .checkArgument(b.compareTo(BigDecimal.ZERO) == 1,
+              "Proportions in hbase.offheap.slab.proportions must be greater than 0!");
+      sumProportions = sumProportions.add(b);
+    }
+
+    /* If the sum is greater than 1 */
+    Preconditions
+        .checkArgument(sumProportions.compareTo(BigDecimal.ONE) != 1,
+            "Sum of all proportions in hbase.offheap.slab.proportions must be less than 1");
+
+    /* If the sum of all proportions is less than 0.99 */
+    if (sumProportions.compareTo(new BigDecimal("0.99")) == -1) {
+      LOG.warn("Sum of hbase.offheap.slab.proportions is less than 0.99! Memory is being wasted");
+    }
+    for (int i = 0; i < parsedProportions.length; i++) {
+      int blockSize = parsedSizes[i].intValue();
+      int numBlocks = new BigDecimal(this.size).multiply(parsedProportions[i])
+          .divide(parsedSizes[i], BigDecimal.ROUND_DOWN).intValue();
+      addSlab(blockSize, numBlocks);
+    }
+  }
+
+  /**
+   * Gets the size of the slab cache a ByteBuffer of this size would be
+   * allocated to.
+   *
+   * @param size Size of the ByteBuffer we are checking.
+   *
+   * @return the Slab that the above bytebuffer would be allocated towards. If
+   *         object is too large, returns null.
+   */
+  Entry getHigherBlock(int size) {
+    return sizer.higherEntry(size - 1);
+  }
+
+  private BigDecimal[] stringArrayToBigDecimalArray(String[] parsee) {
+    BigDecimal[] parsed = new BigDecimal[parsee.length];
+    for (int i = 0; i < parsee.length; i++) {
+      parsed[i] = new BigDecimal(parsee[i].trim());
+    }
+    return parsed;
+  }
+
+  private void addSlab(int blockSize, int numBlocks) {
+    LOG.info("Creating a slab of blockSize " + blockSize + " with " + numBlocks
+        + " blocks.");
+    sizer.put(blockSize, new SingleSizeCache(blockSize, numBlocks, this));
+  }
+
+  /**
+   * Cache the block with the specified key and buffer. First finds what size
+   * SingleSlabCache it should fit in. If the block doesn't fit in any, it will
+   * return without doing anything.
+   * 
      
+   * It is assumed this will NEVER be called on an already cached block. If that
+   * is done, it is assumed that you are reinserting the same exact block due to
+   * a race condition, and will throw a runtime exception.
+   *
+   * @param cacheKey block cache key
+   * @param cachedItem block buffer
+   */
+  public void cacheBlock(BlockCacheKey cacheKey, Cacheable cachedItem) {
+    Entry scacheEntry = getHigherBlock(cachedItem
+        .getSerializedLength());
+
+    this.requestStats.addin(cachedItem.getSerializedLength());
+
+    if (scacheEntry == null) {
+      return; // we can't cache, something too big.
+    }
+
+    this.successfullyCachedStats.addin(cachedItem.getSerializedLength());
+    SingleSizeCache scache = scacheEntry.getValue();
+
+    /*
+     * This will throw a runtime exception if we try to cache the same value
+     * twice
+     */
+    scache.cacheBlock(cacheKey, cachedItem);
+  } 
+
+  /**
+   * We don't care about whether its in memory or not, so we just pass the call
+   * through.
+   */
+  public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
+    cacheBlock(cacheKey, buf);
+  }
+
+  public CacheStats getStats() {
+    return this.stats;
+  }
+
+  /**
+   * Get the buffer of the block with the specified name.
+   * @param caching
+   * @param key
+   * @param repeat
+   *
+   * @return buffer of specified block name, or null if not in cache
+   */
+  public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat) {
+    SingleSizeCache cachedBlock = backingStore.get(key);
+    if (cachedBlock == null) {
+      if (!repeat) stats.miss(caching);
+      return null;
+    }
+
+    Cacheable contentBlock = cachedBlock.getBlock(key, caching, false);
+
+    if (contentBlock != null) {
+      stats.hit(caching);
+    } else {
+      if (!repeat) stats.miss(caching);
+    }
+    return contentBlock;
+  }
+
+  /**
+   * Evicts a block from the cache. This is public, and thus contributes to the
+   * the evict counter.
+   */
+  public boolean evictBlock(BlockCacheKey cacheKey) {
+    SingleSizeCache cacheEntry = backingStore.get(cacheKey);
+    if (cacheEntry == null) {
+      return false;
+    } else {
+      cacheEntry.evictBlock(cacheKey);
+      return true;
+    }
+  }
+
+  @Override
+  public void onEviction(BlockCacheKey key, SingleSizeCache notifier) {
+    stats.evicted();
+    backingStore.remove(key);
+  }
+  
+  @Override
+  public void onInsertion(BlockCacheKey key, SingleSizeCache notifier) {
+    backingStore.put(key, notifier);
+  }
+
+  /**
+   * Sends a shutdown to all SingleSizeCache's contained by this cache.
+   *
+   * Also terminates the scheduleThreadPool.
+   */
+  public void shutdown() {
+    for (SingleSizeCache s : sizer.values()) {
+      s.shutdown();
+    }
+    this.scheduleThreadPool.shutdown();
+  }
+
+  public long heapSize() {
+    long childCacheSize = 0;
+    for (SingleSizeCache s : sizer.values()) {
+      childCacheSize += s.heapSize();
+    }
+    return SlabCache.CACHE_FIXED_OVERHEAD + childCacheSize;
+  }
+
+  public long size() {
+    return this.size;
+  }
+
+  public long getFreeSize() {
+    return 0; // this cache, by default, allocates all its space.
+  }
+
+  @Override
+  public long getBlockCount() {
+    long count = 0;
+    for (SingleSizeCache cache : backingStore.values()) {
+      count += cache.getBlockCount();
+    }
+    return count;
+  }
+
+  public long getCurrentSize() {
+    return size;
+  }
+
+  public long getEvictedCount() {
+    return stats.getEvictedCount();
+  }
+
+  /*
+   * Statistics thread. Periodically prints the cache statistics to the log.
+   */
+  static class StatisticsThread extends HasThread {
+    SlabCache ourcache;
+
+    public StatisticsThread(SlabCache slabCache) {
+      super("SlabCache.StatisticsThread");
+      setDaemon(true);
+      this.ourcache = slabCache;
+    }
+
+    @Override
+    public void run() {
+      for (SingleSizeCache s : ourcache.sizer.values()) {
+        s.logStats();
+      }
+
+      SlabCache.LOG.info("Current heap size is: "
+          + StringUtils.humanReadableInt(ourcache.heapSize()));
+
+      LOG.info("Request Stats");
+      ourcache.requestStats.logStats();
+      LOG.info("Successfully Cached Stats");
+      ourcache.successfullyCachedStats.logStats();
+    }
+
+  }
+
+  /**
+   * Just like CacheStats, but more Slab specific. Finely grained profiling of
+   * sizes we store using logs.
+   *
+   */
+  static class SlabStats {
+    // the maximum size somebody will ever try to cache, then we multiply by
+    // 10
+    // so we have finer grained stats.
+    final int MULTIPLIER = 10;
+    final int NUMDIVISIONS = (int) (Math.log(Integer.MAX_VALUE) * MULTIPLIER);
+    private final AtomicLong[] counts = new AtomicLong[NUMDIVISIONS];
+
+    public SlabStats() {
+      for (int i = 0; i < NUMDIVISIONS; i++) {
+        counts[i] = new AtomicLong();
+      }
+    }
+
+    public void addin(int size) {
+      int index = (int) (Math.log(size) * MULTIPLIER);
+      counts[index].incrementAndGet();
+    }
+
+    public AtomicLong[] getUsage() {
+      return counts;
+    }
+
+    double getUpperBound(int index) {
+      return Math.pow(Math.E, ((index + 0.5) / MULTIPLIER));
+    }
+
+    double getLowerBound(int index) {
+      return Math.pow(Math.E, ((index - 0.5) / MULTIPLIER));
+    }
+
+    public void logStats() {
+      AtomicLong[] fineGrainedStats = getUsage();
+      for (int i = 0; i < fineGrainedStats.length; i++) {
+
+        if (fineGrainedStats[i].get() > 0) {
+          SlabCache.LOG.info("From  "
+              + StringUtils.humanReadableInt((long) getLowerBound(i)) + "- "
+              + StringUtils.humanReadableInt((long) getUpperBound(i)) + ": "
+              + StringUtils.humanReadableInt(fineGrainedStats[i].get())
+              + " requests");
+
+        }
+      }
+    }
+  }
+
+  public int evictBlocksByHfileName(String hfileName) {
+    int numEvicted = 0;
+    for (BlockCacheKey key : backingStore.keySet()) {
+      if (key.getHfileName().equals(hfileName)) {
+        if (evictBlock(key))
+          ++numEvicted;
+      }
+    }
+    return numEvicted;
+  }
+
+  /*
+   * Not implemented. Extremely costly to do this from the off heap cache, you'd
+   * need to copy every object on heap once
+   */
+  @Override
+  public List getBlockCacheColumnFamilySummaries(
+      Configuration conf) {
+    throw new UnsupportedOperationException();
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java
new file mode 100644
index 0000000..dfd727f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java
@@ -0,0 +1,45 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io.hfile.slab;
+
+import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
+
+/**
+ * Interface for objects that want to know when actions occur in a SingleSizeCache.
+ * */
+interface SlabItemActionWatcher {
+
+  /**
+   * This is called as a callback when an item is removed from a SingleSizeCache.
+   *
+   * @param key the key of the item being evicted
+   * @param notifier the object notifying the SlabCache of the eviction.
+   */
+  void onEviction(BlockCacheKey key, SingleSizeCache notifier);
+  
+  /**
+   * This is called as a callback when an item is inserted into a SingleSizeCache.
+   *
+   * @param key the key of the item being added
+   * @param notifier the object notifying the SlabCache of the insertion..
+   */
+  void onInsertion(BlockCacheKey key, SingleSizeCache notifier);
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/CallerDisconnectedException.java b/src/main/java/org/apache/hadoop/hbase/ipc/CallerDisconnectedException.java
new file mode 100644
index 0000000..00e450f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/CallerDisconnectedException.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+
+/**
+ * Exception indicating that the remote host making this IPC lost its
+ * IPC connection. This will never be returned back to a client,
+ * but is only used for logging on the server side, etc.
+ */
+public class CallerDisconnectedException extends IOException {
+  public CallerDisconnectedException(String msg) {
+    super(msg);
+  }
+
+  private static final long serialVersionUID = 1L;
+
+  
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/ConnectionHeader.java b/src/main/java/org/apache/hadoop/hbase/ipc/ConnectionHeader.java
new file mode 100644
index 0000000..6328310
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/ConnectionHeader.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.hbase.security.User;
+
+/**
+ * The IPC connection header sent by the client to the server
+ * on connection establishment.
+ */
+class ConnectionHeader implements Writable {
+  protected String protocol;
+
+  public ConnectionHeader() {}
+
+  /**
+   * Create a new {@link ConnectionHeader} with the given protocol
+   * and {@link User}.
+   * @param protocol protocol used for communication between the IPC client
+   *                 and the server
+   * @param user {@link User} of the client communicating with
+   *            the server
+   */
+  public ConnectionHeader(String protocol, User user) {
+    this.protocol = protocol;
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    protocol = Text.readString(in);
+    if (protocol.isEmpty()) {
+      protocol = null;
+    }
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    Text.writeString(out, (protocol == null) ? "" : protocol);
+  }
+
+  public String getProtocol() {
+    return protocol;
+  }
+
+  public User getUser() {
+    return null;
+  }
+
+  public String toString() {
+    return protocol;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorProtocol.java b/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorProtocol.java
new file mode 100644
index 0000000..8211f03
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorProtocol.java
@@ -0,0 +1,39 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+/**
+ * All custom RPC protocols to be exported by Coprocessors must extend this interface.
+ *
+ * 
      
+ * Note that all callable methods must have a return type handled by
+ * {@link org.apache.hadoop.hbase.io.HbaseObjectWritable#writeObject(java.io.DataOutput, Object, Class, org.apache.hadoop.conf.Configuration)}.
+ * That is:
+ * 
        
+ *   
      • a Java primitive type ({@code int}, {@code float}, etc)
        • 
+ *   
      • a Java {@code String}
        • 
+ *   
      • a {@link org.apache.hadoop.io.Writable}
        • 
+ *   
      • an array or {@code java.util.List} of one of the above
        • 
+ * 
      
+ * 
      
+ */
+public interface CoprocessorProtocol extends VersionedProtocol {
+  public static final long VERSION = 1L;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/Delayable.java b/src/main/java/org/apache/hadoop/hbase/ipc/Delayable.java
new file mode 100644
index 0000000..04eb02d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/Delayable.java
@@ -0,0 +1,71 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+
+/**
+ * A call whose response can be delayed by the server.
+ */
+public interface Delayable {
+  /**
+   * Signal that the call response should be delayed, thus freeing the RPC
+   * server to handle different requests.
+   *
+   * @param delayReturnValue Controls whether the return value of the call
+   * should be set when ending the delay or right away.  There are cases when
+   * the return value can be set right away, even if the call is delayed.
+   */
+  public void startDelay(boolean delayReturnValue);
+
+  /**
+   * @return is the call delayed?
+   */
+  public boolean isDelayed();
+
+  /**
+   * @return is the return value delayed?
+   */
+  public boolean isReturnValueDelayed();
+
+  /**
+   * Signal that the  RPC server is now allowed to send the response.
+   * @param result The value to return to the caller.  If the corresponding
+   * delay response specified that the return value should
+   * not be delayed, this parameter must be null.
+   * @throws IOException
+   */
+  public void endDelay(Object result) throws IOException;
+
+  /**
+   * Signal the end of a delayed RPC, without specifying the return value.  Use
+   * this only if the return value was not delayed
+   * @throws IOException
+   */
+  public void endDelay() throws IOException;
+
+  /**
+   * End the call, throwing and exception to the caller.  This works regardless
+   * of the return value being delayed.
+   * @param t Object to throw to the client.
+   * @throws IOException
+   */
+  public void endDelayThrowing(Throwable t) throws IOException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/ExecRPCInvoker.java b/src/main/java/org/apache/hadoop/hbase/ipc/ExecRPCInvoker.java
new file mode 100644
index 0000000..b8b290c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/ExecRPCInvoker.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.client.coprocessor.Exec;
+import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+
+/**
+ * Backs a {@link CoprocessorProtocol} subclass proxy and forwards method
+ * invocations for server execution.  Note that internally this will issue a
+ * separate RPC call for each method invocation (using a
+ * {@link org.apache.hadoop.hbase.client.ServerCallable} instance).
+ */
+public class ExecRPCInvoker implements InvocationHandler {
+  // LOG is NOT in hbase subpackage intentionally so that the default HBase
+  // DEBUG log level does NOT emit RPC-level logging. 
+  private static final Log LOG = LogFactory.getLog("org.apache.hadoop.ipc.ExecRPCInvoker");
+
+  private Configuration conf;
+  private final HConnection connection;
+  private Class protocol;
+  private final byte[] table;
+  private final byte[] row;
+  private byte[] regionName;
+
+  public ExecRPCInvoker(Configuration conf,
+      HConnection connection,
+      Class protocol,
+      byte[] table,
+      byte[] row) {
+    this.conf = conf;
+    this.connection = connection;
+    this.protocol = protocol;
+    this.table = table;
+    this.row = row;
+  }
+
+  @Override
+  public Object invoke(Object instance, final Method method, final Object[] args)
+      throws Throwable {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Call: "+method.getName()+", "+(args != null ? args.length : 0));
+    }
+
+    if (row != null) {
+      final Exec exec = new Exec(conf, row, protocol, method, args);
+      ServerCallable callable =
+          new ServerCallable(connection, table, row) {
+            public ExecResult call() throws Exception {
+              return server.execCoprocessor(location.getRegionInfo().getRegionName(),
+                  exec);
+            }
+          };
+      ExecResult result = callable.withRetries();
+      this.regionName = result.getRegionName();
+      LOG.debug("Result is region="+ Bytes.toStringBinary(regionName) +
+          ", value="+result.getValue());
+      return result.getValue();
+    }
+
+    return null;
+  }
+
+  public byte[] getRegionName() {
+    return regionName;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
new file mode 100644
index 0000000..c5b76c9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -0,0 +1,1186 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.ConnectException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.net.SocketFactory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.PoolMap;
+import org.apache.hadoop.hbase.util.PoolMap.PoolType;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/** A client for an IPC service.  IPC calls take a single {@link Writable} as a
+ * parameter, and return a {@link Writable} as their value.  A service runs on
+ * a port and is defined by a parameter class and a value class.
+ *
+ * 
      This is the org.apache.hadoop.ipc.Client renamed as HBaseClient and
+ * moved into this package so can access package-private methods.
+ *
+ * @see HBaseServer
+ */
+public class HBaseClient {
+
+  private static final Log LOG = LogFactory
+      .getLog("org.apache.hadoop.ipc.HBaseClient");
+  protected final PoolMap connections;
+
+  protected final Class valueClass;   // class of call values
+  protected int counter;                            // counter for call ids
+  protected final AtomicBoolean running = new AtomicBoolean(true); // if client runs
+  final protected Configuration conf;
+  final protected int maxIdleTime; // connections will be culled if it was idle for
+                           // maxIdleTime microsecs
+  final protected int maxRetries; //the max. no. of retries for socket connections
+  final protected long failureSleep; // Time to sleep before retry on failure.
+  protected final boolean tcpNoDelay; // if T then disable Nagle's Algorithm
+  protected final boolean tcpKeepAlive; // if T then use keepalives
+  protected int pingInterval; // how often sends ping to the server in msecs
+  protected int socketTimeout; // socket timeout
+  protected FailedServers failedServers;
+
+  protected final SocketFactory socketFactory;           // how to create sockets
+  protected String clusterId;
+
+  final private static String PING_INTERVAL_NAME = "ipc.ping.interval";
+  final private static String SOCKET_TIMEOUT = "ipc.socket.timeout";
+  final static int DEFAULT_PING_INTERVAL = 60000;  // 1 min
+  final static int DEFAULT_SOCKET_TIMEOUT = 20000; // 20 seconds
+  final static int PING_CALL_ID = -1;
+
+  public final static String FAILED_SERVER_EXPIRY_KEY = "hbase.ipc.client.failed.servers.expiry";
+  public final static int FAILED_SERVER_EXPIRY_DEFAULT = 2000;
+
+  /**
+   * A class to manage a list of servers that failed recently.
+   */
+  static class FailedServers {
+    private final LinkedList> failedServers = new
+        LinkedList>();
+    private final int recheckServersTimeout;
+
+    FailedServers(Configuration conf) {
+      this.recheckServersTimeout = conf.getInt(
+          FAILED_SERVER_EXPIRY_KEY, FAILED_SERVER_EXPIRY_DEFAULT);
+    }
+
+    /**
+     * Add an address to the list of the failed servers list.
+     */
+    public synchronized void addToFailedServers(InetSocketAddress address) {
+      final long expiry = EnvironmentEdgeManager.currentTimeMillis() + recheckServersTimeout;
+      failedServers.addFirst(new Pair(expiry, address.toString()));
+    }
+
+    /**
+     * Check if the server should be considered as bad. Clean the old entries of the list.
+     *
+     * @return true if the server is in the failed servers list
+     */
+    public synchronized boolean isFailedServer(final InetSocketAddress address) {
+      if (failedServers.isEmpty()) {
+        return false;
+      }
+
+      final String lookup = address.toString();
+      final long now = EnvironmentEdgeManager.currentTimeMillis();
+
+      // iterate, looking for the search entry and cleaning expired entries
+      Iterator> it = failedServers.iterator();
+      while (it.hasNext()) {
+        Pair cur = it.next();
+        if (cur.getFirst() < now) {
+          it.remove();
+        } else {
+          if (lookup.equals(cur.getSecond())) {
+            return true;
+          }
+        }
+      }
+
+      return false;
+    }
+
+  }
+
+  public static class FailedServerException extends IOException {
+    public FailedServerException(String s) {
+      super(s);
+    }
+  }
+
+
+  /**
+   * set the ping interval value in configuration
+   *
+   * @param conf Configuration
+   * @param pingInterval the ping interval
+   */
+  public static void setPingInterval(Configuration conf, int pingInterval) {
+    conf.setInt(PING_INTERVAL_NAME, pingInterval);
+  }
+
+  /**
+   * Get the ping interval from configuration;
+   * If not set in the configuration, return the default value.
+   *
+   * @param conf Configuration
+   * @return the ping interval
+   */
+  static int getPingInterval(Configuration conf) {
+    return conf.getInt(PING_INTERVAL_NAME, DEFAULT_PING_INTERVAL);
+  }
+
+  /**
+   * Set the socket timeout
+   * @param conf Configuration
+   * @param socketTimeout the socket timeout
+   */
+  public static void setSocketTimeout(Configuration conf, int socketTimeout) {
+    conf.setInt(SOCKET_TIMEOUT, socketTimeout);
+  }
+
+  /**
+   * @return the socket timeout
+   */
+  static int getSocketTimeout(Configuration conf) {
+    return conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT);
+  }
+
+  /** A call waiting for a value. */
+  protected class Call {
+    final int id;                                       // call id
+    final Writable param;                               // parameter
+    Writable value;                               // value, null if error
+    IOException error;                            // exception, null if value
+    boolean done;                                 // true when call is done
+    long startTime;
+
+    protected Call(Writable param) {
+      this.param = param;
+      this.startTime = System.currentTimeMillis();
+      synchronized (HBaseClient.this) {
+        this.id = counter++;
+      }
+    }
+
+    /** Indicate when the call is complete and the
+     * value or error are available.  Notifies by default.  */
+    protected synchronized void callComplete() {
+      this.done = true;
+      notify();                                 // notify caller
+    }
+
+    /** Set the exception when there is an error.
+     * Notify the caller the call is done.
+     *
+     * @param error exception thrown by the call; either local or remote
+     */
+    public synchronized void setException(IOException error) {
+      this.error = error;
+      callComplete();
+    }
+
+    /** Set the return value when there is no error.
+     * Notify the caller the call is done.
+     *
+     * @param value return value of the call.
+     */
+    public synchronized void setValue(Writable value) {
+      this.value = value;
+      callComplete();
+    }
+
+    public long getStartTime() {
+      return this.startTime;
+    }
+  }
+
+  /**
+   * Creates a connection. Can be overridden by a subclass for testing.
+   *
+   * @param remoteId - the ConnectionId to use for the connection creation.
+   */
+  protected Connection createConnection(ConnectionId remoteId) throws IOException {
+    return new Connection(remoteId);
+  }
+
+  /** Thread that reads responses and notifies callers.  Each connection owns a
+   * socket connected to a remote address.  Calls are multiplexed through this
+   * socket: responses may be delivered out of order. */
+  protected class Connection extends Thread {
+    private ConnectionHeader header;              // connection header
+    protected ConnectionId remoteId;
+    protected Socket socket = null;                 // connected socket
+    protected DataInputStream in;
+    protected DataOutputStream out;
+
+    // currently active calls
+    protected final ConcurrentSkipListMap calls = new ConcurrentSkipListMap();
+    protected final AtomicLong lastActivity = new AtomicLong();// last I/O activity time
+    protected final AtomicBoolean shouldCloseConnection = new AtomicBoolean();  // indicate if the connection is closed
+    protected IOException closeException; // close reason
+
+    Connection(ConnectionId remoteId) throws IOException {
+      if (remoteId.getAddress().isUnresolved()) {
+        throw new UnknownHostException("unknown host: " +
+                                       remoteId.getAddress().getHostName());
+      }
+      this.remoteId = remoteId;
+      User ticket = remoteId.getTicket();
+      Class protocol = remoteId.getProtocol();
+
+      header = new ConnectionHeader(
+          protocol == null ? null : protocol.getName(), ticket);
+
+      this.setName("IPC Client (" + socketFactory.hashCode() +") connection to " +
+        remoteId.getAddress().toString() +
+        ((ticket==null)?" from an unknown user": (" from " + ticket.getName())));
+      this.setDaemon(true);
+    }
+
+    /** Update lastActivity with the current time. */
+    protected void touch() {
+      lastActivity.set(System.currentTimeMillis());
+    }
+
+    /**
+     * Add a call to this connection's call queue and notify
+     * a listener; synchronized. If the connection is dead, the call is not added, and the
+     * caller is notified.
+     * This function can return a connection that is already marked as 'shouldCloseConnection'
+     *  It is up to the user code to check this status.
+     * @param call to add
+     */
+    protected synchronized void addCall(Call call) {
+      // If the connection is about to close, we manage this as if the call was already added
+      //  to the connection calls list. If not, the connection creations are serialized, as
+      //  mentioned in HBASE-6364
+      if (this.shouldCloseConnection.get()) {
+        if (this.closeException == null) {
+          call.setException(new IOException(
+              "Call " + call.id + " not added as the connection " + remoteId + " is closing"));
+        } else {
+          call.setException(this.closeException);
+        }
+        synchronized (call) {
+          call.notifyAll();
+        }
+      } else {
+        calls.put(call.id, call);
+        notify();
+      }
+    }
+
+    /** This class sends a ping to the remote side when timeout on
+     * reading. If no failure is detected, it retries until at least
+     * a byte is read.
+     */
+    protected class PingInputStream extends FilterInputStream {
+      /* constructor */
+      protected PingInputStream(InputStream in) {
+        super(in);
+      }
+
+      /* Process timeout exception
+       * if the connection is not going to be closed, send a ping.
+       * otherwise, throw the timeout exception.
+       */
+      private void handleTimeout(SocketTimeoutException e) throws IOException {
+        if (shouldCloseConnection.get() || !running.get() ||
+            remoteId.rpcTimeout > 0) {
+          throw e;
+        }
+        sendPing();
+      }
+
+      /** Read a byte from the stream.
+       * Send a ping if timeout on read. Retries if no failure is detected
+       * until a byte is read.
+       * @throws IOException for any IO problem other than socket timeout
+       */
+      @Override
+      public int read() throws IOException {
+        do {
+          try {
+            return super.read();
+          } catch (SocketTimeoutException e) {
+            handleTimeout(e);
+          }
+        } while (true);
+      }
+
+      /** Read bytes into a buffer starting from offset off
+       * Send a ping if timeout on read. Retries if no failure is detected
+       * until a byte is read.
+       *
+       * @return the total number of bytes read; -1 if the connection is closed.
+       */
+      @Override
+      public int read(byte[] buf, int off, int len) throws IOException {
+        do {
+          try {
+            return super.read(buf, off, len);
+          } catch (SocketTimeoutException e) {
+            handleTimeout(e);
+          }
+        } while (true);
+      }
+    }
+
+    protected synchronized void setupConnection() throws IOException {
+      short ioFailures = 0;
+      short timeoutFailures = 0;
+      while (true) {
+        try {
+          this.socket = socketFactory.createSocket();
+          this.socket.setTcpNoDelay(tcpNoDelay);
+          this.socket.setKeepAlive(tcpKeepAlive);
+          // connection time out is 20s
+          NetUtils.connect(this.socket, remoteId.getAddress(),
+              getSocketTimeout(conf));
+          if (remoteId.rpcTimeout > 0) {
+            pingInterval = remoteId.rpcTimeout; // overwrite pingInterval
+          }
+          this.socket.setSoTimeout(pingInterval);
+          return;
+        } catch (SocketTimeoutException toe) {
+          /* The max number of retries is 45,
+           * which amounts to 20s*45 = 15 minutes retries.
+           */
+          handleConnectionFailure(timeoutFailures++, maxRetries, toe);
+        } catch (IOException ie) {
+          handleConnectionFailure(ioFailures++, maxRetries, ie);
+        }
+      }
+    }
+
+    /** Connect to the server and set up the I/O streams. It then sends
+     * a header to the server and starts
+     * the connection thread that waits for responses.
+     * @throws java.io.IOException e
+     */
+    protected synchronized void setupIOstreams()
+        throws IOException, InterruptedException {
+
+      if (socket != null || shouldCloseConnection.get()) {
+        return;
+      }
+
+      if (failedServers.isFailedServer(remoteId.getAddress())) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Not trying to connect to " + remoteId.getAddress() +
+              " this server is in the failed servers list");
+        }
+        IOException e = new FailedServerException(
+            "This server is in the failed servers list: " + remoteId.getAddress());
+        markClosed(e);
+        close();
+        throw e;
+      }
+
+      try {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Connecting to "+remoteId);
+        }
+        setupConnection();
+        this.in = new DataInputStream(new BufferedInputStream
+            (new PingInputStream(NetUtils.getInputStream(socket))));
+        this.out = new DataOutputStream
+            (new BufferedOutputStream(NetUtils.getOutputStream(socket)));
+        writeHeader();
+
+        // update last activity time
+        touch();
+
+        // start the receiver thread after the socket connection has been set up
+        start();
+      } catch (IOException e) {
+        failedServers.addToFailedServers(remoteId.address);
+        markClosed(e);
+        close();
+
+        throw e;
+      }
+    }
+
+    protected void closeConnection() {
+      // close the current connection
+      if (socket != null) {
+        try {
+          socket.close();
+        } catch (IOException e) {
+          LOG.warn("Not able to close a socket", e);
+        }
+      }
+      // set socket to null so that the next call to setupIOstreams
+      // can start the process of connect all over again.
+      socket = null;
+    }
+
+    /**
+     *  Handle connection failures
+     *
+     * If the current number of retries is equal to the max number of retries,
+     * stop retrying and throw the exception; Otherwise backoff N seconds and
+     * try connecting again.
+     *
+     * This Method is only called from inside setupIOstreams(), which is
+     * synchronized. Hence the sleep is synchronized; the locks will be retained.
+     *
+     * @param curRetries current number of retries
+     * @param maxRetries max number of retries allowed
+     * @param ioe failure reason
+     * @throws IOException if max number of retries is reached
+     */
+    private void handleConnectionFailure(
+        int curRetries, int maxRetries, IOException ioe) throws IOException {
+
+      closeConnection();
+
+      // throw the exception if the maximum number of retries is reached
+      if (curRetries >= maxRetries) {
+        throw ioe;
+      }
+
+      // otherwise back off and retry
+      try {
+        Thread.sleep(failureSleep);
+      } catch (InterruptedException ignored) {}
+
+      LOG.info("Retrying connect to server: " + remoteId.getAddress() +
+        " after sleeping " + failureSleep + "ms. Already tried " + curRetries +
+        " time(s).");
+    }
+
+    /* Write the header for each connection
+     * Out is not synchronized because only the first thread does this.
+     */
+    private void writeHeader() throws IOException {
+      out.write(HBaseServer.HEADER.array());
+      out.write(HBaseServer.CURRENT_VERSION);
+      //When there are more fields we can have ConnectionHeader Writable.
+      DataOutputBuffer buf = new DataOutputBuffer();
+      header.write(buf);
+
+      int bufLen = buf.getLength();
+      out.writeInt(bufLen);
+      out.write(buf.getData(), 0, bufLen);
+    }
+
+    /* wait till someone signals us to start reading RPC response or
+     * it is idle too long, it is marked as to be closed,
+     * or the client is marked as not running.
+     *
+     * Return true if it is time to read a response; false otherwise.
+     */
+    @SuppressWarnings({"ThrowableInstanceNeverThrown"})
+    protected synchronized boolean waitForWork() {
+      if (calls.isEmpty() && !shouldCloseConnection.get()  && running.get())  {
+        long timeout = maxIdleTime-
+              (System.currentTimeMillis()-lastActivity.get());
+        if (timeout>0) {
+          try {
+            wait(timeout);
+          } catch (InterruptedException ignored) {}
+        }
+      }
+
+      if (!calls.isEmpty() && !shouldCloseConnection.get() && running.get()) {
+        return true;
+      } else if (shouldCloseConnection.get()) {
+        return false;
+      } else if (calls.isEmpty()) { // idle connection closed or stopped
+        markClosed(null);
+        return false;
+      } else { // get stopped but there are still pending requests
+        markClosed((IOException)new IOException().initCause(
+            new InterruptedException()));
+        return false;
+      }
+    }
+
+    public InetSocketAddress getRemoteAddress() {
+      return remoteId.getAddress();
+    }
+
+    /* Send a ping to the server if the time elapsed
+     * since last I/O activity is equal to or greater than the ping interval
+     */
+    protected synchronized void sendPing() throws IOException {
+      long curTime = System.currentTimeMillis();
+      if ( curTime - lastActivity.get() >= pingInterval) {
+        lastActivity.set(curTime);
+        //noinspection SynchronizeOnNonFinalField
+        synchronized (this.out) {
+          out.writeInt(PING_CALL_ID);
+          out.flush();
+        }
+      }
+    }
+
+    @Override
+    public void run() {
+      if (LOG.isDebugEnabled())
+        LOG.debug(getName() + ": starting, having connections "
+            + connections.size());
+
+      try {
+        while (waitForWork()) {//wait here for work - read or close connection
+          receiveResponse();
+        }
+      } catch (Throwable t) {
+        LOG.warn("Unexpected exception receiving call responses", t);
+        markClosed(new IOException("Unexpected exception receiving call responses", t));
+      }
+
+      close();
+
+      if (LOG.isDebugEnabled())
+        LOG.debug(getName() + ": stopped, remaining connections "
+            + connections.size());
+    }
+
+    /* Initiates a call by sending the parameter to the remote server.
+     * Note: this is not called from the Connection thread, but by other
+     * threads.
+     */
+    protected void sendParam(Call call) {
+      if (shouldCloseConnection.get()) {
+        return;
+      }
+
+      // For serializing the data to be written.
+
+      final DataOutputBuffer d = new DataOutputBuffer();
+      try {
+        if (LOG.isDebugEnabled())
+          LOG.debug(getName() + " sending #" + call.id);
+
+        d.writeInt(0xdeadbeef); // placeholder for data length
+        d.writeInt(call.id);
+        call.param.write(d);
+        byte[] data = d.getData();
+        int dataLength = d.getLength();
+        // fill in the placeholder
+        Bytes.putInt(data, 0, dataLength - 4);
+        //noinspection SynchronizeOnNonFinalField
+        synchronized (this.out) { // FindBugs IS2_INCONSISTENT_SYNC
+          out.write(data, 0, dataLength);
+          out.flush();
+        }
+      } catch(IOException e) {
+        markClosed(e);
+      } finally {
+        //the buffer is just an in-memory buffer, but it is still polite to
+        // close early
+        IOUtils.closeStream(d);
+      }
+    }
+
+    /* Receive a response.
+     * Because only one receiver, so no synchronization on in.
+     */
+    protected void receiveResponse() {
+      if (shouldCloseConnection.get()) {
+        return;
+      }
+      touch();
+
+      try {
+        // See HBaseServer.Call.setResponse for where we write out the response.
+        // It writes the call.id (int), a flag byte, then optionally the length
+        // of the response (int) followed by data.
+
+        // Read the call id.
+        int id = in.readInt();
+
+        if (LOG.isDebugEnabled())
+          LOG.debug(getName() + " got value #" + id);
+        Call call = calls.get(id);
+
+        // Read the flag byte
+        byte flag = in.readByte();
+        boolean isError = ResponseFlag.isError(flag);
+        if (ResponseFlag.isLength(flag)) {
+          // Currently length if present is unused.
+          in.readInt();
+        }
+        int state = in.readInt(); // Read the state.  Currently unused.
+        if (isError) {
+          if (call != null) {
+            //noinspection ThrowableInstanceNeverThrown
+            call.setException(new RemoteException(WritableUtils.readString(in),
+                WritableUtils.readString(in)));
+          }
+        } else {
+          Writable value = ReflectionUtils.newInstance(valueClass, conf);
+          value.readFields(in);                 // read value
+          // it's possible that this call may have been cleaned up due to a RPC
+          // timeout, so check if it still exists before setting the value.
+          if (call != null) {
+            call.setValue(value);
+          }
+        }
+        calls.remove(id);
+      } catch (IOException e) {
+        if (e instanceof SocketTimeoutException && remoteId.rpcTimeout > 0) {
+          // Clean up open calls but don't treat this as a fatal condition,
+          // since we expect certain responses to not make it by the specified
+          // {@link ConnectionId#rpcTimeout}.
+          closeException = e;
+        } else {
+          // Since the server did not respond within the default ping interval
+          // time, treat this as a fatal condition and close this connection
+          markClosed(e);
+        }
+      } finally {
+        if (remoteId.rpcTimeout > 0) {
+          cleanupCalls(remoteId.rpcTimeout);
+        }
+      }
+    }
+
+    protected synchronized void markClosed(IOException e) {
+      if (shouldCloseConnection.compareAndSet(false, true)) {
+        closeException = e;
+        notifyAll();
+      }
+    }
+
+    /** Close the connection. */
+    protected synchronized void close() {
+      if (!shouldCloseConnection.get()) {
+        LOG.error("The connection is not in the closed state");
+        return;
+      }
+
+      // release the resources
+      // first thing to do;take the connection out of the connection list
+      synchronized (connections) {
+        connections.remove(remoteId, this);
+      }
+
+      // close the streams and therefore the socket
+      IOUtils.closeStream(out);
+      IOUtils.closeStream(in);
+
+      // clean up all calls
+      if (closeException == null) {
+        if (!calls.isEmpty()) {
+          LOG.warn(
+              "A connection is closed for no cause and calls are not empty");
+
+          // clean up calls anyway
+          closeException = new IOException("Unexpected closed connection");
+          cleanupCalls();
+        }
+      } else {
+        // log the info
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("closing ipc connection to " + remoteId.address + ": " +
+              closeException.getMessage(),closeException);
+        }
+
+        // cleanup calls
+        cleanupCalls();
+      }
+      if (LOG.isDebugEnabled())
+        LOG.debug(getName() + ": closed");
+    }
+
+    /* Cleanup all calls and mark them as done */
+    protected void cleanupCalls() {
+      cleanupCalls(0);
+    }
+
+    protected void cleanupCalls(long rpcTimeout) {
+      Iterator> itor = calls.entrySet().iterator();
+      while (itor.hasNext()) {
+        Call c = itor.next().getValue();
+        long waitTime = System.currentTimeMillis() - c.getStartTime();
+        if (waitTime >= rpcTimeout) {
+          if (this.closeException == null) {
+            // There may be no exception in the case that there are many calls
+            // being multiplexed over this connection and these are succeeding
+            // fine while this Call object is taking a long time to finish
+            // over on the server; e.g. I just asked the regionserver to bulk
+            // open 3k regions or its a big fat multiput into a heavily-loaded
+            // server (Perhaps this only happens at the extremes?)
+            this.closeException = new CallTimeoutException("Call id=" + c.id +
+              ", waitTime=" + waitTime + ", rpcTimetout=" + rpcTimeout);
+          }
+          c.setException(this.closeException);
+          synchronized (c) {
+            c.notifyAll();
+          }
+          itor.remove();
+        } else {
+          break;
+        }
+      }
+      try {
+        if (!calls.isEmpty()) {
+          Call firstCall = calls.get(calls.firstKey());
+          long maxWaitTime = System.currentTimeMillis() - firstCall.getStartTime();
+          if (maxWaitTime < rpcTimeout) {
+            rpcTimeout -= maxWaitTime;
+          }
+        }
+        if (!shouldCloseConnection.get()) {
+          closeException = null;
+          if (socket != null) {
+            socket.setSoTimeout((int) rpcTimeout);
+          }
+        }
+      } catch (SocketException e) {
+        LOG.debug("Couldn't lower timeout, which may result in longer than expected calls");
+      }
+    }
+  }
+
+  /**
+   * Client-side call timeout
+   */
+  public static class CallTimeoutException extends IOException {
+    public CallTimeoutException(final String msg) {
+      super(msg);
+    }
+  }
+
+  /** Call implementation used for parallel calls. */
+  protected class ParallelCall extends Call {
+    private final ParallelResults results;
+    protected final int index;
+
+    public ParallelCall(Writable param, ParallelResults results, int index) {
+      super(param);
+      this.results = results;
+      this.index = index;
+    }
+
+    /** Deliver result to result collector. */
+    @Override
+    protected void callComplete() {
+      results.callComplete(this);
+    }
+  }
+
+  /** Result collector for parallel calls. */
+  protected static class ParallelResults {
+    protected final Writable[] values;
+    protected int size;
+    protected int count;
+
+    public ParallelResults(int size) {
+      this.values = new Writable[size];
+      this.size = size;
+    }
+
+    /*
+     * Collect a result.
+     */
+    synchronized void callComplete(ParallelCall call) {
+      // FindBugs IS2_INCONSISTENT_SYNC
+      values[call.index] = call.value;            // store the value
+      count++;                                    // count it
+      if (count == size)                          // if all values are in
+        notify();                                 // then notify waiting caller
+    }
+  }
+
+  /**
+   * Construct an IPC client whose values are of the given {@link Writable}
+   * class.
+   * @param valueClass value class
+   * @param conf configuration
+   * @param factory socket factory
+   */
+  public HBaseClient(Class valueClass, Configuration conf,
+      SocketFactory factory) {
+    this.valueClass = valueClass;
+    this.maxIdleTime =
+      conf.getInt("hbase.ipc.client.connection.maxidletime", 10000); //10s
+    this.maxRetries = conf.getInt("hbase.ipc.client.connect.max.retries", 0);
+    this.failureSleep = conf.getInt("hbase.client.pause", 1000);
+    this.tcpNoDelay = conf.getBoolean("hbase.ipc.client.tcpnodelay", false);
+    this.tcpKeepAlive = conf.getBoolean("hbase.ipc.client.tcpkeepalive", true);
+    this.pingInterval = getPingInterval(conf);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("The ping interval is" + this.pingInterval + "ms.");
+    }
+    this.conf = conf;
+    this.socketFactory = factory;
+    this.clusterId = conf.get(HConstants.CLUSTER_ID, "default");
+    this.connections = new PoolMap(
+        getPoolType(conf), getPoolSize(conf));
+    this.failedServers = new FailedServers(conf);
+  }
+
+  /**
+   * Construct an IPC client with the default SocketFactory
+   * @param valueClass value class
+   * @param conf configuration
+   */
+  public HBaseClient(Class valueClass, Configuration conf) {
+    this(valueClass, conf, NetUtils.getDefaultSocketFactory(conf));
+  }
+
+  /**
+   * Return the pool type specified in the configuration, which must be set to
+   * either {@link PoolType#RoundRobin} or {@link PoolType#ThreadLocal},
+   * otherwise default to the former.
+   *
+   * For applications with many user threads, use a small round-robin pool. For
+   * applications with few user threads, you may want to try using a
+   * thread-local pool. In any case, the number of {@link HBaseClient} instances
+   * should not exceed the operating system's hard limit on the number of
+   * connections.
+   *
+   * @param config configuration
+   * @return either a {@link PoolType#RoundRobin} or
+   *         {@link PoolType#ThreadLocal}
+   */
+  protected static PoolType getPoolType(Configuration config) {
+    return PoolType.valueOf(config.get(HConstants.HBASE_CLIENT_IPC_POOL_TYPE),
+        PoolType.RoundRobin, PoolType.ThreadLocal);
+  }
+
+  /**
+   * Return the pool size specified in the configuration, which is applicable only if
+   * the pool type is {@link PoolType#RoundRobin}.
+   *
+   * @param config
+   * @return the maximum pool size
+   */
+  protected static int getPoolSize(Configuration config) {
+    return config.getInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, 1);
+  }
+
+  /** Return the socket factory of this client
+   *
+   * @return this client's socket factory
+   */
+  SocketFactory getSocketFactory() {
+    return socketFactory;
+  }
+
+  /** Stop all threads related to this client.  No further calls may be made
+   * using this client. */
+  public void stop() {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Stopping client");
+    }
+
+    if (!running.compareAndSet(true, false)) {
+      return;
+    }
+
+    // wake up all connections
+    synchronized (connections) {
+      for (Connection conn : connections.values()) {
+        conn.interrupt();
+      }
+    }
+
+    // wait until all connections are closed
+    while (!connections.isEmpty()) {
+      try {
+        Thread.sleep(100);
+      } catch (InterruptedException ignored) {
+      }
+    }
+  }
+
+  /** Make a call, passing param, to the IPC server running at
+   * address, returning the value.  Throws exceptions if there are
+   * network problems or if the remote code threw an exception.
+   * @param param writable parameter
+   * @param address network address
+   * @return Writable
+   * @throws IOException e
+   */
+  public Writable call(Writable param, InetSocketAddress address)
+  throws IOException, InterruptedException {
+      return call(param, address, null, 0);
+  }
+
+  public Writable call(Writable param, InetSocketAddress addr,
+                       User ticket, int rpcTimeout)
+                       throws IOException, InterruptedException {
+    return call(param, addr, null, ticket, rpcTimeout);
+  }
+
+  /** Make a call, passing param, to the IPC server running at
+   * address which is servicing the protocol protocol,
+   * with the ticket credentials, returning the value.
+   * Throws exceptions if there are network problems or if the remote code
+   * threw an exception. */
+  public Writable call(Writable param, InetSocketAddress addr,
+                       Class protocol,
+                       User ticket, int rpcTimeout)
+      throws InterruptedException, IOException {
+    Call call = new Call(param);
+    Connection connection = getConnection(addr, protocol, ticket, rpcTimeout, call);
+    connection.sendParam(call);                 // send the parameter
+    boolean interrupted = false;
+    //noinspection SynchronizationOnLocalVariableOrMethodParameter
+    synchronized (call) {
+      while (!call.done) {
+        try {
+          call.wait();                           // wait for the result
+        } catch (InterruptedException ignored) {
+          // save the fact that we were interrupted
+          interrupted = true;
+        }
+      }
+
+      if (interrupted) {
+        // set the interrupt flag now that we are done waiting
+        Thread.currentThread().interrupt();
+      }
+
+      if (call.error != null) {
+        if (call.error instanceof RemoteException) {
+          call.error.fillInStackTrace();
+          throw call.error;
+        }
+        // local exception
+        throw wrapException(addr, call.error);
+      }
+      return call.value;
+    }
+  }
+
+  /**
+   * Take an IOException and the address we were trying to connect to
+   * and return an IOException with the input exception as the cause.
+   * The new exception provides the stack trace of the place where
+   * the exception is thrown and some extra diagnostics information.
+   * If the exception is ConnectException or SocketTimeoutException,
+   * return a new one of the same type; Otherwise return an IOException.
+   *
+   * @param addr target address
+   * @param exception the relevant exception
+   * @return an exception to throw
+   */
+  @SuppressWarnings({"ThrowableInstanceNeverThrown"})
+  protected IOException wrapException(InetSocketAddress addr,
+                                         IOException exception) {
+    if (exception instanceof ConnectException) {
+      //connection refused; include the host:port in the error
+      return (ConnectException)new ConnectException(
+           "Call to " + addr + " failed on connection exception: " + exception)
+                    .initCause(exception);
+    } else if (exception instanceof SocketTimeoutException) {
+      return (SocketTimeoutException)new SocketTimeoutException(
+           "Call to " + addr + " failed on socket timeout exception: "
+                      + exception).initCause(exception);
+    } else {
+      return (IOException)new IOException(
+           "Call to " + addr + " failed on local exception: " + exception)
+                                 .initCause(exception);
+
+    }
+  }
+
+  /** Makes a set of calls in parallel.  Each parameter is sent to the
+   * corresponding address.  When all values are available, or have timed out
+   * or errored, the collected results are returned in an array.  The array
+   * contains nulls for calls that timed out or errored.
+   * @param params writable parameters
+   * @param addresses socket addresses
+   * @return  Writable[]
+   * @throws IOException e
+   * @deprecated Use {@link #call(Writable[], InetSocketAddress[], Class, User)} instead
+   */
+  @Deprecated
+  public Writable[] call(Writable[] params, InetSocketAddress[] addresses)
+    throws IOException, InterruptedException {
+    return call(params, addresses, null, null);
+  }
+
+  /** Makes a set of calls in parallel.  Each parameter is sent to the
+   * corresponding address.  When all values are available, or have timed out
+   * or errored, the collected results are returned in an array.  The array
+   * contains nulls for calls that timed out or errored.  */
+  public Writable[] call(Writable[] params, InetSocketAddress[] addresses,
+                         Class protocol,
+                         User ticket)
+      throws IOException, InterruptedException {
+    if (addresses.length == 0) return new Writable[0];
+
+    ParallelResults results = new ParallelResults(params.length);
+    // TODO this synchronization block doesnt make any sense, we should possibly fix it
+    //noinspection SynchronizationOnLocalVariableOrMethodParameter
+    synchronized (results) {
+      for (int i = 0; i < params.length; i++) {
+        ParallelCall call = new ParallelCall(params[i], results, i);
+        try {
+          Connection connection =
+              getConnection(addresses[i], protocol, ticket, 0, call);
+          connection.sendParam(call);             // send each parameter
+        } catch (IOException e) {
+          // log errors
+          LOG.info("Calling "+addresses[i]+" caught: " +
+                   e.getMessage(),e);
+          results.size--;                         //  wait for one fewer result
+        }
+      }
+      while (results.count != results.size) {
+        try {
+          results.wait();                    // wait for all results
+        } catch (InterruptedException ignored) {}
+      }
+
+      return results.values;
+    }
+  }
+
+  /* Get a connection from the pool, or create a new one and add it to the
+   * pool.  Connections to a given host/port are reused. */
+  protected Connection getConnection(InetSocketAddress addr,
+                                   Class protocol,
+                                   User ticket,
+                                   int rpcTimeout,
+                                   Call call)
+                                   throws IOException, InterruptedException {
+    if (!running.get()) {
+      // the client is stopped
+      throw new IOException("The client is stopped");
+    }
+    Connection connection;
+    /* we could avoid this allocation for each RPC by having a
+     * connectionsId object and with set() method. We need to manage the
+     * refs for keys in HashMap properly. For now its ok.
+     */
+    ConnectionId remoteId = new ConnectionId(addr, protocol, ticket, rpcTimeout);
+    synchronized (connections) {
+      connection = connections.get(remoteId);
+      if (connection == null) {
+        connection = createConnection(remoteId);
+        connections.put(remoteId, connection);
+      }
+    }
+    connection.addCall(call);
+
+    //we don't invoke the method below inside "synchronized (connections)"
+    //block above. The reason for that is if the server happens to be slow,
+    //it will take longer to establish a connection and that will slow the
+    //entire system down.
+    //Moreover, if the connection is currently created, there will be many threads
+    // waiting here; as setupIOstreams is synchronized. If the connection fails with a
+    // timeout, they will all fail simultaneously. This is checked in setupIOstreams.
+    connection.setupIOstreams();
+    return connection;
+  }
+
+  /**
+   * This class holds the address and the user ticket. The client connections
+   * to servers are uniquely identified by 
+   */
+  protected static class ConnectionId {
+    final InetSocketAddress address;
+    final User ticket;
+    final int rpcTimeout;
+    Class protocol;
+    private static final int PRIME = 16777619;
+
+    ConnectionId(InetSocketAddress address,
+        Class protocol,
+        User ticket,
+        int rpcTimeout) {
+      this.protocol = protocol;
+      this.address = address;
+      this.ticket = ticket;
+      this.rpcTimeout = rpcTimeout;
+    }
+
+    InetSocketAddress getAddress() {
+      return address;
+    }
+
+    Class getProtocol() {
+      return protocol;
+    }
+
+    User getTicket() {
+      return ticket;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+     if (obj instanceof ConnectionId) {
+       ConnectionId id = (ConnectionId) obj;
+       return address.equals(id.address) && protocol == id.protocol &&
+              ((ticket != null && ticket.equals(id.ticket)) ||
+               (ticket == id.ticket)) && rpcTimeout == id.rpcTimeout;
+     }
+     return false;
+    }
+
+    @Override  // simply use the default Object#hashcode() ?
+    public int hashCode() {
+      return (address.hashCode() + PRIME * (
+                  PRIME * System.identityHashCode(protocol) ^
+             (ticket == null ? 0 : ticket.hashCode()) )) ^ rpcTimeout;
+    }
+  }
+
+  /**
+   * @return the clusterId
+   */
+  public String getClusterId() {
+    return clusterId;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
new file mode 100644
index 0000000..27884f9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
@@ -0,0 +1,353 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.RetriesExhaustedException;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.ReflectionUtils;
+import javax.net.SocketFactory;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.net.ConnectException;
+import java.net.InetSocketAddress;
+import java.net.SocketTimeoutException;
+import java.util.HashMap;
+import java.util.Map;
+
+/** A simple RPC mechanism.
+ *
+ * This is a local hbase copy of the hadoop RPC so we can do things like
+ * address HADOOP-414 for hbase-only and try other hbase-specific
+ * optimizations like using our own version of ObjectWritable.  Class has been
+ * renamed to avoid confusing it w/ hadoop versions.
+ * 
      
+ *
+ *
+ * A protocol is a Java interface.  All parameters and return types must
+ * be one of:
+ *
+ * 
         
      • a primitive type, boolean, byte,
+ * char, short, int, long,
+ * float, double, or void; or
        • 
+ *
+ * 
      • a {@link String}; or
        • 
+ *
+ * 
      • a {@link Writable}; or
        • 
+ *
+ * 
      • an array of the above types
        •  
      
+ *
+ * All methods in the protocol should throw only IOException.  No field data of
+ * the protocol instance is transmitted.
+ */
+public class HBaseRPC {
+  // Leave this out in the hadoop ipc package but keep class name.  Do this
+  // so that we dont' get the logging of this class's invocations by doing our
+  // blanket enabling DEBUG on the o.a.h.h. package.
+  protected static final Log LOG =
+    LogFactory.getLog("org.apache.hadoop.ipc.HBaseRPC");
+
+  private HBaseRPC() {
+    super();
+  }                                  // no public ctor
+
+  /**
+   * Configuration key for the {@link RpcEngine} implementation to load to
+   * handle connection protocols.  Handlers for individual protocols can be
+   * configured using {@code "hbase.rpc.engine." + protocol.class.name}.
+   */
+  public static final String RPC_ENGINE_PROP = "hbase.rpc.engine";
+
+  // thread-specific RPC timeout, which may override that of RpcEngine
+  private static ThreadLocal rpcTimeout = new ThreadLocal() {
+    @Override
+      protected Integer initialValue() {
+        return HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT;
+      }
+    };
+
+  /**
+   * Returns a new instance of the configured {@link RpcEngine} implementation.
+   */
+  public static synchronized RpcEngine getProtocolEngine(Configuration conf) {
+    // check for a configured default engine
+    Class impl =
+        conf.getClass(RPC_ENGINE_PROP, WritableRpcEngine.class);
+
+    LOG.debug("Using RpcEngine: "+impl.getName());
+    RpcEngine engine = (RpcEngine) ReflectionUtils.newInstance(impl, conf);
+    return engine;
+  }
+
+  /**
+   * A version mismatch for the RPC protocol.
+   */
+  @SuppressWarnings("serial")
+  public static class VersionMismatch extends IOException {
+    private static final long serialVersionUID = 0;
+    private String interfaceName;
+    private long clientVersion;
+    private long serverVersion;
+
+    /**
+     * Create a version mismatch exception
+     * @param interfaceName the name of the protocol mismatch
+     * @param clientVersion the client's version of the protocol
+     * @param serverVersion the server's version of the protocol
+     */
+    public VersionMismatch(String interfaceName, long clientVersion,
+                           long serverVersion) {
+      super("Protocol " + interfaceName + " version mismatch. (client = " +
+            clientVersion + ", server = " + serverVersion + ")");
+      this.interfaceName = interfaceName;
+      this.clientVersion = clientVersion;
+      this.serverVersion = serverVersion;
+    }
+
+    /**
+     * Get the interface name
+     * @return the java class name
+     *          (eg. org.apache.hadoop.mapred.InterTrackerProtocol)
+     */
+    public String getInterfaceName() {
+      return interfaceName;
+    }
+
+    /**
+     * @return the client's preferred version
+     */
+    public long getClientVersion() {
+      return clientVersion;
+    }
+
+    /**
+     * @return the server's agreed to version.
+     */
+    public long getServerVersion() {
+      return serverVersion;
+    }
+  }
+
+  /**
+   * An error requesting an RPC protocol that the server is not serving.
+   */
+  public static class UnknownProtocolException extends DoNotRetryIOException {
+    private Class protocol;
+
+    public UnknownProtocolException(String mesg) {
+      // required for unwrapping from a RemoteException
+      super(mesg);
+    }
+
+    public UnknownProtocolException(Class protocol) {
+      this(protocol, "Server is not handling protocol "+protocol.getName());
+    }
+
+    public UnknownProtocolException(Class protocol, String mesg) {
+      super(mesg);
+      this.protocol = protocol;
+    }
+
+    public Class getProtocol() {
+      return protocol;
+    }
+  }
+
+  /**
+   * @param protocol protocol interface
+   * @param clientVersion which client version we expect
+   * @param addr address of remote service
+   * @param conf configuration
+   * @param maxAttempts max attempts
+   * @param rpcTimeout timeout for each RPC
+   * @param timeout timeout in milliseconds
+   * @return proxy
+   * @throws IOException e
+   */
+  @SuppressWarnings("unchecked")
+  public static  T waitForProxy(RpcEngine rpcClient,
+                                               Class protocol,
+                                               long clientVersion,
+                                               InetSocketAddress addr,
+                                               Configuration conf,
+                                               int maxAttempts,
+                                               int rpcTimeout,
+                                               long timeout
+  ) throws IOException {
+    // HBase does limited number of reconnects which is different from hadoop.
+    long startTime = System.currentTimeMillis();
+    IOException ioe;
+    int reconnectAttempts = 0;
+    while (true) {
+      try {
+        return rpcClient.getProxy(protocol, clientVersion, addr, conf, rpcTimeout);
+      } catch(SocketTimeoutException te) {  // namenode is busy
+        LOG.info("Problem connecting to server: " + addr);
+        ioe = te;
+      } catch (IOException ioex) {
+        // We only handle the ConnectException.
+        ConnectException ce = null;
+        if (ioex instanceof ConnectException) {
+          ce = (ConnectException) ioex;
+          ioe = ce;
+        } else if (ioex.getCause() != null
+            && ioex.getCause() instanceof ConnectException) {
+          ce = (ConnectException) ioex.getCause();
+          ioe = ce;
+        } else if (ioex.getMessage().toLowerCase()
+            .contains("connection refused")) {
+          ce = new ConnectException(ioex.getMessage());
+          ioe = ce;
+        } else {
+          // This is the exception we can't handle.
+          ioe = ioex;
+        }
+        if (ce != null) {
+          handleConnectionException(++reconnectAttempts, maxAttempts, protocol,
+              addr, ce);
+        }
+      }
+      // check if timed out
+      if (System.currentTimeMillis() - timeout >= startTime) {
+        throw ioe;
+      }
+
+      // wait for retry
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException ie) {
+        // IGNORE
+      }
+    }
+  }
+
+  /**
+   * @param retries current retried times.
+   * @param maxAttmpts max attempts
+   * @param protocol protocol interface
+   * @param addr address of remote service
+   * @param ce ConnectException
+   * @throws RetriesExhaustedException
+   */
+  private static void handleConnectionException(int retries, int maxAttmpts,
+      Class protocol, InetSocketAddress addr, ConnectException ce)
+      throws RetriesExhaustedException {
+    if (maxAttmpts >= 0 && retries >= maxAttmpts) {
+      LOG.info("Server at " + addr + " could not be reached after "
+          + maxAttmpts + " tries, giving up.");
+      throw new RetriesExhaustedException("Failed setting up proxy " + protocol
+          + " to " + addr.toString() + " after attempts=" + maxAttmpts, ce);
+    }
+  }
+  
+  /**
+   * Expert: Make multiple, parallel calls to a set of servers.
+   *
+   * @param method method to invoke
+   * @param params array of parameters
+   * @param addrs array of addresses
+   * @param conf configuration
+   * @return values
+   * @throws IOException e
+   * @deprecated Instead of calling statically, use
+   *     {@link HBaseRPC#getProtocolEngine(org.apache.hadoop.conf.Configuration)}
+   *     to obtain an {@link RpcEngine} instance and then use
+   *     {@link RpcEngine#call(java.lang.reflect.Method, Object[][], java.net.InetSocketAddress[], Class, org.apache.hadoop.hbase.security.User, org.apache.hadoop.conf.Configuration)}
+   */
+  @Deprecated
+  public static Object[] call(Method method, Object[][] params,
+      InetSocketAddress[] addrs,
+      Class protocol,
+      User ticket,
+      Configuration conf)
+    throws IOException, InterruptedException {
+    Object[] result = null;
+    RpcEngine engine = null;
+    try {
+      engine = getProtocolEngine(conf);
+      result = engine.call(method, params, addrs, protocol, ticket, conf);
+    } finally {
+      engine.close();
+    }
+    return result;
+  }
+
+  /**
+   * Construct a server for a protocol implementation instance listening on a
+   * port and address.
+   *
+   * @param instance instance
+   * @param bindAddress bind address
+   * @param port port to bind to
+   * @param numHandlers number of handlers to start
+   * @param verbose verbose flag
+   * @param conf configuration
+   * @return Server
+   * @throws IOException e
+   */
+  public static RpcServer getServer(final Object instance,
+                                 final Class[] ifaces,
+                                 final String bindAddress, final int port,
+                                 final int numHandlers,
+                                 int metaHandlerCount, final boolean verbose, Configuration conf, int highPriorityLevel)
+    throws IOException {
+    return getServer(instance.getClass(), instance, ifaces, bindAddress, port, numHandlers, metaHandlerCount, verbose, conf, highPriorityLevel);
+  }
+
+  /** Construct a server for a protocol implementation instance. */
+  public static RpcServer getServer(Class protocol,
+                                 final Object instance,
+                                 final Class[] ifaces, String bindAddress,
+                                 int port,
+                                 final int numHandlers,
+                                 int metaHandlerCount, final boolean verbose, Configuration conf, int highPriorityLevel)
+    throws IOException {
+    return getProtocolEngine(conf)
+        .getServer(protocol, instance, ifaces, bindAddress, port, numHandlers, metaHandlerCount, verbose, conf, highPriorityLevel);
+  }
+
+  public static void setRpcTimeout(int rpcTimeout) {
+    HBaseRPC.rpcTimeout.set(rpcTimeout);
+  }
+
+  public static int getRpcTimeout() {
+    return HBaseRPC.rpcTimeout.get();
+  }
+
+  public static void resetRpcTimeout() {
+    HBaseRPC.rpcTimeout.remove();
+  }
+
+  /**
+   * Returns the lower of the thread-local RPC time from {@link #setRpcTimeout(int)} and the given
+   * default timeout.
+   */
+  public static int getRpcTimeout(int defaultTimeout) {
+    return Math.min(defaultTimeout, HBaseRPC.rpcTimeout.get());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCErrorHandler.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCErrorHandler.java
new file mode 100644
index 0000000..ad790b5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCErrorHandler.java
@@ -0,0 +1,33 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+/**
+ * An interface for calling out of RPC for error conditions.
+ */
+public interface HBaseRPCErrorHandler {
+	/**
+	 * Take actions on the event of an OutOfMemoryError.
+	 * @param e the throwable
+	 * @return if the server should be shut down
+	 */
+	public boolean checkOOME(final Throwable e) ;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCStatistics.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCStatistics.java
new file mode 100644
index 0000000..b9fa0dd
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCStatistics.java
@@ -0,0 +1,52 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import org.apache.hadoop.metrics.util.MBeanUtil;
+import org.apache.hadoop.metrics.util.MetricsDynamicMBeanBase;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+
+import javax.management.ObjectName;
+
+/**
+ * Exports HBase RPC statistics recorded in {@link HBaseRpcMetrics} as an MBean
+ * for JMX monitoring.
+ */
+public class HBaseRPCStatistics extends MetricsDynamicMBeanBase {
+  private final ObjectName mbeanName;
+
+  @SuppressWarnings({"UnusedDeclaration"})
+  public HBaseRPCStatistics(MetricsRegistry registry,
+      String hostName, String port) {
+	  super(registry, "Metrics for RPC server instance");
+
+    String name = String.format("RPCStatistics-%s",
+        (port != null ? port : "unknown"));
+
+    mbeanName = MBeanUtil.registerMBean("HBase", name, this);
+  }
+
+  public void shutdown() {
+    if (mbeanName != null)
+      MBeanUtil.unregisterMBean(mbeanName);
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
new file mode 100644
index 0000000..bc897d9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
@@ -0,0 +1,218 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.ipc.VersionedProtocol;
+import org.apache.hadoop.metrics.MetricsContext;
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.MetricsUtil;
+import org.apache.hadoop.metrics.Updater;
+import org.apache.hadoop.metrics.util.*;
+
+import java.lang.reflect.Method;
+
+/**
+ *
+ * This class is for maintaining  the various RPC statistics
+ * and publishing them through the metrics interfaces.
+ * This also registers the JMX MBean for RPC.
+ * 
      
+ * This class has a number of metrics variables that are publicly accessible;
+ * these variables (objects) have methods to update their values;
+ * for example:
+ *  
       {@link #rpcQueueTime}.inc(time)
+ *
+ */
+public class HBaseRpcMetrics implements Updater {
+  public static final String NAME_DELIM = "$";
+  private final MetricsRegistry registry = new MetricsRegistry();
+  private final MetricsRecord metricsRecord;
+  private static Log LOG = LogFactory.getLog(HBaseRpcMetrics.class);
+  private final HBaseRPCStatistics rpcStatistics;
+
+  public HBaseRpcMetrics(String hostName, String port) {
+    MetricsContext context = MetricsUtil.getContext("rpc");
+    metricsRecord = MetricsUtil.createRecord(context, "metrics");
+
+    metricsRecord.setTag("port", port);
+
+    LOG.info("Initializing RPC Metrics with hostName="
+        + hostName + ", port=" + port);
+
+    context.registerUpdater(this);
+
+    initMethods(HMasterInterface.class);
+    initMethods(HMasterRegionInterface.class);
+    initMethods(HRegionInterface.class);
+    rpcStatistics = new HBaseRPCStatistics(this.registry, hostName, port);
+  }
+
+
+  /**
+   * The metrics variables are public:
+   *  - they can be set directly by calling their set/inc methods
+   *  -they can also be read directly - e.g. JMX does this.
+   */
+
+  public final MetricsTimeVaryingLong receivedBytes =
+         new MetricsTimeVaryingLong("ReceivedBytes", registry);
+  public final MetricsTimeVaryingLong sentBytes =
+         new MetricsTimeVaryingLong("SentBytes", registry);
+  public final MetricsTimeVaryingRate rpcQueueTime =
+          new MetricsTimeVaryingRate("RpcQueueTime", registry);
+  public MetricsTimeVaryingRate rpcProcessingTime =
+          new MetricsTimeVaryingRate("RpcProcessingTime", registry);
+  public final MetricsIntValue numOpenConnections =
+          new MetricsIntValue("NumOpenConnections", registry);
+  public final MetricsIntValue callQueueLen =
+          new MetricsIntValue("callQueueLen", registry);
+  public final MetricsIntValue priorityCallQueueLen =
+          new MetricsIntValue("priorityCallQueueLen", registry);
+  public final MetricsTimeVaryingInt authenticationFailures = 
+          new MetricsTimeVaryingInt("rpcAuthenticationFailures", registry);
+  public final MetricsTimeVaryingInt authenticationSuccesses =
+          new MetricsTimeVaryingInt("rpcAuthenticationSuccesses", registry);
+  public final MetricsTimeVaryingInt authorizationFailures =
+          new MetricsTimeVaryingInt("rpcAuthorizationFailures", registry);
+  public final MetricsTimeVaryingInt authorizationSuccesses =
+         new MetricsTimeVaryingInt("rpcAuthorizationSuccesses", registry);
+  public MetricsTimeVaryingRate rpcSlowResponseTime =
+      new MetricsTimeVaryingRate("RpcSlowResponse", registry);
+  public final MetricsIntValue replicationCallQueueLen =
+    new MetricsIntValue("replicationCallQueueLen", registry);
+
+  private void initMethods(Class protocol) {
+    for (Method m : protocol.getDeclaredMethods()) {
+      if (get(m.getName()) == null)
+        create(m.getName());
+    }
+  }
+
+  private MetricsTimeVaryingRate get(String key) {
+    return (MetricsTimeVaryingRate) registry.get(key);
+  }
+  private MetricsTimeVaryingRate create(String key) {
+    return new MetricsTimeVaryingRate(key, this.registry);
+  }
+
+  public void inc(String name, int amt) {
+    MetricsTimeVaryingRate m = get(name);
+    if (m == null) {
+      LOG.warn("Got inc() request for method that doesnt exist: " +
+      name);
+      return; // ignore methods that dont exist.
+    }
+    m.inc(amt);
+  }
+
+  /**
+   * Generate metrics entries for all the methods defined in the list of
+   * interfaces.  A {@link MetricsTimeVaryingRate} counter will be created for
+   * each {@code Class.getMethods().getName()} entry.
+   * @param ifaces Define metrics for all methods in the given classes
+   */
+  public void createMetrics(Class[] ifaces) {
+    createMetrics(ifaces, false);
+  }
+
+  /**
+   * Generate metrics entries for all the methods defined in the list of
+   * interfaces.  A {@link MetricsTimeVaryingRate} counter will be created for
+   * each {@code Class.getMethods().getName()} entry.
+   *
+   * 
      
+   * If {@code prefixWithClass} is {@code true}, each metric will be named as
+   * {@code [Class.getSimpleName()].[Method.getName()]}.  Otherwise each metric
+   * will just be named according to the method -- {@code Method.getName()}.
+   * 
      
+   * @param ifaces Define metrics for all methods in the given classes
+   * @param prefixWithClass If {@code true}, each metric will be named as
+   *     "classname.method"
+   */
+  public void createMetrics(Class[] ifaces, boolean prefixWithClass) {
+    createMetrics(ifaces, prefixWithClass, null);
+  }
+
+  /**
+  * Generate metrics entries for all the methods defined in the list of
+  * interfaces. A {@link MetricsTimeVaryingRate} counter will be created for
+  * each {@code Class.getMethods().getName()} entry.
+  *
+  * 
      
+  * If {@code prefixWithClass} is {@code true}, each metric will be named as
+  * {@code [Class.getSimpleName()].[Method.getName()]}. Otherwise each metric
+  * will just be named according to the method -- {@code Method.getName()}.
+  * 
      
+  *
+  * 
      
+  * Additionally, if {@code suffixes} is defined, additional metrics will be
+  * created for each method named as the original metric concatenated with
+  * the suffix.
+  * 
      
+  * @param ifaces Define metrics for all methods in the given classes
+  * @param prefixWithClass If {@code true}, each metric will be named as
+  * "classname.method"
+  * @param suffixes If not null, each method will get additional metrics ending
+  * in each of the suffixes.
+  */
+  public void createMetrics(Class[] ifaces, boolean prefixWithClass,
+      String [] suffixes) {
+    for (Class iface : ifaces) {
+      Method[] methods = iface.getMethods();
+      for (Method method : methods) {
+        String attrName = prefixWithClass ?
+        getMetricName(iface, method.getName()) : method.getName();
+        if (get(attrName) == null)
+          create(attrName);
+        if (suffixes != null) {
+          // create metrics for each requested suffix
+          for (String s : suffixes) {
+            String metricName = attrName + s;
+            if (get(metricName) == null)
+              create(metricName);
+          }
+        }
+      }
+    }
+  }
+
+  public static String getMetricName(Class c, String method) {
+    return c.getSimpleName() + NAME_DELIM + method;
+  }
+
+  /**
+   * Push the metrics to the monitoring subsystem on doUpdate() call.
+   */
+  public void doUpdates(final MetricsContext context) {
+    // Both getMetricsList() and pushMetric() are thread-safe
+    for (MetricsBase m : registry.getMetricsList()) {
+      m.pushMetric(metricsRecord);
+    }
+    metricsRecord.update();
+  }
+
+  public void shutdown() {
+    if (rpcStatistics != null)
+      rpcStatistics.shutdown();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
new file mode 100644
index 0000000..8813a00
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
@@ -0,0 +1,1857 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.BindException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.nio.channels.CancelledKeyException;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.nio.channels.WritableByteChannel;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.HbaseObjectWritable;
+import org.apache.hadoop.hbase.io.WritableWithSize;
+import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.ByteBufferOutputStream;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.SizeBasedThrottler;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.ipc.RPC.VersionMismatch;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.cliffc.high_scale_lib.Counter;
+
+import com.google.common.base.Function;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/** An abstract IPC service.  IPC calls take a single {@link Writable} as a
+ * parameter, and return a {@link Writable} as their value.  A service runs on
+ * a port and is defined by a parameter class and a value class.
+ *
+ *
+ * 
      Copied local so can fix HBASE-900.
+ *
+ * @see HBaseClient
+ */
+public abstract class HBaseServer implements RpcServer {
+
+  /**
+   * The first four bytes of Hadoop RPC connections
+   */
+  public static final ByteBuffer HEADER = ByteBuffer.wrap("hrpc".getBytes());
+  public static final byte CURRENT_VERSION = 3;
+
+  /**
+   * How many calls/handler are allowed in the queue.
+   */
+  private static final int DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER = 10;
+
+  /**
+   * The maximum size that we can hold in the IPC queue
+   */
+  private static final int DEFAULT_MAX_CALLQUEUE_SIZE =
+    1024 * 1024 * 1024;
+
+  static final int BUFFER_INITIAL_SIZE = 1024;
+
+  private static final String WARN_DELAYED_CALLS =
+      "hbase.ipc.warn.delayedrpc.number";
+
+  private static final int DEFAULT_WARN_DELAYED_CALLS = 1000;
+
+  private final int warnDelayedCalls;
+
+  private AtomicInteger delayedCalls;
+
+  public static final Log LOG =
+    LogFactory.getLog("org.apache.hadoop.ipc.HBaseServer");
+  protected static final Log TRACELOG =
+      LogFactory.getLog("org.apache.hadoop.ipc.HBaseServer.trace");
+
+  protected static final ThreadLocal SERVER =
+    new ThreadLocal();
+  private volatile boolean started = false;
+
+  private static final Map>
+      PROTOCOL_CACHE =
+      new ConcurrentHashMap>();
+
+  static Class getProtocolClass(
+      String protocolName, Configuration conf)
+  throws ClassNotFoundException {
+    Class protocol =
+        PROTOCOL_CACHE.get(protocolName);
+
+    if (protocol == null) {
+      protocol = (Class)
+          conf.getClassByName(protocolName);
+      PROTOCOL_CACHE.put(protocolName, protocol);
+    }
+    return protocol;
+  }
+
+  /** Returns the server instance called under or null.  May be called under
+   * {@link #call(Class, Writable, long, MonitoredRPCHandler)} implementations,
+   * and under {@link Writable} methods of paramters and return values.
+   * Permits applications to access the server context.
+   * @return HBaseServer
+   */
+  public static RpcServer get() {
+    return SERVER.get();
+  }
+
+  /** This is set to Call object before Handler invokes an RPC and reset
+   * after the call returns.
+   */
+  protected static final ThreadLocal CurCall = new ThreadLocal();
+
+  /** Returns the remote side ip address when invoked inside an RPC
+   *  Returns null incase of an error.
+   *  @return InetAddress
+   */
+  public static InetAddress getRemoteIp() {
+    Call call = CurCall.get();
+    if (call != null) {
+      return call.connection.socket.getInetAddress();
+    }
+    return null;
+  }
+  /** Returns remote address as a string when invoked inside an RPC.
+   *  Returns null in case of an error.
+   *  @return String
+   */
+  public static String getRemoteAddress() {
+    Call call = CurCall.get();
+    if (call != null) {
+      return call.connection.getHostAddress();
+    }
+    return null;
+  }
+
+  protected String bindAddress;
+  protected int port;                             // port we listen on
+  private int handlerCount;                       // number of handler threads
+  private int priorityHandlerCount;
+  private int readThreads;                        // number of read threads
+  protected Class paramClass; // class of call parameters
+  protected int maxIdleTime;                      // the maximum idle time after
+                                                  // which a client may be
+                                                  // disconnected
+  protected int thresholdIdleConnections;         // the number of idle
+                                                  // connections after which we
+                                                  // will start cleaning up idle
+                                                  // connections
+  int maxConnectionsToNuke;                       // the max number of
+                                                  // connections to nuke
+                                                  // during a cleanup
+
+  protected HBaseRpcMetrics  rpcMetrics;
+
+  protected Configuration conf;
+
+  private int maxQueueLength;
+  private int maxQueueSize;
+  protected int socketSendBufferSize;
+  protected final boolean tcpNoDelay;   // if T then disable Nagle's Algorithm
+  protected final boolean tcpKeepAlive; // if T then use keepalives
+  protected final long purgeTimeout;    // in milliseconds
+
+  // responseQueuesSizeThrottler is shared among all responseQueues,
+  // it bounds memory occupied by responses in all responseQueues
+  final SizeBasedThrottler responseQueuesSizeThrottler;
+
+  // RESPONSE_QUEUE_MAX_SIZE limits total size of responses in every response queue
+  private static final long DEFAULT_RESPONSE_QUEUES_MAX_SIZE = 1024 * 1024 * 1024; // 1G
+  private static final String RESPONSE_QUEUES_MAX_SIZE = "ipc.server.response.queue.maxsize";
+
+  volatile protected boolean running = true;         // true while server runs
+  protected BlockingQueue callQueue; // queued calls
+  protected final Counter callQueueSize = new Counter();
+  protected BlockingQueue priorityCallQueue;
+
+  protected int highPriorityLevel;  // what level a high priority call is at
+
+  protected final List connectionList =
+    Collections.synchronizedList(new LinkedList());
+  //maintain a list
+  //of client connections
+  private Listener listener = null;
+  protected Responder responder = null;
+  protected int numConnections = 0;
+  private Handler[] handlers = null;
+  private Handler[] priorityHandlers = null;
+  /** replication related queue; */
+  protected BlockingQueue replicationQueue;
+  private int numOfReplicationHandlers = 0;
+  private Handler[] replicationHandlers = null;
+  protected HBaseRPCErrorHandler errorHandler = null;
+
+  /**
+   * A convenience method to bind to a given address and report
+   * better exceptions if the address is not a valid host.
+   * @param socket the socket to bind
+   * @param address the address to bind to
+   * @param backlog the number of connections allowed in the queue
+   * @throws BindException if the address can't be bound
+   * @throws UnknownHostException if the address isn't a valid host name
+   * @throws IOException other random errors from bind
+   */
+  public static void bind(ServerSocket socket, InetSocketAddress address,
+                          int backlog) throws IOException {
+    try {
+      socket.bind(address, backlog);
+    } catch (BindException e) {
+      BindException bindException =
+        new BindException("Problem binding to " + address + " : " +
+            e.getMessage());
+      bindException.initCause(e);
+      throw bindException;
+    } catch (SocketException e) {
+      // If they try to bind to a different host's address, give a better
+      // error message.
+      if ("Unresolved address".equals(e.getMessage())) {
+        throw new UnknownHostException("Invalid hostname for server: " +
+                                       address.getHostName());
+      }
+      throw e;
+    }
+  }
+
+  /** A call queued for handling. */
+  protected class Call implements RpcCallContext {
+    protected int id;                             // the client's call id
+    protected Writable param;                     // the parameter passed
+    protected Connection connection;              // connection to client
+    protected long timestamp;      // the time received when response is null
+                                   // the time served when response is not null
+    protected ByteBuffer response;                // the response for this call
+    protected boolean delayResponse;
+    protected Responder responder;
+    protected boolean delayReturnValue;           // if the return value should be
+                                                  // set at call completion
+    protected long size;                          // size of current call
+    protected boolean isError;
+
+    public Call(int id, Writable param, Connection connection,
+        Responder responder, long size) {
+      this.id = id;
+      this.param = param;
+      this.connection = connection;
+      this.timestamp = System.currentTimeMillis();
+      this.response = null;
+      this.delayResponse = false;
+      this.responder = responder;
+      this.isError = false;
+      this.size = size;
+    }
+
+    @Override
+    public String toString() {
+      return param.toString() + " from " + connection.toString();
+    }
+
+    protected synchronized void setResponse(Object value, Status status,
+        String errorClass, String error) {
+      // Avoid overwriting an error value in the response.  This can happen if
+      // endDelayThrowing is called by another thread before the actual call
+      // returning.
+      if (this.isError)
+        return;
+      if (errorClass != null) {
+        this.isError = true;
+      }
+      Writable result = null;
+      if (value instanceof Writable) {
+        result = (Writable) value;
+      } else {
+        /* We might have a null value and errors. Avoid creating a
+         * HbaseObjectWritable, because the constructor fails on null. */
+        if (value != null) {
+          result = new HbaseObjectWritable(value);
+        }
+      }
+
+      int size = BUFFER_INITIAL_SIZE;
+      if (result instanceof WritableWithSize) {
+        // get the size hint.
+        WritableWithSize ohint = (WritableWithSize) result;
+        long hint = ohint.getWritableSize() + Bytes.SIZEOF_BYTE +
+          (2 * Bytes.SIZEOF_INT);
+        if (hint > Integer.MAX_VALUE) {
+          // oops, new problem.
+          IOException ioe =
+            new IOException("Result buffer size too large: " + hint);
+          errorClass = ioe.getClass().getName();
+          error = StringUtils.stringifyException(ioe);
+        } else {
+          size = (int)hint;
+        }
+      }
+
+      ByteBufferOutputStream buf = new ByteBufferOutputStream(size);
+      DataOutputStream out = new DataOutputStream(buf);
+      try {
+        // Call id.
+        out.writeInt(this.id);
+        // Write flag.
+        byte flag = (error != null)?
+          ResponseFlag.getErrorAndLengthSet(): ResponseFlag.getLengthSetOnly();
+        out.writeByte(flag);
+        // Place holder for length set later below after we
+        // fill the buffer with data.
+        out.writeInt(0xdeadbeef);
+        out.writeInt(status.state);
+      } catch (IOException e) {
+        errorClass = e.getClass().getName();
+        error = StringUtils.stringifyException(e);
+      }
+
+      try {
+        if (error == null) {
+          result.write(out);
+        } else {
+          WritableUtils.writeString(out, errorClass);
+          WritableUtils.writeString(out, error);
+        }
+      } catch (IOException e) {
+        LOG.warn("Error sending response to call: ", e);
+      }
+
+      // Set the length into the ByteBuffer after call id and after
+      // byte flag.
+      ByteBuffer bb = buf.getByteBuffer();
+      int bufSiz = bb.remaining();
+      // Move to the size location in our ByteBuffer past call.id
+      // and past the byte flag.
+      bb.position(Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE); 
+      bb.putInt(bufSiz);
+      bb.position(0);
+      this.response = bb;
+    }
+
+    @Override
+    public synchronized void endDelay(Object result) throws IOException {
+      assert this.delayResponse;
+      assert this.delayReturnValue || result == null;
+      this.delayResponse = false;
+      delayedCalls.decrementAndGet();
+      if (this.delayReturnValue)
+        this.setResponse(result, Status.SUCCESS, null, null);
+      this.responder.doRespond(this);
+    }
+
+    @Override
+    public synchronized void endDelay() throws IOException {
+      this.endDelay(null);
+    }
+
+    @Override
+    public synchronized void startDelay(boolean delayReturnValue) {
+      assert !this.delayResponse;
+      this.delayResponse = true;
+      this.delayReturnValue = delayReturnValue;
+      int numDelayed = delayedCalls.incrementAndGet();
+      if (numDelayed > warnDelayedCalls) {
+        LOG.warn("Too many delayed calls: limit " + warnDelayedCalls +
+            " current " + numDelayed);
+      }
+    }
+
+    @Override
+    public synchronized void endDelayThrowing(Throwable t) throws IOException {
+      this.setResponse(null, Status.ERROR, t.getClass().toString(),
+          StringUtils.stringifyException(t));
+      this.delayResponse = false;
+      this.sendResponseIfReady();
+    }
+
+    @Override
+    public synchronized boolean isDelayed() {
+      return this.delayResponse;
+    }
+
+    @Override
+    public synchronized boolean isReturnValueDelayed() {
+      return this.delayReturnValue;
+    }
+    
+    @Override
+    public void throwExceptionIfCallerDisconnected() throws CallerDisconnectedException {
+      if (!connection.channel.isOpen()) {
+        long afterTime = System.currentTimeMillis() - timestamp;
+        throw new CallerDisconnectedException(
+            "Aborting call " + this + " after " + afterTime + " ms, since " +
+            "caller disconnected");
+      }
+    }
+
+    public long getSize() {
+      return this.size;
+    }
+
+    /**
+     * If we have a response, and delay is not set, then respond
+     * immediately.  Otherwise, do not respond to client.  This is
+     * called the by the RPC code in the context of the Handler thread.
+     */
+    public synchronized void sendResponseIfReady() throws IOException {
+      if (!this.delayResponse) {
+        this.responder.doRespond(this);
+      }
+    }
+  }
+
+  /** Listens on the socket. Creates jobs for the handler threads*/
+  private class Listener extends Thread {
+
+    private ServerSocketChannel acceptChannel = null; //the accept channel
+    private Selector selector = null; //the selector that we use for the server
+    private Reader[] readers = null;
+    private int currentReader = 0;
+    private InetSocketAddress address; //the address we bind at
+    private Random rand = new Random();
+    private long lastCleanupRunTime = 0; //the last time when a cleanup connec-
+                                         //-tion (for idle connections) ran
+    private long cleanupInterval = 10000; //the minimum interval between
+                                          //two cleanup runs
+    private int backlogLength = conf.getInt("ipc.server.listen.queue.size", 128);
+
+    private ExecutorService readPool;
+
+    public Listener() throws IOException {
+      address = new InetSocketAddress(bindAddress, port);
+      // Create a new server socket and set to non blocking mode
+      acceptChannel = ServerSocketChannel.open();
+      acceptChannel.configureBlocking(false);
+
+      // Bind the server socket to the local host and port
+      bind(acceptChannel.socket(), address, backlogLength);
+      port = acceptChannel.socket().getLocalPort(); //Could be an ephemeral port
+      // create a selector;
+      selector= Selector.open();
+
+      readers = new Reader[readThreads];
+      readPool = Executors.newFixedThreadPool(readThreads,
+        new ThreadFactoryBuilder().setNameFormat(
+          "IPC Reader %d on port " + port).setDaemon(true).build());
+      for (int i = 0; i < readThreads; ++i) {
+        Reader reader = new Reader();
+        readers[i] = reader;
+        readPool.execute(reader);
+      }
+
+      // Register accepts on the server socket with the selector.
+      acceptChannel.register(selector, SelectionKey.OP_ACCEPT);
+      this.setName("IPC Server listener on " + port);
+      this.setDaemon(true);
+    }
+
+
+    private class Reader implements Runnable {
+      private volatile boolean adding = false;
+      private final Selector readSelector;
+
+      Reader() throws IOException {
+        this.readSelector = Selector.open();
+      }
+      public void run() {
+        LOG.info("Starting " + getName());
+        try {
+          doRunLoop();
+        } finally {
+          try {
+            readSelector.close();
+          } catch (IOException ioe) {
+            LOG.error("Error closing read selector in " + getName(), ioe);
+          }
+        }
+      }
+
+      private synchronized void doRunLoop() {
+        while (running) {
+          SelectionKey key = null;
+          try {
+            readSelector.select();
+            while (adding) {
+              this.wait(1000);
+            }
+
+            Iterator iter = readSelector.selectedKeys().iterator();
+            while (iter.hasNext()) {
+              key = iter.next();
+              iter.remove();
+              if (key.isValid()) {
+                if (key.isReadable()) {
+                  doRead(key);
+                }
+              }
+              key = null;
+            }
+          } catch (InterruptedException e) {
+            if (running) {                      // unexpected -- log it
+              LOG.info(getName() + " unexpectedly interrupted: " +
+                  StringUtils.stringifyException(e));
+            }
+          } catch (IOException ex) {
+            LOG.error("Error in Reader", ex);
+          }
+        }
+      }
+
+      /**
+       * This gets reader into the state that waits for the new channel
+       * to be registered with readSelector. If it was waiting in select()
+       * the thread will be woken up, otherwise whenever select() is called
+       * it will return even if there is nothing to read and wait
+       * in while(adding) for finishAdd call
+       */
+      public void startAdd() {
+        adding = true;
+        readSelector.wakeup();
+      }
+
+      public synchronized SelectionKey registerChannel(SocketChannel channel)
+        throws IOException {
+        return channel.register(readSelector, SelectionKey.OP_READ);
+      }
+
+      public synchronized void finishAdd() {
+        adding = false;
+        this.notify();
+      }
+    }
+
+    /** cleanup connections from connectionList. Choose a random range
+     * to scan and also have a limit on the number of the connections
+     * that will be cleanedup per run. The criteria for cleanup is the time
+     * for which the connection was idle. If 'force' is true then all
+     * connections will be looked at for the cleanup.
+     * @param force all connections will be looked at for cleanup
+     */
+    private void cleanupConnections(boolean force) {
+      if (force || numConnections > thresholdIdleConnections) {
+        long currentTime = System.currentTimeMillis();
+        if (!force && (currentTime - lastCleanupRunTime) < cleanupInterval) {
+          return;
+        }
+        int start = 0;
+        int end = numConnections - 1;
+        if (!force) {
+          start = rand.nextInt() % numConnections;
+          end = rand.nextInt() % numConnections;
+          int temp;
+          if (end < start) {
+            temp = start;
+            start = end;
+            end = temp;
+          }
+        }
+        int i = start;
+        int numNuked = 0;
+        while (i <= end) {
+          Connection c;
+          synchronized (connectionList) {
+            try {
+              c = connectionList.get(i);
+            } catch (Exception e) {return;}
+          }
+          if (c.timedOut(currentTime)) {
+            if (LOG.isDebugEnabled())
+              LOG.debug(getName() + ": disconnecting client " + c.getHostAddress());
+            closeConnection(c);
+            numNuked++;
+            end--;
+            //noinspection UnusedAssignment
+            c = null;
+            if (!force && numNuked == maxConnectionsToNuke) break;
+          }
+          else i++;
+        }
+        lastCleanupRunTime = System.currentTimeMillis();
+      }
+    }
+
+    @Override
+    public void run() {
+      LOG.info(getName() + ": starting");
+      SERVER.set(HBaseServer.this);
+
+      while (running) {
+        SelectionKey key = null;
+        try {
+          selector.select(); // FindBugs IS2_INCONSISTENT_SYNC
+          Iterator iter = selector.selectedKeys().iterator();
+          while (iter.hasNext()) {
+            key = iter.next();
+            iter.remove();
+            try {
+              if (key.isValid()) {
+                if (key.isAcceptable())
+                  doAccept(key);
+              }
+            } catch (IOException ignored) {
+            }
+            key = null;
+          }
+        } catch (OutOfMemoryError e) {
+          if (errorHandler != null) {
+            if (errorHandler.checkOOME(e)) {
+              LOG.info(getName() + ": exiting on OOME");
+              closeCurrentConnection(key, e);
+              cleanupConnections(true);
+              return;
+            }
+          } else {
+            // we can run out of memory if we have too many threads
+            // log the event and sleep for a minute and give
+            // some thread(s) a chance to finish
+            LOG.warn("Out of Memory in server select", e);
+            closeCurrentConnection(key, e);
+            cleanupConnections(true);
+            try { Thread.sleep(60000); } catch (Exception ignored) {}
+      }
+        } catch (Exception e) {
+          closeCurrentConnection(key, e);
+        }
+        cleanupConnections(false);
+      }
+      LOG.info("Stopping " + this.getName());
+
+      synchronized (this) {
+        try {
+          acceptChannel.close();
+          selector.close();
+        } catch (IOException ignored) { }
+
+        selector= null;
+        acceptChannel= null;
+
+        // clean up all connections
+        while (!connectionList.isEmpty()) {
+          closeConnection(connectionList.remove(0));
+        }
+      }
+    }
+
+    private void closeCurrentConnection(SelectionKey key, Throwable e) {
+      if (key != null) {
+        Connection c = (Connection)key.attachment();
+        if (c != null) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug(getName() + ": disconnecting client " + c.getHostAddress() +
+                (e != null ? " on error " + e.getMessage() : ""));
+          }
+          closeConnection(c);
+          key.attach(null);
+        }
+      }
+    }
+
+    InetSocketAddress getAddress() {
+      return (InetSocketAddress)acceptChannel.socket().getLocalSocketAddress();
+    }
+
+    void doAccept(SelectionKey key) throws IOException, OutOfMemoryError {
+      Connection c;
+      ServerSocketChannel server = (ServerSocketChannel) key.channel();
+
+      SocketChannel channel;
+      while ((channel = server.accept()) != null) {
+        channel.configureBlocking(false);
+        channel.socket().setTcpNoDelay(tcpNoDelay);
+        channel.socket().setKeepAlive(tcpKeepAlive);
+
+        Reader reader = getReader();
+        try {
+          reader.startAdd();
+          SelectionKey readKey = reader.registerChannel(channel);
+          c = getConnection(channel, System.currentTimeMillis());
+          readKey.attach(c);
+          synchronized (connectionList) {
+            connectionList.add(numConnections, c);
+            numConnections++;
+          }
+          if (LOG.isDebugEnabled())
+            LOG.debug("Server connection from " + c.toString() +
+                "; # active connections: " + numConnections +
+                "; # queued calls: " + callQueue.size());
+        } finally {
+          reader.finishAdd();
+        }
+      }
+      rpcMetrics.numOpenConnections.set(numConnections);
+    }
+
+    void doRead(SelectionKey key) throws InterruptedException {
+      int count = 0;
+      Connection c = (Connection)key.attachment();
+      if (c == null) {
+        return;
+      }
+      c.setLastContact(System.currentTimeMillis());
+
+      try {
+        count = c.readAndProcess();
+      } catch (InterruptedException ieo) {
+        throw ieo;
+      } catch (Exception e) {
+        LOG.warn(getName() + ": readAndProcess threw exception " + e + ". Count of bytes read: " + count, e);
+        count = -1; //so that the (count < 0) block is executed
+      }
+      if (count < 0) {
+        if (LOG.isDebugEnabled())
+          LOG.debug(getName() + ": disconnecting client " +
+                    c.getHostAddress() + ". Number of active connections: "+
+                    numConnections);
+        closeConnection(c);
+        // c = null;
+      }
+      else {
+        c.setLastContact(System.currentTimeMillis());
+      }
+    }
+
+    synchronized void doStop() {
+      if (selector != null) {
+        selector.wakeup();
+        Thread.yield();
+      }
+      if (acceptChannel != null) {
+        try {
+          acceptChannel.socket().close();
+        } catch (IOException e) {
+          LOG.info(getName() + ":Exception in closing listener socket. " + e);
+        }
+      }
+      readPool.shutdownNow();
+    }
+
+    // The method that will return the next reader to work with
+    // Simplistic implementation of round robin for now
+    Reader getReader() {
+      currentReader = (currentReader + 1) % readers.length;
+      return readers[currentReader];
+    }
+  }
+
+  // Sends responses of RPC back to clients.
+  protected class Responder extends Thread {
+    private final Selector writeSelector;
+    private int pending;         // connections waiting to register
+
+    Responder() throws IOException {
+      this.setName("IPC Server Responder");
+      this.setDaemon(true);
+      writeSelector = Selector.open(); // create a selector
+      pending = 0;
+    }
+
+    @Override
+    public void run() {
+      LOG.info(getName() + ": starting");
+      SERVER.set(HBaseServer.this);
+      try {
+        doRunLoop();
+      } finally {
+        LOG.info("Stopping " + this.getName());
+        try {
+          writeSelector.close();
+        } catch (IOException ioe) {
+          LOG.error("Couldn't close write selector in " + this.getName(), ioe);
+        }
+      }
+    }
+
+    private void doRunLoop() {
+      long lastPurgeTime = 0;   // last check for old calls.
+
+      while (running) {
+        try {
+          waitPending();     // If a channel is being registered, wait.
+          writeSelector.select(purgeTimeout);
+          Iterator iter = writeSelector.selectedKeys().iterator();
+          while (iter.hasNext()) {
+            SelectionKey key = iter.next();
+            iter.remove();
+            try {
+              if (key.isValid() && key.isWritable()) {
+                  doAsyncWrite(key);
+              }
+            } catch (IOException e) {
+              LOG.info(getName() + ": doAsyncWrite threw exception " + e);
+            }
+          }
+          long now = System.currentTimeMillis();
+          if (now < lastPurgeTime + purgeTimeout) {
+            continue;
+          }
+          lastPurgeTime = now;
+          //
+          // If there were some calls that have not been sent out for a
+          // long time, discard them.
+          //
+          LOG.debug("Checking for old call responses.");
+          ArrayList calls;
+
+          // get the list of channels from list of keys.
+          synchronized (writeSelector.keys()) {
+            calls = new ArrayList(writeSelector.keys().size());
+            iter = writeSelector.keys().iterator();
+            while (iter.hasNext()) {
+              SelectionKey key = iter.next();
+              Call call = (Call)key.attachment();
+              if (call != null && key.channel() == call.connection.channel) {
+                calls.add(call);
+              }
+            }
+          }
+
+          for(Call call : calls) {
+            try {
+              doPurge(call, now);
+            } catch (IOException e) {
+              LOG.warn("Error in purging old calls " + e);
+            }
+          }
+        } catch (OutOfMemoryError e) {
+          if (errorHandler != null) {
+            if (errorHandler.checkOOME(e)) {
+              LOG.info(getName() + ": exiting on OOME");
+              return;
+            }
+          } else {
+            //
+            // we can run out of memory if we have too many threads
+            // log the event and sleep for a minute and give
+            // some thread(s) a chance to finish
+            //
+            LOG.warn("Out of Memory in server select", e);
+            try { Thread.sleep(60000); } catch (Exception ignored) {}
+          }
+        } catch (Exception e) {
+          LOG.warn("Exception in Responder " +
+                   StringUtils.stringifyException(e));
+        }
+      }
+      LOG.info("Stopping " + this.getName());
+    }
+
+    private void doAsyncWrite(SelectionKey key) throws IOException {
+      Call call = (Call)key.attachment();
+      if (call == null) {
+        return;
+      }
+      if (key.channel() != call.connection.channel) {
+        throw new IOException("doAsyncWrite: bad channel");
+      }
+
+      synchronized(call.connection.responseQueue) {
+        if (processResponse(call.connection.responseQueue, false)) {
+          try {
+            key.interestOps(0);
+          } catch (CancelledKeyException e) {
+            /* The Listener/reader might have closed the socket.
+             * We don't explicitly cancel the key, so not sure if this will
+             * ever fire.
+             * This warning could be removed.
+             */
+            LOG.warn("Exception while changing ops : " + e);
+          }
+        }
+      }
+    }
+
+    //
+    // Remove calls that have been pending in the responseQueue
+    // for a long time.
+    //
+    private void doPurge(Call call, long now) throws IOException {
+      synchronized (call.connection.responseQueue) {
+        Iterator iter = call.connection.responseQueue.listIterator(0);
+        while (iter.hasNext()) {
+          Call nextCall = iter.next();
+          if (now > nextCall.timestamp + purgeTimeout) {
+            closeConnection(nextCall.connection);
+            break;
+          }
+        }
+      }
+    }
+
+    // Processes one response. Returns true if there are no more pending
+    // data for this channel.
+    //
+    private boolean processResponse(final LinkedList responseQueue,
+                                    boolean inHandler) throws IOException {
+      boolean error = true;
+      boolean done = false;       // there is more data for this channel.
+      int numElements;
+      Call call = null;
+      try {
+        //noinspection SynchronizationOnLocalVariableOrMethodParameter
+        synchronized (responseQueue) {
+          //
+          // If there are no items for this channel, then we are done
+          //
+          numElements = responseQueue.size();
+          if (numElements == 0) {
+            error = false;
+            return true;              // no more data for this channel.
+          }
+          //
+          // Extract the first call
+          //
+          call = responseQueue.peek();
+          SocketChannel channel = call.connection.channel;
+          if (LOG.isDebugEnabled()) {
+            LOG.debug(getName() + ": responding to #" + call.id + " from " +
+                      call.connection);
+          }
+          //
+          // Send as much data as we can in the non-blocking fashion
+          //
+          int numBytes = channelWrite(channel, call.response);
+          if (numBytes < 0) {
+            // Error flag is set, so returning here closes connection and
+            // clears responseQueue.                   
+            return true;
+          }
+          if (!call.response.hasRemaining()) {
+            responseQueue.poll();
+            responseQueuesSizeThrottler.decrease(call.response.limit());    
+            call.connection.decRpcCount();
+            //noinspection RedundantIfStatement
+            if (numElements == 1) {    // last call fully processes.
+              done = true;             // no more data for this channel.
+            } else {
+              done = false;            // more calls pending to be sent.
+            }
+            if (LOG.isDebugEnabled()) {
+              LOG.debug(getName() + ": responding to #" + call.id + " from " +
+                        call.connection + " Wrote " + numBytes + " bytes.");
+            }
+          } else {
+            if (inHandler) {
+              // set the serve time when the response has to be sent later
+              call.timestamp = System.currentTimeMillis();
+              if (enqueueInSelector(call))
+                done = true;
+            }
+            if (LOG.isDebugEnabled()) {
+              LOG.debug(getName() + ": responding to #" + call.id + " from " +
+                        call.connection + " Wrote partial " + numBytes +
+                        " bytes.");
+            }
+          }
+          error = false;              // everything went off well
+        }
+      } finally {
+        if (error && call != null) {
+          LOG.warn(getName()+", call " + call + ": output error");
+          done = true;               // error. no more data for this channel.
+          closeConnection(call.connection);
+        }
+      }
+      return done;
+    }
+
+    //
+    // Enqueue for background thread to send responses out later.
+    //
+    private boolean enqueueInSelector(Call call) throws IOException {
+      boolean done = false;
+      incPending();
+      try {
+        // Wake up the thread blocked on select, only then can the call
+        // to channel.register() complete.
+        SocketChannel channel = call.connection.channel;
+        writeSelector.wakeup();
+        channel.register(writeSelector, SelectionKey.OP_WRITE, call);
+      } catch (ClosedChannelException e) {
+        //It's OK.  Channel might be closed else where.
+        done = true;
+      } finally {
+        decPending();
+      }
+      return done;
+    }
+
+    //
+    // Enqueue a response from the application.
+    //
+    void doRespond(Call call) throws IOException {
+      // set the serve time when the response has to be sent later
+      call.timestamp = System.currentTimeMillis();
+
+      boolean doRegister = false;
+      boolean closed;
+      try {
+        responseQueuesSizeThrottler.increase(call.response.remaining());
+      } catch (InterruptedException ie) {
+        throw new InterruptedIOException(ie.getMessage());
+      }
+      synchronized (call.connection.responseQueue) {
+        closed = call.connection.closed;
+        if (!closed) {
+          call.connection.responseQueue.addLast(call);
+
+          if (call.connection.responseQueue.size() == 1) {
+            doRegister = !processResponse(call.connection.responseQueue, false);
+          }
+        }
+      }
+      if (doRegister) {
+        enqueueInSelector(call);
+      }
+      if (closed) {
+        // Connection was closed when we tried to submit response, but we
+        // increased responseQueues size already. It shoud be
+        // decreased here.
+        responseQueuesSizeThrottler.decrease(call.response.remaining());
+      }      
+    }
+
+    private synchronized void incPending() {   // call waiting to be enqueued.
+      pending++;
+    }
+
+    private synchronized void decPending() { // call done enqueueing.
+      pending--;
+      notify();
+    }
+
+    private synchronized void waitPending() throws InterruptedException {
+      while (pending > 0) {
+        wait();
+      }
+    }
+  }
+
+  /** Reads calls from a connection and queues them for handling. */
+  protected class Connection {
+    private boolean versionRead = false; //if initial signature and
+                                         //version are read
+    private boolean headerRead = false;  //if the connection header that
+                                         //follows version is read.
+
+    protected volatile boolean closed = false;    // indicates if connection was closed
+    protected SocketChannel channel;
+    private ByteBuffer data;
+    private ByteBuffer dataLengthBuffer;
+    protected final LinkedList responseQueue;
+    private volatile int rpcCount = 0; // number of outstanding rpcs
+    private long lastContact;
+    private int dataLength;
+    protected Socket socket;
+    // Cache the remote host & port info so that even if the socket is
+    // disconnected, we can say where it used to connect to.
+    protected String hostAddress;
+    protected int remotePort;
+    ConnectionHeader header = new ConnectionHeader();
+    Class protocol;
+    protected User ticket = null;
+
+    public Connection(SocketChannel channel, long lastContact) {
+      this.channel = channel;
+      this.lastContact = lastContact;
+      this.data = null;
+      this.dataLengthBuffer = ByteBuffer.allocate(4);
+      this.socket = channel.socket();
+      InetAddress addr = socket.getInetAddress();
+      if (addr == null) {
+        this.hostAddress = "*Unknown*";
+      } else {
+        this.hostAddress = addr.getHostAddress();
+      }
+      this.remotePort = socket.getPort();
+      this.responseQueue = new LinkedList();
+      if (socketSendBufferSize != 0) {
+        try {
+          socket.setSendBufferSize(socketSendBufferSize);
+        } catch (IOException e) {
+          LOG.warn("Connection: unable to set socket send buffer size to " +
+                   socketSendBufferSize);
+        }
+      }
+    }
+
+    @Override
+    public String toString() {
+      return getHostAddress() + ":" + remotePort;
+    }
+
+    public String getHostAddress() {
+      return hostAddress;
+    }
+
+    public int getRemotePort() {
+      return remotePort;
+    }
+
+    public void setLastContact(long lastContact) {
+      this.lastContact = lastContact;
+    }
+
+    public long getLastContact() {
+      return lastContact;
+    }
+
+    /* Return true if the connection has no outstanding rpc */
+    private boolean isIdle() {
+      return rpcCount == 0;
+    }
+
+    /* Decrement the outstanding RPC count */
+    protected void decRpcCount() {
+      rpcCount--;
+    }
+
+    /* Increment the outstanding RPC count */
+    protected void incRpcCount() {
+      rpcCount++;
+    }
+
+    protected boolean timedOut(long currentTime) {
+      return isIdle() && currentTime - lastContact > maxIdleTime;
+    }
+
+    public int readAndProcess() throws IOException, InterruptedException {
+      while (true) {
+        /* Read at most one RPC. If the header is not read completely yet
+         * then iterate until we read first RPC or until there is no data left.
+         */
+        int count;
+        if (dataLengthBuffer.remaining() > 0) {
+          count = channelRead(channel, dataLengthBuffer);
+          if (count < 0 || dataLengthBuffer.remaining() > 0)
+            return count;
+        }
+
+        if (!versionRead) {
+          //Every connection is expected to send the header.
+          ByteBuffer versionBuffer = ByteBuffer.allocate(1);
+          count = channelRead(channel, versionBuffer);
+          if (count <= 0) {
+            return count;
+          }
+          int version = versionBuffer.get(0);
+
+          dataLengthBuffer.flip();
+          if (!HEADER.equals(dataLengthBuffer) || version != CURRENT_VERSION) {
+            //Warning is ok since this is not supposed to happen.
+            LOG.warn("Incorrect header or version mismatch from " +
+                     hostAddress + ":" + remotePort +
+                     " got version " + version +
+                     " expected version " + CURRENT_VERSION);
+            setupBadVersionResponse(version);
+            return -1;
+          }
+          dataLengthBuffer.clear();
+          versionRead = true;
+          continue;
+        }
+
+        if (data == null) {
+          dataLengthBuffer.flip();
+          dataLength = dataLengthBuffer.getInt();
+
+          if (dataLength == HBaseClient.PING_CALL_ID) {
+            dataLengthBuffer.clear();
+            return 0;  //ping message
+          }
+          data = ByteBuffer.allocate(dataLength);
+          incRpcCount();  // Increment the rpc count
+        }
+
+        count = channelRead(channel, data);
+
+        if (data.remaining() == 0) {
+          dataLengthBuffer.clear();
+          data.flip();
+          if (headerRead) {
+            processData(data.array());
+            data = null;
+            return count;
+          }
+          processHeader();
+          headerRead = true;
+          data = null;
+          continue;
+        }
+        return count;
+      }
+    }
+
+    /**
+     * Try to set up the response to indicate that the client version
+     * is incompatible with the server. This can contain special-case
+     * code to speak enough of past IPC protocols to pass back
+     * an exception to the caller.
+     * @param clientVersion the version the caller is using
+     * @throws IOException
+     */
+    private void setupBadVersionResponse(int clientVersion) throws IOException {
+      String errMsg = "Server IPC version " + CURRENT_VERSION +
+      " cannot communicate with client version " + clientVersion;
+      ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+      if (clientVersion >= 3) {
+        // We used to return an id of -1 which caused server to close the
+        // connection without telling the client what the problem was.  Now
+        // we return 0 which will keep the socket up -- bad clients, unless
+        // they switch to suit the running server -- will fail later doing
+        // getProtocolVersion.
+        Call fakeCall =  new Call(0, null, this, responder, 0);
+        // Versions 3 and greater can interpret this exception
+        // response in the same manner
+        setupResponse(buffer, fakeCall, Status.FATAL,
+            null, VersionMismatch.class.getName(), errMsg);
+
+        responder.doRespond(fakeCall);
+      }
+    }
+
+    /// Reads the connection header following version
+    private void processHeader() throws IOException {
+      DataInputStream in =
+        new DataInputStream(new ByteArrayInputStream(data.array()));
+      header.readFields(in);
+      try {
+        String protocolClassName = header.getProtocol();
+        if (protocolClassName == null) {
+          protocolClassName = "org.apache.hadoop.hbase.ipc.HRegionInterface";
+        }
+        protocol = getProtocolClass(protocolClassName, conf);
+      } catch (ClassNotFoundException cnfe) {
+        throw new IOException("Unknown protocol: " + header.getProtocol());
+      }
+
+      ticket = header.getUser();
+    }
+
+    protected void processData(byte[] buf) throws  IOException, InterruptedException {
+      DataInputStream dis =
+        new DataInputStream(new ByteArrayInputStream(buf));
+      int id = dis.readInt();                    // try to read an id
+      long callSize = buf.length;
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(" got call #" + id + ", " + callSize + " bytes");
+      }
+
+      // Enforcing the call queue size, this triggers a retry in the client
+      if ((callSize + callQueueSize.get()) > maxQueueSize) {
+        final Call callTooBig =
+          new Call(id, null, this, responder, callSize);
+        ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
+        setupResponse(responseBuffer, callTooBig, Status.FATAL, null,
+            IOException.class.getName(),
+            "Call queue is full, is ipc.server.max.callqueue.size too small?");
+        responder.doRespond(callTooBig);
+        return;
+      }
+
+      Writable param;
+      try {
+        param = ReflectionUtils.newInstance(paramClass, conf);//read param
+        param.readFields(dis);
+      } catch (Throwable t) {
+        LOG.warn("Unable to read call parameters for client " +
+                 getHostAddress(), t);
+        final Call readParamsFailedCall =
+          new Call(id, null, this, responder, callSize);
+        ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
+
+        setupResponse(responseBuffer, readParamsFailedCall, Status.FATAL, null,
+            t.getClass().getName(),
+            "IPC server unable to read call parameters: " + t.getMessage());
+        responder.doRespond(readParamsFailedCall);
+        return;
+      }
+      Call call = new Call(id, param, this, responder, callSize);
+      callQueueSize.add(callSize);
+
+      if (priorityCallQueue != null && getQosLevel(param) > highPriorityLevel) {
+        priorityCallQueue.put(call);
+        updateCallQueueLenMetrics(priorityCallQueue);
+      } else if (replicationQueue != null && getQosLevel(param) == HConstants.REPLICATION_QOS) {
+        replicationQueue.put(call);
+        updateCallQueueLenMetrics(replicationQueue);
+      } else {
+        callQueue.put(call); // queue the call; maybe blocked here
+        updateCallQueueLenMetrics(callQueue);
+      }
+    }
+
+    protected synchronized void close() {
+      closed = true;
+      data = null;
+      dataLengthBuffer = null;
+      if (!channel.isOpen())
+        return;
+      try {socket.shutdownOutput();} catch(Exception ignored) {} // FindBugs DE_MIGHT_IGNORE
+      if (channel.isOpen()) {
+        try {channel.close();} catch(Exception ignored) {}
+      }
+      try {socket.close();} catch(Exception ignored) {}
+    }
+  }
+
+  /**
+   * Reports length of the call queue to HBaseRpcMetrics.
+   * @param queue Which queue to report
+   */
+  protected void updateCallQueueLenMetrics(BlockingQueue queue) {
+    if (queue == callQueue) {
+      rpcMetrics.callQueueLen.set(callQueue.size());
+    } else if (queue == priorityCallQueue) {
+      rpcMetrics.priorityCallQueueLen.set(priorityCallQueue.size());
+    } else if (queue == replicationQueue) {
+      rpcMetrics.replicationCallQueueLen.set(replicationQueue.size());
+    } else {
+      LOG.warn("Unknown call queue");
+    }
+  }
+
+  /** Handles queued calls . */
+  private class Handler extends Thread {
+    private final BlockingQueue myCallQueue;
+    private MonitoredRPCHandler status;
+
+    public Handler(final BlockingQueue cq, int instanceNumber) {
+      this.myCallQueue = cq;
+      this.setDaemon(true);
+
+      String threadName = "IPC Server handler " + instanceNumber + " on " + port;
+      if (cq == priorityCallQueue) {
+        // this is just an amazing hack, but it works.
+        threadName = "PRI " + threadName;
+      } else if (cq == replicationQueue) {
+        threadName = "REPL " + threadName;
+      }
+      this.setName(threadName);
+      this.status = TaskMonitor.get().createRPCStatus(threadName);
+    }
+
+    @Override
+    public void run() {
+      LOG.info(getName() + ": starting");
+      status.setStatus("starting");
+      SERVER.set(HBaseServer.this);
+      while (running) {
+        try {
+          status.pause("Waiting for a call");
+          Call call = myCallQueue.take(); // pop the queue; maybe blocked here
+          updateCallQueueLenMetrics(myCallQueue);
+          status.setStatus("Setting up call");
+          status.setConnection(call.connection.getHostAddress(), 
+              call.connection.getRemotePort());
+
+          if (LOG.isDebugEnabled())
+            LOG.debug(getName() + ": has #" + call.id + " from " +
+                      call.connection);
+
+          String errorClass = null;
+          String error = null;
+          Writable value = null;
+
+          CurCall.set(call);
+          try {
+            if (!started)
+              throw new ServerNotRunningYetException("Server is not running yet");
+
+            if (LOG.isDebugEnabled()) {
+              User remoteUser = call.connection.ticket;
+              LOG.debug(getName() + ": call #" + call.id + " executing as "
+                  + (remoteUser == null ? "NULL principal" : remoteUser.getName()));
+            }
+
+            RequestContext.set(call.connection.ticket, getRemoteIp(),
+                call.connection.protocol);
+            // make the call
+            value = call(call.connection.protocol, call.param, call.timestamp, 
+                status);
+          } catch (Throwable e) {
+            LOG.debug(getName()+", call "+call+": error: " + e, e);
+            errorClass = e.getClass().getName();
+            error = StringUtils.stringifyException(e);
+          } finally {
+            // Must always clear the request context to avoid leaking
+            // credentials between requests.
+            RequestContext.clear();
+          }
+          CurCall.set(null);
+          callQueueSize.add(call.getSize() * -1);
+          // Set the response for undelayed calls and delayed calls with
+          // undelayed responses.
+          if (!call.isDelayed() || !call.isReturnValueDelayed()) {
+            call.setResponse(value,
+              errorClass == null? Status.SUCCESS: Status.ERROR,
+                errorClass, error);
+          }
+          call.sendResponseIfReady();
+          status.markComplete("Sent response");
+        } catch (InterruptedException e) {
+          if (running) {                          // unexpected -- log it
+            LOG.info(getName() + " caught: " +
+                     StringUtils.stringifyException(e));
+          }
+        } catch (OutOfMemoryError e) {
+          if (errorHandler != null) {
+            if (errorHandler.checkOOME(e)) {
+              LOG.info(getName() + ": exiting on OOME");
+              return;
+            }
+          } else {
+            // rethrow if no handler
+            throw e;
+          }
+       } catch (ClosedChannelException cce) {
+          LOG.warn(getName() + " caught a ClosedChannelException, " +
+            "this means that the server was processing a " +
+            "request but the client went away. The error message was: " +
+            cce.getMessage());
+        } catch (Exception e) {
+          LOG.warn(getName() + " caught: " +
+                   StringUtils.stringifyException(e));
+        }
+      }
+      LOG.info(getName() + ": exiting");
+    }
+
+  }
+
+
+  private Function qosFunction = null;
+
+  /**
+   * Gets the QOS level for this call.  If it is higher than the highPriorityLevel and there
+   * are priorityHandlers available it will be processed in it's own thread set.
+   *
+   * @param newFunc
+   */
+  @Override
+  public void setQosFunction(Function newFunc) {
+    qosFunction = newFunc;
+  }
+
+  protected int getQosLevel(Writable param) {
+    if (qosFunction == null) {
+      return 0;
+    }
+
+    Integer res = qosFunction.apply(param);
+    if (res == null) {
+      return 0;
+    }
+    return res;
+  }
+
+  /* Constructs a server listening on the named port and address.  Parameters passed must
+   * be of the named class.  The handlerCount determines
+   * the number of handler threads that will be used to process calls.
+   *
+   */
+  protected HBaseServer(String bindAddress, int port,
+                        Class paramClass, int handlerCount,
+                        int priorityHandlerCount, Configuration conf, String serverName,
+                        int highPriorityLevel)
+    throws IOException {
+    this.bindAddress = bindAddress;
+    this.conf = conf;
+    this.port = port;
+    this.paramClass = paramClass;
+    this.handlerCount = handlerCount;
+    this.priorityHandlerCount = priorityHandlerCount;
+    this.socketSendBufferSize = 0;
+
+    // temporary backward compatibility
+    String oldMaxQueueSize = this.conf.get("ipc.server.max.queue.size");
+    if (oldMaxQueueSize == null) {
+      this.maxQueueLength =
+        this.conf.getInt("ipc.server.max.callqueue.length",
+          handlerCount * DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER);
+    } else {
+      LOG.warn("ipc.server.max.queue.size was renamed " +
+               "ipc.server.max.callqueue.length, " +
+               "please update your configuration");
+      this.maxQueueLength = Integer.getInteger(oldMaxQueueSize);
+    }
+
+    this.maxQueueSize =
+      this.conf.getInt("ipc.server.max.callqueue.size",
+        DEFAULT_MAX_CALLQUEUE_SIZE);
+     this.readThreads = conf.getInt(
+        "ipc.server.read.threadpool.size",
+        10);
+    this.callQueue  = new LinkedBlockingQueue(maxQueueLength);
+    if (priorityHandlerCount > 0) {
+      this.priorityCallQueue = new LinkedBlockingQueue(maxQueueLength); // TODO hack on size
+    } else {
+      this.priorityCallQueue = null;
+    }
+    this.highPriorityLevel = highPriorityLevel;
+    this.maxIdleTime = 2*conf.getInt("ipc.client.connection.maxidletime", 1000);
+    this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10);
+    this.thresholdIdleConnections = conf.getInt("ipc.client.idlethreshold", 4000);
+    this.purgeTimeout = conf.getLong("ipc.client.call.purge.timeout",
+                                     2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
+    this.numOfReplicationHandlers = 
+      conf.getInt("hbase.regionserver.replication.handler.count", 3);
+    if (numOfReplicationHandlers > 0) {
+      this.replicationQueue = new LinkedBlockingQueue(maxQueueSize);
+    }
+    // Start the listener here and let it bind to the port
+    listener = new Listener();
+    this.port = listener.getAddress().getPort();
+    this.rpcMetrics = new HBaseRpcMetrics(
+        serverName, Integer.toString(this.port));
+    this.tcpNoDelay = conf.getBoolean("ipc.server.tcpnodelay", false);
+    this.tcpKeepAlive = conf.getBoolean("ipc.server.tcpkeepalive", true);
+
+    this.warnDelayedCalls = conf.getInt(WARN_DELAYED_CALLS,
+                                        DEFAULT_WARN_DELAYED_CALLS);
+    this.delayedCalls = new AtomicInteger(0);
+
+
+    this.responseQueuesSizeThrottler = new SizeBasedThrottler(
+        conf.getLong(RESPONSE_QUEUES_MAX_SIZE, DEFAULT_RESPONSE_QUEUES_MAX_SIZE));
+
+    // Create the responder here
+    responder = new Responder();
+  }
+
+  /**
+   * Subclasses of HBaseServer can override this to provide their own
+   * Connection implementations.
+   */
+  protected Connection getConnection(SocketChannel channel, long time) {
+    return new Connection(channel, time);
+  }
+
+  /**
+   * Setup response for the IPC Call.
+   *
+   * @param response buffer to serialize the response into
+   * @param call {@link Call} to which we are setting up the response
+   * @param status {@link Status} of the IPC call
+   * @param rv return value for the IPC Call, if the call was successful
+   * @param errorClass error class, if the the call failed
+   * @param error error message, if the call failed
+   * @throws IOException
+   */
+  private void setupResponse(ByteArrayOutputStream response,
+                             Call call, Status status,
+                             Writable rv, String errorClass, String error)
+  throws IOException {
+    response.reset();
+    DataOutputStream out = new DataOutputStream(response);
+
+    if (status == Status.SUCCESS) {
+      try {
+        rv.write(out);
+        call.setResponse(rv, status, null, null);
+      } catch (Throwable t) {
+        LOG.warn("Error serializing call response for call " + call, t);
+        // Call back to same function - this is OK since the
+        // buffer is reset at the top, and since status is changed
+        // to ERROR it won't infinite loop.
+        call.setResponse(null, status.ERROR, t.getClass().getName(),
+            StringUtils.stringifyException(t));
+      }
+    } else {
+      call.setResponse(rv, status, errorClass, error);
+    }
+  }
+
+  protected void closeConnection(Connection connection) {
+    synchronized (connectionList) {
+      if (connectionList.remove(connection)) {
+        numConnections--;
+      }
+    }
+    connection.close();
+    long bytes = 0;
+    synchronized (connection.responseQueue) {
+      for (Call c : connection.responseQueue) {
+        bytes += c.response.limit();
+      }
+      connection.responseQueue.clear();
+    }
+    responseQueuesSizeThrottler.decrease(bytes);    
+    rpcMetrics.numOpenConnections.set(numConnections);
+  }
+
+  /** Sets the socket buffer size used for responding to RPCs.
+   * @param size send size
+   */
+  @Override
+  public void setSocketSendBufSize(int size) { this.socketSendBufferSize = size; }
+
+  /** Starts the service.  Must be called before any calls will be handled. */
+  @Override
+  public void start() {
+    startThreads();
+    openServer();
+  }
+
+  /**
+   * Open a previously started server.
+   */
+  @Override
+  public void openServer() {
+    started = true;
+  }
+
+  /**
+   * Starts the service threads but does not allow requests to be responded yet.
+   * Client will get {@link ServerNotRunningYetException} instead.
+   */
+  @Override
+  public synchronized void startThreads() {
+    responder.start();
+    listener.start();
+    handlers = startHandlers(callQueue, handlerCount);
+    priorityHandlers = startHandlers(priorityCallQueue, priorityHandlerCount);
+    replicationHandlers = startHandlers(replicationQueue, numOfReplicationHandlers);
+    }
+
+  private Handler[] startHandlers(BlockingQueue queue, int numOfHandlers) {
+    if (numOfHandlers <= 0) {
+      return null;
+    }
+    Handler[] handlers = new Handler[numOfHandlers];
+    for (int i = 0; i < numOfHandlers; i++) {
+      handlers[i] = new Handler(queue, i);
+      handlers[i].start();
+    }
+    return handlers;
+  }
+
+  /** Stops the service.  No new calls will be handled after this is called. */
+  @Override
+  public synchronized void stop() {
+    LOG.info("Stopping server on " + port);
+    running = false;
+    stopHandlers(handlers);
+    stopHandlers(priorityHandlers);
+    stopHandlers(replicationHandlers);
+    listener.interrupt();
+    listener.doStop();
+    responder.interrupt();
+    notifyAll();
+    if (this.rpcMetrics != null) {
+      this.rpcMetrics.shutdown();
+    }
+  }
+
+  private void stopHandlers(Handler[] handlers) {
+    if (handlers != null) {
+      for (Handler handler : handlers) {
+        if (handler != null) {
+          handler.interrupt();
+        }
+      }
+    }
+  }
+
+  /** Wait for the server to be stopped.
+   * Does not wait for all subthreads to finish.
+   *  See {@link #stop()}.
+   * @throws InterruptedException e
+   */
+  @Override
+  public synchronized void join() throws InterruptedException {
+    while (running) {
+      wait();
+    }
+  }
+
+  /**
+   * Return the socket (ip+port) on which the RPC server is listening to.
+   * @return the socket (ip+port) on which the RPC server is listening to.
+   */
+  @Override
+  public synchronized InetSocketAddress getListenerAddress() {
+    return listener.getAddress();
+  }
+
+  /**
+   * Set the handler for calling out of RPC for error conditions.
+   * @param handler the handler implementation
+   */
+  @Override
+  public void setErrorHandler(HBaseRPCErrorHandler handler) {
+    this.errorHandler = handler;
+  }
+
+  /**
+   * Returns the metrics instance for reporting RPC call statistics
+   */
+  public HBaseRpcMetrics getRpcMetrics() {
+    return rpcMetrics;
+  }
+
+  /**
+   * When the read or write buffer size is larger than this limit, i/o will be
+   * done in chunks of this size. Most RPC requests and responses would be
+   * be smaller.
+   */
+  private static int NIO_BUFFER_LIMIT = 64 * 1024; //should not be more than 64KB.
+
+  /**
+   * This is a wrapper around {@link java.nio.channels.WritableByteChannel#write(java.nio.ByteBuffer)}.
+   * If the amount of data is large, it writes to channel in smaller chunks.
+   * This is to avoid jdk from creating many direct buffers as the size of
+   * buffer increases. This also minimizes extra copies in NIO layer
+   * as a result of multiple write operations required to write a large
+   * buffer.
+   *
+   * @param channel writable byte channel to write to
+   * @param buffer buffer to write
+   * @return number of bytes written
+   * @throws java.io.IOException e
+   * @see java.nio.channels.WritableByteChannel#write(java.nio.ByteBuffer)
+   */
+  protected int channelWrite(WritableByteChannel channel,
+                                    ByteBuffer buffer) throws IOException {
+
+    int count =  (buffer.remaining() <= NIO_BUFFER_LIMIT) ?
+           channel.write(buffer) : channelIO(null, channel, buffer);
+    if (count > 0) {
+      rpcMetrics.sentBytes.inc(count);
+    }
+    return count;
+  }
+
+  /**
+   * This is a wrapper around {@link java.nio.channels.ReadableByteChannel#read(java.nio.ByteBuffer)}.
+   * If the amount of data is large, it writes to channel in smaller chunks.
+   * This is to avoid jdk from creating many direct buffers as the size of
+   * ByteBuffer increases. There should not be any performance degredation.
+   *
+   * @param channel writable byte channel to write on
+   * @param buffer buffer to write
+   * @return number of bytes written
+   * @throws java.io.IOException e
+   * @see java.nio.channels.ReadableByteChannel#read(java.nio.ByteBuffer)
+   */
+  protected int channelRead(ReadableByteChannel channel,
+                                   ByteBuffer buffer) throws IOException {
+
+    int count = (buffer.remaining() <= NIO_BUFFER_LIMIT) ?
+           channel.read(buffer) : channelIO(channel, null, buffer);
+    if (count > 0) {
+      rpcMetrics.receivedBytes.inc(count);
+  }
+    return count;
+  }
+
+  /**
+   * Helper for {@link #channelRead(java.nio.channels.ReadableByteChannel, java.nio.ByteBuffer)}
+   * and {@link #channelWrite(java.nio.channels.WritableByteChannel, java.nio.ByteBuffer)}. Only
+   * one of readCh or writeCh should be non-null.
+   *
+   * @param readCh read channel
+   * @param writeCh write channel
+   * @param buf buffer to read or write into/out of
+   * @return bytes written
+   * @throws java.io.IOException e
+   * @see #channelRead(java.nio.channels.ReadableByteChannel, java.nio.ByteBuffer)
+   * @see #channelWrite(java.nio.channels.WritableByteChannel, java.nio.ByteBuffer)
+   */
+  private static int channelIO(ReadableByteChannel readCh,
+                               WritableByteChannel writeCh,
+                               ByteBuffer buf) throws IOException {
+
+    int originalLimit = buf.limit();
+    int initialRemaining = buf.remaining();
+    int ret = 0;
+
+    while (buf.remaining() > 0) {
+      try {
+        int ioSize = Math.min(buf.remaining(), NIO_BUFFER_LIMIT);
+        buf.limit(buf.position() + ioSize);
+
+        ret = (readCh == null) ? writeCh.write(buf) : readCh.read(buf);
+
+        if (ret < ioSize) {
+          break;
+        }
+
+      } finally {
+        buf.limit(originalLimit);
+      }
+    }
+
+    int nBytes = initialRemaining - buf.remaining();
+    return (nBytes > 0) ? nBytes : ret;
+  }
+
+  /**
+   * Needed for delayed calls.  We need to be able to store the current call
+   * so that we can complete it later.
+   * @return Call the server is currently handling.
+   */
+  public static RpcCallContext getCurrentCall() {
+    return CurCall.get();
+  }
+
+  public long getResponseQueueSize(){
+    return responseQueuesSizeThrottler.getCurrentValue();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
new file mode 100644
index 0000000..48cdc78
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
@@ -0,0 +1,316 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.UnknownRegionException;
+import org.apache.hadoop.hbase.client.coprocessor.Exec;
+import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
+import org.apache.hadoop.hbase.security.TokenInfo;
+import org.apache.hadoop.hbase.security.KerberosInfo;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * Clients interact with the HMasterInterface to gain access to meta-level
+ * HBase functionality, like finding an HRegionServer and creating/destroying
+ * tables.
+ *
+ * 
      NOTE: if you change the interface, you must change the RPC version
+ * number in HBaseRPCProtocolVersion
+ *
+ */
+@KerberosInfo(
+    serverPrincipal = "hbase.master.kerberos.principal")
+@TokenInfo("HBASE_AUTH_TOKEN")
+public interface HMasterInterface extends VersionedProtocol {
+  /**
+   * This Interfaces' version. Version changes when the Interface changes.
+   */
+  // All HBase Interfaces used derive from HBaseRPCProtocolVersion.  It
+  // maintained a single global version number on all HBase Interfaces.  This
+  // meant all HBase RPC was broke though only one of the three RPC Interfaces
+  // had changed.  This has since been undone.
+  // 29:  4/3/2010 - changed ClusterStatus serialization
+  // 30: 3/20/2012 - HBASE-5589: Added offline method 
+	
+  // NOTE: Not bumped from 29 to maintain compatibility since this addition is
+  // after the v0.92.0 releases this is applied to.  This is not bumped for
+  // 0.94.0 to maintain rolling restart compatibility with 0.92.x. 
+  public static final long VERSION = 29L;
+
+  /** @return true if master is available */
+  public boolean isMasterRunning();
+
+  // Admin tools would use these cmds
+
+  /**
+   * Creates a new table asynchronously.  If splitKeys are specified, then the
+   * table will be created with an initial set of multiple regions.
+   * If splitKeys is null, the table will be created with a single region.
+   * @param desc table descriptor
+   * @param splitKeys
+   * @throws IOException
+   */
+  public void createTable(HTableDescriptor desc, byte [][] splitKeys)
+  throws IOException;
+
+  /**
+   * Deletes a table
+   * @param tableName table to delete
+   * @throws IOException e
+   */
+  public void deleteTable(final byte [] tableName) throws IOException;
+
+  /**
+   * Used by the client to get the number of regions that have received the
+   * updated schema
+   *
+   * @param tableName
+   * @return Pair indicating the number of regions updated Pair.getFirst() is the
+   *         regions that are yet to be updated Pair.getSecond() is the total number
+   *         of regions of the table
+   * @throws IOException
+   */
+  public Pair getAlterStatus(byte[] tableName)
+  throws IOException;
+
+  /**
+   * Adds a column to the specified table
+   * @param tableName table to modify
+   * @param column column descriptor
+   * @throws IOException e
+   */
+  public void addColumn(final byte [] tableName, HColumnDescriptor column)
+  throws IOException;
+
+  /**
+   * Modifies an existing column on the specified table
+   * @param tableName table name
+   * @param descriptor new column descriptor
+   * @throws IOException e
+   */
+  public void modifyColumn(final byte [] tableName, HColumnDescriptor descriptor)
+  throws IOException;
+
+
+  /**
+   * Deletes a column from the specified table. Table must be disabled.
+   * @param tableName table to alter
+   * @param columnName column family to remove
+   * @throws IOException e
+   */
+  public void deleteColumn(final byte [] tableName, final byte [] columnName)
+  throws IOException;
+
+  /**
+   * Puts the table on-line (only needed if table has been previously taken offline)
+   * @param tableName table to enable
+   * @throws IOException e
+   */
+  public void enableTable(final byte [] tableName) throws IOException;
+
+  /**
+   * Take table offline
+   *
+   * @param tableName table to take offline
+   * @throws IOException e
+   */
+  public void disableTable(final byte [] tableName) throws IOException;
+
+  /**
+   * Modify a table's metadata
+   *
+   * @param tableName table to modify
+   * @param htd new descriptor for table
+   * @throws IOException e
+   */
+  public void modifyTable(byte[] tableName, HTableDescriptor htd)
+  throws IOException;
+
+  /**
+   * Shutdown an HBase cluster.
+   * @throws IOException e
+   */
+  public void shutdown() throws IOException;
+
+  /**
+   * Stop HBase Master only.
+   * Does not shutdown the cluster.
+   * @throws IOException e
+   */
+  public void stopMaster() throws IOException;
+
+  /**
+   * Return cluster status.
+   * @return status object
+   */
+  public ClusterStatus getClusterStatus();
+
+
+  /**
+   * Move the region r to dest.
+   * @param encodedRegionName The encoded region name; i.e. the hash that makes
+   * up the region name suffix: e.g. if regionname is
+   * TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.,
+   * then the encoded region name is: 527db22f95c8a9e0116f0cc13c680396.
+   * @param destServerName The servername of the destination regionserver.  If
+   * passed the empty byte array we'll assign to a random server.  A server name
+   * is made of host, port and startcode.  Here is an example:
+   *  host187.example.com,60020,1289493121758.
+   * @throws UnknownRegionException Thrown if we can't find a region named
+   * encodedRegionName
+   */
+  public void move(final byte [] encodedRegionName, final byte [] destServerName)
+  throws UnknownRegionException;
+
+  /**
+   * Assign a region to a server chosen at random.
+   * @param regionName Region to assign.  Will use existing RegionPlan if one
+   * found.
+   * @param force If true, will force the assignment.
+   * @throws IOException
+   * @deprecated The force is unused.Use {@link #assign(byte[])}
+   */
+  public void assign(final byte [] regionName, final boolean force)
+  throws IOException;
+
+  /**
+   * Assign a region to a server chosen at random.
+   * 
+   * @param regionName
+   *          Region to assign. Will use existing RegionPlan if one found.
+   * @throws IOException
+   */
+  public void assign(final byte[] regionName) throws IOException;
+  
+  /**
+   * Unassign a region from current hosting regionserver.  Region will then be
+   * assigned to a regionserver chosen at random.  Region could be reassigned
+   * back to the same server.  Use {@link #move(byte[], byte[])} if you want
+   * to control the region movement.
+   * @param regionName Region to unassign. Will clear any existing RegionPlan
+   * if one found.
+   * @param force If true, force unassign (Will remove region from
+   * regions-in-transition too if present as well as from assigned regions --
+   * radical!.If results in double assignment use hbck -fix to resolve.
+   * @throws IOException
+   */
+  public void unassign(final byte [] regionName, final boolean force)
+  throws IOException;
+
+  /**
+   * Offline a region from the assignment manager's in-memory state.  The
+   * region should be in a closed state and there will be no attempt to
+   * automatically reassign the region as in unassign.   This is a special
+   * method, and should only be used by experts or hbck.
+   * @param regionName Region to offline.  Will clear any existing RegionPlan
+   * if one found.
+   * @throws IOException
+   */
+  public void offline(final byte[] regionName) throws IOException;
+
+  /**
+   * Run the balancer.  Will run the balancer and if regions to move, it will
+   * go ahead and do the reassignments.  Can NOT run for various reasons.  Check
+   * logs.
+   * @return True if balancer ran and was able to tell the region servers to
+   * unassign all the regions to balance (the re-assignment itself is async),
+   * false otherwise.
+   */
+  public boolean balance();
+
+  /**
+   * Turn the load balancer on or off.
+   * @param b If true, enable balancer. If false, disable balancer.
+   * @return Previous balancer value
+   */
+  public boolean balanceSwitch(final boolean b);
+
+  /**
+   * Turn the load balancer on or off.
+   * It waits until current balance() call, if outstanding, to return.
+   * @param b If true, enable balancer. If false, disable balancer.
+   * @return Previous balancer value
+   */
+  public boolean synchronousBalanceSwitch(final boolean b);
+
+  /**
+   * Get array of all HTDs.
+   * @return array of HTableDescriptor
+   */
+  public HTableDescriptor[] getHTableDescriptors();
+
+  /**
+   * Get array of HTDs for requested tables.
+   * @param tableNames
+   * @return array of HTableDescriptor
+   */
+  public HTableDescriptor[] getHTableDescriptors(List tableNames);
+
+  /**
+   * Set the state to ENABLED or DISABLED in the zookeeper
+   * @param tableName
+   * @param state
+   * @throws IOException
+   */
+  public void setTableState(String tableName, String state) throws IOException;
+
+  /**
+   * Executes a single {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol}
+   * method using the registered protocol handlers.
+   * {@link CoprocessorProtocol} implementations must be registered via the
+   * {@link org.apache.hadoop.hbase.master.MasterServices#registerProtocol(Class, CoprocessorProtocol)}
+   * method before they are available.
+   *
+   * @param call an {@code Exec} instance identifying the protocol, method name,
+   *     and parameters for the method invocation
+   * @return an {@code ExecResult} instance containing the region name of the
+   *     invocation and the return value
+   * @throws IOException if no registered protocol handler is found or an error
+   *     occurs during the invocation
+   * @see org.apache.hadoop.hbase.master.MasterServices#registerProtocol(Class, CoprocessorProtocol)
+   */
+  public ExecResult execCoprocessor(Exec call)
+      throws IOException;
+
+  public long snapshot(final HSnapshotDescription snapshot)
+    throws IOException;
+
+  public List getCompletedSnapshots()
+    throws IOException;
+
+  public void deleteSnapshot(final HSnapshotDescription snapshot)
+    throws IOException;
+
+  public boolean isSnapshotDone(final HSnapshotDescription snapshot)
+    throws IOException;
+
+  public void restoreSnapshot(final HSnapshotDescription request)
+    throws IOException;
+
+  public boolean isRestoreSnapshotDone(final HSnapshotDescription request)
+    throws IOException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java
new file mode 100644
index 0000000..dfb9133
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java
@@ -0,0 +1,76 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.security.KerberosInfo;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.hbase.ipc.VersionedProtocol;
+
+/**
+ * The Master publishes this Interface for RegionServers to register themselves
+ * on.
+ */
+@KerberosInfo(
+    serverPrincipal = "hbase.master.kerberos.principal",
+    clientPrincipal = "hbase.regionserver.kerberos.principal")
+public interface HMasterRegionInterface extends VersionedProtocol {
+  /**
+   * This Interfaces' version. Version changes when the Interface changes.
+   */
+  // All HBase Interfaces used derive from HBaseRPCProtocolVersion.  It
+  // maintained a single global version number on all HBase Interfaces.  This
+  // meant all HBase RPC was broke though only one of the three RPC Interfaces
+  // had changed.  This has since been undone.
+  public static final long VERSION = 29L;
+
+  /**
+   * Called when a region server first starts.
+   * @param port Port number this regionserver is up on.
+   * @param serverStartcode This servers' startcode.
+   * @param serverCurrentTime The current time of the region server in ms
+   * @throws IOException e
+   * @return Configuration for the regionserver to use: e.g. filesystem,
+   * hbase rootdir, the hostname to use creating the RegionServer ServerName,
+   * etc.
+   */
+  public MapWritable regionServerStartup(final int port,
+    final long serverStartcode, final long serverCurrentTime)
+  throws IOException;
+
+  /**
+   * @param sn {@link ServerName#getVersionedBytes()}
+   * @param hsl Server load.
+   * @throws IOException
+   */
+  public void regionServerReport(byte [] sn, HServerLoad hsl)
+  throws IOException;
+  
+  /**
+   * Called by a region server to report a fatal error that is causing
+   * it to abort.
+   * @param sn {@link ServerName#getVersionedBytes()}
+   * @param errorMessage informative text to expose in the master logs and UI
+   */
+  public void reportRSFatalError(byte [] sn, String errorMessage);
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
new file mode 100644
index 0000000..9886b3a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
@@ -0,0 +1,650 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.MultiAction;
+import org.apache.hadoop.hbase.client.MultiResponse;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.Exec;
+import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary;
+import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.CompactionState;
+import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.security.TokenInfo;
+import org.apache.hadoop.hbase.security.KerberosInfo;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.hbase.ipc.VersionedProtocol;
+
+/**
+ * Clients interact with HRegionServers using a handle to the HRegionInterface.
+ *
+ * 
      NOTE: if you change the interface, you must change the RPC version
+ * number in HBaseRPCProtocolVersion
+ */
+@KerberosInfo(
+    serverPrincipal = "hbase.regionserver.kerberos.principal")
+@TokenInfo("HBASE_AUTH_TOKEN")
+public interface HRegionInterface extends VersionedProtocol, Stoppable, Abortable {
+  /**
+   * This Interfaces' version. Version changes when the Interface changes.
+   */
+  // All HBase Interfaces used derive from HBaseRPCProtocolVersion.  It
+  // maintained a single global version number on all HBase Interfaces.  This
+  // meant all HBase RPC was broke though only one of the three RPC Interfaces
+  // had changed.  This has since been undone.
+  public static final long VERSION = 29L;
+
+  /**
+   * Get metainfo about an HRegion
+   *
+   * @param regionName name of the region
+   * @return HRegionInfo object for region
+   * @throws NotServingRegionException
+   * @throws ConnectException
+   * @throws IOException This can manifest as an Hadoop ipc {@link RemoteException}
+   */
+  public HRegionInfo getRegionInfo(final byte [] regionName)
+  throws NotServingRegionException, ConnectException, IOException;
+
+  /**
+   * Flush the given region
+   * @param region name
+   */
+  public void flushRegion(byte[] regionName)
+    throws IllegalArgumentException, IOException;
+
+  /**
+   * Flush the given region if lastFlushTime < ifOlderThanTS
+   * @param region name
+   * @param timestamp
+   */
+  public void flushRegion(byte[] regionName, long ifOlderThanTS)
+    throws IllegalArgumentException, IOException;
+
+  /**
+   * Gets last flush time for the given region
+   * @return the last flush time for a region
+   */
+  public long getLastFlushTime(byte[] regionName);
+
+  /**
+   * Get a list of store files for a particular CF in a particular region
+   * @param region name
+   * @param CF name
+   * @return the list of store files
+   */
+  public List getStoreFileList(byte[] regionName, byte[] columnFamily)
+    throws IllegalArgumentException;
+
+  /**
+   * Get a list of store files for a set of CFs in a particular region
+   * @param region name
+   * @param CF names
+   * @return the list of store files
+   */
+  public List getStoreFileList(byte[] regionName, byte[][] columnFamilies)
+    throws IllegalArgumentException;
+
+  /**
+   * Get a list of store files for all CFs in a particular region
+   * @param region name
+   * @return the list of store files
+   */
+  public List getStoreFileList(byte[] regionName)
+    throws IllegalArgumentException;
+
+  /**
+   * Return all the data for the row that matches row exactly,
+   * or the one that immediately preceeds it.
+   *
+   * @param regionName region name
+   * @param row row key
+   * @param family Column family to look for row in.
+   * @return map of values
+   * @throws IOException e
+   */
+  public Result getClosestRowBefore(final byte [] regionName,
+    final byte [] row, final byte [] family)
+  throws IOException;
+
+  /**
+   * Perform Get operation.
+   * @param regionName name of region to get from
+   * @param get Get operation
+   * @return Result
+   * @throws IOException e
+   */
+  public Result get(byte [] regionName, Get get) throws IOException;
+
+  /**
+   * Perform exists operation.
+   * @param regionName name of region to get from
+   * @param get Get operation describing cell to test
+   * @return true if exists
+   * @throws IOException e
+   */
+  public boolean exists(byte [] regionName, Get get) throws IOException;
+
+  /**
+   * Put data into the specified region
+   * @param regionName region name
+   * @param put the data to be put
+   * @throws IOException e
+   */
+  public void put(final byte [] regionName, final Put put)
+  throws IOException;
+
+  /**
+   * Put an array of puts into the specified region
+   *
+   * @param regionName region name
+   * @param puts List of puts to execute
+   * @return The number of processed put's.  Returns -1 if all Puts
+   * processed successfully.
+   * @throws IOException e
+   */
+  public int put(final byte[] regionName, final List puts)
+  throws IOException;
+
+  /**
+   * Deletes all the KeyValues that match those found in the Delete object,
+   * if their ts <= to the Delete. In case of a delete with a specific ts it
+   * only deletes that specific KeyValue.
+   * @param regionName region name
+   * @param delete delete object
+   * @throws IOException e
+   */
+  public void delete(final byte[] regionName, final Delete delete)
+  throws IOException;
+
+  /**
+   * Put an array of deletes into the specified region
+   *
+   * @param regionName region name
+   * @param deletes delete List to execute
+   * @return The number of processed deletes.  Returns -1 if all Deletes
+   * processed successfully.
+   * @throws IOException e
+   */
+  public int delete(final byte[] regionName, final List deletes)
+  throws IOException;
+
+  /**
+   * Atomically checks if a row/family/qualifier value match the expectedValue.
+   * If it does, it adds the put. If passed expected value is null, then the
+   * check is for non-existance of the row/column.
+   *
+   * @param regionName region name
+   * @param row row to check
+   * @param family column family
+   * @param qualifier column qualifier
+   * @param value the expected value
+   * @param put data to put if check succeeds
+   * @throws IOException e
+   * @return true if the new put was execute, false otherwise
+   */
+  public boolean checkAndPut(final byte[] regionName, final byte [] row,
+      final byte [] family, final byte [] qualifier, final byte [] value,
+      final Put put)
+  throws IOException;
+
+
+  /**
+   * Atomically checks if a row/family/qualifier value match the expectedValue.
+   * If it does, it adds the delete. If passed expected value is null, then the
+   * check is for non-existance of the row/column.
+   *
+   * @param regionName region name
+   * @param row row to check
+   * @param family column family
+   * @param qualifier column qualifier
+   * @param value the expected value
+   * @param delete data to delete if check succeeds
+   * @throws IOException e
+   * @return true if the new delete was execute, false otherwise
+   */
+  public boolean checkAndDelete(final byte[] regionName, final byte [] row,
+      final byte [] family, final byte [] qualifier, final byte [] value,
+      final Delete delete)
+  throws IOException;
+
+  /**
+   * Atomically increments a column value. If the column value isn't long-like,
+   * this could throw an exception. If passed expected value is null, then the
+   * check is for non-existance of the row/column.
+   *
+   * @param regionName region name
+   * @param row row to check
+   * @param family column family
+   * @param qualifier column qualifier
+   * @param amount long amount to increment
+   * @param writeToWAL whether to write the increment to the WAL
+   * @return new incremented column value
+   * @throws IOException e
+   */
+  public long incrementColumnValue(byte [] regionName, byte [] row,
+      byte [] family, byte [] qualifier, long amount, boolean writeToWAL)
+  throws IOException;
+
+  public void mutateRow(byte[] regionName, RowMutations rm)
+      throws IOException;
+
+  /**
+   * Appends values to one or more columns values in a row. Optionally
+   * Returns the updated keys after the append.
+   * 
      
+   * This operation does not appear atomic to readers. Appends are done
+   * under a row lock but readers do not take row locks.
+   * @param regionName region name
+   * @param append Append operation
+   * @return changed cells (maybe null)
+   */
+  public Result append(byte[] regionName, Append append)
+  throws IOException;
+
+  /**
+   * Increments one or more columns values in a row.  Returns the
+   * updated keys after the increment.
+   * 
      
+   * This operation does not appear atomic to readers.  Increments are done
+   * under a row lock but readers do not take row locks.
+   * @param regionName region name
+   * @param increment increment operation
+   * @return incremented cells
+   */
+  public Result increment(byte[] regionName, Increment increment)
+  throws IOException;
+
+  //
+  // remote scanner interface
+  //
+
+  /**
+   * Opens a remote scanner with a RowFilter.
+   *
+   * @param regionName name of region to scan
+   * @param scan configured scan object
+   * @return scannerId scanner identifier used in other calls
+   * @throws IOException e
+   */
+  public long openScanner(final byte [] regionName, final Scan scan)
+  throws IOException;
+
+  /**
+   * Get the next set of values
+   * @param scannerId clientId passed to openScanner
+   * @return map of values; returns null if no results.
+   * @throws IOException e
+   */
+  public Result next(long scannerId) throws IOException;
+
+  /**
+   * Get the next set of values
+   * @param scannerId clientId passed to openScanner
+   * @param numberOfRows the number of rows to fetch
+   * @return Array of Results (map of values); array is empty if done with this
+   * region and null if we are NOT to go to the next region (happens when a
+   * filter rules that the scan is done).
+   * @throws IOException e
+   */
+  public Result [] next(long scannerId, int numberOfRows) throws IOException;
+
+  /**
+   * Close a scanner
+   *
+   * @param scannerId the scanner id returned by openScanner
+   * @throws IOException e
+   */
+  public void close(long scannerId) throws IOException;
+
+  /**
+   * Opens a remote row lock.
+   *
+   * @param regionName name of region
+   * @param row row to lock
+   * @return lockId lock identifier
+   * @throws IOException e
+   */
+  public long lockRow(final byte [] regionName, final byte [] row)
+  throws IOException;
+
+  /**
+   * Releases a remote row lock.
+   *
+   * @param regionName region name
+   * @param lockId the lock id returned by lockRow
+   * @throws IOException e
+   */
+  public void unlockRow(final byte [] regionName, final long lockId)
+  throws IOException;
+
+
+  /**
+   * @return All regions online on this region server
+   * @throws IOException e
+   */
+  public List getOnlineRegions() throws IOException;
+
+  /**
+   * Method used when a master is taking the place of another failed one.
+   * @return This servers {@link HServerInfo}; it has RegionServer POV on the
+   * hostname which may not agree w/ how the Master sees this server.
+   * @throws IOException e
+   * @deprecated
+   */
+  public HServerInfo getHServerInfo() throws IOException;
+
+  /**
+   * Method used for doing multiple actions(Deletes, Gets and Puts) in one call
+   * @param multi
+   * @return MultiResult
+   * @throws IOException
+   */
+  public  MultiResponse multi(MultiAction multi) throws IOException;
+
+  /**
+   * Atomically bulk load multiple HFiles (say from different column families)
+   * into an open region.
+   * 
+   * @param familyPaths List of (family, hfile path) pairs
+   * @param regionName name of region to load hfiles into
+   * @return true if successful, false if failed recoverably
+   * @throws IOException if fails unrecoverably
+   */
+  public boolean bulkLoadHFiles(List> familyPaths, byte[] regionName)
+  throws IOException;
+
+  // Master methods
+
+  /**
+   * Opens the specified region.
+   * 
+   * @param region
+   *          region to open
+   * @return RegionOpeningState 
+   *         OPENED         - if region open request was successful.
+   *         ALREADY_OPENED - if the region was already opened. 
+   *         FAILED_OPENING - if region opening failed.
+   *
+   * @throws IOException
+   */
+  public RegionOpeningState openRegion(final HRegionInfo region) throws IOException;
+
+  /**
+   * Opens the specified region.
+   * @param region
+   *          region to open
+   * @param versionOfOfflineNode
+   *          the version of znode to compare when RS transitions the znode from
+   *          OFFLINE state.
+   * @return RegionOpeningState 
+   *         OPENED         - if region open request was successful.
+   *         ALREADY_OPENED - if the region was already opened. 
+   *         FAILED_OPENING - if region opening failed.
+   * @throws IOException
+   */
+  public RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNode)
+      throws IOException;
+  
+  /**
+   * Opens the specified regions.
+   * @param regions regions to open
+   * @throws IOException
+   */
+  public void openRegions(final List regions) throws IOException;
+
+  /**
+   * Closes the specified region.
+   * @param region region to close
+   * @return true if closing region, false if not
+   * @throws IOException
+   */
+  public boolean closeRegion(final HRegionInfo region)
+  throws IOException;
+
+  /**
+   * Closes the specified region.
+   * @param region region to close
+   * @param versionOfClosingNode
+   *          the version of znode to compare when RS transitions the znode
+   *          from CLOSING state.
+   * @return true if closing region, false if not
+   * @throws IOException
+   */
+  public boolean closeRegion(final HRegionInfo region,
+    final int versionOfClosingNode)
+  throws IOException;
+
+  /**
+   * Closes the specified region and will use or not use ZK during the close
+   * according to the specified flag.
+   * @param region region to close
+   * @param zk true if transitions should be done in ZK, false if not
+   * @return true if closing region, false if not
+   * @throws IOException
+   */
+  public boolean closeRegion(final HRegionInfo region, final boolean zk)
+  throws IOException;
+  
+  /**
+   * Closes the region in the RS with the specified encoded regionName and will
+   * use or not use ZK during the close according to the specified flag. Note
+   * that the encoded region name is in byte format.
+   * 
+   * @param encodedRegionName
+   *          in bytes
+   * @param zk
+   *          true if to use zookeeper, false if need not.
+   * @return true if region is closed, false if not.
+   * @throws IOException
+   */
+  public boolean closeRegion(byte[] encodedRegionName, final boolean zk)
+      throws IOException;
+
+  // Region administrative methods
+
+  /**
+   * Flushes the MemStore of the specified region.
+   * 
      
+   * This method is synchronous.
+   * @param regionInfo region to flush
+   * @throws NotServingRegionException
+   * @throws IOException
+   * @deprecated use {@link #flushRegion(byte[])} instead
+   */
+  void flushRegion(HRegionInfo regionInfo)
+  throws NotServingRegionException, IOException;
+
+  /**
+   * Splits the specified region.
+   * 
      
+   * This method currently flushes the region and then forces a compaction which
+   * will then trigger a split.  The flush is done synchronously but the
+   * compaction is asynchronous.
+   * @param regionInfo region to split
+   * @throws NotServingRegionException
+   * @throws IOException
+   */
+  void splitRegion(HRegionInfo regionInfo)
+  throws NotServingRegionException, IOException;
+
+  /**
+   * Splits the specified region.
+   * 
      
+   * This method currently flushes the region and then forces a compaction which
+   * will then trigger a split.  The flush is done synchronously but the
+   * compaction is asynchronous.
+   * @param regionInfo region to split
+   * @param splitPoint the explicit row to split on
+   * @throws NotServingRegionException
+   * @throws IOException
+   */
+  void splitRegion(HRegionInfo regionInfo, byte[] splitPoint)
+  throws NotServingRegionException, IOException;
+
+  /**
+   * Compacts the specified region.  Performs a major compaction if specified.
+   * 
      
+   * This method is asynchronous.
+   * @param regionInfo region to compact
+   * @param major true to force major compaction
+   * @throws NotServingRegionException
+   * @throws IOException
+   */
+  void compactRegion(HRegionInfo regionInfo, boolean major)
+  throws NotServingRegionException, IOException;
+
+  /**
+   * Compacts a column-family within a specified region.
+   * Performs a major compaction if specified.
+   * 
      
+   * This method is asynchronous.
+   * @param regionInfo region to compact
+   * @param major true to force major compaction
+   * @param columnFamily column family within a region to compact
+   * @throws NotServingRegionException
+   * @throws IOException
+   */
+  void compactRegion(HRegionInfo regionInfo, boolean major, byte[] columnFamily)
+  throws NotServingRegionException, IOException;
+   
+  /**
+   * Replicates the given entries. The guarantee is that the given entries
+   * will be durable on the slave cluster if this method returns without
+   * any exception.
+   * hbase.replication has to be set to true for this to work.
+   *
+   * @param entries entries to replicate
+   * @throws IOException
+   */
+  public void replicateLogEntries(HLog.Entry[] entries) throws IOException;
+
+  /**
+   * Executes a single {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol}
+   * method using the registered protocol handlers.
+   * {@link CoprocessorProtocol} implementations must be registered via the
+   * {@link org.apache.hadoop.hbase.regionserver.HRegion#registerProtocol(Class, org.apache.hadoop.hbase.ipc.CoprocessorProtocol)}
+   * method before they are available.
+   *
+   * @param regionName name of the region against which the invocation is executed
+   * @param call an {@code Exec} instance identifying the protocol, method name,
+   *     and parameters for the method invocation
+   * @return an {@code ExecResult} instance containing the region name of the
+   *     invocation and the return value
+   * @throws IOException if no registered protocol handler is found or an error
+   *     occurs during the invocation
+   * @see org.apache.hadoop.hbase.regionserver.HRegion#registerProtocol(Class, org.apache.hadoop.hbase.ipc.CoprocessorProtocol)
+   */
+  ExecResult execCoprocessor(byte[] regionName, Exec call)
+      throws IOException;
+
+  /**
+   * Atomically checks if a row/family/qualifier value match the expectedValue.
+   * If it does, it adds the put. If passed expected value is null, then the
+   * check is for non-existance of the row/column.
+   *
+   * @param regionName
+   * @param row
+   * @param family
+   * @param qualifier
+   * @param compareOp
+   * @param comparator
+   * @param put
+   * @throws IOException
+   * @return true if the new put was execute, false otherwise
+   */
+  public boolean checkAndPut(final byte[] regionName, final byte[] row,
+      final byte[] family, final byte[] qualifier, final CompareOp compareOp,
+      final WritableByteArrayComparable comparator, final Put put)
+  throws IOException;
+
+  /**
+   * Atomically checks if a row/family/qualifier value match the expectedValue.
+   * If it does, it adds the delete. If passed expected value is null, then the
+   * check is for non-existance of the row/column.
+   *
+   * @param regionName
+   * @param row
+   * @param family
+   * @param qualifier
+   * @param compareOp
+   * @param comparator
+   * @param delete
+   * @throws IOException
+   * @return true if the new put was execute, false otherwise
+   */
+  public boolean checkAndDelete(final byte[] regionName, final byte[] row,
+     final byte[] family, final byte[] qualifier, final CompareOp compareOp,
+     final WritableByteArrayComparable comparator, final Delete delete)
+     throws IOException;
+  
+  /**
+   * Performs a BlockCache summary and returns a List of BlockCacheColumnFamilySummary objects.
+   * This method could be fairly heavyweight in that it evaluates the entire HBase file-system
+   * against what is in the RegionServer BlockCache. 
+   * 
+   * @return BlockCacheColumnFamilySummary
+   * @throws IOException exception
+   */
+  public List getBlockCacheColumnFamilySummaries() throws IOException;
+  /**
+   * Roll the log writer. That is, start writing log messages to a new file.
+   * 
+   * @throws IOException
+   * @throws FailedLogCloseException
+   * @return If lots of logs, flush the returned regions so next time through
+   * we can clean logs. Returns null if nothing to flush.  Names are actual
+   * region names as returned by {@link HRegionInfo#getEncodedName()} 
+   */
+  public byte[][] rollHLogWriter() throws IOException, FailedLogCloseException;
+
+  /**
+   * Get the current compaction state of the region.
+   *
+   * @param regionName the name of the region to check compaction statte.
+   * @return the compaction state name.
+   * @throws IOException exception
+   */
+  public String getCompactionState(final byte[] regionName) throws IOException;
+
+  @Override
+  public void stop(String why);
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java b/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
new file mode 100644
index 0000000..1006570
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.HbaseObjectWritable;
+import org.apache.hadoop.io.VersionMismatchException;
+import org.apache.hadoop.io.VersionedWritable;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+
+/** A method invocation, including the method name and its parameters.*/
+public class Invocation extends VersionedWritable implements Configurable {
+  protected String methodName;
+  @SuppressWarnings("rawtypes")
+  protected Class[] parameterClasses;
+  protected Object[] parameters;
+  protected Configuration conf;
+  private long clientVersion;
+  private int clientMethodsHash;
+
+  private static byte RPC_VERSION = 1;
+
+  public Invocation() {}
+
+  public Invocation(Method method,
+      Class declaringClass, Object[] parameters) {
+    this.methodName = method.getName();
+    this.parameterClasses = method.getParameterTypes();
+    this.parameters = parameters;
+    if (declaringClass.equals(VersionedProtocol.class)) {
+      //VersionedProtocol is exempted from version check.
+      clientVersion = 0;
+      clientMethodsHash = 0;
+    } else {
+      try {
+        Field versionField = declaringClass.getField("VERSION");
+        versionField.setAccessible(true);
+        this.clientVersion = versionField.getLong(declaringClass);
+      } catch (NoSuchFieldException ex) {
+        throw new RuntimeException("The " + declaringClass, ex);
+      } catch (IllegalAccessException ex) {
+        throw new RuntimeException(ex);
+      }
+      this.clientMethodsHash = ProtocolSignature.getFingerprint(
+          declaringClass.getMethods());
+    }
+  }
+
+  /** @return The name of the method invoked. */
+  public String getMethodName() { return methodName; }
+
+  /** @return The parameter classes. */
+  @SuppressWarnings({ "rawtypes" })
+  public Class[] getParameterClasses() { return parameterClasses; }
+
+  /** @return The parameter instances. */
+  public Object[] getParameters() { return parameters; }
+
+  long getProtocolVersion() {
+    return clientVersion;
+  }
+
+  protected int getClientMethodsHash() {
+    return clientMethodsHash;
+  }
+
+  /**
+   * Returns the rpc version used by the client.
+   * @return rpcVersion
+   */
+  public long getRpcVersion() {
+    return RPC_VERSION;
+  }
+
+  public void readFields(DataInput in) throws IOException {
+    try {
+      super.readFields(in);
+      methodName = in.readUTF();
+      clientVersion = in.readLong();
+      clientMethodsHash = in.readInt();
+    } catch (VersionMismatchException e) {
+      // VersionMismatchException doesn't provide an API to access
+      // expectedVersion and foundVersion.  This is really sad.
+      if (e.toString().endsWith("found v0")) {
+        // Try to be a bit backwards compatible.  In previous versions of
+        // HBase (before HBASE-3939 in 0.92) Invocation wasn't a
+        // VersionedWritable and thus the first thing on the wire was always
+        // the 2-byte length of the method name.  Because no method name is
+        // longer than 255 characters, and all method names are in ASCII,
+        // The following code is equivalent to `in.readUTF()', which we can't
+        // call again here, because `super.readFields(in)' already consumed
+        // the first byte of input, which can't be "unread" back into `in'.
+        final short len = (short) (in.readByte() & 0xFF);  // Unsigned byte.
+        final byte[] buf = new byte[len];
+        in.readFully(buf, 0, len);
+        methodName = new String(buf);
+      }
+    }
+    parameters = new Object[in.readInt()];
+    parameterClasses = new Class[parameters.length];
+    HbaseObjectWritable objectWritable = new HbaseObjectWritable();
+    for (int i = 0; i < parameters.length; i++) {
+      parameters[i] = HbaseObjectWritable.readObject(in, objectWritable,
+        this.conf);
+      parameterClasses[i] = objectWritable.getDeclaredClass();
+    }
+  }
+
+  public void write(DataOutput out) throws IOException {
+    super.write(out);
+    out.writeUTF(this.methodName);
+    out.writeLong(clientVersion);
+    out.writeInt(clientMethodsHash);
+    out.writeInt(parameterClasses.length);
+    for (int i = 0; i < parameterClasses.length; i++) {
+      HbaseObjectWritable.writeObject(out, parameters[i], parameterClasses[i],
+                                 conf);
+    }
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder buffer = new StringBuilder(256);
+    buffer.append(methodName);
+    buffer.append("(");
+    for (int i = 0; i < parameters.length; i++) {
+      if (i != 0)
+        buffer.append(", ");
+      buffer.append(parameters[i]);
+    }
+    buffer.append(")");
+    buffer.append(", rpc version="+RPC_VERSION);
+    buffer.append(", client version="+clientVersion);
+    buffer.append(", methodsFingerPrint="+clientMethodsHash);
+    return buffer.toString();
+  }
+
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  @Override
+  public byte getVersion() {
+    return RPC_VERSION;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/MasterExecRPCInvoker.java b/src/main/java/org/apache/hadoop/hbase/ipc/MasterExecRPCInvoker.java
new file mode 100644
index 0000000..caaa430
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/MasterExecRPCInvoker.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.ServerCallable;
+import org.apache.hadoop.hbase.client.coprocessor.Exec;
+import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+
+/**
+ * Backs a {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol} subclass proxy and forwards method
+ * invocations for server execution.  Note that internally this will issue a
+ * separate RPC call for each method invocation.
+ */
+public class MasterExecRPCInvoker implements InvocationHandler {
+  // LOG is NOT in hbase subpackage intentionally so that the default HBase
+  // DEBUG log level does NOT emit RPC-level logging.
+  private static final Log LOG = LogFactory.getLog("org.apache.hadoop.ipc.MasterExecRPCInvoker");
+
+  private Configuration conf;
+  private final HConnection connection;
+  private Class protocol;
+
+  public MasterExecRPCInvoker(Configuration conf,
+                              HConnection connection,
+                              Class protocol) {
+    this.conf = conf;
+    this.connection = connection;
+    this.protocol = protocol;
+  }
+
+  @Override
+  public Object invoke(Object instance, final Method method, final Object[] args)
+      throws Throwable {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Call: "+method.getName()+", "+(args != null ? args.length : 0));
+    }
+    Exec exec = new Exec(conf, protocol, method, args);
+    ExecResult result = connection.getMaster().execCoprocessor(exec);
+    LOG.debug("Master Result is value="+result.getValue());
+    return result.getValue();
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/ProtocolSignature.java b/src/main/java/org/apache/hadoop/hbase/ipc/ProtocolSignature.java
new file mode 100644
index 0000000..f345cee
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/ProtocolSignature.java
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.HashMap;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableFactories;
+import org.apache.hadoop.io.WritableFactory;
+
+public class ProtocolSignature implements Writable {
+  static {               // register a ctor
+    WritableFactories.setFactory
+      (ProtocolSignature.class,
+       new WritableFactory() {
+         public Writable newInstance() { return new ProtocolSignature(); }
+       });
+  }
+
+  private long version;
+  private int[] methods = null; // an array of method hash codes
+
+  /**
+   * default constructor
+   */
+  public ProtocolSignature() {
+  }
+
+  /**
+   * Constructor
+   *
+   * @param version server version
+   * @param methodHashcodes hash codes of the methods supported by server
+   */
+  public ProtocolSignature(long version, int[] methodHashcodes) {
+    this.version = version;
+    this.methods = methodHashcodes;
+  }
+
+  public long getVersion() {
+    return version;
+  }
+
+  public int[] getMethods() {
+    return methods;
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    version = in.readLong();
+    boolean hasMethods = in.readBoolean();
+    if (hasMethods) {
+      int numMethods = in.readInt();
+      methods = new int[numMethods];
+      for (int i=0; i type : method.getParameterTypes()) {
+      hashcode = 31*hashcode ^ type.getName().hashCode();
+    }
+    return hashcode;
+  }
+
+  /**
+   * Convert an array of Method into an array of hash codes
+   *
+   * @param methods
+   * @return array of hash codes
+   */
+  private static int[] getFingerprints(Method[] methods) {
+    if (methods == null) {
+      return null;
+    }
+    int[] hashCodes = new int[methods.length];
+    for (int i = 0; i 
+     PROTOCOL_FINGERPRINT_CACHE =
+       new HashMap();
+
+  /**
+   * Return a protocol's signature and finger print from cache
+   *
+   * @param protocol a protocol class
+   * @param serverVersion protocol version
+   * @return its signature and finger print
+   */
+  private static ProtocolSigFingerprint getSigFingerprint(
+      Class  protocol, long serverVersion) {
+    String protocolName = protocol.getName();
+    synchronized (PROTOCOL_FINGERPRINT_CACHE) {
+      ProtocolSigFingerprint sig = PROTOCOL_FINGERPRINT_CACHE.get(protocolName);
+      if (sig == null) {
+        int[] serverMethodHashcodes = getFingerprints(protocol.getMethods());
+        sig = new ProtocolSigFingerprint(
+            new ProtocolSignature(serverVersion, serverMethodHashcodes),
+            getFingerprint(serverMethodHashcodes));
+        PROTOCOL_FINGERPRINT_CACHE.put(protocolName, sig);
+      }
+      return sig;
+    }
+  }
+
+  /**
+   * Get a server protocol's signature
+   *
+   * @param clientMethodsHashCode client protocol methods hashcode
+   * @param serverVersion server protocol version
+   * @param protocol protocol
+   * @return the server's protocol signature
+   */
+  static ProtocolSignature getProtocolSignature(
+      int clientMethodsHashCode,
+      long serverVersion,
+      Class protocol) {
+    // try to get the finger print & signature from the cache
+    ProtocolSigFingerprint sig = getSigFingerprint(protocol, serverVersion);
+
+    // check if the client side protocol matches the one on the server side
+    if (clientMethodsHashCode == sig.fingerprint) {
+      return new ProtocolSignature(serverVersion, null);  // null indicates a match
+    }
+
+    return sig.signature;
+  }
+
+  /**
+   * Get a server protocol's signature
+   *
+   * @param server server implementation
+   * @param protocol server protocol
+   * @param clientVersion client's version
+   * @param clientMethodsHash client's protocol's hash code
+   * @return the server protocol's signature
+   * @throws IOException if any error occurs
+   */
+  @SuppressWarnings("unchecked")
+  public static ProtocolSignature getProtocolSignature(VersionedProtocol server,
+      String protocol,
+      long clientVersion, int clientMethodsHash) throws IOException {
+    Class inter;
+    try {
+      inter = (Class)Class.forName(protocol);
+    } catch (Exception e) {
+      throw new IOException(e);
+    }
+    long serverVersion = server.getProtocolVersion(protocol, clientVersion);
+    return ProtocolSignature.getProtocolSignature(
+        clientMethodsHash, serverVersion, inter);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/RequestContext.java b/src/main/java/org/apache/hadoop/hbase/ipc/RequestContext.java
new file mode 100644
index 0000000..7de87c1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/RequestContext.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import org.apache.hadoop.hbase.security.User;
+
+import java.net.InetAddress;
+
+/**
+ * Represents client information (authenticated username, remote address, protocol)
+ * for the currently executing request within a RPC server handler thread.  If
+ * called outside the context of a RPC request, all values will be
+ * null.
+ */
+public class RequestContext {
+  private static ThreadLocal instance =
+      new ThreadLocal() {
+        protected RequestContext initialValue() {
+          return new RequestContext(null, null, null);
+        }
+      };
+
+  public static RequestContext get() {
+    return instance.get();
+  }
+
+
+  /**
+   * Returns the user credentials associated with the current RPC request or
+   * null if no credentials were provided.
+   * @return A User
+   */
+  public static User getRequestUser() {
+    RequestContext ctx = instance.get();
+    if (ctx != null) {
+      return ctx.getUser();
+    }
+    return null;
+  }
+
+  /**
+   * Returns the username for any user associated with the current RPC
+   * request or null if no user is set.
+   */
+  public static String getRequestUserName() {
+    User user = getRequestUser();
+    if (user != null) {
+      return user.getShortName();
+    }
+    return null;
+  }
+
+  /**
+   * Indicates whether or not the current thread is within scope of executing
+   * an RPC request.
+   */
+  public static boolean isInRequestContext() {
+    RequestContext ctx = instance.get();
+    if (ctx != null) {
+      return ctx.isInRequest();
+    }
+    return false;
+  }
+
+  /**
+   * Initializes the client credentials for the current request.
+   * @param user
+   * @param remoteAddress
+   * @param protocol
+   */
+  public static void set(User user,
+      InetAddress remoteAddress,
+      Class protocol) {
+    RequestContext ctx = instance.get();
+    ctx.user = user;
+    ctx.remoteAddress = remoteAddress;
+    ctx.protocol = protocol;
+    ctx.inRequest = true;
+  }
+
+  /**
+   * Clears out the client credentials for a given request.
+   */
+  public static void clear() {
+    RequestContext ctx = instance.get();
+    ctx.user = null;
+    ctx.remoteAddress = null;
+    ctx.protocol = null;
+    ctx.inRequest = false;
+  }
+
+  private User user;
+  private InetAddress remoteAddress;
+  private Class protocol;
+  // indicates we're within a RPC request invocation
+  private boolean inRequest;
+
+  private RequestContext(User user, InetAddress remoteAddr,
+      Class protocol) {
+    this.user = user;
+    this.remoteAddress = remoteAddr;
+    this.protocol = protocol;
+  }
+
+  public User getUser() {
+    return user;
+  }
+
+  public InetAddress getRemoteAddress() {
+    return remoteAddress;
+  }
+
+  public Class getProtocol() {
+    return protocol;
+  }
+
+  public boolean isInRequest() {
+    return inRequest;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/ResponseFlag.java b/src/main/java/org/apache/hadoop/hbase/ipc/ResponseFlag.java
new file mode 100644
index 0000000..88f5115
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/ResponseFlag.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+/**
+ * Utility for managing the flag byte passed in response to a
+ * {@link HBaseServer.Call}
+ */
+class ResponseFlag {
+  private static final byte ERROR_BIT = 0x1;
+  private static final byte LENGTH_BIT = 0x2;
+
+  private ResponseFlag() {
+    // Make it so this class cannot be constructed.
+  }
+
+  static boolean isError(final byte flag) {
+    return (flag & ERROR_BIT) != 0;
+  }
+
+  static boolean isLength(final byte flag) {
+    return (flag & LENGTH_BIT) != 0;
+  }
+
+  static byte getLengthSetOnly() {
+    return LENGTH_BIT;
+  }
+
+  static byte getErrorAndLengthSet() {
+    return LENGTH_BIT | ERROR_BIT;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/RpcCallContext.java b/src/main/java/org/apache/hadoop/hbase/ipc/RpcCallContext.java
new file mode 100644
index 0000000..60236d6
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/RpcCallContext.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+public interface RpcCallContext extends Delayable {
+
+  /**
+   * Throw an exception if the caller who made this IPC call has disconnected.
+   * If called from outside the context of IPC, this does nothing.
+   * @throws CallerDisconnectedException
+   */
+  void throwExceptionIfCallerDisconnected() throws CallerDisconnectedException;
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
new file mode 100644
index 0000000..faf725c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import java.lang.reflect.Method;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.conf.Configuration;
+
+/** An RPC implementation. */
+@InterfaceAudience.Private
+public interface RpcEngine extends Configurable {
+
+  /* Client-related methods */
+  /** Construct a client-side proxy object. */
+   T getProxy(Class protocol,
+                                           long clientVersion, InetSocketAddress addr,
+                                           Configuration conf, int rpcTimeout) throws IOException;
+
+  /** Shutdown this instance */
+  void close();
+
+  /** Expert: Make multiple, parallel calls to a set of servers. */
+  Object[] call(Method method, Object[][] params, InetSocketAddress[] addrs,
+                Class protocol,
+                User ticket, Configuration conf)
+    throws IOException, InterruptedException;
+
+  /* Server-related methods */
+  /** Construct a server for a protocol implementation instance. */
+  RpcServer getServer(Class protocol, Object instance,
+                       Class[] ifaces, String bindAddress,
+                       int port, int numHandlers, int metaHandlerCount,
+                       boolean verbose, Configuration conf, int highPriorityLevel)
+      throws IOException;
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
new file mode 100644
index 0000000..716fa22
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import com.google.common.base.Function;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.hbase.ipc.VersionedProtocol;
+import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+/**
+ */
+public interface RpcServer {
+
+  void setSocketSendBufSize(int size);
+
+  void start();
+
+  void stop();
+
+  void join() throws InterruptedException;
+
+  InetSocketAddress getListenerAddress();
+
+  /** Called for each call.
+   * @param param writable parameter
+   * @param receiveTime time
+   * @return Writable
+   * @throws java.io.IOException e
+   */
+  Writable call(Class protocol,
+      Writable param, long receiveTime, MonitoredRPCHandler status)
+      throws IOException;
+
+  void setErrorHandler(HBaseRPCErrorHandler handler);
+
+  void setQosFunction(Function newFunc);
+
+  void openServer();
+
+  void startThreads();
+
+
+  /**
+   * Returns the metrics instance for reporting RPC call statistics
+   */
+  HBaseRpcMetrics getRpcMetrics();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.java b/src/main/java/org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.java
new file mode 100644
index 0000000..7dd9b19
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+
+public class ServerNotRunningYetException extends IOException {
+  public ServerNotRunningYetException(String s) {
+    super(s);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/Status.java b/src/main/java/org/apache/hadoop/hbase/ipc/Status.java
new file mode 100644
index 0000000..c61282f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/Status.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+/**
+ * Status of a Hadoop IPC call.
+ */
+enum Status {
+  SUCCESS (0),
+  ERROR (1),
+  FATAL (-1);
+
+  int state;
+  private Status(int state) {
+    this.state = state;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/VersionedProtocol.java b/src/main/java/org/apache/hadoop/hbase/ipc/VersionedProtocol.java
new file mode 100644
index 0000000..9568b1b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/VersionedProtocol.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+
+/**
+ * Superclass of all protocols that use Hadoop RPC.
+ * Subclasses of this interface are also supposed to have
+ * a static final long versionID field.
+ */
+public interface VersionedProtocol {
+
+  /**
+   * Return protocol version corresponding to protocol interface.
+   * @param protocol The classname of the protocol interface
+   * @param clientVersion The version of the protocol that the client speaks
+   * @return the version that the server will speak
+   * @throws IOException if any IO error occurs
+   */
+  @Deprecated
+  public long getProtocolVersion(String protocol,
+                                 long clientVersion) throws IOException;
+
+  /**
+   * Return protocol version corresponding to protocol interface.
+   * @param protocol The classname of the protocol interface
+   * @param clientVersion The version of the protocol that the client speaks
+   * @param clientMethodsHash the hashcode of client protocol methods
+   * @return the server protocol signature containing its version and
+   *         a list of its supported methods
+   * @see ProtocolSignature#getProtocolSignature(VersionedProtocol, String,
+   *                long, int) for a default implementation
+   */
+  public ProtocolSignature getProtocolSignature(String protocol,
+                                 long clientVersion,
+                                 int clientMethodsHash) throws IOException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
new file mode 100644
index 0000000..a93fa49
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
@@ -0,0 +1,445 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import java.lang.reflect.Proxy;
+import java.lang.reflect.Method;
+import java.lang.reflect.Array;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+
+import java.net.InetSocketAddress;
+import java.io.*;
+import java.util.Map;
+import java.util.HashMap;
+
+import javax.net.SocketFactory;
+
+import org.apache.commons.logging.*;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.client.Operation;
+import org.apache.hadoop.hbase.io.HbaseObjectWritable;
+import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Objects;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.hbase.ipc.VersionedProtocol;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
+import org.apache.hadoop.conf.*;
+
+import org.codehaus.jackson.map.ObjectMapper;
+
+/** An RpcEngine implementation for Writable data. */
+class WritableRpcEngine implements RpcEngine {
+  // LOG is NOT in hbase subpackage intentionally so that the default HBase
+  // DEBUG log level does NOT emit RPC-level logging. 
+  private static final Log LOG = LogFactory.getLog("org.apache.hadoop.ipc.RPCEngine");
+
+  private static class Invoker implements InvocationHandler {
+    private Class protocol;
+    private InetSocketAddress address;
+    private User ticket;
+    private HBaseClient client;
+    final private int rpcTimeout;
+
+    public Invoker(HBaseClient client,
+                   Class protocol,
+                   InetSocketAddress address, User ticket,
+                   Configuration conf, int rpcTimeout) {
+      this.protocol = protocol;
+      this.address = address;
+      this.ticket = ticket;
+      this.client = client;
+      this.rpcTimeout = rpcTimeout;
+    }
+
+    public Object invoke(Object proxy, Method method, Object[] args)
+        throws Throwable {
+      final boolean logDebug = LOG.isDebugEnabled();
+      long startTime = 0;
+      if (logDebug) {
+        startTime = System.currentTimeMillis();
+      }
+
+      HbaseObjectWritable value = (HbaseObjectWritable)
+        client.call(new Invocation(method, protocol, args), address,
+                    protocol, ticket, rpcTimeout);
+      if (logDebug) {
+        // FIGURE HOW TO TURN THIS OFF!
+        long callTime = System.currentTimeMillis() - startTime;
+        LOG.debug("Call: " + method.getName() + " " + callTime);
+      }
+      return value.get();
+    }
+  }
+
+  private Configuration conf;
+  private HBaseClient client;
+
+  @Override
+  public void setConf(Configuration config) {
+    this.conf = config;
+    // check for an already created client
+    if (this.client != null) {
+      this.client.stop();
+    }
+    this.client = new HBaseClient(HbaseObjectWritable.class, conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  /** Construct a client-side proxy object that implements the named protocol,
+   * talking to a server at the named address. */
+  @Override
+  public  T getProxy(
+      Class protocol, long clientVersion,
+      InetSocketAddress addr, Configuration conf, int rpcTimeout)
+    throws IOException {
+    if (this.client == null) {
+      throw new IOException("Client must be initialized by calling setConf(Configuration)");
+    }
+
+    T proxy =
+          (T) Proxy.newProxyInstance(
+              protocol.getClassLoader(), new Class[] { protocol },
+              new Invoker(client, protocol, addr, User.getCurrent(), conf,
+                  HBaseRPC.getRpcTimeout(rpcTimeout)));
+
+    /*
+     * TODO: checking protocol version only needs to be done once when we setup a new
+     * HBaseClient.Connection.  Doing it every time we retrieve a proxy instance is resulting
+     * in unnecessary RPC traffic.
+     */
+    long serverVersion = ((VersionedProtocol)proxy)
+      .getProtocolVersion(protocol.getName(), clientVersion);
+    if (serverVersion != clientVersion) {
+      throw new HBaseRPC.VersionMismatch(protocol.getName(), clientVersion,
+                                    serverVersion);
+    }
+
+    return proxy;
+  }
+
+
+
+  /** Expert: Make multiple, parallel calls to a set of servers. */
+  @Override
+  public Object[] call(Method method, Object[][] params,
+                       InetSocketAddress[] addrs,
+                       Class protocol,
+                       User ticket, Configuration conf)
+    throws IOException, InterruptedException {
+    if (this.client == null) {
+      throw new IOException("Client must be initialized by calling setConf(Configuration)");
+    }
+
+    Invocation[] invocations = new Invocation[params.length];
+    for (int i = 0; i < params.length; i++) {
+      invocations[i] = new Invocation(method, protocol, params[i]);
+    }
+
+    Writable[] wrappedValues =
+        client.call(invocations, addrs, protocol, ticket);
+
+    if (method.getReturnType() == Void.TYPE) {
+      return null;
+    }
+
+    Object[] values =
+        (Object[])Array.newInstance(method.getReturnType(), wrappedValues.length);
+    for (int i = 0; i < values.length; i++) {
+      if (wrappedValues[i] != null) {
+        values[i] = ((HbaseObjectWritable)wrappedValues[i]).get();
+      }
+    }
+
+    return values;
+  }
+
+  @Override
+  public void close() {
+    if (this.client != null) {
+      this.client.stop();
+    }
+  }
+
+  /** Construct a server for a protocol implementation instance listening on a
+   * port and address. */
+  public Server getServer(Class protocol,
+                          Object instance,
+                          Class[] ifaces,
+                          String bindAddress, int port,
+                          int numHandlers,
+                          int metaHandlerCount, boolean verbose,
+                          Configuration conf, int highPriorityLevel)
+    throws IOException {
+    return new Server(instance, ifaces, conf, bindAddress, port, numHandlers,
+        metaHandlerCount, verbose, highPriorityLevel);
+  }
+
+  /** An RPC Server. */
+  public static class Server extends HBaseServer {
+    private Object instance;
+    private Class implementation;
+    private Class[] ifaces;
+    private boolean verbose;
+    private boolean authorize = false;
+
+    // for JSON encoding
+    private static ObjectMapper mapper = new ObjectMapper();
+
+    private static final String WARN_RESPONSE_TIME =
+      "hbase.ipc.warn.response.time";
+    private static final String WARN_RESPONSE_SIZE =
+      "hbase.ipc.warn.response.size";
+
+    /** Default value for above params */
+    private static final int DEFAULT_WARN_RESPONSE_TIME = 10000; // milliseconds
+    private static final int DEFAULT_WARN_RESPONSE_SIZE = 100 * 1024 * 1024;
+
+    /** Names for suffixed metrics */
+    private static final String ABOVE_ONE_SEC_METRIC = ".aboveOneSec.";
+
+    private final int warnResponseTime;
+    private final int warnResponseSize;
+
+    private static String classNameBase(String className) {
+      String[] names = className.split("\\.", -1);
+      if (names == null || names.length == 0) {
+        return className;
+      }
+      return names[names.length-1];
+    }
+
+    /** Construct an RPC server.
+     * @param instance the instance whose methods will be called
+     * @param conf the configuration to use
+     * @param bindAddress the address to bind on to listen for connection
+     * @param port the port to listen for connections on
+     * @param numHandlers the number of method handler threads to run
+     * @param verbose whether each call should be logged
+     * @throws IOException e
+     */
+    public Server(Object instance, final Class[] ifaces,
+                  Configuration conf, String bindAddress,  int port,
+                  int numHandlers, int metaHandlerCount, boolean verbose,
+                  int highPriorityLevel) throws IOException {
+      super(bindAddress, port, Invocation.class, numHandlers, metaHandlerCount,
+          conf, classNameBase(instance.getClass().getName()),
+          highPriorityLevel);
+      this.instance = instance;
+      this.implementation = instance.getClass();
+      this.verbose = verbose;
+
+      this.ifaces = ifaces;
+
+      // create metrics for the advertised interfaces this server implements.
+      String [] metricSuffixes = new String [] {ABOVE_ONE_SEC_METRIC};
+      this.rpcMetrics.createMetrics(this.ifaces, false, metricSuffixes);
+
+      this.authorize =
+        conf.getBoolean(
+            ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, false);
+
+      this.warnResponseTime = conf.getInt(WARN_RESPONSE_TIME,
+          DEFAULT_WARN_RESPONSE_TIME);
+      this.warnResponseSize = conf.getInt(WARN_RESPONSE_SIZE,
+          DEFAULT_WARN_RESPONSE_SIZE);
+    }
+
+    @Override
+    public Writable call(Class protocol,
+        Writable param, long receivedTime, MonitoredRPCHandler status)
+    throws IOException {
+      try {
+        Invocation call = (Invocation)param;
+        if(call.getMethodName() == null) {
+          throw new IOException("Could not find requested method, the usual " +
+              "cause is a version mismatch between client and server.");
+        }
+        if (verbose) log("Call: " + call);
+        status.setRPC(call.getMethodName(), call.getParameters(), receivedTime);
+        status.setRPCPacket(param);
+        status.resume("Servicing call");
+
+        Method method =
+          protocol.getMethod(call.getMethodName(),
+                                   call.getParameterClasses());
+        method.setAccessible(true);
+
+        //Verify protocol version.
+        //Bypass the version check for VersionedProtocol
+        if (!method.getDeclaringClass().equals(VersionedProtocol.class)) {
+          long clientVersion = call.getProtocolVersion();
+          ProtocolSignature serverInfo = ((VersionedProtocol) instance)
+              .getProtocolSignature(protocol.getCanonicalName(), call
+                  .getProtocolVersion(), call.getClientMethodsHash());
+          long serverVersion = serverInfo.getVersion();
+          if (serverVersion != clientVersion) {
+            LOG.warn("Version mismatch: client version=" + clientVersion
+                + ", server version=" + serverVersion);
+            throw new RPC.VersionMismatch(protocol.getName(), clientVersion,
+                serverVersion);
+          }
+        }
+        Object impl = null;
+        if (protocol.isAssignableFrom(this.implementation)) {
+          impl = this.instance;
+        }
+        else {
+          throw new HBaseRPC.UnknownProtocolException(protocol);
+        }
+
+        long startTime = System.currentTimeMillis();
+        Object[] params = call.getParameters();
+        Object value = method.invoke(impl, params);
+        int processingTime = (int) (System.currentTimeMillis() - startTime);
+        int qTime = (int) (startTime-receivedTime);
+        if (TRACELOG.isDebugEnabled()) {
+          TRACELOG.debug("Call #" + CurCall.get().id +
+              "; Served: " + protocol.getSimpleName()+"#"+call.getMethodName() +
+              " queueTime=" + qTime +
+              " processingTime=" + processingTime +
+              " contents=" + Objects.describeQuantity(params));
+        }
+        rpcMetrics.rpcQueueTime.inc(qTime);
+        rpcMetrics.rpcProcessingTime.inc(processingTime);
+        rpcMetrics.inc(call.getMethodName(), processingTime);
+        if (verbose) log("Return: "+value);
+
+        HbaseObjectWritable retVal =
+          new HbaseObjectWritable(method.getReturnType(), value);
+        long responseSize = retVal.getWritableSize();
+        // log any RPC responses that are slower than the configured warn
+        // response time or larger than configured warning size
+        boolean tooSlow = (processingTime > warnResponseTime
+            && warnResponseTime > -1);
+        boolean tooLarge = (responseSize > warnResponseSize
+            && warnResponseSize > -1);
+        if (tooSlow || tooLarge) {
+          // when tagging, we let TooLarge trump TooSmall to keep output simple
+          // note that large responses will often also be slow.
+          logResponse(call, (tooLarge ? "TooLarge" : "TooSlow"),
+              status.getClient(), startTime, processingTime, qTime,
+              responseSize);
+          // provides a count of log-reported slow responses
+          if (tooSlow) {
+            rpcMetrics.rpcSlowResponseTime.inc(processingTime);
+          }
+        }
+        if (processingTime > 1000) {
+          // we use a hard-coded one second period so that we can clearly
+          // indicate the time period we're warning about in the name of the 
+          // metric itself
+          rpcMetrics.inc(call.getMethodName() + ABOVE_ONE_SEC_METRIC,
+              processingTime);
+        }
+
+        return retVal;
+      } catch (InvocationTargetException e) {
+        Throwable target = e.getTargetException();
+        if (target instanceof IOException) {
+          throw (IOException)target;
+        }
+        IOException ioe = new IOException(target.toString());
+        ioe.setStackTrace(target.getStackTrace());
+        throw ioe;
+      } catch (Throwable e) {
+        if (!(e instanceof IOException)) {
+          LOG.error("Unexpected throwable object ", e);
+        }
+        IOException ioe = new IOException(e.toString());
+        ioe.setStackTrace(e.getStackTrace());
+        throw ioe;
+      }
+    }
+
+    /**
+     * Logs an RPC response to the LOG file, producing valid JSON objects for
+     * client Operations.
+     * @param call The call to log.
+     * @param tag  The tag that will be used to indicate this event in the log.
+     * @param clientAddress   The address of the client who made this call.
+     * @param startTime       The time that the call was initiated, in ms.
+     * @param processingTime  The duration that the call took to run, in ms.
+     * @param qTime           The duration that the call spent on the queue 
+     *                        prior to being initiated, in ms.
+     * @param responseSize    The size in bytes of the response buffer.
+     */
+    private void logResponse(Invocation call, String tag, String clientAddress,
+        long startTime, int processingTime, int qTime, long responseSize)
+      throws IOException {
+      Object params[] = call.getParameters();
+      // for JSON encoding
+      ObjectMapper mapper = new ObjectMapper();
+      // base information that is reported regardless of type of call
+      Map responseInfo = new HashMap();
+      responseInfo.put("starttimems", startTime);
+      responseInfo.put("processingtimems", processingTime);
+      responseInfo.put("queuetimems", qTime);
+      responseInfo.put("responsesize", responseSize);
+      responseInfo.put("client", clientAddress);
+      responseInfo.put("class", instance.getClass().getSimpleName());
+      responseInfo.put("method", call.getMethodName());
+      if (params.length == 2 && instance instanceof HRegionServer &&
+          params[0] instanceof byte[] &&
+          params[1] instanceof Operation) {
+        // if the slow process is a query, we want to log its table as well 
+        // as its own fingerprint
+        byte [] tableName =
+          HRegionInfo.parseRegionName((byte[]) params[0])[0];
+        responseInfo.put("table", Bytes.toStringBinary(tableName));
+        // annotate the response map with operation details
+        responseInfo.putAll(((Operation) params[1]).toMap());
+        // report to the log file
+        LOG.warn("(operation" + tag + "): " +
+            mapper.writeValueAsString(responseInfo));
+      } else if (params.length == 1 && instance instanceof HRegionServer &&
+          params[0] instanceof Operation) {
+        // annotate the response map with operation details
+        responseInfo.putAll(((Operation) params[0]).toMap());
+        // report to the log file
+        LOG.warn("(operation" + tag + "): " +
+            mapper.writeValueAsString(responseInfo));
+      } else {
+        // can't get JSON details, so just report call.toString() along with 
+        // a more generic tag.
+        responseInfo.put("call", call.toString());
+        LOG.warn("(response" + tag + "): " +
+            mapper.writeValueAsString(responseInfo));
+      }
+    }
+  }
+
+  protected static void log(String value) {
+    String v = value;
+    if (v != null && v.length() > 55)
+      v = v.substring(0, 55)+"...";
+    LOG.info(v);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/Driver.java b/src/main/java/org/apache/hadoop/hbase/mapred/Driver.java
new file mode 100644
index 0000000..d38956c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/Driver.java
@@ -0,0 +1,41 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import org.apache.hadoop.util.ProgramDriver;
+
+/**
+ * Driver for hbase mapreduce jobs. Select which to run by passing
+ * name of job to this main.
+ */
+@Deprecated
+public class Driver {
+  /**
+   * @param args
+   * @throws Throwable
+   */
+  public static void main(String[] args) throws Throwable {
+    ProgramDriver pgd = new ProgramDriver();
+    pgd.addClass(RowCounter.NAME, RowCounter.class,
+      "Count rows in HBase table");
+    ProgramDriver.class.getMethod("driver", new Class [] {String[].class}).
+      invoke(pgd, new Object[]{args});
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java b/src/main/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java
new file mode 100644
index 0000000..c368140
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java
@@ -0,0 +1,162 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+
+
+/**
+ * Extract grouping columns from input record
+ */
+@Deprecated
+public class GroupingTableMap
+extends MapReduceBase
+implements TableMap {
+
+  /**
+   * JobConf parameter to specify the columns used to produce the key passed to
+   * collect from the map phase
+   */
+  public static final String GROUP_COLUMNS =
+    "hbase.mapred.groupingtablemap.columns";
+
+  protected byte [][] columns;
+
+  /**
+   * Use this before submitting a TableMap job. It will appropriately set up the
+   * JobConf.
+   *
+   * @param table table to be processed
+   * @param columns space separated list of columns to fetch
+   * @param groupColumns space separated list of columns used to form the key
+   * used in collect
+   * @param mapper map class
+   * @param job job configuration object
+   */
+  @SuppressWarnings("unchecked")
+  public static void initJob(String table, String columns, String groupColumns,
+    Class mapper, JobConf job) {
+
+    TableMapReduceUtil.initTableMapJob(table, columns, mapper,
+        ImmutableBytesWritable.class, Result.class, job);
+    job.set(GROUP_COLUMNS, groupColumns);
+  }
+
+  @Override
+  public void configure(JobConf job) {
+    super.configure(job);
+    String[] cols = job.get(GROUP_COLUMNS, "").split(" ");
+    columns = new byte[cols.length][];
+    for(int i = 0; i < cols.length; i++) {
+      columns[i] = Bytes.toBytes(cols[i]);
+    }
+  }
+
+  /**
+   * Extract the grouping columns from value to construct a new key.
+   *
+   * Pass the new key and value to reduce.
+   * If any of the grouping columns are not found in the value, the record is skipped.
+   * @param key
+   * @param value
+   * @param output
+   * @param reporter
+   * @throws IOException
+   */
+  public void map(ImmutableBytesWritable key, Result value,
+      OutputCollector output,
+      Reporter reporter) throws IOException {
+
+    byte[][] keyVals = extractKeyValues(value);
+    if(keyVals != null) {
+      ImmutableBytesWritable tKey = createGroupKey(keyVals);
+      output.collect(tKey, value);
+    }
+  }
+
+  /**
+   * Extract columns values from the current record. This method returns
+   * null if any of the columns are not found.
+   *
+   * Override this method if you want to deal with nulls differently.
+   *
+   * @param r
+   * @return array of byte values
+   */
+  protected byte[][] extractKeyValues(Result r) {
+    byte[][] keyVals = null;
+    ArrayList foundList = new ArrayList();
+    int numCols = columns.length;
+    if (numCols > 0) {
+      for (KeyValue value: r.list()) {
+        byte [] column = KeyValue.makeColumn(value.getFamily(),
+            value.getQualifier());
+        for (int i = 0; i < numCols; i++) {
+          if (Bytes.equals(column, columns[i])) {
+            foundList.add(value.getValue());
+            break;
+          }
+        }
+      }
+      if(foundList.size() == numCols) {
+        keyVals = foundList.toArray(new byte[numCols][]);
+      }
+    }
+    return keyVals;
+  }
+
+  /**
+   * Create a key by concatenating multiple column values.
+   * Override this function in order to produce different types of keys.
+   *
+   * @param vals
+   * @return key generated by concatenating multiple column values
+   */
+  protected ImmutableBytesWritable createGroupKey(byte[][] vals) {
+    if(vals == null) {
+      return null;
+    }
+    StringBuilder sb =  new StringBuilder();
+    for(int i = 0; i < vals.length; i++) {
+      if(i > 0) {
+        sb.append(" ");
+      }
+      try {
+        sb.append(new String(vals[i], HConstants.UTF8_ENCODING));
+      } catch (UnsupportedEncodingException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    return new ImmutableBytesWritable(Bytes.toBytes(sb.toString()));
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java b/src/main/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java
new file mode 100644
index 0000000..b58c5c7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java
@@ -0,0 +1,91 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Partitioner;
+
+
+/**
+ * This is used to partition the output keys into groups of keys.
+ * Keys are grouped according to the regions that currently exist
+ * so that each reducer fills a single region so load is distributed.
+ *
+ * @param 
+ * @param 
+ */
+@Deprecated
+public class HRegionPartitioner
+implements Partitioner {
+  private final Log LOG = LogFactory.getLog(TableInputFormat.class);
+  private HTable table;
+  private byte[][] startKeys;
+
+  public void configure(JobConf job) {
+    try {
+      this.table = new HTable(HBaseConfiguration.create(job),
+        job.get(TableOutputFormat.OUTPUT_TABLE));
+    } catch (IOException e) {
+      LOG.error(e);
+    }
+
+    try {
+      this.startKeys = this.table.getStartKeys();
+    } catch (IOException e) {
+      LOG.error(e);
+    }
+  }
+
+  public int getPartition(ImmutableBytesWritable key,
+      V2 value, int numPartitions) {
+    byte[] region = null;
+    // Only one region return 0
+    if (this.startKeys.length == 1){
+      return 0;
+    }
+    try {
+      // Not sure if this is cached after a split so we could have problems
+      // here if a region splits while mapping
+      region = table.getRegionLocation(key.get()).getRegionInfo().getStartKey();
+    } catch (IOException e) {
+      LOG.error(e);
+    }
+    for (int i = 0; i < this.startKeys.length; i++){
+      if (Bytes.compareTo(region, this.startKeys[i]) == 0 ){
+        if (i >= numPartitions-1){
+          // cover if we have less reduces then regions.
+          return (Integer.toString(i).hashCode()
+              & Integer.MAX_VALUE) % numPartitions;
+        }
+        return i;
+      }
+    }
+    // if above fails to find start key that match we need to return something
+    return 0;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableMap.java b/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableMap.java
new file mode 100644
index 0000000..0f67a9e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableMap.java
@@ -0,0 +1,76 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * Pass the given key and record as-is to reduce
+ */
+@Deprecated
+public class IdentityTableMap
+extends MapReduceBase
+implements TableMap {
+
+  /** constructor */
+  public IdentityTableMap() {
+    super();
+  }
+
+  /**
+   * Use this before submitting a TableMap job. It will
+   * appropriately set up the JobConf.
+   *
+   * @param table table name
+   * @param columns columns to scan
+   * @param mapper mapper class
+   * @param job job configuration
+   */
+  @SuppressWarnings("unchecked")
+  public static void initJob(String table, String columns,
+    Class mapper, JobConf job) {
+    TableMapReduceUtil.initTableMapJob(table, columns, mapper,
+      ImmutableBytesWritable.class,
+      Result.class, job);
+  }
+
+  /**
+   * Pass the key, value to reduce
+   * @param key
+   * @param value
+   * @param output
+   * @param reporter
+   * @throws IOException
+   */
+  public void map(ImmutableBytesWritable key, Result value,
+      OutputCollector output,
+      Reporter reporter) throws IOException {
+
+    // convert
+    output.collect(key, value);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java b/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java
new file mode 100644
index 0000000..be0a6bd
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java
@@ -0,0 +1,61 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * Write to table each key, record pair
+ */
+@Deprecated
+public class IdentityTableReduce
+extends MapReduceBase
+implements TableReduce {
+  @SuppressWarnings("unused")
+  private static final Log LOG =
+    LogFactory.getLog(IdentityTableReduce.class.getName());
+
+  /**
+   * No aggregation, output pairs of (key, record)
+   * @param key
+   * @param values
+   * @param output
+   * @param reporter
+   * @throws IOException
+   */
+  public void reduce(ImmutableBytesWritable key, Iterator values,
+      OutputCollector output,
+      Reporter reporter)
+      throws IOException {
+
+    while(values.hasNext()) {
+      output.collect(key, values.next());
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/RowCounter.java b/src/main/java/org/apache/hadoop/hbase/mapred/RowCounter.java
new file mode 100644
index 0000000..ac50bd7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/RowCounter.java
@@ -0,0 +1,124 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.lib.IdentityReducer;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * A job with a map to count rows.
+ * Map outputs table rows IF the input row has columns that have content.
+ * Uses an {@link IdentityReducer}
+ */
+@Deprecated
+public class RowCounter extends Configured implements Tool {
+  // Name of this 'program'
+  static final String NAME = "rowcounter";
+
+  /**
+   * Mapper that runs the count.
+   */
+  static class RowCounterMapper
+  implements TableMap {
+    private static enum Counters {ROWS}
+
+    public void map(ImmutableBytesWritable row, Result values,
+        OutputCollector output,
+        Reporter reporter)
+    throws IOException {
+        // Count every row containing data, whether it's in qualifiers or values
+        reporter.incrCounter(Counters.ROWS, 1);
+    }
+
+    public void configure(JobConf jc) {
+      // Nothing to do.
+    }
+
+    public void close() throws IOException {
+      // Nothing to do.
+    }
+  }
+
+  /**
+   * @param args
+   * @return the JobConf
+   * @throws IOException
+   */
+  public JobConf createSubmittableJob(String[] args) throws IOException {
+    JobConf c = new JobConf(getConf(), getClass());
+    c.setJobName(NAME);
+    // Columns are space delimited
+    StringBuilder sb = new StringBuilder();
+    final int columnoffset = 2;
+    for (int i = columnoffset; i < args.length; i++) {
+      if (i > columnoffset) {
+        sb.append(" ");
+      }
+      sb.append(args[i]);
+    }
+    // Second argument is the table name.
+    TableMapReduceUtil.initTableMapJob(args[1], sb.toString(),
+      RowCounterMapper.class, ImmutableBytesWritable.class, Result.class, c);
+    c.setNumReduceTasks(0);
+    // First arg is the output directory.
+    FileOutputFormat.setOutputPath(c, new Path(args[0]));
+    return c;
+  }
+
+  static int printUsage() {
+    System.out.println(NAME +
+      "    [...]");
+    return -1;
+  }
+
+  public int run(final String[] args) throws Exception {
+    // Make sure there are at least 3 parameters
+    if (args.length < 3) {
+      System.err.println("ERROR: Wrong number of parameters: " + args.length);
+      return printUsage();
+    }
+    JobClient.runJob(createSubmittableJob(args));
+    return 0;
+  }
+
+  /**
+   * @param args
+   * @throws Exception
+   */
+  public static void main(String[] args) throws Exception {
+    int errCode = ToolRunner.run(HBaseConfiguration.create(), new RowCounter(), args);
+    System.exit(errCode);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormat.java
new file mode 100644
index 0000000..395a626
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormat.java
@@ -0,0 +1,83 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobConfigurable;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Convert HBase tabular data into a format that is consumable by Map/Reduce.
+ */
+@Deprecated
+public class TableInputFormat extends TableInputFormatBase implements
+    JobConfigurable {
+  private final Log LOG = LogFactory.getLog(TableInputFormat.class);
+
+  /**
+   * space delimited list of columns
+   */
+  public static final String COLUMN_LIST = "hbase.mapred.tablecolumns";
+
+  public void configure(JobConf job) {
+    Path[] tableNames = FileInputFormat.getInputPaths(job);
+    String colArg = job.get(COLUMN_LIST);
+    String[] colNames = colArg.split(" ");
+    byte [][] m_cols = new byte[colNames.length][];
+    for (int i = 0; i < m_cols.length; i++) {
+      m_cols[i] = Bytes.toBytes(colNames[i]);
+    }
+    setInputColumns(m_cols);
+    try {
+      setHTable(new HTable(HBaseConfiguration.create(job), tableNames[0].getName()));
+    } catch (Exception e) {
+      LOG.error(StringUtils.stringifyException(e));
+    }
+  }
+
+  public void validateInput(JobConf job) throws IOException {
+    // expecting exactly one path
+    Path [] tableNames = FileInputFormat.getInputPaths(job);
+    if (tableNames == null || tableNames.length > 1) {
+      throw new IOException("expecting one table name");
+    }
+
+    // connected to table?
+    if (getHTable() == null) {
+      throw new IOException("could not connect to table '" +
+        tableNames[0].getName() + "'");
+    }
+
+    // expecting at least one column
+    String colArg = job.get(COLUMN_LIST);
+    if (colArg == null || colArg.length() == 0) {
+      throw new IOException("expecting at least one column");
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
new file mode 100644
index 0000000..a959e5d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
@@ -0,0 +1,189 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * A Base for {@link TableInputFormat}s. Receives a {@link HTable}, a
+ * byte[] of input columns and optionally a {@link Filter}.
+ * Subclasses may use other TableRecordReader implementations.
+ * 
      
+ * An example of a subclass:
+ * 
      + *   class ExampleTIF extends TableInputFormatBase implements JobConfigurable {
+ *
+ *     public void configure(JobConf job) {
+ *       HTable exampleTable = new HTable(HBaseConfiguration.create(job),
+ *         Bytes.toBytes("exampleTable"));
+ *       // mandatory
+ *       setHTable(exampleTable);
+ *       Text[] inputColumns = new byte [][] { Bytes.toBytes("columnA"),
+ *         Bytes.toBytes("columnB") };
+ *       // mandatory
+ *       setInputColumns(inputColumns);
+ *       RowFilterInterface exampleFilter = new RegExpRowFilter("keyPrefix.*");
+ *       // optional
+ *       setRowFilter(exampleFilter);
+ *     }
+ *
+ *     public void validateInput(JobConf job) throws IOException {
+ *     }
+ *  }
+ * 
      
+ */
+
+@Deprecated
+public abstract class TableInputFormatBase
+implements InputFormat {
+  final Log LOG = LogFactory.getLog(TableInputFormatBase.class);
+  private byte [][] inputColumns;
+  private HTable table;
+  private TableRecordReader tableRecordReader;
+  private Filter rowFilter;
+
+  /**
+   * Builds a TableRecordReader. If no TableRecordReader was provided, uses
+   * the default.
+   *
+   * @see org.apache.hadoop.mapred.InputFormat#getRecordReader(InputSplit,
+   *      JobConf, Reporter)
+   */
+  public RecordReader getRecordReader(
+      InputSplit split, JobConf job, Reporter reporter)
+  throws IOException {
+    TableSplit tSplit = (TableSplit) split;
+    TableRecordReader trr = this.tableRecordReader;
+    // if no table record reader was provided use default
+    if (trr == null) {
+      trr = new TableRecordReader();
+    }
+    trr.setStartRow(tSplit.getStartRow());
+    trr.setEndRow(tSplit.getEndRow());
+    trr.setHTable(this.table);
+    trr.setInputColumns(this.inputColumns);
+    trr.setRowFilter(this.rowFilter);
+    trr.init();
+    return trr;
+  }
+
+  /**
+   * Calculates the splits that will serve as input for the map tasks.
+   * 
        
+   * Splits are created in number equal to the smallest between numSplits and
+   * the number of {@link HRegion}s in the table. If the number of splits is
+   * smaller than the number of {@link HRegion}s then splits are spanned across
+   * multiple {@link HRegion}s and are grouped the most evenly possible. In the
+   * case splits are uneven the bigger splits are placed first in the
+   * {@link InputSplit} array.
+   *
+   * @param job the map task {@link JobConf}
+   * @param numSplits a hint to calculate the number of splits (mapred.map.tasks).
+   *
+   * @return the input splits
+   *
+   * @see org.apache.hadoop.mapred.InputFormat#getSplits(org.apache.hadoop.mapred.JobConf, int)
+   */
+  public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
+    if (this.table == null) {
+      throw new IOException("No table was provided");
+    }
+    byte [][] startKeys = this.table.getStartKeys();
+    if (startKeys == null || startKeys.length == 0) {
+      throw new IOException("Expecting at least one region");
+    }
+    if (this.inputColumns == null || this.inputColumns.length == 0) {
+      throw new IOException("Expecting at least one column");
+    }
+    int realNumSplits = numSplits > startKeys.length? startKeys.length:
+      numSplits;
+    InputSplit[] splits = new InputSplit[realNumSplits];
+    int middle = startKeys.length / realNumSplits;
+    int startPos = 0;
+    for (int i = 0; i < realNumSplits; i++) {
+      int lastPos = startPos + middle;
+      lastPos = startKeys.length % realNumSplits > i ? lastPos + 1 : lastPos;
+      String regionLocation = table.getRegionLocation(startKeys[startPos]).
+        getHostname();
+      splits[i] = new TableSplit(this.table.getTableName(),
+        startKeys[startPos], ((i + 1) < realNumSplits) ? startKeys[lastPos]:
+          HConstants.EMPTY_START_ROW, regionLocation);
+      LOG.info("split: " + i + "->" + splits[i]);
+      startPos = lastPos;
+    }
+    return splits;
+  }
+
+  /**
+   * @param inputColumns to be passed in {@link Result} to the map task.
+   */
+  protected void setInputColumns(byte [][] inputColumns) {
+    this.inputColumns = inputColumns;
+  }
+
+  /**
+   * Allows subclasses to get the {@link HTable}.
+   */
+  protected HTable getHTable() {
+    return this.table;
+  }
+
+  /**
+   * Allows subclasses to set the {@link HTable}.
+   *
+   * @param table to get the data from
+   */
+  protected void setHTable(HTable table) {
+    this.table = table;
+  }
+
+  /**
+   * Allows subclasses to set the {@link TableRecordReader}.
+   *
+   * @param tableRecordReader
+   *                to provide other {@link TableRecordReader} implementations.
+   */
+  protected void setTableRecordReader(TableRecordReader tableRecordReader) {
+    this.tableRecordReader = tableRecordReader;
+  }
+
+  /**
+   * Allows subclasses to set the {@link Filter} to be used.
+   *
+   * @param rowFilter
+   */
+  protected void setRowFilter(Filter rowFilter) {
+    this.rowFilter = rowFilter;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableMap.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableMap.java
new file mode 100644
index 0000000..597f3ef
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableMap.java
@@ -0,0 +1,39 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.Mapper;
+
+/**
+ * Scan an HBase table to sort by a specified sort column.
+ * If the column does not exist, the record is not passed to Reduce.
+ *
+ * @param  WritableComparable key class
+ * @param  Writable value class
+ */
+@Deprecated
+public interface TableMap, V extends Writable>
+extends Mapper {
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
new file mode 100644
index 0000000..bd09879
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
@@ -0,0 +1,275 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
+
+/**
+ * Utility for {@link TableMap} and {@link TableReduce}
+ */
+@Deprecated
+@SuppressWarnings("unchecked")
+public class TableMapReduceUtil {
+
+  /**
+   * Use this before submitting a TableMap job. It will
+   * appropriately set up the JobConf.
+   *
+   * @param table  The table name to read from.
+   * @param columns  The columns to scan.
+   * @param mapper  The mapper class to use.
+   * @param outputKeyClass  The class of the output key.
+   * @param outputValueClass  The class of the output value.
+   * @param job  The current job configuration to adjust.
+   */
+  public static void initTableMapJob(String table, String columns,
+    Class mapper,
+    Class outputKeyClass,
+    Class outputValueClass, JobConf job) {
+    initTableMapJob(table, columns, mapper, outputKeyClass, outputValueClass, job, true);
+  }
+
+  /**
+   * Use this before submitting a TableMap job. It will
+   * appropriately set up the JobConf.
+   *
+   * @param table  The table name to read from.
+   * @param columns  The columns to scan.
+   * @param mapper  The mapper class to use.
+   * @param outputKeyClass  The class of the output key.
+   * @param outputValueClass  The class of the output value.
+   * @param job  The current job configuration to adjust.
+   * @param addDependencyJars upload HBase jars and jars for any of the configured
+   *           job classes via the distributed cache (tmpjars).
+   */
+  public static void initTableMapJob(String table, String columns,
+    Class mapper,
+    Class outputKeyClass,
+    Class outputValueClass, JobConf job, boolean addDependencyJars) {
+
+    job.setInputFormat(TableInputFormat.class);
+    job.setMapOutputValueClass(outputValueClass);
+    job.setMapOutputKeyClass(outputKeyClass);
+    job.setMapperClass(mapper);
+    FileInputFormat.addInputPaths(job, table);
+    job.set(TableInputFormat.COLUMN_LIST, columns);
+    if (addDependencyJars) {
+      try {
+        addDependencyJars(job);
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
+    }
+    try {
+      initCredentials(job);
+    } catch (IOException ioe) {
+      // just spit out the stack trace?  really?
+      ioe.printStackTrace();
+    }
+  }
+
+  /**
+   * Use this before submitting a TableReduce job. It will
+   * appropriately set up the JobConf.
+   *
+   * @param table  The output table.
+   * @param reducer  The reducer class to use.
+   * @param job  The current job configuration to adjust.
+   * @throws IOException When determining the region count fails.
+   */
+  public static void initTableReduceJob(String table,
+    Class reducer, JobConf job)
+  throws IOException {
+    initTableReduceJob(table, reducer, job, null);
+  }
+
+  /**
+   * Use this before submitting a TableReduce job. It will
+   * appropriately set up the JobConf.
+   *
+   * @param table  The output table.
+   * @param reducer  The reducer class to use.
+   * @param job  The current job configuration to adjust.
+   * @param partitioner  Partitioner to use. Pass null to use
+   * default partitioner.
+   * @throws IOException When determining the region count fails.
+   */
+  public static void initTableReduceJob(String table,
+    Class reducer, JobConf job, Class partitioner)
+  throws IOException {
+    initTableReduceJob(table, reducer, job, partitioner, true);
+  }
+
+  /**
+   * Use this before submitting a TableReduce job. It will
+   * appropriately set up the JobConf.
+   *
+   * @param table  The output table.
+   * @param reducer  The reducer class to use.
+   * @param job  The current job configuration to adjust.
+   * @param partitioner  Partitioner to use. Pass null to use
+   * default partitioner.
+   * @param addDependencyJars upload HBase jars and jars for any of the configured
+   *           job classes via the distributed cache (tmpjars).
+   * @throws IOException When determining the region count fails.
+   */
+  public static void initTableReduceJob(String table,
+    Class reducer, JobConf job, Class partitioner,
+    boolean addDependencyJars) throws IOException {
+    job.setOutputFormat(TableOutputFormat.class);
+    job.setReducerClass(reducer);
+    job.set(TableOutputFormat.OUTPUT_TABLE, table);
+    job.setOutputKeyClass(ImmutableBytesWritable.class);
+    job.setOutputValueClass(Put.class);
+    if (partitioner == HRegionPartitioner.class) {
+      job.setPartitionerClass(HRegionPartitioner.class);
+      HTable outputTable = new HTable(HBaseConfiguration.create(job), table);
+      int regions = outputTable.getRegionsInfo().size();
+      if (job.getNumReduceTasks() > regions) {
+        job.setNumReduceTasks(outputTable.getRegionsInfo().size());
+      }
+    } else if (partitioner != null) {
+      job.setPartitionerClass(partitioner);
+    }
+    if (addDependencyJars) {
+      addDependencyJars(job);
+    }
+    initCredentials(job);
+  }
+
+  public static void initCredentials(JobConf job) throws IOException {
+    if (User.isHBaseSecurityEnabled(job)) {
+      try {
+        User.getCurrent().obtainAuthTokenForJob(job);
+      } catch (InterruptedException ie) {
+        ie.printStackTrace();
+        Thread.interrupted();
+      }
+    }
+  }
+
+  /**
+   * Ensures that the given number of reduce tasks for the given job
+   * configuration does not exceed the number of regions for the given table.
+   *
+   * @param table  The table to get the region count for.
+   * @param job  The current job configuration to adjust.
+   * @throws IOException When retrieving the table details fails.
+   */
+  public static void limitNumReduceTasks(String table, JobConf job)
+  throws IOException {
+    HTable outputTable = new HTable(HBaseConfiguration.create(job), table);
+    int regions = outputTable.getRegionsInfo().size();
+    if (job.getNumReduceTasks() > regions)
+      job.setNumReduceTasks(regions);
+  }
+
+  /**
+   * Ensures that the given number of map tasks for the given job
+   * configuration does not exceed the number of regions for the given table.
+   *
+   * @param table  The table to get the region count for.
+   * @param job  The current job configuration to adjust.
+   * @throws IOException When retrieving the table details fails.
+   */
+  public static void limitNumMapTasks(String table, JobConf job)
+  throws IOException {
+    HTable outputTable = new HTable(HBaseConfiguration.create(job), table);
+    int regions = outputTable.getRegionsInfo().size();
+    if (job.getNumMapTasks() > regions)
+      job.setNumMapTasks(regions);
+  }
+
+  /**
+   * Sets the number of reduce tasks for the given job configuration to the
+   * number of regions the given table has.
+   *
+   * @param table  The table to get the region count for.
+   * @param job  The current job configuration to adjust.
+   * @throws IOException When retrieving the table details fails.
+   */
+  public static void setNumReduceTasks(String table, JobConf job)
+  throws IOException {
+    HTable outputTable = new HTable(HBaseConfiguration.create(job), table);
+    int regions = outputTable.getRegionsInfo().size();
+    job.setNumReduceTasks(regions);
+  }
+
+  /**
+   * Sets the number of map tasks for the given job configuration to the
+   * number of regions the given table has.
+   *
+   * @param table  The table to get the region count for.
+   * @param job  The current job configuration to adjust.
+   * @throws IOException When retrieving the table details fails.
+   */
+  public static void setNumMapTasks(String table, JobConf job)
+  throws IOException {
+    HTable outputTable = new HTable(HBaseConfiguration.create(job), table);
+    int regions = outputTable.getRegionsInfo().size();
+    job.setNumMapTasks(regions);
+  }
+
+  /**
+   * Sets the number of rows to return and cache with each scanner iteration.
+   * Higher caching values will enable faster mapreduce jobs at the expense of
+   * requiring more heap to contain the cached rows.
+   *
+   * @param job The current job configuration to adjust.
+   * @param batchSize The number of rows to return in batch with each scanner
+   * iteration.
+   */
+  public static void setScannerCaching(JobConf job, int batchSize) {
+    job.setInt("hbase.client.scanner.caching", batchSize);
+  }
+
+  /**
+   * @see org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil#addDependencyJars(Job)
+   */
+  public static void addDependencyJars(JobConf job) throws IOException {
+    org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(
+      job,
+      org.apache.zookeeper.ZooKeeper.class,
+      com.google.common.base.Function.class,
+      com.google.protobuf.Message.class,
+      job.getMapOutputKeyClass(),
+      job.getMapOutputValueClass(),
+      job.getOutputKeyClass(),
+      job.getOutputValueClass(),
+      job.getPartitionerClass(),
+      job.getClass("mapred.input.format.class", TextInputFormat.class, InputFormat.class),
+      job.getClass("mapred.output.format.class", TextOutputFormat.class, OutputFormat.class),
+      job.getCombinerClass());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java
new file mode 100644
index 0000000..732de97
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java
@@ -0,0 +1,106 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.mapred.FileAlreadyExistsException;
+import org.apache.hadoop.mapred.InvalidJobConfException;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.Progressable;
+
+/**
+ * Convert Map/Reduce output and write it to an HBase table
+ */
+@Deprecated
+public class TableOutputFormat extends
+FileOutputFormat {
+
+  /** JobConf parameter that specifies the output table */
+  public static final String OUTPUT_TABLE = "hbase.mapred.outputtable";
+  private final Log LOG = LogFactory.getLog(TableOutputFormat.class);
+
+  /**
+   * Convert Reduce output (key, value) to (HStoreKey, KeyedDataArrayWritable)
+   * and write to an HBase table
+   */
+  protected static class TableRecordWriter
+    implements RecordWriter {
+    private HTable m_table;
+
+    /**
+     * Instantiate a TableRecordWriter with the HBase HClient for writing.
+     *
+     * @param table
+     */
+    public TableRecordWriter(HTable table) {
+      m_table = table;
+    }
+
+    public void close(Reporter reporter)
+      throws IOException {
+      m_table.close();
+    }
+
+    public void write(ImmutableBytesWritable key,
+        Put value) throws IOException {
+      m_table.put(new Put(value));
+    }
+  }
+
+  @Override
+  @SuppressWarnings("unchecked")
+  public RecordWriter getRecordWriter(FileSystem ignored,
+      JobConf job, String name, Progressable progress) throws IOException {
+
+    // expecting exactly one path
+
+    String tableName = job.get(OUTPUT_TABLE);
+    HTable table = null;
+    try {
+      table = new HTable(HBaseConfiguration.create(job), tableName);
+    } catch(IOException e) {
+      LOG.error(e);
+      throw e;
+    }
+    table.setAutoFlush(false);
+    return new TableRecordWriter(table);
+  }
+
+  @Override
+  public void checkOutputSpecs(FileSystem ignored, JobConf job)
+  throws FileAlreadyExistsException, InvalidJobConfException, IOException {
+
+    String tableName = job.get(OUTPUT_TABLE);
+    if(tableName == null) {
+      throw new IOException("Must specify table name");
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReader.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReader.java
new file mode 100644
index 0000000..7133860
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReader.java
@@ -0,0 +1,138 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.mapred.RecordReader;
+
+
+/**
+ * Iterate over an HBase table data, return (Text, RowResult) pairs
+ */
+public class TableRecordReader
+implements RecordReader {
+
+  private TableRecordReaderImpl recordReaderImpl = new TableRecordReaderImpl();
+
+  /**
+   * Restart from survivable exceptions by creating a new scanner.
+   *
+   * @param firstRow
+   * @throws IOException
+   */
+  public void restart(byte[] firstRow) throws IOException {
+    this.recordReaderImpl.restart(firstRow);
+  }
+
+  /**
+   * Build the scanner. Not done in constructor to allow for extension.
+   *
+   * @throws IOException
+   */
+  public void init() throws IOException {
+    this.recordReaderImpl.restart(this.recordReaderImpl.getStartRow());
+  }
+
+  /**
+   * @param htable the {@link HTable} to scan.
+   */
+  public void setHTable(HTable htable) {
+    this.recordReaderImpl.setHTable(htable);
+  }
+
+  /**
+   * @param inputColumns the columns to be placed in {@link Result}.
+   */
+  public void setInputColumns(final byte [][] inputColumns) {
+    this.recordReaderImpl.setInputColumns(inputColumns);
+  }
+
+  /**
+   * @param startRow the first row in the split
+   */
+  public void setStartRow(final byte [] startRow) {
+    this.recordReaderImpl.setStartRow(startRow);
+  }
+
+  /**
+   *
+   * @param endRow the last row in the split
+   */
+  public void setEndRow(final byte [] endRow) {
+    this.recordReaderImpl.setEndRow(endRow);
+  }
+
+  /**
+   * @param rowFilter the {@link Filter} to be used.
+   */
+  public void setRowFilter(Filter rowFilter) {
+    this.recordReaderImpl.setRowFilter(rowFilter);
+  }
+
+  public void close() {
+    this.recordReaderImpl.close();
+  }
+
+  /**
+   * @return ImmutableBytesWritable
+   *
+   * @see org.apache.hadoop.mapred.RecordReader#createKey()
+   */
+  public ImmutableBytesWritable createKey() {
+    return this.recordReaderImpl.createKey();
+  }
+
+  /**
+   * @return RowResult
+   *
+   * @see org.apache.hadoop.mapred.RecordReader#createValue()
+   */
+  public Result createValue() {
+    return this.recordReaderImpl.createValue();
+  }
+
+  public long getPos() {
+
+    // This should be the ordinal tuple in the range;
+    // not clear how to calculate...
+    return this.recordReaderImpl.getPos();
+  }
+
+  public float getProgress() {
+    // Depends on the total number of tuples and getPos
+    return this.recordReaderImpl.getPos();
+  }
+
+  /**
+   * @param key HStoreKey as input key.
+   * @param value MapWritable as input value
+   * @return true if there was more data
+   * @throws IOException
+   */
+  public boolean next(ImmutableBytesWritable key, Result value)
+  throws IOException {
+    return this.recordReaderImpl.next(key, value);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
new file mode 100644
index 0000000..72e5774
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
@@ -0,0 +1,247 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.ScannerCallable;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.util.StringUtils;
+
+import static org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl.LOG_PER_ROW_COUNT;
+
+/**
+ * Iterate over an HBase table data, return (Text, RowResult) pairs
+ */
+public class TableRecordReaderImpl {
+  static final Log LOG = LogFactory.getLog(TableRecordReaderImpl.class);
+
+  private byte [] startRow;
+  private byte [] endRow;
+  private byte [] lastSuccessfulRow;
+  private Filter trrRowFilter;
+  private ResultScanner scanner;
+  private HTable htable;
+  private byte [][] trrInputColumns;
+  private long timestamp;
+  private int rowcount;
+  private boolean logScannerActivity = false;
+  private int logPerRowCount = 100;
+
+  /**
+   * Restart from survivable exceptions by creating a new scanner.
+   *
+   * @param firstRow
+   * @throws IOException
+   */
+  public void restart(byte[] firstRow) throws IOException {
+    Scan currentScan;
+    if ((endRow != null) && (endRow.length > 0)) {
+      if (trrRowFilter != null) {
+        Scan scan = new Scan(firstRow, endRow);
+        TableInputFormat.addColumns(scan, trrInputColumns);
+        scan.setFilter(trrRowFilter);
+        scan.setCacheBlocks(false);
+        this.scanner = this.htable.getScanner(scan);
+        currentScan = scan;
+      } else {
+        LOG.debug("TIFB.restart, firstRow: " +
+            Bytes.toStringBinary(firstRow) + ", endRow: " +
+            Bytes.toStringBinary(endRow));
+        Scan scan = new Scan(firstRow, endRow);
+        TableInputFormat.addColumns(scan, trrInputColumns);
+        this.scanner = this.htable.getScanner(scan);
+        currentScan = scan;
+      }
+    } else {
+      LOG.debug("TIFB.restart, firstRow: " +
+          Bytes.toStringBinary(firstRow) + ", no endRow");
+
+      Scan scan = new Scan(firstRow);
+      TableInputFormat.addColumns(scan, trrInputColumns);
+      scan.setFilter(trrRowFilter);
+      this.scanner = this.htable.getScanner(scan);
+      currentScan = scan;
+    }
+    if (logScannerActivity) {
+      LOG.info("Current scan=" + currentScan.toString());
+      timestamp = System.currentTimeMillis();
+      rowcount = 0;
+    }
+  }
+
+  /**
+   * Build the scanner. Not done in constructor to allow for extension.
+   *
+   * @throws IOException
+   */
+  public void init() throws IOException {
+    restart(startRow);
+  }
+
+  byte[] getStartRow() {
+    return this.startRow;
+  }
+  /**
+   * @param htable the {@link HTable} to scan.
+   */
+  public void setHTable(HTable htable) {
+    Configuration conf = htable.getConfiguration();
+    logScannerActivity = conf.getBoolean(
+      ScannerCallable.LOG_SCANNER_ACTIVITY, false);
+    logPerRowCount = conf.getInt(LOG_PER_ROW_COUNT, 100);
+    this.htable = htable;
+  }
+
+  /**
+   * @param inputColumns the columns to be placed in {@link Result}.
+   */
+  public void setInputColumns(final byte [][] inputColumns) {
+    this.trrInputColumns = inputColumns;
+  }
+
+  /**
+   * @param startRow the first row in the split
+   */
+  public void setStartRow(final byte [] startRow) {
+    this.startRow = startRow;
+  }
+
+  /**
+   *
+   * @param endRow the last row in the split
+   */
+  public void setEndRow(final byte [] endRow) {
+    this.endRow = endRow;
+  }
+
+  /**
+   * @param rowFilter the {@link Filter} to be used.
+   */
+  public void setRowFilter(Filter rowFilter) {
+    this.trrRowFilter = rowFilter;
+  }
+
+  public void close() {
+    this.scanner.close();
+  }
+
+  /**
+   * @return ImmutableBytesWritable
+   *
+   * @see org.apache.hadoop.mapred.RecordReader#createKey()
+   */
+  public ImmutableBytesWritable createKey() {
+    return new ImmutableBytesWritable();
+  }
+
+  /**
+   * @return RowResult
+   *
+   * @see org.apache.hadoop.mapred.RecordReader#createValue()
+   */
+  public Result createValue() {
+    return new Result();
+  }
+
+  public long getPos() {
+    // This should be the ordinal tuple in the range;
+    // not clear how to calculate...
+    return 0;
+  }
+
+  public float getProgress() {
+    // Depends on the total number of tuples and getPos
+    return 0;
+  }
+
+  /**
+   * @param key HStoreKey as input key.
+   * @param value MapWritable as input value
+   * @return true if there was more data
+   * @throws IOException
+   */
+  public boolean next(ImmutableBytesWritable key, Result value)
+  throws IOException {
+    Result result;
+    try {
+      try {
+        result = this.scanner.next();
+        if (logScannerActivity) {
+          rowcount ++;
+          if (rowcount >= logPerRowCount) {
+            long now = System.currentTimeMillis();
+            LOG.info("Mapper took " + (now-timestamp)
+              + "ms to process " + rowcount + " rows");
+            timestamp = now;
+            rowcount = 0;
+          }
+        }
+      } catch (IOException e) {
+        // try to handle all IOExceptions by restarting
+        // the scanner, if the second call fails, it will be rethrown
+        LOG.debug("recovered from " + StringUtils.stringifyException(e));
+        if (lastSuccessfulRow == null) {
+          LOG.warn("We are restarting the first next() invocation," +
+              " if your mapper has restarted a few other times like this" +
+              " then you should consider killing this job and investigate" +
+              " why it's taking so long.");
+        }
+        if (lastSuccessfulRow == null) {
+          restart(startRow);
+        } else {
+          restart(lastSuccessfulRow);
+          this.scanner.next();    // skip presumed already mapped row
+        }
+        result = this.scanner.next();
+      }
+
+      if (result != null && result.size() > 0) {
+        key.set(result.getRow());
+        lastSuccessfulRow = key.get();
+        value.copyFrom(result);
+        return true;
+      }
+      return false;
+    } catch (IOException ioe) {
+      if (logScannerActivity) {
+        long now = System.currentTimeMillis();
+        LOG.info("Mapper took " + (now-timestamp)
+          + "ms to process " + rowcount + " rows");
+        LOG.info(ioe);
+        String lastRow = lastSuccessfulRow == null ?
+          "null" : Bytes.toStringBinary(lastSuccessfulRow);
+        LOG.info("lastSuccessfulRow=" + lastRow);
+      }
+      throw ioe;
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableReduce.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableReduce.java
new file mode 100644
index 0000000..155ce82
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableReduce.java
@@ -0,0 +1,39 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.Reducer;
+
+/**
+ * Write a table, sorting by the input key
+ *
+ * @param  key class
+ * @param  value class
+ */
+@Deprecated
+@SuppressWarnings("unchecked")
+public interface TableReduce
+extends Reducer {
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableSplit.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableSplit.java
new file mode 100644
index 0000000..5956ee8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableSplit.java
@@ -0,0 +1,113 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapred.InputSplit;
+
+/**
+ * A table split corresponds to a key range [low, high)
+ */
+@Deprecated
+public class TableSplit implements InputSplit, Comparable {
+  private byte [] m_tableName;
+  private byte [] m_startRow;
+  private byte [] m_endRow;
+  private String m_regionLocation;
+
+  /** default constructor */
+  public TableSplit() {
+    this(HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY,
+      HConstants.EMPTY_BYTE_ARRAY, "");
+  }
+
+  /**
+   * Constructor
+   * @param tableName
+   * @param startRow
+   * @param endRow
+   * @param location
+   */
+  public TableSplit(byte [] tableName, byte [] startRow, byte [] endRow,
+      final String location) {
+    this.m_tableName = tableName;
+    this.m_startRow = startRow;
+    this.m_endRow = endRow;
+    this.m_regionLocation = location;
+  }
+
+  /** @return table name */
+  public byte [] getTableName() {
+    return this.m_tableName;
+  }
+
+  /** @return starting row key */
+  public byte [] getStartRow() {
+    return this.m_startRow;
+  }
+
+  /** @return end row key */
+  public byte [] getEndRow() {
+    return this.m_endRow;
+  }
+
+  /** @return the region's hostname */
+  public String getRegionLocation() {
+    return this.m_regionLocation;
+  }
+
+  public String[] getLocations() {
+    return new String[] {this.m_regionLocation};
+  }
+
+  public long getLength() {
+    // Not clear how to obtain this... seems to be used only for sorting splits
+    return 0;
+  }
+
+  public void readFields(DataInput in) throws IOException {
+    this.m_tableName = Bytes.readByteArray(in);
+    this.m_startRow = Bytes.readByteArray(in);
+    this.m_endRow = Bytes.readByteArray(in);
+    this.m_regionLocation = Bytes.toString(Bytes.readByteArray(in));
+  }
+
+  public void write(DataOutput out) throws IOException {
+    Bytes.writeByteArray(out, this.m_tableName);
+    Bytes.writeByteArray(out, this.m_startRow);
+    Bytes.writeByteArray(out, this.m_endRow);
+    Bytes.writeByteArray(out, Bytes.toBytes(this.m_regionLocation));
+  }
+
+  @Override
+  public String toString() {
+    return m_regionLocation + ":" +
+      Bytes.toStringBinary(m_startRow) + "," + Bytes.toStringBinary(m_endRow);
+  }
+
+  public int compareTo(TableSplit o) {
+    return Bytes.compareTo(getStartRow(), o.getStartRow());
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/package-info.java b/src/main/java/org/apache/hadoop/hbase/mapred/package-info.java
new file mode 100644
index 0000000..cc5228a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapred/package-info.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+Provides HBase MapReduce
+Input/OutputFormats, a table indexing MapReduce job, and utility
+
+
        Table of Contents
        
+
+
+
        HBase, MapReduce and the CLASSPATH
        
+
+
        MapReduce jobs deployed to a MapReduce cluster do not by default have access
+to the HBase configuration under $HBASE_CONF_DIR nor to HBase classes.
+You could add hbase-site.xml to $HADOOP_HOME/conf and add
+hbase-X.X.X.jar to the $HADOOP_HOME/lib and copy these
+changes across your cluster but the cleanest means of adding hbase configuration
+and classes to the cluster CLASSPATH is by uncommenting
+HADOOP_CLASSPATH in $HADOOP_HOME/conf/hadoop-env.sh
+adding hbase dependencies here.  For example, here is how you would amend
+hadoop-env.sh adding the
+built hbase jar, zookeeper (needed by hbase client), hbase conf, and the
+PerformanceEvaluation class from the built hbase test jar to the
+hadoop CLASSPATH:
+
+
        # Extra Java CLASSPATH elements. Optional.
+# export HADOOP_CLASSPATH=
+export HADOOP_CLASSPATH=$HBASE_HOME/build/hbase-X.X.X.jar:$HBASE_HOME/build/hbase-X.X.X-test.jar:$HBASE_HOME/conf:${HBASE_HOME}/lib/zookeeper-X.X.X.jar
        
+
+
        Expand $HBASE_HOME in the above appropriately to suit your
+local environment.
        
+
+
        After copying the above change around your cluster (and restarting), this is
+how you would run the PerformanceEvaluation MR job to put up 4 clients (Presumes
+a ready mapreduce cluster):
+
+
        $HADOOP_HOME/bin/hadoop org.apache.hadoop.hbase.PerformanceEvaluation sequentialWrite 4
        
+
+The PerformanceEvaluation class wil be found on the CLASSPATH because you
+added $HBASE_HOME/build/test to HADOOP_CLASSPATH
+
        
+
+
        Another possibility, if for example you do not have access to hadoop-env.sh or
+are unable to restart the hadoop cluster, is bundling the hbase jar into a mapreduce
+job jar adding it and its dependencies under the job jar lib/
+directory and the hbase conf into a job jar conf/ directory.
+
+
+
        HBase as MapReduce job data source and sink
        
+
+
        HBase can be used as a data source, {@link org.apache.hadoop.hbase.mapred.TableInputFormat TableInputFormat},
+and data sink, {@link org.apache.hadoop.hbase.mapred.TableOutputFormat TableOutputFormat}, for MapReduce jobs.
+Writing MapReduce jobs that read or write HBase, you'll probably want to subclass
+{@link org.apache.hadoop.hbase.mapred.TableMap TableMap} and/or
+{@link org.apache.hadoop.hbase.mapred.TableReduce TableReduce}.  See the do-nothing
+pass-through classes {@link org.apache.hadoop.hbase.mapred.IdentityTableMap IdentityTableMap} and
+{@link org.apache.hadoop.hbase.mapred.IdentityTableReduce IdentityTableReduce} for basic usage.  For a more
+involved example, see BuildTableIndex
+or review the org.apache.hadoop.hbase.mapred.TestTableMapReduce unit test.
+
        
+
+
        Running mapreduce jobs that have hbase as source or sink, you'll need to
+specify source/sink table and column names in your configuration.
        
+
+
        Reading from hbase, the TableInputFormat asks hbase for the list of
+regions and makes a map-per-region or mapred.map.tasks maps,
+whichever is smaller (If your job only has two maps, up mapred.map.tasks
+to a number > number of regions). Maps will run on the adjacent TaskTracker
+if you are running a TaskTracer and RegionServer per node.
+Writing, it may make sense to avoid the reduce step and write yourself back into
+hbase from inside your map. You'd do this when your job does not need the sort
+and collation that mapreduce does on the map emitted data; on insert,
+hbase 'sorts' so there is no point double-sorting (and shuffling data around
+your mapreduce cluster) unless you need to. If you do not need the reduce,
+you might just have your map emit counts of records processed just so the
+framework's report at the end of your job has meaning or set the number of
+reduces to zero and use TableOutputFormat. See example code
+below. If running the reduce step makes sense in your case, its usually better
+to have lots of reducers so load is spread across the hbase cluster.
        
+
+
        There is also a new hbase partitioner that will run as many reducers as
+currently existing regions.  The
+{@link org.apache.hadoop.hbase.mapred.HRegionPartitioner} is suitable
+when your table is large and your upload is not such that it will greatly
+alter the number of existing regions when done; other use the default
+partitioner.
+
        
+
+
        Example Code
        
+
        Sample Row Counter
        
+
        See {@link org.apache.hadoop.hbase.mapred.RowCounter}.  You should be able to run
+it by doing: % ./bin/hadoop jar hbase-X.X.X.jar.  This will invoke
+the hbase MapReduce Driver class.  Select 'rowcounter' from the choice of jobs
+offered. You may need to add the hbase conf directory to $HADOOP_HOME/conf/hadoop-env.sh#HADOOP_CLASSPATH
+so the rowcounter gets pointed at the right hbase cluster (or, build a new jar
+with an appropriate hbase-site.xml built into your job jar).
+
        
+
        PerformanceEvaluation
        
+
        See org.apache.hadoop.hbase.PerformanceEvaluation from hbase src/test.  It runs
+a mapreduce job to run concurrent clients reading and writing hbase.
+
        
+
+*/
+package org.apache.hadoop.hbase.mapred;
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
new file mode 100644
index 0000000..46d8c71
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
@@ -0,0 +1,258 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.*;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.io.Text;
+
+
+/**
+ * A job with a a map and reduce phase to count cells in a table.
+ * The counter lists the following stats for a given table:
+ * 
        + * 1. Total number of rows in the table
+ * 2. Total number of CFs across all rows
+ * 3. Total qualifiers across all rows
+ * 4. Total occurrence of each CF
+ * 5. Total occurrence  of each qualifier
+ * 6. Total number of versions of each qualifier.
+ * 
        
+ *
+ * The cellcounter takes two optional parameters one to use a user
+ * supplied row/family/qualifier string to use in the report and
+ * second a regex based or prefix based row filter to restrict the
+ * count operation to a limited subset of rows from the table.
+ */
+public class CellCounter {
+  private static final Log LOG =
+    LogFactory.getLog(CellCounter.class.getName());
+
+
+  /**
+   * Name of this 'program'.
+   */
+  static final String NAME = "CellCounter";
+
+  /**
+   * Mapper that runs the count.
+   */
+  static class CellCounterMapper
+  extends TableMapper {
+    /**
+     * Counter enumeration to count the actual rows.
+     */
+    public static enum Counters {
+      ROWS
+    }
+
+    /**
+     * Maps the data.
+     *
+     * @param row     The current table row key.
+     * @param values  The columns.
+     * @param context The current context.
+     * @throws IOException When something is broken with the data.
+     * @see org.apache.hadoop.mapreduce.Mapper#map(KEYIN, VALUEIN,
+     *      org.apache.hadoop.mapreduce.Mapper.Context)
+     */
+
+    @Override
+    public void map(ImmutableBytesWritable row, Result values,
+                    Context context)
+        throws IOException {
+      String currentFamilyName = null;
+      String currentQualifierName = null;
+      String currentRowKey = null;
+      Configuration config = context.getConfiguration();
+      String separator = config.get("ReportSeparator",":");
+
+      try {
+        if (values != null) {
+          context.getCounter(Counters.ROWS).increment(1);
+          context.write(new Text("Total ROWS"), new IntWritable(1));
+        }
+
+        for (KeyValue value : values.list()) {
+          currentRowKey = Bytes.toStringBinary(value.getRow());
+          String thisRowFamilyName = Bytes.toStringBinary(value.getFamily());
+          if (thisRowFamilyName != null &&
+              !thisRowFamilyName.equals(currentFamilyName)) {
+            currentFamilyName = thisRowFamilyName;
+            context.getCounter("CF", thisRowFamilyName).increment(1);
+            context.write(new Text("Total Families Across all Rows"),
+              new IntWritable(1));
+            context.write(new Text(thisRowFamilyName), new IntWritable(1));
+          }
+          String thisRowQualifierName =
+            thisRowFamilyName + separator + Bytes.toStringBinary(value.getQualifier());
+          if (thisRowQualifierName != null &&
+              !thisRowQualifierName.equals(currentQualifierName)) {
+            currentQualifierName = thisRowQualifierName;
+            context.getCounter("CFQL", thisRowQualifierName).increment(1);
+            context.write(new Text("Total Qualifiers across all Rows"),
+              new IntWritable(1));
+            context.write(new Text(thisRowQualifierName), new IntWritable(1));
+            // Intialize versions
+            context.getCounter("QL_VERSIONS", currentRowKey + separator +
+              thisRowQualifierName).increment(1);
+            context.write(new Text(currentRowKey + separator + thisRowQualifierName +
+              "_Versions"), new IntWritable(1));
+
+          } else {
+            // Increment versions
+            currentQualifierName = thisRowQualifierName;
+            context.getCounter("QL_VERSIONS", currentRowKey + separator +
+              thisRowQualifierName).increment(1);
+            context.write(new Text(currentRowKey + separator + thisRowQualifierName + "_Versions"),
+              new IntWritable(1));
+          }
+        }
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+  }
+
+  static class IntSumReducer extends Reducer {
+
+    private IntWritable result = new IntWritable();
+    public void reduce(Key key, Iterable values,
+      Context context)
+    throws IOException, InterruptedException {
+      int sum = 0;
+      for (IntWritable val : values) {
+        sum += val.get();
+      }
+      result.set(sum);
+      context.write(key, result);
+    }
+  }
+
+  /**
+   * Sets up the actual job.
+   *
+   * @param conf The current configuration.
+   * @param args The command line parameters.
+   * @return The newly created job.
+   * @throws IOException When setting up the job fails.
+   */
+  public static Job createSubmittableJob(Configuration conf, String[] args)
+      throws IOException {
+    String tableName = args[0];
+    Path outputDir = new Path(args[1]);
+    String reportSeparatorString = (args.length > 2) ? args[2]: ":";
+    conf.set("ReportSeparator", reportSeparatorString);
+    Job job = new Job(conf, NAME + "_" + tableName);
+    job.setJarByClass(CellCounter.class);
+    Scan scan = getConfiguredScanForJob(conf, args);
+    TableMapReduceUtil.initTableMapperJob(tableName, scan,
+        CellCounterMapper.class, ImmutableBytesWritable.class, Result.class, job);
+    job.setNumReduceTasks(1);
+    job.setMapOutputKeyClass(Text.class);
+    job.setMapOutputValueClass(IntWritable.class);
+    job.setOutputFormatClass(TextOutputFormat.class);
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(IntWritable.class);
+    FileOutputFormat.setOutputPath(job, outputDir);
+    job.setReducerClass(IntSumReducer.class);
+    return job;
+  }
+
+  private static Scan getConfiguredScanForJob(Configuration conf, String[] args) throws IOException {
+    Scan s = new Scan();
+    // Set Scan Versions
+    s.setMaxVersions(Integer.MAX_VALUE);
+    s.setCacheBlocks(false);
+    // Set Scan Column Family
+    if (conf.get(TableInputFormat.SCAN_COLUMN_FAMILY) != null) {
+      s.addFamily(Bytes.toBytes(conf.get(TableInputFormat.SCAN_COLUMN_FAMILY)));
+    }
+    // Set RowFilter or Prefix Filter if applicable.
+    Filter rowFilter = getRowFilter(args);
+    if (rowFilter!= null) {
+      LOG.info("Setting Row Filter for counter.");
+      s.setFilter(rowFilter);
+    }
+    return s;
+  }
+
+
+  private static Filter getRowFilter(String[] args) {
+    Filter rowFilter = null;
+    String filterCriteria = (args.length > 3) ? args[3]: null;
+    if (filterCriteria == null) return null;
+    if (filterCriteria.startsWith("^")) {
+      String regexPattern = filterCriteria.substring(1, filterCriteria.length());
+      rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regexPattern));
+    } else {
+      rowFilter = new PrefixFilter(Bytes.toBytes(filterCriteria));
+    }
+    return rowFilter;
+  }
+
+  /**
+   * Main entry point.
+   *
+   * @param args The command line parameters.
+   * @throws Exception When running the job fails.
+   */
+  public static void main(String[] args) throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+    if (otherArgs.length < 1) {
+      System.err.println("ERROR: Wrong number of parameters: " + args.length);
+      System.err.println("Usage: CellCounter    " +
+          "[^[regex pattern] or [Prefix] for row filter]] ");
+      System.err.println("  Note: -D properties will be applied to the conf used. ");
+      System.err.println("  Additionally, the following SCAN properties can be specified");
+      System.err.println("  to get fine grained control on what is counted..");
+      System.err.println("   -D " + TableInputFormat.SCAN_COLUMN_FAMILY + "=");
+      System.err.println("  parameter can be used to override the default report separator " +
+          "string : used to separate the rowId/column family name and qualifier name.");
+      System.err.println(" [^[regex pattern] or [Prefix] parameter can be used to limit the cell counter count " +
+          "operation to a limited subset of rows from the table based on regex or prefix pattern.");
+      System.exit(-1);
+    }
+    Job job = createSubmittableJob(conf, otherArgs);
+    System.exit(job.waitForCompletion(true) ? 0 : 1);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
new file mode 100644
index 0000000..d8e9860
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -0,0 +1,240 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.GenericOptionsParser;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Tool used to copy a table to another one which can be on a different setup.
+ * It is also configurable with a start and time as well as a specification
+ * of the region server implementation if different from the local cluster.
+ */
+public class CopyTable {
+
+  final static String NAME = "copytable";
+  static long startTime = 0;
+  static long endTime = 0;
+  static int versions = -1;
+  static String tableName = null;
+  static String newTableName = null;
+  static String peerAddress = null;
+  static String families = null;
+  static boolean allCells = false;
+
+  /**
+   * Sets up the actual job.
+   *
+   * @param conf  The current configuration.
+   * @param args  The command line parameters.
+   * @return The newly created job.
+   * @throws IOException When setting up the job fails.
+   */
+  public static Job createSubmittableJob(Configuration conf, String[] args)
+  throws IOException {
+    if (!doCommandLine(args)) {
+      return null;
+    }
+    Job job = new Job(conf, NAME + "_" + tableName);
+    job.setJarByClass(CopyTable.class);
+    Scan scan = new Scan();
+    scan.setCacheBlocks(false);
+    if (startTime != 0) {
+      scan.setTimeRange(startTime,
+          endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
+    }
+    if (allCells) {
+      scan.setRaw(true);
+    }
+    if (versions >= 0) {
+      scan.setMaxVersions(versions);
+    }
+    if(families != null) {
+      String[] fams = families.split(",");
+      Map cfRenameMap = new HashMap();
+      for(String fam : fams) {
+        String sourceCf;
+        if(fam.contains(":")) { 
+            // fam looks like "sourceCfName:destCfName"
+            String[] srcAndDest = fam.split(":", 2);
+            sourceCf = srcAndDest[0];
+            String destCf = srcAndDest[1];
+            cfRenameMap.put(sourceCf, destCf);
+        } else {
+            // fam is just "sourceCf"
+            sourceCf = fam; 
+        }
+        scan.addFamily(Bytes.toBytes(sourceCf));
+      }
+      Import.configureCfRenaming(job.getConfiguration(), cfRenameMap);
+    }
+    TableMapReduceUtil.initTableMapperJob(tableName, scan,
+        Import.Importer.class, null, null, job);
+    TableMapReduceUtil.initTableReducerJob(
+        newTableName == null ? tableName : newTableName, null, job,
+        null, peerAddress, null, null);
+    job.setNumReduceTasks(0);
+    return job;
+  }
+
+  /*
+   * @param errorMsg Error message.  Can be null.
+   */
+  private static void printUsage(final String errorMsg) {
+    if (errorMsg != null && errorMsg.length() > 0) {
+      System.err.println("ERROR: " + errorMsg);
+    }
+    System.err.println("Usage: CopyTable [general options] [--starttime=X] [--endtime=Y] " +
+        "[--new.name=NEW] [--peer.adr=ADR] ");
+    System.err.println();
+    System.err.println("Options:");
+    System.err.println(" rs.class     hbase.regionserver.class of the peer cluster");
+    System.err.println("              specify if different from current cluster");
+    System.err.println(" rs.impl      hbase.regionserver.impl of the peer cluster");
+    System.err.println(" starttime    beginning of the time range (unixtime in millis)");
+    System.err.println("              without endtime means from starttime to forever");
+    System.err.println(" endtime      end of the time range.  Ignored if no starttime specified.");
+    System.err.println(" versions     number of cell versions to copy");
+    System.err.println(" new.name     new table's name");
+    System.err.println(" peer.adr     Address of the peer cluster given in the format");
+    System.err.println("              hbase.zookeeer.quorum:hbase.zookeeper.client.port:zookeeper.znode.parent");
+    System.err.println(" families     comma-separated list of families to copy");
+    System.err.println("              To copy from cf1 to cf2, give sourceCfName:destCfName. ");
+    System.err.println("              To keep the same name, just give \"cfName\"");
+    System.err.println(" all.cells    also copy delete markers and deleted cells");
+    System.err.println();
+    System.err.println("Args:");
+    System.err.println(" tablename    Name of the table to copy");
+    System.err.println();
+    System.err.println("Examples:");
+    System.err.println(" To copy 'TestTable' to a cluster that uses replication for a 1 hour window:");
+    System.err.println(" $ bin/hbase " +
+        "org.apache.hadoop.hbase.mapreduce.CopyTable --starttime=1265875194289 --endtime=1265878794289 " +
+        "--peer.adr=server1,server2,server3:2181:/hbase --families=myOldCf:myNewCf,cf2,cf3 TestTable ");
+    System.err.println("For performance consider the following general options:\n"
+        + "-Dhbase.client.scanner.caching=100\n"
+        + "-Dmapred.map.tasks.speculative.execution=false");
+  }
+
+  private static boolean doCommandLine(final String[] args) {
+    // Process command-line args. TODO: Better cmd-line processing
+    // (but hopefully something not as painful as cli options).
+    if (args.length < 1) {
+      printUsage(null);
+      return false;
+    }
+    try {
+      for (int i = 0; i < args.length; i++) {
+        String cmd = args[i];
+        if (cmd.equals("-h") || cmd.startsWith("--h")) {
+          printUsage(null);
+          return false;
+        }
+
+        final String startTimeArgKey = "--starttime=";
+        if (cmd.startsWith(startTimeArgKey)) {
+          startTime = Long.parseLong(cmd.substring(startTimeArgKey.length()));
+          continue;
+        }
+
+        final String endTimeArgKey = "--endtime=";
+        if (cmd.startsWith(endTimeArgKey)) {
+          endTime = Long.parseLong(cmd.substring(endTimeArgKey.length()));
+          continue;
+        }
+
+        final String versionsArgKey = "--versions=";
+        if (cmd.startsWith(versionsArgKey)) {
+          versions = Integer.parseInt(cmd.substring(versionsArgKey.length()));
+          continue;
+        }
+
+        final String newNameArgKey = "--new.name=";
+        if (cmd.startsWith(newNameArgKey)) {
+          newTableName = cmd.substring(newNameArgKey.length());
+          continue;
+        }
+
+        final String peerAdrArgKey = "--peer.adr=";
+        if (cmd.startsWith(peerAdrArgKey)) {
+          peerAddress = cmd.substring(peerAdrArgKey.length());
+          continue;
+        }
+
+        final String familiesArgKey = "--families=";
+        if (cmd.startsWith(familiesArgKey)) {
+          families = cmd.substring(familiesArgKey.length());
+          continue;
+        }
+
+        if (cmd.startsWith("--all.cells")) {
+          allCells = true;
+          continue;
+        }
+
+        if (i == args.length-1) {
+          tableName = cmd;
+        } else {
+          printUsage("Invalid argument '" + cmd + "'" );
+          return false;
+        }
+      }
+      if (newTableName == null && peerAddress == null) {
+        printUsage("At least a new table name or a " +
+            "peer address must be specified");
+        return false;
+      }
+      if (startTime > endTime) {
+        printUsage("Invalid time range filter: starttime=" + startTime + " >  endtime=" + endTime);
+        return false;
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      printUsage("Can't start because " + e.getMessage());
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Main entry point.
+   *
+   * @param args  The command line parameters.
+   * @throws Exception When running the job fails.
+   */
+  public static void main(String[] args) throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    String[] otherArgs =
+      new GenericOptionsParser(conf, args).getRemainingArgs();
+    Job job = createSubmittableJob(conf, otherArgs);
+    if (job != null) {
+      System.exit(job.waitForCompletion(true) ? 0 : 1);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Driver.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Driver.java
new file mode 100644
index 0000000..dda4241
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Driver.java
@@ -0,0 +1,54 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication;
+import org.apache.hadoop.util.ProgramDriver;
+
+/**
+ * Driver for hbase mapreduce jobs. Select which to run by passing
+ * name of job to this main.
+ */
+public class Driver {
+  /**
+   * @param args
+   * @throws Throwable
+   */
+  public static void main(String[] args) throws Throwable {
+    ProgramDriver pgd = new ProgramDriver();
+    pgd.addClass(RowCounter.NAME, RowCounter.class,
+      "Count rows in HBase table");
+    pgd.addClass(CellCounter.NAME, CellCounter.class,
+      "Count cells in HBase table");
+    pgd.addClass(Export.NAME, Export.class, "Write table data to HDFS.");
+    pgd.addClass(Import.NAME, Import.class, "Import data written by Export.");
+    pgd.addClass(ImportTsv.NAME, ImportTsv.class, "Import data in TSV format.");
+    pgd.addClass(LoadIncrementalHFiles.NAME, LoadIncrementalHFiles.class,
+                 "Complete a bulk data load.");
+    pgd.addClass(CopyTable.NAME, CopyTable.class,
+        "Export a table from local cluster to peer cluster");
+    pgd.addClass(VerifyReplication.NAME, VerifyReplication.class, "Compare" +
+        " the data from tables in two different clusters. WARNING: It" +
+        " doesn't work for incrementColumnValues'd cells since the" +
+        " timestamp is changed after being appended to the log.");
+    ProgramDriver.class.getMethod("driver", new Class [] {String[].class}).
+      invoke(pgd, new Object[]{args});
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
new file mode 100644
index 0000000..2c53f6d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
@@ -0,0 +1,191 @@
+/**
+* Copyright 2009 The Apache Software Foundation
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
+import org.apache.hadoop.hbase.filter.RowFilter;
+import org.apache.hadoop.hbase.filter.RegexStringComparator;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+* Export an HBase table.
+* Writes content to sequence files up in HDFS.  Use {@link Import} to read it
+* back in again.
+*/
+public class Export {
+  private static final Log LOG = LogFactory.getLog(Export.class);
+  final static String NAME = "export";
+  final static String RAW_SCAN="hbase.mapreduce.include.deleted.rows";
+
+  /**
+   * Mapper.
+   */
+  static class Exporter
+  extends TableMapper {
+    /**
+     * @param row  The current table row key.
+     * @param value  The columns.
+     * @param context  The current context.
+     * @throws IOException When something is broken with the data.
+     * @see org.apache.hadoop.mapreduce.Mapper#map(KEYIN, VALUEIN,
+     *   org.apache.hadoop.mapreduce.Mapper.Context)
+     */
+    @Override
+    public void map(ImmutableBytesWritable row, Result value,
+      Context context)
+    throws IOException {
+      try {
+        context.write(row, value);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+  }
+
+  /**
+   * Sets up the actual job.
+   *
+   * @param conf  The current configuration.
+   * @param args  The command line parameters.
+   * @return The newly created job.
+   * @throws IOException When setting up the job fails.
+   */
+  public static Job createSubmittableJob(Configuration conf, String[] args)
+  throws IOException {
+    String tableName = args[0];
+    Path outputDir = new Path(args[1]);
+    Job job = new Job(conf, NAME + "_" + tableName);
+    job.setJobName(NAME + "_" + tableName);
+    job.setJarByClass(Exporter.class);
+    // Set optional scan parameters
+    Scan s = getConfiguredScanForJob(conf, args);
+    TableMapReduceUtil.initTableMapperJob(tableName, s, Exporter.class, null,
+      null, job);
+    // No reducers.  Just write straight to output files.
+    job.setNumReduceTasks(0);
+    job.setOutputFormatClass(SequenceFileOutputFormat.class);
+    job.setOutputKeyClass(ImmutableBytesWritable.class);
+    job.setOutputValueClass(Result.class);
+    FileOutputFormat.setOutputPath(job, outputDir);
+    return job;
+  }
+
+  private static Scan getConfiguredScanForJob(Configuration conf, String[] args) throws IOException {
+    Scan s = new Scan();
+    // Optional arguments.
+    // Set Scan Versions
+    int versions = args.length > 2? Integer.parseInt(args[2]): 1;
+    s.setMaxVersions(versions);
+    // Set Scan Range
+    long startTime = args.length > 3? Long.parseLong(args[3]): 0L;
+    long endTime = args.length > 4? Long.parseLong(args[4]): Long.MAX_VALUE;
+    s.setTimeRange(startTime, endTime);
+    // Set cache blocks
+    s.setCacheBlocks(false);
+    // Set Scan Column Family
+    boolean raw = Boolean.parseBoolean(conf.get(RAW_SCAN));
+    if (raw) {
+      s.setRaw(raw);
+    }
+    
+    if (conf.get(TableInputFormat.SCAN_COLUMN_FAMILY) != null) {
+      s.addFamily(Bytes.toBytes(conf.get(TableInputFormat.SCAN_COLUMN_FAMILY)));
+    }
+    // Set RowFilter or Prefix Filter if applicable.
+    Filter exportFilter = getExportFilter(args);
+    if (exportFilter!= null) {
+        LOG.info("Setting Scan Filter for Export.");
+      s.setFilter(exportFilter);
+    }
+    LOG.info("versions=" + versions + ", starttime=" + startTime +
+      ", endtime=" + endTime + ", keepDeletedCells=" + raw);
+    return s;
+  }
+
+  private static Filter getExportFilter(String[] args) {
+    Filter exportFilter = null;
+    String filterCriteria = (args.length > 5) ? args[5]: null;
+    if (filterCriteria == null) return null;
+    if (filterCriteria.startsWith("^")) {
+      String regexPattern = filterCriteria.substring(1, filterCriteria.length());
+      exportFilter = new RowFilter(CompareOp.EQUAL, new RegexStringComparator(regexPattern));
+    } else {
+      exportFilter = new PrefixFilter(Bytes.toBytes(filterCriteria));
+    }
+    return exportFilter;
+  }
+
+  /*
+   * @param errorMsg Error message.  Can be null.
+   */
+  private static void usage(final String errorMsg) {
+    if (errorMsg != null && errorMsg.length() > 0) {
+      System.err.println("ERROR: " + errorMsg);
+    }
+    System.err.println("Usage: Export [-D ]*   [ " +
+      "[ []] [^[regex pattern] or [Prefix] to filter]]\n");
+    System.err.println("  Note: -D properties will be applied to the conf used. ");
+    System.err.println("  For example: ");
+    System.err.println("   -D mapred.output.compress=true");
+    System.err.println("   -D mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec");
+    System.err.println("   -D mapred.output.compression.type=BLOCK");
+    System.err.println("  Additionally, the following SCAN properties can be specified");
+    System.err.println("  to control/limit what is exported..");
+    System.err.println("   -D " + TableInputFormat.SCAN_COLUMN_FAMILY + "=");
+    System.err.println("   -D " + RAW_SCAN + "=true");
+    System.err.println("For performance consider the following properties:\n"
+        + "   -Dhbase.client.scanner.caching=100\n"
+        + "   -Dmapred.map.tasks.speculative.execution=false\n"
+        + "   -Dmapred.reduce.tasks.speculative.execution=false");
+  }
+
+  /**
+   * Main entry point.
+   *
+   * @param args  The command line parameters.
+   * @throws Exception When running the job fails.
+   */
+  public static void main(String[] args) throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+    if (otherArgs.length < 2) {
+      usage("Wrong number of arguments: " + otherArgs.length);
+      System.exit(-1);
+    }
+    Job job = createSubmittableJob(conf, otherArgs);
+    System.exit(job.waitForCompletion(true)? 0 : 1);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/GroupingTableMapper.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/GroupingTableMapper.java
new file mode 100644
index 0000000..c38337b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/GroupingTableMapper.java
@@ -0,0 +1,180 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Job;
+
+/**
+ * Extract grouping columns from input record.
+ */
+public class GroupingTableMapper
+extends TableMapper implements Configurable {
+
+  /**
+   * JobConf parameter to specify the columns used to produce the key passed to
+   * collect from the map phase.
+   */
+  public static final String GROUP_COLUMNS =
+    "hbase.mapred.groupingtablemap.columns";
+
+  /** The grouping columns. */
+  protected byte [][] columns;
+  /** The current configuration. */
+  private Configuration conf = null;
+
+  /**
+   * Use this before submitting a TableMap job. It will appropriately set up
+   * the job.
+   *
+   * @param table The table to be processed.
+   * @param scan  The scan with the columns etc.
+   * @param groupColumns  A space separated list of columns used to form the
+   * key used in collect.
+   * @param mapper  The mapper class.
+   * @param job  The current job.
+   * @throws IOException When setting up the job fails.
+   */
+  @SuppressWarnings("unchecked")
+  public static void initJob(String table, Scan scan, String groupColumns,
+    Class mapper, Job job) throws IOException {
+    TableMapReduceUtil.initTableMapperJob(table, scan, mapper,
+        ImmutableBytesWritable.class, Result.class, job);
+    job.getConfiguration().set(GROUP_COLUMNS, groupColumns);
+  }
+
+  /**
+   * Extract the grouping columns from value to construct a new key. Pass the
+   * new key and value to reduce. If any of the grouping columns are not found
+   * in the value, the record is skipped.
+   *
+   * @param key  The current key.
+   * @param value  The current value.
+   * @param context  The current context.
+   * @throws IOException When writing the record fails.
+   * @throws InterruptedException When the job is aborted.
+   */
+  @Override
+  public void map(ImmutableBytesWritable key, Result value, Context context)
+  throws IOException, InterruptedException {
+    byte[][] keyVals = extractKeyValues(value);
+    if(keyVals != null) {
+      ImmutableBytesWritable tKey = createGroupKey(keyVals);
+      context.write(tKey, value);
+    }
+  }
+
+  /**
+   * Extract columns values from the current record. This method returns
+   * null if any of the columns are not found.
+   * 
        
+   * Override this method if you want to deal with nulls differently.
+   *
+   * @param r  The current values.
+   * @return Array of byte values.
+   */
+  protected byte[][] extractKeyValues(Result r) {
+    byte[][] keyVals = null;
+    ArrayList foundList = new ArrayList();
+    int numCols = columns.length;
+    if (numCols > 0) {
+      for (KeyValue value: r.list()) {
+        byte [] column = KeyValue.makeColumn(value.getFamily(),
+            value.getQualifier());
+        for (int i = 0; i < numCols; i++) {
+          if (Bytes.equals(column, columns[i])) {
+            foundList.add(value.getValue());
+            break;
+          }
+        }
+      }
+      if(foundList.size() == numCols) {
+        keyVals = foundList.toArray(new byte[numCols][]);
+      }
+    }
+    return keyVals;
+  }
+
+  /**
+   * Create a key by concatenating multiple column values.
+   * 
        
+   * Override this function in order to produce different types of keys.
+   *
+   * @param vals  The current key/values.
+   * @return A key generated by concatenating multiple column values.
+   */
+  protected ImmutableBytesWritable createGroupKey(byte[][] vals) {
+    if(vals == null) {
+      return null;
+    }
+    StringBuilder sb =  new StringBuilder();
+    for(int i = 0; i < vals.length; i++) {
+      if(i > 0) {
+        sb.append(" ");
+      }
+      try {
+        sb.append(new String(vals[i], HConstants.UTF8_ENCODING));
+      } catch (UnsupportedEncodingException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    return new ImmutableBytesWritable(Bytes.toBytes(sb.toString()));
+  }
+
+  /**
+   * Returns the current configuration.
+   *
+   * @return The current configuration.
+   * @see org.apache.hadoop.conf.Configurable#getConf()
+   */
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  /**
+   * Sets the configuration. This is used to set up the grouping details.
+   *
+   * @param configuration  The configuration to set.
+   * @see org.apache.hadoop.conf.Configurable#setConf(
+   *   org.apache.hadoop.conf.Configuration)
+   */
+  @Override
+  public void setConf(Configuration configuration) {
+    this.conf = configuration;
+    String[] cols = conf.get(GROUP_COLUMNS, "").split(" ");
+    columns = new byte[cols.length][];
+    for(int i = 0; i < cols.length; i++) {
+      columns[i] = Bytes.toBytes(cols[i]);
+    }
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
new file mode 100644
index 0000000..acb7d73
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
@@ -0,0 +1,507 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.AbstractHFileWriter;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+/**
+ * Writes HFiles. Passed KeyValues must arrive in order.
+ * Currently, can only write files to a single column family at a
+ * time.  Multiple column families requires coordinating keys cross family.
+ * Writes current time as the sequence id for the file. Sets the major compacted
+ * attribute on created hfiles. Calling write(null,null) will forceably roll
+ * all HFiles being written.
+ * @see KeyValueSortReducer
+ */
+public class HFileOutputFormat extends FileOutputFormat {
+  static Log LOG = LogFactory.getLog(HFileOutputFormat.class);
+  static final String COMPRESSION_CONF_KEY = "hbase.hfileoutputformat.families.compression";
+  private static final String BLOOM_TYPE_CONF_KEY = "hbase.hfileoutputformat.families.bloomtype";
+  private static final String DATABLOCK_ENCODING_CONF_KEY = 
+     "hbase.mapreduce.hfileoutputformat.datablock.encoding";
+
+  public RecordWriter getRecordWriter(final TaskAttemptContext context)
+  throws IOException, InterruptedException {
+    // Get the path of the temporary output file
+    final Path outputPath = FileOutputFormat.getOutputPath(context);
+    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
+    final Configuration conf = context.getConfiguration();
+    final FileSystem fs = outputdir.getFileSystem(conf);
+    // These configs. are from hbase-*.xml
+    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
+        HConstants.DEFAULT_MAX_FILE_SIZE);
+    final int blocksize = conf.getInt("hbase.mapreduce.hfileoutputformat.blocksize",
+        HFile.DEFAULT_BLOCKSIZE);
+    // Invented config.  Add to hbase-*.xml if other than default compression.
+    final String defaultCompression = conf.get("hfile.compression",
+        Compression.Algorithm.NONE.getName());
+    final boolean compactionExclude = conf.getBoolean(
+        "hbase.mapreduce.hfileoutputformat.compaction.exclude", false);
+
+    // create a map from column family to the compression algorithm
+    final Map compressionMap = createFamilyCompressionMap(conf);
+    final Map bloomTypeMap = createFamilyBloomMap(conf);
+    
+    String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_CONF_KEY);
+    final HFileDataBlockEncoder encoder;
+    if (dataBlockEncodingStr == null) {
+      encoder = NoOpDataBlockEncoder.INSTANCE;
+    } else {
+      try {
+        encoder = new HFileDataBlockEncoderImpl(DataBlockEncoding
+            .valueOf(dataBlockEncodingStr));
+      } catch (IllegalArgumentException ex) {
+        throw new RuntimeException(
+            "Invalid data block encoding type configured for the param "
+                + DATABLOCK_ENCODING_CONF_KEY + " : "
+                + dataBlockEncodingStr);
+      }
+    }
+
+    return new RecordWriter() {
+      // Map of families to writers and how much has been output on the writer.
+      private final Map writers =
+        new TreeMap(Bytes.BYTES_COMPARATOR);
+      private byte [] previousRow = HConstants.EMPTY_BYTE_ARRAY;
+      private final byte [] now = Bytes.toBytes(System.currentTimeMillis());
+      private boolean rollRequested = false;
+
+      public void write(ImmutableBytesWritable row, KeyValue kv)
+      throws IOException {
+        // null input == user explicitly wants to flush
+        if (row == null && kv == null) {
+          rollWriters();
+          return;
+        }
+
+        byte [] rowKey = kv.getRow();
+        long length = kv.getLength();
+        byte [] family = kv.getFamily();
+        WriterLength wl = this.writers.get(family);
+
+        // If this is a new column family, verify that the directory exists
+        if (wl == null) {
+          fs.mkdirs(new Path(outputdir, Bytes.toString(family)));
+        }
+
+        // If any of the HFiles for the column families has reached
+        // maxsize, we need to roll all the writers
+        if (wl != null && wl.written + length >= maxsize) {
+          this.rollRequested = true;
+        }
+
+        // This can only happen once a row is finished though
+        if (rollRequested && Bytes.compareTo(this.previousRow, rowKey) != 0) {
+          rollWriters();
+        }
+
+        // create a new HLog writer, if necessary
+        if (wl == null || wl.writer == null) {
+          wl = getNewWriter(family, conf);
+        }
+
+        // we now have the proper HLog writer. full steam ahead
+        kv.updateLatestStamp(this.now);
+        wl.writer.append(kv);
+        wl.written += length;
+
+        // Copy the row so we know when a row transition.
+        this.previousRow = rowKey;
+      }
+
+      private void rollWriters() throws IOException {
+        for (WriterLength wl : this.writers.values()) {
+          if (wl.writer != null) {
+            LOG.info("Writer=" + wl.writer.getPath() +
+                ((wl.written == 0)? "": ", wrote=" + wl.written));
+            close(wl.writer);
+          }
+          wl.writer = null;
+          wl.written = 0;
+        }
+        this.rollRequested = false;
+      }
+
+      /* Create a new StoreFile.Writer.
+       * @param family
+       * @return A WriterLength, containing a new StoreFile.Writer.
+       * @throws IOException
+       */
+      private WriterLength getNewWriter(byte[] family, Configuration conf)
+          throws IOException {
+        WriterLength wl = new WriterLength();
+        Path familydir = new Path(outputdir, Bytes.toString(family));
+        String compression = compressionMap.get(family);
+        compression = compression == null ? defaultCompression : compression;
+        String bloomTypeStr = bloomTypeMap.get(family);
+        BloomType bloomType = BloomType.NONE;
+        if (bloomTypeStr != null) {
+          bloomType = BloomType.valueOf(bloomTypeStr);
+        }
+        Configuration tempConf = new Configuration(conf);
+        tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
+        wl.writer = new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs, blocksize)
+            .withOutputDir(familydir)
+            .withCompression(AbstractHFileWriter.compressionByName(compression))
+            .withBloomType(bloomType)
+            .withComparator(KeyValue.COMPARATOR)
+            .withDataBlockEncoder(encoder)
+            .withChecksumType(Store.getChecksumType(conf))
+            .withBytesPerChecksum(Store.getBytesPerChecksum(conf))
+            .build();
+
+        this.writers.put(family, wl);
+        return wl;
+      }
+
+      private void close(final StoreFile.Writer w) throws IOException {
+        if (w != null) {
+          w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY,
+              Bytes.toBytes(System.currentTimeMillis()));
+          w.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY,
+              Bytes.toBytes(context.getTaskAttemptID().toString()));
+          w.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY,
+              Bytes.toBytes(true));
+          w.appendFileInfo(StoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY,
+              Bytes.toBytes(compactionExclude));
+          w.appendTrackedTimestampsToMetadata();
+          w.close();
+        }
+      }
+
+      public void close(TaskAttemptContext c)
+      throws IOException, InterruptedException {
+        for (WriterLength wl: this.writers.values()) {
+          close(wl.writer);
+        }
+      }
+    };
+  }
+
+  /*
+   * Data structure to hold a Writer and amount of data written on it.
+   */
+  static class WriterLength {
+    long written = 0;
+    StoreFile.Writer writer = null;
+  }
+
+  /**
+   * Return the start keys of all of the regions in this table,
+   * as a list of ImmutableBytesWritable.
+   */
+  private static List getRegionStartKeys(HTable table)
+  throws IOException {
+    byte[][] byteKeys = table.getStartKeys();
+    ArrayList ret =
+      new ArrayList(byteKeys.length);
+    for (byte[] byteKey : byteKeys) {
+      ret.add(new ImmutableBytesWritable(byteKey));
+    }
+    return ret;
+  }
+
+  /**
+   * Write out a SequenceFile that can be read by TotalOrderPartitioner
+   * that contains the split points in startKeys.
+   * @param partitionsPath output path for SequenceFile
+   * @param startKeys the region start keys
+   */
+  private static void writePartitions(Configuration conf, Path partitionsPath,
+      List startKeys) throws IOException {
+    if (startKeys.isEmpty()) {
+      throw new IllegalArgumentException("No regions passed");
+    }
+
+    // We're generating a list of split points, and we don't ever
+    // have keys < the first region (which has an empty start key)
+    // so we need to remove it. Otherwise we would end up with an
+    // empty reducer with index 0
+    TreeSet sorted =
+      new TreeSet(startKeys);
+
+    ImmutableBytesWritable first = sorted.first();
+    if (!first.equals(HConstants.EMPTY_BYTE_ARRAY)) {
+      throw new IllegalArgumentException(
+          "First region of table should have empty start key. Instead has: "
+          + Bytes.toStringBinary(first.get()));
+    }
+    sorted.remove(first);
+
+    // Write the actual file
+    FileSystem fs = partitionsPath.getFileSystem(conf);
+    SequenceFile.Writer writer = SequenceFile.createWriter(fs,
+        conf, partitionsPath, ImmutableBytesWritable.class, NullWritable.class);
+
+    try {
+      for (ImmutableBytesWritable startKey : sorted) {
+        writer.append(startKey, NullWritable.get());
+      }
+    } finally {
+      writer.close();
+    }
+  }
+
+  /**
+   * Configure a MapReduce Job to perform an incremental load into the given
+   * table. This
+   * 
          
+   *   
        • Inspects the table to configure a total order partitioner
          • 
+   *   
        • Uploads the partitions file to the cluster and adds it to the DistributedCache
          • 
+   *   
        • Sets the number of reduce tasks to match the current number of regions
          • 
+   *   
        • Sets the output key/value class to match HFileOutputFormat's requirements
          • 
+   *   
        • Sets the reducer up to perform the appropriate sorting (either KeyValueSortReducer or
+   *     PutSortReducer)
          • 
+   * 
        
+   * The user should be sure to set the map output value class to either KeyValue or Put before
+   * running this function.
+   */
+  public static void configureIncrementalLoad(Job job, HTable table)
+  throws IOException {
+    Configuration conf = job.getConfiguration();
+    Class topClass;
+    try {
+      topClass = getTotalOrderPartitionerClass();
+    } catch (ClassNotFoundException e) {
+      throw new IOException("Failed getting TotalOrderPartitioner", e);
+    }
+    job.setPartitionerClass(topClass);
+    job.setOutputKeyClass(ImmutableBytesWritable.class);
+    job.setOutputValueClass(KeyValue.class);
+    job.setOutputFormatClass(HFileOutputFormat.class);
+
+    // Based on the configured map output class, set the correct reducer to properly
+    // sort the incoming values.
+    // TODO it would be nice to pick one or the other of these formats.
+    if (KeyValue.class.equals(job.getMapOutputValueClass())) {
+      job.setReducerClass(KeyValueSortReducer.class);
+    } else if (Put.class.equals(job.getMapOutputValueClass())) {
+      job.setReducerClass(PutSortReducer.class);
+    } else {
+      LOG.warn("Unknown map output value type:" + job.getMapOutputValueClass());
+    }
+
+    LOG.info("Looking up current regions for table " + table);
+    List startKeys = getRegionStartKeys(table);
+    LOG.info("Configuring " + startKeys.size() + " reduce partitions " +
+        "to match current region count");
+    job.setNumReduceTasks(startKeys.size());
+
+    Path partitionsPath = new Path(job.getWorkingDirectory(),
+                                   "partitions_" + UUID.randomUUID());
+    LOG.info("Writing partition information to " + partitionsPath);
+
+    FileSystem fs = partitionsPath.getFileSystem(conf);
+    writePartitions(conf, partitionsPath, startKeys);
+    partitionsPath.makeQualified(fs);
+
+    URI cacheUri;
+    try {
+      // Below we make explicit reference to the bundled TOP.  Its cheating.
+      // We are assume the define in the hbase bundled TOP is as it is in
+      // hadoop (whether 0.20 or 0.22, etc.)
+      cacheUri = new URI(partitionsPath.toString() + "#" +
+        org.apache.hadoop.hbase.mapreduce.hadoopbackport.TotalOrderPartitioner.DEFAULT_PATH);
+    } catch (URISyntaxException e) {
+      throw new IOException(e);
+    }
+    DistributedCache.addCacheFile(cacheUri, conf);
+    DistributedCache.createSymlink(conf);
+
+    // Set compression algorithms based on column families
+    configureCompression(table, conf);
+    configureBloomType(table, conf);
+    
+    TableMapReduceUtil.addDependencyJars(job);
+    LOG.info("Incremental table output configured.");
+  }
+
+  /**
+   * If > hadoop 0.20, then we want to use the hadoop TotalOrderPartitioner.
+   * If 0.20, then we want to use the TOP that we have under hadoopbackport.
+   * This method is about hbase being able to run on different versions of
+   * hadoop.  In 0.20.x hadoops, we have to use the TOP that is bundled with
+   * hbase.  Otherwise, we use the one in Hadoop.
+   * @return Instance of the TotalOrderPartitioner class
+   * @throws ClassNotFoundException If can't find a TotalOrderPartitioner.
+   */
+  private static Class getTotalOrderPartitionerClass()
+  throws ClassNotFoundException {
+    Class clazz = null;
+    try {
+      clazz = (Class) Class.forName("org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner");
+    } catch (ClassNotFoundException e) {
+      clazz =
+        (Class) Class.forName("org.apache.hadoop.hbase.mapreduce.hadoopbackport.TotalOrderPartitioner");
+    }
+    return clazz;
+  }
+
+  /**
+   * Run inside the task to deserialize column family to compression algorithm
+   * map from the
+   * configuration.
+   *
+   * Package-private for unit tests only.
+   *
+   * @return a map from column family to the name of the configured compression
+   *         algorithm
+   */
+  static Map createFamilyCompressionMap(Configuration conf) {
+    return createFamilyConfValueMap(conf, COMPRESSION_CONF_KEY);
+  }
+
+  private static Map createFamilyBloomMap(Configuration conf) {
+    return createFamilyConfValueMap(conf, BLOOM_TYPE_CONF_KEY);
+  }
+  
+  /**
+   * Run inside the task to deserialize column family to given conf value map.
+   * 
+   * @param conf
+   * @param confName
+   * @return a map of column family to the given configuration value
+   */
+  private static Map createFamilyConfValueMap(Configuration conf, String confName) {
+    Map confValMap = new TreeMap(Bytes.BYTES_COMPARATOR);
+    String confVal = conf.get(confName, "");
+    for (String familyConf : confVal.split("&")) {
+      String[] familySplit = familyConf.split("=");
+      if (familySplit.length != 2) {
+        continue;
+      }
+      try {
+        confValMap.put(URLDecoder.decode(familySplit[0], "UTF-8").getBytes(),
+            URLDecoder.decode(familySplit[1], "UTF-8"));
+      } catch (UnsupportedEncodingException e) {
+        // will not happen with UTF-8 encoding
+        throw new AssertionError(e);
+      }
+    }
+    return confValMap;
+  }
+  
+  /**
+   * Serialize column family to compression algorithm map to configuration.
+   * Invoked while configuring the MR job for incremental load.
+   *
+   * Package-private for unit tests only.
+   *
+   * @throws IOException
+   *           on failure to read column family descriptors
+   */
+  static void configureCompression(HTable table, Configuration conf) throws IOException {
+    StringBuilder compressionConfigValue = new StringBuilder();
+    HTableDescriptor tableDescriptor = table.getTableDescriptor();
+    if(tableDescriptor == null){
+      // could happen with mock table instance
+      return;
+    }
+    Collection families = tableDescriptor.getFamilies();
+    int i = 0;
+    for (HColumnDescriptor familyDescriptor : families) {
+      if (i++ > 0) {
+        compressionConfigValue.append('&');
+      }
+      compressionConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8"));
+      compressionConfigValue.append('=');
+      compressionConfigValue.append(URLEncoder.encode(familyDescriptor.getCompression().getName(), "UTF-8"));
+    }
+    // Get rid of the last ampersand
+    conf.set(COMPRESSION_CONF_KEY, compressionConfigValue.toString());
+  }
+  
+  /**
+   * Serialize column family to bloom type map to configuration.
+   * Invoked while configuring the MR job for incremental load.
+   *
+   * @throws IOException
+   *           on failure to read column family descriptors
+   */
+  static void configureBloomType(HTable table, Configuration conf) throws IOException {
+    HTableDescriptor tableDescriptor = table.getTableDescriptor();
+    if (tableDescriptor == null) {
+      // could happen with mock table instance
+      return;
+    }
+    StringBuilder bloomTypeConfigValue = new StringBuilder();
+    Collection families = tableDescriptor.getFamilies();
+    int i = 0;
+    for (HColumnDescriptor familyDescriptor : families) {
+      if (i++ > 0) {
+        bloomTypeConfigValue.append('&');
+      }
+      bloomTypeConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8"));
+      bloomTypeConfigValue.append('=');
+      String bloomType = familyDescriptor.getBloomFilterType().toString();
+      if (bloomType == null) {
+        bloomType = HColumnDescriptor.DEFAULT_BLOOMFILTER;
+      }
+      bloomTypeConfigValue.append(URLEncoder.encode(bloomType, "UTF-8"));
+    }
+    conf.set(BLOOM_TYPE_CONF_KEY, bloomTypeConfigValue.toString());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java
new file mode 100644
index 0000000..bdac1f4
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java
@@ -0,0 +1,266 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * Simple {@link InputFormat} for {@link HLog} files.
+ */
+public class HLogInputFormat extends InputFormat {
+  private static Log LOG = LogFactory.getLog(HLogInputFormat.class);
+
+  public static String START_TIME_KEY = "hlog.start.time";
+  public static String END_TIME_KEY = "hlog.end.time";
+
+  /**
+   * {@link InputSplit} for {@link HLog} files. Each split represent
+   * exactly one log file.
+   */
+  static class HLogSplit extends InputSplit implements Writable {
+    private String logFileName;
+    private long fileSize;
+    private long startTime;
+    private long endTime;
+
+    /** for serialization */
+    public HLogSplit() {}
+
+    /**
+     * Represent an HLogSplit, i.e. a single HLog file.
+     * Start- and EndTime are managed by the split, so that HLog files can be
+     * filtered before WALEdits are passed to the mapper(s).
+     * @param logFileName
+     * @param fileSize
+     * @param startTime
+     * @param endTime
+     */
+    public HLogSplit(String logFileName, long fileSize, long startTime, long endTime) {
+      this.logFileName = logFileName;
+      this.fileSize = fileSize;
+      this.startTime = startTime;
+      this.endTime = endTime;
+    }
+
+    @Override
+    public long getLength() throws IOException, InterruptedException {
+      return fileSize;
+    }
+
+    @Override
+    public String[] getLocations() throws IOException, InterruptedException {
+      // TODO: Find the data node with the most blocks for this HLog?
+      return new String[] {};
+    }
+
+    public String getLogFileName() {
+      return logFileName;
+    }
+
+    public long getStartTime() {
+      return startTime;
+    }
+
+    public long getEndTime() {
+      return endTime;
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+      logFileName = in.readUTF();
+      fileSize = in.readLong();
+      startTime = in.readLong();
+      endTime = in.readLong();
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+      out.writeUTF(logFileName);
+      out.writeLong(fileSize);
+      out.writeLong(startTime);
+      out.writeLong(endTime);
+    }
+
+    @Override
+    public String toString() {
+      return logFileName + " (" + startTime + ":" + endTime + ") length:" + fileSize;
+    }
+  }
+
+  /**
+   * {@link RecordReader} for an {@link HLog} file.
+   */
+  static class HLogRecordReader extends RecordReader {
+    private HLog.Reader reader = null;
+    private HLog.Entry currentEntry = new HLog.Entry();
+    private long startTime;
+    private long endTime;
+
+    @Override
+    public void initialize(InputSplit split, TaskAttemptContext context)
+        throws IOException, InterruptedException {
+      HLogSplit hsplit = (HLogSplit)split;
+      Path logFile = new Path(hsplit.getLogFileName());
+      Configuration conf = context.getConfiguration();
+      LOG.info("Opening reader for "+split);
+      try {
+        this.reader = HLog.getReader(logFile.getFileSystem(conf), logFile, conf);
+      } catch (EOFException x) {
+        LOG.info("Ignoring corrupted HLog file: " + logFile
+            + " (This is normal when a RegionServer crashed.)");
+      }
+      this.startTime = hsplit.getStartTime();
+      this.endTime = hsplit.getEndTime();
+    }
+
+    @Override
+    public boolean nextKeyValue() throws IOException, InterruptedException {
+      if (reader == null) return false;
+
+      HLog.Entry temp;
+      long i = -1;
+      do {
+        // skip older entries
+        try {
+          temp = reader.next(currentEntry);
+          i++;
+        } catch (EOFException x) {
+          LOG.info("Corrupted entry detected. Ignoring the rest of the file."
+              + " (This is normal when a RegionServer crashed.)");
+          return false;
+        }
+      }
+      while(temp != null && temp.getKey().getWriteTime() < startTime);
+
+      if (temp == null) {
+        if (i > 0) LOG.info("Skipped " + i + " entries.");
+        LOG.info("Reached end of file.");
+        return false;
+      } else if (i > 0) {
+        LOG.info("Skipped " + i + " entries, until ts: " + temp.getKey().getWriteTime() + ".");
+      }
+      boolean res = temp.getKey().getWriteTime() <= endTime;
+      if (!res) {
+        LOG.info("Reached ts: " + temp.getKey().getWriteTime() + " ignoring the rest of the file.");
+      }
+      return res;
+    }
+
+    @Override
+    public HLogKey getCurrentKey() throws IOException, InterruptedException {
+      return currentEntry.getKey();
+    }
+
+    @Override
+    public WALEdit getCurrentValue() throws IOException, InterruptedException {
+      return currentEntry.getEdit();
+    }
+
+    @Override
+    public float getProgress() throws IOException, InterruptedException {
+      // N/A depends on total number of entries, which is unknown
+      return 0;
+    }
+
+    @Override
+    public void close() throws IOException {
+      LOG.info("Closing reader");
+      if (reader != null) this.reader.close();
+    }
+  }
+
+  @Override
+  public List getSplits(JobContext context) throws IOException,
+      InterruptedException {
+    Configuration conf = context.getConfiguration();
+    Path inputDir = new Path(conf.get("mapred.input.dir"));
+
+    long startTime = conf.getLong(START_TIME_KEY, Long.MIN_VALUE);
+    long endTime = conf.getLong(END_TIME_KEY, Long.MAX_VALUE);
+
+    FileSystem fs = inputDir.getFileSystem(conf);
+    List files = getFiles(fs, inputDir, startTime, endTime);
+
+    List splits = new ArrayList(files.size());
+    for (FileStatus file : files) {
+      splits.add(new HLogSplit(file.getPath().toString(), file.getLen(), startTime, endTime));
+    }
+    return splits;
+  }
+
+  private List getFiles(FileSystem fs, Path dir, long startTime, long endTime)
+      throws IOException {
+    List result = new ArrayList();
+    LOG.debug("Scanning " + dir.toString() + " for HLog files");
+
+    FileStatus[] files = fs.listStatus(dir);
+    if (files == null) return Collections.emptyList();
+    for (FileStatus file : files) {
+      if (file.isDir()) {
+        // recurse into sub directories
+        result.addAll(getFiles(fs, file.getPath(), startTime, endTime));
+      } else {
+        String name = file.getPath().toString();
+        int idx = name.lastIndexOf('.');
+        if (idx > 0) {
+          try {
+            long fileStartTime = Long.parseLong(name.substring(idx+1));
+            if (fileStartTime <= endTime) {
+              LOG.info("Found: " + name);
+              result.add(file);
+            }
+          } catch (NumberFormatException x) {
+            idx = 0;
+          }
+        }
+        if (idx == 0) {
+          LOG.warn("File " + name + " does not appear to be an HLog file. Skipping...");
+        }
+      }
+    }
+    return result;
+  }
+
+  @Override
+  public RecordReader createRecordReader(InputSplit split,
+      TaskAttemptContext context) throws IOException, InterruptedException {
+    return new HLogRecordReader();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java
new file mode 100644
index 0000000..363ba51
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java
@@ -0,0 +1,132 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Partitioner;
+
+/**
+ * This is used to partition the output keys into groups of keys.
+ * Keys are grouped according to the regions that currently exist
+ * so that each reducer fills a single region so load is distributed.
+ *
+ * 
        This class is not suitable as partitioner creating hfiles
+ * for incremental bulk loads as region spread will likely change between time of
+ * hfile creation and load time. See {@link LoadIncrementalHFiles}
+ * and Bulk Load.
+ *
+ * @param   The type of the key.
+ * @param   The type of the value.
+ */
+public class HRegionPartitioner
+extends Partitioner
+implements Configurable {
+
+  private final Log LOG = LogFactory.getLog(TableInputFormat.class);
+  private Configuration conf = null;
+  private HTable table;
+  private byte[][] startKeys;
+
+  /**
+   * Gets the partition number for a given key (hence record) given the total
+   * number of partitions i.e. number of reduce-tasks for the job.
+   *
+   * 
        Typically a hash function on a all or a subset of the key.
        
+   *
+   * @param key  The key to be partitioned.
+   * @param value  The entry value.
+   * @param numPartitions  The total number of partitions.
+   * @return The partition number for the key.
+   * @see org.apache.hadoop.mapreduce.Partitioner#getPartition(
+   *   java.lang.Object, java.lang.Object, int)
+   */
+  @Override
+  public int getPartition(ImmutableBytesWritable key,
+      VALUE value, int numPartitions) {
+    byte[] region = null;
+    // Only one region return 0
+    if (this.startKeys.length == 1){
+      return 0;
+    }
+    try {
+      // Not sure if this is cached after a split so we could have problems
+      // here if a region splits while mapping
+      region = table.getRegionLocation(key.get()).getRegionInfo().getStartKey();
+    } catch (IOException e) {
+      LOG.error(e);
+    }
+    for (int i = 0; i < this.startKeys.length; i++){
+      if (Bytes.compareTo(region, this.startKeys[i]) == 0 ){
+        if (i >= numPartitions-1){
+          // cover if we have less reduces then regions.
+          return (Integer.toString(i).hashCode()
+              & Integer.MAX_VALUE) % numPartitions;
+        }
+        return i;
+      }
+    }
+    // if above fails to find start key that match we need to return something
+    return 0;
+  }
+
+  /**
+   * Returns the current configuration.
+   *
+   * @return The current configuration.
+   * @see org.apache.hadoop.conf.Configurable#getConf()
+   */
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  /**
+   * Sets the configuration. This is used to determine the start keys for the
+   * given table.
+   *
+   * @param configuration  The configuration to set.
+   * @see org.apache.hadoop.conf.Configurable#setConf(
+   *   org.apache.hadoop.conf.Configuration)
+   */
+  @Override
+  public void setConf(Configuration configuration) {
+    this.conf = HBaseConfiguration.create(configuration);
+    try {
+      this.table = new HTable(this.conf,
+        configuration.get(TableOutputFormat.OUTPUT_TABLE));
+    } catch (IOException e) {
+      LOG.error(e);
+    }
+    try {
+      this.startKeys = this.table.getStartKeys();
+    } catch (IOException e) {
+      LOG.error(e);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableMapper.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableMapper.java
new file mode 100644
index 0000000..fd5d8fe
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableMapper.java
@@ -0,0 +1,66 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.mapreduce.Job;
+
+/**
+ * Pass the given key and record as-is to the reduce phase.
+ */
+public class IdentityTableMapper
+extends TableMapper {
+
+  /**
+   * Use this before submitting a TableMap job. It will appropriately set up
+   * the job.
+   *
+   * @param table  The table name.
+   * @param scan  The scan with the columns to scan.
+   * @param mapper  The mapper class.
+   * @param job  The job configuration.
+   * @throws IOException When setting up the job fails.
+   */
+  @SuppressWarnings("unchecked")
+  public static void initJob(String table, Scan scan,
+    Class mapper, Job job) throws IOException {
+    TableMapReduceUtil.initTableMapperJob(table, scan, mapper,
+      ImmutableBytesWritable.class, Result.class, job);
+  }
+
+  /**
+   * Pass the key, value to reduce.
+   *
+   * @param key  The current key.
+   * @param value  The current value.
+   * @param context  The current context.
+   * @throws IOException When writing the record fails.
+   * @throws InterruptedException When the job is aborted.
+   */
+  public void map(ImmutableBytesWritable key, Result value, Context context)
+  throws IOException, InterruptedException {
+    context.write(key, value);
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableReducer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableReducer.java
new file mode 100644
index 0000000..25f466e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/IdentityTableReducer.java
@@ -0,0 +1,78 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.OutputFormat;
+
+/**
+ * Convenience class that simply writes all values (which must be
+ * {@link org.apache.hadoop.hbase.client.Put Put} or
+ * {@link org.apache.hadoop.hbase.client.Delete Delete} instances)
+ * passed to it out to the configured HBase table. This works in combination
+ * with {@link TableOutputFormat} which actually does the writing to HBase.
        
+ *
+ * Keys are passed along but ignored in TableOutputFormat.  However, they can
+ * be used to control how your values will be divided up amongst the specified
+ * number of reducers. 
        
+ *
+ * You can also use the {@link TableMapReduceUtil} class to set up the two
+ * classes in one step:
+ * 
        
+ * TableMapReduceUtil.initTableReducerJob("table", IdentityTableReducer.class, job);
+ * 
        
+ * This will also set the proper {@link TableOutputFormat} which is given the
+ * table parameter. The
+ * {@link org.apache.hadoop.hbase.client.Put Put} or
+ * {@link org.apache.hadoop.hbase.client.Delete Delete} define the
+ * row and columns implicitly.
+ */
+public class IdentityTableReducer
+extends TableReducer {
+
+  @SuppressWarnings("unused")
+  private static final Log LOG = LogFactory.getLog(IdentityTableReducer.class);
+
+  /**
+   * Writes each given record, consisting of the row key and the given values,
+   * to the configured {@link OutputFormat}. It is emitting the row key and each
+   * {@link org.apache.hadoop.hbase.client.Put Put} or
+   * {@link org.apache.hadoop.hbase.client.Delete Delete} as separate pairs.
+   *
+   * @param key  The current row key.
+   * @param values  The {@link org.apache.hadoop.hbase.client.Put Put} or
+   *   {@link org.apache.hadoop.hbase.client.Delete Delete} list for the given
+   *   row.
+   * @param context  The context of the reduce.
+   * @throws IOException When writing the record fails.
+   * @throws InterruptedException When the job gets interrupted.
+   */
+  @Override
+  public void reduce(Writable key, Iterable values, Context context)
+  throws IOException, InterruptedException {
+    for(Writable putOrDelete : values) {
+      context.write(key, putOrDelete);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
new file mode 100644
index 0000000..e5a797f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -0,0 +1,455 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Import data written by {@link Export}.
+ */
+public class Import {
+  private static final Log LOG = LogFactory.getLog(Import.class);
+  final static String NAME = "import";
+  final static String CF_RENAME_PROP = "HBASE_IMPORTER_RENAME_CFS";
+  final static String BULK_OUTPUT_CONF_KEY = "import.bulk.output";
+  final static String FILTER_CLASS_CONF_KEY = "import.filter.class";
+  final static String FILTER_ARGS_CONF_KEY = "import.filter.args";
+
+  // Optional filter to use for mappers
+  private static Filter filter;
+
+  /**
+   * A mapper that just writes out KeyValues.
+   */
+  static class KeyValueImporter
+  extends TableMapper {
+    private Map cfRenameMap;
+      
+    /**
+     * @param row  The current table row key.
+     * @param value  The columns.
+     * @param context  The current context.
+     * @throws IOException When something is broken with the data.
+     * @see org.apache.hadoop.mapreduce.Mapper#map(KEYIN, VALUEIN,
+     *   org.apache.hadoop.mapreduce.Mapper.Context)
+     */
+    @Override
+    public void map(ImmutableBytesWritable row, Result value,
+      Context context)
+    throws IOException {
+      try {
+        for (KeyValue kv : value.raw()) {
+          kv = filterKv(kv);
+          // skip if we filtered it out
+          if (kv == null) continue;
+
+          context.write(row, convertKv(kv, cfRenameMap));
+        }
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+
+    @Override
+    public void setup(Context context) {
+      cfRenameMap = createCfRenameMap(context.getConfiguration());
+      filter = instantiateFilter(context.getConfiguration());
+    }
+  }
+
+  /**
+   * Write table content out to files in hdfs.
+   */
+  static class Importer
+  extends TableMapper {
+    private Map cfRenameMap;
+    private UUID clusterId;
+      
+    /**
+     * @param row  The current table row key.
+     * @param value  The columns.
+     * @param context  The current context.
+     * @throws IOException When something is broken with the data.
+     * @see org.apache.hadoop.mapreduce.Mapper#map(KEYIN, VALUEIN,
+     *   org.apache.hadoop.mapreduce.Mapper.Context)
+     */
+    @Override
+    public void map(ImmutableBytesWritable row, Result value,
+      Context context)
+    throws IOException {
+      try {
+        writeResult(row, value, context);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+
+    private void writeResult(ImmutableBytesWritable key, Result result, Context context)
+    throws IOException, InterruptedException {
+      Put put = null;
+      Delete delete = null;
+      for (KeyValue kv : result.raw()) {
+        kv = filterKv(kv);
+        // skip if we filter it out
+        if (kv == null) continue;
+
+        kv = convertKv(kv, cfRenameMap);
+        // Deletes and Puts are gathered and written when finished
+        if (kv.isDelete()) {
+          if (delete == null) {
+            delete = new Delete(key.get());
+          }
+          delete.addDeleteMarker(kv);
+        } else {
+          if (put == null) { 
+            put = new Put(key.get());
+          }
+          put.add(kv);
+        }
+      }
+      if (put != null) {
+        put.setClusterId(clusterId);
+        context.write(key, put);
+      }
+      if (delete != null) {
+        delete.setClusterId(clusterId);
+        context.write(key, delete);
+      }
+    }
+
+    @Override
+    public void setup(Context context) {
+      Configuration conf = context.getConfiguration();
+      cfRenameMap = createCfRenameMap(conf);
+      filter = instantiateFilter(conf);
+
+      try {
+        HConnection connection = HConnectionManager.getConnection(conf);
+        ZooKeeperWatcher zkw = connection.getZooKeeperWatcher();
+        ReplicationZookeeper zkHelper = new ReplicationZookeeper(connection, conf, zkw);
+        clusterId = zkHelper.getUUIDForCluster(zkw);
+      } catch (ZooKeeperConnectionException e) {
+        LOG.error("Problem connecting to ZooKeper during task setup", e);
+      } catch (KeeperException e) {
+        LOG.error("Problem reading ZooKeeper data during task setup", e);
+      } catch (IOException e) {
+        LOG.error("Problem setting up task", e);
+      }
+
+    }
+  }
+
+  /**
+   * Create a {@link Filter} to apply to all incoming keys ({@link KeyValue KeyValues}) to
+   * optionally not include in the job output
+   * @param conf {@link Configuration} from which to load the filter
+   * @return the filter to use for the task, or null if no filter to should be used
+   * @throws IllegalArgumentException if the filter is misconfigured
+   */
+  private static Filter instantiateFilter(Configuration conf) {
+    // get the filter, if it was configured
+    Class filterClass = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class);
+    if (filterClass == null) {
+      LOG.debug("No configured filter class, accepting all keyvalues.");
+      return null;
+    }
+    LOG.debug("Attempting to create filter:" + filterClass);
+
+    try {
+      Method m = filterClass.getMethod("createFilterFromArguments", ArrayList.class);
+      return (Filter) m.invoke(null, getFilterArgs(conf));
+    } catch (IllegalAccessException e) {
+      LOG.error("Couldn't instantiate filter!", e);
+      throw new RuntimeException(e);
+    } catch (SecurityException e) {
+      LOG.error("Couldn't instantiate filter!", e);
+      throw new RuntimeException(e);
+    } catch (NoSuchMethodException e) {
+      LOG.error("Couldn't instantiate filter!", e);
+      throw new RuntimeException(e);
+    } catch (IllegalArgumentException e) {
+      LOG.error("Couldn't instantiate filter!", e);
+      throw new RuntimeException(e);
+    } catch (InvocationTargetException e) {
+      LOG.error("Couldn't instantiate filter!", e);
+      throw new RuntimeException(e);
+    }
+  }
+
+  private static ArrayList getFilterArgs(Configuration conf) {
+    ArrayList args = new ArrayList();
+    String[] sargs = conf.getStrings(FILTER_ARGS_CONF_KEY);
+    for (String arg : sargs) {
+      // all the filters' instantiation methods expected quoted args since they are coming from
+      // the shell, so add them here, though its shouldn't really be needed :-/
+      args.add(Bytes.toBytes("'" + arg + "'"));
+    }
+    return args;
+  }
+
+  /**
+   * Attempt to filter out the keyvalue
+   * @param kv {@link KeyValue} on which to apply the filter
+   * @return null if the key should not be written, otherwise returns the original
+   *         {@link KeyValue}
+   */
+  private static KeyValue filterKv(KeyValue kv) {
+    // apply the filter and skip this kv if the filter doesn't apply
+    if (filter != null) {
+      Filter.ReturnCode code = filter.filterKeyValue(kv);
+      System.out.println("Filter returned:" + code);
+      // if its not an accept type, then skip this kv
+      if (!(code.equals(Filter.ReturnCode.INCLUDE) || code
+          .equals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL))) {
+        if (LOG.isDebugEnabled()) {
+          System.out.println("Skipping key: " + kv + " from filter decision: " + code);
+        }
+        return null;
+      }
+    }
+    return kv;
+  }
+
+  // helper: create a new KeyValue based on CF rename map
+  private static KeyValue convertKv(KeyValue kv, Map cfRenameMap) {
+    if(cfRenameMap != null) {
+      // If there's a rename mapping for this CF, create a new KeyValue
+      byte[] newCfName = cfRenameMap.get(kv.getFamily());
+      if(newCfName != null) {
+          kv = new KeyValue(kv.getBuffer(), // row buffer 
+                  kv.getRowOffset(),        // row offset
+                  kv.getRowLength(),        // row length
+                  newCfName,                // CF buffer
+                  0,                        // CF offset 
+                  newCfName.length,         // CF length 
+                  kv.getBuffer(),           // qualifier buffer
+                  kv.getQualifierOffset(),  // qualifier offset
+                  kv.getQualifierLength(),  // qualifier length
+                  kv.getTimestamp(),        // timestamp
+                  KeyValue.Type.codeToType(kv.getType()), // KV Type
+                  kv.getBuffer(),           // value buffer 
+                  kv.getValueOffset(),      // value offset
+                  kv.getValueLength());     // value length
+      }
+    }
+    return kv;
+  }
+
+  // helper: make a map from sourceCfName to destCfName by parsing a config key
+  private static Map createCfRenameMap(Configuration conf) {
+    Map cfRenameMap = null;
+    String allMappingsPropVal = conf.get(CF_RENAME_PROP);
+    if(allMappingsPropVal != null) {
+      // The conf value format should be sourceCf1:destCf1,sourceCf2:destCf2,...
+      String[] allMappings = allMappingsPropVal.split(",");
+      for (String mapping: allMappings) {
+        if(cfRenameMap == null) {
+            cfRenameMap = new TreeMap(Bytes.BYTES_COMPARATOR);
+        }
+        String [] srcAndDest = mapping.split(":");
+        if(srcAndDest.length != 2) {
+            continue;
+        }
+        cfRenameMap.put(srcAndDest[0].getBytes(), srcAndDest[1].getBytes());
+      }
+    }
+    return cfRenameMap;
+  }
+
+  /**
+   * 
        Sets a configuration property with key {@link #CF_RENAME_PROP} in conf that tells
+   * the mapper how to rename column families.
+   * 
+   * 
        Alternately, instead of calling this function, you could set the configuration key 
+   * {@link #CF_RENAME_PROP} yourself. The value should look like 
+   * 
        srcCf1:destCf1,srcCf2:destCf2,....
        . This would have the same effect on
+   * the mapper behavior.
+   * 
+   * @param conf the Configuration in which the {@link #CF_RENAME_PROP} key will be
+   *  set
+   * @param renameMap a mapping from source CF names to destination CF names
+   */
+  static public void configureCfRenaming(Configuration conf, 
+          Map renameMap) {
+    StringBuilder sb = new StringBuilder();
+    for(Map.Entry entry: renameMap.entrySet()) {
+      String sourceCf = entry.getKey();
+      String destCf = entry.getValue();
+
+      if(sourceCf.contains(":") || sourceCf.contains(",") || 
+              destCf.contains(":") || destCf.contains(",")) {
+        throw new IllegalArgumentException("Illegal character in CF names: " 
+              + sourceCf + ", " + destCf);
+      }
+
+      if(sb.length() != 0) {
+        sb.append(",");
+      }
+      sb.append(sourceCf + ":" + destCf);
+    }
+    conf.set(CF_RENAME_PROP, sb.toString());
+  }
+  
+  /**
+   * Add a Filter to be instantiated on import
+   * @param conf Configuration to update (will be passed to the job)
+   * @param clazz {@link Filter} subclass to instantiate on the server.
+   * @param args List of arguments to pass to the filter on instantiation
+   */
+  public static void addFilterAndArguments(Configuration conf, Class clazz,
+      List args) {
+    conf.set(Import.FILTER_CLASS_CONF_KEY, clazz.getName());
+
+    // build the param string for the key
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < args.size(); i++) {
+      String arg = args.get(i);
+      builder.append(arg);
+      if (i != args.size() - 1) {
+        builder.append(",");
+      }
+    }
+    conf.set(Import.FILTER_ARGS_CONF_KEY, builder.toString());
+  }
+
+  /**
+   * Sets up the actual job.
+   *
+   * @param conf  The current configuration.
+   * @param args  The command line parameters.
+   * @return The newly created job.
+   * @throws IOException When setting up the job fails.
+   */
+  public static Job createSubmittableJob(Configuration conf, String[] args)
+  throws IOException {
+    String tableName = args[0];
+    Path inputDir = new Path(args[1]);
+    Job job = new Job(conf, NAME + "_" + tableName);
+    job.setJarByClass(Importer.class);
+    FileInputFormat.setInputPaths(job, inputDir);
+    job.setInputFormatClass(SequenceFileInputFormat.class);
+    String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY);
+
+    // make sure we get the filter in the jars
+    try {
+      Class filter = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class);
+      if (filter != null) {
+        TableMapReduceUtil.addDependencyJars(conf, filter);
+      }
+    } catch (Exception e) {
+      throw new IOException(e);
+    }
+
+    if (hfileOutPath != null) {
+      job.setMapperClass(KeyValueImporter.class);
+      HTable table = new HTable(conf, tableName);
+      job.setReducerClass(KeyValueSortReducer.class);
+      Path outputDir = new Path(hfileOutPath);
+      FileOutputFormat.setOutputPath(job, outputDir);
+      job.setMapOutputKeyClass(ImmutableBytesWritable.class);
+      job.setMapOutputValueClass(KeyValue.class);
+      HFileOutputFormat.configureIncrementalLoad(job, table);
+      TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+          com.google.common.base.Preconditions.class);
+    } else {
+      // No reducers.  Just write straight to table.  Call initTableReducerJob
+      // because it sets up the TableOutputFormat.
+      job.setMapperClass(Importer.class);
+      TableMapReduceUtil.initTableReducerJob(tableName, null, job);
+      job.setNumReduceTasks(0);
+    }
+    return job;
+  }
+
+  /*
+   * @param errorMsg Error message.  Can be null.
+   */
+  private static void usage(final String errorMsg) {
+    if (errorMsg != null && errorMsg.length() > 0) {
+      System.err.println("ERROR: " + errorMsg);
+    }
+    System.err.println("Usage: Import [options]  ");
+    System.err.println("By default Import will load data directly into HBase. To instead generate");
+    System.err.println("HFiles of data to prepare for a bulk data load, pass the option:");
+    System.err.println("  -D" + BULK_OUTPUT_CONF_KEY + "=/path/for/output");
+    System.err
+        .println(" To apply a generic org.apache.hadoop.hbase.filter.Filter to the input, use");
+    System.err.println("  -D" + FILTER_CLASS_CONF_KEY + "=");
+    System.err.println("  -D" + FILTER_ARGS_CONF_KEY + "= columnStrings = Lists.newArrayList(
+        Splitter.on(',').trimResults().split(columnsSpecification));
+
+      maxColumnCount = columnStrings.size();
+      families = new byte[maxColumnCount][];
+      qualifiers = new byte[maxColumnCount][];
+
+      for (int i = 0; i < columnStrings.size(); i++) {
+        String str = columnStrings.get(i);
+        if (ROWKEY_COLUMN_SPEC.equals(str)) {
+          rowKeyColumnIndex = i;
+          continue;
+        }
+        
+        if (TIMESTAMPKEY_COLUMN_SPEC.equals(str)) {
+          timestampKeyColumnIndex = i;
+          continue;
+        }
+        
+        String[] parts = str.split(":", 2);
+        if (parts.length == 1) {
+          families[i] = str.getBytes();
+          qualifiers[i] = HConstants.EMPTY_BYTE_ARRAY;
+        } else {
+          families[i] = parts[0].getBytes();
+          qualifiers[i] = parts[1].getBytes();
+        }
+      }
+    }
+
+    public boolean hasTimestamp() {
+      return timestampKeyColumnIndex != DEFAULT_TIMESTAMP_COLUMN_INDEX;
+    }
+
+    public int getTimestampKeyColumnIndex() {
+      return timestampKeyColumnIndex;
+    }
+
+    public int getRowKeyColumnIndex() {
+      return rowKeyColumnIndex;
+    }
+    public byte[] getFamily(int idx) {
+      return families[idx];
+    }
+    public byte[] getQualifier(int idx) {
+      return qualifiers[idx];
+    }
+
+    public ParsedLine parse(byte[] lineBytes, int length)
+    throws BadTsvLineException {
+      // Enumerate separator offsets
+      ArrayList tabOffsets = new ArrayList(maxColumnCount);
+      for (int i = 0; i < length; i++) {
+        if (lineBytes[i] == separatorByte) {
+          tabOffsets.add(i);
+        }
+      }
+      if (tabOffsets.isEmpty()) {
+        throw new BadTsvLineException("No delimiter");
+      }
+
+      tabOffsets.add(length);
+
+      if (tabOffsets.size() > maxColumnCount) {
+        throw new BadTsvLineException("Excessive columns");
+      } else if (tabOffsets.size() <= getRowKeyColumnIndex()) {
+        throw new BadTsvLineException("No row key");
+      } else if (hasTimestamp()
+          && tabOffsets.size() <= getTimestampKeyColumnIndex()) {
+        throw new BadTsvLineException("No timestamp");
+      }
+      return new ParsedLine(tabOffsets, lineBytes);
+    }
+
+    class ParsedLine {
+      private final ArrayList tabOffsets;
+      private byte[] lineBytes;
+
+      ParsedLine(ArrayList tabOffsets, byte[] lineBytes) {
+        this.tabOffsets = tabOffsets;
+        this.lineBytes = lineBytes;
+      }
+
+      public int getRowKeyOffset() {
+        return getColumnOffset(rowKeyColumnIndex);
+      }
+      public int getRowKeyLength() {
+        return getColumnLength(rowKeyColumnIndex);
+      }
+      
+      public long getTimestamp(long ts) throws BadTsvLineException {
+        // Return ts if HBASE_TS_KEY is not configured in column spec
+        if (!hasTimestamp()) {
+          return ts;
+        }
+
+        String timeStampStr = Bytes.toString(lineBytes,
+            getColumnOffset(timestampKeyColumnIndex),
+            getColumnLength(timestampKeyColumnIndex));
+        try {
+          return Long.parseLong(timeStampStr);
+        } catch (NumberFormatException nfe) {
+          // treat this record as bad record
+          throw new BadTsvLineException("Invalid timestamp " + timeStampStr);
+        }
+      }
+      
+      public int getColumnOffset(int idx) {
+        if (idx > 0)
+          return tabOffsets.get(idx - 1) + 1;
+        else
+          return 0;
+      }
+      public int getColumnLength(int idx) {
+        return tabOffsets.get(idx) - getColumnOffset(idx);
+      }
+      public int getColumnCount() {
+        return tabOffsets.size();
+      }
+      public byte[] getLineBytes() {
+        return lineBytes;
+      }
+    }
+
+    public static class BadTsvLineException extends Exception {
+      public BadTsvLineException(String err) {
+        super(err);
+      }
+      private static final long serialVersionUID = 1L;
+    }
+  }
+
+  /**
+   * Sets up the actual job.
+   *
+   * @param conf  The current configuration.
+   * @param args  The command line parameters.
+   * @return The newly created job.
+   * @throws IOException When setting up the job fails.
+   */
+  public static Job createSubmittableJob(Configuration conf, String[] args)
+  throws IOException, ClassNotFoundException {
+
+    // Support non-XML supported characters
+    // by re-encoding the passed separator as a Base64 string.
+    String actualSeparator = conf.get(SEPARATOR_CONF_KEY);
+    if (actualSeparator != null) {
+      conf.set(SEPARATOR_CONF_KEY,
+               Base64.encodeBytes(actualSeparator.getBytes()));
+    }
+
+    // See if a non-default Mapper was set
+    String mapperClassName = conf.get(MAPPER_CONF_KEY);
+    Class mapperClass = mapperClassName != null ?
+        Class.forName(mapperClassName) : DEFAULT_MAPPER;
+
+    String tableName = args[0];
+    Path inputDir = new Path(args[1]);
+    Job job = new Job(conf, NAME + "_" + tableName);
+    job.setJarByClass(mapperClass);
+    FileInputFormat.setInputPaths(job, inputDir);
+    job.setInputFormatClass(TextInputFormat.class);
+    job.setMapperClass(mapperClass);
+
+    String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY);
+    if (hfileOutPath != null) {
+      if (!doesTableExist(tableName)) {
+        createTable(conf, tableName);
+      }
+      HTable table = new HTable(conf, tableName);
+      job.setReducerClass(PutSortReducer.class);
+      Path outputDir = new Path(hfileOutPath);
+      FileOutputFormat.setOutputPath(job, outputDir);
+      job.setMapOutputKeyClass(ImmutableBytesWritable.class);
+      job.setMapOutputValueClass(Put.class);
+      HFileOutputFormat.configureIncrementalLoad(job, table);
+    } else {
+      // No reducers.  Just write straight to table.  Call initTableReducerJob
+      // to set up the TableOutputFormat.
+      TableMapReduceUtil.initTableReducerJob(tableName, null, job);
+      job.setNumReduceTasks(0);
+    }
+
+    TableMapReduceUtil.addDependencyJars(job);
+    TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+        com.google.common.base.Function.class /* Guava used by TsvParser */);
+    return job;
+  }
+
+  private static boolean doesTableExist(String tableName) throws IOException {
+    return hbaseAdmin.tableExists(tableName.getBytes());
+  }
+
+  private static void createTable(Configuration conf, String tableName)
+      throws IOException {
+    HTableDescriptor htd = new HTableDescriptor(tableName.getBytes());
+    String columns[] = conf.getStrings(COLUMNS_CONF_KEY);
+    Set cfSet = new HashSet();
+    for (String aColumn : columns) {
+      if (TsvParser.ROWKEY_COLUMN_SPEC.equals(aColumn)) continue;
+      // we are only concerned with the first one (in case this is a cf:cq)
+      cfSet.add(aColumn.split(":", 2)[0]);
+    }
+    for (String cf : cfSet) {
+      HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes(cf));
+      htd.addFamily(hcd);
+    }
+    hbaseAdmin.createTable(htd);
+  }
+
+  /*
+   * @param errorMsg Error message.  Can be null.
+   */
+  private static void usage(final String errorMsg) {
+    if (errorMsg != null && errorMsg.length() > 0) {
+      System.err.println("ERROR: " + errorMsg);
+    }
+    String usage = 
+      "Usage: " + NAME + " -Dimporttsv.columns=a,b,c  \n" +
+      "\n" +
+      "Imports the given input directory of TSV data into the specified table.\n" +
+      "\n" +
+      "The column names of the TSV data must be specified using the -Dimporttsv.columns\n" +
+      "option. This option takes the form of comma-separated column names, where each\n" +
+      "column name is either a simple column family, or a columnfamily:qualifier. The special\n" +
+      "column name HBASE_ROW_KEY is used to designate that this column should be used\n" +
+      "as the row key for each imported record. You must specify exactly one column\n" +
+      "to be the row key, and you must specify a column name for every column that exists in the\n" +
+      "input data. Another special column HBASE_TS_KEY designates that this column should be\n" +
+      "used as timestamp for each record. Unlike HBASE_ROW_KEY, HBASE_TS_KEY is optional.\n" +
+      "You must specify atmost one column as timestamp key for each imported record.\n" +
+      "Record with invalid timestamps (blank, non-numeric) will be treated as bad record.\n" +
+      "Note: if you use this option, then 'importtsv.timestamp' option will be ignored.\n" +
+      "\n" +
+      "By default importtsv will load data directly into HBase. To instead generate\n" +
+      "HFiles of data to prepare for a bulk data load, pass the option:\n" +
+      "  -D" + BULK_OUTPUT_CONF_KEY + "=/path/for/output\n" +
+      "  Note: if you do not use this option, then the target table must already exist in HBase\n" +
+      "\n" +
+      "Other options that may be specified with -D include:\n" +
+      "  -D" + SKIP_LINES_CONF_KEY + "=false - fail if encountering an invalid line\n" +
+      "  '-D" + SEPARATOR_CONF_KEY + "=|' - eg separate on pipes instead of tabs\n" +
+      "  -D" + TIMESTAMP_CONF_KEY + "=currentTimeAsLong - use the specified timestamp for the import\n" +
+      "  -D" + MAPPER_CONF_KEY + "=my.Mapper - A user-defined Mapper to use instead of " + DEFAULT_MAPPER.getName() + "\n" +
+      "For performance consider the following options:\n" +
+      "  -Dmapred.map.tasks.speculative.execution=false\n" +
+      "  -Dmapred.reduce.tasks.speculative.execution=false";
+
+    System.err.println(usage);
+  }
+
+  /**
+   * Used only by test method
+   * @param conf
+   */
+  static void createHbaseAdmin(Configuration conf) throws IOException {
+    hbaseAdmin = new HBaseAdmin(conf);
+  }
+
+  /**
+   * Main entry point.
+   *
+   * @param args  The command line parameters.
+   * @throws Exception When running the job fails.
+   */
+  public static void main(String[] args) throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+    if (otherArgs.length < 2) {
+      usage("Wrong number of arguments: " + otherArgs.length);
+      System.exit(-1);
+    }
+
+    // Make sure columns are specified
+    String columns[] = conf.getStrings(COLUMNS_CONF_KEY);
+    if (columns == null) {
+      usage("No columns specified. Please specify with -D" +
+            COLUMNS_CONF_KEY+"=...");
+      System.exit(-1);
+    }
+
+    // Make sure they specify exactly one column as the row key
+    int rowkeysFound=0;
+    for (String col : columns) {
+      if (col.equals(TsvParser.ROWKEY_COLUMN_SPEC)) rowkeysFound++;
+    }
+    if (rowkeysFound != 1) {
+      usage("Must specify exactly one column as " + TsvParser.ROWKEY_COLUMN_SPEC);
+      System.exit(-1);
+    }
+
+    // Make sure we have at most one column as the timestamp key
+    int tskeysFound = 0;
+    for (String col : columns) {
+      if (col.equals(TsvParser.TIMESTAMPKEY_COLUMN_SPEC))
+        tskeysFound++;
+    }
+    if (tskeysFound > 1) {
+      usage("Must specify at most one column as "
+          + TsvParser.TIMESTAMPKEY_COLUMN_SPEC);
+      System.exit(-1);
+    }
+    
+    // Make sure one or more columns are specified excluding rowkey and
+    // timestamp key
+    if (columns.length - (rowkeysFound + tskeysFound) < 1) {
+      usage("One or more columns in addition to the row key and timestamp(optional) are required");
+      System.exit(-1);
+    }
+
+    // If timestamp option is not specified, use current system time.
+    long timstamp = conf
+        .getLong(TIMESTAMP_CONF_KEY, System.currentTimeMillis());
+
+    // Set it back to replace invalid timestamp (non-numeric) with current
+    // system time
+    conf.setLong(TIMESTAMP_CONF_KEY, timstamp); 
+    
+    hbaseAdmin = new HBaseAdmin(conf);
+    Job job = createSubmittableJob(conf, otherArgs);
+    System.exit(job.waitForCompletion(true) ? 0 : 1);
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSortReducer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSortReducer.java
new file mode 100644
index 0000000..1f1567e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSortReducer.java
@@ -0,0 +1,50 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.mapreduce.Reducer;
+
+/**
+ * Emits sorted KeyValues.
+ * Reads in all KeyValues from passed Iterator, sorts them, then emits
+ * KeyValues in sorted order.  If lots of columns per row, it will use lots of
+ * memory sorting.
+ * @see HFileOutputFormat
+ */
+public class KeyValueSortReducer extends Reducer {
+  protected void reduce(ImmutableBytesWritable row, java.lang.Iterable kvs,
+      org.apache.hadoop.mapreduce.Reducer.Context context)
+  throws java.io.IOException, InterruptedException {
+    TreeSet map = new TreeSet(KeyValue.COMPARATOR);
+    for (KeyValue kv: kvs) {
+      map.add(kv.clone());
+    }
+    context.setStatus("Read " + map.getClass());
+    int index = 0;
+    for (KeyValue kv: map) {
+      context.write(row, kv);
+      if (index > 0 && index % 100 == 0) context.setStatus("Wrote " + index);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
new file mode 100644
index 0000000..7366469
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -0,0 +1,789 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.ServerCallable;
+import org.apache.hadoop.hbase.coprocessor.SecureBulkLoadClient;
+import org.apache.hadoop.hbase.io.HalfStoreFileReader;
+import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.io.Reference.Range;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * Tool to load the output of HFileOutputFormat into an existing table.
+ * @see #usage()
+ */
+public class LoadIncrementalHFiles extends Configured implements Tool {
+
+  private static Log LOG = LogFactory.getLog(LoadIncrementalHFiles.class);
+  private static final int  TABLE_CREATE_MAX_RETRIES = 20;
+  private static final long TABLE_CREATE_SLEEP = 60000;
+  static AtomicLong regionCount = new AtomicLong(0);
+  private HBaseAdmin hbAdmin;
+  private Configuration cfg;
+
+  public static String NAME = "completebulkload";
+
+  private boolean useSecure;
+  private Token userToken;
+  private String bulkToken;
+
+  //package private for testing
+  LoadIncrementalHFiles(Configuration conf, Boolean useSecure) throws Exception {
+    super(conf);
+    this.cfg = conf;
+    this.hbAdmin = new HBaseAdmin(conf);
+    //added simple for testing
+    this.useSecure = useSecure != null ? useSecure : User.isHBaseSecurityEnabled(conf);
+  }
+
+  public LoadIncrementalHFiles(Configuration conf) throws Exception {
+    this(conf, null);
+  }
+
+  private void usage() {
+    System.err.println("usage: " + NAME +
+        " /path/to/hfileoutputformat-output " +
+        "tablename");
+  }
+
+  /**
+   * Represents an HFile waiting to be loaded. An queue is used
+   * in this class in order to support the case where a region has
+   * split during the process of the load. When this happens,
+   * the HFile is split into two physical parts across the new
+   * region boundary, and each part is added back into the queue.
+   * The import process finishes when the queue is empty.
+   */
+  static class LoadQueueItem {
+    final byte[] family;
+    final Path hfilePath;
+
+    public LoadQueueItem(byte[] family, Path hfilePath) {
+      this.family = family;
+      this.hfilePath = hfilePath;
+    }
+
+    public String toString() {
+      return "family:"+ Bytes.toString(family) + " path:" + hfilePath.toString();
+    }
+  }
+
+  /**
+   * Walk the given directory for all HFiles, and return a Queue
+   * containing all such files.
+   */
+  private void discoverLoadQueue(Deque ret, Path hfofDir)
+  throws IOException {
+    FileSystem fs = hfofDir.getFileSystem(getConf());
+
+    if (!fs.exists(hfofDir)) {
+      throw new FileNotFoundException("HFileOutputFormat dir " +
+          hfofDir + " not found");
+    }
+
+    FileStatus[] familyDirStatuses = fs.listStatus(hfofDir);
+    if (familyDirStatuses == null) {
+      throw new FileNotFoundException("No families found in " + hfofDir);
+    }
+
+    for (FileStatus stat : familyDirStatuses) {
+      if (!stat.isDir()) {
+        LOG.warn("Skipping non-directory " + stat.getPath());
+        continue;
+      }
+      Path familyDir = stat.getPath();
+      // Skip _logs, etc
+      if (familyDir.getName().startsWith("_")) continue;
+      byte[] family = familyDir.getName().getBytes();
+      Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
+      for (Path hfile : hfiles) {
+        if (hfile.getName().startsWith("_")) continue;
+        ret.add(new LoadQueueItem(family, hfile));
+      }
+    }
+  }
+
+  /**
+   * Perform a bulk load of the given directory into the given
+   * pre-existing table.  This method is not threadsafe.
+   * 
+   * @param hfofDir the directory that was provided as the output path
+   * of a job using HFileOutputFormat
+   * @param table the table to load into
+   * @throws TableNotFoundException if table does not yet exist
+   */
+  public void doBulkLoad(Path hfofDir, final HTable table)
+    throws TableNotFoundException, IOException
+  {
+    final HConnection conn = table.getConnection();
+
+    if (!conn.isTableAvailable(table.getTableName())) {
+      throw new TableNotFoundException("Table " +
+          Bytes.toStringBinary(table.getTableName()) +
+          "is not currently available.");
+    }
+
+    // initialize thread pools
+    int nrThreads = cfg.getInt("hbase.loadincremental.threads.max",
+        Runtime.getRuntime().availableProcessors());
+    ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
+    builder.setNameFormat("LoadIncrementalHFiles-%1$d");
+    ExecutorService pool = new ThreadPoolExecutor(nrThreads, nrThreads,
+        60, TimeUnit.SECONDS,
+        new LinkedBlockingQueue(),
+        builder.build());
+    ((ThreadPoolExecutor)pool).allowCoreThreadTimeOut(true);
+
+    // LQI queue does not need to be threadsafe -- all operations on this queue
+    // happen in this thread
+    Deque queue = new LinkedList();
+    try {
+      discoverLoadQueue(queue, hfofDir);
+      int count = 0;
+
+      if (queue.isEmpty()) {
+        LOG.warn("Bulk load operation did not find any files to load in " +
+            "directory " + hfofDir.toUri() + ".  Does it contain files in " +
+            "subdirectories that correspond to column family names?");
+        return;
+      }
+
+      //If using secure bulk load
+      //prepare staging directory and token
+      if(useSecure) {
+        //This condition is here for unit testing
+        //Since delegation token doesn't work in mini cluster
+        if(User.isSecurityEnabled()) {
+          FileSystem fs = FileSystem.get(cfg);
+          userToken = fs.getDelegationToken("renewer");
+        }
+        bulkToken = new SecureBulkLoadClient(table).prepareBulkLoad(table.getTableName());
+      }
+
+      // Assumes that region splits can happen while this occurs.
+      while (!queue.isEmpty()) {
+        // need to reload split keys each iteration.
+        final Pair startEndKeys = table.getStartEndKeys();
+        if (count != 0) {
+          LOG.info("Split occured while grouping HFiles, retry attempt " +
+              + count + " with " + queue.size() + " files remaining to group or split");
+        }
+
+        int maxRetries = cfg.getInt("hbase.bulkload.retries.number", 0);
+        if (maxRetries != 0 && count >= maxRetries) {
+          LOG.error("Retry attempted " + count +  " times without completing, bailing out");
+          return;
+        }
+        count++;
+
+        // Using ByteBuffer for byte[] equality semantics
+        Multimap regionGroups = groupOrSplitPhase(table,
+            pool, queue, startEndKeys);
+
+        bulkLoadPhase(table, conn, pool, queue, regionGroups);
+
+        // NOTE: The next iteration's split / group could happen in parallel to
+        // atomic bulkloads assuming that there are splits and no merges, and
+        // that we can atomically pull out the groups we want to retry.
+      }
+
+    } finally {
+      if(useSecure) {
+        if(userToken != null) {
+          try {
+            userToken.cancel(cfg);
+          } catch (Exception e) {
+            LOG.warn("Failed to cancel HDFS delegation token.", e);
+          }
+        }
+        if(bulkToken != null) {
+          new SecureBulkLoadClient(table).cleanupBulkLoad(bulkToken);
+        }
+      }
+      pool.shutdown();
+      if (queue != null && !queue.isEmpty()) {
+        StringBuilder err = new StringBuilder();
+        err.append("-------------------------------------------------\n");
+        err.append("Bulk load aborted with some files not yet loaded:\n");
+        err.append("-------------------------------------------------\n");
+        for (LoadQueueItem q : queue) {
+          err.append("  ").append(q.hfilePath).append('\n');
+        }
+        LOG.error(err);
+      }
+    }
+    
+    if (queue != null && !queue.isEmpty()) {
+        throw new RuntimeException("Bulk load aborted with some files not yet loaded."
+          + "Please check log for more details.");
+    }
+  }
+
+  /**
+   * This takes the LQI's grouped by likely regions and attempts to bulk load
+   * them.  Any failures are re-queued for another pass with the
+   * groupOrSplitPhase.
+   */
+  protected void bulkLoadPhase(final HTable table, final HConnection conn,
+      ExecutorService pool, Deque queue,
+      final Multimap regionGroups) throws IOException {
+    // atomically bulk load the groups.
+    Set>> loadingFutures = new HashSet>>();
+    for (Entry> e: regionGroups.asMap().entrySet()) {
+      final byte[] first = e.getKey().array();
+      final Collection lqis =  e.getValue();
+
+      final Callable> call = new Callable>() {
+        public List call() throws Exception {
+          List toRetry = tryAtomicRegionLoad(conn, table.getTableName(), first, lqis);
+          return toRetry;
+        }
+      };
+      loadingFutures.add(pool.submit(call));
+    }
+
+    // get all the results.
+    for (Future> future : loadingFutures) {
+      try {
+        List toRetry = future.get();
+
+        // LQIs that are requeued to be regrouped.
+        queue.addAll(toRetry);
+
+      } catch (ExecutionException e1) {
+        Throwable t = e1.getCause();
+        if (t instanceof IOException) {
+          // At this point something unrecoverable has happened.
+          // TODO Implement bulk load recovery
+          throw new IOException("BulkLoad encountered an unrecoverable problem", t);
+        }
+        LOG.error("Unexpected execution exception during bulk load", e1);
+        throw new IllegalStateException(t);
+      } catch (InterruptedException e1) {
+        LOG.error("Unexpected interrupted exception during bulk load", e1);
+        throw new IllegalStateException(e1);
+      }
+    }
+  }
+
+  /**
+   * @return A Multimap that groups LQI by likely
+   * bulk load region targets.
+   */
+  private Multimap groupOrSplitPhase(final HTable table,
+      ExecutorService pool, Deque queue,
+      final Pair startEndKeys) throws IOException {
+    //  need synchronized only within this scope of this
+    // phase because of the puts that happen in futures.
+    Multimap rgs = HashMultimap.create();
+    final Multimap regionGroups = Multimaps.synchronizedMultimap(rgs);
+
+    // drain LQIs and figure out bulk load groups
+    Set>> splittingFutures = new HashSet>>();
+    while (!queue.isEmpty()) {
+      final LoadQueueItem item = queue.remove();
+      
+      final Callable> call = new Callable>() {
+        public List call() throws Exception {
+          List splits = groupOrSplit(regionGroups, item, table, startEndKeys);
+          return splits;
+        }
+      };
+      splittingFutures.add(pool.submit(call));
+    }
+    // get all the results.  All grouping and splitting must finish before
+    // we can attempt the atomic loads.
+    for (Future> lqis : splittingFutures) {
+      try {
+        List splits = lqis.get();
+        if (splits != null) {
+          queue.addAll(splits);
+        }
+      } catch (ExecutionException e1) {
+        Throwable t = e1.getCause();
+        if (t instanceof IOException) {
+          LOG.error("IOException during splitting", e1);
+          throw (IOException)t; // would have been thrown if not parallelized,
+        }
+        LOG.error("Unexpected execution exception during splitting", e1);
+        throw new IllegalStateException(t);
+      } catch (InterruptedException e1) {
+        LOG.error("Unexpected interrupted exception during splitting", e1);
+        throw new IllegalStateException(e1);
+      }
+    }
+    return regionGroups;
+  }
+
+  // unique file name for the table
+  String getUniqueName(byte[] tableName) {
+    String name = Bytes.toStringBinary(tableName) + "," + regionCount.incrementAndGet();
+    return name;
+  }
+
+  protected List splitStoreFile(final LoadQueueItem item,
+      final HTable table, byte[] startKey,
+      byte[] splitKey) throws IOException {
+    final Path hfilePath = item.hfilePath;
+
+    // We use a '_' prefix which is ignored when walking directory trees
+    // above.
+    final Path tmpDir = new Path(item.hfilePath.getParent(), "_tmp");
+
+    LOG.info("HFile at " + hfilePath + " no longer fits inside a single " +
+    "region. Splitting...");
+
+    String uniqueName = getUniqueName(table.getTableName());
+    HColumnDescriptor familyDesc = table.getTableDescriptor().getFamily(item.family);
+    Path botOut = new Path(tmpDir, uniqueName + ".bottom");
+    Path topOut = new Path(tmpDir, uniqueName + ".top");
+    splitStoreFile(getConf(), hfilePath, familyDesc, splitKey,
+        botOut, topOut);
+
+    // Add these back at the *front* of the queue, so there's a lower
+    // chance that the region will just split again before we get there.
+    List lqis = new ArrayList(2);
+    lqis.add(new LoadQueueItem(item.family, botOut));
+    lqis.add(new LoadQueueItem(item.family, topOut));
+
+    LOG.info("Successfully split into new HFiles " + botOut + " and " + topOut);
+    return lqis;
+  }
+
+  /**
+   * Attempt to assign the given load queue item into its target region group.
+   * If the hfile boundary no longer fits into a region, physically splits
+   * the hfile such that the new bottom half will fit and returns the list of
+   * LQI's corresponding to the resultant hfiles.
+   *
+   * protected for testing
+   */
+  protected List groupOrSplit(Multimap regionGroups,
+      final LoadQueueItem item, final HTable table,
+      final Pair startEndKeys)
+      throws IOException {
+    final Path hfilePath = item.hfilePath;
+    final FileSystem fs = hfilePath.getFileSystem(getConf());
+    HFile.Reader hfr = HFile.createReader(fs, hfilePath,
+        new CacheConfig(getConf()));
+    final byte[] first, last;
+    try {
+      hfr.loadFileInfo();
+      first = hfr.getFirstRowKey();
+      last = hfr.getLastRowKey();
+    }  finally {
+      hfr.close();
+    }
+
+    LOG.info("Trying to load hfile=" + hfilePath +
+        " first=" + Bytes.toStringBinary(first) +
+        " last="  + Bytes.toStringBinary(last));
+    if (first == null || last == null) {
+      assert first == null && last == null;
+      // TODO what if this is due to a bad HFile?
+      LOG.info("hfile " + hfilePath + " has no entries, skipping");
+      return null;
+    }
+    if (Bytes.compareTo(first, last) > 0) {
+      throw new IllegalArgumentException(
+      "Invalid range: " + Bytes.toStringBinary(first) +
+      " > " + Bytes.toStringBinary(last));
+    }
+    int idx = Arrays.binarySearch(startEndKeys.getFirst(), first,
+        Bytes.BYTES_COMPARATOR);
+    if (idx < 0) {
+      // not on boundary, returns -(insertion index).  Calculate region it
+      // would be in.
+      idx = -(idx + 1) - 1;
+    }
+    final int indexForCallable = idx;
+    boolean lastKeyInRange =
+      Bytes.compareTo(last, startEndKeys.getSecond()[idx]) < 0 ||
+      Bytes.equals(startEndKeys.getSecond()[idx], HConstants.EMPTY_BYTE_ARRAY);
+    if (!lastKeyInRange) {
+      List lqis = splitStoreFile(item, table,
+          startEndKeys.getFirst()[indexForCallable],
+          startEndKeys.getSecond()[indexForCallable]);
+      return lqis;
+    }
+
+    // group regions.
+    regionGroups.put(ByteBuffer.wrap(startEndKeys.getFirst()[idx]), item);
+    return null;
+  }
+
+  /**
+   * Attempts to do an atomic load of many hfiles into a region.  If it fails,
+   * it returns a list of hfiles that need to be retried.  If it is successful
+   * it will return an empty list.
+   * 
+   * NOTE: To maintain row atomicity guarantees, region server callable should
+   * succeed atomically and fails atomically.
+   * 
+   * Protected for testing.
+   * 
+   * @return empty list if success, list of items to retry on recoverable
+   * failure
+   */
+  protected List tryAtomicRegionLoad(final HConnection conn,
+      byte[] tableName, final byte[] first, Collection lqis) throws IOException {
+
+    final List> famPaths =
+      new ArrayList>(lqis.size());
+    for (LoadQueueItem lqi : lqis) {
+      famPaths.add(Pair.newPair(lqi.family, lqi.hfilePath.toString()));
+    }
+
+    final ServerCallable svrCallable = new ServerCallable(conn,
+        tableName, first) {
+      @Override
+      public Boolean call() throws Exception {
+        SecureBulkLoadClient secureClient = null;
+        boolean success = false;
+
+        try {
+          LOG.debug("Going to connect to server " + location + " for row "
+              + Bytes.toStringBinary(row));
+          byte[] regionName = location.getRegionInfo().getRegionName();
+          if(!useSecure) {
+            success = server.bulkLoadHFiles(famPaths, regionName);
+          } else {
+            HTable table = new HTable(conn.getConfiguration(), tableName);
+            secureClient = new SecureBulkLoadClient(table, location.getRegionInfo().getStartKey());
+            success = secureClient.bulkLoadHFiles(famPaths, userToken, bulkToken);
+          }
+          return success;
+        } finally {
+          //Best effort copying of files that might not have been imported
+          //from the staging directory back to original location
+          //in user directory
+          if(secureClient != null && !success) {
+            FileSystem fs = FileSystem.get(cfg);
+            for(Pair el : famPaths) {
+              Path hfileStagingPath = null;
+              Path hfileOrigPath = new Path(el.getSecond());
+              try {
+                hfileStagingPath= new Path(secureClient.getStagingPath(bulkToken, el.getFirst()),
+                    hfileOrigPath.getName());
+                if(fs.rename(hfileStagingPath, hfileOrigPath)) {
+                  LOG.debug("Moved back file " + hfileOrigPath + " from " +
+                      hfileStagingPath);
+                } else if(fs.exists(hfileStagingPath)){
+                  LOG.debug("Unable to move back file " + hfileOrigPath + " from " +
+                      hfileStagingPath);
+                }
+              } catch(Exception ex) {
+                LOG.debug("Unable to move back file " + hfileOrigPath + " from " +
+                    hfileStagingPath, ex);
+              }
+            }
+          }
+        }
+      }
+
+    };
+
+    try {
+      List toRetry = new ArrayList();
+      boolean success = svrCallable.withRetries();
+      if (!success) {
+        LOG.warn("Attempt to bulk load region containing "
+            + Bytes.toStringBinary(first) + " into table "
+            + Bytes.toStringBinary(tableName)  + " with files " + lqis
+            + " failed.  This is recoverable and they will be retried.");
+        toRetry.addAll(lqis); // return lqi's to retry
+      }
+      // success
+      return toRetry;
+    } catch (IOException e) {
+      LOG.error("Encountered unrecoverable error from region server", e);
+      throw e;
+    }
+  }
+
+  /**
+   * Split a storefile into a top and bottom half, maintaining
+   * the metadata, recreating bloom filters, etc.
+   */
+  static void splitStoreFile(
+      Configuration conf, Path inFile,
+      HColumnDescriptor familyDesc, byte[] splitKey,
+      Path bottomOut, Path topOut) throws IOException
+  {
+    // Open reader with no block cache, and not in-memory
+    Reference topReference = new Reference(splitKey, Range.top);
+    Reference bottomReference = new Reference(splitKey, Range.bottom);
+
+    copyHFileHalf(conf, inFile, topOut, topReference, familyDesc);
+    copyHFileHalf(conf, inFile, bottomOut, bottomReference, familyDesc);
+  }
+
+  /**
+   * Copy half of an HFile into a new HFile.
+   */
+  private static void copyHFileHalf(
+      Configuration conf, Path inFile, Path outFile, Reference reference,
+      HColumnDescriptor familyDescriptor)
+  throws IOException {
+    FileSystem fs = inFile.getFileSystem(conf);
+    CacheConfig cacheConf = new CacheConfig(conf);
+    HalfStoreFileReader halfReader = null;
+    StoreFile.Writer halfWriter = null;
+    HFileDataBlockEncoder dataBlockEncoder = new HFileDataBlockEncoderImpl(
+        familyDescriptor.getDataBlockEncodingOnDisk(),
+        familyDescriptor.getDataBlockEncoding());
+    try {
+      halfReader = new HalfStoreFileReader(fs, inFile, cacheConf,
+          reference, DataBlockEncoding.NONE);
+      Map fileInfo = halfReader.loadFileInfo();
+
+      int blocksize = familyDescriptor.getBlocksize();
+      Algorithm compression = familyDescriptor.getCompression();
+      BloomType bloomFilterType = familyDescriptor.getBloomFilterType();
+
+      halfWriter = new StoreFile.WriterBuilder(conf, cacheConf,
+          fs, blocksize)
+              .withFilePath(outFile)
+              .withCompression(compression)
+              .withDataBlockEncoder(dataBlockEncoder)
+              .withBloomType(bloomFilterType)
+              .withChecksumType(Store.getChecksumType(conf))
+              .withBytesPerChecksum(Store.getBytesPerChecksum(conf))
+              .build();
+      HFileScanner scanner = halfReader.getScanner(false, false, false);
+      scanner.seekTo();
+      do {
+        KeyValue kv = scanner.getKeyValue();
+        halfWriter.append(kv);
+      } while (scanner.next());
+
+      for (Map.Entry entry : fileInfo.entrySet()) {
+        if (shouldCopyHFileMetaKey(entry.getKey())) {
+          halfWriter.appendFileInfo(entry.getKey(), entry.getValue());
+        }
+      }
+    } finally {
+      if (halfWriter != null) halfWriter.close();
+      if (halfReader != null) halfReader.close(cacheConf.shouldEvictOnClose());
+    }
+  }
+
+  private static boolean shouldCopyHFileMetaKey(byte[] key) {
+    return !HFile.isReservedFileInfoKey(key);
+  }
+
+  private boolean doesTableExist(String tableName) throws Exception {
+    return hbAdmin.tableExists(tableName);
+  }
+  
+  /*
+   * Infers region boundaries for a new table.
+   * Parameter:
+   *   bdryMap is a map between keys to an integer belonging to {+1, -1}
+   *     If a key is a start key of a file, then it maps to +1
+   *     If a key is an end key of a file, then it maps to -1
+   * Algo:
+   * 1) Poll on the keys in order: 
+   *    a) Keep adding the mapped values to these keys (runningSum) 
+   *    b) Each time runningSum reaches 0, add the start Key from when the runningSum had started to a boundary list.
+   * 2) Return the boundary list. 
+   */
+  public static byte[][] inferBoundaries(TreeMap bdryMap) {
+    ArrayList keysArray = new ArrayList();
+    int runningValue = 0;
+    byte[] currStartKey = null;
+    boolean firstBoundary = true;
+    
+    for (Map.Entry item: bdryMap.entrySet()) {
+      if (runningValue == 0) currStartKey = item.getKey();
+      runningValue += item.getValue();
+      if (runningValue == 0) {
+        if (!firstBoundary) keysArray.add(currStartKey);
+        firstBoundary = false;
+      } 
+    }
+    
+    return keysArray.toArray(new byte[0][0]);
+  }
+ 
+  /*
+   * If the table is created for the first time, then "completebulkload" reads the files twice.
+   * More modifications necessary if we want to avoid doing it.
+   */
+  private void createTable(String tableName, String dirPath) throws Exception {
+    Path hfofDir = new Path(dirPath);
+    FileSystem fs = hfofDir.getFileSystem(getConf());
+
+    if (!fs.exists(hfofDir)) {
+      throw new FileNotFoundException("HFileOutputFormat dir " +
+          hfofDir + " not found");
+    }
+
+    FileStatus[] familyDirStatuses = fs.listStatus(hfofDir);
+    if (familyDirStatuses == null) {
+      throw new FileNotFoundException("No families found in " + hfofDir);
+    }
+
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    HColumnDescriptor hcd = null;
+
+    // Add column families
+    // Build a set of keys
+    byte[][] keys = null;
+    TreeMap map = new TreeMap(Bytes.BYTES_COMPARATOR);
+    
+    for (FileStatus stat : familyDirStatuses) {
+      if (!stat.isDir()) {
+        LOG.warn("Skipping non-directory " + stat.getPath());
+        continue;
+      }
+      Path familyDir = stat.getPath();
+      // Skip _logs, etc
+      if (familyDir.getName().startsWith("_")) continue;
+      byte[] family = familyDir.getName().getBytes();
+     
+      hcd = new HColumnDescriptor(family);
+      htd.addFamily(hcd);
+      
+      Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
+      for (Path hfile : hfiles) {
+        if (hfile.getName().startsWith("_")) continue;
+        HFile.Reader reader = HFile.createReader(fs, hfile,
+            new CacheConfig(getConf()));
+        final byte[] first, last;
+        try {
+          if (hcd.getCompressionType() != reader.getCompressionAlgorithm()) {
+            hcd.setCompressionType(reader.getCompressionAlgorithm());
+            LOG.info("Setting compression " + hcd.getCompressionType().name() +
+                     " for family " + hcd.toString());
+          }
+          reader.loadFileInfo();
+          first = reader.getFirstRowKey();
+          last =  reader.getLastRowKey();
+
+          LOG.info("Trying to figure out region boundaries hfile=" + hfile +
+            " first=" + Bytes.toStringBinary(first) +
+            " last="  + Bytes.toStringBinary(last));
+          
+          // To eventually infer start key-end key boundaries
+          Integer value = map.containsKey(first)?(Integer)map.get(first):0;
+          map.put(first, value+1);
+
+          value = map.containsKey(last)?(Integer)map.get(last):0;
+          map.put(last, value-1);
+        }  finally {
+          reader.close();
+        }
+      }
+    }
+    
+    keys = LoadIncrementalHFiles.inferBoundaries(map);
+    this.hbAdmin.createTable(htd,keys);
+
+    LOG.info("Table "+ tableName +" is available!!");
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    if (args.length != 2) {
+      usage();
+      return -1;
+    }
+
+    String dirPath   = args[0];
+    String tableName = args[1];
+
+    boolean tableExists   = this.doesTableExist(tableName);
+    if (!tableExists) this.createTable(tableName,dirPath);
+
+    Path hfofDir = new Path(dirPath);
+    HTable table = new HTable(this.cfg, tableName);
+
+    doBulkLoad(hfofDir, table);
+    return 0;
+  }
+
+  public static void main(String[] args) throws Exception {
+    int ret = ToolRunner.run(new LoadIncrementalHFiles(HBaseConfiguration.create()), args);
+    System.exit(ret);
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormat.java
new file mode 100644
index 0000000..3a4ceab
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormat.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Scan;
+
+/**
+ * Convert HBase tabular data from multiple scanners into a format that 
+ * is consumable by Map/Reduce.
+ *
+ * 
        
+ * Usage example
+ * 
        
+ *
+ * 
        + * List scans = new ArrayList();
+ * 
+ * Scan scan1 = new Scan();
+ * scan1.setStartRow(firstRow1);
+ * scan1.setStopRow(lastRow1);
+ * scan1.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, table1);
+ * scans.add(scan1);
+ *
+ * Scan scan2 = new Scan();
+ * scan2.setStartRow(firstRow2);
+ * scan2.setStopRow(lastRow2);
+ * scan1.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, table2);
+ * scans.add(scan2);
+ *
+ * TableMapReduceUtil.initTableMapperJob(scans, TableMapper.class, Text.class,
+ *     IntWritable.class, job);
+ * 
        
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class MultiTableInputFormat extends MultiTableInputFormatBase implements
+    Configurable {
+
+  /** Job parameter that specifies the scan list. */
+  public static final String SCANS = "hbase.mapreduce.scans";
+
+  /** The configuration. */
+  private Configuration conf = null;
+
+  /**
+   * Returns the current configuration.
+   *
+   * @return The current configuration.
+   * @see org.apache.hadoop.conf.Configurable#getConf()
+   */
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  /**
+   * Sets the configuration. This is used to set the details for the tables to
+   *  be scanned.
+   *
+   * @param configuration The configuration to set.
+   * @see org.apache.hadoop.conf.Configurable#setConf(
+   *        org.apache.hadoop.conf.Configuration)
+   */
+  @Override
+  public void setConf(Configuration configuration) {
+    this.conf = configuration;
+    String[] rawScans = conf.getStrings(SCANS);
+    if (rawScans.length <= 0) {
+      throw new IllegalArgumentException("There must be at least 1 scan configuration set to : "
+          + SCANS);
+    }
+    List scans = new ArrayList();
+
+    for (int i = 0; i < rawScans.length; i++) {
+      try {
+        scans.add(TableMapReduceUtil.convertStringToScan(rawScans[i]));
+      } catch (IOException e) {
+        throw new RuntimeException("Failed to convert Scan : " + rawScans[i] + " to string", e);
+      }
+    }
+    this.setScans(scans);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
new file mode 100644
index 0000000..76a1632
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * A base for {@link MultiTableInputFormat}s. Receives a list of
+ * {@link Scan} instances that define the input tables and
+ * filters etc. Subclasses may use other TableRecordReader implementations.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class MultiTableInputFormatBase extends
+    InputFormat {
+
+  final Log LOG = LogFactory.getLog(MultiTableInputFormatBase.class);
+
+  /** Holds the set of scans used to define the input. */
+  private List scans;
+
+  /** The reader scanning the table, can be a custom one. */
+  private TableRecordReader tableRecordReader = null;
+
+  /**
+   * Builds a TableRecordReader. If no TableRecordReader was provided, uses the
+   * default.
+   *
+   * @param split The split to work with.
+   * @param context The current context.
+   * @return The newly created record reader.
+   * @throws IOException When creating the reader fails.
+   * @throws InterruptedException when record reader initialization fails
+   * @see org.apache.hadoop.mapreduce.InputFormat#createRecordReader(
+   *      org.apache.hadoop.mapreduce.InputSplit,
+   *      org.apache.hadoop.mapreduce.TaskAttemptContext)
+   */
+  @Override
+  public RecordReader createRecordReader(
+      InputSplit split, TaskAttemptContext context)
+      throws IOException, InterruptedException {
+    TableSplit tSplit = (TableSplit) split;
+
+    if (tSplit.getTableName() == null) {
+      throw new IOException("Cannot create a record reader because of a"
+          + " previous error. Please look at the previous logs lines from"
+          + " the task's full log for more details.");
+    }
+    HTable table =
+        new HTable(context.getConfiguration(), tSplit.getTableName());
+
+    TableRecordReader trr = this.tableRecordReader;
+    // if no table record reader was provided use default
+    if (trr == null) {
+      trr = new TableRecordReader();
+    }
+    Scan sc = tSplit.getScan();
+    sc.setStartRow(tSplit.getStartRow());
+    sc.setStopRow(tSplit.getEndRow());
+    trr.setScan(sc);
+    trr.setHTable(table);
+    trr.initialize(split, context);
+    return trr;
+  }
+
+  /**
+   * Calculates the splits that will serve as input for the map tasks. The
+   * number of splits matches the number of regions in a table.
+   *
+   * @param context The current job context.
+   * @return The list of input splits.
+   * @throws IOException When creating the list of splits fails.
+   * @see org.apache.hadoop.mapreduce.InputFormat#getSplits(org.apache.hadoop.mapreduce.JobContext)
+   */
+  @Override
+  public List getSplits(JobContext context) throws IOException {
+    if (scans.isEmpty()) {
+      throw new IOException("No scans were provided.");
+    }
+    List splits = new ArrayList();
+
+    for (Scan scan : scans) {
+      byte[] tableName = scan.getAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME);
+      if (tableName == null) 
+        throw new IOException("A scan object did not have a table name");
+      HTable table = new HTable(context.getConfiguration(), tableName);
+      Pair keys = table.getStartEndKeys();
+      if (keys == null || keys.getFirst() == null ||
+          keys.getFirst().length == 0) {
+        throw new IOException("Expecting at least one region for table : "
+            + Bytes.toString(tableName));
+      }
+      int count = 0;
+      
+      byte[] startRow = scan.getStartRow();
+      byte[] stopRow = scan.getStopRow();
+
+      for (int i = 0; i < keys.getFirst().length; i++) {
+        if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
+          continue;
+        }
+        String regionLocation =
+            table.getRegionLocation(keys.getFirst()[i], false).getHostname();
+        
+        // determine if the given start and stop keys fall into the range
+        if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
+            Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
+            (stopRow.length == 0 ||
+                Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) {
+          byte[] splitStart =
+              startRow.length == 0 ||
+                  Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? keys
+                  .getFirst()[i] : startRow;
+          byte[] splitStop =
+              (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i],
+                  stopRow) <= 0) && keys.getSecond()[i].length > 0 ? keys
+                  .getSecond()[i] : stopRow;
+          InputSplit split =
+              new TableSplit(tableName, scan, splitStart,
+                  splitStop, regionLocation);
+          splits.add(split);
+          if (LOG.isDebugEnabled())
+            LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
+        }
+      }
+      table.close();
+    }
+    return splits;
+  }
+
+  /**
+   * Test if the given region is to be included in the InputSplit while
+   * splitting the regions of a table.
+   * 
        
+   * This optimization is effective when there is a specific reasoning to
+   * exclude an entire region from the M-R job, (and hence, not contributing to
+   * the InputSplit), given the start and end keys of the same. 
        
+   * Useful when we need to remember the last-processed top record and revisit
+   * the [last, current) interval for M-R processing, continuously. In addition
+   * to reducing InputSplits, reduces the load on the region server as well, due
+   * to the ordering of the keys. 
        
+   * 
        
+   * Note: It is possible that endKey.length() == 0  , for the last
+   * (recent) region. 
        
+   * Override this method, if you want to bulk exclude regions altogether from
+   * M-R. By default, no region is excluded( i.e. all regions are included).
+   *
+   * @param startKey Start key of the region
+   * @param endKey End key of the region
+   * @return true, if this region needs to be included as part of the input
+   *         (default).
+   */
+  protected boolean includeRegionInSplit(final byte[] startKey,
+      final byte[] endKey) {
+    return true;
+  }
+
+  /**
+   * Allows subclasses to get the list of {@link Scan} objects.
+   */
+  protected List getScans() {
+    return this.scans;
+  }
+
+  /**
+   * Allows subclasses to set the list of {@link Scan} objects.
+   *
+   * @param scans The list of {@link Scan} used to define the input
+   */
+  protected void setScans(List scans) {
+    this.scans = scans;
+  }
+
+  /**
+   * Allows subclasses to set the {@link TableRecordReader}.
+   *
+   * @param tableRecordReader A different {@link TableRecordReader}
+   *          implementation.
+   */
+  protected void setTableRecordReader(TableRecordReader tableRecordReader) {
+    this.tableRecordReader = tableRecordReader;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableOutputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableOutputFormat.java
new file mode 100644
index 0000000..81d2746
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableOutputFormat.java
@@ -0,0 +1,163 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * 
        
+ * Hadoop output format that writes to one or more HBase tables. The key is
+ * taken to be the table name while the output value must be either a
+ * {@link Put} or a {@link Delete} instance. All tables must already exist, and
+ * all Puts and Deletes must reference only valid column families.
+ * 
        
+ *
+ * 
        
+ * Write-ahead logging (HLog) for Puts can be disabled by setting
+ * {@link #WAL_PROPERTY} to {@link #WAL_OFF}. Default value is {@link #WAL_ON}.
+ * Note that disabling write-ahead logging is only appropriate for jobs where
+ * loss of data due to region server failure can be tolerated (for example,
+ * because it is easy to rerun a bulk import).
+ * 
        
+ */
+public class MultiTableOutputFormat extends OutputFormat {
+  /** Set this to {@link #WAL_OFF} to turn off write-ahead logging (HLog) */
+  public static final String WAL_PROPERTY = "hbase.mapreduce.multitableoutputformat.wal";
+  /** Property value to use write-ahead logging */
+  public static final boolean WAL_ON = true;
+  /** Property value to disable write-ahead logging */
+  public static final boolean WAL_OFF = false;
+  /**
+   * Record writer for outputting to multiple HTables.
+   */
+  protected static class MultiTableRecordWriter extends
+      RecordWriter {
+    private static final Log LOG = LogFactory.getLog(MultiTableRecordWriter.class);
+    Map tables;
+    Configuration conf;
+    boolean useWriteAheadLogging;
+
+    /**
+     * @param conf
+     *          HBaseConfiguration to used
+     * @param useWriteAheadLogging
+     *          whether to use write ahead logging. This can be turned off (
+     *          false) to improve performance when bulk loading data.
+     */
+    public MultiTableRecordWriter(Configuration conf,
+        boolean useWriteAheadLogging) {
+      LOG.debug("Created new MultiTableRecordReader with WAL "
+          + (useWriteAheadLogging ? "on" : "off"));
+      this.tables = new HashMap();
+      this.conf = conf;
+      this.useWriteAheadLogging = useWriteAheadLogging;
+    }
+
+    /**
+     * @param tableName
+     *          the name of the table, as a string
+     * @return the named table
+     * @throws IOException
+     *           if there is a problem opening a table
+     */
+    HTable getTable(ImmutableBytesWritable tableName) throws IOException {
+      if (!tables.containsKey(tableName)) {
+        LOG.debug("Opening HTable \"" + Bytes.toString(tableName.get())+ "\" for writing");
+        HTable table = new HTable(conf, tableName.get());
+        table.setAutoFlush(false);
+        tables.put(tableName, table);
+      }
+      return tables.get(tableName);
+    }
+
+    @Override
+    public void close(TaskAttemptContext context) throws IOException {
+      for (HTable table : tables.values()) {
+        table.flushCommits();
+      }
+    }
+
+    /**
+     * Writes an action (Put or Delete) to the specified table.
+     *
+     * @param tableName
+     *          the table being updated.
+     * @param action
+     *          the update, either a put or a delete.
+     * @throws IllegalArgumentException
+     *          if the action is not a put or a delete.
+     */
+    @Override
+    public void write(ImmutableBytesWritable tableName, Writable action) throws IOException {
+      HTable table = getTable(tableName);
+      // The actions are not immutable, so we defensively copy them
+      if (action instanceof Put) {
+        Put put = new Put((Put) action);
+        put.setWriteToWAL(useWriteAheadLogging);
+        table.put(put);
+      } else if (action instanceof Delete) {
+        Delete delete = new Delete((Delete) action);
+        table.delete(delete);
+      } else
+        throw new IllegalArgumentException(
+            "action must be either Delete or Put");
+    }
+  }
+
+  @Override
+  public void checkOutputSpecs(JobContext context) throws IOException,
+      InterruptedException {
+    // we can't know ahead of time if it's going to blow up when the user
+    // passes a table name that doesn't exist, so nothing useful here.
+  }
+
+  @Override
+  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
+      throws IOException, InterruptedException {
+    return new TableOutputCommitter();
+  }
+
+  @Override
+  public RecordWriter getRecordWriter(TaskAttemptContext context)
+      throws IOException, InterruptedException {
+    Configuration conf = context.getConfiguration();
+    return new MultiTableRecordWriter(HBaseConfiguration.create(conf),
+        conf.getBoolean(WAL_PROPERTY, WAL_ON));
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java
new file mode 100644
index 0000000..8d1aaf0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java
@@ -0,0 +1,302 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.MapContext;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.StatusReporter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.util.ReflectionUtils;
+
+
+/**
+ * Multithreaded implementation for @link org.apache.hbase.mapreduce.TableMapper
+ * 
        
+ * It can be used instead when the Map operation is not CPU
+ * bound in order to improve throughput.
+ * 
        
+ * Mapper implementations using this MapRunnable must be thread-safe.
+ * 
        
+ * The Map-Reduce job has to be configured with the mapper to use via
+ * {@link #setMapperClass(Configuration, Class)} and
+ * the number of thread the thread-pool can use with the
+ * {@link #getNumberOfThreads(Configuration) method. The default
+ * value is 10 threads.
+ * 
        
+ */
+
+public class MultithreadedTableMapper extends TableMapper {
+  private static final Log LOG = LogFactory.getLog(MultithreadedTableMapper.class);
+  private Class> mapClass;
+  private Context outer;
+  private ExecutorService executor;
+  public static final String NUMBER_OF_THREADS = "hbase.mapreduce.multithreadedmapper.threads";
+  public static final String MAPPER_CLASS = "hbase.mapreduce.multithreadedmapper.mapclass";
+
+  /**
+   * The number of threads in the thread pool that will run the map function.
+   * @param job the job
+   * @return the number of threads
+   */
+  public static int getNumberOfThreads(JobContext job) {
+    return job.getConfiguration().
+        getInt(NUMBER_OF_THREADS, 10);
+  }
+
+  /**
+   * Set the number of threads in the pool for running maps.
+   * @param job the job to modify
+   * @param threads the new number of threads
+   */
+  public static void setNumberOfThreads(Job job, int threads) {
+    job.getConfiguration().setInt(NUMBER_OF_THREADS,
+        threads);
+  }
+
+  /**
+   * Get the application's mapper class.
+   * @param  the map's output key type
+   * @param  the map's output value type
+   * @param job the job
+   * @return the mapper class to run
+   */
+  @SuppressWarnings("unchecked")
+  public static 
+  Class> getMapperClass(JobContext job) {
+    return (Class>)
+        job.getConfiguration().getClass( MAPPER_CLASS,
+            Mapper.class);
+  }
+
+  /**
+   * Set the application's mapper class.
+   * @param  the map output key type
+   * @param  the map output value type
+   * @param job the job to modify
+   * @param cls the class to use as the mapper
+   */
+  public static 
+  void setMapperClass(Job job,
+      Class> cls) {
+    if (MultithreadedTableMapper.class.isAssignableFrom(cls)) {
+      throw new IllegalArgumentException("Can't have recursive " +
+          "MultithreadedTableMapper instances.");
+    }
+    job.getConfiguration().setClass(MAPPER_CLASS,
+        cls, Mapper.class);
+  }
+
+  /**
+   * Run the application's maps using a thread pool.
+   */
+  @Override
+  public void run(Context context) throws IOException, InterruptedException {
+    outer = context;
+    int numberOfThreads = getNumberOfThreads(context);
+    mapClass = getMapperClass(context);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Configuring multithread runner to use " + numberOfThreads +
+          " threads");
+    }
+    executor = Executors.newFixedThreadPool(numberOfThreads);
+    for(int i=0; i < numberOfThreads; ++i) {
+      MapRunner thread = new MapRunner(context);
+      executor.execute(thread);
+    }
+    executor.shutdown();
+    while (!executor.isTerminated()) {
+      // wait till all the threads are done
+      Thread.sleep(1000);
+    }
+  }
+
+  private class SubMapRecordReader
+  extends RecordReader {
+    private ImmutableBytesWritable key;
+    private Result value;
+    private Configuration conf;
+
+    @Override
+    public void close() throws IOException {
+    }
+
+    @Override
+    public float getProgress() throws IOException, InterruptedException {
+      return 0;
+    }
+
+    @Override
+    public void initialize(InputSplit split,
+        TaskAttemptContext context
+        ) throws IOException, InterruptedException {
+      conf = context.getConfiguration();
+    }
+
+    @Override
+    public boolean nextKeyValue() throws IOException, InterruptedException {
+      synchronized (outer) {
+        if (!outer.nextKeyValue()) {
+          return false;
+        }
+        key = ReflectionUtils.copy(outer.getConfiguration(),
+            outer.getCurrentKey(), key);
+        value = ReflectionUtils.copy(conf, outer.getCurrentValue(), value);
+        return true;
+      }
+    }
+
+    public ImmutableBytesWritable getCurrentKey() {
+      return key;
+    }
+
+    @Override
+    public Result getCurrentValue() {
+      return value;
+    }
+  }
+
+  private class SubMapRecordWriter extends RecordWriter {
+
+    @Override
+    public void close(TaskAttemptContext context) throws IOException,
+    InterruptedException {
+    }
+
+    @Override
+    public void write(K2 key, V2 value) throws IOException,
+    InterruptedException {
+      synchronized (outer) {
+        outer.write(key, value);
+      }
+    }
+  }
+
+  private class SubMapStatusReporter extends StatusReporter {
+
+    @Override
+    public Counter getCounter(Enum name) {
+      return outer.getCounter(name);
+    }
+
+    @Override
+    public Counter getCounter(String group, String name) {
+      return outer.getCounter(group, name);
+    }
+
+    @Override
+    public void progress() {
+      outer.progress();
+    }
+
+    @Override
+    public void setStatus(String status) {
+      outer.setStatus(status);
+    }
+
+    public float getProgress() {
+      return 0;
+    }
+  }
+
+  private class MapRunner implements Runnable {
+    private Mapper mapper;
+    private Context subcontext;
+    private Throwable throwable;
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    MapRunner(Context context) throws IOException, InterruptedException {
+      mapper = ReflectionUtils.newInstance(mapClass,
+          context.getConfiguration());
+      try {
+        Constructor c = context.getClass().getConstructor(
+          Mapper.class,
+          Configuration.class,
+          TaskAttemptID.class,
+          RecordReader.class,
+          RecordWriter.class,
+          OutputCommitter.class,
+          StatusReporter.class,
+          InputSplit.class);
+        c.setAccessible(true);
+        subcontext = (Context) c.newInstance(
+          mapper,
+          outer.getConfiguration(), 
+          outer.getTaskAttemptID(),
+          new SubMapRecordReader(),
+          new SubMapRecordWriter(),
+          context.getOutputCommitter(),
+          new SubMapStatusReporter(),
+          outer.getInputSplit());
+      } catch (Exception e) {
+        try {
+          Constructor c = Class.forName("org.apache.hadoop.mapreduce.task.MapContextImpl").getConstructor(
+            Configuration.class,
+            TaskAttemptID.class,
+            RecordReader.class,
+            RecordWriter.class,
+            OutputCommitter.class,
+            StatusReporter.class,
+            InputSplit.class);
+          c.setAccessible(true);
+          MapContext mc = (MapContext) c.newInstance(
+            outer.getConfiguration(), 
+            outer.getTaskAttemptID(),
+            new SubMapRecordReader(),
+            new SubMapRecordWriter(),
+            context.getOutputCommitter(),
+            new SubMapStatusReporter(),
+            outer.getInputSplit());
+          Class wrappedMapperClass = Class.forName("org.apache.hadoop.mapreduce.lib.map.WrappedMapper");
+          Method getMapContext = wrappedMapperClass.getMethod("getMapContext", MapContext.class);
+          subcontext = (Context) getMapContext.invoke(wrappedMapperClass.newInstance(), mc);
+        } catch (Exception ee) {
+          // rethrow as IOE
+          throw new IOException(e);
+        }
+      }
+    }
+
+    @Override
+    public void run() {
+      try {
+        mapper.run(subcontext);
+      } catch (Throwable ie) {
+        throwable = ie;
+      }
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
new file mode 100644
index 0000000..e76df8c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
@@ -0,0 +1,84 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Emits sorted Puts.
+ * Reads in all Puts from passed Iterator, sorts them, then emits
+ * Puts in sorted order.  If lots of columns per row, it will use lots of
+ * memory sorting.
+ * @see HFileOutputFormat
+ * @see KeyValueSortReducer
+ */
+public class PutSortReducer extends
+    Reducer {
+  
+  @Override
+  protected void reduce(
+      ImmutableBytesWritable row,
+      java.lang.Iterable puts,
+      Reducer.Context context)
+      throws java.io.IOException, InterruptedException
+  {
+    // although reduce() is called per-row, handle pathological case
+    long threshold = context.getConfiguration().getLong(
+        "putsortreducer.row.threshold", 2L * (1<<30));
+    Iterator iter = puts.iterator();
+    while (iter.hasNext()) {
+      TreeSet map = new TreeSet(KeyValue.COMPARATOR);
+      long curSize = 0;
+      // stop at the end or the RAM threshold
+      while (iter.hasNext() && curSize < threshold) {
+        Put p = iter.next();
+        for (List kvs : p.getFamilyMap().values()) {
+          for (KeyValue kv : kvs) {
+            map.add(kv);
+            curSize += kv.getLength();
+          }
+        }
+      }
+      context.setStatus("Read " + map.size() + " entries of " + map.getClass()
+          + "(" + StringUtils.humanReadableInt(curSize) + ")");
+      int index = 0;
+      for (KeyValue kv : map) {
+        context.write(row, kv);
+        if (index > 0 && index % 100 == 0)
+          context.setStatus("Wrote " + index);
+      }
+
+      // if we have more entries to process
+      if (iter.hasNext()) {
+        // force flush because we cannot guarantee intra-row sorted order
+        context.write(null, null);
+      }
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
new file mode 100644
index 0000000..5ebe712
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
@@ -0,0 +1,175 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+
+/**
+ * A job with a just a map phase to count rows. Map outputs table rows IF the
+ * input row has columns that have content.
+ */
+public class RowCounter {
+
+  /** Name of this 'program'. */
+  static final String NAME = "rowcounter";
+
+  /**
+   * Mapper that runs the count.
+   */
+  static class RowCounterMapper
+  extends TableMapper {
+
+    /** Counter enumeration to count the actual rows. */
+    public static enum Counters {ROWS}
+
+    /**
+     * Maps the data.
+     *
+     * @param row  The current table row key.
+     * @param values  The columns.
+     * @param context  The current context.
+     * @throws IOException When something is broken with the data.
+     * @see org.apache.hadoop.mapreduce.Mapper#map(KEYIN, VALUEIN,
+     *   org.apache.hadoop.mapreduce.Mapper.Context)
+     */
+    @Override
+    public void map(ImmutableBytesWritable row, Result values,
+      Context context)
+    throws IOException {
+      // Count every row containing data, whether it's in qualifiers or values
+      context.getCounter(Counters.ROWS).increment(1);
+    }
+  }
+
+  /**
+   * Sets up the actual job.
+   *
+   * @param conf  The current configuration.
+   * @param args  The command line parameters.
+   * @return The newly created job.
+   * @throws IOException When setting up the job fails.
+   */
+  public static Job createSubmittableJob(Configuration conf, String[] args)
+  throws IOException {
+    String tableName = args[0];
+    String startKey = null;
+    String endKey = null;
+    StringBuilder sb = new StringBuilder();
+
+    final String rangeSwitch = "--range=";
+
+    // First argument is table name, starting from second
+    for (int i = 1; i < args.length; i++) {
+      if (args[i].startsWith(rangeSwitch)) {
+        String[] startEnd = args[i].substring(rangeSwitch.length()).split(",", 2);
+        if (startEnd.length != 2 || startEnd[1].contains(",")) {
+          printUsage("Please specify range in such format as \"--range=a,b\" " +
+              "or, with only one boundary, \"--range=,b\" or \"--range=a,\"");
+          return null;
+        }
+        startKey = startEnd[0];
+        endKey = startEnd[1];
+      }
+      else {
+        // if no switch, assume column names
+        sb.append(args[i]);
+        sb.append(" ");
+      }
+    }
+
+    Job job = new Job(conf, NAME + "_" + tableName);
+    job.setJarByClass(RowCounter.class);
+    Scan scan = new Scan();
+    scan.setCacheBlocks(false);
+    if (startKey != null && !startKey.equals("")) {
+      scan.setStartRow(Bytes.toBytes(startKey));
+    }
+    if (endKey != null && !endKey.equals("")) {
+      scan.setStopRow(Bytes.toBytes(endKey));
+    }
+    scan.setFilter(new FirstKeyOnlyFilter());
+    if (sb.length() > 0) {
+      for (String columnName : sb.toString().trim().split(" ")) {
+        String [] fields = columnName.split(":");
+        if(fields.length == 1) {
+          scan.addFamily(Bytes.toBytes(fields[0]));
+        } else {
+          scan.addColumn(Bytes.toBytes(fields[0]), Bytes.toBytes(fields[1]));
+        }
+      }
+    }
+    job.setOutputFormatClass(NullOutputFormat.class);
+    TableMapReduceUtil.initTableMapperJob(tableName, scan,
+      RowCounterMapper.class, ImmutableBytesWritable.class, Result.class, job);
+    job.setNumReduceTasks(0);
+    return job;
+  }
+
+  /*
+   * @param errorMessage Can attach a message when error occurs.
+   */
+  private static void printUsage(String errorMessage) {
+    System.err.println("ERROR: " + errorMessage);
+    printUsage();
+  }
+
+  /*
+   * Prints usage without error message
+   */
+  private static void printUsage() {
+    System.err.println("Usage: RowCounter [options]  " +
+        "[--range=[startKey],[endKey]] [ ...]");
+    System.err.println("For performance consider the following options:\n"
+        + "-Dhbase.client.scanner.caching=100\n"
+        + "-Dmapred.map.tasks.speculative.execution=false");
+  }
+
+  /**
+   * Main entry point.
+   *
+   * @param args  The command line parameters.
+   * @throws Exception When running the job fails.
+   */
+  public static void main(String[] args) throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+    if (otherArgs.length < 1) {
+      printUsage("Wrong number of parameters: " + args.length);
+      System.exit(-1);
+    }
+    Job job = createSubmittableJob(conf, otherArgs);
+    if (job == null) {
+      System.exit(-1);
+    }
+    System.exit(job.waitForCompletion(true) ? 0 : 1);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java
new file mode 100644
index 0000000..a7524cb
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java
@@ -0,0 +1,139 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Base64;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Partitioner;
+
+/**
+ * A partitioner that takes start and end keys and uses bigdecimal to figure
+ * which reduce a key belongs to.  Pass the start and end
+ * keys in the Configuration using hbase.simpletotalorder.start
+ * and hbase.simpletotalorder.end.  The end key needs to be
+ * exclusive; i.e. one larger than the biggest key in your key space.
+ * You may be surprised at how this class partitions the space; it may not
+ * align with preconceptions; e.g. a start key of zero and an end key of 100
+ * divided in ten will not make regions whose range is 0-10, 10-20, and so on.
+ * Make your own partitioner if you need the region spacing to come out a
+ * particular way.
+ * @param 
+ * @see #START
+ * @see #END
+ */
+public class SimpleTotalOrderPartitioner extends Partitioner
+implements Configurable {
+  private final static Log LOG = LogFactory.getLog(SimpleTotalOrderPartitioner.class);
+
+  @Deprecated
+  public static final String START = "hbase.simpletotalorder.start";
+  @Deprecated
+  public static final String END = "hbase.simpletotalorder.end";
+  
+  static final String START_BASE64 = "hbase.simpletotalorder.start.base64";
+  static final String END_BASE64 = "hbase.simpletotalorder.end.base64";
+  
+  private Configuration c;
+  private byte [] startkey;
+  private byte [] endkey;
+  private byte [][] splits;
+  private int lastReduces = -1;
+
+  public static void setStartKey(Configuration conf, byte[] startKey) {
+    conf.set(START_BASE64, Base64.encodeBytes(startKey));
+  }
+  
+  public static void setEndKey(Configuration conf, byte[] endKey) {
+    conf.set(END_BASE64, Base64.encodeBytes(endKey));
+  }
+  
+  @SuppressWarnings("deprecation")
+  static byte[] getStartKey(Configuration conf) {
+    return getKeyFromConf(conf, START_BASE64, START);
+  }
+  
+  @SuppressWarnings("deprecation")
+  static byte[] getEndKey(Configuration conf) {
+    return getKeyFromConf(conf, END_BASE64, END);
+  }
+  
+  private static byte[] getKeyFromConf(Configuration conf,
+      String base64Key, String deprecatedKey) {
+    String encoded = conf.get(base64Key);
+    if (encoded != null) {
+      return Base64.decode(encoded);
+    }
+    String oldStyleVal = conf.get(deprecatedKey);
+    if (oldStyleVal == null) {
+      return null;
+    }
+    LOG.warn("Using deprecated configuration " + deprecatedKey +
+        " - please use static accessor methods instead.");
+    return Bytes.toBytes(oldStyleVal);
+  }
+  
+  @Override
+  public int getPartition(final ImmutableBytesWritable key, final VALUE value,
+      final int reduces) {
+    if (reduces == 1) return 0;
+    if (this.lastReduces != reduces) {
+      this.splits = Bytes.split(this.startkey, this.endkey, reduces - 1);
+      for (int i = 0; i < splits.length; i++) {
+        LOG.info(Bytes.toStringBinary(splits[i]));
+      }
+    }
+    int pos = Bytes.binarySearch(this.splits, key.get(), key.getOffset(),
+      key.getLength(), Bytes.BYTES_RAWCOMPARATOR);
+    // Below code is from hfile index search.
+    if (pos < 0) {
+      pos++;
+      pos *= -1;
+      if (pos == 0) {
+        // falls before the beginning of the file.
+        throw new RuntimeException("Key outside start/stop range: " +
+          key.toString());
+      }
+      pos--;
+    }
+    return pos;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.c;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.c = conf;
+    this.startkey = getStartKey(conf);
+    this.endkey = getEndKey(conf);
+    if (startkey == null || endkey == null) {
+      throw new RuntimeException(this.getClass() + " not configured");
+    }
+    LOG.info("startkey=" + Bytes.toStringBinary(startkey) +
+        ", endkey=" + Bytes.toStringBinary(endkey));
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
new file mode 100644
index 0000000..27abad5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
@@ -0,0 +1,206 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Convert HBase tabular data into a format that is consumable by Map/Reduce.
+ */
+public class TableInputFormat extends TableInputFormatBase
+implements Configurable {
+
+  private final Log LOG = LogFactory.getLog(TableInputFormat.class);
+
+  /** Job parameter that specifies the input table. */
+  public static final String INPUT_TABLE = "hbase.mapreduce.inputtable";
+  /** Base-64 encoded scanner. All other SCAN_ confs are ignored if this is specified.
+   * See {@link TableMapReduceUtil#convertScanToString(Scan)} for more details.
+   */
+  public static final String SCAN = "hbase.mapreduce.scan";
+  /** Scan start row */
+  public static final String SCAN_ROW_START = "hbase.mapreduce.scan.row.start";
+  /** Scan stop row */
+  public static final String SCAN_ROW_STOP = "hbase.mapreduce.scan.row.stop";
+  /** Column Family to Scan */
+  public static final String SCAN_COLUMN_FAMILY = "hbase.mapreduce.scan.column.family";
+  /** Space delimited list of columns to scan. */
+  public static final String SCAN_COLUMNS = "hbase.mapreduce.scan.columns";
+  /** The timestamp used to filter columns with a specific timestamp. */
+  public static final String SCAN_TIMESTAMP = "hbase.mapreduce.scan.timestamp";
+  /** The starting timestamp used to filter columns with a specific range of versions. */
+  public static final String SCAN_TIMERANGE_START = "hbase.mapreduce.scan.timerange.start";
+  /** The ending timestamp used to filter columns with a specific range of versions. */
+  public static final String SCAN_TIMERANGE_END = "hbase.mapreduce.scan.timerange.end";
+  /** The maximum number of version to return. */
+  public static final String SCAN_MAXVERSIONS = "hbase.mapreduce.scan.maxversions";
+  /** Set to false to disable server-side caching of blocks for this scan. */
+  public static final String SCAN_CACHEBLOCKS = "hbase.mapreduce.scan.cacheblocks";
+  /** The number of rows for caching that will be passed to scanners. */
+  public static final String SCAN_CACHEDROWS = "hbase.mapreduce.scan.cachedrows";
+
+  /** The configuration. */
+  private Configuration conf = null;
+
+  /**
+   * Returns the current configuration.
+   *
+   * @return The current configuration.
+   * @see org.apache.hadoop.conf.Configurable#getConf()
+   */
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  /**
+   * Sets the configuration. This is used to set the details for the table to
+   * be scanned.
+   *
+   * @param configuration  The configuration to set.
+   * @see org.apache.hadoop.conf.Configurable#setConf(
+   *   org.apache.hadoop.conf.Configuration)
+   */
+  @Override
+  public void setConf(Configuration configuration) {
+    this.conf = configuration;
+    String tableName = conf.get(INPUT_TABLE);
+    try {
+      setHTable(new HTable(new Configuration(conf), tableName));
+    } catch (Exception e) {
+      LOG.error(StringUtils.stringifyException(e));
+    }
+
+    Scan scan = null;
+
+    if (conf.get(SCAN) != null) {
+      try {
+        scan = TableMapReduceUtil.convertStringToScan(conf.get(SCAN));
+      } catch (IOException e) {
+        LOG.error("An error occurred.", e);
+      }
+    } else {
+      try {
+        scan = new Scan();
+
+        if (conf.get(SCAN_ROW_START) != null) {
+          scan.setStartRow(Bytes.toBytes(conf.get(SCAN_ROW_START)));
+        }
+
+        if (conf.get(SCAN_ROW_STOP) != null) {
+          scan.setStopRow(Bytes.toBytes(conf.get(SCAN_ROW_STOP)));
+        }
+
+        if (conf.get(SCAN_COLUMNS) != null) {
+          addColumns(scan, conf.get(SCAN_COLUMNS));
+        }
+
+        if (conf.get(SCAN_COLUMN_FAMILY) != null) {
+          scan.addFamily(Bytes.toBytes(conf.get(SCAN_COLUMN_FAMILY)));
+        }
+
+        if (conf.get(SCAN_TIMESTAMP) != null) {
+          scan.setTimeStamp(Long.parseLong(conf.get(SCAN_TIMESTAMP)));
+        }
+
+        if (conf.get(SCAN_TIMERANGE_START) != null && conf.get(SCAN_TIMERANGE_END) != null) {
+          scan.setTimeRange(
+              Long.parseLong(conf.get(SCAN_TIMERANGE_START)),
+              Long.parseLong(conf.get(SCAN_TIMERANGE_END)));
+        }
+
+        if (conf.get(SCAN_MAXVERSIONS) != null) {
+          scan.setMaxVersions(Integer.parseInt(conf.get(SCAN_MAXVERSIONS)));
+        }
+
+        if (conf.get(SCAN_CACHEDROWS) != null) {
+          scan.setCaching(Integer.parseInt(conf.get(SCAN_CACHEDROWS)));
+        }
+
+        // false by default, full table scans generate too much BC churn
+        scan.setCacheBlocks((conf.getBoolean(SCAN_CACHEBLOCKS, false)));
+      } catch (Exception e) {
+          LOG.error(StringUtils.stringifyException(e));
+      }
+    }
+
+    setScan(scan);
+  }
+  
+  /**
+   * Parses a combined family and qualifier and adds either both or just the
+   * family in case there is not qualifier. This assumes the older colon
+   * divided notation, e.g. "data:contents" or "meta:".
+   * 
        
+   * Note: It will through an error when the colon is missing.
+   *
+   * @param familyAndQualifier family and qualifier
+   * @return A reference to this instance.
+   * @throws IllegalArgumentException When the colon is missing.
+   */
+  private static void addColumn(Scan scan, byte[] familyAndQualifier) {
+    byte [][] fq = KeyValue.parseColumn(familyAndQualifier);
+    if (fq.length > 1 && fq[1] != null && fq[1].length > 0) {
+      scan.addColumn(fq[0], fq[1]);
+    } else {
+      scan.addFamily(fq[0]);
+    }
+  }
+
+  /**
+   * Adds an array of columns specified using old format, family:qualifier.
+   * 
        
+   * Overrides previous calls to addFamily for any families in the input.
+   *
+   * @param columns array of columns, formatted as 
        family:qualifier
        
+   */
+  public static void addColumns(Scan scan, byte [][] columns) {
+    for (byte[] column : columns) {
+      addColumn(scan, column);
+    }
+  }
+
+  /**
+   * Convenience method to help parse old style (or rather user entry on the
+   * command line) column definitions, e.g. "data:contents mime:". The columns
+   * must be space delimited and always have a colon (":") to denote family
+   * and qualifier.
+   *
+   * @param columns  The columns to parse.
+   * @return A reference to this instance.
+   */
+  private static void addColumns(Scan scan, String columns) {
+    String[] cols = columns.split(" ");
+    for (String col : cols) {
+      addColumn(scan, Bytes.toBytes(col));
+    }
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
new file mode 100644
index 0000000..ad855e2
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
@@ -0,0 +1,295 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import javax.naming.NamingException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Addressing;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Strings;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.net.DNS;
+
+/**
+ * A base for {@link TableInputFormat}s. Receives a {@link HTable}, an
+ * {@link Scan} instance that defines the input columns etc. Subclasses may use
+ * other TableRecordReader implementations.
+ * 
        
+ * An example of a subclass:
+ * 
        + *   class ExampleTIF extends TableInputFormatBase implements JobConfigurable {
+ *
+ *     public void configure(JobConf job) {
+ *       HTable exampleTable = new HTable(HBaseConfiguration.create(job),
+ *         Bytes.toBytes("exampleTable"));
+ *       // mandatory
+ *       setHTable(exampleTable);
+ *       Text[] inputColumns = new byte [][] { Bytes.toBytes("columnA"),
+ *         Bytes.toBytes("columnB") };
+ *       // mandatory
+ *       setInputColumns(inputColumns);
+ *       RowFilterInterface exampleFilter = new RegExpRowFilter("keyPrefix.*");
+ *       // optional
+ *       setRowFilter(exampleFilter);
+ *     }
+ *
+ *     public void validateInput(JobConf job) throws IOException {
+ *     }
+ *  }
+ * 
        
+ */
+public abstract class TableInputFormatBase
+extends InputFormat {
+
+  final Log LOG = LogFactory.getLog(TableInputFormatBase.class);
+
+  /** Holds the details for the internal scanner. */
+  private Scan scan = null;
+  /** The table to scan. */
+  private HTable table = null;
+  /** The reader scanning the table, can be a custom one. */
+  private TableRecordReader tableRecordReader = null;
+
+
+  /** The reverse DNS lookup cache mapping: IPAddress => HostName */
+  private HashMap reverseDNSCacheMap =
+    new HashMap();
+  
+  /** The NameServer address */
+  private String nameServer = null;
+  
+  /**
+   * Builds a TableRecordReader. If no TableRecordReader was provided, uses
+   * the default.
+   *
+   * @param split  The split to work with.
+   * @param context  The current context.
+   * @return The newly created record reader.
+   * @throws IOException When creating the reader fails.
+   * @see org.apache.hadoop.mapreduce.InputFormat#createRecordReader(
+   *   org.apache.hadoop.mapreduce.InputSplit,
+   *   org.apache.hadoop.mapreduce.TaskAttemptContext)
+   */
+  @Override
+  public RecordReader createRecordReader(
+      InputSplit split, TaskAttemptContext context)
+  throws IOException {
+    if (table == null) {
+      throw new IOException("Cannot create a record reader because of a" +
+          " previous error. Please look at the previous logs lines from" +
+          " the task's full log for more details.");
+    }
+    TableSplit tSplit = (TableSplit) split;
+    TableRecordReader trr = this.tableRecordReader;
+    // if no table record reader was provided use default
+    if (trr == null) {
+      trr = new TableRecordReader();
+    }
+    Scan sc = new Scan(this.scan);
+    sc.setStartRow(tSplit.getStartRow());
+    sc.setStopRow(tSplit.getEndRow());
+    trr.setScan(sc);
+    trr.setHTable(table);
+    try {
+      trr.initialize(tSplit, context);
+    } catch (InterruptedException e) {
+      throw new InterruptedIOException(e.getMessage());
+    }
+    return trr;
+  }
+
+  /**
+   * Calculates the splits that will serve as input for the map tasks. The
+   * number of splits matches the number of regions in a table.
+   *
+   * @param context  The current job context.
+   * @return The list of input splits.
+   * @throws IOException When creating the list of splits fails.
+   * @see org.apache.hadoop.mapreduce.InputFormat#getSplits(
+   *   org.apache.hadoop.mapreduce.JobContext)
+   */
+  @Override
+  public List getSplits(JobContext context) throws IOException {
+	if (table == null) {
+	    throw new IOException("No table was provided.");
+	}
+    // Get the name server address and the default value is null.
+    this.nameServer =
+      context.getConfiguration().get("hbase.nameserver.address", null);
+    
+    Pair keys = table.getStartEndKeys();
+    if (keys == null || keys.getFirst() == null ||
+        keys.getFirst().length == 0) {
+      HRegionLocation regLoc = table.getRegionLocation(
+          HConstants.EMPTY_BYTE_ARRAY, false);
+      if (null == regLoc) {
+        throw new IOException("Expecting at least one region.");
+      }
+      List splits = new ArrayList(1);
+      InputSplit split = new TableSplit(table.getTableName(),
+          HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, regLoc
+              .getHostnamePort().split(Addressing.HOSTNAME_PORT_SEPARATOR)[0]);
+      splits.add(split);
+      return splits;
+    }
+    List splits = new ArrayList(keys.getFirst().length);
+    for (int i = 0; i < keys.getFirst().length; i++) {
+      if ( !includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
+        continue;
+      }
+      HServerAddress regionServerAddress = 
+        table.getRegionLocation(keys.getFirst()[i]).getServerAddress();
+      InetAddress regionAddress =
+        regionServerAddress.getInetSocketAddress().getAddress();
+      String regionLocation;
+      try {
+        regionLocation = reverseDNS(regionAddress);
+      } catch (NamingException e) {
+        LOG.error("Cannot resolve the host name for " + regionAddress +
+            " because of " + e);
+        regionLocation = regionServerAddress.getHostname();
+      }
+
+			byte[] startRow = scan.getStartRow();
+			byte[] stopRow = scan.getStopRow();
+			// determine if the given start an stop key fall into the region
+      if ((startRow.length == 0 || keys.getSecond()[i].length == 0 ||
+					 Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) &&
+          (stopRow.length == 0 ||
+           Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) {
+        byte[] splitStart = startRow.length == 0 ||
+          Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ?
+            keys.getFirst()[i] : startRow;
+        byte[] splitStop = (stopRow.length == 0 ||
+          Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) &&
+          keys.getSecond()[i].length > 0 ?
+            keys.getSecond()[i] : stopRow;
+        InputSplit split = new TableSplit(table.getTableName(),
+          splitStart, splitStop, regionLocation);
+        splits.add(split);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("getSplits: split -> " + i + " -> " + split);
+        }
+      }
+    }
+    return splits;
+  }
+  
+  private String reverseDNS(InetAddress ipAddress) throws NamingException {
+    String hostName = this.reverseDNSCacheMap.get(ipAddress);
+    if (hostName == null) {
+      hostName = Strings.domainNamePointerToHostName(DNS.reverseDns(ipAddress, this.nameServer));
+      this.reverseDNSCacheMap.put(ipAddress, hostName);
+    }
+    return hostName;
+  }
+
+  /**
+   *
+   *
+   * Test if the given region is to be included in the InputSplit while splitting
+   * the regions of a table.
+   * 
        
+   * This optimization is effective when there is a specific reasoning to exclude an entire region from the M-R job,
+   * (and hence, not contributing to the InputSplit), given the start and end keys of the same. 
        
+   * Useful when we need to remember the last-processed top record and revisit the [last, current) interval for M-R processing,
+   * continuously. In addition to reducing InputSplits, reduces the load on the region server as well, due to the ordering of the keys.
+   * 
        
+   * 
        
+   * Note: It is possible that endKey.length() == 0  , for the last (recent) region.
+   * 
        
+   * Override this method, if you want to bulk exclude regions altogether from M-R. By default, no region is excluded( i.e. all regions are included).
+   *
+   *
+   * @param startKey Start key of the region
+   * @param endKey End key of the region
+   * @return true, if this region needs to be included as part of the input (default).
+   *
+   */
+  protected boolean includeRegionInSplit(final byte[] startKey, final byte [] endKey) {
+    return true;
+  }
+
+  /**
+   * Allows subclasses to get the {@link HTable}.
+   */
+  protected HTable getHTable() {
+    return this.table;
+  }
+
+  /**
+   * Allows subclasses to set the {@link HTable}.
+   *
+   * @param table  The table to get the data from.
+   */
+  protected void setHTable(HTable table) {
+    this.table = table;
+  }
+
+  /**
+   * Gets the scan defining the actual details like columns etc.
+   *
+   * @return The internal scan instance.
+   */
+  public Scan getScan() {
+    if (this.scan == null) this.scan = new Scan();
+    return scan;
+  }
+
+  /**
+   * Sets the scan defining the actual details like columns etc.
+   *
+   * @param scan  The scan to set.
+   */
+  public void setScan(Scan scan) {
+    this.scan = scan;
+  }
+
+  /**
+   * Allows subclasses to set the {@link TableRecordReader}.
+   *
+   * @param tableRecordReader A different {@link TableRecordReader}
+   *   implementation.
+   */
+  protected void setTableRecordReader(TableRecordReader tableRecordReader) {
+    this.tableRecordReader = tableRecordReader;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
new file mode 100644
index 0000000..cac8dd4
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -0,0 +1,699 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.hadoopbackport.JarFinder;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.Base64;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Utility for {@link TableMapper} and {@link TableReducer}
+ */
+@SuppressWarnings("unchecked")
+public class TableMapReduceUtil {
+  static Log LOG = LogFactory.getLog(TableMapReduceUtil.class);
+  
+  /**
+   * Use this before submitting a TableMap job. It will appropriately set up
+   * the job.
+   *
+   * @param table  The table name to read from.
+   * @param scan  The scan instance with the columns, time range etc.
+   * @param mapper  The mapper class to use.
+   * @param outputKeyClass  The class of the output key.
+   * @param outputValueClass  The class of the output value.
+   * @param job  The current job to adjust.  Make sure the passed job is
+   * carrying all necessary HBase configuration.
+   * @throws IOException When setting up the details fails.
+   */
+  public static void initTableMapperJob(String table, Scan scan,
+      Class mapper,
+      Class outputKeyClass,
+      Class outputValueClass, Job job)
+  throws IOException {
+    initTableMapperJob(table, scan, mapper, outputKeyClass, outputValueClass,
+        job, true);
+  }
+
+
+  /**
+   * Use this before submitting a TableMap job. It will appropriately set up
+   * the job.
+   *
+   * @param table Binary representation of the table name to read from.
+   * @param scan  The scan instance with the columns, time range etc.
+   * @param mapper  The mapper class to use.
+   * @param outputKeyClass  The class of the output key.
+   * @param outputValueClass  The class of the output value.
+   * @param job  The current job to adjust.  Make sure the passed job is
+   * carrying all necessary HBase configuration.
+   * @throws IOException When setting up the details fails.
+   */
+   public static void initTableMapperJob(byte[] table, Scan scan,
+      Class mapper,
+      Class outputKeyClass,
+      Class outputValueClass, Job job)
+  throws IOException {
+      initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass, outputValueClass,
+              job, true);
+  }
+
+  /**
+   * Use this before submitting a TableMap job. It will appropriately set up
+   * the job.
+   *
+   * @param table  The table name to read from.
+   * @param scan  The scan instance with the columns, time range etc.
+   * @param mapper  The mapper class to use.
+   * @param outputKeyClass  The class of the output key.
+   * @param outputValueClass  The class of the output value.
+   * @param job  The current job to adjust.  Make sure the passed job is
+   * carrying all necessary HBase configuration.
+   * @param addDependencyJars upload HBase jars and jars for any of the configured
+   *           job classes via the distributed cache (tmpjars).
+   * @throws IOException When setting up the details fails.
+   */
+  public static void initTableMapperJob(String table, Scan scan,
+      Class mapper,
+      Class outputKeyClass,
+      Class outputValueClass, Job job,
+      boolean addDependencyJars, Class inputFormatClass)
+  throws IOException {
+    job.setInputFormatClass(inputFormatClass);
+    if (outputValueClass != null) job.setMapOutputValueClass(outputValueClass);
+    if (outputKeyClass != null) job.setMapOutputKeyClass(outputKeyClass);
+    job.setMapperClass(mapper);
+    Configuration conf = job.getConfiguration();
+    HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
+    conf.set(TableInputFormat.INPUT_TABLE, table);
+    conf.set(TableInputFormat.SCAN, convertScanToString(scan));
+    if (addDependencyJars) {
+      addDependencyJars(job);
+    }
+    initCredentials(job);
+  }
+  
+  /**
+   * Use this before submitting a TableMap job. It will appropriately set up
+   * the job.
+   *
+   * @param table Binary representation of the table name to read from.
+   * @param scan  The scan instance with the columns, time range etc.
+   * @param mapper  The mapper class to use.
+   * @param outputKeyClass  The class of the output key.
+   * @param outputValueClass  The class of the output value.
+   * @param job  The current job to adjust.  Make sure the passed job is
+   * carrying all necessary HBase configuration.
+   * @param addDependencyJars upload HBase jars and jars for any of the configured
+   *           job classes via the distributed cache (tmpjars).
+   * @param inputFormatClass The class of the input format
+   * @throws IOException When setting up the details fails.
+   */
+  public static void initTableMapperJob(byte[] table, Scan scan,
+      Class mapper,
+      Class outputKeyClass,
+      Class outputValueClass, Job job,
+      boolean addDependencyJars, Class inputFormatClass)
+  throws IOException {
+      initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
+              outputValueClass, job, addDependencyJars, inputFormatClass);
+  }
+  
+  /**
+   * Use this before submitting a TableMap job. It will appropriately set up
+   * the job.
+   *
+   * @param table Binary representation of the table name to read from.
+   * @param scan  The scan instance with the columns, time range etc.
+   * @param mapper  The mapper class to use.
+   * @param outputKeyClass  The class of the output key.
+   * @param outputValueClass  The class of the output value.
+   * @param job  The current job to adjust.  Make sure the passed job is
+   * carrying all necessary HBase configuration.
+   * @param addDependencyJars upload HBase jars and jars for any of the configured
+   *           job classes via the distributed cache (tmpjars).
+   * @throws IOException When setting up the details fails.
+   */
+  public static void initTableMapperJob(byte[] table, Scan scan,
+      Class mapper,
+      Class outputKeyClass,
+      Class outputValueClass, Job job,
+      boolean addDependencyJars)
+  throws IOException {
+      initTableMapperJob(Bytes.toString(table), scan, mapper, outputKeyClass,
+              outputValueClass, job, addDependencyJars, TableInputFormat.class);
+  }
+  
+  /**
+   * Use this before submitting a TableMap job. It will appropriately set up
+   * the job.
+   *
+   * @param table The table name to read from.
+   * @param scan  The scan instance with the columns, time range etc.
+   * @param mapper  The mapper class to use.
+   * @param outputKeyClass  The class of the output key.
+   * @param outputValueClass  The class of the output value.
+   * @param job  The current job to adjust.  Make sure the passed job is
+   * carrying all necessary HBase configuration.
+   * @param addDependencyJars upload HBase jars and jars for any of the configured
+   *           job classes via the distributed cache (tmpjars).
+   * @throws IOException When setting up the details fails.
+   */
+  public static void initTableMapperJob(String table, Scan scan,
+      Class mapper,
+      Class outputKeyClass,
+      Class outputValueClass, Job job,
+      boolean addDependencyJars)
+  throws IOException {
+      initTableMapperJob(table, scan, mapper, outputKeyClass,
+              outputValueClass, job, addDependencyJars, TableInputFormat.class);
+  }
+  
+  /**
+   * Use this before submitting a Multi TableMap job. It will appropriately set
+   * up the job.
+   *
+   * @param scans The list of {@link Scan} objects to read from.
+   * @param mapper The mapper class to use.
+   * @param outputKeyClass The class of the output key.
+   * @param outputValueClass The class of the output value.
+   * @param job The current job to adjust. Make sure the passed job is carrying
+   *          all necessary HBase configuration.
+   * @throws IOException When setting up the details fails.
+   */
+  public static void initTableMapperJob(List scans,
+      Class mapper,
+      Class outputKeyClass,
+      Class outputValueClass, Job job) throws IOException {
+    initTableMapperJob(scans, mapper, outputKeyClass, outputValueClass, job,
+        true);
+  }
+
+  /**
+   * Use this before submitting a Multi TableMap job. It will appropriately set
+   * up the job.
+   *
+   * @param scans The list of {@link Scan} objects to read from.
+   * @param mapper The mapper class to use.
+   * @param outputKeyClass The class of the output key.
+   * @param outputValueClass The class of the output value.
+   * @param job The current job to adjust. Make sure the passed job is carrying
+   *          all necessary HBase configuration.
+   * @param addDependencyJars upload HBase jars and jars for any of the
+   *          configured job classes via the distributed cache (tmpjars).
+   * @throws IOException When setting up the details fails.
+   */
+  public static void initTableMapperJob(List scans,
+      Class mapper,
+      Class outputKeyClass,
+      Class outputValueClass, Job job,
+      boolean addDependencyJars) throws IOException {
+    job.setInputFormatClass(MultiTableInputFormat.class);
+    if (outputValueClass != null) {
+      job.setMapOutputValueClass(outputValueClass);
+    }
+    if (outputKeyClass != null) {
+      job.setMapOutputKeyClass(outputKeyClass);
+    }
+    job.setMapperClass(mapper);
+    HBaseConfiguration.addHbaseResources(job.getConfiguration());
+    List scanStrings = new ArrayList();
+
+    for (Scan scan : scans) {
+      scanStrings.add(convertScanToString(scan));
+    }
+    job.getConfiguration().setStrings(MultiTableInputFormat.SCANS,
+      scanStrings.toArray(new String[scanStrings.size()]));
+
+    if (addDependencyJars) {
+      addDependencyJars(job);
+    }
+  }
+
+  public static void initCredentials(Job job) throws IOException {
+    if (User.isHBaseSecurityEnabled(job.getConfiguration())) {
+      try {
+        // init credentials for remote cluster
+        String quorumAddress = job.getConfiguration().get(
+            TableOutputFormat.QUORUM_ADDRESS);
+        if (quorumAddress != null) {
+          String[] parts = ZKUtil.transformClusterKey(quorumAddress);
+          Configuration peerConf = HBaseConfiguration.create(job
+              .getConfiguration());
+          peerConf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
+          peerConf.set("hbase.zookeeper.client.port", parts[1]);
+          peerConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]);
+          User.getCurrent().obtainAuthTokenForJob(peerConf, job);
+        }
+        
+        User.getCurrent().obtainAuthTokenForJob(job.getConfiguration(), job);
+      } catch (InterruptedException ie) {
+        LOG.info("Interrupted obtaining user authentication token");
+        Thread.interrupted();
+      }
+    }
+  }
+
+  /**
+   * Writes the given scan into a Base64 encoded string.
+   *
+   * @param scan  The scan to write out.
+   * @return The scan saved in a Base64 encoded string.
+   * @throws IOException When writing the scan fails.
+   */
+  static String convertScanToString(Scan scan) throws IOException {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    DataOutputStream dos = new DataOutputStream(out);
+    scan.write(dos);
+    return Base64.encodeBytes(out.toByteArray());
+  }
+
+  /**
+   * Converts the given Base64 string back into a Scan instance.
+   *
+   * @param base64  The scan details.
+   * @return The newly created Scan instance.
+   * @throws IOException When reading the scan instance fails.
+   */
+  static Scan convertStringToScan(String base64) throws IOException {
+    ByteArrayInputStream bis = new ByteArrayInputStream(Base64.decode(base64));
+    DataInputStream dis = new DataInputStream(bis);
+    Scan scan = new Scan();
+    scan.readFields(dis);
+    return scan;
+  }
+
+  /**
+   * Use this before submitting a TableReduce job. It will
+   * appropriately set up the JobConf.
+   *
+   * @param table  The output table.
+   * @param reducer  The reducer class to use.
+   * @param job  The current job to adjust.
+   * @throws IOException When determining the region count fails.
+   */
+  public static void initTableReducerJob(String table,
+    Class reducer, Job job)
+  throws IOException {
+    initTableReducerJob(table, reducer, job, null);
+  }
+
+  /**
+   * Use this before submitting a TableReduce job. It will
+   * appropriately set up the JobConf.
+   *
+   * @param table  The output table.
+   * @param reducer  The reducer class to use.
+   * @param job  The current job to adjust.
+   * @param partitioner  Partitioner to use. Pass null to use
+   * default partitioner.
+   * @throws IOException When determining the region count fails.
+   */
+  public static void initTableReducerJob(String table,
+    Class reducer, Job job,
+    Class partitioner) throws IOException {
+    initTableReducerJob(table, reducer, job, partitioner, null, null, null);
+  }
+
+  /**
+   * Use this before submitting a TableReduce job. It will
+   * appropriately set up the JobConf.
+   *
+   * @param table  The output table.
+   * @param reducer  The reducer class to use.
+   * @param job  The current job to adjust.  Make sure the passed job is
+   * carrying all necessary HBase configuration.
+   * @param partitioner  Partitioner to use. Pass null to use
+   * default partitioner.
+   * @param quorumAddress Distant cluster to write to; default is null for
+   * output to the cluster that is designated in hbase-site.xml.
+   * Set this String to the zookeeper ensemble of an alternate remote cluster
+   * when you would have the reduce write a cluster that is other than the
+   * default; e.g. copying tables between clusters, the source would be
+   * designated by hbase-site.xml and this param would have the
+   * ensemble address of the remote cluster.  The format to pass is particular.
+   * Pass  <hbase.zookeeper.quorum>:<hbase.zookeeper.client.port>:<zookeeper.znode.parent>
+   *  such as server,server2,server3:2181:/hbase.
+   * @param serverClass redefined hbase.regionserver.class
+   * @param serverImpl redefined hbase.regionserver.impl
+   * @throws IOException When determining the region count fails.
+   */
+  public static void initTableReducerJob(String table,
+    Class reducer, Job job,
+    Class partitioner, String quorumAddress, String serverClass,
+    String serverImpl) throws IOException {
+    initTableReducerJob(table, reducer, job, partitioner, quorumAddress,
+        serverClass, serverImpl, true);
+  }
+
+  /**
+   * Use this before submitting a TableReduce job. It will
+   * appropriately set up the JobConf.
+   *
+   * @param table  The output table.
+   * @param reducer  The reducer class to use.
+   * @param job  The current job to adjust.  Make sure the passed job is
+   * carrying all necessary HBase configuration.
+   * @param partitioner  Partitioner to use. Pass null to use
+   * default partitioner.
+   * @param quorumAddress Distant cluster to write to; default is null for
+   * output to the cluster that is designated in hbase-site.xml.
+   * Set this String to the zookeeper ensemble of an alternate remote cluster
+   * when you would have the reduce write a cluster that is other than the
+   * default; e.g. copying tables between clusters, the source would be
+   * designated by hbase-site.xml and this param would have the
+   * ensemble address of the remote cluster.  The format to pass is particular.
+   * Pass  <hbase.zookeeper.quorum>:<hbase.zookeeper.client.port>:<zookeeper.znode.parent>
+   *  such as server,server2,server3:2181:/hbase.
+   * @param serverClass redefined hbase.regionserver.class
+   * @param serverImpl redefined hbase.regionserver.impl
+   * @param addDependencyJars upload HBase jars and jars for any of the configured
+   *           job classes via the distributed cache (tmpjars).
+   * @throws IOException When determining the region count fails.
+   */
+  public static void initTableReducerJob(String table,
+    Class reducer, Job job,
+    Class partitioner, String quorumAddress, String serverClass,
+    String serverImpl, boolean addDependencyJars) throws IOException {
+
+    Configuration conf = job.getConfiguration();    
+    HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
+    job.setOutputFormatClass(TableOutputFormat.class);
+    if (reducer != null) job.setReducerClass(reducer);
+    conf.set(TableOutputFormat.OUTPUT_TABLE, table);
+    // If passed a quorum/ensemble address, pass it on to TableOutputFormat.
+    if (quorumAddress != null) {
+      // Calling this will validate the format
+      ZKUtil.transformClusterKey(quorumAddress);
+      conf.set(TableOutputFormat.QUORUM_ADDRESS,quorumAddress);
+    }
+    if (serverClass != null && serverImpl != null) {
+      conf.set(TableOutputFormat.REGION_SERVER_CLASS, serverClass);
+      conf.set(TableOutputFormat.REGION_SERVER_IMPL, serverImpl);
+    }
+    job.setOutputKeyClass(ImmutableBytesWritable.class);
+    job.setOutputValueClass(Writable.class);
+    if (partitioner == HRegionPartitioner.class) {
+      job.setPartitionerClass(HRegionPartitioner.class);
+      HTable outputTable = new HTable(conf, table);
+      int regions = outputTable.getRegionsInfo().size();
+      if (job.getNumReduceTasks() > regions) {
+        job.setNumReduceTasks(outputTable.getRegionsInfo().size());
+      }
+    } else if (partitioner != null) {
+      job.setPartitionerClass(partitioner);
+    }
+
+    if (addDependencyJars) {
+      addDependencyJars(job);
+    }
+
+    initCredentials(job);
+  }
+
+  /**
+   * Ensures that the given number of reduce tasks for the given job
+   * configuration does not exceed the number of regions for the given table.
+   *
+   * @param table  The table to get the region count for.
+   * @param job  The current job to adjust.
+   * @throws IOException When retrieving the table details fails.
+   */
+  public static void limitNumReduceTasks(String table, Job job)
+  throws IOException {
+    HTable outputTable = new HTable(job.getConfiguration(), table);
+    int regions = outputTable.getRegionsInfo().size();
+    if (job.getNumReduceTasks() > regions)
+      job.setNumReduceTasks(regions);
+  }
+
+  /**
+   * Sets the number of reduce tasks for the given job configuration to the
+   * number of regions the given table has.
+   *
+   * @param table  The table to get the region count for.
+   * @param job  The current job to adjust.
+   * @throws IOException When retrieving the table details fails.
+   */
+  public static void setNumReduceTasks(String table, Job job)
+  throws IOException {
+    HTable outputTable = new HTable(job.getConfiguration(), table);
+    int regions = outputTable.getRegionsInfo().size();
+    job.setNumReduceTasks(regions);
+  }
+
+  /**
+   * Sets the number of rows to return and cache with each scanner iteration.
+   * Higher caching values will enable faster mapreduce jobs at the expense of
+   * requiring more heap to contain the cached rows.
+   *
+   * @param job The current job to adjust.
+   * @param batchSize The number of rows to return in batch with each scanner
+   * iteration.
+   */
+  public static void setScannerCaching(Job job, int batchSize) {
+    job.getConfiguration().setInt("hbase.client.scanner.caching", batchSize);
+  }
+
+  /**
+   * Add the HBase dependency jars as well as jars for any of the configured
+   * job classes to the job configuration, so that JobClient will ship them
+   * to the cluster and add them to the DistributedCache.
+   */
+  public static void addDependencyJars(Job job) throws IOException {
+    try {
+      addDependencyJars(job.getConfiguration(),
+          org.apache.zookeeper.ZooKeeper.class,
+          com.google.protobuf.Message.class,
+          com.google.common.collect.ImmutableSet.class,
+          org.apache.hadoop.hbase.util.Bytes.class, //one class from hbase.jar
+          job.getMapOutputKeyClass(),
+          job.getMapOutputValueClass(),
+          job.getInputFormatClass(),
+          job.getOutputKeyClass(),
+          job.getOutputValueClass(),
+          job.getOutputFormatClass(),
+          job.getPartitionerClass(),
+          job.getCombinerClass());
+    } catch (ClassNotFoundException e) {
+      throw new IOException(e);
+    }    
+  }
+  
+  /**
+   * Add the jars containing the given classes to the job's configuration
+   * such that JobClient will ship them to the cluster and add them to
+   * the DistributedCache.
+   */
+  public static void addDependencyJars(Configuration conf,
+      Class... classes) throws IOException {
+
+    FileSystem localFs = FileSystem.getLocal(conf);
+    Set jars = new HashSet();
+    // Add jars that are already in the tmpjars variable
+    jars.addAll(conf.getStringCollection("tmpjars"));
+
+    // add jars as we find them to a map of contents jar name so that we can avoid
+    // creating new jars for classes that have already been packaged.
+    Map packagedClasses = new HashMap();
+
+    // Add jars containing the specified classes
+    for (Class clazz : classes) {
+      if (clazz == null) continue;
+
+      Path path = findOrCreateJar(clazz, localFs, packagedClasses);
+      if (path == null) {
+        LOG.warn("Could not find jar for class " + clazz +
+                 " in order to ship it to the cluster.");
+        continue;
+      }
+      if (!localFs.exists(path)) {
+        LOG.warn("Could not validate jar file " + path + " for class "
+                 + clazz);
+        continue;
+      }
+      jars.add(path.toString());
+    }
+    if (jars.isEmpty()) return;
+
+    conf.set("tmpjars",
+             StringUtils.arrayToString(jars.toArray(new String[0])));
+  }
+
+  /**
+   * If org.apache.hadoop.util.JarFinder is available (0.23+ hadoop), finds
+   * the Jar for a class or creates it if it doesn't exist. If the class is in
+   * a directory in the classpath, it creates a Jar on the fly with the
+   * contents of the directory and returns the path to that Jar. If a Jar is
+   * created, it is created in the system temporary directory. Otherwise,
+   * returns an existing jar that contains a class of the same name. Maintains
+   * a mapping from jar contents to the tmp jar created.
+   * @param my_class the class to find.
+   * @param fs the FileSystem with which to qualify the returned path.
+   * @param packagedClasses a map of class name to path.
+   * @return a jar file that contains the class.
+   * @throws IOException
+   */
+  private static Path findOrCreateJar(Class my_class, FileSystem fs,
+      Map packagedClasses)
+  throws IOException {
+    // attempt to locate an existing jar for the class.
+    String jar = findContainingJar(my_class, packagedClasses);
+    if (null == jar || jar.isEmpty()) {
+      jar = getJar(my_class);
+      updateMap(jar, packagedClasses);
+    }
+
+    if (null == jar || jar.isEmpty()) {
+      throw new IOException("Cannot locate resource for class " + my_class.getName());
+    }
+
+    LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar));
+    return new Path(jar).makeQualified(fs);
+  }
+
+  /**
+   * Add entries to packagedClasses corresponding to class files
+   * contained in jar.
+   * @param jar The jar who's content to list.
+   * @param packagedClasses map[class -> jar]
+   */
+  private static void updateMap(String jar, Map packagedClasses) throws IOException {
+    ZipFile zip = null;
+    try {
+      zip = new ZipFile(jar);
+      for (Enumeration iter = zip.entries(); iter.hasMoreElements();) {
+        ZipEntry entry = iter.nextElement();
+        if (entry.getName().endsWith("class")) {
+          packagedClasses.put(entry.getName(), jar);
+        }
+      }
+    } finally {
+      if (null != zip) zip.close();
+    }
+  }
+
+  /**
+   * Find a jar that contains a class of the same name, if any. It will return
+   * a jar file, even if that is not the first thing on the class path that
+   * has a class with the same name. Looks first on the classpath and then in
+   * the packagedClasses map.
+   * @param my_class the class to find.
+   * @return a jar file that contains the class, or null.
+   * @throws IOException
+   */
+  private static String findContainingJar(Class my_class, Map packagedClasses)
+      throws IOException {
+    ClassLoader loader = my_class.getClassLoader();
+    String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+
+    // first search the classpath
+    for (Enumeration itr = loader.getResources(class_file); itr.hasMoreElements();) {
+      URL url = itr.nextElement();
+      if ("jar".equals(url.getProtocol())) {
+        String toReturn = url.getPath();
+        if (toReturn.startsWith("file:")) {
+          toReturn = toReturn.substring("file:".length());
+        }
+        // URLDecoder is a misnamed class, since it actually decodes
+        // x-www-form-urlencoded MIME type rather than actual
+        // URL encoding (which the file path has). Therefore it would
+        // decode +s to ' 's which is incorrect (spaces are actually
+        // either unencoded or encoded as "%20"). Replace +s first, so
+        // that they are kept sacred during the decoding process.
+        toReturn = toReturn.replaceAll("\\+", "%2B");
+        toReturn = URLDecoder.decode(toReturn, "UTF-8");
+        return toReturn.replaceAll("!.*$", "");
+      }
+    }
+
+    // now look in any jars we've packaged using JarFinder. Returns null when
+    // no jar is found.
+    return packagedClasses.get(class_file);
+  }
+
+  /**
+   * Invoke 'getJar' on a JarFinder implementation. Useful for some job
+   * configuration contexts (HBASE-8140) and also for testing on MRv2. First
+   * check if we have HADOOP-9426. Lacking that, fall back to the backport.
+   * @param my_class the class to find.
+   * @return a jar file that contains the class, or null.
+   */
+  private static String getJar(Class my_class) {
+    String ret = null;
+    String hadoopJarFinder = "org.apache.hadoop.util.JarFinder";
+    Class jarFinder = null;
+    try {
+      LOG.debug("Looking for " + hadoopJarFinder + ".");
+      jarFinder = Class.forName(hadoopJarFinder);
+      LOG.debug(hadoopJarFinder + " found.");
+      Method getJar = jarFinder.getMethod("getJar", Class.class);
+      ret = (String) getJar.invoke(null, my_class);
+    } catch (ClassNotFoundException e) {
+      LOG.debug("Using backported JarFinder.");
+      ret = JarFinder.getJar(my_class);
+    } catch (InvocationTargetException e) {
+      // function was properly called, but threw it's own exception. Unwrap it
+      // and pass it on.
+      throw new RuntimeException(e.getCause());
+    } catch (Exception e) {
+      // toss all other exceptions, related to reflection failure
+      throw new RuntimeException("getJar invocation failed.", e);
+    }
+
+    return ret;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapper.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapper.java
new file mode 100644
index 0000000..bbceb63
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapper.java
@@ -0,0 +1,37 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.mapreduce.Mapper;
+
+/**
+ * Extends the base Mapper class to add the required input key
+ * and value classes.
+ *
+ * @param   The type of the key.
+ * @param   The type of the value.
+ * @see org.apache.hadoop.mapreduce.Mapper
+ */
+public abstract class TableMapper
+extends Mapper {
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputCommitter.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputCommitter.java
new file mode 100644
index 0000000..5289da7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputCommitter.java
@@ -0,0 +1,58 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * Small committer class that does not do anything.
+ */
+public class TableOutputCommitter extends OutputCommitter {
+
+  @Override
+  public void abortTask(TaskAttemptContext arg0) throws IOException {
+  }
+
+  @Override
+  public void cleanupJob(JobContext arg0) throws IOException {
+  }
+
+  @Override
+  public void commitTask(TaskAttemptContext arg0) throws IOException {
+  }
+
+  @Override
+  public boolean needsTaskCommit(TaskAttemptContext arg0) throws IOException {
+    return false;
+  }
+
+  @Override
+  public void setupJob(JobContext arg0) throws IOException {
+  }
+
+  @Override
+  public void setupTask(TaskAttemptContext arg0) throws IOException {
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
new file mode 100644
index 0000000..89c9603
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
@@ -0,0 +1,209 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * Convert Map/Reduce output and write it to an HBase table. The KEY is ignored
+ * while the output value must be either a {@link Put} or a
+ * {@link Delete} instance.
+ *
+ * @param   The type of the key. Ignored in this class.
+ */
+public class TableOutputFormat extends OutputFormat
+implements Configurable {
+
+  private final Log LOG = LogFactory.getLog(TableOutputFormat.class);
+
+  /** Job parameter that specifies the output table. */
+  public static final String OUTPUT_TABLE = "hbase.mapred.outputtable";
+
+  /**
+   * Optional job parameter to specify a peer cluster.
+   * Used specifying remote cluster when copying between hbase clusters (the
+   * source is picked up from hbase-site.xml).
+   * @see TableMapReduceUtil#initTableReducerJob(String, Class, org.apache.hadoop.mapreduce.Job, Class, String, String, String)
+   */
+  public static final String QUORUM_ADDRESS = "hbase.mapred.output.quorum";
+
+  /** Optional job parameter to specify peer cluster's ZK client port */
+  public static final String QUORUM_PORT = "hbase.mapred.output.quorum.port";
+
+  /** Optional specification of the rs class name of the peer cluster */
+  public static final String
+      REGION_SERVER_CLASS = "hbase.mapred.output.rs.class";
+  /** Optional specification of the rs impl name of the peer cluster */
+  public static final String
+      REGION_SERVER_IMPL = "hbase.mapred.output.rs.impl";
+
+  /** The configuration. */
+  private Configuration conf = null;
+
+  private HTable table;
+
+  /**
+   * Writes the reducer output to an HBase table.
+   *
+   * @param   The type of the key.
+   */
+  protected static class TableRecordWriter
+  extends RecordWriter {
+
+    /** The table to write to. */
+    private HTable table;
+
+    /**
+     * Instantiate a TableRecordWriter with the HBase HClient for writing.
+     *
+     * @param table  The table to write to.
+     */
+    public TableRecordWriter(HTable table) {
+      this.table = table;
+    }
+
+    /**
+     * Closes the writer, in this case flush table commits.
+     *
+     * @param context  The context.
+     * @throws IOException When closing the writer fails.
+     * @see org.apache.hadoop.mapreduce.RecordWriter#close(org.apache.hadoop.mapreduce.TaskAttemptContext)
+     */
+    @Override
+    public void close(TaskAttemptContext context)
+    throws IOException {
+      table.close();
+    }
+
+    /**
+     * Writes a key/value pair into the table.
+     *
+     * @param key  The key.
+     * @param value  The value.
+     * @throws IOException When writing fails.
+     * @see org.apache.hadoop.mapreduce.RecordWriter#write(java.lang.Object, java.lang.Object)
+     */
+    @Override
+    public void write(KEY key, Writable value)
+    throws IOException {
+      if (value instanceof Put) this.table.put(new Put((Put)value));
+      else if (value instanceof Delete) this.table.delete(new Delete((Delete)value));
+      else throw new IOException("Pass a Delete or a Put");
+    }
+  }
+
+  /**
+   * Creates a new record writer.
+   *
+   * @param context  The current task context.
+   * @return The newly created writer instance.
+   * @throws IOException When creating the writer fails.
+   * @throws InterruptedException When the jobs is cancelled.
+   * @see org.apache.hadoop.mapreduce.lib.output.FileOutputFormat#getRecordWriter(org.apache.hadoop.mapreduce.TaskAttemptContext)
+   */
+  @Override
+  public RecordWriter getRecordWriter(
+    TaskAttemptContext context)
+  throws IOException, InterruptedException {
+    return new TableRecordWriter(this.table);
+  }
+
+  /**
+   * Checks if the output target exists.
+   *
+   * @param context  The current context.
+   * @throws IOException When the check fails.
+   * @throws InterruptedException When the job is aborted.
+   * @see org.apache.hadoop.mapreduce.OutputFormat#checkOutputSpecs(org.apache.hadoop.mapreduce.JobContext)
+   */
+  @Override
+  public void checkOutputSpecs(JobContext context) throws IOException,
+      InterruptedException {
+    // TODO Check if the table exists?
+
+  }
+
+  /**
+   * Returns the output committer.
+   *
+   * @param context  The current context.
+   * @return The committer.
+   * @throws IOException When creating the committer fails.
+   * @throws InterruptedException When the job is aborted.
+   * @see org.apache.hadoop.mapreduce.OutputFormat#getOutputCommitter(org.apache.hadoop.mapreduce.TaskAttemptContext)
+   */
+  @Override
+  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
+  throws IOException, InterruptedException {
+    return new TableOutputCommitter();
+  }
+
+  public Configuration getConf() {
+    return conf;
+  }
+
+  @Override
+  public void setConf(Configuration otherConf) {
+    this.conf = HBaseConfiguration.create(otherConf);
+    String tableName = this.conf.get(OUTPUT_TABLE);
+    if(tableName == null || tableName.length() <= 0) {
+      throw new IllegalArgumentException("Must specify table name");
+    }
+    String address = this.conf.get(QUORUM_ADDRESS);
+    int zkClientPort = conf.getInt(QUORUM_PORT, 0);
+    String serverClass = this.conf.get(REGION_SERVER_CLASS);
+    String serverImpl = this.conf.get(REGION_SERVER_IMPL);
+    try {
+      if (address != null) {
+        ZKUtil.applyClusterKeyToConf(this.conf, address);
+      }
+      if (serverClass != null) {
+        this.conf.set(HConstants.REGION_SERVER_CLASS, serverClass);
+        this.conf.set(HConstants.REGION_SERVER_IMPL, serverImpl);
+      }
+      if (zkClientPort != 0) {
+        conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
+      }
+      this.table = new HTable(this.conf, tableName);
+      this.table.setAutoFlush(false);
+      LOG.info("Created table instance for "  + tableName);
+    } catch(IOException e) {
+      LOG.error(e);
+      throw new RuntimeException(e);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReader.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReader.java
new file mode 100644
index 0000000..a55f82a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReader.java
@@ -0,0 +1,148 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * Iterate over an HBase table data, return (ImmutableBytesWritable, Result)
+ * pairs.
+ */
+public class TableRecordReader
+extends RecordReader {
+
+  private TableRecordReaderImpl recordReaderImpl = new TableRecordReaderImpl();
+
+  /**
+   * Restart from survivable exceptions by creating a new scanner.
+   *
+   * @param firstRow  The first row to start at.
+   * @throws IOException When restarting fails.
+   */
+  public void restart(byte[] firstRow) throws IOException {
+    this.recordReaderImpl.restart(firstRow);
+  }
+
+
+  /**
+   * Sets the HBase table.
+   *
+   * @param htable  The {@link HTable} to scan.
+   */
+  public void setHTable(HTable htable) {
+    this.recordReaderImpl.setHTable(htable);
+  }
+
+  /**
+   * Sets the scan defining the actual details like columns etc.
+   *
+   * @param scan  The scan to set.
+   */
+  public void setScan(Scan scan) {
+    this.recordReaderImpl.setScan(scan);
+  }
+
+  /**
+   * Closes the split.
+   *
+   * @see org.apache.hadoop.mapreduce.RecordReader#close()
+   */
+  @Override
+  public void close() {
+    this.recordReaderImpl.close();
+  }
+
+  /**
+   * Returns the current key.
+   *
+   * @return The current key.
+   * @throws IOException
+   * @throws InterruptedException When the job is aborted.
+   * @see org.apache.hadoop.mapreduce.RecordReader#getCurrentKey()
+   */
+  @Override
+  public ImmutableBytesWritable getCurrentKey() throws IOException,
+      InterruptedException {
+    return this.recordReaderImpl.getCurrentKey();
+  }
+
+  /**
+   * Returns the current value.
+   *
+   * @return The current value.
+   * @throws IOException When the value is faulty.
+   * @throws InterruptedException When the job is aborted.
+   * @see org.apache.hadoop.mapreduce.RecordReader#getCurrentValue()
+   */
+  @Override
+  public Result getCurrentValue() throws IOException, InterruptedException {
+    return this.recordReaderImpl.getCurrentValue();
+  }
+
+  /**
+   * Initializes the reader.
+   *
+   * @param inputsplit  The split to work with.
+   * @param context  The current task context.
+   * @throws IOException When setting up the reader fails.
+   * @throws InterruptedException When the job is aborted.
+   * @see org.apache.hadoop.mapreduce.RecordReader#initialize(
+   *   org.apache.hadoop.mapreduce.InputSplit,
+   *   org.apache.hadoop.mapreduce.TaskAttemptContext)
+   */
+  @Override
+  public void initialize(InputSplit inputsplit,
+      TaskAttemptContext context) throws IOException,
+      InterruptedException {
+    this.recordReaderImpl.initialize(inputsplit, context);
+  }
+
+  /**
+   * Positions the record reader to the next record.
+   *
+   * @return true if there was another record.
+   * @throws IOException When reading the record failed.
+   * @throws InterruptedException When the job was aborted.
+   * @see org.apache.hadoop.mapreduce.RecordReader#nextKeyValue()
+   */
+  @Override
+  public boolean nextKeyValue() throws IOException, InterruptedException {
+    return this.recordReaderImpl.nextKeyValue();
+  }
+
+  /**
+   * The current progress of the record reader through its data.
+   *
+   * @return A number between 0.0 and 1.0, the fraction of the data read.
+   * @see org.apache.hadoop.mapreduce.RecordReader#getProgress()
+   */
+  @Override
+  public float getProgress() {
+    return this.recordReaderImpl.getProgress();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
new file mode 100644
index 0000000..d91b555
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.ScannerCallable;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Iterate over an HBase table data, return (ImmutableBytesWritable, Result)
+ * pairs.
+ */
+public class TableRecordReaderImpl {
+  public static final String LOG_PER_ROW_COUNT
+    = "hbase.mapreduce.log.scanner.rowcount";
+
+  static final Log LOG = LogFactory.getLog(TableRecordReader.class);
+
+  // HBASE_COUNTER_GROUP_NAME is the name of mapreduce counter group for HBase
+  private static final String HBASE_COUNTER_GROUP_NAME =
+    "HBase Counters";
+  private ResultScanner scanner = null;
+  private Scan scan = null;
+  private Scan currentScan = null;
+  private HTable htable = null;
+  private byte[] lastSuccessfulRow = null;
+  private ImmutableBytesWritable key = null;
+  private Result value = null;
+  private TaskAttemptContext context = null;
+  private Method getCounter = null;
+  private long numRestarts = 0;
+  private long timestamp;
+  private int rowcount;
+  private boolean logScannerActivity = false;
+  private int logPerRowCount = 100;
+
+  /**
+   * Restart from survivable exceptions by creating a new scanner.
+   *
+   * @param firstRow  The first row to start at.
+   * @throws IOException When restarting fails.
+   */
+  public void restart(byte[] firstRow) throws IOException {
+    currentScan = new Scan(scan);
+    currentScan.setStartRow(firstRow);
+    currentScan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE,
+      Bytes.toBytes(Boolean.TRUE));
+    this.scanner = this.htable.getScanner(currentScan);
+    if (logScannerActivity) {
+      LOG.info("Current scan=" + currentScan.toString());
+      timestamp = System.currentTimeMillis();
+      rowcount = 0;
+    }
+  }
+
+  /**
+   * In new mapreduce APIs, TaskAttemptContext has two getCounter methods
+   * Check if getCounter(String, String) method is available.
+   * @return The getCounter method or null if not available.
+   * @throws IOException
+   */
+  private Method retrieveGetCounterWithStringsParams(TaskAttemptContext context)
+  throws IOException {
+    Method m = null;
+    try {
+      m = context.getClass().getMethod("getCounter",
+        new Class [] {String.class, String.class});
+    } catch (SecurityException e) {
+      throw new IOException("Failed test for getCounter", e);
+    } catch (NoSuchMethodException e) {
+      // Ignore
+    }
+    return m;
+  }
+
+  /**
+   * Sets the HBase table.
+   *
+   * @param htable  The {@link HTable} to scan.
+   */
+  public void setHTable(HTable htable) {
+    Configuration conf = htable.getConfiguration();
+    logScannerActivity = conf.getBoolean(
+      ScannerCallable.LOG_SCANNER_ACTIVITY, false);
+    logPerRowCount = conf.getInt(LOG_PER_ROW_COUNT, 100);
+    this.htable = htable;
+  }
+
+  /**
+   * Sets the scan defining the actual details like columns etc.
+   *
+   * @param scan  The scan to set.
+   */
+  public void setScan(Scan scan) {
+    this.scan = scan;
+  }
+
+  /**
+   * Build the scanner. Not done in constructor to allow for extension.
+   *
+   * @throws IOException, InterruptedException
+   */
+  public void initialize(InputSplit inputsplit,
+      TaskAttemptContext context) throws IOException,
+      InterruptedException {
+    if (context != null) {
+      this.context = context;
+      getCounter = retrieveGetCounterWithStringsParams(context);
+    }
+    restart(scan.getStartRow());
+  }
+
+  /**
+   * Closes the split.
+   *
+   *
+   */
+  public void close() {
+    this.scanner.close();
+  }
+
+  /**
+   * Returns the current key.
+   *
+   * @return The current key.
+   * @throws IOException
+   * @throws InterruptedException When the job is aborted.
+   */
+  public ImmutableBytesWritable getCurrentKey() throws IOException,
+      InterruptedException {
+    return key;
+  }
+
+  /**
+   * Returns the current value.
+   *
+   * @return The current value.
+   * @throws IOException When the value is faulty.
+   * @throws InterruptedException When the job is aborted.
+   */
+  public Result getCurrentValue() throws IOException, InterruptedException {
+    return value;
+  }
+
+
+  /**
+   * Positions the record reader to the next record.
+   *
+   * @return true if there was another record.
+   * @throws IOException When reading the record failed.
+   * @throws InterruptedException When the job was aborted.
+   */
+  public boolean nextKeyValue() throws IOException, InterruptedException {
+    if (key == null) key = new ImmutableBytesWritable();
+    if (value == null) value = new Result();
+    try {
+      try {
+        value = this.scanner.next();
+        if (logScannerActivity) {
+          rowcount ++;
+          if (rowcount >= logPerRowCount) {
+            long now = System.currentTimeMillis();
+            LOG.info("Mapper took " + (now-timestamp)
+              + "ms to process " + rowcount + " rows");
+            timestamp = now;
+            rowcount = 0;
+          }
+        }
+      } catch (IOException e) {
+        // try to handle all IOExceptions by restarting
+        // the scanner, if the second call fails, it will be rethrown
+        LOG.info("recovered from " + StringUtils.stringifyException(e));
+        if (lastSuccessfulRow == null) {
+          LOG.warn("We are restarting the first next() invocation," +
+              " if your mapper has restarted a few other times like this" +
+              " then you should consider killing this job and investigate" +
+              " why it's taking so long.");
+        }
+        if (lastSuccessfulRow == null) {
+          restart(scan.getStartRow());
+        } else {
+          restart(lastSuccessfulRow);
+          scanner.next();    // skip presumed already mapped row
+        }
+        value = scanner.next();
+        numRestarts++;
+      }
+      if (value != null && value.size() > 0) {
+        key.set(value.getRow());
+        lastSuccessfulRow = key.get();
+        return true;
+      }
+
+      updateCounters();
+      return false;
+    } catch (IOException ioe) {
+      if (logScannerActivity) {
+        long now = System.currentTimeMillis();
+        LOG.info("Mapper took " + (now-timestamp)
+          + "ms to process " + rowcount + " rows");
+        LOG.info(ioe);
+        String lastRow = lastSuccessfulRow == null ?
+          "null" : Bytes.toStringBinary(lastSuccessfulRow);
+        LOG.info("lastSuccessfulRow=" + lastRow);
+      }
+      throw ioe;
+    }
+  }
+
+  /**
+   * If hbase runs on new version of mapreduce, RecordReader has access to
+   * counters thus can update counters based on scanMetrics.
+   * If hbase runs on old version of mapreduce, it won't be able to get
+   * access to counters and TableRecorderReader can't update counter values.
+   * @throws IOException
+   */
+  private void updateCounters() throws IOException {
+    // we can get access to counters only if hbase uses new mapreduce APIs
+    if (this.getCounter == null) {
+      return;
+    }
+
+    byte[] serializedMetrics = currentScan.getAttribute(
+        Scan.SCAN_ATTRIBUTES_METRICS_DATA);
+    if (serializedMetrics == null || serializedMetrics.length == 0 ) {
+      return;
+    }
+
+    DataInputBuffer in = new DataInputBuffer();
+    in.reset(serializedMetrics, 0, serializedMetrics.length);
+    ScanMetrics scanMetrics = new ScanMetrics();
+    scanMetrics.readFields(in);
+    MetricsTimeVaryingLong[] mlvs =
+      scanMetrics.getMetricsTimeVaryingLongArray();
+
+    try {
+      for (MetricsTimeVaryingLong mlv : mlvs) {
+        Counter ct = (Counter)this.getCounter.invoke(context,
+          HBASE_COUNTER_GROUP_NAME, mlv.getName());
+        ct.increment(mlv.getCurrentIntervalValue());
+      }
+      ((Counter) this.getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME,
+          "NUM_SCANNER_RESTARTS")).increment(numRestarts);
+    } catch (Exception e) {
+      LOG.debug("can't update counter." + StringUtils.stringifyException(e));
+    }
+  }
+
+  /**
+   * The current progress of the record reader through its data.
+   *
+   * @return A number between 0.0 and 1.0, the fraction of the data read.
+   */
+  public float getProgress() {
+    // Depends on the total number of tuples
+    return 0;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableReducer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableReducer.java
new file mode 100644
index 0000000..d087f85
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableReducer.java
@@ -0,0 +1,44 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Reducer;
+
+/**
+ * Extends the basic Reducer class to add the required key and
+ * value input/output classes. While the input key and value as well as the
+ * output key can be anything handed in from the previous map phase the output
+ * value must be either a {@link org.apache.hadoop.hbase.client.Put Put}
+ * or a {@link org.apache.hadoop.hbase.client.Delete Delete} instance when
+ * using the {@link TableOutputFormat} class.
+ * 
        
+ * This class is extended by {@link IdentityTableReducer} but can also be
+ * subclassed to implement similar features or any custom code needed. It has
+ * the advantage to enforce the output value to a specific basic type.
+ *
+ * @param   The type of the input key.
+ * @param   The type of the input value.
+ * @param   The type of the output key.
+ * @see org.apache.hadoop.mapreduce.Reducer
+ */
+public abstract class TableReducer
+extends Reducer {
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSplit.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSplit.java
new file mode 100644
index 0000000..bb9fb46
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSplit.java
@@ -0,0 +1,292 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapreduce.InputSplit;
+
+/**
+ * A table split corresponds to a key range (low, high) and an optional scanner.
+ * All references to row below refer to the key of the row.
+ */
+public class TableSplit extends InputSplit
+implements Writable, Comparable { 
+  public static final Log LOG = LogFactory.getLog(TableSplit.class);
+
+  // should be < 0 (@see #readFields(DataInput))
+  // version 1 supports Scan data member
+  enum Version {
+    UNVERSIONED(0),
+    // Initial number we put on TableSplit when we introduced versioning.
+    INITIAL(-1);
+
+    final int code;
+    static final Version[] byCode;
+    static {
+      byCode = Version.values();
+      for (int i = 0; i < byCode.length; i++) {
+        if (byCode[i].code != -1 * i) {
+          throw new AssertionError("Values in this enum should be descending by one");
+        }
+      }
+    }
+
+    Version(int code) {
+      this.code = code;
+    }
+
+    boolean atLeast(Version other) {
+      return code <= other.code;
+    }
+
+    static Version fromCode(int code) {
+      return byCode[code * -1];
+    }
+  }
+
+  private static final Version VERSION = Version.INITIAL;
+
+  private byte [] tableName;
+  private byte [] startRow;
+  private byte [] endRow;
+  private String regionLocation;
+  private String scan = ""; // stores the serialized form of the Scan
+
+  /** Default constructor. */
+  public TableSplit() {
+    this(HConstants.EMPTY_BYTE_ARRAY, null, HConstants.EMPTY_BYTE_ARRAY,
+      HConstants.EMPTY_BYTE_ARRAY, "");
+  }
+
+  /**
+   * Creates a new instance while assigning all variables.
+   *
+   * @param tableName  The name of the current table.
+   * @param scan The scan associated with this split.
+   * @param startRow  The start row of the split.
+   * @param endRow  The end row of the split.
+   * @param location  The location of the region.
+   */
+  public TableSplit(byte [] tableName, Scan scan, byte [] startRow, byte [] endRow,
+      final String location) {
+    this.tableName = tableName;
+    try {
+      this.scan =
+        (null == scan) ? "" : TableMapReduceUtil.convertScanToString(scan);
+    } catch (IOException e) {
+      LOG.warn("Failed to convert Scan to String", e);
+    }
+    this.startRow = startRow;
+    this.endRow = endRow;
+    this.regionLocation = location;
+  }
+  
+  /**
+   * Creates a new instance without a scanner.
+   *
+   * @param tableName The name of the current table.
+   * @param startRow The start row of the split.
+   * @param endRow The end row of the split.
+   * @param location The location of the region.
+   */
+  public TableSplit(byte[] tableName, byte[] startRow, byte[] endRow,
+      final String location) {
+    this(tableName, null, startRow, endRow, location);
+  }
+
+  /**
+   * Returns a Scan object from the stored string representation.
+   *
+   * @return Returns a Scan object based on the stored scanner.
+   * @throws IOException
+   */
+  public Scan getScan() throws IOException {
+    return TableMapReduceUtil.convertStringToScan(this.scan);
+  }
+
+  /**
+   * Returns the table name.
+   *
+   * @return The table name.
+   */
+  public byte [] getTableName() {
+    return tableName;
+  }
+
+  /**
+   * Returns the start row.
+   *
+   * @return The start row.
+   */
+  public byte [] getStartRow() {
+    return startRow;
+  }
+
+  /**
+   * Returns the end row.
+   *
+   * @return The end row.
+   */
+  public byte [] getEndRow() {
+    return endRow;
+  }
+
+  /**
+   * Returns the region location.
+   *
+   * @return The region's location.
+   */
+  public String getRegionLocation() {
+    return regionLocation;
+  }
+
+  /**
+   * Returns the region's location as an array.
+   *
+   * @return The array containing the region location.
+   * @see org.apache.hadoop.mapreduce.InputSplit#getLocations()
+   */
+  @Override
+  public String[] getLocations() {
+    return new String[] {regionLocation};
+  }
+
+  /**
+   * Returns the length of the split.
+   *
+   * @return The length of the split.
+   * @see org.apache.hadoop.mapreduce.InputSplit#getLength()
+   */
+  @Override
+  public long getLength() {
+    // Not clear how to obtain this... seems to be used only for sorting splits
+    return 0;
+  }
+
+  /**
+   * Reads the values of each field.
+   *
+   * @param in  The input to read from.
+   * @throws IOException When reading the input fails.
+   */
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    Version version = Version.UNVERSIONED;
+    // TableSplit was not versioned in the beginning.
+    // In order to introduce it now, we make use of the fact
+    // that tableName was written with Bytes.writeByteArray,
+    // which encodes the array length as a vint which is >= 0.
+    // Hence if the vint is >= 0 we have an old version and the vint
+    // encodes the length of tableName.
+    // If < 0 we just read the version and the next vint is the length.
+    // @see Bytes#readByteArray(DataInput)
+    int len = WritableUtils.readVInt(in);
+    if (len < 0) {
+      // what we just read was the version
+      version = Version.fromCode(len);
+      len = WritableUtils.readVInt(in);
+    }
+    tableName = new byte[len];
+    in.readFully(tableName);
+    startRow = Bytes.readByteArray(in);
+    endRow = Bytes.readByteArray(in);
+    regionLocation = Bytes.toString(Bytes.readByteArray(in));
+    if (version.atLeast(Version.INITIAL)) {
+      scan = Bytes.toString(Bytes.readByteArray(in));
+    }
+  }
+
+  /**
+   * Writes the field values to the output.
+   *
+   * @param out  The output to write to.
+   * @throws IOException When writing the values to the output fails.
+   */
+  @Override
+  public void write(DataOutput out) throws IOException {
+    WritableUtils.writeVInt(out, VERSION.code);
+    Bytes.writeByteArray(out, tableName);
+    Bytes.writeByteArray(out, startRow);
+    Bytes.writeByteArray(out, endRow);
+    Bytes.writeByteArray(out, Bytes.toBytes(regionLocation));
+    Bytes.writeByteArray(out, Bytes.toBytes(scan));
+  }
+
+  /**
+   * Returns the details about this instance as a string.
+   *
+   * @return The values of this instance as a string.
+   * @see java.lang.Object#toString()
+   */
+  @Override
+  public String toString() {
+    return regionLocation + ":" +
+      Bytes.toStringBinary(startRow) + "," + Bytes.toStringBinary(endRow);
+  }
+
+  /**
+   * Compares this split against the given one.
+   *
+   * @param split  The split to compare to.
+   * @return The result of the comparison.
+   * @see java.lang.Comparable#compareTo(java.lang.Object)
+   */
+  @Override
+  public int compareTo(TableSplit split) {
+    // If The table name of the two splits is the same then compare start row
+    // otherwise compare based on table names
+    int tableNameComparison =
+        Bytes.compareTo(getTableName(), split.getTableName());
+    return tableNameComparison != 0 ? tableNameComparison : Bytes.compareTo(
+        getStartRow(), split.getStartRow());
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (o == null || !(o instanceof TableSplit)) {
+      return false;
+    }
+    return Bytes.equals(tableName, ((TableSplit)o).tableName) &&
+      Bytes.equals(startRow, ((TableSplit)o).startRow) &&
+      Bytes.equals(endRow, ((TableSplit)o).endRow) &&
+      regionLocation.equals(((TableSplit)o).regionLocation);
+  }
+
+    @Override
+    public int hashCode() {
+        int result = tableName != null ? Arrays.hashCode(tableName) : 0;
+        result = 31 * result + (scan != null ? scan.hashCode() : 0);
+        result = 31 * result + (startRow != null ? Arrays.hashCode(startRow) : 0);
+        result = 31 * result + (endRow != null ? Arrays.hashCode(endRow) : 0);
+        result = 31 * result + (regionLocation != null ? regionLocation.hashCode() : 0);
+        return result;
+    }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TsvImporterMapper.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TsvImporterMapper.java
new file mode 100644
index 0000000..398c708
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TsvImporterMapper.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Base64;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.IOException;
+
+/**
+ * Write table content out to files in hdfs.
+ */
+public class TsvImporterMapper
+extends Mapper
+{
+
+  /** Timestamp for all inserted rows */
+  private long ts;
+
+  /** Column seperator */
+  private String separator;
+
+  /** Should skip bad lines */
+  private boolean skipBadLines;
+  private Counter badLineCount;
+
+  private ImportTsv.TsvParser parser;
+
+  public long getTs() {
+    return ts;
+  }
+
+  public boolean getSkipBadLines() {
+    return skipBadLines;
+  }
+
+  public Counter getBadLineCount() {
+    return badLineCount;
+  }
+
+  public void incrementBadLineCount(int count) {
+    this.badLineCount.increment(count);
+  }
+
+  /**
+   * Handles initializing this class with objects specific to it (i.e., the parser).
+   * Common initialization that might be leveraged by a subsclass is done in
+   * doSetup. Hence a subclass may choose to override this method
+   * and call doSetup as well before handling it's own custom params.
+   *
+   * @param context
+   */
+  @Override
+  protected void setup(Context context) {
+    doSetup(context);
+
+    Configuration conf = context.getConfiguration();
+
+    parser = new ImportTsv.TsvParser(conf.get(ImportTsv.COLUMNS_CONF_KEY),
+                           separator);
+    if (parser.getRowKeyColumnIndex() == -1) {
+      throw new RuntimeException("No row key column specified");
+    }
+  }
+
+  /**
+   * Handles common parameter initialization that a subclass might want to leverage.
+   * @param context
+   */
+  protected void doSetup(Context context) {
+    Configuration conf = context.getConfiguration();
+
+    // If a custom separator has been used,
+    // decode it back from Base64 encoding.
+    separator = conf.get(ImportTsv.SEPARATOR_CONF_KEY);
+    if (separator == null) {
+      separator = ImportTsv.DEFAULT_SEPARATOR;
+    } else {
+      separator = new String(Base64.decode(separator));
+    }
+
+    // Should never get 0 as we are setting this to a valid value in job
+    // configuration.
+    ts = conf.getLong(ImportTsv.TIMESTAMP_CONF_KEY, 0);
+
+    skipBadLines = context.getConfiguration().getBoolean(
+        ImportTsv.SKIP_LINES_CONF_KEY, true);
+    badLineCount = context.getCounter("ImportTsv", "Bad Lines");
+  }
+
+  /**
+   * Convert a line of TSV text into an HBase table row.
+   */
+  @Override
+  public void map(LongWritable offset, Text value,
+    Context context)
+  throws IOException {
+    byte[] lineBytes = value.getBytes();
+
+    try {
+      ImportTsv.TsvParser.ParsedLine parsed = parser.parse(
+          lineBytes, value.getLength());
+      ImmutableBytesWritable rowKey =
+        new ImmutableBytesWritable(lineBytes,
+            parsed.getRowKeyOffset(),
+            parsed.getRowKeyLength());
+      // Retrieve timestamp if exists
+      ts = parsed.getTimestamp(ts);
+
+      Put put = new Put(rowKey.copyBytes());
+      for (int i = 0; i < parsed.getColumnCount(); i++) {
+        if (i == parser.getRowKeyColumnIndex()
+            || i == parser.getTimestampKeyColumnIndex()) {
+          continue;
+        }
+        KeyValue kv = new KeyValue(
+            lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(),
+            parser.getFamily(i), 0, parser.getFamily(i).length,
+            parser.getQualifier(i), 0, parser.getQualifier(i).length,
+            ts,
+            KeyValue.Type.Put,
+            lineBytes, parsed.getColumnOffset(i), parsed.getColumnLength(i));
+        put.add(kv);
+      }
+      context.write(rowKey, put);
+    } catch (ImportTsv.TsvParser.BadTsvLineException badLine) {
+      if (skipBadLines) {
+        System.err.println(
+            "Bad line at offset: " + offset.get() + ":\n" +
+            badLine.getMessage());
+        incrementBadLineCount(1);
+        return;
+      } else {
+        throw new IOException(badLine);
+      }
+    } catch (IllegalArgumentException e) {
+      if (skipBadLines) {
+        System.err.println(
+            "Bad line at offset: " + offset.get() + ":\n" +
+            e.getMessage());
+        incrementBadLineCount(1);
+        return;
+      } else {
+        throw new IOException(e);
+      }
+    } catch (InterruptedException e) {
+      e.printStackTrace();
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
new file mode 100644
index 0000000..b0a7b69
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
@@ -0,0 +1,305 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * A tool to replay WAL files as a M/R job.
+ * The WAL can be replayed for a set of tables or all tables,
+ * and a timerange can be provided (in milliseconds).
+ * The WAL is filtered to the passed set of tables and  the output
+ * can optionally be mapped to another set of tables.
+ *
+ * WAL replay can also generate HFiles for later bulk importing,
+ * in that case the WAL is replayed for a single table only.
+ */
+public class WALPlayer extends Configured implements Tool {
+  final static String NAME = "WALPlayer";
+  final static String BULK_OUTPUT_CONF_KEY = "hlog.bulk.output";
+  final static String HLOG_INPUT_KEY = "hlog.input.dir";
+  final static String TABLES_KEY = "hlog.input.tables";
+  final static String TABLE_MAP_KEY = "hlog.input.tablesmap";
+
+  /**
+   * A mapper that just writes out KeyValues.
+   * This one can be used together with {@link KeyValueSortReducer}
+   */
+  static class HLogKeyValueMapper
+  extends Mapper {
+    private byte[] table;
+
+    @Override
+    public void map(HLogKey key, WALEdit value,
+      Context context)
+    throws IOException {
+      try {
+        // skip all other tables
+        if (Bytes.equals(table, key.getTablename())) {
+          for (KeyValue kv : value.getKeyValues()) {
+            if (HLog.isMetaFamily(kv.getFamily())) continue;
+            context.write(new ImmutableBytesWritable(kv.getRow()), kv);
+          }
+        }
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+
+    @Override
+    public void setup(Context context) throws IOException {
+      // only a single table is supported when HFiles are generated with HFileOutputFormat
+      String tables[] = context.getConfiguration().getStrings(TABLES_KEY);
+      if (tables == null || tables.length != 1) {
+        // this can only happen when HLogMapper is used directly by a class other than WALPlayer
+        throw new IOException("Exactly one table must be specified for bulk HFile case.");
+      }
+      table = Bytes.toBytes(tables[0]);
+    }
+  }
+
+  /**
+   * A mapper that writes out {@link Mutation} to be directly applied to
+   * a running HBase instance.
+   */
+  static class HLogMapper
+  extends Mapper {
+    private Map tables = new TreeMap(Bytes.BYTES_COMPARATOR);
+
+    @Override
+    public void map(HLogKey key, WALEdit value,
+      Context context)
+    throws IOException {
+      try {
+        if (tables.isEmpty() || tables.containsKey(key.getTablename())) {
+          byte[] targetTable = tables.isEmpty() ?
+                key.getTablename() :
+                tables.get(key.getTablename());
+          ImmutableBytesWritable tableOut = new ImmutableBytesWritable(targetTable);
+          Put put = null;
+          Delete del = null;
+          KeyValue lastKV = null;
+          for (KeyValue kv : value.getKeyValues()) {
+            // filtering HLog meta entries, see HLog.completeCacheFlushLogEdit
+            if (HLog.isMetaFamily(kv.getFamily())) continue;
+
+            // A WALEdit may contain multiple operations (HBASE-3584) and/or
+            // multiple rows (HBASE-5229).
+            // Aggregate as much as possible into a single Put/Delete
+            // operation before writing to the context.
+            if (lastKV == null || lastKV.getType() != kv.getType() || !lastKV.matchingRow(kv)) {
+              // row or type changed, write out aggregate KVs.
+              if (put != null) context.write(tableOut, put);
+              if (del != null) context.write(tableOut, del);
+
+              if (kv.isDelete()) {
+                del = new Delete(kv.getRow());
+              } else {
+                put = new Put(kv.getRow());
+              }
+            }
+            if (kv.isDelete()) {
+              del.addDeleteMarker(kv);
+            } else {
+              put.add(kv);
+            }
+            lastKV = kv;
+          }
+          // write residual KVs
+          if (put != null) context.write(tableOut, put);
+          if (del != null) context.write(tableOut, del);
+        }
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+
+    @Override
+    public void setup(Context context) throws IOException {
+      String[] tableMap = context.getConfiguration().getStrings(TABLE_MAP_KEY);
+      String[] tablesToUse = context.getConfiguration().getStrings(TABLES_KEY);
+      if (tablesToUse == null || tableMap == null || tablesToUse.length != tableMap.length) {
+        // this can only happen when HLogMapper is used directly by a class other than WALPlayer
+        throw new IOException("No tables or incorrect table mapping specified.");
+      }
+      int i = 0;
+      for (String table : tablesToUse) {
+        tables.put(Bytes.toBytes(table), Bytes.toBytes(tableMap[i++]));
+      }
+    }
+  }
+
+  /**
+   * @param conf The {@link Configuration} to use.
+   */
+  public WALPlayer(Configuration conf) {
+    super(conf);
+  }
+
+  void setupTime(Configuration conf, String option) throws IOException {
+    String val = conf.get(option);
+    if (val == null) return;
+    long ms;
+    try {
+      // first try to parse in user friendly form
+      ms = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SS").parse(val).getTime();
+    } catch (ParseException pe) {
+      try {
+        // then see if just a number of ms's was specified
+        ms = Long.parseLong(val);
+      } catch (NumberFormatException nfe) {
+        throw new IOException(option
+            + " must be specified either in the form 2001-02-20T16:35:06.99 "
+            + "or as number of milliseconds");
+      }
+    }
+    conf.setLong(option, ms);
+  }
+
+  /**
+   * Sets up the actual job.
+   *
+   * @param args  The command line parameters.
+   * @return The newly created job.
+   * @throws IOException When setting up the job fails.
+   */
+  public Job createSubmittableJob(String[] args)
+  throws IOException {
+    Configuration conf = getConf();
+    setupTime(conf, HLogInputFormat.START_TIME_KEY);
+    setupTime(conf, HLogInputFormat.END_TIME_KEY);
+    Path inputDir = new Path(args[0]);
+    String[] tables = args[1].split(",");
+    String[] tableMap;
+    if (args.length > 2) {
+      tableMap = args[2].split(",");
+      if (tableMap.length != tables.length) {
+        throw new IOException("The same number of tables and mapping must be provided.");
+      }
+    } else {
+      // if not mapping is specified map each table to itself
+      tableMap = tables;
+    }
+    conf.setStrings(TABLES_KEY, tables);
+    conf.setStrings(TABLE_MAP_KEY, tableMap);
+    Job job = new Job(conf, NAME + "_" + inputDir);
+    job.setJarByClass(WALPlayer.class);
+    FileInputFormat.setInputPaths(job, inputDir);
+    job.setInputFormatClass(HLogInputFormat.class);
+    job.setMapOutputKeyClass(ImmutableBytesWritable.class);
+    String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY);
+    if (hfileOutPath != null) {
+      // the bulk HFile case
+      if (tables.length != 1) {
+        throw new IOException("Exactly one table must be specified for the bulk export option");
+      }
+      HTable table = new HTable(conf, tables[0]);
+      job.setMapperClass(HLogKeyValueMapper.class);
+      job.setReducerClass(KeyValueSortReducer.class);
+      Path outputDir = new Path(hfileOutPath);
+      FileOutputFormat.setOutputPath(job, outputDir);
+      job.setMapOutputValueClass(KeyValue.class);
+      HFileOutputFormat.configureIncrementalLoad(job, table);
+      TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+          com.google.common.base.Preconditions.class);
+    } else {
+      // output to live cluster
+      job.setMapperClass(HLogMapper.class);
+      job.setOutputFormatClass(MultiTableOutputFormat.class);
+      TableMapReduceUtil.addDependencyJars(job);
+      // No reducers.
+      job.setNumReduceTasks(0);
+    }
+    return job;
+  }
+
+  /*
+   * @param errorMsg Error message.  Can be null.
+   */
+  private void usage(final String errorMsg) {
+    if (errorMsg != null && errorMsg.length() > 0) {
+      System.err.println("ERROR: " + errorMsg);
+    }
+    System.err.println("Usage: " + NAME + " [options]   []");
+    System.err.println("Read all WAL entries for .");
+    System.err.println("If no tables (\"\") are specific, all tables are imported.");
+    System.err.println("(Careful, even -ROOT- and .META. entries will be imported in that case.)");
+    System.err.println("Otherwise  is a comma separated list of tables.\n");
+    System.err.println("The WAL entries can be mapped to new set of tables via .");
+    System.err.println(" is a command separated list of targettables.");
+    System.err.println("If specified, each table in  must have a mapping.\n");
+    System.err.println("By default " + NAME + " will load data directly into HBase.");
+    System.err.println("To generate HFiles for a bulk data load instead, pass the option:");
+    System.err.println("  -D" + BULK_OUTPUT_CONF_KEY + "=/path/for/output");
+    System.err.println("  (Only one table can be specified, and no mapping is allowed!)");
+    System.err.println("Other options: (specify time range to WAL edit to consider)");
+    System.err.println("  -D" + HLogInputFormat.START_TIME_KEY + "=[date|ms]");
+    System.err.println("  -D" + HLogInputFormat.END_TIME_KEY + "=[date|ms]");
+    System.err.println("For performance also consider the following options:\n"
+        + "  -Dmapred.map.tasks.speculative.execution=false\n"
+        + "  -Dmapred.reduce.tasks.speculative.execution=false");
+  }
+
+  /**
+   * Main entry point.
+   *
+   * @param args  The command line parameters.
+   * @throws Exception When running the job fails.
+   */
+  public static void main(String[] args) throws Exception {
+    int ret = ToolRunner.run(new WALPlayer(HBaseConfiguration.create()), args);
+    System.exit(ret);
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
+    if (otherArgs.length < 2) {
+      usage("Wrong number of arguments: " + otherArgs.length);
+      System.exit(-1);
+    }
+    Job job = createSubmittableJob(otherArgs);
+    return job.waitForCompletion(true) ? 0 : 1;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
new file mode 100644
index 0000000..89c3428
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/InputSampler.java
@@ -0,0 +1,443 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.mapreduce.hadoopbackport;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Utility for collecting samples and writing a partition file for
+ * {@link TotalOrderPartitioner}.
+ *
+ * This is an identical copy of o.a.h.mapreduce.lib.partition.TotalOrderPartitioner
+ * from Hadoop trunk at r961542, with the exception of replacing
+ * TaskAttemptContextImpl with TaskAttemptContext.
+ */
+public class InputSampler extends Configured implements Tool  {
+
+  private static final Log LOG = LogFactory.getLog(InputSampler.class);
+
+  static int printUsage() {
+    System.out.println("sampler -r \n" +
+      "      [-inFormat ]\n" +
+      "      [-keyClass ]\n" +
+      "      [-splitRandom    | " +
+      "             // Sample from random splits at random (general)\n" +
+      "       -splitSample   | " +
+      "             // Sample from first records in splits (random data)\n"+
+      "       -splitInterval  ]" +
+      "             // Sample from splits at intervals (sorted data)");
+    System.out.println("Default sampler: -splitRandom 0.1 10000 10");
+    ToolRunner.printGenericCommandUsage(System.out);
+    return -1;
+  }
+
+  public InputSampler(Configuration conf) {
+    setConf(conf);
+  }
+
+  /**
+   * Interface to sample using an 
+   * {@link org.apache.hadoop.mapreduce.InputFormat}.
+   */
+  public interface Sampler {
+    /**
+     * For a given job, collect and return a subset of the keys from the
+     * input data.
+     */
+    K[] getSample(InputFormat inf, Job job) 
+    throws IOException, InterruptedException;
+  }
+
+  /**
+   * Samples the first n records from s splits.
+   * Inexpensive way to sample random data.
+   */
+  public static class SplitSampler implements Sampler {
+
+    private final int numSamples;
+    private final int maxSplitsSampled;
+
+    /**
+     * Create a SplitSampler sampling all splits.
+     * Takes the first numSamples / numSplits records from each split.
+     * @param numSamples Total number of samples to obtain from all selected
+     *                   splits.
+     */
+    public SplitSampler(int numSamples) {
+      this(numSamples, Integer.MAX_VALUE);
+    }
+
+    /**
+     * Create a new SplitSampler.
+     * @param numSamples Total number of samples to obtain from all selected
+     *                   splits.
+     * @param maxSplitsSampled The maximum number of splits to examine.
+     */
+    public SplitSampler(int numSamples, int maxSplitsSampled) {
+      this.numSamples = numSamples;
+      this.maxSplitsSampled = maxSplitsSampled;
+    }
+
+    /**
+     * From each split sampled, take the first numSamples / numSplits records.
+     */
+    @SuppressWarnings("unchecked") // ArrayList::toArray doesn't preserve type
+    public K[] getSample(InputFormat inf, Job job) 
+        throws IOException, InterruptedException {
+      List splits = inf.getSplits(job);
+      ArrayList samples = new ArrayList(numSamples);
+      int splitsToSample = Math.min(maxSplitsSampled, splits.size());
+      int samplesPerSplit = numSamples / splitsToSample;
+      long records = 0;
+      for (int i = 0; i < splitsToSample; ++i) {
+        TaskAttemptContext samplingContext = getTaskAttemptContext(job);
+        RecordReader reader = inf.createRecordReader(
+            splits.get(i), samplingContext);
+        reader.initialize(splits.get(i), samplingContext);
+        while (reader.nextKeyValue()) {
+          samples.add(ReflectionUtils.copy(job.getConfiguration(),
+                                           reader.getCurrentKey(), null));
+          ++records;
+          if ((i+1) * samplesPerSplit <= records) {
+            break;
+          }
+        }
+        reader.close();
+      }
+      return (K[])samples.toArray();
+    }
+  }
+
+  /**
+   * This method is about making hbase portable, making it so it can run on
+   * more than just hadoop 0.20.  In later hadoops, TaskAttemptContext became
+   * an Interface.  But in hadoops where TAC is an Interface, we shouldn't
+   * be using the classes that are in this package; we should be using the
+   * native Hadoop ones (We'll throw a ClassNotFoundException if end up in
+   * here when we should be using native hadoop TotalOrderPartitioner).
+   * @param job
+   * @return Context
+   * @throws IOException 
+   */
+  public static TaskAttemptContext getTaskAttemptContext(final Job job)
+  throws IOException {
+    Constructor c;
+    try {
+      c = TaskAttemptContext.class.getConstructor(Configuration.class, TaskAttemptID.class);
+    } catch (Exception e) {
+      throw new IOException("Failed getting constructor", e);
+    }
+    try {
+      return c.newInstance(job.getConfiguration(), new TaskAttemptID());
+    } catch (Exception e) {
+      throw new IOException("Failed creating instance", e);
+    }
+  }
+
+  /**
+   * Sample from random points in the input.
+   * General-purpose sampler. Takes numSamples / maxSplitsSampled inputs from
+   * each split.
+   */
+  public static class RandomSampler implements Sampler {
+    private double freq;
+    private final int numSamples;
+    private final int maxSplitsSampled;
+
+    /**
+     * Create a new RandomSampler sampling all splits.
+     * This will read every split at the client, which is very expensive.
+     * @param freq Probability with which a key will be chosen.
+     * @param numSamples Total number of samples to obtain from all selected
+     *                   splits.
+     */
+    public RandomSampler(double freq, int numSamples) {
+      this(freq, numSamples, Integer.MAX_VALUE);
+    }
+
+    /**
+     * Create a new RandomSampler.
+     * @param freq Probability with which a key will be chosen.
+     * @param numSamples Total number of samples to obtain from all selected
+     *                   splits.
+     * @param maxSplitsSampled The maximum number of splits to examine.
+     */
+    public RandomSampler(double freq, int numSamples, int maxSplitsSampled) {
+      this.freq = freq;
+      this.numSamples = numSamples;
+      this.maxSplitsSampled = maxSplitsSampled;
+    }
+
+    /**
+     * Randomize the split order, then take the specified number of keys from
+     * each split sampled, where each key is selected with the specified
+     * probability and possibly replaced by a subsequently selected key when
+     * the quota of keys from that split is satisfied.
+     */
+    @SuppressWarnings("unchecked") // ArrayList::toArray doesn't preserve type
+    public K[] getSample(InputFormat inf, Job job) 
+        throws IOException, InterruptedException {
+      List splits = inf.getSplits(job);
+      ArrayList samples = new ArrayList(numSamples);
+      int splitsToSample = Math.min(maxSplitsSampled, splits.size());
+
+      Random r = new Random();
+      long seed = r.nextLong();
+      r.setSeed(seed);
+      LOG.debug("seed: " + seed);
+      // shuffle splits
+      for (int i = 0; i < splits.size(); ++i) {
+        InputSplit tmp = splits.get(i);
+        int j = r.nextInt(splits.size());
+        splits.set(i, splits.get(j));
+        splits.set(j, tmp);
+      }
+      // our target rate is in terms of the maximum number of sample splits,
+      // but we accept the possibility of sampling additional splits to hit
+      // the target sample keyset
+      for (int i = 0; i < splitsToSample ||
+                     (i < splits.size() && samples.size() < numSamples); ++i) {
+        TaskAttemptContext samplingContext = getTaskAttemptContext(job);
+        RecordReader reader = inf.createRecordReader(
+            splits.get(i), samplingContext);
+        reader.initialize(splits.get(i), samplingContext);
+        while (reader.nextKeyValue()) {
+          if (r.nextDouble() <= freq) {
+            if (samples.size() < numSamples) {
+              samples.add(ReflectionUtils.copy(job.getConfiguration(),
+                                               reader.getCurrentKey(), null));
+            } else {
+              // When exceeding the maximum number of samples, replace a
+              // random element with this one, then adjust the frequency
+              // to reflect the possibility of existing elements being
+              // pushed out
+              int ind = r.nextInt(numSamples);
+              if (ind != numSamples) {
+                samples.set(ind, ReflectionUtils.copy(job.getConfiguration(),
+                                 reader.getCurrentKey(), null));
+              }
+              freq *= (numSamples - 1) / (double) numSamples;
+            }
+          }
+        }
+        reader.close();
+      }
+      return (K[])samples.toArray();
+    }
+  }
+
+  /**
+   * Sample from s splits at regular intervals.
+   * Useful for sorted data.
+   */
+  public static class IntervalSampler implements Sampler {
+    private final double freq;
+    private final int maxSplitsSampled;
+
+    /**
+     * Create a new IntervalSampler sampling all splits.
+     * @param freq The frequency with which records will be emitted.
+     */
+    public IntervalSampler(double freq) {
+      this(freq, Integer.MAX_VALUE);
+    }
+
+    /**
+     * Create a new IntervalSampler.
+     * @param freq The frequency with which records will be emitted.
+     * @param maxSplitsSampled The maximum number of splits to examine.
+     * @see #getSample
+     */
+    public IntervalSampler(double freq, int maxSplitsSampled) {
+      this.freq = freq;
+      this.maxSplitsSampled = maxSplitsSampled;
+    }
+
+    /**
+     * For each split sampled, emit when the ratio of the number of records
+     * retained to the total record count is less than the specified
+     * frequency.
+     */
+    @SuppressWarnings("unchecked") // ArrayList::toArray doesn't preserve type
+    public K[] getSample(InputFormat inf, Job job) 
+        throws IOException, InterruptedException {
+      List splits = inf.getSplits(job);
+      ArrayList samples = new ArrayList();
+      int splitsToSample = Math.min(maxSplitsSampled, splits.size());
+      long records = 0;
+      long kept = 0;
+      for (int i = 0; i < splitsToSample; ++i) {
+        TaskAttemptContext samplingContext = getTaskAttemptContext(job);
+        RecordReader reader = inf.createRecordReader(
+            splits.get(i), samplingContext);
+        reader.initialize(splits.get(i), samplingContext);
+        while (reader.nextKeyValue()) {
+          ++records;
+          if ((double) kept / records < freq) {
+            samples.add(ReflectionUtils.copy(job.getConfiguration(),
+                                 reader.getCurrentKey(), null));
+            ++kept;
+          }
+        }
+        reader.close();
+      }
+      return (K[])samples.toArray();
+    }
+  }
+
+  /**
+   * Write a partition file for the given job, using the Sampler provided.
+   * Queries the sampler for a sample keyset, sorts by the output key
+   * comparator, selects the keys for each rank, and writes to the destination
+   * returned from {@link TotalOrderPartitioner#getPartitionFile}.
+   */
+  @SuppressWarnings("unchecked") // getInputFormat, getOutputKeyComparator
+  public static  void writePartitionFile(Job job, Sampler sampler) 
+      throws IOException, ClassNotFoundException, InterruptedException {
+    Configuration conf = job.getConfiguration();
+    final InputFormat inf = 
+        ReflectionUtils.newInstance(job.getInputFormatClass(), conf);
+    int numPartitions = job.getNumReduceTasks();
+    K[] samples = sampler.getSample(inf, job);
+    LOG.info("Using " + samples.length + " samples");
+    RawComparator comparator =
+      (RawComparator) job.getSortComparator();
+    Arrays.sort(samples, comparator);
+    Path dst = new Path(TotalOrderPartitioner.getPartitionFile(conf));
+    FileSystem fs = dst.getFileSystem(conf);
+    if (fs.exists(dst)) {
+      fs.delete(dst, false);
+    }
+    SequenceFile.Writer writer = SequenceFile.createWriter(fs, 
+      conf, dst, job.getMapOutputKeyClass(), NullWritable.class);
+    NullWritable nullValue = NullWritable.get();
+    float stepSize = samples.length / (float) numPartitions;
+    int last = -1;
+    for(int i = 1; i < numPartitions; ++i) {
+      int k = Math.round(stepSize * i);
+      while (last >= k && comparator.compare(samples[last], samples[k]) == 0) {
+        ++k;
+      }
+      writer.append(samples[k], nullValue);
+      last = k;
+    }
+    writer.close();
+  }
+
+  /**
+   * Driver for InputSampler from the command line.
+   * Configures a JobConf instance and calls {@link #writePartitionFile}.
+   */
+  public int run(String[] args) throws Exception {
+    Job job = new Job(getConf());
+    ArrayList otherArgs = new ArrayList();
+    Sampler sampler = null;
+    for(int i=0; i < args.length; ++i) {
+      try {
+        if ("-r".equals(args[i])) {
+          job.setNumReduceTasks(Integer.parseInt(args[++i]));
+        } else if ("-inFormat".equals(args[i])) {
+          job.setInputFormatClass(
+              Class.forName(args[++i]).asSubclass(InputFormat.class));
+        } else if ("-keyClass".equals(args[i])) {
+          job.setMapOutputKeyClass(
+              Class.forName(args[++i]).asSubclass(WritableComparable.class));
+        } else if ("-splitSample".equals(args[i])) {
+          int numSamples = Integer.parseInt(args[++i]);
+          int maxSplits = Integer.parseInt(args[++i]);
+          if (0 >= maxSplits) maxSplits = Integer.MAX_VALUE;
+          sampler = new SplitSampler(numSamples, maxSplits);
+        } else if ("-splitRandom".equals(args[i])) {
+          double pcnt = Double.parseDouble(args[++i]);
+          int numSamples = Integer.parseInt(args[++i]);
+          int maxSplits = Integer.parseInt(args[++i]);
+          if (0 >= maxSplits) maxSplits = Integer.MAX_VALUE;
+          sampler = new RandomSampler(pcnt, numSamples, maxSplits);
+        } else if ("-splitInterval".equals(args[i])) {
+          double pcnt = Double.parseDouble(args[++i]);
+          int maxSplits = Integer.parseInt(args[++i]);
+          if (0 >= maxSplits) maxSplits = Integer.MAX_VALUE;
+          sampler = new IntervalSampler(pcnt, maxSplits);
+        } else {
+          otherArgs.add(args[i]);
+        }
+      } catch (NumberFormatException except) {
+        System.out.println("ERROR: Integer expected instead of " + args[i]);
+        return printUsage();
+      } catch (ArrayIndexOutOfBoundsException except) {
+        System.out.println("ERROR: Required parameter missing from " +
+            args[i-1]);
+        return printUsage();
+      }
+    }
+    if (job.getNumReduceTasks() <= 1) {
+      System.err.println("Sampler requires more than one reducer");
+      return printUsage();
+    }
+    if (otherArgs.size() < 2) {
+      System.out.println("ERROR: Wrong number of parameters: ");
+      return printUsage();
+    }
+    if (null == sampler) {
+      sampler = new RandomSampler(0.1, 10000, 10);
+    }
+
+    Path outf = new Path(otherArgs.remove(otherArgs.size() - 1));
+    TotalOrderPartitioner.setPartitionFile(getConf(), outf);
+    for (String s : otherArgs) {
+      FileInputFormat.addInputPath(job, new Path(s));
+    }
+    InputSampler.writePartitionFile(job, sampler);
+
+    return 0;
+  }
+
+  public static void main(String[] args) throws Exception {
+    InputSampler sampler = new InputSampler(new Configuration());
+    int res = ToolRunner.run(sampler, args);
+    System.exit(res);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java
new file mode 100644
index 0000000..14e7949
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.hbase.mapreduce.hadoopbackport;
+
+import com.google.common.base.Preconditions;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.text.MessageFormat;
+import java.util.Enumeration;
+import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * Finds the Jar for a class. If the class is in a directory in the
+ * classpath, it creates a Jar on the fly with the contents of the directory
+ * and returns the path to that Jar. If a Jar is created, it is created in
+ * the system temporary directory.
+ * 
+ * This file was forked from hadoop/common/branches/branch-2@1377176.
+ */
+public class JarFinder {
+
+  private static void copyToZipStream(InputStream is, ZipEntry entry,
+                              ZipOutputStream zos) throws IOException {
+    zos.putNextEntry(entry);
+    byte[] arr = new byte[4096];
+    int read = is.read(arr);
+    while (read > -1) {
+      zos.write(arr, 0, read);
+      read = is.read(arr);
+    }
+    is.close();
+    zos.closeEntry();
+  }
+
+  public static void jarDir(File dir, String relativePath, ZipOutputStream zos)
+    throws IOException {
+    Preconditions.checkNotNull(relativePath, "relativePath");
+    Preconditions.checkNotNull(zos, "zos");
+
+    // by JAR spec, if there is a manifest, it must be the first entry in the
+    // ZIP.
+    File manifestFile = new File(dir, JarFile.MANIFEST_NAME);
+    ZipEntry manifestEntry = new ZipEntry(JarFile.MANIFEST_NAME);
+    if (!manifestFile.exists()) {
+      zos.putNextEntry(manifestEntry);
+      new Manifest().write(new BufferedOutputStream(zos));
+      zos.closeEntry();
+    } else {
+      InputStream is = new FileInputStream(manifestFile);
+      copyToZipStream(is, manifestEntry, zos);
+    }
+    zos.closeEntry();
+    zipDir(dir, relativePath, zos, true);
+    zos.close();
+  }
+
+  private static void zipDir(File dir, String relativePath, ZipOutputStream zos,
+                             boolean start) throws IOException {
+    String[] dirList = dir.list();
+    for (String aDirList : dirList) {
+      File f = new File(dir, aDirList);
+      if (!f.isHidden()) {
+        if (f.isDirectory()) {
+          if (!start) {
+            ZipEntry dirEntry = new ZipEntry(relativePath + f.getName() + "/");
+            zos.putNextEntry(dirEntry);
+            zos.closeEntry();
+          }
+          String filePath = f.getPath();
+          File file = new File(filePath);
+          zipDir(file, relativePath + f.getName() + "/", zos, false);
+        }
+        else {
+          String path = relativePath + f.getName();
+          if (!path.equals(JarFile.MANIFEST_NAME)) {
+            ZipEntry anEntry = new ZipEntry(path);
+            InputStream is = new FileInputStream(f);
+            copyToZipStream(is, anEntry, zos);
+          }
+        }
+      }
+    }
+  }
+
+  private static void createJar(File dir, File jarFile) throws IOException {
+    Preconditions.checkNotNull(dir, "dir");
+    Preconditions.checkNotNull(jarFile, "jarFile");
+    File jarDir = jarFile.getParentFile();
+    if (!jarDir.exists()) {
+      if (!jarDir.mkdirs()) {
+        throw new IOException(MessageFormat.format("could not create dir [{0}]",
+                                                   jarDir));
+      }
+    }
+    JarOutputStream zos = new JarOutputStream(new FileOutputStream(jarFile));
+    jarDir(dir, "", zos);
+  }
+
+  /**
+   * Returns the full path to the Jar containing the class. It always return a
+   * JAR.
+   *
+   * @param klass class.
+   *
+   * @return path to the Jar containing the class.
+   */
+  public static String getJar(Class klass) {
+    Preconditions.checkNotNull(klass, "klass");
+    ClassLoader loader = klass.getClassLoader();
+    if (loader != null) {
+      String class_file = klass.getName().replaceAll("\\.", "/") + ".class";
+      try {
+        for (Enumeration itr = loader.getResources(class_file);
+             itr.hasMoreElements(); ) {
+          URL url = (URL) itr.nextElement();
+          String path = url.getPath();
+          if (path.startsWith("file:")) {
+            path = path.substring("file:".length());
+          }
+          path = URLDecoder.decode(path, "UTF-8");
+          if ("jar".equals(url.getProtocol())) {
+            path = URLDecoder.decode(path, "UTF-8");
+            return path.replaceAll("!.*$", "");
+          }
+          else if ("file".equals(url.getProtocol())) {
+            String klassName = klass.getName();
+            klassName = klassName.replace(".", "/") + ".class";
+            path = path.substring(0, path.length() - klassName.length());
+            File baseDir = new File(path);
+            File testDir = new File(System.getProperty("test.build.dir", "target/test-dir"));
+            testDir = testDir.getAbsoluteFile();
+            if (!testDir.exists()) {
+              testDir.mkdirs();
+            }
+            File tempJar = File.createTempFile("hadoop-", "", testDir);
+            tempJar = new File(tempJar.getAbsolutePath() + ".jar");
+            createJar(baseDir, tempJar);
+            return tempJar.getAbsolutePath();
+          }
+        }
+      }
+      catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    return null;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TotalOrderPartitioner.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TotalOrderPartitioner.java
new file mode 100644
index 0000000..065e844
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TotalOrderPartitioner.java
@@ -0,0 +1,401 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce.hadoopbackport;
+
+import java.io.IOException;
+import java.lang.reflect.Array;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BinaryComparable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Partitioner effecting a total order by reading split points from
+ * an externally generated source.
+ * 
+ * This is an identical copy of o.a.h.mapreduce.lib.partition.TotalOrderPartitioner
+ * from Hadoop trunk at r910774.
+ */
+public class TotalOrderPartitioner,V>
+    extends Partitioner implements Configurable {
+
+  private Node partitions;
+  public static final String DEFAULT_PATH = "_partition.lst";
+  public static final String PARTITIONER_PATH = 
+    "mapreduce.totalorderpartitioner.path";
+  public static final String MAX_TRIE_DEPTH = 
+    "mapreduce.totalorderpartitioner.trie.maxdepth"; 
+  public static final String NATURAL_ORDER = 
+    "mapreduce.totalorderpartitioner.naturalorder";
+  Configuration conf;
+
+  public TotalOrderPartitioner() { }
+
+  /**
+   * Read in the partition file and build indexing data structures.
+   * If the keytype is {@link org.apache.hadoop.io.BinaryComparable} and
+   * total.order.partitioner.natural.order is not false, a trie
+   * of the first total.order.partitioner.max.trie.depth(2) + 1 bytes
+   * will be built. Otherwise, keys will be located using a binary search of
+   * the partition keyset using the {@link org.apache.hadoop.io.RawComparator}
+   * defined for this job. The input file must be sorted with the same
+   * comparator and contain {@link Job#getNumReduceTasks()} - 1 keys.
+   */
+  @SuppressWarnings("unchecked") // keytype from conf not static
+  public void setConf(Configuration conf) {
+    try {
+      this.conf = conf;
+      String parts = getPartitionFile(conf);
+      final Path partFile = new Path(parts);
+      final FileSystem fs = (DEFAULT_PATH.equals(parts))
+        ? FileSystem.getLocal(conf)     // assume in DistributedCache
+        : partFile.getFileSystem(conf);
+
+      Job job = new Job(conf);
+      Class keyClass = (Class)job.getMapOutputKeyClass();
+      K[] splitPoints = readPartitions(fs, partFile, keyClass, conf);
+      if (splitPoints.length != job.getNumReduceTasks() - 1) {
+        throw new IOException("Wrong number of partitions in keyset:"
+            + splitPoints.length);
+      }
+      RawComparator comparator =
+        (RawComparator) job.getSortComparator();
+      for (int i = 0; i < splitPoints.length - 1; ++i) {
+        if (comparator.compare(splitPoints[i], splitPoints[i+1]) >= 0) {
+          throw new IOException("Split points are out of order");
+        }
+      }
+      boolean natOrder =
+        conf.getBoolean(NATURAL_ORDER, true);
+      if (natOrder && BinaryComparable.class.isAssignableFrom(keyClass)) {
+        partitions = buildTrie((BinaryComparable[])splitPoints, 0,
+            splitPoints.length, new byte[0],
+            // Now that blocks of identical splitless trie nodes are 
+            // represented reentrantly, and we develop a leaf for any trie
+            // node with only one split point, the only reason for a depth
+            // limit is to refute stack overflow or bloat in the pathological
+            // case where the split points are long and mostly look like bytes 
+            // iii...iixii...iii   .  Therefore, we make the default depth
+            // limit large but not huge.
+            conf.getInt(MAX_TRIE_DEPTH, 200));
+      } else {
+        partitions = new BinarySearchNode(splitPoints, comparator);
+      }
+    } catch (IOException e) {
+      throw new IllegalArgumentException("Can't read partitions file", e);
+    }
+  }
+
+  public Configuration getConf() {
+    return conf;
+  }
+  
+  // by construction, we know if our keytype
+  @SuppressWarnings("unchecked") // is memcmp-able and uses the trie
+  public int getPartition(K key, V value, int numPartitions) {
+    return partitions.findPartition(key);
+  }
+
+  /**
+   * Set the path to the SequenceFile storing the sorted partition keyset.
+   * It must be the case that for R reduces, there are R-1
+   * keys in the SequenceFile.
+   */
+  public static void setPartitionFile(Configuration conf, Path p) {
+    conf.set(PARTITIONER_PATH, p.toString());
+  }
+
+  /**
+   * Get the path to the SequenceFile storing the sorted partition keyset.
+   * @see #setPartitionFile(Configuration, Path)
+   */
+  public static String getPartitionFile(Configuration conf) {
+    return conf.get(PARTITIONER_PATH, DEFAULT_PATH);
+  }
+
+  /**
+   * Interface to the partitioner to locate a key in the partition keyset.
+   */
+  interface Node {
+    /**
+     * Locate partition in keyset K, st [Ki..Ki+1) defines a partition,
+     * with implicit K0 = -inf, Kn = +inf, and |K| = #partitions - 1.
+     */
+    int findPartition(T key);
+  }
+
+  /**
+   * Base class for trie nodes. If the keytype is memcomp-able, this builds
+   * tries of the first total.order.partitioner.max.trie.depth
+   * bytes.
+   */
+  static abstract class TrieNode implements Node {
+    private final int level;
+    TrieNode(int level) {
+      this.level = level;
+    }
+    int getLevel() {
+      return level;
+    }
+  }
+
+  /**
+   * For types that are not {@link org.apache.hadoop.io.BinaryComparable} or
+   * where disabled by total.order.partitioner.natural.order,
+   * search the partition keyset with a binary search.
+   */
+  class BinarySearchNode implements Node {
+    private final K[] splitPoints;
+    private final RawComparator comparator;
+    BinarySearchNode(K[] splitPoints, RawComparator comparator) {
+      this.splitPoints = splitPoints;
+      this.comparator = comparator;
+    }
+    public int findPartition(K key) {
+      final int pos = Arrays.binarySearch(splitPoints, key, comparator) + 1;
+      return (pos < 0) ? -pos : pos;
+    }
+  }
+
+  /**
+   * An inner trie node that contains 256 children based on the next
+   * character.
+   */
+  class InnerTrieNode extends TrieNode {
+    private TrieNode[] child = new TrieNode[256];
+
+    InnerTrieNode(int level) {
+      super(level);
+    }
+    public int findPartition(BinaryComparable key) {
+      int level = getLevel();
+      if (key.getLength() <= level) {
+        return child[0].findPartition(key);
+      }
+      return child[0xFF & key.getBytes()[level]].findPartition(key);
+    }
+  }
+  
+  /**
+   * @param level        the tree depth at this node
+   * @param splitPoints  the full split point vector, which holds
+   *                     the split point or points this leaf node
+   *                     should contain
+   * @param lower        first INcluded element of splitPoints
+   * @param upper        first EXcluded element of splitPoints
+   * @return  a leaf node.  They come in three kinds: no split points 
+   *          [and the findParttion returns a canned index], one split
+   *          point [and we compare with a single comparand], or more
+   *          than one [and we do a binary search].  The last case is
+   *          rare.
+   */
+  private TrieNode LeafTrieNodeFactory
+             (int level, BinaryComparable[] splitPoints, int lower, int upper) {
+      switch (upper - lower) {
+      case 0:
+          return new UnsplitTrieNode(level, lower);
+          
+      case 1:
+          return new SinglySplitTrieNode(level, splitPoints, lower);
+          
+      default:
+          return new LeafTrieNode(level, splitPoints, lower, upper);
+      }
+  }
+
+  /**
+   * A leaf trie node that scans for the key between lower..upper.
+   * 
+   * We don't generate many of these now, since we usually continue trie-ing 
+   * when more than one split point remains at this level. and we make different
+   * objects for nodes with 0 or 1 split point.
+   */
+  private class LeafTrieNode extends TrieNode {
+    final int lower;
+    final int upper;
+    final BinaryComparable[] splitPoints;
+    LeafTrieNode(int level, BinaryComparable[] splitPoints, int lower, int upper) {
+      super(level);
+      this.lower = lower;
+      this.upper = upper;
+      this.splitPoints = splitPoints;
+    }
+    public int findPartition(BinaryComparable key) {
+      final int pos = Arrays.binarySearch(splitPoints, lower, upper, key) + 1;
+      return (pos < 0) ? -pos : pos;
+    }
+  }
+  
+  private class UnsplitTrieNode extends TrieNode {
+      final int result;
+      
+      UnsplitTrieNode(int level, int value) {
+          super(level);
+          this.result = value;
+      }
+      
+      public int findPartition(BinaryComparable key) {
+          return result;
+      }
+  }
+  
+  private class SinglySplitTrieNode extends TrieNode {
+      final int               lower;
+      final BinaryComparable  mySplitPoint;
+      
+      SinglySplitTrieNode(int level, BinaryComparable[] splitPoints, int lower) {
+          super(level);
+          this.lower = lower;
+          this.mySplitPoint = splitPoints[lower];
+      }
+      
+      public int findPartition(BinaryComparable key) {
+          return lower + (key.compareTo(mySplitPoint) < 0 ? 0 : 1);
+      }
+  }
+
+
+  /**
+   * Read the cut points from the given IFile.
+   * @param fs The file system
+   * @param p The path to read
+   * @param keyClass The map output key class
+   * @param job The job config
+   * @throws IOException
+   */
+                                 // matching key types enforced by passing in
+  @SuppressWarnings("unchecked") // map output key class
+  private K[] readPartitions(FileSystem fs, Path p, Class keyClass,
+      Configuration conf) throws IOException {
+    SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf);
+    ArrayList parts = new ArrayList();
+    K key = ReflectionUtils.newInstance(keyClass, conf);
+    NullWritable value = NullWritable.get();
+    while (reader.next(key, value)) {
+      parts.add(key);
+      key = ReflectionUtils.newInstance(keyClass, conf);
+    }
+    reader.close();
+    return parts.toArray((K[])Array.newInstance(keyClass, parts.size()));
+  }
+  
+  /**
+   * 
+   * This object contains a TrieNodeRef if there is such a thing that
+   * can be repeated.  Two adjacent trie node slots that contain no 
+   * split points can be filled with the same trie node, even if they
+   * are not on the same level.  See buildTreeRec, below.
+   *
+   */  
+  private class CarriedTrieNodeRef
+  {
+      TrieNode   content;
+      
+      CarriedTrieNodeRef() {
+          content = null;
+      }
+  }
+
+  
+  /**
+   * Given a sorted set of cut points, build a trie that will find the correct
+   * partition quickly.
+   * @param splits the list of cut points
+   * @param lower the lower bound of partitions 0..numPartitions-1
+   * @param upper the upper bound of partitions 0..numPartitions-1
+   * @param prefix the prefix that we have already checked against
+   * @param maxDepth the maximum depth we will build a trie for
+   * @return the trie node that will divide the splits correctly
+   */
+  private TrieNode buildTrie(BinaryComparable[] splits, int lower,
+          int upper, byte[] prefix, int maxDepth) {
+      return buildTrieRec
+               (splits, lower, upper, prefix, maxDepth, new CarriedTrieNodeRef());
+  }
+  
+  /**
+   * This is the core of buildTrie.  The interface, and stub, above, just adds
+   * an empty CarriedTrieNodeRef.  
+   * 
+   * We build trie nodes in depth first order, which is also in key space
+   * order.  Every leaf node is referenced as a slot in a parent internal
+   * node.  If two adjacent slots [in the DFO] hold leaf nodes that have
+   * no split point, then they are not separated by a split point either, 
+   * because there's no place in key space for that split point to exist.
+   * 
+   * When that happens, the leaf nodes would be semantically identical, and
+   * we reuse the object.  A single CarriedTrieNodeRef "ref" lives for the 
+   * duration of the tree-walk.  ref carries a potentially reusable, unsplit
+   * leaf node for such reuse until a leaf node with a split arises, which 
+   * breaks the chain until we need to make a new unsplit leaf node.
+   * 
+   * Note that this use of CarriedTrieNodeRef means that for internal nodes, 
+   * for internal nodes if this code is modified in any way we still need 
+   * to make or fill in the subnodes in key space order.
+   */
+  private TrieNode buildTrieRec(BinaryComparable[] splits, int lower,
+      int upper, byte[] prefix, int maxDepth, CarriedTrieNodeRef ref) {
+    final int depth = prefix.length;
+    // We generate leaves for a single split point as well as for 
+    // no split points.
+    if (depth >= maxDepth || lower >= upper - 1) {
+        // If we have two consecutive requests for an unsplit trie node, we
+        // can deliver the same one the second time.
+        if (lower == upper && ref.content != null) {
+            return ref.content;
+        }
+        TrieNode  result = LeafTrieNodeFactory(depth, splits, lower, upper);
+        ref.content = lower == upper ? result : null;
+        return result;
+    }
+    InnerTrieNode result = new InnerTrieNode(depth);
+    byte[] trial = Arrays.copyOf(prefix, prefix.length + 1);
+    // append an extra byte on to the prefix
+    int         currentBound = lower;
+    for(int ch = 0; ch < 0xFF; ++ch) {
+      trial[depth] = (byte) (ch + 1);
+      lower = currentBound;
+      while (currentBound < upper) {
+        if (splits[currentBound].compareTo(trial, 0, trial.length) >= 0) {
+          break;
+        }
+        currentBound += 1;
+      }
+      trial[depth] = (byte) ch;
+      result.child[0xFF & ch]
+                   = buildTrieRec(splits, lower, currentBound, trial, maxDepth, ref);
+    }
+    // pick up the rest
+    trial[depth] = (byte)0xFF;
+    result.child[0xFF] 
+                 = buildTrieRec(splits, lower, currentBound, trial, maxDepth, ref);
+    
+    return result;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java
new file mode 100644
index 0000000..62d4296
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+Provides HBase MapReduce
+Input/OutputFormats, a table indexing MapReduce job, and utility
+
+
        Table of Contents
        
+
+
+
        HBase, MapReduce and the CLASSPATH
        
+
+
        MapReduce jobs deployed to a MapReduce cluster do not by default have access
+to the HBase configuration under $HBASE_CONF_DIR nor to HBase classes.
+You could add hbase-site.xml to
+$HADOOP_HOME/conf and add
+HBase jars to the $HADOOP_HOME/lib and copy these
+changes across your cluster (or edit conf/hadoop-env.sh and add them to the
+HADOOP_CLASSPATH variable) but this will pollute your
+hadoop install with HBase references; its also obnoxious requiring restart of
+the hadoop cluster before it'll notice your HBase additions.
        
+
+
        As of 0.90.x, HBase will just add its dependency jars to the job
+configuration; the dependencies just need to be available on the local
+CLASSPATH.  For example, to run the bundled HBase
+{@link org.apache.hadoop.hbase.mapreduce.RowCounter} mapreduce job against a table named usertable,
+type:
+
+
        +$ HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/hbase-0.90.0.jar rowcounter usertable
+
        
+
+Expand $HBASE_HOME and $HADOOP_HOME in the above
+appropriately to suit your local environment.  The content of HADOOP_CLASSPATH
+is set to the HBase CLASSPATH via backticking the command
+${HBASE_HOME}/bin/hbase classpath.
+
+
        When the above runs, internally, the HBase jar finds its zookeeper and
+guava,
+etc., dependencies on the passed
+HADOOP_CLASSPATH and adds the found jars to the mapreduce
+job configuration. See the source at
+TableMapReduceUtil#addDependencyJars(org.apache.hadoop.mapreduce.Job)
+for how this is done.
+
        
+
        The above may not work if you are running your HBase from its build directory;
+i.e. you've done $ mvn test install at
+${HBASE_HOME} and you are now
+trying to use this build in your mapreduce job.  If you get
+
        java.lang.RuntimeException: java.lang.ClassNotFoundException: org.apache.hadoop.hbase.mapreduce.RowCounter$RowCounterMapper
+...
+
        
+exception thrown, try doing the following:
+
        +$ HADOOP_CLASSPATH=${HBASE_HOME}/target/hbase-0.90.0-SNAPSHOT.jar:`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/target/hbase-0.90.0-SNAPSHOT.jar rowcounter usertable
+
        
+Notice how we preface the backtick invocation setting
+HADOOP_CLASSPATH with reference to the built HBase jar over in
+the target directory.
+
        
+
+
        Bundled HBase MapReduce Jobs
        
+
        The HBase jar also serves as a Driver for some bundled mapreduce jobs. To
+learn about the bundled mapreduce jobs run:
+
        +$ ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/hbase-0.90.0-SNAPSHOT.jar
+An example program must be given as the first argument.
+Valid program names are:
+  copytable: Export a table from local cluster to peer cluster
+  completebulkload: Complete a bulk data load.
+  export: Write table data to HDFS.
+  import: Import data written by Export.
+  importtsv: Import data in TSV format.
+  rowcounter: Count rows in HBase table
+
        
+
+
        HBase as MapReduce job data source and sink
        
+
+
        HBase can be used as a data source, {@link org.apache.hadoop.hbase.mapreduce.TableInputFormat TableInputFormat},
+and data sink, {@link org.apache.hadoop.hbase.mapreduce.TableOutputFormat TableOutputFormat}
+or {@link org.apache.hadoop.hbase.mapreduce.MultiTableOutputFormat MultiTableOutputFormat},
+for MapReduce jobs.
+Writing MapReduce jobs that read or write HBase, you'll probably want to subclass
+{@link org.apache.hadoop.hbase.mapreduce.TableMapper TableMapper} and/or
+{@link org.apache.hadoop.hbase.mapreduce.TableReducer TableReducer}.  See the do-nothing
+pass-through classes {@link org.apache.hadoop.hbase.mapreduce.IdentityTableMapper IdentityTableMapper} and
+{@link org.apache.hadoop.hbase.mapreduce.IdentityTableReducer IdentityTableReducer} for basic usage.  For a more
+involved example, see {@link org.apache.hadoop.hbase.mapreduce.RowCounter}
+or review the org.apache.hadoop.hbase.mapreduce.TestTableMapReduce unit test.
+
        
+
+
        Running mapreduce jobs that have HBase as source or sink, you'll need to
+specify source/sink table and column names in your configuration.
        
+
+
        Reading from HBase, the TableInputFormat asks HBase for the list of
+regions and makes a map-per-region or mapred.map.tasks maps,
+whichever is smaller (If your job only has two maps, up mapred.map.tasks
+to a number > number of regions). Maps will run on the adjacent TaskTracker
+if you are running a TaskTracer and RegionServer per node.
+Writing, it may make sense to avoid the reduce step and write yourself back into
+HBase from inside your map. You'd do this when your job does not need the sort
+and collation that mapreduce does on the map emitted data; on insert,
+HBase 'sorts' so there is no point double-sorting (and shuffling data around
+your mapreduce cluster) unless you need to. If you do not need the reduce,
+you might just have your map emit counts of records processed just so the
+framework's report at the end of your job has meaning or set the number of
+reduces to zero and use TableOutputFormat. See example code
+below. If running the reduce step makes sense in your case, its usually better
+to have lots of reducers so load is spread across the HBase cluster.
        
+
+
        There is also a new HBase partitioner that will run as many reducers as
+currently existing regions.  The
+{@link org.apache.hadoop.hbase.mapreduce.HRegionPartitioner} is suitable
+when your table is large and your upload is not such that it will greatly
+alter the number of existing regions when done; otherwise use the default
+partitioner.
+
        
+
+
        Bulk import writing HFiles directly
        
+
        If importing into a new table, its possible to by-pass the HBase API
+and write your content directly to the filesystem properly formatted as
+HBase data files (HFiles).  Your import will run faster, perhaps an order of
+magnitude faster if not more.  For more on how this mechanism works, see
+Bulk Loads
+documentation.
+
        
+
+
        Example Code
        
+
        Sample Row Counter
        
+
        See {@link org.apache.hadoop.hbase.mapreduce.RowCounter}.  This job uses
+{@link org.apache.hadoop.hbase.mapreduce.TableInputFormat TableInputFormat} and
+does a count of all rows in specified table.
+You should be able to run
+it by doing: % ./bin/hadoop jar hbase-X.X.X.jar.  This will invoke
+the hbase MapReduce Driver class.  Select 'rowcounter' from the choice of jobs
+offered. This will emit rowcouner 'usage'.  Specify tablename, column to count
+and output directory.  You may need to add the hbase conf directory to $HADOOP_HOME/conf/hadoop-env.sh#HADOOP_CLASSPATH
+so the rowcounter gets pointed at the right hbase cluster (or, build a new jar
+with an appropriate hbase-site.xml built into your job jar).
+
        
+*/
+package org.apache.hadoop.hbase.mapreduce;
diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
new file mode 100644
index 0000000..b7d540b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -0,0 +1,296 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce.replication;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
+import org.apache.hadoop.hbase.mapreduce.TableMapper;
+import org.apache.hadoop.hbase.replication.ReplicationPeer;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * This map-only job compares the data from a local table with a remote one.
+ * Every cell is compared and must have exactly the same keys (even timestamp)
+ * as well as same value. It is possible to restrict the job by time range and
+ * families. The peer id that's provided must match the one given when the
+ * replication stream was setup.
+ * 
        
+ * Two counters are provided, Verifier.Counters.GOODROWS and BADROWS. The reason
+ * for a why a row is different is shown in the map's log.
+ */
+public class VerifyReplication {
+
+  private static final Log LOG =
+      LogFactory.getLog(VerifyReplication.class);
+
+  public final static String NAME = "verifyrep";
+  static long startTime = 0;
+  static long endTime = 0;
+  static String tableName = null;
+  static String families = null;
+  static String peerId = null;
+
+  /**
+   * Map-only comparator for 2 tables
+   */
+  public static class Verifier
+      extends TableMapper {
+
+    public static enum Counters {GOODROWS, BADROWS}
+
+    private ResultScanner replicatedScanner;
+
+    /**
+     * Map method that compares every scanned row with the equivalent from
+     * a distant cluster.
+     * @param row  The current table row key.
+     * @param value  The columns.
+     * @param context  The current context.
+     * @throws IOException When something is broken with the data.
+     */
+    @Override
+    public void map(ImmutableBytesWritable row, final Result value,
+                    Context context)
+        throws IOException {
+      if (replicatedScanner == null) {
+        Configuration conf = context.getConfiguration();
+        final Scan scan = new Scan();
+        scan.setCaching(conf.getInt(TableInputFormat.SCAN_CACHEDROWS, 1));
+        long startTime = conf.getLong(NAME + ".startTime", 0);
+        long endTime = conf.getLong(NAME + ".endTime", 0);
+        String families = conf.get(NAME + ".families", null);
+        if(families != null) {
+          String[] fams = families.split(",");
+          for(String fam : fams) {
+            scan.addFamily(Bytes.toBytes(fam));
+          }
+        }
+        if (startTime != 0) {
+          scan.setTimeRange(startTime,
+              endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
+        }
+        HConnectionManager.execute(new HConnectable(conf) {
+          @Override
+          public Void connect(HConnection conn) throws IOException {
+            try {
+              ReplicationZookeeper zk = new ReplicationZookeeper(conn, conf,
+                  conn.getZooKeeperWatcher());
+              ReplicationPeer peer = zk.getPeer(conf.get(NAME+".peerId"));
+              HTable replicatedTable = new HTable(peer.getConfiguration(),
+                  conf.get(NAME+".tableName"));
+              scan.setStartRow(value.getRow());
+              replicatedScanner = replicatedTable.getScanner(scan);
+            } catch (KeeperException e) {
+              throw new IOException("Got a ZK exception", e);
+            }
+            return null;
+          }
+        });
+      }
+      Result res = replicatedScanner.next();
+      try {
+        Result.compareResults(value, res);
+        context.getCounter(Counters.GOODROWS).increment(1);
+      } catch (Exception e) {
+        LOG.warn("Bad row", e);
+        context.getCounter(Counters.BADROWS).increment(1);
+      }
+    }
+
+    protected void cleanup(Context context) {
+      if (replicatedScanner != null) {
+        replicatedScanner.close();
+        replicatedScanner = null;
+      }
+    }
+  }
+
+  /**
+   * Sets up the actual job.
+   *
+   * @param conf  The current configuration.
+   * @param args  The command line parameters.
+   * @return The newly created job.
+   * @throws java.io.IOException When setting up the job fails.
+   */
+  public static Job createSubmittableJob(Configuration conf, String[] args)
+  throws IOException {
+    if (!doCommandLine(args)) {
+      return null;
+    }
+    if (!conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false)) {
+      throw new IOException("Replication needs to be enabled to verify it.");
+    }
+    HConnectionManager.execute(new HConnectable(conf) {
+      @Override
+      public Void connect(HConnection conn) throws IOException {
+        try {
+          ReplicationZookeeper zk = new ReplicationZookeeper(conn, conf,
+              conn.getZooKeeperWatcher());
+          // Just verifying it we can connect
+          ReplicationPeer peer = zk.getPeer(peerId);
+          if (peer == null) {
+            throw new IOException("Couldn't get access to the slave cluster," +
+                "please see the log");
+          }
+        } catch (KeeperException ex) {
+          throw new IOException("Couldn't get access to the slave cluster" +
+              " because: ", ex);
+        }
+        return null;
+      }
+    });
+    conf.set(NAME+".peerId", peerId);
+    conf.set(NAME+".tableName", tableName);
+    conf.setLong(NAME+".startTime", startTime);
+    conf.setLong(NAME+".endTime", endTime);
+    if (families != null) {
+      conf.set(NAME+".families", families);
+    }
+    Job job = new Job(conf, NAME + "_" + tableName);
+    job.setJarByClass(VerifyReplication.class);
+
+    Scan scan = new Scan();
+    if (startTime != 0) {
+      scan.setTimeRange(startTime,
+          endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
+    }
+    if(families != null) {
+      String[] fams = families.split(",");
+      for(String fam : fams) {
+        scan.addFamily(Bytes.toBytes(fam));
+      }
+    }
+    TableMapReduceUtil.initTableMapperJob(tableName, scan,
+        Verifier.class, null, null, job);
+    job.setOutputFormatClass(NullOutputFormat.class);
+    job.setNumReduceTasks(0);
+    return job;
+  }
+
+  private static boolean doCommandLine(final String[] args) {
+    if (args.length < 2) {
+      printUsage(null);
+      return false;
+    }
+    try {
+      for (int i = 0; i < args.length; i++) {
+        String cmd = args[i];
+        if (cmd.equals("-h") || cmd.startsWith("--h")) {
+          printUsage(null);
+          return false;
+        }
+
+        final String startTimeArgKey = "--starttime=";
+        if (cmd.startsWith(startTimeArgKey)) {
+          startTime = Long.parseLong(cmd.substring(startTimeArgKey.length()));
+          continue;
+        }
+
+        final String endTimeArgKey = "--endtime=";
+        if (cmd.startsWith(endTimeArgKey)) {
+          endTime = Long.parseLong(cmd.substring(endTimeArgKey.length()));
+          continue;
+        }
+
+        final String familiesArgKey = "--families=";
+        if (cmd.startsWith(familiesArgKey)) {
+          families = cmd.substring(familiesArgKey.length());
+          continue;
+        }
+
+        if (i == args.length-2) {
+          peerId = cmd;
+        }
+
+        if (i == args.length-1) {
+          tableName = cmd;
+        }
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      printUsage("Can't start because " + e.getMessage());
+      return false;
+    }
+    return true;
+  }
+
+  /*
+   * @param errorMsg Error message.  Can be null.
+   */
+  private static void printUsage(final String errorMsg) {
+    if (errorMsg != null && errorMsg.length() > 0) {
+      System.err.println("ERROR: " + errorMsg);
+    }
+    System.err.println("Usage: verifyrep [--starttime=X]" +
+        " [--stoptime=Y] [--families=A]  ");
+    System.err.println();
+    System.err.println("Options:");
+    System.err.println(" starttime    beginning of the time range");
+    System.err.println("              without endtime means from starttime to forever");
+    System.err.println(" stoptime     end of the time range");
+    System.err.println(" families     comma-separated list of families to copy");
+    System.err.println();
+    System.err.println("Args:");
+    System.err.println(" peerid       Id of the peer used for verification, must match the one given for replication");
+    System.err.println(" tablename    Name of the table to verify");
+    System.err.println();
+    System.err.println("Examples:");
+    System.err.println(" To verify the data replicated from TestTable for a 1 hour window with peer #5 ");
+    System.err.println(" $ bin/hbase " +
+        "org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication" +
+        " --starttime=1265875194289 --stoptime=1265878794289 5 TestTable ");
+  }
+
+  /**
+   * Main entry point.
+   *
+   * @param args  The command line parameters.
+   * @throws Exception When running the job fails.
+   */
+  public static void main(String[] args) throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    Job job = createSubmittableJob(conf, args);
+    if (job != null) {
+      System.exit(job.waitForCompletion(true) ? 0 : 1);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
new file mode 100644
index 0000000..c1c4294
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -0,0 +1,245 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
+
+/**
+ * Handles everything on master-side related to master election.
+ *
+ * 
        Listens and responds to ZooKeeper notifications on the master znode,
+ * both nodeCreated and nodeDeleted.
+ *
+ * 
        Contains blocking methods which will hold up backup masters, waiting
+ * for the active master to fail.
+ *
+ * 
        This class is instantiated in the HMaster constructor and the method
+ * #blockUntilBecomingActiveMaster() is called to wait until becoming
+ * the active master of the cluster.
+ */
+class ActiveMasterManager extends ZooKeeperListener {
+  private static final Log LOG = LogFactory.getLog(ActiveMasterManager.class);
+
+  final AtomicBoolean clusterHasActiveMaster = new AtomicBoolean(false);
+
+  private final ServerName sn;
+  private final Server master;
+
+  /**
+   * @param watcher
+   * @param sn ServerName
+   * @param master In an instance of a Master.
+   */
+  ActiveMasterManager(ZooKeeperWatcher watcher, ServerName sn, Server master) {
+    super(watcher);
+    this.sn = sn;
+    this.master = master;
+  }
+
+  @Override
+  public void nodeCreated(String path) {
+    if(path.equals(watcher.masterAddressZNode) && !master.isStopped()) {
+      handleMasterNodeChange();
+    }
+  }
+
+  @Override
+  public void nodeDeleted(String path) {
+    if(path.equals(watcher.masterAddressZNode) && !master.isStopped()) {
+      handleMasterNodeChange();
+    }
+  }
+
+  /**
+   * Handle a change in the master node.  Doesn't matter whether this was called
+   * from a nodeCreated or nodeDeleted event because there are no guarantees
+   * that the current state of the master node matches the event at the time of
+   * our next ZK request.
+   *
+   * 
        Uses the watchAndCheckExists method which watches the master address node
+   * regardless of whether it exists or not.  If it does exist (there is an
+   * active master), it returns true.  Otherwise it returns false.
+   *
+   * 
        A watcher is set which guarantees that this method will get called again if
+   * there is another change in the master node.
+   */
+  private void handleMasterNodeChange() {
+    // Watch the node and check if it exists.
+    try {
+      synchronized(clusterHasActiveMaster) {
+        if(ZKUtil.watchAndCheckExists(watcher, watcher.masterAddressZNode)) {
+          // A master node exists, there is an active master
+          LOG.debug("A master is now available");
+          clusterHasActiveMaster.set(true);
+        } else {
+          // Node is no longer there, cluster does not have an active master
+          LOG.debug("No master available. Notifying waiting threads");
+          clusterHasActiveMaster.set(false);
+          // Notify any thread waiting to become the active master
+          clusterHasActiveMaster.notifyAll();
+        }
+      }
+    } catch (KeeperException ke) {
+      master.abort("Received an unexpected KeeperException, aborting", ke);
+    }
+  }
+
+  /**
+   * Block until becoming the active master.
+   *
+   * Method blocks until there is not another active master and our attempt
+   * to become the new active master is successful.
+   *
+   * This also makes sure that we are watching the master znode so will be
+   * notified if another master dies.
+   * @param startupStatus
+   * @return True if no issue becoming active master else false if another
+   * master was running or if some other problem (zookeeper, stop flag has been
+   * set on this Master)
+   */
+  boolean blockUntilBecomingActiveMaster(MonitoredTask startupStatus,
+    ClusterStatusTracker clusterStatusTracker) {
+    while (true) {
+      startupStatus.setStatus("Trying to register in ZK as active master");
+      // Try to become the active master, watch if there is another master.
+      // Write out our ServerName as versioned bytes.
+      try {
+        String backupZNode = ZKUtil.joinZNode(
+          this.watcher.backupMasterAddressesZNode, this.sn.toString());
+        if (ZKUtil.createEphemeralNodeAndWatch(this.watcher,
+          this.watcher.masterAddressZNode, this.sn.getVersionedBytes())) {
+          // If we were a backup master before, delete our ZNode from the backup
+          // master directory since we are the active now
+          LOG.info("Deleting ZNode for " + backupZNode +
+            " from backup master directory");
+          ZKUtil.deleteNodeFailSilent(this.watcher, backupZNode);
+
+          // We are the master, return
+          startupStatus.setStatus("Successfully registered as active master.");
+          this.clusterHasActiveMaster.set(true);
+          LOG.info("Master=" + this.sn);
+          return true;
+        }
+
+        // There is another active master running elsewhere or this is a restart
+        // and the master ephemeral node has not expired yet.
+        this.clusterHasActiveMaster.set(true);
+
+        /*
+         * Add a ZNode for ourselves in the backup master directory since we are
+         * not the active master.
+         *
+         * If we become the active master later, ActiveMasterManager will delete
+         * this node explicitly.  If we crash before then, ZooKeeper will delete
+         * this node for us since it is ephemeral.
+         */
+        LOG.info("Adding ZNode for " + backupZNode +
+          " in backup master directory");
+        ZKUtil.createEphemeralNodeAndWatch(this.watcher, backupZNode,
+          this.sn.getVersionedBytes());
+
+        String msg;
+        byte [] bytes =
+          ZKUtil.getDataAndWatch(this.watcher, this.watcher.masterAddressZNode);
+        if (bytes == null) {
+          msg = ("A master was detected, but went down before its address " +
+            "could be read.  Attempting to become the next active master");
+        } else {
+          ServerName currentMaster = ServerName.parseVersionedServerName(bytes);
+          if (ServerName.isSameHostnameAndPort(currentMaster, this.sn)) {
+            msg = ("Current master has this master's address, " +
+              currentMaster + "; master was restarted? Deleting node.");
+            // Hurry along the expiration of the znode.
+            ZKUtil.deleteNode(this.watcher, this.watcher.masterAddressZNode);
+          } else {
+            msg = "Another master is the active master, " + currentMaster +
+              "; waiting to become the next active master";
+          }
+        }
+        LOG.info(msg);
+        startupStatus.setStatus(msg);
+      } catch (KeeperException ke) {
+        master.abort("Received an unexpected KeeperException, aborting", ke);
+        return false;
+      }
+      synchronized (this.clusterHasActiveMaster) {
+        while (this.clusterHasActiveMaster.get() && !this.master.isStopped()) {
+          try {
+            this.clusterHasActiveMaster.wait();
+          } catch (InterruptedException e) {
+            // We expect to be interrupted when a master dies, will fall out if so
+            LOG.debug("Interrupted waiting for master to die", e);
+          }
+        }
+        if (!clusterStatusTracker.isClusterUp()) {
+          this.master.stop("Cluster went down before this master became active");
+        }
+        if (this.master.isStopped()) {
+          return false;
+        }
+        // Try to become active master again now that there is no active master
+      }
+    }
+  }
+
+  /**
+   * @return True if cluster has an active master.
+   */
+  public boolean isActiveMaster() {
+    try {
+      if (ZKUtil.checkExists(watcher, watcher.masterAddressZNode) >= 0) {
+        return true;
+      }
+    } 
+    catch (KeeperException ke) {
+      LOG.info("Received an unexpected KeeperException when checking " +
+          "isActiveMaster : "+ ke);
+    }
+    return false;
+  }
+
+  public void stop() {
+    try {
+      // If our address is in ZK, delete it on our way out
+      byte [] bytes =
+        ZKUtil.getDataAndWatch(watcher, watcher.masterAddressZNode);
+      // TODO: redo this to make it atomic (only added for tests)
+      ServerName master = bytes == null ? null : ServerName.parseVersionedServerName(bytes);
+      if (master != null &&  master.equals(this.sn)) {
+        ZKUtil.deleteNode(watcher, watcher.masterAddressZNode);
+      }
+    } catch (KeeperException e) {
+      LOG.error(this.watcher.prefix("Error deleting our own master address node"), e);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignCallable.java b/src/main/java/org/apache/hadoop/hbase/master/AssignCallable.java
new file mode 100644
index 0000000..b233d10
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignCallable.java
@@ -0,0 +1,47 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.util.concurrent.Callable;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+
+/**
+ * A callable object that invokes the corresponding action that needs to be
+ * taken for assignment of a region in transition. 
+ * Implementing as future callable we are able to act on the timeout
+ * asynchronously.
+ */
+public class AssignCallable implements Callable {
+  private AssignmentManager assignmentManager;
+
+  private HRegionInfo hri;
+
+  public AssignCallable(AssignmentManager assignmentManager, HRegionInfo hri) {
+    this.assignmentManager = assignmentManager;
+    this.hri = hri;
+  }
+
+  @Override
+  public Object call() throws Exception {
+    assignmentManager.assign(hri, true, true, true);
+    return null;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
new file mode 100644
index 0000000..4de5683
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -0,0 +1,3737 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.Thread.UncaughtExceptionHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.catalog.RootLocationEditor;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State;
+import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler;
+import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
+import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
+import org.apache.hadoop.hbase.master.handler.OpenedRegionHandler;
+import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
+import org.apache.hadoop.hbase.master.handler.SplitRegionHandler;
+import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException;
+import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
+import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKTable;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.KeeperException.NodeExistsException;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ * Manages and performs region assignment.
+ * 
        
+ * Monitors ZooKeeper for events related to regions in transition.
+ * 
        
+ * Handles existing regions in transition during master failover.
+ */
+public class AssignmentManager extends ZooKeeperListener {
+
+  private static final Log LOG = LogFactory.getLog(AssignmentManager.class);
+
+  protected Server master;
+
+  private ServerManager serverManager;
+
+  private CatalogTracker catalogTracker;
+
+  private TimeoutMonitor timeoutMonitor;
+
+  private TimerUpdater timerUpdater;
+
+  private LoadBalancer balancer;
+
+  /**
+   * Map of regions to reopen after the schema of a table is changed. Key -
+   * encoded region name, value - HRegionInfo
+   */
+  private final Map  regionsToReopen;
+
+  /*
+   * Maximum times we recurse an assignment.  See below in {@link #assign()}.
+   */
+  private final int maximumAssignmentAttempts;
+
+  /**
+   * Regions currently in transition.  Map of encoded region names to the master
+   * in-memory state for that region.
+   */
+  final ConcurrentSkipListMap regionsInTransition =
+    new ConcurrentSkipListMap();
+
+  /** Plans for region movement. Key is the encoded version of a region name*/
+  // TODO: When do plans get cleaned out?  Ever? In server open and in server
+  // shutdown processing -- St.Ack
+  // All access to this Map must be synchronized.
+  final NavigableMap regionPlans =
+    new TreeMap();
+
+  private final ZKTable zkTable;
+
+  // store all the table names in disabling state
+  Set disablingTables = new HashSet(1);
+  // store all the enabling state table names and corresponding online servers' regions.
+  // This may be needed to avoid calling assign twice for the regions of the ENABLING table
+  // that could have been assigned through processRIT.
+  Map> enablingTables = new HashMap>(1);
+  /**
+   * Server to regions assignment map.
+   * Contains the set of regions currently assigned to a given server.
+   * This Map and {@link #regions} are tied.  Always update this in tandem
+   * with the other under a lock on {@link #regions}.
+   * @see #regions
+   */
+  private final NavigableMap> servers =
+    new TreeMap>();
+
+  /**
+   * Contains the server which need to update timer, these servers will be
+   * handled by {@link TimerUpdater}
+   */
+  private final ConcurrentSkipListSet serversInUpdatingTimer = 
+    new ConcurrentSkipListSet();
+
+  /**
+   * Region to server assignment map.
+   * Contains the server a given region is currently assigned to.
+   * This Map and {@link #servers} are tied.  Always update this in tandem
+   * with the other under a lock on {@link #regions}.
+   * @see #servers
+   */
+  private final SortedMap regions =
+    new TreeMap();
+
+  private final ExecutorService executorService;
+
+  //Thread pool executor service for timeout monitor
+  private java.util.concurrent.ExecutorService threadPoolExecutorService;
+  
+  private List ignoreStatesRSOffline = Arrays.asList(new EventType[]{
+      EventType.RS_ZK_REGION_FAILED_OPEN, EventType.RS_ZK_REGION_CLOSED });
+
+  /**
+   * Set when we are doing master failover processing; cleared when failover
+   * completes.
+   */
+  private volatile boolean failover = false;
+
+  // Set holding all the regions which got processed while RIT was not 
+  // populated during master failover. 
+  private Map failoverProcessedRegions =
+    new HashMap();
+
+  /**
+   * Constructs a new assignment manager.
+   *
+   * @param master
+   * @param serverManager
+   * @param catalogTracker
+   * @param service
+   * @throws KeeperException
+   * @throws IOException 
+   */
+  public AssignmentManager(Server master, ServerManager serverManager,
+      CatalogTracker catalogTracker, final LoadBalancer balancer,
+      final ExecutorService service) throws KeeperException, IOException {
+    super(master.getZooKeeper());
+    this.master = master;
+    this.serverManager = serverManager;
+    this.catalogTracker = catalogTracker;
+    this.executorService = service;
+    this.regionsToReopen = Collections.synchronizedMap
+                           (new HashMap ());
+    Configuration conf = master.getConfiguration();
+    this.timeoutMonitor = new TimeoutMonitor(
+      conf.getInt("hbase.master.assignment.timeoutmonitor.period", 10000),
+      master, serverManager,
+      conf.getInt("hbase.master.assignment.timeoutmonitor.timeout", 1800000));
+    this.timerUpdater = new TimerUpdater(conf.getInt(
+        "hbase.master.assignment.timerupdater.period", 10000), master);
+    Threads.setDaemonThreadRunning(timerUpdater.getThread(),
+        master.getServerName() + ".timerUpdater");
+    this.zkTable = new ZKTable(this.master.getZooKeeper());
+    this.maximumAssignmentAttempts =
+      this.master.getConfiguration().getInt("hbase.assignment.maximum.attempts", 10);
+    this.balancer = balancer;
+    this.threadPoolExecutorService = Executors.newCachedThreadPool();
+  }
+  
+  void startTimeOutMonitor() {
+    Threads.setDaemonThreadRunning(timeoutMonitor.getThread(), master.getServerName()
+        + ".timeoutMonitor");
+  }
+
+  /**
+   * Compute the average load across all region servers.
+   * Currently, this uses a very naive computation - just uses the number of
+   * regions being served, ignoring stats about number of requests.
+   * @return the average load
+   */
+  double getAverageLoad() {
+    int totalLoad = 0;
+    int numServers = 0;
+    // Sync on this.regions because access to this.servers always synchronizes
+    // in this order.
+    synchronized (this.regions) {
+      for (Map.Entry> e: servers.entrySet()) {
+        numServers++;
+        totalLoad += e.getValue().size();
+      }
+    }
+    return (double)totalLoad / (double)numServers;
+  }
+
+  /**
+   * @return Instance of ZKTable.
+   */
+  public ZKTable getZKTable() {
+    // These are 'expensive' to make involving trip to zk ensemble so allow
+    // sharing.
+    return this.zkTable;
+  }
+  /**
+   * Returns the RegionServer to which hri is assigned.
+   *
+   * @param hri
+   *          HRegion for which this function returns the region server
+   * @return HServerInfo The region server to which hri belongs
+   */
+  public ServerName getRegionServerOfRegion(HRegionInfo hri) {
+    synchronized (this.regions ) {
+      return regions.get(hri);
+    }
+  }
+
+  /**
+   * Checks whether the region is assigned.
+   * @param hri HRegion for which this function returns the result
+   * @return True iff assigned.
+   */
+  public boolean isRegionAssigned(HRegionInfo hri) {
+    synchronized (this.regions ) {
+      return regions.containsKey(hri);
+    }
+  }
+
+  /**
+   * Gives enabling table regions.
+   * 
+   * @param tableName
+   * @return list of regionInfos
+   */
+  public List getEnablingTableRegions(String tableName){
+    return this.enablingTables.get(tableName);
+  }
+
+  /**
+   * Add a regionPlan for the specified region.
+   * @param encodedName 
+   * @param plan 
+   */
+  public void addPlan(String encodedName, RegionPlan plan) {
+    synchronized (regionPlans) {
+      regionPlans.put(encodedName, plan);
+    }
+  }
+
+  /**
+   * Add a map of region plans.
+   */
+  public void addPlans(Map plans) {
+    synchronized (regionPlans) {
+      regionPlans.putAll(plans);
+    }
+  }
+
+  /**
+   * Set the list of regions that will be reopened
+   * because of an update in table schema
+   *
+   * @param regions
+   *          list of regions that should be tracked for reopen
+   */
+  public void setRegionsToReopen(List  regions) {
+    for(HRegionInfo hri : regions) {
+      regionsToReopen.put(hri.getEncodedName(), hri);
+    }
+  }
+
+  /**
+   * Used by the client to identify if all regions have the schema updates
+   *
+   * @param tableName
+   * @return Pair indicating the status of the alter command
+   * @throws IOException
+   */
+  public Pair getReopenStatus(byte[] tableName)
+  throws IOException {
+    List  hris =
+      MetaReader.getTableRegions(this.master.getCatalogTracker(), tableName);
+    Integer pending = 0;
+    for(HRegionInfo hri : hris) {
+      String name = hri.getEncodedName();
+      if (regionsToReopen.containsKey(name) || regionsInTransition.containsKey(name)) {
+        pending++;
+      }
+    }
+    return new Pair(pending, hris.size());
+  }
+  /**
+   * Reset all unassigned znodes.  Called on startup of master.
+   * Call {@link #assignAllUserRegions()} after root and meta have been assigned.
+   * @throws IOException
+   * @throws KeeperException
+   */
+  void cleanoutUnassigned() throws IOException, KeeperException {
+    // Cleanup any existing ZK nodes and start watching
+    ZKAssign.deleteAllNodes(watcher);
+    ZKUtil.listChildrenAndWatchForNewChildren(this.watcher,
+      this.watcher.assignmentZNode);
+  }
+
+  /**
+   * Called on startup.
+   * Figures whether a fresh cluster start of we are joining extant running cluster.
+   * @throws IOException
+   * @throws KeeperException
+   * @throws InterruptedException
+   */
+  void joinCluster() throws IOException,
+      KeeperException, InterruptedException {
+    // Concurrency note: In the below the accesses on regionsInTransition are
+    // outside of a synchronization block where usually all accesses to RIT are
+    // synchronized.  The presumption is that in this case it is safe since this
+    // method is being played by a single thread on startup.
+
+    // TODO: Regions that have a null location and are not in regionsInTransitions
+    // need to be handled.
+
+    // Scan META to build list of existing regions, servers, and assignment
+    // Returns servers who have not checked in (assumed dead) and their regions
+    Map>> deadServers = rebuildUserRegions();
+
+    processDeadServersAndRegionsInTransition(deadServers);
+
+    // Recover the tables that were not fully moved to DISABLED state.
+    // These tables are in DISABLING state when the master restarted/switched.
+    boolean isWatcherCreated = recoverTableInDisablingState(this.disablingTables);
+    recoverTableInEnablingState(this.enablingTables.keySet(), isWatcherCreated);
+    this.enablingTables.clear();
+    this.disablingTables.clear();
+  }
+
+  /**
+   * Process all regions that are in transition up in zookeeper.  Used by
+   * master joining an already running cluster.
+   * @throws KeeperException
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  void processDeadServersAndRegionsInTransition()
+  throws KeeperException, IOException, InterruptedException {
+    // Pass null to signify no dead servers in this context.
+    processDeadServersAndRegionsInTransition(null);
+  }
+
+  /**
+   * Process all regions that are in transition in zookeeper and also
+   * processes the list of dead servers by scanning the META. 
+   * Used by master joining an cluster.
+   * @param deadServers
+   *          Map of dead servers and their regions. Can be null.
+   * @throws KeeperException
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  void processDeadServersAndRegionsInTransition(
+      final Map>> deadServers)
+  throws KeeperException, IOException, InterruptedException {
+    List nodes = ZKUtil.listChildrenAndWatchForNewChildren(watcher,
+      watcher.assignmentZNode);
+    
+    if (nodes == null) {
+      String errorMessage = "Failed to get the children from ZK";
+      master.abort(errorMessage, new IOException(errorMessage));
+      return;
+    }
+    // Run through all regions.  If they are not assigned and not in RIT, then
+    // its a clean cluster startup, else its a failover.
+    synchronized (this.regions) {
+      for (Map.Entry e : this.regions.entrySet()) {
+        if (!e.getKey().isMetaTable() && e.getValue() != null) {
+          LOG.debug("Found " + e + " out on cluster");
+          this.failover = true;
+          break;
+        }
+        if (nodes.contains(e.getKey().getEncodedName())) {
+          LOG.debug("Found " + e.getKey().getRegionNameAsString() + " in RITs");
+          // Could be a meta region.
+          this.failover = true;
+          break;
+        }
+      }
+    }
+
+    // Remove regions in RIT, they are possibly being processed by
+    // ServerShutdownHandler.
+    synchronized (regionsInTransition) {
+      nodes.removeAll(regionsInTransition.keySet());
+    }
+
+    // If some dead servers are processed by ServerShutdownHandler, we shouldn't
+    // assign all user regions( some would be assigned by
+    // ServerShutdownHandler), consider it as a failover
+    if (!this.serverManager.getDeadServers().isEmpty()) {
+      this.failover = true;
+    }
+
+    // If we found user regions out on cluster, its a failover.
+    if (this.failover) {
+      LOG.info("Found regions out on cluster or in RIT; failover");
+      // Process list of dead servers and regions in RIT.
+      // See HBASE-4580 for more information.
+      processDeadServersAndRecoverLostRegions(deadServers, nodes);
+      this.failover = false;
+      failoverProcessedRegions.clear();
+    } else {
+      // Fresh cluster startup.
+      LOG.info("Clean cluster startup. Assigning userregions");
+      cleanoutUnassigned();
+      assignAllUserRegions();
+    }
+  }
+
+  /**
+   * If region is up in zk in transition, then do fixup and block and wait until
+   * the region is assigned and out of transition.  Used on startup for
+   * catalog regions.
+   * @param hri Region to look for.
+   * @return True if we processed a region in transition else false if region
+   * was not up in zk in transition.
+   * @throws InterruptedException
+   * @throws KeeperException
+   * @throws IOException
+   */
+  boolean processRegionInTransitionAndBlockUntilAssigned(final HRegionInfo hri)
+  throws InterruptedException, KeeperException, IOException {
+    boolean intransistion =
+      processRegionInTransition(hri.getEncodedName(), hri, null);
+    if (!intransistion) return intransistion;
+    LOG.debug("Waiting on " + HRegionInfo.prettyPrint(hri.getEncodedName()));
+    synchronized(this.regionsInTransition) {
+      while (!this.master.isStopped() &&
+          this.regionsInTransition.containsKey(hri.getEncodedName())) {
+        // We expect a notify, but by security we set a timout
+        this.regionsInTransition.wait(100);
+      }
+    }
+    return intransistion;
+  }
+
+  /**
+   * Process failover of new master for region encodedRegionName
+   * up in zookeeper.
+   * @param encodedRegionName Region to process failover for.
+   * @param regionInfo If null we'll go get it from meta table.
+   * @param deadServers Can be null 
+   * @return True if we processed regionInfo as a RIT.
+   * @throws KeeperException
+   * @throws IOException
+   */
+  boolean processRegionInTransition(final String encodedRegionName,
+      final HRegionInfo regionInfo,
+      final Map>> deadServers)
+  throws KeeperException, IOException {
+    Stat stat = new Stat();
+    RegionTransitionData data = ZKAssign.getDataAndWatch(watcher,
+        encodedRegionName, stat);
+    if (data == null) return false;
+    HRegionInfo hri = regionInfo;
+    if (hri == null) {
+      if ((hri = getHRegionInfo(data)) == null) return false; 
+    }
+    processRegionsInTransition(data, hri, deadServers, stat.getVersion());
+    return true;
+  }
+
+  void processRegionsInTransition(final RegionTransitionData data,
+      final HRegionInfo regionInfo,
+      final Map>> deadServers,
+      int expectedVersion)
+  throws KeeperException {
+    String encodedRegionName = regionInfo.getEncodedName();
+    LOG.info("Processing region " + regionInfo.getRegionNameAsString() +
+      " in state " + data.getEventType());
+    List hris = this.enablingTables.get(regionInfo.getTableNameAsString());
+    if (hris != null && !hris.isEmpty()) {
+      hris.remove(regionInfo);
+    }
+    synchronized (regionsInTransition) {
+      RegionState regionState = regionsInTransition.get(encodedRegionName);
+      if (regionState != null ||
+          failoverProcessedRegions.containsKey(encodedRegionName)) {
+        // Just return
+        return;
+      }
+      switch (data.getEventType()) {
+      case M_ZK_REGION_CLOSING:
+        // If zk node of the region was updated by a live server skip this
+        // region and just add it into RIT.
+        if (isOnDeadServer(regionInfo, deadServers) &&
+            (data.getOrigin() == null || !serverManager.isServerOnline(data.getOrigin()))) {
+          // If was on dead server, its closed now. Force to OFFLINE and this
+          // will get it reassigned if appropriate
+          forceOffline(regionInfo, data);
+        } else {
+          // Just insert region into RIT.
+          // If this never updates the timeout will trigger new assignment
+          regionsInTransition.put(encodedRegionName, new RegionState(
+            regionInfo, RegionState.State.CLOSING,
+            data.getStamp(), data.getOrigin()));
+        }
+        failoverProcessedRegions.put(encodedRegionName, regionInfo);
+        break;
+
+      case RS_ZK_REGION_CLOSED:
+      case RS_ZK_REGION_FAILED_OPEN:
+        // Region is closed, insert into RIT and handle it
+        addToRITandCallClose(regionInfo, RegionState.State.CLOSED, data);
+        failoverProcessedRegions.put(encodedRegionName, regionInfo);
+        break;
+
+      case M_ZK_REGION_OFFLINE:
+        // If zk node of the region was updated by a live server skip this
+        // region and just add it into RIT.
+        if (isOnDeadServer(regionInfo, deadServers) &&
+            (data.getOrigin() == null ||
+              !serverManager.isServerOnline(data.getOrigin()))) {
+          // Region is offline, insert into RIT and handle it like a closed
+          addToRITandCallClose(regionInfo, RegionState.State.OFFLINE, data);
+        } else if (data.getOrigin() != null &&
+            !serverManager.isServerOnline(data.getOrigin())) {
+          // to handle cases where offline node is created but sendRegionOpen
+          // RPC is not yet sent
+          addToRITandCallClose(regionInfo, RegionState.State.OFFLINE, data);
+        } else {
+          regionsInTransition.put(encodedRegionName, new RegionState(
+              regionInfo, RegionState.State.PENDING_OPEN, data.getStamp(), data
+                  .getOrigin()));
+        }
+        failoverProcessedRegions.put(encodedRegionName, regionInfo);
+        break;
+
+      case RS_ZK_REGION_OPENING:
+        // TODO: Could check if it was on deadServers.  If it was, then we could
+        // do what happens in TimeoutMonitor when it sees this condition.
+
+        // Just insert region into RIT
+        // If this never updates the timeout will trigger new assignment
+        if (regionInfo.isMetaTable()) {
+          regionsInTransition.put(encodedRegionName, new RegionState(
+              regionInfo, RegionState.State.OPENING, data.getStamp(), data
+                  .getOrigin()));
+          // If ROOT or .META. table is waiting for timeout monitor to assign
+          // it may take lot of time when the assignment.timeout.period is
+          // the default value which may be very long.  We will not be able
+          // to serve any request during this time.
+          // So we will assign the ROOT and .META. region immediately.
+          processOpeningState(regionInfo);
+          break;
+        }
+        regionsInTransition.put(encodedRegionName, new RegionState(regionInfo,
+            RegionState.State.OPENING, data.getStamp(), data.getOrigin()));
+        failoverProcessedRegions.put(encodedRegionName, regionInfo);
+        putRegionPlan(regionInfo, data.getOrigin());
+        break;
+
+      case RS_ZK_REGION_OPENED:
+        // Region is opened, insert into RIT and handle it
+        regionsInTransition.put(encodedRegionName, new RegionState(
+            regionInfo, RegionState.State.OPEN,
+            data.getStamp(), data.getOrigin()));
+        ServerName sn = data.getOrigin() == null? null: data.getOrigin();
+        // sn could be null if this server is no longer online.  If
+        // that is the case, just let this RIT timeout; it'll be assigned
+        // to new server then.
+        if (sn == null) {
+          LOG.warn("Region in transition " + regionInfo.getEncodedName() +
+            " references a null server; letting RIT timeout so will be " +
+            "assigned elsewhere");
+        } else if (!serverManager.isServerOnline(sn)
+            && (isOnDeadServer(regionInfo, deadServers)
+                || regionInfo.isMetaRegion() || regionInfo.isRootRegion())) {
+          forceOffline(regionInfo, data);
+        } else {
+          new OpenedRegionHandler(master, this, regionInfo, sn, expectedVersion)
+              .process();
+        }
+        failoverProcessedRegions.put(encodedRegionName, regionInfo);
+        break;
+      }
+    }
+  }
+
+  /**
+   * Put the region hri into an offline state up in zk.
+   * @param hri
+   * @param oldData
+   * @throws KeeperException
+   */
+  private void forceOffline(final HRegionInfo hri,
+      final RegionTransitionData oldData)
+  throws KeeperException {
+    // If was on dead server, its closed now.  Force to OFFLINE and then
+    // handle it like a close; this will get it reassigned if appropriate
+    LOG.debug("RIT " + hri.getEncodedName() + " in state=" +
+      oldData.getEventType() + " was on deadserver; forcing offline");
+    ZKAssign.createOrForceNodeOffline(this.watcher, hri,
+      this.master.getServerName());
+    addToRITandCallClose(hri, RegionState.State.OFFLINE, oldData);
+  }
+
+  /**
+   * Add to the in-memory copy of regions in transition and then call close
+   * handler on passed region hri
+   * @param hri
+   * @param state
+   * @param oldData
+   */
+  private void addToRITandCallClose(final HRegionInfo hri,
+      final RegionState.State state, final RegionTransitionData oldData) {
+    this.regionsInTransition.put(hri.getEncodedName(),
+      new RegionState(hri, state, oldData.getStamp(), oldData.getOrigin()));
+    new ClosedRegionHandler(this.master, this, hri).process();
+  }
+
+  /**
+   * When a region is closed, it should be removed from the regionsToReopen
+   * @param hri HRegionInfo of the region which was closed
+   */
+  public void removeClosedRegion(HRegionInfo hri) {
+    if (!regionsToReopen.isEmpty()) {
+      if (regionsToReopen.remove(hri.getEncodedName()) != null) {
+          LOG.debug("Removed region from reopening regions because it was closed");
+      }
+    }
+  }
+
+  /**
+   * @param regionInfo
+   * @param deadServers Map of deadServers and the regions they were carrying;
+   * can be null.
+   * @return True if the passed regionInfo in the passed map of deadServers?
+   */
+  private boolean isOnDeadServer(final HRegionInfo regionInfo,
+      final Map>> deadServers) {
+    if (deadServers == null) return false;
+    for (Map.Entry>> deadServer:
+        deadServers.entrySet()) {
+      for (Pair e: deadServer.getValue()) {
+        if (e.getFirst().equals(regionInfo)) return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Handles various states an unassigned node can be in.
+   * 
        
+   * Method is called when a state change is suspected for an unassigned node.
+   * 
        
+   * This deals with skipped transitions (we got a CLOSED but didn't see CLOSING
+   * yet).
+   * @param data
+   * @param expectedVersion
+   */
+  private void handleRegion(final RegionTransitionData data, int expectedVersion) {
+    synchronized(regionsInTransition) {
+      HRegionInfo hri = null;
+      if (data == null || data.getOrigin() == null) {
+        LOG.warn("Unexpected NULL input " + data);
+        return;
+      }
+      ServerName sn = data.getOrigin();
+      // Check if this is a special HBCK transition
+      if (sn.equals(HConstants.HBCK_CODE_SERVERNAME)) {
+        handleHBCK(data);
+        return;
+      }
+      String encodedName = HRegionInfo.encodeRegionName(data.getRegionName());
+      String prettyPrintedRegionName = HRegionInfo.prettyPrint(encodedName);
+      // Verify this is a known server
+      if (!serverManager.isServerOnline(sn) &&
+          !this.master.getServerName().equals(sn)
+          && !ignoreStatesRSOffline.contains(data.getEventType())) {
+        LOG.warn("Attempted to handle region transition for server but " +
+          "server is not online: " + prettyPrintedRegionName);
+        return;
+      }
+      // Printing if the event was created a long time ago helps debugging
+      boolean lateEvent = data.getStamp() <
+          (System.currentTimeMillis() - 15000);
+      LOG.debug("Handling transition=" + data.getEventType() +
+        ", server=" + data.getOrigin() + ", region=" +
+          (prettyPrintedRegionName == null? "null": prettyPrintedRegionName)  +
+          (lateEvent? ", which is more than 15 seconds late" : ""));
+      RegionState regionState = regionsInTransition.get(encodedName);
+      switch (data.getEventType()) {
+        case M_ZK_REGION_OFFLINE:
+          // Nothing to do.
+          break;
+
+        case RS_ZK_REGION_SPLITTING:
+          if (!isInStateForSplitting(regionState)) break;
+          addSplittingToRIT(sn, encodedName);
+          break;
+
+        case RS_ZK_REGION_SPLIT:
+          // RegionState must be null, or SPLITTING or PENDING_CLOSE.
+          if (!isInStateForSplitting(regionState)) break;
+          // If null, add SPLITTING state before going to SPLIT
+          if (regionState == null) {
+            regionState = addSplittingToRIT(sn, encodedName);
+            String message = "Received SPLIT for region " + prettyPrintedRegionName +
+              " from server " + sn;
+            // If still null, it means we cannot find it and it was already processed
+            if (regionState == null) {
+              LOG.warn(message + " but it doesn't exist anymore," +
+                  " probably already processed its split");
+              break;
+            }
+            LOG.info(message +
+                " but region was not first in SPLITTING state; continuing");
+          }
+          // Check it has daughters.
+          byte [] payload = data.getPayload();
+          List daughters = null;
+          try {
+            daughters = Writables.getHRegionInfos(payload, 0, payload.length);
+          } catch (IOException e) {
+            LOG.error("Dropped split! Failed reading split payload for " +
+              prettyPrintedRegionName);
+            break;
+          }
+          assert daughters.size() == 2;
+          // Assert that we can get a serverinfo for this server.
+          if (!this.serverManager.isServerOnline(sn)) {
+            LOG.error("Dropped split! ServerName=" + sn + " unknown.");
+            break;
+          }
+          // Run handler to do the rest of the SPLIT handling.
+          this.executorService.submit(new SplitRegionHandler(master, this,
+            regionState.getRegion(), sn, daughters));
+          break;
+
+        case M_ZK_REGION_CLOSING:
+          hri = checkIfInFailover(regionState, encodedName, data);
+          if (hri != null) {
+            regionState = new RegionState(hri, RegionState.State.CLOSING, data
+               .getStamp(), data.getOrigin());
+            regionsInTransition.put(encodedName, regionState);
+            failoverProcessedRegions.put(encodedName, hri);
+            break;
+          }
+          // Should see CLOSING after we have asked it to CLOSE or additional
+          // times after already being in state of CLOSING
+          if (regionState == null ||
+              (!regionState.isPendingClose() && !regionState.isClosing())) {
+            LOG.warn("Received CLOSING for region " + prettyPrintedRegionName +
+              " from server " + data.getOrigin() + " but region was in " +
+              " the state " + regionState + " and not " +
+              "in expected PENDING_CLOSE or CLOSING states");
+            return;
+          }
+          // Transition to CLOSING (or update stamp if already CLOSING)
+          regionState.update(RegionState.State.CLOSING,
+              data.getStamp(), data.getOrigin());
+          break;
+
+        case RS_ZK_REGION_CLOSED:
+          hri = checkIfInFailover(regionState, encodedName, data);
+          if (hri != null) {
+            regionState = new RegionState(hri, RegionState.State.CLOSED, data
+                .getStamp(), data.getOrigin());
+            regionsInTransition.put(encodedName, regionState);
+            removeClosedRegion(regionState.getRegion());
+            new ClosedRegionHandler(master, this, regionState.getRegion())
+              .process();
+            failoverProcessedRegions.put(encodedName, hri);
+            break;
+          }
+          // Should see CLOSED after CLOSING but possible after PENDING_CLOSE
+          if (regionState == null ||
+              (!regionState.isPendingClose() && !regionState.isClosing())) {
+            LOG.warn("Received CLOSED for region " + prettyPrintedRegionName +
+                " from server " + data.getOrigin() + " but region was in " +
+                " the state " + regionState + " and not " +
+                "in expected PENDING_CLOSE or CLOSING states");
+            return;
+          }
+          // Handle CLOSED by assigning elsewhere or stopping if a disable
+          // If we got here all is good.  Need to update RegionState -- else
+          // what follows will fail because not in expected state.
+          regionState.update(RegionState.State.CLOSED,
+              data.getStamp(), data.getOrigin());
+          removeClosedRegion(regionState.getRegion());
+          this.executorService.submit(new ClosedRegionHandler(master,
+            this, regionState.getRegion()));
+          break;
+          
+        case RS_ZK_REGION_FAILED_OPEN:
+          hri = checkIfInFailover(regionState, encodedName, data);
+          if (hri != null) {
+            regionState = new RegionState(hri, RegionState.State.CLOSED, data
+                .getStamp(), data.getOrigin());
+            regionsInTransition.put(encodedName, regionState);
+            new ClosedRegionHandler(master, this, regionState.getRegion())
+              .process();
+            failoverProcessedRegions.put(encodedName, hri);
+            break;
+          }
+          if (regionState == null ||
+              (!regionState.isOffline() && !regionState.isPendingOpen() && !regionState.isOpening())) {
+            LOG.warn("Received FAILED_OPEN for region " + prettyPrintedRegionName +
+                " from server " + data.getOrigin() + " but region was in " +
+                " the state " + regionState + " and not in OFFLINE, PENDING_OPEN or OPENING");
+            return;
+          }
+          // Handle this the same as if it were opened and then closed.
+          regionState.update(RegionState.State.CLOSED,
+              data.getStamp(), data.getOrigin());
+          // When there are more than one region server a new RS is selected as the 
+          // destination and the same is updated in the regionplan. (HBASE-5546)
+          getRegionPlan(regionState, sn, true);
+          this.executorService.submit(new ClosedRegionHandler(master,
+            this, regionState.getRegion()));
+          break;
+
+        case RS_ZK_REGION_OPENING:
+          hri = checkIfInFailover(regionState, encodedName, data);       
+          if (hri != null) {
+            regionState = new RegionState(hri, RegionState.State.OPENING, data
+                .getStamp(), data.getOrigin());
+            regionsInTransition.put(encodedName, regionState);
+            failoverProcessedRegions.put(encodedName, hri);
+            break;
+          }
+          if (regionState == null ||
+              (!regionState.isOffline() && !regionState.isPendingOpen() &&
+                  !regionState.isOpening())) {
+              LOG.warn("Received OPENING for region " + prettyPrintedRegionName +  " from server " +
+                sn + " but region was in " + " the state " + regionState + " and not " +
+                "in expected OFFLINE, PENDING_OPEN or OPENING states");
+              return;
+          }
+          // Transition to OPENING (or update stamp if already OPENING)
+          regionState.update(RegionState.State.OPENING,
+              data.getStamp(), data.getOrigin());
+          break;
+
+        case RS_ZK_REGION_OPENED:
+          hri = checkIfInFailover(regionState, encodedName, data);
+          if (hri != null) {
+            regionState = new RegionState(hri, RegionState.State.OPEN, data
+                .getStamp(), data.getOrigin());
+            regionsInTransition.put(encodedName, regionState);
+            new OpenedRegionHandler(master, this, regionState.getRegion(), data
+              .getOrigin(), expectedVersion).process();
+            failoverProcessedRegions.put(encodedName, hri);
+            break;
+          }
+          // Should see OPENED after OPENING but possible after PENDING_OPEN
+          if (regionState == null ||
+              (!regionState.isOffline() && !regionState.isPendingOpen() && !regionState.isOpening())) {
+            LOG.warn("Received OPENED for region " +
+                prettyPrintedRegionName +
+                " from server " + data.getOrigin() + " but region was in " +
+                " the state " + regionState + " and not " +
+                "in expected OFFLINE, PENDING_OPEN or OPENING states");
+            return;
+          }
+          // Handle OPENED by removing from transition and deleted zk node
+          regionState.update(RegionState.State.OPEN,
+              data.getStamp(), data.getOrigin());
+          this.executorService.submit(
+            new OpenedRegionHandler(master, this, regionState.getRegion(),
+              data.getOrigin(), expectedVersion));
+          break;
+      }
+    }
+  }
+
+  /**
+   * Checks whether the callback came while RIT was not yet populated during
+   * master failover.
+   * @param regionState
+   * @param encodedName
+   * @param data
+   * @return hri
+   */
+  private HRegionInfo checkIfInFailover(RegionState regionState,
+      String encodedName, RegionTransitionData data) {
+    if (regionState == null && this.failover &&
+        (failoverProcessedRegions.containsKey(encodedName) == false ||
+          failoverProcessedRegions.get(encodedName) == null)) {
+      HRegionInfo hri = this.failoverProcessedRegions.get(encodedName);
+      if (hri == null) hri = getHRegionInfo(data);
+      return hri;
+    }
+    return null;
+  }
+  
+  /**
+   * Gets the HRegionInfo from the META table
+   * @param  data
+   * @return HRegionInfo hri for the region 
+   */
+  private HRegionInfo getHRegionInfo(RegionTransitionData data) {
+    Pair p = null;
+    try {
+      p = MetaReader.getRegion(catalogTracker, data.getRegionName());
+      if (p == null) return null;
+      return p.getFirst();
+    } catch (IOException e) {
+      master.abort("Aborting because error occoured while reading "
+          + data.getRegionName() + " from .META.", e);
+      return null;
+    }
+  }
+
+  /**
+   * @return Returns true if this RegionState is splittable; i.e. the
+   * RegionState is currently in splitting state or pending_close or
+   * null (Anything else will return false). (Anything else will return false).
+   */
+  private boolean isInStateForSplitting(final RegionState rs) {
+    if (rs == null) return true;
+    if (rs.isSplitting()) return true;
+    if (convertPendingCloseToSplitting(rs)) return true;
+    LOG.warn("Dropped region split! Not in state good for SPLITTING; rs=" + rs);
+    return false;
+  }
+
+  /**
+   * If the passed regionState is in PENDING_CLOSE, clean up PENDING_CLOSE
+   * state and convert it to SPLITTING instead.
+   * This can happen in case where master wants to close a region at same time
+   * a regionserver starts a split.  The split won.  Clean out old PENDING_CLOSE
+   * state.
+   * @param rs
+   * @return True if we converted from PENDING_CLOSE to SPLITTING
+   */
+  private boolean convertPendingCloseToSplitting(final RegionState rs) {
+    if (!rs.isPendingClose()) return false;
+    LOG.debug("Converting PENDING_CLOSE to SPLITING; rs=" + rs);
+    rs.update(RegionState.State.SPLITTING);
+    // Clean up existing state.  Clear from region plans seems all we
+    // have to do here by way of clean up of PENDING_CLOSE.
+    clearRegionPlan(rs.getRegion());
+    return true;
+  }
+
+  /**
+   * @param serverName
+   * @param encodedName
+   * @return The SPLITTING RegionState we added to RIT for the passed region
+   * encodedName
+   */
+  private RegionState addSplittingToRIT(final ServerName serverName,
+      final String encodedName) {
+    RegionState regionState = null;
+    synchronized (this.regions) {
+      regionState = findHRegionInfoThenAddToRIT(serverName, encodedName);
+      if (regionState != null) {
+        regionState.update(RegionState.State.SPLITTING,
+          System.currentTimeMillis(), serverName);
+      }
+    }
+    return regionState;
+  }
+
+  /**
+   * Caller must hold lock on this.regions.
+   * @param serverName
+   * @param encodedName
+   * @return The instance of RegionState that was added to RIT or null if error.
+   */
+  private RegionState findHRegionInfoThenAddToRIT(final ServerName serverName,
+      final String encodedName) {
+    HRegionInfo hri = findHRegionInfo(serverName, encodedName);
+    if (hri == null) {
+      LOG.warn("Region " + encodedName + " not found on server " + serverName +
+        "; failed processing");
+      return null;
+    }
+    // Add to regions in transition, then update state to SPLITTING.
+    return addToRegionsInTransition(hri);
+  }
+
+  /**
+   * Caller must hold lock on this.regions.
+   * @param serverName
+   * @param encodedName
+   * @return Found HRegionInfo or null.
+   */
+  private HRegionInfo findHRegionInfo(final ServerName sn,
+      final String encodedName) {
+    if (!this.serverManager.isServerOnline(sn)) return null;
+    Set hris = this.servers.get(sn);
+    HRegionInfo foundHri = null;
+    for (HRegionInfo hri: hris) {
+      if (hri.getEncodedName().equals(encodedName)) {
+        foundHri = hri;
+        break;
+      }
+    }
+    return foundHri;
+  }
+
+  /**
+   * Handle a ZK unassigned node transition triggered by HBCK repair tool.
+   * 
        
+   * This is handled in a separate code path because it breaks the normal rules.
+   * @param data
+   */
+  private void handleHBCK(RegionTransitionData data) {
+    String encodedName = HRegionInfo.encodeRegionName(data.getRegionName());
+    LOG.info("Handling HBCK triggered transition=" + data.getEventType() +
+      ", server=" + data.getOrigin() + ", region=" +
+      HRegionInfo.prettyPrint(encodedName));
+    RegionState regionState = regionsInTransition.get(encodedName);
+    switch (data.getEventType()) {
+      case M_ZK_REGION_OFFLINE:
+        HRegionInfo regionInfo = null;
+        if (regionState != null) {
+          regionInfo = regionState.getRegion();
+        } else {
+          try {
+            byte[] name = data.getRegionName();
+            Pair p = MetaReader.getRegion(catalogTracker, name);
+            regionInfo = p.getFirst();
+          } catch (IOException e) {
+            LOG.info("Exception reading META doing HBCK repair operation", e);
+            return;
+          }
+        }
+        LOG.info("HBCK repair is triggering assignment of region=" +
+            regionInfo.getRegionNameAsString());
+        // trigger assign, node is already in OFFLINE so don't need to update ZK
+        assign(regionInfo, false);
+        break;
+
+      default:
+        LOG.warn("Received unexpected region state from HBCK (" +
+            data.getEventType() + ")");
+        break;
+    }
+  }
+
+  // ZooKeeper events
+
+  /**
+   * New unassigned node has been created.
+   *
+   * 
        This happens when an RS begins the OPENING or CLOSING of a region by
+   * creating an unassigned node.
+   *
+   * 
        When this happens we must:
+   * 
          
+   *   
        1. Watch the node for further events
          2. 
+   *   
        2. Read and handle the state in the node
          3. 
+   * 
        
+   */
+  @Override
+  public void nodeCreated(String path) {
+    if(path.startsWith(watcher.assignmentZNode)) {
+      try {
+        Stat stat = new Stat();
+        RegionTransitionData data = ZKAssign.getDataAndWatch(watcher, path, stat);
+        if (data == null) {
+          return;
+        }
+        handleRegion(data, stat.getVersion());
+      } catch (KeeperException e) {
+        master.abort("Unexpected ZK exception reading unassigned node data", e);
+      }
+    }
+  }
+
+  /**
+   * Existing unassigned node has had data changed.
+   *
+   * 
        This happens when an RS transitions from OFFLINE to OPENING, or between
+   * OPENING/OPENED and CLOSING/CLOSED.
+   *
+   * 
        When this happens we must:
+   * 
          
+   *   
        1. Watch the node for further events
          2. 
+   *   
        2. Read and handle the state in the node
          3. 
+   * 
        
+   */
+  @Override
+  public void nodeDataChanged(String path) {
+    if(path.startsWith(watcher.assignmentZNode)) {
+      try {
+        Stat stat = new Stat();
+        RegionTransitionData data = ZKAssign.getDataAndWatch(watcher, path, stat);
+        if (data == null) {
+          return;
+        }
+        handleRegion(data, stat.getVersion());
+      } catch (KeeperException e) {
+        master.abort("Unexpected ZK exception reading unassigned node data", e);
+      }
+    }
+  }
+
+  @Override
+  public void nodeDeleted(final String path) {
+    if (path.startsWith(this.watcher.assignmentZNode)) {
+      String regionName = ZKAssign.getRegionName(this.master.getZooKeeper(), path);
+      RegionState rs = this.regionsInTransition.get(regionName);
+      if (rs != null) {
+        HRegionInfo regionInfo = rs.getRegion();
+        if (rs.isSplit()) {
+          LOG.debug("Ephemeral node deleted, regionserver crashed?, offlining the region"
+              + rs.getRegion() + " clearing from RIT;");
+          regionOffline(rs.getRegion());
+        } else if (rs.isSplitting()) {
+          LOG.debug("Ephemeral node deleted.  Found in SPLITTING state. " + "Removing from RIT "
+              + rs.getRegion());
+          synchronized(this.regionsInTransition) {
+            this.regionsInTransition.remove(regionName);
+          }
+        } else {
+          LOG.debug("The znode of region " + regionInfo.getRegionNameAsString()
+              + " has been deleted.");
+          if (rs.isOpened()) {
+            makeRegionOnline(rs, regionInfo);
+          }
+        }
+      }
+    }
+  }
+
+  private void makeRegionOnline(RegionState rs, HRegionInfo regionInfo) {
+    regionOnline(regionInfo, rs.serverName);
+    LOG.info("The master has opened the region "
+        + regionInfo.getRegionNameAsString() + " that was online on "
+        + rs.serverName);
+    if (this.getZKTable().isDisablingOrDisabledTable(
+        regionInfo.getTableNameAsString())) {
+      LOG.debug("Opened region "
+          + regionInfo.getRegionNameAsString() + " but "
+          + "this table is disabled, triggering close of region");
+      unassign(regionInfo);
+    }
+  }
+
+  /**
+   * New unassigned node has been created.
+   *
+   * 
        This happens when an RS begins the OPENING, SPLITTING or CLOSING of a
+   * region by creating a znode.
+   *
+   * 
        When this happens we must:
+   * 
          
+   *   
        1. Watch the node for further children changed events
          2. 
+   *   
        2. Watch all new children for changed events
          3. 
+   * 
        
+   */
+  @Override
+  public void nodeChildrenChanged(String path) {
+    if(path.equals(watcher.assignmentZNode)) {
+      try {
+        List children = ZKUtil.listChildrenAndWatchForNewChildren(watcher,
+            watcher.assignmentZNode);
+        if (children != null) {
+          Stat stat = new Stat();
+          for (String child : children) {
+            stat.setVersion(0);
+            RegionTransitionData data = ZKAssign.getDataAndWatch(watcher,
+                ZKUtil.joinZNode(watcher.assignmentZNode, child), stat);
+            // See HBASE-7551, handle splitting here as well, in case we miss the node change event
+            if (stat.getVersion() > 0 && data.getEventType() == EventType.RS_ZK_REGION_SPLITTING) {
+              handleRegion(data, stat.getVersion());
+            }
+          }
+        }
+      } catch(KeeperException e) {
+        master.abort("Unexpected ZK exception reading unassigned children", e);
+      }
+    }
+  }
+
+  /**
+   * Marks the region as online.  Removes it from regions in transition and
+   * updates the in-memory assignment information.
+   * 
        
+   * Used when a region has been successfully opened on a region server.
+   * @param regionInfo
+   * @param sn
+   */
+  void regionOnline(HRegionInfo regionInfo, ServerName sn) {
+    synchronized (this.regionsInTransition) {
+      RegionState rs =
+        this.regionsInTransition.remove(regionInfo.getEncodedName());
+      if (rs != null) {
+        this.regionsInTransition.notifyAll();
+      }
+    }
+    synchronized (this.regions) {
+      // Add check
+      ServerName oldSn = this.regions.get(regionInfo);
+      if (oldSn != null) LOG.warn("Overwriting " + regionInfo.getEncodedName() +
+        " on " + oldSn + " with " + sn);
+      
+      if (isServerOnline(sn)) {
+        this.regions.put(regionInfo, sn);
+        addToServers(sn, regionInfo);
+        this.regions.notifyAll();
+      } else {
+        LOG.info("The server is not in online servers, ServerName=" + 
+          sn.getServerName() + ", region=" + regionInfo.getEncodedName());
+      }
+    }
+    // Remove plan if one.
+    clearRegionPlan(regionInfo);
+    // Add the server to serversInUpdatingTimer
+    addToServersInUpdatingTimer(sn);
+  }
+
+  /**
+   * Add the server to the set serversInUpdatingTimer, then {@link TimerUpdater}
+   * will update timers for this server in background
+   * @param sn
+   */
+  private void addToServersInUpdatingTimer(final ServerName sn) {
+    this.serversInUpdatingTimer.add(sn);
+  }
+
+  /**
+   * Touch timers for all regions in transition that have the passed
+   * sn in common.
+   * Call this method whenever a server checks in.  Doing so helps the case where
+   * a new regionserver has joined the cluster and its been given 1k regions to
+   * open.  If this method is tickled every time the region reports in a
+   * successful open then the 1k-th region won't be timed out just because its
+   * sitting behind the open of 999 other regions.  This method is NOT used
+   * as part of bulk assign -- there we have a different mechanism for extending
+   * the regions in transition timer (we turn it off temporarily -- because
+   * there is no regionplan involved when bulk assigning.
+   * @param sn
+   */
+  private void updateTimers(final ServerName sn) {
+    // This loop could be expensive.
+    // First make a copy of current regionPlan rather than hold sync while
+    // looping because holding sync can cause deadlock.  Its ok in this loop
+    // if the Map we're going against is a little stale
+    Map copy = new HashMap();
+    synchronized(this.regionPlans) {
+      copy.putAll(this.regionPlans);
+    }
+    for (Map.Entry e: copy.entrySet()) {
+      if (e.getValue() == null || e.getValue().getDestination() == null) continue;
+      if (!e.getValue().getDestination().equals(sn)) continue;
+      RegionState rs = null;
+      synchronized (this.regionsInTransition) {
+        rs = this.regionsInTransition.get(e.getKey());
+      }
+      if (rs == null) continue;
+      rs.updateTimestampToNow();
+    }
+  }
+
+  /**
+   * Marks the region as offline.  Removes it from regions in transition and
+   * removes in-memory assignment information.
+   * 
        
+   * Used when a region has been closed and should remain closed.
+   * @param regionInfo
+   */
+  public void regionOffline(final HRegionInfo regionInfo) {
+    // remove the region plan as well just in case.
+    clearRegionPlan(regionInfo);
+    setOffline(regionInfo);
+
+    // This is needed in case of secondary index load balancer so that the
+    // region needs to be cleared from our map
+    clearRegionPlanFromBalancerPlan(regionInfo);
+
+    synchronized(this.regionsInTransition) {
+      if (this.regionsInTransition.remove(regionInfo.getEncodedName()) != null) {
+        this.regionsInTransition.notifyAll();
+      }
+    }
+  }
+
+  /**
+   * Sets the region as offline by removing in-memory assignment information but
+   * retaining transition information.
+   * 
        
+   * Used when a region has been closed but should be reassigned.
+   * @param regionInfo
+   */
+  public void setOffline(HRegionInfo regionInfo) {
+    synchronized (this.regions) {
+      ServerName sn = this.regions.remove(regionInfo);
+      if (sn == null) return;
+      Set serverRegions = this.servers.get(sn);
+      if (!serverRegions.remove(regionInfo)) {
+        LOG.warn("No " + regionInfo + " on " + sn);
+      }
+    }
+  }
+
+  public void offlineDisabledRegion(HRegionInfo regionInfo) {
+    // Disabling so should not be reassigned, just delete the CLOSED node
+    LOG.debug("Table being disabled so deleting ZK node and removing from " +
+        "regions in transition, skipping assignment of region " +
+          regionInfo.getRegionNameAsString());
+    try {
+      if (!ZKAssign.deleteClosedNode(watcher, regionInfo.getEncodedName())) {
+        // Could also be in OFFLINE mode
+        ZKAssign.deleteOfflineNode(watcher, regionInfo.getEncodedName());
+      }
+    } catch (KeeperException.NoNodeException nne) {
+      LOG.debug("Tried to delete closed node for " + regionInfo + " but it " +
+          "does not exist so just offlining");
+    } catch (KeeperException e) {
+      this.master.abort("Error deleting CLOSED node in ZK", e);
+    }
+    regionOffline(regionInfo);
+  }
+
+  // Assignment methods
+
+  /**
+   * Assigns the specified region.
+   * 
        
+   * If a RegionPlan is available with a valid destination then it will be used
+   * to determine what server region is assigned to.  If no RegionPlan is
+   * available, region will be assigned to a random available server.
+   * 
        
+   * Updates the RegionState and sends the OPEN RPC.
+   * 
        
+   * This will only succeed if the region is in transition and in a CLOSED or
+   * OFFLINE state or not in transition (in-memory not zk), and of course, the
+   * chosen server is up and running (It may have just crashed!).  If the
+   * in-memory checks pass, the zk node is forced to OFFLINE before assigning.
+   *
+   * @param region server to be assigned
+   * @param setOfflineInZK whether ZK node should be created/transitioned to an
+   *                       OFFLINE state before assigning the region
+   */
+  public void assign(HRegionInfo region, boolean setOfflineInZK) {
+    assign(region, setOfflineInZK, false);
+  }
+
+  public void assign(HRegionInfo region, boolean setOfflineInZK,
+      boolean forceNewPlan) {
+    assign(region, setOfflineInZK, forceNewPlan, false);
+  }
+
+  /**
+   * @param region
+   * @param setOfflineInZK
+   * @param forceNewPlan
+   * @param hijack
+   *          - true new assignment is needed, false otherwise
+   */
+  public void assign(HRegionInfo region, boolean setOfflineInZK,
+      boolean forceNewPlan, boolean hijack) {
+    // If hijack is true do not call disableRegionIfInRIT as 
+    // we have not yet moved the znode to OFFLINE state.
+    if (!hijack && isDisabledorDisablingRegionInRIT(region)) {
+      return;
+    }
+    if (this.serverManager.isClusterShutdown()) {
+      LOG.info("Cluster shutdown is set; skipping assign of " +
+        region.getRegionNameAsString());
+      return;
+    }
+    RegionState state = addToRegionsInTransition(region,
+        hijack);
+    synchronized (state) {
+      assign(region, state, setOfflineInZK, forceNewPlan, hijack);
+    }
+  }
+
+  /**
+   * Bulk assign regions to destination.
+   * @param destination
+   * @param regions Regions to assign.
+   */
+  void assign(final ServerName destination,
+      final List regions) {
+    if (regions.size() == 0) {
+      return;
+    }
+    LOG.debug("Bulk assigning " + regions.size() + " region(s) to " +
+      destination.toString());
+
+    List states = new ArrayList(regions.size());
+    synchronized (this.regionsInTransition) {
+      for (HRegionInfo region: regions) {
+        states.add(forceRegionStateToOffline(region));
+      }
+    }
+    // Add region plans, so we can updateTimers when one region is opened so
+    // that unnecessary timeout on RIT is reduced.
+    Map plans=new HashMap();
+    for (HRegionInfo region : regions) {
+      plans.put(region.getEncodedName(), new RegionPlan(region, null,
+          destination));
+    }
+    this.addPlans(plans);
+    
+    // Presumption is that only this thread will be updating the state at this
+    // time; i.e. handlers on backend won't be trying to set it to OPEN, etc.
+    AtomicInteger counter = new AtomicInteger(0);
+    CreateUnassignedAsyncCallback cb =
+      new CreateUnassignedAsyncCallback(this.watcher, destination, counter);
+    for (RegionState state: states) {
+      if (!asyncSetOfflineInZooKeeper(state, cb, state)) {
+        return;
+      }
+    }
+    // Wait until all unassigned nodes have been put up and watchers set.
+    int total = regions.size();
+    for (int oldCounter = 0; true;) {
+      int count = counter.get();
+      if (oldCounter != count) {
+        LOG.info(destination.toString() + " outstanding calls=" + count +
+          " of total=" + total);
+        oldCounter = count;
+      }
+      if (count == total) break;
+      Threads.sleep(1);
+    }
+    // Check if any failed.
+    if (cb.hasErrors()) {
+      // TODO: createOrForceNodeOffline actually handles this condition; whereas this
+      //       code used to just abort master. Now, it will bail more "gracefully".
+      LOG.error("Error creating nodes for some of the regions we are trying to bulk assign");
+      return;
+    }
+
+    // Move on to open regions.
+    try {
+      // Send OPEN RPC. If it fails on a IOE or RemoteException, the
+      // TimeoutMonitor will pick up the pieces.
+      long maxWaitTime = System.currentTimeMillis() +
+        this.master.getConfiguration().
+          getLong("hbase.regionserver.rpc.startup.waittime", 60000);
+      while (!this.master.isStopped()) {
+        try {
+          this.serverManager.sendRegionOpen(destination, regions);
+          break;
+        } catch (RemoteException e) {
+          IOException decodedException = e.unwrapRemoteException();
+          if (decodedException instanceof RegionServerStoppedException) {
+            LOG.warn("The region server was shut down, ", decodedException);
+            // No need to retry, the region server is a goner.
+            return;
+          } else if (decodedException instanceof ServerNotRunningYetException) {
+            // This is the one exception to retry.  For all else we should just fail
+            // the startup.
+            long now = System.currentTimeMillis();
+            if (now > maxWaitTime) throw e;
+            LOG.debug("Server is not yet up; waiting up to " +
+                (maxWaitTime - now) + "ms", e);
+            Thread.sleep(1000);
+          }
+
+          throw decodedException;
+        }
+      }
+    } catch (IOException e) {
+      // Can be a socket timeout, EOF, NoRouteToHost, etc
+      LOG.info("Unable to communicate with the region server in order" +
+          " to assign regions", e);
+    } catch (InterruptedException e) {
+      throw new RuntimeException(e);
+    }
+    LOG.debug("Bulk assigning done for " + destination.toString());
+  }
+
+  /**
+   * Callback handler for create unassigned znodes used during bulk assign.
+   */
+  static class CreateUnassignedAsyncCallback implements AsyncCallback.StringCallback {
+    private final Log LOG = LogFactory.getLog(CreateUnassignedAsyncCallback.class);
+    private final ZooKeeperWatcher zkw;
+    private final ServerName destination;
+    private final AtomicInteger counter;
+    private final AtomicInteger errorCount = new AtomicInteger(0);
+
+    CreateUnassignedAsyncCallback(final ZooKeeperWatcher zkw,
+        final ServerName destination, final AtomicInteger counter) {
+      this.zkw = zkw;
+      this.destination = destination;
+      this.counter = counter;
+    }
+
+    boolean hasErrors() {
+      return this.errorCount.get() > 0;
+    }
+
+    @Override
+    public void processResult(int rc, String path, Object ctx, String name) {
+      if (rc == KeeperException.Code.NODEEXISTS.intValue()) {
+        LOG.warn("Node for " + path + " already exists");
+        reportCompletion(false);
+        return;
+      }
+      if (rc != 0) {
+        // This is resultcode. If non-zero, we will abort :(
+        LOG.warn("rc != 0 for " + path + " -- some error, may be retryable connection loss -- "
+            + "FIX see http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A2");
+        this.zkw.abort("Some error, may be connection loss writing unassigned at " + path +
+          ", rc=" + rc, null);
+        return;
+      }
+      LOG.debug("rs=" + (RegionState)ctx + ", server=" + this.destination.toString());
+      // Async exists to set a watcher so we'll get triggered when
+      // unassigned node changes.
+      this.zkw.getRecoverableZooKeeper().getZooKeeper().exists(path, this.zkw,
+        new ExistsUnassignedAsyncCallback(this, destination), ctx);
+    }
+
+    void reportCompletion(boolean success) {
+      if (!success) {
+        this.errorCount.incrementAndGet();
+      }
+      this.counter.incrementAndGet();
+    }
+  }
+
+  /**
+   * Callback handler for the exists call that sets watcher on unassigned znodes.
+   * Used during bulk assign on startup.
+   */
+  static class ExistsUnassignedAsyncCallback implements AsyncCallback.StatCallback {
+    private final Log LOG = LogFactory.getLog(ExistsUnassignedAsyncCallback.class);
+    private ServerName destination;
+    private CreateUnassignedAsyncCallback parent;
+
+    ExistsUnassignedAsyncCallback(
+        CreateUnassignedAsyncCallback parent, ServerName destination) {
+      this.parent = parent;
+      this.destination = destination;
+    }
+
+    @Override
+    public void processResult(int rc, String path, Object ctx, Stat stat) {
+      if (rc != 0) {
+        // This is resultcode.  If non-zero, need to resubmit.
+        LOG.warn("rc != 0 for " + path + " -- some error, may be connection loss -- " +
+          "FIX see http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A2");
+        parent.reportCompletion(false);
+        return;
+      }
+      RegionState state = (RegionState)ctx;
+      LOG.debug("rs=" + state);
+      // Transition RegionState to PENDING_OPEN here in master; means we've
+      // sent the open.  We're a little ahead of ourselves here since we've not
+      // yet sent out the actual open but putting this state change after the
+      // call to open risks our writing PENDING_OPEN after state has been moved
+      // to OPENING by the regionserver.
+      state.update(RegionState.State.PENDING_OPEN, System.currentTimeMillis(), destination);
+      parent.reportCompletion(true);
+    }
+  }
+
+  /**
+   * @param region
+   * @return The current RegionState
+   */
+  private RegionState addToRegionsInTransition(final HRegionInfo region) {
+    return addToRegionsInTransition(region, false);
+  }
+  /**
+   * @param region
+   * @param hijack
+   * @return The current RegionState
+   */
+  private RegionState addToRegionsInTransition(final HRegionInfo region,
+      boolean hijack) {
+    synchronized (regionsInTransition) {
+      return forceRegionStateToOffline(region, hijack);
+    }
+  }
+  /**
+   * Sets regions {@link RegionState} to {@link RegionState.State#OFFLINE}.
+   * Caller must hold lock on this.regionsInTransition.
+   * @param region
+   * @return Amended RegionState.
+   */
+  private RegionState forceRegionStateToOffline(final HRegionInfo region) {
+    return forceRegionStateToOffline(region, false);
+  }
+
+  /**
+   * Sets regions {@link RegionState} to {@link RegionState.State#OFFLINE}.
+   * Caller must hold lock on this.regionsInTransition.
+   * @param region
+   * @param hijack
+   * @return Amended RegionState.
+   */
+  private RegionState forceRegionStateToOffline(final HRegionInfo region,
+      boolean hijack) {
+    String encodedName = region.getEncodedName();
+    RegionState state = this.regionsInTransition.get(encodedName);
+    if (state == null) {
+      state = new RegionState(region, RegionState.State.OFFLINE);
+      this.regionsInTransition.put(encodedName, state);
+    } else {
+      // If we are reassigning the node do not force in-memory state to OFFLINE.
+      // Based on the znode state we will decide if to change in-memory state to
+      // OFFLINE or not. It will be done before setting znode to OFFLINE state.
+
+      // We often get here with state == CLOSED because ClosedRegionHandler will
+      // assign on its tail as part of the handling of a region close.
+      if (!hijack) {
+        LOG.debug("Forcing OFFLINE; was=" + state);
+        state.update(RegionState.State.OFFLINE);
+      }
+    }
+    return state;
+  }
+
+  /**
+   * Caller must hold lock on the passed state object.
+   * @param state
+   * @param setOfflineInZK
+   * @param forceNewPlan
+   * @param hijack
+   */
+  private void assign(final HRegionInfo region, final RegionState state,
+      final boolean setOfflineInZK, final boolean forceNewPlan,
+      boolean hijack) {
+    boolean regionAlreadyInTransitionException = false;
+    boolean serverNotRunningYet = false;
+    long maxRegionServerStartupWaitTime = -1;
+    for (int i = 0; i < this.maximumAssignmentAttempts; i++) {
+      int versionOfOfflineNode = -1;
+      if (setOfflineInZK) {
+        // get the version of the znode after setting it to OFFLINE.
+        // versionOfOfflineNode will be -1 if the znode was not set to OFFLINE
+        versionOfOfflineNode = setOfflineInZooKeeper(state, hijack,
+            regionAlreadyInTransitionException);
+        if(versionOfOfflineNode != -1){
+          if (isDisabledorDisablingRegionInRIT(region)) {
+            return;
+          }
+          // In case of assign from EnableTableHandler table state is ENABLING. Any how
+          // EnableTableHandler will set ENABLED after assigning all the table regions. If we
+          // try to set to ENABLED directly then client api may think ENABLE table is completed.
+          // When we have a case like all the regions are added directly into META and we call
+          // assignRegion then we need to make the table ENABLED. Hence in such case the table
+          // will not be in ENABLING or ENABLED state.
+          String tableName = region.getTableNameAsString();
+          if (!zkTable.isEnablingTable(tableName) && !zkTable.isEnabledTable(tableName)) {
+            LOG.debug("Setting table " + tableName + " to ENABLED state.");
+            setEnabledTable(region);
+          }
+        }
+      }
+      
+      if (setOfflineInZK && versionOfOfflineNode == -1) {
+        return;
+      }
+      
+      if (this.master.isStopped()) {
+        LOG.debug("Server stopped; skipping assign of " + state);
+        return;
+      }
+      RegionPlan plan = getRegionPlan(state, !regionAlreadyInTransitionException
+          && !serverNotRunningYet && forceNewPlan);
+      if (plan == null) {
+        LOG.debug("Unable to determine a plan to assign " + state);
+        this.timeoutMonitor.setAllRegionServersOffline(true);
+        return; // Should get reassigned later when RIT times out.
+      }
+      try {
+        LOG.debug("Assigning region " + state.getRegion().getRegionNameAsString() +
+          " to " + plan.getDestination().toString());
+        long currentOfflineTimeStamp = state.getStamp();
+        RegionOpeningState regionOpenState = serverManager.sendRegionOpen(plan.getDestination(),
+            state.getRegion(), versionOfOfflineNode);
+        if (regionOpenState == RegionOpeningState.OPENED) {
+          // Transition RegionState to PENDING_OPEN
+          // Check if already the offline state has been updated due to a
+          // failure in prev assign
+          if (state.isOffline() && currentOfflineTimeStamp != state.getStamp()) {
+            return;
+          }
+          if (state.isOffline() && !state.isOpening()) {
+            state.update(RegionState.State.PENDING_OPEN,
+                System.currentTimeMillis(), plan.getDestination());
+          }
+          if (state.isOpening()) return;
+          if (state.isOpened()) return;
+        } else if (regionOpenState == RegionOpeningState.ALREADY_OPENED) {
+          // Remove region from in-memory transition and unassigned node from ZK
+          // While trying to enable the table the regions of the table were
+          // already enabled.
+          LOG.debug("ALREADY_OPENED region " + state.getRegion().getRegionNameAsString() +
+              " to " + plan.getDestination().toString());
+          String encodedRegionName = state.getRegion()
+              .getEncodedName();
+          try {
+            ZKAssign.deleteOfflineNode(master.getZooKeeper(), encodedRegionName);
+          } catch (KeeperException.NoNodeException e) {
+            if(LOG.isDebugEnabled()){
+              LOG.debug("The unassigned node "+encodedRegionName+" doesnot exist.");
+            }
+          } catch (KeeperException e) {
+            master.abort(
+                "Error deleting OFFLINED node in ZK for transition ZK node ("
+                    + encodedRegionName + ")", e);
+          }
+          synchronized (this.regionsInTransition) {
+            this.regionsInTransition.remove(plan.getRegionInfo()
+                .getEncodedName());
+          }
+          synchronized (this.regions) {
+            this.regions.put(plan.getRegionInfo(), plan.getDestination());
+          }
+        }
+        break;
+      } catch (Throwable t) {
+        if (t instanceof RemoteException) {
+          t = ((RemoteException) t).unwrapRemoteException();
+        }
+        regionAlreadyInTransitionException = false;
+        serverNotRunningYet = false;
+        if (t instanceof RegionAlreadyInTransitionException) {
+          regionAlreadyInTransitionException = true;
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Failed assignment in: " + plan.getDestination() + " due to "
+                + t.getMessage());
+          }
+        } else if (t instanceof ServerNotRunningYetException) {
+          if (maxRegionServerStartupWaitTime < 0) {
+            maxRegionServerStartupWaitTime = System.currentTimeMillis()
+                + this.master.getConfiguration().getLong("hbase.regionserver.rpc.startup.waittime",
+                    60000);
+          }
+          try {
+            long now = System.currentTimeMillis();
+            if (now < maxRegionServerStartupWaitTime) {
+              LOG.debug("Server is not yet up; waiting up to "
+                  + (maxRegionServerStartupWaitTime - now) + "ms", t);
+              serverNotRunningYet = true;
+              Thread.sleep(100);
+              i--; // reset the try count
+            } else {
+              LOG.debug("Server is not up for a while; try a new one", t);
+            }
+          } catch (InterruptedException ie) {
+            LOG.warn("Failed to assign " + state.getRegion().getRegionNameAsString()
+                + " since interrupted", ie);
+            Thread.currentThread().interrupt();
+            return;
+          }
+        } else if (t instanceof java.net.SocketTimeoutException 
+            && this.serverManager.isServerOnline(plan.getDestination())) {
+          LOG.warn("Call openRegion() to " + plan.getDestination()
+              + " has timed out when trying to assign "
+              + region.getRegionNameAsString()
+              + ", but the region might already be opened on "
+              + plan.getDestination() + ".", t);
+          return;
+        }
+        LOG.warn("Failed assignment of "
+          + state.getRegion().getRegionNameAsString()
+          + " to "
+          + plan.getDestination()
+          + ", trying to assign "
+          + (regionAlreadyInTransitionException || serverNotRunningYet
+            ? "to the same region server because of "
+            + "RegionAlreadyInTransitionException/ServerNotRunningYetException;"
+            : "elsewhere instead; ")
+          + "retry=" + i, t);
+        // Clean out plan we failed execute and one that doesn't look like it'll
+        // succeed anyways; we need a new plan!
+        // Transition back to OFFLINE
+        state.update(RegionState.State.OFFLINE);
+        // If region opened on destination of present plan, reassigning to new
+        // RS may cause double assignments. In case of RegionAlreadyInTransitionException
+        // reassigning to same RS.
+        RegionPlan newPlan = plan;
+        if (!regionAlreadyInTransitionException && !serverNotRunningYet) {
+          // Force a new plan and reassign. Will return null if no servers.
+          // The new plan could be the same as the existing plan since we don't
+          // exclude the server of the original plan, which should not be
+          // excluded since it could be the only server up now.
+          newPlan = getRegionPlan(state, true);
+        }
+        if (newPlan == null) {
+          this.timeoutMonitor.setAllRegionServersOffline(true);
+          LOG.warn("Unable to find a viable location to assign region " +
+            state.getRegion().getRegionNameAsString());
+          return;
+        }
+      }
+    }
+  }
+
+  private boolean isDisabledorDisablingRegionInRIT(final HRegionInfo region) {
+    String tableName = region.getTableNameAsString();
+    boolean disabled = this.zkTable.isDisabledTable(tableName);
+    if (disabled || this.zkTable.isDisablingTable(tableName)) {
+      LOG.info("Table " + tableName + (disabled ? " disabled;" : " disabling;") +
+        " skipping assign of " + region.getRegionNameAsString());
+      offlineDisabledRegion(region);
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Set region as OFFLINED up in zookeeper
+   * 
+   * @param state
+   * @param hijack
+   *          - true if needs to be hijacked and reassigned, false otherwise.
+   * @param regionAlreadyInTransitionException  
+   *          - true if we need to retry assignment because of RegionAlreadyInTransitionException.       
+   * @return the version of the offline node if setting of the OFFLINE node was
+   *         successful, -1 otherwise.
+   */
+  int setOfflineInZooKeeper(final RegionState state, boolean hijack,
+      boolean regionAlreadyInTransitionException) {
+    // In case of reassignment the current state in memory need not be
+    // OFFLINE. 
+    if (!hijack && !state.isClosed() && !state.isOffline()) {
+      if (!regionAlreadyInTransitionException ) {
+        String msg = "Unexpected state : " + state + " .. Cannot transit it to OFFLINE.";
+        this.master.abort(msg, new IllegalStateException(msg));
+        return -1;
+      } 
+      LOG.debug("Unexpected state : " + state
+          + " but retrying to assign because RegionAlreadyInTransitionException.");
+    }
+    boolean allowZNodeCreation = false;
+    // Under reassignment if the current state is PENDING_OPEN
+    // or OPENING then refresh the in-memory state to PENDING_OPEN. This is
+    // important because if the region was in 
+    // RS_OPENING state for a long time the master will try to force the znode
+    // to OFFLINE state meanwhile the RS could have opened the corresponding
+    // region and the state in znode will be RS_ZK_REGION_OPENED.
+    // For all other cases we can change the in-memory state to OFFLINE.
+    if (hijack &&
+        (state.getState().equals(RegionState.State.PENDING_OPEN) || 
+            state.getState().equals(RegionState.State.OPENING))) {
+      state.update(RegionState.State.PENDING_OPEN);
+      allowZNodeCreation = false;
+    } else {
+      state.update(RegionState.State.OFFLINE);
+      allowZNodeCreation = true;
+    }
+    int versionOfOfflineNode = -1;
+    try {
+      // get the version after setting the znode to OFFLINE
+      versionOfOfflineNode = ZKAssign.createOrForceNodeOffline(master.getZooKeeper(), 
+          state.getRegion(), this.master.getServerName(),
+          hijack, allowZNodeCreation);
+      if (versionOfOfflineNode == -1) {
+        LOG.warn("Attempted to create/force node into OFFLINE state before "
+            + "completing assignment but failed to do so for " + state);
+        return -1;
+      }
+    } catch (KeeperException e) {
+      master.abort("Unexpected ZK exception creating/setting node OFFLINE", e);
+      return -1;
+    }
+    return versionOfOfflineNode;
+  }
+
+  /**
+   * Set region as OFFLINED up in zookeeper asynchronously.
+   * @param state
+   * @return True if we succeeded, false otherwise (State was incorrect or failed
+   * updating zk).
+   */
+  boolean asyncSetOfflineInZooKeeper(final RegionState state,
+      final AsyncCallback.StringCallback cb, final Object ctx) {
+    if (!state.isClosed() && !state.isOffline()) {
+        new RuntimeException("Unexpected state trying to OFFLINE; " + state);
+      this.master.abort("Unexpected state trying to OFFLINE; " + state,
+        new IllegalStateException());
+      return false;
+    }
+    state.update(RegionState.State.OFFLINE);
+    try {
+      ZKAssign.asyncCreateNodeOffline(master.getZooKeeper(), state.getRegion(),
+        this.master.getServerName(), cb, ctx);
+    } catch (KeeperException e) {
+      // TODO: this error handling will never execute, as the callback is async.
+      if (e instanceof NodeExistsException) {
+        LOG.warn("Node for " + state.getRegion() + " already exists");
+      } else { 
+        master.abort("Unexpected ZK exception creating/setting node OFFLINE", e);
+      }
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * @param state
+   * @return Plan for passed state (If none currently, it creates one or
+   * if no servers to assign, it returns null).
+   */
+  RegionPlan getRegionPlan(final RegionState state,
+      final boolean forceNewPlan) {
+    return getRegionPlan(state, null, forceNewPlan);
+  }
+
+  /**
+   * @param state
+   * @param serverToExclude Server to exclude (we know its bad). Pass null if
+   * all servers are thought to be assignable.
+   * @param forceNewPlan If true, then if an existing plan exists, a new plan
+   * will be generated.
+   * @return Plan for passed state (If none currently, it creates one or
+   * if no servers to assign, it returns null).
+   */
+  RegionPlan getRegionPlan(final RegionState state,
+      final ServerName serverToExclude, final boolean forceNewPlan) {
+    // Pickup existing plan or make a new one
+    final String encodedName = state.getRegion().getEncodedName();
+    final List servers = this.serverManager.getOnlineServersList();
+    final List drainingServers = this.serverManager.getDrainingServersList();
+
+
+    if (serverToExclude != null) servers.remove(serverToExclude);
+
+    // Loop through the draining server list and remove them from the server
+    // list.
+    if (!drainingServers.isEmpty()) {
+      for (final ServerName server: drainingServers) {
+        LOG.debug("Removing draining server: " + server +
+            " from eligible server pool.");
+        servers.remove(server);
+      }
+    }
+
+    // Remove the deadNotExpired servers from the server list.
+    removeDeadNotExpiredServers(servers);
+
+
+
+    if (servers.isEmpty()) return null;
+
+    RegionPlan randomPlan = null;
+    boolean newPlan = false;
+    RegionPlan existingPlan = null;
+
+    synchronized (this.regionPlans) {
+      existingPlan = this.regionPlans.get(encodedName);
+
+      if (existingPlan != null && existingPlan.getDestination() != null) {
+        LOG.debug("Found an existing plan for " +
+            state.getRegion().getRegionNameAsString() +
+       " destination server is " + existingPlan.getDestination().toString());
+      }
+
+      if (forceNewPlan
+          || existingPlan == null
+          || existingPlan.getDestination() == null
+          || drainingServers.contains(existingPlan.getDestination())) {
+        newPlan = true;
+        randomPlan = new RegionPlan(state.getRegion(), null, balancer.randomAssignment(state
+          .getRegion(), servers));
+        this.regionPlans.put(encodedName, randomPlan);
+      }
+    }
+
+    if (newPlan) {
+      LOG.debug("No previous transition plan was found (or we are ignoring " +
+        "an existing plan) for " + state.getRegion().getRegionNameAsString() +
+        " so generated a random one; " + randomPlan + "; " +
+        serverManager.countOfRegionServers() +
+               " (online=" + serverManager.getOnlineServers().size() +
+               ", available=" + servers.size() + ") available servers");
+        return randomPlan;
+      }
+    LOG.debug("Using pre-existing plan for region " +
+               state.getRegion().getRegionNameAsString() + "; plan=" + existingPlan);
+      return existingPlan;
+  }
+
+  /**
+   * Loop through the deadNotExpired server list and remove them from the
+   * servers.
+   * @param servers
+   */
+  public void removeDeadNotExpiredServers(List servers) {
+    Set deadNotExpiredServers = this.serverManager
+        .getDeadNotExpiredServers();
+    if (!deadNotExpiredServers.isEmpty()) {
+      for (ServerName server : deadNotExpiredServers) {
+        LOG.debug("Removing dead but not expired server: " + server
+            + " from eligible server pool.");
+        servers.remove(server);
+      }
+    }
+  }
+
+  /**
+   * Unassign the list of regions. Configuration knobs:
+   * hbase.bulk.waitbetween.reopen indicates the number of milliseconds to
+   * wait before unassigning another region from this region server
+   *
+   * @param regions
+   * @throws InterruptedException
+   */
+  public void unassign(List regions) {
+    int waitTime = this.master.getConfiguration().getInt(
+        "hbase.bulk.waitbetween.reopen", 0);
+    for (HRegionInfo region : regions) {
+      if (isRegionInTransition(region) != null)
+        continue;
+      unassign(region, false);
+      while (isRegionInTransition(region) != null) {
+        try {
+          Thread.sleep(10);
+        } catch (InterruptedException e) {
+          // Do nothing, continue
+        }
+      }
+      if (waitTime > 0)
+        try {
+          Thread.sleep(waitTime);
+        } catch (InterruptedException e) {
+          // Do nothing, continue
+        }
+    }
+  }
+
+  /**
+   * Unassigns the specified region.
+   * 
        
+   * Updates the RegionState and sends the CLOSE RPC unless region is being
+   * split by regionserver; then the unassign fails (silently) because we
+   * presume the region being unassigned no longer exists (its been split out
+   * of existence). TODO: What to do if split fails and is rolled back and
+   * parent is revivified?
+   * 
        
+   * If a RegionPlan is already set, it will remain.
+   *
+   * @param region server to be unassigned
+   */
+  public void unassign(HRegionInfo region) {
+    unassign(region, false);
+  }
+
+  /**
+   * Unassigns the specified region.
+   * 
        
+   * Updates the RegionState and sends the CLOSE RPC unless region is being
+   * split by regionserver; then the unassign fails (silently) because we
+   * presume the region being unassigned no longer exists (its been split out
+   * of existence). TODO: What to do if split fails and is rolled back and
+   * parent is revivified?
+   * 
        
+   * If a RegionPlan is already set, it will remain.
+   *
+   * @param region server to be unassigned
+   * @param force if region should be closed even if already closing
+   */
+  public void unassign(HRegionInfo region, boolean force) {
+    // TODO: Method needs refactoring.  Ugly buried returns throughout.  Beware!
+    LOG.debug("Starting unassignment of region " +
+      region.getRegionNameAsString() + " (offlining)");
+
+    synchronized (this.regions) {
+      // Check if this region is currently assigned
+      if (!regions.containsKey(region)) {
+        LOG.debug("Attempted to unassign region " +
+          region.getRegionNameAsString() + " but it is not " +
+          "currently assigned anywhere");
+        return;
+      }
+    }
+    String encodedName = region.getEncodedName();
+    // Grab the state of this region and synchronize on it
+    RegionState state;
+    int versionOfClosingNode = -1;
+    synchronized (regionsInTransition) {
+      state = regionsInTransition.get(encodedName);
+      if (state == null) {
+         // Create the znode in CLOSING state
+        try {
+          versionOfClosingNode = ZKAssign.createNodeClosing(
+            master.getZooKeeper(), region, master.getServerName());
+          if (versionOfClosingNode == -1) {
+            LOG.debug("Attempting to unassign region " +
+                region.getRegionNameAsString() + " but ZK closing node "
+                + "can't be created.");
+            return;
+          }
+        } catch (KeeperException e) {
+          if (e instanceof NodeExistsException) {
+            // Handle race between master initiated close and regionserver
+            // orchestrated splitting. See if existing node is in a
+            // SPLITTING or SPLIT state.  If so, the regionserver started
+            // an op on node before we could get our CLOSING in.  Deal.
+            NodeExistsException nee = (NodeExistsException)e;
+            String path = nee.getPath();
+            try {
+              if (isSplitOrSplitting(path)) {
+                LOG.debug(path + " is SPLIT or SPLITTING; " +
+                  "skipping unassign because region no longer exists -- its split");
+                return;
+              }
+            } catch (KeeperException.NoNodeException ke) {
+              LOG.warn("Failed getData on SPLITTING/SPLIT at " + path +
+                "; presuming split and that the region to unassign, " +
+                encodedName + ", no longer exists -- confirm", ke);
+              return;
+            } catch (KeeperException ke) {
+              LOG.error("Unexpected zk state", ke);
+              ke = e;
+            }
+          }
+          // If we get here, don't understand whats going on -- abort.
+          master.abort("Unexpected ZK exception creating node CLOSING", e);
+          return;
+        }
+        state = new RegionState(region, RegionState.State.PENDING_CLOSE);
+        regionsInTransition.put(encodedName, state);
+      } else if (force && (state.isPendingClose() || state.isClosing())) {
+        LOG.debug("Attempting to unassign region " + region.getRegionNameAsString() + 
+          " which is already " + state.getState()  + 
+          " but forcing to send a CLOSE RPC again ");
+        state.update(state.getState());
+      } else {
+        LOG.debug("Attempting to unassign region " +
+          region.getRegionNameAsString() + " but it is " +
+          "already in transition (" + state.getState() + ", force=" + force + ")");
+        return;
+      }
+    } 
+    // Send CLOSE RPC
+    ServerName server = null;
+    synchronized (this.regions) {
+      server = regions.get(region);
+    }
+    // ClosedRegionhandler can remove the server from this.regions
+    if (server == null) {
+      // Possibility of disable flow removing from RIT.
+      synchronized (regionsInTransition) {
+        state = regionsInTransition.get(encodedName);
+        if (state != null) {
+          // remove only if the state is PENDING_CLOSE or CLOSING
+          State presentState = state.getState();
+          if (presentState == State.PENDING_CLOSE
+              || presentState == State.CLOSING) {
+            this.regionsInTransition.remove(encodedName);
+          }
+        }
+      }
+      // delete the node. if no node exists need not bother.
+      deleteClosingOrClosedNode(region);
+      return;
+    }
+    try {
+      // TODO: We should consider making this look more like it does for the
+      // region open where we catch all throwables and never abort
+      if (serverManager.sendRegionClose(server, state.getRegion(),
+        versionOfClosingNode)) {
+        LOG.debug("Sent CLOSE to " + server + " for region " +
+          region.getRegionNameAsString());
+        return;
+      }
+      // This never happens. Currently regionserver close always return true.
+      LOG.warn("Server " + server + " region CLOSE RPC returned false for " +
+        region.getRegionNameAsString());
+    } catch (NotServingRegionException nsre) {
+      LOG.info("Server " + server + " returned " + nsre + " for " +
+        region.getRegionNameAsString());
+      // Presume that master has stale data.  Presume remote side just split.
+      // Presume that the split message when it comes in will fix up the master's
+      // in memory cluster state.
+    } catch (Throwable t) {
+      if (t instanceof RemoteException) {
+        t = ((RemoteException)t).unwrapRemoteException();
+        if (t instanceof NotServingRegionException) {
+          if (checkIfRegionBelongsToDisabling(region)
+              || checkIfRegionBelongsToDisabled(region)) {
+            // Remove from the regionsinTransition map
+            LOG.info("While trying to recover the table "
+                + region.getTableNameAsString()
+                + " to DISABLED state the region " + region
+                + " was offlined but the table was in DISABLING state");
+            synchronized (this.regionsInTransition) {
+              this.regionsInTransition.remove(region.getEncodedName());
+            }
+            // Remove from the regionsMap
+            synchronized (this.regions) {
+              this.regions.remove(region);
+            }
+            deleteClosingOrClosedNode(region);
+          }
+        }
+        // RS is already processing this region, only need to update the timestamp
+        if (t instanceof RegionAlreadyInTransitionException) {
+          LOG.debug("update " + state + " the timestamp.");
+          state.update(state.getState());
+        }
+      }
+      LOG.info("Server " + server + " returned " + t + " for " +
+        region.getEncodedName());
+      // Presume retry or server will expire.
+    }
+  }
+  
+  /**
+   * 
+   * @param region regioninfo of znode to be deleted.
+   */
+  public void deleteClosingOrClosedNode(HRegionInfo region) {
+    try {
+      if (!ZKAssign.deleteNode(master.getZooKeeper(), region.getEncodedName(),
+          EventHandler.EventType.M_ZK_REGION_CLOSING)) {
+        boolean deleteNode = ZKAssign.deleteNode(master.getZooKeeper(), region
+            .getEncodedName(), EventHandler.EventType.RS_ZK_REGION_CLOSED);
+        // TODO : We don't abort if the delete node returns false. Is there any
+        // such corner case?
+        if (!deleteNode) {
+          LOG.error("The deletion of the CLOSED node for the region "
+              + region.getEncodedName() + " returned " + deleteNode);
+        }
+      }
+    } catch (NoNodeException e) {
+      LOG.debug("CLOSING/CLOSED node for the region " + region.getEncodedName()
+          + " already deleted");
+    } catch (KeeperException ke) {
+      master.abort(
+          "Unexpected ZK exception deleting node CLOSING/CLOSED for the region "
+              + region.getEncodedName(), ke);
+      return;
+    }
+  }
+
+  /**
+   * @param path
+   * @return True if znode is in SPLIT or SPLITTING state.
+   * @throws KeeperException Can happen if the znode went away in meantime.
+   */
+  private boolean isSplitOrSplitting(final String path) throws KeeperException {
+    boolean result = false;
+    // This may fail if the SPLIT or SPLITTING znode gets cleaned up before we
+    // can get data from it.
+    RegionTransitionData data = ZKAssign.getData(master.getZooKeeper(), path);
+    EventType evt = data.getEventType();
+    switch (evt) {
+    case RS_ZK_REGION_SPLIT:
+    case RS_ZK_REGION_SPLITTING:
+      result = true;
+      break;
+    default:
+      break;
+    }
+    return result;
+  }
+
+  /**
+   * Waits until the specified region has completed assignment.
+   * 
        
+   * If the region is already assigned, returns immediately.  Otherwise, method
+   * blocks until the region is assigned.
+   * @param regionInfo region to wait on assignment for
+   * @throws InterruptedException
+   */
+  public void waitForAssignment(HRegionInfo regionInfo)
+  throws InterruptedException {
+    synchronized(regions) {
+      while (!this.master.isStopped() && !regions.containsKey(regionInfo)) {
+        // We should receive a notification, but it's
+        //  better to have a timeout to recheck the condition here:
+        //  it lowers the impact of a race condition if any
+        regions.wait(100);
+      }
+    }
+  }
+
+  /**
+   * Waits until the specified region has completed assignment or timeout elapsed.
+   * 
        
+   * If the region is already assigned, returns immediately. Otherwise, method blocks until the
+   * region is assigned.
+   * @param regionInfo region to wait on assignment for
+   * @param timeout How long to wait.
+   * @throws InterruptedException
+   */
+  public void waitForAssignment(HRegionInfo regionInfo, long timeout) throws InterruptedException {
+    synchronized (regions) {
+      long startTime = System.currentTimeMillis();
+      long remaining = timeout;
+      while (!regions.containsKey(regionInfo) && remaining > 0) {
+        // We should receive a notification, but it's
+        // better to have a timeout to recheck the condition here:
+        // it lowers the impact of a race condition if any
+        regions.wait(remaining);
+        remaining = timeout - (System.currentTimeMillis() - startTime);
+      }
+    }
+  }
+
+  /**
+   * Assigns the ROOT region.
+   * 
        
+   * Assumes that ROOT is currently closed and is not being actively served by
+   * any RegionServer.
+   * 
        
+   * Forcibly unsets the current root region location in ZooKeeper and assigns
+   * ROOT to a random RegionServer.
+   * @throws KeeperException
+   */
+  public void assignRoot() throws KeeperException {
+    RootLocationEditor.deleteRootLocation(this.master.getZooKeeper());
+    assign(HRegionInfo.ROOT_REGIONINFO, true);
+  }
+
+  /**
+   * Assigns the META region.
+   * 
        
+   * Assumes that META is currently closed and is not being actively served by
+   * any RegionServer.
+   * 
        
+   * Forcibly assigns META to a random RegionServer.
+   */
+  public void assignMeta() {
+    // Force assignment to a random server
+    assign(HRegionInfo.FIRST_META_REGIONINFO, true);
+  }
+
+  /**
+   * Assigns all user regions to online servers. Use round-robin assignment.
+   * 
+   * @param regions
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public void assignUserRegionsToOnlineServers(List regions)
+      throws IOException,
+      InterruptedException {
+    List servers = this.serverManager.getOnlineServersList();
+    removeDeadNotExpiredServers(servers);
+    assignUserRegions(regions, servers);
+  }
+
+  /**
+   * Assigns all user regions, if any.  Used during cluster startup.
+   * 
        
+   * This is a synchronous call and will return once every region has been
+   * assigned.  If anything fails, an exception is thrown
+   * @throws InterruptedException
+   * @throws IOException
+   */
+  public void assignUserRegions(List regions, List servers)
+  throws IOException, InterruptedException {
+    if (regions == null)
+      return;
+    Map> bulkPlan = null;
+    // Generate a round-robin bulk assignment plan
+    bulkPlan = balancer.roundRobinAssignment(regions, servers);
+    LOG.info("Bulk assigning " + regions.size() + " region(s) round-robin across " +
+               servers.size() + " server(s)");
+    // Use fixed count thread pool assigning.
+    BulkAssigner ba = new StartupBulkAssigner(this.master, bulkPlan, this);
+    ba.bulkAssign();
+    LOG.info("Bulk assigning done");
+  }
+
+  private void setEnabledTable(HRegionInfo hri) {
+    String tableName = hri.getTableNameAsString();
+    boolean isTableEnabled = this.zkTable.isEnabledTable(tableName);
+    if (!isTableEnabled) {
+      setEnabledTable(tableName);
+    }    
+  }
+
+  /**
+   * Assigns all user regions, if any exist.  Used during cluster startup.
+   * 
        
+   * This is a synchronous call and will return once every region has been
+   * assigned.  If anything fails, an exception is thrown and the cluster
+   * should be shutdown.
+   * @throws InterruptedException
+   * @throws IOException
+   */
+  public void assignAllUserRegions() throws IOException, InterruptedException {
+    // Skip assignment for regions of tables in DISABLING state also because
+    // during clean cluster startup no RS is alive and regions map also doesn't
+    // have any information about the regions. See HBASE-6281.
+    Set disablingDisabledAndEnablingTables = new HashSet(this.disablingTables);
+    disablingDisabledAndEnablingTables.addAll(this.zkTable.getDisabledTables());
+    disablingDisabledAndEnablingTables.addAll(this.enablingTables.keySet());
+    // Scan META for all user regions, skipping any disabled tables
+    Map allRegions = MetaReader.fullScan(catalogTracker,
+        disablingDisabledAndEnablingTables, true);
+    if (allRegions == null || allRegions.isEmpty()) return;
+
+    // Get all available servers
+    List servers = serverManager.getOnlineServersList();
+
+    // Remove the deadNotExpired servers from the server list.
+    removeDeadNotExpiredServers(servers);
+
+    // If there are no servers we need not proceed with region assignment.
+    if(servers.isEmpty()) return;
+
+    // Determine what type of assignment to do on startup
+    boolean retainAssignment = master.getConfiguration().
+      getBoolean("hbase.master.startup.retainassign", true);
+
+    Map> bulkPlan = null;
+    if (retainAssignment) {
+      // Reuse existing assignment info
+      bulkPlan = balancer.retainAssignment(allRegions, servers);
+    } else {
+      // assign regions in round-robin fashion
+      assignUserRegions(new ArrayList(allRegions.keySet()), servers);
+      for (HRegionInfo hri : allRegions.keySet()) {
+        setEnabledTable(hri);
+      }
+      return;
+    }
+    LOG.info("Bulk assigning " + allRegions.size() + " region(s) across " +
+      servers.size() + " server(s), retainAssignment=" + retainAssignment);
+
+    // Use fixed count thread pool assigning.
+    BulkAssigner ba = new StartupBulkAssigner(this.master, bulkPlan, this);
+    ba.bulkAssign();
+    for (HRegionInfo hri : allRegions.keySet()) {
+      setEnabledTable(hri);
+    }
+    LOG.info("Bulk assigning done");
+  }
+
+  /**
+   * Run bulk assign on startup.  Does one RCP per regionserver passing a
+   * batch of reginons using {@link SingleServerBulkAssigner}.
+   * Uses default {@link #getUncaughtExceptionHandler()}
+   * which will abort the Server if exception.
+   */
+  static class StartupBulkAssigner extends BulkAssigner {
+    final Map> bulkPlan;
+    final AssignmentManager assignmentManager;
+
+    StartupBulkAssigner(final Server server,
+        final Map> bulkPlan,
+        final AssignmentManager am) {
+      super(server);
+      this.bulkPlan = bulkPlan;
+      this.assignmentManager = am;
+    }
+
+    @Override
+    public boolean bulkAssign(boolean sync) throws InterruptedException,
+        IOException {
+      // Disable timing out regions in transition up in zk while bulk assigning.
+      this.assignmentManager.timeoutMonitor.bulkAssign(true);
+      try {
+        return super.bulkAssign(sync);
+      } finally {
+        // Reenable timing out regions in transition up in zi.
+        this.assignmentManager.timeoutMonitor.bulkAssign(false);
+      }
+    }
+
+    @Override
+    protected String getThreadNamePrefix() {
+      return this.server.getServerName() + "-StartupBulkAssigner";
+    }
+
+    @Override
+    protected void populatePool(java.util.concurrent.ExecutorService pool) {
+      for (Map.Entry> e: this.bulkPlan.entrySet()) {
+        pool.execute(new SingleServerBulkAssigner(e.getKey(), e.getValue(),
+          this.assignmentManager));
+      }
+    }
+
+    protected boolean waitUntilDone(final long timeout)
+    throws InterruptedException {
+      Set regionSet = new HashSet();
+      for (List regionList : bulkPlan.values()) {
+        regionSet.addAll(regionList);
+      }
+      return this.assignmentManager.waitUntilNoRegionsInTransition(timeout, regionSet);
+    }
+
+    @Override
+    protected long getTimeoutOnRIT() {
+      // Guess timeout.  Multiply the number of regions on a random server
+      // by how long we thing one region takes opening.
+      long perRegionOpenTimeGuesstimate =
+        this.server.getConfiguration().getLong("hbase.bulk.assignment.perregion.open.time", 1000);
+      int regionsPerServer =
+        this.bulkPlan.entrySet().iterator().next().getValue().size();
+      long timeout = perRegionOpenTimeGuesstimate * regionsPerServer;
+      LOG.debug("Timeout-on-RIT=" + timeout);
+      return timeout;
+    }
+  }
+
+  /**
+   * Bulk user region assigner.
+   * If failed assign, lets timeout in RIT do cleanup.
+   */
+  static class GeneralBulkAssigner extends StartupBulkAssigner {
+    GeneralBulkAssigner(final Server server,
+        final Map> bulkPlan,
+        final AssignmentManager am) {
+      super(server, bulkPlan, am);
+    }
+
+    @Override
+    protected UncaughtExceptionHandler getUncaughtExceptionHandler() {
+      return new UncaughtExceptionHandler() {
+        @Override
+        public void uncaughtException(Thread t, Throwable e) {
+          LOG.warn("Assigning regions in " + t.getName(), e);
+        }
+      };
+    }
+  }
+
+  /**
+   * Manage bulk assigning to a server.
+   */
+  static class SingleServerBulkAssigner implements Runnable {
+    private final ServerName regionserver;
+    private final List regions;
+    private final AssignmentManager assignmentManager;
+
+    SingleServerBulkAssigner(final ServerName regionserver,
+        final List regions, final AssignmentManager am) {
+      this.regionserver = regionserver;
+      this.regions = regions;
+      this.assignmentManager = am;
+    }
+    @Override
+    public void run() {
+      this.assignmentManager.assign(this.regionserver, this.regions);
+    }
+  }
+
+  /**
+   * Wait until no regions in transition.
+   * @param timeout How long to wait.
+   * @return True if nothing in regions in transition.
+   * @throws InterruptedException
+   */
+  public boolean waitUntilNoRegionsInTransition(final long timeout)
+  throws InterruptedException {
+    // Blocks until there are no regions in transition. It is possible that
+    // there
+    // are regions in transition immediately after this returns but guarantees
+    // that if it returns without an exception that there was a period of time
+    // with no regions in transition from the point-of-view of the in-memory
+    // state of the Master.
+    long startTime = System.currentTimeMillis();
+    long remaining = timeout;
+    synchronized (regionsInTransition) {
+      while (regionsInTransition.size() > 0 && !this.master.isStopped()
+          && remaining > 0) {
+        regionsInTransition.wait(remaining);
+        remaining = timeout - (System.currentTimeMillis() - startTime);
+      }
+    }
+    return regionsInTransition.isEmpty();
+  }
+
+  /**
+   * Wait until no regions from set regions are in transition.
+   * @param timeout How long to wait.
+   * @param regions set of regions to wait for
+   * @return True if nothing in regions in transition.
+   * @throws InterruptedException
+   */
+  public boolean waitUntilNoRegionsInTransition(final long timeout, Set regions)
+  throws InterruptedException {
+    // Blocks until there are no regions in transition.
+    long startTime = System.currentTimeMillis();
+    long remaining = timeout;
+    boolean stillInTransition = true;
+    synchronized (regionsInTransition) {
+      while (regionsInTransition.size() > 0 && !this.master.isStopped() &&
+          remaining > 0 && stillInTransition) {
+        int count = 0;
+        for (RegionState rs : regionsInTransition.values()) {
+          if (regions.contains(rs.getRegion())) {
+            count++;
+            break;
+          }
+        }
+        if (count == 0) {
+          stillInTransition = false;
+          break;
+        }
+        regionsInTransition.wait(remaining);
+        remaining = timeout - (System.currentTimeMillis() - startTime);
+      }
+    }
+    return stillInTransition;
+  }
+
+  /**
+   * Rebuild the list of user regions and assignment information.
+   * 
        
+   * Returns a map of servers that are not found to be online and the regions
+   * they were hosting.
+   * @return map of servers not online to their assigned regions, as stored
+   *         in META
+   * @throws IOException
+   */
+  Map>> rebuildUserRegions() throws IOException,
+      KeeperException {
+    // Region assignment from META
+    List results = MetaReader.fullScan(this.catalogTracker);
+    // Get any new but slow to checkin region server that joined the cluster
+    Set onlineServers = serverManager.getOnlineServers().keySet();    
+    // Map of offline servers and their regions to be returned
+    Map>> offlineServers =
+      new TreeMap>>();
+    // Iterate regions in META
+    for (Result result : results) {
+      boolean disabled = false;
+      boolean disablingOrEnabling = false;
+      Pair region = MetaReader.parseCatalogResult(result);
+      if (region == null) continue;
+      HRegionInfo regionInfo = region.getFirst();
+      ServerName regionLocation = region.getSecond();
+      if (regionInfo == null) continue;
+      String tableName = regionInfo.getTableNameAsString();
+      if (regionLocation == null) {
+        // regionLocation could be null if createTable didn't finish properly.
+        // When createTable is in progress, HMaster restarts.
+        // Some regions have been added to .META., but have not been assigned.
+        // When this happens, the region's table must be in ENABLING state.
+        // It can't be in ENABLED state as that is set when all regions are
+        // assigned.
+        // It can't be in DISABLING state, because DISABLING state transitions
+        // from ENABLED state when application calls disableTable.
+        // It can't be in DISABLED state, because DISABLED states transitions
+        // from DISABLING state.
+        boolean enabling = checkIfRegionsBelongsToEnabling(regionInfo);
+        addTheTablesInPartialState(regionInfo);
+        if (enabling) {
+          addToEnablingTableRegions(regionInfo);
+        } else {
+          LOG.warn("Region " + regionInfo.getEncodedName() + " has null regionLocation."
+              + " But its table " + tableName + " isn't in ENABLING state.");
+        }
+      } else if (!onlineServers.contains(regionLocation)) {
+        // Region is located on a server that isn't online
+        List> offlineRegions =
+          offlineServers.get(regionLocation);
+        if (offlineRegions == null) {
+          offlineRegions = new ArrayList>(1);
+          offlineServers.put(regionLocation, offlineRegions);
+        }
+        offlineRegions.add(new Pair(regionInfo, result));
+        disabled = checkIfRegionBelongsToDisabled(regionInfo);
+        disablingOrEnabling = addTheTablesInPartialState(regionInfo);
+        // need to enable the table if not disabled or disabling or enabling
+        // this will be used in rolling restarts
+        enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
+            disablingOrEnabling, tableName);
+      } else {
+        // If region is in offline and split state check the ZKNode
+        if (regionInfo.isOffline() && regionInfo.isSplit()) {
+          String node = ZKAssign.getNodeName(this.watcher, regionInfo
+              .getEncodedName());
+          Stat stat = new Stat();
+          byte[] data = ZKUtil.getDataNoWatch(this.watcher, node, stat);
+          // If znode does not exist dont consider this region
+          if (data == null) {
+            LOG.debug("Region "+ regionInfo.getRegionNameAsString() + " split is completed. " 
+                + "Hence need not add to regions list");
+            continue;
+          }
+        }
+        // Region is being served and on an active server
+        // add only if region not in disabled and enabling table
+        boolean enabling = checkIfRegionsBelongsToEnabling(regionInfo);
+        disabled = checkIfRegionBelongsToDisabled(regionInfo);
+        if (!enabling && !disabled) {
+          synchronized (this.regions) {
+            regions.put(regionInfo, regionLocation);
+            addToServers(regionLocation, regionInfo);
+          }
+          putRegionPlan(regionInfo, regionLocation);
+        }
+        disablingOrEnabling = addTheTablesInPartialState(regionInfo);
+        if (enabling) {
+          addToEnablingTableRegions(regionInfo);
+        }
+        // need to enable the table if not disabled or disabling or enabling
+        // this will be used in rolling restarts
+        enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
+            disablingOrEnabling, tableName);
+      }
+    }
+    return offlineServers;
+  }
+
+  private void addToEnablingTableRegions(HRegionInfo regionInfo) {
+    String tableName = regionInfo.getTableNameAsString();
+    List hris = this.enablingTables.get(tableName);
+    if (!hris.contains(regionInfo)) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Adding region" + regionInfo.getRegionNameAsString()
+            + " to enabling table " + tableName + ".");
+      }
+      hris.add(regionInfo);
+    }
+  }
+  
+  private void enableTableIfNotDisabledOrDisablingOrEnabling(boolean disabled,
+      boolean disablingOrEnabling, String tableName) {
+    if (!disabled && !disablingOrEnabling
+        && !getZKTable().isEnabledTable(tableName)) {
+      setEnabledTable(tableName);
+    }
+  }
+
+  private Boolean addTheTablesInPartialState(HRegionInfo regionInfo) {
+    String tableName = regionInfo.getTableNameAsString();
+    if (checkIfRegionBelongsToDisabling(regionInfo)) {
+      this.disablingTables.add(tableName);
+      return true;
+    } else if (checkIfRegionsBelongsToEnabling(regionInfo)) {
+      if (!this.enablingTables.containsKey(tableName)) {
+        this.enablingTables.put(tableName, new ArrayList());
+      } 
+      return true;
+    } 
+    return false;
+  }
+
+  /**
+   * Recover the tables that were not fully moved to DISABLED state. These
+   * tables are in DISABLING state when the master restarted/switched.
+   * 
+   * @param disablingTables
+   * @return
+   * @throws KeeperException
+   * @throws TableNotFoundException
+   * @throws IOException
+   */
+  private boolean recoverTableInDisablingState(Set disablingTables)
+      throws KeeperException, TableNotFoundException, IOException {
+    boolean isWatcherCreated = false;
+    if (disablingTables.size() != 0) {
+      // Create a watcher on the zookeeper node
+      ZKUtil.listChildrenAndWatchForNewChildren(watcher,
+          watcher.assignmentZNode);
+      isWatcherCreated = true;
+      for (String tableName : disablingTables) {
+        // Recover by calling DisableTableHandler
+        LOG.info("The table " + tableName
+            + " is in DISABLING state.  Hence recovering by moving the table"
+            + " to DISABLED state.");
+        new DisableTableHandler(this.master, tableName.getBytes(),
+            catalogTracker, this, true).process();
+      }
+    }
+    return isWatcherCreated;
+  }
+
+  /**
+   * Recover the tables that are not fully moved to ENABLED state. These tables
+   * are in ENABLING state when the master restarted/switched
+   * 
+   * @param enablingTables
+   * @param isWatcherCreated
+   * @throws KeeperException
+   * @throws TableNotFoundException
+   * @throws IOException
+   */
+  private void recoverTableInEnablingState(Set enablingTables,
+      boolean isWatcherCreated) throws KeeperException, TableNotFoundException,
+      IOException {
+    if (enablingTables.size() != 0) {
+      if (false == isWatcherCreated) {
+        ZKUtil.listChildrenAndWatchForNewChildren(watcher,
+            watcher.assignmentZNode);
+      }
+      for (String tableName : enablingTables) {
+        // Recover by calling EnableTableHandler
+        LOG.info("The table " + tableName
+            + " is in ENABLING state.  Hence recovering by moving the table"
+            + " to ENABLED state.");
+        // enableTable in sync way during master startup,
+        // no need to invoke coprocessor
+        new EnableTableHandler(this.master, tableName.getBytes(),
+            catalogTracker, this, true).process();
+      }
+    }
+  }
+
+  private boolean checkIfRegionsBelongsToEnabling(HRegionInfo regionInfo) {
+    String tableName = regionInfo.getTableNameAsString();
+    return getZKTable().isEnablingTable(tableName);
+  }
+
+  private boolean checkIfRegionBelongsToDisabled(HRegionInfo regionInfo) {
+    String tableName = regionInfo.getTableNameAsString();
+    return getZKTable().isDisabledTable(tableName);
+  }
+
+  private boolean checkIfRegionBelongsToDisabling(HRegionInfo regionInfo) {
+    String tableName = regionInfo.getTableNameAsString();
+    return getZKTable().isDisablingTable(tableName);
+  }
+
+  /**
+   * Processes list of dead servers from result of META scan and regions in RIT
+   * 
        
+   * This is used for failover to recover the lost regions that belonged to
+   * RegionServers which failed while there was no active master or regions 
+   * that were in RIT.
+   * 
        
+   * 
+   * @param deadServers
+   *          The list of dead servers which failed while there was no active
+   *          master. Can be null.
+   * @param nodes
+   *          The regions in RIT
+   * @throws IOException
+   * @throws KeeperException
+   */
+  private void processDeadServersAndRecoverLostRegions(
+      Map>> deadServers,
+      List nodes) throws IOException, KeeperException {
+    if (null != deadServers) {
+      Set actualDeadServers = this.serverManager.getDeadServers();
+      for (Map.Entry>> deadServer : 
+        deadServers.entrySet()) {
+        // skip regions of dead servers because SSH will process regions during rs expiration.
+        // see HBASE-5916
+        if (actualDeadServers.contains(deadServer.getKey())) {
+          for (Pair deadRegion : deadServer.getValue()) {
+            HRegionInfo hri = deadRegion.getFirst();
+            // Delete znode of region in transition if table is disabled or disabling. If a region
+            // server went down during master initialization then SSH cannot handle the regions of
+            // partially disabled tables because in memory region state information may not be
+            // available with master.
+            deleteNodeAndOfflineRegion(hri);
+            nodes.remove(deadRegion.getFirst().getEncodedName());
+          }
+          continue;
+        }
+        List> regions = deadServer.getValue();
+        for (Pair region : regions) {
+          HRegionInfo regionInfo = region.getFirst();
+          Result result = region.getSecond();
+          // If region was in transition (was in zk) force it offline for
+          // reassign
+          try {
+            RegionTransitionData data = ZKAssign.getData(watcher,
+                regionInfo.getEncodedName());
+
+            // If zk node of this region has been updated by a live server,
+            // we consider that this region is being handled.
+            // So we should skip it and process it in
+            // processRegionsInTransition.
+            if (data != null && data.getOrigin() != null && 
+                serverManager.isServerOnline(data.getOrigin())) {
+              LOG.info("The region " + regionInfo.getEncodedName()
+                  + "is being handled on " + data.getOrigin());
+              continue;
+            }
+            // Process with existing RS shutdown code
+            boolean assign = ServerShutdownHandler.processDeadRegion(
+                regionInfo, result, this, this.catalogTracker);
+            if (assign) {
+              ZKAssign.createOrForceNodeOffline(watcher, regionInfo,
+                  master.getServerName());
+              if (!nodes.contains(regionInfo.getEncodedName())) {
+                nodes.add(regionInfo.getEncodedName());
+              }
+            }
+          } catch (KeeperException.NoNodeException nne) {
+            // This is fine
+          }
+        }
+      }
+    }
+
+    if (!nodes.isEmpty()) {
+      for (String encodedRegionName : nodes) {
+        processRegionInTransition(encodedRegionName, null, deadServers);
+      }
+    }
+  }
+
+  /**
+   * Delete znode of region in transition if table is disabling/disabled and offline the region.
+   * @param hri region to offline.
+   */
+  public void deleteNodeAndOfflineRegion(HRegionInfo hri) {
+    if (zkTable.isDisablingOrDisabledTable(hri.getTableNameAsString())) {
+      try {
+        // If table is partially disabled then delete znode if exists in any state.
+        ZKAssign.deleteNodeFailSilent(this.master.getZooKeeper(), hri);
+      } catch (KeeperException ke) {
+        this.master.abort("Unexpected ZK exception deleting unassigned node " + hri, ke);
+      }
+      regionOffline(hri);
+    }
+  }
+
+  /*
+   * Presumes caller has taken care of necessary locking modifying servers Map.
+   * @param hsi
+   * @param hri
+   */
+  private void addToServers(final ServerName sn, final HRegionInfo hri) {
+    Set hris = servers.get(sn);
+    if (hris == null) {
+      hris = new ConcurrentSkipListSet();
+      servers.put(sn, hris);
+    }
+    if (!hris.contains(hri)) hris.add(hri);
+  }
+
+  /**
+   * @return A copy of the Map of regions currently in transition.
+   */
+  public NavigableMap getRegionsInTransition() {
+    synchronized (this.regionsInTransition) {
+      return new TreeMap(this.regionsInTransition);
+    }
+  }
+
+  /**
+   * @return True if regions in transition.
+   */
+  public boolean isRegionsInTransition() {
+    synchronized (this.regionsInTransition) {
+      return !this.regionsInTransition.isEmpty();
+    }
+  }
+
+  /**
+   * @param hri Region to check.
+   * @return Returns null if passed region is not in transition else the current
+   * RegionState
+   */
+  public RegionState isRegionInTransition(final HRegionInfo hri) {
+    synchronized (this.regionsInTransition) {
+      return this.regionsInTransition.get(hri.getEncodedName());
+    }
+  }
+
+  /**
+   * Clears the specified region from being in transition.
+   * 
        
+   * @param hri Region to remove.
+   * @deprecated This is a dupe of {@link #regionOffline(HRegionInfo)}.
+   *   Please use that method instead.
+   */
+  public void clearRegionFromTransition(HRegionInfo hri) {
+    synchronized (this.regionsInTransition) {
+      this.regionsInTransition.remove(hri.getEncodedName());
+    }
+    synchronized (this.regions) {
+      this.regions.remove(hri);
+      for (Set regions : this.servers.values()) {
+        regions.remove(hri);
+      }
+    }
+    clearRegionPlan(hri);
+  }
+
+  /**
+   * @param region Region whose plan we are to clear.
+   */
+  void clearRegionPlan(final HRegionInfo region) {
+    synchronized (this.regionPlans) {
+      this.regionPlans.remove(region.getEncodedName());
+    }
+  }
+
+  /**
+   * Wait on region to clear regions-in-transition.
+   * @param hri Region to wait on.
+   * @throws IOException
+   */
+  public void waitOnRegionToClearRegionsInTransition(final HRegionInfo hri)
+  throws IOException {
+    if (isRegionInTransition(hri) == null) return;
+    RegionState rs = null;
+    // There is already a timeout monitor on regions in transition so I
+    // should not have to have one here too?
+    while(!this.master.isStopped() && (rs = isRegionInTransition(hri)) != null) {
+      Threads.sleep(1000);
+      LOG.info("Waiting on " + rs + " to clear regions-in-transition");
+    }
+    if (this.master.isStopped()) {
+      LOG.info("Giving up wait on regions in " +
+        "transition because stoppable.isStopped is set");
+    }
+  }
+
+
+  /**
+   * Gets the online regions of the specified table.
+   * This method looks at the in-memory state.  It does not go to .META..
+   * Only returns online regions.  If a region on this table has been
+   * closed during a disable, etc., it will be included in the returned list.
+   * So, the returned list may not necessarily be ALL regions in this table, its
+   * all the ONLINE regions in the table.
+   * @param tableName
+   * @return Online regions from tableName
+   */
+  public List getRegionsOfTable(byte[] tableName) {
+    List tableRegions = new ArrayList();
+    // boundary needs to have table's name but regionID 0 so that it is sorted 
+    // before all table's regions.
+    HRegionInfo boundary =
+      new HRegionInfo(tableName, null, null, false, 0L);
+    synchronized (this.regions) {
+      for (HRegionInfo regionInfo: this.regions.tailMap(boundary).keySet()) {
+        if(Bytes.equals(regionInfo.getTableName(), tableName)) {
+          tableRegions.add(regionInfo);
+        } else {
+          break;
+        }
+      }
+    }
+    return tableRegions;
+  }
+
+  /**
+   * Update timers for all regions in transition going against the server in the
+   * serversInUpdatingTimer.
+   */
+  public class TimerUpdater extends Chore {
+
+    public TimerUpdater(final int period, final Stoppable stopper) {
+      super("AssignmentTimerUpdater", period, stopper);
+    }
+
+    @Override
+    protected void chore() {
+      ServerName serverToUpdateTimer = null;
+      while (!serversInUpdatingTimer.isEmpty() && !stopper.isStopped()) {
+        if (serverToUpdateTimer == null) {
+          serverToUpdateTimer = serversInUpdatingTimer.first();
+        } else {
+          serverToUpdateTimer = serversInUpdatingTimer
+              .higher(serverToUpdateTimer);
+        }
+        if (serverToUpdateTimer == null) {
+          break;
+        }
+        updateTimers(serverToUpdateTimer);
+        serversInUpdatingTimer.remove(serverToUpdateTimer);
+      }
+    }
+  }
+
+  /**
+   * Monitor to check for time outs on region transition operations
+   */
+  public class TimeoutMonitor extends Chore {
+    private final int timeout;
+    private boolean bulkAssign = false;
+    private boolean allRegionServersOffline = false;
+    private ServerManager serverManager;
+
+    /**
+     * Creates a periodic monitor to check for time outs on region transition
+     * operations.  This will deal with retries if for some reason something
+     * doesn't happen within the specified timeout.
+     * @param period
+   * @param stopper When {@link Stoppable#isStopped()} is true, this thread will
+   * cleanup and exit cleanly.
+     * @param timeout
+     */
+    public TimeoutMonitor(final int period, final Stoppable stopper,
+        ServerManager serverManager,
+        final int timeout) {
+      super("AssignmentTimeoutMonitor", period, stopper);
+      this.timeout = timeout;
+      this.serverManager = serverManager;
+    }
+
+    /**
+     * @param bulkAssign If true, we'll suspend checking regions in transition
+     * up in zookeeper.  If false, will reenable check.
+     * @return Old setting for bulkAssign.
+     */
+    public boolean bulkAssign(final boolean bulkAssign) {
+      boolean result = this.bulkAssign;
+      this.bulkAssign = bulkAssign;
+      return result;
+    }
+
+    private synchronized void setAllRegionServersOffline(
+      boolean allRegionServersOffline) {
+      this.allRegionServersOffline = allRegionServersOffline;
+    }
+
+    @Override
+    protected void chore() {
+      // If bulkAssign in progress, suspend checks
+      if (this.bulkAssign) return;
+      boolean allRSsOffline = this.serverManager.getOnlineServersList().
+        isEmpty();
+
+      synchronized (regionsInTransition) {
+        // Iterate all regions in transition checking for time outs
+        long now = System.currentTimeMillis();
+        for (RegionState regionState : regionsInTransition.values()) {
+          if (regionState.getStamp() + timeout <= now) {
+           //decide on action upon timeout
+            actOnTimeOut(regionState);
+          } else if (this.allRegionServersOffline && !allRSsOffline) {
+            RegionPlan existingPlan = regionPlans.get(regionState.getRegion().getEncodedName());
+            if (existingPlan == null
+                || !this.serverManager.isServerOnline(existingPlan.getDestination())) {
+              // if some RSs just came back online, we can start the
+              // the assignment right away
+              actOnTimeOut(regionState);
+            }
+          }
+        }
+      }
+      setAllRegionServersOffline(allRSsOffline);
+    }
+
+    private void actOnTimeOut(RegionState regionState) {
+      HRegionInfo regionInfo = regionState.getRegion();
+      LOG.info("Regions in transition timed out:  " + regionState);
+      // Expired! Do a retry.
+      switch (regionState.getState()) {
+      case CLOSED:
+        LOG.info("Region " + regionInfo.getEncodedName()
+            + " has been CLOSED for too long, waiting on queued "
+            + "ClosedRegionHandler to run or server shutdown");
+        // Update our timestamp.
+        regionState.updateTimestampToNow();
+        break;
+      case OFFLINE:
+        LOG.info("Region has been OFFLINE for too long, " + "reassigning "
+            + regionInfo.getRegionNameAsString() + " to a random server");
+        invokeAssign(regionInfo);
+        break;
+      case PENDING_OPEN:
+        LOG.info("Region has been PENDING_OPEN for too "
+            + "long, reassigning region=" + regionInfo.getRegionNameAsString());
+        invokeAssign(regionInfo);
+        break;
+      case OPENING:
+        processOpeningState(regionInfo);
+        break;
+      case OPEN:
+        LOG.error("Region has been OPEN for too long, " +
+            "we don't know where region was opened so can't do anything");
+        synchronized (regionState) {
+          regionState.updateTimestampToNow();
+        }
+        break;
+
+      case PENDING_CLOSE:
+        LOG.info("Region has been PENDING_CLOSE for too "
+            + "long, running forced unassign again on region="
+            + regionInfo.getRegionNameAsString());
+        invokeUnassign(regionInfo);
+        break;
+      case CLOSING:
+        LOG.info("Region has been CLOSING for too " +
+          "long, this should eventually complete or the server will " +
+          "expire, send RPC again");
+        invokeUnassign(regionInfo);
+        break;
+      }
+    }
+  }
+  
+  private void processOpeningState(HRegionInfo regionInfo) {
+    LOG.info("Region has been OPENING for too " + "long, reassigning region="
+        + regionInfo.getRegionNameAsString());
+    // Should have a ZK node in OPENING state
+    try {
+      String node = ZKAssign.getNodeName(watcher, regionInfo.getEncodedName());
+      Stat stat = new Stat();
+      RegionTransitionData dataInZNode = ZKAssign.getDataNoWatch(watcher, node,
+          stat);
+      if (dataInZNode == null) {
+        LOG.warn("Data is null, node " + node + " no longer exists");
+        return;
+      }
+      if (dataInZNode.getEventType() == EventType.RS_ZK_REGION_OPENED) {
+        LOG.debug("Region has transitioned to OPENED, allowing "
+            + "watched event handlers to process");
+        return;
+      } else if (dataInZNode.getEventType() != EventType.RS_ZK_REGION_OPENING &&
+          dataInZNode.getEventType() != EventType.RS_ZK_REGION_FAILED_OPEN ) {
+        LOG.warn("While timing out a region in state OPENING, "
+            + "found ZK node in unexpected state: "
+            + dataInZNode.getEventType());
+        return;
+      }
+      invokeAssign(regionInfo);
+    } catch (KeeperException ke) {
+      LOG.error("Unexpected ZK exception timing out CLOSING region", ke);
+      return;
+    }
+    return;
+  }
+
+  private void invokeAssign(HRegionInfo regionInfo) {
+    threadPoolExecutorService.submit(new AssignCallable(this, regionInfo));
+  }
+
+  private void invokeUnassign(HRegionInfo regionInfo) {
+    threadPoolExecutorService.submit(new UnAssignCallable(this, regionInfo));
+  }
+
+  public boolean isCarryingRoot(ServerName serverName) {
+    return isCarryingRegion(serverName, HRegionInfo.ROOT_REGIONINFO);
+  }
+
+  public boolean isCarryingMeta(ServerName serverName) {
+    return isCarryingRegion(serverName, HRegionInfo.FIRST_META_REGIONINFO);
+  }
+  /**
+   * Check if the shutdown server carries the specific region.
+   * We have a bunch of places that store region location
+   * Those values aren't consistent. There is a delay of notification.
+   * The location from zookeeper unassigned node has the most recent data;
+   * but the node could be deleted after the region is opened by AM.
+   * The AM's info could be old when OpenedRegionHandler
+   * processing hasn't finished yet when server shutdown occurs.
+   * @return whether the serverName currently hosts the region
+   */
+  public boolean isCarryingRegion(ServerName serverName, HRegionInfo hri) {
+    RegionTransitionData data = null;
+    try {
+      data = ZKAssign.getData(master.getZooKeeper(), hri.getEncodedName());
+    } catch (KeeperException e) {
+      master.abort("Unexpected ZK exception reading unassigned node for region="
+        + hri.getEncodedName(), e);
+    }
+
+    ServerName addressFromZK = (data != null && data.getOrigin() != null) ?
+      data.getOrigin() : null;
+    if (addressFromZK != null) {
+      // if we get something from ZK, we will use the data
+      boolean matchZK = (addressFromZK != null &&
+        addressFromZK.equals(serverName));
+      LOG.debug("based on ZK, current region=" + hri.getRegionNameAsString() +
+          " is on server=" + addressFromZK +
+          " server being checked=: " + serverName);
+      return matchZK;
+    }
+
+    ServerName addressFromAM = getRegionServerOfRegion(hri);
+    boolean matchAM = (addressFromAM != null &&
+      addressFromAM.equals(serverName));
+    LOG.debug("based on AM, current region=" + hri.getRegionNameAsString() +
+      " is on server=" + (addressFromAM != null ? addressFromAM : "null") +
+      " server being checked: " + serverName);
+
+    return matchAM;
+  }
+
+  /**
+   * Start processing of shutdown server.
+   * @param sn Server that went down.
+   * @return Pair that has a set of regions in transition TO the dead server and
+   * a list of regions that were in transition, and also ON this server.
+   */
+  public Pair, List> processServerShutdown(final ServerName sn) {
+    // Clean out any existing assignment plans for this server
+    synchronized (this.regionPlans) {
+      for (Iterator > i =
+          this.regionPlans.entrySet().iterator(); i.hasNext();) {
+        Map.Entry e = i.next();
+        ServerName otherSn = e.getValue().getDestination();
+        // The name will be null if the region is planned for a random assign.
+        if (otherSn != null && otherSn.equals(sn)) {
+          // Use iterator's remove else we'll get CME
+          i.remove();
+        }
+      }
+    }
+    // TODO: Do we want to sync on RIT here?
+    // Remove this server from map of servers to regions, and remove all regions
+    // of this server from online map of regions.
+    Set deadRegions = new TreeSet();
+    synchronized (this.regions) {
+      Set assignedRegions = this.servers.remove(sn);
+      if (assignedRegions != null && !assignedRegions.isEmpty()) {
+        deadRegions.addAll(assignedRegions);
+        for (HRegionInfo region : deadRegions) {
+          this.regions.remove(region);
+        }
+      }
+    }
+    // See if any of the regions that were online on this server were in RIT
+    // If they are, normal timeouts will deal with them appropriately so
+    // let's skip a manual re-assignment.
+    Set ritsGoingToServer = new ConcurrentSkipListSet();
+    List ritsOnServer = new ArrayList();
+    synchronized (regionsInTransition) {
+      for (RegionState state : this.regionsInTransition.values()) {
+        // If destination server in RegionState is same as dead server then add to regions to assign
+        // Skip the region in OFFLINE state because destionation server in RegionState is master
+        // server name. Skip the region if the destionation server in RegionState is other than dead
+        // server.
+        if ((state.getServerName() != null) && state.getServerName().equals(sn)) {
+          ritsGoingToServer.add(state.getRegion());
+        }
+        if (deadRegions.contains(state.getRegion())) {
+          ritsOnServer.add(state);
+        }
+      }
+    }
+    return new Pair, List>(ritsGoingToServer, ritsOnServer);
+  }
+
+  /**
+   * Update inmemory structures.
+   * @param sn Server that reported the split
+   * @param parent Parent region that was split
+   * @param a Daughter region A
+   * @param b Daughter region B
+   */
+  public void handleSplitReport(final ServerName sn, final HRegionInfo parent,
+      final HRegionInfo a, final HRegionInfo b) {
+    regionOffline(parent);
+    regionOnline(a, sn);
+    regionOnline(b, sn);
+
+    // There's a possibility that the region was splitting while a user asked
+    // the master to disable, we need to make sure we close those regions in
+    // that case. This is not racing with the region server itself since RS
+    // report is done after the split transaction completed.
+    if (this.zkTable.isDisablingOrDisabledTable(
+        parent.getTableNameAsString())) {
+      unassign(a);
+      unassign(b);
+    }
+  }
+
+  /**
+   * This is an EXPENSIVE clone.  Cloning though is the safest thing to do.
+   * Can't let out original since it can change and at least the loadbalancer
+   * wants to iterate this exported list.  We need to synchronize on regions
+   * since all access to this.servers is under a lock on this.regions.
+   * 
+   * @return A clone of current assignments by table.
+   */
+  Map>> getAssignmentsByTable() {
+    Map>> result = null;
+    synchronized (this.regions) {
+      result = new TreeMap>>();
+      if (!this.master.getConfiguration().
+          getBoolean("hbase.master.loadbalance.bytable", true)) {
+        result.put("ensemble", getAssignments());
+      } else {
+        for (Map.Entry> e: this.servers.entrySet()) {
+          for (HRegionInfo hri : e.getValue()) {
+            if (hri.isMetaRegion() || hri.isRootRegion()) continue;
+            String tablename = hri.getTableNameAsString();
+            Map> svrToRegions = result.get(tablename);
+            if (svrToRegions == null) {
+              svrToRegions = new HashMap>(this.servers.size());
+              result.put(tablename, svrToRegions);
+            }
+            List regions = null;
+            if (!svrToRegions.containsKey(e.getKey())) {
+              regions = new ArrayList();
+              svrToRegions.put(e.getKey(), regions);
+            } else {
+              regions = svrToRegions.get(e.getKey());
+            }
+            regions.add(hri);
+          }
+        }
+      }
+    }
+    Map onlineSvrs = this.serverManager.getOnlineServers();
+    // Take care of servers w/o assignments.
+    for (Map> map : result.values()) {
+      for (Map.Entry svrEntry: onlineSvrs.entrySet()) {
+        if (!map.containsKey(svrEntry.getKey())) {
+          map.put(svrEntry.getKey(), new ArrayList());
+        }
+      }
+    }
+    return result;
+  }
+  
+  /**
+   * @return A clone of current assignments. Note, this is assignments only.
+   * If a new server has come in and it has no regions, it will not be included
+   * in the returned Map.
+   */
+  Map> getAssignments() {
+    // This is an EXPENSIVE clone.  Cloning though is the safest thing to do.
+    // Can't let out original since it can change and at least the loadbalancer
+    // wants to iterate this exported list.  We need to synchronize on regions
+    // since all access to this.servers is under a lock on this.regions.
+    Map> result = null;
+    synchronized (this.regions) {
+      result = new HashMap>(this.servers.size());
+      for (Map.Entry> e: this.servers.entrySet()) {
+        result.put(e.getKey(), new ArrayList(e.getValue()));
+      }
+    }
+    return result;
+  }
+
+  /**
+   * @param encodedRegionName Region encoded name.
+   * @return Null or a {@link Pair} instance that holds the full {@link HRegionInfo}
+   * and the hosting servers {@link ServerName}.
+   */
+  Pair getAssignment(final byte [] encodedRegionName) {
+    String name = Bytes.toString(encodedRegionName);
+    synchronized(this.regions) {
+      for (Map.Entry e: this.regions.entrySet()) {
+        if (e.getKey().getEncodedName().equals(name)) {
+          return new Pair(e.getKey(), e.getValue());
+        }
+      }
+    }
+    return null;
+  }
+
+  /**
+   * @param plan Plan to execute.
+   */
+  void balance(final RegionPlan plan) {
+    synchronized (this.regionPlans) {
+      this.regionPlans.put(plan.getRegionName(), plan);
+    }
+    unassign(plan.getRegionInfo());
+  }
+
+  /**
+   * Run through remaining regionservers and unassign all catalog regions.
+   */
+  void unassignCatalogRegions() {
+    synchronized (this.regions) {
+      for (Map.Entry> e: this.servers.entrySet()) {
+        Set regions = e.getValue();
+        if (regions == null || regions.isEmpty()) continue;
+        for (HRegionInfo hri: regions) {
+          if (hri.isMetaRegion()) {
+            unassign(hri);
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * State of a Region while undergoing transitions.
+   */
+  public static class RegionState implements org.apache.hadoop.io.Writable {
+    private HRegionInfo region;
+
+    public enum State {
+      OFFLINE,        // region is in an offline state
+      PENDING_OPEN,   // sent rpc to server to open but has not begun
+      OPENING,        // server has begun to open but not yet done
+      OPEN,           // server opened region and updated meta
+      PENDING_CLOSE,  // sent rpc to server to close but has not begun
+      CLOSING,        // server has begun to close but not yet done
+      CLOSED,         // server closed region and updated meta
+      SPLITTING,      // server started split of a region
+      SPLIT           // server completed split of a region
+    }
+
+    private State state;
+    // Many threads can update the state at the stamp at the same time
+    private final AtomicLong stamp;
+    private ServerName serverName;
+
+    public RegionState() {
+      this.stamp = new AtomicLong(System.currentTimeMillis());
+    }
+
+    RegionState(HRegionInfo region, State state) {
+      this(region, state, System.currentTimeMillis(), null);
+    }
+
+    RegionState(HRegionInfo region, State state, long stamp, ServerName serverName) {
+      this.region = region;
+      this.state = state;
+      this.stamp = new AtomicLong(stamp);
+      this.serverName = serverName;
+    }
+
+    public void update(State state, long stamp, ServerName serverName) {
+      this.state = state;
+      updateTimestamp(stamp);
+      this.serverName = serverName;
+    }
+
+    public void update(State state) {
+      this.state = state;
+      updateTimestampToNow();
+      this.serverName = null;
+    }
+
+    public void updateTimestamp(long stamp) {
+      this.stamp.set(stamp);
+    }
+
+    public void updateTimestampToNow() {
+      this.stamp.set(System.currentTimeMillis());
+    }
+
+    public State getState() {
+      return state;
+    }
+
+    public long getStamp() {
+      return stamp.get();
+    }
+
+    public HRegionInfo getRegion() {
+      return region;
+    }
+
+    public ServerName getServerName() {
+      return serverName;
+    }
+
+    public boolean isClosing() {
+      return state == State.CLOSING;
+    }
+
+    public boolean isClosed() {
+      return state == State.CLOSED;
+    }
+
+    public boolean isPendingClose() {
+      return state == State.PENDING_CLOSE;
+    }
+
+    public boolean isOpening() {
+      return state == State.OPENING;
+    }
+
+    public boolean isOpened() {
+      return state == State.OPEN;
+    }
+
+    public boolean isPendingOpen() {
+      return state == State.PENDING_OPEN;
+    }
+
+    public boolean isOffline() {
+      return state == State.OFFLINE;
+    }
+
+    public boolean isSplitting() {
+      return state == State.SPLITTING;
+    }
+ 
+    public boolean isSplit() {
+      return state == State.SPLIT;
+    }
+
+    @Override
+    public String toString() {
+      return region.getRegionNameAsString()
+        + " state=" + state
+        + ", ts=" + stamp
+        + ", server=" + serverName;
+    }
+
+    /**
+     * A slower (but more easy-to-read) stringification 
+     */
+    public String toDescriptiveString() {
+      long lstamp = stamp.get();
+      long relTime = System.currentTimeMillis() - lstamp;
+      
+      return region.getRegionNameAsString()
+        + " state=" + state
+        + ", ts=" + new Date(lstamp) + " (" + (relTime/1000) + "s ago)"
+        + ", server=" + serverName;
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+      region = new HRegionInfo();
+      region.readFields(in);
+      state = State.valueOf(in.readUTF());
+      stamp.set(in.readLong());
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+      region.write(out);
+      out.writeUTF(state.name());
+      out.writeLong(stamp.get());
+    }
+  }
+  
+  public void stop() {
+    this.timeoutMonitor.interrupt();
+    this.timerUpdater.interrupt();
+  }
+  
+  /**
+   * Check whether the RegionServer is online.
+   * @param serverName 
+   * @return True if online.
+   */
+  public boolean isServerOnline(ServerName serverName) {
+    return this.serverManager.isServerOnline(serverName);
+  }
+  /**
+   * Shutdown the threadpool executor service
+   */
+  public void shutdown() {
+    if (null != threadPoolExecutorService) {
+      this.threadPoolExecutorService.shutdown();
+    }
+  }
+
+  protected void setEnabledTable(String tableName) {
+    try {
+      this.zkTable.setEnabledTable(tableName);
+    } catch (KeeperException e) {
+      // here we can abort as it is the start up flow
+      String errorMsg = "Unable to ensure that the table " + tableName
+          + " will be" + " enabled because of a ZooKeeper issue";
+      LOG.error(errorMsg);
+      this.master.abort(errorMsg, e);
+    }
+  }
+
+  protected void setDisabledTable(String tableName) {
+    try {
+      this.zkTable.setDisabledTable(tableName);
+    } catch (KeeperException e) {
+      // here we can abort as it is the start up flow
+      String errorMsg =
+          "Unable to ensure that the table " + tableName + " will be"
+              + " enabled because of a ZooKeeper issue";
+      LOG.error(errorMsg);
+      this.master.abort(errorMsg, e);
+    }
+  }
+
+  public void putRegionPlan(HRegionInfo hri, ServerName dest) {
+    try {
+      if (LoadBalancerFactory.secIndexLoadBalancerKlass.isInstance(this.balancer)) {
+        Method method =
+            LoadBalancerFactory.secIndexLoadBalancerKlass.getMethod("putRegionPlan",
+              HRegionInfo.class, ServerName.class);
+        method.invoke(this.balancer, hri, dest);
+      }
+    } catch (Throwable t) {
+      // Nothing to do
+    }
+  }
+
+  public void clearRegionPlanFromBalancerPlan(HRegionInfo hri) {
+    try {
+      if (LoadBalancerFactory.secIndexLoadBalancerKlass.isInstance(this.balancer)) {
+        Method method =
+            LoadBalancerFactory.secIndexLoadBalancerKlass.getMethod(
+              "clearRegionInfoFromRegionPlan", HRegionInfo.class);
+        method.invoke(this.balancer, hri);
+      }
+    } catch (Throwable t) {
+      // Nothing to do
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/BulkAssigner.java b/src/main/java/org/apache/hadoop/hbase/master/BulkAssigner.java
new file mode 100644
index 0000000..588cf35
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/BulkAssigner.java
@@ -0,0 +1,121 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.concurrent.Executors;
+
+import org.apache.hadoop.hbase.Server;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * Base class used bulk assigning and unassigning regions.
+ * Encapsulates a fixed size thread pool of executors to run assignment/unassignment.
+ * Implement {@link #populatePool(java.util.concurrent.ExecutorService)} and
+ * {@link #waitUntilDone(long)}.  The default implementation of
+ * the {@link #getUncaughtExceptionHandler()} is to abort the hosting
+ * Server.
+ */
+public abstract class BulkAssigner {
+  protected final Server server;
+
+  /**
+   * @param server An instance of Server
+   */
+  public BulkAssigner(final Server server) {
+    this.server = server;
+  }
+
+  /**
+   * @return What to use for a thread prefix when executor runs.
+   */
+  protected String getThreadNamePrefix() {
+    return this.server.getServerName() + "-" + this.getClass().getName(); 
+  }
+
+  protected UncaughtExceptionHandler getUncaughtExceptionHandler() {
+    return new UncaughtExceptionHandler() {
+      @Override
+      public void uncaughtException(Thread t, Throwable e) {
+        // Abort if exception of any kind.
+        server.abort("Uncaught exception in " + t.getName(), e);
+      }
+    };
+  }
+
+  protected int getThreadCount() {
+    return this.server.getConfiguration().
+      getInt("hbase.bulk.assignment.threadpool.size", 20);
+  }
+
+  protected long getTimeoutOnRIT() {
+    return this.server.getConfiguration().
+      getLong("hbase.bulk.assignment.waiton.empty.rit", 5 * 60 * 1000);
+  }
+
+  protected abstract void populatePool(
+      final java.util.concurrent.ExecutorService pool) throws IOException;
+
+  public boolean bulkAssign() throws InterruptedException, IOException {
+    return bulkAssign(true);
+  }
+
+  /**
+   * Run the bulk assign.
+   * 
+   * @param sync
+   *          Whether to assign synchronously.
+   * @throws InterruptedException
+   * @return True if done.
+   * @throws IOException
+   */
+  public boolean bulkAssign(boolean sync) throws InterruptedException,
+      IOException {
+    boolean result = false;
+    ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
+    builder.setDaemon(true);
+    builder.setNameFormat(getThreadNamePrefix() + "-%1$d");
+    builder.setUncaughtExceptionHandler(getUncaughtExceptionHandler());
+    int threadCount = getThreadCount();
+    java.util.concurrent.ExecutorService pool =
+      Executors.newFixedThreadPool(threadCount, builder.build());
+    try {
+      populatePool(pool);
+      // How long to wait on empty regions-in-transition.  If we timeout, the
+      // RIT monitor should do fixup.
+      if (sync) result = waitUntilDone(getTimeoutOnRIT());
+    } finally {
+      // We're done with the pool.  It'll exit when its done all in queue.
+      pool.shutdown();
+    }
+    return result;
+  }
+
+  /**
+   * Wait until bulk assign is done.
+   * @param timeout How long to wait.
+   * @throws InterruptedException
+   * @return True if the condition we were waiting on happened.
+   */
+  protected abstract boolean waitUntilDone(final long timeout)
+  throws InterruptedException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/BulkReOpen.java b/src/main/java/org/apache/hadoop/hbase/master/BulkReOpen.java
new file mode 100644
index 0000000..37e22cc
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/BulkReOpen.java
@@ -0,0 +1,100 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+
+/**
+ * Performs bulk reopen of the list of regions provided to it.
+ */
+public class BulkReOpen extends BulkAssigner {
+  private final Map> rsToRegions;
+  private final AssignmentManager assignmentManager;
+  private static final Log LOG = LogFactory.getLog(BulkReOpen.class);
+
+  public BulkReOpen(final Server server,
+      final Map> serverToRegions,
+    final AssignmentManager am) {
+    super(server);
+    this.assignmentManager = am;
+    this.rsToRegions = serverToRegions;
+  }
+
+  /**
+   * Unassign all regions, so that they go through the regular region
+   * assignment flow (in assignment manager) and are re-opened.
+   */
+  @Override
+  protected void populatePool(ExecutorService pool) {
+    LOG.debug("Creating threads for each region server ");
+    for (Map.Entry> e : rsToRegions
+        .entrySet()) {
+      final List hris = e.getValue();
+      // add plans for the regions that need to be reopened
+      Map plans = new HashMap();
+      for (HRegionInfo hri : hris) {
+        RegionPlan reOpenPlan = new RegionPlan(hri, null,
+            assignmentManager.getRegionServerOfRegion(hri));
+        plans.put(hri.getEncodedName(), reOpenPlan);
+      }
+      assignmentManager.addPlans(plans);
+      pool.execute(new Runnable() {
+        public void run() {
+          assignmentManager.unassign(hris);
+        }
+      });
+    }
+  }
+
+ /**
+  * Reopen the regions asynchronously, so always returns true immediately.
+  * @return true
+  */
+  @Override
+  protected boolean waitUntilDone(long timeout) {
+    return true;
+  }
+
+  /**
+   * Configuration knobs "hbase.bulk.reopen.threadpool.size" number of regions
+   * that can be reopened concurrently. The maximum number of threads the master
+   * creates is never more than the number of region servers.
+   * If configuration is not defined it defaults to 20
+   */
+  protected int getThreadCount() {
+    int defaultThreadCount = super.getThreadCount();
+    return this.server.getConfiguration().getInt(
+        "hbase.bulk.reopen.threadpool.size", defaultThreadCount);
+  }
+
+  public boolean bulkReOpen() throws InterruptedException, IOException {
+    return bulkAssign();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
new file mode 100644
index 0000000..ef6dad0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -0,0 +1,330 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.client.MetaScanner;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Writables;
+
+/**
+ * A janitor for the catalog tables.  Scans the .META. catalog
+ * table on a period looking for unused regions to garbage collect.
+ */
+class CatalogJanitor extends Chore {
+  private static final Log LOG = LogFactory.getLog(CatalogJanitor.class.getName());
+  private final Server server;
+  private final MasterServices services;
+  private boolean enabled = true;
+
+  CatalogJanitor(final Server server, final MasterServices services) {
+    super(server.getServerName() + "-CatalogJanitor",
+      server.getConfiguration().getInt("hbase.catalogjanitor.interval", 300000),
+      server);
+    this.server = server;
+    this.services = services;
+  }
+
+  @Override
+  protected boolean initialChore() {
+    try {
+      if (this.enabled) scan();
+    } catch (IOException e) {
+      LOG.warn("Failed initial scan of catalog table", e);
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * @param enabled
+   */
+  public void setEnabled(final boolean enabled) {
+    this.enabled = enabled;
+  }
+
+  @Override
+  protected void chore() {
+    try {
+      scan();
+    } catch (IOException e) {
+      LOG.warn("Failed scan of catalog table", e);
+    }
+  }
+
+  /**
+   * Scans META and returns a number of scanned rows, and
+   * an ordered map of split parents.
+   */
+  Pair> getSplitParents() throws IOException {
+    // TODO: Only works with single .META. region currently.  Fix.
+    final AtomicInteger count = new AtomicInteger(0);
+    // Keep Map of found split parents.  There are candidates for cleanup.
+    // Use a comparator that has split parents come before its daughters.
+    final Map splitParents =
+      new TreeMap(new SplitParentFirstComparator());
+    // This visitor collects split parents and counts rows in the .META. table
+
+    MetaScannerVisitor visitor = new MetaScanner.BlockingMetaScannerVisitor(server.getConfiguration()) {
+      @Override
+      public boolean processRowInternal(Result r) throws IOException {
+        if (r == null || r.isEmpty()) return true;
+        count.incrementAndGet();
+        HRegionInfo info = getHRegionInfo(r);
+        if (info == null) return true; // Keep scanning
+        if (info.isSplitParent()) splitParents.put(info, r);
+        // Returning true means "keep scanning"
+        return true;
+      }
+    };
+
+    // Run full scan of .META. catalog table passing in our custom visitor
+    MetaScanner.metaScan(server.getConfiguration(), visitor);
+
+    return new Pair>(count.get(), splitParents);
+  }
+
+  /**
+   * Run janitorial scan of catalog .META. table looking for
+   * garbage to collect.
+   * @throws IOException
+   */
+  int scan() throws IOException {
+    Pair> pair = getSplitParents();
+    int count = pair.getFirst();
+    Map splitParents = pair.getSecond();
+
+    // Now work on our list of found parents. See if any we can clean up.
+    int cleaned = 0;
+    HashSet parentNotCleaned = new HashSet(); //regions whose parents are still around
+    for (Map.Entry e : splitParents.entrySet()) {
+      if (!parentNotCleaned.contains(e.getKey().getEncodedName()) && cleanParent(e.getKey(), e.getValue())) {
+        cleaned++;
+      } else {
+        // We could not clean the parent, so it's daughters should not be cleaned either (HBASE-6160)
+        parentNotCleaned.add(getDaughterRegionInfo(
+              e.getValue(), HConstants.SPLITA_QUALIFIER).getEncodedName());
+        parentNotCleaned.add(getDaughterRegionInfo(
+              e.getValue(), HConstants.SPLITB_QUALIFIER).getEncodedName());
+      }
+    }
+    if (cleaned != 0) {
+      LOG.info("Scanned " + count + " catalog row(s) and gc'd " + cleaned +
+        " unreferenced parent region(s)");
+    } else if (LOG.isDebugEnabled()) {
+      LOG.debug("Scanned " + count + " catalog row(s) and gc'd " + cleaned +
+      " unreferenced parent region(s)");
+    }
+    return cleaned;
+  }
+
+  /**
+   * Compare HRegionInfos in a way that has split parents sort BEFORE their
+   * daughters.
+   */
+  static class SplitParentFirstComparator implements Comparator {
+    Comparator rowEndKeyComparator = new Bytes.RowEndKeyComparator();
+    @Override
+    public int compare(HRegionInfo left, HRegionInfo right) {
+      // This comparator differs from the one HRegionInfo in that it sorts
+      // parent before daughters.
+      if (left == null) return -1;
+      if (right == null) return 1;
+      // Same table name.
+      int result = Bytes.compareTo(left.getTableName(),
+          right.getTableName());
+      if (result != 0) return result;
+      // Compare start keys.
+      result = Bytes.compareTo(left.getStartKey(), right.getStartKey());
+      if (result != 0) return result;
+      // Compare end keys.
+      result = rowEndKeyComparator.compare(left.getEndKey(), right.getEndKey());
+
+      return -result; // Flip the result so parent comes first.
+    }
+  }
+
+  /**
+   * Get HRegionInfo from passed Map of row values.
+   * @param result Map to do lookup in.
+   * @return Null if not found (and logs fact that expected COL_REGIONINFO
+   * was missing) else deserialized {@link HRegionInfo}
+   * @throws IOException
+   */
+  static HRegionInfo getHRegionInfo(final Result result)
+  throws IOException {
+    byte [] bytes =
+      result.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+    if (bytes == null) {
+      LOG.warn("REGIONINFO_QUALIFIER is empty in " + result);
+      return null;
+    }
+    return Writables.getHRegionInfo(bytes);
+  }
+
+  /**
+   * If daughters no longer hold reference to the parents, delete the parent.
+   * @param server HRegionInterface of meta server to talk to
+   * @param parent HRegionInfo of split offlined parent
+   * @param rowContent Content of parent row in
+   * metaRegionName
+   * @return True if we removed parent from meta table and from
+   * the filesystem.
+   * @throws IOException
+   */
+  boolean cleanParent(final HRegionInfo parent, Result rowContent)
+  throws IOException {
+    boolean result = false;
+    // Run checks on each daughter split.
+    HRegionInfo a_region = getDaughterRegionInfo(rowContent, HConstants.SPLITA_QUALIFIER);
+    HRegionInfo b_region = getDaughterRegionInfo(rowContent, HConstants.SPLITB_QUALIFIER);
+    Pair a =
+      checkDaughterInFs(parent, a_region, HConstants.SPLITA_QUALIFIER);
+    Pair b =
+      checkDaughterInFs(parent, b_region, HConstants.SPLITB_QUALIFIER);
+    if (hasNoReferences(a) && hasNoReferences(b)) {
+      LOG.debug("Deleting region " + parent.getRegionNameAsString() +
+        " because daughter splits no longer hold references");
+
+      // This latter regionOffline should not be necessary but is done for now
+      // until we let go of regionserver to master heartbeats.  See HBASE-3368.
+      if (this.services.getAssignmentManager() != null) {
+        // The mock used in testing catalogjanitor returns null for getAssignmnetManager.
+        // Allow for null result out of getAssignmentManager.
+        this.services.getAssignmentManager().regionOffline(parent);
+      }
+      FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
+      HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
+      MetaEditor.deleteRegion(this.server.getCatalogTracker(), parent);
+      result = true;
+    }
+    return result;
+  }
+
+  /**
+   * @param p A pair where the first boolean says whether or not the daughter
+   * region directory exists in the filesystem and then the second boolean says
+   * whether the daughter has references to the parent.
+   * @return True the passed p signifies no references.
+   */
+  private boolean hasNoReferences(final Pair p) {
+    return !p.getFirst() || !p.getSecond();
+  }
+
+  /**
+   * Get daughter HRegionInfo out of parent info:splitA/info:splitB columns.
+   * @param result
+   * @param which Whether "info:splitA" or "info:splitB" column
+   * @return Deserialized content of the info:splitA or info:splitB as a
+   * HRegionInfo
+   * @throws IOException
+   */
+  private HRegionInfo getDaughterRegionInfo(final Result result,
+    final byte [] which)
+  throws IOException {
+    byte [] bytes = result.getValue(HConstants.CATALOG_FAMILY, which);
+    return Writables.getHRegionInfoOrNull(bytes);
+  }
+
+  /**
+   * Checks if a daughter region -- either splitA or splitB -- still holds
+   * references to parent.
+   * @param parent Parent region name.
+   * @param split Which column family.
+   * @param qualifier Which of the daughters to look at, splitA or splitB.
+   * @return A pair where the first boolean says whether or not the daughter
+   * region directory exists in the filesystem and then the second boolean says
+   * whether the daughter has references to the parent.
+   * @throws IOException
+   */
+  Pair checkDaughterInFs(final HRegionInfo parent,
+    final HRegionInfo split,
+    final byte [] qualifier)
+  throws IOException {
+    boolean references = false;
+    boolean exists = false;
+    if (split == null)  {
+      return new Pair(Boolean.FALSE, Boolean.FALSE);
+    }
+    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
+    Path rootdir = this.services.getMasterFileSystem().getRootDir();
+    Path tabledir = new Path(rootdir, split.getTableNameAsString());
+    Path regiondir = new Path(tabledir, split.getEncodedName());
+    exists = fs.exists(regiondir);
+    if (!exists) {
+      LOG.warn("Daughter regiondir does not exist: " + regiondir.toString());
+      return new Pair(exists, Boolean.FALSE);
+    }
+    HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTableName());
+
+    for (HColumnDescriptor family: parentDescriptor.getFamilies()) {
+      Path p = Store.getStoreHomedir(tabledir, split.getEncodedName(),
+        family.getName());
+      if (!fs.exists(p)) continue;
+      // Look for reference files.  Call listStatus with anonymous instance of PathFilter.
+      FileStatus [] ps = FSUtils.listStatus(fs, p,
+          new PathFilter () {
+            public boolean accept(Path path) {
+              return StoreFile.isReference(path);
+            }
+          }
+      );
+
+      if (ps != null && ps.length > 0) {
+        references = true;
+        break;
+      }
+    }
+    return new Pair(Boolean.valueOf(exists),
+      Boolean.valueOf(references));
+  }
+
+  private HTableDescriptor getTableDescriptor(byte[] tableName)
+  throws FileNotFoundException, IOException {
+    return this.services.getTableDescriptors().get(Bytes.toString(tableName));
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java b/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
new file mode 100644
index 0000000..26e5714
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
@@ -0,0 +1,172 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.hbase.ServerName;
+
+/**
+ * Class to hold dead servers list and utility querying dead server list.
+ */
+public class DeadServer implements Set {
+  /**
+   * Set of known dead servers.  On znode expiration, servers are added here.
+   * This is needed in case of a network partitioning where the server's lease
+   * expires, but the server is still running. After the network is healed,
+   * and it's server logs are recovered, it will be told to call server startup
+   * because by then, its regions have probably been reassigned.
+   */
+  private final Set deadServers = new HashSet();
+
+  /** Number of dead servers currently being processed */
+  private int numProcessing;
+
+  public DeadServer() {
+    super();
+    this.numProcessing = 0;
+  }
+
+  /**
+   * @param serverName Server name
+   * @return true if server is dead
+   */
+  public boolean isDeadServer(final String serverName) {
+    return isDeadServer(new ServerName(serverName));
+  }
+
+  /**
+   * A dead server that comes back alive has a different start code.
+   * @param newServerName Servername as either host:port or
+   * host,port,startcode.
+   * @return true if this server was dead before and coming back alive again
+   */
+  public boolean cleanPreviousInstance(final ServerName newServerName) {
+    ServerName sn =
+      ServerName.findServerWithSameHostnamePort(this.deadServers, newServerName);
+    if (sn == null) return false;
+    return this.deadServers.remove(sn);
+  }
+
+  /**
+   * @param serverName
+   * @return true if this server is on the dead servers list.
+   */
+  boolean isDeadServer(final ServerName serverName) {
+    return this.deadServers.contains(serverName);
+  }
+
+  /**
+   * @return True if we have a server with matching hostname and port.
+   */
+  boolean isDeadServerWithSameHostnamePort(final ServerName serverName) {
+    return ServerName.findServerWithSameHostnamePort(this.deadServers,
+      serverName) != null;
+  }
+
+  /**
+   * Checks if there are currently any dead servers being processed by the
+   * master.  Returns true if at least one region server is currently being
+   * processed as dead.
+   * @return true if any RS are being processed as dead
+   */
+  public boolean areDeadServersInProgress() {
+    return numProcessing != 0;
+  }
+
+  public synchronized Set clone() {
+    Set clone = new HashSet(this.deadServers.size());
+    clone.addAll(this.deadServers);
+    return clone;
+  }
+
+  public synchronized boolean add(ServerName e) {
+    this.numProcessing++;
+    return deadServers.add(e);
+  }
+
+  public synchronized void finish(ServerName e) {
+    this.numProcessing--;
+  }
+
+  public synchronized int size() {
+    return deadServers.size();
+  }
+
+  public synchronized boolean isEmpty() {
+    return deadServers.isEmpty();
+  }
+
+  public synchronized boolean contains(Object o) {
+    return deadServers.contains(o);
+  }
+
+  public Iterator iterator() {
+    return this.deadServers.iterator();
+  }
+
+  public synchronized Object[] toArray() {
+    return deadServers.toArray();
+  }
+
+  public synchronized  T[] toArray(T[] a) {
+    return deadServers.toArray(a);
+  }
+
+  public synchronized boolean remove(Object o) {
+    return this.deadServers.remove(o);
+  }
+
+  public synchronized boolean containsAll(Collection c) {
+    return deadServers.containsAll(c);
+  }
+
+  public synchronized boolean addAll(Collection c) {
+    return deadServers.addAll(c);
+  }
+
+  public synchronized boolean retainAll(Collection c) {
+    return deadServers.retainAll(c);
+  }
+
+  public synchronized boolean removeAll(Collection c) {
+    return deadServers.removeAll(c);
+  }
+
+  public synchronized void clear() {
+    throw new NotImplementedException();
+  }
+
+  public synchronized boolean equals(Object o) {
+    return deadServers.equals(o);
+  }
+
+  public synchronized int hashCode() {
+    return deadServers.hashCode();
+  }
+
+  public synchronized String toString() {
+    return this.deadServers.toString();
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java b/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
new file mode 100644
index 0000000..78f41de
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/DefaultLoadBalancer.java
@@ -0,0 +1,779 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.MinMaxPriorityQueue;
+import com.google.common.collect.Sets;
+
+/**
+ * Makes decisions about the placement and movement of Regions across
+ * RegionServers.
+ *
+ * 
        Cluster-wide load balancing will occur only when there are no regions in
+ * transition and according to a fixed period of a time using {@link #balanceCluster(Map)}.
+ *
+ * 
        Inline region placement with {@link #immediateAssignment} can be used when
+ * the Master needs to handle closed regions that it currently does not have
+ * a destination set for.  This can happen during master failover.
+ *
+ * 
        On cluster startup, bulk assignment can be used to determine
+ * locations for all Regions in a cluster.
+ *
+ * 
        This classes produces plans for the {@link AssignmentManager} to execute.
+ */
+public class DefaultLoadBalancer implements LoadBalancer {
+  private static final Log LOG = LogFactory.getLog(LoadBalancer.class);
+  private static final Random RANDOM = new Random(System.currentTimeMillis());
+  // slop for regions
+  private float slop;
+  private Configuration config;
+  private ClusterStatus status;
+  private MasterServices services;
+
+  public void setClusterStatus(ClusterStatus st) {
+    this.status = st;
+  }
+
+  public void setMasterServices(MasterServices masterServices) {
+    this.services = masterServices;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.slop = conf.getFloat("hbase.regions.slop", (float) 0.2);
+    if (slop < 0) slop = 0;
+    else if (slop > 1) slop = 1;
+    this.config = conf;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.config;
+  }
+
+  /*
+  * The following comparator assumes that RegionId from HRegionInfo can
+  * represent the age of the region - larger RegionId means the region
+  * is younger.
+  * This comparator is used in balanceCluster() to account for the out-of-band
+  * regions which were assigned to the server after some other region server
+  * crashed.
+  */
+   private static class RegionInfoComparator implements Comparator {
+       @Override
+       public int compare(HRegionInfo l, HRegionInfo r) {
+          long diff = r.getRegionId() - l.getRegionId();
+          if (diff < 0) return -1;
+          if (diff > 0) return 1;
+          return 0;
+       }
+   }
+
+
+   RegionInfoComparator riComparator = new RegionInfoComparator();
+   
+   private class RegionPlanComparator implements Comparator {
+    @Override
+    public int compare(RegionPlan l, RegionPlan r) {
+      long diff = r.getRegionInfo().getRegionId() - l.getRegionInfo().getRegionId();
+      if (diff < 0) return -1;
+      if (diff > 0) return 1;
+      return 0;
+    }
+  }
+
+  RegionPlanComparator rpComparator = new RegionPlanComparator();
+
+  /**
+   * Generate a global load balancing plan according to the specified map of
+   * server information to the most loaded regions of each server.
+   *
+   * The load balancing invariant is that all servers are within 1 region of the
+   * average number of regions per server.  If the average is an integer number,
+   * all servers will be balanced to the average.  Otherwise, all servers will
+   * have either floor(average) or ceiling(average) regions.
+   *
+   * HBASE-3609 Modeled regionsToMove using Guava's MinMaxPriorityQueue so that
+   *   we can fetch from both ends of the queue. 
+   * At the beginning, we check whether there was empty region server 
+   *   just discovered by Master. If so, we alternately choose new / old
+   *   regions from head / tail of regionsToMove, respectively. This alternation
+   *   avoids clustering young regions on the newly discovered region server.
+   *   Otherwise, we choose new regions from head of regionsToMove.
+   *   
+   * Another improvement from HBASE-3609 is that we assign regions from
+   *   regionsToMove to underloaded servers in round-robin fashion.
+   *   Previously one underloaded server would be filled before we move onto
+   *   the next underloaded server, leading to clustering of young regions.
+   *   
+   * Finally, we randomly shuffle underloaded servers so that they receive
+   *   offloaded regions relatively evenly across calls to balanceCluster().
+   *         
+   * The algorithm is currently implemented as such:
+   *
+   * 
          
+   * 
        1. Determine the two valid numbers of regions each server should have,
+   *     MIN=floor(average) and MAX=ceiling(average).
+   *
+   * 
        2. Iterate down the most loaded servers, shedding regions from each so
+   *     each server hosts exactly MAX regions.  Stop once you reach a
+   *     server that already has <= MAX regions.
+   *     
          
+   *     Order the regions to move from most recent to least.
+   *
+   * 
        3. Iterate down the least loaded servers, assigning regions so each server
+   *     has exactly MIN regions.  Stop once you reach a server that
+   *     already has >= MIN regions.
+   *
+   *     Regions being assigned to underloaded servers are those that were shed
+   *     in the previous step.  It is possible that there were not enough
+   *     regions shed to fill each underloaded server to MIN.  If so we
+   *     end up with a number of regions required to do so, neededRegions.
+   *
+   *     It is also possible that we were able to fill each underloaded but ended
+   *     up with regions that were unassigned from overloaded servers but that
+   *     still do not have assignment.
+   *
+   *     If neither of these conditions hold (no regions needed to fill the
+   *     underloaded servers, no regions leftover from overloaded servers),
+   *     we are done and return.  Otherwise we handle these cases below.
+   *
+   * 
        4. If neededRegions is non-zero (still have underloaded servers),
+   *     we iterate the most loaded servers again, shedding a single server from
+   *     each (this brings them from having MAX regions to having
+   *     MIN regions).
+   *
+   * 
        5. We now definitely have more regions that need assignment, either from
+   *     the previous step or from the original shedding from overloaded servers.
+   *     Iterate the least loaded servers filling each to MIN.
+   *
+   * 
        6. If we still have more regions that need assignment, again iterate the
+   *     least loaded servers, this time giving each one (filling them to
+   *     MAX) until we run out.
+   *
+   * 
        7. All servers will now either host MIN or MAX regions.
+   *
+   *     In addition, any server hosting >= MAX regions is guaranteed
+   *     to end up with MAX regions at the end of the balancing.  This
+   *     ensures the minimal number of regions possible are moved.
+   * 
        
+   *
+   * TODO: We can at-most reassign the number of regions away from a particular
+   *       server to be how many they report as most loaded.
+   *       Should we just keep all assignment in memory?  Any objections?
+   *       Does this mean we need HeapSize on HMaster?  Or just careful monitor?
+   *       (current thinking is we will hold all assignments in memory)
+   *
+   * @param clusterState Map of regionservers and their load/region information to
+   *                   a list of their most loaded regions
+   * @return a list of regions to be moved, including source and destination,
+   *         or null if cluster is already balanced
+   */
+  public List balanceCluster(
+      Map> clusterState) {
+    boolean emptyRegionServerPresent = false;
+    long startTime = System.currentTimeMillis();
+
+    int numServers = clusterState.size();
+    if (numServers == 0) {
+      LOG.debug("numServers=0 so skipping load balancing");
+      return null;
+    }
+    NavigableMap> serversByLoad =
+      new TreeMap>();
+    int numRegions = 0;
+    int maxRegionCountPerServer = 0;
+    // Iterate so we can count regions as we build the map
+    for (Map.Entry> server: clusterState.entrySet()) {
+      List regions = server.getValue();
+      int sz = regions.size();
+      if (sz == 0) emptyRegionServerPresent = true;
+      numRegions += sz;
+      if (maxRegionCountPerServer < sz) maxRegionCountPerServer = sz;
+      serversByLoad.put(new ServerAndLoad(server.getKey(), sz), regions);
+    }
+    // Check if we even need to do any load balancing
+    float average = (float)numRegions / numServers; // for logging
+    // HBASE-3681 check sloppiness first
+    int floor = (int) Math.floor(average * (1 - slop));
+    int ceiling = (int) Math.ceil(average * (1 + slop));
+    if (serversByLoad.lastKey().getLoad() <= ceiling &&
+       serversByLoad.firstKey().getLoad() >= floor) {
+      // Skipped because no server outside (min,max) range
+      LOG.info("Skipping load balancing because balanced cluster; " +
+        "servers=" + numServers + " " +
+        "regions=" + numRegions + " average=" + average + " " +
+        "mostloaded=" + serversByLoad.lastKey().getLoad() +
+        " leastloaded=" + serversByLoad.firstKey().getLoad());
+      return null;
+    }
+    int min = numRegions / numServers;
+    int max = numRegions % numServers == 0 ? min : min + 1;
+    if (maxRegionCountPerServer == 1) return null; // table is balanced
+
+    // Using to check balance result.
+    StringBuilder strBalanceParam = new StringBuilder();
+    strBalanceParam.append("Balance parameter: numRegions=").append(numRegions)
+        .append(", numServers=").append(numServers).append(", max=").append(max)
+        .append(", min=").append(min);
+    LOG.debug(strBalanceParam.toString());
+
+    // Balance the cluster
+    // TODO: Look at data block locality or a more complex load to do this
+    MinMaxPriorityQueue regionsToMove =
+      MinMaxPriorityQueue.orderedBy(rpComparator).create();
+    List regionsToReturn = new ArrayList();
+
+    // Walk down most loaded, pruning each to the max
+    int serversOverloaded = 0;
+    // flag used to fetch regions from head and tail of list, alternately
+    boolean fetchFromTail = false;
+    Map serverBalanceInfo =
+      new TreeMap();
+    for (Map.Entry> server:
+        serversByLoad.descendingMap().entrySet()) {
+      ServerAndLoad sal = server.getKey();
+      int regionCount = sal.getLoad();
+      if (regionCount <= max) {
+        serverBalanceInfo.put(sal.getServerName(), new BalanceInfo(0, 0));
+        break;
+      }
+      serversOverloaded++;
+      List regions = server.getValue();
+      int numToOffload = Math.min(regionCount - max, regions.size());
+      // account for the out-of-band regions which were assigned to this server
+      // after some other region server crashed 
+      Collections.sort(regions, riComparator);
+      int numTaken = 0;
+      for (int i = 0; i <= numToOffload; ) {
+        HRegionInfo hri = regions.get(i); // fetch from head
+        if (fetchFromTail) {
+          hri = regions.get(regions.size() - 1 - i);
+        }
+        i++;
+        // Don't rebalance meta regions.
+        if (hri.isMetaRegion()) continue;
+        regionsToMove.add(new RegionPlan(hri, sal.getServerName(), null));
+        numTaken++;
+        if (numTaken >= numToOffload) break;
+        // fetch in alternate order if there is new region server
+        if (emptyRegionServerPresent) {
+          fetchFromTail = !fetchFromTail;
+        }
+      }
+      serverBalanceInfo.put(sal.getServerName(),
+        new BalanceInfo(numToOffload, (-1)*numTaken));
+    }
+    int totalNumMoved = regionsToMove.size();
+
+    // Walk down least loaded, filling each to the min
+    int neededRegions = 0; // number of regions needed to bring all up to min
+    fetchFromTail = false;
+
+    Map underloadedServers = new HashMap();
+    int maxToTake = numRegions - (int)average;
+    for (Map.Entry> server:
+        serversByLoad.entrySet()) {
+      if (maxToTake == 0) break; // no more to take
+      int regionCount = server.getKey().getLoad();
+      if (regionCount >= min && regionCount > 0) {
+        continue; // look for other servers which haven't reached min
+      }
+      int regionsToPut = min - regionCount;
+      if (regionsToPut == 0)
+      {
+        regionsToPut = 1;
+        maxToTake--;
+      }
+      underloadedServers.put(server.getKey().getServerName(), regionsToPut);
+    }
+    // number of servers that get new regions
+    int serversUnderloaded = underloadedServers.size();
+    int incr = 1;
+    List sns =
+      Arrays.asList(underloadedServers.keySet().toArray(new ServerName[serversUnderloaded]));
+    Collections.shuffle(sns, RANDOM);
+    while (regionsToMove.size() > 0) {
+      int cnt = 0;
+      int i = incr > 0 ? 0 : underloadedServers.size()-1;
+      for (; i >= 0 && i < underloadedServers.size(); i += incr) {
+        if (regionsToMove.isEmpty()) break;
+        ServerName si = sns.get(i);
+        int numToTake = underloadedServers.get(si);
+        if (numToTake == 0) continue;
+
+        addRegionPlan(regionsToMove, fetchFromTail, si, regionsToReturn);
+        if (emptyRegionServerPresent) {
+          fetchFromTail = !fetchFromTail;
+        }
+
+        underloadedServers.put(si, numToTake-1);
+        cnt++;
+        BalanceInfo bi = serverBalanceInfo.get(si);
+        if (bi == null) {
+          bi = new BalanceInfo(0, 0);
+          serverBalanceInfo.put(si, bi);
+        }
+        bi.setNumRegionsAdded(bi.getNumRegionsAdded()+1);
+      }
+      if (cnt == 0) break;
+      // iterates underloadedServers in the other direction
+      incr = -incr;
+    }
+    for (Integer i : underloadedServers.values()) {
+      // If we still want to take some, increment needed
+      neededRegions += i;
+    }
+
+    // If none needed to fill all to min and none left to drain all to max,
+    // we are done
+    if (neededRegions == 0 && regionsToMove.isEmpty()) {
+      long endTime = System.currentTimeMillis();
+      LOG.info("Calculated a load balance in " + (endTime-startTime) + "ms. " +
+          "Moving " + totalNumMoved + " regions off of " +
+          serversOverloaded + " overloaded servers onto " +
+          serversUnderloaded + " less loaded servers");
+      return regionsToReturn;
+    }
+
+    // Need to do a second pass.
+    // Either more regions to assign out or servers that are still underloaded
+
+    // If we need more to fill min, grab one from each most loaded until enough
+    if (neededRegions != 0) {
+      // Walk down most loaded, grabbing one from each until we get enough
+      for (Map.Entry> server :
+        serversByLoad.descendingMap().entrySet()) {
+        BalanceInfo balanceInfo =
+          serverBalanceInfo.get(server.getKey().getServerName());
+        int idx =
+          balanceInfo == null ? 0 : balanceInfo.getNextRegionForUnload();
+        if (idx >= server.getValue().size()) break;
+        HRegionInfo region = server.getValue().get(idx);
+        if (region.isMetaRegion()) continue; // Don't move meta regions.
+        regionsToMove.add(new RegionPlan(region, server.getKey().getServerName(), null));
+        totalNumMoved++;
+        if (--neededRegions == 0) {
+          // No more regions needed, done shedding
+          break;
+        }
+      }
+    }
+
+    // Now we have a set of regions that must be all assigned out
+    // Assign each underloaded up to the min, then if leftovers, assign to max
+
+    // Walk down least loaded, assigning to each to fill up to min
+    for (Map.Entry> server :
+        serversByLoad.entrySet()) {
+      int regionCount = server.getKey().getLoad();
+      if (regionCount >= min) break;
+      BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey().getServerName());
+      if(balanceInfo != null) {
+        regionCount += balanceInfo.getNumRegionsAdded();
+      }
+      if(regionCount >= min) {
+        continue;
+      }
+      int numToTake = min - regionCount;
+      int numTaken = 0;
+      while(numTaken < numToTake && 0 < regionsToMove.size()) {
+        addRegionPlan(regionsToMove, fetchFromTail,
+          server.getKey().getServerName(), regionsToReturn);
+        numTaken++;
+        if (emptyRegionServerPresent) {
+          fetchFromTail = !fetchFromTail;
+        }
+      }
+    }
+
+    // If we still have regions to dish out, assign underloaded to max
+    if (0 < regionsToMove.size()) {
+      for (Map.Entry> server :
+        serversByLoad.entrySet()) {
+        int regionCount = server.getKey().getLoad();
+        if(regionCount >= max) {
+          break;
+        }
+        addRegionPlan(regionsToMove, fetchFromTail,
+          server.getKey().getServerName(), regionsToReturn);
+        if (emptyRegionServerPresent) {
+          fetchFromTail = !fetchFromTail;
+        }
+        if (regionsToMove.isEmpty()) {
+          break;
+        }
+      }
+    }
+
+    long endTime = System.currentTimeMillis();
+
+    if (!regionsToMove.isEmpty() || neededRegions != 0) {
+      // Emit data so can diagnose how balancer went astray.
+      LOG.warn("regionsToMove=" + totalNumMoved +
+        ", numServers=" + numServers + ", serversOverloaded=" + serversOverloaded +
+        ", serversUnderloaded=" + serversUnderloaded);
+      StringBuilder sb = new StringBuilder();
+      for (Map.Entry> e: clusterState.entrySet()) {
+        if (sb.length() > 0) sb.append(", ");
+        sb.append(e.getKey().toString());
+        sb.append(" ");
+        sb.append(e.getValue().size());
+      }
+      LOG.warn("Input " + sb.toString());
+    }
+
+    // All done!
+    LOG.info("Done. Calculated a load balance in " + (endTime-startTime) + "ms. " +
+        "Moving " + totalNumMoved + " regions off of " +
+        serversOverloaded + " overloaded servers onto " +
+        serversUnderloaded + " less loaded servers");
+
+    return regionsToReturn;
+  }
+
+  /**
+   * Add a region from the head or tail to the List of regions to return.
+   */
+  void addRegionPlan(final MinMaxPriorityQueue regionsToMove,
+      final boolean fetchFromTail, final ServerName sn, List regionsToReturn) {
+    RegionPlan rp = null;
+    if (!fetchFromTail) rp = regionsToMove.remove();
+    else rp = regionsToMove.removeLast();
+    rp.setDestination(sn);
+    regionsToReturn.add(rp);
+  }
+
+  /**
+   * Stores additional per-server information about the regions added/removed
+   * during the run of the balancing algorithm.
+   *
+   * For servers that shed regions, we need to track which regions we have
+   * already shed.  nextRegionForUnload contains the index in the list
+   * of regions on the server that is the next to be shed.
+   */
+  private static class BalanceInfo {
+
+    private final int nextRegionForUnload;
+    private int numRegionsAdded;
+
+    public BalanceInfo(int nextRegionForUnload, int numRegionsAdded) {
+      this.nextRegionForUnload = nextRegionForUnload;
+      this.numRegionsAdded = numRegionsAdded;
+    }
+
+    public int getNextRegionForUnload() {
+      return nextRegionForUnload;
+    }
+
+    public int getNumRegionsAdded() {
+      return numRegionsAdded;
+    }
+
+    public void setNumRegionsAdded(int numAdded) {
+      this.numRegionsAdded = numAdded;
+    }
+  }
+
+  /**
+   * Generates a bulk assignment plan to be used on cluster startup using a
+   * simple round-robin assignment.
+   * 
        
+   * Takes a list of all the regions and all the servers in the cluster and
+   * returns a map of each server to the regions that it should be assigned.
+   * 
        
+   * Currently implemented as a round-robin assignment.  Same invariant as
+   * load balancing, all servers holding floor(avg) or ceiling(avg).
+   *
+   * TODO: Use block locations from HDFS to place regions with their blocks
+   *
+   * @param regions all regions
+   * @param servers all servers
+   * @return map of server to the regions it should take, or null if no
+   *         assignment is possible (ie. no regions or no servers)
+   */
+  public Map> roundRobinAssignment(
+      List regions, List servers) {
+    if (regions.isEmpty() || servers.isEmpty()) {
+      return null;
+    }
+    Map> assignments =
+      new TreeMap>();
+    int numRegions = regions.size();
+    int numServers = servers.size();
+    int max = (int)Math.ceil((float)numRegions/numServers);
+    int serverIdx = 0;
+    if (numServers > 1) {
+      serverIdx = RANDOM.nextInt(numServers);
+    }
+    int regionIdx = 0;
+    for (int j = 0; j < numServers; j++) {
+      ServerName server = servers.get((j + serverIdx) % numServers);
+      List serverRegions = new ArrayList(max);
+      for (int i=regionIdx; i
+   * Takes a map of all regions to their existing assignment from META.  Also
+   * takes a list of online servers for regions to be assigned to.  Attempts to
+   * retain all assignment, so in some instances initial assignment will not be
+   * completely balanced.
+   * 
        
+   * Any leftover regions without an existing server to be assigned to will be
+   * assigned randomly to available servers.
+   * @param regions regions and existing assignment from meta
+   * @param servers available servers
+   * @return map of servers and regions to be assigned to them
+   */
+  public Map> retainAssignment(
+      Map regions, List servers) {
+    // Group all of the old assignments by their hostname.
+    // We can't group directly by ServerName since the servers all have
+    // new start-codes.
+    
+    // Group the servers by their hostname. It's possible we have multiple
+    // servers on the same host on different ports.
+    ArrayListMultimap serversByHostname =
+        ArrayListMultimap.create();
+    for (ServerName server : servers) {
+      serversByHostname.put(server.getHostname(), server);
+    }
+    
+    // Now come up with new assignments
+    Map> assignments =
+      new TreeMap>();
+    
+    for (ServerName server : servers) {
+      assignments.put(server, new ArrayList());
+    }
+    
+    // Collection of the hostnames that used to have regions
+    // assigned, but for which we no longer have any RS running
+    // after the cluster restart.
+    Set oldHostsNoLongerPresent = Sets.newTreeSet();
+    
+    int numRandomAssignments = 0;
+    int numRetainedAssigments = 0;
+    for (Map.Entry entry : regions.entrySet()) {
+      HRegionInfo region = entry.getKey();
+      ServerName oldServerName = entry.getValue();
+      List localServers = new ArrayList();
+      if (oldServerName != null) {
+        localServers = serversByHostname.get(oldServerName.getHostname());
+      }
+      if (localServers.isEmpty()) {
+        // No servers on the new cluster match up with this hostname,
+        // assign randomly.
+        ServerName randomServer = servers.get(RANDOM.nextInt(servers.size()));
+        assignments.get(randomServer).add(region);
+        numRandomAssignments++;
+        if (oldServerName != null) oldHostsNoLongerPresent.add(oldServerName.getHostname());
+      } else if (localServers.size() == 1) {
+        // the usual case - one new server on same host
+        assignments.get(localServers.get(0)).add(region);
+        numRetainedAssigments++;
+      } else {
+        // multiple new servers in the cluster on this same host
+        int size = localServers.size();
+        ServerName target = localServers.get(RANDOM.nextInt(size));
+        assignments.get(target).add(region);
+        numRetainedAssigments++;
+      }
+    }
+    
+    String randomAssignMsg = "";
+    if (numRandomAssignments > 0) {
+      randomAssignMsg = numRandomAssignments + " regions were assigned " +
+      		"to random hosts, since the old hosts for these regions are no " +
+      		"longer present in the cluster. These hosts were:\n  " +
+          Joiner.on("\n  ").join(oldHostsNoLongerPresent);
+    }
+    
+    LOG.info("Reassigned " + regions.size() + " regions. " +
+        numRetainedAssigments + " retained the pre-restart assignment. " +
+        randomAssignMsg);
+    return assignments;
+  }
+
+  /**
+   * Returns an ordered list of hosts that are hosting the blocks for this
+   * region.  The weight of each host is the sum of the block lengths of all
+   * files on that host, so the first host in the list is the server which
+   * holds the most bytes of the given region's HFiles.
+   *
+   * @param fs the filesystem
+   * @param region region
+   * @return ordered list of hosts holding blocks of the specified region
+   */
+  @SuppressWarnings("unused")
+  private List getTopBlockLocations(FileSystem fs,
+    HRegionInfo region) {
+    List topServerNames = null;
+    try {
+      HTableDescriptor tableDescriptor = getTableDescriptor(
+        region.getTableName());
+      if (tableDescriptor != null) {
+        HDFSBlocksDistribution blocksDistribution =
+          HRegion.computeHDFSBlocksDistribution(config, tableDescriptor,
+          region.getEncodedName());
+        List topHosts = blocksDistribution.getTopHosts();
+        topServerNames = mapHostNameToServerName(topHosts);
+      }
+    } catch (IOException ioe) {
+      LOG.debug("IOException during HDFSBlocksDistribution computation. for " +
+        "region = " + region.getEncodedName() , ioe);
+    }
+    
+    return topServerNames;
+  }
+
+  /**
+   * return HTableDescriptor for a given tableName
+   * @param tableName the table name
+   * @return HTableDescriptor
+   * @throws IOException
+   */
+  private HTableDescriptor getTableDescriptor(byte[] tableName)
+    throws IOException {
+    HTableDescriptor tableDescriptor = null;
+    try {
+      if ( this.services != null)
+      {
+        tableDescriptor = this.services.getTableDescriptors().
+          get(Bytes.toString(tableName));
+      }
+    } catch (FileNotFoundException fnfe) {
+      LOG.debug("FileNotFoundException during getTableDescriptors." +
+        " Current table name = " + tableName , fnfe);
+    }
+
+    return tableDescriptor;
+  }
+
+  /**
+   * Map hostname to ServerName, The output ServerName list will have the same
+   * order as input hosts.
+   * @param hosts the list of hosts
+   * @return ServerName list
+   */  
+  private List mapHostNameToServerName(List hosts) {
+    if ( hosts == null || status == null) {
+      return null;
+    }
+
+    List topServerNames = new ArrayList();
+    Collection regionServers = status.getServers();
+
+    // create a mapping from hostname to ServerName for fast lookup
+    HashMap hostToServerName =
+      new HashMap();
+    for (ServerName sn : regionServers) {
+      hostToServerName.put(sn.getHostname(), sn);
+        }
+
+    for (String host : hosts ) {
+      ServerName sn = hostToServerName.get(host);
+      // it is possible that HDFS is up ( thus host is valid ),
+      // but RS is down ( thus sn is null )
+      if (sn != null) {
+        topServerNames.add(sn);
+      }
+    }
+    return topServerNames;
+  }
+
+
+  /**
+   * Generates an immediate assignment plan to be used by a new master for
+   * regions in transition that do not have an already known destination.
+   *
+   * Takes a list of regions that need immediate assignment and a list of
+   * all available servers.  Returns a map of regions to the server they
+   * should be assigned to.
+   *
+   * This method will return quickly and does not do any intelligent
+   * balancing.  The goal is to make a fast decision not the best decision
+   * possible.
+   *
+   * Currently this is random.
+   *
+   * @param regions
+   * @param servers
+   * @return map of regions to the server it should be assigned to
+   */
+  public Map immediateAssignment(
+      List regions, List servers) {
+    Map assignments =
+      new TreeMap();
+    for(HRegionInfo region : regions) {
+      assignments.put(region, servers.get(RANDOM.nextInt(servers.size())));
+    }
+    return assignments;
+  }
+
+  public ServerName randomAssignment(HRegionInfo regionInfo, List servers) {
+    if (servers == null || servers.isEmpty()) {
+      LOG.warn("Wanted to do random assignment but no servers to assign to");
+      return null;
+    }
+    return servers.get(RANDOM.nextInt(servers.size()));
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
new file mode 100644
index 0000000..401ef52
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -0,0 +1,2273 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.management.ObjectName;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HealthCheckChore;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.PleaseHoldException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableDescriptors;
+import org.apache.hadoop.hbase.TableNotDisabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.UnknownRegionException;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.MetaScanner;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.coprocessor.Exec;
+import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+import org.apache.hadoop.hbase.ipc.HBaseRPC;
+import org.apache.hadoop.hbase.ipc.HBaseServer;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.ipc.HMasterRegionInterface;
+import org.apache.hadoop.hbase.ipc.ProtocolSignature;
+import org.apache.hadoop.hbase.ipc.RpcServer;
+import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
+import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
+import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
+import org.apache.hadoop.hbase.master.handler.DeleteTableHandler;
+import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
+import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
+import org.apache.hadoop.hbase.master.handler.ModifyTableHandler;
+import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
+import org.apache.hadoop.hbase.master.handler.TableAddFamilyHandler;
+import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
+import org.apache.hadoop.hbase.master.handler.TableEventHandler;
+import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.replication.regionserver.Replication;
+import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.util.HasThread;
+import org.apache.hadoop.hbase.util.InfoServer;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Sleeper;
+import org.apache.hadoop.hbase.util.Strings;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.VersionInfo;
+import org.apache.hadoop.hbase.zookeeper.ClusterId;
+import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
+import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
+import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.metrics.util.MBeanUtil;
+import org.apache.hadoop.net.DNS;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
+
+import com.google.common.collect.ClassToInstanceMap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.MutableClassToInstanceMap;
+
+/**
+ * HMaster is the "master server" for HBase. An HBase cluster has one active
+ * master.  If many masters are started, all compete.  Whichever wins goes on to
+ * run the cluster.  All others park themselves in their constructor until
+ * master or cluster shutdown or until the active master loses its lease in
+ * zookeeper.  Thereafter, all running master jostle to take over master role.
+ *
+ * 
        The Master can be asked shutdown the cluster. See {@link #shutdown()}.  In
+ * this case it will tell all regionservers to go down and then wait on them
+ * all reporting in that they are down.  This master will then shut itself down.
+ *
+ * 
        You can also shutdown just this master.  Call {@link #stopMaster()}.
+ *
+ * @see HMasterInterface
+ * @see HMasterRegionInterface
+ * @see Watcher
+ */
+public class HMaster extends HasThread
+implements HMasterInterface, HMasterRegionInterface, MasterServices,
+Server {
+  private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
+
+  // MASTER is name of the webapp and the attribute name used stuffing this
+  //instance into web context.
+  public static final String MASTER = "master";
+
+  // The configuration for the Master
+  private final Configuration conf;
+  // server for the web ui
+  private InfoServer infoServer;
+
+  // Our zk client.
+  private ZooKeeperWatcher zooKeeper;
+  // Manager and zk listener for master election
+  private ActiveMasterManager activeMasterManager;
+  // Region server tracker
+  private RegionServerTracker regionServerTracker;
+  // Draining region server tracker
+  private DrainingServerTracker drainingServerTracker;
+
+  // RPC server for the HMaster
+  private final RpcServer rpcServer;
+
+  /**
+   * This servers address.
+   */
+  private final InetSocketAddress isa;
+
+  // Metrics for the HMaster
+  private final MasterMetrics metrics;
+  // file system manager for the master FS operations
+  private MasterFileSystem fileSystemManager;
+
+  // server manager to deal with region server info
+  private ServerManager serverManager;
+
+  // manager of assignment nodes in zookeeper
+  AssignmentManager assignmentManager;
+  // manager of catalog regions
+  private CatalogTracker catalogTracker;
+  // Cluster status zk tracker and local setter
+  private ClusterStatusTracker clusterStatusTracker;
+
+  // buffer for "fatal error" notices from region servers
+  // in the cluster. This is only used for assisting
+  // operations/debugging.
+  private MemoryBoundedLogMessageBuffer rsFatals;
+
+  // This flag is for stopping this Master instance.  Its set when we are
+  // stopping or aborting
+  private volatile boolean stopped = false;
+  // Set on abort -- usually failure of our zk session.
+  private volatile boolean abort = false;
+  // flag set after we become the active master (used for testing)
+  private volatile boolean isActiveMaster = false;
+
+  // flag set after we complete initialization once active,
+  // it is not private since it's used in unit tests
+  volatile boolean initialized = false;
+
+  // flag set after we complete assignRootAndMeta.
+  private volatile boolean serverShutdownHandlerEnabled = false;
+  // flag to indicate that we should be handling meta hlogs differently for splitting
+  private volatile boolean shouldSplitMetaSeparately;
+
+  // Instance of the hbase executor service.
+  ExecutorService executorService;
+
+  private LoadBalancer balancer;
+  private Thread balancerChore;
+  // If 'true', the balancer is 'on'.  If 'false', the balancer will not run.
+  private volatile boolean balanceSwitch = true;
+
+  private CatalogJanitor catalogJanitorChore;
+  private LogCleaner logCleaner;
+  private HFileCleaner hfileCleaner;
+
+  private MasterCoprocessorHost cpHost;
+  private final ServerName serverName;
+
+  private TableDescriptors tableDescriptors;
+
+  // Time stamps for when a hmaster was started and when it became active
+  private long masterStartTime;
+  private long masterActiveTime;
+
+  // monitor for snapshot of hbase tables
+  private SnapshotManager snapshotManager;
+
+  /**
+   * MX Bean for MasterInfo
+   */
+  private ObjectName mxBean = null;
+
+  // Registered master protocol handlers
+  private ClassToInstanceMap
+      protocolHandlers = MutableClassToInstanceMap.create();
+
+  private Map>
+      protocolHandlerNames = Maps.newHashMap();
+
+  /** The health check chore. */
+  private HealthCheckChore healthCheckChore;
+
+  /** flag when true, Master waits for log splitting complete before start up */
+  private boolean waitingOnLogSplitting = false;
+
+  /** flag used in test cases in order to simulate RS failures during master initialization */
+  private volatile boolean initializationBeforeMetaAssignment = false;
+
+  /** The following is used in master recovery scenario to re-register listeners */
+  private List registeredZKListenersBeforeRecovery;
+
+  /**
+   * Initializes the HMaster. The steps are as follows:
+   * 
        
+   * 
          
+   * 
        1. Initialize HMaster RPC and address
+   * 
        2. Connect to ZooKeeper.
+   * 
        
+   * 
        
+   * Remaining steps of initialization occur in {@link #run()} so that they
+   * run in their own thread rather than within the context of the constructor.
+   * @throws InterruptedException
+   */
+  public HMaster(final Configuration conf)
+  throws IOException, KeeperException, InterruptedException {
+    this.conf = new Configuration(conf);
+    // Disable the block cache on the master
+    this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
+    // Set how many times to retry talking to another server over HConnection.
+    HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG);
+    // Server to handle client requests.
+    String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(
+      conf.get("hbase.master.dns.interface", "default"),
+      conf.get("hbase.master.dns.nameserver", "default")));
+    int port = conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT);
+    // Test that the hostname is reachable
+    InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
+    if (initialIsa.getAddress() == null) {
+      throw new IllegalArgumentException("Failed resolve of hostname " + initialIsa);
+    }
+    // Verify that the bind address is reachable if set
+    String bindAddress = conf.get("hbase.master.ipc.address");
+    if (bindAddress != null) {
+      initialIsa = new InetSocketAddress(bindAddress, port);
+      if (initialIsa.getAddress() == null) {
+        throw new IllegalArgumentException("Failed resolve of bind address " + initialIsa);
+      }
+    }
+    int numHandlers = conf.getInt("hbase.master.handler.count",
+      conf.getInt("hbase.regionserver.handler.count", 25));
+    this.rpcServer = HBaseRPC.getServer(this,
+      new Class[]{HMasterInterface.class, HMasterRegionInterface.class},
+        initialIsa.getHostName(), // This is bindAddress if set else it's hostname
+        initialIsa.getPort(),
+        numHandlers,
+        0, // we dont use high priority handlers in master
+        conf.getBoolean("hbase.rpc.verbose", false), conf,
+        0); // this is a DNC w/o high priority handlers
+    // Set our address.
+    this.isa = this.rpcServer.getListenerAddress();
+    this.serverName = new ServerName(hostname,
+      this.isa.getPort(), System.currentTimeMillis());
+    this.rsFatals = new MemoryBoundedLogMessageBuffer(
+        conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));
+
+    // login the zookeeper client principal (if using security)
+    ZKUtil.loginClient(this.conf, "hbase.zookeeper.client.keytab.file",
+      "hbase.zookeeper.client.kerberos.principal", this.isa.getHostName());
+
+    // initialize server principal (if using secure Hadoop)
+    User.login(conf, "hbase.master.keytab.file",
+      "hbase.master.kerberos.principal", this.isa.getHostName());
+
+    // set the thread name now we have an address
+    setName(MASTER + "-" + this.serverName.toString());
+
+    Replication.decorateMasterConfiguration(this.conf);
+
+    // Hack! Maps DFSClient => Master for logs.  HDFS made this
+    // config param for task trackers, but we can piggyback off of it.
+    if (this.conf.get("mapred.task.id") == null) {
+      this.conf.set("mapred.task.id", "hb_m_" + this.serverName.toString());
+    }
+
+    this.zooKeeper = new ZooKeeperWatcher(conf, MASTER + ":" + isa.getPort(), this, true);
+    this.rpcServer.startThreads();
+    this.metrics = new MasterMetrics(getServerName().toString());
+
+    // Health checker thread.
+    int sleepTime = this.conf.getInt(HConstants.HEALTH_CHORE_WAKE_FREQ,
+      HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);
+    if (isHealthCheckerConfigured()) {
+      healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration());
+    }
+
+    this.shouldSplitMetaSeparately = conf.getBoolean(HLog.SEPARATE_HLOG_FOR_META, false);
+    waitingOnLogSplitting = this.conf.getBoolean("hbase.master.wait.for.log.splitting", false);
+  }
+
+  /**
+   * Stall startup if we are designated a backup master; i.e. we want someone
+   * else to become the master before proceeding.
+   * @param c
+   * @param amm
+   * @throws InterruptedException
+   */
+  private static void stallIfBackupMaster(final Configuration c,
+      final ActiveMasterManager amm)
+  throws InterruptedException {
+    // If we're a backup master, stall until a primary to writes his address
+    if (!c.getBoolean(HConstants.MASTER_TYPE_BACKUP,
+      HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
+      return;
+    }
+    LOG.debug("HMaster started in backup mode.  " +
+      "Stalling until master znode is written.");
+    // This will only be a minute or so while the cluster starts up,
+    // so don't worry about setting watches on the parent znode
+    while (!amm.isActiveMaster()) {
+      LOG.debug("Waiting for master address ZNode to be written " +
+        "(Also watching cluster state node)");
+      Thread.sleep(c.getInt("zookeeper.session.timeout", 180 * 1000));
+    }
+
+  }
+
+  /**
+   * Main processing loop for the HMaster.
+   * 
          
+   * 
        1. Block until becoming active master
+   * 
        2. Finish initialization via finishInitialization(MonitoredTask)
+   * 
        3. Enter loop until we are stopped
+   * 
        4. Stop services and perform cleanup once stopped
+   * 
        
+   */
+  @Override
+  public void run() {
+    MonitoredTask startupStatus =
+      TaskMonitor.get().createStatus("Master startup");
+    startupStatus.setDescription("Master startup");
+    masterStartTime = System.currentTimeMillis();
+    try {
+      this.registeredZKListenersBeforeRecovery = this.zooKeeper.getListeners();
+      /*
+       * Block on becoming the active master.
+       *
+       * We race with other masters to write our address into ZooKeeper.  If we
+       * succeed, we are the primary/active master and finish initialization.
+       *
+       * If we do not succeed, there is another active master and we should
+       * now wait until it dies to try and become the next active master.  If we
+       * do not succeed on our first attempt, this is no longer a cluster startup.
+       */
+      becomeActiveMaster(startupStatus);
+
+      // We are either the active master or we were asked to shutdown
+      if (!this.stopped) {
+        finishInitialization(startupStatus, false);
+        loop();
+      }
+    } catch (Throwable t) {
+      // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility
+      if (t instanceof NoClassDefFoundError &&
+          t.getMessage().contains("org/apache/hadoop/hdfs/protocol/FSConstants$SafeModeAction")) {
+          // improved error message for this special case
+          abort("HBase is having a problem with its Hadoop jars.  You may need to "
+              + "recompile HBase against Hadoop version "
+              +  org.apache.hadoop.util.VersionInfo.getVersion()
+              + " or change your hadoop jars to start properly", t);
+      } else {
+        abort("Unhandled exception. Starting shutdown.", t);
+      }
+    } finally {
+      startupStatus.cleanup();
+
+      stopChores();
+      // Wait for all the remaining region servers to report in IFF we were
+      // running a cluster shutdown AND we were NOT aborting.
+      if (!this.abort && this.serverManager != null &&
+          this.serverManager.isClusterShutdown()) {
+        this.serverManager.letRegionServersShutdown();
+      }
+      stopServiceThreads();
+      // Stop services started for both backup and active masters
+      if (this.activeMasterManager != null) this.activeMasterManager.stop();
+      if (this.catalogTracker != null) this.catalogTracker.stop();
+      if (this.serverManager != null) this.serverManager.stop();
+      if (this.assignmentManager != null) this.assignmentManager.stop();
+      if (this.fileSystemManager != null) this.fileSystemManager.stop();
+      if (this.snapshotManager != null) this.snapshotManager.stop("server shutting down.");
+      this.zooKeeper.close();
+    }
+    LOG.info("HMaster main thread exiting");
+  }
+
+  /**
+   * Try becoming active master.
+   * @param startupStatus
+   * @return True if we could successfully become the active master.
+   * @throws InterruptedException
+   */
+  private boolean becomeActiveMaster(MonitoredTask startupStatus)
+  throws InterruptedException {
+    // TODO: This is wrong!!!! Should have new servername if we restart ourselves,
+    // if we come back to life.
+    this.activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName,
+        this);
+    this.zooKeeper.registerListener(activeMasterManager);
+    stallIfBackupMaster(this.conf, this.activeMasterManager);
+
+    // The ClusterStatusTracker is setup before the other
+    // ZKBasedSystemTrackers because it's needed by the activeMasterManager
+    // to check if the cluster should be shutdown.
+    this.clusterStatusTracker = new ClusterStatusTracker(getZooKeeper(), this);
+    this.clusterStatusTracker.start();
+    return this.activeMasterManager.blockUntilBecomingActiveMaster(startupStatus,
+        this.clusterStatusTracker);
+  }
+
+  /**
+   * Initialize all ZK based system trackers.
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  private void initializeZKBasedSystemTrackers() throws IOException,
+      InterruptedException, KeeperException {
+    this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf, this);
+    this.catalogTracker.start();
+
+    this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
+    this.assignmentManager = new AssignmentManager(this, serverManager,
+        this.catalogTracker, this.balancer, this.executorService);
+    zooKeeper.registerListenerFirst(assignmentManager);
+
+    this.regionServerTracker = new RegionServerTracker(zooKeeper, this,
+        this.serverManager);
+    this.regionServerTracker.start();
+
+    this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this,
+      this.serverManager);
+    this.drainingServerTracker.start();
+
+    // Set the cluster as up.  If new RSs, they'll be waiting on this before
+    // going ahead with their startup.
+    boolean wasUp = this.clusterStatusTracker.isClusterUp();
+    if (!wasUp) this.clusterStatusTracker.setClusterUp();
+
+    LOG.info("Server active/primary master; " + this.serverName +
+        ", sessionid=0x" +
+        Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +
+        ", cluster-up flag was=" + wasUp);
+
+    // create the snapshot manager
+    this.snapshotManager = new SnapshotManager(this, this.metrics);
+  }
+
+  // Check if we should stop every second.
+  private Sleeper stopSleeper = new Sleeper(1000, this);
+  private void loop() {
+    while (!this.stopped) {
+      stopSleeper.sleep();
+    }
+  }
+
+  /**
+   * Finish initialization of HMaster after becoming the primary master.
+   *
+   * 
          
+   * 
        1. Initialize master components - file system manager, server manager,
+   *     assignment manager, region server tracker, catalog tracker, etc
          2. 
+   * 
        2. Start necessary service threads - rpc server, info server,
+   *     executor services, etc
          3. 
+   * 
        3. Set cluster as UP in ZooKeeper
          4. 
+   * 
        4. Wait for RegionServers to check-in
          5. 
+   * 
        5. Split logs and perform data recovery, if necessary
          6. 
+   * 
        6. Ensure assignment of root and meta regions
        7. 
+   * 
        8. Handle either fresh cluster start or master failover
          9. 
+   * 
        
+   * @param masterRecovery
+   *
+   * @throws IOException
+   * @throws InterruptedException
+   * @throws KeeperException
+   */
+  private void finishInitialization(MonitoredTask status, boolean masterRecovery)
+  throws IOException, InterruptedException, KeeperException {
+
+    isActiveMaster = true;
+
+    /*
+     * We are active master now... go initialize components we need to run.
+     * Note, there may be dross in zk from previous runs; it'll get addressed
+     * below after we determine if cluster startup or failover.
+     */
+
+    status.setStatus("Initializing Master file system");
+    this.masterActiveTime = System.currentTimeMillis();
+    // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
+    this.fileSystemManager = new MasterFileSystem(this, this, metrics, masterRecovery);
+
+    this.tableDescriptors =
+      new FSTableDescriptors(this.fileSystemManager.getFileSystem(),
+      this.fileSystemManager.getRootDir());
+
+    // publish cluster ID
+    status.setStatus("Publishing Cluster ID in ZooKeeper");
+    ClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
+    if (!masterRecovery) {
+      this.executorService = new ExecutorService(getServerName().toString());
+      this.serverManager = new ServerManager(this, this);
+    }
+
+
+    status.setStatus("Initializing ZK system trackers");
+    initializeZKBasedSystemTrackers();
+
+    if (!masterRecovery) {
+      // initialize master side coprocessors before we start handling requests
+      status.setStatus("Initializing master coprocessors");
+      this.cpHost = new MasterCoprocessorHost(this, this.conf);
+
+      // start up all service threads.
+      status.setStatus("Initializing master service threads");
+      startServiceThreads();
+    }
+
+    // Wait for region servers to report in.
+    this.serverManager.waitForRegionServers(status);
+    // Check zk for regionservers that are up but didn't register
+    for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
+      if (!this.serverManager.isServerOnline(sn)) {
+        // Not registered; add it.
+        LOG.info("Registering server found up in zk but who has not yet " +
+          "reported in: " + sn);
+        this.serverManager.recordNewServer(sn, HServerLoad.EMPTY_HSERVERLOAD);
+      }
+    }
+    if (!masterRecovery) {
+      this.assignmentManager.startTimeOutMonitor();
+    }
+
+    // get a list for previously failed RS which need recovery work
+    Set failedServers = this.fileSystemManager.getFailedServersFromLogFolders();
+    if (waitingOnLogSplitting) {
+      List servers = new ArrayList(failedServers);
+      this.fileSystemManager.splitAllLogs(servers);
+      failedServers.clear();
+    }
+
+    ServerName preRootServer = this.catalogTracker.getRootLocation();
+    if (preRootServer != null && failedServers.contains(preRootServer)) {
+      // create recovered edits file for _ROOT_ server
+      this.fileSystemManager.splitAllLogs(preRootServer);
+      failedServers.remove(preRootServer);
+    }
+
+    this.initializationBeforeMetaAssignment = true;
+    // Make sure root assigned before proceeding.
+    if (!assignRoot(status)) return;
+
+    // SSH should enabled for ROOT before META region assignment
+    // because META region assignment is depending on ROOT server online.
+    this.serverManager.enableSSHForRoot();
+
+    // log splitting for .META. server
+    ServerName preMetaServer = this.catalogTracker.getMetaLocationOrReadLocationFromRoot();
+    if (preMetaServer != null && failedServers.contains(preMetaServer)) {
+      // create recovered edits file for .META. server
+      this.fileSystemManager.splitAllLogs(preMetaServer);
+      failedServers.remove(preMetaServer);
+    }
+
+    // Make sure meta assigned before proceeding.
+    if (!assignMeta(status, ((masterRecovery) ? null : preMetaServer), preRootServer)) return;
+
+    enableServerShutdownHandler();
+
+    // handle other dead servers in SSH
+    status.setStatus("Submit log splitting work of non-meta region servers");
+    for (ServerName curServer : failedServers) {
+      this.serverManager.expireServer(curServer);
+    }
+
+    // Update meta with new HRI if required. i.e migrate all HRI with HTD to
+    // HRI with out HTD in meta and update the status in ROOT. This must happen
+    // before we assign all user regions or else the assignment will fail.
+    // TODO: Remove this when we do 0.94.
+    org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD.
+      updateMetaWithNewHRI(this);
+
+    // Fixup assignment manager status
+    status.setStatus("Starting assignment manager");
+    this.assignmentManager.joinCluster();
+
+    this.balancer.setClusterStatus(getClusterStatus());
+    this.balancer.setMasterServices(this);
+
+    // Fixing up missing daughters if any
+    status.setStatus("Fixing up missing daughters");
+    fixupDaughters(status);
+
+    if (!masterRecovery) {
+      // Start meta catalog janitor after meta and regions have
+      // been assigned.
+      status.setStatus("Starting catalog janitor");
+      this.catalogJanitorChore = new CatalogJanitor(this, this);
+      startCatalogJanitorChore();
+      registerMBean();
+    }
+
+    status.markComplete("Initialization successful");
+    LOG.info("Master has completed initialization");
+    if (this.cpHost != null) {
+      try {
+        this.cpHost.preMasterInitialization();
+      } catch (IOException e) {
+        LOG.error("Coprocessor preMasterInitialization() hook failed", e);
+      }
+    }
+    if (!masterRecovery) {
+      status.setStatus("Starting balancer");
+      this.balancerChore = getAndStartBalancerChore(this);
+    }
+    initialized = true;
+
+    // clear the dead servers with same host name and port of online server because we are not
+    // removing dead server with same hostname and port of rs which is trying to check in before
+    // master initialization. See HBASE-5916.
+    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
+
+    if (!masterRecovery) {
+      if (this.cpHost != null) {
+        // don't let cp initialization errors kill the master
+        try {
+          this.cpHost.postStartMaster();
+        } catch (IOException ioe) {
+          LOG.error("Coprocessor postStartMaster() hook failed", ioe);
+        }
+      }
+    }
+  }
+
+  /**
+   * If ServerShutdownHandler is disabled, we enable it and expire those dead
+   * but not expired servers.
+   *
+   * @throws IOException
+   */
+  private void enableServerShutdownHandler() throws IOException {
+    if (!serverShutdownHandlerEnabled) {
+      serverShutdownHandlerEnabled = true;
+      this.serverManager.expireDeadNotExpiredServers();
+    }
+  }
+
+  /**
+   * Useful for testing purpose also where we have
+   * master restart scenarios.
+   */
+  protected void startCatalogJanitorChore() {
+    Threads.setDaemonThreadRunning(catalogJanitorChore.getThread());
+  }
+
+  /**
+   * Check -ROOT- is assigned. If not, assign it.
+   * @param status MonitoredTask
+   * @throws InterruptedException
+   * @throws IOException
+   * @throws KeeperException
+   */
+  private boolean assignRoot(MonitoredTask status)
+  throws InterruptedException, IOException, KeeperException {
+    int assigned = 0;
+    long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
+
+    // Work on ROOT region.  Is it in zk in transition?
+    status.setStatus("Assigning ROOT region");
+    boolean rit = this.assignmentManager.
+      processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.ROOT_REGIONINFO);
+    ServerName currentRootServer = null;
+    boolean rootRegionLocation = catalogTracker.verifyRootRegionLocation(timeout);
+    if (!rit && !rootRegionLocation) {
+      currentRootServer = this.catalogTracker.getRootLocation();
+      splitLogAndExpireIfOnline(currentRootServer);
+      this.assignmentManager.assignRoot();
+      waitForRootAssignment();
+      if (!this.assignmentManager.isRegionAssigned(HRegionInfo.ROOT_REGIONINFO) || this.stopped) {
+        return false;
+      }
+      assigned++;
+    } else if (rit && !rootRegionLocation) {
+      waitForRootAssignment();
+      if (!this.assignmentManager.isRegionAssigned(HRegionInfo.ROOT_REGIONINFO) || this.stopped) {
+        return false;
+      }
+      assigned++;
+    } else {
+      // Region already assigned. We didn't assign it. Add to in-memory state.
+      this.assignmentManager.regionOnline(HRegionInfo.ROOT_REGIONINFO,
+          this.catalogTracker.getRootLocation());
+    }
+    // Enable the ROOT table if on process fail over the RS containing ROOT
+    // was active.
+    enableCatalogTables(Bytes.toString(HConstants.ROOT_TABLE_NAME));
+    LOG.info("-ROOT- assigned=" + assigned + ", rit=" + rit +
+      ", location=" + catalogTracker.getRootLocation());
+
+    status.setStatus("ROOT assigned.");
+    return true;
+  }
+
+  /**
+   * Check .META. is assigned. If not, assign it.
+   * @param status MonitoredTask
+   * @param previousMetaServer ServerName of previous meta region server before current start up
+   * @param previousRootServer ServerName of previous root region server before current start up
+   * @throws InterruptedException
+   * @throws IOException
+   * @throws KeeperException
+   */
+  private boolean assignMeta(MonitoredTask status, ServerName previousMetaServer,
+      ServerName previousRootServer)
+      throws InterruptedException,
+      IOException, KeeperException {
+    int assigned = 0;
+    long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);
+
+    status.setStatus("Assigning META region");
+    boolean rit =
+        this.assignmentManager
+            .processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);
+    boolean metaRegionLocation = this.catalogTracker.verifyMetaRegionLocation(timeout);
+    if (!rit && !metaRegionLocation) {
+      ServerName currentMetaServer =
+          (previousMetaServer != null) ? previousMetaServer : this.catalogTracker
+              .getMetaLocationOrReadLocationFromRoot();
+      if (currentMetaServer != null && !currentMetaServer.equals(previousRootServer)) {
+        fileSystemManager.splitAllLogs(currentMetaServer);
+        if (this.serverManager.isServerOnline(currentMetaServer)) {
+          this.serverManager.expireServer(currentMetaServer);
+        }
+      }
+      assignmentManager.assignMeta();
+      enableSSHandWaitForMeta();
+      if (!this.assignmentManager.isRegionAssigned(HRegionInfo.FIRST_META_REGIONINFO)
+          || this.stopped) {
+        return false;
+      }
+      assigned++;
+    } else if (rit && !metaRegionLocation) {
+      enableSSHandWaitForMeta();
+      if (!this.assignmentManager.isRegionAssigned(HRegionInfo.FIRST_META_REGIONINFO)
+          || this.stopped) {
+        return false;
+      }
+      assigned++;
+    } else {
+      // Region already assigned. We didnt' assign it. Add to in-memory state.
+      this.assignmentManager.regionOnline(HRegionInfo.FIRST_META_REGIONINFO,
+        this.catalogTracker.getMetaLocation());
+    }
+    enableCatalogTables(Bytes.toString(HConstants.META_TABLE_NAME));
+    LOG.info(".META. assigned=" + assigned + ", rit=" + rit + ", location="
+        + catalogTracker.getMetaLocation());
+    status.setStatus("META assigned.");
+    return true;
+  }
+
+  private void enableSSHandWaitForMeta() throws IOException,
+      InterruptedException {
+    enableServerShutdownHandler();
+    this.catalogTracker.waitForMeta();
+    // Above check waits for general meta availability but this does not
+    // guarantee that the transition has completed
+    this.assignmentManager
+        .waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO);
+  }
+
+  private void waitForRootAssignment() throws InterruptedException, IOException {
+    // Enable SSH for ROOT to prevent a newly assigned ROOT crashes again before global SSH is
+    // enabled
+    this.serverManager.enableSSHForRoot();
+    this.catalogTracker.waitForRoot();
+    // This guarantees that the transition has completed
+    this.assignmentManager.waitForAssignment(HRegionInfo.ROOT_REGIONINFO);
+  }
+
+  private void enableCatalogTables(String catalogTableName) {
+    if (!this.assignmentManager.getZKTable().isEnabledTable(catalogTableName)) {
+      this.assignmentManager.setEnabledTable(catalogTableName);
+    }
+  }
+
+  void fixupDaughters(final MonitoredTask status) throws IOException {
+    final Map offlineSplitParents =
+      new HashMap();
+    // This visitor collects offline split parents in the .META. table
+    MetaReader.Visitor visitor = new MetaReader.Visitor() {
+      @Override
+      public boolean visit(Result r) throws IOException {
+        if (r == null || r.isEmpty()) return true;
+        HRegionInfo info =
+          MetaReader.parseHRegionInfoFromCatalogResult(
+            r, HConstants.REGIONINFO_QUALIFIER);
+        if (info == null) return true; // Keep scanning
+        if (info.isOffline() && info.isSplit()) {
+          offlineSplitParents.put(info, r);
+        }
+        // Returning true means "keep scanning"
+        return true;
+      }
+    };
+    // Run full scan of .META. catalog table passing in our custom visitor
+    MetaReader.fullScan(this.catalogTracker, visitor);
+    // Now work on our list of found parents. See if any we can clean up.
+    int fixups = 0;
+    for (Map.Entry e : offlineSplitParents.entrySet()) {
+      fixups += ServerShutdownHandler.fixupDaughters(
+          e.getValue(), assignmentManager, catalogTracker);
+    }
+    if (fixups != 0) {
+      LOG.info("Scanned the catalog and fixed up " + fixups +
+        " missing daughter region(s)");
+    }
+  }
+
+  /**
+   * Expire a server if we find it is one of the online servers.
+   * @param sn ServerName to check.
+   * @throws IOException
+   */
+  private void splitLogAndExpireIfOnline(final ServerName sn)
+      throws IOException {
+    if (sn == null || !serverManager.isServerOnline(sn)) {
+      return;
+    }
+    LOG.info("Forcing splitLog and expire of " + sn);
+    if (this.shouldSplitMetaSeparately) {
+      fileSystemManager.splitMetaLog(sn);
+      fileSystemManager.splitLog(sn);
+    } else {
+      fileSystemManager.splitAllLogs(sn);
+    }
+    serverManager.expireServer(sn);
+  }
+
+  @Override
+  public ProtocolSignature getProtocolSignature(
+      String protocol, long version, int clientMethodsHashCode)
+  throws IOException {
+    if (HMasterInterface.class.getName().equals(protocol)) {
+      return new ProtocolSignature(HMasterInterface.VERSION, null);
+    } else if (HMasterRegionInterface.class.getName().equals(protocol)) {
+      return new ProtocolSignature(HMasterRegionInterface.VERSION, null);
+    }
+    throw new IOException("Unknown protocol: " + protocol);
+  }
+
+  public long getProtocolVersion(String protocol, long clientVersion) {
+    if (HMasterInterface.class.getName().equals(protocol)) {
+      return HMasterInterface.VERSION;
+    } else if (HMasterRegionInterface.class.getName().equals(protocol)) {
+      return HMasterRegionInterface.VERSION;
+    }
+    // unknown protocol
+    LOG.warn("Version requested for unimplemented protocol: "+protocol);
+    return -1;
+  }
+
+  @Override
+  public TableDescriptors getTableDescriptors() {
+    return this.tableDescriptors;
+  }
+
+  /** @return InfoServer object. Maybe null.*/
+  public InfoServer getInfoServer() {
+    return this.infoServer;
+  }
+
+  @Override
+  public Configuration getConfiguration() {
+    return this.conf;
+  }
+
+  @Override
+  public ServerManager getServerManager() {
+    return this.serverManager;
+  }
+
+  @Override
+  public ExecutorService getExecutorService() {
+    return this.executorService;
+  }
+
+  @Override
+  public MasterFileSystem getMasterFileSystem() {
+    return this.fileSystemManager;
+  }
+
+  /**
+   * Get the ZK wrapper object - needed by master_jsp.java
+   * @return the zookeeper wrapper
+   */
+  public ZooKeeperWatcher getZooKeeperWatcher() {
+    return this.zooKeeper;
+  }
+
+  /*
+   * Start up all services. If any of these threads gets an unhandled exception
+   * then they just die with a logged message.  This should be fine because
+   * in general, we do not expect the master to get such unhandled exceptions
+   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if
+   *  need to install an unexpected exception handler.
+   */
+  private void startServiceThreads() throws IOException{
+
+   // Start the executor service pools
+   this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,
+      conf.getInt("hbase.master.executor.openregion.threads", 5));
+   this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,
+      conf.getInt("hbase.master.executor.closeregion.threads", 5));
+   this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,
+      conf.getInt("hbase.master.executor.serverops.threads", 3));
+   this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,
+      conf.getInt("hbase.master.executor.serverops.threads", 5));
+
+   // We depend on there being only one instance of this executor running
+   // at a time.  To do concurrency, would need fencing of enable/disable of
+   // tables.
+   this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
+
+   // Start log cleaner thread
+   String n = Thread.currentThread().getName();
+   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000);
+   this.logCleaner =
+      new LogCleaner(cleanerInterval,
+         this, conf, getMasterFileSystem().getFileSystem(),
+         getMasterFileSystem().getOldLogDir());
+         Threads.setDaemonThreadRunning(logCleaner.getThread(), n + ".oldLogCleaner");
+
+   //start the hfile archive cleaner thread
+    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
+    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()
+        .getFileSystem(), archiveDir);
+    Threads.setDaemonThreadRunning(hfileCleaner.getThread(), n + ".archivedHFileCleaner");
+
+   // Put up info server.
+   int port = this.conf.getInt("hbase.master.info.port", 60010);
+   if (port >= 0) {
+     String a = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0");
+     this.infoServer = new InfoServer(MASTER, a, port, false, this.conf);
+     this.infoServer.addServlet("status", "/master-status", MasterStatusServlet.class);
+     this.infoServer.addServlet("dump", "/dump", MasterDumpServlet.class);
+     this.infoServer.setAttribute(MASTER, this);
+     this.infoServer.start();
+    }
+
+   // Start the health checker
+   if (this.healthCheckChore != null) {
+     Threads.setDaemonThreadRunning(this.healthCheckChore.getThread(), n + ".healthChecker");
+   }
+
+    // Start allowing requests to happen.
+    this.rpcServer.openServer();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Started service threads");
+    }
+
+  }
+
+  private void stopServiceThreads() {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Stopping service threads");
+    }
+    if (this.rpcServer != null) this.rpcServer.stop();
+    // Clean up and close up shop
+    if (this.logCleaner!= null) this.logCleaner.interrupt();
+    if (this.hfileCleaner != null) this.hfileCleaner.interrupt();
+
+    if (this.infoServer != null) {
+      LOG.info("Stopping infoServer");
+      try {
+        this.infoServer.stop();
+      } catch (Exception ex) {
+        ex.printStackTrace();
+      }
+    }
+    if (this.executorService != null) this.executorService.shutdown();
+    if (this.healthCheckChore != null) {
+      this.healthCheckChore.interrupt();
+    }
+  }
+
+  private static Thread getAndStartBalancerChore(final HMaster master) {
+    String name = master.getServerName() + "-BalancerChore";
+    int balancerPeriod =
+      master.getConfiguration().getInt("hbase.balancer.period", 300000);
+    // Start up the load balancer chore
+    Chore chore = new Chore(name, balancerPeriod, master) {
+      @Override
+      protected void chore() {
+        master.balance();
+      }
+    };
+    return Threads.setDaemonThreadRunning(chore.getThread());
+  }
+
+  private void stopChores() {
+    if (this.balancerChore != null) {
+      this.balancerChore.interrupt();
+    }
+    if (this.catalogJanitorChore != null) {
+      this.catalogJanitorChore.interrupt();
+    }
+  }
+
+  @Override
+  public MapWritable regionServerStartup(final int port,
+    final long serverStartCode, final long serverCurrentTime)
+  throws IOException {
+    // Register with server manager
+    InetAddress ia = HBaseServer.getRemoteIp();
+    ServerName rs = this.serverManager.regionServerStartup(ia, port,
+      serverStartCode, serverCurrentTime);
+    // Send back some config info
+    MapWritable mw = createConfigurationSubset();
+    mw.put(new Text(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER),
+      new Text(rs.getHostname()));
+    return mw;
+  }
+
+  /**
+   * @return Subset of configuration to pass initializing regionservers: e.g.
+   * the filesystem to use and root directory to use.
+   */
+  protected MapWritable createConfigurationSubset() {
+    MapWritable mw = addConfig(new MapWritable(), HConstants.HBASE_DIR);
+    return addConfig(mw, "fs.default.name");
+  }
+
+  private MapWritable addConfig(final MapWritable mw, final String key) {
+    mw.put(new Text(key), new Text(this.conf.get(key)));
+    return mw;
+  }
+
+  @Override
+  public void regionServerReport(final byte [] sn, final HServerLoad hsl)
+  throws IOException {
+    this.serverManager.regionServerReport(ServerName.parseVersionedServerName(sn), hsl);
+    if (hsl != null && this.metrics != null) {
+      // Up our metrics.
+      this.metrics.incrementRequests(hsl.getTotalNumberOfRequests());
+    }
+  }
+
+  @Override
+  public void reportRSFatalError(byte [] sn, String errorText) {
+    String msg = "Region server " + Bytes.toString(sn) +
+      " reported a fatal error:\n" + errorText;
+    LOG.error(msg);
+    rsFatals.add(msg);
+  }
+
+  public boolean isMasterRunning() {
+    return !isStopped();
+  }
+
+  /**
+   * @return Maximum time we should run balancer for
+   */
+  private int getBalancerCutoffTime() {
+    int balancerCutoffTime =
+      getConfiguration().getInt("hbase.balancer.max.balancing", -1);
+    if (balancerCutoffTime == -1) {
+      // No time period set so create one -- do half of balancer period.
+      int balancerPeriod =
+        getConfiguration().getInt("hbase.balancer.period", 300000);
+      balancerCutoffTime = balancerPeriod / 2;
+      // If nonsense period, set it to balancerPeriod
+      if (balancerCutoffTime <= 0) balancerCutoffTime = balancerPeriod;
+    }
+    return balancerCutoffTime;
+  }
+
+  @Override
+  public boolean balance() {
+    // if master not initialized, don't run balancer.
+    if (!this.initialized) {
+      LOG.debug("Master has not been initialized, don't run balancer.");
+      return false;
+    }
+    // If balance not true, don't run balancer.
+    if (!this.balanceSwitch) return false;
+    return balanceInternals();
+  }
+
+  public boolean balanceInternals() {
+    // Do this call outside of synchronized block.
+    int maximumBalanceTime = getBalancerCutoffTime();
+    long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;
+    boolean balancerRan;
+    synchronized (this.balancer) {
+      // Only allow one balance run at at time.
+      if (this.assignmentManager.isRegionsInTransition()) {
+        LOG.debug("Not running balancer because " +
+          this.assignmentManager.getRegionsInTransition().size() +
+          " region(s) in transition: " +
+          org.apache.commons.lang.StringUtils.
+            abbreviate(this.assignmentManager.getRegionsInTransition().toString(), 256));
+        return false;
+      }
+      if (this.serverManager.areDeadServersInProgress()) {
+        LOG.debug("Not running balancer because processing dead regionserver(s): " +
+          this.serverManager.getDeadServers());
+        return false;
+      }
+
+      if (this.cpHost != null) {
+        try {
+          if (this.cpHost.preBalance()) {
+            LOG.debug("Coprocessor bypassing balancer request");
+            return false;
+          }
+        } catch (IOException ioe) {
+          LOG.error("Error invoking master coprocessor preBalance()", ioe);
+          return false;
+        }
+      }
+
+      Map>> assignmentsByTable =
+        this.assignmentManager.getAssignmentsByTable();
+
+      List plans = new ArrayList();
+      for (Map> assignments : assignmentsByTable.values()) {
+        List partialPlans = this.balancer.balanceCluster(assignments);
+        if (partialPlans != null) plans.addAll(partialPlans);
+      }
+      int rpCount = 0;  // number of RegionPlans balanced so far
+      long totalRegPlanExecTime = 0;
+      balancerRan = plans != null;
+      if (plans != null && !plans.isEmpty()) {
+        for (RegionPlan plan: plans) {
+          LOG.info("balance " + plan);
+          long balStartTime = System.currentTimeMillis();
+          this.assignmentManager.balance(plan);
+          totalRegPlanExecTime += System.currentTimeMillis()-balStartTime;
+          rpCount++;
+          if (rpCount < plans.size() &&
+              // if performing next balance exceeds cutoff time, exit the loop
+              (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {
+            LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +
+              maximumBalanceTime);
+            break;
+          }
+        }
+      }
+      if (this.cpHost != null) {
+        try {
+          this.cpHost.postBalance();
+        } catch (IOException ioe) {
+          // balancing already succeeded so don't change the result
+          LOG.error("Error invoking master coprocessor postBalance()", ioe);
+        }
+      }
+    }
+    return balancerRan;
+  }
+
+  enum BalanceSwitchMode {
+    SYNC,
+    ASYNC
+  }
+  /**
+   * Assigns balancer switch according to BalanceSwitchMode
+   * @param b new balancer switch
+   * @param mode BalanceSwitchMode
+   * @return old balancer switch
+   */
+  public boolean switchBalancer(final boolean b, BalanceSwitchMode mode) {
+    boolean oldValue = this.balanceSwitch;
+    boolean newValue = b;
+    try {
+      if (this.cpHost != null) {
+        newValue = this.cpHost.preBalanceSwitch(newValue);
+      }
+      if (mode == BalanceSwitchMode.SYNC) {
+        synchronized (this.balancer) {
+          this.balanceSwitch = newValue;
+        }
+      } else {
+        this.balanceSwitch = newValue;
+      }
+      LOG.info("BalanceSwitch=" + newValue);
+      if (this.cpHost != null) {
+        this.cpHost.postBalanceSwitch(oldValue, newValue);
+      }
+    } catch (IOException ioe) {
+      LOG.warn("Error flipping balance switch", ioe);
+    }
+    return oldValue;
+  }
+
+  @Override
+  public boolean synchronousBalanceSwitch(final boolean b) {
+    return switchBalancer(b, BalanceSwitchMode.SYNC);
+  }
+
+  @Override
+  public boolean balanceSwitch(final boolean b) {
+    return switchBalancer(b, BalanceSwitchMode.ASYNC);
+  }
+
+  /**
+   * Switch for the background CatalogJanitor thread.
+   * Used for testing.  The thread will continue to run.  It will just be a noop
+   * if disabled.
+   * @param b If false, the catalog janitor won't do anything.
+   */
+  public void setCatalogJanitorEnabled(final boolean b) {
+    ((CatalogJanitor)this.catalogJanitorChore).setEnabled(b);
+  }
+
+  @Override
+  public void move(final byte[] encodedRegionName, final byte[] destServerName)
+  throws UnknownRegionException {
+    Pair p =
+      this.assignmentManager.getAssignment(encodedRegionName);
+    if (p == null)
+      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));
+    ServerName dest = null;
+    if (destServerName == null || destServerName.length == 0) {
+      LOG.info("Passed destination servername is null or empty so choosing a server at random");
+      List destServers = this.serverManager.getOnlineServersList();
+      if (destServers.size() > 1) {
+        destServers.remove(p.getSecond());
+      }
+      // If i have only one RS then destination can be null.
+      dest = balancer.randomAssignment(p.getFirst(), destServers);
+    } else {
+      dest = new ServerName(Bytes.toString(destServerName));
+    }
+
+    // Now we can do the move
+    RegionPlan rp = new RegionPlan(p.getFirst(), p.getSecond(), dest);
+
+    try {
+      checkInitialized();
+      if (this.cpHost != null) {
+        if (this.cpHost.preMove(p.getFirst(), p.getSecond(), dest)) {
+          return;
+        }
+      }
+      LOG.info("Added move plan " + rp + ", running balancer");
+      this.assignmentManager.balance(rp);
+      if (this.cpHost != null) {
+        this.cpHost.postMove(p.getFirst(), p.getSecond(), dest);
+      }
+    } catch (IOException ioe) {
+      UnknownRegionException ure = new UnknownRegionException(
+          Bytes.toStringBinary(encodedRegionName));
+      ure.initCause(ioe);
+      throw ure;
+    }
+  }
+
+  public void createTable(HTableDescriptor hTableDescriptor,
+    byte [][] splitKeys)
+  throws IOException {
+    if (!isMasterRunning()) {
+      throw new MasterNotRunningException();
+    }
+
+    HRegionInfo [] newRegions = getHRegionInfos(hTableDescriptor, splitKeys);
+    checkInitialized();
+    if (cpHost != null) {
+      cpHost.preCreateTable(hTableDescriptor, newRegions);
+    }
+
+    this.executorService.submit(new CreateTableHandler(this,
+      this.fileSystemManager, this.serverManager, hTableDescriptor, conf,
+      newRegions, catalogTracker, assignmentManager));
+
+    if (cpHost != null) {
+      cpHost.postCreateTable(hTableDescriptor, newRegions);
+    }
+  }
+
+  private HRegionInfo[] getHRegionInfos(HTableDescriptor hTableDescriptor,
+    byte[][] splitKeys) {
+    HRegionInfo[] hRegionInfos = null;
+    if (splitKeys == null || splitKeys.length == 0) {
+      hRegionInfos = new HRegionInfo[]{
+          new HRegionInfo(hTableDescriptor.getName(), null, null)};
+    } else {
+      int numRegions = splitKeys.length + 1;
+      hRegionInfos = new HRegionInfo[numRegions];
+      byte[] startKey = null;
+      byte[] endKey = null;
+      for (int i = 0; i < numRegions; i++) {
+        endKey = (i == splitKeys.length) ? null : splitKeys[i];
+        hRegionInfos[i] =
+            new HRegionInfo(hTableDescriptor.getName(), startKey, endKey);
+        startKey = endKey;
+      }
+    }
+    return hRegionInfos;
+  }
+
+  private static boolean isCatalogTable(final byte [] tableName) {
+    return Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME) ||
+           Bytes.equals(tableName, HConstants.META_TABLE_NAME);
+  }
+
+  @Override
+  public void deleteTable(final byte [] tableName) throws IOException {
+    checkInitialized();
+    if (cpHost != null) {
+      cpHost.preDeleteTable(tableName);
+    }
+    this.executorService.submit(new DeleteTableHandler(tableName, this, this));
+    if (cpHost != null) {
+      cpHost.postDeleteTable(tableName);
+    }
+  }
+
+  /**
+   * Get the number of regions of the table that have been updated by the alter.
+   *
+   * @return Pair indicating the number of regions updated Pair.getFirst is the
+   *         regions that are yet to be updated Pair.getSecond is the total number
+   *         of regions of the table
+   * @throws IOException
+   */
+  public Pair getAlterStatus(byte[] tableName)
+  throws IOException {
+    return this.assignmentManager.getReopenStatus(tableName);
+  }
+
+  public void addColumn(byte [] tableName, HColumnDescriptor column)
+  throws IOException {
+    checkInitialized();
+    if (cpHost != null) {
+      if (cpHost.preAddColumn(tableName, column)) {
+        return;
+      }
+    }
+    new TableAddFamilyHandler(tableName, column, this, this).process();
+    if (cpHost != null) {
+      cpHost.postAddColumn(tableName, column);
+    }
+  }
+
+  public void modifyColumn(byte [] tableName, HColumnDescriptor descriptor)
+  throws IOException {
+    checkInitialized();
+    if (cpHost != null) {
+      if (cpHost.preModifyColumn(tableName, descriptor)) {
+        return;
+      }
+    }
+    new TableModifyFamilyHandler(tableName, descriptor, this, this).process();
+    if (cpHost != null) {
+      cpHost.postModifyColumn(tableName, descriptor);
+    }
+  }
+
+  public void deleteColumn(final byte [] tableName, final byte [] c)
+  throws IOException {
+    checkInitialized();
+    if (cpHost != null) {
+      if (cpHost.preDeleteColumn(tableName, c)) {
+        return;
+      }
+    }
+    new TableDeleteFamilyHandler(tableName, c, this, this).process();
+    if (cpHost != null) {
+      cpHost.postDeleteColumn(tableName, c);
+    }
+  }
+
+  public void enableTable(final byte [] tableName) throws IOException {
+    checkInitialized();
+    if (cpHost != null) {
+      cpHost.preEnableTable(tableName);
+    }
+    this.executorService.submit(new EnableTableHandler(this, tableName,
+      catalogTracker, assignmentManager, false));
+
+    if (cpHost != null) {
+      cpHost.postEnableTable(tableName);
+    }
+  }
+
+  public void disableTable(final byte [] tableName) throws IOException {
+    checkInitialized();
+    if (cpHost != null) {
+      cpHost.preDisableTable(tableName);
+    }
+    this.executorService.submit(new DisableTableHandler(this, tableName,
+        catalogTracker, assignmentManager, false));
+
+    if (cpHost != null) {
+      cpHost.postDisableTable(tableName);
+    }
+  }
+
+  /**
+   * Return the region and current deployment for the region containing
+   * the given row. If the region cannot be found, returns null. If it
+   * is found, but not currently deployed, the second element of the pair
+   * may be null.
+   */
+  Pair getTableRegionForRow(
+      final byte [] tableName, final byte [] rowKey)
+  throws IOException {
+    final AtomicReference> result =
+      new AtomicReference>(null);
+
+    MetaScannerVisitor visitor =
+      new MetaScannerVisitorBase() {
+        @Override
+        public boolean processRow(Result data) throws IOException {
+          if (data == null || data.size() <= 0) {
+            return true;
+          }
+          Pair pair = MetaReader.parseCatalogResult(data);
+          if (pair == null) {
+            return false;
+          }
+          if (!Bytes.equals(pair.getFirst().getTableName(), tableName)) {
+            return false;
+          }
+          result.set(pair);
+          return true;
+        }
+    };
+
+    MetaScanner.metaScan(conf, visitor, tableName, rowKey, 1);
+    return result.get();
+  }
+
+  @Override
+  public void modifyTable(final byte[] tableName, HTableDescriptor htd)
+      throws IOException {
+    checkInitialized();
+    if (cpHost != null) {
+      cpHost.preModifyTable(tableName, htd);
+    }
+    TableEventHandler tblHandler = new ModifyTableHandler(tableName, htd, this, this);
+    this.executorService.submit(tblHandler);
+    // prevent client from querying status even before the event is being handled.
+    tblHandler.waitForEventBeingHandled();
+    if (cpHost != null) {
+      cpHost.postModifyTable(tableName, htd);
+    }
+  }
+
+  @Override
+  public void checkTableModifiable(final byte [] tableName)
+  throws IOException {
+    String tableNameStr = Bytes.toString(tableName);
+    if (isCatalogTable(tableName)) {
+      throw new IOException("Can't modify catalog tables");
+    }
+    if (!MetaReader.tableExists(getCatalogTracker(), tableNameStr)) {
+      throw new TableNotFoundException(tableNameStr);
+    }
+    if (!getAssignmentManager().getZKTable().
+        isDisabledTable(Bytes.toString(tableName))) {
+      throw new TableNotDisabledException(tableName);
+    }
+  }
+
+  public void clearFromTransition(HRegionInfo hri) {
+    if (this.assignmentManager.isRegionInTransition(hri) != null) {
+      this.assignmentManager.regionOffline(hri);
+    }
+  }
+
+  /**
+   * @return cluster status
+   */
+  public ClusterStatus getClusterStatus() {
+    // Build Set of backup masters from ZK nodes
+    List backupMasterStrings;
+    try {
+      backupMasterStrings = ZKUtil.listChildrenNoWatch(this.zooKeeper,
+                              this.zooKeeper.backupMasterAddressesZNode);
+    } catch (KeeperException e) {
+      LOG.warn(this.zooKeeper.prefix("Unable to list backup servers"), e);
+      backupMasterStrings = new ArrayList(0);
+    }
+    List backupMasters = new ArrayList(
+                                          backupMasterStrings.size());
+    for (String s: backupMasterStrings) {
+      try {
+        byte[] bytes = ZKUtil.getData(this.zooKeeper, ZKUtil.joinZNode(this.zooKeeper.backupMasterAddressesZNode, s));
+        if (bytes != null) {
+          backupMasters.add(ServerName.parseVersionedServerName(bytes));
+        }
+      } catch (KeeperException e) {
+        LOG.warn(this.zooKeeper.prefix("Unable to get information about " +
+                 "backup servers"), e);
+      }
+    }
+    Collections.sort(backupMasters, new Comparator() {
+      public int compare(ServerName s1, ServerName s2) {
+        return s1.getServerName().compareTo(s2.getServerName());
+      }});
+
+    return new ClusterStatus(VersionInfo.getVersion(),
+      this.fileSystemManager.getClusterId(),
+      this.serverManager.getOnlineServers(),
+      this.serverManager.getDeadServers(),
+      this.serverName,
+      backupMasters,
+      this.assignmentManager.getRegionsInTransition(),
+      this.getCoprocessors());
+  }
+
+  public String getClusterId() {
+    return fileSystemManager.getClusterId();
+  }
+
+  /**
+   * The set of loaded coprocessors is stored in a static set. Since it's
+   * statically allocated, it does not require that HMaster's cpHost be
+   * initialized prior to accessing it.
+   * @return a String representation of the set of names of the loaded
+   * coprocessors.
+   */
+  public static String getLoadedCoprocessors() {
+    return CoprocessorHost.getLoadedCoprocessors().toString();
+  }
+
+  /**
+   * @return timestamp in millis when HMaster was started.
+   */
+  public long getMasterStartTime() {
+    return masterStartTime;
+  }
+
+  /**
+   * @return timestamp in millis when HMaster became the active master.
+   */
+  public long getMasterActiveTime() {
+    return masterActiveTime;
+  }
+
+  /**
+   * @return array of coprocessor SimpleNames.
+   */
+  public String[] getCoprocessors() {
+    Set masterCoprocessors =
+        getCoprocessorHost().getCoprocessors();
+    return masterCoprocessors.toArray(new String[0]);
+  }
+
+  @Override
+  public void abort(final String msg, final Throwable t) {
+    if (cpHost != null) {
+      // HBASE-4014: dump a list of loaded coprocessors.
+      LOG.fatal("Master server abort: loaded coprocessors are: " +
+          getLoadedCoprocessors());
+    }
+
+    if (abortNow(msg, t)) {
+      if (t != null) LOG.fatal(msg, t);
+      else LOG.fatal(msg);
+      this.abort = true;
+      stop("Aborting");
+    }
+  }
+
+  /**
+   * We do the following in a different thread.  If it is not completed
+   * in time, we will time it out and assume it is not easy to recover.
+   *
+   * 1. Create a new ZK session. (since our current one is expired)
+   * 2. Try to become a primary master again
+   * 3. Initialize all ZK based system trackers.
+   * 4. Assign root and meta. (they are already assigned, but we need to update our
+   * internal memory state to reflect it)
+   * 5. Process any RIT if any during the process of our recovery.
+   *
+   * @return True if we could successfully recover from ZK session expiry.
+   * @throws InterruptedException
+   * @throws IOException
+   * @throws KeeperException
+   * @throws ExecutionException
+   */
+  private boolean tryRecoveringExpiredZKSession() throws InterruptedException,
+      IOException, KeeperException, ExecutionException {
+
+    this.zooKeeper.unregisterAllListeners();
+    // add back listeners which were registered before master initialization
+    // because they won't be added back in below Master re-initialization code
+    if (this.registeredZKListenersBeforeRecovery != null) {
+      for (ZooKeeperListener curListener : this.registeredZKListenersBeforeRecovery) {
+        this.zooKeeper.registerListener(curListener);
+      }
+    }
+
+    this.zooKeeper.reconnectAfterExpiration();
+
+    Callable callable = new Callable () {
+      public Boolean call() throws InterruptedException,
+          IOException, KeeperException {
+        MonitoredTask status =
+          TaskMonitor.get().createStatus("Recovering expired ZK session");
+        try {
+          if (!becomeActiveMaster(status)) {
+            return Boolean.FALSE;
+          }
+          serverManager.disableSSHForRoot();
+          serverShutdownHandlerEnabled = false;
+          initialized = false;
+          finishInitialization(status, true);
+          return Boolean.TRUE;
+        } finally {
+          status.cleanup();
+        }
+      }
+    };
+
+    long timeout =
+      conf.getLong("hbase.master.zksession.recover.timeout", 300000);
+    java.util.concurrent.ExecutorService executor =
+      Executors.newSingleThreadExecutor();
+    Future result = executor.submit(callable);
+    executor.shutdown();
+    if (executor.awaitTermination(timeout, TimeUnit.MILLISECONDS)
+        && result.isDone()) {
+      Boolean recovered = result.get();
+      if (recovered != null) {
+        return recovered.booleanValue();
+      }
+    }
+    executor.shutdownNow();
+    return false;
+  }
+
+  /**
+   * Check to see if the current trigger for abort is due to ZooKeeper session
+   * expiry, and If yes, whether we can recover from ZK session expiry.
+   *
+   * @param msg Original abort message
+   * @param t   The cause for current abort request
+   * @return true if we should proceed with abort operation, false other wise.
+   */
+  private boolean abortNow(final String msg, final Throwable t) {
+    if (!this.isActiveMaster) {
+      return true;
+    }
+    if (t != null && t instanceof KeeperException.SessionExpiredException) {
+      try {
+        LOG.info("Primary Master trying to recover from ZooKeeper session " +
+            "expiry.");
+        return !tryRecoveringExpiredZKSession();
+      } catch (Throwable newT) {
+        LOG.error("Primary master encountered unexpected exception while " +
+            "trying to recover from ZooKeeper session" +
+            " expiry. Proceeding with server abort.", newT);
+      }
+    }
+    return true;
+  }
+
+  @Override
+  public ZooKeeperWatcher getZooKeeper() {
+    return zooKeeper;
+  }
+
+  @Override
+  public MasterCoprocessorHost getCoprocessorHost() {
+    return cpHost;
+  }
+
+  @Override
+  public ServerName getServerName() {
+    return this.serverName;
+  }
+
+  @Override
+  public CatalogTracker getCatalogTracker() {
+    return catalogTracker;
+  }
+
+  @Override
+  public AssignmentManager getAssignmentManager() {
+    return this.assignmentManager;
+  }
+
+  public MemoryBoundedLogMessageBuffer getRegionServerFatalLogBuffer() {
+    return rsFatals;
+  }
+
+  @SuppressWarnings("deprecation")
+  @Override
+  public void shutdown() {
+    if (cpHost != null) {
+      try {
+        cpHost.preShutdown();
+      } catch (IOException ioe) {
+        LOG.error("Error call master coprocessor preShutdown()", ioe);
+      }
+    }
+    if (mxBean != null) {
+      MBeanUtil.unregisterMBean(mxBean);
+      mxBean = null;
+    }
+    if (this.assignmentManager != null) this.assignmentManager.shutdown();
+    if (this.serverManager != null) this.serverManager.shutdownCluster();
+
+    try {
+      if (this.clusterStatusTracker != null){
+        this.clusterStatusTracker.setClusterDown();
+      }
+    } catch (KeeperException e) {
+      if (e instanceof KeeperException.SessionExpiredException) {
+        LOG.warn("ZK session expired. Retry a new connection...");
+        try {
+          this.zooKeeper.reconnectAfterExpiration();
+          this.clusterStatusTracker.setClusterDown();
+        } catch (Exception ex) {
+          LOG.error("Retry setClusterDown failed", ex);
+        }
+      } else {
+        LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
+      }
+    }
+  }
+
+  @Override
+  public void stopMaster() {
+    if (cpHost != null) {
+      try {
+        cpHost.preStopMaster();
+      } catch (IOException ioe) {
+        LOG.error("Error call master coprocessor preStopMaster()", ioe);
+      }
+    }
+    stop("Stopped by " + Thread.currentThread().getName());
+  }
+
+  @Override
+  public void stop(final String why) {
+    LOG.info(why);
+    this.stopped = true;
+    // We wake up the stopSleeper to stop immediately
+    stopSleeper.skipSleepCycle();
+    // If we are a backup master, we need to interrupt wait
+    if (this.activeMasterManager != null) {
+      synchronized (this.activeMasterManager.clusterHasActiveMaster) {
+        this.activeMasterManager.clusterHasActiveMaster.notifyAll();
+      }
+    }
+    // If no region server is online then master may stuck waiting on -ROOT- and .META. to come on
+    // line. See HBASE-8422.
+    if (this.catalogTracker != null && this.serverManager.getOnlineServers().isEmpty()) {
+      this.catalogTracker.stop();
+    }
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.stopped;
+  }
+
+  public boolean isAborted() {
+    return this.abort;
+  }
+
+  void checkInitialized() throws PleaseHoldException {
+    if (!this.initialized) {
+      throw new PleaseHoldException("Master is initializing");
+    }
+  }
+
+  /**
+   * Report whether this master is currently the active master or not.
+   * If not active master, we are parked on ZK waiting to become active.
+   *
+   * This method is used for testing.
+   *
+   * @return true if active master, false if not.
+   */
+  public boolean isActiveMaster() {
+    return isActiveMaster;
+  }
+
+  /**
+   * Report whether this master has completed with its initialization and is
+   * ready.  If ready, the master is also the active master.  A standby master
+   * is never ready.
+   *
+   * This method is used for testing.
+   *
+   * @return true if master is ready to go, false if not.
+   */
+  public boolean isInitialized() {
+    return initialized;
+  }
+
+  /**
+   * ServerShutdownHandlerEnabled is set false before completing
+   * assignRootAndMeta to prevent processing of ServerShutdownHandler.
+   * @return true if assignRootAndMeta has completed;
+   */
+  public boolean isServerShutdownHandlerEnabled() {
+    return this.serverShutdownHandlerEnabled;
+  }
+
+  public boolean shouldSplitMetaSeparately() {
+    return this.shouldSplitMetaSeparately;
+  }
+
+  /**
+   * Report whether this master has started initialization and is about to do meta region assignment
+   * @return true if master is in initialization & about to assign ROOT & META regions
+   */
+  public boolean isInitializationStartsMetaRegoinAssignment() {
+    return this.initializationBeforeMetaAssignment;
+  }
+
+  @Override
+  @Deprecated
+  public void assign(final byte[] regionName, final boolean force)
+      throws IOException {
+    assign(regionName);
+  }
+
+  @Override
+  public void assign(final byte [] regionName)throws IOException {
+    checkInitialized();
+    Pair pair =
+      MetaReader.getRegion(this.catalogTracker, regionName);
+    if (pair == null) throw new UnknownRegionException(Bytes.toString(regionName));
+    if (cpHost != null) {
+      if (cpHost.preAssign(pair.getFirst())) {
+        return;
+      }
+    }
+    assignRegion(pair.getFirst());
+    if (cpHost != null) {
+      cpHost.postAssign(pair.getFirst());
+    }
+  }
+
+
+
+  public void assignRegion(HRegionInfo hri) {
+    assignmentManager.assign(hri, true);
+  }
+
+  @Override
+  public void unassign(final byte [] regionName, final boolean force)
+  throws IOException {
+    checkInitialized();
+    Pair pair =
+      MetaReader.getRegion(this.catalogTracker, regionName);
+    if (pair == null) throw new UnknownRegionException(Bytes.toString(regionName));
+    HRegionInfo hri = pair.getFirst();
+    if (cpHost != null) {
+      if (cpHost.preUnassign(hri, force)) {
+        return;
+      }
+    }
+    if (force) {
+      this.assignmentManager.regionOffline(hri);
+      assignRegion(hri);
+    } else {
+      this.assignmentManager.unassign(hri, force);
+    }
+    if (cpHost != null) {
+      cpHost.postUnassign(hri, force);
+    }
+  }
+
+  /**
+   * Get HTD array for given tables
+   * @param tableNames
+   * @return HTableDescriptor[]
+   */
+  public HTableDescriptor[] getHTableDescriptors(List tableNames) {
+    List list =
+      new ArrayList(tableNames.size());
+    for (String s: tableNames) {
+      HTableDescriptor htd = null;
+      try {
+        htd = this.tableDescriptors.get(s);
+      } catch (IOException e) {
+        LOG.warn("Failed getting descriptor for " + s, e);
+      }
+      if (htd == null) continue;
+      list.add(htd);
+    }
+    return list.toArray(new HTableDescriptor [] {});
+  }
+
+  @Override
+  public  boolean registerProtocol(
+      Class protocol, T handler) {
+
+    /* No stacking of protocol handlers is currently allowed.  The
+     * first to claim wins!
+     */
+    if (protocolHandlers.containsKey(protocol)) {
+      LOG.error("Protocol "+protocol.getName()+
+          " already registered, rejecting request from "+
+          handler
+      );
+      return false;
+    }
+
+    protocolHandlers.putInstance(protocol, handler);
+    protocolHandlerNames.put(protocol.getName(), protocol);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Registered master protocol handler: protocol="+protocol.getName());
+    }
+    return true;
+  }
+
+  @Override
+  public ExecResult execCoprocessor(Exec call) throws IOException {
+    Class protocol = call.getProtocol();
+    if (protocol == null) {
+      String protocolName = call.getProtocolName();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Received dynamic protocol exec call with protocolName " + protocolName);
+      }
+      // detect the actual protocol class
+      protocol  = protocolHandlerNames.get(protocolName);
+      if (protocol == null) {
+        throw new HBaseRPC.UnknownProtocolException(protocol,
+            "No matching handler for master protocol "+protocolName);
+      }
+    }
+    if (!protocolHandlers.containsKey(protocol)) {
+      throw new HBaseRPC.UnknownProtocolException(protocol,
+          "No matching handler for protocol ");
+    }
+
+    CoprocessorProtocol handler = protocolHandlers.getInstance(protocol);
+    Object value;
+
+    try {
+      Method method = protocol.getMethod(
+          call.getMethodName(), call.getParameterClasses());
+      method.setAccessible(true);
+
+      value = method.invoke(handler, call.getParameters());
+    } catch (InvocationTargetException e) {
+      Throwable target = e.getTargetException();
+      if (target instanceof IOException) {
+        throw (IOException)target;
+      }
+      IOException ioe = new IOException(target.toString());
+      ioe.setStackTrace(target.getStackTrace());
+      throw ioe;
+    } catch (Throwable e) {
+      if (!(e instanceof IOException)) {
+        LOG.error("Unexpected throwable object ", e);
+      }
+      IOException ioe = new IOException(e.toString());
+      ioe.setStackTrace(e.getStackTrace());
+      throw ioe;
+    }
+
+    return new ExecResult(value);
+  }
+
+  /**
+   * Get all table descriptors
+   * @return All descriptors or null if none.
+   */
+  public HTableDescriptor [] getHTableDescriptors() {
+    Map descriptors = null;
+    try {
+      descriptors = this.tableDescriptors.getAll();
+    } catch (IOException e) {
+      LOG.warn("Failed getting all descriptors", e);
+    }
+    return descriptors == null?
+      null: descriptors.values().toArray(new HTableDescriptor [] {});
+  }
+
+  /**
+   * Compute the average load across all region servers.
+   * Currently, this uses a very naive computation - just uses the number of
+   * regions being served, ignoring stats about number of requests.
+   * @return the average load
+   */
+  public double getAverageLoad() {
+    return this.assignmentManager.getAverageLoad();
+  }
+
+  /**
+   * Special method, only used by hbck.
+   */
+  @Override
+  public void offline(final byte[] regionName) throws IOException {
+    Pair pair =
+      MetaReader.getRegion(this.catalogTracker, regionName);
+    if (pair == null) throw new UnknownRegionException(Bytes.toStringBinary(regionName));
+    HRegionInfo hri = pair.getFirst();
+    this.assignmentManager.regionOffline(hri);
+  }
+
+  /**
+   * Utility for constructing an instance of the passed HMaster class.
+   * @param masterClass
+   * @param conf
+   * @return HMaster instance.
+   */
+  public static HMaster constructMaster(Class masterClass,
+      final Configuration conf)  {
+    try {
+      Constructor c =
+        masterClass.getConstructor(Configuration.class);
+      return c.newInstance(conf);
+    } catch (InvocationTargetException ite) {
+      Throwable target = ite.getTargetException() != null?
+        ite.getTargetException(): ite;
+      if (target.getCause() != null) target = target.getCause();
+      throw new RuntimeException("Failed construction of Master: " +
+        masterClass.toString(), target);
+    } catch (Exception e) {
+      throw new RuntimeException("Failed construction of Master: " +
+        masterClass.toString() + ((e.getCause() != null)?
+          e.getCause().getMessage(): ""), e);
+    }
+  }
+
+  /**
+   * @see org.apache.hadoop.hbase.master.HMasterCommandLine
+   */
+  public static void main(String [] args) throws Exception {
+	VersionInfo.logVersion();
+    new HMasterCommandLine(HMaster.class).doMain(args);
+  }
+
+  /**
+   * Register bean with platform management server
+   */
+  @SuppressWarnings("deprecation")
+  void registerMBean() {
+    MXBeanImpl mxBeanInfo = MXBeanImpl.init(this);
+    MBeanUtil.registerMBean("Master", "Master", mxBeanInfo);
+    LOG.info("Registered HMaster MXBean");
+  }
+
+  public LoadBalancer getBalancer() {
+    return this.balancer;
+  }
+
+
+  @Override
+  public void setTableState(String tableName, String state) throws IOException {
+    TableState tableState = TableState.valueOf(state);
+    if (tableState == TableState.ENABLED) {
+      List tableRegions =
+          MetaReader.getTableRegions(this.catalogTracker, Bytes.toBytes(tableName));
+      List onlineRegions =
+          this.assignmentManager.getRegionsOfTable(Bytes.toBytes(tableName));
+      if (tableRegions.size() == onlineRegions.size()) {
+        this.assignmentManager.setEnabledTable(tableName);
+      } else {
+        LOG.warn("All the regions for the table are not assigned. Cannot set table " + tableName
+            + " as ENABLED.");
+      }
+    } else {
+      List onlineRegions =
+          this.assignmentManager.getRegionsOfTable(Bytes.toBytes(tableName));
+      if (onlineRegions.size() == 0) {
+        this.assignmentManager.setDisabledTable(tableName);
+      } else {
+        LOG.warn("All the regions for the table are not unassigned. Cannot set table " + tableName
+            + " as DISABLED.");
+      }
+    }
+  }
+
+  /**
+   * Exposed for Testing!
+   * @return the current hfile cleaner
+   */
+  public HFileCleaner getHFileCleaner() {
+    return this.hfileCleaner;
+  }
+
+  private boolean isHealthCheckerConfigured() {
+    String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
+    return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
+  }
+
+  /**
+   * Exposed for TESTING!
+   * @return the underlying snapshot manager
+   */
+  public SnapshotManager getSnapshotManagerForTesting() {
+    return this.snapshotManager;
+   }
+
+
+  /**
+   * Triggers an asynchronous attempt to take a snapshot.
+   * {@inheritDoc}
+   */
+  @Override
+  public long snapshot(final HSnapshotDescription request) throws IOException {
+    LOG.debug("Submitting snapshot request for:" +
+        SnapshotDescriptionUtils.toString(request.getProto()));
+    try {
+      this.snapshotManager.checkSnapshotSupport();
+    } catch (UnsupportedOperationException e) {
+      throw new IOException(e);
+    }
+
+    // get the snapshot information
+    SnapshotDescription snapshot = SnapshotDescriptionUtils.validate(request.getProto(),
+      this.conf);
+
+    snapshotManager.takeSnapshot(snapshot);
+
+    // send back the max amount of time the client should wait for the snapshot to complete
+    long waitTime = SnapshotDescriptionUtils.getMaxMasterTimeout(conf, snapshot.getType(),
+      SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME);
+    return waitTime;
+  }
+
+  /**
+   * List the currently available/stored snapshots. Any in-progress snapshots are ignored
+   */
+  @Override
+  public List getCompletedSnapshots() throws IOException {
+    List availableSnapshots = new ArrayList();
+    List snapshots = snapshotManager.getCompletedSnapshots();
+
+    // convert to writables
+    for (SnapshotDescription snapshot: snapshots) {
+      availableSnapshots.add(new HSnapshotDescription(snapshot));
+    }
+
+    return availableSnapshots;
+  }
+
+  /**
+   * Execute Delete Snapshot operation.
+   * @throws ServiceException wrapping SnapshotDoesNotExistException if specified snapshot did not
+   * exist.
+   */
+  @Override
+  public void deleteSnapshot(final HSnapshotDescription request) throws IOException {
+    try {
+      this.snapshotManager.checkSnapshotSupport();
+    } catch (UnsupportedOperationException e) {
+      throw new IOException(e);
+    }
+
+    snapshotManager.deleteSnapshot(request.getProto());
+  }
+
+  /**
+   * Checks if the specified snapshot is done.
+   * @return true if the snapshot is in file system ready to use,
+   * false if the snapshot is in the process of completing
+   * @throws ServiceException wrapping UnknownSnapshotException if invalid snapshot, or
+   * a wrapped HBaseSnapshotException with progress failure reason.
+   */
+  @Override
+  public boolean isSnapshotDone(final HSnapshotDescription request) throws IOException {
+    LOG.debug("Checking to see if snapshot from request:" +
+      SnapshotDescriptionUtils.toString(request.getProto()) + " is done");
+    return snapshotManager.isSnapshotDone(request.getProto());
+  }
+
+  /**
+   * Execute Restore/Clone snapshot operation.
+   *
+   * 
        If the specified table exists a "Restore" is executed, replacing the table
+   * schema and directory data with the content of the snapshot.
+   * The table must be disabled, or a UnsupportedOperationException will be thrown.
+   *
+   * 
        If the table doesn't exist a "Clone" is executed, a new table is created
+   * using the schema at the time of the snapshot, and the content of the snapshot.
+   *
+   * 
        The restore/clone operation does not require copying HFiles. Since HFiles
+   * are immutable the table can point to and use the same files as the original one.
+   */
+  @Override
+  public void restoreSnapshot(final HSnapshotDescription request) throws IOException {
+    try {
+      this.snapshotManager.checkSnapshotSupport();
+    } catch (UnsupportedOperationException e) {
+      throw new IOException(e);
+    }
+
+    snapshotManager.restoreSnapshot(request.getProto());
+  }
+
+  /**
+   * Returns the status of the requested snapshot restore/clone operation.
+   * This method is not exposed to the user, it is just used internally by HBaseAdmin
+   * to verify if the restore is completed.
+   *
+   * No exceptions are thrown if the restore is not running, the result will be "done".
+   *
+   * @return done true if the restore/clone operation is completed.
+   * @throws RestoreSnapshotExcepton if the operation failed.
+   */
+  @Override
+  public boolean isRestoreSnapshotDone(final HSnapshotDescription request) throws IOException {
+    return snapshotManager.isRestoreDone(request.getProto());
+  }
+}
+
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
new file mode 100644
index 0000000..e6fcce5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
@@ -0,0 +1,240 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.apache.hadoop.hbase.util.ServerCommandLine;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.zookeeper.KeeperException;
+
+public class HMasterCommandLine extends ServerCommandLine {
+  private static final Log LOG = LogFactory.getLog(HMasterCommandLine.class);
+
+  private static final String USAGE =
+    "Usage: Master [opts] start|stop\n" +
+    " start  Start Master. If local mode, start Master and RegionServer in same JVM\n" +
+    " stop   Start cluster shutdown; Master signals RegionServer shutdown\n" +
+    " where [opts] are:\n" +
+    "   --minServers=    Minimum RegionServers needed to host user tables.\n" +
+    "   --backup                  Master should start in backup mode";
+
+  private final Class masterClass;
+
+  public HMasterCommandLine(Class masterClass) {
+    this.masterClass = masterClass;
+  }
+
+  protected String getUsage() {
+    return USAGE;
+  }
+
+
+  public int run(String args[]) throws Exception {
+    Options opt = new Options();
+    opt.addOption("minServers", true, "Minimum RegionServers needed to host user tables");
+    opt.addOption("backup", false, "Do not try to become HMaster until the primary fails");
+
+
+    CommandLine cmd;
+    try {
+      cmd = new GnuParser().parse(opt, args);
+    } catch (ParseException e) {
+      LOG.error("Could not parse: ", e);
+      usage(null);
+      return -1;
+    }
+
+
+    if (cmd.hasOption("minServers")) {
+      String val = cmd.getOptionValue("minServers");
+      getConf().setInt("hbase.regions.server.count.min",
+                  Integer.valueOf(val));
+      LOG.debug("minServers set to " + val);
+    }
+
+    // check if we are the backup master - override the conf if so
+    if (cmd.hasOption("backup")) {
+      getConf().setBoolean(HConstants.MASTER_TYPE_BACKUP, true);
+    }
+
+    List remainingArgs = cmd.getArgList();
+    if (remainingArgs.size() != 1) {
+      usage(null);
+      return -1;
+    }
+
+    String command = remainingArgs.get(0);
+
+    if ("start".equals(command)) {
+      return startMaster();
+    } else if ("stop".equals(command)) {
+      return stopMaster();
+    } else {
+      usage("Invalid command: " + command);
+      return -1;
+    }
+  }
+
+  private int startMaster() {
+    Configuration conf = getConf();
+    try {
+      // If 'local', defer to LocalHBaseCluster instance.  Starts master
+      // and regionserver both in the one JVM.
+      if (LocalHBaseCluster.isLocal(conf)) {
+        final MiniZooKeeperCluster zooKeeperCluster =
+          new MiniZooKeeperCluster();
+        File zkDataPath = new File(conf.get(HConstants.ZOOKEEPER_DATA_DIR));
+        int zkClientPort = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 0);
+        if (zkClientPort == 0) {
+          throw new IOException("No config value for "
+              + HConstants.ZOOKEEPER_CLIENT_PORT);
+        }
+        zooKeeperCluster.setDefaultClientPort(zkClientPort);
+
+        // login the zookeeper server principal (if using security)
+        ZKUtil.loginServer(conf, "hbase.zookeeper.server.keytab.file",
+          "hbase.zookeeper.server.kerberos.principal", null);
+
+        int clientPort = zooKeeperCluster.startup(zkDataPath);
+        if (clientPort != zkClientPort) {
+          String errorMsg = "Could not start ZK at requested port of " +
+            zkClientPort + ".  ZK was started at port: " + clientPort +
+            ".  Aborting as clients (e.g. shell) will not be able to find " +
+            "this ZK quorum.";
+          System.err.println(errorMsg);
+          throw new IOException(errorMsg);
+        }
+        conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,
+                 Integer.toString(clientPort));
+        // Need to have the zk cluster shutdown when master is shutdown.
+        // Run a subclass that does the zk cluster shutdown on its way out.
+        LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 1,
+                                                          LocalHMaster.class, HRegionServer.class);
+        ((LocalHMaster)cluster.getMaster(0)).setZKCluster(zooKeeperCluster);
+        cluster.startup();
+        waitOnMasterThreads(cluster);
+      } else {
+        HMaster master = HMaster.constructMaster(masterClass, conf);
+        if (master.isStopped()) {
+          LOG.info("Won't bring the Master up as a shutdown is requested");
+          return -1;
+        }
+        master.start();
+        master.join();
+        if(master.isAborted())
+          throw new RuntimeException("HMaster Aborted");
+      }
+    } catch (Throwable t) {
+      LOG.error("Failed to start master", t);
+      return -1;
+    }
+    return 0;
+  }
+
+  private int stopMaster() {
+    HBaseAdmin adm = null;
+    try {
+      Configuration conf = getConf();
+      // Don't try more than once
+      conf.setInt("hbase.client.retries.number", 1);
+      adm = new HBaseAdmin(getConf());
+    } catch (MasterNotRunningException e) {
+      LOG.error("Master not running");
+      return -1;
+    } catch (ZooKeeperConnectionException e) {
+      LOG.error("ZooKeeper not available");
+      return -1;
+    }
+    try {
+      adm.shutdown();
+    } catch (Throwable t) {
+      LOG.error("Failed to stop master", t);
+      return -1;
+    }
+    return 0;
+  }
+
+  private void waitOnMasterThreads(LocalHBaseCluster cluster) throws InterruptedException{
+    List masters = cluster.getMasters();
+    List regionservers = cluster.getRegionServers();
+	  
+    if (masters != null) { 
+      for (JVMClusterUtil.MasterThread t : masters) {
+        t.join();
+        if(t.getMaster().isAborted()) {
+          closeAllRegionServerThreads(regionservers);
+          throw new RuntimeException("HMaster Aborted");
+        }
+      }
+    }
+  }
+
+  private static void closeAllRegionServerThreads(List regionservers) {
+    for(JVMClusterUtil.RegionServerThread t : regionservers){
+      t.getRegionServer().stop("HMaster Aborted; Bringing down regions servers");
+    }
+  }
+  
+  /*
+   * Version of master that will shutdown the passed zk cluster on its way out.
+   */
+  public static class LocalHMaster extends HMaster {
+    private MiniZooKeeperCluster zkcluster = null;
+
+    public LocalHMaster(Configuration conf)
+    throws IOException, KeeperException, InterruptedException {
+      super(conf);
+    }
+
+    @Override
+    public void run() {
+      super.run();
+      if (this.zkcluster != null) {
+        try {
+          this.zkcluster.shutdown();
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+      }
+    }
+
+    void setZKCluster(final MiniZooKeeperCluster zkcluster) {
+      this.zkcluster = zkcluster;
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java b/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
new file mode 100644
index 0000000..3999c5d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
@@ -0,0 +1,99 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Makes decisions about the placement and movement of Regions across
+ * RegionServers.
+ *
+ * 
        Cluster-wide load balancing will occur only when there are no regions in
+ * transition and according to a fixed period of a time using {@link #balanceCluster(Map)}.
+ *
+ * 
        Inline region placement with {@link #immediateAssignment} can be used when
+ * the Master needs to handle closed regions that it currently does not have
+ * a destination set for.  This can happen during master failover.
+ *
+ * 
        On cluster startup, bulk assignment can be used to determine
+ * locations for all Regions in a cluster.
+ *
+ * 
        This classes produces plans for the {@link AssignmentManager} to execute.
+ */
+public interface LoadBalancer extends Configurable {
+
+  /**
+   * Set the current cluster status.  This allows a LoadBalancer to map host name to a server
+   * @param st
+   */
+  public void setClusterStatus(ClusterStatus st);
+
+
+  /**
+   * Set the master service.
+   * @param masterServices
+   */
+  public void setMasterServices(MasterServices masterServices);
+
+  /**
+   * Perform the major balance operation
+   * @param clusterState
+   * @return List of plans
+   */
+  public List balanceCluster(Map> clusterState);
+
+  /**
+   * Perform a Round Robin assignment of regions.
+   * @param regions
+   * @param servers
+   * @return Map of servername to regioninfos
+   */
+  public Map> roundRobinAssignment(List regions, List servers);
+
+  /**
+   * Assign regions to the previously hosting region server
+   * @param regions
+   * @param servers
+   * @return List of plans
+   */
+  public Map> retainAssignment(Map regions, List servers);
+
+  /**
+   * Sync assign a region
+   * @param regions
+   * @param servers
+    * @return Map regioninfos to servernames
+   */
+  public Map immediateAssignment(List regions, List servers);
+
+  /**
+   * Get a random region server from the list
+   * @param regionInfo Region for which this selection is being done.
+   * @param servers
+   * @return Servername
+   */
+  public ServerName randomAssignment(HRegionInfo regionInfo, List servers);
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/LoadBalancerFactory.java b/src/main/java/org/apache/hadoop/hbase/master/LoadBalancerFactory.java
new file mode 100644
index 0000000..ab1d603
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/LoadBalancerFactory.java
@@ -0,0 +1,72 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master;
+
+import java.lang.reflect.Method;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * The class that creates a load balancer from a conf.
+ */
+public class LoadBalancerFactory {
+
+  private static final Log LOG = LogFactory.getLog(LoadBalancerFactory.class);
+
+  static Class secIndexLoadBalancerKlass = LoadBalancer.class;
+
+  /**
+   * Create a loadblanacer from the given conf.
+   * @param conf
+   * @return A {@link LoadBalancer}
+   */
+  public static LoadBalancer getLoadBalancer(Configuration conf) {
+    // Create the balancer
+    Class balancerKlass =
+        conf.getClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, DefaultLoadBalancer.class,
+          LoadBalancer.class);
+    boolean secondaryIndex = conf.getBoolean("hbase.use.secondary.index", false);
+    String secIndexBalancer = conf.get("hbase.index.loadbalancer.class");
+    if (secondaryIndex) {
+      if (secIndexBalancer == null) {
+        throw new RuntimeException(
+            "Secondary index load  balancer not configured. Configure the property hbase.index.loadbalancer.class");
+      }
+      try {
+        secIndexLoadBalancerKlass = Class.forName(secIndexBalancer.trim());
+        Object secIndexLoadBalancerInstance = secIndexLoadBalancerKlass.newInstance();
+        Method method = secIndexLoadBalancerKlass.getMethod("setDelegator", LoadBalancer.class);
+        method.invoke(secIndexLoadBalancerInstance,
+          (LoadBalancer) ReflectionUtils.newInstance(balancerKlass, conf));
+        return (LoadBalancer) secIndexLoadBalancerInstance;
+      } catch (Throwable t) {
+        LOG.error("Error while initializing/invoking method of seconday index load balancer.", t);
+        throw new RuntimeException("Not able to load the secondary index load balancer class.");
+      }
+
+    }
+    return ReflectionUtils.newInstance(balancerKlass, conf);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MXBean.java b/src/main/java/org/apache/hadoop/hbase/master/MXBean.java
new file mode 100644
index 0000000..41da8fb
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/MXBean.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.util.Map;
+
+import org.apache.hadoop.hbase.HServerLoad;
+
+/**
+ * This is the JMX management interface for Hbase master information
+ */
+public interface MXBean {
+
+  /**
+   * Required for MXBean implementation
+   */
+  public static interface RegionsInTransitionInfo {
+    /**
+     * Name of region in transition
+     */
+    public String getRegionName();
+    /**
+     * Current transition state
+     */
+    public String getRegionState();
+    /**
+     * Get Region Server name
+     */
+    public String getRegionServerName();
+    /**
+     * Get last update time
+     */
+    public long getLastUpdateTime();
+  }
+
+  /**
+   * Get ServerName
+   */
+  public String getServerName();
+
+  /**
+   * Get Average Load
+   * @return Average Load
+   */
+  public double getAverageLoad();
+
+  /**
+   * Get the Cluster ID
+   * @return Cluster ID
+   */
+  public String getClusterId();
+
+  /**
+   * Get the Zookeeper Quorum Info
+   * @return Zookeeper Quorum Info
+   */
+  public String getZookeeperQuorum();
+
+  /**
+   * Get the co-processors
+   * @return Co-processors
+   */
+  public String[] getCoprocessors();
+
+  /**
+   * Get hbase master start time
+   * @return Start time of master in milliseconds
+   */
+  public long getMasterStartTime();
+
+  /**
+   * Get the hbase master active time
+   * @return Time in milliseconds when master became active
+   */
+  public long getMasterActiveTime();
+
+  /**
+   * Whether this master is the active master
+   * @return True if this is the active master
+   */
+  public boolean getIsActiveMaster();
+
+  /**
+   * Get the live region servers
+   * @return Live region servers
+   */
+  public Map getRegionServers();
+
+  /**
+   * Get the dead region servers
+   * @return Dead region Servers
+   */
+  public String[] getDeadRegionServers();
+
+  /**
+   * Get information on regions in transition
+   * @return Regions in transition
+   */
+  public RegionsInTransitionInfo[] getRegionsInTransition();
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MXBeanImpl.java b/src/main/java/org/apache/hadoop/hbase/master/MXBeanImpl.java
new file mode 100644
index 0000000..45b8fe7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/MXBeanImpl.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
+
+/**
+ * Impl for exposing HMaster Information through JMX
+ */
+public class MXBeanImpl implements MXBean {
+
+  private final HMaster master;
+
+  private static MXBeanImpl instance = null;
+  public synchronized static MXBeanImpl init(final HMaster master) {
+    if (instance == null) {
+      instance = new MXBeanImpl(master);
+    }
+    return instance;
+  }
+
+  protected MXBeanImpl(final HMaster master) {
+    this.master = master;
+  }
+
+  @Override
+  public double getAverageLoad() {
+    return master.getAverageLoad();
+  }
+
+  @Override
+  public String getClusterId() {
+    return master.getClusterId();
+  }
+
+  @Override
+  public String getZookeeperQuorum() {
+    return master.getZooKeeperWatcher().getQuorum();
+  }
+
+  @Override
+  public String[] getCoprocessors() {
+    return master.getCoprocessors();
+  }
+
+  @Override
+  public long getMasterStartTime() {
+    return master.getMasterStartTime();
+  }
+
+  @Override
+  public long getMasterActiveTime() {
+    return master.getMasterActiveTime();
+  }
+
+  @Override
+  public Map getRegionServers() {
+    Map data = new HashMap();
+    for (final Entry entry :
+      master.getServerManager().getOnlineServers().entrySet()) {
+      data.put(entry.getKey().getServerName(),
+          entry.getValue());
+    }
+    return data;
+  }
+
+  @Override
+  public String[] getDeadRegionServers() {
+    List deadServers = new ArrayList();
+    for (ServerName name : master.getServerManager().getDeadServers()) {
+      deadServers.add(name.getHostAndPort());
+    }
+    return deadServers.toArray(new String[0]);
+  }
+
+  @Override
+  public RegionsInTransitionInfo[] getRegionsInTransition() {
+    List info =
+        new ArrayList();
+    for (final Entry entry :
+      master.getAssignmentManager().getRegionsInTransition().entrySet()) {
+      RegionsInTransitionInfo innerinfo = new RegionsInTransitionInfo() {
+
+        @Override
+        public String getRegionState() {
+          return entry.getValue().getState().toString();
+        }
+
+        @Override
+        public String getRegionName() {
+          return entry.getKey();
+        }
+
+        @Override
+        public long getLastUpdateTime() {
+          return entry.getValue().getStamp();
+        }
+
+        @Override
+        public String getRegionServerName() {
+          ServerName serverName = entry.getValue().getServerName();
+          if (serverName != null) {
+            return serverName.getServerName();
+          }
+          else {
+            return "";
+          }
+        }
+      };
+      info.add(innerinfo);
+    }
+    RegionsInTransitionInfo[] data =
+        new RegionsInTransitionInfo[info.size()];
+    info.toArray(data);
+    return data;
+  }
+
+  @Override
+  public String getServerName() {
+    return master.getServerName().getServerName();
+  }
+
+  @Override
+  public boolean getIsActiveMaster() {
+    return master.isActiveMaster();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
new file mode 100644
index 0000000..1e6c004
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -0,0 +1,1078 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.coprocessor.*;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+
+import java.io.IOException;
+
+/**
+ * Provides the coprocessor framework and environment for master oriented
+ * operations.  {@link HMaster} interacts with the loaded coprocessors
+ * through this class.
+ */
+public class MasterCoprocessorHost
+    extends CoprocessorHost {
+
+  private static final Log LOG = LogFactory.getLog(MasterCoprocessorHost.class);
+
+  /**
+   * Coprocessor environment extension providing access to master related
+   * services.
+   */
+  static class MasterEnvironment extends CoprocessorHost.Environment
+      implements MasterCoprocessorEnvironment {
+    private MasterServices masterServices;
+
+    public MasterEnvironment(final Class implClass, final Coprocessor impl,
+        final int priority, final int seq, final Configuration conf,
+        final MasterServices services) {
+      super(impl, priority, seq, conf);
+      this.masterServices = services;
+    }
+
+    public MasterServices getMasterServices() {
+      return masterServices;
+    }
+  }
+
+  private MasterServices masterServices;
+
+  MasterCoprocessorHost(final MasterServices services, final Configuration conf) {
+    this.conf = conf;
+    this.masterServices = services;
+    loadSystemCoprocessors(conf, MASTER_COPROCESSOR_CONF_KEY);
+  }
+
+  @Override
+  public MasterEnvironment createEnvironment(final Class implClass,
+      final Coprocessor instance, final int priority, final int seq,
+      final Configuration conf) {
+    for (Class c : implClass.getInterfaces()) {
+      if (CoprocessorProtocol.class.isAssignableFrom(c)) {
+        masterServices.registerProtocol(c, (CoprocessorProtocol)instance);
+        break;
+      }
+    }
+    return new MasterEnvironment(implClass, instance, priority, seq, conf,
+        masterServices);
+  }
+
+  @Override
+  protected void abortServer(final CoprocessorEnvironment env, final Throwable e) {
+    abortServer("master", masterServices, env, e);
+  }
+
+  /* Implementation of hooks for invoking MasterObservers */
+  public void preCreateTable(HTableDescriptor htd, HRegionInfo[] regions)
+    throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preCreateTable(ctx, htd, regions);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postCreateTable(HTableDescriptor htd, HRegionInfo[] regions)
+    throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+            ((MasterObserver)env.getInstance()).postCreateTable(ctx, htd, regions);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preCreateTableHandler(HTableDescriptor htd, HRegionInfo[] regions) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).preCreateTableHandler(ctx, htd, regions);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postCreateTableHandler(HTableDescriptor htd, HRegionInfo[] regions)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).postCreateTableHandler(ctx, htd, regions);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preDeleteTable(byte[] tableName) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preDeleteTable(ctx, tableName);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postDeleteTable(byte[] tableName) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postDeleteTable(ctx, tableName);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preDeleteTableHandler(byte[] tableName) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).preDeleteTableHandler(ctx, tableName);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postDeleteTableHandler(byte[] tableName) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).postDeleteTableHandler(ctx, tableName);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preModifyTable(final byte[] tableName, HTableDescriptor htd)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preModifyTable(ctx, tableName,
+              htd);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postModifyTable(final byte[] tableName, HTableDescriptor htd)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postModifyTable(ctx, tableName,
+              htd);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preModifyTableHandler(final byte[] tableName, HTableDescriptor htd)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).preModifyTableHandler(ctx, tableName, htd);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postModifyTableHandler(final byte[] tableName, HTableDescriptor htd)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).postModifyTableHandler(ctx, tableName, htd);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public boolean preAddColumn(byte [] tableName, HColumnDescriptor column)
+      throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preAddColumn(ctx, tableName, column);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  public void postAddColumn(byte [] tableName, HColumnDescriptor column)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postAddColumn(ctx, tableName,
+              column);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public boolean preAddColumnHandler(byte[] tableName, HColumnDescriptor column) throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).preAddColumnHandler(ctx, tableName, column);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  public void postAddColumnHandler(byte[] tableName, HColumnDescriptor column) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).postAddColumnHandler(ctx, tableName, column);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public boolean preModifyColumn(byte [] tableName, HColumnDescriptor descriptor)
+      throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preModifyColumn(
+            ctx, tableName, descriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  public void postModifyColumn(byte [] tableName, HColumnDescriptor descriptor)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postModifyColumn(
+              ctx, tableName, descriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public boolean preModifyColumnHandler(byte[] tableName, HColumnDescriptor descriptor)
+      throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).preModifyColumnHandler(ctx, tableName, descriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  public void postModifyColumnHandler(byte[] tableName, HColumnDescriptor descriptor)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).postModifyColumnHandler(ctx, tableName, descriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  boolean preDeleteColumn(final byte [] tableName, final byte [] c)
+      throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preDeleteColumn(ctx, tableName, c);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  public void postDeleteColumn(final byte [] tableName, final byte [] c)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postDeleteColumn(ctx, tableName,
+              c);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public boolean preDeleteColumnHandler(final byte[] tableName, final byte[] c) throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).preDeleteColumnHandler(ctx, tableName, c);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  public void postDeleteColumnHandler(final byte[] tableName, final byte[] c) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).postDeleteColumnHandler(ctx, tableName, c);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preEnableTable(final byte [] tableName) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preEnableTable(ctx, tableName);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postEnableTable(final byte [] tableName) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postEnableTable(ctx, tableName);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preEnableTableHandler(final byte[] tableName) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).preEnableTableHandler(ctx, tableName);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postEnableTableHandler(final byte[] tableName) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).postEnableTableHandler(ctx, tableName);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preDisableTable(final byte [] tableName) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preDisableTable(ctx, tableName);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postDisableTable(final byte[] tableName) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postDisableTable(ctx, tableName);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preDisableTableHandler(final byte[] tableName) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).preDisableTableHandler(ctx, tableName);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postDisableTableHandler(final byte[] tableName) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).postDisableTableHandler(ctx, tableName);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public boolean preMove(final HRegionInfo region, final ServerName srcServer,
+      final ServerName destServer) throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preMove(
+              ctx, region, srcServer, destServer);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  public void postMove(final HRegionInfo region, final ServerName srcServer,
+      final ServerName destServer) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postMove(
+              ctx, region, srcServer, destServer);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  boolean preAssign(final HRegionInfo regionInfo) throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver) env.getInstance()).preAssign(ctx, regionInfo);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  void postAssign(final HRegionInfo regionInfo) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postAssign(ctx, regionInfo);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  boolean preUnassign(final HRegionInfo regionInfo, final boolean force)
+      throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preUnassign(
+              ctx, regionInfo, force);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  void postUnassign(final HRegionInfo regionInfo, final boolean force)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postUnassign(
+              ctx, regionInfo, force);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  boolean preBalance() throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preBalance(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  void postBalance() throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postBalance(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  boolean preBalanceSwitch(final boolean b) throws IOException {
+    boolean balance = b;
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          balance = ((MasterObserver)env.getInstance()).preBalanceSwitch(
+              ctx, balance);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return balance;
+  }
+
+  void postBalanceSwitch(final boolean oldValue, final boolean newValue)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postBalanceSwitch(
+              ctx, oldValue, newValue);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  void preShutdown() throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preShutdown(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  void preStopMaster() throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preStopMaster(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  void postStartMaster() throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postStartMaster(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preCloneSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preCloneSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postCloneSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postCloneSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preRestoreSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preRestoreSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postRestoreSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postRestoreSnapshot(ctx, snapshot, hTableDescriptor);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preDeleteSnapshot(final SnapshotDescription snapshot) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).preDeleteSnapshot(ctx, snapshot);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postDeleteSnapshot(final SnapshotDescription snapshot) throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof MasterObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserver)env.getInstance()).postDeleteSnapshot(ctx, snapshot);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preMasterInitialization() throws IOException {
+    ObserverContext ctx = null;
+    for (MasterEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof MasterObserverExt) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((MasterObserverExt) env.getInstance()).preMasterInitialization(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
new file mode 100644
index 0000000..876eda4
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
@@ -0,0 +1,121 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.Date;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
+import org.apache.hadoop.hbase.monitoring.LogMonitoring;
+import org.apache.hadoop.hbase.monitoring.StateDumpServlet;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.util.ReflectionUtils;
+
+public class MasterDumpServlet extends StateDumpServlet {
+  private static final long serialVersionUID = 1L;
+  private static final String LINE =
+    "===========================================================";
+  
+  @Override
+  public void doGet(HttpServletRequest request, HttpServletResponse response)
+      throws IOException {
+    HMaster master = (HMaster) getServletContext().getAttribute(HMaster.MASTER);
+    assert master != null : "No Master in context!";
+
+    response.setContentType("text/plain");
+    OutputStream os = response.getOutputStream();
+    PrintWriter out = new PrintWriter(os);
+    
+    out.println("Master status for " + master.getServerName()
+        + " as of " + new Date());
+    
+    out.println("\n\nVersion Info:");
+    out.println(LINE);
+    dumpVersionInfo(out);
+
+    out.println("\n\nTasks:");
+    out.println(LINE);
+    TaskMonitor.get().dumpAsText(out);
+    
+    out.println("\n\nServers:");
+    out.println(LINE);
+    dumpServers(master, out);
+    
+    out.println("\n\nRegions-in-transition:");
+    out.println(LINE);
+    dumpRIT(master, out);
+    
+    out.println("\n\nExecutors:");
+    out.println(LINE);
+    dumpExecutors(master.getExecutorService(), out);
+    
+    out.println("\n\nStacks:");
+    out.println(LINE);
+    ReflectionUtils.printThreadInfo(out, "");
+    
+    out.println("\n\nMaster configuration:");
+    out.println(LINE);
+    Configuration conf = master.getConfiguration();
+    out.flush();
+    conf.writeXml(os);
+    os.flush();
+    
+    out.println("\n\nRecent regionserver aborts:");
+    out.println(LINE);
+    master.getRegionServerFatalLogBuffer().dumpTo(out);
+    
+    out.println("\n\nLogs");
+    out.println(LINE);
+    long tailKb = getTailKbParam(request);
+    LogMonitoring.dumpTailOfLogs(out, tailKb);
+    
+    out.flush();
+  }
+  
+
+  private void dumpRIT(HMaster master, PrintWriter out) {
+    NavigableMap regionsInTransition =
+      master.getAssignmentManager().getRegionsInTransition();
+    for (Map.Entry e : regionsInTransition.entrySet()) {
+      String rid = e.getKey();
+      RegionState rs = e.getValue();
+      out.println("Region " + rid + ": " + rs.toDescriptiveString());
+    }
+  }
+
+  private void dumpServers(HMaster master, PrintWriter out) {
+    Map servers =
+      master.getServerManager().getOnlineServers();
+    for (Map.Entry e : servers.entrySet()) {
+      out.println(e.getKey() + ": " + e.getValue());
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
new file mode 100644
index 0000000..acce4b5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -0,0 +1,699 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.InvalidFamilyOperationException;
+import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
+import org.apache.hadoop.hbase.regionserver.wal.OrphanHLogAfterSplitException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * This class abstracts a bunch of operations the HMaster needs to interact with
+ * the underlying file system, including splitting log files, checking file
+ * system status, etc.
+ */
+public class MasterFileSystem {
+  private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
+  // HBase configuration
+  Configuration conf;
+  // master status
+  Server master;
+  // metrics for master
+  MasterMetrics metrics;
+  // Persisted unique cluster ID
+  private String clusterId;
+  // Keep around for convenience.
+  private final FileSystem fs;
+  // Is the fileystem ok?
+  private volatile boolean fsOk = true;
+  // The Path to the old logs dir
+  private final Path oldLogDir;
+  // root hbase directory on the FS
+  private final Path rootdir;
+  // hbase temp directory used for table construction and deletion
+  private final Path tempdir;
+  // create the split log lock
+  final Lock splitLogLock = new ReentrantLock();
+  final boolean distributedLogSplitting;
+  final SplitLogManager splitLogManager;
+  private final MasterServices services;
+
+  private final static PathFilter META_FILTER = new PathFilter() {
+    public boolean accept(Path p) {
+      return HLog.isMetaFile(p);
+    }
+  };
+
+  private final static PathFilter NON_META_FILTER = new PathFilter() {
+    public boolean accept(Path p) {
+      return !HLog.isMetaFile(p);
+    }
+  };
+
+  public MasterFileSystem(Server master, MasterServices services,
+      MasterMetrics metrics, boolean masterRecovery)
+  throws IOException {
+    this.conf = master.getConfiguration();
+    this.master = master;
+    this.services = services;
+    this.metrics = metrics;
+    // Set filesystem to be that of this.rootdir else we get complaints about
+    // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
+    // default localfs.  Presumption is that rootdir is fully-qualified before
+    // we get to here with appropriate fs scheme.
+    this.rootdir = FSUtils.getRootDir(conf);
+    this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
+    // Cover both bases, the old way of setting default fs and the new.
+    // We're supposed to run on 0.20 and 0.21 anyways.
+    this.fs = this.rootdir.getFileSystem(conf);
+    String fsUri = this.fs.getUri().toString();
+    conf.set("fs.default.name", fsUri);
+    conf.set("fs.defaultFS", fsUri);
+    // make sure the fs has the same conf
+    fs.setConf(conf);
+    this.distributedLogSplitting =
+      conf.getBoolean("hbase.master.distributed.log.splitting", true);
+    if (this.distributedLogSplitting) {
+      this.splitLogManager = new SplitLogManager(master.getZooKeeper(),
+          master.getConfiguration(), master, this.services, master.getServerName().toString());
+      this.splitLogManager.finishInitialization(masterRecovery);
+    } else {
+      this.splitLogManager = null;
+    }
+    // setup the filesystem variable
+    // set up the archived logs path
+    this.oldLogDir = createInitialFileSystemLayout();
+  }
+
+  /**
+   * Create initial layout in filesystem.
+   * 
          
+   * 
        1. Check if the root region exists and is readable, if not create it.
+   * Create hbase.version and the -ROOT- directory if not one.
+   * 
          2. 
+   * 
        2. Create a log archive directory for RS to put archived logs
          3. 
+   * 
        
+   * Idempotent.
+   */
+  private Path createInitialFileSystemLayout() throws IOException {
+    // check if the root directory exists
+    checkRootDir(this.rootdir, conf, this.fs);
+
+    // check if temp directory exists and clean it
+    checkTempDir(this.tempdir, conf, this.fs);
+
+    Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
+
+    // Make sure the region servers can archive their old logs
+    if(!this.fs.exists(oldLogDir)) {
+      HBaseFileSystem.makeDirOnFileSystem(fs, oldLogDir);
+    }
+
+    return oldLogDir;
+  }
+
+  public FileSystem getFileSystem() {
+    return this.fs;
+  }
+
+  /**
+   * Get the directory where old logs go
+   * @return the dir
+   */
+  public Path getOldLogDir() {
+    return this.oldLogDir;
+  }
+
+  /**
+   * Checks to see if the file system is still accessible.
+   * If not, sets closed
+   * @return false if file system is not available
+   */
+  public boolean checkFileSystem() {
+    if (this.fsOk) {
+      try {
+        FSUtils.checkFileSystemAvailable(this.fs);
+        FSUtils.checkDfsSafeMode(this.conf);
+      } catch (IOException e) {
+        master.abort("Shutting down HBase cluster: file system not available", e);
+        this.fsOk = false;
+      }
+    }
+    return this.fsOk;
+  }
+
+  /**
+   * @return HBase root dir.
+   */
+  public Path getRootDir() {
+    return this.rootdir;
+  }
+
+  /**
+   * @return HBase temp dir.
+   */
+  public Path getTempDir() {
+    return this.tempdir;
+  }
+
+  /**
+   * @return The unique identifier generated for this cluster
+   */
+  public String getClusterId() {
+    return clusterId;
+  }
+
+  /**
+   * Inspect the log directory to find dead servers which need log splitting
+   */
+  Set getFailedServersFromLogFolders() {
+    boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
+      HLog.SPLIT_SKIP_ERRORS_DEFAULT);
+    
+    Set serverNames = new HashSet();
+    Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
+
+    do {
+      if (master.isStopped()) {
+        LOG.warn("Master stopped while trying to get failed servers.");
+        break;
+      }
+      try {
+        if (!this.fs.exists(logsDirPath)) return serverNames;
+        FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
+        // Get online servers after getting log folders to avoid log folder deletion of newly
+        // checked in region servers . see HBASE-5916
+        Set onlineServers =
+            ((HMaster) master).getServerManager().getOnlineServers().keySet();
+
+        if (logFolders == null || logFolders.length == 0) {
+          LOG.debug("No log files to split, proceeding...");
+          return serverNames;
+        }
+        for (FileStatus status : logFolders) {
+          String sn = status.getPath().getName();
+          // truncate splitting suffix if present (for ServerName parsing)
+          if (sn.endsWith(HLog.SPLITTING_EXT)) {
+            sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
+          }
+          ServerName serverName = ServerName.parseServerName(sn);
+          if (!onlineServers.contains(serverName)) {
+            LOG.info("Log folder " + status.getPath() + " doesn't belong "
+                + "to a known region server, splitting");
+            serverNames.add(serverName);
+          } else {
+            LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
+          }
+        }
+        retrySplitting = false;
+      } catch (IOException ioe) {
+        LOG.warn("Failed getting failed servers to be recovered.", ioe);
+        if (!checkFileSystem()) {
+          LOG.warn("Bad Filesystem, exiting");
+          Runtime.getRuntime().halt(1);
+        }
+        try {
+          if (retrySplitting) {
+            Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
+          }
+        } catch (InterruptedException e) {
+          LOG.warn("Interrupted, aborting since cannot return w/o splitting");
+          Thread.currentThread().interrupt();
+          retrySplitting = false;
+          Runtime.getRuntime().halt(1);
+        }
+      }
+    } while (retrySplitting);
+
+    return serverNames;
+  }
+
+  public void splitLog(final ServerName serverName) throws IOException {
+    List serverNames = new ArrayList();
+    serverNames.add(serverName);
+    splitLog(serverNames);
+  }
+
+  public void splitAllLogs(final ServerName serverName) throws IOException {
+    List serverNames = new ArrayList();
+    serverNames.add(serverName);
+    splitAllLogs(serverNames);
+  }
+
+  /**
+   * Specialized method to handle the splitting for meta HLog
+   * @param serverName
+   * @throws IOException
+   */
+  public void splitMetaLog(final ServerName serverName) throws IOException {
+    long splitTime = 0, splitLogSize = 0;
+    List serverNames = new ArrayList();
+    serverNames.add(serverName);
+    List logDirs = getLogDirs(serverNames);
+    if (logDirs.isEmpty()) {
+      LOG.info("No meta logs to split");
+      return;
+    }
+    splitLogManager.handleDeadWorkers(serverNames);
+    splitTime = EnvironmentEdgeManager.currentTimeMillis();
+    splitLogSize = splitLogManager.splitLogDistributed(logDirs, META_FILTER);
+    splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
+    if (this.metrics != null) {
+      this.metrics.addSplit(splitTime, splitLogSize);
+    }
+  }
+
+  private List getLogDirs(final List serverNames) throws IOException {
+    List logDirs = new ArrayList();
+    for(ServerName serverName: serverNames){
+      Path logDir = new Path(this.rootdir,
+        HLog.getHLogDirectoryName(serverName.toString()));
+      Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
+      // rename the directory so a rogue RS doesn't create more HLogs
+      if (fs.exists(logDir)) {
+        if (!HBaseFileSystem.renameDirForFileSystem(fs, logDir, splitDir)) {
+          throw new IOException("Failed fs.rename for log split: " + logDir);
+        }
+        logDir = splitDir;
+        LOG.debug("Renamed region directory: " + splitDir);
+      } else if (!fs.exists(splitDir)) {
+        LOG.info("Log dir for server " + serverName + " does not exist");
+        continue;
+      }
+      logDirs.add(splitDir);
+    }
+    return logDirs;
+  }
+
+  public void splitLog(final List serverNames) throws IOException {
+    splitLog(serverNames, NON_META_FILTER);
+  }
+
+  public void splitAllLogs(final List serverNames) throws IOException {
+    splitLog(serverNames, null); //no filter
+  }
+
+  /**
+   * This method is the base split method that splits HLog files matching a filter.
+   * Callers should pass the appropriate filter for meta and non-meta HLogs.
+   * @param serverNames
+   * @param filter
+   * @throws IOException
+   */
+  public void splitLog(final List serverNames, PathFilter filter) throws IOException {
+    long splitTime = 0, splitLogSize = 0;
+    List logDirs = getLogDirs(serverNames);
+
+    if (logDirs.isEmpty()) {
+      LOG.info("No logs to split");
+      return;
+    }
+
+    boolean lockAcquired = false;
+    if (distributedLogSplitting) {
+      try {
+        if (!this.services.isServerShutdownHandlerEnabled()) {
+          // process one log splitting task at one time before SSH is enabled.
+          // because ROOT SSH and HMaster#assignMeta could both log split a same server
+          this.splitLogLock.lock();
+          lockAcquired = true;
+        }
+        splitLogManager.handleDeadWorkers(serverNames);
+        splitTime = EnvironmentEdgeManager.currentTimeMillis();
+        splitLogSize = splitLogManager.splitLogDistributed(logDirs, filter);
+        splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
+      } finally {
+        if (lockAcquired) {
+          this.splitLogLock.unlock();
+        }
+      }
+    } else {
+      for(Path logDir: logDirs){
+        // splitLogLock ensures that dead region servers' logs are processed
+        // one at a time
+        this.splitLogLock.lock();
+        try {
+          HLogSplitter splitter = HLogSplitter.createLogSplitter(
+            conf, rootdir, logDir, oldLogDir, this.fs);
+          try {
+            // If FS is in safe mode, just wait till out of it.
+            FSUtils.waitOnSafeMode(conf, conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 1000));
+            splitter.splitLog();
+          } catch (OrphanHLogAfterSplitException e) {
+            LOG.warn("Retrying splitting because of:", e);
+            //An HLogSplitter instance can only be used once.  Get new instance.
+            splitter = HLogSplitter.createLogSplitter(conf, rootdir, logDir,
+              oldLogDir, this.fs);
+            splitter.splitLog();
+          }
+          splitTime = splitter.getTime();
+          splitLogSize = splitter.getSize();
+        } finally {
+          this.splitLogLock.unlock();
+        }
+      }
+    }
+
+    if (this.metrics != null) {
+      this.metrics.addSplit(splitTime, splitLogSize);
+    }
+  }
+
+  /**
+   * Get the rootdir.  Make sure its wholesome and exists before returning.
+   * @param rd
+   * @param conf
+   * @param fs
+   * @return hbase.rootdir (after checks for existence and bootstrapping if
+   * needed populating the directory with necessary bootup files).
+   * @throws IOException
+   */
+  private Path checkRootDir(final Path rd, final Configuration c,
+    final FileSystem fs)
+  throws IOException {
+    // If FS is in safe mode wait till out of it.
+    FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+        10 * 1000));
+    // Filesystem is good. Go ahead and check for hbase.rootdir.
+    try {
+      if (!fs.exists(rd)) {
+        HBaseFileSystem.makeDirOnFileSystem(fs, rd);
+        // DFS leaves safe mode with 0 DNs when there are 0 blocks.
+        // We used to handle this by checking the current DN count and waiting until
+        // it is nonzero. With security, the check for datanode count doesn't work --
+        // it is a privileged op. So instead we adopt the strategy of the jobtracker
+        // and simply retry file creation during bootstrap indefinitely. As soon as
+        // there is one datanode it will succeed. Permission problems should have
+        // already been caught by mkdirs above.
+        FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+          10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
+        		  HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
+      } else {
+        if (!fs.isDirectory(rd)) {
+          throw new IllegalArgumentException(rd.toString() + " is not a directory");
+        }
+        // as above
+        FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+          10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
+        		  HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
+      }
+    } catch (IllegalArgumentException iae) {
+      LOG.fatal("Please fix invalid configuration for "
+        + HConstants.HBASE_DIR + " " + rd.toString(), iae);
+      throw iae;
+    }
+    // Make sure cluster ID exists
+    if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
+        HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
+      FSUtils.setClusterId(fs, rd, UUID.randomUUID().toString(), c.getInt(
+          HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
+    }
+    clusterId = FSUtils.getClusterId(fs, rd);
+
+    // Make sure the root region directory exists!
+    if (!FSUtils.rootRegionExists(fs, rd)) {
+      bootstrap(rd, c);
+    }
+    createRootTableInfo(rd);
+    return rd;
+  }
+
+  private void createRootTableInfo(Path rd) throws IOException {
+    // Create ROOT tableInfo if required.
+    if (!FSTableDescriptors.isTableInfoExists(fs, rd,
+        Bytes.toString(HRegionInfo.ROOT_REGIONINFO.getTableName()))) {
+      FSTableDescriptors.createTableDescriptor(HTableDescriptor.ROOT_TABLEDESC, this.conf);
+    }
+  }
+
+  /**
+   * Make sure the hbase temp directory exists and is empty.
+   * NOTE that this method is only executed once just after the master becomes the active one.
+   */
+  private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
+      throws IOException {
+    // If the temp directory exists, clear the content (left over, from the previous run)
+    if (fs.exists(tmpdir)) {
+      // Archive table in temp, maybe left over from failed deletion,
+      // if not the cleaner will take care of them.
+      for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
+        for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
+          HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
+        }
+      }
+      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, tmpdir)) {
+        throw new IOException("Unable to clean the temp directory: " + tmpdir);
+      }
+    }
+
+    // Create the temp directory
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, tmpdir)) {
+      throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
+    }
+  }
+
+  private static void bootstrap(final Path rd, final Configuration c)
+  throws IOException {
+    LOG.info("BOOTSTRAP: creating ROOT and first META regions");
+    try {
+      // Bootstrapping, make sure blockcache is off.  Else, one will be
+      // created here in bootstap and it'll need to be cleaned up.  Better to
+      // not make it in first place.  Turn off block caching for bootstrap.
+      // Enable after.
+      HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
+      setInfoFamilyCachingForRoot(false);
+      HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
+      setInfoFamilyCachingForMeta(false);
+      HRegion root = HRegion.createHRegion(rootHRI, rd, c,
+          HTableDescriptor.ROOT_TABLEDESC);
+      HRegion meta = HRegion.createHRegion(metaHRI, rd, c,
+          HTableDescriptor.META_TABLEDESC);
+      setInfoFamilyCachingForRoot(true);
+      setInfoFamilyCachingForMeta(true);
+      // Add first region from the META table to the ROOT region.
+      HRegion.addRegionToMETA(root, meta);
+      root.close();
+      root.getLog().closeAndDelete();
+      meta.close();
+      meta.getLog().closeAndDelete();
+    } catch (IOException e) {
+      e = RemoteExceptionHandler.checkIOException(e);
+      LOG.error("bootstrap", e);
+      throw e;
+    }
+  }
+
+  /**
+   * Enable in-memory caching for -ROOT-
+   */
+  public static void setInfoFamilyCachingForRoot(final boolean b) {
+    for (HColumnDescriptor hcd:
+        HTableDescriptor.ROOT_TABLEDESC.getColumnFamilies()) {
+       if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
+         hcd.setBlockCacheEnabled(b);
+         hcd.setInMemory(b);
+     }
+    }
+  }
+
+  /**
+   * Enable in memory caching for .META.
+   */
+  public static void setInfoFamilyCachingForMeta(final boolean b) {
+    for (HColumnDescriptor hcd:
+        HTableDescriptor.META_TABLEDESC.getColumnFamilies()) {
+      if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
+        hcd.setBlockCacheEnabled(b);
+        hcd.setInMemory(b);
+      }
+    }
+  }
+
+
+  public void deleteRegion(HRegionInfo region) throws IOException {
+    HFileArchiver.archiveRegion(conf, fs, region);
+  }
+
+  public void deleteTable(byte[] tableName) throws IOException {
+    HBaseFileSystem.deleteDirFromFileSystem(fs, new Path(rootdir, Bytes.toString(tableName)));
+  }
+
+  /**
+   * Move the specified file/directory to the hbase temp directory.
+   * @param path The path of the file/directory to move
+   * @return The temp location of the file/directory moved
+   * @throws IOException in case of file-system failure
+   */
+  public Path moveToTemp(final Path path) throws IOException {
+    Path tempPath = new Path(this.tempdir, path.getName());
+
+    // Ensure temp exists
+    if (!fs.exists(tempdir) && !HBaseFileSystem.makeDirOnFileSystem(fs, tempdir)) {
+      throw new IOException("HBase temp directory '" + tempdir + "' creation failure.");
+    }
+
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, path, tempPath)) {
+      throw new IOException("Unable to move '" + path + "' to temp '" + tempPath + "'");
+    }
+
+    return tempPath;
+  }
+
+  /**
+   * Move the specified table to the hbase temp directory
+   * @param tableName Table name to move
+   * @return The temp location of the table moved
+   * @throws IOException in case of file-system failure
+   */
+  public Path moveTableToTemp(byte[] tableName) throws IOException {
+    return moveToTemp(HTableDescriptor.getTableDir(this.rootdir, tableName));
+  }
+
+  public void updateRegionInfo(HRegionInfo region) {
+    // TODO implement this.  i think this is currently broken in trunk i don't
+    //      see this getting updated.
+    //      @see HRegion.checkRegioninfoOnFilesystem()
+  }
+
+  public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
+      throws IOException {
+    // archive family store files
+    Path tableDir = new Path(rootdir, region.getTableNameAsString());
+    HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
+
+    // delete the family folder
+    Path familyDir = new Path(tableDir,
+      new Path(region.getEncodedName(), Bytes.toString(familyName)));
+    if (!HBaseFileSystem.deleteDirFromFileSystem(fs, familyDir)) {
+      throw new IOException("Could not delete family "
+          + Bytes.toString(familyName) + " from FileSystem for region "
+          + region.getRegionNameAsString() + "(" + region.getEncodedName()
+          + ")");
+    }
+  }
+
+  public void stop() {
+    if (splitLogManager != null) {
+      this.splitLogManager.stop();
+    }
+  }
+
+  /**
+   * Create new HTableDescriptor in HDFS.
+   *
+   * @param htableDescriptor
+   */
+  public void createTableDescriptor(HTableDescriptor htableDescriptor)
+      throws IOException {
+    FSTableDescriptors.createTableDescriptor(htableDescriptor, conf);
+  }
+
+  /**
+   * Delete column of a table
+   * @param tableName
+   * @param familyName
+   * @return Modified HTableDescriptor with requested column deleted.
+   * @throws IOException
+   */
+  public HTableDescriptor deleteColumn(byte[] tableName, byte[] familyName)
+      throws IOException {
+    LOG.info("DeleteColumn. Table = " + Bytes.toString(tableName)
+        + " family = " + Bytes.toString(familyName));
+    HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
+    htd.removeFamily(familyName);
+    this.services.getTableDescriptors().add(htd);
+    return htd;
+  }
+
+  /**
+   * Modify Column of a table
+   * @param tableName
+   * @param hcd HColumnDesciptor
+   * @return Modified HTableDescriptor with the column modified.
+   * @throws IOException
+   */
+  public HTableDescriptor modifyColumn(byte[] tableName, HColumnDescriptor hcd)
+      throws IOException {
+    LOG.info("AddModifyColumn. Table = " + Bytes.toString(tableName)
+        + " HCD = " + hcd.toString());
+
+    HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
+    byte [] familyName = hcd.getName();
+    if(!htd.hasFamily(familyName)) {
+      throw new InvalidFamilyOperationException("Family '" +
+        Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
+    }
+    htd.addFamily(hcd);
+    this.services.getTableDescriptors().add(htd);
+    return htd;
+  }
+
+  /**
+   * Add column to a table
+   * @param tableName
+   * @param hcd
+   * @return Modified HTableDescriptor with new column added.
+   * @throws IOException
+   */
+  public HTableDescriptor addColumn(byte[] tableName, HColumnDescriptor hcd)
+      throws IOException {
+    LOG.info("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
+      hcd.toString());
+    HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
+    if (htd == null) {
+      throw new InvalidFamilyOperationException("Family '" +
+        hcd.getNameAsString() + "' cannot be modified as HTD is null");
+    }
+    htd.addFamily(hcd);
+    this.services.getTableDescriptors().add(htd);
+    return htd;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
new file mode 100644
index 0000000..3eefdfc
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -0,0 +1,169 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableDescriptors;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+
+/**
+ * Services Master supplies
+ */
+public interface MasterServices extends Server {
+  /**
+   * @return Master's instance of the {@link AssignmentManager}
+   */
+  public AssignmentManager getAssignmentManager();
+
+  /**
+   * @return Master's filesystem {@link MasterFileSystem} utility class.
+   */
+  public MasterFileSystem getMasterFileSystem();
+
+  /**
+   * @return Master's {@link ServerManager} instance.
+   */
+  public ServerManager getServerManager();
+
+  /**
+   * @return Master's instance of {@link ExecutorService}
+   */
+  public ExecutorService getExecutorService();
+
+  /**
+   * Check table is modifiable; i.e. exists and is offline.
+   * @param tableName Name of table to check.
+   * @throws TableNotDisabledException
+   * @throws TableNotFoundException 
+   */
+  public void checkTableModifiable(final byte [] tableName) throws IOException;
+
+  /**
+   * Create a table using the given table definition.
+   * @param desc The table definition
+   * @param splitKeys Starting row keys for the initial table regions.  If null
+   *     a single region is created.
+   */
+  public void createTable(HTableDescriptor desc, byte [][] splitKeys)
+      throws IOException;
+
+  /**
+   * Delete a table
+   * @param tableName The table name
+   * @throws IOException
+   */
+  public void deleteTable(final byte[] tableName) throws IOException;
+
+  /**
+   * Modify the descriptor of an existing table
+   * @param tableName The table name
+   * @param descriptor The updated table descriptor
+   * @throws IOException
+   */
+  public void modifyTable(final byte[] tableName, final HTableDescriptor descriptor)
+      throws IOException;
+
+  /**
+   * Enable an existing table
+   * @param tableName The table name
+   * @throws IOException
+   */
+  public void enableTable(final byte[] tableName) throws IOException;
+
+  /**
+   * Disable an existing table
+   * @param tableName The table name
+   * @throws IOException
+   */
+  public void disableTable(final byte[] tableName) throws IOException;
+
+  /**
+   * Add a new column to an existing table
+   * @param tableName The table name
+   * @param column The column definition
+   * @throws IOException
+   */
+  public void addColumn(final byte[] tableName, final HColumnDescriptor column)
+      throws IOException;
+
+  /**
+   * Modify the column descriptor of an existing column in an existing table
+   * @param tableName The table name
+   * @param descriptor The updated column definition
+   * @throws IOException
+   */
+  public void modifyColumn(byte[] tableName, HColumnDescriptor descriptor)
+      throws IOException;
+
+  /**
+   * Delete a column from an existing table
+   * @param tableName The table name
+   * @param columnName The column name
+   * @throws IOException
+   */
+  public void deleteColumn(final byte[] tableName, final byte[] columnName)
+      throws IOException;
+
+  /**
+   * @return Return table descriptors implementation.
+   */
+  public TableDescriptors getTableDescriptors();
+
+  /**
+   * @return true if master enables ServerShutdownHandler;
+   */
+  public boolean isServerShutdownHandlerEnabled();
+  
+  /**
+   * @return true if master thinks that meta hlogs should be split separately
+   */
+  public boolean shouldSplitMetaSeparately();
+
+  /**
+   * @return returns the master coprocessor host
+   */
+  public MasterCoprocessorHost getCoprocessorHost();
+
+  /**
+   * Registers a new CoprocessorProtocol subclass and instance to
+   * be available for handling
+   * {@link HMaster#execCoprocessor(org.apache.hadoop.hbase.client.coprocessor.Exec)} calls.
+   *
+   * 
        
+   * Only a single protocol type/handler combination may be registered.
+   *
+   * After the first registration, subsequent calls with the same protocol type
+   * will fail with a return value of {@code false}.
+   * 
        
+   * @param protocol a {@code CoprocessorProtocol} subinterface defining the
+   * protocol methods
+   * @param handler an instance implementing the interface
+   * @param  the protocol type
+   * @return {@code true} if the registration was successful, {@code false}
+   * otherwise
+   */
+  public  boolean registerProtocol(
+      Class protocol, T handler);
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java b/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java
new file mode 100644
index 0000000..862db2e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java
@@ -0,0 +1,109 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl;
+
+/**
+ * The servlet responsible for rendering the index page of the
+ * master.
+ */
+public class MasterStatusServlet extends HttpServlet {
+  private static final Log LOG = LogFactory.getLog(MasterStatusServlet.class);
+  private static final long serialVersionUID = 1L;
+
+  @Override
+  public void doGet(HttpServletRequest request, HttpServletResponse response)
+    throws IOException
+  {
+    HMaster master = (HMaster) getServletContext().getAttribute(HMaster.MASTER);
+    assert master != null : "No Master in context!";
+    
+    Configuration conf = master.getConfiguration();
+    HBaseAdmin admin = new HBaseAdmin(conf);    
+        
+    Map frags = getFragmentationInfo(master, conf);
+    
+    ServerName rootLocation = getRootLocationOrNull(master);
+    ServerName metaLocation = master.getCatalogTracker().getMetaLocation();
+    List servers = master.getServerManager().getOnlineServersList();
+    Set deadServers = master.getServerManager().getDeadServers();
+
+    response.setContentType("text/html");
+    MasterStatusTmpl tmpl = new MasterStatusTmpl()
+      .setFrags(frags)
+      .setShowAppendWarning(shouldShowAppendWarning(conf))
+      .setRootLocation(rootLocation)
+      .setMetaLocation(metaLocation)
+      .setServers(servers)
+      .setDeadServers(deadServers);
+    if (request.getParameter("filter") != null)
+      tmpl.setFilter(request.getParameter("filter"));
+    if (request.getParameter("format") != null)
+      tmpl.setFormat(request.getParameter("format"));
+    tmpl.render(response.getWriter(),
+          master, admin);
+  }
+
+  private ServerName getRootLocationOrNull(HMaster master) {
+    try {
+      return master.getCatalogTracker().getRootLocation();
+    } catch (InterruptedException e) {
+      LOG.warn("Unable to get root location", e);
+      return null;
+    }
+  }
+
+  private Map getFragmentationInfo(
+      HMaster master, Configuration conf) throws IOException {
+    boolean showFragmentation = conf.getBoolean(
+        "hbase.master.ui.fragmentation.enabled", false);    
+    if (showFragmentation) {
+      return FSUtils.getTableFragmentation(master);
+    } else {
+      return null;
+    }
+  }
+
+  static boolean shouldShowAppendWarning(Configuration conf) {
+    try {
+      return !FSUtils.isAppendSupported(conf) && FSUtils.isHDFS(conf);
+    } catch (IOException e) {
+      LOG.warn("Unable to determine if append is supported", e);
+      return false;
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/RegionPlan.java b/src/main/java/org/apache/hadoop/hbase/master/RegionPlan.java
new file mode 100644
index 0000000..3c1fe83
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/RegionPlan.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+
+/**
+ * Stores the plan for the move of an individual region.
+ *
+ * Contains info for the region being moved, info for the server the region
+ * should be moved from, and info for the server the region should be moved
+ * to.
+ *
+ * The comparable implementation of this class compares only the region
+ * information and not the source/dest server info.
+ */
+public class RegionPlan implements Comparable {
+  private final HRegionInfo hri;
+  private final ServerName source;
+  private ServerName dest;
+
+  /**
+   * Instantiate a plan for a region move, moving the specified region from
+   * the specified source server to the specified destination server.
+   *
+   * Destination server can be instantiated as null and later set
+   * with {@link #setDestination(ServerName)}.
+   *
+   * @param hri region to be moved
+   * @param source regionserver region should be moved from
+   * @param dest regionserver region should be moved to
+   */
+  public RegionPlan(final HRegionInfo hri, ServerName source, ServerName dest) {
+    this.hri = hri;
+    this.source = source;
+    this.dest = dest;
+  }
+
+  /**
+   * Set the destination server for the plan for this region.
+   */
+  public void setDestination(ServerName dest) {
+    this.dest = dest;
+  }
+
+  /**
+   * Get the source server for the plan for this region.
+   * @return server info for source
+   */
+  public ServerName getSource() {
+    return source;
+  }
+
+  /**
+   * Get the destination server for the plan for this region.
+   * @return server info for destination
+   */
+  public ServerName getDestination() {
+    return dest;
+  }
+
+  /**
+   * Get the encoded region name for the region this plan is for.
+   * @return Encoded region name
+   */
+  public String getRegionName() {
+    return this.hri.getEncodedName();
+  }
+
+  public HRegionInfo getRegionInfo() {
+    return this.hri;
+  }
+
+  /**
+   * Compare the region info.
+   * @param o region plan you are comparing against
+   */
+  @Override
+  public int compareTo(RegionPlan o) {
+    return getRegionName().compareTo(o.getRegionName());
+  }
+
+  @Override
+  public String toString() {
+    return "hri=" + this.hri.getRegionNameAsString() + ", src=" +
+      (this.source == null? "": this.source.toString()) +
+      ", dest=" + (this.dest == null? "": this.dest.toString());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerAndLoad.java b/src/main/java/org/apache/hadoop/hbase/master/ServerAndLoad.java
new file mode 100644
index 0000000..17d0c76
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerAndLoad.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+
+import org.apache.hadoop.hbase.ServerName;
+
+/**
+ * Data structure that holds servername and 'load'.
+ */
+class ServerAndLoad implements Comparable {
+  private final ServerName sn;
+  private final int load;
+
+  ServerAndLoad(final ServerName sn, final int load) {
+    this.sn = sn;
+    this.load = load;
+  }
+
+  ServerName getServerName() {
+    return this.sn;
+  }
+
+  int getLoad() {
+    return this.load;
+  }
+
+  @Override
+  public int compareTo(ServerAndLoad other) {
+    int diff = this.load - other.load;
+    return diff != 0 ? diff : this.sn.compareTo(other.getServerName());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
new file mode 100644
index 0000000..1b6d29e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -0,0 +1,744 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ClockOutOfSyncException;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.PleaseHoldException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.YouAreDeadException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.RetriesExhaustedException;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler;
+import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
+
+/**
+ * The ServerManager class manages info about region servers.
+ * 
        
+ * Maintains lists of online and dead servers.  Processes the startups,
+ * shutdowns, and deaths of region servers.
+ * 
        
+ * Servers are distinguished in two different ways.  A given server has a
+ * location, specified by hostname and port, and of which there can only be one
+ * online at any given time.  A server instance is specified by the location
+ * (hostname and port) as well as the startcode (timestamp from when the server
+ * was started).  This is used to differentiate a restarted instance of a given
+ * server from the original instance.
+ */
+public class ServerManager {
+  public static final String WAIT_ON_REGIONSERVERS_MAXTOSTART =
+      "hbase.master.wait.on.regionservers.maxtostart";
+
+  public static final String WAIT_ON_REGIONSERVERS_MINTOSTART =
+      "hbase.master.wait.on.regionservers.mintostart";
+
+  public static final String WAIT_ON_REGIONSERVERS_TIMEOUT =
+      "hbase.master.wait.on.regionservers.timeout";
+
+  public static final String WAIT_ON_REGIONSERVERS_INTERVAL =
+      "hbase.master.wait.on.regionservers.interval";
+
+  private static final Log LOG = LogFactory.getLog(ServerManager.class);
+
+  // Set if we are to shutdown the cluster.
+  private volatile boolean clusterShutdown = false;
+
+  /** Map of registered servers to their current load */
+  private final Map onlineServers =
+    new ConcurrentHashMap();
+
+  // TODO: This is strange to have two maps but HSI above is used on both sides
+  /**
+   * Map from full server-instance name to the RPC connection for this server.
+   */
+  private final Map serverConnections =
+    new HashMap();
+
+  /**
+   * List of region servers  that should not get any more new
+   * regions.
+   */
+  private final ArrayList drainingServers =
+    new ArrayList();
+
+  private final Server master;
+  private final MasterServices services;
+  private final HConnection connection;
+
+  private final DeadServer deadservers;
+
+  private final long maxSkew;
+  private final long warningSkew;
+
+  /**
+   * Set of region servers which are dead but not expired immediately. If one
+   * server died before master enables ServerShutdownHandler, the server will be
+   * added to set and will be expired through calling
+   * {@link ServerManager#expireDeadNotExpiredServers()} by master.
+   */
+  private Set deadNotExpiredServers = new HashSet();
+
+  /**
+   * Flag to enable SSH for ROOT region server. It's used in master initialization to enable SSH for
+   * ROOT before META assignment.
+   */
+  private boolean isSSHForRootEnabled = false;
+
+  /**
+   * Constructor.
+   * @param master
+   * @param services
+   * @throws ZooKeeperConnectionException
+   */
+  public ServerManager(final Server master, final MasterServices services)
+      throws ZooKeeperConnectionException {
+    this(master, services, true);
+  }
+
+  ServerManager(final Server master, final MasterServices services,
+      final boolean connect) throws ZooKeeperConnectionException {
+    this.master = master;
+    this.services = services;
+    Configuration c = master.getConfiguration();
+    maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
+    warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
+    this.deadservers = new DeadServer();
+    this.connection = connect ? HConnectionManager.getConnection(c) : null;
+  }
+
+  /**
+   * Let the server manager know a new regionserver has come online
+   * @param ia The remote address
+   * @param port The remote port
+   * @param serverStartcode
+   * @param serverCurrentTime The current time of the region server in ms
+   * @return The ServerName we know this server as.
+   * @throws IOException
+   */
+  ServerName regionServerStartup(final InetAddress ia, final int port,
+    final long serverStartcode, long serverCurrentTime)
+  throws IOException {
+    // Test for case where we get a region startup message from a regionserver
+    // that has been quickly restarted but whose znode expiration handler has
+    // not yet run, or from a server whose fail we are currently processing.
+    // Test its host+port combo is present in serverAddresstoServerInfo.  If it
+    // is, reject the server and trigger its expiration. The next time it comes
+    // in, it should have been removed from serverAddressToServerInfo and queued
+    // for processing by ProcessServerShutdown.
+    ServerName sn = new ServerName(ia.getHostName(), port, serverStartcode);
+    checkClockSkew(sn, serverCurrentTime);
+    checkIsDead(sn, "STARTUP");
+    checkAlreadySameHostPort(sn);
+    recordNewServer(sn, HServerLoad.EMPTY_HSERVERLOAD);
+    return sn;
+  }
+
+  void regionServerReport(ServerName sn, HServerLoad hsl)
+  throws YouAreDeadException, PleaseHoldException {
+    checkIsDead(sn, "REPORT");
+    if (!this.onlineServers.containsKey(sn)) {
+      // Already have this host+port combo and its just different start code?
+      checkAlreadySameHostPort(sn);
+      // Just let the server in. Presume master joining a running cluster.
+      // recordNewServer is what happens at the end of reportServerStartup.
+      // The only thing we are skipping is passing back to the regionserver
+      // the ServerName to use. Here we presume a master has already done
+      // that so we'll press on with whatever it gave us for ServerName.
+      recordNewServer(sn, hsl);
+    } else {
+      this.onlineServers.put(sn, hsl);
+    }
+  }
+
+  /**
+   * Test to see if we have a server of same host and port already.
+   * @param serverName
+   * @throws PleaseHoldException
+   */
+  void checkAlreadySameHostPort(final ServerName serverName)
+  throws PleaseHoldException {
+    ServerName existingServer =
+      ServerName.findServerWithSameHostnamePort(getOnlineServersList(), serverName);
+    if (existingServer != null) {
+      String message = "Server serverName=" + serverName +
+        " rejected; we already have " + existingServer.toString() +
+        " registered with same hostname and port";
+      LOG.info(message);
+      if (existingServer.getStartcode() < serverName.getStartcode()) {
+        LOG.info("Triggering server recovery; existingServer " +
+          existingServer + " looks stale, new server:" + serverName);
+        expireServer(existingServer);
+      }
+      if (services.isServerShutdownHandlerEnabled()) {
+        // master has completed the initialization
+        throw new PleaseHoldException(message);
+      }
+    }
+  }
+
+  /**
+   * Checks if the clock skew between the server and the master. If the clock skew exceeds the 
+   * configured max, it will throw an exception; if it exceeds the configured warning threshold, 
+   * it will log a warning but start normally.
+   * @param serverName Incoming servers's name
+   * @param serverCurrentTime
+   * @throws ClockOutOfSyncException if the skew exceeds the configured max value
+   */
+  private void checkClockSkew(final ServerName serverName, final long serverCurrentTime)
+  throws ClockOutOfSyncException {
+    long skew = System.currentTimeMillis() - serverCurrentTime;
+    if (skew > maxSkew) {
+      String message = "Server " + serverName + " has been " +
+        "rejected; Reported time is too far out of sync with master.  " +
+        "Time difference of " + skew + "ms > max allowed of " + maxSkew + "ms";
+      LOG.warn(message);
+      throw new ClockOutOfSyncException(message);
+    } else if (skew > warningSkew){
+      String message = "Reported time for server " + serverName + " is out of sync with master " +
+        "by " + skew + "ms. (Warning threshold is " + warningSkew + "ms; " + 
+        "error threshold is " + maxSkew + "ms)";
+      LOG.warn(message);
+    }
+  }
+
+  /**
+   * If this server is on the dead list, reject it with a YouAreDeadException.
+   * If it was dead but came back with a new start code, remove the old entry
+   * from the dead list.
+   * @param serverName
+   * @param what START or REPORT
+   * @throws YouAreDeadException
+   */
+  private void checkIsDead(final ServerName serverName, final String what)
+      throws YouAreDeadException {
+    if (this.deadservers.isDeadServer(serverName)) {
+      // host name, port and start code all match with existing one of the
+      // dead servers. So, this server must be dead.
+      String message = "Server " + what + " rejected; currently processing " +
+          serverName + " as dead server";
+      LOG.debug(message);
+      throw new YouAreDeadException(message);
+    }
+
+    // remove dead server with same hostname and port of newly checking in rs after master
+    // initialization.See HBASE-5916 for more information.
+    if ((this.services == null || ((HMaster) this.services).isInitialized())
+        && this.deadservers.cleanPreviousInstance(serverName)) {
+      // This server has now become alive after we marked it as dead.
+      // We removed it's previous entry from the dead list to reflect it.
+      LOG.debug(what + ":" + " Server " + serverName + " came back up," +
+          " removed it from the dead servers list");
+    }
+  }
+
+  /**
+   * Adds the onlineServers list.
+   * @param hsl
+   * @param serverName The remote servers name.
+   */
+  void recordNewServer(final ServerName serverName, final  HServerLoad hsl) {
+    LOG.info("Registering server=" + serverName);
+    this.onlineServers.put(serverName, hsl);
+    this.serverConnections.remove(serverName);
+  }
+
+  /**
+   * @param serverName
+   * @return HServerLoad if serverName is known else null
+   */
+  public HServerLoad getLoad(final ServerName serverName) {
+    return this.onlineServers.get(serverName);
+  }
+
+  /**
+   * @param address
+   * @return HServerLoad if serverName is known else null
+   * @deprecated Use {@link #getLoad(HServerAddress)}
+   */
+  public HServerLoad getLoad(final HServerAddress address) {
+    ServerName sn = new ServerName(address.toString(), ServerName.NON_STARTCODE);
+    ServerName actual =
+      ServerName.findServerWithSameHostnamePort(this.getOnlineServersList(), sn);
+    return actual == null? null: getLoad(actual);
+  }
+
+  /**
+   * Compute the average load across all region servers.
+   * Currently, this uses a very naive computation - just uses the number of
+   * regions being served, ignoring stats about number of requests.
+   * @return the average load
+   */
+  public double getAverageLoad() {
+    int totalLoad = 0;
+    int numServers = 0;
+    double averageLoad = 0.0;
+    for (HServerLoad hsl: this.onlineServers.values()) {
+        numServers++;
+        totalLoad += hsl.getNumberOfRegions();
+    }
+    averageLoad = (double)totalLoad / (double)numServers;
+    return averageLoad;
+  }
+
+  /** @return the count of active regionservers */
+  int countOfRegionServers() {
+    // Presumes onlineServers is a concurrent map
+    return this.onlineServers.size();
+  }
+
+  /**
+   * @return Read-only map of servers to serverinfo
+   */
+  public Map getOnlineServers() {
+    // Presumption is that iterating the returned Map is OK.
+    synchronized (this.onlineServers) {
+      return Collections.unmodifiableMap(this.onlineServers);
+    }
+  }
+
+  public Set getDeadServers() {
+    return this.deadservers.clone();
+  }
+
+  /**
+   * Checks if any dead servers are currently in progress.
+   * @return true if any RS are being processed as dead, false if not
+   */
+  public boolean areDeadServersInProgress() {
+    return this.deadservers.areDeadServersInProgress();
+  }
+
+  void letRegionServersShutdown() {
+    long previousLogTime = 0;
+    while (!onlineServers.isEmpty()) {
+
+      if (System.currentTimeMillis() > (previousLogTime + 1000)) {
+        StringBuilder sb = new StringBuilder();
+        for (ServerName key : this.onlineServers.keySet()) {
+          if (sb.length() > 0) {
+            sb.append(", ");
+          }
+          sb.append(key);
+        }
+        LOG.info("Waiting on regionserver(s) to go down " + sb.toString());
+        previousLogTime = System.currentTimeMillis();
+      }
+
+      synchronized (onlineServers) {
+        try {
+          onlineServers.wait(100);
+        } catch (InterruptedException ignored) {
+          // continue
+        }
+      }
+    }
+  }
+
+  /*
+   * Expire the passed server.  Add it to list of deadservers and queue a
+   * shutdown processing.
+   */
+  public synchronized void expireServer(final ServerName serverName) {
+    boolean carryingRoot = services.getAssignmentManager().isCarryingRoot(serverName);
+    if (!services.isServerShutdownHandlerEnabled() && (!carryingRoot || !this.isSSHForRootEnabled)) {
+      LOG.info("Master doesn't enable ServerShutdownHandler during initialization, "
+          + "delay expiring server " + serverName);
+      this.deadNotExpiredServers.add(serverName);
+      return;
+    }
+    if (!this.onlineServers.containsKey(serverName)) {
+      LOG.warn("Received expiration of " + serverName +
+        " but server is not currently online");
+    }
+    if (this.deadservers.contains(serverName)) {
+      // TODO: Can this happen?  It shouldn't be online in this case?
+      LOG.warn("Received expiration of " + serverName +
+          " but server shutdown is already in progress");
+      return;
+    }
+    // Remove the server from the known servers lists and update load info BUT
+    // add to deadservers first; do this so it'll show in dead servers list if
+    // not in online servers list.
+    this.deadservers.add(serverName);
+    this.onlineServers.remove(serverName);
+    synchronized (onlineServers) {
+      onlineServers.notifyAll();
+    }
+    this.serverConnections.remove(serverName);
+    // If cluster is going down, yes, servers are going to be expiring; don't
+    // process as a dead server
+    if (this.clusterShutdown) {
+      LOG.info("Cluster shutdown set; " + serverName +
+        " expired; onlineServers=" + this.onlineServers.size());
+      if (this.onlineServers.isEmpty()) {
+        master.stop("Cluster shutdown set; onlineServer=0");
+      }
+      return;
+    }
+
+    boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName);
+    if (carryingRoot || carryingMeta) {
+      this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master,
+        this.services, this.deadservers, serverName, carryingRoot, carryingMeta));
+    } else {
+      this.services.getExecutorService().submit(new ServerShutdownHandler(this.master,
+        this.services, this.deadservers, serverName, true));
+    }
+    LOG.debug("Added=" + serverName +
+      " to dead servers, submitted shutdown handler to be executed, root=" +
+        carryingRoot + ", meta=" + carryingMeta);
+  }
+
+  /**
+   * Expire the servers which died during master's initialization. It will be
+   * called after HMaster#assignRootAndMeta.
+   * @throws IOException
+   * */
+  synchronized void expireDeadNotExpiredServers() throws IOException {
+    if (!services.isServerShutdownHandlerEnabled()) {
+      throw new IOException("Master hasn't enabled ServerShutdownHandler ");
+    }
+    Iterator serverIterator = deadNotExpiredServers.iterator();
+    while (serverIterator.hasNext()) {
+      expireServer(serverIterator.next());
+      serverIterator.remove();
+    }
+  }
+
+  /**
+   * Enable SSH for ROOT region server and expire ROOT which died during master's initialization. It
+   * will be called before Meta assignment.
+   * @throws IOException
+   */
+  void enableSSHForRoot() throws IOException {
+    if (this.isSSHForRootEnabled) {
+      return;
+    }
+    this.isSSHForRootEnabled = true;
+    Iterator serverIterator = deadNotExpiredServers.iterator();
+    while (serverIterator.hasNext()) {
+      ServerName curServerName = serverIterator.next();
+      if (services.getAssignmentManager().isCarryingRoot(curServerName)) {
+        expireServer(curServerName);
+        serverIterator.remove();
+      }
+    }
+  }
+
+  /**
+   * Reset flag isSSHForRootEnabled to false
+   */
+  void disableSSHForRoot() {
+    this.isSSHForRootEnabled = false;
+  }
+
+  /*
+   * Remove the server from the drain list.
+   */
+  public boolean removeServerFromDrainList(final ServerName sn) {
+    // Warn if the server (sn) is not online.  ServerName is of the form:
+    //  ,  , 
+
+    if (!this.isServerOnline(sn)) {
+      LOG.warn("Server " + sn + " is not currently online. " +
+               "Removing from draining list anyway, as requested.");
+    }
+    // Remove the server from the draining servers lists.
+    return this.drainingServers.remove(sn);
+  }
+
+  /*
+   * Add the server to the drain list.
+   */
+  public boolean addServerToDrainList(final ServerName sn) {
+    // Warn if the server (sn) is not online.  ServerName is of the form:
+    //  ,  , 
+
+    if (!this.isServerOnline(sn)) {
+      LOG.warn("Server " + sn + " is not currently online. " +
+               "Ignoring request to add it to draining list.");
+      return false;
+    }
+    // Add the server to the draining servers lists, if it's not already in
+    // it.
+    if (this.drainingServers.contains(sn)) {
+      LOG.warn("Server " + sn + " is already in the draining server list." +
+               "Ignoring request to add it again.");
+      return false;
+    }
+    return this.drainingServers.add(sn);
+  }
+
+  // RPC methods to region servers
+
+  /**
+   * Sends an OPEN RPC to the specified server to open the specified region.
+   * 
        
+   * Open should not fail but can if server just crashed.
+   * 
        
+   * @param server server to open a region
+   * @param region region to open
+   * @param versionOfOfflineNode that needs to be present in the offline node
+   * when RS tries to change the state from OFFLINE to other states.
+   */
+  public RegionOpeningState sendRegionOpen(final ServerName server,
+      HRegionInfo region, int versionOfOfflineNode)
+  throws IOException {
+    HRegionInterface hri = getServerConnection(server);
+    if (hri == null) {
+      LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
+        " failed because no RPC connection found to this server");
+      return RegionOpeningState.FAILED_OPENING;
+    }
+    return (versionOfOfflineNode == -1) ? hri.openRegion(region) : hri
+        .openRegion(region, versionOfOfflineNode);
+  }
+
+  /**
+   * Sends an OPEN RPC to the specified server to open the specified region.
+   * 
        
+   * Open should not fail but can if server just crashed.
+   * 
        
+   * @param server server to open a region
+   * @param regions regions to open
+   */
+  public void sendRegionOpen(ServerName server, List regions)
+  throws IOException {
+    HRegionInterface hri = getServerConnection(server);
+    if (hri == null) {
+      LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
+        " failed because no RPC connection found to this server");
+      return;
+    }
+    hri.openRegions(regions);
+  }
+
+  /**
+   * Sends an CLOSE RPC to the specified server to close the specified region.
+   * 
        
+   * A region server could reject the close request because it either does not
+   * have the specified region or the region is being split.
+   * @param server server to open a region
+   * @param region region to open
+   * @param versionOfClosingNode
+   *   the version of znode to compare when RS transitions the znode from
+   *   CLOSING state.
+   * @return true if server acknowledged close, false if not
+   * @throws IOException
+   */
+  public boolean sendRegionClose(ServerName server, HRegionInfo region,
+    int versionOfClosingNode) throws IOException {
+    if (server == null) throw new NullPointerException("Passed server is null");
+    HRegionInterface hri = getServerConnection(server);
+    if (hri == null) {
+      throw new IOException("Attempting to send CLOSE RPC to server " +
+        server.toString() + " for region " +
+        region.getRegionNameAsString() +
+        " failed because no RPC connection found to this server");
+    }
+    return hri.closeRegion(region, versionOfClosingNode);
+  }
+
+  /**
+   * @param sn
+   * @return
+   * @throws IOException
+   * @throws RetriesExhaustedException wrapping a ConnectException if failed
+   * putting up proxy.
+   */
+  private HRegionInterface getServerConnection(final ServerName sn)
+  throws IOException {
+    HRegionInterface hri = this.serverConnections.get(sn);
+    if (hri == null) {
+      LOG.debug("New connection to " + sn.toString());
+      hri = this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());
+      this.serverConnections.put(sn, hri);
+    }
+    return hri;
+  }
+
+  /**
+   * Wait for the region servers to report in.
+   * We will wait until one of this condition is met:
+   *  - the master is stopped
+   *  - the 'hbase.master.wait.on.regionservers.maxtostart' number of
+   *    region servers is reached
+   *  - the 'hbase.master.wait.on.regionservers.mintostart' is reached AND
+   *   there have been no new region server in for
+   *      'hbase.master.wait.on.regionservers.interval' time AND
+   *   the 'hbase.master.wait.on.regionservers.timeout' is reached
+   *
+   * @throws InterruptedException
+   */
+  public void waitForRegionServers(MonitoredTask status)
+  throws InterruptedException {
+    final long interval = this.master.getConfiguration().
+      getLong(WAIT_ON_REGIONSERVERS_INTERVAL, 1500);
+    final long timeout = this.master.getConfiguration().
+      getLong(WAIT_ON_REGIONSERVERS_TIMEOUT, 4500);
+    int minToStart = this.master.getConfiguration().
+      getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
+    if (minToStart < 1) {
+      LOG.warn(String.format(
+        "The value of '%s' (%d) can not be less than 1, ignoring.",
+        WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
+      minToStart = 1;
+    }
+    int maxToStart = this.master.getConfiguration().
+      getInt(WAIT_ON_REGIONSERVERS_MAXTOSTART, Integer.MAX_VALUE);
+    if (maxToStart < minToStart) {
+        LOG.warn(String.format(
+            "The value of '%s' (%d) is set less than '%s' (%d), ignoring.",
+            WAIT_ON_REGIONSERVERS_MAXTOSTART, maxToStart,
+            WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
+        maxToStart = Integer.MAX_VALUE;
+    }
+
+    long now =  System.currentTimeMillis();
+    final long startTime = now;
+    long slept = 0;
+    long lastLogTime = 0;
+    long lastCountChange = startTime;
+    int count = countOfRegionServers();
+    int oldCount = 0;
+    while (
+      !this.master.isStopped() &&
+        count < maxToStart &&
+        (lastCountChange+interval > now || timeout > slept || count < minToStart)
+      ){
+
+      // Log some info at every interval time or if there is a change
+      if (oldCount != count || lastLogTime+interval < now){
+        lastLogTime = now;
+        String msg =
+          "Waiting for region servers count to settle; currently"+
+            " checked in " + count + ", slept for " + slept + " ms," +
+            " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+
+            ", timeout of "+timeout+" ms, interval of "+interval+" ms.";
+        LOG.info(msg);
+        status.setStatus(msg);
+      }
+
+      // We sleep for some time
+      final long sleepTime = 50;
+      Thread.sleep(sleepTime);
+      now =  System.currentTimeMillis();
+      slept = now - startTime;
+
+      oldCount = count;
+      count = countOfRegionServers();
+      if (count != oldCount) {
+        lastCountChange = now;
+      }
+    }
+
+    LOG.info("Finished waiting for region servers count to settle;" +
+      " checked in " + count + ", slept for " + slept + " ms," +
+      " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+","+
+      " master is "+ (this.master.isStopped() ? "stopped.": "running.")
+    );
+  }
+
+  /**
+   * @return A copy of the internal list of online servers.
+   */
+  public List getOnlineServersList() {
+    // TODO: optimize the load balancer call so we don't need to make a new list
+    // TODO: FIX. THIS IS POPULAR CALL.
+    return new ArrayList(this.onlineServers.keySet());
+  }
+
+  /**
+   * @return A copy of the internal list of draining servers.
+   */
+  public List getDrainingServersList() {
+    return new ArrayList(this.drainingServers);
+  }
+
+  /**
+   * @return A copy of the internal set of deadNotExpired servers.
+   */
+  Set getDeadNotExpiredServers() {
+    return new HashSet(this.deadNotExpiredServers);
+  }
+
+  public boolean isServerOnline(ServerName serverName) {
+    return onlineServers.containsKey(serverName);
+  }
+
+  public void shutdownCluster() {
+    this.clusterShutdown = true;
+    this.master.stop("Cluster shutdown requested");
+  }
+
+  public boolean isClusterShutdown() {
+    return this.clusterShutdown;
+  }
+
+  /**
+   * Stop the ServerManager.  Currently closes the connection to the master.
+   */
+  public void stop() {
+    if (connection != null) {
+      try {
+        connection.close();
+      } catch (IOException e) {
+        LOG.error("Attempt to close connection to master failed", e);
+      }
+    }
+  }
+    
+  /**
+   * To clear any dead server with same host name and port of any online server
+   */
+  void clearDeadServersWithSameHostNameAndPortOfOnlineServer() {
+    ServerName sn = null;
+    for (ServerName serverName : getOnlineServersList()) {
+      while ((sn = ServerName.
+          findServerWithSameHostnamePort(this.deadservers, serverName)) != null) {
+        this.deadservers.remove(sn);
+      }
+    }
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java b/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
new file mode 100644
index 0000000..d621d77
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/SnapshotSentinel.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+
+/**
+ * Watch the current snapshot under process
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface SnapshotSentinel {
+
+  /**
+   * Check to see if the snapshot is finished, where finished may be success or failure.
+   * @return false if the snapshot is still in progress, true if the snapshot has
+   *         finished
+   */
+  public boolean isFinished();
+
+  /**
+   * @return -1 if the snapshot is in progress, otherwise the completion timestamp.
+   */
+  public long getCompletionTimestamp();
+
+  /**
+   * Actively cancel a running snapshot.
+   * @param why Reason for cancellation.
+   */
+  public void cancel(String why);
+
+  /**
+   * @return the description of the snapshot being run
+   */
+  public SnapshotDescription getSnapshot();
+
+  /**
+   * Get the exception that caused the snapshot to fail, if the snapshot has failed.
+   * @return {@link ForeignException} that caused the snapshot to fail, or null if the
+   *  snapshot is still in progress or has succeeded
+   */
+  public ForeignException getExceptionIfFailed();
+
+  /**
+   * Rethrow the exception returned by {@link SnapshotSentinel#getExceptionIfFailed}.
+   * If there is no exception this is a no-op.
+   *
+   * @throws ForeignException all exceptions from remote sources are procedure exceptions
+   */
+  public void rethrowExceptionIfFailed() throws ForeignException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
new file mode 100644
index 0000000..3c9fcc5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -0,0 +1,1300 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.master.SplitLogManager.TaskFinisher.Status;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.regionserver.SplitLogWorker;
+import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
+import org.apache.hadoop.hbase.zookeeper.ZKSplitLog.TaskState;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.data.Stat;
+
+import com.google.common.base.Strings;
+
+import static org.apache.hadoop.hbase.master.SplitLogManager.ResubmitDirective.*;
+import static org.apache.hadoop.hbase.master.SplitLogManager.TerminationStatus.*;
+
+/**
+ * Distributes the task of log splitting to the available region servers.
+ * Coordination happens via zookeeper. For every log file that has to be split a
+ * znode is created under /hbase/splitlog. SplitLogWorkers race to grab a task.
+ *
+ * SplitLogManager monitors the task znodes that it creates using the
+ * timeoutMonitor thread. If a task's progress is slow then
+ * resubmit(String, boolean) will take away the task from the owner
+ * {@link SplitLogWorker} and the task will be
+ * upforgrabs again. When the task is done then the task's znode is deleted by
+ * SplitLogManager.
+ *
+ * Clients call {@link #splitLogDistributed(Path)} to split a region server's
+ * log files. The caller thread waits in this method until all the log files
+ * have been split.
+ *
+ * All the zookeeper calls made by this class are asynchronous. This is mainly
+ * to help reduce response time seen by the callers.
+ *
+ * There is race in this design between the SplitLogManager and the
+ * SplitLogWorker. SplitLogManager might re-queue a task that has in reality
+ * already been completed by a SplitLogWorker. We rely on the idempotency of
+ * the log splitting task for correctness.
+ *
+ * It is also assumed that every log splitting task is unique and once
+ * completed (either with success or with error) it will be not be submitted
+ * again. If a task is resubmitted then there is a risk that old "delete task"
+ * can delete the re-submission.
+ */
+public class SplitLogManager extends ZooKeeperListener {
+  private static final Log LOG = LogFactory.getLog(SplitLogManager.class);
+
+  private final Stoppable stopper;
+  private final MasterServices master;
+  private final String serverName;
+  private final TaskFinisher taskFinisher;
+  private FileSystem fs;
+  private Configuration conf;
+
+  private long zkretries;
+  private long resubmit_threshold;
+  private long timeout;
+  private long unassignedTimeout;
+  private long lastNodeCreateTime = Long.MAX_VALUE;
+  public boolean ignoreZKDeleteForTesting = false;
+
+  private final ConcurrentMap tasks =
+    new ConcurrentHashMap();
+  private TimeoutMonitor timeoutMonitor;
+
+  private Set deadWorkers = null;
+  private final Object deadWorkersLock = new Object();
+
+  private Set failedDeletions = null;
+
+  /**
+   * Wrapper around {@link #SplitLogManager(ZooKeeperWatcher, Configuration,
+   * Stoppable, String, TaskFinisher)} that provides a task finisher for
+   * copying recovered edits to their final destination. The task finisher
+   * has to be robust because it can be arbitrarily restarted or called
+   * multiple times.
+   * 
+   * @param zkw
+   * @param conf
+   * @param stopper
+   * @param serverName
+   */
+  public SplitLogManager(ZooKeeperWatcher zkw, final Configuration conf,
+      Stoppable stopper, MasterServices master, String serverName) {
+    this(zkw, conf, stopper, master, serverName, new TaskFinisher() {
+      @Override
+      public Status finish(String workerName, String logfile) {
+        try {
+          HLogSplitter.finishSplitLogFile(logfile, conf);
+        } catch (IOException e) {
+          LOG.warn("Could not finish splitting of log file " + logfile, e);
+          return Status.ERR;
+        }
+        return Status.DONE;
+      }
+    });
+  }
+
+  /**
+   * Its OK to construct this object even when region-servers are not online. It
+   * does lookup the orphan tasks in zk but it doesn't block waiting for them
+   * to be done.
+   *
+   * @param zkw
+   * @param conf
+   * @param stopper
+   * @param serverName
+   * @param tf task finisher 
+   */
+  public SplitLogManager(ZooKeeperWatcher zkw, Configuration conf,
+      Stoppable stopper, MasterServices master, String serverName, TaskFinisher tf) {
+    super(zkw);
+    this.taskFinisher = tf;
+    this.conf = conf;
+    this.stopper = stopper;
+    this.master = master;
+    this.zkretries = conf.getLong("hbase.splitlog.zk.retries",
+        ZKSplitLog.DEFAULT_ZK_RETRIES);
+    this.resubmit_threshold = conf.getLong("hbase.splitlog.max.resubmit",
+        ZKSplitLog.DEFAULT_MAX_RESUBMIT);
+    this.timeout = conf.getInt("hbase.splitlog.manager.timeout",
+        ZKSplitLog.DEFAULT_TIMEOUT);
+    this.unassignedTimeout =
+      conf.getInt("hbase.splitlog.manager.unassigned.timeout",
+        ZKSplitLog.DEFAULT_UNASSIGNED_TIMEOUT);
+    LOG.info("timeout = " + timeout);
+    LOG.info("unassigned timeout = " + unassignedTimeout);
+    LOG.info("resubmit threshold = " + this.resubmit_threshold);
+
+    this.serverName = serverName;
+    this.timeoutMonitor = new TimeoutMonitor(
+        conf.getInt("hbase.splitlog.manager.timeoutmonitor.period",
+            1000),
+        stopper);
+
+    this.failedDeletions = Collections.synchronizedSet(new HashSet());
+  }
+
+  public void finishInitialization(boolean masterRecovery) {
+    if (!masterRecovery) {
+      Threads.setDaemonThreadRunning(timeoutMonitor.getThread(), serverName
+          + ".splitLogManagerTimeoutMonitor");
+    }
+    // Watcher can be null during tests with Mock'd servers.
+    if (this.watcher != null) {
+      this.watcher.registerListener(this);
+      lookForOrphans();
+    }
+  }
+
+  private FileStatus[] getFileList(List logDirs, PathFilter filter) throws IOException {
+    List fileStatus = new ArrayList();
+    for (Path hLogDir : logDirs) {
+      this.fs = hLogDir.getFileSystem(conf);
+      if (!fs.exists(hLogDir)) {
+        LOG.warn(hLogDir + " doesn't exist. Nothing to do!");
+        continue;
+      }
+      FileStatus[] logfiles = FSUtils.listStatus(fs, hLogDir, filter);
+      if (logfiles == null || logfiles.length == 0) {
+        LOG.info(hLogDir + " is empty dir, no logs to split");
+      } else {
+        for (FileStatus status : logfiles)
+          fileStatus.add(status);
+      }
+    }
+    FileStatus[] a = new FileStatus[fileStatus.size()];
+    return fileStatus.toArray(a);
+  }
+
+  /**
+   * @param logDir
+   *            one region sever hlog dir path in .logs
+   * @throws IOException
+   *             if there was an error while splitting any log file
+   * @return cumulative size of the logfiles split
+   * @throws IOException 
+   */
+  public long splitLogDistributed(final Path logDir) throws IOException {
+    List logDirs = new ArrayList();
+    logDirs.add(logDir);
+    return splitLogDistributed(logDirs);
+  }
+
+  /**
+   * The caller will block until all the log files of the given region server
+   * have been processed - successfully split or an error is encountered - by an
+   * available worker region server. This method must only be called after the
+   * region servers have been brought online.
+   *
+   * @param logDirs
+   * @throws IOException
+   *          if there was an error while splitting any log file
+   * @return cumulative size of the logfiles split
+   */
+  public long splitLogDistributed(final List logDirs) throws IOException {
+    return splitLogDistributed(logDirs, null);
+  }
+
+  /**
+   * The caller will block until all the META log files of the given region server
+   * have been processed - successfully split or an error is encountered - by an
+   * available worker region server. This method must only be called after the
+   * region servers have been brought online.
+   *
+   * @param logDirs List of log dirs to split
+   * @param filter the Path filter to select specific files for considering
+   * @throws IOException If there was an error while splitting any log file
+   * @return cumulative size of the logfiles split
+   */
+  public long splitLogDistributed(final List logDirs, PathFilter filter) 
+      throws IOException {
+    MonitoredTask status = TaskMonitor.get().createStatus(
+          "Doing distributed log split in " + logDirs);
+    FileStatus[] logfiles = getFileList(logDirs, filter);
+    status.setStatus("Checking directory contents...");
+    LOG.debug("Scheduling batch of logs to split");
+    tot_mgr_log_split_batch_start.incrementAndGet();
+    LOG.info("started splitting logs in " + logDirs);
+    long t = EnvironmentEdgeManager.currentTimeMillis();
+    long totalSize = 0;
+    TaskBatch batch = new TaskBatch();
+    for (FileStatus lf : logfiles) {
+      // TODO If the log file is still being written to - which is most likely
+      // the case for the last log file - then its length will show up here
+      // as zero. The size of such a file can only be retrieved after
+      // recover-lease is done. totalSize will be under in most cases and the
+      // metrics that it drives will also be under-reported.
+      totalSize += lf.getLen();
+      if (enqueueSplitTask(lf.getPath().toString(), batch) == false) {
+        throw new IOException("duplicate log split scheduled for "
+            + lf.getPath());
+      }
+    }
+    waitForSplittingCompletion(batch, status);
+    if (batch.done != batch.installed) {
+      batch.isDead = true;
+      tot_mgr_log_split_batch_err.incrementAndGet();
+      LOG.warn("error while splitting logs in " + logDirs +
+      " installed = " + batch.installed + " but only " + batch.done + " done");
+      String msg = "error or interrupted while splitting logs in "
+        + logDirs + " Task = " + batch;
+      status.abort(msg);
+      throw new IOException(msg);
+    }
+    for(Path logDir: logDirs){
+      status.setStatus("Cleaning up log directory...");
+      try {
+        if (fs.exists(logDir) && !HBaseFileSystem.deleteFileFromFileSystem(fs, logDir)) {
+          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);
+        }
+      } catch (IOException ioe) {
+        FileStatus[] files = fs.listStatus(logDir);
+        if (files != null && files.length > 0) {
+          LOG.warn("returning success without actually splitting and " + 
+              "deleting all the log files in path " + logDir);
+        } else {
+          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);
+        }
+      }
+      tot_mgr_log_split_batch_success.incrementAndGet();
+    }
+    String msg = "finished splitting (more than or equal to) " + totalSize +
+        " bytes in " + batch.installed + " log files in " + logDirs + " in " +
+        (EnvironmentEdgeManager.currentTimeMillis() - t) + "ms";
+    status.markComplete(msg);
+    LOG.info(msg);
+    return totalSize;
+  }
+
+  /**
+   * Add a task entry to splitlog znode if it is not already there.
+   * 
+   * @param taskname the path of the log to be split
+   * @param batch the batch this task belongs to
+   * @return true if a new entry is created, false if it is already there.
+   */
+  boolean enqueueSplitTask(String taskname, TaskBatch batch) {
+    tot_mgr_log_split_start.incrementAndGet();
+    String path = ZKSplitLog.getEncodedNodeName(watcher, taskname);
+    Task oldtask = createTaskIfAbsent(path, batch);
+    if (oldtask == null) {
+      // publish the task in zk
+      createNode(path, zkretries);
+      return true;
+    }
+    return false;
+  }
+
+  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {
+    synchronized (batch) {
+      while ((batch.done + batch.error) != batch.installed) {
+        try {
+          status.setStatus("Waiting for distributed tasks to finish. "
+              + " scheduled=" + batch.installed
+              + " done=" + batch.done
+              + " error=" + batch.error);
+          int remaining = batch.installed - (batch.done + batch.error);
+          int actual = activeTasks(batch);
+          if (remaining != actual) {
+            LOG.warn("Expected " + remaining
+              + " active tasks, but actually there are " + actual);
+          }
+          int remainingInZK = remainingTasksInZK();
+          if (remainingInZK >= 0 && actual > remainingInZK) {
+            LOG.warn("Expected at least" + actual
+              + " tasks in ZK, but actually there are " + remainingInZK);
+          }
+          if (remainingInZK == 0 || actual == 0) {
+            LOG.warn("No more task remaining (ZK or task map), splitting "
+              + "should have completed. Remaining tasks in ZK " + remainingInZK
+              + ", active tasks in map " + actual);
+            if (remainingInZK == 0 && actual == 0) {
+              return;
+            }
+          }
+          batch.wait(100);
+          if (stopper.isStopped()) {
+            LOG.warn("Stopped while waiting for log splits to be completed");
+            return;
+          }
+        } catch (InterruptedException e) {
+          LOG.warn("Interrupted while waiting for log splits to be completed");
+          Thread.currentThread().interrupt();
+          return;
+        }
+      }
+    }
+  }
+
+  private int activeTasks(final TaskBatch batch) {
+    int count = 0;
+    for (Task t: tasks.values()) {
+      if (t.batch == batch && t.status == TerminationStatus.IN_PROGRESS) {
+        count++;
+      }
+    }
+    return count;
+  }
+
+  private int remainingTasksInZK() {
+    int count = 0;
+    try {
+      List tasks =
+        ZKUtil.listChildrenNoWatch(watcher, watcher.splitLogZNode);
+      if (tasks != null) {
+        for (String t: tasks) {
+          if (!ZKSplitLog.isRescanNode(watcher, t)) {
+            count++;
+          }
+        }
+      }
+    } catch (KeeperException ke) {
+      LOG.warn("Failed to check remaining tasks", ke);
+      count = -1;
+    }
+    return count;
+  }
+
+  private void setDone(String path, TerminationStatus status) {
+    Task task = tasks.get(path);
+    if (task == null) {
+      if (!ZKSplitLog.isRescanNode(watcher, path)) {
+        tot_mgr_unacquired_orphan_done.incrementAndGet();
+        LOG.debug("unacquired orphan task is done " + path);
+      }
+    } else {
+      synchronized (task) {
+        if (task.status == IN_PROGRESS) {
+          if (status == SUCCESS) {
+            tot_mgr_log_split_success.incrementAndGet();
+            LOG.info("Done splitting " + path);
+          } else {
+            tot_mgr_log_split_err.incrementAndGet();
+            LOG.warn("Error splitting " + path);
+          }
+          task.status = status;
+          if (task.batch != null) {
+            synchronized (task.batch) {
+              if (status == SUCCESS) {
+                task.batch.done++;
+              } else {
+                task.batch.error++;
+              }
+              task.batch.notify();
+            }
+          }
+        }
+      }
+    }
+    // delete the task node in zk. It's an async
+    // call and no one is blocked waiting for this node to be deleted. All
+    // task names are unique (log.) there is no risk of deleting
+    // a future task.
+    // if a deletion fails, TimeoutMonitor will retry the same deletion later
+    deleteNode(path, zkretries);
+    return;
+  }
+
+  private void createNode(String path, Long retry_count) {
+    ZKUtil.asyncCreate(this.watcher, path,
+        TaskState.TASK_UNASSIGNED.get(serverName), new CreateAsyncCallback(),
+        retry_count);
+    tot_mgr_node_create_queued.incrementAndGet();
+    return;
+  }
+
+  private void createNodeSuccess(String path) {
+    lastNodeCreateTime = EnvironmentEdgeManager.currentTimeMillis();
+    LOG.debug("put up splitlog task at znode " + path);
+    getDataSetWatch(path, zkretries);
+  }
+
+  private void createNodeFailure(String path) {
+    // TODO the Manager should split the log locally instead of giving up
+    LOG.warn("failed to create task node" + path);
+    setDone(path, FAILURE);
+  }
+
+
+  private void getDataSetWatch(String path, Long retry_count) {
+    this.watcher.getRecoverableZooKeeper().getZooKeeper().
+        getData(path, this.watcher,
+        new GetDataAsyncCallback(), retry_count);
+    tot_mgr_get_data_queued.incrementAndGet();
+  }
+
+  private void tryGetDataSetWatch(String path) {
+    // A negative retry count will lead to ignoring all error processing.
+    this.watcher.getRecoverableZooKeeper().getZooKeeper().
+        getData(path, this.watcher,
+        new GetDataAsyncCallback(), new Long(-1) /* retry count */);
+    tot_mgr_get_data_queued.incrementAndGet();
+  }
+
+  private void getDataSetWatchSuccess(String path, byte[] data, int version) {
+    if (data == null) {
+      if (version == Integer.MIN_VALUE) {
+        // assume all done. The task znode suddenly disappeared.
+        setDone(path, SUCCESS);
+        return;
+      }
+      tot_mgr_null_data.incrementAndGet();
+      LOG.fatal("logic error - got null data " + path);
+      setDone(path, FAILURE);
+      return;
+    }
+    data = this.watcher.getRecoverableZooKeeper().removeMetaData(data);
+    // LOG.debug("set watch on " + path + " got data " + new String(data));
+    if (TaskState.TASK_UNASSIGNED.equals(data)) {
+      LOG.debug("task not yet acquired " + path + " ver = " + version);
+      handleUnassignedTask(path);
+    } else if (TaskState.TASK_OWNED.equals(data)) {
+      heartbeat(path, version,
+          TaskState.TASK_OWNED.getWriterName(data));
+    } else if (TaskState.TASK_RESIGNED.equals(data)) {
+      LOG.info("task " + path + " entered state " + new String(data));
+      resubmitOrFail(path, FORCE);
+    } else if (TaskState.TASK_DONE.equals(data)) {
+      LOG.info("task " + path + " entered state " + new String(data));
+      if (taskFinisher != null && !ZKSplitLog.isRescanNode(watcher, path)) {
+        if (taskFinisher.finish(TaskState.TASK_DONE.getWriterName(data),
+            ZKSplitLog.getFileName(path)) == Status.DONE) {
+          setDone(path, SUCCESS);
+        } else {
+          resubmitOrFail(path, CHECK);
+        }
+      } else {
+        setDone(path, SUCCESS);
+      }
+    } else if (TaskState.TASK_ERR.equals(data)) {
+      LOG.info("task " + path + " entered state " + new String(data));
+      resubmitOrFail(path, CHECK);
+    } else {
+      LOG.fatal("logic error - unexpected zk state for path = " + path
+          + " data = " + new String(data));
+      setDone(path, FAILURE);
+    }
+  }
+
+  private void getDataSetWatchFailure(String path) {
+    LOG.warn("failed to set data watch " + path);
+    setDone(path, FAILURE);
+  }
+
+  /**
+   * It is possible for a task to stay in UNASSIGNED state indefinitely - say
+   * SplitLogManager wants to resubmit a task. It forces the task to UNASSIGNED
+   * state but it dies before it could create the RESCAN task node to signal
+   * the SplitLogWorkers to pick up the task. To prevent this scenario the
+   * SplitLogManager resubmits all orphan and UNASSIGNED tasks at startup.
+   *
+   * @param path
+   */
+  private void handleUnassignedTask(String path) {
+    if (ZKSplitLog.isRescanNode(watcher, path)) {
+      return;
+    }
+    Task task = findOrCreateOrphanTask(path);
+    if (task.isOrphan() && (task.incarnation == 0)) {
+      LOG.info("resubmitting unassigned orphan task " + path);
+      // ignore failure to resubmit. The timeout-monitor will handle it later
+      // albeit in a more crude fashion
+      resubmit(path, task, FORCE);
+    }
+  }
+
+  /**
+   * Helper function to check whether to abandon retries in ZooKeeper AsyncCallback functions
+   * @param statusCode integer value of a ZooKeeper exception code
+   * @param action description message about the retried action
+   * @return true when need to abandon retries, otherwise false
+   */
+  private boolean shouldAbandonRetries(int statusCode, String action) {
+    if (statusCode == KeeperException.Code.SESSIONEXPIRED.intValue()) {
+      LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries for "
+          + "action=" + action);
+      return true;
+    }
+    return false;
+  }
+
+  private void heartbeat(String path, int new_version,
+      String workerName) {
+    Task task = findOrCreateOrphanTask(path);
+    if (new_version != task.last_version) {
+      if (task.isUnassigned()) {
+        LOG.info("task " + path + " acquired by " + workerName);
+      }
+      task.heartbeat(EnvironmentEdgeManager.currentTimeMillis(),
+          new_version, workerName);
+      tot_mgr_heartbeat.incrementAndGet();
+    } else {
+      // duplicate heartbeats - heartbeats w/o zk node version
+      // changing - are possible. The timeout thread does
+      // getDataSetWatch() just to check whether a node still
+      // exists or not
+    }
+    return;
+  }
+
+  private boolean resubmit(String path, Task task,
+      ResubmitDirective directive) {
+    // its ok if this thread misses the update to task.deleted. It will
+    // fail later
+    if (task.status != IN_PROGRESS) {
+      return false;
+    }
+    int version;
+    if (directive != FORCE) {
+      // We're going to resubmit:
+      // 1) immediately if the worker server is now marked as dead
+      // 2) after a configurable timeout if the server is not marked as dead but has still not
+      // finished the task. This allows to continue if the worker cannot actually handle it,
+      // for any reason.
+      final long time = EnvironmentEdgeManager.currentTimeMillis() - task.last_update;
+      ServerName curWorker = null;
+      if (!Strings.isNullOrEmpty(task.cur_worker_name)) {
+        try {
+          curWorker = ServerName.parseServerName(task.cur_worker_name);
+        } catch (IllegalArgumentException ie) {
+          LOG.error("Got invalid server name:" + task.cur_worker_name + " - task for path:" + path
+              + " won't be resubmitted before timeout");
+        }
+      } else {
+        LOG.error("Got empty/null server name:" + task.cur_worker_name + " - task for path:" + path
+            + " won't be resubmitted before timeout");
+      }
+      final boolean alive =
+          (master.getServerManager() != null && curWorker != null) ? master.getServerManager()
+              .isServerOnline(curWorker) : true;
+      if (alive && time < timeout) {
+        LOG.trace("Skipping the resubmit of " + task.toString() + "  because the server "
+            + task.cur_worker_name + " is not marked as dead, we waited for " + time
+            + " while the timeout is " + timeout);
+        return false;
+      }
+      if (task.unforcedResubmits >= resubmit_threshold) {
+        if (!task.resubmitThresholdReached) {
+          task.resubmitThresholdReached = true;
+          tot_mgr_resubmit_threshold_reached.incrementAndGet();
+          LOG.info("Skipping resubmissions of task " + path +
+              " because threshold " + resubmit_threshold + " reached");
+        }
+        return false;
+      }
+      // race with heartbeat() that might be changing last_version
+      version = task.last_version;
+    } else {
+      version = -1;
+    }
+    LOG.info("resubmitting task " + path);
+    task.incarnation++;
+    try {
+      // blocking zk call but this is done from the timeout thread
+      if (ZKUtil.setData(this.watcher, path,
+          TaskState.TASK_UNASSIGNED.get(serverName),
+          version) == false) {
+        LOG.debug("failed to resubmit task " + path +
+            " version changed");
+        task.heartbeatNoDetails(EnvironmentEdgeManager.currentTimeMillis());
+        return false;
+      }
+    } catch (NoNodeException e) {
+      LOG.warn("failed to resubmit because znode doesn't exist " + path +
+          " task done (or forced done by removing the znode)");
+      getDataSetWatchSuccess(path, null, Integer.MIN_VALUE);
+      return false;
+    } catch (KeeperException.BadVersionException e) {
+      LOG.debug("failed to resubmit task " + path +
+          " version changed");
+      task.heartbeatNoDetails(EnvironmentEdgeManager.currentTimeMillis());
+      return false;
+    } catch (KeeperException e) {
+      tot_mgr_resubmit_failed.incrementAndGet();
+      LOG.warn("failed to resubmit " + path, e);
+      return false;
+    }
+    // don't count forced resubmits
+    if (directive != FORCE) {
+      task.unforcedResubmits++;
+    }
+    task.setUnassigned();
+    createRescanNode(Long.MAX_VALUE);
+    tot_mgr_resubmit.incrementAndGet();
+    return true;
+  }
+
+  private void resubmitOrFail(String path, ResubmitDirective directive) {
+    if (resubmit(path, findOrCreateOrphanTask(path), directive) == false) {
+      setDone(path, FAILURE);
+    }
+  }
+
+  private void deleteNode(String path, Long retries) {
+    tot_mgr_node_delete_queued.incrementAndGet();
+    // Once a task znode is ready for delete, that is it is in the TASK_DONE
+    // state, then no one should be writing to it anymore. That is no one
+    // will be updating the znode version any more.
+    this.watcher.getRecoverableZooKeeper().getZooKeeper().
+      delete(path, -1, new DeleteAsyncCallback(),
+        retries);
+  }
+
+  private void deleteNodeSuccess(String path) {
+    if (ignoreZKDeleteForTesting) {
+      return;
+    }
+    Task task;
+    task = tasks.remove(path);
+    if (task == null) {
+      if (ZKSplitLog.isRescanNode(watcher, path)) {
+        tot_mgr_rescan_deleted.incrementAndGet();
+      }
+      tot_mgr_missing_state_in_delete.incrementAndGet();
+      LOG.debug("deleted task without in memory state " + path);
+      return;
+    }
+    synchronized (task) {
+      task.status = DELETED;
+      task.notify();
+    }
+    tot_mgr_task_deleted.incrementAndGet();
+  }
+
+  private void deleteNodeFailure(String path) {
+    LOG.info("Failed to delete node " + path + " and will retry soon.");
+    return;
+  }
+
+  /**
+   * signal the workers that a task was resubmitted by creating the
+   * RESCAN node.
+   * @throws KeeperException 
+   */
+  private void createRescanNode(long retries) {
+    // The RESCAN node will be deleted almost immediately by the
+    // SplitLogManager as soon as it is created because it is being
+    // created in the DONE state. This behavior prevents a buildup
+    // of RESCAN nodes. But there is also a chance that a SplitLogWorker
+    // might miss the watch-trigger that creation of RESCAN node provides.
+    // Since the TimeoutMonitor will keep resubmitting UNASSIGNED tasks
+    // therefore this behavior is safe.
+    this.watcher.getRecoverableZooKeeper().getZooKeeper().
+      create(ZKSplitLog.getRescanNode(watcher),
+        TaskState.TASK_DONE.get(serverName), Ids.OPEN_ACL_UNSAFE,
+        CreateMode.EPHEMERAL_SEQUENTIAL,
+        new CreateRescanAsyncCallback(), Long.valueOf(retries));
+  }
+
+  private void createRescanSuccess(String path) {
+    lastNodeCreateTime = EnvironmentEdgeManager.currentTimeMillis();
+    tot_mgr_rescan.incrementAndGet();
+    getDataSetWatch(path, zkretries);
+  }
+
+  private void createRescanFailure() {
+    LOG.fatal("logic failure, rescan failure must not happen");
+  }
+
+  /**
+   * @param path
+   * @param batch
+   * @return null on success, existing task on error
+   */
+  private Task createTaskIfAbsent(String path, TaskBatch batch) {
+    Task oldtask;
+    // batch.installed is only changed via this function and
+    // a single thread touches batch.installed.
+    Task newtask = new Task();
+    newtask.batch = batch;
+    oldtask = tasks.putIfAbsent(path, newtask);
+    if (oldtask == null) {
+      batch.installed++;
+      return  null;
+    }
+    // new task was not used.
+    synchronized (oldtask) {
+      if (oldtask.isOrphan()) {
+        if (oldtask.status == SUCCESS) {
+          // The task is already done. Do not install the batch for this
+          // task because it might be too late for setDone() to update
+          // batch.done. There is no need for the batch creator to wait for
+          // this task to complete.
+          return (null);
+        }
+        if (oldtask.status == IN_PROGRESS) {
+          oldtask.batch = batch;
+          batch.installed++;
+          LOG.debug("Previously orphan task " + path +
+              " is now being waited upon");
+          return null;
+        }
+        while (oldtask.status == FAILURE) {
+          LOG.debug("wait for status of task " + path +
+              " to change to DELETED");
+          tot_mgr_wait_for_zk_delete.incrementAndGet();
+          try {
+            oldtask.wait();
+          } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            LOG.warn("Interrupted when waiting for znode delete callback");
+            // fall through to return failure
+            break;
+          }
+        }
+        if (oldtask.status != DELETED) {
+          LOG.warn("Failure because previously failed task" +
+              " state still present. Waiting for znode delete callback" +
+              " path=" + path);
+          return oldtask;
+        }
+        // reinsert the newTask and it must succeed this time
+        Task t = tasks.putIfAbsent(path, newtask);
+        if (t == null) {
+          batch.installed++;
+          return  null;
+        }
+        LOG.fatal("Logic error. Deleted task still present in tasks map");
+        assert false : "Deleted task still present in tasks map";
+        return t;
+      }
+      LOG.warn("Failure because two threads can't wait for the same task. " +
+          " path=" + path);
+      return oldtask;
+    }
+  }
+
+  Task findOrCreateOrphanTask(String path) {
+    Task orphanTask = new Task();
+    Task task;
+    task = tasks.putIfAbsent(path, orphanTask);
+    if (task == null) {
+      LOG.info("creating orphan task " + path);
+      tot_mgr_orphan_task_acquired.incrementAndGet();
+      task = orphanTask;
+    }
+    return task;
+  }
+
+  @Override
+  public void nodeDataChanged(String path) {
+    Task task;
+    task = tasks.get(path);
+    if (task != null || ZKSplitLog.isRescanNode(watcher, path)) {
+      if (task != null) {
+        task.heartbeatNoDetails(EnvironmentEdgeManager.currentTimeMillis());
+      }
+      getDataSetWatch(path, zkretries);
+    }
+  }
+
+  public void stop() {
+    if (timeoutMonitor != null) {
+      timeoutMonitor.interrupt();
+    }
+  }
+
+  private void lookForOrphans() {
+    List orphans;
+    try {
+       orphans = ZKUtil.listChildrenNoWatch(this.watcher,
+          this.watcher.splitLogZNode);
+      if (orphans == null) {
+        LOG.warn("could not get children of " + this.watcher.splitLogZNode);
+        return;
+      }
+    } catch (KeeperException e) {
+      LOG.warn("could not get children of " + this.watcher.splitLogZNode +
+          " " + StringUtils.stringifyException(e));
+      return;
+    }
+    int rescan_nodes = 0;
+    for (String path : orphans) {
+      String nodepath = ZKUtil.joinZNode(watcher.splitLogZNode, path);
+      if (ZKSplitLog.isRescanNode(watcher, nodepath)) {
+        rescan_nodes++;
+        LOG.debug("found orphan rescan node " + path);
+      } else {
+        LOG.info("found orphan task " + path);
+      }
+      getDataSetWatch(nodepath, zkretries);
+    }
+    LOG.info("found " + (orphans.size() - rescan_nodes) + " orphan tasks and " +
+        rescan_nodes + " rescan nodes");
+  }
+
+  /**
+   * Keeps track of the batch of tasks submitted together by a caller in
+   * splitLogDistributed(). Clients threads use this object to wait for all
+   * their tasks to be done.
+   * 
        
+   * All access is synchronized.
+   */
+  static class TaskBatch {
+    int installed = 0;
+    int done = 0;
+    int error = 0;
+    volatile boolean isDead = false;
+
+    @Override
+    public String toString() {
+      return ("installed = " + installed + " done = " + done + " error = "
+          + error);
+    }
+  }
+
+  /**
+   * in memory state of an active task.
+   */
+  static class Task {
+    volatile long last_update;
+    volatile int last_version;
+    volatile String cur_worker_name;
+    TaskBatch batch;
+    volatile TerminationStatus status;
+    volatile int incarnation;
+    volatile int unforcedResubmits;
+    volatile boolean resubmitThresholdReached;
+
+    @Override
+    public String toString() {
+      return ("last_update = " + last_update +
+          " last_version = " + last_version +
+          " cur_worker_name = " + cur_worker_name +
+          " status = " + status +
+          " incarnation = " + incarnation +
+          " resubmits = " + unforcedResubmits +
+          " batch = " + batch);
+    }
+
+    Task() {
+      incarnation = 0;
+      last_version = -1;
+      status = IN_PROGRESS;
+      setUnassigned();
+    }
+
+    public boolean isOrphan() {
+      return (batch == null || batch.isDead);
+    }
+
+    public boolean isUnassigned() {
+      return (cur_worker_name == null);
+    }
+
+    public void heartbeatNoDetails(long time) {
+      last_update = time;
+    }
+
+    public void heartbeat(long time, int version, String worker) {
+      last_version = version;
+      last_update = time;
+      cur_worker_name = worker;
+    }
+
+    public void setUnassigned() {
+      cur_worker_name = null;
+      last_update = -1;
+    }
+  }
+
+  void handleDeadWorker(String workerName) {
+    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier
+    // to reason about concurrency. Makes it easier to retry.
+    synchronized (deadWorkersLock) {
+      if (deadWorkers == null) {
+        deadWorkers = new HashSet(100);
+      }
+      deadWorkers.add(workerName);
+    }
+    LOG.info("dead splitlog worker " + workerName);
+  }
+
+  void handleDeadWorkers(List serverNames) {
+    List workerNames = new ArrayList(serverNames.size());
+    for (ServerName serverName : serverNames) {
+      workerNames.add(serverName.toString());
+    }
+    synchronized (deadWorkersLock) {
+      if (deadWorkers == null) {
+        deadWorkers = new HashSet(100);
+      }
+      deadWorkers.addAll(workerNames);
+    }
+    LOG.info("dead splitlog workers " + workerNames);
+  }
+
+  /**
+   * Periodically checks all active tasks and resubmits the ones that have timed
+   * out
+   */
+  private class TimeoutMonitor extends Chore {
+    public TimeoutMonitor(final int period, Stoppable stopper) {
+      super("SplitLogManager Timeout Monitor", period, stopper);
+    }
+
+    @Override
+    protected void chore() {
+      int resubmitted = 0;
+      int unassigned = 0;
+      int tot = 0;
+      boolean found_assigned_task = false;
+      Set localDeadWorkers;
+
+      synchronized (deadWorkersLock) {
+        localDeadWorkers = deadWorkers;
+        deadWorkers = null;
+      }
+
+      for (Map.Entry e : tasks.entrySet()) {
+        String path = e.getKey();
+        Task task = e.getValue();
+        String cur_worker = task.cur_worker_name;
+        tot++;
+        // don't easily resubmit a task which hasn't been picked up yet. It
+        // might be a long while before a SplitLogWorker is free to pick up a
+        // task. This is because a SplitLogWorker picks up a task one at a
+        // time. If we want progress when there are no region servers then we
+        // will have to run a SplitLogWorker thread in the Master.
+        if (task.isUnassigned()) {
+          unassigned++;
+          continue;
+        }
+        found_assigned_task = true;
+        if (localDeadWorkers != null && localDeadWorkers.contains(cur_worker)) {
+          tot_mgr_resubmit_dead_server_task.incrementAndGet();
+          if (resubmit(path, task, FORCE)) {
+            resubmitted++;
+          } else {
+            handleDeadWorker(cur_worker);
+            LOG.warn("Failed to resubmit task " + path + " owned by dead " +
+                cur_worker + ", will retry.");
+          }
+        } else if (resubmit(path, task, CHECK)) {
+          resubmitted++;
+        }
+      }
+      if (tot > 0) {
+        LOG.debug("total tasks = " + tot + " unassigned = " + unassigned);
+      }
+      if (resubmitted > 0) {
+        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");
+      }
+      // If there are pending tasks and all of them have been unassigned for
+      // some time then put up a RESCAN node to ping the workers.
+      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes
+      // because a. it is very unlikely that every worker had a
+      // transient error when trying to grab the task b. if there are no
+      // workers then all tasks wills stay unassigned indefinitely and the
+      // manager will be indefinitely creating RESCAN nodes. TODO may be the
+      // master should spawn both a manager and a worker thread to guarantee
+      // that there is always one worker in the system
+      if (tot > 0 && !found_assigned_task &&
+          ((EnvironmentEdgeManager.currentTimeMillis() - lastNodeCreateTime) >
+          unassignedTimeout)) {
+        for (Map.Entry e : tasks.entrySet()) {
+          String path = e.getKey();
+          Task task = e.getValue();
+          // we have to do task.isUnassigned() check again because tasks might
+          // have been asynchronously assigned. There is no locking required
+          // for these checks ... it is OK even if tryGetDataSetWatch() is
+          // called unnecessarily for a task
+          if (task.isUnassigned() && (task.status != FAILURE)) {
+            // We just touch the znode to make sure its still there
+            tryGetDataSetWatch(path);
+          }
+        }
+        createRescanNode(Long.MAX_VALUE);
+        tot_mgr_resubmit_unassigned.incrementAndGet();
+        LOG.debug("resubmitting unassigned task(s) after timeout");
+      }
+
+      // Retry previously failed deletes
+      if (failedDeletions.size() > 0) {
+        List tmpPaths = new ArrayList(failedDeletions);
+        failedDeletions.removeAll(tmpPaths);
+        for (String tmpPath : tmpPaths) {
+          // deleteNode is an async call
+          deleteNode(tmpPath, zkretries);
+        }
+      }
+    }
+  }
+
+  /**
+   * Asynchronous handler for zk create node results.
+   * Retries on failures.
+   */
+  class CreateAsyncCallback implements AsyncCallback.StringCallback {
+    private final Log LOG = LogFactory.getLog(CreateAsyncCallback.class);
+
+    @Override
+    public void processResult(int rc, String path, Object ctx, String name) {
+      tot_mgr_node_create_result.incrementAndGet();
+      if (rc != 0) {
+        if (shouldAbandonRetries(rc, "Create znode " + path)) {
+          createNodeFailure(path);
+          return;
+        }
+        if (rc == KeeperException.Code.NODEEXISTS.intValue()) {
+          // What if there is a delete pending against this pre-existing
+          // znode? Then this soon-to-be-deleted task znode must be in TASK_DONE
+          // state. Only operations that will be carried out on this node by
+          // this manager are get-znode-data, task-finisher and delete-znode.
+          // And all code pieces correctly handle the case of suddenly
+          // disappearing task-znode.
+          LOG.debug("found pre-existing znode " + path);
+          tot_mgr_node_already_exists.incrementAndGet();
+        } else {
+          Long retry_count = (Long)ctx;
+          LOG.warn("create rc =" + KeeperException.Code.get(rc) + " for " +
+              path + " remaining retries=" + retry_count);
+          if (retry_count == 0) {
+            tot_mgr_node_create_err.incrementAndGet();
+            createNodeFailure(path);
+          } else {
+            tot_mgr_node_create_retry.incrementAndGet();
+            createNode(path, retry_count - 1);
+          }
+          return;
+        }
+      }
+      createNodeSuccess(path);
+    }
+  }
+
+  /**
+   * Asynchronous handler for zk get-data-set-watch on node results.
+   * Retries on failures.
+   */
+  class GetDataAsyncCallback implements AsyncCallback.DataCallback {
+    private final Log LOG = LogFactory.getLog(GetDataAsyncCallback.class);
+
+    @Override
+    public void processResult(int rc, String path, Object ctx, byte[] data,
+        Stat stat) {
+      tot_mgr_get_data_result.incrementAndGet();
+      if (rc != 0) {
+        if (shouldAbandonRetries(rc, "GetData from znode " + path)) {
+          return;
+        }
+        if (rc == KeeperException.Code.NONODE.intValue()) {
+          tot_mgr_get_data_nonode.incrementAndGet();
+          // The task znode has been deleted. Must be some pending delete
+          // that deleted the task. Assume success because a task-znode is
+          // is only deleted after TaskFinisher is successful.
+          LOG.warn("task znode " + path + " vanished.");
+          getDataSetWatchSuccess(path, null, Integer.MIN_VALUE);
+          return;
+        }
+        Long retry_count = (Long) ctx;
+
+        if (retry_count < 0) {
+          LOG.warn("getdata rc = " + KeeperException.Code.get(rc) + " " +
+              path + ". Ignoring error. No error handling. No retrying.");
+          return;
+        }
+        LOG.warn("getdata rc = " + KeeperException.Code.get(rc) + " " +
+            path + " remaining retries=" + retry_count);
+        if (retry_count == 0) {
+          tot_mgr_get_data_err.incrementAndGet();
+          getDataSetWatchFailure(path);
+        } else {
+          tot_mgr_get_data_retry.incrementAndGet();
+          getDataSetWatch(path, retry_count - 1);
+        }
+        return;
+      }
+      getDataSetWatchSuccess(path, data, stat.getVersion());
+      return;
+    }
+  }
+
+  /**
+   * Asynchronous handler for zk delete node results.
+   * Retries on failures.
+   */
+  class DeleteAsyncCallback implements AsyncCallback.VoidCallback {
+    private final Log LOG = LogFactory.getLog(DeleteAsyncCallback.class);
+
+    @Override
+    public void processResult(int rc, String path, Object ctx) {
+      tot_mgr_node_delete_result.incrementAndGet();
+      if (rc != 0) {
+        if (shouldAbandonRetries(rc, "Delete znode " + path)) {
+          failedDeletions.add(path);
+          return;
+        }
+        if (rc != KeeperException.Code.NONODE.intValue()) {
+          tot_mgr_node_delete_err.incrementAndGet();
+          Long retry_count = (Long) ctx;
+          LOG.warn("delete rc=" + KeeperException.Code.get(rc) + " for " +
+              path + " remaining retries=" + retry_count);
+          if (retry_count == 0) {
+            LOG.warn("delete failed " + path);
+            failedDeletions.add(path);
+            deleteNodeFailure(path);
+          } else {
+            deleteNode(path, retry_count - 1);
+          }
+          return;
+        } else {
+        LOG.debug(path +
+            " does not exist. Either was created but deleted behind our" +
+            " back by another pending delete OR was deleted" +
+            " in earlier retry rounds. zkretries = " + (Long) ctx);
+        }
+      } else {
+        LOG.debug("deleted " + path);
+      }
+      deleteNodeSuccess(path);
+    }
+  }
+
+  /**
+   * Asynchronous handler for zk create RESCAN-node results.
+   * Retries on failures.
+   * 
        
+   * A RESCAN node is created using PERSISTENT_SEQUENTIAL flag. It is a signal
+   * for all the {@link SplitLogWorker}s to rescan for new tasks.
+   */
+  class CreateRescanAsyncCallback implements AsyncCallback.StringCallback {
+    private final Log LOG = LogFactory.getLog(CreateRescanAsyncCallback.class);
+
+    @Override
+    public void processResult(int rc, String path, Object ctx, String name) {
+      if (rc != 0) {
+        if (shouldAbandonRetries(rc, "CreateRescan znode " + path)) {
+          return;
+        }
+        Long retry_count = (Long)ctx;
+        LOG.warn("rc=" + KeeperException.Code.get(rc) + " for "+ path +
+            " remaining retries=" + retry_count);
+        if (retry_count == 0) {
+          createRescanFailure();
+        } else {
+          createRescanNode(retry_count - 1);
+        }
+        return;
+      }
+      // path is the original arg, name is the actual name that was created
+      createRescanSuccess(name);
+    }
+  }
+
+  /**
+   * {@link SplitLogManager} can use objects implementing this interface to
+   * finish off a partially done task by {@link SplitLogWorker}. This provides
+   * a serialization point at the end of the task processing. Must be
+   * restartable and idempotent.
+   */
+  static public interface TaskFinisher {
+    /**
+     * status that can be returned finish()
+     */
+    static public enum Status {
+      /**
+       * task completed successfully
+       */
+      DONE(),
+      /**
+       * task completed with error
+       */
+      ERR();
+    }
+    /**
+     * finish the partially done task. workername provides clue to where the
+     * partial results of the partially done tasks are present. taskname is the
+     * name of the task that was put up in zookeeper.
+     * 
        
+     * @param workerName
+     * @param taskname
+     * @return DONE if task completed successfully, ERR otherwise
+     */
+    public Status finish(String workerName, String taskname);
+  }
+  enum ResubmitDirective {
+    CHECK(),
+    FORCE();
+  }
+  enum TerminationStatus {
+    IN_PROGRESS("in_progress"),
+    SUCCESS("success"),
+    FAILURE("failure"),
+    DELETED("deleted");
+
+    String statusMsg;
+    TerminationStatus(String msg) {
+      statusMsg = msg;
+    }
+    
+    @Override
+    public String toString() {
+      return statusMsg;
+    }
+  }
+  
+  /**
+   * Completes the initialization
+   */
+  public void finishInitialization() {
+    finishInitialization(false);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/UnAssignCallable.java b/src/main/java/org/apache/hadoop/hbase/master/UnAssignCallable.java
new file mode 100644
index 0000000..2cbe7e0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/UnAssignCallable.java
@@ -0,0 +1,46 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.util.concurrent.Callable;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+
+/**
+ * A callable object that invokes the corresponding action that needs to be
+ * taken for unassignment of a region in transition. Implementing as future
+ * callable we are able to act on the timeout asynchronously.
+ */
+public class UnAssignCallable implements Callable {
+  private AssignmentManager assignmentManager;
+
+  private HRegionInfo hri;
+
+  public UnAssignCallable(AssignmentManager assignmentManager, HRegionInfo hri) {
+    this.assignmentManager = assignmentManager;
+    this.hri = hri;
+  }
+
+  @Override
+  public Object call() throws Exception {
+    assignmentManager.unassign(hri, true);
+    return null;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
new file mode 100644
index 0000000..ed890ac
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.BaseConfigurable;
+
+/**
+ * Base class for the hfile cleaning function inside the master. By default, only the
+ * {@link TimeToLiveHFileCleaner} is called.
+ * 
        
+ * If other effects are needed, implement your own LogCleanerDelegate and add it to the
+ * configuration "hbase.master.hfilecleaner.plugins", which is a comma-separated list of fully
+ * qualified class names. The HFileCleaner will build the cleaner chain in 
+ * order the order specified by the configuration.
+ * 
        
+ * For subclasses, setConf will be called exactly once before using the cleaner.
+ * 
        
+ * Since {@link BaseHFileCleanerDelegate HFileCleanerDelegates} are created in
+ * HFileCleaner by reflection, classes that implements this interface must
+ * provide a default constructor.
+ */
+@InterfaceAudience.Private
+public abstract class BaseHFileCleanerDelegate extends BaseConfigurable implements
+    FileCleanerDelegate {
+
+  private boolean stopped = false;
+
+  @Override
+  public void stop(String why) {
+    this.stopped = true;
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.stopped;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
new file mode 100644
index 0000000..d62d347
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.BaseConfigurable;
+
+/**
+ * Base class for the log cleaning function inside the master. By default, two
+ * cleaners: TimeToLiveLogCleaner and
+ * ReplicationLogCleaner are called in order. So if other effects
+ * are needed, implement your own LogCleanerDelegate and add it to the
+ * configuration "hbase.master.logcleaner.plugins", which is a comma-separated
+ * list of fully qualified class names. LogsCleaner will add it to the chain.
+ * 
        
+ * HBase ships with LogsCleaner as the default implementation.
+ * 
        
+ * This interface extends Configurable, so setConf needs to be called once
+ * before using the cleaner. Since LogCleanerDelegates are created in
+ * LogsCleaner by reflection. Classes that implements this interface should
+ * provide a default constructor.
+ */
+@InterfaceAudience.Private
+public abstract class BaseLogCleanerDelegate extends BaseConfigurable implements FileCleanerDelegate {
+
+  @Override
+  public boolean isFileDeletable(Path file) {
+    return isLogDeletable(file);
+  }
+
+  /**
+   * Should the master delete the log or keep it?
+   * 
        
+   * Implementing classes should override {@link #isFileDeletable(Path)} instead.
+   * @param filePath full path to log.
+   * @return true if the log is deletable, false if not
+   */
+  @Deprecated
+  public abstract boolean isLogDeletable(Path filePath);
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
new file mode 100644
index 0000000..0fa53c0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Abstract Cleaner that uses a chain of delegates to clean a directory of files
+ * @param  Cleaner delegate class that is dynamically loaded from configuration
+ */
+public abstract class CleanerChore extends Chore {
+
+  private static final Log LOG = LogFactory.getLog(CleanerChore.class.getName());
+
+  private final FileSystem fs;
+  private final Path oldFileDir;
+  private final Configuration conf;
+  List cleanersChain;
+
+  /**
+   * @param name name of the chore being run
+   * @param sleepPeriod the period of time to sleep between each run
+   * @param s the stopper
+   * @param conf configuration to use
+   * @param fs handle to the FS
+   * @param oldFileDir the path to the archived files
+   * @param confKey configuration key for the classes to instantiate
+   */
+  public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf,
+      FileSystem fs, Path oldFileDir, String confKey) {
+    super(name, sleepPeriod, s);
+    this.fs = fs;
+    this.oldFileDir = oldFileDir;
+    this.conf = conf;
+
+    initCleanerChain(confKey);
+  }
+
+  /**
+   * Validate the file to see if it even belongs in the directory. If it is valid, then the file
+   * will go through the cleaner delegates, but otherwise the file is just deleted.
+   * @param file full {@link Path} of the file to be checked
+   * @return true if the file is valid, false otherwise
+   */
+  protected abstract boolean validate(Path file);
+
+  /**
+   * Instanitate and initialize all the file cleaners set in the configuration
+   * @param confKey key to get the file cleaner classes from the configuration
+   */
+  private void initCleanerChain(String confKey) {
+    this.cleanersChain = new LinkedList();
+    String[] logCleaners = conf.getStrings(confKey);
+    if (logCleaners != null) {
+      for (String className : logCleaners) {
+        T logCleaner = newFileCleaner(className, conf);
+        if (logCleaner != null) {
+          LOG.debug("initialize cleaner=" + className);
+          this.cleanersChain.add(logCleaner);
+        }
+      }
+    }
+  }
+
+  /**
+   * A utility method to create new instances of LogCleanerDelegate based on the class name of the
+   * LogCleanerDelegate.
+   * @param className fully qualified class name of the LogCleanerDelegate
+   * @param conf
+   * @return the new instance
+   */
+  public T newFileCleaner(String className, Configuration conf) {
+    try {
+      Class c = Class.forName(className).asSubclass(
+        FileCleanerDelegate.class);
+      @SuppressWarnings("unchecked")
+      T cleaner = (T) c.newInstance();
+      cleaner.setConf(conf);
+      return cleaner;
+    } catch (Exception e) {
+      LOG.warn("Can NOT create CleanerDelegate: " + className, e);
+      // skipping if can't instantiate
+      return null;
+    }
+  }
+
+  @Override
+  protected void chore() {
+    try {
+      FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir, null);
+      // if the path (file or directory) doesn't exist, then we can just return
+      if (files == null) return;
+      // loop over the found files and see if they should be deleted
+      for (FileStatus file : files) {
+        try {
+          if (file.isDir()) checkAndDeleteDirectory(file.getPath());
+          else checkAndDelete(file.getPath());
+        } catch (IOException e) {
+          e = RemoteExceptionHandler.checkIOException(e);
+          LOG.warn("Error while cleaning the logs", e);
+        }
+      }
+    } catch (IOException e) {
+      LOG.warn("Failed to get status of:" + oldFileDir);
+    }
+
+  }
+
+  /**
+   * Attempt to delete a directory and all files under that directory. Each child file is passed
+   * through the delegates to see if it can be deleted. If the directory has not children when the
+   * cleaners have finished it is deleted.
+   * 
        
+   * If new children files are added between checks of the directory, the directory will not
+   * be deleted.
+   * @param toCheck directory to check
+   * @return true if the directory was deleted, false otherwise.
+   * @throws IOException if there is an unexpected filesystem error
+   */
+  public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Checking directory: " + toCheck);
+    }
+    FileStatus[] children = FSUtils.listStatus(fs, toCheck, null);
+    // if the directory doesn't exist, then we are done
+    if (children == null) {
+      try {
+        return HBaseFileSystem.deleteFileFromFileSystem(fs, toCheck);
+      } catch (IOException e) {
+        if (LOG.isTraceEnabled()) {
+          LOG.trace("Couldn't delete directory: " + toCheck, e);
+        }
+      }
+      // couldn't delete w/o exception, so we can't return success.
+      return false;
+    }     
+
+    boolean canDeleteThis = true;
+    for (FileStatus child : children) {
+      Path path = child.getPath();
+      // attempt to delete all the files under the directory
+      if (child.isDir()) {
+        if (!checkAndDeleteDirectory(path)) {
+          canDeleteThis = false;
+        }
+      }
+      // otherwise we can just check the file
+      else if (!checkAndDelete(path)) {
+        canDeleteThis = false;
+      }
+    }
+
+    // if the directory has children, we can't delete it, so we are done
+    if (!canDeleteThis) return false;
+
+    // otherwise, all the children (that we know about) have been deleted, so we should try to
+    // delete this directory. However, don't do so recursively so we don't delete files that have
+    // been added since we last checked.
+    try {
+      return HBaseFileSystem.deleteFileFromFileSystem(fs, toCheck);
+    } catch (IOException e) {
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Couldn't delete directory: " + toCheck, e);
+      }
+    }
+
+    // couldn't delete w/o exception, so we can't return success.
+    return false;
+  }
+
+  /**
+   * Run the given file through each of the cleaners to see if it should be deleted, deleting it if
+   * necessary.
+   * @param filePath path of the file to check (and possibly delete)
+   * @throws IOException if cann't delete a file because of a filesystem issue
+   * @throws IllegalArgumentException if the file is a directory and has children
+   */
+  private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumentException {
+    // first check to see if the path is valid
+    if (!validate(filePath)) {
+      LOG.warn("Found a wrongly formatted file: " + filePath.getName() + " deleting it.");
+      boolean success = HBaseFileSystem.deleteDirFromFileSystem(fs, filePath);
+      if (!success) LOG.warn("Attempted to delete:" + filePath
+          + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
+
+      return success;
+    }
+    // check each of the cleaners for the file
+    for (T cleaner : cleanersChain) {
+      if (cleaner.isStopped() || this.stopper.isStopped()) {
+        LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any file in:"
+            + this.oldFileDir);
+        return false;
+      }
+
+      if (!cleaner.isFileDeletable(filePath)) {
+        // this file is not deletable, then we are done
+        if (LOG.isTraceEnabled()) {
+          LOG.trace(filePath + " is not deletable according to:" + cleaner);
+        }
+        return false;
+      }
+    }
+    // delete this file if it passes all the cleaners
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Removing:" + filePath + " from archive");
+    }
+    boolean success = HBaseFileSystem.deleteFileFromFileSystem(fs, filePath);
+    if (!success) {
+      LOG.warn("Attempted to delete:" + filePath
+          + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
+    }
+    return success;
+  }
+
+  @Override
+  public void cleanup() {
+    for (T lc : this.cleanersChain) {
+      try {
+        lc.stop("Exiting");
+      } catch (Throwable t) {
+        LOG.warn("Stopping", t);
+      }
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
new file mode 100644
index 0000000..4cf5038
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
+
+/**
+ * General interface for cleaning files from a folder (generally an archive or
+ * backup folder). These are chained via the {@link CleanerChore} to determine
+ * if a given file should be deleted.
+ */
+@InterfaceAudience.Private
+public interface FileCleanerDelegate extends Configurable, Stoppable {
+
+  /**
+   * Should the master delete the file or keep it?
+   * @param file full path to the file to check
+   * @return true if the file is deletable, false if not
+   */
+  public boolean isFileDeletable(Path file);
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
new file mode 100644
index 0000000..6773fbd
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+/**
+ * This Chore, every time it runs, will clear the HFiles in the hfile archive
+ * folder that are deletable for each HFile cleaner in the chain.
+ */
+@InterfaceAudience.Private
+public class HFileCleaner extends CleanerChore {
+
+  public static final String MASTER_HFILE_CLEANER_PLUGINS = "hbase.master.hfilecleaner.plugins";
+
+  /**
+   * @param period the period of time to sleep between each run
+   * @param stopper the stopper
+   * @param conf configuration to use
+   * @param fs handle to the FS
+   * @param directory directory to be cleaned
+   */
+  public HFileCleaner(final int period, final Stoppable stopper, Configuration conf, FileSystem fs,
+      Path directory) {
+    super("HFileCleaner", period, stopper, conf, fs, directory, MASTER_HFILE_CLEANER_PLUGINS);
+  }
+
+  @Override
+  protected boolean validate(Path file) {
+    if (HFileLink.isBackReferencesDir(file) || HFileLink.isBackReferencesDir(file.getParent())) {
+      return true;
+    }
+    return StoreFile.validateStoreFileName(file.getName());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
new file mode 100644
index 0000000..4da1b9b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
+
+/**
+ * HFileLink cleaner that determines if a hfile should be deleted.
+ * HFiles can be deleted only if there're no links to them.
+ *
+ * When a HFileLink is created a back reference file is created in:
+ *      /hbase/archive/table/region/cf/.links-hfile/ref-region.ref-table
+ * To check if the hfile can be deleted the back references folder must be empty.
+ */
+@InterfaceAudience.Private
+public class HFileLinkCleaner extends BaseHFileCleanerDelegate {
+  private static final Log LOG = LogFactory.getLog(HFileLinkCleaner.class);
+
+  private FileSystem fs = null;
+
+  @Override
+  public synchronized boolean isFileDeletable(Path filePath) {
+    if (this.fs == null) return false;
+
+    // HFile Link is always deletable
+    if (HFileLink.isHFileLink(filePath)) return true;
+
+    // If the file is inside a link references directory, means that is a back ref link.
+    // The back ref can be deleted only if the referenced file doesn't exists.
+    Path parentDir = filePath.getParent();
+    if (HFileLink.isBackReferencesDir(parentDir)) {
+      try {
+        Path hfilePath = HFileLink.getHFileFromBackReference(getConf(), filePath);
+        return !fs.exists(hfilePath);
+      } catch (IOException e) {
+        LOG.error("Couldn't verify if the referenced file still exists, keep it just in case");
+        return false;
+      }
+    }
+
+    // HFile is deletable only if has no links
+    try {
+      Path backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName());
+      return FSUtils.listStatus(fs, backRefDir) == null;
+    } catch (IOException e) {
+      LOG.error("Couldn't get the references, not deleting file, just in case");
+      return false;
+    }
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+
+    // setup filesystem
+    try {
+      this.fs = FileSystem.get(this.getConf());
+    } catch (IOException e) {
+      LOG.error("Couldn't instantiate the file system, not deleting file, just in case");
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java
new file mode 100644
index 0000000..0e5bd3d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+
+/**
+ * This Chore, every time it runs, will attempt to delete the HLogs in the old logs folder. The HLog
+ * is only deleted if none of the cleaner delegates says otherwise.
+ * @see BaseLogCleanerDelegate
+ */
+@InterfaceAudience.Private
+public class LogCleaner extends CleanerChore {
+  static final Log LOG = LogFactory.getLog(LogCleaner.class.getName());
+
+  /**
+   * @param p the period of time to sleep between each run
+   * @param s the stopper
+   * @param conf configuration to use
+   * @param fs handle to the FS
+   * @param oldLogDir the path to the archived logs
+   */
+  public LogCleaner(final int p, final Stoppable s, Configuration conf, FileSystem fs,
+      Path oldLogDir) {
+    super("LogsCleaner", p, s, conf, fs, oldLogDir, HBASE_MASTER_LOGCLEANER_PLUGINS);
+  }
+
+  @Override
+  protected boolean validate(Path file) {
+    return HLog.validateHLogFilename(file.getName());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
new file mode 100644
index 0000000..82231af
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+/**
+ * HFile cleaner that uses the timestamp of the hfile to determine if it should be deleted. By
+ * default they are allowed to live for {@value TimeToLiveHFileCleaner#DEFAULT_TTL}
+ */
+@InterfaceAudience.Private
+public class TimeToLiveHFileCleaner extends BaseHFileCleanerDelegate {
+
+  public static final Log LOG = LogFactory.getLog(TimeToLiveHFileCleaner.class.getName());
+  public static final String TTL_CONF_KEY = "hbase.master.hfilecleaner.ttl";
+  // default ttl = 5 minutes
+  private static final long DEFAULT_TTL = 60000 * 5;
+  // Configured time a hfile can be kept after it was moved to the archive
+  private long ttl;
+  private FileSystem fs;
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.ttl = conf.getLong(TTL_CONF_KEY, DEFAULT_TTL);
+    super.setConf(conf);
+  }
+
+  @Override
+  public boolean isFileDeletable(Path filePath) {
+    if (!instantiateFS()) {
+      return false;
+    }
+    long time = 0;
+    long currentTime = EnvironmentEdgeManager.currentTimeMillis();
+    try {
+      FileStatus fStat = fs.getFileStatus(filePath);
+      time = fStat.getModificationTime();
+    } catch (IOException e) {
+      LOG.error("Unable to get modification time of file " + filePath.getName()
+          + ", not deleting it.", e);
+      return false;
+    }
+    long life = currentTime - time;
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("HFile life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: "
+          + time);
+    }
+    if (life < 0) {
+      LOG.warn("Found a log (" + filePath + ") newer than current time (" + currentTime + " < "
+          + time + "), probably a clock skew");
+      return false;
+    }
+    return life > ttl;
+  }
+
+  /**
+   * setup the filesystem, if it hasn't been already
+   */
+  private synchronized boolean instantiateFS() {
+    if (this.fs == null) {
+      try {
+        this.fs = FileSystem.get(this.getConf());
+      } catch (IOException e) {
+        LOG.error("Couldn't instantiate the file system, not deleting file, just incase");
+        return false;
+      }
+    }
+    return true;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
new file mode 100644
index 0000000..6043a8a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Log cleaner that uses the timestamp of the hlog to determine if it should
+ * be deleted. By default they are allowed to live for 10 minutes.
+ */
+@InterfaceAudience.Private
+public class TimeToLiveLogCleaner extends BaseLogCleanerDelegate {
+  static final Log LOG = LogFactory.getLog(TimeToLiveLogCleaner.class.getName());
+  // Configured time a log can be kept after it was closed
+  private long ttl;
+  private boolean stopped = false;
+
+  @Override
+  public boolean isLogDeletable(Path filePath) {
+    long time = 0;
+    long currentTime = System.currentTimeMillis();
+    try {
+      FileStatus fStat = filePath.getFileSystem(this.getConf()).getFileStatus(filePath);
+      time = fStat.getModificationTime();
+    } catch (IOException e) {
+      LOG.error("Unable to get modification time of file " + filePath.getName() +
+      ", not deleting it.", e);
+      return false;
+    }
+    long life = currentTime - time;
+    if (life < 0) {
+      LOG.warn("Found a log newer than current time, " +
+          "probably a clock skew");
+      return false;
+    }
+    return life > ttl;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+    this.ttl = conf.getLong("hbase.master.logcleaner.ttl", 600000);
+  }
+
+
+  @Override
+  public void stop(String why) {
+    this.stopped = true;
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.stopped;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java
new file mode 100644
index 0000000..88f207a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java
@@ -0,0 +1,107 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+
+/**
+ * Handles CLOSED region event on Master.
+ * 
        
+ * If table is being disabled, deletes ZK unassigned node and removes from
+ * regions in transition.
+ * 
        
+ * Otherwise, assigns the region to another server.
+ */
+public class ClosedRegionHandler extends EventHandler implements TotesHRegionInfo {
+  private static final Log LOG = LogFactory.getLog(ClosedRegionHandler.class);
+  private final AssignmentManager assignmentManager;
+  private final HRegionInfo regionInfo;
+  private final ClosedPriority priority;
+
+  private enum ClosedPriority {
+    ROOT (1),
+    META (2),
+    USER (3);
+
+    private final int value;
+    ClosedPriority(int value) {
+      this.value = value;
+    }
+    public int getValue() {
+      return value;
+    }
+  };
+
+  public ClosedRegionHandler(Server server, AssignmentManager assignmentManager,
+      HRegionInfo regionInfo) {
+    super(server, EventType.RS_ZK_REGION_CLOSED);
+    this.assignmentManager = assignmentManager;
+    this.regionInfo = regionInfo;
+    if(regionInfo.isRootRegion()) {
+      priority = ClosedPriority.ROOT;
+    } else if(regionInfo.isMetaRegion()) {
+      priority = ClosedPriority.META;
+    } else {
+      priority = ClosedPriority.USER;
+    }
+  }
+
+  @Override
+  public int getPriority() {
+    return priority.getValue();
+  }
+
+  @Override
+  public HRegionInfo getHRegionInfo() {
+    return this.regionInfo;
+  }
+
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    return getClass().getSimpleName() + "-" + name + "-" + getSeqid();
+  }
+  
+  @Override
+  public void process() {
+    LOG.debug("Handling CLOSED event for " + regionInfo.getEncodedName());
+    // Check if this table is being disabled or not
+    if (this.assignmentManager.getZKTable().
+        isDisablingOrDisabledTable(this.regionInfo.getTableNameAsString())) {
+      assignmentManager.offlineDisabledRegion(regionInfo);
+      return;
+    }
+    // ZK Node is in CLOSED state, assign it.
+    // TODO: Should we remove the region from RIT too?  We don't?  Makes for
+    // a 'forcing' log message when we go to update state from CLOSED to OFFLINE
+    assignmentManager.setOffline(regionInfo);
+    // This below has to do w/ online enable/disable of a table
+    assignmentManager.removeClosedRegion(regionInfo);
+    assignmentManager.assign(regionInfo, true);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
new file mode 100644
index 0000000..ddf4228
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
@@ -0,0 +1,237 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.ModifyRegionUtils;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Handler to create a table.
+ */
+@InterfaceAudience.Private
+public class CreateTableHandler extends EventHandler {
+  private static final Log LOG = LogFactory.getLog(CreateTableHandler.class);
+  protected MasterFileSystem fileSystemManager;
+  protected final HTableDescriptor hTableDescriptor;
+  protected Configuration conf;
+  protected final AssignmentManager assignmentManager;
+  protected final CatalogTracker catalogTracker;
+  protected final ServerManager serverManager;
+  private final HRegionInfo [] newRegions;
+
+  public CreateTableHandler(Server server, MasterFileSystem fileSystemManager,
+    ServerManager serverManager, HTableDescriptor hTableDescriptor,
+    Configuration conf, HRegionInfo [] newRegions,
+    CatalogTracker catalogTracker, AssignmentManager assignmentManager)
+    throws NotAllMetaRegionsOnlineException, TableExistsException,
+    IOException {
+    super(server, EventType.C_M_CREATE_TABLE);
+
+    this.fileSystemManager = fileSystemManager;
+    this.serverManager = serverManager;
+    this.hTableDescriptor = hTableDescriptor;
+    this.conf = conf;
+    this.newRegions = newRegions;
+    this.catalogTracker = catalogTracker;
+    this.assignmentManager = assignmentManager;
+
+    int timeout = conf.getInt("hbase.client.catalog.timeout", 10000);
+    // Need META availability to create a table
+    try {
+      if(catalogTracker.waitForMeta(timeout) == null) {
+        throw new NotAllMetaRegionsOnlineException();
+      }
+    } catch (InterruptedException e) {
+      LOG.warn("Interrupted waiting for meta availability", e);
+      throw new IOException(e);
+    }
+
+    String tableName = this.hTableDescriptor.getNameAsString();
+    if (MetaReader.tableExists(catalogTracker, tableName)) {
+      throw new TableExistsException(tableName);
+    }
+
+    // If we have multiple client threads trying to create the table at the
+    // same time, given the async nature of the operation, the table
+    // could be in a state where .META. table hasn't been updated yet in
+    // the process() function.
+    // Use enabling state to tell if there is already a request for the same
+    // table in progress. This will introduce a new zookeeper call. Given
+    // createTable isn't a frequent operation, that should be ok.
+    try {
+      if (!this.assignmentManager.getZKTable().checkAndSetEnablingTable(tableName))
+        throw new TableExistsException(tableName);
+    } catch (KeeperException e) {
+      throw new IOException("Unable to ensure that the table will be" +
+        " enabling because of a ZooKeeper issue", e);
+    }
+  }
+
+
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    return getClass().getSimpleName() + "-" + name + "-" + getSeqid() + "-" +
+      this.hTableDescriptor.getNameAsString();
+  }
+
+  @Override
+  public void process() {
+    String tableName = this.hTableDescriptor.getNameAsString();
+    try {
+      LOG.info("Attempting to create the table " + tableName);
+      MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost();
+      if (cpHost != null) {
+        cpHost.preCreateTableHandler(this.hTableDescriptor, this.newRegions);
+      }
+      handleCreateTable(tableName);
+      if (cpHost != null) {
+        cpHost.postCreateTableHandler(this.hTableDescriptor, this.newRegions);
+      }
+      completed(null);
+    } catch (Throwable e) {
+      LOG.error("Error trying to create the table " + tableName, e);
+      completed(e);
+    }
+  }
+
+  /**
+   * Called after that process() is completed.
+   * @param exception null if process() is successful or not null if something has failed.
+   */
+  protected void completed(final Throwable exception) {
+    // Try deleting the enabling node
+    // If this does not happen then if the client tries to create the table
+    // again with the same Active master
+    // It will block the creation saying TableAlreadyExists.
+    if (exception != null) {
+      try {
+        this.assignmentManager.getZKTable().removeEnablingTable(
+            this.hTableDescriptor.getNameAsString(), false);
+      } catch (KeeperException e) {
+        // Keeper exception should not happen here
+        LOG.error("Got a keeper exception while removing the ENABLING table znode "
+            + this.hTableDescriptor.getNameAsString(), e);
+      }
+    }
+
+  }
+
+  /**
+   * Responsible of table creation (on-disk and META) and assignment.
+   * - Create the table directory and descriptor (temp folder)
+   * - Create the on-disk regions (temp folder)
+   *   [If something fails here: we've just some trash in temp]
+   * - Move the table from temp to the root directory
+   *   [If something fails here: we've the table in place but some of the rows required
+   *    present in META. (hbck needed)]
+   * - Add regions to META
+   *   [If something fails here: we don't have regions assigned: table disabled]
+   * - Assign regions to Region Servers
+   *   [If something fails here: we still have the table in disabled state]
+   * - Update ZooKeeper with the enabled state
+   */
+  private void handleCreateTable(String tableName) throws IOException, KeeperException {
+    Path tempdir = fileSystemManager.getTempDir();
+    FileSystem fs = fileSystemManager.getFileSystem();
+
+    // 1. Create Table Descriptor
+    FSTableDescriptors.createTableDescriptor(fs, tempdir, this.hTableDescriptor);
+    Path tempTableDir = new Path(tempdir, tableName);
+    Path tableDir = new Path(fileSystemManager.getRootDir(), tableName);
+
+    // 2. Create Regions
+    List regionInfos = handleCreateHdfsRegions(tempdir, tableName);
+
+    // 3. Move Table temp directory to the hbase root location
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, tempTableDir, tableDir)) {
+      throw new IOException("Unable to move table from temp=" + tempTableDir +
+        " to hbase root=" + tableDir);
+    }
+    
+    if (regionInfos != null && regionInfos.size() > 0) {
+      // 4. Add regions to META
+      MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
+
+      // 5. Trigger immediate assignment of the regions in round-robin fashion
+      List servers = serverManager.getOnlineServersList();
+      // Remove the deadNotExpired servers from the server list.
+      assignmentManager.removeDeadNotExpiredServers(servers);
+      try {
+        this.assignmentManager.assignUserRegions(regionInfos, servers);
+      } catch (InterruptedException e) {
+        LOG.error("Caught " + e + " during round-robin assignment");
+        InterruptedIOException ie = new InterruptedIOException(e.getMessage());
+        ie.initCause(e);
+        throw ie;
+      }
+    }
+
+    // 6. Set table enabled flag up in zk.
+    try {
+      assignmentManager.getZKTable().setEnabledTable(tableName);
+    } catch (KeeperException e) {
+      throw new IOException("Unable to ensure that " + tableName + " will be" +
+        " enabled because of a ZooKeeper issue", e);
+    }
+  }
+
+  /**
+   * Create the on-disk structure for the table, and returns the regions info.
+   * @param tableRootDir directory where the table is being created
+   * @param tableName name of the table under construction
+   * @return the list of regions created
+   */
+  protected List handleCreateHdfsRegions(final Path tableRootDir,
+    final String tableName)
+      throws IOException {
+    return ModifyRegionUtils.createRegions(conf, tableRootDir,
+        hTableDescriptor, newRegions, null);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
new file mode 100644
index 0000000..3b96342
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
@@ -0,0 +1,122 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.zookeeper.KeeperException;
+
+public class DeleteTableHandler extends TableEventHandler {
+  private static final Log LOG = LogFactory.getLog(DeleteTableHandler.class);
+
+  public DeleteTableHandler(byte [] tableName, Server server,
+      final MasterServices masterServices)
+  throws IOException {
+    super(EventType.C_M_DELETE_TABLE, tableName, server, masterServices);
+    // The next call fails if no such table.
+    getTableDescriptor();
+  }
+
+  @Override
+  protected void handleTableOperation(List regions)
+  throws IOException, KeeperException {
+    MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost();
+    if (cpHost != null) {
+      cpHost.preDeleteTableHandler(this.tableName);
+    }
+    // 1. Wait because of region in transition
+    AssignmentManager am = this.masterServices.getAssignmentManager();
+    long waitTime = server.getConfiguration().
+      getLong("hbase.master.wait.on.region", 5 * 60 * 1000);
+    for (HRegionInfo region : regions) {
+      long done = System.currentTimeMillis() + waitTime;
+      while (System.currentTimeMillis() < done) {
+        AssignmentManager.RegionState rs = am.isRegionInTransition(region);
+        if (rs == null) break;
+        Threads.sleep(waitingTimeForEvents);
+        LOG.debug("Waiting on  region to clear regions in transition; " + rs);
+      }
+      if (am.isRegionInTransition(region) != null) {
+        throw new IOException("Waited hbase.master.wait.on.region (" +
+          waitTime + "ms) for region to leave region " +
+          region.getRegionNameAsString() + " in transitions");
+      }
+    }
+
+    // 2. Remove regions from META
+    LOG.debug("Deleting regions from META");
+    MetaEditor.deleteRegions(this.server.getCatalogTracker(), regions);
+
+    // 3. Move the table in /hbase/.tmp
+    LOG.debug("Moving table directory to a temp directory");
+    MasterFileSystem mfs = this.masterServices.getMasterFileSystem();
+    Path tempTableDir = mfs.moveTableToTemp(tableName);
+
+    try {
+      // 4. Delete regions from FS (temp directory)
+      FileSystem fs = mfs.getFileSystem();
+      for (HRegionInfo hri: regions) {
+        LOG.debug("Archiving region " + hri.getRegionNameAsString() + " from FS");
+        HFileArchiver.archiveRegion(fs, mfs.getRootDir(),
+            tempTableDir, new Path(tempTableDir, hri.getEncodedName()));
+      }
+
+      // 5. Delete table from FS (temp directory)
+      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, tempTableDir)) {
+        LOG.error("Couldn't delete " + tempTableDir);
+      }
+    } finally {
+      // 6. Update table descriptor cache
+      this.masterServices.getTableDescriptors().remove(Bytes.toString(tableName));
+
+      // 7. If entry for this table in zk, and up in AssignmentManager, remove it.
+      am.getZKTable().setDeletedTable(Bytes.toString(tableName));
+      if (cpHost != null) {
+        cpHost.postDeleteTableHandler(this.tableName);
+      }
+    }
+  }
+
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    return getClass().getSimpleName() + "-" + name + "-" + getSeqid() + "-" + tableNameStr;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
new file mode 100644
index 0000000..af138c6
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
@@ -0,0 +1,189 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableNotEnabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.BulkAssigner;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Handler to run disable of a table.
+ */
+public class DisableTableHandler extends EventHandler {
+  private static final Log LOG = LogFactory.getLog(DisableTableHandler.class);
+  private final byte [] tableName;
+  private final String tableNameStr;
+  private final AssignmentManager assignmentManager;
+
+  public DisableTableHandler(Server server, byte [] tableName,
+      CatalogTracker catalogTracker, AssignmentManager assignmentManager,
+      boolean skipTableStateCheck)
+  throws TableNotFoundException, TableNotEnabledException, IOException {
+    super(server, EventType.C_M_DISABLE_TABLE);
+    this.tableName = tableName;
+    this.tableNameStr = Bytes.toString(this.tableName);
+    this.assignmentManager = assignmentManager;
+    // Check if table exists
+    // TODO: do we want to keep this in-memory as well?  i guess this is
+    //       part of old master rewrite, schema to zk to check for table
+    //       existence and such
+    if (!MetaReader.tableExists(catalogTracker, this.tableNameStr)) {
+      throw new TableNotFoundException(this.tableNameStr);
+    }
+
+    // There could be multiple client requests trying to disable or enable
+    // the table at the same time. Ensure only the first request is honored
+    // After that, no other requests can be accepted until the table reaches
+    // DISABLED or ENABLED.
+    if (!skipTableStateCheck)
+    {
+      try {
+        if (!this.assignmentManager.getZKTable().checkEnabledAndSetDisablingTable
+          (this.tableNameStr)) {
+          LOG.info("Table " + tableNameStr + " isn't enabled; skipping disable");
+          throw new TableNotEnabledException(this.tableNameStr);
+        }
+      } catch (KeeperException e) {
+        throw new IOException("Unable to ensure that the table will be" +
+          " disabling because of a ZooKeeper issue", e);
+      }
+    }
+  }
+
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    return getClass().getSimpleName() + "-" + name + "-" + getSeqid() + "-" +
+      tableNameStr;
+  }
+
+  @Override
+  public void process() {
+    try {
+      LOG.info("Attemping to disable table " + this.tableNameStr);
+      MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost();
+      if (cpHost != null) {
+        cpHost.preDisableTableHandler(this.tableName);
+      }
+      handleDisableTable();
+      if (cpHost != null) {
+        cpHost.postDisableTableHandler(this.tableName);
+      }
+    } catch (IOException e) {
+      LOG.error("Error trying to disable table " + this.tableNameStr, e);
+    } catch (KeeperException e) {
+      LOG.error("Error trying to disable table " + this.tableNameStr, e);
+    }
+  }
+
+  private void handleDisableTable() throws IOException, KeeperException {
+    // Set table disabling flag up in zk.
+    this.assignmentManager.getZKTable().setDisablingTable(this.tableNameStr);
+    boolean done = false;
+    while (true) {
+      // Get list of online regions that are of this table.  Regions that are
+      // already closed will not be included in this list; i.e. the returned
+      // list is not ALL regions in a table, its all online regions according
+      // to the in-memory state on this master.
+      final List regions =
+        this.assignmentManager.getRegionsOfTable(tableName);
+      if (regions.size() == 0) {
+        done = true;
+        break;
+      }
+      LOG.info("Offlining " + regions.size() + " regions.");
+      BulkDisabler bd = new BulkDisabler(this.server, regions);
+      try {
+        if (bd.bulkAssign()) {
+          done = true;
+          break;
+        }
+      } catch (InterruptedException e) {
+        LOG.warn("Disable was interrupted");
+        // Preserve the interrupt.
+        Thread.currentThread().interrupt();
+        break;
+      }
+    }
+    // Flip the table to disabled if success.
+    if (done) this.assignmentManager.getZKTable().setDisabledTable(this.tableNameStr);
+    LOG.info("Disabled table is done=" + done);
+  }
+
+  /**
+   * Run bulk disable.
+   */
+  class BulkDisabler extends BulkAssigner {
+    private final List regions;
+
+    BulkDisabler(final Server server, final List regions) {
+      super(server);
+      this.regions = regions;
+    }
+
+    @Override
+    protected void populatePool(ExecutorService pool) {
+      for (HRegionInfo region: regions) {
+        if (assignmentManager.isRegionInTransition(region) != null) continue;
+        final HRegionInfo hri = region;
+        pool.execute(new Runnable() {
+          public void run() {
+            assignmentManager.unassign(hri);
+          }
+        });
+      }
+    }
+
+    @Override
+    protected boolean waitUntilDone(long timeout)
+    throws InterruptedException {
+      long startTime = System.currentTimeMillis();
+      long remaining = timeout;
+      List regions = null;
+      while (!server.isStopped() && remaining > 0) {
+        Thread.sleep(waitingTimeForEvents);
+        regions = assignmentManager.getRegionsOfTable(tableName);
+        LOG.debug("Disable waiting until done; " + remaining + " ms remaining; " + regions);
+        if (regions.isEmpty()) break;
+        remaining = timeout - (System.currentTimeMillis() - startTime);
+      }
+      return regions != null && regions.isEmpty();
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
new file mode 100644
index 0000000..74b2e64
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
@@ -0,0 +1,278 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableNotDisabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.BulkAssigner;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.RegionPlan;
+import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Handler to run enable of a table.
+ */
+public class EnableTableHandler extends EventHandler {
+  private static final Log LOG = LogFactory.getLog(EnableTableHandler.class);
+  private final byte [] tableName;
+  private final String tableNameStr;
+  private final AssignmentManager assignmentManager;
+  private final CatalogTracker ct;
+  private boolean retainAssignment = false;
+
+  public EnableTableHandler(Server server, byte [] tableName,
+      CatalogTracker catalogTracker, AssignmentManager assignmentManager,
+      boolean skipTableStateCheck)
+  throws TableNotFoundException, TableNotDisabledException, IOException {
+    super(server, EventType.C_M_ENABLE_TABLE);
+    this.tableName = tableName;
+    this.tableNameStr = Bytes.toString(tableName);
+    this.ct = catalogTracker;
+    this.assignmentManager = assignmentManager;
+    this.retainAssignment = skipTableStateCheck;
+    // Check if table exists
+    if (!MetaReader.tableExists(catalogTracker, this.tableNameStr)) {
+      // retainAssignment is true only during recovery. In normal case it is
+      // false
+      if (!this.retainAssignment) {
+        throw new TableNotFoundException(tableNameStr);
+      }
+      try {
+        this.assignmentManager.getZKTable().removeEnablingTable(tableNameStr, true);
+      } catch (KeeperException e) {
+        // TODO : Use HBCK to clear such nodes
+        LOG.warn("Failed to delete the ENABLING node for the table " + tableNameStr
+            + ".  The table will remain unusable. Run HBCK to manually fix the problem.");
+      }
+    }
+
+    // There could be multiple client requests trying to disable or enable
+    // the table at the same time. Ensure only the first request is honored
+    // After that, no other requests can be accepted until the table reaches
+    // DISABLED or ENABLED.
+    if (!skipTableStateCheck)
+    {
+      try {
+        if (!this.assignmentManager.getZKTable().checkDisabledAndSetEnablingTable
+          (this.tableNameStr)) {
+          LOG.info("Table " + tableNameStr + " isn't disabled; skipping enable");
+          throw new TableNotDisabledException(this.tableNameStr);
+        }
+      } catch (KeeperException e) {
+        throw new IOException("Unable to ensure that the table will be" +
+          " enabling because of a ZooKeeper issue", e);
+      }
+    }
+  }
+
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    return getClass().getSimpleName() + "-" + name + "-" + getSeqid() + "-" +
+      tableNameStr;
+  }
+
+  @Override
+  public void process() {
+    try {
+      LOG.info("Attemping to enable the table " + this.tableNameStr);
+      MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost();
+      if (cpHost != null) {
+        cpHost.preEnableTableHandler(this.tableName);
+      }
+      handleEnableTable();
+      if (cpHost != null) {
+        cpHost.postEnableTableHandler(this.tableName);
+      }
+    } catch (IOException e) {
+      LOG.error("Error trying to enable the table " + this.tableNameStr, e);
+    } catch (KeeperException e) {
+      LOG.error("Error trying to enable the table " + this.tableNameStr, e);
+    } catch (InterruptedException e) {
+      LOG.error("Error trying to enable the table " + this.tableNameStr, e);
+    }
+  }
+
+  private void handleEnableTable() throws IOException, KeeperException, InterruptedException {
+    // I could check table is disabling and if so, not enable but require
+    // that user first finish disabling but that might be obnoxious.
+
+    // Set table enabling flag up in zk.
+    this.assignmentManager.getZKTable().setEnablingTable(this.tableNameStr);
+    boolean done = false;
+    // Get the regions of this table. We're done when all listed
+    // tables are onlined.
+    List> tableRegionsAndLocations = MetaReader
+        .getTableRegionsAndLocations(this.ct, tableName, true);
+    int countOfRegionsInTable = tableRegionsAndLocations.size();
+    List regions = regionsToAssignWithServerName(tableRegionsAndLocations);
+    int regionsCount = regions.size();
+    if (regionsCount == 0) {
+      done = true;
+    }
+    LOG.info("Table has " + countOfRegionsInTable + " regions of which " +
+      regionsCount + " are offline.");
+    BulkEnabler bd = new BulkEnabler(this.server, regions, countOfRegionsInTable,
+        this.retainAssignment);
+    try {
+      if (bd.bulkAssign()) {
+        done = true;
+      }
+    } catch (InterruptedException e) {
+      LOG.warn("Enable was interrupted");
+      // Preserve the interrupt.
+      Thread.currentThread().interrupt();
+    }
+    // Flip the table to enabled.
+    if (done) this.assignmentManager.getZKTable().setEnabledTable(
+      this.tableNameStr);
+    LOG.info("Enabled table is done=" + done);
+  }
+
+  /**
+   * @param regionsInMeta This datastructure is edited by this method.
+   * @return List of regions neither in transition nor assigned.
+   * @throws IOException
+   */
+  private List regionsToAssignWithServerName(
+      final List> regionsInMeta) throws IOException {
+    ServerManager serverManager = ((HMaster) this.server).getServerManager();
+    List regions = new ArrayList();
+    List enablingTableRegions = this.assignmentManager
+        .getEnablingTableRegions(this.tableNameStr);
+    final List onlineRegions = this.assignmentManager.getRegionsOfTable(tableName);
+    for (Pair regionLocation : regionsInMeta) {
+      HRegionInfo hri = regionLocation.getFirst();
+      ServerName sn = regionLocation.getSecond();
+      if (this.retainAssignment) {
+        // Region may be available in enablingTableRegions during master startup only.
+        if (enablingTableRegions != null && enablingTableRegions.contains(hri)) {
+          regions.add(hri);
+          if (sn != null && serverManager.isServerOnline(sn)) {
+            this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, sn));
+            this.assignmentManager.putRegionPlan(hri, sn);
+          }
+        }
+      } else if (onlineRegions.contains(hri)) {
+        continue;
+      } else {
+        regions.add(hri);
+      }
+    }
+    return regions;
+  }
+
+  /**
+   * Run bulk enable.
+   */
+  class BulkEnabler extends BulkAssigner {
+    private final List regions;
+    // Count of regions in table at time this assign was launched.
+    private final int countOfRegionsInTable;
+    private final boolean retainAssignment;
+
+    BulkEnabler(final Server server, final List regions,
+        final int countOfRegionsInTable,final boolean retainAssignment) {
+      super(server);
+      this.regions = regions;
+      this.countOfRegionsInTable = countOfRegionsInTable;
+      this.retainAssignment = retainAssignment;
+    }
+
+    @Override
+    protected void populatePool(ExecutorService pool) throws IOException {
+      boolean roundRobinAssignment = this.server.getConfiguration().getBoolean(
+          "hbase.master.enabletable.roundrobin", false);
+
+      if (retainAssignment || !roundRobinAssignment) {
+        for (HRegionInfo region : regions) {
+          if (assignmentManager.isRegionInTransition(region) != null) {
+            continue;
+          }
+          final HRegionInfo hri = region;
+          pool.execute(new Runnable() {
+            public void run() {
+              if (retainAssignment) {
+                assignmentManager.assign(hri, true, false, false);
+              } else {
+                assignmentManager.assign(hri, true);
+              }
+            }
+          });
+        }
+      } else {
+        try {
+          assignmentManager.assignUserRegionsToOnlineServers(regions);
+        } catch (InterruptedException e) {
+          LOG.warn("Assignment was interrupted");
+          Thread.currentThread().interrupt();
+        }
+      }
+    }
+
+    @Override
+    protected boolean waitUntilDone(long timeout)
+    throws InterruptedException {
+      long startTime = System.currentTimeMillis();
+      long remaining = timeout;
+      List regions = null;
+      int lastNumberOfRegions = 0;
+      while (!server.isStopped() && remaining > 0) {
+        Thread.sleep(waitingTimeForEvents);
+        regions = assignmentManager.getRegionsOfTable(tableName);
+        if (isDone(regions)) break;
+
+        // Punt on the timeout as long we make progress
+        if (regions.size() > lastNumberOfRegions) {
+          lastNumberOfRegions = regions.size();
+          timeout += waitingTimeForEvents;
+        }
+        remaining = timeout - (System.currentTimeMillis() - startTime);
+      }
+      return isDone(regions);
+    }
+
+    private boolean isDone(final List regions) {
+      return regions != null && regions.size() >= this.countOfRegionsInTable;
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
new file mode 100644
index 0000000..a241e20
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
@@ -0,0 +1,199 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.DeadServer;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Shutdown handler for the server hosting -ROOT-,
+ * .META., or both.
+ */
+public class MetaServerShutdownHandler extends ServerShutdownHandler {
+  private final boolean carryingRoot;
+  private final boolean carryingMeta;
+  private static final Log LOG = LogFactory.getLog(MetaServerShutdownHandler.class);
+  public MetaServerShutdownHandler(final Server server,
+      final MasterServices services,
+      final DeadServer deadServers, final ServerName serverName,
+      final boolean carryingRoot, final boolean carryingMeta) {
+    super(server, services, deadServers, serverName,
+      EventType.M_META_SERVER_SHUTDOWN, true);
+    this.carryingRoot = carryingRoot;
+    this.carryingMeta = carryingMeta;
+  }
+
+  @Override
+  public void process() throws IOException {
+
+    boolean gotException = true;
+    try {
+      try {
+        if (this.shouldSplitHlog) {
+          if (this.services.shouldSplitMetaSeparately()) {
+            LOG.info("Splitting META logs for " + serverName);
+            this.services.getMasterFileSystem().splitMetaLog(serverName);
+          } else {
+            LOG.info("Splitting all logs for " + serverName);
+            this.services.getMasterFileSystem().splitAllLogs(serverName);
+          }
+        }
+      } catch (IOException ioe) {
+        this.services.getExecutorService().submit(this);
+        this.deadServers.add(serverName);
+        throw new IOException("failed log splitting for " +
+            serverName + ", will retry", ioe);
+      }
+  
+      // Assign root and meta if we were carrying them.
+      if (isCarryingRoot()) { // -ROOT-
+        // Check again: region may be assigned to other where because of RIT
+        // timeout
+        if (this.services.getAssignmentManager().isCarryingRoot(serverName)) {
+          LOG.info("Server " + serverName
+              + " was carrying ROOT. Trying to assign.");
+          this.services.getAssignmentManager().regionOffline(
+              HRegionInfo.ROOT_REGIONINFO);
+          verifyAndAssignRootWithRetries();
+        } else {
+          LOG.info("ROOT has been assigned to otherwhere, skip assigning.");
+        }
+      }
+      
+      if(!this.services.isServerShutdownHandlerEnabled()) {
+        // resubmit in case we're in master initialization and SSH hasn't been enabled yet.
+        this.services.getExecutorService().submit(this);
+        this.deadServers.add(serverName);
+        return;
+      }
+  
+      // Carrying meta?
+      if (isCarryingMeta()) {
+        // Check again: region may be assigned to other where because of RIT
+        // timeout
+        if (this.services.getAssignmentManager().isCarryingMeta(serverName)) {
+          LOG.info("Server " + serverName
+              + " was carrying META. Trying to assign.");
+          this.services.getAssignmentManager().regionOffline(
+              HRegionInfo.FIRST_META_REGIONINFO);
+          this.services.getAssignmentManager().assignMeta();
+        } else {
+          LOG.info("META has been assigned to otherwhere, skip assigning.");
+        }
+      }
+      
+      gotException = false;
+    } finally {
+      if (gotException){
+        // If we had an exception, this.deadServers.finish will be skipped in super.process()
+        this.deadServers.finish(serverName);
+      }
+    }
+
+    super.process();
+  }
+  /**
+   * Before assign the ROOT region, ensure it haven't
+   *  been assigned by other place
+   * 
        
+   * Under some scenarios, the ROOT region can be opened twice, so it seemed online
+   * in two regionserver at the same time.
+   * If the ROOT region has been assigned, so the operation can be canceled.
+   * @throws InterruptedException
+   * @throws IOException
+   * @throws KeeperException
+   */
+  private void verifyAndAssignRoot()
+  throws InterruptedException, IOException, KeeperException {
+    long timeout = this.server.getConfiguration().
+      getLong("hbase.catalog.verification.timeout", 1000);
+    if (!this.server.getCatalogTracker().verifyRootRegionLocation(timeout)) {
+      this.services.getAssignmentManager().assignRoot();
+    } else if (serverName.equals(server.getCatalogTracker().getRootLocation())) {
+      throw new IOException("-ROOT- is onlined on the dead server "
+          + serverName);
+    } else {
+      LOG.info("Skip assigning -ROOT-, because it is online on the "
+          + server.getCatalogTracker().getRootLocation());
+    }
+  }
+
+  /**
+   * Failed many times, shutdown processing
+   * @throws IOException
+   */
+  private void verifyAndAssignRootWithRetries() throws IOException {
+    int iTimes = this.server.getConfiguration().getInt(
+        "hbase.catalog.verification.retries", 10);
+
+    long waitTime = this.server.getConfiguration().getLong(
+        "hbase.catalog.verification.timeout", 1000);
+
+    int iFlag = 0;
+    while (true) {
+      try {
+        verifyAndAssignRoot();
+        break;
+      } catch (KeeperException e) {
+        this.server.abort("In server shutdown processing, assigning root", e);
+        throw new IOException("Aborting", e);
+      } catch (Exception e) {
+        if (iFlag >= iTimes) {
+          this.server.abort("verifyAndAssignRoot failed after" + iTimes
+              + " times retries, aborting", e);
+          throw new IOException("Aborting", e);
+        }
+        try {
+          Thread.sleep(waitTime);
+        } catch (InterruptedException e1) {
+          LOG.warn("Interrupted when is the thread sleep", e1);
+          Thread.currentThread().interrupt();
+          throw new IOException("Interrupted", e1);
+        }
+        iFlag++;
+      }
+    }
+  }
+
+  boolean isCarryingRoot() {
+    return this.carryingRoot;
+  }
+
+  boolean isCarryingMeta() {
+    return this.carryingMeta;
+  }
+  
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    return getClass().getSimpleName() + "-" + name + "-" + getSeqid();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
new file mode 100644
index 0000000..0dc9ffa
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
@@ -0,0 +1,69 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.MasterServices;
+
+public class ModifyTableHandler extends TableEventHandler {
+  private final HTableDescriptor htd;
+
+  public ModifyTableHandler(final byte [] tableName,
+      final HTableDescriptor htd, final Server server,
+      final MasterServices masterServices)
+  throws IOException {
+    super(EventType.C_M_MODIFY_TABLE, tableName, server, masterServices);
+    // Check table exists.
+    getTableDescriptor();
+    // This is the new schema we are going to write out as this modification.
+    this.htd = htd;
+  }
+
+  @Override
+  protected void handleTableOperation(List hris)
+  throws IOException {
+    MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost();
+    if (cpHost != null) {
+      cpHost.preModifyTableHandler(this.tableName, this.htd);
+    }
+    // Update descriptor
+    this.masterServices.getTableDescriptors().add(this.htd);
+    if (cpHost != null) {
+      cpHost.postModifyTableHandler(this.tableName, this.htd);
+    }
+  }
+
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    return getClass().getSimpleName() + "-" + name + "-" + getSeqid() + "-" +
+      tableNameStr;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java
new file mode 100644
index 0000000..f171a5a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java
@@ -0,0 +1,150 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Handles OPENED region event on Master.
+ */
+public class OpenedRegionHandler extends EventHandler implements TotesHRegionInfo {
+  private static final Log LOG = LogFactory.getLog(OpenedRegionHandler.class);
+  private final AssignmentManager assignmentManager;
+  private final HRegionInfo regionInfo;
+  private final ServerName sn;
+  private final OpenedPriority priority;
+  private final int expectedVersion;
+
+  private enum OpenedPriority {
+    ROOT (1),
+    META (2),
+    USER (3);
+
+    private final int value;
+    OpenedPriority(int value) {
+      this.value = value;
+    }
+    public int getValue() {
+      return value;
+    }
+  };
+
+  public OpenedRegionHandler(Server server,
+      AssignmentManager assignmentManager, HRegionInfo regionInfo,
+      ServerName sn, int expectedVersion) {
+    super(server, EventType.RS_ZK_REGION_OPENED);
+    this.assignmentManager = assignmentManager;
+    this.regionInfo = regionInfo;
+    this.sn = sn;
+    this.expectedVersion = expectedVersion;
+    if(regionInfo.isRootRegion()) {
+      priority = OpenedPriority.ROOT;
+    } else if(regionInfo.isMetaRegion()) {
+      priority = OpenedPriority.META;
+    } else {
+      priority = OpenedPriority.USER;
+    }
+  }
+
+  @Override
+  public int getPriority() {
+    return priority.getValue();
+  }
+
+  @Override
+  public HRegionInfo getHRegionInfo() {
+    return this.regionInfo;
+  }
+  
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    return getClass().getSimpleName() + "-" + name + "-" + getSeqid();
+  }
+
+  @Override
+  public void process() {
+    // Code to defend against case where we get SPLIT before region open
+    // processing completes; temporary till we make SPLITs go via zk -- 0.92.
+    RegionState regionState = this.assignmentManager.isRegionInTransition(regionInfo);
+    boolean openedNodeDeleted = false;
+    if (regionState != null
+        && regionState.getState().equals(RegionState.State.OPEN)) {
+      openedNodeDeleted = deleteOpenedNode(expectedVersion);
+      if (!openedNodeDeleted) {
+        LOG.error("The znode of region " + regionInfo.getRegionNameAsString()
+            + " could not be deleted.");
+      }
+    } else {
+      LOG.warn("Skipping the onlining of " + regionInfo.getRegionNameAsString() +
+        " because regions is NOT in RIT -- presuming this is because it SPLIT");
+    }
+    if (!openedNodeDeleted) {
+      if (this.assignmentManager.getZKTable().isDisablingOrDisabledTable(
+          regionInfo.getTableNameAsString())) {
+        debugLog(regionInfo, "Opened region "
+            + regionInfo.getRegionNameAsString() + " but "
+            + "this table is disabled, triggering close of region");
+        assignmentManager.unassign(regionInfo);
+      }
+    }
+  }
+
+  private boolean deleteOpenedNode(int expectedVersion) {
+    debugLog(regionInfo, "Handling OPENED event for " +
+      this.regionInfo.getRegionNameAsString() + " from " + this.sn.toString() +
+      "; deleting unassigned node");
+    try {
+      // delete the opened znode only if the version matches.
+      return ZKAssign.deleteNode(server.getZooKeeper(),
+          regionInfo.getEncodedName(), EventType.RS_ZK_REGION_OPENED, expectedVersion);
+    } catch(KeeperException.NoNodeException e){
+      // Getting no node exception here means that already the region has been opened.
+      LOG.warn("The znode of the region " + regionInfo.getRegionNameAsString() +
+        " would have already been deleted");
+      return false;
+    } catch (KeeperException e) {
+      server.abort("Error deleting OPENED node in ZK (" +
+        regionInfo.getRegionNameAsString() + ")", e);
+    }
+    return false;
+  }
+
+  private void debugLog(HRegionInfo region, String string) {
+    if (region.isMetaTable()) {
+      LOG.info(string);
+    } else {
+      LOG.debug(string);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
new file mode 100644
index 0000000..41efd27
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
@@ -0,0 +1,493 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
+import org.apache.hadoop.hbase.master.DeadServer;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Process server shutdown.
+ * Server-to-handle must be already in the deadservers lists.  See
+ * {@link ServerManager#expireServer(ServerName)}
+ */
+public class ServerShutdownHandler extends EventHandler {
+  private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class);
+  protected final ServerName serverName;
+  protected final MasterServices services;
+  protected final DeadServer deadServers;
+  protected final boolean shouldSplitHlog; // whether to split HLog or not
+
+  public ServerShutdownHandler(final Server server, final MasterServices services,
+      final DeadServer deadServers, final ServerName serverName,
+      final boolean shouldSplitHlog) {
+    this(server, services, deadServers, serverName, EventType.M_SERVER_SHUTDOWN,
+        shouldSplitHlog);
+  }
+
+  ServerShutdownHandler(final Server server, final MasterServices services,
+      final DeadServer deadServers, final ServerName serverName, EventType type,
+      final boolean shouldSplitHlog) {
+    super(server, type);
+    this.serverName = serverName;
+    this.server = server;
+    this.services = services;
+    this.deadServers = deadServers;
+    if (!this.deadServers.contains(this.serverName)) {
+      LOG.warn(this.serverName + " is NOT in deadservers; it should be!");
+    }
+    this.shouldSplitHlog = shouldSplitHlog;
+  }
+
+  @Override
+  public String getInformativeName() {
+    if (serverName != null) {
+      return this.getClass().getSimpleName() + " for " + serverName;
+    } else {
+      return super.getInformativeName();
+    }
+  }
+  
+  /**
+   * @return True if the server we are processing was carrying -ROOT-
+   */
+  boolean isCarryingRoot() {
+    return false;
+  }
+
+  /**
+   * @return True if the server we are processing was carrying .META.
+   */
+  boolean isCarryingMeta() {
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    return getClass().getSimpleName() + "-" + name + "-" + getSeqid();
+  }
+
+  @Override
+  public void process() throws IOException {
+    final ServerName serverName = this.serverName;
+    try {
+      if (this.server.isStopped()) {
+        throw new IOException("Server is stopped");
+      }
+
+      try {
+        if (this.shouldSplitHlog) {
+          LOG.info("Splitting logs for " + serverName);
+          this.services.getMasterFileSystem().splitLog(serverName);
+        } else {
+          LOG.info("Skipping log splitting for " + serverName);
+        }
+      } catch (IOException ioe) {
+        //typecast to SSH so that we make sure that it is the SSH instance that
+        //gets submitted as opposed to MSSH or some other derived instance of SSH
+        this.services.getExecutorService().submit((ServerShutdownHandler)this);
+        this.deadServers.add(serverName);
+        throw new IOException("failed log splitting for " +
+          serverName + ", will retry", ioe);
+      }
+
+      // We don't want worker thread in the MetaServerShutdownHandler
+      // executor pool to block by waiting availability of -ROOT-
+      // and .META. server. Otherwise, it could run into the following issue:
+      // 1. The current MetaServerShutdownHandler instance For RS1 waits for the .META.
+      //    to come online.
+      // 2. The newly assigned .META. region server RS2 was shutdown right after
+      //    it opens the .META. region. So the MetaServerShutdownHandler
+      //    instance For RS1 will still be blocked.
+      // 3. The new instance of MetaServerShutdownHandler for RS2 is queued.
+      // 4. The newly assigned .META. region server RS3 was shutdown right after
+      //    it opens the .META. region. So the MetaServerShutdownHandler
+      //    instance For RS1 and RS2 will still be blocked.
+      // 5. The new instance of MetaServerShutdownHandler for RS3 is queued.
+      // 6. Repeat until we run out of MetaServerShutdownHandler worker threads
+      // The solution here is to resubmit a ServerShutdownHandler request to process
+      // user regions on that server so that MetaServerShutdownHandler
+      // executor pool is always available.
+      if (isCarryingRoot() || isCarryingMeta()) { // -ROOT- or .META.
+        this.services.getExecutorService().submit(new ServerShutdownHandler(
+          this.server, this.services, this.deadServers, serverName, false));
+        this.deadServers.add(serverName);
+        return;
+      }
+
+
+      // Wait on meta to come online; we need it to progress.
+      // TODO: Best way to hold strictly here?  We should build this retry logic
+      // into the MetaReader operations themselves.
+      // TODO: Is the reading of .META. necessary when the Master has state of
+      // cluster in its head?  It should be possible to do without reading .META.
+      // in all but one case. On split, the RS updates the .META.
+      // table and THEN informs the master of the split via zk nodes in
+      // 'unassigned' dir.  Currently the RS puts ephemeral nodes into zk so if
+      // the regionserver dies, these nodes do not stick around and this server
+      // shutdown processing does fixup (see the fixupDaughters method below).
+      // If we wanted to skip the .META. scan, we'd have to change at least the
+      // final SPLIT message to be permanent in zk so in here we'd know a SPLIT
+      // completed (zk is updated after edits to .META. have gone in).  See
+      // {@link SplitTransaction}.  We'd also have to be figure another way for
+      // doing the below .META. daughters fixup.
+      NavigableMap hris = null;
+      while (!this.server.isStopped()) {
+        try {
+          this.server.getCatalogTracker().waitForMeta();
+          hris = MetaReader.getServerUserRegions(this.server.getCatalogTracker(),
+            this.serverName);
+          break;
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+          throw new IOException("Interrupted", e);
+        } catch (IOException ioe) {
+          LOG.info("Received exception accessing META during server shutdown of " +
+              serverName + ", retrying META read", ioe);
+        }
+      }
+
+      // Returns set of regions that had regionplans against the downed server and a list of
+      // the intersection of regions-in-transition and regions that were on the server that died.
+      Pair, List> p = this.services.getAssignmentManager()
+          .processServerShutdown(this.serverName);
+      Set ritsGoingToServer = p.getFirst();
+      List ritsOnServer = p.getSecond();
+
+      List regionsToAssign = getRegionsToAssign(hris, ritsOnServer, ritsGoingToServer);
+      for (HRegionInfo hri : ritsGoingToServer) {
+        if (!this.services.getAssignmentManager().isRegionAssigned(hri)) {
+          if (!regionsToAssign.contains(hri)) {
+            regionsToAssign.add(hri);
+            RegionState rit =
+                services.getAssignmentManager().getRegionsInTransition().get(hri.getEncodedName());
+            removeRITsOfRregionInDisablingOrDisabledTables(regionsToAssign, rit,
+              services.getAssignmentManager(), hri);
+          }
+        }
+      }
+
+      // re-assign regions
+      for (HRegionInfo hri : regionsToAssign) {
+        this.services.getAssignmentManager().assign(hri, true);
+      }
+      LOG.info(regionsToAssign.size() + " regions which were planned to open on " + this.serverName
+          + " have been re-assigned.");
+    } finally {
+      this.deadServers.finish(serverName);
+    }
+    LOG.info("Finished processing of shutdown of " + serverName);
+  }
+
+  /**
+   * Figure what to assign from the dead server considering state of RIT and whats up in .META.
+   * @param metaHRIs Regions that .META. says were assigned to the dead server
+   * @param ritsOnServer Regions that were in transition, and on the dead server.
+   * @param ritsGoingToServer Regions that were in transition to the dead server.
+   * @return List of regions to assign or null if aborting.
+   * @throws IOException
+   */
+  private List getRegionsToAssign(final NavigableMap metaHRIs,
+      final List ritsOnServer, Set ritsGoingToServer) throws IOException {
+    List toAssign = new ArrayList();
+    // If no regions on the server, then nothing to assign (Regions that were currently being
+    // assigned will be retried over in the AM#assign method).
+    if (metaHRIs == null || metaHRIs.isEmpty()) return toAssign;
+    // Remove regions that we do not want to reassign such as regions that are
+    // OFFLINE. If region is OFFLINE against this server, its probably being assigned over
+    // in the single region assign method in AM; do not assign it here too. TODO: VERIFY!!!
+    // TODO: Currently OFFLINE is too messy. Its done on single assign but bulk done when bulk
+    // assigning and then there is special handling when master joins a cluster.
+    //
+    // If split, the zk callback will have offlined. Daughters will be in the
+    // list of hris we got from scanning the .META. These should be reassigned. Not the parent.
+    for (RegionState rs : ritsOnServer) {
+      if (!rs.isClosing() && !rs.isPendingClose() && !rs.isSplitting()) {
+        LOG.debug("Removed " + rs.getRegion().getRegionNameAsString()
+            + " from list of regions to assign because region state: " + rs.getState());
+        metaHRIs.remove(rs.getRegion());
+      }
+    }
+
+    AssignmentManager assignmentManager = this.services.getAssignmentManager();
+    for (Map.Entry e : metaHRIs.entrySet()) {
+      RegionState rit =
+          assignmentManager.getRegionsInTransition().get(e.getKey().getEncodedName());
+
+      if (processDeadRegion(e.getKey(), e.getValue(), assignmentManager,
+        this.server.getCatalogTracker())) {
+        ServerName addressFromAM = assignmentManager.getRegionServerOfRegion(e.getKey());
+        if (rit != null && !rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting()
+            && !ritsGoingToServer.contains(e.getKey())) {
+          // Skip regions that were in transition unless CLOSING or
+          // PENDING_CLOSE
+          LOG.info("Skip assigning region " + rit.toString());
+        } else if (addressFromAM != null && !addressFromAM.equals(this.serverName)) {
+          LOG.debug("Skip assigning region " + e.getKey().getRegionNameAsString()
+              + " because it has been opened in " + addressFromAM.getServerName());
+          ritsGoingToServer.remove(e.getKey());
+        } else {
+          if (rit != null) {
+            // clean zk node
+            try {
+              LOG.info("Reassigning region with rs =" + rit + " and deleting zk node if exists");
+              ZKAssign.deleteNodeFailSilent(services.getZooKeeper(), e.getKey());
+            } catch (KeeperException ke) {
+              this.server.abort("Unexpected ZK exception deleting unassigned node " + e.getKey(),
+                ke);
+              return null;
+            }
+          }
+          toAssign.add(e.getKey());
+        }
+      } else if (rit != null && (rit.isSplitting() || rit.isSplit())) {
+        // This will happen when the RS went down and the call back for the SPLIITING or SPLIT
+        // has not yet happened for node Deleted event. In that case if the region was actually
+        // split but the RS had gone down before completing the split process then will not try
+        // to assign the parent region again. In that case we should make the region offline
+        // and also delete the region from RIT.
+        HRegionInfo region = rit.getRegion();
+        AssignmentManager am = assignmentManager;
+        am.regionOffline(region);
+        ritsGoingToServer.remove(region);
+      }
+      // If the table was partially disabled and the RS went down, we should clear the RIT
+      // and remove the node for the region. The rit that we use may be stale in case the table
+      // was in DISABLING state but though we did assign we will not be clearing the znode in
+      // CLOSING state. Doing this will have no harm. The rit can be null if region server went
+      // down during master startup. In that case If any znodes' exists for partially disabled 
+      // table regions deleting them during startup only. See HBASE-8127. 
+      removeRITsOfRregionInDisablingOrDisabledTables(toAssign, rit, assignmentManager, e.getKey());
+    }
+
+    return toAssign;
+  }
+
+  private void removeRITsOfRregionInDisablingOrDisabledTables(List toAssign,
+      RegionState rit, AssignmentManager assignmentManager, HRegionInfo hri) {
+
+    if (!assignmentManager.getZKTable().isDisablingOrDisabledTable(hri.getTableNameAsString())) {
+      return;
+    }
+
+    // To avoid region assignment if table is in disabling or disabled state.
+    toAssign.remove(hri);
+
+    if (rit != null) {
+      assignmentManager.deleteNodeAndOfflineRegion(hri);
+    }
+  }
+
+  /**
+   * Process a dead region from a dead RS. Checks if the region is disabled or
+   * disabling or if the region has a partially completed split.
+   * @param hri
+   * @param result
+   * @param assignmentManager
+   * @param catalogTracker
+   * @return Returns true if specified region should be assigned, false if not.
+   * @throws IOException
+   */
+  public static boolean processDeadRegion(HRegionInfo hri, Result result,
+      AssignmentManager assignmentManager, CatalogTracker catalogTracker)
+  throws IOException {
+    boolean tablePresent = assignmentManager.getZKTable().isTablePresent(
+        hri.getTableNameAsString());
+    if (!tablePresent) {
+      LOG.info("The table " + hri.getTableNameAsString()
+          + " was deleted.  Hence not proceeding.");
+      return false;
+    }
+    // If table is not disabled but the region is offlined,
+    boolean disabled = assignmentManager.getZKTable().isDisabledTable(
+        hri.getTableNameAsString());
+    if (disabled){
+      LOG.info("The table " + hri.getTableNameAsString()
+          + " was disabled.  Hence not proceeding.");
+      return false;
+    }
+    if (hri.isOffline() && hri.isSplit()) {
+      LOG.debug("Offlined and split region " + hri.getRegionNameAsString() +
+        "; checking daughter presence");
+      if (MetaReader.getRegion(catalogTracker, hri.getRegionName()) == null) {
+        return false;
+      }
+      fixupDaughters(result, assignmentManager, catalogTracker);
+      return false;
+    }
+    boolean disabling = assignmentManager.getZKTable().isDisablingTable(
+        hri.getTableNameAsString());
+    if (disabling) {
+      LOG.info("The table " + hri.getTableNameAsString()
+          + " is disabled.  Hence not assigning region" + hri.getEncodedName());
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Check that daughter regions are up in .META. and if not, add them.
+   * @param hris All regions for this server in meta.
+   * @param result The contents of the parent row in .META.
+   * @return the number of daughters missing and fixed
+   * @throws IOException
+   */
+  public static int fixupDaughters(final Result result,
+      final AssignmentManager assignmentManager,
+      final CatalogTracker catalogTracker)
+  throws IOException {
+    int fixedA = fixupDaughter(result, HConstants.SPLITA_QUALIFIER,
+      assignmentManager, catalogTracker);
+    int fixedB = fixupDaughter(result, HConstants.SPLITB_QUALIFIER,
+      assignmentManager, catalogTracker);
+    return fixedA + fixedB;
+  }
+
+  /**
+   * Check individual daughter is up in .META.; fixup if its not.
+   * @param result The contents of the parent row in .META.
+   * @param qualifier Which daughter to check for.
+   * @return 1 if the daughter is missing and fixed. Otherwise 0
+   * @throws IOException
+   */
+  static int fixupDaughter(final Result result, final byte [] qualifier,
+      final AssignmentManager assignmentManager,
+      final CatalogTracker catalogTracker)
+  throws IOException {
+    HRegionInfo daughter =
+      MetaReader.parseHRegionInfoFromCatalogResult(result, qualifier);
+    if (daughter == null) return 0;
+    if (isDaughterMissing(catalogTracker, daughter)) {
+      LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString());
+      MetaEditor.addDaughter(catalogTracker, daughter, null);
+
+      // TODO: Log WARN if the regiondir does not exist in the fs.  If its not
+      // there then something wonky about the split -- things will keep going
+      // but could be missing references to parent region.
+
+      // And assign it.
+      assignmentManager.assign(daughter, true);
+      return 1;
+    } else {
+      LOG.debug("Daughter " + daughter.getRegionNameAsString() + " present");
+    }
+    return 0;
+  }
+
+  /**
+   * Look for presence of the daughter OR of a split of the daughter in .META.
+   * Daughter could have been split over on regionserver before a run of the
+   * catalogJanitor had chance to clear reference from parent.
+   * @param daughter Daughter region to search for.
+   * @throws IOException 
+   */
+  private static boolean isDaughterMissing(final CatalogTracker catalogTracker,
+      final HRegionInfo daughter) throws IOException {
+    FindDaughterVisitor visitor = new FindDaughterVisitor(daughter);
+    // Start the scan at what should be the daughter's row in the .META.
+    // We will either 1., find the daughter or some derivative split of the
+    // daughter (will have same table name and start row at least but will sort
+    // after because has larger regionid -- the regionid is timestamp of region
+    // creation), OR, we will not find anything with same table name and start
+    // row.  If the latter, then assume daughter missing and do fixup.
+    byte [] startrow = daughter.getRegionName();
+    MetaReader.fullScan(catalogTracker, visitor, startrow);
+    return !visitor.foundDaughter();
+  }
+
+  /**
+   * Looks for daughter.  Sets a flag if daughter or some progeny of daughter
+   * is found up in .META..
+   */
+  static class FindDaughterVisitor implements MetaReader.Visitor {
+    private final HRegionInfo daughter;
+    private boolean found = false;
+
+    FindDaughterVisitor(final HRegionInfo daughter) {
+      this.daughter = daughter;
+    }
+
+    /**
+     * @return True if we found a daughter region during our visiting.
+     */
+    boolean foundDaughter() {
+      return this.found;
+    }
+
+    @Override
+    public boolean visit(Result r) throws IOException {
+      HRegionInfo hri =
+        MetaReader.parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
+      if (hri == null) {
+        LOG.warn("No serialized HRegionInfo in " + r);
+        return true;
+      }
+      byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
+          HConstants.SERVER_QUALIFIER);
+      // See if daughter is assigned to some server
+      if (value == null) return false;
+
+      // Now see if we have gone beyond the daughter's startrow.
+      if (!Bytes.equals(daughter.getTableName(),
+          hri.getTableName())) {
+        // We fell into another table.  Stop scanning.
+        return false;
+      }
+      // If our start rows do not compare, move on.
+      if (!Bytes.equals(daughter.getStartKey(), hri.getStartKey())) {
+        return false;
+      }
+      // Else, table name and start rows compare.  It means that the daughter
+      // or some derivative split of the daughter is up in .META.  Daughter
+      // exists.
+      this.found = true;
+      return false;
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/SplitRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/SplitRegionHandler.java
new file mode 100644
index 0000000..2d544dd
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/SplitRegionHandler.java
@@ -0,0 +1,118 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+/**
+ * Handles SPLIT region event on Master.
+ */
+public class SplitRegionHandler extends EventHandler implements TotesHRegionInfo {
+  private static final Log LOG = LogFactory.getLog(SplitRegionHandler.class);
+  private final AssignmentManager assignmentManager;
+  private final HRegionInfo parent;
+  private final ServerName sn;
+  private final List daughters;
+  /**
+   * For testing only!  Set to true to skip handling of split.
+   */
+  public static boolean TEST_SKIP = false;
+
+  public SplitRegionHandler(Server server,
+      AssignmentManager assignmentManager, HRegionInfo regionInfo,
+      ServerName sn, final List daughters) {
+    super(server, EventType.RS_ZK_REGION_SPLIT);
+    this.assignmentManager = assignmentManager;
+    this.parent = regionInfo;
+    this.sn = sn;
+    this.daughters = daughters;
+  }
+
+  @Override
+  public HRegionInfo getHRegionInfo() {
+    return this.parent;
+  }
+  
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    String parentRegion = "UnknownRegion";
+    if(parent != null) {
+      parentRegion = parent.getRegionNameAsString();
+    }
+    return getClass().getSimpleName() + "-" + name + "-" + getSeqid() + "-" + parentRegion;
+  }
+
+  @Override
+  public void process() {
+    String encodedRegionName = this.parent.getEncodedName();
+    LOG.debug("Handling SPLIT event for " + encodedRegionName +
+      "; deleting node");
+    // The below is for testing ONLY!  We can't do fault injection easily, so
+    // resort to this kinda uglyness -- St.Ack 02/25/2011.
+    if (TEST_SKIP) {
+      LOG.warn("Skipping split message, TEST_SKIP is set");
+      return;
+    }
+    this.assignmentManager.handleSplitReport(this.sn, this.parent,
+      this.daughters.get(0), this.daughters.get(1));
+    // Remove region from ZK
+    try {
+
+      boolean successful = false;
+      while (!successful) {
+        // It's possible that the RS tickles in between the reading of the
+        // znode and the deleting, so it's safe to retry.
+        successful = ZKAssign.deleteNode(this.server.getZooKeeper(),
+          encodedRegionName,
+          EventHandler.EventType.RS_ZK_REGION_SPLIT);
+      }
+    } catch (KeeperException e) {
+      if (e instanceof NoNodeException) {
+        String znodePath = ZKUtil.joinZNode(
+            this.server.getZooKeeper().splitLogZNode, encodedRegionName);
+        LOG.debug("The znode " + znodePath
+            + " does not exist.  May be deleted already.");
+      } else {
+        server.abort("Error deleting SPLIT node in ZK for transition ZK node (" +
+            parent.getEncodedName() + ")", e);
+      }
+    }
+    LOG.info("Handled SPLIT event; parent=" +
+      this.parent.getRegionNameAsString() +
+      " daughter a=" + this.daughters.get(0).getRegionNameAsString() +
+      "daughter b=" + this.daughters.get(1).getRegionNameAsString());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
new file mode 100644
index 0000000..267b0d4
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
@@ -0,0 +1,80 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.InvalidFamilyOperationException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.MasterServices;
+
+/**
+ * Handles adding a new family to an existing table.
+ */
+public class TableAddFamilyHandler extends TableEventHandler {
+
+  private final HColumnDescriptor familyDesc;
+
+  public TableAddFamilyHandler(byte[] tableName, HColumnDescriptor familyDesc,
+      Server server, final MasterServices masterServices) throws IOException {
+    super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices);
+    HTableDescriptor htd = getTableDescriptor();
+    if (htd.hasFamily(familyDesc.getName())) {
+      throw new InvalidFamilyOperationException("Family '" +
+        familyDesc.getNameAsString() + "' already exists so cannot be added");
+    }
+    this.familyDesc = familyDesc;
+  }
+
+  @Override
+  protected void handleTableOperation(List hris)
+  throws IOException {
+    MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost();
+    if (cpHost != null) {
+      cpHost.preAddColumnHandler(this.tableName, this.familyDesc);
+    }
+    // Update table descriptor
+    this.masterServices.getMasterFileSystem().addColumn(tableName, familyDesc);
+    if (cpHost != null) {
+      cpHost.postAddColumnHandler(this.tableName, this.familyDesc);
+    }
+  }
+  
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    String family = "UnknownFamily";
+    if(familyDesc != null) {
+      family = familyDesc.getNameAsString();
+    }
+    return getClass().getSimpleName() + "-" + name + "-" +
+                              getSeqid() + "-" + tableNameStr + "-" + family;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
new file mode 100644
index 0000000..76298dc
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
@@ -0,0 +1,79 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Handles adding a new family to an existing table.
+ */
+public class TableDeleteFamilyHandler extends TableEventHandler {
+
+  private final byte [] familyName;
+
+  public TableDeleteFamilyHandler(byte[] tableName, byte [] familyName,
+      Server server, final MasterServices masterServices) throws IOException {
+    super(EventType.C_M_ADD_FAMILY, tableName, server, masterServices);
+    HTableDescriptor htd = getTableDescriptor();
+    this.familyName = hasColumnFamily(htd, familyName);
+  }
+
+  @Override
+  protected void handleTableOperation(List hris) throws IOException {
+    MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost();
+    if (cpHost != null) {
+      cpHost.preDeleteColumnHandler(this.tableName, this.familyName);
+    }
+    MasterFileSystem mfs = this.masterServices.getMasterFileSystem();
+    // Update table descriptor
+    mfs.deleteColumn(tableName, familyName);
+    // Remove the column family from the file system
+    for (HRegionInfo hri : hris) {
+      // Delete the family directory in FS for all the regions one by one
+      mfs.deleteFamilyFromFS(hri, familyName);
+    }
+    if (cpHost != null) {
+      cpHost.postDeleteColumnHandler(this.tableName, this.familyName);
+    }
+  }
+
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    String family = "UnknownFamily";
+    if(familyName != null) {
+      family = Bytes.toString(familyName);
+    }
+    return getClass().getSimpleName() + "-" + name + "-" + getSeqid() + "-" + tableNameStr + "-" + family;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
new file mode 100644
index 0000000..b73f25d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
@@ -0,0 +1,238 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.InvalidFamilyOperationException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.TableNotDisabledException;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.BulkReOpen;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.zookeeper.KeeperException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ * Base class for performing operations against tables.
+ * Checks on whether the process can go forward are done in constructor rather
+ * than later on in {@link #process()}.  The idea is to fail fast rather than
+ * later down in an async invocation of {@link #process()} (which currently has
+ * no means of reporting back issues once started).
+ */
+public abstract class TableEventHandler extends EventHandler {
+  private static final Log LOG = LogFactory.getLog(TableEventHandler.class);
+  protected final MasterServices masterServices;
+  protected final byte [] tableName;
+  protected final String tableNameStr;
+  protected boolean isEventBeingHandled = false;
+
+  public TableEventHandler(EventType eventType, byte [] tableName, Server server,
+      MasterServices masterServices)
+  throws IOException {
+    super(server, eventType);
+    this.masterServices = masterServices;
+    this.tableName = tableName;
+    try {
+      this.masterServices.checkTableModifiable(tableName);
+    } catch (TableNotDisabledException ex)  {
+      if (isOnlineSchemaChangeAllowed()
+          && eventType.isOnlineSchemaChangeSupported()) {
+        LOG.debug("Ignoring table not disabled exception " +
+            "for supporting online schema changes.");
+      }	else {
+        throw ex;
+      }
+    }
+    this.tableNameStr = Bytes.toString(this.tableName);
+  }
+
+  private boolean isOnlineSchemaChangeAllowed() {
+    return this.server.getConfiguration().getBoolean(
+      "hbase.online.schema.update.enable", false);
+  }
+
+  @Override
+  public void process() {
+    try {
+      LOG.info("Handling table operation " + eventType + " on table " +
+          Bytes.toString(tableName));
+      List hris =
+        MetaReader.getTableRegions(this.server.getCatalogTracker(),
+          tableName);
+      handleTableOperation(hris);
+      if (eventType.isOnlineSchemaChangeSupported() && this.masterServices.
+          getAssignmentManager().getZKTable().
+          isEnabledTable(Bytes.toString(tableName))) {
+        if (reOpenAllRegions(hris)) {
+          LOG.info("Completed table operation " + eventType + " on table " +
+              Bytes.toString(tableName));
+        } else {
+          LOG.warn("Error on reopening the regions");
+        }
+      }
+      completed(null);
+    } catch (IOException e) {
+      LOG.error("Error manipulating table " + Bytes.toString(tableName), e);
+      completed(e);
+    } catch (KeeperException e) {
+      LOG.error("Error manipulating table " + Bytes.toString(tableName), e);
+      completed(e);
+    } finally {
+      notifyEventBeingHandled();
+    }
+  }
+
+  /**
+   * Called after that process() is completed.
+   * @param exception null if process() is successful or not null if something has failed.
+   */
+  protected void completed(final Throwable exception) {
+  }
+
+  public boolean reOpenAllRegions(List regions) throws IOException {
+    boolean done = false;
+    HTable table = null;
+    TreeMap> serverToRegions = Maps.newTreeMap();
+    NavigableMap hriHserverMapping;
+
+    LOG.info("Bucketing regions by region server...");
+
+    try {
+      table = new HTable(masterServices.getConfiguration(), tableName);
+      hriHserverMapping = table.getRegionLocations();
+    } finally {
+      if (table != null) {
+        table.close();
+      }
+    }
+    List reRegions = new ArrayList();
+    for (HRegionInfo hri : regions) {
+      ServerName rsLocation = hriHserverMapping.get(hri);
+
+      // Skip the offlined split parent region
+      // See HBASE-4578 for more information.
+      if (null == rsLocation) {
+        LOG.info("Skip " + hri);
+        continue;
+      }
+      if (!serverToRegions.containsKey(rsLocation)) {
+        LinkedList hriList = Lists.newLinkedList();
+        serverToRegions.put(rsLocation, hriList);
+      }
+      reRegions.add(hri);
+      serverToRegions.get(rsLocation).add(hri);
+    }
+    
+    LOG.info("Reopening " + reRegions.size() + " regions on "
+        + serverToRegions.size() + " region servers.");
+    this.masterServices.getAssignmentManager().setRegionsToReopen(reRegions);
+    notifyEventBeingHandled();
+    BulkReOpen bulkReopen = new BulkReOpen(this.server, serverToRegions,
+        this.masterServices.getAssignmentManager());
+    while (true) {
+      try {
+        if (bulkReopen.bulkReOpen()) {
+          done = true;
+          break;
+        } else {
+          LOG.warn("Timeout before reopening all regions");
+        }
+      } catch (InterruptedException e) {
+        LOG.warn("Reopen was interrupted");
+        // Preserve the interrupt.
+        Thread.currentThread().interrupt();
+        break;
+      }
+    }
+    return done;
+  }
+
+  /**
+   * Gets a TableDescriptor from the masterServices.  Can Throw exceptions.
+   *
+   * @return Table descriptor for this table
+   * @throws TableExistsException
+   * @throws FileNotFoundException
+   * @throws IOException
+   */
+  public HTableDescriptor getTableDescriptor()
+  throws FileNotFoundException, IOException {
+    final String name = Bytes.toString(tableName);
+    HTableDescriptor htd =
+      this.masterServices.getTableDescriptors().get(name);
+    if (htd == null) {
+      throw new IOException("HTableDescriptor missing for " + name);
+    }
+    return htd;
+  }
+
+  byte [] hasColumnFamily(final HTableDescriptor htd, final byte [] cf)
+  throws InvalidFamilyOperationException {
+    if (!htd.hasFamily(cf)) {
+      throw new InvalidFamilyOperationException("Column family '" +
+        Bytes.toString(cf) + "' does not exist");
+    }
+    return cf;
+  }
+
+  protected abstract void handleTableOperation(List regions)
+  throws IOException, KeeperException;
+
+  /**
+   * Table modifications are processed asynchronously, but provide an API for you to query their
+   * status.
+   * @throws IOException
+   */
+  public synchronized void waitForEventBeingHandled() throws IOException {
+    if (!this.isEventBeingHandled) {
+      try {
+        wait();
+      } catch (InterruptedException ie) {
+        throw (IOException) new InterruptedIOException().initCause(ie);
+      }
+    }
+  }
+
+  private synchronized void notifyEventBeingHandled() {
+    if (!this.isEventBeingHandled) {
+      isEventBeingHandled = true;
+      notify();
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
new file mode 100644
index 0000000..ecb760b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
@@ -0,0 +1,79 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.InvalidFamilyOperationException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Handles adding a new family to an existing table.
+ */
+public class TableModifyFamilyHandler extends TableEventHandler {
+  private final HColumnDescriptor familyDesc;
+
+  public TableModifyFamilyHandler(byte[] tableName,
+      HColumnDescriptor familyDesc, Server server,
+      final MasterServices masterServices) throws IOException {
+    super(EventType.C_M_MODIFY_FAMILY, tableName, server, masterServices);
+    HTableDescriptor htd = getTableDescriptor();
+    hasColumnFamily(htd, familyDesc.getName());
+    this.familyDesc = familyDesc;
+  }
+
+  @Override
+  protected void handleTableOperation(List regions) throws IOException {
+    MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost();
+    if (cpHost != null) {
+      cpHost.preModifyColumnHandler(this.tableName, this.familyDesc);
+    }
+    // Update table descriptor in HDFS
+    HTableDescriptor htd =
+      this.masterServices.getMasterFileSystem().modifyColumn(tableName, familyDesc);
+    // Update in-memory descriptor cache
+    this.masterServices.getTableDescriptors().add(htd);
+    if (cpHost != null) {
+      cpHost.postModifyColumnHandler(this.tableName, this.familyDesc);
+    }
+  }
+
+  @Override
+  public String toString() {
+    String name = "UnknownServerName";
+    if(server != null && server.getServerName() != null) {
+      name = server.getServerName().toString();
+    }
+    String family = "UnknownFamily";
+    if(familyDesc != null) {
+      family = familyDesc.getNameAsString();
+    }
+    return getClass().getSimpleName() + "-" + name + "-" + getSeqid() + "-" + tableNameStr + "-" + family;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java
new file mode 100644
index 0000000..d08f649
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java
@@ -0,0 +1,36 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.handler;
+
+import java.beans.EventHandler;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+
+/**
+ * Implementors tote an HRegionInfo instance.
+ * This is a marker interface that can be put on {@link EventHandler}s that
+ * have an {@link HRegionInfo}.
+ */
+public interface TotesHRegionInfo {
+  /**
+   * @return HRegionInfo instance.
+   */
+  public HRegionInfo getHRegionInfo();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java b/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
new file mode 100644
index 0000000..ccb8a38
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.metrics;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.metrics.HBaseInfo;
+import org.apache.hadoop.hbase.metrics.MetricsRate;
+import org.apache.hadoop.hbase.metrics.PersistentMetricsTimeVaryingRate;
+import org.apache.hadoop.metrics.ContextFactory;
+import org.apache.hadoop.metrics.MetricsContext;
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.MetricsUtil;
+import org.apache.hadoop.metrics.Updater;
+import org.apache.hadoop.metrics.jvm.JvmMetrics;
+import org.apache.hadoop.metrics.util.MetricsLongValue;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+
+
+/**
+ * This class is for maintaining the various master statistics
+ * and publishing them through the metrics interfaces.
+ * 
        
+ * This class has a number of metrics variables that are publicly accessible;
+ * these variables (objects) have methods to update their values.
+ */
+public class MasterMetrics implements Updater {
+  private final Log LOG = LogFactory.getLog(this.getClass());
+  private final MetricsRecord metricsRecord;
+  private final MetricsRegistry registry = new MetricsRegistry();
+  private final MasterStatistics masterStatistics;
+
+  private long lastUpdate = System.currentTimeMillis();
+  private long lastExtUpdate = System.currentTimeMillis();
+  private long extendedPeriod = 0;
+
+  /*
+   * Count of requests to the cluster since last call to metrics update
+   */
+  private final MetricsRate cluster_requests =
+    new MetricsRate("cluster_requests", registry);
+
+  /** Time it takes to finish HLog.splitLog() */
+  final PersistentMetricsTimeVaryingRate splitTime =
+    new PersistentMetricsTimeVaryingRate("splitTime", registry);
+
+  /** Size of HLog files being split */
+  final PersistentMetricsTimeVaryingRate splitSize =
+    new PersistentMetricsTimeVaryingRate("splitSize", registry);
+
+  /** Time it takes to finish snapshot() */
+  final PersistentMetricsTimeVaryingRate snapshotTime =
+    new PersistentMetricsTimeVaryingRate("snapshotTime", registry);
+
+  /** Time it takes to finish restoreSnapshot() */
+  final PersistentMetricsTimeVaryingRate snapshotRestoreTime =
+    new PersistentMetricsTimeVaryingRate("snapshotRestoreTime", registry);
+
+  /** Time it takes to finish cloneSnapshotTime() */
+  final PersistentMetricsTimeVaryingRate snapshotCloneTime =
+    new PersistentMetricsTimeVaryingRate("snapshotCloneTime", registry);
+
+  public MasterMetrics(final String name) {
+    MetricsContext context = MetricsUtil.getContext("hbase");
+    metricsRecord = MetricsUtil.createRecord(context, "master");
+    metricsRecord.setTag("Master", name);
+    context.registerUpdater(this);
+    JvmMetrics.init("Master", name);
+    HBaseInfo.init();
+
+    // expose the MBean for metrics
+    masterStatistics = new MasterStatistics(this.registry);
+
+    // get custom attributes
+    try {
+      Object m = 
+        ContextFactory.getFactory().getAttribute("hbase.extendedperiod");
+      if (m instanceof String) {
+        this.extendedPeriod = Long.parseLong((String) m)*1000;
+      }
+    } catch (IOException ioe) {
+      LOG.info("Couldn't load ContextFactory for Metrics config info");
+    }
+
+    LOG.info("Initialized");
+  }
+
+  public void shutdown() {
+    if (masterStatistics != null)
+      masterStatistics.shutdown();
+  }
+
+  /**
+   * Since this object is a registered updater, this method will be called
+   * periodically, e.g. every 5 seconds.
+   * @param unused
+   */
+  public void doUpdates(MetricsContext unused) {
+    synchronized (this) {
+      this.lastUpdate = System.currentTimeMillis();
+
+      // has the extended period for long-living stats elapsed?
+      if (this.extendedPeriod > 0 &&
+          this.lastUpdate - this.lastExtUpdate >= this.extendedPeriod) {
+        this.lastExtUpdate = this.lastUpdate;
+        this.splitTime.resetMinMaxAvg();
+        this.splitSize.resetMinMaxAvg();
+        this.resetAllMinMax();
+      }
+
+      this.cluster_requests.pushMetric(metricsRecord);
+      this.splitTime.pushMetric(metricsRecord);
+      this.splitSize.pushMetric(metricsRecord);
+    }
+    this.metricsRecord.update();
+  }
+
+  public void resetAllMinMax() {
+    // Nothing to do
+  }
+  
+  /**
+   * Record a single instance of a split
+   * @param time time that the split took
+   * @param size length of original HLogs that were split
+   */
+  public synchronized void addSplit(long time, long size) {
+    splitTime.inc(time);
+    splitSize.inc(size);
+  }
+
+  /**
+   * @return Count of requests.
+   */
+  public float getRequests() {
+    return this.cluster_requests.getPreviousIntervalValue();
+  }
+
+  /**
+   * @param inc How much to add to requests.
+   */
+  public void incrementRequests(final int inc) {
+    this.cluster_requests.inc(inc);
+  }
+
+  /**
+   * Record a single instance of a snapshot
+   * @param time time that the snapshot took
+   */
+  public void addSnapshot(long time) {
+    snapshotTime.inc(time);
+  }
+
+  /**
+   * Record a single instance of a snapshot
+   * @param time time that the snapshot restore took
+   */
+  public void addSnapshotRestore(long time) {
+    snapshotRestoreTime.inc(time);
+  }
+
+  /**
+   * Record a single instance of a snapshot cloned table
+   * @param time time that the snapshot clone took
+   */
+  public void addSnapshotClone(long time) {
+    snapshotCloneTime.inc(time);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterStatistics.java b/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterStatistics.java
new file mode 100644
index 0000000..d885348
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterStatistics.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.metrics;
+
+import javax.management.ObjectName;
+
+import org.apache.hadoop.hbase.metrics.MetricsMBeanBase;
+import org.apache.hadoop.metrics.util.MBeanUtil;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+
+/**
+ * Exports the {@link MasterMetrics} statistics as an MBean
+ * for JMX.
+ */
+public class MasterStatistics extends MetricsMBeanBase {
+  private final ObjectName mbeanName;
+
+  public MasterStatistics(MetricsRegistry registry) {
+    super(registry, "MasterStatistics");
+    mbeanName = MBeanUtil.registerMBean("Master", "MasterStatistics", this);
+  }
+
+  public void shutdown() {
+    if (mbeanName != null)
+      MBeanUtil.unregisterMBean(mbeanName);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
new file mode 100644
index 0000000..bd0349f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
@@ -0,0 +1,182 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CancellationException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Handler to Clone a snapshot.
+ *
+ * 
        Uses {@link RestoreSnapshotHelper} to create a new table with the same
+ * content of the specified snapshot.
+ */
+@InterfaceAudience.Private
+public class CloneSnapshotHandler extends CreateTableHandler implements SnapshotSentinel {
+  private static final Log LOG = LogFactory.getLog(CloneSnapshotHandler.class);
+
+  private final static String NAME = "Master CloneSnapshotHandler";
+
+  private final SnapshotDescription snapshot;
+
+  private final ForeignExceptionDispatcher monitor;
+  private final MasterMetrics metricsMaster;
+  private final MonitoredTask status;
+
+  private volatile boolean stopped = false;
+
+  public CloneSnapshotHandler(final MasterServices masterServices,
+      final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor,
+      final MasterMetrics metricsMaster)
+      throws NotAllMetaRegionsOnlineException, TableExistsException, IOException {
+    super(masterServices, masterServices.getMasterFileSystem(),
+      masterServices.getServerManager(), hTableDescriptor,
+      masterServices.getConfiguration(), null, masterServices.getCatalogTracker(),
+      masterServices.getAssignmentManager());
+    this.metricsMaster = metricsMaster;
+
+    // Snapshot information
+    this.snapshot = snapshot;
+
+    // Monitor
+    this.monitor = new ForeignExceptionDispatcher();
+    this.status = TaskMonitor.get().createStatus("Cloning  snapshot '" + snapshot.getName() +
+      "' to table " + hTableDescriptor.getNameAsString());
+  }
+
+  /**
+   * Create the on-disk regions, using the tableRootDir provided by the CreateTableHandler.
+   * The cloned table will be created in a temp directory, and then the CreateTableHandler
+   * will be responsible to add the regions returned by this method to META and do the assignment.
+   */
+  @Override
+  protected List handleCreateHdfsRegions(final Path tableRootDir,
+      final String tableName) throws IOException {
+    status.setStatus("Creating regions for table: " + tableName);
+    FileSystem fs = fileSystemManager.getFileSystem();
+    Path rootDir = fileSystemManager.getRootDir();
+    Path tableDir = new Path(tableRootDir, tableName);
+
+    try {
+      // 1. Execute the on-disk Clone
+      Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+      RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(conf, fs,
+          snapshot, snapshotDir, hTableDescriptor, tableDir, monitor, status);
+      RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
+
+      // Clone operation should not have stuff to restore or remove
+      Preconditions.checkArgument(!metaChanges.hasRegionsToRestore(),
+          "A clone should not have regions to restore");
+      Preconditions.checkArgument(!metaChanges.hasRegionsToRemove(),
+          "A clone should not have regions to remove");
+
+      // At this point the clone is complete. Next step is enabling the table.
+      String msg = "Clone snapshot=" + snapshot.getName() +" on table=" + tableName + " completed!";
+      LOG.info(msg);
+      status.setStatus(msg + " Waiting for table to be enabled...");
+
+      // 2. let the CreateTableHandler add the regions to meta
+      return metaChanges.getRegionsToAdd();
+    } catch (Exception e) {
+      String msg = "clone snapshot=" + SnapshotDescriptionUtils.toString(snapshot) + " failed";
+      LOG.error(msg, e);
+      IOException rse = new RestoreSnapshotException(msg, e, snapshot);
+
+      // these handlers aren't futures so we need to register the error here.
+      this.monitor.receive(new ForeignException(NAME, rse));
+      throw rse;
+    }
+  }
+
+  @Override
+  protected void completed(final Throwable exception) {
+    this.stopped = true;
+    if (exception != null) {
+     status.abort("Snapshot '" + snapshot.getName() + "' clone failed because " +
+        exception.getMessage());
+    } else {
+      status.markComplete("Snapshot '"+ snapshot.getName() +"' clone completed and table enabled!");
+    }
+    metricsMaster.addSnapshotClone(status.getCompletionTimestamp() - status.getStartTime());
+    super.completed(exception);
+  }
+
+  @Override
+  public boolean isFinished() {
+    return this.stopped;
+  }
+
+  @Override
+  public long getCompletionTimestamp() {
+    return this.status.getCompletionTimestamp();
+  }
+
+  @Override
+  public SnapshotDescription getSnapshot() {
+    return snapshot;
+  }
+
+  @Override
+  public void cancel(String why) {
+    if (this.stopped) return;
+    this.stopped = true;
+    String msg = "Stopping clone snapshot=" + snapshot + " because: " + why;
+    LOG.info(msg);
+    status.abort(msg);
+    this.monitor.receive(new ForeignException(NAME, new CancellationException(why)));
+  }
+
+  @Override
+  public ForeignException getExceptionIfFailed() {
+    return this.monitor.getException();
+  }
+
+  @Override
+  public void rethrowExceptionIfFailed() throws ForeignException {
+    monitor.rethrowException();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
new file mode 100644
index 0000000..eee4548
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.snapshot.CopyRecoveredEditsTask;
+import org.apache.hadoop.hbase.snapshot.ReferenceRegionHFilesTask;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Take a snapshot of a disabled table.
+ * 
        
+ * Table must exist when taking the snapshot, or results are undefined.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class DisabledTableSnapshotHandler extends TakeSnapshotHandler {
+  private static final Log LOG = LogFactory.getLog(DisabledTableSnapshotHandler.class);
+  private final TimeoutExceptionInjector timeoutInjector;
+
+  /**
+   * @param snapshot descriptor of the snapshot to take
+   * @param masterServices master services provider
+   * @throws IOException on unexpected error
+   */
+  public DisabledTableSnapshotHandler(SnapshotDescription snapshot,
+      final MasterServices masterServices, final MasterMetrics metricsMaster) {
+    super(snapshot, masterServices, metricsMaster);
+
+    // setup the timer
+    timeoutInjector = TakeSnapshotUtils.getMasterTimerAndBindToMonitor(snapshot, conf, monitor);
+  }
+
+  // TODO consider parallelizing these operations since they are independent. Right now its just
+  // easier to keep them serial though
+  @Override
+  public void snapshotRegions(List> regionsAndLocations)
+      throws IOException, KeeperException {
+    try {
+      timeoutInjector.start();
+
+      // 1. get all the regions hosting this table.
+
+      // extract each pair to separate lists
+      Set serverNames = new HashSet();
+      Set regions = new HashSet();
+      for (Pair p : regionsAndLocations) {
+        regions.add(p.getFirst());
+        serverNames.add(p.getSecond().toString());
+      }
+
+      // 2. for each region, write all the info to disk
+      LOG.info("Starting to write region info and WALs for regions for offline snapshot:"
+          + SnapshotDescriptionUtils.toString(snapshot));
+      for (HRegionInfo regionInfo : regions) {
+        // 2.1 copy the regionInfo files to the snapshot
+        Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(snapshot, rootDir,
+          regionInfo.getEncodedName());
+        HRegion.writeRegioninfoOnFilesystem(regionInfo, snapshotRegionDir, fs, conf);
+        // check for error for each region
+        monitor.rethrowException();
+
+        // 2.2 for each region, copy over its recovered.edits directory
+        Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
+        new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call();
+        monitor.rethrowException();
+        status.setStatus("Completed copying recovered edits for offline snapshot of table: "
+            + snapshot.getTable());
+
+        // 2.3 reference all the files in the region
+        new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call();
+        monitor.rethrowException();
+        status.setStatus("Completed referencing HFiles for offline snapshot of table: " +
+          snapshot.getTable());
+      }
+
+      // 3. write the table info to disk
+      LOG.info("Starting to copy tableinfo for offline snapshot: " +
+      SnapshotDescriptionUtils.toString(snapshot));
+      TableInfoCopyTask tableInfoCopyTask = new TableInfoCopyTask(this.monitor, snapshot, fs,
+          FSUtils.getRootDir(conf));
+      tableInfoCopyTask.call();
+      monitor.rethrowException();
+      status.setStatus("Finished copying tableinfo for snapshot of table: " + snapshot.getTable());
+    } catch (Exception e) {
+      // make sure we capture the exception to propagate back to the client later
+      String reason = "Failed snapshot " + SnapshotDescriptionUtils.toString(snapshot)
+          + " due to exception:" + e.getMessage();
+      ForeignException ee = new ForeignException(reason, e);
+      monitor.receive(ee);
+      status.abort("Snapshot of table: "+ snapshot.getTable() +" failed because " + e.getMessage());
+    } finally {
+      LOG.debug("Marking snapshot" + SnapshotDescriptionUtils.toString(snapshot)
+          + " as finished.");
+
+      // 6. mark the timer as finished - even if we got an exception, we don't need to time the
+      // operation any further
+      timeoutInjector.complete();
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
new file mode 100644
index 0000000..4bbdef3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.procedure.Procedure;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
+import org.apache.hadoop.hbase.util.Pair;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Handle the master side of taking a snapshot of an online table, regardless of snapshot type.
+ * Uses a {@link Procedure} to run the snapshot across all the involved region servers.
+ * @see ProcedureCoordinator
+ */
+@InterfaceAudience.Private
+public class EnabledTableSnapshotHandler extends TakeSnapshotHandler {
+
+  private static final Log LOG = LogFactory.getLog(EnabledTableSnapshotHandler.class);
+  private final ProcedureCoordinator coordinator;
+
+  public EnabledTableSnapshotHandler(SnapshotDescription snapshot, MasterServices master,
+      final SnapshotManager manager, final MasterMetrics metricsMaster) {
+    super(snapshot, master, metricsMaster);
+    this.coordinator = manager.getCoordinator();
+  }
+
+  // TODO consider switching over to using regionnames, rather than server names. This would allow
+  // regions to migrate during a snapshot, and then be involved when they are ready. Still want to
+  // enforce a snapshot time constraints, but lets us be potentially a bit more robust.
+
+  /**
+   * This method kicks off a snapshot procedure.  Other than that it hangs around for various
+   * phases to complete.
+   */
+  @Override
+  protected void snapshotRegions(List> regions)
+      throws HBaseSnapshotException {
+    Set regionServers = new HashSet(regions.size());
+    for (Pair region : regions) {
+      regionServers.add(region.getSecond().toString());
+    }
+
+    // start the snapshot on the RS
+    Procedure proc = coordinator.startProcedure(this.monitor, this.snapshot.getName(),
+      this.snapshot.toByteArray(), Lists.newArrayList(regionServers));
+    if (proc == null) {
+      String msg = "Failed to submit distributed procedure for snapshot '"
+          + snapshot.getName() + "'";
+      LOG.error(msg);
+      throw new HBaseSnapshotException(msg);
+    }
+
+    try {
+      // wait for the snapshot to complete.  A timer thread is kicked off that should cancel this
+      // if it takes too long.
+      proc.waitForCompleted();
+      LOG.info("Done waiting - snapshot for " + this.snapshot.getName() + " finished!");
+    } catch (InterruptedException e) {
+      ForeignException ee =
+          new ForeignException("Interrupted while waiting for snapshot to finish", e);
+      monitor.receive(ee);
+      Thread.currentThread().interrupt();
+    } catch (ForeignException e) {
+      monitor.receive(e);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
new file mode 100644
index 0000000..db89e66
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
@@ -0,0 +1,249 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+
+/**
+ * General snapshot verification on the master.
+ * 
        
+ * This is a light-weight verification mechanism for all the files in a snapshot. It doesn't
+ * attempt to verify that the files are exact copies (that would be paramount to taking the
+ * snapshot again!), but instead just attempts to ensure that the files match the expected
+ * files and are the same length.
+ * 
        
+ * Taking an online snapshots can race against other operations and this is an last line of
+ * defense.  For example, if meta changes between when snapshots are taken not all regions of a
+ * table may be present.  This can be caused by a region split (daughters present on this scan,
+ * but snapshot took parent), or move (snapshots only checks lists of region servers, a move could
+ * have caused a region to be skipped or done twice).
+ * 
        
+ * Current snapshot files checked:
+ * 
          
+ * 
        1. SnapshotDescription is readable
          2. 
+ * 
        2. Table info is readable
          3. 
+ * 
        3. Regions
          4. 
+ * 
            
+ * 
          • Matching regions in the snapshot as currently in the table
            • 
+ * 
          • {@link HRegionInfo} matches the current and stored regions
            • 
+ * 
          • All referenced hfiles have valid names
            • 
+ * 
          • All the hfiles are present (either in .archive directory in the region)
            • 
+ * 
          • All recovered.edits files are present (by name) and have the correct file size
            • 
+ * 
          
+ * 
        
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class MasterSnapshotVerifier {
+
+  private SnapshotDescription snapshot;
+  private FileSystem fs;
+  private Path rootDir;
+  private String tableName;
+  private MasterServices services;
+
+  /**
+   * @param services services for the master
+   * @param snapshot snapshot to check
+   * @param rootDir root directory of the hbase installation.
+   */
+  public MasterSnapshotVerifier(MasterServices services, SnapshotDescription snapshot, Path rootDir) {
+    this.fs = services.getMasterFileSystem().getFileSystem();
+    this.services = services;
+    this.snapshot = snapshot;
+    this.rootDir = rootDir;
+    this.tableName = snapshot.getTable();
+  }
+
+  /**
+   * Verify that the snapshot in the directory is a valid snapshot
+   * @param snapshotDir snapshot directory to check
+   * @param snapshotServers {@link ServerName} of the servers that are involved in the snapshot
+   * @throws CorruptedSnapshotException if the snapshot is invalid
+   * @throws IOException if there is an unexpected connection issue to the filesystem
+   */
+  public void verifySnapshot(Path snapshotDir, Set snapshotServers)
+      throws CorruptedSnapshotException, IOException {
+    // verify snapshot info matches
+    verifySnapshotDescription(snapshotDir);
+
+    // check that tableinfo is a valid table description
+    verifyTableInfo(snapshotDir);
+
+    // check that each region is valid
+    verifyRegions(snapshotDir);
+  }
+
+  /**
+   * Check that the snapshot description written in the filesystem matches the current snapshot
+   * @param snapshotDir snapshot directory to check
+   */
+  private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException {
+    SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+    if (!this.snapshot.equals(found)) {
+      throw new CorruptedSnapshotException("Snapshot read (" + found
+          + ") doesn't equal snapshot we ran (" + snapshot + ").", snapshot);
+    }
+  }
+
+  /**
+   * Check that the table descriptor for the snapshot is a valid table descriptor
+   * @param snapshotDir snapshot directory to check
+   */
+  private void verifyTableInfo(Path snapshotDir) throws IOException {
+    FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
+  }
+
+  /**
+   * Check that all the regions in the snapshot are valid, and accounted for.
+   * @param snapshotDir snapshot directory to check
+   * @throws IOException if we can't reach .META. or read the files from the FS
+   */
+  private void verifyRegions(Path snapshotDir) throws IOException {
+    List regions = MetaReader.getTableRegions(this.services.getCatalogTracker(),
+      Bytes.toBytes(tableName));
+    for (HRegionInfo region : regions) {
+      // if offline split parent, skip it
+      if (region.isOffline() && (region.isSplit() || region.isSplitParent())) {
+        continue;
+      }
+
+      verifyRegion(fs, snapshotDir, region);
+    }
+  }
+
+  /**
+   * Verify that the region (regioninfo, hfiles) are valid
+   * @param fs the FileSystem instance
+   * @param snapshotDir snapshot directory to check
+   * @param region the region to check
+   */
+  private void verifyRegion(FileSystem fs, Path snapshotDir, HRegionInfo region) throws IOException {
+    // make sure we have region in the snapshot
+    Path regionDir = new Path(snapshotDir, region.getEncodedName());
+    if (!fs.exists(regionDir)) {
+      // could happen due to a move or split race.
+      throw new CorruptedSnapshotException("No region directory found for region:" + region,
+          snapshot);
+    }
+    // make sure we have the region info in the snapshot
+    Path regionInfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
+    // make sure the file exists
+    if (!fs.exists(regionInfo)) {
+      throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot);
+    }
+    FSDataInputStream in = fs.open(regionInfo);
+    HRegionInfo found = new HRegionInfo();
+    try {
+      found.readFields(in);
+      if (!region.equals(found)) {
+        throw new CorruptedSnapshotException("Found region info (" + found
+           + ") doesn't match expected region:" + region, snapshot);
+      }
+    } finally {
+      in.close();
+    }
+
+    // make sure we have the expected recovered edits files
+    TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot);
+
+    // check for the existance of each hfile
+    PathFilter familiesDirs = new FSUtils.FamilyDirFilter(fs);
+    FileStatus[] columnFamilies = FSUtils.listStatus(fs, regionDir, familiesDirs);
+    // should we do some checking here to make sure the cfs are correct?
+    if (columnFamilies == null) return;
+
+    // setup the suffixes for the snapshot directories
+    Path tableNameSuffix = new Path(tableName);
+    Path regionNameSuffix = new Path(tableNameSuffix, region.getEncodedName());
+
+    // get the potential real paths
+    Path archivedRegion = new Path(HFileArchiveUtil.getArchivePath(services.getConfiguration()),
+        regionNameSuffix);
+    Path realRegion = new Path(rootDir, regionNameSuffix);
+
+    // loop through each cf and check we can find each of the hfiles
+    for (FileStatus cf : columnFamilies) {
+      FileStatus[] hfiles = FSUtils.listStatus(fs, cf.getPath(), null);
+      // should we check if there should be hfiles?
+      if (hfiles == null || hfiles.length == 0) continue;
+
+      Path realCfDir = new Path(realRegion, cf.getPath().getName());
+      Path archivedCfDir = new Path(archivedRegion, cf.getPath().getName());
+      for (FileStatus hfile : hfiles) {
+        // make sure the name is correct
+        if (!StoreFile.validateStoreFileName(hfile.getPath().getName())) {
+          throw new CorruptedSnapshotException("HFile: " + hfile.getPath()
+              + " is not a valid hfile name.", snapshot);
+        }
+
+        // check to see if hfile is present in the real table
+        String fileName = hfile.getPath().getName();
+        Path file = new Path(realCfDir, fileName);
+        Path archived = new Path(archivedCfDir, fileName);
+        if (!fs.exists(file) && !fs.exists(archived)) {
+          throw new CorruptedSnapshotException("Can't find hfile: " + hfile.getPath()
+              + " in the real (" + realCfDir + ") or archive (" + archivedCfDir
+              + ") directory for the primary table.", snapshot);
+        }
+      }
+    }
+  }
+
+  /**
+   * Check that the logs stored in the log directory for the snapshot are valid - it contains all
+   * the expected logs for all servers involved in the snapshot.
+   * @param snapshotDir snapshot directory to check
+   * @param snapshotServers list of the names of servers involved in the snapshot.
+   * @throws CorruptedSnapshotException if the hlogs in the snapshot are not correct
+   * @throws IOException if we can't reach the filesystem
+   */
+  private void verifyLogs(Path snapshotDir, Set snapshotServers)
+      throws CorruptedSnapshotException, IOException {
+    Path snapshotLogDir = new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME);
+    Path logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
+    TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir, snapshotServers, snapshot,
+      snapshotLogDir);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
new file mode 100644
index 0000000..1660972
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
@@ -0,0 +1,191 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CancellationException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.handler.TableEventHandler;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Handler to Restore a snapshot.
+ *
+ * 
        Uses {@link RestoreSnapshotHelper} to replace the table content with the
+ * data available in the snapshot.
+ */
+@InterfaceAudience.Private
+public class RestoreSnapshotHandler extends TableEventHandler implements SnapshotSentinel {
+  private static final Log LOG = LogFactory.getLog(RestoreSnapshotHandler.class);
+
+  private final HTableDescriptor hTableDescriptor;
+  private final SnapshotDescription snapshot;
+
+  private final ForeignExceptionDispatcher monitor;
+  private final MasterMetrics metricsMaster;
+  private final MonitoredTask status;
+
+  private volatile boolean stopped = false;
+
+  public RestoreSnapshotHandler(final MasterServices masterServices,
+      final SnapshotDescription snapshot, final HTableDescriptor htd,
+      final MasterMetrics metricsMaster) throws IOException {
+    super(EventType.C_M_RESTORE_SNAPSHOT, htd.getName(), masterServices, masterServices);
+    this.metricsMaster = metricsMaster;
+
+    // Snapshot information
+    this.snapshot = snapshot;
+
+    // Monitor
+    this.monitor = new ForeignExceptionDispatcher();
+
+    // Check table exists.
+    getTableDescriptor();
+
+    // This is the new schema we are going to write out as this modification.
+    this.hTableDescriptor = htd;
+
+    this.status = TaskMonitor.get().createStatus(
+      "Restoring  snapshot '" + snapshot.getName() + "' to table "
+          + hTableDescriptor.getNameAsString());
+  }
+
+  /**
+   * The restore table is executed in place.
+   *  - The on-disk data will be restored - reference files are put in place without moving data
+   *  -  [if something fail here: you need to delete the table and re-run the restore]
+   *  - META will be updated
+   *  -  [if something fail here: you need to run hbck to fix META entries]
+   * The passed in list gets changed in this method
+   */
+  @Override
+  protected void handleTableOperation(List hris) throws IOException {
+    MasterFileSystem fileSystemManager = masterServices.getMasterFileSystem();
+    CatalogTracker catalogTracker = masterServices.getCatalogTracker();
+    FileSystem fs = fileSystemManager.getFileSystem();
+    Path rootDir = fileSystemManager.getRootDir();
+    byte[] tableName = hTableDescriptor.getName();
+    Path tableDir = HTableDescriptor.getTableDir(rootDir, tableName);
+
+    try {
+      // 1. Update descriptor
+      this.masterServices.getTableDescriptors().add(hTableDescriptor);
+
+      // 2. Execute the on-disk Restore
+      LOG.debug("Starting restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot));
+      Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+      RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
+          masterServices.getConfiguration(), fs,
+          snapshot, snapshotDir, hTableDescriptor, tableDir, monitor, status);
+      RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
+
+      // 3. Applies changes to .META.
+      hris.clear();
+      status.setStatus("Preparing to restore each region");
+      if (metaChanges.hasRegionsToAdd()) hris.addAll(metaChanges.getRegionsToAdd());
+      if (metaChanges.hasRegionsToRestore()) hris.addAll(metaChanges.getRegionsToRestore());
+      List hrisToRemove = metaChanges.getRegionsToRemove();
+      MetaEditor.mutateRegions(catalogTracker, hrisToRemove, hris);
+
+      // At this point the restore is complete. Next step is enabling the table.
+      LOG.info("Restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot) + " on table=" +
+        Bytes.toString(tableName) + " completed!");
+    } catch (IOException e) {
+      String msg = "restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot)
+          + " failed. Try re-running the restore command.";
+      LOG.error(msg, e);
+      monitor.receive(new ForeignException(masterServices.getServerName().toString(), e));
+      throw new RestoreSnapshotException(msg, e);
+    } finally {
+      this.stopped = true;
+    }
+  }
+
+  @Override
+  protected void completed(final Throwable exception) {
+    this.stopped = true;
+    if (exception != null) {
+      status.abort("Restore snapshot '" + snapshot.getName() + "' failed because " +
+          exception.getMessage());
+    } else {
+      status.markComplete("Restore snapshot '"+ snapshot.getName() +"' completed!");
+    }
+    metricsMaster.addSnapshotRestore(status.getCompletionTimestamp() - status.getStartTime());
+    super.completed(exception);
+  }
+
+  @Override
+  public boolean isFinished() {
+    return this.stopped;
+  }
+
+  @Override
+  public long getCompletionTimestamp() {
+    return this.status.getCompletionTimestamp();
+  }
+
+  @Override
+  public SnapshotDescription getSnapshot() {
+    return snapshot;
+  }
+
+  @Override
+  public void cancel(String why) {
+    if (this.stopped) return;
+    this.stopped = true;
+    String msg = "Stopping restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot)
+        + " because: " + why;
+    LOG.info(msg);
+    CancellationException ce = new CancellationException(why);
+    this.monitor.receive(new ForeignException(masterServices.getServerName().toString(), ce));
+  }
+
+  @Override
+  public ForeignException getExceptionIfFailed() {
+    return this.monitor.getException();
+  }
+
+  @Override
+  public void rethrowExceptionIfFailed() throws ForeignException {
+    monitor.rethrowException();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
new file mode 100644
index 0000000..1385741
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -0,0 +1,312 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Intelligently keep track of all the files for all the snapshots.
+ * 
        
+ * A cache of files is kept to avoid querying the {@link FileSystem} frequently. If there is a cache
+ * miss the directory modification time is used to ensure that we don't rescan directories that we
+ * already have in cache. We only check the modification times of the snapshot directories
+ * (/hbase/.snapshot/[snapshot_name]) to determine if the files need to be loaded into the cache.
+ * 
        
+ * New snapshots will be added to the cache and deleted snapshots will be removed when we refresh
+ * the cache. If the files underneath a snapshot directory are changed, but not the snapshot itself,
+ * we will ignore updates to that snapshot's files.
+ * 
        
+ * This is sufficient because each snapshot has its own directory and is added via an atomic rename
+ * once, when the snapshot is created. We don't need to worry about the data in the snapshot
+ * being run.
+ * 
        
+ * Further, the cache is periodically refreshed ensure that files in snapshots that were deleted are
+ * also removed from the cache.
+ * 
        
+ * A SnapshotFileInspector must be passed when creating this to allow extraction
+ * of files under the /hbase/.snapshot/[snapshot name] directory, for each snapshot.
+ * This allows you to only cache files under, for instance, all the logs in the .logs directory or
+ * all the files under all the regions.
+ * 
        
+ * this also considers all running snapshots (those under /hbase/.snapshot/.tmp) as valid
+ * snapshots and will attempt to cache files from those snapshots as well.
+ * 
        
+ * Queries about a given file are thread-safe with respect to multiple queries and cache refreshes.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SnapshotFileCache implements Stoppable {
+  interface SnapshotFileInspector {
+    /**
+     * Returns a collection of file names needed by the snapshot.
+     * @param snapshotDir {@link Path} to the snapshot directory to scan.
+     * @return the collection of file names needed by the snapshot.
+     */
+    Collection filesUnderSnapshot(final Path snapshotDir) throws IOException;
+  }
+
+  private static final Log LOG = LogFactory.getLog(SnapshotFileCache.class);
+  private volatile boolean stop = false;
+  private final FileSystem fs;
+  private final SnapshotFileInspector fileInspector;
+  private final Path snapshotDir;
+  private final Set cache = new HashSet();
+  /**
+   * This is a helper map of information about the snapshot directories so we don't need to rescan
+   * them if they haven't changed since the last time we looked.
+   */
+  private final Map snapshots =
+      new HashMap();
+  private final Timer refreshTimer;
+
+  private long lastModifiedTime = Long.MIN_VALUE;
+
+  /**
+   * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
+   * filesystem.
+   * 
        
+   * Immediately loads the file cache.
+   * @param conf to extract the configured {@link FileSystem} where the snapshots are stored and
+   *          hbase root directory
+   * @param cacheRefreshPeriod frequency (ms) with which the cache should be refreshed
+   * @param refreshThreadName name of the cache refresh thread
+   * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
+   * @throws IOException if the {@link FileSystem} or root directory cannot be loaded
+   */
+  public SnapshotFileCache(Configuration conf, long cacheRefreshPeriod, String refreshThreadName,
+      SnapshotFileInspector inspectSnapshotFiles) throws IOException {
+    this(FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), 0, cacheRefreshPeriod,
+        refreshThreadName, inspectSnapshotFiles);
+  }
+
+  /**
+   * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
+   * filesystem
+   * @param fs {@link FileSystem} where the snapshots are stored
+   * @param rootDir hbase root directory
+   * @param cacheRefreshPeriod period (ms) with which the cache should be refreshed
+   * @param cacheRefreshDelay amount of time to wait for the cache to be refreshed
+   * @param refreshThreadName name of the cache refresh thread
+   * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
+   */
+  public SnapshotFileCache(FileSystem fs, Path rootDir, long cacheRefreshPeriod,
+      long cacheRefreshDelay, String refreshThreadName, SnapshotFileInspector inspectSnapshotFiles) {
+    this.fs = fs;
+    this.fileInspector = inspectSnapshotFiles;
+    this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+    // periodically refresh the file cache to make sure we aren't superfluously saving files.
+    this.refreshTimer = new Timer(refreshThreadName, true);
+    this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay,
+      cacheRefreshPeriod);
+  }
+
+  /**
+   * Trigger a cache refresh, even if its before the next cache refresh. Does not affect pending
+   * cache refreshes.
+   * 
        
+   * Blocks until the cache is refreshed.
+   * 
        
+   * Exposed for TESTING.
+   */
+  public void triggerCacheRefreshForTesting() {
+    try {
+      SnapshotFileCache.this.refreshCache();
+    } catch (IOException e) {
+      LOG.warn("Failed to refresh snapshot hfile cache!", e);
+    }
+    LOG.debug("Current cache:" + cache);
+  }
+
+  /**
+   * Check to see if the passed file name is contained in any of the snapshots. First checks an
+   * in-memory cache of the files to keep. If its not in the cache, then the cache is refreshed and
+   * the cache checked again for that file. This ensures that we always return true for a
+   * files that exists.
+   * 
        
+   * Note this may lead to periodic false positives for the file being referenced. Periodically, the
+   * cache is refreshed even if there are no requests to ensure that the false negatives get removed
+   * eventually. For instance, suppose you have a file in the snapshot and it gets loaded into the
+   * cache. Then at some point later that snapshot is deleted. If the cache has not been refreshed
+   * at that point, cache will still think the file system contains that file and return
+   * true, even if it is no longer present (false positive). However, if the file never was
+   * on the filesystem, we will never find it and always return false.
+   * @param fileName file to check
+   * @return false if the file is not referenced in any current or running snapshot,
+   *         true if the file is in the cache.
+   * @throws IOException if there is an unexpected error reaching the filesystem.
+   */
+  // XXX this is inefficient to synchronize on the method, when what we really need to guard against
+  // is an illegal access to the cache. Really we could do a mutex-guarded pointer swap on the
+  // cache, but that seems overkill at the moment and isn't necessarily a bottleneck.
+  public synchronized boolean contains(String fileName) throws IOException {
+    if (this.cache.contains(fileName)) return true;
+
+    refreshCache();
+
+    // then check again
+    return this.cache.contains(fileName);
+  }
+
+  private synchronized void refreshCache() throws IOException {
+    // get the status of the snapshots directory
+    FileStatus status;
+    try {
+      status = fs.getFileStatus(snapshotDir);
+    } catch (FileNotFoundException e) {
+      if (this.cache.size() > 0) {
+        LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist");
+      }
+      return;
+    }
+    // if the snapshot directory wasn't modified since we last check, we are done
+    if (status.getModificationTime() <= lastModifiedTime) return;
+
+    // directory was modified, so we need to reload our cache
+    // there could be a slight race here where we miss the cache, check the directory modification
+    // time, then someone updates the directory, causing us to not scan the directory again.
+    // However, snapshot directories are only created once, so this isn't an issue.
+
+    // 1. update the modified time
+    this.lastModifiedTime = status.getModificationTime();
+
+    // 2.clear the cache
+    this.cache.clear();
+    Map known = new HashMap();
+
+    // 3. check each of the snapshot directories
+    FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir);
+    if (snapshots == null) {
+      // remove all the remembered snapshots because we don't have any left
+      if (LOG.isDebugEnabled() && this.snapshots.size() > 0) {
+        LOG.debug("No snapshots on-disk, cache empty");
+      }
+      this.snapshots.clear();
+      return;
+    }
+
+    // 3.1 iterate through the on-disk snapshots
+    for (FileStatus snapshot : snapshots) {
+      String name = snapshot.getPath().getName();
+      // its the tmp dir
+      if (name.equals(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
+        // only add those files to the cache, but not to the known snapshots
+        FileStatus[] running = FSUtils.listStatus(fs, snapshot.getPath());
+        if (running == null) continue;
+        for (FileStatus run : running) {
+          this.cache.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+        }
+      } else {
+        SnapshotDirectoryInfo files = this.snapshots.remove(name);
+        // 3.1.1 if we don't know about the snapshot or its been modified, we need to update the files
+        // the latter could occur where I create a snapshot, then delete it, and then make a new
+        // snapshot with the same name. We will need to update the cache the information from that new
+        // snapshot, even though it has the same name as the files referenced have probably changed.
+        if (files == null || files.hasBeenModified(snapshot.getModificationTime())) {
+          // get all files for the snapshot and create a new info
+          Collection storedFiles = fileInspector.filesUnderSnapshot(snapshot.getPath());
+          files = new SnapshotDirectoryInfo(snapshot.getModificationTime(), storedFiles);
+        }
+        // 3.2 add all the files to cache
+        this.cache.addAll(files.getFiles());
+        known.put(name, files);
+      }
+    }
+
+    // 4. set the snapshots we are tracking
+    this.snapshots.clear();
+    this.snapshots.putAll(known);
+  }
+
+  /**
+   * Simple helper task that just periodically attempts to refresh the cache
+   */
+  public class RefreshCacheTask extends TimerTask {
+    @Override
+    public void run() {
+      try {
+        SnapshotFileCache.this.refreshCache();
+      } catch (IOException e) {
+        LOG.warn("Failed to refresh snapshot hfile cache!", e);
+      }
+    }
+  }
+
+  @Override
+  public void stop(String why) {
+    if (!this.stop) {
+      this.stop = true;
+      this.refreshTimer.cancel();
+    }
+
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.stop;
+  }
+
+  /**
+   * Information about a snapshot directory
+   */
+  private static class SnapshotDirectoryInfo {
+    long lastModified;
+    Collection files;
+
+    public SnapshotDirectoryInfo(long mtime, Collection files) {
+      this.lastModified = mtime;
+      this.files = files;
+    }
+
+    /**
+     * @return the hfiles in the snapshot when this was made.
+     */
+    public Collection getFiles() {
+      return this.files;
+    }
+
+    /**
+     * Check if the snapshot directory has been modified
+     * @param mtime current modification time of the directory
+     * @return true if it the modification time of the directory is newer time when we
+     *         created this
+     */
+    public boolean hasBeenModified(long mtime) {
+      return this.lastModified < mtime;
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
new file mode 100644
index 0000000..7b75fd2
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Implementation of a file cleaner that checks if a hfile is still used by snapshots of HBase
+ * tables.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
+  private static final Log LOG = LogFactory.getLog(SnapshotHFileCleaner.class);
+
+  /**
+   * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in
+   * snapshots (ms)
+   */
+  public static final String HFILE_CACHE_REFRESH_PERIOD_CONF_KEY =
+      "hbase.master.hfilecleaner.plugins.snapshot.period";
+
+  /** Refresh cache, by default, every 5 minutes */
+  private static final long DEFAULT_HFILE_CACHE_REFRESH_PERIOD = 300000;
+
+  /** File cache for HFiles in the completed and currently running snapshots */
+  private SnapshotFileCache cache;
+
+  @Override
+  public synchronized boolean isFileDeletable(Path filePath) {
+    try {
+      return !cache.contains(filePath.getName());
+    } catch (IOException e) {
+      LOG.error("Exception while checking if:" + filePath + " was valid, keeping it just in case.",
+        e);
+      return false;
+    }
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+    try {
+      long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
+        DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
+      final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
+      Path rootDir = FSUtils.getRootDir(conf);
+      cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
+          "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
+            public Collection filesUnderSnapshot(final Path snapshotDir)
+                throws IOException {
+              return SnapshotReferenceUtil.getHFileNames(fs, snapshotDir);
+            }
+          });
+    } catch (IOException e) {
+      LOG.error("Failed to create cleaner util", e);
+    }
+  }
+
+  @Override
+  public void stop(String why) {
+    this.cache.stop(why);
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.cache.isStopped();
+  }
+
+  /**
+   * Exposed for Testing!
+   * @return the cache of all hfiles
+   */
+  public SnapshotFileCache getFileCacheForTesting() {
+    return this.cache;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
new file mode 100644
index 0000000..12af8c4
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Implementation of a log cleaner that checks if a log is still used by
+ * snapshots of HBase tables.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SnapshotLogCleaner extends BaseLogCleanerDelegate {
+  private static final Log LOG = LogFactory.getLog(SnapshotLogCleaner.class);
+
+  /**
+   * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in
+   * snapshots (ms)
+   */
+  static final String HLOG_CACHE_REFRESH_PERIOD_CONF_KEY =
+      "hbase.master.hlogcleaner.plugins.snapshot.period";
+
+  /** Refresh cache, by default, every 5 minutes */
+  private static final long DEFAULT_HLOG_CACHE_REFRESH_PERIOD = 300000;
+
+  private SnapshotFileCache cache;
+
+  @Override
+  public synchronized boolean isLogDeletable(Path filePath) {
+    try {
+      if (null == cache) return false;
+      return !cache.contains(filePath.getName());
+    } catch (IOException e) {
+      LOG.error("Exception while checking if:" + filePath + " was valid, keeping it just in case.",
+        e);
+      return false;
+    }
+  }
+
+  /**
+   * This method should only be called once, as it starts a thread to keep the cache
+   * up-to-date.
+   * 
        
+   * {@inheritDoc}
+   */
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+    try {
+      long cacheRefreshPeriod = conf.getLong(
+        HLOG_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_HLOG_CACHE_REFRESH_PERIOD);
+      final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
+      Path rootDir = FSUtils.getRootDir(conf);
+      cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
+          "snapshot-log-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
+            public Collection filesUnderSnapshot(final Path snapshotDir)
+                throws IOException {
+              return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
+            }
+          });
+    } catch (IOException e) {
+      LOG.error("Failed to create snapshot log cleaner", e);
+    }
+  }
+
+  @Override
+  public void stop(String why) {
+    this.cache.stop(why);
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.cache.isStopped();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
new file mode 100644
index 0000000..439989c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -0,0 +1,974 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
+import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.procedure.Procedure;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
+import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
+import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
+import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
+import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
+import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * This class manages the procedure of taking and restoring snapshots. There is only one
+ * SnapshotManager for the master.
+ * 
        
+ * The class provides methods for monitoring in-progress snapshot actions.
+ * 
        
+ * Note: Currently there can only be one snapshot being taken at a time over the cluster. This is a
+ * simplification in the current implementation.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class SnapshotManager implements Stoppable {
+  private static final Log LOG = LogFactory.getLog(SnapshotManager.class);
+
+  /** By default, check to see if the snapshot is complete every WAKE MILLIS (ms) */
+  private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
+
+  /**
+   * Wait time before removing a finished sentinel from the in-progress map
+   *
+   * NOTE: This is used as a safety auto cleanup.
+   * The snapshot and restore handlers map entries are removed when a user asks if a snapshot or
+   * restore is completed. This operation is part of the HBaseAdmin snapshot/restore API flow.
+   * In case something fails on the client side and the snapshot/restore state is not reclaimed
+   * after a default timeout, the entry is removed from the in-progress map.
+   * At this point, if the user asks for the snapshot/restore status, the result will be
+   * snapshot done if exists or failed if it doesn't exists.
+   */
+  private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
+
+  /** Enable or disable snapshot support */
+  public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
+
+  /**
+   * Conf key for # of ms elapsed between checks for snapshot errors while waiting for
+   * completion.
+   */
+  private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
+
+  /** By default, check to see if the snapshot is complete (ms) */
+  private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 5000;
+
+  /**
+   * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for
+   * completion.
+   */
+  private static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
+
+  /** Name of the operation to use in the controller */
+  public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
+
+  /** Conf key for # of threads used by the SnapshotManager thread pool */
+  private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
+
+  /** number of current operations running on the master */
+  private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
+
+  private boolean stopped;
+  private final long wakeFrequency;
+  private final MasterServices master;  // Needed by TableEventHandlers
+  private final MasterMetrics metricsMaster;
+  private final ProcedureCoordinator coordinator;
+
+  // Is snapshot feature enabled?
+  private boolean isSnapshotSupported = false;
+
+  // Snapshot handlers map, with table name as key.
+  // The map is always accessed and modified under the object lock using synchronized.
+  // snapshotTable() will insert an Handler in the table.
+  // isSnapshotDone() will remove the handler requested if the operation is finished.
+  private Map snapshotHandlers = new HashMap();
+
+  // Restore Sentinels map, with table name as key.
+  // The map is always accessed and modified under the object lock using synchronized.
+  // restoreSnapshot()/cloneSnapshot() will insert an Handler in the table.
+  // isRestoreDone() will remove the handler requested if the operation is finished.
+  private Map restoreHandlers = new HashMap();
+
+  private final Path rootDir;
+  private final ExecutorService executorService;
+
+  /**
+   * Construct a snapshot manager.
+   * @param master
+   */
+  public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster)
+      throws KeeperException, IOException, UnsupportedOperationException {
+    this.master = master;
+    this.metricsMaster = metricsMaster;
+
+    this.rootDir = master.getMasterFileSystem().getRootDir();
+    checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
+
+    // get the configuration for the coordinator
+    Configuration conf = master.getConfiguration();
+    this.wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
+    long keepAliveTime = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
+    int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
+
+    // setup the default procedure coordinator
+    String name = master.getServerName().toString();
+    ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, keepAliveTime, opThreads, wakeFrequency);
+    ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
+        master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
+    this.coordinator = new ProcedureCoordinator(comms, tpool);
+    this.executorService = master.getExecutorService();
+    resetTempDir();
+  }
+
+  /**
+   * Fully specify all necessary components of a snapshot manager. Exposed for testing.
+   * @param master services for the master where the manager is running
+   * @param coordinator procedure coordinator instance.  exposed for testing.
+   * @param pool HBase ExecutorServcie instance, exposed for testing.
+   */
+  public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster,
+      ProcedureCoordinator coordinator, ExecutorService pool)
+      throws IOException, UnsupportedOperationException {
+    this.master = master;
+    this.metricsMaster = metricsMaster;
+
+    this.rootDir = master.getMasterFileSystem().getRootDir();
+    checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
+
+    this.wakeFrequency = master.getConfiguration().getInt(SNAPSHOT_WAKE_MILLIS_KEY,
+      SNAPSHOT_WAKE_MILLIS_DEFAULT);
+    this.coordinator = coordinator;
+    this.executorService = pool;
+    resetTempDir();
+  }
+
+  /**
+   * Gets the list of all completed snapshots.
+   * @return list of SnapshotDescriptions
+   * @throws IOException File system exception
+   */
+  public List getCompletedSnapshots() throws IOException {
+    return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
+  }
+
+  /**
+   * Gets the list of all completed snapshots.
+   * @param snapshotDir snapshot directory
+   * @return list of SnapshotDescriptions
+   * @throws IOException File system exception
+   */
+  private List getCompletedSnapshots(Path snapshotDir) throws IOException {
+    List snapshotDescs = new ArrayList();
+    // first create the snapshot root path and check to see if it exists
+    FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+
+    // if there are no snapshots, return an empty list
+    if (!fs.exists(snapshotDir)) {
+      return snapshotDescs;
+    }
+
+    // ignore all the snapshots in progress
+    FileStatus[] snapshots = fs.listStatus(snapshotDir,
+      new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
+    // loop through all the completed snapshots
+    for (FileStatus snapshot : snapshots) {
+      Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
+      // if the snapshot is bad
+      if (!fs.exists(info)) {
+        LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
+        continue;
+      }
+      FSDataInputStream in = null;
+      try {
+        in = fs.open(info);
+        SnapshotDescription desc = SnapshotDescription.parseFrom(in);
+        snapshotDescs.add(desc);
+      } catch (IOException e) {
+        LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
+      } finally {
+        if (in != null) {
+          in.close();
+        }
+      }
+    }
+    return snapshotDescs;
+  }
+
+  /**
+   * Cleans up any snapshots in the snapshot/.tmp directory that were left from failed
+   * snapshot attempts.
+   *
+   * @throws IOException if we can't reach the filesystem
+   */
+  void resetTempDir() throws IOException {
+    // cleanup any existing snapshots.
+    Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
+    if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
+      LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
+    }
+  }
+
+  /**
+   * Delete the specified snapshot
+   * @param snapshot
+   * @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
+   * @throws IOException For filesystem IOExceptions
+   */
+  public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
+
+    // call coproc pre hook
+    MasterCoprocessorHost cpHost = master.getCoprocessorHost();
+    if (cpHost != null) {
+      cpHost.preDeleteSnapshot(snapshot);
+    }
+
+    // check to see if it is completed
+    if (!isSnapshotCompleted(snapshot)) {
+      throw new SnapshotDoesNotExistException(snapshot);
+    }
+
+    String snapshotName = snapshot.getName();
+    LOG.debug("Deleting snapshot: " + snapshotName);
+    // first create the snapshot description and check to see if it exists
+    MasterFileSystem fs = master.getMasterFileSystem();
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+
+    // delete the existing snapshot
+    if (!fs.getFileSystem().delete(snapshotDir, true)) {
+      throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
+    }
+
+    // call coproc post hook
+    if (cpHost != null) {
+      cpHost.postDeleteSnapshot(snapshot);
+    }
+
+  }
+
+  /**
+   * Check if the specified snapshot is done
+   *
+   * @param expected
+   * @return true if snapshot is ready to be restored, false if it is still being taken.
+   * @throws IOException IOException if error from HDFS or RPC
+   * @throws UnknownSnapshotException if snapshot is invalid or does not exist.
+   */
+  public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
+    // check the request to make sure it has a snapshot
+    if (expected == null) {
+      throw new UnknownSnapshotException(
+         "No snapshot name passed in request, can't figure out which snapshot you want to check.");
+    }
+
+    String ssString = SnapshotDescriptionUtils.toString(expected);
+
+    // check to see if the sentinel exists,
+    // and if the task is complete removes it from the in-progress snapshots map.
+    SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
+
+    // stop tracking "abandoned" handlers
+    cleanupSentinels();
+
+    if (handler == null) {
+      // If there's no handler in the in-progress map, it means one of the following:
+      //   - someone has already requested the snapshot state
+      //   - the requested snapshot was completed long time ago (cleanupSentinels() timeout)
+      //   - the snapshot was never requested
+      // In those cases returns to the user the "done state" if the snapshots exists on disk,
+      // otherwise raise an exception saying that the snapshot is not running and doesn't exist.
+      if (!isSnapshotCompleted(expected)) {
+        throw new UnknownSnapshotException("Snapshot " + ssString
+            + " is not currently running or one of the known completed snapshots.");
+      }
+      // was done, return true;
+      return true;
+    }
+
+    // pass on any failure we find in the sentinel
+    try {
+      handler.rethrowExceptionIfFailed();
+    } catch (ForeignException e) {
+      // Give some procedure info on an exception.
+      String status;
+      Procedure p = coordinator.getProcedure(expected.getName());
+      if (p != null) {
+        status = p.getStatus();
+      } else {
+        status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
+      }
+      throw new HBaseSnapshotException("Snapshot " + ssString +  " had an error.  " + status, e,
+          expected);
+    }
+
+    // check to see if we are done
+    if (handler.isFinished()) {
+      LOG.debug("Snapshot '" + ssString + "' has completed, notifying client.");
+      return true;
+    } else if (LOG.isDebugEnabled()) {
+      LOG.debug("Snapshoting '" + ssString + "' is still in progress!");
+    }
+    return false;
+  }
+
+  /**
+   * Check to see if the specified table has a snapshot in progress.  Currently we have a
+   * limitation only allowing a single snapshot per table at a time.
+   * @param tableName name of the table being snapshotted.
+   * @return true if there is a snapshot in progress on the specified table.
+   */
+  synchronized boolean isTakingSnapshot(final String tableName) {
+    SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
+    return handler != null && !handler.isFinished();
+  }
+
+  /**
+   * Check to make sure that we are OK to run the passed snapshot. Checks to make sure that we
+   * aren't already running a snapshot or restore on the requested table.
+   * @param snapshot description of the snapshot we want to start
+   * @throws HBaseSnapshotException if the filesystem could not be prepared to start the snapshot
+   */
+  private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
+      throws HBaseSnapshotException {
+    FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
+
+    // make sure we aren't already running a snapshot
+    if (isTakingSnapshot(snapshot.getTable())) {
+      SnapshotSentinel handler = this.snapshotHandlers.get(snapshot.getTable());
+      throw new SnapshotCreationException("Rejected taking "
+          + SnapshotDescriptionUtils.toString(snapshot)
+          + " because we are already running another snapshot "
+          + SnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
+    }
+
+    // make sure we aren't running a restore on the same table
+    if (isRestoringTable(snapshot.getTable())) {
+      SnapshotSentinel handler = restoreHandlers.get(snapshot.getTable());
+      throw new SnapshotCreationException("Rejected taking "
+          + SnapshotDescriptionUtils.toString(snapshot)
+          + " because we are already have a restore in progress on the same snapshot "
+          + SnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
+    }
+
+    try {
+      // delete the working directory, since we aren't running the snapshot. Likely leftovers
+      // from a failed attempt.
+      fs.delete(workingDir, true);
+
+      // recreate the working directory for the snapshot
+      if (!fs.mkdirs(workingDir)) {
+        throw new SnapshotCreationException("Couldn't create working directory (" + workingDir
+            + ") for snapshot" , snapshot);
+      }
+    } catch (HBaseSnapshotException e) {
+      throw e;
+    } catch (IOException e) {
+      throw new SnapshotCreationException(
+          "Exception while checking to see if snapshot could be started.", e, snapshot);
+    }
+  }
+
+  /**
+   * Take a snapshot of a disabled table.
+   * @param snapshot description of the snapshot to take. Modified to be {@link Type#DISABLED}.
+   * @throws HBaseSnapshotException if the snapshot could not be started
+   */
+  private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
+      throws HBaseSnapshotException {
+    // setup the snapshot
+    prepareToTakeSnapshot(snapshot);
+
+    // set the snapshot to be a disabled snapshot, since the client doesn't know about that
+    snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
+
+    // Take the snapshot of the disabled table
+    DisabledTableSnapshotHandler handler =
+        new DisabledTableSnapshotHandler(snapshot, master, metricsMaster);
+    snapshotTable(snapshot, handler);
+  }
+
+  /**
+   * Take a snapshot of an enabled table.
+   * @param snapshot description of the snapshot to take.
+   * @throws HBaseSnapshotException if the snapshot could not be started
+   */
+  private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
+      throws HBaseSnapshotException {
+    // setup the snapshot
+    prepareToTakeSnapshot(snapshot);
+
+    // Take the snapshot of the enabled table
+    EnabledTableSnapshotHandler handler =
+        new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster);
+    snapshotTable(snapshot, handler);
+  }
+
+  /**
+   * Take a snapshot using the specified handler.
+   * On failure the snapshot temporary working directory is removed.
+   * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
+   *       snapshot request if the table is busy with another snapshot/restore operation.
+   * @param snapshot the snapshot description
+   * @param handler the snapshot handler
+   */
+  private synchronized void snapshotTable(SnapshotDescription snapshot,
+      final TakeSnapshotHandler handler) throws HBaseSnapshotException {
+    try {
+      handler.prepare();
+      this.executorService.submit(handler);
+      this.snapshotHandlers.put(snapshot.getTable(), handler);
+    } catch (Exception e) {
+      // cleanup the working directory by trying to delete it from the fs.
+      Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
+      try {
+        if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
+          LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
+              SnapshotDescriptionUtils.toString(snapshot));
+        }
+      } catch (IOException e1) {
+        LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
+            SnapshotDescriptionUtils.toString(snapshot));
+      }
+      // fail the snapshot
+      throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
+    }
+  }
+
+  /**
+   * Take a snapshot based on the enabled/disabled state of the table.
+   *
+   * @param snapshot
+   * @throws HBaseSnapshotException when a snapshot specific exception occurs.
+   * @throws IOException when some sort of generic IO exception occurs.
+   */
+  public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
+    // check to see if we already completed the snapshot
+    if (isSnapshotCompleted(snapshot)) {
+      throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
+          + "' already stored on the filesystem.", snapshot);
+    }
+
+    LOG.debug("No existing snapshot, attempting snapshot...");
+
+    // stop tracking "abandoned" handlers
+    cleanupSentinels();
+
+    // check to see if the table exists
+    HTableDescriptor desc = null;
+    try {
+      desc = master.getTableDescriptors().get(snapshot.getTable());
+    } catch (FileNotFoundException e) {
+      String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
+      LOG.error(msg);
+      throw new SnapshotCreationException(msg, e, snapshot);
+    } catch (IOException e) {
+      throw new SnapshotCreationException("Error while geting table description for table "
+          + snapshot.getTable(), e, snapshot);
+    }
+    if (desc == null) {
+      throw new SnapshotCreationException("Table '" + snapshot.getTable()
+          + "' doesn't exist, can't take snapshot.", snapshot);
+    }
+
+    // set the snapshot version, now that we are ready to take it
+    snapshot = snapshot.toBuilder().setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION)
+        .build();
+
+    // call pre coproc hook
+    MasterCoprocessorHost cpHost = master.getCoprocessorHost();
+    if (cpHost != null) {
+      cpHost.preSnapshot(snapshot, desc);
+    }
+
+    // if the table is enabled, then have the RS run actually the snapshot work
+    AssignmentManager assignmentMgr = master.getAssignmentManager();
+    if (assignmentMgr.getZKTable().isEnabledTable(snapshot.getTable())) {
+      LOG.debug("Table enabled, starting distributed snapshot.");
+      snapshotEnabledTable(snapshot);
+      LOG.debug("Started snapshot: " + SnapshotDescriptionUtils.toString(snapshot));
+    }
+    // For disabled table, snapshot is created by the master
+    else if (assignmentMgr.getZKTable().isDisabledTable(snapshot.getTable())) {
+      LOG.debug("Table is disabled, running snapshot entirely on master.");
+      snapshotDisabledTable(snapshot);
+      LOG.debug("Started snapshot: " + SnapshotDescriptionUtils.toString(snapshot));
+    } else {
+      LOG.error("Can't snapshot table '" + snapshot.getTable()
+          + "', isn't open or closed, we don't know what to do!");
+      TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
+          + " isn't fully open.");
+      throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
+    }
+
+    // call post coproc hook
+    if (cpHost != null) {
+      cpHost.postSnapshot(snapshot, desc);
+    }
+  }
+
+  /**
+   * Set the handler for the current snapshot
+   * 
        
+   * Exposed for TESTING
+   * @param tableName
+   * @param handler handler the master should use
+   *
+   * TODO get rid of this if possible, repackaging, modify tests.
+   */
+  public synchronized void setSnapshotHandlerForTesting(final String tableName,
+      final SnapshotSentinel handler) {
+    if (handler != null) {
+      this.snapshotHandlers.put(tableName, handler);
+    } else {
+      this.snapshotHandlers.remove(tableName);
+    }
+  }
+
+  /**
+   * @return distributed commit coordinator for all running snapshots
+   */
+  ProcedureCoordinator getCoordinator() {
+    return coordinator;
+  }
+
+  /**
+   * Check to see if the snapshot is one of the currently completed snapshots
+   * Returns true if the snapshot exists in the "completed snapshots folder".
+   *
+   * @param snapshot expected snapshot to check
+   * @return true if the snapshot is stored on the {@link FileSystem}, false if is
+   *         not stored
+   * @throws IOException if the filesystem throws an unexpected exception,
+   * @throws IllegalArgumentException if snapshot name is invalid.
+   */
+  private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
+    try {
+      final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+      FileSystem fs = master.getMasterFileSystem().getFileSystem();
+
+      // check to see if the snapshot already exists
+      return fs.exists(snapshotDir);
+    } catch (IllegalArgumentException iae) {
+      throw new UnknownSnapshotException("Unexpected exception thrown", iae);
+    }
+  }
+
+  /**
+   * Clone the specified snapshot into a new table.
+   * The operation will fail if the destination table has a snapshot or restore in progress.
+   *
+   * @param snapshot Snapshot Descriptor
+   * @param hTableDescriptor Table Descriptor of the table to create
+   */
+  synchronized void cloneSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
+    String tableName = hTableDescriptor.getNameAsString();
+
+    // make sure we aren't running a snapshot on the same table
+    if (isTakingSnapshot(tableName)) {
+      throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
+    }
+
+    // make sure we aren't running a restore on the same table
+    if (isRestoringTable(tableName)) {
+      throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
+    }
+
+    try {
+      CloneSnapshotHandler handler =
+        new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
+      this.executorService.submit(handler);
+      this.restoreHandlers.put(tableName, handler);
+    } catch (Exception e) {
+      String msg = "Couldn't clone the snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
+        " on table=" + tableName;
+      LOG.error(msg, e);
+      throw new RestoreSnapshotException(msg, e);
+    }
+  }
+
+  /**
+   * Restore the specified snapshot
+   * @param reqSnapshot
+   * @throws IOException
+   */
+  public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
+    FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
+    MasterCoprocessorHost cpHost = master.getCoprocessorHost();
+
+    // check if the snapshot exists
+    if (!fs.exists(snapshotDir)) {
+      LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
+      throw new SnapshotDoesNotExistException(reqSnapshot);
+    }
+
+    // read snapshot information
+    SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+    HTableDescriptor snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
+    String tableName = reqSnapshot.getTable();
+
+    // stop tracking "abandoned" handlers
+    cleanupSentinels();
+
+    // Execute the restore/clone operation
+    if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
+      if (master.getAssignmentManager().getZKTable().isEnabledTable(fsSnapshot.getTable())) {
+        throw new UnsupportedOperationException("Table '" +
+          fsSnapshot.getTable() + "' must be disabled in order to perform a restore operation.");
+      }
+
+      // call coproc pre hook
+      if (cpHost != null) {
+        cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
+      }
+      restoreSnapshot(fsSnapshot, snapshotTableDesc);
+      LOG.info("Restore snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
+
+      if (cpHost != null) {
+        cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
+      }
+    } else {
+      HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc,
+                                                         Bytes.toBytes(tableName));
+      if (cpHost != null) {
+        cpHost.preCloneSnapshot(reqSnapshot, htd);
+      }
+      cloneSnapshot(fsSnapshot, htd);
+      LOG.info("Clone snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
+
+      if (cpHost != null) {
+        cpHost.postCloneSnapshot(reqSnapshot, htd);
+      }
+    }
+  }
+
+  /**
+   * Restore the specified snapshot.
+   * The restore will fail if the destination table has a snapshot or restore in progress.
+   *
+   * @param snapshot Snapshot Descriptor
+   * @param hTableDescriptor Table Descriptor
+   */
+  private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
+      final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
+    String tableName = hTableDescriptor.getNameAsString();
+
+    // make sure we aren't running a snapshot on the same table
+    if (isTakingSnapshot(tableName)) {
+      throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
+    }
+
+    // make sure we aren't running a restore on the same table
+    if (isRestoringTable(tableName)) {
+      throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
+    }
+
+    try {
+      RestoreSnapshotHandler handler =
+        new RestoreSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
+      this.executorService.submit(handler);
+      restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
+    } catch (Exception e) {
+      String msg = "Couldn't restore the snapshot=" + SnapshotDescriptionUtils.toString(
+          snapshot)  +
+          " on table=" + tableName;
+      LOG.error(msg, e);
+      throw new RestoreSnapshotException(msg, e);
+    }
+  }
+
+  /**
+   * Verify if the restore of the specified table is in progress.
+   *
+   * @param tableName table under restore
+   * @return true if there is a restore in progress of the specified table.
+   */
+  private synchronized boolean isRestoringTable(final String tableName) {
+    SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
+    return(sentinel != null && !sentinel.isFinished());
+  }
+
+  /**
+   * Returns the status of a restore operation.
+   * If the in-progress restore is failed throws the exception that caused the failure.
+   *
+   * @param snapshot
+   * @return false if in progress, true if restore is completed or not requested.
+   * @throws IOException if there was a failure during the restore
+   */
+  public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
+    // check to see if the sentinel exists,
+    // and if the task is complete removes it from the in-progress restore map.
+    SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
+
+    // stop tracking "abandoned" handlers
+    cleanupSentinels();
+
+    if (sentinel == null) {
+      // there is no sentinel so restore is not in progress.
+      return true;
+    }
+
+    LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
+        + sentinel.getSnapshot().getName() + " table=" + snapshot.getTable());
+
+    // If the restore is failed, rethrow the exception
+    sentinel.rethrowExceptionIfFailed();
+
+    // check to see if we are done
+    if (sentinel.isFinished()) {
+      LOG.debug("Restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
+          " has completed. Notifying the client.");
+      return true;
+    }
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
+          SnapshotDescriptionUtils.toString(snapshot));
+    }
+    return false;
+  }
+
+  /**
+   * Return the handler if it is currently live and has the same snapshot target name.
+   * The handler is removed from the sentinels map if completed.
+   * @param sentinels live handlers
+   * @param snapshot snapshot description
+   * @return null if doesn't match, else a live handler.
+   */
+  private synchronized SnapshotSentinel removeSentinelIfFinished(
+      final Map sentinels, final SnapshotDescription snapshot) {
+    SnapshotSentinel h = sentinels.get(snapshot.getTable());
+    if (h == null) {
+      return null;
+    }
+
+    if (!h.getSnapshot().getName().equals(snapshot.getName())) {
+      // specified snapshot is to the one currently running
+      return null;
+    }
+
+    // Remove from the "in-progress" list once completed
+    if (h.isFinished()) {
+      sentinels.remove(snapshot.getTable());
+    }
+
+    return h;
+  }
+
+  /**
+   * Removes "abandoned" snapshot/restore requests.
+   * As part of the HBaseAdmin snapshot/restore API the operation status is checked until completed,
+   * and the in-progress maps are cleaned up when the status of a completed task is requested.
+   * To avoid having sentinels staying around for long time if something client side is failed,
+   * each operation tries to clean up the in-progress maps sentinels finished from a long time.
+   */
+  private void cleanupSentinels() {
+    cleanupSentinels(this.snapshotHandlers);
+    cleanupSentinels(this.restoreHandlers);
+  }
+
+  /**
+   * Remove the sentinels that are marked as finished and the completion time
+   * has exceeded the removal timeout.
+   * @param sentinels map of sentinels to clean
+   */
+  private synchronized void cleanupSentinels(final Map sentinels) {
+    long currentTime = EnvironmentEdgeManager.currentTimeMillis();
+    Iterator> it = sentinels.entrySet().iterator();
+    while (it.hasNext()) {
+      Map.Entry entry = it.next();
+      SnapshotSentinel sentinel = entry.getValue();
+      if (sentinel.isFinished() &&
+          (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
+      {
+        it.remove();
+      }
+    }
+  }
+
+  //
+  // Implementing Stoppable interface
+  //
+
+  @Override
+  public void stop(String why) {
+    // short circuit
+    if (this.stopped) return;
+    // make sure we get stop
+    this.stopped = true;
+    // pass the stop onto take snapshot handlers
+    for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
+      snapshotHandler.cancel(why);
+    }
+
+    // pass the stop onto all the restore handlers
+    for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
+      restoreHandler.cancel(why);
+    }
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.stopped;
+  }
+
+  /**
+   * Throws an exception if snapshot operations (take a snapshot, restore, clone) are not supported.
+   * Called at the beginning of snapshot() and restoreSnapshot() methods.
+   * @throws UnsupportedOperationException if snapshot are not supported
+   */
+  public void checkSnapshotSupport() throws UnsupportedOperationException {
+    if (!this.isSnapshotSupported) {
+      throw new UnsupportedOperationException(
+        "To use snapshots, You must add to the hbase-site.xml of the HBase Master: '" +
+          HBASE_SNAPSHOT_ENABLED + "' property with value 'true'.");
+    }
+  }
+
+  /**
+   * Called at startup, to verify if snapshot operation is supported, and to avoid
+   * starting the master if there're snapshots present but the cleaners needed are missing.
+   * Otherwise we can end up with snapshot data loss.
+   * @param conf The {@link Configuration} object to use
+   * @param mfs The MasterFileSystem to use
+   * @throws IOException in case of file-system operation failure
+   * @throws UnsupportedOperationException in case cleaners are missing and
+   *         there're snapshot in the system
+   */
+  private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
+      throws IOException, UnsupportedOperationException {
+    // Verify if snapshot is disabled by the user
+    String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
+    boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
+    boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
+
+    // Extract cleaners from conf
+    Set hfileCleaners = new HashSet();
+    String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
+    if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);
+
+    Set logCleaners = new HashSet();
+    cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
+    if (cleaners != null) Collections.addAll(logCleaners, cleaners);
+
+    // check if an older version of snapshot directory was present
+    Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
+    FileSystem fs = mfs.getFileSystem();
+    List ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
+    if (ss != null && !ss.isEmpty()) {
+      LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
+      LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
+    }
+
+    // If the user has enabled the snapshot, we force the cleaners to be present
+    // otherwise we still need to check if cleaners are enabled or not and verify
+    // that there're no snapshot in the .snapshot folder.
+    if (snapshotEnabled) {
+      // Inject snapshot cleaners, if snapshot.enable is true
+      hfileCleaners.add(SnapshotHFileCleaner.class.getName());
+      hfileCleaners.add(HFileLinkCleaner.class.getName());
+      logCleaners.add(SnapshotLogCleaner.class.getName());
+
+      // Set cleaners conf
+      conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
+        hfileCleaners.toArray(new String[hfileCleaners.size()]));
+      conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
+        logCleaners.toArray(new String[logCleaners.size()]));
+    } else {
+      // Verify if cleaners are present
+      snapshotEnabled = logCleaners.contains(SnapshotLogCleaner.class.getName()) &&
+        hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
+        hfileCleaners.contains(HFileLinkCleaner.class.getName());
+
+      // Warn if the cleaners are enabled but the snapshot.enabled property is false/not set.
+      if (snapshotEnabled) {
+        LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
+          "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
+          (userDisabled ? "is set to 'false'." : "is not set."));
+      }
+    }
+
+    // Mark snapshot feature as enabled if cleaners are present and user has not disabled it.
+    this.isSnapshotSupported = snapshotEnabled && !userDisabled;
+
+    // If cleaners are not enabled, verify that there're no snapshot in the .snapshot folder
+    // otherwise we end up with snapshot data loss.
+    if (!snapshotEnabled) {
+      LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
+      Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
+      if (fs.exists(snapshotDir)) {
+        FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
+          new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
+        if (snapshots != null) {
+          LOG.error("Snapshots are present, but cleaners are not enabled.");
+          checkSnapshotSupport();
+        }
+      }
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
new file mode 100644
index 0000000..48ed967
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CancellationException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * A handler for taking snapshots from the master.
+ *
+ * This is not a subclass of TableEventHandler because using that would incur an extra META scan.
+ *
+ * The {@link #snapshotRegions(List)} call should get implemented for each snapshot flavor.
+ */
+@InterfaceAudience.Private
+public abstract class TakeSnapshotHandler extends EventHandler implements SnapshotSentinel,
+    ForeignExceptionSnare {
+  private static final Log LOG = LogFactory.getLog(TakeSnapshotHandler.class);
+
+  private volatile boolean finished;
+
+  // none of these should ever be null
+  protected final MasterServices master;
+  protected final MasterMetrics metricsMaster;
+  protected final SnapshotDescription snapshot;
+  protected final Configuration conf;
+  protected final FileSystem fs;
+  protected final Path rootDir;
+  private final Path snapshotDir;
+  protected final Path workingDir;
+  private final MasterSnapshotVerifier verifier;
+  protected final ForeignExceptionDispatcher monitor;
+  protected final MonitoredTask status;
+
+  /**
+   * @param snapshot descriptor of the snapshot to take
+   * @param masterServices master services provider
+   * @throws IOException on unexpected error
+   */
+  public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices masterServices,
+      final MasterMetrics metricsMaster) {
+    super(masterServices, EventType.C_M_SNAPSHOT_TABLE);
+    assert snapshot != null : "SnapshotDescription must not be nul1";
+    assert masterServices != null : "MasterServices must not be nul1";
+
+    this.master = masterServices;
+    this.metricsMaster = metricsMaster;
+    this.snapshot = snapshot;
+    this.conf = this.master.getConfiguration();
+    this.fs = this.master.getMasterFileSystem().getFileSystem();
+    this.rootDir = this.master.getMasterFileSystem().getRootDir();
+    this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+    this.workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
+    this.monitor =  new ForeignExceptionDispatcher();
+
+    // prepare the verify
+    this.verifier = new MasterSnapshotVerifier(masterServices, snapshot, rootDir);
+    // update the running tasks
+    this.status = TaskMonitor.get().createStatus(
+      "Taking " + snapshot.getType() + " snapshot on table: " + snapshot.getTable());
+  }
+
+  private HTableDescriptor loadTableDescriptor()
+      throws FileNotFoundException, IOException {
+    final String name = snapshot.getTable();
+    HTableDescriptor htd =
+      this.master.getTableDescriptors().get(name);
+    if (htd == null) {
+      throw new IOException("HTableDescriptor missing for " + name);
+    }
+    return htd;
+  }
+
+  public TakeSnapshotHandler prepare() throws Exception {
+    loadTableDescriptor(); // check that .tableinfo is present
+    return this;
+  }
+
+  /**
+   * Execute the core common portions of taking a snapshot. The {@link #snapshotRegions(List)}
+   * call should get implemented for each snapshot flavor.
+   */
+  @Override
+  public void process() {
+    String msg = "Running " + snapshot.getType() + " table snapshot " + snapshot.getName() + " "
+        + eventType + " on table " + snapshot.getTable();
+    LOG.info(msg);
+    status.setStatus(msg);
+    try {
+      // If regions move after this meta scan, the region specific snapshot should fail, triggering
+      // an external exception that gets captured here.
+
+      // write down the snapshot info in the working directory
+      SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, this.fs);
+      new TableInfoCopyTask(monitor, snapshot, fs, rootDir).call();
+      monitor.rethrowException();
+
+      List> regionsAndLocations =
+          MetaReader.getTableRegionsAndLocations(this.server.getCatalogTracker(),
+            Bytes.toBytes(snapshot.getTable()), true);
+
+      // run the snapshot
+      snapshotRegions(regionsAndLocations);
+
+      // extract each pair to separate lists
+      Set serverNames = new HashSet();
+      for (Pair p : regionsAndLocations) {
+        serverNames.add(p.getSecond().toString());
+      }
+
+      // verify the snapshot is valid
+      status.setStatus("Verifying snapshot: " + snapshot.getName());
+      verifier.verifySnapshot(this.workingDir, serverNames);
+
+      // complete the snapshot, atomically moving from tmp to .snapshot dir.
+      completeSnapshot(this.snapshotDir, this.workingDir, this.fs);
+      status.markComplete("Snapshot " + snapshot.getName() + " of table " + snapshot.getTable()
+          + " completed");
+      metricsMaster.addSnapshot(status.getCompletionTimestamp() - status.getStartTime());
+    } catch (Exception e) {
+      status.abort("Failed to complete snapshot " + snapshot.getName() + " on table " +
+          snapshot.getTable() + " because " + e.getMessage());
+      String reason = "Failed taking snapshot " + SnapshotDescriptionUtils.toString(snapshot)
+          + " due to exception:" + e.getMessage();
+      LOG.error(reason, e);
+      ForeignException ee = new ForeignException(reason, e);
+      monitor.receive(ee);
+      // need to mark this completed to close off and allow cleanup to happen.
+      cancel("Failed to take snapshot '" + SnapshotDescriptionUtils.toString(snapshot)
+          + "' due to exception");
+    } finally {
+      LOG.debug("Launching cleanup of working dir:" + workingDir);
+      try {
+        // if the working dir is still present, the snapshot has failed.  it is present we delete
+        // it.
+        if (fs.exists(workingDir) && !this.fs.delete(workingDir, true)) {
+          LOG.error("Couldn't delete snapshot working directory:" + workingDir);
+        }
+      } catch (IOException e) {
+        LOG.error("Couldn't delete snapshot working directory:" + workingDir);
+      }
+    }
+  }
+
+  /**
+   * Reset the manager to allow another snapshot to proceed
+   *
+   * @param snapshotDir final path of the snapshot
+   * @param workingDir directory where the in progress snapshot was built
+   * @param fs {@link FileSystem} where the snapshot was built
+   * @throws SnapshotCreationException if the snapshot could not be moved
+   * @throws IOException the filesystem could not be reached
+   */
+  public void completeSnapshot(Path snapshotDir, Path workingDir, FileSystem fs)
+      throws SnapshotCreationException, IOException {
+    LOG.debug("Sentinel is done, just moving the snapshot from " + workingDir + " to "
+        + snapshotDir);
+    if (!fs.rename(workingDir, snapshotDir)) {
+      throw new SnapshotCreationException("Failed to move working directory(" + workingDir
+          + ") to completed directory(" + snapshotDir + ").");
+    }
+    finished = true;
+  }
+
+  /**
+   * Snapshot the specified regions
+   */
+  protected abstract void snapshotRegions(List> regions)
+      throws IOException, KeeperException;
+
+  @Override
+  public void cancel(String why) {
+    if (finished) return;
+
+    this.finished = true;
+    LOG.info("Stop taking snapshot=" + SnapshotDescriptionUtils.toString(snapshot) + " because: "
+        + why);
+    CancellationException ce = new CancellationException(why);
+    monitor.receive(new ForeignException(master.getServerName().toString(), ce));
+  }
+
+  @Override
+  public boolean isFinished() {
+    return finished;
+  }
+
+  @Override
+  public long getCompletionTimestamp() {
+    return this.status.getCompletionTimestamp();
+  }
+
+  @Override
+  public SnapshotDescription getSnapshot() {
+    return snapshot;
+  }
+
+  @Override
+  public ForeignException getExceptionIfFailed() {
+    return monitor.getException();
+  }
+
+  @Override
+  public void rethrowExceptionIfFailed() throws ForeignException {
+    monitor.rethrowException();
+  }
+
+  @Override
+  public void rethrowException() throws ForeignException {
+    monitor.rethrowException();
+  }
+
+  @Override
+  public boolean hasException() {
+    return monitor.hasException();
+  }
+
+  @Override
+  public ForeignException getException() {
+    return monitor.getException();
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/ExactCounterMetric.java b/src/main/java/org/apache/hadoop/hbase/metrics/ExactCounterMetric.java
new file mode 100644
index 0000000..40e29eb
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/ExactCounterMetric.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.metrics;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.util.MetricsBase;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+import org.cliffc.high_scale_lib.Counter;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.MapMaker;
+
+public class ExactCounterMetric extends MetricsBase {
+
+  private static final int DEFAULT_TOP_N = 5;
+
+  // only publish stats on the topN items (default to DEFAULT_TOP_N)
+  private final int topN;
+  private final Map counts;
+
+  // all access to the 'counts' map should use this lock.
+  // take a write lock iff you want to guarantee exclusive access
+  // (the map stripes locks internally, so it's already thread safe -
+  // this lock is just so you can take a consistent snapshot of data)
+  private final ReadWriteLock lock;
+  
+  
+  /**
+   * Constructor to create a new counter metric
+   * @param nam         the name to publish this metric under
+   * @param registry    where the metrics object will be registered
+   * @param description metrics description
+   * @param topN        how many 'keys' to publish metrics on 
+   */
+  public ExactCounterMetric(final String nam, final MetricsRegistry registry, 
+      final String description, int topN) {
+    super(nam, description);
+
+    this.counts = new MapMaker().makeComputingMap(
+        new Function() {
+          @Override
+          public Counter apply(String input) {
+            return new Counter();
+          }    
+        });
+
+    this.lock = new ReentrantReadWriteLock();
+    this.topN = topN;
+
+    if (registry != null) {
+      registry.add(nam, this);      
+    }
+  }
+
+  /**
+   * Constructor creates a new ExactCounterMetric
+   * @param nam       the name of the metrics to be used to publish the metric
+   * @param registry  where the metrics object will be registered
+   */
+  public ExactCounterMetric(final String nam, MetricsRegistry registry) {
+    this(nam, registry, NO_DESCRIPTION, DEFAULT_TOP_N);
+  }
+
+  
+  public void update(String type) {
+    this.lock.readLock().lock();
+    try {
+      this.counts.get(type).increment();
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+  
+  public void update(String type, long count) {
+    this.lock.readLock().lock();
+    try {
+      this.counts.get(type).add(count);
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+  
+  public List> getTop(int n) {
+    final List> countsSnapshot = 
+        Lists.newArrayListWithCapacity(this.counts.size());
+    
+    // no updates are allowed while I'm holding this lock, so move fast
+    this.lock.writeLock().lock();
+    try {
+      for(Entry entry : this.counts.entrySet()) {
+        countsSnapshot.add(Pair.newPair(entry.getKey(), 
+            entry.getValue().get()));
+      }
+    } finally {
+      this.lock.writeLock().unlock();
+    }
+    
+    Collections.sort(countsSnapshot, new Comparator>() {
+      @Override
+      public int compare(Pair a, Pair b) {
+        return b.getSecond().compareTo(a.getSecond());
+      }      
+    });
+    
+    return countsSnapshot.subList(0, Math.min(n, countsSnapshot.size()));
+  }
+  
+  @Override
+  public void pushMetric(MetricsRecord mr) {
+    final List> topKeys = getTop(Integer.MAX_VALUE);
+    int sum = 0;
+    
+    int counter = 0;
+    for (Pair keyCount : topKeys) {
+      counter++;
+      // only push stats on the topN keys
+      if (counter <= this.topN) {
+        mr.setMetric(getName() + "_" + keyCount.getFirst(), 
+            keyCount.getSecond());        
+      }
+      sum += keyCount.getSecond();
+    }
+    mr.setMetric(getName() + "_map_size", this.counts.size());
+    mr.setMetric(getName() + "_total_count", sum);
+  }
+  
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/HBaseInfo.java b/src/main/java/org/apache/hadoop/hbase/metrics/HBaseInfo.java
new file mode 100644
index 0000000..38ed872
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/HBaseInfo.java
@@ -0,0 +1,98 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics;
+
+import org.apache.hadoop.hbase.metrics.MetricsMBeanBase;
+import org.apache.hadoop.metrics.MetricsContext;
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.MetricsUtil;
+import org.apache.hadoop.metrics.util.MBeanUtil;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+
+import javax.management.ObjectName;
+
+/**
+ * Exports HBase system information as an MBean for JMX observation.
+ */
+public class HBaseInfo {
+  protected static class HBaseInfoMBean extends MetricsMBeanBase {
+    private final ObjectName mbeanName;
+  
+    public HBaseInfoMBean(MetricsRegistry registry, String rsName) {
+      super(registry, "HBase cluster information");
+      // The name seems wrong to me; should include clusterid IMO.
+      // That would make it harder to locate and rare we have
+      // two clusters up on single machine. St.Ack 20120309
+      mbeanName = MBeanUtil.registerMBean("HBase", "Info", this);
+    }
+  
+    public void shutdown() {
+      if (mbeanName != null)
+        MBeanUtil.unregisterMBean(mbeanName);
+    }
+  }
+
+  protected final MetricsRecord mr;
+  protected final HBaseInfoMBean mbean;
+  protected MetricsRegistry registry = new MetricsRegistry();
+
+  private static HBaseInfo theInstance = null;
+  public synchronized static HBaseInfo init() {
+      if (theInstance == null) {
+        theInstance = new HBaseInfo();
+      }
+      return theInstance;
+  }
+  
+  // HBase jar info
+  private MetricsString date = new MetricsString("date", registry,
+      org.apache.hadoop.hbase.util.VersionInfo.getDate());
+  private MetricsString revision = new MetricsString("revision", registry, 
+      org.apache.hadoop.hbase.util.VersionInfo.getRevision());
+  private MetricsString url = new MetricsString("url", registry,
+      org.apache.hadoop.hbase.util.VersionInfo.getUrl());
+  private MetricsString user = new MetricsString("user", registry,
+      org.apache.hadoop.hbase.util.VersionInfo.getUser());
+  private MetricsString version = new MetricsString("version", registry,
+      org.apache.hadoop.hbase.util.VersionInfo.getVersion());
+
+  // Info on the HDFS jar that HBase has (aka: HDFS Client)
+  private MetricsString hdfsDate = new MetricsString("hdfsDate", registry,
+      org.apache.hadoop.util.VersionInfo.getDate());
+  private MetricsString hdfsRev = new MetricsString("hdfsRevision", registry,
+      org.apache.hadoop.util.VersionInfo.getRevision());
+  private MetricsString hdfsUrl = new MetricsString("hdfsUrl", registry,
+      org.apache.hadoop.util.VersionInfo.getUrl());
+  private MetricsString hdfsUser = new MetricsString("hdfsUser", registry,
+      org.apache.hadoop.util.VersionInfo.getUser());
+  private MetricsString hdfsVer = new MetricsString("hdfsVersion", registry,
+      org.apache.hadoop.util.VersionInfo.getVersion());
+
+  protected HBaseInfo() {
+    MetricsContext context = MetricsUtil.getContext("hbase");
+    mr = MetricsUtil.createRecord(context, "info");
+    String name = Thread.currentThread().getName();
+    mr.setTag("Info", name);
+
+    // export for JMX
+    mbean = new HBaseInfoMBean(this.registry, name);
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
new file mode 100644
index 0000000..00d514c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsMBeanBase.java
@@ -0,0 +1,241 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.management.AttributeNotFoundException;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanException;
+import javax.management.MBeanInfo;
+import javax.management.ReflectionException;
+import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
+
+import com.yammer.metrics.stats.Snapshot;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics.util.MetricsBase;
+import org.apache.hadoop.metrics.util.MetricsDynamicMBeanBase;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+
+/**
+ * Extends the Hadoop MetricsDynamicMBeanBase class to provide JMX support for
+ * custom HBase MetricsBase implementations.  MetricsDynamicMBeanBase ignores
+ * registered MetricsBase instance that are not instances of one of the
+ * org.apache.hadoop.metrics.util implementations.
+ *
+ */
+public class MetricsMBeanBase extends MetricsDynamicMBeanBase {
+
+  private static final Log LOG = LogFactory.getLog("org.apache.hadoop.hbase.metrics");
+
+  protected final MetricsRegistry registry;
+  protected final String description;
+  protected int registryLength;
+  /** HBase MetricsBase implementations that MetricsDynamicMBeanBase does
+   * not understand
+   */
+  protected Map extendedAttributes =
+      new ConcurrentHashMap();
+  protected MBeanInfo extendedInfo;
+
+  protected MetricsMBeanBase( MetricsRegistry mr, String description ) {
+    super(copyMinusHBaseMetrics(mr), description);
+    this.registry = mr;
+    this.description = description;
+    this.init();
+  }
+
+  /*
+   * @param mr MetricsRegistry.
+   * @return A copy of the passed MetricsRegistry minus the hbase metrics
+   */
+  private static MetricsRegistry copyMinusHBaseMetrics(final MetricsRegistry mr) {
+    MetricsRegistry copy = new MetricsRegistry();
+    for (MetricsBase metric : mr.getMetricsList()) {
+      if (metric instanceof MetricsRate || metric instanceof MetricsString ||
+          metric instanceof MetricsHistogram || metric instanceof ExactCounterMetric) {
+        continue;
+      }
+      copy.add(metric.getName(), metric);
+    }
+    return copy;
+  }
+
+  protected void init() {
+    List attributes = new ArrayList();
+    MBeanInfo parentInfo = super.getMBeanInfo();
+    List parentAttributes = new ArrayList();
+    for (MBeanAttributeInfo attr : parentInfo.getAttributes()) {
+      attributes.add(attr);
+      parentAttributes.add(attr.getName());
+    }
+
+    this.registryLength = this.registry.getMetricsList().size();
+
+    for (MetricsBase metric : this.registry.getMetricsList()) {
+      if (metric.getName() == null || parentAttributes.contains(metric.getName()))
+        continue;
+
+      // add on custom HBase metric types
+      if (metric instanceof MetricsRate) {
+        attributes.add( new MBeanAttributeInfo(metric.getName(),
+            "java.lang.Float", metric.getDescription(), true, false, false) );
+        extendedAttributes.put(metric.getName(), metric);
+      } else if (metric instanceof MetricsString) {
+        attributes.add( new MBeanAttributeInfo(metric.getName(),
+            "java.lang.String", metric.getDescription(), true, false, false) );
+        extendedAttributes.put(metric.getName(), metric);
+        LOG.info("MetricsString added: " + metric.getName());
+      } else if (metric instanceof MetricsHistogram) {
+
+        String metricName = metric.getName() + MetricsHistogram.NUM_OPS_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Long", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.MIN_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Long", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.MAX_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Long", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.MEAN_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Float", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.STD_DEV_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Float", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.MEDIAN_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Float", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.SEVENTY_FIFTH_PERCENTILE_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Float", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.NINETY_FIFTH_PERCENTILE_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Float", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+
+        metricName = metric.getName() + MetricsHistogram.NINETY_NINETH_PERCENTILE_METRIC_NAME;
+        attributes.add(new MBeanAttributeInfo(metricName,
+            "java.lang.Float", metric.getDescription(), true, false, false));
+        extendedAttributes.put(metricName, metric);
+      }
+      // else, its probably a hadoop metric already registered. Skip it.
+    }
+
+    LOG.info("new MBeanInfo");
+    this.extendedInfo = new MBeanInfo( this.getClass().getName(),
+        this.description, attributes.toArray( new MBeanAttributeInfo[0] ),
+        parentInfo.getConstructors(), parentInfo.getOperations(),
+        parentInfo.getNotifications() );
+  }
+
+  private void checkAndUpdateAttributes() {
+    if (this.registryLength != this.registry.getMetricsList().size())
+      this.init();
+  }
+
+  @Override
+  public Object getAttribute( String name )
+      throws AttributeNotFoundException, MBeanException,
+      ReflectionException {
+
+    if (name == null) {
+      throw new IllegalArgumentException("Attribute name is NULL");
+    }
+
+    /*
+     * Ugly.  Since MetricsDynamicMBeanBase implementation is private,
+     * we need to first check the parent class for the attribute.
+     * In case that the MetricsRegistry contents have changed, this will
+     * allow the parent to update it's internal structures (which we rely on
+     * to update our own.
+     */
+    try {
+      return super.getAttribute(name);
+    } catch (AttributeNotFoundException ex) {
+
+      checkAndUpdateAttributes();
+
+      MetricsBase metric = this.extendedAttributes.get(name);
+      if (metric != null) {
+        if (metric instanceof MetricsRate) {
+          return ((MetricsRate) metric).getPreviousIntervalValue();
+        } else if (metric instanceof MetricsString) {
+          return ((MetricsString)metric).getValue();
+        } else if (metric instanceof MetricsHistogram)  {
+          MetricsHistogram hist = (MetricsHistogram) metric;
+          if (name.endsWith(MetricsHistogram.NUM_OPS_METRIC_NAME)) {
+            return hist.getCount();
+          } else if (name.endsWith(MetricsHistogram.MIN_METRIC_NAME)) {
+            return hist.getMin();
+          } else if (name.endsWith(MetricsHistogram.MAX_METRIC_NAME)) {
+            return hist.getMax();
+          } else if (name.endsWith(MetricsHistogram.MEAN_METRIC_NAME)) {
+            return (float) hist.getMean();
+          } else if (name.endsWith(MetricsHistogram.STD_DEV_METRIC_NAME)) {
+            return (float) hist.getStdDev();
+          } else if (name.endsWith(MetricsHistogram.MEDIAN_METRIC_NAME)) {
+            Snapshot s = hist.getSnapshot();
+            return (float) s.getMedian();
+          } else if (name.endsWith(MetricsHistogram.SEVENTY_FIFTH_PERCENTILE_METRIC_NAME)) {
+            Snapshot s = hist.getSnapshot();
+            return (float) s.get75thPercentile();
+          } else if (name.endsWith(MetricsHistogram.NINETY_FIFTH_PERCENTILE_METRIC_NAME)) {
+            Snapshot s = hist.getSnapshot();
+            return (float) s.get95thPercentile();
+          } else if (name.endsWith(MetricsHistogram.NINETY_NINETH_PERCENTILE_METRIC_NAME)) {
+            Snapshot s = hist.getSnapshot();
+            return (float) s.get99thPercentile();
+          }
+
+        } else {
+          LOG.warn( String.format("unknown metrics type %s for attribute %s",
+                        metric.getClass().getName(), name) );
+        }
+      }
+    }
+
+    throw new AttributeNotFoundException();
+  }
+
+  @Override
+  public MBeanInfo getMBeanInfo() {
+    return this.extendedInfo;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/MetricsRate.java b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsRate.java
new file mode 100644
index 0000000..9e00d15
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsRate.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.util.MetricsBase;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Publishes a rate based on a counter - you increment the counter each
+ * time an event occurs (eg: an RPC call) and this publishes a rate.
+ */
+public class MetricsRate extends MetricsBase {
+  private static final Log LOG = LogFactory.getLog("org.apache.hadoop.hbase.metrics");
+
+  private int value;
+  private float prevRate;
+  private long ts;
+
+  public MetricsRate(final String name, final MetricsRegistry registry,
+      final String description) {
+    super(name, description);
+    this.value = 0;
+    this.prevRate = 0;
+    this.ts = System.currentTimeMillis();
+    registry.add(name, this);
+  }
+
+  public MetricsRate(final String name, final MetricsRegistry registry) {
+    this(name, registry, NO_DESCRIPTION);
+  }
+
+  public synchronized void inc(final int incr) {
+    value += incr;
+  }
+
+  public synchronized void inc() {
+    value++;
+  }
+
+  public synchronized void intervalHeartBeat() {
+    long now = System.currentTimeMillis();
+    long diff = (now-ts) / 1000;
+    if (diff < 1){
+        // To make sure our averages aren't skewed by fast repeated calls,
+        // we simply ignore fast repeated calls.
+    	return;
+    }
+    this.prevRate = (float)value / diff;
+    this.value = 0;
+    this.ts = now;
+  }
+
+  @Override
+  public synchronized void pushMetric(final MetricsRecord mr) {
+    intervalHeartBeat();
+    try {
+      mr.setMetric(getName(), getPreviousIntervalValue());
+    } catch (Exception e) {
+      LOG.info("pushMetric failed for " + getName() + "\n" +
+          StringUtils.stringifyException(e));
+    }
+  }
+
+  public synchronized float getPreviousIntervalValue() {
+    return this.prevRate;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/MetricsString.java b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsString.java
new file mode 100644
index 0000000..2ee8066
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/MetricsString.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.util.MetricsBase;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+
+/**
+ * Publishes a string to the metrics collector
+ */
+public class MetricsString extends MetricsBase {
+  private static final Log LOG = LogFactory.getLog("org.apache.hadoop.hbase.metrics");
+
+  private String value;
+
+  public MetricsString(final String name, final MetricsRegistry registry, 
+      final String value) {
+    super(name, NO_DESCRIPTION);
+    this.value = value;
+    registry.add(name, this);
+  }
+  public MetricsString(final String name, final String description, 
+      final MetricsRegistry registry, final String value) {
+    super(name, description);
+    this.value = value;
+    registry.add(name, this);
+  }
+  
+  public String getValue() {
+    return this.value;
+  }
+
+  @Override
+  public synchronized void pushMetric(final MetricsRecord mr) {
+    // NOOP
+    // MetricsMBeanBase.getAttribute is where we actually fill the data
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/PersistentMetricsTimeVaryingRate.java b/src/main/java/org/apache/hadoop/hbase/metrics/PersistentMetricsTimeVaryingRate.java
new file mode 100644
index 0000000..cf2fc28
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/PersistentMetricsTimeVaryingRate.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * This class extends MetricsTimeVaryingRate to let the metrics
+ * persist past a pushMetric() call
+ */
+public class PersistentMetricsTimeVaryingRate extends MetricsTimeVaryingRate {
+  protected static final Log LOG =
+    LogFactory.getLog("org.apache.hadoop.hbase.metrics");
+
+  protected boolean reset = false;
+  protected long lastOper = 0;
+  protected long totalOps = 0;
+
+  /**
+   * Constructor - create a new metric
+   * @param nam the name of the metrics to be used to publish the metric
+   * @param registry - where the metrics object will be registered
+   * @param description metrics description
+   */
+  public PersistentMetricsTimeVaryingRate(final String nam, 
+      final MetricsRegistry registry, 
+      final String description) {
+    super(nam, registry, description);
+  }
+
+  /**
+   * Constructor - create a new metric
+   * @param nam the name of the metrics to be used to publish the metric
+   * @param registry - where the metrics object will be registered
+   */
+  public PersistentMetricsTimeVaryingRate(final String nam, 
+      MetricsRegistry registry) {
+    this(nam, registry, NO_DESCRIPTION);
+  }
+
+  /**
+   * Push updated metrics to the mr.
+   * 
+   * Note this does NOT push to JMX
+   * (JMX gets the info via {@link #getPreviousIntervalAverageTime()} and
+   * {@link #getPreviousIntervalNumOps()}
+   *
+   * @param mr owner of this metric
+   */
+  @Override
+  public synchronized void pushMetric(final MetricsRecord mr) {
+    // this will reset the currentInterval & num_ops += prevInterval()
+    super.pushMetric(mr);
+    // since we're retaining prevInterval(), we don't want to do the incr
+    // instead, we want to set that value because we have absolute ops
+    try {
+      mr.setMetric(getName() + "_num_ops", totalOps);
+    } catch (Exception e) {
+      LOG.info("pushMetric failed for " + getName() + "\n" +
+          StringUtils.stringifyException(e));
+    }
+    if (reset) {
+      // use the previous avg as our starting min/max/avg
+      super.inc(getPreviousIntervalAverageTime());
+      reset = false;
+    } else {
+      // maintain the stats that pushMetric() cleared
+      maintainStats();
+    }
+  }
+  
+  /**
+   * Increment the metrics for numOps operations
+   * @param numOps - number of operations
+   * @param time - time for numOps operations
+   */
+  @Override
+  public synchronized void inc(final int numOps, final long time) {
+    super.inc(numOps, time);
+    totalOps += numOps;
+  }
+  
+  /**
+   * Increment the metrics for numOps operations
+   * @param time - time for numOps operations
+   */
+  @Override
+  public synchronized void inc(final long time) {
+    super.inc(time);
+    ++totalOps;
+  }
+  
+  /**
+   * Rollover to a new interval
+   * NOTE: does not reset numOps.  this is an absolute value
+   */
+  public synchronized void resetMinMaxAvg() {
+    reset = true;
+  }
+
+  /* MetricsTimeVaryingRate will reset every time pushMetric() is called
+   * This is annoying for long-running stats that might not get a single 
+   * operation in the polling period.  This function ensures that values
+   * for those stat entries don't get reset.
+   */
+  protected void maintainStats() {
+    int curOps = this.getPreviousIntervalNumOps();
+    if (curOps > 0) {
+      long curTime = this.getPreviousIntervalAverageTime();
+      long totalTime = curTime * curOps;
+      if (curTime == 0 || totalTime / curTime == curOps) {
+        super.inc(curOps, totalTime);
+      } else {
+        LOG.info("Stats for " + this.getName() + " overflowed! resetting");
+      }
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/file/TimeStampingFileContext.java b/src/main/java/org/apache/hadoop/hbase/metrics/file/TimeStampingFileContext.java
new file mode 100644
index 0000000..000e0d3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/file/TimeStampingFileContext.java
@@ -0,0 +1,111 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics.file;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.hadoop.metrics.ContextFactory;
+import org.apache.hadoop.metrics.file.FileContext;
+import org.apache.hadoop.metrics.spi.OutputRecord;
+
+/**
+ * Add timestamp to {@link org.apache.hadoop.metrics.file.FileContext#emitRecord(String, String, OutputRecord)}.
+ */
+public class TimeStampingFileContext extends FileContext {
+  // Copies bunch of FileContext here because writer and file are private in
+  // superclass.
+  private File file = null;
+  private PrintWriter writer = null;
+  private final SimpleDateFormat sdf;
+
+  public TimeStampingFileContext() {
+    super();
+    this.sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+  }
+
+  @Override
+  public void init(String contextName, ContextFactory factory) {
+    super.init(contextName, factory);
+    String fileName = getAttribute(FILE_NAME_PROPERTY);
+    if (fileName != null) {
+      file = new File(fileName);
+    }
+  }
+
+  @Override
+  public void startMonitoring() throws IOException {
+    if (file == null) {
+      writer = new PrintWriter(new BufferedOutputStream(System.out));
+    } else {
+      writer = new PrintWriter(new FileWriter(file, true));
+    }
+    super.startMonitoring();
+  }
+
+  @Override
+  public void stopMonitoring() {
+    super.stopMonitoring();
+    if (writer != null) {
+      writer.close();
+      writer = null;
+    }
+  }
+
+  private synchronized String iso8601() {
+    return this.sdf.format(new Date());
+  }
+
+  @Override
+  public void emitRecord(String contextName, String recordName,
+      OutputRecord outRec) {
+    writer.print(iso8601());
+    writer.print(" ");
+    writer.print(contextName);
+    writer.print(".");
+    writer.print(recordName);
+    String separator = ": ";
+    for (String tagName : outRec.getTagNames()) {
+      writer.print(separator);
+      separator = ", ";
+      writer.print(tagName);
+      writer.print("=");
+      writer.print(outRec.getTag(tagName));
+    }
+    for (String metricName : outRec.getMetricNames()) {
+      writer.print(separator);
+      separator = ", ";
+      writer.print(metricName);
+      writer.print("=");
+      writer.print(outRec.getMetric(metricName));
+    }
+    writer.println();
+  }
+
+  @Override
+  public void flush() {
+    writer.flush();
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java
new file mode 100644
index 0000000..392cbf9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ 
+
+package org.apache.hadoop.hbase.metrics.histogram;
+
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.util.MetricsBase;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+
+import com.yammer.metrics.stats.Sample;
+import com.yammer.metrics.stats.Snapshot;
+import com.yammer.metrics.stats.UniformSample;
+import com.yammer.metrics.stats.ExponentiallyDecayingSample;
+
+public class MetricsHistogram extends MetricsBase {
+  
+  // 1028 items implies 99.9% CI w/ 5% margin of error 
+  // (assuming a normal distribution on the underlying data)
+  private static final int DEFAULT_SAMPLE_SIZE = 1028;
+
+  // the bias towards sampling from more recent data. 
+  // Per Cormode et al. an alpha of 0.015 strongly biases to the last 5 minutes
+  private static final double DEFAULT_ALPHA = 0.015;
+  public static final String NUM_OPS_METRIC_NAME = "_num_ops";
+  public static final String MIN_METRIC_NAME = "_min";
+  public static final String MAX_METRIC_NAME = "_max";
+  public static final String MEAN_METRIC_NAME = "_mean";
+  public static final String STD_DEV_METRIC_NAME = "_std_dev";
+  public static final String MEDIAN_METRIC_NAME = "_median";
+  public static final String SEVENTY_FIFTH_PERCENTILE_METRIC_NAME = "_75th_percentile";
+  public static final String NINETY_FIFTH_PERCENTILE_METRIC_NAME = "_95th_percentile";
+  public static final String NINETY_NINETH_PERCENTILE_METRIC_NAME = "_99th_percentile";
+
+  /**
+   * Constructor to create a new histogram metric
+   * @param nam           the name to publish the metric under
+   * @param registry      where the metrics object will be registered
+   * @param description   the metric's description
+   * @param forwardBiased true if you want this histogram to give more 
+   *                      weight to recent data, 
+   *                      false if you want all data to have uniform weight
+   */
+  public MetricsHistogram(final String nam, final MetricsRegistry registry, 
+      final String description, boolean forwardBiased) {
+    super(nam, description);
+
+    this.min = new AtomicLong();
+    this.max = new AtomicLong();
+    this.sum = new AtomicLong();
+    this.sample = forwardBiased ? 
+        new ExponentiallyDecayingSample(DEFAULT_SAMPLE_SIZE, DEFAULT_ALPHA) 
+    : new UniformSample(DEFAULT_SAMPLE_SIZE);
+
+    this.variance =  new AtomicReference(new double[]{-1, 0});
+    this.count = new AtomicLong();
+
+    this.clear();
+
+    if (registry != null) {
+      registry.add(nam, this);      
+    }
+  }
+
+  /**
+   * Constructor create a new (forward biased) histogram metric
+   * @param nam         the name to publish the metric under
+   * @param registry    where the metrics object will be registered
+   * @param description the metric's description
+   */
+  public MetricsHistogram(final String nam, MetricsRegistry registry, 
+      final String description) {
+    this(nam, registry, NO_DESCRIPTION, true);
+  }
+    
+  /**
+   * Constructor - create a new (forward biased) histogram metric
+   * @param nam the name of the metrics to be used to publish the metric
+   * @param registry - where the metrics object will be registered
+   */
+  public MetricsHistogram(final String nam, MetricsRegistry registry) {
+    this(nam, registry, NO_DESCRIPTION);
+  }
+
+  private final Sample sample;
+  private final AtomicLong min;
+  private final AtomicLong max;
+  private final AtomicLong sum;
+
+  // these are for computing a running-variance, 
+  // without letting floating point errors accumulate via Welford's algorithm
+  private final AtomicReference variance;
+  private final AtomicLong count;
+
+  /**
+   * Clears all recorded values.
+   */
+  public void clear() {
+    this.sample.clear();
+    this.count.set(0);
+    this.max.set(Long.MIN_VALUE);
+    this.min.set(Long.MAX_VALUE);
+    this.sum.set(0);
+    variance.set(new double[]{-1, 0});
+  }
+
+  public void update(int val) {
+    update((long) val);
+  }
+
+  public void update(final long val) {
+    count.incrementAndGet();
+    sample.update(val);
+    setMax(val);
+    setMin(val);
+    sum.getAndAdd(val);
+    updateVariance(val);
+  }
+
+  private void setMax(final long potentialMax) {
+    boolean done = false;
+    while (!done) {
+      final long currentMax = max.get();
+      done = currentMax >= potentialMax 
+          || max.compareAndSet(currentMax, potentialMax);
+    }
+  }
+
+  private void setMin(long potentialMin) {
+    boolean done = false;
+    while (!done) {
+      final long currentMin = min.get();
+      done = currentMin <= potentialMin 
+          || min.compareAndSet(currentMin, potentialMin);
+    }
+  }
+
+  private void updateVariance(long value) {
+    boolean done = false;
+    while (!done) {
+      final double[] oldValues = variance.get();
+      final double[] newValues = new double[2];
+      if (oldValues[0] == -1) {
+        newValues[0] = value;
+        newValues[1] = 0;
+      } else {
+        final double oldM = oldValues[0];
+        final double oldS = oldValues[1];
+
+        final double newM = oldM + ((value - oldM) / getCount());
+        final double newS = oldS + ((value - oldM) * (value - newM));
+
+        newValues[0] = newM;
+        newValues[1] = newS;
+      }
+      done = variance.compareAndSet(oldValues, newValues);
+    }
+  }
+
+
+  public long getCount() {
+    return count.get();
+  }
+
+  public long getMax() {
+    if (getCount() > 0) {
+      return max.get();
+    }
+    return 0L;
+  }
+
+  public long getMin() {
+    if (getCount() > 0) {
+      return min.get();
+    }
+    return 0L;
+  }
+
+  public double getMean() {
+    if (getCount() > 0) {
+      return sum.get() / (double) getCount();
+    }
+    return 0.0;
+  }
+
+  public double getStdDev() {
+    if (getCount() > 0) {
+      return Math.sqrt(getVariance());
+    }
+    return 0.0;
+  }
+
+  public Snapshot getSnapshot() {
+    return sample.getSnapshot();
+  }
+
+  private double getVariance() {
+    if (getCount() <= 1) {
+      return 0.0;
+    }
+    return variance.get()[1] / (getCount() - 1);
+  }
+
+  @Override
+  public void pushMetric(MetricsRecord mr) {
+    final Snapshot s = this.getSnapshot();
+    mr.setMetric(getName() + NUM_OPS_METRIC_NAME, this.getCount());
+    mr.setMetric(getName() + MIN_METRIC_NAME, this.getMin());
+    mr.setMetric(getName() + MAX_METRIC_NAME, this.getMax());
+
+    mr.setMetric(getName() + MEAN_METRIC_NAME, (float) this.getMean());
+    mr.setMetric(getName() + STD_DEV_METRIC_NAME, (float) this.getStdDev());
+
+    mr.setMetric(getName() + MEDIAN_METRIC_NAME, (float) s.getMedian());
+    mr.setMetric(getName() + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME,
+        (float) s.get75thPercentile());
+    mr.setMetric(getName() + NINETY_FIFTH_PERCENTILE_METRIC_NAME,
+        (float) s.get95thPercentile());
+    mr.setMetric(getName() + NINETY_NINETH_PERCENTILE_METRIC_NAME,
+        (float) s.get99thPercentile());
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/migration/HRegionInfo090x.java b/src/main/java/org/apache/hadoop/hbase/migration/HRegionInfo090x.java
new file mode 100644
index 0000000..eeb18e8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/migration/HRegionInfo090x.java
@@ -0,0 +1,680 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.migration;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.KVComparator;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JenkinsHash;
+import org.apache.hadoop.hbase.util.MD5Hash;
+import org.apache.hadoop.io.VersionedWritable;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * HRegion information.
+ * Contains HRegion id, start and end keys, a reference to this
+ * HRegions' table descriptor, etc.
+ */
+public class HRegionInfo090x extends VersionedWritable implements
+    WritableComparable{
+  private static final byte VERSION = 0;
+  private static final Log LOG = LogFactory.getLog(HRegionInfo090x.class);
+
+  /**
+   * The new format for a region name contains its encodedName at the end.
+   * The encoded name also serves as the directory name for the region
+   * in the filesystem.
+   *
+   * New region name format:
+   *    <tablename>,,<startkey>,<regionIdTimestamp>.<encodedName>.
+   * where,
+   *    <encodedName> is a hex version of the MD5 hash of
+   *    <tablename>,<startkey>,<regionIdTimestamp>
+   *
+   * The old region name format:
+   *    <tablename>,<startkey>,<regionIdTimestamp>
+   * For region names in the old format, the encoded name is a 32-bit
+   * JenkinsHash integer value (in its decimal notation, string form).
+   *
        
+   * **NOTE**
+   *
+   * ROOT, the first META region, and regions created by an older
+   * version of HBase (0.20 or prior) will continue to use the
+   * old region name format.
+   */
+
+  /** Separator used to demarcate the encodedName in a region name
+   * in the new format. See description on new format above.
+   */
+  private static final int ENC_SEPARATOR = '.';
+  public  static final int MD5_HEX_LENGTH   = 32;
+
+  /**
+   * Does region name contain its encoded name?
+   * @param regionName region name
+   * @return boolean indicating if this a new format region
+   *         name which contains its encoded name.
+   */
+  private static boolean hasEncodedName(final byte[] regionName) {
+    // check if region name ends in ENC_SEPARATOR
+    if ((regionName.length >= 1)
+        && (regionName[regionName.length - 1] == ENC_SEPARATOR)) {
+      // region name is new format. it contains the encoded name.
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * @param regionName
+   * @return the encodedName
+   */
+  public static String encodeRegionName(final byte [] regionName) {
+    String encodedName;
+    if (hasEncodedName(regionName)) {
+      // region is in new format:
+      // ,,/encodedName/
+      encodedName = Bytes.toString(regionName,
+          regionName.length - MD5_HEX_LENGTH - 1,
+          MD5_HEX_LENGTH);
+    } else {
+      // old format region name. ROOT and first META region also
+      // use this format.EncodedName is the JenkinsHash value.
+      int hashVal = Math.abs(JenkinsHash.getInstance().hash(regionName,
+        regionName.length, 0));
+      encodedName = String.valueOf(hashVal);
+    }
+    return encodedName;
+  }
+
+  /**
+   * Use logging.
+   * @param encodedRegionName The encoded regionname.
+   * @return -ROOT- if passed 70236052 or
+   * .META. if passed 1028785192 else returns
+   * encodedRegionName
+   */
+  public static String prettyPrint(final String encodedRegionName) {
+    if (encodedRegionName.equals("70236052")) {
+      return encodedRegionName + "/-ROOT-";
+    } else if (encodedRegionName.equals("1028785192")) {
+      return encodedRegionName + "/.META.";
+    }
+    return encodedRegionName;
+  }
+
+  /** delimiter used between portions of a region name */
+  public static final int DELIMITER = ',';
+
+  /** HRegionInfo for root region */
+  public static final HRegionInfo090x ROOT_REGIONINFO =
+    new HRegionInfo090x(0L, HTableDescriptor.ROOT_TABLEDESC);
+
+  /** HRegionInfo for first meta region */
+  public static final HRegionInfo090x FIRST_META_REGIONINFO =
+    new HRegionInfo090x(1L, HTableDescriptor.META_TABLEDESC);
+
+  private byte [] endKey = HConstants.EMPTY_BYTE_ARRAY;
+  // This flag is in the parent of a split while the parent is still referenced
+  // by daughter regions.  We USED to set this flag when we disabled a table
+  // but now table state is kept up in zookeeper as of 0.90.0 HBase.
+  private boolean offLine = false;
+  private long regionId = -1;
+  private transient byte [] regionName = HConstants.EMPTY_BYTE_ARRAY;
+  private String regionNameStr = "";
+  private boolean split = false;
+  private byte [] startKey = HConstants.EMPTY_BYTE_ARRAY;
+  protected HTableDescriptor tableDesc = null;
+  private int hashCode = -1;
+  //TODO: Move NO_HASH to HStoreFile which is really the only place it is used.
+  public static final String NO_HASH = null;
+  private volatile String encodedName = NO_HASH;
+  private byte [] encodedNameAsBytes = null;
+
+  private void setHashCode() {
+    int result = Arrays.hashCode(this.regionName);
+    result ^= this.regionId;
+    result ^= Arrays.hashCode(this.startKey);
+    result ^= Arrays.hashCode(this.endKey);
+    result ^= Boolean.valueOf(this.offLine).hashCode();
+    result ^= this.tableDesc.hashCode();
+    this.hashCode = result;
+  }
+
+  /**
+   * Private constructor used constructing HRegionInfo for the catalog root and
+   * first meta regions
+   */
+  private HRegionInfo090x(long regionId, HTableDescriptor tableDesc) {
+    super();
+    this.regionId = regionId;
+    this.tableDesc = tableDesc;
+
+    // Note: Root & First Meta regions names are still in old format
+    this.regionName = createRegionName(tableDesc.getName(), null,
+                                       regionId, false);
+    this.regionNameStr = Bytes.toStringBinary(this.regionName);
+    setHashCode();
+  }
+
+  /** Default constructor - creates empty object */
+  public HRegionInfo090x() {
+    super();
+    this.tableDesc = new HTableDescriptor();
+  }
+
+  /**
+   * Construct HRegionInfo with explicit parameters
+   *
+   * @param tableDesc the table descriptor
+   * @param startKey first key in region
+   * @param endKey end of key range
+   * @throws IllegalArgumentException
+   */
+  public HRegionInfo090x(final HTableDescriptor tableDesc, final byte[] startKey,
+                         final byte[] endKey)
+  throws IllegalArgumentException {
+    this(tableDesc, startKey, endKey, false);
+  }
+
+  /**
+   * Construct HRegionInfo with explicit parameters
+   *
+   * @param tableDesc the table descriptor
+   * @param startKey first key in region
+   * @param endKey end of key range
+   * @param split true if this region has split and we have daughter regions
+   * regions that may or may not hold references to this region.
+   * @throws IllegalArgumentException
+   */
+  public HRegionInfo090x(HTableDescriptor tableDesc, final byte[] startKey,
+                         final byte[] endKey, final boolean split)
+  throws IllegalArgumentException {
+    this(tableDesc, startKey, endKey, split, System.currentTimeMillis());
+  }
+
+  /**
+   * Construct HRegionInfo with explicit parameters
+   *
+   * @param tableDesc the table descriptor
+   * @param startKey first key in region
+   * @param endKey end of key range
+   * @param split true if this region has split and we have daughter regions
+   * regions that may or may not hold references to this region.
+   * @param regionid Region id to use.
+   * @throws IllegalArgumentException
+   */
+  public HRegionInfo090x(HTableDescriptor tableDesc, final byte[] startKey,
+                         final byte[] endKey, final boolean split, final long regionid)
+  throws IllegalArgumentException {
+    super();
+    if (tableDesc == null) {
+      throw new IllegalArgumentException("tableDesc cannot be null");
+    }
+    this.offLine = false;
+    this.regionId = regionid;
+    this.regionName = createRegionName(tableDesc.getName(), startKey, regionId, true);
+    this.regionNameStr = Bytes.toStringBinary(this.regionName);
+    this.split = split;
+    this.endKey = endKey == null? HConstants.EMPTY_END_ROW: endKey.clone();
+    this.startKey = startKey == null?
+      HConstants.EMPTY_START_ROW: startKey.clone();
+    this.tableDesc = tableDesc;
+    setHashCode();
+  }
+
+  /**
+   * Costruct a copy of another HRegionInfo
+   *
+   * @param other
+   */
+  public HRegionInfo090x(HRegionInfo090x other) {
+    super();
+    this.endKey = other.getEndKey();
+    this.offLine = other.isOffline();
+    this.regionId = other.getRegionId();
+    this.regionName = other.getRegionName();
+    this.regionNameStr = Bytes.toStringBinary(this.regionName);
+    this.split = other.isSplit();
+    this.startKey = other.getStartKey();
+    this.tableDesc = other.getTableDesc();
+    this.hashCode = other.hashCode();
+    this.encodedName = other.getEncodedName();
+  }
+
+  /**
+   * Make a region name of passed parameters.
+   * @param tableName
+   * @param startKey Can be null
+   * @param regionid Region id (Usually timestamp from when region was created).
+   * @param newFormat should we create the region name in the new format
+   *                  (such that it contains its encoded name?).
+   * @return Region name made of passed tableName, startKey and id
+   */
+  public static byte [] createRegionName(final byte [] tableName,
+      final byte [] startKey, final long regionid, boolean newFormat) {
+    return createRegionName(tableName, startKey, Long.toString(regionid), newFormat);
+  }
+
+  /**
+   * Make a region name of passed parameters.
+   * @param tableName
+   * @param startKey Can be null
+   * @param id Region id (Usually timestamp from when region was created).
+   * @param newFormat should we create the region name in the new format
+   *                  (such that it contains its encoded name?).
+   * @return Region name made of passed tableName, startKey and id
+   */
+  public static byte [] createRegionName(final byte [] tableName,
+      final byte [] startKey, final String id, boolean newFormat) {
+    return createRegionName(tableName, startKey, Bytes.toBytes(id), newFormat);
+  }
+
+  /**
+   * Make a region name of passed parameters.
+   * @param tableName
+   * @param startKey Can be null
+   * @param id Region id (Usually timestamp from when region was created).
+   * @param newFormat should we create the region name in the new format
+   *                  (such that it contains its encoded name?).
+   * @return Region name made of passed tableName, startKey and id
+   */
+  public static byte [] createRegionName(final byte [] tableName,
+      final byte [] startKey, final byte [] id, boolean newFormat) {
+    byte [] b = new byte [tableName.length + 2 + id.length +
+       (startKey == null? 0: startKey.length) +
+       (newFormat ? (MD5_HEX_LENGTH + 2) : 0)];
+
+    int offset = tableName.length;
+    System.arraycopy(tableName, 0, b, 0, offset);
+    b[offset++] = DELIMITER;
+    if (startKey != null && startKey.length > 0) {
+      System.arraycopy(startKey, 0, b, offset, startKey.length);
+      offset += startKey.length;
+    }
+    b[offset++] = DELIMITER;
+    System.arraycopy(id, 0, b, offset, id.length);
+    offset += id.length;
+
+    if (newFormat) {
+      //
+      // Encoded name should be built into the region name.
+      //
+      // Use the region name thus far (namely, ,,)
+      // to compute a MD5 hash to be used as the encoded name, and append
+      // it to the byte buffer.
+      //
+      String md5Hash = MD5Hash.getMD5AsHex(b, 0, offset);
+      byte [] md5HashBytes = Bytes.toBytes(md5Hash);
+
+      if (md5HashBytes.length != MD5_HEX_LENGTH) {
+        LOG.error("MD5-hash length mismatch: Expected=" + MD5_HEX_LENGTH +
+                  "; Got=" + md5HashBytes.length);
+      }
+
+      // now append the bytes '..' to the end
+      b[offset++] = ENC_SEPARATOR;
+      System.arraycopy(md5HashBytes, 0, b, offset, MD5_HEX_LENGTH);
+      offset += MD5_HEX_LENGTH;
+      b[offset++] = ENC_SEPARATOR;
+    }
+
+    return b;
+  }
+
+  /**
+   * Gets the table name from the specified region name.
+   * @param regionName
+   * @return Table name.
+   */
+  public static byte [] getTableName(byte [] regionName) {
+    int offset = -1;
+    for (int i = 0; i < regionName.length; i++) {
+      if (regionName[i] == DELIMITER) {
+        offset = i;
+        break;
+      }
+    }
+    byte [] tableName = new byte[offset];
+    System.arraycopy(regionName, 0, tableName, 0, offset);
+    return tableName;
+  }
+
+  /**
+   * Separate elements of a regionName.
+   * @param regionName
+   * @return Array of byte[] containing tableName, startKey and id
+   * @throws IOException
+   */
+  public static byte [][] parseRegionName(final byte [] regionName)
+  throws IOException {
+    int offset = -1;
+    for (int i = 0; i < regionName.length; i++) {
+      if (regionName[i] == DELIMITER) {
+        offset = i;
+        break;
+      }
+    }
+    if(offset == -1) throw new IOException("Invalid regionName format");
+    byte [] tableName = new byte[offset];
+    System.arraycopy(regionName, 0, tableName, 0, offset);
+    offset = -1;
+    for (int i = regionName.length - 1; i > 0; i--) {
+      if(regionName[i] == DELIMITER) {
+        offset = i;
+        break;
+      }
+    }
+    if(offset == -1) throw new IOException("Invalid regionName format");
+    byte [] startKey = HConstants.EMPTY_BYTE_ARRAY;
+    if(offset != tableName.length + 1) {
+      startKey = new byte[offset - tableName.length - 1];
+      System.arraycopy(regionName, tableName.length + 1, startKey, 0,
+          offset - tableName.length - 1);
+    }
+    byte [] id = new byte[regionName.length - offset - 1];
+    System.arraycopy(regionName, offset + 1, id, 0,
+        regionName.length - offset - 1);
+    byte [][] elements = new byte[3][];
+    elements[0] = tableName;
+    elements[1] = startKey;
+    elements[2] = id;
+    return elements;
+  }
+
+  /** @return the regionId */
+  public long getRegionId(){
+    return regionId;
+  }
+
+  /**
+   * @return the regionName as an array of bytes.
+   * @see #getRegionNameAsString()
+   */
+  public byte [] getRegionName(){
+    return regionName;
+  }
+
+  /**
+   * @return Region name as a String for use in logging, etc.
+   */
+  public String getRegionNameAsString() {
+    if (hasEncodedName(this.regionName)) {
+      // new format region names already have their encoded name.
+      return this.regionNameStr;
+    }
+
+    // old format. regionNameStr doesn't have the region name.
+    //
+    //
+    return this.regionNameStr + "." + this.getEncodedName();
+  }
+
+  /** @return the encoded region name */
+  public synchronized String getEncodedName() {
+    if (this.encodedName == NO_HASH) {
+      this.encodedName = encodeRegionName(this.regionName);
+    }
+    return this.encodedName;
+  }
+
+  public synchronized byte [] getEncodedNameAsBytes() {
+    if (this.encodedNameAsBytes == null) {
+      this.encodedNameAsBytes = Bytes.toBytes(getEncodedName());
+    }
+    return this.encodedNameAsBytes;
+  }
+
+  /** @return the startKey */
+  public byte [] getStartKey(){
+    return startKey;
+  }
+
+  /** @return the endKey */
+  public byte [] getEndKey(){
+    return endKey;
+  }
+
+  /**
+   * Returns true if the given inclusive range of rows is fully contained
+   * by this region. For example, if the region is foo,a,g and this is
+   * passed ["b","c"] or ["a","c"] it will return true, but if this is passed
+   * ["b","z"] it will return false.
+   * @throws IllegalArgumentException if the range passed is invalid (ie end < start)
+   */
+  public boolean containsRange(byte[] rangeStartKey, byte[] rangeEndKey) {
+    if (Bytes.compareTo(rangeStartKey, rangeEndKey) > 0) {
+      throw new IllegalArgumentException(
+      "Invalid range: " + Bytes.toStringBinary(rangeStartKey) +
+      " > " + Bytes.toStringBinary(rangeEndKey));
+    }
+
+    boolean firstKeyInRange = Bytes.compareTo(rangeStartKey, startKey) >= 0;
+    boolean lastKeyInRange =
+      Bytes.compareTo(rangeEndKey, endKey) < 0 ||
+      Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY);
+    return firstKeyInRange && lastKeyInRange;
+  }
+
+  /**
+   * Return true if the given row falls in this region.
+   */
+  public boolean containsRow(byte[] row) {
+    return Bytes.compareTo(row, startKey) >= 0 &&
+      (Bytes.compareTo(row, endKey) < 0 ||
+       Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY));
+  }
+
+  /** @return the tableDesc */
+  public HTableDescriptor getTableDesc(){
+    return tableDesc;
+  }
+
+  /**
+   * @param newDesc new table descriptor to use
+   */
+  public void setTableDesc(HTableDescriptor newDesc) {
+    this.tableDesc = newDesc;
+  }
+
+  /** @return true if this is the root region */
+  public boolean isRootRegion() {
+    return this.tableDesc.isRootRegion();
+  }
+
+  /** @return true if this region is from a table that is a meta table,
+   * either .META. or -ROOT-
+   */
+  public boolean isMetaTable() {
+    return this.tableDesc.isMetaTable();
+  }
+
+  /** @return true if this region is a meta region */
+  public boolean isMetaRegion() {
+    return this.tableDesc.isMetaRegion();
+  }
+
+  /**
+   * @return True if has been split and has daughters.
+   */
+  public boolean isSplit() {
+    return this.split;
+  }
+
+  /**
+   * @param split set split status
+   */
+  public void setSplit(boolean split) {
+    this.split = split;
+  }
+
+  /**
+   * @return True if this region is offline.
+   */
+  public boolean isOffline() {
+    return this.offLine;
+  }
+
+  /**
+   * The parent of a region split is offline while split daughters hold
+   * references to the parent. Offlined regions are closed.
+   * @param offLine Set online/offline status.
+   */
+  public void setOffline(boolean offLine) {
+    this.offLine = offLine;
+  }
+
+
+  /**
+   * @return True if this is a split parent region.
+   */
+  public boolean isSplitParent() {
+    if (!isSplit()) return false;
+    if (!isOffline()) {
+      LOG.warn("Region is split but NOT offline: " + getRegionNameAsString());
+    }
+    return true;
+  }
+
+  /**
+   * @see java.lang.Object#toString()
+   */
+  @Override
+  public String toString() {
+    return "REGION => {" + HConstants.NAME + " => '" +
+      this.regionNameStr +
+      "', STARTKEY => '" +
+      Bytes.toStringBinary(this.startKey) + "', ENDKEY => '" +
+      Bytes.toStringBinary(this.endKey) +
+      "', ENCODED => " + getEncodedName() + "," +
+      (isOffline()? " OFFLINE => true,": "") +
+      (isSplit()? " SPLIT => true,": "") +
+      " TABLE => {" + this.tableDesc.toString() + "}";
+  }
+
+  /**
+   * @see java.lang.Object#equals(java.lang.Object)
+   */
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null) {
+      return false;
+    }
+    if (!(o instanceof HRegionInfo090x)) {
+      return false;
+    }
+    return this.compareTo((HRegionInfo090x)o) == 0;
+  }
+
+  /**
+   * @see java.lang.Object#hashCode()
+   */
+  @Override
+  public int hashCode() {
+    return this.hashCode;
+  }
+
+  /** @return the object version number */
+  @Override
+  public byte getVersion() {
+    return VERSION;
+  }
+
+  //
+  // Writable
+  //
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    super.write(out);
+    Bytes.writeByteArray(out, endKey);
+    out.writeBoolean(offLine);
+    out.writeLong(regionId);
+    Bytes.writeByteArray(out, regionName);
+    out.writeBoolean(split);
+    Bytes.writeByteArray(out, startKey);
+    tableDesc.write(out);
+    out.writeInt(hashCode);
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    super.readFields(in);
+    this.endKey = Bytes.readByteArray(in);
+    this.offLine = in.readBoolean();
+    this.regionId = in.readLong();
+    this.regionName = Bytes.readByteArray(in);
+    this.regionNameStr = Bytes.toStringBinary(this.regionName);
+    this.split = in.readBoolean();
+    this.startKey = Bytes.readByteArray(in);
+    try {
+      this.tableDesc.readFields(in);
+    } catch(EOFException eofe) {
+       throw new IOException("HTD not found in input buffer");
+    }
+    this.hashCode = in.readInt();
+  }
+
+  //
+  // Comparable
+  //
+
+  public int compareTo(HRegionInfo090x o) {
+    if (o == null) {
+      return 1;
+    }
+
+    // Are regions of same table?
+    int result = Bytes.compareTo(this.tableDesc.getName(), o.tableDesc.getName());
+    if (result != 0) {
+      return result;
+    }
+
+    // Compare start keys.
+    result = Bytes.compareTo(this.startKey, o.startKey);
+    if (result != 0) {
+      return result;
+    }
+
+    // Compare end keys.
+    return Bytes.compareTo(this.endKey, o.endKey);
+  }
+
+  /**
+   * @return Comparator to use comparing {@link org.apache.hadoop.hbase.KeyValue}s.
+   */
+  public KVComparator getComparator() {
+    return isRootRegion()? KeyValue.ROOT_COMPARATOR: isMetaRegion()?
+      KeyValue.META_COMPARATOR: KeyValue.COMPARATOR;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java b/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java
new file mode 100644
index 0000000..d121ee1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java
@@ -0,0 +1,95 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.monitoring;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.nio.channels.FileChannel;
+import java.util.Enumeration;
+import java.util.Set;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.log4j.Appender;
+import org.apache.log4j.FileAppender;
+import org.apache.log4j.Logger;
+
+import com.google.common.collect.Sets;
+
+/**
+ * Utility functions for reading the log4j logs that are
+ * being written by HBase.
+ */
+public abstract class LogMonitoring {
+  public static Set getActiveLogFiles() throws IOException {
+    Set ret = Sets.newHashSet();
+    Appender a;
+    @SuppressWarnings("unchecked")
+    Enumeration e = Logger.getRootLogger().getAllAppenders();
+    while (e.hasMoreElements()) {
+      a = e.nextElement();
+      if (a instanceof FileAppender) {
+        FileAppender fa = (FileAppender) a;
+        String filename = fa.getFile();
+        ret.add(new File(filename));
+      }
+    }
+    return ret;
+  }
+  
+
+  public static void dumpTailOfLogs(
+      PrintWriter out, long tailKb) throws IOException {
+    Set logs = LogMonitoring.getActiveLogFiles();
+    for (File f : logs) {
+      out.println("+++++++++++++++++++++++++++++++");
+      out.println(f.getAbsolutePath());
+      out.println("+++++++++++++++++++++++++++++++");
+      try {
+        dumpTailOfLog(f, out, tailKb);
+      } catch (IOException ioe) {
+        out.println("Unable to dump log at " + f);
+        ioe.printStackTrace(out);
+      }
+      out.println("\n\n");
+    }
+  }
+
+  private static void dumpTailOfLog(File f, PrintWriter out, long tailKb)
+      throws IOException {
+    FileInputStream fis = new FileInputStream(f);
+    try {
+      FileChannel channel = fis.getChannel();
+      channel.position(Math.max(0, channel.size() - tailKb*1024));
+      BufferedReader r = new BufferedReader(
+          new InputStreamReader(fis));
+      r.readLine(); // skip the first partial line
+      String line;
+      while ((line = r.readLine()) != null) {
+        out.println(line);
+      }
+    } finally {
+      IOUtils.closeStream(fis);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/MemoryBoundedLogMessageBuffer.java b/src/main/java/org/apache/hadoop/hbase/monitoring/MemoryBoundedLogMessageBuffer.java
new file mode 100644
index 0000000..e8b7416
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/MemoryBoundedLogMessageBuffer.java
@@ -0,0 +1,114 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.monitoring;
+
+import java.io.PrintWriter;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.LinkedList;
+import java.util.List;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+/**
+ * A size-bounded repository of alerts, which are kept
+ * in a linked list. Alerts can be added, and they will
+ * automatically be removed one by one when the specified heap
+ * usage is exhausted.
+ */
+public class MemoryBoundedLogMessageBuffer {
+  private final long maxSizeBytes;
+  private long usage = 0;
+  private LinkedList messages;
+  
+  public MemoryBoundedLogMessageBuffer(long maxSizeBytes) {
+    Preconditions.checkArgument(
+        maxSizeBytes > 0);
+    this.maxSizeBytes = maxSizeBytes;
+    this.messages = Lists.newLinkedList();
+  }
+  
+  /**
+   * Append the given message to this buffer, automatically evicting
+   * older messages until the desired memory limit is achieved.
+   */
+  public synchronized void add(String messageText) {
+    LogMessage message = new LogMessage(messageText, System.currentTimeMillis());
+    
+    usage += message.estimateHeapUsage();
+    messages.add(message);
+    while (usage > maxSizeBytes) {
+      LogMessage removed = messages.remove();
+      usage -= removed.estimateHeapUsage();
+      assert usage >= 0;
+    }
+  }
+  
+  /**
+   * Dump the contents of the buffer to the given stream.
+   */
+  public synchronized void dumpTo(PrintWriter out) {
+    SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+
+    for (LogMessage msg : messages) {
+      out.write(df.format(new Date(msg.timestamp)));
+      out.write(" ");
+      out.println(new String(msg.message, Charsets.UTF_8));
+    }
+  }
+  
+  synchronized List getMessages() {
+    // defensive copy
+    return Lists.newArrayList(messages);
+  }
+ 
+  /**
+   * Estimate the number of bytes this buffer is currently
+   * using.
+   */
+  synchronized long estimateHeapUsage() {
+    return usage;
+  }
+  
+  private static class LogMessage {
+    /** the error text, encoded in bytes to save memory */
+    public final byte[] message;
+    public final long timestamp;
+    
+    /**
+     * Completely non-scientific estimate of how much one of these
+     * objects takes, along with the LinkedList overhead. This doesn't
+     * need to be exact, since we don't expect a ton of these alerts.
+     */
+    private static final long BASE_USAGE=100;
+    
+    public LogMessage(String message, long timestamp) {
+      this.message = message.getBytes(Charsets.UTF_8);
+      this.timestamp = timestamp;
+    }
+    
+    public long estimateHeapUsage() {
+      return message.length + BASE_USAGE;
+    }
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandler.java b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandler.java
new file mode 100644
index 0000000..d4f9714
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandler.java
@@ -0,0 +1,44 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.monitoring;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * A MonitoredTask implementation optimized for use with RPC Handlers 
+ * handling frequent, short duration tasks. String concatenations and object 
+ * allocations are avoided in methods that will be hit by every RPC call.
+ */
+public interface MonitoredRPCHandler extends MonitoredTask {
+  public abstract String getRPC();
+  public abstract String getRPC(boolean withParams);
+  public abstract long getRPCPacketLength();
+  public abstract String getClient();
+  public abstract long getRPCStartTime();
+  public abstract long getRPCQueueTime();
+  public abstract boolean isRPCRunning();
+  public abstract boolean isOperationRunning();
+  
+  public abstract void setRPC(String methodName, Object [] params,
+      long queueTime);
+  public abstract void setRPCPacket(Writable param);
+  public abstract void setConnection(String clientAddress, int remotePort);
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandlerImpl.java b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandlerImpl.java
new file mode 100644
index 0000000..d68468b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandlerImpl.java
@@ -0,0 +1,263 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.monitoring;
+
+import org.apache.hadoop.hbase.client.Operation;
+import org.apache.hadoop.hbase.io.WritableWithSize;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
+
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A MonitoredTask implementation designed for use with RPC Handlers 
+ * handling frequent, short duration tasks. String concatenations and object 
+ * allocations are avoided in methods that will be hit by every RPC call.
+ */
+public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl 
+  implements MonitoredRPCHandler {
+  private String clientAddress;
+  private int remotePort;
+  private long rpcQueueTime;
+  private long rpcStartTime;
+  private String methodName = "";
+  private Object [] params = {};
+  private Writable packet;
+
+  public MonitoredRPCHandlerImpl() {
+    super();
+    // in this implementation, WAITING indicates that the handler is not 
+    // actively servicing an RPC call.
+    setState(State.WAITING);
+  }
+
+  @Override
+  public synchronized MonitoredRPCHandlerImpl clone() {
+    return (MonitoredRPCHandlerImpl) super.clone();
+  }
+
+  /**
+   * Gets the status of this handler; if it is currently servicing an RPC, 
+   * this status will include the RPC information.
+   * @return a String describing the current status.
+   */
+  @Override
+  public String getStatus() {
+    if (getState() != State.RUNNING) {
+      return super.getStatus();
+    }
+    return super.getStatus() + " from " + getClient() + ": " + getRPC();
+  }
+
+  /**
+   * Accesses the queue time for the currently running RPC on the 
+   * monitored Handler.
+   * @return the queue timestamp or -1 if there is no RPC currently running.
+   */
+  public long getRPCQueueTime() {
+    if (getState() != State.RUNNING) {
+      return -1;
+    }
+    return rpcQueueTime;
+  }
+
+  /**
+   * Accesses the start time for the currently running RPC on the 
+   * monitored Handler.
+   * @return the start timestamp or -1 if there is no RPC currently running.
+   */
+  public long getRPCStartTime() {
+    if (getState() != State.RUNNING) {
+      return -1;
+    }
+    return rpcStartTime;
+  }
+
+  /**
+   * Produces a string representation of the method currently being serviced
+   * by this Handler.
+   * @return a string representing the method call without parameters
+   */
+  public String getRPC() {
+    return getRPC(false);
+  }
+
+  /**
+   * Produces a string representation of the method currently being serviced
+   * by this Handler.
+   * @param withParams toggle inclusion of parameters in the RPC String
+   * @return A human-readable string representation of the method call.
+   */
+  public synchronized String getRPC(boolean withParams) {
+    if (getState() != State.RUNNING) {
+      // no RPC is currently running
+      return "";
+    }
+    StringBuilder buffer = new StringBuilder(256);
+    buffer.append(methodName);
+    if (withParams) {
+      buffer.append("(");
+      for (int i = 0; i < params.length; i++) {
+        if (i != 0)
+          buffer.append(", ");
+        buffer.append(params[i]);
+      }
+      buffer.append(")");
+    }
+    return buffer.toString();
+  }
+
+  /**
+   * Produces a string representation of the method currently being serviced
+   * by this Handler.
+   * @return A human-readable string representation of the method call.
+   */
+  public long getRPCPacketLength() {
+    if (getState() != State.RUNNING || packet == null) {
+      // no RPC is currently running, or we don't have an RPC's packet info
+      return -1L;
+    }
+    if (!(packet instanceof WritableWithSize)) {
+      // the packet passed to us doesn't expose size information
+      return -1L;
+    }
+    return ((WritableWithSize) packet).getWritableSize();
+  }
+
+  /**
+   * If an RPC call is currently running, produces a String representation of 
+   * the connection from which it was received.
+   * @return A human-readable string representation of the address and port 
+   *  of the client.
+   */
+  public String getClient() {
+    return clientAddress + ":" + remotePort;
+  }
+
+  /**
+   * Indicates to the client whether this task is monitoring a currently active 
+   * RPC call.
+   * @return true if the monitored handler is currently servicing an RPC call.
+   */
+  public boolean isRPCRunning() {
+    return getState() == State.RUNNING;
+  }
+
+  /**
+   * Indicates to the client whether this task is monitoring a currently active 
+   * RPC call to a database command. (as defined by 
+   * o.a.h.h.client.Operation)
+   * @return true if the monitored handler is currently servicing an RPC call
+   * to a database command.
+   */
+  public boolean isOperationRunning() {
+    if(!isRPCRunning()) {
+      return false;
+    }
+    for(Object param : params) {
+      if (param instanceof Operation) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Tells this instance that it is monitoring a new RPC call.
+   * @param methodName The name of the method that will be called by the RPC.
+   * @param params The parameters that will be passed to the indicated method.
+   */
+  public synchronized void setRPC(String methodName, Object [] params, 
+      long queueTime) {
+    this.methodName = methodName;
+    this.params = params;
+    this.rpcStartTime = System.currentTimeMillis();
+    this.rpcQueueTime = queueTime;
+    this.state = State.RUNNING;
+  }
+
+  /**
+   * Gives this instance a reference to the Writable received by the RPC, so 
+   * that it can later compute its size if asked for it.
+   * @param param The Writable received by the RPC for this call
+   */
+  public void setRPCPacket(Writable param) {
+    this.packet = param;
+  }
+
+  /**
+   * Registers current handler client details.
+   * @param clientAddress the address of the current client
+   * @param remotePort the port from which the client connected
+   */
+  public void setConnection(String clientAddress, int remotePort) {
+    this.clientAddress = clientAddress;
+    this.remotePort = remotePort;
+  }
+
+  @Override
+  public void markComplete(String status) {
+    super.markComplete(status);
+    this.params = null;
+    this.packet = null;
+  }
+
+  public synchronized Map toMap() {
+    // only include RPC info if the Handler is actively servicing an RPC call
+    Map map = super.toMap();
+    if (getState() != State.RUNNING) {
+      return map;
+    }
+    Map rpcJSON = new HashMap();
+    ArrayList paramList = new ArrayList();
+    map.put("rpcCall", rpcJSON);
+    rpcJSON.put("queuetimems", getRPCQueueTime());
+    rpcJSON.put("starttimems", getRPCStartTime());
+    rpcJSON.put("clientaddress", clientAddress);
+    rpcJSON.put("remoteport", remotePort);
+    rpcJSON.put("packetlength", getRPCPacketLength());
+    rpcJSON.put("method", methodName);
+    rpcJSON.put("params", paramList);
+    for(Object param : params) {
+      if(param instanceof byte []) {
+        paramList.add(Bytes.toStringBinary((byte []) param));
+      } else if (param instanceof Operation) {
+        paramList.add(((Operation) param).toMap());
+      } else {
+        paramList.add(param.toString());
+      }
+    }
+    return map;
+  }
+
+  @Override
+  public String toString() {
+    if (getState() != State.RUNNING) {
+      return super.toString();
+    }
+    return super.toString() + ", rpcMethod=" + getRPC();
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTask.java b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTask.java
new file mode 100644
index 0000000..c59d415
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTask.java
@@ -0,0 +1,76 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.monitoring;
+
+import java.io.IOException;
+import java.util.Map;
+
+public interface MonitoredTask extends Cloneable {
+  enum State {
+    RUNNING,
+    WAITING,
+    COMPLETE,
+    ABORTED;
+  }
+
+  public abstract long getStartTime();
+  public abstract String getDescription();
+  public abstract String getStatus();
+  public abstract long getStatusTime();
+  public abstract State getState();
+  public abstract long getStateTime();
+  public abstract long getCompletionTimestamp();
+
+  public abstract void markComplete(String msg);
+  public abstract void pause(String msg);
+  public abstract void resume(String msg);
+  public abstract void abort(String msg);
+  public abstract void expireNow();
+
+  public abstract void setStatus(String status);
+  public abstract void setDescription(String description);
+
+  /**
+   * Explicitly mark this status as able to be cleaned up,
+   * even though it might not be complete.
+   */
+  public abstract void cleanup();
+
+  /**
+   * Public exposure of Object.clone() in order to allow clients to easily 
+   * capture current state.
+   * @return a copy of the object whose references will not change
+   */
+  public abstract MonitoredTask clone();
+
+  /**
+   * Creates a string map of internal details for extensible exposure of 
+   * monitored tasks.
+   * @return A Map containing information for this task.
+   */
+  public abstract Map toMap() throws IOException;
+
+  /**
+   * Creates a JSON object for parseable exposure of monitored tasks.
+   * @return An encoded JSON object containing information for this task.
+   */
+  public abstract String toJSON() throws IOException;
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
new file mode 100644
index 0000000..394129c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
@@ -0,0 +1,179 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.monitoring;
+
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+class MonitoredTaskImpl implements MonitoredTask {
+  private long startTime;
+  private long statusTime;
+  private long stateTime;
+  
+  private volatile String status;
+  private volatile String description;
+  
+  protected volatile State state = State.RUNNING;
+  
+  public MonitoredTaskImpl() {
+    startTime = System.currentTimeMillis();
+    statusTime = startTime;
+    stateTime = startTime;
+  }
+
+  @Override
+  public synchronized MonitoredTaskImpl clone() {
+    try {
+      return (MonitoredTaskImpl) super.clone();
+    } catch (CloneNotSupportedException e) {
+      throw new AssertionError(); // Won't happen
+    }
+  }
+
+  @Override
+  public long getStartTime() {
+    return startTime;
+  }
+  
+  @Override
+  public String getDescription() {
+    return description;
+  }
+
+  @Override
+  public String getStatus() {
+    return status;
+  }
+
+  @Override
+  public long getStatusTime() {
+    return statusTime;
+  }
+  
+  @Override
+  public State getState() {
+    return state;
+  }
+  
+  @Override
+  public long getStateTime() {
+    return stateTime;
+  }
+  
+  @Override
+  public long getCompletionTimestamp() {
+    if (state == State.COMPLETE || state == State.ABORTED) {
+      return stateTime;
+    }
+    return -1;
+  }
+
+  @Override
+  public void markComplete(String status) {
+    setState(State.COMPLETE);
+    setStatus(status);
+  }
+
+  @Override
+  public void pause(String msg) {
+    setState(State.WAITING);
+    setStatus(msg);
+  }
+
+  @Override
+  public void resume(String msg) {
+    setState(State.RUNNING);
+    setStatus(msg);
+  }
+
+  @Override
+  public void abort(String msg) {
+    setStatus(msg);
+    setState(State.ABORTED);
+  }
+  
+  @Override
+  public void setStatus(String status) {
+    this.status = status;
+    statusTime = System.currentTimeMillis();
+  }
+
+  protected void setState(State state) {
+    this.state = state;
+    stateTime = System.currentTimeMillis();
+  }
+
+  @Override
+  public void setDescription(String description) {
+    this.description = description;
+  }
+
+  @Override
+  public void cleanup() {
+    if (state == State.RUNNING) {
+      setState(State.ABORTED);
+    }
+  }
+
+  /**
+   * Force the completion timestamp backwards so that
+   * it expires now.
+   */
+  public void expireNow() {
+    stateTime -= 180 * 1000;
+  }
+
+  @Override
+  public Map toMap() {
+    Map map = new HashMap();
+    map.put("description", getDescription());
+    map.put("status", getStatus());
+    map.put("state", getState());
+    map.put("starttimems", getStartTime());
+    map.put("statustimems", getCompletionTimestamp());
+    map.put("statetimems", getCompletionTimestamp());
+    return map;
+  }
+
+  @Override
+  public String toJSON() throws IOException {
+    ObjectMapper mapper = new ObjectMapper();
+    return mapper.writeValueAsString(toMap());
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder(512);
+    sb.append(getDescription());
+    sb.append(": status=");
+    sb.append(getStatus());
+    sb.append(", state=");
+    sb.append(getState());
+    sb.append(", startTime=");
+    sb.append(getStartTime());
+    sb.append(", completionTime=");
+    sb.append(getCompletionTimestamp());
+    return sb.toString();
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java
new file mode 100644
index 0000000..604f10d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java
@@ -0,0 +1,62 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.monitoring;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Map;
+
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorStatus;
+import org.apache.hadoop.hbase.util.VersionInfo;
+
+public abstract class StateDumpServlet extends HttpServlet {
+  static final long DEFAULT_TAIL_KB = 100;
+  private static final long serialVersionUID = 1L;
+
+  protected void dumpVersionInfo(PrintWriter out) {
+    VersionInfo.writeTo(out);
+  
+    out.println("Hadoop " + org.apache.hadoop.util.VersionInfo.getVersion());
+    out.println("Subversion " + org.apache.hadoop.util.VersionInfo.getUrl() + " -r " +
+        org.apache.hadoop.util.VersionInfo.getRevision());
+    out.println("Compiled by " + org.apache.hadoop.util.VersionInfo.getUser() +
+        " on " + org.apache.hadoop.util.VersionInfo.getDate());
+  }
+
+  protected long getTailKbParam(HttpServletRequest request) {
+    String param = request.getParameter("tailkb");
+    if (param == null) {
+      return DEFAULT_TAIL_KB;
+    }
+    return Long.parseLong(param);
+  }
+
+  protected void dumpExecutors(ExecutorService service, PrintWriter out)
+      throws IOException {
+    Map statuses = service.getAllExecutorStatuses();
+    for (ExecutorStatus status : statuses.values()) {
+      status.dumpTo(out, "  ");
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java b/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
new file mode 100644
index 0000000..942b7c1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
@@ -0,0 +1,221 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.monitoring;
+
+import java.io.PrintWriter;
+import java.lang.ref.WeakReference;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+
+/**
+ * Singleton which keeps track of tasks going on in this VM.
+ * A Task here is anything which takes more than a few seconds
+ * and the user might want to inquire about the status
+ */
+public class TaskMonitor {
+  private static final Log LOG = LogFactory.getLog(TaskMonitor.class);
+
+  // Don't keep around any tasks that have completed more than
+  // 60 seconds ago
+  private static final long EXPIRATION_TIME = 60*1000;
+
+  @VisibleForTesting
+  static final int MAX_TASKS = 1000;
+  
+  private static TaskMonitor instance;
+  private List tasks =
+    Lists.newArrayList();
+
+  /**
+   * Get singleton instance.
+   * TODO this would be better off scoped to a single daemon
+   */
+  public static synchronized TaskMonitor get() {
+    if (instance == null) {
+      instance = new TaskMonitor();
+    }
+    return instance;
+  }
+  
+  public synchronized MonitoredTask createStatus(String description) {
+    MonitoredTask stat = new MonitoredTaskImpl();
+    stat.setDescription(description);
+    MonitoredTask proxy = (MonitoredTask) Proxy.newProxyInstance(
+        stat.getClass().getClassLoader(),
+        new Class[] { MonitoredTask.class },
+        new PassthroughInvocationHandler(stat));
+    TaskAndWeakRefPair pair = new TaskAndWeakRefPair(stat, proxy);
+    synchronized (this) {
+      tasks.add(pair);
+    }
+    return proxy;
+  }
+
+  public synchronized MonitoredRPCHandler createRPCStatus(String description) {
+    MonitoredRPCHandler stat = new MonitoredRPCHandlerImpl();
+    stat.setDescription(description);
+    MonitoredRPCHandler proxy = (MonitoredRPCHandler) Proxy.newProxyInstance(
+        stat.getClass().getClassLoader(),
+        new Class[] { MonitoredRPCHandler.class },
+        new PassthroughInvocationHandler(stat));
+    TaskAndWeakRefPair pair = new TaskAndWeakRefPair(stat, proxy);
+    synchronized (this) {
+      tasks.add(pair);
+    }
+    return proxy;
+  }
+
+  private synchronized void purgeExpiredTasks() {
+    int size = 0;
+    
+    for (Iterator it = tasks.iterator();
+         it.hasNext();) {
+      TaskAndWeakRefPair pair = it.next();
+      MonitoredTask stat = pair.get();
+      
+      if (pair.isDead()) {
+        // The class who constructed this leaked it. So we can
+        // assume it's done.
+        if (stat.getState() == MonitoredTaskImpl.State.RUNNING) {
+          LOG.warn("Status " + stat + " appears to have been leaked");
+          stat.cleanup();
+        }
+      }
+      
+      if (canPurge(stat)) {
+        it.remove();
+      } else {
+        size++;
+      }
+    }
+    
+    if (size > MAX_TASKS) {
+      LOG.warn("Too many actions in action monitor! Purging some.");
+      tasks = tasks.subList(size - MAX_TASKS, size);
+    }
+  }
+
+  /**
+   * Produces a list containing copies of the current state of all non-expired 
+   * MonitoredTasks handled by this TaskMonitor.
+   * @return A complete list of MonitoredTasks.
+   */
+  public synchronized List getTasks() {
+    purgeExpiredTasks();
+    ArrayList ret = Lists.newArrayListWithCapacity(tasks.size());
+    for (TaskAndWeakRefPair pair : tasks) {
+      MonitoredTask t = pair.get();
+      ret.add(t.clone());
+    }
+    return ret;
+  }
+
+  private boolean canPurge(MonitoredTask stat) {
+    long cts = stat.getCompletionTimestamp();
+    return (cts > 0 && System.currentTimeMillis() - cts > EXPIRATION_TIME);
+  }
+  
+
+  public void dumpAsText(PrintWriter out) {
+    long now = System.currentTimeMillis();
+    
+    List tasks = getTasks();
+    for (MonitoredTask task : tasks) {
+      out.println("Task: " + task.getDescription());
+      out.println("Status: " + task.getState() + ":" + task.getStatus());
+      long running = (now - task.getStartTime())/1000;
+      if (task.getCompletionTimestamp() != -1) {
+        long completed = (now - task.getCompletionTimestamp()) / 1000;
+        out.println("Completed " + completed + "s ago");
+        out.println("Ran for " +
+            (task.getCompletionTimestamp() - task.getStartTime())/1000
+            + "s");
+      } else {
+        out.println("Running for " + running + "s");
+      }
+      out.println();
+    }
+  }
+
+  /**
+   * This class encapsulates an object as well as a weak reference to a proxy
+   * that passes through calls to that object. In art form:
+   * 
+   *     Proxy  <------------------
+   *       |                       \
+   *       v                        \
+   * PassthroughInvocationHandler   |  weak reference
+   *       |                       /
+   * MonitoredTaskImpl            / 
+   *       |                     /
+   * StatAndWeakRefProxy  ------/
+   *
+   * Since we only return the Proxy to the creator of the MonitorableStatus,
+   * this means that they can leak that object, and we'll detect it
+   * since our weak reference will go null. But, we still have the actual
+   * object, so we can log it and display it as a leaked (incomplete) action.
+   */
+  private static class TaskAndWeakRefPair {
+    private MonitoredTask impl;
+    private WeakReference weakProxy;
+    
+    public TaskAndWeakRefPair(MonitoredTask stat,
+        MonitoredTask proxy) {
+      this.impl = stat;
+      this.weakProxy = new WeakReference(proxy);
+    }
+    
+    public MonitoredTask get() {
+      return impl;
+    }
+    
+    public boolean isDead() {
+      return weakProxy.get() == null;
+    }
+  }
+  
+  /**
+   * An InvocationHandler that simply passes through calls to the original 
+   * object.
+   */
+  private static class PassthroughInvocationHandler implements InvocationHandler {
+    private T delegatee;
+    
+    public PassthroughInvocationHandler(T delegatee) {
+      this.delegatee = delegatee;
+    }
+
+    @Override
+    public Object invoke(Object proxy, Method method, Object[] args)
+        throws Throwable {
+      return method.invoke(delegatee, args);
+    }    
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadMonitoring.java b/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadMonitoring.java
new file mode 100644
index 0000000..a3fa706
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadMonitoring.java
@@ -0,0 +1,96 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.monitoring;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+
+public abstract class ThreadMonitoring {
+
+  private static final ThreadMXBean threadBean = 
+    ManagementFactory.getThreadMXBean();
+  private static final int STACK_DEPTH = 20;
+
+  public static ThreadInfo getThreadInfo(Thread t) {
+    long tid = t.getId();
+    return threadBean.getThreadInfo(tid, STACK_DEPTH);
+  }
+    
+
+  /**
+   * Format the given ThreadInfo object as a String.
+   * @param indent a prefix for each line, used for nested indentation
+   */
+  public static String formatThreadInfo(ThreadInfo threadInfo, String indent) {
+    StringBuilder sb = new StringBuilder();
+    appendThreadInfo(sb, threadInfo, indent);
+    return sb.toString();
+  }
+
+  /**
+   * Print all of the thread's information and stack traces.
+   * 
+   * @param sb
+   * @param info
+   * @param indent
+   */
+  public static void appendThreadInfo(StringBuilder sb,
+                                      ThreadInfo info,
+                                      String indent) {
+    boolean contention = threadBean.isThreadContentionMonitoringEnabled();
+
+    if (info == null) {
+      sb.append(indent).append("Inactive (perhaps exited while monitoring was done)\n");
+      return;
+    }
+    String taskName = getTaskName(info.getThreadId(), info.getThreadName());
+    sb.append(indent).append("Thread ").append(taskName).append(":\n");
+    
+    Thread.State state = info.getThreadState();
+    sb.append(indent).append("  State: ").append(state).append("\n");
+    sb.append(indent).append("  Blocked count: ").append(info.getBlockedCount()).append("\n");
+    sb.append(indent).append("  Waited count: ").append(info.getWaitedCount()).append("\n");
+    if (contention) {
+      sb.append(indent).append("  Blocked time: " + info.getBlockedTime()).append("\n");
+      sb.append(indent).append("  Waited time: " + info.getWaitedTime()).append("\n");
+    }
+    if (state == Thread.State.WAITING) {
+      sb.append(indent).append("  Waiting on ").append(info.getLockName()).append("\n");
+    } else  if (state == Thread.State.BLOCKED) {
+      sb.append(indent).append("  Blocked on ").append(info.getLockName()).append("\n");
+      sb.append(indent).append("  Blocked by ").append(
+        getTaskName(info.getLockOwnerId(), info.getLockOwnerName())).append("\n");
+    }
+    sb.append(indent).append("  Stack:").append("\n");
+    for (StackTraceElement frame: info.getStackTrace()) {
+      sb.append(indent).append("    ").append(frame.toString()).append("\n");
+    }
+  }
+  
+  private static String getTaskName(long id, String name) {
+    if (name == null) {
+      return Long.toString(id);
+    }
+    return id + " (" + name + ")";
+  }
+
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java b/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java
new file mode 100644
index 0000000..533a896
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java
@@ -0,0 +1,377 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
+import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
+
+import com.google.common.collect.Lists;
+
+/**
+ * A globally-barriered distributed procedure.  This class encapsulates state and methods for
+ * tracking and managing a distributed procedure, as well as aborting if any member encounters
+ * a problem or if a cancellation is requested.
+ * 
        
+ * All procedures first attempt to reach a barrier point with the {@link #sendGlobalBarrierStart()}
+ * method.  The procedure contacts all members and waits for all subprocedures to execute
+ * {@link Subprocedure#acquireBarrier} to acquire its local piece of the global barrier and then
+ * send acquisition info back to the coordinator.  If all acquisitions at subprocedures succeed,
+ * the coordinator then will call {@link #sendGlobalBarrierReached()}.  This notifies members to
+ * execute the {@link Subprocedure#insideBarrier()} method.  The procedure is blocked until all
+ * {@link Subprocedure#insideBarrier} executions complete at the members.  When
+ * {@link Subprocedure#insideBarrier} completes at each member, the member sends notification to
+ * the coordinator.  Once all members complete, the coordinator calls
+ * {@link #sendGlobalBarrierComplete()}.
+ * 
        
+ * If errors are encountered remotely, they are forwarded to the coordinator, and
+ * {@link Subprocedure#cleanup(Exception)} is called.
+ * 
        
+ * Each Procedure and each Subprocedure enforces a time limit on the execution time. If the time
+ * limit expires before the procedure completes the {@link TimeoutExceptionInjector} will trigger
+ * an {@link ForeignException} to abort the procedure.  This is particularly useful for situations
+ * when running a distributed {@link Subprocedure} so participants can avoid blocking for extreme
+ * amounts of time if one of the participants fails or takes a really long time (e.g. GC pause).
+ * 
        
+ * Users should generally not directly create or subclass instances of this.  They are created
+ * for them implicitly via {@link ProcedureCoordinator#startProcedure(ForeignExceptionDispatcher,
+ * String, byte[], List)}}
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Procedure implements Callable, ForeignExceptionListener {
+  private static final Log LOG = LogFactory.getLog(Procedure.class);
+
+  //
+  // Arguments and naming
+  //
+
+  // Name of the procedure
+  final private String procName;
+  // Arguments for this procedure execution
+  final private byte[] args;
+
+  //
+  // Execution State
+  //
+  /** latch for waiting until all members have acquire in barrier state */
+  final CountDownLatch acquiredBarrierLatch;
+  /** latch for waiting until all members have executed and released their in barrier state */
+  final CountDownLatch releasedBarrierLatch;
+  /** latch for waiting until a procedure has completed */
+  final CountDownLatch completedLatch;
+  /** monitor to check for errors */
+  private final ForeignExceptionDispatcher monitor;
+
+  //
+  // Execution Timeout Handling.
+  //
+
+  /** frequency to check for errors (ms) */
+  protected final long wakeFrequency;
+  protected final TimeoutExceptionInjector timeoutInjector;
+
+  //
+  // Members' and Coordinator's state
+  //
+
+  /** lock to prevent nodes from acquiring and then releasing before we can track them */
+  private Object joinBarrierLock = new Object();
+  private final List acquiringMembers;
+  private final List inBarrierMembers;
+  private ProcedureCoordinator coord;
+
+  /**
+   * Creates a procedure. (FOR TESTING)
+   *
+   * {@link Procedure} state to be run by a {@link ProcedureCoordinator}.
+   * @param coord coordinator to call back to for general errors (e.g.
+   *          {@link ProcedureCoordinator#rpcConnectionFailure(String, IOException)}).
+   * @param monitor error monitor to check for external errors
+   * @param wakeFreq frequency to check for errors while waiting
+   * @param timeout amount of time to allow the procedure to run before cancelling
+   * @param procName name of the procedure instance
+   * @param args argument data associated with the procedure instance
+   * @param expectedMembers names of the expected members
+   */
+  public Procedure(ProcedureCoordinator coord, ForeignExceptionDispatcher monitor, long wakeFreq,
+      long timeout, String procName, byte[] args, List expectedMembers) {
+    this.coord = coord;
+    this.acquiringMembers = new ArrayList(expectedMembers);
+    this.inBarrierMembers = new ArrayList(acquiringMembers.size());
+    this.procName = procName;
+    this.args = args;
+    this.monitor = monitor;
+    this.wakeFrequency = wakeFreq;
+
+    int count = expectedMembers.size();
+    this.acquiredBarrierLatch = new CountDownLatch(count);
+    this.releasedBarrierLatch = new CountDownLatch(count);
+    this.completedLatch = new CountDownLatch(1);
+    this.timeoutInjector = new TimeoutExceptionInjector(monitor, timeout);
+  }
+
+  /**
+   * Create a procedure.
+   *
+   * Users should generally not directly create instances of this.  They are created them
+   * implicitly via {@link ProcedureCoordinator#createProcedure(ForeignExceptionDispatcher,
+   * String, byte[], List)}}
+   *
+   * @param coord coordinator to call back to for general errors (e.g.
+   *          {@link ProcedureCoordinator#rpcConnectionFailure(String, IOException)}).
+   * @param wakeFreq frequency to check for errors while waiting
+   * @param timeout amount of time to allow the procedure to run before cancelling
+   * @param procName name of the procedure instance
+   * @param args argument data associated with the procedure instance
+   * @param expectedMembers names of the expected members
+   */
+  public Procedure(ProcedureCoordinator coord, long wakeFreq, long timeout,
+      String procName, byte[] args, List expectedMembers) {
+    this(coord, new ForeignExceptionDispatcher(), wakeFreq, timeout, procName, args,
+        expectedMembers);
+  }
+
+  public String getName() {
+    return procName;
+  }
+
+  /**
+   * @return String of the procedure members both trying to enter the barrier and already in barrier
+   */
+  public String getStatus() {
+    String waiting, done;
+    synchronized (joinBarrierLock) {
+      waiting = acquiringMembers.toString();
+      done = inBarrierMembers.toString();
+    }
+    return "Procedure " + procName + " { waiting=" + waiting + " done="+ done + " }";
+  }
+  
+  /**
+   * Get the ForeignExceptionDispatcher
+   * @return the Procedure's monitor.
+   */
+  public ForeignExceptionDispatcher getErrorMonitor() {
+    return monitor;
+  }
+
+  /**
+   * This call is the main execution thread of the barriered procedure.  It sends messages and
+   * essentially blocks until all procedure members acquire or later complete but periodically
+   * checks for foreign exceptions.
+   */
+  @Override
+  @SuppressWarnings("finally")
+  final public Void call() {
+    LOG.info("Starting procedure '" + procName + "'");
+    // start the timer
+    timeoutInjector.start();
+
+    // run the procedure
+    try {
+      // start by checking for error first
+      monitor.rethrowException();
+      LOG.debug("Procedure '" + procName + "' starting 'acquire'");
+      sendGlobalBarrierStart();
+
+      // wait for all the members to report acquisition
+      LOG.debug("Waiting for all members to 'acquire'");
+      waitForLatch(acquiredBarrierLatch, monitor, wakeFrequency, "acquired");
+      monitor.rethrowException();
+
+      LOG.debug("Procedure '" + procName + "' starting 'in-barrier' execution.");
+      sendGlobalBarrierReached();
+
+      // wait for all members to report barrier release
+      waitForLatch(releasedBarrierLatch, monitor, wakeFrequency, "released");
+
+      // make sure we didn't get an error during in barrier execution and release
+      monitor.rethrowException();
+      LOG.info("Procedure '" + procName + "' execution completed");
+    } catch (Exception e) {
+      if (e instanceof InterruptedException) {
+        Thread.currentThread().interrupt();
+      }
+      String msg = "Procedure '" + procName +"' execution failed!";
+      LOG.error(msg, e);
+      receive(new ForeignException(getName(), e));
+    } finally {
+      LOG.debug("Running finish phase.");
+      sendGlobalBarrierComplete();
+      completedLatch.countDown();
+
+      // tell the timer we are done, if we get here successfully
+      timeoutInjector.complete();
+      return null;
+    }
+  }
+
+  /**
+   * Sends a message to Members to create a new {@link Subprocedure} for this Procedure and execute
+   * the {@link Subprocedure#acquireBarrier} step.
+   * @throws ForeignException
+   */
+  public void sendGlobalBarrierStart() throws ForeignException {
+    // start the procedure
+    LOG.debug("Starting procedure '" + procName + "', kicking off acquire phase on members.");
+    try {
+      // send procedure barrier start to specified list of members. cloning the list to avoid
+      // concurrent modification from the controller setting the prepared nodes
+      coord.getRpcs().sendGlobalBarrierAcquire(this, args, Lists.newArrayList(this.acquiringMembers));
+    } catch (IOException e) {
+      coord.rpcConnectionFailure("Can't reach controller.", e);
+    } catch (IllegalArgumentException e) {
+      throw new ForeignException(getName(), e);
+    }
+  }
+
+  /**
+   * Sends a message to all members that the global barrier condition has been satisfied.  This
+   * should only be executed after all members have completed its
+   * {@link Subprocedure#acquireBarrier()} call successfully.  This triggers the member
+   * {@link Subprocedure#insideBarrier} method.
+   * @throws ForeignException
+   */
+  public void sendGlobalBarrierReached() throws ForeignException {
+    try {
+      // trigger to have member run {@link Subprocedure#insideBarrier}
+      coord.getRpcs().sendGlobalBarrierReached(this, Lists.newArrayList(inBarrierMembers));
+    } catch (IOException e) {
+      coord.rpcConnectionFailure("Can't reach controller.", e);
+    }
+  }
+
+  /**
+   * Sends a message to members that all {@link Subprocedure#insideBarrier} calls have completed.
+   * After this executes, the coordinator can assume that any state resources about this barrier
+   * procedure state has been released.
+   */
+  public void sendGlobalBarrierComplete() {
+    LOG.debug("Finished coordinator procedure - removing self from list of running procedures");
+    try {
+      coord.getRpcs().resetMembers(this);
+    } catch (IOException e) {
+      coord.rpcConnectionFailure("Failed to reset procedure:" + procName, e);
+    }
+  }
+
+  //
+  // Call backs from other external processes.
+  //
+
+  /**
+   * Call back triggered by an individual member upon successful local barrier acquisition
+   * @param member
+   */
+  public void barrierAcquiredByMember(String member) {
+    LOG.debug("member: '" + member + "' joining prepared barrier for procedure '" + procName
+        + "' on coordinator");
+    if (this.acquiringMembers.contains(member)) {
+      synchronized (joinBarrierLock) {
+        if (this.acquiringMembers.remove(member)) {
+          this.inBarrierMembers.add(member);
+          acquiredBarrierLatch.countDown();
+        }
+      }
+      LOG.debug("Waiting on: " + acquiredBarrierLatch + " remaining members to acquire global barrier");
+    } else {
+      LOG.warn("Member " + member + " joined barrier, but we weren't waiting on it to join." +
+          " Continuing on.");
+    }
+  }
+
+  /**
+   * Call back triggered by a individual member upon successful local in-barrier execution and
+   * release
+   * @param member
+   */
+  public void barrierReleasedByMember(String member) {
+    boolean removed = false;
+    synchronized (joinBarrierLock) {
+      removed = this.inBarrierMembers.remove(member);
+      if (removed) {
+        releasedBarrierLatch.countDown();
+      }
+    }
+    if (removed) {
+      LOG.debug("Member: '" + member + "' released barrier for procedure'" + procName
+          + "', counting down latch.  Waiting for " + releasedBarrierLatch.getCount()
+          + " more");
+    } else {
+      LOG.warn("Member: '" + member + "' released barrier for procedure'" + procName
+          + "', but we weren't waiting on it to release!");
+    }
+  }
+
+  /**
+   * Waits until the entire procedure has globally completed, or has been aborted.
+   * @throws ForeignException
+   * @throws InterruptedException
+   */
+  public void waitForCompleted() throws ForeignException, InterruptedException {
+    waitForLatch(completedLatch, monitor, wakeFrequency, procName + " completed");
+  }
+
+  /**
+   * A callback that handles incoming ForeignExceptions.
+   */
+  @Override
+  public void receive(ForeignException e) {
+    monitor.receive(e);
+  }
+
+  /**
+   * Wait for latch to count to zero, ignoring any spurious wake-ups, but waking periodically to
+   * check for errors
+   * @param latch latch to wait on
+   * @param monitor monitor to check for errors while waiting
+   * @param wakeFrequency frequency to wake up and check for errors (in
+   *          {@link TimeUnit#MILLISECONDS})
+   * @param latchDescription description of the latch, for logging
+   * @throws ForeignException type of error the monitor can throw, if the task fails
+   * @throws InterruptedException if we are interrupted while waiting on latch
+   */
+  public static void waitForLatch(CountDownLatch latch, ForeignExceptionSnare monitor,
+      long wakeFrequency, String latchDescription) throws ForeignException,
+      InterruptedException {
+    boolean released = false;
+    while (!released) {
+      if (monitor != null) {
+        monitor.rethrowException();
+      }
+      /*
+      ForeignExceptionDispatcher.LOG.debug("Waiting for '" + latchDescription + "' latch. (sleep:"
+          + wakeFrequency + " ms)"); */
+      released = latch.await(wakeFrequency, TimeUnit.MILLISECONDS);
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java
new file mode 100644
index 0000000..f9c6259
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.DaemonThreadFactory;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+
+import com.google.common.collect.MapMaker;
+
+/**
+ * This is the master side of a distributed complex procedure execution.
+ * 
        
+ * The {@link Procedure} is generic and subclassing or customization shouldn't be
+ * necessary -- any customization should happen just in {@link Subprocedure}s.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ProcedureCoordinator {
+  private static final Log LOG = LogFactory.getLog(ProcedureCoordinator.class);
+
+  final static long TIMEOUT_MILLIS_DEFAULT = 60000;
+  final static long WAKE_MILLIS_DEFAULT = 500;
+
+  private final ProcedureCoordinatorRpcs rpcs;
+  private final ExecutorService pool;
+
+  // Running procedure table.  Maps procedure name to running procedure reference
+  private final ConcurrentMap procedures =
+      new MapMaker().concurrencyLevel(4).weakValues().makeMap();
+
+  /**
+   * Create and start a ProcedureCoordinator.
+   *
+   * The rpc object registers the ProcedureCoordinator and starts any threads in this
+   * constructor.
+   *
+   * @param rpcs
+   * @param pool Used for executing procedures.
+   */
+  public ProcedureCoordinator(ProcedureCoordinatorRpcs rpcs, ThreadPoolExecutor pool) {
+    this.rpcs = rpcs;
+    this.pool = pool;
+    this.rpcs.start(this);
+  }
+
+  /**
+   * Default thread pool for the procedure
+   */
+  public static ThreadPoolExecutor defaultPool(String coordName, long keepAliveTime, int opThreads,
+      long wakeFrequency) {
+    return new ThreadPoolExecutor(1, opThreads, keepAliveTime, TimeUnit.SECONDS,
+        new SynchronousQueue(),
+        new DaemonThreadFactory("(" + coordName + ")-proc-coordinator-pool"));
+  }
+
+  /**
+   * Shutdown the thread pools and release rpc resources
+   * @throws IOException
+   */
+  public void close() throws IOException {
+    // have to use shutdown now to break any latch waiting
+    pool.shutdownNow();
+    rpcs.close();
+  }
+
+  /**
+   * Submit an procedure to kick off its dependent subprocedures.
+   * @param proc Procedure to execute
+   * @return true if the procedure was started correctly, false if the
+   *         procedure or any subprocedures could not be started.  Failure could be due to
+   *         submitting a procedure multiple times (or one with the same name), or some sort
+   *         of IO problem.  On errors, the procedure's monitor holds a reference to the exception
+   *         that caused the failure.
+   */
+  boolean submitProcedure(Procedure proc) {
+    // if the submitted procedure was null, then we don't want to run it
+    if (proc == null) {
+      return false;
+    }
+    String procName = proc.getName();
+
+    // make sure we aren't already running a procedure of that name
+    synchronized (procedures) {
+      Procedure oldProc = procedures.get(procName);
+      if (oldProc != null) {
+        // procedures are always eventually completed on both successful and failed execution
+        if (oldProc.completedLatch.getCount() != 0) {
+          LOG.warn("Procedure " + procName + " currently running.  Rejecting new request");
+          return false;
+        }
+        LOG.debug("Procedure " + procName + " was in running list but was completed.  Accepting new attempt.");
+        procedures.remove(procName);
+      }
+    }
+
+    // kick off the procedure's execution in a separate thread
+    Future f = null;
+    try {
+      synchronized (procedures) {
+        f = this.pool.submit(proc);
+        // if everything got started properly, we can add it known running procedures
+        this.procedures.put(procName, proc);
+      }
+      return true;
+    } catch (RejectedExecutionException e) {
+      LOG.warn("Procedure " + procName + " rejected by execution pool.  Propagating error and " +
+          "cancelling operation.", e);
+      // the thread pool is full and we can't run the procedure
+      proc.receive(new ForeignException(procName, e));
+
+      // cancel procedure proactively
+      if (f != null) {
+        f.cancel(true);
+      }
+    }
+    return false;
+  }
+
+  /**
+   * The connection to the rest of the procedure group (members and coordinator) has been
+   * broken/lost/failed. This should fail any interested procedures, but not attempt to notify other
+   * members since we cannot reach them anymore.
+   * @param message description of the error
+   * @param cause the actual cause of the failure
+   */
+  void rpcConnectionFailure(final String message, final IOException cause) {
+    Collection toNotify = procedures.values();
+
+    for (Procedure proc : toNotify) {
+      if (proc == null) {
+        continue;
+      }
+      // notify the elements, if they aren't null
+      proc.receive(new ForeignException(proc.getName(), cause));
+    }
+  }
+
+  /**
+   * Abort the procedure with the given name
+   * @param procName name of the procedure to abort
+   * @param reason serialized information about the abort
+   */
+  public void abortProcedure(String procName, ForeignException reason) {
+    // if we know about the Procedure, notify it
+    synchronized(procedures) {
+      Procedure proc = procedures.get(procName);
+      if (proc == null) {
+        return;
+      }
+      proc.receive(reason);
+    }
+  }
+
+  /**
+   * Exposed for hooking with unit tests.
+   * @param procName
+   * @param procArgs
+   * @param expectedMembers
+   * @return
+   */
+  Procedure createProcedure(ForeignExceptionDispatcher fed, String procName, byte[] procArgs,
+      List expectedMembers) {
+    // build the procedure
+    return new Procedure(this, fed, WAKE_MILLIS_DEFAULT, TIMEOUT_MILLIS_DEFAULT,
+        procName, procArgs, expectedMembers);
+  }
+
+  /**
+   * Kick off the named procedure
+   * @param procName name of the procedure to start
+   * @param procArgs arguments for the procedure
+   * @param expectedMembers expected members to start
+   * @return handle to the running procedure, if it was started correctly, null otherwise
+   * @throws RejectedExecutionException if there are no more available threads to run the procedure
+   */
+  public Procedure startProcedure(ForeignExceptionDispatcher fed, String procName, byte[] procArgs,
+      List expectedMembers) throws RejectedExecutionException {
+    Procedure proc = createProcedure(fed, procName, procArgs, expectedMembers);
+    if (!this.submitProcedure(proc)) {
+      LOG.error("Failed to submit procedure '" + procName + "'");
+      return null;
+    }
+    return proc;
+  }
+
+  /**
+   * Notification that the procedure had the specified member acquired its part of the barrier
+   * via {@link Subprocedure#acquireBarrier()}.
+   * @param procName name of the procedure that acquired
+   * @param member name of the member that acquired
+   */
+  void memberAcquiredBarrier(String procName, final String member) {
+    Procedure proc = procedures.get(procName);
+    if (proc != null) {
+      proc.barrierAcquiredByMember(member);
+    }
+  }
+
+  /**
+   * Notification that the procedure had another member finished executing its in-barrier subproc
+   * via {@link Subprocedure#insideBarrier()}.
+   * @param procName name of the subprocedure that finished
+   * @param member name of the member that executed and released its barrier
+   */
+  void memberFinishedBarrier(String procName, final String member) {
+    Procedure proc = procedures.get(procName);
+    if (proc != null) {
+      proc.barrierReleasedByMember(member);
+    }
+  }
+
+  /**
+   * @return the rpcs implementation for all current procedures
+   */
+  ProcedureCoordinatorRpcs getRpcs() {
+    return rpcs;
+  }
+
+  /**
+   * Returns the procedure.  This Procedure is a live instance so should not be modified but can
+   * be inspected.
+   * @param name Name of the procedure
+   * @return Procedure or null if not present any more
+   */
+  public Procedure getProcedure(String name) {
+    return procedures.get(name);
+  }
+
+  /**
+   * @return Return set of all procedure names.
+   */
+  public Set getProcedureNames() {
+    return new HashSet(procedures.keySet());
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinatorRpcs.java b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinatorRpcs.java
new file mode 100644
index 0000000..209c671
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinatorRpcs.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+
+/**
+ * RPCs for the coordinator to run a barriered procedure with subprocedures executed at
+ * distributed members.
+ * @see ProcedureCoordinator
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface ProcedureCoordinatorRpcs extends Closeable {
+
+  /**
+   * Initialize and start threads necessary to connect an implementation's rpc mechanisms.
+   * @param listener
+   * @return true if succeed, false if encountered initialization errors.
+   */
+  public boolean start(final ProcedureCoordinator listener);
+
+  /**
+   * Notify the members that the coordinator has aborted the procedure and that it should release
+   * barrier resources.
+   *
+   * @param procName name of the procedure that was aborted
+   * @param cause the reason why the procedure needs to be aborted
+   * @throws IOException if the rpcs can't reach the other members of the procedure (and can't
+   *           recover).
+   */
+  public void sendAbortToMembers(Procedure procName, ForeignException cause) throws IOException;
+
+  /**
+   * Notify the members to acquire barrier for the procedure
+   *
+   * @param procName name of the procedure to start
+   * @param info information that should be passed to all members
+   * @param members names of the members requested to reach the acquired phase
+   * @throws IllegalArgumentException if the procedure was already marked as failed
+   * @throws IOException if we can't reach the remote notification mechanism
+   */
+  public void sendGlobalBarrierAcquire(Procedure procName, byte[] info, List members)
+      throws IOException, IllegalArgumentException;
+
+  /**
+   * Notify members that all members have acquired their parts of the barrier and that they can
+   * now execute under the global barrier.
+   *
+   * Must come after calling {@link #sendGlobalBarrierAcquire(Procedure, byte[], List)}
+   *
+   * @param procName name of the procedure to start
+   * @param members members to tell we have reached in-barrier phase
+   * @throws IOException if we can't reach the remote notification mechanism
+   */
+  public void sendGlobalBarrierReached(Procedure procName, List members) throws IOException;
+
+  /**
+   * Notify Members to reset the distributed state for procedure
+   * @param procName name of the procedure to reset
+   * @throws IOException if the remote notification mechanism cannot be reached
+   */
+  public void resetMembers(Procedure procName) throws IOException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java
new file mode 100644
index 0000000..876f5c4
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java
@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.DaemonThreadFactory;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+
+import com.google.common.collect.MapMaker;
+
+/**
+ * Process to kick off and manage a running {@link Subprocedure} on a member. This is the
+ * specialized part of a {@link Procedure} that actually does procedure type-specific work
+ * and reports back to the coordinator as it completes each phase.
+ * 
        
+ * If there is a connection error ({@link #controllerConnectionFailure(String, IOException)}), all
+ * currently running subprocedures are notify to failed since there is no longer a way to reach any
+ * other members or coordinators since the rpcs are down.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ProcedureMember implements Closeable {
+  private static final Log LOG = LogFactory.getLog(ProcedureMember.class);
+
+  private final SubprocedureFactory builder;
+  private final ProcedureMemberRpcs rpcs;
+
+  private final ConcurrentMap subprocs =
+      new MapMaker().concurrencyLevel(4).weakValues().makeMap();
+  private final ExecutorService pool;
+
+  /**
+   * Instantiate a new ProcedureMember.  This is a slave that executes subprocedures.
+   *
+   * @param rpcs controller used to send notifications to the procedure coordinator
+   * @param pool thread pool to submit subprocedures
+   * @param factory class that creates instances of a subprocedure.
+   */
+  public ProcedureMember(ProcedureMemberRpcs rpcs, ThreadPoolExecutor pool,
+      SubprocedureFactory factory) {
+    this.pool = pool;
+    this.rpcs = rpcs;
+    this.builder = factory;
+  }
+
+  public static ThreadPoolExecutor defaultPool(long wakeFrequency, long keepAlive,
+      int procThreads, String memberName) {
+    return new ThreadPoolExecutor(1, procThreads, keepAlive, TimeUnit.SECONDS,
+        new SynchronousQueue(),
+        new DaemonThreadFactory("member: '" + memberName + "' subprocedure-pool"));
+  }
+
+  /**
+   * Package exposed.  Not for public use.
+   *
+   * @return reference to the Procedure member's rpcs object
+   */
+  ProcedureMemberRpcs getRpcs() {
+     return rpcs;
+  }
+
+
+  /**
+   * This is separated from execution so that we can detect and handle the case where the
+   * subprocedure is invalid and inactionable due to bad info (like DISABLED snapshot type being
+   * sent here)
+   * @param opName
+   * @param data
+   * @return subprocedure
+   */
+  public Subprocedure createSubprocedure(String opName, byte[] data) {
+    return builder.buildSubprocedure(opName, data);
+  }
+
+  /**
+   * Submit an subprocedure for execution.  This starts the local acquire phase.
+   * @param subproc the subprocedure to execute.
+   * @return true if the subprocedure was started correctly, false if it
+   *         could not be started. In the latter case, the subprocedure holds a reference to
+   *         the exception that caused the failure.
+   */
+  public boolean submitSubprocedure(Subprocedure subproc) {
+     // if the submitted subprocedure was null, bail.
+    if (subproc == null) {
+      LOG.warn("Submitted null subprocedure, nothing to run here.");
+      return false;
+    }
+
+    String procName = subproc.getName();
+    if (procName == null || procName.length() == 0) {
+      LOG.error("Subproc name cannot be null or the empty string");
+      return false;
+    }
+
+    // make sure we aren't already running an subprocedure of that name
+    Subprocedure rsub;
+    synchronized (subprocs) {
+      rsub = subprocs.get(procName);
+    }
+    if (rsub != null) {
+      if (!rsub.isComplete()) {
+        LOG.error("Subproc '" + procName + "' is already running. Bailing out");
+        return false;
+      }
+      LOG.warn("A completed old subproc "  +  procName + " is still present, removing");
+      subprocs.remove(procName);
+    }
+
+    LOG.debug("Submitting new Subprocedure:" + procName);
+
+    // kick off the subprocedure
+    Future future = null;
+    try {
+      future = this.pool.submit(subproc);
+      synchronized (subprocs) {
+        subprocs.put(procName, subproc);
+      }
+      return true;
+    } catch (RejectedExecutionException e) {
+      // the thread pool is full and we can't run the subprocedure
+      String msg = "Subprocedure pool is full!";
+      subproc.cancel(msg, e.getCause());
+
+      // cancel all subprocedures proactively
+      if (future != null) {
+        future.cancel(true);
+      }
+    }
+
+    LOG.error("Failed to start subprocedure '" + procName + "'");
+    return false;
+  }
+
+   /**
+    * Notification that procedure coordinator has reached the global barrier
+    * @param procName name of the subprocedure that should start running the the in-barrier phase
+    */
+   public void receivedReachedGlobalBarrier(String procName) {
+     Subprocedure subproc = subprocs.get(procName);
+     if (subproc == null) {
+       LOG.warn("Unexpected reached glabal barrier message for Sub-Procedure '" + procName + "'");
+       return;
+     }
+     subproc.receiveReachedGlobalBarrier();
+   }
+
+  /**
+   * Best effort attempt to close the threadpool via Thread.interrupt.
+   */
+  @Override
+  public void close() throws IOException {
+    // have to use shutdown now to break any latch waiting
+    pool.shutdownNow();
+  }
+
+  /**
+   * Shutdown the threadpool, and wait for upto timeoutMs millis before bailing
+   * @param timeoutMs timeout limit in millis
+   * @return true if successfully, false if bailed due to timeout.
+   * @throws InterruptedException
+   */
+  boolean closeAndWait(long timeoutMs) throws InterruptedException {
+    pool.shutdown();
+    return pool.awaitTermination(timeoutMs, TimeUnit.MILLISECONDS);
+  }
+
+  /**
+   * The connection to the rest of the procedure group (member and coordinator) has been
+   * broken/lost/failed. This should fail any interested subprocedure, but not attempt to notify
+   * other members since we cannot reach them anymore.
+   * @param message description of the error
+   * @param cause the actual cause of the failure
+   *
+   * TODO i'm tempted to just remove this code completely and treat it like any other abort.
+   * Implementation wise, if this happens it is a ZK failure which means the RS will abort.
+   */
+  public void controllerConnectionFailure(final String message, final IOException cause) {
+    Collection toNotify = subprocs.values();
+    LOG.error(message, cause);
+    for (Subprocedure sub : toNotify) {
+      // TODO notify the elements, if they aren't null
+      sub.cancel(message, cause);
+    }
+  }
+
+  /**
+   * Send abort to the specified procedure
+   * @param procName name of the procedure to about
+   * @param ee exception information about the abort
+   */
+  public void receiveAbortProcedure(String procName, ForeignException ee) {
+    LOG.debug("Request received to abort procedure " + procName, ee);
+    // if we know about the procedure, notify it
+    Subprocedure sub = subprocs.get(procName);
+    if (sub == null) {
+      LOG.info("Received abort on procedure with no local subprocedure " + procName +
+          ", ignoring it.", ee);
+      return; // Procedure has already completed
+    }
+    LOG.error("Propagating foreign exception to subprocedure " + sub.getName(), ee);
+    sub.monitor.receive(ee);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMemberRpcs.java b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMemberRpcs.java
new file mode 100644
index 0000000..72cec2b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMemberRpcs.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+
+/**
+ * This is the notification interface for Procedures that encapsulates message passing from
+ * members to a coordinator.  Each of these calls should send a message to the coordinator.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface ProcedureMemberRpcs extends Closeable {
+
+  /**
+   * Initialize and start any threads or connections the member needs.
+   */
+  public void start(ProcedureMember member);
+
+  /**
+   * Each subprocedure is being executed on a member.  This is the identifier for the member.
+   * @return the member name
+   */
+  public String getMemberName();
+
+  /**
+   * Notify the coordinator that we aborted the specified {@link Subprocedure}
+   *
+   * @param sub the {@link Subprocedure} we are aborting
+   * @param cause the reason why the member's subprocedure aborted
+   * @throws IOException thrown when the rpcs can't reach the other members of the procedure (and
+   *  thus can't recover).
+   */
+  public void sendMemberAborted(Subprocedure sub, ForeignException cause) throws IOException;
+
+  /**
+   * Notify the coordinator that the specified {@link Subprocedure} has acquired the locally required
+   * barrier condition.
+   *
+   * @param sub the specified {@link Subprocedure}
+   * @throws IOException if we can't reach the coordinator
+   */
+  public void sendMemberAcquired(Subprocedure sub) throws IOException;
+
+  /**
+   * Notify the coordinator that the specified {@link Subprocedure} has completed the work that
+   * needed to be done under the global barrier.
+   *
+   * @param sub the specified {@link Subprocedure}
+   * @throws IOException if we can't reach the coordinator
+   */
+  public void sendMemberCompleted(Subprocedure sub) throws IOException;
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java b/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java
new file mode 100644
index 0000000..ea669a0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java
@@ -0,0 +1,331 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener;
+import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
+
+/**
+ * Distributed procedure member's Subprocedure.  A procedure is sarted on a ProcedureCoordinator
+ * which communicates with ProcedureMembers who create and start its part of the Procedure.  This
+ * sub part is called a Subprocedure
+ *
+ * Users should subclass this and implement {@link #acquireBarrier()} (get local barrier for this
+ * member), {@link #insideBarrier()} (execute while globally barriered and release barrier) and
+ * {@link #cleanup(Exception)} (release state associated with subprocedure.)
+ *
+ * When submitted to a ProcedureMemeber, the call method is executed in a separate thread.
+ * Latches are use too block its progress and trigger continuations when barrier conditions are
+ * met.
+ *
+ * Exception that makes it out of calls to {@link #acquireBarrier()} or {@link #insideBarrier()}
+ * gets converted into {@link ForeignException}, which will get propagated to the
+ * {@link ProcedureCoordinator}.
+ *
+ * There is a category of procedure (ex: online-snapshots), and a user-specified instance-specific
+ * barrierName. (ex: snapshot121126).
+ */
+abstract public class Subprocedure implements Callable {
+  private static final Log LOG = LogFactory.getLog(Subprocedure.class);
+
+  // Name of the procedure
+  final private String barrierName;
+
+  //
+  // Execution state
+  //
+
+  /** wait on before allowing the in barrier phase to proceed */
+  private final CountDownLatch inGlobalBarrier;
+  /** counted down when the Subprocedure has completed */
+  private final CountDownLatch releasedLocalBarrier;
+
+  //
+  // Error handling
+  //
+  /** monitor to check for errors */
+  protected final ForeignExceptionDispatcher monitor;
+  /** frequency to check for errors (ms) */
+  protected final long wakeFrequency;
+  protected final TimeoutExceptionInjector executionTimeoutTimer;
+  protected final ProcedureMemberRpcs rpcs;
+
+  private volatile boolean complete = false;
+
+  /**
+   * @param member reference to the member managing this subprocedure
+   * @param procName name of the procedure this subprocedure is associated with
+   * @param monitor notified if there is an error in the subprocedure
+   * @param wakeFrequency time in millis to wake to check if there is an error via the monitor (in
+   *          milliseconds).
+   * @param timeout time in millis that will trigger a subprocedure abort if it has not completed
+   */
+  public Subprocedure(ProcedureMember member, String procName, ForeignExceptionDispatcher monitor,
+      long wakeFrequency, long timeout) {
+    // Asserts should be caught during unit testing
+    assert member != null : "procedure member should be non-null";
+    assert member.getRpcs() != null : "rpc handlers should be non-null";
+    assert procName != null : "procedure name should be non-null";
+    assert monitor != null : "monitor should be non-null";
+
+    // Default to a very large timeout
+    this.rpcs = member.getRpcs();
+    this.barrierName = procName;
+    this.monitor = monitor;
+    // forward any failures to coordinator.  Since this is a dispatcher, resend loops should not be
+    // possible.
+    this.monitor.addListener(new ForeignExceptionListener() {
+      @Override
+      public void receive(ForeignException ee) {
+        // if this is a notification from a remote source, just log
+        if (ee.isRemote()) {
+          LOG.debug("Was remote foreign exception, not redispatching error", ee);
+          return;
+        }
+
+        // if it is local, then send it to the coordinator
+        try {
+          rpcs.sendMemberAborted(Subprocedure.this, ee);
+        } catch (IOException e) {
+          // this will fail all the running procedures, since the connection is down
+          LOG.error("Can't reach controller, not propagating error", e);
+        }
+      }
+    });
+
+    this.wakeFrequency = wakeFrequency;
+    this.inGlobalBarrier = new CountDownLatch(1);
+    this.releasedLocalBarrier = new CountDownLatch(1);
+
+    // accept error from timer thread, this needs to be started.
+    this.executionTimeoutTimer = new TimeoutExceptionInjector(monitor, timeout);
+  }
+
+  public String getName() {
+     return barrierName;
+  }
+
+  public String getMemberName() {
+    return rpcs.getMemberName();
+  }
+
+  private void rethrowException() throws ForeignException {
+    monitor.rethrowException();
+  }
+
+  /**
+   * Execute the Subprocedure {@link #acquireBarrier()} and {@link #insideBarrier()} methods
+   * while keeping some state for other threads to access.
+   *
+   * This would normally be executed by the ProcedureMemeber when a acquire message comes from the
+   * coordinator.  Rpcs are used to spend message back to the coordinator after different phases
+   * are executed.  Any exceptions caught during the execution (except for InterrupedException) get
+   * converted and propagated to coordinator via {@link ProcedureMemberRpcs#sendMemberAborted(
+   * Subprocedure, ForeignException)}.
+   */
+  @SuppressWarnings("finally")
+  final public Void call() {
+    LOG.debug("Starting subprocedure '" + barrierName + "' with timeout " +
+        executionTimeoutTimer.getMaxTime() + "ms");
+    // start the execution timeout timer
+    executionTimeoutTimer.start();
+
+    try {
+      // start by checking for error first
+      rethrowException();
+      LOG.debug("Subprocedure '" + barrierName + "' starting 'acquire' stage");
+      acquireBarrier();
+      LOG.debug("Subprocedure '" + barrierName + "' locally acquired");
+
+      // vote yes to coordinator about being prepared
+      rpcs.sendMemberAcquired(this);
+      LOG.debug("Subprocedure '" + barrierName + "' coordinator notified of 'acquire', waiting on" +
+          " 'reached' or 'abort' from coordinator");
+
+      // wait for the procedure to reach global barrier before proceding
+      waitForReachedGlobalBarrier();
+      rethrowException(); // if Coordinator aborts, will bail from here with exception
+
+      // In traditional 2PC, if a member reaches this state the TX has been committed and the
+      // member is responsible for rolling forward and recovering and completing the subsequent
+      // operations in the case of failure.  It cannot rollback.
+      //
+      // This implementation is not 2PC since it can still rollback here, and thus has different
+      // semantics.
+
+      LOG.debug("Subprocedure '" + barrierName + "' received 'reached' from coordinator.");
+      insideBarrier();
+      LOG.debug("Subprocedure '" + barrierName + "' locally completed");
+
+      // Ack that the member has executed and released local barrier
+      rpcs.sendMemberCompleted(this);
+      LOG.debug("Subprocedure '" + barrierName + "' has notified controller of completion");
+
+      // make sure we didn't get an external exception
+      rethrowException();
+    } catch (Exception e) {
+      String msg = null;
+      if (e instanceof InterruptedException) {
+        msg = "Procedure '" + barrierName + "' aborting due to interrupt!" +
+            " Likely due to pool shutdown.";
+        Thread.currentThread().interrupt();
+      } else if (e instanceof ForeignException) {
+        msg = "Subprocedure '" + barrierName + "' aborting due to a ForeignException!";
+      } else {
+        msg = "Subprocedure '" + barrierName + "' failed!";
+      }
+      LOG.error(msg , e);
+      cancel(msg, e);
+
+      LOG.debug("Subprocedure '" + barrierName + "' running cleanup.");
+      cleanup(e);
+    } finally {
+      releasedLocalBarrier.countDown();
+
+      // tell the timer we are done, if we get here successfully
+      executionTimeoutTimer.complete();
+      complete = true;
+      LOG.debug("Subprocedure '" + barrierName + "' completed.");
+      return null;
+    }
+  }
+
+  boolean isComplete() {
+    return complete;
+  }
+
+  /**
+   * exposed for testing.
+   */
+  ForeignExceptionSnare getErrorCheckable() {
+    return this.monitor;
+  }
+
+  /**
+   * The implementation of this method should gather and hold required resources (locks, disk
+   * space, etc) to satisfy the Procedures barrier condition.  For example, this would be where
+   * to make all the regions on a RS on the quiescent for an procedure that required all regions
+   * to be globally quiesed.
+   *
+   * Users should override this method.  If a quiescent is not required, this is overkill but
+   * can still be used to execute a procedure on all members and to propagate any exceptions.
+   *
+   * @throws ForeignException
+   */
+  abstract public void acquireBarrier() throws ForeignException;
+
+  /**
+   * The implementation of this method should act with the assumption that the barrier condition
+   * has been satisfied.  Continuing the previous example, a condition could be that all RS's
+   * globally have been quiesced, and procedures that require this precondition could be
+   * implemented here.
+   *
+   * Users should override this method.  If quiescense is not required, this can be a no-op
+   *
+   * @throws ForeignException
+   */
+  abstract public void insideBarrier() throws ForeignException;
+
+  /**
+   * Users should override this method. This implementation of this method should rollback and
+   * cleanup any temporary or partially completed state that the {@link #acquireBarrier()} may have
+   * created.
+   * @param e
+   */
+  abstract public void cleanup(Exception e);
+
+  /**
+   * Method to cancel the Subprocedure by injecting an exception from and external source.
+   * @param cause
+   */
+  public void cancel(String msg, Throwable cause) {
+    LOG.error(msg, cause);
+    if (cause instanceof ForeignException) {
+      monitor.receive((ForeignException) cause);
+    } else {
+      monitor.receive(new ForeignException(getMemberName(), cause));
+    }
+  }
+
+  /**
+   * Callback for the member rpcs to call when the global barrier has been reached.  This
+   * unblocks the main subprocedure exectuion thread so that the Subprocedure's
+   * {@link #insideBarrier()} method can be run.
+   */
+  public void receiveReachedGlobalBarrier() {
+    inGlobalBarrier.countDown();
+  }
+
+  //
+  // Subprocedure Internal State interface
+  //
+
+  /**
+   * Wait for the reached global barrier notification.
+   *
+   * Package visibility for testing
+   *
+   * @throws ForeignException
+   * @throws InterruptedException
+   */
+  void waitForReachedGlobalBarrier() throws ForeignException, InterruptedException {
+    Procedure.waitForLatch(inGlobalBarrier, monitor, wakeFrequency,
+        barrierName + ":remote acquired");
+  }
+
+  /**
+   * Waits until the entire procedure has globally completed, or has been aborted.
+   * @throws ForeignException
+   * @throws InterruptedException
+   */
+  public void waitForLocallyCompleted() throws ForeignException, InterruptedException {
+    Procedure.waitForLatch(releasedLocalBarrier, monitor, wakeFrequency,
+        barrierName + ":completed");
+  }
+
+  /**
+   * Empty Subprocedure for testing.
+   *
+   * Must be public for stubbing used in testing to work.
+   */
+  public static class SubprocedureImpl extends Subprocedure {
+
+    public SubprocedureImpl(ProcedureMember member, String opName,
+        ForeignExceptionDispatcher monitor, long wakeFrequency, long timeout) {
+      super(member, opName, monitor, wakeFrequency, timeout);
+    }
+
+    @Override
+    public void acquireBarrier() throws ForeignException {}
+
+    @Override
+    public void insideBarrier() throws ForeignException {}
+
+    @Override
+    public void cleanup(Exception e) {}
+  };
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/SubprocedureFactory.java b/src/main/java/org/apache/hadoop/hbase/procedure/SubprocedureFactory.java
new file mode 100644
index 0000000..0b94c89
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/SubprocedureFactory.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Task builder to build instances of a {@link ProcedureMember}'s {@link Subprocedure}s.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface SubprocedureFactory {
+
+  /**
+   * Build {@link Subprocedure} when requested.
+   * @param procName name of the procedure associated with this subprocedure
+   * @param procArgs  arguments passed from the coordinator about the procedure
+   * @return {@link Subprocedure} to run or null if the no operation should be run
+   * @throws IllegalArgumentException if the operation could not be run because of errors in the
+   *           request
+   * @throws IllegalStateException if the current runner cannot accept any more new requests
+   */
+  public Subprocedure buildSubprocedure(String procName, byte[] procArgs);
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
new file mode 100644
index 0000000..4595335
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
@@ -0,0 +1,267 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * ZooKeeper based {@link ProcedureCoordinatorRpcs} for a {@link ProcedureCoordinator}
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ZKProcedureCoordinatorRpcs implements ProcedureCoordinatorRpcs {
+  public static final Log LOG = LogFactory.getLog(ZKProcedureCoordinatorRpcs.class);
+  private ZKProcedureUtil zkProc = null;
+  protected ProcedureCoordinator coordinator = null;  // if started this should be non-null
+
+  ZooKeeperWatcher watcher;
+  String procedureType;
+  String coordName;
+
+  /**
+   * @param watcher zookeeper watcher. Owned by this and closed via {@link #close()}
+   * @param procedureClass procedure type name is a category for when there are multiple kinds of
+   *    procedures.-- this becomes a znode so be aware of the naming restrictions
+   * @param coordName name of the node running the coordinator
+   * @throws KeeperException if an unexpected zk error occurs
+   */
+  public ZKProcedureCoordinatorRpcs(ZooKeeperWatcher watcher,
+      String procedureClass, String coordName) throws KeeperException {
+    this.watcher = watcher;
+    this.procedureType = procedureClass;
+    this.coordName = coordName;
+  }
+
+  /**
+   * The "acquire" phase.  The coordinator creates a new procType/acquired/ znode dir. If znodes
+   * appear, first acquire to relevant listener or sets watch waiting for notification of
+   * the acquire node
+   *
+   * @param proc the Procedure
+   * @param info data to be stored in the acquire node
+   * @param nodeNames children of the acquire phase
+   * @throws IOException if any failure occurs.
+   */
+  @Override
+  final public void sendGlobalBarrierAcquire(Procedure proc, byte[] info, List nodeNames)
+      throws IOException, IllegalArgumentException {
+    String procName = proc.getName();
+    // start watching for the abort node
+    String abortNode = zkProc.getAbortZNode(procName);
+    try {
+      // check to see if the abort node already exists
+      if (ZKUtil.watchAndCheckExists(zkProc.getWatcher(), abortNode)) {
+        abort(abortNode);
+      }
+      // If we get an abort node watch triggered here, we'll go complete creating the acquired
+      // znode but then handle the acquire znode and bail out
+    } catch (KeeperException e) {
+      LOG.error("Failed to watch abort", e);
+      throw new IOException("Failed while watching abort node:" + abortNode, e);
+    }
+
+    // create the acquire barrier
+    String acquire = zkProc.getAcquiredBarrierNode(procName);
+    LOG.debug("Creating acquire znode:" + acquire);
+    try {
+      // notify all the procedure listeners to look for the acquire node
+      byte[] data = ProtobufUtil.prependPBMagic(info);
+      ZKUtil.createWithParents(zkProc.getWatcher(), acquire, data);
+      // loop through all the children of the acquire phase and watch for them
+      for (String node : nodeNames) {
+        String znode = ZKUtil.joinZNode(acquire, node);
+        LOG.debug("Watching for acquire node:" + znode);
+        if (ZKUtil.watchAndCheckExists(zkProc.getWatcher(), znode)) {
+          coordinator.memberAcquiredBarrier(procName, node);
+        }
+      }
+    } catch (KeeperException e) {
+      throw new IOException("Failed while creating acquire node:" + acquire, e);
+    }
+  }
+
+  @Override
+  public void sendGlobalBarrierReached(Procedure proc, List nodeNames) throws IOException {
+    String procName = proc.getName();
+    String reachedNode = zkProc.getReachedBarrierNode(procName);
+    LOG.debug("Creating reached barrier zk node:" + reachedNode);
+    try {
+      // create the reached znode and watch for the reached znodes
+      ZKUtil.createWithParents(zkProc.getWatcher(), reachedNode);
+      // loop through all the children of the acquire phase and watch for them
+      for (String node : nodeNames) {
+        String znode = ZKUtil.joinZNode(reachedNode, node);
+        if (ZKUtil.watchAndCheckExists(zkProc.getWatcher(), znode)) {
+          coordinator.memberFinishedBarrier(procName, node);
+        }
+      }
+    } catch (KeeperException e) {
+      throw new IOException("Failed while creating reached node:" + reachedNode, e);
+    }
+  }
+
+
+  /**
+   * Delete znodes that are no longer in use.
+   */
+  @Override
+  final public void resetMembers(Procedure proc) throws IOException {
+    String procName = proc.getName();
+    boolean stillGettingNotifications = false;
+    do {
+      try {
+        LOG.debug("Attempting to clean out zk node for op:" + procName);
+        zkProc.clearZNodes(procName);
+        stillGettingNotifications = false;
+      } catch (KeeperException.NotEmptyException e) {
+        // recursive delete isn't transactional (yet) so we need to deal with cases where we get
+        // children trickling in
+        stillGettingNotifications = true;
+      } catch (KeeperException e) {
+        throw new IOException("Failed to complete reset procedure " + procName, e);
+      }
+    } while (stillGettingNotifications);
+  }
+
+  /**
+   * Start monitoring znodes in ZK - subclass hook to start monitoring znodes they are about.
+   * @return true if succeed, false if encountered initialization errors.
+   */
+  final public boolean start(final ProcedureCoordinator coordinator) {
+    if (this.coordinator != null) {
+      throw new IllegalStateException(
+        "ZKProcedureCoordinator already started and already has listener installed");
+    }
+    this.coordinator = coordinator;
+
+    try {
+      this.zkProc = new ZKProcedureUtil(watcher, procedureType, coordName) {
+        @Override
+        public void nodeCreated(String path) {
+          if (!isInProcedurePath(path)) return;
+          LOG.debug("Node created: " + path);
+          logZKTree(this.baseZNode);
+          if (isAcquiredPathNode(path)) {
+            // node wasn't present when we created the watch so zk event triggers acquire
+            coordinator.memberAcquiredBarrier(ZKUtil.getNodeName(ZKUtil.getParent(path)),
+              ZKUtil.getNodeName(path));
+          } else if (isReachedPathNode(path)) {
+            // node was absent when we created the watch so zk event triggers the finished barrier.
+
+            // TODO Nothing enforces that acquire and reached znodes from showing up in wrong order.
+            coordinator.memberFinishedBarrier(ZKUtil.getNodeName(ZKUtil.getParent(path)),
+              ZKUtil.getNodeName(path));
+          } else if (isAbortPathNode(path)) {
+            abort(path);
+          }
+        }
+      };
+      zkProc.clearChildZNodes();
+    } catch (KeeperException e) {
+      LOG.error("Unable to start the ZK-based Procedure Coordinator rpcs.", e);
+      return false;
+    }
+
+    LOG.debug("Starting the controller for procedure member:" + zkProc.getMemberName());
+    return true;
+  }
+
+  /**
+   * This is the abort message being sent by the coordinator to member
+   *
+   * TODO this code isn't actually used but can be used to issue a cancellation from the
+   * coordinator.
+   */
+  @Override
+  final public void sendAbortToMembers(Procedure proc, ForeignException ee) {
+    String procName = proc.getName();
+    LOG.debug("Aborting procedure '" + procName + "' in zk");
+    String procAbortNode = zkProc.getAbortZNode(procName);
+    try {
+      LOG.debug("Creating abort znode:" + procAbortNode);
+      String source = (ee.getSource() == null) ? coordName : ee.getSource();
+      byte[] errorInfo = ProtobufUtil.prependPBMagic(ForeignException.serialize(source, ee));
+      // first create the znode for the procedure
+      ZKUtil.createAndFailSilent(zkProc.getWatcher(), procAbortNode, errorInfo);
+      LOG.debug("Finished creating abort node:" + procAbortNode);
+    } catch (KeeperException e) {
+      // possible that we get this error for the procedure if we already reset the zk state, but in
+      // that case we should still get an error for that procedure anyways
+      zkProc.logZKTree(zkProc.baseZNode);
+      coordinator.rpcConnectionFailure("Failed to post zk node:" + procAbortNode
+          + " to abort procedure '" + procName + "'", new IOException(e));
+    }
+  }
+
+  /**
+   * Receive a notification and propagate it to the local coordinator
+   * @param abortNode full znode path to the failed procedure information
+   */
+  protected void abort(String abortNode) {
+    String procName = ZKUtil.getNodeName(abortNode);
+    ForeignException ee = null;
+    try {
+      byte[] data = ZKUtil.getData(zkProc.getWatcher(), abortNode);
+      if (!ProtobufUtil.isPBMagicPrefix(data)) {
+        LOG.warn("Got an error notification for op:" + abortNode
+            + " but we can't read the information. Killing the procedure.");
+        // we got a remote exception, but we can't describe it
+        ee = new ForeignException(coordName, "Data in abort node is illegally formatted.  ignoring content.");
+      } else {
+
+        data = Arrays.copyOfRange(data, ProtobufUtil.lengthOfPBMagic(), data.length);
+        ee = ForeignException.deserialize(data);
+      }
+    } catch (InvalidProtocolBufferException e) {
+      LOG.warn("Got an error notification for op:" + abortNode
+          + " but we can't read the information. Killing the procedure.");
+      // we got a remote exception, but we can't describe it
+      ee = new ForeignException(coordName, e);
+    } catch (KeeperException e) {
+      coordinator.rpcConnectionFailure("Failed to get data for abort node:" + abortNode
+          + zkProc.getAbortZnode(), new IOException(e));
+    }
+    coordinator.abortProcedure(procName, ee);
+  }
+
+  @Override
+  final public void close() throws IOException {
+    zkProc.close();
+  }
+
+  /**
+   * Used in testing
+   */
+  final ZKProcedureUtil getZkProcedureUtil() {
+    return zkProc;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
new file mode 100644
index 0000000..f5a552a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
@@ -0,0 +1,350 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * ZooKeeper based controller for a procedure member.
+ * 
        
+ * There can only be one {@link ZKProcedureMemberRpcs} per procedure type per member,
+ * since each procedure type is bound to a single set of znodes. You can have multiple
+ * {@link ZKProcedureMemberRpcs} on the same server, each serving a different member
+ * name, but each individual rpcs is still bound to a single member name (and since they are
+ * used to determine global progress, its important to not get this wrong).
+ * 
        
+ * To make this slightly more confusing, you can run multiple, concurrent procedures at the same
+ * time (as long as they have different types), from the same controller, but the same node name
+ * must be used for each procedure (though there is no conflict between the two procedure as long
+ * as they have distinct names).
+ * 
        
+ * There is no real error recovery with this mechanism currently -- if any the coordinator fails,
+ * its re-initialization will delete the znodes and require all in progress subprocedures to start
+ * anew.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ZKProcedureMemberRpcs implements ProcedureMemberRpcs {
+
+  private static final Log LOG = LogFactory.getLog(ZKProcedureMemberRpcs.class);
+  private final String memberName;
+
+  protected ProcedureMember member;
+  private ZKProcedureUtil zkController;
+
+  /**
+   * Must call {@link #start(ProcedureMember)} before this can be used.
+   * @param watcher {@link ZooKeeperWatcher} to be owned by this. Closed via
+   *          {@link #close()}.
+   * @param procType name of the znode describing the procedure type
+   * @param memberName name of the member to join the procedure
+   * @throws KeeperException if we can't reach zookeeper
+   */
+  public ZKProcedureMemberRpcs(ZooKeeperWatcher watcher,
+      String procType, String memberName) throws KeeperException {
+    this.zkController = new ZKProcedureUtil(watcher, procType, memberName) {
+      @Override
+      public void nodeCreated(String path) {
+        if (!isInProcedurePath(path)) {
+          return;
+        }
+
+        LOG.info("Received created event:" + path);
+        // if it is a simple start/end/abort then we just rewatch the node
+        if (isAcquiredNode(path)) {
+          waitForNewProcedures();
+          return;
+        } else if (isAbortNode(path)) {
+          watchForAbortedProcedures();
+          return;
+        }
+        String parent = ZKUtil.getParent(path);
+        // if its the end barrier, the procedure can be completed
+        if (isReachedNode(parent)) {
+          receivedReachedGlobalBarrier(path);
+          return;
+        } else if (isAbortNode(parent)) {
+          abort(path);
+          return;
+        } else if (isAcquiredNode(parent)) {
+          startNewSubprocedure(path);
+        } else {
+          LOG.debug("Ignoring created notification for node:" + path);
+        }
+      }
+
+      @Override
+      public void nodeChildrenChanged(String path) {
+        if (path.equals(this.acquiredZnode)) {
+          LOG.info("Received procedure start children changed event: " + path);
+          waitForNewProcedures();
+        } else if (path.equals(this.abortZnode)) {
+          LOG.info("Received procedure abort children changed event: " + path);
+          watchForAbortedProcedures();
+        }
+      }
+    };
+    this.memberName = memberName;
+  }
+
+  public ZKProcedureUtil getZkController() {
+    return zkController;
+  }
+
+  @Override
+  public String getMemberName() {
+    return memberName;
+  }
+
+  /**
+   * Pass along the procedure global barrier notification to any listeners
+   * @param path full znode path that cause the notification
+   */
+  private void receivedReachedGlobalBarrier(String path) {
+    LOG.debug("Recieved reached global barrier:" + path);
+    String procName = ZKUtil.getNodeName(path);
+    this.member.receivedReachedGlobalBarrier(procName);
+  }
+
+  private void watchForAbortedProcedures() {
+    LOG.debug("Checking for aborted procedures on node: '" + zkController.getAbortZnode() + "'");
+    try {
+      // this is the list of the currently aborted procedues
+      for (String node : ZKUtil.listChildrenAndWatchForNewChildren(zkController.getWatcher(),
+        zkController.getAbortZnode())) {
+        String abortNode = ZKUtil.joinZNode(zkController.getAbortZnode(), node);
+        abort(abortNode);
+      }
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("Failed to list children for abort node:"
+          + zkController.getAbortZnode(), new IOException(e));
+    }
+  }
+
+  private void waitForNewProcedures() {
+    // watch for new procedues that we need to start subprocedures for
+    LOG.debug("Looking for new procedures under znode:'" + zkController.getAcquiredBarrier() + "'");
+    List runningProcedures = null;
+    try {
+      runningProcedures = ZKUtil.listChildrenAndWatchForNewChildren(zkController.getWatcher(),
+        zkController.getAcquiredBarrier());
+      if (runningProcedures == null) {
+        LOG.debug("No running procedures.");
+        return;
+      }
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("General failure when watching for new procedures",
+        new IOException(e));
+    }
+    if (runningProcedures == null) {
+      LOG.debug("No running procedures.");
+      return;
+    }
+    for (String procName : runningProcedures) {
+      // then read in the procedure information
+      String path = ZKUtil.joinZNode(zkController.getAcquiredBarrier(), procName);
+      startNewSubprocedure(path);
+    }
+  }
+
+  /**
+   * Kick off a new sub-procedure on the listener with the data stored in the passed znode.
+   * 
        
+   * Will attempt to create the same procedure multiple times if an procedure znode with the same
+   * name is created. It is left up the coordinator to ensure this doesn't occur.
+   * @param path full path to the znode for the procedure to start
+   */
+  private synchronized void startNewSubprocedure(String path) {
+    LOG.debug("Found procedure znode: " + path);
+    String opName = ZKUtil.getNodeName(path);
+    // start watching for an abort notification for the procedure
+    String abortZNode = zkController.getAbortZNode(opName);
+    try {
+      if (ZKUtil.watchAndCheckExists(zkController.getWatcher(), abortZNode)) {
+        LOG.debug("Not starting:" + opName + " because we already have an abort notification.");
+        return;
+      }
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("Failed to get the abort znode (" + abortZNode
+          + ") for procedure :" + opName, new IOException(e));
+      return;
+    }
+
+    // get the data for the procedure
+    Subprocedure subproc = null;
+    try {
+      byte[] data = ZKUtil.getData(zkController.getWatcher(), path);
+      LOG.debug("start proc data length is " + data.length);
+      if (!ProtobufUtil.isPBMagicPrefix(data)) {
+        String msg = "Data in for starting procuedure " + opName + " is illegally formatted. "
+            + "Killing the procedure.";
+        LOG.error(msg);
+        throw new IllegalArgumentException(msg);
+      }
+      data = Arrays.copyOfRange(data, ProtobufUtil.lengthOfPBMagic(), data.length);
+      LOG.debug("Found data for znode:" + path);
+      subproc = member.createSubprocedure(opName, data);
+      member.submitSubprocedure(subproc);
+    } catch (IllegalArgumentException iae ) {
+      LOG.error("Illegal argument exception", iae);
+      sendMemberAborted(subproc, new ForeignException(getMemberName(), iae));
+    } catch (IllegalStateException ise) {
+      LOG.error("Illegal state exception ", ise);
+      sendMemberAborted(subproc, new ForeignException(getMemberName(), ise));
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("Failed to get data for new procedure:" + opName,
+        new IOException(e));
+    }
+  }
+
+  /**
+   * This attempts to create an acquired state znode for the procedure (snapshot name).
+   *
+   * It then looks for the reached znode to trigger in-barrier execution.  If not present we
+   * have a watcher, if present then trigger the in-barrier action.
+   */
+  @Override
+  public void sendMemberAcquired(Subprocedure sub) throws IOException {
+    String procName = sub.getName();
+    try {
+      LOG.debug("Member: '" + memberName + "' joining acquired barrier for procedure (" + procName
+          + ") in zk");
+      String acquiredZNode = ZKUtil.joinZNode(ZKProcedureUtil.getAcquireBarrierNode(
+        zkController, procName), memberName);
+      ZKUtil.createAndFailSilent(zkController.getWatcher(), acquiredZNode);
+
+      // watch for the complete node for this snapshot
+      String reachedBarrier = zkController.getReachedBarrierNode(procName);
+      LOG.debug("Watch for global barrier reached:" + reachedBarrier);
+      if (ZKUtil.watchAndCheckExists(zkController.getWatcher(), reachedBarrier)) {
+        receivedReachedGlobalBarrier(reachedBarrier);
+      }
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("Failed to acquire barrier for procedure: "
+          + procName + " and member: " + memberName, new IOException(e));
+    }
+  }
+
+  /**
+   * This acts as the ack for a completed snapshot
+   */
+  @Override
+  public void sendMemberCompleted(Subprocedure sub) throws IOException {
+    String procName = sub.getName();
+    LOG.debug("Marking procedure  '" + procName + "' completed for member '" + memberName
+        + "' in zk");
+    String joinPath = ZKUtil.joinZNode(zkController.getReachedBarrierNode(procName), memberName);
+    try {
+      ZKUtil.createAndFailSilent(zkController.getWatcher(), joinPath);
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("Failed to post zk node:" + joinPath
+          + " to join procedure barrier.", new IOException(e));
+    }
+  }
+
+  /**
+   * This should be called by the member and should write a serialized root cause exception as
+   * to the abort znode.
+   */
+  @Override
+  public void sendMemberAborted(Subprocedure sub, ForeignException ee) {
+    if (sub == null) {
+      LOG.error("Failed due to null subprocedure", ee);
+      return;
+    }
+    String procName = sub.getName();
+    LOG.debug("Aborting procedure (" + procName + ") in zk");
+    String procAbortZNode = zkController.getAbortZNode(procName);
+    try {
+      String source = (ee.getSource() == null) ? memberName: ee.getSource();
+      byte[] errorInfo = ProtobufUtil.prependPBMagic(ForeignException.serialize(source, ee));
+      ZKUtil.createAndFailSilent(zkController.getWatcher(), procAbortZNode, errorInfo);
+      LOG.debug("Finished creating abort znode:" + procAbortZNode);
+    } catch (KeeperException e) {
+      // possible that we get this error for the procedure if we already reset the zk state, but in
+      // that case we should still get an error for that procedure anyways
+      zkController.logZKTree(zkController.getBaseZnode());
+      member.controllerConnectionFailure("Failed to post zk node:" + procAbortZNode
+          + " to abort procedure", new IOException(e));
+    }
+  }
+
+  /**
+   * Pass along the found abort notification to the listener
+   * @param abortZNode full znode path to the failed procedure information
+   */
+  protected void abort(String abortZNode) {
+    LOG.debug("Aborting procedure member for znode " + abortZNode);
+    String opName = ZKUtil.getNodeName(abortZNode);
+    try {
+      byte[] data = ZKUtil.getData(zkController.getWatcher(), abortZNode);
+
+      // figure out the data we need to pass
+      ForeignException ee;
+      try {
+        if (!ProtobufUtil.isPBMagicPrefix(data)) {
+          String msg = "Illegally formatted data in abort node for proc " + opName
+              + ".  Killing the procedure.";
+          LOG.error(msg);
+          // we got a remote exception, but we can't describe it so just return exn from here
+          ee = new ForeignException(getMemberName(), new IllegalArgumentException(msg));
+        } else {
+          data = Arrays.copyOfRange(data, ProtobufUtil.lengthOfPBMagic(), data.length);
+          ee = ForeignException.deserialize(data);
+        }
+      } catch (InvalidProtocolBufferException e) {
+        LOG.warn("Got an error notification for op:" + opName
+            + " but we can't read the information. Killing the procedure.");
+        // we got a remote exception, but we can't describe it so just return exn from here
+        ee = new ForeignException(getMemberName(), e);
+      }
+
+      this.member.receiveAbortProcedure(opName, ee);
+    } catch (KeeperException e) {
+      member.controllerConnectionFailure("Failed to get data for abort znode:" + abortZNode
+          + zkController.getAbortZnode(), new IOException(e));
+    }
+  }
+
+  public void start(ProcedureMember listener) {
+    LOG.debug("Starting procedure member '" + this.memberName + "'");
+    this.member = listener;
+    watchForAbortedProcedures();
+    waitForNewProcedures();
+  }
+
+  @Override
+  public void close() throws IOException {
+    zkController.close();
+  }
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
new file mode 100644
index 0000000..0a4dd4a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
@@ -0,0 +1,285 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * This is a shared ZooKeeper-based znode management utils for distributed procedure.  All znode
+ * operations should go through the provided methods in coordinators and members.
+ *
+ * Layout of nodes in ZK is
+ * /hbase/[op name]/acquired/
+ *                    [op instance] - op data/
+ *                        /[nodes that have acquired]
+ *                 /reached/
+ *                    [op instance]/
+ *                        /[nodes that have completed]
+ *                 /abort/
+ *                    [op instance] - failure data
+ *
+ * NOTE: while acquired and completed are znode dirs, abort is actually just a znode.
+ *
+ * Assumption here that procedure names are unique
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class ZKProcedureUtil
+    extends ZooKeeperListener implements Closeable {
+
+  private static final Log LOG = LogFactory.getLog(ZKProcedureUtil.class);
+
+  public static final String ACQUIRED_BARRIER_ZNODE_DEFAULT = "acquired";
+  public static final String REACHED_BARRIER_ZNODE_DEFAULT = "reached";
+  public static final String ABORT_ZNODE_DEFAULT = "abort";
+
+  public final String baseZNode;
+  protected final String acquiredZnode;
+  protected final String reachedZnode;
+  protected final String abortZnode;
+
+  protected final String memberName;
+
+  /**
+   * Top-level watcher/controller for procedures across the cluster.
+   * 
        
+   * On instantiation, this ensures the procedure znodes exist.  This however requires the passed in
+   *  watcher has been started.
+   * @param watcher watcher for the cluster ZK. Owned by this and closed via
+   *          {@link #close()}
+   * @param procDescription name of the znode describing the procedure to run
+   * @param memberName name of the member from which we are interacting with running procedures
+   * @throws KeeperException when the procedure znodes cannot be created
+   */
+  public ZKProcedureUtil(ZooKeeperWatcher watcher, String procDescription,
+      String memberName) throws KeeperException {
+    super(watcher);
+    this.memberName = memberName;
+    // make sure we are listening for events
+    watcher.registerListener(this);
+    // setup paths for the zknodes used in procedures
+    this.baseZNode = ZKUtil.joinZNode(watcher.baseZNode, procDescription);
+    acquiredZnode = ZKUtil.joinZNode(baseZNode, ACQUIRED_BARRIER_ZNODE_DEFAULT);
+    reachedZnode = ZKUtil.joinZNode(baseZNode, REACHED_BARRIER_ZNODE_DEFAULT);
+    abortZnode = ZKUtil.joinZNode(baseZNode, ABORT_ZNODE_DEFAULT);
+
+    // first make sure all the ZK nodes exist
+    // make sure all the parents exist (sometimes not the case in tests)
+    ZKUtil.createWithParents(watcher, acquiredZnode);
+    // regular create because all the parents exist
+    ZKUtil.createAndFailSilent(watcher, reachedZnode);
+    ZKUtil.createAndFailSilent(watcher, abortZnode);
+  }
+
+  @Override
+  public void close() throws IOException {
+    // the watcher is passed from either Master or Region Server
+    // watcher.close() will be called by the owner so no need to call close() here
+  }
+
+  public String getAcquiredBarrierNode(String opInstanceName) {
+    return ZKProcedureUtil.getAcquireBarrierNode(this, opInstanceName);
+  }
+
+  public String getReachedBarrierNode(String opInstanceName) {
+    return ZKProcedureUtil.getReachedBarrierNode(this, opInstanceName);
+  }
+
+  public String getAbortZNode(String opInstanceName) {
+    return ZKProcedureUtil.getAbortNode(this, opInstanceName);
+  }
+
+  public String getAbortZnode() {
+    return abortZnode;
+  }
+
+  public String getBaseZnode() {
+    return baseZNode;
+  }
+
+  public String getAcquiredBarrier() {
+    return acquiredZnode;
+  }
+
+  public String getMemberName() {
+    return memberName;
+  }
+
+  /**
+   * Get the full znode path for the node used by the coordinator to trigger a global barrier
+   * acquire on each subprocedure.
+   * @param controller controller running the procedure
+   * @param opInstanceName name of the running procedure instance (not the procedure description).
+   * @return full znode path to the prepare barrier/start node
+   */
+  public static String getAcquireBarrierNode(ZKProcedureUtil controller,
+      String opInstanceName) {
+    return ZKUtil.joinZNode(controller.acquiredZnode, opInstanceName);
+  }
+
+  /**
+   * Get the full znode path for the node used by the coordinator to trigger a global barrier
+   * execution and release on each subprocedure.
+   * @param controller controller running the procedure
+   * @param opInstanceName name of the running procedure instance (not the procedure description).
+   * @return full znode path to the commit barrier
+   */
+  public static String getReachedBarrierNode(ZKProcedureUtil controller,
+      String opInstanceName) {
+    return ZKUtil.joinZNode(controller.reachedZnode, opInstanceName);
+  }
+
+  /**
+   * Get the full znode path for the node used by the coordinator or member to trigger an abort
+   * of the global barrier acquisition or execution in subprocedures.
+   * @param controller controller running the procedure
+   * @param opInstanceName name of the running procedure instance (not the procedure description).
+   * @return full znode path to the abort znode
+   */
+  public static String getAbortNode(ZKProcedureUtil controller, String opInstanceName) {
+    return ZKUtil.joinZNode(controller.abortZnode, opInstanceName);
+  }
+
+  public ZooKeeperWatcher getWatcher() {
+    return watcher;
+  }
+
+  /**
+   * Is this a procedure related znode path?
+   *
+   * TODO: this is not strict, can return true if had name just starts with same prefix but is
+   * different zdir.
+   *
+   * @return true if starts with baseZnode
+   */
+  boolean isInProcedurePath(String path) {
+    return path.startsWith(baseZNode);
+  }
+
+  /**
+   * Is this the exact procedure barrier acquired znode
+   */
+  boolean isAcquiredNode(String path) {
+    return path.equals(acquiredZnode);
+  }
+
+  
+  /**
+   * Is this in the procedure barrier acquired znode path
+   */
+  boolean isAcquiredPathNode(String path) {
+    return path.startsWith(this.acquiredZnode) && !path.equals(acquiredZnode);
+  }
+
+  /**
+   * Is this the exact procedure barrier reached znode
+   */
+  boolean isReachedNode(String path) {
+    return path.equals(reachedZnode);
+  }
+
+  /**
+   * Is this in the procedure barrier reached znode path
+   */
+  boolean isReachedPathNode(String path) {
+    return path.startsWith(this.reachedZnode) && !path.equals(reachedZnode);
+  }
+
+
+  /**
+   * Is this in the procedure barrier abort znode path
+   */
+  boolean isAbortNode(String path) {
+    return path.equals(abortZnode);
+  }
+
+  /**
+   * Is this in the procedure barrier abort znode path
+   */
+  public boolean isAbortPathNode(String path) {
+    return path.startsWith(this.abortZnode) && !path.equals(abortZnode);
+  }
+
+  // --------------------------------------------------------------------------
+  // internal debugging methods
+  // --------------------------------------------------------------------------
+  /**
+   * Recursively print the current state of ZK (non-transactional)
+   * @param root name of the root directory in zk to print
+   * @throws KeeperException
+   */
+  void logZKTree(String root) {
+    if (!LOG.isDebugEnabled()) return;
+    LOG.debug("Current zk system:");
+    String prefix = "|-";
+    LOG.debug(prefix + root);
+    try {
+      logZKTree(root, prefix);
+    } catch (KeeperException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Helper method to print the current state of the ZK tree.
+   * @see #logZKTree(String)
+   * @throws KeeperException if an unexpected exception occurs
+   */
+  protected void logZKTree(String root, String prefix) throws KeeperException {
+    List children = ZKUtil.listChildrenNoWatch(watcher, root);
+    if (children == null) return;
+    for (String child : children) {
+      LOG.debug(prefix + child);
+      String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child);
+      logZKTree(node, prefix + "---");
+    }
+  }
+
+  public void clearChildZNodes() throws KeeperException {
+    // TODO This is potentially racy since not atomic. update when we support zk that has multi
+    LOG.info("Clearing all procedure znodes: " + acquiredZnode + " " + reachedZnode + " "
+        + abortZnode);
+
+    // If the coordinator was shutdown mid-procedure, then we are going to lose
+    // an procedure that was previously started by cleaning out all the previous state. Its much
+    // harder to figure out how to keep an procedure going and the subject of HBASE-5487.
+    ZKUtil.deleteChildrenRecursively(watcher, acquiredZnode);
+    ZKUtil.deleteChildrenRecursively(watcher, reachedZnode);
+    ZKUtil.deleteChildrenRecursively(watcher, abortZnode);
+  }
+
+  public void clearZNodes(String procedureName) throws KeeperException {
+    // TODO This is potentially racy since not atomic. update when we support zk that has multi
+    LOG.info("Clearing all znodes for procedure " + procedureName + "including nodes "
+        + acquiredZnode + " " + reachedZnode + " " + abortZnode);
+    ZKUtil.deleteNodeRecursively(watcher, getAcquiredBarrierNode(procedureName));
+    ZKUtil.deleteNodeRecursively(watcher, getReachedBarrierNode(procedureName));
+    ZKUtil.deleteNodeRecursively(watcher, getAbortZNode(procedureName));
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
new file mode 100644
index 0000000..65c87d3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.protobuf;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Protobufs utility.
+ */
+@SuppressWarnings("deprecation")
+public final class ProtobufUtil {
+
+  private ProtobufUtil() {
+  }
+
+  /**
+   * Magic we put ahead of a serialized protobuf message.
+   * For example, all znode content is protobuf messages with the below magic
+   * for preamble.
+   */
+  public static final byte [] PB_MAGIC = new byte [] {'P', 'B', 'U', 'F'};
+  private static final String PB_MAGIC_STR = Bytes.toString(PB_MAGIC);
+
+  /**
+   * Prepend the passed bytes with four bytes of magic, {@link #PB_MAGIC}, to flag what
+   * follows as a protobuf in hbase.  Prepend these bytes to all content written to znodes, etc.
+   * @param bytes Bytes to decorate
+   * @return The passed bytes with magic prepended (Creates a new
+   * byte array that is bytes.length plus {@link #PB_MAGIC}.length.
+   */
+  public static byte [] prependPBMagic(final byte [] bytes) {
+    return Bytes.add(PB_MAGIC, bytes);
+  }
+
+  /**
+   * @param bytes Bytes to check.
+   * @return True if passed bytes has {@link #PB_MAGIC} for a prefix.
+   */
+  public static boolean isPBMagicPrefix(final byte [] bytes) {
+    if (bytes == null || bytes.length < PB_MAGIC.length) return false;
+    return Bytes.compareTo(PB_MAGIC, 0, PB_MAGIC.length, bytes, 0, PB_MAGIC.length) == 0;
+  }
+
+  /**
+   * @return Length of {@link #PB_MAGIC}
+   */
+  public static int lengthOfPBMagic() {
+    return PB_MAGIC.length;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ErrorHandlingProtos.java b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ErrorHandlingProtos.java
new file mode 100644
index 0000000..067321f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ErrorHandlingProtos.java
@@ -0,0 +1,2185 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: ErrorHandling.proto
+
+package org.apache.hadoop.hbase.protobuf.generated;
+
+public final class ErrorHandlingProtos {
+  private ErrorHandlingProtos() {}
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistry registry) {
+  }
+  public interface StackTraceElementMessageOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string declaringClass = 1;
+    boolean hasDeclaringClass();
+    String getDeclaringClass();
+    
+    // optional string methodName = 2;
+    boolean hasMethodName();
+    String getMethodName();
+    
+    // optional string fileName = 3;
+    boolean hasFileName();
+    String getFileName();
+    
+    // optional int32 lineNumber = 4;
+    boolean hasLineNumber();
+    int getLineNumber();
+  }
+  public static final class StackTraceElementMessage extends
+      com.google.protobuf.GeneratedMessage
+      implements StackTraceElementMessageOrBuilder {
+    // Use StackTraceElementMessage.newBuilder() to construct.
+    private StackTraceElementMessage(Builder builder) {
+      super(builder);
+    }
+    private StackTraceElementMessage(boolean noInit) {}
+    
+    private static final StackTraceElementMessage defaultInstance;
+    public static StackTraceElementMessage getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public StackTraceElementMessage getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_StackTraceElementMessage_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_StackTraceElementMessage_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string declaringClass = 1;
+    public static final int DECLARINGCLASS_FIELD_NUMBER = 1;
+    private java.lang.Object declaringClass_;
+    public boolean hasDeclaringClass() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getDeclaringClass() {
+      java.lang.Object ref = declaringClass_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          declaringClass_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getDeclaringClassBytes() {
+      java.lang.Object ref = declaringClass_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        declaringClass_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string methodName = 2;
+    public static final int METHODNAME_FIELD_NUMBER = 2;
+    private java.lang.Object methodName_;
+    public boolean hasMethodName() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getMethodName() {
+      java.lang.Object ref = methodName_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          methodName_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getMethodNameBytes() {
+      java.lang.Object ref = methodName_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        methodName_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string fileName = 3;
+    public static final int FILENAME_FIELD_NUMBER = 3;
+    private java.lang.Object fileName_;
+    public boolean hasFileName() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public String getFileName() {
+      java.lang.Object ref = fileName_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          fileName_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getFileNameBytes() {
+      java.lang.Object ref = fileName_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        fileName_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int32 lineNumber = 4;
+    public static final int LINENUMBER_FIELD_NUMBER = 4;
+    private int lineNumber_;
+    public boolean hasLineNumber() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    public int getLineNumber() {
+      return lineNumber_;
+    }
+    
+    private void initFields() {
+      declaringClass_ = "";
+      methodName_ = "";
+      fileName_ = "";
+      lineNumber_ = 0;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getDeclaringClassBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getMethodNameBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeBytes(3, getFileNameBytes());
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        output.writeInt32(4, lineNumber_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getDeclaringClassBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getMethodNameBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(3, getFileNameBytes());
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(4, lineNumber_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage)) {
+        return super.equals(obj);
+      }
+      org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage other = (org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage) obj;
+      
+      boolean result = true;
+      result = result && (hasDeclaringClass() == other.hasDeclaringClass());
+      if (hasDeclaringClass()) {
+        result = result && getDeclaringClass()
+            .equals(other.getDeclaringClass());
+      }
+      result = result && (hasMethodName() == other.hasMethodName());
+      if (hasMethodName()) {
+        result = result && getMethodName()
+            .equals(other.getMethodName());
+      }
+      result = result && (hasFileName() == other.hasFileName());
+      if (hasFileName()) {
+        result = result && getFileName()
+            .equals(other.getFileName());
+      }
+      result = result && (hasLineNumber() == other.hasLineNumber());
+      if (hasLineNumber()) {
+        result = result && (getLineNumber()
+            == other.getLineNumber());
+      }
+      result = result &&
+          getUnknownFields().equals(other.getUnknownFields());
+      return result;
+    }
+    
+    @java.lang.Override
+    public int hashCode() {
+      int hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      if (hasDeclaringClass()) {
+        hash = (37 * hash) + DECLARINGCLASS_FIELD_NUMBER;
+        hash = (53 * hash) + getDeclaringClass().hashCode();
+      }
+      if (hasMethodName()) {
+        hash = (37 * hash) + METHODNAME_FIELD_NUMBER;
+        hash = (53 * hash) + getMethodName().hashCode();
+      }
+      if (hasFileName()) {
+        hash = (37 * hash) + FILENAME_FIELD_NUMBER;
+        hash = (53 * hash) + getFileName().hashCode();
+      }
+      if (hasLineNumber()) {
+        hash = (37 * hash) + LINENUMBER_FIELD_NUMBER;
+        hash = (53 * hash) + getLineNumber();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      return hash;
+    }
+    
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_StackTraceElementMessage_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_StackTraceElementMessage_fieldAccessorTable;
+      }
+      
+      // Construct using org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        declaringClass_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        methodName_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        fileName_ = "";
+        bitField0_ = (bitField0_ & ~0x00000004);
+        lineNumber_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000008);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.getDescriptor();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage getDefaultInstanceForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.getDefaultInstance();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage build() {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage buildPartial() {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage result = new org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.declaringClass_ = declaringClass_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.methodName_ = methodName_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.fileName_ = fileName_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
+        }
+        result.lineNumber_ = lineNumber_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage) {
+          return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage other) {
+        if (other == org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.getDefaultInstance()) return this;
+        if (other.hasDeclaringClass()) {
+          setDeclaringClass(other.getDeclaringClass());
+        }
+        if (other.hasMethodName()) {
+          setMethodName(other.getMethodName());
+        }
+        if (other.hasFileName()) {
+          setFileName(other.getFileName());
+        }
+        if (other.hasLineNumber()) {
+          setLineNumber(other.getLineNumber());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              declaringClass_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              methodName_ = input.readBytes();
+              break;
+            }
+            case 26: {
+              bitField0_ |= 0x00000004;
+              fileName_ = input.readBytes();
+              break;
+            }
+            case 32: {
+              bitField0_ |= 0x00000008;
+              lineNumber_ = input.readInt32();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string declaringClass = 1;
+      private java.lang.Object declaringClass_ = "";
+      public boolean hasDeclaringClass() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getDeclaringClass() {
+        java.lang.Object ref = declaringClass_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          declaringClass_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setDeclaringClass(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        declaringClass_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearDeclaringClass() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        declaringClass_ = getDefaultInstance().getDeclaringClass();
+        onChanged();
+        return this;
+      }
+      void setDeclaringClass(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        declaringClass_ = value;
+        onChanged();
+      }
+      
+      // optional string methodName = 2;
+      private java.lang.Object methodName_ = "";
+      public boolean hasMethodName() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getMethodName() {
+        java.lang.Object ref = methodName_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          methodName_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setMethodName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        methodName_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearMethodName() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        methodName_ = getDefaultInstance().getMethodName();
+        onChanged();
+        return this;
+      }
+      void setMethodName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        methodName_ = value;
+        onChanged();
+      }
+      
+      // optional string fileName = 3;
+      private java.lang.Object fileName_ = "";
+      public boolean hasFileName() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public String getFileName() {
+        java.lang.Object ref = fileName_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          fileName_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setFileName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000004;
+        fileName_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearFileName() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        fileName_ = getDefaultInstance().getFileName();
+        onChanged();
+        return this;
+      }
+      void setFileName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000004;
+        fileName_ = value;
+        onChanged();
+      }
+      
+      // optional int32 lineNumber = 4;
+      private int lineNumber_ ;
+      public boolean hasLineNumber() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      public int getLineNumber() {
+        return lineNumber_;
+      }
+      public Builder setLineNumber(int value) {
+        bitField0_ |= 0x00000008;
+        lineNumber_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearLineNumber() {
+        bitField0_ = (bitField0_ & ~0x00000008);
+        lineNumber_ = 0;
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:StackTraceElementMessage)
+    }
+    
+    static {
+      defaultInstance = new StackTraceElementMessage(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:StackTraceElementMessage)
+  }
+  
+  public interface GenericExceptionMessageOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string className = 1;
+    boolean hasClassName();
+    String getClassName();
+    
+    // optional string message = 2;
+    boolean hasMessage();
+    String getMessage();
+    
+    // optional bytes errorInfo = 3;
+    boolean hasErrorInfo();
+    com.google.protobuf.ByteString getErrorInfo();
+    
+    // repeated .StackTraceElementMessage trace = 4;
+    java.util.List 
+        getTraceList();
+    org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage getTrace(int index);
+    int getTraceCount();
+    java.util.List 
+        getTraceOrBuilderList();
+    org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder getTraceOrBuilder(
+        int index);
+  }
+  public static final class GenericExceptionMessage extends
+      com.google.protobuf.GeneratedMessage
+      implements GenericExceptionMessageOrBuilder {
+    // Use GenericExceptionMessage.newBuilder() to construct.
+    private GenericExceptionMessage(Builder builder) {
+      super(builder);
+    }
+    private GenericExceptionMessage(boolean noInit) {}
+    
+    private static final GenericExceptionMessage defaultInstance;
+    public static GenericExceptionMessage getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public GenericExceptionMessage getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_GenericExceptionMessage_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_GenericExceptionMessage_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string className = 1;
+    public static final int CLASSNAME_FIELD_NUMBER = 1;
+    private java.lang.Object className_;
+    public boolean hasClassName() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getClassName() {
+      java.lang.Object ref = className_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          className_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getClassNameBytes() {
+      java.lang.Object ref = className_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        className_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string message = 2;
+    public static final int MESSAGE_FIELD_NUMBER = 2;
+    private java.lang.Object message_;
+    public boolean hasMessage() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getMessage() {
+      java.lang.Object ref = message_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          message_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getMessageBytes() {
+      java.lang.Object ref = message_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        message_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional bytes errorInfo = 3;
+    public static final int ERRORINFO_FIELD_NUMBER = 3;
+    private com.google.protobuf.ByteString errorInfo_;
+    public boolean hasErrorInfo() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public com.google.protobuf.ByteString getErrorInfo() {
+      return errorInfo_;
+    }
+    
+    // repeated .StackTraceElementMessage trace = 4;
+    public static final int TRACE_FIELD_NUMBER = 4;
+    private java.util.List trace_;
+    public java.util.List getTraceList() {
+      return trace_;
+    }
+    public java.util.List 
+        getTraceOrBuilderList() {
+      return trace_;
+    }
+    public int getTraceCount() {
+      return trace_.size();
+    }
+    public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage getTrace(int index) {
+      return trace_.get(index);
+    }
+    public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder getTraceOrBuilder(
+        int index) {
+      return trace_.get(index);
+    }
+    
+    private void initFields() {
+      className_ = "";
+      message_ = "";
+      errorInfo_ = com.google.protobuf.ByteString.EMPTY;
+      trace_ = java.util.Collections.emptyList();
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getClassNameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getMessageBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeBytes(3, errorInfo_);
+      }
+      for (int i = 0; i < trace_.size(); i++) {
+        output.writeMessage(4, trace_.get(i));
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getClassNameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getMessageBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(3, errorInfo_);
+      }
+      for (int i = 0; i < trace_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(4, trace_.get(i));
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage)) {
+        return super.equals(obj);
+      }
+      org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage other = (org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage) obj;
+      
+      boolean result = true;
+      result = result && (hasClassName() == other.hasClassName());
+      if (hasClassName()) {
+        result = result && getClassName()
+            .equals(other.getClassName());
+      }
+      result = result && (hasMessage() == other.hasMessage());
+      if (hasMessage()) {
+        result = result && getMessage()
+            .equals(other.getMessage());
+      }
+      result = result && (hasErrorInfo() == other.hasErrorInfo());
+      if (hasErrorInfo()) {
+        result = result && getErrorInfo()
+            .equals(other.getErrorInfo());
+      }
+      result = result && getTraceList()
+          .equals(other.getTraceList());
+      result = result &&
+          getUnknownFields().equals(other.getUnknownFields());
+      return result;
+    }
+    
+    @java.lang.Override
+    public int hashCode() {
+      int hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      if (hasClassName()) {
+        hash = (37 * hash) + CLASSNAME_FIELD_NUMBER;
+        hash = (53 * hash) + getClassName().hashCode();
+      }
+      if (hasMessage()) {
+        hash = (37 * hash) + MESSAGE_FIELD_NUMBER;
+        hash = (53 * hash) + getMessage().hashCode();
+      }
+      if (hasErrorInfo()) {
+        hash = (37 * hash) + ERRORINFO_FIELD_NUMBER;
+        hash = (53 * hash) + getErrorInfo().hashCode();
+      }
+      if (getTraceCount() > 0) {
+        hash = (37 * hash) + TRACE_FIELD_NUMBER;
+        hash = (53 * hash) + getTraceList().hashCode();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      return hash;
+    }
+    
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_GenericExceptionMessage_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_GenericExceptionMessage_fieldAccessorTable;
+      }
+      
+      // Construct using org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getTraceFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        className_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        message_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        errorInfo_ = com.google.protobuf.ByteString.EMPTY;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        if (traceBuilder_ == null) {
+          trace_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000008);
+        } else {
+          traceBuilder_.clear();
+        }
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDescriptor();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage getDefaultInstanceForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage build() {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage buildPartial() {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage result = new org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.className_ = className_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.message_ = message_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.errorInfo_ = errorInfo_;
+        if (traceBuilder_ == null) {
+          if (((bitField0_ & 0x00000008) == 0x00000008)) {
+            trace_ = java.util.Collections.unmodifiableList(trace_);
+            bitField0_ = (bitField0_ & ~0x00000008);
+          }
+          result.trace_ = trace_;
+        } else {
+          result.trace_ = traceBuilder_.build();
+        }
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage) {
+          return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage other) {
+        if (other == org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance()) return this;
+        if (other.hasClassName()) {
+          setClassName(other.getClassName());
+        }
+        if (other.hasMessage()) {
+          setMessage(other.getMessage());
+        }
+        if (other.hasErrorInfo()) {
+          setErrorInfo(other.getErrorInfo());
+        }
+        if (traceBuilder_ == null) {
+          if (!other.trace_.isEmpty()) {
+            if (trace_.isEmpty()) {
+              trace_ = other.trace_;
+              bitField0_ = (bitField0_ & ~0x00000008);
+            } else {
+              ensureTraceIsMutable();
+              trace_.addAll(other.trace_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.trace_.isEmpty()) {
+            if (traceBuilder_.isEmpty()) {
+              traceBuilder_.dispose();
+              traceBuilder_ = null;
+              trace_ = other.trace_;
+              bitField0_ = (bitField0_ & ~0x00000008);
+              traceBuilder_ = 
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+                   getTraceFieldBuilder() : null;
+            } else {
+              traceBuilder_.addAllMessages(other.trace_);
+            }
+          }
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              className_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              message_ = input.readBytes();
+              break;
+            }
+            case 26: {
+              bitField0_ |= 0x00000004;
+              errorInfo_ = input.readBytes();
+              break;
+            }
+            case 34: {
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder subBuilder = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.newBuilder();
+              input.readMessage(subBuilder, extensionRegistry);
+              addTrace(subBuilder.buildPartial());
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string className = 1;
+      private java.lang.Object className_ = "";
+      public boolean hasClassName() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getClassName() {
+        java.lang.Object ref = className_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          className_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setClassName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        className_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearClassName() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        className_ = getDefaultInstance().getClassName();
+        onChanged();
+        return this;
+      }
+      void setClassName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        className_ = value;
+        onChanged();
+      }
+      
+      // optional string message = 2;
+      private java.lang.Object message_ = "";
+      public boolean hasMessage() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getMessage() {
+        java.lang.Object ref = message_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          message_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setMessage(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        message_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearMessage() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        message_ = getDefaultInstance().getMessage();
+        onChanged();
+        return this;
+      }
+      void setMessage(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        message_ = value;
+        onChanged();
+      }
+      
+      // optional bytes errorInfo = 3;
+      private com.google.protobuf.ByteString errorInfo_ = com.google.protobuf.ByteString.EMPTY;
+      public boolean hasErrorInfo() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public com.google.protobuf.ByteString getErrorInfo() {
+        return errorInfo_;
+      }
+      public Builder setErrorInfo(com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000004;
+        errorInfo_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearErrorInfo() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        errorInfo_ = getDefaultInstance().getErrorInfo();
+        onChanged();
+        return this;
+      }
+      
+      // repeated .StackTraceElementMessage trace = 4;
+      private java.util.List trace_ =
+        java.util.Collections.emptyList();
+      private void ensureTraceIsMutable() {
+        if (!((bitField0_ & 0x00000008) == 0x00000008)) {
+          trace_ = new java.util.ArrayList(trace_);
+          bitField0_ |= 0x00000008;
+         }
+      }
+      
+      private com.google.protobuf.RepeatedFieldBuilder<
+          org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder> traceBuilder_;
+      
+      public java.util.List getTraceList() {
+        if (traceBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(trace_);
+        } else {
+          return traceBuilder_.getMessageList();
+        }
+      }
+      public int getTraceCount() {
+        if (traceBuilder_ == null) {
+          return trace_.size();
+        } else {
+          return traceBuilder_.getCount();
+        }
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage getTrace(int index) {
+        if (traceBuilder_ == null) {
+          return trace_.get(index);
+        } else {
+          return traceBuilder_.getMessage(index);
+        }
+      }
+      public Builder setTrace(
+          int index, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage value) {
+        if (traceBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureTraceIsMutable();
+          trace_.set(index, value);
+          onChanged();
+        } else {
+          traceBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      public Builder setTrace(
+          int index, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder builderForValue) {
+        if (traceBuilder_ == null) {
+          ensureTraceIsMutable();
+          trace_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          traceBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addTrace(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage value) {
+        if (traceBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureTraceIsMutable();
+          trace_.add(value);
+          onChanged();
+        } else {
+          traceBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      public Builder addTrace(
+          int index, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage value) {
+        if (traceBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureTraceIsMutable();
+          trace_.add(index, value);
+          onChanged();
+        } else {
+          traceBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      public Builder addTrace(
+          org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder builderForValue) {
+        if (traceBuilder_ == null) {
+          ensureTraceIsMutable();
+          trace_.add(builderForValue.build());
+          onChanged();
+        } else {
+          traceBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addTrace(
+          int index, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder builderForValue) {
+        if (traceBuilder_ == null) {
+          ensureTraceIsMutable();
+          trace_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          traceBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAllTrace(
+          java.lang.Iterable values) {
+        if (traceBuilder_ == null) {
+          ensureTraceIsMutable();
+          super.addAll(values, trace_);
+          onChanged();
+        } else {
+          traceBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      public Builder clearTrace() {
+        if (traceBuilder_ == null) {
+          trace_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000008);
+          onChanged();
+        } else {
+          traceBuilder_.clear();
+        }
+        return this;
+      }
+      public Builder removeTrace(int index) {
+        if (traceBuilder_ == null) {
+          ensureTraceIsMutable();
+          trace_.remove(index);
+          onChanged();
+        } else {
+          traceBuilder_.remove(index);
+        }
+        return this;
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder getTraceBuilder(
+          int index) {
+        return getTraceFieldBuilder().getBuilder(index);
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder getTraceOrBuilder(
+          int index) {
+        if (traceBuilder_ == null) {
+          return trace_.get(index);  } else {
+          return traceBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      public java.util.List 
+           getTraceOrBuilderList() {
+        if (traceBuilder_ != null) {
+          return traceBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(trace_);
+        }
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder addTraceBuilder() {
+        return getTraceFieldBuilder().addBuilder(
+            org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.getDefaultInstance());
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder addTraceBuilder(
+          int index) {
+        return getTraceFieldBuilder().addBuilder(
+            index, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.getDefaultInstance());
+      }
+      public java.util.List 
+           getTraceBuilderList() {
+        return getTraceFieldBuilder().getBuilderList();
+      }
+      private com.google.protobuf.RepeatedFieldBuilder<
+          org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder> 
+          getTraceFieldBuilder() {
+        if (traceBuilder_ == null) {
+          traceBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessageOrBuilder>(
+                  trace_,
+                  ((bitField0_ & 0x00000008) == 0x00000008),
+                  getParentForChildren(),
+                  isClean());
+          trace_ = null;
+        }
+        return traceBuilder_;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:GenericExceptionMessage)
+    }
+    
+    static {
+      defaultInstance = new GenericExceptionMessage(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:GenericExceptionMessage)
+  }
+  
+  public interface ForeignExceptionMessageOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string source = 1;
+    boolean hasSource();
+    String getSource();
+    
+    // optional .GenericExceptionMessage genericException = 2;
+    boolean hasGenericException();
+    org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage getGenericException();
+    org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder getGenericExceptionOrBuilder();
+  }
+  public static final class ForeignExceptionMessage extends
+      com.google.protobuf.GeneratedMessage
+      implements ForeignExceptionMessageOrBuilder {
+    // Use ForeignExceptionMessage.newBuilder() to construct.
+    private ForeignExceptionMessage(Builder builder) {
+      super(builder);
+    }
+    private ForeignExceptionMessage(boolean noInit) {}
+    
+    private static final ForeignExceptionMessage defaultInstance;
+    public static ForeignExceptionMessage getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public ForeignExceptionMessage getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_ForeignExceptionMessage_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_ForeignExceptionMessage_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string source = 1;
+    public static final int SOURCE_FIELD_NUMBER = 1;
+    private java.lang.Object source_;
+    public boolean hasSource() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getSource() {
+      java.lang.Object ref = source_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          source_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getSourceBytes() {
+      java.lang.Object ref = source_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        source_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional .GenericExceptionMessage genericException = 2;
+    public static final int GENERICEXCEPTION_FIELD_NUMBER = 2;
+    private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage genericException_;
+    public boolean hasGenericException() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage getGenericException() {
+      return genericException_;
+    }
+    public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder getGenericExceptionOrBuilder() {
+      return genericException_;
+    }
+    
+    private void initFields() {
+      source_ = "";
+      genericException_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance();
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getSourceBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeMessage(2, genericException_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getSourceBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(2, genericException_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage)) {
+        return super.equals(obj);
+      }
+      org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage other = (org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage) obj;
+      
+      boolean result = true;
+      result = result && (hasSource() == other.hasSource());
+      if (hasSource()) {
+        result = result && getSource()
+            .equals(other.getSource());
+      }
+      result = result && (hasGenericException() == other.hasGenericException());
+      if (hasGenericException()) {
+        result = result && getGenericException()
+            .equals(other.getGenericException());
+      }
+      result = result &&
+          getUnknownFields().equals(other.getUnknownFields());
+      return result;
+    }
+    
+    @java.lang.Override
+    public int hashCode() {
+      int hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      if (hasSource()) {
+        hash = (37 * hash) + SOURCE_FIELD_NUMBER;
+        hash = (53 * hash) + getSource().hashCode();
+      }
+      if (hasGenericException()) {
+        hash = (37 * hash) + GENERICEXCEPTION_FIELD_NUMBER;
+        hash = (53 * hash) + getGenericException().hashCode();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      return hash;
+    }
+    
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_ForeignExceptionMessage_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.internal_static_ForeignExceptionMessage_fieldAccessorTable;
+      }
+      
+      // Construct using org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getGenericExceptionFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        source_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        if (genericExceptionBuilder_ == null) {
+          genericException_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance();
+        } else {
+          genericExceptionBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDescriptor();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage getDefaultInstanceForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage build() {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage buildPartial() {
+        org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage result = new org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.source_ = source_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        if (genericExceptionBuilder_ == null) {
+          result.genericException_ = genericException_;
+        } else {
+          result.genericException_ = genericExceptionBuilder_.build();
+        }
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage) {
+          return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage other) {
+        if (other == org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance()) return this;
+        if (other.hasSource()) {
+          setSource(other.getSource());
+        }
+        if (other.hasGenericException()) {
+          mergeGenericException(other.getGenericException());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              source_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder subBuilder = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.newBuilder();
+              if (hasGenericException()) {
+                subBuilder.mergeFrom(getGenericException());
+              }
+              input.readMessage(subBuilder, extensionRegistry);
+              setGenericException(subBuilder.buildPartial());
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string source = 1;
+      private java.lang.Object source_ = "";
+      public boolean hasSource() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getSource() {
+        java.lang.Object ref = source_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          source_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setSource(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        source_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearSource() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        source_ = getDefaultInstance().getSource();
+        onChanged();
+        return this;
+      }
+      void setSource(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        source_ = value;
+        onChanged();
+      }
+      
+      // optional .GenericExceptionMessage genericException = 2;
+      private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage genericException_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance();
+      private com.google.protobuf.SingleFieldBuilder<
+          org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder> genericExceptionBuilder_;
+      public boolean hasGenericException() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage getGenericException() {
+        if (genericExceptionBuilder_ == null) {
+          return genericException_;
+        } else {
+          return genericExceptionBuilder_.getMessage();
+        }
+      }
+      public Builder setGenericException(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage value) {
+        if (genericExceptionBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          genericException_ = value;
+          onChanged();
+        } else {
+          genericExceptionBuilder_.setMessage(value);
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      public Builder setGenericException(
+          org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder builderForValue) {
+        if (genericExceptionBuilder_ == null) {
+          genericException_ = builderForValue.build();
+          onChanged();
+        } else {
+          genericExceptionBuilder_.setMessage(builderForValue.build());
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      public Builder mergeGenericException(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage value) {
+        if (genericExceptionBuilder_ == null) {
+          if (((bitField0_ & 0x00000002) == 0x00000002) &&
+              genericException_ != org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance()) {
+            genericException_ =
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.newBuilder(genericException_).mergeFrom(value).buildPartial();
+          } else {
+            genericException_ = value;
+          }
+          onChanged();
+        } else {
+          genericExceptionBuilder_.mergeFrom(value);
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      public Builder clearGenericException() {
+        if (genericExceptionBuilder_ == null) {
+          genericException_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.getDefaultInstance();
+          onChanged();
+        } else {
+          genericExceptionBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder getGenericExceptionBuilder() {
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return getGenericExceptionFieldBuilder().getBuilder();
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder getGenericExceptionOrBuilder() {
+        if (genericExceptionBuilder_ != null) {
+          return genericExceptionBuilder_.getMessageOrBuilder();
+        } else {
+          return genericException_;
+        }
+      }
+      private com.google.protobuf.SingleFieldBuilder<
+          org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder> 
+          getGenericExceptionFieldBuilder() {
+        if (genericExceptionBuilder_ == null) {
+          genericExceptionBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessageOrBuilder>(
+                  genericException_,
+                  getParentForChildren(),
+                  isClean());
+          genericException_ = null;
+        }
+        return genericExceptionBuilder_;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:ForeignExceptionMessage)
+    }
+    
+    static {
+      defaultInstance = new ForeignExceptionMessage(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:ForeignExceptionMessage)
+  }
+  
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_StackTraceElementMessage_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_StackTraceElementMessage_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_GenericExceptionMessage_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_GenericExceptionMessage_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_ForeignExceptionMessage_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_ForeignExceptionMessage_fieldAccessorTable;
+  
+  public static com.google.protobuf.Descriptors.FileDescriptor
+      getDescriptor() {
+    return descriptor;
+  }
+  private static com.google.protobuf.Descriptors.FileDescriptor
+      descriptor;
+  static {
+    java.lang.String[] descriptorData = {
+      "\n\023ErrorHandling.proto\"l\n\030StackTraceEleme" +
+      "ntMessage\022\026\n\016declaringClass\030\001 \001(\t\022\022\n\nmet" +
+      "hodName\030\002 \001(\t\022\020\n\010fileName\030\003 \001(\t\022\022\n\nlineN" +
+      "umber\030\004 \001(\005\"z\n\027GenericExceptionMessage\022\021" +
+      "\n\tclassName\030\001 \001(\t\022\017\n\007message\030\002 \001(\t\022\021\n\ter" +
+      "rorInfo\030\003 \001(\014\022(\n\005trace\030\004 \003(\0132\031.StackTrac" +
+      "eElementMessage\"]\n\027ForeignExceptionMessa" +
+      "ge\022\016\n\006source\030\001 \001(\t\0222\n\020genericException\030\002" +
+      " \001(\0132\030.GenericExceptionMessageBF\n*org.ap" +
+      "ache.hadoop.hbase.protobuf.generatedB\023Er",
+      "rorHandlingProtosH\001\240\001\001"
+    };
+    com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
+      new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
+        public com.google.protobuf.ExtensionRegistry assignDescriptors(
+            com.google.protobuf.Descriptors.FileDescriptor root) {
+          descriptor = root;
+          internal_static_StackTraceElementMessage_descriptor =
+            getDescriptor().getMessageTypes().get(0);
+          internal_static_StackTraceElementMessage_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_StackTraceElementMessage_descriptor,
+              new java.lang.String[] { "DeclaringClass", "MethodName", "FileName", "LineNumber", },
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.class,
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage.Builder.class);
+          internal_static_GenericExceptionMessage_descriptor =
+            getDescriptor().getMessageTypes().get(1);
+          internal_static_GenericExceptionMessage_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_GenericExceptionMessage_descriptor,
+              new java.lang.String[] { "ClassName", "Message", "ErrorInfo", "Trace", },
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.class,
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage.Builder.class);
+          internal_static_ForeignExceptionMessage_descriptor =
+            getDescriptor().getMessageTypes().get(2);
+          internal_static_ForeignExceptionMessage_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_ForeignExceptionMessage_descriptor,
+              new java.lang.String[] { "Source", "GenericException", },
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.class,
+              org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder.class);
+          return null;
+        }
+      };
+    com.google.protobuf.Descriptors.FileDescriptor
+      .internalBuildGeneratedFileFrom(descriptorData,
+        new com.google.protobuf.Descriptors.FileDescriptor[] {
+        }, assigner);
+  }
+  
+  // @@protoc_insertion_point(outer_class_scope)
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
new file mode 100644
index 0000000..84abf01
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
@@ -0,0 +1,851 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: hbase.proto
+
+package org.apache.hadoop.hbase.protobuf.generated;
+
+public final class HBaseProtos {
+  private HBaseProtos() {}
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistry registry) {
+  }
+  public interface SnapshotDescriptionOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // required string name = 1;
+    boolean hasName();
+    String getName();
+    
+    // optional string table = 2;
+    boolean hasTable();
+    String getTable();
+    
+    // optional int64 creationTime = 3 [default = 0];
+    boolean hasCreationTime();
+    long getCreationTime();
+    
+    // optional .SnapshotDescription.Type type = 4 [default = FLUSH];
+    boolean hasType();
+    org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type getType();
+    
+    // optional int32 version = 5;
+    boolean hasVersion();
+    int getVersion();
+  }
+  public static final class SnapshotDescription extends
+      com.google.protobuf.GeneratedMessage
+      implements SnapshotDescriptionOrBuilder {
+    // Use SnapshotDescription.newBuilder() to construct.
+    private SnapshotDescription(Builder builder) {
+      super(builder);
+    }
+    private SnapshotDescription(boolean noInit) {}
+    
+    private static final SnapshotDescription defaultInstance;
+    public static SnapshotDescription getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public SnapshotDescription getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_SnapshotDescription_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_SnapshotDescription_fieldAccessorTable;
+    }
+    
+    public enum Type
+        implements com.google.protobuf.ProtocolMessageEnum {
+      DISABLED(0, 0),
+      FLUSH(1, 1),
+      ;
+      
+      public static final int DISABLED_VALUE = 0;
+      public static final int FLUSH_VALUE = 1;
+      
+      
+      public final int getNumber() { return value; }
+      
+      public static Type valueOf(int value) {
+        switch (value) {
+          case 0: return DISABLED;
+          case 1: return FLUSH;
+          default: return null;
+        }
+      }
+      
+      public static com.google.protobuf.Internal.EnumLiteMap
+          internalGetValueMap() {
+        return internalValueMap;
+      }
+      private static com.google.protobuf.Internal.EnumLiteMap
+          internalValueMap =
+            new com.google.protobuf.Internal.EnumLiteMap() {
+              public Type findValueByNumber(int number) {
+                return Type.valueOf(number);
+              }
+            };
+      
+      public final com.google.protobuf.Descriptors.EnumValueDescriptor
+          getValueDescriptor() {
+        return getDescriptor().getValues().get(index);
+      }
+      public final com.google.protobuf.Descriptors.EnumDescriptor
+          getDescriptorForType() {
+        return getDescriptor();
+      }
+      public static final com.google.protobuf.Descriptors.EnumDescriptor
+          getDescriptor() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.getDescriptor().getEnumTypes().get(0);
+      }
+      
+      private static final Type[] VALUES = {
+        DISABLED, FLUSH, 
+      };
+      
+      public static Type valueOf(
+          com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+        if (desc.getType() != getDescriptor()) {
+          throw new java.lang.IllegalArgumentException(
+            "EnumValueDescriptor is not for this type.");
+        }
+        return VALUES[desc.getIndex()];
+      }
+      
+      private final int index;
+      private final int value;
+      
+      private Type(int index, int value) {
+        this.index = index;
+        this.value = value;
+      }
+      
+      // @@protoc_insertion_point(enum_scope:SnapshotDescription.Type)
+    }
+    
+    private int bitField0_;
+    // required string name = 1;
+    public static final int NAME_FIELD_NUMBER = 1;
+    private java.lang.Object name_;
+    public boolean hasName() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getName() {
+      java.lang.Object ref = name_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          name_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getNameBytes() {
+      java.lang.Object ref = name_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        name_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string table = 2;
+    public static final int TABLE_FIELD_NUMBER = 2;
+    private java.lang.Object table_;
+    public boolean hasTable() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getTable() {
+      java.lang.Object ref = table_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          table_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getTableBytes() {
+      java.lang.Object ref = table_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        table_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int64 creationTime = 3 [default = 0];
+    public static final int CREATIONTIME_FIELD_NUMBER = 3;
+    private long creationTime_;
+    public boolean hasCreationTime() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public long getCreationTime() {
+      return creationTime_;
+    }
+    
+    // optional .SnapshotDescription.Type type = 4 [default = FLUSH];
+    public static final int TYPE_FIELD_NUMBER = 4;
+    private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type type_;
+    public boolean hasType() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type getType() {
+      return type_;
+    }
+    
+    // optional int32 version = 5;
+    public static final int VERSION_FIELD_NUMBER = 5;
+    private int version_;
+    public boolean hasVersion() {
+      return ((bitField0_ & 0x00000010) == 0x00000010);
+    }
+    public int getVersion() {
+      return version_;
+    }
+    
+    private void initFields() {
+      name_ = "";
+      table_ = "";
+      creationTime_ = 0L;
+      type_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type.FLUSH;
+      version_ = 0;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      if (!hasName()) {
+        memoizedIsInitialized = 0;
+        return false;
+      }
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getNameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getTableBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeInt64(3, creationTime_);
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        output.writeEnum(4, type_.getNumber());
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        output.writeInt32(5, version_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getNameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getTableBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(3, creationTime_);
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeEnumSize(4, type_.getNumber());
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(5, version_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription)) {
+        return super.equals(obj);
+      }
+      org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription other = (org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription) obj;
+      
+      boolean result = true;
+      result = result && (hasName() == other.hasName());
+      if (hasName()) {
+        result = result && getName()
+            .equals(other.getName());
+      }
+      result = result && (hasTable() == other.hasTable());
+      if (hasTable()) {
+        result = result && getTable()
+            .equals(other.getTable());
+      }
+      result = result && (hasCreationTime() == other.hasCreationTime());
+      if (hasCreationTime()) {
+        result = result && (getCreationTime()
+            == other.getCreationTime());
+      }
+      result = result && (hasType() == other.hasType());
+      if (hasType()) {
+        result = result &&
+            (getType() == other.getType());
+      }
+      result = result && (hasVersion() == other.hasVersion());
+      if (hasVersion()) {
+        result = result && (getVersion()
+            == other.getVersion());
+      }
+      result = result &&
+          getUnknownFields().equals(other.getUnknownFields());
+      return result;
+    }
+    
+    @java.lang.Override
+    public int hashCode() {
+      int hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      if (hasName()) {
+        hash = (37 * hash) + NAME_FIELD_NUMBER;
+        hash = (53 * hash) + getName().hashCode();
+      }
+      if (hasTable()) {
+        hash = (37 * hash) + TABLE_FIELD_NUMBER;
+        hash = (53 * hash) + getTable().hashCode();
+      }
+      if (hasCreationTime()) {
+        hash = (37 * hash) + CREATIONTIME_FIELD_NUMBER;
+        hash = (53 * hash) + hashLong(getCreationTime());
+      }
+      if (hasType()) {
+        hash = (37 * hash) + TYPE_FIELD_NUMBER;
+        hash = (53 * hash) + hashEnum(getType());
+      }
+      if (hasVersion()) {
+        hash = (37 * hash) + VERSION_FIELD_NUMBER;
+        hash = (53 * hash) + getVersion();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      return hash;
+    }
+    
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescriptionOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_SnapshotDescription_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.internal_static_SnapshotDescription_fieldAccessorTable;
+      }
+      
+      // Construct using org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        name_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        table_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        creationTime_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        type_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type.FLUSH;
+        bitField0_ = (bitField0_ & ~0x00000008);
+        version_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000010);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.getDescriptor();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription getDefaultInstanceForType() {
+        return org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.getDefaultInstance();
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription build() {
+        org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription buildPartial() {
+        org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription result = new org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.name_ = name_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.table_ = table_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.creationTime_ = creationTime_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
+        }
+        result.type_ = type_;
+        if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+          to_bitField0_ |= 0x00000010;
+        }
+        result.version_ = version_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription) {
+          return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription other) {
+        if (other == org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.getDefaultInstance()) return this;
+        if (other.hasName()) {
+          setName(other.getName());
+        }
+        if (other.hasTable()) {
+          setTable(other.getTable());
+        }
+        if (other.hasCreationTime()) {
+          setCreationTime(other.getCreationTime());
+        }
+        if (other.hasType()) {
+          setType(other.getType());
+        }
+        if (other.hasVersion()) {
+          setVersion(other.getVersion());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        if (!hasName()) {
+          
+          return false;
+        }
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              name_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              table_ = input.readBytes();
+              break;
+            }
+            case 24: {
+              bitField0_ |= 0x00000004;
+              creationTime_ = input.readInt64();
+              break;
+            }
+            case 32: {
+              int rawValue = input.readEnum();
+              org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type value = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(4, rawValue);
+              } else {
+                bitField0_ |= 0x00000008;
+                type_ = value;
+              }
+              break;
+            }
+            case 40: {
+              bitField0_ |= 0x00000010;
+              version_ = input.readInt32();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // required string name = 1;
+      private java.lang.Object name_ = "";
+      public boolean hasName() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getName() {
+        java.lang.Object ref = name_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          name_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        name_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearName() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        name_ = getDefaultInstance().getName();
+        onChanged();
+        return this;
+      }
+      void setName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        name_ = value;
+        onChanged();
+      }
+      
+      // optional string table = 2;
+      private java.lang.Object table_ = "";
+      public boolean hasTable() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getTable() {
+        java.lang.Object ref = table_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          table_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setTable(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        table_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearTable() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        table_ = getDefaultInstance().getTable();
+        onChanged();
+        return this;
+      }
+      void setTable(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        table_ = value;
+        onChanged();
+      }
+      
+      // optional int64 creationTime = 3 [default = 0];
+      private long creationTime_ ;
+      public boolean hasCreationTime() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public long getCreationTime() {
+        return creationTime_;
+      }
+      public Builder setCreationTime(long value) {
+        bitField0_ |= 0x00000004;
+        creationTime_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearCreationTime() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        creationTime_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // optional .SnapshotDescription.Type type = 4 [default = FLUSH];
+      private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type type_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type.FLUSH;
+      public boolean hasType() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type getType() {
+        return type_;
+      }
+      public Builder setType(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        bitField0_ |= 0x00000008;
+        type_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearType() {
+        bitField0_ = (bitField0_ & ~0x00000008);
+        type_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type.FLUSH;
+        onChanged();
+        return this;
+      }
+      
+      // optional int32 version = 5;
+      private int version_ ;
+      public boolean hasVersion() {
+        return ((bitField0_ & 0x00000010) == 0x00000010);
+      }
+      public int getVersion() {
+        return version_;
+      }
+      public Builder setVersion(int value) {
+        bitField0_ |= 0x00000010;
+        version_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearVersion() {
+        bitField0_ = (bitField0_ & ~0x00000010);
+        version_ = 0;
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:SnapshotDescription)
+    }
+    
+    static {
+      defaultInstance = new SnapshotDescription(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:SnapshotDescription)
+  }
+  
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_SnapshotDescription_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_SnapshotDescription_fieldAccessorTable;
+  
+  public static com.google.protobuf.Descriptors.FileDescriptor
+      getDescriptor() {
+    return descriptor;
+  }
+  private static com.google.protobuf.Descriptors.FileDescriptor
+      descriptor;
+  static {
+    java.lang.String[] descriptorData = {
+      "\n\013hbase.proto\"\255\001\n\023SnapshotDescription\022\014\n" +
+      "\004name\030\001 \002(\t\022\r\n\005table\030\002 \001(\t\022\027\n\014creationTi" +
+      "me\030\003 \001(\003:\0010\022.\n\004type\030\004 \001(\0162\031.SnapshotDesc" +
+      "ription.Type:\005FLUSH\022\017\n\007version\030\005 \001(\005\"\037\n\004" +
+      "Type\022\014\n\010DISABLED\020\000\022\t\n\005FLUSH\020\001B>\n*org.apa" +
+      "che.hadoop.hbase.protobuf.generatedB\013HBa" +
+      "seProtosH\001\240\001\001"
+    };
+    com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
+      new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
+        public com.google.protobuf.ExtensionRegistry assignDescriptors(
+            com.google.protobuf.Descriptors.FileDescriptor root) {
+          descriptor = root;
+          internal_static_SnapshotDescription_descriptor =
+            getDescriptor().getMessageTypes().get(0);
+          internal_static_SnapshotDescription_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_SnapshotDescription_descriptor,
+              new java.lang.String[] { "Name", "Table", "CreationTime", "Type", "Version", },
+              org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.class,
+              org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Builder.class);
+          return null;
+        }
+      };
+    com.google.protobuf.Descriptors.FileDescriptor
+      .internalBuildGeneratedFileFrom(descriptorData,
+        new com.google.protobuf.Descriptors.FileDescriptor[] {
+        }, assigner);
+  }
+  
+  // @@protoc_insertion_point(outer_class_scope)
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
new file mode 100644
index 0000000..82894e2
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
@@ -0,0 +1,35 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+
+/**
+ * If set of MapFile.Readers in Store change, implementors are notified.
+ */
+public interface ChangedReadersObserver {
+  /**
+   * Notify observers.
+   * @throws IOException e
+   */
+  void updateReaders() throws IOException;
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnCount.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnCount.java
new file mode 100644
index 0000000..a617d68
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnCount.java
@@ -0,0 +1,109 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+/**
+ * Simple wrapper for a byte buffer and a counter.  Does not copy.
+ * 
        
+ * NOT thread-safe because it is not used in a multi-threaded context, yet.
+ */
+public class ColumnCount {
+  private final byte [] bytes;
+  private final int offset;
+  private final int length;
+  private int count;
+
+  /**
+   * Constructor
+   * @param column the qualifier to count the versions for
+   */
+  public ColumnCount(byte [] column) {
+    this(column, 0);
+  }
+
+  /**
+   * Constructor
+   * @param column the qualifier to count the versions for
+   * @param count initial count
+   */
+  public ColumnCount(byte [] column, int count) {
+    this(column, 0, column.length, count);
+  }
+
+  /**
+   * Constuctor
+   * @param column the qualifier to count the versions for
+   * @param offset in the passed buffer where to start the qualifier from
+   * @param length of the qualifier
+   * @param count initial count
+   */
+  public ColumnCount(byte [] column, int offset, int length, int count) {
+    this.bytes = column;
+    this.offset = offset;
+    this.length = length;
+    this.count = count;
+  }
+
+  /**
+   * @return the buffer
+   */
+  public byte [] getBuffer(){
+    return this.bytes;
+  }
+
+  /**
+   * @return the offset
+   */
+  public int getOffset(){
+    return this.offset;
+  }
+
+  /**
+   * @return the length
+   */
+  public int getLength(){
+    return this.length;
+  }
+
+  /**
+   * Decrement the current version count
+   * @return current count
+   */
+  public int decrement() {
+    return --count;
+  }
+
+  /**
+   * Increment the current version count
+   * @return current count
+   */
+  public int increment() {
+    return ++count;
+  }
+
+  /**
+   * Set the current count to a new count
+   * @param count new count to set
+   */
+  public void setCount(int count) {
+    this.count = count;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
new file mode 100644
index 0000000..2eeaab1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
@@ -0,0 +1,105 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
+
+/**
+ * Implementing classes of this interface will be used for the tracking
+ * and enforcement of columns and numbers of versions and timeToLive during
+ * the course of a Get or Scan operation.
+ * 
        
+ * Currently there are two different types of Store/Family-level queries.
+ * 
        • {@link ExplicitColumnTracker} is used when the query specifies
+ * one or more column qualifiers to return in the family.
+ * 
          
+ * This class is utilized by {@link ScanQueryMatcher} through two methods:
+ * 
          • {@link #checkColumn} is called when a Put satisfies all other
+ * conditions of the query.  This method returns a {@link org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode} to define
+ * what action should be taken.
+ * 
          • {@link #update} is called at the end of every StoreFile or memstore.
+ * 
            
+ * This class is NOT thread-safe as queries are never multi-threaded
+ */
+public interface ColumnTracker {
+  /**
+   * Keeps track of the number of versions for the columns asked for
+   * @param bytes
+   * @param offset
+   * @param length
+   * @param ttl The timeToLive to enforce.
+   * @param type The type of the KeyValue
+   * @param ignoreCount indicates if the KV needs to be excluded while counting
+   *   (used during compactions. We only count KV's that are older than all the
+   *   scanners' read points.)
+   * @return The match code instance.
+   * @throws IOException in case there is an internal consistency problem
+   *      caused by a data corruption.
+   */
+  public ScanQueryMatcher.MatchCode checkColumn(byte[] bytes, int offset,
+      int length, long ttl, byte type, boolean ignoreCount)
+      throws IOException;
+
+  /**
+   * Updates internal variables in between files
+   */
+  public void update();
+
+  /**
+   * Resets the Matcher
+   */
+  public void reset();
+
+  /**
+   *
+   * @return true when done.
+   */
+  public boolean done();
+
+  /**
+   * Used by matcher and scan/get to get a hint of the next column
+   * to seek to after checkColumn() returns SKIP.  Returns the next interesting
+   * column we want, or NULL there is none (wildcard scanner).
+   *
+   * Implementations aren't required to return anything useful unless the most recent
+   * call was to checkColumn() and the return code was SKIP.  This is pretty implementation
+   * detail-y, but optimizations are like that.
+   *
+   * @return null, or a ColumnCount that we should seek to
+   */
+  public ColumnCount getColumnHint();
+
+  /**
+   * Retrieve the MatchCode for the next row or column
+   */
+  public MatchCode getNextRowOrNextColumn(byte[] bytes, int offset,
+      int qualLength);
+
+  /**
+   * Give the tracker a chance to declare it's done based on only the timestamp
+   * to allow an early out.
+   *
+   * @param timestamp
+   * @return true to early out based on timestamp.
+   */
+  public boolean isDone(long timestamp);
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
new file mode 100644
index 0000000..9d0ad62
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -0,0 +1,315 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.concurrent.Executors;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Compact region on request and then run split if appropriate
+ */
+public class CompactSplitThread implements CompactionRequestor {
+  static final Log LOG = LogFactory.getLog(CompactSplitThread.class);
+
+  private final HRegionServer server;
+  private final Configuration conf;
+
+  private final ThreadPoolExecutor largeCompactions;
+  private final ThreadPoolExecutor smallCompactions;
+  private final ThreadPoolExecutor splits;
+
+  /**
+   * Splitting should not take place if the total number of regions exceed this.
+   * This is not a hard limit to the number of regions but it is a guideline to
+   * stop splitting after number of online regions is greater than this.
+   */
+  private int regionSplitLimit;
+
+  /** @param server */
+  CompactSplitThread(HRegionServer server) {
+    super();
+    this.server = server;
+    this.conf = server.getConfiguration();
+    this.regionSplitLimit = conf.getInt("hbase.regionserver.regionSplitLimit",
+        Integer.MAX_VALUE);
+
+    int largeThreads = Math.max(1, conf.getInt(
+        "hbase.regionserver.thread.compaction.large", 1));
+    int smallThreads = conf.getInt(
+        "hbase.regionserver.thread.compaction.small", 1);
+
+    int splitThreads = conf.getInt("hbase.regionserver.thread.split", 1);
+
+    // if we have throttle threads, make sure the user also specified size
+    Preconditions.checkArgument(largeThreads > 0 && smallThreads > 0);
+
+    final String n = Thread.currentThread().getName();
+
+    this.largeCompactions = new ThreadPoolExecutor(largeThreads, largeThreads,
+        60, TimeUnit.SECONDS, new PriorityBlockingQueue(),
+        new ThreadFactory() {
+          @Override
+          public Thread newThread(Runnable r) {
+            Thread t = new Thread(r);
+            t.setName(n + "-largeCompactions-" + System.currentTimeMillis());
+            return t;
+          }
+      });
+    this.largeCompactions
+        .setRejectedExecutionHandler(new CompactionRequest.Rejection());
+    this.smallCompactions = new ThreadPoolExecutor(smallThreads, smallThreads,
+        60, TimeUnit.SECONDS, new PriorityBlockingQueue(),
+        new ThreadFactory() {
+          @Override
+          public Thread newThread(Runnable r) {
+            Thread t = new Thread(r);
+            t.setName(n + "-smallCompactions-" + System.currentTimeMillis());
+            return t;
+          }
+      });
+    this.smallCompactions
+        .setRejectedExecutionHandler(new CompactionRequest.Rejection());
+    this.splits = (ThreadPoolExecutor)
+        Executors.newFixedThreadPool(splitThreads,
+            new ThreadFactory() {
+          @Override
+          public Thread newThread(Runnable r) {
+            Thread t = new Thread(r);
+            t.setName(n + "-splits-" + System.currentTimeMillis());
+            return t;
+          }
+      });
+  }
+
+  @Override
+  public String toString() {
+    return "compaction_queue=("
+        + largeCompactions.getQueue().size() + ":"
+        + smallCompactions.getQueue().size() + ")"
+        + ", split_queue=" + splits.getQueue().size();
+  }
+
+  public String dumpQueue() {
+    StringBuffer queueLists = new StringBuffer();
+    queueLists.append("Compaction/Split Queue dump:\n");
+    queueLists.append("  LargeCompation Queue:\n");
+    BlockingQueue lq = largeCompactions.getQueue();
+    Iterator it = lq.iterator();
+    while(it.hasNext()){
+      queueLists.append("    "+it.next().toString());
+      queueLists.append("\n");
+    }
+    
+    if( smallCompactions != null ){
+      queueLists.append("\n");
+      queueLists.append("  SmallCompation Queue:\n");
+      lq = smallCompactions.getQueue();
+      it = lq.iterator();
+      while(it.hasNext()){
+        queueLists.append("    "+it.next().toString());
+        queueLists.append("\n");
+      }
+    }
+    
+    queueLists.append("\n");
+    queueLists.append("  Split Queue:\n");
+    lq = splits.getQueue();
+    it = lq.iterator();
+    while(it.hasNext()){
+      queueLists.append("    "+it.next().toString());
+      queueLists.append("\n");
+    }
+    
+    return queueLists.toString();
+  }
+
+  public synchronized boolean requestSplit(final HRegion r) {
+    // don't split regions that are blocking
+    if (shouldSplitRegion() && r.getCompactPriority() >= Store.PRIORITY_USER) {
+      byte[] midKey = r.checkSplit();
+      if (midKey != null) {
+        requestSplit(r, midKey);
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public synchronized void requestSplit(final HRegion r, byte[] midKey) {
+    if (midKey == null) {
+      LOG.debug("Region " + r.getRegionNameAsString() +
+        " not splittable because midkey=null");
+      return;
+    }
+    boolean indexUsed = this.conf.getBoolean("hbase.use.secondary.index", false);
+    if (indexUsed) {
+      if (r.getRegionInfo().getTableNameAsString().endsWith("_idx")) {
+        LOG.warn("Split issued on the index region which is not allowed."
+            + "Returning without splitting the region.");
+        return;
+      }
+    }
+    try {
+      this.splits.execute(new SplitRequest(r, midKey, this.server));
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Split requested for " + r + ".  " + this);
+      }
+    } catch (RejectedExecutionException ree) {
+      LOG.info("Could not execute split for " + r, ree);
+    }
+  }
+
+  @Override
+  public synchronized List requestCompaction(final HRegion r, final String why)
+      throws IOException {
+    return requestCompaction(r, why, null);
+  }
+
+  @Override
+  public synchronized List requestCompaction(final HRegion r, final String why,
+      List requests) throws IOException {
+    return requestCompaction(r, why, Store.NO_PRIORITY, requests);
+  }
+
+  @Override
+  public synchronized CompactionRequest requestCompaction(final HRegion r, final Store s,
+      final String why,
+      CompactionRequest request) throws IOException {
+    return requestCompaction(r, s, why, Store.NO_PRIORITY, request);
+  }
+
+  @Override
+  public synchronized List requestCompaction(final HRegion r, final String why,
+      int pri, final List requests) throws IOException {
+    List ret;
+    // not a special compaction request, so make out own list
+    if (requests == null) {
+      ret = new ArrayList(r.getStores().size());
+      for (Store s : r.getStores().values()) {
+        ret.add(requestCompaction(r, s, why, pri, null));
+      }
+    } else {
+      ret = new ArrayList(requests.size());
+      for (CompactionRequest request : requests) {
+        ret.add(requestCompaction(r, request.getStore(), why, pri, request));
+      }
+    }
+    return ret;
+  }
+
+  @Override
+  public synchronized CompactionRequest requestCompaction(final HRegion r, final Store s,
+      final String why, int priority, CompactionRequest request) throws IOException {
+    if (this.server.isStopped()) {
+      return null;
+    }
+    CompactionRequest cr = s.requestCompaction(priority, request);
+    if (cr != null) {
+      cr.setServer(server);
+      if (priority != Store.NO_PRIORITY) {
+        cr.setPriority(priority);
+      }
+      ThreadPoolExecutor pool = s.throttleCompaction(cr.getSize())
+          ? largeCompactions : smallCompactions;
+      pool.execute(cr);
+      if (LOG.isDebugEnabled()) {
+        String type = (pool == smallCompactions) ? "Small " : "Large ";
+        LOG.debug(type + "Compaction requested: " + cr
+            + (why != null && !why.isEmpty() ? "; Because: " + why : "")
+            + "; " + this);
+      }
+    } else {
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Not compacting " + r.getRegionNameAsString() + 
+            " because compaction request was cancelled");
+      }
+    }
+    return cr;
+  }
+
+  /**
+   * Only interrupt once it's done with a run through the work loop.
+   */
+  void interruptIfNecessary() {
+    splits.shutdown();
+    largeCompactions.shutdown();
+    smallCompactions.shutdown();
+  }
+
+  private void waitFor(ThreadPoolExecutor t, String name) {
+    boolean done = false;
+    while (!done) {
+      try {
+        done = t.awaitTermination(60, TimeUnit.SECONDS);
+        LOG.debug("Waiting for " + name + " to finish...");
+        if (!done) {
+          t.shutdownNow();
+        }
+      } catch (InterruptedException ie) {
+        LOG.debug("Interrupted waiting for " + name + " to finish...");
+      }
+    }
+  }
+
+  void join() {
+    waitFor(splits, "Split Thread");
+    waitFor(largeCompactions, "Large Compaction Thread");
+    waitFor(smallCompactions, "Small Compaction Thread");
+  }
+
+  /**
+   * Returns the current size of the queue containing regions that are
+   * processed.
+   *
+   * @return The current size of the regions queue.
+   */
+  public int getCompactionQueueSize() {
+    return largeCompactions.getQueue().size() + smallCompactions.getQueue().size();
+  }
+
+  private boolean shouldSplitRegion() {
+    return (regionSplitLimit > server.getNumberOfOnlineRegions());
+  }
+
+  /**
+   * @return the regionSplitLimit
+   */
+  public int getRegionSplitLimit() {
+    return this.regionSplitLimit;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
new file mode 100644
index 0000000..978854f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
@@ -0,0 +1,90 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+
+public interface CompactionRequestor {
+  /**
+   * @param r Region to compact
+   * @param why Why compaction was requested -- used in debug messages
+   * @return the created {@link CompactionRequest CompactionRequets} or, if no compactions were
+   *         started, an empty list
+   * @throws IOException
+   */
+  public List requestCompaction(final HRegion r, final String why)
+      throws IOException;
+
+  /**
+   * @param r Region to compact
+   * @param why Why compaction was requested -- used in debug messages
+   * @param requests custom compaction requests. Each compaction must specify the store on which it
+   *          is acting. Can be null in which case a compaction will be attempted on all
+   *          stores for the region.
+   * @return The created {@link CompactionRequest CompactionRequests} or an empty list if no
+   *         compactions were started
+   * @throws IOException
+   */
+  public List requestCompaction(final HRegion r, final String why,
+      List requests) throws IOException;
+
+  /**
+   * @param r Region to compact
+   * @param s Store within region to compact
+   * @param why Why compaction was requested -- used in debug messages
+   * @param request custom compaction request for the {@link HRegion} and {@link Store}. Custom
+   *          request must be null or be constructed with matching region and store.
+   * @return The created {@link CompactionRequest} or null if no compaction was started.
+   * @throws IOException
+   */
+  public CompactionRequest requestCompaction(final HRegion r, final Store s, final String why,
+      CompactionRequest request) throws IOException;
+
+  /**
+   * @param r Region to compact
+   * @param why Why compaction was requested -- used in debug messages
+   * @param pri Priority of this compaction. minHeap. <=0 is critical
+   * @param requests custom compaction requests. Each compaction must specify the store on which it
+   *          is acting. Can be null in which case a compaction will be attempted on all
+   *          stores for the region.
+   * @return The created {@link CompactionRequest CompactionRequests} or an empty list if no
+   *         compactions were started.
+   * @throws IOException
+   */
+  public List requestCompaction(final HRegion r, final String why, int pri,
+      List requests) throws IOException;
+
+  /**
+   * @param r Region to compact
+   * @param s Store within region to compact
+   * @param why Why compaction was requested -- used in debug messages
+   * @param pri Priority of this compaction. minHeap. <=0 is critical
+   * @param request request custom compaction request to run. {@link Store} and {@link HRegion} for
+   *          the request must match the region and store specified here.
+   * @return The created {@link CompactionRequest} or null if no compaction was started
+   * @throws IOException
+   */
+  public CompactionRequest requestCompaction(final HRegion r, final Store s, final String why,
+      int pri, CompactionRequest request) throws IOException;
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
new file mode 100644
index 0000000..e123c22
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
@@ -0,0 +1,468 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.LineReader;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/*
+ * The CompactionTool allows to execute a compaction specifying a:
+ * 
              
+ *  
            • table folder (all regions and families will be compacted)
+ *  
            • region folder (all families in the region will be compacted)
+ *  
            • family folder (the store files will be compacted)
+ * 
            
+ */
+@InterfaceAudience.Public
+public class CompactionTool extends Configured implements Tool {
+  private static final Log LOG = LogFactory.getLog(CompactionTool.class);
+
+  private final static String CONF_TMP_DIR = "hbase.tmp.dir";
+  private final static String CONF_COMPACT_ONCE = "hbase.compactiontool.compact.once";
+  private final static String CONF_DELETE_COMPACTED = "hbase.compactiontool.delete";
+  private final static String CONF_COMPLETE_COMPACTION = "hbase.hstore.compaction.complete";
+
+  /**
+   * Class responsible to execute the Compaction on the specified path.
+   * The path can be a table, region or family directory.
+   */
+  private static class CompactionWorker {
+    private final boolean keepCompactedFiles;
+    private final boolean deleteCompacted;
+    private final Configuration conf;
+    private final FileSystem fs;
+    private final Path tmpDir;
+
+    public CompactionWorker(final FileSystem fs, final Configuration conf) {
+      this.conf = conf;
+      this.keepCompactedFiles = !conf.getBoolean(CONF_COMPLETE_COMPACTION, true);
+      this.deleteCompacted = conf.getBoolean(CONF_DELETE_COMPACTED, false);
+      this.tmpDir = new Path(conf.get(CONF_TMP_DIR));
+      this.fs = fs;
+    }
+
+    /**
+     * Execute the compaction on the specified path.
+     *
+     * @param path Directory path on which run a
+     * @param compactOnce Execute just a single step of compaction.
+     */
+    public void compact(final Path path, final boolean compactOnce) throws IOException {
+      if (isFamilyDir(fs, path)) {
+        Path regionDir = path.getParent();
+        Path tableDir = regionDir.getParent();
+        HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
+        HRegion region = loadRegion(fs, conf, htd, regionDir);
+        compactStoreFiles(region, path, compactOnce);
+      } else if (isRegionDir(fs, path)) {
+        Path tableDir = path.getParent();
+        HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
+        compactRegion(htd, path, compactOnce);
+      } else if (isTableDir(fs, path)) {
+        compactTable(path, compactOnce);
+      } else {
+        throw new IOException(
+          "Specified path is not a table, region or family directory. path=" + path);
+      }
+    }
+
+    private void compactTable(final Path tableDir, final boolean compactOnce)
+        throws IOException {
+      HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir);
+      LOG.info("Compact table=" + htd.getNameAsString());
+      for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
+        compactRegion(htd, regionDir, compactOnce);
+      }
+    }
+
+    private void compactRegion(final HTableDescriptor htd, final Path regionDir,
+        final boolean compactOnce) throws IOException {
+      HRegion region = loadRegion(fs, conf, htd, regionDir);
+      LOG.info("Compact table=" + htd.getNameAsString() +
+        " region=" + region.getRegionNameAsString());
+      for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
+        compactStoreFiles(region, familyDir, compactOnce);
+      }
+    }
+
+    /**
+     * Execute the actual compaction job.
+     * If the compact once flag is not specified, execute the compaction until
+     * no more compactions are needed. Uses the Configuration settings provided.
+     */
+    private void compactStoreFiles(final HRegion region, final Path familyDir,
+        final boolean compactOnce) throws IOException {
+      LOG.info("Compact table=" + region.getTableDesc().getNameAsString() +
+        " region=" + region.getRegionNameAsString() +
+        " family=" + familyDir.getName());
+      Store store = getStore(region, familyDir);
+      do {
+        CompactionRequest cr = store.requestCompaction();
+        StoreFile storeFile = store.compact(cr);
+        if (storeFile != null) {
+          if (keepCompactedFiles && deleteCompacted) {
+            fs.delete(storeFile.getPath(), false);
+          }
+        }
+      } while (store.needsCompaction() && !compactOnce);
+    }
+
+    /**
+     * Create a "mock" HStore that uses the tmpDir specified by the user and
+     * the store dir to compact as source.
+     */
+    private Store getStore(final HRegion region, final Path storeDir) throws IOException {
+      byte[] familyName = Bytes.toBytes(storeDir.getName());
+      HColumnDescriptor hcd = region.getTableDesc().getFamily(familyName);
+      // Create a Store w/ check of hbase.rootdir blanked out and return our
+      // list of files instead of have Store search its home dir.
+      return new Store(tmpDir, region, hcd, fs, conf) {
+        @Override
+        public FileStatus[] getStoreFiles() throws IOException {
+          return this.fs.listStatus(getHomedir());
+        }
+
+        @Override
+        Path createStoreHomeDir(FileSystem fs, Path homedir) throws IOException {
+          return storeDir;
+        }
+      };
+    }
+
+    private static HRegion loadRegion(final FileSystem fs, final Configuration conf,
+        final HTableDescriptor htd, final Path regionDir) throws IOException {
+      Path rootDir = regionDir.getParent().getParent();
+      HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
+      return HRegion.createHRegion(hri, rootDir, conf, htd, null, false, true);
+    }
+  }
+
+  private static boolean isRegionDir(final FileSystem fs, final Path path) throws IOException {
+    Path regionInfo = new Path(path, HRegion.REGIONINFO_FILE);
+    return fs.exists(regionInfo);
+  }
+
+  private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException {
+    return FSTableDescriptors.getTableInfoPath(fs, path) != null;
+  }
+
+  private static boolean isFamilyDir(final FileSystem fs, final Path path) throws IOException {
+    return isRegionDir(fs, path.getParent());
+  }
+
+  private static class CompactionMapper
+      extends Mapper {
+    private CompactionWorker compactor = null;
+    private boolean compactOnce = false;
+
+    @Override
+    public void setup(Context context) {
+      Configuration conf = context.getConfiguration();
+      compactOnce = conf.getBoolean(CONF_COMPACT_ONCE, false);
+
+      try {
+        FileSystem fs = FileSystem.get(conf);
+        this.compactor = new CompactionWorker(fs, conf);
+      } catch (IOException e) {
+        throw new RuntimeException("Could not get the input FileSystem", e);
+      }
+    }
+
+    @Override
+    public void map(LongWritable key, Text value, Context context)
+        throws InterruptedException, IOException {
+      Path path = new Path(value.toString());
+      this.compactor.compact(path, compactOnce);
+    }
+  }
+
+  /**
+   * Input format that uses store files block location as input split locality.
+   */
+  private static class CompactionInputFormat extends TextInputFormat {
+    @Override
+    protected boolean isSplitable(JobContext context, Path file) {
+      return true;
+    }
+
+    /**
+     * Returns a split for each store files directory using the block location
+     * of each file as locality reference.
+     */
+    @Override
+    public List getSplits(JobContext job) throws IOException {
+      List splits = new ArrayList();
+      List files = listStatus(job);
+
+      Text key = new Text();
+      for (FileStatus file: files) {
+        Path path = file.getPath();
+        FileSystem fs = path.getFileSystem(job.getConfiguration());
+        LineReader reader = new LineReader(fs.open(path));
+        long pos = 0;
+        int n;
+        try {
+          while ((n = reader.readLine(key)) > 0) {
+            String[] hosts = getStoreDirHosts(fs, path);
+            splits.add(new FileSplit(path, pos, n, hosts));
+            pos += n;
+          }
+        } finally {
+          reader.close();
+        }
+      }
+
+      return splits;
+    }
+
+    /**
+     * return the top hosts of the store files, used by the Split
+     */
+    private static String[] getStoreDirHosts(final FileSystem fs, final Path path)
+        throws IOException {
+      FileStatus[] files = FSUtils.listStatus(fs, path, null);
+      if (files == null) {
+        return new String[] {};
+      }
+
+      HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
+      for (FileStatus hfileStatus: files) {
+        HDFSBlocksDistribution storeFileBlocksDistribution =
+          FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0, hfileStatus.getLen());
+        hdfsBlocksDistribution.add(storeFileBlocksDistribution);
+      }
+
+      List hosts = hdfsBlocksDistribution.getTopHosts();
+      return hosts.toArray(new String[hosts.size()]);
+    }
+
+    /**
+     * Create the input file for the given directories to compact.
+     * The file is a TextFile with each line corrisponding to a
+     * store files directory to compact.
+     */
+    public static void createInputFile(final FileSystem fs, final Path path,
+        final Set toCompactDirs) throws IOException {
+      // Extract the list of store dirs
+      List storeDirs = new LinkedList();
+      for (Path compactDir: toCompactDirs) {
+        if (isFamilyDir(fs, compactDir)) {
+          storeDirs.add(compactDir);
+        } else if (isRegionDir(fs, compactDir)) {
+          for (Path familyDir: FSUtils.getFamilyDirs(fs, compactDir)) {
+            storeDirs.add(familyDir);
+          }
+        } else if (isTableDir(fs, compactDir)) {
+          // Lookup regions
+          for (Path regionDir: FSUtils.getRegionDirs(fs, compactDir)) {
+            for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
+              storeDirs.add(familyDir);
+            }
+          }
+        } else {
+          throw new IOException(
+            "Specified path is not a table, region or family directory. path=" + compactDir);
+        }
+      }
+
+      // Write Input File
+      FSDataOutputStream stream = fs.create(path);
+      LOG.info("Create input file=" + path + " with " + storeDirs.size() + " dirs to compact.");
+      try {
+        final byte[] newLine = Bytes.toBytes("\n");
+        for (Path storeDir: storeDirs) {
+          stream.write(Bytes.toBytes(storeDir.toString()));
+          stream.write(newLine);
+        }
+      } finally {
+        stream.close();
+      }
+    }
+  }
+
+  /**
+   * Execute compaction, using a Map-Reduce job.
+   */
+  private int doMapReduce(final FileSystem fs, final Set toCompactDirs,
+      final boolean compactOnce) throws Exception {
+    Configuration conf = getConf();
+    conf.setBoolean(CONF_COMPACT_ONCE, compactOnce);
+
+    Job job = new Job(conf);
+    job.setJobName("CompactionTool");
+    job.setJarByClass(CompactionTool.class);
+    job.setMapperClass(CompactionMapper.class);
+    job.setInputFormatClass(CompactionInputFormat.class);
+    job.setOutputFormatClass(NullOutputFormat.class);
+    job.setMapSpeculativeExecution(false);
+    job.setNumReduceTasks(0);
+
+    String stagingName = "compact-" + EnvironmentEdgeManager.currentTimeMillis();
+    Path stagingDir = new Path(conf.get(CONF_TMP_DIR), stagingName);
+    fs.mkdirs(stagingDir);
+    try {
+      // Create input file with the store dirs
+      Path inputPath = new Path(stagingDir, stagingName);
+      CompactionInputFormat.createInputFile(fs, inputPath, toCompactDirs);
+      CompactionInputFormat.addInputPath(job, inputPath);
+
+      // Initialize credential for secure cluster
+      TableMapReduceUtil.initCredentials(job);
+
+      // Start the MR Job and wait
+      return job.waitForCompletion(true) ? 0 : 1;
+    } finally {
+      fs.delete(stagingDir, true);
+    }
+  }
+
+  /**
+   * Execute compaction, from this client, one path at the time.
+   */
+  private int doClient(final FileSystem fs, final Set toCompactDirs,
+      final boolean compactOnce) throws IOException {
+    CompactionWorker worker = new CompactionWorker(fs, getConf());
+    for (Path path: toCompactDirs) {
+      worker.compact(path, compactOnce);
+    }
+    return 0;
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    Set toCompactDirs = new HashSet();
+    boolean compactOnce = false;
+    boolean mapred = false;
+
+    Configuration conf = getConf();
+    FileSystem fs = FileSystem.get(conf);
+
+    try {
+      for (int i = 0; i < args.length; ++i) {
+        String opt = args[i];
+        if (opt.equals("-compactOnce")) {
+          compactOnce = true;
+        } else if (opt.equals("-mapred")) {
+          mapred = true;
+        } else if (!opt.startsWith("-")) {
+          Path path = new Path(opt);
+          FileStatus status = fs.getFileStatus(path);
+          if (!status.isDir()) {
+            printUsage("Specified path is not a directory. path=" + path);
+            return 1;
+          }
+          toCompactDirs.add(path);
+        } else {
+          printUsage();
+        }
+      }
+    } catch (Exception e) {
+      printUsage(e.getMessage());
+      return 1;
+    }
+
+    if (toCompactDirs.size() == 0) {
+      printUsage("No directories to compact specified.");
+      return 1;
+    }
+
+    // Execute compaction!
+    if (mapred) {
+      return doMapReduce(fs, toCompactDirs, compactOnce);
+    } else {
+      return doClient(fs, toCompactDirs, compactOnce);
+    }
+  }
+
+  private void printUsage() {
+    printUsage(null);
+  }
+
+  private void printUsage(final String message) {
+    if (message != null && message.length() > 0) {
+      System.err.println(message);
+    }
+    System.err.println("Usage: java " + this.getClass().getName() + " \\");
+    System.err.println("  [-compactOnce] [-mapred] [-D]* files...");
+    System.err.println();
+    System.err.println("Options:");
+    System.err.println(" mapred         Use MapReduce to run compaction.");
+    System.err.println(" compactOnce    Execute just one compaction step. (default: while needed)");
+    System.err.println();
+    System.err.println("Note: -D properties will be applied to the conf used. ");
+    System.err.println("For example: ");
+    System.err.println(" To preserve input files, pass -D"+CONF_COMPLETE_COMPACTION+"=false");
+    System.err.println(" To stop delete of compacted file, pass -D"+CONF_DELETE_COMPACTED+"=false");
+    System.err.println(" To set tmp dir, pass -D"+CONF_TMP_DIR+"=ALTERNATE_DIR");
+    System.err.println();
+    System.err.println("Examples:");
+    System.err.println(" To compact the full 'TestTable' using MapReduce:");
+    System.err.println(" $ bin/hbase " + this.getClass().getName() + " -mapred hdfs:///hbase/TestTable");
+    System.err.println();
+    System.err.println(" To compact column family 'x' of the table 'TestTable' region 'abc':");
+    System.err.println(" $ bin/hbase " + this.getClass().getName() + " hdfs:///hbase/TestTable/abc/x");
+  }
+
+  public static void main(String[] args) throws Exception {
+    System.exit(ToolRunner.run(HBaseConfiguration.create(), new CompactionTool(), args));
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
new file mode 100644
index 0000000..7d5f77f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFileWriterV2;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Compact passed set of files.
+ * Create an instance and then call {@ink #compact(Store, Collection, boolean, long)}.
+ */
+@InterfaceAudience.Private
+class Compactor extends Configured {
+  private static final Log LOG = LogFactory.getLog(Compactor.class);
+  private CompactionProgress progress;
+
+  Compactor(final Configuration c) {
+    super(c);
+  }
+
+  /**
+   * Compact a list of files for testing. Creates a fake {@link CompactionRequest} to pass to the
+   * actual compaction method
+   * @param store store which should be compacted
+   * @param conf configuration to use when generating the compaction selection
+   * @param filesToCompact the files to compact. They are used a the compaction selection for the
+   *          generated {@link CompactionRequest}
+   * @param isMajor true to initiate a major compaction (prune all deletes, max versions,
+   *          etc)
+   * @param maxId maximum sequenceID == the last key of all files in the compaction
+   * @return product of the compaction or null if all cells expired or deleted and nothing made it
+   *         through the compaction.
+   * @throws IOException
+   */
+  public StoreFile.Writer compactForTesting(final Store store, Configuration conf,
+      final Collection filesToCompact,
+      boolean isMajor, long maxId) throws IOException {
+    return compact(CompactionRequest.getRequestForTesting(store, conf, filesToCompact, isMajor),
+      maxId);
+  }
+
+  /**
+   * Do a minor/major compaction on an explicit set of storefiles from a Store.
+   * @param request the requested compaction that contains all necessary information to complete the
+   *          compaction (i.e. the store, the files, etc.)
+   * @return Product of compaction or null if all cells expired or deleted and nothing made it
+   *         through the compaction.
+   * @throws IOException
+   */
+  StoreFile.Writer compact(CompactionRequest request, long maxId) throws IOException {
+    // Calculate maximum key count after compaction (for blooms)
+    // Also calculate earliest put timestamp if major compaction
+    int maxKeyCount = 0;
+    long earliestPutTs = HConstants.LATEST_TIMESTAMP;
+    long maxMVCCReadpoint = 0;
+
+    // pull out the interesting things from the CR for ease later
+    final Store store = request.getStore();
+    final boolean majorCompaction = request.isMajor();
+    final List filesToCompact = request.getFiles();
+
+    for (StoreFile file : filesToCompact) {
+      StoreFile.Reader r = file.getReader();
+      if (r == null) {
+        LOG.warn("Null reader for " + file.getPath());
+        continue;
+      }
+      // NOTE: getFilterEntries could cause under-sized blooms if the user
+      //       switches bloom type (e.g. from ROW to ROWCOL)
+      long keyCount = (r.getBloomFilterType() == store.getFamily()
+          .getBloomFilterType()) ?
+          r.getFilterEntries() : r.getEntries();
+      maxKeyCount += keyCount;
+      // Calculate the maximum MVCC readpoint used in any of the involved files
+      Map fileInfo = r.loadFileInfo();
+      byte[] tmp = fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
+      if (tmp != null) {
+        maxMVCCReadpoint = Math.max(maxMVCCReadpoint, Bytes.toLong(tmp));
+      }
+      // For major compactions calculate the earliest put timestamp
+      // of all involved storefiles. This is used to remove 
+      // family delete marker during the compaction.
+      if (majorCompaction) {
+        tmp = fileInfo.get(StoreFile.EARLIEST_PUT_TS);
+        if (tmp == null) {
+          // There's a file with no information, must be an old one
+          // assume we have very old puts
+          earliestPutTs = HConstants.OLDEST_TIMESTAMP;
+        } else {
+          earliestPutTs = Math.min(earliestPutTs, Bytes.toLong(tmp));
+        }
+      }
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Compacting " + file +
+          ", keycount=" + keyCount +
+          ", bloomtype=" + r.getBloomFilterType().toString() +
+          ", size=" + StringUtils.humanReadableInt(r.length()) +
+          ", encoding=" + r.getHFileReader().getEncodingOnDisk() +
+          (majorCompaction? ", earliestPutTs=" + earliestPutTs: ""));
+      }
+    }
+
+    // keep track of compaction progress
+    this.progress = new CompactionProgress(maxKeyCount);
+    // Get some configs
+    int compactionKVMax = getConf().getInt("hbase.hstore.compaction.kv.max", 10);
+    Compression.Algorithm compression = store.getFamily().getCompression();
+    // Avoid overriding compression setting for major compactions if the user
+    // has not specified it separately
+    Compression.Algorithm compactionCompression =
+      (store.getFamily().getCompactionCompression() != Compression.Algorithm.NONE) ?
+      store.getFamily().getCompactionCompression(): compression;
+
+    // For each file, obtain a scanner:
+    List scanners = StoreFileScanner
+      .getScannersForStoreFiles(filesToCompact, false, false, true);
+
+    // Make the instantiation lazy in case compaction produces no product; i.e.
+    // where all source cells are expired or deleted.
+    StoreFile.Writer writer = null;
+    // Find the smallest read point across all the Scanners.
+    long smallestReadPoint = store.getHRegion().getSmallestReadPoint();
+    MultiVersionConsistencyControl.setThreadReadPoint(smallestReadPoint);
+    try {
+      InternalScanner scanner = null;
+      try {
+        if (store.getHRegion().getCoprocessorHost() != null) {
+          scanner = store.getHRegion()
+              .getCoprocessorHost()
+              .preCompactScannerOpen(store, scanners,
+                majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs,
+                request);
+        }
+        if (scanner == null) {
+          Scan scan = new Scan();
+          scan.setMaxVersions(store.getFamily().getMaxVersions());
+          /* Include deletes, unless we are doing a major compaction */
+          scanner = new StoreScanner(store, store.getScanInfo(), scan, scanners,
+            majorCompaction? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT,
+            smallestReadPoint, earliestPutTs);
+        }
+        if (store.getHRegion().getCoprocessorHost() != null) {
+          InternalScanner cpScanner =
+            store.getHRegion().getCoprocessorHost().preCompact(store, scanner, request);
+          // NULL scanner returned from coprocessor hooks means skip normal processing
+          if (cpScanner == null) {
+            return null;
+          }
+          scanner = cpScanner;
+        }
+
+        int bytesWritten = 0;
+        // since scanner.next() can return 'false' but still be delivering data,
+        // we have to use a do/while loop.
+        List kvs = new ArrayList();
+        // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
+        boolean hasMore;
+        do {
+          hasMore = scanner.next(kvs, compactionKVMax);
+          // Create the writer even if no kv(Empty store file is also ok),
+          // because we need record the max seq id for the store file, see
+          // HBASE-6059
+          if (writer == null) {
+            writer = store.createWriterInTmp(maxKeyCount, compactionCompression, true,
+                maxMVCCReadpoint >= smallestReadPoint);
+          }
+          if (writer != null) {
+            // output to writer:
+            for (KeyValue kv : kvs) {
+              if (kv.getMemstoreTS() <= smallestReadPoint) {
+                kv.setMemstoreTS(0);
+              }
+              writer.append(kv);
+              // update progress per key
+              ++progress.currentCompactedKVs;
+
+              // check periodically to see if a system stop is requested
+              if (Store.closeCheckInterval > 0) {
+                bytesWritten += kv.getLength();
+                if (bytesWritten > Store.closeCheckInterval) {
+                  bytesWritten = 0;
+                  isInterrupted(store, writer);
+                }
+              }
+            }
+          }
+          kvs.clear();
+        } while (hasMore);
+      } finally {
+        if (scanner != null) {
+          scanner.close();
+        }
+      }
+    } finally {
+      if (writer != null) {
+        writer.appendMetadata(maxId, majorCompaction);
+        writer.close();
+      }
+    }
+    return writer;
+  }
+
+  void isInterrupted(final Store store, final StoreFile.Writer writer)
+  throws IOException {
+    if (store.getHRegion().areWritesEnabled()) return;
+    // Else cleanup.
+    writer.close();
+    store.getFileSystem().delete(writer.getPath(), false);
+    throw new InterruptedIOException( "Aborting compaction of store " + store +
+      " in region " + store.getHRegion() + " because user requested stop.");
+  }
+
+  CompactionProgress getProgress() {
+    return this.progress;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java b/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
new file mode 100644
index 0000000..0197a60
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
@@ -0,0 +1,466 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Do a shallow merge of multiple KV configuration pools. This is a very useful
+ * utility class to easily add per-object configurations in addition to wider
+ * scope settings. This is different from Configuration.addResource()
+ * functionality, which performs a deep merge and mutates the common data
+ * structure.
+ * 
            
+ * For clarity: the shallow merge allows the user to mutate either of the
+ * configuration objects and have changes reflected everywhere. In contrast to a
+ * deep merge, that requires you to explicitly know all applicable copies to
+ * propagate changes.
+ * 
            
+ * This class is package private because we expect significant refactoring here
+ * on the HBase side when certain HDFS changes are added & ubiquitous. Will
+ * revisit expanding access at that point.
+ */
+@InterfaceAudience.Private
+class CompoundConfiguration extends Configuration {
+  /**
+   * Default Constructor. Initializes empty configuration
+   */
+  public CompoundConfiguration() {
+  }
+
+  // Devs: these APIs are the same contract as their counterparts in
+  // Configuration.java
+  private static interface ImmutableConfigMap {
+    String get(String key);
+    String getRaw(String key);
+    Class getClassByName(String name) throws ClassNotFoundException;
+    int size();
+  }
+
+  protected List configs
+    = new ArrayList();
+
+  /****************************************************************************
+   * These initial APIs actually required original thought
+   ***************************************************************************/
+
+  /**
+   * Add Hadoop Configuration object to config list
+   * @param conf configuration object
+   * @return this, for builder pattern
+   */
+  public CompoundConfiguration add(final Configuration conf) {
+    if (conf instanceof CompoundConfiguration) {
+      this.configs.addAll(0, ((CompoundConfiguration) conf).configs);
+      return this;
+    }
+    // put new config at the front of the list (top priority)
+    this.configs.add(0, new ImmutableConfigMap() {
+      Configuration c = conf;
+
+      @Override
+      public String get(String key) {
+        return c.get(key);
+      }
+
+      @Override
+      public String getRaw(String key) {
+        return c.getRaw(key);
+      }
+
+      @Override
+      public Class getClassByName(String name)
+          throws ClassNotFoundException {
+        return c.getClassByName(name);
+      }
+
+      @Override
+      public int size() {
+        return c.size();
+      }
+
+      @Override
+      public String toString() {
+        return c.toString();
+      }
+    });
+    return this;
+  }
+
+  /**
+   * Add ImmutableBytesWritable map to config list. This map is generally
+   * created by HTableDescriptor or HColumnDescriptor, but can be abstractly
+   * used.
+   *
+   * @param map
+   *          ImmutableBytesWritable map
+   * @return this, for builder pattern
+   */
+  public CompoundConfiguration add(
+      final Map map) {
+    // put new map at the front of the list (top priority)
+    this.configs.add(0, new ImmutableConfigMap() {
+      Map m = map;
+
+      @Override
+      public String get(String key) {
+        ImmutableBytesWritable ibw = new ImmutableBytesWritable(Bytes
+            .toBytes(key));
+        if (!m.containsKey(ibw))
+          return null;
+        ImmutableBytesWritable value = m.get(ibw);
+        if (value == null || value.get() == null)
+          return null;
+        return Bytes.toString(value.get());
+      }
+
+      @Override
+      public String getRaw(String key) {
+        return get(key);
+      }
+
+      @Override
+      public Class getClassByName(String name)
+      throws ClassNotFoundException {
+        return null;
+      }
+
+      @Override
+      public int size() {
+        // TODO Auto-generated method stub
+        return m.size();
+      }
+
+      @Override
+      public String toString() {
+        return m.toString();
+      }
+    });
+    return this;
+  }
+
+  @Override
+  public String toString() {
+    StringBuffer sb = new StringBuffer();
+    sb.append("CompoundConfiguration: " + this.configs.size() + " configs");
+    for (ImmutableConfigMap m : this.configs) {
+      sb.append(this.configs);
+    }
+    return sb.toString();
+  }
+
+  @Override
+  public String get(String key) {
+    for (ImmutableConfigMap m : this.configs) {
+      String value = m.get(key);
+      if (value != null) {
+        return value;
+      }
+    }
+    return null;
+  }
+
+  @Override
+  public String getRaw(String key) {
+    for (ImmutableConfigMap m : this.configs) {
+      String value = m.getRaw(key);
+      if (value != null) {
+        return value;
+      }
+    }
+    return null;
+  }
+
+  @Override
+  public Class getClassByName(String name) throws ClassNotFoundException {
+    for (ImmutableConfigMap m : this.configs) {
+      try {
+        Class value = m.getClassByName(name);
+        if (value != null) {
+          return value;
+        }
+      } catch (ClassNotFoundException e) {
+        // don't propagate an exception until all configs fail
+        continue;
+      }
+    }
+    throw new ClassNotFoundException();
+  }
+
+  @Override
+  public int size() {
+    int ret = 0;
+    for (ImmutableConfigMap m : this.configs) {
+      ret += m.size();
+    }
+    return ret;
+  }
+
+  /***************************************************************************
+   * You should just ignore everything below this line unless there's a bug in
+   * Configuration.java...
+   *
+   * Below get APIs are directly copied from Configuration.java Oh, how I wish
+   * this wasn't so! A tragically-sad example of why you use interfaces instead
+   * of inheritance.
+   *
+   * Why the duplication? We basically need to override Configuration.getProps
+   * or we'd need protected access to Configuration.properties so we can modify
+   * that pointer. There are a bunch of functions in the base Configuration that
+   * call getProps() and we need to use our derived version instead of the base
+   * version. We need to make a generic implementation that works across all
+   * HDFS versions. We should modify Configuration.properties in HDFS 1.0 to be
+   * protected, but we still need to have this code until that patch makes it to
+   * all the HDFS versions we support.
+   ***************************************************************************/
+
+  @Override
+  public String get(String name, String defaultValue) {
+    String ret = get(name);
+    return ret == null ? defaultValue : ret;
+  }
+
+  @Override
+  public int getInt(String name, int defaultValue) {
+    String valueString = get(name);
+    if (valueString == null)
+      return defaultValue;
+    try {
+      String hexString = getHexDigits(valueString);
+      if (hexString != null) {
+        return Integer.parseInt(hexString, 16);
+      }
+      return Integer.parseInt(valueString);
+    } catch (NumberFormatException e) {
+      return defaultValue;
+    }
+  }
+
+  @Override
+  public long getLong(String name, long defaultValue) {
+    String valueString = get(name);
+    if (valueString == null)
+      return defaultValue;
+    try {
+      String hexString = getHexDigits(valueString);
+      if (hexString != null) {
+        return Long.parseLong(hexString, 16);
+      }
+      return Long.parseLong(valueString);
+    } catch (NumberFormatException e) {
+      return defaultValue;
+    }
+  }
+
+  protected String getHexDigits(String value) {
+    boolean negative = false;
+    String str = value;
+    String hexString = null;
+    if (value.startsWith("-")) {
+      negative = true;
+      str = value.substring(1);
+    }
+    if (str.startsWith("0x") || str.startsWith("0X")) {
+      hexString = str.substring(2);
+      if (negative) {
+        hexString = "-" + hexString;
+      }
+      return hexString;
+    }
+    return null;
+  }
+
+  @Override
+  public float getFloat(String name, float defaultValue) {
+    String valueString = get(name);
+    if (valueString == null)
+      return defaultValue;
+    try {
+      return Float.parseFloat(valueString);
+    } catch (NumberFormatException e) {
+      return defaultValue;
+    }
+  }
+
+  @Override
+  public boolean getBoolean(String name, boolean defaultValue) {
+    String valueString = get(name);
+    if ("true".equals(valueString))
+      return true;
+    else if ("false".equals(valueString))
+      return false;
+    else return defaultValue;
+  }
+
+  @Override
+  public IntegerRanges getRange(String name, String defaultValue) {
+    return new IntegerRanges(get(name, defaultValue));
+  }
+
+  @Override
+  public Collection getStringCollection(String name) {
+    String valueString = get(name);
+    return StringUtils.getStringCollection(valueString);
+  }
+
+  @Override
+  public String[] getStrings(String name) {
+    String valueString = get(name);
+    return StringUtils.getStrings(valueString);
+  }
+
+  @Override
+  public String[] getStrings(String name, String... defaultValue) {
+    String valueString = get(name);
+    if (valueString == null) {
+      return defaultValue;
+    } else {
+      return StringUtils.getStrings(valueString);
+    }
+  }
+
+  @Override
+  public Class[] getClasses(String name, Class... defaultValue) {
+    String[] classnames = getStrings(name);
+    if (classnames == null)
+      return defaultValue;
+    try {
+      Class[] classes = new Class[classnames.length];
+      for (int i = 0; i < classnames.length; i++) {
+        classes[i] = getClassByName(classnames[i]);
+      }
+      return classes;
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public Class getClass(String name, Class defaultValue) {
+    String valueString = get(name);
+    if (valueString == null)
+      return defaultValue;
+    try {
+      return getClassByName(valueString);
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public  Class getClass(String name,
+      Class defaultValue, Class xface) {
+    try {
+      Class theClass = getClass(name, defaultValue);
+      if (theClass != null && !xface.isAssignableFrom(theClass))
+        throw new RuntimeException(theClass + " not " + xface.getName());
+      else if (theClass != null)
+        return theClass.asSubclass(xface);
+      else
+        return null;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /*******************************************************************
+   * This class is immutable. Quickly abort any attempts to alter it *
+   *******************************************************************/
+
+  @Override
+  public void clear() {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+
+  @Override
+  public Iterator> iterator() {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+
+  @Override
+  public void set(String name, String value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setIfUnset(String name, String value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setInt(String name, int value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setLong(String name, long value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setFloat(String name, float value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setBoolean(String name, boolean value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setBooleanIfUnset(String name, boolean value) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setStrings(String name, String... values) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setClass(String name, Class theClass, Class xface) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+  @Override
+  public void setClassLoader(ClassLoader classLoader) {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+
+  @Override
+  public void writeXml(OutputStream out) throws IOException {
+    throw new UnsupportedOperationException("Immutable Configuration");
+  }
+};
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
new file mode 100644
index 0000000..90ba281
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+
+/**
+ * A {@link RegionSplitPolicy} implementation which splits a region
+ * as soon as any of its store files exceeds a maximum configurable
+ * size.
+ * 
            
+ * This is the default split policy. From 0.94.0 on the default split policy has
+ * changed to {@link IncreasingToUpperBoundRegionSplitPolicy}
+ * 
            
+ */
+public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
+  private long desiredMaxFileSize;
+
+  @Override
+  protected void configureForRegion(HRegion region) {
+    super.configureForRegion(region);
+    Configuration conf = getConf();
+    HTableDescriptor desc = region.getTableDesc();
+    if (desc != null) {
+      this.desiredMaxFileSize = desc.getMaxFileSize();
+    }
+    if (this.desiredMaxFileSize <= 0) {
+      this.desiredMaxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
+        HConstants.DEFAULT_MAX_FILE_SIZE);
+    }
+  }
+
+  @Override
+  protected boolean shouldSplit() {
+    boolean force = region.shouldForceSplit();
+    boolean foundABigStore = false;
+
+    for (Store store : region.getStores().values()) {
+      // If any of the stores are unable to split (eg they contain reference files)
+      // then don't split
+      if ((!store.canSplit())) {
+        return false;
+      }
+
+      // Mark if any store is big enough
+      if (store.getSize() > desiredMaxFileSize) {
+        foundABigStore = true;
+      }
+    }
+
+    return foundABigStore || force;
+  }
+
+  long getDesiredMaxFileSize() {
+    return desiredMaxFileSize;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/DebugPrint.java b/src/main/java/org/apache/hadoop/hbase/regionserver/DebugPrint.java
new file mode 100644
index 0000000..e1d69c7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/DebugPrint.java
@@ -0,0 +1,69 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class DebugPrint {
+
+private static final AtomicBoolean enabled = new AtomicBoolean(false);
+  private static final Object sync = new Object();
+  public static StringBuilder out = new StringBuilder();
+
+  static public void enable() {
+    enabled.set(true);
+  }
+  static public void disable() {
+    enabled.set(false);
+  }
+
+  static public void reset() {
+    synchronized (sync) {
+      enable(); // someone wants us enabled basically.
+
+      out = new StringBuilder();
+    }
+  }
+  static public void dumpToFile(String file) throws IOException {
+    FileWriter f = new FileWriter(file);
+    synchronized (sync) {
+      f.write(out.toString());
+    }
+    f.close();
+  }
+
+  public static void println(String m) {
+    if (!enabled.get()) {
+      System.out.println(m);
+      return;
+    }
+
+    synchronized (sync) {
+      String threadName = Thread.currentThread().getName();
+      out.append("<");
+      out.append(threadName);
+      out.append("> ");
+      out.append(m);
+      out.append("\n");
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java
new file mode 100644
index 0000000..2963084
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java
@@ -0,0 +1,110 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+/**
+ * This interface is used for the tracking and enforcement of Deletes
+ * during the course of a Get or Scan operation.
+ * 
            
+ * This class is utilized through three methods:
+ * 
            • {@link #add} when encountering a Delete
+ * 
            • {@link #isDeleted} when checking if a Put KeyValue has been deleted
+ * 
            • {@link #update} when reaching the end of a StoreFile
+ */
+public interface DeleteTracker {
+
+  /**
+   * Add the specified KeyValue to the list of deletes to check against for
+   * this row operation.
+   * 
              
+   * This is called when a Delete is encountered in a StoreFile.
+   * @param buffer KeyValue buffer
+   * @param qualifierOffset column qualifier offset
+   * @param qualifierLength column qualifier length
+   * @param timestamp timestamp
+   * @param type delete type as byte
+   */
+  public void add(byte [] buffer, int qualifierOffset, int qualifierLength,
+      long timestamp, byte type);
+
+  /**
+   * Check if the specified KeyValue buffer has been deleted by a previously
+   * seen delete.
+   * @param buffer KeyValue buffer
+   * @param qualifierOffset column qualifier offset
+   * @param qualifierLength column qualifier length
+   * @param timestamp timestamp
+   * @return deleteResult The result tells whether the KeyValue is deleted and why
+   */
+  public DeleteResult isDeleted(byte [] buffer, int qualifierOffset,
+      int qualifierLength, long timestamp);
+
+  /**
+   * @return true if there are no current delete, false otherwise
+   */
+  public boolean isEmpty();
+
+  /**
+   * Called at the end of every StoreFile.
+   * 
              
+   * Many optimized implementations of Trackers will require an update at
+   * when the end of each StoreFile is reached.
+   */
+  public void update();
+
+  /**
+   * Called between rows.
+   * 
              
+   * This clears everything as if a new DeleteTracker was instantiated.
+   */
+  public void reset();
+
+
+  /**
+   * Return codes for comparison of two Deletes.
+   * 
              
+   * The codes tell the merging function what to do.
+   * 
              
+   * INCLUDE means add the specified Delete to the merged list.
+   * NEXT means move to the next element in the specified list(s).
+   */
+  enum DeleteCompare {
+    INCLUDE_OLD_NEXT_OLD,
+    INCLUDE_OLD_NEXT_BOTH,
+    INCLUDE_NEW_NEXT_NEW,
+    INCLUDE_NEW_NEXT_BOTH,
+    NEXT_OLD,
+    NEXT_NEW
+  }
+
+  /**
+   * Returns codes for delete result.
+   * The codes tell the ScanQueryMatcher whether the kv is deleted and why.
+   * Based on the delete result, the ScanQueryMatcher will decide the next
+   * operation
+   */
+  public static enum DeleteResult {
+    FAMILY_DELETED, // The KeyValue is deleted by a delete family.
+    COLUMN_DELETED, // The KeyValue is deleted by a delete column.
+    VERSION_DELETED, // The KeyValue is deleted by a version delete.
+    NOT_DELETED
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/DelimitedKeyPrefixRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/DelimitedKeyPrefixRegionSplitPolicy.java
new file mode 100644
index 0000000..c0940ed
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/DelimitedKeyPrefixRegionSplitPolicy.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.Arrays;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * A custom RegionSplitPolicy implementing a SplitPolicy that groups
+ * rows by a prefix of the row-key with a delimiter. Only the first delimiter
+ * for the row key will define the prefix of the row key that is used for grouping.
+ *
+ * This ensures that a region is not split "inside" a prefix of a row key.
+ * I.e. rows can be co-located in a region by their prefix.
+ *
+ * As an example, if you have row keys delimited with _, like
+ * userid_eventtype_eventid, and use prefix delimiter _, this split policy
+ * ensures that all rows starting with the same userid, belongs to the same region.
+ * @see KeyPrefixRegionSplitPolicy
+ */
+@InterfaceAudience.Private
+public class DelimitedKeyPrefixRegionSplitPolicy extends IncreasingToUpperBoundRegionSplitPolicy {
+
+  private static final Log LOG = LogFactory
+      .getLog(DelimitedKeyPrefixRegionSplitPolicy.class);
+  public static final String DELIMITER_KEY = "DelimitedKeyPrefixRegionSplitPolicy.delimiter";
+
+  private byte[] delimiter = null;
+
+  @Override
+  protected void configureForRegion(HRegion region) {
+    super.configureForRegion(region);
+    if (region != null) {
+
+      // read the prefix length from the table descriptor
+      String delimiterString = region.getTableDesc().getValue(
+          DELIMITER_KEY);
+      if (delimiterString == null || delimiterString.length() == 0) {
+        LOG.error(DELIMITER_KEY + " not specified for table "
+            + region.getTableDesc().getNameAsString()
+            + ". Using default RegionSplitPolicy");
+        return;
+      }
+
+      delimiter = Bytes.toBytes(delimiterString);
+    }
+  }
+
+  @Override
+  protected byte[] getSplitPoint() {
+    byte[] splitPoint = super.getSplitPoint();
+    if (delimiter != null) {
+
+      //find the first occurrence of delimiter in split point
+      int index = com.google.common.primitives.Bytes.indexOf(splitPoint, delimiter);
+      if (index < 0) {
+        LOG.warn("Delimiter " + Bytes.toString(delimiter) + "  not found for split key "
+            + Bytes.toString(splitPoint));
+        return splitPoint;
+      }
+
+      // group split keys by a prefix
+      return Arrays.copyOf(splitPoint, Math.min(index, splitPoint.length));
+    } else {
+      return splitPoint;
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/DisabledRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/DisabledRegionSplitPolicy.java
new file mode 100644
index 0000000..c5f40b3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/DisabledRegionSplitPolicy.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+/**
+ * A {@link RegionSplitPolicy} that disables region splits.
+ * This should be used with care, since it will disable automatic sharding.
+ * Most of the time, using {@link ConstantSizeRegionSplitPolicy} with a
+ * large region size (10GB, etc) is safer.
+ */
+public class DisabledRegionSplitPolicy extends RegionSplitPolicy {
+  @Override
+  protected boolean shouldSplit() {
+    return false;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
new file mode 100644
index 0000000..49375cf
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
@@ -0,0 +1,279 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableSet;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * This class is used for the tracking and enforcement of columns and numbers
+ * of versions during the course of a Get or Scan operation, when explicit
+ * column qualifiers have been asked for in the query.
+ *
+ * With a little magic (see {@link ScanQueryMatcher}), we can use this matcher
+ * for both scans and gets.  The main difference is 'next' and 'done' collapse
+ * for the scan case (since we see all columns in order), and we only reset
+ * between rows.
+ *
+ * 
              
+ * This class is utilized by {@link ScanQueryMatcher} through two methods:
+ * 
              • {@link #checkColumn} is called when a Put satisfies all other
+ * conditions of the query.  This method returns a {@link org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode} to define
+ * what action should be taken.
+ * 
              • {@link #update} is called at the end of every StoreFile or memstore.
+ * 
                
+ * This class is NOT thread-safe as queries are never multi-threaded
+ */
+public class ExplicitColumnTracker implements ColumnTracker {
+
+  private final int maxVersions;
+  private final int minVersions;
+
+ /**
+  * Contains the list of columns that the ExplicitColumnTracker is tracking.
+  * Each ColumnCount instance also tracks how many versions of the requested
+  * column have been returned.
+  */
+  private final List columns;
+  private final List columnsToReuse;
+  private int index;
+  private ColumnCount column;
+  /** Keeps track of the latest timestamp included for current column.
+   * Used to eliminate duplicates. */
+  private long latestTSOfCurrentColumn;
+  private long oldestStamp;
+
+  /**
+   * Default constructor.
+   * @param columns columns specified user in query
+   * @param minVersions minimum number of versions to keep
+   * @param maxVersions maximum versions to return per column
+   * @param oldestUnexpiredTS the oldest timestamp we are interested in,
+   *  based on TTL 
+   * @param ttl The timeToLive to enforce
+   */
+  public ExplicitColumnTracker(NavigableSet columns, int minVersions,
+      int maxVersions, long oldestUnexpiredTS) {
+    this.maxVersions = maxVersions;
+    this.minVersions = minVersions;
+    this.oldestStamp = oldestUnexpiredTS;
+    this.columns = new ArrayList(columns.size());
+    this.columnsToReuse = new ArrayList(columns.size());
+    for(byte [] column : columns) {
+      this.columnsToReuse.add(new ColumnCount(column));
+    }
+    reset();
+  }
+
+    /**
+   * Done when there are no more columns to match against.
+   */
+  public boolean done() {
+    return this.columns.size() == 0;
+  }
+
+  public ColumnCount getColumnHint() {
+    return this.column;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
+      int length, long timestamp, byte type, boolean ignoreCount) {
+    // delete markers should never be passed to an
+    // *Explicit*ColumnTracker
+    assert !KeyValue.isDelete(type);
+    do {
+      // No more columns left, we are done with this query
+      if(this.columns.size() == 0) {
+        return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row
+      }
+
+      // No more columns to match against, done with storefile
+      if(this.column == null) {
+        return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row
+      }
+
+      // Compare specific column to current column
+      int ret = Bytes.compareTo(column.getBuffer(), column.getOffset(),
+          column.getLength(), bytes, offset, length);
+
+      // Column Matches. If it is not a duplicate key, increment the version count
+      // and include.
+      if(ret == 0) {
+        if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE;
+
+        //If column matches, check if it is a duplicate timestamp
+        if (sameAsPreviousTS(timestamp)) {
+          //If duplicate, skip this Key
+          return ScanQueryMatcher.MatchCode.SKIP;
+        }
+        int count = this.column.increment();
+        if(count >= maxVersions || (count >= minVersions && isExpired(timestamp))) {
+          // Done with versions for this column
+          // Note: because we are done with this column, and are removing
+          // it from columns, we don't do a ++this.index. The index stays
+          // the same but the columns have shifted within the array such
+          // that index now points to the next column we are interested in.
+          this.columns.remove(this.index);
+
+          resetTS();
+          if (this.columns.size() == this.index) {
+            // We have served all the requested columns.
+            this.column = null;
+            return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW;
+          } else {
+            // We are done with current column; advance to next column
+            // of interest.
+            this.column = this.columns.get(this.index);
+            return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
+          }
+        } else {
+          setTS(timestamp);
+        }
+        return ScanQueryMatcher.MatchCode.INCLUDE;
+      }
+
+      resetTS();
+
+      if (ret > 0) {
+        // The current KV is smaller than the column the ExplicitColumnTracker
+        // is interested in, so seek to that column of interest.
+        return ScanQueryMatcher.MatchCode.SEEK_NEXT_COL;
+      }
+
+      // The current KV is bigger than the column the ExplicitColumnTracker
+      // is interested in. That means there is no more data for the column
+      // of interest. Advance the ExplicitColumnTracker state to next
+      // column of interest, and check again.
+      if (ret <= -1) {
+        if (++this.index >= this.columns.size()) {
+          // No more to match, do not include, done with this row.
+          return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row
+        }
+        // This is the recursive case.
+        this.column = this.columns.get(this.index);
+      }
+    } while(true);
+  }
+
+  /**
+   * Called at the end of every StoreFile or memstore.
+   */
+  public void update() {
+    if(this.columns.size() != 0) {
+      this.index = 0;
+      this.column = this.columns.get(this.index);
+    } else {
+      this.index = -1;
+      this.column = null;
+    }
+  }
+
+  // Called between every row.
+  public void reset() {
+    buildColumnList();
+    this.index = 0;
+    this.column = this.columns.get(this.index);
+    resetTS();
+  }
+
+  private void resetTS() {
+    latestTSOfCurrentColumn = HConstants.LATEST_TIMESTAMP;
+  }
+
+  private void setTS(long timestamp) {
+    latestTSOfCurrentColumn = timestamp;
+  }
+
+  private boolean sameAsPreviousTS(long timestamp) {
+    return timestamp == latestTSOfCurrentColumn;
+  }
+
+  private boolean isExpired(long timestamp) {
+    return timestamp < oldestStamp;
+  }
+
+  private void buildColumnList() {
+    this.columns.clear();
+    this.columns.addAll(this.columnsToReuse);
+    for(ColumnCount col : this.columns) {
+      col.setCount(0);
+    }
+  }
+
+  /**
+   * This method is used to inform the column tracker that we are done with
+   * this column. We may get this information from external filters or
+   * timestamp range and we then need to indicate this information to
+   * tracker. It is required only in case of ExplicitColumnTracker.
+   * @param bytes
+   * @param offset
+   * @param length
+   */
+  public void doneWithColumn(byte [] bytes, int offset, int length) {
+    while (this.column != null) {
+      int compare = Bytes.compareTo(column.getBuffer(), column.getOffset(),
+          column.getLength(), bytes, offset, length);
+      resetTS();
+      if (compare == 0) {
+        this.columns.remove(this.index);
+        if (this.columns.size() == this.index) {
+          // Will not hit any more columns in this storefile
+          this.column = null;
+        } else {
+          this.column = this.columns.get(this.index);
+        }
+        return;
+      } else if ( compare <= -1) {
+        if(++this.index != this.columns.size()) {
+          this.column = this.columns.get(this.index);
+        } else {
+          this.column = null;
+        }
+      } else {
+        return;
+      }
+    }
+  }
+
+  public MatchCode getNextRowOrNextColumn(byte[] bytes, int offset,
+      int qualLength) {
+    doneWithColumn(bytes, offset,qualLength);
+
+    if (getColumnHint() == null) {
+      return MatchCode.SEEK_NEXT_ROW;
+    } else {
+      return MatchCode.SEEK_NEXT_COL;
+    }
+  }
+
+  public boolean isDone(long timestamp) {
+    return minVersions <= 0 && isExpired(timestamp);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java b/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
new file mode 100644
index 0000000..aecd9c3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
@@ -0,0 +1,41 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+/**
+ * Request a flush.
+ */
+public interface FlushRequester {
+  /**
+   * Tell the listener the cache needs to be flushed.
+   *
+   * @param region the HRegion requesting the cache flush
+   */
+  void requestFlush(HRegion region);
+
+  /**
+   * Tell the listener the cache needs to be flushed after a delay
+   *
+   * @param region the HRegion requesting the cache flush
+   * @param delay after how much time should the flush happen
+   */
+  void requestDelayedFlush(HRegion region, long delay);
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java
new file mode 100644
index 0000000..3a26bbb
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java
@@ -0,0 +1,240 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.KVComparator;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * State and utility processing {@link HRegion#getClosestRowBefore(byte[], byte[])}.
+ * Like {@link ScanDeleteTracker} and {@link ScanDeleteTracker} but does not
+ * implement the {@link DeleteTracker} interface since state spans rows (There
+ * is no update nor reset method).
+ */
+class GetClosestRowBeforeTracker {
+  private final KeyValue targetkey;
+  // Any cell w/ a ts older than this is expired.
+  private final long oldestts;
+  private KeyValue candidate = null;
+  private final KVComparator kvcomparator;
+  // Flag for whether we're doing getclosest on a metaregion.
+  private final boolean metaregion;
+  // Offset and length into targetkey demarking table name (if in a metaregion).
+  private final int rowoffset;
+  private final int tablenamePlusDelimiterLength;
+
+  // Deletes keyed by row.  Comparator compares on row portion of KeyValue only.
+  private final NavigableMap> deletes;
+
+  /**
+   * @param c
+   * @param kv Presume first on row: i.e. empty column, maximum timestamp and
+   * a type of Type.Maximum
+   * @param ttl Time to live in ms for this Store
+   * @param metaregion True if this is .META. or -ROOT- region.
+   */
+  GetClosestRowBeforeTracker(final KVComparator c, final KeyValue kv,
+      final long ttl, final boolean metaregion) {
+    super();
+    this.metaregion = metaregion;
+    this.targetkey = kv;
+    // If we are in a metaregion, then our table name is the prefix on the
+    // targetkey.
+    this.rowoffset = kv.getRowOffset();
+    int l = -1;
+    if (metaregion) {
+      l = KeyValue.getDelimiter(kv.getBuffer(), rowoffset, kv.getRowLength(),
+        HRegionInfo.DELIMITER) - this.rowoffset;
+    }
+    this.tablenamePlusDelimiterLength = metaregion? l + 1: -1;
+    this.oldestts = System.currentTimeMillis() - ttl;
+    this.kvcomparator = c;
+    KeyValue.RowComparator rc = new KeyValue.RowComparator(this.kvcomparator);
+    this.deletes = new TreeMap>(rc);
+  }
+
+  /**
+   * @param kv
+   * @return True if this kv is expired.
+   */
+  boolean isExpired(final KeyValue kv) {
+    return Store.isExpired(kv, this.oldestts);
+  }
+
+  /*
+   * Add the specified KeyValue to the list of deletes.
+   * @param kv
+   */
+  private void addDelete(final KeyValue kv) {
+    NavigableSet rowdeletes = this.deletes.get(kv);
+    if (rowdeletes == null) {
+      rowdeletes = new TreeSet(this.kvcomparator);
+      this.deletes.put(kv, rowdeletes);
+    }
+    rowdeletes.add(kv);
+  }
+
+  /*
+   * @param kv Adds candidate if nearer the target than previous candidate.
+   * @return True if updated candidate.
+   */
+  private boolean addCandidate(final KeyValue kv) {
+    if (!isDeleted(kv) && isBetterCandidate(kv)) {
+      this.candidate = kv;
+      return true;
+    }
+    return false;
+  }
+
+  boolean isBetterCandidate(final KeyValue contender) {
+    return this.candidate == null ||
+      (this.kvcomparator.compareRows(this.candidate, contender) < 0 &&
+        this.kvcomparator.compareRows(contender, this.targetkey) <= 0);
+  }
+
+  /*
+   * Check if specified KeyValue buffer has been deleted by a previously
+   * seen delete.
+   * @param kv
+   * @return true is the specified KeyValue is deleted, false if not
+   */
+  private boolean isDeleted(final KeyValue kv) {
+    if (this.deletes.isEmpty()) return false;
+    NavigableSet rowdeletes = this.deletes.get(kv);
+    if (rowdeletes == null || rowdeletes.isEmpty()) return false;
+    return isDeleted(kv, rowdeletes);
+  }
+
+  /**
+   * Check if the specified KeyValue buffer has been deleted by a previously
+   * seen delete.
+   * @param kv
+   * @param ds
+   * @return True is the specified KeyValue is deleted, false if not
+   */
+  public boolean isDeleted(final KeyValue kv, final NavigableSet ds) {
+    if (deletes == null || deletes.isEmpty()) return false;
+    for (KeyValue d: ds) {
+      long kvts = kv.getTimestamp();
+      long dts = d.getTimestamp();
+      if (d.isDeleteFamily()) {
+        if (kvts <= dts) return true;
+        continue;
+      }
+      // Check column
+      int ret = Bytes.compareTo(kv.getBuffer(), kv.getQualifierOffset(),
+          kv.getQualifierLength(),
+        d.getBuffer(), d.getQualifierOffset(), d.getQualifierLength());
+      if (ret <= -1) {
+        // This delete is for an earlier column.
+        continue;
+      } else if (ret >= 1) {
+        // Beyond this kv.
+        break;
+      }
+      // Check Timestamp
+      if (kvts > dts) return false;
+
+      // Check Type
+      switch (KeyValue.Type.codeToType(d.getType())) {
+        case Delete: return kvts == dts;
+        case DeleteColumn: return true;
+        default: continue;
+      }
+    }
+    return false;
+  }
+
+  /*
+   * Handle keys whose values hold deletes.
+   * Add to the set of deletes and then if the candidate keys contain any that
+   * might match, then check for a match and remove it.  Implies candidates
+   * is made with a Comparator that ignores key type.
+   * @param kv
+   * @return True if we removed k from candidates.
+   */
+  boolean handleDeletes(final KeyValue kv) {
+    addDelete(kv);
+    boolean deleted = false;
+    if (!hasCandidate()) return deleted;
+    if (isDeleted(this.candidate)) {
+      this.candidate = null;
+      deleted = true;
+    }
+    return deleted;
+  }
+
+  /**
+   * Do right thing with passed key, add to deletes or add to candidates.
+   * @param kv
+   * @return True if we added a candidate
+   */
+  boolean handle(final KeyValue kv) {
+    if (kv.isDelete()) {
+      handleDeletes(kv);
+      return false;
+    }
+    return addCandidate(kv);
+  }
+
+  /**
+   * @return True if has candidate
+   */
+  public boolean hasCandidate() {
+    return this.candidate != null;
+  }
+
+  /**
+   * @return Best candidate or null.
+   */
+  public KeyValue getCandidate() {
+    return this.candidate;
+  }
+
+  public KeyValue getTargetKey() {
+    return this.targetkey;
+  }
+
+  /**
+   * @param kv Current kv
+   * @param First on row kv.
+   * @param state
+   * @return True if we went too far, past the target key.
+   */
+  boolean isTooFar(final KeyValue kv, final KeyValue firstOnRow) {
+    return this.kvcomparator.compareRows(kv, firstOnRow) > 0;
+  }
+
+  boolean isTargetTable(final KeyValue kv) {
+    if (!metaregion) return true;
+    // Compare start of keys row.  Compare including delimiter.  Saves having
+    // to calculate where tablename ends in the candidate kv.
+    return Bytes.compareTo(this.targetkey.getBuffer(), this.rowoffset,
+        this.tablenamePlusDelimiterLength,
+      kv.getBuffer(), kv.getRowOffset(), this.tablenamePlusDelimiterLength) == 0;
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
new file mode 100644
index 0000000..179e4aa
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -0,0 +1,6076 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.text.ParseException;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.DroppedSnapshotException;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.RegionTooBusyException;
+import org.apache.hadoop.hbase.UnknownScannerException;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.IsolationLevel;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowLock;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.Exec;
+import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
+import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.TimeRange;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+import org.apache.hadoop.hbase.ipc.HBaseRPC;
+import org.apache.hadoop.hbase.ipc.HBaseServer;
+import org.apache.hadoop.hbase.ipc.RpcCallContext;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.metrics.OperationMetrics;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CancelableProgressable;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.hbase.util.CompressionTest;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HashedBytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.MultipleIOException;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.StringUtils;
+import org.cliffc.high_scale_lib.Counter;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ClassToInstanceMap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.MutableClassToInstanceMap;
+
+/**
+ * HRegion stores data for a certain region of a table.  It stores all columns
+ * for each row. A given table consists of one or more HRegions.
+ *
+ * 
                We maintain multiple HStores for a single HRegion.
+ *
+ * 
                An Store is a set of rows with some column data; together,
+ * they make up all the data for the rows.
+ *
+ * 
                Each HRegion has a 'startKey' and 'endKey'.
+ * 
                The first is inclusive, the second is exclusive (except for
+ * the final region)  The endKey of region 0 is the same as
+ * startKey for region 1 (if it exists).  The startKey for the
+ * first region is null. The endKey for the final region is null.
+ *
+ * 
                Locking at the HRegion level serves only one purpose: preventing the
+ * region from being closed (and consequently split) while other operations
+ * are ongoing. Each row level operation obtains both a row lock and a region
+ * read lock for the duration of the operation. While a scanner is being
+ * constructed, getScanner holds a read lock. If the scanner is successfully
+ * constructed, it holds a read lock until it is closed. A close takes out a
+ * write lock and consequently will block for ongoing operations and will block
+ * new operations from starting while the close is in progress.
+ *
+ * 
                An HRegion is defined by its table and its key extent.
+ *
+ * 
                It consists of at least one Store.  The number of Stores should be
+ * configurable, so that data which is accessed together is stored in the same
+ * Store.  Right now, we approximate that by building a single Store for
+ * each column family.  (This config info will be communicated via the
+ * tabledesc.)
+ *
+ * 
                The HTableDescriptor contains metainfo about the HRegion's table.
+ * regionName is a unique identifier for this HRegion. (startKey, endKey]
+ * defines the keyspace for this HRegion.
+ */
+public class HRegion implements HeapSize { // , Writable{
+  public static final Log LOG = LogFactory.getLog(HRegion.class);
+  private static final String MERGEDIR = ".merges";
+
+  public static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY = "hbase.hregion.scan.loadColumnFamiliesOnDemand";
+
+  final AtomicBoolean closed = new AtomicBoolean(false);
+  /* Closing can take some time; use the closing flag if there is stuff we don't
+   * want to do while in closing state; e.g. like offer this region up to the
+   * master as a region to close if the carrying regionserver is overloaded.
+   * Once set, it is never cleared.
+   */
+  final AtomicBoolean closing = new AtomicBoolean(false);
+
+  //////////////////////////////////////////////////////////////////////////////
+  // Members
+  //////////////////////////////////////////////////////////////////////////////
+
+  private final ConcurrentHashMap lockedRows =
+    new ConcurrentHashMap();
+  private final ConcurrentHashMap lockIds =
+    new ConcurrentHashMap();
+  private final AtomicInteger lockIdGenerator = new AtomicInteger(1);
+  static private Random rand = new Random();
+
+  protected final Map stores =
+    new ConcurrentSkipListMap(Bytes.BYTES_RAWCOMPARATOR);
+
+  // Registered region protocol handlers
+  private ClassToInstanceMap
+      protocolHandlers = MutableClassToInstanceMap.create();
+
+  private Map>
+      protocolHandlerNames = Maps.newHashMap();
+
+  /**
+   * Temporary subdirectory of the region directory used for compaction output.
+   */
+  public static final String REGION_TEMP_SUBDIR = ".tmp";
+
+  //These variable are just used for getting data out of the region, to test on
+  //client side
+  // private int numStores = 0;
+  // private int [] storeSize = null;
+  // private byte [] name = null;
+
+  final AtomicLong memstoreSize = new AtomicLong(0);
+
+  // Debug possible data loss due to WAL off
+  final AtomicLong numPutsWithoutWAL = new AtomicLong(0);
+  final AtomicLong dataInMemoryWithoutWAL = new AtomicLong(0);
+
+  final Counter readRequestsCount = new Counter();
+  final Counter writeRequestsCount = new Counter();
+  final Counter updatesBlockedMs = new Counter();
+
+  /**
+   * The directory for the table this region is part of.
+   * This directory contains the directory for this region.
+   */
+  private final Path tableDir;
+
+  private final HLog log;
+  private final FileSystem fs;
+  private final Configuration conf;
+  final Configuration baseConf;
+  private final int rowLockWaitDuration;
+  static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;
+
+  // The internal wait duration to acquire a lock before read/update
+  // from the region. It is not per row. The purpose of this wait time
+  // is to avoid waiting a long time while the region is busy, so that
+  // we can release the IPC handler soon enough to improve the
+  // availability of the region server. It can be adjusted by
+  // tuning configuration "hbase.busy.wait.duration".
+  final long busyWaitDuration;
+  static final long DEFAULT_BUSY_WAIT_DURATION = HConstants.DEFAULT_HBASE_RPC_TIMEOUT;
+
+  // If updating multiple rows in one call, wait longer,
+  // i.e. waiting for busyWaitDuration * # of rows. However,
+  // we can limit the max multiplier.
+  final int maxBusyWaitMultiplier;
+
+  // Max busy wait duration. There is no point to wait longer than the RPC
+  // purge timeout, when a RPC call will be terminated by the RPC engine.
+  final long maxBusyWaitDuration;
+
+  private final HRegionInfo regionInfo;
+  private final Path regiondir;
+  KeyValue.KVComparator comparator;
+
+  private ConcurrentHashMap scannerReadPoints;
+  /**
+   * The default setting for whether to enable on-demand CF loading for
+   * scan requests to this region. Requests can override it.
+   */
+  private boolean isLoadingCfsOnDemandDefault = false;
+
+  /**
+   * @return The smallest mvcc readPoint across all the scanners in this
+   * region. Writes older than this readPoint, are included  in every
+   * read operation.
+   */
+  public long getSmallestReadPoint() {
+    long minimumReadPoint;
+    // We need to ensure that while we are calculating the smallestReadPoint
+    // no new RegionScanners can grab a readPoint that we are unaware of.
+    // We achieve this by synchronizing on the scannerReadPoints object.
+    synchronized(scannerReadPoints) {
+      minimumReadPoint = mvcc.memstoreReadPoint();
+
+      for (Long readPoint: this.scannerReadPoints.values()) {
+        if (readPoint < minimumReadPoint) {
+          minimumReadPoint = readPoint;
+        }
+      }
+    }
+    return minimumReadPoint;
+  }
+  /*
+   * Data structure of write state flags used coordinating flushes,
+   * compactions and closes.
+   */
+  static class WriteState {
+    // Set while a memstore flush is happening.
+    volatile boolean flushing = false;
+    // Set when a flush has been requested.
+    volatile boolean flushRequested = false;
+    // Number of compactions running.
+    volatile int compacting = 0;
+    // Gets set in close. If set, cannot compact or flush again.
+    volatile boolean writesEnabled = true;
+    // Set if region is read-only
+    volatile boolean readOnly = false;
+
+    /**
+     * Set flags that make this region read-only.
+     *
+     * @param onOff flip value for region r/o setting
+     */
+    synchronized void setReadOnly(final boolean onOff) {
+      this.writesEnabled = !onOff;
+      this.readOnly = onOff;
+    }
+
+    boolean isReadOnly() {
+      return this.readOnly;
+    }
+
+    boolean isFlushRequested() {
+      return this.flushRequested;
+    }
+
+    static final long HEAP_SIZE = ClassSize.align(
+        ClassSize.OBJECT + 5 * Bytes.SIZEOF_BOOLEAN);
+  }
+
+  final WriteState writestate = new WriteState();
+
+  long memstoreFlushSize;
+  final long timestampSlop;
+  private volatile long lastFlushTime;
+  final RegionServerServices rsServices;
+  private RegionServerAccounting rsAccounting;
+  private List> recentFlushes = new ArrayList>();
+  private long flushCheckInterval;
+  private long blockingMemStoreSize;
+  final long threadWakeFrequency;
+  // Used to guard closes
+  final ReentrantReadWriteLock lock =
+    new ReentrantReadWriteLock();
+
+  // Stop updates lock
+  private final ReentrantReadWriteLock updatesLock =
+    new ReentrantReadWriteLock();
+  private boolean splitRequest;
+  private byte[] explicitSplitPoint = null;
+
+  private final MultiVersionConsistencyControl mvcc =
+      new MultiVersionConsistencyControl();
+
+  // Coprocessor host
+  private RegionCoprocessorHost coprocessorHost;
+
+  /**
+   * Name of the region info file that resides just under the region directory.
+   */
+  public final static String REGIONINFO_FILE = ".regioninfo";
+  private HTableDescriptor htableDescriptor = null;
+  private RegionSplitPolicy splitPolicy;
+  private final OperationMetrics opMetrics;
+  private final boolean deferredLogSyncDisabled;
+
+  /**
+   * Should only be used for testing purposes
+   */
+  public HRegion(){
+    this.tableDir = null;
+    this.blockingMemStoreSize = 0L;
+    this.conf = null;
+    this.rowLockWaitDuration = DEFAULT_ROWLOCK_WAIT_DURATION;
+    this.rsServices = null;
+    this.baseConf = null;
+    this.fs = null;
+    this.timestampSlop = HConstants.LATEST_TIMESTAMP;
+    this.memstoreFlushSize = 0L;
+    this.log = null;
+    this.regiondir = null;
+    this.regionInfo = null;
+    this.htableDescriptor = null;
+    this.threadWakeFrequency = 0L;
+    this.coprocessorHost = null;
+    this.scannerReadPoints = new ConcurrentHashMap();
+    this.opMetrics = new OperationMetrics();
+
+    this.maxBusyWaitDuration = 2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT;
+    this.busyWaitDuration = DEFAULT_BUSY_WAIT_DURATION;
+    this.maxBusyWaitMultiplier = 2;
+    this.deferredLogSyncDisabled = false;
+  }
+
+  
+  /**
+   * HRegion copy constructor. Useful when reopening a closed region (normally
+   * for unit tests)
+   * @param other original object
+   */
+  public HRegion(HRegion other) {
+    this(other.getTableDir(), other.getLog(), other.getFilesystem(),
+        other.baseConf, other.getRegionInfo(), other.getTableDesc(), null);
+  }
+  
+  /**
+   * HRegion constructor.  his constructor should only be used for testing and
+   * extensions.  Instances of HRegion should be instantiated with the
+   * {@link HRegion#newHRegion(Path, HLog, FileSystem, Configuration, HRegionInfo, HTableDescriptor, RegionServerServices)} method.
+   *
+   *
+   * @param tableDir qualified path of directory where region should be located,
+   * usually the table directory.
+   * @param log The HLog is the outbound log for any updates to the HRegion
+   * (There's a single HLog for all the HRegions on a single HRegionServer.)
+   * The log file is a logfile from the previous execution that's
+   * custom-computed for this HRegion. The HRegionServer computes and sorts the
+   * appropriate log info for this HRegion. If there is a previous log file
+   * (implying that the HRegion has been written-to before), then read it from
+   * the supplied path.
+   * @param fs is the filesystem.
+   * @param conf is global configuration settings.
+   * @param regionInfo - HRegionInfo that describes the region
+   * is new), then read them from the supplied path.
+   * @param rsServices reference to {@link RegionServerServices} or null
+   *
+   * @see HRegion#newHRegion(Path, HLog, FileSystem, Configuration, HRegionInfo, HTableDescriptor, RegionServerServices)
+   */
+  public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration confParam,
+    final HRegionInfo regionInfo, final HTableDescriptor htd,
+      RegionServerServices rsServices) {
+    this.tableDir = tableDir;
+    this.comparator = regionInfo.getComparator();
+    this.log = log;
+    this.fs = fs;
+    if (confParam instanceof CompoundConfiguration) {
+       throw new IllegalArgumentException("Need original base configuration");
+    }
+    // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
+    this.baseConf = confParam;
+    if (htd != null) {
+      this.conf = new CompoundConfiguration().add(confParam).add(htd.getValues());
+    }
+    else {
+      this.conf = new CompoundConfiguration().add(confParam);
+    }
+    this.flushCheckInterval = conf.getInt(MEMSTORE_PERIODIC_FLUSH_INTERVAL,
+        DEFAULT_CACHE_FLUSH_INTERVAL);
+    this.rowLockWaitDuration = conf.getInt("hbase.rowlock.wait.duration",
+                    DEFAULT_ROWLOCK_WAIT_DURATION);
+
+    this.isLoadingCfsOnDemandDefault = conf.getBoolean(LOAD_CFS_ON_DEMAND_CONFIG_KEY, false);
+    this.regionInfo = regionInfo;
+    this.htableDescriptor = htd;
+    this.rsServices = rsServices;
+    this.threadWakeFrequency = conf.getLong(HConstants.THREAD_WAKE_FREQUENCY,
+        10 * 1000);
+    String encodedNameStr = this.regionInfo.getEncodedName();
+    setHTableSpecificConf();
+    this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
+    this.scannerReadPoints = new ConcurrentHashMap();
+    this.opMetrics = new OperationMetrics(conf, this.regionInfo);
+
+    this.busyWaitDuration = conf.getLong(
+      "hbase.busy.wait.duration", DEFAULT_BUSY_WAIT_DURATION);
+    this.maxBusyWaitMultiplier = conf.getInt("hbase.busy.wait.multiplier.max", 2);
+    if (busyWaitDuration * maxBusyWaitMultiplier <= 0L) {
+      throw new IllegalArgumentException("Invalid hbase.busy.wait.duration ("
+        + busyWaitDuration + ") or hbase.busy.wait.multiplier.max ("
+        + maxBusyWaitMultiplier + "). Their product should be positive");
+    }
+    this.maxBusyWaitDuration = conf.getLong("ipc.client.call.purge.timeout",
+      2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
+
+    /*
+     * timestamp.slop provides a server-side constraint on the timestamp. This
+     * assumes that you base your TS around currentTimeMillis(). In this case,
+     * throw an error to the user if the user-specified TS is newer than now +
+     * slop. LATEST_TIMESTAMP == don't use this functionality
+     */
+    this.timestampSlop = conf.getLong(
+        "hbase.hregion.keyvalue.timestamp.slop.millisecs",
+        HConstants.LATEST_TIMESTAMP);
+    // When hbase.regionserver.optionallogflushinterval <= 0 , deferred log sync is disabled.
+    this.deferredLogSyncDisabled = conf.getLong("hbase.regionserver.optionallogflushinterval",
+        1 * 1000) <= 0;
+    
+    if (rsServices != null) {
+      this.rsAccounting = this.rsServices.getRegionServerAccounting();
+      // don't initialize coprocessors if not running within a regionserver
+      // TODO: revisit if coprocessors should load in other cases
+      this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
+    }
+    if (LOG.isDebugEnabled()) {
+      // Write out region name as string and its encoded name.
+      LOG.debug("Instantiated " + this);
+    }
+  }
+
+  void setHTableSpecificConf() {
+    if (this.htableDescriptor == null) return;
+    LOG.info("Setting up tabledescriptor config now ...");
+    long flushSize = this.htableDescriptor.getMemStoreFlushSize();
+
+    if (flushSize <= 0) {
+      flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
+        HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE);
+    }
+    this.memstoreFlushSize = flushSize;
+    this.blockingMemStoreSize = this.memstoreFlushSize *
+        conf.getLong("hbase.hregion.memstore.block.multiplier", 2);
+  }
+
+  /**
+   * Initialize this region.
+   * @return What the next sequence (edit) id should be.
+   * @throws IOException e
+   */
+  public long initialize() throws IOException {
+    return initialize(null);
+  }
+
+  /**
+   * Initialize this region.
+   *
+   * @param reporter Tickle every so often if initialize is taking a while.
+   * @return What the next sequence (edit) id should be.
+   * @throws IOException e
+   */
+  public long initialize(final CancelableProgressable reporter)
+      throws IOException {
+
+    MonitoredTask status = TaskMonitor.get().createStatus(
+        "Initializing region " + this);
+
+    long nextSeqId = -1;
+    try {
+      nextSeqId = initializeRegionInternals(reporter, status);
+      return nextSeqId;
+    } finally {
+      // nextSeqid will be -1 if the initialization fails.
+      // At least it will be 0 otherwise.
+      if (nextSeqId == -1) {
+        status.abort("Exception during region " + this.getRegionNameAsString()
+            + " initialization.");
+      }
+    }
+  }
+
+  private long initializeRegionInternals(final CancelableProgressable reporter,
+      MonitoredTask status) throws IOException, UnsupportedEncodingException {
+    if (coprocessorHost != null) {
+      status.setStatus("Running coprocessor pre-open hook");
+      coprocessorHost.preOpen();
+    }
+
+    // Write HRI to a file in case we need to recover .META.
+    status.setStatus("Writing region info on filesystem");
+    checkRegioninfoOnFilesystem();
+
+    // Remove temporary data left over from old regions
+    status.setStatus("Cleaning up temporary data from old regions");
+    cleanupTmpDir();
+
+    // Load in all the HStores.
+    //
+    // Context: During replay we want to ensure that we do not lose any data. So, we
+    // have to be conservative in how we replay logs. For each store, we calculate
+    // the maxSeqId up to which the store was flushed. And, skip the edits which
+    // is equal to or lower than maxSeqId for each store.
+    Map maxSeqIdInStores = new TreeMap(
+        Bytes.BYTES_COMPARATOR);
+    long maxSeqId = -1;
+    // initialized to -1 so that we pick up MemstoreTS from column families
+    long maxMemstoreTS = -1;
+
+    if (this.htableDescriptor != null &&
+        !htableDescriptor.getFamilies().isEmpty()) {
+      // initialize the thread pool for opening stores in parallel.
+      ThreadPoolExecutor storeOpenerThreadPool =
+        getStoreOpenAndCloseThreadPool(
+          "StoreOpenerThread-" + this.regionInfo.getRegionNameAsString());
+      CompletionService completionService =
+        new ExecutorCompletionService(storeOpenerThreadPool);
+
+      // initialize each store in parallel
+      for (final HColumnDescriptor family : htableDescriptor.getFamilies()) {
+        status.setStatus("Instantiating store for column family " + family);
+        completionService.submit(new Callable() {
+          public Store call() throws IOException {
+            return instantiateHStore(tableDir, family);
+          }
+        });
+      }
+      try {
+        for (int i = 0; i < htableDescriptor.getFamilies().size(); i++) {
+          Future future = completionService.take();
+          Store store = future.get();
+
+          this.stores.put(store.getColumnFamilyName().getBytes(), store);
+          long storeSeqId = store.getMaxSequenceId();
+          maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(),
+              storeSeqId);
+          if (maxSeqId == -1 || storeSeqId > maxSeqId) {
+            maxSeqId = storeSeqId;
+          }
+          long maxStoreMemstoreTS = store.getMaxMemstoreTS();
+          if (maxStoreMemstoreTS > maxMemstoreTS) {
+            maxMemstoreTS = maxStoreMemstoreTS;
+          }
+        }
+      } catch (InterruptedException e) {
+        throw new IOException(e);
+      } catch (ExecutionException e) {
+        throw new IOException(e.getCause());
+      } finally {
+        storeOpenerThreadPool.shutdownNow();
+      }
+    }
+    mvcc.initialize(maxMemstoreTS + 1);
+    // Recover any edits if available.
+    maxSeqId = Math.max(maxSeqId, replayRecoveredEditsIfAny(
+        this.regiondir, maxSeqIdInStores, reporter, status));
+
+    status.setStatus("Cleaning up detritus from prior splits");
+    // Get rid of any splits or merges that were lost in-progress.  Clean out
+    // these directories here on open.  We may be opening a region that was
+    // being split but we crashed in the middle of it all.
+    SplitTransaction.cleanupAnySplitDetritus(this);
+    FSUtils.deleteDirectory(this.fs, new Path(regiondir, MERGEDIR));
+
+    this.writestate.setReadOnly(this.htableDescriptor.isReadOnly());
+
+    this.writestate.flushRequested = false;
+    this.writestate.compacting = 0;
+
+    // Initialize split policy
+    this.splitPolicy = RegionSplitPolicy.create(this, conf);
+
+    this.lastFlushTime = EnvironmentEdgeManager.currentTimeMillis();
+    // Use maximum of log sequenceid or that which was found in stores
+    // (particularly if no recovered edits, seqid will be -1).
+    long nextSeqid = maxSeqId + 1;
+    LOG.info("Onlined " + this.toString() + "; next sequenceid=" + nextSeqid);
+
+    // A region can be reopened if failed a split; reset flags
+    this.closing.set(false);
+    this.closed.set(false);
+
+    if (coprocessorHost != null) {
+      status.setStatus("Running coprocessor post-open hooks");
+      coprocessorHost.postOpen();
+    }
+
+    status.markComplete("Region opened successfully");
+    return nextSeqid;
+  }
+
+  /*
+   * Move any passed HStore files into place (if any).  Used to pick up split
+   * files and any merges from splits and merges dirs.
+   * @param initialFiles
+   * @throws IOException
+   */
+  static void moveInitialFilesIntoPlace(final FileSystem fs,
+    final Path initialFiles, final Path regiondir)
+  throws IOException {
+    if (initialFiles != null && fs.exists(initialFiles)) {
+      if (!HBaseFileSystem.renameDirForFileSystem(fs, initialFiles, regiondir)) {
+        LOG.warn("Unable to rename " + initialFiles + " to " + regiondir);
+      }
+    }
+  }
+
+  /**
+   * @return True if this region has references.
+   */
+  public boolean hasReferences() {
+    for (Store store : this.stores.values()) {
+      for (StoreFile sf : store.getStorefiles()) {
+        // Found a reference, return.
+        if (sf.isReference()) return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * This function will return the HDFS blocks distribution based on the data
+   * captured when HFile is created
+   * @return The HDFS blocks distribution for the region.
+   */
+  public HDFSBlocksDistribution getHDFSBlocksDistribution() {
+    HDFSBlocksDistribution hdfsBlocksDistribution =
+      new HDFSBlocksDistribution();
+    synchronized (this.stores) {
+      for (Store store : this.stores.values()) {
+        for (StoreFile sf : store.getStorefiles()) {
+          HDFSBlocksDistribution storeFileBlocksDistribution =
+            sf.getHDFSBlockDistribution();
+          hdfsBlocksDistribution.add(storeFileBlocksDistribution);
+        }
+      }
+    }
+    return hdfsBlocksDistribution;
+  }
+
+  /**
+   * This is a helper function to compute HDFS block distribution on demand
+   * @param conf configuration
+   * @param tableDescriptor HTableDescriptor of the table
+   * @param regionEncodedName encoded name of the region
+   * @return The HDFS blocks distribution for the given region.
+ * @throws IOException
+   */
+  static public HDFSBlocksDistribution computeHDFSBlocksDistribution(
+    Configuration conf, HTableDescriptor tableDescriptor,
+    String regionEncodedName) throws IOException {
+    HDFSBlocksDistribution hdfsBlocksDistribution =
+      new HDFSBlocksDistribution();
+    Path tablePath = FSUtils.getTablePath(FSUtils.getRootDir(conf),
+      tableDescriptor.getName());
+    FileSystem fs = tablePath.getFileSystem(conf);
+
+    for (HColumnDescriptor family: tableDescriptor.getFamilies()) {
+      Path storeHomeDir = Store.getStoreHomedir(tablePath, regionEncodedName,
+      family.getName());
+      if (!fs.exists(storeHomeDir))continue;
+
+      FileStatus[] hfilesStatus = null;
+      hfilesStatus = fs.listStatus(storeHomeDir);
+
+      for (FileStatus hfileStatus : hfilesStatus) {
+        HDFSBlocksDistribution storeFileBlocksDistribution =
+          FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0,
+          hfileStatus.getLen());
+        hdfsBlocksDistribution.add(storeFileBlocksDistribution);
+      }
+    }
+    return hdfsBlocksDistribution;
+  }
+
+  public AtomicLong getMemstoreSize() {
+    return memstoreSize;
+  }
+
+  /**
+   * Increase the size of mem store in this region and the size of global mem
+   * store
+   * @param memStoreSize
+   * @return the size of memstore in this region
+   */
+  public long addAndGetGlobalMemstoreSize(long memStoreSize) {
+    if (this.rsAccounting != null) {
+      rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
+    }
+    return this.memstoreSize.getAndAdd(memStoreSize);
+  }
+
+  /*
+   * Write out an info file under the region directory.  Useful recovering
+   * mangled regions.
+   * @throws IOException
+   */
+  private void checkRegioninfoOnFilesystem() throws IOException {
+    checkRegioninfoOnFilesystem(this.regiondir);
+  }
+
+  /**
+   * Write out an info file under the region directory. Useful recovering mangled regions.
+   * @param regiondir directory under which to write out the region info
+   * @throws IOException
+   */
+  private void checkRegioninfoOnFilesystem(Path regiondir) throws IOException {
+    writeRegioninfoOnFilesystem(regionInfo, regiondir, getFilesystem(), conf);
+  }
+
+  /**
+   * Write out an info file under the region directory. Useful recovering mangled regions. If the
+   * regioninfo already exists on disk and there is information in the file, then we fast exit.
+   * @param regionInfo information about the region
+   * @param regiondir directory under which to write out the region info
+   * @param fs {@link FileSystem} on which to write the region info
+   * @param conf {@link Configuration} from which to extract specific file locations
+   * @throws IOException on unexpected error.
+   */
+  public static void writeRegioninfoOnFilesystem(HRegionInfo regionInfo, Path regiondir,
+      FileSystem fs, Configuration conf) throws IOException {
+    Path regioninfoPath = new Path(regiondir, REGIONINFO_FILE);
+    if (fs.exists(regioninfoPath)) {
+      if (fs.getFileStatus(regioninfoPath).getLen() > 0) {
+        return;
+      }
+
+      LOG.info("Rewriting .regioninfo file at: " + regioninfoPath);
+      if (!fs.delete(regioninfoPath, false)) {
+        throw new IOException("Unable to remove existing " + regioninfoPath);
+      }
+    }
+
+    // Create in tmpdir and then move into place in case we crash after
+    // create but before close.  If we don't successfully close the file,
+    // subsequent region reopens will fail the below because create is
+    // registered in NN.
+
+    // first check to get the permissions
+    FsPermission perms = FSUtils.getFilePermissions(fs, conf,
+        HConstants.DATA_FILE_UMASK_KEY);
+
+    // and then create the file
+    Path tmpPath = new Path(getTmpDir(regiondir), REGIONINFO_FILE);
+
+    // if datanode crashes or if the RS goes down just before the close is called while trying to
+    // close the created regioninfo file in the .tmp directory then on next
+    // creation we will be getting AlreadyCreatedException.
+    // Hence delete and create the file if exists.
+    if (FSUtils.isExists(fs, tmpPath)) {
+      FSUtils.delete(fs, tmpPath, true);
+    }
+
+    FSDataOutputStream out = FSUtils.create(fs, tmpPath, perms);
+
+    try {
+      regionInfo.write(out);
+      out.write('\n');
+      out.write('\n');
+      out.write(Bytes.toBytes(regionInfo.toString()));
+    } finally {
+      out.close();
+    }
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, tmpPath, regioninfoPath)) {
+      throw new IOException("Unable to rename " + tmpPath + " to " +
+        regioninfoPath);
+    }
+  }
+
+  /**
+   * @param fs
+   * @param dir
+   * @return An HRegionInfo instance gotten from the .regioninfo file under region dir
+   * @throws IOException
+   */
+  public static HRegionInfo loadDotRegionInfoFileContent(final FileSystem fs, final Path dir)
+  throws IOException {
+    Path regioninfo = new Path(dir, HRegion.REGIONINFO_FILE);
+    if (!fs.exists(regioninfo)) throw new FileNotFoundException(regioninfo.toString());
+    FSDataInputStream in = fs.open(regioninfo);
+    try {
+      HRegionInfo hri = new HRegionInfo();
+      hri.readFields(in);
+      return hri;
+    } finally {
+      in.close();
+    }
+  }
+
+  /** @return a HRegionInfo object for this region */
+  public HRegionInfo getRegionInfo() {
+    return this.regionInfo;
+  }
+
+  /**
+   * @return Instance of {@link RegionServerServices} used by this HRegion.
+   * Can be null.
+   */
+  RegionServerServices getRegionServerServices() {
+    return this.rsServices;
+  }
+
+  /** @return requestsCount for this region */
+  public long getRequestsCount() {
+    return this.readRequestsCount.get() + this.writeRequestsCount.get();
+  }
+
+  /** @return readRequestsCount for this region */
+  public long getReadRequestsCount() {
+    return this.readRequestsCount.get();
+  }
+
+  /** @return writeRequestsCount for this region */
+  public long getWriteRequestsCount() {
+    return this.writeRequestsCount.get();
+  }
+
+  /** @return true if region is closed */
+  public boolean isClosed() {
+    return this.closed.get();
+  }
+
+  /**
+   * @return True if closing process has started.
+   */
+  public boolean isClosing() {
+    return this.closing.get();
+  }
+
+  /** @return true if region is available (not closed and not closing) */
+  public boolean isAvailable() {
+    return !isClosed() && !isClosing();
+  }
+
+  /** @return true if region is splittable */
+  public boolean isSplittable() {
+    return isAvailable() && !hasReferences();
+  }
+
+  boolean areWritesEnabled() {
+    synchronized(this.writestate) {
+      return this.writestate.writesEnabled;
+    }
+  }
+
+   public MultiVersionConsistencyControl getMVCC() {
+     return mvcc;
+   }
+
+   public boolean isLoadingCfsOnDemandDefault() {
+     return this.isLoadingCfsOnDemandDefault;
+   }
+
+  /**
+   * Close down this HRegion.  Flush the cache, shut down each HStore, don't
+   * service any more calls.
+   *
+   * 
                This method could take some time to execute, so don't call it from a
+   * time-sensitive thread.
+   *
+   * @return Vector of all the storage files that the HRegion's component
+   * HStores make use of.  It's a list of all HStoreFile objects. Returns empty
+   * vector if already closed and null if judged that it should not close.
+   *
+   * @throws IOException e
+   */
+  public List close() throws IOException {
+    return close(false);
+  }
+
+  private final Object closeLock = new Object();
+
+  /** Conf key for the periodic flush interval */
+  public static final String MEMSTORE_PERIODIC_FLUSH_INTERVAL = 
+      "hbase.regionserver.optionalcacheflushinterval";
+  /** Default interval for the memstore flush */
+  public static final int DEFAULT_CACHE_FLUSH_INTERVAL = 3600000;
+
+  /**
+   * Close down this HRegion.  Flush the cache unless abort parameter is true,
+   * Shut down each HStore, don't service any more calls.
+   *
+   * This method could take some time to execute, so don't call it from a
+   * time-sensitive thread.
+   *
+   * @param abort true if server is aborting (only during testing)
+   * @return Vector of all the storage files that the HRegion's component
+   * HStores make use of.  It's a list of HStoreFile objects.  Can be null if
+   * we are not to close at this time or we are already closed.
+   *
+   * @throws IOException e
+   */
+  public List close(final boolean abort) throws IOException {
+    // Only allow one thread to close at a time. Serialize them so dual
+    // threads attempting to close will run up against each other.
+    MonitoredTask status = TaskMonitor.get().createStatus(
+        "Closing region " + this +
+        (abort ? " due to abort" : ""));
+
+    status.setStatus("Waiting for close lock");
+    try {
+      synchronized (closeLock) {
+        return doClose(abort, status);
+      }
+    } finally {
+      status.cleanup();
+    }
+  }
+
+  private List doClose(
+      final boolean abort, MonitoredTask status)
+  throws IOException {
+    if (isClosed()) {
+      LOG.warn("Region " + this + " already closed");
+      return null;
+    }
+
+    if (coprocessorHost != null) {
+      status.setStatus("Running coprocessor pre-close hooks");
+      this.coprocessorHost.preClose(abort);
+    }
+
+    status.setStatus("Disabling compacts and flushes for region");
+    boolean wasFlushing = false;
+    synchronized (writestate) {
+      // Disable compacting and flushing by background threads for this
+      // region.
+      writestate.writesEnabled = false;
+      wasFlushing = writestate.flushing;
+      LOG.debug("Closing " + this + ": disabling compactions & flushes");
+      waitForFlushesAndCompactions();
+    }
+    // If we were not just flushing, is it worth doing a preflush...one
+    // that will clear out of the bulk of the memstore before we put up
+    // the close flag?
+    if (!abort && !wasFlushing && worthPreFlushing()) {
+      status.setStatus("Pre-flushing region before close");
+      LOG.info("Running close preflush of " + this.getRegionNameAsString());
+      internalFlushcache(status);
+    }
+
+    this.closing.set(true);
+    status.setStatus("Disabling writes for close");
+    // block waiting for the lock for closing
+    lock.writeLock().lock();
+    try {
+      if (this.isClosed()) {
+        status.abort("Already got closed by another process");
+        // SplitTransaction handles the null
+        return null;
+      }
+      LOG.debug("Updates disabled for region " + this);
+      // Don't flush the cache if we are aborting
+      if (!abort) {
+        internalFlushcache(status);
+      }
+
+      List result = new ArrayList();
+      if (!stores.isEmpty()) {
+        // initialize the thread pool for closing stores in parallel.
+        ThreadPoolExecutor storeCloserThreadPool =
+          getStoreOpenAndCloseThreadPool("StoreCloserThread-"
+            + this.regionInfo.getRegionNameAsString());
+        CompletionService> completionService =
+          new ExecutorCompletionService>(
+            storeCloserThreadPool);
+
+        // close each store in parallel
+        for (final Store store : stores.values()) {
+          completionService
+              .submit(new Callable>() {
+                public ImmutableList call() throws IOException {
+                  return store.close();
+                }
+              });
+        }
+        try {
+          for (int i = 0; i < stores.size(); i++) {
+            Future> future = completionService
+                .take();
+            ImmutableList storeFileList = future.get();
+            result.addAll(storeFileList);
+          }
+        } catch (InterruptedException e) {
+          throw new IOException(e);
+        } catch (ExecutionException e) {
+          throw new IOException(e.getCause());
+        } finally {
+          storeCloserThreadPool.shutdownNow();
+        }
+      }
+      this.closed.set(true);
+
+      if (coprocessorHost != null) {
+        status.setStatus("Running coprocessor post-close hooks");
+        this.coprocessorHost.postClose(abort);
+      }
+      this.opMetrics.closeMetrics(this.getRegionInfo().getEncodedName());
+      status.markComplete("Closed");
+      LOG.info("Closed " + this);
+      return result;
+    } finally {
+      lock.writeLock().unlock();
+    }
+  }
+
+  /**
+   * Wait for all current flushes and compactions of the region to complete.
+   * 
                
+   * Exposed for TESTING.
+   */
+  public void waitForFlushesAndCompactions() {
+    synchronized (writestate) {
+      while (writestate.compacting > 0 || writestate.flushing) {
+        LOG.debug("waiting for " + writestate.compacting + " compactions"
+            + (writestate.flushing ? " & cache flush" : "") + " to complete for region " + this);
+        try {
+          writestate.wait();
+        } catch (InterruptedException iex) {
+          // essentially ignore and propagate the interrupt back up
+          Thread.currentThread().interrupt();
+        }
+      }
+    }
+  }
+
+  protected ThreadPoolExecutor getStoreOpenAndCloseThreadPool(
+      final String threadNamePrefix) {
+    int numStores = Math.max(1, this.htableDescriptor.getFamilies().size());
+    int maxThreads = Math.min(numStores,
+        conf.getInt(HConstants.HSTORE_OPEN_AND_CLOSE_THREADS_MAX,
+            HConstants.DEFAULT_HSTORE_OPEN_AND_CLOSE_THREADS_MAX));
+    return getOpenAndCloseThreadPool(maxThreads, threadNamePrefix);
+  }
+
+  protected ThreadPoolExecutor getStoreFileOpenAndCloseThreadPool(
+      final String threadNamePrefix) {
+    int numStores = Math.max(1, this.htableDescriptor.getFamilies().size());
+    int maxThreads = Math.max(1,
+        conf.getInt(HConstants.HSTORE_OPEN_AND_CLOSE_THREADS_MAX,
+            HConstants.DEFAULT_HSTORE_OPEN_AND_CLOSE_THREADS_MAX)
+            / numStores);
+    return getOpenAndCloseThreadPool(maxThreads, threadNamePrefix);
+  }
+
+  static ThreadPoolExecutor getOpenAndCloseThreadPool(int maxThreads,
+      final String threadNamePrefix) {
+    return Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
+      new ThreadFactory() {
+        private int count = 1;
+
+        public Thread newThread(Runnable r) {
+          return new Thread(r, threadNamePrefix + "-" + count++);
+        }
+      });
+  }
+
+   /**
+    * @return True if its worth doing a flush before we put up the close flag.
+    */
+  private boolean worthPreFlushing() {
+    return this.memstoreSize.get() >
+      this.conf.getLong("hbase.hregion.preclose.flush.size", 1024 * 1024 * 5);
+  }
+
+  //////////////////////////////////////////////////////////////////////////////
+  // HRegion accessors
+  //////////////////////////////////////////////////////////////////////////////
+
+  /** @return start key for region */
+  public byte [] getStartKey() {
+    return this.regionInfo.getStartKey();
+  }
+
+  /** @return end key for region */
+  public byte [] getEndKey() {
+    return this.regionInfo.getEndKey();
+  }
+
+  /** @return region id */
+  public long getRegionId() {
+    return this.regionInfo.getRegionId();
+  }
+
+  /** @return region name */
+  public byte [] getRegionName() {
+    return this.regionInfo.getRegionName();
+  }
+
+  /** @return region name as string for logging */
+  public String getRegionNameAsString() {
+    return this.regionInfo.getRegionNameAsString();
+  }
+
+  /** @return HTableDescriptor for this region */
+  public HTableDescriptor getTableDesc() {
+    return this.htableDescriptor;
+  }
+
+  /** @return HLog in use for this region */
+  public HLog getLog() {
+    return this.log;
+  }
+
+  /** @return Configuration object */
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+    /**
+   * A split takes the config from the parent region & passes it to the daughter
+   * region's constructor. If 'conf' was passed, you would end up using the HTD
+   * of the parent region in addition to the new daughter HTD. Pass 'baseConf'
+   * to the daughter regions to avoid this tricky dedupe problem.
+   * @return Configuration object
+   */
+  Configuration getBaseConf() {
+    return this.baseConf;
+  }
+
+  /** @return region directory Path */
+  public Path getRegionDir() {
+    return this.regiondir;
+  }
+
+  /**
+   * Computes the Path of the HRegion
+   *
+   * @param tabledir qualified path for table
+   * @param name ENCODED region name
+   * @return Path of HRegion directory
+   */
+  public static Path getRegionDir(final Path tabledir, final String name) {
+    return new Path(tabledir, name);
+  }
+
+  /** @return FileSystem being used by this region */
+  public FileSystem getFilesystem() {
+    return this.fs;
+  }
+
+  /** @return the last time the region was flushed */
+  public long getLastFlushTime() {
+    return this.lastFlushTime;
+  }
+
+  /** @return info about the last flushes  */
+  public List> getRecentFlushInfo() {
+    this.lock.readLock().lock();
+    List> ret = this.recentFlushes;
+    this.recentFlushes = new ArrayList>();
+    this.lock.readLock().unlock();
+    return ret;
+  }
+
+  //////////////////////////////////////////////////////////////////////////////
+  // HRegion maintenance.
+  //
+  // These methods are meant to be called periodically by the HRegionServer for
+  // upkeep.
+  //////////////////////////////////////////////////////////////////////////////
+
+  /** @return returns size of largest HStore. */
+  public long getLargestHStoreSize() {
+    long size = 0;
+    for (Store h: stores.values()) {
+      long storeSize = h.getSize();
+      if (storeSize > size) {
+        size = storeSize;
+      }
+    }
+    return size;
+  }
+
+  /*
+   * Do preparation for pending compaction.
+   * @throws IOException
+   */
+  void doRegionCompactionPrep() throws IOException {
+  }
+
+  /*
+   * Removes the temporary directory for this Store.
+   */
+  private void cleanupTmpDir() throws IOException {
+    FSUtils.deleteDirectory(this.fs, getTmpDir());
+  }
+
+  /**
+   * Get the temporary directory for this region. This directory
+   * will have its contents removed when the region is reopened.
+   */
+  Path getTmpDir() {
+    return getTmpDir(getRegionDir());
+  }
+
+  static Path getTmpDir(Path regionDir) {
+    return new Path(regionDir, REGION_TEMP_SUBDIR);
+  }
+
+  void triggerMajorCompaction() {
+    for (Store h: stores.values()) {
+      h.triggerMajorCompaction();
+    }
+  }
+
+  /**
+   * This is a helper function that compact all the stores synchronously
+   * It is used by utilities and testing
+   *
+   * @param majorCompaction True to force a major compaction regardless of thresholds
+   * @throws IOException e
+   */
+  public void compactStores(final boolean majorCompaction)
+  throws IOException {
+    if (majorCompaction) {
+      this.triggerMajorCompaction();
+    }
+    compactStores();
+  }
+
+  /**
+   * This is a helper function that compact all the stores synchronously
+   * It is used by utilities and testing
+   *
+   * @throws IOException e
+   */
+  public void compactStores() throws IOException {
+    for(Store s : getStores().values()) {
+      CompactionRequest cr = s.requestCompaction();
+      if(cr != null) {
+        try {
+          compact(cr);
+        } finally {
+          s.finishRequest(cr);
+        }
+      }
+    }
+  }
+
+  /*
+   * Called by compaction thread and after region is opened to compact the
+   * HStores if necessary.
+   *
+   * 
                This operation could block for a long time, so don't call it from a
+   * time-sensitive thread.
+   *
+   * Note that no locking is necessary at this level because compaction only
+   * conflicts with a region split, and that cannot happen because the region
+   * server does them sequentially and not in parallel.
+   *
+   * @param cr Compaction details, obtained by requestCompaction()
+   * @return whether the compaction completed
+   * @throws IOException e
+   */
+  public boolean compact(CompactionRequest cr)
+  throws IOException {
+    if (cr == null) {
+      return false;
+    }
+    if (this.closing.get() || this.closed.get()) {
+      LOG.debug("Skipping compaction on " + this + " because closing/closed");
+      return false;
+    }
+    Preconditions.checkArgument(cr.getHRegion().equals(this));
+    // block waiting for the lock for compaction
+    lock.readLock().lock();
+    MonitoredTask status = TaskMonitor.get().createStatus(
+        "Compacting " + cr.getStore() + " in " + this);
+    try {
+      if (this.closed.get()) {
+        LOG.debug("Skipping compaction on " + this + " because closed");
+        return false;
+      }
+      boolean decr = true;
+      try {
+        synchronized (writestate) {
+          if (writestate.writesEnabled) {
+            ++writestate.compacting;
+          } else {
+            String msg = "NOT compacting region " + this + ". Writes disabled.";
+            LOG.info(msg);
+            status.abort(msg);
+            decr = false;
+            return false;
+          }
+        }
+        LOG.info("Starting compaction on " + cr.getStore() + " in region "
+            + this + (cr.getCompactSelection().isOffPeakCompaction()?" as an off-peak compaction":""));
+        doRegionCompactionPrep();
+        try {
+          status.setStatus("Compacting store " + cr.getStore());
+          cr.getStore().compact(cr);
+        } catch (InterruptedIOException iioe) {
+          String msg = "compaction interrupted by user";
+          LOG.info(msg, iioe);
+          status.abort(msg);
+          return false;
+        }
+      } finally {
+        if (decr) {
+          synchronized (writestate) {
+            --writestate.compacting;
+            if (writestate.compacting <= 0) {
+              writestate.notifyAll();
+            }
+          }
+        }
+      }
+      status.markComplete("Compaction complete");
+      return true;
+    } finally {
+      status.cleanup();
+      lock.readLock().unlock();
+    }
+  }
+
+  /**
+   * Flush the cache.
+   *
+   * When this method is called the cache will be flushed unless:
+   * 
                  
+   *   
                1. the cache is empty
                  2. 
+   *   
                2. the region is closed.
                  3. 
+   *   
                3. a flush is already in progress
                  4. 
+   *   
                4. writes are disabled
                  5. 
+   * 
                
+   *
+   * 
                This method may block for some time, so it should not be called from a
+   * time-sensitive thread.
+   *
+   * @return true if the region needs compaction
+   *
+   * @throws IOException general io exceptions
+   * @throws DroppedSnapshotException Thrown when replay of hlog is required
+   * because a Snapshot was not properly persisted.
+   */
+  public boolean flushcache() throws IOException {
+    // fail-fast instead of waiting on the lock
+    if (this.closing.get()) {
+      LOG.debug("Skipping flush on " + this + " because closing");
+      return false;
+    }
+    MonitoredTask status = TaskMonitor.get().createStatus("Flushing " + this);
+    status.setStatus("Acquiring readlock on region");
+    // block waiting for the lock for flushing cache
+    lock.readLock().lock();
+    try {
+      if (this.closed.get()) {
+        LOG.debug("Skipping flush on " + this + " because closed");
+        status.abort("Skipped: closed");
+        return false;
+      }
+      if (coprocessorHost != null) {
+        status.setStatus("Running coprocessor pre-flush hooks");
+        coprocessorHost.preFlush();
+      }
+      if (numPutsWithoutWAL.get() > 0) {
+        numPutsWithoutWAL.set(0);
+        dataInMemoryWithoutWAL.set(0);
+      }
+      synchronized (writestate) {
+        if (!writestate.flushing && writestate.writesEnabled) {
+          this.writestate.flushing = true;
+        } else {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("NOT flushing memstore for region " + this
+                + ", flushing=" + writestate.flushing + ", writesEnabled="
+                + writestate.writesEnabled);
+          }
+          status.abort("Not flushing since "
+              + (writestate.flushing ? "already flushing"
+                  : "writes not enabled"));
+          return false;
+        }
+      }
+      try {
+        boolean result = internalFlushcache(status);
+
+        if (coprocessorHost != null) {
+          status.setStatus("Running post-flush coprocessor hooks");
+          coprocessorHost.postFlush();
+        }
+
+        status.markComplete("Flush successful");
+        return result;
+      } finally {
+        synchronized (writestate) {
+          writestate.flushing = false;
+          this.writestate.flushRequested = false;
+          writestate.notifyAll();
+        }
+      }
+    } finally {
+      lock.readLock().unlock();
+      status.cleanup();
+    }
+  }
+
+  /**
+   * Should the memstore be flushed now
+   */
+  boolean shouldFlush() {
+    if (flushCheckInterval <= 0) { //disabled
+      return false;
+    }
+    long now = EnvironmentEdgeManager.currentTimeMillis();
+    //if we flushed in the recent past, we don't need to do again now
+    if ((now - getLastFlushTime() < flushCheckInterval)) {
+      return false;
+    }
+    //since we didn't flush in the recent past, flush now if certain conditions
+    //are met. Return true on first such memstore hit.
+    for (Store s : this.getStores().values()) {
+      if (s.timeOfOldestEdit() < now - flushCheckInterval) {
+        // we have an old enough edit in the memstore, flush
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Flush the memstore.
+   *
+   * Flushing the memstore is a little tricky. We have a lot of updates in the
+   * memstore, all of which have also been written to the log. We need to
+   * write those updates in the memstore out to disk, while being able to
+   * process reads/writes as much as possible during the flush operation. Also,
+   * the log has to state clearly the point in time at which the memstore was
+   * flushed. (That way, during recovery, we know when we can rely on the
+   * on-disk flushed structures and when we have to recover the memstore from
+   * the log.)
+   *
+   * 
                So, we have a three-step process:
+   *
+   * 
                • A. Flush the memstore to the on-disk stores, noting the current
+   * sequence ID for the log.
                • 
+   *
+   * 
                • B. Write a FLUSHCACHE-COMPLETE message to the log, using the sequence
+   * ID that was current at the time of memstore-flush.
                  • 
+   *
+   * 
                • C. Get rid of the memstore structures that are now redundant, as
+   * they've been flushed to the on-disk HStores.
                  • 
+   * 
                
+   * 
                This method is protected, but can be accessed via several public
+   * routes.
+   *
+   * 
                 This method may block for some time.
+   * @param status
+   *
+   * @return true if the region needs compacting
+   *
+   * @throws IOException general io exceptions
+   * @throws DroppedSnapshotException Thrown when replay of hlog is required
+   * because a Snapshot was not properly persisted.
+   */
+  protected boolean internalFlushcache(MonitoredTask status)
+      throws IOException {
+    return internalFlushcache(this.log, -1, status);
+  }
+
+  /**
+   * @param wal Null if we're NOT to go via hlog/wal.
+   * @param myseqid The seqid to use if wal is null writing out
+   * flush file.
+   * @param status
+   * @return true if the region needs compacting
+   * @throws IOException
+   * @see #internalFlushcache(MonitoredTask)
+   */
+  protected boolean internalFlushcache(
+      final HLog wal, final long myseqid, MonitoredTask status)
+  throws IOException {
+    if (this.rsServices != null && this.rsServices.isAborted()) {
+      // Don't flush when server aborting, it's unsafe
+      throw new IOException("Aborting flush because server is abortted...");
+    }
+    final long startTime = EnvironmentEdgeManager.currentTimeMillis();
+    // Clear flush flag.
+    // Record latest flush time
+    this.lastFlushTime = startTime;
+    // If nothing to flush, return and avoid logging start/stop flush.
+    if (this.memstoreSize.get() <= 0) {
+      return false;
+    }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Started memstore flush for " + this +
+        ", current region memstore size " +
+        StringUtils.humanReadableInt(this.memstoreSize.get()) +
+        ((wal != null)? "": "; wal is null, using passed sequenceid=" + myseqid));
+    }
+
+    // Stop updates while we snapshot the memstore of all stores. We only have
+    // to do this for a moment.  Its quick.  The subsequent sequence id that
+    // goes into the HLog after we've flushed all these snapshots also goes
+    // into the info file that sits beside the flushed files.
+    // We also set the memstore size to zero here before we allow updates
+    // again so its value will represent the size of the updates received
+    // during the flush
+    long sequenceId = -1L;
+    long completeSequenceId = -1L;
+    MultiVersionConsistencyControl.WriteEntry w = null;
+
+    // We have to take a write lock during snapshot, or else a write could
+    // end up in both snapshot and memstore (makes it difficult to do atomic
+    // rows then)
+    status.setStatus("Obtaining lock to block concurrent updates");
+    // block waiting for the lock for internal flush
+    this.updatesLock.writeLock().lock();
+    long flushsize = this.memstoreSize.get();
+    status.setStatus("Preparing to flush by snapshotting stores");
+    List storeFlushers = new ArrayList(stores.size());
+    try {
+      // Record the mvcc for all transactions in progress.
+      w = mvcc.beginMemstoreInsert();
+      mvcc.advanceMemstore(w);
+
+      sequenceId = (wal == null)? myseqid:
+        wal.startCacheFlush(this.regionInfo.getEncodedNameAsBytes());
+      completeSequenceId = this.getCompleteCacheFlushSequenceId(sequenceId);
+
+      for (Store s : stores.values()) {
+        storeFlushers.add(s.getStoreFlusher(completeSequenceId));
+      }
+
+      // prepare flush (take a snapshot)
+      for (StoreFlusher flusher : storeFlushers) {
+        flusher.prepare();
+      }
+    } finally {
+      this.updatesLock.writeLock().unlock();
+    }
+    String s = "Finished snapshotting " + this +
+      ", commencing wait for mvcc, flushsize=" + flushsize;
+    status.setStatus(s);
+    LOG.debug(s);
+
+    // sync unflushed WAL changes when deferred log sync is enabled
+    // see HBASE-8208 for details
+    if (wal != null && isDeferredLogSyncEnabled()) {
+      wal.sync();
+    }
+
+    // wait for all in-progress transactions to commit to HLog before
+    // we can start the flush. This prevents
+    // uncommitted transactions from being written into HFiles.
+    // We have to block before we start the flush, otherwise keys that
+    // were removed via a rollbackMemstore could be written to Hfiles.
+    mvcc.waitForRead(w);
+
+    status.setStatus("Flushing stores");
+    LOG.debug("Finished snapshotting, commencing flushing stores");
+
+    // Any failure from here on out will be catastrophic requiring server
+    // restart so hlog content can be replayed and put back into the memstore.
+    // Otherwise, the snapshot content while backed up in the hlog, it will not
+    // be part of the current running servers state.
+    boolean compactionRequested = false;
+    try {
+      // A.  Flush memstore to all the HStores.
+      // Keep running vector of all store files that includes both old and the
+      // just-made new flush store file. The new flushed file is still in the
+      // tmp directory.
+
+      for (StoreFlusher flusher : storeFlushers) {
+        flusher.flushCache(status);
+      }
+
+      // Switch snapshot (in memstore) -> new hfile (thus causing
+      // all the store scanners to reset/reseek).
+      for (StoreFlusher flusher : storeFlushers) {
+        boolean needsCompaction = flusher.commit(status);
+        if (needsCompaction) {
+          compactionRequested = true;
+        }
+      }
+      storeFlushers.clear();
+
+      // Set down the memstore size by amount of flush.
+      this.addAndGetGlobalMemstoreSize(-flushsize);
+    } catch (Throwable t) {
+      // An exception here means that the snapshot was not persisted.
+      // The hlog needs to be replayed so its content is restored to memstore.
+      // Currently, only a server restart will do this.
+      // We used to only catch IOEs but its possible that we'd get other
+      // exceptions -- e.g. HBASE-659 was about an NPE -- so now we catch
+      // all and sundry.
+      if (wal != null) {
+        wal.abortCacheFlush(this.regionInfo.getEncodedNameAsBytes());
+      }
+      DroppedSnapshotException dse = new DroppedSnapshotException("region: " +
+          Bytes.toStringBinary(getRegionName()));
+      dse.initCause(t);
+      status.abort("Flush failed: " + StringUtils.stringifyException(t));
+      throw dse;
+    }
+
+    // If we get to here, the HStores have been written. If we get an
+    // error in completeCacheFlush it will release the lock it is holding
+
+    // B.  Write a FLUSHCACHE-COMPLETE message to the log.
+    //     This tells future readers that the HStores were emitted correctly,
+    //     and that all updates to the log for this regionName that have lower
+    //     log-sequence-ids can be safely ignored.
+    if (wal != null) {
+      wal.completeCacheFlush(this.regionInfo.getEncodedNameAsBytes(),
+        regionInfo.getTableName(), completeSequenceId,
+        this.getRegionInfo().isMetaRegion());
+    }
+
+    // C. Finally notify anyone waiting on memstore to clear:
+    // e.g. checkResources().
+    synchronized (this) {
+      notifyAll(); // FindBugs NN_NAKED_NOTIFY
+    }
+
+    long time = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+    long memstoresize = this.memstoreSize.get();
+    String msg = "Finished memstore flush of ~" +
+      StringUtils.humanReadableInt(flushsize) + "/" + flushsize +
+      ", currentsize=" +
+      StringUtils.humanReadableInt(memstoresize) + "/" + memstoresize +
+      " for region " + this + " in " + time + "ms, sequenceid=" + sequenceId +
+      ", compaction requested=" + compactionRequested +
+      ((wal == null)? "; wal=null": "");
+    LOG.info(msg);
+    status.setStatus(msg);
+    this.recentFlushes.add(new Pair(time/1000, flushsize));
+
+    return compactionRequested;
+  }
+
+   /**
+   * Get the sequence number to be associated with this cache flush. Used by
+   * TransactionalRegion to not complete pending transactions.
+   *
+   *
+   * @param currentSequenceId
+   * @return sequence id to complete the cache flush with
+   */
+  protected long getCompleteCacheFlushSequenceId(long currentSequenceId) {
+    return currentSequenceId;
+  }
+
+  //////////////////////////////////////////////////////////////////////////////
+  // get() methods for client use.
+  //////////////////////////////////////////////////////////////////////////////
+  /**
+   * Return all the data for the row that matches row exactly,
+   * or the one that immediately preceeds it, at or immediately before
+   * ts.
+   *
+   * @param row row key
+   * @return map of values
+   * @throws IOException
+   */
+  Result getClosestRowBefore(final byte [] row)
+  throws IOException{
+    return getClosestRowBefore(row, HConstants.CATALOG_FAMILY);
+  }
+
+  /**
+   * Return all the data for the row that matches row exactly,
+   * or the one that immediately preceeds it, at or immediately before
+   * ts.
+   *
+   * @param row row key
+   * @param family column family to find on
+   * @return map of values
+   * @throws IOException read exceptions
+   */
+  public Result getClosestRowBefore(final byte [] row, final byte [] family)
+  throws IOException {
+    if (coprocessorHost != null) {
+      Result result = new Result();
+      if (coprocessorHost.preGetClosestRowBefore(row, family, result)) {
+        return result;
+      }
+    }
+    // look across all the HStores for this region and determine what the
+    // closest key is across all column families, since the data may be sparse
+    checkRow(row, "getClosestRowBefore");
+    startRegionOperation();
+    this.readRequestsCount.increment();
+    this.opMetrics.setReadRequestCountMetrics(this.readRequestsCount.get());   
+    try {
+      Store store = getStore(family);
+      // get the closest key. (HStore.getRowKeyAtOrBefore can return null)
+      KeyValue key = store.getRowKeyAtOrBefore(row);
+      Result result = null;
+      if (key != null) {
+        Get get = new Get(key.getRow());
+        get.addFamily(family);
+        result = get(get, null);
+      }
+      if (coprocessorHost != null) {
+        coprocessorHost.postGetClosestRowBefore(row, family, result);
+      }
+      return result;
+    } finally {
+      closeRegionOperation();
+    }
+  }
+
+  /**
+   * Return an iterator that scans over the HRegion, returning the indicated
+   * columns and rows specified by the {@link Scan}.
+   * 
                
+   * This Iterator must be closed by the caller.
+   *
+   * @param scan configured {@link Scan}
+   * @return RegionScanner
+   * @throws IOException read exceptions
+   */
+  public RegionScanner getScanner(Scan scan) throws IOException {
+   return getScanner(scan, null);
+  }
+
+  void prepareScanner(Scan scan) throws IOException {
+    if(!scan.hasFamilies()) {
+      // Adding all families to scanner
+      for(byte[] family: this.htableDescriptor.getFamiliesKeys()){
+        scan.addFamily(family);
+      }
+    }
+  }
+
+  protected RegionScanner getScanner(Scan scan,
+      List additionalScanners) throws IOException {
+    startRegionOperation();
+    try {
+      // Verify families are all valid
+      prepareScanner(scan);
+      if(scan.hasFamilies()) {
+        for(byte [] family : scan.getFamilyMap().keySet()) {
+          checkFamily(family);
+        }
+      }
+      return instantiateRegionScanner(scan, additionalScanners);
+    } finally {
+      closeRegionOperation();
+    }
+  }
+
+  protected RegionScanner instantiateRegionScanner(Scan scan,
+      List additionalScanners) throws IOException {
+    return new RegionScannerImpl(scan, additionalScanners, this);
+  }
+
+  /*
+   * @param delete The passed delete is modified by this method. WARNING!
+   */
+  private void prepareDelete(Delete delete) throws IOException {
+    // Check to see if this is a deleteRow insert
+    if(delete.getFamilyMap().isEmpty()){
+      for(byte [] family : this.htableDescriptor.getFamiliesKeys()){
+        // Don't eat the timestamp
+        delete.deleteFamily(family, delete.getTimeStamp());
+      }
+    } else {
+      for(byte [] family : delete.getFamilyMap().keySet()) {
+        if(family == null) {
+          throw new NoSuchColumnFamilyException("Empty family is invalid");
+        }
+        checkFamily(family);
+      }
+    }
+  }
+
+  //////////////////////////////////////////////////////////////////////////////
+  // set() methods for client use.
+  //////////////////////////////////////////////////////////////////////////////
+
+  /**
+   * @param delete delete object
+   * @param writeToWAL append to the write ahead lock or not
+   * @throws IOException read exceptions
+   */
+  public void delete(Delete delete, boolean writeToWAL)
+  throws IOException {
+    delete(delete, null, writeToWAL);
+  }
+
+  /**
+   * @param delete delete object
+   * @param lockid existing lock id, or null for grab a lock
+   * @param writeToWAL append to the write ahead lock or not
+   * @throws IOException read exceptions
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
+   */
+  public void delete(Delete delete, Integer lockid, boolean writeToWAL)
+  throws IOException {
+    checkReadOnly();
+    checkResources();
+    Integer lid = null;
+    startRegionOperation();
+    this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
+    try {
+      byte [] row = delete.getRow();
+      // If we did not pass an existing row lock, obtain a new one
+      lid = getLock(lockid, row, true);
+
+      try {
+        // All edits for the given row (across all column families) must happen atomically.
+        doBatchMutate(delete, lid);
+      } finally {
+        if(lockid == null) releaseRowLock(lid);
+      }
+    } finally {
+      closeRegionOperation();
+    }
+  }
+
+  /**
+   * This is used only by unit tests. Not required to be a public API.
+   * @param familyMap map of family to edits for the given family.
+   * @param writeToWAL
+   * @throws IOException
+   */
+  void delete(Map> familyMap, UUID clusterId,
+      boolean writeToWAL) throws IOException {
+    Delete delete = new Delete(new byte[0]);
+    delete.setFamilyMap(familyMap);
+    delete.setClusterId(clusterId);
+    delete.setWriteToWAL(writeToWAL);
+    doBatchMutate(delete, null);
+  }
+
+  /**
+   * Setup correct timestamps in the KVs in Delete object.
+   * Caller should have the row and region locks.
+   * @param familyMap
+   * @param now
+   * @throws IOException
+   */
+  private void prepareDeleteTimestamps(Map> familyMap, byte[] byteNow)
+      throws IOException {
+    for (Map.Entry> e : familyMap.entrySet()) {
+
+      byte[] family = e.getKey();
+      List kvs = e.getValue();
+      Map kvCount = new TreeMap(Bytes.BYTES_COMPARATOR);
+
+      for (KeyValue kv: kvs) {
+        //  Check if time is LATEST, change to time of most recent addition if so
+        //  This is expensive.
+        if (kv.isLatestTimestamp() && kv.isDeleteType()) {
+          byte[] qual = kv.getQualifier();
+          if (qual == null) qual = HConstants.EMPTY_BYTE_ARRAY;
+
+          Integer count = kvCount.get(qual);
+          if (count == null) {
+            kvCount.put(qual, 1);
+          } else {
+            kvCount.put(qual, count + 1);
+          }
+          count = kvCount.get(qual);
+
+          Get get = new Get(kv.getRow());
+          get.setMaxVersions(count);
+          get.addColumn(family, qual);
+
+          List result = get(get, false);
+
+          if (result.size() < count) {
+            // Nothing to delete
+            kv.updateLatestStamp(byteNow);
+            continue;
+          }
+          if (result.size() > count) {
+            throw new RuntimeException("Unexpected size: " + result.size());
+          }
+          KeyValue getkv = result.get(count - 1);
+          Bytes.putBytes(kv.getBuffer(), kv.getTimestampOffset(),
+              getkv.getBuffer(), getkv.getTimestampOffset(), Bytes.SIZEOF_LONG);
+        } else {
+          kv.updateLatestStamp(byteNow);
+        }
+      }
+    }
+  }
+
+  /**
+   * @param put
+   * @throws IOException
+   */
+  public void put(Put put) throws IOException {
+    this.put(put, null, put.getWriteToWAL());
+  }
+
+  /**
+   * @param put
+   * @param writeToWAL
+   * @throws IOException
+   */
+  public void put(Put put, boolean writeToWAL) throws IOException {
+    this.put(put, null, writeToWAL);
+  }
+
+  /**
+   * @param put
+   * @param lockid
+   * @throws IOException
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
+   */
+  public void put(Put put, Integer lockid) throws IOException {
+    this.put(put, lockid, put.getWriteToWAL());
+  }
+
+
+
+  /**
+   * @param put
+   * @param lockid
+   * @param writeToWAL
+   * @throws IOException
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
+   */
+  public void put(Put put, Integer lockid, boolean writeToWAL)
+  throws IOException {
+    checkReadOnly();
+
+    // Do a rough check that we have resources to accept a write.  The check is
+    // 'rough' in that between the resource check and the call to obtain a
+    // read lock, resources may run out.  For now, the thought is that this
+    // will be extremely rare; we'll deal with it when it happens.
+    checkResources();
+    startRegionOperation();
+    this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
+    try {
+      // We obtain a per-row lock, so other clients will block while one client
+      // performs an update. The read lock is released by the client calling
+      // #commit or #abort or if the HRegionServer lease on the lock expires.
+      // See HRegionServer#RegionListener for how the expire on HRegionServer
+      // invokes a HRegion#abort.
+      byte [] row = put.getRow();
+      // If we did not pass an existing row lock, obtain a new one
+      Integer lid = getLock(lockid, row, true);
+
+      try {
+        // All edits for the given row (across all column families) must happen atomically.
+        doBatchMutate(put, lid);
+      } finally {
+        if(lockid == null) releaseRowLock(lid);
+      }
+    } finally {
+      closeRegionOperation();
+    }
+  }
+
+  /**
+   * Struct-like class that tracks the progress of a batch operation,
+   * accumulating status codes and tracking the index at which processing
+   * is proceeding.
+   */
+  private static class BatchOperationInProgress {
+    T[] operations;
+    int nextIndexToProcess = 0;
+    OperationStatus[] retCodeDetails;
+    WALEdit[] walEditsFromCoprocessors;
+
+    public BatchOperationInProgress(T[] operations) {
+      this.operations = operations;
+      this.retCodeDetails = new OperationStatus[operations.length];
+      this.walEditsFromCoprocessors = new WALEdit[operations.length];
+      Arrays.fill(this.retCodeDetails, OperationStatus.NOT_RUN);
+    }
+
+    public boolean isDone() {
+      return nextIndexToProcess == operations.length;
+    }
+  }
+
+  /**
+   * Perform a batch put with no pre-specified locks
+   * @see HRegion#batchMutate(Pair[])
+   */
+  public OperationStatus[] put(Put[] puts) throws IOException {
+    @SuppressWarnings("unchecked")
+    Pair putsAndLocks[] = new Pair[puts.length];
+
+    for (int i = 0; i < puts.length; i++) {
+      putsAndLocks[i] = new Pair(puts[i], null);
+    }
+    return batchMutate(putsAndLocks);
+  }
+
+  /**
+   * Perform a batch of puts.
+   * @param putsAndLocks
+   *          the list of puts paired with their requested lock IDs.
+   * @return an array of OperationStatus which internally contains the OperationStatusCode and the
+   *         exceptionMessage if any.
+   * @throws IOException
+   * @deprecated Instead use {@link HRegion#batchMutate(Pair[])}
+   */
+  @Deprecated
+  public OperationStatus[] put(Pair[] putsAndLocks) throws IOException {
+    Pair[] mutationsAndLocks = new Pair[putsAndLocks.length];
+    System.arraycopy(putsAndLocks, 0, mutationsAndLocks, 0, putsAndLocks.length);
+    return batchMutate(mutationsAndLocks);
+  }
+
+  /**
+   * Perform a batch of mutations.
+   * It supports only Put and Delete mutations and will ignore other types passed.
+   * @param mutationsAndLocks
+   *          the list of mutations paired with their requested lock IDs.
+   * @return an array of OperationStatus which internally contains the
+   *         OperationStatusCode and the exceptionMessage if any.
+   * @throws IOException
+   */
+  public OperationStatus[] batchMutate(
+      Pair[] mutationsAndLocks) throws IOException {
+    BatchOperationInProgress> batchOp =
+      new BatchOperationInProgress>(mutationsAndLocks);
+
+    boolean initialized = false;
+
+    while (!batchOp.isDone()) {
+      checkReadOnly();
+      checkResources();
+
+      long newSize;
+      startRegionOperation();
+
+      if (coprocessorHost != null) {
+        coprocessorHost.postStartRegionOperation();
+      }
+
+      try {
+        if (!initialized) {
+          this.writeRequestsCount.increment();
+          this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
+          doPreMutationHook(batchOp);
+          initialized = true;
+        }
+        long addedSize = doMiniBatchMutation(batchOp);
+        newSize = this.addAndGetGlobalMemstoreSize(addedSize);
+      } finally {
+        closeRegionOperation();
+        if (coprocessorHost != null) {
+          coprocessorHost.postCloseRegionOperation();
+        }
+      }
+      if (isFlushSize(newSize)) {
+        requestFlush();
+      }
+    }
+    return batchOp.retCodeDetails;
+  }
+
+  /**
+   * Not to be used by external users. This is only for the IndexRegionObserver.
+   * @param mutationsAndLocks
+   * @return
+   * @throws IOException
+   */
+  public OperationStatus[] batchMutateForIndex(Pair[] mutationsAndLocks)
+      throws IOException {
+    BatchOperationInProgress> batchOp =
+        new BatchOperationInProgress>(mutationsAndLocks);
+
+    boolean initialized = false;
+
+    while (!batchOp.isDone()) {
+      long newSize;
+
+      if (coprocessorHost != null) {
+        coprocessorHost.postStartRegionOperation();
+      }
+
+      try {
+        if (!initialized) {
+          this.writeRequestsCount.increment();
+          doPreMutationHook(batchOp);
+          initialized = true;
+        }
+        long addedSize = doMiniBatchMutation(batchOp);
+        newSize = this.addAndGetGlobalMemstoreSize(addedSize);
+      } finally {
+        if (coprocessorHost != null) {
+          coprocessorHost.postCloseRegionOperation();
+        }
+      }
+      if (isFlushSize(newSize)) {
+        requestFlush();
+      }
+    }
+    return batchOp.retCodeDetails;
+  }
+
+  private void doPreMutationHook(BatchOperationInProgress> batchOp)
+      throws IOException {
+    /* Run coprocessor pre hook outside of locks to avoid deadlock */
+    WALEdit walEdit = new WALEdit();
+    if (coprocessorHost != null) {
+      for (int i = 0; i < batchOp.operations.length; i++) {
+        Pair nextPair = batchOp.operations[i];
+        Mutation m = nextPair.getFirst();
+        if (m instanceof Put) {
+          if (coprocessorHost.prePut((Put) m, walEdit, m.getWriteToWAL())) {
+            // pre hook says skip this Put
+            // mark as success and skip in doMiniBatchMutation
+            batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
+          }
+        } else if (m instanceof Delete) {
+          if (coprocessorHost.preDelete((Delete) m, walEdit, m.getWriteToWAL())) {
+            // pre hook says skip this Delete
+            // mark as success and skip in doMiniBatchMutation
+            batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
+          }
+        } else {
+          // In case of passing Append mutations along with the Puts and Deletes in batchMutate
+          // mark the operation return code as failure so that it will not be considered in
+          // the doMiniBatchMutation
+          batchOp.retCodeDetails[i] = new OperationStatus(OperationStatusCode.FAILURE,
+              "Put/Delete mutations only supported in batchMutate() now");
+        }
+        if (!walEdit.isEmpty()) {
+          batchOp.walEditsFromCoprocessors[i] = walEdit;
+          walEdit = new WALEdit();
+        }
+      }
+    }
+  }
+
+  // The mutation will be either a Put or Delete.
+  @SuppressWarnings("unchecked")
+  private long doMiniBatchMutation(
+      BatchOperationInProgress> batchOp) throws IOException {
+
+    // The set of columnFamilies first seen for Put.
+    Set putsCfSet = null;
+    // variable to note if all Put items are for the same CF -- metrics related
+    boolean putsCfSetConsistent = true;
+    // The set of columnFamilies first seen for Delete.
+    Set deletesCfSet = null;
+    // variable to note if all Delete items are for the same CF -- metrics related
+    boolean deletesCfSetConsistent = true;
+    long startTimeMs = EnvironmentEdgeManager.currentTimeMillis();
+
+    WALEdit walEdit = new WALEdit();
+
+    MultiVersionConsistencyControl.WriteEntry w = null;
+    long txid = 0;
+    boolean walSyncSuccessful = false;
+    boolean locked = false;
+
+    /** Keep track of the locks we hold so we can release them in finally clause */
+    List acquiredLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);
+    // reference family maps directly so coprocessors can mutate them if desired
+    Map>[] familyMaps = new Map[batchOp.operations.length];
+    // We try to set up a batch in the range [firstIndex,lastIndexExclusive)
+    int firstIndex = batchOp.nextIndexToProcess;
+    int lastIndexExclusive = firstIndex;
+    boolean success = false;
+    int noOfPuts = 0, noOfDeletes = 0;
+    try {
+      // ------------------------------------
+      // STEP 1. Try to acquire as many locks as we can, and ensure
+      // we acquire at least one.
+      // ----------------------------------
+      int numReadyToWrite = 0;
+      long now = EnvironmentEdgeManager.currentTimeMillis();
+      while (lastIndexExclusive < batchOp.operations.length) {
+        Pair nextPair = batchOp.operations[lastIndexExclusive];
+        Mutation mutation = nextPair.getFirst();
+        Integer providedLockId = nextPair.getSecond();
+
+        Map> familyMap = mutation.getFamilyMap();
+        // store the family map reference to allow for mutations
+        familyMaps[lastIndexExclusive] = familyMap;
+
+        // skip anything that "ran" already
+        if (batchOp.retCodeDetails[lastIndexExclusive].getOperationStatusCode()
+            != OperationStatusCode.NOT_RUN) {
+          lastIndexExclusive++;
+          continue;
+        }
+
+        try {
+          if (mutation instanceof Put) {
+            checkFamilies(familyMap.keySet());
+            checkTimestamps(mutation.getFamilyMap(), now);
+          } else {
+            prepareDelete((Delete) mutation);
+          }
+        } catch (NoSuchColumnFamilyException nscf) {
+          LOG.warn("No such column family in batch mutation", nscf);
+          batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(
+              OperationStatusCode.BAD_FAMILY, nscf.getMessage());
+          lastIndexExclusive++;
+          continue;
+        } catch (DoNotRetryIOException fsce) {
+          // The only thing that throws a generic DoNotRetryIOException in the above code is
+          // checkTimestamps so that DoNotRetryIOException means that timestamps were invalid.
+          // If more checks are added, be sure to revisit this assumption.
+          LOG.warn("Batch Mutation did not pass sanity check", fsce);
+          batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(
+              OperationStatusCode.SANITY_CHECK_FAILURE, fsce.getMessage());
+          lastIndexExclusive++;
+          continue;
+        }
+        // If we haven't got any rows in our batch, we should block to
+        // get the next one.
+        boolean shouldBlock = numReadyToWrite == 0;
+        Integer acquiredLockId = null;
+        try {
+          acquiredLockId = getLock(providedLockId, mutation.getRow(),
+              shouldBlock);
+        } catch (IOException ioe) {
+          LOG.warn("Failed getting lock in batch put, row="
+                  + Bytes.toStringBinary(mutation.getRow()), ioe);
+        }
+        if (acquiredLockId == null) {
+          // We failed to grab another lock
+          assert !shouldBlock : "Should never fail to get lock when blocking";
+          break; // stop acquiring more rows for this batch
+        }
+        if (providedLockId == null) {
+          acquiredLocks.add(acquiredLockId);
+        }
+        lastIndexExclusive++;
+        numReadyToWrite++;
+
+        if (mutation instanceof Put) {
+          // If Column Families stay consistent through out all of the
+          // individual puts then metrics can be reported as a mutliput across
+          // column families in the first put.
+          if (putsCfSet == null) {
+            putsCfSet = mutation.getFamilyMap().keySet();
+          } else {
+            putsCfSetConsistent = putsCfSetConsistent
+                && mutation.getFamilyMap().keySet().equals(putsCfSet);
+          }
+        } else {
+          if (deletesCfSet == null) {
+            deletesCfSet = mutation.getFamilyMap().keySet();
+          } else {
+            deletesCfSetConsistent = deletesCfSetConsistent
+                && mutation.getFamilyMap().keySet().equals(deletesCfSet);
+          }
+        }
+      }
+
+      // we should record the timestamp only after we have acquired the rowLock,
+      // otherwise, newer puts/deletes are not guaranteed to have a newer timestamp
+      now = EnvironmentEdgeManager.currentTimeMillis();
+      byte[] byteNow = Bytes.toBytes(now);
+
+      // Nothing to put/delete -- an exception in the above such as NoSuchColumnFamily?
+      if (numReadyToWrite <= 0) return 0L;
+
+      // We've now grabbed as many mutations off the list as we can
+
+      // ------------------------------------
+      // STEP 2. Update any LATEST_TIMESTAMP timestamps
+      // ----------------------------------
+      for (int i = firstIndex; i < lastIndexExclusive; i++) {
+        // skip invalid
+        if (batchOp.retCodeDetails[i].getOperationStatusCode()
+            != OperationStatusCode.NOT_RUN) continue;
+        Mutation mutation = batchOp.operations[i].getFirst();
+        if (mutation instanceof Put) {
+          updateKVTimestamps(familyMaps[i].values(), byteNow);
+          noOfPuts++;
+        } else {
+          prepareDeleteTimestamps(familyMaps[i], byteNow);
+          noOfDeletes++;
+        }
+      }
+
+      lock(this.updatesLock.readLock(), numReadyToWrite);
+      locked = true;
+
+      //
+      // ------------------------------------
+      // Acquire the latest mvcc number
+      // ----------------------------------
+      w = mvcc.beginMemstoreInsert();
+
+      // ------------------------------------------------
+      // Call the coprocessor hook to do the actual put.
+      // Here the walDdit is updated with the actual timestamp
+      // for the kv.
+      // This hook will be called for all the puts that are currently
+      // acquired the locks
+      // -------------------------------------------------
+      if (coprocessorHost != null) {
+        List> mutationVsBatchOp =
+            new ArrayList>(lastIndexExclusive - firstIndex);
+        for (int i = firstIndex; i < lastIndexExclusive; i++) {
+          mutationVsBatchOp.add(new Pair(batchOp.operations[i]
+              .getFirst(), batchOp.retCodeDetails[i]));
+        }
+        if (coprocessorHost.preBatchMutate(mutationVsBatchOp, walEdit)) {
+          return 0;
+        }
+        boolean fullBatchFailed = true;
+        for (int i = 0; i < mutationVsBatchOp.size(); i++) {
+          OperationStatus opStatus = mutationVsBatchOp.get(i).getSecond();
+          if (opStatus.getOperationStatusCode() != OperationStatusCode.NOT_RUN) {
+            batchOp.retCodeDetails[firstIndex + i] = mutationVsBatchOp.get(i).getSecond();
+          } else {
+            fullBatchFailed = false;
+          }
+        }
+        if (fullBatchFailed) {
+          return 0;
+        }
+      }
+
+      // ------------------------------------
+      // STEP 3. Write back to memstore
+      // Write to memstore. It is ok to write to memstore
+      // first without updating the HLog because we do not roll
+      // forward the memstore MVCC. The MVCC will be moved up when
+      // the complete operation is done. These changes are not yet
+      // visible to scanners till we update the MVCC. The MVCC is
+      // moved only when the sync is complete.
+      // ----------------------------------
+      long addedSize = 0;
+      for (int i = firstIndex; i < lastIndexExclusive; i++) {
+        if (batchOp.retCodeDetails[i].getOperationStatusCode()
+            != OperationStatusCode.NOT_RUN) {
+          continue;
+        }
+        addedSize += applyFamilyMapToMemstore(familyMaps[i], w);
+      }
+
+      // ------------------------------------
+      // STEP 4. Build WAL edit
+      // ----------------------------------
+      Durability durability = Durability.USE_DEFAULT;
+
+      for (int i = firstIndex; i < lastIndexExclusive; i++) {
+        // Skip puts that were determined to be invalid during preprocessing
+        if (batchOp.retCodeDetails[i].getOperationStatusCode()
+            != OperationStatusCode.NOT_RUN) {
+          continue;
+        }
+        batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
+
+        Mutation m = batchOp.operations[i].getFirst();
+        Durability tmpDur = m.getDurability(); 
+        if (tmpDur.ordinal() > durability.ordinal()) {
+          durability = tmpDur;
+        }
+        if (tmpDur == Durability.SKIP_WAL) {
+          if (m instanceof Put) {
+            recordPutWithoutWal(m.getFamilyMap());
+          }
+          continue;
+        }
+
+        // Add WAL edits by CP
+        WALEdit fromCP = batchOp.walEditsFromCoprocessors[i];
+        if (fromCP != null) {
+          for (KeyValue kv : fromCP.getKeyValues()) {
+            walEdit.add(kv);
+          }
+        }
+        addFamilyMapToWALEdit(familyMaps[i], walEdit);
+      }
+
+      // -------------------------
+      // STEP 5. Append the edit to WAL. Do not sync wal.
+      // -------------------------
+      Mutation first = batchOp.operations[firstIndex].getFirst();
+      txid = this.log.appendNoSync(regionInfo, this.htableDescriptor.getName(),
+               walEdit, first.getClusterId(), now, this.htableDescriptor);
+
+      // -------------------------------
+      // STEP 6. Release row locks, etc.
+      // -------------------------------
+      if (locked) {
+        this.updatesLock.readLock().unlock();
+        locked = false;
+      }
+      if (acquiredLocks != null) {
+        for (Integer toRelease : acquiredLocks) {
+          releaseRowLock(toRelease);
+        }
+        acquiredLocks = null;
+      }
+      // -------------------------
+      // STEP 7. Sync wal.
+      // -------------------------
+      if (walEdit.size() > 0) {
+        syncOrDefer(txid, durability);
+      }
+      walSyncSuccessful = true;
+
+      // hook to complete the actual put
+      if (coprocessorHost != null) {
+        List mutations = new ArrayList();
+        for (int i = firstIndex; i < lastIndexExclusive; i++) {
+          // only for successful puts
+          if (batchOp.retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) {
+            continue;
+          }
+          Mutation m = batchOp.operations[i].getFirst();
+          mutations.add(m);
+        }
+        coprocessorHost.postBatchMutate(mutations, walEdit);
+      }
+      
+      // ------------------------------------------------------------------
+      // STEP 8. Advance mvcc. This will make this put visible to scanners and getters.
+      // ------------------------------------------------------------------
+      if (w != null) {
+        mvcc.completeMemstoreInsert(w);
+        w = null;
+      }
+
+      // ------------------------------------
+      // STEP 9. Run coprocessor post hooks. This should be done after the wal is
+      // synced so that the coprocessor contract is adhered to.
+      // ------------------------------------
+      if (coprocessorHost != null) {
+        for (int i = firstIndex; i < lastIndexExclusive; i++) {
+          // only for successful puts
+          if (batchOp.retCodeDetails[i].getOperationStatusCode()
+              != OperationStatusCode.SUCCESS) {
+            continue;
+          }
+          Mutation m = batchOp.operations[i].getFirst();
+          if (m instanceof Put) {
+            coprocessorHost.postPut((Put) m, walEdit, m.getWriteToWAL());
+          } else {
+            coprocessorHost.postDelete((Delete) m, walEdit, m.getWriteToWAL());
+          }
+        }
+      }
+      success = true;
+      return addedSize;
+    } finally {
+
+      // if the wal sync was unsuccessful, remove keys from memstore
+      if (!walSyncSuccessful) {
+        rollbackMemstore(batchOp, familyMaps, firstIndex, lastIndexExclusive);
+      }
+      if (w != null) mvcc.completeMemstoreInsert(w);
+
+      if (locked) {
+        this.updatesLock.readLock().unlock();
+      }
+
+      if (acquiredLocks != null) {
+        for (Integer toRelease : acquiredLocks) {
+          releaseRowLock(toRelease);
+        }
+      }
+
+      // call the coprocessor hook to do to any finalization steps
+      // after the put is done
+      if (coprocessorHost != null) {
+        List mutations = new ArrayList();
+        for (int i = firstIndex; i < lastIndexExclusive; i++) {
+          // only for successful puts
+          if (batchOp.retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) {
+            continue;
+          }
+          Mutation m = batchOp.operations[i].getFirst();
+          mutations.add(m);
+        }
+        coprocessorHost.postCompleteBatchMutate(mutations);
+      }
+
+      // do after lock
+      final long netTimeMs = EnvironmentEdgeManager.currentTimeMillis()- startTimeMs;
+
+      // See if the column families were consistent through the whole thing.
+      // if they were then keep them. If they were not then pass a null.
+      // null will be treated as unknown.
+      // Total time taken might be involving Puts and Deletes.
+      // Split the time for puts and deletes based on the total number of Puts and Deletes.
+      long timeTakenForPuts = 0;
+      if (noOfPuts > 0) {
+        // There were some Puts in the batch.
+        double noOfMutations = noOfPuts + noOfDeletes;
+        timeTakenForPuts = (long) (netTimeMs * (noOfPuts / noOfMutations));
+        final Set keptCfs = putsCfSetConsistent ? putsCfSet : null;
+        this.opMetrics.updateMultiPutMetrics(keptCfs, timeTakenForPuts);
+      }
+      if (noOfDeletes > 0) {
+        // There were some Deletes in the batch.
+        final Set keptCfs = deletesCfSetConsistent ? deletesCfSet : null;
+        this.opMetrics.updateMultiDeleteMetrics(keptCfs, netTimeMs - timeTakenForPuts);
+      }
+      if (!success) {
+        for (int i = firstIndex; i < lastIndexExclusive; i++) {
+          if (batchOp.retCodeDetails[i].getOperationStatusCode() == OperationStatusCode.NOT_RUN) {
+            batchOp.retCodeDetails[i] = OperationStatus.FAILURE;
+          }
+        }
+      }
+      batchOp.nextIndexToProcess = lastIndexExclusive;
+    }
+  }
+
+  //TODO, Think that gets/puts and deletes should be refactored a bit so that
+  //the getting of the lock happens before, so that you would just pass it into
+  //the methods. So in the case of checkAndMutate you could just do lockRow,
+  //get, put, unlockRow or something
+ /**
+  *
+  * @param row
+  * @param family
+  * @param qualifier
+  * @param compareOp
+  * @param comparator
+  * @param writeToWAL
+  * @throws IOException
+  * @return true if the new put was execute, false otherwise
+  */
+ public boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
+     CompareOp compareOp, WritableByteArrayComparable comparator, Writable w,
+     boolean writeToWAL)
+ throws IOException {
+   return checkAndMutate(row, family, qualifier, compareOp, comparator, w, null, writeToWAL);
+ }
+  
+  /**
+   *
+   * @param row
+   * @param family
+   * @param qualifier
+   * @param compareOp
+   * @param comparator
+   * @param lockId
+   * @param writeToWAL
+   * @throws IOException
+   * @return true if the new put was execute, false otherwise
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
+   */
+  public boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
+      CompareOp compareOp, WritableByteArrayComparable comparator, Writable w,
+      Integer lockId, boolean writeToWAL)
+  throws IOException{
+    checkReadOnly();
+    //TODO, add check for value length or maybe even better move this to the
+    //client if this becomes a global setting
+    checkResources();
+    boolean isPut = w instanceof Put;
+    if (!isPut && !(w instanceof Delete))
+      throw new DoNotRetryIOException("Action must be Put or Delete");
+    Row r = (Row)w;
+    if (!Bytes.equals(row, r.getRow())) {
+      throw new DoNotRetryIOException("Action's getRow must match the passed row");
+    }
+
+    startRegionOperation();
+    this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
+    try {
+      RowLock lock = isPut ? ((Put)w).getRowLock() : ((Delete)w).getRowLock();
+      Get get = new Get(row, lock);
+      checkFamily(family);
+      get.addColumn(family, qualifier);
+
+      // Lock row
+      Integer lid = getLock(lockId, get.getRow(), true);
+      // wait for all previous transactions to complete (with lock held)
+      mvcc.completeMemstoreInsert(mvcc.beginMemstoreInsert());
+      List result = new ArrayList();
+      try {
+        result = get(get, false);
+
+        boolean valueIsNull = comparator.getValue() == null ||
+          comparator.getValue().length == 0;
+        boolean matches = false;
+        if (result.size() == 0 && valueIsNull) {
+          matches = true;
+        } else if (result.size() > 0 && result.get(0).getValue().length == 0 &&
+            valueIsNull) {
+          matches = true;
+        } else if (result.size() == 1 && !valueIsNull) {
+          KeyValue kv = result.get(0);
+          int compareResult = comparator.compareTo(kv.getBuffer(),
+              kv.getValueOffset(), kv.getValueLength());
+          switch (compareOp) {
+          case LESS:
+            matches = compareResult <= 0;
+            break;
+          case LESS_OR_EQUAL:
+            matches = compareResult < 0;
+            break;
+          case EQUAL:
+            matches = compareResult == 0;
+            break;
+          case NOT_EQUAL:
+            matches = compareResult != 0;
+            break;
+          case GREATER_OR_EQUAL:
+            matches = compareResult > 0;
+            break;
+          case GREATER:
+            matches = compareResult >= 0;
+            break;
+          default:
+            throw new RuntimeException("Unknown Compare op " + compareOp.name());
+          }
+        }
+        //If matches put the new put or delete the new delete
+        if (matches) {
+          // All edits for the given row (across all column families) must
+          // happen atomically.
+          //
+          // Using default cluster id, as this can only happen in the
+          // originating cluster. A slave cluster receives the result as a Put
+          // or Delete
+          doBatchMutate((Mutation) w, lid);
+          return true;
+        }
+        return false;
+      } finally {
+        if(lockId == null) releaseRowLock(lid);
+      }
+    } finally {
+      closeRegionOperation();
+    }
+  }
+
+  private void doBatchMutate(Mutation mutation, Integer lid) throws IOException,
+      DoNotRetryIOException {
+    Pair mutateWithLocks[] = new Pair[1];
+    mutateWithLocks[0] = new Pair(mutation, lid);
+    OperationStatus[] batchMutate = this.batchMutate(mutateWithLocks);
+    if (batchMutate[0].getOperationStatusCode().equals(OperationStatusCode.SANITY_CHECK_FAILURE)) {
+      throw new DoNotRetryIOException(batchMutate[0].getExceptionMsg());
+    }
+  }
+
+  /**
+   * Complete taking the snapshot on the region. Writes the region info and adds references to the
+   * working snapshot directory.
+   *
+   * TODO for api consistency, consider adding another version with no {@link ForeignExceptionSnare}
+   * arg.  (In the future other cancellable HRegion methods could eventually add a
+   * {@link ForeignExceptionSnare}, or we could do something fancier).
+   *
+   * @param desc snasphot description object
+   * @param exnSnare ForeignExceptionSnare that captures external exeptions in case we need to
+   *   bail out.  This is allowed to be null and will just be ignored in that case.
+   * @throws IOException if there is an external or internal error causing the snapshot to fail
+   */
+  public void addRegionToSnapshot(SnapshotDescription desc,
+      ForeignExceptionSnare exnSnare) throws IOException {
+    // This should be "fast" since we don't rewrite store files but instead
+    // back up the store files by creating a reference
+    Path rootDir = FSUtils.getRootDir(this.rsServices.getConfiguration());
+    Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(desc, rootDir,
+      regionInfo.getEncodedName());
+
+    // 1. dump region meta info into the snapshot directory
+    LOG.debug("Storing region-info for snapshot.");
+    checkRegioninfoOnFilesystem(snapshotRegionDir);
+
+    // 2. iterate through all the stores in the region
+    LOG.debug("Creating references for hfiles");
+
+    // This ensures that we have an atomic view of the directory as long as we have < ls limit
+    // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
+    // batches and may miss files being added/deleted. This could be more robust (iteratively
+    // checking to see if we have all the files until we are sure), but the limit is currently 1000
+    // files/batch, far more than the number of store files under a single column family.
+    for (Store store : stores.values()) {
+      // 2.1. build the snapshot reference directory for the store
+      Path dstStoreDir = TakeSnapshotUtils.getStoreSnapshotDirectory(snapshotRegionDir,
+        Bytes.toString(store.getFamily().getName()));
+      List storeFiles = store.getStorefiles();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Adding snapshot references for " + storeFiles  + " hfiles");
+      }
+
+      // 2.2. iterate through all the store's files and create "references".
+      int sz = storeFiles.size();
+      for (int i = 0; i < sz; i++) {
+        if (exnSnare != null) {
+          exnSnare.rethrowException();
+        }
+        Path file = storeFiles.get(i).getPath();
+        // create "reference" to this store file.  It is intentionally an empty file -- all
+        // necessary infomration is captured by its fs location and filename.  This allows us to
+        // only figure out what needs to be done via a single nn operation (instead of having to
+        // open and read the files as well).
+        LOG.debug("Creating reference for file (" + (i+1) + "/" + sz + ") : " + file);
+        Path referenceFile = new Path(dstStoreDir, file.getName());
+        boolean success = HBaseFileSystem.createNewFileOnFileSystem(fs, referenceFile);
+        if (!success) {
+          throw new IOException("Failed to create reference file:" + referenceFile);
+        }
+      }
+    }
+  }
+
+  /**
+   * Replaces any KV timestamps set to {@link HConstants#LATEST_TIMESTAMP} with the provided current
+   * timestamp.
+   */
+  private void updateKVTimestamps(
+      final Iterable> keyLists, final byte[] now) {
+    for (List keys: keyLists) {
+      if (keys == null) continue;
+      for (KeyValue key : keys) {
+        key.updateLatestStamp(now);
+      }
+    }
+  }
+
+  /*
+   * Check if resources to support an update.
+   *
+   * Here we synchronize on HRegion, a broad scoped lock.  Its appropriate
+   * given we're figuring in here whether this region is able to take on
+   * writes.  This is only method with a synchronize (at time of writing),
+   * this and the synchronize on 'this' inside in internalFlushCache to send
+   * the notify.
+   */
+  public void checkResources()
+      throws RegionTooBusyException, InterruptedIOException {
+
+    // If catalog region, do not impose resource constraints or block updates.
+    if (this.getRegionInfo().isMetaRegion()) return;
+
+    boolean blocked = false;
+    long startTime = 0;
+    while (this.memstoreSize.get() > this.blockingMemStoreSize) {
+      requestFlush();
+      if (!blocked) {
+        startTime = EnvironmentEdgeManager.currentTimeMillis();
+        LOG.info("Blocking updates for '" + Thread.currentThread().getName() +
+          "' on region " + Bytes.toStringBinary(getRegionName()) +
+          ": memstore size " +
+          StringUtils.humanReadableInt(this.memstoreSize.get()) +
+          " is >= than blocking " +
+          StringUtils.humanReadableInt(this.blockingMemStoreSize) + " size");
+      }
+      long now = EnvironmentEdgeManager.currentTimeMillis();
+      long timeToWait = startTime + busyWaitDuration - now;
+      if (timeToWait <= 0L) {
+        final long totalTime = now - startTime;
+        this.updatesBlockedMs.add(totalTime);
+        LOG.info("Failed to unblock updates for region " + this + " '"
+          + Thread.currentThread().getName() + "' in " + totalTime
+          + "ms. The region is still busy.");
+        throw new RegionTooBusyException("region is flushing");
+      }
+      blocked = true;
+      synchronized(this) {
+        try {
+          wait(Math.min(timeToWait, threadWakeFrequency));
+        } catch (InterruptedException ie) {
+          final long totalTime = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+          if (totalTime > 0) {
+            this.updatesBlockedMs.add(totalTime);
+          }
+          LOG.info("Interrupted while waiting to unblock updates for region "
+            + this + " '" + Thread.currentThread().getName() + "'");
+          InterruptedIOException iie = new InterruptedIOException();
+          iie.initCause(ie);
+          throw iie;
+        }
+      }
+    }
+    if (blocked) {
+      // Add in the blocked time if appropriate
+      final long totalTime = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+      if(totalTime > 0 ){
+        this.updatesBlockedMs.add(totalTime);
+      }
+      LOG.info("Unblocking updates for region " + this + " '"
+          + Thread.currentThread().getName() + "'");
+    }
+  }
+
+  /**
+   * @throws IOException Throws exception if region is in read-only mode.
+   */
+  protected void checkReadOnly() throws IOException {
+    if (this.writestate.isReadOnly()) {
+      throw new IOException("region is read only");
+    }
+  }
+
+  /**
+   * Add updates first to the hlog and then add values to memstore.
+   * Warning: Assumption is caller has lock on passed in row.
+   * @param family
+   * @param edits Cell updates by column
+   * @praram now
+   * @throws IOException
+   */
+  private void put(byte[] family, List edits, Integer lid) throws IOException {
+    Map> familyMap;
+    familyMap = new HashMap>();
+
+    familyMap.put(family, edits);
+    Put p = new Put();
+    p.setFamilyMap(familyMap);
+    p.setClusterId(HConstants.DEFAULT_CLUSTER_ID);
+    p.setWriteToWAL(true);
+    doBatchMutate(p, lid);
+  }
+
+  /**
+   * Atomically apply the given map of family->edits to the memstore.
+   * This handles the consistency control on its own, but the caller
+   * should already have locked updatesLock.readLock(). This also does
+   * not check the families for validity.
+   *
+   * @param familyMap Map of kvs per family
+   * @param localizedWriteEntry The WriteEntry of the MVCC for this transaction.
+   *        If null, then this method internally creates a mvcc transaction.
+   * @return the additional memory usage of the memstore caused by the
+   * new entries.
+   */
+  private long applyFamilyMapToMemstore(Map> familyMap,
+    MultiVersionConsistencyControl.WriteEntry localizedWriteEntry) {
+    long size = 0;
+    boolean freemvcc = false;
+
+    try {
+      if (localizedWriteEntry == null) {
+        localizedWriteEntry = mvcc.beginMemstoreInsert();
+        freemvcc = true;
+      }
+
+      for (Map.Entry> e : familyMap.entrySet()) {
+        byte[] family = e.getKey();
+        List edits = e.getValue();
+
+        Store store = getStore(family);
+        for (KeyValue kv: edits) {
+          kv.setMemstoreTS(localizedWriteEntry.getWriteNumber());
+          size += store.add(kv);
+        }
+      }
+    } finally {
+      if (freemvcc) {
+        mvcc.completeMemstoreInsert(localizedWriteEntry);
+      }
+    }
+
+     return size;
+   }
+
+  /**
+   * Remove all the keys listed in the map from the memstore. This method is
+   * called when a Put/Delete has updated memstore but subequently fails to update
+   * the wal. This method is then invoked to rollback the memstore.
+   */
+  private void rollbackMemstore(BatchOperationInProgress> batchOp,
+                                Map>[] familyMaps,
+                                int start, int end) {
+    int kvsRolledback = 0;
+    for (int i = start; i < end; i++) {
+      // skip over request that never succeeded in the first place.
+      if (batchOp.retCodeDetails[i].getOperationStatusCode()
+            != OperationStatusCode.SUCCESS) {
+        continue;
+      }
+
+      // Rollback all the kvs for this row.
+      Map> familyMap  = familyMaps[i];
+      for (Map.Entry> e : familyMap.entrySet()) {
+        byte[] family = e.getKey();
+        List edits = e.getValue();
+
+        // Remove those keys from the memstore that matches our
+        // key's (row, cf, cq, timestamp, memstoreTS). The interesting part is
+        // that even the memstoreTS has to match for keys that will be rolleded-back.
+        Store store = getStore(family);
+        for (KeyValue kv: edits) {
+          store.rollback(kv);
+          kvsRolledback++;
+        }
+      }
+    }
+    LOG.debug("rollbackMemstore rolled back " + kvsRolledback +
+        " keyvalues from start:" + start + " to end:" + end);
+  }
+
+  /**
+   * Check the collection of families for validity.
+   * @throws NoSuchColumnFamilyException if a family does not exist.
+   */
+  private void checkFamilies(Collection families)
+  throws NoSuchColumnFamilyException {
+    for (byte[] family : families) {
+      checkFamily(family);
+    }
+  }
+
+  private void checkTimestamps(final Map> familyMap,
+      long now) throws DoNotRetryIOException {
+    if (timestampSlop == HConstants.LATEST_TIMESTAMP) {
+      return;
+    }
+    long maxTs = now + timestampSlop;
+    for (List kvs : familyMap.values()) {
+      for (KeyValue kv : kvs) {
+        // see if the user-side TS is out of range. latest = server-side
+        if (!kv.isLatestTimestamp() && kv.getTimestamp() > maxTs) {
+          throw new DoNotRetryIOException("Timestamp for KV out of range "
+              + kv + " (too.new=" + timestampSlop + ")");
+        }
+      }
+    }
+  }
+
+  /**
+   * Append the given map of family->edits to a WALEdit data structure.
+   * This does not write to the HLog itself.
+   * @param familyMap map of family->edits
+   * @param walEdit the destination entry to append into
+   */
+  private void addFamilyMapToWALEdit(Map> familyMap,
+      WALEdit walEdit) {
+    for (List edits : familyMap.values()) {
+      for (KeyValue kv : edits) {
+        walEdit.add(kv);
+      }
+    }
+  }
+
+  private void requestFlush() {
+    if (this.rsServices == null) {
+      return;
+    }
+    synchronized (writestate) {
+      if (this.writestate.isFlushRequested()) {
+        return;
+      }
+      writestate.flushRequested = true;
+    }
+    // Make request outside of synchronize block; HBASE-818.
+    this.rsServices.getFlushRequester().requestFlush(this);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Flush requested on " + this);
+    }
+  }
+
+  /*
+   * @param size
+   * @return True if size is over the flush threshold
+   */
+  private boolean isFlushSize(final long size) {
+    return size > this.memstoreFlushSize;
+  }
+
+  /**
+   * Read the edits log put under this region by wal log splitting process.  Put
+   * the recovered edits back up into this region.
+   *
+   * 
                We can ignore any log message that has a sequence ID that's equal to or
+   * lower than minSeqId.  (Because we know such log messages are already
+   * reflected in the HFiles.)
+   *
+   * 
                While this is running we are putting pressure on memory yet we are
+   * outside of our usual accounting because we are not yet an onlined region
+   * (this stuff is being run as part of Region initialization).  This means
+   * that if we're up against global memory limits, we'll not be flagged to flush
+   * because we are not online. We can't be flushed by usual mechanisms anyways;
+   * we're not yet online so our relative sequenceids are not yet aligned with
+   * HLog sequenceids -- not till we come up online, post processing of split
+   * edits.
+   *
+   * 
                But to help relieve memory pressure, at least manage our own heap size
+   * flushing if are in excess of per-region limits.  Flushing, though, we have
+   * to be careful and avoid using the regionserver/hlog sequenceid.  Its running
+   * on a different line to whats going on in here in this region context so if we
+   * crashed replaying these edits, but in the midst had a flush that used the
+   * regionserver log with a sequenceid in excess of whats going on in here
+   * in this region and with its split editlogs, then we could miss edits the
+   * next time we go to recover. So, we have to flush inline, using seqids that
+   * make sense in a this single region context only -- until we online.
+   *
+   * @param regiondir
+   * @param maxSeqIdInStores Any edit found in split editlogs needs to be in excess of
+   * the maxSeqId for the store to be applied, else its skipped.
+   * @param reporter
+   * @return the sequence id of the last edit added to this region out of the
+   * recovered edits log or minSeqId if nothing added from editlogs.
+   * @throws UnsupportedEncodingException
+   * @throws IOException
+   */
+  protected long replayRecoveredEditsIfAny(final Path regiondir,
+      Map maxSeqIdInStores,
+      final CancelableProgressable reporter, final MonitoredTask status)
+      throws UnsupportedEncodingException, IOException {
+    long minSeqIdForTheRegion = -1;
+    for (Long maxSeqIdInStore : maxSeqIdInStores.values()) {
+      if (maxSeqIdInStore < minSeqIdForTheRegion || minSeqIdForTheRegion == -1) {
+        minSeqIdForTheRegion = maxSeqIdInStore;
+      }
+    }
+    long seqid = minSeqIdForTheRegion;
+    NavigableSet files = HLog.getSplitEditFilesSorted(this.fs, regiondir);
+    if (files == null || files.isEmpty()) return seqid;
+
+    for (Path edits: files) {
+      if (edits == null || !this.fs.exists(edits)) {
+        LOG.warn("Null or non-existent edits file: " + edits);
+        continue;
+      }
+      if (isZeroLengthThenDelete(this.fs, edits)) continue;
+
+      long maxSeqId = Long.MAX_VALUE;
+      String fileName = edits.getName();
+      maxSeqId = Math.abs(Long.parseLong(fileName));
+      if (maxSeqId <= minSeqIdForTheRegion) {
+        String msg = "Maximum sequenceid for this log is " + maxSeqId
+            + " and minimum sequenceid for the region is " + minSeqIdForTheRegion
+            + ", skipped the whole file, path=" + edits;
+        LOG.debug(msg);
+        continue;
+      }
+
+      try {
+        seqid = replayRecoveredEdits(edits, maxSeqIdInStores, reporter);
+      } catch (IOException e) {
+        boolean skipErrors = conf.getBoolean("hbase.skip.errors", false);
+        if (skipErrors) {
+          Path p = HLog.moveAsideBadEditsFile(fs, edits);
+          LOG.error("hbase.skip.errors=true so continuing. Renamed " + edits +
+            " as " + p, e);
+        } else {
+          throw e;
+        }
+      }
+      // The edits size added into rsAccounting during this replaying will not
+      // be required any more. So just clear it.
+      if (this.rsAccounting != null) {
+        this.rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName());
+      }
+    }
+    if (seqid > minSeqIdForTheRegion) {
+      // Then we added some edits to memory. Flush and cleanup split edit files.
+      internalFlushcache(null, seqid, status);
+    }
+    // Now delete the content of recovered edits.  We're done w/ them.
+    for (Path file: files) {
+      if (!HBaseFileSystem.deleteFileFromFileSystem(fs, file)) {
+        LOG.error("Failed delete of " + file);
+      } else {
+        LOG.debug("Deleted recovered.edits file=" + file);
+      }
+    }
+    return seqid;
+  }
+
+  /*
+   * @param edits File of recovered edits.
+   * @param maxSeqIdInStores Maximum sequenceid found in each store.  Edits in log
+   * must be larger than this to be replayed for each store.
+   * @param reporter
+   * @return the sequence id of the last edit added to this region out of the
+   * recovered edits log or minSeqId if nothing added from editlogs.
+   * @throws IOException
+   */
+  private long replayRecoveredEdits(final Path edits,
+      Map maxSeqIdInStores, final CancelableProgressable reporter)
+    throws IOException {
+    String msg = "Replaying edits from " + edits;
+    LOG.info(msg);
+    MonitoredTask status = TaskMonitor.get().createStatus(msg);
+
+    status.setStatus("Opening logs");
+    HLog.Reader reader = null;
+    try {
+      reader = HLog.getReader(this.fs, edits, conf);
+      long currentEditSeqId = -1;
+      long firstSeqIdInLog = -1;
+      long skippedEdits = 0;
+      long editsCount = 0;
+      long intervalEdits = 0;
+      HLog.Entry entry;
+      Store store = null;
+      boolean reported_once = false;
+
+      try {
+        // How many edits seen before we check elapsed time
+        int interval = this.conf.getInt("hbase.hstore.report.interval.edits",
+            2000);
+        // How often to send a progress report (default 1/2 master timeout)
+        int period = this.conf.getInt("hbase.hstore.report.period",
+            this.conf.getInt("hbase.master.assignment.timeoutmonitor.timeout",
+                180000) / 2);
+        long lastReport = EnvironmentEdgeManager.currentTimeMillis();
+
+        while ((entry = reader.next()) != null) {
+          HLogKey key = entry.getKey();
+          WALEdit val = entry.getEdit();
+
+          if (reporter != null) {
+            intervalEdits += val.size();
+            if (intervalEdits >= interval) {
+              // Number of edits interval reached
+              intervalEdits = 0;
+              long cur = EnvironmentEdgeManager.currentTimeMillis();
+              if (lastReport + period <= cur) {
+                status.setStatus("Replaying edits..." +
+                    " skipped=" + skippedEdits +
+                    " edits=" + editsCount);
+                // Timeout reached
+                if(!reporter.progress()) {
+                  msg = "Progressable reporter failed, stopping replay";
+                  LOG.warn(msg);
+                  status.abort(msg);
+                  throw new IOException(msg);
+                }
+                reported_once = true;
+                lastReport = cur;
+              }
+            }
+          }
+
+          // Start coprocessor replay here. The coprocessor is for each WALEdit
+          // instead of a KeyValue.
+          if (coprocessorHost != null) {
+            status.setStatus("Running pre-WAL-restore hook in coprocessors");
+            if (coprocessorHost.preWALRestore(this.getRegionInfo(), key, val)) {
+              // if bypass this log entry, ignore it ...
+              continue;
+            }
+          }
+
+          if (firstSeqIdInLog == -1) {
+            firstSeqIdInLog = key.getLogSeqNum();
+          }
+          boolean flush = false;
+          for (KeyValue kv: val.getKeyValues()) {
+            // Check this edit is for me. Also, guard against writing the special
+            // METACOLUMN info such as HBASE::CACHEFLUSH entries
+            if (kv.matchingFamily(HLog.METAFAMILY) ||
+                !Bytes.equals(key.getEncodedRegionName(), this.regionInfo.getEncodedNameAsBytes())) {
+              skippedEdits++;
+              continue;
+                }
+            // Figure which store the edit is meant for.
+            if (store == null || !kv.matchingFamily(store.getFamily().getName())) {
+              store = this.stores.get(kv.getFamily());
+            }
+            if (store == null) {
+              // This should never happen.  Perhaps schema was changed between
+              // crash and redeploy?
+              LOG.warn("No family for " + kv);
+              skippedEdits++;
+              continue;
+            }
+            // Now, figure if we should skip this edit.
+            if (key.getLogSeqNum() <= maxSeqIdInStores.get(store.getFamily()
+                .getName())) {
+              skippedEdits++;
+              continue;
+            }
+            currentEditSeqId = key.getLogSeqNum();
+            // Once we are over the limit, restoreEdit will keep returning true to
+            // flush -- but don't flush until we've played all the kvs that make up
+            // the WALEdit.
+            flush = restoreEdit(store, kv);
+            editsCount++;
+          }
+          if (flush) internalFlushcache(null, currentEditSeqId, status);
+
+          if (coprocessorHost != null) {
+            coprocessorHost.postWALRestore(this.getRegionInfo(), key, val);
+          }
+        }
+      } catch (EOFException eof) {
+        Path p = HLog.moveAsideBadEditsFile(fs, edits);
+        msg = "Encountered EOF. Most likely due to Master failure during " +
+            "log spliting, so we have this data in another edit.  " +
+            "Continuing, but renaming " + edits + " as " + p;
+        LOG.warn(msg, eof);
+        status.abort(msg);
+      } catch (IOException ioe) {
+        // If the IOE resulted from bad file format,
+        // then this problem is idempotent and retrying won't help
+        if (ioe.getCause() instanceof ParseException) {
+          Path p = HLog.moveAsideBadEditsFile(fs, edits);
+          msg = "File corruption encountered!  " +
+              "Continuing, but renaming " + edits + " as " + p;
+          LOG.warn(msg, ioe);
+          status.setStatus(msg);
+        } else {
+          status.abort(StringUtils.stringifyException(ioe));
+          // other IO errors may be transient (bad network connection,
+          // checksum exception on one datanode, etc).  throw & retry
+          throw ioe;
+        }
+      }
+      if (reporter != null && !reported_once) {
+        reporter.progress();
+      }
+      msg = "Applied " + editsCount + ", skipped " + skippedEdits +
+        ", firstSequenceidInLog=" + firstSeqIdInLog +
+        ", maxSequenceidInLog=" + currentEditSeqId + ", path=" + edits;
+      status.markComplete(msg);
+      LOG.debug(msg);
+      return currentEditSeqId;
+    } finally {
+      status.cleanup();
+      if (reader != null) {
+         reader.close();
+      }
+    }
+  }
+
+  /**
+   * Used by tests
+   * @param s Store to add edit too.
+   * @param kv KeyValue to add.
+   * @return True if we should flush.
+   */
+  protected boolean restoreEdit(final Store s, final KeyValue kv) {
+    long kvSize = s.add(kv);
+    if (this.rsAccounting != null) {
+      rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize);
+    }
+    return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize));
+  }
+
+  /*
+   * @param fs
+   * @param p File to check.
+   * @return True if file was zero-length (and if so, we'll delete it in here).
+   * @throws IOException
+   */
+  private static boolean isZeroLengthThenDelete(final FileSystem fs, final Path p)
+      throws IOException {
+    FileStatus stat = fs.getFileStatus(p);
+    if (stat.getLen() > 0) return false;
+    LOG.warn("File " + p + " is zero-length, deleting.");
+    HBaseFileSystem.deleteFileFromFileSystem(fs, p);
+    return true;
+  }
+
+  protected Store instantiateHStore(Path tableDir, HColumnDescriptor c)
+      throws IOException {
+    return new Store(tableDir, this, c, this.fs, this.conf);
+  }
+
+  /**
+   * Return HStore instance.
+   * Use with caution.  Exposed for use of fixup utilities.
+   * @param column Name of column family hosted by this region.
+   * @return Store that goes with the family on passed column.
+   * TODO: Make this lookup faster.
+   */
+  public Store getStore(final byte [] column) {
+    return this.stores.get(column);
+  }
+
+  public Map getStores() {
+    return this.stores;
+  }
+
+  /**
+   * Return list of storeFiles for the set of CFs.
+   * Uses closeLock to prevent the race condition where a region closes
+   * in between the for loop - closing the stores one by one, some stores
+   * will return 0 files.
+   * @return List of storeFiles.
+   */
+  public List getStoreFileList(final byte [][] columns)
+    throws IllegalArgumentException {
+    List storeFileNames = new ArrayList();
+    synchronized(closeLock) {
+      for(byte[] column : columns) {
+        Store store = this.stores.get(column);
+        if (store == null) {
+          throw new IllegalArgumentException("No column family : " +
+              new String(column) + " available");
+        }
+        List storeFiles = store.getStorefiles();
+        for (StoreFile storeFile: storeFiles) {
+          storeFileNames.add(storeFile.getPath().toString());
+        }
+      }
+    }
+    return storeFileNames;
+  }
+  //////////////////////////////////////////////////////////////////////////////
+  // Support code
+  //////////////////////////////////////////////////////////////////////////////
+
+  /** Make sure this is a valid row for the HRegion */
+  void checkRow(final byte [] row, String op) throws IOException {
+    if(!rowIsInRange(regionInfo, row)) {
+      throw new WrongRegionException("Requested row out of range for " +
+          op + " on HRegion " + this + ", startKey='" +
+          Bytes.toStringBinary(regionInfo.getStartKey()) + "', getEndKey()='" +
+          Bytes.toStringBinary(regionInfo.getEndKey()) + "', row='" +
+          Bytes.toStringBinary(row) + "'");
+    }
+  }
+
+  /**
+   * Obtain a lock on the given row.  Blocks until success.
+   *
+   * I know it's strange to have two mappings:
+   * 
                +   *   ROWS  ==> LOCKS
+   * 
                
+   * as well as
+   * 
                +   *   LOCKS ==> ROWS
+   * 
                
+   *
+   * But it acts as a guard on the client; a miswritten client just can't
+   * submit the name of a row and start writing to it; it must know the correct
+   * lockid, which matches the lock list in memory.
+   *
+   * 
                It would be more memory-efficient to assume a correctly-written client,
+   * which maybe we'll do in the future.
+   *
+   * @param row Name of row to lock.
+   * @throws IOException
+   * @return The id of the held lock.
+   */
+  public Integer obtainRowLock(final byte [] row) throws IOException {
+    startRegionOperation();
+    this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics( this.writeRequestsCount.get());
+    try {
+      return internalObtainRowLock(row, true);
+    } finally {
+      closeRegionOperation();
+    }
+  }
+
+  /**
+   * Obtains or tries to obtain the given row lock.
+   * @param waitForLock if true, will block until the lock is available.
+   *        Otherwise, just tries to obtain the lock and returns
+   *        null if unavailable.
+   */
+  private Integer internalObtainRowLock(final byte[] row, boolean waitForLock)
+      throws IOException {
+    checkRow(row, "row lock");
+    startRegionOperation();
+    try {
+      HashedBytes rowKey = new HashedBytes(row);
+      CountDownLatch rowLatch = new CountDownLatch(1);
+
+      // loop until we acquire the row lock (unless !waitForLock)
+      while (true) {
+        CountDownLatch existingLatch = lockedRows.putIfAbsent(rowKey, rowLatch);
+        if (existingLatch == null) {
+          break;
+        } else {
+          // row already locked
+          if (!waitForLock) {
+            return null;
+          }
+          try {
+            if (!existingLatch.await(this.rowLockWaitDuration,
+                            TimeUnit.MILLISECONDS)) {
+              throw new IOException("Timed out on getting lock for row="
+                  + Bytes.toStringBinary(row));
+            }
+          } catch (InterruptedException ie) {
+            // Empty
+          }
+        }
+      }
+
+      // loop until we generate an unused lock id
+      while (true) {
+        Integer lockId = lockIdGenerator.incrementAndGet();
+        HashedBytes existingRowKey = lockIds.putIfAbsent(lockId, rowKey);
+        if (existingRowKey == null) {
+          return lockId;
+        } else {
+          // lockId already in use, jump generator to a new spot
+          lockIdGenerator.set(rand.nextInt());
+        }
+      }
+    } finally {
+      closeRegionOperation();
+    }
+  }
+
+  /**
+   * Used by unit tests.
+   * @param lockid
+   * @return Row that goes with lockid
+   */
+  byte[] getRowFromLock(final Integer lockid) {
+    HashedBytes rowKey = lockIds.get(lockid);
+    return rowKey == null ? null : rowKey.getBytes();
+  }
+
+  /**
+   * Release the row lock!
+   * @param lockId  The lock ID to release.
+   */
+  public void releaseRowLock(final Integer lockId) {
+    if (lockId == null) return; // null lock id, do nothing
+    HashedBytes rowKey = lockIds.remove(lockId);
+    if (rowKey == null) {
+      LOG.warn("Release unknown lockId: " + lockId);
+      return;
+    }
+    CountDownLatch rowLatch = lockedRows.remove(rowKey);
+    if (rowLatch == null) {
+      LOG.error("Releases row not locked, lockId: " + lockId + " row: "
+          + rowKey);
+      return;
+    }
+    rowLatch.countDown();
+  }
+
+  /**
+   * See if row is currently locked.
+   * @param lockid
+   * @return boolean
+   */
+  boolean isRowLocked(final Integer lockId) {
+    return lockIds.containsKey(lockId);
+  }
+
+  /**
+   * Returns existing row lock if found, otherwise
+   * obtains a new row lock and returns it.
+   * @param lockid requested by the user, or null if the user didn't already hold lock
+   * @param row the row to lock
+   * @param waitForLock if true, will block until the lock is available, otherwise will
+   * simply return null if it could not acquire the lock.
+   * @return lockid or null if waitForLock is false and the lock was unavailable.
+   */
+  public Integer getLock(Integer lockid, byte [] row, boolean waitForLock)
+  throws IOException {
+    Integer lid = null;
+    if (lockid == null) {
+      lid = internalObtainRowLock(row, waitForLock);
+    } else {
+      if (!isRowLocked(lockid)) {
+        throw new IOException("Invalid row lock");
+      }
+      lid = lockid;
+    }
+    return lid;
+  }
+
+  /**
+   * Determines whether multiple column families are present
+   * Precondition: familyPaths is not null
+   *
+   * @param familyPaths List of Pair
+   */
+  private static boolean hasMultipleColumnFamilies(
+      List> familyPaths) {
+    boolean multipleFamilies = false;
+    byte[] family = null;
+    for (Pair pair : familyPaths) {
+      byte[] fam = pair.getFirst();
+      if (family == null) {
+        family = fam;
+      } else if (!Bytes.equals(family, fam)) {
+        multipleFamilies = true;
+        break;
+      }
+    }
+    return multipleFamilies;
+  }
+
+  /**
+   * Attempts to atomically load a group of hfiles.  This is critical for loading
+   * rows with multiple column families atomically.
+   *
+   * @param familyPaths List of Pair
+   * @return true if successful, false if failed recoverably
+   * @throws IOException if failed unrecoverably.
+   */
+  public boolean bulkLoadHFiles(List> familyPaths) throws IOException {
+    return bulkLoadHFiles(familyPaths, null);
+  }
+
+  /**
+   * Attempts to atomically load a group of hfiles.  This is critical for loading
+   * rows with multiple column families atomically.
+   *
+   * @param familyPaths List of Pair
+   * @param bulkLoadListener Internal hooks enabling massaging/preparation of a
+   * file about to be bulk loaded
+   * @return true if successful, false if failed recoverably
+   * @throws IOException if failed unrecoverably.
+   */
+  public boolean bulkLoadHFiles(List> familyPaths,
+      BulkLoadListener bulkLoadListener) throws IOException {
+    Preconditions.checkNotNull(familyPaths);
+    // we need writeLock for multi-family bulk load
+    startBulkRegionOperation(hasMultipleColumnFamilies(familyPaths));
+    try {
+      this.writeRequestsCount.increment();
+      this.opMetrics.setWriteRequestCountMetrics( this.writeRequestsCount.get());
+
+      // There possibly was a split that happend between when the split keys
+      // were gathered and before the HReiogn's write lock was taken.  We need
+      // to validate the HFile region before attempting to bulk load all of them
+      List ioes = new ArrayList();
+      List> failures = new ArrayList>();
+      for (Pair p : familyPaths) {
+        byte[] familyName = p.getFirst();
+        String path = p.getSecond();
+
+        Store store = getStore(familyName);
+        if (store == null) {
+          IOException ioe = new DoNotRetryIOException(
+              "No such column family " + Bytes.toStringBinary(familyName));
+          ioes.add(ioe);
+        } else {
+          try {
+            store.assertBulkLoadHFileOk(new Path(path));
+          } catch (WrongRegionException wre) {
+            // recoverable (file doesn't fit in region)
+            failures.add(p);
+          } catch (IOException ioe) {
+            // unrecoverable (hdfs problem)
+            ioes.add(ioe);
+          }
+        }
+      }
+
+      // validation failed because of some sort of IO problem.
+      if (ioes.size() != 0) {
+        IOException e = MultipleIOException.createIOException(ioes);
+        LOG.error("There were one or more IO errors when checking if the bulk load is ok.", e);
+        throw e;
+      }
+
+      // validation failed, bail out before doing anything permanent.
+      if (failures.size() != 0) {
+        StringBuilder list = new StringBuilder();
+        for (Pair p : failures) {
+          list.append("\n").append(Bytes.toString(p.getFirst())).append(" : ")
+            .append(p.getSecond());
+        }
+        // problem when validating
+        LOG.warn("There was a recoverable bulk load failure likely due to a" +
+            " split.  These (family, HFile) pairs were not loaded: " + list);
+        return false;
+      }
+
+      for (Pair p : familyPaths) {
+        byte[] familyName = p.getFirst();
+        String path = p.getSecond();
+        Store store = getStore(familyName);
+        try {
+          String finalPath = path;
+          if(bulkLoadListener != null) {
+            finalPath = bulkLoadListener.prepareBulkLoad(familyName, path);
+          }
+          store.bulkLoadHFile(finalPath);
+          if(bulkLoadListener != null) {
+            bulkLoadListener.doneBulkLoad(familyName, path);
+          }
+        } catch (IOException ioe) {
+          // a failure here causes an atomicity violation that we currently
+          // cannot recover from since it is likely a failed hdfs operation.
+
+          // TODO Need a better story for reverting partial failures due to HDFS.
+          LOG.error("There was a partial failure due to IO when attempting to" +
+              " load " + Bytes.toString(p.getFirst()) + " : "+ p.getSecond());
+          if(bulkLoadListener != null) {
+            try {
+              bulkLoadListener.failedBulkLoad(familyName, path);
+            } catch (Exception ex) {
+              LOG.error("Error while calling failedBulkLoad for family "+
+                  Bytes.toString(familyName)+" with path "+path, ex);
+            }
+          }
+          throw ioe;
+        }
+      }
+      return true;
+    } finally {
+      closeBulkRegionOperation();
+    }
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (!(o instanceof HRegion)) {
+      return false;
+    }
+    return Bytes.equals(this.getRegionName(), ((HRegion) o).getRegionName());
+  }
+
+  @Override
+  public int hashCode() {
+    return Bytes.hashCode(this.getRegionName());
+  }
+
+  @Override
+  public String toString() {
+    return this.regionInfo.getRegionNameAsString();
+  }
+
+  /** @return Path of region base directory */
+  public Path getTableDir() {
+    return this.tableDir;
+  }
+
+  /**
+   * RegionScannerImpl is used to combine scanners from multiple Stores (aka column families).
+   */
+  class RegionScannerImpl implements RegionScanner {
+    // Package local for testability
+    KeyValueHeap storeHeap = null;
+    /** Heap of key-values that are not essential for the provided filters and are thus read
+     * on demand, if on-demand column family loading is enabled.*/
+    KeyValueHeap joinedHeap = null;
+    /**
+     * If the joined heap data gathering is interrupted due to scan limits, this will
+     * contain the row for which we are populating the values.*/
+    private KeyValue joinedContinuationRow = null;
+    // KeyValue indicating that limit is reached when scanning
+    private final KeyValue KV_LIMIT = new KeyValue();
+    private final byte [] stopRow;
+    private Filter filter;
+    private int batch;
+    private int isScan;
+    private boolean filterClosed = false;
+    private long readPt;
+    private HRegion region;
+
+    public HRegionInfo getRegionInfo() {
+      return regionInfo;
+    }
+    
+    RegionScannerImpl(Scan scan, List additionalScanners, HRegion region)
+        throws IOException {
+      // DebugPrint.println("HRegionScanner.");
+      this.region = region;
+      this.filter = scan.getFilter();
+      this.batch = scan.getBatch();
+      if (Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
+        this.stopRow = null;
+      } else {
+        this.stopRow = scan.getStopRow();
+      }
+      // If we are doing a get, we want to be [startRow,endRow] normally
+      // it is [startRow,endRow) and if startRow=endRow we get nothing.
+      this.isScan = scan.isGetScan() ? -1 : 0;
+
+      // synchronize on scannerReadPoints so that nobody calculates
+      // getSmallestReadPoint, before scannerReadPoints is updated.
+      IsolationLevel isolationLevel = scan.getIsolationLevel();
+      synchronized(scannerReadPoints) {
+        if (isolationLevel == IsolationLevel.READ_UNCOMMITTED) {
+          // This scan can read even uncommitted transactions
+          this.readPt = Long.MAX_VALUE;
+          MultiVersionConsistencyControl.setThreadReadPoint(this.readPt);
+        } else {
+          this.readPt = MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
+        }
+        scannerReadPoints.put(this, this.readPt);
+      }
+
+      // Here we separate all scanners into two lists - scanner that provide data required
+      // by the filter to operate (scanners list) and all others (joinedScanners list).
+      List scanners = new ArrayList();
+      List joinedScanners = new ArrayList();
+      if (additionalScanners != null) {
+        scanners.addAll(additionalScanners);
+      }
+
+      for (Map.Entry> entry :
+          scan.getFamilyMap().entrySet()) {
+        Store store = stores.get(entry.getKey());
+        KeyValueScanner scanner = store.getScanner(scan, entry.getValue());
+        if (this.filter == null || !scan.doLoadColumnFamiliesOnDemand()
+          || FilterBase.isFamilyEssential(this.filter, entry.getKey())) {
+          scanners.add(scanner);
+        } else {
+          joinedScanners.add(scanner);
+        }
+      }
+      this.storeHeap = new KeyValueHeap(scanners, comparator);
+      if (!joinedScanners.isEmpty()) {
+        this.joinedHeap = new KeyValueHeap(joinedScanners, comparator);
+      }
+    }
+
+    RegionScannerImpl(Scan scan, HRegion region) throws IOException {
+      this(scan, null, region);
+    }
+
+    @Override
+    public long getMvccReadPoint() {
+      return this.readPt;
+    }
+    /**
+     * Reset both the filter and the old filter.
+     */
+    protected void resetFilters() {
+      if (filter != null) {
+        filter.reset();
+      }
+    }
+
+    @Override
+    public boolean next(List outResults, int limit)
+        throws IOException {
+      return next(outResults, limit, null);
+    }
+
+    @Override
+    public synchronized boolean next(List outResults, int limit,
+        String metric) throws IOException {
+      if (this.filterClosed) {
+        throw new UnknownScannerException("Scanner was closed (timed out?) " +
+            "after we renewed it. Could be caused by a very slow scanner " +
+            "or a lengthy garbage collection");
+      }
+      startRegionOperation();
+      readRequestsCount.increment();
+      opMetrics.setReadRequestCountMetrics(readRequestsCount.get());
+      try {
+
+        // This could be a new thread from the last time we called next().
+        MultiVersionConsistencyControl.setThreadReadPoint(this.readPt);
+
+        return nextRaw(outResults, limit, metric);
+      } finally {
+        closeRegionOperation();
+      }
+    }
+
+    @Override
+    public boolean nextRaw(List outResults, String metric)
+        throws IOException {
+      return nextRaw(outResults, batch, metric);
+    }
+
+    @Override
+    public boolean nextRaw(List outResults, int limit,
+        String metric) throws IOException {
+      boolean returnResult;
+      if (outResults.isEmpty()) {
+        // Usually outResults is empty. This is true when next is called
+        // to handle scan or get operation.
+        returnResult = nextInternal(outResults, limit, metric);
+      } else {
+        List tmpList = new ArrayList();
+        returnResult = nextInternal(tmpList, limit, metric);
+        outResults.addAll(tmpList);
+      }
+      resetFilters();
+      if (isFilterDone()) {
+        return false;
+      }
+      return returnResult;
+    }
+
+    @Override
+    public boolean next(List outResults)
+        throws IOException {
+      // apply the batching limit by default
+      return next(outResults, batch, null);
+    }
+
+    @Override
+    public boolean next(List outResults, String metric)
+        throws IOException {
+      // apply the batching limit by default
+      return next(outResults, batch, metric);
+    }
+
+    private void populateFromJoinedHeap(List results, int limit, String metric)
+        throws IOException {
+      assert joinedContinuationRow != null;
+      KeyValue kv = populateResult(results, this.joinedHeap, limit,
+        joinedContinuationRow.getBuffer(), joinedContinuationRow.getRowOffset(),
+        joinedContinuationRow.getRowLength(), metric);
+      if (kv != KV_LIMIT) {
+        // We are done with this row, reset the continuation.
+        joinedContinuationRow = null;
+      }
+      // As the data is obtained from two independent heaps, we need to
+      // ensure that result list is sorted, because Result relies on that.
+      Collections.sort(results, comparator);
+    }
+
+    /**
+     * Fetches records with this row into result list, until next row or limit (if not -1).
+     * @param results
+     * @param heap KeyValueHeap to fetch data from. It must be positioned on correct row before call.
+     * @param limit Max amount of KVs to place in result list, -1 means no limit.
+     * @param currentRow Byte array with key we are fetching.
+     * @param offset offset for currentRow
+     * @param length length for currentRow
+     * @param metric Metric key to be passed into KeyValueHeap::next().
+     * @return true if limit reached, false otherwise.
+     */
+    private KeyValue populateResult(List results, KeyValueHeap heap, int limit,
+        byte[] currentRow, int offset, short length, String metric) throws IOException {
+      KeyValue nextKv;
+      do {
+        heap.next(results, limit - results.size(), metric);
+        if (limit > 0 && results.size() == limit) {
+          return KV_LIMIT;
+        }
+        nextKv = heap.peek();
+      } while (nextKv != null && nextKv.matchingRow(currentRow, offset, length));
+      return nextKv;
+    }
+
+    /*
+     * @return True if a filter rules the scanner is over, done.
+     */
+    public synchronized boolean isFilterDone() {
+      return this.filter != null && this.filter.filterAllRemaining();
+    }
+
+    private boolean nextInternal(List results, int limit, String metric)
+    throws IOException {
+      if (!results.isEmpty()) {
+        throw new IllegalArgumentException("First parameter should be an empty list");
+      }
+      RpcCallContext rpcCall = HBaseServer.getCurrentCall();
+      // The loop here is used only when at some point during the next we determine
+      // that due to effects of filters or otherwise, we have an empty row in the result.
+      // Then we loop and try again. Otherwise, we must get out on the first iteration via return,
+      // "true" if there's more data to read, "false" if there isn't (storeHeap is at a stop row,
+      // and joinedHeap has no more data to read for the last row (if set, joinedContinuationRow).
+      while (true) {
+        if (rpcCall != null) {
+          // If a user specifies a too-restrictive or too-slow scanner, the
+          // client might time out and disconnect while the server side
+          // is still processing the request. We should abort aggressively
+          // in that case.
+          rpcCall.throwExceptionIfCallerDisconnected();
+        }
+
+        // Let's see what we have in the storeHeap.
+        KeyValue current = this.storeHeap.peek();
+
+        byte[] currentRow = null;
+        int offset = 0;
+        short length = 0;
+        if (current != null) {
+          currentRow = current.getBuffer();
+          offset = current.getRowOffset();
+          length = current.getRowLength();
+        }
+        boolean stopRow = isStopRow(currentRow, offset, length);
+        // Check if we were getting data from the joinedHeap abd hit the limit.
+        // If not, then it's main path - getting results from storeHeap.
+        if (joinedContinuationRow == null) {
+          // First, check if we are at a stop row. If so, there are no more results.
+          if (stopRow) {
+            if (filter != null && filter.hasFilterRow()) {
+              filter.filterRow(results);
+            }
+            if (filter != null && filter.filterRow()) {
+              results.clear();
+            }
+            return false;
+          }
+
+          // Check if rowkey filter wants to exclude this row. If so, loop to next.
+          // Techically, if we hit limits before on this row, we don't need this call.
+          if (filterRowKey(currentRow, offset, length)) {
+            results.clear();
+            boolean moreRows = nextRow(currentRow, offset, length);
+            if (!moreRows) return false;
+            continue;
+          }
+
+          // Ok, we are good, let's try to get some results from the main heap.
+          KeyValue nextKv = populateResult(results, this.storeHeap, limit, currentRow, offset,
+              length, metric);
+          if (nextKv == KV_LIMIT) {
+            if (this.filter != null && filter.hasFilterRow()) {
+              throw new IncompatibleFilterException(
+                "Filter whose hasFilterRow() returns true is incompatible with scan with limit!");
+            }
+            return true; // We hit the limit.
+          }
+          stopRow = nextKv == null
+              || isStopRow(nextKv.getBuffer(), nextKv.getRowOffset(), nextKv.getRowLength());
+          // save that the row was empty before filters applied to it.
+          final boolean isEmptyRow = results.isEmpty();
+
+          // We have the part of the row necessary for filtering (all of it, usually).
+          // First filter with the filterRow(List).            
+          if (filter != null && filter.hasFilterRow()) {
+            filter.filterRow(results);
+          }
+
+          if (isEmptyRow || filterRow()) {
+            results.clear();
+            boolean moreRows = nextRow(currentRow, offset, length);
+            if (!moreRows) return false;
+
+            // This row was totally filtered out, if this is NOT the last row,
+            // we should continue on. Otherwise, nothing else to do.
+            if (!stopRow) continue;
+            return false;
+          }
+
+          // Ok, we are done with storeHeap for this row.
+          // Now we may need to fetch additional, non-essential data into row.
+          // These values are not needed for filter to work, so we postpone their
+          // fetch to (possibly) reduce amount of data loads from disk.
+          if (this.joinedHeap != null) {
+            KeyValue nextJoinedKv = joinedHeap.peek();
+            // If joinedHeap is pointing to some other row, try to seek to a correct one.
+            boolean mayHaveData =
+              (nextJoinedKv != null && nextJoinedKv.matchingRow(currentRow, offset, length))
+                || (this.joinedHeap.requestSeek(
+                    KeyValue.createFirstOnRow(currentRow, offset, length), true, true)
+                  && joinedHeap.peek() != null
+                  && joinedHeap.peek().matchingRow(currentRow, offset, length));
+            if (mayHaveData) {
+              joinedContinuationRow = current;
+              populateFromJoinedHeap(results, limit, metric);
+            }
+          }
+        } else {
+          // Populating from the joined map was stopped by limits, populate some more.
+          populateFromJoinedHeap(results, limit, metric);
+        }
+
+        // We may have just called populateFromJoinedMap and hit the limits. If that is
+        // the case, we need to call it again on the next next() invocation.
+        if (joinedContinuationRow != null) {
+          return true;
+        }
+
+        // Finally, we are done with both joinedHeap and storeHeap.
+        // Double check to prevent empty rows from appearing in result. It could be
+        // the case when SingleValueExcludeFilter is used.
+        if (results.isEmpty()) {
+          boolean moreRows = nextRow(currentRow, offset, length);
+          if (!moreRows) return false;
+          if (!stopRow) continue;
+        }
+
+        // We are done. Return the result.
+        return !stopRow;
+      }
+    }
+
+    private boolean filterRow() {
+      return filter != null
+          && filter.filterRow();
+    }
+    private boolean filterRowKey(byte[] row, int offset, short length) {
+      return filter != null
+          && filter.filterRowKey(row, offset, length);
+    }
+
+    protected boolean nextRow(byte [] currentRow, int offset, short length) throws IOException {
+      KeyValue next;
+      while((next = this.storeHeap.peek()) != null && next.matchingRow(currentRow, offset, length)) {
+        this.storeHeap.next(MOCKED_LIST);       
+      }
+      resetFilters();
+      // Calling the hook in CP which allows it to do a fast forward
+      if (this.region.getCoprocessorHost() != null) {
+        return this.region.getCoprocessorHost().postScannerFilterRow(this, currentRow);
+      }
+      return true;
+    }
+
+    private boolean isStopRow(byte [] currentRow, int offset, short length) {
+      return currentRow == null ||
+          (stopRow != null &&
+          comparator.compareRows(stopRow, 0, stopRow.length,
+              currentRow, offset, length) <= isScan);
+    }
+
+    @Override
+    public synchronized void close() {
+      if (storeHeap != null) {
+        storeHeap.close();
+        storeHeap = null;
+      }
+      if (joinedHeap != null) {
+        joinedHeap.close();
+        joinedHeap = null;
+      }
+      // no need to sychronize here.
+      scannerReadPoints.remove(this);
+      this.filterClosed = true;
+    }
+
+    KeyValueHeap getStoreHeapForTesting() {
+      return storeHeap;
+    }
+
+    @Override
+    public synchronized boolean reseek(byte[] row) throws IOException {
+      if (row == null) {
+        throw new IllegalArgumentException("Row cannot be null.");
+      }
+      boolean result = false;
+      startRegionOperation();
+      try {
+        // This could be a new thread from the last time we called next().
+        MultiVersionConsistencyControl.setThreadReadPoint(this.readPt);
+        KeyValue kv = KeyValue.createFirstOnRow(row);
+        // use request seek to make use of the lazy seek option. See HBASE-5520
+        result = this.storeHeap.requestSeek(kv, true, true);
+        if (this.joinedHeap != null) {
+          result = this.joinedHeap.requestSeek(kv, true, true) || result;
+        }
+      } finally {
+        closeRegionOperation();
+      }
+      return result;
+    }
+  }
+
+  // Utility methods
+  /**
+   * A utility method to create new instances of HRegion based on the
+   * {@link HConstants#REGION_IMPL} configuration property.
+   * @param tableDir qualified path of directory where region should be located,
+   * usually the table directory.
+   * @param log The HLog is the outbound log for any updates to the HRegion
+   * (There's a single HLog for all the HRegions on a single HRegionServer.)
+   * The log file is a logfile from the previous execution that's
+   * custom-computed for this HRegion. The HRegionServer computes and sorts the
+   * appropriate log info for this HRegion. If there is a previous log file
+   * (implying that the HRegion has been written-to before), then read it from
+   * the supplied path.
+   * @param fs is the filesystem.
+   * @param conf is global configuration settings.
+   * @param regionInfo - HRegionInfo that describes the region
+   * is new), then read them from the supplied path.
+   * @param htd
+   * @param rsServices
+   * @return the new instance
+   */
+  public static HRegion newHRegion(Path tableDir, HLog log, FileSystem fs,
+      Configuration conf, HRegionInfo regionInfo, final HTableDescriptor htd,
+      RegionServerServices rsServices) {
+    try {
+      @SuppressWarnings("unchecked")
+      Class regionClass =
+          (Class) conf.getClass(HConstants.REGION_IMPL, HRegion.class);
+
+      Constructor c =
+          regionClass.getConstructor(Path.class, HLog.class, FileSystem.class,
+              Configuration.class, HRegionInfo.class, HTableDescriptor.class,
+              RegionServerServices.class);
+
+      return c.newInstance(tableDir, log, fs, conf, regionInfo, htd, rsServices);
+    } catch (Throwable e) {
+      // todo: what should I throw here?
+      throw new IllegalStateException("Could not instantiate a region instance.", e);
+    }
+  }
+
+  /**
+   * Convenience method creating new HRegions. Used by createTable and by the
+   * bootstrap code in the HMaster constructor.
+   * Note, this method creates an {@link HLog} for the created region. It
+   * needs to be closed explicitly.  Use {@link HRegion#getLog()} to get
+   * access.  When done with a region created using this method, you will
+   * need to explicitly close the {@link HLog} it created too; it will not be
+   * done for you.  Not closing the log will leave at least a daemon thread
+   * running.  Call {@link #closeHRegion(HRegion)} and it will do
+   * necessary cleanup for you.
+   * @param info Info for region to create.
+   * @param rootDir Root directory for HBase instance
+   * @param conf
+   * @param hTableDescriptor
+   * @return new HRegion
+   *
+   * @throws IOException
+   */
+  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
+      final Configuration conf, final HTableDescriptor hTableDescriptor)
+  throws IOException {
+    return createHRegion(info, rootDir, conf, hTableDescriptor, null);
+  }
+
+  /**
+   * This will do the necessary cleanup a call to {@link #createHRegion(HRegionInfo, Path, Configuration, HTableDescriptor)}
+   * requires.  This method will close the region and then close its
+   * associated {@link HLog} file.  You use it if you call the other createHRegion,
+   * the one that takes an {@link HLog} instance but don't be surprised by the
+   * call to the {@link HLog#closeAndDelete()} on the {@link HLog} the
+   * HRegion was carrying.
+   * @param r
+   * @throws IOException
+   */
+  public static void closeHRegion(final HRegion r) throws IOException {
+    if (r == null) return;
+    r.close();
+    if (r.getLog() == null) return;
+    r.getLog().closeAndDelete();
+  }
+
+  /**
+   * Convenience method creating new HRegions. Used by createTable.
+   * The {@link HLog} for the created region needs to be closed explicitly.
+   * Use {@link HRegion#getLog()} to get access.
+   *
+   * @param info Info for region to create.
+   * @param rootDir Root directory for HBase instance
+   * @param conf
+   * @param hTableDescriptor
+   * @param hlog shared HLog
+   * @param boolean initialize - true to initialize the region
+   * @return new HRegion
+   *
+   * @throws IOException
+   */
+  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
+                                      final Configuration conf,
+                                      final HTableDescriptor hTableDescriptor,
+                                      final HLog hlog,
+                                      final boolean initialize)
+      throws IOException {
+    return createHRegion(info, rootDir, conf, hTableDescriptor,
+        hlog, initialize, false);
+  }
+
+  /**
+   * Convenience method creating new HRegions. Used by createTable.
+   * The {@link HLog} for the created region needs to be closed
+   * explicitly, if it is not null.
+   * Use {@link HRegion#getLog()} to get access.
+   *
+   * @param info Info for region to create.
+   * @param rootDir Root directory for HBase instance
+   * @param conf
+   * @param hTableDescriptor
+   * @param hlog shared HLog
+   * @param boolean initialize - true to initialize the region
+   * @param boolean ignoreHLog
+      - true to skip generate new hlog if it is null, mostly for createTable
+   * @return new HRegion
+   *
+   * @throws IOException
+   */
+  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
+                                      final Configuration conf,
+                                      final HTableDescriptor hTableDescriptor,
+                                      final HLog hlog,
+                                      final boolean initialize, final boolean ignoreHLog)
+      throws IOException {
+    LOG.info("creating HRegion " + info.getTableNameAsString()
+        + " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
+        " Table name == " + info.getTableNameAsString());
+
+    Path tableDir =
+        HTableDescriptor.getTableDir(rootDir, info.getTableName());
+    Path regionDir = HRegion.getRegionDir(tableDir, info.getEncodedName());
+    FileSystem fs = FileSystem.get(conf);
+    HBaseFileSystem.makeDirOnFileSystem(fs, regionDir);
+    // Write HRI to a file in case we need to recover .META.
+    writeRegioninfoOnFilesystem(info, regionDir, fs, conf);
+    HLog effectiveHLog = hlog;
+    if (hlog == null && !ignoreHLog) {
+      effectiveHLog = new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
+          new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf);
+    }
+    HRegion region = HRegion.newHRegion(tableDir,
+        effectiveHLog, fs, conf, info, hTableDescriptor, null);
+    if (initialize) {
+      region.initialize();
+    }
+    return region;
+  }
+
+  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
+                                      final Configuration conf,
+                                      final HTableDescriptor hTableDescriptor,
+                                      final HLog hlog)
+    throws IOException {
+    return createHRegion(info, rootDir, conf, hTableDescriptor, hlog, true);
+  }
+
+  /**
+   * Open a Region.
+   * @param info Info for region to be opened.
+   * @param wal HLog for region to use. This method will call
+   * HLog#setSequenceNumber(long) passing the result of the call to
+   * HRegion#getMinSequenceId() to ensure the log id is properly kept
+   * up.  HRegionStore does this every time it opens a new region.
+   * @param conf
+   * @return new HRegion
+   *
+   * @throws IOException
+   */
+  public static HRegion openHRegion(final HRegionInfo info,
+      final HTableDescriptor htd, final HLog wal,
+      final Configuration conf)
+  throws IOException {
+    return openHRegion(info, htd, wal, conf, null, null);
+  }
+
+  /**
+   * Open a Region.
+   * @param info Info for region to be opened
+   * @param htd
+   * @param wal HLog for region to use. This method will call
+   * HLog#setSequenceNumber(long) passing the result of the call to
+   * HRegion#getMinSequenceId() to ensure the log id is properly kept
+   * up.  HRegionStore does this every time it opens a new region.
+   * @param conf
+   * @param rsServices An interface we can request flushes against.
+   * @param reporter An interface we can report progress against.
+   * @return new HRegion
+   *
+   * @throws IOException
+   */
+  public static HRegion openHRegion(final HRegionInfo info,
+    final HTableDescriptor htd, final HLog wal, final Configuration conf,
+    final RegionServerServices rsServices,
+    final CancelableProgressable reporter)
+  throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Opening region: " + info);
+    }
+    if (info == null) {
+      throw new NullPointerException("Passed region info is null");
+    }
+    Path dir = HTableDescriptor.getTableDir(FSUtils.getRootDir(conf),
+      info.getTableName());
+    FileSystem fs = null;
+    if (rsServices != null) {
+      fs = rsServices.getFileSystem();
+    }
+    if (fs == null) {
+      fs = FileSystem.get(conf);
+    }
+    HRegion r = HRegion.newHRegion(dir, wal, fs, conf, info,
+      htd, rsServices);
+    return r.openHRegion(reporter);
+  }
+
+  public static HRegion openHRegion(Path tableDir, final HRegionInfo info,
+      final HTableDescriptor htd, final HLog wal, final Configuration conf)
+  throws IOException {
+    return openHRegion(tableDir, info, htd, wal, conf, null, null);
+  }
+
+  /**
+   * Open a Region.
+   * @param tableDir Table directory
+   * @param info Info for region to be opened.
+   * @param wal HLog for region to use. This method will call
+   * HLog#setSequenceNumber(long) passing the result of the call to
+   * HRegion#getMinSequenceId() to ensure the log id is properly kept
+   * up.  HRegionStore does this every time it opens a new region.
+   * @param conf
+   * @param reporter An interface we can report progress against.
+   * @return new HRegion
+   *
+   * @throws IOException
+   */
+  public static HRegion openHRegion(final Path tableDir, final HRegionInfo info,
+      final HTableDescriptor htd, final HLog wal, final Configuration conf,
+      final RegionServerServices rsServices,
+      final CancelableProgressable reporter)
+  throws IOException {
+    if (info == null) throw new NullPointerException("Passed region info is null");
+    LOG.info("HRegion.openHRegion Region name ==" + info.getRegionNameAsString());
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Opening region: " + info);
+    }
+    Path dir = HTableDescriptor.getTableDir(tableDir,
+        info.getTableName());
+    HRegion r = HRegion.newHRegion(dir, wal, FileSystem.get(conf), conf, info,
+        htd, rsServices);
+    return r.openHRegion(reporter);
+  }
+
+
+  /**
+   * Open HRegion.
+   * Calls initialize and sets sequenceid.
+   * @param reporter
+   * @return Returns this
+   * @throws IOException
+   */
+  protected HRegion openHRegion(final CancelableProgressable reporter)
+  throws IOException {
+    checkCompressionCodecs();
+
+    long seqid = initialize(reporter);
+    if (this.log != null) {
+      this.log.setSequenceNumber(seqid);
+    }
+    return this;
+  }
+
+  private void checkCompressionCodecs() throws IOException {
+    for (HColumnDescriptor fam: this.htableDescriptor.getColumnFamilies()) {
+      CompressionTest.testCompression(fam.getCompression());
+      CompressionTest.testCompression(fam.getCompactionCompression());
+    }
+  }
+
+  /**
+   * Inserts a new region's meta information into the passed
+   * meta region. Used by the HMaster bootstrap code adding
+   * new table to ROOT table.
+   *
+   * @param meta META HRegion to be updated
+   * @param r HRegion to add to meta
+   *
+   * @throws IOException
+   */
+  public static void addRegionToMETA(HRegion meta, HRegion r)
+  throws IOException {
+    meta.checkResources();
+    // The row key is the region name
+    byte[] row = r.getRegionName();
+    Integer lid = meta.obtainRowLock(row);
+    try {
+      final long now = EnvironmentEdgeManager.currentTimeMillis();
+      final List edits = new ArrayList(2);
+      edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
+        HConstants.REGIONINFO_QUALIFIER, now,
+        Writables.getBytes(r.getRegionInfo())));
+      // Set into the root table the version of the meta table.
+      edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
+        HConstants.META_VERSION_QUALIFIER, now,
+        Bytes.toBytes(HConstants.META_VERSION)));
+      meta.put(HConstants.CATALOG_FAMILY, edits, lid);
+    } finally {
+      meta.releaseRowLock(lid);
+    }
+  }
+
+  /**
+   * Deletes all the files for a HRegion
+   *
+   * @param fs the file system object
+   * @param rootdir qualified path of HBase root directory
+   * @param info HRegionInfo for region to be deleted
+   * @throws IOException
+   */
+  public static void deleteRegion(FileSystem fs, Path rootdir, HRegionInfo info)
+  throws IOException {
+    deleteRegion(fs, HRegion.getRegionDir(rootdir, info));
+  }
+
+  private static void deleteRegion(FileSystem fs, Path regiondir)
+  throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("DELETING region " + regiondir.toString());
+    }
+    if (!HBaseFileSystem.deleteDirFromFileSystem(fs, regiondir)) {
+      LOG.warn("Failed delete of " + regiondir);
+    }
+  }
+
+  /**
+   * Computes the Path of the HRegion
+   *
+   * @param rootdir qualified path of HBase root directory
+   * @param info HRegionInfo for the region
+   * @return qualified path of region directory
+   */
+  public static Path getRegionDir(final Path rootdir, final HRegionInfo info) {
+    return new Path(
+      HTableDescriptor.getTableDir(rootdir, info.getTableName()),
+                                   info.getEncodedName());
+  }
+
+  /**
+   * Determines if the specified row is within the row range specified by the
+   * specified HRegionInfo
+   *
+   * @param info HRegionInfo that specifies the row range
+   * @param row row to be checked
+   * @return true if the row is within the range specified by the HRegionInfo
+   */
+  public static boolean rowIsInRange(HRegionInfo info, final byte [] row) {
+    return ((info.getStartKey().length == 0) ||
+        (Bytes.compareTo(info.getStartKey(), row) <= 0)) &&
+        ((info.getEndKey().length == 0) ||
+            (Bytes.compareTo(info.getEndKey(), row) > 0));
+  }
+
+  /**
+   * Make the directories for a specific column family
+   *
+   * @param fs the file system
+   * @param tabledir base directory where region will live (usually the table dir)
+   * @param hri
+   * @param colFamily the column family
+   * @throws IOException
+   */
+  public static void makeColumnFamilyDirs(FileSystem fs, Path tabledir,
+    final HRegionInfo hri, byte [] colFamily)
+  throws IOException {
+    Path dir = Store.getStoreHomedir(tabledir, hri.getEncodedName(), colFamily);
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, dir)) {
+      LOG.warn("Failed to create " + dir);
+    }
+  }
+
+  /**
+   * Merge two HRegions.  The regions must be adjacent and must not overlap.
+   *
+   * @param srcA
+   * @param srcB
+   * @return new merged HRegion
+   * @throws IOException
+   */
+  public static HRegion mergeAdjacent(final HRegion srcA, final HRegion srcB)
+  throws IOException {
+    HRegion a = srcA;
+    HRegion b = srcB;
+
+    // Make sure that srcA comes first; important for key-ordering during
+    // write of the merged file.
+    if (srcA.getStartKey() == null) {
+      if (srcB.getStartKey() == null) {
+        throw new IOException("Cannot merge two regions with null start key");
+      }
+      // A's start key is null but B's isn't. Assume A comes before B
+    } else if ((srcB.getStartKey() == null) ||
+      (Bytes.compareTo(srcA.getStartKey(), srcB.getStartKey()) > 0)) {
+      a = srcB;
+      b = srcA;
+    }
+
+    if (!(Bytes.compareTo(a.getEndKey(), b.getStartKey()) == 0)) {
+      throw new IOException("Cannot merge non-adjacent regions");
+    }
+    return merge(a, b);
+  }
+
+  /**
+   * Merge two regions whether they are adjacent or not.
+   *
+   * @param a region a
+   * @param b region b
+   * @return new merged region
+   * @throws IOException
+   */
+  public static HRegion merge(HRegion a, HRegion b)
+  throws IOException {
+    if (!a.getRegionInfo().getTableNameAsString().equals(
+        b.getRegionInfo().getTableNameAsString())) {
+      throw new IOException("Regions do not belong to the same table");
+    }
+
+    FileSystem fs = a.getFilesystem();
+
+    // Make sure each region's cache is empty
+
+    a.flushcache();
+    b.flushcache();
+
+    // Compact each region so we only have one store file per family
+
+    a.compactStores(true);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Files for region: " + a);
+      listPaths(fs, a.getRegionDir());
+    }
+    b.compactStores(true);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Files for region: " + b);
+      listPaths(fs, b.getRegionDir());
+    }
+
+    Configuration conf = a.getBaseConf();
+    HTableDescriptor tabledesc = a.getTableDesc();
+    HLog log = a.getLog();
+    Path tableDir = a.getTableDir();
+    // Presume both are of same region type -- i.e. both user or catalog
+    // table regions.  This way can use comparator.
+    final byte[] startKey =
+      (a.comparator.matchingRows(a.getStartKey(), 0, a.getStartKey().length,
+           HConstants.EMPTY_BYTE_ARRAY, 0, HConstants.EMPTY_BYTE_ARRAY.length)
+       || b.comparator.matchingRows(b.getStartKey(), 0,
+              b.getStartKey().length, HConstants.EMPTY_BYTE_ARRAY, 0,
+              HConstants.EMPTY_BYTE_ARRAY.length))
+      ? HConstants.EMPTY_BYTE_ARRAY
+      : (a.comparator.compareRows(a.getStartKey(), 0, a.getStartKey().length,
+             b.getStartKey(), 0, b.getStartKey().length) <= 0
+         ? a.getStartKey()
+         : b.getStartKey());
+    final byte[] endKey =
+      (a.comparator.matchingRows(a.getEndKey(), 0, a.getEndKey().length,
+           HConstants.EMPTY_BYTE_ARRAY, 0, HConstants.EMPTY_BYTE_ARRAY.length)
+       || a.comparator.matchingRows(b.getEndKey(), 0, b.getEndKey().length,
+              HConstants.EMPTY_BYTE_ARRAY, 0,
+              HConstants.EMPTY_BYTE_ARRAY.length))
+      ? HConstants.EMPTY_BYTE_ARRAY
+      : (a.comparator.compareRows(a.getEndKey(), 0, a.getEndKey().length,
+             b.getEndKey(), 0, b.getEndKey().length) <= 0
+         ? b.getEndKey()
+         : a.getEndKey());
+
+    HRegionInfo newRegionInfo =
+        new HRegionInfo(tabledesc.getName(), startKey, endKey);
+    LOG.info("Creating new region " + newRegionInfo.toString());
+    String encodedName = newRegionInfo.getEncodedName();
+    Path newRegionDir = HRegion.getRegionDir(a.getTableDir(), encodedName);
+    if(fs.exists(newRegionDir)) {
+      throw new IOException("Cannot merge; target file collision at " +
+          newRegionDir);
+    }
+    HBaseFileSystem.makeDirOnFileSystem(fs, newRegionDir);
+
+    LOG.info("starting merge of regions: " + a + " and " + b +
+      " into new region " + newRegionInfo.toString() +
+        " with start key <" + Bytes.toStringBinary(startKey) + "> and end key <" +
+        Bytes.toStringBinary(endKey) + ">");
+
+    // Move HStoreFiles under new region directory
+    Map> byFamily =
+      new TreeMap>(Bytes.BYTES_COMPARATOR);
+    byFamily = filesByFamily(byFamily, a.close());
+    byFamily = filesByFamily(byFamily, b.close());
+    for (Map.Entry> es : byFamily.entrySet()) {
+      byte [] colFamily = es.getKey();
+      makeColumnFamilyDirs(fs, tableDir, newRegionInfo, colFamily);
+      // Because we compacted the source regions we should have no more than two
+      // HStoreFiles per family and there will be no reference store
+      List srcFiles = es.getValue();
+      if (srcFiles.size() == 2) {
+        long seqA = srcFiles.get(0).getMaxSequenceId();
+        long seqB = srcFiles.get(1).getMaxSequenceId();
+        if (seqA == seqB) {
+          // Can't have same sequenceid since on open of a store, this is what
+          // distingushes the files (see the map of stores how its keyed by
+          // sequenceid).
+          throw new IOException("Files have same sequenceid: " + seqA);
+        }
+      }
+      for (StoreFile hsf: srcFiles) {
+        StoreFile.rename(fs, hsf.getPath(),
+          StoreFile.getUniqueFile(fs, Store.getStoreHomedir(tableDir,
+            newRegionInfo.getEncodedName(), colFamily)));
+      }
+    }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Files for new region");
+      listPaths(fs, newRegionDir);
+    }
+    HRegion dstRegion = HRegion.newHRegion(tableDir, log, fs, conf,
+        newRegionInfo, a.getTableDesc(), null);
+    long totalReadRequestCount = a.readRequestsCount.get() + b.readRequestsCount.get();
+    dstRegion.readRequestsCount.set(totalReadRequestCount);
+    dstRegion.opMetrics.setReadRequestCountMetrics(totalReadRequestCount);
+    
+    long totalWriteRequestCount = a.writeRequestsCount.get() + b.writeRequestsCount.get();
+    dstRegion.writeRequestsCount.set(totalWriteRequestCount);
+    dstRegion.opMetrics.setWriteRequestCountMetrics(totalWriteRequestCount);
+    
+    dstRegion.initialize();
+    dstRegion.compactStores();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Files for new region");
+      listPaths(fs, dstRegion.getRegionDir());
+    }
+
+    // delete out the 'A' region
+    HFileArchiver.archiveRegion(fs, FSUtils.getRootDir(a.getConf()),
+        a.getTableDir(), a.getRegionDir());
+    // delete out the 'B' region
+    HFileArchiver.archiveRegion(fs, FSUtils.getRootDir(b.getConf()),
+        b.getTableDir(), b.getRegionDir());
+
+    LOG.info("merge completed. New region is " + dstRegion);
+
+    return dstRegion;
+  }
+
+  /*
+   * Fills a map with a vector of store files keyed by column family.
+   * @param byFamily Map to fill.
+   * @param storeFiles Store files to process.
+   * @param family
+   * @return Returns byFamily
+   */
+  private static Map> filesByFamily(
+      Map> byFamily, List storeFiles) {
+    for (StoreFile src: storeFiles) {
+      byte [] family = src.getFamily();
+      List v = byFamily.get(family);
+      if (v == null) {
+        v = new ArrayList();
+        byFamily.put(family, v);
+      }
+      v.add(src);
+    }
+    return byFamily;
+  }
+
+  /**
+   * @return True if needs a mojor compaction.
+   * @throws IOException
+   */
+  boolean isMajorCompaction() throws IOException {
+    for (Store store: this.stores.values()) {
+      if (store.isMajorCompaction()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /*
+   * List the files under the specified directory
+   *
+   * @param fs
+   * @param dir
+   * @throws IOException
+   */
+  private static void listPaths(FileSystem fs, Path dir) throws IOException {
+    if (LOG.isDebugEnabled()) {
+      FileStatus[] stats = FSUtils.listStatus(fs, dir, null);
+      if (stats == null || stats.length == 0) {
+        return;
+      }
+      for (int i = 0; i < stats.length; i++) {
+        String path = stats[i].getPath().toString();
+        if (stats[i].isDir()) {
+          LOG.debug("d " + path);
+          listPaths(fs, stats[i].getPath());
+        } else {
+          LOG.debug("f " + path + " size=" + stats[i].getLen());
+        }
+      }
+    }
+  }
+
+
+  //
+  // HBASE-880
+  //
+  /**
+   * @param get get object
+   * @return result
+   * @throws IOException read exceptions
+   */
+  public Result get(final Get get) throws IOException {
+    return get(get, null);
+  }
+
+  /**
+   * @param get get object
+   * @param lockid existing lock id, or null for no previous lock
+   * @return result
+   * @throws IOException read exceptions
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
+   */
+  public Result get(final Get get, final Integer lockid) throws IOException {
+    checkRow(get.getRow(), "Get");
+    // Verify families are all valid
+    if (get.hasFamilies()) {
+      for (byte [] family: get.familySet()) {
+        checkFamily(family);
+      }
+    } else { // Adding all families to scanner
+      for (byte[] family: this.htableDescriptor.getFamiliesKeys()) {
+        get.addFamily(family);
+      }
+    }
+    List results = get(get, true);
+    return new Result(results);
+  }
+
+  /*
+   * Do a get based on the get parameter.
+   * @param withCoprocessor invoke coprocessor or not. We don't want to
+   * always invoke cp for this private method.
+   */
+  private List get(Get get, boolean withCoprocessor)
+  throws IOException {
+    long now = EnvironmentEdgeManager.currentTimeMillis();
+
+    List results = new ArrayList();
+
+    // pre-get CP hook
+    if (withCoprocessor && (coprocessorHost != null)) {
+       if (coprocessorHost.preGet(get, results)) {
+         return results;
+       }
+    }
+
+    Scan scan = new Scan(get);
+
+    RegionScanner scanner = null;
+    try {
+      scanner = getScanner(scan);
+      scanner.next(results, SchemaMetrics.METRIC_GETSIZE);
+    } finally {
+      if (scanner != null)
+        scanner.close();
+    }
+
+    // post-get CP hook
+    if (withCoprocessor && (coprocessorHost != null)) {
+      coprocessorHost.postGet(get, results);
+    }
+
+    // do after lock
+    final long after = EnvironmentEdgeManager.currentTimeMillis();
+    this.opMetrics.updateGetMetrics(get.familySet(), after - now);
+
+    return results;
+  }
+
+  public void mutateRow(RowMutations rm) throws IOException {
+    mutateRowsWithLocks(rm.getMutations(), Collections.singleton(rm.getRow()));
+  }
+
+  /**
+   * Perform atomic mutations within the region.
+   * @param mutationsList The list of mutations to perform.
+   * mutations can contain operations for multiple rows.
+   * Caller has to ensure that all rows are contained in this region.
+   * @param rowsToLock Rows to lock
+   * If multiple rows are locked care should be taken that
+   * rowsToLock is sorted in order to avoid deadlocks.
+   * @throws IOException
+   */
+  public void mutateRowsWithLocks(Collection mutationsList,
+      Collection rowsToLock) throws IOException {
+    boolean flush = false;
+
+    checkReadOnly();
+    checkResources();
+
+    startRegionOperation();
+    List acquiredLocks = null;
+    try {
+      // 1. run all pre-hooks before the atomic operation
+      // if any pre hook indicates "bypass", bypass the entire operation
+
+      // one WALEdit is used for all edits.
+      WALEdit walEdit = new WALEdit();
+      if (coprocessorHost != null) {
+        for (Mutation m : mutationsList) {
+          if (m instanceof Put) {
+            if (coprocessorHost.prePut((Put) m, walEdit, m.getWriteToWAL())) {
+              // by pass everything
+              return;
+            }
+          } else if (m instanceof Delete) {
+            Delete d = (Delete) m;
+            prepareDelete(d);
+            if (coprocessorHost.preDelete(d, walEdit, d.getWriteToWAL())) {
+              // by pass everything
+              return;
+            }
+          }
+        }
+      }
+
+      long txid = 0;
+      boolean walSyncSuccessful = false;
+      boolean locked = false;
+
+      // 2. acquire the row lock(s)
+      acquiredLocks = new ArrayList(rowsToLock.size());
+      Map locksMap = new TreeMap(Bytes.BYTES_COMPARATOR);
+      for (byte[] row : rowsToLock) {
+        // attempt to lock all involved rows, fail if one lock times out
+        Integer lid = getLock(null, row, true);
+        if (lid == null) {
+          throw new IOException("Failed to acquire lock on "
+              + Bytes.toStringBinary(row));
+        }
+        locksMap.put(row, lid);
+        acquiredLocks.add(lid);
+      }
+
+      // 3. acquire the region lock
+      lock(this.updatesLock.readLock(), acquiredLocks.size());
+      locked = true;
+
+      // 4. Get a mvcc write number
+      MultiVersionConsistencyControl.WriteEntry w = mvcc.beginMemstoreInsert();
+
+      BatchOperationInProgress> batchOp = null;
+      if (coprocessorHost != null) {
+        Pair putsAndLocks[] = new Pair[mutationsList.size()];
+        int count = 0;
+        for (Mutation m : mutationsList) {
+          Integer rowLock = locksMap.get(m.getRow());
+          putsAndLocks[count] = new Pair(m, rowLock);
+          ++count;
+        }
+        batchOp = new BatchOperationInProgress>(putsAndLocks);
+        List> mutationVsBatchOp =
+            new ArrayList>();
+        for (int i = 0; i < mutationsList.size(); i++) {
+          mutationVsBatchOp.add(new Pair(batchOp.operations[i]
+              .getFirst(), batchOp.retCodeDetails[i]));
+        }
+        if (coprocessorHost.preBatchMutate(mutationVsBatchOp, walEdit)) {
+          return;
+        }
+        boolean fullBatchFailed = true;
+        for (int i = 0; i < mutationVsBatchOp.size(); i++) {
+          OperationStatus opStatus = mutationVsBatchOp.get(i).getSecond();
+          if (opStatus.getOperationStatusCode() != OperationStatusCode.NOT_RUN) {
+            batchOp.retCodeDetails[i] = mutationVsBatchOp.get(i).getSecond();
+          } else {
+            fullBatchFailed = false;
+          }
+        }
+        if (fullBatchFailed) {
+          return;
+        }
+      }
+
+      long now = EnvironmentEdgeManager.currentTimeMillis();
+      byte[] byteNow = Bytes.toBytes(now);
+      Durability durability = Durability.USE_DEFAULT;
+      try {
+        // 5. Check mutations and apply edits to a single WALEdit
+        for (Mutation m : mutationsList) {
+          if (m instanceof Put) {
+            Map> familyMap = m.getFamilyMap();
+            checkFamilies(familyMap.keySet());
+            checkTimestamps(familyMap, now);
+            updateKVTimestamps(familyMap.values(), byteNow);
+          } else if (m instanceof Delete) {
+            Delete d = (Delete) m;
+            prepareDelete(d);
+            prepareDeleteTimestamps(d.getFamilyMap(), byteNow);
+          } else {
+            throw new DoNotRetryIOException(
+                "Action must be Put or Delete. But was: "
+                    + m.getClass().getName());
+          }
+          Durability tmpDur = m.getDurability(); 
+          if (tmpDur.ordinal() > durability.ordinal()) {
+            durability = tmpDur;
+          }
+          if (tmpDur != Durability.SKIP_WAL) {
+            addFamilyMapToWALEdit(m.getFamilyMap(), walEdit);
+          }
+        }
+
+        if (coprocessorHost != null) {
+          for (int i = 0; i < mutationsList.size(); i++) {
+            batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
+          }
+        }
+
+        // 6. append all edits at once (don't sync)
+        if (walEdit.size() > 0) {
+          txid = this.log.appendNoSync(regionInfo,
+              this.htableDescriptor.getName(), walEdit,
+              HConstants.DEFAULT_CLUSTER_ID, now, this.htableDescriptor);
+        }
+
+        // 7. apply to memstore
+        long addedSize = 0;
+        for (Mutation m : mutationsList) {
+          addedSize += applyFamilyMapToMemstore(m.getFamilyMap(), w);
+        }
+        flush = isFlushSize(this.addAndGetGlobalMemstoreSize(addedSize));
+
+        // 8. release region and row lock(s)
+        this.updatesLock.readLock().unlock();
+        locked = false;
+        if (acquiredLocks != null) {
+          for (Integer lid : acquiredLocks) {
+            releaseRowLock(lid);
+          }
+          acquiredLocks = null;
+        }
+
+        // 9. sync WAL if required
+        if (walEdit.size() > 0) {
+          syncOrDefer(txid, durability);
+        }
+        walSyncSuccessful = true;
+
+        if (coprocessorHost != null) {
+          List mutations = new ArrayList();
+          for (int i = 0; i < mutationsList.size(); i++) {
+            // only for successful puts
+            if (batchOp.retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) {
+              continue;
+            }
+            Mutation m = batchOp.operations[i].getFirst();
+            mutations.add(m);
+          }
+          coprocessorHost.postBatchMutate(mutations, walEdit);
+        }
+
+        // 10. advance mvcc
+        mvcc.completeMemstoreInsert(w);
+        w = null;
+
+        // 11. run coprocessor post host hooks
+        // after the WAL is sync'ed and all locks are released
+        // (similar to doMiniBatchPut)
+        if (coprocessorHost != null) {
+          for (Mutation m : mutationsList) {
+            if (m instanceof Put) {
+              coprocessorHost.postPut((Put) m, walEdit, m.getWriteToWAL());
+            } else if (m instanceof Delete) {
+              coprocessorHost.postDelete((Delete) m, walEdit, m.getWriteToWAL());
+            }
+          }
+        }
+      } finally {
+        // 12. clean up if needed
+        if (!walSyncSuccessful) {
+          int kvsRolledback = 0;
+          for (Mutation m : mutationsList) {
+            for (Map.Entry> e : m.getFamilyMap()
+                .entrySet()) {
+              List kvs = e.getValue();
+              byte[] family = e.getKey();
+              Store store = getStore(family);
+              // roll back each kv
+              for (KeyValue kv : kvs) {
+                store.rollback(kv);
+                kvsRolledback++;
+              }
+            }
+          }
+          LOG.info("mutateRowWithLocks: rolled back " + kvsRolledback
+              + " KeyValues");
+        }
+
+        if (w != null) {
+          mvcc.completeMemstoreInsert(w);
+        }
+
+        if (locked) {
+          this.updatesLock.readLock().unlock();
+        }
+
+        if (acquiredLocks != null) {
+          for (Integer lid : acquiredLocks) {
+            releaseRowLock(lid);
+          }
+        }
+
+        if (coprocessorHost != null) {
+          List mutations = new ArrayList();
+          for (int i = 0; i < mutationsList.size(); i++) {
+            // only for successful puts
+            if (batchOp.retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) {
+              continue;
+            }
+            Mutation m = batchOp.operations[i].getFirst();
+            mutations.add(m);
+          }
+          coprocessorHost.postCompleteBatchMutate(mutations);
+        }
+      }
+    } finally {
+      if (flush) {
+        // 13. Flush cache if needed. Do it outside update lock.
+        requestFlush();
+      }
+      closeRegionOperation();
+    }
+  }
+
+  // TODO: There's a lot of boiler plate code identical
+  // to increment... See how to better unify that.
+
+  /**
+  *
+  * Perform one or more append operations on a row.
+  * 
                
+  * Appends performed are done under row lock but reads do not take locks out
+  * so this can be seen partially complete by gets and scans.
+  *
+  * @param append
+  * @param writeToWAL
+  * @return new keyvalues after increment
+  * @throws IOException
+  */
+ public Result append(Append append, boolean writeToWAL)
+     throws IOException {
+   return append(append, null, writeToWAL);
+ }
+  /**
+   *
+   * Perform one or more append operations on a row.
+   * 
                
+   * Appends performed are done under row lock but reads do not take locks out
+   * so this can be seen partially complete by gets and scans.
+   *
+   * @param append
+   * @param lockid
+   * @param writeToWAL
+   * @return new keyvalues after increment
+   * @throws IOException
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
+   */
+  public Result append(Append append, Integer lockid, boolean writeToWAL)
+      throws IOException {
+    // TODO: Use MVCC to make this set of appends atomic to reads
+    byte[] row = append.getRow();
+    checkRow(row, "append");
+    boolean flush = false;
+    WALEdit walEdits = null;
+    List allKVs = new ArrayList(append.size());
+    Map> tempMemstore = new HashMap>();
+    long before = EnvironmentEdgeManager.currentTimeMillis();
+    long size = 0;
+    long txid = 0;
+
+    checkReadOnly();
+    // Lock row
+    startRegionOperation();
+    this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
+    try {
+      Integer lid = getLock(lockid, row, true);
+      lock(this.updatesLock.readLock());
+      try {
+        long now = EnvironmentEdgeManager.currentTimeMillis();
+        // Process each family
+        for (Map.Entry> family : append.getFamilyMap()
+            .entrySet()) {
+
+          Store store = stores.get(family.getKey());
+          List kvs = new ArrayList(family.getValue().size());
+
+          // Get previous values for all columns in this family
+          Get get = new Get(row);
+          for (KeyValue kv : family.getValue()) {
+            get.addColumn(family.getKey(), kv.getQualifier());
+          }
+          List results = get(get, false);
+
+          // Iterate the input columns and update existing values if they were
+          // found, otherwise add new column initialized to the append value
+
+          // Avoid as much copying as possible. Every byte is copied at most
+          // once.
+          // Would be nice if KeyValue had scatter/gather logic
+          int idx = 0;
+          for (KeyValue kv : family.getValue()) {
+            KeyValue newKV;
+            if (idx < results.size()
+                && results.get(idx).matchingQualifier(kv.getBuffer(),
+                    kv.getQualifierOffset(), kv.getQualifierLength())) {
+              KeyValue oldKv = results.get(idx);
+              // allocate an empty kv once
+              newKV = new KeyValue(row.length, kv.getFamilyLength(),
+                  kv.getQualifierLength(), now, KeyValue.Type.Put,
+                  oldKv.getValueLength() + kv.getValueLength());
+              // copy in the value
+              System.arraycopy(oldKv.getBuffer(), oldKv.getValueOffset(),
+                  newKV.getBuffer(), newKV.getValueOffset(),
+                  oldKv.getValueLength());
+              System.arraycopy(kv.getBuffer(), kv.getValueOffset(),
+                  newKV.getBuffer(),
+                  newKV.getValueOffset() + oldKv.getValueLength(),
+                  kv.getValueLength());
+              idx++;
+            } else {
+              // allocate an empty kv once
+              newKV = new KeyValue(row.length, kv.getFamilyLength(),
+                  kv.getQualifierLength(), now, KeyValue.Type.Put,
+                  kv.getValueLength());
+              // copy in the value
+              System.arraycopy(kv.getBuffer(), kv.getValueOffset(),
+                  newKV.getBuffer(), newKV.getValueOffset(),
+                  kv.getValueLength());
+            }
+            // copy in row, family, and qualifier
+            System.arraycopy(kv.getBuffer(), kv.getRowOffset(),
+                newKV.getBuffer(), newKV.getRowOffset(), kv.getRowLength());
+            System.arraycopy(kv.getBuffer(), kv.getFamilyOffset(),
+                newKV.getBuffer(), newKV.getFamilyOffset(),
+                kv.getFamilyLength());
+            System.arraycopy(kv.getBuffer(), kv.getQualifierOffset(),
+                newKV.getBuffer(), newKV.getQualifierOffset(),
+                kv.getQualifierLength());
+
+            kvs.add(newKV);
+
+            // Append update to WAL
+            if (writeToWAL) {
+              if (walEdits == null) {
+                walEdits = new WALEdit();
+              }
+              walEdits.add(newKV);
+            }
+          }
+
+          // store the kvs to the temporary memstore before writing HLog
+          tempMemstore.put(store, kvs);
+        }
+
+        // Actually write to WAL now
+        if (writeToWAL) {
+          // Using default cluster id, as this can only happen in the orginating
+          // cluster. A slave cluster receives the final value (not the delta)
+          // as a Put.
+          txid = this.log.appendNoSync(regionInfo,
+              this.htableDescriptor.getName(), walEdits,
+              HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(),
+              this.htableDescriptor);
+        }
+        // Actually write to Memstore now
+        for (Map.Entry> entry : tempMemstore.entrySet()) {
+          Store store = entry.getKey();
+          size += store.upsert(entry.getValue());
+          allKVs.addAll(entry.getValue());
+        }
+        size = this.addAndGetGlobalMemstoreSize(size);
+        flush = isFlushSize(size);
+      } finally {
+        this.updatesLock.readLock().unlock();
+        releaseRowLock(lid);
+      }
+      if (writeToWAL) {
+        // sync the transaction log outside the rowlock
+        syncOrDefer(txid, append.getDurability());
+      }
+    } finally {
+      closeRegionOperation();
+    }
+
+
+    long after = EnvironmentEdgeManager.currentTimeMillis();
+    this.opMetrics.updateAppendMetrics(append.getFamilyMap().keySet(), after - before);
+
+    if (flush) {
+      // Request a cache flush. Do it outside update lock.
+      requestFlush();
+    }
+
+    return append.isReturnResults() ? new Result(allKVs) : null;
+  }
+
+  /**
+  *
+  * Perform one or more increment operations on a row.
+  * 
                
+  * Increments performed are done under row lock but reads do not take locks
+  * out so this can be seen partially complete by gets and scans.
+  * @param increment
+  * @param writeToWAL
+  * @return new keyvalues after increment
+  * @throws IOException
+  */
+  public Result increment(Increment increment, boolean writeToWAL)
+  throws IOException {
+    return increment(increment, null, writeToWAL);
+  }
+
+  /**
+   *
+   * Perform one or more increment operations on a row.
+   * 
                
+   * Increments performed are done under row lock but reads do not take locks
+   * out so this can be seen partially complete by gets and scans.
+   * @param increment
+   * @param lockid
+   * @param writeToWAL
+   * @return new keyvalues after increment
+   * @throws IOException
+   * @deprecated row locks (lockId) held outside the extent of the operation are deprecated.
+
+   */
+  public Result increment(Increment increment, Integer lockid,
+      boolean writeToWAL)
+  throws IOException {
+    // TODO: Use MVCC to make this set of increments atomic to reads
+    byte [] row = increment.getRow();
+    checkRow(row, "increment");
+    TimeRange tr = increment.getTimeRange();
+    boolean flush = false;
+    WALEdit walEdits = null;
+    List allKVs = new ArrayList(increment.numColumns());
+    Map> tempMemstore = new HashMap>();
+    long before = EnvironmentEdgeManager.currentTimeMillis();
+    long size = 0;
+    long txid = 0;
+
+    checkReadOnly();
+    // Lock row
+    startRegionOperation();
+    this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
+    try {
+      Integer lid = getLock(lockid, row, true);
+      lock(this.updatesLock.readLock());
+      try {
+        long now = EnvironmentEdgeManager.currentTimeMillis();
+        // Process each family
+        for (Map.Entry> family :
+          increment.getFamilyMap().entrySet()) {
+
+          Store store = stores.get(family.getKey());
+          List kvs = new ArrayList(family.getValue().size());
+
+          // Get previous values for all columns in this family
+          Get get = new Get(row);
+          for (Map.Entry column : family.getValue().entrySet()) {
+            get.addColumn(family.getKey(), column.getKey());
+          }
+          get.setTimeRange(tr.getMin(), tr.getMax());
+          List results = get(get, false);
+
+          // Iterate the input columns and update existing values if they were
+          // found, otherwise add new column initialized to the increment amount
+          int idx = 0;
+          for (Map.Entry column : family.getValue().entrySet()) {
+            long amount = column.getValue();
+            if (idx < results.size() &&
+                results.get(idx).matchingQualifier(column.getKey())) {
+              KeyValue kv = results.get(idx);
+              if(kv.getValueLength() == Bytes.SIZEOF_LONG) {
+                amount += Bytes.toLong(kv.getBuffer(), kv.getValueOffset(), Bytes.SIZEOF_LONG);
+              } else {
+                // throw DoNotRetryIOException instead of IllegalArgumentException
+                throw new DoNotRetryIOException(
+                    "Attempted to increment field that isn't 64 bits wide");
+              }
+              idx++;
+            }
+
+            // Append new incremented KeyValue to list
+            KeyValue newKV = new KeyValue(row, family.getKey(), column.getKey(),
+                now, Bytes.toBytes(amount));
+            kvs.add(newKV);
+
+            // Append update to WAL
+            if (writeToWAL) {
+              if (walEdits == null) {
+                walEdits = new WALEdit();
+              }
+              walEdits.add(newKV);
+            }
+          }
+
+          //store the kvs to the temporary memstore before writing HLog
+          tempMemstore.put(store, kvs);
+        }
+
+        // Actually write to WAL now
+        if (writeToWAL) {
+          // Using default cluster id, as this can only happen in the orginating
+          // cluster. A slave cluster receives the final value (not the delta)
+          // as a Put.
+          txid = this.log.appendNoSync(regionInfo, this.htableDescriptor.getName(),
+              walEdits, HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(),
+              this.htableDescriptor);
+        }
+
+        //Actually write to Memstore now
+        for (Map.Entry> entry : tempMemstore.entrySet()) {
+          Store store = entry.getKey();
+          size += store.upsert(entry.getValue());
+          allKVs.addAll(entry.getValue());
+        }
+        size = this.addAndGetGlobalMemstoreSize(size);
+        flush = isFlushSize(size);
+      } finally {
+        this.updatesLock.readLock().unlock();
+        releaseRowLock(lid);
+      }
+      if (writeToWAL) {
+        // sync the transaction log outside the rowlock
+        syncOrDefer(txid, Durability.USE_DEFAULT);
+      }
+    } finally {
+      closeRegionOperation();
+      long after = EnvironmentEdgeManager.currentTimeMillis();
+      this.opMetrics.updateIncrementMetrics(increment.getFamilyMap().keySet(), after - before);
+    }
+
+    if (flush) {
+      // Request a cache flush.  Do it outside update lock.
+      requestFlush();
+    }
+
+    return new Result(allKVs);
+  }
+
+  /**
+   * @param row
+   * @param family
+   * @param qualifier
+   * @param amount
+   * @param writeToWAL
+   * @return The new value.
+   * @throws IOException
+   */
+  public long incrementColumnValue(byte [] row, byte [] family,
+      byte [] qualifier, long amount, boolean writeToWAL)
+  throws IOException {
+    // to be used for metrics
+    long before = EnvironmentEdgeManager.currentTimeMillis();
+
+    checkRow(row, "increment");
+    boolean flush = false;
+    boolean wrongLength = false;
+    long txid = 0;
+    // Lock row
+    long result = amount;
+    startRegionOperation();
+    this.writeRequestsCount.increment();
+    this.opMetrics.setWriteRequestCountMetrics(this.writeRequestsCount.get());
+    try {
+      Integer lid = obtainRowLock(row);
+      lock(this.updatesLock.readLock());
+      try {
+        Store store = stores.get(family);
+
+        // Get the old value:
+        Get get = new Get(row);
+        get.addColumn(family, qualifier);
+
+        // we don't want to invoke coprocessor in this case; ICV is wrapped
+        // in HRegionServer, so we leave getLastIncrement alone
+        List results = get(get, false);
+
+        if (!results.isEmpty()) {
+          KeyValue kv = results.get(0);
+          if(kv.getValueLength() == Bytes.SIZEOF_LONG){
+            byte [] buffer = kv.getBuffer();
+            int valueOffset = kv.getValueOffset();
+            result += Bytes.toLong(buffer, valueOffset, Bytes.SIZEOF_LONG);
+          }
+          else{
+            wrongLength = true;
+          }
+        }
+        if(!wrongLength){
+          // build the KeyValue now:
+          KeyValue newKv = new KeyValue(row, family,
+            qualifier, EnvironmentEdgeManager.currentTimeMillis(),
+            Bytes.toBytes(result));
+
+          // now log it:
+          if (writeToWAL) {
+            long now = EnvironmentEdgeManager.currentTimeMillis();
+            WALEdit walEdit = new WALEdit();
+            walEdit.add(newKv);
+            // Using default cluster id, as this can only happen in the
+            // orginating cluster. A slave cluster receives the final value (not
+            // the delta) as a Put.
+            txid = this.log.appendNoSync(regionInfo, this.htableDescriptor.getName(),
+                walEdit, HConstants.DEFAULT_CLUSTER_ID, now,
+                this.htableDescriptor);
+          }
+
+          // Now request the ICV to the store, this will set the timestamp
+          // appropriately depending on if there is a value in memcache or not.
+          // returns the change in the size of the memstore from operation
+          long size = store.updateColumnValue(row, family, qualifier, result);
+
+          size = this.addAndGetGlobalMemstoreSize(size);
+          flush = isFlushSize(size);
+        }
+      } finally {
+        this.updatesLock.readLock().unlock();
+        releaseRowLock(lid);
+      }
+      if (writeToWAL) {
+        // sync the transaction log outside the rowlock
+        syncOrDefer(txid, Durability.USE_DEFAULT);
+      }
+    } finally {
+      closeRegionOperation();
+    }
+
+    // do after lock
+    long after = EnvironmentEdgeManager.currentTimeMillis();
+    this.opMetrics.updateIncrementColumnValueMetrics(family, after - before);
+
+    if (flush) {
+      // Request a cache flush.  Do it outside update lock.
+      requestFlush();
+    }
+    if(wrongLength){
+      throw new DoNotRetryIOException(
+          "Attempted to increment field that isn't 64 bits wide");
+    }
+    return result;
+  }
+
+
+  //
+  // New HBASE-880 Helpers
+  //
+
+  private void checkFamily(final byte [] family)
+  throws NoSuchColumnFamilyException {
+    if (!this.htableDescriptor.hasFamily(family)) {
+      throw new NoSuchColumnFamilyException("Column family " +
+          Bytes.toString(family) + " does not exist in region " + this
+          + " in table " + this.htableDescriptor);
+  	}
+  }
+
+  public static final long FIXED_OVERHEAD = ClassSize.align(
+      ClassSize.OBJECT +
+      ClassSize.ARRAY +
+      36 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
+      (8 * Bytes.SIZEOF_LONG) +
+      Bytes.SIZEOF_BOOLEAN);
+
+  public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
+      ClassSize.OBJECT + // closeLock
+      (2 * ClassSize.ATOMIC_BOOLEAN) + // closed, closing
+      (3 * ClassSize.ATOMIC_LONG) + // memStoreSize, numPutsWithoutWAL, dataInMemoryWithoutWAL
+      ClassSize.ATOMIC_INTEGER + // lockIdGenerator
+      (3 * ClassSize.CONCURRENT_HASHMAP) +  // lockedRows, lockIds, scannerReadPoints
+      WriteState.HEAP_SIZE + // writestate
+      ClassSize.CONCURRENT_SKIPLISTMAP + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + // stores
+      (2 * ClassSize.REENTRANT_LOCK) + // lock, updatesLock
+      ClassSize.ARRAYLIST + // recentFlushes
+      MultiVersionConsistencyControl.FIXED_SIZE // mvcc
+      ;
+
+  @Override
+  public long heapSize() {
+    long heapSize = DEEP_OVERHEAD;
+    for(Store store : this.stores.values()) {
+      heapSize += store.heapSize();
+    }
+    // this does not take into account row locks, recent flushes, mvcc entries
+    return heapSize;
+  }
+
+  /*
+   * This method calls System.exit.
+   * @param message Message to print out.  May be null.
+   */
+  private static void printUsageAndExit(final String message) {
+    if (message != null && message.length() > 0) System.out.println(message);
+    System.out.println("Usage: HRegion CATLALOG_TABLE_DIR [major_compact]");
+    System.out.println("Options:");
+    System.out.println(" major_compact  Pass this option to major compact " +
+      "passed region.");
+    System.out.println("Default outputs scan of passed region.");
+    System.exit(1);
+  }
+
+  /**
+   * Registers a new CoprocessorProtocol subclass and instance to
+   * be available for handling {@link HRegion#exec(Exec)} calls.
+   *
+   * 
                
+   * Only a single protocol type/handler combination may be registered per
+   * region.
+   * After the first registration, subsequent calls with the same protocol type
+   * will fail with a return value of {@code false}.
+   * 
                
+   * @param protocol a {@code CoprocessorProtocol} subinterface defining the
+   * protocol methods
+   * @param handler an instance implementing the interface
+   * @param  the protocol type
+   * @return {@code true} if the registration was successful, {@code false}
+   * otherwise
+   */
+  public  boolean registerProtocol(
+      Class protocol, T handler) {
+
+    /* No stacking of protocol handlers is currently allowed.  The
+     * first to claim wins!
+     */
+    if (protocolHandlers.containsKey(protocol)) {
+      LOG.error("Protocol "+protocol.getName()+
+          " already registered, rejecting request from "+
+          handler
+      );
+      return false;
+    }
+
+    protocolHandlers.putInstance(protocol, handler);
+    protocolHandlerNames.put(protocol.getName(), protocol);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Registered protocol handler: region="+
+          Bytes.toStringBinary(getRegionName())+" protocol="+protocol.getName());
+    }
+    return true;
+  }
+
+  /**
+   * Executes a single {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol}
+   * method using the registered protocol handlers.
+   * {@link CoprocessorProtocol} implementations must be registered via the
+   * {@link org.apache.hadoop.hbase.regionserver.HRegion#registerProtocol(Class, org.apache.hadoop.hbase.ipc.CoprocessorProtocol)}
+   * method before they are available.
+   *
+   * @param call an {@code Exec} instance identifying the protocol, method name,
+   *     and parameters for the method invocation
+   * @return an {@code ExecResult} instance containing the region name of the
+   *     invocation and the return value
+   * @throws IOException if no registered protocol handler is found or an error
+   *     occurs during the invocation
+   * @see org.apache.hadoop.hbase.regionserver.HRegion#registerProtocol(Class, org.apache.hadoop.hbase.ipc.CoprocessorProtocol)
+   */
+  public ExecResult exec(Exec call)
+      throws IOException {
+    Class protocol = call.getProtocol();
+    if (protocol == null) {
+      String protocolName = call.getProtocolName();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Received dynamic protocol exec call with protocolName " + protocolName);
+      }
+      // detect the actual protocol class
+      protocol  = protocolHandlerNames.get(protocolName);
+      if (protocol == null) {
+        throw new HBaseRPC.UnknownProtocolException(protocol,
+            "No matching handler for protocol "+protocolName+
+            " in region "+Bytes.toStringBinary(getRegionName()));
+      }
+    }
+    if (!protocolHandlers.containsKey(protocol)) {
+      throw new HBaseRPC.UnknownProtocolException(protocol,
+          "No matching handler for protocol "+protocol.getName()+
+          " in region "+Bytes.toStringBinary(getRegionName()));
+    }
+
+    CoprocessorProtocol handler = protocolHandlers.getInstance(protocol);
+    Object value;
+
+    try {
+      Method method = protocol.getMethod(
+          call.getMethodName(), call.getParameterClasses());
+      method.setAccessible(true);
+
+      value = method.invoke(handler, call.getParameters());
+    } catch (InvocationTargetException e) {
+      Throwable target = e.getTargetException();
+      if (target instanceof IOException) {
+        throw (IOException)target;
+      }
+      IOException ioe = new IOException(target.toString());
+      ioe.setStackTrace(target.getStackTrace());
+      throw ioe;
+    } catch (Throwable e) {
+      if (!(e instanceof IOException)) {
+        LOG.error("Unexpected throwable object ", e);
+      }
+      IOException ioe = new IOException(e.toString());
+      ioe.setStackTrace(e.getStackTrace());
+      throw ioe;
+    }
+
+    return new ExecResult(getRegionName(), value);
+  }
+
+  /*
+   * Process table.
+   * Do major compaction or list content.
+   * @param fs
+   * @param p
+   * @param log
+   * @param c
+   * @param majorCompact
+   * @throws IOException
+   */
+  private static void processTable(final FileSystem fs, final Path p,
+      final HLog log, final Configuration c,
+      final boolean majorCompact)
+  throws IOException {
+    HRegion region = null;
+    String rootStr = Bytes.toString(HConstants.ROOT_TABLE_NAME);
+    String metaStr = Bytes.toString(HConstants.META_TABLE_NAME);
+    // Currently expects tables have one region only.
+    if (p.getName().startsWith(rootStr)) {
+      region = HRegion.newHRegion(p, log, fs, c, HRegionInfo.ROOT_REGIONINFO,
+        HTableDescriptor.ROOT_TABLEDESC, null);
+    } else if (p.getName().startsWith(metaStr)) {
+      region = HRegion.newHRegion(p, log, fs, c,
+        HRegionInfo.FIRST_META_REGIONINFO, HTableDescriptor.META_TABLEDESC, null);
+    } else {
+      throw new IOException("Not a known catalog table: " + p.toString());
+    }
+    try {
+      region.initialize();
+      if (majorCompact) {
+        region.compactStores(true);
+      } else {
+        // Default behavior
+        Scan scan = new Scan();
+        // scan.addFamily(HConstants.CATALOG_FAMILY);
+        RegionScanner scanner = region.getScanner(scan);
+        try {
+          List kvs = new ArrayList();
+          boolean done = false;
+          do {
+            kvs.clear();
+            done = scanner.next(kvs);
+            if (kvs.size() > 0) LOG.info(kvs);
+          } while (done);
+        } finally {
+          scanner.close();
+        }
+      }
+    } finally {
+      region.close();
+    }
+  }
+
+  boolean shouldForceSplit() {
+    return this.splitRequest;
+  }
+
+  byte[] getExplicitSplitPoint() {
+    return this.explicitSplitPoint;
+  }
+
+  public void forceSplit(byte[] sp) {
+    // NOTE : this HRegion will go away after the forced split is successfull
+    //        therefore, no reason to clear this value
+    this.splitRequest = true;
+    if (sp != null) {
+      this.explicitSplitPoint = sp;
+    }
+  }
+
+  void clearSplit_TESTS_ONLY() {
+    this.splitRequest = false;
+  }
+
+  /**
+   * Give the region a chance to prepare before it is split.
+   */
+  protected void prepareToSplit() {
+    // nothing
+  }
+
+  /**
+   * Return the splitpoint. null indicates the region isn't splittable
+   * If the splitpoint isn't explicitly specified, it will go over the stores
+   * to find the best splitpoint. Currently the criteria of best splitpoint
+   * is based on the size of the store.
+   */
+  public byte[] checkSplit() {
+    // Can't split ROOT/META
+    if (this.regionInfo.isMetaTable()) {
+      if (shouldForceSplit()) {
+        LOG.warn("Cannot split root/meta regions in HBase 0.20 and above");
+      }
+      return null;
+    }
+
+    if (!splitPolicy.shouldSplit()) {
+      return null;
+    }
+
+    byte[] ret = splitPolicy.getSplitPoint();
+
+    if (ret != null) {
+      try {
+        checkRow(ret, "calculated split");
+      } catch (IOException e) {
+        LOG.error("Ignoring invalid split", e);
+        return null;
+      }
+    }
+    return ret;
+  }
+
+  /**
+   * @return The priority that this region should have in the compaction queue
+   */
+  public int getCompactPriority() {
+    int count = Integer.MAX_VALUE;
+    for(Store store : stores.values()) {
+      count = Math.min(count, store.getCompactPriority());
+    }
+    return count;
+  }
+
+  /**
+   * Checks every store to see if one has too many
+   * store files
+   * @return true if any store has too many store files
+   */
+  public boolean needsCompaction() {
+    for(Store store : stores.values()) {
+      if(store.needsCompaction()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /** @return the coprocessor host */
+  public RegionCoprocessorHost getCoprocessorHost() {
+    return coprocessorHost;
+  }
+
+  /*
+   * Set the read request count defined in opMetrics
+   * @param value absolute value of read request count
+   */
+  public void setOpMetricsReadRequestCount(long value)
+  {
+    this.opMetrics.setReadRequestCountMetrics(value);
+  }
+  
+  /*
+   * Set the write request count defined in opMetrics
+   * @param value absolute value of write request count
+   */
+  public void setOpMetricsWriteRequestCount(long value)
+  {
+    this.opMetrics.setWriteRequestCountMetrics(value);
+  }
+  
+  /** @param coprocessorHost the new coprocessor host */
+  public void setCoprocessorHost(final RegionCoprocessorHost coprocessorHost) {
+    this.coprocessorHost = coprocessorHost;
+  }
+
+  /**
+   * This method needs to be called before any public call that reads or modifies data. It has to be
+   * called just before a try. #closeRegionOperation needs to be called in the try's finally block
+   * Acquires a read lock and checks if the region is closing or closed.
+   * 
                
+   * Note: This method changed to public to support changes done in the IndexRegionObserver. Not
+   * advisable to used by users.
+   * 
                
+   * @throws NotServingRegionException when the region is closing or closed
+   * @throws RegionTooBusyException if failed to get the lock in time
+   * @throws InterruptedIOException if interrupted while waiting for a lock
+   */
+  public void startRegionOperation()
+      throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
+    if (this.closing.get()) {
+      throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
+          " is closing");
+    }
+    lock(lock.readLock());
+    if (this.closed.get()) {
+      lock.readLock().unlock();
+      throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
+          " is closed");
+    }
+  }
+
+  /**
+   * Closes the lock. This needs to be called in the finally block corresponding to the try block of
+   * #startRegionOperation
+   * 
                
+   * Note: This method changed to public to support changes done in the IndexRegionObserver. Not
+   * advisable to used by users.
+   * 
                
+   */
+  public void closeRegionOperation(){
+    lock.readLock().unlock();
+  }
+
+  /**
+   * This method needs to be called before any public call that reads or
+   * modifies stores in bulk. It has to be called just before a try.
+   * #closeBulkRegionOperation needs to be called in the try's finally block
+   * Acquires a writelock and checks if the region is closing or closed.
+   * @throws NotServingRegionException when the region is closing or closed
+   * @throws RegionTooBusyException if failed to get the lock in time
+   * @throws InterruptedIOException if interrupted while waiting for a lock
+   */
+  private void startBulkRegionOperation(boolean writeLockNeeded)
+      throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
+    if (this.closing.get()) {
+      throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
+          " is closing");
+    }
+    if (writeLockNeeded) lock(lock.writeLock());
+    else lock(lock.readLock());
+    if (this.closed.get()) {
+      if (writeLockNeeded) lock.writeLock().unlock();
+      else lock.readLock().unlock();
+      throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
+          " is closed");
+    }
+  }
+
+  /**
+   * Closes the lock. This needs to be called in the finally block corresponding
+   * to the try block of #startRegionOperation
+   */
+  private void closeBulkRegionOperation() {
+    if (lock.writeLock().isHeldByCurrentThread()) lock.writeLock().unlock();
+    else lock.readLock().unlock();
+  }
+
+  /**
+   * Update counters for numer of puts without wal and the size of possible data loss.
+   * These information are exposed by the region server metrics.
+   */
+  private void recordPutWithoutWal(final Map> familyMap) {
+    if (numPutsWithoutWAL.getAndIncrement() == 0) {
+      LOG.info("writing data to region " + this +
+               " with WAL disabled. Data may be lost in the event of a crash.");
+    }
+
+    long putSize = 0;
+    for (List edits : familyMap.values()) {
+      for (KeyValue kv : edits) {
+        putSize += kv.getKeyLength() + kv.getValueLength();
+      }
+    }
+
+    dataInMemoryWithoutWAL.addAndGet(putSize);
+  }
+
+  private void lock(final Lock lock)
+      throws RegionTooBusyException, InterruptedIOException {
+    lock(lock, 1);
+  }
+
+  /**
+   * Try to acquire a lock.  Throw RegionTooBusyException
+   * if failed to get the lock in time. Throw InterruptedIOException
+   * if interrupted while waiting for the lock.
+   */
+  private void lock(final Lock lock, final int multiplier)
+      throws RegionTooBusyException, InterruptedIOException {
+    try {
+      final long waitTime = Math.min(maxBusyWaitDuration,
+        busyWaitDuration * Math.min(multiplier, maxBusyWaitMultiplier));
+      if (!lock.tryLock(waitTime, TimeUnit.MILLISECONDS)) {
+        throw new RegionTooBusyException(
+          "failed to get a lock in " + waitTime + "ms");
+      }
+    } catch (InterruptedException ie) {
+      LOG.info("Interrupted while waiting for a lock");
+      InterruptedIOException iie = new InterruptedIOException();
+      iie.initCause(ie);
+      throw iie;
+    }
+  }
+
+  /**
+   * Calls sync with the given transaction ID if the region's table is not
+   * deferring it.
+   * @param txid should sync up to which transaction
+   * @throws IOException If anything goes wrong with DFS
+   */
+  private void syncOrDefer(long txid, Durability durability) throws IOException {
+    if (this.getRegionInfo().isMetaRegion()) {
+      this.log.sync(txid);
+    } else {
+      switch(durability) {
+      case USE_DEFAULT:
+        // do what CF defaults to
+        if (!isDeferredLogSyncEnabled()) {
+          this.log.sync(txid);
+        }
+        break;
+      case SKIP_WAL:
+        // nothing do to
+        break;
+      case ASYNC_WAL:
+        // defer the sync, unless we globally can't
+        if (this.deferredLogSyncDisabled) {
+          this.log.sync(txid);
+        }
+        break;
+      case SYNC_WAL:
+      case FSYNC_WAL:
+        // sync the WAL edit (SYNC and FSYNC treated the same for now)
+        this.log.sync(txid);
+        break;
+      }
+    }
+  }
+
+  /**
+   * check if current region is deferred sync enabled.
+   */
+  private boolean isDeferredLogSyncEnabled() {
+    return (this.htableDescriptor.isDeferredLogFlush() && !this.deferredLogSyncDisabled);
+  }
+
+  /**
+   * A mocked list implementaion - discards all updates.
+   */
+  private static final List MOCKED_LIST = new AbstractList() {
+
+    @Override
+    public void add(int index, KeyValue element) {
+      // do nothing
+    }
+
+    @Override
+    public boolean addAll(int index, Collection c) {
+      return false; // this list is never changed as a result of an update
+    }
+
+    @Override
+    public KeyValue get(int index) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int size() {
+      return 0;
+    }
+  };
+
+  public ReentrantReadWriteLock getUpdateLock() {
+    return this.updatesLock;
+  }
+
+  public void updateLock() {
+    this.updatesLock.readLock().lock();
+  }
+
+  public void releaseLock() {
+    this.updatesLock.readLock().unlock();
+  }
+
+  /**
+   * Facility for dumping and compacting catalog tables.
+   * Only does catalog tables since these are only tables we for sure know
+   * schema on.  For usage run:
+   * 
                +   *   ./bin/hbase org.apache.hadoop.hbase.regionserver.HRegion
+   * 
                
+   * @param args
+   * @throws IOException
+   */
+  public static void main(String[] args) throws IOException {
+    if (args.length < 1) {
+      printUsageAndExit(null);
+    }
+    boolean majorCompact = false;
+    if (args.length > 1) {
+      if (!args[1].toLowerCase().startsWith("major")) {
+        printUsageAndExit("ERROR: Unrecognized option <" + args[1] + ">");
+      }
+      majorCompact = true;
+    }
+    final Path tableDir = new Path(args[0]);
+    final Configuration c = HBaseConfiguration.create();
+    final FileSystem fs = FileSystem.get(c);
+    final Path logdir = new Path(c.get("hbase.tmp.dir"),
+        "hlog" + tableDir.getName()
+        + EnvironmentEdgeManager.currentTimeMillis());
+    final Path oldLogDir = new Path(c.get("hbase.tmp.dir"),
+        HConstants.HREGION_OLDLOGDIR_NAME);
+    final HLog log = new HLog(fs, logdir, oldLogDir, c);
+    try {
+      processTable(fs, tableDir, log, c, majorCompact);
+    } finally {
+       log.close();
+       // TODO: is this still right?
+       BlockCache bc = new CacheConfig(c).getBlockCache();
+       if (bc != null) bc.shutdown();
+    }
+  }
+
+  /**
+   * Listener class to enable callers of
+   * bulkLoadHFile() to perform any necessary
+   * pre/post processing of a given bulkload call
+   */
+  public static interface BulkLoadListener {
+
+    /**
+     * Called before an HFile is actually loaded
+     * @param family family being loaded to
+     * @param srcPath path of HFile
+     * @return final path to be used for actual loading
+     * @throws IOException
+     */
+    String prepareBulkLoad(byte[] family, String srcPath) throws IOException;
+
+    /**
+     * Called after a successful HFile load
+     * @param family family being loaded to
+     * @param srcPath path of HFile
+     * @throws IOException
+     */
+    void doneBulkLoad(byte[] family, String srcPath) throws IOException;
+
+    /**
+     * Called after a failed HFile load
+     * @param family family being loaded to
+     * @param srcPath path of HFile
+     * @throws IOException
+     */
+    void failedBulkLoad(byte[] family, String srcPath) throws IOException;
+
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
new file mode 100644
index 0000000..fe4cb68
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+
+/**
+ * Acts as an abstraction layer b/w HBase and underlying fs. Used for making non-idempotent calls.
+ * This is useful as it can have a retry logic for such operations, as they are not retried at
+ * hdfs level.
+ * Region specific methods that access fs should be added here.
+ *
+ */
+public class HRegionFileSystem extends HBaseFileSystem {
+  public static final Log LOG = LogFactory.getLog(HRegionFileSystem.class);
+
+  public HRegionFileSystem(Configuration conf) {
+    setRetryCounts(conf);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
new file mode 100644
index 0000000..b168855
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -0,0 +1,4148 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.lang.Thread.UncaughtExceptionHandler;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.net.BindException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import javax.management.ObjectName;
+
+import org.apache.commons.lang.mutable.MutableDouble;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.ClockOutOfSyncException;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HealthCheckChore;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MasterAddressTracker;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.TableDescriptors;
+import org.apache.hadoop.hbase.UnknownRowLockException;
+import org.apache.hadoop.hbase.UnknownScannerException;
+import org.apache.hadoop.hbase.YouAreDeadException;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.catalog.RootLocationEditor;
+import org.apache.hadoop.hbase.client.Action;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.MultiAction;
+import org.apache.hadoop.hbase.client.MultiResponse;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowLock;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.Exec;
+import org.apache.hadoop.hbase.client.coprocessor.ExecResult;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.CacheStats;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+import org.apache.hadoop.hbase.ipc.HBaseRPC;
+import org.apache.hadoop.hbase.ipc.HBaseRPCErrorHandler;
+import org.apache.hadoop.hbase.ipc.HBaseRpcMetrics;
+import org.apache.hadoop.hbase.ipc.HBaseServer;
+import org.apache.hadoop.hbase.ipc.HMasterRegionInterface;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.ipc.Invocation;
+import org.apache.hadoop.hbase.ipc.ProtocolSignature;
+import org.apache.hadoop.hbase.ipc.RpcEngine;
+import org.apache.hadoop.hbase.ipc.RpcServer;
+import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
+import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.handler.CloseMetaHandler;
+import org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler;
+import org.apache.hadoop.hbase.regionserver.handler.CloseRootHandler;
+import org.apache.hadoop.hbase.regionserver.handler.OpenMetaHandler;
+import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler;
+import org.apache.hadoop.hbase.regionserver.handler.OpenRootHandler;
+import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage;
+import org.apache.hadoop.hbase.regionserver.metrics.RegionServerDynamicMetrics;
+import org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.StoreMetricType;
+import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager;
+import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CompressionTest;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.InfoServer;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Sleeper;
+import org.apache.hadoop.hbase.util.Strings;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.VersionInfo;
+import org.apache.hadoop.hbase.zookeeper.ClusterId;
+import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.metrics.util.MBeanUtil;
+import org.apache.hadoop.net.DNS;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.zookeeper.KeeperException;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+
+/**
+ * HRegionServer makes a set of HRegions available to clients. It checks in with
+ * the HMaster. There are many HRegionServers in a single HBase deployment.
+ */
+public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
+    Runnable, RegionServerServices {
+
+  public static final Log LOG = LogFactory.getLog(HRegionServer.class);
+
+  // Set when a report to the master comes back with a message asking us to
+  // shutdown. Also set by call to stop when debugging or running unit tests
+  // of HRegionServer in isolation.
+  protected volatile boolean stopped = false;
+
+  // A state before we go into stopped state.  At this stage we're closing user
+  // space regions.
+  private boolean stopping = false;
+
+  // Go down hard. Used if file system becomes unavailable and also in
+  // debugging and unit tests.
+  protected volatile boolean abortRequested;
+
+  private volatile boolean killed = false;
+
+  // If false, the file system has become unavailable
+  protected volatile boolean fsOk;
+
+  protected final Configuration conf;
+
+  protected final AtomicBoolean haveRootRegion = new AtomicBoolean(false);
+  private HFileSystem fs;
+  private boolean useHBaseChecksum; // verify hbase checksums?
+  private Path rootDir;
+  private final Random rand;
+
+  //RegionName vs current action in progress
+  //true - if open region action in progress
+  //false - if close region action in progress
+  private final ConcurrentSkipListMap regionsInTransitionInRS =
+      new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR);
+
+  /**
+   * Map of regions currently being served by this region server. Key is the
+   * encoded region name.  All access should be synchronized.
+   */
+  protected final Map onlineRegions =
+    new ConcurrentHashMap();
+
+  protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+  final int numRetries;
+  protected final int threadWakeFrequency;
+  private final int msgInterval;
+
+  protected final int numRegionsToReport;
+
+  private final long maxScannerResultSize;
+
+  // Remote HMaster
+  private HMasterRegionInterface hbaseMaster;
+
+  // RPC Engine for master connection
+  private RpcEngine rpcEngine;
+
+  // Server to handle client requests. Default access so can be accessed by
+  // unit tests.
+  RpcServer rpcServer;
+
+  // Server to handle client requests.
+  private HBaseServer server;  
+
+  private final InetSocketAddress isa;
+  private UncaughtExceptionHandler uncaughtExceptionHandler;
+
+  // Leases
+  private Leases leases;
+
+  // Request counter.
+  // Do we need this?  Can't we just sum region counters?  St.Ack 20110412
+  private AtomicInteger requestCount = new AtomicInteger();
+
+  // Info server. Default access so can be used by unit tests. REGIONSERVER
+  // is name of the webapp and the attribute name used stuffing this instance
+  // into web context.
+  InfoServer infoServer;
+
+  /** region server process name */
+  public static final String REGIONSERVER = "regionserver";
+  
+  /** region server configuration name */
+  public static final String REGIONSERVER_CONF = "regionserver_conf";
+
+  /*
+   * Space is reserved in HRS constructor and then released when aborting to
+   * recover from an OOME. See HBASE-706. TODO: Make this percentage of the heap
+   * or a minimum.
+   */
+  private final LinkedList reservedSpace = new LinkedList();
+
+  private RegionServerMetrics metrics;
+
+  private RegionServerDynamicMetrics dynamicMetrics;
+
+  // Compactions
+  public CompactSplitThread compactSplitThread;
+
+  // Cache flushing
+  MemStoreFlusher cacheFlusher;
+
+  /*
+   * Check for compactions requests.
+   */
+  Chore compactionChecker;
+
+  /*
+   * Check for flushes
+   */
+  Chore periodicFlusher;
+
+  // HLog and HLog roller. log is protected rather than private to avoid
+  // eclipse warning when accessed by inner classes
+  protected volatile HLog hlog;
+  // The meta updates are written to a different hlog. If this
+  // regionserver holds meta regions, then this field will be non-null.
+  protected volatile HLog hlogForMeta;
+
+  LogRoller hlogRoller;
+  LogRoller metaHLogRoller;
+
+  private final boolean separateHLogForMeta;
+
+  // flag set after we're done setting up server threads (used for testing)
+  protected volatile boolean isOnline;
+
+  final Map scanners =
+    new ConcurrentHashMap();
+
+  // zookeeper connection and watcher
+  private ZooKeeperWatcher zooKeeper;
+
+  // master address manager and watcher
+  private MasterAddressTracker masterAddressManager;
+
+  // catalog tracker
+  private CatalogTracker catalogTracker;
+
+  // Cluster Status Tracker
+  private ClusterStatusTracker clusterStatusTracker;
+
+  // Log Splitting Worker
+  private SplitLogWorker splitLogWorker;
+
+  // A sleeper that sleeps for msgInterval.
+  private final Sleeper sleeper;
+
+  private final int rpcTimeout;
+
+  // Instance of the hbase executor service.
+  private ExecutorService service;
+
+  // Replication services. If no replication, this handler will be null.
+  private ReplicationSourceService replicationSourceHandler;
+  private ReplicationSinkService replicationSinkHandler;
+
+  private final RegionServerAccounting regionServerAccounting;
+
+  // Cache configuration and block cache reference
+  private final CacheConfig cacheConfig;
+
+  // reference to the Thrift Server.
+  volatile private HRegionThriftServer thriftServer;
+
+  /**
+   * The server name the Master sees us as.  Its made from the hostname the
+   * master passes us, port, and server startcode. Gets set after registration
+   * against  Master.  The hostname can differ from the hostname in {@link #isa}
+   * but usually doesn't if both servers resolve .
+   */
+  private ServerName serverNameFromMasterPOV;
+
+  // Port we put up the webui on.
+  private int webuiport = -1;
+
+  /**
+   * This servers startcode.
+   */
+  private final long startcode;
+
+  /**
+   * Go here to get table descriptors.
+   */
+  private TableDescriptors tableDescriptors;
+
+  /*
+   * Strings to be used in forming the exception message for
+   * RegionsAlreadyInTransitionException.
+   */
+  private static final String OPEN = "OPEN";
+  private static final String CLOSE = "CLOSE";
+
+  /**
+   * MX Bean for RegionServerInfo
+   */
+  private ObjectName mxBean = null;
+
+  /**
+   * ClusterId
+   */
+  private ClusterId clusterId = null;
+
+  private RegionServerCoprocessorHost rsHost;
+
+  /** The health check chore. */
+  private HealthCheckChore healthCheckChore;
+
+  /**
+   * Starts a HRegionServer at the default location
+   *
+   * @param conf
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public HRegionServer(Configuration conf)
+  throws IOException, InterruptedException {
+    this.fsOk = true;
+    this.conf = conf;
+    // Set how many times to retry talking to another server over HConnection.
+    HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG);
+    this.isOnline = false;
+    checkCodecs(this.conf);
+
+    // do we use checksum verfication in the hbase? If hbase checksum verification
+    // is enabled, then we automatically switch off hdfs checksum verification.
+    this.useHBaseChecksum = conf.getBoolean(
+      HConstants.HBASE_CHECKSUM_VERIFICATION, false);
+
+    // Config'ed params
+    this.separateHLogForMeta = conf.getBoolean(HLog.SEPARATE_HLOG_FOR_META, false);
+    this.numRetries = conf.getInt("hbase.client.retries.number", 10);
+    this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+      10 * 1000);
+    this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000);
+
+    this.sleeper = new Sleeper(this.msgInterval, this);
+
+    this.maxScannerResultSize = conf.getLong(
+      HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
+      HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
+
+    this.numRegionsToReport = conf.getInt(
+      "hbase.regionserver.numregionstoreport", 10);
+
+    this.rpcTimeout = conf.getInt(
+      HConstants.HBASE_RPC_TIMEOUT_KEY,
+      HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
+
+    this.abortRequested = false;
+    this.stopped = false;
+
+    // Server to handle client requests.
+    String hostname = conf.get("hbase.regionserver.ipc.address",
+      Strings.domainNamePointerToHostName(DNS.getDefaultHost(
+        conf.get("hbase.regionserver.dns.interface", "default"),
+        conf.get("hbase.regionserver.dns.nameserver", "default"))));
+    int port = conf.getInt(HConstants.REGIONSERVER_PORT,
+      HConstants.DEFAULT_REGIONSERVER_PORT);
+    // Creation of a HSA will force a resolve.
+    InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
+    if (initialIsa.getAddress() == null) {
+      throw new IllegalArgumentException("Failed resolve of " + initialIsa);
+    }
+
+    this.rand = new Random(initialIsa.hashCode());
+    this.rpcServer = HBaseRPC.getServer(this,
+      new Class[]{HRegionInterface.class, HBaseRPCErrorHandler.class,
+        OnlineRegions.class},
+        initialIsa.getHostName(), // BindAddress is IP we got for this server.
+        initialIsa.getPort(),
+        conf.getInt("hbase.regionserver.handler.count", 10),
+        conf.getInt("hbase.regionserver.metahandler.count", 10),
+        conf.getBoolean("hbase.rpc.verbose", false),
+        conf, HConstants.QOS_THRESHOLD);
+    if (rpcServer instanceof HBaseServer) server = (HBaseServer) rpcServer;
+    // Set our address.
+    this.isa = this.rpcServer.getListenerAddress();
+
+    this.rpcServer.setErrorHandler(this);
+    this.rpcServer.setQosFunction(new QosFunction());
+    this.startcode = System.currentTimeMillis();
+
+    // login the zookeeper client principal (if using security)
+    ZKUtil.loginClient(this.conf, "hbase.zookeeper.client.keytab.file",
+      "hbase.zookeeper.client.kerberos.principal", this.isa.getHostName());
+
+    // login the server principal (if using secure Hadoop)
+    User.login(this.conf, "hbase.regionserver.keytab.file",
+      "hbase.regionserver.kerberos.principal", this.isa.getHostName());
+    regionServerAccounting = new RegionServerAccounting();
+    cacheConfig = new CacheConfig(conf);
+    uncaughtExceptionHandler = new UncaughtExceptionHandler() {
+      public void uncaughtException(Thread t, Throwable e) {
+        abort("Uncaught exception in service thread " + t.getName(), e);
+      }
+    };
+  }
+
+  /** Handle all the snapshot requests to this server */
+  RegionServerSnapshotManager snapshotManager;
+
+  /**
+   * Run test on configured codecs to make sure supporting libs are in place.
+   * @param c
+   * @throws IOException
+   */
+  private static void checkCodecs(final Configuration c) throws IOException {
+    // check to see if the codec list is available:
+    String [] codecs = c.getStrings("hbase.regionserver.codecs", (String[])null);
+    if (codecs == null) return;
+    for (String codec : codecs) {
+      if (!CompressionTest.testCompression(codec)) {
+        throw new IOException("Compression codec " + codec +
+          " not supported, aborting RS construction");
+      }
+    }
+  }
+
+
+  @Retention(RetentionPolicy.RUNTIME)
+  private @interface QosPriority {
+    int priority() default 0;
+  }
+
+  /**
+   * Utility used ensuring higher quality of service for priority rpcs; e.g.
+   * rpcs to .META. and -ROOT-, etc.
+   */
+  class QosFunction implements Function {
+    private final Map annotatedQos;
+
+    public QosFunction() {
+      Map qosMap = new HashMap();
+      for (Method m : HRegionServer.class.getMethods()) {
+        QosPriority p = m.getAnnotation(QosPriority.class);
+        if (p != null) {
+          qosMap.put(m.getName(), p.priority());
+        }
+      }
+
+      annotatedQos = qosMap;
+    }
+
+    public boolean isMetaTable(byte[] regionName) {
+      HRegion region;
+      try {
+        region = getRegion(regionName);
+      } catch (NotServingRegionException ignored) {
+        return false;
+      }
+      return region.getRegionInfo().isMetaTable();
+    }
+
+    @Override
+    public Integer apply(Writable from) {
+      if (!(from instanceof Invocation)) return HConstants.NORMAL_QOS;
+
+      Invocation inv = (Invocation) from;
+      String methodName = inv.getMethodName();
+
+      Integer priorityByAnnotation = annotatedQos.get(methodName);
+      if (priorityByAnnotation != null) {
+        return priorityByAnnotation;
+      }
+
+      // scanner methods...
+      if (methodName.equals("next") || methodName.equals("close")) {
+        // translate!
+        Long scannerId;
+        try {
+          scannerId = (Long) inv.getParameters()[0];
+        } catch (ClassCastException ignored) {
+          // LOG.debug("Low priority: " + from);
+          return HConstants.NORMAL_QOS;
+        }
+        String scannerIdString = Long.toString(scannerId);
+        RegionScanner scanner = scanners.get(scannerIdString);
+        if (scanner != null && scanner.getRegionInfo().isMetaTable()) {
+          // LOG.debug("High priority scanner request: " + scannerId);
+          return HConstants.HIGH_QOS;
+        }
+      } else if (inv.getParameterClasses().length == 0) {
+       // Just let it through.  This is getOnlineRegions, etc.
+      } else if (inv.getParameterClasses()[0] == byte[].class) {
+        // first arg is byte array, so assume this is a regionname:
+        if (isMetaTable((byte[]) inv.getParameters()[0])) {
+          // LOG.debug("High priority with method: " + methodName +
+          // " and region: "
+          // + Bytes.toString((byte[]) inv.getParameters()[0]));
+          return HConstants.HIGH_QOS;
+        }
+      } else if (inv.getParameterClasses()[0] == MultiAction.class) {
+        MultiAction ma = (MultiAction) inv.getParameters()[0];
+        Set regions = ma.getRegions();
+        // ok this sucks, but if any single of the actions touches a meta, the
+        // whole
+        // thing gets pingged high priority. This is a dangerous hack because
+        // people
+        // can get their multi action tagged high QOS by tossing a Get(.META.)
+        // AND this
+        // regionserver hosts META/-ROOT-
+        for (byte[] region : regions) {
+          if (isMetaTable(region)) {
+            // LOG.debug("High priority multi with region: " +
+            // Bytes.toString(region));
+            return HConstants.HIGH_QOS; // short circuit for the win.
+          }
+        }
+      }
+      // LOG.debug("Low priority: " + from.toString());
+      return HConstants.NORMAL_QOS;
+    }
+  }
+
+  /**
+   * All initialization needed before we go register with Master.
+   *
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  private void preRegistrationInitialization(){
+    try {
+      initializeZooKeeper();
+
+      clusterId = new ClusterId(zooKeeper, this);
+      if(clusterId.hasId()) {
+        conf.set(HConstants.CLUSTER_ID, clusterId.getId());
+      }
+
+      initializeThreads();
+      int nbBlocks = conf.getInt("hbase.regionserver.nbreservationblocks", 4);
+      for (int i = 0; i < nbBlocks; i++) {
+        reservedSpace.add(new byte[HConstants.DEFAULT_SIZE_RESERVATION_BLOCK]);
+      }
+
+      this.rpcEngine = HBaseRPC.getProtocolEngine(conf);
+    } catch (Throwable t) {
+      // Call stop if error or process will stick around for ever since server
+      // puts up non-daemon threads.
+      this.rpcServer.stop();
+      abort("Initialization of RS failed.  Hence aborting RS.", t);
+    }
+  }
+
+  /**
+   * Bring up connection to zk ensemble and then wait until a master for this
+   * cluster and then after that, wait until cluster 'up' flag has been set.
+   * This is the order in which master does things.
+   * Finally put up a catalog tracker.
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  private void initializeZooKeeper() throws IOException, InterruptedException {
+    // Open connection to zookeeper and set primary watcher
+    this.zooKeeper = new ZooKeeperWatcher(conf, REGIONSERVER + ":" +
+      this.isa.getPort(), this);
+
+    // Create the master address manager, register with zk, and start it.  Then
+    // block until a master is available.  No point in starting up if no master
+    // running.
+    this.masterAddressManager = new MasterAddressTracker(this.zooKeeper, this);
+    this.masterAddressManager.start();
+    blockAndCheckIfStopped(this.masterAddressManager);
+
+    // Wait on cluster being up.  Master will set this flag up in zookeeper
+    // when ready.
+    this.clusterStatusTracker = new ClusterStatusTracker(this.zooKeeper, this);
+    this.clusterStatusTracker.start();
+    blockAndCheckIfStopped(this.clusterStatusTracker);
+
+    // Create the catalog tracker and start it;
+    this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf, this);
+    catalogTracker.start();
+
+    // watch for snapshots
+    try {
+      this.snapshotManager = new RegionServerSnapshotManager(this);
+    } catch (KeeperException e) {
+      this.abort("Failed to reach zk cluster when creating snapshot handler.");
+    }
+  }
+
+  /**
+   * Utilty method to wait indefinitely on a znode availability while checking
+   * if the region server is shut down
+   * @param tracker znode tracker to use
+   * @throws IOException any IO exception, plus if the RS is stopped
+   * @throws InterruptedException
+   */
+  private void blockAndCheckIfStopped(ZooKeeperNodeTracker tracker)
+      throws IOException, InterruptedException {
+    while (tracker.blockUntilAvailable(this.msgInterval, false) == null) {
+      if (this.stopped) {
+        throw new IOException("Received the shutdown message while waiting.");
+      }
+    }
+  }
+
+  /**
+   * @return False if cluster shutdown in progress
+   */
+  private boolean isClusterUp() {
+    return this.clusterStatusTracker.isClusterUp();
+  }
+
+  private void initializeThreads() throws IOException {
+    // Cache flushing thread.
+    this.cacheFlusher = new MemStoreFlusher(conf, this);
+
+    // Compaction thread
+    this.compactSplitThread = new CompactSplitThread(this);
+
+    // Background thread to check for compactions; needed if region
+    // has not gotten updates in a while. Make it run at a lesser frequency.
+    int multiplier = this.conf.getInt(HConstants.THREAD_WAKE_FREQUENCY +
+      ".multiplier", 1000);
+    this.compactionChecker = new CompactionChecker(this,
+      this.threadWakeFrequency * multiplier, this);
+
+    this.periodicFlusher = new PeriodicMemstoreFlusher(this.threadWakeFrequency, this);
+
+    // Health checker thread.
+    int sleepTime = this.conf.getInt(HConstants.HEALTH_CHORE_WAKE_FREQ,
+      HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);
+    if (isHealthCheckerConfigured()) {
+      healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration());
+    }
+
+    this.leases = new Leases((int) conf.getLong(
+        HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
+        HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD),
+        this.threadWakeFrequency);
+
+    // Create the thread for the ThriftServer.
+    if (conf.getBoolean("hbase.regionserver.export.thrift", false)) {
+      thriftServer = new HRegionThriftServer(this, conf);
+      thriftServer.start();
+      LOG.info("Started Thrift API from Region Server.");
+    }
+  }
+
+  /**
+   * The HRegionServer sticks in this loop until closed.
+   */
+  @SuppressWarnings("deprecation")
+  public void run() {
+    try {
+      // Do pre-registration initializations; zookeeper, lease threads, etc.
+      preRegistrationInitialization();
+    } catch (Throwable e) {
+      abort("Fatal exception during initialization", e);
+    }
+
+    try {
+      // Try and register with the Master; tell it we are here.  Break if
+      // server is stopped or the clusterup flag is down or hdfs went wacky.
+      while (keepLooping()) {
+        MapWritable w = reportForDuty();
+        if (w == null) {
+          LOG.warn("reportForDuty failed; sleeping and then retrying.");
+          this.sleeper.sleep();
+        } else {
+          handleReportForDutyResponse(w);
+          break;
+        }
+      }
+      registerMBean();
+
+      // start the snapshot handler, since the server is ready to run
+      this.snapshotManager.start();
+
+      // We registered with the Master.  Go into run mode.
+      long lastMsg = 0;
+      long oldRequestCount = -1;
+      // The main run loop.
+      while (!this.stopped && isHealthy()) {
+        if (!isClusterUp()) {
+          if (isOnlineRegionsEmpty()) {
+            stop("Exiting; cluster shutdown set and not carrying any regions");
+          } else if (!this.stopping) {
+            this.stopping = true;
+            LOG.info("Closing user regions");
+            closeUserRegions(this.abortRequested);
+          } else if (this.stopping) {
+            boolean allUserRegionsOffline = areAllUserRegionsOffline();
+            if (allUserRegionsOffline) {
+              // Set stopped if no requests since last time we went around the loop.
+              // The remaining meta regions will be closed on our way out.
+              if (oldRequestCount == this.requestCount.get()) {
+                stop("Stopped; only catalog regions remaining online");
+                break;
+              }
+              oldRequestCount = this.requestCount.get();
+            } else {
+              // Make sure all regions have been closed -- some regions may
+              // have not got it because we were splitting at the time of
+              // the call to closeUserRegions.
+              closeUserRegions(this.abortRequested);
+            }
+            LOG.debug("Waiting on " + getOnlineRegionsAsPrintableString());
+          }
+        }
+        long now = System.currentTimeMillis();
+        if ((now - lastMsg) >= msgInterval) {
+          doMetrics();
+          tryRegionServerReport();
+          lastMsg = System.currentTimeMillis();
+        }
+        if (!this.stopped) this.sleeper.sleep();
+      } // for
+    } catch (Throwable t) {
+      if (!checkOOME(t)) {
+        abort("Unhandled exception: " + t.getMessage(), t);
+      }
+    }
+    // Run shutdown.
+    if (mxBean != null) {
+      MBeanUtil.unregisterMBean(mxBean);
+      mxBean = null;
+    }
+    if (this.thriftServer != null) this.thriftServer.shutdown();
+    this.leases.closeAfterLeasesExpire();
+    this.rpcServer.stop();
+    if (this.splitLogWorker != null) {
+      splitLogWorker.stop();
+    }
+    if (this.infoServer != null) {
+      LOG.info("Stopping infoServer");
+      try {
+        this.infoServer.stop();
+      } catch (Exception e) {
+        e.printStackTrace();
+      }
+    }
+    // Send cache a shutdown.
+    if (cacheConfig.isBlockCacheEnabled()) {
+      cacheConfig.getBlockCache().shutdown();
+    }
+
+    // Send interrupts to wake up threads if sleeping so they notice shutdown.
+    // TODO: Should we check they are alive? If OOME could have exited already
+    if (this.cacheFlusher != null) this.cacheFlusher.interruptIfNecessary();
+    if (this.compactSplitThread != null) this.compactSplitThread.interruptIfNecessary();
+    if (this.hlogRoller != null) this.hlogRoller.interruptIfNecessary();
+    if (this.metaHLogRoller != null) this.metaHLogRoller.interruptIfNecessary();
+    if (this.compactionChecker != null)
+      this.compactionChecker.interrupt();
+    if (this.healthCheckChore != null) {
+      this.healthCheckChore.interrupt();
+    }
+
+    try {
+      if (snapshotManager != null) snapshotManager.stop(this.abortRequested);
+    } catch (IOException e) {
+      LOG.warn("Failed to close snapshot handler cleanly", e);
+    }
+
+    if (this.killed) {
+      // Just skip out w/o closing regions.  Used when testing.
+    } else if (abortRequested) {
+      if (this.fsOk) {
+        closeUserRegions(abortRequested); // Don't leave any open file handles
+      }
+      LOG.info("aborting server " + this.serverNameFromMasterPOV);
+    } else {
+      closeUserRegions(abortRequested);
+      closeAllScanners();
+      LOG.info("stopping server " + this.serverNameFromMasterPOV);
+    }
+    // Interrupt catalog tracker here in case any regions being opened out in
+    // handlers are stuck waiting on meta or root.
+    if (this.catalogTracker != null) this.catalogTracker.stop();
+
+    // stop the snapshot handler, forcefully killing all running tasks
+    try {
+      if (snapshotManager != null) snapshotManager.stop(this.abortRequested || this.killed);
+    } catch (IOException e) {
+      LOG.warn("Failed to close snapshot handler cleanly", e);
+    }
+
+    // Closing the compactSplit thread before closing meta regions
+    if (!this.killed && containsMetaTableRegions()) {
+      if (!abortRequested || this.fsOk) {
+        if (this.compactSplitThread != null) {
+          this.compactSplitThread.join();
+          this.compactSplitThread = null;
+        }
+        closeMetaTableRegions(abortRequested);
+      }
+    }
+
+    if (!this.killed && this.fsOk) {
+      waitOnAllRegionsToClose(abortRequested);
+      LOG.info("stopping server " + this.serverNameFromMasterPOV +
+        "; all regions closed.");
+    }
+
+    //fsOk flag may be changed when closing regions throws exception.
+    if (!this.killed && this.fsOk) {
+      closeWAL(abortRequested ? false : true);
+    }
+
+    // Make sure the proxy is down.
+    this.hbaseMaster = null;
+    this.rpcEngine.close();
+    this.leases.close();
+
+    if (!killed) {
+      join();
+    }
+
+    try {
+      deleteMyEphemeralNode();
+    } catch (KeeperException e) {
+      LOG.warn("Failed deleting my ephemeral node", e);
+    }
+    this.zooKeeper.close();
+    LOG.info("stopping server " + this.serverNameFromMasterPOV +
+      "; zookeeper connection closed.");
+
+    LOG.info(Thread.currentThread().getName() + " exiting");
+  }
+
+  private boolean containsMetaTableRegions() {
+    return onlineRegions.containsKey(HRegionInfo.ROOT_REGIONINFO.getEncodedName())
+        || onlineRegions.containsKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
+  }
+
+  private boolean areAllUserRegionsOffline() {
+    if (getNumberOfOnlineRegions() > 2) return false;
+    boolean allUserRegionsOffline = true;
+    for (Map.Entry e: this.onlineRegions.entrySet()) {
+      if (!e.getValue().getRegionInfo().isMetaTable()) {
+        allUserRegionsOffline = false;
+        break;
+      }
+    }
+    return allUserRegionsOffline;
+  }
+
+  void tryRegionServerReport()
+  throws IOException {
+    HServerLoad hsl = buildServerLoad();
+    // Why we do this?
+    this.requestCount.set(0);
+    try {
+      this.hbaseMaster.regionServerReport(this.serverNameFromMasterPOV.getVersionedBytes(), hsl);
+    } catch (IOException ioe) {
+      if (ioe instanceof RemoteException) {
+        ioe = ((RemoteException)ioe).unwrapRemoteException();
+      }
+      if (ioe instanceof YouAreDeadException) {
+        // This will be caught and handled as a fatal error in run()
+        throw ioe;
+      }
+      // Couldn't connect to the master, get location from zk and reconnect
+      // Method blocks until new master is found or we are stopped
+      getMaster();
+    }
+  }
+
+  HServerLoad buildServerLoad() {
+    Collection regions = getOnlineRegionsLocalContext();
+    TreeMap regionLoads =
+      new TreeMap(Bytes.BYTES_COMPARATOR);
+    for (HRegion region: regions) {
+      regionLoads.put(region.getRegionName(), createRegionLoad(region));
+    }
+    MemoryUsage memory =
+      ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+    return new HServerLoad(requestCount.get(),(int)metrics.getRequests(),
+      (int)(memory.getUsed() / 1024 / 1024),
+      (int) (memory.getMax() / 1024 / 1024), regionLoads,
+      this.hlog.getCoprocessorHost().getCoprocessors());
+  }
+
+  String getOnlineRegionsAsPrintableString() {
+    StringBuilder sb = new StringBuilder();
+    for (HRegion r: this.onlineRegions.values()) {
+      if (sb.length() > 0) sb.append(", ");
+      sb.append(r.getRegionInfo().getEncodedName());
+    }
+    return sb.toString();
+  }
+
+  /**
+   * Wait on regions close.
+   */
+  private void waitOnAllRegionsToClose(final boolean abort) {
+    // Wait till all regions are closed before going out.
+    int lastCount = -1;
+    long previousLogTime = 0;
+    Set closedRegions = new HashSet();
+    while (!isOnlineRegionsEmpty()) {
+      int count = getNumberOfOnlineRegions();
+      // Only print a message if the count of regions has changed.
+      if (count != lastCount) {
+        // Log every second at most
+        if (System.currentTimeMillis() > (previousLogTime + 1000)) {
+          previousLogTime = System.currentTimeMillis();
+          lastCount = count;
+          LOG.info("Waiting on " + count + " regions to close");
+          // Only print out regions still closing if a small number else will
+          // swamp the log.
+          if (count < 10 && LOG.isDebugEnabled()) {
+            LOG.debug(this.onlineRegions);
+          }
+        }
+      }
+      // Ensure all user regions have been sent a close. Use this to
+      // protect against the case where an open comes in after we start the
+      // iterator of onlineRegions to close all user regions.
+      for (Map.Entry e : this.onlineRegions.entrySet()) {
+        HRegionInfo hri = e.getValue().getRegionInfo();
+        if (!this.regionsInTransitionInRS.containsKey(hri.getEncodedNameAsBytes())
+            && !closedRegions.contains(hri.getEncodedName())) {
+          closedRegions.add(hri.getEncodedName());
+          // Don't update zk with this close transition; pass false.
+          closeRegion(hri, abort, false);
+        }
+      }
+      // No regions in RIT, we could stop waiting now.
+      if (this.regionsInTransitionInRS.isEmpty()) {
+        if (!isOnlineRegionsEmpty()) {
+          LOG.info("We were exiting though online regions are not empty, because some regions failed closing");
+        }
+        break;
+      }
+      Threads.sleep(200);
+    }
+  }
+
+  private void closeWAL(final boolean delete) {
+    if (this.hlogForMeta != null) {
+      // All hlogs (meta and non-meta) are in the same directory. Don't call
+      // closeAndDelete here since that would delete all hlogs not just the
+      // meta ones. We will just 'close' the hlog for meta here, and leave
+      // the directory cleanup to the follow-on closeAndDelete call.
+      try { //Part of the patch from HBASE-7982 to do with exception handling 
+        this.hlogForMeta.close();
+      } catch (Throwable e) {
+        LOG.error("Metalog close and delete failed", RemoteExceptionHandler.checkThrowable(e));
+      }
+    }
+    if (this.hlog != null) {
+      try {
+        if (delete) {
+          hlog.closeAndDelete();
+        } else {
+          hlog.close();
+        }
+      } catch (Throwable e) {
+        LOG.error("Close and delete failed", RemoteExceptionHandler.checkThrowable(e));
+      }
+    }
+  }
+
+  private void closeAllScanners() {
+    // Close any outstanding scanners. Means they'll get an UnknownScanner
+    // exception next time they come in.
+    for (Map.Entry e : this.scanners.entrySet()) {
+      try {
+        e.getValue().close();
+      } catch (IOException ioe) {
+        LOG.warn("Closing scanner " + e.getKey(), ioe);
+      }
+    }
+  }
+
+  /*
+   * Run init. Sets up hlog and starts up all server threads.
+   *
+   * @param c Extra configuration.
+   */
+  protected void handleReportForDutyResponse(final MapWritable c)
+  throws IOException {
+    try {
+      for (Map.Entry e :c.entrySet()) {
+        String key = e.getKey().toString();
+        // The hostname the master sees us as.
+        if (key.equals(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)) {
+          String hostnameFromMasterPOV = e.getValue().toString();
+          this.serverNameFromMasterPOV = new ServerName(hostnameFromMasterPOV,
+            this.isa.getPort(), this.startcode);
+          LOG.info("Master passed us hostname to use. Was=" +
+            this.isa.getHostName() + ", Now=" +
+            this.serverNameFromMasterPOV.getHostname());
+          continue;
+        }
+        String value = e.getValue().toString();
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Config from master: " + key + "=" + value);
+        }
+        this.conf.set(key, value);
+      }
+
+      // hack! Maps DFSClient => RegionServer for logs.  HDFS made this
+      // config param for task trackers, but we can piggyback off of it.
+      if (this.conf.get("mapred.task.id") == null) {
+        this.conf.set("mapred.task.id", "hb_rs_" +
+          this.serverNameFromMasterPOV.toString());
+      }
+      // Set our ephemeral znode up in zookeeper now we have a name.
+      createMyEphemeralNode();
+
+      // Master sent us hbase.rootdir to use. Should be fully qualified
+      // path with file system specification included. Set 'fs.defaultFS'
+      // to match the filesystem on hbase.rootdir else underlying hadoop hdfs
+      // accessors will be going against wrong filesystem (unless all is set
+      // to defaults).
+      this.conf.set("fs.defaultFS", this.conf.get("hbase.rootdir"));
+      // Get fs instance used by this RS
+      this.fs = new HFileSystem(this.conf, this.useHBaseChecksum);
+      this.rootDir = new Path(this.conf.get(HConstants.HBASE_DIR));
+      this.tableDescriptors = new FSTableDescriptors(this.fs, this.rootDir, true);
+      this.hlog = setupWALAndReplication();
+      // Init in here rather than in constructor after thread name has been set
+      this.metrics = new RegionServerMetrics();
+      this.dynamicMetrics = RegionServerDynamicMetrics.newInstance(this);
+      this.rsHost = new RegionServerCoprocessorHost(this, this.conf);
+      startServiceThreads();
+      LOG.info("Serving as " + this.serverNameFromMasterPOV +
+        ", RPC listening on " + this.isa +
+        ", sessionid=0x" +
+        Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()));
+      isOnline = true;
+    } catch (Throwable e) {
+      LOG.warn("Exception in region server : ", e);
+      this.isOnline = false;
+      stop("Failed initialization");
+      throw convertThrowableToIOE(cleanup(e, "Failed init"),
+          "Region server startup failed");
+    } finally {
+      sleeper.skipSleepCycle();
+    }
+  }
+
+  private String getMyEphemeralNodePath() {
+    return ZKUtil.joinZNode(this.zooKeeper.rsZNode, getServerName().toString());
+  }
+
+  private void createMyEphemeralNode() throws KeeperException {
+    ZKUtil.createEphemeralNodeAndWatch(this.zooKeeper, getMyEphemeralNodePath(),
+      HConstants.EMPTY_BYTE_ARRAY);
+  }
+
+  private void deleteMyEphemeralNode() throws KeeperException {
+    ZKUtil.deleteNode(this.zooKeeper, getMyEphemeralNodePath());
+  }
+
+  public RegionServerAccounting getRegionServerAccounting() {
+    return regionServerAccounting;
+  }
+
+  /*
+   * @param r Region to get RegionLoad for.
+   *
+   * @return RegionLoad instance.
+   *
+   * @throws IOException
+   */
+  private HServerLoad.RegionLoad createRegionLoad(final HRegion r) {
+    byte[] name = r.getRegionName();
+    int stores = 0;
+    int storefiles = 0;
+    int storeUncompressedSizeMB = 0;
+    int storefileSizeMB = 0;
+    int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024);
+    int storefileIndexSizeMB = 0;
+    int rootIndexSizeKB = 0;
+    int totalStaticIndexSizeKB = 0;
+    int totalStaticBloomSizeKB = 0;
+    long totalCompactingKVs = 0;
+    long currentCompactedKVs = 0;
+    synchronized (r.stores) {
+      stores += r.stores.size();
+      for (Store store : r.stores.values()) {
+        storefiles += store.getStorefilesCount();
+        storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed()
+            / 1024 / 1024);
+        storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024);
+        storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024);
+        CompactionProgress progress = store.getCompactionProgress();
+        if (progress != null) {
+          totalCompactingKVs += progress.totalCompactingKVs;
+          currentCompactedKVs += progress.currentCompactedKVs;
+        }
+
+        rootIndexSizeKB +=
+            (int) (store.getStorefilesIndexSize() / 1024);
+
+        totalStaticIndexSizeKB +=
+          (int) (store.getTotalStaticIndexSize() / 1024);
+
+        totalStaticBloomSizeKB +=
+          (int) (store.getTotalStaticBloomSize() / 1024);
+      }
+    }
+    return new HServerLoad.RegionLoad(name, stores, storefiles,
+        storeUncompressedSizeMB,
+        storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, rootIndexSizeKB,
+        totalStaticIndexSizeKB, totalStaticBloomSizeKB,
+        (int) r.readRequestsCount.get(), (int) r.writeRequestsCount.get(),
+        totalCompactingKVs, currentCompactedKVs);
+  }
+
+  /**
+   * @param encodedRegionName
+   * @return An instance of RegionLoad.
+   */
+  public HServerLoad.RegionLoad createRegionLoad(final String encodedRegionName) {
+    HRegion r = null;
+    r = this.onlineRegions.get(encodedRegionName);
+    return r != null ? createRegionLoad(r) : null;
+  }
+
+  /*
+   * Cleanup after Throwable caught invoking method. Converts t to
+   * IOE if it isn't already.
+   *
+   * @param t Throwable
+   *
+   * @return Throwable converted to an IOE; methods can only let out IOEs.
+   */
+  private Throwable cleanup(final Throwable t) {
+    return cleanup(t, null);
+  }
+
+  /*
+   * Cleanup after Throwable caught invoking method. Converts t to
+   * IOE if it isn't already.
+   *
+   * @param t Throwable
+   *
+   * @param msg Message to log in error. Can be null.
+   *
+   * @return Throwable converted to an IOE; methods can only let out IOEs.
+   */
+  private Throwable cleanup(final Throwable t, final String msg) {
+    // Don't log as error if NSRE; NSRE is 'normal' operation.
+    if (t instanceof NotServingRegionException) {
+      LOG.debug("NotServingRegionException; " +  t.getMessage());
+      return t;
+    }
+    if (msg == null) {
+      LOG.error("", RemoteExceptionHandler.checkThrowable(t));
+    } else {
+      LOG.error(msg, RemoteExceptionHandler.checkThrowable(t));
+    }
+    if (!checkOOME(t)) {
+      checkFileSystem();
+    }
+    return t;
+  }
+
+  /*
+   * @param t
+   *
+   * @return Make t an IOE if it isn't already.
+   */
+  private IOException convertThrowableToIOE(final Throwable t) {
+    return convertThrowableToIOE(t, null);
+  }
+
+  /*
+   * @param t
+   *
+   * @param msg Message to put in new IOE if passed t is not an IOE
+   *
+   * @return Make t an IOE if it isn't already.
+   */
+  private IOException convertThrowableToIOE(final Throwable t, final String msg) {
+    return (t instanceof IOException ? (IOException) t : msg == null
+        || msg.length() == 0 ? new IOException(t) : new IOException(msg, t));
+  }
+
+  /*
+   * Check if an OOME and, if so, abort immediately to avoid creating more objects.
+   *
+   * @param e
+   *
+   * @return True if we OOME'd and are aborting.
+   */
+  public boolean checkOOME(final Throwable e) {
+    boolean stop = false;
+    try {
+      if (e instanceof OutOfMemoryError
+          || (e.getCause() != null && e.getCause() instanceof OutOfMemoryError)
+          || (e.getMessage() != null && e.getMessage().contains(
+              "java.lang.OutOfMemoryError"))) {
+        stop = true;
+        LOG.fatal(
+          "Run out of memory; HRegionServer will abort itself immediately", e);
+      }
+    } finally {
+      if (stop) {
+        Runtime.getRuntime().halt(1);
+      }
+    }
+    return stop;
+  }
+
+  /**
+   * Checks to see if the file system is still accessible. If not, sets
+   * abortRequested and stopRequested
+   *
+   * @return false if file system is not available
+   */
+  public boolean checkFileSystem() {
+    if (this.fsOk && this.fs != null) {
+      try {
+        FSUtils.checkFileSystemAvailable(this.fs);
+      } catch (IOException e) {
+        abort("File System not available", e);
+        this.fsOk = false;
+      }
+    }
+    return this.fsOk;
+  }
+
+  /*
+   * Inner class that runs on a long period checking if regions need compaction.
+   */
+  private static class CompactionChecker extends Chore {
+    private final HRegionServer instance;
+    private final int majorCompactPriority;
+    private final static int DEFAULT_PRIORITY = Integer.MAX_VALUE;
+
+    CompactionChecker(final HRegionServer h, final int sleepTime,
+        final Stoppable stopper) {
+      super("CompactionChecker", sleepTime, h);
+      this.instance = h;
+      LOG.info("Runs every " + StringUtils.formatTime(sleepTime));
+
+      /* MajorCompactPriority is configurable.
+       * If not set, the compaction will use default priority.
+       */
+      this.majorCompactPriority = this.instance.conf.
+        getInt("hbase.regionserver.compactionChecker.majorCompactPriority",
+        DEFAULT_PRIORITY);
+    }
+
+    @Override
+    protected void chore() {
+      for (HRegion r : this.instance.onlineRegions.values()) {
+        if (r == null)
+          continue;
+        for (Store s : r.getStores().values()) {
+          try {
+            if (s.needsCompaction()) {
+              // Queue a compaction. Will recognize if major is needed.
+              this.instance.compactSplitThread.requestCompaction(r, s, getName()
+                  + " requests compaction", null);
+            } else if (s.isMajorCompaction()) {
+              if (majorCompactPriority == DEFAULT_PRIORITY
+                  || majorCompactPriority > r.getCompactPriority()) {
+                this.instance.compactSplitThread.requestCompaction(r, s, getName()
+                    + " requests major compaction; use default priority", null);
+              } else {
+                this.instance.compactSplitThread.requestCompaction(r, s, getName()
+                    + " requests major compaction; use configured priority",
+                  this.majorCompactPriority, null);
+              }
+            }
+          } catch (IOException e) {
+            LOG.warn("Failed major compaction check on " + r, e);
+          }
+        }
+      }
+    }
+  }
+
+  class PeriodicMemstoreFlusher extends Chore {
+    final HRegionServer server;
+    final static int RANGE_OF_DELAY = 20000; //millisec
+    final static int MIN_DELAY_TIME = 3000; //millisec
+    public PeriodicMemstoreFlusher(int cacheFlushInterval, final HRegionServer server) {
+      super(server.getServerName() + "-MemstoreFlusherChore", cacheFlushInterval, server);
+      this.server = server;
+    }
+
+    @Override
+    protected void chore() {
+      for (HRegion r : this.server.onlineRegions.values()) {
+        if (r == null)
+          continue;
+        if (r.shouldFlush()) {
+          FlushRequester requester = server.getFlushRequester();
+          if (requester != null) {
+            long randomDelay = rand.nextInt(RANGE_OF_DELAY) + MIN_DELAY_TIME;
+            LOG.info(getName() + " requesting flush for region " + r.getRegionNameAsString() + 
+                " after a delay of " + randomDelay);
+            //Throttle the flushes by putting a delay. If we don't throttle, and there
+            //is a balanced write-load on the regions in a table, we might end up 
+            //overwhelming the filesystem with too many flushes at once.
+            requester.requestDelayedFlush(r, randomDelay);
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Report the status of the server. A server is online once all the startup is
+   * completed (setting up filesystem, starting service threads, etc.). This
+   * method is designed mostly to be useful in tests.
+   *
+   * @return true if online, false if not.
+   */
+  public boolean isOnline() {
+    return isOnline;
+  }
+
+  /**
+   * Setup WAL log and replication if enabled.
+   * Replication setup is done in here because it wants to be hooked up to WAL.
+   * @return A WAL instance.
+   * @throws IOException
+   */
+  private HLog setupWALAndReplication() throws IOException {
+    final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+    Path logdir = new Path(rootDir,
+      HLog.getHLogDirectoryName(this.serverNameFromMasterPOV.toString()));
+    if (LOG.isDebugEnabled()) LOG.debug("logdir=" + logdir);
+    if (this.fs.exists(logdir)) {
+      throw new RegionServerRunningException("Region server has already " +
+        "created directory at " + this.serverNameFromMasterPOV.toString());
+    }
+
+    // Instantiate replication manager if replication enabled.  Pass it the
+    // log directories.
+    createNewReplicationInstance(conf, this, this.fs, logdir, oldLogDir);
+    return instantiateHLog(logdir, oldLogDir);
+  }
+
+  // The method is synchronized to guarantee atomic update to hlogForMeta - 
+  // It is possible that multiple calls could be made to this method almost 
+  // at the same time, one for _ROOT_ and another for .META. (if they happen
+  // to be assigned to the same RS). Also, we want to use the same log for both
+  private synchronized HLog getMetaWAL() throws IOException {
+    if (this.hlogForMeta == null) {
+      final String logName
+      = HLog.getHLogDirectoryName(this.serverNameFromMasterPOV.toString());
+
+      Path logdir = new Path(rootDir, logName);
+      final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+      if (LOG.isDebugEnabled()) LOG.debug("logdir=" + logdir);
+      this.hlogForMeta = new HLog(this.fs.getBackingFs(), logdir, oldLogDir, this.conf,
+          getMetaWALActionListeners(), false, this.serverNameFromMasterPOV.toString(), true);
+    }
+    return this.hlogForMeta;
+  }
+
+  /**
+   * Called by {@link #setupWALAndReplication()} creating WAL instance.
+   * @param logdir
+   * @param oldLogDir
+   * @return WAL instance.
+   * @throws IOException
+   */
+  protected HLog instantiateHLog(Path logdir, Path oldLogDir) throws IOException {
+    return new HLog(this.fs.getBackingFs(), logdir, oldLogDir, this.conf,
+      getWALActionListeners(), this.serverNameFromMasterPOV.toString());
+  }
+
+  /**
+   * Called by {@link #instantiateHLog(Path, Path)} setting up WAL instance.
+   * Add any {@link WALActionsListener}s you want inserted before WAL startup.
+   * @return List of WALActionsListener that will be passed in to
+   * {@link HLog} on construction.
+   */
+  protected List getWALActionListeners() {
+    List listeners = new ArrayList();
+    // Log roller.
+    this.hlogRoller = new LogRoller(this, this);
+    listeners.add(this.hlogRoller);
+    if (this.replicationSourceHandler != null &&
+        this.replicationSourceHandler.getWALActionsListener() != null) {
+      // Replication handler is an implementation of WALActionsListener.
+      listeners.add(this.replicationSourceHandler.getWALActionsListener());
+    }
+    return listeners;
+  }
+
+  protected List getMetaWALActionListeners() {
+    List listeners = new ArrayList();
+    // Using a tmp log roller to ensure metaLogRoller is alive once it is not
+    // null (addendum patch on HBASE-7213)
+    MetaLogRoller tmpLogRoller = new MetaLogRoller(this, this);
+    String n = Thread.currentThread().getName();
+    Threads.setDaemonThreadRunning(tmpLogRoller.getThread(),
+        n + "MetaLogRoller", uncaughtExceptionHandler);
+    this.metaHLogRoller = tmpLogRoller;
+    tmpLogRoller = null;
+    listeners.add(this.metaHLogRoller);
+    return listeners;
+  }
+
+  protected LogRoller getLogRoller() {
+    return hlogRoller;
+  }
+
+  /*
+   * @param interval Interval since last time metrics were called.
+   */
+  protected void doMetrics() {
+    try {
+      metrics();
+    } catch (Throwable e) {
+      LOG.warn("Failed metrics", e);
+    }
+  }
+
+  protected void metrics() {
+    this.metrics.regions.set(this.onlineRegions.size());
+    this.metrics.incrementRequests(this.requestCount.get());
+    this.metrics.requests.intervalHeartBeat();
+    // Is this too expensive every three seconds getting a lock on onlineRegions
+    // and then per store carried? Can I make metrics be sloppier and avoid
+    // the synchronizations?
+    int stores = 0;
+    int storefiles = 0;
+    long memstoreSize = 0;
+    int readRequestsCount = 0;
+    int writeRequestsCount = 0;
+    long storefileIndexSize = 0;
+    HDFSBlocksDistribution hdfsBlocksDistribution =
+      new HDFSBlocksDistribution();
+    long totalStaticIndexSize = 0;
+    long totalStaticBloomSize = 0;
+    long numPutsWithoutWAL = 0;
+    long dataInMemoryWithoutWAL = 0;
+    long updatesBlockedMs = 0;
+
+    // Note that this is a map of Doubles instead of Longs. This is because we
+    // do effective integer division, which would perhaps truncate more than it
+    // should because we do it only on one part of our sum at a time. Rather
+    // than dividing at the end, where it is difficult to know the proper
+    // factor, everything is exact then truncated.
+    final Map tempVals =
+        new HashMap();
+
+    for (Map.Entry e : this.onlineRegions.entrySet()) {
+      HRegion r = e.getValue();
+      memstoreSize += r.memstoreSize.get();
+      numPutsWithoutWAL += r.numPutsWithoutWAL.get();
+      dataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get();
+      readRequestsCount += r.readRequestsCount.get();
+      writeRequestsCount += r.writeRequestsCount.get();
+      updatesBlockedMs += r.updatesBlockedMs.get();
+      synchronized (r.stores) {
+        stores += r.stores.size();
+        for (Map.Entry ee : r.stores.entrySet()) {
+            final Store store = ee.getValue();
+            final SchemaMetrics schemaMetrics = store.getSchemaMetrics();
+
+            {
+              long tmpStorefiles = store.getStorefilesCount();
+              schemaMetrics.accumulateStoreMetric(tempVals,
+                  StoreMetricType.STORE_FILE_COUNT, tmpStorefiles);
+              storefiles += tmpStorefiles;
+            }
+
+
+            {
+              long tmpStorefileIndexSize = store.getStorefilesIndexSize();
+              schemaMetrics.accumulateStoreMetric(tempVals,
+                  StoreMetricType.STORE_FILE_INDEX_SIZE,
+                  (long) (tmpStorefileIndexSize / (1024.0 * 1024)));
+              storefileIndexSize += tmpStorefileIndexSize;
+            }
+
+            {
+              long tmpStorefilesSize = store.getStorefilesSize();
+              schemaMetrics.accumulateStoreMetric(tempVals,
+                  StoreMetricType.STORE_FILE_SIZE_MB,
+                  (long) (tmpStorefilesSize / (1024.0 * 1024)));
+            }
+
+            {
+              long tmpStaticBloomSize = store.getTotalStaticBloomSize();
+              schemaMetrics.accumulateStoreMetric(tempVals,
+                  StoreMetricType.STATIC_BLOOM_SIZE_KB,
+                  (long) (tmpStaticBloomSize / 1024.0));
+              totalStaticBloomSize += tmpStaticBloomSize;
+            }
+
+            {
+              long tmpStaticIndexSize = store.getTotalStaticIndexSize();
+              schemaMetrics.accumulateStoreMetric(tempVals,
+                  StoreMetricType.STATIC_INDEX_SIZE_KB,
+                  (long) (tmpStaticIndexSize / 1024.0));
+              totalStaticIndexSize += tmpStaticIndexSize;
+            }
+
+            schemaMetrics.accumulateStoreMetric(tempVals,
+                StoreMetricType.MEMSTORE_SIZE_MB,
+                (long) (store.getMemStoreSize() / (1024.0 * 1024)));
+        }
+      }
+
+      hdfsBlocksDistribution.add(r.getHDFSBlocksDistribution());
+    }
+
+    for (Entry e : tempVals.entrySet()) {
+      RegionMetricsStorage.setNumericMetric(e.getKey(), e.getValue().longValue());
+    }
+
+    this.metrics.stores.set(stores);
+    this.metrics.storefiles.set(storefiles);
+    this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024)));
+    this.metrics.mbInMemoryWithoutWAL.set((int) (dataInMemoryWithoutWAL / (1024 * 1024)));
+    this.metrics.numPutsWithoutWAL.set(numPutsWithoutWAL);
+    this.metrics.storefileIndexSizeMB.set(
+        (int) (storefileIndexSize / (1024 * 1024)));
+    this.metrics.rootIndexSizeKB.set(
+        (int) (storefileIndexSize / 1024));
+    this.metrics.totalStaticIndexSizeKB.set(
+        (int) (totalStaticIndexSize / 1024));
+    this.metrics.totalStaticBloomSizeKB.set(
+        (int) (totalStaticBloomSize / 1024));
+    this.metrics.readRequestsCount.set(readRequestsCount);
+    this.metrics.writeRequestsCount.set(writeRequestsCount);
+    this.metrics.compactionQueueSize.set(compactSplitThread
+        .getCompactionQueueSize());
+    this.metrics.flushQueueSize.set(cacheFlusher
+        .getFlushQueueSize());
+    this.metrics.updatesBlockedSeconds.update(updatesBlockedMs > 0 ? 
+        updatesBlockedMs/1000: 0);
+    final long updatesBlockedMsHigherWater = cacheFlusher.getUpdatesBlockedMsHighWater().get();
+    this.metrics.updatesBlockedSecondsHighWater.update(updatesBlockedMsHigherWater > 0 ? 
+        updatesBlockedMsHigherWater/1000: 0);
+
+    BlockCache blockCache = cacheConfig.getBlockCache();
+    if (blockCache != null) {
+      this.metrics.blockCacheCount.set(blockCache.size());
+      this.metrics.blockCacheFree.set(blockCache.getFreeSize());
+      this.metrics.blockCacheSize.set(blockCache.getCurrentSize());
+      CacheStats cacheStats = blockCache.getStats();
+      this.metrics.blockCacheHitCount.set(cacheStats.getHitCount());
+      this.metrics.blockCacheMissCount.set(cacheStats.getMissCount());
+      this.metrics.blockCacheEvictedCount.set(blockCache.getEvictedCount());
+      double ratio = blockCache.getStats().getHitRatio();
+      int percent = (int) (ratio * 100);
+      this.metrics.blockCacheHitRatio.set(percent);
+      ratio = blockCache.getStats().getHitCachingRatio();
+      percent = (int) (ratio * 100);
+      this.metrics.blockCacheHitCachingRatio.set(percent);
+      // past N period block cache hit / hit caching ratios
+      cacheStats.rollMetricsPeriod();
+      ratio = cacheStats.getHitRatioPastNPeriods();
+      percent = (int) (ratio * 100);
+      this.metrics.blockCacheHitRatioPastNPeriods.set(percent);
+      ratio = cacheStats.getHitCachingRatioPastNPeriods();
+      percent = (int) (ratio * 100);
+      this.metrics.blockCacheHitCachingRatioPastNPeriods.set(percent);
+    }
+    float localityIndex = hdfsBlocksDistribution.getBlockLocalityIndex(
+      getServerName().getHostname());
+    int percent = (int) (localityIndex * 100);
+    this.metrics.hdfsBlocksLocalityIndex.set(percent);
+
+  }
+
+  /**
+   * @return Region server metrics instance.
+   */
+  public RegionServerMetrics getMetrics() {
+    return this.metrics;
+  }
+
+  /**
+   * @return Master address tracker instance.
+   */
+  public MasterAddressTracker getMasterAddressManager() {
+    return this.masterAddressManager;
+  }
+
+  /*
+   * Start maintanence Threads, Server, Worker and lease checker threads.
+   * Install an UncaughtExceptionHandler that calls abort of RegionServer if we
+   * get an unhandled exception. We cannot set the handler on all threads.
+   * Server's internal Listener thread is off limits. For Server, if an OOME, it
+   * waits a while then retries. Meantime, a flush or a compaction that tries to
+   * run should trigger same critical condition and the shutdown will run. On
+   * its way out, this server will shut down Server. Leases are sort of
+   * inbetween. It has an internal thread that while it inherits from Chore, it
+   * keeps its own internal stop mechanism so needs to be stopped by this
+   * hosting server. Worker logs the exception and exits.
+   */
+  private void startServiceThreads() throws IOException {
+    String n = Thread.currentThread().getName();
+    // Start executor services
+    this.service = new ExecutorService(getServerName().toString());
+    this.service.startExecutorService(ExecutorType.RS_OPEN_REGION,
+      conf.getInt("hbase.regionserver.executor.openregion.threads", 3));
+    this.service.startExecutorService(ExecutorType.RS_OPEN_ROOT,
+      conf.getInt("hbase.regionserver.executor.openroot.threads", 1));
+    this.service.startExecutorService(ExecutorType.RS_OPEN_META,
+      conf.getInt("hbase.regionserver.executor.openmeta.threads", 1));
+    this.service.startExecutorService(ExecutorType.RS_CLOSE_REGION,
+      conf.getInt("hbase.regionserver.executor.closeregion.threads", 3));
+    this.service.startExecutorService(ExecutorType.RS_CLOSE_ROOT,
+      conf.getInt("hbase.regionserver.executor.closeroot.threads", 1));
+    this.service.startExecutorService(ExecutorType.RS_CLOSE_META,
+      conf.getInt("hbase.regionserver.executor.closemeta.threads", 1));
+
+    Threads.setDaemonThreadRunning(this.hlogRoller.getThread(), n + ".logRoller",
+        uncaughtExceptionHandler);
+    Threads.setDaemonThreadRunning(this.cacheFlusher.getThread(), n + ".cacheFlusher",
+        uncaughtExceptionHandler);
+    Threads.setDaemonThreadRunning(this.compactionChecker.getThread(), n +
+      ".compactionChecker", uncaughtExceptionHandler);
+    Threads.setDaemonThreadRunning(this.periodicFlusher.getThread(), n +
+        ".periodicFlusher", uncaughtExceptionHandler);
+    if (this.healthCheckChore != null) {
+      Threads.setDaemonThreadRunning(this.healthCheckChore.getThread(), n + ".healthChecker",
+          uncaughtExceptionHandler);
+    }
+
+    // Leases is not a Thread. Internally it runs a daemon thread. If it gets
+    // an unhandled exception, it will just exit.
+    this.leases.setName(n + ".leaseChecker");
+    this.leases.start();
+
+    // Put up the webui.  Webui may come up on port other than configured if
+    // that port is occupied. Adjust serverInfo if this is the case.
+    this.webuiport = putUpWebUI();
+
+    if (this.replicationSourceHandler == this.replicationSinkHandler &&
+        this.replicationSourceHandler != null) {
+      this.replicationSourceHandler.startReplicationService();
+    } else if (this.replicationSourceHandler != null) {
+      this.replicationSourceHandler.startReplicationService();
+    } else if (this.replicationSinkHandler != null) {
+      this.replicationSinkHandler.startReplicationService();
+    }
+
+    // Start Server.  This service is like leases in that it internally runs
+    // a thread.
+    this.rpcServer.start();
+
+    // Create the log splitting worker and start it
+    this.splitLogWorker = new SplitLogWorker(this.zooKeeper,
+        this.getConfiguration(), this.getServerName().toString());
+    splitLogWorker.start();
+    
+  }
+
+  /**
+   * Puts up the webui.
+   * @return Returns final port -- maybe different from what we started with.
+   * @throws IOException
+   */
+  private int putUpWebUI() throws IOException {
+    int port = this.conf.getInt("hbase.regionserver.info.port", 60030);
+    // -1 is for disabling info server
+    if (port < 0) return port;
+    String addr = this.conf.get("hbase.regionserver.info.bindAddress", "0.0.0.0");
+    // check if auto port bind enabled
+    boolean auto = this.conf.getBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO,
+        false);
+    while (true) {
+      try {
+        this.infoServer = new InfoServer("regionserver", addr, port, false, this.conf);
+        this.infoServer.addServlet("status", "/rs-status", RSStatusServlet.class);
+        this.infoServer.addServlet("dump", "/dump", RSDumpServlet.class);
+        this.infoServer.setAttribute(REGIONSERVER, this);
+        this.infoServer.setAttribute(REGIONSERVER_CONF, conf);
+        this.infoServer.start();
+        break;
+      } catch (BindException e) {
+        if (!auto) {
+          // auto bind disabled throw BindException
+          throw e;
+        }
+        // auto bind enabled, try to use another port
+        LOG.info("Failed binding http info server to port: " + port);
+        port++;
+      }
+    }
+    return port;
+  }
+
+  /*
+   * Verify that server is healthy
+   */
+  private boolean isHealthy() {
+    if (!fsOk) {
+      // File system problem
+      return false;
+    }
+    // Verify that all threads are alive
+    if (!(leases.isAlive()
+        && cacheFlusher.isAlive() && hlogRoller.isAlive()
+        && this.compactionChecker.isAlive())
+        && this.periodicFlusher.isAlive()) {
+      stop("One or more threads are no longer alive -- stop");
+      return false;
+    }
+    if (metaHLogRoller != null && !metaHLogRoller.isAlive()) {
+      stop("Meta HLog roller thread is no longer alive -- stop");
+      return false;
+    }
+    return true;
+  }
+
+  public HLog getWAL() {
+    try {
+      return getWAL(null);
+    } catch (IOException e) {
+      LOG.warn("getWAL threw exception " + e);
+      return null; 
+    }
+  }
+
+  @Override
+  public HLog getWAL(HRegionInfo regionInfo) throws IOException {
+    //TODO: at some point this should delegate to the HLogFactory
+    //currently, we don't care about the region as much as we care about the 
+    //table.. (hence checking the tablename below)
+    //_ROOT_ and .META. regions have separate WAL. 
+    if (this.separateHLogForMeta && 
+        regionInfo != null && 
+        regionInfo.isMetaTable()) {
+      return getMetaWAL();
+    }
+    return this.hlog;
+  }
+
+  @Override
+  public CatalogTracker getCatalogTracker() {
+    return this.catalogTracker;
+  }
+
+  @Override
+  public void stop(final String msg) {
+    try {
+      if (this.rsHost != null) {
+        this.rsHost.preStop(msg);
+      }
+      this.stopped = true;
+      LOG.info("STOPPED: " + msg);
+      // Wakes run() if it is sleeping
+      sleeper.skipSleepCycle();
+    } catch (IOException exp) {
+      LOG.warn("The region server did not stop", exp);
+    }
+  }
+
+  public void waitForServerOnline(){
+    while (!isOnline() && !isStopped()){
+       sleeper.sleep();
+    }
+  }
+
+  @Override
+  public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct,
+      final boolean daughter)
+  throws KeeperException, IOException {
+    checkOpen();
+    LOG.info("Post open deploy tasks for region=" + r.getRegionNameAsString() +
+      ", daughter=" + daughter);
+    // Do checks to see if we need to compact (references or too many files)
+    for (Store s : r.getStores().values()) {
+      if (s.hasReferences() || s.needsCompaction()) {
+        getCompactionRequester().requestCompaction(r, s, "Opening Region", null);
+      }
+    }
+    // Update ZK, ROOT or META
+    if (r.getRegionInfo().isRootRegion()) {
+      RootLocationEditor.setRootLocation(getZooKeeper(),
+       this.serverNameFromMasterPOV);
+    } else if (r.getRegionInfo().isMetaRegion()) {
+      MetaEditor.updateMetaLocation(ct, r.getRegionInfo(),
+        this.serverNameFromMasterPOV);
+    } else {
+      if (daughter) {
+        // If daughter of a split, update whole row, not just location.
+        MetaEditor.addDaughter(ct, r.getRegionInfo(),
+          this.serverNameFromMasterPOV);
+      } else {
+        MetaEditor.updateRegionLocation(ct, r.getRegionInfo(),
+          this.serverNameFromMasterPOV);
+      }
+    }
+    LOG.info("Done with post open deploy task for region=" +
+      r.getRegionNameAsString() + ", daughter=" + daughter);
+
+  }
+
+  /**
+   * Return a reference to the metrics instance used for counting RPC calls.
+   * @return Metrics instance.
+   */
+  public HBaseRpcMetrics getRpcMetrics() {
+    return rpcServer.getRpcMetrics();
+  }
+
+  @Override
+  public RpcServer getRpcServer() {
+    return rpcServer;
+  }
+
+  /**
+   * Cause the server to exit without closing the regions it is serving, the log
+   * it is using and without notifying the master. Used unit testing and on
+   * catastrophic events such as HDFS is yanked out from under hbase or we OOME.
+   *
+   * @param reason
+   *          the reason we are aborting
+   * @param cause
+   *          the exception that caused the abort, or null
+   */
+  public void abort(String reason, Throwable cause) {
+    String msg = "ABORTING region server " + this + ": " + reason;
+    if (cause != null) {
+      LOG.fatal(msg, cause);
+    } else {
+      LOG.fatal(msg);
+    }
+    this.abortRequested = true;
+    this.reservedSpace.clear();
+    // HBASE-4014: show list of coprocessors that were loaded to help debug
+    // regionserver crashes.Note that we're implicitly using
+    // java.util.HashSet's toString() method to print the coprocessor names.
+    LOG.fatal("RegionServer abort: loaded coprocessors are: " +
+        CoprocessorHost.getLoadedCoprocessors());
+    if (this.metrics != null) {
+      LOG.info("Dump of metrics: " + this.metrics);
+    }
+    // Do our best to report our abort to the master, but this may not work
+    try {
+      if (cause != null) {
+        msg += "\nCause:\n" + StringUtils.stringifyException(cause);
+      }
+      if (hbaseMaster != null) {
+        hbaseMaster.reportRSFatalError(
+            this.serverNameFromMasterPOV.getVersionedBytes(), msg);
+      }
+    } catch (Throwable t) {
+      LOG.warn("Unable to report fatal error to master", t);
+    }
+    stop(reason);
+  }
+
+  /**
+   * @see HRegionServer#abort(String, Throwable)
+   */
+  public void abort(String reason) {
+    abort(reason, null);
+  }
+
+  public boolean isAborted() {
+    return this.abortRequested;
+  }
+
+  /*
+   * Simulate a kill -9 of this server. Exits w/o closing regions or cleaninup
+   * logs but it does close socket in case want to bring up server on old
+   * hostname+port immediately.
+   */
+  protected void kill() {
+    this.killed = true;
+    abort("Simulated kill");
+  }
+
+  /**
+   * Wait on all threads to finish. Presumption is that all closes and stops
+   * have already been called.
+   */
+  protected void join() {
+    Threads.shutdown(this.compactionChecker.getThread());
+    Threads.shutdown(this.periodicFlusher.getThread());
+    Threads.shutdown(this.cacheFlusher.getThread());
+    if (this.healthCheckChore != null) {
+      Threads.shutdown(this.healthCheckChore.getThread());
+    }
+    if (this.hlogRoller != null) {
+      Threads.shutdown(this.hlogRoller.getThread());
+    }
+    if (this.metaHLogRoller != null) {
+      Threads.shutdown(this.metaHLogRoller.getThread());
+    }
+    if (this.compactSplitThread != null) {
+      this.compactSplitThread.join();
+    }
+    if (this.service != null) this.service.shutdown();
+    if (this.replicationSourceHandler != null &&
+        this.replicationSourceHandler == this.replicationSinkHandler) {
+      this.replicationSourceHandler.stopReplicationService();
+    } else if (this.replicationSourceHandler != null) {
+      this.replicationSourceHandler.stopReplicationService();
+    } else if (this.replicationSinkHandler != null) {
+      this.replicationSinkHandler.stopReplicationService();
+    }
+  }
+
+  /**
+   * @return Return the object that implements the replication
+   * source service.
+   */
+  ReplicationSourceService getReplicationSourceService() {
+    return replicationSourceHandler;
+  }
+
+  /**
+   * @return Return the object that implements the replication
+   * sink service.
+   */
+  ReplicationSinkService getReplicationSinkService() {
+    return replicationSinkHandler;
+  }
+
+  /**
+   * Get the current master from ZooKeeper and open the RPC connection to it.
+   *
+   * Method will block until a master is available. You can break from this
+   * block by requesting the server stop.
+   *
+   * @return master + port, or null if server has been stopped
+   */
+  private ServerName getMaster() {
+    ServerName masterServerName = null;
+    long previousLogTime = 0;
+    HMasterRegionInterface master = null;
+    InetSocketAddress masterIsa = null;
+    while (keepLooping() && master == null) {
+      masterServerName = this.masterAddressManager.getMasterAddress();
+      if (masterServerName == null) {
+        if (!keepLooping()) {
+          // give up with no connection.
+          LOG.debug("No master found and cluster is stopped; bailing out");
+          return null;
+        }
+        LOG.debug("No master found; retry");
+        previousLogTime = System.currentTimeMillis();
+
+        sleeper.sleep();
+        continue;
+      }
+
+      masterIsa =
+        new InetSocketAddress(masterServerName.getHostname(), masterServerName.getPort());
+
+      LOG.info("Attempting connect to Master server at " +
+        this.masterAddressManager.getMasterAddress());
+      try {
+        // Do initial RPC setup. The final argument indicates that the RPC
+        // should retry indefinitely.
+        master = HBaseRPC.waitForProxy(this.rpcEngine,
+            HMasterRegionInterface.class, HMasterRegionInterface.VERSION,
+            masterIsa, this.conf, -1,
+            this.rpcTimeout, this.rpcTimeout);
+      } catch (IOException e) {
+        e = e instanceof RemoteException ?
+            ((RemoteException)e).unwrapRemoteException() : e;
+        if (e instanceof ServerNotRunningYetException) {
+          if (System.currentTimeMillis() > (previousLogTime+1000)){
+            LOG.info("Master isn't available yet, retrying");
+            previousLogTime = System.currentTimeMillis();
+          }
+        } else {
+          if (System.currentTimeMillis() > (previousLogTime + 1000)) {
+            LOG.warn("Unable to connect to master. Retrying. Error was:", e);
+            previousLogTime = System.currentTimeMillis();
+          }
+        }
+        try {
+          Thread.sleep(200);
+        } catch (InterruptedException ignored) {
+        }
+      }
+    }
+    LOG.info("Connected to master at " + masterIsa);
+    this.hbaseMaster = master;
+    return masterServerName;
+  }
+
+  /**
+   * @return True if we should break loop because cluster is going down or
+   * this server has been stopped or hdfs has gone bad.
+   */
+  private boolean keepLooping() {
+    return !this.stopped && isClusterUp();
+  }
+
+  /*
+   * Let the master know we're here Run initialization using parameters passed
+   * us by the master.
+   * @return A Map of key/value configurations we got from the Master else
+   * null if we failed to register.
+   * @throws IOException
+   */
+  private MapWritable reportForDuty() throws IOException {
+    MapWritable result = null;
+    ServerName masterServerName = getMaster();
+    if (masterServerName == null) return result;
+    try {
+      this.requestCount.set(0);
+      LOG.info("Telling master at " + masterServerName + " that we are up " +
+        "with port=" + this.isa.getPort() + ", startcode=" + this.startcode);
+      long now = EnvironmentEdgeManager.currentTimeMillis();
+      int port = this.isa.getPort();
+      result = this.hbaseMaster.regionServerStartup(port, this.startcode, now);
+    } catch (RemoteException e) {
+      IOException ioe = e.unwrapRemoteException();
+      if (ioe instanceof ClockOutOfSyncException) {
+        LOG.fatal("Master rejected startup because clock is out of sync", ioe);
+        // Re-throw IOE will cause RS to abort
+        throw ioe;
+      } else {
+        LOG.warn("remote error telling master we are up", e);
+      }
+    } catch (IOException e) {
+      LOG.warn("error telling master we are up", e);
+    }
+    return result;
+  }
+
+  /**
+   * Closes all regions.  Called on our way out.
+   * Assumes that its not possible for new regions to be added to onlineRegions
+   * while this method runs.
+   */
+  protected void closeAllRegions(final boolean abort) {
+    closeUserRegions(abort);
+    closeMetaTableRegions(abort);
+  }
+
+  /**
+   * Close root and meta regions if we carry them
+   * @param abort Whether we're running an abort.
+   */
+  void closeMetaTableRegions(final boolean abort) {
+    HRegion meta = null;
+    HRegion root = null;
+    this.lock.writeLock().lock();
+    try {
+      for (Map.Entry e: onlineRegions.entrySet()) {
+        HRegionInfo hri = e.getValue().getRegionInfo();
+        if (hri.isRootRegion()) {
+          root = e.getValue();
+        } else if (hri.isMetaRegion()) {
+          meta = e.getValue();
+        }
+        if (meta != null && root != null) break;
+      }
+    } finally {
+      this.lock.writeLock().unlock();
+    }
+    if (meta != null) closeRegion(meta.getRegionInfo(), abort, false);
+    if (root != null) closeRegion(root.getRegionInfo(), abort, false);
+  }
+
+  /**
+   * Schedule closes on all user regions.
+   * Should be safe calling multiple times because it wont' close regions
+   * that are already closed or that are closing.
+   * @param abort Whether we're running an abort.
+   */
+  void closeUserRegions(final boolean abort) {
+    this.lock.writeLock().lock();
+    try {
+      for (Map.Entry e: this.onlineRegions.entrySet()) {
+        HRegion r = e.getValue();
+        if (!r.getRegionInfo().isMetaTable() && r.isAvailable()) {
+          // Don't update zk with this close transition; pass false.
+          closeRegion(r.getRegionInfo(), abort, false);
+        }
+      }
+    } finally {
+      this.lock.writeLock().unlock();
+    }
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public HRegionInfo getRegionInfo(final byte[] regionName)
+  throws NotServingRegionException, IOException {
+    checkOpen();
+    requestCount.incrementAndGet();
+    return getRegion(regionName).getRegionInfo();
+  }
+
+  public Result getClosestRowBefore(final byte[] regionName, final byte[] row,
+      final byte[] family) throws IOException {
+    checkOpen();
+    requestCount.incrementAndGet();
+    try {
+      // locate the region we're operating on
+      HRegion region = getRegion(regionName);
+      // ask the region for all the data
+
+      Result r = region.getClosestRowBefore(row, family);
+      return r;
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t));
+    }
+  }
+
+  /** {@inheritDoc} */
+  public Result get(byte[] regionName, Get get) throws IOException {
+    checkOpen();
+    requestCount.incrementAndGet();
+    try {
+      HRegion region = getRegion(regionName);
+      return region.get(get, getLockFromId(get.getLockId()));
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t));
+    }
+  }
+
+  public boolean exists(byte[] regionName, Get get) throws IOException {
+    checkOpen();
+    requestCount.incrementAndGet();
+    try {
+      HRegion region = getRegion(regionName);
+      Integer lock = getLockFromId(get.getLockId());
+      if (region.getCoprocessorHost() != null) {
+        Boolean result = region.getCoprocessorHost().preExists(get);
+        if (result != null) {
+          return result.booleanValue();
+        }
+      }
+      Result r = region.get(get, lock);
+      boolean result = r != null && !r.isEmpty();
+      if (region.getCoprocessorHost() != null) {
+        result = region.getCoprocessorHost().postExists(get, result);
+      }
+      return result;
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t));
+    }
+  }
+
+  public void put(final byte[] regionName, final Put put) throws IOException {
+    if (put.getRow() == null) {
+      throw new IllegalArgumentException("update has null row");
+    }
+
+    checkOpen();
+    this.requestCount.incrementAndGet();
+    HRegion region = getRegion(regionName);
+    try {
+      if (!region.getRegionInfo().isMetaTable()) {
+        this.cacheFlusher.reclaimMemStoreMemory();
+      }
+      boolean writeToWAL = put.getWriteToWAL();
+      region.put(put, getLockFromId(put.getLockId()), writeToWAL);
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t));
+    }
+  }
+
+  public int put(final byte[] regionName, final List puts)
+      throws IOException {
+    checkOpen();
+    HRegion region = null;
+    int i = 0;
+
+    try {
+      region = getRegion(regionName);
+      if (!region.getRegionInfo().isMetaTable()) {
+        this.cacheFlusher.reclaimMemStoreMemory();
+      }
+
+      @SuppressWarnings("unchecked")
+      Pair[] putsWithLocks = new Pair[puts.size()];
+
+      for (Put p : puts) {
+        Integer lock = getLockFromId(p.getLockId());
+        putsWithLocks[i++] = new Pair(p, lock);
+      }
+
+      this.requestCount.addAndGet(puts.size());
+      OperationStatus codes[] = region.batchMutate(putsWithLocks);
+      for (i = 0; i < codes.length; i++) {
+        if (codes[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) {
+          return i;
+        }
+      }
+      return -1;
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t));
+    }
+  }
+
+  private boolean checkAndMutate(final byte[] regionName, final byte[] row,
+      final byte[] family, final byte[] qualifier, final CompareOp compareOp,
+      final WritableByteArrayComparable comparator, final Writable w,
+      Integer lock) throws IOException {
+    checkOpen();
+    this.requestCount.incrementAndGet();
+    HRegion region = getRegion(regionName);
+    try {
+      if (!region.getRegionInfo().isMetaTable()) {
+        this.cacheFlusher.reclaimMemStoreMemory();
+      }
+      return region.checkAndMutate(row, family, qualifier, compareOp,
+        comparator, w, lock, true);
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t));
+    }
+  }
+
+  /**
+   *
+   * @param regionName
+   * @param row
+   * @param family
+   * @param qualifier
+   * @param value
+   *          the expected value
+   * @param put
+   * @throws IOException
+   * @return true if the new put was execute, false otherwise
+   */
+  public boolean checkAndPut(final byte[] regionName, final byte[] row,
+      final byte[] family, final byte[] qualifier, final byte[] value,
+      final Put put) throws IOException {
+    checkOpen();
+    if (regionName == null) {
+      throw new IOException("Invalid arguments to checkAndPut "
+          + "regionName is null");
+    }
+    HRegion region = getRegion(regionName);
+    Integer lock = getLockFromId(put.getLockId());
+    WritableByteArrayComparable comparator = new BinaryComparator(value);
+    if (region.getCoprocessorHost() != null) {
+      Boolean result = region.getCoprocessorHost()
+        .preCheckAndPut(row, family, qualifier, CompareOp.EQUAL, comparator,
+          put);
+      if (result != null) {
+        return result.booleanValue();
+      }
+    }
+    boolean result = checkAndMutate(regionName, row, family, qualifier,
+        CompareOp.EQUAL, comparator, put,
+      lock);
+    if (region.getCoprocessorHost() != null) {
+      result = region.getCoprocessorHost().postCheckAndPut(row, family,
+        qualifier, CompareOp.EQUAL, comparator, put, result);
+    }
+    return result;
+  }
+
+  /**
+   *
+   * @param regionName
+   * @param row
+   * @param family
+   * @param qualifier
+   * @param compareOp
+   * @param comparator
+   * @param put
+   * @throws IOException
+   * @return true if the new put was execute, false otherwise
+   */
+  public boolean checkAndPut(final byte[] regionName, final byte[] row,
+      final byte[] family, final byte[] qualifier, final CompareOp compareOp,
+      final WritableByteArrayComparable comparator, final Put put)
+       throws IOException {
+    checkOpen();
+    if (regionName == null) {
+      throw new IOException("Invalid arguments to checkAndPut "
+          + "regionName is null");
+    }
+    HRegion region = getRegion(regionName);
+    Integer lock = getLockFromId(put.getLockId());
+    if (region.getCoprocessorHost() != null) {
+      Boolean result = region.getCoprocessorHost()
+        .preCheckAndPut(row, family, qualifier, compareOp, comparator, put);
+      if (result != null) {
+        return result.booleanValue();
+      }
+    }
+    boolean result = checkAndMutate(regionName, row, family, qualifier,
+      compareOp, comparator, put, lock);
+    if (region.getCoprocessorHost() != null) {
+      result = region.getCoprocessorHost().postCheckAndPut(row, family,
+        qualifier, compareOp, comparator, put, result);
+    }
+    return result;
+  }
+
+  /**
+   *
+   * @param regionName
+   * @param row
+   * @param family
+   * @param qualifier
+   * @param value
+   *          the expected value
+   * @param delete
+   * @throws IOException
+   * @return true if the new put was execute, false otherwise
+   */
+  public boolean checkAndDelete(final byte[] regionName, final byte[] row,
+      final byte[] family, final byte[] qualifier, final byte[] value,
+      final Delete delete) throws IOException {
+    checkOpen();
+
+    if (regionName == null) {
+      throw new IOException("Invalid arguments to checkAndDelete "
+          + "regionName is null");
+    }
+    HRegion region = getRegion(regionName);
+    Integer lock = getLockFromId(delete.getLockId());
+    WritableByteArrayComparable comparator = new BinaryComparator(value);
+    if (region.getCoprocessorHost() != null) {
+      Boolean result = region.getCoprocessorHost().preCheckAndDelete(row,
+        family, qualifier, CompareOp.EQUAL, comparator, delete);
+      if (result != null) {
+        return result.booleanValue();
+      }
+    }
+    boolean result = checkAndMutate(regionName, row, family, qualifier,
+      CompareOp.EQUAL, comparator, delete, lock);
+    if (region.getCoprocessorHost() != null) {
+      result = region.getCoprocessorHost().postCheckAndDelete(row, family,
+        qualifier, CompareOp.EQUAL, comparator, delete, result);
+    }
+    return result;
+  }
+
+  @Override
+  public List getStoreFileList(byte[] regionName, byte[] columnFamily)
+    throws IllegalArgumentException {
+    return getStoreFileList(regionName, new byte[][]{columnFamily});
+  }
+
+  @Override
+  public List getStoreFileList(byte[] regionName, byte[][] columnFamilies)
+    throws IllegalArgumentException {
+    HRegion region = getOnlineRegion(regionName);
+    if (region == null) {
+      throw new IllegalArgumentException("No region: " + new String(regionName)
+          + " available");
+    }
+    return region.getStoreFileList(columnFamilies);
+  }
+
+  public List getStoreFileList(byte[] regionName)
+    throws IllegalArgumentException {
+    HRegion region = getOnlineRegion(regionName);
+    if (region == null) {
+      throw new IllegalArgumentException("No region: " + new String(regionName)
+          + " available");
+    }
+    Set columnFamilies = region.getStores().keySet();
+    int nCF = columnFamilies.size();
+    return region.getStoreFileList(columnFamilies.toArray(new byte[nCF][]));
+  }
+  
+ /**
+  * Flushes the given region
+  */
+  public void flushRegion(byte[] regionName)
+    throws IllegalArgumentException, IOException {
+    HRegion region = getOnlineRegion(regionName);
+    if (region == null) {
+      throw new IllegalArgumentException("No region : " + new String(regionName)
+      + " available");
+    }
+    boolean needsCompaction = region.flushcache();
+    if (needsCompaction) {
+      this.compactSplitThread.requestCompaction(region, "Compaction through user triggered flush");
+    }
+  }
+
+ /**
+   * Flushes the given region if lastFlushTime < ifOlderThanTS
+   */
+   public void flushRegion(byte[] regionName, long ifOlderThanTS)
+     throws IllegalArgumentException, IOException {
+     HRegion region = getOnlineRegion(regionName);
+     if (region == null) {
+       throw new IllegalArgumentException("No region : " + new String(regionName)
+       + " available");
+     }
+     if (region.getLastFlushTime() < ifOlderThanTS) {
+      boolean needsCompaction = region.flushcache();
+      if (needsCompaction) {
+        this.compactSplitThread
+            .requestCompaction(region, "Compaction through user triggered flush");
+      }
+    }
+   }
+
+  /**
+   * Gets last flush time for the given region
+   * @return the last flush time for a region
+   */
+  public long getLastFlushTime(byte[] regionName) {
+    HRegion region = getOnlineRegion(regionName);
+    if (region == null) {
+      throw new IllegalArgumentException("No region : " + new String(regionName)
+      + " available");
+    }
+    return region.getLastFlushTime();
+  }
+ 
+  /**
+   *
+   * @param regionName
+   * @param row
+   * @param family
+   * @param qualifier
+   * @param compareOp
+   * @param comparator
+   * @param delete
+   * @throws IOException
+   * @return true if the new put was execute, false otherwise
+   */
+  public boolean checkAndDelete(final byte[] regionName, final byte[] row,
+      final byte[] family, final byte[] qualifier, final CompareOp compareOp,
+      final WritableByteArrayComparable comparator, final Delete delete)
+      throws IOException {
+    checkOpen();
+
+    if (regionName == null) {
+      throw new IOException("Invalid arguments to checkAndDelete "
+        + "regionName is null");
+    }
+    HRegion region = getRegion(regionName);
+    Integer lock = getLockFromId(delete.getLockId());
+    if (region.getCoprocessorHost() != null) {
+      Boolean result = region.getCoprocessorHost().preCheckAndDelete(row,
+        family, qualifier, compareOp, comparator, delete);
+     if (result != null) {
+       return result.booleanValue();
+     }
+    }
+    boolean result = checkAndMutate(regionName, row, family, qualifier,
+      compareOp, comparator, delete, lock);
+   if (region.getCoprocessorHost() != null) {
+     result = region.getCoprocessorHost().postCheckAndDelete(row, family,
+       qualifier, compareOp, comparator, delete, result);
+   }
+   return result;
+ }
+
+  //
+  // remote scanner interface
+  //
+
+  public long openScanner(byte[] regionName, Scan scan) throws IOException {
+    checkOpen();
+    NullPointerException npe = null;
+    if (regionName == null) {
+      npe = new NullPointerException("regionName is null");
+    } else if (scan == null) {
+      npe = new NullPointerException("scan is null");
+    }
+    if (npe != null) {
+      throw new IOException("Invalid arguments to openScanner", npe);
+    }
+    requestCount.incrementAndGet();
+    try {
+      HRegion r = getRegion(regionName);
+      r.checkRow(scan.getStartRow(), "Scan");
+      scan.setLoadColumnFamiliesOnDemand(r.isLoadingCfsOnDemandDefault()
+          || scan.doLoadColumnFamiliesOnDemand());
+      r.prepareScanner(scan);
+      RegionScanner s = null;
+      if (r.getCoprocessorHost() != null) {
+        s = r.getCoprocessorHost().preScannerOpen(scan);
+      }
+      if (s == null) {
+        s = r.getScanner(scan);
+      }
+      if (r.getCoprocessorHost() != null) {
+        RegionScanner savedScanner = r.getCoprocessorHost().postScannerOpen(
+            scan, s);
+        if (savedScanner == null) {
+          LOG.warn("PostScannerOpen impl returning null. "
+              + "Check the RegionObserver implementation.");
+        } else {
+          s = savedScanner;
+        }
+      }
+      return addScanner(s);
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t, "Failed openScanner"));
+    }
+  }
+
+  protected long addScanner(RegionScanner s) throws LeaseStillHeldException {
+    long scannerId = -1L;
+    scannerId = rand.nextLong();
+    String scannerName = String.valueOf(scannerId);
+    scanners.put(scannerName, s);
+    this.leases.createLease(scannerName, new ScannerListener(scannerName));
+    return scannerId;
+  }
+
+  public Result next(final long scannerId) throws IOException {
+    Result[] res = next(scannerId, 1);
+    if (res == null || res.length == 0) {
+      return null;
+    }
+    return res[0];
+  }
+
+  public Result[] next(final long scannerId, int nbRows) throws IOException {
+    String scannerName = String.valueOf(scannerId);
+    RegionScanner s = this.scanners.get(scannerName);
+    if (s == null) throw new UnknownScannerException("Name: " + scannerName);
+    try {
+      checkOpen();
+    } catch (IOException e) {
+      // If checkOpen failed, server not running or filesystem gone,
+      // cancel this lease; filesystem is gone or we're closing or something.
+      try {
+        this.leases.cancelLease(scannerName);
+      } catch (LeaseException le) {
+        LOG.info("Server shutting down and client tried to access missing scanner " +
+          scannerName);
+      }
+      throw e;
+    }
+    Leases.Lease lease = null;
+    try {
+      // Remove lease while its being processed in server; protects against case
+      // where processing of request takes > lease expiration time.
+      lease = this.leases.removeLease(scannerName);
+      List results = new ArrayList(nbRows);
+      long currentScanResultSize = 0;
+      List values = new ArrayList();
+
+      // Call coprocessor. Get region info from scanner.
+      HRegion region = getRegion(s.getRegionInfo().getRegionName());
+      if (region != null && region.getCoprocessorHost() != null) {
+        Boolean bypass = region.getCoprocessorHost().preScannerNext(s,
+            results, nbRows);
+        if (!results.isEmpty()) {
+          for (Result r : results) {
+            if (maxScannerResultSize < Long.MAX_VALUE){
+              for (KeyValue kv : r.raw()) {
+                currentScanResultSize += kv.heapSize();
+              }
+            }
+          }
+        }
+        if (bypass != null) {
+          return s.isFilterDone() && results.isEmpty() ? null
+              : results.toArray(new Result[0]);
+        }
+      }
+
+      MultiVersionConsistencyControl.setThreadReadPoint(s.getMvccReadPoint());
+      region.startRegionOperation();
+      try {
+        int i = 0;
+        synchronized(s) {
+          for (; i < nbRows
+              && currentScanResultSize < maxScannerResultSize; i++) {
+            // Collect values to be returned here
+            boolean moreRows = s.nextRaw(values, SchemaMetrics.METRIC_NEXTSIZE);
+            if (!values.isEmpty()) {
+              if (maxScannerResultSize < Long.MAX_VALUE){
+                for (KeyValue kv : values) {
+                  currentScanResultSize += kv.heapSize();
+                }
+              }
+              results.add(new Result(values));
+            }
+            if (!moreRows) {
+              break;
+            }
+            values.clear();
+          }
+        }
+        requestCount.addAndGet(i);
+        region.readRequestsCount.add(i);
+        region.setOpMetricsReadRequestCount(region.readRequestsCount.get());
+      } finally {
+        region.closeRegionOperation();
+      }
+      // coprocessor postNext hook
+      if (region != null && region.getCoprocessorHost() != null) {
+        region.getCoprocessorHost().postScannerNext(s, results, nbRows, true);
+      }
+
+      // If the scanner's filter - if any - is done with the scan
+      // and wants to tell the client to stop the scan. This is done by passing
+      // a null result.
+      return s.isFilterDone() && results.isEmpty() ? null
+          : results.toArray(new Result[0]);
+    } catch (Throwable t) {
+      if (t instanceof NotServingRegionException) {
+        this.scanners.remove(scannerName);
+      }
+      throw convertThrowableToIOE(cleanup(t));
+    } finally {
+      // We're done. On way out readd the above removed lease.  Adding resets
+      // expiration time on lease.
+      if (this.scanners.containsKey(scannerName)) {
+        if (lease != null) this.leases.addLease(lease);
+      }
+    }
+  }
+
+  public void close(final long scannerId) throws IOException {
+    try {
+      checkOpen();
+      requestCount.incrementAndGet();
+      String scannerName = String.valueOf(scannerId);
+      RegionScanner s = scanners.get(scannerName);
+
+      HRegion region = null;
+      if (s != null) {
+        // call coprocessor.
+        region = getRegion(s.getRegionInfo().getRegionName());
+        if (region != null && region.getCoprocessorHost() != null) {
+          if (region.getCoprocessorHost().preScannerClose(s)) {
+            return; // bypass
+          }
+        }
+      }
+
+      s = scanners.remove(scannerName);
+      if (s != null) {
+        s.close();
+        this.leases.cancelLease(scannerName);
+
+        if (region != null && region.getCoprocessorHost() != null) {
+          region.getCoprocessorHost().postScannerClose(s);
+        }
+      }
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t));
+    }
+  }
+
+  /**
+   * Instantiated as a scanner lease. If the lease times out, the scanner is
+   * closed
+   */
+  private class ScannerListener implements LeaseListener {
+    private final String scannerName;
+
+    ScannerListener(final String n) {
+      this.scannerName = n;
+    }
+
+    public void leaseExpired() {
+      RegionScanner s = scanners.remove(this.scannerName);
+      if (s != null) {
+        LOG.info("Scanner " + this.scannerName + " lease expired on region "
+            + s.getRegionInfo().getRegionNameAsString());
+        try {
+          HRegion region = getRegion(s.getRegionInfo().getRegionName());
+          if (region != null && region.getCoprocessorHost() != null) {
+            region.getCoprocessorHost().preScannerClose(s);
+          }
+
+          s.close();
+          if (region != null && region.getCoprocessorHost() != null) {
+            region.getCoprocessorHost().postScannerClose(s);
+          }
+        } catch (IOException e) {
+          LOG.error("Closing scanner for "
+              + s.getRegionInfo().getRegionNameAsString(), e);
+        }
+      } else {
+        LOG.info("Scanner " + this.scannerName + " lease expired");
+      }
+    }
+  }
+
+  //
+  // Methods that do the actual work for the remote API
+  //
+  public void delete(final byte[] regionName, final Delete delete)
+      throws IOException {
+    checkOpen();
+    try {
+      boolean writeToWAL = delete.getWriteToWAL();
+      this.requestCount.incrementAndGet();
+      HRegion region = getRegion(regionName);
+      if (!region.getRegionInfo().isMetaTable()) {
+        this.cacheFlusher.reclaimMemStoreMemory();
+      }
+      Integer lid = getLockFromId(delete.getLockId());
+      region.delete(delete, lid, writeToWAL);
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t));
+    }
+  }
+
+  public int delete(final byte[] regionName, final List deletes)
+      throws IOException {
+    checkOpen();
+    // Count of Deletes processed.
+    int i = 0;
+    HRegion region = null;
+    try {
+      region = getRegion(regionName);
+      if (!region.getRegionInfo().isMetaTable()) {
+        this.cacheFlusher.reclaimMemStoreMemory();
+      }
+      int size = deletes.size();
+      Integer[] locks = new Integer[size];
+      for (Delete delete : deletes) {
+        this.requestCount.incrementAndGet();
+        locks[i] = getLockFromId(delete.getLockId());
+        region.delete(delete, locks[i], delete.getWriteToWAL());
+        i++;
+      }
+    } catch (WrongRegionException ex) {
+      LOG.debug("Batch deletes: " + i, ex);
+      return i;
+    } catch (NotServingRegionException ex) {
+      return i;
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t));
+    }
+    return -1;
+  }
+
+  /**
+   * @deprecated {@link RowLock} and associated operations are deprecated.
+   */
+  public long lockRow(byte[] regionName, byte[] row) throws IOException {
+    checkOpen();
+    NullPointerException npe = null;
+    if (regionName == null) {
+      npe = new NullPointerException("regionName is null");
+    } else if (row == null) {
+      npe = new NullPointerException("row to lock is null");
+    }
+    if (npe != null) {
+      IOException io = new IOException("Invalid arguments to lockRow");
+      io.initCause(npe);
+      throw io;
+    }
+    requestCount.incrementAndGet();
+    try {
+      HRegion region = getRegion(regionName);
+      if (region.getCoprocessorHost() != null) {
+        region.getCoprocessorHost().preLockRow(regionName, row);
+      }
+      Integer r = region.obtainRowLock(row);
+      long lockId = addRowLock(r, region);
+      LOG.debug("Row lock " + lockId + " explicitly acquired by client");
+      return lockId;
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t, "Error obtaining row lock (fsOk: "
+          + this.fsOk + ")"));
+    }
+  }
+
+  protected long addRowLock(Integer r, HRegion region)
+      throws LeaseStillHeldException {
+    long lockId = -1L;
+    lockId = rand.nextLong();
+    String lockName = String.valueOf(lockId);
+    rowlocks.put(lockName, r);
+    this.leases.createLease(lockName, new RowLockListener(lockName, region));
+    return lockId;
+  }
+
+  /**
+   * Method to get the Integer lock identifier used internally from the long
+   * lock identifier used by the client.
+   *
+   * @param lockId
+   *          long row lock identifier from client
+   * @return intId Integer row lock used internally in HRegion
+   * @throws IOException
+   *           Thrown if this is not a valid client lock id.
+   */
+  Integer getLockFromId(long lockId) throws IOException {
+    if (lockId == -1L) {
+      return null;
+    }
+    String lockName = String.valueOf(lockId);
+    Integer rl = rowlocks.get(lockName);
+    if (rl == null) {
+      throw new UnknownRowLockException("Invalid row lock");
+    }
+    this.leases.renewLease(lockName);
+    return rl;
+  }
+
+  /**
+   * @deprecated {@link RowLock} and associated operations are deprecated.
+   */
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public void unlockRow(byte[] regionName, long lockId) throws IOException {
+    checkOpen();
+    NullPointerException npe = null;
+    if (regionName == null) {
+      npe = new NullPointerException("regionName is null");
+    } else if (lockId == -1L) {
+      npe = new NullPointerException("lockId is null");
+    }
+    if (npe != null) {
+      IOException io = new IOException("Invalid arguments to unlockRow");
+      io.initCause(npe);
+      throw io;
+    }
+    requestCount.incrementAndGet();
+    try {
+      HRegion region = getRegion(regionName);
+      if (region.getCoprocessorHost() != null) {
+        region.getCoprocessorHost().preUnLockRow(regionName, lockId);
+      }
+      String lockName = String.valueOf(lockId);
+      Integer r = rowlocks.remove(lockName);
+      if (r == null) {
+        throw new UnknownRowLockException(lockName);
+      }
+      region.releaseRowLock(r);
+      this.leases.cancelLease(lockName);
+      LOG.debug("Row lock " + lockId
+          + " has been explicitly released by client");
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t));
+    }
+  }
+
+  /**
+   * Atomically bulk load several HFiles into an open region
+   * @return true if successful, false is failed but recoverably (no action)
+   * @throws IOException if failed unrecoverably
+   */
+  @Override
+  public boolean bulkLoadHFiles(List> familyPaths,
+      byte[] regionName) throws IOException {
+    checkOpen();
+    HRegion region = getRegion(regionName);
+    boolean bypass = false;
+    if (region.getCoprocessorHost() != null) {
+      bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths);
+    }
+    boolean loaded = false;
+    if (!bypass) {
+      loaded = region.bulkLoadHFiles(familyPaths);
+    }
+    if (region.getCoprocessorHost() != null) {
+      loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded);
+    }
+    return loaded;
+  }
+
+  Map rowlocks = new ConcurrentHashMap();
+
+  /**
+   * Instantiated as a row lock lease. If the lease times out, the row lock is
+   * released
+   */
+  private class RowLockListener implements LeaseListener {
+    private final String lockName;
+    private final HRegion region;
+
+    RowLockListener(final String lockName, final HRegion region) {
+      this.lockName = lockName;
+      this.region = region;
+    }
+
+    public void leaseExpired() {
+      LOG.info("Row Lock " + this.lockName + " lease expired");
+      Integer r = rowlocks.remove(this.lockName);
+      if (r != null) {
+        region.releaseRowLock(r);
+      }
+    }
+  }
+
+  // Region open/close direct RPCs
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public RegionOpeningState openRegion(HRegionInfo region)
+  throws IOException {
+    return openRegion(region, -1);
+  }
+
+  @Override
+  @QosPriority(priority = HConstants.HIGH_QOS)
+  public RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNode)
+      throws IOException {
+    return openRegion(region, versionOfOfflineNode, null);
+  }
+
+  private RegionOpeningState openRegion(HRegionInfo region, int versionOfOfflineNode,
+      Map htds) throws IOException {
+    checkOpen();
+    HRegion onlineRegion = this.getFromOnlineRegions(region.getEncodedName());
+    if (null != onlineRegion) {
+      // See HBASE-5094. Cross check with META if still this RS is owning the
+      // region.
+      Pair p = MetaReader.getRegion(
+          this.catalogTracker, region.getRegionName());
+      if (this.getServerName().equals(p.getSecond())) {
+        LOG.warn("Attempted open of " + region.getEncodedName()
+            + " but already online on this server");
+        return RegionOpeningState.ALREADY_OPENED;
+      } else {
+        LOG.warn("The region " + region.getEncodedName()
+            + " is online on this server but META does not have this server.");
+        this.removeFromOnlineRegions(region.getEncodedName());
+      }
+    }
+    // Added to in-memory RS RIT that we are trying to open this region.
+    // Clear it if we fail queuing an open executor.
+    boolean isNewRit = addRegionsInTransition(region, OPEN);
+    if (!isNewRit) {
+      // An open is in progress. This is supported, but let's log this.
+      LOG.info("Receiving OPEN for the region:" +
+          region.getRegionNameAsString() + " , which we are already trying to OPEN" +
+          " - ignoring this new request for this region.");
+      return RegionOpeningState.OPENED;
+    }
+    try {
+      LOG.info("Received request to open region: " +
+        region.getRegionNameAsString());
+      HTableDescriptor htd = null;
+      if (htds == null) {
+        htd = this.tableDescriptors.get(region.getTableName());
+      } else {
+        htd = htds.get(region.getTableNameAsString());
+        if (htd == null) {
+          htd = this.tableDescriptors.get(region.getTableName());
+          htds.put(region.getTableNameAsString(), htd);
+        }
+      }
+
+      // Mark the region as OPENING up in zk.  This is how we tell the master control of the
+      // region has passed to this regionserver.
+      int version = transitionZookeeperOfflineToOpening(region, versionOfOfflineNode);
+      // Need to pass the expected version in the constructor.
+      if (region.isRootRegion()) {
+        this.service.submit(new OpenRootHandler(this, this, region, htd, version));
+      } else if (region.isMetaRegion()) {
+        this.service.submit(new OpenMetaHandler(this, this, region, htd, version));
+      } else {
+        this.service.submit(new OpenRegionHandler(this, this, region, htd, version));
+      }
+    } catch (IOException ie) {
+      // Clear from this server's RIT list else will stick around for ever.
+      removeFromRegionsInTransition(region);
+      throw ie;
+    }
+    return RegionOpeningState.OPENED;
+  }
+
+  /**
+   * Transition ZK node from OFFLINE to OPENING. The master will get a callback
+   * and will know that the region is now ours.
+   *
+   * @param hri
+   *          HRegionInfo whose znode we are updating
+   * @param versionOfOfflineNode
+   *          Version Of OfflineNode that needs to be compared before changing
+   *          the node's state from OFFLINE
+   * @throws IOException
+   */
+  int transitionZookeeperOfflineToOpening(final HRegionInfo hri, int versionOfOfflineNode)
+      throws IOException {
+    // TODO: should also handle transition from CLOSED?
+    int version = -1;
+    try {
+      // Initialize the znode version.
+      version = ZKAssign.transitionNode(this.zooKeeper, hri, this.getServerName(),
+          EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_OPENING, versionOfOfflineNode);
+    } catch (KeeperException e) {
+      LOG.error("Error transition from OFFLINE to OPENING for region=" + hri.getEncodedName(), e);
+    }
+    if (version == -1) {
+      // TODO: Fix this sloppyness. The exception should be coming off zk
+      // directly, not an
+      // intepretation at this high-level (-1 when we call transitionNode can
+      // mean many things).
+      throw new IOException("Failed transition from OFFLINE to OPENING for region="
+          + hri.getEncodedName());
+    }
+    return version;
+  }
+
+   /**
+    * String currentAction) throws RegionAlreadyInTransitionException { Add
+    * region to this regionservers list of in transitions regions ONLY if its not
+    * already byte[] encodedName = region.getEncodedNameAsBytes(); in transition.
+    * If a region already in RIT, we throw
+    * {@link RegionAlreadyInTransitionException}. if
+    * (this.regionsInTransitionInRS.containsKey(encodedName)) { Callers need to
+    * call {@link #removeFromRegionsInTransition(HRegionInfo)} when done or if
+    * boolean openAction = this.regionsInTransitionInRS.get(encodedName); error
+    * processing.
+    *
+    * @param region
+    *          Region to add
+    * @param currentAction
+    *          Whether OPEN or CLOSE.
+    * @throws RegionAlreadyInTransitionException
+    */
+   protected boolean addRegionsInTransition(final HRegionInfo region, final String currentAction)
+       throws RegionAlreadyInTransitionException {
+     boolean isOpen = currentAction.equals(OPEN);
+     Boolean action = this.regionsInTransitionInRS.putIfAbsent(
+         region.getEncodedNameAsBytes(), isOpen);
+     if (action == null) return true;
+     if (isOpen && action.booleanValue()) {
+       return false;
+     }
+     // The below exception message will be used in master.
+     throw new RegionAlreadyInTransitionException("Received:" + currentAction
+         + " for the region:" + region.getRegionNameAsString()
+         + ", which we are already trying to " + (action ? OPEN : CLOSE) + ".");
+   }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public void openRegions(List regions)
+  throws IOException {
+    checkOpen();
+    LOG.info("Received request to open " + regions.size() + " region(s)");
+    Map htds = new HashMap(regions.size());
+    for (HRegionInfo region : regions) openRegion(region, -1, htds);
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public boolean closeRegion(HRegionInfo region)
+  throws IOException {
+    return closeRegion(region, true, -1);
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public boolean closeRegion(final HRegionInfo region,
+    final int versionOfClosingNode)
+  throws IOException {
+    return closeRegion(region, true, versionOfClosingNode);
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public boolean closeRegion(HRegionInfo region, final boolean zk)
+  throws IOException {
+    return closeRegion(region, zk, -1);
+  }
+
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  protected boolean closeRegion(HRegionInfo region, final boolean zk,
+    final int versionOfClosingNode)
+  throws IOException {
+    checkOpen();
+    //Check for permissions to close.
+    HRegion actualRegion = this.getFromOnlineRegions(region.getEncodedName());
+    if (actualRegion != null && actualRegion.getCoprocessorHost() != null) {
+      actualRegion.getCoprocessorHost().preClose(false);
+    }
+    LOG.info("Received close region: " + region.getRegionNameAsString() +
+      ". Version of ZK closing node:" + versionOfClosingNode);
+    boolean hasit = this.onlineRegions.containsKey(region.getEncodedName());
+    if (!hasit) {
+      LOG.warn("Received close for region we are not serving; " +
+        region.getEncodedName());
+      throw new NotServingRegionException("Received close for "
+        + region.getRegionNameAsString() + " but we are not serving it");
+    }
+    return closeRegion(region, false, zk, versionOfClosingNode);
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public boolean closeRegion(byte[] encodedRegionName, boolean zk)
+    throws IOException {
+    return closeRegion(encodedRegionName, false, zk);
+  }
+
+  /**
+   * @param region Region to close
+   * @param abort True if we are aborting
+   * @param zk True if we are to update zk about the region close; if the close
+   * was orchestrated by master, then update zk.  If the close is being run by
+   * the regionserver because its going down, don't update zk.
+   * @return True if closed a region.
+   */
+  protected boolean closeRegion(HRegionInfo region, final boolean abort,
+      final boolean zk) {
+    return closeRegion(region, abort, zk, -1);
+  }
+
+
+  /**
+   * @param region Region to close
+   * @param abort True if we are aborting
+   * @param zk True if we are to update zk about the region close; if the close
+   * was orchestrated by master, then update zk.  If the close is being run by
+   * the regionserver because its going down, don't update zk.
+   * @param versionOfClosingNode
+   *   the version of znode to compare when RS transitions the znode from
+   *   CLOSING state.
+   * @return True if closed a region.
+   */
+  protected boolean closeRegion(HRegionInfo region, final boolean abort,
+      final boolean zk, final int versionOfClosingNode) {
+    
+    HRegion actualRegion = this.getFromOnlineRegions(region.getEncodedName());
+    if (actualRegion != null && actualRegion.getCoprocessorHost() != null) {
+      try {
+        actualRegion.getCoprocessorHost().preClose(abort);
+      } catch (IOException e) {
+        LOG.warn(e);
+        return false;
+      }
+    }
+    try {
+      addRegionsInTransition(region, CLOSE);
+    } catch (RegionAlreadyInTransitionException rate) {
+      LOG.warn("Received close for region we are already opening or closing; "
+          + region.getEncodedName());
+      return false;
+    }
+    boolean success = false;
+    try {
+      CloseRegionHandler crh = null;
+      if (region.isRootRegion()) {
+        crh = new CloseRootHandler(this, this, region, abort, zk, versionOfClosingNode);
+      } else if (region.isMetaRegion()) {
+        crh = new CloseMetaHandler(this, this, region, abort, zk, versionOfClosingNode);
+      } else {
+        crh = new CloseRegionHandler(this, this, region, abort, zk, versionOfClosingNode);
+      }
+      this.service.submit(crh);
+      success = true;
+    } finally {
+      // Remove from this server's RIT.
+      if (!success) removeFromRegionsInTransition(region);
+    }
+    return true;
+  }
+
+  /**
+   * @param encodedRegionName
+   *          encodedregionName to close
+   * @param abort
+   *          True if we are aborting
+   * @param zk
+   *          True if we are to update zk about the region close; if the close
+   *          was orchestrated by master, then update zk. If the close is being
+   *          run by the regionserver because its going down, don't update zk.
+   * @return True if closed a region.
+   */
+  protected boolean closeRegion(byte[] encodedRegionName, final boolean abort,
+      final boolean zk) throws IOException {
+    String encodedRegionNameStr = Bytes.toString(encodedRegionName);
+    HRegion region = this.getFromOnlineRegions(encodedRegionNameStr);
+    if (null != region) {
+      return closeRegion(region.getRegionInfo(), abort, zk);
+    }
+    LOG.error("The specified region name" + encodedRegionNameStr
+        + " does not exist to close the region.");
+    return false;
+  }
+
+  // Manual remote region administration RPCs
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public void flushRegion(HRegionInfo regionInfo)
+      throws NotServingRegionException, IOException {
+    checkOpen();
+    LOG.info("Flushing " + regionInfo.getRegionNameAsString());
+    HRegion region = getRegion(regionInfo.getRegionName());
+    boolean needsCompaction = region.flushcache();
+    if (needsCompaction) {
+      this.compactSplitThread.requestCompaction(region, "Compaction through user triggered flush");
+    }
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public void splitRegion(HRegionInfo regionInfo)
+      throws NotServingRegionException, IOException {
+    splitRegion(regionInfo, null);
+  }
+
+  @Override
+  public void splitRegion(HRegionInfo regionInfo, byte[] splitPoint)
+      throws NotServingRegionException, IOException {
+    checkOpen();
+    HRegion region = getRegion(regionInfo.getRegionName());
+    region.flushcache();
+    region.forceSplit(splitPoint);
+    compactSplitThread.requestSplit(region, region.checkSplit());
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public void compactRegion(HRegionInfo regionInfo, boolean major)
+      throws NotServingRegionException, IOException {
+    compactRegion(regionInfo, major, null);
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public void compactRegion(HRegionInfo regionInfo, boolean major,  byte[] family)
+      throws NotServingRegionException, IOException {
+    checkOpen();
+    HRegion region = getRegion(regionInfo.getRegionName());
+    Store store = null;
+    if (family != null) {
+      store = region.getStore(family);
+      if (store == null) {
+        throw new IOException("column family " + Bytes.toString(family) +
+          " does not exist in region " + new String(region.getRegionNameAsString()));
+      }
+    }
+
+    if (major) {
+      if (family != null) {
+        store.triggerMajorCompaction();
+      } else {
+        region.triggerMajorCompaction();
+      }
+    }
+    String familyLogMsg = (family != null)?" for column family: " + Bytes.toString(family):"";
+    LOG.trace("User-triggered compaction requested for region " +
+      region.getRegionNameAsString() + familyLogMsg);
+    String log = "User-triggered " + (major ? "major " : "") + "compaction" + familyLogMsg;
+    if (family != null) {
+      compactSplitThread.requestCompaction(region, store, log,
+        Store.PRIORITY_USER, null);
+    } else {
+      compactSplitThread.requestCompaction(region, log,
+        Store.PRIORITY_USER, null);
+    }
+  }
+
+  /** @return the info server */
+  public InfoServer getInfoServer() {
+    return infoServer;
+  }
+
+  /**
+   * @return true if a stop has been requested.
+   */
+  public boolean isStopped() {
+    return this.stopped;
+  }
+
+  @Override
+  public boolean isStopping() {
+    return this.stopping;
+  }
+
+  /**
+   *
+   * @return the configuration
+   */
+  public Configuration getConfiguration() {
+    return conf;
+  }
+
+  /** @return the write lock for the server */
+  ReentrantReadWriteLock.WriteLock getWriteLock() {
+    return lock.writeLock();
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public List getOnlineRegions() throws IOException {
+    checkOpen();
+    List list = new ArrayList(onlineRegions.size());
+    for (Map.Entry e: this.onlineRegions.entrySet()) {
+      list.add(e.getValue().getRegionInfo());
+    }
+    Collections.sort(list);
+    return list;
+  }
+
+  public int getNumberOfOnlineRegions() {
+    return this.onlineRegions.size();
+  }
+
+  boolean isOnlineRegionsEmpty() {
+    return this.onlineRegions.isEmpty();
+  }
+
+  /**
+   * @param encodedRegionName
+   * @return JSON Map of labels to values for passed in encodedRegionName
+   * @throws IOException
+   */
+  public byte [] getRegionStats(final String encodedRegionName)
+  throws IOException {
+    HRegion r = null;
+    synchronized (this.onlineRegions) {
+      r = this.onlineRegions.get(encodedRegionName);
+    }
+    if (r == null) return null;
+    ObjectMapper mapper = new ObjectMapper();
+    int stores = 0;
+    int storefiles = 0;
+    int storefileSizeMB = 0;
+    int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024);
+    int storefileIndexSizeMB = 0;
+    long totalCompactingKVs = 0;
+    long currentCompactedKVs = 0;
+    synchronized (r.stores) {
+      stores += r.stores.size();
+      for (Store store : r.stores.values()) {
+        storefiles += store.getStorefilesCount();
+        storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024);
+        storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024);
+      }
+    }
+    Map map = new TreeMap();
+    map.put("stores", stores);
+    map.put("storefiles", storefiles);
+    map.put("storefileSizeMB", storefileIndexSizeMB);
+    map.put("memstoreSizeMB", memstoreSizeMB);
+    StringWriter w = new StringWriter();
+    mapper.writeValue(w, map);
+    w.close();
+    return Bytes.toBytes(w.toString());
+  }
+
+  /**
+   * For tests and web ui.
+   * This method will only work if HRegionServer is in the same JVM as client;
+   * HRegion cannot be serialized to cross an rpc.
+   * @see #getOnlineRegions()
+   */
+  public Collection getOnlineRegionsLocalContext() {
+    Collection regions = this.onlineRegions.values();
+    return Collections.unmodifiableCollection(regions);
+  }
+
+  @Override
+  public void addToOnlineRegions(HRegion region) {
+    this.onlineRegions.put(region.getRegionInfo().getEncodedName(), region);
+  }
+
+  @Override
+  public boolean removeFromOnlineRegions(final String encodedName) {
+    HRegion toReturn = null;
+    toReturn = this.onlineRegions.remove(encodedName);
+    
+    //Clear all of the dynamic metrics as they are now probably useless.
+    //This is a clear because dynamic metrics could include metrics per cf and
+    //per hfile.  Figuring out which cfs, hfiles, and regions are still relevant to
+    //this region server would be an onerous task.  Instead just clear everything
+    //and on the next tick of the metrics everything that is still relevant will be
+    //re-added.
+    this.dynamicMetrics.clear();
+    return toReturn != null;
+  }
+
+  /**
+   * @return A new Map of online regions sorted by region size with the first
+   *         entry being the biggest.
+   */
+  public SortedMap getCopyOfOnlineRegionsSortedBySize() {
+    // we'll sort the regions in reverse
+    SortedMap sortedRegions = new TreeMap(
+        new Comparator() {
+          public int compare(Long a, Long b) {
+            return -1 * a.compareTo(b);
+          }
+        });
+    // Copy over all regions. Regions are sorted by size with biggest first.
+    for (HRegion region : this.onlineRegions.values()) {
+      sortedRegions.put(Long.valueOf(region.memstoreSize.get()), region);
+    }
+    return sortedRegions;
+  }
+
+  @Override
+  public HRegion getFromOnlineRegions(final String encodedRegionName) {
+    HRegion r = null;
+    r = this.onlineRegions.get(encodedRegionName);
+    return r;
+  }
+
+  /**
+   * @param regionName
+   * @return HRegion for the passed binary regionName or null if
+   *         named region is not member of the online regions.
+   */
+  public HRegion getOnlineRegion(final byte[] regionName) {
+    return getFromOnlineRegions(HRegionInfo.encodeRegionName(regionName));
+  }
+
+  /** @return the request count */
+  public AtomicInteger getRequestCount() {
+    return this.requestCount;
+  }
+
+  /**
+   * @return time stamp in millis of when this region server was started
+   */
+  public long getStartcode() {
+    return this.startcode;
+  }
+
+  /** @return reference to FlushRequester */
+  public FlushRequester getFlushRequester() {
+    return this.cacheFlusher;
+  }
+
+  /**
+   * Protected utility method for safely obtaining an HRegion handle.
+   *
+   * @param regionName
+   *          Name of online {@link HRegion} to return
+   * @return {@link HRegion} for regionName
+   * @throws NotServingRegionException
+   */
+  protected HRegion getRegion(final byte[] regionName)
+      throws NotServingRegionException {
+    HRegion region = null;
+    region = getOnlineRegion(regionName);
+    if (region == null) {
+      throw new NotServingRegionException("Region is not online: " +
+        Bytes.toStringBinary(regionName));
+    }
+    return region;
+  }
+
+  /**
+   * Get the top N most loaded regions this server is serving so we can tell the
+   * master which regions it can reallocate if we're overloaded. TODO: actually
+   * calculate which regions are most loaded. (Right now, we're just grabbing
+   * the first N regions being served regardless of load.)
+   */
+  protected HRegionInfo[] getMostLoadedRegions() {
+    ArrayList regions = new ArrayList();
+    for (HRegion r : onlineRegions.values()) {
+      if (!r.isAvailable()) {
+        continue;
+      }
+      if (regions.size() < numRegionsToReport) {
+        regions.add(r.getRegionInfo());
+      } else {
+        break;
+      }
+    }
+    return regions.toArray(new HRegionInfo[regions.size()]);
+  }
+
+  /**
+   * Called to verify that this server is up and running.
+   *
+   * @throws IOException
+   */
+  protected void checkOpen() throws IOException {
+    if (this.stopped || this.abortRequested) {
+      throw new RegionServerStoppedException("Server " + getServerName() +
+        " not running" + (this.abortRequested ? ", aborting" : ""));
+    }
+    if (!fsOk) {
+      throw new RegionServerStoppedException("File system not available");
+    }
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public ProtocolSignature getProtocolSignature(
+      String protocol, long version, int clientMethodsHashCode)
+  throws IOException {
+    if (protocol.equals(HRegionInterface.class.getName())) {
+      return new ProtocolSignature(HRegionInterface.VERSION, null);
+    }
+    throw new IOException("Unknown protocol: " + protocol);
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public long getProtocolVersion(final String protocol, final long clientVersion)
+  throws IOException {
+    if (protocol.equals(HRegionInterface.class.getName())) {
+      return HRegionInterface.VERSION;
+    }
+    throw new IOException("Unknown protocol: " + protocol);
+  }
+
+  @Override
+  public Leases getLeases() {
+    return leases;
+  }
+
+  /**
+   * @return Return the rootDir.
+   */
+  protected Path getRootDir() {
+    return rootDir;
+  }
+
+  /**
+   * @return Return the fs.
+   */
+  public FileSystem getFileSystem() {
+    return fs;
+  }
+
+  /**
+   * @return This servers {@link HServerInfo}
+   */
+  // TODO: Deprecate and do getServerName instead.
+  public HServerInfo getServerInfo() {
+    try {
+      return getHServerInfo();
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+    return null;
+  }
+
+  @Override
+  public void mutateRow(byte[] regionName, RowMutations rm)
+      throws IOException {
+    checkOpen();
+    if (regionName == null) {
+      throw new IOException("Invalid arguments to mutateRow " +
+      "regionName is null");
+    }
+    requestCount.incrementAndGet();
+    try {
+      HRegion region = getRegion(regionName);
+      if (!region.getRegionInfo().isMetaTable()) {
+        this.cacheFlusher.reclaimMemStoreMemory();
+      }
+      region.mutateRow(rm);
+    } catch (IOException e) {
+      checkFileSystem();
+      throw e;
+    }
+  }
+
+  @Override
+  public Result append(byte[] regionName, Append append)
+  throws IOException {
+    checkOpen();
+    if (regionName == null) {
+      throw new IOException("Invalid arguments to increment " +
+      "regionName is null");
+    }
+    requestCount.incrementAndGet();
+    try {
+      HRegion region = getRegion(regionName);
+      Integer lock = getLockFromId(append.getLockId());
+      Append appVal = append;
+      Result resVal;
+      if (region.getCoprocessorHost() != null) {
+        resVal = region.getCoprocessorHost().preAppend(appVal);
+        if (resVal != null) {
+          return resVal;
+        }
+      }
+      resVal = region.append(appVal, lock, append.getWriteToWAL());
+      if (region.getCoprocessorHost() != null) {
+        region.getCoprocessorHost().postAppend(appVal, resVal);
+      }
+      return resVal;
+    } catch (IOException e) {
+      checkFileSystem();
+      throw e;
+    }
+  }
+
+  @Override
+  public Result increment(byte[] regionName, Increment increment)
+  throws IOException {
+    checkOpen();
+    if (regionName == null) {
+      throw new IOException("Invalid arguments to increment " +
+      "regionName is null");
+    }
+    requestCount.incrementAndGet();
+    try {
+      HRegion region = getRegion(regionName);
+      Integer lock = getLockFromId(increment.getLockId());
+      Increment incVal = increment;
+      Result resVal;
+      if (region.getCoprocessorHost() != null) {
+        resVal = region.getCoprocessorHost().preIncrement(incVal);
+        if (resVal != null) {
+          return resVal;
+        }
+      }
+      resVal = region.increment(incVal, lock,
+          increment.getWriteToWAL());
+      if (region.getCoprocessorHost() != null) {
+        resVal = region.getCoprocessorHost().postIncrement(incVal, resVal);
+      }
+      return resVal;
+    } catch (IOException e) {
+      checkFileSystem();
+      throw e;
+    }
+  }
+
+  /** {@inheritDoc} */
+  public long incrementColumnValue(byte[] regionName, byte[] row,
+      byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
+      throws IOException {
+    checkOpen();
+
+    if (regionName == null) {
+      throw new IOException("Invalid arguments to incrementColumnValue "
+          + "regionName is null");
+    }
+    requestCount.incrementAndGet();
+    try {
+      HRegion region = getRegion(regionName);
+      if (region.getCoprocessorHost() != null) {
+        Long amountVal = region.getCoprocessorHost().preIncrementColumnValue(row,
+          family, qualifier, amount, writeToWAL);
+        if (amountVal != null) {
+          return amountVal.longValue();
+        }
+      }
+      long retval = region.incrementColumnValue(row, family, qualifier, amount,
+        writeToWAL);
+      if (region.getCoprocessorHost() != null) {
+        retval = region.getCoprocessorHost().postIncrementColumnValue(row,
+          family, qualifier, amount, writeToWAL, retval);
+      }
+      return retval;
+    } catch (IOException e) {
+      checkFileSystem();
+      throw e;
+    }
+  }
+
+  /** {@inheritDoc}
+   * @deprecated Use {@link #getServerName()} instead.
+   */
+  @Override
+  @QosPriority(priority=HConstants.HIGH_QOS)
+  public HServerInfo getHServerInfo() throws IOException {
+    checkOpen();
+    return new HServerInfo(new HServerAddress(this.isa),
+      this.startcode, this.webuiport);
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public  MultiResponse multi(MultiAction multi) throws IOException {
+    checkOpen();
+    MultiResponse response = new MultiResponse();
+    for (Map.Entry>> e : multi.actions.entrySet()) {
+      byte[] regionName = e.getKey();
+      List> actionsForRegion = e.getValue();
+      // sort based on the row id - this helps in the case where we reach the
+      // end of a region, so that we don't have to try the rest of the
+      // actions in the list.
+      Collections.sort(actionsForRegion);
+      Row action;
+      List> mutations = new ArrayList>();
+      for (Action a : actionsForRegion) {
+        action = a.getAction();
+        int originalIndex = a.getOriginalIndex();
+
+        try {
+          if (action instanceof Delete || action instanceof Put) {
+            mutations.add(a); 
+          } else if (action instanceof Get) {
+            response.add(regionName, originalIndex,
+                get(regionName, (Get)action));
+          } else if (action instanceof Exec) {
+            ExecResult result = execCoprocessor(regionName, (Exec)action);
+            response.add(regionName, new Pair(
+                a.getOriginalIndex(), result.getValue()
+            ));
+          } else if (action instanceof Increment) {
+            response.add(regionName, originalIndex,
+                increment(regionName, (Increment)action));
+          } else if (action instanceof Append) {
+            response.add(regionName, originalIndex,
+                append(regionName, (Append)action));
+          } else if (action instanceof RowMutations) {
+            mutateRow(regionName, (RowMutations)action);
+            response.add(regionName, originalIndex, new Result());
+          } else {
+            LOG.debug("Error: invalid Action, row must be a Get, Delete, " +
+                "Put, Exec, Increment, or Append.");
+            throw new DoNotRetryIOException("Invalid Action, row must be a " +
+                "Get, Delete, Put, Exec, Increment, or Append.");
+          }
+        } catch (IOException ex) {
+          response.add(regionName, originalIndex, ex);
+        }
+      }
+
+      // We do the puts with result.put so we can get the batching efficiency
+      // we so need. All this data munging doesn't seem great, but at least
+      // we arent copying bytes or anything.
+      if (!mutations.isEmpty()) {
+        try {
+          HRegion region = getRegion(regionName);
+
+          if (!region.getRegionInfo().isMetaTable()) {
+            this.cacheFlusher.reclaimMemStoreMemory();
+          }
+
+          List> mutationsWithLocks =
+              Lists.newArrayListWithCapacity(mutations.size());
+          for (Action a : mutations) {
+            Mutation m = (Mutation) a.getAction();
+
+            Integer lock;
+            try {
+              lock = getLockFromId(m.getLockId());
+            } catch (UnknownRowLockException ex) {
+              response.add(regionName, a.getOriginalIndex(), ex);
+              continue;
+            }
+            mutationsWithLocks.add(new Pair(m, lock));
+          }
+
+          this.requestCount.addAndGet(mutations.size());
+
+          OperationStatus[] codes =
+              region.batchMutate(mutationsWithLocks.toArray(new Pair[]{}));
+
+          for( int i = 0 ; i < codes.length ; i++) {
+            OperationStatus code = codes[i];
+
+            Action theAction = mutations.get(i);
+            Object result = null;
+
+            if (code.getOperationStatusCode() == OperationStatusCode.SUCCESS) {
+              result = new Result();
+            } else if (code.getOperationStatusCode()
+                == OperationStatusCode.SANITY_CHECK_FAILURE) {
+              // Don't send a FailedSanityCheckException as older clients will not know about
+              // that class being a subclass of DoNotRetryIOException
+              // and will retry mutations that will never succeed.
+              result = new DoNotRetryIOException(code.getExceptionMsg());
+            } else if (code.getOperationStatusCode() == OperationStatusCode.BAD_FAMILY) {
+              result = new NoSuchColumnFamilyException(code.getExceptionMsg());
+            }
+            // FAILURE && NOT_RUN becomes null, aka: need to run again.
+
+            response.add(regionName, theAction.getOriginalIndex(), result);
+          }
+        } catch (IOException ioe) {
+          // fail all the puts with the ioe in question.
+          for (Action a: mutations) {
+            response.add(regionName, a.getOriginalIndex(), ioe);
+          }
+        }
+      }
+    }
+    return response;
+  }
+
+  /**
+   * Executes a single {@link org.apache.hadoop.hbase.ipc.CoprocessorProtocol}
+   * method using the registered protocol handlers.
+   * {@link CoprocessorProtocol} implementations must be registered per-region
+   * via the
+   * {@link org.apache.hadoop.hbase.regionserver.HRegion#registerProtocol(Class, org.apache.hadoop.hbase.ipc.CoprocessorProtocol)}
+   * method before they are available.
+   *
+   * @param regionName name of the region against which the invocation is executed
+   * @param call an {@code Exec} instance identifying the protocol, method name,
+   *     and parameters for the method invocation
+   * @return an {@code ExecResult} instance containing the region name of the
+   *     invocation and the return value
+   * @throws IOException if no registered protocol handler is found or an error
+   *     occurs during the invocation
+   * @see org.apache.hadoop.hbase.regionserver.HRegion#registerProtocol(Class, org.apache.hadoop.hbase.ipc.CoprocessorProtocol)
+   */
+  @Override
+  public ExecResult execCoprocessor(byte[] regionName, Exec call)
+      throws IOException {
+    checkOpen();
+    requestCount.incrementAndGet();
+    try {
+      HRegion region = getRegion(regionName);
+      return region.exec(call);
+    } catch (Throwable t) {
+      throw convertThrowableToIOE(cleanup(t));
+    }
+  }
+
+  public String toString() {
+    return getServerName().toString();
+  }
+
+  /**
+   * Interval at which threads should run
+   *
+   * @return the interval
+   */
+  public int getThreadWakeFrequency() {
+    return threadWakeFrequency;
+  }
+
+  @Override
+  public ZooKeeperWatcher getZooKeeper() {
+    return zooKeeper;
+  }
+
+  @Override
+  public ServerName getServerName() {
+    // Our servername could change after we talk to the master.
+    return this.serverNameFromMasterPOV == null?
+      new ServerName(this.isa.getHostName(), this.isa.getPort(), this.startcode):
+        this.serverNameFromMasterPOV;
+  }
+
+  @Override
+  public CompactionRequestor getCompactionRequester() {
+    return this.compactSplitThread;
+  }
+
+  public ZooKeeperWatcher getZooKeeperWatcher() {
+    return this.zooKeeper;
+  }
+
+  public RegionServerCoprocessorHost getCoprocessorHost(){
+    return this.rsHost;
+  }
+
+  @Override
+  public boolean removeFromRegionsInTransition(final HRegionInfo hri) {
+    return this.regionsInTransitionInRS.remove(hri.getEncodedNameAsBytes());
+  }
+
+  @Override
+  public boolean containsKeyInRegionsInTransition(final HRegionInfo hri) {
+    return this.regionsInTransitionInRS.containsKey(hri.getEncodedNameAsBytes());
+  }
+
+  public ExecutorService getExecutorService() {
+    return service;
+  }
+
+  //
+  // Main program and support routines
+  //
+
+  /**
+   * Load the replication service objects, if any
+   */
+  static private void createNewReplicationInstance(Configuration conf,
+    HRegionServer server, FileSystem fs, Path logDir, Path oldLogDir) throws IOException{
+
+    // If replication is not enabled, then return immediately.
+    if (!conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false)) {
+      return;
+    }
+
+    // read in the name of the source replication class from the config file.
+    String sourceClassname = conf.get(HConstants.REPLICATION_SOURCE_SERVICE_CLASSNAME,
+                               HConstants.REPLICATION_SERVICE_CLASSNAME_DEFAULT);
+
+    // read in the name of the sink replication class from the config file.
+    String sinkClassname = conf.get(HConstants.REPLICATION_SINK_SERVICE_CLASSNAME,
+                             HConstants.REPLICATION_SERVICE_CLASSNAME_DEFAULT);
+
+    // If both the sink and the source class names are the same, then instantiate
+    // only one object.
+    if (sourceClassname.equals(sinkClassname)) {
+      server.replicationSourceHandler = (ReplicationSourceService)
+                                         newReplicationInstance(sourceClassname,
+                                         conf, server, fs, logDir, oldLogDir);
+      server.replicationSinkHandler = (ReplicationSinkService)
+                                         server.replicationSourceHandler;
+    }
+    else {
+      server.replicationSourceHandler = (ReplicationSourceService)
+                                         newReplicationInstance(sourceClassname,
+                                         conf, server, fs, logDir, oldLogDir);
+      server.replicationSinkHandler = (ReplicationSinkService)
+                                         newReplicationInstance(sinkClassname,
+                                         conf, server, fs, logDir, oldLogDir);
+    }
+  }
+
+  static private ReplicationService newReplicationInstance(String classname,
+    Configuration conf, HRegionServer server, FileSystem fs, Path logDir,
+    Path oldLogDir) throws IOException{
+
+    Class clazz = null;
+    try {
+      ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+      clazz = Class.forName(classname, true, classLoader);
+    } catch (java.lang.ClassNotFoundException nfe) {
+      throw new IOException("Cound not find class for " + classname);
+    }
+
+    // create an instance of the replication object.
+    ReplicationService service = (ReplicationService)
+                              ReflectionUtils.newInstance(clazz, conf);
+    service.initialize(server, fs, logDir, oldLogDir);
+    return service;
+  }
+
+  /**
+   * @param hrs
+   * @return Thread the RegionServer is running in correctly named.
+   * @throws IOException
+   */
+  public static Thread startRegionServer(final HRegionServer hrs)
+      throws IOException {
+    return startRegionServer(hrs, "regionserver" + hrs.isa.getPort());
+  }
+
+  /**
+   * @param hrs
+   * @param name
+   * @return Thread the RegionServer is running in correctly named.
+   * @throws IOException
+   */
+  public static Thread startRegionServer(final HRegionServer hrs,
+      final String name) throws IOException {
+    Thread t = new Thread(hrs);
+    t.setName(name);
+    t.start();
+    // Install shutdown hook that will catch signals and run an orderly shutdown
+    // of the hrs.
+    ShutdownHook.install(hrs.getConfiguration(), FileSystem.get(hrs
+        .getConfiguration()), hrs, t);
+    return t;
+  }
+
+  /**
+   * Utility for constructing an instance of the passed HRegionServer class.
+   *
+   * @param regionServerClass
+   * @param conf2
+   * @return HRegionServer instance.
+   */
+  public static HRegionServer constructRegionServer(
+      Class regionServerClass,
+      final Configuration conf2) {
+    try {
+      Constructor c = regionServerClass
+          .getConstructor(Configuration.class);
+      return c.newInstance(conf2);
+    } catch (Exception e) {
+      throw new RuntimeException("Failed construction of " + "Regionserver: "
+          + regionServerClass.toString(), e);
+    }
+  }
+
+  @Override
+  @QosPriority(priority=HConstants.REPLICATION_QOS)
+  public void replicateLogEntries(final HLog.Entry[] entries)
+  throws IOException {
+    checkOpen();
+    if (this.replicationSinkHandler == null) return;
+    this.replicationSinkHandler.replicateLogEntries(entries);
+  }
+
+  /**
+   * @see org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine
+   */
+  public static void main(String[] args) throws Exception {
+	VersionInfo.logVersion();
+    Configuration conf = HBaseConfiguration.create();
+    @SuppressWarnings("unchecked")
+    Class regionServerClass = (Class) conf
+        .getClass(HConstants.REGION_SERVER_IMPL, HRegionServer.class);
+
+    new HRegionServerCommandLine(regionServerClass).doMain(args);
+  }
+
+  @Override
+  public List getBlockCacheColumnFamilySummaries() throws IOException {
+    BlockCache c = new CacheConfig(this.conf).getBlockCache();
+    return c.getBlockCacheColumnFamilySummaries(this.conf);
+  }
+
+  @Override
+  public byte[][] rollHLogWriter() throws IOException, FailedLogCloseException {
+    HLog wal = this.getWAL();
+    return wal.rollWriter(true);
+  }
+
+  /**
+   * Gets the online regions of the specified table.
+   * This method looks at the in-memory onlineRegions.  It does not go to .META..
+   * Only returns online regions.  If a region on this table has been
+   * closed during a disable, etc., it will not be included in the returned list.
+   * So, the returned list may not necessarily be ALL regions in this table, its
+   * all the ONLINE regions in the table.
+   * @param tableName
+   * @return Online regions from tableName
+   */
+   public List getOnlineRegions(byte[] tableName) {
+     List tableRegions = new ArrayList();
+     synchronized (this.onlineRegions) {
+       for (HRegion region: this.onlineRegions.values()) {
+         HRegionInfo regionInfo = region.getRegionInfo();
+         if(Bytes.equals(regionInfo.getTableName(), tableName)) {
+           tableRegions.add(region);
+         }
+       }
+     }
+     return tableRegions;
+   }
+
+  // used by org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon (HBASE-4070).
+  public String[] getCoprocessors() {
+    TreeSet coprocessors = new TreeSet(
+        this.hlog.getCoprocessorHost().getCoprocessors());
+    Collection regions = getOnlineRegionsLocalContext();
+    for (HRegion region: regions) {
+      coprocessors.addAll(region.getCoprocessorHost().getCoprocessors());
+    }
+    return coprocessors.toArray(new String[0]);
+  }
+
+  /**
+   * Register bean with platform management server
+   */
+  @SuppressWarnings("deprecation")
+  void registerMBean() {
+    MXBeanImpl mxBeanInfo = MXBeanImpl.init(this);
+    mxBean = MBeanUtil.registerMBean("RegionServer", "RegionServer",
+        mxBeanInfo);
+    LOG.info("Registered RegionServer MXBean");
+  }
+
+  /**
+   * Get the current compaction state of the region.
+   *
+   * @param regionName the name of the region to check compaction statte.
+   * @return the compaction state name.
+   * @throws IOException exception
+   */
+  public String getCompactionState(final byte[] regionName) throws IOException {
+      checkOpen();
+      requestCount.incrementAndGet();
+      HRegion region = getRegion(regionName);
+      HRegionInfo info = region.getRegionInfo();
+      return CompactionRequest.getCompactionState(info.getRegionId()).name();
+  }
+
+  public long getResponseQueueSize(){
+    if (server != null) {
+      return server.getResponseQueueSize();
+    }
+    return 0;
+  }
+
+  private boolean isHealthCheckerConfigured() {
+    String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
+    return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
+  }
+
+  /**
+   * @return the underlying {@link CompactSplitThread} for the servers
+   */
+  public CompactSplitThread getCompactSplitThread() {
+    return this.compactSplitThread;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
new file mode 100644
index 0000000..71b9985
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
@@ -0,0 +1,87 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.util.ServerCommandLine;
+
+/**
+ * Class responsible for parsing the command line and starting the
+ * RegionServer.
+ */
+public class HRegionServerCommandLine extends ServerCommandLine {
+  private static final Log LOG = LogFactory.getLog(HRegionServerCommandLine.class);
+
+  private final Class regionServerClass;
+
+  private static final String USAGE =
+    "Usage: HRegionServer [-D conf.param=value] start";
+
+  public HRegionServerCommandLine(Class clazz) {
+    this.regionServerClass = clazz;
+  }
+
+  protected String getUsage() {
+    return USAGE;
+  }
+
+  private int start() throws Exception {
+    Configuration conf = getConf();
+
+    // If 'local', don't start a region server here. Defer to
+    // LocalHBaseCluster. It manages 'local' clusters.
+    if (LocalHBaseCluster.isLocal(conf)) {
+      LOG.warn("Not starting a distinct region server because "
+               + HConstants.CLUSTER_DISTRIBUTED + " is false");
+    } else {
+      logJVMInfo();
+      HRegionServer hrs = HRegionServer.constructRegionServer(regionServerClass, conf);
+      HRegionServer.startRegionServer(hrs);
+    }
+    return 0;
+  }
+
+  public int run(String args[]) throws Exception {
+    if (args.length != 1) {
+      usage(null);
+      return -1;
+    }
+
+    String cmd = args[0];
+
+    if ("start".equals(cmd)) {
+      return start();
+    } else if ("stop".equals(cmd)) {
+      System.err.println(
+        "To shutdown the regionserver run " +
+        "bin/hbase-daemon.sh stop regionserver or send a kill signal to" +
+        "the regionserver pid");
+      return -1;
+    } else {
+      usage("Unknown command: " + args[0]);
+      return -1;
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionThriftServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionThriftServer.java
new file mode 100644
index 0000000..7acfe9b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionThriftServer.java
@@ -0,0 +1,159 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.apache.hadoop.hbase.thrift.ThriftServerRunner.HBaseHandler.toBytes;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.thrift.ThriftServerRunner;
+import org.apache.hadoop.hbase.thrift.ThriftUtilities;
+import org.apache.hadoop.hbase.thrift.generated.IOError;
+import org.apache.hadoop.hbase.thrift.generated.TRowResult;
+
+/**
+ * HRegionThriftServer - this class starts up a Thrift server in the same
+ * JVM where the RegionServer is running. It inherits most of the
+ * functionality from the standard ThriftServer. This is good because
+ * we can maintain compatibility with applications that use the
+ * standard Thrift interface. For performance reasons, we can override
+ * methods to directly invoke calls into the HRegionServer and avoid the hop.
+ * 
                
+ * This can be enabled with hbase.regionserver.export.thrift set to true.
+ */
+public class HRegionThriftServer extends Thread {
+
+  public static final Log LOG = LogFactory.getLog(HRegionThriftServer.class);
+
+  private final HRegionServer rs;
+  private final ThriftServerRunner serverRunner;
+
+  /**
+   * Create an instance of the glue object that connects the
+   * RegionServer with the standard ThriftServer implementation
+   */
+  HRegionThriftServer(HRegionServer regionServer, Configuration conf)
+      throws IOException {
+    super("Region Thrift Server");
+    this.rs = regionServer;
+    this.serverRunner =
+        new ThriftServerRunner(conf, new HBaseHandlerRegion(conf));
+  }
+
+  /**
+   * Stop ThriftServer
+   */
+  void shutdown() {
+    serverRunner.shutdown();
+  }
+
+  @Override
+  public void run() {
+    serverRunner.run();
+  }
+
+  /**
+   * Inherit the Handler from the standard ThriftServerRunner. This allows us
+   * to use the default implementation for most calls. We override certain calls
+   * for performance reasons
+   */
+  private class HBaseHandlerRegion extends ThriftServerRunner.HBaseHandler {
+
+    /**
+     * Whether requests should be redirected to other RegionServers if the
+     * specified region is not hosted by this RegionServer.
+     */
+    private boolean redirect;
+
+    HBaseHandlerRegion(final Configuration conf) throws IOException {
+      super(conf);
+      initialize(conf);
+    }
+
+    /**
+     * Read and initialize config parameters
+     */
+    private void initialize(Configuration conf) {
+      this.redirect = conf.getBoolean("hbase.regionserver.thrift.redirect",
+          false);
+    }
+
+    // TODO: Override more methods to short-circuit for performance
+
+    /**
+     * Get a record. Short-circuit to get better performance.
+     */
+    @Override
+    public List getRowWithColumnsTs(ByteBuffer tableName,
+                                                ByteBuffer rowb,
+                                                List columns,
+                                                long timestamp,
+      Map attributes) throws IOError {
+      try {
+        byte[] row = toBytes(rowb);
+        HTable table = getTable(toBytes(tableName));
+        HRegionLocation location = table.getRegionLocation(row, false);
+        byte[] regionName = location.getRegionInfo().getRegionName();
+
+        if (columns == null) {
+          Get get = new Get(row);
+          get.setTimeRange(Long.MIN_VALUE, timestamp);
+          Result result = rs.get(regionName, get);
+          return ThriftUtilities.rowResultFromHBase(result);
+        }
+        Get get = new Get(row);
+        for(ByteBuffer column : columns) {
+          byte [][] famAndQf = KeyValue.parseColumn(toBytes(column));
+          if (famAndQf.length == 1) {
+            get.addFamily(famAndQf[0]);
+          } else {
+            get.addColumn(famAndQf[0], famAndQf[1]);
+          }
+        }
+        get.setTimeRange(Long.MIN_VALUE, timestamp);
+        Result result = rs.get(regionName, get);
+        return ThriftUtilities.rowResultFromHBase(result);
+      } catch (NotServingRegionException e) {
+        if (!redirect) {
+          LOG.warn(e.getMessage(), e);
+          throw new IOError(e.getMessage());
+        }
+        LOG.debug("ThriftServer redirecting getRowWithColumnsTs");
+        return super.getRowWithColumnsTs(tableName, rowb, columns, timestamp,
+                                         attributes);
+      } catch (IOException e) {
+        LOG.warn(e.getMessage(), e);
+        throw new IOError(e.getMessage());
+      }
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
new file mode 100644
index 0000000..0c9f151
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Split size is the number of regions that are on this server that all are
+ * of the same table, squared, times the region flush size OR the maximum
+ * region split size, whichever is smaller.  For example, if the flush size
+ * is 128M, then on first flush we will split which will make two regions
+ * that will split when their size is 2 * 2 * 128M = 512M.  If one of these
+ * regions splits, then there are three regions and now the split size is
+ * 3 * 3 * 128M =  1152M, and so on until we reach the configured
+ * maximum filesize and then from there on out, we'll use that.
+ */
+public class IncreasingToUpperBoundRegionSplitPolicy
+extends ConstantSizeRegionSplitPolicy {
+  static final Log LOG =
+    LogFactory.getLog(IncreasingToUpperBoundRegionSplitPolicy.class);
+  private long flushSize;
+
+  @Override
+  protected void configureForRegion(HRegion region) {
+    super.configureForRegion(region);
+    Configuration conf = getConf();
+    HTableDescriptor desc = region.getTableDesc();
+    if (desc != null) {
+      this.flushSize = desc.getMemStoreFlushSize();
+    }
+    if (this.flushSize <= 0) {
+      this.flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
+        HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE);
+    }
+  }
+
+  @Override
+  protected boolean shouldSplit() {
+    if (region.shouldForceSplit()) return true;
+    boolean foundABigStore = false;
+    // Get count of regions that have the same common table as this.region
+    int tableRegionsCount = getCountOfCommonTableRegions();
+    // Get size to check
+    long sizeToCheck = getSizeToCheck(tableRegionsCount);
+
+    for (Store store : region.getStores().values()) {
+      // If any of the stores is unable to split (eg they contain reference files)
+      // then don't split
+      if ((!store.canSplit())) {
+        return false;
+      }
+
+      // Mark if any store is big enough
+      long size = store.getSize();
+      if (size > sizeToCheck) {
+        LOG.debug("ShouldSplit because " + store.getColumnFamilyName() +
+          " size=" + size + ", sizeToCheck=" + sizeToCheck +
+          ", regionsWithCommonTable=" + tableRegionsCount);
+        foundABigStore = true;
+        break;
+      }
+    }
+
+    return foundABigStore;
+  }
+
+  /**
+   * @return Region max size or count of regions squared * flushsize, which ever is
+   * smaller; guard against there being zero regions on this server.
+   */
+  long getSizeToCheck(final int tableRegionsCount) {
+    return tableRegionsCount == 0? getDesiredMaxFileSize():
+      Math.min(getDesiredMaxFileSize(),
+        this.flushSize * (tableRegionsCount * tableRegionsCount));
+  }
+
+  /**
+   * @return Count of regions on this server that share the table this.region
+   * belongs to
+   */
+  private int getCountOfCommonTableRegions() {
+    RegionServerServices rss = this.region.getRegionServerServices();
+    // Can be null in tests
+    if (rss == null) return 0;
+    byte [] tablename = this.region.getTableDesc().getName();
+    int tableRegionsCount = 0;
+    try {
+      List hri = rss.getOnlineRegions(tablename);
+      tableRegionsCount = hri == null || hri.isEmpty()? 0: hri.size();
+    } catch (IOException e) {
+      LOG.debug("Failed getOnlineRegions " + Bytes.toString(tablename), e);
+    }
+    return tableRegionsCount;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java
new file mode 100644
index 0000000..db2e02d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java
@@ -0,0 +1,78 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Scan;
+
+/**
+ * Special internal-only scanner, currently used for increment operations to
+ * allow additional server-side arguments for Scan operations.
+ * 
                
+ * Rather than adding new options/parameters to the public Scan API, this new
+ * class has been created.
+ * 
                
+ * Supports adding an option to only read from the MemStore with
+ * {@link #checkOnlyMemStore()} or to only read from StoreFiles with
+ * {@link #checkOnlyStoreFiles()}.
+ */
+class InternalScan extends Scan {
+  private boolean memOnly = false;
+  private boolean filesOnly = false;
+
+  /**
+   * @param get get to model scan after
+   */
+  public InternalScan(Get get) {
+    super(get);
+  }
+
+  /**
+   * StoreFiles will not be scanned. Only MemStore will be scanned.
+   */
+  public void checkOnlyMemStore() {
+    memOnly = true;
+    filesOnly = false;
+  }
+
+  /**
+   * MemStore will not be scanned. Only StoreFiles will be scanned.
+   */
+  public void checkOnlyStoreFiles() {
+    memOnly = false;
+    filesOnly = true;
+  }
+
+  /**
+   * Returns true if only the MemStore should be checked.  False if not.
+   * @return true to only check MemStore
+   */
+  public boolean isCheckOnlyMemStore() {
+    return (memOnly);
+  }
+
+  /**
+   * Returns true if only StoreFiles should be checked.  False if not.
+   * @return true if only check StoreFiles
+   */
+  public boolean isCheckOnlyStoreFiles() {
+    return (filesOnly);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java
new file mode 100644
index 0000000..6cbed5a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java
@@ -0,0 +1,86 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Internal scanners differ from client-side scanners in that they operate on
+ * HStoreKeys and byte[] instead of RowResults. This is because they are
+ * actually close to how the data is physically stored, and therefore it is more
+ * convenient to interact with them that way. It is also much easier to merge
+ * the results across SortedMaps than RowResults.
+ *
+ * 
                Additionally, we need to be able to determine if the scanner is doing
+ * wildcard column matches (when only a column family is specified or if a
+ * column regex is specified) or if multiple members of the same column family
+ * were specified. If so, we need to ignore the timestamp to ensure that we get
+ * all the family members, as they may have been last updated at different
+ * times.
+ */
+public interface InternalScanner extends Closeable {
+  /**
+   * Grab the next row's worth of values.
+   * @param results return output array
+   * @return true if more rows exist after this one, false if scanner is done
+   * @throws IOException e
+   */
+  public boolean next(List results) throws IOException;
+  
+  /**
+   * Grab the next row's worth of values.
+   * @param results return output array
+   * @param metric the metric name
+   * @return true if more rows exist after this one, false if scanner is done
+   * @throws IOException e
+   */
+  public boolean next(List results, String metric) throws IOException;
+
+  /**
+   * Grab the next row's worth of values with a limit on the number of values
+   * to return.
+   * @param result return output array
+   * @param limit limit on row count to get
+   * @return true if more rows exist after this one, false if scanner is done
+   * @throws IOException e
+   */
+  public boolean next(List result, int limit) throws IOException;
+  
+  /**
+   * Grab the next row's worth of values with a limit on the number of values
+   * to return.
+   * @param result return output array
+   * @param limit limit on row count to get
+   * @param metric the metric name
+   * @return true if more rows exist after this one, false if scanner is done
+   * @throws IOException e
+   */
+  public boolean next(List result, int limit, String metric) throws IOException;
+
+  /**
+   * Closes the scanner and releases any resources it has allocated
+   * @throws IOException
+   */
+  public void close() throws IOException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java
new file mode 100644
index 0000000..85c3dd9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.Arrays;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * A custom RegionSplitPolicy implementing a SplitPolicy that groups
+ * rows by a prefix of the row-key
+ *
+ * This ensures that a region is not split "inside" a prefix of a row key.
+ * I.e. rows can be co-located in a region by their prefix.
+ */
+public class KeyPrefixRegionSplitPolicy extends IncreasingToUpperBoundRegionSplitPolicy {
+  private static final Log LOG = LogFactory
+      .getLog(KeyPrefixRegionSplitPolicy.class);
+  @Deprecated
+  public static final String PREFIX_LENGTH_KEY_DEPRECATED = "prefix_split_key_policy.prefix_length";
+  public static final String PREFIX_LENGTH_KEY = "KeyPrefixRegionSplitPolicy.prefix_length";
+
+  private int prefixLength = 0;
+
+  @Override
+  protected void configureForRegion(HRegion region) {
+    super.configureForRegion(region);
+    if (region != null) {
+      prefixLength = 0;
+
+      // read the prefix length from the table descriptor
+      String prefixLengthString = region.getTableDesc().getValue(
+          PREFIX_LENGTH_KEY);
+      if (prefixLengthString == null) {
+        //read the deprecated value
+        prefixLengthString = region.getTableDesc().getValue(PREFIX_LENGTH_KEY_DEPRECATED);
+        if (prefixLengthString == null) {
+          LOG.error(PREFIX_LENGTH_KEY + " not specified for table "
+              + region.getTableDesc().getNameAsString()
+              + ". Using default RegionSplitPolicy");
+          return;
+        }
+      }
+      try {
+        prefixLength = Integer.parseInt(prefixLengthString);
+      } catch (NumberFormatException nfe) {
+        // ignore
+      }
+      if (prefixLength <= 0) {
+        LOG.error("Invalid value for " + PREFIX_LENGTH_KEY + " for table "
+            + region.getTableDesc().getNameAsString() + ":"
+            + prefixLengthString + ". Using default RegionSplitPolicy");
+      }
+    }
+  }
+
+  @Override
+  protected byte[] getSplitPoint() {
+    byte[] splitPoint = super.getSplitPoint();
+    if (prefixLength > 0 && splitPoint != null && splitPoint.length > 0) {
+      // group split keys by a prefix
+      return Arrays.copyOf(splitPoint,
+          Math.min(prefixLength, splitPoint.length));
+    } else {
+      return splitPoint;
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
new file mode 100644
index 0000000..8613a87
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
@@ -0,0 +1,406 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.KVComparator;
+
+/**
+ * Implements a heap merge across any number of KeyValueScanners.
+ * 
                
+ * Implements KeyValueScanner itself.
+ * 
                
+ * This class is used at the Region level to merge across Stores
+ * and at the Store level to merge across the memstore and StoreFiles.
+ * 
                
+ * In the Region case, we also need InternalScanner.next(List), so this class
+ * also implements InternalScanner.  WARNING: As is, if you try to use this
+ * as an InternalScanner at the Store level, you will get runtime exceptions.
+ */
+public class KeyValueHeap extends NonLazyKeyValueScanner
+    implements KeyValueScanner, InternalScanner {
+  private PriorityQueue heap = null;
+
+  /**
+   * The current sub-scanner, i.e. the one that contains the next key/value
+   * to return to the client. This scanner is NOT included in {@link #heap}
+   * (but we frequently add it back to the heap and pull the new winner out).
+   * We maintain an invariant that the current sub-scanner has already done
+   * a real seek, and that current.peek() is always a real key/value (or null)
+   * except for the fake last-key-on-row-column supplied by the multi-column
+   * Bloom filter optimization, which is OK to propagate to StoreScanner. In
+   * order to ensure that, always use {@link #pollRealKV()} to update current.
+   */
+  private KeyValueScanner current = null;
+
+  private KVScannerComparator comparator;
+
+  /**
+   * Constructor.  This KeyValueHeap will handle closing of passed in
+   * KeyValueScanners.
+   * @param scanners
+   * @param comparator
+   */
+  public KeyValueHeap(List scanners,
+      KVComparator comparator) throws IOException {
+    this.comparator = new KVScannerComparator(comparator);
+    if (!scanners.isEmpty()) {
+      this.heap = new PriorityQueue(scanners.size(),
+          this.comparator);
+      for (KeyValueScanner scanner : scanners) {
+        if (scanner.peek() != null) {
+          this.heap.add(scanner);
+        } else {
+          scanner.close();
+        }
+      }
+      this.current = pollRealKV();
+    }
+  }
+
+  public KeyValue peek() {
+    if (this.current == null) {
+      return null;
+    }
+    return this.current.peek();
+  }
+
+  public KeyValue next()  throws IOException {
+    if(this.current == null) {
+      return null;
+    }
+    KeyValue kvReturn = this.current.next();
+    KeyValue kvNext = this.current.peek();
+    if (kvNext == null) {
+      this.current.close();
+      this.current = pollRealKV();
+    } else {
+      KeyValueScanner topScanner = this.heap.peek();
+      if (topScanner == null ||
+          this.comparator.compare(kvNext, topScanner.peek()) >= 0) {
+        this.heap.add(this.current);
+        this.current = pollRealKV();
+      }
+    }
+    return kvReturn;
+  }
+
+  /**
+   * Gets the next row of keys from the top-most scanner.
+   * 
                
+   * This method takes care of updating the heap.
+   * 
                
+   * This can ONLY be called when you are using Scanners that implement
+   * InternalScanner as well as KeyValueScanner (a {@link StoreScanner}).
+   * @param result
+   * @param limit
+   * @return true if there are more keys, false if all scanners are done
+   */
+  public boolean next(List result, int limit) throws IOException {
+    return next(result, limit, null);
+  }
+
+  /**
+   * Gets the next row of keys from the top-most scanner.
+   * 
                
+   * This method takes care of updating the heap.
+   * 
                
+   * This can ONLY be called when you are using Scanners that implement
+   * InternalScanner as well as KeyValueScanner (a {@link StoreScanner}).
+   * @param result output result list
+   * @param limit limit on row count to get
+   * @param metric the metric name
+   * @return true if there are more keys, false if all scanners are done
+   */
+  public boolean next(List result, int limit, String metric) throws IOException {
+    if (this.current == null) {
+      return false;
+    }
+    InternalScanner currentAsInternal = (InternalScanner)this.current;
+    boolean mayContainMoreRows = currentAsInternal.next(result, limit, metric);
+    KeyValue pee = this.current.peek();
+    /*
+     * By definition, any InternalScanner must return false only when it has no
+     * further rows to be fetched. So, we can close a scanner if it returns
+     * false. All existing implementations seem to be fine with this. It is much
+     * more efficient to close scanners which are not needed than keep them in
+     * the heap. This is also required for certain optimizations.
+     */
+    if (pee == null || !mayContainMoreRows) {
+      this.current.close();
+    } else {
+      this.heap.add(this.current);
+    }
+    this.current = pollRealKV();
+    return (this.current != null);
+  }
+
+  /**
+   * Gets the next row of keys from the top-most scanner.
+   * 
                
+   * This method takes care of updating the heap.
+   * 
                
+   * This can ONLY be called when you are using Scanners that implement
+   * InternalScanner as well as KeyValueScanner (a {@link StoreScanner}).
+   * @param result
+   * @return true if there are more keys, false if all scanners are done
+   */
+  public boolean next(List result) throws IOException {
+    return next(result, -1);
+  }
+
+  @Override
+  public boolean next(List result, String metric) throws IOException {
+    return next(result, -1, metric);
+  }
+
+  private static class KVScannerComparator implements Comparator {
+    private KVComparator kvComparator;
+    /**
+     * Constructor
+     * @param kvComparator
+     */
+    public KVScannerComparator(KVComparator kvComparator) {
+      this.kvComparator = kvComparator;
+    }
+    public int compare(KeyValueScanner left, KeyValueScanner right) {
+      int comparison = compare(left.peek(), right.peek());
+      if (comparison != 0) {
+        return comparison;
+      } else {
+        // Since both the keys are exactly the same, we break the tie in favor
+        // of the key which came latest.
+        long leftSequenceID = left.getSequenceID();
+        long rightSequenceID = right.getSequenceID();
+        if (leftSequenceID > rightSequenceID) {
+          return -1;
+        } else if (leftSequenceID < rightSequenceID) {
+          return 1;
+        } else {
+          return 0;
+        }
+      }
+    }
+    /**
+     * Compares two KeyValue
+     * @param left
+     * @param right
+     * @return less than 0 if left is smaller, 0 if equal etc..
+     */
+    public int compare(KeyValue left, KeyValue right) {
+      return this.kvComparator.compare(left, right);
+    }
+    /**
+     * @return KVComparator
+     */
+    public KVComparator getComparator() {
+      return this.kvComparator;
+    }
+  }
+
+  public void close() {
+    if (this.current != null) {
+      this.current.close();
+    }
+    if (this.heap != null) {
+      KeyValueScanner scanner;
+      while ((scanner = this.heap.poll()) != null) {
+        scanner.close();
+      }
+    }
+  }
+
+  /**
+   * Seeks all scanners at or below the specified seek key.  If we earlied-out
+   * of a row, we may end up skipping values that were never reached yet.
+   * Rather than iterating down, we want to give the opportunity to re-seek.
+   * 
                
+   * As individual scanners may run past their ends, those scanners are
+   * automatically closed and removed from the heap.
+   * 
                
+   * This function (and {@link #reseek(KeyValue)}) does not do multi-column
+   * Bloom filter and lazy-seek optimizations. To enable those, call
+   * {@link #requestSeek(KeyValue, boolean, boolean)}.
+   * @param seekKey KeyValue to seek at or after
+   * @return true if KeyValues exist at or after specified key, false if not
+   * @throws IOException
+   */
+  @Override
+  public boolean seek(KeyValue seekKey) throws IOException {
+    return generalizedSeek(false,    // This is not a lazy seek
+                           seekKey,
+                           false,    // forward (false: this is not a reseek)
+                           false);   // Not using Bloom filters
+  }
+
+  /**
+   * This function is identical to the {@link #seek(KeyValue)} function except
+   * that scanner.seek(seekKey) is changed to scanner.reseek(seekKey).
+   */
+  @Override
+  public boolean reseek(KeyValue seekKey) throws IOException {
+    return generalizedSeek(false,    // This is not a lazy seek
+                           seekKey,
+                           true,     // forward (true because this is reseek)
+                           false);   // Not using Bloom filters
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public boolean requestSeek(KeyValue key, boolean forward,
+      boolean useBloom) throws IOException {
+    return generalizedSeek(true, key, forward, useBloom);
+  }
+
+  /**
+   * @param isLazy whether we are trying to seek to exactly the given row/col.
+   *          Enables Bloom filter and most-recent-file-first optimizations for
+   *          multi-column get/scan queries.
+   * @param seekKey key to seek to
+   * @param forward whether to seek forward (also known as reseek)
+   * @param useBloom whether to optimize seeks using Bloom filters
+   */
+  private boolean generalizedSeek(boolean isLazy, KeyValue seekKey,
+      boolean forward, boolean useBloom) throws IOException {
+    if (!isLazy && useBloom) {
+      throw new IllegalArgumentException("Multi-column Bloom filter " +
+          "optimization requires a lazy seek");
+    }
+
+    if (current == null) {
+      return false;
+    }
+    heap.add(current);
+    current = null;
+
+    KeyValueScanner scanner;
+    while ((scanner = heap.poll()) != null) {
+      KeyValue topKey = scanner.peek();
+      if (comparator.getComparator().compare(seekKey, topKey) <= 0) {
+        // Top KeyValue is at-or-after Seek KeyValue. We only know that all
+        // scanners are at or after seekKey (because fake keys of
+        // scanners where a lazy-seek operation has been done are not greater
+        // than their real next keys) but we still need to enforce our
+        // invariant that the top scanner has done a real seek. This way
+        // StoreScanner and RegionScanner do not have to worry about fake keys.
+        heap.add(scanner);
+        current = pollRealKV();
+        return current != null;
+      }
+
+      boolean seekResult;
+      if (isLazy) {
+        seekResult = scanner.requestSeek(seekKey, forward, useBloom);
+      } else {
+        seekResult = NonLazyKeyValueScanner.doRealSeek(
+            scanner, seekKey, forward);
+      }
+
+      if (!seekResult) {
+        scanner.close();
+      } else {
+        heap.add(scanner);
+      }
+    }
+
+    // Heap is returning empty, scanner is done
+    return false;
+  }
+
+  /**
+   * Fetches the top sub-scanner from the priority queue, ensuring that a real
+   * seek has been done on it. Works by fetching the top sub-scanner, and if it
+   * has not done a real seek, making it do so (which will modify its top KV),
+   * putting it back, and repeating this until success. Relies on the fact that
+   * on a lazy seek we set the current key of a StoreFileScanner to a KV that
+   * is not greater than the real next KV to be read from that file, so the
+   * scanner that bubbles up to the top of the heap will have global next KV in
+   * this scanner heap if (1) it has done a real seek and (2) its KV is the top
+   * among all top KVs (some of which are fake) in the scanner heap.
+   */
+  private KeyValueScanner pollRealKV() throws IOException {
+    KeyValueScanner kvScanner = heap.poll();
+    if (kvScanner == null) {
+      return null;
+    }
+
+    while (kvScanner != null && !kvScanner.realSeekDone()) {
+      if (kvScanner.peek() != null) {
+        kvScanner.enforceSeek();
+        KeyValue curKV = kvScanner.peek();
+        if (curKV != null) {
+          KeyValueScanner nextEarliestScanner = heap.peek();
+          if (nextEarliestScanner == null) {
+            // The heap is empty. Return the only possible scanner.
+            return kvScanner;
+          }
+
+          // Compare the current scanner to the next scanner. We try to avoid
+          // putting the current one back into the heap if possible.
+          KeyValue nextKV = nextEarliestScanner.peek();
+          if (nextKV == null || comparator.compare(curKV, nextKV) < 0) {
+            // We already have the scanner with the earliest KV, so return it.
+            return kvScanner;
+          }
+
+          // Otherwise, put the scanner back into the heap and let it compete
+          // against all other scanners (both those that have done a "real
+          // seek" and a "lazy seek").
+          heap.add(kvScanner);
+        } else {
+          // Close the scanner because we did a real seek and found out there
+          // are no more KVs.
+          kvScanner.close();
+        }
+      } else {
+        // Close the scanner because it has already run out of KVs even before
+        // we had to do a real seek on it.
+        kvScanner.close();
+      }
+      kvScanner = heap.poll();
+    }
+
+    return kvScanner;
+  }
+
+  /**
+   * @return the current Heap
+   */
+  public PriorityQueue getHeap() {
+    return this.heap;
+  }
+
+  @Override
+  public long getSequenceID() {
+    return 0;
+  }
+
+  KeyValueScanner getCurrentForTesting() {
+    return current;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
new file mode 100644
index 0000000..6a7d5c6
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
@@ -0,0 +1,124 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.SortedSet;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Scan;
+
+/**
+ * Scanner that returns the next KeyValue.
+ */
+public interface KeyValueScanner {
+  /**
+   * Look at the next KeyValue in this scanner, but do not iterate scanner.
+   * @return the next KeyValue
+   */
+  public KeyValue peek();
+
+  /**
+   * Return the next KeyValue in this scanner, iterating the scanner
+   * @return the next KeyValue
+   */
+  public KeyValue next() throws IOException;
+
+  /**
+   * Seek the scanner at or after the specified KeyValue.
+   * @param key seek value
+   * @return true if scanner has values left, false if end of scanner
+   */
+  public boolean seek(KeyValue key) throws IOException;
+
+  /**
+   * Reseek the scanner at or after the specified KeyValue.
+   * This method is guaranteed to seek at or after the required key only if the
+   * key comes after the current position of the scanner. Should not be used
+   * to seek to a key which may come before the current position.
+   * @param key seek value (should be non-null)
+   * @return true if scanner has values left, false if end of scanner
+   */
+  public boolean reseek(KeyValue key) throws IOException;
+
+  /**
+   * Get the sequence id associated with this KeyValueScanner. This is required
+   * for comparing multiple files to find out which one has the latest data.
+   * The default implementation for this would be to return 0. A file having
+   * lower sequence id will be considered to be the older one.
+   */
+  public long getSequenceID();
+
+  /**
+   * Close the KeyValue scanner.
+   */
+  public void close();
+
+  /**
+   * Allows to filter out scanners (both StoreFile and memstore) that we don't
+   * want to use based on criteria such as Bloom filters and timestamp ranges.
+   * @param scan the scan that we are selecting scanners for
+   * @param columns the set of columns in the current column family, or null if
+   *          not specified by the scan
+   * @param oldestUnexpiredTS the oldest timestamp we are interested in for
+   *          this query, based on TTL
+   * @return true if the scanner should be included in the query
+   */
+  public boolean shouldUseScanner(Scan scan, SortedSet columns,
+      long oldestUnexpiredTS);
+
+  // "Lazy scanner" optimizations
+
+  /**
+   * Similar to {@link #seek} (or {@link #reseek} if forward is true) but only
+   * does a seek operation after checking that it is really necessary for the
+   * row/column combination specified by the kv parameter. This function was
+   * added to avoid unnecessary disk seeks by checking row-column Bloom filters
+   * before a seek on multi-column get/scan queries, and to optimize by looking
+   * up more recent files first.
+   * @param forward do a forward-only "reseek" instead of a random-access seek
+   * @param useBloom whether to enable multi-column Bloom filter optimization
+   */
+  public boolean requestSeek(KeyValue kv, boolean forward, boolean useBloom)
+      throws IOException;
+
+  /**
+   * We optimize our store scanners by checking the most recent store file
+   * first, so we sometimes pretend we have done a seek but delay it until the
+   * store scanner bubbles up to the top of the key-value heap. This method is
+   * then used to ensure the top store file scanner has done a seek operation.
+   */
+  public boolean realSeekDone();
+
+  /**
+   * Does the real seek operation in case it was skipped by
+   * seekToRowCol(KeyValue, boolean) (TODO: Whats this?). Note that this function should
+   * be never called on scanners that always do real seek operations (i.e. most
+   * of the scanners). The easiest way to achieve this is to call
+   * {@link #realSeekDone()} first.
+   */
+  public void enforceSeek() throws IOException;
+
+  /**
+   * @return true if this is a file scanner. Otherwise a memory scanner is
+   *         assumed.
+   */
+  public boolean isFileScanner();  
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
new file mode 100644
index 0000000..51df1ee
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java
@@ -0,0 +1,183 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.NavigableSet;
+import java.util.SortedSet;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+/**
+ * A {@link java.util.Set} of {@link KeyValue}s implemented on top of a
+ * {@link java.util.concurrent.ConcurrentSkipListMap}.  Works like a
+ * {@link java.util.concurrent.ConcurrentSkipListSet} in all but one regard:
+ * An add will overwrite if already an entry for the added key.  In other words,
+ * where CSLS does "Adds the specified element to this set if it is not already
+ * present.", this implementation "Adds the specified element to this set EVEN
+ * if it is already present overwriting what was there previous".  The call to
+ * add returns true if no value in the backing map or false if there was an
+ * entry with same key (though value may be different).
+ * 
                Otherwise,
+ * has same attributes as ConcurrentSkipListSet: e.g. tolerant of concurrent
+ * get and set and won't throw ConcurrentModificationException when iterating.
+ */
+class KeyValueSkipListSet implements NavigableSet {
+  private final ConcurrentNavigableMap delegatee;
+
+  KeyValueSkipListSet(final KeyValue.KVComparator c) {
+    this.delegatee = new ConcurrentSkipListMap(c);
+  }
+
+  KeyValueSkipListSet(final ConcurrentNavigableMap m) {
+    this.delegatee = m;
+  }
+
+  public KeyValue ceiling(KeyValue e) {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public Iterator descendingIterator() {
+    return this.delegatee.descendingMap().values().iterator();
+  }
+
+  public NavigableSet descendingSet() {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public KeyValue floor(KeyValue e) {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public SortedSet headSet(final KeyValue toElement) {
+    return headSet(toElement, false);
+  }
+
+  public NavigableSet headSet(final KeyValue toElement,
+      boolean inclusive) {
+    return new KeyValueSkipListSet(this.delegatee.headMap(toElement, inclusive));
+  }
+
+  public KeyValue higher(KeyValue e) {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public Iterator iterator() {
+    return this.delegatee.values().iterator();
+  }
+
+  public KeyValue lower(KeyValue e) {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public KeyValue pollFirst() {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public KeyValue pollLast() {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public SortedSet subSet(KeyValue fromElement, KeyValue toElement) {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public NavigableSet subSet(KeyValue fromElement,
+      boolean fromInclusive, KeyValue toElement, boolean toInclusive) {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public SortedSet tailSet(KeyValue fromElement) {
+    return tailSet(fromElement, true);
+  }
+
+  public NavigableSet tailSet(KeyValue fromElement, boolean inclusive) {
+    return new KeyValueSkipListSet(this.delegatee.tailMap(fromElement, inclusive));
+  }
+
+  public Comparator comparator() {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public KeyValue first() {
+    return this.delegatee.get(this.delegatee.firstKey());
+  }
+
+  public KeyValue last() {
+    return this.delegatee.get(this.delegatee.lastKey());
+  }
+
+  public boolean add(KeyValue e) {
+    return this.delegatee.put(e, e) == null;
+  }
+
+  public boolean addAll(Collection c) {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public void clear() {
+    this.delegatee.clear();
+  }
+
+  public boolean contains(Object o) {
+    //noinspection SuspiciousMethodCalls
+    return this.delegatee.containsKey(o);
+  }
+
+  public boolean containsAll(Collection c) {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public boolean isEmpty() {
+    return this.delegatee.isEmpty();
+  }
+
+  public boolean remove(Object o) {
+    return this.delegatee.remove(o) != null;
+  }
+
+  public boolean removeAll(Collection c) {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public boolean retainAll(Collection c) {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public KeyValue get(KeyValue kv) {
+    return this.delegatee.get(kv);
+  }
+
+  public int size() {
+    return this.delegatee.size();
+  }
+
+  public Object[] toArray() {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  public  T[] toArray(T[] a) {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java
new file mode 100644
index 0000000..cafbb28
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java
@@ -0,0 +1,42 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+
+/**
+ * Reports a problem with a lease
+ */
+public class LeaseException extends DoNotRetryIOException {
+
+  private static final long serialVersionUID = 8179703995292418650L;
+
+  /** default constructor */
+  public LeaseException() {
+    super();
+  }
+
+  /**
+   * @param message
+   */
+  public LeaseException(String message) {
+    super(message);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseListener.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseListener.java
new file mode 100644
index 0000000..a843736
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseListener.java
@@ -0,0 +1,34 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+
+/**
+ * LeaseListener is an interface meant to be implemented by users of the Leases
+ * class.
+ *
+ * It receives events from the Leases class about the status of its accompanying
+ * lease.  Users of the Leases class can use a LeaseListener subclass to, for
+ * example, clean up resources after a lease has expired.
+ */
+public interface LeaseListener {
+  /** When a lease expires, this method is called. */
+  public void leaseExpired();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Leases.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Leases.java
new file mode 100644
index 0000000..c518521
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Leases.java
@@ -0,0 +1,300 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.util.HasThread;
+
+import java.util.ConcurrentModificationException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.DelayQueue;
+import java.util.concurrent.TimeUnit;
+
+import java.io.IOException;
+
+/**
+ * Leases
+ *
+ * There are several server classes in HBase that need to track external
+ * clients that occasionally send heartbeats.
+ *
+ * 
                These external clients hold resources in the server class.
+ * Those resources need to be released if the external client fails to send a
+ * heartbeat after some interval of time passes.
+ *
+ * 
                The Leases class is a general reusable class for this kind of pattern.
+ * An instance of the Leases class will create a thread to do its dirty work.
+ * You should close() the instance if you want to clean up the thread properly.
+ *
+ * 
                
+ * NOTE: This class extends Thread rather than Chore because the sleep time
+ * can be interrupted when there is something to do, rather than the Chore
+ * sleep time which is invariant.
+ */
+public class Leases extends HasThread {
+  private static final Log LOG = LogFactory.getLog(Leases.class.getName());
+  private final int leasePeriod;
+  private final int leaseCheckFrequency;
+  private volatile DelayQueue leaseQueue = new DelayQueue();
+  protected final Map leases = new HashMap();
+  private volatile boolean stopRequested = false;
+
+  /**
+   * Creates a lease monitor
+   *
+   * @param leasePeriod - length of time (milliseconds) that the lease is valid
+   * @param leaseCheckFrequency - how often the lease should be checked
+   * (milliseconds)
+   */
+  public Leases(final int leasePeriod, final int leaseCheckFrequency) {
+    this.leasePeriod = leasePeriod;
+    this.leaseCheckFrequency = leaseCheckFrequency;
+    setDaemon(true);
+  }
+
+  /**
+   * @see java.lang.Thread#run()
+   */
+  @Override
+  public void run() {
+    while (!stopRequested || (stopRequested && leaseQueue.size() > 0) ) {
+      Lease lease = null;
+      try {
+        lease = leaseQueue.poll(leaseCheckFrequency, TimeUnit.MILLISECONDS);
+      } catch (InterruptedException e) {
+        continue;
+      } catch (ConcurrentModificationException e) {
+        continue;
+      } catch (Throwable e) {
+        LOG.fatal("Unexpected exception killed leases thread", e);
+        break;
+      }
+      if (lease == null) {
+        continue;
+      }
+      // A lease expired.  Run the expired code before removing from queue
+      // since its presence in queue is used to see if lease exists still.
+      if (lease.getListener() == null) {
+        LOG.error("lease listener is null for lease " + lease.getLeaseName());
+      } else {
+        lease.getListener().leaseExpired();
+      }
+      synchronized (leaseQueue) {
+        leases.remove(lease.getLeaseName());
+      }
+    }
+    close();
+  }
+
+  /**
+   * Shuts down this lease instance when all outstanding leases expire.
+   * Like {@link #close()} but rather than violently end all leases, waits
+   * first on extant leases to finish.  Use this method if the lease holders
+   * could loose data, leak locks, etc.  Presumes client has shutdown
+   * allocation of new leases.
+   */
+  public void closeAfterLeasesExpire() {
+    this.stopRequested = true;
+  }
+
+  /**
+   * Shut down this Leases instance.  All pending leases will be destroyed,
+   * without any cancellation calls.
+   */
+  public void close() {
+    LOG.info(Thread.currentThread().getName() + " closing leases");
+    this.stopRequested = true;
+    synchronized (leaseQueue) {
+      leaseQueue.clear();
+      leases.clear();
+      leaseQueue.notifyAll();
+    }
+    LOG.info(Thread.currentThread().getName() + " closed leases");
+  }
+
+  /**
+   * Obtain a lease
+   *
+   * @param leaseName name of the lease
+   * @param listener listener that will process lease expirations
+   * @throws LeaseStillHeldException
+   */
+  public void createLease(String leaseName, final LeaseListener listener)
+  throws LeaseStillHeldException {
+    addLease(new Lease(leaseName, listener));
+  }
+
+  /**
+   * Inserts lease.  Resets expiration before insertion.
+   * @param lease
+   * @throws LeaseStillHeldException
+   */
+  public void addLease(final Lease lease) throws LeaseStillHeldException {
+    if (this.stopRequested) {
+      return;
+    }
+    lease.setExpirationTime(System.currentTimeMillis() + this.leasePeriod);
+    synchronized (leaseQueue) {
+      if (leases.containsKey(lease.getLeaseName())) {
+        throw new LeaseStillHeldException(lease.getLeaseName());
+      }
+      leases.put(lease.getLeaseName(), lease);
+      leaseQueue.add(lease);
+    }
+  }
+
+  /**
+   * Thrown if we are asked create a lease but lease on passed name already
+   * exists.
+   */
+  @SuppressWarnings("serial")
+  public static class LeaseStillHeldException extends IOException {
+    private final String leaseName;
+
+    /**
+     * @param name
+     */
+    public LeaseStillHeldException(final String name) {
+      this.leaseName = name;
+    }
+
+    /** @return name of lease */
+    public String getName() {
+      return this.leaseName;
+    }
+  }
+
+  /**
+   * Renew a lease
+   *
+   * @param leaseName name of lease
+   * @throws LeaseException
+   */
+  public void renewLease(final String leaseName) throws LeaseException {
+    synchronized (leaseQueue) {
+      Lease lease = leases.get(leaseName);
+      // We need to check to see if the remove is successful as the poll in the run()
+      // method could have completed between the get and the remove which will result
+      // in a corrupt leaseQueue.
+      if (lease == null || !leaseQueue.remove(lease)) {
+        throw new LeaseException("lease '" + leaseName +
+        "' does not exist or has already expired");
+      }
+      lease.setExpirationTime(System.currentTimeMillis() + leasePeriod);
+      leaseQueue.add(lease);
+    }
+  }
+
+  /**
+   * Client explicitly cancels a lease.
+   * @param leaseName name of lease
+   * @throws LeaseException
+   */
+  public void cancelLease(final String leaseName) throws LeaseException {
+    removeLease(leaseName);
+  }
+
+  /**
+   * Remove named lease.
+   * Lease is removed from the list of leases and removed from the delay queue.
+   * Lease can be resinserted using {@link #addLease(Lease)}
+   *
+   * @param leaseName name of lease
+   * @throws LeaseException
+   * @return Removed lease
+   */
+  Lease removeLease(final String leaseName) throws LeaseException {
+    Lease lease =  null;
+    synchronized (leaseQueue) {
+      lease = leases.remove(leaseName);
+      if (lease == null) {
+        throw new LeaseException("lease '" + leaseName + "' does not exist");
+      }
+      leaseQueue.remove(lease);
+    }
+    return lease;
+  }
+
+  /** This class tracks a single Lease. */
+  static class Lease implements Delayed {
+    private final String leaseName;
+    private final LeaseListener listener;
+    private long expirationTime;
+
+    Lease(final String leaseName, LeaseListener listener) {
+      this(leaseName, listener, 0);
+    }
+
+    Lease(final String leaseName, LeaseListener listener, long expirationTime) {
+      this.leaseName = leaseName;
+      this.listener = listener;
+      this.expirationTime = expirationTime;
+    }
+
+    /** @return the lease name */
+    public String getLeaseName() {
+      return leaseName;
+    }
+
+    /** @return listener */
+    public LeaseListener getListener() {
+      return this.listener;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) {
+        return true;
+      }
+      if (obj == null) {
+        return false;
+      }
+      if (getClass() != obj.getClass()) {
+        return false;
+      }
+      return this.hashCode() == ((Lease) obj).hashCode();
+    }
+
+    @Override
+    public int hashCode() {
+      return this.leaseName.hashCode();
+    }
+
+    public long getDelay(TimeUnit unit) {
+      return unit.convert(this.expirationTime - System.currentTimeMillis(),
+          TimeUnit.MILLISECONDS);
+    }
+
+    public int compareTo(Delayed o) {
+      long delta = this.getDelay(TimeUnit.MILLISECONDS) -
+        o.getDelay(TimeUnit.MILLISECONDS);
+
+      return this.equals(o) ? 0 : (delta > 0 ? 1 : -1);
+    }
+
+    /** @param expirationTime the expirationTime to set */
+    public void setExpirationTime(long expirationTime) {
+      this.expirationTime = expirationTime;
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
new file mode 100644
index 0000000..01ae295
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -0,0 +1,199 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.HasThread;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Runs periodically to determine if the HLog should be rolled.
+ *
+ * NOTE: This class extends Thread rather than Chore because the sleep time
+ * can be interrupted when there is something to do, rather than the Chore
+ * sleep time which is invariant.
+ */
+class LogRoller extends HasThread implements WALActionsListener {
+  static final Log LOG = LogFactory.getLog(LogRoller.class);
+  private final ReentrantLock rollLock = new ReentrantLock();
+  private final AtomicBoolean rollLog = new AtomicBoolean(false);
+  private final Server server;
+  protected final RegionServerServices services;
+  private volatile long lastrolltime = System.currentTimeMillis();
+  // Period to roll log.
+  private final long rollperiod;
+  private final int threadWakeFrequency;
+
+  /** @param server */
+  public LogRoller(final Server server, final RegionServerServices services) {
+    super();
+    this.server = server;
+    this.services = services;
+    this.rollperiod = this.server.getConfiguration().
+      getLong("hbase.regionserver.logroll.period", 3600000);
+    this.threadWakeFrequency = this.server.getConfiguration().
+      getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
+  }
+
+  @Override
+  public void run() {
+    while (!server.isStopped()) {
+      long now = System.currentTimeMillis();
+      boolean periodic = false;
+      if (!rollLog.get()) {
+        periodic = (now - this.lastrolltime) > this.rollperiod;
+        if (!periodic) {
+          synchronized (rollLog) {
+            try {
+              rollLog.wait(this.threadWakeFrequency);
+            } catch (InterruptedException e) {
+              // Fall through
+            }
+          }
+          continue;
+        }
+        // Time for periodic roll
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Hlog roll period " + this.rollperiod + "ms elapsed");
+        }
+      } else if (LOG.isDebugEnabled()) {
+        LOG.debug("HLog roll requested");
+      }
+      rollLock.lock(); // FindBugs UL_UNRELEASED_LOCK_EXCEPTION_PATH
+      try {
+        this.lastrolltime = now;
+        // This is array of actual region names.
+        byte [][] regionsToFlush = getWAL().rollWriter(rollLog.get());
+        if (regionsToFlush != null) {
+          for (byte [] r: regionsToFlush) scheduleFlush(r);
+        }
+      } catch (FailedLogCloseException e) {
+        server.abort("Failed log close in log roller", e);
+      } catch (java.net.ConnectException e) {
+        server.abort("Failed log close in log roller", e);
+      } catch (IOException ex) {
+        // Abort if we get here.  We probably won't recover an IOE. HBASE-1132
+        server.abort("IOE in log roller",
+          RemoteExceptionHandler.checkIOException(ex));
+      } catch (Exception ex) {
+        LOG.error("Log rolling failed", ex);
+        server.abort("Log rolling failed", ex);
+      } finally {
+        rollLog.set(false);
+        rollLock.unlock();
+      }
+    }
+    LOG.info("LogRoller exiting.");
+  }
+
+  /**
+   * @param encodedRegionName Encoded name of region to flush.
+   */
+  private void scheduleFlush(final byte [] encodedRegionName) {
+    boolean scheduled = false;
+    HRegion r = this.services.getFromOnlineRegions(Bytes.toString(encodedRegionName));
+    FlushRequester requester = null;
+    if (r != null) {
+      requester = this.services.getFlushRequester();
+      if (requester != null) {
+        requester.requestFlush(r);
+        scheduled = true;
+      }
+    }
+    if (!scheduled) {
+      LOG.warn("Failed to schedule flush of " +
+        Bytes.toString(encodedRegionName) + ", region=" + r + ", requester=" +
+        requester);
+    }
+  }
+
+  public void logRollRequested() {
+    synchronized (rollLog) {
+      rollLog.set(true);
+      rollLog.notifyAll();
+    }
+  }
+
+  /**
+   * Called by region server to wake up this thread if it sleeping.
+   * It is sleeping if rollLock is not held.
+   */
+  public void interruptIfNecessary() {
+    try {
+      rollLock.lock();
+      this.interrupt();
+    } finally {
+      rollLock.unlock();
+    }
+  }
+
+  protected HLog getWAL() throws IOException {
+    return this.services.getWAL(null);
+  }
+
+  @Override
+  public void preLogRoll(Path oldPath, Path newPath) throws IOException {
+    // Not interested
+  }
+
+  @Override
+  public void postLogRoll(Path oldPath, Path newPath) throws IOException {
+    // Not interested
+  }
+
+  @Override
+  public void preLogArchive(Path oldPath, Path newPath) throws IOException {
+    // Not interested
+  }
+
+  @Override
+  public void postLogArchive(Path oldPath, Path newPath) throws IOException {
+    // Not interested
+  }
+
+  @Override
+  public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey,
+      WALEdit logEdit) {
+    // Not interested.
+  }
+
+  @Override
+  public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey,
+                                       WALEdit logEdit) {
+    //Not interested
+  }
+
+  @Override
+  public void logCloseRequested() {
+    // not interested
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/LruHashMap.java b/src/main/java/org/apache/hadoop/hbase/regionserver/LruHashMap.java
new file mode 100644
index 0000000..161ae18
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/LruHashMap.java
@@ -0,0 +1,1099 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * The LruHashMap is a memory-aware HashMap with a configurable maximum
+ * memory footprint.
+ * 
                
+ * It maintains an ordered list of all entries in the map ordered by
+ * access time.  When space needs to be freed becase the maximum has been
+ * reached, or the application has asked to free memory, entries will be
+ * evicted according to an LRU (least-recently-used) algorithm.  That is,
+ * those entries which have not been accessed the longest will be evicted
+ * first.
+ * 
                
+ * Both the Key and Value Objects used for this class must extend
+ * HeapSize in order to track heap usage.
+ * 
                
+ * This class contains internal synchronization and is thread-safe.
+ */
+public class LruHashMap
+implements HeapSize, Map {
+
+  static final Log LOG = LogFactory.getLog(LruHashMap.class);
+
+  /** The default size (in bytes) of the LRU */
+  private static final long DEFAULT_MAX_MEM_USAGE = 50000;
+  /** The default capacity of the hash table */
+  private static final int DEFAULT_INITIAL_CAPACITY = 16;
+  /** The maxmum capacity of the hash table */
+  private static final int MAXIMUM_CAPACITY = 1 << 30;
+  /** The default load factor to use */
+  private static final float DEFAULT_LOAD_FACTOR = 0.75f;
+
+  /** Memory overhead of this Object (for HeapSize) */
+  private static final int OVERHEAD = 5 * Bytes.SIZEOF_LONG +
+    2 * Bytes.SIZEOF_INT + 2 * Bytes.SIZEOF_FLOAT + 3 * ClassSize.REFERENCE +
+    1 * ClassSize.ARRAY;
+
+  /** Load factor allowed (usually 75%) */
+  private final float loadFactor;
+  /** Number of key/vals in the map */
+  private int size;
+  /** Size at which we grow hash */
+  private int threshold;
+  /** Entries in the map */
+  private Entry [] entries;
+
+  /** Pointer to least recently used entry */
+  private Entry headPtr;
+  /** Pointer to most recently used entry */
+  private Entry tailPtr;
+
+  /** Maximum memory usage of this map */
+  private long memTotal = 0;
+  /** Amount of available memory */
+  private long memFree = 0;
+
+  /** Number of successful (found) get() calls */
+  private long hitCount = 0;
+  /** Number of unsuccessful (not found) get() calls */
+  private long missCount = 0;
+
+  /**
+   * Constructs a new, empty map with the specified initial capacity,
+   * load factor, and maximum memory usage.
+   *
+   * @param initialCapacity the initial capacity
+   * @param loadFactor the load factor
+   * @param maxMemUsage the maximum total memory usage
+   * @throws IllegalArgumentException if the initial capacity is less than one
+   * @throws IllegalArgumentException if the initial capacity is greater than
+   * the maximum capacity
+   * @throws IllegalArgumentException if the load factor is <= 0
+   * @throws IllegalArgumentException if the max memory usage is too small
+   * to support the base overhead
+   */
+  public LruHashMap(int initialCapacity, float loadFactor,
+  long maxMemUsage) {
+    if (initialCapacity < 1) {
+      throw new IllegalArgumentException("Initial capacity must be > 0");
+    }
+    if (initialCapacity > MAXIMUM_CAPACITY) {
+      throw new IllegalArgumentException("Initial capacity is too large");
+    }
+    if (loadFactor <= 0 || Float.isNaN(loadFactor)) {
+      throw new IllegalArgumentException("Load factor must be > 0");
+    }
+    if (maxMemUsage <= (OVERHEAD + initialCapacity * ClassSize.REFERENCE)) {
+      throw new IllegalArgumentException("Max memory usage too small to " +
+      "support base overhead");
+    }
+
+    /** Find a power of 2 >= initialCapacity */
+    int capacity = calculateCapacity(initialCapacity);
+    this.loadFactor = loadFactor;
+    this.threshold = calculateThreshold(capacity,loadFactor);
+    this.entries = new Entry[capacity];
+    this.memFree = maxMemUsage;
+    this.memTotal = maxMemUsage;
+    init();
+  }
+
+  /**
+   * Constructs a new, empty map with the specified initial capacity and
+   * load factor, and default maximum memory usage.
+   *
+   * @param initialCapacity the initial capacity
+   * @param loadFactor the load factor
+   * @throws IllegalArgumentException if the initial capacity is less than one
+   * @throws IllegalArgumentException if the initial capacity is greater than
+   * the maximum capacity
+   * @throws IllegalArgumentException if the load factor is <= 0
+   */
+  public LruHashMap(int initialCapacity, float loadFactor) {
+    this(initialCapacity, loadFactor, DEFAULT_MAX_MEM_USAGE);
+  }
+
+  /**
+   * Constructs a new, empty map with the specified initial capacity and
+   * with the default load factor and maximum memory usage.
+   *
+   * @param initialCapacity the initial capacity
+   * @throws IllegalArgumentException if the initial capacity is less than one
+   * @throws IllegalArgumentException if the initial capacity is greater than
+   * the maximum capacity
+   */
+  public LruHashMap(int initialCapacity) {
+    this(initialCapacity, DEFAULT_LOAD_FACTOR, DEFAULT_MAX_MEM_USAGE);
+  }
+
+  /**
+   * Constructs a new, empty map with the specified maximum memory usage
+   * and with default initial capacity and load factor.
+   *
+   * @param maxMemUsage the maximum total memory usage
+   * @throws IllegalArgumentException if the max memory usage is too small
+   * to support the base overhead
+   */
+  public LruHashMap(long maxMemUsage) {
+    this(DEFAULT_INITIAL_CAPACITY, DEFAULT_LOAD_FACTOR,
+    maxMemUsage);
+  }
+
+  /**
+   * Constructs a new, empty map with the default initial capacity,
+   * load factor and maximum memory usage.
+   */
+  public LruHashMap() {
+    this(DEFAULT_INITIAL_CAPACITY, DEFAULT_LOAD_FACTOR,
+    DEFAULT_MAX_MEM_USAGE);
+  }
+
+  //--------------------------------------------------------------------------
+  /**
+   * Get the currently available memory for this LRU in bytes.
+   * This is (maxAllowed - currentlyUsed).
+   *
+   * @return currently available bytes
+   */
+  public long getMemFree() {
+    return memFree;
+  }
+
+  /**
+   * Get the maximum memory allowed for this LRU in bytes.
+   *
+   * @return maximum allowed bytes
+   */
+  public long getMemMax() {
+    return memTotal;
+  }
+
+  /**
+   * Get the currently used memory for this LRU in bytes.
+   *
+   * @return currently used memory in bytes
+   */
+  public long getMemUsed() {
+    return (memTotal - memFree); // FindBugs IS2_INCONSISTENT_SYNC
+  }
+
+  /**
+   * Get the number of hits to the map.  This is the number of times
+   * a call to get() returns a matched key.
+   *
+   * @return number of hits
+   */
+  public long getHitCount() {
+    return hitCount;
+  }
+
+  /**
+   * Get the number of misses to the map.  This is the number of times
+   * a call to get() returns null.
+   *
+   * @return number of misses
+   */
+  public long getMissCount() {
+    return missCount; // FindBugs IS2_INCONSISTENT_SYNC
+  }
+
+  /**
+   * Get the hit ratio.  This is the number of hits divided by the
+   * total number of requests.
+   *
+   * @return hit ratio (double between 0 and 1)
+   */
+  public double getHitRatio() {
+    return (double)((double)hitCount/
+      ((double)(hitCount+missCount)));
+  }
+
+  /**
+   * Free the requested amount of memory from the LRU map.
+   *
+   * This will do LRU eviction from the map until at least as much
+   * memory as requested is freed.  This does not affect the maximum
+   * memory usage parameter.
+   *
+   * @param requestedAmount memory to free from LRU in bytes
+   * @return actual amount of memory freed in bytes
+   */
+  public synchronized long freeMemory(long requestedAmount) throws Exception {
+    if(requestedAmount > (getMemUsed() - getMinimumUsage())) {
+      return clearAll();
+    }
+    long freedMemory = 0;
+    while(freedMemory < requestedAmount) {
+      freedMemory += evictFromLru();
+    }
+    return freedMemory;
+  }
+
+  /**
+   * The total memory usage of this map
+   *
+   * @return memory usage of map in bytes
+   */
+  public long heapSize() {
+    return (memTotal - memFree);
+  }
+
+  //--------------------------------------------------------------------------
+  /**
+   * Retrieves the value associated with the specified key.
+   *
+   * If an entry is found, it is updated in the LRU as the most recently
+   * used (last to be evicted) entry in the map.
+   *
+   * @param key the key
+   * @return the associated value, or null if none found
+   * @throws NullPointerException if key is null
+   */
+  public synchronized V get(Object key) {
+    checkKey((K)key);
+    int hash = hash(key);
+    int i = hashIndex(hash, entries.length);
+    Entry e = entries[i];
+    while (true) {
+      if (e == null) {
+        missCount++;
+        return null;
+      }
+      if (e.hash == hash && isEqual(key, e.key))  {
+        // Hit!  Update position in LRU
+        hitCount++;
+        updateLru(e);
+        return e.value;
+      }
+      e = e.next;
+    }
+  }
+
+  /**
+   * Insert a key-value mapping into the map.
+   *
+   * Entry will be inserted as the most recently used.
+   *
+   * Both the key and value are required to be Objects and must
+   * implement the HeapSize interface.
+   *
+   * @param key the key
+   * @param value the value
+   * @return the value that was previously mapped to this key, null if none
+   * @throws UnsupportedOperationException if either objects do not
+   * implement HeapSize
+   * @throws NullPointerException if the key or value is null
+   */
+  public synchronized V put(K key, V value) {
+    checkKey(key);
+    checkValue(value);
+    int hash = hash(key);
+    int i = hashIndex(hash, entries.length);
+
+    // For old values
+    for (Entry e = entries[i]; e != null; e = e.next) {
+      if (e.hash == hash && isEqual(key, e.key)) {
+        V oldValue = e.value;
+        long memChange = e.replaceValue(value);
+        checkAndFreeMemory(memChange);
+        // If replacing an old value for this key, update in LRU
+        updateLru(e);
+        return oldValue;
+      }
+    }
+    long memChange = addEntry(hash, key, value, i);
+    checkAndFreeMemory(memChange);
+    return null;
+  }
+
+  /**
+   * Deletes the mapping for the specified key if it exists.
+   *
+   * @param key the key of the entry to be removed from the map
+   * @return the value associated with the specified key, or null
+   * if no mapping exists.
+   */
+  public synchronized V remove(Object key) {
+    Entry e = removeEntryForKey((K)key);
+    if(e == null) return null;
+    // Add freed memory back to available
+    memFree += e.heapSize();
+    return e.value;
+  }
+
+  /**
+   * Gets the size (number of entries) of the map.
+   *
+   * @return size of the map
+   */
+  public int size() {
+    return size;
+  }
+
+  /**
+   * Checks whether the map is currently empty.
+   *
+   * @return true if size of map is zero
+   */
+  public boolean isEmpty() {
+    return size == 0;
+  }
+
+  /**
+   * Clears all entries from the map.
+   *
+   * This frees all entries, tracking memory usage along the way.
+   * All references to entries are removed so they can be GC'd.
+   */
+  public synchronized void clear() {
+    memFree += clearAll();
+  }
+
+  //--------------------------------------------------------------------------
+  /**
+   * Checks whether there is a value in the map for the specified key.
+   *
+   * Does not affect the LRU.
+   *
+   * @param key the key to check
+   * @return true if the map contains a value for this key, false if not
+   * @throws NullPointerException if the key is null
+   */
+  public synchronized boolean containsKey(Object key) {
+    checkKey((K)key);
+    int hash = hash(key);
+    int i = hashIndex(hash, entries.length);
+    Entry e = entries[i];
+    while (e != null) {
+      if (e.hash == hash && isEqual(key, e.key))
+          return true;
+      e = e.next;
+    }
+    return false;
+  }
+
+  /**
+   * Checks whether this is a mapping which contains the specified value.
+   *
+   * Does not affect the LRU.  This is an inefficient operation.
+   *
+   * @param value the value to check
+   * @return true if the map contains an entry for this value, false
+   * if not
+   * @throws NullPointerException if the value is null
+   */
+  public synchronized boolean containsValue(Object value) {
+    checkValue((V)value);
+    Entry[] tab = entries;
+    for (int i = 0; i < tab.length ; i++)
+      for (Entry e = tab[i] ; e != null ; e = e.next)
+          if (value.equals(e.value))
+            return true;
+    return false;
+  }
+
+  //--------------------------------------------------------------------------
+  /**
+   * Enforces key constraints.  Null keys are not permitted and key must
+   * implement HeapSize.  It should not be necessary to verify the second
+   * constraint because that's enforced on instantiation?
+   *
+   * Can add other constraints in the future.
+   *
+   * @param key the key
+   * @throws NullPointerException if the key is null
+   * @throws UnsupportedOperationException if the key class does not
+   * implement the HeapSize interface
+   */
+  private void checkKey(K key) {
+    if(key == null) {
+      throw new NullPointerException("null keys are not allowed");
+    }
+  }
+
+  /**
+   * Enforces value constraints.  Null values are not permitted and value must
+   * implement HeapSize.  It should not be necessary to verify the second
+   * constraint because that's enforced on instantiation?
+   *
+   * Can add other contraints in the future.
+   *
+   * @param value the value
+   * @throws NullPointerException if the value is null
+   * @throws UnsupportedOperationException if the value class does not
+   * implement the HeapSize interface
+   */
+  private void checkValue(V value) {
+    if(value == null) {
+      throw new NullPointerException("null values are not allowed");
+    }
+  }
+
+  /**
+   * Returns the minimum memory usage of the base map structure.
+   *
+   * @return baseline memory overhead of object in bytes
+   */
+  private long getMinimumUsage() {
+    return OVERHEAD + (entries.length * ClassSize.REFERENCE);
+  }
+
+  //--------------------------------------------------------------------------
+  /**
+   * Evicts and frees based on LRU until at least as much memory as requested
+   * is available.
+   *
+   * @param memNeeded the amount of memory needed in bytes
+   */
+  private void checkAndFreeMemory(long memNeeded) {
+    while(memFree < memNeeded) {
+      evictFromLru();
+    }
+    memFree -= memNeeded;
+  }
+
+  /**
+   * Evicts based on LRU.  This removes all references and updates available
+   * memory.
+   *
+   * @return amount of memory freed in bytes
+   */
+  private long evictFromLru() {
+    long freed = headPtr.heapSize();
+    memFree += freed;
+    removeEntry(headPtr);
+    return freed;
+  }
+
+  /**
+   * Moves the specified entry to the most recently used slot of the
+   * LRU.  This is called whenever an entry is fetched.
+   *
+   * @param e entry that was accessed
+   */
+  private void updateLru(Entry e) {
+    Entry prev = e.getPrevPtr();
+    Entry next = e.getNextPtr();
+    if(next != null) {
+      if(prev != null) {
+        prev.setNextPtr(next);
+        next.setPrevPtr(prev);
+      } else {
+        headPtr = next;
+        headPtr.setPrevPtr(null);
+      }
+      e.setNextPtr(null);
+      e.setPrevPtr(tailPtr);
+      tailPtr.setNextPtr(e);
+      tailPtr = e;
+    }
+  }
+
+  /**
+   * Removes the specified entry from the map and LRU structure.
+   *
+   * @param entry entry to be removed
+   */
+  private void removeEntry(Entry entry) {
+    K k = entry.key;
+    int hash = entry.hash;
+    int i = hashIndex(hash, entries.length);
+    Entry prev = entries[i];
+    Entry e = prev;
+
+    while (e != null) {
+      Entry next = e.next;
+      if (e.hash == hash && isEqual(k, e.key)) {
+          size--;
+          if (prev == e) {
+            entries[i] = next;
+          } else {
+            prev.next = next;
+          }
+
+          Entry prevPtr = e.getPrevPtr();
+          Entry nextPtr = e.getNextPtr();
+
+          if(prevPtr != null && nextPtr != null) {
+            prevPtr.setNextPtr(nextPtr);
+            nextPtr.setPrevPtr(prevPtr);
+          } else if(prevPtr != null) {
+            tailPtr = prevPtr;
+            prevPtr.setNextPtr(null);
+          } else if(nextPtr != null) {
+            headPtr = nextPtr;
+            nextPtr.setPrevPtr(null);
+          }
+
+          return;
+      }
+      prev = e;
+      e = next;
+    }
+  }
+
+  /**
+   * Removes and returns the entry associated with the specified
+   * key.
+   *
+   * @param key key of the entry to be deleted
+   * @return entry that was removed, or null if none found
+   */
+  private Entry removeEntryForKey(K key) {
+    int hash = hash(key);
+    int i = hashIndex(hash, entries.length);
+    Entry prev = entries[i];
+    Entry e = prev;
+
+    while (e != null) {
+      Entry next = e.next;
+      if (e.hash == hash && isEqual(key, e.key)) {
+          size--;
+          if (prev == e) {
+            entries[i] = next;
+          } else {
+            prev.next = next;
+          }
+
+          // Updating LRU
+          Entry prevPtr = e.getPrevPtr();
+          Entry nextPtr = e.getNextPtr();
+          if(prevPtr != null && nextPtr != null) {
+            prevPtr.setNextPtr(nextPtr);
+            nextPtr.setPrevPtr(prevPtr);
+          } else if(prevPtr != null) {
+            tailPtr = prevPtr;
+            prevPtr.setNextPtr(null);
+          } else if(nextPtr != null) {
+            headPtr = nextPtr;
+            nextPtr.setPrevPtr(null);
+          }
+
+          return e;
+      }
+      prev = e;
+      e = next;
+    }
+
+    return e;
+  }
+
+ /**
+  * Adds a new entry with the specified key, value, hash code, and
+  * bucket index to the map.
+  *
+  * Also puts it in the bottom (most-recent) slot of the list and
+  * checks to see if we need to grow the array.
+  *
+  * @param hash hash value of key
+  * @param key the key
+  * @param value the value
+  * @param bucketIndex index into hash array to store this entry
+  * @return the amount of heap size used to store the new entry
+  */
+  private long addEntry(int hash, K key, V value, int bucketIndex) {
+    Entry e = entries[bucketIndex];
+    Entry newE = new Entry(hash, key, value, e, tailPtr);
+    entries[bucketIndex] = newE;
+    // add as most recently used in lru
+    if (size == 0) {
+      headPtr = newE;
+      tailPtr = newE;
+    } else {
+      newE.setPrevPtr(tailPtr);
+      tailPtr.setNextPtr(newE);
+      tailPtr = newE;
+    }
+    // Grow table if we are past the threshold now
+    if (size++ >= threshold) {
+      growTable(2 * entries.length);
+    }
+    return newE.heapSize();
+  }
+
+  /**
+   * Clears all the entries in the map.  Tracks the amount of memory being
+   * freed along the way and returns the total.
+   *
+   * Cleans up all references to allow old entries to be GC'd.
+   *
+   * @return total memory freed in bytes
+   */
+  private long clearAll() {
+    Entry cur;
+    long freedMemory = 0;
+    for(int i=0; i entry = oldTable[i];
+      if(entry != null) {
+        // Set to null for GC
+        oldTable[i] = null;
+        do {
+          Entry next = entry.next;
+          int idx = hashIndex(entry.hash, newCapacity);
+          entry.next = newTable[idx];
+          newTable[idx] = entry;
+          entry = next;
+        } while(entry != null);
+      }
+    }
+
+    entries = newTable;
+    threshold = (int)(newCapacity * loadFactor);
+  }
+
+  /**
+   * Gets the hash code for the specified key.
+   * This implementation uses the additional hashing routine
+   * from JDK 1.4.
+   *
+   * @param key the key to get a hash value for
+   * @return the hash value
+   */
+  private int hash(Object key) {
+    int h = key.hashCode();
+    h += ~(h << 9);
+    h ^=  (h >>> 14);
+    h +=  (h << 4);
+    h ^=  (h >>> 10);
+    return h;
+  }
+
+  /**
+   * Compares two objects for equality.  Method uses equals method and
+   * assumes neither value is null.
+   *
+   * @param x the first value
+   * @param y the second value
+   * @return true if equal
+   */
+  private boolean isEqual(Object x, Object y) {
+    return (x == y || x.equals(y));
+  }
+
+  /**
+   * Determines the index into the current hash table for the specified
+   * hashValue.
+   *
+   * @param hashValue the hash value
+   * @param length the current number of hash buckets
+   * @return the index of the current hash array to use
+   */
+  private int hashIndex(int hashValue, int length) {
+    return hashValue & (length - 1);
+  }
+
+  /**
+   * Calculates the capacity of the array backing the hash
+   * by normalizing capacity to a power of 2 and enforcing
+   * capacity limits.
+   *
+   * @param proposedCapacity the proposed capacity
+   * @return the normalized capacity
+   */
+  private int calculateCapacity(int proposedCapacity) {
+    int newCapacity = 1;
+    if(proposedCapacity > MAXIMUM_CAPACITY) {
+      newCapacity = MAXIMUM_CAPACITY;
+    } else {
+      while(newCapacity < proposedCapacity) {
+        newCapacity <<= 1;
+      }
+      if(newCapacity > MAXIMUM_CAPACITY) {
+        newCapacity = MAXIMUM_CAPACITY;
+      }
+    }
+    return newCapacity;
+  }
+
+  /**
+   * Calculates the threshold of the map given the capacity and load
+   * factor.  Once the number of entries in the map grows to the
+   * threshold we will double the size of the array.
+   *
+   * @param capacity the size of the array
+   * @param factor the load factor of the hash
+   */
+  private int calculateThreshold(int capacity, float factor) {
+    return (int)(capacity * factor);
+  }
+
+  /**
+   * Set the initial heap usage of this class.  Includes class variable
+   * overhead and the entry array.
+   */
+  private void init() {
+    memFree -= OVERHEAD;
+    memFree -= (entries.length * ClassSize.REFERENCE);
+  }
+
+  //--------------------------------------------------------------------------
+  /**
+   * Debugging function that returns a List sorted by access time.
+   *
+   * The order is oldest to newest (first in list is next to be evicted).
+   *
+   * @return Sorted list of entries
+   */
+  public List> entryLruList() {
+    List> entryList = new ArrayList>();
+    Entry entry = headPtr;
+    while(entry != null) {
+      entryList.add(entry);
+      entry = entry.getNextPtr();
+    }
+    return entryList;
+  }
+
+  /**
+   * Debugging function that returns a Set of all entries in the hash table.
+   *
+   * @return Set of entries in hash
+   */
+  public Set> entryTableSet() {
+    Set> entrySet = new HashSet>();
+    Entry [] table = entries; // FindBugs IS2_INCONSISTENT_SYNC
+    for(int i=0;i> entrySet() {
+    throw new UnsupportedOperationException(
+    "entrySet() is intentionally unimplemented");
+  }
+
+  /**
+   * Intentionally unimplemented.
+   */
+  public boolean equals(Object o) {
+    throw new UnsupportedOperationException(
+    "equals(Object) is intentionally unimplemented");
+  }
+
+  /**
+   * Intentionally unimplemented.
+   */
+  public int hashCode() {
+    throw new UnsupportedOperationException(
+    "hashCode(Object) is intentionally unimplemented");
+  }
+
+  /**
+   * Intentionally unimplemented.
+   */
+  public Set keySet() {
+    throw new UnsupportedOperationException(
+    "keySet() is intentionally unimplemented");
+  }
+
+  /**
+   * Intentionally unimplemented.
+   */
+  public void putAll(Map m) {
+    throw new UnsupportedOperationException(
+    "putAll() is intentionally unimplemented");
+  }
+
+  /**
+   * Intentionally unimplemented.
+   */
+  public Collection values() {
+    throw new UnsupportedOperationException(
+    "values() is intentionally unimplemented");
+  }
+
+  //--------------------------------------------------------------------------
+  /**
+   * Entry to store key/value mappings.
+   * 
                
+   * Contains previous and next pointers for the doubly linked-list which is
+   * used for LRU eviction.
+   * 
                
+   * Instantiations of this class are memory aware.  Both the key and value
+   * classes used must also implement HeapSize.
+   */
+  protected static class Entry
+  implements Map.Entry, HeapSize {
+    /** The baseline overhead memory usage of this class */
+    static final int OVERHEAD = 1 * Bytes.SIZEOF_LONG +
+      5 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT;
+
+    /** The key */
+    protected final K key;
+    /** The value */
+    protected V value;
+    /** The hash value for this entries key */
+    protected final int hash;
+    /** The next entry in the hash chain (for collisions) */
+    protected Entry next;
+
+    /** The previous entry in the LRU list (towards LRU) */
+    protected Entry prevPtr;
+    /** The next entry in the LRU list (towards MRU) */
+    protected Entry nextPtr;
+
+    /** The precomputed heap size of this entry */
+    protected long heapSize;
+
+    /**
+     * Create a new entry.
+     *
+     * @param h the hash value of the key
+     * @param k the key
+     * @param v the value
+     * @param nextChainPtr the next entry in the hash chain, null if none
+     * @param prevLruPtr the previous entry in the LRU
+     */
+    Entry(int h, K k, V v, Entry nextChainPtr, Entry prevLruPtr) {
+      value = v;
+      next = nextChainPtr;
+      key = k;
+      hash = h;
+      prevPtr = prevLruPtr;
+      nextPtr = null;
+      // Pre-compute heap size
+      heapSize = OVERHEAD + k.heapSize() + v.heapSize();
+    }
+
+    /**
+     * Get the key of this entry.
+     *
+     * @return the key associated with this entry
+     */
+    public K getKey() {
+      return key;
+    }
+
+    /**
+     * Get the value of this entry.
+     *
+     * @return the value currently associated with this entry
+     */
+    public V getValue() {
+      return value;
+    }
+
+    /**
+     * Set the value of this entry.
+     *
+     * It is not recommended to use this method when changing the value.
+     * Rather, using replaceValue will return the difference
+     * in heap usage between the previous and current values.
+     *
+     * @param newValue the new value to associate with this entry
+     * @return the value previously associated with this entry
+     */
+    public V setValue(V newValue) {
+      V oldValue = value;
+      value = newValue;
+      return oldValue;
+    }
+
+    /**
+     * Replace the value of this entry.
+     *
+     * Computes and returns the difference in heap size when changing
+     * the value associated with this entry.
+     *
+     * @param newValue the new value to associate with this entry
+     * @return the change in heap usage of this entry in bytes
+     */
+    protected long replaceValue(V newValue) {
+      long sizeDiff = newValue.heapSize() - value.heapSize();
+      value = newValue;
+      heapSize += sizeDiff;
+      return sizeDiff;
+    }
+
+    /**
+     * Returns true is the specified entry has the same key and the
+     * same value as this entry.
+     *
+     * @param o entry to test against current
+     * @return true is entries have equal key and value, false if no
+     */
+    public boolean equals(Object o) {
+      if (!(o instanceof Map.Entry))
+          return false;
+      Map.Entry e = (Map.Entry)o;
+      Object k1 = getKey();
+      Object k2 = e.getKey();
+      if (k1 == k2 || (k1 != null && k1.equals(k2))) {
+          Object v1 = getValue();
+          Object v2 = e.getValue();
+          if (v1 == v2 || (v1 != null && v1.equals(v2)))
+            return true;
+      }
+      return false;
+    }
+
+    /**
+     * Returns the hash code of the entry by xor'ing the hash values
+     * of the key and value of this entry.
+     *
+     * @return hash value of this entry
+     */
+    public int hashCode() {
+      return (key.hashCode() ^ value.hashCode());
+    }
+
+    /**
+     * Returns String representation of the entry in form "key=value"
+     *
+     * @return string value of entry
+     */
+    public String toString() {
+      return getKey() + "=" + getValue();
+    }
+
+    //------------------------------------------------------------------------
+    /**
+     * Sets the previous pointer for the entry in the LRU.
+     * @param prevPtr previous entry
+     */
+    protected void setPrevPtr(Entry prevPtr){
+      this.prevPtr = prevPtr;
+    }
+
+    /**
+     * Returns the previous pointer for the entry in the LRU.
+     * @return previous entry
+     */
+    protected Entry getPrevPtr(){
+      return prevPtr;
+    }
+
+    /**
+     * Sets the next pointer for the entry in the LRU.
+     * @param nextPtr next entry
+     */
+    protected void setNextPtr(Entry nextPtr){
+      this.nextPtr = nextPtr;
+    }
+
+    /**
+     * Returns the next pointer for the entry in teh LRU.
+     * @return next entry
+     */
+    protected Entry getNextPtr(){
+      return nextPtr;
+    }
+
+    /**
+     * Returns the pre-computed and "deep" size of the Entry
+     * @return size of the entry in bytes
+     */
+    public long heapSize() {
+      return heapSize;
+    }
+  }
+}
+
+
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MXBean.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MXBean.java
new file mode 100644
index 0000000..2d75ab5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MXBean.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+/**
+ * This is the JMX management interface for HBase Region Server information
+ */
+public interface MXBean {
+
+  /**
+   * Return RegionServer's ServerName
+   * @return ServerName
+   */
+  public String getServerName();
+
+  /**
+   * Get loaded co-processors
+   * @return Loaded Co-processors
+   */
+  public String[] getCoprocessors();
+
+  /**
+   * Get Zookeeper Quorum
+   * @return Comma-separated list of Zookeeper Quorum servers
+   */
+  public String getZookeeperQuorum();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MXBeanImpl.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MXBeanImpl.java
new file mode 100644
index 0000000..78f3b6f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MXBeanImpl.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+/**
+ * Impl for exposing Region Server Information through JMX
+ */
+public class MXBeanImpl implements MXBean {
+
+  private final HRegionServer regionServer;
+
+  private static MXBeanImpl instance = null;
+  public synchronized static MXBeanImpl init(final HRegionServer rs){
+    if (instance == null) {
+      instance = new MXBeanImpl(rs);
+    }
+    return instance;
+  }
+
+  protected MXBeanImpl(final HRegionServer rs) {
+    this.regionServer = rs;
+  }
+
+  @Override
+  public String[] getCoprocessors() {
+    return regionServer.getCoprocessors();
+  }
+
+  @Override
+  public String getZookeeperQuorum() {
+    return regionServer.getZooKeeper().getQuorum();
+  }
+
+  @Override
+  public String getServerName() {
+    return regionServer.getServerName().getServerName();
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
new file mode 100644
index 0000000..113af80
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -0,0 +1,1011 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.rmi.UnexpectedException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.SortedSet;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.regionserver.MemStoreLAB.Allocation;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+/**
+ * The MemStore holds in-memory modifications to the Store.  Modifications
+ * are {@link KeyValue}s.  When asked to flush, current memstore is moved
+ * to snapshot and is cleared.  We continue to serve edits out of new memstore
+ * and backing snapshot until flusher reports in that the flush succeeded. At
+ * this point we let the snapshot go.
+ * TODO: Adjust size of the memstore when we remove items because they have
+ * been deleted.
+ * TODO: With new KVSLS, need to make sure we update HeapSize with difference
+ * in KV size.
+ */
+public class MemStore implements HeapSize {
+  private static final Log LOG = LogFactory.getLog(MemStore.class);
+
+  static final String USEMSLAB_KEY =
+    "hbase.hregion.memstore.mslab.enabled";
+  private static final boolean USEMSLAB_DEFAULT = true;
+
+  private Configuration conf;
+
+  // MemStore.  Use a KeyValueSkipListSet rather than SkipListSet because of the
+  // better semantics.  The Map will overwrite if passed a key it already had
+  // whereas the Set will not add new KV if key is same though value might be
+  // different.  Value is not important -- just make sure always same
+  // reference passed.
+  volatile KeyValueSkipListSet kvset;
+
+  // Snapshot of memstore.  Made for flusher.
+  volatile KeyValueSkipListSet snapshot;
+
+  final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+  final KeyValue.KVComparator comparator;
+
+  // Used comparing versions -- same r/c and ts but different type.
+  final KeyValue.KVComparator comparatorIgnoreType;
+
+  // Used comparing versions -- same r/c and type but different timestamp.
+  final KeyValue.KVComparator comparatorIgnoreTimestamp;
+
+  // Used to track own heapSize
+  final AtomicLong size;
+
+  // Used to track when to flush
+  volatile long timeOfOldestEdit = Long.MAX_VALUE;
+
+  TimeRangeTracker timeRangeTracker;
+  TimeRangeTracker snapshotTimeRangeTracker;
+
+  MemStoreLAB allocator;
+
+
+
+  /**
+   * Default constructor. Used for tests.
+   */
+  public MemStore() {
+    this(HBaseConfiguration.create(), KeyValue.COMPARATOR);
+  }
+
+  /**
+   * Constructor.
+   * @param c Comparator
+   */
+  public MemStore(final Configuration conf,
+                  final KeyValue.KVComparator c) {
+    this.conf = conf;
+    this.comparator = c;
+    this.comparatorIgnoreTimestamp =
+      this.comparator.getComparatorIgnoringTimestamps();
+    this.comparatorIgnoreType = this.comparator.getComparatorIgnoringType();
+    this.kvset = new KeyValueSkipListSet(c);
+    this.snapshot = new KeyValueSkipListSet(c);
+    timeRangeTracker = new TimeRangeTracker();
+    snapshotTimeRangeTracker = new TimeRangeTracker();
+    this.size = new AtomicLong(DEEP_OVERHEAD);
+    if (conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT)) {
+      this.allocator = new MemStoreLAB(conf);
+    } else {
+      this.allocator = null;
+    }
+  }
+
+  void dump() {
+    for (KeyValue kv: this.kvset) {
+      LOG.info(kv);
+    }
+    for (KeyValue kv: this.snapshot) {
+      LOG.info(kv);
+    }
+  }
+
+  /**
+   * Creates a snapshot of the current memstore.
+   * Snapshot must be cleared by call to {@link #clearSnapshot(SortedSet)}
+   * To get the snapshot made by this method, use {@link #getSnapshot()}
+   */
+  void snapshot() {
+    this.lock.writeLock().lock();
+    try {
+      // If snapshot currently has entries, then flusher failed or didn't call
+      // cleanup.  Log a warning.
+      if (!this.snapshot.isEmpty()) {
+        LOG.warn("Snapshot called again without clearing previous. " +
+          "Doing nothing. Another ongoing flush or did we fail last attempt?");
+      } else {
+        if (!this.kvset.isEmpty()) {
+          this.snapshot = this.kvset;
+          this.kvset = new KeyValueSkipListSet(this.comparator);
+          this.snapshotTimeRangeTracker = this.timeRangeTracker;
+          this.timeRangeTracker = new TimeRangeTracker();
+          // Reset heap to not include any keys
+          this.size.set(DEEP_OVERHEAD);
+          // Reset allocator so we get a fresh buffer for the new memstore
+          if (allocator != null) {
+            this.allocator = new MemStoreLAB(conf);
+          }
+          timeOfOldestEdit = Long.MAX_VALUE;
+        }
+      }
+    } finally {
+      this.lock.writeLock().unlock();
+    }
+  }
+
+  /**
+   * Return the current snapshot.
+   * Called by flusher to get current snapshot made by a previous
+   * call to {@link #snapshot()}
+   * @return Return snapshot.
+   * @see {@link #snapshot()}
+   * @see {@link #clearSnapshot(SortedSet)}
+   */
+  KeyValueSkipListSet getSnapshot() {
+    return this.snapshot;
+  }
+
+  /**
+   * The passed snapshot was successfully persisted; it can be let go.
+   * @param ss The snapshot to clean out.
+   * @throws UnexpectedException
+   * @see {@link #snapshot()}
+   */
+  void clearSnapshot(final SortedSet ss)
+  throws UnexpectedException {
+    this.lock.writeLock().lock();
+    try {
+      if (this.snapshot != ss) {
+        throw new UnexpectedException("Current snapshot is " +
+          this.snapshot + ", was passed " + ss);
+      }
+      // OK. Passed in snapshot is same as current snapshot.  If not-empty,
+      // create a new snapshot and let the old one go.
+      if (!ss.isEmpty()) {
+        this.snapshot = new KeyValueSkipListSet(this.comparator);
+        this.snapshotTimeRangeTracker = new TimeRangeTracker();
+      }
+    } finally {
+      this.lock.writeLock().unlock();
+    }
+  }
+
+  /**
+   * Write an update
+   * @param kv
+   * @return approximate size of the passed key and value.
+   */
+  long add(final KeyValue kv) {
+    this.lock.readLock().lock();
+    try {
+      KeyValue toAdd = maybeCloneWithAllocator(kv);
+      return internalAdd(toAdd);
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+
+  long timeOfOldestEdit() {
+    return timeOfOldestEdit;
+  }
+
+  private boolean addToKVSet(KeyValue e) {
+    boolean b = this.kvset.add(e);
+    setOldestEditTimeToNow();
+    return b;
+  }
+
+  private boolean removeFromKVSet(KeyValue e) {
+    boolean b = this.kvset.remove(e);
+    setOldestEditTimeToNow();
+    return b;
+  }
+
+  void setOldestEditTimeToNow() {
+    if (timeOfOldestEdit == Long.MAX_VALUE) {
+      timeOfOldestEdit = EnvironmentEdgeManager.currentTimeMillis();
+    }
+  }
+
+  /**
+   * Internal version of add() that doesn't clone KVs with the
+   * allocator, and doesn't take the lock.
+   *
+   * Callers should ensure they already have the read lock taken
+   */
+  private long internalAdd(final KeyValue toAdd) {
+    long s = heapSizeChange(toAdd, addToKVSet(toAdd));
+    timeRangeTracker.includeTimestamp(toAdd);
+    this.size.addAndGet(s);
+    return s;
+  }
+
+  private KeyValue maybeCloneWithAllocator(KeyValue kv) {
+    if (allocator == null) {
+      return kv;
+    }
+
+    int len = kv.getLength();
+    Allocation alloc = allocator.allocateBytes(len);
+    if (alloc == null) {
+      // The allocation was too large, allocator decided
+      // not to do anything with it.
+      return kv;
+    }
+    assert alloc != null && alloc.getData() != null;
+    System.arraycopy(kv.getBuffer(), kv.getOffset(), alloc.getData(), alloc.getOffset(), len);
+    KeyValue newKv = new KeyValue(alloc.getData(), alloc.getOffset(), len);
+    newKv.setMemstoreTS(kv.getMemstoreTS());
+    return newKv;
+  }
+
+  /**
+   * Remove n key from the memstore. Only kvs that have the same key and the
+   * same memstoreTS are removed.  It is ok to not update timeRangeTracker
+   * in this call. It is possible that we can optimize this method by using
+   * tailMap/iterator, but since this method is called rarely (only for
+   * error recovery), we can leave those optimization for the future.
+   * @param kv
+   */
+  void rollback(final KeyValue kv) {
+    this.lock.readLock().lock();
+    try {
+      // If the key is in the snapshot, delete it. We should not update
+      // this.size, because that tracks the size of only the memstore and
+      // not the snapshot. The flush of this snapshot to disk has not
+      // yet started because Store.flush() waits for all rwcc transactions to
+      // commit before starting the flush to disk.
+      KeyValue found = this.snapshot.get(kv);
+      if (found != null && found.getMemstoreTS() == kv.getMemstoreTS()) {
+        this.snapshot.remove(kv);
+      }
+      // If the key is in the memstore, delete it. Update this.size.
+      found = this.kvset.get(kv);
+      if (found != null && found.getMemstoreTS() == kv.getMemstoreTS()) {
+        removeFromKVSet(kv);
+        long s = heapSizeChange(kv, true);
+        this.size.addAndGet(-s);
+      }
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+
+  /**
+   * Write a delete
+   * @param delete
+   * @return approximate size of the passed key and value.
+   */
+  long delete(final KeyValue delete) {
+    long s = 0;
+    this.lock.readLock().lock();
+    try {
+      KeyValue toAdd = maybeCloneWithAllocator(delete);
+      s += heapSizeChange(toAdd, addToKVSet(toAdd));
+      timeRangeTracker.includeTimestamp(toAdd);
+    } finally {
+      this.lock.readLock().unlock();
+    }
+    this.size.addAndGet(s);
+    return s;
+  }
+
+  /**
+   * @param kv Find the row that comes after this one.  If null, we return the
+   * first.
+   * @return Next row or null if none found.
+   */
+  KeyValue getNextRow(final KeyValue kv) {
+    this.lock.readLock().lock();
+    try {
+      return getLowest(getNextRow(kv, this.kvset), getNextRow(kv, this.snapshot));
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+
+  /*
+   * @param a
+   * @param b
+   * @return Return lowest of a or b or null if both a and b are null
+   */
+  private KeyValue getLowest(final KeyValue a, final KeyValue b) {
+    if (a == null) {
+      return b;
+    }
+    if (b == null) {
+      return a;
+    }
+    return comparator.compareRows(a, b) <= 0? a: b;
+  }
+
+  /*
+   * @param key Find row that follows this one.  If null, return first.
+   * @param map Set to look in for a row beyond row.
+   * @return Next row or null if none found.  If one found, will be a new
+   * KeyValue -- can be destroyed by subsequent calls to this method.
+   */
+  private KeyValue getNextRow(final KeyValue key,
+      final NavigableSet set) {
+    KeyValue result = null;
+    SortedSet tail = key == null? set: set.tailSet(key);
+    // Iterate until we fall into the next row; i.e. move off current row
+    for (KeyValue kv: tail) {
+      if (comparator.compareRows(kv, key) <= 0)
+        continue;
+      // Note: Not suppressing deletes or expired cells.  Needs to be handled
+      // by higher up functions.
+      result = kv;
+      break;
+    }
+    return result;
+  }
+
+  /**
+   * @param state column/delete tracking state
+   */
+  void getRowKeyAtOrBefore(final GetClosestRowBeforeTracker state) {
+    this.lock.readLock().lock();
+    try {
+      getRowKeyAtOrBefore(kvset, state);
+      getRowKeyAtOrBefore(snapshot, state);
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+
+  /*
+   * @param set
+   * @param state Accumulates deletes and candidates.
+   */
+  private void getRowKeyAtOrBefore(final NavigableSet set,
+      final GetClosestRowBeforeTracker state) {
+    if (set.isEmpty()) {
+      return;
+    }
+    if (!walkForwardInSingleRow(set, state.getTargetKey(), state)) {
+      // Found nothing in row.  Try backing up.
+      getRowKeyBefore(set, state);
+    }
+  }
+
+  /*
+   * Walk forward in a row from firstOnRow.  Presumption is that
+   * we have been passed the first possible key on a row.  As we walk forward
+   * we accumulate deletes until we hit a candidate on the row at which point
+   * we return.
+   * @param set
+   * @param firstOnRow First possible key on this row.
+   * @param state
+   * @return True if we found a candidate walking this row.
+   */
+  private boolean walkForwardInSingleRow(final SortedSet set,
+      final KeyValue firstOnRow, final GetClosestRowBeforeTracker state) {
+    boolean foundCandidate = false;
+    SortedSet tail = set.tailSet(firstOnRow);
+    if (tail.isEmpty()) return foundCandidate;
+    for (Iterator i = tail.iterator(); i.hasNext();) {
+      KeyValue kv = i.next();
+      // Did we go beyond the target row? If so break.
+      if (state.isTooFar(kv, firstOnRow)) break;
+      if (state.isExpired(kv)) {
+        i.remove();
+        continue;
+      }
+      // If we added something, this row is a contender. break.
+      if (state.handle(kv)) {
+        foundCandidate = true;
+        break;
+      }
+    }
+    return foundCandidate;
+  }
+
+  /*
+   * Walk backwards through the passed set a row at a time until we run out of
+   * set or until we get a candidate.
+   * @param set
+   * @param state
+   */
+  private void getRowKeyBefore(NavigableSet set,
+      final GetClosestRowBeforeTracker state) {
+    KeyValue firstOnRow = state.getTargetKey();
+    for (Member p = memberOfPreviousRow(set, state, firstOnRow);
+        p != null; p = memberOfPreviousRow(p.set, state, firstOnRow)) {
+      // Make sure we don't fall out of our table.
+      if (!state.isTargetTable(p.kv)) break;
+      // Stop looking if we've exited the better candidate range.
+      if (!state.isBetterCandidate(p.kv)) break;
+      // Make into firstOnRow
+      firstOnRow = new KeyValue(p.kv.getRow(), HConstants.LATEST_TIMESTAMP);
+      // If we find something, break;
+      if (walkForwardInSingleRow(p.set, firstOnRow, state)) break;
+    }
+  }
+
+  /**
+   * Given the specs of a column, update it, first by inserting a new record,
+   * then removing the old one.  Since there is only 1 KeyValue involved, the memstoreTS
+   * will be set to 0, thus ensuring that they instantly appear to anyone. The underlying
+   * store will ensure that the insert/delete each are atomic. A scanner/reader will either
+   * get the new value, or the old value and all readers will eventually only see the new
+   * value after the old was removed.
+   *
+   * @param row
+   * @param family
+   * @param qualifier
+   * @param newValue
+   * @param now
+   * @return  Timestamp
+   */
+  public long updateColumnValue(byte[] row,
+                                byte[] family,
+                                byte[] qualifier,
+                                long newValue,
+                                long now) {
+   this.lock.readLock().lock();
+    try {
+      KeyValue firstKv = KeyValue.createFirstOnRow(
+          row, family, qualifier);
+      // Is there a KeyValue in 'snapshot' with the same TS? If so, upgrade the timestamp a bit.
+      SortedSet snSs = snapshot.tailSet(firstKv);
+      if (!snSs.isEmpty()) {
+        KeyValue snKv = snSs.first();
+        // is there a matching KV in the snapshot?
+        if (snKv.matchingRow(firstKv) && snKv.matchingQualifier(firstKv)) {
+          if (snKv.getTimestamp() == now) {
+            // poop,
+            now += 1;
+          }
+        }
+      }
+
+      // logic here: the new ts MUST be at least 'now'. But it could be larger if necessary.
+      // But the timestamp should also be max(now, mostRecentTsInMemstore)
+
+      // so we cant add the new KV w/o knowing what's there already, but we also
+      // want to take this chance to delete some kvs. So two loops (sad)
+
+      SortedSet ss = kvset.tailSet(firstKv);
+      Iterator it = ss.iterator();
+      while ( it.hasNext() ) {
+        KeyValue kv = it.next();
+
+        // if this isnt the row we are interested in, then bail:
+        if (!kv.matchingColumn(family,qualifier) || !kv.matchingRow(firstKv) ) {
+          break; // rows dont match, bail.
+        }
+
+        // if the qualifier matches and it's a put, just RM it out of the kvset.
+        if (kv.getType() == KeyValue.Type.Put.getCode() &&
+            kv.getTimestamp() > now && firstKv.matchingQualifier(kv)) {
+          now = kv.getTimestamp();
+        }
+      }
+
+      // create or update (upsert) a new KeyValue with
+      // 'now' and a 0 memstoreTS == immediately visible
+      return upsert(Arrays.asList(
+          new KeyValue(row, family, qualifier, now, Bytes.toBytes(newValue)))
+      );
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+
+  /**
+   * Update or insert the specified KeyValues.
+   * 
                
+   * For each KeyValue, insert into MemStore.  This will atomically upsert the
+   * value for that row/family/qualifier.  If a KeyValue did already exist,
+   * it will then be removed.
+   * 
                
+   * Currently the memstoreTS is kept at 0 so as each insert happens, it will
+   * be immediately visible.  May want to change this so it is atomic across
+   * all KeyValues.
+   * 
                
+   * This is called under row lock, so Get operations will still see updates
+   * atomically.  Scans will only see each KeyValue update as atomic.
+   *
+   * @param kvs
+   * @return change in memstore size
+   */
+  public long upsert(List kvs) {
+   this.lock.readLock().lock();
+    try {
+      long size = 0;
+      for (KeyValue kv : kvs) {
+        kv.setMemstoreTS(0);
+        size += upsert(kv);
+      }
+      return size;
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+
+  /**
+   * Inserts the specified KeyValue into MemStore and deletes any existing
+   * versions of the same row/family/qualifier as the specified KeyValue.
+   * 
                
+   * First, the specified KeyValue is inserted into the Memstore.
+   * 
                
+   * If there are any existing KeyValues in this MemStore with the same row,
+   * family, and qualifier, they are removed.
+   * 
                
+   * Callers must hold the read lock.
+   *
+   * @param kv
+   * @return change in size of MemStore
+   */
+  private long upsert(KeyValue kv) {
+    // Add the KeyValue to the MemStore
+    // Use the internalAdd method here since we (a) already have a lock
+    // and (b) cannot safely use the MSLAB here without potentially
+    // hitting OOME - see TestMemStore.testUpsertMSLAB for a
+    // test that triggers the pathological case if we don't avoid MSLAB
+    // here.
+    long addedSize = internalAdd(kv);
+
+    // Get the KeyValues for the row/family/qualifier regardless of timestamp.
+    // For this case we want to clean up any other puts
+    KeyValue firstKv = KeyValue.createFirstOnRow(
+        kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
+        kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(),
+        kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength());
+    SortedSet ss = kvset.tailSet(firstKv);
+    Iterator it = ss.iterator();
+    while ( it.hasNext() ) {
+      KeyValue cur = it.next();
+
+      if (kv == cur) {
+        // ignore the one just put in
+        continue;
+      }
+      // if this isn't the row we are interested in, then bail
+      if (!kv.matchingRow(cur)) {
+        break;
+      }
+
+      // if the qualifier matches and it's a put, remove it
+      if (kv.matchingQualifier(cur)) {
+
+        // to be extra safe we only remove Puts that have a memstoreTS==0
+        if (kv.getType() == KeyValue.Type.Put.getCode() &&
+            kv.getMemstoreTS() == 0) {
+          // false means there was a change, so give us the size.
+          long delta = heapSizeChange(cur, true);
+          addedSize -= delta;
+          this.size.addAndGet(-delta);
+          it.remove();
+          setOldestEditTimeToNow();
+        }
+      } else {
+        // past the column, done
+        break;
+      }
+    }
+    return addedSize;
+  }
+
+  /*
+   * Immutable data structure to hold member found in set and the set it was
+   * found in.  Include set because it is carrying context.
+   */
+  private static class Member {
+    final KeyValue kv;
+    final NavigableSet set;
+    Member(final NavigableSet s, final KeyValue kv) {
+      this.kv = kv;
+      this.set = s;
+    }
+  }
+
+  /*
+   * @param set Set to walk back in.  Pass a first in row or we'll return
+   * same row (loop).
+   * @param state Utility and context.
+   * @param firstOnRow First item on the row after the one we want to find a
+   * member in.
+   * @return Null or member of row previous to firstOnRow
+   */
+  private Member memberOfPreviousRow(NavigableSet set,
+      final GetClosestRowBeforeTracker state, final KeyValue firstOnRow) {
+    NavigableSet head = set.headSet(firstOnRow, false);
+    if (head.isEmpty()) return null;
+    for (Iterator i = head.descendingIterator(); i.hasNext();) {
+      KeyValue found = i.next();
+      if (state.isExpired(found)) {
+        i.remove();
+        continue;
+      }
+      return new Member(head, found);
+    }
+    return null;
+  }
+
+  /**
+   * @return scanner on memstore and snapshot in this order.
+   */
+  List getScanners() {
+    this.lock.readLock().lock();
+    try {
+      return Collections.singletonList(
+          new MemStoreScanner());
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+
+  /**
+   * Check if this memstore may contain the required keys
+   * @param scan
+   * @return False if the key definitely does not exist in this Memstore
+   */
+  public boolean shouldSeek(Scan scan, long oldestUnexpiredTS) {
+    return (timeRangeTracker.includesTimeRange(scan.getTimeRange()) ||
+        snapshotTimeRangeTracker.includesTimeRange(scan.getTimeRange()))
+        && (Math.max(timeRangeTracker.getMaximumTimestamp(),
+                     snapshotTimeRangeTracker.getMaximumTimestamp()) >=
+            oldestUnexpiredTS);
+  }
+
+  public TimeRangeTracker getSnapshotTimeRangeTracker() {
+    return this.snapshotTimeRangeTracker;
+  }
+
+  /*
+   * MemStoreScanner implements the KeyValueScanner.
+   * It lets the caller scan the contents of a memstore -- both current
+   * map and snapshot.
+   * This behaves as if it were a real scanner but does not maintain position.
+   */
+  protected class MemStoreScanner extends NonLazyKeyValueScanner {
+    // Next row information for either kvset or snapshot
+    private KeyValue kvsetNextRow = null;
+    private KeyValue snapshotNextRow = null;
+
+    // last iterated KVs for kvset and snapshot (to restore iterator state after reseek)
+    private KeyValue kvsetItRow = null;
+    private KeyValue snapshotItRow = null;
+    
+    // iterator based scanning.
+    private Iterator kvsetIt;
+    private Iterator snapshotIt;
+
+    // The kvset and snapshot at the time of creating this scanner
+    volatile KeyValueSkipListSet kvsetAtCreation;
+    volatile KeyValueSkipListSet snapshotAtCreation;
+
+    // the pre-calculated KeyValue to be returned by peek() or next()
+    private KeyValue theNext;
+
+    /*
+    Some notes...
+
+     So memstorescanner is fixed at creation time. this includes pointers/iterators into
+    existing kvset/snapshot.  during a snapshot creation, the kvset is null, and the
+    snapshot is moved.  since kvset is null there is no point on reseeking on both,
+      we can save us the trouble. During the snapshot->hfile transition, the memstore
+      scanner is re-created by StoreScanner#updateReaders().  StoreScanner should
+      potentially do something smarter by adjusting the existing memstore scanner.
+
+      But there is a greater problem here, that being once a scanner has progressed
+      during a snapshot scenario, we currently iterate past the kvset then 'finish' up.
+      if a scan lasts a little while, there is a chance for new entries in kvset to
+      become available but we will never see them.  This needs to be handled at the
+      StoreScanner level with coordination with MemStoreScanner.
+
+      Currently, this problem is only partly managed: during the small amount of time
+      when the StoreScanner has not yet created a new MemStoreScanner, we will miss
+      the adds to kvset in the MemStoreScanner.
+    */
+
+    MemStoreScanner() {
+      super();
+
+      kvsetAtCreation = kvset;
+      snapshotAtCreation = snapshot;
+    }
+
+    private KeyValue getNext(Iterator it) {
+      long readPoint = MultiVersionConsistencyControl.getThreadReadPoint();
+
+      KeyValue v = null;
+      try {
+        while (it.hasNext()) {
+          v = it.next();
+          if (v.getMemstoreTS() <= readPoint) {
+            return v;
+          }
+        }
+
+        return null;
+      } finally {
+        if (v != null) {
+          // in all cases, remember the last KV iterated to
+          if (it == snapshotIt) {
+            snapshotItRow = v;
+          } else {
+            kvsetItRow = v;
+          }
+        }
+      }
+    }
+
+    /**
+     *  Set the scanner at the seek key.
+     *  Must be called only once: there is no thread safety between the scanner
+     *   and the memStore.
+     * @param key seek value
+     * @return false if the key is null or if there is no data
+     */
+    @Override
+    public synchronized boolean seek(KeyValue key) {
+      if (key == null) {
+        close();
+        return false;
+      }
+
+      // kvset and snapshot will never be null.
+      // if tailSet can't find anything, SortedSet is empty (not null).
+      kvsetIt = kvsetAtCreation.tailSet(key).iterator();
+      snapshotIt = snapshotAtCreation.tailSet(key).iterator();
+      kvsetItRow = null;
+      snapshotItRow = null;
+
+      return seekInSubLists(key);
+    }
+
+
+    /**
+     * (Re)initialize the iterators after a seek or a reseek.
+     */
+    private synchronized boolean seekInSubLists(KeyValue key){
+      kvsetNextRow = getNext(kvsetIt);
+      snapshotNextRow = getNext(snapshotIt);
+
+      // Calculate the next value
+      theNext = getLowest(kvsetNextRow, snapshotNextRow);
+
+      // has data
+      return (theNext != null);
+    }
+
+
+    /**
+     * Move forward on the sub-lists set previously by seek.
+     * @param key seek value (should be non-null)
+     * @return true if there is at least one KV to read, false otherwise
+     */
+    @Override
+    public synchronized boolean reseek(KeyValue key) {
+      /*
+      See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation.
+      This code is executed concurrently with flush and puts, without locks.
+      Two points must be known when working on this code:
+      1) It's not possible to use the 'kvTail' and 'snapshot'
+       variables, as they are modified during a flush.
+      2) The ideal implementation for performance would use the sub skip list
+       implicitly pointed by the iterators 'kvsetIt' and
+       'snapshotIt'. Unfortunately the Java API does not offer a method to
+       get it. So we remember the last keys we iterated to and restore
+       the reseeked set to at least that point.
+       */
+
+      kvsetIt = kvsetAtCreation.tailSet(getHighest(key, kvsetItRow)).iterator();
+      snapshotIt = snapshotAtCreation.tailSet(getHighest(key, snapshotItRow)).iterator();
+
+      return seekInSubLists(key);
+    }
+
+
+    @Override
+    public synchronized KeyValue peek() {
+      //DebugPrint.println(" MS@" + hashCode() + " peek = " + getLowest());
+      return theNext;
+    }
+
+    @Override
+    public synchronized KeyValue next() {
+      if (theNext == null) {
+          return null;
+      }
+
+      final KeyValue ret = theNext;
+
+      // Advance one of the iterators
+      if (theNext == kvsetNextRow) {
+        kvsetNextRow = getNext(kvsetIt);
+      } else {
+        snapshotNextRow = getNext(snapshotIt);
+      }
+
+      // Calculate the next value
+      theNext = getLowest(kvsetNextRow, snapshotNextRow);
+
+      //long readpoint = ReadWriteConsistencyControl.getThreadReadPoint();
+      //DebugPrint.println(" MS@" + hashCode() + " next: " + theNext + " next_next: " +
+      //    getLowest() + " threadpoint=" + readpoint);
+      return ret;
+    }
+
+    /*
+     * Returns the lower of the two key values, or null if they are both null.
+     * This uses comparator.compare() to compare the KeyValue using the memstore
+     * comparator.
+     */
+    private KeyValue getLowest(KeyValue first, KeyValue second) {
+      if (first == null && second == null) {
+        return null;
+      }
+      if (first != null && second != null) {
+        int compare = comparator.compare(first, second);
+        return (compare <= 0 ? first : second);
+      }
+      return (first != null ? first : second);
+    }
+
+    /*
+     * Returns the higher of the two key values, or null if they are both null.
+     * This uses comparator.compare() to compare the KeyValue using the memstore
+     * comparator.
+     */
+    private KeyValue getHighest(KeyValue first, KeyValue second) {
+      if (first == null && second == null) {
+        return null;
+      }
+      if (first != null && second != null) {
+        int compare = comparator.compare(first, second);
+        return (compare > 0 ? first : second);
+      }
+      return (first != null ? first : second);
+    }
+
+    public synchronized void close() {
+      this.kvsetNextRow = null;
+      this.snapshotNextRow = null;
+
+      this.kvsetIt = null;
+      this.snapshotIt = null;
+
+      this.kvsetItRow = null;
+      this.snapshotItRow = null;
+    }
+
+    /**
+     * MemStoreScanner returns max value as sequence id because it will
+     * always have the latest data among all files.
+     */
+    @Override
+    public long getSequenceID() {
+      return Long.MAX_VALUE;
+    }
+
+    @Override
+    public boolean shouldUseScanner(Scan scan, SortedSet columns,
+        long oldestUnexpiredTS) {
+      return shouldSeek(scan, oldestUnexpiredTS);
+    }
+  }
+
+  public final static long FIXED_OVERHEAD = ClassSize.align(
+      ClassSize.OBJECT + (11 * ClassSize.REFERENCE) + Bytes.SIZEOF_LONG);
+
+  public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
+      ClassSize.REENTRANT_LOCK + ClassSize.ATOMIC_LONG +
+      ClassSize.COPYONWRITE_ARRAYSET + ClassSize.COPYONWRITE_ARRAYLIST +
+      (2 * ClassSize.CONCURRENT_SKIPLISTMAP));
+
+  /** Used for readability when we don't store memstore timestamp in HFile */
+  public static final boolean NO_PERSISTENT_TS = false;
+
+  /*
+   * Calculate how the MemStore size has changed.  Includes overhead of the
+   * backing Map.
+   * @param kv
+   * @param notpresent True if the kv was NOT present in the set.
+   * @return Size
+   */
+  long heapSizeChange(final KeyValue kv, final boolean notpresent) {
+    return notpresent ?
+        ClassSize.align(ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + kv.heapSize()):
+        0;
+  }
+
+  /**
+   * Get the entire heap usage for this MemStore not including keys in the
+   * snapshot.
+   */
+  @Override
+  public long heapSize() {
+    return size.get();
+  }
+
+  /**
+   * Get the heap usage of KVs in this MemStore.
+   */
+  public long keySize() {
+    return heapSize() - DEEP_OVERHEAD;
+  }
+
+  /**
+   * Code to help figure if our approximation of object heap sizes is close
+   * enough.  See hbase-900.  Fills memstores then waits so user can heap
+   * dump and bring up resultant hprof in something like jprofiler which
+   * allows you get 'deep size' on objects.
+   * @param args main args
+   */
+  public static void main(String [] args) {
+    RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
+    LOG.info("vmName=" + runtime.getVmName() + ", vmVendor=" +
+      runtime.getVmVendor() + ", vmVersion=" + runtime.getVmVersion());
+    LOG.info("vmInputArguments=" + runtime.getInputArguments());
+    MemStore memstore1 = new MemStore();
+    // TODO: x32 vs x64
+    long size = 0;
+    final int count = 10000;
+    byte [] fam = Bytes.toBytes("col");
+    byte [] qf = Bytes.toBytes("umn");
+    byte [] empty = new byte[0];
+    for (int i = 0; i < count; i++) {
+      // Give each its own ts
+      size += memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty));
+    }
+    LOG.info("memstore1 estimated size=" + size);
+    for (int i = 0; i < count; i++) {
+      size += memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty));
+    }
+    LOG.info("memstore1 estimated size (2nd loading of same data)=" + size);
+    // Make a variably sized memstore.
+    MemStore memstore2 = new MemStore();
+    for (int i = 0; i < count; i++) {
+      size += memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i,
+        new byte[i]));
+    }
+    LOG.info("memstore2 estimated size=" + size);
+    final int seconds = 30;
+    LOG.info("Waiting " + seconds + " seconds while heap dump is taken");
+    for (int i = 0; i < seconds; i++) {
+      // Thread.sleep(1000);
+    }
+    LOG.info("Exiting.");
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
new file mode 100644
index 0000000..7f704d8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -0,0 +1,610 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.ConcurrentModificationException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.DelayQueue;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.DroppedSnapshotException;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.HasThread;
+import org.apache.hadoop.util.StringUtils;
+import org.cliffc.high_scale_lib.Counter;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Thread that flushes cache on request
+ *
+ * NOTE: This class extends Thread rather than Chore because the sleep time
+ * can be interrupted when there is something to do, rather than the Chore
+ * sleep time which is invariant.
+ *
+ * @see FlushRequester
+ */
+class MemStoreFlusher extends HasThread implements FlushRequester {
+  static final Log LOG = LogFactory.getLog(MemStoreFlusher.class);
+  // These two data members go together.  Any entry in the one must have
+  // a corresponding entry in the other.
+  private final BlockingQueue flushQueue =
+    new DelayQueue();
+  private final Map regionsInQueue =
+    new HashMap();
+  private AtomicBoolean wakeupPending = new AtomicBoolean();
+
+  private final long threadWakeFrequency;
+  private final HRegionServer server;
+  private final ReentrantLock lock = new ReentrantLock();
+  private final Condition flushOccurred = lock.newCondition();
+
+  protected final long globalMemStoreLimit;
+  protected final long globalMemStoreLimitLowMark;
+
+  private static final float DEFAULT_UPPER = 0.4f;
+  private static final float DEFAULT_LOWER = 0.35f;
+  private static final String UPPER_KEY =
+    "hbase.regionserver.global.memstore.upperLimit";
+  private static final String LOWER_KEY =
+    "hbase.regionserver.global.memstore.lowerLimit";
+  
+  private long blockingWaitTime;
+  private final Counter updatesBlockedMsHighWater = new Counter();
+
+  /**
+   * @param conf
+   * @param server
+   */
+  public MemStoreFlusher(final Configuration conf,
+      final HRegionServer server) {
+    super();
+    this.server = server;
+    this.threadWakeFrequency =
+      conf.getLong(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
+    long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
+    this.globalMemStoreLimit = globalMemStoreLimit(max, DEFAULT_UPPER,
+      UPPER_KEY, conf);
+    long lower = globalMemStoreLimit(max, DEFAULT_LOWER, LOWER_KEY, conf);
+    if (lower > this.globalMemStoreLimit) {
+      lower = this.globalMemStoreLimit;
+      LOG.info("Setting globalMemStoreLimitLowMark == globalMemStoreLimit " +
+        "because supplied " + LOWER_KEY + " was > " + UPPER_KEY);
+    }
+    this.globalMemStoreLimitLowMark = lower;
+    
+    this.blockingWaitTime = conf.getInt("hbase.hstore.blockingWaitTime",
+      90000);
+    LOG.info("globalMemStoreLimit=" +
+      StringUtils.humanReadableInt(this.globalMemStoreLimit) +
+      ", globalMemStoreLimitLowMark=" +
+      StringUtils.humanReadableInt(this.globalMemStoreLimitLowMark) +
+      ", maxHeap=" + StringUtils.humanReadableInt(max));
+  }
+
+  /**
+   * Calculate size using passed key for configured
+   * percentage of max.
+   * @param max
+   * @param defaultLimit
+   * @param key
+   * @param c
+   * @return Limit.
+   */
+  static long globalMemStoreLimit(final long max,
+     final float defaultLimit, final String key, final Configuration c) {
+    float limit = c.getFloat(key, defaultLimit);
+    return getMemStoreLimit(max, limit, defaultLimit);
+  }
+
+  static long getMemStoreLimit(final long max, final float limit,
+      final float defaultLimit) {
+    float effectiveLimit = limit;
+    if (limit >= 0.9f || limit < 0.1f) {
+      LOG.warn("Setting global memstore limit to default of " + defaultLimit +
+        " because supplied value outside allowed range of 0.1 -> 0.9");
+      effectiveLimit = defaultLimit;
+    }
+    return (long)(max * effectiveLimit);
+  }
+
+  public Counter getUpdatesBlockedMsHighWater() {
+    return this.updatesBlockedMsHighWater;
+  }
+
+  /**
+   * The memstore across all regions has exceeded the low water mark. Pick
+   * one region to flush and flush it synchronously (this is called from the
+   * flush thread)
+   * @return true if successful
+   */
+  private boolean flushOneForGlobalPressure() {
+    SortedMap regionsBySize =
+        server.getCopyOfOnlineRegionsSortedBySize();
+
+    Set excludedRegions = new HashSet();
+
+    boolean flushedOne = false;
+    while (!flushedOne) {
+      // Find the biggest region that doesn't have too many storefiles
+      // (might be null!)
+      HRegion bestFlushableRegion = getBiggestMemstoreRegion(
+          regionsBySize, excludedRegions, true);
+      // Find the biggest region, total, even if it might have too many flushes.
+      HRegion bestAnyRegion = getBiggestMemstoreRegion(
+          regionsBySize, excludedRegions, false);
+
+      if (bestAnyRegion == null) {
+        LOG.error("Above memory mark but there are no flushable regions!");
+        return false;
+      }
+
+      HRegion regionToFlush;
+      if (bestFlushableRegion != null &&
+          bestAnyRegion.memstoreSize.get() > 2 * bestFlushableRegion.memstoreSize.get()) {
+        // Even if it's not supposed to be flushed, pick a region if it's more than twice
+        // as big as the best flushable one - otherwise when we're under pressure we make
+        // lots of little flushes and cause lots of compactions, etc, which just makes
+        // life worse!
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Under global heap pressure: " +
+            "Region " + bestAnyRegion.getRegionNameAsString() + " has too many " +
+            "store files, but is " +
+            StringUtils.humanReadableInt(bestAnyRegion.memstoreSize.get()) +
+            " vs best flushable region's " +
+            StringUtils.humanReadableInt(bestFlushableRegion.memstoreSize.get()) +
+            ". Choosing the bigger.");
+        }
+        regionToFlush = bestAnyRegion;
+      } else {
+        if (bestFlushableRegion == null) {
+          regionToFlush = bestAnyRegion;
+        } else {
+          regionToFlush = bestFlushableRegion;
+        }
+      }
+
+      Preconditions.checkState(regionToFlush.memstoreSize.get() > 0);
+
+      LOG.info("Flush of region " + regionToFlush + " due to global heap pressure");
+      flushedOne = flushRegion(regionToFlush, true);
+      if (!flushedOne) {
+        LOG.info("Excluding unflushable region " + regionToFlush +
+          " - trying to find a different region to flush.");
+        excludedRegions.add(regionToFlush);
+      }
+    }
+    return true;
+  }
+
+  @Override
+  public void run() {
+    while (!this.server.isStopped()) {
+      FlushQueueEntry fqe = null;
+      try {
+        wakeupPending.set(false); // allow someone to wake us up again
+        fqe = flushQueue.poll(threadWakeFrequency, TimeUnit.MILLISECONDS);
+        if (fqe == null || fqe instanceof WakeupFlushThread) {
+          if (isAboveLowWaterMark()) {
+            LOG.debug("Flush thread woke up because memory above low water=" +
+              StringUtils.humanReadableInt(this.globalMemStoreLimitLowMark));
+            if (!flushOneForGlobalPressure()) {
+              // Wasn't able to flush any region, but we're above low water mark
+              // This is unlikely to happen, but might happen when closing the
+              // entire server - another thread is flushing regions. We'll just
+              // sleep a little bit to avoid spinning, and then pretend that
+              // we flushed one, so anyone blocked will check again
+              lock.lock();
+              try {
+                Thread.sleep(1000);
+                flushOccurred.signalAll();
+              } finally {
+                lock.unlock();
+              }
+            }
+            // Enqueue another one of these tokens so we'll wake up again
+            wakeupFlushThread();
+          }
+          continue;
+        }
+        FlushRegionEntry fre = (FlushRegionEntry)fqe;
+        if (!flushRegion(fre)) {
+          break;
+        }
+      } catch (InterruptedException ex) {
+        continue;
+      } catch (ConcurrentModificationException ex) {
+        continue;
+      } catch (Exception ex) {
+        LOG.error("Cache flusher failed for entry " + fqe, ex);
+        if (!server.checkFileSystem()) {
+          break;
+        }
+      }
+    }
+    this.regionsInQueue.clear();
+    this.flushQueue.clear();
+
+    // Signal anyone waiting, so they see the close flag
+    lock.lock();
+    try {
+      flushOccurred.signalAll();
+    } finally {
+      lock.unlock();
+    }
+    LOG.info(getName() + " exiting");
+  }
+
+  private void wakeupFlushThread() {
+    if (wakeupPending.compareAndSet(false, true)) {
+      flushQueue.add(new WakeupFlushThread());
+    }
+  }
+
+  private HRegion getBiggestMemstoreRegion(
+      SortedMap regionsBySize,
+      Set excludedRegions,
+      boolean checkStoreFileCount) {
+    synchronized (regionsInQueue) {
+      for (HRegion region : regionsBySize.values()) {
+        if (excludedRegions.contains(region)) {
+          continue;
+        }
+
+        if (checkStoreFileCount && isTooManyStoreFiles(region)) {
+          continue;
+        }
+        return region;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Return true if global memory usage is above the high watermark
+   */
+  private boolean isAboveHighWaterMark() {
+    return server.getRegionServerAccounting().
+      getGlobalMemstoreSize() >= globalMemStoreLimit;
+  }
+
+  /**
+   * Return true if we're above the high watermark
+   */
+  private boolean isAboveLowWaterMark() {
+    return server.getRegionServerAccounting().
+      getGlobalMemstoreSize() >= globalMemStoreLimitLowMark;
+  }
+
+  public void requestFlush(HRegion r) {
+    synchronized (regionsInQueue) {
+      if (!regionsInQueue.containsKey(r)) {
+        // This entry has no delay so it will be added at the top of the flush
+        // queue.  It'll come out near immediately.
+        FlushRegionEntry fqe = new FlushRegionEntry(r);
+        this.regionsInQueue.put(r, fqe);
+        this.flushQueue.add(fqe);
+      }
+    }
+  }
+
+  public void requestDelayedFlush(HRegion r, long delay) {
+    synchronized (regionsInQueue) {
+      if (!regionsInQueue.containsKey(r)) {
+        // This entry has some delay
+        FlushRegionEntry fqe = new FlushRegionEntry(r);
+        fqe.requeue(delay);
+        this.regionsInQueue.put(r, fqe);
+        this.flushQueue.add(fqe);
+      }
+    }
+  }
+
+  public int getFlushQueueSize() {
+    return flushQueue.size();
+  }
+
+  /**
+   * Only interrupt once it's done with a run through the work loop.
+   */
+  void interruptIfNecessary() {
+    lock.lock();
+    try {
+      this.interrupt();
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  /*
+   * A flushRegion that checks store file count.  If too many, puts the flush
+   * on delay queue to retry later.
+   * @param fqe
+   * @return true if the region was successfully flushed, false otherwise. If
+   * false, there will be accompanying log messages explaining why the log was
+   * not flushed.
+   */
+  private boolean flushRegion(final FlushRegionEntry fqe) {
+    HRegion region = fqe.region;
+    if (!fqe.region.getRegionInfo().isMetaRegion() &&
+        isTooManyStoreFiles(region)) {
+      if (fqe.isMaximumWait(this.blockingWaitTime)) {
+        LOG.info("Waited " + (System.currentTimeMillis() - fqe.createTime) +
+          "ms on a compaction to clean up 'too many store files'; waited " +
+          "long enough... proceeding with flush of " +
+          region.getRegionNameAsString());
+      } else {
+        // If this is first time we've been put off, then emit a log message.
+        if (fqe.getRequeueCount() <= 0) {
+          // Note: We don't impose blockingStoreFiles constraint on meta regions
+          LOG.warn("Region " + region.getRegionNameAsString() + " has too many " +
+            "store files; delaying flush up to " + this.blockingWaitTime + "ms");
+          if (!this.server.compactSplitThread.requestSplit(region)) {
+            try {
+              this.server.compactSplitThread.requestCompaction(region, getName());
+            }  catch (IOException e) {
+              LOG.error("Cache flush failed" +
+                (region != null ? (" for region " + Bytes.toStringBinary(region.getRegionName())) : ""),
+                RemoteExceptionHandler.checkIOException(e));
+            }
+          }
+        }
+
+        // Put back on the queue.  Have it come back out of the queue
+        // after a delay of this.blockingWaitTime / 100 ms.
+        this.flushQueue.add(fqe.requeue(this.blockingWaitTime / 100));
+        // Tell a lie, it's not flushed but it's ok
+        return true;
+      }
+    }
+    return flushRegion(region, false);
+  }
+
+  /*
+   * Flush a region.
+   * @param region Region to flush.
+   * @param emergencyFlush Set if we are being force flushed. If true the region
+   * needs to be removed from the flush queue. If false, when we were called
+   * from the main flusher run loop and we got the entry to flush by calling
+   * poll on the flush queue (which removed it).
+   *
+   * @return true if the region was successfully flushed, false otherwise. If
+   * false, there will be accompanying log messages explaining why the log was
+   * not flushed.
+   */
+  private boolean flushRegion(final HRegion region, final boolean emergencyFlush) {
+    synchronized (this.regionsInQueue) {
+      FlushRegionEntry fqe = this.regionsInQueue.remove(region);
+      if (fqe != null && emergencyFlush) {
+        // Need to remove from region from delay queue.  When NOT an
+        // emergencyFlush, then item was removed via a flushQueue.poll.
+        flushQueue.remove(fqe);
+     }
+     lock.lock();
+    }
+    try {
+      boolean shouldCompact = region.flushcache();
+      // We just want to check the size
+      boolean shouldSplit = region.checkSplit() != null;
+      if (shouldSplit) {
+        this.server.compactSplitThread.requestSplit(region);
+      } else if (shouldCompact) {
+        server.compactSplitThread.requestCompaction(region, getName());
+      }
+
+      server.getMetrics().addFlush(region.getRecentFlushInfo());
+    } catch (DroppedSnapshotException ex) {
+      // Cache flush can fail in a few places. If it fails in a critical
+      // section, we get a DroppedSnapshotException and a replay of hlog
+      // is required. Currently the only way to do this is a restart of
+      // the server. Abort because hdfs is probably bad (HBASE-644 is a case
+      // where hdfs was bad but passed the hdfs check).
+      server.abort("Replay of HLog required. Forcing server shutdown", ex);
+      return false;
+    } catch (IOException ex) {
+      LOG.error("Cache flush failed" +
+        (region != null ? (" for region " + Bytes.toStringBinary(region.getRegionName())) : ""),
+        RemoteExceptionHandler.checkIOException(ex));
+      if (!server.checkFileSystem()) {
+        return false;
+      }
+    } finally {
+      flushOccurred.signalAll();
+      lock.unlock();
+    }
+    return true;
+  }
+
+  private boolean isTooManyStoreFiles(HRegion region) {
+    for (Store hstore: region.stores.values()) {
+      if (hstore.hasTooManyStoreFiles()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Check if the regionserver's memstore memory usage is greater than the
+   * limit. If so, flush regions with the biggest memstores until we're down
+   * to the lower limit. This method blocks callers until we're down to a safe
+   * amount of memstore consumption.
+   */
+  public void reclaimMemStoreMemory() {
+    if (isAboveHighWaterMark()) {
+      lock.lock();
+      try {
+        boolean blocked = false;
+        long startTime = 0;
+        while (isAboveHighWaterMark() && !server.isStopped()) {
+          if(!blocked){
+            startTime = EnvironmentEdgeManager.currentTimeMillis();
+            LOG.info("Blocking updates on " + server.toString() +
+            ": the global memstore size " +
+            StringUtils.humanReadableInt(server.getRegionServerAccounting().getGlobalMemstoreSize()) +
+            " is >= than blocking " +
+            StringUtils.humanReadableInt(globalMemStoreLimit) + " size");
+          }
+          blocked = true;
+          wakeupFlushThread();
+          try {
+            // we should be able to wait forever, but we've seen a bug where
+            // we miss a notify, so put a 5 second bound on it at least.
+            flushOccurred.await(5, TimeUnit.SECONDS);
+          } catch (InterruptedException ie) {
+            Thread.currentThread().interrupt();
+          }
+        }
+        if(blocked){
+          final long totalTime = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+          if(totalTime > 0){
+            this.updatesBlockedMsHighWater.add(totalTime);
+          }
+          LOG.info("Unblocking updates for server " + server.toString());
+        }
+      } finally {
+        lock.unlock();
+      }
+    } else if (isAboveLowWaterMark()) {
+      wakeupFlushThread();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "flush_queue="
+        + flushQueue.size();
+  }
+  
+  public String dumpQueue() {
+    StringBuilder queueList = new StringBuilder();
+    queueList.append("Flush Queue Queue dump:\n");
+    queueList.append("  Flush Queue:\n");
+    java.util.Iterator it = flushQueue.iterator();
+    
+    while(it.hasNext()){
+      queueList.append("    "+it.next().toString());
+      queueList.append("\n");
+    }
+    
+    return queueList.toString();
+  }
+  
+  interface FlushQueueEntry extends Delayed {}
+
+  /**
+   * Token to insert into the flush queue that ensures that the flusher does not sleep
+   */
+  static class WakeupFlushThread implements FlushQueueEntry {
+    @Override
+    public long getDelay(TimeUnit unit) {
+      return 0;
+    }
+
+    @Override
+    public int compareTo(Delayed o) {
+      return -1;
+    }
+  }
+
+  /**
+   * Datastructure used in the flush queue.  Holds region and retry count.
+   * Keeps tabs on how old this object is.  Implements {@link Delayed}.  On
+   * construction, the delay is zero. When added to a delay queue, we'll come
+   * out near immediately.  Call {@link #requeue(long)} passing delay in
+   * milliseconds before readding to delay queue if you want it to stay there
+   * a while.
+   */
+  static class FlushRegionEntry implements FlushQueueEntry {
+    private final HRegion region;
+
+    private final long createTime;
+    private long whenToExpire;
+    private int requeueCount = 0;
+
+    FlushRegionEntry(final HRegion r) {
+      this.region = r;
+      this.createTime = System.currentTimeMillis();
+      this.whenToExpire = this.createTime;
+    }
+
+    /**
+     * @param maximumWait
+     * @return True if we have been delayed > maximumWait milliseconds.
+     */
+    public boolean isMaximumWait(final long maximumWait) {
+      return (System.currentTimeMillis() - this.createTime) > maximumWait;
+    }
+
+    /**
+     * @return Count of times {@link #resetDelay()} was called; i.e this is
+     * number of times we've been requeued.
+     */
+    public int getRequeueCount() {
+      return this.requeueCount;
+    }
+
+    /**
+     * @param when When to expire, when to come up out of the queue.
+     * Specify in milliseconds.  This method adds System.currentTimeMillis()
+     * to whatever you pass.
+     * @return This.
+     */
+    public FlushRegionEntry requeue(final long when) {
+      this.whenToExpire = System.currentTimeMillis() + when;
+      this.requeueCount++;
+      return this;
+    }
+
+    @Override
+    public long getDelay(TimeUnit unit) {
+      return unit.convert(this.whenToExpire - System.currentTimeMillis(),
+          TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public int compareTo(Delayed other) {
+      return Long.valueOf(getDelay(TimeUnit.MILLISECONDS) -
+        other.getDelay(TimeUnit.MILLISECONDS)).intValue();
+    }
+
+    @Override
+    public String toString() {
+      return "[flush region " + Bytes.toStringBinary(region.getRegionName()) + "]";
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
new file mode 100644
index 0000000..cbb76e8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
@@ -0,0 +1,273 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.conf.Configuration;
+import com.google.common.base.Preconditions;
+
+/**
+ * A memstore-local allocation buffer.
+ * 
                
+ * The MemStoreLAB is basically a bump-the-pointer allocator that allocates
+ * big (2MB) byte[] chunks from and then doles it out to threads that request
+ * slices into the array.
+ * 
                
+ * The purpose of this class is to combat heap fragmentation in the
+ * regionserver. By ensuring that all KeyValues in a given memstore refer
+ * only to large chunks of contiguous memory, we ensure that large blocks
+ * get freed up when the memstore is flushed.
+ * 
                
+ * Without the MSLAB, the byte array allocated during insertion end up
+ * interleaved throughout the heap, and the old generation gets progressively
+ * more fragmented until a stop-the-world compacting collection occurs.
+ * 
                
+ * TODO: we should probably benchmark whether word-aligning the allocations
+ * would provide a performance improvement - probably would speed up the
+ * Bytes.toLong/Bytes.toInt calls in KeyValue, but some of those are cached
+ * anyway
+ */
+public class MemStoreLAB {
+  private AtomicReference curChunk = new AtomicReference();
+
+  final static String CHUNK_SIZE_KEY = "hbase.hregion.memstore.mslab.chunksize";
+  final static int CHUNK_SIZE_DEFAULT = 2048 * 1024;
+  final int chunkSize;
+
+  final static String MAX_ALLOC_KEY = "hbase.hregion.memstore.mslab.max.allocation";
+  final static int MAX_ALLOC_DEFAULT = 256  * 1024; // allocs bigger than this don't go through allocator
+  final int maxAlloc;
+
+  public MemStoreLAB() {
+    this(new Configuration());
+  }
+
+  public MemStoreLAB(Configuration conf) {
+    chunkSize = conf.getInt(CHUNK_SIZE_KEY, CHUNK_SIZE_DEFAULT);
+    maxAlloc = conf.getInt(MAX_ALLOC_KEY, MAX_ALLOC_DEFAULT);
+
+    // if we don't exclude allocations >CHUNK_SIZE, we'd infiniteloop on one!
+    Preconditions.checkArgument(
+      maxAlloc <= chunkSize,
+      MAX_ALLOC_KEY + " must be less than " + CHUNK_SIZE_KEY);
+  }
+
+  /**
+   * Allocate a slice of the given length.
+   *
+   * If the size is larger than the maximum size specified for this
+   * allocator, returns null.
+   */
+  public Allocation allocateBytes(int size) {
+    Preconditions.checkArgument(size >= 0, "negative size");
+
+    // Callers should satisfy large allocations directly from JVM since they
+    // don't cause fragmentation as badly.
+    if (size > maxAlloc) {
+      return null;
+    }
+
+    while (true) {
+      Chunk c = getOrMakeChunk();
+
+      // Try to allocate from this chunk
+      int allocOffset = c.alloc(size);
+      if (allocOffset != -1) {
+        // We succeeded - this is the common case - small alloc
+        // from a big buffer
+        return new Allocation(c.data, allocOffset);
+      }
+
+      // not enough space!
+      // try to retire this chunk
+      tryRetireChunk(c);
+    }
+  }
+
+  /**
+   * Try to retire the current chunk if it is still
+   * c. Postcondition is that curChunk.get()
+   * != c
+   */
+  private void tryRetireChunk(Chunk c) {
+    @SuppressWarnings("unused")
+    boolean weRetiredIt = curChunk.compareAndSet(c, null);
+    // If the CAS succeeds, that means that we won the race
+    // to retire the chunk. We could use this opportunity to
+    // update metrics on external fragmentation.
+    //
+    // If the CAS fails, that means that someone else already
+    // retired the chunk for us.
+  }
+
+  /**
+   * Get the current chunk, or, if there is no current chunk,
+   * allocate a new one from the JVM.
+   */
+  private Chunk getOrMakeChunk() {
+    while (true) {
+      // Try to get the chunk
+      Chunk c = curChunk.get();
+      if (c != null) {
+        return c;
+      }
+
+      // No current chunk, so we want to allocate one. We race
+      // against other allocators to CAS in an uninitialized chunk
+      // (which is cheap to allocate)
+      c = new Chunk(chunkSize);
+      if (curChunk.compareAndSet(null, c)) {
+        // we won race - now we need to actually do the expensive
+        // allocation step
+        c.init();
+        return c;
+      }
+      // someone else won race - that's fine, we'll try to grab theirs
+      // in the next iteration of the loop.
+    }
+  }
+
+  /**
+   * A chunk of memory out of which allocations are sliced.
+   */
+  private static class Chunk {
+    /** Actual underlying data */
+    private byte[] data;
+
+    private static final int UNINITIALIZED = -1;
+    private static final int OOM = -2;
+    /**
+     * Offset for the next allocation, or the sentinel value -1
+     * which implies that the chunk is still uninitialized.
+     * */
+    private AtomicInteger nextFreeOffset = new AtomicInteger(UNINITIALIZED);
+
+    /** Total number of allocations satisfied from this buffer */
+    private AtomicInteger allocCount = new AtomicInteger();
+
+    /** Size of chunk in bytes */
+    private final int size;
+
+    /**
+     * Create an uninitialized chunk. Note that memory is not allocated yet, so
+     * this is cheap.
+     * @param size in bytes
+     */
+    private Chunk(int size) {
+      this.size = size;
+    }
+
+    /**
+     * Actually claim the memory for this chunk. This should only be called from
+     * the thread that constructed the chunk. It is thread-safe against other
+     * threads calling alloc(), who will block until the allocation is complete.
+     */
+    public void init() {
+      assert nextFreeOffset.get() == UNINITIALIZED;
+      try {
+        data = new byte[size];
+      } catch (OutOfMemoryError e) {
+        boolean failInit = nextFreeOffset.compareAndSet(UNINITIALIZED, OOM);
+        assert failInit; // should be true.
+        throw e;
+      }
+      // Mark that it's ready for use
+      boolean initted = nextFreeOffset.compareAndSet(
+          UNINITIALIZED, 0);
+      // We should always succeed the above CAS since only one thread
+      // calls init()!
+      Preconditions.checkState(initted,
+          "Multiple threads tried to init same chunk");
+    }
+
+    /**
+     * Try to allocate size bytes from the chunk.
+     * @return the offset of the successful allocation, or -1 to indicate not-enough-space
+     */
+    public int alloc(int size) {
+      while (true) {
+        int oldOffset = nextFreeOffset.get();
+        if (oldOffset == UNINITIALIZED) {
+          // The chunk doesn't have its data allocated yet.
+          // Since we found this in curChunk, we know that whoever
+          // CAS-ed it there is allocating it right now. So spin-loop
+          // shouldn't spin long!
+          Thread.yield();
+          continue;
+        }
+        if (oldOffset == OOM) {
+          // doh we ran out of ram. return -1 to chuck this away.
+          return -1;
+        }
+
+        if (oldOffset + size > data.length) {
+          return -1; // alloc doesn't fit
+        }
+
+        // Try to atomically claim this chunk
+        if (nextFreeOffset.compareAndSet(oldOffset, oldOffset + size)) {
+          // we got the alloc
+          allocCount.incrementAndGet();
+          return oldOffset;
+        }
+        // we raced and lost alloc, try again
+      }
+    }
+
+    @Override
+    public String toString() {
+      return "Chunk@" + System.identityHashCode(this) +
+        " allocs=" + allocCount.get() + "waste=" +
+        (data.length - nextFreeOffset.get());
+    }
+  }
+
+  /**
+   * The result of a single allocation. Contains the chunk that the
+   * allocation points into, and the offset in this array where the
+   * slice begins.
+   */
+  public static class Allocation {
+    private final byte[] data;
+    private final int offset;
+
+    private Allocation(byte[] data, int off) {
+      this.data = data;
+      this.offset = off;
+    }
+
+    @Override
+    public String toString() {
+      return "Allocation(data=" + data +
+        " with capacity=" + data.length +
+        ", off=" + offset + ")";
+    }
+
+    byte[] getData() {
+      return data;
+    }
+
+    int getOffset() {
+      return offset;
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MetaLogRoller.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MetaLogRoller.java
new file mode 100644
index 0000000..d1ebd59
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MetaLogRoller.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+
+@InterfaceAudience.Private
+class MetaLogRoller extends LogRoller {
+  public MetaLogRoller(Server server, RegionServerServices services) {
+    super(server, services);
+  }
+  @Override
+  protected HLog getWAL() throws IOException {
+    //The argument to getWAL below could either be HRegionInfo.FIRST_META_REGIONINFO or
+    //HRegionInfo.ROOT_REGIONINFO. Both these share the same WAL.
+    return services.getWAL(HRegionInfo.FIRST_META_REGIONINFO);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java
new file mode 100644
index 0000000..ce96aa7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+
+/**
+ * Wraps together the mutations which are applied as a batch to the region and their operation
+ * status and WALEdits. 
+ * @see RegionObserver#preBatchMutate(ObserverContext, MiniBatchOperationInProgress)
+ * @see RegionObserver#postBatchMutate(ObserverContext, MiniBatchOperationInProgress)
+ * @param  Pair pair of Mutations and associated rowlock ids.
+ */
+public class MiniBatchOperationInProgress {
+  private final T[] operations;
+  private final OperationStatus[] retCodeDetails;
+  private final WALEdit[] walEditsFromCoprocessors;
+  private final int firstIndex;
+  private final int lastIndexExclusive;
+
+  public MiniBatchOperationInProgress(T[] operations, OperationStatus[] retCodeDetails,
+      WALEdit[] walEditsFromCoprocessors, int firstIndex, int lastIndexExclusive) {
+    this.operations = operations;
+    this.retCodeDetails = retCodeDetails;
+    this.walEditsFromCoprocessors = walEditsFromCoprocessors;
+    this.firstIndex = firstIndex;
+    this.lastIndexExclusive = lastIndexExclusive;
+  }
+
+  /**
+   * @return The number of operations(Mutations) involved in this batch.
+   */
+  public int size() {
+    return this.lastIndexExclusive - this.firstIndex;
+  }
+
+  /**
+   * @param index
+   * @return The operation(Mutation) at the specified position.
+   */
+  public T getOperation(int index) {
+    return operations[getAbsoluteIndex(index)];
+  }
+
+  /**
+   * Sets the status code for the operation(Mutation) at the specified position.
+   * By setting this status, {@link RegionObserver} can make HRegion to skip Mutations.
+   * @param index
+   * @param opStatus
+   */
+  public void setOperationStatus(int index, OperationStatus opStatus) {
+    this.retCodeDetails[getAbsoluteIndex(index)] = opStatus;
+  }
+
+  /**
+   * @param index
+   * @return Gets the status code for the operation(Mutation) at the specified position.
+   */
+  public OperationStatus getOperationStatus(int index) {
+    return this.retCodeDetails[getAbsoluteIndex(index)];
+  }
+
+  /**
+   * Sets the walEdit for the operation(Mutation) at the specified position.
+   * @param index
+   * @param walEdit
+   */
+  public void setWalEdit(int index, WALEdit walEdit) {
+    this.walEditsFromCoprocessors[getAbsoluteIndex(index)] = walEdit;
+  }
+
+  /**
+   * @param index
+   * @return Gets the walEdit for the operation(Mutation) at the specified position.
+   */
+  public WALEdit getWalEdit(int index) {
+    return this.walEditsFromCoprocessors[getAbsoluteIndex(index)];
+  }
+
+  private int getAbsoluteIndex(int index) {
+    if (index < 0 || this.firstIndex + index >= this.lastIndexExclusive) {
+      throw new ArrayIndexOutOfBoundsException(index);
+    }
+    return this.firstIndex + index;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
new file mode 100644
index 0000000..6b28f03
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
@@ -0,0 +1,213 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.LinkedList;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+
+/**
+ * Manages the read/write consistency within memstore. This provides
+ * an interface for readers to determine what entries to ignore, and
+ * a mechanism for writers to obtain new write numbers, then "commit"
+ * the new writes for readers to read (thus forming atomic transactions).
+ */
+public class MultiVersionConsistencyControl {
+  private volatile long memstoreRead = 0;
+  private volatile long memstoreWrite = 0;
+
+  private final Object readWaiters = new Object();
+
+  // This is the pending queue of writes.
+  private final LinkedList writeQueue =
+      new LinkedList();
+
+  private static final ThreadLocal perThreadReadPoint =
+      new ThreadLocal() {
+       @Override
+      protected
+       Long initialValue() {
+         return Long.MAX_VALUE;
+       }
+  };
+
+  /**
+   * Default constructor. Initializes the memstoreRead/Write points to 0.
+   */
+  public MultiVersionConsistencyControl() {
+    this.memstoreRead = this.memstoreWrite = 0;
+  }
+
+  /**
+   * Initializes the memstoreRead/Write points appropriately.
+   * @param startPoint
+   */
+  public void initialize(long startPoint) {
+    synchronized (writeQueue) {
+      if (this.memstoreWrite != this.memstoreRead) {
+        throw new RuntimeException("Already used this mvcc. Too late to initialize");
+      }
+
+      this.memstoreRead = this.memstoreWrite = startPoint;
+    }
+  }
+
+  /**
+   * Get this thread's read point. Used primarily by the memstore scanner to
+   * know which values to skip (ie: have not been completed/committed to
+   * memstore).
+   */
+  public static long getThreadReadPoint() {
+      return perThreadReadPoint.get();
+  }
+
+  /**
+   * Set the thread read point to the given value. The thread MVCC
+   * is used by the Memstore scanner so it knows which values to skip.
+   * Give it a value of 0 if you want everything.
+   */
+  public static void setThreadReadPoint(long readPoint) {
+    perThreadReadPoint.set(readPoint);
+  }
+
+  /**
+   * Set the thread MVCC read point to whatever the current read point is in
+   * this particular instance of MVCC.  Returns the new thread read point value.
+   */
+  public static long resetThreadReadPoint(MultiVersionConsistencyControl mvcc) {
+    perThreadReadPoint.set(mvcc.memstoreReadPoint());
+    return getThreadReadPoint();
+  }
+
+  /**
+   * Set the thread MVCC read point to 0 (include everything).
+   */
+  public static void resetThreadReadPoint() {
+    perThreadReadPoint.set(0L);
+  }
+
+  public WriteEntry beginMemstoreInsert() {
+    synchronized (writeQueue) {
+      long nextWriteNumber = ++memstoreWrite;
+      WriteEntry e = new WriteEntry(nextWriteNumber);
+      writeQueue.add(e);
+      return e;
+    }
+  }
+
+  public void completeMemstoreInsert(WriteEntry e) {
+    advanceMemstore(e);
+    waitForRead(e);
+  }
+
+  boolean advanceMemstore(WriteEntry e) {
+    synchronized (writeQueue) {
+      e.markCompleted();
+
+      long nextReadValue = -1;
+      boolean ranOnce=false;
+      while (!writeQueue.isEmpty()) {
+        ranOnce=true;
+        WriteEntry queueFirst = writeQueue.getFirst();
+
+        if (nextReadValue > 0) {
+          if (nextReadValue+1 != queueFirst.getWriteNumber()) {
+            throw new RuntimeException("invariant in completeMemstoreInsert violated, prev: "
+                + nextReadValue + " next: " + queueFirst.getWriteNumber());
+          }
+        }
+
+        if (queueFirst.isCompleted()) {
+          nextReadValue = queueFirst.getWriteNumber();
+          writeQueue.removeFirst();
+        } else {
+          break;
+        }
+      }
+
+      if (!ranOnce) {
+        throw new RuntimeException("never was a first");
+      }
+
+      if (nextReadValue > 0) {
+        synchronized (readWaiters) {
+          memstoreRead = nextReadValue;
+          readWaiters.notifyAll();
+        }
+      }
+      if (memstoreRead >= e.getWriteNumber()) {
+        return true;
+      }
+      return false;
+    }
+  }
+
+  /**
+   * Wait for the global readPoint to advance upto
+   * the specified transaction number.
+   */
+  public void waitForRead(WriteEntry e) {
+    boolean interrupted = false;
+    synchronized (readWaiters) {
+      while (memstoreRead < e.getWriteNumber()) {
+        try {
+          readWaiters.wait(0);
+        } catch (InterruptedException ie) {
+          // We were interrupted... finish the loop -- i.e. cleanup --and then
+          // on our way out, reset the interrupt flag.
+          interrupted = true;
+        }
+      }
+    }
+    if (interrupted) Thread.currentThread().interrupt();
+  }
+
+  public long memstoreReadPoint() {
+    return memstoreRead;
+  }
+
+
+  public static class WriteEntry {
+    private long writeNumber;
+    private boolean completed = false;
+    WriteEntry(long writeNumber) {
+      this.writeNumber = writeNumber;
+    }
+    void markCompleted() {
+      this.completed = true;
+    }
+    boolean isCompleted() {
+      return this.completed;
+    }
+    long getWriteNumber() {
+      return this.writeNumber;
+    }
+  }
+
+  public static final long FIXED_SIZE = ClassSize.align(
+      ClassSize.OBJECT +
+      2 * Bytes.SIZEOF_LONG +
+      2 * ClassSize.REFERENCE);
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/NoSuchColumnFamilyException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/NoSuchColumnFamilyException.java
new file mode 100644
index 0000000..4881fc0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/NoSuchColumnFamilyException.java
@@ -0,0 +1,41 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+
+/**
+ * Thrown if request for nonexistent column family.
+ */
+public class NoSuchColumnFamilyException extends DoNotRetryIOException {
+  private static final long serialVersionUID = -6569952730832331274L;
+
+  /** default constructor */
+  public NoSuchColumnFamilyException() {
+    super();
+  }
+
+  /**
+   * @param message exception message
+   */
+  public NoSuchColumnFamilyException(String message) {
+    super(message);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java
new file mode 100644
index 0000000..6534e2c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/NonLazyKeyValueScanner.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.SortedSet;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Scan;
+
+/**
+ * A "non-lazy" scanner which always does a real seek operation. Most scanners
+ * are inherited from this class.
+ */
+public abstract class NonLazyKeyValueScanner implements KeyValueScanner {
+
+  @Override
+  public boolean requestSeek(KeyValue kv, boolean forward, boolean useBloom)
+      throws IOException {
+    return doRealSeek(this, kv, forward);
+  }
+
+  @Override
+  public boolean realSeekDone() {
+    return true;
+  }
+
+  @Override
+  public void enforceSeek() throws IOException {
+    throw new NotImplementedException("enforceSeek must not be called on a " +
+        "non-lazy scanner");
+  }
+
+  public static boolean doRealSeek(KeyValueScanner scanner,
+      KeyValue kv, boolean forward) throws IOException {
+    return forward ? scanner.reseek(kv) : scanner.seek(kv);
+  }
+
+  @Override
+  public boolean shouldUseScanner(Scan scan, SortedSet columns,
+      long oldestUnexpiredTS) {
+    // No optimizations implemented by default.
+    return true;
+  }
+
+  @Override
+  public boolean isFileScanner() {
+    // Not a file by default.
+    return false;
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java b/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
new file mode 100644
index 0000000..847efb6
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
@@ -0,0 +1,63 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.Server;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Interface to Map of online regions.  In the  Map, the key is the region's
+ * encoded name and the value is an {@link HRegion} instance.
+ */
+interface OnlineRegions extends Server {
+  /**
+   * Add to online regions.
+   * @param r
+   */
+  public void addToOnlineRegions(final HRegion r);
+
+  /**
+   * This method removes HRegion corresponding to hri from the Map of onlineRegions.
+   *
+   * @param encodedRegionName
+   * @return True if we removed a region from online list.
+   */
+  public boolean removeFromOnlineRegions(String encodedRegionName);
+
+  /**
+   * Return {@link HRegion} instance.
+   * Only works if caller is in same context, in same JVM. HRegion is not
+   * serializable.
+   * @param encodedRegionName
+   * @return HRegion for the passed encoded encodedRegionName or
+   * null if named region is not member of the online regions.
+   */
+  public HRegion getFromOnlineRegions(String encodedRegionName);
+
+   /**
+    * Get all online regions of a table in this RS.
+    * @param tableName
+    * @return List of HRegion
+    * @throws java.io.IOException
+    */
+   public List getOnlineRegions(byte[] tableName) throws IOException;
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java b/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java
new file mode 100644
index 0000000..1b94ab5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
+/**
+ * 
+ * This class stores the Operation status code and the exception message
+ * that occurs in case of failure of operations like put, delete, etc.
+ * This class is added with a purpose of adding more details or info regarding
+ * the operation status in future.
+ *
+ */
+public class OperationStatus {
+
+  /** Singleton for successful operations.  */
+  static final OperationStatus SUCCESS =
+    new OperationStatus(OperationStatusCode.SUCCESS);
+
+  /** Singleton for failed operations.  */
+  static final OperationStatus FAILURE =
+    new OperationStatus(OperationStatusCode.FAILURE);
+
+  /** Singleton for operations not yet run.  */
+  static final OperationStatus NOT_RUN =
+    new OperationStatus(OperationStatusCode.NOT_RUN);
+
+  private final OperationStatusCode code;
+
+  private final String exceptionMsg;
+
+  public OperationStatus(OperationStatusCode code) {
+    this(code, "");
+  }
+
+  public OperationStatus(OperationStatusCode code, String exceptionMsg) {
+    this.code = code;
+    this.exceptionMsg = exceptionMsg;
+  }
+
+  
+  /**
+   * @return OperationStatusCode
+   */
+  public OperationStatusCode getOperationStatusCode() {
+    return code;
+  }
+
+  /**
+   * @return ExceptionMessge
+   */
+  public String getExceptionMsg() {
+    return exceptionMsg;
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
new file mode 100644
index 0000000..a4ec6d3
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
@@ -0,0 +1,113 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.Date;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.monitoring.LogMonitoring;
+import org.apache.hadoop.hbase.monitoring.StateDumpServlet;
+import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.util.ReflectionUtils;
+
+public class RSDumpServlet extends StateDumpServlet {
+  private static final long serialVersionUID = 1L;
+  private static final String LINE =
+    "===========================================================";
+  
+  @Override
+  public void doGet(HttpServletRequest request, HttpServletResponse response)
+      throws IOException {
+    HRegionServer hrs = (HRegionServer)getServletContext().getAttribute(
+        HRegionServer.REGIONSERVER);
+    assert hrs != null : "No RS in context!";
+    
+    Configuration hrsconf = (Configuration)getServletContext().getAttribute(
+        HRegionServer.REGIONSERVER_CONF);
+    assert hrsconf != null : "No RS conf in context";
+
+    response.setContentType("text/plain");
+    OutputStream os = response.getOutputStream();
+    PrintWriter out = new PrintWriter(os);
+    
+    out.println("Master status for " + hrs.getServerName()
+        + " as of " + new Date());
+    
+    out.println("\n\nVersion Info:");
+    out.println(LINE);
+    dumpVersionInfo(out);
+
+    out.println("\n\nTasks:");
+    out.println(LINE);
+    TaskMonitor.get().dumpAsText(out);
+    
+    out.println("\n\nExecutors:");
+    out.println(LINE);
+    dumpExecutors(hrs.getExecutorService(), out);
+    
+    out.println("\n\nStacks:");
+    out.println(LINE);
+    ReflectionUtils.printThreadInfo(out, "");
+    
+    out.println("\n\nRS Configuration:");
+    out.println(LINE);
+    Configuration conf = hrs.getConfiguration();
+    out.flush();
+    conf.writeXml(os);
+    os.flush();
+    
+    out.println("\n\nLogs");
+    out.println(LINE);
+    long tailKb = getTailKbParam(request);
+    LogMonitoring.dumpTailOfLogs(out, tailKb);
+    
+    out.println("\n\nRS Queue:");
+    out.println(LINE);
+    if(isShowQueueDump(hrsconf)) {
+      dumpQueue(hrs, out);
+    }
+    
+    out.flush();
+  }
+  
+  private boolean isShowQueueDump(Configuration conf){
+    return conf.getBoolean("hbase.regionserver.servlet.show.queuedump", true);
+  }
+  
+  private void dumpQueue(HRegionServer hrs, PrintWriter out)
+      throws IOException {
+    // 1. Print out Compaction/Split Queue
+    out.println("Compaction/Split Queue summary: " 
+        + hrs.compactSplitThread.toString() );
+    out.println(hrs.compactSplitThread.dumpQueue());
+
+    // 2. Print out flush Queue
+    out.println("\nFlush Queue summary: "
+        + hrs.cacheFlusher.toString());
+    out.println(hrs.cacheFlusher.dumpQueue());
+  }
+  
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RSStatusServlet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RSStatusServlet.java
new file mode 100644
index 0000000..7521cd4
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RSStatusServlet.java
@@ -0,0 +1,51 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl;
+
+public class RSStatusServlet extends HttpServlet {
+  private static final long serialVersionUID = 1L;
+
+  @Override
+  protected void doGet(HttpServletRequest req, HttpServletResponse resp)
+      throws ServletException, IOException
+    {
+    HRegionServer hrs = (HRegionServer)getServletContext().getAttribute(
+        HRegionServer.REGIONSERVER);
+    assert hrs != null : "No RS in context!";
+    
+    resp.setContentType("text/html");
+    RSStatusTmpl tmpl = new RSStatusTmpl();
+    if (req.getParameter("format") != null)
+      tmpl.setFormat(req.getParameter("format"));
+    if (req.getParameter("filter") != null)
+      tmpl.setFilter(req.getParameter("filter"));
+    tmpl.render(resp.getWriter(), hrs);
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionAlreadyInTransitionException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionAlreadyInTransitionException.java
new file mode 100644
index 0000000..1c21825
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionAlreadyInTransitionException.java
@@ -0,0 +1,34 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+/**
+ * This exception is thrown when a region server is asked to open or close
+ * a region but it's already processing it
+ */
+public class RegionAlreadyInTransitionException extends IOException {
+
+  public RegionAlreadyInTransitionException(String s) {
+    super(s);
+  }
+  
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
new file mode 100644
index 0000000..8e25292
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -0,0 +1,1748 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Matcher;
+
+import org.apache.commons.collections.map.AbstractReferenceMap;
+import org.apache.commons.collections.map.ReferenceMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.coprocessor.RegionObserverExt;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+import org.apache.hadoop.hbase.regionserver.SplitTransaction.SplitInfo;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Implements the coprocessor environment and runtime support for coprocessors
+ * loaded within a {@link HRegion}.
+ */
+public class RegionCoprocessorHost
+    extends CoprocessorHost {
+
+  private static final Log LOG = LogFactory.getLog(RegionCoprocessorHost.class);
+  // The shared data map
+  private static ReferenceMap sharedDataMap =
+      new ReferenceMap(AbstractReferenceMap.HARD, AbstractReferenceMap.WEAK);
+
+  /**
+   * Encapsulation of the environment of each coprocessor
+   */
+  static class RegionEnvironment extends CoprocessorHost.Environment
+      implements RegionCoprocessorEnvironment {
+
+    private HRegion region;
+    private RegionServerServices rsServices;
+    ConcurrentMap sharedData;
+
+    /**
+     * Constructor
+     * @param impl the coprocessor instance
+     * @param priority chaining priority
+     */
+    public RegionEnvironment(final Coprocessor impl, final int priority,
+        final int seq, final Configuration conf, final HRegion region,
+        final RegionServerServices services, final ConcurrentMap sharedData) {
+      super(impl, priority, seq, conf);
+      this.region = region;
+      this.rsServices = services;
+      this.sharedData = sharedData;
+    }
+
+    /** @return the region */
+    @Override
+    public HRegion getRegion() {
+      return region;
+    }
+
+    /** @return reference to the region server services */
+    @Override
+    public RegionServerServices getRegionServerServices() {
+      return rsServices;
+    }
+
+    public void shutdown() {
+      super.shutdown();
+    }
+
+    @Override
+    public ConcurrentMap getSharedData() {
+      return sharedData;
+    }
+  }
+
+  /** The region server services */
+  RegionServerServices rsServices;
+  /** The region */
+  HRegion region;
+
+  /**
+   * Constructor
+   * @param region the region
+   * @param rsServices interface to available region server functionality
+   * @param conf the configuration
+   */
+  public RegionCoprocessorHost(final HRegion region,
+      final RegionServerServices rsServices, final Configuration conf) {
+    this.conf = conf;
+    this.rsServices = rsServices;
+    this.region = region;
+    this.pathPrefix = Integer.toString(this.region.getRegionInfo().hashCode());
+
+    // load system default cp's from configuration.
+    loadSystemCoprocessors(conf, REGION_COPROCESSOR_CONF_KEY);
+
+    // load system default cp's for user tables from configuration.
+    if (!HTableDescriptor.isMetaTable(region.getRegionInfo().getTableName())) {
+      loadSystemCoprocessors(conf, USER_REGION_COPROCESSOR_CONF_KEY);
+    }
+
+    // load Coprocessor From HDFS
+    loadTableCoprocessors(conf);
+  }
+
+  void loadTableCoprocessors(final Configuration conf) {
+    // scan the table attributes for coprocessor load specifications
+    // initialize the coprocessors
+    List configured = new ArrayList();
+    for (Map.Entry e:
+        region.getTableDesc().getValues().entrySet()) {
+      String key = Bytes.toString(e.getKey().get()).trim();
+      String spec = Bytes.toString(e.getValue().get()).trim();
+      if (HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher(key).matches()) {
+        // found one
+        try {
+          Matcher matcher = HConstants.CP_HTD_ATTR_VALUE_PATTERN.matcher(spec);
+          if (matcher.matches()) {
+            // jar file path can be empty if the cp class can be loaded
+            // from class loader.
+            Path path = matcher.group(1).trim().isEmpty() ?
+                null : new Path(matcher.group(1).trim());
+            String className = matcher.group(2).trim();
+            int priority = matcher.group(3).trim().isEmpty() ?
+                Coprocessor.PRIORITY_USER : Integer.valueOf(matcher.group(3));
+            String cfgSpec = null;
+            try {
+              cfgSpec = matcher.group(4);
+            } catch (IndexOutOfBoundsException ex) {
+              // ignore
+            }
+            if (cfgSpec != null) {
+              cfgSpec = cfgSpec.substring(cfgSpec.indexOf('|') + 1);
+              Configuration newConf = new Configuration(conf);
+              Matcher m = HConstants.CP_HTD_ATTR_VALUE_PARAM_PATTERN.matcher(cfgSpec);
+              while (m.find()) {
+                newConf.set(m.group(1), m.group(2));
+              }
+              configured.add(load(path, className, priority, newConf));
+            } else {
+              configured.add(load(path, className, priority, conf));
+            }
+            LOG.info("Load coprocessor " + className + " from HTD of " +
+              Bytes.toString(region.getTableDesc().getName()) +
+                " successfully.");
+          } else {
+            throw new RuntimeException("specification does not match pattern");
+          }
+        } catch (Exception ex) {
+          LOG.warn("attribute '" + key +
+            "' has invalid coprocessor specification '" + spec + "'");
+          LOG.warn(StringUtils.stringifyException(ex));
+        }
+      }
+    }
+    // add together to coprocessor set for COW efficiency
+    coprocessors.addAll(configured);
+  }
+
+  @Override
+  public RegionEnvironment createEnvironment(Class implClass,
+      Coprocessor instance, int priority, int seq, Configuration conf) {
+    // Check if it's an Endpoint.
+    // Due to current dynamic protocol design, Endpoint
+    // uses a different way to be registered and executed.
+    // It uses a visitor pattern to invoke registered Endpoint
+    // method.
+    for (Class c : implClass.getInterfaces()) {
+      if (CoprocessorProtocol.class.isAssignableFrom(c)) {
+        region.registerProtocol(c, (CoprocessorProtocol)instance);
+        break;
+      }
+    }
+    ConcurrentMap classData;
+    // make sure only one thread can add maps
+    synchronized (sharedDataMap) {
+      // as long as at least one RegionEnvironment holds on to its classData it will
+      // remain in this map
+      classData = (ConcurrentMap)sharedDataMap.get(implClass.getName());
+      if (classData == null) {
+        classData = new ConcurrentHashMap();
+        sharedDataMap.put(implClass.getName(), classData);
+      }
+    }
+    return new RegionEnvironment(instance, priority, seq, conf, region,
+        rsServices, classData);
+  }
+
+  @Override
+  protected void abortServer(final CoprocessorEnvironment env, final Throwable e) {
+    abortServer("regionserver", rsServices, env, e);
+  }
+
+  /**
+   * HBASE-4014 : This is used by coprocessor hooks which are not declared to throw exceptions.
+   *
+   * For example, {@link
+   * org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost#preOpen()} and
+   * {@link org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost#postOpen()} are such hooks.
+   *
+   * See also {@link org.apache.hadoop.hbase.master.MasterCoprocessorHost#handleCoprocessorThrowable()}
+   * @param env The coprocessor that threw the exception.
+   * @param e The exception that was thrown.
+   */
+  private void handleCoprocessorThrowableNoRethrow(
+      final CoprocessorEnvironment env, final Throwable e) {
+    try {
+      handleCoprocessorThrowable(env,e);
+    } catch (IOException ioe) {
+      // We cannot throw exceptions from the caller hook, so ignore.
+      LOG.warn("handleCoprocessorThrowable() threw an IOException while attempting to handle Throwable " + e
+        + ". Ignoring.",e);
+    }
+  }
+
+  /**
+   * Invoked before a region open
+   */
+  public void preOpen(){
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+         try {
+          ((RegionObserver)env.getInstance()).preOpen(ctx);
+         } catch (Throwable e) {
+           handleCoprocessorThrowableNoRethrow(env, e);
+         }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Invoked after a region open
+   */
+  public void postOpen(){
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postOpen(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowableNoRethrow(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Invoked before a region is closed
+   * @param abortRequested true if the server is aborting
+   */
+  public void preClose(boolean abortRequested) throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).preClose(ctx, abortRequested);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+      }
+    }
+  }
+
+  /**
+   * Invoked after a region is closed
+   * @param abortRequested true if the server is aborting
+   */
+  public void postClose(boolean abortRequested){
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postClose(ctx, abortRequested);
+        } catch (Throwable e) {
+          handleCoprocessorThrowableNoRethrow(env, e);
+        }
+
+      }
+      shutdown(env);
+    }
+  }
+
+  /**
+   * See
+   * {@link RegionObserver#preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, InternalScanner, CompactionRequest)}
+   */
+  public InternalScanner preCompactScannerOpen(Store store, List scanners,
+      ScanType scanType, long earliestPutTs, CompactionRequest request) throws IOException {
+    ObserverContext ctx = null;
+    InternalScanner s = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          s = ((RegionObserver) env.getInstance()).preCompactScannerOpen(ctx, store, scanners,
+            scanType, earliestPutTs, s, request);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env,e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return s;
+  }
+
+  /**
+   * Called prior to selecting the {@link StoreFile}s for compaction from the list of currently
+   * available candidates.
+   * @param store The store where compaction is being requested
+   * @param candidates The currently available store files
+   * @param request custom compaction request
+   * @return If {@code true}, skip the normal selection process and use the current list
+   * @throws IOException
+   */
+  public boolean preCompactSelection(Store store, List candidates,
+      CompactionRequest request) throws IOException {
+    ObserverContext ctx = null;
+    boolean bypass = false;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver) env.getInstance()).preCompactSelection(ctx, store, candidates, request);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env,e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  /**
+   * Called after the {@link StoreFile}s to be compacted have been selected from the available
+   * candidates.
+   * @param store The store where compaction is being requested
+   * @param selected The store files selected to compact
+   * @param request custom compaction
+   */
+  public void postCompactSelection(Store store, ImmutableList selected,
+      CompactionRequest request) {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver) env.getInstance()).postCompactSelection(ctx, store, selected, request);
+        } catch (Throwable e) {
+          handleCoprocessorThrowableNoRethrow(env,e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Called prior to rewriting the store files selected for compaction
+   * @param store the store being compacted
+   * @param scanner the scanner used to read store data during compaction
+   * @param request the compaction that will be executed
+   * @throws IOException
+   */
+  public InternalScanner preCompact(Store store, InternalScanner scanner, 
+      CompactionRequest request) throws IOException {
+    ObserverContext ctx = null;
+    boolean bypass = false;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          scanner = ((RegionObserver) env.getInstance()).preCompact(ctx, store, scanner, request);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass ? null : scanner;
+  }
+
+  /**
+   * Called after the store compaction has completed.
+   * @param store the store being compacted
+   * @param resultFile the new store file written during compaction
+   * @param request the compaction that is being executed
+   * @throws IOException
+   */
+  public void postCompact(Store store, StoreFile resultFile, CompactionRequest request)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver) env.getInstance()).postCompact(ctx, store, resultFile, request);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Invoked before a memstore flush
+   * @throws IOException
+   */
+  public InternalScanner preFlush(Store store, InternalScanner scanner) throws IOException {
+    ObserverContext ctx = null;
+    boolean bypass = false;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          scanner = ((RegionObserver)env.getInstance()).preFlush(
+              ctx, store, scanner);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env,e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass ? null : scanner;
+  }
+
+  /**
+   * Invoked before a memstore flush
+   * @throws IOException 
+   */
+  public void preFlush() throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).preFlush(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * See
+   * {@link RegionObserver#preFlush(ObserverContext, Store, KeyValueScanner)}
+   */
+  public InternalScanner preFlushScannerOpen(Store store, KeyValueScanner memstoreScanner) throws IOException {
+    ObserverContext ctx = null;
+    InternalScanner s = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          s = ((RegionObserver) env.getInstance()).preFlushScannerOpen(ctx, store, memstoreScanner, s);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return s;
+  }
+
+  /**
+   * Invoked after a memstore flush
+   * @throws IOException
+   */
+  public void postFlush() throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postFlush(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Invoked after a memstore flush
+   * @throws IOException
+   */
+  public void postFlush(final Store store, final StoreFile storeFile) throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postFlush(ctx, store, storeFile);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Invoked just before a split
+   * @throws IOException
+   */
+  public void preSplit() throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).preSplit(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Invoked just after a split
+   * @param l the new left-hand daughter region
+   * @param r the new right-hand daughter region
+   * @throws IOException 
+   */
+  public void postSplit(HRegion l, HRegion r) throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postSplit(ctx, l, r);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  // RegionObserver support
+
+  /**
+   * @param row the row key
+   * @param family the family
+   * @param result the result set from the region
+   * @return true if default processing should be bypassed
+   * @exception IOException Exception
+   */
+  public boolean preGetClosestRowBefore(final byte[] row, final byte[] family,
+      final Result result) throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).preGetClosestRowBefore(ctx, row,
+              family, result);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  /**
+   * @param row the row key
+   * @param family the family
+   * @param result the result set from the region
+   * @exception IOException Exception
+   */
+  public void postGetClosestRowBefore(final byte[] row, final byte[] family,
+      final Result result) throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postGetClosestRowBefore(ctx, row,
+              family, result);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * @param get the Get request
+   * @return true if default processing should be bypassed
+   * @exception IOException Exception
+   */
+  public boolean preGet(final Get get, final List results)
+      throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).preGet(ctx, get, results);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  /**
+   * @param get the Get request
+   * @param results the result set
+   * @exception IOException Exception
+   */
+  public void postGet(final Get get, final List results)
+  throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postGet(ctx, get, results);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * @param get the Get request
+   * @return true or false to return to client if bypassing normal operation,
+   * or null otherwise
+   * @exception IOException Exception
+   */
+  public Boolean preExists(final Get get) throws IOException {
+    boolean bypass = false;
+    boolean exists = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          exists = ((RegionObserver)env.getInstance()).preExists(ctx, get, exists);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass ? exists : null;
+  }
+
+  /**
+   * @param get the Get request
+   * @param exists the result returned by the region server
+   * @return the result to return to the client
+   * @exception IOException Exception
+   */
+  public boolean postExists(final Get get, boolean exists)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          exists = ((RegionObserver)env.getInstance()).postExists(ctx, get, exists);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return exists;
+  }
+
+  /**
+   * @param put The Put object
+   * @param edit The WALEdit object.
+   * @param writeToWAL true if the change should be written to the WAL
+   * @return true if default processing should be bypassed
+   * @exception IOException Exception
+   */
+  public boolean prePut(Put put, WALEdit edit,
+      final boolean writeToWAL) throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).prePut(ctx, put, edit, writeToWAL);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  /**
+   * @param put The Put object
+   * @param edit The WALEdit object.
+   * @param writeToWAL true if the change should be written to the WAL
+   * @exception IOException Exception
+   */
+  public void postPut(Put put, WALEdit edit,
+      final boolean writeToWAL) throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postPut(ctx, put, edit, writeToWAL);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * @param delete The Delete object
+   * @param edit The WALEdit object.
+   * @param writeToWAL true if the change should be written to the WAL
+   * @return true if default processing should be bypassed
+   * @exception IOException Exception
+   */
+  public boolean preDelete(Delete delete, WALEdit edit,
+      final boolean writeToWAL) throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).preDelete(ctx, delete, edit, writeToWAL);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  /**
+   * @param delete The Delete object
+   * @param edit The WALEdit object.
+   * @param writeToWAL true if the change should be written to the WAL
+   * @exception IOException Exception
+   */
+  public void postDelete(Delete delete, WALEdit edit,
+      final boolean writeToWAL) throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postDelete(ctx, delete, edit, writeToWAL);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+  
+  /**
+   * @param miniBatchOp
+   * @return true if default processing should be bypassed
+   * @throws IOException
+   */
+  public boolean preBatchMutate(
+      final MiniBatchOperationInProgress> miniBatchOp) throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver) env.getInstance()).preBatchMutate(ctx, miniBatchOp);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  /**
+   * @param miniBatchOp
+   * @throws IOException
+   */
+  public void postBatchMutate(
+      final MiniBatchOperationInProgress> miniBatchOp) throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver) env.getInstance()).postBatchMutate(ctx, miniBatchOp);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * @param row row to check
+   * @param family column family
+   * @param qualifier column qualifier
+   * @param compareOp the comparison operation
+   * @param comparator the comparator
+   * @param put data to put if check succeeds
+   * @return true or false to return to client if default processing should
+   * be bypassed, or null otherwise
+   * @throws IOException e
+   */
+  public Boolean preCheckAndPut(final byte [] row, final byte [] family,
+      final byte [] qualifier, final CompareOp compareOp,
+      final WritableByteArrayComparable comparator, Put put)
+    throws IOException {
+    boolean bypass = false;
+    boolean result = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          result = ((RegionObserver)env.getInstance()).preCheckAndPut(ctx, row, family,
+            qualifier, compareOp, comparator, put, result);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+
+
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass ? result : null;
+  }
+
+  /**
+   * @param row row to check
+   * @param family column family
+   * @param qualifier column qualifier
+   * @param compareOp the comparison operation
+   * @param comparator the comparator
+   * @param put data to put if check succeeds
+   * @throws IOException e
+   */
+  public boolean postCheckAndPut(final byte [] row, final byte [] family,
+      final byte [] qualifier, final CompareOp compareOp,
+      final WritableByteArrayComparable comparator, final Put put,
+      boolean result)
+    throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          result = ((RegionObserver)env.getInstance()).postCheckAndPut(ctx, row,
+            family, qualifier, compareOp, comparator, put, result);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return result;
+  }
+
+  /**
+   * @param row row to check
+   * @param family column family
+   * @param qualifier column qualifier
+   * @param compareOp the comparison operation
+   * @param comparator the comparator
+   * @param delete delete to commit if check succeeds
+   * @return true or false to return to client if default processing should
+   * be bypassed, or null otherwise
+   * @throws IOException e
+   */
+  public Boolean preCheckAndDelete(final byte [] row, final byte [] family,
+      final byte [] qualifier, final CompareOp compareOp,
+      final WritableByteArrayComparable comparator, Delete delete)
+      throws IOException {
+    boolean bypass = false;
+    boolean result = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          result = ((RegionObserver)env.getInstance()).preCheckAndDelete(ctx, row,
+            family, qualifier, compareOp, comparator, delete, result);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass ? result : null;
+  }
+
+  /**
+   * @param row row to check
+   * @param family column family
+   * @param qualifier column qualifier
+   * @param compareOp the comparison operation
+   * @param comparator the comparator
+   * @param delete delete to commit if check succeeds
+   * @throws IOException e
+   */
+  public boolean postCheckAndDelete(final byte [] row, final byte [] family,
+      final byte [] qualifier, final CompareOp compareOp,
+      final WritableByteArrayComparable comparator, final Delete delete,
+      boolean result)
+    throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          result = ((RegionObserver)env.getInstance())
+            .postCheckAndDelete(ctx, row, family, qualifier, compareOp,
+              comparator, delete, result);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return result;
+  }
+
+  /**
+   * @param row row to check
+   * @param family column family
+   * @param qualifier column qualifier
+   * @param amount long amount to increment
+   * @param writeToWAL true if the change should be written to the WAL
+   * @return return value for client if default operation should be bypassed,
+   * or null otherwise
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  public Long preIncrementColumnValue(final byte [] row, final byte [] family,
+      final byte [] qualifier, long amount, final boolean writeToWAL)
+      throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          amount = ((RegionObserver)env.getInstance()).preIncrementColumnValue(ctx,
+              row, family, qualifier, amount, writeToWAL);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass ? amount : null;
+  }
+
+  /**
+   * @param row row to check
+   * @param family column family
+   * @param qualifier column qualifier
+   * @param amount long amount to increment
+   * @param writeToWAL true if the change should be written to the WAL
+   * @param result the result returned by incrementColumnValue
+   * @return the result to return to the client
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  public long postIncrementColumnValue(final byte [] row, final byte [] family,
+      final byte [] qualifier, final long amount, final boolean writeToWAL,
+      long result) throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          result = ((RegionObserver)env.getInstance()).postIncrementColumnValue(ctx,
+              row, family, qualifier, amount, writeToWAL, result);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return result;
+  }
+
+  /**
+   * @param append append object
+   * @return result to return to client if default operation should be
+   * bypassed, null otherwise
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  public Result preAppend(Append append)
+      throws IOException {
+    boolean bypass = false;
+    Result result = null;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          result = ((RegionObserver)env.getInstance()).preAppend(ctx, append);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass ? result : null;
+  }
+
+  /**
+   * @param increment increment object
+   * @return result to return to client if default operation should be
+   * bypassed, null otherwise
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  public Result preIncrement(Increment increment)
+      throws IOException {
+    boolean bypass = false;
+    Result result = null;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          result = ((RegionObserver)env.getInstance()).preIncrement(ctx, increment);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass ? result : null;
+  }
+
+  /**
+   * @param append Append object
+   * @param result the result returned by postAppend
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  public void postAppend(final Append append, Result result)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postAppend(ctx, append, result);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * @param increment increment object
+   * @param result the result returned by postIncrement
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  public Result postIncrement(final Increment increment, Result result)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          result = ((RegionObserver)env.getInstance()).postIncrement(ctx, increment, result);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return result;
+  }
+
+  /**
+   * @param scan the Scan specification
+   * @return scanner id to return to client if default operation should be
+   * bypassed, false otherwise
+   * @exception IOException Exception
+   */
+  public RegionScanner preScannerOpen(Scan scan) throws IOException {
+    boolean bypass = false;
+    RegionScanner s = null;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          s = ((RegionObserver)env.getInstance()).preScannerOpen(ctx, scan, s);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass ? s : null;
+  }
+
+  /**
+   * See
+   * {@link RegionObserver#preStoreScannerOpen(ObserverContext, Store, Scan, NavigableSet, KeyValueScanner)}
+   */
+  public KeyValueScanner preStoreScannerOpen(Store store, Scan scan,
+      final NavigableSet targetCols) throws IOException {
+    KeyValueScanner s = null;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          s = ((RegionObserver) env.getInstance()).preStoreScannerOpen(ctx, store, scan,
+              targetCols, s);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return s;
+  }
+
+  /**
+   * @param scan the Scan specification
+   * @param s the scanner
+   * @return the scanner instance to use
+   * @exception IOException Exception
+   */
+  public RegionScanner postScannerOpen(final Scan scan, RegionScanner s)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          s = ((RegionObserver)env.getInstance()).postScannerOpen(ctx, scan, s);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return s;
+  }
+
+  /**
+   * @param s the scanner
+   * @param results the result set returned by the region server
+   * @param limit the maximum number of results to return
+   * @return 'has next' indication to client if bypassing default behavior, or
+   * null otherwise
+   * @exception IOException Exception
+   */
+  public Boolean preScannerNext(final InternalScanner s,
+      final List results, int limit) throws IOException {
+    boolean bypass = false;
+    boolean hasNext = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          hasNext = ((RegionObserver)env.getInstance()).preScannerNext(ctx, s, results,
+            limit, hasNext);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass ? hasNext : null;
+  }
+
+  /**
+   * @param s the scanner
+   * @param results the result set returned by the region server
+   * @param limit the maximum number of results to return
+   * @param hasMore
+   * @return 'has more' indication to give to client
+   * @exception IOException Exception
+   */
+  public boolean postScannerNext(final InternalScanner s,
+      final List results, final int limit, boolean hasMore)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          hasMore = ((RegionObserver)env.getInstance()).postScannerNext(ctx, s,
+            results, limit, hasMore);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return hasMore;
+  }
+
+  /**
+   * This will be called by the scan flow when the current scanned row is being filtered out by the
+   * filter.
+   * @param s the scanner
+   * @param currentRow The current rowkey which got filtered out
+   * @return whether more rows are available for the scanner or not
+   * @throws IOException
+   */
+  public boolean postScannerFilterRow(final InternalScanner s, final byte[] currentRow)
+      throws IOException {
+    boolean hasMore = true; // By default assume more rows there.
+    ObserverContext ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          hasMore = ((RegionObserver) env.getInstance()).postScannerFilterRow(ctx, s, currentRow,
+              hasMore);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return hasMore;
+  }
+ 
+  /**
+   * @param s the scanner
+   * @return true if default behavior should be bypassed, false otherwise
+   * @exception IOException Exception
+   */
+  public boolean preScannerClose(final InternalScanner s)
+      throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).preScannerClose(ctx, s);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  /**
+   * @param s the scanner
+   * @exception IOException Exception
+   */
+  public void postScannerClose(final InternalScanner s)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postScannerClose(ctx, s);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * @param info
+   * @param logKey
+   * @param logEdit
+   * @return true if default behavior should be bypassed, false otherwise
+   * @throws IOException
+   */
+  public boolean preWALRestore(HRegionInfo info, HLogKey logKey,
+      WALEdit logEdit) throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).preWALRestore(ctx, info, logKey,
+              logEdit);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  /**
+   * @param info
+   * @param logKey
+   * @param logEdit
+   * @throws IOException
+   */
+  public void postWALRestore(HRegionInfo info, HLogKey logKey,
+      WALEdit logEdit) throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).postWALRestore(ctx, info,
+              logKey, logEdit);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * @param familyPaths pairs of { CF, file path } submitted for bulk load
+   * @return true if the default operation should be bypassed
+   * @throws IOException
+   */
+  public boolean preBulkLoadHFile(List> familyPaths) throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserver)env.getInstance()).preBulkLoadHFile(ctx, familyPaths);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+
+    return bypass;
+  }
+
+  /**
+   * @param familyPaths pairs of { CF, file path } submitted for bulk load
+   * @param hasLoaded whether load was successful or not
+   * @return the possibly modified value of hasLoaded
+   * @throws IOException
+   */
+  public boolean postBulkLoadHFile(List> familyPaths, boolean hasLoaded)
+      throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env: coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          hasLoaded = ((RegionObserver)env.getInstance()).postBulkLoadHFile(ctx,
+            familyPaths, hasLoaded);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+
+    return hasLoaded;
+  }
+
+  /**
+   * @param mutations
+   * @param edit
+   * @return
+   * @throws IOException
+   */
+  public boolean preBatchMutate(final List> mutationVsBatchOp,
+      final WALEdit edit) throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserverExt) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserverExt) env.getInstance()).preBatchMutate(ctx, mutationVsBatchOp, edit);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  /**
+   * @param mutations
+   * @param walEdit
+   * @return
+   * @throws IOException
+   */
+  public boolean postBatchMutate(final List mutations, final WALEdit walEdit)
+      throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserverExt) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserverExt) env.getInstance()).postBatchMutate(ctx, mutations, walEdit);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  /**
+   * @param mutations
+   * @return
+   * @throws IOException
+   */
+  public boolean postCompleteBatchMutate(List mutations) throws IOException {
+    boolean bypass = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserverExt) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserverExt) env.getInstance()).postCompleteBatchMutate(ctx, mutations);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        bypass |= ctx.shouldBypass();
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return bypass;
+  }
+
+  public void preLockRow(byte[] regionName, byte[] row) throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        ((RegionObserver) env.getInstance()).preLockRow(ctx, regionName, row);
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void preUnLockRow(byte[] regionName, long lockId) throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        ((RegionObserver) env.getInstance()).preUnlockRow(ctx, regionName, lockId);
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public SplitInfo preSplitBeforePONR(byte[] splitKey) throws IOException {
+    SplitInfo info = null;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserverExt) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          info = ((RegionObserverExt) env.getInstance()).preSplitBeforePONR(ctx, splitKey);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+    return info;
+  }
+
+  public void preRollBack() throws IOException {
+    boolean result = false;
+    ObserverContext ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserverExt) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserverExt) env.getInstance()).preRollBack(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  public void postCloseRegionOperation() throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserverExt) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserverExt) env.getInstance()).postCloseRegionOperation(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  // May be we can return boolean from here and use that boolean over there
+  // in HRegion before calling postCloseRegionOperation if there was no lock acquired
+  public void postStartRegionOperation() throws IOException {
+    ObserverContext ctx = null;
+    for (RegionEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionObserverExt) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        try {
+          ((RegionObserverExt) env.getInstance()).postStartRegionOperation(ctx);
+        } catch (Throwable e) {
+          handleCoprocessorThrowable(env, e);
+        }
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionOpeningState.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionOpeningState.java
new file mode 100644
index 0000000..c5bcb4c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionOpeningState.java
@@ -0,0 +1,29 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+public enum RegionOpeningState {
+
+  OPENED,
+
+  ALREADY_OPENED,
+
+  FAILED_OPENING;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
new file mode 100644
index 0000000..7b6762c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
@@ -0,0 +1,101 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.KeyValue;
+
+/**
+ * RegionScanner describes iterators over rows in an HRegion.
+ */
+public interface RegionScanner extends InternalScanner {
+  /**
+   * @return The RegionInfo for this scanner.
+   */
+  public HRegionInfo getRegionInfo();
+
+  /**
+   * @return True if a filter indicates that this scanner will return no
+   *         further rows.
+   */
+  public boolean isFilterDone();
+
+  /**
+   * Do a reseek to the required row. Should not be used to seek to a key which
+   * may come before the current position. Always seeks to the beginning of a
+   * row boundary.
+   *
+   * @throws IOException
+   * @throws IllegalArgumentException
+   *           if row is null
+   *
+   */
+  public boolean reseek(byte[] row) throws IOException;
+
+  /**
+   * @return The Scanner's MVCC readPt see {@link MultiVersionConsistencyControl}
+   */
+  public long getMvccReadPoint();
+
+  /**
+   * Grab the next row's worth of values with the default limit on the number of values
+   * to return.
+   * This is a special internal method to be called from coprocessor hooks to avoid expensive setup.
+   * Caller must set the thread's readpoint, start and close a region operation, an synchronize on the scanner object.
+   * See {@link #nextRaw(List, int, String)}
+   * @param result return output array
+   * @param metric the metric name
+   * @return true if more rows exist after this one, false if scanner is done
+   * @throws IOException e
+   */
+  public boolean nextRaw(List result, String metric) throws IOException;
+
+  /**
+   * Grab the next row's worth of values with a limit on the number of values
+   * to return.
+   * This is a special internal method to be called from coprocessor hooks to avoid expensive setup.
+   * Caller must set the thread's readpoint, start and close a region operation, an synchronize on the scanner object.
+   * Example:
+   * 
                +   * HRegion region = ...;
+   * RegionScanner scanner = ...
+   * MultiVersionConsistencyControl.setThreadReadPoint(scanner.getMvccReadPoint());
+   * region.startRegionOperation();
+   * try {
+   *   synchronized(scanner) {
+   *     ...
+   *     boolean moreRows = scanner.nextRaw(values);
+   *     ...
+   *   }
+   * } finally {
+   *   region.closeRegionOperation();
+   * }
+   * 
                
+   * @param result return output array
+   * @param limit limit on row count to get
+   * @param metric the metric name
+   * @return true if more rows exist after this one, false if scanner is done
+   * @throws IOException e
+   */
+  public boolean nextRaw(List result, int limit, String metric) throws IOException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
new file mode 100644
index 0000000..3fb7c0d
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
@@ -0,0 +1,98 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * RegionServerAccounting keeps record of some basic real time information about
+ * the Region Server. Currently, it only keeps record the global memstore size. 
+ */
+public class RegionServerAccounting {
+
+  private final AtomicLong atomicGlobalMemstoreSize = new AtomicLong(0);
+  
+  // Store the edits size during replaying HLog. Use this to roll back the  
+  // global memstore size once a region opening failed.
+  private final ConcurrentMap replayEditsPerRegion = 
+    new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR);
+  
+  /**
+   * @return the global Memstore size in the RegionServer
+   */
+  public long getGlobalMemstoreSize() {
+    return atomicGlobalMemstoreSize.get();
+  }
+  
+  /**
+   * @param memStoreSize the Memstore size will be added to 
+   *        the global Memstore size 
+   * @return the global Memstore size in the RegionServer 
+   */
+  public long addAndGetGlobalMemstoreSize(long memStoreSize) {
+    return atomicGlobalMemstoreSize.addAndGet(memStoreSize);
+  }
+
+  /***
+   * Add memStoreSize to replayEditsPerRegion.
+   * 
+   * @param regionName region name.
+   * @param memStoreSize the Memstore size will be added to replayEditsPerRegion.
+   * @return the replay edits size for region hri.
+   */
+  public long addAndGetRegionReplayEditsSize(byte[] regionName, long memStoreSize) {
+    AtomicLong replayEdistsSize = replayEditsPerRegion.get(regionName);
+    if (replayEdistsSize == null) {
+      replayEdistsSize = new AtomicLong(0);
+      replayEditsPerRegion.put(regionName, replayEdistsSize);
+    }
+    return replayEdistsSize.addAndGet(memStoreSize);
+  }
+
+  /**
+   * Roll back the global MemStore size for a specified region when this region
+   * can't be opened.
+   * 
+   * @param regionName the region which could not open.
+   * @return the global Memstore size in the RegionServer
+   */
+  public long rollbackRegionReplayEditsSize(byte[] regionName) {
+    AtomicLong replayEditsSize = replayEditsPerRegion.get(regionName);
+    long editsSizeLong = 0L;
+    if (replayEditsSize != null) {
+      editsSizeLong = -replayEditsSize.get();
+      clearRegionReplayEditsSize(regionName);
+    }
+    return addAndGetGlobalMemstoreSize(editsSizeLong);
+  }
+
+  /**
+   * Clear a region from replayEditsPerRegion.
+   * 
+   * @param regionName region name.
+   */
+  public void clearRegionReplayEditsSize(byte[] regionName) {
+    replayEditsPerRegion.remove(regionName);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
new file mode 100644
index 0000000..2302f9e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.Comparator;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
+
+public class RegionServerCoprocessorHost extends
+    CoprocessorHost {
+
+  private RegionServerServices rsServices;
+
+  public RegionServerCoprocessorHost(RegionServerServices rsServices, Configuration conf) {
+    this.rsServices = rsServices;
+    this.conf = conf;
+    // load system default cp's from configuration.
+    loadSystemCoprocessors(conf, REGIONSERVER_COPROCESSOR_CONF_KEY);
+  }
+
+  @Override
+  public RegionServerEnvironment createEnvironment(Class implClass, Coprocessor instance,
+      int priority, int sequence, Configuration conf) {
+    return new RegionServerEnvironment(implClass, instance, priority, sequence, conf,
+        this.rsServices);
+  }
+
+  public void preStop(String message) throws IOException {
+    ObserverContext ctx = null;
+    for (RegionServerEnvironment env : coprocessors) {
+      if (env.getInstance() instanceof RegionServerObserver) {
+        ctx = ObserverContext.createAndPrepare(env, ctx);
+        ((RegionServerObserver) env.getInstance()).preStopRegionServer(ctx);
+        if (ctx.shouldComplete()) {
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Coprocessor environment extension providing access to region server related services.
+   */
+  static class RegionServerEnvironment extends CoprocessorHost.Environment implements
+      RegionServerCoprocessorEnvironment {
+
+    private RegionServerServices regionServerServices;
+
+    public RegionServerEnvironment(final Class implClass, final Coprocessor impl,
+        final int priority, final int seq, final Configuration conf,
+        final RegionServerServices services) {
+      super(impl, priority, seq, conf);
+      this.regionServerServices = services;
+    }
+
+    @Override
+    public RegionServerServices getRegionServerServices() {
+      return regionServerServices;
+    }
+  }
+
+  /**
+   * Environment priority comparator. Coprocessors are chained in sorted order.
+   */
+  static class EnvironmentPriorityComparator implements Comparator {
+    public int compare(final CoprocessorEnvironment env1, final CoprocessorEnvironment env2) {
+      if (env1.getPriority() < env2.getPriority()) {
+        return -1;
+      } else if (env1.getPriority() > env2.getPriority()) {
+        return 1;
+      }
+      if (env1.getLoadSequence() < env2.getLoadSequence()) {
+        return -1;
+      } else if (env1.getLoadSequence() > env2.getLoadSequence()) {
+        return 1;
+      }
+      return 0;
+    }
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerRunningException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerRunningException.java
new file mode 100644
index 0000000..ed36ed7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerRunningException.java
@@ -0,0 +1,44 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+/**
+ * Thrown if the region server log directory exists (which indicates another
+ * region server is running at the same address)
+ */
+public class RegionServerRunningException extends IOException {
+  private static final long serialVersionUID = 1L << 31 - 1L;
+
+  /** Default Constructor */
+  public RegionServerRunningException() {
+    super();
+  }
+
+  /**
+   * Constructs the exception and supplies a string as the message
+   * @param s - message
+   */
+  public RegionServerRunningException(String s) {
+    super(s);
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
new file mode 100644
index 0000000..c3473c6
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -0,0 +1,101 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.ipc.RpcServer;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Services provided by {@link HRegionServer}
+ */
+public interface RegionServerServices extends OnlineRegions {
+  /**
+   * @return True if this regionserver is stopping.
+   */
+  public boolean isStopping();
+
+  /** @return the HLog for a particular region. Pass null for getting the
+   * default (common) WAL */
+  public HLog getWAL(HRegionInfo regionInfo) throws IOException;
+
+  /**
+   * @return Implementation of {@link CompactionRequestor} or null.
+   */
+  public CompactionRequestor getCompactionRequester();
+
+  /**
+   * @return Implementation of {@link FlushRequester} or null.
+   */
+  public FlushRequester getFlushRequester();
+
+  /**
+   * @return the RegionServerAccounting for this Region Server
+   */
+  public RegionServerAccounting getRegionServerAccounting();
+
+  /**
+   * Tasks to perform after region open to complete deploy of region on
+   * regionserver
+   * 
+   * @param r Region to open.
+   * @param ct Instance of {@link CatalogTracker}
+   * @param daughter True if this is daughter of a split
+   * @throws KeeperException
+   * @throws IOException
+   */
+  public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct,
+      final boolean daughter)
+  throws KeeperException, IOException;
+
+  /**
+   * Returns a reference to the region server's RPC server
+   */
+  public RpcServer getRpcServer();
+
+  /**
+   * Remove passed hri from the internal list of regions in transition on this
+   * regionserver.
+   * @param hri Region to remove.
+   * @return True if removed
+   */
+  public boolean removeFromRegionsInTransition(HRegionInfo hri);
+  /**
+   * @param hri
+   * @return True if the internal list of regions in transition includes the
+   *         passed hri.
+   */
+  public boolean containsKeyInRegionsInTransition(HRegionInfo hri);
+
+  /**
+   * @return Return the FileSystem object used by the regionserver
+   */
+  public FileSystem getFileSystem();
+
+  /**
+   * @return The RegionServer's "Leases" service
+   */
+  public Leases getLeases();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerStoppedException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerStoppedException.java
new file mode 100644
index 0000000..45acb17
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerStoppedException.java
@@ -0,0 +1,33 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+/**
+ * Thrown by the region server when it is in shutting down state.
+ */
+@SuppressWarnings("serial")
+public class RegionServerStoppedException extends IOException {
+
+  public RegionServerStoppedException(String s) {
+    super(s);
+  }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
new file mode 100644
index 0000000..a526544
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A split policy determines when a region should be split.
+ * @see IncreasingToUpperBoundRegionSplitPolicy Default split policy since
+ *      0.94.0
+ * @see ConstantSizeRegionSplitPolicy Default split policy before 0.94.0
+ */
+public abstract class RegionSplitPolicy extends Configured {
+  private static final Class
+    DEFAULT_SPLIT_POLICY_CLASS = IncreasingToUpperBoundRegionSplitPolicy.class;
+
+  /**
+   * The region configured for this split policy.
+   */
+  protected HRegion region;
+  
+  /**
+   * Upon construction, this method will be called with the region
+   * to be governed. It will be called once and only once.
+   */
+  protected void configureForRegion(HRegion region) {
+    Preconditions.checkState(
+        this.region == null,
+        "Policy already configured for region {}",
+        this.region);
+    
+    this.region = region;
+  }
+
+  /**
+   * @return true if the specified region should be split.
+   */
+  protected abstract boolean shouldSplit();
+
+  /**
+   * @return the key at which the region should be split, or null
+   * if it cannot be split. This will only be called if shouldSplit
+   * previously returned true.
+   */
+  protected byte[] getSplitPoint() {
+    byte[] explicitSplitPoint = this.region.getExplicitSplitPoint();
+    if (explicitSplitPoint != null) {
+      return explicitSplitPoint;
+    }
+    Map stores = region.getStores();
+
+    byte[] splitPointFromLargestStore = null;
+    long largestStoreSize = 0;
+    for (Store s : stores.values()) {
+      byte[] splitPoint = s.getSplitPoint();
+      long storeSize = s.getSize();
+      if (splitPoint != null && largestStoreSize < storeSize) {
+        splitPointFromLargestStore = splitPoint;
+        largestStoreSize = storeSize;
+      }
+    }
+    
+    return splitPointFromLargestStore;
+  }
+
+  /**
+   * Create the RegionSplitPolicy configured for the given table.
+   * Each
+   * @param htd
+   * @param conf
+   * @return
+   * @throws IOException
+   */
+  public static RegionSplitPolicy create(HRegion region,
+      Configuration conf) throws IOException {
+    
+    Class clazz = getSplitPolicyClass(
+        region.getTableDesc(), conf);
+    RegionSplitPolicy policy = ReflectionUtils.newInstance(clazz, conf);
+    policy.configureForRegion(region);
+    return policy;
+  }
+  
+  static Class getSplitPolicyClass(
+      HTableDescriptor htd, Configuration conf) throws IOException {
+    String className = htd.getRegionSplitPolicyClassName();
+    if (className == null) {
+      className = conf.get(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+          DEFAULT_SPLIT_POLICY_CLASS.getName());
+    }
+
+    try {
+      Class clazz =
+        Class.forName(className).asSubclass(RegionSplitPolicy.class);
+      return clazz;
+    } catch (Exception  e) {
+      throw new IOException(
+          "Unable to load configured region split policy '" +
+          className + "' for table '" + htd.getNameAsString() + "'",
+          e);
+    }
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationService.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationService.java
new file mode 100644
index 0000000..83ffcc7
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationService.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Gateway to Cluster Replication.  
+ * Used by {@link org.apache.hadoop.hbase.regionserver.HRegionServer}.
+ * One such application is a cross-datacenter
+ * replication service that can keep two hbase clusters in sync.
+ */
+public interface ReplicationService {
+
+  /**
+   * Initializes the replication service object.
+   * @throws IOException
+   */
+  public void initialize(Server rs, FileSystem fs, Path logdir,
+                         Path oldLogDir) throws IOException;
+
+  /**
+   * Start replication services.
+   * @throws IOException
+   */
+  public void startReplicationService() throws IOException;
+
+  /**
+   * Stops replication service.
+   */
+  public void stopReplicationService();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSinkService.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSinkService.java
new file mode 100644
index 0000000..1d22192
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSinkService.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+
+/**
+ * A sink for a replication stream has to expose this service.
+ * This service allows an application to hook into the
+ * regionserver and behave as a replication sink.
+ */
+public interface ReplicationSinkService extends ReplicationService {
+
+ /**
+   * Carry on the list of log entries down to the sink
+   * @param entries list of entries to replicate
+   * @throws IOException
+   */
+  public void replicateLogEntries(HLog.Entry[] entries) throws IOException;
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
new file mode 100644
index 0000000..9e16d02
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
+
+/**
+ * A source for a replication stream has to expose this service.
+ * This service allows an application to hook into the
+ * regionserver and watch for new transactions.
+ */
+public interface ReplicationSourceService extends ReplicationService {
+
+  /**
+   * Returns a WALObserver for the service. This is needed to 
+   * observe log rolls and log archival events.
+   */
+  public WALActionsListener getWALActionsListener();
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java
new file mode 100644
index 0000000..cc26e3a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java
@@ -0,0 +1,164 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * This class is responsible for the tracking and enforcement of Deletes
+ * during the course of a Scan operation.
+ *
+ * It only has to enforce Delete and DeleteColumn, since the
+ * DeleteFamily is handled at a higher level.
+ *
+ * 
                
+ * This class is utilized through three methods:
+ * 
                • {@link #add} when encountering a Delete or DeleteColumn
+ * 
                • {@link #isDeleted} when checking if a Put KeyValue has been deleted
+ * 
                • {@link #update} when reaching the end of a StoreFile or row for scans
+ * 
                  
+ * This class is NOT thread-safe as queries are never multi-threaded
+ */
+public class ScanDeleteTracker implements DeleteTracker {
+
+  private boolean hasFamilyStamp = false;
+  private long familyStamp = 0L;
+  private byte [] deleteBuffer = null;
+  private int deleteOffset = 0;
+  private int deleteLength = 0;
+  private byte deleteType = 0;
+  private long deleteTimestamp = 0L;
+
+  /**
+   * Constructor for ScanDeleteTracker
+   */
+  public ScanDeleteTracker() {
+    super();
+  }
+
+  /**
+   * Add the specified KeyValue to the list of deletes to check against for
+   * this row operation.
+   * 
                  
+   * This is called when a Delete is encountered in a StoreFile.
+   * @param buffer KeyValue buffer
+   * @param qualifierOffset column qualifier offset
+   * @param qualifierLength column qualifier length
+   * @param timestamp timestamp
+   * @param type delete type as byte
+   */
+  @Override
+  public void add(byte[] buffer, int qualifierOffset, int qualifierLength,
+      long timestamp, byte type) {
+    if (!hasFamilyStamp || timestamp > familyStamp) {
+      if (type == KeyValue.Type.DeleteFamily.getCode()) {
+        hasFamilyStamp = true;
+        familyStamp = timestamp;
+        return;
+      }
+
+      if (deleteBuffer != null && type < deleteType) {
+        // same column, so ignore less specific delete
+        if (Bytes.equals(deleteBuffer, deleteOffset, deleteLength,
+            buffer, qualifierOffset, qualifierLength)){
+          return;
+        }
+      }
+      // new column, or more general delete type
+      deleteBuffer = buffer;
+      deleteOffset = qualifierOffset;
+      deleteLength = qualifierLength;
+      deleteType = type;
+      deleteTimestamp = timestamp;
+    }
+    // missing else is never called.
+  }
+
+  /**
+   * Check if the specified KeyValue buffer has been deleted by a previously
+   * seen delete.
+   *
+   * @param buffer KeyValue buffer
+   * @param qualifierOffset column qualifier offset
+   * @param qualifierLength column qualifier length
+   * @param timestamp timestamp
+   * @return deleteResult
+   */
+  @Override
+  public DeleteResult isDeleted(byte [] buffer, int qualifierOffset,
+      int qualifierLength, long timestamp) {
+    if (hasFamilyStamp && timestamp <= familyStamp) {
+      return DeleteResult.FAMILY_DELETED;
+    }
+
+    if (deleteBuffer != null) {
+      int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength,
+          buffer, qualifierOffset, qualifierLength);
+
+      if (ret == 0) {
+        if (deleteType == KeyValue.Type.DeleteColumn.getCode()) {
+          return DeleteResult.COLUMN_DELETED;
+        }
+        // Delete (aka DeleteVersion)
+        // If the timestamp is the same, keep this one
+        if (timestamp == deleteTimestamp) {
+          return DeleteResult.VERSION_DELETED;
+        }
+        // use assert or not?
+        assert timestamp < deleteTimestamp;
+
+        // different timestamp, let's clear the buffer.
+        deleteBuffer = null;
+      } else if(ret < 0){
+        // Next column case.
+        deleteBuffer = null;
+      } else {
+        throw new IllegalStateException("isDelete failed: deleteBuffer="
+            + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength)
+            + ", qualifier="
+            + Bytes.toStringBinary(buffer, qualifierOffset, qualifierLength)
+            + ", timestamp=" + timestamp + ", comparison result: " + ret);
+      }
+    }
+
+    return DeleteResult.NOT_DELETED;
+  }
+
+  @Override
+  public boolean isEmpty() {
+    return deleteBuffer == null && !hasFamilyStamp;
+  }
+
+  @Override
+  // called between every row.
+  public void reset() {
+    hasFamilyStamp = false;
+    familyStamp = 0L;
+    deleteBuffer = null;
+  }
+
+  @Override
+  // should not be called at all even (!)
+  public void update() {
+    this.reset();
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
new file mode 100644
index 0000000..4b52159
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -0,0 +1,511 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.NavigableSet;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
+import org.apache.hadoop.hbase.io.TimeRange;
+import org.apache.hadoop.hbase.regionserver.DeleteTracker.DeleteResult;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+/**
+ * A query matcher that is specifically designed for the scan case.
+ */
+public class ScanQueryMatcher {
+  // Optimization so we can skip lots of compares when we decide to skip
+  // to the next row.
+  private boolean stickyNextRow;
+  private final byte[] stopRow;
+
+  private final TimeRange tr;
+
+  private final Filter filter;
+
+  /** Keeps track of deletes */
+  private final DeleteTracker deletes;
+
+  /*
+   * The following three booleans define how we deal with deletes.
+   * There are three different aspects:
+   * 1. Whether to keep delete markers. This is used in compactions.
+   *    Minor compactions always keep delete markers.
+   * 2. Whether to keep deleted rows. This is also used in compactions,
+   *    if the store is set to keep deleted rows. This implies keeping
+   *    the delete markers as well.
+   *    In this case deleted rows are subject to the normal max version
+   *    and TTL/min version rules just like "normal" rows.
+   * 3. Whether a scan can do time travel queries even before deleted
+   *    marker to reach deleted rows.
+   */
+  /** whether to retain delete markers */
+  private final boolean retainDeletesInOutput;
+  /** whether to return deleted rows */
+  private final boolean keepDeletedCells;
+  /** whether time range queries can see rows "behind" a delete */
+  private final boolean seePastDeleteMarkers;
+
+
+  /** Keeps track of columns and versions */
+  private final ColumnTracker columns;
+
+  /** Key to seek to in memstore and StoreFiles */
+  private final KeyValue startKey;
+
+  /** Row comparator for the region this query is for */
+  private final KeyValue.KeyComparator rowComparator;
+
+  /* row is not private for tests */
+  /** Row the query is on */
+  byte [] row;
+  int rowOffset;
+  short rowLength;
+  
+  /**
+   * Oldest put in any of the involved store files
+   * Used to decide whether it is ok to delete
+   * family delete marker of this store keeps
+   * deleted KVs.
+   */
+  private final long earliestPutTs;
+
+  /** readPoint over which the KVs are unconditionally included */
+  protected long maxReadPointToTrackVersions;
+
+  /**
+   * This variable shows whether there is an null column in the query. There
+   * always exists a null column in the wildcard column query.
+   * There maybe exists a null column in the explicit column query based on the
+   * first column.
+   * */
+  private boolean hasNullColumn = true;
+
+  // By default, when hbase.hstore.time.to.purge.deletes is 0ms, a delete
+  // marker is always removed during a major compaction. If set to non-zero
+  // value then major compaction will try to keep a delete marker around for
+  // the given number of milliseconds. We want to keep the delete markers
+  // around a bit longer because old puts might appear out-of-order. For
+  // example, during log replication between two clusters.
+  //
+  // If the delete marker has lived longer than its column-family's TTL then
+  // the delete marker will be removed even if time.to.purge.deletes has not
+  // passed. This is because all the Puts that this delete marker can influence
+  // would have also expired. (Removing of delete markers on col family TTL will
+  // not happen if min-versions is set to non-zero)
+  //
+  // But, if time.to.purge.deletes has not expired then a delete
+  // marker will not be removed just because there are no Puts that it is
+  // currently influencing. This is because Puts, that this delete can
+  // influence.  may appear out of order.
+  private final long timeToPurgeDeletes;
+  
+  private final boolean isUserScan;
+
+  /**
+   * Construct a QueryMatcher for a scan
+   * @param scan
+   * @param scanInfo The store's immutable scan info
+   * @param columns
+   * @param scanType Type of the scan
+   * @param earliestPutTs Earliest put seen in any of the store files.
+   * @param oldestUnexpiredTS the oldest timestamp we are interested in,
+   *  based on TTL
+   */
+  public ScanQueryMatcher(Scan scan, Store.ScanInfo scanInfo,
+      NavigableSet columns, ScanType scanType,
+      long readPointToUse, long earliestPutTs, long oldestUnexpiredTS) {
+    this.tr = scan.getTimeRange();
+    this.rowComparator = scanInfo.getComparator().getRawComparator();
+    this.deletes =  new ScanDeleteTracker();
+    this.stopRow = scan.getStopRow();
+    this.startKey = KeyValue.createFirstDeleteFamilyOnRow(scan.getStartRow(),
+        scanInfo.getFamily());
+    this.filter = scan.getFilter();
+    this.earliestPutTs = earliestPutTs;
+    this.maxReadPointToTrackVersions = readPointToUse;
+    this.timeToPurgeDeletes = scanInfo.getTimeToPurgeDeletes();
+
+    /* how to deal with deletes */
+    this.isUserScan = scanType == ScanType.USER_SCAN;
+    // keep deleted cells: if compaction or raw scan
+    this.keepDeletedCells = (scanInfo.getKeepDeletedCells() && !isUserScan) || scan.isRaw();
+    // retain deletes: if minor compaction or raw scan
+    this.retainDeletesInOutput = scanType == ScanType.MINOR_COMPACT || scan.isRaw();
+    // seePastDeleteMarker: user initiated scans
+    this.seePastDeleteMarkers = scanInfo.getKeepDeletedCells() && isUserScan;
+
+    int maxVersions = Math.min(scan.getMaxVersions(), scanInfo.getMaxVersions());
+    // Single branch to deal with two types of reads (columns vs all in family)
+    if (columns == null || columns.size() == 0) {
+      // there is always a null column in the wildcard column query.
+      hasNullColumn = true;
+
+      // use a specialized scan for wildcard column tracker.
+      this.columns = new ScanWildcardColumnTracker(
+          scanInfo.getMinVersions(), maxVersions, oldestUnexpiredTS);
+    } else {
+      // whether there is null column in the explicit column query
+      hasNullColumn = (columns.first().length == 0);
+
+      // We can share the ExplicitColumnTracker, diff is we reset
+      // between rows, not between storefiles.
+      this.columns = new ExplicitColumnTracker(columns,
+          scanInfo.getMinVersions(), maxVersions, oldestUnexpiredTS);
+    }
+  }
+
+  /*
+   * Constructor for tests
+   */
+  ScanQueryMatcher(Scan scan, Store.ScanInfo scanInfo,
+      NavigableSet columns, long oldestUnexpiredTS) {
+    this(scan, scanInfo, columns, ScanType.USER_SCAN,
+          Long.MAX_VALUE, /* max Readpoint to track versions */
+        HConstants.LATEST_TIMESTAMP, oldestUnexpiredTS);
+  }
+
+  /**
+   *
+   * @return  whether there is an null column in the query
+   */
+  public boolean hasNullColumnInQuery() {
+    return hasNullColumn;
+  }
+
+  /**
+   * Determines if the caller should do one of several things:
+   * - seek/skip to the next row (MatchCode.SEEK_NEXT_ROW)
+   * - seek/skip to the next column (MatchCode.SEEK_NEXT_COL)
+   * - include the current KeyValue (MatchCode.INCLUDE)
+   * - ignore the current KeyValue (MatchCode.SKIP)
+   * - got to the next row (MatchCode.DONE)
+   *
+   * @param kv KeyValue to check
+   * @return The match code instance.
+   * @throws IOException in case there is an internal consistency problem
+   *      caused by a data corruption.
+   */
+  public MatchCode match(KeyValue kv) throws IOException {
+    if (filter != null && filter.filterAllRemaining()) {
+      return MatchCode.DONE_SCAN;
+    }
+
+    byte [] bytes = kv.getBuffer();
+    int offset = kv.getOffset();
+    int initialOffset = offset;
+
+    int keyLength = Bytes.toInt(bytes, offset, Bytes.SIZEOF_INT);
+    offset += KeyValue.ROW_OFFSET;
+
+    short rowLength = Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT);
+    offset += Bytes.SIZEOF_SHORT;
+
+    int ret = this.rowComparator.compareRows(row, this.rowOffset, this.rowLength,
+        bytes, offset, rowLength);
+    if (ret <= -1) {
+      return MatchCode.DONE;
+    } else if (ret >= 1) {
+      // could optimize this, if necessary?
+      // Could also be called SEEK_TO_CURRENT_ROW, but this
+      // should be rare/never happens.
+      return MatchCode.SEEK_NEXT_ROW;
+    }
+
+    // optimize case.
+    if (this.stickyNextRow)
+        return MatchCode.SEEK_NEXT_ROW;
+
+    if (this.columns.done()) {
+      stickyNextRow = true;
+      return MatchCode.SEEK_NEXT_ROW;
+    }
+
+    //Passing rowLength
+    offset += rowLength;
+
+    //Skipping family
+    byte familyLength = bytes [offset];
+    offset += familyLength + 1;
+
+    int qualLength = keyLength + KeyValue.ROW_OFFSET -
+      (offset - initialOffset) - KeyValue.TIMESTAMP_TYPE_SIZE;
+
+    long timestamp = kv.getTimestamp();
+    // check for early out based on timestamp alone
+    if (columns.isDone(timestamp)) {
+        return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
+    }
+
+    /*
+     * The delete logic is pretty complicated now.
+     * This is corroborated by the following:
+     * 1. The store might be instructed to keep deleted rows around.
+     * 2. A scan can optionally see past a delete marker now.
+     * 3. If deleted rows are kept, we have to find out when we can
+     *    remove the delete markers.
+     * 4. Family delete markers are always first (regardless of their TS)
+     * 5. Delete markers should not be counted as version
+     * 6. Delete markers affect puts of the *same* TS
+     * 7. Delete marker need to be version counted together with puts
+     *    they affect
+     */
+    byte type = kv.getType();
+    if (kv.isDelete()) {
+      if (!keepDeletedCells) {
+        // first ignore delete markers if the scanner can do so, and the
+        // range does not include the marker
+        //
+        // during flushes and compactions also ignore delete markers newer
+        // than the readpoint of any open scanner, this prevents deleted
+        // rows that could still be seen by a scanner from being collected
+        boolean includeDeleteMarker = seePastDeleteMarkers ?
+            tr.withinTimeRange(timestamp) :
+            tr.withinOrAfterTimeRange(timestamp);
+        if (includeDeleteMarker
+            && kv.getMemstoreTS() <= maxReadPointToTrackVersions) {
+          this.deletes.add(bytes, offset, qualLength, timestamp, type);
+        }
+        // Can't early out now, because DelFam come before any other keys
+      }
+      if (retainDeletesInOutput
+          || (!isUserScan && (EnvironmentEdgeManager.currentTimeMillis() - timestamp) <= timeToPurgeDeletes)
+          || kv.getMemstoreTS() > maxReadPointToTrackVersions) {
+        // always include or it is not time yet to check whether it is OK
+        // to purge deltes or not
+        return MatchCode.INCLUDE;
+      } else if (keepDeletedCells) {
+        if (timestamp < earliestPutTs) {
+          // keeping delete rows, but there are no puts older than
+          // this delete in the store files.
+          return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
+        }
+        // else: fall through and do version counting on the
+        // delete markers
+      } else {
+        return MatchCode.SKIP;
+      }
+      // note the following next else if...
+      // delete marker are not subject to other delete markers
+    } else if (!this.deletes.isEmpty()) {
+      DeleteResult deleteResult = deletes.isDeleted(bytes, offset, qualLength,
+          timestamp);
+      switch (deleteResult) {
+        case FAMILY_DELETED:
+        case COLUMN_DELETED:
+          return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
+        case VERSION_DELETED:
+          return MatchCode.SKIP;
+        case NOT_DELETED:
+          break;
+        default:
+          throw new RuntimeException("UNEXPECTED");
+        }
+    }
+
+    int timestampComparison = tr.compare(timestamp);
+    if (timestampComparison >= 1) {
+      return MatchCode.SKIP;
+    } else if (timestampComparison <= -1) {
+      return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
+    }
+
+    /**
+     * Filters should be checked before checking column trackers. If we do
+     * otherwise, as was previously being done, ColumnTracker may increment its
+     * counter for even that KV which may be discarded later on by Filter. This
+     * would lead to incorrect results in certain cases.
+     */
+    ReturnCode filterResponse = ReturnCode.SKIP;
+    if (filter != null) {
+      filterResponse = filter.filterKeyValue(kv);
+      if (filterResponse == ReturnCode.SKIP) {
+        return MatchCode.SKIP;
+      } else if (filterResponse == ReturnCode.NEXT_COL) {
+        return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
+      } else if (filterResponse == ReturnCode.NEXT_ROW) {
+        stickyNextRow = true;
+        return MatchCode.SEEK_NEXT_ROW;
+      } else if (filterResponse == ReturnCode.SEEK_NEXT_USING_HINT) {
+        return MatchCode.SEEK_NEXT_USING_HINT;
+      }
+    }
+
+    MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength,
+        timestamp, type, kv.getMemstoreTS() > maxReadPointToTrackVersions);
+    /*
+     * According to current implementation, colChecker can only be
+     * SEEK_NEXT_COL, SEEK_NEXT_ROW, SKIP or INCLUDE. Therefore, always return
+     * the MatchCode. If it is SEEK_NEXT_ROW, also set stickyNextRow.
+     */
+    if (colChecker == MatchCode.SEEK_NEXT_ROW) {
+      stickyNextRow = true;
+    } else if (filter != null && colChecker == MatchCode.INCLUDE &&
+               filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL) {
+      return MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
+    }
+    return colChecker;
+
+  }
+
+  public boolean moreRowsMayExistAfter(KeyValue kv) {
+    if (!Bytes.equals(stopRow , HConstants.EMPTY_END_ROW) &&
+        rowComparator.compareRows(kv.getBuffer(),kv.getRowOffset(),
+            kv.getRowLength(), stopRow, 0, stopRow.length) >= 0) {
+      // KV >= STOPROW
+      // then NO there is nothing left.
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  /**
+   * Set current row
+   * @param row
+   */
+  public void setRow(byte [] row, int offset, short length) {
+    this.row = row;
+    this.rowOffset = offset;
+    this.rowLength = length;
+    reset();
+  }
+
+  public void reset() {
+    this.deletes.reset();
+    this.columns.reset();
+
+    stickyNextRow = false;
+  }
+
+  /**
+   *
+   * @return the start key
+   */
+  public KeyValue getStartKey() {
+    return this.startKey;
+  }
+
+  /**
+   *
+   * @return the Filter
+   */
+  Filter getFilter() {
+    return this.filter;
+  }
+
+  public KeyValue getNextKeyHint(KeyValue kv) {
+    if (filter == null) {
+      return null;
+    } else {
+      return filter.getNextKeyHint(kv);
+    }
+  }
+
+  public KeyValue getKeyForNextColumn(KeyValue kv) {
+    ColumnCount nextColumn = columns.getColumnHint();
+    if (nextColumn == null) {
+      return KeyValue.createLastOnRow(
+          kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
+          kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(),
+          kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength());
+    } else {
+      return KeyValue.createFirstOnRow(
+          kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
+          kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(),
+          nextColumn.getBuffer(), nextColumn.getOffset(), nextColumn.getLength());
+    }
+  }
+
+  public KeyValue getKeyForNextRow(KeyValue kv) {
+    return KeyValue.createLastOnRow(
+        kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
+        null, 0, 0,
+        null, 0, 0);
+  }
+
+  /**
+   * {@link #match} return codes.  These instruct the scanner moving through
+   * memstores and StoreFiles what to do with the current KeyValue.
+   * 
                  
+   * Additionally, this contains "early-out" language to tell the scanner to
+   * move on to the next File (memstore or Storefile), or to return immediately.
+   */
+  public static enum MatchCode {
+    /**
+     * Include KeyValue in the returned result
+     */
+    INCLUDE,
+
+    /**
+     * Do not include KeyValue in the returned result
+     */
+    SKIP,
+
+    /**
+     * Do not include, jump to next StoreFile or memstore (in time order)
+     */
+    NEXT,
+
+    /**
+     * Do not include, return current result
+     */
+    DONE,
+
+    /**
+     * These codes are used by the ScanQueryMatcher
+     */
+
+    /**
+     * Done with the row, seek there.
+     */
+    SEEK_NEXT_ROW,
+    /**
+     * Done with column, seek to next.
+     */
+    SEEK_NEXT_COL,
+
+    /**
+     * Done with scan, thanks to the row filter.
+     */
+    DONE_SCAN,
+
+    /*
+     * Seek to next key which is given as hint.
+     */
+    SEEK_NEXT_USING_HINT,
+
+    /**
+     * Include KeyValue and done with column, seek to next.
+     */
+    INCLUDE_AND_SEEK_NEXT_COL,
+
+    /**
+     * Include KeyValue and done with row, seek to next.
+     */
+    INCLUDE_AND_SEEK_NEXT_ROW,
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java
new file mode 100644
index 0000000..5f96c6a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+/**
+ * Enum to distinguish general scan types.
+ */
+public enum ScanType {
+  MAJOR_COMPACT,
+  MINOR_COMPACT,
+  USER_SCAN
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
new file mode 100644
index 0000000..1402455
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
@@ -0,0 +1,203 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Keeps track of the columns for a scan if they are not explicitly specified
+ */
+public class ScanWildcardColumnTracker implements ColumnTracker {
+  private byte [] columnBuffer = null;
+  private int columnOffset = 0;
+  private int columnLength = 0;
+  private int currentCount = 0;
+  private int maxVersions;
+  private int minVersions;
+  /* Keeps track of the latest timestamp and type included for current column.
+   * Used to eliminate duplicates. */
+  private long latestTSOfCurrentColumn;
+  private byte latestTypeOfCurrentColumn;
+
+  private long oldestStamp;
+
+  /**
+   * Return maxVersions of every row.
+   * @param minVersion Minimum number of versions to keep
+   * @param maxVersion Maximum number of versions to return
+   * @param oldestUnexpiredTS oldest timestamp that has not expired according
+   *          to the TTL.
+   */
+  public ScanWildcardColumnTracker(int minVersion, int maxVersion,
+      long oldestUnexpiredTS) {
+    this.maxVersions = maxVersion;
+    this.minVersions = minVersion;
+    this.oldestStamp = oldestUnexpiredTS;
+  }
+
+  /**
+   * {@inheritDoc}
+   * This receives puts *and* deletes.
+   * Deletes do not count as a version, but rather take the version
+   * of the previous put (so eventually all but the last can be reclaimed).
+   */
+  @Override
+  public MatchCode checkColumn(byte[] bytes, int offset, int length,
+      long timestamp, byte type, boolean ignoreCount) throws IOException {
+    
+    if (columnBuffer == null) {
+      // first iteration.
+      resetBuffer(bytes, offset, length);
+      if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE;
+      // do not count a delete marker as another version
+      return checkVersion(type, timestamp);
+    }
+    int cmp = Bytes.compareTo(bytes, offset, length,
+        columnBuffer, columnOffset, columnLength);
+    if (cmp == 0) {
+      if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE;
+
+      //If column matches, check if it is a duplicate timestamp
+      if (sameAsPreviousTSAndType(timestamp, type)) {
+        return ScanQueryMatcher.MatchCode.SKIP;
+      }
+      return checkVersion(type, timestamp);
+    }
+
+    resetTSAndType();
+
+    // new col > old col
+    if (cmp > 0) {
+      // switched columns, lets do something.x
+      resetBuffer(bytes, offset, length);
+      if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE;
+      return checkVersion(type, timestamp);
+    }
+
+    // new col < oldcol
+    // WARNING: This means that very likely an edit for some other family
+    // was incorrectly stored into the store for this one. Throw an exception,
+    // because this might lead to data corruption.
+    throw new IOException(
+        "ScanWildcardColumnTracker.checkColumn ran into a column actually " +
+        "smaller than the previous column: " +
+        Bytes.toStringBinary(bytes, offset, length));
+  }
+
+  private void resetBuffer(byte[] bytes, int offset, int length) {
+    columnBuffer = bytes;
+    columnOffset = offset;
+    columnLength = length;
+    currentCount = 0;
+  }
+
+  /**
+   * Check whether this version should be retained.
+   * There are 4 variables considered:
+   * If this version is past max versions -> skip it
+   * If this kv has expired or was deleted, check min versions
+   * to decide whther to skip it or not.
+   *
+   * Increase the version counter unless this is a delete
+   */
+  private MatchCode checkVersion(byte type, long timestamp) {
+    if (!KeyValue.isDelete(type)) {
+      currentCount++;
+    }
+    if (currentCount > maxVersions) {
+      return ScanQueryMatcher.MatchCode.SEEK_NEXT_COL; // skip to next col
+    }
+    // keep the KV if required by minversions or it is not expired, yet
+    if (currentCount <= minVersions || !isExpired(timestamp)) {
+      setTSAndType(timestamp, type);
+      return ScanQueryMatcher.MatchCode.INCLUDE;
+    } else {
+      return MatchCode.SEEK_NEXT_COL;
+    }
+
+  }
+
+  @Override
+  public void update() {
+    // no-op, shouldn't even be called
+    throw new UnsupportedOperationException(
+        "ScanWildcardColumnTracker.update should never be called!");
+  }
+
+  @Override
+  public void reset() {
+    columnBuffer = null;
+    resetTSAndType();
+  }
+
+  private void resetTSAndType() {
+    latestTSOfCurrentColumn = HConstants.LATEST_TIMESTAMP;
+    latestTypeOfCurrentColumn = 0;
+  }
+
+  private void setTSAndType(long timestamp, byte type) {
+    latestTSOfCurrentColumn = timestamp;
+    latestTypeOfCurrentColumn = type;
+  }
+
+  private boolean sameAsPreviousTSAndType(long timestamp, byte type) {
+    return timestamp == latestTSOfCurrentColumn && type == latestTypeOfCurrentColumn;
+  }
+
+  private boolean isExpired(long timestamp) {
+    return timestamp < oldestStamp;
+  }
+
+  /**
+   * Used by matcher and scan/get to get a hint of the next column
+   * to seek to after checkColumn() returns SKIP.  Returns the next interesting
+   * column we want, or NULL there is none (wildcard scanner).
+   *
+   * @return The column count.
+   */
+  public ColumnCount getColumnHint() {
+    return null;
+  }
+
+
+  /**
+   * We can never know a-priori if we are done, so always return false.
+   * @return false
+   */
+  @Override
+  public boolean done() {
+    return false;
+  }
+
+  public MatchCode getNextRowOrNextColumn(byte[] bytes, int offset,
+      int qualLength) {
+    return MatchCode.SEEK_NEXT_COL;
+  }
+
+  public boolean isDone(long timestamp) {
+    return minVersions <= 0 && isExpired(timestamp);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
new file mode 100644
index 0000000..5f16535
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHook.java
@@ -0,0 +1,259 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.util.ShutdownHookManager;
+import org.apache.hadoop.hbase.util.Threads;
+
+/**
+ * Manage regionserver shutdown hooks.
+ * @see #install(Configuration, FileSystem, Stoppable, Thread)
+ */
+public class ShutdownHook {
+  private static final Log LOG = LogFactory.getLog(ShutdownHook.class);
+  private static final String CLIENT_FINALIZER_DATA_METHOD = "clientFinalizer";
+
+  /**
+   * Key for boolean configuration whose default is true.
+   */
+  public static final String RUN_SHUTDOWN_HOOK = "hbase.shutdown.hook";
+
+  /**
+   * Key for a long configuration on how much time to wait on the fs shutdown
+   * hook. Default is 30 seconds.
+   */
+  public static final String FS_SHUTDOWN_HOOK_WAIT = "hbase.fs.shutdown.hook.wait";
+
+  /**
+   * A place for keeping track of all the filesystem shutdown hooks that need
+   * to be executed after the last regionserver referring to a given filesystem
+   * stops. We keep track of the # of regionserver references in values of the map.
+   */
+  private final static Map fsShutdownHooks = new HashMap();
+
+  /**
+   * Install a shutdown hook that calls stop on the passed Stoppable
+   * and then thread joins against the passed threadToJoin.
+   * When this thread completes, it then runs the hdfs thread (This install
+   * removes the hdfs shutdown hook keeping a handle on it to run it after
+   * threadToJoin has stopped).
+   *
+   * 
                  To suppress all shutdown hook  handling -- both the running of the
+   * regionserver hook and of the hdfs hook code -- set
+   * {@link ShutdownHook#RUN_SHUTDOWN_HOOK} in {@link Configuration} to
+   * false.
+   * This configuration value is checked when the hook code runs.
+   * @param conf
+   * @param fs Instance of Filesystem used by the RegionServer
+   * @param stop Installed shutdown hook will call stop against this passed
+   * Stoppable instance.
+   * @param threadToJoin After calling stop on stop will then
+   * join this thread.
+   */
+  public static void install(final Configuration conf, final FileSystem fs,
+      final Stoppable stop, final Thread threadToJoin) {
+    Runnable fsShutdownHook = suppressHdfsShutdownHook(fs);
+    Thread t = new ShutdownHookThread(conf, stop, threadToJoin, fsShutdownHook);
+    ShutdownHookManager.affixShutdownHook(t, 0);
+    LOG.info("Installed shutdown hook thread: " + t.getName());
+  }
+
+  /*
+   * Thread run by shutdown hook.
+   */
+  private static class ShutdownHookThread extends Thread {
+    private final Stoppable stop;
+    private final Thread threadToJoin;
+    private final Runnable fsShutdownHook;
+    private final Configuration conf;
+
+    ShutdownHookThread(final Configuration conf, final Stoppable stop,
+        final Thread threadToJoin, final Runnable fsShutdownHook) {
+      super("Shutdownhook:" + threadToJoin.getName());
+      this.stop = stop;
+      this.threadToJoin = threadToJoin;
+      this.conf = conf;
+      this.fsShutdownHook = fsShutdownHook;
+    }
+
+    @Override
+    public void run() {
+      boolean b = this.conf.getBoolean(RUN_SHUTDOWN_HOOK, true);
+      LOG.info("Shutdown hook starting; " + RUN_SHUTDOWN_HOOK + "=" + b +
+        "; fsShutdownHook=" + this.fsShutdownHook);
+      if (b) {
+        this.stop.stop("Shutdown hook");
+        Threads.shutdown(this.threadToJoin);
+        if (this.fsShutdownHook != null) {
+          synchronized (fsShutdownHooks) {
+            int refs = fsShutdownHooks.get(fsShutdownHook);
+            if (refs == 1) {
+              LOG.info("Starting fs shutdown hook thread.");
+              Thread fsShutdownHookThread = (fsShutdownHook instanceof Thread) ?
+                (Thread)fsShutdownHook : new Thread(fsShutdownHook);
+              fsShutdownHookThread.start();
+              Threads.shutdown(fsShutdownHookThread,
+              this.conf.getLong(FS_SHUTDOWN_HOOK_WAIT, 30000));
+            }
+            if (refs > 0) {
+              fsShutdownHooks.put(fsShutdownHook, refs - 1);
+            }
+          }
+        }
+      }
+      LOG.info("Shutdown hook finished.");
+    }
+  }
+
+  /*
+   * So, HDFS keeps a static map of all FS instances. In order to make sure
+   * things are cleaned up on our way out, it also creates a shutdown hook
+   * so that all filesystems can be closed when the process is terminated; it
+   * calls FileSystem.closeAll. This inconveniently runs concurrently with our
+   * own shutdown handler, and therefore causes all the filesystems to be closed
+   * before the server can do all its necessary cleanup.
+   *
+   * 
                  The dirty reflection in this method sneaks into the FileSystem class
+   * and grabs the shutdown hook, removes it from the list of active shutdown
+   * hooks, and returns the hook for the caller to run at its convenience.
+   *
+   * 
                  This seems quite fragile and susceptible to breaking if Hadoop changes
+   * anything about the way this cleanup is managed. Keep an eye on things.
+   * @return The fs shutdown hook
+   * @throws RuntimeException if we fail to find or grap the shutdown hook.
+   */
+  private static Runnable suppressHdfsShutdownHook(final FileSystem fs) {
+    try {
+      // This introspection has been updated to work for hadoop 0.20, 0.21 and for
+      // cloudera 0.20.  0.21 and cloudera 0.20 both have hadoop-4829.  With the
+      // latter in place, things are a little messy in that there are now two
+      // instances of the data member clientFinalizer; an uninstalled one in
+      // FileSystem and one in the innner class named Cache that actually gets
+      // registered as a shutdown hook.  If the latter is present, then we are
+      // on 0.21 or cloudera patched 0.20.
+      Runnable hdfsClientFinalizer = null;
+      // Look into the FileSystem#Cache class for clientFinalizer
+      Class [] classes = FileSystem.class.getDeclaredClasses();
+      Class cache = null;
+      for (Class c: classes) {
+        if (c.getSimpleName().equals("Cache")) {
+          cache = c;
+          break;
+        }
+      }
+      Field field = null;
+      try {
+        field = cache.getDeclaredField(CLIENT_FINALIZER_DATA_METHOD);
+      } catch (NoSuchFieldException e) {
+        // We can get here if the Cache class does not have a clientFinalizer
+        // instance: i.e. we're running on straight 0.20 w/o hadoop-4829.
+      }
+      if (field != null) {
+        field.setAccessible(true);
+        Field cacheField = FileSystem.class.getDeclaredField("CACHE");
+        cacheField.setAccessible(true);
+        Object cacheInstance = cacheField.get(fs);
+        hdfsClientFinalizer = (Runnable)field.get(cacheInstance);
+      } else {
+        // Then we didnt' find clientFinalizer in Cache.  Presume clean 0.20 hadoop.
+        field = FileSystem.class.getDeclaredField(CLIENT_FINALIZER_DATA_METHOD);
+        field.setAccessible(true);
+        hdfsClientFinalizer = (Runnable)field.get(null);
+      }
+      if (hdfsClientFinalizer == null) {
+        throw new RuntimeException("Client finalizer is null, can't suppress!");
+      }
+      if (!fsShutdownHooks.containsKey(hdfsClientFinalizer) &&
+          !ShutdownHookManager.deleteShutdownHook(hdfsClientFinalizer)) {
+        throw new RuntimeException("Failed suppression of fs shutdown hook: " +
+          hdfsClientFinalizer);
+      }
+      synchronized (fsShutdownHooks) {
+        Integer refs = fsShutdownHooks.get(hdfsClientFinalizer);
+        fsShutdownHooks.put(hdfsClientFinalizer, refs == null ? 1 : refs + 1);
+      }
+      return hdfsClientFinalizer;
+    } catch (NoSuchFieldException nsfe) {
+      LOG.fatal("Couldn't find field 'clientFinalizer' in FileSystem!", nsfe);
+      throw new RuntimeException("Failed to suppress HDFS shutdown hook");
+    } catch (IllegalAccessException iae) {
+      LOG.fatal("Couldn't access field 'clientFinalizer' in FileSystem!", iae);
+      throw new RuntimeException("Failed to suppress HDFS shutdown hook");
+    }
+  }
+
+  // Thread that does nothing. Used in below main testing.
+  static class DoNothingThread extends Thread {
+    DoNothingThread() {
+      super("donothing");
+    }
+    @Override
+    public void run() {
+      super.run();
+    }
+  }
+
+  // Stoppable with nothing to stop.  Used below in main testing.
+  static class DoNothingStoppable implements Stoppable {
+    @Override
+    public boolean isStopped() {
+      // TODO Auto-generated method stub
+      return false;
+    }
+
+    @Override
+    public void stop(String why) {
+      // TODO Auto-generated method stub
+    }
+  }
+
+  /**
+   * Main to test basic functionality.  Run with clean hadoop 0.20 and hadoop
+   * 0.21 and cloudera patched hadoop to make sure our shutdown hook handling
+   * works for all compbinations.
+   * Pass '-Dhbase.shutdown.hook=false' to test turning off the running of
+   * shutdown hooks.
+   * @param args
+   * @throws IOException
+   */
+  public static void main(final String [] args) throws IOException {
+    Configuration conf = HBaseConfiguration.create();
+    String prop = System.getProperty(RUN_SHUTDOWN_HOOK);
+    if (prop != null) {
+      conf.setBoolean(RUN_SHUTDOWN_HOOK, Boolean.parseBoolean(prop));
+    }
+    // Instantiate a FileSystem. This will register the fs shutdown hook.
+    FileSystem fs = FileSystem.get(conf);
+    Thread donothing = new DoNothingThread();
+    donothing.start();
+    ShutdownHook.install(conf, fs, new DoNothingStoppable(), donothing);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
new file mode 100644
index 0000000..0dbc4f9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
@@ -0,0 +1,582 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.*;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.master.SplitLogManager;
+import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
+import org.apache.hadoop.hbase.util.CancelableProgressable;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
+import org.apache.hadoop.hbase.zookeeper.ZKSplitLog.TaskState;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ * This worker is spawned in every regionserver (should we also spawn one in
+ * the master?). The Worker waits for log splitting tasks to be put up by the
+ * {@link SplitLogManager} running in the master and races with other workers
+ * in other serves to acquire those tasks. The coordination is done via
+ * zookeeper. All the action takes place at /hbase/splitlog znode.
+ * 
                  
+ * If a worker has successfully moved the task from state UNASSIGNED to
+ * OWNED then it owns the task. It keeps heart beating the manager by
+ * periodically moving the task from UNASSIGNED to OWNED state. On success it
+ * moves the task to TASK_DONE. On unrecoverable error it moves task state to
+ * ERR. If it cannot continue but wants the master to retry the task then it
+ * moves the task state to RESIGNED.
+ * 
                  
+ * The manager can take a task away from a worker by moving the task from
+ * OWNED to UNASSIGNED. In the absence of a global lock there is a
+ * unavoidable race here - a worker might have just finished its task when it
+ * is stripped of its ownership. Here we rely on the idempotency of the log
+ * splitting task for correctness
+ */
+public class SplitLogWorker extends ZooKeeperListener implements Runnable {
+  private static final Log LOG = LogFactory.getLog(SplitLogWorker.class);
+
+  Thread worker;
+  private final String serverName;
+  private final TaskExecutor splitTaskExecutor;
+  private long zkretries;
+
+  private Object taskReadyLock = new Object();
+  volatile int taskReadySeq = 0;
+  private volatile String currentTask = null;
+  private int currentVersion;
+  private volatile boolean exitWorker;
+  private Object grabTaskLock = new Object();
+  private boolean workerInGrabTask = false;
+
+
+  public SplitLogWorker(ZooKeeperWatcher watcher, Configuration conf,
+      String serverName, TaskExecutor splitTaskExecutor) {
+    super(watcher);
+    this.serverName = serverName;
+    this.splitTaskExecutor = splitTaskExecutor;
+    this.zkretries = conf.getLong("hbase.splitlog.zk.retries", 3);
+  }
+
+  public SplitLogWorker(ZooKeeperWatcher watcher, final Configuration conf,
+      final String serverName) {
+    this(watcher, conf, serverName, new TaskExecutor () {
+      @Override
+      public Status exec(String filename, CancelableProgressable p) {
+        Path rootdir;
+        FileSystem fs;
+        try {
+          rootdir = FSUtils.getRootDir(conf);
+          fs = rootdir.getFileSystem(conf);
+        } catch (IOException e) {
+          LOG.warn("could not find root dir or fs", e);
+          return Status.RESIGNED;
+        }
+        // TODO have to correctly figure out when log splitting has been
+        // interrupted or has encountered a transient error and when it has
+        // encountered a bad non-retry-able persistent error.
+        try {          
+          String relativeLogPath = getRelativeLogPath(filename);
+          if (HLogSplitter.splitLogFile(rootdir,
+              fs.getFileStatus(new Path(rootdir, relativeLogPath)), fs, conf, p) == false) {
+            return Status.PREEMPTED;
+          }
+        } catch (InterruptedIOException iioe) {
+          LOG.warn("log splitting of " + filename + " interrupted, resigning",
+              iioe);
+          return Status.RESIGNED;
+        } catch (IOException e) {
+          Throwable cause = e.getCause();
+          if (cause instanceof InterruptedException) {
+            LOG.warn("log splitting of " + filename + " interrupted, resigning",
+                e);
+            return Status.RESIGNED;
+          }
+          LOG.warn("log splitting of " + filename + " failed, returning error",
+              e);
+          return Status.ERR;
+        }
+        return Status.DONE;
+      }
+
+      private String getRelativeLogPath(String logPath) {
+        StringBuilder sb = new StringBuilder();
+        String znodeDelimiter = Character.toString(Path.SEPARATOR_CHAR);
+        String[] filenameSplits = logPath.split(znodeDelimiter);
+        int len = filenameSplits.length;
+        String relativeLogPath = logPath;
+        if (len > 3) {
+          // the last three terms are .logs/server/log-file
+          relativeLogPath = sb.append(filenameSplits[len - 3]).append(znodeDelimiter)
+            .append(filenameSplits[len - 2]).append(znodeDelimiter)
+            .append(filenameSplits[len - 1]).toString();
+        }
+        return relativeLogPath;
+      }
+    });
+  }
+
+  @Override
+  public void run() {
+   try {
+    LOG.info("SplitLogWorker " + this.serverName + " starting");
+    this.watcher.registerListener(this);
+    int res;
+    // wait for master to create the splitLogZnode
+    res = -1;
+    while (res == -1) {
+      try {
+        res = ZKUtil.checkExists(watcher, watcher.splitLogZNode);
+      } catch (KeeperException e) {
+        // ignore
+        LOG.warn("Exception when checking for " + watcher.splitLogZNode +
+            " ... retrying", e);
+      }
+      if (res == -1) {
+        try {
+          LOG.info(watcher.splitLogZNode + " znode does not exist," +
+              " waiting for master to create one");
+          Thread.sleep(1000);
+        } catch (InterruptedException e) {
+          LOG.debug("Interrupted while waiting for " + watcher.splitLogZNode);
+          assert exitWorker == true;
+        }
+      }
+    }
+
+    taskLoop();
+   } catch (Throwable t) {
+	   // only a logical error can cause here. Printing it out 
+	   // to make debugging easier
+	   LOG.error("unexpected error ", t);
+   } finally {
+	   LOG.info("SplitLogWorker " + this.serverName + " exiting");
+   }
+  }
+
+  /**
+   * Wait for tasks to become available at /hbase/splitlog zknode. Grab a task
+   * one at a time. This policy puts an upper-limit on the number of
+   * simultaneous log splitting that could be happening in a cluster.
+   * 
                  
+   * Synchronization using {@link #task_ready_signal_seq} ensures that it will
+   * try to grab every task that has been put up
+   */
+  private void taskLoop() {
+    while (true) {
+      int seq_start = taskReadySeq;
+      List paths = getTaskList();
+      if (paths == null) {
+        LOG.warn("Could not get tasks, did someone remove " +
+            this.watcher.splitLogZNode + " ... worker thread exiting.");
+        return;
+      }
+      int offset = (int)(Math.random() * paths.size());
+      for (int i = 0; i < paths.size(); i ++) {
+        int idx = (i + offset) % paths.size();
+        // don't call ZKSplitLog.getNodeName() because that will lead to
+        // double encoding of the path name
+        grabTask(ZKUtil.joinZNode(watcher.splitLogZNode, paths.get(idx)));
+        if (exitWorker == true) {
+          return;
+        }
+      }
+      synchronized (taskReadyLock) {
+        while (seq_start == taskReadySeq) {
+          try {
+            taskReadyLock.wait();
+          } catch (InterruptedException e) {
+            LOG.info("SplitLogWorker interrupted while waiting for task," +
+              " exiting: " + e.toString());
+            assert exitWorker == true;
+            return;
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * try to grab a 'lock' on the task zk node to own and execute the task.
+   * 
                  
+   * @param path zk node for the task
+   */
+  private void grabTask(String path) {
+    Stat stat = new Stat();
+    long t = -1;
+    byte[] data;
+    synchronized (grabTaskLock) {
+      currentTask = path;
+      workerInGrabTask = true;
+      if (Thread.interrupted()) {
+        return;
+      }
+    }
+    try {
+      try {
+        if ((data = ZKUtil.getDataNoWatch(this.watcher, path, stat)) == null) {
+          tot_wkr_failed_to_grab_task_no_data.incrementAndGet();
+          return;
+        }
+      } catch (KeeperException e) {
+        LOG.warn("Failed to get data for znode " + path, e);
+        tot_wkr_failed_to_grab_task_exception.incrementAndGet();
+        return;
+      }
+      if (TaskState.TASK_UNASSIGNED.equals(data) == false) {
+        tot_wkr_failed_to_grab_task_owned.incrementAndGet();
+        return;
+      }
+
+      currentVersion = stat.getVersion();
+      if (attemptToOwnTask(true) == false) {
+        tot_wkr_failed_to_grab_task_lost_race.incrementAndGet();
+        return;
+      }
+
+      if (ZKSplitLog.isRescanNode(watcher, currentTask)) {
+        endTask(TaskState.TASK_DONE, tot_wkr_task_acquired_rescan);
+        return;
+      }
+      LOG.info("worker " + serverName + " acquired task " + path);
+      tot_wkr_task_acquired.incrementAndGet();
+      getDataSetWatchAsync();
+
+      t = System.currentTimeMillis();
+      TaskExecutor.Status status;
+
+      status = splitTaskExecutor.exec(ZKSplitLog.getFileName(currentTask),
+          new CancelableProgressable() {
+
+        @Override
+        public boolean progress() {
+          if (attemptToOwnTask(false) == false) {
+            LOG.warn("Failed to heartbeat the task" + currentTask);
+            return false;
+          }
+          return true;
+        }
+      });
+      switch (status) {
+        case DONE:
+          endTask(TaskState.TASK_DONE, tot_wkr_task_done);
+          break;
+        case PREEMPTED:
+          tot_wkr_preempt_task.incrementAndGet();
+          LOG.warn("task execution prempted " + path);
+          break;
+        case ERR:
+          if (!exitWorker) {
+            endTask(TaskState.TASK_ERR, tot_wkr_task_err);
+            break;
+          }
+          // if the RS is exiting then there is probably a tons of stuff
+          // that can go wrong. Resign instead of signaling error.
+          //$FALL-THROUGH$
+        case RESIGNED:
+          if (exitWorker) {
+            LOG.info("task execution interrupted because worker is exiting " +
+                path);
+            endTask(TaskState.TASK_RESIGNED, tot_wkr_task_resigned);
+          } else {
+            tot_wkr_preempt_task.incrementAndGet();
+            LOG.info("task execution interrupted via zk by manager " +
+                path);
+          }
+          break;
+      }
+    } finally {
+      if (t > 0) {
+        LOG.info("worker " + serverName + " done with task " + path +
+            " in " + (System.currentTimeMillis() - t) + "ms");
+      }
+      synchronized (grabTaskLock) {
+        workerInGrabTask = false;
+        // clear the interrupt from stopTask() otherwise the next task will
+        // suffer
+        Thread.interrupted();
+      }
+    }
+    return;
+  }
+
+  /**
+   * Try to own the task by transitioning the zk node data from UNASSIGNED to
+   * OWNED.
+   * 
                  
+   * This method is also used to periodically heartbeat the task progress by
+   * transitioning the node from OWNED to OWNED.
+   * 
                  
+   * @return true if task path is successfully locked
+   */
+  private boolean attemptToOwnTask(boolean isFirstTime) {
+    try {
+      Stat stat = this.watcher.getRecoverableZooKeeper().setData(currentTask,
+          TaskState.TASK_OWNED.get(serverName), currentVersion);
+      if (stat == null) {
+        LOG.warn("zk.setData() returned null for path " + currentTask);
+        tot_wkr_task_heartbeat_failed.incrementAndGet();
+        return (false);
+      }
+      currentVersion = stat.getVersion();
+      tot_wkr_task_heartbeat.incrementAndGet();
+      return (true);
+    } catch (KeeperException e) {
+      if (!isFirstTime) {
+        if (e.code().equals(KeeperException.Code.NONODE)) {
+          LOG.warn("NONODE failed to assert ownership for " + currentTask, e);
+        } else if (e.code().equals(KeeperException.Code.BADVERSION)) {
+          LOG.warn("BADVERSION failed to assert ownership for " +
+              currentTask, e);
+        } else {
+          LOG.warn("failed to assert ownership for " + currentTask, e);
+        }
+      }
+    } catch (InterruptedException e1) {
+      LOG.warn("Interrupted while trying to assert ownership of " +
+          currentTask + " " + StringUtils.stringifyException(e1));
+      Thread.currentThread().interrupt();
+    }
+    tot_wkr_task_heartbeat_failed.incrementAndGet();
+    return (false);
+  }
+
+  /**
+   * endTask() can fail and the only way to recover out of it is for the
+   * {@link SplitLogManager} to timeout the task node.
+   * @param ts
+   * @param ctr
+   */
+  private void endTask(ZKSplitLog.TaskState ts, AtomicLong ctr) {
+    String path = currentTask;
+    currentTask = null;
+    try {
+      if (ZKUtil.setData(this.watcher, path, ts.get(serverName),
+          currentVersion)) {
+        LOG.info("successfully transitioned task " + path +
+            " to final state " + ts);
+        ctr.incrementAndGet();
+        return;
+      }
+      LOG.warn("failed to transistion task " + path + " to end state " + ts +
+          " because of version mismatch ");
+    } catch (KeeperException.BadVersionException bve) {
+      LOG.warn("transisition task " + path + " to " + ts +
+          " failed because of version mismatch", bve);
+    } catch (KeeperException.NoNodeException e) {
+      LOG.fatal("logic error - end task " + path + " " + ts +
+          " failed because task doesn't exist", e);
+    } catch (KeeperException e) {
+      LOG.warn("failed to end task, " + path + " " + ts, e);
+    }
+    tot_wkr_final_transistion_failed.incrementAndGet();
+    return;
+  }
+
+  void getDataSetWatchAsync() {
+    this.watcher.getRecoverableZooKeeper().getZooKeeper().
+      getData(currentTask, this.watcher,
+      new GetDataAsyncCallback(), null);
+    tot_wkr_get_data_queued.incrementAndGet();
+  }
+
+  void getDataSetWatchSuccess(String path, byte[] data) {
+    synchronized (grabTaskLock) {
+      if (workerInGrabTask) {
+        // currentTask can change but that's ok
+        String taskpath = currentTask;
+        if (taskpath != null && taskpath.equals(path)) {
+          // have to compare data. cannot compare version because then there
+          // will be race with attemptToOwnTask()
+          // cannot just check whether the node has been transitioned to
+          // UNASSIGNED because by the time this worker sets the data watch
+          // the node might have made two transitions - from owned by this
+          // worker to unassigned to owned by another worker
+          if (! TaskState.TASK_OWNED.equals(data, serverName) &&
+              ! TaskState.TASK_DONE.equals(data, serverName) &&
+              ! TaskState.TASK_ERR.equals(data, serverName) &&
+              ! TaskState.TASK_RESIGNED.equals(data, serverName)) {
+            LOG.info("task " + taskpath + " preempted from " +
+                serverName + ", current task state and owner=" +
+                new String(data));
+            stopTask();
+          }
+        }
+      }
+    }
+  }
+
+  void getDataSetWatchFailure(String path) {
+    synchronized (grabTaskLock) {
+      if (workerInGrabTask) {
+        // currentTask can change but that's ok
+        String taskpath = currentTask;
+        if (taskpath != null && taskpath.equals(path)) {
+          LOG.info("retrying data watch on " + path);
+          tot_wkr_get_data_retry.incrementAndGet();
+          getDataSetWatchAsync();
+        } else {
+          // no point setting a watch on the task which this worker is not
+          // working upon anymore
+        }
+      }
+    }
+  }
+
+
+
+
+  @Override
+  public void nodeDataChanged(String path) {
+    // there will be a self generated dataChanged event every time attemptToOwnTask()
+    // heartbeats the task znode by upping its version
+    synchronized (grabTaskLock) {
+      if (workerInGrabTask) {
+        // currentTask can change
+        String taskpath = currentTask;
+        if (taskpath!= null && taskpath.equals(path)) {
+          getDataSetWatchAsync();
+        }
+      }
+    }
+  }
+
+
+  private List getTaskList() {
+    List childrenPaths = null;
+    long sleepTime = 1000;
+    // It will be in loop till it gets the list of children or
+    // it will come out if worker thread exited.
+    while (!exitWorker) {
+      try {
+        childrenPaths = ZKUtil.listChildrenAndWatchForNewChildren(this.watcher,
+            this.watcher.splitLogZNode);
+        if (childrenPaths != null) {
+          return childrenPaths;
+        }
+      } catch (KeeperException e) {
+        LOG.warn("Could not get children of znode "
+            + this.watcher.splitLogZNode, e);
+      }
+      try {
+        LOG.debug("Retry listChildren of znode " + this.watcher.splitLogZNode
+            + " after sleep for " + sleepTime + "ms!");
+        Thread.sleep(sleepTime);
+      } catch (InterruptedException e1) {
+        LOG.warn("Interrupted while trying to get task list ...", e1);
+        Thread.currentThread().interrupt();
+      }
+    }
+    return childrenPaths;
+  }
+
+
+  @Override
+  public void nodeChildrenChanged(String path) {
+    if(path.equals(watcher.splitLogZNode)) {
+      LOG.debug("tasks arrived or departed");
+      synchronized (taskReadyLock) {
+        taskReadySeq++;
+        taskReadyLock.notify();
+      }
+    }
+  }
+
+  /**
+   * If the worker is doing a task i.e. splitting a log file then stop the task.
+   * It doesn't exit the worker thread.
+   */
+  void stopTask() {
+    LOG.info("Sending interrupt to stop the worker thread");
+    worker.interrupt(); // TODO interrupt often gets swallowed, do what else?
+  }
+
+
+  /**
+   * start the SplitLogWorker thread
+   */
+  public void start() {
+    worker = new Thread(null, this, "SplitLogWorker-" + serverName);
+    exitWorker = false;
+    worker.start();
+    return;
+  }
+
+  /**
+   * stop the SplitLogWorker thread
+   */
+  public void stop() {
+    exitWorker = true;
+    stopTask();
+  }
+
+  /**
+   * Asynchronous handler for zk get-data-set-watch on node results.
+   */
+  class GetDataAsyncCallback implements AsyncCallback.DataCallback {
+    private final Log LOG = LogFactory.getLog(GetDataAsyncCallback.class);
+
+    @Override
+    public void processResult(int rc, String path, Object ctx, byte[] data,
+        Stat stat) {
+      tot_wkr_get_data_result.incrementAndGet();
+      if (rc != 0) {
+        LOG.warn("getdata rc = " + KeeperException.Code.get(rc) + " " + path);
+        getDataSetWatchFailure(path);
+        return;
+      }
+      data = watcher.getRecoverableZooKeeper().removeMetaData(data);
+      getDataSetWatchSuccess(path, data);
+      return;
+    }
+  }
+
+  /**
+   * Objects implementing this interface actually do the task that has been
+   * acquired by a {@link SplitLogWorker}. Since there isn't a water-tight
+   * guarantee that two workers will not be executing the same task therefore it
+   * is better to have workers prepare the task and then have the
+   * {@link SplitLogManager} commit the work in SplitLogManager.TaskFinisher
+   */
+  static public interface TaskExecutor {
+    static public enum Status {
+      DONE(),
+      ERR(),
+      RESIGNED(),
+      PREEMPTED();
+    }
+    public Status exec(String name, CancelableProgressable p);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
new file mode 100644
index 0000000..9ca424f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
@@ -0,0 +1,108 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Handles processing region splits. Put in a queue, owned by HRegionServer.
+ */
+class SplitRequest implements Runnable {
+  static final Log LOG = LogFactory.getLog(SplitRequest.class);
+  private final HRegion parent;
+  private final byte[] midKey;
+  private final HRegionServer server;
+
+  SplitRequest(HRegion region, byte[] midKey, HRegionServer hrs) {
+    Preconditions.checkNotNull(hrs);
+    this.parent = region;
+    this.midKey = midKey;
+    this.server = hrs;
+  }
+
+  @Override
+  public String toString() {
+    return "regionName=" + parent + ", midKey=" + Bytes.toStringBinary(midKey);
+  }
+
+  @Override
+  public void run() {
+    if (this.server.isStopping() || this.server.isStopped()) {
+      LOG.debug("Skipping split because server is stopping=" +
+        this.server.isStopping() + " or stopped=" + this.server.isStopped());
+      return;
+    }
+    try {
+      final long startTime = System.currentTimeMillis();
+      SplitTransaction st = new SplitTransaction(parent, midKey);
+      // If prepare does not return true, for some reason -- logged inside in
+      // the prepare call -- we are not ready to split just now. Just return.
+      if (!st.prepare()) return;
+      try {
+        st.execute(this.server, this.server);
+        this.server.getMetrics().incrementSplitSuccessCount();
+      } catch (Exception e) {
+        if (this.server.isStopping() || this.server.isStopped()) {
+          LOG.info(
+              "Skip rollback/cleanup of failed split of "
+                  + parent.getRegionNameAsString() + " because server is"
+                  + (this.server.isStopping() ? " stopping" : " stopped"), e);
+          return;
+        }
+        try {
+          LOG.info("Running rollback/cleanup of failed split of " +
+            parent.getRegionNameAsString() + "; " + e.getMessage(), e);
+          if (st.rollback(this.server, this.server)) {
+            LOG.info("Successful rollback of failed split of " +
+              parent.getRegionNameAsString());
+            this.server.getMetrics().incrementSplitFailureCount();
+          } else {
+            this.server.abort("Abort; we got an error after point-of-no-return");
+          }
+        } catch (RuntimeException ee) {
+          String msg = "Failed rollback of failed split of " +
+            parent.getRegionNameAsString() + " -- aborting server";
+          // If failed rollback, kill this server to avoid having a hole in table.
+          LOG.info(msg, ee);
+          this.server.abort(msg);
+        }
+        return;
+      }
+      LOG.info("Region split, META updated, and report to master. Parent="
+          + parent.getRegionInfo().getRegionNameAsString() + ", new regions: "
+          + st.getFirstDaughter().getRegionNameAsString() + ", "
+          + st.getSecondDaughter().getRegionNameAsString() + ". Split took "
+          + StringUtils.formatTimeDiff(System.currentTimeMillis(), startTime));
+    } catch (IOException ex) {
+      LOG.error("Split failed " + this, RemoteExceptionHandler
+          .checkIOException(ex));
+      this.server.getMetrics().incrementSplitFailureCount();
+      server.checkFileSystem();
+    }
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
new file mode 100644
index 0000000..a3a0c03
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -0,0 +1,1089 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
+import org.apache.hadoop.hbase.executor.RegionTransitionData;
+import org.apache.hadoop.hbase.io.Reference.Range;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CancelableProgressable;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HasThread;
+import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NodeExistsException;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * Executes region split as a "transaction".  Call {@link #prepare()} to setup
+ * the transaction, {@link #execute(Server, RegionServerServices)} to run the
+ * transaction and {@link #rollback(Server, RegionServerServices)} to cleanup if execute fails.
+ *
+ * 
                  Here is an example of how you would use this class:
+ * 
                  + *  SplitTransaction st = new SplitTransaction(this.conf, parent, midKey)
+ *  if (!st.prepare()) return;
+ *  try {
+ *    st.execute(server, services);
+ *  } catch (IOException ioe) {
+ *    try {
+ *      st.rollback(server, services);
+ *      return;
+ *    } catch (RuntimeException e) {
+ *      myAbortable.abort("Failed split, abort");
+ *    }
+ *  }
+ * 
                  
+ * 
                  This class is not thread safe.  Caller needs ensure split is run by
+ * one thread only.
+ */
+public class SplitTransaction {
+  private static final Log LOG = LogFactory.getLog(SplitTransaction.class);
+  private static final String SPLITDIR = ".splits";
+
+  /*
+   * Region to split
+   */
+  private final HRegion parent;
+  private HRegionInfo hri_a;
+  private HRegionInfo hri_b;
+  private Path splitdir;
+  private long fileSplitTimeout = 30000;
+  private int znodeVersion = -1;
+
+  /*
+   * Row to split around
+   */
+  private final byte [] splitrow;
+
+  /**
+   * Types to add to the transaction journal.
+   * Each enum is a step in the split transaction. Used to figure how much
+   * we need to rollback.
+   */
+  enum JournalEntry {
+    /**
+     * Set region as in transition, set it into SPLITTING state.
+     */
+    SET_SPLITTING_IN_ZK,
+    /**
+     * We created the temporary split data directory.
+     */
+    CREATE_SPLIT_DIR,
+    /**
+     * Closed the parent region.
+     */
+    CLOSED_PARENT_REGION,
+    /**
+     * The parent has been taken out of the server's online regions list.
+     */
+    OFFLINED_PARENT,
+    /**
+     * Started in on creation of the first daughter region.
+     */
+    STARTED_REGION_A_CREATION,
+    /**
+     * Started in on the creation of the second daughter region.
+     */
+    STARTED_REGION_B_CREATION,
+    /**
+     * Point of no return.
+     * If we got here, then transaction is not recoverable other than by
+     * crashing out the regionserver.
+     */
+    PONR
+  }
+
+  /*
+   * Journal of how far the split transaction has progressed.
+   */
+  private final List journal = new ArrayList();
+
+  static final String INDEX_TABLE_SUFFIX = "_idx";
+
+  /**
+   * Constructor
+   * @param r Region to split
+   * @param splitrow Row to split around
+   */
+  public SplitTransaction(final HRegion r, final byte [] splitrow) {
+    this.parent = r;
+    this.splitrow = splitrow;
+    this.splitdir = getSplitDir(this.parent);
+  }
+
+  /**
+   * Does checks on split inputs.
+   * @return true if the region is splittable else
+   * false if it is not (e.g. its already closed, etc.).
+   */
+  public boolean prepare() {
+    if (!this.parent.isSplittable()) return false;
+    // Split key can be null if this region is unsplittable; i.e. has refs.
+    if (this.splitrow == null) return false;
+    HRegionInfo hri = this.parent.getRegionInfo();
+    parent.prepareToSplit();
+    // Check splitrow.
+    byte [] startKey = hri.getStartKey();
+    byte [] endKey = hri.getEndKey();
+    if (Bytes.equals(startKey, splitrow) ||
+        !this.parent.getRegionInfo().containsRow(splitrow)) {
+      LOG.info("Split row is not inside region key range or is equal to " +
+          "startkey: " + Bytes.toStringBinary(this.splitrow));
+      return false;
+    }
+    long rid = getDaughterRegionIdTimestamp(hri);
+    this.hri_a = new HRegionInfo(hri.getTableName(), startKey, this.splitrow,
+      false, rid);
+    this.hri_b = new HRegionInfo(hri.getTableName(), this.splitrow, endKey,
+      false, rid);
+    return true;
+  }
+
+  /**
+   * Calculate daughter regionid to use.
+   * @param hri Parent {@link HRegionInfo}
+   * @return Daughter region id (timestamp) to use.
+   */
+  private static long getDaughterRegionIdTimestamp(final HRegionInfo hri) {
+    long rid = EnvironmentEdgeManager.currentTimeMillis();
+    // Regionid is timestamp.  Can't be less than that of parent else will insert
+    // at wrong location in .META. (See HBASE-710).
+    if (rid < hri.getRegionId()) {
+      LOG.warn("Clock skew; parent regions id is " + hri.getRegionId() +
+        " but current time here is " + rid);
+      rid = hri.getRegionId() + 1;
+    }
+    return rid;
+  }
+
+  private static IOException closedByOtherException = new IOException(
+      "Failed to close region: already closed by another thread");
+  
+  /**
+   * Prepare the regions and region files.
+   * @param server Hosting server instance.  Can be null when testing (won't try
+   * and update in zk if a null server)
+   * @param services Used to online/offline regions.
+   * @throws IOException If thrown, transaction failed. Call {@link #rollback(Server, RegionServerServices)}
+   * @return Regions created
+   */
+  /* package */PairOfSameType createDaughters(final Server server,
+      final RegionServerServices services) throws IOException {
+    LOG.info("Starting split of region " + this.parent);
+    boolean secondaryIndex =
+        server == null ? false : server.getConfiguration().getBoolean("hbase.use.secondary.index",
+          false);
+    boolean indexRegionAvailable = false;
+    if ((server != null && server.isStopped()) ||
+        (services != null && services.isStopping())) {
+      throw new IOException("Server is stopped or stopping");
+    }
+    assert !this.parent.lock.writeLock().isHeldByCurrentThread(): "Unsafe to hold write lock while performing RPCs";
+
+    // Coprocessor callback
+    if (this.parent.getCoprocessorHost() != null) {
+      this.parent.getCoprocessorHost().preSplit();
+    }
+
+    boolean testing = server == null? true:
+      server.getConfiguration().getBoolean("hbase.testing.nocluster", false);
+
+    PairOfSameType daughterRegionsPair = stepsBeforeAddingPONR(server, services, testing);
+
+    SplitInfo info = null;
+    // Coprocessor callback
+    if (secondaryIndex) {
+      if (this.parent.getCoprocessorHost() != null) {
+        info = this.parent.getCoprocessorHost().preSplitBeforePONR(this.splitrow);
+        if (info == null) {
+          throw new IOException("Pre split of Index region has failed.");
+        }
+        if ((info.getSplitTransaction() != null && info.getDaughters() != null)) {
+          indexRegionAvailable = true;
+        }
+      }
+    }
+
+    // add one hook
+    // do the step till started_region_b_creation
+    // This is the point of no return.  Adding subsequent edits to .META. as we
+    // do below when we do the daughter opens adding each to .META. can fail in
+    // various interesting ways the most interesting of which is a timeout
+    // BUT the edits all go through (See HBASE-3872).  IF we reach the PONR
+    // then subsequent failures need to crash out this regionserver; the
+    // server shutdown processing should be able to fix-up the incomplete split.
+    // The offlined parent will have the daughters as extra columns.  If
+    // we leave the daughter regions in place and do not remove them when we
+    // crash out, then they will have their references to the parent in place
+    // still and the server shutdown fixup of .META. will point to these
+    // regions.
+    // We should add PONR JournalEntry before offlineParentInMeta,so even if
+    // OfflineParentInMeta timeout,this will cause regionserver exit,and then
+    // master ServerShutdownHandler will fix daughter & avoid data loss. (See
+    // HBase-4562).
+    this.journal.add(JournalEntry.PONR);
+
+    // Edit parent in meta.  Offlines parent region and adds splita and splitb.
+    if (!testing) {
+      if (!indexRegionAvailable) {
+        MetaEditor.offlineParentInMeta(server.getCatalogTracker(), this.parent.getRegionInfo(),
+          daughterRegionsPair.getFirst().getRegionInfo(), daughterRegionsPair.getSecond()
+              .getRegionInfo());
+      } else {
+        offlineParentInMetaBothIndexAndMainRegion(server.getCatalogTracker(),
+          this.parent.getRegionInfo(), daughterRegionsPair.getFirst().getRegionInfo(),
+          daughterRegionsPair.getSecond().getRegionInfo(),
+          info.getSplitTransaction().parent.getRegionInfo(), info.getDaughters().getFirst()
+              .getRegionInfo(), info.getDaughters().getSecond().getRegionInfo());
+      }
+    }
+    return daughterRegionsPair;
+  }
+
+  private static void offlineParentInMetaBothIndexAndMainRegion(CatalogTracker catalogTracker,
+      HRegionInfo parent, final HRegionInfo a, final HRegionInfo b, final HRegionInfo parentIdx,
+      final HRegionInfo idxa, final HRegionInfo idxb) throws NotAllMetaRegionsOnlineException,
+      IOException {
+    HRegionInfo copyOfParent = new HRegionInfo(parent);
+    copyOfParent.setOffline(true);
+    copyOfParent.setSplit(true);
+    List list = new ArrayList();
+    Put put = new Put(copyOfParent.getRegionName());
+    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+      Writables.getBytes(copyOfParent));
+    put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER, Writables.getBytes(a));
+    put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER, Writables.getBytes(b));
+    list.add(put);
+
+    HRegionInfo copyOfIdxParent = new HRegionInfo(parentIdx);
+    copyOfIdxParent.setOffline(true);
+    copyOfIdxParent.setSplit(true);
+    Put putForIdxRegion = new Put(copyOfIdxParent.getRegionName());
+    putForIdxRegion.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+      Writables.getBytes(copyOfIdxParent));
+    putForIdxRegion.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER,
+      Writables.getBytes(idxa));
+    putForIdxRegion.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER,
+      Writables.getBytes(idxb));
+    list.add(putForIdxRegion);
+    putToMetaTable(catalogTracker, list);
+    LOG.info("Offlined parent region " + parent.getRegionNameAsString() + " in META");
+  }
+
+  private static void putToMetaTable(final CatalogTracker ct, final List p) throws IOException {
+    org.apache.hadoop.hbase.client.HConnection c = ct.getConnection();
+    if (c == null) throw new NullPointerException("No connection");
+    put(new HTable(ct.getConnection().getConfiguration(), HConstants.META_TABLE_NAME), p);
+  }
+
+  private static void put(final HTable t, final List p) throws IOException {
+    try {
+      t.put(p);
+    } finally {
+      t.close();
+    }
+  }
+
+  public PairOfSameType stepsBeforeAddingPONR(final Server server, final RegionServerServices services,
+      boolean testing) throws IOException {
+    // Set ephemeral SPLITTING znode up in zk.  Mocked servers sometimes don't
+    // have zookeeper so don't do zk stuff if server or zookeeper is null
+    if (server != null && server.getZooKeeper() != null) {
+      try {
+        createNodeSplitting(server.getZooKeeper(),
+          this.parent.getRegionInfo(), server.getServerName());
+      } catch (KeeperException e) {
+        throw new IOException("Failed creating SPLITTING znode on " +
+          this.parent.getRegionNameAsString(), e);
+      }
+    }
+    this.journal.add(JournalEntry.SET_SPLITTING_IN_ZK);
+    if (server != null && server.getZooKeeper() != null) {
+      try {
+        // Transition node from SPLITTING to SPLITTING after creating the split node.
+        // Master will get the callback for node change only if the transition is successful.
+        // Note that if the transition fails then the rollback will delete the created znode
+        // TODO : May be we can add some new state to znode and handle the new state incase of success/failure
+        this.znodeVersion = transitionNodeSplitting(server.getZooKeeper(),
+            this.parent.getRegionInfo(), server.getServerName(), -1);
+      } catch (KeeperException e) {
+        throw new IOException("Failed setting SPLITTING znode on "
+            + this.parent.getRegionNameAsString(), e);
+      }
+    }
+    createSplitDir(this.parent.getFilesystem(), this.splitdir);
+    this.journal.add(JournalEntry.CREATE_SPLIT_DIR);
+ 
+    List hstoreFilesToSplit = null;
+    Exception exceptionToThrow = null;
+    try{
+      hstoreFilesToSplit = this.parent.close(false);
+    } catch (Exception e) {
+      exceptionToThrow = e;
+    }
+    if (exceptionToThrow == null && hstoreFilesToSplit == null) {
+      // The region was closed by a concurrent thread.  We can't continue
+      // with the split, instead we must just abandon the split.  If we
+      // reopen or split this could cause problems because the region has
+      // probably already been moved to a different server, or is in the
+      // process of moving to a different server.
+      exceptionToThrow = closedByOtherException;
+    }
+    if (exceptionToThrow != closedByOtherException) {
+      this.journal.add(JournalEntry.CLOSED_PARENT_REGION);
+    }
+    if (exceptionToThrow != null) {
+      if (exceptionToThrow instanceof IOException) throw (IOException)exceptionToThrow;
+      throw new IOException(exceptionToThrow);
+    }
+
+    if (!testing) {
+      services.removeFromOnlineRegions(this.parent.getRegionInfo().getEncodedName());
+    }
+    this.journal.add(JournalEntry.OFFLINED_PARENT);
+
+    // TODO: If splitStoreFiles were multithreaded would we complete steps in
+    // less elapsed time?  St.Ack 20100920
+    //
+    // splitStoreFiles creates daughter region dirs under the parent splits dir
+    // Nothing to unroll here if failure -- clean up of CREATE_SPLIT_DIR will
+    // clean this up.
+    splitStoreFiles(this.splitdir, hstoreFilesToSplit);
+
+    // Log to the journal that we are creating region A, the first daughter
+    // region.  We could fail halfway through.  If we do, we could have left
+    // stuff in fs that needs cleanup -- a storefile or two.  Thats why we
+    // add entry to journal BEFORE rather than AFTER the change.
+    this.journal.add(JournalEntry.STARTED_REGION_A_CREATION);
+    HRegion a = createDaughterRegion(this.hri_a, this.parent.rsServices);
+
+    // Ditto
+    this.journal.add(JournalEntry.STARTED_REGION_B_CREATION);
+    HRegion b = createDaughterRegion(this.hri_b, this.parent.rsServices);
+    return new PairOfSameType(a,b);
+  }
+
+  /**
+   * Perform time consuming opening of the daughter regions.
+   * @param server Hosting server instance.  Can be null when testing (won't try
+   * and update in zk if a null server)
+   * @param services Used to online/offline regions.
+   * @param a first daughter region
+   * @param a second daughter region
+   * @throws IOException If thrown, transaction failed. Call {@link #rollback(Server, RegionServerServices)}
+   */
+  /* package */void openDaughters(final Server server,
+      final RegionServerServices services, HRegion a, HRegion b)
+      throws IOException {
+    boolean stopped = server != null && server.isStopped();
+    boolean stopping = services != null && services.isStopping();
+    // TODO: Is this check needed here?
+    if (stopped || stopping) {
+      LOG.info("Not opening daughters " +
+          b.getRegionInfo().getRegionNameAsString() +
+          " and " +
+          a.getRegionInfo().getRegionNameAsString() +
+          " because stopping=" + stopping + ", stopped=" + stopped);
+    } else {
+      // Open daughters in parallel.
+      DaughterOpener aOpener = new DaughterOpener(server, a);
+      DaughterOpener bOpener = new DaughterOpener(server, b);
+      aOpener.start();
+      bOpener.start();
+      try {
+        aOpener.join();
+        bOpener.join();
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        throw new IOException("Interrupted " + e.getMessage());
+      }
+      if (aOpener.getException() != null) {
+        throw new IOException("Failed " +
+          aOpener.getName(), aOpener.getException());
+      }
+      if (bOpener.getException() != null) {
+        throw new IOException("Failed " +
+          bOpener.getName(), bOpener.getException());
+      }
+      if (services != null) {
+        try {
+          // add 2nd daughter first (see HBASE-4335)
+          services.postOpenDeployTasks(b, server.getCatalogTracker(), true);
+          // Should add it to OnlineRegions
+          services.addToOnlineRegions(b);
+          services.postOpenDeployTasks(a, server.getCatalogTracker(), true);
+          services.addToOnlineRegions(a);
+        } catch (KeeperException ke) {
+          throw new IOException(ke);
+        }
+      }
+    }
+  }
+
+  /**
+   * Finish off split transaction, transition the zknode
+   * @param server Hosting server instance.  Can be null when testing (won't try
+   * and update in zk if a null server)
+   * @param services Used to online/offline regions.
+   * @param a first daughter region
+   * @param a second daughter region
+   * @throws IOException If thrown, transaction failed. Call {@link #rollback(Server, RegionServerServices)}
+   */
+  /* package */void transitionZKNode(final Server server,
+      final RegionServerServices services, HRegion a, HRegion b)
+      throws IOException {
+    // Tell master about split by updating zk.  If we fail, abort.
+    if (server != null && server.getZooKeeper() != null) {
+      try {
+        this.znodeVersion = transitionNodeSplit(server.getZooKeeper(),
+          parent.getRegionInfo(), a.getRegionInfo(), b.getRegionInfo(),
+          server.getServerName(), this.znodeVersion);
+
+        int spins = 0;
+        // Now wait for the master to process the split. We know it's done
+        // when the znode is deleted. The reason we keep tickling the znode is
+        // that it's possible for the master to miss an event.
+        do {
+          if (spins % 10 == 0) {
+            LOG.debug("Still waiting on the master to process the split for " +
+                this.parent.getRegionInfo().getEncodedName());
+          }
+          Thread.sleep(100);
+          // When this returns -1 it means the znode doesn't exist
+          this.znodeVersion = tickleNodeSplit(server.getZooKeeper(),
+            parent.getRegionInfo(), a.getRegionInfo(), b.getRegionInfo(),
+            server.getServerName(), this.znodeVersion);
+          spins++;
+        } while (this.znodeVersion != -1 && !server.isStopped()
+            && !services.isStopping());
+      } catch (Exception e) {
+        if (e instanceof InterruptedException) {
+          Thread.currentThread().interrupt();
+        }
+        throw new IOException("Failed telling master about split", e);
+      }
+    }
+
+    // Coprocessor callback
+    if (this.parent.getCoprocessorHost() != null) {
+      this.parent.getCoprocessorHost().postSplit(a,b);
+    }
+
+    // Leaving here, the splitdir with its dross will be in place but since the
+    // split was successful, just leave it; it'll be cleaned when parent is
+    // deleted and cleaned up.
+  }
+
+  /**
+   * Run the transaction.
+   * @param server Hosting server instance.  Can be null when testing (won't try
+   * and update in zk if a null server)
+   * @param services Used to online/offline regions.
+   * @throws IOException If thrown, transaction failed. Call {@link #rollback(Server, RegionServerServices)}
+   * @return Regions created
+   * @throws IOException
+   * @see #rollback(Server, RegionServerServices)
+   */
+  public PairOfSameType execute(final Server server,
+      final RegionServerServices services)
+  throws IOException {
+    PairOfSameType regions = createDaughters(server, services);
+    stepsAfterPONR(server, services, regions);
+    return regions;
+  }
+
+  public void stepsAfterPONR(final Server server, final RegionServerServices services,
+      PairOfSameType regions) throws IOException {
+    openDaughters(server, services, regions.getFirst(), regions.getSecond());
+    transitionZKNode(server, services, regions.getFirst(), regions.getSecond());
+  }
+
+  /*
+   * Open daughter region in its own thread.
+   * If we fail, abort this hosting server.
+   */
+  class DaughterOpener extends HasThread {
+    private final Server server;
+    private final HRegion r;
+    private Throwable t = null;
+
+    DaughterOpener(final Server s, final HRegion r) {
+      super((s == null? "null-services": s.getServerName()) +
+        "-daughterOpener=" + r.getRegionInfo().getEncodedName());
+      setDaemon(true);
+      this.server = s;
+      this.r = r;
+    }
+
+    /**
+     * @return Null if open succeeded else exception that causes us fail open.
+     * Call it after this thread exits else you may get wrong view on result.
+     */
+    Throwable getException() {
+      return this.t;
+    }
+
+    @Override
+    public void run() {
+      try {
+        openDaughterRegion(this.server, r);
+      } catch (Throwable t) {
+        this.t = t;
+      }
+    }
+  }
+
+  /**
+   * Open daughter regions, add them to online list and update meta.
+   * @param server
+   * @param services Can be null when testing.
+   * @param daughter
+   * @throws IOException
+   * @throws KeeperException
+   */
+  void openDaughterRegion(final Server server, final HRegion daughter)
+  throws IOException, KeeperException {
+    HRegionInfo hri = daughter.getRegionInfo();
+    LoggingProgressable reporter = server == null? null:
+      new LoggingProgressable(hri, server.getConfiguration());
+    daughter.openHRegion(reporter);
+  }
+
+  static class LoggingProgressable implements CancelableProgressable {
+    private final HRegionInfo hri;
+    private long lastLog = -1;
+    private final long interval;
+
+    LoggingProgressable(final HRegionInfo hri, final Configuration c) {
+      this.hri = hri;
+      this.interval = c.getLong("hbase.regionserver.split.daughter.open.log.interval",
+        10000);
+    }
+
+    @Override
+    public boolean progress() {
+      long now = System.currentTimeMillis();
+      if (now - lastLog > this.interval) {
+        LOG.info("Opening " + this.hri.getRegionNameAsString());
+        this.lastLog = now;
+      }
+      return true;
+    }
+  }
+
+  private static Path getSplitDir(final HRegion r) {
+    return new Path(r.getRegionDir(), SPLITDIR);
+  }
+
+  /**
+   * @param fs Filesystem to use
+   * @param splitdir Directory to store temporary split data in
+   * @throws IOException If splitdir already exists or we fail
+   * to create it.
+   * @see #cleanupSplitDir(FileSystem, Path)
+   */
+  void createSplitDir(final FileSystem fs, final Path splitdir)
+  throws IOException {
+    if (fs.exists(splitdir)) {
+      LOG.info("The " + splitdir
+          + " directory exists.  Hence deleting it to recreate it");
+      if (!HBaseFileSystem.deleteDirFromFileSystem(fs, splitdir)) {
+        throw new IOException("Failed deletion of " + splitdir
+            + " before creating them again.");
+      }
+    }
+    if (!HBaseFileSystem.makeDirOnFileSystem(fs, splitdir))
+        throw new IOException("Failed create of " + splitdir);
+  }
+
+  private static void cleanupSplitDir(final FileSystem fs, final Path splitdir)
+  throws IOException {
+    // Splitdir may have been cleaned up by reopen of the parent dir.
+    deleteDir(fs, splitdir, false);
+  }
+
+  /**
+   * @param fs Filesystem to use
+   * @param dir Directory to delete
+   * @param mustPreExist If true, we'll throw exception if dir
+   * does not preexist, else we'll just pass.
+   * @throws IOException Thrown if we fail to delete passed dir
+   */
+  private static void deleteDir(final FileSystem fs, final Path dir,
+      final boolean mustPreExist)
+  throws IOException {
+    if (!fs.exists(dir)) {
+      if (mustPreExist) throw new IOException(dir.toString() + " does not exist!");
+    } else if (!HBaseFileSystem.deleteDirFromFileSystem(fs, dir)) {
+      throw new IOException("Failed delete of " + dir);
+    }
+  }
+
+  protected void splitStoreFiles(final Path splitdir,
+      final List hstoreFilesToSplit)
+  throws IOException {
+    if (hstoreFilesToSplit == null) {
+      // Could be null because close didn't succeed -- for now consider it fatal
+      throw new IOException("Close returned empty list of StoreFiles");
+    }
+    // The following code sets up a thread pool executor with as many slots as
+    // there's files to split. It then fires up everything, waits for
+    // completion and finally checks for any exception
+    int nbFiles = hstoreFilesToSplit.size();
+    boolean secondaryIndex = this.parent.getConf().getBoolean("hbase.use.secondary.index", false);
+    if (secondaryIndex) {
+      String idxTableName = this.parent.getTableDesc().getNameAsString();
+      if (isIndexTable(idxTableName)) if (nbFiles == 0) {
+        LOG.debug("Setting number of threads for ThreadPoolExecutor to 1 since IndexTable "
+            + idxTableName + " doesn't have any store files ");
+        nbFiles = 1;
+      }
+    }
+
+    ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
+    builder.setNameFormat("StoreFileSplitter-%1$d");
+    ThreadFactory factory = builder.build();
+    ThreadPoolExecutor threadPool =
+      (ThreadPoolExecutor) Executors.newFixedThreadPool(nbFiles, factory);
+    List> futures = new ArrayList>(nbFiles);
+
+     // Split each store file.
+    for (StoreFile sf: hstoreFilesToSplit) {
+      //splitStoreFile(sf, splitdir);
+      StoreFileSplitter sfs = new StoreFileSplitter(sf, splitdir);
+      futures.add(threadPool.submit(sfs));
+    }
+    // Shutdown the pool
+    threadPool.shutdown();
+
+    // Wait for all the tasks to finish
+    try {
+      boolean stillRunning = !threadPool.awaitTermination(
+          this.fileSplitTimeout, TimeUnit.MILLISECONDS);
+      if (stillRunning) {
+        threadPool.shutdownNow();
+        // wait for the thread to shutdown completely.
+        while (!threadPool.isTerminated()) {
+          Thread.sleep(50);
+        }
+        throw new IOException("Took too long to split the" +
+            " files and create the references, aborting split");
+      }
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new IOException("Interrupted while waiting for file splitters", e);
+    }
+
+    // Look for any exception
+    for (Future future: futures) {
+      try {
+        future.get();
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        throw new IOException(
+            "Interrupted while trying to get the results of file splitters", e);
+      } catch (ExecutionException e) {
+        throw new IOException(e);
+      }
+    }
+  }
+
+  private boolean isIndexTable(String idxTableName) {
+    return idxTableName.endsWith(INDEX_TABLE_SUFFIX);
+  }
+
+  private void splitStoreFile(final StoreFile sf, final Path splitdir)
+  throws IOException {
+    FileSystem fs = this.parent.getFilesystem();
+    byte [] family = sf.getFamily();
+    String encoded = this.hri_a.getEncodedName();
+    Path storedir = Store.getStoreHomedir(splitdir, encoded, family);
+    StoreFile.split(fs, storedir, sf, this.splitrow, Range.bottom);
+    encoded = this.hri_b.getEncodedName();
+    storedir = Store.getStoreHomedir(splitdir, encoded, family);
+    StoreFile.split(fs, storedir, sf, this.splitrow, Range.top);
+  }
+
+  /**
+   * Utility class used to do the file splitting / reference writing
+   * in parallel instead of sequentially.
+   */
+  class StoreFileSplitter implements Callable {
+
+    private final StoreFile sf;
+    private final Path splitdir;
+
+    /**
+     * Constructor that takes what it needs to split
+     * @param sf which file
+     * @param splitdir where the splitting is done
+     */
+    public StoreFileSplitter(final StoreFile sf, final Path splitdir) {
+      this.sf = sf;
+      this.splitdir = splitdir;
+    }
+
+    public Void call() throws IOException {
+      splitStoreFile(sf, splitdir);
+      return null;
+    }
+  }
+
+  /**
+   * @param hri Spec. for daughter region to open.
+   * @param flusher Flusher this region should use.
+   * @return Created daughter HRegion.
+   * @throws IOException
+   * @see #cleanupDaughterRegion(FileSystem, Path, HRegionInfo)
+   */
+  HRegion createDaughterRegion(final HRegionInfo hri,
+      final RegionServerServices rsServices)
+  throws IOException {
+    // Package private so unit tests have access.
+    FileSystem fs = this.parent.getFilesystem();
+    Path regionDir = getSplitDirForDaughter(this.parent.getFilesystem(),
+      this.splitdir, hri);
+    HRegion r = HRegion.newHRegion(this.parent.getTableDir(),
+      this.parent.getLog(), fs, this.parent.getBaseConf(),
+      hri, this.parent.getTableDesc(), rsServices);
+    long halfParentReadRequestCount = this.parent.getReadRequestsCount() / 2;
+    r.readRequestsCount.set(halfParentReadRequestCount);
+    r.setOpMetricsReadRequestCount(halfParentReadRequestCount);
+    long halfParentWriteRequest = this.parent.getWriteRequestsCount() / 2;
+    r.writeRequestsCount.set(halfParentWriteRequest);
+    r.setOpMetricsWriteRequestCount(halfParentWriteRequest);    
+    HRegion.moveInitialFilesIntoPlace(fs, regionDir, r.getRegionDir());
+    return r;
+  }
+
+  private static void cleanupDaughterRegion(final FileSystem fs,
+    final Path tabledir, final String encodedName)
+  throws IOException {
+    Path regiondir = HRegion.getRegionDir(tabledir, encodedName);
+    // Dir may not preexist.
+    deleteDir(fs, regiondir, false);
+  }
+
+  /*
+   * Get the daughter directories in the splits dir.  The splits dir is under
+   * the parent regions' directory.
+   * @param fs
+   * @param splitdir
+   * @param hri
+   * @return Path to daughter split dir.
+   * @throws IOException
+   */
+  private static Path getSplitDirForDaughter(final FileSystem fs,
+      final Path splitdir, final HRegionInfo hri)
+  throws IOException {
+    return new Path(splitdir, hri.getEncodedName());
+  }
+
+  /**
+   * @param server Hosting server instance (May be null when testing).
+   * @param services
+   * @throws IOException If thrown, rollback failed.  Take drastic action.
+   * @return True if we successfully rolled back, false if we got to the point
+   * of no return and so now need to abort the server to minimize damage.
+   */
+  public boolean rollback(final Server server, final RegionServerServices services)
+  throws IOException {
+    // Coprocessor callback
+    boolean secondaryIndex =
+        server == null ? false : server.getConfiguration().getBoolean("hbase.use.secondary.index",
+          false);
+    if (secondaryIndex) {
+      if (this.parent.getCoprocessorHost() != null) {
+        this.parent.getCoprocessorHost().preRollBack();
+      }
+    }
+    boolean result = true;
+    FileSystem fs = this.parent.getFilesystem();
+    ListIterator iterator =
+      this.journal.listIterator(this.journal.size());
+    // Iterate in reverse.
+    while (iterator.hasPrevious()) {
+      JournalEntry je = iterator.previous();
+      switch(je) {
+      
+      case SET_SPLITTING_IN_ZK:
+        if (server != null && server.getZooKeeper() != null) {
+          cleanZK(server, this.parent.getRegionInfo());
+        }
+        break;
+
+      case CREATE_SPLIT_DIR:
+    	this.parent.writestate.writesEnabled = true;
+        cleanupSplitDir(fs, this.splitdir);
+        break;
+
+      case CLOSED_PARENT_REGION:
+        try {
+          // So, this returns a seqid but if we just closed and then reopened, we
+          // should be ok. On close, we flushed using sequenceid obtained from
+          // hosting regionserver so no need to propagate the sequenceid returned
+          // out of initialize below up into regionserver as we normally do.
+          // TODO: Verify.
+          this.parent.initialize();
+        } catch (IOException e) {
+          LOG.error("Failed rollbacking CLOSED_PARENT_REGION of region " +
+            this.parent.getRegionNameAsString(), e);
+          throw new RuntimeException(e);
+        }
+        break;
+
+      case STARTED_REGION_A_CREATION:
+        cleanupDaughterRegion(fs, this.parent.getTableDir(),
+          this.hri_a.getEncodedName());
+        break;
+
+      case STARTED_REGION_B_CREATION:
+        cleanupDaughterRegion(fs, this.parent.getTableDir(),
+          this.hri_b.getEncodedName());
+        break;
+
+      case OFFLINED_PARENT:
+        if (services != null) services.addToOnlineRegions(this.parent);
+        break;
+
+      case PONR:
+        // We got to the point-of-no-return so we need to just abort. Return
+        // immediately.  Do not clean up created daughter regions.  They need
+        // to be in place so we don't delete the parent region mistakenly.
+        // See HBASE-3872.
+        return false;
+
+      default:
+        throw new RuntimeException("Unhandled journal entry: " + je);
+      }
+    }
+    return result;
+  }
+
+  HRegionInfo getFirstDaughter() {
+    return hri_a;
+  }
+
+  HRegionInfo getSecondDaughter() {
+    return hri_b;
+  }
+
+  // For unit testing.
+  Path getSplitDir() {
+    return this.splitdir;
+  }
+
+  /**
+   * Clean up any split detritus that may have been left around from previous
+   * split attempts.
+   * Call this method on initial region deploy.  Cleans up any mess
+   * left by previous deploys of passed r region.
+   * @param r
+   * @throws IOException
+   */
+  static void cleanupAnySplitDetritus(final HRegion r) throws IOException {
+    Path splitdir = getSplitDir(r);
+    FileSystem fs = r.getFilesystem();
+    if (!fs.exists(splitdir)) return;
+    // Look at the splitdir.  It could have the encoded names of the daughter
+    // regions we tried to make.  See if the daughter regions actually got made
+    // out under the tabledir.  If here under splitdir still, then the split did
+    // not complete.  Try and do cleanup.  This code WILL NOT catch the case
+    // where we successfully created daughter a but regionserver crashed during
+    // the creation of region b.  In this case, there'll be an orphan daughter
+    // dir in the filesystem.  TOOD: Fix.
+    FileStatus [] daughters = fs.listStatus(splitdir, new FSUtils.DirFilter(fs));
+    for (int i = 0; i < daughters.length; i++) {
+      cleanupDaughterRegion(fs, r.getTableDir(),
+        daughters[i].getPath().getName());
+    }
+    cleanupSplitDir(r.getFilesystem(), splitdir);
+    LOG.info("Cleaned up old failed split transaction detritus: " + splitdir);
+  }
+
+  private static void cleanZK(final Server server, final HRegionInfo hri) {
+    try {
+      // Only delete if its in expected state; could have been hijacked.
+      ZKAssign.deleteNode(server.getZooKeeper(), hri.getEncodedName(),
+        EventType.RS_ZK_REGION_SPLITTING);
+    } catch (KeeperException e) {
+      server.abort("Failed cleanup of " + hri.getRegionNameAsString(), e);
+    }
+  }
+
+  /**
+   * Creates a new ephemeral node in the SPLITTING state for the specified region.
+   * Create it ephemeral in case regionserver dies mid-split.
+   *
+   * 
                  Does not transition nodes from other states.  If a node already exists
+   * for this region, a {@link NodeExistsException} will be thrown.
+   *
+   * @param zkw zk reference
+   * @param region region to be created as offline
+   * @param serverName server event originates from
+   * @return Version of znode created.
+   * @throws KeeperException 
+   * @throws IOException 
+   */
+  void createNodeSplitting(final ZooKeeperWatcher zkw, final HRegionInfo region,
+      final ServerName serverName) throws KeeperException, IOException {
+    LOG.debug(zkw.prefix("Creating ephemeral node for " +
+      region.getEncodedName() + " in SPLITTING state"));
+    RegionTransitionData data =
+      new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
+        region.getRegionName(), serverName);
+
+    String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
+    if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, data.getBytes())) {
+      throw new IOException("Failed create of ephemeral " + node);
+    }
+  }
+
+  /**
+   * Transitions an existing node for the specified region which is
+   * currently in the SPLITTING state to be in the SPLIT state.  Converts the
+   * ephemeral SPLITTING znode to an ephemeral SPLIT node.  Master cleans up
+   * SPLIT znode when it reads it (or if we crash, zk will clean it up).
+   *
+   * 
                  Does not transition nodes from other states.  If for some reason the
+   * node could not be transitioned, the method returns -1.  If the transition
+   * is successful, the version of the node after transition is returned.
+   *
+   * 
                  This method can fail and return false for three different reasons:
+   * 
                  • Node for this region does not exist
                    • 
+   * 
                  • Node for this region is not in SPLITTING state
                    • 
+   * 
                  • After verifying SPLITTING state, update fails because of wrong version
+   * (this should never actually happen since an RS only does this transition
+   * following a transition to SPLITTING.  if two RS are conflicting, one would
+   * fail the original transition to SPLITTING and not this transition)
                    • 
+   * 
                  
+   *
+   * 
                  Does not set any watches.
+   *
+   * 
                  This method should only be used by a RegionServer when completing the
+   * open of a region.
+   *
+   * @param zkw zk reference
+   * @param parent region to be transitioned to opened
+   * @param a Daughter a of split
+   * @param b Daughter b of split
+   * @param serverName server event originates from
+   * @return version of node after transition, -1 if unsuccessful transition
+   * @throws KeeperException if unexpected zookeeper exception
+   * @throws IOException 
+   */
+  private static int transitionNodeSplit(ZooKeeperWatcher zkw,
+      HRegionInfo parent, HRegionInfo a, HRegionInfo b, ServerName serverName,
+      final int znodeVersion)
+  throws KeeperException, IOException {
+    byte [] payload = Writables.getBytes(a, b);
+    return ZKAssign.transitionNode(zkw, parent, serverName,
+      EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLIT,
+      znodeVersion, payload);
+  }
+
+  public static class SplitInfo {
+    PairOfSameType pairOfSameType;
+    SplitTransaction st;
+
+    public void setDaughtersAndTransaction(PairOfSameType pairOfSameType,
+        SplitTransaction st) {
+      this.pairOfSameType = pairOfSameType;
+      this.st = st;
+    }
+
+    public PairOfSameType getDaughters() {
+      return this.pairOfSameType;
+    }
+
+    public SplitTransaction getSplitTransaction() {
+      return this.st;
+    }
+  }
+
+  /**
+   * Added for secondary index. Needed in the hooks to get the parentRegion name
+   * @return
+   */
+  public HRegion getParent() {
+    return this.parent;
+  }
+
+  /**
+   *
+   * @param zkw zk reference
+   * @param parent region to be transitioned to splitting
+   * @param serverName server event originates from
+   * @param version znode version
+   * @return version of node after transition, -1 if unsuccessful transition
+   * @throws KeeperException
+   * @throws IOException
+   */
+  int transitionNodeSplitting(final ZooKeeperWatcher zkw, final HRegionInfo parent,
+      final ServerName serverName, final int version) throws KeeperException, IOException {
+    return ZKAssign.transitionNode(zkw, parent, serverName,
+      EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
+  }
+
+  private static int tickleNodeSplit(ZooKeeperWatcher zkw,
+      HRegionInfo parent, HRegionInfo a, HRegionInfo b, ServerName serverName,
+      final int znodeVersion)
+  throws KeeperException, IOException {
+    byte [] payload = Writables.getBytes(a, b);
+    return ZKAssign.transitionNode(zkw, parent, serverName,
+      EventType.RS_ZK_REGION_SPLIT, EventType.RS_ZK_REGION_SPLIT,
+      znodeVersion, payload);
+  }
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
new file mode 100644
index 0000000..a20892e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -0,0 +1,2447 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.Random;
+import java.util.SortedSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.KVComparator;
+import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
+import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.io.hfile.InvalidHFileException;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactSelection;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hadoop.hbase.util.CollectionBackedScanner;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+/**
+ * A Store holds a column family in a Region.  Its a memstore and a set of zero
+ * or more StoreFiles, which stretch backwards over time.
+ *
+ * 
                  There's no reason to consider append-logging at this level; all logging
+ * and locking is handled at the HRegion level.  Store just provides
+ * services to manage sets of StoreFiles.  One of the most important of those
+ * services is compaction services where files are aggregated once they pass
+ * a configurable threshold.
+ *
+ * 
                  The only thing having to do with logs that Store needs to deal with is
+ * the reconstructionLog.  This is a segment of an HRegion's log that might
+ * NOT be present upon startup.  If the param is NULL, there's nothing to do.
+ * If the param is non-NULL, we need to process the log to reconstruct
+ * a TreeMap that might not have been written to disk before the process
+ * died.
+ *
+ * 
                  It's assumed that after this constructor returns, the reconstructionLog
+ * file will be deleted (by whoever has instantiated the Store).
+ *
+ * 
                  Locking and transactions are handled at a higher level.  This API should
+ * not be called directly but by an HRegion manager.
+ */
+public class Store extends SchemaConfigured implements HeapSize {
+  static final Log LOG = LogFactory.getLog(Store.class);
+  
+  public static final String BLOCKING_STOREFILES_KEY = "hbase.hstore.blockingStoreFiles";
+  public static final int DEFAULT_BLOCKING_STOREFILE_COUNT = 7;
+
+  protected final MemStore memstore;
+  // This stores directory in the filesystem.
+  private final Path homedir;
+  private final HRegion region;
+  private final HColumnDescriptor family;
+  final FileSystem fs;
+  final Configuration conf;
+  final CacheConfig cacheConf;
+  // ttl in milliseconds.
+  private long ttl;
+  private final int minFilesToCompact;
+  private final int maxFilesToCompact;
+  private final long minCompactSize;
+  private final long maxCompactSize;
+  private long lastCompactSize = 0;
+  volatile boolean forceMajor = false;
+  /* how many bytes to write between status checks */
+  static int closeCheckInterval = 0;
+  private final int blockingStoreFileCount;
+  private volatile long storeSize = 0L;
+  private volatile long totalUncompressedBytes = 0L;
+  private final Object flushLock = new Object();
+  final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+  private final boolean verifyBulkLoads;
+  
+  private long blockingFileCount;
+
+  /* The default priority for user-specified compaction requests.
+   * The user gets top priority unless we have blocking compactions. (Pri <= 0)
+   */
+  public static final int PRIORITY_USER = 1;
+  public static final int NO_PRIORITY = Integer.MIN_VALUE;
+
+  // not private for testing
+  /* package */ScanInfo scanInfo;
+  /*
+   * List of store files inside this store. This is an immutable list that
+   * is atomically replaced when its contents change.
+   */
+  private volatile ImmutableList storefiles = null;
+
+  List filesCompacting = Lists.newArrayList();
+
+  // All access must be synchronized.
+  private final CopyOnWriteArraySet changedReaderObservers =
+    new CopyOnWriteArraySet();
+
+  private final int blocksize;
+  private HFileDataBlockEncoder dataBlockEncoder;
+
+  /** Checksum configuration */
+  private ChecksumType checksumType;
+  private int bytesPerChecksum;
+
+  // Comparing KeyValues
+  final KeyValue.KVComparator comparator;
+
+  private final Compactor compactor;
+
+  private static final int DEFAULT_FLUSH_RETRIES_NUMBER = 10;
+  private static int flush_retries_number;
+  private static int pauseTime;
+
+  /**
+   * Constructor
+   * @param basedir qualified path under which the region directory lives;
+   * generally the table subdirectory
+   * @param region
+   * @param family HColumnDescriptor for this column
+   * @param fs file system object
+   * @param confParam configuration object
+   * failed.  Can be null.
+   * @throws IOException
+   */
+  protected Store(Path basedir, HRegion region, HColumnDescriptor family,
+      FileSystem fs, Configuration confParam)
+  throws IOException {
+    super(new CompoundConfiguration().add(confParam).add(
+        family.getValues()), region.getTableDesc().getNameAsString(),
+        Bytes.toString(family.getName()));
+    HRegionInfo info = region.getRegionInfo();
+    this.fs = fs;
+    Path p = getStoreHomedir(basedir, info.getEncodedName(), family.getName());
+    this.homedir = createStoreHomeDir(this.fs, p);
+    this.region = region;
+    this.family = family;
+    // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
+    this.conf = new CompoundConfiguration().add(confParam).add(family.getValues());
+    this.blocksize = family.getBlocksize();
+
+    this.dataBlockEncoder =
+        new HFileDataBlockEncoderImpl(family.getDataBlockEncodingOnDisk(),
+            family.getDataBlockEncoding());
+
+    this.comparator = info.getComparator();
+    // getTimeToLive returns ttl in seconds.  Convert to milliseconds.
+    this.ttl = family.getTimeToLive();
+    if (ttl == HConstants.FOREVER) {
+      // default is unlimited ttl.
+      ttl = Long.MAX_VALUE;
+    } else if (ttl == -1) {
+      ttl = Long.MAX_VALUE;
+    } else {
+      // second -> ms adjust for user data
+      this.ttl *= 1000;
+    }
+    // used by ScanQueryMatcher
+    long timeToPurgeDeletes =
+        Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
+    LOG.info("time to purge deletes set to " + timeToPurgeDeletes +
+        "ms in store " + this);
+    scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator);
+    this.memstore = new MemStore(conf, this.comparator);
+
+    // By default, compact if storefile.count >= minFilesToCompact
+    this.minFilesToCompact = Math.max(2,
+      conf.getInt("hbase.hstore.compaction.min",
+        /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));
+
+    LOG.info("hbase.hstore.compaction.min = " + this.minFilesToCompact);
+    
+    // Setting up cache configuration for this family
+    this.cacheConf = new CacheConfig(conf, family);
+    this.blockingStoreFileCount =
+      conf.getInt("hbase.hstore.blockingStoreFiles", 7);
+
+    this.maxFilesToCompact = conf.getInt("hbase.hstore.compaction.max", 10);
+    this.minCompactSize = conf.getLong("hbase.hstore.compaction.min.size",
+      this.region.memstoreFlushSize);
+    this.maxCompactSize
+      = conf.getLong("hbase.hstore.compaction.max.size", Long.MAX_VALUE);
+
+    this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify", false);
+    
+    this.blockingFileCount =
+                conf.getInt(BLOCKING_STOREFILES_KEY, DEFAULT_BLOCKING_STOREFILE_COUNT);
+    
+    if (Store.closeCheckInterval == 0) {
+      Store.closeCheckInterval = conf.getInt(
+          "hbase.hstore.close.check.interval", 10*1000*1000 /* 10 MB */);
+    }
+    this.storefiles = sortAndClone(loadStoreFiles());
+
+    // Initialize checksum type from name. The names are CRC32, CRC32C, etc.
+    this.checksumType = getChecksumType(conf);
+    // initilize bytes per checksum
+    this.bytesPerChecksum = getBytesPerChecksum(conf);
+    // Create a compaction tool instance
+    this.compactor = new Compactor(this.conf);
+    if (Store.flush_retries_number == 0) {
+      Store.flush_retries_number = conf.getInt(
+          "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER);
+      Store.pauseTime = conf.getInt(HConstants.HBASE_SERVER_PAUSE,
+          HConstants.DEFAULT_HBASE_SERVER_PAUSE);
+      if (Store.flush_retries_number <= 0) {
+        throw new IllegalArgumentException(
+            "hbase.hstore.flush.retries.number must be > 0, not "
+                + Store.flush_retries_number);
+      }
+    }
+  }
+
+  /**
+   * @param family
+   * @return
+   */
+  long getTTL(final HColumnDescriptor family) {
+    // HCD.getTimeToLive returns ttl in seconds.  Convert to milliseconds.
+    long ttl = family.getTimeToLive();
+    if (ttl == HConstants.FOREVER) {
+      // Default is unlimited ttl.
+      ttl = Long.MAX_VALUE;
+    } else if (ttl == -1) {
+      ttl = Long.MAX_VALUE;
+    } else {
+      // Second -> ms adjust for user data
+      ttl *= 1000;
+    }
+    return ttl;
+  }
+
+  /**
+   * Create this store's homedir
+   * @param fs
+   * @param homedir
+   * @return Return homedir
+   * @throws IOException
+   */
+  Path createStoreHomeDir(final FileSystem fs,
+      final Path homedir) throws IOException {
+    if (!fs.exists(homedir) && !HBaseFileSystem.makeDirOnFileSystem(fs, homedir)) {
+        throw new IOException("Failed create of: " + homedir.toString());
+    }
+    return homedir;
+  }
+
+  FileSystem getFileSystem() {
+    return this.fs;
+  }
+
+  /**
+   * Returns the configured bytesPerChecksum value.
+   * @param conf The configuration
+   * @return The bytesPerChecksum that is set in the configuration
+   */
+  public static int getBytesPerChecksum(Configuration conf) {
+    return conf.getInt(HConstants.BYTES_PER_CHECKSUM,
+                       HFile.DEFAULT_BYTES_PER_CHECKSUM);
+  }
+
+  /**
+   * Returns the configured checksum algorithm.
+   * @param conf The configuration
+   * @return The checksum algorithm that is set in the configuration
+   */
+  public static ChecksumType getChecksumType(Configuration conf) {
+    String checksumName = conf.get(HConstants.CHECKSUM_TYPE_NAME);
+    if (checksumName == null) {
+      return HFile.DEFAULT_CHECKSUM_TYPE;
+    } else {
+      return ChecksumType.nameToType(checksumName);
+    }
+  }
+
+  public HColumnDescriptor getFamily() {
+    return this.family;
+  }
+
+  /**
+   * @return The maximum sequence id in all store files.
+   */
+  long getMaxSequenceId() {
+    return StoreFile.getMaxSequenceIdInList(this.getStorefiles());
+  }
+
+  /**
+   * @return The maximum memstoreTS in all store files.
+   */
+  public long getMaxMemstoreTS() {
+    return StoreFile.getMaxMemstoreTSInList(this.getStorefiles());
+  }
+
+  /**
+   * @param tabledir
+   * @param encodedName Encoded region name.
+   * @param family
+   * @return Path to family/Store home directory.
+   */
+  public static Path getStoreHomedir(final Path tabledir,
+      final String encodedName, final byte [] family) {
+     return getStoreHomedir(tabledir, encodedName, Bytes.toString(family));
+   }
+
+  /**
+   * @param tabledir
+   * @param encodedName Encoded region name.
+   * @param family
+   * @return Path to family/Store home directory.
+   */
+  public static Path getStoreHomedir(final Path tabledir,
+      final String encodedName, final String family) {
+    return new Path(tabledir, new Path(encodedName, new Path(family)));
+  }
+
+  /**
+   * @param parentRegionDirectory directory for the parent region
+   * @param family family name of this store
+   * @return Path to the family/Store home directory
+   */
+  public static Path getStoreHomedir(final Path parentRegionDirectory, final byte[] family) {
+    return new Path(parentRegionDirectory, new Path(Bytes.toString(family)));
+  }
+
+  /**
+   * Return the directory in which this store stores its
+   * StoreFiles
+   */
+  Path getHomedir() {
+    return homedir;
+  }
+
+  /**
+   * @return the data block encoder
+   */
+  public HFileDataBlockEncoder getDataBlockEncoder() {
+    return dataBlockEncoder;
+  }
+
+  /**
+   * Should be used only in tests.
+   * @param blockEncoder the block delta encoder to use
+   */
+  void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {
+    this.dataBlockEncoder = blockEncoder;
+  }
+
+  FileStatus [] getStoreFiles() throws IOException {
+    return FSUtils.listStatus(this.fs, this.homedir, null);
+  }
+
+  /**
+   * Creates an unsorted list of StoreFile loaded in parallel
+   * from the given directory.
+   * @throws IOException
+   */
+  private List loadStoreFiles() throws IOException {
+    ArrayList results = new ArrayList();
+    FileStatus files[] = getStoreFiles();
+
+    if (files == null || files.length == 0) {
+      return results;
+    }
+    // initialize the thread pool for opening store files in parallel..
+    ThreadPoolExecutor storeFileOpenerThreadPool =
+      this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-" +
+          this.family.getNameAsString());
+    CompletionService completionService =
+      new ExecutorCompletionService(storeFileOpenerThreadPool);
+
+    int totalValidStoreFile = 0;
+    for (int i = 0; i < files.length; i++) {
+      // Skip directories.
+      if (files[i].isDir()) {
+        continue;
+      }
+      final Path p = files[i].getPath();
+      // Check for empty hfile. Should never be the case but can happen
+      // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646
+      // NOTE: that the HFileLink is just a name, so it's an empty file.
+      if (!HFileLink.isHFileLink(p) && this.fs.getFileStatus(p).getLen() <= 0) {
+        LOG.warn("Skipping " + p + " because its empty. HBASE-646 DATA LOSS?");
+        continue;
+      }
+
+      // open each store file in parallel
+      completionService.submit(new Callable() {
+        public StoreFile call() throws IOException {
+          StoreFile storeFile = new StoreFile(fs, p, conf, cacheConf,
+              family.getBloomFilterType(), dataBlockEncoder);
+          passSchemaMetricsTo(storeFile);
+          storeFile.createReader();
+          return storeFile;
+        }
+      });
+      totalValidStoreFile++;
+    }
+
+    IOException ioe = null;
+    try {
+      for (int i = 0; i < totalValidStoreFile; i++) {
+        try {
+          Future future = completionService.take();
+          StoreFile storeFile = future.get();
+          long length = storeFile.getReader().length();
+          this.storeSize += length;
+          this.totalUncompressedBytes +=
+              storeFile.getReader().getTotalUncompressedBytes();
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("loaded " + storeFile.toStringDetailed());
+          }
+          results.add(storeFile);
+        } catch (InterruptedException e) {
+          if (ioe == null) ioe = new InterruptedIOException(e.getMessage());
+        } catch (ExecutionException e) {
+          if (ioe == null) ioe = new IOException(e.getCause());
+        } 
+      } 
+    } finally {
+      storeFileOpenerThreadPool.shutdownNow();
+    }
+    if (ioe != null) {
+      // close StoreFile readers
+      try {
+        for (StoreFile file : results) {
+          if (file != null) file.closeReader(true);
+        }
+      } catch (IOException e) { }
+      throw ioe;
+    }
+
+    return results;
+  }
+
+  /**
+   * Adds a value to the memstore
+   *
+   * @param kv
+   * @return memstore size delta
+   */
+  protected long add(final KeyValue kv) {
+    lock.readLock().lock();
+    try {
+      return this.memstore.add(kv);
+    } finally {
+      lock.readLock().unlock();
+    }
+  }
+
+  /**
+   * When was the oldest edit done in the memstore
+   */
+  public long timeOfOldestEdit() {
+    return memstore.timeOfOldestEdit();
+  }
+
+  /**
+   * Adds a value to the memstore
+   *
+   * @param kv
+   * @return memstore size delta
+   */
+  protected long delete(final KeyValue kv) {
+    lock.readLock().lock();
+    try {
+      return this.memstore.delete(kv);
+    } finally {
+      lock.readLock().unlock();
+    }
+  }
+
+  /**
+   * Removes a kv from the memstore. The KeyValue is removed only
+   * if its key & memstoreTS matches the key & memstoreTS value of the
+   * kv parameter.
+   *
+   * @param kv
+   */
+  protected void rollback(final KeyValue kv) {
+    lock.readLock().lock();
+    try {
+      this.memstore.rollback(kv);
+    } finally {
+      lock.readLock().unlock();
+    }
+  }
+
+  /**
+   * @return All store files.
+   */
+  public List getStorefiles() {
+    return this.storefiles;
+  }
+
+  /**
+   * This throws a WrongRegionException if the HFile does not fit in this
+   * region, or an InvalidHFileException if the HFile is not valid.
+   */
+  void assertBulkLoadHFileOk(Path srcPath) throws IOException {
+    HFile.Reader reader  = null;
+    try {
+      LOG.info("Validating hfile at " + srcPath + " for inclusion in "
+          + "store " + this + " region " + this.region);
+      reader = HFile.createReader(srcPath.getFileSystem(conf),
+          srcPath, cacheConf);
+      reader.loadFileInfo();
+
+      byte[] firstKey = reader.getFirstRowKey();
+      byte[] lk = reader.getLastKey();
+      byte[] lastKey =
+          (lk == null) ? null :
+              KeyValue.createKeyValueFromKey(lk).getRow();
+
+      LOG.debug("HFile bounds: first=" + Bytes.toStringBinary(firstKey) +
+          " last=" + Bytes.toStringBinary(lastKey));
+      LOG.debug("Region bounds: first=" +
+          Bytes.toStringBinary(region.getStartKey()) +
+          " last=" + Bytes.toStringBinary(region.getEndKey()));
+
+      HRegionInfo hri = region.getRegionInfo();
+      if (!hri.containsRange(firstKey, lastKey)) {
+        throw new WrongRegionException(
+            "Bulk load file " + srcPath.toString() + " does not fit inside region "
+            + this.region);
+      }
+
+      if (verifyBulkLoads) {
+        KeyValue prevKV = null;
+        HFileScanner scanner = reader.getScanner(false, false, false);
+        scanner.seekTo();
+        do {
+          KeyValue kv = scanner.getKeyValue();
+          if (prevKV != null) {
+            if (Bytes.compareTo(prevKV.getBuffer(), prevKV.getRowOffset(),
+                prevKV.getRowLength(), kv.getBuffer(), kv.getRowOffset(),
+                kv.getRowLength()) > 0) {
+              throw new InvalidHFileException("Previous row is greater than"
+                  + " current row: path=" + srcPath + " previous="
+                  + Bytes.toStringBinary(prevKV.getKey()) + " current="
+                  + Bytes.toStringBinary(kv.getKey()));
+            }
+            if (Bytes.compareTo(prevKV.getBuffer(), prevKV.getFamilyOffset(),
+                prevKV.getFamilyLength(), kv.getBuffer(), kv.getFamilyOffset(),
+                kv.getFamilyLength()) != 0) {
+              throw new InvalidHFileException("Previous key had different"
+                  + " family compared to current key: path=" + srcPath
+                  + " previous=" + Bytes.toStringBinary(prevKV.getFamily())
+                  + " current=" + Bytes.toStringBinary(kv.getFamily()));
+            }
+          }
+          prevKV = kv;
+        } while (scanner.next());
+      }
+    } finally {
+      if (reader != null) reader.close();
+    }
+  }
+
+  /**
+   * This method should only be called from HRegion.  It is assumed that the
+   * ranges of values in the HFile fit within the stores assigned region.
+   * (assertBulkLoadHFileOk checks this)
+   */
+  void bulkLoadHFile(String srcPathStr) throws IOException {
+    Path srcPath = new Path(srcPathStr);
+
+    // Move the file if it's on another filesystem
+    FileSystem srcFs = srcPath.getFileSystem(conf);
+    FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs;
+    //We can't compare FileSystem instances as
+    //equals() includes UGI instance as part of the comparison
+    //and won't work when doing SecureBulkLoad
+    //TODO deal with viewFS
+    if (!srcFs.getUri().equals(desFs.getUri())) {
+      LOG.info("File " + srcPath + " on different filesystem than " +
+          "destination store - moving to this filesystem.");
+      Path tmpPath = getTmpPath();
+      FileUtil.copy(srcFs, srcPath, fs, tmpPath, false, conf);
+      LOG.info("Copied to temporary path on dst filesystem: " + tmpPath);
+      srcPath = tmpPath;
+    }
+
+    Path dstPath = StoreFile.getRandomFilename(fs, homedir);
+    LOG.debug("Renaming bulk load file " + srcPath + " to " + dstPath);
+    StoreFile.rename(fs, srcPath, dstPath);
+
+    StoreFile sf = new StoreFile(fs, dstPath, this.conf, this.cacheConf,
+        this.family.getBloomFilterType(), this.dataBlockEncoder);
+    passSchemaMetricsTo(sf);
+
+    StoreFile.Reader r = sf.createReader();
+    this.storeSize += r.length();
+    this.totalUncompressedBytes += r.getTotalUncompressedBytes();
+
+    LOG.info("Moved hfile " + srcPath + " into store directory " +
+        homedir + " - updating store file list.");
+
+    // Append the new storefile into the list
+    this.lock.writeLock().lock();
+    try {
+      ArrayList newFiles = new ArrayList(storefiles);
+      newFiles.add(sf);
+      this.storefiles = sortAndClone(newFiles);
+    } finally {
+      // We need the lock, as long as we are updating the storefiles
+      // or changing the memstore. Let us release it before calling
+      // notifyChangeReadersObservers. See HBASE-4485 for a possible
+      // deadlock scenario that could have happened if continue to hold
+      // the lock.
+      this.lock.writeLock().unlock();
+    }
+    notifyChangedReadersObservers();
+    LOG.info("Successfully loaded store file " + srcPath
+        + " into store " + this + " (new location: " + dstPath + ")");
+  }
+
+  /**
+   * Get a temporary path in this region. These temporary files
+   * will get cleaned up when the region is re-opened if they are
+   * still around.
+   */
+  private Path getTmpPath() throws IOException {
+    return StoreFile.getRandomFilename(
+        fs, region.getTmpDir());
+  }
+
+  /**
+   * Close all the readers
+   *
+   * We don't need to worry about subsequent requests because the HRegion holds
+   * a write lock that will prevent any more reads or writes.
+   *
+   * @throws IOException
+   */
+  ImmutableList close() throws IOException {
+    this.lock.writeLock().lock();
+    try {
+      ImmutableList result = storefiles;
+
+      // Clear so metrics doesn't find them.
+      storefiles = ImmutableList.of();
+
+      if (!result.isEmpty()) {
+        // initialize the thread pool for closing store files in parallel.
+        ThreadPoolExecutor storeFileCloserThreadPool = this.region
+            .getStoreFileOpenAndCloseThreadPool("StoreFileCloserThread-"
+                + this.family.getNameAsString());
+
+        // close each store file in parallel
+        CompletionService completionService =
+          new ExecutorCompletionService(storeFileCloserThreadPool);
+        for (final StoreFile f : result) {
+          completionService.submit(new Callable() {
+            public Void call() throws IOException {
+              f.closeReader(true);
+              return null;
+            }
+          });
+        }
+
+        IOException ioe = null;
+        try {
+          for (int i = 0; i < result.size(); i++) {
+            try {
+              Future future = completionService.take();
+              future.get();
+            } catch (InterruptedException e) {
+              if (ioe == null) {
+                ioe = new InterruptedIOException();
+                ioe.initCause(e);
+              }
+            } catch (ExecutionException e) {
+              if (ioe == null) ioe = new IOException(e.getCause());
+            }
+          }
+        } finally {
+          storeFileCloserThreadPool.shutdownNow();
+        }
+        if (ioe != null) throw ioe;
+      }
+      LOG.info("Closed " + this);
+      return result;
+    } finally {
+      this.lock.writeLock().unlock();
+    }
+  }
+
+  /**
+   * Snapshot this stores memstore.  Call before running
+   * {@link #flushCache(long, SortedSet)} so it has some work to do.
+   */
+  void snapshot() {
+    this.memstore.snapshot();
+  }
+
+  /**
+   * Write out current snapshot.  Presumes {@link #snapshot()} has been called
+   * previously.
+   * @param logCacheFlushId flush sequence number
+   * @param snapshot
+   * @param snapshotTimeRangeTracker
+   * @param flushedSize The number of bytes flushed
+   * @param status
+   * @return Path The path name of the tmp file to which the store was flushed
+   * @throws IOException
+   */
+  protected Path flushCache(final long logCacheFlushId,
+      SortedSet snapshot,
+      TimeRangeTracker snapshotTimeRangeTracker,
+      AtomicLong flushedSize,
+      MonitoredTask status) throws IOException {
+    // If an exception happens flushing, we let it out without clearing
+    // the memstore snapshot.  The old snapshot will be returned when we say
+    // 'snapshot', the next time flush comes around.
+    // Retry after catching exception when flushing, otherwise server will abort
+    // itself
+    IOException lastException = null;
+    for (int i = 0; i < Store.flush_retries_number; i++) {
+      try {
+        Path pathName = internalFlushCache(snapshot, logCacheFlushId,
+            snapshotTimeRangeTracker, flushedSize, status);
+        try {
+          // Path name is null if there is no entry to flush
+          if (pathName != null) {
+            validateStoreFile(pathName);
+          }
+          return pathName;
+        } catch (Exception e) {
+          LOG.warn("Failed validating store file " + pathName
+              + ", retring num=" + i, e);
+          if (e instanceof IOException) {
+            lastException = (IOException) e;
+          } else {
+            lastException = new IOException(e);
+          }
+        }
+      } catch (IOException e) {
+        LOG.warn("Failed flushing store file, retring num=" + i, e);
+        lastException = e;
+      }
+      if (lastException != null) {
+        try {
+          Thread.sleep(pauseTime);
+        } catch (InterruptedException e) {
+          IOException iie = new InterruptedIOException();
+          iie.initCause(e);
+          throw iie;
+        }
+      }
+    }
+    throw lastException;
+  }
+
+  /*
+   * @param cache
+   * @param logCacheFlushId
+   * @param snapshotTimeRangeTracker
+   * @param flushedSize The number of bytes flushed
+   * @return Path The path name of the tmp file to which the store was flushed
+   * @throws IOException
+   */
+  private Path internalFlushCache(final SortedSet set,
+      final long logCacheFlushId,
+      TimeRangeTracker snapshotTimeRangeTracker,
+      AtomicLong flushedSize,
+      MonitoredTask status)
+      throws IOException {
+    StoreFile.Writer writer;
+    // Find the smallest read point across all the Scanners.
+    long smallestReadPoint = region.getSmallestReadPoint();
+    long flushed = 0;
+    Path pathName;
+    // Don't flush if there are no entries.
+    if (set.size() == 0) {
+      return null;
+    }
+    // Use a store scanner to find which rows to flush.
+    // Note that we need to retain deletes, hence
+    // treat this as a minor compaction.
+    InternalScanner scanner = null;
+    KeyValueScanner memstoreScanner = new CollectionBackedScanner(set, this.comparator);
+    if (getHRegion().getCoprocessorHost() != null) {
+      scanner = getHRegion().getCoprocessorHost().preFlushScannerOpen(this, memstoreScanner);
+    }
+    if (scanner == null) {
+      Scan scan = new Scan();
+      scan.setMaxVersions(scanInfo.getMaxVersions());
+      scanner = new StoreScanner(this, scanInfo, scan,
+          Collections.singletonList(memstoreScanner), ScanType.MINOR_COMPACT,
+          this.region.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
+    }
+    if (getHRegion().getCoprocessorHost() != null) {
+      InternalScanner cpScanner =
+        getHRegion().getCoprocessorHost().preFlush(this, scanner);
+      // NULL scanner returned from coprocessor hooks means skip normal processing
+      if (cpScanner == null) {
+        return null;
+      }
+      scanner = cpScanner;
+    }
+    try {
+      int compactionKVMax = conf.getInt(HConstants.COMPACTION_KV_MAX, 10);
+      // TODO:  We can fail in the below block before we complete adding this
+      // flush to list of store files.  Add cleanup of anything put on filesystem
+      // if we fail.
+      synchronized (flushLock) {
+        status.setStatus("Flushing " + this + ": creating writer");
+        // A. Write the map out to the disk
+        writer = createWriterInTmp(set.size());
+        writer.setTimeRangeTracker(snapshotTimeRangeTracker);
+        pathName = writer.getPath();
+        try {
+          List kvs = new ArrayList();
+          boolean hasMore;
+          do {
+            hasMore = scanner.next(kvs, compactionKVMax);
+            if (!kvs.isEmpty()) {
+              for (KeyValue kv : kvs) {
+                // If we know that this KV is going to be included always, then let us
+                // set its memstoreTS to 0. This will help us save space when writing to disk.
+                if (kv.getMemstoreTS() <= smallestReadPoint) {
+                  // let us not change the original KV. It could be in the memstore
+                  // changing its memstoreTS could affect other threads/scanners.
+                  kv = kv.shallowCopy();
+                  kv.setMemstoreTS(0);
+                }
+                writer.append(kv);
+                flushed += this.memstore.heapSizeChange(kv, true);
+              }
+              kvs.clear();
+            }
+          } while (hasMore);
+        } finally {
+          // Write out the log sequence number that corresponds to this output
+          // hfile.  The hfile is current up to and including logCacheFlushId.
+          status.setStatus("Flushing " + this + ": appending metadata");
+          writer.appendMetadata(logCacheFlushId, false);
+          status.setStatus("Flushing " + this + ": closing flushed file");
+          writer.close();
+        }
+      }
+    } finally {
+      flushedSize.set(flushed);
+      scanner.close();
+    }
+    if (LOG.isInfoEnabled()) {
+      LOG.info("Flushed " +
+               ", sequenceid=" + logCacheFlushId +
+               ", memsize=" + StringUtils.humanReadableInt(flushed) +
+               ", into tmp file " + pathName);
+    }
+    return pathName;
+  }
+
+  /*
+   * @param path The pathname of the tmp file into which the store was flushed
+   * @param logCacheFlushId
+   * @return StoreFile created.
+   * @throws IOException
+   */
+  private StoreFile commitFile(final Path path,
+      final long logCacheFlushId,
+      TimeRangeTracker snapshotTimeRangeTracker,
+      AtomicLong flushedSize,
+      MonitoredTask status)
+      throws IOException {
+    // Write-out finished successfully, move into the right spot
+    String fileName = path.getName();
+    Path dstPath = new Path(homedir, fileName);
+    String msg = "Renaming flushed file at " + path + " to " + dstPath;
+    LOG.debug(msg);
+    status.setStatus("Flushing " + this + ": " + msg);
+    if (!HBaseFileSystem.renameDirForFileSystem(fs, path, dstPath)) {
+      LOG.warn("Unable to rename " + path + " to " + dstPath);
+    }
+
+    status.setStatus("Flushing " + this + ": reopening flushed file");
+    StoreFile sf = new StoreFile(this.fs, dstPath, this.conf, this.cacheConf,
+        this.family.getBloomFilterType(), this.dataBlockEncoder);
+    passSchemaMetricsTo(sf);
+
+    StoreFile.Reader r = sf.createReader();
+    this.storeSize += r.length();
+    this.totalUncompressedBytes += r.getTotalUncompressedBytes();
+
+    // This increments the metrics associated with total flushed bytes for this
+    // family. The overall flush count is stored in the static metrics and
+    // retrieved from HRegion.recentFlushes, which is set within
+    // HRegion.internalFlushcache, which indirectly calls this to actually do
+    // the flushing through the StoreFlusherImpl class
+    getSchemaMetrics().updatePersistentStoreMetric(
+        SchemaMetrics.StoreMetricType.FLUSH_SIZE, flushedSize.longValue());
+    if (LOG.isInfoEnabled()) {
+      LOG.info("Added " + sf + ", entries=" + r.getEntries() +
+        ", sequenceid=" + logCacheFlushId +
+        ", filesize=" + StringUtils.humanReadableInt(r.length()));
+    }
+    return sf;
+  }
+
+  /*
+   * @param maxKeyCount
+   * @return Writer for a new StoreFile in the tmp dir.
+   */
+  private StoreFile.Writer createWriterInTmp(int maxKeyCount)
+  throws IOException {
+    return createWriterInTmp(maxKeyCount, this.family.getCompression(), false, true);
+  }
+
+  /*
+   * @param maxKeyCount
+   * @param compression Compression algorithm to use
+   * @param isCompaction whether we are creating a new file in a compaction
+   * @return Writer for a new StoreFile in the tmp dir.
+   */
+  public StoreFile.Writer createWriterInTmp(int maxKeyCount,
+    Compression.Algorithm compression, boolean isCompaction, boolean includeMVCCReadpoint)
+  throws IOException {
+    final CacheConfig writerCacheConf;
+    if (isCompaction) {
+      // Don't cache data on write on compactions.
+      writerCacheConf = new CacheConfig(cacheConf);
+      writerCacheConf.setCacheDataOnWrite(false);
+    } else {
+      writerCacheConf = cacheConf;
+    }
+    StoreFile.Writer w = new StoreFile.WriterBuilder(conf, writerCacheConf,
+        fs, blocksize)
+            .withOutputDir(region.getTmpDir())
+            .withDataBlockEncoder(dataBlockEncoder)
+            .withComparator(comparator)
+            .withBloomType(family.getBloomFilterType())
+            .withMaxKeyCount(maxKeyCount)
+            .withChecksumType(checksumType)
+            .withBytesPerChecksum(bytesPerChecksum)
+            .withCompression(compression)
+            .includeMVCCReadpoint(includeMVCCReadpoint)
+            .build();
+    // The store file writer's path does not include the CF name, so we need
+    // to configure the HFile writer directly.
+    SchemaConfigured sc = (SchemaConfigured) w.writer;
+    SchemaConfigured.resetSchemaMetricsConf(sc);
+    passSchemaMetricsTo(sc);
+    return w;
+  }
+
+  /*
+   * Change storefiles adding into place the Reader produced by this new flush.
+   * @param sf
+   * @param set That was used to make the passed file p.
+   * @throws IOException
+   * @return Whether compaction is required.
+   */
+  private boolean updateStorefiles(final StoreFile sf,
+                                   final SortedSet set)
+  throws IOException {
+    this.lock.writeLock().lock();
+    try {
+      ArrayList newList = new ArrayList(storefiles);
+      newList.add(sf);
+      storefiles = sortAndClone(newList);
+
+      this.memstore.clearSnapshot(set);
+    } finally {
+      // We need the lock, as long as we are updating the storefiles
+      // or changing the memstore. Let us release it before calling
+      // notifyChangeReadersObservers. See HBASE-4485 for a possible
+      // deadlock scenario that could have happened if continue to hold
+      // the lock.
+      this.lock.writeLock().unlock();
+    }
+
+    // Tell listeners of the change in readers.
+    notifyChangedReadersObservers();
+
+    return needsCompaction();
+  }
+
+  /*
+   * Notify all observers that set of Readers has changed.
+   * @throws IOException
+   */
+  private void notifyChangedReadersObservers() throws IOException {
+    for (ChangedReadersObserver o: this.changedReaderObservers) {
+      o.updateReaders();
+    }
+  }
+
+  /**
+   * Get all scanners with no filtering based on TTL (that happens further down
+   * the line).
+   * @return all scanners for this store
+   */
+  protected List getScanners(boolean cacheBlocks,
+      boolean isGet,
+      boolean isCompaction,
+      ScanQueryMatcher matcher) throws IOException {
+    List storeFiles;
+    List memStoreScanners;
+    this.lock.readLock().lock();
+    try {
+      storeFiles = this.getStorefiles();
+      memStoreScanners = this.memstore.getScanners();
+    } finally {
+      this.lock.readLock().unlock();
+    }
+
+    // First the store file scanners
+
+    // TODO this used to get the store files in descending order,
+    // but now we get them in ascending order, which I think is
+    // actually more correct, since memstore get put at the end.
+    List sfScanners = StoreFileScanner
+      .getScannersForStoreFiles(storeFiles, cacheBlocks, isGet, isCompaction, matcher);
+    List scanners =
+      new ArrayList(sfScanners.size()+1);
+    scanners.addAll(sfScanners);
+    // Then the memstore scanners
+    scanners.addAll(memStoreScanners);
+    return scanners;
+  }
+
+  /*
+   * @param o Observer who wants to know about changes in set of Readers
+   */
+  void addChangedReaderObserver(ChangedReadersObserver o) {
+    this.changedReaderObservers.add(o);
+  }
+
+  /*
+   * @param o Observer no longer interested in changes in set of Readers.
+   */
+  void deleteChangedReaderObserver(ChangedReadersObserver o) {
+    // We don't check if observer present; it may not be (legitimately)
+    this.changedReaderObservers.remove(o);
+  }
+
+  //////////////////////////////////////////////////////////////////////////////
+  // Compaction
+  //////////////////////////////////////////////////////////////////////////////
+
+  /**
+   * Compact the StoreFiles.  This method may take some time, so the calling
+   * thread must be able to block for long periods.
+   *
+   * 
                  During this time, the Store can work as usual, getting values from
+   * StoreFiles and writing new StoreFiles from the memstore.
+   *
+   * Existing StoreFiles are not destroyed until the new compacted StoreFile is
+   * completely written-out to disk.
+   *
+   * 
                  The compactLock prevents multiple simultaneous compactions.
+   * The structureLock prevents us from interfering with other write operations.
+   *
+   * 
                  We don't want to hold the structureLock for the whole time, as a compact()
+   * can be lengthy and we want to allow cache-flushes during this period.
+   *
+   * @param cr
+   *          compaction details obtained from requestCompaction()
+   * @throws IOException
+   * @return Storefile we compacted into or null if we failed or opted out early.
+   */
+  StoreFile compact(CompactionRequest cr) throws IOException {
+    if (cr == null || cr.getFiles().isEmpty()) return null;
+    Preconditions.checkArgument(cr.getStore().toString().equals(this.toString()));
+    List filesToCompact = cr.getFiles();
+    synchronized (filesCompacting) {
+      // sanity check: we're compacting files that this store knows about
+      // TODO: change this to LOG.error() after more debugging
+      Preconditions.checkArgument(filesCompacting.containsAll(filesToCompact));
+    }
+
+    // Max-sequenceID is the last key in the files we're compacting
+    long maxId = StoreFile.getMaxSequenceIdInList(filesToCompact);
+
+    // Ready to go. Have list of files to compact.
+    LOG.info("Starting compaction of " + filesToCompact.size() + " file(s) in "
+        + this + " of "
+        + this.region.getRegionInfo().getRegionNameAsString()
+        + " into tmpdir=" + region.getTmpDir() + ", seqid=" + maxId + ", totalSize="
+        + StringUtils.humanReadableInt(cr.getSize()));
+
+    StoreFile sf = null;
+    try {
+      StoreFile.Writer writer = this.compactor.compact(cr, maxId);
+      // Move the compaction into place.
+      if (this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
+        sf = completeCompaction(filesToCompact, writer);
+        if (region.getCoprocessorHost() != null) {
+          region.getCoprocessorHost().postCompact(this, sf, cr);
+        }
+      } else {
+        // Create storefile around what we wrote with a reader on it.
+        sf = new StoreFile(this.fs, writer.getPath(), this.conf, this.cacheConf,
+          this.family.getBloomFilterType(), this.dataBlockEncoder);
+        sf.createReader();
+      }
+    } finally {
+      synchronized (filesCompacting) {
+        filesCompacting.removeAll(filesToCompact);
+      }
+    }
+
+    LOG.info("Completed" + (cr.isMajor() ? " major " : " ") + "compaction of "
+        + filesToCompact.size() + " file(s) in " + this + " of "
+        + this.region.getRegionInfo().getRegionNameAsString()
+        + " into " +
+        (sf == null ? "none" : sf.getPath().getName()) +
+        ", size=" + (sf == null ? "none" :
+          StringUtils.humanReadableInt(sf.getReader().length()))
+        + "; total size for store is "
+        + StringUtils.humanReadableInt(storeSize));
+    return sf;
+  }
+
+  /**
+   * Compact the most recent N files. Used in testing.
+   */
+  public void compactRecentForTesting(int N) throws IOException {
+    List filesToCompact;
+    long maxId;
+    boolean isMajor;
+
+    this.lock.readLock().lock();
+    try {
+      synchronized (filesCompacting) {
+        filesToCompact = Lists.newArrayList(storefiles);
+        if (!filesCompacting.isEmpty()) {
+          // exclude all files older than the newest file we're currently
+          // compacting. this allows us to preserve contiguity (HBASE-2856)
+          StoreFile last = filesCompacting.get(filesCompacting.size() - 1);
+          int idx = filesToCompact.indexOf(last);
+          Preconditions.checkArgument(idx != -1);
+          filesToCompact.subList(0, idx + 1).clear();
+        }
+        int count = filesToCompact.size();
+        if (N > count) {
+          throw new RuntimeException("Not enough files");
+        }
+
+        filesToCompact = filesToCompact.subList(count - N, count);
+        maxId = StoreFile.getMaxSequenceIdInList(filesToCompact);
+        isMajor = (filesToCompact.size() == storefiles.size());
+        filesCompacting.addAll(filesToCompact);
+        Collections.sort(filesCompacting, StoreFile.Comparators.FLUSH_TIME);
+      }
+    } finally {
+      this.lock.readLock().unlock();
+    }
+
+    try {
+      // Ready to go. Have list of files to compact.
+      StoreFile.Writer writer = this.compactor.compactForTesting(this, conf, filesToCompact,
+        isMajor, maxId);
+      // Move the compaction into place.
+      StoreFile sf = completeCompaction(filesToCompact, writer);
+      if (region.getCoprocessorHost() != null) {
+        region.getCoprocessorHost().postCompact(this, sf, null);
+      }
+    } finally {
+      synchronized (filesCompacting) {
+        filesCompacting.removeAll(filesToCompact);
+      }
+    }
+  }
+
+  boolean hasReferences() {
+    return hasReferences(this.storefiles);
+  }
+
+  /*
+   * @param files
+   * @return True if any of the files in files are References.
+   */
+  private boolean hasReferences(Collection files) {
+    if (files != null && files.size() > 0) {
+      for (StoreFile hsf: files) {
+        if (hsf.isReference()) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  /*
+   * Gets lowest timestamp from candidate StoreFiles
+   *
+   * @param fs
+   * @param dir
+   * @throws IOException
+   */
+  public static long getLowestTimestamp(final List candidates)
+      throws IOException {
+    long minTs = Long.MAX_VALUE;
+    for (StoreFile storeFile : candidates) {
+      minTs = Math.min(minTs, storeFile.getModificationTimeStamp());
+    }
+    return minTs;
+  }
+
+  /** getter for CompactionProgress object
+   * @return CompactionProgress object; can be null
+   */
+  public CompactionProgress getCompactionProgress() {
+    return this.compactor.getProgress();
+  }
+
+  /*
+   * @return True if we should run a major compaction.
+   */
+  boolean isMajorCompaction() throws IOException {
+    for (StoreFile sf : this.storefiles) {
+      if (sf.getReader() == null) {
+        LOG.debug("StoreFile " + sf + " has null Reader");
+        return false;
+      }
+    }
+
+    List candidates = new ArrayList(this.storefiles);
+
+    // exclude files above the max compaction threshold
+    // except: save all references. we MUST compact them
+    int pos = 0;
+    while (pos < candidates.size() &&
+           candidates.get(pos).getReader().length() > this.maxCompactSize &&
+           !candidates.get(pos).isReference()) ++pos;
+    candidates.subList(0, pos).clear();
+
+    return isMajorCompaction(candidates);
+  }
+
+  /*
+   * @param filesToCompact Files to compact. Can be null.
+   * @return True if we should run a major compaction.
+   */
+  private boolean isMajorCompaction(final List filesToCompact) throws IOException {
+    boolean result = false;
+    long mcTime = getNextMajorCompactTime();
+    if (filesToCompact == null || filesToCompact.isEmpty() || mcTime == 0) {
+      return result;
+    }
+    // TODO: Use better method for determining stamp of last major (HBASE-2990)
+    long lowTimestamp = getLowestTimestamp(filesToCompact);
+    long now = System.currentTimeMillis();
+    if (lowTimestamp > 0l && lowTimestamp < (now - mcTime)) {
+      // Major compaction time has elapsed.
+      if (filesToCompact.size() == 1) {
+        // Single file
+        StoreFile sf = filesToCompact.get(0);
+        long oldest =
+            (sf.getReader().timeRangeTracker == null) ?
+                Long.MIN_VALUE :
+                now - sf.getReader().timeRangeTracker.minimumTimestamp;
+        if (sf.isMajorCompaction() &&
+            (this.ttl == HConstants.FOREVER || oldest < this.ttl)) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Skipping major compaction of " + this +
+                " because one (major) compacted file only and oldestTime " +
+                oldest + "ms is < ttl=" + this.ttl);
+          }
+        } else if (this.ttl != HConstants.FOREVER && oldest > this.ttl) {
+          LOG.debug("Major compaction triggered on store " + this +
+            ", because keyvalues outdated; time since last major compaction " +
+            (now - lowTimestamp) + "ms");
+          result = true;
+        }
+      } else {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Major compaction triggered on store " + this +
+              "; time since last major compaction " + (now - lowTimestamp) + "ms");
+        }
+        result = true;
+      }
+    }
+    return result;
+  }
+
+  long getNextMajorCompactTime() {
+    // default = 24hrs
+    long ret = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24);
+    if (family.getValue(HConstants.MAJOR_COMPACTION_PERIOD) != null) {
+      String strCompactionTime =
+        family.getValue(HConstants.MAJOR_COMPACTION_PERIOD);
+      ret = (new Long(strCompactionTime)).longValue();
+    }
+
+    if (ret > 0) {
+      // default = 20% = +/- 4.8 hrs
+      double jitterPct =  conf.getFloat("hbase.hregion.majorcompaction.jitter",
+          0.20F);
+      if (jitterPct > 0) {
+        long jitter = Math.round(ret * jitterPct);
+        // deterministic jitter avoids a major compaction storm on restart
+        ImmutableList snapshot = storefiles;
+        if (snapshot != null && !snapshot.isEmpty()) {
+          String seed = snapshot.get(0).getPath().getName();
+          double curRand = new Random(seed.hashCode()).nextDouble();
+          ret += jitter - Math.round(2L * jitter * curRand);
+        } else {
+          ret = 0; // no storefiles == no major compaction
+        }
+      }
+    }
+    return ret;
+  }
+
+  public CompactionRequest requestCompaction() throws IOException {
+    return requestCompaction(NO_PRIORITY, null);
+  }
+
+  public CompactionRequest requestCompaction(int priority, CompactionRequest request)
+      throws IOException {
+    // don't even select for compaction if writes are disabled
+    if (!this.region.areWritesEnabled()) {
+      return null;
+    }
+
+    this.lock.readLock().lock();
+    try {
+      synchronized (filesCompacting) {
+        // candidates = all storefiles not already in compaction queue
+        List candidates = Lists.newArrayList(storefiles);
+        if (!filesCompacting.isEmpty()) {
+          // exclude all files older than the newest file we're currently
+          // compacting. this allows us to preserve contiguity (HBASE-2856)
+          StoreFile last = filesCompacting.get(filesCompacting.size() - 1);
+          int idx = candidates.indexOf(last);
+          Preconditions.checkArgument(idx != -1);
+          candidates.subList(0, idx + 1).clear();
+        }
+
+        boolean override = false;
+        if (region.getCoprocessorHost() != null) {
+          override = region.getCoprocessorHost().preCompactSelection(this, candidates, request);
+        }
+        CompactSelection filesToCompact;
+        if (override) {
+          // coprocessor is overriding normal file selection
+          filesToCompact = new CompactSelection(conf, candidates);
+        } else {
+          filesToCompact = compactSelection(candidates, priority);
+        }
+
+        if (region.getCoprocessorHost() != null) {
+          region.getCoprocessorHost().postCompactSelection(this,
+            ImmutableList.copyOf(filesToCompact.getFilesToCompact()), request);
+        }
+
+        // no files to compact
+        if (filesToCompact.getFilesToCompact().isEmpty()) {
+          return null;
+        }
+
+        // basic sanity check: do not try to compact the same StoreFile twice.
+        if (!Collections.disjoint(filesCompacting, filesToCompact.getFilesToCompact())) {
+          // TODO: change this from an IAE to LOG.error after sufficient testing
+          Preconditions.checkArgument(false, "%s overlaps with %s",
+              filesToCompact, filesCompacting);
+        }
+        filesCompacting.addAll(filesToCompact.getFilesToCompact());
+        Collections.sort(filesCompacting, StoreFile.Comparators.FLUSH_TIME);
+
+        // major compaction iff all StoreFiles are included
+        boolean isMajor = (filesToCompact.getFilesToCompact().size() == this.storefiles.size());
+        if (isMajor) {
+          // since we're enqueuing a major, update the compaction wait interval
+          this.forceMajor = false;
+        }
+
+        // everything went better than expected. create a compaction request
+        int pri = getCompactPriority(priority);
+        //not a special compaction request, so we need to make one
+        if(request == null){
+          request = new CompactionRequest(region, this, filesToCompact, isMajor, pri);
+        } else {
+          // update the request with what the system thinks the request should be
+          // its up to the request if it wants to listen
+          request.setSelection(filesToCompact);
+          request.setIsMajor(isMajor);
+          request.setPriority(pri);
+        }
+      }
+    } finally {
+      this.lock.readLock().unlock();
+    }
+    if (request != null) {
+      CompactionRequest.preRequest(request);
+    }
+    return request;
+  }
+
+  public void finishRequest(CompactionRequest cr) {
+    CompactionRequest.postRequest(cr);
+    cr.finishRequest();
+    synchronized (filesCompacting) {
+      filesCompacting.removeAll(cr.getFiles());
+    }
+  }
+
+  /**
+   * Algorithm to choose which files to compact, see {@link #compactSelection(java.util.List, int)}
+   * @param candidates
+   * @return
+   * @throws IOException
+   */
+  CompactSelection compactSelection(List candidates) throws IOException {
+    return compactSelection(candidates,NO_PRIORITY);
+  }
+
+  /**
+   * Algorithm to choose which files to compact
+   *
+   * Configuration knobs:
+   *  "hbase.hstore.compaction.ratio"
+   *    normal case: minor compact when file <= sum(smaller_files) * ratio
+   *  "hbase.hstore.compaction.min.size"
+   *    unconditionally compact individual files below this size
+   *  "hbase.hstore.compaction.max.size"
+   *    never compact individual files above this size (unless splitting)
+   *  "hbase.hstore.compaction.min"
+   *    min files needed to minor compact
+   *  "hbase.hstore.compaction.max"
+   *    max files to compact at once (avoids OOM)
+   *
+   * @param candidates candidate files, ordered from oldest to newest
+   * @return subset copy of candidate list that meets compaction criteria
+   * @throws IOException
+   */
+  CompactSelection compactSelection(List candidates, int priority)
+      throws IOException {
+    // ASSUMPTION!!! filesCompacting is locked when calling this function
+
+    /* normal skew:
+     *
+     *         older ----> newer
+     *     _
+     *    | |   _
+     *    | |  | |   _
+     *  --|-|- |-|- |-|---_-------_-------  minCompactSize
+     *    | |  | |  | |  | |  _  | |
+     *    | |  | |  | |  | | | | | |
+     *    | |  | |  | |  | | | | | |
+     */
+    CompactSelection compactSelection = new CompactSelection(conf, candidates);
+
+    boolean forcemajor = this.forceMajor && filesCompacting.isEmpty();
+    if (!forcemajor) {
+      // Delete the expired store files before the compaction selection.
+      if (conf.getBoolean("hbase.store.delete.expired.storefile", true)
+          && (ttl != Long.MAX_VALUE) && (this.scanInfo.minVersions == 0)) {
+        CompactSelection expiredSelection = compactSelection
+            .selectExpiredStoreFilesToCompact(
+                EnvironmentEdgeManager.currentTimeMillis() - this.ttl);
+
+        // If there is any expired store files, delete them  by compaction.
+        if (expiredSelection != null) {
+          return expiredSelection;
+        }
+      }
+      // do not compact old files above a configurable threshold
+      // save all references. we MUST compact them
+      int pos = 0;
+      while (pos < compactSelection.getFilesToCompact().size() &&
+             compactSelection.getFilesToCompact().get(pos).getReader().length()
+               > maxCompactSize &&
+             !compactSelection.getFilesToCompact().get(pos).isReference()) ++pos;
+      if (pos != 0) compactSelection.clearSubList(0, pos);
+    }
+
+    if (compactSelection.getFilesToCompact().isEmpty()) {
+      LOG.debug(this.getHRegionInfo().getEncodedName() + " - " +
+        this + ": no store files to compact");
+      compactSelection.emptyFileList();
+      return compactSelection;
+    }
+
+    // Force a major compaction if this is a user-requested major compaction,
+    // or if we do not have too many files to compact and this was requested
+    // as a major compaction
+    boolean majorcompaction = (forcemajor && priority == PRIORITY_USER) ||
+      (forcemajor || isMajorCompaction(compactSelection.getFilesToCompact())) &&
+      (compactSelection.getFilesToCompact().size() < this.maxFilesToCompact
+    );
+    LOG.debug(this.getHRegionInfo().getEncodedName() + " - " +
+      this.getColumnFamilyName() + ": Initiating " +
+      (majorcompaction ? "major" : "minor") + "compaction");
+
+    if (!majorcompaction &&
+        !hasReferences(compactSelection.getFilesToCompact())) {
+      // we're doing a minor compaction, let's see what files are applicable
+      int start = 0;
+      double r = compactSelection.getCompactSelectionRatio();
+
+      // remove bulk import files that request to be excluded from minors
+      compactSelection.getFilesToCompact().removeAll(Collections2.filter(
+          compactSelection.getFilesToCompact(),
+          new Predicate() {
+            public boolean apply(StoreFile input) {
+              return input.excludeFromMinorCompaction();
+            }
+          }));
+
+      // skip selection algorithm if we don't have enough files
+      if (compactSelection.getFilesToCompact().size() < this.minFilesToCompact) {
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("Not compacting files because we only have " +
+            compactSelection.getFilesToCompact().size() +
+            " files ready for compaction.  Need " + this.minFilesToCompact + " to initiate.");
+        }
+        compactSelection.emptyFileList();
+        return compactSelection;
+      }
+
+      /* TODO: add sorting + unit test back in when HBASE-2856 is fixed
+      // Sort files by size to correct when normal skew is altered by bulk load.
+      Collections.sort(filesToCompact, StoreFile.Comparators.FILE_SIZE);
+       */
+
+      // get store file sizes for incremental compacting selection.
+      int countOfFiles = compactSelection.getFilesToCompact().size();
+      long [] fileSizes = new long[countOfFiles];
+      long [] sumSize = new long[countOfFiles];
+      for (int i = countOfFiles-1; i >= 0; --i) {
+        StoreFile file = compactSelection.getFilesToCompact().get(i);
+        fileSizes[i] = file.getReader().length();
+        // calculate the sum of fileSizes[i,i+maxFilesToCompact-1) for algo
+        int tooFar = i + this.maxFilesToCompact - 1;
+        sumSize[i] = fileSizes[i]
+                   + ((i+1    < countOfFiles) ? sumSize[i+1]      : 0)
+                   - ((tooFar < countOfFiles) ? fileSizes[tooFar] : 0);
+      }
+
+      /* Start at the oldest file and stop when you find the first file that
+       * meets compaction criteria:
+       *   (1) a recently-flushed, small file (i.e. <= minCompactSize)
+       *      OR
+       *   (2) within the compactRatio of sum(newer_files)
+       * Given normal skew, any newer files will also meet this criteria
+       *
+       * Additional Note:
+       * If fileSizes.size() >> maxFilesToCompact, we will recurse on
+       * compact().  Consider the oldest files first to avoid a
+       * situation where we always compact [end-threshold,end).  Then, the
+       * last file becomes an aggregate of the previous compactions.
+       */
+      while(countOfFiles - start >= this.minFilesToCompact &&
+            fileSizes[start] >
+              Math.max(minCompactSize, (long)(sumSize[start+1] * r))) {
+        ++start;
+      }
+      int end = Math.min(countOfFiles, start + this.maxFilesToCompact);
+      long totalSize = fileSizes[start]
+                     + ((start+1 < countOfFiles) ? sumSize[start+1] : 0);
+      compactSelection = compactSelection.getSubList(start, end);
+
+      // if we don't have enough files to compact, just wait
+      if (compactSelection.getFilesToCompact().size() < this.minFilesToCompact) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Skipped compaction of " + this
+            + ".  Only " + (end - start) + " file(s) of size "
+            + StringUtils.humanReadableInt(totalSize)
+            + " have met compaction criteria.");
+        }
+        compactSelection.emptyFileList();
+        return compactSelection;
+      }
+    } else {
+      if(majorcompaction) {
+        if (compactSelection.getFilesToCompact().size() > this.maxFilesToCompact) {
+          LOG.debug("Warning, compacting more than " + this.maxFilesToCompact +
+            " files, probably because of a user-requested major compaction");
+          if(priority != PRIORITY_USER) {
+            LOG.error("Compacting more than max files on a non user-requested compaction");
+          }
+        }
+      } else if (compactSelection.getFilesToCompact().size() > this.maxFilesToCompact) {
+        // all files included in this compaction, up to max
+        int pastMax = compactSelection.getFilesToCompact().size() - this.maxFilesToCompact;
+        compactSelection.getFilesToCompact().subList(0, pastMax).clear();
+      }
+    }
+    return compactSelection;
+  }
+
+  /**
+   * Validates a store file by opening and closing it. In HFileV2 this should
+   * not be an expensive operation.
+   *
+   * @param path the path to the store file
+   */
+  private void validateStoreFile(Path path)
+      throws IOException {
+    StoreFile storeFile = null;
+    try {
+      storeFile = new StoreFile(this.fs, path, this.conf,
+          this.cacheConf, this.family.getBloomFilterType(),
+          NoOpDataBlockEncoder.INSTANCE);
+      passSchemaMetricsTo(storeFile);
+      storeFile.createReader();
+    } catch (IOException e) {
+      LOG.error("Failed to open store file : " + path
+          + ", keeping it in tmp location", e);
+      throw e;
+    } finally {
+      if (storeFile != null) {
+        storeFile.closeReader(false);
+      }
+    }
+  }
+
+  /*
+   * 
                  It works by processing a compaction that's been written to disk.
+   *
+   * 
                  It is usually invoked at the end of a compaction, but might also be
+   * invoked at HStore startup, if the prior execution died midway through.
+   *
+   * 
                  Moving the compacted TreeMap into place means:
+   * 
                  +   * 1) Moving the new compacted StoreFile into place
+   * 2) Unload all replaced StoreFile, close and collect list to delete.
+   * 3) Loading the new TreeMap.
+   * 4) Compute new store size
+   * 
                  
+   *
+   * @param compactedFiles list of files that were compacted
+   * @param compactedFile StoreFile that is the result of the compaction
+   * @return StoreFile created. May be null.
+   * @throws IOException
+   */
+  StoreFile completeCompaction(final Collection compactedFiles,
+                                       final StoreFile.Writer compactedFile)
+      throws IOException {
+    // 1. Moving the new files into place -- if there is a new file (may not
+    // be if all cells were expired or deleted).
+    StoreFile result = null;
+    if (compactedFile != null) {
+      validateStoreFile(compactedFile.getPath());
+      // Move the file into the right spot
+      Path origPath = compactedFile.getPath();
+      Path destPath = new Path(homedir, origPath.getName());
+      LOG.info("Renaming compacted file at " + origPath + " to " + destPath);
+      if (!HBaseFileSystem.renameDirForFileSystem(fs, origPath, destPath)) {
+        LOG.error("Failed move of compacted file " + origPath + " to " +
+            destPath);
+        throw new IOException("Failed move of compacted file " + origPath +
+            " to " + destPath);
+      }
+      result = new StoreFile(this.fs, destPath, this.conf, this.cacheConf,
+          this.family.getBloomFilterType(), this.dataBlockEncoder);
+      passSchemaMetricsTo(result);
+      result.createReader();
+    }
+    try {
+      this.lock.writeLock().lock();
+      try {
+        // Change this.storefiles so it reflects new state but do not
+        // delete old store files until we have sent out notification of
+        // change in case old files are still being accessed by outstanding
+        // scanners.
+        ArrayList newStoreFiles = Lists.newArrayList(storefiles);
+        newStoreFiles.removeAll(compactedFiles);
+        filesCompacting.removeAll(compactedFiles); // safe bc: lock.writeLock()
+
+        // If a StoreFile result, move it into place.  May be null.
+        if (result != null) {
+          newStoreFiles.add(result);
+        }
+
+        this.storefiles = sortAndClone(newStoreFiles);
+      } finally {
+        // We need the lock, as long as we are updating the storefiles
+        // or changing the memstore. Let us release it before calling
+        // notifyChangeReadersObservers. See HBASE-4485 for a possible
+        // deadlock scenario that could have happened if continue to hold
+        // the lock.
+        this.lock.writeLock().unlock();
+      }
+
+      // Tell observers that list of StoreFiles has changed.
+      notifyChangedReadersObservers();
+
+      // let the archive util decide if we should archive or delete the files
+      LOG.debug("Removing store files after compaction...");
+      HFileArchiver.archiveStoreFiles(this.conf, this.fs, this.region, this.family.getName(),
+        compactedFiles);
+
+    } catch (IOException e) {
+      e = RemoteExceptionHandler.checkIOException(e);
+      LOG.error("Failed replacing compacted files in " + this +
+        ". Compacted file is " + (result == null? "none": result.toString()) +
+        ".  Files replaced " + compactedFiles.toString() +
+        " some of which may have been already removed", e);
+    }
+
+    // 4. Compute new store size
+    this.storeSize = 0L;
+    this.totalUncompressedBytes = 0L;
+    for (StoreFile hsf : this.storefiles) {
+      StoreFile.Reader r = hsf.getReader();
+      if (r == null) {
+        LOG.warn("StoreFile " + hsf + " has a null Reader");
+        continue;
+      }
+      this.storeSize += r.length();
+      this.totalUncompressedBytes += r.getTotalUncompressedBytes();
+    }
+    return result;
+  }
+
+  public ImmutableList sortAndClone(List storeFiles) {
+    Collections.sort(storeFiles, StoreFile.Comparators.FLUSH_TIME);
+    ImmutableList newList = ImmutableList.copyOf(storeFiles);
+    return newList;
+  }
+
+  // ////////////////////////////////////////////////////////////////////////////
+  // Accessors.
+  // (This is the only section that is directly useful!)
+  //////////////////////////////////////////////////////////////////////////////
+  /**
+   * @return the number of files in this store
+   */
+  public int getNumberOfStoreFiles() {
+    return this.storefiles.size();
+  }
+
+  /*
+   * @param wantedVersions How many versions were asked for.
+   * @return wantedVersions or this families' {@link HConstants#VERSIONS}.
+   */
+  int versionsToReturn(final int wantedVersions) {
+    if (wantedVersions <= 0) {
+      throw new IllegalArgumentException("Number of versions must be > 0");
+    }
+    // Make sure we do not return more than maximum versions for this store.
+    int maxVersions = this.family.getMaxVersions();
+    return wantedVersions > maxVersions ? maxVersions: wantedVersions;
+  }
+
+  static boolean isExpired(final KeyValue key, final long oldestTimestamp) {
+    return key.getTimestamp() < oldestTimestamp;
+  }
+
+  /**
+   * Find the key that matches row exactly, or the one that immediately
+   * precedes it. WARNING: Only use this method on a table where writes occur
+   * with strictly increasing timestamps. This method assumes this pattern of
+   * writes in order to make it reasonably performant.  Also our search is
+   * dependent on the axiom that deletes are for cells that are in the container
+   * that follows whether a memstore snapshot or a storefile, not for the
+   * current container: i.e. we'll see deletes before we come across cells we
+   * are to delete. Presumption is that the memstore#kvset is processed before
+   * memstore#snapshot and so on.
+   * @param row The row key of the targeted row.
+   * @return Found keyvalue or null if none found.
+   * @throws IOException
+   */
+  KeyValue getRowKeyAtOrBefore(final byte[] row) throws IOException {
+    // If minVersions is set, we will not ignore expired KVs.
+    // As we're only looking for the latest matches, that should be OK.
+    // With minVersions > 0 we guarantee that any KV that has any version
+    // at all (expired or not) has at least one version that will not expire.
+    // Note that this method used to take a KeyValue as arguments. KeyValue
+    // can be back-dated, a row key cannot.
+    long ttlToUse = scanInfo.getMinVersions() > 0 ? Long.MAX_VALUE : this.ttl;
+
+    KeyValue kv = new KeyValue(row, HConstants.LATEST_TIMESTAMP);
+
+    GetClosestRowBeforeTracker state = new GetClosestRowBeforeTracker(
+      this.comparator, kv, ttlToUse, this.region.getRegionInfo().isMetaRegion());
+    this.lock.readLock().lock();
+    try {
+      // First go to the memstore.  Pick up deletes and candidates.
+      this.memstore.getRowKeyAtOrBefore(state);
+      // Check if match, if we got a candidate on the asked for 'kv' row.
+      // Process each store file. Run through from newest to oldest.
+      for (StoreFile sf : Lists.reverse(storefiles)) {
+        // Update the candidate keys from the current map file
+        rowAtOrBeforeFromStoreFile(sf, state);
+      }
+      return state.getCandidate();
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+
+  /*
+   * Check an individual MapFile for the row at or before a given row.
+   * @param f
+   * @param state
+   * @throws IOException
+   */
+  private void rowAtOrBeforeFromStoreFile(final StoreFile f,
+                                          final GetClosestRowBeforeTracker state)
+      throws IOException {
+    StoreFile.Reader r = f.getReader();
+    if (r == null) {
+      LOG.warn("StoreFile " + f + " has a null Reader");
+      return;
+    }
+    if (r.getEntries() == 0) {
+      LOG.warn("StoreFile " + f + " is a empty store file");
+      return;
+    }
+    // TODO: Cache these keys rather than make each time?
+    byte [] fk = r.getFirstKey();
+    if (fk == null) return;
+    KeyValue firstKV = KeyValue.createKeyValueFromKey(fk, 0, fk.length);
+    byte [] lk = r.getLastKey();
+    KeyValue lastKV = KeyValue.createKeyValueFromKey(lk, 0, lk.length);
+    KeyValue firstOnRow = state.getTargetKey();
+    if (this.comparator.compareRows(lastKV, firstOnRow) < 0) {
+      // If last key in file is not of the target table, no candidates in this
+      // file.  Return.
+      if (!state.isTargetTable(lastKV)) return;
+      // If the row we're looking for is past the end of file, set search key to
+      // last key. TODO: Cache last and first key rather than make each time.
+      firstOnRow = new KeyValue(lastKV.getRow(), HConstants.LATEST_TIMESTAMP);
+    }
+    // Get a scanner that caches blocks and that uses pread.
+    HFileScanner scanner = r.getScanner(true, true, false);
+    // Seek scanner.  If can't seek it, return.
+    if (!seekToScanner(scanner, firstOnRow, firstKV)) return;
+    // If we found candidate on firstOnRow, just return. THIS WILL NEVER HAPPEN!
+    // Unlikely that there'll be an instance of actual first row in table.
+    if (walkForwardInSingleRow(scanner, firstOnRow, state)) return;
+    // If here, need to start backing up.
+    while (scanner.seekBefore(firstOnRow.getBuffer(), firstOnRow.getKeyOffset(),
+       firstOnRow.getKeyLength())) {
+      KeyValue kv = scanner.getKeyValue();
+      if (!state.isTargetTable(kv)) break;
+      if (!state.isBetterCandidate(kv)) break;
+      // Make new first on row.
+      firstOnRow = new KeyValue(kv.getRow(), HConstants.LATEST_TIMESTAMP);
+      // Seek scanner.  If can't seek it, break.
+      if (!seekToScanner(scanner, firstOnRow, firstKV)) break;
+      // If we find something, break;
+      if (walkForwardInSingleRow(scanner, firstOnRow, state)) break;
+    }
+  }
+
+  /*
+   * Seek the file scanner to firstOnRow or first entry in file.
+   * @param scanner
+   * @param firstOnRow
+   * @param firstKV
+   * @return True if we successfully seeked scanner.
+   * @throws IOException
+   */
+  private boolean seekToScanner(final HFileScanner scanner,
+                                final KeyValue firstOnRow,
+                                final KeyValue firstKV)
+      throws IOException {
+    KeyValue kv = firstOnRow;
+    // If firstOnRow < firstKV, set to firstKV
+    if (this.comparator.compareRows(firstKV, firstOnRow) == 0) kv = firstKV;
+    int result = scanner.seekTo(kv.getBuffer(), kv.getKeyOffset(),
+      kv.getKeyLength());
+    return result >= 0;
+  }
+
+  /*
+   * When we come in here, we are probably at the kv just before we break into
+   * the row that firstOnRow is on.  Usually need to increment one time to get
+   * on to the row we are interested in.
+   * @param scanner
+   * @param firstOnRow
+   * @param state
+   * @return True we found a candidate.
+   * @throws IOException
+   */
+  private boolean walkForwardInSingleRow(final HFileScanner scanner,
+                                         final KeyValue firstOnRow,
+                                         final GetClosestRowBeforeTracker state)
+      throws IOException {
+    boolean foundCandidate = false;
+    do {
+      KeyValue kv = scanner.getKeyValue();
+      // If we are not in the row, skip.
+      if (this.comparator.compareRows(kv, firstOnRow) < 0) continue;
+      // Did we go beyond the target row? If so break.
+      if (state.isTooFar(kv, firstOnRow)) break;
+      if (state.isExpired(kv)) {
+        continue;
+      }
+      // If we added something, this row is a contender. break.
+      if (state.handle(kv)) {
+        foundCandidate = true;
+        break;
+      }
+    } while(scanner.next());
+    return foundCandidate;
+  }
+
+  public boolean canSplit() {
+    this.lock.readLock().lock();
+    try {
+      // Not splitable if we find a reference store file present in the store.
+      for (StoreFile sf : storefiles) {
+        if (sf.isReference()) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug(sf + " is not splittable");
+          }
+          return false;
+        }
+      }
+
+      return true;
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+  /**
+   * Determines if Store should be split
+   * @return byte[] if store should be split, null otherwise.
+   */
+  public byte[] getSplitPoint() {
+    this.lock.readLock().lock();
+    try {
+      // sanity checks
+      if (this.storefiles.isEmpty()) {
+        return null;
+      }
+      // Should already be enforced by the split policy!
+      assert !this.region.getRegionInfo().isMetaRegion();
+
+      // Not splitable if we find a reference store file present in the store.
+      long maxSize = 0L;
+      StoreFile largestSf = null;
+      for (StoreFile sf : storefiles) {
+        if (sf.isReference()) {
+          // Should already be enforced since we return false in this case
+          assert false : "getSplitPoint() called on a region that can't split!";
+          return null;
+        }
+
+        StoreFile.Reader r = sf.getReader();
+        if (r == null) {
+          LOG.warn("Storefile " + sf + " Reader is null");
+          continue;
+        }
+
+        long size = r.length();
+        if (size > maxSize) {
+          // This is the largest one so far
+          maxSize = size;
+          largestSf = sf;
+        }
+      }
+
+      StoreFile.Reader r = largestSf.getReader();
+      if (r == null) {
+        LOG.warn("Storefile " + largestSf + " Reader is null");
+        return null;
+      }
+      // Get first, last, and mid keys.  Midkey is the key that starts block
+      // in middle of hfile.  Has column and timestamp.  Need to return just
+      // the row we want to split on as midkey.
+      byte [] midkey = r.midkey();
+      if (midkey != null) {
+        KeyValue mk = KeyValue.createKeyValueFromKey(midkey, 0, midkey.length);
+        byte [] fk = r.getFirstKey();
+        KeyValue firstKey = KeyValue.createKeyValueFromKey(fk, 0, fk.length);
+        byte [] lk = r.getLastKey();
+        KeyValue lastKey = KeyValue.createKeyValueFromKey(lk, 0, lk.length);
+        // if the midkey is the same as the first or last keys, then we cannot
+        // (ever) split this region.
+        if (this.comparator.compareRows(mk, firstKey) == 0 ||
+            this.comparator.compareRows(mk, lastKey) == 0) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("cannot split because midkey is the same as first or " +
+              "last row");
+          }
+          return null;
+        }
+        return mk.getRow();
+      }
+    } catch(IOException e) {
+      LOG.warn("Failed getting store size for " + this, e);
+    } finally {
+      this.lock.readLock().unlock();
+    }
+    return null;
+  }
+
+  /** @return aggregate size of all HStores used in the last compaction */
+  public long getLastCompactSize() {
+    return this.lastCompactSize;
+  }
+
+  /** @return aggregate size of HStore */
+  public long getSize() {
+    return storeSize;
+  }
+
+  public void triggerMajorCompaction() {
+    this.forceMajor = true;
+  }
+
+  boolean getForceMajorCompaction() {
+    return this.forceMajor;
+  }
+
+  //////////////////////////////////////////////////////////////////////////////
+  // File administration
+  //////////////////////////////////////////////////////////////////////////////
+
+  /**
+   * Return a scanner for both the memstore and the HStore files. Assumes we
+   * are not in a compaction.
+   * @throws IOException
+   */
+  public KeyValueScanner getScanner(Scan scan,
+      final NavigableSet targetCols) throws IOException {
+    lock.readLock().lock();
+    try {
+      KeyValueScanner scanner = null;
+      if (getHRegion().getCoprocessorHost() != null) {
+        scanner = getHRegion().getCoprocessorHost().preStoreScannerOpen(this, scan, targetCols);
+      }
+      if (scanner == null) {
+        scanner = new StoreScanner(this, getScanInfo(), scan, targetCols);
+      }
+      return scanner;
+    } finally {
+      lock.readLock().unlock();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return getColumnFamilyName();
+  }
+
+  /**
+   * @return Count of store files
+   */
+  int getStorefilesCount() {
+    return this.storefiles.size();
+  }
+
+  /**
+   * @return The size of the store files, in bytes, uncompressed.
+   */
+  long getStoreSizeUncompressed() {
+    return this.totalUncompressedBytes;
+  }
+
+  /**
+   * @return The size of the store files, in bytes.
+   */
+  long getStorefilesSize() {
+    long size = 0;
+    for (StoreFile s: storefiles) {
+      StoreFile.Reader r = s.getReader();
+      if (r == null) {
+        LOG.warn("StoreFile " + s + " has a null Reader");
+        continue;
+      }
+      size += r.length();
+    }
+    return size;
+  }
+
+  /**
+   * @return The size of the store file indexes, in bytes.
+   */
+  long getStorefilesIndexSize() {
+    long size = 0;
+    for (StoreFile s: storefiles) {
+      StoreFile.Reader r = s.getReader();
+      if (r == null) {
+        LOG.warn("StoreFile " + s + " has a null Reader");
+        continue;
+      }
+      size += r.indexSize();
+    }
+    return size;
+  }
+
+  /**
+   * Returns the total size of all index blocks in the data block indexes,
+   * including the root level, intermediate levels, and the leaf level for
+   * multi-level indexes, or just the root level for single-level indexes.
+   *
+   * @return the total size of block indexes in the store
+   */
+  long getTotalStaticIndexSize() {
+    long size = 0;
+    for (StoreFile s : storefiles) {
+      size += s.getReader().getUncompressedDataIndexSize();
+    }
+    return size;
+  }
+
+  /**
+   * Returns the total byte size of all Bloom filter bit arrays. For compound
+   * Bloom filters even the Bloom blocks currently not loaded into the block
+   * cache are counted.
+   *
+   * @return the total size of all Bloom filters in the store
+   */
+  long getTotalStaticBloomSize() {
+    long size = 0;
+    for (StoreFile s : storefiles) {
+      StoreFile.Reader r = s.getReader();
+      size += r.getTotalBloomSize();
+    }
+    return size;
+  }
+
+  /**
+   * @return The size of this store's memstore, in bytes
+   */
+  long getMemStoreSize() {
+    return this.memstore.heapSize();
+  }
+
+  public int getCompactPriority() {
+    return getCompactPriority(NO_PRIORITY);
+  }
+
+  /**
+   * @return The priority that this store should have in the compaction queue
+   * @param priority
+   */
+  public int getCompactPriority(int priority) {
+    // If this is a user-requested compaction, leave this at the highest priority
+    if(priority == PRIORITY_USER) {
+      return PRIORITY_USER;
+    } else {
+      return this.blockingStoreFileCount - this.storefiles.size();
+    }
+  }
+
+  boolean throttleCompaction(long compactionSize) {
+    long throttlePoint = conf.getLong(
+        "hbase.regionserver.thread.compaction.throttle",
+        2 * this.minFilesToCompact * this.region.memstoreFlushSize);
+    return compactionSize > throttlePoint;
+  }
+
+  public HRegion getHRegion() {
+    return this.region;
+  }
+
+  HRegionInfo getHRegionInfo() {
+    return this.region.getRegionInfo();
+  }
+
+  /**
+   * Increments the value for the given row/family/qualifier.
+   *
+   * This function will always be seen as atomic by other readers
+   * because it only puts a single KV to memstore. Thus no
+   * read/write control necessary.
+   *
+   * @param row
+   * @param f
+   * @param qualifier
+   * @param newValue the new value to set into memstore
+   * @return memstore size delta
+   * @throws IOException
+   */
+  public long updateColumnValue(byte [] row, byte [] f,
+                                byte [] qualifier, long newValue)
+      throws IOException {
+
+    this.lock.readLock().lock();
+    try {
+      long now = EnvironmentEdgeManager.currentTimeMillis();
+
+      return this.memstore.updateColumnValue(row,
+          f,
+          qualifier,
+          newValue,
+          now);
+
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+
+  /**
+   * Adds or replaces the specified KeyValues.
+   * 
                  
+   * For each KeyValue specified, if a cell with the same row, family, and
+   * qualifier exists in MemStore, it will be replaced.  Otherwise, it will just
+   * be inserted to MemStore.
+   * 
                  
+   * This operation is atomic on each KeyValue (row/family/qualifier) but not
+   * necessarily atomic across all of them.
+   * @param kvs
+   * @return memstore size delta
+   * @throws IOException
+   */
+  public long upsert(List kvs)
+      throws IOException {
+    this.lock.readLock().lock();
+    try {
+      // TODO: Make this operation atomic w/ MVCC
+      return this.memstore.upsert(kvs);
+    } finally {
+      this.lock.readLock().unlock();
+    }
+  }
+
+  public StoreFlusher getStoreFlusher(long cacheFlushId) {
+    return new StoreFlusherImpl(cacheFlushId);
+  }
+
+  private class StoreFlusherImpl implements StoreFlusher {
+
+    private long cacheFlushId;
+    private SortedSet snapshot;
+    private StoreFile storeFile;
+    private Path storeFilePath;
+    private TimeRangeTracker snapshotTimeRangeTracker;
+    private AtomicLong flushedSize;
+
+    private StoreFlusherImpl(long cacheFlushId) {
+      this.cacheFlushId = cacheFlushId;
+      this.flushedSize = new AtomicLong();
+    }
+
+    @Override
+    public void prepare() {
+      memstore.snapshot();
+      this.snapshot = memstore.getSnapshot();
+      this.snapshotTimeRangeTracker = memstore.getSnapshotTimeRangeTracker();
+    }
+
+    @Override
+    public void flushCache(MonitoredTask status) throws IOException {
+      storeFilePath = Store.this.flushCache(
+        cacheFlushId, snapshot, snapshotTimeRangeTracker, flushedSize, status);
+    }
+
+    @Override
+    public boolean commit(MonitoredTask status) throws IOException {
+      if (storeFilePath == null) {
+        return false;
+      }
+      storeFile = Store.this.commitFile(storeFilePath, cacheFlushId,
+                               snapshotTimeRangeTracker, flushedSize, status);
+      if (Store.this.getHRegion().getCoprocessorHost() != null) {
+        Store.this.getHRegion()
+            .getCoprocessorHost()
+            .postFlush(Store.this, storeFile);
+      }
+
+      // Add new file to store files.  Clear snapshot too while we have
+      // the Store write lock.
+      return Store.this.updateStorefiles(storeFile, snapshot);
+    }
+  }
+
+  /**
+   * See if there's too much store files in this store
+   * @return true if number of store files is greater than
+   *  the number defined in minFilesToCompact
+   */
+  public boolean needsCompaction() {
+    return (storefiles.size() - filesCompacting.size()) > minFilesToCompact;
+  }
+
+  /**
+   * Used for tests. Get the cache configuration for this Store.
+   */
+  public CacheConfig getCacheConfig() {
+    return this.cacheConf;
+  }
+
+  public static final long FIXED_OVERHEAD =
+      ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
+          + (17 * ClassSize.REFERENCE) + (7 * Bytes.SIZEOF_LONG)
+          + (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
+
+  public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
+      + ClassSize.OBJECT + ClassSize.REENTRANT_LOCK
+      + ClassSize.CONCURRENT_SKIPLISTMAP
+      + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + ClassSize.OBJECT
+      + ScanInfo.FIXED_OVERHEAD);
+
+  @Override
+  public long heapSize() {
+    return DEEP_OVERHEAD + this.memstore.heapSize();
+  }
+
+  public KeyValue.KVComparator getComparator() {
+    return comparator;
+  }
+
+  public ScanInfo getScanInfo() {
+    return scanInfo;
+  }
+  
+  public boolean hasTooManyStoreFiles() {
+    return getStorefilesCount() > this.blockingFileCount;
+  }
+
+  /**
+   * Immutable information for scans over a store.
+   */
+  public static class ScanInfo {
+    private byte[] family;
+    private int minVersions;
+    private int maxVersions;
+    private long ttl;
+    private boolean keepDeletedCells;
+    private long timeToPurgeDeletes;
+    private KVComparator comparator;
+
+    public static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT
+        + (2 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_INT)
+        + Bytes.SIZEOF_LONG + Bytes.SIZEOF_BOOLEAN);
+
+    /**
+     * @param family {@link HColumnDescriptor} describing the column family
+     * @param ttl Store's TTL (in ms)
+     * @param timeToPurgeDeletes duration in ms after which a delete marker can
+     *        be purged during a major compaction.
+     * @param comparator The store's comparator
+     */
+    public ScanInfo(HColumnDescriptor family, long ttl, long timeToPurgeDeletes, KVComparator comparator) {
+      this(family.getName(), family.getMinVersions(), family.getMaxVersions(), ttl, family
+          .getKeepDeletedCells(), timeToPurgeDeletes, comparator);
+    }
+    /**
+     * @param family Name of this store's column family
+     * @param minVersions Store's MIN_VERSIONS setting
+     * @param maxVersions Store's VERSIONS setting
+     * @param ttl Store's TTL (in ms)
+     * @param timeToPurgeDeletes duration in ms after which a delete marker can
+     *        be purged during a major compaction.
+     * @param keepDeletedCells Store's keepDeletedCells setting
+     * @param comparator The store's comparator
+     */
+    public ScanInfo(byte[] family, int minVersions, int maxVersions, long ttl,
+        boolean keepDeletedCells, long timeToPurgeDeletes,
+        KVComparator comparator) {
+
+      this.family = family;
+      this.minVersions = minVersions;
+      this.maxVersions = maxVersions;
+      this.ttl = ttl;
+      this.keepDeletedCells = keepDeletedCells;
+      this.timeToPurgeDeletes = timeToPurgeDeletes;
+      this.comparator = comparator;
+    }
+
+    public byte[] getFamily() {
+      return family;
+    }
+
+    public int getMinVersions() {
+      return minVersions;
+    }
+
+    public int getMaxVersions() {
+      return maxVersions;
+    }
+
+    public long getTtl() {
+      return ttl;
+    }
+
+    public boolean getKeepDeletedCells() {
+      return keepDeletedCells;
+    }
+
+    public long getTimeToPurgeDeletes() {
+      return timeToPurgeDeletes;
+    }
+
+    public KVComparator getComparator() {
+      return comparator;
+    }
+  }
+
+}
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
new file mode 100644
index 0000000..f93fbc0
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -0,0 +1,1934 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.DataInput;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseFileSystem;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.KVComparator;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.HalfStoreFileReader;
+import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.BlockType;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.io.hfile.HFileWriterV1;
+import org.apache.hadoop.hbase.io.hfile.HFileWriterV2;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.util.BloomFilter;
+import org.apache.hadoop.hbase.util.BloomFilterFactory;
+import org.apache.hadoop.hbase.util.BloomFilterWriter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.WritableUtils;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Ordering;
+
+/**
+ * A Store data file.  Stores usually have one or more of these files.  They
+ * are produced by flushing the memstore to disk.  To
+ * create, instantiate a writer using {@link StoreFile#WriterBuilder}
+ * and append data. Be sure to add any metadata before calling close on the
+ * Writer (Use the appendMetadata convenience methods). On close, a StoreFile
+ * is sitting in the Filesystem.  To refer to it, create a StoreFile instance
+ * passing filesystem and path.  To read, call {@link #createReader()}.
+ * 
                  StoreFiles may also reference store files in another Store.
+ *
+ * The reason for this weird pattern where you use a different instance for the
+ * writer and a reader is that we write once but read a lot more.
+ */
+public class StoreFile extends SchemaConfigured {
+  static final Log LOG = LogFactory.getLog(StoreFile.class.getName());
+
+  public static enum BloomType {
+    /**
+     * Bloomfilters disabled
+     */
+    NONE,
+    /**
+     * Bloom enabled with Table row as Key
+     */
+    ROW,
+    /**
+     * Bloom enabled with Table row & column (family+qualifier) as Key
+     */
+    ROWCOL
+  }
+
+  // Keys for fileinfo values in HFile
+
+  /** Max Sequence ID in FileInfo */
+  public static final byte [] MAX_SEQ_ID_KEY = Bytes.toBytes("MAX_SEQ_ID_KEY");
+
+  /** Major compaction flag in FileInfo */
+  public static final byte[] MAJOR_COMPACTION_KEY =
+      Bytes.toBytes("MAJOR_COMPACTION_KEY");
+
+  /** Major compaction flag in FileInfo */
+  public static final byte[] EXCLUDE_FROM_MINOR_COMPACTION_KEY =
+      Bytes.toBytes("EXCLUDE_FROM_MINOR_COMPACTION");
+
+  /** Bloom filter Type in FileInfo */
+  public static final byte[] BLOOM_FILTER_TYPE_KEY =
+      Bytes.toBytes("BLOOM_FILTER_TYPE");
+
+  /** Delete Family Count in FileInfo */
+  public static final byte[] DELETE_FAMILY_COUNT =
+      Bytes.toBytes("DELETE_FAMILY_COUNT");
+
+  /** Last Bloom filter key in FileInfo */
+  private static final byte[] LAST_BLOOM_KEY = Bytes.toBytes("LAST_BLOOM_KEY");
+
+  /** Key for Timerange information in metadata*/
+  public static final byte[] TIMERANGE_KEY = Bytes.toBytes("TIMERANGE");
+
+  /** Key for timestamp of earliest-put in metadata*/
+  public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS");
+
+  // Make default block size for StoreFiles 8k while testing.  TODO: FIX!
+  // Need to make it 8k for testing.
+  public static final int DEFAULT_BLOCKSIZE_SMALL = 8 * 1024;
+  
+  private static boolean useIndex;
+
+  private final FileSystem fs;
+
+  // This file's path.
+  private final Path path;
+
+  // If this storefile references another, this is the reference instance.
+  private Reference reference;
+
+  // If this StoreFile references another, this is the other files path.
+  private Path referencePath;
+
+  // If this storefile is a link to another, this is the link instance.
+  private HFileLink link;
+
+  // Block cache configuration and reference.
+  private final CacheConfig cacheConf;
+
+  private Configuration conf;
+
+  // What kind of data block encoding will be used
+  private final HFileDataBlockEncoder dataBlockEncoder;
+
+  // HDFS blocks distribution information
+  private HDFSBlocksDistribution hdfsBlocksDistribution;
+
+  // Keys for metadata stored in backing HFile.
+  // Set when we obtain a Reader.
+  private long sequenceid = -1;
+
+  // max of the MemstoreTS in the KV's in this store
+  // Set when we obtain a Reader.
+  private long maxMemstoreTS = -1;
+
+  public long getMaxMemstoreTS() {
+    return maxMemstoreTS;
+  }
+
+  public void setMaxMemstoreTS(long maxMemstoreTS) {
+    this.maxMemstoreTS = maxMemstoreTS;
+  }
+
+  // If true, this file was product of a major compaction.  Its then set
+  // whenever you get a Reader.
+  private AtomicBoolean majorCompaction = null;
+
+  // If true, this file should not be included in minor compactions.
+  // It's set whenever you get a Reader.
+  private boolean excludeFromMinorCompaction = false;
+
+  /** Meta key set when store file is a result of a bulk load */
+  public static final byte[] BULKLOAD_TASK_KEY =
+    Bytes.toBytes("BULKLOAD_SOURCE_TASK");
+  public static final byte[] BULKLOAD_TIME_KEY =
+    Bytes.toBytes("BULKLOAD_TIMESTAMP");
+
+  /**
+   * Map of the metadata entries in the corresponding HFile
+   */
+  private Map metadataMap;
+
+  /**
+   * A non-capture group, for hfiles, so that this can be embedded.
+   * HFiles are uuid ([0-9a-z]+). Bulk loaded hfiles has (_SeqId_[0-9]+_) has suffix.
+   */
+  public static final String HFILE_NAME_REGEX = "[0-9a-f]+(?:_SeqId_[0-9]+_)?";
+
+  /** Regex that will work for hfiles */
+  private static final Pattern HFILE_NAME_PATTERN =
+    Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
+
+  /**
+   * Regex that will work for straight reference names (.)
+   * and hfilelink reference names (

      +<path> must point to a jar, can be on any filesystem supported by the +Hadoop FileSystem object. +

=-.) + * If reference, then the regex has more than just one group. + * Group 1, hfile/hfilelink pattern, is this file's id. + * Group 2 '(.+)' is the reference's parent region name. + */ + private static final Pattern REF_NAME_PATTERN = + Pattern.compile(String.format("^(%s|%s)\\.(.+)$", + HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX)); + + // StoreFile.Reader + private volatile Reader reader; + + /** + * Bloom filter type specified in column family configuration. Does not + * necessarily correspond to the Bloom filter type present in the HFile. + */ + private final BloomType cfBloomType; + + // the last modification time stamp + private long modificationTimeStamp = 0L; + + /** + * Constructor, loads a reader and it's indices, etc. May allocate a + * substantial amount of ram depending on the underlying files (10-20MB?). + * + * @param fs The current file system to use. + * @param p The path of the file. + * @param blockcache true if the block cache is enabled. + * @param conf The current configuration. + * @param cacheConf The cache configuration and block cache reference. + * @param cfBloomType The bloom type to use for this store file as specified + * by column family configuration. This may or may not be the same + * as the Bloom filter type actually present in the HFile, because + * column family configuration might change. If this is + * {@link BloomType#NONE}, the existing Bloom filter is ignored. + * @param dataBlockEncoder data block encoding algorithm. + * @throws IOException When opening the reader fails. + */ + public StoreFile(final FileSystem fs, + final Path p, + final Configuration conf, + final CacheConfig cacheConf, + final BloomType cfBloomType, + final HFileDataBlockEncoder dataBlockEncoder) + throws IOException { + this.fs = fs; + this.path = p; + this.cacheConf = cacheConf; + this.conf = conf; + useIndex = conf.getBoolean("hbase.use.secondary.index", false); + this.dataBlockEncoder = + dataBlockEncoder == null ? NoOpDataBlockEncoder.INSTANCE + : dataBlockEncoder; + + if (HFileLink.isHFileLink(p)) { + this.link = new HFileLink(conf, p); + LOG.debug("Store file " + p + " is a link"); + } else if (isReference(p)) { + this.reference = Reference.read(fs, p); + this.referencePath = getReferredToFile(this.path); + if (HFileLink.isHFileLink(this.referencePath)) { + this.link = new HFileLink(conf, this.referencePath); + } + LOG.debug("Store file " + p + " is a " + reference.getFileRegion() + + " reference to " + this.referencePath); + } else if (!isHFile(p)) { + throw new IOException("path=" + path + " doesn't look like a valid StoreFile"); + } + + if (BloomFilterFactory.isGeneralBloomEnabled(conf)) { + this.cfBloomType = cfBloomType; + } else { + LOG.info("Ignoring bloom filter check for file " + path + ": " + + "cfBloomType=" + cfBloomType + " (disabled in config)"); + this.cfBloomType = BloomType.NONE; + } + + // cache the modification time stamp of this store file + FileStatus[] stats = FSUtils.listStatus(fs, p, null); + if (stats != null && stats.length == 1) { + this.modificationTimeStamp = stats[0].getModificationTime(); + } else { + this.modificationTimeStamp = 0; + } + + SchemaMetrics.configureGlobally(conf); + } + + /** + * @return Path or null if this StoreFile was made with a Stream. + */ + public Path getPath() { + return this.path; + } + + /** + * @return The Store/ColumnFamily this file belongs to. + */ + byte [] getFamily() { + return Bytes.toBytes(this.path.getParent().getName()); + } + + /** + * @return True if this is a StoreFile Reference; call after {@link #open()} + * else may get wrong answer. + */ + boolean isReference() { + return this.reference != null; + } + + /** + * @return true if this StoreFile is an HFileLink + */ + boolean isLink() { + return this.link != null && this.reference == null; + } + + private static boolean isHFile(final Path path) { + Matcher m = HFILE_NAME_PATTERN.matcher(path.getName()); + return m.matches() && m.groupCount() > 0; + } + + /** + * @param p Path to check. + * @return True if the path has format of a HStoreFile reference. + */ + public static boolean isReference(final Path p) { + return isReference(p.getName()); + } + + /** + * @param name file name to check. + * @return True if the path has format of a HStoreFile reference. + */ + public static boolean isReference(final String name) { + Matcher m = REF_NAME_PATTERN.matcher(name); + return m.matches() && m.groupCount() > 1; + } + + /* + * Return path to the file referred to by a Reference. Presumes a directory + * hierarchy of ${hbase.rootdir}/tablename/regionname/familyname. + * @param p Path to a Reference file. + * @return Calculated path to parent region file. + * @throws IllegalArgumentException when path regex fails to match. + */ + public static Path getReferredToFile(final Path p) { + Matcher m = REF_NAME_PATTERN.matcher(p.getName()); + if (m == null || !m.matches()) { + LOG.warn("Failed match of store file name " + p.toString()); + throw new IllegalArgumentException("Failed match of store file name " + + p.toString()); + } + // Other region name is suffix on the passed Reference file name + String otherRegion = m.group(2); + // Tabledir is up two directories from where Reference was written. + Path tableDir = p.getParent().getParent().getParent(); + String nameStrippedOfSuffix = m.group(1); + LOG.debug("reference '" + p + "' to region=" + otherRegion + " hfile=" + nameStrippedOfSuffix); + + // Build up new path with the referenced region in place of our current + // region in the reference path. Also strip regionname suffix from name. + return new Path(new Path(new Path(tableDir, otherRegion), + p.getParent().getName()), nameStrippedOfSuffix); + } + + /** + * @return True if this file was made by a major compaction. + */ + boolean isMajorCompaction() { + if (this.majorCompaction == null) { + throw new NullPointerException("This has not been set yet"); + } + return this.majorCompaction.get(); + } + + /** + * @return True if this file should not be part of a minor compaction. + */ + boolean excludeFromMinorCompaction() { + return this.excludeFromMinorCompaction; + } + + /** + * @return This files maximum edit sequence id. + */ + public long getMaxSequenceId() { + return this.sequenceid; + } + + public long getModificationTimeStamp() { + return modificationTimeStamp; + } + + /** + * Return the largest memstoreTS found across all storefiles in + * the given list. Store files that were created by a mapreduce + * bulk load are ignored, as they do not correspond to any specific + * put operation, and thus do not have a memstoreTS associated with them. + * @return 0 if no non-bulk-load files are provided or, this is Store that + * does not yet have any store files. + */ + public static long getMaxMemstoreTSInList(Collection sfs) { + long max = 0; + for (StoreFile sf : sfs) { + if (!sf.isBulkLoadResult()) { + max = Math.max(max, sf.getMaxMemstoreTS()); + } + } + return max; + } + + /** + * Return the highest sequence ID found across all storefiles in + * the given list. Store files that were created by a mapreduce + * bulk load are ignored, as they do not correspond to any edit + * log items. + * @return 0 if no non-bulk-load files are provided or, this is Store that + * does not yet have any store files. + */ + public static long getMaxSequenceIdInList(Collection sfs) { + long max = 0; + for (StoreFile sf : sfs) { + if (!sf.isBulkLoadResult()) { + max = Math.max(max, sf.getMaxSequenceId()); + } + } + return max; + } + + /** + * @return true if this storefile was created by HFileOutputFormat + * for a bulk load. + */ + boolean isBulkLoadResult() { + return metadataMap.containsKey(BULKLOAD_TIME_KEY); + } + + /** + * Return the timestamp at which this bulk load file was generated. + */ + public long getBulkLoadTimestamp() { + return Bytes.toLong(metadataMap.get(BULKLOAD_TIME_KEY)); + } + + /** + * @return the cached value of HDFS blocks distribution. The cached value is + * calculated when store file is opened. + */ + public HDFSBlocksDistribution getHDFSBlockDistribution() { + return this.hdfsBlocksDistribution; + } + + /** + * helper function to compute HDFS blocks distribution of a given reference + * file.For reference file, we don't compute the exact value. We use some + * estimate instead given it might be good enough. we assume bottom part + * takes the first half of reference file, top part takes the second half + * of the reference file. This is just estimate, given + * midkey ofregion != midkey of HFile, also the number and size of keys vary. + * If this estimate isn't good enough, we can improve it later. + * @param fs The FileSystem + * @param reference The reference + * @param status The reference FileStatus + * @return HDFS blocks distribution + */ + static private HDFSBlocksDistribution computeRefFileHDFSBlockDistribution( + FileSystem fs, Reference reference, FileStatus status) throws IOException { + if (status == null) { + return null; + } + + long start = 0; + long length = 0; + + if (Reference.isTopFileRegion(reference.getFileRegion())) { + start = status.getLen()/2; + length = status.getLen() - status.getLen()/2; + } else { + start = 0; + length = status.getLen()/2; + } + return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length); + } + + /** + * compute HDFS block distribution, for reference file, it is an estimate + */ + private void computeHDFSBlockDistribution() throws IOException { + if (isReference()) { + FileStatus status; + if (this.link != null) { + status = this.link.getFileStatus(fs); + } else { + status = fs.getFileStatus(this.referencePath); + } + this.hdfsBlocksDistribution = computeRefFileHDFSBlockDistribution( + this.fs, this.reference, status); + } else { + FileStatus status; + if (isLink()) { + status = link.getFileStatus(fs); + } else { + status = this.fs.getFileStatus(path); + } + long length = status.getLen(); + this.hdfsBlocksDistribution = FSUtils.computeHDFSBlocksDistribution( + this.fs, status, 0, length); + } + } + + /** + * Opens reader on this store file. Called by Constructor. + * @return Reader for the store file. + * @throws IOException + * @see #closeReader() + */ + private Reader open() throws IOException { + if (this.reader != null) { + throw new IllegalAccessError("Already open"); + } + if (isReference()) { + if (this.link != null) { + this.reader = new HalfStoreFileReader(this.fs, this.referencePath, this.link, + this.cacheConf, this.reference, dataBlockEncoder.getEncodingInCache()); + } else { + if (conf.getBoolean("hbase.use.secondary.index", false) && isIndexRegionReference()) { + String indexHalfStoreFileClass = conf.get("hbase.index.half.storefile.reader.class"); + if (indexHalfStoreFileClass == null) { + throw new RuntimeException( + "Class for index half store files should be present. Configure the property hbase.index.half.storefile.reader.class"); + } + try { + Class indexHalfStoreReader = Class.forName(indexHalfStoreFileClass.trim()); + Constructor constructor = + indexHalfStoreReader.getConstructor(FileSystem.class, Path.class, + CacheConfig.class, Reference.class, DataBlockEncoding.class); + this.reader = + (Reader) constructor.newInstance(this.fs, this.referencePath, this.cacheConf, + this.reference, dataBlockEncoder.getEncodingInCache()); + } catch (Throwable e) { + LOG.error("Error while initializing/invoking constructor of IndexHalfStoreFileReader.", + e); + throw new RuntimeException( + "Error while initializing/invoking constructor of IndexHalfStoreFileReader."); + } + } else { + this.reader = + new HalfStoreFileReader(this.fs, this.referencePath, this.cacheConf, this.reference, + dataBlockEncoder.getEncodingInCache()); + } + } + } else if (isLink()) { + long size = link.getFileStatus(fs).getLen(); + this.reader = new Reader(this.fs, this.path, link, size, this.cacheConf, + dataBlockEncoder.getEncodingInCache(), true); + } else { + this.reader = new Reader(this.fs, this.path, this.cacheConf, + dataBlockEncoder.getEncodingInCache()); + } + + if (isSchemaConfigured()) { + SchemaConfigured.resetSchemaMetricsConf(reader); + passSchemaMetricsTo(reader); + } + + computeHDFSBlockDistribution(); + + // Load up indices and fileinfo. This also loads Bloom filter type. + metadataMap = Collections.unmodifiableMap(this.reader.loadFileInfo()); + + // Read in our metadata. + byte [] b = metadataMap.get(MAX_SEQ_ID_KEY); + if (b != null) { + // By convention, if halfhfile, top half has a sequence number > bottom + // half. Thats why we add one in below. Its done for case the two halves + // are ever merged back together --rare. Without it, on open of store, + // since store files are distinguished by sequence id, the one half would + // subsume the other. + this.sequenceid = Bytes.toLong(b); + if (isReference()) { + if (Reference.isTopFileRegion(this.reference.getFileRegion())) { + this.sequenceid += 1; + } + } + } + this.reader.setSequenceID(this.sequenceid); + + b = metadataMap.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY); + if (b != null) { + this.maxMemstoreTS = Bytes.toLong(b); + } + + b = metadataMap.get(MAJOR_COMPACTION_KEY); + if (b != null) { + boolean mc = Bytes.toBoolean(b); + if (this.majorCompaction == null) { + this.majorCompaction = new AtomicBoolean(mc); + } else { + this.majorCompaction.set(mc); + } + } else { + // Presume it is not major compacted if it doesn't explicity say so + // HFileOutputFormat explicitly sets the major compacted key. + this.majorCompaction = new AtomicBoolean(false); + } + + b = metadataMap.get(EXCLUDE_FROM_MINOR_COMPACTION_KEY); + this.excludeFromMinorCompaction = (b != null && Bytes.toBoolean(b)); + + BloomType hfileBloomType = reader.getBloomFilterType(); + if (cfBloomType != BloomType.NONE) { + reader.loadBloomfilter(BlockType.GENERAL_BLOOM_META); + if (hfileBloomType != cfBloomType) { + LOG.info("HFile Bloom filter type for " + + reader.getHFileReader().getName() + ": " + hfileBloomType + + ", but " + cfBloomType + " specified in column family " + + "configuration"); + } + } else if (hfileBloomType != BloomType.NONE) { + LOG.info("Bloom filter turned off by CF config for " + + reader.getHFileReader().getName()); + } + + // load delete family bloom filter + reader.loadBloomfilter(BlockType.DELETE_FAMILY_BLOOM_META); + + try { + byte [] timerangeBytes = metadataMap.get(TIMERANGE_KEY); + if (timerangeBytes != null) { + this.reader.timeRangeTracker = new TimeRangeTracker(); + Writables.copyWritable(timerangeBytes, this.reader.timeRangeTracker); + } + } catch (IllegalArgumentException e) { + LOG.error("Error reading timestamp range data from meta -- " + + "proceeding without", e); + this.reader.timeRangeTracker = null; + } + return this.reader; + } + + private boolean isIndexRegionReference() { + String tablePath = this.referencePath.getParent().getParent().getParent().getName(); + return tablePath.endsWith("_idx"); + } + + /** + * @return Reader for StoreFile. creates if necessary + * @throws IOException + */ + public Reader createReader() throws IOException { + if (this.reader == null) { + try { + this.reader = open(); + } catch (IOException e) { + try { + this.closeReader(true); + } catch (IOException ee) { + } + throw e; + } + + } + return this.reader; + } + + /** + * @return Current reader. Must call createReader first else returns null. + * @see #createReader() + */ + public Reader getReader() { + return this.reader; + } + + /** + * @param evictOnClose whether to evict blocks belonging to this file + * @throws IOException + */ + public synchronized void closeReader(boolean evictOnClose) + throws IOException { + if (this.reader != null) { + this.reader.close(evictOnClose); + this.reader = null; + } + } + + /** + * Delete this file + * @throws IOException + */ + public void deleteReader() throws IOException { + closeReader(true); + HBaseFileSystem.deleteDirFromFileSystem(fs, getPath()); + } + + @Override + public String toString() { + return this.path.toString() + + (isReference()? "-" + this.referencePath + "-" + reference.toString(): ""); + } + + /** + * @return a length description of this StoreFile, suitable for debug output + */ + public String toStringDetailed() { + StringBuilder sb = new StringBuilder(); + sb.append(this.path.toString()); + sb.append(", isReference=").append(isReference()); + sb.append(", isBulkLoadResult=").append(isBulkLoadResult()); + if (isBulkLoadResult()) { + sb.append(", bulkLoadTS=").append(getBulkLoadTimestamp()); + } else { + sb.append(", seqid=").append(getMaxSequenceId()); + } + sb.append(", majorCompaction=").append(isMajorCompaction()); + + return sb.toString(); + } + + /** + * Utility to help with rename. + * @param fs + * @param src + * @param tgt + * @return True if succeeded. + * @throws IOException + */ + public static Path rename(final FileSystem fs, + final Path src, + final Path tgt) + throws IOException { + + if (!fs.exists(src)) { + throw new FileNotFoundException(src.toString()); + } + if (!HBaseFileSystem.renameDirForFileSystem(fs, src, tgt)) { + throw new IOException("Failed rename of " + src + " to " + tgt); + } + return tgt; + } + + public static class WriterBuilder { + private final Configuration conf; + private final CacheConfig cacheConf; + private final FileSystem fs; + private final int blockSize; + + private Compression.Algorithm compressAlgo = + HFile.DEFAULT_COMPRESSION_ALGORITHM; + private HFileDataBlockEncoder dataBlockEncoder = + NoOpDataBlockEncoder.INSTANCE; + private KeyValue.KVComparator comparator = KeyValue.COMPARATOR; + private BloomType bloomType = BloomType.NONE; + private long maxKeyCount = 0; + private Path dir; + private Path filePath; + private ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE; + private int bytesPerChecksum = HFile.DEFAULT_BYTES_PER_CHECKSUM; + private boolean includeMVCCReadpoint = true; + + public WriterBuilder(Configuration conf, CacheConfig cacheConf, + FileSystem fs, int blockSize) { + this.conf = conf; + this.cacheConf = cacheConf; + this.fs = fs; + this.blockSize = blockSize; + } + + /** + * Use either this method or {@link #withFilePath}, but not both. + * @param dir Path to column family directory. The directory is created if + * does not exist. The file is given a unique name within this + * directory. + * @return this (for chained invocation) + */ + public WriterBuilder withOutputDir(Path dir) { + Preconditions.checkNotNull(dir); + this.dir = dir; + return this; + } + + /** + * Use either this method or {@link #withOutputDir}, but not both. + * @param filePath the StoreFile path to write + * @return this (for chained invocation) + */ + public WriterBuilder withFilePath(Path filePath) { + Preconditions.checkNotNull(filePath); + this.filePath = filePath; + return this; + } + + public WriterBuilder withCompression(Compression.Algorithm compressAlgo) { + Preconditions.checkNotNull(compressAlgo); + this.compressAlgo = compressAlgo; + return this; + } + + public WriterBuilder withDataBlockEncoder(HFileDataBlockEncoder encoder) { + Preconditions.checkNotNull(encoder); + this.dataBlockEncoder = encoder; + return this; + } + + public WriterBuilder withComparator(KeyValue.KVComparator comparator) { + Preconditions.checkNotNull(comparator); + this.comparator = comparator; + return this; + } + + public WriterBuilder withBloomType(BloomType bloomType) { + Preconditions.checkNotNull(bloomType); + this.bloomType = bloomType; + return this; + } + + /** + * @param maxKeyCount estimated maximum number of keys we expect to add + * @return this (for chained invocation) + */ + public WriterBuilder withMaxKeyCount(long maxKeyCount) { + this.maxKeyCount = maxKeyCount; + return this; + } + + /** + * @param checksumType the type of checksum + * @return this (for chained invocation) + */ + public WriterBuilder withChecksumType(ChecksumType checksumType) { + this.checksumType = checksumType; + return this; + } + + /** + * @param bytesPerChecksum the number of bytes per checksum chunk + * @return this (for chained invocation) + */ + public WriterBuilder withBytesPerChecksum(int bytesPerChecksum) { + this.bytesPerChecksum = bytesPerChecksum; + return this; + } + + /** + * @param includeMVCCReadpoint whether to write the mvcc readpoint to the file for each KV + * @return this (for chained invocation) + */ + public WriterBuilder includeMVCCReadpoint(boolean includeMVCCReadpoint) { + this.includeMVCCReadpoint = includeMVCCReadpoint; + return this; + } + + /** + * Create a store file writer. Client is responsible for closing file when + * done. If metadata, add BEFORE closing using + * {@link Writer#appendMetadata}. + */ + public Writer build() throws IOException { + if ((dir == null ? 0 : 1) + (filePath == null ? 0 : 1) != 1) { + throw new IllegalArgumentException("Either specify parent directory " + + "or file path"); + } + + if (dir == null) { + dir = filePath.getParent(); + } + + if (!fs.exists(dir)) { + HBaseFileSystem.makeDirOnFileSystem(fs, dir); + } + + if (filePath == null) { + filePath = getUniqueFile(fs, dir); + if (!BloomFilterFactory.isGeneralBloomEnabled(conf)) { + bloomType = BloomType.NONE; + } + } + + if (compressAlgo == null) { + compressAlgo = HFile.DEFAULT_COMPRESSION_ALGORITHM; + } + if (comparator == null) { + comparator = KeyValue.COMPARATOR; + } + return new Writer(fs, filePath, blockSize, compressAlgo, dataBlockEncoder, + conf, cacheConf, comparator, bloomType, maxKeyCount, checksumType, + bytesPerChecksum, includeMVCCReadpoint); + } + } + + /** + * @param fs + * @param dir Directory to create file in. + * @return random filename inside passed dir + */ + public static Path getUniqueFile(final FileSystem fs, final Path dir) + throws IOException { + if (!fs.getFileStatus(dir).isDir()) { + throw new IOException("Expecting " + dir.toString() + + " to be a directory"); + } + return getRandomFilename(fs, dir); + } + + /** + * + * @param fs + * @param dir + * @return Path to a file that doesn't exist at time of this invocation. + * @throws IOException + */ + static Path getRandomFilename(final FileSystem fs, final Path dir) + throws IOException { + return getRandomFilename(fs, dir, null); + } + + /** + * + * @param fs + * @param dir + * @param suffix + * @return Path to a file that doesn't exist at time of this invocation. + * @throws IOException + */ + static Path getRandomFilename(final FileSystem fs, + final Path dir, + final String suffix) + throws IOException { + return new Path(dir, UUID.randomUUID().toString().replaceAll("-", "") + + (suffix == null ? "" : suffix)); + } + + /** + * Validate the store file name. + * @param fileName name of the file to validate + * @return true if the file could be a valid store file, false otherwise + */ + public static boolean validateStoreFileName(String fileName) { + if (HFileLink.isHFileLink(fileName)) + return true; + if (isReference(fileName)) + return true; + return !fileName.contains("-"); + } + + /** + * Write out a split reference. Package local so it doesnt leak out of + * regionserver. + * @param fs + * @param splitDir Presumes path format is actually + * SOME_DIRECTORY/REGIONNAME/FAMILY. + * @param f File to split. + * @param splitRow + * @param range + * @return Path to created reference. + * @throws IOException + */ + static Path split(final FileSystem fs, + final Path splitDir, + final StoreFile f, + final byte [] splitRow, + final Reference.Range range) + throws IOException { + + // Check whether the split row lies in the range of the store file + // If it is outside the range, return directly. + if (!useIndex || !isIndexTable(f.getPath())) { + if (range == Reference.Range.bottom) { + //check if smaller than first key + KeyValue splitKey = KeyValue.createLastOnRow(splitRow); + byte[] firstKey = f.createReader().getFirstKey(); + if (firstKey == null) return null; + if (f.getReader().getComparator().compare(splitKey.getBuffer(), + splitKey.getKeyOffset(), splitKey.getKeyLength(), + firstKey, 0, firstKey.length) < 0) { + return null; + } + } else { + //check if larger than last key. + KeyValue splitKey = KeyValue.createFirstOnRow(splitRow); + byte[] lastKey = f.createReader().getLastKey(); + if (lastKey == null) return null; + if (f.getReader().getComparator().compare(splitKey.getBuffer(), + splitKey.getKeyOffset(), splitKey.getKeyLength(), + lastKey, 0, lastKey.length) > 0) { + return null; + } + } + } + + // A reference to the bottom half of the hsf store file. + Reference r = new Reference(splitRow, range); + // Add the referred-to regions name as a dot separated suffix. + // See REF_NAME_REGEX regex above. The referred-to regions name is + // up in the path of the passed in f -- parentdir is family, + // then the directory above is the region name. + String parentRegionName = f.getPath().getParent().getParent().getName(); + // Write reference with same file id only with the other region name as + // suffix and into the new region location (under same family). + Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName); + return r.write(fs, p); + } + + + private static boolean isIndexTable(Path path) { + String tablePath = path.getParent().getParent().getParent().getName(); + return tablePath.endsWith("_idx"); + } + + /** + * A StoreFile writer. Use this to read/write HBase Store Files. It is package + * local because it is an implementation detail of the HBase regionserver. + */ + public static class Writer { + private final BloomFilterWriter generalBloomFilterWriter; + private final BloomFilterWriter deleteFamilyBloomFilterWriter; + private final BloomType bloomType; + private byte[] lastBloomKey; + private int lastBloomKeyOffset, lastBloomKeyLen; + private KVComparator kvComparator; + private KeyValue lastKv = null; + private long earliestPutTs = HConstants.LATEST_TIMESTAMP; + private KeyValue lastDeleteFamilyKV = null; + private long deleteFamilyCnt = 0; + + protected HFileDataBlockEncoder dataBlockEncoder; + + /** Checksum type */ + protected ChecksumType checksumType; + + /** Bytes per Checksum */ + protected int bytesPerChecksum; + + TimeRangeTracker timeRangeTracker = new TimeRangeTracker(); + /* isTimeRangeTrackerSet keeps track if the timeRange has already been set + * When flushing a memstore, we set TimeRange and use this variable to + * indicate that it doesn't need to be calculated again while + * appending KeyValues. + * It is not set in cases of compactions when it is recalculated using only + * the appended KeyValues*/ + boolean isTimeRangeTrackerSet = false; + + protected HFile.Writer writer; + + /** + * Creates an HFile.Writer that also write helpful meta data. + * @param fs file system to write to + * @param path file name to create + * @param blocksize HDFS block size + * @param compress HDFS block compression + * @param conf user configuration + * @param comparator key comparator + * @param bloomType bloom filter setting + * @param maxKeys the expected maximum number of keys to be added. Was used + * for Bloom filter size in {@link HFile} format version 1. + * @param checksumType the checksum type + * @param bytesPerChecksum the number of bytes per checksum value + * @param includeMVCCReadpoint whether to write the mvcc readpoint to the file for each KV + * @throws IOException problem writing to FS + */ + private Writer(FileSystem fs, Path path, int blocksize, + Compression.Algorithm compress, + HFileDataBlockEncoder dataBlockEncoder, final Configuration conf, + CacheConfig cacheConf, + final KVComparator comparator, BloomType bloomType, long maxKeys, + final ChecksumType checksumType, final int bytesPerChecksum, boolean includeMVCCReadpoint) + throws IOException { + this.dataBlockEncoder = dataBlockEncoder != null ? + dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE; + writer = HFile.getWriterFactory(conf, cacheConf) + .withPath(fs, path) + .withBlockSize(blocksize) + .withCompression(compress) + .withDataBlockEncoder(dataBlockEncoder) + .withComparator(comparator.getRawComparator()) + .withChecksumType(checksumType) + .withBytesPerChecksum(bytesPerChecksum) + .includeMVCCReadpoint(includeMVCCReadpoint) + .create(); + + this.kvComparator = comparator; + + generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite( + conf, cacheConf, bloomType, + (int) Math.min(maxKeys, Integer.MAX_VALUE), writer); + + if (generalBloomFilterWriter != null) { + this.bloomType = bloomType; + LOG.info("Bloom filter type for " + path + ": " + this.bloomType + ", " + + generalBloomFilterWriter.getClass().getSimpleName()); + } else { + // Not using Bloom filters. + this.bloomType = BloomType.NONE; + } + + // initialize delete family Bloom filter when there is NO RowCol Bloom + // filter + if (this.bloomType != BloomType.ROWCOL) { + this.deleteFamilyBloomFilterWriter = BloomFilterFactory + .createDeleteBloomAtWrite(conf, cacheConf, + (int) Math.min(maxKeys, Integer.MAX_VALUE), writer); + } else { + deleteFamilyBloomFilterWriter = null; + } + if (deleteFamilyBloomFilterWriter != null) { + LOG.info("Delete Family Bloom filter type for " + path + ": " + + deleteFamilyBloomFilterWriter.getClass().getSimpleName()); + } + this.checksumType = checksumType; + this.bytesPerChecksum = bytesPerChecksum; + } + + /** + * Writes meta data. + * Call before {@link #close()} since its written as meta data to this file. + * @param maxSequenceId Maximum sequence id. + * @param majorCompaction True if this file is product of a major compaction + * @throws IOException problem writing to FS + */ + public void appendMetadata(final long maxSequenceId, final boolean majorCompaction) + throws IOException { + writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(maxSequenceId)); + writer.appendFileInfo(MAJOR_COMPACTION_KEY, + Bytes.toBytes(majorCompaction)); + appendTrackedTimestampsToMetadata(); + } + + /** + * Add TimestampRange and earliest put timestamp to Metadata + */ + public void appendTrackedTimestampsToMetadata() throws IOException { + appendFileInfo(TIMERANGE_KEY,WritableUtils.toByteArray(timeRangeTracker)); + appendFileInfo(EARLIEST_PUT_TS, Bytes.toBytes(earliestPutTs)); + } + + /** + * Set TimeRangeTracker + * @param trt + */ + public void setTimeRangeTracker(final TimeRangeTracker trt) { + this.timeRangeTracker = trt; + isTimeRangeTrackerSet = true; + } + + /** + * Record the earlest Put timestamp. + * + * If the timeRangeTracker is not set, + * update TimeRangeTracker to include the timestamp of this key + * @param kv + */ + public void trackTimestamps(final KeyValue kv) { + if (KeyValue.Type.Put.getCode() == kv.getType()) { + earliestPutTs = Math.min(earliestPutTs, kv.getTimestamp()); + } + if (!isTimeRangeTrackerSet) { + timeRangeTracker.includeTimestamp(kv); + } + } + + private void appendGeneralBloomfilter(final KeyValue kv) throws IOException { + if (this.generalBloomFilterWriter != null) { + // only add to the bloom filter on a new, unique key + boolean newKey = true; + if (this.lastKv != null) { + switch(bloomType) { + case ROW: + newKey = ! kvComparator.matchingRows(kv, lastKv); + break; + case ROWCOL: + newKey = ! kvComparator.matchingRowColumn(kv, lastKv); + break; + case NONE: + newKey = false; + break; + default: + throw new IOException("Invalid Bloom filter type: " + bloomType + + " (ROW or ROWCOL expected)"); + } + } + if (newKey) { + /* + * http://2.bp.blogspot.com/_Cib_A77V54U/StZMrzaKufI/AAAAAAAAADo/ZhK7bGoJdMQ/s400/KeyValue.png + * Key = RowLen + Row + FamilyLen + Column [Family + Qualifier] + TimeStamp + * + * 2 Types of Filtering: + * 1. Row = Row + * 2. RowCol = Row + Qualifier + */ + byte[] bloomKey; + int bloomKeyOffset, bloomKeyLen; + + switch (bloomType) { + case ROW: + bloomKey = kv.getBuffer(); + bloomKeyOffset = kv.getRowOffset(); + bloomKeyLen = kv.getRowLength(); + break; + case ROWCOL: + // merge(row, qualifier) + // TODO: could save one buffer copy in case of compound Bloom + // filters when this involves creating a KeyValue + bloomKey = generalBloomFilterWriter.createBloomKey(kv.getBuffer(), + kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(), + kv.getQualifierOffset(), kv.getQualifierLength()); + bloomKeyOffset = 0; + bloomKeyLen = bloomKey.length; + break; + default: + throw new IOException("Invalid Bloom filter type: " + bloomType + + " (ROW or ROWCOL expected)"); + } + generalBloomFilterWriter.add(bloomKey, bloomKeyOffset, bloomKeyLen); + if (lastBloomKey != null + && generalBloomFilterWriter.getComparator().compare(bloomKey, + bloomKeyOffset, bloomKeyLen, lastBloomKey, + lastBloomKeyOffset, lastBloomKeyLen) <= 0) { + throw new IOException("Non-increasing Bloom keys: " + + Bytes.toStringBinary(bloomKey, bloomKeyOffset, bloomKeyLen) + + " after " + + Bytes.toStringBinary(lastBloomKey, lastBloomKeyOffset, + lastBloomKeyLen)); + } + lastBloomKey = bloomKey; + lastBloomKeyOffset = bloomKeyOffset; + lastBloomKeyLen = bloomKeyLen; + this.lastKv = kv; + } + } + } + + private void appendDeleteFamilyBloomFilter(final KeyValue kv) + throws IOException { + if (!kv.isDeleteFamily()) { + return; + } + + // increase the number of delete family in the store file + deleteFamilyCnt++; + if (null != this.deleteFamilyBloomFilterWriter) { + boolean newKey = true; + if (lastDeleteFamilyKV != null) { + newKey = !kvComparator.matchingRows(kv, lastDeleteFamilyKV); + } + if (newKey) { + this.deleteFamilyBloomFilterWriter.add(kv.getBuffer(), + kv.getRowOffset(), kv.getRowLength()); + this.lastDeleteFamilyKV = kv; + } + } + } + + public void append(final KeyValue kv) throws IOException { + appendGeneralBloomfilter(kv); + appendDeleteFamilyBloomFilter(kv); + writer.append(kv); + trackTimestamps(kv); + } + + public Path getPath() { + return this.writer.getPath(); + } + + boolean hasGeneralBloom() { + return this.generalBloomFilterWriter != null; + } + + /** + * For unit testing only. + * + * @return the Bloom filter used by this writer. + */ + BloomFilterWriter getGeneralBloomWriter() { + return generalBloomFilterWriter; + } + + private boolean closeBloomFilter(BloomFilterWriter bfw) throws IOException { + boolean haveBloom = (bfw != null && bfw.getKeyCount() > 0); + if (haveBloom) { + bfw.compactBloom(); + } + return haveBloom; + } + + private boolean closeGeneralBloomFilter() throws IOException { + boolean hasGeneralBloom = closeBloomFilter(generalBloomFilterWriter); + + // add the general Bloom filter writer and append file info + if (hasGeneralBloom) { + writer.addGeneralBloomFilter(generalBloomFilterWriter); + writer.appendFileInfo(BLOOM_FILTER_TYPE_KEY, + Bytes.toBytes(bloomType.toString())); + if (lastBloomKey != null) { + writer.appendFileInfo(LAST_BLOOM_KEY, Arrays.copyOfRange( + lastBloomKey, lastBloomKeyOffset, lastBloomKeyOffset + + lastBloomKeyLen)); + } + } + return hasGeneralBloom; + } + + private boolean closeDeleteFamilyBloomFilter() throws IOException { + boolean hasDeleteFamilyBloom = closeBloomFilter(deleteFamilyBloomFilterWriter); + + // add the delete family Bloom filter writer + if (hasDeleteFamilyBloom) { + writer.addDeleteFamilyBloomFilter(deleteFamilyBloomFilterWriter); + } + + // append file info about the number of delete family kvs + // even if there is no delete family Bloom. + writer.appendFileInfo(DELETE_FAMILY_COUNT, + Bytes.toBytes(this.deleteFamilyCnt)); + + return hasDeleteFamilyBloom; + } + + public void close() throws IOException { + boolean hasGeneralBloom = this.closeGeneralBloomFilter(); + boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter(); + + writer.close(); + + // Log final Bloom filter statistics. This needs to be done after close() + // because compound Bloom filters might be finalized as part of closing. + StoreFile.LOG.info((hasGeneralBloom ? "" : "NO ") + "General Bloom and " + + (hasDeleteFamilyBloom ? "" : "NO ") + "DeleteFamily" + + " was added to HFile (" + getPath() + ") "); + + } + + public void appendFileInfo(byte[] key, byte[] value) throws IOException { + writer.appendFileInfo(key, value); + } + + /** For use in testing, e.g. {@link CreateRandomStoreFile} */ + HFile.Writer getHFileWriter() { + return writer; + } + } + + /** + * Reader for a StoreFile. + */ + public static class Reader extends SchemaConfigured { + static final Log LOG = LogFactory.getLog(Reader.class.getName()); + + protected BloomFilter generalBloomFilter = null; + protected BloomFilter deleteFamilyBloomFilter = null; + protected BloomType bloomFilterType; + private final HFile.Reader reader; + protected TimeRangeTracker timeRangeTracker = null; + protected long sequenceID = -1; + private byte[] lastBloomKey; + private long deleteFamilyCnt = -1; + + public Reader(FileSystem fs, Path path, CacheConfig cacheConf, + DataBlockEncoding preferredEncodingInCache) throws IOException { + super(path); + reader = HFile.createReaderWithEncoding(fs, path, cacheConf, + preferredEncodingInCache); + bloomFilterType = BloomType.NONE; + } + + public Reader(FileSystem fs, Path path, HFileLink hfileLink, long size, + CacheConfig cacheConf, DataBlockEncoding preferredEncodingInCache, + boolean closeIStream) throws IOException { + super(path); + + FSDataInputStream in = hfileLink.open(fs); + FSDataInputStream inNoChecksum = in; + if (fs instanceof HFileSystem) { + FileSystem noChecksumFs = ((HFileSystem)fs).getNoChecksumFs(); + inNoChecksum = hfileLink.open(noChecksumFs); + } + + reader = HFile.createReaderWithEncoding(fs, path, in, inNoChecksum, + size, cacheConf, preferredEncodingInCache, closeIStream); + bloomFilterType = BloomType.NONE; + } + + /** + * ONLY USE DEFAULT CONSTRUCTOR FOR UNIT TESTS + */ + Reader() { + this.reader = null; + } + + public RawComparator getComparator() { + return reader.getComparator(); + } + + /** + * Get a scanner to scan over this StoreFile. Do not use + * this overload if using this scanner for compactions. + * + * @param cacheBlocks should this scanner cache blocks? + * @param pread use pread (for highly concurrent small readers) + * @return a scanner + */ + public StoreFileScanner getStoreFileScanner(boolean cacheBlocks, + boolean pread) { + return getStoreFileScanner(cacheBlocks, pread, false); + } + + /** + * Get a scanner to scan over this StoreFile. + * + * @param cacheBlocks should this scanner cache blocks? + * @param pread use pread (for highly concurrent small readers) + * @param isCompaction is scanner being used for compaction? + * @return a scanner + */ + public StoreFileScanner getStoreFileScanner(boolean cacheBlocks, + boolean pread, + boolean isCompaction) { + return new StoreFileScanner(this, + getScanner(cacheBlocks, pread, + isCompaction), !isCompaction); + } + + /** + * Warning: Do not write further code which depends on this call. Instead + * use getStoreFileScanner() which uses the StoreFileScanner class/interface + * which is the preferred way to scan a store with higher level concepts. + * + * @param cacheBlocks should we cache the blocks? + * @param pread use pread (for concurrent small readers) + * @return the underlying HFileScanner + */ + @Deprecated + public HFileScanner getScanner(boolean cacheBlocks, boolean pread) { + return getScanner(cacheBlocks, pread, false); + } + + /** + * Warning: Do not write further code which depends on this call. Instead + * use getStoreFileScanner() which uses the StoreFileScanner class/interface + * which is the preferred way to scan a store with higher level concepts. + * + * @param cacheBlocks + * should we cache the blocks? + * @param pread + * use pread (for concurrent small readers) + * @param isCompaction + * is scanner being used for compaction? + * @return the underlying HFileScanner + */ + @Deprecated + public HFileScanner getScanner(boolean cacheBlocks, boolean pread, + boolean isCompaction) { + return reader.getScanner(cacheBlocks, pread, isCompaction); + } + + public void close(boolean evictOnClose) throws IOException { + reader.close(evictOnClose); + } + + /** + * Check if this storeFile may contain keys within the TimeRange that + * have not expired (i.e. not older than oldestUnexpiredTS). + * @param scan the current scan + * @param oldestUnexpiredTS the oldest timestamp that is not expired, as + * determined by the column family's TTL + * @return false if queried keys definitely don't exist in this StoreFile + */ + boolean passesTimerangeFilter(Scan scan, long oldestUnexpiredTS) { + if (timeRangeTracker == null) { + return true; + } else { + return timeRangeTracker.includesTimeRange(scan.getTimeRange()) && + timeRangeTracker.getMaximumTimestamp() >= oldestUnexpiredTS; + } + } + + /** + * Checks whether the given scan passes the Bloom filter (if present). Only + * checks Bloom filters for single-row or single-row-column scans. Bloom + * filter checking for multi-gets is implemented as part of the store + * scanner system (see {@link StoreFileScanner#seekExactly}) and uses + * the lower-level API {@link #passesGeneralBloomFilter(byte[], int, int, byte[], + * int, int)}. + * + * @param scan the scan specification. Used to determine the row, and to + * check whether this is a single-row ("get") scan. + * @param columns the set of columns. Only used for row-column Bloom + * filters. + * @return true if the scan with the given column set passes the Bloom + * filter, or if the Bloom filter is not applicable for the scan. + * False if the Bloom filter is applicable and the scan fails it. + */ + boolean passesBloomFilter(Scan scan, + final SortedSet columns) { + // Multi-column non-get scans will use Bloom filters through the + // lower-level API function that this function calls. + if (!scan.isGetScan()) { + return true; + } + + byte[] row = scan.getStartRow(); + switch (this.bloomFilterType) { + case ROW: + return passesGeneralBloomFilter(row, 0, row.length, null, 0, 0); + + case ROWCOL: + if (columns != null && columns.size() == 1) { + byte[] column = columns.first(); + return passesGeneralBloomFilter(row, 0, row.length, column, 0, + column.length); + } + + // For multi-column queries the Bloom filter is checked from the + // seekExact operation. + return true; + + default: + return true; + } + } + + public boolean passesDeleteFamilyBloomFilter(byte[] row, int rowOffset, + int rowLen) { + // Cache Bloom filter as a local variable in case it is set to null by + // another thread on an IO error. + BloomFilter bloomFilter = this.deleteFamilyBloomFilter; + + // Empty file or there is no delete family at all + if (reader.getTrailer().getEntryCount() == 0 || deleteFamilyCnt == 0) { + return false; + } + + if (bloomFilter == null) { + return true; + } + + try { + if (!bloomFilter.supportsAutoLoading()) { + return true; + } + return bloomFilter.contains(row, rowOffset, rowLen, null); + } catch (IllegalArgumentException e) { + LOG.error("Bad Delete Family bloom filter data -- proceeding without", + e); + setDeleteFamilyBloomFilterFaulty(); + } + + return true; + } + + /** + * A method for checking Bloom filters. Called directly from + * StoreFileScanner in case of a multi-column query. + * + * @param row + * @param rowOffset + * @param rowLen + * @param col + * @param colOffset + * @param colLen + * @return True if passes + */ + public boolean passesGeneralBloomFilter(byte[] row, int rowOffset, + int rowLen, byte[] col, int colOffset, int colLen) { + if (generalBloomFilter == null) + return true; + + byte[] key; + switch (bloomFilterType) { + case ROW: + if (col != null) { + throw new RuntimeException("Row-only Bloom filter called with " + + "column specified"); + } + if (rowOffset != 0 || rowLen != row.length) { + throw new AssertionError("For row-only Bloom filters the row " + + "must occupy the whole array"); + } + key = row; + break; + + case ROWCOL: + key = generalBloomFilter.createBloomKey(row, rowOffset, rowLen, col, + colOffset, colLen); + break; + + default: + return true; + } + + // Cache Bloom filter as a local variable in case it is set to null by + // another thread on an IO error. + BloomFilter bloomFilter = this.generalBloomFilter; + + if (bloomFilter == null) { + return true; + } + + // Empty file + if (reader.getTrailer().getEntryCount() == 0) + return false; + + try { + boolean shouldCheckBloom; + ByteBuffer bloom; + if (bloomFilter.supportsAutoLoading()) { + bloom = null; + shouldCheckBloom = true; + } else { + bloom = reader.getMetaBlock(HFileWriterV1.BLOOM_FILTER_DATA_KEY, + true); + shouldCheckBloom = bloom != null; + } + + if (shouldCheckBloom) { + boolean exists; + + // Whether the primary Bloom key is greater than the last Bloom key + // from the file info. For row-column Bloom filters this is not yet + // a sufficient condition to return false. + boolean keyIsAfterLast = lastBloomKey != null + && bloomFilter.getComparator().compare(key, lastBloomKey) > 0; + + if (bloomFilterType == BloomType.ROWCOL) { + // Since a Row Delete is essentially a DeleteFamily applied to all + // columns, a file might be skipped if using row+col Bloom filter. + // In order to ensure this file is included an additional check is + // required looking only for a row bloom. + byte[] rowBloomKey = bloomFilter.createBloomKey(row, 0, row.length, + null, 0, 0); + + if (keyIsAfterLast + && bloomFilter.getComparator().compare(rowBloomKey, + lastBloomKey) > 0) { + exists = false; + } else { + exists = + bloomFilter.contains(key, 0, key.length, bloom) || + bloomFilter.contains(rowBloomKey, 0, rowBloomKey.length, + bloom); + } + } else { + exists = !keyIsAfterLast + && bloomFilter.contains(key, 0, key.length, bloom); + } + + getSchemaMetrics().updateBloomMetrics(exists); + return exists; + } + } catch (IOException e) { + LOG.error("Error reading bloom filter data -- proceeding without", + e); + setGeneralBloomFilterFaulty(); + } catch (IllegalArgumentException e) { + LOG.error("Bad bloom filter data -- proceeding without", e); + setGeneralBloomFilterFaulty(); + } + + return true; + } + + /** + * Checks whether the given scan rowkey range overlaps with the current storefile's + * @param scan the scan specification. Used to determine the rowkey range. + * @return true if there is overlap, false otherwise + */ + public boolean passesKeyRangeFilter(Scan scan) { + if (this.getFirstKey() == null || this.getLastKey() == null) { + // the file is empty + return false; + } + if (Bytes.equals(scan.getStartRow(), HConstants.EMPTY_START_ROW) + && Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) { + return true; + } + KeyValue startKeyValue = KeyValue.createFirstOnRow(scan.getStartRow()); + KeyValue stopKeyValue = KeyValue.createLastOnRow(scan.getStopRow()); + boolean nonOverLapping = (getComparator().compare(this.getFirstKey(), + stopKeyValue.getKey()) > 0 && !Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) + || getComparator().compare(this.getLastKey(), startKeyValue.getKey()) < 0; + return !nonOverLapping; + } + + public Map loadFileInfo() throws IOException { + Map fi = reader.loadFileInfo(); + + byte[] b = fi.get(BLOOM_FILTER_TYPE_KEY); + if (b != null) { + bloomFilterType = BloomType.valueOf(Bytes.toString(b)); + } + + lastBloomKey = fi.get(LAST_BLOOM_KEY); + byte[] cnt = fi.get(DELETE_FAMILY_COUNT); + if (cnt != null) { + deleteFamilyCnt = Bytes.toLong(cnt); + } + + return fi; + } + + public void loadBloomfilter() { + this.loadBloomfilter(BlockType.GENERAL_BLOOM_META); + this.loadBloomfilter(BlockType.DELETE_FAMILY_BLOOM_META); + } + + private void loadBloomfilter(BlockType blockType) { + try { + if (blockType == BlockType.GENERAL_BLOOM_META) { + if (this.generalBloomFilter != null) + return; // Bloom has been loaded + + DataInput bloomMeta = reader.getGeneralBloomFilterMetadata(); + if (bloomMeta != null) { + // sanity check for NONE Bloom filter + if (bloomFilterType == BloomType.NONE) { + throw new IOException( + "valid bloom filter type not found in FileInfo"); + } else { + generalBloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, + reader); + LOG.info("Loaded " + bloomFilterType.toString() + " (" + + generalBloomFilter.getClass().getSimpleName() + + ") metadata for " + reader.getName()); + } + } + } else if (blockType == BlockType.DELETE_FAMILY_BLOOM_META) { + if (this.deleteFamilyBloomFilter != null) + return; // Bloom has been loaded + + DataInput bloomMeta = reader.getDeleteBloomFilterMetadata(); + if (bloomMeta != null) { + deleteFamilyBloomFilter = BloomFilterFactory.createFromMeta( + bloomMeta, reader); + LOG.info("Loaded Delete Family Bloom (" + + deleteFamilyBloomFilter.getClass().getSimpleName() + + ") metadata for " + reader.getName()); + } + } else { + throw new RuntimeException("Block Type: " + blockType.toString() + + "is not supported for Bloom filter"); + } + } catch (IOException e) { + LOG.error("Error reading bloom filter meta for " + blockType + + " -- proceeding without", e); + setBloomFilterFaulty(blockType); + } catch (IllegalArgumentException e) { + LOG.error("Bad bloom filter meta " + blockType + + " -- proceeding without", e); + setBloomFilterFaulty(blockType); + } + } + + private void setBloomFilterFaulty(BlockType blockType) { + if (blockType == BlockType.GENERAL_BLOOM_META) { + setGeneralBloomFilterFaulty(); + } else if (blockType == BlockType.DELETE_FAMILY_BLOOM_META) { + setDeleteFamilyBloomFilterFaulty(); + } + } + + /** + * The number of Bloom filter entries in this store file, or an estimate + * thereof, if the Bloom filter is not loaded. This always returns an upper + * bound of the number of Bloom filter entries. + * + * @return an estimate of the number of Bloom filter entries in this file + */ + public long getFilterEntries() { + return generalBloomFilter != null ? generalBloomFilter.getKeyCount() + : reader.getEntries(); + } + + public void setGeneralBloomFilterFaulty() { + generalBloomFilter = null; + } + + public void setDeleteFamilyBloomFilterFaulty() { + this.deleteFamilyBloomFilter = null; + } + + public byte[] getLastKey() { + return reader.getLastKey(); + } + + public byte[] midkey() throws IOException { + return reader.midkey(); + } + + public long length() { + return reader.length(); + } + + public long getTotalUncompressedBytes() { + return reader.getTrailer().getTotalUncompressedBytes(); + } + + public long getEntries() { + return reader.getEntries(); + } + + public long getDeleteFamilyCnt() { + return deleteFamilyCnt; + } + + public byte[] getFirstKey() { + return reader.getFirstKey(); + } + + public long indexSize() { + return reader.indexSize(); + } + + public String getColumnFamilyName() { + return reader.getColumnFamilyName(); + } + + public BloomType getBloomFilterType() { + return this.bloomFilterType; + } + + public long getSequenceID() { + return sequenceID; + } + + public void setSequenceID(long sequenceID) { + this.sequenceID = sequenceID; + } + + BloomFilter getGeneralBloomFilter() { + return generalBloomFilter; + } + + long getUncompressedDataIndexSize() { + return reader.getTrailer().getUncompressedDataIndexSize(); + } + + public long getTotalBloomSize() { + if (generalBloomFilter == null) + return 0; + return generalBloomFilter.getByteSize(); + } + + public int getHFileVersion() { + return reader.getTrailer().getMajorVersion(); + } + + HFile.Reader getHFileReader() { + return reader; + } + + void disableBloomFilterForTesting() { + generalBloomFilter = null; + this.deleteFamilyBloomFilter = null; + } + + public long getMaxTimestamp() { + return timeRangeTracker == null ? Long.MAX_VALUE : timeRangeTracker.maximumTimestamp; + } + + @Override + public void schemaConfigurationChanged() { + passSchemaMetricsTo((SchemaConfigured) reader); + } + } + + /** + * Useful comparators for comparing StoreFiles. + */ + abstract static class Comparators { + /** + * Comparator that compares based on the flush time of + * the StoreFiles. All bulk loads are placed before all non- + * bulk loads, and then all files are sorted by sequence ID. + * If there are ties, the path name is used as a tie-breaker. + */ + static final Comparator FLUSH_TIME = + Ordering.compound(ImmutableList.of( + Ordering.natural().onResultOf(new GetBulkTime()), + Ordering.natural().onResultOf(new GetSeqId()), + Ordering.natural().onResultOf(new GetPathName()) + )); + + private static class GetBulkTime implements Function { + @Override + public Long apply(StoreFile sf) { + if (!sf.isBulkLoadResult()) return Long.MAX_VALUE; + return sf.getBulkLoadTimestamp(); + } + } + private static class GetSeqId implements Function { + @Override + public Long apply(StoreFile sf) { + if (sf.isBulkLoadResult()) return -1L; + return sf.getMaxSequenceId(); + } + } + private static class GetPathName implements Function { + @Override + public String apply(StoreFile sf) { + return sf.getPath().getName(); + } + } + + /** + * FILE_SIZE = descending sort StoreFiles (largest --> smallest in size) + */ + static final Comparator FILE_SIZE = + Ordering.natural().reverse().onResultOf(new Function() { + @Override + public Long apply(StoreFile sf) { + return sf.getReader().length(); + } + }); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java new file mode 100644 index 0000000..d69fc9f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java @@ -0,0 +1,375 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.SortedSet; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.regionserver.StoreFile.Reader; + +/** + * KeyValueScanner adaptor over the Reader. It also provides hooks into + * bloom filter things. + */ +public class StoreFileScanner implements KeyValueScanner { + static final Log LOG = LogFactory.getLog(Store.class); + + // the reader it comes from: + private final StoreFile.Reader reader; + private final HFileScanner hfs; + private KeyValue cur = null; + + private boolean realSeekDone; + private boolean delayedReseek; + private KeyValue delayedSeekKV; + + private boolean enforceMVCC = false; + + //The variable, realSeekDone, may cheat on store file scanner for the + // multi-column bloom-filter optimization. + // So this flag shows whether this storeFileScanner could do a reseek. + private boolean isReseekable = false; + + private static final AtomicLong seekCount = new AtomicLong(); + + private ScanQueryMatcher matcher; + + /** + * Implements a {@link KeyValueScanner} on top of the specified {@link HFileScanner} + * @param hfs HFile scanner + */ + public StoreFileScanner(StoreFile.Reader reader, HFileScanner hfs, boolean useMVCC) { + this.reader = reader; + this.hfs = hfs; + this.enforceMVCC = useMVCC; + } + + /** + * Return an array of scanners corresponding to the given + * set of store files. + */ + public static List getScannersForStoreFiles( + Collection files, + boolean cacheBlocks, + boolean usePread) throws IOException { + return getScannersForStoreFiles(files, cacheBlocks, + usePread, false); + } + + /** + * Return an array of scanners corresponding to the given set of store files. + */ + public static List getScannersForStoreFiles( + Collection files, boolean cacheBlocks, boolean usePread, + boolean isCompaction) throws IOException { + return getScannersForStoreFiles(files, cacheBlocks, usePread, isCompaction, + null); + } + + /** + * Return an array of scanners corresponding to the given set of store files, + * And set the ScanQueryMatcher for each store file scanner for further + * optimization + */ + public static List getScannersForStoreFiles( + Collection files, boolean cacheBlocks, boolean usePread, + boolean isCompaction, ScanQueryMatcher matcher) throws IOException { + List scanners = new ArrayList( + files.size()); + for (StoreFile file : files) { + StoreFile.Reader r = file.createReader(); + StoreFileScanner scanner = r.getStoreFileScanner(cacheBlocks, usePread, + isCompaction); + scanner.setScanQueryMatcher(matcher); + scanners.add(scanner); + } + return scanners; + } + + public String toString() { + return "StoreFileScanner[" + hfs.toString() + ", cur=" + cur + "]"; + } + + public KeyValue peek() { + return cur; + } + + public KeyValue next() throws IOException { + KeyValue retKey = cur; + + try { + // only seek if we aren't at the end. cur == null implies 'end'. + if (cur != null) { + hfs.next(); + cur = hfs.getKeyValue(); + skipKVsNewerThanReadpoint(); + } + } catch(IOException e) { + throw new IOException("Could not iterate " + this, e); + } + return retKey; + } + + public boolean seek(KeyValue key) throws IOException { + seekCount.incrementAndGet(); + + try { + try { + if(!seekAtOrAfter(hfs, key)) { + close(); + return false; + } + + this.isReseekable = true; + cur = hfs.getKeyValue(); + + return skipKVsNewerThanReadpoint(); + } finally { + realSeekDone = true; + } + } catch (IOException ioe) { + throw new IOException("Could not seek " + this + " to key " + key, ioe); + } + } + + public boolean reseek(KeyValue key) throws IOException { + seekCount.incrementAndGet(); + + try { + try { + if (!reseekAtOrAfter(hfs, key)) { + close(); + return false; + } + cur = hfs.getKeyValue(); + + return skipKVsNewerThanReadpoint(); + } finally { + realSeekDone = true; + } + } catch (IOException ioe) { + throw new IOException("Could not reseek " + this + " to key " + key, + ioe); + } + } + + protected boolean skipKVsNewerThanReadpoint() throws IOException { + long readPoint = MultiVersionConsistencyControl.getThreadReadPoint(); + + // We want to ignore all key-values that are newer than our current + // readPoint + while(enforceMVCC + && cur != null + && (cur.getMemstoreTS() > readPoint)) { + hfs.next(); + cur = hfs.getKeyValue(); + } + + if (cur == null) { + close(); + return false; + } + + // For the optimisation in HBASE-4346, we set the KV's memstoreTS to + // 0, if it is older than all the scanners' read points. It is possible + // that a newer KV's memstoreTS was reset to 0. But, there is an + // older KV which was not reset to 0 (because it was + // not old enough during flush). Make sure that we set it correctly now, + // so that the comparision order does not change. + if (cur.getMemstoreTS() <= readPoint) { + cur.setMemstoreTS(0); + } + return true; + } + + public void close() { + // Nothing to close on HFileScanner? + cur = null; + } + + /** + * + * @param s + * @param k + * @return + * @throws IOException + */ + public static boolean seekAtOrAfter(HFileScanner s, KeyValue k) + throws IOException { + int result = s.seekTo(k.getBuffer(), k.getKeyOffset(), k.getKeyLength()); + if(result < 0) { + // Passed KV is smaller than first KV in file, work from start of file + return s.seekTo(); + } else if(result > 0) { + // Passed KV is larger than current KV in file, if there is a next + // it is the "after", if not then this scanner is done. + return s.next(); + } + // Seeked to the exact key + return true; + } + + static boolean reseekAtOrAfter(HFileScanner s, KeyValue k) + throws IOException { + //This function is similar to seekAtOrAfter function + int result = s.reseekTo(k.getBuffer(), k.getKeyOffset(), k.getKeyLength()); + if (result <= 0) { + return true; + } else { + // passed KV is larger than current KV in file, if there is a next + // it is after, if not then this scanner is done. + return s.next(); + } + } + + @Override + public long getSequenceID() { + return reader.getSequenceID(); + } + + /** + * Pretend we have done a seek but don't do it yet, if possible. The hope is + * that we find requested columns in more recent files and won't have to seek + * in older files. Creates a fake key/value with the given row/column and the + * highest (most recent) possible timestamp we might get from this file. When + * users of such "lazy scanner" need to know the next KV precisely (e.g. when + * this scanner is at the top of the heap), they run {@link #enforceSeek()}. + *

+ * Note that this function does guarantee that the current KV of this scanner + * will be advanced to at least the given KV. Because of this, it does have + * to do a real seek in cases when the seek timestamp is older than the + * highest timestamp of the file, e.g. when we are trying to seek to the next + * row/column and use OLDEST_TIMESTAMP in the seek key. + */ + @Override + public boolean requestSeek(KeyValue kv, boolean forward, boolean useBloom) + throws IOException { + if (kv.getFamilyLength() == 0) { + useBloom = false; + } + + boolean haveToSeek = true; + if (useBloom) { + // check ROWCOL Bloom filter first. + if (reader.getBloomFilterType() == StoreFile.BloomType.ROWCOL) { + haveToSeek = reader.passesGeneralBloomFilter(kv.getBuffer(), + kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(), + kv.getQualifierOffset(), kv.getQualifierLength()); + } else if (this.matcher != null && !matcher.hasNullColumnInQuery() && + kv.isDeleteFamily()) { + // if there is no such delete family kv in the store file, + // then no need to seek. + haveToSeek = reader.passesDeleteFamilyBloomFilter(kv.getBuffer(), + kv.getRowOffset(), kv.getRowLength()); + } + } + + delayedReseek = forward; + delayedSeekKV = kv; + + if (haveToSeek) { + // This row/column might be in this store file (or we did not use the + // Bloom filter), so we still need to seek. + realSeekDone = false; + long maxTimestampInFile = reader.getMaxTimestamp(); + long seekTimestamp = kv.getTimestamp(); + if (seekTimestamp > maxTimestampInFile) { + // Create a fake key that is not greater than the real next key. + // (Lower timestamps correspond to higher KVs.) + // To understand this better, consider that we are asked to seek to + // a higher timestamp than the max timestamp in this file. We know that + // the next point when we have to consider this file again is when we + // pass the max timestamp of this file (with the same row/column). + cur = kv.createFirstOnRowColTS(maxTimestampInFile); + } else { + // This will be the case e.g. when we need to seek to the next + // row/column, and we don't know exactly what they are, so we set the + // seek key's timestamp to OLDEST_TIMESTAMP to skip the rest of this + // row/column. + enforceSeek(); + } + return cur != null; + } + + // Multi-column Bloom filter optimization. + // Create a fake key/value, so that this scanner only bubbles up to the top + // of the KeyValueHeap in StoreScanner after we scanned this row/column in + // all other store files. The query matcher will then just skip this fake + // key/value and the store scanner will progress to the next column. This + // is obviously not a "real real" seek, but unlike the fake KV earlier in + // this method, we want this to be propagated to ScanQueryMatcher. + cur = kv.createLastOnRowCol(); + + realSeekDone = true; + return true; + } + + Reader getReaderForTesting() { + return reader; + } + + @Override + public boolean realSeekDone() { + return realSeekDone; + } + + @Override + public void enforceSeek() throws IOException { + if (realSeekDone) + return; + + if (delayedReseek && this.isReseekable) { + reseek(delayedSeekKV); + } else { + seek(delayedSeekKV); + } + } + + public void setScanQueryMatcher(ScanQueryMatcher matcher) { + this.matcher = matcher; + } + + @Override + public boolean isFileScanner() { + return true; + } + + // Test methods + + static final long getSeekCount() { + return seekCount.get(); + } + + @Override + public boolean shouldUseScanner(Scan scan, SortedSet columns, long oldestUnexpiredTS) { + return reader.passesTimerangeFilter(scan, oldestUnexpiredTS) + && reader.passesKeyRangeFilter(scan) && reader.passesBloomFilter(scan, columns); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java new file mode 100644 index 0000000..d2eb697 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java @@ -0,0 +1,64 @@ +/* + * Copyright 2010 The Apache Software Foundation + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; + +import org.apache.hadoop.hbase.monitoring.MonitoredTask; + +/** + * A package protected interface for a store flushing. + * A store flusher carries the state required to prepare/flush/commit the + * store's cache. + */ +interface StoreFlusher { + + /** + * Prepare for a store flush (create snapshot) + * + * Requires pausing writes. + * + * A very short operation. + */ + void prepare(); + + /** + * Flush the cache (create the new store file) + * + * A length operation which doesn't require locking out any function + * of the store. + * + * @throws IOException in case the flush fails + */ + void flushCache(MonitoredTask status) throws IOException; + + /** + * Commit the flush - add the store file to the store and clear the + * memstore snapshot. + * + * Requires pausing scans. + * + * A very short operation + * + * @return + * @throws IOException + */ + boolean commit(MonitoredTask status) throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java new file mode 100644 index 0000000..db2bd8d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -0,0 +1,572 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.regionserver.Store.ScanInfo; +import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; + +/** + * Scanner scans both the memstore and the HStore. Coalesce KeyValue stream + * into List for a single row. + */ +public class StoreScanner extends NonLazyKeyValueScanner + implements KeyValueScanner, InternalScanner, ChangedReadersObserver { + static final Log LOG = LogFactory.getLog(StoreScanner.class); + private Store store; + private ScanQueryMatcher matcher; + private KeyValueHeap heap; + private boolean cacheBlocks; + + + private String metricNamePrefix; + // Used to indicate that the scanner has closed (see HBASE-1107) + // Doesnt need to be volatile because it's always accessed via synchronized methods + private boolean closing = false; + private final boolean isGet; + private final boolean explicitColumnQuery; + private final boolean useRowColBloom; + private final Scan scan; + private final NavigableSet columns; + private final long oldestUnexpiredTS; + private final int minVersions; + + /** We don't ever expect to change this, the constant is just for clarity. */ + static final boolean LAZY_SEEK_ENABLED_BY_DEFAULT = true; + + /** Used during unit testing to ensure that lazy seek does save seek ops */ + private static boolean lazySeekEnabledGlobally = + LAZY_SEEK_ENABLED_BY_DEFAULT; + + // if heap == null and lastTop != null, you need to reseek given the key below + private KeyValue lastTop = null; + + /** An internal constructor. */ + private StoreScanner(Store store, boolean cacheBlocks, Scan scan, + final NavigableSet columns, long ttl, int minVersions) { + this.store = store; + this.cacheBlocks = cacheBlocks; + isGet = scan.isGetScan(); + int numCol = columns == null ? 0 : columns.size(); + explicitColumnQuery = numCol > 0; + this.scan = scan; + this.columns = columns; + oldestUnexpiredTS = EnvironmentEdgeManager.currentTimeMillis() - ttl; + this.minVersions = minVersions; + + // We look up row-column Bloom filters for multi-column queries as part of + // the seek operation. However, we also look the row-column Bloom filter + // for multi-row (non-"get") scans because this is not done in + // StoreFile.passesBloomFilter(Scan, SortedSet). + useRowColBloom = numCol > 1 || (!isGet && numCol == 1); + } + + /** + * Opens a scanner across memstore, snapshot, and all StoreFiles. Assumes we + * are not in a compaction. + * + * @param store who we scan + * @param scan the spec + * @param columns which columns we are scanning + * @throws IOException + */ + public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, final NavigableSet columns) + throws IOException { + this(store, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(), + scanInfo.getMinVersions()); + initializeMetricNames(); + if (columns != null && scan.isRaw()) { + throw new DoNotRetryIOException( + "Cannot specify any column for a raw scan"); + } + matcher = new ScanQueryMatcher(scan, scanInfo, columns, + ScanType.USER_SCAN, Long.MAX_VALUE, HConstants.LATEST_TIMESTAMP, + oldestUnexpiredTS); + + // Pass columns to try to filter out unnecessary StoreFiles. + List scanners = getScannersNoCompaction(); + + // Seek all scanners to the start of the Row (or if the exact matching row + // key does not exist, then to the start of the next matching Row). + // Always check bloom filter to optimize the top row seek for delete + // family marker. + if (explicitColumnQuery && lazySeekEnabledGlobally) { + for (KeyValueScanner scanner : scanners) { + scanner.requestSeek(matcher.getStartKey(), false, true); + } + } else { + for (KeyValueScanner scanner : scanners) { + scanner.seek(matcher.getStartKey()); + } + } + + // Combine all seeked scanners with a heap + heap = new KeyValueHeap(scanners, store.comparator); + + this.store.addChangedReaderObserver(this); + } + + /** + * Used for major compactions.

+ * + * Opens a scanner across specified StoreFiles. + * @param store who we scan + * @param scan the spec + * @param scanners ancillary scanners + * @param smallestReadPoint the readPoint that we should use for tracking + * versions + */ + public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, + List scanners, ScanType scanType, + long smallestReadPoint, long earliestPutTs) throws IOException { + this(store, false, scan, null, scanInfo.getTtl(), + scanInfo.getMinVersions()); + initializeMetricNames(); + matcher = new ScanQueryMatcher(scan, scanInfo, null, scanType, + smallestReadPoint, earliestPutTs, oldestUnexpiredTS); + + // Filter the list of scanners using Bloom filters, time range, TTL, etc. + scanners = selectScannersFrom(scanners); + + // Seek all scanners to the initial key + for(KeyValueScanner scanner : scanners) { + scanner.seek(matcher.getStartKey()); + } + + // Combine all seeked scanners with a heap + heap = new KeyValueHeap(scanners, store.comparator); + } + + /** Constructor for testing. */ + StoreScanner(final Scan scan, Store.ScanInfo scanInfo, + ScanType scanType, final NavigableSet columns, + final List scanners) throws IOException { + this(scan, scanInfo, scanType, columns, scanners, + HConstants.LATEST_TIMESTAMP); + } + + // Constructor for testing. + StoreScanner(final Scan scan, Store.ScanInfo scanInfo, + ScanType scanType, final NavigableSet columns, + final List scanners, long earliestPutTs) + throws IOException { + this(null, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(), + scanInfo.getMinVersions()); + this.initializeMetricNames(); + this.matcher = new ScanQueryMatcher(scan, scanInfo, columns, scanType, + Long.MAX_VALUE, earliestPutTs, oldestUnexpiredTS); + + // Seek all scanners to the initial key + for (KeyValueScanner scanner : scanners) { + scanner.seek(matcher.getStartKey()); + } + heap = new KeyValueHeap(scanners, scanInfo.getComparator()); + } + + /** + * Method used internally to initialize metric names throughout the + * constructors. + * + * To be called after the store variable has been initialized! + */ + private void initializeMetricNames() { + String tableName = SchemaMetrics.UNKNOWN; + String family = SchemaMetrics.UNKNOWN; + if (store != null) { + tableName = store.getTableName(); + family = Bytes.toString(store.getFamily().getName()); + } + this.metricNamePrefix = + SchemaMetrics.generateSchemaMetricsPrefix(tableName, family); + } + + /** + * Get a filtered list of scanners. Assumes we are not in a compaction. + * @return list of scanners to seek + */ + private List getScannersNoCompaction() throws IOException { + final boolean isCompaction = false; + return selectScannersFrom(store.getScanners(cacheBlocks, isGet, + isCompaction, matcher)); + } + + /** + * Filters the given list of scanners using Bloom filter, time range, and + * TTL. + */ + private List selectScannersFrom( + final List allScanners) { + boolean memOnly; + boolean filesOnly; + if (scan instanceof InternalScan) { + InternalScan iscan = (InternalScan)scan; + memOnly = iscan.isCheckOnlyMemStore(); + filesOnly = iscan.isCheckOnlyStoreFiles(); + } else { + memOnly = false; + filesOnly = false; + } + + List scanners = + new ArrayList(allScanners.size()); + + // We can only exclude store files based on TTL if minVersions is set to 0. + // Otherwise, we might have to return KVs that have technically expired. + long expiredTimestampCutoff = minVersions == 0 ? oldestUnexpiredTS : + Long.MIN_VALUE; + + // include only those scan files which pass all filters + for (KeyValueScanner kvs : allScanners) { + boolean isFile = kvs.isFileScanner(); + if ((!isFile && filesOnly) || (isFile && memOnly)) { + continue; + } + + if (kvs.shouldUseScanner(scan, columns, expiredTimestampCutoff)) { + scanners.add(kvs); + } + } + return scanners; + } + + @Override + public synchronized KeyValue peek() { + if (this.heap == null) { + return this.lastTop; + } + return this.heap.peek(); + } + + @Override + public KeyValue next() { + // throw runtime exception perhaps? + throw new RuntimeException("Never call StoreScanner.next()"); + } + + @Override + public synchronized void close() { + if (this.closing) return; + this.closing = true; + // under test, we dont have a this.store + if (this.store != null) + this.store.deleteChangedReaderObserver(this); + if (this.heap != null) + this.heap.close(); + this.heap = null; // CLOSED! + this.lastTop = null; // If both are null, we are closed. + } + + @Override + public synchronized boolean seek(KeyValue key) throws IOException { + if (this.heap == null) { + + List scanners = getScannersNoCompaction(); + + heap = new KeyValueHeap(scanners, store.comparator); + } + + return this.heap.seek(key); + } + + /** + * Get the next row of values from this Store. + * @param outResult + * @param limit + * @return true if there are more rows, false if scanner is done + */ + @Override + public synchronized boolean next(List outResult, int limit) throws IOException { + return next(outResult, limit, null); + } + + /** + * Get the next row of values from this Store. + * @param outResult + * @param limit + * @return true if there are more rows, false if scanner is done + */ + @Override + public synchronized boolean next(List outResult, int limit, + String metric) throws IOException { + + if (checkReseek()) { + return true; + } + + // if the heap was left null, then the scanners had previously run out anyways, close and + // return. + if (this.heap == null) { + close(); + return false; + } + + KeyValue peeked = this.heap.peek(); + if (peeked == null) { + close(); + return false; + } + + // only call setRow if the row changes; avoids confusing the query matcher + // if scanning intra-row + byte[] row = peeked.getBuffer(); + int offset = peeked.getRowOffset(); + short length = peeked.getRowLength(); + if (limit < 0 || matcher.row == null || !Bytes.equals(row, offset, length, matcher.row, matcher.rowOffset, matcher.rowLength)) { + matcher.setRow(row, offset, length); + } + + KeyValue kv; + KeyValue prevKV = null; + + // Only do a sanity-check if store and comparator are available. + KeyValue.KVComparator comparator = + store != null ? store.getComparator() : null; + + long cumulativeMetric = 0; + int count = 0; + try { + LOOP: while((kv = this.heap.peek()) != null) { + // Check that the heap gives us KVs in an increasing order. + assert prevKV == null || comparator == null || comparator.compare(prevKV, kv) <= 0 : + "Key " + prevKV + " followed by a " + "smaller key " + kv + " in cf " + store; + prevKV = kv; + ScanQueryMatcher.MatchCode qcode = matcher.match(kv); + switch(qcode) { + case INCLUDE: + case INCLUDE_AND_SEEK_NEXT_ROW: + case INCLUDE_AND_SEEK_NEXT_COL: + + Filter f = matcher.getFilter(); + outResult.add(f == null ? kv : f.transform(kv)); + count++; + + if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW) { + if (!matcher.moreRowsMayExistAfter(kv)) { + return false; + } + reseek(matcher.getKeyForNextRow(kv)); + } else if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL) { + reseek(matcher.getKeyForNextColumn(kv)); + } else { + this.heap.next(); + } + + cumulativeMetric += kv.getLength(); + if (limit > 0 && (count == limit)) { + break LOOP; + } + continue; + + case DONE: + return true; + + case DONE_SCAN: + close(); + + return false; + + case SEEK_NEXT_ROW: + // This is just a relatively simple end of scan fix, to short-cut end + // us if there is an endKey in the scan. + if (!matcher.moreRowsMayExistAfter(kv)) { + return false; + } + + reseek(matcher.getKeyForNextRow(kv)); + break; + + case SEEK_NEXT_COL: + reseek(matcher.getKeyForNextColumn(kv)); + break; + + case SKIP: + this.heap.next(); + break; + + case SEEK_NEXT_USING_HINT: + KeyValue nextKV = matcher.getNextKeyHint(kv); + if (nextKV != null) { + reseek(nextKV); + } else { + heap.next(); + } + break; + + default: + throw new RuntimeException("UNEXPECTED"); + } + } + } finally { + if (cumulativeMetric > 0 && metric != null) { + RegionMetricsStorage.incrNumericMetric(this.metricNamePrefix + metric, + cumulativeMetric); + } + } + + if (count > 0) { + return true; + } + + // No more keys + close(); + return false; + } + + @Override + public synchronized boolean next(List outResult) throws IOException { + return next(outResult, -1, null); + } + + @Override + public synchronized boolean next(List outResult, String metric) + throws IOException { + return next(outResult, -1, metric); + } + + // Implementation of ChangedReadersObserver + @Override + public synchronized void updateReaders() throws IOException { + if (this.closing) return; + + // All public synchronized API calls will call 'checkReseek' which will cause + // the scanner stack to reseek if this.heap==null && this.lastTop != null. + // But if two calls to updateReaders() happen without a 'next' or 'peek' then we + // will end up calling this.peek() which would cause a reseek in the middle of a updateReaders + // which is NOT what we want, not to mention could cause an NPE. So we early out here. + if (this.heap == null) return; + + // this could be null. + this.lastTop = this.peek(); + + //DebugPrint.println("SS updateReaders, topKey = " + lastTop); + + // close scanners to old obsolete Store files + this.heap.close(); // bubble thru and close all scanners. + this.heap = null; // the re-seeks could be slow (access HDFS) free up memory ASAP + + // Let the next() call handle re-creating and seeking + } + + /** + * @return true if top of heap has changed (and KeyValueHeap has to try the + * next KV) + * @throws IOException + */ + private boolean checkReseek() throws IOException { + if (this.heap == null && this.lastTop != null) { + resetScannerStack(this.lastTop); + if (this.heap.peek() == null + || store.comparator.compareRows(this.lastTop, this.heap.peek()) != 0) { + LOG.debug("Storescanner.peek() is changed where before = " + + this.lastTop.toString() + ",and after = " + this.heap.peek()); + this.lastTop = null; + return true; + } + this.lastTop = null; // gone! + } + // else dont need to reseek + return false; + } + + private void resetScannerStack(KeyValue lastTopKey) throws IOException { + if (heap != null) { + throw new RuntimeException("StoreScanner.reseek run on an existing heap!"); + } + + /* When we have the scan object, should we not pass it to getScanners() + * to get a limited set of scanners? We did so in the constructor and we + * could have done it now by storing the scan object from the constructor */ + List scanners = getScannersNoCompaction(); + + for(KeyValueScanner scanner : scanners) { + scanner.seek(lastTopKey); + } + + // Combine all seeked scanners with a heap + heap = new KeyValueHeap(scanners, store.comparator); + + // Reset the state of the Query Matcher and set to top row. + // Only reset and call setRow if the row changes; avoids confusing the + // query matcher if scanning intra-row. + KeyValue kv = heap.peek(); + if (kv == null) { + kv = lastTopKey; + } + byte[] row = kv.getBuffer(); + int offset = kv.getRowOffset(); + short length = kv.getRowLength(); + if ((matcher.row == null) || !Bytes.equals(row, offset, length, matcher.row, matcher.rowOffset, matcher.rowLength)) { + matcher.reset(); + matcher.setRow(row, offset, length); + } + } + + @Override + public synchronized boolean reseek(KeyValue kv) throws IOException { + //Heap will not be null, if this is called from next() which. + //If called from RegionScanner.reseek(...) make sure the scanner + //stack is reset if needed. + checkReseek(); + if (explicitColumnQuery && lazySeekEnabledGlobally) { + return heap.requestSeek(kv, true, useRowColBloom); + } else { + return heap.reseek(kv); + } + } + + @Override + public long getSequenceID() { + return 0; + } + + /** + * Used in testing. + * @return all scanners in no particular order + */ + List getAllScannersForTesting() { + List allScanners = new ArrayList(); + KeyValueScanner current = heap.getCurrentForTesting(); + if (current != null) + allScanners.add(current); + for (KeyValueScanner scanner : heap.getHeap()) + allScanners.add(scanner); + return allScanners; + } + + static void enableLazySeekGlobally(boolean enable) { + lazySeekEnabledGlobally = enable; + } +} + diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java b/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java new file mode 100644 index 0000000..ec028d1 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java @@ -0,0 +1,152 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.io.TimeRange; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Writable; + +/** + * Stores the minimum and maximum timestamp values (both are inclusive). + * Can be used to find if any given time range overlaps with its time range + * MemStores use this class to track its minimum and maximum timestamps. + * When writing StoreFiles, this information is stored in meta blocks and used + * at read time to match against the required TimeRange. + */ +public class TimeRangeTracker implements Writable { + + long minimumTimestamp = -1; + long maximumTimestamp = -1; + + /** + * Default constructor. + * Initializes TimeRange to be null + */ + public TimeRangeTracker() { + + } + + /** + * Copy Constructor + * @param trt source TimeRangeTracker + */ + public TimeRangeTracker(final TimeRangeTracker trt) { + this.minimumTimestamp = trt.getMinimumTimestamp(); + this.maximumTimestamp = trt.getMaximumTimestamp(); + } + + public TimeRangeTracker(long minimumTimestamp, long maximumTimestamp) { + this.minimumTimestamp = minimumTimestamp; + this.maximumTimestamp = maximumTimestamp; + } + + /** + * Update the current TimestampRange to include the timestamp from KeyValue + * If the Key is of type DeleteColumn or DeleteFamily, it includes the + * entire time range from 0 to timestamp of the key. + * @param kv the KeyValue to include + */ + public void includeTimestamp(final KeyValue kv) { + includeTimestamp(kv.getTimestamp()); + if (kv.isDeleteColumnOrFamily()) { + includeTimestamp(0); + } + } + + /** + * Update the current TimestampRange to include the timestamp from Key. + * If the Key is of type DeleteColumn or DeleteFamily, it includes the + * entire time range from 0 to timestamp of the key. + * @param key + */ + public void includeTimestamp(final byte[] key) { + includeTimestamp(Bytes.toLong(key,key.length-KeyValue.TIMESTAMP_TYPE_SIZE)); + int type = key[key.length - 1]; + if (type == Type.DeleteColumn.getCode() || + type == Type.DeleteFamily.getCode()) { + includeTimestamp(0); + } + } + + /** + * If required, update the current TimestampRange to include timestamp + * @param timestamp the timestamp value to include + */ + private void includeTimestamp(final long timestamp) { + if (maximumTimestamp == -1) { + minimumTimestamp = timestamp; + maximumTimestamp = timestamp; + } + else if (minimumTimestamp > timestamp) { + minimumTimestamp = timestamp; + } + else if (maximumTimestamp < timestamp) { + maximumTimestamp = timestamp; + } + return; + } + + /** + * Check if the range has any overlap with TimeRange + * @param tr TimeRange + * @return True if there is overlap, false otherwise + */ + public boolean includesTimeRange(final TimeRange tr) { + return (this.minimumTimestamp < tr.getMax() && + this.maximumTimestamp >= tr.getMin()); + } + + /** + * @return the minimumTimestamp + */ + public long getMinimumTimestamp() { + return minimumTimestamp; + } + + /** + * @return the maximumTimestamp + */ + public long getMaximumTimestamp() { + return maximumTimestamp; + } + + public void write(final DataOutput out) throws IOException { + out.writeLong(minimumTimestamp); + out.writeLong(maximumTimestamp); + } + + public void readFields(final DataInput in) throws IOException { + this.minimumTimestamp = in.readLong(); + this.maximumTimestamp = in.readLong(); + } + + @Override + public String toString() { + return "[" + minimumTimestamp + "," + maximumTimestamp + "]"; + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/WrongRegionException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/WrongRegionException.java new file mode 100644 index 0000000..52b9a6c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/WrongRegionException.java @@ -0,0 +1,42 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; + +/** + * Thrown when a request contains a key which is not part of this region + */ +public class WrongRegionException extends IOException { + private static final long serialVersionUID = 993179627856392526L; + + /** constructor */ + public WrongRegionException() { + super(); + } + + /** + * Constructor + * @param s message + */ + public WrongRegionException(String s) { + super(s); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactSelection.java b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactSelection.java new file mode 100644 index 0000000..26338e1 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactSelection.java @@ -0,0 +1,214 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.compactions; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.regionserver.StoreFile; + +public class CompactSelection { + private static final long serialVersionUID = 1L; + static final Log LOG = LogFactory.getLog(CompactSelection.class); + // the actual list - this is needed to handle methods like "sublist" + // correctly + List filesToCompact = new ArrayList(); + + /** + * Number of off peak compactions either in the compaction queue or + * happening now. Please lock compactionCountLock before modifying. + */ + static long numOutstandingOffPeakCompactions = 0; + + /** + * Lock object for numOutstandingOffPeakCompactions + */ + private final static Object compactionCountLock = new Object(); + + // HBase conf object + Configuration conf; + // was this compaction promoted to an off-peak + boolean isOffPeakCompaction = false; + // compactRatio: double on purpose! Float.MAX < Long.MAX < Double.MAX + // With float, java will downcast your long to float for comparisons (bad) + private double compactRatio; + // compaction ratio off-peak + private double compactRatioOffPeak; + // offpeak start time + private int offPeakStartHour = -1; + // off peak end time + private int offPeakEndHour = -1; + + public CompactSelection(Configuration conf, List filesToCompact) { + this.filesToCompact = filesToCompact; + this.conf = conf; + this.compactRatio = conf.getFloat("hbase.hstore.compaction.ratio", 1.2F); + this.compactRatioOffPeak = conf.getFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F); + + // Peak time is from [offPeakStartHour, offPeakEndHour). Valid numbers are [0, 23] + this.offPeakStartHour = conf.getInt("hbase.offpeak.start.hour", -1); + this.offPeakEndHour = conf.getInt("hbase.offpeak.end.hour", -1); + if (!isValidHour(this.offPeakStartHour) || !isValidHour(this.offPeakEndHour)) { + if (!(this.offPeakStartHour == -1 && this.offPeakEndHour == -1)) { + LOG.warn("Invalid start/end hour for peak hour : start = " + + this.offPeakStartHour + " end = " + this.offPeakEndHour + + ". Valid numbers are [0-23]"); + } + this.offPeakStartHour = this.offPeakEndHour = -1; + } + } + + /** + * Select the expired store files to compact + * + * @param maxExpiredTimeStamp + * The store file will be marked as expired if its max time stamp is + * less than this maxExpiredTimeStamp. + * @return A CompactSelection contains the expired store files as + * filesToCompact + */ + public CompactSelection selectExpiredStoreFilesToCompact( + long maxExpiredTimeStamp) { + if (filesToCompact == null || filesToCompact.size() == 0) + return null; + ArrayList expiredStoreFiles = null; + boolean hasExpiredStoreFiles = false; + CompactSelection expiredSFSelection = null; + + for (StoreFile storeFile : this.filesToCompact) { + if (storeFile.getReader().getMaxTimestamp() < maxExpiredTimeStamp) { + LOG.info("Deleting the expired store file by compaction: " + + storeFile.getPath() + " whose maxTimeStamp is " + + storeFile.getReader().getMaxTimestamp() + + " while the max expired timestamp is " + maxExpiredTimeStamp); + if (!hasExpiredStoreFiles) { + expiredStoreFiles = new ArrayList(); + hasExpiredStoreFiles = true; + } + expiredStoreFiles.add(storeFile); + } + } + + if (hasExpiredStoreFiles) { + expiredSFSelection = new CompactSelection(conf, expiredStoreFiles); + } + return expiredSFSelection; + } + + /** + * If the current hour falls in the off peak times and there are no + * outstanding off peak compactions, the current compaction is + * promoted to an off peak compaction. Currently only one off peak + * compaction is present in the compaction queue. + * + * @param currentHour + * @return + */ + public double getCompactSelectionRatio() { + double r = this.compactRatio; + synchronized(compactionCountLock) { + if (isOffPeakHour() && numOutstandingOffPeakCompactions == 0) { + r = this.compactRatioOffPeak; + numOutstandingOffPeakCompactions++; + isOffPeakCompaction = true; + } + } + if(isOffPeakCompaction) { + LOG.info("Running an off-peak compaction, selection ratio = " + + compactRatioOffPeak + ", numOutstandingOffPeakCompactions is now " + + numOutstandingOffPeakCompactions); + } + return r; + } + + /** + * The current compaction finished, so reset the off peak compactions count + * if this was an off peak compaction. + */ + public void finishRequest() { + if (isOffPeakCompaction) { + synchronized(compactionCountLock) { + numOutstandingOffPeakCompactions--; + isOffPeakCompaction = false; + } + LOG.info("Compaction done, numOutstandingOffPeakCompactions is now " + + numOutstandingOffPeakCompactions); + } + } + + public List getFilesToCompact() { + return filesToCompact; + } + + /** + * Removes all files from the current compaction list, and resets off peak + * compactions is set. + */ + public void emptyFileList() { + filesToCompact.clear(); + if (isOffPeakCompaction) { + synchronized(compactionCountLock) { + // reset the off peak count + numOutstandingOffPeakCompactions--; + isOffPeakCompaction = false; + } + LOG.info("Nothing to compact, numOutstandingOffPeakCompactions is now " + + numOutstandingOffPeakCompactions); + } + } + + public boolean isOffPeakCompaction() { + return this.isOffPeakCompaction; + } + + private boolean isOffPeakHour() { + int currentHour = (new GregorianCalendar()).get(Calendar.HOUR_OF_DAY); + // If offpeak time checking is disabled just return false. + if (this.offPeakStartHour == this.offPeakEndHour) { + return false; + } + if (this.offPeakStartHour < this.offPeakEndHour) { + return (currentHour >= this.offPeakStartHour && currentHour < this.offPeakEndHour); + } + return (currentHour >= this.offPeakStartHour || currentHour < this.offPeakEndHour); + } + + public CompactSelection subList(int start, int end) { + throw new UnsupportedOperationException(); + } + + public CompactSelection getSubList(int start, int end) { + filesToCompact = filesToCompact.subList(start, end); + return this; + } + + public void clearSubList(int start, int end) { + filesToCompact.subList(start, end).clear(); + } + + private boolean isValidHour(int hour) { + return (hour >= 0 && hour <= 23); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java new file mode 100644 index 0000000..f91b782 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java @@ -0,0 +1,52 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.compactions; + +/** + * This class holds information relevant for tracking the progress of a + * compaction. + * + *

The metrics tracked allow one to calculate the percent completion of the + * compaction based on the number of Key/Value pairs already compacted vs. + * total amount scheduled to be compacted. + * + */ +public class CompactionProgress { + + /** the total compacting key values in currently running compaction */ + public long totalCompactingKVs; + /** the completed count of key values in currently running compaction */ + public long currentCompactedKVs = 0; + + /** Constructor + * @param totalCompactingKVs the total Key/Value pairs to be compacted + */ + public CompactionProgress(long totalCompactingKVs) { + this.totalCompactingKVs = totalCompactingKVs; + } + + /** getter for calculated percent complete + * @return float + */ + public float getProgressPct() { + return currentCompactedKVs / totalCompactingKVs; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java new file mode 100644 index 0000000..65f14be --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java @@ -0,0 +1,355 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.compactions; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.RemoteExceptionHandler; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; + +/** + * This class holds all details necessary to run a compaction. + */ +public class CompactionRequest implements Comparable, + Runnable { + static final Log LOG = LogFactory.getLog(CompactionRequest.class); + private final HRegion r; + private final Store s; + private CompactSelection compactSelection; + private long totalSize; + private boolean isMajor; + private int p; + private final Long timeInNanos; + private HRegionServer server = null; + + /** + * Map to track the number of compaction requested per region (id) + */ + private static final ConcurrentHashMap + majorCompactions = new ConcurrentHashMap(); + private static final ConcurrentHashMap + minorCompactions = new ConcurrentHashMap(); + + /** + * Create a simple compaction request just for testing - this lets you specify everything you + * would need in the general case of testing compactions from an external perspective (e.g. + * requesting a compaction through the HRegion). + * @param store + * @param conf + * @param selection + * @param isMajor + * @return a request that is useful in requesting compactions for testing + */ + public static CompactionRequest getRequestForTesting(Store store, Configuration conf, + Collection selection, boolean isMajor) { + return new CompactionRequest(store.getHRegion(), store, new CompactSelection(conf, + new ArrayList( + selection)), isMajor, 0, System.nanoTime()); + } + + /** + * Constructor for a custom compaction. Uses the setXXX methods to update the state of the + * compaction before being used. Uses the current system time on creation as the start time. + * @param region region that is being compacted + * @param store store which is being compacted + * @param priority specified priority with which this compaction should enter the queue. + */ + public CompactionRequest(HRegion region, Store store, int priority) { + this(region, store, null, false, priority, System.nanoTime()); + } + + public CompactionRequest(HRegion r, Store s, CompactSelection files, boolean isMajor, int p) { + // delegate to the internal constructor after checking basic preconditions + this(Preconditions.checkNotNull(r), s, Preconditions.checkNotNull(files), isMajor, p, System + .nanoTime()); + } + + private CompactionRequest(HRegion region, Store store, CompactSelection files, boolean isMajor, + int priority, long startTime) { + this.r = region; + this.s = store; + this.isMajor = isMajor; + this.p = priority; + this.timeInNanos = startTime; + if (files != null) { + this.setSelection(files); + } + } + + /** + * Find out if a given region in compaction now. + * + * @param regionId + * @return + */ + public static CompactionState getCompactionState( + final long regionId) { + Long key = Long.valueOf(regionId); + AtomicInteger major = majorCompactions.get(key); + AtomicInteger minor = minorCompactions.get(key); + int state = 0; + if (minor != null && minor.get() > 0) { + state += 1; // use 1 to indicate minor here + } + if (major != null && major.get() > 0) { + state += 2; // use 2 to indicate major here + } + switch (state) { + case 3: // 3 = 2 + 1, so both major and minor + return CompactionState.MAJOR_AND_MINOR; + case 2: + return CompactionState.MAJOR; + case 1: + return CompactionState.MINOR; + default: + return CompactionState.NONE; + } + } + + public static void preRequest(final CompactionRequest cr){ + Long key = Long.valueOf(cr.getHRegion().getRegionId()); + ConcurrentHashMap compactions = + cr.isMajor() ? majorCompactions : minorCompactions; + AtomicInteger count = compactions.get(key); + if (count == null) { + compactions.putIfAbsent(key, new AtomicInteger(0)); + count = compactions.get(key); + } + count.incrementAndGet(); + } + + public static void postRequest(final CompactionRequest cr){ + Long key = Long.valueOf(cr.getHRegion().getRegionId()); + ConcurrentHashMap compactions = + cr.isMajor() ? majorCompactions : minorCompactions; + AtomicInteger count = compactions.get(key); + if (count != null) { + count.decrementAndGet(); + } + } + + public void finishRequest() { + this.compactSelection.finishRequest(); + } + + /** + * This function will define where in the priority queue the request will + * end up. Those with the highest priorities will be first. When the + * priorities are the same it will first compare priority then date + * to maintain a FIFO functionality. + * + *

Note: The date is only accurate to the millisecond which means it is + * possible that two requests were inserted into the queue within a + * millisecond. When that is the case this function will break the tie + * arbitrarily. + */ + @Override + public int compareTo(CompactionRequest request) { + //NOTE: The head of the priority queue is the least element + if (this.equals(request)) { + return 0; //they are the same request + } + int compareVal; + + compareVal = p - request.p; //compare priority + if (compareVal != 0) { + return compareVal; + } + + compareVal = timeInNanos.compareTo(request.timeInNanos); + if (compareVal != 0) { + return compareVal; + } + + // break the tie based on hash code + return this.hashCode() - request.hashCode(); + } + + /** Gets the HRegion for the request */ + public HRegion getHRegion() { + return r; + } + + /** Gets the Store for the request */ + public Store getStore() { + return s; + } + + /** Gets the compact selection object for the request */ + public CompactSelection getCompactSelection() { + return compactSelection; + } + + /** Gets the StoreFiles for the request */ + public List getFiles() { + return compactSelection.getFilesToCompact(); + } + + /** Gets the total size of all StoreFiles in compaction */ + public long getSize() { + return totalSize; + } + + public boolean isMajor() { + return this.isMajor; + } + + /** Gets the priority for the request */ + public int getPriority() { + return p; + } + + /** Gets the priority for the request */ + public void setPriority(int p) { + this.p = p; + } + + public void setServer(HRegionServer hrs) { + this.server = hrs; + } + + /** + * Set the files (and, implicitly, the size of the compaction based on those files) + * @param files files that should be included in the compaction + */ + public void setSelection(CompactSelection files) { + long sz = 0; + for (StoreFile sf : files.getFilesToCompact()) { + sz += sf.getReader().length(); + } + this.totalSize = sz; + this.compactSelection = files; + } + + /** + * Specify if this compaction should be a major compaction based on the state of the store + * @param isMajor true if the system determines that this compaction should be a major + * compaction + */ + public void setIsMajor(boolean isMajor) { + this.isMajor = isMajor; + } + + @Override + public String toString() { + String fsList = Joiner.on(", ").join( + Collections2.transform(Collections2.filter( + compactSelection.getFilesToCompact(), + new Predicate() { + public boolean apply(StoreFile sf) { + return sf.getReader() != null; + } + }), new Function() { + public String apply(StoreFile sf) { + return StringUtils.humanReadableInt(sf.getReader().length()); + } + })); + + return "regionName=" + r.getRegionNameAsString() + + ", storeName=" + new String(s.getFamily().getName()) + + ", fileCount=" + compactSelection.getFilesToCompact().size() + + ", fileSize=" + StringUtils.humanReadableInt(totalSize) + + ((fsList.isEmpty()) ? "" : " (" + fsList + ")") + + ", priority=" + p + ", time=" + timeInNanos; + } + + @Override + public void run() { + Preconditions.checkNotNull(server); + if (server.isStopped()) { + return; + } + try { + long start = EnvironmentEdgeManager.currentTimeMillis(); + boolean completed = r.compact(this); + long now = EnvironmentEdgeManager.currentTimeMillis(); + LOG.info(((completed) ? "completed" : "aborted") + " compaction: " + + this + "; duration=" + StringUtils.formatTimeDiff(now, start)); + if (completed) { + server.getMetrics().addCompaction(now - start, this.totalSize); + // degenerate case: blocked regions require recursive enqueues + if (s.getCompactPriority() <= 0) { + server.getCompactSplitThread() + .requestCompaction(r, s, "Recursive enqueue", null); + } else { + // see if the compaction has caused us to exceed max region size + server.getCompactSplitThread().requestSplit(r); + } + } + } catch (IOException ex) { + LOG.error("Compaction failed " + this, RemoteExceptionHandler + .checkIOException(ex)); + server.checkFileSystem(); + } catch (Exception ex) { + LOG.error("Compaction failed " + this, ex); + server.checkFileSystem(); + } finally { + s.finishRequest(this); + LOG.debug("CompactSplitThread status: " + server.getCompactSplitThread()); + } + } + + /** + * An enum for the region compaction state + */ + public static enum CompactionState { + NONE, + MINOR, + MAJOR, + MAJOR_AND_MINOR; + } + + /** + * Cleanup class to use when rejecting a compaction request from the queue. + */ + public static class Rejection implements RejectedExecutionHandler { + + @Override + public void rejectedExecution(Runnable request, ThreadPoolExecutor pool) { + if (request instanceof CompactionRequest) { + CompactionRequest cr = (CompactionRequest) request; + LOG.debug("Compaction Rejected: " + cr); + cr.getStore().finishRequest(cr); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java new file mode 100644 index 0000000..83544e2 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java @@ -0,0 +1,44 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.handler; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; + +/** + * Handles closing of the root region on a region server. + */ +public class CloseMetaHandler extends CloseRegionHandler { + // Called when master tells us shutdown a region via close rpc + public CloseMetaHandler(final Server server, + final RegionServerServices rsServices, final HRegionInfo regionInfo) { + this(server, rsServices, regionInfo, false, true, -1); + } + + // Called when regionserver determines its to go down; not master orchestrated + public CloseMetaHandler(final Server server, + final RegionServerServices rsServices, + final HRegionInfo regionInfo, + final boolean abort, final boolean zk, final int versionOfClosingNode) { + super(server, rsServices, regionInfo, abort, zk, versionOfClosingNode, + EventType.M_RS_CLOSE_META); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java new file mode 100644 index 0000000..286a6da --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java @@ -0,0 +1,184 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.handler; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.executor.EventHandler; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.zookeeper.KeeperException; + +/** + * Handles closing of a region on a region server. + */ +public class CloseRegionHandler extends EventHandler { + // NOTE on priorities shutting down. There are none for close. There are some + // for open. I think that is right. On shutdown, we want the meta to close + // before root and both to close after the user regions have closed. What + // about the case where master tells us to shutdown a catalog region and we + // have a running queue of user regions to close? + private static final Log LOG = LogFactory.getLog(CloseRegionHandler.class); + + private final int FAILED = -1; + int expectedVersion = FAILED; + + private final RegionServerServices rsServices; + + private final HRegionInfo regionInfo; + + // If true, the hosting server is aborting. Region close process is different + // when we are aborting. + private final boolean abort; + + // Update zk on closing transitions. Usually true. Its false if cluster + // is going down. In this case, its the rs that initiates the region + // close -- not the master process so state up in zk will unlikely be + // CLOSING. + private final boolean zk; + + // This is executed after receiving an CLOSE RPC from the master. + public CloseRegionHandler(final Server server, + final RegionServerServices rsServices, HRegionInfo regionInfo) { + this(server, rsServices, regionInfo, false, true, -1); + } + + /** + * This method used internally by the RegionServer to close out regions. + * @param server + * @param rsServices + * @param regionInfo + * @param abort If the regionserver is aborting. + * @param zk If the close should be noted out in zookeeper. + */ + public CloseRegionHandler(final Server server, + final RegionServerServices rsServices, + final HRegionInfo regionInfo, final boolean abort, final boolean zk, + final int versionOfClosingNode) { + this(server, rsServices, regionInfo, abort, zk, versionOfClosingNode, + EventType.M_RS_CLOSE_REGION); + } + + protected CloseRegionHandler(final Server server, + final RegionServerServices rsServices, HRegionInfo regionInfo, + boolean abort, final boolean zk, final int versionOfClosingNode, + EventType eventType) { + super(server, eventType); + this.server = server; + this.rsServices = rsServices; + this.regionInfo = regionInfo; + this.abort = abort; + this.zk = zk; + this.expectedVersion = versionOfClosingNode; + } + + public HRegionInfo getRegionInfo() { + return regionInfo; + } + + @Override + public void process() { + try { + String name = regionInfo.getRegionNameAsString(); + LOG.debug("Processing close of " + name); + String encodedRegionName = regionInfo.getEncodedName(); + // Check that this region is being served here + HRegion region = this.rsServices.getFromOnlineRegions(encodedRegionName); + if (region == null) { + LOG.warn("Received CLOSE for region " + name + + " but currently not serving"); + return; + } + + // Close the region + try { + // TODO: If we need to keep updating CLOSING stamp to prevent against + // a timeout if this is long-running, need to spin up a thread? + if (region.close(abort) == null) { + // This region got closed. Most likely due to a split. So instead + // of doing the setClosedState() below, let's just ignore cont + // The split message will clean up the master state. + LOG.warn("Can't close region: was already closed during close(): " + + regionInfo.getRegionNameAsString()); + return; + } + } catch (Throwable t) { + // A throwable here indicates that we couldn't successfully flush the + // memstore before closing. So, we need to abort the server and allow + // the master to split our logs in order to recover the data. + server.abort("Unrecoverable exception while closing region " + + regionInfo.getRegionNameAsString() + ", still finishing close", t); + throw new RuntimeException(t); + } + + this.rsServices.removeFromOnlineRegions(regionInfo.getEncodedName()); + + if (this.zk) { + if (setClosedState(this.expectedVersion, region)) { + LOG.debug("set region closed state in zk successfully for region " + + name + " sn name: " + this.server.getServerName()); + } else { + LOG.debug("set region closed state in zk unsuccessfully for region " + + name + " sn name: " + this.server.getServerName()); + } + } + + // Done! Region is closed on this RS + LOG.debug("Closed region " + region.getRegionNameAsString()); + } finally { + this.rsServices.removeFromRegionsInTransition(this.regionInfo); + } + } + + /** + * Transition ZK node to CLOSED + * @param expectedVersion + * @return If the state is set successfully + */ + private boolean setClosedState(final int expectedVersion, final HRegion region) { + try { + if (ZKAssign.transitionNodeClosed(server.getZooKeeper(), regionInfo, + server.getServerName(), expectedVersion) == FAILED) { + LOG.warn("Completed the CLOSE of a region but when transitioning from " + + " CLOSING to CLOSED got a version mismatch, someone else clashed " + + "so now unassigning"); + region.close(); + return false; + } + } catch (NullPointerException e) { + // I've seen NPE when table was deleted while close was running in unit tests. + LOG.warn("NPE during close -- catching and continuing...", e); + return false; + } catch (KeeperException e) { + LOG.error("Failed transitioning node from CLOSING to CLOSED", e); + return false; + } catch (IOException e) { + LOG.error("Failed to close region after failing to transition", e); + return false; + } + return true; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRootHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRootHandler.java new file mode 100644 index 0000000..4cfeec6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRootHandler.java @@ -0,0 +1,44 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.handler; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; + +/** + * Handles closing of the root region on a region server. + */ +public class CloseRootHandler extends CloseRegionHandler { + // This is executed after receiving an CLOSE RPC from the master for root. + public CloseRootHandler(final Server server, + final RegionServerServices rsServices, HRegionInfo regionInfo) { + this(server, rsServices, regionInfo, false, true, -1); + } + + // This is called directly by the regionserver when its determined its + // shutting down. + public CloseRootHandler(final Server server, + final RegionServerServices rsServices, HRegionInfo regionInfo, + final boolean abort, final boolean zk, final int versionOfClosingNode) { + super(server, rsServices, regionInfo, abort, zk, versionOfClosingNode, + EventType.M_RS_CLOSE_ROOT); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenMetaHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenMetaHandler.java new file mode 100644 index 0000000..66e5706 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenMetaHandler.java @@ -0,0 +1,44 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.handler; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; + +/** + * Handles opening of a meta region on a region server. + *

+ * This is executed after receiving an OPEN RPC from the master for meta. + */ +public class OpenMetaHandler extends OpenRegionHandler { + public OpenMetaHandler(final Server server, + final RegionServerServices rsServices, HRegionInfo regionInfo, + final HTableDescriptor htd) { + this(server, rsServices, regionInfo, htd, -1); + } + public OpenMetaHandler(final Server server, + final RegionServerServices rsServices, HRegionInfo regionInfo, + final HTableDescriptor htd, int versionOfOfflineNode) { + super(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_META, + versionOfOfflineNode); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java new file mode 100644 index 0000000..1fbf8f0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java @@ -0,0 +1,396 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.handler; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.executor.EventHandler; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionServerAccounting; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.CancelableProgressable; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.zookeeper.KeeperException; + +/** + * Handles opening of a region on a region server. + *

+ * This is executed after receiving an OPEN RPC from the master or client. + */ +public class OpenRegionHandler extends EventHandler { + private static final Log LOG = LogFactory.getLog(OpenRegionHandler.class); + + protected final RegionServerServices rsServices; + + private final HRegionInfo regionInfo; + private final HTableDescriptor htd; + + // We get version of our znode at start of open process and monitor it across + // the total open. We'll fail the open if someone hijacks our znode; we can + // tell this has happened if version is not as expected. + private volatile int version = -1; + + + public OpenRegionHandler(final Server server, + final RegionServerServices rsServices, HRegionInfo regionInfo, + HTableDescriptor htd) { + this(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION, -1); + } + public OpenRegionHandler(final Server server, + final RegionServerServices rsServices, HRegionInfo regionInfo, + HTableDescriptor htd, int version) { + this(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION, + version); + } + + protected OpenRegionHandler(final Server server, + final RegionServerServices rsServices, final HRegionInfo regionInfo, + final HTableDescriptor htd, EventType eventType, + final int version) { + super(server, eventType); + this.rsServices = rsServices; + this.regionInfo = regionInfo; + this.htd = htd; + this.version = version; + } + + public HRegionInfo getRegionInfo() { + return regionInfo; + } + + @Override + public void process() throws IOException { + boolean transitionToFailedOpen = false; + boolean openSuccessful = false; + try { + final String name = regionInfo.getRegionNameAsString(); + if (this.server.isStopped() || this.rsServices.isStopping()) { + return; + } + final String encodedName = regionInfo.getEncodedName(); + + // Check that this region is not already online + HRegion region = this.rsServices.getFromOnlineRegions(encodedName); + + // Open region. After a successful open, failures in subsequent + // processing needs to do a close as part of cleanup. + region = openRegion(); + if (region == null) { + tryTransitionToFailedOpen(regionInfo); + transitionToFailedOpen = true; + return; + } + boolean failed = true; + if (tickleOpening("post_region_open")) { + if (updateMeta(region)) { + failed = false; + } + } + if (failed || this.server.isStopped() || + this.rsServices.isStopping()) { + cleanupFailedOpen(region); + tryTransitionToFailedOpen(regionInfo); + transitionToFailedOpen = true; + return; + } + + if (!transitionToOpened(region)) { + // If we fail to transition to opened, it's because of one of two cases: + // (a) we lost our ZK lease + // OR (b) someone else opened the region before us + // In either case, we don't need to transition to FAILED_OPEN state. + // In case (a), the Master will process us as a dead server. In case + // (b) the region is already being handled elsewhere anyway. + cleanupFailedOpen(region); + transitionToFailedOpen = true; + return; + } + // Successful region open, and add it to OnlineRegions + this.rsServices.addToOnlineRegions(region); + openSuccessful = true; + // Done! Successful region open + LOG.debug("Opened " + name + " on server:" + + this.server.getServerName()); + } finally { + this.rsServices.removeFromRegionsInTransition(this.regionInfo); + if (!openSuccessful && !transitionToFailedOpen) { + tryTransitionToFailedOpen(regionInfo); + } + } + } + + /** + * Update ZK, ROOT or META. This can take a while if for example the + * .META. is not available -- if server hosting .META. crashed and we are + * waiting on it to come back -- so run in a thread and keep updating znode + * state meantime so master doesn't timeout our region-in-transition. + * Caller must cleanup region if this fails. + */ + boolean updateMeta(final HRegion r) { + if (this.server.isStopped() || this.rsServices.isStopping()) { + return false; + } + // Object we do wait/notify on. Make it boolean. If set, we're done. + // Else, wait. + final AtomicBoolean signaller = new AtomicBoolean(false); + PostOpenDeployTasksThread t = new PostOpenDeployTasksThread(r, + this.server, this.rsServices, signaller); + t.start(); + int assignmentTimeout = this.server.getConfiguration(). + getInt("hbase.master.assignment.timeoutmonitor.period", 10000); + // Total timeout for meta edit. If we fail adding the edit then close out + // the region and let it be assigned elsewhere. + long timeout = assignmentTimeout * 10; + long now = System.currentTimeMillis(); + long endTime = now + timeout; + // Let our period at which we update OPENING state to be be 1/3rd of the + // regions-in-transition timeout period. + long period = Math.max(1, assignmentTimeout/ 3); + long lastUpdate = now; + boolean tickleOpening = true; + while (!signaller.get() && t.isAlive() && !this.server.isStopped() && + !this.rsServices.isStopping() && (endTime > now)) { + long elapsed = now - lastUpdate; + if (elapsed > period) { + // Only tickle OPENING if postOpenDeployTasks is taking some time. + lastUpdate = now; + tickleOpening = tickleOpening("post_open_deploy"); + } + synchronized (signaller) { + try { + signaller.wait(period); + } catch (InterruptedException e) { + // Go to the loop check. + } + } + now = System.currentTimeMillis(); + } + // Is thread still alive? We may have left above loop because server is + // stopping or we timed out the edit. Is so, interrupt it. + if (t.isAlive()) { + if (!signaller.get()) { + // Thread still running; interrupt + LOG.debug("Interrupting thread " + t); + t.interrupt(); + } + try { + t.join(); + } catch (InterruptedException ie) { + LOG.warn("Interrupted joining " + + r.getRegionInfo().getRegionNameAsString(), ie); + Thread.currentThread().interrupt(); + } + } + + // Was there an exception opening the region? This should trigger on + // InterruptedException too. If so, we failed. Even if tickle opening fails + // then it is a failure. + return ((!Thread.interrupted() && t.getException() == null) && tickleOpening); + } + + /** + * Thread to run region post open tasks. Call {@link #getException()} after + * the thread finishes to check for exceptions running + * {@link RegionServerServices#postOpenDeployTasks(HRegion, org.apache.hadoop.hbase.catalog.CatalogTracker, boolean)} + * . + */ + static class PostOpenDeployTasksThread extends Thread { + private Exception exception = null; + private final Server server; + private final RegionServerServices services; + private final HRegion region; + private final AtomicBoolean signaller; + + PostOpenDeployTasksThread(final HRegion region, final Server server, + final RegionServerServices services, final AtomicBoolean signaller) { + super("PostOpenDeployTasks:" + region.getRegionInfo().getEncodedName()); + this.setDaemon(true); + this.server = server; + this.services = services; + this.region = region; + this.signaller = signaller; + } + + public void run() { + try { + this.services.postOpenDeployTasks(this.region, + this.server.getCatalogTracker(), false); + } catch (Exception e) { + LOG.warn("Exception running postOpenDeployTasks; region=" + + this.region.getRegionInfo().getEncodedName(), e); + this.exception = e; + } + // We're done. Set flag then wake up anyone waiting on thread to complete. + this.signaller.set(true); + synchronized (this.signaller) { + this.signaller.notify(); + } + } + + /** + * @return Null or the run exception; call this method after thread is done. + */ + Exception getException() { + return this.exception; + } + } + + + /** + * @param r Region we're working on. + * @return whether znode is successfully transitioned to OPENED state. + * @throws IOException + */ + private boolean transitionToOpened(final HRegion r) throws IOException { + boolean result = false; + HRegionInfo hri = r.getRegionInfo(); + final String name = hri.getRegionNameAsString(); + // Finally, Transition ZK node to OPENED + try { + if (ZKAssign.transitionNodeOpened(this.server.getZooKeeper(), hri, + this.server.getServerName(), this.version) == -1) { + LOG.warn("Completed the OPEN of region " + name + + " but when transitioning from " + + " OPENING to OPENED got a version mismatch, someone else clashed " + + "so now unassigning -- closing region on server: " + + this.server.getServerName()); + } else { + LOG.debug("region transitioned to opened in zookeeper: " + + r.getRegionInfo() + ", server: " + this.server.getServerName()); + result = true; + } + } catch (KeeperException e) { + LOG.error("Failed transitioning node " + name + + " from OPENING to OPENED -- closing region", e); + } + return result; + } + + /** + * @param Region we're working on. + * This is not guaranteed to succeed, we just do our best. + * @return whether znode is successfully transitioned to FAILED_OPEN state. + */ + private boolean tryTransitionToFailedOpen(final HRegionInfo hri) { + boolean result = false; + final String name = hri.getRegionNameAsString(); + try { + LOG.info("Opening of region " + hri + " failed, marking as FAILED_OPEN in ZK"); + if (ZKAssign.transitionNode( + this.server.getZooKeeper(), hri, + this.server.getServerName(), + EventType.RS_ZK_REGION_OPENING, + EventType.RS_ZK_REGION_FAILED_OPEN, + this.version) == -1) { + LOG.warn("Unable to mark region " + hri + " as FAILED_OPEN. " + + "It's likely that the master already timed out this open " + + "attempt, and thus another RS already has the region."); + } else { + result = true; + } + } catch (KeeperException e) { + LOG.error("Failed transitioning node " + name + + " from OPENING to FAILED_OPEN", e); + } + return result; + } + + /** + * @return Instance of HRegion if successful open else null. + */ + HRegion openRegion() { + HRegion region = null; + try { + // Instantiate the region. This also periodically tickles our zk OPENING + // state so master doesn't timeout this region in transition. + region = HRegion.openHRegion(this.regionInfo, this.htd, + this.rsServices.getWAL(this.regionInfo), + this.server.getConfiguration(), + this.rsServices, + new CancelableProgressable() { + public boolean progress() { + // We may lose the znode ownership during the open. Currently its + // too hard interrupting ongoing region open. Just let it complete + // and check we still have the znode after region open. + return tickleOpening("open_region_progress"); + } + }); + } catch (Throwable t) { + // We failed open. Our caller will see the 'null' return value + // and transition the node back to FAILED_OPEN. If that fails, + // we rely on the Timeout Monitor in the master to reassign. + LOG.error( + "Failed open of region=" + this.regionInfo.getRegionNameAsString() + + ", starting to roll back the global memstore size.", t); + // Decrease the global memstore size. + if (this.rsServices != null) { + RegionServerAccounting rsAccounting = + this.rsServices.getRegionServerAccounting(); + if (rsAccounting != null) { + rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName()); + } + } + } + return region; + } + + void cleanupFailedOpen(final HRegion region) throws IOException { + if (region != null) region.close(); + } + + + /** + * Update our OPENING state in zookeeper. + * Do this so master doesn't timeout this region-in-transition. + * @param context Some context to add to logs if failure + * @return True if successful transition. + */ + boolean tickleOpening(final String context) { + // If previous checks failed... do not try again. + if (!isGoodVersion()) return false; + String encodedName = this.regionInfo.getEncodedName(); + try { + this.version = + ZKAssign.retransitionNodeOpening(server.getZooKeeper(), + this.regionInfo, this.server.getServerName(), this.version); + } catch (KeeperException e) { + server.abort("Exception refreshing OPENING; region=" + encodedName + + ", context=" + context, e); + this.version = -1; + } + boolean b = isGoodVersion(); + if (!b) { + LOG.warn("Failed refreshing OPENING; region=" + encodedName + + ", context=" + context); + } + return b; + } + + private boolean isGoodVersion() { + return this.version != -1; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRootHandler.java b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRootHandler.java new file mode 100644 index 0000000..9a4f01a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRootHandler.java @@ -0,0 +1,44 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.handler; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; + +/** + * Handles opening of the root region on a region server. + *

+ * This is executed after receiving an OPEN RPC from the master for root. + */ +public class OpenRootHandler extends OpenRegionHandler { + public OpenRootHandler(final Server server, + final RegionServerServices rsServices, HRegionInfo regionInfo, + final HTableDescriptor htd) { + super(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_ROOT, -1); + } + public OpenRootHandler(final Server server, + final RegionServerServices rsServices, HRegionInfo regionInfo, + final HTableDescriptor htd, int versionOfOfflineNode) { + super(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_ROOT, + versionOfOfflineNode); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java new file mode 100644 index 0000000..9e99857 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java @@ -0,0 +1,259 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.regionserver.metrics; + +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * This class provides a simplified interface to expose time varying metrics + * about GET/DELETE/PUT/ICV operations on a region and on Column Families. All + * metrics are stored in {@link RegionMetricsStorage} and exposed to hadoop + * metrics through {@link RegionServerDynamicMetrics}. + */ +public class OperationMetrics { + + private static final String DELETE_KEY = "delete_"; + private static final String PUT_KEY = "put_"; + private static final String GET_KEY = "get_"; + private static final String ICV_KEY = "incrementColumnValue_"; + private static final String INCREMENT_KEY = "increment_"; + private static final String MULTIPUT_KEY = "multiput_"; + private static final String MULTIDELETE_KEY = "multidelete_"; + private static final String APPEND_KEY = "append_"; + private static final String READREQUESTCOUNT_KEY = "readrequestcount"; + private static final String WRITEREQUESTCOUNT_KEY = "writerequestcount"; + + /** Conf key controlling whether we should expose metrics.*/ + private static final String CONF_KEY = + "hbase.metrics.exposeOperationTimes"; + + private final String tableName; + private final String regionName; + private final String regionMetrixPrefix; + private final Configuration conf; + private final boolean exposeTimes; + + + /** + * Create a new OperationMetrics + * @param conf The Configuration of the HRegion reporting operations coming in. + * @param regionInfo The region info + */ + public OperationMetrics(Configuration conf, HRegionInfo regionInfo) { + // Configure SchemaMetrics before trying to create a RegionOperationMetrics instance as + // RegionOperationMetrics relies on SchemaMetrics to do naming. + if (conf != null) { + SchemaMetrics.configureGlobally(conf); + + this.conf = conf; + if (regionInfo != null) { + this.tableName = regionInfo.getTableNameAsString(); + this.regionName = regionInfo.getEncodedName(); + } else { + this.tableName = SchemaMetrics.UNKNOWN; + this.regionName = SchemaMetrics.UNKNOWN; + } + this.regionMetrixPrefix = + SchemaMetrics.generateRegionMetricsPrefix(this.tableName, this.regionName); + this.exposeTimes = this.conf.getBoolean(CONF_KEY, true); + } else { + //Make all the final values happy. + this.conf = null; + this.tableName = null; + this.regionName = null; + this.regionMetrixPrefix = null; + this.exposeTimes = false; + } + } + + /** + * This is used in creating a testing HRegion where the regionInfo is unknown + * @param conf + */ + public OperationMetrics() { + this(null, null); + } + + /* + * This is used in set the read request count that is going to be exposed to + * hadoop metric framework. + * @param value absolute value of read account + */ + public void setReadRequestCountMetrics(long value) { + doSetNumericPersistentMetrics(READREQUESTCOUNT_KEY, value); + } + + /* + * This is used in set the read request count that is going to be exposed to + * hadoop metric framework. + * @param value absolute value of write account + */ + public void setWriteRequestCountMetrics(long value) { + doSetNumericPersistentMetrics(WRITEREQUESTCOUNT_KEY, value); + } + + private void doSetNumericPersistentMetrics(String key, long value) { + RegionMetricsStorage.setNumericPersistentMetric(this.regionMetrixPrefix+key, value); + } + + /** + * Update the stats associated with {@link HTable#put(java.util.List)}. + * + * @param columnFamilies Set of CF's this multiput is associated with + * @param value the time + */ + public void updateMultiPutMetrics(Set columnFamilies, long value) { + doUpdateTimeVarying(columnFamilies, MULTIPUT_KEY, value); + } + + /** + * Update the stats associated with {@link HTable#delete(java.util.List)}. + * + * @param columnFamilies Set of CF's this multidelete is associated with + * @param value the time + */ + public void updateMultiDeleteMetrics(Set columnFamilies, long value) { + doUpdateTimeVarying(columnFamilies, MULTIDELETE_KEY, value); + } + + /** + * Update the metrics associated with a {@link Get} + * + * @param columnFamilies + * Set of Column Families in this get. + * @param value + * the time + */ + public void updateGetMetrics(Set columnFamilies, long value) { + doUpdateTimeVarying(columnFamilies, GET_KEY, value); + } + + /** + * Update metrics associated with an {@link Increment} + * @param columnFamilies + * @param value + */ + public void updateIncrementMetrics(Set columnFamilies, long value) { + doUpdateTimeVarying(columnFamilies, INCREMENT_KEY, value); + } + + + /** + * Update the metrics associated with an {@link Append} + * @param columnFamilies + * @param value + */ + public void updateAppendMetrics(Set columnFamilies, long value) { + doUpdateTimeVarying(columnFamilies, APPEND_KEY, value); + } + + + /** + * Update the metrics associated with + * {@link HTable#incrementColumnValue(byte[], byte[], byte[], long)} + * + * @param columnFamily + * The single column family associated with an ICV + * @param value + * the time + */ + public void updateIncrementColumnValueMetrics(byte[] columnFamily, long value) { + String cfMetricPrefix = + SchemaMetrics.generateSchemaMetricsPrefix(this.tableName, Bytes.toString(columnFamily)); + doSafeIncTimeVarying(cfMetricPrefix, ICV_KEY, value); + doSafeIncTimeVarying(this.regionMetrixPrefix, ICV_KEY, value); + } + + /** + * update metrics associated with a {@link Put} + * + * @param columnFamilies + * Set of column families involved. + * @param value + * the time. + */ + public void updatePutMetrics(Set columnFamilies, long value) { + doUpdateTimeVarying(columnFamilies, PUT_KEY, value); + } + + /** + * update metrics associated with a {@link Delete} + * + * @param columnFamilies + * @param value + * the time. + */ + public void updateDeleteMetrics(Set columnFamilies, long value) { + doUpdateTimeVarying(columnFamilies, DELETE_KEY, value); + } + + + + /** + * This deletes all old non-persistent metrics this instance has ever created or updated. + * for persistent metrics, only delete for the region to be closed + * @param regionEncodedName the region that is to be closed + */ + public void closeMetrics(String regionEncodedName) { + RegionMetricsStorage.clear(regionEncodedName); + } + + /** + * Method to send updates for cf and region metrics. This is the normal method + * used if the naming of stats and CF's are in line with put/delete/multiput. + * + * @param columnFamilies + * the set of column families involved. + * @param key + * the metric name. + * @param value + * the time. + */ + private void doUpdateTimeVarying(Set columnFamilies, String key, long value) { + String cfPrefix = null; + if (columnFamilies != null) { + cfPrefix = SchemaMetrics.generateSchemaMetricsPrefix(tableName, columnFamilies); + } else { + cfPrefix = SchemaMetrics.generateSchemaMetricsPrefix(tableName, SchemaMetrics.UNKNOWN); + } + + doSafeIncTimeVarying(cfPrefix, key, value); + doSafeIncTimeVarying(this.regionMetrixPrefix, key, value); + } + + private void doSafeIncTimeVarying(String prefix, String key, long value) { + if (exposeTimes) { + if (prefix != null && !prefix.isEmpty() && key != null && !key.isEmpty()) { + String m = prefix + key; + RegionMetricsStorage.incrTimeVaryingMetric(m, value); + } + } + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java new file mode 100644 index 0000000..6e65cb4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java @@ -0,0 +1,147 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hadoop.hbase.util.Pair; + +/** + * This class if for maintaining the maps used to power metrics for hfiles, + * regions, and regionservers. It has methods to mutate and get state of metrics + * numbers. These numbers are exposed to Hadoop metrics through + * RegionServerDynamicMetrics. + */ +public class RegionMetricsStorage { + + // for simple numeric metrics (# of blocks read from block cache) + private static final ConcurrentMap numericMetrics = + new ConcurrentHashMap(); + + // for simple numeric metrics (current block cache size) + // These ones are not reset to zero when queried, unlike the previous. + private static final ConcurrentMap numericPersistentMetrics = + new ConcurrentHashMap(); + + /** + * Used for metrics where we want track a metrics (such as latency) over a + * number of operations. + */ + private static final ConcurrentMap> timeVaryingMetrics = + new ConcurrentHashMap>(); + + public static Map getNumericMetrics() { + return numericMetrics; + } + + public static Map getNumericPersistentMetrics() { + return numericPersistentMetrics; + } + + public static Map> getTimeVaryingMetrics() { + return timeVaryingMetrics; + } + + public static void incrNumericMetric(String key, long amount) { + AtomicLong oldVal = numericMetrics.get(key); + if (oldVal == null) { + oldVal = numericMetrics.putIfAbsent(key, new AtomicLong(amount)); + if (oldVal == null) + return; + } + oldVal.addAndGet(amount); + } + + public static void incrTimeVaryingMetric(String key, long amount) { + Pair oldVal = timeVaryingMetrics.get(key); + if (oldVal == null) { + oldVal = + timeVaryingMetrics.putIfAbsent(key, + new Pair( + new AtomicLong(amount), + new AtomicInteger(1))); + if (oldVal == null) + return; + } + oldVal.getFirst().addAndGet(amount); // total time + oldVal.getSecond().incrementAndGet(); // increment ops by 1 + } + + public static void setNumericPersistentMetric(String key, long amount) { + numericPersistentMetrics.put(key, new AtomicLong(amount)); + } + public static void incrNumericPersistentMetric(String key, long amount) { + AtomicLong oldVal = numericPersistentMetrics.get(key); + if (oldVal == null) { + oldVal = numericPersistentMetrics.putIfAbsent(key, new AtomicLong(amount)); + if (oldVal == null) + return; + } + oldVal.addAndGet(amount); + } + + public static void setNumericMetric(String key, long amount) { + numericMetrics.put(key, new AtomicLong(amount)); + } + + public static long getNumericMetric(String key) { + AtomicLong m = numericMetrics.get(key); + if (m == null) + return 0; + return m.get(); + } + + public static Pair getTimeVaryingMetric(String key) { + Pair pair = timeVaryingMetrics.get(key); + if (pair == null) { + return new Pair(0L, 0); + } + + return new Pair(pair.getFirst().get(), pair.getSecond().get()); + } + + public static long getNumericPersistentMetric(String key) { + AtomicLong m = numericPersistentMetrics.get(key); + if (m == null) + return 0; + return m.get(); + } + + /** + * Clear the timevarying and numeric metrics for all regions in this region server + * Clear the numericPersistentMerics for only the region being closed. + */ + public static void clear(String regionEncodedName) { + timeVaryingMetrics.clear(); + numericMetrics.clear(); + for (Entry entry : RegionMetricsStorage.getNumericPersistentMetrics().entrySet()) { + if (entry.getKey().contains(regionEncodedName)) + { + String keyName = entry.getKey(); + numericPersistentMetrics.remove(keyName); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java new file mode 100644 index 0000000..cd76211 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java @@ -0,0 +1,238 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.metrics.MetricsContext; +import org.apache.hadoop.metrics.MetricsRecord; +import org.apache.hadoop.metrics.MetricsUtil; +import org.apache.hadoop.metrics.Updater; +import org.apache.hadoop.metrics.util.MetricsBase; +import org.apache.hadoop.metrics.util.MetricsLongValue; +import org.apache.hadoop.metrics.util.MetricsRegistry; +import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate; + +/** + * + * This class is for maintaining the various RPC statistics + * and publishing them through the metrics interfaces. + * This also registers the JMX MBean for RPC. + *

+ * This class has a number of metrics variables that are publicly accessible; + * these variables (objects) have methods to update their values; + * for example: rpcQueueTime.inc(time) + * + */ +public class RegionServerDynamicMetrics implements Updater { + private static final String UNABLE_TO_CLEAR = "Unable to clear RegionServerDynamicMetrics"; + + private MetricsRecord metricsRecord; + private MetricsContext context; + private final RegionServerDynamicStatistics rsDynamicStatistics; + private Method updateMbeanInfoIfMetricsListChanged = null; + private HRegionServer regionServer; + private static final Log LOG = + LogFactory.getLog(RegionServerDynamicStatistics.class); + + private boolean reflectionInitialized = false; + private boolean needsUpdateMessage = false; + private Field recordMetricMapField; + private Field registryMetricMapField; + + /** + * The metrics variables are public: + * - they can be set directly by calling their set/inc methods + * -they can also be read directly - e.g. JMX does this. + */ + public final MetricsRegistry registry = new MetricsRegistry(); + + private RegionServerDynamicMetrics(HRegionServer regionServer) { + this.context = MetricsUtil.getContext("hbase"); + this.metricsRecord = MetricsUtil.createRecord( + this.context, + "RegionServerDynamicStatistics"); + context.registerUpdater(this); + this.rsDynamicStatistics = new RegionServerDynamicStatistics(this.registry); + this.regionServer = regionServer; + try { + updateMbeanInfoIfMetricsListChanged = + this.rsDynamicStatistics.getClass().getSuperclass() + .getDeclaredMethod("updateMbeanInfoIfMetricsListChanged", + new Class[]{}); + updateMbeanInfoIfMetricsListChanged.setAccessible(true); + } catch (Exception e) { + LOG.error(e); + } + } + + public static RegionServerDynamicMetrics newInstance(HRegionServer regionServer) { + RegionServerDynamicMetrics metrics = + new RegionServerDynamicMetrics(regionServer); + return metrics; + } + + public synchronized void setNumericMetric(String name, long amt) { + MetricsLongValue m = (MetricsLongValue)registry.get(name); + if (m == null) { + m = new MetricsLongValue(name, this.registry); + this.needsUpdateMessage = true; + } + m.set(amt); + } + + public synchronized void incrTimeVaryingMetric( + String name, + long amt, + int numOps) { + MetricsTimeVaryingRate m = (MetricsTimeVaryingRate)registry.get(name); + if (m == null) { + m = new MetricsTimeVaryingRate(name, this.registry); + this.needsUpdateMessage = true; + } + if (numOps > 0) { + m.inc(numOps, amt); + } + } + + /** + * Clear all metrics this exposes. + * Uses reflection to clear them from hadoop metrics side as well. + */ + @SuppressWarnings("rawtypes") + public void clear() { + this.needsUpdateMessage = true; + // If this is the first clear use reflection to get the two maps that hold copies of our + // metrics on the hadoop metrics side. We have to use reflection because there is not + // remove metrics on the hadoop side. If we can't get them then clearing old metrics + // is not possible and bailing out early is our best option. + if (!this.reflectionInitialized) { + this.reflectionInitialized = true; + try { + this.recordMetricMapField = this.metricsRecord.getClass().getDeclaredField("metricTable"); + this.recordMetricMapField.setAccessible(true); + } catch (SecurityException e) { + LOG.debug(UNABLE_TO_CLEAR); + return; + } catch (NoSuchFieldException e) { + LOG.debug(UNABLE_TO_CLEAR); + return; + } + + try { + this.registryMetricMapField = this.registry.getClass().getDeclaredField("metricsList"); + this.registryMetricMapField.setAccessible(true); + } catch (SecurityException e) { + LOG.debug(UNABLE_TO_CLEAR); + return; + } catch (NoSuchFieldException e) { + LOG.debug(UNABLE_TO_CLEAR); + return; + } + } + + + //If we found both fields then try and clear the maps. + if (this.recordMetricMapField != null && this.registryMetricMapField != null) { + try { + Map recordMap = (Map) this.recordMetricMapField.get(this.metricsRecord); + recordMap.clear(); + Map registryMap = (Map) this.registryMetricMapField.get(this.registry); + registryMap.clear(); + } catch (IllegalArgumentException e) { + LOG.debug(UNABLE_TO_CLEAR); + } catch (IllegalAccessException e) { + LOG.debug(UNABLE_TO_CLEAR); + } + } else { + LOG.debug(UNABLE_TO_CLEAR); + } + } + + /** + * Push the metrics to the monitoring subsystem on doUpdate() call. + * @param context ctx + */ + public void doUpdates(MetricsContext context) { + /* get dynamically created numeric metrics, and push the metrics */ + for (Entry entry : RegionMetricsStorage.getNumericMetrics().entrySet()) { + this.setNumericMetric(entry.getKey(), entry.getValue().getAndSet(0)); + } + + /* export estimated size of all response queues */ + if (regionServer != null) { + long responseQueueSize = regionServer.getResponseQueueSize(); + this.setNumericMetric("responseQueuesSize", responseQueueSize); + } + + /* get dynamically created numeric metrics, and push the metrics. + * These ones aren't to be reset; they are cumulative. */ + for (Entry entry : RegionMetricsStorage.getNumericPersistentMetrics().entrySet()) { + this.setNumericMetric(entry.getKey(), entry.getValue().get()); + } + /* get dynamically created time varying metrics, and push the metrics */ + for (Entry> entry : + RegionMetricsStorage.getTimeVaryingMetrics().entrySet()) { + Pair value = entry.getValue(); + this.incrTimeVaryingMetric(entry.getKey(), + value.getFirst().getAndSet(0), + value.getSecond().getAndSet(0)); + } + + // If there are new metrics sending this message to jmx tells it to update everything. + // This is not ideal we should just move to metrics2 that has full support for dynamic metrics. + if (needsUpdateMessage) { + try { + if (updateMbeanInfoIfMetricsListChanged != null) { + updateMbeanInfoIfMetricsListChanged.invoke(this.rsDynamicStatistics, + new Object[]{}); + } + } catch (Exception e) { + LOG.error(e); + } + needsUpdateMessage = false; + } + + + synchronized (registry) { + // Iterate through the registry to propagate the different rpc metrics. + for (String metricName : registry.getKeyList() ) { + MetricsBase value = registry.get(metricName); + value.pushMetric(metricsRecord); + } + } + metricsRecord.update(); + } + + public void shutdown() { + if (rsDynamicStatistics != null) + rsDynamicStatistics.shutdown(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java new file mode 100644 index 0000000..ea96cf5 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java @@ -0,0 +1,47 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.metrics.util.MBeanUtil; +import org.apache.hadoop.metrics.util.MetricsDynamicMBeanBase; +import org.apache.hadoop.metrics.util.MetricsRegistry; + +import javax.management.ObjectName; + +/** + * Exports dynamic region server metric recorded in + * {@link RegionServerDynamicMetrics} as an MBean + * for JMX monitoring. + */ +public class RegionServerDynamicStatistics extends MetricsDynamicMBeanBase { + private final ObjectName mbeanName; + + public RegionServerDynamicStatistics(MetricsRegistry registry) { + super(registry, "RegionServerDynamicStatistics"); + mbeanName = MBeanUtil.registerMBean("RegionServer", "RegionServerDynamicStatistics", this); + } + + public void shutdown() { + if (mbeanName != null) + MBeanUtil.unregisterMBean(mbeanName); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java new file mode 100644 index 0000000..e6590f5 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java @@ -0,0 +1,613 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.metrics; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryUsage; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.metrics.HBaseInfo; +import org.apache.hadoop.hbase.metrics.MetricsRate; +import org.apache.hadoop.hbase.metrics.PersistentMetricsTimeVaryingRate; +import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram; +import com.yammer.metrics.stats.Snapshot; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.Strings; +import org.apache.hadoop.metrics.ContextFactory; +import org.apache.hadoop.metrics.MetricsContext; +import org.apache.hadoop.metrics.MetricsRecord; +import org.apache.hadoop.metrics.MetricsUtil; +import org.apache.hadoop.metrics.Updater; +import org.apache.hadoop.metrics.jvm.JvmMetrics; +import org.apache.hadoop.metrics.util.MetricsIntValue; +import org.apache.hadoop.metrics.util.MetricsLongValue; +import org.apache.hadoop.metrics.util.MetricsRegistry; +import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate; +import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong; +import org.apache.hadoop.util.StringUtils; + +/** + * This class is for maintaining the various regionserver statistics + * and publishing them through the metrics interfaces. + *

+ * This class has a number of metrics variables that are publicly accessible; + * these variables (objects) have methods to update their values. + */ +public class RegionServerMetrics implements Updater { + @SuppressWarnings({"FieldCanBeLocal"}) + private final Log LOG = LogFactory.getLog(this.getClass()); + private final MetricsRecord metricsRecord; + private long lastUpdate = System.currentTimeMillis(); + private long lastExtUpdate = System.currentTimeMillis(); + private long extendedPeriod = 0; + private static final int MB = 1024*1024; + private MetricsRegistry registry = new MetricsRegistry(); + private final RegionServerStatistics statistics; + + public final MetricsTimeVaryingRate atomicIncrementTime = + new MetricsTimeVaryingRate("atomicIncrementTime", registry); + + /** + * Count of regions carried by this regionserver + */ + public final MetricsIntValue regions = + new MetricsIntValue("regions", registry); + + /** + * Block cache size. + */ + public final MetricsLongValue blockCacheSize = + new MetricsLongValue("blockCacheSize", registry); + + /** + * Block cache free size. + */ + public final MetricsLongValue blockCacheFree = + new MetricsLongValue("blockCacheFree", registry); + + /** + * Block cache item count. + */ + public final MetricsLongValue blockCacheCount = + new MetricsLongValue("blockCacheCount", registry); + + /** + * Block cache hit count. + */ + public final MetricsLongValue blockCacheHitCount = + new MetricsLongValue("blockCacheHitCount", registry); + + /** + * Block cache miss count. + */ + public final MetricsLongValue blockCacheMissCount = + new MetricsLongValue("blockCacheMissCount", registry); + + /** + * Block cache evict count. + */ + public final MetricsLongValue blockCacheEvictedCount = + new MetricsLongValue("blockCacheEvictedCount", registry); + + /** + * Block hit ratio. + */ + public final MetricsIntValue blockCacheHitRatio = + new MetricsIntValue("blockCacheHitRatio", registry); + + /** + * Block hit caching ratio. This only includes the requests to the block + * cache where caching was turned on. See HBASE-2253. + */ + public final MetricsIntValue blockCacheHitCachingRatio = + new MetricsIntValue("blockCacheHitCachingRatio", registry); + + /** Block hit ratio for past N periods. */ + public final MetricsIntValue blockCacheHitRatioPastNPeriods = new MetricsIntValue("blockCacheHitRatioPastNPeriods", registry); + + /** Block hit caching ratio for past N periods */ + public final MetricsIntValue blockCacheHitCachingRatioPastNPeriods = new MetricsIntValue("blockCacheHitCachingRatioPastNPeriods", registry); + + /* + * Count of requests to the regionservers since last call to metrics update + */ + public final MetricsRate requests = new MetricsRate("requests", registry); + + /** + * Count of stores open on the regionserver. + */ + public final MetricsIntValue stores = new MetricsIntValue("stores", registry); + + /** + * Count of storefiles open on the regionserver. + */ + public final MetricsIntValue storefiles = + new MetricsIntValue("storefiles", registry); + + /** + * Count of read requests + */ + public final MetricsLongValue readRequestsCount = + new MetricsLongValue("readRequestsCount", registry); + + /** + * Count of write requests + */ + public final MetricsLongValue writeRequestsCount = + new MetricsLongValue("writeRequestsCount", registry); + + /** + */ + public final MetricsIntValue storefileIndexSizeMB = + new MetricsIntValue("storefileIndexSizeMB", registry); + + /** The total size of block index root levels in this regionserver in KB. */ + public final MetricsIntValue rootIndexSizeKB = + new MetricsIntValue("rootIndexSizeKB", registry); + + /** Total size of all block indexes (not necessarily loaded in memory) */ + public final MetricsIntValue totalStaticIndexSizeKB = + new MetricsIntValue("totalStaticIndexSizeKB", registry); + + /** Total size of all Bloom filters (not necessarily loaded in memory) */ + public final MetricsIntValue totalStaticBloomSizeKB = + new MetricsIntValue("totalStaticBloomSizeKB", registry); + + /** + * HDFS blocks locality index + */ + public final MetricsIntValue hdfsBlocksLocalityIndex = + new MetricsIntValue("hdfsBlocksLocalityIndex", registry); + + /** + * Sum of all the memstore sizes in this regionserver in MB + */ + public final MetricsIntValue memstoreSizeMB = + new MetricsIntValue("memstoreSizeMB", registry); + + /** + * Number of put with WAL disabled in this regionserver in MB + */ + public final MetricsLongValue numPutsWithoutWAL = + new MetricsLongValue("numPutsWithoutWAL", registry); + + /** + * Possible data loss sizes (due to put with WAL disabled) in this regionserver in MB + */ + public final MetricsIntValue mbInMemoryWithoutWAL = + new MetricsIntValue("mbInMemoryWithoutWAL", registry); + + /** + * Size of the compaction queue. + */ + public final MetricsIntValue compactionQueueSize = + new MetricsIntValue("compactionQueueSize", registry); + + /** + * Size of the flush queue. + */ + public final MetricsIntValue flushQueueSize = + new MetricsIntValue("flushQueueSize", registry); + + /** + * filesystem sequential read latency distribution + */ + public final MetricsHistogram fsReadLatencyHistogram = + new MetricsHistogram("fsReadLatencyHistogram", registry); + + /** + * filesystem pread latency distribution + */ + public final MetricsHistogram fsPreadLatencyHistogram = + new MetricsHistogram("fsPreadLatencyHistogram", registry); + + /** + * Metrics on the distribution of filesystem write latencies (improved version of fsWriteLatency) + */ + public final MetricsHistogram fsWriteLatencyHistogram = + new MetricsHistogram("fsWriteLatencyHistogram", registry); + + + /** + * filesystem read latency + */ + public final MetricsTimeVaryingRate fsReadLatency = + new MetricsTimeVaryingRate("fsReadLatency", registry); + + /** + * filesystem positional read latency + */ + public final MetricsTimeVaryingRate fsPreadLatency = + new MetricsTimeVaryingRate("fsPreadLatency", registry); + + /** + * filesystem write latency + */ + public final MetricsTimeVaryingRate fsWriteLatency = + new MetricsTimeVaryingRate("fsWriteLatency", registry); + + /** + * size (in bytes) of data in HLog append calls + */ + public final MetricsTimeVaryingRate fsWriteSize = + new MetricsTimeVaryingRate("fsWriteSize", registry); + + /** + * filesystem sync latency + */ + public final MetricsTimeVaryingRate fsSyncLatency = + new MetricsTimeVaryingRate("fsSyncLatency", registry); + + + /** + * time each scheduled compaction takes + */ + protected final PersistentMetricsTimeVaryingRate compactionTime = + new PersistentMetricsTimeVaryingRate("compactionTime", registry); + + protected final PersistentMetricsTimeVaryingRate compactionSize = + new PersistentMetricsTimeVaryingRate("compactionSize", registry); + + /** + * time each scheduled flush takes + */ + protected final PersistentMetricsTimeVaryingRate flushTime = + new PersistentMetricsTimeVaryingRate("flushTime", registry); + + protected final PersistentMetricsTimeVaryingRate flushSize = + new PersistentMetricsTimeVaryingRate("flushSize", registry); + + public final MetricsLongValue slowHLogAppendCount = + new MetricsLongValue("slowHLogAppendCount", registry); + + public final MetricsTimeVaryingRate slowHLogAppendTime = + new MetricsTimeVaryingRate("slowHLogAppendTime", registry); + + public final MetricsTimeVaryingLong regionSplitSuccessCount = + new MetricsTimeVaryingLong("regionSplitSuccessCount", registry); + + public final MetricsTimeVaryingLong regionSplitFailureCount = + new MetricsTimeVaryingLong("regionSplitFailureCount", registry); + + /** + * Number of times checksum verification failed. + */ + public final MetricsLongValue checksumFailuresCount = + new MetricsLongValue("checksumFailuresCount", registry); + + /** + * time blocked on lack of resources + */ + public final MetricsHistogram updatesBlockedSeconds = new MetricsHistogram( + "updatesBlockedSeconds", registry); + + /** + * time blocked on memstoreHW + */ + public final MetricsHistogram updatesBlockedSecondsHighWater = new MetricsHistogram( + "updatesBlockedSecondsHighWater",registry); + + public RegionServerMetrics() { + MetricsContext context = MetricsUtil.getContext("hbase"); + metricsRecord = MetricsUtil.createRecord(context, "regionserver"); + String name = Thread.currentThread().getName(); + metricsRecord.setTag("RegionServer", name); + context.registerUpdater(this); + // Add jvmmetrics. + JvmMetrics.init("RegionServer", name); + // Add Hbase Info metrics + HBaseInfo.init(); + + // export for JMX + statistics = new RegionServerStatistics(this.registry, name); + + // get custom attributes + try { + Object m = ContextFactory.getFactory().getAttribute("hbase.extendedperiod"); + if (m instanceof String) { + this.extendedPeriod = Long.parseLong((String) m)*1000; + } + } catch (IOException ioe) { + LOG.info("Couldn't load ContextFactory for Metrics config info"); + } + + LOG.info("Initialized"); + } + + public void shutdown() { + if (statistics != null) + statistics.shutdown(); + } + + /** + * Since this object is a registered updater, this method will be called + * periodically, e.g. every 5 seconds. + * @param caller the metrics context that this responsible for calling us + */ + public void doUpdates(MetricsContext caller) { + synchronized (this) { + this.lastUpdate = System.currentTimeMillis(); + + // has the extended period for long-living stats elapsed? + if (this.extendedPeriod > 0 && + this.lastUpdate - this.lastExtUpdate >= this.extendedPeriod) { + this.lastExtUpdate = this.lastUpdate; + this.compactionTime.resetMinMaxAvg(); + this.compactionSize.resetMinMaxAvg(); + this.flushTime.resetMinMaxAvg(); + this.flushSize.resetMinMaxAvg(); + this.resetAllMinMax(); + } + + this.stores.pushMetric(this.metricsRecord); + this.storefiles.pushMetric(this.metricsRecord); + this.storefileIndexSizeMB.pushMetric(this.metricsRecord); + this.rootIndexSizeKB.pushMetric(this.metricsRecord); + this.totalStaticIndexSizeKB.pushMetric(this.metricsRecord); + this.totalStaticBloomSizeKB.pushMetric(this.metricsRecord); + this.memstoreSizeMB.pushMetric(this.metricsRecord); + this.mbInMemoryWithoutWAL.pushMetric(this.metricsRecord); + this.numPutsWithoutWAL.pushMetric(this.metricsRecord); + this.readRequestsCount.pushMetric(this.metricsRecord); + this.writeRequestsCount.pushMetric(this.metricsRecord); + this.regions.pushMetric(this.metricsRecord); + this.requests.pushMetric(this.metricsRecord); + this.compactionQueueSize.pushMetric(this.metricsRecord); + this.flushQueueSize.pushMetric(this.metricsRecord); + this.blockCacheSize.pushMetric(this.metricsRecord); + this.blockCacheFree.pushMetric(this.metricsRecord); + this.blockCacheCount.pushMetric(this.metricsRecord); + this.blockCacheHitCount.pushMetric(this.metricsRecord); + this.blockCacheMissCount.pushMetric(this.metricsRecord); + this.blockCacheEvictedCount.pushMetric(this.metricsRecord); + this.blockCacheHitRatio.pushMetric(this.metricsRecord); + this.blockCacheHitCachingRatio.pushMetric(this.metricsRecord); + this.hdfsBlocksLocalityIndex.pushMetric(this.metricsRecord); + this.blockCacheHitRatioPastNPeriods.pushMetric(this.metricsRecord); + this.blockCacheHitCachingRatioPastNPeriods.pushMetric(this.metricsRecord); + + // Mix in HFile and HLog metrics + // Be careful. Here is code for MTVR from up in hadoop: + // public synchronized void inc(final int numOps, final long time) { + // currentData.numOperations += numOps; + // currentData.time += time; + // long timePerOps = time/numOps; + // minMax.update(timePerOps); + // } + // Means you can't pass a numOps of zero or get a ArithmeticException / by zero. + // HLog metrics + addHLogMetric(HLog.getWriteTime(), this.fsWriteLatency); + addHLogMetric(HLog.getWriteSize(), this.fsWriteSize); + addHLogMetric(HLog.getSyncTime(), this.fsSyncLatency); + addHLogMetric(HLog.getSlowAppendTime(), this.slowHLogAppendTime); + this.slowHLogAppendCount.set(HLog.getSlowAppendCount()); + // HFile metrics, sequential reads + int ops = HFile.getReadOps(); + if (ops != 0) this.fsReadLatency.inc(ops, HFile.getReadTimeMs()); + // HFile metrics, positional reads + ops = HFile.getPreadOps(); + if (ops != 0) this.fsPreadLatency.inc(ops, HFile.getPreadTimeMs()); + this.checksumFailuresCount.set(HFile.getChecksumFailuresCount()); + + /* NOTE: removed HFile write latency. 2 reasons: + * 1) Mixing HLog latencies are far higher priority since they're + * on-demand and HFile is used in background (compact/flush) + * 2) HFile metrics are being handled at a higher level + * by compaction & flush metrics. + */ + + for(Long latency : HFile.getReadLatenciesNanos()) { + this.fsReadLatencyHistogram.update(latency); + } + for(Long latency : HFile.getPreadLatenciesNanos()) { + this.fsPreadLatencyHistogram.update(latency); + } + for(Long latency : HFile.getWriteLatenciesNanos()) { + this.fsWriteLatencyHistogram.update(latency); + } + + + // push the result + this.fsPreadLatency.pushMetric(this.metricsRecord); + this.fsReadLatency.pushMetric(this.metricsRecord); + this.fsWriteLatency.pushMetric(this.metricsRecord); + this.fsWriteSize.pushMetric(this.metricsRecord); + + this.fsReadLatencyHistogram.pushMetric(this.metricsRecord); + this.fsWriteLatencyHistogram.pushMetric(this.metricsRecord); + this.fsPreadLatencyHistogram.pushMetric(this.metricsRecord); + + this.fsSyncLatency.pushMetric(this.metricsRecord); + this.compactionTime.pushMetric(this.metricsRecord); + this.compactionSize.pushMetric(this.metricsRecord); + this.flushTime.pushMetric(this.metricsRecord); + this.flushSize.pushMetric(this.metricsRecord); + this.slowHLogAppendCount.pushMetric(this.metricsRecord); + this.regionSplitSuccessCount.pushMetric(this.metricsRecord); + this.regionSplitFailureCount.pushMetric(this.metricsRecord); + this.checksumFailuresCount.pushMetric(this.metricsRecord); + this.updatesBlockedSeconds.pushMetric(this.metricsRecord); + this.updatesBlockedSecondsHighWater.pushMetric(this.metricsRecord); + } + this.metricsRecord.update(); + } + + private void addHLogMetric(HLog.Metric logMetric, + MetricsTimeVaryingRate hadoopMetric) { + if (logMetric.count > 0) + hadoopMetric.inc(logMetric.min); + if (logMetric.count > 1) + hadoopMetric.inc(logMetric.max); + if (logMetric.count > 2) { + int ops = logMetric.count - 2; + hadoopMetric.inc(ops, logMetric.total - logMetric.max - logMetric.min); + } + } + + public void resetAllMinMax() { + this.atomicIncrementTime.resetMinMax(); + this.fsReadLatency.resetMinMax(); + this.fsWriteLatency.resetMinMax(); + this.fsWriteSize.resetMinMax(); + this.fsSyncLatency.resetMinMax(); + this.slowHLogAppendTime.resetMinMax(); + } + + /** + * @return Count of requests. + */ + public float getRequests() { + return this.requests.getPreviousIntervalValue(); + } + + /** + * @param time time that compaction took + * @param size bytesize of storefiles in the compaction + */ + public synchronized void addCompaction(long time, long size) { + this.compactionTime.inc(time); + this.compactionSize.inc(size); + } + + /** + * @param flushes history in + */ + public synchronized void addFlush(final List> flushes) { + for (Pair f : flushes) { + this.flushTime.inc(f.getFirst()); + this.flushSize.inc(f.getSecond()); + } + } + + /** + * @param inc How much to add to requests. + */ + public void incrementRequests(final int inc) { + this.requests.inc(inc); + } + + public void incrementSplitSuccessCount() { + this.regionSplitSuccessCount.inc(); + } + + public void incrementSplitFailureCount() { + this.regionSplitFailureCount.inc(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb = Strings.appendKeyValue(sb, "requestsPerSecond", Integer + .valueOf((int) this.requests.getPreviousIntervalValue())); + sb = Strings.appendKeyValue(sb, "numberOfOnlineRegions", + Integer.valueOf(this.regions.get())); + sb = Strings.appendKeyValue(sb, "numberOfStores", + Integer.valueOf(this.stores.get())); + sb = Strings.appendKeyValue(sb, "numberOfStorefiles", + Integer.valueOf(this.storefiles.get())); + sb = Strings.appendKeyValue(sb, this.storefileIndexSizeMB.getName(), + Integer.valueOf(this.storefileIndexSizeMB.get())); + sb = Strings.appendKeyValue(sb, "rootIndexSizeKB", + Integer.valueOf(this.rootIndexSizeKB.get())); + sb = Strings.appendKeyValue(sb, "totalStaticIndexSizeKB", + Integer.valueOf(this.totalStaticIndexSizeKB.get())); + sb = Strings.appendKeyValue(sb, "totalStaticBloomSizeKB", + Integer.valueOf(this.totalStaticBloomSizeKB.get())); + sb = Strings.appendKeyValue(sb, this.memstoreSizeMB.getName(), + Integer.valueOf(this.memstoreSizeMB.get())); + sb = Strings.appendKeyValue(sb, "mbInMemoryWithoutWAL", + Integer.valueOf(this.mbInMemoryWithoutWAL.get())); + sb = Strings.appendKeyValue(sb, "numberOfPutsWithoutWAL", + Long.valueOf(this.numPutsWithoutWAL.get())); + sb = Strings.appendKeyValue(sb, "readRequestsCount", + Long.valueOf(this.readRequestsCount.get())); + sb = Strings.appendKeyValue(sb, "writeRequestsCount", + Long.valueOf(this.writeRequestsCount.get())); + sb = Strings.appendKeyValue(sb, "compactionQueueSize", + Integer.valueOf(this.compactionQueueSize.get())); + sb = Strings.appendKeyValue(sb, "flushQueueSize", + Integer.valueOf(this.flushQueueSize.get())); + // Duplicate from jvmmetrics because metrics are private there so + // inaccessible. + MemoryUsage memory = + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + sb = Strings.appendKeyValue(sb, "usedHeapMB", + Long.valueOf(memory.getUsed()/MB)); + sb = Strings.appendKeyValue(sb, "maxHeapMB", + Long.valueOf(memory.getMax()/MB)); + sb = Strings.appendKeyValue(sb, this.blockCacheSize.getName()+"MB", + StringUtils.limitDecimalTo2((float)this.blockCacheSize.get()/MB)); + sb = Strings.appendKeyValue(sb, this.blockCacheFree.getName()+"MB", + StringUtils.limitDecimalTo2((float)this.blockCacheFree.get()/MB)); + sb = Strings.appendKeyValue(sb, this.blockCacheCount.getName(), + Long.valueOf(this.blockCacheCount.get())); + sb = Strings.appendKeyValue(sb, this.blockCacheHitCount.getName(), + Long.valueOf(this.blockCacheHitCount.get())); + sb = Strings.appendKeyValue(sb, this.blockCacheMissCount.getName(), + Long.valueOf(this.blockCacheMissCount.get())); + sb = Strings.appendKeyValue(sb, this.blockCacheEvictedCount.getName(), + Long.valueOf(this.blockCacheEvictedCount.get())); + sb = Strings.appendKeyValue(sb, this.blockCacheHitRatio.getName(), + Long.valueOf(this.blockCacheHitRatio.get())+"%"); + sb = Strings.appendKeyValue(sb, this.blockCacheHitCachingRatio.getName(), + Long.valueOf(this.blockCacheHitCachingRatio.get())+"%"); + sb = Strings.appendKeyValue(sb, this.hdfsBlocksLocalityIndex.getName(), + Long.valueOf(this.hdfsBlocksLocalityIndex.get())); + sb = Strings.appendKeyValue(sb, "slowHLogAppendCount", + Long.valueOf(this.slowHLogAppendCount.get())); + sb = appendHistogram(sb, this.fsReadLatencyHistogram); + sb = appendHistogram(sb, this.fsPreadLatencyHistogram); + sb = appendHistogram(sb, this.fsWriteLatencyHistogram); + + return sb.toString(); + } + + private StringBuilder appendHistogram(StringBuilder sb, + MetricsHistogram histogram) { + sb = Strings.appendKeyValue(sb, + histogram.getName() + "Mean", + StringUtils.limitDecimalTo2(histogram.getMean())); + sb = Strings.appendKeyValue(sb, + histogram.getName() + "Count", + StringUtils.limitDecimalTo2(histogram.getCount())); + final Snapshot s = histogram.getSnapshot(); + sb = Strings.appendKeyValue(sb, + histogram.getName() + "Median", + StringUtils.limitDecimalTo2(s.getMedian())); + sb = Strings.appendKeyValue(sb, + histogram.getName() + "75th", + StringUtils.limitDecimalTo2(s.get75thPercentile())); + sb = Strings.appendKeyValue(sb, + histogram.getName() + "95th", + StringUtils.limitDecimalTo2(s.get95thPercentile())); + sb = Strings.appendKeyValue(sb, + histogram.getName() + "99th", + StringUtils.limitDecimalTo2(s.get99thPercentile())); + sb = Strings.appendKeyValue(sb, + histogram.getName() + "999th", + StringUtils.limitDecimalTo2(s.get999thPercentile())); + return sb; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerStatistics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerStatistics.java new file mode 100644 index 0000000..04fe7b1 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerStatistics.java @@ -0,0 +1,47 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.hbase.metrics.MetricsMBeanBase; +import org.apache.hadoop.metrics.util.MBeanUtil; +import org.apache.hadoop.metrics.util.MetricsRegistry; + +import javax.management.ObjectName; + +/** + * Exports metrics recorded by {@link RegionServerMetrics} as an MBean + * for JMX monitoring. + */ +public class RegionServerStatistics extends MetricsMBeanBase { + + private final ObjectName mbeanName; + + public RegionServerStatistics(MetricsRegistry registry, String rsName) { + super(registry, "RegionServerStatistics"); + mbeanName = MBeanUtil.registerMBean("RegionServer", + "RegionServerStatistics", this); + } + + public void shutdown() { + if (mbeanName != null) + MBeanUtil.unregisterMBean(mbeanName); + } + +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java new file mode 100644 index 0000000..7493951 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.SchemaAware; +import org.apache.hadoop.hbase.util.ClassSize; + +/** + * A base class for objects that are associated with a particular table and + * column family. Provides a way to obtain the schema metrics object. + *

+ * Due to the variety of things that can be associated with a table/CF, there + * are many ways to initialize this base class, either in the constructor, or + * from another similar object. For example, an HFile reader configures HFile + * blocks it reads with its own table/CF name. + */ +public class SchemaConfigured implements HeapSize, SchemaAware { + private static final Log LOG = LogFactory.getLog(SchemaConfigured.class); + + // These are not final because we set them at runtime e.g. for HFile blocks. + private String cfName; + private String tableName; + + /** + * Schema metrics. Can only be initialized when we know our column family + * name, table name, and have had a chance to take a look at the + * configuration (in {@link SchemaMetrics#configureGlobally(Configuration)) + * so we know whether we are using per-table metrics. Therefore, initialized + * lazily. We don't make this volatile because even if a thread sees a stale + * value of null, it will be re-initialized to the same value that other + * threads see. + */ + private SchemaMetrics schemaMetrics; + + static { + if (ClassSize.OBJECT <= 0 || ClassSize.REFERENCE <= 0) { + throw new AssertionError("Class sizes are not initialized"); + } + } + + /** + * Estimated heap size of this object. We don't count table name and column + * family name characters because these strings are shared among many + * objects. We need unaligned size to reuse this in subclasses. + */ + public static final int SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE = + ClassSize.OBJECT + 3 * ClassSize.REFERENCE; + + private static final int SCHEMA_CONFIGURED_ALIGNED_HEAP_SIZE = + ClassSize.align(SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE); + + /** A helper constructor that configures the "use table name" flag. */ + private SchemaConfigured(Configuration conf) { + SchemaMetrics.configureGlobally(conf); + // Even though we now know if table-level metrics are used, we can't + // initialize schemaMetrics yet, because CF and table name are only known + // to the calling constructor. + } + + /** + * Creates an instance corresponding to an unknown table and column family. + * Used in unit tests. + */ + public static SchemaConfigured createUnknown() { + return new SchemaConfigured(null, SchemaMetrics.UNKNOWN, + SchemaMetrics.UNKNOWN); + } + + /** + * Default constructor. Only use when column/family name are not known at + * construction (i.e. for HFile blocks). + */ + public SchemaConfigured() { + } + + /** + * Initialize table and column family name from an HFile path. If + * configuration is null, + * {@link SchemaMetrics#configureGlobally(Configuration)} should have been + * called already. + */ + public SchemaConfigured(Configuration conf, Path path) { + this(conf); + + if (path != null) { + String splits[] = path.toString().split("/"); + int numPathLevels = splits.length; + if (numPathLevels > 0 && splits[0].isEmpty()) { + // This path starts with an '/'. + --numPathLevels; + } + if (numPathLevels < HFile.MIN_NUM_HFILE_PATH_LEVELS) { + LOG.warn("Could not determine table and column family of the HFile " + + "path " + path + ". Expecting at least " + + HFile.MIN_NUM_HFILE_PATH_LEVELS + " path components."); + path = null; + } else { + cfName = splits[splits.length - 2]; + if (cfName.equals(HRegion.REGION_TEMP_SUBDIR)) { + // This is probably a compaction or flush output file. We will set + // the real CF name later. + cfName = null; + } else { + cfName = cfName.intern(); + } + tableName = splits[splits.length - 4].intern(); + return; + } + } + + // This might also happen if we were passed an incorrect path. + cfName = SchemaMetrics.UNKNOWN; + tableName = SchemaMetrics.UNKNOWN; + } + + /** + * Used when we know an HFile path to deduce table and CF name from, but do + * not have a configuration. + * @param path an HFile path + */ + public SchemaConfigured(Path path) { + this(null, path); + } + + /** + * Used when we know table and column family name. If configuration is null, + * {@link SchemaMetrics#configureGlobally(Configuration)} should have been + * called already. + */ + public SchemaConfigured(Configuration conf, String tableName, String cfName) + { + this(conf); + this.tableName = tableName != null ? tableName.intern() : tableName; + this.cfName = cfName != null ? cfName.intern() : cfName; + } + + public SchemaConfigured(SchemaAware that) { + tableName = that.getTableName().intern(); + cfName = that.getColumnFamilyName().intern(); + schemaMetrics = that.getSchemaMetrics(); + } + + @Override + public String getTableName() { + return tableName; + } + + @Override + public String getColumnFamilyName() { + return cfName; + } + + @Override + public SchemaMetrics getSchemaMetrics() { + if (schemaMetrics == null) { + if (tableName == null || cfName == null) { + throw new IllegalStateException("Schema metrics requested before " + + "table/CF name initialization: " + schemaConfAsJSON()); + } + schemaMetrics = SchemaMetrics.getInstance(tableName, cfName); + } + return schemaMetrics; + } + + /** + * Configures the given object (e.g. an HFile block) with the current table + * and column family name, and the associated collection of metrics. Please + * note that this method configures the other object, not this + * object. + */ + public void passSchemaMetricsTo(SchemaConfigured target) { + if (isNull()) { + resetSchemaMetricsConf(target); + return; + } + + if (!isSchemaConfigured()) { + // Cannot configure another object if we are not configured ourselves. + throw new IllegalStateException("Table name/CF not initialized: " + + schemaConfAsJSON()); + } + + if (conflictingWith(target)) { + // Make sure we don't try to change table or CF name. + throw new IllegalArgumentException("Trying to change table name to \"" + + tableName + "\", CF name to \"" + cfName + "\" from " + + target.schemaConfAsJSON()); + } + + target.tableName = tableName.intern(); + target.cfName = cfName.intern(); + target.schemaMetrics = schemaMetrics; + target.schemaConfigurationChanged(); + } + + /** + * Reset schema metrics configuration in this particular instance. Used when + * legitimately need to re-initialize the object with another table/CF. + * This is a static method because its use is discouraged and reserved for + * when it is really necessary (e.g. writing HFiles in a temp direcdtory + * on compaction). + */ + public static void resetSchemaMetricsConf(SchemaConfigured target) { + target.tableName = null; + target.cfName = null; + target.schemaMetrics = null; + target.schemaConfigurationChanged(); + } + + @Override + public long heapSize() { + return SCHEMA_CONFIGURED_ALIGNED_HEAP_SIZE; + } + + public String schemaConfAsJSON() { + return "{\"tableName\":\"" + tableName + "\",\"cfName\":\"" + cfName + + "\"}"; + } + + protected boolean isSchemaConfigured() { + return tableName != null && cfName != null; + } + + private boolean isNull() { + return tableName == null && cfName == null && schemaMetrics == null; + } + + /** + * Determines if the current object's table/CF settings are not in conflict + * with the other object's table and CF. If the other object's table/CF are + * undefined, they are not considered to be in conflict. Used to sanity-check + * configuring the other object with this object's table/CF. + */ + boolean conflictingWith(SchemaConfigured other) { + return (other.tableName != null && !tableName.equals(other.tableName)) || + (other.cfName != null && !cfName.equals(other.cfName)); + } + + /** + * A hook method called when schema configuration changes. Can be used to + * update schema-aware member fields. + */ + protected void schemaConfigurationChanged() { + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java new file mode 100644 index 0000000..62d3f8e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java @@ -0,0 +1,989 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLongArray; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang.mutable.MutableDouble; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +/** + * A collection of metric names in a given column family or a (table, column + * family) combination. The following "dimensions" are supported: + *

    + *
  • Table name (optional; enabled based on configuration)
    • + *
  • Per-column family vs. aggregated. The aggregated mode is only supported + * when table name is not included.
    • + *
  • Block category (data, index, bloom filter, etc.)
    • + *
  • Whether the request is part of a compaction
    • + *
  • Metric type (read time, block read count, cache hits/misses, etc.)
    • + *
+ *

+ * An instance of this class does not store any metric values. It just allows + * to determine the correct metric name for each combination of the above + * dimensions. + *

+ *

+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Metric keyPer-table metrics conf settingDescription
OnOff
tbl.T.cf.CF.M Include Skip A specific column family of a specific table
tbl.T.M Skip Skip All column families in the given table
cf.CF.M Skip Include A specific column family in all tables
M Include Include All column families in all tables
+ */ +public class SchemaMetrics { + + public interface SchemaAware { + public String getTableName(); + public String getColumnFamilyName(); + public SchemaMetrics getSchemaMetrics(); + } + + private static final Log LOG = LogFactory.getLog(SchemaMetrics.class); + + public static enum BlockMetricType { + // Metric configuration: compactionAware, timeVarying + READ_TIME("Read", true, true), + READ_COUNT("BlockReadCnt", true, false), + CACHE_HIT("BlockReadCacheHitCnt", true, false), + CACHE_MISS("BlockReadCacheMissCnt", true, false), + + CACHE_SIZE("blockCacheSize", false, false), + CACHED("blockCacheNumCached", false, false), + EVICTED("blockCacheNumEvicted", false, false); + + private final String metricStr; + private final boolean compactionAware; + private final boolean timeVarying; + + BlockMetricType(String metricStr, boolean compactionAware, + boolean timeVarying) { + this.metricStr = metricStr; + this.compactionAware = compactionAware; + this.timeVarying = timeVarying; + } + + @Override + public String toString() { + return metricStr; + } + + private static final String BLOCK_METRIC_TYPE_RE; + static { + StringBuilder sb = new StringBuilder(); + for (BlockMetricType bmt : values()) { + if (sb.length() > 0) + sb.append("|"); + sb.append(bmt); + } + BLOCK_METRIC_TYPE_RE = sb.toString(); + } + }; + + public static enum StoreMetricType { + STORE_FILE_COUNT("storeFileCount"), + STORE_FILE_INDEX_SIZE("storeFileIndexSizeMB"), + STORE_FILE_SIZE_MB("storeFileSizeMB"), + STATIC_BLOOM_SIZE_KB("staticBloomSizeKB"), + MEMSTORE_SIZE_MB("memstoreSizeMB"), + STATIC_INDEX_SIZE_KB("staticIndexSizeKB"), + FLUSH_SIZE("flushSize"); + + private final String metricStr; + + StoreMetricType(String metricStr) { + this.metricStr = metricStr; + } + + @Override + public String toString() { + return metricStr; + } + }; + + // Constants + /** + * A string used when column family or table name is unknown, and in some + * unit tests. This should not normally show up in metric names but if it + * does it is better than creating a silent discrepancy in total vs. + * per-CF/table metrics. + */ + public static final String UNKNOWN = "__unknown"; + + public static final String TABLE_PREFIX = "tbl."; + public static final String CF_PREFIX = "cf."; + public static final String BLOCK_TYPE_PREFIX = "bt."; + public static final String REGION_PREFIX = "region."; + + public static final String CF_UNKNOWN_PREFIX = CF_PREFIX + UNKNOWN + "."; + public static final String CF_BAD_FAMILY_PREFIX = CF_PREFIX + "__badfamily."; + + /** Use for readability when obtaining non-compaction counters */ + public static final boolean NO_COMPACTION = false; + + public static final String METRIC_GETSIZE = "getsize"; + public static final String METRIC_NEXTSIZE = "nextsize"; + + /** + * A special schema metric value that means "all tables aggregated" or + * "all column families aggregated" when used as a table name or a column + * family name. + */ + public static final String TOTAL_KEY = ""; + + /** + * Special handling for meta-block-specific metrics for + * backwards-compatibility. + */ + private static final String META_BLOCK_CATEGORY_STR = "Meta"; + + private static final int NUM_BLOCK_CATEGORIES = + BlockCategory.values().length; + + private static final int NUM_METRIC_TYPES = + BlockMetricType.values().length; + + static final boolean[] BOOL_VALUES = new boolean[] { false, true }; + + private static final int NUM_BLOCK_METRICS = + NUM_BLOCK_CATEGORIES * // blockCategory + BOOL_VALUES.length * // isCompaction + NUM_METRIC_TYPES; // metricType + + private static final int NUM_STORE_METRIC_TYPES = + StoreMetricType.values().length; + + /** Conf key controlling whether we include table name in metric names */ + private static final String SHOW_TABLE_NAME_CONF_KEY = + "hbase.metrics.showTableName"; + + /** We use this when too many column families are involved in a request. */ + private static final String MORE_CFS_OMITTED_STR = "__more"; + + /** + * Maximum length of a metric name prefix. Used when constructing metric + * names from a set of column families participating in a request. + */ + private static final int MAX_METRIC_PREFIX_LENGTH = + 256 - MORE_CFS_OMITTED_STR.length(); + + // Global variables + /** + * Maps a string key consisting of table name and column family name, with + * table name optionally replaced with {@link #TOTAL_KEY} if per-table + * metrics are disabled, to an instance of this class. + */ + private static final ConcurrentHashMap + tableAndFamilyToMetrics = new ConcurrentHashMap(); + + /** Metrics for all tables and column families. */ + // This has to be initialized after cfToMetrics. + public static final SchemaMetrics ALL_SCHEMA_METRICS = + getInstance(TOTAL_KEY, TOTAL_KEY); + + /** Threshold for flush the metrics, currently used only for "on cache hit" */ + private static final long THRESHOLD_METRICS_FLUSH = 100l; + + /** + * Whether to include table name in metric names. If this is null, it has not + * been initialized. This is a global instance, but we also have a copy of it + * per a {@link SchemaMetrics} object to avoid synchronization overhead. + */ + private static volatile Boolean useTableNameGlobally; + + /** Whether we logged a message about configuration inconsistency */ + private static volatile boolean loggedConfInconsistency; + + // Instance variables + private final String[] blockMetricNames = new String[NUM_BLOCK_METRICS]; + private final boolean[] blockMetricTimeVarying = + new boolean[NUM_BLOCK_METRICS]; + + private final String[] bloomMetricNames = new String[2]; + private final String[] storeMetricNames = new String[NUM_STORE_METRIC_TYPES]; + private final String[] storeMetricNamesMax = new String[NUM_STORE_METRIC_TYPES]; + private final AtomicLongArray onHitCacheMetrics= + new AtomicLongArray(NUM_BLOCK_CATEGORIES * BOOL_VALUES.length); + + private SchemaMetrics(final String tableName, final String cfName) { + String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix( + tableName, cfName); + + for (BlockCategory blockCategory : BlockCategory.values()) { + for (boolean isCompaction : BOOL_VALUES) { + // initialize the cache metrics + onHitCacheMetrics.set(getCacheHitMetricIndex(blockCategory, isCompaction), 0); + + for (BlockMetricType metricType : BlockMetricType.values()) { + if (!metricType.compactionAware && isCompaction) { + continue; + } + + StringBuilder sb = new StringBuilder(metricPrefix); + if (blockCategory != BlockCategory.ALL_CATEGORIES + && blockCategory != BlockCategory.META) { + String categoryStr = blockCategory.toString(); + categoryStr = categoryStr.charAt(0) + + categoryStr.substring(1).toLowerCase(); + sb.append(BLOCK_TYPE_PREFIX + categoryStr + "."); + } + + if (metricType.compactionAware) { + sb.append(isCompaction ? "compaction" : "fs"); + } + + // A special-case for meta blocks for backwards-compatibility. + if (blockCategory == BlockCategory.META) { + sb.append(META_BLOCK_CATEGORY_STR); + } + + sb.append(metricType); + + int i = getBlockMetricIndex(blockCategory, isCompaction, metricType); + blockMetricNames[i] = sb.toString().intern(); + blockMetricTimeVarying[i] = metricType.timeVarying; + } + } + } + + for (boolean isInBloom : BOOL_VALUES) { + bloomMetricNames[isInBloom ? 1 : 0] = metricPrefix + + (isInBloom ? "keyMaybeInBloomCnt" : "keyNotInBloomCnt"); + } + + for (StoreMetricType storeMetric : StoreMetricType.values()) { + String coreName = metricPrefix + storeMetric.toString(); + storeMetricNames[storeMetric.ordinal()] = coreName; + storeMetricNamesMax[storeMetric.ordinal()] = coreName + ".max"; + } + } + + /** + * Returns a {@link SchemaMetrics} object for the given table and column + * family, instantiating it if necessary. + * + * @param tableName table name (null is interpreted as "unknown"). This is + * ignored + * @param cfName column family name (null is interpreted as "unknown") + */ + public static SchemaMetrics getInstance(String tableName, String cfName) { + if (tableName == null) { + tableName = UNKNOWN; + } + + if (cfName == null) { + cfName = UNKNOWN; + } + + tableName = getEffectiveTableName(tableName); + + final String instanceKey = tableName + "\t" + cfName; + SchemaMetrics schemaMetrics = tableAndFamilyToMetrics.get(instanceKey); + if (schemaMetrics != null) { + return schemaMetrics; + } + + schemaMetrics = new SchemaMetrics(tableName, cfName); + SchemaMetrics existingMetrics = + tableAndFamilyToMetrics.putIfAbsent(instanceKey, schemaMetrics); + return existingMetrics != null ? existingMetrics : schemaMetrics; + } + + private static final int getCacheHitMetricIndex (BlockCategory blockCategory, + boolean isCompaction) { + return blockCategory.ordinal() * BOOL_VALUES.length + (isCompaction ? 1 : 0); + } + + private static final int getBlockMetricIndex(BlockCategory blockCategory, + boolean isCompaction, BlockMetricType metricType) { + int i = 0; + i = i * NUM_BLOCK_CATEGORIES + blockCategory.ordinal(); + i = i * BOOL_VALUES.length + (isCompaction ? 1 : 0); + i = i * NUM_METRIC_TYPES + metricType.ordinal(); + return i; + } + + public String getBlockMetricName(BlockCategory blockCategory, + boolean isCompaction, BlockMetricType metricType) { + if (isCompaction && !metricType.compactionAware) { + throw new IllegalArgumentException("isCompaction cannot be true for " + + metricType); + } + return blockMetricNames[getBlockMetricIndex(blockCategory, isCompaction, + metricType)]; + } + + public String getBloomMetricName(boolean isInBloom) { + return bloomMetricNames[isInBloom ? 1 : 0]; + } + + /** + * Increments the given metric, both per-CF and aggregate, for both the given + * category and all categories in aggregate (four counters total). + */ + private void incrNumericMetric(BlockCategory blockCategory, + boolean isCompaction, BlockMetricType metricType) { + incrNumericMetric (blockCategory, isCompaction, metricType, 1); + } + + /** + * Increments the given metric, both per-CF and aggregate, for both the given + * category and all categories in aggregate (four counters total). + */ + private void incrNumericMetric(BlockCategory blockCategory, + boolean isCompaction, BlockMetricType metricType, long amount) { + if (blockCategory == null) { + blockCategory = BlockCategory.UNKNOWN; // So that we see this in stats. + } + RegionMetricsStorage.incrNumericMetric(getBlockMetricName(blockCategory, + isCompaction, metricType), amount); + + if (blockCategory != BlockCategory.ALL_CATEGORIES) { + incrNumericMetric(BlockCategory.ALL_CATEGORIES, isCompaction, + metricType, amount); + } + } + + private void addToReadTime(BlockCategory blockCategory, + boolean isCompaction, long timeMs) { + RegionMetricsStorage.incrTimeVaryingMetric(getBlockMetricName(blockCategory, + isCompaction, BlockMetricType.READ_TIME), timeMs); + + // Also update the read time aggregated across all block categories + if (blockCategory != BlockCategory.ALL_CATEGORIES) { + addToReadTime(BlockCategory.ALL_CATEGORIES, isCompaction, timeMs); + } + } + + /** + * Used to accumulate store metrics across multiple regions in a region + * server. These metrics are not "persistent", i.e. we keep overriding them + * on every update instead of incrementing, so we need to accumulate them in + * a temporary map before pushing them to the global metric collection. + * @param tmpMap a temporary map for accumulating store metrics + * @param storeMetricType the store metric type to increment + * @param val the value to add to the metric + */ + public void accumulateStoreMetric(final Map tmpMap, + StoreMetricType storeMetricType, double val) { + final String key = getStoreMetricName(storeMetricType); + if (tmpMap.get(key) == null) { + tmpMap.put(key, new MutableDouble(val)); + } else { + tmpMap.get(key).add(val); + } + + if (this == ALL_SCHEMA_METRICS) { + // also compute the max value across all Stores on this server + final String maxKey = getStoreMetricNameMax(storeMetricType); + MutableDouble cur = tmpMap.get(maxKey); + if (cur == null) { + tmpMap.put(maxKey, new MutableDouble(val)); + } else if (cur.doubleValue() < val) { + cur.setValue(val); + } + } else { + ALL_SCHEMA_METRICS.accumulateStoreMetric(tmpMap, storeMetricType, val); + } + } + + public String getStoreMetricName(StoreMetricType storeMetricType) { + return storeMetricNames[storeMetricType.ordinal()]; + } + + public String getStoreMetricNameMax(StoreMetricType storeMetricType) { + return storeMetricNamesMax[storeMetricType.ordinal()]; + } + + /** + * Update a metric that does not get reset on every poll. + * @param storeMetricType the store metric to update + * @param value the value to update the metric to + */ + public void updatePersistentStoreMetric(StoreMetricType storeMetricType, + long value) { + RegionMetricsStorage.incrNumericPersistentMetric( + storeMetricNames[storeMetricType.ordinal()], value); + } + + /** + * Updates the number of hits and the total number of block reads on a block + * cache hit. + */ + public void updateOnCacheHit(BlockCategory blockCategory, + boolean isCompaction) { + updateOnCacheHit(blockCategory, isCompaction, 1); + } + + /** + * Updates the number of hits and the total number of block reads on a block + * cache hit. + */ + public void updateOnCacheHit(BlockCategory blockCategory, + boolean isCompaction, long count) { + blockCategory.expectSpecific(); + int idx = getCacheHitMetricIndex(blockCategory, isCompaction); + + if (this.onHitCacheMetrics.addAndGet(idx, count) > THRESHOLD_METRICS_FLUSH) { + flushCertainOnCacheHitMetrics(blockCategory, isCompaction); + } + + if (this != ALL_SCHEMA_METRICS) { + ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction, count); + } + } + + private void flushCertainOnCacheHitMetrics(BlockCategory blockCategory, boolean isCompaction) { + int idx = getCacheHitMetricIndex(blockCategory, isCompaction); + long tempCount = this.onHitCacheMetrics.getAndSet(idx, 0); + + if (tempCount > 0) { + incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT, tempCount); + incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT, tempCount); + } + } + + /** + * Flush the on cache hit metrics; + */ + private void flushOnCacheHitMetrics() { + for (BlockCategory blockCategory : BlockCategory.values()) { + for (boolean isCompaction : BOOL_VALUES) { + flushCertainOnCacheHitMetrics (blockCategory, isCompaction); + } + } + + if (this != ALL_SCHEMA_METRICS) { + ALL_SCHEMA_METRICS.flushOnCacheHitMetrics(); + } + } + + /** + * Notify the SchemaMetrics to flush all of the the metrics + */ + public void flushMetrics() { + // currently only for "on cache hit metrics" + flushOnCacheHitMetrics(); + } + + /** + * Updates read time, the number of misses, and the total number of block + * reads on a block cache miss. + */ + public void updateOnCacheMiss(BlockCategory blockCategory, + boolean isCompaction, long timeMs) { + blockCategory.expectSpecific(); + addToReadTime(blockCategory, isCompaction, timeMs); + incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_MISS); + incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT); + if (this != ALL_SCHEMA_METRICS) { + ALL_SCHEMA_METRICS.updateOnCacheMiss(blockCategory, isCompaction, + timeMs); + } + } + + /** + * Adds the given delta to the cache size for the given block category and + * the aggregate metric for all block categories. Updates both the per-CF + * counter and the counter for all CFs (four metrics total). The cache size + * metric is "persistent", i.e. it does not get reset when metrics are + * collected. + */ + public void addToCacheSize(BlockCategory category, long cacheSizeDelta) { + if (category == null) { + category = BlockCategory.ALL_CATEGORIES; + } + RegionMetricsStorage.incrNumericPersistentMetric(getBlockMetricName(category, false, + BlockMetricType.CACHE_SIZE), cacheSizeDelta); + + if (category != BlockCategory.ALL_CATEGORIES) { + addToCacheSize(BlockCategory.ALL_CATEGORIES, cacheSizeDelta); + } + } + + public void updateOnCachePutOrEvict(BlockCategory blockCategory, + long cacheSizeDelta, boolean isEviction) { + addToCacheSize(blockCategory, cacheSizeDelta); + incrNumericMetric(blockCategory, false, + isEviction ? BlockMetricType.EVICTED : BlockMetricType.CACHED); + if (this != ALL_SCHEMA_METRICS) { + ALL_SCHEMA_METRICS.updateOnCachePutOrEvict(blockCategory, cacheSizeDelta, + isEviction); + } + } + + /** + * Increments both the per-CF and the aggregate counter of bloom + * positives/negatives as specified by the argument. + */ + public void updateBloomMetrics(boolean isInBloom) { + RegionMetricsStorage.incrNumericMetric(getBloomMetricName(isInBloom), 1); + if (this != ALL_SCHEMA_METRICS) { + ALL_SCHEMA_METRICS.updateBloomMetrics(isInBloom); + } + } + + /** + * Sets the flag whether to use table name in metric names according to the + * given configuration. This must be called at least once before + * instantiating HFile readers/writers. + */ + public static void configureGlobally(Configuration conf) { + if (conf != null) { + final boolean useTableNameNew = + conf.getBoolean(SHOW_TABLE_NAME_CONF_KEY, false); + setUseTableName(useTableNameNew); + } else { + setUseTableName(false); + } + } + + /** + * Determine the table name to be included in metric keys. If the global + * configuration says that we should not use table names in metrics, + * we always return {@link #TOTAL_KEY} even if nontrivial table name is + * provided. + * + * @param tableName a table name or {@link #TOTAL_KEY} when aggregating + * across all tables + * @return the table name to use in metric keys + */ + private static String getEffectiveTableName(String tableName) { + if (!tableName.equals(TOTAL_KEY)) { + // We are provided with a non-trivial table name (including "unknown"). + // We need to know whether table name should be included into metrics. + if (useTableNameGlobally == null) { + throw new IllegalStateException("The value of the " + + SHOW_TABLE_NAME_CONF_KEY + " conf option has not been specified " + + "in SchemaMetrics"); + } + final boolean useTableName = useTableNameGlobally; + if (!useTableName) { + // Don't include table name in metric keys. + tableName = TOTAL_KEY; + } + } + return tableName; + } + + /** + * Method to transform a combination of a table name and a column family name + * into a metric key prefix. Tables/column family names equal to + * {@link #TOTAL_KEY} are omitted from the prefix. + * + * @param tableName the table name or {@link #TOTAL_KEY} for all tables + * @param cfName the column family name or {@link #TOTAL_KEY} for all CFs + * @return the metric name prefix, ending with a dot. + */ + public static String generateSchemaMetricsPrefix(String tableName, + final String cfName) { + tableName = getEffectiveTableName(tableName); + String schemaMetricPrefix = + tableName.equals(TOTAL_KEY) ? "" : TABLE_PREFIX + tableName + "."; + schemaMetricPrefix += + cfName.equals(TOTAL_KEY) ? "" : CF_PREFIX + cfName + "."; + return schemaMetricPrefix; + } + + public static String generateSchemaMetricsPrefix(byte[] tableName, + byte[] cfName) { + return generateSchemaMetricsPrefix(Bytes.toString(tableName), + Bytes.toString(cfName)); + } + + /** + * Method to transform a set of column families in byte[] format with table + * name into a metric key prefix. + * + * @param tableName the table name or {@link #TOTAL_KEY} for all tables + * @param families the ordered set of column families + * @return the metric name prefix, ending with a dot, or an empty string in + * case of invalid arguments. This is OK since we always expect + * some CFs to be included. + */ + public static String generateSchemaMetricsPrefix(String tableName, + Set families) { + if (families == null || families.isEmpty() || + tableName == null || tableName.isEmpty()) { + return ""; + } + + if (families.size() == 1) { + return generateSchemaMetricsPrefix(tableName, + Bytes.toString(families.iterator().next())); + } + + tableName = getEffectiveTableName(tableName); + List sortedFamilies = new ArrayList(families); + Collections.sort(sortedFamilies, Bytes.BYTES_COMPARATOR); + + StringBuilder sb = new StringBuilder(); + + int numCFsLeft = families.size(); + for (byte[] family : sortedFamilies) { + if (sb.length() > MAX_METRIC_PREFIX_LENGTH) { + sb.append(MORE_CFS_OMITTED_STR); + break; + } + --numCFsLeft; + sb.append(Bytes.toString(family)); + if (numCFsLeft > 0) { + sb.append("~"); + } + } + + return SchemaMetrics.generateSchemaMetricsPrefix(tableName, sb.toString()); + } + + /** + * Get the prefix for metrics generated about a single region. + * + * @param tableName + * the table name or {@link #TOTAL_KEY} for all tables + * @param regionName + * regionName + * @return the prefix for this table/region combination. + */ + static String generateRegionMetricsPrefix(String tableName, String regionName) { + tableName = getEffectiveTableName(tableName); + String schemaMetricPrefix = tableName.equals(TOTAL_KEY) ? "" : TABLE_PREFIX + tableName + "."; + schemaMetricPrefix += regionName.equals(TOTAL_KEY) ? "" : REGION_PREFIX + regionName + "."; + + return schemaMetricPrefix; + } + + /** + * Sets the flag of whether to use table name in metric names. This flag + * is specified in configuration and is not expected to change at runtime, + * so we log an error message when it does change. + */ + private static void setUseTableName(final boolean useTableNameNew) { + if (useTableNameGlobally == null) { + // This configuration option has not yet been set. + useTableNameGlobally = useTableNameNew; + } else if (useTableNameGlobally != useTableNameNew + && !loggedConfInconsistency) { + // The configuration is inconsistent and we have not reported it + // previously. Once we report it, just keep ignoring the new setting. + LOG.error("Inconsistent configuration. Previous configuration " + + "for using table name in metrics: " + useTableNameGlobally + ", " + + "new configuration: " + useTableNameNew); + loggedConfInconsistency = true; + } + } + + // Methods used in testing + + private static final String regexEscape(String s) { + return s.replace(".", "\\."); + } + + /** + * Assume that table names used in tests don't contain dots, except for the + * META table. + */ + private static final String WORD_AND_DOT_RE_STR = "([^.]+|" + + regexEscape(Bytes.toString(HConstants.META_TABLE_NAME)) + + ")\\."; + + /** "tbl.." */ + private static final String TABLE_NAME_RE_STR = + "\\b" + regexEscape(TABLE_PREFIX) + WORD_AND_DOT_RE_STR; + + /** "cf.." */ + private static final String CF_NAME_RE_STR = + "\\b" + regexEscape(CF_PREFIX) + WORD_AND_DOT_RE_STR; + private static final Pattern CF_NAME_RE = Pattern.compile(CF_NAME_RE_STR); + + /** "tbl..cf.." */ + private static final Pattern TABLE_AND_CF_NAME_RE = Pattern.compile( + TABLE_NAME_RE_STR + CF_NAME_RE_STR); + + private static final Pattern BLOCK_CATEGORY_RE = Pattern.compile( + "\\b" + regexEscape(BLOCK_TYPE_PREFIX) + "[^.]+\\." + + // Also remove the special-case block type marker for meta blocks + "|" + META_BLOCK_CATEGORY_STR + "(?=" + + BlockMetricType.BLOCK_METRIC_TYPE_RE + ")"); + + /** + * A suffix for the "number of operations" part of "time-varying metrics". We + * only use this for metric verification in unit testing. Time-varying + * metrics are handled by a different code path in production. + */ + private static String NUM_OPS_SUFFIX = "numops"; + + /** + * A custom suffix that we use for verifying the second component of + * a "time-varying metric". + */ + private static String TOTAL_SUFFIX = "_total"; + private static final Pattern TIME_VARYING_SUFFIX_RE = Pattern.compile( + "(" + NUM_OPS_SUFFIX + "|" + TOTAL_SUFFIX + ")$"); + + void printMetricNames() { + for (BlockCategory blockCategory : BlockCategory.values()) { + for (boolean isCompaction : BOOL_VALUES) { + for (BlockMetricType metricType : BlockMetricType.values()) { + int i = getBlockMetricIndex(blockCategory, isCompaction, metricType); + LOG.debug("blockCategory=" + blockCategory + ", " + + "metricType=" + metricType + ", isCompaction=" + isCompaction + + ", metricName=" + blockMetricNames[i]); + } + } + } + } + + private Collection getAllMetricNames() { + List allMetricNames = new ArrayList(); + for (int i = 0; i < blockMetricNames.length; ++i) { + final String blockMetricName = blockMetricNames[i]; + final boolean timeVarying = blockMetricTimeVarying[i]; + if (blockMetricName != null) { + if (timeVarying) { + allMetricNames.add(blockMetricName + NUM_OPS_SUFFIX); + allMetricNames.add(blockMetricName + TOTAL_SUFFIX); + } else { + allMetricNames.add(blockMetricName); + } + } + } + allMetricNames.addAll(Arrays.asList(bloomMetricNames)); + return allMetricNames; + } + + private static final boolean isTimeVaryingKey(String metricKey) { + return metricKey.endsWith(NUM_OPS_SUFFIX) + || metricKey.endsWith(TOTAL_SUFFIX); + } + + private static final String stripTimeVaryingSuffix(String metricKey) { + return TIME_VARYING_SUFFIX_RE.matcher(metricKey).replaceAll(""); + } + + public static Map getMetricsSnapshot() { + Map metricsSnapshot = new TreeMap(); + for (SchemaMetrics cfm : tableAndFamilyToMetrics.values()) { + cfm.flushMetrics(); + for (String metricName : cfm.getAllMetricNames()) { + long metricValue; + if (isTimeVaryingKey(metricName)) { + Pair totalAndCount = + RegionMetricsStorage.getTimeVaryingMetric(stripTimeVaryingSuffix(metricName)); + metricValue = metricName.endsWith(TOTAL_SUFFIX) ? + totalAndCount.getFirst() : totalAndCount.getSecond(); + } else { + metricValue = RegionMetricsStorage.getNumericMetric(metricName); + } + + metricsSnapshot.put(metricName, metricValue); + } + } + return metricsSnapshot; + } + + public static long getLong(Map m, String k) { + Long l = m.get(k); + return l != null ? l : 0; + } + + private static void putLong(Map m, String k, long v) { + if (v != 0) { + m.put(k, v); + } else { + m.remove(k); + } + } + + /** + * @return the difference between two sets of metrics (second minus first). + * Only includes keys that have nonzero difference. + */ + public static Map diffMetrics(Map a, + Map b) { + Set allKeys = new TreeSet(a.keySet()); + allKeys.addAll(b.keySet()); + Map diff = new TreeMap(); + for (String k : allKeys) { + long aVal = getLong(a, k); + long bVal = getLong(b, k); + if (aVal != bVal) { + diff.put(k, bVal - aVal); + } + } + return diff; + } + + public static void validateMetricChanges(Map oldMetrics) { + final Map newMetrics = getMetricsSnapshot(); + final Map allCfDeltas = new TreeMap(); + final Map allBlockCategoryDeltas = + new TreeMap(); + final Map deltas = diffMetrics(oldMetrics, newMetrics); + final Pattern cfTableMetricRE = + useTableNameGlobally ? TABLE_AND_CF_NAME_RE : CF_NAME_RE; + final Set allKeys = new TreeSet(oldMetrics.keySet()); + allKeys.addAll(newMetrics.keySet()); + + for (SchemaMetrics cfm : tableAndFamilyToMetrics.values()) { + for (String metricName : cfm.getAllMetricNames()) { + if (metricName.startsWith(CF_PREFIX + CF_PREFIX)) { + throw new AssertionError("Column family prefix used twice: " + + metricName); + } + + final long oldValue = getLong(oldMetrics, metricName); + final long newValue = getLong(newMetrics, metricName); + final long delta = newValue - oldValue; + + // Re-calculate values of metrics with no column family (or CF/table) + // specified based on all metrics with CF (or CF/table) specified. + if (delta != 0) { + if (cfm != ALL_SCHEMA_METRICS) { + final String aggregateMetricName = + cfTableMetricRE.matcher(metricName).replaceAll(""); + if (!aggregateMetricName.equals(metricName)) { + LOG.debug("Counting " + delta + " units of " + metricName + + " towards " + aggregateMetricName); + + putLong(allCfDeltas, aggregateMetricName, + getLong(allCfDeltas, aggregateMetricName) + delta); + } + } else { + LOG.debug("Metric=" + metricName + ", delta=" + delta); + } + } + + Matcher matcher = BLOCK_CATEGORY_RE.matcher(metricName); + if (matcher.find()) { + // Only process per-block-category metrics + String metricNoBlockCategory = matcher.replaceAll(""); + + putLong(allBlockCategoryDeltas, metricNoBlockCategory, + getLong(allBlockCategoryDeltas, metricNoBlockCategory) + delta); + } + } + } + + StringBuilder errors = new StringBuilder(); + for (String key : ALL_SCHEMA_METRICS.getAllMetricNames()) { + long actual = getLong(deltas, key); + long expected = getLong(allCfDeltas, key); + if (actual != expected) { + if (errors.length() > 0) + errors.append("\n"); + errors.append("The all-CF metric " + key + " changed by " + + actual + " but the aggregation of per-CF/table metrics " + + "yields " + expected); + } + } + + // Verify metrics computed for all block types based on the aggregation + // of per-block-type metrics. + for (String key : allKeys) { + if (BLOCK_CATEGORY_RE.matcher(key).find() || + key.contains(ALL_SCHEMA_METRICS.getBloomMetricName(false)) || + key.contains(ALL_SCHEMA_METRICS.getBloomMetricName(true))){ + // Skip per-block-category metrics. Also skip bloom filters, because + // they are not aggregated per block type. + continue; + } + long actual = getLong(deltas, key); + long expected = getLong(allBlockCategoryDeltas, key); + if (actual != expected) { + if (errors.length() > 0) + errors.append("\n"); + errors.append("The all-block-category metric " + key + + " changed by " + actual + " but the aggregation of " + + "per-block-category metrics yields " + expected); + } + } + + if (errors.length() > 0) { + throw new AssertionError(errors.toString()); + } + } + + /** + * Creates an instance pretending both the table and column family are + * unknown. Used in unit tests. + */ + public static SchemaMetrics getUnknownInstanceForTest() { + return getInstance(UNKNOWN, UNKNOWN); + } + + /** + * Set the flag to use or not use table name in metric names. Used in unit + * tests, so the flag can be set arbitrarily. + */ + public static void setUseTableNameInTest(final boolean useTableNameNew) { + useTableNameGlobally = useTableNameNew; + } + + /** Formats the given map of metrics in a human-readable way. */ + public static String formatMetrics(Map metrics) { + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : metrics.entrySet()) { + if (sb.length() > 0) { + sb.append('\n'); + } + sb.append(entry.getKey() + " : " + entry.getValue()); + } + return sb.toString(); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java new file mode 100644 index 0000000..b5d194c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.snapshot; + +import java.util.List; +import java.util.concurrent.Callable; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.errorhandling.ForeignException; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.procedure.ProcedureMember; +import org.apache.hadoop.hbase.procedure.Subprocedure; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager.SnapshotSubprocedurePool; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; + +/** + * This online snapshot implementation uses the distributed procedure framework to force a + * store flush and then records the hfiles. Its enter stage does nothing. Its leave stage then + * flushes the memstore, builds the region server's snapshot manifest from its hfiles list, and + * copies .regioninfos into the snapshot working directory. At the master side, there is an atomic + * rename of the working dir into the proper snapshot directory. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class FlushSnapshotSubprocedure extends Subprocedure { + private static final Log LOG = LogFactory.getLog(FlushSnapshotSubprocedure.class); + + private final List regions; + private final SnapshotDescription snapshot; + private final SnapshotSubprocedurePool taskManager; + + public FlushSnapshotSubprocedure(ProcedureMember member, + ForeignExceptionDispatcher errorListener, long wakeFrequency, long timeout, + List regions, SnapshotDescription snapshot, + SnapshotSubprocedurePool taskManager) { + super(member, snapshot.getName(), errorListener, wakeFrequency, timeout); + this.snapshot = snapshot; + this.regions = regions; + this.taskManager = taskManager; + } + + /** + * Callable for adding files to snapshot manifest working dir. Ready for multithreading. + */ + private class RegionSnapshotTask implements Callable { + HRegion region; + RegionSnapshotTask(HRegion region) { + this.region = region; + } + + @Override + public Void call() throws Exception { + // Taking the region read lock prevents the individual region from being closed while a + // snapshot is in progress. This is helpful but not sufficient for preventing races with + // snapshots that involve multiple regions and regionservers. It is still possible to have + // an interleaving such that globally regions are missing, so we still need the verification + // step. + LOG.debug("Starting region operation on " + region); + region.startRegionOperation(); + try { + LOG.debug("Flush Snapshotting region " + region.toString() + " started..."); + region.flushcache(); + region.addRegionToSnapshot(snapshot, monitor); + LOG.debug("... Flush Snapshotting region " + region.toString() + " completed."); + } finally { + LOG.debug("Closing region operation on " + region); + region.closeRegionOperation(); + } + return null; + } + } + + private void flushSnapshot() throws ForeignException { + if (regions.isEmpty()) { + // No regions on this RS, we are basically done. + return; + } + + monitor.rethrowException(); + + // assert that the taskManager is empty. + if (taskManager.hasTasks()) { + throw new IllegalStateException("Attempting to take snapshot " + + SnapshotDescriptionUtils.toString(snapshot) + + " but we currently have outstanding tasks"); + } + + // Add all hfiles already existing in region. + for (HRegion region : regions) { + // submit one task per region for parallelize by region. + taskManager.submitTask(new RegionSnapshotTask(region)); + monitor.rethrowException(); + } + + // wait for everything to complete. + LOG.debug("Flush Snapshot Tasks submitted for " + regions.size() + " regions"); + try { + taskManager.waitForOutstandingTasks(); + } catch (InterruptedException e) { + throw new ForeignException(getMemberName(), e); + } + } + + /** + * do nothing, core of snapshot is executed in {@link #insideBarrier} step. + */ + @Override + public void acquireBarrier() throws ForeignException { + // NO OP + } + + /** + * do a flush snapshot of every region on this rs from the target table. + */ + @Override + public void insideBarrier() throws ForeignException { + flushSnapshot(); + } + + /** + * Cancel threads if they haven't finished. + */ + @Override + public void cleanup(Exception e) { + LOG.info("Aborting all online FLUSH snapshot subprocedure task threads for '" + + snapshot.getName() + "' due to error", e); + try { + taskManager.cancelTasks(); + } catch (InterruptedException e1) { + Thread.currentThread().interrupt(); + } + } + + /** + * Hooray! + */ + public void releaseBarrier() { + // NO OP + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java new file mode 100644 index 0000000..987a713 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java @@ -0,0 +1,377 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.snapshot; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.DaemonThreadFactory; +import org.apache.hadoop.hbase.errorhandling.ForeignException; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.master.snapshot.MasterSnapshotVerifier; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.procedure.ProcedureMember; +import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs; +import org.apache.hadoop.hbase.procedure.Subprocedure; +import org.apache.hadoop.hbase.procedure.SubprocedureFactory; +import org.apache.hadoop.hbase.procedure.ZKProcedureMemberRpcs; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +import com.google.protobuf.InvalidProtocolBufferException; + +/** + * This manager class handles the work dealing with snapshots for a {@link HRegionServer}. + *

+ * This provides the mechanism necessary to kick off a online snapshot specific + * {@link Subprocedure} that is responsible for the regions being served by this region server. + * If any failures occur with the subprocedure, the RegionSeverSnapshotManager's subprocedure + * handler, {@link ProcedureMember}, notifies the master's ProcedureCoordinator to abort all + * others. + *

+ * On startup, requires {@link #start()} to be called. + *

+ * On shutdown, requires {@link #stop(boolean)} to be called + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class RegionServerSnapshotManager { + private static final Log LOG = LogFactory.getLog(RegionServerSnapshotManager.class); + + /** Maximum number of snapshot region tasks that can run concurrently */ + private static final String CONCURENT_SNAPSHOT_TASKS_KEY = "hbase.snapshot.region.concurrentTasks"; + private static final int DEFAULT_CONCURRENT_SNAPSHOT_TASKS = 3; + + /** Conf key for number of request threads to start snapshots on regionservers */ + public static final String SNAPSHOT_REQUEST_THREADS_KEY = "hbase.snapshot.region.pool.threads"; + /** # of threads for snapshotting regions on the rs. */ + public static final int SNAPSHOT_REQUEST_THREADS_DEFAULT = 10; + + /** Conf key for max time to keep threads in snapshot request pool waiting */ + public static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.region.timeout"; + /** Keep threads alive in request pool for max of 60 seconds */ + public static final long SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000; + + /** Conf key for millis between checks to see if snapshot completed or if there are errors*/ + public static final String SNAPSHOT_REQUEST_WAKE_MILLIS_KEY = "hbase.snapshot.region.wakefrequency"; + /** Default amount of time to check for errors while regions finish snapshotting */ + private static final long SNAPSHOT_REQUEST_WAKE_MILLIS_DEFAULT = 500; + + private final RegionServerServices rss; + private final ProcedureMemberRpcs memberRpcs; + private final ProcedureMember member; + + /** + * Exposed for testing. + * @param conf HBase configuration. + * @param parent parent running the snapshot handler + * @param memberRpc use specified memberRpc instance + * @param procMember use specified ProcedureMember + */ + RegionServerSnapshotManager(Configuration conf, HRegionServer parent, + ProcedureMemberRpcs memberRpc, ProcedureMember procMember) { + this.rss = parent; + this.memberRpcs = memberRpc; + this.member = procMember; + } + + /** + * Create a default snapshot handler - uses a zookeeper based member controller. + * @param rss region server running the handler + * @throws KeeperException if the zookeeper cluster cannot be reached + */ + public RegionServerSnapshotManager(RegionServerServices rss) + throws KeeperException { + this.rss = rss; + ZooKeeperWatcher zkw = rss.getZooKeeper(); + String nodeName = rss.getServerName().toString(); + this.memberRpcs = new ZKProcedureMemberRpcs(zkw, + SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, nodeName); + + // read in the snapshot request configuration properties + Configuration conf = rss.getConfiguration(); + long wakeMillis = conf.getLong(SNAPSHOT_REQUEST_WAKE_MILLIS_KEY, SNAPSHOT_REQUEST_WAKE_MILLIS_DEFAULT); + long keepAlive = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT); + int opThreads = conf.getInt(SNAPSHOT_REQUEST_THREADS_KEY, SNAPSHOT_REQUEST_THREADS_DEFAULT); + + // create the actual snapshot procedure member + ThreadPoolExecutor pool = ProcedureMember.defaultPool(wakeMillis, keepAlive, opThreads, nodeName); + this.member = new ProcedureMember(memberRpcs, pool, new SnapshotSubprocedureBuilder()); + } + + /** + * Start accepting snapshot requests. + */ + public void start() { + this.memberRpcs.start(member); + } + + /** + * Close this and all running snapshot tasks + * @param force forcefully stop all running tasks + * @throws IOException + */ + public void stop(boolean force) throws IOException { + String mode = force ? "abruptly" : "gracefully"; + LOG.info("Stopping RegionServerSnapshotManager " + mode + "."); + + try { + this.member.close(); + } finally { + this.memberRpcs.close(); + } + } + + /** + * If in a running state, creates the specified subprocedure for handling an online snapshot. + * + * Because this gets the local list of regions to snapshot and not the set the master had, + * there is a possibility of a race where regions may be missed. This detected by the master in + * the snapshot verification step. + * + * @param snapshot + * @return Subprocedure to submit to the ProcedureMemeber. + */ + public Subprocedure buildSubprocedure(SnapshotDescription snapshot) { + + // don't run a snapshot if the parent is stop(ping) + if (rss.isStopping() || rss.isStopped()) { + throw new IllegalStateException("Can't start snapshot on RS: " + rss.getServerName() + + ", because stopping/stopped!"); + } + + // check to see if this server is hosting any regions for the snapshots + // check to see if we have regions for the snapshot + List involvedRegions; + try { + involvedRegions = getRegionsToSnapshot(snapshot); + } catch (IOException e1) { + throw new IllegalStateException("Failed to figure out if we should handle a snapshot - " + + "something has gone awry with the online regions.", e1); + } + + // We need to run the subprocedure even if we have no relevant regions. The coordinator + // expects participation in the procedure and without sending message the snapshot attempt + // will hang and fail. + + LOG.debug("Launching subprocedure for snapshot " + snapshot.getName() + " from table " + + snapshot.getTable()); + ForeignExceptionDispatcher exnDispatcher = new ForeignExceptionDispatcher(); + Configuration conf = rss.getConfiguration(); + long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, + SNAPSHOT_TIMEOUT_MILLIS_DEFAULT); + long wakeMillis = conf.getLong(SNAPSHOT_REQUEST_WAKE_MILLIS_KEY, + SNAPSHOT_REQUEST_WAKE_MILLIS_DEFAULT); + + switch (snapshot.getType()) { + case FLUSH: + SnapshotSubprocedurePool taskManager = + new SnapshotSubprocedurePool(rss.getServerName().toString(), conf); + return new FlushSnapshotSubprocedure(member, exnDispatcher, wakeMillis, + timeoutMillis, involvedRegions, snapshot, taskManager); + default: + throw new UnsupportedOperationException("Unrecognized snapshot type:" + snapshot.getType()); + } + } + + /** + * Determine if the snapshot should be handled on this server + * + * NOTE: This is racy -- the master expects a list of regionservers. + * This means if a region moves somewhere between the calls we'll miss some regions. + * For example, a region move during a snapshot could result in a region to be skipped or done + * twice. This is manageable because the {@link MasterSnapshotVerifier} will double check the + * region lists after the online portion of the snapshot completes and will explicitly fail the + * snapshot. + * + * @param snapshot + * @return the list of online regions. Empty list is returned if no regions are responsible for + * the given snapshot. + * @throws IOException + */ + private List getRegionsToSnapshot(SnapshotDescription snapshot) throws IOException { + byte[] table = Bytes.toBytes(snapshot.getTable()); + return rss.getOnlineRegions(table); + } + + /** + * Build the actual snapshot runner that will do all the 'hard' work + */ + public class SnapshotSubprocedureBuilder implements SubprocedureFactory { + + @Override + public Subprocedure buildSubprocedure(String name, byte[] data) { + try { + // unwrap the snapshot information + SnapshotDescription snapshot = SnapshotDescription.parseFrom(data); + return RegionServerSnapshotManager.this.buildSubprocedure(snapshot); + } catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException("Could not read snapshot information from request."); + } + } + + } + + /** + * We use the SnapshotSubprocedurePool, a class specific thread pool instead of + * {@link org.apache.hadoop.hbase.executor.ExecutorService}. + * + * It uses a {@link java.util.concurrent.ExecutorCompletionService} which provides queuing of + * completed tasks which lets us efficiently cancel pending tasks upon the earliest operation + * failures. + * + * HBase's ExecutorService (different from {@link java.util.concurrent.ExecutorService}) isn't + * really built for coordinated tasks where multiple threads as part of one larger task. In + * RS's the HBase Executor services are only used for open and close and not other threadpooled + * operations such as compactions and replication sinks. + */ + static class SnapshotSubprocedurePool { + private final ExecutorCompletionService taskPool; + private final ThreadPoolExecutor executor; + private volatile boolean stopped; + private final List> futures = new ArrayList>(); + private final String name; + + SnapshotSubprocedurePool(String name, Configuration conf) { + // configure the executor service + long keepAlive = conf.getLong( + RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_KEY, + RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT); + int threads = conf.getInt(CONCURENT_SNAPSHOT_TASKS_KEY, DEFAULT_CONCURRENT_SNAPSHOT_TASKS); + this.name = name; + executor = new ThreadPoolExecutor(1, threads, keepAlive, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue(), new DaemonThreadFactory("rs(" + + name + ")-snapshot-pool")); + taskPool = new ExecutorCompletionService(executor); + } + + boolean hasTasks() { + return futures.size() != 0; + } + + /** + * Submit a task to the pool. + * + * NOTE: all must be submitted before you can safely {@link #waitForOutstandingTasks()}. This + * version does not support issuing tasks from multiple concurrent table snapshots requests. + */ + void submitTask(final Callable task) { + Future f = this.taskPool.submit(task); + futures.add(f); + } + + /** + * Wait for all of the currently outstanding tasks submitted via {@link #submitTask(Callable)}. + * This *must* be called after all tasks are submitted via submitTask. + * + * @return true on success, false otherwise + * @throws InterruptedException + * @throws SnapshotCreationException if the snapshot failed while we were waiting + */ + boolean waitForOutstandingTasks() throws ForeignException, InterruptedException { + LOG.debug("Waiting for local region snapshots to finish."); + + int sz = futures.size(); + try { + // Using the completion service to process the futures that finish first first. + for (int i = 0; i < sz; i++) { + Future f = taskPool.take(); + f.get(); + if (!futures.remove(f)) { + LOG.warn("unexpected future" + f); + } + LOG.debug("Completed " + (i+1) + "/" + sz + " local region snapshots."); + } + LOG.debug("Completed " + sz + " local region snapshots."); + return true; + } catch (InterruptedException e) { + LOG.warn("Got InterruptedException in SnapshotSubprocedurePool", e); + if (!stopped) { + Thread.currentThread().interrupt(); + throw new ForeignException("SnapshotSubprocedurePool", e); + } + // we are stopped so we can just exit. + } catch (ExecutionException e) { + if (e.getCause() instanceof ForeignException) { + LOG.warn("Rethrowing ForeignException from SnapshotSubprocedurePool", e); + throw (ForeignException)e.getCause(); + } + LOG.warn("Got Exception in SnapshotSubprocedurePool", e); + throw new ForeignException(name, e.getCause()); + } finally { + cancelTasks(); + } + return false; + } + + /** + * This attempts to cancel out all pending and in progress tasks (interruptions issues) + * @throws InterruptedException + */ + void cancelTasks() throws InterruptedException { + Collection> tasks = futures; + LOG.debug("cancelling " + tasks.size() + " tasks for snapshot " + name); + for (Future f: tasks) { + // TODO Ideally we'd interrupt hbase threads when we cancel. However it seems that there + // are places in the HBase code where row/region locks are taken and not released in a + // finally block. Thus we cancel without interrupting. Cancellations will be slower to + // complete but we won't suffer from unreleased locks due to poor code discipline. + f.cancel(false); + } + + // evict remaining tasks and futures from taskPool. + LOG.debug(taskPool); + while (!futures.isEmpty()) { + // block to remove cancelled futures; + LOG.warn("Removing cancelled elements from taskPool"); + futures.remove(taskPool.take()); + } + stop(); + } + + /** + * Abruptly shutdown the thread pool. Call when exiting a region server. + */ + void stop() { + if (this.stopped) return; + + this.stopped = true; + this.executor.shutdownNow(); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java new file mode 100644 index 0000000..10aec00 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +/** + * Context that holds the various dictionaries for compression in HLog. + */ +class CompressionContext { + final Dictionary regionDict; + final Dictionary tableDict; + final Dictionary familyDict; + final Dictionary qualifierDict; + final Dictionary rowDict; + + public CompressionContext(Class dictType) + throws SecurityException, NoSuchMethodException, InstantiationException, + IllegalAccessException, InvocationTargetException { + Constructor dictConstructor = + dictType.getConstructor(); + regionDict = dictConstructor.newInstance(); + tableDict = dictConstructor.newInstance(); + familyDict = dictConstructor.newInstance(); + qualifierDict = dictConstructor.newInstance(); + rowDict = dictConstructor.newInstance(); + } + + void clear() { + regionDict.clear(); + tableDict.clear(); + familyDict.clear(); + qualifierDict.clear(); + rowDict.clear(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java new file mode 100644 index 0000000..c19d5b3 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Compressor.java @@ -0,0 +1,191 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import com.google.common.base.Preconditions; + +/** + * A set of static functions for running our custom WAL compression/decompression. + * Also contains a command line tool to compress and uncompress HLogs. + */ +public class Compressor { + /** + * Command line tool to compress and uncompress WALs. + */ + public static void main(String[] args) throws IOException { + if (args.length != 2 || args[0].equals("--help") || args[0].equals("-h")) { + printHelp(); + System.exit(-1); + } + + Path inputPath = new Path(args[0]); + Path outputPath = new Path(args[1]); + + transformFile(inputPath, outputPath); + } + + private static void printHelp() { + System.err.println("usage: Compressor "); + System.err.println("If HLog is compressed, will be decompressed."); + System.err.println("If HLog is uncompressed, will be compressed."); + return; + } + + private static void transformFile(Path input, Path output) + throws IOException { + SequenceFileLogReader in = new SequenceFileLogReader(); + SequenceFileLogWriter out = new SequenceFileLogWriter(); + + try { + Configuration conf = HBaseConfiguration.create(); + + FileSystem inFS = input.getFileSystem(conf); + FileSystem outFS = output.getFileSystem(conf); + + in.init(inFS, input, conf); + boolean compress = in.reader.isWALCompressionEnabled(); + + conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, !compress); + out.init(outFS, output, conf); + + Entry e = null; + while ((e = in.next()) != null) out.append(e); + } finally { + in.close(); + out.close(); + } + } + + /** + * Reads the next compressed entry and returns it as a byte array + * + * @param in the DataInput to read from + * @param dict the dictionary we use for our read. + * + * @param the uncompressed array. + */ + static byte[] readCompressed(DataInput in, Dictionary dict) + throws IOException { + byte status = in.readByte(); + + if (status == Dictionary.NOT_IN_DICTIONARY) { + int length = WritableUtils.readVInt(in); + // if this isn't in the dictionary, we need to add to the dictionary. + byte[] arr = new byte[length]; + in.readFully(arr); + if (dict != null) dict.addEntry(arr, 0, length); + return arr; + } else { + // Status here is the higher-order byte of index of the dictionary entry + // (when its not Dictionary.NOT_IN_DICTIONARY -- dictionary indices are + // shorts). + short dictIdx = toShort(status, in.readByte()); + byte[] entry = dict.getEntry(dictIdx); + if (entry == null) { + throw new IOException("Missing dictionary entry for index " + + dictIdx); + } + return entry; + } + } + + /** + * Reads a compressed entry into an array. + * The output into the array ends up length-prefixed. + * + * @param to the array to write into + * @param offset array offset to start writing to + * @param in the DataInput to read from + * @param dict the dictionary to use for compression + * + * @return the length of the uncompressed data + */ + static int uncompressIntoArray(byte[] to, int offset, DataInput in, + Dictionary dict) throws IOException { + byte status = in.readByte(); + + if (status == Dictionary.NOT_IN_DICTIONARY) { + // status byte indicating that data to be read is not in dictionary. + // if this isn't in the dictionary, we need to add to the dictionary. + int length = WritableUtils.readVInt(in); + in.readFully(to, offset, length); + dict.addEntry(to, offset, length); + return length; + } else { + // the status byte also acts as the higher order byte of the dictionary + // entry + short dictIdx = toShort(status, in.readByte()); + byte[] entry; + try { + entry = dict.getEntry(dictIdx); + } catch (Exception ex) { + throw new IOException("Unable to uncompress the log entry", ex); + } + if (entry == null) { + throw new IOException("Missing dictionary entry for index " + + dictIdx); + } + // now we write the uncompressed value. + Bytes.putBytes(to, offset, entry, 0, entry.length); + return entry.length; + } + } + + /** + * Compresses and writes an array to a DataOutput + * + * @param data the array to write. + * @param out the DataOutput to write into + * @param dict the dictionary to use for compression + */ + static void writeCompressed(byte[] data, int offset, int length, + DataOutput out, Dictionary dict) + throws IOException { + short dictIdx = Dictionary.NOT_IN_DICTIONARY; + if (dict != null) { + dictIdx = dict.findEntry(data, offset, length); + } + if (dictIdx == Dictionary.NOT_IN_DICTIONARY) { + // not in dict + out.writeByte(Dictionary.NOT_IN_DICTIONARY); + WritableUtils.writeVInt(out, length); + out.write(data, offset, length); + } else { + out.writeShort(dictIdx); + } + } + + static short toShort(byte hi, byte lo) { + short s = (short) (((hi & 0xFF) << 8) | (lo & 0xFF)); + Preconditions.checkArgument(s >= 0); + return s; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Dictionary.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Dictionary.java new file mode 100644 index 0000000..5dbf3bf --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/Dictionary.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + + +/** + * Dictionary interface + * + * Dictionary indexes should be either bytes or shorts, only positive. (The + * first bit is reserved for detecting whether something is compressed or not). + */ +interface Dictionary { + static final byte NOT_IN_DICTIONARY = -1; + + /** + * Gets an entry from the dictionary. + * + * @param idx index of the entry + * @return the entry, or null if non existent + */ + public byte[] getEntry(short idx); + + /** + * Finds the index of an entry. + * If no entry found, we add it. + * + * @param data the byte array that we're looking up + * @param offset Offset into data to add to Dictionary. + * @param length Length beyond offset that comprises entry; must be > 0. + * @return the index of the entry, or {@link #NOT_IN_DICTIONARY} if not found + */ + public short findEntry(byte[] data, int offset, int length); + + /** + * Adds an entry to the dictionary. + * Be careful using this method. It will add an entry to the + * dictionary even if it already has an entry for the same data. + * Call {{@link #findEntry(byte[], int, int)}} to add without duplicating + * dictionary entries. + * + * @param data the entry to add + * @param offset Offset into data to add to Dictionary. + * @param length Length beyond offset that comprises entry; must be > 0. + * @return the index of the entry + */ + + public short addEntry(byte[] data, int offset, int length); + + /** + * Flushes the dictionary, empties all values. + */ + public void clear(); +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FailedLogCloseException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FailedLogCloseException.java new file mode 100644 index 0000000..393b1d2 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FailedLogCloseException.java @@ -0,0 +1,44 @@ +/** + * Copyright 2008 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; + +/** + * Thrown when we fail close of the write-ahead-log file. + * Package private. Only used inside this package. + */ +public class FailedLogCloseException extends IOException { + private static final long serialVersionUID = 1759152841462990925L; + + /** + * + */ + public FailedLogCloseException() { + super(); + } + + /** + * @param arg0 + */ + public FailedLogCloseException(String arg0) { + super(arg0); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java new file mode 100644 index 0000000..b8d90a2 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java @@ -0,0 +1,1954 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.UUID; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.Syncable; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseFileSystem; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.HasThread; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.util.StringUtils; + +/** + * HLog stores all the edits to the HStore. Its the hbase write-ahead-log + * implementation. + * + * It performs logfile-rolling, so external callers are not aware that the + * underlying file is being rolled. + * + *

+ * There is one HLog per RegionServer. All edits for all Regions carried by + * a particular RegionServer are entered first in the HLog. + * + *

+ * Each HRegion is identified by a unique long int. HRegions do + * not need to declare themselves before using the HLog; they simply include + * their HRegion-id in the append or + * completeCacheFlush calls. + * + *

+ * An HLog consists of multiple on-disk files, which have a chronological order. + * As data is flushed to other (better) on-disk structures, the log becomes + * obsolete. We can destroy all the log messages for a given HRegion-id up to + * the most-recent CACHEFLUSH message from that HRegion. + * + *

+ * It's only practical to delete entire files. Thus, we delete an entire on-disk + * file F when all of the messages in F have a log-sequence-id that's older + * (smaller) than the most-recent CACHEFLUSH message for every HRegion that has + * a message in F. + * + *

+ * Synchronized methods can never execute in parallel. However, between the + * start of a cache flush and the completion point, appends are allowed but log + * rolling is not. To prevent log rolling taking place during this period, a + * separate reentrant lock is used. + * + *

To read an HLog, call {@link #getReader(org.apache.hadoop.fs.FileSystem, + * org.apache.hadoop.fs.Path, org.apache.hadoop.conf.Configuration)}. + * + */ +public class HLog implements Syncable { + static final Log LOG = LogFactory.getLog(HLog.class); + public static final byte [] METAFAMILY = Bytes.toBytes("METAFAMILY"); + static final byte [] METAROW = Bytes.toBytes("METAROW"); + + /** File Extension used while splitting an HLog into regions (HBASE-2312) */ + public static final String SPLITTING_EXT = "-splitting"; + public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false; + /** The META region's HLog filename extension */ + public static final String META_HLOG_FILE_EXTN = ".meta"; + public static final String SEPARATE_HLOG_FOR_META = "hbase.regionserver.separate.hlog.for.meta"; + + /* + * Name of directory that holds recovered edits written by the wal log + * splitting code, one per region + */ + public static final String RECOVERED_EDITS_DIR = "recovered.edits"; + private static final Pattern EDITFILES_NAME_PATTERN = + Pattern.compile("-?[0-9]+"); + public static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp"; + + private final FileSystem fs; + private final Path dir; + private final Configuration conf; + private final HLogFileSystem hlogFs; + // Listeners that are called on WAL events. + private List listeners = + new CopyOnWriteArrayList(); + private final long optionalFlushInterval; + private final long blocksize; + private final String prefix; + private final AtomicLong unflushedEntries = new AtomicLong(0); + private volatile long syncedTillHere = 0; + private long lastDeferredTxid; + private final Path oldLogDir; + private volatile boolean logRollRunning; + + private static Class logWriterClass; + private static Class logReaderClass; + + private WALCoprocessorHost coprocessorHost; + + static void resetLogReaderClass() { + HLog.logReaderClass = null; + } + + private FSDataOutputStream hdfs_out; // FSDataOutputStream associated with the current SequenceFile.writer + // Minimum tolerable replicas, if the actual value is lower than it, + // rollWriter will be triggered + private int minTolerableReplication; + private Method getNumCurrentReplicas; // refers to DFSOutputStream.getNumCurrentReplicas + final static Object [] NO_ARGS = new Object []{}; + + public interface Reader { + void init(FileSystem fs, Path path, Configuration c) throws IOException; + void close() throws IOException; + Entry next() throws IOException; + Entry next(Entry reuse) throws IOException; + void seek(long pos) throws IOException; + long getPosition() throws IOException; + void reset() throws IOException; + } + + public interface Writer { + void init(FileSystem fs, Path path, Configuration c) throws IOException; + void close() throws IOException; + void sync() throws IOException; + void append(Entry entry) throws IOException; + long getLength() throws IOException; + } + + /* + * Current log file. + */ + Writer writer; + + /* + * Map of all log files but the current one. + */ + final SortedMap outputfiles = + Collections.synchronizedSortedMap(new TreeMap()); + + /* + * Map of encoded region names to their most recent sequence/edit id in their + * memstore. + */ + private final ConcurrentSkipListMap lastSeqWritten = + new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR); + + private volatile boolean closed = false; + + private final AtomicLong logSeqNum = new AtomicLong(0); + + private boolean forMeta = false; + + // The timestamp (in ms) when the log file was created. + private volatile long filenum = -1; + + //number of transactions in the current Hlog. + private final AtomicInteger numEntries = new AtomicInteger(0); + + // If live datanode count is lower than the default replicas value, + // RollWriter will be triggered in each sync(So the RollWriter will be + // triggered one by one in a short time). Using it as a workaround to slow + // down the roll frequency triggered by checkLowReplication(). + private volatile int consecutiveLogRolls = 0; + private final int lowReplicationRollLimit; + + // If consecutiveLogRolls is larger than lowReplicationRollLimit, + // then disable the rolling in checkLowReplication(). + // Enable it if the replications recover. + private volatile boolean lowReplicationRollEnabled = true; + + // If > than this size, roll the log. This is typically 0.95 times the size + // of the default Hdfs block size. + private final long logrollsize; + + // This lock prevents starting a log roll during a cache flush. + // synchronized is insufficient because a cache flush spans two method calls. + private final Lock cacheFlushLock = new ReentrantLock(); + + // We synchronize on updateLock to prevent updates and to prevent a log roll + // during an update + // locked during appends + private final Object updateLock = new Object(); + private final Object flushLock = new Object(); + + private final boolean enabled; + + /* + * If more than this many logs, force flush of oldest region to oldest edit + * goes to disk. If too many and we crash, then will take forever replaying. + * Keep the number of logs tidy. + */ + private final int maxLogs; + + /** + * Thread that handles optional sync'ing + */ + private final LogSyncer logSyncer; + + /** Number of log close errors tolerated before we abort */ + private final int closeErrorsTolerated; + + private final AtomicInteger closeErrorCount = new AtomicInteger(); + + /** + * Pattern used to validate a HLog file name + */ + private static final Pattern pattern = + Pattern.compile(".*\\.\\d*("+HLog.META_HLOG_FILE_EXTN+")*"); + + static byte [] COMPLETE_CACHE_FLUSH; + static { + try { + COMPLETE_CACHE_FLUSH = + "HBASE::CACHEFLUSH".getBytes(HConstants.UTF8_ENCODING); + } catch (UnsupportedEncodingException e) { + assert(false); + } + } + + public static class Metric { + public long min = Long.MAX_VALUE; + public long max = 0; + public long total = 0; + public int count = 0; + + synchronized void inc(final long val) { + min = Math.min(min, val); + max = Math.max(max, val); + total += val; + ++count; + } + + synchronized Metric get() { + Metric copy = new Metric(); + copy.min = min; + copy.max = max; + copy.total = total; + copy.count = count; + this.min = Long.MAX_VALUE; + this.max = 0; + this.total = 0; + this.count = 0; + return copy; + } + } + + // For measuring latency of writes + private static Metric writeTime = new Metric(); + private static Metric writeSize = new Metric(); + // For measuring latency of syncs + private static Metric syncTime = new Metric(); + //For measuring slow HLog appends + private static AtomicLong slowHLogAppendCount = new AtomicLong(); + private static Metric slowHLogAppendTime = new Metric(); + + public static Metric getWriteTime() { + return writeTime.get(); + } + + public static Metric getWriteSize() { + return writeSize.get(); + } + + public static Metric getSyncTime() { + return syncTime.get(); + } + + public static long getSlowAppendCount() { + return slowHLogAppendCount.get(); + } + + public static Metric getSlowAppendTime() { + return slowHLogAppendTime.get(); + } + + /** + * Constructor. + * + * @param fs filesystem handle + * @param dir path to where hlogs are stored + * @param oldLogDir path to where hlogs are archived + * @param conf configuration to use + * @throws IOException + */ + public HLog(final FileSystem fs, final Path dir, final Path oldLogDir, + final Configuration conf) + throws IOException { + this(fs, dir, oldLogDir, conf, null, true, null, false); + } + + /** + * Create an edit log at the given dir location. + * + * You should never have to load an existing log. If there is a log at + * startup, it should have already been processed and deleted by the time the + * HLog object is started up. + * + * @param fs filesystem handle + * @param dir path to where hlogs are stored + * @param oldLogDir path to where hlogs are archived + * @param conf configuration to use + * @param listeners Listeners on WAL events. Listeners passed here will + * be registered before we do anything else; e.g. the + * Constructor {@link #rollWriter()}. + * @param prefix should always be hostname and port in distributed env and + * it will be URL encoded before being used. + * If prefix is null, "hlog" will be used + * @throws IOException + */ + public HLog(final FileSystem fs, final Path dir, final Path oldLogDir, + final Configuration conf, final List listeners, + final String prefix) throws IOException { + this(fs, dir, oldLogDir, conf, listeners, true, prefix, false); + } + + /** + * Create an edit log at the given dir location. + * + * You should never have to load an existing log. If there is a log at + * startup, it should have already been processed and deleted by the time the + * HLog object is started up. + * + * @param fs filesystem handle + * @param dir path to where hlogs are stored + * @param oldLogDir path to where hlogs are archived + * @param conf configuration to use + * @param listeners Listeners on WAL events. Listeners passed here will + * be registered before we do anything else; e.g. the + * Constructor {@link #rollWriter()}. + * @param failIfLogDirExists If true IOException will be thrown if dir already exists. + * @param prefix should always be hostname and port in distributed env and + * it will be URL encoded before being used. + * If prefix is null, "hlog" will be used + * @param forMeta if this hlog is meant for meta updates + * @throws IOException + */ + public HLog(final FileSystem fs, final Path dir, final Path oldLogDir, + final Configuration conf, final List listeners, + final boolean failIfLogDirExists, final String prefix, boolean forMeta) + throws IOException { + super(); + this.fs = fs; + this.dir = dir; + this.conf = conf; + this.hlogFs = new HLogFileSystem(conf); + if (listeners != null) { + for (WALActionsListener i: listeners) { + registerWALActionsListener(i); + } + } + this.blocksize = conf.getLong("hbase.regionserver.hlog.blocksize", + FSUtils.getDefaultBlockSize(this.fs, this.dir)); + // Roll at 95% of block size. + float multi = conf.getFloat("hbase.regionserver.logroll.multiplier", 0.95f); + this.logrollsize = (long)(this.blocksize * multi); + this.optionalFlushInterval = + conf.getLong("hbase.regionserver.optionallogflushinterval", 1 * 1000); + boolean dirExists = false; + if (failIfLogDirExists && (dirExists = this.fs.exists(dir))) { + throw new IOException("Target HLog directory already exists: " + dir); + } + if (!dirExists && !HBaseFileSystem.makeDirOnFileSystem(fs, dir)) { + throw new IOException("Unable to mkdir " + dir); + } + this.oldLogDir = oldLogDir; + if (!fs.exists(oldLogDir) && !HBaseFileSystem.makeDirOnFileSystem(fs, oldLogDir)) { + throw new IOException("Unable to mkdir " + this.oldLogDir); + } + this.forMeta = forMeta; + this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 32); + this.minTolerableReplication = conf.getInt( + "hbase.regionserver.hlog.tolerable.lowreplication", + FSUtils.getDefaultReplication(this.fs, this.dir)); + this.lowReplicationRollLimit = conf.getInt( + "hbase.regionserver.hlog.lowreplication.rolllimit", 5); + this.enabled = conf.getBoolean("hbase.regionserver.hlog.enabled", true); + this.closeErrorsTolerated = conf.getInt( + "hbase.regionserver.logroll.errors.tolerated", 0); + + LOG.info("HLog configuration: blocksize=" + + StringUtils.byteDesc(this.blocksize) + + ", rollsize=" + StringUtils.byteDesc(this.logrollsize) + + ", enabled=" + this.enabled + + ", optionallogflushinternal=" + this.optionalFlushInterval + "ms"); + // If prefix is null||empty then just name it hlog + this.prefix = prefix == null || prefix.isEmpty() ? + "hlog" : URLEncoder.encode(prefix, "UTF8"); + // rollWriter sets this.hdfs_out if it can. + rollWriter(); + + // handle the reflection necessary to call getNumCurrentReplicas() + this.getNumCurrentReplicas = getGetNumCurrentReplicas(this.hdfs_out); + + logSyncer = new LogSyncer(this.optionalFlushInterval); + // When optionalFlushInterval is set as 0, don't start a thread for deferred log sync. + if (this.optionalFlushInterval > 0) { + Threads.setDaemonThreadRunning(logSyncer.getThread(), Thread.currentThread().getName() + + ".logSyncer"); + } else { + LOG.info("hbase.regionserver.optionallogflushinterval is set as " + + this.optionalFlushInterval + ". Deferred log syncing won't work. " + + "Any Mutation, marked to be deferred synced, will be flushed immediately."); + } + coprocessorHost = new WALCoprocessorHost(this, conf); + } + + /** + * Find the 'getNumCurrentReplicas' on the passed os stream. + * @return Method or null. + */ + private Method getGetNumCurrentReplicas(final FSDataOutputStream os) { + Method m = null; + if (os != null) { + Class wrappedStreamClass = os.getWrappedStream() + .getClass(); + try { + m = wrappedStreamClass.getDeclaredMethod("getNumCurrentReplicas", + new Class[] {}); + m.setAccessible(true); + } catch (NoSuchMethodException e) { + LOG.info("FileSystem's output stream doesn't support" + + " getNumCurrentReplicas; --HDFS-826 not available; fsOut=" + + wrappedStreamClass.getName()); + } catch (SecurityException e) { + LOG.info("Doesn't have access to getNumCurrentReplicas on " + + "FileSystems's output stream --HDFS-826 not available; fsOut=" + + wrappedStreamClass.getName(), e); + m = null; // could happen on setAccessible() + } + } + if (m != null) { + LOG.info("Using getNumCurrentReplicas--HDFS-826"); + } + return m; + } + + public void registerWALActionsListener(final WALActionsListener listener) { + this.listeners.add(listener); + } + + public boolean unregisterWALActionsListener(final WALActionsListener listener) { + return this.listeners.remove(listener); + } + + /** + * @return Current state of the monotonically increasing file id. + */ + public long getFilenum() { + return this.filenum; + } + + /** + * Called by HRegionServer when it opens a new region to ensure that log + * sequence numbers are always greater than the latest sequence number of the + * region being brought on-line. + * + * @param newvalue We'll set log edit/sequence number to this value if it + * is greater than the current value. + */ + public void setSequenceNumber(final long newvalue) { + for (long id = this.logSeqNum.get(); id < newvalue && + !this.logSeqNum.compareAndSet(id, newvalue); id = this.logSeqNum.get()) { + // This could spin on occasion but better the occasional spin than locking + // every increment of sequence number. + LOG.debug("Changed sequenceid from " + logSeqNum + " to " + newvalue); + } + } + + /** + * @return log sequence number + */ + public long getSequenceNumber() { + return logSeqNum.get(); + } + + /** + * Method used internal to this class and for tests only. + * @return The wrapped stream our writer is using; its not the + * writer's 'out' FSDatoOutputStream but the stream that this 'out' wraps + * (In hdfs its an instance of DFSDataOutputStream). + */ + // usage: see TestLogRolling.java + OutputStream getOutputStream() { + return this.hdfs_out.getWrappedStream(); + } + + /** + * Roll the log writer. That is, start writing log messages to a new file. + * + * Because a log cannot be rolled during a cache flush, and a cache flush + * spans two method calls, a special lock needs to be obtained so that a cache + * flush cannot start when the log is being rolled and the log cannot be + * rolled during a cache flush. + * + *

Note that this method cannot be synchronized because it is possible that + * startCacheFlush runs, obtaining the cacheFlushLock, then this method could + * start which would obtain the lock on this but block on obtaining the + * cacheFlushLock and then completeCacheFlush could be called which would wait + * for the lock on this and consequently never release the cacheFlushLock + * + * @return If lots of logs, flush the returned regions so next time through + * we can clean logs. Returns null if nothing to flush. Names are actual + * region names as returned by {@link HRegionInfo#getEncodedName()} + * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException + * @throws IOException + */ + public byte [][] rollWriter() throws FailedLogCloseException, IOException { + return rollWriter(false); + } + + /** + * Roll the log writer. That is, start writing log messages to a new file. + * + * Because a log cannot be rolled during a cache flush, and a cache flush + * spans two method calls, a special lock needs to be obtained so that a cache + * flush cannot start when the log is being rolled and the log cannot be + * rolled during a cache flush. + * + *

Note that this method cannot be synchronized because it is possible that + * startCacheFlush runs, obtaining the cacheFlushLock, then this method could + * start which would obtain the lock on this but block on obtaining the + * cacheFlushLock and then completeCacheFlush could be called which would wait + * for the lock on this and consequently never release the cacheFlushLock + * + * @param force If true, force creation of a new writer even if no entries + * have been written to the current writer + * @return If lots of logs, flush the returned regions so next time through + * we can clean logs. Returns null if nothing to flush. Names are actual + * region names as returned by {@link HRegionInfo#getEncodedName()} + * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException + * @throws IOException + */ + public byte [][] rollWriter(boolean force) + throws FailedLogCloseException, IOException { + // Return if nothing to flush. + if (!force && this.writer != null && this.numEntries.get() <= 0) { + return null; + } + byte [][] regionsToFlush = null; + this.cacheFlushLock.lock(); + this.logRollRunning = true; + try { + if (closed) { + LOG.debug("HLog closed. Skipping rolling of writer"); + return regionsToFlush; + } + // Do all the preparation outside of the updateLock to block + // as less as possible the incoming writes + long currentFilenum = this.filenum; + Path oldPath = null; + if (currentFilenum > 0) { + //computeFilename will take care of meta hlog filename + oldPath = computeFilename(currentFilenum); + } + this.filenum = System.currentTimeMillis(); + Path newPath = computeFilename(); + + // Tell our listeners that a new log is about to be created + if (!this.listeners.isEmpty()) { + for (WALActionsListener i : this.listeners) { + i.preLogRoll(oldPath, newPath); + } + } + HLog.Writer nextWriter = this.createWriterInstance(fs, newPath, conf); + // Can we get at the dfsclient outputstream? If an instance of + // SFLW, it'll have done the necessary reflection to get at the + // protected field name. + FSDataOutputStream nextHdfsOut = null; + if (nextWriter instanceof SequenceFileLogWriter) { + nextHdfsOut = ((SequenceFileLogWriter)nextWriter).getWriterFSDataOutputStream(); + } + + synchronized (updateLock) { + // Clean up current writer. + Path oldFile = cleanupCurrentWriter(currentFilenum); + this.writer = nextWriter; + this.hdfs_out = nextHdfsOut; + + LOG.info((oldFile != null? + "Roll " + FSUtils.getPath(oldFile) + ", entries=" + + this.numEntries.get() + + ", filesize=" + + this.fs.getFileStatus(oldFile).getLen() + ". ": "") + + " for " + FSUtils.getPath(newPath)); + this.numEntries.set(0); + } + // Tell our listeners that a new log was created + if (!this.listeners.isEmpty()) { + for (WALActionsListener i : this.listeners) { + i.postLogRoll(oldPath, newPath); + } + } + + // Can we delete any of the old log files? + if (this.outputfiles.size() > 0) { + if (this.lastSeqWritten.isEmpty()) { + LOG.debug("Last sequenceid written is empty. Deleting all old hlogs"); + // If so, then no new writes have come in since all regions were + // flushed (and removed from the lastSeqWritten map). Means can + // remove all but currently open log file. + for (Map.Entry e : this.outputfiles.entrySet()) { + archiveLogFile(e.getValue(), e.getKey()); + } + this.outputfiles.clear(); + } else { + regionsToFlush = cleanOldLogs(); + } + } + } finally { + this.logRollRunning = false; + this.cacheFlushLock.unlock(); + } + return regionsToFlush; + } + + /** + * This method allows subclasses to inject different writers without having to + * extend other methods like rollWriter(). + * + * @param fs + * @param path + * @param conf + * @return Writer instance + * @throws IOException + */ + protected Writer createWriterInstance(final FileSystem fs, final Path path, + final Configuration conf) throws IOException { + if (forMeta) { + //TODO: set a higher replication for the hlog files (HBASE-6773) + } + return this.hlogFs.createWriter(fs, conf, path); + } + + /** + * Get a reader for the WAL. + * The proper way to tail a log that can be under construction is to first use this method + * to get a reader then call {@link HLog.Reader#reset()} to see the new data. It will also + * take care of keeping implementation-specific context (like compression). + * @param fs + * @param path + * @param conf + * @return A WAL reader. Close when done with it. + * @throws IOException + */ + public static Reader getReader(final FileSystem fs, final Path path, + Configuration conf) + throws IOException { + try { + + if (logReaderClass == null) { + + logReaderClass = conf.getClass("hbase.regionserver.hlog.reader.impl", + SequenceFileLogReader.class, Reader.class); + } + + + HLog.Reader reader = logReaderClass.newInstance(); + reader.init(fs, path, conf); + return reader; + } catch (IOException e) { + throw e; + } + catch (Exception e) { + throw new IOException("Cannot get log reader", e); + } + } + + /** + * Get a writer for the WAL. + * @param path + * @param conf + * @return A WAL writer. Close when done with it. + * @throws IOException + */ + public static Writer createWriter(final FileSystem fs, + final Path path, Configuration conf) + throws IOException { + try { + if (logWriterClass == null) { + logWriterClass = conf.getClass("hbase.regionserver.hlog.writer.impl", + SequenceFileLogWriter.class, Writer.class); + } + HLog.Writer writer = (HLog.Writer) logWriterClass.newInstance(); + writer.init(fs, path, conf); + return writer; + } catch (Exception e) { + throw new IOException("cannot get log writer", e); + } + } + + /* + * Clean up old commit logs. + * @return If lots of logs, flush the returned region so next time through + * we can clean logs. Returns null if nothing to flush. Returns array of + * encoded region names to flush. + * @throws IOException + */ + private byte [][] cleanOldLogs() throws IOException { + Long oldestOutstandingSeqNum = getOldestOutstandingSeqNum(); + // Get the set of all log files whose last sequence number is smaller than + // the oldest edit's sequence number. + TreeSet sequenceNumbers = + new TreeSet(this.outputfiles.headMap( + (Long.valueOf(oldestOutstandingSeqNum.longValue()))).keySet()); + // Now remove old log files (if any) + int logsToRemove = sequenceNumbers.size(); + if (logsToRemove > 0) { + if (LOG.isDebugEnabled()) { + // Find associated region; helps debugging. + byte [] oldestRegion = getOldestRegion(oldestOutstandingSeqNum); + LOG.debug("Found " + logsToRemove + " hlogs to remove" + + " out of total " + this.outputfiles.size() + ";" + + " oldest outstanding sequenceid is " + oldestOutstandingSeqNum + + " from region " + Bytes.toStringBinary(oldestRegion)); + } + for (Long seq : sequenceNumbers) { + archiveLogFile(this.outputfiles.remove(seq), seq); + } + } + + // If too many log files, figure which regions we need to flush. + // Array is an array of encoded region names. + byte [][] regions = null; + int logCount = this.outputfiles == null? 0: this.outputfiles.size(); + if (logCount > this.maxLogs && logCount > 0) { + // This is an array of encoded region names. + regions = findMemstoresWithEditsEqualOrOlderThan(this.outputfiles.firstKey(), + this.lastSeqWritten); + if (regions != null) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < regions.length; i++) { + if (i > 0) sb.append(", "); + sb.append(Bytes.toStringBinary(regions[i])); + } + LOG.info("Too many hlogs: logs=" + logCount + ", maxlogs=" + + this.maxLogs + "; forcing flush of " + regions.length + " regions(s): " + + sb.toString()); + } + } + return regions; + } + + /** + * Return regions (memstores) that have edits that are equal or less than + * the passed oldestWALseqid. + * @param oldestWALseqid + * @param regionsToSeqids Encoded region names to sequence ids + * @return All regions whose seqid is < than oldestWALseqid (Not + * necessarily in order). Null if no regions found. + */ + static byte [][] findMemstoresWithEditsEqualOrOlderThan(final long oldestWALseqid, + final Map regionsToSeqids) { + // This method is static so it can be unit tested the easier. + List regions = null; + for (Map.Entry e: regionsToSeqids.entrySet()) { + if (e.getValue().longValue() <= oldestWALseqid) { + if (regions == null) regions = new ArrayList(); + // Key is encoded region name. + regions.add(e.getKey()); + } + } + return regions == null? + null: regions.toArray(new byte [][] {HConstants.EMPTY_BYTE_ARRAY}); + } + + /* + * @return Logs older than this id are safe to remove. + */ + private Long getOldestOutstandingSeqNum() { + return Collections.min(this.lastSeqWritten.values()); + } + + /** + * @param oldestOutstandingSeqNum + * @return (Encoded) name of oldest outstanding region. + */ + private byte [] getOldestRegion(final Long oldestOutstandingSeqNum) { + byte [] oldestRegion = null; + for (Map.Entry e: this.lastSeqWritten.entrySet()) { + if (e.getValue().longValue() == oldestOutstandingSeqNum.longValue()) { + // Key is encoded region name. + oldestRegion = e.getKey(); + break; + } + } + return oldestRegion; + } + + /* + * Cleans up current writer closing and adding to outputfiles. + * Presumes we're operating inside an updateLock scope. + * @return Path to current writer or null if none. + * @throws IOException + */ + Path cleanupCurrentWriter(final long currentfilenum) throws IOException { + Path oldFile = null; + if (this.writer != null) { + // Close the current writer, get a new one. + try { + // Wait till all current transactions are written to the hlog. + // No new transactions can occur because we have the updatelock. + if (this.unflushedEntries.get() != this.syncedTillHere) { + LOG.debug("cleanupCurrentWriter " + + " waiting for transactions to get synced " + + " total " + this.unflushedEntries.get() + + " synced till here " + syncedTillHere); + sync(); + } + this.writer.close(); + this.writer = null; + closeErrorCount.set(0); + } catch (IOException e) { + LOG.error("Failed close of HLog writer", e); + int errors = closeErrorCount.incrementAndGet(); + if (errors <= closeErrorsTolerated && !hasDeferredEntries()) { + LOG.warn("Riding over HLog close failure! error count="+errors); + } else { + if (hasDeferredEntries()) { + LOG.error("Aborting due to unflushed edits in HLog"); + } + // Failed close of log file. Means we're losing edits. For now, + // shut ourselves down to minimize loss. Alternative is to try and + // keep going. See HBASE-930. + FailedLogCloseException flce = + new FailedLogCloseException("#" + currentfilenum); + flce.initCause(e); + throw flce; + } + } + if (currentfilenum >= 0) { + oldFile = computeFilename(currentfilenum); + this.outputfiles.put(Long.valueOf(this.logSeqNum.get()), oldFile); + } + } + return oldFile; + } + + private void archiveLogFile(final Path p, final Long seqno) throws IOException { + Path newPath = getHLogArchivePath(this.oldLogDir, p); + LOG.info("moving old hlog file " + FSUtils.getPath(p) + + " whose highest sequenceid is " + seqno + " to " + + FSUtils.getPath(newPath)); + + // Tell our listeners that a log is going to be archived. + if (!this.listeners.isEmpty()) { + for (WALActionsListener i : this.listeners) { + i.preLogArchive(p, newPath); + } + } + if (!HBaseFileSystem.renameDirForFileSystem(fs, p, newPath)) { + throw new IOException("Unable to rename " + p + " to " + newPath); + } + // Tell our listeners that a log has been archived. + if (!this.listeners.isEmpty()) { + for (WALActionsListener i : this.listeners) { + i.postLogArchive(p, newPath); + } + } + } + + /** + * This is a convenience method that computes a new filename with a given + * using the current HLog file-number + * @return Path + */ + protected Path computeFilename() { + return computeFilename(this.filenum); + } + + /** + * This is a convenience method that computes a new filename with a given + * file-number. + * @param filenum to use + * @return Path + */ + protected Path computeFilename(long filenum) { + if (filenum < 0) { + throw new RuntimeException("hlog file number can't be < 0"); + } + String child = prefix + "." + filenum; + if (forMeta) { + child += HLog.META_HLOG_FILE_EXTN; + } + return new Path(dir, child); + } + + public static boolean isMetaFile(Path p) { + if (p.getName().endsWith(HLog.META_HLOG_FILE_EXTN)) { + return true; + } + return false; + } + + /** + * Shut down the log and delete the log directory + * + * @throws IOException + */ + public void closeAndDelete() throws IOException { + close(); + if (!fs.exists(this.dir)) return; + FileStatus[] files = fs.listStatus(this.dir); + for(FileStatus file : files) { + + Path p = getHLogArchivePath(this.oldLogDir, file.getPath()); + // Tell our listeners that a log is going to be archived. + if (!this.listeners.isEmpty()) { + for (WALActionsListener i : this.listeners) { + i.preLogArchive(file.getPath(), p); + } + } + + if (!HBaseFileSystem.renameDirForFileSystem(fs, file.getPath(), p)) { + throw new IOException("Unable to rename " + file.getPath() + " to " + p); + } + // Tell our listeners that a log was archived. + if (!this.listeners.isEmpty()) { + for (WALActionsListener i : this.listeners) { + i.postLogArchive(file.getPath(), p); + } + } + } + LOG.debug("Moved " + files.length + " log files to " + + FSUtils.getPath(this.oldLogDir)); + if (!HBaseFileSystem.deleteDirFromFileSystem(fs, dir)) { + LOG.info("Unable to delete " + dir); + } + } + + /** + * Shut down the log. + * + * @throws IOException + */ + public void close() throws IOException { + // When optionalFlushInterval is 0, the logSyncer is not started as a Thread. + if (this.optionalFlushInterval > 0) { + try { + logSyncer.close(); + // Make sure we synced everything + logSyncer.join(this.optionalFlushInterval * 2); + } catch (InterruptedException e) { + LOG.error("Exception while waiting for syncer thread to die", e); + } + } + + cacheFlushLock.lock(); + try { + // Tell our listeners that the log is closing + if (!this.listeners.isEmpty()) { + for (WALActionsListener i : this.listeners) { + i.logCloseRequested(); + } + } + synchronized (updateLock) { + this.closed = true; + if (LOG.isDebugEnabled()) { + LOG.debug("closing hlog writer in " + this.dir.toString()); + } + if (this.writer != null) { + this.writer.close(); + } + } + } finally { + cacheFlushLock.unlock(); + } + } + + /** + * @param now + * @param regionName + * @param tableName + * @param clusterId + * @return New log key. + */ + protected HLogKey makeKey(byte[] regionName, byte[] tableName, long seqnum, + long now, UUID clusterId) { + return new HLogKey(regionName, tableName, seqnum, now, clusterId); + } + + + /** Append an entry to the log. + * + * @param regionInfo + * @param logEdit + * @param logKey + * @param doSync shall we sync after writing the transaction + * @return The txid of this transaction + * @throws IOException + */ + public long append(HRegionInfo regionInfo, HLogKey logKey, WALEdit logEdit, + HTableDescriptor htd, boolean doSync) + throws IOException { + if (this.closed) { + throw new IOException("Cannot append; log is closed"); + } + long txid = 0; + synchronized (updateLock) { + long seqNum = obtainSeqNum(); + logKey.setLogSeqNum(seqNum); + // The 'lastSeqWritten' map holds the sequence number of the oldest + // write for each region (i.e. the first edit added to the particular + // memstore). When the cache is flushed, the entry for the + // region being flushed is removed if the sequence number of the flush + // is greater than or equal to the value in lastSeqWritten. + this.lastSeqWritten.putIfAbsent(regionInfo.getEncodedNameAsBytes(), + Long.valueOf(seqNum)); + doWrite(regionInfo, logKey, logEdit, htd); + txid = this.unflushedEntries.incrementAndGet(); + this.numEntries.incrementAndGet(); + if (htd.isDeferredLogFlush()) { + lastDeferredTxid = txid; + } + } + + // Sync if catalog region, and if not then check if that table supports + // deferred log flushing + if (doSync && + (regionInfo.isMetaRegion() || + !htd.isDeferredLogFlush())) { + // sync txn to file system + this.sync(txid); + } + return txid; + } + + /** + * Only used in tests. + * + * @param info + * @param tableName + * @param edits + * @param now + * @param htd + * @throws IOException + */ + public void append(HRegionInfo info, byte [] tableName, WALEdit edits, + final long now, HTableDescriptor htd) + throws IOException { + append(info, tableName, edits, HConstants.DEFAULT_CLUSTER_ID, now, htd); + } + + /** + * Append a set of edits to the log. Log edits are keyed by (encoded) + * regionName, rowname, and log-sequence-id. + * + * Later, if we sort by these keys, we obtain all the relevant edits for a + * given key-range of the HRegion (TODO). Any edits that do not have a + * matching COMPLETE_CACHEFLUSH message can be discarded. + * + *

+ * Logs cannot be restarted once closed, or once the HLog process dies. Each + * time the HLog starts, it must create a new log. This means that other + * systems should process the log appropriately upon each startup (and prior + * to initializing HLog). + * + * synchronized prevents appends during the completion of a cache flush or for + * the duration of a log roll. + * + * @param info + * @param tableName + * @param edits + * @param clusterId The originating clusterId for this edit (for replication) + * @param now + * @param doSync shall we sync? + * @return txid of this transaction + * @throws IOException + */ + private long append(HRegionInfo info, byte [] tableName, WALEdit edits, UUID clusterId, + final long now, HTableDescriptor htd, boolean doSync) + throws IOException { + if (edits.isEmpty()) return this.unflushedEntries.get();; + if (this.closed) { + throw new IOException("Cannot append; log is closed"); + } + long txid = 0; + synchronized (this.updateLock) { + long seqNum = obtainSeqNum(); + // The 'lastSeqWritten' map holds the sequence number of the oldest + // write for each region (i.e. the first edit added to the particular + // memstore). . When the cache is flushed, the entry for the + // region being flushed is removed if the sequence number of the flush + // is greater than or equal to the value in lastSeqWritten. + // Use encoded name. Its shorter, guaranteed unique and a subset of + // actual name. + byte [] encodedRegionName = info.getEncodedNameAsBytes(); + this.lastSeqWritten.putIfAbsent(encodedRegionName, seqNum); + HLogKey logKey = makeKey(encodedRegionName, tableName, seqNum, now, clusterId); + doWrite(info, logKey, edits, htd); + this.numEntries.incrementAndGet(); + txid = this.unflushedEntries.incrementAndGet(); + if (htd.isDeferredLogFlush()) { + lastDeferredTxid = txid; + } + } + // Sync if catalog region, and if not then check if that table supports + // deferred log flushing + if (doSync && + (info.isMetaRegion() || + !htd.isDeferredLogFlush())) { + // sync txn to file system + this.sync(txid); + } + return txid; + } + + /** + * Append a set of edits to the log. Log edits are keyed by (encoded) + * regionName, rowname, and log-sequence-id. The HLog is not flushed + * after this transaction is written to the log. + * + * @param info + * @param tableName + * @param edits + * @param clusterId The originating clusterId for this edit (for replication) + * @param now + * @return txid of this transaction + * @throws IOException + */ + public long appendNoSync(HRegionInfo info, byte [] tableName, WALEdit edits, + UUID clusterId, final long now, HTableDescriptor htd) + throws IOException { + return append(info, tableName, edits, clusterId, now, htd, false); + } + + /** + * Append a set of edits to the log. Log edits are keyed by (encoded) + * regionName, rowname, and log-sequence-id. The HLog is flushed + * after this transaction is written to the log. + * + * @param info + * @param tableName + * @param edits + * @param clusterId The originating clusterId for this edit (for replication) + * @param now + * @return txid of this transaction + * @throws IOException + */ + public long append(HRegionInfo info, byte [] tableName, WALEdit edits, + UUID clusterId, final long now, HTableDescriptor htd) + throws IOException { + return append(info, tableName, edits, clusterId, now, htd, true); + } + + /** + * This class is responsible to hold the HLog's appended Entry list + * and to sync them according to a configurable interval. + * + * Deferred log flushing works first by piggy backing on this process by + * simply not sync'ing the appended Entry. It can also be sync'd by other + * non-deferred log flushed entries outside of this thread. + */ + class LogSyncer extends HasThread { + + private final long optionalFlushInterval; + + private AtomicBoolean closeLogSyncer = new AtomicBoolean(false); + + // List of pending writes to the HLog. There corresponds to transactions + // that have not yet returned to the client. We keep them cached here + // instead of writing them to HDFS piecemeal, because the HDFS write + // method is pretty heavyweight as far as locking is concerned. The + // goal is to increase the batchsize for writing-to-hdfs as well as + // sync-to-hdfs, so that we can get better system throughput. + private List pendingWrites = new LinkedList(); + + LogSyncer(long optionalFlushInterval) { + this.optionalFlushInterval = optionalFlushInterval; + } + + @Override + public void run() { + try { + // awaiting with a timeout doesn't always + // throw exceptions on interrupt + while(!this.isInterrupted() && !closeLogSyncer.get()) { + + try { + if (unflushedEntries.get() <= syncedTillHere) { + synchronized (closeLogSyncer) { + closeLogSyncer.wait(this.optionalFlushInterval); + } + } + // Calling sync since we waited or had unflushed entries. + // Entries appended but not sync'd are taken care of here AKA + // deferred log flush + sync(); + } catch (IOException e) { + LOG.error("Error while syncing, requesting close of hlog ", e); + requestLogRoll(); + } + } + } catch (InterruptedException e) { + LOG.debug(getName() + " interrupted while waiting for sync requests"); + } finally { + LOG.info(getName() + " exiting"); + } + } + + // appends new writes to the pendingWrites. It is better to keep it in + // our own queue rather than writing it to the HDFS output stream because + // HDFSOutputStream.writeChunk is not lightweight at all. + synchronized void append(Entry e) throws IOException { + pendingWrites.add(e); + } + + // Returns all currently pending writes. New writes + // will accumulate in a new list. + synchronized List getPendingWrites() { + List save = this.pendingWrites; + this.pendingWrites = new LinkedList(); + return save; + } + + // writes out pending entries to the HLog + void hlogFlush(Writer writer, List pending) throws IOException { + if (pending == null) return; + + // write out all accumulated Entries to hdfs. + for (Entry e : pending) { + writer.append(e); + } + } + + void close() { + synchronized (closeLogSyncer) { + closeLogSyncer.set(true); + closeLogSyncer.notifyAll(); + } + } + } + + // sync all known transactions + private void syncer() throws IOException { + syncer(this.unflushedEntries.get()); // sync all pending items + } + + // sync all transactions upto the specified txid + private void syncer(long txid) throws IOException { + // if the transaction that we are interested in is already + // synced, then return immediately. + if (txid <= this.syncedTillHere) { + return; + } + Writer tempWriter; + synchronized (this.updateLock) { + if (this.closed) return; + tempWriter = this.writer; // guaranteed non-null + } + try { + long doneUpto; + long now = System.currentTimeMillis(); + // First flush all the pending writes to HDFS. Then + // issue the sync to HDFS. If sync is successful, then update + // syncedTillHere to indicate that transactions till this + // number has been successfully synced. + IOException ioe = null; + List pending = null; + synchronized (flushLock) { + if (txid <= this.syncedTillHere) { + return; + } + doneUpto = this.unflushedEntries.get(); + pending = logSyncer.getPendingWrites(); + try { + logSyncer.hlogFlush(tempWriter, pending); + } catch(IOException io) { + ioe = io; + LOG.error("syncer encountered error, will retry. txid=" + txid, ioe); + } + } + if (ioe != null && pending != null) { + synchronized (this.updateLock) { + synchronized (flushLock) { + // HBASE-4387, HBASE-5623, retry with updateLock held + tempWriter = this.writer; + logSyncer.hlogFlush(tempWriter, pending); + } + } + } + // another thread might have sync'ed avoid double-sync'ing + if (txid <= this.syncedTillHere) { + return; + } + try { + tempWriter.sync(); + } catch (IOException io) { + synchronized (this.updateLock) { + // HBASE-4387, HBASE-5623, retry with updateLock held + tempWriter = this.writer; + tempWriter.sync(); + } + } + this.syncedTillHere = Math.max(this.syncedTillHere, doneUpto); + + syncTime.inc(System.currentTimeMillis() - now); + if (!this.logRollRunning) { + checkLowReplication(); + try { + if (tempWriter.getLength() > this.logrollsize) { + requestLogRoll(); + } + } catch (IOException x) { + LOG.debug("Log roll failed and will be retried. (This is not an error)"); + } + } + } catch (IOException e) { + LOG.fatal("Could not sync. Requesting close of hlog", e); + requestLogRoll(); + throw e; + } + } + + private void checkLowReplication() { + // if the number of replicas in HDFS has fallen below the configured + // value, then roll logs. + try { + int numCurrentReplicas = getLogReplication(); + if (numCurrentReplicas != 0 + && numCurrentReplicas < this.minTolerableReplication) { + if (this.lowReplicationRollEnabled) { + if (this.consecutiveLogRolls < this.lowReplicationRollLimit) { + LOG.warn("HDFS pipeline error detected. " + "Found " + + numCurrentReplicas + " replicas but expecting no less than " + + this.minTolerableReplication + " replicas. " + + " Requesting close of hlog."); + requestLogRoll(); + // If rollWriter is requested, increase consecutiveLogRolls. Once it + // is larger than lowReplicationRollLimit, disable the + // LowReplication-Roller + this.consecutiveLogRolls++; + } else { + LOG.warn("Too many consecutive RollWriter requests, it's a sign of " + + "the total number of live datanodes is lower than the tolerable replicas."); + this.consecutiveLogRolls = 0; + this.lowReplicationRollEnabled = false; + } + } + } else if (numCurrentReplicas >= this.minTolerableReplication) { + + if (!this.lowReplicationRollEnabled) { + // The new writer's log replicas is always the default value. + // So we should not enable LowReplication-Roller. If numEntries + // is lower than or equals 1, we consider it as a new writer. + if (this.numEntries.get() <= 1) { + return; + } + // Once the live datanode number and the replicas return to normal, + // enable the LowReplication-Roller. + this.lowReplicationRollEnabled = true; + LOG.info("LowReplication-Roller was enabled."); + } + } + } catch (Exception e) { + LOG.warn("Unable to invoke DFSOutputStream.getNumCurrentReplicas" + e + + " still proceeding ahead..."); + } + } + + /** + * This method gets the datanode replication count for the current HLog. + * + * If the pipeline isn't started yet or is empty, you will get the default + * replication factor. Therefore, if this function returns 0, it means you + * are not properly running with the HDFS-826 patch. + * @throws InvocationTargetException + * @throws IllegalAccessException + * @throws IllegalArgumentException + * + * @throws Exception + */ + int getLogReplication() + throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { + if (this.getNumCurrentReplicas != null && this.hdfs_out != null) { + Object repl = this.getNumCurrentReplicas.invoke(getOutputStream(), NO_ARGS); + if (repl instanceof Integer) { + return ((Integer)repl).intValue(); + } + } + return 0; + } + + boolean canGetCurReplicas() { + return this.getNumCurrentReplicas != null; + } + + public void hsync() throws IOException { + syncer(); + } + + public void hflush() throws IOException { + syncer(); + } + + public void sync() throws IOException { + syncer(); + } + + public void sync(long txid) throws IOException { + syncer(txid); + } + + private void requestLogRoll() { + if (!this.listeners.isEmpty()) { + for (WALActionsListener i: this.listeners) { + i.logRollRequested(); + } + } + } + + protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit, + HTableDescriptor htd) + throws IOException { + if (!this.enabled) { + return; + } + if (!this.listeners.isEmpty()) { + for (WALActionsListener i: this.listeners) { + i.visitLogEntryBeforeWrite(htd, logKey, logEdit); + } + } + try { + long now = System.currentTimeMillis(); + // coprocessor hook: + if (!coprocessorHost.preWALWrite(info, logKey, logEdit)) { + // write to our buffer for the Hlog file. + logSyncer.append(new HLog.Entry(logKey, logEdit)); + } + long took = System.currentTimeMillis() - now; + coprocessorHost.postWALWrite(info, logKey, logEdit); + writeTime.inc(took); + long len = 0; + for (KeyValue kv : logEdit.getKeyValues()) { + len += kv.getLength(); + } + writeSize.inc(len); + if (took > 1000) { + LOG.warn(String.format( + "%s took %d ms appending an edit to hlog; editcount=%d, len~=%s", + Thread.currentThread().getName(), took, this.numEntries.get(), + StringUtils.humanReadableInt(len))); + slowHLogAppendCount.incrementAndGet(); + slowHLogAppendTime.inc(took); + } + } catch (IOException e) { + LOG.fatal("Could not append. Requesting close of hlog", e); + requestLogRoll(); + throw e; + } + } + + + /** @return How many items have been added to the log */ + int getNumEntries() { + return numEntries.get(); + } + + /** + * Obtain a log sequence number. + */ + private long obtainSeqNum() { + return this.logSeqNum.incrementAndGet(); + } + + /** @return the number of log files in use */ + int getNumLogFiles() { + return outputfiles.size(); + } + + private byte[] getSnapshotName(byte[] encodedRegionName) { + byte snp[] = new byte[encodedRegionName.length + 3]; + // an encoded region name has only hex digits. s, n or p are not hex + // and therefore snapshot-names will never collide with + // encoded-region-names + snp[0] = 's'; snp[1] = 'n'; snp[2] = 'p'; + for (int i = 0; i < encodedRegionName.length; i++) { + snp[i+3] = encodedRegionName[i]; + } + return snp; + } + + /** + * By acquiring a log sequence ID, we can allow log messages to continue while + * we flush the cache. + * + * Acquire a lock so that we do not roll the log between the start and + * completion of a cache-flush. Otherwise the log-seq-id for the flush will + * not appear in the correct logfile. + * + * Ensuring that flushes and log-rolls don't happen concurrently also allows + * us to temporarily put a log-seq-number in lastSeqWritten against the region + * being flushed that might not be the earliest in-memory log-seq-number for + * that region. By the time the flush is completed or aborted and before the + * cacheFlushLock is released it is ensured that lastSeqWritten again has the + * oldest in-memory edit's lsn for the region that was being flushed. + * + * In this method, by removing the entry in lastSeqWritten for the region + * being flushed we ensure that the next edit inserted in this region will be + * correctly recorded in {@link #append(HRegionInfo, byte[], WALEdit, long, HTableDescriptor)} The + * lsn of the earliest in-memory lsn - which is now in the memstore snapshot - + * is saved temporarily in the lastSeqWritten map while the flush is active. + * + * @return sequence ID to pass + * {@link #completeCacheFlush(byte[], byte[], long, boolean)} (byte[], + * byte[], long)} + * @see #completeCacheFlush(byte[], byte[], long, boolean) + * @see #abortCacheFlush(byte[]) + */ + public long startCacheFlush(final byte[] encodedRegionName) { + this.cacheFlushLock.lock(); + Long seq = this.lastSeqWritten.remove(encodedRegionName); + // seq is the lsn of the oldest edit associated with this region. If a + // snapshot already exists - because the last flush failed - then seq will + // be the lsn of the oldest edit in the snapshot + if (seq != null) { + // keeping the earliest sequence number of the snapshot in + // lastSeqWritten maintains the correctness of + // getOldestOutstandingSeqNum(). But it doesn't matter really because + // everything is being done inside of cacheFlush lock. + Long oldseq = + lastSeqWritten.put(getSnapshotName(encodedRegionName), seq); + if (oldseq != null) { + LOG.error("Logic Error Snapshot seq id from earlier flush still" + + " present! for region " + Bytes.toString(encodedRegionName) + + " overwritten oldseq=" + oldseq + "with new seq=" + seq); + Runtime.getRuntime().halt(1); + } + } + return obtainSeqNum(); + } + + + /** + * Complete the cache flush + * + * Protected by cacheFlushLock + * + * @param encodedRegionName + * @param tableName + * @param logSeqId + * @throws IOException + */ + public void completeCacheFlush(final byte [] encodedRegionName, + final byte [] tableName, final long logSeqId, final boolean isMetaRegion) + throws IOException { + try { + if (this.closed) { + return; + } + long txid = 0; + synchronized (updateLock) { + long now = System.currentTimeMillis(); + WALEdit edit = completeCacheFlushLogEdit(); + HLogKey key = makeKey(encodedRegionName, tableName, logSeqId, + System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID); + logSyncer.append(new Entry(key, edit)); + txid = this.unflushedEntries.incrementAndGet(); + writeTime.inc(System.currentTimeMillis() - now); + long len = 0; + for (KeyValue kv : edit.getKeyValues()) { + len += kv.getLength(); + } + writeSize.inc(len); + this.numEntries.incrementAndGet(); + } + // sync txn to file system + this.sync(txid); + + } finally { + // updateLock not needed for removing snapshot's entry + // Cleaning up of lastSeqWritten is in the finally clause because we + // don't want to confuse getOldestOutstandingSeqNum() + this.lastSeqWritten.remove(getSnapshotName(encodedRegionName)); + this.cacheFlushLock.unlock(); + } + } + + private WALEdit completeCacheFlushLogEdit() { + KeyValue kv = new KeyValue(METAROW, METAFAMILY, null, + System.currentTimeMillis(), COMPLETE_CACHE_FLUSH); + WALEdit e = new WALEdit(); + e.add(kv); + return e; + } + + /** + * Abort a cache flush. + * Call if the flush fails. Note that the only recovery for an aborted flush + * currently is a restart of the regionserver so the snapshot content dropped + * by the failure gets restored to the memstore. + */ + public void abortCacheFlush(byte[] encodedRegionName) { + Long snapshot_seq = + this.lastSeqWritten.remove(getSnapshotName(encodedRegionName)); + if (snapshot_seq != null) { + // updateLock not necessary because we are racing against + // lastSeqWritten.putIfAbsent() in append() and we will always win + // before releasing cacheFlushLock make sure that the region's entry in + // lastSeqWritten points to the earliest edit in the region + Long current_memstore_earliest_seq = + this.lastSeqWritten.put(encodedRegionName, snapshot_seq); + if (current_memstore_earliest_seq != null && + (current_memstore_earliest_seq.longValue() <= + snapshot_seq.longValue())) { + LOG.error("Logic Error region " + Bytes.toString(encodedRegionName) + + "acquired edits out of order current memstore seq=" + + current_memstore_earliest_seq + " snapshot seq=" + snapshot_seq); + Runtime.getRuntime().halt(1); + } + } + this.cacheFlushLock.unlock(); + } + + /** + * @param family + * @return true if the column is a meta column + */ + public static boolean isMetaFamily(byte [] family) { + return Bytes.equals(METAFAMILY, family); + } + + /** + * Get LowReplication-Roller status + * + * @return lowReplicationRollEnabled + */ + public boolean isLowReplicationRollEnabled() { + return lowReplicationRollEnabled; + } + + @SuppressWarnings("unchecked") + public static Class getKeyClass(Configuration conf) { + return (Class) + conf.getClass("hbase.regionserver.hlog.keyclass", HLogKey.class); + } + + public static HLogKey newKey(Configuration conf) throws IOException { + Class keyClass = getKeyClass(conf); + try { + return keyClass.newInstance(); + } catch (InstantiationException e) { + throw new IOException("cannot create hlog key"); + } catch (IllegalAccessException e) { + throw new IOException("cannot create hlog key"); + } + } + + /** + * Utility class that lets us keep track of the edit with it's key + * Only used when splitting logs + */ + public static class Entry implements Writable { + private WALEdit edit; + private HLogKey key; + + public Entry() { + edit = new WALEdit(); + key = new HLogKey(); + } + + /** + * Constructor for both params + * @param edit log's edit + * @param key log's key + */ + public Entry(HLogKey key, WALEdit edit) { + super(); + this.key = key; + this.edit = edit; + } + /** + * Gets the edit + * @return edit + */ + public WALEdit getEdit() { + return edit; + } + /** + * Gets the key + * @return key + */ + public HLogKey getKey() { + return key; + } + + /** + * Set compression context for this entry. + * @param compressionContext Compression context + */ + public void setCompressionContext(CompressionContext compressionContext) { + edit.setCompressionContext(compressionContext); + key.setCompressionContext(compressionContext); + } + + @Override + public String toString() { + return this.key + "=" + this.edit; + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + this.key.write(dataOutput); + this.edit.write(dataOutput); + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + this.key.readFields(dataInput); + this.edit.readFields(dataInput); + } + } + + /** + * Construct the HLog directory name + * + * @param serverName Server name formatted as described in {@link ServerName} + * @return the HLog directory name + */ + public static String getHLogDirectoryName(final String serverName) { + StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME); + dirName.append("/"); + dirName.append(serverName); + return dirName.toString(); + } + + /** + * Get the directory we are making logs in. + * + * @return dir + */ + protected Path getDir() { + return dir; + } + + /** + * @param filename name of the file to validate + * @return true if the filename matches an HLog, false + * otherwise + */ + public static boolean validateHLogFilename(String filename) { + return pattern.matcher(filename).matches(); + } + + static Path getHLogArchivePath(Path oldLogDir, Path p) { + return new Path(oldLogDir, p.getName()); + } + + static String formatRecoveredEditsFileName(final long seqid) { + return String.format("%019d", seqid); + } + + /** + * Returns sorted set of edit files made by wal-log splitter, excluding files + * with '.temp' suffix. + * @param fs + * @param regiondir + * @return Files in passed regiondir as a sorted set. + * @throws IOException + */ + public static NavigableSet getSplitEditFilesSorted(final FileSystem fs, + final Path regiondir) + throws IOException { + NavigableSet filesSorted = new TreeSet(); + Path editsdir = getRegionDirRecoveredEditsDir(regiondir); + if (!fs.exists(editsdir)) return filesSorted; + FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() { + @Override + public boolean accept(Path p) { + boolean result = false; + try { + // Return files and only files that match the editfile names pattern. + // There can be other files in this directory other than edit files. + // In particular, on error, we'll move aside the bad edit file giving + // it a timestamp suffix. See moveAsideBadEditsFile. + Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName()); + result = fs.isFile(p) && m.matches(); + // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX, + // because it means splithlog thread is writting this file. + if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) { + result = false; + } + } catch (IOException e) { + LOG.warn("Failed isFile check on " + p); + } + return result; + } + }); + if (files == null) return filesSorted; + for (FileStatus status: files) { + filesSorted.add(status.getPath()); + } + return filesSorted; + } + + /** + * Move aside a bad edits file. + * @param fs + * @param edits Edits file to move aside. + * @return The name of the moved aside file. + * @throws IOException + */ + public static Path moveAsideBadEditsFile(final FileSystem fs, + final Path edits) + throws IOException { + Path moveAsideName = new Path(edits.getParent(), edits.getName() + "." + + System.currentTimeMillis()); + if (!HBaseFileSystem.renameDirForFileSystem(fs, edits, moveAsideName)) { + LOG.warn("Rename failed from " + edits + " to " + moveAsideName); + } + return moveAsideName; + } + + /** + * @param regiondir This regions directory in the filesystem. + * @return The directory that holds recovered edits files for the region + * regiondir + */ + public static Path getRegionDirRecoveredEditsDir(final Path regiondir) { + return new Path(regiondir, RECOVERED_EDITS_DIR); + } + + public static final long FIXED_OVERHEAD = ClassSize.align( + ClassSize.OBJECT + (5 * ClassSize.REFERENCE) + + ClassSize.ATOMIC_INTEGER + Bytes.SIZEOF_INT + (3 * Bytes.SIZEOF_LONG)); + + private static void usage() { + System.err.println("Usage: HLog "); + System.err.println("Arguments:"); + System.err.println(" --dump Dump textual representation of passed one or more files"); + System.err.println(" For example: HLog --dump hdfs://example.com:9000/hbase/.logs/MACHINE/LOGFILE"); + System.err.println(" --split Split the passed directory of WAL logs"); + System.err.println(" For example: HLog --split hdfs://example.com:9000/hbase/.logs/DIR"); + } + + private static void split(final Configuration conf, final Path p) + throws IOException { + FileSystem fs = FileSystem.get(conf); + if (!fs.exists(p)) { + throw new FileNotFoundException(p.toString()); + } + final Path baseDir = new Path(conf.get(HConstants.HBASE_DIR)); + final Path oldLogDir = new Path(baseDir, HConstants.HREGION_OLDLOGDIR_NAME); + if (!fs.getFileStatus(p).isDir()) { + throw new IOException(p + " is not a directory"); + } + + HLogSplitter logSplitter = HLogSplitter.createLogSplitter( + conf, baseDir, p, oldLogDir, fs); + logSplitter.splitLog(); + } + + /** + * @return Coprocessor host. + */ + public WALCoprocessorHost getCoprocessorHost() { + return coprocessorHost; + } + + /** Provide access to currently deferred sequence num for tests */ + boolean hasDeferredEntries() { + return lastDeferredTxid > syncedTillHere; + } + + /** + * Pass one or more log file names and it will either dump out a text version + * on stdout or split the specified log files. + * + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + if (args.length < 2) { + usage(); + System.exit(-1); + } + // either dump using the HLogPrettyPrinter or split, depending on args + if (args[0].compareTo("--dump") == 0) { + HLogPrettyPrinter.run(Arrays.copyOfRange(args, 1, args.length)); + } else if (args[0].compareTo("--split") == 0) { + Configuration conf = HBaseConfiguration.create(); + for (int i = 1; i < args.length; i++) { + try { + conf.set("fs.default.name", args[i]); + conf.set("fs.defaultFS", args[i]); + Path logPath = new Path(args[i]); + split(conf, logPath); + } catch (Throwable t) { + t.printStackTrace(System.err); + System.exit(-1); + } + } + } else { + usage(); + System.exit(-1); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogFileSystem.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogFileSystem.java new file mode 100644 index 0000000..9bcf1db --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogFileSystem.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseFileSystem; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.regionserver.wal.HLog.Writer; + +/** + * Acts as an abstraction between the HLog and the underlying filesystem. This is analogous to the + * {@link HRegionFileSystem} class. + */ +public class HLogFileSystem extends HBaseFileSystem { + public static final Log LOG = LogFactory.getLog(HLogFileSystem.class); + + /** + * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the + * client level. + */ + + public HLogFileSystem(Configuration conf) { + setRetryCounts(conf); + } + + /** + * Creates writer for the given path. + * @param fs + * @param conf + * @param hlogFile + * @return an init'ed writer for the given path. + * @throws IOException + */ + public Writer createWriter(FileSystem fs, Configuration conf, Path hlogFile) throws IOException { + int i = 0; + IOException lastIOE = null; + do { + try { + return HLog.createWriter(fs, hlogFile, conf); + } catch (IOException ioe) { + lastIOE = ioe; + sleepBeforeRetry("Create Writer", i+1); + } + } while (++i <= hdfsClientRetriesNumber); + throw new IOException("Exception in createWriter", lastIOE); + + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java new file mode 100644 index 0000000..9511481 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java @@ -0,0 +1,327 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.EOFException; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableUtils; + +/** + * A Key for an entry in the change log. + * + * The log intermingles edits to many tables and rows, so each log entry + * identifies the appropriate table and row. Within a table and row, they're + * also sorted. + * + *

Some Transactional edits (START, COMMIT, ABORT) will not have an + * associated row. + */ +public class HLogKey implements WritableComparable { + // should be < 0 (@see #readFields(DataInput)) + // version 2 supports HLog compression + enum Version { + UNVERSIONED(0), + // Initial number we put on HLogKey when we introduced versioning. + INITIAL(-1), + // Version -2 introduced a dictionary compression facility. Only this + // dictionary-based compression is available in version -2. + COMPRESSED(-2); + + final int code; + static final Version[] byCode; + static { + byCode = Version.values(); + for (int i = 0; i < byCode.length; i++) { + if (byCode[i].code != -1 * i) { + throw new AssertionError("Values in this enum should be descending by one"); + } + } + } + + Version(int code) { + this.code = code; + } + + boolean atLeast(Version other) { + return code <= other.code; + } + + static Version fromCode(int code) { + return byCode[code * -1]; + } + } + + private static final Version VERSION = Version.COMPRESSED; + + // The encoded region name. + private byte [] encodedRegionName; + private byte [] tablename; + private long logSeqNum; + // Time at which this edit was written. + private long writeTime; + + private UUID clusterId; + + private CompressionContext compressionContext; + + /** Writable Constructor -- Do not use. */ + public HLogKey() { + this(null, null, 0L, HConstants.LATEST_TIMESTAMP, + HConstants.DEFAULT_CLUSTER_ID); + } + + /** + * Create the log key! + * We maintain the tablename mainly for debugging purposes. + * A regionName is always a sub-table object. + * + * @param encodedRegionName Encoded name of the region as returned by + * HRegionInfo#getEncodedNameAsBytes(). + * @param tablename - name of table + * @param logSeqNum - log sequence number + * @param now Time at which this edit was written. + * @param clusterId of the cluster (used in Replication) + */ + public HLogKey(final byte [] encodedRegionName, final byte [] tablename, + long logSeqNum, final long now, UUID clusterId) { + this.encodedRegionName = encodedRegionName; + this.tablename = tablename; + this.logSeqNum = logSeqNum; + this.writeTime = now; + this.clusterId = clusterId; + } + + /** + * @param compressionContext Compression context to use + */ + public void setCompressionContext(CompressionContext compressionContext) { + this.compressionContext = compressionContext; + } + + /** @return encoded region name */ + public byte [] getEncodedRegionName() { + return encodedRegionName; + } + + /** @return table name */ + public byte [] getTablename() { + return tablename; + } + + /** @return log sequence number */ + public long getLogSeqNum() { + return logSeqNum; + } + + void setLogSeqNum(long logSeqNum) { + this.logSeqNum = logSeqNum; + } + + /** + * @return the write time + */ + public long getWriteTime() { + return this.writeTime; + } + + /** + * Get the id of the original cluster + * @return Cluster id. + */ + public UUID getClusterId() { + return clusterId; + } + + /** + * Set the cluster id of this key + * @param clusterId + */ + public void setClusterId(UUID clusterId) { + this.clusterId = clusterId; + } + + @Override + public String toString() { + return Bytes.toString(tablename) + "/" + Bytes.toString(encodedRegionName) + "/" + + logSeqNum; + } + + /** + * Produces a string map for this key. Useful for programmatic use and + * manipulation of the data stored in an HLogKey, for example, printing + * as JSON. + * + * @return a Map containing data from this key + */ + public Map toStringMap() { + Map stringMap = new HashMap(); + stringMap.put("table", Bytes.toStringBinary(tablename)); + stringMap.put("region", Bytes.toStringBinary(encodedRegionName)); + stringMap.put("sequence", logSeqNum); + return stringMap; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + return compareTo((HLogKey)obj) == 0; + } + + @Override + public int hashCode() { + int result = Bytes.hashCode(this.encodedRegionName); + result ^= this.logSeqNum; + result ^= this.writeTime; + result ^= this.clusterId.hashCode(); + return result; + } + + public int compareTo(HLogKey o) { + int result = Bytes.compareTo(this.encodedRegionName, o.encodedRegionName); + if (result == 0) { + if (this.logSeqNum < o.logSeqNum) { + result = -1; + } else if (this.logSeqNum > o.logSeqNum) { + result = 1; + } + if (result == 0) { + if (this.writeTime < o.writeTime) { + result = -1; + } else if (this.writeTime > o.writeTime) { + return 1; + } + } + } + // why isn't cluster id accounted for? + return result; + } + + /** + * Drop this instance's tablename byte array and instead + * hold a reference to the provided tablename. This is not + * meant to be a general purpose setter - it's only used + * to collapse references to conserve memory. + */ + void internTableName(byte []tablename) { + // We should not use this as a setter - only to swap + // in a new reference to the same table name. + assert Bytes.equals(tablename, this.tablename); + this.tablename = tablename; + } + + /** + * Drop this instance's region name byte array and instead + * hold a reference to the provided region name. This is not + * meant to be a general purpose setter - it's only used + * to collapse references to conserve memory. + */ + void internEncodedRegionName(byte []encodedRegionName) { + // We should not use this as a setter - only to swap + // in a new reference to the same table name. + assert Bytes.equals(this.encodedRegionName, encodedRegionName); + this.encodedRegionName = encodedRegionName; + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, VERSION.code); + if (compressionContext == null) { + Bytes.writeByteArray(out, this.encodedRegionName); + Bytes.writeByteArray(out, this.tablename); + } else { + Compressor.writeCompressed(this.encodedRegionName, 0, + this.encodedRegionName.length, out, + compressionContext.regionDict); + Compressor.writeCompressed(this.tablename, 0, this.tablename.length, out, + compressionContext.tableDict); + } + out.writeLong(this.logSeqNum); + out.writeLong(this.writeTime); + // avoid storing 16 bytes when replication is not enabled + if (this.clusterId == HConstants.DEFAULT_CLUSTER_ID) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeLong(this.clusterId.getMostSignificantBits()); + out.writeLong(this.clusterId.getLeastSignificantBits()); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + Version version = Version.UNVERSIONED; + // HLogKey was not versioned in the beginning. + // In order to introduce it now, we make use of the fact + // that encodedRegionName was written with Bytes.writeByteArray, + // which encodes the array length as a vint which is >= 0. + // Hence if the vint is >= 0 we have an old version and the vint + // encodes the length of encodedRegionName. + // If < 0 we just read the version and the next vint is the length. + // @see Bytes#readByteArray(DataInput) + int len = WritableUtils.readVInt(in); + if (len < 0) { + // what we just read was the version + version = Version.fromCode(len); + // We only compress V2 of HLogkey. + // If compression is on, the length is handled by the dictionary + if (compressionContext == null || !version.atLeast(Version.COMPRESSED)) { + len = WritableUtils.readVInt(in); + } + } + if (compressionContext == null || !version.atLeast(Version.COMPRESSED)) { + this.encodedRegionName = new byte[len]; + in.readFully(this.encodedRegionName); + this.tablename = Bytes.readByteArray(in); + } else { + this.encodedRegionName = Compressor.readCompressed(in, compressionContext.regionDict); + this.tablename = Compressor.readCompressed(in, compressionContext.tableDict); + } + + this.logSeqNum = in.readLong(); + this.writeTime = in.readLong(); + this.clusterId = HConstants.DEFAULT_CLUSTER_ID; + if (version.atLeast(Version.INITIAL)) { + if (in.readBoolean()) { + this.clusterId = new UUID(in.readLong(), in.readLong()); + } + } else { + try { + // dummy read (former byte cluster id) + in.readByte(); + } catch(EOFException e) { + // Means it's a very old key, just continue + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java new file mode 100644 index 0000000..d0e85f3 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java @@ -0,0 +1,375 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader; +import org.apache.hadoop.hbase.util.Bytes; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * HLogPrettyPrinter prints the contents of a given HLog with a variety of + * options affecting formatting and extent of content. + * + * It targets two usage cases: pretty printing for ease of debugging directly by + * humans, and JSON output for consumption by monitoring and/or maintenance + * scripts. + * + * It can filter by row, region, or sequence id. + * + * It can also toggle output of values. + * + */ +public class HLogPrettyPrinter { + private boolean outputValues; + private boolean outputJSON; + // The following enable filtering by sequence, region, and row, respectively + private long sequence; + private String region; + private String row; + // enable in order to output a single list of transactions from several files + private boolean persistentOutput; + private boolean firstTxn; + // useful for programatic capture of JSON output + private PrintStream out; + // for JSON encoding + private ObjectMapper mapper; + + /** + * Basic constructor that simply initializes values to reasonable defaults. + */ + public HLogPrettyPrinter() { + outputValues = false; + outputJSON = false; + sequence = -1; + region = null; + row = null; + persistentOutput = false; + firstTxn = true; + out = System.out; + mapper = new ObjectMapper(); + } + + /** + * Fully specified constructor. + * + * @param outputValues + * when true, enables output of values along with other log + * information + * @param outputJSON + * when true, enables output in JSON format rather than a + * "pretty string" + * @param sequence + * when nonnegative, serves as a filter; only log entries with this + * sequence id will be printed + * @param region + * when not null, serves as a filter; only log entries from this + * region will be printed + * @param row + * when not null, serves as a filter; only log entries from this row + * will be printed + * @param persistentOutput + * keeps a single list running for multiple files. if enabled, the + * endPersistentOutput() method must be used! + * @param out + * Specifies an alternative to stdout for the destination of this + * PrettyPrinter's output. + */ + public HLogPrettyPrinter(boolean outputValues, boolean outputJSON, + long sequence, String region, String row, boolean persistentOutput, + PrintStream out) { + this.outputValues = outputValues; + this.outputJSON = outputJSON; + this.sequence = sequence; + this.region = region; + this.row = row; + this.persistentOutput = persistentOutput; + if (persistentOutput) { + beginPersistentOutput(); + } + this.out = out; + this.firstTxn = true; + } + + /** + * turns value output on + */ + public void enableValues() { + outputValues = true; + } + + /** + * turns value output off + */ + public void disableValues() { + outputValues = false; + } + + /** + * turns JSON output on + */ + public void enableJSON() { + outputJSON = true; + } + + /** + * turns JSON output off, and turns on "pretty strings" for human consumption + */ + public void disableJSON() { + outputJSON = false; + } + + /** + * sets the region by which output will be filtered + * + * @param sequence + * when nonnegative, serves as a filter; only log entries with this + * sequence id will be printed + */ + public void setSequenceFilter(long sequence) { + this.sequence = sequence; + } + + /** + * sets the region by which output will be filtered + * + * @param region + * when not null, serves as a filter; only log entries from this + * region will be printed + */ + public void setRegionFilter(String region) { + this.region = region; + } + + /** + * sets the region by which output will be filtered + * + * @param row + * when not null, serves as a filter; only log entries from this row + * will be printed + */ + public void setRowFilter(String row) { + this.row = row; + } + + /** + * enables output as a single, persistent list. at present, only relevant in + * the case of JSON output. + */ + public void beginPersistentOutput() { + if (persistentOutput) + return; + persistentOutput = true; + firstTxn = true; + if (outputJSON) + out.print("["); + } + + /** + * ends output of a single, persistent list. at present, only relevant in the + * case of JSON output. + */ + public void endPersistentOutput() { + if (!persistentOutput) + return; + persistentOutput = false; + if (outputJSON) + out.print("]"); + } + + /** + * reads a log file and outputs its contents, one transaction at a time, as + * specified by the currently configured options + * + * @param conf + * the HBase configuration relevant to this log file + * @param p + * the path of the log file to be read + * @throws IOException + * may be unable to access the configured filesystem or requested + * file. + */ + public void processFile(final Configuration conf, final Path p) + throws IOException { + FileSystem fs = FileSystem.get(conf); + if (!fs.exists(p)) { + throw new FileNotFoundException(p.toString()); + } + if (!fs.isFile(p)) { + throw new IOException(p + " is not a file"); + } + if (outputJSON && !persistentOutput) { + out.print("["); + firstTxn = true; + } + Reader log = HLog.getReader(fs, p, conf); + try { + HLog.Entry entry; + while ((entry = log.next()) != null) { + HLogKey key = entry.getKey(); + WALEdit edit = entry.getEdit(); + // begin building a transaction structure + Map txn = key.toStringMap(); + // check output filters + if (sequence >= 0 && ((Long) txn.get("sequence")) != sequence) + continue; + if (region != null && !((String) txn.get("region")).equals(region)) + continue; + // initialize list into which we will store atomic actions + List actions = new ArrayList(); + for (KeyValue kv : edit.getKeyValues()) { + // add atomic operation to txn + Map op = + new HashMap(kv.toStringMap()); + if (outputValues) + op.put("value", Bytes.toStringBinary(kv.getValue())); + // check row output filter + if (row == null || ((String) op.get("row")).equals(row)) + actions.add(op); + } + if (actions.size() == 0) + continue; + txn.put("actions", actions); + if (outputJSON) { + // JSON output is a straightforward "toString" on the txn object + if (firstTxn) + firstTxn = false; + else + out.print(","); + // encode and print JSON + out.print(mapper.writeValueAsString(txn)); + } else { + // Pretty output, complete with indentation by atomic action + out.println("Sequence " + txn.get("sequence") + " " + + "from region " + txn.get("region") + " " + "in table " + + txn.get("table")); + for (int i = 0; i < actions.size(); i++) { + Map op = actions.get(i); + out.println(" Action:"); + out.println(" row: " + op.get("row")); + out.println(" column: " + op.get("family") + ":" + + op.get("qualifier")); + out.println(" at time: " + + (new Date((Long) op.get("timestamp")))); + if (outputValues) + out.println(" value: " + op.get("value")); + } + } + } + } finally { + log.close(); + } + if (outputJSON && !persistentOutput) { + out.print("]"); + } + } + + public static void main(String[] args) throws IOException { + run(args); + } + + /** + * Pass one or more log file names and formatting options and it will dump out + * a text version of the contents on stdout. + * + * @param args + * Command line arguments + * @throws IOException + * Thrown upon file system errors etc. + * @throws ParseException + * Thrown if command-line parsing fails. + */ + public static void run(String[] args) throws IOException { + // create options + Options options = new Options(); + options.addOption("h", "help", false, "Output help message"); + options.addOption("j", "json", false, "Output JSON"); + options.addOption("p", "printvals", false, "Print values"); + options.addOption("r", "region", true, + "Region to filter by. Pass region name; e.g. '.META.,,1'"); + options.addOption("s", "sequence", true, + "Sequence to filter by. Pass sequence number."); + options.addOption("w", "row", true, "Row to filter by. Pass row name."); + + HLogPrettyPrinter printer = new HLogPrettyPrinter(); + CommandLineParser parser = new PosixParser(); + List files = null; + try { + CommandLine cmd = parser.parse(options, args); + files = cmd.getArgList(); + if (files.size() == 0 || cmd.hasOption("h")) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("HLog ", options, true); + System.exit(-1); + } + // configure the pretty printer using command line options + if (cmd.hasOption("p")) + printer.enableValues(); + if (cmd.hasOption("j")) + printer.enableJSON(); + if (cmd.hasOption("r")) + printer.setRegionFilter(cmd.getOptionValue("r")); + if (cmd.hasOption("s")) + printer.setSequenceFilter(Long.parseLong(cmd.getOptionValue("s"))); + if (cmd.hasOption("w")) + printer.setRowFilter(cmd.getOptionValue("w")); + } catch (ParseException e) { + e.printStackTrace(); + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("HFile filename(s) ", options, true); + System.exit(-1); + } + // get configuration, file system, and process the given files + Configuration conf = HBaseConfiguration.create(); + conf.set("fs.defaultFS", + conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR)); + conf.set("fs.default.name", + conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR)); + // begin output + printer.beginPersistentOutput(); + for (Object f : files) { + Path file = new Path((String) f); + FileSystem fs = file.getFileSystem(conf); + if (!fs.exists(file)) { + System.err.println("ERROR, file doesnt exist: " + file); + return; + } + printer.processFile(conf, file); + } + printer.endPersistentOutput(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java new file mode 100644 index 0000000..57f75b7 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java @@ -0,0 +1,1392 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.CountDownLatch; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseFileSystem; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.RemoteExceptionHandler; +import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.monitoring.MonitoredTask; +import org.apache.hadoop.hbase.monitoring.TaskMonitor; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry; +import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader; +import org.apache.hadoop.hbase.regionserver.wal.HLog.Writer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CancelableProgressable; +import org.apache.hadoop.hbase.util.ClassSize; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.zookeeper.ZKSplitLog; +import org.apache.hadoop.io.MultipleIOException; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +/** + * This class is responsible for splitting up a bunch of regionserver commit log + * files that are no longer being written to, into new files, one per region for + * region to replay on startup. Delete the old log files when finished. + */ +public class HLogSplitter { + private static final String LOG_SPLITTER_IMPL = "hbase.hlog.splitter.impl"; + + /** + * Name of file that holds recovered edits written by the wal log splitting + * code, one per region + */ + public static final String RECOVERED_EDITS = "recovered.edits"; + + + static final Log LOG = LogFactory.getLog(HLogSplitter.class); + + private boolean hasSplit = false; + private long splitTime = 0; + private long splitSize = 0; + + + // Parameters for split process + protected final Path rootDir; + protected final Path srcDir; + protected final Path oldLogDir; + protected final FileSystem fs; + protected final Configuration conf; + private final HLogFileSystem hlogFs; + + // Major subcomponents of the split process. + // These are separated into inner classes to make testing easier. + OutputSink outputSink; + EntryBuffers entryBuffers; + + // If an exception is thrown by one of the other threads, it will be + // stored here. + protected AtomicReference thrown = new AtomicReference(); + + // Wait/notify for when data has been produced by the reader thread, + // consumed by the reader thread, or an exception occurred + Object dataAvailable = new Object(); + + private MonitoredTask status; + + + /** + * Create a new HLogSplitter using the given {@link Configuration} and the + * hbase.hlog.splitter.impl property to derived the instance + * class to use. + *

+ * @param conf + * @param rootDir hbase directory + * @param srcDir logs directory + * @param oldLogDir directory where processed logs are archived to + * @param fs FileSystem + * @return New HLogSplitter instance + */ + public static HLogSplitter createLogSplitter(Configuration conf, + final Path rootDir, final Path srcDir, + Path oldLogDir, final FileSystem fs) { + + @SuppressWarnings("unchecked") + Class splitterClass = (Class) conf + .getClass(LOG_SPLITTER_IMPL, HLogSplitter.class); + try { + Constructor constructor = + splitterClass.getConstructor( + Configuration.class, // conf + Path.class, // rootDir + Path.class, // srcDir + Path.class, // oldLogDir + FileSystem.class); // fs + return constructor.newInstance(conf, rootDir, srcDir, oldLogDir, fs); + } catch (IllegalArgumentException e) { + throw new RuntimeException(e); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } catch (SecurityException e) { + throw new RuntimeException(e); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + + public HLogSplitter(Configuration conf, Path rootDir, Path srcDir, + Path oldLogDir, FileSystem fs) { + this.conf = conf; + this.rootDir = rootDir; + this.srcDir = srcDir; + this.oldLogDir = oldLogDir; + this.fs = fs; + + entryBuffers = new EntryBuffers( + conf.getInt("hbase.regionserver.hlog.splitlog.buffersize", + 128*1024*1024)); + outputSink = new OutputSink(); + this.hlogFs = new HLogFileSystem(conf); + } + + /** + * Split up a bunch of regionserver commit log files that are no longer being + * written to, into new files, one per region for region to replay on startup. + * Delete the old log files when finished. + * + * @throws IOException will throw if corrupted hlogs aren't tolerated + * @return the list of splits + */ + public List splitLog() + throws IOException { + return splitLog((CountDownLatch) null); + } + + /** + * Split up a bunch of regionserver commit log files that are no longer being + * written to, into new files, one per region for region to replay on startup. + * Delete the old log files when finished. + * + * @param latch + * @throws IOException will throw if corrupted hlogs aren't tolerated + * @return the list of splits + */ + public List splitLog(CountDownLatch latch) + throws IOException { + Preconditions.checkState(!hasSplit, + "An HLogSplitter instance may only be used once"); + hasSplit = true; + + status = TaskMonitor.get().createStatus( + "Splitting logs in " + srcDir); + + long startTime = EnvironmentEdgeManager.currentTimeMillis(); + + status.setStatus("Determining files to split..."); + List splits = null; + if (!fs.exists(srcDir)) { + // Nothing to do + status.markComplete("No log directory existed to split."); + return splits; + } + FileStatus[] logfiles = fs.listStatus(srcDir); + if (logfiles == null || logfiles.length == 0) { + // Nothing to do + return splits; + } + logAndReport("Splitting " + logfiles.length + " hlog(s) in " + + srcDir.toString()); + splits = splitLog(logfiles, latch); + + splitTime = EnvironmentEdgeManager.currentTimeMillis() - startTime; + String msg = "hlog file splitting completed in " + splitTime + + " ms for " + srcDir.toString(); + status.markComplete(msg); + LOG.info(msg); + return splits; + } + + private void logAndReport(String msg) { + status.setStatus(msg); + LOG.info(msg); + } + + /** + * @return time that this split took + */ + public long getTime() { + return this.splitTime; + } + + /** + * @return aggregate size of hlogs that were split + */ + public long getSize() { + return this.splitSize; + } + + /** + * @return a map from encoded region ID to the number of edits written out + * for that region. + */ + Map getOutputCounts() { + Preconditions.checkState(hasSplit); + return outputSink.getOutputCounts(); + } + + /** + * Splits the HLog edits in the given list of logfiles (that are a mix of edits + * on multiple regions) by region and then splits them per region directories, + * in batches of (hbase.hlog.split.batch.size) + *

+ * This process is split into multiple threads. In the main thread, we loop + * through the logs to be split. For each log, we: + *

    + *
  • Recover it (take and drop HDFS lease) to ensure no other process can write
    • + *
  • Read each edit (see {@link #parseHLog}
    • + *
  • Mark as "processed" or "corrupt" depending on outcome
    • + *
+ *

+ * Each edit is passed into the EntryBuffers instance, which takes care of + * memory accounting and splitting the edits by region. + *

+ * The OutputSink object then manages N other WriterThreads which pull chunks + * of edits from EntryBuffers and write them to the output region directories. + *

+ * After the process is complete, the log files are archived to a separate + * directory. + */ + private List splitLog(final FileStatus[] logfiles, CountDownLatch latch) + throws IOException { + List processedLogs = new ArrayList(); + List corruptedLogs = new ArrayList(); + List splits = null; + + boolean skipErrors = conf.getBoolean("hbase.hlog.split.skip.errors", true); + + countTotalBytes(logfiles); + splitSize = 0; + + outputSink.startWriterThreads(entryBuffers); + + try { + int i = 0; + for (FileStatus log : logfiles) { + Path logPath = log.getPath(); + long logLength = log.getLen(); + splitSize += logLength; + logAndReport("Splitting hlog " + (i++ + 1) + " of " + logfiles.length + + ": " + logPath + ", length=" + logLength); + Reader in; + try { + //actually, for meta-only hlogs, we don't need to go thru the process + //of parsing and segregating by regions since all the logs are for + //meta only. However, there is a sequence number that can be obtained + //only by parsing.. so we parse for all files currently + //TODO: optimize this part somehow + in = getReader(fs, log, conf, skipErrors); + if (in != null) { + parseHLog(in, logPath, entryBuffers, fs, conf, skipErrors); + try { + in.close(); + } catch (IOException e) { + LOG.warn("Close log reader threw exception -- continuing", + e); + } + } + processedLogs.add(logPath); + } catch (CorruptedLogFileException e) { + LOG.info("Got while parsing hlog " + logPath + + ". Marking as corrupted", e); + corruptedLogs.add(logPath); + continue; + } + } + status.setStatus("Log splits complete. Checking for orphaned logs."); + + if (latch != null) { + try { + latch.await(); + } catch (InterruptedException ie) { + LOG.warn("wait for latch interrupted"); + Thread.currentThread().interrupt(); + } + } + FileStatus[] currFiles = fs.listStatus(srcDir); + if (currFiles.length > processedLogs.size() + + corruptedLogs.size()) { + throw new OrphanHLogAfterSplitException( + "Discovered orphan hlog after split. Maybe the " + + "HRegionServer was not dead when we started"); + } + } finally { + status.setStatus("Finishing writing output logs and closing down."); + splits = outputSink.finishWritingAndClose(); + } + status.setStatus("Archiving logs after completed split"); + archiveLogs(srcDir, corruptedLogs, processedLogs, oldLogDir, fs, conf); + return splits; + } + + /** + * @return the total size of the passed list of files. + */ + private static long countTotalBytes(FileStatus[] logfiles) { + long ret = 0; + for (FileStatus stat : logfiles) { + ret += stat.getLen(); + } + return ret; + } + + /** + * Splits a HLog file into region's recovered-edits directory + *

+ * If the log file has N regions then N recovered.edits files will be + * produced. There is no buffering in this code. Instead it relies on the + * buffering in the SequenceFileWriter. + *

+ * @param rootDir + * @param logfile + * @param fs + * @param conf + * @param reporter + * @return false if it is interrupted by the progress-able. + * @throws IOException + */ + static public boolean splitLogFile(Path rootDir, FileStatus logfile, + FileSystem fs, Configuration conf, CancelableProgressable reporter) + throws IOException { + HLogSplitter s = new HLogSplitter(conf, rootDir, null, null /* oldLogDir */, + fs); + return s.splitLogFile(logfile, reporter); + } + + public boolean splitLogFile(FileStatus logfile, + CancelableProgressable reporter) throws IOException { + final Map logWriters = Collections. + synchronizedMap(new TreeMap(Bytes.BYTES_COMPARATOR)); + boolean isCorrupted = false; + + Preconditions.checkState(status == null); + status = TaskMonitor.get().createStatus( + "Splitting log file " + logfile.getPath() + + "into a temporary staging area."); + + Object BAD_WRITER = new Object(); + + boolean progress_failed = false; + + boolean skipErrors = conf.getBoolean("hbase.hlog.split.skip.errors", + HLog.SPLIT_SKIP_ERRORS_DEFAULT); + int interval = conf.getInt("hbase.splitlog.report.interval.loglines", 1024); + // How often to send a progress report (default 1/2 the zookeeper session + // timeout of if that not set, the split log DEFAULT_TIMEOUT) + int period = conf.getInt("hbase.splitlog.report.period", + conf.getInt("hbase.splitlog.manager.timeout", ZKSplitLog.DEFAULT_TIMEOUT) / 2); + int numOpenedFilesBeforeReporting = + conf.getInt("hbase.splitlog.report.openedfiles", 3); + Path logPath = logfile.getPath(); + long logLength = logfile.getLen(); + LOG.info("Splitting hlog: " + logPath + ", length=" + logLength); + status.setStatus("Opening log file"); + Reader in = null; + try { + in = getReader(fs, logfile, conf, skipErrors); + } catch (CorruptedLogFileException e) { + LOG.warn("Could not get reader, corrupted log file " + logPath, e); + ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs); + isCorrupted = true; + } + if (in == null) { + status.markComplete("Was nothing to split in log file"); + LOG.warn("Nothing to split in log file " + logPath); + return true; + } + long t = EnvironmentEdgeManager.currentTimeMillis(); + long last_report_at = t; + if (reporter != null && reporter.progress() == false) { + status.markComplete("Failed: reporter.progress asked us to terminate"); + return false; + } + // Report progress every so many edits and/or files opened (opening a file + // takes a bit of time). + int editsCount = 0; + int numNewlyOpenedFiles = 0; + Entry entry; + try { + while ((entry = getNextLogLine(in,logPath, skipErrors)) != null) { + byte[] region = entry.getKey().getEncodedRegionName(); + Object o = logWriters.get(region); + if (o == BAD_WRITER) { + continue; + } + WriterAndPath wap = (WriterAndPath)o; + if (wap == null) { + wap = createWAP(region, entry, rootDir, fs, conf); + numNewlyOpenedFiles++; + if (wap == null) { + // ignore edits from this region. It doesn't exist anymore. + // It was probably already split. + logWriters.put(region, BAD_WRITER); + continue; + } else { + logWriters.put(region, wap); + } + } + wap.w.append(entry); + outputSink.updateRegionMaximumEditLogSeqNum(entry); + editsCount++; + // If sufficient edits have passed OR we've opened a few files, check if + // we should report progress. + if (editsCount % interval == 0 || + (numNewlyOpenedFiles > numOpenedFilesBeforeReporting)) { + // Zero out files counter each time we fall in here. + numNewlyOpenedFiles = 0; + String countsStr = "edits=" + editsCount + ", files=" + logWriters.size(); + status.setStatus("Split " + countsStr); + long t1 = EnvironmentEdgeManager.currentTimeMillis(); + if ((t1 - last_report_at) > period) { + last_report_at = t; + if (reporter != null && reporter.progress() == false) { + status.markComplete("Failed: reporter.progress asked us to terminate; " + countsStr); + progress_failed = true; + return false; + } + } + } + } + } catch (CorruptedLogFileException e) { + LOG.warn("Could not parse, corrupted log file " + logPath, e); + ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs); + isCorrupted = true; + } catch (IOException e) { + e = RemoteExceptionHandler.checkIOException(e); + throw e; + } finally { + boolean allWritersClosed = false; + try { + int n = 0; + for (Map.Entry logWritersEntry : logWriters.entrySet()) { + Object o = logWritersEntry.getValue(); + long t1 = EnvironmentEdgeManager.currentTimeMillis(); + if ((t1 - last_report_at) > period) { + last_report_at = t; + if ((progress_failed == false) && (reporter != null) && (reporter.progress() == false)) { + progress_failed = true; + } + } + if (o == BAD_WRITER) { + continue; + } + n++; + WriterAndPath wap = (WriterAndPath) o; + wap.writerClosed = true; + wap.w.close(); + LOG.debug("Closed " + wap.p); + Path dst = getCompletedRecoveredEditsFilePath(wap.p, + outputSink.getRegionMaximumEditLogSeqNum(logWritersEntry.getKey())); + if (!dst.equals(wap.p) && fs.exists(dst)) { + LOG.warn("Found existing old edits file. It could be the " + + "result of a previous failed split attempt. Deleting " + dst + ", length=" + + fs.getFileStatus(dst).getLen()); + if (!HBaseFileSystem.deleteFileFromFileSystem(fs, dst)) { + LOG.warn("Failed deleting of old " + dst); + throw new IOException("Failed deleting of old " + dst); + } + } + // Skip the unit tests which create a splitter that reads and writes the + // data without touching disk. TestHLogSplit#testThreading is an + // example. + if (fs.exists(wap.p)) { + if (!HBaseFileSystem.renameDirForFileSystem(fs, wap.p, dst)) { + throw new IOException("Failed renaming " + wap.p + " to " + dst); + } + LOG.debug("Rename " + wap.p + " to " + dst); + } + } + allWritersClosed = true; + String msg = "Processed " + editsCount + " edits across " + n + " regions" + + " threw away edits for " + (logWriters.size() - n) + " regions" + "; log file=" + + logPath + " is corrupted = " + isCorrupted + " progress failed = " + progress_failed; + LOG.info(msg); + status.markComplete(msg); + } finally { + if (!allWritersClosed) { + for (Map.Entry logWritersEntry : logWriters.entrySet()) { + Object o = logWritersEntry.getValue(); + if (o != BAD_WRITER) { + WriterAndPath wap = (WriterAndPath) o; + try { + if (!wap.writerClosed) { + wap.writerClosed = true; + wap.w.close(); + } + } catch (IOException e) { + LOG.debug("Exception while closing the writer :", e); + } + } + } + } + in.close(); + } + } + return !progress_failed; + } + + /** + * Completes the work done by splitLogFile by archiving logs + *

+ * It is invoked by SplitLogManager once it knows that one of the + * SplitLogWorkers have completed the splitLogFile() part. If the master + * crashes then this function might get called multiple times. + *

+ * @param logfile + * @param conf + * @throws IOException + */ + public static void finishSplitLogFile(String logfile, Configuration conf) + throws IOException { + Path rootdir = FSUtils.getRootDir(conf); + Path oldLogDir = new Path(rootdir, HConstants.HREGION_OLDLOGDIR_NAME); + finishSplitLogFile(rootdir, oldLogDir, logfile, conf); + } + + public static void finishSplitLogFile(Path rootdir, Path oldLogDir, + String logfile, Configuration conf) throws IOException { + List processedLogs = new ArrayList(); + List corruptedLogs = new ArrayList(); + FileSystem fs; + fs = rootdir.getFileSystem(conf); + Path logPath = new Path(logfile); + if (ZKSplitLog.isCorrupted(rootdir, logPath.getName(), fs)) { + corruptedLogs.add(logPath); + } else { + processedLogs.add(logPath); + } + archiveLogs(null, corruptedLogs, processedLogs, oldLogDir, fs, conf); + Path stagingDir = ZKSplitLog.getSplitLogDir(rootdir, logPath.getName()); + HBaseFileSystem.deleteDirFromFileSystem(fs, stagingDir); + } + + /** + * Moves processed logs to a oldLogDir after successful processing Moves + * corrupted logs (any log that couldn't be successfully parsed to corruptDir + * (.corrupt) for later investigation + * + * @param corruptedLogs + * @param processedLogs + * @param oldLogDir + * @param fs + * @param conf + * @throws IOException + */ + private static void archiveLogs( + final Path srcDir, + final List corruptedLogs, + final List processedLogs, final Path oldLogDir, + final FileSystem fs, final Configuration conf) throws IOException { + final Path corruptDir = new Path(conf.get(HConstants.HBASE_DIR), conf.get( + "hbase.regionserver.hlog.splitlog.corrupt.dir", HConstants.CORRUPT_DIR_NAME)); + + if (!HBaseFileSystem.makeDirOnFileSystem(fs, corruptDir)) { + LOG.info("Unable to mkdir " + corruptDir); + } + HBaseFileSystem.makeDirOnFileSystem(fs, oldLogDir); + + // this method can get restarted or called multiple times for archiving + // the same log files. + for (Path corrupted : corruptedLogs) { + Path p = new Path(corruptDir, corrupted.getName()); + if (fs.exists(corrupted)) { + if (!HBaseFileSystem.renameDirForFileSystem(fs, corrupted, p)) { + LOG.warn("Unable to move corrupted log " + corrupted + " to " + p); + } else { + LOG.warn("Moving corrupted log " + corrupted + " to " + p); + } + } + } + + for (Path p : processedLogs) { + Path newPath = HLog.getHLogArchivePath(oldLogDir, p); + if (fs.exists(p)) { + if (!HBaseFileSystem.renameDirForFileSystem(fs, p, newPath)) { + LOG.warn("Unable to move " + p + " to " + newPath); + } else { + LOG.debug("Archived processed log " + p + " to " + newPath); + } + } + } + + // distributed log splitting removes the srcDir (region's log dir) later + // when all the log files in that srcDir have been successfully processed + if (srcDir != null && !HBaseFileSystem.deleteDirFromFileSystem(fs, srcDir)) { + throw new IOException("Unable to delete src dir: " + srcDir); + } + } + + /** + * Path to a file under RECOVERED_EDITS_DIR directory of the region found in + * logEntry named for the sequenceid in the passed + * logEntry: e.g. /hbase/some_table/2323432434/recovered.edits/2332. + * This method also ensures existence of RECOVERED_EDITS_DIR under the region + * creating it if necessary. + * @param fs + * @param logEntry + * @param rootDir HBase root dir. + * @return Path to file into which to dump split log edits. + * @throws IOException + */ + static Path getRegionSplitEditsPath(final FileSystem fs, + final Entry logEntry, final Path rootDir, boolean isCreate) + throws IOException { + Path tableDir = HTableDescriptor.getTableDir(rootDir, logEntry.getKey() + .getTablename()); + Path regiondir = HRegion.getRegionDir(tableDir, + Bytes.toString(logEntry.getKey().getEncodedRegionName())); + Path dir = HLog.getRegionDirRecoveredEditsDir(regiondir); + + if (!fs.exists(regiondir)) { + LOG.info("This region's directory doesn't exist: " + + regiondir.toString() + ". It is very likely that it was" + + " already split so it's safe to discard those edits."); + return null; + } + if (isCreate && !fs.exists(dir) && + !HBaseFileSystem.makeDirOnFileSystem(fs, dir)) { + LOG.warn("mkdir failed on " + dir); + } + // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure + // region's replayRecoveredEdits will not delete it + String fileName = formatRecoveredEditsFileName(logEntry.getKey() + .getLogSeqNum()); + fileName = getTmpRecoveredEditsFileName(fileName); + return new Path(dir, fileName); + } + + static String getTmpRecoveredEditsFileName(String fileName) { + return fileName + HLog.RECOVERED_LOG_TMPFILE_SUFFIX; + } + + /** + * Get the completed recovered edits file path, renaming it to be by last edit + * in the file from its first edit. Then we could use the name to skip + * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}. + * @param srcPath + * @param maximumEditLogSeqNum + * @return dstPath take file's last edit log seq num as the name + */ + static Path getCompletedRecoveredEditsFilePath(Path srcPath, + Long maximumEditLogSeqNum) { + String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum); + return new Path(srcPath.getParent(), fileName); + } + + static String formatRecoveredEditsFileName(final long seqid) { + return String.format("%019d", seqid); + } + + /** + * Parse a single hlog and put the edits in entryBuffers + * + * @param in the hlog reader + * @param path the path of the log file + * @param entryBuffers the buffer to hold the parsed edits + * @param fs the file system + * @param conf the configuration + * @param skipErrors indicator if CorruptedLogFileException should be thrown instead of IOException + * @throws IOException + * @throws CorruptedLogFileException if hlog is corrupted + */ + private void parseHLog(final Reader in, Path path, + EntryBuffers entryBuffers, final FileSystem fs, + final Configuration conf, boolean skipErrors) + throws IOException, CorruptedLogFileException { + int editsCount = 0; + try { + Entry entry; + while ((entry = getNextLogLine(in, path, skipErrors)) != null) { + entryBuffers.appendEntry(entry); + editsCount++; + } + } catch (InterruptedException ie) { + IOException t = new InterruptedIOException(); + t.initCause(ie); + throw t; + } finally { + LOG.debug("Pushed=" + editsCount + " entries from " + path); + } + } + + /** + * Create a new {@link Reader} for reading logs to split. + * + * @param fs + * @param file + * @param conf + * @return A new Reader instance + * @throws IOException + * @throws CorruptedLogFile + */ + protected Reader getReader(FileSystem fs, FileStatus file, Configuration conf, + boolean skipErrors) + throws IOException, CorruptedLogFileException { + Path path = file.getPath(); + long length = file.getLen(); + Reader in; + + + // Check for possibly empty file. With appends, currently Hadoop reports a + // zero length even if the file has been sync'd. Revisit if HDFS-376 or + // HDFS-878 is committed. + if (length <= 0) { + LOG.warn("File " + path + " might be still open, length is 0"); + } + + try { + FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf); + try { + in = getReader(fs, path, conf); + } catch (EOFException e) { + if (length <= 0) { + // TODO should we ignore an empty, not-last log file if skip.errors + // is false? Either way, the caller should decide what to do. E.g. + // ignore if this is the last log in sequence. + // TODO is this scenario still possible if the log has been + // recovered (i.e. closed) + LOG.warn("Could not open " + path + " for reading. File is empty", e); + return null; + } else { + // EOFException being ignored + return null; + } + } + } catch (IOException e) { + if (!skipErrors) { + throw e; + } + CorruptedLogFileException t = + new CorruptedLogFileException("skipErrors=true Could not open hlog " + + path + " ignoring"); + t.initCause(e); + throw t; + } + return in; + } + + static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors) + throws CorruptedLogFileException, IOException { + try { + return in.next(); + } catch (EOFException eof) { + // truncated files are expected if a RS crashes (see HBASE-2643) + LOG.info("EOF from hlog " + path + ". continuing"); + return null; + } catch (IOException e) { + // If the IOE resulted from bad file format, + // then this problem is idempotent and retrying won't help + if (e.getCause() != null && + (e.getCause() instanceof ParseException || + e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) { + LOG.warn("Parse exception " + e.getCause().toString() + " from hlog " + + path + ". continuing"); + return null; + } + if (!skipErrors) { + throw e; + } + CorruptedLogFileException t = + new CorruptedLogFileException("skipErrors=true Ignoring exception" + + " while parsing hlog " + path + ". Marking as corrupted"); + t.initCause(e); + throw t; + } + } + + + private void writerThreadError(Throwable t) { + thrown.compareAndSet(null, t); + } + + /** + * Check for errors in the writer threads. If any is found, rethrow it. + */ + private void checkForErrors() throws IOException { + Throwable thrown = this.thrown.get(); + if (thrown == null) return; + if (thrown instanceof IOException) { + throw (IOException)thrown; + } else { + throw new RuntimeException(thrown); + } + } + /** + * Create a new {@link Writer} for writing log splits. + */ + protected Writer createWriter(FileSystem fs, Path logfile, Configuration conf) + throws IOException { + return hlogFs.createWriter(fs, conf, logfile); + } + + /** + * Create a new {@link Reader} for reading logs to split. + */ + protected Reader getReader(FileSystem fs, Path curLogFile, Configuration conf) + throws IOException { + return HLog.getReader(fs, curLogFile, conf); + } + + /** + * Class which accumulates edits and separates them into a buffer per region + * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses + * a predefined threshold. + * + * Writer threads then pull region-specific buffers from this class. + */ + class EntryBuffers { + Map buffers = + new TreeMap(Bytes.BYTES_COMPARATOR); + + /* Track which regions are currently in the middle of writing. We don't allow + an IO thread to pick up bytes from a region if we're already writing + data for that region in a different IO thread. */ + Set currentlyWriting = new TreeSet(Bytes.BYTES_COMPARATOR); + + long totalBuffered = 0; + long maxHeapUsage; + + EntryBuffers(long maxHeapUsage) { + this.maxHeapUsage = maxHeapUsage; + } + + /** + * Append a log entry into the corresponding region buffer. + * Blocks if the total heap usage has crossed the specified threshold. + * + * @throws InterruptedException + * @throws IOException + */ + void appendEntry(Entry entry) throws InterruptedException, IOException { + HLogKey key = entry.getKey(); + + RegionEntryBuffer buffer; + long incrHeap; + synchronized (this) { + buffer = buffers.get(key.getEncodedRegionName()); + if (buffer == null) { + buffer = new RegionEntryBuffer(key.getTablename(), key.getEncodedRegionName()); + buffers.put(key.getEncodedRegionName(), buffer); + } + incrHeap= buffer.appendEntry(entry); + } + + // If we crossed the chunk threshold, wait for more space to be available + synchronized (dataAvailable) { + totalBuffered += incrHeap; + while (totalBuffered > maxHeapUsage && thrown.get() == null) { + LOG.debug("Used " + totalBuffered + " bytes of buffered edits, waiting for IO threads..."); + dataAvailable.wait(3000); + } + dataAvailable.notifyAll(); + } + checkForErrors(); + } + + synchronized RegionEntryBuffer getChunkToWrite() { + long biggestSize=0; + byte[] biggestBufferKey=null; + + for (Map.Entry entry : buffers.entrySet()) { + long size = entry.getValue().heapSize(); + if (size > biggestSize && !currentlyWriting.contains(entry.getKey())) { + biggestSize = size; + biggestBufferKey = entry.getKey(); + } + } + if (biggestBufferKey == null) { + return null; + } + + RegionEntryBuffer buffer = buffers.remove(biggestBufferKey); + currentlyWriting.add(biggestBufferKey); + return buffer; + } + + void doneWriting(RegionEntryBuffer buffer) { + synchronized (this) { + boolean removed = currentlyWriting.remove(buffer.encodedRegionName); + assert removed; + } + long size = buffer.heapSize(); + + synchronized (dataAvailable) { + totalBuffered -= size; + // We may unblock writers + dataAvailable.notifyAll(); + } + } + + synchronized boolean isRegionCurrentlyWriting(byte[] region) { + return currentlyWriting.contains(region); + } + } + + /** + * A buffer of some number of edits for a given region. + * This accumulates edits and also provides a memory optimization in order to + * share a single byte array instance for the table and region name. + * Also tracks memory usage of the accumulated edits. + */ + static class RegionEntryBuffer implements HeapSize { + long heapInBuffer = 0; + List entryBuffer; + byte[] tableName; + byte[] encodedRegionName; + + RegionEntryBuffer(byte[] table, byte[] region) { + this.tableName = table; + this.encodedRegionName = region; + this.entryBuffer = new LinkedList(); + } + + long appendEntry(Entry entry) { + internify(entry); + entryBuffer.add(entry); + long incrHeap = entry.getEdit().heapSize() + + ClassSize.align(2 * ClassSize.REFERENCE) + // HLogKey pointers + 0; // TODO linkedlist entry + heapInBuffer += incrHeap; + return incrHeap; + } + + private void internify(Entry entry) { + HLogKey k = entry.getKey(); + k.internTableName(this.tableName); + k.internEncodedRegionName(this.encodedRegionName); + } + + public long heapSize() { + return heapInBuffer; + } + } + + + class WriterThread extends Thread { + private volatile boolean shouldStop = false; + + WriterThread(int i) { + super("WriterThread-" + i); + } + + public void run() { + try { + doRun(); + } catch (Throwable t) { + LOG.error("Error in log splitting write thread", t); + writerThreadError(t); + } + } + + private void doRun() throws IOException { + LOG.debug("Writer thread " + this + ": starting"); + while (true) { + RegionEntryBuffer buffer = entryBuffers.getChunkToWrite(); + if (buffer == null) { + // No data currently available, wait on some more to show up + synchronized (dataAvailable) { + if (shouldStop) return; + try { + dataAvailable.wait(1000); + } catch (InterruptedException ie) { + if (!shouldStop) { + throw new RuntimeException(ie); + } + } + } + continue; + } + + assert buffer != null; + try { + writeBuffer(buffer); + } finally { + entryBuffers.doneWriting(buffer); + } + } + } + + + private void writeBuffer(RegionEntryBuffer buffer) throws IOException { + List entries = buffer.entryBuffer; + if (entries.isEmpty()) { + LOG.warn(this.getName() + " got an empty buffer, skipping"); + return; + } + + WriterAndPath wap = null; + + long startTime = System.nanoTime(); + try { + int editsCount = 0; + + for (Entry logEntry : entries) { + if (wap == null) { + wap = outputSink.getWriterAndPath(logEntry); + if (wap == null) { + // getWriterAndPath decided we don't need to write these edits + // Message was already logged + return; + } + } + wap.w.append(logEntry); + outputSink.updateRegionMaximumEditLogSeqNum(logEntry); + editsCount++; + } + // Pass along summary statistics + wap.incrementEdits(editsCount); + wap.incrementNanoTime(System.nanoTime() - startTime); + } catch (IOException e) { + e = RemoteExceptionHandler.checkIOException(e); + LOG.fatal(this.getName() + " Got while writing log entry to log", e); + throw e; + } + } + + void finish() { + synchronized (dataAvailable) { + shouldStop = true; + dataAvailable.notifyAll(); + } + } + } + + private WriterAndPath createWAP(byte[] region, Entry entry, Path rootdir, + FileSystem fs, Configuration conf) + throws IOException { + Path regionedits = getRegionSplitEditsPath(fs, entry, rootdir, true); + if (regionedits == null) { + return null; + } + if (fs.exists(regionedits)) { + LOG.warn("Found existing old edits file. It could be the " + + "result of a previous failed split attempt. Deleting " + + regionedits + ", length=" + + fs.getFileStatus(regionedits).getLen()); + if (!HBaseFileSystem.deleteFileFromFileSystem(fs, regionedits)) { + LOG.warn("Failed delete of old " + regionedits); + } + } + Writer w = createWriter(fs, regionedits, conf); + LOG.debug("Creating writer path=" + regionedits + " region=" + + Bytes.toStringBinary(region)); + return (new WriterAndPath(regionedits, w)); + } + + Path convertRegionEditsToTemp(Path rootdir, Path edits, String tmpname) { + List components = new ArrayList(10); + do { + components.add(edits.getName()); + edits = edits.getParent(); + } while (edits.depth() > rootdir.depth()); + Path ret = ZKSplitLog.getSplitLogDir(rootdir, tmpname); + for (int i = components.size() - 1; i >= 0; i--) { + ret = new Path(ret, components.get(i)); + } + try { + if (fs.exists(ret)) { + LOG.warn("Found existing old temporary edits file. It could be the " + + "result of a previous failed split attempt. Deleting " + + ret + ", length=" + + fs.getFileStatus(ret).getLen()); + if (!HBaseFileSystem.deleteFileFromFileSystem(fs, ret)) { + LOG.warn("Failed delete of old " + ret); + } + } + Path dir = ret.getParent(); + if (!fs.exists(dir) && !HBaseFileSystem.makeDirOnFileSystem(fs, dir)) { + LOG.warn("mkdir failed on " + dir); + } + } catch (IOException e) { + LOG.warn("Could not prepare temp staging area ", e); + // ignore, exceptions will be thrown elsewhere + } + return ret; + } + + /** + * Class that manages the output streams from the log splitting process. + */ + class OutputSink { + private final Map logWriters = Collections.synchronizedMap( + new TreeMap(Bytes.BYTES_COMPARATOR)); + private final Map regionMaximumEditLogSeqNum = Collections + .synchronizedMap(new TreeMap(Bytes.BYTES_COMPARATOR)); + private final List writerThreads = Lists.newArrayList(); + + /* Set of regions which we've decided should not output edits */ + private final Set blacklistedRegions = Collections.synchronizedSet( + new TreeSet(Bytes.BYTES_COMPARATOR)); + + private boolean closeAndCleanCompleted = false; + + private boolean logWritersClosed = false; + + /** + * Start the threads that will pump data from the entryBuffers + * to the output files. + * @return the list of started threads + */ + synchronized void startWriterThreads(EntryBuffers entryBuffers) { + // More threads could potentially write faster at the expense + // of causing more disk seeks as the logs are split. + // 3. After a certain setting (probably around 3) the + // process will be bound on the reader in the current + // implementation anyway. + int numThreads = conf.getInt( + "hbase.regionserver.hlog.splitlog.writer.threads", 3); + + for (int i = 0; i < numThreads; i++) { + WriterThread t = new WriterThread(i); + t.start(); + writerThreads.add(t); + } + } + + List finishWritingAndClose() throws IOException { + LOG.info("Waiting for split writer threads to finish"); + try { + for (WriterThread t : writerThreads) { + t.finish(); + } + for (WriterThread t : writerThreads) { + try { + t.join(); + } catch (InterruptedException ie) { + throw new IOException(ie); + } + checkForErrors(); + } + LOG.info("Split writers finished"); + + return closeStreams(); + } finally { + List thrown = closeLogWriters(null); + if (thrown != null && !thrown.isEmpty()) { + throw MultipleIOException.createIOException(thrown); + } + } + } + + /** + * Close all of the output streams. + * @return the list of paths written. + */ + private List closeStreams() throws IOException { + Preconditions.checkState(!closeAndCleanCompleted); + + List paths = new ArrayList(); + List thrown = Lists.newArrayList(); + closeLogWriters(thrown); + for (Map.Entry logWritersEntry : logWriters + .entrySet()) { + WriterAndPath wap = logWritersEntry.getValue(); + Path dst = getCompletedRecoveredEditsFilePath(wap.p, + regionMaximumEditLogSeqNum.get(logWritersEntry.getKey())); + try { + if (!dst.equals(wap.p) && fs.exists(dst)) { + LOG.warn("Found existing old edits file. It could be the " + + "result of a previous failed split attempt. Deleting " + dst + + ", length=" + fs.getFileStatus(dst).getLen()); + if (!HBaseFileSystem.deleteFileFromFileSystem(fs, dst)) { + LOG.warn("Failed deleting of old " + dst); + throw new IOException("Failed deleting of old " + dst); + } + } + // Skip the unit tests which create a splitter that reads and writes + // the data without touching disk. TestHLogSplit#testThreading is an + // example. + if (fs.exists(wap.p)) { + if (!HBaseFileSystem.renameDirForFileSystem(fs, wap.p, dst)) { + throw new IOException("Failed renaming " + wap.p + " to " + dst); + } + LOG.debug("Rename " + wap.p + " to " + dst); + } + } catch (IOException ioe) { + LOG.error("Couldn't rename " + wap.p + " to " + dst, ioe); + thrown.add(ioe); + continue; + } + paths.add(dst); + } + if (!thrown.isEmpty()) { + throw MultipleIOException.createIOException(thrown); + } + + closeAndCleanCompleted = true; + return paths; + } + + private List closeLogWriters(List thrown) + throws IOException { + if (!logWritersClosed) { + if (thrown == null) { + thrown = Lists.newArrayList(); + } + try { + for (WriterThread t : writerThreads) { + while (t.isAlive()) { + t.shouldStop = true; + t.interrupt(); + try { + t.join(10); + } catch (InterruptedException e) { + IOException iie = new InterruptedIOException(); + iie.initCause(e); + throw iie; + } + } + } + } finally { + synchronized (logWriters) { + for (WriterAndPath wap : logWriters.values()) { + try { + wap.w.close(); + } catch (IOException ioe) { + LOG.error("Couldn't close log at " + wap.p, ioe); + thrown.add(ioe); + continue; + } + LOG.info("Closed path " + wap.p + " (wrote " + wap.editsWritten + + " edits in " + (wap.nanosSpent / 1000 / 1000) + "ms)"); + } + } + logWritersClosed = true; + } + } + return thrown; + } + + /** + * Get a writer and path for a log starting at the given entry. + * + * This function is threadsafe so long as multiple threads are always + * acting on different regions. + * + * @return null if this region shouldn't output any logs + */ + WriterAndPath getWriterAndPath(Entry entry) throws IOException { + byte region[] = entry.getKey().getEncodedRegionName(); + WriterAndPath ret = logWriters.get(region); + if (ret != null) { + return ret; + } + // If we already decided that this region doesn't get any output + // we don't need to check again. + if (blacklistedRegions.contains(region)) { + return null; + } + ret = createWAP(region, entry, rootDir, fs, conf); + if (ret == null) { + blacklistedRegions.add(region); + return null; + } + logWriters.put(region, ret); + return ret; + } + + /** + * Update region's maximum edit log SeqNum. + */ + void updateRegionMaximumEditLogSeqNum(Entry entry) { + synchronized (regionMaximumEditLogSeqNum) { + Long currentMaxSeqNum=regionMaximumEditLogSeqNum.get(entry.getKey().getEncodedRegionName()); + if (currentMaxSeqNum == null + || entry.getKey().getLogSeqNum() > currentMaxSeqNum) { + regionMaximumEditLogSeqNum.put(entry.getKey().getEncodedRegionName(), + entry.getKey().getLogSeqNum()); + } + } + + } + + Long getRegionMaximumEditLogSeqNum(byte[] region) { + return regionMaximumEditLogSeqNum.get(region); + } + + /** + * @return a map from encoded region ID to the number of edits written out + * for that region. + */ + private Map getOutputCounts() { + TreeMap ret = new TreeMap( + Bytes.BYTES_COMPARATOR); + synchronized (logWriters) { + for (Map.Entry entry : logWriters.entrySet()) { + ret.put(entry.getKey(), entry.getValue().editsWritten); + } + } + return ret; + } + } + + + + /** + * Private data structure that wraps a Writer and its Path, + * also collecting statistics about the data written to this + * output. + */ + private final static class WriterAndPath { + final Path p; + final Writer w; + + /* Count of edits written to this path */ + long editsWritten = 0; + /* Number of nanos spent writing to this log */ + long nanosSpent = 0; + + /* To check whether a close has already been tried on the + * writer + */ + boolean writerClosed = false; + + WriterAndPath(final Path p, final Writer w) { + this.p = p; + this.w = w; + } + + void incrementEdits(int edits) { + editsWritten += edits; + } + + void incrementNanoTime(long nanos) { + nanosSpent += nanos; + } + } + + static class CorruptedLogFileException extends Exception { + private static final long serialVersionUID = 1L; + CorruptedLogFileException(String s) { + super(s); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java new file mode 100644 index 0000000..0f9743f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; + +/** + * Compression class for {@link KeyValue}s written to the WAL. This is not + * synchronized, so synchronization should be handled outside. + * + * Class only compresses and uncompresses row keys, family names, and the + * qualifier. More may be added depending on use patterns. + */ +class KeyValueCompression { + /** + * Uncompresses a KeyValue from a DataInput and returns it. + * + * @param in the DataInput + * @param readContext the compressionContext to use. + * @return an uncompressed KeyValue + * @throws IOException + */ + + public static KeyValue readKV(DataInput in, CompressionContext readContext) + throws IOException { + int keylength = WritableUtils.readVInt(in); + int vlength = WritableUtils.readVInt(in); + int length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength; + + byte[] backingArray = new byte[length]; + int pos = 0; + pos = Bytes.putInt(backingArray, pos, keylength); + pos = Bytes.putInt(backingArray, pos, vlength); + + // the row + int elemLen = Compressor.uncompressIntoArray(backingArray, + pos + Bytes.SIZEOF_SHORT, in, readContext.rowDict); + checkLength(elemLen, Short.MAX_VALUE); + pos = Bytes.putShort(backingArray, pos, (short)elemLen); + pos += elemLen; + + // family + elemLen = Compressor.uncompressIntoArray(backingArray, + pos + Bytes.SIZEOF_BYTE, in, readContext.familyDict); + checkLength(elemLen, Byte.MAX_VALUE); + pos = Bytes.putByte(backingArray, pos, (byte)elemLen); + pos += elemLen; + + // qualifier + elemLen = Compressor.uncompressIntoArray(backingArray, pos, in, + readContext.qualifierDict); + pos += elemLen; + + // the rest + in.readFully(backingArray, pos, length - pos); + + return new KeyValue(backingArray); + } + + private static void checkLength(int len, int max) throws IOException { + if (len < 0 || len > max) { + throw new IOException( + "Invalid length for compresesed portion of keyvalue: " + len); + } + } + + /** + * Compresses and writes ourKV to out, a DataOutput. + * + * @param out the DataOutput + * @param keyVal the KV to compress and write + * @param writeContext the compressionContext to use. + * @throws IOException + */ + public static void writeKV(final DataOutput out, KeyValue keyVal, + CompressionContext writeContext) throws IOException { + byte[] backingArray = keyVal.getBuffer(); + int offset = keyVal.getOffset(); + + // we first write the KeyValue infrastructure as VInts. + WritableUtils.writeVInt(out, keyVal.getKeyLength()); + WritableUtils.writeVInt(out, keyVal.getValueLength()); + + // now we write the row key, as the row key is likely to be repeated + // We save space only if we attempt to compress elements with duplicates + Compressor.writeCompressed(keyVal.getBuffer(), keyVal.getRowOffset(), + keyVal.getRowLength(), out, writeContext.rowDict); + + + // now family, if it exists. if it doesn't, we write a 0 length array. + Compressor.writeCompressed(keyVal.getBuffer(), keyVal.getFamilyOffset(), + keyVal.getFamilyLength(), out, writeContext.familyDict); + + // qualifier next + Compressor.writeCompressed(keyVal.getBuffer(), keyVal.getQualifierOffset(), + keyVal.getQualifierLength(), out, + writeContext.qualifierDict); + + // now we write the rest uncompressed + int pos = keyVal.getTimestampOffset(); + int remainingLength = keyVal.getLength() + offset - (pos); + out.write(backingArray, pos, remainingLength); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LRUDictionary.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LRUDictionary.java new file mode 100644 index 0000000..5ee18fc --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/LRUDictionary.java @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import java.util.HashMap; + +import org.apache.hadoop.hbase.util.Bytes; + +import com.google.common.base.Preconditions; + +/** + * WALDictionary using an LRU eviction algorithm. Uses a linked list running + * through a hashtable. Currently has max of 2^15 entries. Will start + * evicting if exceeds this number The maximum memory we expect this dictionary + * to take in the worst case is about: + * (2 ^ 15) * 5 (Regionname, Row key, CF, Column qual, table) * 100 bytes (these are some big names) = ~16MB. + * If you want to get silly, even at 1kb entries, it maxes out at 160 megabytes. + */ +public class LRUDictionary implements Dictionary { + private final BidirectionalLRUMap backingStore = new BidirectionalLRUMap(); + + @Override + public byte[] getEntry(short idx) { + return backingStore.get(idx); + } + + @Override + public short findEntry(byte[] data, int offset, int length) { + short ret = backingStore.findIdx(data, offset, length); + if (ret == NOT_IN_DICTIONARY) { + addEntry(data, offset, length); + } + return ret; + } + + @Override + public short addEntry(byte[] data, int offset, int length) { + if (length <= 0) return NOT_IN_DICTIONARY; + return backingStore.put(data, offset, length); + } + + @Override + public void clear() { + backingStore.clear(); + } + + /* + * Internal class used to implement LRU eviction and dual lookup (by key and + * value). + * + * This is not thread safe. Don't use in multi-threaded applications. + */ + static class BidirectionalLRUMap { + static final int MAX_SIZE = Short.MAX_VALUE; + private int currSize = 0; + + // Head and tail of the LRU list. + private Node head; + private Node tail; + + private HashMap nodeToIndex = new HashMap(); + private Node[] indexToNode = new Node[MAX_SIZE]; + + public BidirectionalLRUMap() { + for (int i = 0; i < MAX_SIZE; i++) { + indexToNode[i] = new Node(); + } + } + + private short put(byte[] array, int offset, int length) { + // We copy the bytes we want, otherwise we might be holding references to + // massive arrays in our dictionary (or those arrays might change) + byte[] stored = new byte[length]; + Bytes.putBytes(stored, 0, array, offset, length); + + if (currSize < MAX_SIZE) { + // There is space to add without evicting. + indexToNode[currSize].setContents(stored, 0, stored.length); + setHead(indexToNode[currSize]); + short ret = (short) currSize++; + nodeToIndex.put(indexToNode[ret], ret); + return ret; + } else { + short s = nodeToIndex.remove(tail); + tail.setContents(stored, 0, stored.length); + // we need to rehash this. + nodeToIndex.put(tail, s); + moveToHead(tail); + return s; + } + } + + private short findIdx(byte[] array, int offset, int length) { + Short s; + final Node comparisonNode = new Node(); + comparisonNode.setContents(array, offset, length); + if ((s = nodeToIndex.get(comparisonNode)) != null) { + moveToHead(indexToNode[s]); + return s; + } else { + return -1; + } + } + + private byte[] get(short idx) { + Preconditions.checkElementIndex(idx, currSize); + moveToHead(indexToNode[idx]); + return indexToNode[idx].container; + } + + private void moveToHead(Node n) { + if (head == n) { + // no-op -- it's already the head. + return; + } + // At this point we definitely have prev, since it's not the head. + assert n.prev != null; + // Unlink prev. + n.prev.next = n.next; + + // Unlink next + if (n.next != null) { + n.next.prev = n.prev; + } else { + assert n == tail; + tail = n.prev; + } + // Node is now removed from the list. Re-add it at the head. + setHead(n); + } + + private void setHead(Node n) { + // assume it's already unlinked from the list at this point. + n.prev = null; + n.next = head; + if (head != null) { + assert head.prev == null; + head.prev = n; + } + + head = n; + + // First entry + if (tail == null) { + tail = n; + } + } + + private void clear() { + currSize = 0; + nodeToIndex.clear(); + tail = null; + head = null; + + for (Node n : indexToNode) { + n.container = null; + } + + for (int i = 0; i < MAX_SIZE; i++) { + indexToNode[i].next = null; + indexToNode[i].prev = null; + } + } + + private static class Node { + byte[] container; + int offset; + int length; + Node next; // link towards the tail + Node prev; // link towards the head + + public Node() { + } + + private void setContents(byte[] container, int offset, int length) { + this.container = container; + this.offset = offset; + this.length = length; + } + + @Override + public int hashCode() { + return Bytes.hashCode(container, offset, length); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Node)) { + return false; + } + + Node casted = (Node) other; + return Bytes.equals(container, offset, length, casted.container, + casted.offset, casted.length); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/OrphanHLogAfterSplitException.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/OrphanHLogAfterSplitException.java new file mode 100644 index 0000000..1c93def --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/OrphanHLogAfterSplitException.java @@ -0,0 +1,40 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; + +public class OrphanHLogAfterSplitException extends IOException { + + /** + * Create this exception without a message + */ + public OrphanHLogAfterSplitException() { + super(); + } + + /** + * Create this exception with a message + * @param message why it failed + */ + public OrphanHLogAfterSplitException(String message) { + super(message); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java new file mode 100644 index 0000000..a56be62 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java @@ -0,0 +1,310 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.SequenceFile; + +public class SequenceFileLogReader implements HLog.Reader { + private static final Log LOG = LogFactory.getLog(SequenceFileLogReader.class); + + /** + * Hack just to set the correct file length up in SequenceFile.Reader. + * See HADOOP-6307. The below is all about setting the right length on the + * file we are reading. fs.getFileStatus(file).getLen() is passed down to + * a private SequenceFile.Reader constructor. This won't work. Need to do + * the available on the stream. The below is ugly. It makes getPos, the + * first time its called, return length of the file -- i.e. tell a lie -- just + * so this line up in SF.Reader's constructor ends up with right answer: + * + * this.end = in.getPos() + length; + * + */ + static class WALReader extends SequenceFile.Reader { + + WALReader(final FileSystem fs, final Path p, final Configuration c) + throws IOException { + super(fs, p, c); + } + + @Override + protected FSDataInputStream openFile(FileSystem fs, Path file, + int bufferSize, long length) + throws IOException { + return new WALReaderFSDataInputStream(super.openFile(fs, file, + bufferSize, length), length); + } + + /** + * Call this method after init() has been executed + * + * @return whether WAL compression is enabled + */ + public boolean isWALCompressionEnabled() { + return SequenceFileLogWriter.isWALCompressionEnabled(this.getMetadata()); + } + + /** + * Override just so can intercept first call to getPos. + */ + static class WALReaderFSDataInputStream extends FSDataInputStream { + private boolean firstGetPosInvocation = true; + private long length; + + WALReaderFSDataInputStream(final FSDataInputStream is, final long l) + throws IOException { + super(is); + this.length = l; + } + + // This section can be confusing. It is specific to how HDFS works. + // Let me try to break it down. This is the problem: + // + // 1. HDFS DataNodes update the NameNode about a filename's length + // on block boundaries or when a file is closed. Therefore, + // if an RS dies, then the NN's fs.getLength() can be out of date + // 2. this.in.available() would work, but it returns int & + // therefore breaks for files > 2GB (happens on big clusters) + // 3. DFSInputStream.getFileLength() gets the actual length from the DNs + // 4. DFSInputStream is wrapped 2 levels deep : this.in.in + // + // So, here we adjust getPos() using getFileLength() so the + // SequenceFile.Reader constructor (aka: first invocation) comes out + // with the correct end of the file: + // this.end = in.getPos() + length; + @Override + public long getPos() throws IOException { + if (this.firstGetPosInvocation) { + this.firstGetPosInvocation = false; + long adjust = 0; + + try { + Field fIn = FilterInputStream.class.getDeclaredField("in"); + fIn.setAccessible(true); + Object realIn = fIn.get(this.in); + // In hadoop 0.22, DFSInputStream is a standalone class. Before this, + // it was an inner class of DFSClient. + if (realIn.getClass().getName().endsWith("DFSInputStream")) { + Method getFileLength = realIn.getClass(). + getDeclaredMethod("getFileLength", new Class []{}); + getFileLength.setAccessible(true); + long realLength = ((Long)getFileLength. + invoke(realIn, new Object []{})).longValue(); + assert(realLength >= this.length); + adjust = realLength - this.length; + } else { + LOG.info("Input stream class: " + realIn.getClass().getName() + + ", not adjusting length"); + } + } catch(Exception e) { + SequenceFileLogReader.LOG.warn( + "Error while trying to get accurate file length. " + + "Truncation / data loss may occur if RegionServers die.", e); + } + + return adjust + super.getPos(); + } + return super.getPos(); + } + } + } + + Configuration conf; + WALReader reader; + FileSystem fs; + + // Needed logging exceptions + Path path; + int edit = 0; + long entryStart = 0; + boolean emptyCompressionContext = true; + /** + * Compression context to use reading. Can be null if no compression. + */ + protected CompressionContext compressionContext = null; + + protected Class keyClass; + + /** + * Default constructor. + */ + public SequenceFileLogReader() { + } + + /** + * This constructor allows a specific HLogKey implementation to override that + * which would otherwise be chosen via configuration property. + * + * @param keyClass + */ + public SequenceFileLogReader(Class keyClass) { + this.keyClass = keyClass; + } + + @Override + public void init(FileSystem fs, Path path, Configuration conf) + throws IOException { + this.conf = conf; + this.path = path; + reader = new WALReader(fs, path, conf); + this.fs = fs; + + // If compression is enabled, new dictionaries are created here. + boolean compression = reader.isWALCompressionEnabled(); + if (compression) { + try { + if (compressionContext == null) { + compressionContext = new CompressionContext(LRUDictionary.class); + } else { + compressionContext.clear(); + } + } catch (Exception e) { + throw new IOException("Failed to initialize CompressionContext", e); + } + } + } + + @Override + public void close() throws IOException { + try { + if (reader != null) { + this.reader.close(); + this.reader = null; + } + } catch (IOException ioe) { + throw addFileInfoToException(ioe); + } + } + + @Override + public HLog.Entry next() throws IOException { + return next(null); + } + + @Override + public HLog.Entry next(HLog.Entry reuse) throws IOException { + this.entryStart = this.reader.getPosition(); + HLog.Entry e = reuse; + if (e == null) { + HLogKey key; + if (keyClass == null) { + key = HLog.newKey(conf); + } else { + try { + key = keyClass.newInstance(); + } catch (InstantiationException ie) { + throw new IOException(ie); + } catch (IllegalAccessException iae) { + throw new IOException(iae); + } + } + + WALEdit val = new WALEdit(); + e = new HLog.Entry(key, val); + } + boolean b = false; + try { + if (compressionContext != null) { + e.setCompressionContext(compressionContext); + } + b = this.reader.next(e.getKey(), e.getEdit()); + } catch (IOException ioe) { + throw addFileInfoToException(ioe); + } + edit++; + if (compressionContext != null && emptyCompressionContext) { + emptyCompressionContext = false; + } + return b? e: null; + } + + @Override + public void seek(long pos) throws IOException { + if (compressionContext != null && emptyCompressionContext) { + while (next() != null) { + if (getPosition() == pos) { + emptyCompressionContext = false; + break; + } + } + } + try { + reader.seek(pos); + } catch (IOException ioe) { + throw addFileInfoToException(ioe); + } + } + + @Override + public long getPosition() throws IOException { + return reader != null ? reader.getPosition() : 0; + } + + protected IOException addFileInfoToException(final IOException ioe) + throws IOException { + long pos = -1; + try { + pos = getPosition(); + } catch (IOException e) { + LOG.warn("Failed getting position to add to throw", e); + } + + // See what SequenceFile.Reader thinks is the end of the file + long end = Long.MAX_VALUE; + try { + Field fEnd = SequenceFile.Reader.class.getDeclaredField("end"); + fEnd.setAccessible(true); + end = fEnd.getLong(this.reader); + } catch(Exception e) { /* reflection fail. keep going */ } + + String msg = (this.path == null? "": this.path.toString()) + + ", entryStart=" + entryStart + ", pos=" + pos + + ((end == Long.MAX_VALUE) ? "" : ", end=" + end) + + ", edit=" + this.edit; + + // Enhance via reflection so we don't change the original class type + try { + return (IOException) ioe.getClass() + .getConstructor(String.class) + .newInstance(msg) + .initCause(ioe); + } catch(Exception e) { /* reflection fail. keep going */ } + + return ioe; + } + + @Override + public void reset() throws IOException { + // Resetting the reader lets us see newly added data if the file is being written to + // We also keep the same compressionContext which was previously populated for this file + reader = new WALReader(fs, path, conf); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java new file mode 100644 index 0000000..0c09c83 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java @@ -0,0 +1,333 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.SequenceFile.CompressionType; +import org.apache.hadoop.io.SequenceFile.Metadata; +import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.io.compress.DefaultCodec; + +/** + * Implementation of {@link HLog.Writer} that delegates to + * SequenceFile.Writer. + */ +public class SequenceFileLogWriter implements HLog.Writer { + static final Text WAL_VERSION_KEY = new Text("version"); + // Let the version be 1. Let absence of a version meta tag be old, version 0. + // Set this version '1' to be the version that introduces compression, + // the COMPRESSION_VERSION. + private static final int COMPRESSION_VERSION = 1; + static final int VERSION = COMPRESSION_VERSION; + static final Text WAL_VERSION = new Text("" + VERSION); + static final Text WAL_COMPRESSION_TYPE_KEY = new Text("compression.type"); + static final Text DICTIONARY_COMPRESSION_TYPE = new Text("dictionary"); + + private final Log LOG = LogFactory.getLog(this.getClass()); + // The sequence file we delegate to. + private SequenceFile.Writer writer; + // This is the FSDataOutputStream instance that is the 'out' instance + // in the SequenceFile.Writer 'writer' instance above. + private FSDataOutputStream writer_out; + + private Class keyClass; + + /** + * Context used by our wal dictionary compressor. Null if we're not to do + * our custom dictionary compression. This custom WAL compression is distinct + * from sequencefile native compression. + */ + private CompressionContext compressionContext; + + private Method syncFs = null; + private Method hflush = null; + + /** + * Default constructor. + */ + public SequenceFileLogWriter() { + super(); + } + + /** + * This constructor allows a specific HLogKey implementation to override that + * which would otherwise be chosen via configuration property. + * + * @param keyClass + */ + public SequenceFileLogWriter(Class keyClass) { + this.keyClass = keyClass; + } + + /** + * Create sequence file Metadata for our WAL file with version and compression + * type (if any). + * @param conf + * @param compress + * @return Metadata instance. + */ + private static Metadata createMetadata(final Configuration conf, + final boolean compress) { + TreeMap metaMap = new TreeMap(); + metaMap.put(WAL_VERSION_KEY, WAL_VERSION); + if (compress) { + // Currently we only do one compression type. + metaMap.put(WAL_COMPRESSION_TYPE_KEY, DICTIONARY_COMPRESSION_TYPE); + } + return new Metadata(metaMap); + } + + /** + * Call this method after init() has been executed + * + * @return whether WAL compression is enabled + */ + static boolean isWALCompressionEnabled(final Metadata metadata) { + // Check version is >= VERSION? + Text txt = metadata.get(WAL_VERSION_KEY); + if (txt == null || Integer.parseInt(txt.toString()) < COMPRESSION_VERSION) { + return false; + } + // Now check that compression type is present. Currently only one value. + txt = metadata.get(WAL_COMPRESSION_TYPE_KEY); + return txt != null && txt.equals(DICTIONARY_COMPRESSION_TYPE); + } + + @Override + public void init(FileSystem fs, Path path, Configuration conf) + throws IOException { + // Should we do our custom WAL compression? + boolean compress = conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, false); + if (compress) { + try { + if (this.compressionContext == null) { + this.compressionContext = new CompressionContext(LRUDictionary.class); + } else { + this.compressionContext.clear(); + } + } catch (Exception e) { + throw new IOException("Failed to initiate CompressionContext", e); + } + } + + if (null == keyClass) { + keyClass = HLog.getKeyClass(conf); + } + + // Create a SF.Writer instance. + try { + // reflection for a version of SequenceFile.createWriter that doesn't + // automatically create the parent directory (see HBASE-2312) + this.writer = (SequenceFile.Writer) SequenceFile.class + .getMethod("createWriter", new Class[] {FileSystem.class, + Configuration.class, Path.class, Class.class, Class.class, + Integer.TYPE, Short.TYPE, Long.TYPE, Boolean.TYPE, + CompressionType.class, CompressionCodec.class, Metadata.class}) + .invoke(null, new Object[] {fs, conf, path, HLog.getKeyClass(conf), + WALEdit.class, + Integer.valueOf(fs.getConf().getInt("io.file.buffer.size", 4096)), + Short.valueOf((short) + conf.getInt("hbase.regionserver.hlog.replication", + FSUtils.getDefaultReplication(fs, path))), + Long.valueOf(conf.getLong("hbase.regionserver.hlog.blocksize", + FSUtils.getDefaultBlockSize(fs, path))), + Boolean.valueOf(false) /*createParent*/, + SequenceFile.CompressionType.NONE, new DefaultCodec(), + createMetadata(conf, compress) + }); + } catch (InvocationTargetException ite) { + // function was properly called, but threw it's own exception + throw new IOException(ite.getCause()); + } catch (Exception e) { + // ignore all other exceptions. related to reflection failure + } + + // if reflection failed, use the old createWriter + if (this.writer == null) { + LOG.debug("new createWriter -- HADOOP-6840 -- not available"); + this.writer = SequenceFile.createWriter(fs, conf, path, + HLog.getKeyClass(conf), WALEdit.class, + fs.getConf().getInt("io.file.buffer.size", 4096), + (short) conf.getInt("hbase.regionserver.hlog.replication", + FSUtils.getDefaultReplication(fs, path)), + conf.getLong("hbase.regionserver.hlog.blocksize", + FSUtils.getDefaultBlockSize(fs, path)), + SequenceFile.CompressionType.NONE, + new DefaultCodec(), + null, + createMetadata(conf, compress)); + } else { + LOG.debug("using new createWriter -- HADOOP-6840"); + } + + this.writer_out = getSequenceFilePrivateFSDataOutputStreamAccessible(); + this.syncFs = getSyncFs(); + this.hflush = getHFlush(); + String msg = "Path=" + path + + ", syncFs=" + (this.syncFs != null) + + ", hflush=" + (this.hflush != null) + + ", compression=" + compress; + if (this.syncFs != null || this.hflush != null) { + LOG.debug(msg); + } else { + LOG.warn("No sync support! " + msg); + } + } + + /** + * Now do dirty work to see if syncFs is available on the backing this.writer. + * It will be available in branch-0.20-append and in CDH3. + * @return The syncFs method or null if not available. + * @throws IOException + */ + private Method getSyncFs() + throws IOException { + Method m = null; + try { + // function pointer to writer.syncFs() method; present when sync is hdfs-200. + m = this.writer.getClass().getMethod("syncFs", new Class []{}); + } catch (SecurityException e) { + throw new IOException("Failed test for syncfs", e); + } catch (NoSuchMethodException e) { + // Not available + } + return m; + } + + /** + * See if hflush (0.21 and 0.22 hadoop) is available. + * @return The hflush method or null if not available. + * @throws IOException + */ + private Method getHFlush() + throws IOException { + Method m = null; + try { + Class c = getWriterFSDataOutputStream().getClass(); + m = c.getMethod("hflush", new Class []{}); + } catch (SecurityException e) { + throw new IOException("Failed test for hflush", e); + } catch (NoSuchMethodException e) { + // Ignore + } + return m; + } + + // Get at the private FSDataOutputStream inside in SequenceFile so we can + // call sync on it. Make it accessible. + private FSDataOutputStream getSequenceFilePrivateFSDataOutputStreamAccessible() + throws IOException { + FSDataOutputStream out = null; + final Field fields [] = this.writer.getClass().getDeclaredFields(); + final String fieldName = "out"; + for (int i = 0; i < fields.length; ++i) { + if (fieldName.equals(fields[i].getName())) { + try { + // Make the 'out' field up in SF.Writer accessible. + fields[i].setAccessible(true); + out = (FSDataOutputStream)fields[i].get(this.writer); + break; + } catch (IllegalAccessException ex) { + throw new IOException("Accessing " + fieldName, ex); + } catch (SecurityException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + return out; + } + + @Override + public void append(HLog.Entry entry) throws IOException { + entry.setCompressionContext(compressionContext); + try { + this.writer.append(entry.getKey(), entry.getEdit()); + } catch (NullPointerException npe) { + // Concurrent close... + throw new IOException(npe); + } + } + + @Override + public void close() throws IOException { + if (this.writer != null) { + try { + this.writer.close(); + } catch (NullPointerException npe) { + // Can get a NPE coming up from down in DFSClient$DFSOutputStream#close + LOG.warn(npe); + } + this.writer = null; + } + } + + @Override + public void sync() throws IOException { + if (this.syncFs != null) { + try { + this.syncFs.invoke(this.writer, HLog.NO_ARGS); + } catch (Exception e) { + throw new IOException("Reflection", e); + } + } else if (this.hflush != null) { + try { + this.hflush.invoke(getWriterFSDataOutputStream(), HLog.NO_ARGS); + } catch (Exception e) { + throw new IOException("Reflection", e); + } + } + } + + @Override + public long getLength() throws IOException { + try { + return this.writer.getLength(); + } catch (NullPointerException npe) { + // Concurrent close... + throw new IOException(npe); + } + } + + /** + * @return The dfsclient out stream up inside SF.Writer made accessible, or + * null if not available. + */ + public FSDataOutputStream getWriterFSDataOutputStream() { + return this.writer_out; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java new file mode 100644 index 0000000..3cdf1bc --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java @@ -0,0 +1,91 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; + +/** + * Get notification of {@link HLog}/WAL log events. The invocations are inline + * so make sure your implementation is fast else you'll slow hbase. + */ +public interface WALActionsListener { + + /** + * The WAL is going to be rolled. The oldPath can be null if this is + * the first log file from the regionserver. + * @param oldPath the path to the old hlog + * @param newPath the path to the new hlog + */ + public void preLogRoll(Path oldPath, Path newPath) throws IOException; + + /** + * The WAL has been rolled. The oldPath can be null if this is + * the first log file from the regionserver. + * @param oldPath the path to the old hlog + * @param newPath the path to the new hlog + */ + public void postLogRoll(Path oldPath, Path newPath) throws IOException; + + /** + * The WAL is going to be archived. + * @param oldPath the path to the old hlog + * @param newPath the path to the new hlog + */ + public void preLogArchive(Path oldPath, Path newPath) throws IOException; + + /** + * The WAL has been archived. + * @param oldPath the path to the old hlog + * @param newPath the path to the new hlog + */ + public void postLogArchive(Path oldPath, Path newPath) throws IOException; + + /** + * A request was made that the WAL be rolled. + */ + public void logRollRequested(); + + /** + * The WAL is about to close. + */ + public void logCloseRequested(); + + /** + * Called before each write. + * @param info + * @param logKey + * @param logEdit + */ + public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, + WALEdit logEdit); + + /** + * + * @param htd + * @param logKey + * @param logEdit + */ + public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, + WALEdit logEdit); + +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java new file mode 100644 index 0000000..b14e190 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java @@ -0,0 +1,134 @@ + +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; + +import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.coprocessor.*; +import org.apache.hadoop.conf.Configuration; + +/** + * Implements the coprocessor environment and runtime support for coprocessors + * loaded within a {@link HLog}. + */ +public class WALCoprocessorHost + extends CoprocessorHost { + + /** + * Encapsulation of the environment of each coprocessor + */ + static class WALEnvironment extends CoprocessorHost.Environment + implements WALCoprocessorEnvironment { + + private HLog wal; + + @Override + public HLog getWAL() { + return wal; + } + + /** + * Constructor + * @param impl the coprocessor instance + * @param priority chaining priority + * @param seq load sequence + * @param conf configuration + * @param hlog HLog + */ + public WALEnvironment(Class implClass, final Coprocessor impl, + final int priority, final int seq, final Configuration conf, + final HLog hlog) { + super(impl, priority, seq, conf); + this.wal = hlog; + } + } + + HLog wal; + /** + * Constructor + * @param log the write ahead log + * @param conf the configuration + */ + public WALCoprocessorHost(final HLog log, final Configuration conf) { + this.wal = log; + // load system default cp's from configuration. + loadSystemCoprocessors(conf, WAL_COPROCESSOR_CONF_KEY); + } + + @Override + public WALEnvironment createEnvironment(final Class implClass, + final Coprocessor instance, final int priority, final int seq, + final Configuration conf) { + return new WALEnvironment(implClass, instance, priority, seq, conf, + this.wal); + } + + /** + * @param info + * @param logKey + * @param logEdit + * @return true if default behavior should be bypassed, false otherwise + * @throws IOException + */ + public boolean preWALWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit) + throws IOException { + boolean bypass = false; + ObserverContext ctx = null; + for (WALEnvironment env: coprocessors) { + if (env.getInstance() instanceof + org.apache.hadoop.hbase.coprocessor.WALObserver) { + ctx = ObserverContext.createAndPrepare(env, ctx); + ((org.apache.hadoop.hbase.coprocessor.WALObserver)env.getInstance()). + preWALWrite(ctx, info, logKey, logEdit); + bypass |= ctx.shouldBypass(); + if (ctx.shouldComplete()) { + break; + } + } + } + return bypass; + } + + /** + * @param info + * @param logKey + * @param logEdit + * @throws IOException + */ + public void postWALWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit) + throws IOException { + ObserverContext ctx = null; + for (WALEnvironment env: coprocessors) { + if (env.getInstance() instanceof + org.apache.hadoop.hbase.coprocessor.WALObserver) { + ctx = ObserverContext.createAndPrepare(env, ctx); + ((org.apache.hadoop.hbase.coprocessor.WALObserver)env.getInstance()). + postWALWrite(ctx, info, logKey, logEdit); + if (ctx.shouldComplete()) { + break; + } + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java new file mode 100644 index 0000000..efd5a32 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java @@ -0,0 +1,202 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableMap; +import java.util.TreeMap; + +import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; +import org.apache.hadoop.io.Writable; + +/** + * WALEdit: Used in HBase's transaction log (WAL) to represent + * the collection of edits (KeyValue objects) corresponding to a + * single transaction. The class implements "Writable" interface + * for serializing/deserializing a set of KeyValue items. + * + * Previously, if a transaction contains 3 edits to c1, c2, c3 for a row R, + * the HLog would have three log entries as follows: + * + * : + * : + * : + * + * This presents problems because row level atomicity of transactions + * was not guaranteed. If we crash after few of the above appends make + * it, then recovery will restore a partial transaction. + * + * In the new world, all the edits for a given transaction are written + * out as a single record, for example: + * + * : + * + * where, the WALEdit is serialized as: + * <-1, # of edits, , , ... > + * For example: + * <-1, 3, , , > + * + * The -1 marker is just a special way of being backward compatible with + * an old HLog which would have contained a single . + * + * The deserializer for WALEdit backward compatibly detects if the record + * is an old style KeyValue or the new style WALEdit. + * + */ +public class WALEdit implements Writable, HeapSize { + + private final int VERSION_2 = -1; + + private final ArrayList kvs = new ArrayList(); + private NavigableMap scopes; + + private CompressionContext compressionContext; + + public WALEdit() { + } + + public void setCompressionContext(final CompressionContext compressionContext) { + this.compressionContext = compressionContext; + } + + public void add(KeyValue kv) { + this.kvs.add(kv); + } + + public boolean isEmpty() { + return kvs.isEmpty(); + } + + public int size() { + return kvs.size(); + } + + public List getKeyValues() { + return kvs; + } + + public NavigableMap getScopes() { + return scopes; + } + + + public void setScopes (NavigableMap scopes) { + // We currently process the map outside of WALEdit, + // TODO revisit when replication is part of core + this.scopes = scopes; + } + + public void readFields(DataInput in) throws IOException { + kvs.clear(); + if (scopes != null) { + scopes.clear(); + } + int versionOrLength = in.readInt(); + if (versionOrLength == VERSION_2) { + // this is new style HLog entry containing multiple KeyValues. + int numEdits = in.readInt(); + for (int idx = 0; idx < numEdits; idx++) { + if (compressionContext != null) { + this.add(KeyValueCompression.readKV(in, compressionContext)); + } else { + KeyValue kv = new KeyValue(); + kv.readFields(in); + this.add(kv); + } + } + int numFamilies = in.readInt(); + if (numFamilies > 0) { + if (scopes == null) { + scopes = new TreeMap(Bytes.BYTES_COMPARATOR); + } + for (int i = 0; i < numFamilies; i++) { + byte[] fam = Bytes.readByteArray(in); + int scope = in.readInt(); + scopes.put(fam, scope); + } + } + } else { + // this is an old style HLog entry. The int that we just + // read is actually the length of a single KeyValue + KeyValue kv = new KeyValue(); + kv.readFields(versionOrLength, in); + this.add(kv); + } + + } + + public void write(DataOutput out) throws IOException { + out.writeInt(VERSION_2); + out.writeInt(kvs.size()); + // We interleave the two lists for code simplicity + for (KeyValue kv : kvs) { + if (compressionContext != null) { + KeyValueCompression.writeKV(out, kv, compressionContext); + } else{ + kv.write(out); + } + } + if (scopes == null) { + out.writeInt(0); + } else { + out.writeInt(scopes.size()); + for (byte[] key : scopes.keySet()) { + Bytes.writeByteArray(out, key); + out.writeInt(scopes.get(key)); + } + } + } + + public long heapSize() { + long ret = 0; + for (KeyValue kv : kvs) { + ret += kv.heapSize(); + } + if (scopes != null) { + ret += ClassSize.TREEMAP; + ret += ClassSize.align(scopes.size() * ClassSize.MAP_ENTRY); + // TODO this isn't quite right, need help here + } + return ret; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + + sb.append("[#edits: " + kvs.size() + " = <"); + for (KeyValue kv : kvs) { + sb.append(kv.toString()); + sb.append("; "); + } + if (scopes != null) { + sb.append(" scopes: " + scopes.toString()); + } + sb.append(">]"); + return sb.toString(); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java new file mode 100644 index 0000000..d977830 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java @@ -0,0 +1,199 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.replication.ReplicationZookeeper.PeerState; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +/** + * This class acts as a wrapper for all the objects used to identify and + * communicate with remote peers and is responsible for answering to expired + * sessions and re-establishing the ZK connections. + */ +public class ReplicationPeer implements Abortable { + private static final Log LOG = LogFactory.getLog(ReplicationPeer.class); + + private final String clusterKey; + private final String id; + private List regionServers = new ArrayList(0); + private final AtomicBoolean peerEnabled = new AtomicBoolean(); + // Cannot be final since a new object needs to be recreated when session fails + private ZooKeeperWatcher zkw; + private final Configuration conf; + + private PeerStateTracker peerStateTracker; + + /** + * Constructor that takes all the objects required to communicate with the + * specified peer, except for the region server addresses. + * @param conf configuration object to this peer + * @param key cluster key used to locate the peer + * @param id string representation of this peer's identifier + */ + public ReplicationPeer(Configuration conf, String key, + String id) throws IOException { + this.conf = conf; + this.clusterKey = key; + this.id = id; + this.reloadZkWatcher(); + } + + /** + * start a state tracker to check whether this peer is enabled or not + * + * @param zookeeper zk watcher for the local cluster + * @param peerStateNode path to zk node which stores peer state + * @throws KeeperException + */ + public void startStateTracker(ZooKeeperWatcher zookeeper, String peerStateNode) + throws KeeperException { + if (ZKUtil.checkExists(zookeeper, peerStateNode) == -1) { + // There is a race b/w PeerWatcher and ReplicationZookeeper#add method to create the + // peer-state znode. This happens while adding a peer. + // The peer state data is set as "ENABLED" by default. + ZKUtil.createNodeIfNotExistsAndWatch(zookeeper, peerStateNode, + Bytes.toBytes(PeerState.ENABLED.name())); + } + this.peerStateTracker = new PeerStateTracker(peerStateNode, zookeeper, + this); + this.peerStateTracker.start(); + this.readPeerStateZnode(); + } + + private void readPeerStateZnode() { + String currentState = Bytes.toString(peerStateTracker.getData(false)); + this.peerEnabled.set(PeerState.ENABLED.equals(PeerState + .valueOf(currentState))); + } + + /** + * Get the cluster key of that peer + * @return string consisting of zk ensemble addresses, client port + * and root znode + */ + public String getClusterKey() { + return clusterKey; + } + + /** + * Get the state of this peer + * @return atomic boolean that holds the status + */ + public AtomicBoolean getPeerEnabled() { + return peerEnabled; + } + + /** + * Get a list of all the addresses of all the region servers + * for this peer cluster + * @return list of addresses + */ + public List getRegionServers() { + return regionServers; + } + + /** + * Set the list of region servers for that peer + * @param regionServers list of addresses for the region servers + */ + public void setRegionServers(List regionServers) { + this.regionServers = regionServers; + } + + /** + * Get the ZK connection to this peer + * @return zk connection + */ + public ZooKeeperWatcher getZkw() { + return zkw; + } + + /** + * Get the identifier of this peer + * @return string representation of the id (short) + */ + public String getId() { + return id; + } + + /** + * Get the configuration object required to communicate with this peer + * @return configuration object + */ + public Configuration getConfiguration() { + return conf; + } + + @Override + public void abort(String why, Throwable e) { + LOG.fatal("The ReplicationPeer coresponding to peer " + clusterKey + + " was aborted for the following reason(s):" + why, e); + } + + /** + * Closes the current ZKW (if not null) and creates a new one + * @throws IOException If anything goes wrong connecting + */ + public void reloadZkWatcher() throws IOException { + if (zkw != null) zkw.close(); + zkw = new ZooKeeperWatcher(conf, + "connection to cluster: " + id, this); + } + + @Override + public boolean isAborted() { + // Currently the replication peer is never "Aborted", we just log when the + // abort method is called. + return false; + } + + /** + * Tracker for state of this peer + */ + public class PeerStateTracker extends ZooKeeperNodeTracker { + + public PeerStateTracker(String peerStateZNode, ZooKeeperWatcher watcher, + Abortable abortable) { + super(watcher, peerStateZNode, abortable); + } + + @Override + public synchronized void nodeDataChanged(String path) { + if (path.equals(node)) { + super.nodeDataChanged(path); + readPeerStateZnode(); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java new file mode 100644 index 0000000..a7aa59e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java @@ -0,0 +1,952 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.replication.regionserver.Replication; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ClusterId; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.ConnectionLossException; +import org.apache.zookeeper.KeeperException.SessionExpiredException; + +/** + * This class serves as a helper for all things related to zookeeper in + * replication. + *

+ * The layout looks something like this under zookeeper.znode.parent for the + * master cluster: + *

+ * + * 

+ * replication/
+ *  state      {contains true or false}
+ *  clusterId  {contains a byte}
+ *  peers/
+ *    1/   {contains a full cluster address}
+ *      peer-state  {contains ENABLED or DISABLED}
+ *    2/
+ *    ...
+ *  rs/ {lists all RS that replicate}
+ *    startcode1/ {lists all peer clusters}
+ *      1/ {lists hlogs to process}
+ *        10.10.1.76%3A53488.123456789 {contains nothing or a position}
+ *        10.10.1.76%3A53488.123456790
+ *        ...
+ *      2/
+ *      ...
+ *    startcode2/
+ *    ...
+ *
+ */ +public class ReplicationZookeeper { + private static final Log LOG = + LogFactory.getLog(ReplicationZookeeper.class); + // Name of znode we use to lock when failover + private final static String RS_LOCK_ZNODE = "lock"; + + // Values of znode which stores state of a peer + public static enum PeerState { + ENABLED, DISABLED + }; + + // Our handle on zookeeper + private final ZooKeeperWatcher zookeeper; + // Map of peer clusters keyed by their id + private Map peerClusters; + // Path to the root replication znode + private String replicationZNode; + // Path to the peer clusters znode + private String peersZNode; + // Path to the znode that contains all RS that replicates + private String rsZNode; + // Path to this region server's name under rsZNode + private String rsServerNameZnode; + // Name node if the replicationState znode + private String replicationStateNodeName; + // Name of zk node which stores peer state + private String peerStateNodeName; + private final Configuration conf; + // Is this cluster replicating at the moment? + private AtomicBoolean replicating; + // The key to our own cluster + private String ourClusterKey; + // Abortable + private Abortable abortable; + private ReplicationStatusTracker statusTracker; + + /** + * Constructor used by clients of replication (like master and HBase clients) + * @param conf conf to use + * @param zk zk connection to use + * @throws IOException + */ + public ReplicationZookeeper(final Abortable abortable, final Configuration conf, + final ZooKeeperWatcher zk) + throws KeeperException { + + this.conf = conf; + this.zookeeper = zk; + this.replicating = new AtomicBoolean(); + setZNodes(abortable); + } + + /** + * Constructor used by region servers, connects to the peer cluster right away. + * + * @param server + * @param replicating atomic boolean to start/stop replication + * @throws IOException + * @throws KeeperException + */ + public ReplicationZookeeper(final Server server, final AtomicBoolean replicating) + throws IOException, KeeperException { + this.abortable = server; + this.zookeeper = server.getZooKeeper(); + this.conf = server.getConfiguration(); + this.replicating = replicating; + setZNodes(server); + + this.peerClusters = new HashMap(); + ZKUtil.createWithParents(this.zookeeper, + ZKUtil.joinZNode(this.replicationZNode, this.replicationStateNodeName)); + this.rsServerNameZnode = ZKUtil.joinZNode(rsZNode, server.getServerName().toString()); + ZKUtil.createWithParents(this.zookeeper, this.rsServerNameZnode); + connectExistingPeers(); + } + + private void setZNodes(Abortable abortable) throws KeeperException { + String replicationZNodeName = + conf.get("zookeeper.znode.replication", "replication"); + String peersZNodeName = + conf.get("zookeeper.znode.replication.peers", "peers"); + this.peerStateNodeName = conf.get( + "zookeeper.znode.replication.peers.state", "peer-state"); + this.replicationStateNodeName = + conf.get("zookeeper.znode.replication.state", "state"); + String rsZNodeName = + conf.get("zookeeper.znode.replication.rs", "rs"); + this.ourClusterKey = ZKUtil.getZooKeeperClusterKey(this.conf); + this.replicationZNode = + ZKUtil.joinZNode(this.zookeeper.baseZNode, replicationZNodeName); + this.peersZNode = ZKUtil.joinZNode(replicationZNode, peersZNodeName); + ZKUtil.createWithParents(this.zookeeper, this.peersZNode); + this.rsZNode = ZKUtil.joinZNode(replicationZNode, rsZNodeName); + ZKUtil.createWithParents(this.zookeeper, this.rsZNode); + + // Set a tracker on replicationStateNodeNode + this.statusTracker = + new ReplicationStatusTracker(this.zookeeper, abortable); + statusTracker.start(); + readReplicationStateZnode(); + } + + private void connectExistingPeers() throws IOException, KeeperException { + List znodes = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode); + if (znodes != null) { + for (String z : znodes) { + connectToPeer(z); + } + } + } + + /** + * List this cluster's peers' IDs + * @return list of all peers' identifiers + */ + public List listPeersIdsAndWatch() { + List ids = null; + try { + ids = ZKUtil.listChildrenAndWatchThem(this.zookeeper, this.peersZNode); + } catch (KeeperException e) { + this.abortable.abort("Cannot get the list of peers ", e); + } + return ids; + } + + /** + * Map of this cluster's peers for display. + * @return A map of peer ids to peer cluster keys + */ + public Map listPeers() { + Map peers = new TreeMap(); + List ids = null; + try { + ids = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode); + for (String id : ids) { + peers.put(id, Bytes.toString(ZKUtil.getData(this.zookeeper, + ZKUtil.joinZNode(this.peersZNode, id)))); + } + } catch (KeeperException e) { + this.abortable.abort("Cannot get the list of peers ", e); + } + return peers; + } + /** + * Returns all region servers from given peer + * + * @param peerClusterId (byte) the cluster to interrogate + * @return addresses of all region servers + */ + public List getSlavesAddresses(String peerClusterId) { + if (this.peerClusters.size() == 0) { + return Collections.emptyList(); + } + ReplicationPeer peer = this.peerClusters.get(peerClusterId); + if (peer == null) { + return Collections.emptyList(); + } + + List addresses; + try { + addresses = fetchSlavesAddresses(peer.getZkw()); + } catch (KeeperException ke) { + reconnectPeer(ke, peer); + addresses = Collections.emptyList(); + } + peer.setRegionServers(addresses); + return peer.getRegionServers(); + } + + /** + * Get the list of all the region servers from the specified peer + * @param zkw zk connection to use + * @return list of region server addresses or an empty list if the slave + * is unavailable + */ + private List fetchSlavesAddresses(ZooKeeperWatcher zkw) + throws KeeperException { + return listChildrenAndGetAsServerNames(zkw, zkw.rsZNode); + } + + /** + * Lists the children of the specified znode, retrieving the data of each + * child as a server address. + * + * Used to list the currently online regionservers and their addresses. + * + * Sets no watches at all, this method is best effort. + * + * Returns an empty list if the node has no children. Returns null if the + * parent node itself does not exist. + * + * @param zkw zookeeper reference + * @param znode node to get children of as addresses + * @return list of data of children of specified znode, empty if no children, + * null if parent does not exist + * @throws KeeperException if unexpected zookeeper exception + */ + public static List listChildrenAndGetAsServerNames( + ZooKeeperWatcher zkw, String znode) + throws KeeperException { + List children = ZKUtil.listChildrenNoWatch(zkw, znode); + if(children == null) { + return Collections.emptyList(); + } + List addresses = new ArrayList(children.size()); + for (String child : children) { + addresses.add(ServerName.parseServerName(child)); + } + return addresses; + } + + /** + * This method connects this cluster to another one and registers it + * in this region server's replication znode + * @param peerId id of the peer cluster + * @throws KeeperException + */ + public boolean connectToPeer(String peerId) + throws IOException, KeeperException { + if (peerClusters == null) { + return false; + } + if (this.peerClusters.containsKey(peerId)) { + return false; + } + ReplicationPeer peer = getPeer(peerId); + if (peer == null) { + return false; + } + this.peerClusters.put(peerId, peer); + ZKUtil.createWithParents(this.zookeeper, ZKUtil.joinZNode( + this.rsServerNameZnode, peerId)); + LOG.info("Added new peer cluster " + peer.getClusterKey()); + return true; + } + + /** + * Helper method to connect to a peer + * @param peerId peer's identifier + * @return object representing the peer + * @throws IOException + * @throws KeeperException + */ + public ReplicationPeer getPeer(String peerId) throws IOException, KeeperException{ + String znode = ZKUtil.joinZNode(this.peersZNode, peerId); + byte [] data = ZKUtil.getData(this.zookeeper, znode); + String otherClusterKey = Bytes.toString(data); + if (this.ourClusterKey.equals(otherClusterKey)) { + LOG.debug("Not connecting to " + peerId + " because it's us"); + return null; + } + // Construct the connection to the new peer + Configuration otherConf = new Configuration(this.conf); + try { + ZKUtil.applyClusterKeyToConf(otherConf, otherClusterKey); + } catch (IOException e) { + LOG.error("Can't get peer because:", e); + return null; + } + + ReplicationPeer peer = new ReplicationPeer(otherConf, peerId, + otherClusterKey); + peer.startStateTracker(this.zookeeper, this.getPeerStateNode(peerId)); + return peer; + } + + /** + * Set the new replication state for this cluster + * @param newState + */ + public void setReplicating(boolean newState) throws KeeperException { + ZKUtil.createWithParents(this.zookeeper, + ZKUtil.joinZNode(this.replicationZNode, this.replicationStateNodeName)); + ZKUtil.setData(this.zookeeper, + ZKUtil.joinZNode(this.replicationZNode, this.replicationStateNodeName), + Bytes.toBytes(Boolean.toString(newState))); + } + + /** + * Remove the peer from zookeeper. which will trigger the watchers on every + * region server and close their sources + * @param id + * @throws IllegalArgumentException Thrown when the peer doesn't exist + */ + public void removePeer(String id) throws IOException { + try { + if (!peerExists(id)) { + throw new IllegalArgumentException("Cannot remove inexisting peer"); + } + ZKUtil.deleteNodeRecursively(this.zookeeper, + ZKUtil.joinZNode(this.peersZNode, id)); + } catch (KeeperException e) { + throw new IOException("Unable to remove a peer", e); + } + } + + /** + * Add a new peer to this cluster + * @param id peer's identifier + * @param clusterKey ZK ensemble's addresses, client port and root znode + * @throws IllegalArgumentException Thrown when the peer doesn't exist + * @throws IllegalStateException Thrown when a peer already exists, since + * multi-slave isn't supported yet. + */ + public void addPeer(String id, String clusterKey) throws IOException { + try { + if (peerExists(id)) { + throw new IllegalArgumentException("Cannot add existing peer"); + } + ZKUtil.createWithParents(this.zookeeper, this.peersZNode); + ZKUtil.createAndWatch(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id), + Bytes.toBytes(clusterKey)); + // There is a race b/w PeerWatcher and ReplicationZookeeper#add method to create the + // peer-state znode. This happens while adding a peer. + // The peer state data is set as "ENABLED" by default. + ZKUtil.createNodeIfNotExistsAndWatch(this.zookeeper, getPeerStateNode(id), + Bytes.toBytes(PeerState.ENABLED.name())); + } catch (KeeperException e) { + throw new IOException("Unable to add peer", e); + } + } + + private boolean peerExists(String id) throws KeeperException { + return ZKUtil.checkExists(this.zookeeper, + ZKUtil.joinZNode(this.peersZNode, id)) >= 0; + } + + /** + * Enable replication to the peer + * + * @param id peer's identifier + * @throws IllegalArgumentException + * Thrown when the peer doesn't exist + */ + public void enablePeer(String id) throws IOException { + changePeerState(id, PeerState.ENABLED); + LOG.info("peer " + id + " is enabled"); + } + + /** + * Disable replication to the peer + * + * @param id peer's identifier + * @throws IllegalArgumentException + * Thrown when the peer doesn't exist + */ + public void disablePeer(String id) throws IOException { + changePeerState(id, PeerState.DISABLED); + LOG.info("peer " + id + " is disabled"); + } + + private void changePeerState(String id, PeerState state) throws IOException { + try { + if (!peerExists(id)) { + throw new IllegalArgumentException("peer " + id + " is not registered"); + } + String peerStateZNode = getPeerStateNode(id); + if (ZKUtil.checkExists(this.zookeeper, peerStateZNode) != -1) { + ZKUtil.setData(this.zookeeper, peerStateZNode, + Bytes.toBytes(state.name())); + } else { + ZKUtil.createAndWatch(zookeeper, peerStateZNode, + Bytes.toBytes(state.name())); + } + LOG.info("state of the peer " + id + " changed to " + state.name()); + } catch (KeeperException e) { + throw new IOException("Unable to change state of the peer " + id, e); + } + } + + /** + * Get state of the peer. This method checks the state by connecting to ZK. + * + * @param id peer's identifier + * @return current state of the peer + */ + public PeerState getPeerState(String id) throws KeeperException { + byte[] peerStateBytes = ZKUtil + .getData(this.zookeeper, getPeerStateNode(id)); + return PeerState.valueOf(Bytes.toString(peerStateBytes)); + } + + /** + * Check whether the peer is enabled or not. This method checks the atomic + * boolean of ReplicationPeer locally. + * + * @param id peer identifier + * @return true if the peer is enabled, otherwise false + * @throws IllegalArgumentException + * Thrown when the peer doesn't exist + */ + public boolean getPeerEnabled(String id) { + if (!this.peerClusters.containsKey(id)) { + throw new IllegalArgumentException("peer " + id + " is not registered"); + } + return this.peerClusters.get(id).getPeerEnabled().get(); + } + + private String getPeerStateNode(String id) { + return ZKUtil.joinZNode(this.peersZNode, + ZKUtil.joinZNode(id, this.peerStateNodeName)); + } + + /** + * This reads the state znode for replication and sets the atomic boolean + */ + private void readReplicationStateZnode() { + try { + this.replicating.set(getReplication()); + LOG.info("Replication is now " + (this.replicating.get()? + "started" : "stopped")); + } catch (KeeperException e) { + this.abortable.abort("Failed getting data on from " + getRepStateNode(), e); + } + } + + /** + * Get the replication status of this cluster. If the state znode doesn't + * exist it will also create it and set it true. + * @return returns true when it's enabled, else false + * @throws KeeperException + */ + public boolean getReplication() throws KeeperException { + byte [] data = this.statusTracker.getData(false); + if (data == null || data.length == 0) { + setReplicating(true); + return true; + } + return Boolean.parseBoolean(Bytes.toString(data)); + } + + private String getRepStateNode() { + return ZKUtil.joinZNode(this.replicationZNode, this.replicationStateNodeName); + } + + /** + * Add a new log to the list of hlogs in zookeeper + * @param filename name of the hlog's znode + * @param peerId name of the cluster's znode + */ + public void addLogToList(String filename, String peerId) + throws KeeperException { + String znode = ZKUtil.joinZNode(this.rsServerNameZnode, peerId); + znode = ZKUtil.joinZNode(znode, filename); + ZKUtil.createWithParents(this.zookeeper, znode); + } + + /** + * Remove a log from the list of hlogs in zookeeper + * @param filename name of the hlog's znode + * @param clusterId name of the cluster's znode + */ + public void removeLogFromList(String filename, String clusterId) { + try { + String znode = ZKUtil.joinZNode(rsServerNameZnode, clusterId); + znode = ZKUtil.joinZNode(znode, filename); + ZKUtil.deleteNode(this.zookeeper, znode); + } catch (KeeperException e) { + this.abortable.abort("Failed remove from list", e); + } + } + + /** + * Set the current position of the specified cluster in the current hlog + * @param filename filename name of the hlog's znode + * @param clusterId clusterId name of the cluster's znode + * @param position the position in the file + * @throws IOException + */ + public void writeReplicationStatus(String filename, String clusterId, + long position) { + try { + String znode = ZKUtil.joinZNode(this.rsServerNameZnode, clusterId); + znode = ZKUtil.joinZNode(znode, filename); + // Why serialize String of Long and note Long as bytes? + ZKUtil.setData(this.zookeeper, znode, + Bytes.toBytes(Long.toString(position))); + } catch (KeeperException e) { + this.abortable.abort("Writing replication status", e); + } + } + + /** + * Get a list of all the other region servers in this cluster + * and set a watch + * @return a list of server nanes + */ + public List getRegisteredRegionServers() { + List result = null; + try { + result = ZKUtil.listChildrenAndWatchThem( + this.zookeeper, this.zookeeper.rsZNode); + } catch (KeeperException e) { + this.abortable.abort("Get list of registered region servers", e); + } + return result; + } + + /** + * Get the list of the replicators that have queues, they can be alive, dead + * or simply from a previous run + * @return a list of server names + */ + public List getListOfReplicators() { + List result = null; + try { + result = ZKUtil.listChildrenNoWatch(this.zookeeper, rsZNode); + } catch (KeeperException e) { + this.abortable.abort("Get list of replicators", e); + } + return result; + } + + /** + * Get the list of peer clusters for the specified server names + * @param rs server names of the rs + * @return a list of peer cluster + */ + public List getListPeersForRS(String rs) { + String znode = ZKUtil.joinZNode(rsZNode, rs); + List result = null; + try { + result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode); + } catch (KeeperException e) { + this.abortable.abort("Get list of peers for rs", e); + } + return result; + } + + /** + * Get the list of hlogs for the specified region server and peer cluster + * @param rs server names of the rs + * @param id peer cluster + * @return a list of hlogs + */ + public List getListHLogsForPeerForRS(String rs, String id) { + String znode = ZKUtil.joinZNode(rsZNode, rs); + znode = ZKUtil.joinZNode(znode, id); + List result = null; + try { + result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode); + } catch (KeeperException e) { + this.abortable.abort("Get list of hlogs for peer", e); + } + return result; + } + + /** + * Try to set a lock in another server's znode. + * @param znode the server names of the other server + * @return true if the lock was acquired, false in every other cases + */ + public boolean lockOtherRS(String znode) { + try { + String parent = ZKUtil.joinZNode(this.rsZNode, znode); + if (parent.equals(rsServerNameZnode)) { + LOG.warn("Won't lock because this is us, we're dead!"); + return false; + } + String p = ZKUtil.joinZNode(parent, RS_LOCK_ZNODE); + ZKUtil.createAndWatch(this.zookeeper, p, Bytes.toBytes(rsServerNameZnode)); + } catch (KeeperException e) { + // This exception will pop up if the znode under which we're trying to + // create the lock is already deleted by another region server, meaning + // that the transfer already occurred. + // NoNode => transfer is done and znodes are already deleted + // NodeExists => lock znode already created by another RS + if (e instanceof KeeperException.NoNodeException || + e instanceof KeeperException.NodeExistsException) { + LOG.info("Won't transfer the queue," + + " another RS took care of it because of: " + e.getMessage()); + } else { + LOG.info("Failed lock other rs", e); + } + return false; + } + return true; + } + + /** + * It "atomically" copies all the hlogs queues from another region server and returns them all + * sorted per peer cluster (appended with the dead server's znode). + * @param znode + * @return HLog queues sorted per peer cluster + */ + public SortedMap> copyQueuesFromRSUsingMulti(String znode) { + SortedMap> queues = new TreeMap>(); + String deadRSZnodePath = ZKUtil.joinZNode(rsZNode, znode);// hbase/replication/rs/deadrs + List peerIdsToProcess = null; + List listOfOps = new ArrayList(); + try { + peerIdsToProcess = ZKUtil.listChildrenNoWatch(this.zookeeper, deadRSZnodePath); + if (peerIdsToProcess == null) return queues; // node already processed + for (String peerId : peerIdsToProcess) { + String newPeerId = peerId + "-" + znode; + String newPeerZnode = ZKUtil.joinZNode(this.rsServerNameZnode, newPeerId); + // check the logs queue for the old peer cluster + String oldClusterZnode = ZKUtil.joinZNode(deadRSZnodePath, peerId); + List hlogs = ZKUtil.listChildrenNoWatch(this.zookeeper, oldClusterZnode); + if (hlogs == null || hlogs.size() == 0) { + listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode)); + continue; // empty log queue. + } + // create the new cluster znode + SortedSet logQueue = new TreeSet(); + queues.put(newPeerId, logQueue); + ZKUtilOp op = ZKUtilOp.createAndFailSilent(newPeerZnode, HConstants.EMPTY_BYTE_ARRAY); + listOfOps.add(op); + // get the offset of the logs and set it to new znodes + for (String hlog : hlogs) { + String oldHlogZnode = ZKUtil.joinZNode(oldClusterZnode, hlog); + byte[] logOffset = ZKUtil.getData(this.zookeeper, oldHlogZnode); + LOG.debug("Creating " + hlog + " with data " + Bytes.toString(logOffset)); + String newLogZnode = ZKUtil.joinZNode(newPeerZnode, hlog); + listOfOps.add(ZKUtilOp.createAndFailSilent(newLogZnode, logOffset)); + // add ops for deleting + listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldHlogZnode)); + logQueue.add(hlog); + } + // add delete op for peer + listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode)); + } + // add delete op for dead rs + listOfOps.add(ZKUtilOp.deleteNodeFailSilent(deadRSZnodePath)); + LOG.debug(" The multi list size is: " + listOfOps.size()); + ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false); + LOG.info("Atomically moved the dead regionserver logs. "); + } catch (KeeperException e) { + // Multi call failed; it looks like some other regionserver took away the logs. + LOG.warn("Got exception in copyQueuesFromRSUsingMulti: ", e); + queues.clear(); + } + return queues; + } + + /** + * This methods copies all the hlogs queues from another region server + * and returns them all sorted per peer cluster (appended with the dead + * server's znode) + * @param znode server names to copy + * @return all hlogs for all peers of that cluster, null if an error occurred + */ + public SortedMap> copyQueuesFromRS(String znode) { + // TODO this method isn't atomic enough, we could start copying and then + // TODO fail for some reason and we would end up with znodes we don't want. + SortedMap> queues = + new TreeMap>(); + try { + String nodePath = ZKUtil.joinZNode(rsZNode, znode); + List clusters = + ZKUtil.listChildrenNoWatch(this.zookeeper, nodePath); + // We have a lock znode in there, it will count as one. + if (clusters == null || clusters.size() <= 1) { + return queues; + } + // The lock isn't a peer cluster, remove it + clusters.remove(RS_LOCK_ZNODE); + for (String cluster : clusters) { + // We add the name of the recovered RS to the new znode, we can even + // do that for queues that were recovered 10 times giving a znode like + // number-startcode-number-otherstartcode-number-anotherstartcode-etc + String newCluster = cluster+"-"+znode; + String newClusterZnode = ZKUtil.joinZNode(rsServerNameZnode, newCluster); + String clusterPath = ZKUtil.joinZNode(nodePath, cluster); + List hlogs = ZKUtil.listChildrenNoWatch(this.zookeeper, clusterPath); + // That region server didn't have anything to replicate for this cluster + if (hlogs == null || hlogs.size() == 0) { + continue; + } + ZKUtil.createNodeIfNotExistsAndWatch(this.zookeeper, newClusterZnode, + HConstants.EMPTY_BYTE_ARRAY); + SortedSet logQueue = new TreeSet(); + queues.put(newCluster, logQueue); + for (String hlog : hlogs) { + String z = ZKUtil.joinZNode(clusterPath, hlog); + byte [] position = ZKUtil.getData(this.zookeeper, z); + LOG.debug("Creating " + hlog + " with data " + Bytes.toString(position)); + String child = ZKUtil.joinZNode(newClusterZnode, hlog); + ZKUtil.createAndWatch(this.zookeeper, child, position); + logQueue.add(hlog); + } + } + } catch (KeeperException e) { + this.abortable.abort("Copy queues from rs", e); + } + return queues; + } + + /** + * Delete a complete queue of hlogs + * @param peerZnode znode of the peer cluster queue of hlogs to delete + */ + public void deleteSource(String peerZnode, boolean closeConnection) { + try { + ZKUtil.deleteNodeRecursively(this.zookeeper, + ZKUtil.joinZNode(rsServerNameZnode, peerZnode)); + if (closeConnection) { + this.peerClusters.get(peerZnode).getZkw().close(); + this.peerClusters.remove(peerZnode); + } + } catch (KeeperException e) { + this.abortable.abort("Failed delete of " + peerZnode, e); + } + } + + /** + * Recursive deletion of all znodes in specified rs' znode + * @param znode + */ + public void deleteRsQueues(String znode) { + String fullpath = ZKUtil.joinZNode(rsZNode, znode); + try { + List clusters = + ZKUtil.listChildrenNoWatch(this.zookeeper, fullpath); + for (String cluster : clusters) { + // We'll delete it later + if (cluster.equals(RS_LOCK_ZNODE)) { + continue; + } + String fullClusterPath = ZKUtil.joinZNode(fullpath, cluster); + ZKUtil.deleteNodeRecursively(this.zookeeper, fullClusterPath); + } + // Finish cleaning up + ZKUtil.deleteNodeRecursively(this.zookeeper, fullpath); + } catch (KeeperException e) { + if (e instanceof KeeperException.NoNodeException || + e instanceof KeeperException.NotEmptyException) { + // Testing a special case where another region server was able to + // create a lock just after we deleted it, but then was also able to + // delete the RS znode before us or its lock znode is still there. + if (e.getPath().equals(fullpath)) { + return; + } + } + this.abortable.abort("Failed delete of " + znode, e); + } + } + + /** + * Delete this cluster's queues + */ + public void deleteOwnRSZNode() { + try { + ZKUtil.deleteNodeRecursively(this.zookeeper, + this.rsServerNameZnode); + } catch (KeeperException e) { + // if the znode is already expired, don't bother going further + if (e instanceof KeeperException.SessionExpiredException) { + return; + } + this.abortable.abort("Failed delete of " + this.rsServerNameZnode, e); + } + } + + /** + * Get the position of the specified hlog in the specified peer znode + * @param peerId znode of the peer cluster + * @param hlog name of the hlog + * @return the position in that hlog + * @throws KeeperException + */ + public long getHLogRepPosition(String peerId, String hlog) + throws KeeperException { + String clusterZnode = ZKUtil.joinZNode(rsServerNameZnode, peerId); + String znode = ZKUtil.joinZNode(clusterZnode, hlog); + String data = Bytes.toString(ZKUtil.getData(this.zookeeper, znode)); + return data == null || data.length() == 0 ? 0 : Long.parseLong(data); + } + + /** + * Returns the UUID of the provided peer id. Should a connection loss or session + * expiration happen, the ZK handler will be reopened once and if it still doesn't + * work then it will bail and return null. + * @param peerId the peer's ID that will be converted into a UUID + * @return a UUID or null if there's a ZK connection issue + */ + public UUID getPeerUUID(String peerId) { + ReplicationPeer peer = getPeerClusters().get(peerId); + UUID peerUUID = null; + try { + peerUUID = getUUIDForCluster(peer.getZkw()); + } catch (KeeperException ke) { + reconnectPeer(ke, peer); + } + return peerUUID; + } + + /** + * Get the UUID for the provided ZK watcher. Doesn't handle any ZK exceptions + * @param zkw watcher connected to an ensemble + * @return the UUID read from zookeeper + * @throws KeeperException + */ + public UUID getUUIDForCluster(ZooKeeperWatcher zkw) throws KeeperException { + return UUID.fromString(ClusterId.readClusterIdZNode(zkw)); + } + + private void reconnectPeer(KeeperException ke, ReplicationPeer peer) { + if (ke instanceof ConnectionLossException + || ke instanceof SessionExpiredException) { + LOG.warn( + "Lost the ZooKeeper connection for peer " + peer.getClusterKey(), + ke); + try { + peer.reloadZkWatcher(); + } catch(IOException io) { + LOG.warn( + "Creation of ZookeeperWatcher failed for peer " + + peer.getClusterKey(), io); + } + } + } + + public void registerRegionServerListener(ZooKeeperListener listener) { + this.zookeeper.registerListener(listener); + } + + /** + * Get a map of all peer clusters + * @return map of peer cluster keyed by id + */ + public Map getPeerClusters() { + return this.peerClusters; + } + + /** + * Extracts the znode name of a peer cluster from a ZK path + * @param fullPath Path to extract the id from + * @return the id or an empty string if path is invalid + */ + public static String getZNodeName(String fullPath) { + String[] parts = fullPath.split("/"); + return parts.length > 0 ? parts[parts.length-1] : ""; + } + + /** + * Get this cluster's zk connection + * @return zk connection + */ + public ZooKeeperWatcher getZookeeperWatcher() { + return this.zookeeper; + } + + + /** + * Get the full path to the peers' znode + * @return path to peers in zk + */ + public String getPeersZNode() { + return peersZNode; + } + + /** + * Tracker for status of the replication + */ + public class ReplicationStatusTracker extends ZooKeeperNodeTracker { + public ReplicationStatusTracker(ZooKeeperWatcher watcher, + Abortable abortable) { + super(watcher, getRepStateNode(), abortable); + } + + @Override + public synchronized void nodeDataChanged(String path) { + if (path.equals(node)) { + super.nodeDataChanged(path); + readReplicationStateZnode(); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java new file mode 100644 index 0000000..ee8941b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java @@ -0,0 +1,173 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication.master; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate; +import org.apache.hadoop.hbase.replication.ReplicationZookeeper; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Implementation of a log cleaner that checks if a log is still scheduled for + * replication before deleting it when its TTL is over. + */ +public class ReplicationLogCleaner extends BaseLogCleanerDelegate implements Abortable { + private static final Log LOG = LogFactory.getLog(ReplicationLogCleaner.class); + private ReplicationZookeeper zkHelper; + private Set hlogs = new HashSet(); + private boolean stopped = false; + private boolean aborted; + + /** + * Instantiates the cleaner, does nothing more. + */ + public ReplicationLogCleaner() {} + + @Override + public boolean isLogDeletable(Path filePath) { + + try { + if (!zkHelper.getReplication()) { + return false; + } + } catch (KeeperException e) { + abort("Cannot get the state of replication", e); + return false; + } + + // all members of this class are null if replication is disabled, and we + // return true since false would render the LogsCleaner useless + if (this.getConf() == null) { + return true; + } + String log = filePath.getName(); + // If we saw the hlog previously, let's consider it's still used + // At some point in the future we will refresh the list and it will be gone + if (this.hlogs.contains(log)) { + return false; + } + + // Let's see it's still there + // This solution makes every miss very expensive to process since we + // almost completely refresh the cache each time + return !refreshHLogsAndSearch(log); + } + + /** + * Search through all the hlogs we have in ZK to refresh the cache + * If a log is specified and found, then we early out and return true + * @param searchedLog log we are searching for, pass null to cache everything + * that's in zookeeper. + * @return false until a specified log is found. + */ + private boolean refreshHLogsAndSearch(String searchedLog) { + this.hlogs.clear(); + final boolean lookForLog = searchedLog != null; + List rss = zkHelper.getListOfReplicators(); + if (rss == null) { + LOG.debug("Didn't find any region server that replicates, deleting: " + + searchedLog); + return false; + } + for (String rs: rss) { + List listOfPeers = zkHelper.getListPeersForRS(rs); + // if rs just died, this will be null + if (listOfPeers == null) { + continue; + } + for (String id : listOfPeers) { + List peersHlogs = zkHelper.getListHLogsForPeerForRS(rs, id); + if (peersHlogs != null) { + this.hlogs.addAll(peersHlogs); + } + // early exit if we found the log + if(lookForLog && this.hlogs.contains(searchedLog)) { + LOG.debug("Found log in ZK, keeping: " + searchedLog); + return true; + } + } + } + LOG.debug("Didn't find this log in ZK, deleting: " + searchedLog); + return false; + } + + @Override + public void setConf(Configuration config) { + // If replication is disabled, keep all members null + if (!config.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false)) { + return; + } + // Make my own Configuration. Then I'll have my own connection to zk that + // I can close myself when comes time. + Configuration conf = new Configuration(config); + super.setConf(conf); + try { + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "replicationLogCleaner", null); + this.zkHelper = new ReplicationZookeeper(this, conf, zkw); + } catch (KeeperException e) { + LOG.error("Error while configuring " + this.getClass().getName(), e); + } catch (IOException e) { + LOG.error("Error while configuring " + this.getClass().getName(), e); + } + refreshHLogsAndSearch(null); + } + + + @Override + public void stop(String why) { + if (this.stopped) return; + this.stopped = true; + if (this.zkHelper != null) { + LOG.info("Stopping " + this.zkHelper.getZookeeperWatcher()); + this.zkHelper.getZookeeperWatcher().close(); + } + // Not sure why we're deleting a connection that we never acquired or used + HConnectionManager.deleteConnection(this.getConf()); + } + + @Override + public boolean isStopped() { + return this.stopped; + } + + @Override + public void abort(String why, Throwable e) { + LOG.warn("Aborting ReplicationLogCleaner because " + why, e); + this.aborted = true; + stop(why); + } + + @Override + public boolean isAborted() { + return this.aborted; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java new file mode 100644 index 0000000..509112a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java @@ -0,0 +1,236 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication.regionserver; + +import java.io.IOException; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.regionserver.ReplicationSourceService; +import org.apache.hadoop.hbase.regionserver.ReplicationSinkService; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; +import org.apache.hadoop.hbase.replication.ReplicationZookeeper; +import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.zookeeper.KeeperException; + +import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS; +import static org.apache.hadoop.hbase.HConstants.REPLICATION_ENABLE_KEY; +import static org.apache.hadoop.hbase.HConstants.REPLICATION_SCOPE_LOCAL; + +/** + * Gateway to Replication. Used by {@link org.apache.hadoop.hbase.regionserver.HRegionServer}. + */ +public class Replication implements WALActionsListener, + ReplicationSourceService, ReplicationSinkService { + private boolean replication; + private ReplicationSourceManager replicationManager; + private final AtomicBoolean replicating = new AtomicBoolean(true); + private ReplicationZookeeper zkHelper; + private Configuration conf; + private ReplicationSink replicationSink; + // Hosting server + private Server server; + + /** + * Instantiate the replication management (if rep is enabled). + * @param server Hosting server + * @param fs handle to the filesystem + * @param logDir + * @param oldLogDir directory where logs are archived + * @throws IOException + */ + public Replication(final Server server, final FileSystem fs, + final Path logDir, final Path oldLogDir) throws IOException{ + initialize(server, fs, logDir, oldLogDir); + } + + /** + * Empty constructor + */ + public Replication() { + } + + public void initialize(final Server server, final FileSystem fs, + final Path logDir, final Path oldLogDir) throws IOException { + this.server = server; + this.conf = this.server.getConfiguration(); + this.replication = isReplication(this.conf); + if (replication) { + try { + this.zkHelper = new ReplicationZookeeper(server, this.replicating); + } catch (KeeperException ke) { + throw new IOException("Failed replication handler create " + + "(replicating=" + this.replicating, ke); + } + this.replicationManager = new ReplicationSourceManager(zkHelper, conf, + this.server, fs, this.replicating, logDir, oldLogDir) ; + } else { + this.replicationManager = null; + this.zkHelper = null; + } + } + + /** + * @param c Configuration to look at + * @return True if replication is enabled. + */ + public static boolean isReplication(final Configuration c) { + return c.getBoolean(REPLICATION_ENABLE_KEY, false); + } + + /* + * Returns an object to listen to new hlog changes + **/ + public WALActionsListener getWALActionsListener() { + return this; + } + /** + * Stops replication service. + */ + public void stopReplicationService() { + join(); + } + + /** + * Join with the replication threads + */ + public void join() { + if (this.replication) { + this.replicationManager.join(); + if (this.replicationSink != null) { + this.replicationSink.stopReplicationSinkServices(); + } + } + } + + /** + * Carry on the list of log entries down to the sink + * @param entries list of entries to replicate + * @throws IOException + */ + public void replicateLogEntries(HLog.Entry[] entries) throws IOException { + if (this.replication) { + this.replicationSink.replicateEntries(entries); + } + } + + /** + * If replication is enabled and this cluster is a master, + * it starts + * @throws IOException + */ + public void startReplicationService() throws IOException { + if (this.replication) { + this.replicationManager.init(); + this.replicationSink = new ReplicationSink(this.conf, this.server); + } + } + + /** + * Get the replication sources manager + * @return the manager if replication is enabled, else returns false + */ + public ReplicationSourceManager getReplicationManager() { + return this.replicationManager; + } + + @Override + public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, + WALEdit logEdit) { + // Not interested + } + + @Override + public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, + WALEdit logEdit) { + NavigableMap scopes = + new TreeMap(Bytes.BYTES_COMPARATOR); + byte[] family; + for (KeyValue kv : logEdit.getKeyValues()) { + family = kv.getFamily(); + int scope = htd.getFamily(family).getScope(); + if (scope != REPLICATION_SCOPE_LOCAL && + !scopes.containsKey(family)) { + scopes.put(family, scope); + } + } + if (!scopes.isEmpty()) { + logEdit.setScopes(scopes); + } + } + + @Override + public void preLogRoll(Path oldPath, Path newPath) throws IOException { + getReplicationManager().preLogRoll(newPath); + } + + @Override + public void postLogRoll(Path oldPath, Path newPath) throws IOException { + getReplicationManager().postLogRoll(newPath); + } + + @Override + public void preLogArchive(Path oldPath, Path newPath) throws IOException { + // Not interested + } + + @Override + public void postLogArchive(Path oldPath, Path newPath) throws IOException { + // Not interested + } + + /** + * This method modifies the master's configuration in order to inject + * replication-related features + * @param conf + */ + public static void decorateMasterConfiguration(Configuration conf) { + if (!isReplication(conf)) { + return; + } + String plugins = conf.get(HBASE_MASTER_LOGCLEANER_PLUGINS); + String cleanerClass = ReplicationLogCleaner.class.getCanonicalName(); + if (!plugins.contains(cleanerClass)) { + conf.set(HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass); + } + } + + @Override + public void logRollRequested() { + // Not interested + } + + @Override + public void logCloseRequested() { + // not interested + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java new file mode 100644 index 0000000..98ff1c6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationHLogReaderManager.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication.regionserver; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.regionserver.wal.HLog; + +import java.io.IOException; + +/** + * Wrapper class around HLog to help manage the implementation details + * such as compression. + */ +@InterfaceAudience.Private +public class ReplicationHLogReaderManager { + + private static final Log LOG = LogFactory.getLog(ReplicationHLogReaderManager.class); + private final FileSystem fs; + private final Configuration conf; + private long position = 0; + private HLog.Reader reader; + private Path lastPath; + + /** + * Creates the helper but doesn't open any file + * Use setInitialPosition after using the constructor if some content needs to be skipped + * @param fs + * @param conf + */ + public ReplicationHLogReaderManager(FileSystem fs, Configuration conf) { + this.fs = fs; + this.conf = conf; + } + + /** + * Opens the file at the current position + * @param path + * @return + * @throws IOException + */ + public HLog.Reader openReader(Path path) throws IOException { + // Detect if this is a new file, if so get a new reader else + // reset the current reader so that we see the new data + if (this.reader == null || !this.lastPath.equals(path)) { + this.closeReader(); + this.reader = HLog.getReader(this.fs, path, this.conf); + this.lastPath = path; + } else { + try { + this.reader.reset(); + } catch (NullPointerException npe) { + throw new IOException("NPE resetting reader, likely HDFS-4380", npe); + } + } + return this.reader; + } + + /** + * Get the next entry, returned and also added in the array + * @param entriesArray + * @param currentNbEntries + * @return a new entry or null + * @throws IOException + */ + public HLog.Entry readNextAndSetPosition(HLog.Entry[] entriesArray, + int currentNbEntries) throws IOException { + HLog.Entry entry = this.reader.next(entriesArray[currentNbEntries]); + // Store the position so that in the future the reader can start + // reading from here. If the above call to next() throws an + // exception, the position won't be changed and retry will happen + // from the last known good position + this.position = this.reader.getPosition(); + // We need to set the CC to null else it will be compressed when sent to the sink + if (entry != null) { + entry.setCompressionContext(null); + } + return entry; + } + + /** + * Advance the reader to the current position + * @throws IOException + */ + public void seek() throws IOException { + if (this.position != 0) { + this.reader.seek(this.position); + } + } + + /** + * Get the position that we stopped reading at + * @return current position, cannot be negative + */ + public long getPosition() { + return this.position; + } + + public void setPosition(long pos) { + this.position = pos; + } + + /** + * Close the current reader + * @throws IOException + */ + public void closeReader() throws IOException { + if (this.reader != null) { + this.reader.close(); + this.reader = null; + } + } + + /** + * Tell the helper to reset internal state + */ + public void finishCurrentFile() { + this.position = 0; + try { + this.closeReader(); + } catch (IOException e) { + LOG.warn("Unable to close reader", e); + } + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java new file mode 100644 index 0000000..41b19ff --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java @@ -0,0 +1,227 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication.regionserver; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Row; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.Stoppable; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * This class is responsible for replicating the edits coming + * from another cluster. + *

+ * This replication process is currently waiting for the edits to be applied + * before the method can return. This means that the replication of edits + * is synchronized (after reading from HLogs in ReplicationSource) and that a + * single region server cannot receive edits from two sources at the same time + *

+ * This class uses the native HBase client in order to replicate entries. + *

+ * + * TODO make this class more like ReplicationSource wrt log handling + */ +public class ReplicationSink { + + private static final Log LOG = LogFactory.getLog(ReplicationSink.class); + // Name of the HDFS directory that contains the temporary rep logs + public static final String REPLICATION_LOG_DIR = ".replogs"; + private final Configuration conf; + private final ExecutorService sharedThreadPool; + private final HConnection sharedHtableCon; + private final ReplicationSinkMetrics metrics; + + /** + * Create a sink for replication + * + * @param conf conf object + * @param stopper boolean to tell this thread to stop + * @throws IOException thrown when HDFS goes bad or bad file name + */ + public ReplicationSink(Configuration conf, Stoppable stopper) + throws IOException { + this.conf = HBaseConfiguration.create(conf); + decorateConf(); + this.sharedHtableCon = HConnectionManager.createConnection(this.conf); + this.sharedThreadPool = new ThreadPoolExecutor(1, + conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE), + conf.getLong("hbase.htable.threads.keepalivetime", 60), TimeUnit.SECONDS, + new SynchronousQueue(), Threads.newDaemonThreadFactory("hbase-repl")); + ((ThreadPoolExecutor)this.sharedThreadPool).allowCoreThreadTimeOut(true); + this.metrics = new ReplicationSinkMetrics(); + } + + /** + * decorate the Configuration object to make replication more receptive to + * delays: lessen the timeout and numTries. + */ + private void decorateConf() { + this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + this.conf.getInt("replication.sink.client.retries.number", 4)); + this.conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, + this.conf.getInt("replication.sink.client.ops.timeout", 10000)); + } + + /** + * Replicate this array of entries directly into the local cluster + * using the native client. + * + * @param entries + * @throws IOException + */ + public void replicateEntries(HLog.Entry[] entries) + throws IOException { + if (entries.length == 0) { + return; + } + // Very simple optimization where we batch sequences of rows going + // to the same table. + try { + long totalReplicated = 0; + // Map of table => list of Rows, we only want to flushCommits once per + // invocation of this method per table. + Map> rows = new TreeMap>(Bytes.BYTES_COMPARATOR); + for (HLog.Entry entry : entries) { + WALEdit edit = entry.getEdit(); + byte[] table = entry.getKey().getTablename(); + Put put = null; + Delete del = null; + KeyValue lastKV = null; + List kvs = edit.getKeyValues(); + for (KeyValue kv : kvs) { + if (lastKV == null || lastKV.getType() != kv.getType() || !lastKV.matchingRow(kv)) { + if (kv.isDelete()) { + del = new Delete(kv.getRow()); + del.setClusterId(entry.getKey().getClusterId()); + addToMultiMap(rows, table, del); + } else { + put = new Put(kv.getRow()); + put.setClusterId(entry.getKey().getClusterId()); + addToMultiMap(rows, table, put); + } + } + if (kv.isDelete()) { + del.addDeleteMarker(kv); + } else { + put.add(kv); + } + lastKV = kv; + } + totalReplicated++; + } + for(byte [] table : rows.keySet()) { + batch(table, rows.get(table)); + } + this.metrics.setAgeOfLastAppliedOp( + entries[entries.length-1].getKey().getWriteTime()); + this.metrics.appliedBatchesRate.inc(1); + LOG.info("Total replicated: " + totalReplicated); + } catch (IOException ex) { + LOG.error("Unable to accept edit because:", ex); + throw ex; + } + } + + /** + * Simple helper to a map from key to (a list of) values + * TODO: Make a general utility method + * @param map + * @param key + * @param value + * @return + */ + private List addToMultiMap(Map> map, K key, V value) { + List values = map.get(key); + if (values == null) { + values = new ArrayList(); + map.put(key, values); + } + values.add(value); + return values; + } + + /** + * stop the thread pool executor. It is called when the regionserver is stopped. + */ + public void stopReplicationSinkServices() { + try { + this.sharedThreadPool.shutdown(); + if (!this.sharedThreadPool.awaitTermination(60000, TimeUnit.MILLISECONDS)) { + this.sharedThreadPool.shutdownNow(); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted while closing the table pool", e); // ignoring it as we are closing. + Thread.currentThread().interrupt(); + } + try { + this.sharedHtableCon.close(); + } catch (IOException e) { + LOG.warn("IOException while closing the connection", e); // ignoring as we are closing. + } + } + + /** + * Do the changes and handle the pool + * @param tableName table to insert into + * @param rows list of actions + * @throws IOException + */ + private void batch(byte[] tableName, List rows) throws IOException { + if (rows.isEmpty()) { + return; + } + HTableInterface table = null; + try { + table = new HTable(tableName, this.sharedHtableCon, this.sharedThreadPool); + table.batch(rows); + this.metrics.appliedOpsRate.inc(rows.size()); + } catch (InterruptedException ix) { + throw new IOException(ix); + } finally { + if (table != null) { + table.close(); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkMetrics.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkMetrics.java new file mode 100644 index 0000000..ae14375 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkMetrics.java @@ -0,0 +1,81 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication.regionserver; +import org.apache.hadoop.hbase.metrics.MetricsRate; +import org.apache.hadoop.metrics.MetricsContext; +import org.apache.hadoop.metrics.MetricsRecord; +import org.apache.hadoop.metrics.MetricsUtil; +import org.apache.hadoop.metrics.Updater; +import org.apache.hadoop.metrics.jvm.JvmMetrics; +import org.apache.hadoop.metrics.util.MetricsIntValue; +import org.apache.hadoop.metrics.util.MetricsLongValue; +import org.apache.hadoop.metrics.util.MetricsRegistry; + +/** + * This class is for maintaining the various replication statistics + * for a sink and publishing them through the metrics interfaces. + */ +public class ReplicationSinkMetrics implements Updater { + private final MetricsRecord metricsRecord; + private MetricsRegistry registry = new MetricsRegistry(); + private static ReplicationSinkMetrics instance; + + /** Rate of operations applied by the sink */ + public final MetricsRate appliedOpsRate = + new MetricsRate("appliedOpsRate", registry); + + /** Rate of batches (of operations) applied by the sink */ + public final MetricsRate appliedBatchesRate = + new MetricsRate("appliedBatchesRate", registry); + + /** Age of the last operation that was applied by the sink */ + private final MetricsLongValue ageOfLastAppliedOp = + new MetricsLongValue("ageOfLastAppliedOp", registry); + + /** + * Constructor used to register the metrics + */ + public ReplicationSinkMetrics() { + MetricsContext context = MetricsUtil.getContext("hbase"); + String name = Thread.currentThread().getName(); + metricsRecord = MetricsUtil.createRecord(context, "replication"); + metricsRecord.setTag("RegionServer", name); + context.registerUpdater(this); + // export for JMX + new ReplicationStatistics(this.registry, "ReplicationSink"); + } + + /** + * Set the age of the last edit that was applied + * @param timestamp write time of the edit + */ + public void setAgeOfLastAppliedOp(long timestamp) { + ageOfLastAppliedOp.set(System.currentTimeMillis() - timestamp); + } + @Override + public void doUpdates(MetricsContext metricsContext) { + synchronized (this) { + this.appliedOpsRate.pushMetric(this.metricsRecord); + this.appliedBatchesRate.pushMetric(this.metricsRecord); + this.ageOfLastAppliedOp.pushMetric(this.metricsRecord); + } + this.metricsRecord.update(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java new file mode 100644 index 0000000..930924d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java @@ -0,0 +1,929 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication.regionserver; + +import java.io.EOFException; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.ConnectException; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.NavigableMap; +import java.util.Random; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.PriorityBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.replication.ReplicationZookeeper; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.ClusterId; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.zookeeper.KeeperException; + +/** + * Class that handles the source of a replication stream. + * Currently does not handle more than 1 slave + * For each slave cluster it selects a random number of peers + * using a replication ratio. For example, if replication ration = 0.1 + * and slave cluster has 100 region servers, 10 will be selected. + *

+ * A stream is considered down when we cannot contact a region server on the + * peer cluster for more than 55 seconds by default. + *

+ * + */ +public class ReplicationSource extends Thread + implements ReplicationSourceInterface { + + private static final Log LOG = LogFactory.getLog(ReplicationSource.class); + // Queue of logs to process + private PriorityBlockingQueue queue; + // container of entries to replicate + private HLog.Entry[] entriesArray; + private HConnection conn; + // Helper class for zookeeper + private ReplicationZookeeper zkHelper; + private Configuration conf; + // ratio of region servers to chose from a slave cluster + private float ratio; + private Random random; + // should we replicate or not? + private AtomicBoolean replicating; + // id of the peer cluster this source replicates to + private String peerId; + // The manager of all sources to which we ping back our progress + private ReplicationSourceManager manager; + // Should we stop everything? + private Stoppable stopper; + // List of chosen sinks (region servers) + private List currentPeers; + // How long should we sleep for each retry + private long sleepForRetries; + // Max size in bytes of entriesArray + private long replicationQueueSizeCapacity; + // Max number of entries in entriesArray + private int replicationQueueNbCapacity; + // Our reader for the current log + private HLog.Reader reader; + // Last position in the log that we sent to ZooKeeper + private long lastLoggedPosition = -1; + // Path of the current log + private volatile Path currentPath; + private FileSystem fs; + // id of this cluster + private UUID clusterId; + // id of the other cluster + private UUID peerClusterId; + // total number of edits we replicated + private long totalReplicatedEdits = 0; + // The znode we currently play with + private String peerClusterZnode; + // Indicates if this queue is recovered (and will be deleted when depleted) + private boolean queueRecovered; + // List of all the dead region servers that had this queue (if recovered) + private List deadRegionServers = new ArrayList(); + // Maximum number of retries before taking bold actions + private int maxRetriesMultiplier; + // Socket timeouts require even bolder actions since we don't want to DDOS + private int socketTimeoutMultiplier; + // Current number of entries that we need to replicate + private int currentNbEntries = 0; + // Current number of operations (Put/Delete) that we need to replicate + private int currentNbOperations = 0; + // Current size of data we need to replicate + private int currentSize = 0; + // Indicates if this particular source is running + private volatile boolean running = true; + // Metrics for this source + private ReplicationSourceMetrics metrics; + // Handle on the log reader helper + private ReplicationHLogReaderManager repLogReader; + + + /** + * Instantiation method used by region servers + * + * @param conf configuration to use + * @param fs file system to use + * @param manager replication manager to ping to + * @param stopper the atomic boolean to use to stop the regionserver + * @param replicating the atomic boolean that starts/stops replication + * @param peerClusterZnode the name of our znode + * @throws IOException + */ + public void init(final Configuration conf, + final FileSystem fs, + final ReplicationSourceManager manager, + final Stoppable stopper, + final AtomicBoolean replicating, + final String peerClusterZnode) + throws IOException { + this.stopper = stopper; + this.conf = conf; + this.replicationQueueSizeCapacity = + this.conf.getLong("replication.source.size.capacity", 1024*1024*64); + this.replicationQueueNbCapacity = + this.conf.getInt("replication.source.nb.capacity", 25000); + this.entriesArray = new HLog.Entry[this.replicationQueueNbCapacity]; + for (int i = 0; i < this.replicationQueueNbCapacity; i++) { + this.entriesArray[i] = new HLog.Entry(); + } + this.maxRetriesMultiplier = this.conf.getInt("replication.source.maxretriesmultiplier", 10); + this.socketTimeoutMultiplier = this.conf.getInt("replication.source.socketTimeoutMultiplier", + maxRetriesMultiplier * maxRetriesMultiplier); + this.queue = + new PriorityBlockingQueue( + conf.getInt("hbase.regionserver.maxlogs", 32), + new LogsComparator()); + this.conn = HConnectionManager.getConnection(conf); + this.zkHelper = manager.getRepZkWrapper(); + this.ratio = this.conf.getFloat("replication.source.ratio", 0.1f); + this.currentPeers = new ArrayList(); + this.random = new Random(); + this.replicating = replicating; + this.manager = manager; + this.sleepForRetries = + this.conf.getLong("replication.source.sleepforretries", 1000); + this.fs = fs; + this.metrics = new ReplicationSourceMetrics(peerClusterZnode); + this.repLogReader = new ReplicationHLogReaderManager(this.fs, this.conf); + try { + this.clusterId = zkHelper.getUUIDForCluster(zkHelper.getZookeeperWatcher()); + } catch (KeeperException ke) { + throw new IOException("Could not read cluster id", ke); + } + + // Finally look if this is a recovered queue + this.checkIfQueueRecovered(peerClusterZnode); + } + + // The passed znode will be either the id of the peer cluster or + // the handling story of that queue in the form of id-servername-* + // + // package access for testing + void checkIfQueueRecovered(String peerClusterZnode) { + String[] parts = peerClusterZnode.split("-", 2); + this.queueRecovered = parts.length != 1; + this.peerId = this.queueRecovered ? + parts[0] : peerClusterZnode; + this.peerClusterZnode = peerClusterZnode; + + if (parts.length < 2) { + // not queue recovered situation + return; + } + + // extract dead servers + extractDeadServersFromZNodeString(parts[1], this.deadRegionServers); + } + + /** + * for tests only + */ + List getDeadRegionServers() { + return Collections.unmodifiableList(this.deadRegionServers); + } + + /** + * Parse dead server names from znode string servername can contain "-" such as + * "ip-10-46-221-101.ec2.internal", so we need skip some "-" during parsing for the following + * cases: 2-ip-10-46-221-101.ec2.internal,52170,1364333181125--... + */ + private static void + extractDeadServersFromZNodeString(String deadServerListStr, List result) { + + if (deadServerListStr == null || result == null || deadServerListStr.isEmpty()) return; + + // valid server name delimiter "-" has to be after "," in a server name + int seenCommaCnt = 0; + int startIndex = 0; + int len = deadServerListStr.length(); + + for (int i = 0; i < len; i++) { + switch (deadServerListStr.charAt(i)) { + case ',': + seenCommaCnt += 1; + break; + case '-': + if (seenCommaCnt >= 2) { + if (i > startIndex) { + result.add(deadServerListStr.substring(startIndex, i)); + startIndex = i + 1; + } + seenCommaCnt = 0; + } + break; + default: + break; + } + } + + // add tail + if (startIndex < len - 1) { + result.add(deadServerListStr.substring(startIndex, len)); + } + + LOG.debug("Found dead servers:" + result); + } + + /** + * Select a number of peers at random using the ratio. Mininum 1. + */ + private void chooseSinks() { + this.currentPeers.clear(); + List addresses = this.zkHelper.getSlavesAddresses(peerId); + Set setOfAddr = new HashSet(); + int nbPeers = (int) (Math.ceil(addresses.size() * ratio)); + LOG.info("Getting " + nbPeers + + " rs from peer cluster # " + peerId); + for (int i = 0; i < nbPeers; i++) { + ServerName sn; + // Make sure we get one address that we don't already have + do { + sn = addresses.get(this.random.nextInt(addresses.size())); + } while (setOfAddr.contains(sn)); + LOG.info("Choosing peer " + sn); + setOfAddr.add(sn); + } + this.currentPeers.addAll(setOfAddr); + } + + @Override + public void enqueueLog(Path log) { + this.queue.put(log); + this.metrics.sizeOfLogQueue.set(queue.size()); + } + + @Override + public void run() { + connectToPeers(); + // We were stopped while looping to connect to sinks, just abort + if (!this.isActive()) { + return; + } + int sleepMultiplier = 1; + // delay this until we are in an asynchronous thread + while (this.peerClusterId == null) { + this.peerClusterId = zkHelper.getPeerUUID(this.peerId); + if (this.peerClusterId == null) { + if (sleepForRetries("Cannot contact the peer's zk ensemble", sleepMultiplier)) { + sleepMultiplier++; + } + } + } + // resetting to 1 to reuse later + sleepMultiplier = 1; + + LOG.info("Replicating "+clusterId + " -> " + peerClusterId); + + // If this is recovered, the queue is already full and the first log + // normally has a position (unless the RS failed between 2 logs) + if (this.queueRecovered) { + try { + this.repLogReader.setPosition(this.zkHelper.getHLogRepPosition( + this.peerClusterZnode, this.queue.peek().getName())); + } catch (KeeperException e) { + this.terminate("Couldn't get the position of this recovered queue " + + peerClusterZnode, e); + } + } + // Loop until we close down + while (isActive()) { + // Sleep until replication is enabled again + if (!isPeerEnabled()) { + if (sleepForRetries("Replication is disabled", sleepMultiplier)) { + sleepMultiplier++; + } + continue; + } + Path oldPath = getCurrentPath(); //note that in the current scenario, + //oldPath will be null when a log roll + //happens. + // Get a new path + boolean hasCurrentPath = getNextPath(); + if (getCurrentPath() != null && oldPath == null) { + sleepMultiplier = 1; //reset the sleepMultiplier on a path change + } + if (!hasCurrentPath) { + if (sleepForRetries("No log to process", sleepMultiplier)) { + sleepMultiplier++; + } + continue; + } + boolean currentWALisBeingWrittenTo = false; + //For WAL files we own (rather than recovered), take a snapshot of whether the + //current WAL file (this.currentPath) is in use (for writing) NOW! + //Since the new WAL paths are enqueued only after the prev WAL file + //is 'closed', presence of an element in the queue means that + //the previous WAL file was closed, else the file is in use (currentPath) + //We take the snapshot now so that we are protected against races + //where a new file gets enqueued while the current file is being processed + //(and where we just finished reading the current file). + if (!this.queueRecovered && queue.size() == 0) { + currentWALisBeingWrittenTo = true; + } + // Open a reader on it + if (!openReader(sleepMultiplier)) { + // Reset the sleep multiplier, else it'd be reused for the next file + sleepMultiplier = 1; + continue; + } + + // If we got a null reader but didn't continue, then sleep and continue + if (this.reader == null) { + if (sleepForRetries("Unable to open a reader", sleepMultiplier)) { + sleepMultiplier++; + } + continue; + } + + boolean gotIOE = false; + currentNbOperations = 0; + currentNbEntries = 0; + currentSize = 0; + try { + if (readAllEntriesToReplicateOrNextFile(currentWALisBeingWrittenTo)) { + continue; + } + } catch (IOException ioe) { + LOG.warn(peerClusterZnode + " Got: ", ioe); + gotIOE = true; + if (ioe.getCause() instanceof EOFException) { + + boolean considerDumping = false; + if (this.queueRecovered) { + try { + FileStatus stat = this.fs.getFileStatus(this.currentPath); + if (stat.getLen() == 0) { + LOG.warn(peerClusterZnode + " Got EOF and the file was empty"); + } + considerDumping = true; + } catch (IOException e) { + LOG.warn(peerClusterZnode + " Got while getting file size: ", e); + } + } else if (currentNbEntries != 0) { + LOG.warn(peerClusterZnode + " Got EOF while reading, " + + "looks like this file is broken? " + currentPath); + considerDumping = true; + currentNbEntries = 0; + } + + if (considerDumping && + sleepMultiplier == this.maxRetriesMultiplier && + processEndOfFile()) { + continue; + } + } + } finally { + try { + this.reader = null; + this.repLogReader.closeReader(); + } catch (IOException e) { + gotIOE = true; + LOG.warn("Unable to finalize the tailing of a file", e); + } + } + + // If we didn't get anything to replicate, or if we hit a IOE, + // wait a bit and retry. + // But if we need to stop, don't bother sleeping + if (this.isActive() && (gotIOE || currentNbEntries == 0)) { + if (this.lastLoggedPosition != this.repLogReader.getPosition()) { + this.manager.logPositionAndCleanOldLogs(this.currentPath, + this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo); + this.lastLoggedPosition = this.repLogReader.getPosition(); + } + if (sleepForRetries("Nothing to replicate", sleepMultiplier)) { + sleepMultiplier++; + } + continue; + } + sleepMultiplier = 1; + shipEdits(currentWALisBeingWrittenTo); + + } + if (this.conn != null) { + try { + this.conn.close(); + } catch (IOException e) { + LOG.debug("Attempt to close connection failed", e); + } + } + LOG.debug("Source exiting " + peerId); + } + + /** + * Read all the entries from the current log files and retain those + * that need to be replicated. Else, process the end of the current file. + * @param currentWALisBeingWrittenTo is the current WAL being written to + * @return true if we got nothing and went to the next file, false if we got + * entries + * @throws IOException + */ + protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingWrittenTo) + throws IOException{ + long seenEntries = 0; + this.repLogReader.seek(); + HLog.Entry entry = + this.repLogReader.readNextAndSetPosition(this.entriesArray, this.currentNbEntries); + while (entry != null) { + WALEdit edit = entry.getEdit(); + this.metrics.logEditsReadRate.inc(1); + seenEntries++; + // Remove all KVs that should not be replicated + HLogKey logKey = entry.getKey(); + // don't replicate if the log entries originated in the peer + if (!logKey.getClusterId().equals(peerClusterId)) { + removeNonReplicableEdits(edit); + // Don't replicate catalog entries, if the WALEdit wasn't + // containing anything to replicate and if we're currently not set to replicate + if (!(Bytes.equals(logKey.getTablename(), HConstants.ROOT_TABLE_NAME) || + Bytes.equals(logKey.getTablename(), HConstants.META_TABLE_NAME)) && + edit.size() != 0 && replicating.get()) { + // Only set the clusterId if is a local key. + // This ensures that the originator sets the cluster id + // and all replicas retain the initial cluster id. + // This is *only* place where a cluster id other than the default is set. + if (HConstants.DEFAULT_CLUSTER_ID == logKey.getClusterId()) { + logKey.setClusterId(this.clusterId); + } + currentNbOperations += countDistinctRowKeys(edit); + currentNbEntries++; + currentSize += entry.getEdit().heapSize(); + } else { + this.metrics.logEditsFilteredRate.inc(1); + } + } + // Stop if too many entries or too big + if (currentSize >= this.replicationQueueSizeCapacity || + currentNbEntries >= this.replicationQueueNbCapacity) { + break; + } + try { + entry = this.repLogReader.readNextAndSetPosition(this.entriesArray, this.currentNbEntries); + } catch (IOException ie) { + LOG.debug("Break on IOE: " + ie.getMessage()); + break; + } + } + LOG.debug("currentNbOperations:" + currentNbOperations + + " and seenEntries:" + seenEntries + + " and size: " + this.currentSize); + if (currentWALisBeingWrittenTo) { + return false; + } + // If we didn't get anything and the queue has an object, it means we + // hit the end of the file for sure + return seenEntries == 0 && processEndOfFile(); + } + + private void connectToPeers() { + int sleepMultiplier = 1; + + // Connect to peer cluster first, unless we have to stop + while (this.isActive() && this.currentPeers.size() == 0) { + + chooseSinks(); + if (this.isActive() && this.currentPeers.size() == 0) { + if (sleepForRetries("Waiting for peers", sleepMultiplier)) { + sleepMultiplier++; + } + } + } + } + + /** + * Poll for the next path + * @return true if a path was obtained, false if not + */ + protected boolean getNextPath() { + try { + if (this.currentPath == null) { + this.currentPath = queue.poll(this.sleepForRetries, TimeUnit.MILLISECONDS); + this.metrics.sizeOfLogQueue.set(queue.size()); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted while reading edits", e); + } + return this.currentPath != null; + } + + /** + * Open a reader on the current path + * + * @param sleepMultiplier by how many times the default sleeping time is augmented + * @return true if we should continue with that file, false if we are over with it + */ + protected boolean openReader(int sleepMultiplier) { + try { + LOG.debug("Opening log for replication " + this.currentPath.getName() + + " at " + this.repLogReader.getPosition()); + try { + this.reader = repLogReader.openReader(this.currentPath); + } catch (FileNotFoundException fnfe) { + if (this.queueRecovered) { + // We didn't find the log in the archive directory, look if it still + // exists in the dead RS folder (there could be a chain of failures + // to look at) + LOG.info("NB dead servers : " + deadRegionServers.size()); + for (String curDeadServerName : deadRegionServers) { + Path deadRsDirectory = + new Path(manager.getLogDir().getParent(), curDeadServerName); + Path[] locs = new Path[] { + new Path(deadRsDirectory, currentPath.getName()), + new Path(deadRsDirectory.suffix(HLog.SPLITTING_EXT), + currentPath.getName()), + }; + for (Path possibleLogLocation : locs) { + LOG.info("Possible location " + possibleLogLocation.toUri().toString()); + if (this.manager.getFs().exists(possibleLogLocation)) { + // We found the right new location + LOG.info("Log " + this.currentPath + " still exists at " + + possibleLogLocation); + // Breaking here will make us sleep since reader is null + return true; + } + } + } + // TODO What happens if the log was missing from every single location? + // Although we need to check a couple of times as the log could have + // been moved by the master between the checks + // It can also happen if a recovered queue wasn't properly cleaned, + // such that the znode pointing to a log exists but the log was + // deleted a long time ago. + // For the moment, we'll throw the IO and processEndOfFile + throw new IOException("File from recovered queue is " + + "nowhere to be found", fnfe); + } else { + // If the log was archived, continue reading from there + Path archivedLogLocation = + new Path(manager.getOldLogDir(), currentPath.getName()); + if (this.manager.getFs().exists(archivedLogLocation)) { + currentPath = archivedLogLocation; + LOG.info("Log " + this.currentPath + " was moved to " + + archivedLogLocation); + // Open the log at the new location + this.openReader(sleepMultiplier); + + } + // TODO What happens the log is missing in both places? + } + } + } catch (IOException ioe) { + if (ioe instanceof EOFException && isCurrentLogEmpty()) return true; + LOG.warn(peerClusterZnode + " Got: ", ioe); + this.reader = null; + if (ioe.getCause() instanceof NullPointerException) { + // Workaround for race condition in HDFS-4380 + // which throws a NPE if we open a file before any data node has the most recent block + // Just sleep and retry. Will require re-reading compressed HLogs for compressionContext. + LOG.warn("Got NPE opening reader, will retry."); + } else if (sleepMultiplier == this.maxRetriesMultiplier) { + // TODO Need a better way to determine if a file is really gone but + // TODO without scanning all logs dir + LOG.warn("Waited too long for this file, considering dumping"); + return !processEndOfFile(); + } + } + return true; + } + + /* + * Checks whether the current log file is empty, and it is not a recovered queue. This is to + * handle scenario when in an idle cluster, there is no entry in the current log and we keep on + * trying to read the log file and get EOFEception. In case of a recovered queue the last log file + * may be empty, and we don't want to retry that. + */ + private boolean isCurrentLogEmpty() { + return (this.repLogReader.getPosition() == 0 && !queueRecovered && queue.size() == 0); + } + + /** + * Do the sleeping logic + * @param msg Why we sleep + * @param sleepMultiplier by how many times the default sleeping time is augmented + * @return True if sleepMultiplier is < maxRetriesMultiplier + */ + protected boolean sleepForRetries(String msg, int sleepMultiplier) { + try { + LOG.debug(msg + ", sleeping " + sleepForRetries + " times " + sleepMultiplier); + Thread.sleep(this.sleepForRetries * sleepMultiplier); + } catch (InterruptedException e) { + LOG.debug("Interrupted while sleeping between retries"); + } + return sleepMultiplier < maxRetriesMultiplier; + } + + /** + * We only want KVs that are scoped other than local + * @param edit The KV to check for replication + */ + protected void removeNonReplicableEdits(WALEdit edit) { + NavigableMap scopes = edit.getScopes(); + List kvs = edit.getKeyValues(); + for (int i = edit.size()-1; i >= 0; i--) { + KeyValue kv = kvs.get(i); + // The scope will be null or empty if + // there's nothing to replicate in that WALEdit + if (scopes == null || !scopes.containsKey(kv.getFamily())) { + kvs.remove(i); + } + } + } + + /** + * Count the number of different row keys in the given edit because of + * mini-batching. We assume that there's at least one KV in the WALEdit. + * @param edit edit to count row keys from + * @return number of different row keys + */ + private int countDistinctRowKeys(WALEdit edit) { + List kvs = edit.getKeyValues(); + int distinctRowKeys = 1; + KeyValue lastKV = kvs.get(0); + for (int i = 0; i < edit.size(); i++) { + if (!kvs.get(i).matchingRow(lastKV)) { + distinctRowKeys++; + } + } + return distinctRowKeys; + } + + /** + * Do the shipping logic + * @param currentWALisBeingWrittenTo was the current WAL being (seemingly) + * written to when this method was called + */ + protected void shipEdits(boolean currentWALisBeingWrittenTo) { + int sleepMultiplier = 1; + if (this.currentNbEntries == 0) { + LOG.warn("Was given 0 edits to ship"); + return; + } + while (this.isActive()) { + if (!isPeerEnabled()) { + if (sleepForRetries("Replication is disabled", sleepMultiplier)) { + sleepMultiplier++; + } + continue; + } + try { + HRegionInterface rrs = getRS(); + LOG.debug("Replicating " + currentNbEntries); + rrs.replicateLogEntries(Arrays.copyOf(this.entriesArray, currentNbEntries)); + if (this.lastLoggedPosition != this.repLogReader.getPosition()) { + this.manager.logPositionAndCleanOldLogs(this.currentPath, + this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo); + this.lastLoggedPosition = this.repLogReader.getPosition(); + } + this.totalReplicatedEdits += currentNbEntries; + this.metrics.shippedBatchesRate.inc(1); + this.metrics.shippedOpsRate.inc( + this.currentNbOperations); + this.metrics.setAgeOfLastShippedOp( + this.entriesArray[currentNbEntries-1].getKey().getWriteTime()); + LOG.debug("Replicated in total: " + this.totalReplicatedEdits); + break; + + } catch (IOException ioe) { + // Didn't ship anything, but must still age the last time we did + this.metrics.refreshAgeOfLastShippedOp(); + if (ioe instanceof RemoteException) { + ioe = ((RemoteException) ioe).unwrapRemoteException(); + LOG.warn("Can't replicate because of an error on the remote cluster: ", ioe); + if (ioe instanceof TableNotFoundException) { + if (sleepForRetries("A table is missing in the peer cluster. " + + "Replication cannot proceed without losing data.", sleepMultiplier)) { + sleepMultiplier++; + } + } + } else { + if (ioe instanceof SocketTimeoutException) { + // This exception means we waited for more than 60s and nothing + // happened, the cluster is alive and calling it right away + // even for a test just makes things worse. + sleepForRetries("Encountered a SocketTimeoutException. Since the " + + "call to the remote cluster timed out, which is usually " + + "caused by a machine failure or a massive slowdown", + this.socketTimeoutMultiplier); + } else if (ioe instanceof ConnectException) { + LOG.warn("Peer is unavailable, rechecking all sinks: ", ioe); + chooseSinks(); + } else { + LOG.warn("Can't replicate because of a local or network error: ", ioe); + } + } + + try { + boolean down; + // Spin while the slave is down and we're not asked to shutdown/close + do { + down = isSlaveDown(); + if (down) { + if (sleepForRetries("Since we are unable to replicate", sleepMultiplier)) { + sleepMultiplier++; + } else { + chooseSinks(); + } + } + } while (this.isActive() && down ); + } catch (InterruptedException e) { + LOG.debug("Interrupted while trying to contact the peer cluster"); + } + } + } + } + + /** + * check whether the peer is enabled or not + * + * @return true if the peer is enabled, otherwise false + */ + protected boolean isPeerEnabled() { + return this.replicating.get() && this.zkHelper.getPeerEnabled(peerId); + } + + /** + * If the queue isn't empty, switch to the next one + * Else if this is a recovered queue, it means we're done! + * Else we'll just continue to try reading the log file + * @return true if we're done with the current file, false if we should + * continue trying to read from it + */ + protected boolean processEndOfFile() { + if (this.queue.size() != 0) { + this.currentPath = null; + this.repLogReader.finishCurrentFile(); + this.reader = null; + return true; + } else if (this.queueRecovered) { + this.manager.closeRecoveredQueue(this); + LOG.info("Finished recovering the queue"); + this.running = false; + return true; + } + return false; + } + + public void startup() { + String n = Thread.currentThread().getName(); + Thread.UncaughtExceptionHandler handler = + new Thread.UncaughtExceptionHandler() { + public void uncaughtException(final Thread t, final Throwable e) { + LOG.error("Unexpected exception in ReplicationSource," + + " currentPath=" + currentPath, e); + } + }; + Threads.setDaemonThreadRunning( + this, n + ".replicationSource," + peerClusterZnode, handler); + } + + public void terminate(String reason) { + terminate(reason, null); + } + + public void terminate(String reason, Exception cause) { + if (cause == null) { + LOG.info("Closing source " + + this.peerClusterZnode + " because: " + reason); + + } else { + LOG.error("Closing source " + this.peerClusterZnode + + " because an error occurred: " + reason, cause); + } + this.running = false; + // Only wait for the thread to die if it's not us + if (!Thread.currentThread().equals(this)) { + Threads.shutdown(this, this.sleepForRetries); + } + } + + /** + * Get a new region server at random from this peer + * @return + * @throws IOException + */ + private HRegionInterface getRS() throws IOException { + if (this.currentPeers.size() == 0) { + throw new IOException(this.peerClusterZnode + " has 0 region servers"); + } + ServerName address = + currentPeers.get(random.nextInt(this.currentPeers.size())); + return this.conn.getHRegionConnection(address.getHostname(), address.getPort()); + } + + /** + * Check if the slave is down by trying to establish a connection + * @return true if down, false if up + * @throws InterruptedException + */ + public boolean isSlaveDown() throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + Thread pingThread = new Thread() { + public void run() { + try { + HRegionInterface rrs = getRS(); + // Dummy call which should fail + rrs.getHServerInfo(); + latch.countDown(); + } catch (IOException ex) { + if (ex instanceof RemoteException) { + ex = ((RemoteException) ex).unwrapRemoteException(); + } + LOG.info("Slave cluster looks down: " + ex.getMessage()); + } + } + }; + pingThread.start(); + // awaits returns true if countDown happened + boolean down = ! latch.await(this.sleepForRetries, TimeUnit.MILLISECONDS); + pingThread.interrupt(); + return down; + } + + public String getPeerClusterZnode() { + return this.peerClusterZnode; + } + + public String getPeerClusterId() { + return this.peerId; + } + + public Path getCurrentPath() { + return this.currentPath; + } + + private boolean isActive() { + return !this.stopper.isStopped() && this.running; + } + + /** + * Comparator used to compare logs together based on their start time + */ + public static class LogsComparator implements Comparator { + + @Override + public int compare(Path o1, Path o2) { + return Long.valueOf(getTS(o1)).compareTo(getTS(o2)); + } + + @Override + public boolean equals(Object o) { + return true; + } + + /** + * Split a path to get the start time + * For example: 10.20.20.171%3A60020.1277499063250 + * @param p path to split + * @return start time + */ + private long getTS(Path p) { + String[] parts = p.getName().split("\\."); + return Long.parseLong(parts[parts.length-1]); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java new file mode 100644 index 0000000..6778c43 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java @@ -0,0 +1,96 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication.regionserver; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Stoppable; + +/** + * Interface that defines a replication source + */ +public interface ReplicationSourceInterface { + + /** + * Initializer for the source + * @param conf the configuration to use + * @param fs the file system to use + * @param manager the manager to use + * @param stopper the stopper object for this region server + * @param replicating the status of the replication on this cluster + * @param peerClusterId the id of the peer cluster + * @throws IOException + */ + public void init(final Configuration conf, + final FileSystem fs, + final ReplicationSourceManager manager, + final Stoppable stopper, + final AtomicBoolean replicating, + final String peerClusterId) throws IOException; + + /** + * Add a log to the list of logs to replicate + * @param log path to the log to replicate + */ + public void enqueueLog(Path log); + + /** + * Get the current log that's replicated + * @return the current log + */ + public Path getCurrentPath(); + + /** + * Start the replication + */ + public void startup(); + + /** + * End the replication + * @param reason why it's terminating + */ + public void terminate(String reason); + + /** + * End the replication + * @param reason why it's terminating + * @param cause the error that's causing it + */ + public void terminate(String reason, Exception cause); + + /** + * Get the id that the source is replicating to + * + * @return peer cluster id + */ + public String getPeerClusterZnode(); + + /** + * Get the id that the source is replicating to. + * + * @return peer cluster id + */ + public String getPeerClusterId(); + +} diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java new file mode 100644 index 0000000..51d0e61 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java @@ -0,0 +1,654 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.replication.ReplicationZookeeper; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * This class is responsible to manage all the replication + * sources. There are two classes of sources: + *

  • Normal sources are persistent and one per peer cluster
    • + *
  • Old sources are recovered from a failed region server and our + * only goal is to finish replicating the HLog queue it had up in ZK
    • + * + * When a region server dies, this class uses a watcher to get notified and it + * tries to grab a lock in order to transfer all the queues in a local + * old source. + */ +public class ReplicationSourceManager { + private static final Log LOG = + LogFactory.getLog(ReplicationSourceManager.class); + // List of all the sources that read this RS's logs + private final List sources; + // List of all the sources we got from died RSs + private final List oldsources; + // Indicates if we are currently replicating + private final AtomicBoolean replicating; + // Helper for zookeeper + private final ReplicationZookeeper zkHelper; + // All about stopping + private final Stoppable stopper; + // All logs we are currently trackign + private final Map> hlogsById; + private final Configuration conf; + private final FileSystem fs; + // The path to the latest log we saw, for new coming sources + private Path latestPath; + // List of all the other region servers in this cluster + private final List otherRegionServers = new ArrayList(); + // Path to the hlogs directories + private final Path logDir; + // Path to the hlog archive + private final Path oldLogDir; + // The number of ms that we wait before moving znodes, HBASE-3596 + private final long sleepBeforeFailover; + // Homemade executer service for replication + private final ThreadPoolExecutor executor; + + private final Random rand; + + + /** + * Creates a replication manager and sets the watch on all the other + * registered region servers + * @param zkHelper the zk helper for replication + * @param conf the configuration to use + * @param stopper the stopper object for this region server + * @param fs the file system to use + * @param replicating the status of the replication on this cluster + * @param logDir the directory that contains all hlog directories of live RSs + * @param oldLogDir the directory where old logs are archived + */ + public ReplicationSourceManager(final ReplicationZookeeper zkHelper, + final Configuration conf, + final Stoppable stopper, + final FileSystem fs, + final AtomicBoolean replicating, + final Path logDir, + final Path oldLogDir) { + this.sources = new ArrayList(); + this.replicating = replicating; + this.zkHelper = zkHelper; + this.stopper = stopper; + this.hlogsById = new HashMap>(); + this.oldsources = new ArrayList(); + this.conf = conf; + this.fs = fs; + this.logDir = logDir; + this.oldLogDir = oldLogDir; + this.sleepBeforeFailover = conf.getLong("replication.sleep.before.failover", 2000); + this.zkHelper.registerRegionServerListener( + new OtherRegionServerWatcher(this.zkHelper.getZookeeperWatcher())); + this.zkHelper.registerRegionServerListener( + new PeersWatcher(this.zkHelper.getZookeeperWatcher())); + this.zkHelper.listPeersIdsAndWatch(); + // It's preferable to failover 1 RS at a time, but with good zk servers + // more could be processed at the same time. + int nbWorkers = conf.getInt("replication.executor.workers", 1); + // use a short 100ms sleep since this could be done inline with a RS startup + // even if we fail, other region servers can take care of it + this.executor = new ThreadPoolExecutor(nbWorkers, nbWorkers, + 100, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue()); + ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); + tfb.setNameFormat("ReplicationExecutor-%d"); + this.executor.setThreadFactory(tfb.build()); + this.rand = new Random(); + } + + /** + * Provide the id of the peer and a log key and this method will figure which + * hlog it belongs to and will log, for this region server, the current + * position. It will also clean old logs from the queue. + * @param log Path to the log currently being replicated from + * replication status in zookeeper. It will also delete older entries. + * @param id id of the peer cluster + * @param position current location in the log + * @param queueRecovered indicates if this queue comes from another region server + * @param holdLogInZK if true then the log is retained in ZK + */ + public void logPositionAndCleanOldLogs(Path log, String id, long position, + boolean queueRecovered, boolean holdLogInZK) { + String key = log.getName(); + LOG.info("Going to report log #" + key + " for position " + position + " in " + log); + this.zkHelper.writeReplicationStatus(key, id, position); + if (holdLogInZK) { + return; + } + synchronized (this.hlogsById) { + SortedSet hlogs = this.hlogsById.get(id); + if (!queueRecovered && hlogs.first() != key) { + SortedSet hlogSet = hlogs.headSet(key); + LOG.info("Removing " + hlogSet.size() + + " logs in the list: " + hlogSet); + for (String hlog : hlogSet) { + this.zkHelper.removeLogFromList(hlog, id); + } + hlogSet.clear(); + } + } + } + + /** + * Adds a normal source per registered peer cluster and tries to process all + * old region server hlog queues + */ + public void init() throws IOException { + for (String id : this.zkHelper.getPeerClusters().keySet()) { + addSource(id); + } + List currentReplicators = this.zkHelper.getListOfReplicators(); + if (currentReplicators == null || currentReplicators.size() == 0) { + return; + } + synchronized (otherRegionServers) { + refreshOtherRegionServersList(); + LOG.info("Current list of replicators: " + currentReplicators + + " other RSs: " + otherRegionServers); + } + // Look if there's anything to process after a restart + for (String rs : currentReplicators) { + synchronized (otherRegionServers) { + if (!this.otherRegionServers.contains(rs)) { + transferQueues(rs); + } + } + } + } + + /** + * Add a new normal source to this region server + * @param id the id of the peer cluster + * @return the source that was created + * @throws IOException + */ + public ReplicationSourceInterface addSource(String id) throws IOException { + ReplicationSourceInterface src = + getReplicationSource(this.conf, this.fs, this, stopper, replicating, id); + synchronized (this.hlogsById) { + this.sources.add(src); + this.hlogsById.put(id, new TreeSet()); + // Add the latest hlog to that source's queue + if (this.latestPath != null) { + String name = this.latestPath.getName(); + this.hlogsById.get(id).add(name); + try { + this.zkHelper.addLogToList(name, src.getPeerClusterZnode()); + } catch (KeeperException ke) { + String message = "Cannot add log to zk for" + + " replication when creating a new source"; + stopper.stop(message); + throw new IOException(message, ke); + } + src.enqueueLog(this.latestPath); + } + } + src.startup(); + return src; + } + + /** + * Terminate the replication on this region server + */ + public void join() { + this.executor.shutdown(); + if (this.sources.size() == 0) { + this.zkHelper.deleteOwnRSZNode(); + } + for (ReplicationSourceInterface source : this.sources) { + source.terminate("Region server is closing"); + } + } + + /** + * Get a copy of the hlogs of the first source on this rs + * @return a sorted set of hlog names + */ + protected Map> getHLogs() { + return Collections.unmodifiableMap(hlogsById); + } + + /** + * Get a list of all the normal sources of this rs + * @return lis of all sources + */ + public List getSources() { + return this.sources; + } + + void preLogRoll(Path newLog) throws IOException { + if (!this.replicating.get()) { + LOG.warn("Replication stopped, won't add new log"); + return; + } + + synchronized (this.hlogsById) { + String name = newLog.getName(); + for (ReplicationSourceInterface source : this.sources) { + try { + this.zkHelper.addLogToList(name, source.getPeerClusterZnode()); + } catch (KeeperException ke) { + throw new IOException("Cannot add log to zk for replication", ke); + } + } + for (SortedSet hlogs : this.hlogsById.values()) { + if (this.sources.isEmpty()) { + // If there's no slaves, don't need to keep the old hlogs since + // we only consider the last one when a new slave comes in + hlogs.clear(); + } + hlogs.add(name); + } + } + + this.latestPath = newLog; + } + + void postLogRoll(Path newLog) throws IOException { + if (!this.replicating.get()) { + LOG.warn("Replication stopped, won't add new log"); + return; + } + + // This only updates the sources we own, not the recovered ones + for (ReplicationSourceInterface source : this.sources) { + source.enqueueLog(newLog); + } + } + + /** + * Get the ZK help of this manager + * @return the helper + */ + public ReplicationZookeeper getRepZkWrapper() { + return zkHelper; + } + + /** + * Factory method to create a replication source + * @param conf the configuration to use + * @param fs the file system to use + * @param manager the manager to use + * @param stopper the stopper object for this region server + * @param replicating the status of the replication on this cluster + * @param peerId the id of the peer cluster + * @return the created source + * @throws IOException + */ + public ReplicationSourceInterface getReplicationSource( + final Configuration conf, + final FileSystem fs, + final ReplicationSourceManager manager, + final Stoppable stopper, + final AtomicBoolean replicating, + final String peerId) throws IOException { + ReplicationSourceInterface src; + try { + @SuppressWarnings("rawtypes") + Class c = Class.forName(conf.get("replication.replicationsource.implementation", + ReplicationSource.class.getCanonicalName())); + src = (ReplicationSourceInterface) c.newInstance(); + } catch (Exception e) { + LOG.warn("Passed replication source implementation throws errors, " + + "defaulting to ReplicationSource", e); + src = new ReplicationSource(); + + } + src.init(conf, fs, manager, stopper, replicating, peerId); + return src; + } + + /** + * Transfer all the queues of the specified to this region server. + * First it tries to grab a lock and if it works it will move the + * znodes and finally will delete the old znodes. + * + * It creates one old source for any type of source of the old rs. + * @param rsZnode + */ + public void transferQueues(String rsZnode) { + NodeFailoverWorker transfer = new NodeFailoverWorker(rsZnode); + try { + this.executor.execute(transfer); + } catch (RejectedExecutionException ex) { + LOG.info("Cancelling the transfer of " + rsZnode + + " because of " + ex.getMessage()); + } + } + + /** + * Clear the references to the specified old source + * @param src source to clear + */ + public void closeRecoveredQueue(ReplicationSourceInterface src) { + LOG.info("Done with the recovered queue " + src.getPeerClusterZnode()); + this.oldsources.remove(src); + this.zkHelper.deleteSource(src.getPeerClusterZnode(), false); + } + + /** + * Thie method first deletes all the recovered sources for the specified + * id, then deletes the normal source (deleting all related data in ZK). + * @param id The id of the peer cluster + */ + public void removePeer(String id) { + LOG.info("Closing the following queue " + id + ", currently have " + + sources.size() + " and another " + + oldsources.size() + " that were recovered"); + String terminateMessage = "Replication stream was removed by a user"; + ReplicationSourceInterface srcToRemove = null; + List oldSourcesToDelete = + new ArrayList(); + // First close all the recovered sources for this peer + for (ReplicationSourceInterface src : oldsources) { + if (id.equals(src.getPeerClusterId())) { + oldSourcesToDelete.add(src); + } + } + for (ReplicationSourceInterface src : oldSourcesToDelete) { + src.terminate(terminateMessage); + closeRecoveredQueue((src)); + } + LOG.info("Number of deleted recovered sources for " + id + ": " + + oldSourcesToDelete.size()); + // Now look for the one on this cluster + for (ReplicationSourceInterface src : this.sources) { + if (id.equals(src.getPeerClusterId())) { + srcToRemove = src; + break; + } + } + if (srcToRemove == null) { + LOG.error("The queue we wanted to close is missing " + id); + return; + } + srcToRemove.terminate(terminateMessage); + this.sources.remove(srcToRemove); + this.zkHelper.deleteSource(id, true); + } + + /** + * Reads the list of region servers from ZK and atomically clears our + * local view of it and replaces it with the updated list. + * + * @return true if the local list of the other region servers was updated + * with the ZK data (even if it was empty), + * false if the data was missing in ZK + */ + private boolean refreshOtherRegionServersList() { + List newRsList = zkHelper.getRegisteredRegionServers(); + if (newRsList == null) { + return false; + } else { + synchronized (otherRegionServers) { + otherRegionServers.clear(); + otherRegionServers.addAll(newRsList); + } + } + return true; + } + + /** + * Watcher used to be notified of the other region server's death + * in the local cluster. It initiates the process to transfer the queues + * if it is able to grab the lock. + */ + public class OtherRegionServerWatcher extends ZooKeeperListener { + + /** + * Construct a ZooKeeper event listener. + */ + public OtherRegionServerWatcher(ZooKeeperWatcher watcher) { + super(watcher); + } + + /** + * Called when a new node has been created. + * @param path full path of the new node + */ + public void nodeCreated(String path) { + refreshListIfRightPath(path); + } + + /** + * Called when a node has been deleted + * @param path full path of the deleted node + */ + public void nodeDeleted(String path) { + if (stopper.isStopped()) { + return; + } + boolean cont = refreshListIfRightPath(path); + if (!cont) { + return; + } + LOG.info(path + " znode expired, trying to lock it"); + transferQueues(ReplicationZookeeper.getZNodeName(path)); + } + + /** + * Called when an existing node has a child node added or removed. + * @param path full path of the node whose children have changed + */ + public void nodeChildrenChanged(String path) { + if (stopper.isStopped()) { + return; + } + refreshListIfRightPath(path); + } + + private boolean refreshListIfRightPath(String path) { + if (!path.startsWith(zkHelper.getZookeeperWatcher().rsZNode)) { + return false; + } + return refreshOtherRegionServersList(); + } + } + + /** + * Watcher used to follow the creation and deletion of peer clusters. + */ + public class PeersWatcher extends ZooKeeperListener { + + /** + * Construct a ZooKeeper event listener. + */ + public PeersWatcher(ZooKeeperWatcher watcher) { + super(watcher); + } + + /** + * Called when a node has been deleted + * @param path full path of the deleted node + */ + public void nodeDeleted(String path) { + List peers = refreshPeersList(path); + if (peers == null) { + return; + } + String id = ReplicationZookeeper.getZNodeName(path); + removePeer(id); + } + + /** + * Called when an existing node has a child node added or removed. + * @param path full path of the node whose children have changed + */ + public void nodeChildrenChanged(String path) { + List peers = refreshPeersList(path); + if (peers == null) { + return; + } + for (String id : peers) { + try { + boolean added = zkHelper.connectToPeer(id); + if (added) { + addSource(id); + } + } catch (IOException e) { + // TODO manage better than that ? + LOG.error("Error while adding a new peer", e); + } catch (KeeperException e) { + LOG.error("Error while adding a new peer", e); + } + } + } + + /** + * Verify if this event is meant for us, and if so then get the latest + * peers' list from ZK. Also reset the watches. + * @param path path to check against + * @return A list of peers' identifiers if the event concerns this watcher, + * else null. + */ + private List refreshPeersList(String path) { + if (!path.startsWith(zkHelper.getPeersZNode())) { + return null; + } + return zkHelper.listPeersIdsAndWatch(); + } + } + + /** + * Class responsible to setup new ReplicationSources to take care of the + * queues from dead region servers. + */ + class NodeFailoverWorker extends Thread { + + private String rsZnode; + + /** + * + * @param rsZnode + */ + public NodeFailoverWorker(String rsZnode) { + super("Failover-for-"+rsZnode); + this.rsZnode = rsZnode; + } + + @Override + public void run() { + // Wait a bit before transferring the queues, we may be shutting down. + // This sleep may not be enough in some cases. + try { + Thread.sleep(sleepBeforeFailover + (long) (rand.nextFloat() * sleepBeforeFailover)); + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting before transferring a queue."); + Thread.currentThread().interrupt(); + } + // We try to lock that rs' queue directory + if (stopper.isStopped()) { + LOG.info("Not transferring queue since we are shutting down"); + return; + } + SortedMap> newQueues = null; + + // check whether there is multi support. If yes, use it. + if (conf.getBoolean(HConstants.ZOOKEEPER_USEMULTI, true)) { + LOG.info("Atomically moving " + rsZnode + "'s hlogs to my queue"); + newQueues = zkHelper.copyQueuesFromRSUsingMulti(rsZnode); + } else { + LOG.info("Moving " + rsZnode + "'s hlogs to my queue"); + if (!zkHelper.lockOtherRS(rsZnode)) { + return; + } + newQueues = zkHelper.copyQueuesFromRS(rsZnode); + zkHelper.deleteRsQueues(rsZnode); + } + // process of copying over the failed queue is completed. + if (newQueues.isEmpty()) { + return; + } + + for (Map.Entry> entry : newQueues.entrySet()) { + String peerId = entry.getKey(); + try { + ReplicationSourceInterface src = getReplicationSource(conf, + fs, ReplicationSourceManager.this, stopper, replicating, peerId); + if (!zkHelper.getPeerClusters().containsKey(src.getPeerClusterId())) { + src.terminate("Recovered queue doesn't belong to any current peer"); + break; + } + oldsources.add(src); + for (String hlog : entry.getValue()) { + src.enqueueLog(new Path(oldLogDir, hlog)); + } + src.startup(); + } catch (IOException e) { + // TODO manage it + LOG.error("Failed creating a source", e); + } + } + } + } + + /** + * Get the directory where hlogs are archived + * @return the directory where hlogs are archived + */ + public Path getOldLogDir() { + return this.oldLogDir; + } + + /** + * Get the directory where hlogs are stored by their RSs + * @return the directory where hlogs are stored by their RSs + */ + public Path getLogDir() { + return this.logDir; + } + + /** + * Get the handle on the local file system + * @return Handle on the local file system + */ + public FileSystem getFs() { + return this.fs; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java new file mode 100644 index 0000000..85b4257 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java @@ -0,0 +1,123 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication.regionserver; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + +import org.apache.hadoop.hbase.metrics.MetricsRate; +import org.apache.hadoop.metrics.MetricsContext; +import org.apache.hadoop.metrics.MetricsRecord; +import org.apache.hadoop.metrics.MetricsUtil; +import org.apache.hadoop.metrics.Updater; +import org.apache.hadoop.metrics.jvm.JvmMetrics; +import org.apache.hadoop.metrics.util.MetricsIntValue; +import org.apache.hadoop.metrics.util.MetricsLongValue; +import org.apache.hadoop.metrics.util.MetricsRegistry; + +/** + * This class is for maintaining the various replication statistics + * for a source and publishing them through the metrics interfaces. + */ +public class ReplicationSourceMetrics implements Updater { + private final MetricsRecord metricsRecord; + private MetricsRegistry registry = new MetricsRegistry(); + + /** Rate of shipped operations by the source */ + public final MetricsRate shippedOpsRate = + new MetricsRate("shippedOpsRate", registry); + + /** Rate of shipped batches by the source */ + public final MetricsRate shippedBatchesRate = + new MetricsRate("shippedBatchesRate", registry); + + /** Rate of log entries (can be multiple Puts) read from the logs */ + public final MetricsRate logEditsReadRate = + new MetricsRate("logEditsReadRate", registry); + + /** Rate of log entries filtered by the source */ + public final MetricsRate logEditsFilteredRate = + new MetricsRate("logEditsFilteredRate", registry); + + /** Age of the last operation that was shipped by the source */ + private final MetricsLongValue ageOfLastShippedOp = + new MetricsLongValue("ageOfLastShippedOp", registry); + + /** + * Current size of the queue of logs to replicate, + * excluding the one being processed at the moment + */ + public final MetricsIntValue sizeOfLogQueue = + new MetricsIntValue("sizeOfLogQueue", registry); + + // It's a little dirty to preset the age to now since if we fail + // to replicate the very first time then it will show that age instead + // of nothing (although that might not be good either). + private long lastTimestampForAge = System.currentTimeMillis(); + + /** + * Constructor used to register the metrics + * @param id Name of the source this class is monitoring + */ + public ReplicationSourceMetrics(String id) { + MetricsContext context = MetricsUtil.getContext("hbase"); + String name = Thread.currentThread().getName(); + metricsRecord = MetricsUtil.createRecord(context, "replication"); + metricsRecord.setTag("RegionServer", name); + context.registerUpdater(this); + try { + id = URLEncoder.encode(id, "UTF8"); + } catch (UnsupportedEncodingException e) { + id = "CAN'T ENCODE UTF8"; + } + // export for JMX + new ReplicationStatistics(this.registry, "ReplicationSource for " + id); + } + + /** + * Set the age of the last edit that was shipped + * @param timestamp write time of the edit + */ + public void setAgeOfLastShippedOp(long timestamp) { + lastTimestampForAge = timestamp; + ageOfLastShippedOp.set(System.currentTimeMillis() - lastTimestampForAge); + } + + /** + * Convenience method to use the last given timestamp to refresh the age + * of the last edit. Used when replication fails and need to keep that + * metric accurate. + */ + public void refreshAgeOfLastShippedOp() { + setAgeOfLastShippedOp(lastTimestampForAge); + } + + @Override + public void doUpdates(MetricsContext metricsContext) { + synchronized (this) { + this.shippedOpsRate.pushMetric(this.metricsRecord); + this.shippedBatchesRate.pushMetric(this.metricsRecord); + this.logEditsReadRate.pushMetric(this.metricsRecord); + this.logEditsFilteredRate.pushMetric(this.metricsRecord); + this.ageOfLastShippedOp.pushMetric(this.metricsRecord); + this.sizeOfLogQueue.pushMetric(this.metricsRecord); + } + this.metricsRecord.update(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java new file mode 100644 index 0000000..54ca3df --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java @@ -0,0 +1,45 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication.regionserver; + +import org.apache.hadoop.hbase.metrics.MetricsMBeanBase; +import org.apache.hadoop.metrics.util.MBeanUtil; +import org.apache.hadoop.metrics.util.MetricsRegistry; + +import javax.management.ObjectName; + +/** + * Exports metrics recorded by {@link ReplicationSourceMetrics} as an MBean + * for JMX monitoring. + */ +public class ReplicationStatistics extends MetricsMBeanBase { + + private final ObjectName mbeanName; + + /** + * Constructor to register the MBean + * @param registry which rehistry to use + * @param name name to get to this bean + */ + public ReplicationStatistics(MetricsRegistry registry, String name) { + super(registry, name); + mbeanName = MBeanUtil.registerMBean("Replication", name, this); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Constants.java b/src/main/java/org/apache/hadoop/hbase/rest/Constants.java new file mode 100644 index 0000000..21d76fe --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/Constants.java @@ -0,0 +1,42 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +/** + * Common constants for org.apache.hadoop.hbase.rest + */ +public interface Constants { + public static final String VERSION_STRING = "0.0.2"; + + public static final int DEFAULT_MAX_AGE = 60 * 60 * 4; // 4 hours + + public static final int DEFAULT_LISTEN_PORT = 8080; + + public static final String MIMETYPE_TEXT = "text/plain"; + public static final String MIMETYPE_HTML = "text/html"; + public static final String MIMETYPE_XML = "text/xml"; + public static final String MIMETYPE_BINARY = "application/octet-stream"; + public static final String MIMETYPE_PROTOBUF = "application/x-protobuf"; + public static final String MIMETYPE_PROTOBUF_IETF = "application/protobuf"; + public static final String MIMETYPE_JSON = "application/json"; + + public static final String CRLF = "\r\n"; +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java new file mode 100644 index 0000000..62a1b0e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/ExistsResource.java @@ -0,0 +1,73 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; + +import javax.ws.rs.GET; +import javax.ws.rs.Produces; +import javax.ws.rs.core.CacheControl; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import javax.ws.rs.core.Response.ResponseBuilder; + +public class ExistsResource extends ResourceBase { + + static CacheControl cacheControl; + static { + cacheControl = new CacheControl(); + cacheControl.setNoCache(true); + cacheControl.setNoTransform(false); + } + + TableResource tableResource; + + /** + * Constructor + * @param tableResource + * @throws IOException + */ + public ExistsResource(TableResource tableResource) throws IOException { + super(); + this.tableResource = tableResource; + } + + @GET + @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF, MIMETYPE_BINARY}) + public Response get(final @Context UriInfo uriInfo) { + try { + if (!tableResource.exists()) { + return Response.status(Response.Status.NOT_FOUND) + .type(MIMETYPE_TEXT).entity("Not found" + CRLF) + .build(); + } + } catch (IOException e) { + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + ResponseBuilder response = Response.ok(); + response.cacheControl(cacheControl); + return response.build(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/Main.java b/src/main/java/org/apache/hadoop/hbase/rest/Main.java new file mode 100644 index 0000000..ca3edc5 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/Main.java @@ -0,0 +1,214 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.cli.ParseException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.rest.filter.GzipFilter; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.InfoServer; +import org.apache.hadoop.hbase.util.Strings; +import org.apache.hadoop.hbase.util.VersionInfo; +import org.apache.hadoop.net.DNS; + +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.Map.Entry; + +import org.mortbay.jetty.Connector; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.nio.SelectChannelConnector; +import org.mortbay.jetty.servlet.Context; +import org.mortbay.jetty.servlet.ServletHolder; +import org.mortbay.thread.QueuedThreadPool; + +import com.sun.jersey.api.json.JSONConfiguration; +import com.sun.jersey.spi.container.servlet.ServletContainer; + +/** + * Main class for launching REST gateway as a servlet hosted by Jetty. + *

    + * The following options are supported: + *

      + *
    • -p --port : service port
      • + *
    • -ro --readonly : server mode
      • + *
    + */ +public class Main implements Constants { + + private static void printUsageAndExit(Options options, int exitCode) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("bin/hbase rest start", "", options, + "\nTo run the REST server as a daemon, execute " + + "bin/hbase-daemon.sh start|stop rest [--infoport ] [-p ] [-ro]\n", true); + System.exit(exitCode); + } + + /** + * The main method for the HBase rest server. + * @param args command-line arguments + * @throws Exception exception + */ + public static void main(String[] args) throws Exception { + Log LOG = LogFactory.getLog("RESTServer"); + + VersionInfo.logVersion(); + Configuration conf = HBaseConfiguration.create(); + // login the server principal (if using secure Hadoop) + if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) { + String machineName = Strings.domainNamePointerToHostName( + DNS.getDefaultHost(conf.get("hbase.rest.dns.interface", "default"), + conf.get("hbase.rest.dns.nameserver", "default"))); + User.login(conf, "hbase.rest.keytab.file", "hbase.rest.kerberos.principal", + machineName); + } + + RESTServlet servlet = RESTServlet.getInstance(conf); + + Options options = new Options(); + options.addOption("p", "port", true, "Port to bind to [default: 8080]"); + options.addOption("ro", "readonly", false, "Respond only to GET HTTP " + + "method requests [default: false]"); + options.addOption(null, "infoport", true, "Port for web UI"); + + CommandLine commandLine = null; + try { + commandLine = new PosixParser().parse(options, args); + } catch (ParseException e) { + LOG.error("Could not parse: ", e); + printUsageAndExit(options, -1); + } + + // check for user-defined port setting, if so override the conf + if (commandLine != null && commandLine.hasOption("port")) { + String val = commandLine.getOptionValue("port"); + servlet.getConfiguration() + .setInt("hbase.rest.port", Integer.valueOf(val)); + LOG.debug("port set to " + val); + } + + // check if server should only process GET requests, if so override the conf + if (commandLine != null && commandLine.hasOption("readonly")) { + servlet.getConfiguration().setBoolean("hbase.rest.readonly", true); + LOG.debug("readonly set to true"); + } + + // check for user-defined info server port setting, if so override the conf + if (commandLine != null && commandLine.hasOption("infoport")) { + String val = commandLine.getOptionValue("infoport"); + servlet.getConfiguration() + .setInt("hbase.rest.info.port", Integer.valueOf(val)); + LOG.debug("Web UI port set to " + val); + } + + @SuppressWarnings("unchecked") + List remainingArgs = commandLine != null ? + commandLine.getArgList() : new ArrayList(); + if (remainingArgs.size() != 1) { + printUsageAndExit(options, 1); + } + + String command = remainingArgs.get(0); + if ("start".equals(command)) { + // continue and start container + } else if ("stop".equals(command)) { + System.exit(1); + } else { + printUsageAndExit(options, 1); + } + + // set up the Jersey servlet container for Jetty + ServletHolder sh = new ServletHolder(ServletContainer.class); + sh.setInitParameter( + "com.sun.jersey.config.property.resourceConfigClass", + ResourceConfig.class.getCanonicalName()); + sh.setInitParameter("com.sun.jersey.config.property.packages", + "jetty"); + // The servlet holder below is instantiated to only handle the case + // of the /status/cluster returning arrays of nodes (live/dead). Without + // this servlet holder, the problem is that the node arrays in the response + // are collapsed to single nodes. We want to be able to treat the + // node lists as POJO in the response to /status/cluster servlet call, + // but not change the behavior for any of the other servlets + // Hence we don't use the servlet holder for all servlets / paths + ServletHolder shPojoMap = new ServletHolder(ServletContainer.class); + @SuppressWarnings("unchecked") + Map shInitMap = sh.getInitParameters(); + for (Entry e : shInitMap.entrySet()) { + shPojoMap.setInitParameter(e.getKey(), e.getValue()); + } + shPojoMap.setInitParameter(JSONConfiguration.FEATURE_POJO_MAPPING, "true"); + + // set up Jetty and run the embedded server + + Server server = new Server(); + + Connector connector = new SelectChannelConnector(); + connector.setPort(servlet.getConfiguration().getInt("hbase.rest.port", 8080)); + connector.setHost(servlet.getConfiguration().get("hbase.rest.host", "0.0.0.0")); + + server.addConnector(connector); + + // Set the default max thread number to 100 to limit + // the number of concurrent requests so that REST server doesn't OOM easily. + // Jetty set the default max thread number to 250, if we don't set it. + // + // Our default min thread number 2 is the same as that used by Jetty. + int maxThreads = servlet.getConfiguration().getInt("hbase.rest.threads.max", 100); + int minThreads = servlet.getConfiguration().getInt("hbase.rest.threads.min", 2); + QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads); + threadPool.setMinThreads(minThreads); + server.setThreadPool(threadPool); + + server.setSendServerVersion(false); + server.setSendDateHeader(false); + server.setStopAtShutdown(true); + + // set up context + Context context = new Context(server, "/", Context.SESSIONS); + context.addServlet(shPojoMap, "/status/cluster"); + context.addServlet(sh, "/*"); + context.addFilter(GzipFilter.class, "/*", 0); + + // Put up info server. + int port = conf.getInt("hbase.rest.info.port", 8085); + if (port >= 0) { + conf.setLong("startcode", System.currentTimeMillis()); + String a = conf.get("hbase.rest.info.bindAddress", "0.0.0.0"); + InfoServer infoServer = new InfoServer("rest", a, port, false, conf); + infoServer.setAttribute("hbase.conf", conf); + infoServer.start(); + } + + // start server + server.start(); + server.join(); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java new file mode 100644 index 0000000..887e8a1 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java @@ -0,0 +1,105 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rest; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.rest.ResourceBase; +import org.apache.hadoop.hbase.rest.RowSpec; +import org.apache.hadoop.hbase.rest.TableResource; +import org.apache.hadoop.hbase.rest.model.CellModel; +import org.apache.hadoop.hbase.rest.model.CellSetModel; +import org.apache.hadoop.hbase.rest.model.RowModel; + +import javax.ws.rs.GET; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import java.io.IOException; + +public class MultiRowResource extends ResourceBase { + public static final String ROW_KEYS_PARAM_NAME = "row"; + + TableResource tableResource; + Integer versions = null; + + /** + * Constructor + * + * @param tableResource + * @param versions + * @throws java.io.IOException + */ + public MultiRowResource(TableResource tableResource, String versions) throws IOException { + super(); + this.tableResource = tableResource; + + if (versions != null) { + this.versions = Integer.valueOf(versions); + + } + } + + @GET + @Produces({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response get(final @Context UriInfo uriInfo) { + MultivaluedMap params = uriInfo.getQueryParameters(); + + servlet.getMetrics().incrementRequests(1); + try { + CellSetModel model = new CellSetModel(); + for (String rk : params.get(ROW_KEYS_PARAM_NAME)) { + RowSpec rowSpec = new RowSpec(rk); + + if (this.versions != null) { + rowSpec.setMaxVersions(this.versions); + } + + ResultGenerator generator = + ResultGenerator.fromRowSpec(this.tableResource.getName(), rowSpec, null); + if (!generator.hasNext()) { + return Response.status(Response.Status.NOT_FOUND) + .type(MIMETYPE_TEXT).entity("Not found" + CRLF) + .build(); + } + + KeyValue value = null; + RowModel rowModel = new RowModel(rk); + + while ((value = generator.next()) != null) { + rowModel.addCell(new CellModel(value.getFamily(), value.getQualifier(), + value.getTimestamp(), value.getValue())); + } + + model.addRow(rowModel); + } + servlet.getMetrics().incrementSucessfulGetRequests(1); + return Response.ok(model).build(); + } catch (IOException e) { + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java b/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java new file mode 100644 index 0000000..405cace --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java @@ -0,0 +1,44 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; + +/** + * Common interface for models capable of supporting protobuf marshalling + * and unmarshalling. Hooks up to the ProtobufMessageBodyConsumer and + * ProtobufMessageBodyProducer adapters. + */ +public abstract interface ProtobufMessageHandler { + /** + * @return the protobuf represention of the model + */ + public byte[] createProtobufOutput(); + + /** + * Initialize the model from a protobuf representation. + * @param message the raw bytes of the protobuf message + * @return reference to self for convenience + * @throws IOException + */ + public ProtobufMessageHandler getObjectFromMessage(byte[] message) + throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java b/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java new file mode 100644 index 0000000..91dca78 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java @@ -0,0 +1,102 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTablePool; +import org.apache.hadoop.hbase.rest.metrics.RESTMetrics; + +/** + * Singleton class encapsulating global REST servlet state and functions. + */ +public class RESTServlet implements Constants { + private static RESTServlet INSTANCE; + private final Configuration conf; + private final HTablePool pool; + private final RESTMetrics metrics = new RESTMetrics(); + private final HBaseAdmin admin; + + /** + * @return the RESTServlet singleton instance + * @throws IOException + */ + public synchronized static RESTServlet getInstance() throws IOException { + assert(INSTANCE != null); + return INSTANCE; + } + + /** + * @param conf Existing configuration to use in rest servlet + * @return the RESTServlet singleton instance + * @throws IOException + */ + public synchronized static RESTServlet getInstance(Configuration conf) + throws IOException { + if (INSTANCE == null) { + INSTANCE = new RESTServlet(conf); + } + return INSTANCE; + } + + public synchronized static void stop() { + if (INSTANCE != null) INSTANCE = null; + } + + /** + * Constructor with existing configuration + * @param conf existing configuration + * @throws IOException. + */ + RESTServlet(Configuration conf) throws IOException { + this.conf = conf; + int maxSize = conf.getInt("hbase.rest.htablepool.size", 10); + this.pool = new HTablePool(conf, maxSize); + this.admin = new HBaseAdmin(conf); + } + + HBaseAdmin getAdmin() { + return admin; + } + + HTablePool getTablePool() { + return pool; + } + + Configuration getConfiguration() { + return conf; + } + + RESTMetrics getMetrics() { + return metrics; + } + + /** + * Helper method to determine if server should + * only respond to GET HTTP method requests. + * @return boolean for server read-only state + */ + boolean isReadOnly() { + return getConfiguration().getBoolean("hbase.rest.readonly", false); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java new file mode 100644 index 0000000..8d4b621 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java @@ -0,0 +1,103 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; +import java.util.Map; + +import javax.ws.rs.GET; +import javax.ws.rs.Produces; +import javax.ws.rs.core.CacheControl; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import javax.ws.rs.core.Response.ResponseBuilder; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.MetaScanner; +import org.apache.hadoop.hbase.rest.model.TableInfoModel; +import org.apache.hadoop.hbase.rest.model.TableRegionModel; +import org.apache.hadoop.hbase.util.Bytes; + +public class RegionsResource extends ResourceBase { + private static final Log LOG = LogFactory.getLog(RegionsResource.class); + + static CacheControl cacheControl; + static { + cacheControl = new CacheControl(); + cacheControl.setNoCache(true); + cacheControl.setNoTransform(false); + } + + TableResource tableResource; + + /** + * Constructor + * @param tableResource + * @throws IOException + */ + public RegionsResource(TableResource tableResource) throws IOException { + super(); + this.tableResource = tableResource; + } + + @GET + @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response get(final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("GET " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + try { + String tableName = tableResource.getName(); + TableInfoModel model = new TableInfoModel(tableName); + Map regions = MetaScanner.allTableRegions( + servlet.getConfiguration(), Bytes.toBytes(tableName), false); + for (Map.Entry e: regions.entrySet()) { + HRegionInfo hri = e.getKey(); + ServerName addr = e.getValue(); + model.add( + new TableRegionModel(tableName, hri.getRegionId(), + hri.getStartKey(), hri.getEndKey(), addr.getHostAndPort())); + } + ResponseBuilder response = Response.ok(model); + response.cacheControl(cacheControl); + servlet.getMetrics().incrementSucessfulGetRequests(1); + return response.build(); + } catch (TableNotFoundException e) { + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.NOT_FOUND) + .type(MIMETYPE_TEXT).entity("Not found" + CRLF) + .build(); + } catch (IOException e) { + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ResourceBase.java b/src/main/java/org/apache/hadoop/hbase/rest/ResourceBase.java new file mode 100644 index 0000000..6167ccc --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/ResourceBase.java @@ -0,0 +1,32 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; + +public class ResourceBase implements Constants { + + RESTServlet servlet; + + public ResourceBase() throws IOException { + servlet = RESTServlet.getInstance(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ResourceConfig.java b/src/main/java/org/apache/hadoop/hbase/rest/ResourceConfig.java new file mode 100644 index 0000000..19c99e7 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/ResourceConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import com.sun.jersey.api.core.PackagesResourceConfig; + +public class ResourceConfig extends PackagesResourceConfig { + public ResourceConfig() { + super("org.apache.hadoop.hbase.rest"); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ResultGenerator.java b/src/main/java/org/apache/hadoop/hbase/rest/ResultGenerator.java new file mode 100644 index 0000000..4e7edf9 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/ResultGenerator.java @@ -0,0 +1,48 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; +import java.util.Iterator; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.rest.model.ScannerModel; + +public abstract class ResultGenerator implements Iterator { + + public static ResultGenerator fromRowSpec(final String table, + final RowSpec rowspec, final Filter filter) throws IOException { + if (rowspec.isSingleRow()) { + return new RowResultGenerator(table, rowspec, filter); + } else { + return new ScannerResultGenerator(table, rowspec, filter); + } + } + + public static Filter buildFilter(final String filter) throws Exception { + return ScannerModel.buildFilter(filter); + } + + public abstract void putBack(KeyValue kv); + + public abstract void close(); +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java new file mode 100644 index 0000000..e1e6d72 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/RootResource.java @@ -0,0 +1,107 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.CacheControl; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import javax.ws.rs.core.Response.ResponseBuilder; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.rest.model.TableListModel; +import org.apache.hadoop.hbase.rest.model.TableModel; + +@Path("/") +public class RootResource extends ResourceBase { + private static final Log LOG = LogFactory.getLog(RootResource.class); + + static CacheControl cacheControl; + static { + cacheControl = new CacheControl(); + cacheControl.setNoCache(true); + cacheControl.setNoTransform(false); + } + + /** + * Constructor + * @throws IOException + */ + public RootResource() throws IOException { + super(); + } + + private final TableListModel getTableList() throws IOException { + TableListModel tableList = new TableListModel(); + HTableDescriptor[] list = servlet.getAdmin().listTables(); + for (HTableDescriptor htd: list) { + tableList.add(new TableModel(htd.getNameAsString())); + } + return tableList; + } + + @GET + @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response get(final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("GET " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + try { + ResponseBuilder response = Response.ok(getTableList()); + response.cacheControl(cacheControl); + servlet.getMetrics().incrementSucessfulGetRequests(1); + return response.build(); + } catch (IOException e) { + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + } + + @Path("status/cluster") + public StorageClusterStatusResource getClusterStatusResource() + throws IOException { + return new StorageClusterStatusResource(); + } + + @Path("version") + public VersionResource getVersionResource() throws IOException { + return new VersionResource(); + } + + @Path("{table}") + public TableResource getTableResource( + final @PathParam("table") String table) throws IOException { + return new TableResource(table); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java new file mode 100644 index 0000000..c348f36 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java @@ -0,0 +1,564 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import javax.ws.rs.core.Response.ResponseBuilder; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.HTablePool; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.rest.model.CellModel; +import org.apache.hadoop.hbase.rest.model.CellSetModel; +import org.apache.hadoop.hbase.rest.model.RowModel; +import org.apache.hadoop.hbase.util.Bytes; + +public class RowResource extends ResourceBase { + private static final Log LOG = LogFactory.getLog(RowResource.class); + + static final String CHECK_PUT = "put"; + static final String CHECK_DELETE = "delete"; + + TableResource tableResource; + RowSpec rowspec; + private String check = null; + + /** + * Constructor + * @param tableResource + * @param rowspec + * @param versions + * @throws IOException + */ + public RowResource(TableResource tableResource, String rowspec, + String versions, String check) throws IOException { + super(); + this.tableResource = tableResource; + this.rowspec = new RowSpec(rowspec); + if (versions != null) { + this.rowspec.setMaxVersions(Integer.valueOf(versions)); + } + this.check = check; + } + + @GET + @Produces({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response get(final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("GET " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + try { + ResultGenerator generator = + ResultGenerator.fromRowSpec(tableResource.getName(), rowspec, null); + if (!generator.hasNext()) { + return Response.status(Response.Status.NOT_FOUND) + .type(MIMETYPE_TEXT).entity("Not found" + CRLF) + .build(); + } + int count = 0; + CellSetModel model = new CellSetModel(); + KeyValue value = generator.next(); + byte[] rowKey = value.getRow(); + RowModel rowModel = new RowModel(rowKey); + do { + if (!Bytes.equals(value.getRow(), rowKey)) { + model.addRow(rowModel); + rowKey = value.getRow(); + rowModel = new RowModel(rowKey); + } + rowModel.addCell(new CellModel(value.getFamily(), value.getQualifier(), + value.getTimestamp(), value.getValue())); + if (++count > rowspec.getMaxValues()) { + break; + } + value = generator.next(); + } while (value != null); + model.addRow(rowModel); + servlet.getMetrics().incrementSucessfulGetRequests(1); + return Response.ok(model).build(); + } catch (RuntimeException e) { + servlet.getMetrics().incrementFailedPutRequests(1); + if (e.getCause() instanceof TableNotFoundException) { + return Response.status(Response.Status.NOT_FOUND) + .type(MIMETYPE_TEXT).entity("Not found" + CRLF) + .build(); + } + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } catch (Exception e) { + servlet.getMetrics().incrementFailedPutRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + } + + @GET + @Produces(MIMETYPE_BINARY) + public Response getBinary(final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("GET " + uriInfo.getAbsolutePath() + " as "+ MIMETYPE_BINARY); + } + servlet.getMetrics().incrementRequests(1); + // doesn't make sense to use a non specific coordinate as this can only + // return a single cell + if (!rowspec.hasColumns() || rowspec.getColumns().length > 1) { + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } + try { + ResultGenerator generator = + ResultGenerator.fromRowSpec(tableResource.getName(), rowspec, null); + if (!generator.hasNext()) { + return Response.status(Response.Status.NOT_FOUND) + .type(MIMETYPE_TEXT).entity("Not found" + CRLF) + .build(); + } + KeyValue value = generator.next(); + ResponseBuilder response = Response.ok(value.getValue()); + response.header("X-Timestamp", value.getTimestamp()); + servlet.getMetrics().incrementSucessfulGetRequests(1); + return response.build(); + } catch (IOException e) { + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + } + + Response update(final CellSetModel model, final boolean replace) { + servlet.getMetrics().incrementRequests(1); + if (servlet.isReadOnly()) { + return Response.status(Response.Status.FORBIDDEN) + .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF) + .build(); + } + + if (CHECK_PUT.equalsIgnoreCase(check)) { + return checkAndPut(model); + } else if (CHECK_DELETE.equalsIgnoreCase(check)) { + return checkAndDelete(model); + } else if (check != null && check.length() > 0) { + LOG.warn("Unknown check value: " + check + ", ignored"); + } + + HTablePool pool = servlet.getTablePool(); + HTableInterface table = null; + try { + List rows = model.getRows(); + List puts = new ArrayList(); + for (RowModel row: rows) { + byte[] key = row.getKey(); + if (key == null) { + key = rowspec.getRow(); + } + if (key == null) { + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } + Put put = new Put(key); + int i = 0; + for (CellModel cell: row.getCells()) { + byte[] col = cell.getColumn(); + if (col == null) try { + col = rowspec.getColumns()[i++]; + } catch (ArrayIndexOutOfBoundsException e) { + col = null; + } + if (col == null) { + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } + byte [][] parts = KeyValue.parseColumn(col); + if (parts.length == 2 && parts[1].length > 0) { + put.add(parts[0], parts[1], cell.getTimestamp(), cell.getValue()); + } else { + put.add(parts[0], null, cell.getTimestamp(), cell.getValue()); + } + } + puts.add(put); + if (LOG.isDebugEnabled()) { + LOG.debug("PUT " + put.toString()); + } + } + table = pool.getTable(tableResource.getName()); + table.put(puts); + table.flushCommits(); + ResponseBuilder response = Response.ok(); + servlet.getMetrics().incrementSucessfulPutRequests(1); + return response.build(); + } catch (IOException e) { + servlet.getMetrics().incrementFailedPutRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } finally { + if (table != null) try { + table.close(); + } catch (IOException ioe) { + LOG.debug("Exception received while closing the table", ioe); + } + } + } + + // This currently supports only update of one row at a time. + Response updateBinary(final byte[] message, final HttpHeaders headers, + final boolean replace) { + servlet.getMetrics().incrementRequests(1); + if (servlet.isReadOnly()) { + return Response.status(Response.Status.FORBIDDEN) + .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF) + .build(); + } + HTablePool pool = servlet.getTablePool(); + HTableInterface table = null; + try { + byte[] row = rowspec.getRow(); + byte[][] columns = rowspec.getColumns(); + byte[] column = null; + if (columns != null) { + column = columns[0]; + } + long timestamp = HConstants.LATEST_TIMESTAMP; + List vals = headers.getRequestHeader("X-Row"); + if (vals != null && !vals.isEmpty()) { + row = Bytes.toBytes(vals.get(0)); + } + vals = headers.getRequestHeader("X-Column"); + if (vals != null && !vals.isEmpty()) { + column = Bytes.toBytes(vals.get(0)); + } + vals = headers.getRequestHeader("X-Timestamp"); + if (vals != null && !vals.isEmpty()) { + timestamp = Long.valueOf(vals.get(0)); + } + if (column == null) { + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } + Put put = new Put(row); + byte parts[][] = KeyValue.parseColumn(column); + if (parts.length == 2 && parts[1].length > 0) { + put.add(parts[0], parts[1], timestamp, message); + } else { + put.add(parts[0], null, timestamp, message); + } + table = pool.getTable(tableResource.getName()); + table.put(put); + if (LOG.isDebugEnabled()) { + LOG.debug("PUT " + put.toString()); + } + servlet.getMetrics().incrementSucessfulPutRequests(1); + return Response.ok().build(); + } catch (IOException e) { + servlet.getMetrics().incrementFailedPutRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } finally { + if (table != null) try { + table.close(); + } catch (IOException ioe) { } + } + } + + @PUT + @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response put(final CellSetModel model, + final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("PUT " + uriInfo.getAbsolutePath() + + " " + uriInfo.getQueryParameters()); + } + return update(model, true); + } + + @PUT + @Consumes(MIMETYPE_BINARY) + public Response putBinary(final byte[] message, + final @Context UriInfo uriInfo, final @Context HttpHeaders headers) { + if (LOG.isDebugEnabled()) { + LOG.debug("PUT " + uriInfo.getAbsolutePath() + " as "+ MIMETYPE_BINARY); + } + return updateBinary(message, headers, true); + } + + @POST + @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response post(final CellSetModel model, + final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("POST " + uriInfo.getAbsolutePath() + + " " + uriInfo.getQueryParameters()); + } + return update(model, false); + } + + @POST + @Consumes(MIMETYPE_BINARY) + public Response postBinary(final byte[] message, + final @Context UriInfo uriInfo, final @Context HttpHeaders headers) { + if (LOG.isDebugEnabled()) { + LOG.debug("POST " + uriInfo.getAbsolutePath() + " as "+MIMETYPE_BINARY); + } + return updateBinary(message, headers, false); + } + + @DELETE + public Response delete(final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("DELETE " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + if (servlet.isReadOnly()) { + return Response.status(Response.Status.FORBIDDEN) + .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF) + .build(); + } + Delete delete = null; + if (rowspec.hasTimestamp()) + delete = new Delete(rowspec.getRow(), rowspec.getTimestamp(), null); + else + delete = new Delete(rowspec.getRow()); + + for (byte[] column: rowspec.getColumns()) { + byte[][] split = KeyValue.parseColumn(column); + if (rowspec.hasTimestamp()) { + if (split.length == 2 && split[1].length != 0) { + delete.deleteColumns(split[0], split[1], rowspec.getTimestamp()); + } else { + delete.deleteFamily(split[0], rowspec.getTimestamp()); + } + } else { + if (split.length == 2 && split[1].length != 0) { + delete.deleteColumns(split[0], split[1]); + } else { + delete.deleteFamily(split[0]); + } + } + } + HTablePool pool = servlet.getTablePool(); + HTableInterface table = null; + try { + table = pool.getTable(tableResource.getName()); + table.delete(delete); + servlet.getMetrics().incrementSucessfulDeleteRequests(1); + if (LOG.isDebugEnabled()) { + LOG.debug("DELETE " + delete.toString()); + } + } catch (IOException e) { + servlet.getMetrics().incrementFailedDeleteRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } finally { + if (table != null) try { + table.close(); + } catch (IOException ioe) { } + } + return Response.ok().build(); + } + + /** + * Validates the input request parameters, parses columns from CellSetModel, + * and invokes checkAndPut on HTable. + * + * @param model instance of CellSetModel + * @return Response 200 OK, 304 Not modified, 400 Bad request + */ + Response checkAndPut(final CellSetModel model) { + HTablePool pool = servlet.getTablePool(); + HTableInterface table = null; + try { + if (model.getRows().size() != 1) { + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } + + RowModel rowModel = model.getRows().get(0); + byte[] key = rowModel.getKey(); + if (key == null) { + key = rowspec.getRow(); + } + + List cellModels = rowModel.getCells(); + int cellModelCount = cellModels.size(); + if (key == null || cellModelCount <= 1) { + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } + + Put put = new Put(key); + CellModel valueToCheckCell = cellModels.get(cellModelCount - 1); + byte[] valueToCheckColumn = valueToCheckCell.getColumn(); + byte[][] valueToPutParts = KeyValue.parseColumn(valueToCheckColumn); + if (valueToPutParts.length == 2 && valueToPutParts[1].length > 0) { + CellModel valueToPutCell = null; + for (int i = 0, n = cellModelCount - 1; i < n ; i++) { + if(Bytes.equals(cellModels.get(i).getColumn(), + valueToCheckCell.getColumn())) { + valueToPutCell = cellModels.get(i); + break; + } + } + if (valueToPutCell != null) { + put.add(valueToPutParts[0], valueToPutParts[1], valueToPutCell + .getTimestamp(), valueToPutCell.getValue()); + } else { + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } + } else { + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } + + table = pool.getTable(this.tableResource.getName()); + boolean retValue = table.checkAndPut(key, valueToPutParts[0], + valueToPutParts[1], valueToCheckCell.getValue(), put); + if (LOG.isDebugEnabled()) { + LOG.debug("CHECK-AND-PUT " + put.toString() + ", returns " + retValue); + } + table.flushCommits(); + ResponseBuilder response = Response.ok(); + if (!retValue) { + response = Response.status(304); + } + return response.build(); + } catch (IOException e) { + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } finally { + if (table != null) try { + table.close(); + } catch (IOException ioe) { } + } + } + + /** + * Validates the input request parameters, parses columns from CellSetModel, + * and invokes checkAndDelete on HTable. + * + * @param model instance of CellSetModel + * @return Response 200 OK, 304 Not modified, 400 Bad request + */ + Response checkAndDelete(final CellSetModel model) { + HTablePool pool = servlet.getTablePool(); + HTableInterface table = null; + Delete delete = null; + try { + if (model.getRows().size() != 1) { + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } + RowModel rowModel = model.getRows().get(0); + byte[] key = rowModel.getKey(); + if (key == null) { + key = rowspec.getRow(); + } + if (key == null) { + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } + + delete = new Delete(key); + CellModel valueToDeleteCell = rowModel.getCells().get(0); + byte[] valueToDeleteColumn = valueToDeleteCell.getColumn(); + if (valueToDeleteColumn == null) { + try { + valueToDeleteColumn = rowspec.getColumns()[0]; + } catch (final ArrayIndexOutOfBoundsException e) { + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } + } + byte[][] parts = KeyValue.parseColumn(valueToDeleteColumn); + if (parts.length == 2 && parts[1].length > 0) { + delete.deleteColumns(parts[0], parts[1]); + } else { + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } + + table = pool.getTable(tableResource.getName()); + boolean retValue = table.checkAndDelete(key, parts[0], parts[1], + valueToDeleteCell.getValue(), delete); + if (LOG.isDebugEnabled()) { + LOG.debug("CHECK-AND-DELETE " + delete.toString() + ", returns " + + retValue); + } + table.flushCommits(); + ResponseBuilder response = Response.ok(); + if (!retValue) { + response = Response.status(304); + } + return response.build(); + } catch (IOException e) { + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } finally { + if (table != null) try { + table.close(); + } catch (IOException ioe) { + LOG.debug("Exception received while closing the table", ioe); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java b/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java new file mode 100644 index 0000000..8d6cfb4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/RowResultGenerator.java @@ -0,0 +1,119 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.HTablePool; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.filter.Filter; + +public class RowResultGenerator extends ResultGenerator { + private static final Log LOG = LogFactory.getLog(RowResultGenerator.class); + + private Iterator valuesI; + private KeyValue cache; + + public RowResultGenerator(final String tableName, final RowSpec rowspec, + final Filter filter) throws IllegalArgumentException, IOException { + HTablePool pool = RESTServlet.getInstance().getTablePool(); + HTableInterface table = pool.getTable(tableName); + try { + Get get = new Get(rowspec.getRow()); + if (rowspec.hasColumns()) { + for (byte[] col: rowspec.getColumns()) { + byte[][] split = KeyValue.parseColumn(col); + if (split.length == 2 && split[1].length != 0) { + get.addColumn(split[0], split[1]); + } else { + get.addFamily(split[0]); + } + } + } + get.setTimeRange(rowspec.getStartTime(), rowspec.getEndTime()); + get.setMaxVersions(rowspec.getMaxVersions()); + if (filter != null) { + get.setFilter(filter); + } + Result result = table.get(get); + if (result != null && !result.isEmpty()) { + valuesI = result.list().iterator(); + } + } catch (DoNotRetryIOException e) { + // Warn here because Stargate will return 404 in the case if multiple + // column families were specified but one did not exist -- currently + // HBase will fail the whole Get. + // Specifying multiple columns in a URI should be uncommon usage but + // help to avoid confusion by leaving a record of what happened here in + // the log. + LOG.warn(StringUtils.stringifyException(e)); + } finally { + table.close(); + } + } + + public void close() { + } + + public boolean hasNext() { + if (cache != null) { + return true; + } + if (valuesI == null) { + return false; + } + return valuesI.hasNext(); + } + + public KeyValue next() { + if (cache != null) { + KeyValue kv = cache; + cache = null; + return kv; + } + if (valuesI == null) { + return null; + } + try { + return valuesI.next(); + } catch (NoSuchElementException e) { + return null; + } + } + + public void putBack(KeyValue kv) { + this.cache = kv; + } + + public void remove() { + throw new UnsupportedOperationException("remove not supported"); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java b/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java new file mode 100644 index 0000000..93d0ed1 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java @@ -0,0 +1,401 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.Collection; +import java.util.TreeSet; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Parses a path based row/column/timestamp specification into its component + * elements. + *

    + * + */ +public class RowSpec { + public static final long DEFAULT_START_TIMESTAMP = 0; + public static final long DEFAULT_END_TIMESTAMP = Long.MAX_VALUE; + + private byte[] row = HConstants.EMPTY_START_ROW; + private byte[] endRow = null; + private TreeSet columns = + new TreeSet(Bytes.BYTES_COMPARATOR); + private long startTime = DEFAULT_START_TIMESTAMP; + private long endTime = DEFAULT_END_TIMESTAMP; + private int maxVersions = HColumnDescriptor.DEFAULT_VERSIONS; + private int maxValues = Integer.MAX_VALUE; + + public RowSpec(String path) throws IllegalArgumentException { + int i = 0; + while (path.charAt(i) == '/') { + i++; + } + i = parseRowKeys(path, i); + i = parseColumns(path, i); + i = parseTimestamp(path, i); + i = parseQueryParams(path, i); + } + + private int parseRowKeys(final String path, int i) + throws IllegalArgumentException { + String startRow = null, endRow = null; + try { + StringBuilder sb = new StringBuilder(); + char c; + while (i < path.length() && (c = path.charAt(i)) != '/') { + sb.append(c); + i++; + } + i++; + String row = startRow = sb.toString(); + int idx = startRow.indexOf(','); + if (idx != -1) { + startRow = URLDecoder.decode(row.substring(0, idx), + HConstants.UTF8_ENCODING); + endRow = URLDecoder.decode(row.substring(idx + 1), + HConstants.UTF8_ENCODING); + } else { + startRow = URLDecoder.decode(row, HConstants.UTF8_ENCODING); + } + } catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException(e); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + // HBase does not support wildcards on row keys so we will emulate a + // suffix glob by synthesizing appropriate start and end row keys for + // table scanning + if (startRow.charAt(startRow.length() - 1) == '*') { + if (endRow != null) + throw new IllegalArgumentException("invalid path: start row "+ + "specified with wildcard"); + this.row = Bytes.toBytes(startRow.substring(0, + startRow.lastIndexOf("*"))); + this.endRow = new byte[this.row.length + 1]; + System.arraycopy(this.row, 0, this.endRow, 0, this.row.length); + this.endRow[this.row.length] = (byte)255; + } else { + this.row = Bytes.toBytes(startRow.toString()); + if (endRow != null) { + this.endRow = Bytes.toBytes(endRow.toString()); + } + } + return i; + } + + private int parseColumns(final String path, int i) + throws IllegalArgumentException { + if (i >= path.length()) { + return i; + } + try { + char c; + StringBuilder column = new StringBuilder(); + while (i < path.length() && (c = path.charAt(i)) != '/') { + if (c == ',') { + if (column.length() < 1) { + throw new IllegalArgumentException("invalid path"); + } + String s = URLDecoder.decode(column.toString(), + HConstants.UTF8_ENCODING); + if (!s.contains(":")) { + this.columns.add(Bytes.toBytes(s + ":")); + } else { + this.columns.add(Bytes.toBytes(s)); + } + column.setLength(0); + i++; + continue; + } + column.append(c); + i++; + } + i++; + // trailing list entry + if (column.length() > 1) { + String s = URLDecoder.decode(column.toString(), + HConstants.UTF8_ENCODING); + if (!s.contains(":")) { + this.columns.add(Bytes.toBytes(s + ":")); + } else { + this.columns.add(Bytes.toBytes(s)); + } + } + } catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException(e); + } catch (UnsupportedEncodingException e) { + // shouldn't happen + throw new RuntimeException(e); + } + return i; + } + + private int parseTimestamp(final String path, int i) + throws IllegalArgumentException { + if (i >= path.length()) { + return i; + } + long time0 = 0, time1 = 0; + try { + char c = 0; + StringBuilder stamp = new StringBuilder(); + while (i < path.length()) { + c = path.charAt(i); + if (c == '/' || c == ',') { + break; + } + stamp.append(c); + i++; + } + try { + time0 = Long.valueOf(URLDecoder.decode(stamp.toString(), + HConstants.UTF8_ENCODING)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(e); + } + if (c == ',') { + stamp = new StringBuilder(); + i++; + while (i < path.length() && ((c = path.charAt(i)) != '/')) { + stamp.append(c); + i++; + } + try { + time1 = Long.valueOf(URLDecoder.decode(stamp.toString(), + HConstants.UTF8_ENCODING)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(e); + } + } + if (c == '/') { + i++; + } + } catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException(e); + } catch (UnsupportedEncodingException e) { + // shouldn't happen + throw new RuntimeException(e); + } + if (time1 != 0) { + startTime = time0; + endTime = time1; + } else { + endTime = time0; + } + return i; + } + + private int parseQueryParams(final String path, int i) { + if (i >= path.length()) { + return i; + } + StringBuilder query = new StringBuilder(); + try { + query.append(URLDecoder.decode(path.substring(i), + HConstants.UTF8_ENCODING)); + } catch (UnsupportedEncodingException e) { + // should not happen + throw new RuntimeException(e); + } + i += query.length(); + int j = 0; + while (j < query.length()) { + char c = query.charAt(j); + if (c != '?' && c != '&') { + break; + } + if (++j > query.length()) { + throw new IllegalArgumentException("malformed query parameter"); + } + char what = query.charAt(j); + if (++j > query.length()) { + break; + } + c = query.charAt(j); + if (c != '=') { + throw new IllegalArgumentException("malformed query parameter"); + } + if (++j > query.length()) { + break; + } + switch (what) { + case 'm': { + StringBuilder sb = new StringBuilder(); + while (j <= query.length()) { + c = query.charAt(i); + if (c < '0' || c > '9') { + j--; + break; + } + sb.append(c); + } + maxVersions = Integer.valueOf(sb.toString()); + } break; + case 'n': { + StringBuilder sb = new StringBuilder(); + while (j <= query.length()) { + c = query.charAt(i); + if (c < '0' || c > '9') { + j--; + break; + } + sb.append(c); + } + maxValues = Integer.valueOf(sb.toString()); + } break; + default: + throw new IllegalArgumentException("unknown parameter '" + c + "'"); + } + } + return i; + } + + public RowSpec(byte[] startRow, byte[] endRow, byte[][] columns, + long startTime, long endTime, int maxVersions) { + this.row = startRow; + this.endRow = endRow; + if (columns != null) { + for (byte[] col: columns) { + this.columns.add(col); + } + } + this.startTime = startTime; + this.endTime = endTime; + this.maxVersions = maxVersions; + } + + public RowSpec(byte[] startRow, byte[] endRow, Collection columns, + long startTime, long endTime, int maxVersions) { + this.row = startRow; + this.endRow = endRow; + if (columns != null) { + this.columns.addAll(columns); + } + this.startTime = startTime; + this.endTime = endTime; + this.maxVersions = maxVersions; + } + + public boolean isSingleRow() { + return endRow == null; + } + + public int getMaxVersions() { + return maxVersions; + } + + public void setMaxVersions(final int maxVersions) { + this.maxVersions = maxVersions; + } + + public int getMaxValues() { + return maxValues; + } + + public void setMaxValues(final int maxValues) { + this.maxValues = maxValues; + } + + public boolean hasColumns() { + return !columns.isEmpty(); + } + + public byte[] getRow() { + return row; + } + + public byte[] getStartRow() { + return row; + } + + public boolean hasEndRow() { + return endRow != null; + } + + public byte[] getEndRow() { + return endRow; + } + + public void addColumn(final byte[] column) { + columns.add(column); + } + + public byte[][] getColumns() { + return columns.toArray(new byte[columns.size()][]); + } + + public boolean hasTimestamp() { + return (startTime == 0) && (endTime != Long.MAX_VALUE); + } + + public long getTimestamp() { + return endTime; + } + + public long getStartTime() { + return startTime; + } + + public void setStartTime(final long startTime) { + this.startTime = startTime; + } + + public long getEndTime() { + return endTime; + } + + public void setEndTime(long endTime) { + this.endTime = endTime; + } + + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("{startRow => '"); + if (row != null) { + result.append(Bytes.toString(row)); + } + result.append("', endRow => '"); + if (endRow != null) { + result.append(Bytes.toString(endRow)); + } + result.append("', columns => ["); + for (byte[] col: columns) { + result.append(" '"); + result.append(Bytes.toString(col)); + result.append("'"); + } + result.append(" ], startTime => "); + result.append(Long.toString(startTime)); + result.append(", endTime => "); + result.append(Long.toString(endTime)); + result.append(", maxVersions => "); + result.append(Integer.toString(maxVersions)); + result.append(", maxValues => "); + result.append(Integer.toString(maxValues)); + result.append("}"); + return result.toString(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java new file mode 100644 index 0000000..7f58e96 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerInstanceResource.java @@ -0,0 +1,204 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; + +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.CacheControl; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.ResponseBuilder; +import javax.ws.rs.core.UriInfo; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.rest.model.CellModel; +import org.apache.hadoop.hbase.rest.model.CellSetModel; +import org.apache.hadoop.hbase.rest.model.RowModel; +import org.apache.hadoop.hbase.util.Base64; +import org.apache.hadoop.hbase.util.Bytes; + +public class ScannerInstanceResource extends ResourceBase { + private static final Log LOG = + LogFactory.getLog(ScannerInstanceResource.class); + + static CacheControl cacheControl; + static { + cacheControl = new CacheControl(); + cacheControl.setNoCache(true); + cacheControl.setNoTransform(false); + } + + ResultGenerator generator = null; + String id = null; + int batch = 1; + + public ScannerInstanceResource() throws IOException { } + + public ScannerInstanceResource(String table, String id, + ResultGenerator generator, int batch) throws IOException { + this.id = id; + this.generator = generator; + this.batch = batch; + } + + @GET + @Produces({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response get(final @Context UriInfo uriInfo, + @QueryParam("n") int maxRows, final @QueryParam("c") int maxValues) { + if (LOG.isDebugEnabled()) { + LOG.debug("GET " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + if (generator == null) { + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.NOT_FOUND) + .type(MIMETYPE_TEXT).entity("Not found" + CRLF) + .build(); + } + CellSetModel model = new CellSetModel(); + RowModel rowModel = null; + byte[] rowKey = null; + int limit = batch; + if (maxValues > 0) { + limit = maxValues; + } + int count = limit; + do { + KeyValue value = null; + try { + value = generator.next(); + } catch (IllegalStateException e) { + if (ScannerResource.delete(id)) { + servlet.getMetrics().incrementSucessfulDeleteRequests(1); + } else { + servlet.getMetrics().incrementFailedDeleteRequests(1); + } + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.GONE) + .type(MIMETYPE_TEXT).entity("Gone" + CRLF) + .build(); + } + if (value == null) { + LOG.info("generator exhausted"); + // respond with 204 (No Content) if an empty cell set would be + // returned + if (count == limit) { + return Response.noContent().build(); + } + break; + } + if (rowKey == null) { + rowKey = value.getRow(); + rowModel = new RowModel(rowKey); + } + if (!Bytes.equals(value.getRow(), rowKey)) { + // if maxRows was given as a query param, stop if we would exceed the + // specified number of rows + if (maxRows > 0) { + if (--maxRows == 0) { + generator.putBack(value); + break; + } + } + model.addRow(rowModel); + rowKey = value.getRow(); + rowModel = new RowModel(rowKey); + } + rowModel.addCell( + new CellModel(value.getFamily(), value.getQualifier(), + value.getTimestamp(), value.getValue())); + } while (--count > 0); + model.addRow(rowModel); + ResponseBuilder response = Response.ok(model); + response.cacheControl(cacheControl); + servlet.getMetrics().incrementSucessfulGetRequests(1); + return response.build(); + } + + @GET + @Produces(MIMETYPE_BINARY) + public Response getBinary(final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("GET " + uriInfo.getAbsolutePath() + " as " + + MIMETYPE_BINARY); + } + servlet.getMetrics().incrementRequests(1); + if (generator == null) { + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.NOT_FOUND) + .type(MIMETYPE_TEXT).entity("Not found" + CRLF) + .build(); + } + try { + KeyValue value = generator.next(); + if (value == null) { + LOG.info("generator exhausted"); + return Response.noContent().build(); + } + ResponseBuilder response = Response.ok(value.getValue()); + response.cacheControl(cacheControl); + response.header("X-Row", Base64.encodeBytes(value.getRow())); + response.header("X-Column", + Base64.encodeBytes( + KeyValue.makeColumn(value.getFamily(), value.getQualifier()))); + response.header("X-Timestamp", value.getTimestamp()); + servlet.getMetrics().incrementSucessfulGetRequests(1); + return response.build(); + } catch (IllegalStateException e) { + if (ScannerResource.delete(id)) { + servlet.getMetrics().incrementSucessfulDeleteRequests(1); + } else { + servlet.getMetrics().incrementFailedDeleteRequests(1); + } + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.GONE) + .type(MIMETYPE_TEXT).entity("Gone" + CRLF) + .build(); + } + } + + @DELETE + public Response delete(final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("DELETE " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + if (servlet.isReadOnly()) { + return Response.status(Response.Status.FORBIDDEN) + .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF) + .build(); + } + if (ScannerResource.delete(id)) { + servlet.getMetrics().incrementSucessfulDeleteRequests(1); + } else { + servlet.getMetrics().incrementFailedDeleteRequests(1); + } + return Response.ok().build(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java new file mode 100644 index 0000000..8ebda4f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java @@ -0,0 +1,155 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriBuilder; +import javax.ws.rs.core.UriInfo; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.rest.model.ScannerModel; + +public class ScannerResource extends ResourceBase { + + private static final Log LOG = LogFactory.getLog(ScannerResource.class); + + static final Map scanners = + Collections.synchronizedMap(new HashMap()); + + TableResource tableResource; + + /** + * Constructor + * @param tableResource + * @throws IOException + */ + public ScannerResource(TableResource tableResource)throws IOException { + super(); + this.tableResource = tableResource; + } + + static boolean delete(final String id) { + ScannerInstanceResource instance = scanners.remove(id); + if (instance != null) { + instance.generator.close(); + return true; + } else { + return false; + } + } + + Response update(final ScannerModel model, final boolean replace, + final UriInfo uriInfo) { + servlet.getMetrics().incrementRequests(1); + if (servlet.isReadOnly()) { + return Response.status(Response.Status.FORBIDDEN) + .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF) + .build(); + } + byte[] endRow = model.hasEndRow() ? model.getEndRow() : null; + RowSpec spec = new RowSpec(model.getStartRow(), endRow, + model.getColumns(), model.getStartTime(), model.getEndTime(), + model.getMaxVersions()); + try { + Filter filter = ScannerResultGenerator.buildFilterFromModel(model); + String tableName = tableResource.getName(); + ScannerResultGenerator gen = + new ScannerResultGenerator(tableName, spec, filter); + String id = gen.getID(); + ScannerInstanceResource instance = + new ScannerInstanceResource(tableName, id, gen, model.getBatch()); + scanners.put(id, instance); + if (LOG.isDebugEnabled()) { + LOG.debug("new scanner: " + id); + } + UriBuilder builder = uriInfo.getAbsolutePathBuilder(); + URI uri = builder.path(id).build(); + servlet.getMetrics().incrementSucessfulPutRequests(1); + return Response.created(uri).build(); + } catch (RuntimeException e) { + servlet.getMetrics().incrementFailedPutRequests(1); + if (e.getCause() instanceof TableNotFoundException) { + return Response.status(Response.Status.NOT_FOUND) + .type(MIMETYPE_TEXT).entity("Not found" + CRLF) + .build(); + } + return Response.status(Response.Status.BAD_REQUEST) + .type(MIMETYPE_TEXT).entity("Bad request" + CRLF) + .build(); + } catch (Exception e) { + servlet.getMetrics().incrementFailedPutRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + } + + @PUT + @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response put(final ScannerModel model, + final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("PUT " + uriInfo.getAbsolutePath()); + } + return update(model, true, uriInfo); + } + + @POST + @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response post(final ScannerModel model, + final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("POST " + uriInfo.getAbsolutePath()); + } + return update(model, false, uriInfo); + } + + @Path("{scanner: .+}") + public ScannerInstanceResource getScannerInstanceResource( + final @PathParam("scanner") String id) throws IOException { + ScannerInstanceResource instance = scanners.get(id); + if (instance == null) { + servlet.getMetrics().incrementFailedGetRequests(1); + return new ScannerInstanceResource(); + } else { + servlet.getMetrics().incrementSucessfulGetRequests(1); + } + return instance; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java new file mode 100644 index 0000000..e068c2e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java @@ -0,0 +1,172 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; +import java.util.Iterator; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.UnknownScannerException; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.HTablePool; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.rest.model.ScannerModel; +import org.apache.hadoop.util.StringUtils; + +public class ScannerResultGenerator extends ResultGenerator { + + private static final Log LOG = + LogFactory.getLog(ScannerResultGenerator.class); + + public static Filter buildFilterFromModel(final ScannerModel model) + throws Exception { + String filter = model.getFilter(); + if (filter == null || filter.length() == 0) { + return null; + } + return buildFilter(filter); + } + + private String id; + private Iterator rowI; + private KeyValue cache; + private ResultScanner scanner; + private Result cached; + + public ScannerResultGenerator(final String tableName, final RowSpec rowspec, + final Filter filter) throws IllegalArgumentException, IOException { + HTablePool pool = RESTServlet.getInstance().getTablePool(); + HTableInterface table = pool.getTable(tableName); + try { + Scan scan; + if (rowspec.hasEndRow()) { + scan = new Scan(rowspec.getStartRow(), rowspec.getEndRow()); + } else { + scan = new Scan(rowspec.getStartRow()); + } + if (rowspec.hasColumns()) { + byte[][] columns = rowspec.getColumns(); + for (byte[] column: columns) { + byte[][] split = KeyValue.parseColumn(column); + if (split.length > 1 && (split[1] != null && split[1].length != 0)) { + scan.addColumn(split[0], split[1]); + } else { + scan.addFamily(split[0]); + } + } + } + scan.setTimeRange(rowspec.getStartTime(), rowspec.getEndTime()); + scan.setMaxVersions(rowspec.getMaxVersions()); + if (filter != null) { + scan.setFilter(filter); + } + // always disable block caching on the cluster when scanning + scan.setCacheBlocks(false); + scanner = table.getScanner(scan); + cached = null; + id = Long.toString(System.currentTimeMillis()) + + Integer.toHexString(scanner.hashCode()); + } finally { + table.close(); + } + } + + public String getID() { + return id; + } + + public void close() { + } + + public boolean hasNext() { + if (cache != null) { + return true; + } + if (rowI != null && rowI.hasNext()) { + return true; + } + if (cached != null) { + return true; + } + try { + Result result = scanner.next(); + if (result != null && !result.isEmpty()) { + cached = result; + } + } catch (UnknownScannerException e) { + throw new IllegalArgumentException(e); + } catch (IOException e) { + LOG.error(StringUtils.stringifyException(e)); + } + return cached != null; + } + + public KeyValue next() { + if (cache != null) { + KeyValue kv = cache; + cache = null; + return kv; + } + boolean loop; + do { + loop = false; + if (rowI != null) { + if (rowI.hasNext()) { + return rowI.next(); + } else { + rowI = null; + } + } + if (cached != null) { + rowI = cached.list().iterator(); + loop = true; + cached = null; + } else { + Result result = null; + try { + result = scanner.next(); + } catch (UnknownScannerException e) { + throw new IllegalArgumentException(e); + } catch (IOException e) { + LOG.error(StringUtils.stringifyException(e)); + } + if (result != null && !result.isEmpty()) { + rowI = result.list().iterator(); + loop = true; + } + } + } while (loop); + return null; + } + + public void putBack(KeyValue kv) { + this.cache = kv; + } + + public void remove() { + throw new UnsupportedOperationException("remove not supported"); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java new file mode 100644 index 0000000..864b2a6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/SchemaResource.java @@ -0,0 +1,268 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; +import java.util.Map; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Produces; +import javax.ws.rs.core.CacheControl; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import javax.ws.rs.core.Response.ResponseBuilder; + +import javax.xml.namespace.QName; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.HTablePool; +import org.apache.hadoop.hbase.rest.model.ColumnSchemaModel; +import org.apache.hadoop.hbase.rest.model.TableSchemaModel; +import org.apache.hadoop.hbase.util.Bytes; + +public class SchemaResource extends ResourceBase { + private static final Log LOG = LogFactory.getLog(SchemaResource.class); + + static CacheControl cacheControl; + static { + cacheControl = new CacheControl(); + cacheControl.setNoCache(true); + cacheControl.setNoTransform(false); + } + + TableResource tableResource; + + /** + * Constructor + * @param tableResource + * @throws IOException + */ + public SchemaResource(TableResource tableResource) throws IOException { + super(); + this.tableResource = tableResource; + } + + private HTableDescriptor getTableSchema() throws IOException, + TableNotFoundException { + HTablePool pool = servlet.getTablePool(); + HTableInterface table = pool.getTable(tableResource.getName()); + try { + return table.getTableDescriptor(); + } finally { + table.close(); + } + } + + @GET + @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response get(final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("GET " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + try { + ResponseBuilder response = + Response.ok(new TableSchemaModel(getTableSchema())); + response.cacheControl(cacheControl); + servlet.getMetrics().incrementSucessfulGetRequests(1); + return response.build(); + } catch (TableNotFoundException e) { + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.NOT_FOUND) + .type(MIMETYPE_TEXT).entity("Not found" + CRLF) + .build(); + } catch (IOException e) { + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + } + + private Response replace(final byte[] name, final TableSchemaModel model, + final UriInfo uriInfo, final HBaseAdmin admin) { + if (servlet.isReadOnly()) { + return Response.status(Response.Status.FORBIDDEN) + .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF) + .build(); + } + try { + HTableDescriptor htd = new HTableDescriptor(name); + for (Map.Entry e: model.getAny().entrySet()) { + htd.setValue(e.getKey().getLocalPart(), e.getValue().toString()); + } + for (ColumnSchemaModel family: model.getColumns()) { + HColumnDescriptor hcd = new HColumnDescriptor(family.getName()); + for (Map.Entry e: family.getAny().entrySet()) { + hcd.setValue(e.getKey().getLocalPart(), e.getValue().toString()); + } + htd.addFamily(hcd); + } + if (admin.tableExists(name)) { + admin.disableTable(name); + admin.modifyTable(name, htd); + admin.enableTable(name); + servlet.getMetrics().incrementSucessfulPutRequests(1); + } else try { + admin.createTable(htd); + servlet.getMetrics().incrementSucessfulPutRequests(1); + } catch (TableExistsException e) { + // race, someone else created a table with the same name + return Response.status(Response.Status.NOT_MODIFIED) + .type(MIMETYPE_TEXT).entity("Not modified" + CRLF) + .build(); + } + return Response.created(uriInfo.getAbsolutePath()).build(); + } catch (IOException e) { + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + } + + private Response update(final byte[] name, final TableSchemaModel model, + final UriInfo uriInfo, final HBaseAdmin admin) { + if (servlet.isReadOnly()) { + return Response.status(Response.Status.FORBIDDEN) + .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF) + .build(); + } + try { + HTableDescriptor htd = admin.getTableDescriptor(name); + admin.disableTable(name); + try { + for (ColumnSchemaModel family: model.getColumns()) { + HColumnDescriptor hcd = new HColumnDescriptor(family.getName()); + for (Map.Entry e: family.getAny().entrySet()) { + hcd.setValue(e.getKey().getLocalPart(), e.getValue().toString()); + } + if (htd.hasFamily(hcd.getName())) { + admin.modifyColumn(name, hcd); + } else { + admin.addColumn(name, hcd); + } + } + } catch (IOException e) { + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } finally { + admin.enableTable(tableResource.getName()); + } + servlet.getMetrics().incrementSucessfulPutRequests(1); + return Response.ok().build(); + } catch (IOException e) { + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + } + + private Response update(final TableSchemaModel model, final boolean replace, + final UriInfo uriInfo) { + try { + byte[] name = Bytes.toBytes(tableResource.getName()); + HBaseAdmin admin = servlet.getAdmin(); + if (replace || !admin.tableExists(name)) { + return replace(name, model, uriInfo, admin); + } else { + return update(name, model, uriInfo, admin); + } + } catch (IOException e) { + servlet.getMetrics().incrementFailedPutRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + } + + @PUT + @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response put(final TableSchemaModel model, + final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("PUT " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + return update(model, true, uriInfo); + } + + @POST + @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response post(final TableSchemaModel model, + final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("PUT " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + return update(model, false, uriInfo); + } + + @DELETE + public Response delete(final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("DELETE " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + try { + HBaseAdmin admin = servlet.getAdmin(); + boolean success = false; + for (int i = 0; i < 10; i++) try { + admin.disableTable(tableResource.getName()); + success = true; + break; + } catch (IOException e) { + } + if (!success) { + throw new IOException("could not disable table"); + } + admin.deleteTable(tableResource.getName()); + servlet.getMetrics().incrementSucessfulDeleteRequests(1); + return Response.ok().build(); + } catch (TableNotFoundException e) { + servlet.getMetrics().incrementFailedDeleteRequests(1); + return Response.status(Response.Status.NOT_FOUND) + .type(MIMETYPE_TEXT).entity("Not found" + CRLF) + .build(); + } catch (IOException e) { + servlet.getMetrics().incrementFailedDeleteRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java new file mode 100644 index 0000000..16af87b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java @@ -0,0 +1,107 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; + +import javax.ws.rs.GET; +import javax.ws.rs.Produces; +import javax.ws.rs.core.CacheControl; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.ResponseBuilder; +import javax.ws.rs.core.UriInfo; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HServerLoad; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.rest.model.StorageClusterStatusModel; + +public class StorageClusterStatusResource extends ResourceBase { + private static final Log LOG = + LogFactory.getLog(StorageClusterStatusResource.class); + + static CacheControl cacheControl; + static { + cacheControl = new CacheControl(); + cacheControl.setNoCache(true); + cacheControl.setNoTransform(false); + } + + /** + * Constructor + * @throws IOException + */ + public StorageClusterStatusResource() throws IOException { + super(); + } + + @GET + @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response get(final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("GET " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + try { + ClusterStatus status = servlet.getAdmin().getClusterStatus(); + StorageClusterStatusModel model = new StorageClusterStatusModel(); + model.setRegions(status.getRegionsCount()); + model.setRequests(status.getRequestsCount()); + model.setAverageLoad(status.getAverageLoad()); + for (ServerName info: status.getServers()) { + HServerLoad load = status.getLoad(info); + StorageClusterStatusModel.Node node = + model.addLiveNode( + info.getHostname() + ":" + + Integer.toString(info.getPort()), + info.getStartcode(), load.getUsedHeapMB(), + load.getMaxHeapMB()); + node.setRequests(load.getNumberOfRequests()); + for (HServerLoad.RegionLoad region: load.getRegionsLoad().values()) { + node.addRegion(region.getName(), region.getStores(), + region.getStorefiles(), region.getStorefileSizeMB(), + region.getMemStoreSizeMB(), region.getStorefileIndexSizeMB(), + region.getReadRequestsCount(), region.getWriteRequestsCount(), + region.getRootIndexSizeKB(), region.getTotalStaticIndexSizeKB(), + region.getTotalStaticBloomSizeKB(), region.getTotalCompactingKVs(), + region.getCurrentCompactedKVs()); + } + } + for (ServerName name: status.getDeadServerNames()) { + model.addDeadNode(name.toString()); + } + ResponseBuilder response = Response.ok(model); + response.cacheControl(cacheControl); + servlet.getMetrics().incrementSucessfulGetRequests(1); + return response.build(); + } catch (IOException e) { + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java new file mode 100644 index 0000000..0ea3004 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterVersionResource.java @@ -0,0 +1,78 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; + +import javax.ws.rs.GET; +import javax.ws.rs.Produces; +import javax.ws.rs.core.CacheControl; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import javax.ws.rs.core.Response.ResponseBuilder; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hbase.rest.model.StorageClusterVersionModel; + +public class StorageClusterVersionResource extends ResourceBase { + private static final Log LOG = + LogFactory.getLog(StorageClusterVersionResource.class); + + static CacheControl cacheControl; + static { + cacheControl = new CacheControl(); + cacheControl.setNoCache(true); + cacheControl.setNoTransform(false); + } + + /** + * Constructor + * @throws IOException + */ + public StorageClusterVersionResource() throws IOException { + super(); + } + + @GET + @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON}) + public Response get(final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("GET " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + try { + StorageClusterVersionModel model = new StorageClusterVersionModel(); + model.setVersion(servlet.getAdmin().getClusterStatus().getHBaseVersion()); + ResponseBuilder response = Response.ok(model); + response.cacheControl(cacheControl); + servlet.getMetrics().incrementSucessfulGetRequests(1); + return response.build(); + } catch (IOException e) { + servlet.getMetrics().incrementFailedGetRequests(1); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF) + .build(); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java b/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java new file mode 100644 index 0000000..88cc68f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java @@ -0,0 +1,92 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; + +import javax.ws.rs.Encoded; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.QueryParam; + +public class TableResource extends ResourceBase { + + String table; + + /** + * Constructor + * @param table + * @throws IOException + */ + public TableResource(String table) throws IOException { + super(); + this.table = table; + } + + /** @return the table name */ + String getName() { + return table; + } + + /** + * @return true if the table exists + * @throws IOException + */ + boolean exists() throws IOException { + return servlet.getAdmin().tableExists(table); + } + + @Path("exists") + public ExistsResource getExistsResource() throws IOException { + return new ExistsResource(this); + } + + @Path("regions") + public RegionsResource getRegionsResource() throws IOException { + return new RegionsResource(this); + } + + @Path("scanner") + public ScannerResource getScannerResource() throws IOException { + return new ScannerResource(this); + } + + @Path("schema") + public SchemaResource getSchemaResource() throws IOException { + return new SchemaResource(this); + } + + @Path("multiget") + public MultiRowResource getMultipleRowResource( + final @QueryParam("v") String versions) throws IOException { + return new MultiRowResource(this, versions); + } + + @Path("{rowspec: .+}") + public RowResource getRowResource( + // We need the @Encoded decorator so Jersey won't urldecode before + // the RowSpec constructor has a chance to parse + final @PathParam("rowspec") @Encoded String rowspec, + final @QueryParam("v") String versions, + final @QueryParam("check") String check) throws IOException { + return new RowResource(this, rowspec, versions, check); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java b/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java new file mode 100644 index 0000000..cdb55dc --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/VersionResource.java @@ -0,0 +1,103 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; + +import javax.servlet.ServletContext; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.CacheControl; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import javax.ws.rs.core.Response.ResponseBuilder; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hbase.rest.model.VersionModel; + +/** + * Implements REST software version reporting + *

    + * /version/rest + *

    + * /version (alias for /version/rest) + */ +public class VersionResource extends ResourceBase { + + private static final Log LOG = LogFactory.getLog(VersionResource.class); + + static CacheControl cacheControl; + static { + cacheControl = new CacheControl(); + cacheControl.setNoCache(true); + cacheControl.setNoTransform(false); + } + + /** + * Constructor + * @throws IOException + */ + public VersionResource() throws IOException { + super(); + } + + /** + * Build a response for a version request. + * @param context servlet context + * @param uriInfo (JAX-RS context variable) request URL + * @return a response for a version request + */ + @GET + @Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + MIMETYPE_PROTOBUF_IETF}) + public Response get(final @Context ServletContext context, + final @Context UriInfo uriInfo) { + if (LOG.isDebugEnabled()) { + LOG.debug("GET " + uriInfo.getAbsolutePath()); + } + servlet.getMetrics().incrementRequests(1); + ResponseBuilder response = Response.ok(new VersionModel(context)); + response.cacheControl(cacheControl); + servlet.getMetrics().incrementSucessfulGetRequests(1); + return response.build(); + } + + /** + * Dispatch to StorageClusterVersionResource + */ + @Path("cluster") + public StorageClusterVersionResource getClusterVersionResource() + throws IOException { + return new StorageClusterVersionResource(); + } + + /** + * Dispatch /version/rest to self. + */ + @Path("rest") + public VersionResource getVersionResource() { + return this; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/Client.java b/src/main/java/org/apache/hadoop/hbase/rest/client/Client.java new file mode 100644 index 0000000..1e0dbbe --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/client/Client.java @@ -0,0 +1,501 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.client; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.HttpVersion; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; +import org.apache.commons.httpclient.URI; +import org.apache.commons.httpclient.methods.ByteArrayRequestEntity; +import org.apache.commons.httpclient.methods.DeleteMethod; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.methods.HeadMethod; +import org.apache.commons.httpclient.methods.PostMethod; +import org.apache.commons.httpclient.methods.PutMethod; +import org.apache.commons.httpclient.params.HttpClientParams; +import org.apache.commons.httpclient.params.HttpConnectionManagerParams; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * A wrapper around HttpClient which provides some useful function and + * semantics for interacting with the REST gateway. + */ +public class Client { + public static final Header[] EMPTY_HEADER_ARRAY = new Header[0]; + + private static final Log LOG = LogFactory.getLog(Client.class); + + private HttpClient httpClient; + private Cluster cluster; + + private Map extraHeaders; + + /** + * Default Constructor + */ + public Client() { + this(null); + } + + /** + * Constructor + * @param cluster the cluster definition + */ + public Client(Cluster cluster) { + this.cluster = cluster; + MultiThreadedHttpConnectionManager manager = + new MultiThreadedHttpConnectionManager(); + HttpConnectionManagerParams managerParams = manager.getParams(); + managerParams.setConnectionTimeout(2000); // 2 s + managerParams.setDefaultMaxConnectionsPerHost(10); + managerParams.setMaxTotalConnections(100); + extraHeaders = new ConcurrentHashMap(); + this.httpClient = new HttpClient(manager); + HttpClientParams clientParams = httpClient.getParams(); + clientParams.setVersion(HttpVersion.HTTP_1_1); + } + + /** + * Shut down the client. Close any open persistent connections. + */ + public void shutdown() { + MultiThreadedHttpConnectionManager manager = + (MultiThreadedHttpConnectionManager) httpClient.getHttpConnectionManager(); + manager.shutdown(); + } + + /** + * @return the wrapped HttpClient + */ + public HttpClient getHttpClient() { + return httpClient; + } + + /** + * Add extra headers. These extra headers will be applied to all http + * methods before they are removed. If any header is not used any more, + * client needs to remove it explicitly. + */ + public void addExtraHeader(final String name, final String value) { + extraHeaders.put(name, value); + } + + /** + * Get an extra header value. + */ + public String getExtraHeader(final String name) { + return extraHeaders.get(name); + } + + /** + * Get all extra headers (read-only). + */ + public Map getExtraHeaders() { + return Collections.unmodifiableMap(extraHeaders); + } + + /** + * Remove an extra header. + */ + public void removeExtraHeader(final String name) { + extraHeaders.remove(name); + } + + /** + * Execute a transaction method given only the path. Will select at random + * one of the members of the supplied cluster definition and iterate through + * the list until a transaction can be successfully completed. The + * definition of success here is a complete HTTP transaction, irrespective + * of result code. + * @param cluster the cluster definition + * @param method the transaction method + * @param headers HTTP header values to send + * @param path the properly urlencoded path + * @return the HTTP response code + * @throws IOException + */ + public int executePathOnly(Cluster cluster, HttpMethod method, + Header[] headers, String path) throws IOException { + IOException lastException; + if (cluster.nodes.size() < 1) { + throw new IOException("Cluster is empty"); + } + int start = (int)Math.round((cluster.nodes.size() - 1) * Math.random()); + int i = start; + do { + cluster.lastHost = cluster.nodes.get(i); + try { + StringBuilder sb = new StringBuilder(); + sb.append("http://"); + sb.append(cluster.lastHost); + sb.append(path); + URI uri = new URI(sb.toString(), true); + return executeURI(method, headers, uri.toString()); + } catch (IOException e) { + lastException = e; + } + } while (++i != start && i < cluster.nodes.size()); + throw lastException; + } + + /** + * Execute a transaction method given a complete URI. + * @param method the transaction method + * @param headers HTTP header values to send + * @param uri a properly urlencoded URI + * @return the HTTP response code + * @throws IOException + */ + public int executeURI(HttpMethod method, Header[] headers, String uri) + throws IOException { + method.setURI(new URI(uri, true)); + for (Map.Entry e: extraHeaders.entrySet()) { + method.addRequestHeader(e.getKey(), e.getValue()); + } + if (headers != null) { + for (Header header: headers) { + method.addRequestHeader(header); + } + } + long startTime = System.currentTimeMillis(); + int code = httpClient.executeMethod(method); + long endTime = System.currentTimeMillis(); + if (LOG.isDebugEnabled()) { + LOG.debug(method.getName() + " " + uri + " " + code + " " + + method.getStatusText() + " in " + (endTime - startTime) + " ms"); + } + return code; + } + + /** + * Execute a transaction method. Will call either executePathOnly + * or executeURI depending on whether a path only is supplied in + * 'path', or if a complete URI is passed instead, respectively. + * @param cluster the cluster definition + * @param method the HTTP method + * @param headers HTTP header values to send + * @param path the properly urlencoded path or URI + * @return the HTTP response code + * @throws IOException + */ + public int execute(Cluster cluster, HttpMethod method, Header[] headers, + String path) throws IOException { + if (path.startsWith("/")) { + return executePathOnly(cluster, method, headers, path); + } + return executeURI(method, headers, path); + } + + /** + * @return the cluster definition + */ + public Cluster getCluster() { + return cluster; + } + + /** + * @param cluster the cluster definition + */ + public void setCluster(Cluster cluster) { + this.cluster = cluster; + } + + /** + * Send a HEAD request + * @param path the path or URI + * @return a Response object with response detail + * @throws IOException + */ + public Response head(String path) throws IOException { + return head(cluster, path, null); + } + + /** + * Send a HEAD request + * @param cluster the cluster definition + * @param path the path or URI + * @param headers the HTTP headers to include in the request + * @return a Response object with response detail + * @throws IOException + */ + public Response head(Cluster cluster, String path, Header[] headers) + throws IOException { + HeadMethod method = new HeadMethod(); + try { + int code = execute(cluster, method, null, path); + headers = method.getResponseHeaders(); + return new Response(code, headers, null); + } finally { + method.releaseConnection(); + } + } + + /** + * Send a GET request + * @param path the path or URI + * @return a Response object with response detail + * @throws IOException + */ + public Response get(String path) throws IOException { + return get(cluster, path); + } + + /** + * Send a GET request + * @param cluster the cluster definition + * @param path the path or URI + * @return a Response object with response detail + * @throws IOException + */ + public Response get(Cluster cluster, String path) throws IOException { + return get(cluster, path, EMPTY_HEADER_ARRAY); + } + + /** + * Send a GET request + * @param path the path or URI + * @param accept Accept header value + * @return a Response object with response detail + * @throws IOException + */ + public Response get(String path, String accept) throws IOException { + return get(cluster, path, accept); + } + + /** + * Send a GET request + * @param cluster the cluster definition + * @param path the path or URI + * @param accept Accept header value + * @return a Response object with response detail + * @throws IOException + */ + public Response get(Cluster cluster, String path, String accept) + throws IOException { + Header[] headers = new Header[1]; + headers[0] = new Header("Accept", accept); + return get(cluster, path, headers); + } + + /** + * Send a GET request + * @param path the path or URI + * @param headers the HTTP headers to include in the request, + * Accept must be supplied + * @return a Response object with response detail + * @throws IOException + */ + public Response get(String path, Header[] headers) throws IOException { + return get(cluster, path, headers); + } + + /** + * Send a GET request + * @param c the cluster definition + * @param path the path or URI + * @param headers the HTTP headers to include in the request + * @return a Response object with response detail + * @throws IOException + */ + public Response get(Cluster c, String path, Header[] headers) + throws IOException { + GetMethod method = new GetMethod(); + try { + int code = execute(c, method, headers, path); + headers = method.getResponseHeaders(); + byte[] body = method.getResponseBody(); + return new Response(code, headers, body); + } finally { + method.releaseConnection(); + } + } + + /** + * Send a PUT request + * @param path the path or URI + * @param contentType the content MIME type + * @param content the content bytes + * @return a Response object with response detail + * @throws IOException + */ + public Response put(String path, String contentType, byte[] content) + throws IOException { + return put(cluster, path, contentType, content); + } + + /** + * Send a PUT request + * @param cluster the cluster definition + * @param path the path or URI + * @param contentType the content MIME type + * @param content the content bytes + * @return a Response object with response detail + * @throws IOException + */ + public Response put(Cluster cluster, String path, String contentType, + byte[] content) throws IOException { + Header[] headers = new Header[1]; + headers[0] = new Header("Content-Type", contentType); + return put(cluster, path, headers, content); + } + + /** + * Send a PUT request + * @param path the path or URI + * @param headers the HTTP headers to include, Content-Type must be + * supplied + * @param content the content bytes + * @return a Response object with response detail + * @throws IOException + */ + public Response put(String path, Header[] headers, byte[] content) + throws IOException { + return put(cluster, path, headers, content); + } + + /** + * Send a PUT request + * @param cluster the cluster definition + * @param path the path or URI + * @param headers the HTTP headers to include, Content-Type must be + * supplied + * @param content the content bytes + * @return a Response object with response detail + * @throws IOException + */ + public Response put(Cluster cluster, String path, Header[] headers, + byte[] content) throws IOException { + PutMethod method = new PutMethod(); + try { + method.setRequestEntity(new ByteArrayRequestEntity(content)); + int code = execute(cluster, method, headers, path); + headers = method.getResponseHeaders(); + content = method.getResponseBody(); + return new Response(code, headers, content); + } finally { + method.releaseConnection(); + } + } + + /** + * Send a POST request + * @param path the path or URI + * @param contentType the content MIME type + * @param content the content bytes + * @return a Response object with response detail + * @throws IOException + */ + public Response post(String path, String contentType, byte[] content) + throws IOException { + return post(cluster, path, contentType, content); + } + + /** + * Send a POST request + * @param cluster the cluster definition + * @param path the path or URI + * @param contentType the content MIME type + * @param content the content bytes + * @return a Response object with response detail + * @throws IOException + */ + public Response post(Cluster cluster, String path, String contentType, + byte[] content) throws IOException { + Header[] headers = new Header[1]; + headers[0] = new Header("Content-Type", contentType); + return post(cluster, path, headers, content); + } + + /** + * Send a POST request + * @param path the path or URI + * @param headers the HTTP headers to include, Content-Type must be + * supplied + * @param content the content bytes + * @return a Response object with response detail + * @throws IOException + */ + public Response post(String path, Header[] headers, byte[] content) + throws IOException { + return post(cluster, path, headers, content); + } + + /** + * Send a POST request + * @param cluster the cluster definition + * @param path the path or URI + * @param headers the HTTP headers to include, Content-Type must be + * supplied + * @param content the content bytes + * @return a Response object with response detail + * @throws IOException + */ + public Response post(Cluster cluster, String path, Header[] headers, + byte[] content) throws IOException { + PostMethod method = new PostMethod(); + try { + method.setRequestEntity(new ByteArrayRequestEntity(content)); + int code = execute(cluster, method, headers, path); + headers = method.getResponseHeaders(); + content = method.getResponseBody(); + return new Response(code, headers, content); + } finally { + method.releaseConnection(); + } + } + + /** + * Send a DELETE request + * @param path the path or URI + * @return a Response object with response detail + * @throws IOException + */ + public Response delete(String path) throws IOException { + return delete(cluster, path); + } + + /** + * Send a DELETE request + * @param cluster the cluster definition + * @param path the path or URI + * @return a Response object with response detail + * @throws IOException + */ + public Response delete(Cluster cluster, String path) throws IOException { + DeleteMethod method = new DeleteMethod(); + try { + int code = execute(cluster, method, null, path); + Header[] headers = method.getResponseHeaders(); + byte[] content = method.getResponseBody(); + return new Response(code, headers, content); + } finally { + method.releaseConnection(); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/Cluster.java b/src/main/java/org/apache/hadoop/hbase/rest/client/Cluster.java new file mode 100644 index 0000000..4672447 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/client/Cluster.java @@ -0,0 +1,99 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.client; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A list of 'host:port' addresses of HTTP servers operating as a single + * entity, for example multiple redundant web service gateways. + */ +public class Cluster { + protected List nodes = + Collections.synchronizedList(new ArrayList()); + protected String lastHost; + + /** + * Constructor + */ + public Cluster() {} + + /** + * Constructor + * @param nodes a list of service locations, in 'host:port' format + */ + public Cluster(List nodes) { + nodes.addAll(nodes); + } + + /** + * @return true if no locations have been added, false otherwise + */ + public boolean isEmpty() { + return nodes.isEmpty(); + } + + /** + * Add a node to the cluster + * @param node the service location in 'host:port' format + */ + public Cluster add(String node) { + nodes.add(node); + return this; + } + + /** + * Add a node to the cluster + * @param name host name + * @param port service port + */ + public Cluster add(String name, int port) { + StringBuilder sb = new StringBuilder(); + sb.append(name); + sb.append(':'); + sb.append(port); + return add(sb.toString()); + } + + /** + * Remove a node from the cluster + * @param node the service location in 'host:port' format + */ + public Cluster remove(String node) { + nodes.remove(node); + return this; + } + + /** + * Remove a node from the cluster + * @param name host name + * @param port service port + */ + public Cluster remove(String name, int port) { + StringBuilder sb = new StringBuilder(); + sb.append(name); + sb.append(':'); + sb.append(port); + return remove(sb.toString()); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java new file mode 100644 index 0000000..a3d0197 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java @@ -0,0 +1,396 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.client; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Unmarshaller; + +import org.apache.hadoop.conf.Configuration; + +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.rest.Constants; +import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; +import org.apache.hadoop.hbase.rest.VersionResource; +import org.apache.hadoop.hbase.rest.model.StorageClusterStatusModel; +import org.apache.hadoop.hbase.rest.model.StorageClusterVersionModel; +import org.apache.hadoop.hbase.rest.model.TableListModel; +import org.apache.hadoop.hbase.rest.model.TableSchemaModel; +import org.apache.hadoop.hbase.rest.model.VersionModel; +import org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus; +import org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage; +import org.apache.hadoop.hbase.util.Bytes; +import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder; +import org.mortbay.jetty.MimeTypes; +import org.xml.sax.InputSource; + +public class RemoteAdmin { + + final Client client; + final Configuration conf; + final String accessToken; + final int maxRetries; + final long sleepTime; + + // This unmarshaller is necessary for getting the /version/cluster resource. + // This resource does not support protobufs. Therefore this is necessary to + // request/interpret it as XML. + private static volatile Unmarshaller versionClusterUnmarshaller; + + /** + * Constructor + * + * @param client + * @param conf + */ + public RemoteAdmin(Client client, Configuration conf) { + this(client, conf, null); + } + + static Unmarshaller getUnmarsheller() throws JAXBException { + + if (versionClusterUnmarshaller == null) { + + RemoteAdmin.versionClusterUnmarshaller = JAXBContext.newInstance( + StorageClusterVersionModel.class).createUnmarshaller(); + } + return RemoteAdmin.versionClusterUnmarshaller; + } + + /** + * Constructor + * @param client + * @param conf + * @param accessToken + */ + public RemoteAdmin(Client client, Configuration conf, String accessToken) { + this.client = client; + this.conf = conf; + this.accessToken = accessToken; + this.maxRetries = conf.getInt("hbase.rest.client.max.retries", 10); + this.sleepTime = conf.getLong("hbase.rest.client.sleep", 1000); + } + + /** + * @param tableName name of table to check + * @return true if all regions of the table are available + * @throws IOException if a remote or network exception occurs + */ + public boolean isTableAvailable(String tableName) throws IOException { + return isTableAvailable(Bytes.toBytes(tableName)); + } + + /** + * @return string representing the rest api's version + * @throws IOEXception + * if the endpoint does not exist, there is a timeout, or some other + * general failure mode + */ + public VersionModel getRestVersion() throws IOException { + + StringBuilder path = new StringBuilder(); + path.append('/'); + if (accessToken != null) { + path.append(accessToken); + path.append('/'); + } + + path.append("version/rest"); + + int code = 0; + for (int i = 0; i < maxRetries; i++) { + Response response = client.get(path.toString(), + Constants.MIMETYPE_PROTOBUF); + code = response.getCode(); + switch (code) { + case 200: + + VersionModel v = new VersionModel(); + return (VersionModel) v.getObjectFromMessage(response.getBody()); + case 404: + throw new IOException("REST version not found"); + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + } + break; + default: + throw new IOException("get request to " + path.toString() + + " returned " + code); + } + } + throw new IOException("get request to " + path.toString() + " timed out"); + } + + /** + * @return string representing the cluster's version + * @throws IOEXception if the endpoint does not exist, there is a timeout, or some other general failure mode + */ + public StorageClusterStatusModel getClusterStatus() throws IOException { + + StringBuilder path = new StringBuilder(); + path.append('/'); + if (accessToken !=null) { + path.append(accessToken); + path.append('/'); + } + + path.append("status/cluster"); + + int code = 0; + for (int i = 0; i < maxRetries; i++) { + Response response = client.get(path.toString(), + Constants.MIMETYPE_PROTOBUF); + code = response.getCode(); + switch (code) { + case 200: + StorageClusterStatusModel s = new StorageClusterStatusModel(); + return (StorageClusterStatusModel) s.getObjectFromMessage(response + .getBody()); + case 404: + throw new IOException("Cluster version not found"); + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + } + break; + default: + throw new IOException("get request to " + path + " returned " + code); + } + } + throw new IOException("get request to " + path + " timed out"); + } + + /** + * @return string representing the cluster's version + * @throws IOEXception + * if the endpoint does not exist, there is a timeout, or some other + * general failure mode + */ + public StorageClusterVersionModel getClusterVersion() throws IOException { + + StringBuilder path = new StringBuilder(); + path.append('/'); + if (accessToken != null) { + path.append(accessToken); + path.append('/'); + } + + path.append("version/cluster"); + + int code = 0; + for (int i = 0; i < maxRetries; i++) { + Response response = client.get(path.toString(), Constants.MIMETYPE_XML); + code = response.getCode(); + switch (code) { + case 200: + try { + + return (StorageClusterVersionModel) getUnmarsheller().unmarshal( + new ByteArrayInputStream(response.getBody())); + } catch (JAXBException jaxbe) { + + throw new IOException( + "Issue parsing StorageClusterVersionModel object in XML form: " + + jaxbe.getLocalizedMessage()); + } + case 404: + throw new IOException("Cluster version not found"); + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + } + break; + default: + throw new IOException(path.toString() + " request returned " + code); + } + } + throw new IOException("get request to " + path.toString() + + " request timed out"); + } + + /** + * @param tableName name of table to check + * @return true if all regions of the table are available + * @throws IOException if a remote or network exception occurs + */ + public boolean isTableAvailable(byte[] tableName) throws IOException { + StringBuilder path = new StringBuilder(); + path.append('/'); + if (accessToken != null) { + path.append(accessToken); + path.append('/'); + } + path.append(Bytes.toStringBinary(tableName)); + path.append('/'); + path.append("exists"); + int code = 0; + for (int i = 0; i < maxRetries; i++) { + Response response = client.get(path.toString(), Constants.MIMETYPE_PROTOBUF); + code = response.getCode(); + switch (code) { + case 200: + return true; + case 404: + return false; + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { } + break; + default: + throw new IOException("get request to " + path.toString() + " returned " + code); + } + } + throw new IOException("get request to " + path.toString() + " timed out"); + } + + /** + * Creates a new table. + * @param desc table descriptor for table + * @throws IOException if a remote or network exception occurs + */ + public void createTable(HTableDescriptor desc) + throws IOException { + TableSchemaModel model = new TableSchemaModel(desc); + StringBuilder path = new StringBuilder(); + path.append('/'); + if (accessToken != null) { + path.append(accessToken); + path.append('/'); + } + path.append(Bytes.toStringBinary(desc.getName())); + path.append('/'); + path.append("schema"); + int code = 0; + for (int i = 0; i < maxRetries; i++) { + Response response = client.put(path.toString(), Constants.MIMETYPE_PROTOBUF, + model.createProtobufOutput()); + code = response.getCode(); + switch (code) { + case 201: + return; + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { } + break; + default: + throw new IOException("create request to " + path.toString() + " returned " + code); + } + } + throw new IOException("create request to " + path.toString() + " timed out"); + } + + /** + * Deletes a table. + * @param tableName name of table to delete + * @throws IOException if a remote or network exception occurs + */ + public void deleteTable(final String tableName) throws IOException { + deleteTable(Bytes.toBytes(tableName)); + } + + /** + * Deletes a table. + * @param tableName name of table to delete + * @throws IOException if a remote or network exception occurs + */ + public void deleteTable(final byte [] tableName) throws IOException { + StringBuilder path = new StringBuilder(); + path.append('/'); + if (accessToken != null) { + path.append(accessToken); + path.append('/'); + } + path.append(Bytes.toStringBinary(tableName)); + path.append('/'); + path.append("schema"); + int code = 0; + for (int i = 0; i < maxRetries; i++) { + Response response = client.delete(path.toString()); + code = response.getCode(); + switch (code) { + case 200: + return; + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { } + break; + default: + throw new IOException("delete request to " + path.toString() + " returned " + code); + } + } + throw new IOException("delete request to " + path.toString() + " timed out"); + } + + /** + * @return string representing the cluster's version + * @throws IOEXception + * if the endpoint does not exist, there is a timeout, or some other + * general failure mode + */ + public TableListModel getTableList() throws IOException { + + StringBuilder path = new StringBuilder(); + path.append('/'); + if (accessToken != null) { + path.append(accessToken); + path.append('/'); + } + + int code = 0; + for (int i = 0; i < maxRetries; i++) { + // Response response = client.get(path.toString(), + // Constants.MIMETYPE_XML); + Response response = client.get(path.toString(), + Constants.MIMETYPE_PROTOBUF); + code = response.getCode(); + switch (code) { + case 200: + TableListModel t = new TableListModel(); + return (TableListModel) t.getObjectFromMessage(response.getBody()); + case 404: + throw new IOException("Table list not found"); + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + } + break; + default: + throw new IOException("get request to " + path.toString() + + " request returned " + code); + } + } + throw new IOException("get request to " + path.toString() + + " request timed out"); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java new file mode 100644 index 0000000..e3be8e1 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java @@ -0,0 +1,805 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.util.StringUtils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.RowMutations; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Row; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.RowLock; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.TimeRange; +import org.apache.hadoop.hbase.rest.Constants; +import org.apache.hadoop.hbase.rest.model.CellModel; +import org.apache.hadoop.hbase.rest.model.CellSetModel; +import org.apache.hadoop.hbase.rest.model.RowModel; +import org.apache.hadoop.hbase.rest.model.ScannerModel; +import org.apache.hadoop.hbase.rest.model.TableSchemaModel; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * HTable interface to remote tables accessed via REST gateway + */ +public class RemoteHTable implements HTableInterface { + + private static final Log LOG = LogFactory.getLog(RemoteHTable.class); + + final Client client; + final Configuration conf; + final byte[] name; + final int maxRetries; + final long sleepTime; + + @SuppressWarnings("rawtypes") + protected String buildRowSpec(final byte[] row, final Map familyMap, + final long startTime, final long endTime, final int maxVersions) { + StringBuffer sb = new StringBuffer(); + sb.append('/'); + sb.append(Bytes.toStringBinary(name)); + sb.append('/'); + sb.append(Bytes.toStringBinary(row)); + Set families = familyMap.entrySet(); + if (families != null) { + Iterator i = familyMap.entrySet().iterator(); + if (i.hasNext()) { + sb.append('/'); + } + while (i.hasNext()) { + Map.Entry e = (Map.Entry)i.next(); + Collection quals = (Collection)e.getValue(); + if (quals != null && !quals.isEmpty()) { + Iterator ii = quals.iterator(); + while (ii.hasNext()) { + sb.append(Bytes.toStringBinary((byte[])e.getKey())); + sb.append(':'); + Object o = ii.next(); + // Puts use byte[] but Deletes use KeyValue + if (o instanceof byte[]) { + sb.append(Bytes.toStringBinary((byte[])o)); + } else if (o instanceof KeyValue) { + sb.append(Bytes.toStringBinary(((KeyValue)o).getQualifier())); + } else { + throw new RuntimeException("object type not handled"); + } + if (ii.hasNext()) { + sb.append(','); + } + } + } else { + sb.append(Bytes.toStringBinary((byte[])e.getKey())); + sb.append(':'); + } + if (i.hasNext()) { + sb.append(','); + } + } + } + if (startTime != 0 && endTime != Long.MAX_VALUE) { + sb.append('/'); + sb.append(startTime); + if (startTime != endTime) { + sb.append(','); + sb.append(endTime); + } + } else if (endTime != Long.MAX_VALUE) { + sb.append('/'); + sb.append(endTime); + } + if (maxVersions > 1) { + sb.append("?v="); + sb.append(maxVersions); + } + return sb.toString(); + } + + protected String buildMultiRowSpec(final byte[][] rows, int maxVersions) { + StringBuilder sb = new StringBuilder(); + sb.append('/'); + sb.append(Bytes.toStringBinary(name)); + sb.append("/multiget/"); + if (rows == null || rows.length == 0) { + return sb.toString(); + } + sb.append("?"); + for(int i=0; i 0) { + if (results.length > 1) { + LOG.warn("too many results for get (" + results.length + ")"); + } + return results[0]; + } else { + return new Result(); + } + } + + public Result[] get(List gets) throws IOException { + byte[][] rows = new byte[gets.size()][]; + int maxVersions = 1; + int count = 0; + + for (Get g : gets) { + + if (count == 0) { + maxVersions = g.getMaxVersions(); + } else if (g.getMaxVersions() != maxVersions) { + LOG.warn("MaxVersions on Gets do not match, using the first in the list (" + + maxVersions +")"); + } + + if (g.getFilter() != null) { + LOG.warn("filters not supported on gets"); + } + + rows[count] = g.getRow(); + count++; + } + + String spec = buildMultiRowSpec(rows, maxVersions); + + return getResults(spec); + } + + private Result[] getResults(String spec) throws IOException { + for (int i = 0; i < maxRetries; i++) { + Response response = client.get(spec, Constants.MIMETYPE_PROTOBUF); + int code = response.getCode(); + switch (code) { + case 200: + CellSetModel model = new CellSetModel(); + model.getObjectFromMessage(response.getBody()); + Result[] results = buildResultFromModel(model); + if (results.length > 0) { + return results; + } + // fall through + case 404: + return new Result[0]; + + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + } + break; + default: + throw new IOException("get request returned " + code); + } + } + throw new IOException("get request timed out"); + } + + public boolean exists(Get get) throws IOException { + LOG.warn("exists() is really get(), just use get()"); + Result result = get(get); + return (result != null && !(result.isEmpty())); + } + + public void put(Put put) throws IOException { + CellSetModel model = buildModelFromPut(put); + StringBuilder sb = new StringBuilder(); + sb.append('/'); + sb.append(Bytes.toStringBinary(name)); + sb.append('/'); + sb.append(Bytes.toStringBinary(put.getRow())); + for (int i = 0; i < maxRetries; i++) { + Response response = client.put(sb.toString(), Constants.MIMETYPE_PROTOBUF, + model.createProtobufOutput()); + int code = response.getCode(); + switch (code) { + case 200: + return; + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { } + break; + default: + throw new IOException("put request failed with " + code); + } + } + throw new IOException("put request timed out"); + } + + public void put(List puts) throws IOException { + // this is a trick: The gateway accepts multiple rows in a cell set and + // ignores the row specification in the URI + + // separate puts by row + TreeMap> map = + new TreeMap>(Bytes.BYTES_COMPARATOR); + for (Put put: puts) { + byte[] row = put.getRow(); + List kvs = map.get(row); + if (kvs == null) { + kvs = new ArrayList(); + map.put(row, kvs); + } + for (List l: put.getFamilyMap().values()) { + kvs.addAll(l); + } + } + + // build the cell set + CellSetModel model = new CellSetModel(); + for (Map.Entry> e: map.entrySet()) { + RowModel row = new RowModel(e.getKey()); + for (KeyValue kv: e.getValue()) { + row.addCell(new CellModel(kv)); + } + model.addRow(row); + } + + // build path for multiput + StringBuilder sb = new StringBuilder(); + sb.append('/'); + sb.append(Bytes.toStringBinary(name)); + sb.append("/$multiput"); // can be any nonexistent row + for (int i = 0; i < maxRetries; i++) { + Response response = client.put(sb.toString(), Constants.MIMETYPE_PROTOBUF, + model.createProtobufOutput()); + int code = response.getCode(); + switch (code) { + case 200: + return; + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { } + break; + default: + throw new IOException("multiput request failed with " + code); + } + } + throw new IOException("multiput request timed out"); + } + + public void delete(Delete delete) throws IOException { + String spec = buildRowSpec(delete.getRow(), delete.getFamilyMap(), + delete.getTimeStamp(), delete.getTimeStamp(), 1); + for (int i = 0; i < maxRetries; i++) { + Response response = client.delete(spec); + int code = response.getCode(); + switch (code) { + case 200: + return; + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { } + break; + default: + throw new IOException("delete request failed with " + code); + } + } + throw new IOException("delete request timed out"); + } + + public void delete(List deletes) throws IOException { + for (Delete delete: deletes) { + delete(delete); + } + } + + public void flushCommits() throws IOException { + // no-op + } + + class Scanner implements ResultScanner { + + String uri; + + public Scanner(Scan scan) throws IOException { + ScannerModel model; + try { + model = ScannerModel.fromScan(scan); + } catch (Exception e) { + throw new IOException(e); + } + StringBuffer sb = new StringBuffer(); + sb.append('/'); + sb.append(Bytes.toStringBinary(name)); + sb.append('/'); + sb.append("scanner"); + for (int i = 0; i < maxRetries; i++) { + Response response = client.post(sb.toString(), + Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); + int code = response.getCode(); + switch (code) { + case 201: + uri = response.getLocation(); + return; + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { } + break; + default: + throw new IOException("scan request failed with " + code); + } + } + throw new IOException("scan request timed out"); + } + + @Override + public Result[] next(int nbRows) throws IOException { + StringBuilder sb = new StringBuilder(uri); + sb.append("?n="); + sb.append(nbRows); + for (int i = 0; i < maxRetries; i++) { + Response response = client.get(sb.toString(), + Constants.MIMETYPE_PROTOBUF); + int code = response.getCode(); + switch (code) { + case 200: + CellSetModel model = new CellSetModel(); + model.getObjectFromMessage(response.getBody()); + return buildResultFromModel(model); + case 204: + case 206: + return null; + case 509: + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { } + break; + default: + throw new IOException("scanner.next request failed with " + code); + } + } + throw new IOException("scanner.next request timed out"); + } + + @Override + public Result next() throws IOException { + Result[] results = next(1); + if (results == null || results.length < 1) { + return null; + } + return results[0]; + } + + class Iter implements Iterator { + + Result cache; + + public Iter() { + try { + cache = Scanner.this.next(); + } catch (IOException e) { + LOG.warn(StringUtils.stringifyException(e)); + } + } + + @Override + public boolean hasNext() { + return cache != null; + } + + @Override + public Result next() { + Result result = cache; + try { + cache = Scanner.this.next(); + } catch (IOException e) { + LOG.warn(StringUtils.stringifyException(e)); + cache = null; + } + return result; + } + + @Override + public void remove() { + throw new RuntimeException("remove() not supported"); + } + + } + + @Override + public Iterator iterator() { + return new Iter(); + } + + @Override + public void close() { + try { + client.delete(uri); + } catch (IOException e) { + LOG.warn(StringUtils.stringifyException(e)); + } + } + + } + + public ResultScanner getScanner(Scan scan) throws IOException { + return new Scanner(scan); + } + + public ResultScanner getScanner(byte[] family) throws IOException { + Scan scan = new Scan(); + scan.addFamily(family); + return new Scanner(scan); + } + + public ResultScanner getScanner(byte[] family, byte[] qualifier) + throws IOException { + Scan scan = new Scan(); + scan.addColumn(family, qualifier); + return new Scanner(scan); + } + + public boolean isAutoFlush() { + return true; + } + + public Result getRowOrBefore(byte[] row, byte[] family) throws IOException { + throw new IOException("getRowOrBefore not supported"); + } + + /** + * @deprecated {@link RowLock} and associated operations are deprecated + */ + public RowLock lockRow(byte[] row) throws IOException { + throw new IOException("lockRow not implemented"); + } + + /** + * @deprecated {@link RowLock} and associated operations are deprecated + */ + public void unlockRow(RowLock rl) throws IOException { + throw new IOException("unlockRow not implemented"); + } + + public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, + byte[] value, Put put) throws IOException { + // column to check-the-value + put.add(new KeyValue(row, family, qualifier, value)); + + CellSetModel model = buildModelFromPut(put); + StringBuilder sb = new StringBuilder(); + sb.append('/'); + sb.append(Bytes.toStringBinary(name)); + sb.append('/'); + sb.append(Bytes.toStringBinary(put.getRow())); + sb.append("?check=put"); + + for (int i = 0; i < maxRetries; i++) { + Response response = client.put(sb.toString(), + Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); + int code = response.getCode(); + switch (code) { + case 200: + return true; + case 304: // NOT-MODIFIED + return false; + case 509: + try { + Thread.sleep(sleepTime); + } catch (final InterruptedException e) { + } + break; + default: + throw new IOException("checkAndPut request failed with " + code); + } + } + throw new IOException("checkAndPut request timed out"); + } + + public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, + byte[] value, Delete delete) throws IOException { + Put put = new Put(row); + // column to check-the-value + put.add(new KeyValue(row, family, qualifier, value)); + CellSetModel model = buildModelFromPut(put); + StringBuilder sb = new StringBuilder(); + sb.append('/'); + sb.append(Bytes.toStringBinary(name)); + sb.append('/'); + sb.append(Bytes.toStringBinary(row)); + sb.append("?check=delete"); + + for (int i = 0; i < maxRetries; i++) { + Response response = client.put(sb.toString(), + Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); + int code = response.getCode(); + switch (code) { + case 200: + return true; + case 304: // NOT-MODIFIED + return false; + case 509: + try { + Thread.sleep(sleepTime); + } catch (final InterruptedException e) { + } + break; + default: + throw new IOException("checkAndDelete request failed with " + code); + } + } + throw new IOException("checkAndDelete request timed out"); + } + + public Result increment(Increment increment) throws IOException { + throw new IOException("Increment not supported"); + } + + public Result append(Append append) throws IOException { + throw new IOException("Append not supported"); + } + + public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, + long amount) throws IOException { + throw new IOException("incrementColumnValue not supported"); + } + + public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, + long amount, boolean writeToWAL) throws IOException { + throw new IOException("incrementColumnValue not supported"); + } + + @Override + public void batch(List actions, Object[] results) throws IOException { + throw new IOException("batch not supported"); + } + + @Override + public Object[] batch(List actions) throws IOException { + throw new IOException("batch not supported"); + } + + @Override + public T coprocessorProxy(Class protocol, + byte[] row) { + throw new + UnsupportedOperationException("coprocessorProxy not implemented"); + } + + @Override + public Map coprocessorExec( + Class protocol, byte[] startKey, byte[] endKey, + Batch.Call callable) + throws IOException, Throwable { + throw new UnsupportedOperationException("coprocessorExec not implemented"); + } + + @Override + public void coprocessorExec( + Class protocol, byte[] startKey, byte[] endKey, + Batch.Call callable, Batch.Callback callback) + throws IOException, Throwable { + throw new UnsupportedOperationException("coprocessorExec not implemented"); + } + + @Override + public void mutateRow(RowMutations rm) throws IOException { + throw new IOException("atomicMutation not supported"); + } + + @Override + public void setAutoFlush(boolean autoFlush) { + throw new UnsupportedOperationException("setAutoFlush not implemented"); + } + + @Override + public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) { + throw new UnsupportedOperationException("setAutoFlush not implemented"); + } + + @Override + public long getWriteBufferSize() { + throw new UnsupportedOperationException("getWriteBufferSize not implemented"); + } + + @Override + public void setWriteBufferSize(long writeBufferSize) throws IOException { + throw new IOException("setWriteBufferSize not supported"); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/client/Response.java b/src/main/java/org/apache/hadoop/hbase/rest/client/Response.java new file mode 100644 index 0000000..421065b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/client/Response.java @@ -0,0 +1,126 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.client; + +import org.apache.commons.httpclient.Header; + +/** + * The HTTP result code, response headers, and body of a HTTP response. + */ +public class Response { + private int code; + private Header[] headers; + private byte[] body; + + /** + * Constructor + * @param code the HTTP response code + */ + public Response(int code) { + this(code, null, null); + } + + /** + * Constructor + * @param code the HTTP response code + * @param headers the HTTP response headers + */ + public Response(int code, Header[] headers) { + this(code, headers, null); + } + + /** + * Constructor + * @param code the HTTP response code + * @param headers the HTTP response headers + * @param body the response body, can be null + */ + public Response(int code, Header[] headers, byte[] body) { + this.code = code; + this.headers = headers; + this.body = body; + } + + /** + * @return the HTTP response code + */ + public int getCode() { + return code; + } + + /** + * @return the HTTP response headers + */ + public Header[] getHeaders() { + return headers; + } + + public String getHeader(String key) { + for (Header header: headers) { + if (header.getName().equalsIgnoreCase(key)) { + return header.getValue(); + } + } + return null; + } + + /** + * @return the value of the Location header + */ + public String getLocation() { + return getHeader("Location"); + } + + /** + * @return true if a response body was sent + */ + public boolean hasBody() { + return body != null; + } + + /** + * @return the HTTP response body + */ + public byte[] getBody() { + return body; + } + + /** + * @param code the HTTP response code + */ + public void setCode(int code) { + this.code = code; + } + + /** + * @param headers the HTTP response headers + */ + public void setHeaders(Header[] headers) { + this.headers = headers; + } + + /** + * @param body the response body + */ + public void setBody(byte[] body) { + this.body = body; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestStream.java b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestStream.java new file mode 100644 index 0000000..0bd5f65 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestStream.java @@ -0,0 +1,56 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.filter; + +import java.io.IOException; +import java.util.zip.GZIPInputStream; + +import javax.servlet.ServletInputStream; +import javax.servlet.http.HttpServletRequest; + +public class GZIPRequestStream extends ServletInputStream +{ + private GZIPInputStream in; + + public GZIPRequestStream(HttpServletRequest request) throws IOException { + this.in = new GZIPInputStream(request.getInputStream()); + } + + @Override + public int read() throws IOException { + return in.read(); + } + + @Override + public int read(byte[] b) throws IOException { + return in.read(b); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return in.read(b, off, len); + } + + @Override + public void close() throws IOException { + in.close(); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestWrapper.java b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestWrapper.java new file mode 100644 index 0000000..764576c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPRequestWrapper.java @@ -0,0 +1,50 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.filter; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; + +import javax.servlet.ServletInputStream; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; + +public class GZIPRequestWrapper extends HttpServletRequestWrapper { + private ServletInputStream is; + private BufferedReader reader; + + public GZIPRequestWrapper(HttpServletRequest request) throws IOException { + super(request); + this.is = new GZIPRequestStream(request); + this.reader = new BufferedReader(new InputStreamReader(this.is)); + } + + @Override + public ServletInputStream getInputStream() throws IOException { + return is; + } + + @Override + public BufferedReader getReader() throws IOException { + return reader; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseStream.java b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseStream.java new file mode 100644 index 0000000..d27b37b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseStream.java @@ -0,0 +1,76 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.filter; + +import java.io.IOException; +import java.util.zip.GZIPOutputStream; + +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; + +public class GZIPResponseStream extends ServletOutputStream +{ + private HttpServletResponse response; + private GZIPOutputStream out; + + public GZIPResponseStream(HttpServletResponse response) throws IOException { + this.response = response; + this.out = new GZIPOutputStream(response.getOutputStream()); + response.addHeader("Content-Encoding", "gzip"); + } + + public void resetBuffer() { + if (out != null && !response.isCommitted()) { + response.setHeader("Content-Encoding", null); + } + out = null; + } + + @Override + public void write(int b) throws IOException { + out.write(b); + } + + @Override + public void write(byte[] b) throws IOException { + out.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + } + + @Override + public void close() throws IOException { + finish(); + out.close(); + } + + @Override + public void flush() throws IOException { + out.flush(); + } + + public void finish() throws IOException { + out.finish(); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseWrapper.java b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseWrapper.java new file mode 100644 index 0000000..08a976b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/filter/GZIPResponseWrapper.java @@ -0,0 +1,145 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.filter; + +import java.io.IOException; +import java.io.PrintWriter; + +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletResponseWrapper; + +public class GZIPResponseWrapper extends HttpServletResponseWrapper { + private HttpServletResponse response; + private ServletOutputStream os; + private PrintWriter writer; + private boolean compress = true; + + public GZIPResponseWrapper(HttpServletResponse response) { + super(response); + this.response = response; + } + + @Override + public void setStatus(int status) { + super.setStatus(status); + if (status < 200 || status >= 300) { + compress = false; + } + } + + @Override + public void addHeader(String name, String value) { + if (!"content-length".equalsIgnoreCase(name)) { + super.addHeader(name, value); + } + } + + @Override + public void setContentLength(int length) { + // do nothing + } + + @Override + public void setIntHeader(String name, int value) { + if (!"content-length".equalsIgnoreCase(name)) { + super.setIntHeader(name, value); + } + } + + @Override + public void setHeader(String name, String value) { + if (!"content-length".equalsIgnoreCase(name)) { + super.setHeader(name, value); + } + } + + @Override + public void flushBuffer() throws IOException { + if (writer != null) { + writer.flush(); + } + if (os != null && (os instanceof GZIPResponseStream)) { + ((GZIPResponseStream)os).finish(); + } else { + getResponse().flushBuffer(); + } + } + + @Override + public void reset() { + super.reset(); + if (os != null && (os instanceof GZIPResponseStream)) { + ((GZIPResponseStream)os).resetBuffer(); + } + writer = null; + os = null; + compress = true; + } + + @Override + public void resetBuffer() { + super.resetBuffer(); + if (os != null && (os instanceof GZIPResponseStream)) { + ((GZIPResponseStream)os).resetBuffer(); + } + writer = null; + os = null; + } + + @Override + public void sendError(int status, String msg) throws IOException { + resetBuffer(); + super.sendError(status, msg); + } + + @Override + public void sendError(int status) throws IOException { + resetBuffer(); + super.sendError(status); + } + + @Override + public void sendRedirect(String location) throws IOException { + resetBuffer(); + super.sendRedirect(location); + } + + @Override + public ServletOutputStream getOutputStream() throws IOException { + if (os == null) { + if (!response.isCommitted() && compress) { + os = (ServletOutputStream)new GZIPResponseStream(response); + } else { + os = response.getOutputStream(); + } + } + return os; + } + + @Override + public PrintWriter getWriter() throws IOException { + if (writer == null) { + writer = new PrintWriter(getOutputStream()); + } + return writer; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/rest/filter/GzipFilter.java b/src/main/java/org/apache/hadoop/hbase/rest/filter/GzipFilter.java new file mode 100644 index 0000000..9c1c3f6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/filter/GzipFilter.java @@ -0,0 +1,82 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.filter; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.HashSet; +import java.util.Set; +import java.util.StringTokenizer; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +public class GzipFilter implements Filter { + private Set mimeTypes = new HashSet(); + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + String s = filterConfig.getInitParameter("mimeTypes"); + if (s != null) { + StringTokenizer tok = new StringTokenizer(s, ",", false); + while (tok.hasMoreTokens()) { + mimeTypes.add(tok.nextToken()); + } + } + } + + @Override + public void destroy() { + } + + @Override + public void doFilter(ServletRequest req, ServletResponse rsp, + FilterChain chain) throws IOException, ServletException { + HttpServletRequest request = (HttpServletRequest)req; + HttpServletResponse response = (HttpServletResponse)rsp; + String contentEncoding = request.getHeader("content-encoding"); + String acceptEncoding = request.getHeader("accept-encoding"); + String contentType = request.getHeader("content-type"); + if ((contentEncoding != null) && + (contentEncoding.toLowerCase().indexOf("gzip") > -1)) { + request = new GZIPRequestWrapper(request); + } + if (((acceptEncoding != null) && + (acceptEncoding.toLowerCase().indexOf("gzip") > -1)) || + ((contentType != null) && mimeTypes.contains(contentType))) { + response = new GZIPResponseWrapper(response); + } + chain.doFilter(request, response); + if (response instanceof GZIPResponseWrapper) { + OutputStream os = response.getOutputStream(); + if (os instanceof GZIPResponseStream) { + ((GZIPResponseStream)os).finish(); + } + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTMetrics.java b/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTMetrics.java new file mode 100644 index 0000000..d5abe6f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTMetrics.java @@ -0,0 +1,190 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.metrics; + +import org.apache.hadoop.hbase.metrics.MetricsRate; + +import org.apache.hadoop.metrics.MetricsContext; +import org.apache.hadoop.metrics.MetricsRecord; +import org.apache.hadoop.metrics.MetricsUtil; +import org.apache.hadoop.metrics.Updater; +import org.apache.hadoop.metrics.jvm.JvmMetrics; +import org.apache.hadoop.metrics.util.MetricsRegistry; + +public class RESTMetrics implements Updater { + private final MetricsRecord metricsRecord; + private final MetricsRegistry registry = new MetricsRegistry(); + private final RESTStatistics restStatistics; + + private MetricsRate requests = new MetricsRate("requests", registry); + private MetricsRate sucessfulGetCount = + new MetricsRate("sucessful.get.count", registry); + private MetricsRate sucessfulPutCount = + new MetricsRate("sucessful.put.count", registry); + private MetricsRate sucessfulDeleteCount = + new MetricsRate("sucessful.delete.count", registry); + + private MetricsRate failedGetCount = + new MetricsRate("failed.get.count", registry); + private MetricsRate failedPutCount = + new MetricsRate("failed.put.count", registry); + private MetricsRate failedDeleteCount = + new MetricsRate("failed.delete.count", registry); + + public RESTMetrics() { + MetricsContext context = MetricsUtil.getContext("rest"); + metricsRecord = MetricsUtil.createRecord(context, "rest"); + String name = Thread.currentThread().getName(); + metricsRecord.setTag("REST", name); + context.registerUpdater(this); + JvmMetrics.init("rest", name); + // expose the MBean for metrics + restStatistics = new RESTStatistics(registry); + + } + + public void shutdown() { + if (restStatistics != null) { + restStatistics.shutdown(); + } + } + + /** + * Since this object is a registered updater, this method will be called + * periodically, e.g. every 5 seconds. + * @param unused + */ + public void doUpdates(MetricsContext unused) { + synchronized (this) { + requests.pushMetric(metricsRecord); + sucessfulGetCount.pushMetric(metricsRecord); + sucessfulPutCount.pushMetric(metricsRecord); + sucessfulDeleteCount.pushMetric(metricsRecord); + failedGetCount.pushMetric(metricsRecord); + failedPutCount.pushMetric(metricsRecord); + failedDeleteCount.pushMetric(metricsRecord); + } + this.metricsRecord.update(); + } + + public void resetAllMinMax() { + // Nothing to do + } + + /** + * @return Count of requests. + */ + public float getRequests() { + return requests.getPreviousIntervalValue(); + } + + /** + * @param inc How much to add to requests. + */ + public void incrementRequests(final int inc) { + requests.inc(inc); + } + + /** + * @return Count of sucessfulGetCount. + */ + public float getSucessfulGetCount() { + return sucessfulGetCount.getPreviousIntervalValue(); + } + + /** + * @param inc How much to add to sucessfulGetCount. + */ + public void incrementSucessfulGetRequests(final int inc) { + sucessfulGetCount.inc(inc); + } + + /** + * @return Count of sucessfulGetCount. + */ + public float getSucessfulPutCount() { + return sucessfulPutCount.getPreviousIntervalValue(); + } + + /** + * @param inc How much to add to sucessfulPutCount. + */ + public void incrementSucessfulPutRequests(final int inc) { + sucessfulPutCount.inc(inc); + } + + /** + * @return Count of failedPutCount. + */ + public float getFailedPutCount() { + return failedPutCount.getPreviousIntervalValue(); + } + + /** + * @param inc How much to add to failedPutCount. + */ + public void incrementFailedPutRequests(final int inc) { + failedPutCount.inc(inc); + } + + /** + * @return Count of failedGetCount. + */ + public float getFailedGetCount() { + return failedGetCount.getPreviousIntervalValue(); + } + + /** + * @param inc How much to add to failedGetCount. + */ + public void incrementFailedGetRequests(final int inc) { + failedGetCount.inc(inc); + } + + /** + * @return Count of sucessfulGetCount. + */ + public float getSucessfulDeleteCount() { + return sucessfulDeleteCount.getPreviousIntervalValue(); + } + + /** + * @param inc How much to add to sucessfulDeleteCount. + */ + public void incrementSucessfulDeleteRequests(final int inc) { + sucessfulDeleteCount.inc(inc); + } + + /** + * @return Count of failedDeleteCount. + */ + public float getFailedDeleteCount() { + return failedDeleteCount.getPreviousIntervalValue(); + } + + /** + * @param inc How much to add to failedDeleteCount. + */ + public void incrementFailedDeleteRequests(final int inc) { + failedDeleteCount.inc(inc); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTStatistics.java b/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTStatistics.java new file mode 100644 index 0000000..d29d50d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/metrics/RESTStatistics.java @@ -0,0 +1,44 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.metrics; + +import javax.management.ObjectName; + +import org.apache.hadoop.hbase.metrics.MetricsMBeanBase; + +import org.apache.hadoop.metrics.util.MBeanUtil; +import org.apache.hadoop.metrics.util.MetricsRegistry; + +public class RESTStatistics extends MetricsMBeanBase { + private final ObjectName mbeanName; + + public RESTStatistics(MetricsRegistry registry) { + super(registry, "restStatistics"); + mbeanName = MBeanUtil.registerMBean("rest", "restStatistics", this); + } + + public void shutdown() { + if (mbeanName != null) { + MBeanUtil.unregisterMBean(mbeanName); + } + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java new file mode 100644 index 0000000..3413d00 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java @@ -0,0 +1,198 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlValue; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; +import org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell; + +import com.google.protobuf.ByteString; + +/** + * Representation of a cell. A cell is a single value associated a column and + * optional qualifier, and either the timestamp when it was stored or the user- + * provided timestamp if one was explicitly supplied. + * + * 

    + * <complexType name="Cell">
+ *   <sequence>
+ *     <element name="value" maxOccurs="1" minOccurs="1">
+ *       <simpleType>
+ *         <restriction base="base64Binary"/>
+ *       </simpleType>
+ *     </element>
+ *   </sequence>
+ *   <attribute name="column" type="base64Binary" />
+ *   <attribute name="timestamp" type="int" />
+ * </complexType>
+ *
    + */ +@XmlRootElement(name="Cell") +public class CellModel implements ProtobufMessageHandler, Serializable { + private static final long serialVersionUID = 1L; + + private long timestamp = HConstants.LATEST_TIMESTAMP; + private byte[] column; + private byte[] value; + + /** + * Default constructor + */ + public CellModel() {} + + /** + * Constructor + * @param column + * @param value + */ + public CellModel(byte[] column, byte[] value) { + this(column, HConstants.LATEST_TIMESTAMP, value); + } + + /** + * Constructor + * @param column + * @param qualifier + * @param value + */ + public CellModel(byte[] column, byte[] qualifier, byte[] value) { + this(column, qualifier, HConstants.LATEST_TIMESTAMP, value); + } + + /** + * Constructor from KeyValue + * @param kv + */ + public CellModel(KeyValue kv) { + this(kv.getFamily(), kv.getQualifier(), kv.getTimestamp(), kv.getValue()); + } + + /** + * Constructor + * @param column + * @param timestamp + * @param value + */ + public CellModel(byte[] column, long timestamp, byte[] value) { + this.column = column; + this.timestamp = timestamp; + this.value = value; + } + + /** + * Constructor + * @param column + * @param qualifier + * @param timestamp + * @param value + */ + public CellModel(byte[] column, byte[] qualifier, long timestamp, + byte[] value) { + this.column = KeyValue.makeColumn(column, qualifier); + this.timestamp = timestamp; + this.value = value; + } + + /** + * @return the column + */ + @XmlAttribute + public byte[] getColumn() { + return column; + } + + /** + * @param column the column to set + */ + public void setColumn(byte[] column) { + this.column = column; + } + + /** + * @return true if the timestamp property has been specified by the + * user + */ + public boolean hasUserTimestamp() { + return timestamp != HConstants.LATEST_TIMESTAMP; + } + + /** + * @return the timestamp + */ + @XmlAttribute + public long getTimestamp() { + return timestamp; + } + + /** + * @param timestamp the timestamp to set + */ + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + /** + * @return the value + */ + @XmlValue + public byte[] getValue() { + return value; + } + + /** + * @param value the value to set + */ + public void setValue(byte[] value) { + this.value = value; + } + + @Override + public byte[] createProtobufOutput() { + Cell.Builder builder = Cell.newBuilder(); + builder.setColumn(ByteString.copyFrom(getColumn())); + builder.setData(ByteString.copyFrom(getValue())); + if (hasUserTimestamp()) { + builder.setTimestamp(getTimestamp()); + } + return builder.build().toByteArray(); + } + + @Override + public ProtobufMessageHandler getObjectFromMessage(byte[] message) + throws IOException { + Cell.Builder builder = Cell.newBuilder(); + builder.mergeFrom(message); + setColumn(builder.getColumn().toByteArray()); + setValue(builder.getData().toByteArray()); + if (builder.hasTimestamp()) { + setTimestamp(builder.getTimestamp()); + } + return this; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java new file mode 100644 index 0000000..7e7073c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java @@ -0,0 +1,149 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlElement; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; +import org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell; +import org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet; + +import com.google.protobuf.ByteString; + +/** + * Representation of a grouping of cells. May contain cells from more than + * one row. Encapsulates RowModel and CellModel models. + * + * 
    + * <complexType name="CellSet">
+ *   <sequence>
+ *     <element name="row" type="tns:Row" maxOccurs="unbounded" 
+ *       minOccurs="1"></element>
+ *   </sequence>
+ * </complexType>
+ * 
+ * <complexType name="Row">
+ *   <sequence>
+ *     <element name="key" type="base64Binary"></element>
+ *     <element name="cell" type="tns:Cell" 
+ *       maxOccurs="unbounded" minOccurs="1"></element>
+ *   </sequence>
+ * </complexType>
+ *
+ * <complexType name="Cell">
+ *   <sequence>
+ *     <element name="value" maxOccurs="1" minOccurs="1">
+ *       <simpleType>
+ *         <restriction base="base64Binary"/>
+ *       </simpleType>
+ *     </element>
+ *   </sequence>
+ *   <attribute name="column" type="base64Binary" />
+ *   <attribute name="timestamp" type="int" />
+ * </complexType>
+ *
    + */ +@XmlRootElement(name="CellSet") +public class CellSetModel implements Serializable, ProtobufMessageHandler { + + private static final long serialVersionUID = 1L; + + private List rows; + + /** + * Constructor + */ + public CellSetModel() { + this.rows = new ArrayList(); + } + + /** + * @param rows the rows + */ + public CellSetModel(List rows) { + super(); + this.rows = rows; + } + + /** + * Add a row to this cell set + * @param row the row + */ + public void addRow(RowModel row) { + rows.add(row); + } + + /** + * @return the rows + */ + @XmlElement(name="Row") + public List getRows() { + return rows; + } + + @Override + public byte[] createProtobufOutput() { + CellSet.Builder builder = CellSet.newBuilder(); + for (RowModel row: getRows()) { + CellSet.Row.Builder rowBuilder = CellSet.Row.newBuilder(); + rowBuilder.setKey(ByteString.copyFrom(row.getKey())); + for (CellModel cell: row.getCells()) { + Cell.Builder cellBuilder = Cell.newBuilder(); + cellBuilder.setColumn(ByteString.copyFrom(cell.getColumn())); + cellBuilder.setData(ByteString.copyFrom(cell.getValue())); + if (cell.hasUserTimestamp()) { + cellBuilder.setTimestamp(cell.getTimestamp()); + } + rowBuilder.addValues(cellBuilder); + } + builder.addRows(rowBuilder); + } + return builder.build().toByteArray(); + } + + @Override + public ProtobufMessageHandler getObjectFromMessage(byte[] message) + throws IOException { + CellSet.Builder builder = CellSet.newBuilder(); + builder.mergeFrom(message); + for (CellSet.Row row: builder.getRowsList()) { + RowModel rowModel = new RowModel(row.getKey().toByteArray()); + for (Cell cell: row.getValuesList()) { + long timestamp = HConstants.LATEST_TIMESTAMP; + if (cell.hasTimestamp()) { + timestamp = cell.getTimestamp(); + } + rowModel.addCell( + new CellModel(cell.getColumn().toByteArray(), timestamp, + cell.getData().toByteArray())); + } + addRow(rowModel); + } + return this; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/ColumnSchemaModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/ColumnSchemaModel.java new file mode 100644 index 0000000..caf5368 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/ColumnSchemaModel.java @@ -0,0 +1,236 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAnyAttribute; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.namespace.QName; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; + +/** + * Representation of a column family schema. + * + * 
    + * <complexType name="ColumnSchema">
+ *   <attribute name="name" type="string"></attribute>
+ *   <anyAttribute></anyAttribute>
+ * </complexType>
+ *
    + */ +@XmlRootElement(name="ColumnSchema") +public class ColumnSchemaModel implements Serializable { + private static final long serialVersionUID = 1L; + private static QName BLOCKCACHE = new QName(HColumnDescriptor.BLOCKCACHE); + private static QName BLOCKSIZE = new QName(HColumnDescriptor.BLOCKSIZE); + private static QName BLOOMFILTER = new QName(HColumnDescriptor.BLOOMFILTER); + private static QName COMPRESSION = new QName(HColumnDescriptor.COMPRESSION); + private static QName IN_MEMORY = new QName(HConstants.IN_MEMORY); + private static QName TTL = new QName(HColumnDescriptor.TTL); + private static QName VERSIONS = new QName(HConstants.VERSIONS); + + private String name; + private Map attrs = new HashMap(); + + /** + * Default constructor + */ + public ColumnSchemaModel() {} + + /** + * Add an attribute to the column family schema + * @param name the attribute name + * @param value the attribute value + */ + public void addAttribute(String name, Object value) { + attrs.put(new QName(name), value); + } + + /** + * @param name the attribute name + * @return the attribute value + */ + public String getAttribute(String name) { + Object o = attrs.get(new QName(name)); + return o != null ? o.toString(): null; + } + + /** + * @return the column name + */ + @XmlAttribute + public String getName() { + return name; + } + + /** + * @return the map for holding unspecified (user) attributes + */ + @XmlAnyAttribute + public Map getAny() { + return attrs; + } + + /** + * @param name the table name + */ + public void setName(String name) { + this.name = name; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{ NAME => '"); + sb.append(name); + sb.append('\''); + for (Map.Entry e: attrs.entrySet()) { + sb.append(", "); + sb.append(e.getKey().getLocalPart()); + sb.append(" => '"); + sb.append(e.getValue().toString()); + sb.append('\''); + } + sb.append(" }"); + return sb.toString(); + } + + // getters and setters for common schema attributes + + // cannot be standard bean type getters and setters, otherwise this would + // confuse JAXB + + /** + * @return true if the BLOCKCACHE attribute is present and true + */ + public boolean __getBlockcache() { + Object o = attrs.get(BLOCKCACHE); + return o != null ? + Boolean.valueOf(o.toString()) : HColumnDescriptor.DEFAULT_BLOCKCACHE; + } + + /** + * @return the value of the BLOCKSIZE attribute or its default if it is unset + */ + public int __getBlocksize() { + Object o = attrs.get(BLOCKSIZE); + return o != null ? + Integer.valueOf(o.toString()) : HColumnDescriptor.DEFAULT_BLOCKSIZE; + } + + /** + * @return the value of the BLOOMFILTER attribute or its default if unset + */ + public String __getBloomfilter() { + Object o = attrs.get(BLOOMFILTER); + return o != null ? o.toString() : HColumnDescriptor.DEFAULT_BLOOMFILTER; + } + + /** + * @return the value of the COMPRESSION attribute or its default if unset + */ + public String __getCompression() { + Object o = attrs.get(COMPRESSION); + return o != null ? o.toString() : HColumnDescriptor.DEFAULT_COMPRESSION; + } + + /** + * @return true if the IN_MEMORY attribute is present and true + */ + public boolean __getInMemory() { + Object o = attrs.get(IN_MEMORY); + return o != null ? + Boolean.valueOf(o.toString()) : HColumnDescriptor.DEFAULT_IN_MEMORY; + } + + /** + * @return the value of the TTL attribute or its default if it is unset + */ + public int __getTTL() { + Object o = attrs.get(TTL); + return o != null ? + Integer.valueOf(o.toString()) : HColumnDescriptor.DEFAULT_TTL; + } + + /** + * @return the value of the VERSIONS attribute or its default if it is unset + */ + public int __getVersions() { + Object o = attrs.get(VERSIONS); + return o != null ? + Integer.valueOf(o.toString()) : HColumnDescriptor.DEFAULT_VERSIONS; + } + + /** + * @param value the desired value of the BLOCKSIZE attribute + */ + public void __setBlocksize(int value) { + attrs.put(BLOCKSIZE, Integer.toString(value)); + } + + /** + * @param value the desired value of the BLOCKCACHE attribute + */ + public void __setBlockcache(boolean value) { + attrs.put(BLOCKCACHE, Boolean.toString(value)); + } + + public void __setBloomfilter(String value) { + attrs.put(BLOOMFILTER, value); + } + + /** + * @param value the desired value of the COMPRESSION attribute + */ + public void __setCompression(String value) { + attrs.put(COMPRESSION, value); + } + + /** + * @param value the desired value of the IN_MEMORY attribute + */ + public void __setInMemory(boolean value) { + attrs.put(IN_MEMORY, Boolean.toString(value)); + } + + /** + * @param value the desired value of the TTL attribute + */ + public void __setTTL(int value) { + attrs.put(TTL, Integer.toString(value)); + } + + /** + * @param value the desired value of the VERSIONS attribute + */ + public void __setVersions(int value) { + attrs.put(VERSIONS, Integer.toString(value)); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java new file mode 100644 index 0000000..a987695 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java @@ -0,0 +1,142 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; + +/** + * Representation of a row. A row is a related set of cells, grouped by common + * row key. RowModels do not appear in results by themselves. They are always + * encapsulated within CellSetModels. + * + * 
    + * <complexType name="Row">
+ *   <sequence>
+ *     <element name="key" type="base64Binary"></element>
+ *     <element name="cell" type="tns:Cell" 
+ *       maxOccurs="unbounded" minOccurs="1"></element>
+ *   </sequence>
+ * </complexType>
+ *
    + */ +@XmlRootElement(name="Row") +public class RowModel implements ProtobufMessageHandler, Serializable { + private static final long serialVersionUID = 1L; + + private byte[] key; + private List cells = new ArrayList(); + + /** + * Default constructor + */ + public RowModel() { } + + /** + * Constructor + * @param key the row key + */ + public RowModel(final String key) { + this(key.getBytes()); + } + + /** + * Constructor + * @param key the row key + */ + public RowModel(final byte[] key) { + this.key = key; + cells = new ArrayList(); + } + + /** + * Constructor + * @param key the row key + * @param cells the cells + */ + public RowModel(final String key, final List cells) { + this(key.getBytes(), cells); + } + + /** + * Constructor + * @param key the row key + * @param cells the cells + */ + public RowModel(final byte[] key, final List cells) { + this.key = key; + this.cells = cells; + } + + /** + * Adds a cell to the list of cells for this row + * @param cell the cell + */ + public void addCell(CellModel cell) { + cells.add(cell); + } + + /** + * @return the row key + */ + @XmlAttribute + public byte[] getKey() { + return key; + } + + /** + * @param key the row key + */ + public void setKey(byte[] key) { + this.key = key; + } + + /** + * @return the cells + */ + @XmlElement(name="Cell") + public List getCells() { + return cells; + } + + @Override + public byte[] createProtobufOutput() { + // there is no standalone row protobuf message + throw new UnsupportedOperationException( + "no protobuf equivalent to RowModel"); + } + + @Override + public ProtobufMessageHandler getObjectFromMessage(byte[] message) + throws IOException { + // there is no standalone row protobuf message + throw new UnsupportedOperationException( + "no protobuf equivalent to RowModel"); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java new file mode 100644 index 0000000..d76bdb1 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java @@ -0,0 +1,740 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.Serializable; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.*; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; +import org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner; +import org.apache.hadoop.hbase.util.Base64; +import org.apache.hadoop.hbase.util.Bytes; + +import com.google.protobuf.ByteString; + +import com.sun.jersey.api.json.JSONConfiguration; +import com.sun.jersey.api.json.JSONJAXBContext; +import com.sun.jersey.api.json.JSONMarshaller; +import com.sun.jersey.api.json.JSONUnmarshaller; + +/** + * A representation of Scanner parameters. + * + * 
    + * <complexType name="Scanner">
+ *   <sequence>
+ *     <element name="column" type="base64Binary" minOccurs="0" maxOccurs="unbounded"/>
+ *   </sequence>
+ *   <element name="filter" type="string" minOccurs="0" maxOccurs="1"></element>
+ *   <attribute name="startRow" type="base64Binary"></attribute>
+ *   <attribute name="endRow" type="base64Binary"></attribute>
+ *   <attribute name="batch" type="int"></attribute>
+ *   <attribute name="startTime" type="int"></attribute>
+ *   <attribute name="endTime" type="int"></attribute>
+ *   <attribute name="maxVersions" type="int"></attribute>
+ * </complexType>
+ *
    + */ +@XmlRootElement(name="Scanner") +public class ScannerModel implements ProtobufMessageHandler, Serializable { + + private static final long serialVersionUID = 1L; + + private byte[] startRow = HConstants.EMPTY_START_ROW; + private byte[] endRow = HConstants.EMPTY_END_ROW;; + private List columns = new ArrayList(); + private int batch = Integer.MAX_VALUE; + private long startTime = 0; + private long endTime = Long.MAX_VALUE; + private String filter = null; + private int maxVersions = Integer.MAX_VALUE; + + @XmlRootElement + static class FilterModel { + + @XmlRootElement + static class WritableByteArrayComparableModel { + @XmlAttribute public String type; + @XmlAttribute public String value; + @XmlAttribute public String op; + + static enum ComparatorType { + BinaryComparator, + BinaryPrefixComparator, + BitComparator, + NullComparator, + RegexStringComparator, + SubstringComparator + } + + public WritableByteArrayComparableModel() { } + + public WritableByteArrayComparableModel( + WritableByteArrayComparable comparator) { + String typeName = comparator.getClass().getSimpleName(); + ComparatorType type = ComparatorType.valueOf(typeName); + this.type = typeName; + switch (type) { + case BinaryComparator: + case BinaryPrefixComparator: + this.value = Base64.encodeBytes(comparator.getValue()); + break; + case BitComparator: + this.value = Base64.encodeBytes(comparator.getValue()); + this.op = ((BitComparator)comparator).getOperator().toString(); + break; + case NullComparator: + break; + case RegexStringComparator: + case SubstringComparator: + this.value = Bytes.toString(comparator.getValue()); + break; + default: + throw new RuntimeException("unhandled filter type: " + type); + } + } + + public WritableByteArrayComparable build() { + WritableByteArrayComparable comparator; + switch (ComparatorType.valueOf(type)) { + case BinaryComparator: + comparator = new BinaryComparator(Base64.decode(value)); + break; + case BinaryPrefixComparator: + comparator = new BinaryPrefixComparator(Base64.decode(value)); + break; + case BitComparator: + comparator = new BitComparator(Base64.decode(value), + BitComparator.BitwiseOp.valueOf(op)); + break; + case NullComparator: + comparator = new NullComparator(); + break; + case RegexStringComparator: + comparator = new RegexStringComparator(value); + break; + case SubstringComparator: + comparator = new SubstringComparator(value); + break; + default: + throw new RuntimeException("unhandled comparator type: " + type); + } + return comparator; + } + + } + + // A grab bag of fields, would have been a union if this were C. + // These are null by default and will only be serialized if set (non null). + @XmlAttribute public String type; + @XmlAttribute public String op; + @XmlElement WritableByteArrayComparableModel comparator; + @XmlAttribute public String value; + @XmlElement public List filters; + @XmlAttribute public Integer limit; + @XmlAttribute public Integer offset; + @XmlAttribute public String family; + @XmlAttribute public String qualifier; + @XmlAttribute public Boolean ifMissing; + @XmlAttribute public Boolean latestVersion; + @XmlAttribute public String minColumn; + @XmlAttribute public Boolean minColumnInclusive; + @XmlAttribute public String maxColumn; + @XmlAttribute public Boolean maxColumnInclusive; + @XmlAttribute public Boolean dropDependentColumn; + @XmlAttribute public Float chance; + @XmlElement public List prefixes; + @XmlElement public List timestamps; + + static enum FilterType { + ColumnCountGetFilter, + ColumnPaginationFilter, + ColumnPrefixFilter, + ColumnRangeFilter, + DependentColumnFilter, + FamilyFilter, + FilterList, + FirstKeyOnlyFilter, + InclusiveStopFilter, + KeyOnlyFilter, + MultipleColumnPrefixFilter, + PageFilter, + PrefixFilter, + QualifierFilter, + RandomRowFilter, + RowFilter, + SingleColumnValueExcludeFilter, + SingleColumnValueFilter, + SkipFilter, + TimestampsFilter, + ValueFilter, + WhileMatchFilter + } + + public FilterModel() { } + + public FilterModel(Filter filter) { + String typeName = filter.getClass().getSimpleName(); + FilterType type = FilterType.valueOf(typeName); + this.type = typeName; + switch (type) { + case ColumnCountGetFilter: + this.limit = ((ColumnCountGetFilter)filter).getLimit(); + break; + case ColumnPaginationFilter: + this.limit = ((ColumnPaginationFilter)filter).getLimit(); + this.offset = ((ColumnPaginationFilter)filter).getOffset(); + break; + case ColumnPrefixFilter: + this.value = Base64.encodeBytes(((ColumnPrefixFilter)filter).getPrefix()); + break; + case ColumnRangeFilter: + this.minColumn = Base64.encodeBytes(((ColumnRangeFilter)filter).getMinColumn()); + this.minColumnInclusive = ((ColumnRangeFilter)filter).getMinColumnInclusive(); + this.maxColumn = Base64.encodeBytes(((ColumnRangeFilter)filter).getMaxColumn()); + this.maxColumnInclusive = ((ColumnRangeFilter)filter).getMaxColumnInclusive(); + break; + case DependentColumnFilter: { + DependentColumnFilter dcf = (DependentColumnFilter)filter; + this.family = Base64.encodeBytes(dcf.getFamily()); + byte[] qualifier = dcf.getQualifier(); + if (qualifier != null) { + this.qualifier = Base64.encodeBytes(qualifier); + } + this.op = dcf.getOperator().toString(); + this.comparator = new WritableByteArrayComparableModel(dcf.getComparator()); + this.dropDependentColumn = dcf.dropDependentColumn(); + } break; + case FilterList: + this.op = ((FilterList)filter).getOperator().toString(); + this.filters = new ArrayList(); + for (Filter child: ((FilterList)filter).getFilters()) { + this.filters.add(new FilterModel(child)); + } + break; + case FirstKeyOnlyFilter: + case KeyOnlyFilter: + break; + case InclusiveStopFilter: + this.value = + Base64.encodeBytes(((InclusiveStopFilter)filter).getStopRowKey()); + break; + case MultipleColumnPrefixFilter: + this.prefixes = new ArrayList(); + for (byte[] prefix: ((MultipleColumnPrefixFilter)filter).getPrefix()) { + this.prefixes.add(Base64.encodeBytes(prefix)); + } + break; + case PageFilter: + this.value = Long.toString(((PageFilter)filter).getPageSize()); + break; + case PrefixFilter: + this.value = Base64.encodeBytes(((PrefixFilter)filter).getPrefix()); + break; + case FamilyFilter: + case QualifierFilter: + case RowFilter: + case ValueFilter: + this.op = ((CompareFilter)filter).getOperator().toString(); + this.comparator = + new WritableByteArrayComparableModel( + ((CompareFilter)filter).getComparator()); + break; + case RandomRowFilter: + this.chance = ((RandomRowFilter)filter).getChance(); + break; + case SingleColumnValueExcludeFilter: + case SingleColumnValueFilter: { + SingleColumnValueFilter scvf = (SingleColumnValueFilter) filter; + this.family = Base64.encodeBytes(scvf.getFamily()); + byte[] qualifier = scvf.getQualifier(); + if (qualifier != null) { + this.qualifier = Base64.encodeBytes(qualifier); + } + this.op = scvf.getOperator().toString(); + this.comparator = + new WritableByteArrayComparableModel(scvf.getComparator()); + if (scvf.getFilterIfMissing()) { + this.ifMissing = true; + } + if (scvf.getLatestVersionOnly()) { + this.latestVersion = true; + } + } break; + case SkipFilter: + this.filters = new ArrayList(); + this.filters.add(new FilterModel(((SkipFilter)filter).getFilter())); + break; + case TimestampsFilter: + this.timestamps = ((TimestampsFilter)filter).getTimestamps(); + break; + case WhileMatchFilter: + this.filters = new ArrayList(); + this.filters.add( + new FilterModel(((WhileMatchFilter)filter).getFilter())); + break; + default: + throw new RuntimeException("unhandled filter type " + type); + } + } + + public Filter build() { + Filter filter; + switch (FilterType.valueOf(type)) { + case ColumnCountGetFilter: + filter = new ColumnCountGetFilter(limit); + break; + case ColumnPaginationFilter: + filter = new ColumnPaginationFilter(limit, offset); + break; + case ColumnPrefixFilter: + filter = new ColumnPrefixFilter(Base64.decode(value)); + break; + case ColumnRangeFilter: + filter = new ColumnRangeFilter(Base64.decode(minColumn), + minColumnInclusive, Base64.decode(maxColumn), + maxColumnInclusive); + break; + case DependentColumnFilter: + filter = new DependentColumnFilter(Base64.decode(family), + qualifier != null ? Base64.decode(qualifier) : null, + dropDependentColumn, CompareOp.valueOf(op), comparator.build()); + break; + case FamilyFilter: + filter = new FamilyFilter(CompareOp.valueOf(op), comparator.build()); + break; + case FilterList: { + List list = new ArrayList(); + for (FilterModel model: filters) { + list.add(model.build()); + } + filter = new FilterList(FilterList.Operator.valueOf(op), list); + } break; + case FirstKeyOnlyFilter: + filter = new FirstKeyOnlyFilter(); + break; + case InclusiveStopFilter: + filter = new InclusiveStopFilter(Base64.decode(value)); + break; + case KeyOnlyFilter: + filter = new KeyOnlyFilter(); + break; + case MultipleColumnPrefixFilter: { + byte[][] values = new byte[prefixes.size()][]; + for (int i = 0; i < prefixes.size(); i++) { + values[i] = Base64.decode(prefixes.get(i)); + } + filter = new MultipleColumnPrefixFilter(values); + } break; + case PageFilter: + filter = new PageFilter(Long.valueOf(value)); + break; + case PrefixFilter: + filter = new PrefixFilter(Base64.decode(value)); + break; + case QualifierFilter: + filter = new QualifierFilter(CompareOp.valueOf(op), comparator.build()); + break; + case RandomRowFilter: + filter = new RandomRowFilter(chance); + break; + case RowFilter: + filter = new RowFilter(CompareOp.valueOf(op), comparator.build()); + break; + case SingleColumnValueFilter: + filter = new SingleColumnValueFilter(Base64.decode(family), + qualifier != null ? Base64.decode(qualifier) : null, + CompareOp.valueOf(op), comparator.build()); + if (ifMissing != null) { + ((SingleColumnValueFilter)filter).setFilterIfMissing(ifMissing); + } + if (latestVersion != null) { + ((SingleColumnValueFilter)filter).setLatestVersionOnly(latestVersion); + } + break; + case SingleColumnValueExcludeFilter: + filter = new SingleColumnValueExcludeFilter(Base64.decode(family), + qualifier != null ? Base64.decode(qualifier) : null, + CompareOp.valueOf(op), comparator.build()); + if (ifMissing != null) { + ((SingleColumnValueExcludeFilter)filter).setFilterIfMissing(ifMissing); + } + if (latestVersion != null) { + ((SingleColumnValueExcludeFilter)filter).setLatestVersionOnly(latestVersion); + } + break; + case SkipFilter: + filter = new SkipFilter(filters.get(0).build()); + break; + case TimestampsFilter: + filter = new TimestampsFilter(timestamps); + break; + case ValueFilter: + filter = new ValueFilter(CompareOp.valueOf(op), comparator.build()); + break; + case WhileMatchFilter: + filter = new WhileMatchFilter(filters.get(0).build()); + break; + default: + throw new RuntimeException("unhandled filter type: " + type); + } + return filter; + } + + } + + /** + * @param s the JSON representation of the filter + * @return the filter + * @throws Exception + */ + public static Filter buildFilter(String s) throws Exception { + JSONJAXBContext context = + new JSONJAXBContext(JSONConfiguration.natural().build(), + FilterModel.class); + JSONUnmarshaller unmarshaller = context.createJSONUnmarshaller(); + FilterModel model = unmarshaller.unmarshalFromJSON(new StringReader(s), + FilterModel.class); + return model.build(); + } + + /** + * @param filter the filter + * @return the JSON representation of the filter + * @throws Exception + */ + public static String stringifyFilter(final Filter filter) throws Exception { + JSONJAXBContext context = + new JSONJAXBContext(JSONConfiguration.natural().build(), + FilterModel.class); + JSONMarshaller marshaller = context.createJSONMarshaller(); + StringWriter writer = new StringWriter(); + marshaller.marshallToJSON(new FilterModel(filter), writer); + return writer.toString(); + } + + private static final byte[] COLUMN_DIVIDER = Bytes.toBytes(":"); + + /** + * @param scan the scan specification + * @throws Exception + */ + public static ScannerModel fromScan(Scan scan) throws Exception { + ScannerModel model = new ScannerModel(); + model.setStartRow(scan.getStartRow()); + model.setEndRow(scan.getStopRow()); + Map> families = scan.getFamilyMap(); + if (families != null) { + for (Map.Entry> entry : families.entrySet()) { + if (entry.getValue() != null) { + for (byte[] qualifier: entry.getValue()) { + model.addColumn(Bytes.add(entry.getKey(), COLUMN_DIVIDER, qualifier)); + } + } else { + model.addColumn(entry.getKey()); + } + } + } + model.setStartTime(scan.getTimeRange().getMin()); + model.setEndTime(scan.getTimeRange().getMax()); + int caching = scan.getCaching(); + if (caching > 0) { + model.setBatch(caching); + } + int maxVersions = scan.getMaxVersions(); + if (maxVersions > 0) { + model.setMaxVersions(maxVersions); + } + Filter filter = scan.getFilter(); + if (filter != null) { + model.setFilter(stringifyFilter(filter)); + } + return model; + } + + /** + * Default constructor + */ + public ScannerModel() {} + + /** + * Constructor + * @param startRow the start key of the row-range + * @param endRow the end key of the row-range + * @param columns the columns to scan + * @param batch the number of values to return in batch + * @param endTime the upper bound on timestamps of values of interest + * @param maxVersions the maximum number of versions to return + * @param filter a filter specification + * (values with timestamps later than this are excluded) + */ + public ScannerModel(byte[] startRow, byte[] endRow, List columns, + int batch, long endTime, int maxVersions, String filter) { + super(); + this.startRow = startRow; + this.endRow = endRow; + this.columns = columns; + this.batch = batch; + this.endTime = endTime; + this.maxVersions = maxVersions; + this.filter = filter; + } + + /** + * Constructor + * @param startRow the start key of the row-range + * @param endRow the end key of the row-range + * @param columns the columns to scan + * @param batch the number of values to return in batch + * @param startTime the lower bound on timestamps of values of interest + * (values with timestamps earlier than this are excluded) + * @param endTime the upper bound on timestamps of values of interest + * (values with timestamps later than this are excluded) + * @param filter a filter specification + */ + public ScannerModel(byte[] startRow, byte[] endRow, List columns, + int batch, long startTime, long endTime, String filter) { + super(); + this.startRow = startRow; + this.endRow = endRow; + this.columns = columns; + this.batch = batch; + this.startTime = startTime; + this.endTime = endTime; + this.filter = filter; + } + + /** + * Add a column to the column set + * @param column the column name, as <column>(:<qualifier>)? + */ + public void addColumn(byte[] column) { + columns.add(column); + } + + /** + * @return true if a start row was specified + */ + public boolean hasStartRow() { + return !Bytes.equals(startRow, HConstants.EMPTY_START_ROW); + } + + /** + * @return start row + */ + @XmlAttribute + public byte[] getStartRow() { + return startRow; + } + + /** + * @return true if an end row was specified + */ + public boolean hasEndRow() { + return !Bytes.equals(endRow, HConstants.EMPTY_END_ROW); + } + + /** + * @return end row + */ + @XmlAttribute + public byte[] getEndRow() { + return endRow; + } + + /** + * @return list of columns of interest in column:qualifier format, or empty for all + */ + @XmlElement(name="column") + public List getColumns() { + return columns; + } + + /** + * @return the number of cells to return in batch + */ + @XmlAttribute + public int getBatch() { + return batch; + } + + /** + * @return the lower bound on timestamps of items of interest + */ + @XmlAttribute + public long getStartTime() { + return startTime; + } + + /** + * @return the upper bound on timestamps of items of interest + */ + @XmlAttribute + public long getEndTime() { + return endTime; + } + + /** + * @return maximum number of versions to return + */ + @XmlAttribute + public int getMaxVersions() { + return maxVersions; + } + + /** + * @return the filter specification + */ + @XmlElement + public String getFilter() { + return filter; + } + + /** + * @param startRow start row + */ + public void setStartRow(byte[] startRow) { + this.startRow = startRow; + } + + /** + * @param endRow end row + */ + public void setEndRow(byte[] endRow) { + this.endRow = endRow; + } + + /** + * @param columns list of columns of interest in column:qualifier format, or empty for all + */ + public void setColumns(List columns) { + this.columns = columns; + } + + /** + * @param batch the number of cells to return in batch + */ + public void setBatch(int batch) { + this.batch = batch; + } + + /** + * @param maxVersions maximum number of versions to return + */ + public void setMaxVersions(int maxVersions) { + this.maxVersions = maxVersions; + } + + /** + * @param startTime the lower bound on timestamps of values of interest + */ + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + /** + * @param endTime the upper bound on timestamps of values of interest + */ + public void setEndTime(long endTime) { + this.endTime = endTime; + } + + /** + * @param filter the filter specification + */ + public void setFilter(String filter) { + this.filter = filter; + } + + @Override + public byte[] createProtobufOutput() { + Scanner.Builder builder = Scanner.newBuilder(); + if (!Bytes.equals(startRow, HConstants.EMPTY_START_ROW)) { + builder.setStartRow(ByteString.copyFrom(startRow)); + } + if (!Bytes.equals(endRow, HConstants.EMPTY_START_ROW)) { + builder.setEndRow(ByteString.copyFrom(endRow)); + } + for (byte[] column: columns) { + builder.addColumns(ByteString.copyFrom(column)); + } + builder.setBatch(batch); + if (startTime != 0) { + builder.setStartTime(startTime); + } + if (endTime != 0) { + builder.setEndTime(endTime); + } + builder.setBatch(getBatch()); + builder.setMaxVersions(maxVersions); + if (filter != null) { + builder.setFilter(filter); + } + return builder.build().toByteArray(); + } + + @Override + public ProtobufMessageHandler getObjectFromMessage(byte[] message) + throws IOException { + Scanner.Builder builder = Scanner.newBuilder(); + builder.mergeFrom(message); + if (builder.hasStartRow()) { + startRow = builder.getStartRow().toByteArray(); + } + if (builder.hasEndRow()) { + endRow = builder.getEndRow().toByteArray(); + } + for (ByteString column: builder.getColumnsList()) { + addColumn(column.toByteArray()); + } + if (builder.hasBatch()) { + batch = builder.getBatch(); + } + if (builder.hasStartTime()) { + startTime = builder.getStartTime(); + } + if (builder.hasEndTime()) { + endTime = builder.getEndTime(); + } + if (builder.hasMaxVersions()) { + maxVersions = builder.getMaxVersions(); + } + if (builder.hasFilter()) { + filter = builder.getFilter(); + } + return this; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java new file mode 100644 index 0000000..c976c31 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java @@ -0,0 +1,787 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; +import org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus; +import org.apache.hadoop.hbase.util.Bytes; + +import com.google.protobuf.ByteString; + +/** + * Representation of the status of a storage cluster: + *

    + *

      + *
    • regions: the total number of regions served by the cluster
      • + *
    • requests: the total number of requests per second handled by the + * cluster in the last reporting interval
      • + *
    • averageLoad: the average load of the region servers in the cluster
      • + *
    • liveNodes: detailed status of the live region servers
      • + *
    • deadNodes: the names of region servers declared dead
      • + *
    + * + * 
    + * <complexType name="StorageClusterStatus">
+ *   <sequence>
+ *     <element name="liveNode" type="tns:Node"
+ *       maxOccurs="unbounded" minOccurs="0">
+ *     </element>
+ *     <element name="deadNode" type="string" maxOccurs="unbounded"
+ *       minOccurs="0">
+ *     </element>
+ *   </sequence>
+ *   <attribute name="regions" type="int"></attribute>
+ *   <attribute name="requests" type="int"></attribute>
+ *   <attribute name="averageLoad" type="float"></attribute>
+ * </complexType>
+ *
+ * <complexType name="Node">
+ *   <sequence>
+ *     <element name="region" type="tns:Region" 
+ *       maxOccurs="unbounded" minOccurs="0"></element>
+ *   </sequence>
+ *   <attribute name="name" type="string"></attribute>
+ *   <attribute name="startCode" type="int"></attribute>
+ *   <attribute name="requests" type="int"></attribute>
+ *   <attribute name="heapSizeMB" type="int"></attribute>
+ *   <attribute name="maxHeapSizeMB" type="int"></attribute>
+ * </complexType>
+ *
+ * <complexType name="Region">
+ *   <attribute name="name" type="base64Binary"></attribute>
+ *   <attribute name="stores" type="int"></attribute>
+ *   <attribute name="storefiles" type="int"></attribute>
+ *   <attribute name="storefileSizeMB" type="int"></attribute>
+ *   <attribute name="memstoreSizeMB" type="int"></attribute>
+ *   <attribute name="storefileIndexSizeMB" type="int"></attribute>
+ *   <attribute name="readRequestsCount" type="int"></attribute>
+ *   <attribute name="writeRequestsCount" type="int"></attribute>
+ *   <attribute name="rootIndexSizeKB" type="int"></attribute>
+ *   <attribute name="totalStaticIndexSizeKB" type="int"></attribute>
+ *   <attribute name="totalStaticBloomSizeKB" type="int"></attribute>
+ *   <attribute name="totalCompactingKVs" type="int"></attribute>
+ *   <attribute name="currentCompactedKVs" type="int"></attribute>
+ * </complexType>
+ *
    + */ +@XmlRootElement(name="ClusterStatus") +public class StorageClusterStatusModel + implements Serializable, ProtobufMessageHandler { + private static final long serialVersionUID = 1L; + + /** + * Represents a region server. + */ + public static class Node { + + /** + * Represents a region hosted on a region server. + */ + public static class Region { + private byte[] name; + private int stores; + private int storefiles; + private int storefileSizeMB; + private int memstoreSizeMB; + private int storefileIndexSizeMB; + private long readRequestsCount; + private long writeRequestsCount; + private int rootIndexSizeKB; + private int totalStaticIndexSizeKB; + private int totalStaticBloomSizeKB; + private long totalCompactingKVs; + private long currentCompactedKVs; + + /** + * Default constructor + */ + public Region() {} + + /** + * Constructor + * @param name the region name + */ + public Region(byte[] name) { + this.name = name; + } + + /** + * Constructor + * @param name the region name + * @param stores the number of stores + * @param storefiles the number of store files + * @param storefileSizeMB total size of store files, in MB + * @param memstoreSizeMB total size of memstore, in MB + * @param storefileIndexSizeMB total size of store file indexes, in MB + */ + public Region(byte[] name, int stores, int storefiles, + int storefileSizeMB, int memstoreSizeMB, int storefileIndexSizeMB, + long readRequestsCount, long writeRequestsCount, int rootIndexSizeKB, + int totalStaticIndexSizeKB, int totalStaticBloomSizeKB, + long totalCompactingKVs, long currentCompactedKVs) { + this.name = name; + this.stores = stores; + this.storefiles = storefiles; + this.storefileSizeMB = storefileSizeMB; + this.memstoreSizeMB = memstoreSizeMB; + this.storefileIndexSizeMB = storefileIndexSizeMB; + this.readRequestsCount = readRequestsCount; + this.writeRequestsCount = writeRequestsCount; + this.rootIndexSizeKB = rootIndexSizeKB; + this.totalStaticIndexSizeKB = totalStaticIndexSizeKB; + this.totalStaticBloomSizeKB = totalStaticBloomSizeKB; + this.totalCompactingKVs = totalCompactingKVs; + this.currentCompactedKVs = currentCompactedKVs; + } + + /** + * @return the region name + */ + @XmlAttribute + public byte[] getName() { + return name; + } + + /** + * @return the number of stores + */ + @XmlAttribute + public int getStores() { + return stores; + } + + /** + * @return the number of store files + */ + @XmlAttribute + public int getStorefiles() { + return storefiles; + } + + /** + * @return the total size of store files, in MB + */ + @XmlAttribute + public int getStorefileSizeMB() { + return storefileSizeMB; + } + + /** + * @return memstore size, in MB + */ + @XmlAttribute + public int getMemstoreSizeMB() { + return memstoreSizeMB; + } + + /** + * @return the total size of store file indexes, in MB + */ + @XmlAttribute + public int getStorefileIndexSizeMB() { + return storefileIndexSizeMB; + } + + /** + * @return the current total read requests made to region + */ + @XmlAttribute + public long getReadRequestsCount() { + return readRequestsCount; + } + + /** + * @return the current total write requests made to region + */ + @XmlAttribute + public long getWriteRequestsCount() { + return writeRequestsCount; + } + + /** + * @return The current total size of root-level indexes for the region, in KB. + */ + @XmlAttribute + public int getRootIndexSizeKB() { + return rootIndexSizeKB; + } + + /** + * @return The total size of static index, in KB + */ + @XmlAttribute + public int getTotalStaticIndexSizeKB() { + return totalStaticIndexSizeKB; + } + + /** + * @return The total size of static bloom, in KB + */ + @XmlAttribute + public int getTotalStaticBloomSizeKB() { + return totalStaticBloomSizeKB; + } + + /** + * @return The total number of compacting key-values + */ + @XmlAttribute + public long getTotalCompactingKVs() { + return totalCompactingKVs; + } + + /** + * @return The number of current compacted key-values + */ + @XmlAttribute + public long getCurrentCompactedKVs() { + return currentCompactedKVs; + } + + /** + * @param readRequestsCount The current total read requests made to region + */ + public void setReadRequestsCount(long readRequestsCount) { + this.readRequestsCount = readRequestsCount; + } + + /** + * @param rootIndexSizeKB The current total size of root-level indexes + * for the region, in KB + */ + public void setRootIndexSizeKB(int rootIndexSizeKB) { + this.rootIndexSizeKB = rootIndexSizeKB; + } + + /** + * @param writeRequestsCount The current total write requests made to region + */ + public void setWriteRequestsCount(long writeRequestsCount) { + this.writeRequestsCount = writeRequestsCount; + } + + /** + * @param currentCompactedKVs The completed count of key values + * in currently running compaction + */ + public void setCurrentCompactedKVs(long currentCompactedKVs) { + this.currentCompactedKVs = currentCompactedKVs; + } + + /** + * @param totalCompactingKVs The total compacting key values + * in currently running compaction + */ + public void setTotalCompactingKVs(long totalCompactingKVs) { + this.totalCompactingKVs = totalCompactingKVs; + } + + /** + * @param totalStaticBloomSizeKB The total size of all Bloom filter blocks, + * not just loaded into the block cache, in KB. + */ + public void setTotalStaticBloomSizeKB(int totalStaticBloomSizeKB) { + this.totalStaticBloomSizeKB = totalStaticBloomSizeKB; + } + + /** + * @param totalStaticIndexSizeKB The total size of all index blocks, + * not just the root level, in KB. + */ + public void setTotalStaticIndexSizeKB(int totalStaticIndexSizeKB) { + this.totalStaticIndexSizeKB = totalStaticIndexSizeKB; + } + + /** + * @param name the region name + */ + public void setName(byte[] name) { + this.name = name; + } + + /** + * @param stores the number of stores + */ + public void setStores(int stores) { + this.stores = stores; + } + + /** + * @param storefiles the number of store files + */ + public void setStorefiles(int storefiles) { + this.storefiles = storefiles; + } + + /** + * @param storefileSizeMB total size of store files, in MB + */ + public void setStorefileSizeMB(int storefileSizeMB) { + this.storefileSizeMB = storefileSizeMB; + } + + /** + * @param memstoreSizeMB memstore size, in MB + */ + public void setMemstoreSizeMB(int memstoreSizeMB) { + this.memstoreSizeMB = memstoreSizeMB; + } + + /** + * @param storefileIndexSizeMB total size of store file indexes, in MB + */ + public void setStorefileIndexSizeMB(int storefileIndexSizeMB) { + this.storefileIndexSizeMB = storefileIndexSizeMB; + } + } + + private String name; + private long startCode; + private int requests; + private int heapSizeMB; + private int maxHeapSizeMB; + private List regions = new ArrayList(); + + /** + * Add a region name to the list + * @param name the region name + */ + public void addRegion(byte[] name, int stores, int storefiles, + int storefileSizeMB, int memstoreSizeMB, int storefileIndexSizeMB, + long readRequestsCount, long writeRequestsCount, int rootIndexSizeKB, + int totalStaticIndexSizeKB, int totalStaticBloomSizeKB, + long totalCompactingKVs, long currentCompactedKVs) { + regions.add(new Region(name, stores, storefiles, storefileSizeMB, + memstoreSizeMB, storefileIndexSizeMB, readRequestsCount, + writeRequestsCount, rootIndexSizeKB, totalStaticIndexSizeKB, + totalStaticBloomSizeKB, totalCompactingKVs, currentCompactedKVs)); + } + + /** + * @param index the index + * @return the region name + */ + public Region getRegion(int index) { + return regions.get(index); + } + + /** + * Default constructor + */ + public Node() {} + + /** + * Constructor + * @param name the region server name + * @param startCode the region server's start code + */ + public Node(String name, long startCode) { + this.name = name; + this.startCode = startCode; + } + + /** + * @return the region server's name + */ + @XmlAttribute + public String getName() { + return name; + } + + /** + * @return the region server's start code + */ + @XmlAttribute + public long getStartCode() { + return startCode; + } + + /** + * @return the current heap size, in MB + */ + @XmlAttribute + public int getHeapSizeMB() { + return heapSizeMB; + } + + /** + * @return the maximum heap size, in MB + */ + @XmlAttribute + public int getMaxHeapSizeMB() { + return maxHeapSizeMB; + } + + /** + * @return the list of regions served by the region server + */ + @XmlElement(name="Region") + public List getRegions() { + return regions; + } + + /** + * @return the number of requests per second processed by the region server + */ + @XmlAttribute + public int getRequests() { + return requests; + } + + /** + * @param name the region server's hostname + */ + public void setName(String name) { + this.name = name; + } + + /** + * @param startCode the region server's start code + */ + public void setStartCode(long startCode) { + this.startCode = startCode; + } + + /** + * @param heapSizeMB the current heap size, in MB + */ + public void setHeapSizeMB(int heapSizeMB) { + this.heapSizeMB = heapSizeMB; + } + + /** + * @param maxHeapSizeMB the maximum heap size, in MB + */ + public void setMaxHeapSizeMB(int maxHeapSizeMB) { + this.maxHeapSizeMB = maxHeapSizeMB; + } + + /** + * @param regions a list of regions served by the region server + */ + public void setRegions(List regions) { + this.regions = regions; + } + + /** + * @param requests the number of requests per second processed by the + * region server + */ + public void setRequests(int requests) { + this.requests = requests; + } + } + + private List liveNodes = new ArrayList(); + private List deadNodes = new ArrayList(); + private int regions; + private int requests; + private double averageLoad; + + /** + * Add a live node to the cluster representation. + * @param name the region server name + * @param startCode the region server's start code + * @param heapSizeMB the current heap size, in MB + * @param maxHeapSizeMB the maximum heap size, in MB + */ + public Node addLiveNode(String name, long startCode, int heapSizeMB, + int maxHeapSizeMB) { + Node node = new Node(name, startCode); + node.setHeapSizeMB(heapSizeMB); + node.setMaxHeapSizeMB(maxHeapSizeMB); + liveNodes.add(node); + return node; + } + + /** + * @param index the index + * @return the region server model + */ + public Node getLiveNode(int index) { + return liveNodes.get(index); + } + + /** + * Add a dead node to the cluster representation. + * @param node the dead region server's name + */ + public void addDeadNode(String node) { + deadNodes.add(node); + } + + /** + * @param index the index + * @return the dead region server's name + */ + public String getDeadNode(int index) { + return deadNodes.get(index); + } + + /** + * Default constructor + */ + public StorageClusterStatusModel() {} + + /** + * @return the list of live nodes + */ + @XmlElement(name="Node") + @XmlElementWrapper(name="LiveNodes") + public List getLiveNodes() { + return liveNodes; + } + + /** + * @return the list of dead nodes + */ + @XmlElement(name="Node") + @XmlElementWrapper(name="DeadNodes") + public List getDeadNodes() { + return deadNodes; + } + + /** + * @return the total number of regions served by the cluster + */ + @XmlAttribute + public int getRegions() { + return regions; + } + + /** + * @return the total number of requests per second handled by the cluster in + * the last reporting interval + */ + @XmlAttribute + public int getRequests() { + return requests; + } + + /** + * @return the average load of the region servers in the cluster + */ + @XmlAttribute + public double getAverageLoad() { + return averageLoad; + } + + /** + * @param nodes the list of live node models + */ + public void setLiveNodes(List nodes) { + this.liveNodes = nodes; + } + + /** + * @param nodes the list of dead node names + */ + public void setDeadNodes(List nodes) { + this.deadNodes = nodes; + } + + /** + * @param regions the total number of regions served by the cluster + */ + public void setRegions(int regions) { + this.regions = regions; + } + + /** + * @param requests the total number of requests per second handled by the + * cluster + */ + public void setRequests(int requests) { + this.requests = requests; + } + + /** + * @param averageLoad the average load of region servers in the cluster + */ + public void setAverageLoad(double averageLoad) { + this.averageLoad = averageLoad; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(String.format("%d live servers, %d dead servers, " + + "%.4f average load\n\n", liveNodes.size(), deadNodes.size(), + averageLoad)); + if (!liveNodes.isEmpty()) { + sb.append(liveNodes.size()); + sb.append(" live servers\n"); + for (Node node: liveNodes) { + sb.append(" "); + sb.append(node.name); + sb.append(' '); + sb.append(node.startCode); + sb.append("\n requests="); + sb.append(node.requests); + sb.append(", regions="); + sb.append(node.regions.size()); + sb.append("\n heapSizeMB="); + sb.append(node.heapSizeMB); + sb.append("\n maxHeapSizeMB="); + sb.append(node.maxHeapSizeMB); + sb.append("\n\n"); + for (Node.Region region: node.regions) { + sb.append(" "); + sb.append(Bytes.toString(region.name)); + sb.append("\n stores="); + sb.append(region.stores); + sb.append("\n storefiless="); + sb.append(region.storefiles); + sb.append("\n storefileSizeMB="); + sb.append(region.storefileSizeMB); + sb.append("\n memstoreSizeMB="); + sb.append(region.memstoreSizeMB); + sb.append("\n storefileIndexSizeMB="); + sb.append(region.storefileIndexSizeMB); + sb.append("\n readRequestsCount="); + sb.append(region.readRequestsCount); + sb.append("\n writeRequestsCount="); + sb.append(region.writeRequestsCount); + sb.append("\n rootIndexSizeKB="); + sb.append(region.rootIndexSizeKB); + sb.append("\n totalStaticIndexSizeKB="); + sb.append(region.totalStaticIndexSizeKB); + sb.append("\n totalStaticBloomSizeKB="); + sb.append(region.totalStaticBloomSizeKB); + sb.append("\n totalCompactingKVs="); + sb.append(region.totalCompactingKVs); + sb.append("\n currentCompactedKVs="); + sb.append(region.currentCompactedKVs); + sb.append('\n'); + } + sb.append('\n'); + } + } + if (!deadNodes.isEmpty()) { + sb.append('\n'); + sb.append(deadNodes.size()); + sb.append(" dead servers\n"); + for (String node: deadNodes) { + sb.append(" "); + sb.append(node); + sb.append('\n'); + } + } + return sb.toString(); + } + + @Override + public byte[] createProtobufOutput() { + StorageClusterStatus.Builder builder = StorageClusterStatus.newBuilder(); + builder.setRegions(regions); + builder.setRequests(requests); + builder.setAverageLoad(averageLoad); + for (Node node: liveNodes) { + StorageClusterStatus.Node.Builder nodeBuilder = + StorageClusterStatus.Node.newBuilder(); + nodeBuilder.setName(node.name); + nodeBuilder.setStartCode(node.startCode); + nodeBuilder.setRequests(node.requests); + nodeBuilder.setHeapSizeMB(node.heapSizeMB); + nodeBuilder.setMaxHeapSizeMB(node.maxHeapSizeMB); + for (Node.Region region: node.regions) { + StorageClusterStatus.Region.Builder regionBuilder = + StorageClusterStatus.Region.newBuilder(); + regionBuilder.setName(ByteString.copyFrom(region.name)); + regionBuilder.setStores(region.stores); + regionBuilder.setStorefiles(region.storefiles); + regionBuilder.setStorefileSizeMB(region.storefileSizeMB); + regionBuilder.setMemstoreSizeMB(region.memstoreSizeMB); + regionBuilder.setStorefileIndexSizeMB(region.storefileIndexSizeMB); + regionBuilder.setReadRequestsCount(region.readRequestsCount); + regionBuilder.setWriteRequestsCount(region.writeRequestsCount); + regionBuilder.setRootIndexSizeKB(region.rootIndexSizeKB); + regionBuilder.setTotalStaticIndexSizeKB(region.totalStaticIndexSizeKB); + regionBuilder.setTotalStaticBloomSizeKB(region.totalStaticBloomSizeKB); + regionBuilder.setTotalCompactingKVs(region.totalCompactingKVs); + regionBuilder.setCurrentCompactedKVs(region.currentCompactedKVs); + nodeBuilder.addRegions(regionBuilder); + } + builder.addLiveNodes(nodeBuilder); + } + for (String node: deadNodes) { + builder.addDeadNodes(node); + } + return builder.build().toByteArray(); + } + + @Override + public ProtobufMessageHandler getObjectFromMessage(byte[] message) + throws IOException { + StorageClusterStatus.Builder builder = StorageClusterStatus.newBuilder(); + builder.mergeFrom(message); + if (builder.hasRegions()) { + regions = builder.getRegions(); + } + if (builder.hasRequests()) { + requests = builder.getRequests(); + } + if (builder.hasAverageLoad()) { + averageLoad = builder.getAverageLoad(); + } + for (StorageClusterStatus.Node node: builder.getLiveNodesList()) { + long startCode = node.hasStartCode() ? node.getStartCode() : -1; + StorageClusterStatusModel.Node nodeModel = + addLiveNode(node.getName(), startCode, node.getHeapSizeMB(), + node.getMaxHeapSizeMB()); + int requests = node.hasRequests() ? node.getRequests() : 0; + nodeModel.setRequests(requests); + for (StorageClusterStatus.Region region: node.getRegionsList()) { + nodeModel.addRegion( + region.getName().toByteArray(), + region.getStores(), + region.getStorefiles(), + region.getStorefileSizeMB(), + region.getMemstoreSizeMB(), + region.getStorefileIndexSizeMB(), + region.getReadRequestsCount(), + region.getWriteRequestsCount(), + region.getRootIndexSizeKB(), + region.getTotalStaticIndexSizeKB(), + region.getTotalStaticBloomSizeKB(), + region.getTotalCompactingKVs(), + region.getCurrentCompactedKVs()); + } + } + for (String node: builder.getDeadNodesList()) { + addDeadNode(node); + } + return this; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterVersionModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterVersionModel.java new file mode 100644 index 0000000..0563479 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterVersionModel.java @@ -0,0 +1,65 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlValue; + +/** + * Simple representation of the version of the storage cluster + * + * 
    + * <complexType name="StorageClusterVersion">
+ *   <attribute name="version" type="string"></attribute>
+ * </complexType>
+ *
    + */ +@XmlRootElement(name="ClusterVersion") +public class StorageClusterVersionModel implements Serializable { + private static final long serialVersionUID = 1L; + + private String version; + + /** + * @return the storage cluster version + */ + @XmlValue + public String getVersion() { + return version; + } + + /** + * @param version the storage cluster version + */ + public void setVersion(String version) { + this.version = version; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return version; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java new file mode 100644 index 0000000..ce6fb96 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java @@ -0,0 +1,159 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; +import org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo; + +import com.google.protobuf.ByteString; + +/** + * Representation of a list of table regions. + * + * 
    + * <complexType name="TableInfo">
+ *   <sequence>
+ *     <element name="region" type="tns:TableRegion" 
+ *       maxOccurs="unbounded" minOccurs="1"></element>
+ *   </sequence>
+ *   <attribute name="name" type="string"></attribute>
+ * </complexType>
+ *
    + */ +@XmlRootElement(name="TableInfo") +public class TableInfoModel implements Serializable, ProtobufMessageHandler { + private static final long serialVersionUID = 1L; + + private String name; + private List regions = new ArrayList(); + + /** + * Default constructor + */ + public TableInfoModel() {} + + /** + * Constructor + * @param name + */ + public TableInfoModel(String name) { + this.name = name; + } + + /** + * Add a region model to the list + * @param region the region + */ + public void add(TableRegionModel region) { + regions.add(region); + } + + /** + * @param index the index + * @return the region model + */ + public TableRegionModel get(int index) { + return regions.get(index); + } + + /** + * @return the table name + */ + @XmlAttribute + public String getName() { + return name; + } + + /** + * @return the regions + */ + @XmlElement(name="Region") + public List getRegions() { + return regions; + } + + /** + * @param name the table name + */ + public void setName(String name) { + this.name = name; + } + + /** + * @param regions the regions to set + */ + public void setRegions(List regions) { + this.regions = regions; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for(TableRegionModel aRegion : regions) { + sb.append(aRegion.toString()); + sb.append('\n'); + } + return sb.toString(); + } + + @Override + public byte[] createProtobufOutput() { + TableInfo.Builder builder = TableInfo.newBuilder(); + builder.setName(name); + for (TableRegionModel aRegion: regions) { + TableInfo.Region.Builder regionBuilder = TableInfo.Region.newBuilder(); + regionBuilder.setName(aRegion.getName()); + regionBuilder.setId(aRegion.getId()); + regionBuilder.setStartKey(ByteString.copyFrom(aRegion.getStartKey())); + regionBuilder.setEndKey(ByteString.copyFrom(aRegion.getEndKey())); + regionBuilder.setLocation(aRegion.getLocation()); + builder.addRegions(regionBuilder); + } + return builder.build().toByteArray(); + } + + @Override + public ProtobufMessageHandler getObjectFromMessage(byte[] message) + throws IOException { + TableInfo.Builder builder = TableInfo.newBuilder(); + builder.mergeFrom(message); + setName(builder.getName()); + for (TableInfo.Region region: builder.getRegionsList()) { + add(new TableRegionModel(builder.getName(), region.getId(), + region.getStartKey().toByteArray(), + region.getEndKey().toByteArray(), + region.getLocation())); + } + return this; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java new file mode 100644 index 0000000..1c276c2 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java @@ -0,0 +1,112 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlElementRef; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; +import org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList; + +/** + * Simple representation of a list of table names. + */ +@XmlRootElement(name="TableList") +public class TableListModel implements Serializable, ProtobufMessageHandler { + + private static final long serialVersionUID = 1L; + + private List tables = new ArrayList(); + + /** + * Default constructor + */ + public TableListModel() {} + + /** + * Add the table name model to the list + * @param table the table model + */ + public void add(TableModel table) { + tables.add(table); + } + + /** + * @param index the index + * @return the table model + */ + public TableModel get(int index) { + return tables.get(index); + } + + /** + * @return the tables + */ + @XmlElementRef(name="table") + public List getTables() { + return tables; + } + + /** + * @param tables the tables to set + */ + public void setTables(List tables) { + this.tables = tables; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for(TableModel aTable : tables) { + sb.append(aTable.toString()); + sb.append('\n'); + } + return sb.toString(); + } + + @Override + public byte[] createProtobufOutput() { + TableList.Builder builder = TableList.newBuilder(); + for (TableModel aTable : tables) { + builder.addName(aTable.getName()); + } + return builder.build().toByteArray(); + } + + @Override + public ProtobufMessageHandler getObjectFromMessage(byte[] message) + throws IOException { + TableList.Builder builder = TableList.newBuilder(); + builder.mergeFrom(message); + for (String table: builder.getNameList()) { + this.add(new TableModel(table)); + } + return this; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/TableModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/TableModel.java new file mode 100644 index 0000000..e1d33cd --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/TableModel.java @@ -0,0 +1,82 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Simple representation of a table name. + * + * 
    + * <complexType name="Table">
+ *   <sequence>
+ *     <element name="name" type="string"></element>
+ *   </sequence>
+ * </complexType>
+ *
    + */ +@XmlRootElement(name="table") +public class TableModel implements Serializable { + + private static final long serialVersionUID = 1L; + + private String name; + + /** + * Default constructor + */ + public TableModel() {} + + /** + * Constructor + * @param name + */ + public TableModel(String name) { + super(); + this.name = name; + } + + /** + * @return the name + */ + @XmlAttribute + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return this.name; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/TableRegionModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/TableRegionModel.java new file mode 100644 index 0000000..67e7a04 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/TableRegionModel.java @@ -0,0 +1,195 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Representation of a region of a table and its current location on the + * storage cluster. + * + * 
    + * <complexType name="TableRegion">
+ *   <attribute name="name" type="string"></attribute>
+ *   <attribute name="id" type="int"></attribute>
+ *   <attribute name="startKey" type="base64Binary"></attribute>
+ *   <attribute name="endKey" type="base64Binary"></attribute>
+ *   <attribute name="location" type="string"></attribute>
+ *  </complexType>
+ *
    + */ +@XmlRootElement(name="Region") +public class TableRegionModel implements Serializable { + + private static final long serialVersionUID = 1L; + + private String table; + private long id; + private byte[] startKey; + private byte[] endKey; + private String location; + + /** + * Constructor + */ + public TableRegionModel() {} + + /** + * Constructor + * @param table the table name + * @param id the encoded id of the region + * @param startKey the start key of the region + * @param endKey the end key of the region + */ + public TableRegionModel(String table, long id, byte[] startKey, + byte[] endKey) { + this(table, id, startKey, endKey, null); + } + + /** + * Constructor + * @param table the table name + * @param id the encoded id of the region + * @param startKey the start key of the region + * @param endKey the end key of the region + * @param location the name and port of the region server hosting the region + */ + public TableRegionModel(String table, long id, byte[] startKey, + byte[] endKey, String location) { + this.table = table; + this.id = id; + this.startKey = startKey; + this.endKey = endKey; + this.location = location; + } + + /** + * @return the region name + */ + @XmlAttribute + public String getName() { + byte [] tableNameAsBytes = Bytes.toBytes(this.table); + byte [] nameAsBytes = HRegionInfo.createRegionName(tableNameAsBytes, + this.startKey, this.id, + !HTableDescriptor.isMetaTable(tableNameAsBytes)); + return Bytes.toString(nameAsBytes); + } + + /** + * @return the encoded region id + */ + @XmlAttribute + public long getId() { + return id; + } + + /** + * @return the start key + */ + @XmlAttribute + public byte[] getStartKey() { + return startKey; + } + + /** + * @return the end key + */ + @XmlAttribute + public byte[] getEndKey() { + return endKey; + } + + /** + * @return the name and port of the region server hosting the region + */ + @XmlAttribute + public String getLocation() { + return location; + } + + /** + * @param name region printable name + */ + public void setName(String name) { + String split[] = name.split(","); + this.table = split[0]; + this.startKey = Bytes.toBytes(split[1]); + String tail = split[2]; + split = tail.split("\\."); + id = Long.valueOf(split[0]); + } + + /** + * @param id the region's encoded id + */ + public void setId(long id) { + this.id = id; + } + + /** + * @param startKey the start key + */ + public void setStartKey(byte[] startKey) { + this.startKey = startKey; + } + + /** + * @param endKey the end key + */ + public void setEndKey(byte[] endKey) { + this.endKey = endKey; + } + + /** + * @param location the name and port of the region server hosting the region + */ + public void setLocation(String location) { + this.location = location; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getName()); + sb.append(" [\n id="); + sb.append(id); + sb.append("\n startKey='"); + sb.append(Bytes.toString(startKey)); + sb.append("'\n endKey='"); + sb.append(Bytes.toString(endKey)); + if (location != null) { + sb.append("'\n location='"); + sb.append(location); + } + sb.append("'\n]\n"); + return sb.toString(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java new file mode 100644 index 0000000..fa6e3a6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java @@ -0,0 +1,353 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAnyAttribute; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.namespace.QName; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; +import org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema; +import org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * A representation of HBase table descriptors. + * + * 
    + * <complexType name="TableSchema">
+ *   <sequence>
+ *     <element name="column" type="tns:ColumnSchema" 
+ *       maxOccurs="unbounded" minOccurs="1"></element>
+ *   </sequence>
+ *   <attribute name="name" type="string"></attribute>
+ *   <anyAttribute></anyAttribute>
+ * </complexType>
+ *
    + */ +@XmlRootElement(name="TableSchema") +public class TableSchemaModel implements Serializable, ProtobufMessageHandler { + private static final long serialVersionUID = 1L; + private static final QName IS_META = new QName(HTableDescriptor.IS_META); + private static final QName IS_ROOT = new QName(HTableDescriptor.IS_ROOT); + private static final QName READONLY = new QName(HTableDescriptor.READONLY); + private static final QName TTL = new QName(HColumnDescriptor.TTL); + private static final QName VERSIONS = new QName(HConstants.VERSIONS); + private static final QName COMPRESSION = + new QName(HColumnDescriptor.COMPRESSION); + + private String name; + private Map attrs = new HashMap(); + private List columns = new ArrayList(); + + /** + * Default constructor. + */ + public TableSchemaModel() {} + + /** + * Constructor + * @param htd the table descriptor + */ + public TableSchemaModel(HTableDescriptor htd) { + setName(htd.getNameAsString()); + for (Map.Entry e: + htd.getValues().entrySet()) { + addAttribute(Bytes.toString(e.getKey().get()), + Bytes.toString(e.getValue().get())); + } + for (HColumnDescriptor hcd: htd.getFamilies()) { + ColumnSchemaModel columnModel = new ColumnSchemaModel(); + columnModel.setName(hcd.getNameAsString()); + for (Map.Entry e: + hcd.getValues().entrySet()) { + columnModel.addAttribute(Bytes.toString(e.getKey().get()), + Bytes.toString(e.getValue().get())); + } + addColumnFamily(columnModel); + } + } + + /** + * Add an attribute to the table descriptor + * @param name attribute name + * @param value attribute value + */ + public void addAttribute(String name, Object value) { + attrs.put(new QName(name), value); + } + + /** + * Return a table descriptor value as a string. Calls toString() on the + * object stored in the descriptor value map. + * @param name the attribute name + * @return the attribute value + */ + public String getAttribute(String name) { + Object o = attrs.get(new QName(name)); + return o != null ? o.toString() : null; + } + + /** + * Add a column family to the table descriptor + * @param family the column family model + */ + public void addColumnFamily(ColumnSchemaModel family) { + columns.add(family); + } + + /** + * Retrieve the column family at the given index from the table descriptor + * @param index the index + * @return the column family model + */ + public ColumnSchemaModel getColumnFamily(int index) { + return columns.get(index); + } + + /** + * @return the table name + */ + @XmlAttribute + public String getName() { + return name; + } + + /** + * @return the map for holding unspecified (user) attributes + */ + @XmlAnyAttribute + public Map getAny() { + return attrs; + } + + /** + * @return the columns + */ + @XmlElement(name="ColumnSchema") + public List getColumns() { + return columns; + } + + /** + * @param name the table name + */ + public void setName(String name) { + this.name = name; + } + + /** + * @param columns the columns to set + */ + public void setColumns(List columns) { + this.columns = columns; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{ NAME=> '"); + sb.append(name); + sb.append('\''); + for (Map.Entry e: attrs.entrySet()) { + sb.append(", "); + sb.append(e.getKey().getLocalPart()); + sb.append(" => '"); + sb.append(e.getValue().toString()); + sb.append('\''); + } + sb.append(", COLUMNS => [ "); + Iterator i = columns.iterator(); + while (i.hasNext()) { + ColumnSchemaModel family = i.next(); + sb.append(family.toString()); + if (i.hasNext()) { + sb.append(','); + } + sb.append(' '); + } + sb.append("] }"); + return sb.toString(); + } + + // getters and setters for common schema attributes + + // cannot be standard bean type getters and setters, otherwise this would + // confuse JAXB + + /** + * @return true if IS_META attribute exists and is truel + */ + public boolean __getIsMeta() { + Object o = attrs.get(IS_META); + return o != null ? Boolean.valueOf(o.toString()) : false; + } + + /** + * @return true if IS_ROOT attribute exists and is truel + */ + public boolean __getIsRoot() { + Object o = attrs.get(IS_ROOT); + return o != null ? Boolean.valueOf(o.toString()) : false; + } + + /** + * @return true if READONLY attribute exists and is truel + */ + public boolean __getReadOnly() { + Object o = attrs.get(READONLY); + return o != null ? + Boolean.valueOf(o.toString()) : HTableDescriptor.DEFAULT_READONLY; + } + + /** + * @param value desired value of IS_META attribute + */ + public void __setIsMeta(boolean value) { + attrs.put(IS_META, Boolean.toString(value)); + } + + /** + * @param value desired value of IS_ROOT attribute + */ + public void __setIsRoot(boolean value) { + attrs.put(IS_ROOT, Boolean.toString(value)); + } + + /** + * @param value desired value of READONLY attribute + */ + public void __setReadOnly(boolean value) { + attrs.put(READONLY, Boolean.toString(value)); + } + + @Override + public byte[] createProtobufOutput() { + TableSchema.Builder builder = TableSchema.newBuilder(); + builder.setName(name); + for (Map.Entry e: attrs.entrySet()) { + TableSchema.Attribute.Builder attrBuilder = + TableSchema.Attribute.newBuilder(); + attrBuilder.setName(e.getKey().getLocalPart()); + attrBuilder.setValue(e.getValue().toString()); + builder.addAttrs(attrBuilder); + } + for (ColumnSchemaModel family: columns) { + Map familyAttrs = family.getAny(); + ColumnSchema.Builder familyBuilder = ColumnSchema.newBuilder(); + familyBuilder.setName(family.getName()); + for (Map.Entry e: familyAttrs.entrySet()) { + ColumnSchema.Attribute.Builder attrBuilder = + ColumnSchema.Attribute.newBuilder(); + attrBuilder.setName(e.getKey().getLocalPart()); + attrBuilder.setValue(e.getValue().toString()); + familyBuilder.addAttrs(attrBuilder); + } + if (familyAttrs.containsKey(TTL)) { + familyBuilder.setTtl( + Integer.valueOf(familyAttrs.get(TTL).toString())); + } + if (familyAttrs.containsKey(VERSIONS)) { + familyBuilder.setMaxVersions( + Integer.valueOf(familyAttrs.get(VERSIONS).toString())); + } + if (familyAttrs.containsKey(COMPRESSION)) { + familyBuilder.setCompression(familyAttrs.get(COMPRESSION).toString()); + } + builder.addColumns(familyBuilder); + } + if (attrs.containsKey(READONLY)) { + builder.setReadOnly( + Boolean.valueOf(attrs.get(READONLY).toString())); + } + return builder.build().toByteArray(); + } + + @Override + public ProtobufMessageHandler getObjectFromMessage(byte[] message) + throws IOException { + TableSchema.Builder builder = TableSchema.newBuilder(); + builder.mergeFrom(message); + this.setName(builder.getName()); + for (TableSchema.Attribute attr: builder.getAttrsList()) { + this.addAttribute(attr.getName(), attr.getValue()); + } + if (builder.hasReadOnly()) { + this.addAttribute(HTableDescriptor.READONLY, builder.getReadOnly()); + } + for (ColumnSchema family: builder.getColumnsList()) { + ColumnSchemaModel familyModel = new ColumnSchemaModel(); + familyModel.setName(family.getName()); + for (ColumnSchema.Attribute attr: family.getAttrsList()) { + familyModel.addAttribute(attr.getName(), attr.getValue()); + } + if (family.hasTtl()) { + familyModel.addAttribute(HColumnDescriptor.TTL, family.getTtl()); + } + if (family.hasMaxVersions()) { + familyModel.addAttribute(HConstants.VERSIONS, + family.getMaxVersions()); + } + if (family.hasCompression()) { + familyModel.addAttribute(HColumnDescriptor.COMPRESSION, + family.getCompression()); + } + this.addColumnFamily(familyModel); + } + return this; + } + + /** + * @return a table descriptor + */ + public HTableDescriptor getTableDescriptor() { + HTableDescriptor htd = new HTableDescriptor(getName()); + for (Map.Entry e: getAny().entrySet()) { + htd.setValue(e.getKey().getLocalPart(), e.getValue().toString()); + } + for (ColumnSchemaModel column: getColumns()) { + HColumnDescriptor hcd = new HColumnDescriptor(column.getName()); + for (Map.Entry e: column.getAny().entrySet()) { + hcd.setValue(e.getKey().getLocalPart(), e.getValue().toString()); + } + htd.addFamily(hcd); + } + return htd; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java new file mode 100644 index 0000000..e4b6b0f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java @@ -0,0 +1,208 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.Serializable; + +import javax.servlet.ServletContext; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; +import org.apache.hadoop.hbase.rest.RESTServlet; +import org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version; + +import com.sun.jersey.spi.container.servlet.ServletContainer; + +/** + * A representation of the collection of versions of the REST gateway software + * components. + *
      + *
    • restVersion: REST gateway revision
      • + *
    • jvmVersion: the JVM vendor and version information
      • + *
    • osVersion: the OS type, version, and hardware architecture
      • + *
    • serverVersion: the name and version of the servlet container
      • + *
    • jerseyVersion: the version of the embedded Jersey framework
      • + *
    + */ +@XmlRootElement(name="Version") +public class VersionModel implements Serializable, ProtobufMessageHandler { + + private static final long serialVersionUID = 1L; + + private String restVersion; + private String jvmVersion; + private String osVersion; + private String serverVersion; + private String jerseyVersion; + + /** + * Default constructor. Do not use. + */ + public VersionModel() {} + + /** + * Constructor + * @param context the servlet context + */ + public VersionModel(ServletContext context) { + restVersion = RESTServlet.VERSION_STRING; + jvmVersion = System.getProperty("java.vm.vendor") + ' ' + + System.getProperty("java.version") + '-' + + System.getProperty("java.vm.version"); + osVersion = System.getProperty("os.name") + ' ' + + System.getProperty("os.version") + ' ' + + System.getProperty("os.arch"); + serverVersion = context.getServerInfo(); + jerseyVersion = ServletContainer.class.getPackage() + .getImplementationVersion(); + } + + /** + * @return the REST gateway version + */ + @XmlAttribute(name="REST") + public String getRESTVersion() { + return restVersion; + } + + /** + * @return the JVM vendor and version + */ + @XmlAttribute(name="JVM") + public String getJVMVersion() { + return jvmVersion; + } + + /** + * @return the OS name, version, and hardware architecture + */ + @XmlAttribute(name="OS") + public String getOSVersion() { + return osVersion; + } + + /** + * @return the servlet container version + */ + @XmlAttribute(name="Server") + public String getServerVersion() { + return serverVersion; + } + + /** + * @return the version of the embedded Jersey framework + */ + @XmlAttribute(name="Jersey") + public String getJerseyVersion() { + return jerseyVersion; + } + + /** + * @param version the REST gateway version string + */ + public void setRESTVersion(String version) { + this.restVersion = version; + } + + /** + * @param version the OS version string + */ + public void setOSVersion(String version) { + this.osVersion = version; + } + + /** + * @param version the JVM version string + */ + public void setJVMVersion(String version) { + this.jvmVersion = version; + } + + /** + * @param version the servlet container version string + */ + public void setServerVersion(String version) { + this.serverVersion = version; + } + + /** + * @param version the Jersey framework version string + */ + public void setJerseyVersion(String version) { + this.jerseyVersion = version; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("rest "); + sb.append(restVersion); + sb.append(" [JVM: "); + sb.append(jvmVersion); + sb.append("] [OS: "); + sb.append(osVersion); + sb.append("] [Server: "); + sb.append(serverVersion); + sb.append("] [Jersey: "); + sb.append(jerseyVersion); + sb.append("]\n"); + return sb.toString(); + } + + @Override + public byte[] createProtobufOutput() { + Version.Builder builder = Version.newBuilder(); + builder.setRestVersion(restVersion); + builder.setJvmVersion(jvmVersion); + builder.setOsVersion(osVersion); + builder.setServerVersion(serverVersion); + builder.setJerseyVersion(jerseyVersion); + return builder.build().toByteArray(); + } + + @Override + public ProtobufMessageHandler getObjectFromMessage(byte[] message) + throws IOException { + Version.Builder builder = Version.newBuilder(); + builder.mergeFrom(message); + if (builder.hasRestVersion()) { + restVersion = builder.getRestVersion(); + } + if (builder.hasJvmVersion()) { + jvmVersion = builder.getJvmVersion(); + } + if (builder.hasOsVersion()) { + osVersion = builder.getOsVersion(); + } + if (builder.hasServerVersion()) { + serverVersion = builder.getServerVersion(); + } + if (builder.hasJerseyVersion()) { + jerseyVersion = builder.getJerseyVersion(); + } + return this; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/package.html b/src/main/java/org/apache/hadoop/hbase/rest/package.html new file mode 100644 index 0000000..9ed0ead --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/package.html @@ -0,0 +1,1444 @@ + + + + + + + +

    HBase REST

    +This package provides a RESTful Web service front end for HBase. +

    + +

    Table Of Contents

    +
      +
    1. Deployment
      2. +
        +
      1. Daemon
        2. +
      2. Servlet
        3. +
      +
    2. Representational State Transfer
      3. +
    3. Resource Identifiers
      4. +
    4. Operations
      5. +
        +
      1. Query Software Version
        2. +
      2. Query Storage Cluster Version
        3. +
      3. Query Storage Cluster Status
        4. +
      4. Query Table List
        5. +
      5. Query Table Schema
        6. +
      6. Create Table Or Update Table Schema
        7. +
      7. Query Table Metadata
        8. +
      8. Delete Table
        9. +
      9. Cell Query (Single Value)
        10. +
      10. Cell or Row Query (Multiple Values)
        11. +
      11. Cell Store (Single)
        12. +
      12. Cell Store (Multiple)
        13. +
      13. Row, Column, or Cell Delete
        14. +
      14. Scanner Creation
        15. +
      15. Scanner Get Next
        16. +
      16. Scanner Deletion
        17. +
      +
    5. XML Schema
      6. +
    6. Protobufs Schema
      7. +
    + +

    + +

    Deployment

    + +

    + +

    + +

    Daemon

    + +

    +HBase REST can run as a daemon which starts an embedded Jetty servlet container +and deploys the servlet into it. +

    +

      +
    1. Start the embedded Jetty servlet container: +
        +
      • In the foreground: +
        + + % ./bin/hbase rest start -p <port> + +
        +

        + where <port> is optional, and is the port the connector should + listen on. (Default is 8080.) +

        +
        • +
      +
      2. +
    + +

    + +

    Representational State Transfer

    + +

    + +The terms "representational state transfer" and "REST" were introduced in 2000 +in the + +doctoral dissertation of Roy Fielding, one of the principal authors of the +Hypertext Transfer Protocol (HTTP) specification. +

    +A GET to an identifier requests a copy of the information in the supplied +content type. +

    +A PUT to an identifier replaces the information. The supplied content type +determines how it is to be interpreted. +

    +POST adds information. +

    +DELETE eliminates information. +

    +

    + + + + + + + + + +
    Database OperationsREST/HTTP Equivalents
     
    CREATEPUT
    READGET
    UPDATEPOST (update) or PUT (replace)
    DELETEDELETE
    +
    + +

    + +

    Resource Identifiers

    + +

    +RFC 3968 defines URL +syntax: +

    +

    +scheme://user:pass@example.net:8080/path/to/file;type=foo?name=val#frag
+\_____/  \_______/\___________/\__/\______/\____/\______/\________/\___/
+   |         |          |       |     |      |       |       |       |
+ scheme   userinfo  hostname  port  path  filename param   query fragment
+         \________________________/
+                  authority
+
    +

    +HBase REST exposes HBase tables, rows, cells, and metadata as URL specified +resources. +

    +NOTE: The characters /, :, and , are reserved +within row keys, column names, and column qualifiers. Clients must escape them +somehow, perhaps by encoding them as hex escapes or by using www-url-encoding. For +example, the key: +

    +

    +    http://www.google.com/
+
    +

    +should first be encoded as: +

    +

    +    http%3A%2F%2Fwww.google.com%2F
+
    +

    +to produce a path like: +

    +    /SomeTable/http%3A%2F%2Fwww.google.com%2F/someColumn:qualifier
+
    +

    +

    Addressing for cell or row query (GET)

    +

    +

    +    path := '/' <table>
+            '/' <row>
+            ( '/' ( <column> ( ':' <qualifier> )?
+                    ( ',' <column> ( ':' <qualifier> )? )+ )?
+                ( '/' ( <start-timestamp> ',' )? <end-timestamp> )? )?
+    query := ( '?' 'v' '=' <num-versions> )?
+
    +

    + +

    Addressing for single value store (PUT)

    +

    +Address with table, row, column (and optional qualifier), and optional timestamp. +

    +

    +    path := '/' <table> '/' <row> '/' <column> ( ':' <qualifier> )?
+              ( '/' <timestamp> )?
+
    +

    + +

    Addressing for multiple (batched) value store (PUT)

    +

    +

    +    path := '/' <table> '/' <false-row-key>
+
    +

    + +

    Addressing for row, column, or cell DELETE

    +

    +

    +    path := '/' <table> 
+            '/' <row>
+            ( '/' <column> ( ':' <qualifier> )?
+              ( '/' <timestamp> )? )?
+
    +

    + +

    Addressing for table creation or schema update (PUT or POST), schema query +(GET), or delete (DELETE)

    +

    +

    +    path := '/' <table> / 'schema'
+
    +

    + +

    Addressing for scanner creation (POST)

    +

    +

    +    path := '/' <table> '/' 'scanner'
+
    +

    + +

    Addressing for scanner next item (GET)

    +

    +

    +    path := '/' <table> '/' 'scanner' '/' <scanner-id>
+
    +

    + +

    Addressing for scanner deletion (DELETE)

    +

    +

    +    path := '/' <table> '/' '%scanner' '/' <scanner-id>
+
    +

    + +

    + +

    Operations

    + +

    + + +

    Query Software Version

    + +

    +

    +GET /version
+
    +

    +Returns the software version. +Set Accept header to text/plain for plain text output. +Set Accept header to text/xml for XML reply. +Set Accept header to application/json for JSON reply. +Set Accept header to application/x-protobuf for protobufs. +

    +If not successful, returns appropriate HTTP error status code. +If successful, returns the software version. +

    +Examples: +

    +

    + +% curl http://localhost:8000/version
    +
    +HTTP/1.1 200 OK
    +Content-Length: 149
    +Cache-Control: no-cache
    +Content-Type: text/plain
    +
    +Stargate 0.0.1 [JVM: Sun Microsystems Inc. 1.6.0_13-11.3-b02] [OS: Linux 2.6.
    +18-128.1.6.el5.centos.plusxen amd64] [Jetty: 6.1.14] [Jersey: 1.1.0-ea]
    +
    +% curl -H "Accept: text/xml" http://localhost:8000/version
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: text/xml
    +Content-Length: 212
    +
    +<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
    +<Version Stargate="0.0.1" OS="Linux 2.6.18-128.1.6.el5.centos.plusxen amd64"
    + JVM="Sun Microsystems Inc. 1.6.0_13-11.3-b02" Jetty="6.1.14" Jersey="1.1.0-e
    +a"/>
    +
    +% curl -H "Accept: application/json" http://localhost:8000/version
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: application/json
    +Transfer-Encoding: chunked
    +
    +{"@Stargate":"0.0.1","@OS":"Linux 2.6.18-128.1.6.el5.centos.plusxen amd64","@
    +JVM":"Sun Microsystems Inc. 1.6.0_13-11.3-b02","@Jetty":"6.1.14","@Jersey":"1
    +.1.0-ea"}
    +
    +% curl -H "Accept: application/x-protobuf" http://localhost:8000/version
    +
    +HTTP/1.1 200 OK
    +Content-Length: 113
    +Cache-Control: no-cache
    +Content-Type: application/x-protobuf
    +
    +000000 0a 05 30 2e 30 2e 31 12 27 53 75 6e 20 4d 69 63
    +000010 72 6f 73 79 73 74 65 6d 73 20 49 6e 63 2e 20 31
    +000020 2e 36 2e 30 5f 31 33 2d 31 31 2e 33 2d 62 30 32
    +000030 1a 2d 4c 69 6e 75 78 20 32 2e 36 2e 31 38 2d 31
    +000040 32 38 2e 31 2e 36 2e 65 6c 35 2e 63 65 6e 74 6f
    +000050 73 2e 70 6c 75 73 78 65 6e 20 61 6d 64 36 34 22
    +000060 06 36 2e 31 2e 31 34 2a 08 31 2e 31 2e 30 2d 65
    +000070 61
    +     +
    +

    + + +

    Query Storage Cluster Version

    + +

    +

    +GET /version/cluster
+
    +

    +Returns version information regarding the HBase cluster backing the Stargate instance. +

    +Examples: +

    +

    + +% curl http://localhost:8000/version/cluster
    +
    +HTTP/1.1 200 OK
    +Content-Length: 6
    +Cache-Control: no-cache
    +Content-Type: text/plain
    +
    +0.20.0
    +
    +% curl -H "Accept: text/xml" http://localhost:8000/version/cluster
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: text/xml
    +Content-Length: 94
    +
    +<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
    +<ClusterVersion>0.20.0</ClusterVersion>
    +
    +% curl -H "Accept: application/json" http://localhost:8000/version/cluster
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: application/json
    +Transfer-Encoding: chunked
    +
    +"0.20.0"
    +     +
    +

    + + +

    Query Storage Cluster Status

    + +

    +

    +GET /status/cluster
+
    +

    +Returns detailed status on the HBase cluster backing the Stargate instance. +

    +Examples: +

    +

    + +% curl http://localhost:8000/status/cluster
    +     +
    +HTTP/1.1 200 OK
+Content-Length: 839
+Cache-Control: no-cache
+Content-Type: text/plain
+
+1 live servers, 0 dead servers, 13.0000 average load
+
+1 live servers
+    test:37154 1244960965781
+        requests=1, regions=13
+
+        urls,http|www.legacy.com|80|site=Legacy|aamsz=300x250||position=1|prod
+          =1,1244851990859
+        urls,http|weather.boston.com|80|LYNX.js,1244851990859
+        .META.,,1
+        content,601292a839b95e50200d8f8767859864,1244869158156
+        content,9d7f3aeb2a5c1e2b45d690a91de3f23c,1244879698031
+        content,7f6d48830ef51d635e9a5b672e79a083,1244879698031
+        content,3ef16d776603bf9b9e775c9ceb64860f,1244869158156
+        urls,,1244851989250
+        urls,http|groups.google.com|80|groups|img|card_left.gif,1244851989250
+        content,deafed2f90f718d72caaf87bd6c27d04,1244870320343
+        content,bcf91ecf78ea72a33faccfb8e6b5d900,1244870320343
+        -ROOT-,,0
+        content,,1244851999187
+
    + +% curl -H "Accept: text/xml" http://localhost:8000/status/cluster
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: text/xml
    +Content-Length: 1301
    +
    +<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
    +<ClusterStatus requests="1" regions="13" averageLoad="13.0"><DeadNodes/><LiveN
    +odes><Node startCode="1244960965781" requests="1" name="test:37154"><Region na
    +me="dXJscyxodHRwfHd3dy5sZWdhY3kuY29tfDgwfHNpdGU9TGVnYWN5fGFhbXN6PTMwMHgyNTB8YX
    +JlYT1DSlDQaElDQUdPVFJJQlVORS4yMXx6b25lPUhvbWV8cG9zaXRpb249MXxwcm9kPTEsMTI0NDg1
    +MTk5MDg1OQ=="/><Region name="dXJscyxodHRwfHdlYXRoZXIuYm9zdG9uLmNvbXw4MHxMWU5YL
    +mpzLDEyNDQ4NTE5OTA4NTk="/><Region name="Lk1FVEEuLCwx"/><Region name="Y29udGVud
    +Cw2MDEyOTJhODM5Yjk1ZTUwMjAwZDhmODc2Nzg1OTg2NCwxMjQ0ODY5MTU4MTU2"/><Region name
    +="Y29udGVudCw5ZDdmM2FlYjJhNWMxZTJiNDVkNjkwYTkxZGUzZjIzYywxMjQ0ODc5Njk4MDMx"/><
    +Region name="Y29udGVudCw3ZjZkNDg4MzBlZjUxZDYzNWU5YTViNjcyZTc5YTA4MywxMjQ0ODc5N
    +jk4MDMx"/><Region name="Y29udGVudCwzZWYxNmQ3NzY2MDNiZjliOWU3NzVjOWNlYjY0ODYwZi
    +wxMjQ0ODY5MTU4MTU2"/><Region name="dXJscywsMTI0NDg1MTk4OTI1MA=="/><Region name
    +="dXJscyxodHRwfGdyb3Vwcy5nb29nbGUuY29tfDgwfGdyb3Vwc3xpbWd8Y2FyZF9sZWZ0LmdpZiwx
    +MjQ0ODUxOTg5MjUw"/><Region name="Y29udGVudCxkZWFmZWQyZjkwZjcxOGQ3MmNhYWY4N2JkN
    +mMyN2QwNCwxMjQ0ODcwMzIwMzQz"/><Region name="Y29udGVudCxiY2Y5MWVjZjc4ZWE3MmEzM2
    +ZhY2NmYjhlNmI1ZDkwMCwxMjQ0ODcwMzIwMzQz"/><Region name="LVJPT1QtLCww"/><Region
    +name="Y29udGVudCwsMTI0NDg1MTk5OTE4Nw=="/></Node></LiveNodes></ClusterStatus>
    +
    +% curl -H "Accept: application/json" http://localhost:8000/status/cluster
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: application/json
    +Transfer-Encoding: chunked
    +
    +{"@requests":"1","@regions":"13","@averageLoad":"13.0","DeadNodes":[],"LiveNod
    +es":{"Node":{"@startCode":"1244960965781","@requests":"1","@name":"test:37154"
    +,"Region":[{"@name":"dXJscyxodHRwfHd3dLmpzy5sZWdhY3kuY29tfDgwfHNpdGU9TGVnYWN5f
    +GFhbXN6PTMwMHgyNTB8YXJlYT1DSElDQUdPVFJJQlVORS4yMXx6b25lPUhvbWV8cG9zaXRpb249MXx
    +wcm9kPTEsMTI0NDg1MTk5MDg1OQ=="},{"@name":"dXJscyxodHRwfHdlYXRoZXIuYm9zdG9uLmNv
    +bXw4MHxMWU5YLmpzLDEyNDQ4NTE5OTA4NTk="},{"@name":"Lk1FVEEuLCwx"},{"@name":"Y29u
    +dGVudCw2MDEyOTJhODM5Yjk1ZTUwMjAwZDhmODc2Nzg1OTg2NCwxMjQ0ODY5MTU4MTU2"},{"@name
    +":"Y29udGVudCw5ZDdmM2FlYjJhNWMxZTJiNDVkNjkwYTkxZGUzZjIzYywxMjQ0ODc5Njk4MDMx"},
    +{"@name":"Y29udGVudCw3ZjZkNDg4MzBlZjUxZDYzNWU5YTViNjcyZTc5YTA4MywxMjQ0ODc5Njk4
    +MDMx"},{"@name":"Y29udGVudCwzZWYxNmQ3NzY2MDNiZjliOWU3NzVjOWNlYjY0ODYwZiwxMjQ0O
    +DY5MTU4MTU2"},{"@name":"dXJscywsMTI0NDg1MTk4OTI1MA=="},{"@name":"dXJscyxodHRwf
    +Gdyb3Vwcy5nb29nbGUuY29tfDgwfGdyb3Vwc3xpbWd8Y2FyZF9sZWZ0LmdpZiwxMjQ0ODUxOTg5MjU
    +w"},{"@name":"Y29udGVudCxkZWFmZWQyZjkwZjcxOGQ3MmNhYWY4N2JkNmMyN2QwNCwxMjQ0ODcw
    +MzIwMzQz"},{"@name":"Y29udGVudCxiY2Y5MWVjZjc4ZWE3MmEzM2ZhY2NmYjhlNmI1ZDkwMCwxM
    +jQ0ODcwMzIwMzQz"},{"@name":"LVJPT1QtLCww"},{"@name":"Y29udGVudCwsMTI0NDg1MTk5O
    +TE4Nw=="}]}}}
    +     +
    +

    + + +

    Query Table List

    + +

    +

    +GET /
+
    +

    +Retrieves the list of available tables. +Set Accept header to text/plain for plain text output. +Set Accept header to text/xml for XML reply. +Set Accept header to application/json for JSON reply. +Set Accept header to application/x-protobuf for protobufs. +If not successful, returns appropriate HTTP error status code. +If successful, returns the table list in the requested encoding. +

    +Examples: +

    +

    + +% curl http://localhost:8000/
    +
    +HTTP/1.1 200 OK
    +Content-Length: 13
    +Cache-Control: no-cache
    +Content-Type: text/plain
    +
    +content
    +urls
    +
    +% curl -H "Accept: text/xml" http://localhost:8000/
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: text/xml
    +Content-Length: 121
    +
    +<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
    +<TableList><table name="content"/><table name="urls"/></TableList>
    +
    +% curl -H "Accept: application/json" http://localhost:8000/
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: application/json
    +Transfer-Encoding: chunked
    +
    +{"table":[{"name":"content"},{"name":"urls"}]}
    +
    +% curl -H "Accept: application/x-protobuf" http://localhost:8000/
    +
    +HTTP/1.1 200 OK
    +Content-Length: 15
    +Cache-Control: no-cache
    +Content-Type: application/x-protobuf
    +
    +000000 0a 07 63 6f 6e 74 65 6e 74 0a 04 75 72 6c 73
    +     +
    +

    + + +

    Query Table Schema

    + +

    +

    +GET /<table>/schema
+
    +

    +Retrieves table schema. +Set Accept header to text/plain for plain text output. +Set Accept header to text/xml for XML reply. +Set Accept header to application/json for JSON reply. +Set Accept header to application/x-protobuf for protobufs. +If not successful, returns appropriate HTTP error status code. +If successful, returns the table schema in the requested encoding. +

    +Examples: +

    +

    + +% curl http://localhost:8000/content/schema
    +
    +HTTP/1.1 200 OK
    +Content-Length: 639
    +Cache-Control: no-cache
    +Content-Type: text/plain
    +
    +{ NAME=> 'content', IS_META => 'false', IS_ROOT => 'false', COLUMNS => [ { NA
    +ME => 'content', BLOCKSIZE => '65536', BLOOMFILTER => 'false', BLOCKCACHE =>
    +'false', COMPRESSION => 'GZ', LENGTH => '2147483647', VERSIONS => '1', TTL =>
    +'-1', IN_MEMORY => 'false' }, { NAME => 'info', BLOCKSIZE => '65536', BLOOMFI
    +LTER => 'false', BLOCKCACHE => 'false', COMPRESSION => 'NONE', LENGTH => '214
    +7483647', VERSIONS => '1', TTL => '-1', IN_MEMORY => 'false' }, { NAME => 'ur
    +l', BLOCKSIZE => '65536', BLOOMFILTER => 'false', BLOCKCACHE => 'false', COMP
    +RESSION => 'NONE', LENGTH => '2147483647', VERSIONS => '1', TTL => '-1', IN_
    +MEMORY => 'false' } ] }
    +
    +% curl -H "Accept: text/xml" http://localhost:8000/content/schema
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: text/xml
    +Content-Length: 618
    +
    +<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
    +<TableSchema name="content" IS_META="false" IS_ROOT="false"><ColumnSchema nam
    +e="content" BLOCKSIZE="65536" BLOOMFILTER="false" BLOCKCACHE="false" COMPRESS
    +ION="GZ" LENGTH="2147483647" VERSIONS="1" TTL="-1" IN_MEMORY="false"/><Column
    +Schema name="info" BLOCKSIZE="65536" BLOOMFILTER="false" BLOCKCACHE="false" C
    +OMPRESSION="NONE" LENGTH="2147483647" VERSIONS="1" TTL="-1" IN_MEMORY="false"
    +/><ColumnSchema name="url" BLOCKSIZE="65536" BLOOMFILTER="false"BLOCKCACHE="f
    +alse" COMPRESSION="NONE" LENGTH="2147483647" VERSIONS="1" TTL="-1" IN_MEMORY=
    +"false"/></TableSchema>
    +
    +% curl -H "Accept: application/json" http://localhost:8000/content/schema
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: application/json
    +Transfer-Encoding: chunked
    +
    +{"@name":"content","@IS_META":"false","@IS_ROOT":"false","ColumnSchema":[{"@n
    +ame":"content","@BLOCKSIZE":"65536","@BLOOMFILTER":"false","@BLOCKCACHE":"fal
    +se","@COMPRESSION":"GZ","@LENGTH":"2147483647","@VERSIONS":"1","@TTL":"-1","@
    +IN_MEMORY":"false"},{"@name":"info","@BLOCKSIZE":"65536","@BLOOMFILTER":"fals
    +e","@BLOCKCACHE":"false","@COMPRESSION":"NONE","@LENGTH":"2147483647","@VERSI
    +ONS":"1","@TTL":"-1","@IN_MEMORY":"false"},{"@name":"url","@BLOCKSIZE":"65536
    +","@BLOOMFILTER":"false","@BLOCKCACHE":"false","@COMPRESSION":"NONE","@LENGTH
    +":"2147483647","@VERSIONS":"1","@TTL":"-1","@IN_MEMORY":"false"}]}
    +
    +% curl -H "Accept: application/x-protobuf" http://localhost:8000/content/schema
    +
    +HTTP/1.1 200 OK
    +Content-Length: 563
    +Cache-Control: no-cache
    +Content-Type: application/x-protobuf
    +
    +000000 0a 07 63 6f 6e 74 65 6e 74 12 10 0a 07 49 53 5f
    +000010 4d 45 54 41 12 05 66 61 6c 73 65 12 10 0a 07 49
    +000020 53 5f 52 4f 4f 54 12 05 66 61 6c 73 65 1a a7 01
    +000030 12 12 0a 09 42 4c 4f 43 4b 53 49 5a 45 12 05 36
    +[...]
    +000230 4f 4e 45
    +     +
    +

    + + +

    Create Table Or Update Table Schema

    + +

    +

    +PUT /<table>/schema
+
+POST /<table>/schema
+
    +

    +Uploads table schema. +PUT or POST creates table as necessary. +PUT fully replaces schema. +POST modifies schema (add or modify column family). +Supply the full table schema for PUT or a well formed schema fragment for POST +in the desired encoding. +Set Content-Type header to text/xml if the desired encoding is XML. +Set Content-Type header to application/json if the desired encoding +is JSON. +Set Content-Type header to application/x-protobuf if the desired +encoding is protobufs. +If not successful, returns appropriate HTTP error status code. +If successful, returns HTTP 200 status. +

    + + +

    Query Table Metadata

    + +

    +

    +GET /<table>/regions
+
    +

    +Retrieves table region metadata. +Set Accept header to text/plain for plain text output. +Set Accept header to text/xml for XML reply. +Set Accept header to application/json for JSON reply. +Set Accept header to application/x-protobuf for protobufs. +If not successful, returns appropriate HTTP error status code. +If successful, returns the table region metadata in the requested encoding. +

    +Examples: +

    +

    + +% curl -H "Accept: text/xml" http://localhost:8000/content/regions
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: text/xml
    +Content-Length: 1555
    +
    +<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
    +<TableInfo name="content"><Region location="test:51025" endKey="M2VmMTZkNzc2Nj
    +AzYmY5YjllNzc1YzljZWI2NDg2MGY=" startKey="" id="1244851999187" name="content,,
    +1244851999187"/><Region location="test:51025" endKey="NjAxMjkyYTgzOWI5NWU1MDIw
    +MGQ4Zjg3Njc4NTk4NjQ=" startKey="M2VmMTZkNzc2NjAzYmY5YjllNzc1YzljZWI2NDg2MGY="
    +id="1244869158156" name="content,3ef16d776603bf9b9e775c9ceb64860f,124486915815
    +6"/><Region location="test:51025" endKey="N2Y2ZDQ4ODMwZWY1MWQ2MzVlOWE1YjY3MmU3
    +OWEwODM=" startKey="NjAxMjkyYTgzOWI5NWU1MDIwMGQ4Zjg3Njc4NTk4NjQ=" id="12448691
    +58156" name="content,601292a839b95e50200d8f8767859864,1244869158156"/><Region
    +location="test:51025" endKey="OWQ3ZjNhZWIyYTVjMWUyYjQ1ZDY5MGE5MWRlM2YyM2M=" st
    +artKey="N2Y2ZDQ4ODMwZWY1MWQ2MzVlOWE1YjY3MmU3OWEwODM=" id="1244879698031" name=
    +"content,7f6d48830ef51d635e9a5b672e79a083,1244879698031"/><Region location="te
    +st:51025" endKey="YmNmOTFlY2Y3OGVhNzJhMzNmYWNjZmI4ZTZiNWQ5MDA=" startKey="OWQ3
    +ZjNhZWIyYTVjMWUyYjQ1ZDY5MGE5MWRlM2YyM2M=" id="1244879698031" name="content,9d7
    +f3aeb2a5c1e2b45d690a91de3f23c,1244879698031"/><Region location="test:51025" en
    +dKey="ZGVhZmVkMmY5MGY3MThkNzJjYWFmODdiZDZjMjdkMDQ=" startKey="YmNmOTFlY2Y3OGVh
    +NzJhMzNmYWNjZmI4ZTZiNWQ5MDA=" id="1244870320343" name="content,bcf91ecf78ea72a
    +33faccfb8e6b5d900,1244870320343"/><Region location="test:51025" endKey="" star
    +tKey="ZGVhZmVkMmY5MGY3MThkNzJjYWFmODdiZDZjMjdkMDQ=" id="1244870320343" name="c
    +ontent,deafed2f90f718d72caaf87bd6c27d04,1244870320343"/></TableInfo>
    +
    +% curl -H "Accept: application/json" http://localhost:8000/content/regions
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: application/json
    +Transfer-Encoding: chunked
    +
    +{"@name":"content","Region":[{"@location":"test:51025","@endKey":"M2VmMTZkNzc2
    +NjAzYmY5YjllNzc1YzljZWI2NDg2MGY=","@startKey":"","@id":"1244851999187","@name"
    +:"content,,1244851999187"},{"@location":"test:51025","@endKey":"NjAxMjkyYTgzOW
    +I5NWU1MDIwMGQ4Zjg3Njc4NTk4NjQ=","@startKey":"M2VmMTZkNzc2NjAzYmY5YjllNzc1YzljZ
    +WI2NDg2MGY=","@id":"1244869158156","@name":"content,3ef16d776603bf9b9e775c9ceb
    +64860f,1244869158156"},{"@location":"test:51025","@endKey":"N2Y2ZDQ4ODMwZWY1MW
    +Q2MzVlOWE1YjY3MmU3OWEwODM=","@startKey":"NjAxMjkyYTgzOWI5NWU1MDIwMGQ4Zjg3Njc4N
    +Tk4NjQ=","@id":"1244869158156","@name":"content,601292a839b95e50200d8f87678598
    +64,1244869158156"},{"@location":"test:51025","@endKey":"OWQ3ZjNhZWIyYTVjMWUyYj
    +Q1ZDY5MGE5MWRlM2YyM2M=","@startKey":"N2Y2ZDQ4ODMwZWY1MWQ2MzVlOWE1YjY3MmU3OWEwO
    +DM=","@id":"1244879698031","@name":"content,7f6d48830ef51d635e9a5b672e79a083,1
    +244879698031"},{"@location":"test:51025","@endKey":"YmNmOTFlY2Y3OGVhNzJhMzNmYW
    +NjZmI4ZTZiNWQ5MDA=","@startKey":"OWQ3ZjNhZWIyYTVjMWUyYjQ1ZDY5MGE5MWRlM2YyM2M="
    +,"@id":"1244879698031","@name":"content,9d7f3aeb2a5c1e2b45d690a91de3f23c,12448
    +79698031"},{"@location":"test:51025","@endKey":"ZGVhZmVkMmY5MGY3MThkNzJjYWFmOD
    +diZDZjMjdkMDQ=","@startKey":"YmNmOTFlY2Y3OGVhNzJhMzNmYWNjZmI4ZTZiNWQ5MDA=","@i
    +d":"1244870320343","@name":"content,bcf91ecf78ea72a33faccfb8e6b5d900,124487032
    +0343"},{"@location":"test:51025","@endKey":"","@startKey":"ZGVhZmVkMmY5MGY3MTh
    +kNzJjYWFmODdiZDZjMjdkMDQ=","@id":"1244870320343","@name":"content,deafed2f90f7
    +18d72caaf87bd6c27d04,1244870320343"}]}
    +
    +% curl -H "Accept: application/x-protobuf" http://localhost:8000/content/regions
    +
    +HTTP/1.1 200 OK
    +Content-Length: 961
    +Cache-Control: no-cache
    +Content-Type: application/x-protobuf
    +
    +000000 0a 07 63 6f 6e 74 65 6e 74 12 53 0a 16 63 6f 6e
    +000010 74 65 6e 74 2c 2c 31 32 34 34 38 35 31 39 39 39
    +000020 31 38 37 12 00 1a 20 33 65 66 31 36 64 37 37 36
    +000030 36 30 33 62 66 39 62 39 65 37 37 35 63 39 63 65
    +[...]
    +0003c0 35
    +     +
    +

    + + +

    Delete Table

    + +

    +

    +DELETE /<table>/schema
+
    +

    +Deletes a table. +If not successful, returns appropriate HTTP error status code. +If successful, returns HTTP 200 status. +

    +NOTE: DELETE /<table> will not work +

    +Examples: +

    +

    + +% telnet localhost 8000
    +DELETE http://localhost:8000/test/schema HTTP/1.0
    +
    +HTTP/1.1 200 OK
    +Content-Length: 0
    +     +
    +

    + + +

    Cell Query (Single Value)

    + +

    +

    +GET /<table>/<row>/
+    <column> ( : <qualifier> )?
+  ( / <timestamp> )?
+
    +

    +Retrieves one cell, with optional specification of timestamp. +Set Accept header to text/xml for XML reply. +Set Accept header to application/x-protobuf for protobufs. +Set Accept header to application/octet-stream for binary. +If not successful, returns appropriate HTTP error status code. +If successful, returns HTTP 200 status and cell data in the response body in +the requested encoding. If the encoding is binary, returns row, column, and +timestamp in X headers: X-Row, X-Column, and +X-Timestamp, respectively. Depending on the precision of the resource +specification, some of the X-headers may be elided as redundant. +

    +Examples: +

    +

    + +% curl -H "Accept: text/xml" http://localhost:8000/content/00012614f7d43df6418523445a6787d6/content:raw
    +
    +HTTP/1.1 200 OK
    +Cache-Control: max-age=14400
    +Content-Type: text/xml
    +Content-Length: 521
    +
    +<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
    +<CellSet><Row key="MDAwMTI2MTRmN2Q0M2RmNjQxODUyMzQ0NWE2Nzg3ZDY="><Cell timesta
    +mp="1244880122250" column="Y29udGVudDpyYXc=">PCFET0NUWVBFIEhUTUwgUFVCTElDICItL
    +y9JRVRGLy9EVEQgSFRNTCAyLjAvL0VOIj4KPGh0bWw+PGhlYWQ+Cjx0aXRsZT4zMDEgTW92ZWQgUGV
    +ybWFuZW50bHk8L3RpdGxlPgo8L2hlYWQ+PGJvZHk+CjxoMT5Nb3ZlZCBQZXJtYW5lbnRseTwvaDE+C
    +jxwPlRoZSBkb2N1bWVudCBoYXMgbW92ZWQgPGEgaHJlZj0iaHR0cDovL3R3aXR0ZXIuY29tL2R1bmN
    +hbnJpbGV5Ij5oZXJlPC9hPi48L3A+CjwvYm9keT48L2h0bWw+Cg==</Cell></Row></CellSet>
    +
    +% curl -H "Accept: application/json" http://localhost:8000/content/00012614f7d43df6418523445a6787d6/content:raw
    +
    +HTTP/1.1 200 OK
    +Cache-Control: max-age=14400
    +Content-Type: application/json
    +Transfer-Encoding: chunked
    +
    +{"Row":{"@key":"MDAwMTI2MTRmN2Q0M2RmNjQxODUyMzQ0NWE2Nzg3ZDY=","Cell":{"@timest
    +amp":"1244880122250","@column":"Y29udGVudDpyYXc=","$":"PCFET0NUWVBFIEhUTUwgUFV
    +CTElDICItLy9JRVRGLy9EVEQgSFRNTCAyLjAvL0VOIj4KPGh0bWw+PGhlYWQ+Cjx0aXRsZT4zMDEgT
    +W92ZWQgUGVybWFuZW50bHk8L3RpdGxlPgo8L2hlYWQ+PGJvZHk+CjxoMT5Nb3ZlZCBQZXJtYW5lbnR
    +seTwvaDE+CjxwPlRoZSBkb2N1bWVudCBoYXMgbW92ZWQgPGEgaHJlZj0iaHR0cDovL3R3aXR0ZXIuY
    +29tL2R1bmNhbnJpbGV5Ij5oZXJlPC9hPi48L3A+CjwvYm9keT48L2h0bWw+Cg=="}}}
    +
    +% curl -H "Accept: application/x-protobuf" http://localhost:8000/content/00012614f7d43df6418523445a6787d6/content:raw
    +
    +HTTP/1.1 200 OK
    +Content-Length: 301
    +Cache-Control: max-age=14400
    +Content-Type: application/x-protobuf
    +
    +000000 0a aa 02 0a 20 30 30 30 31 32 36 31 34 66 37 64
    +000010 34 33 64 66 36 34 31 38 35 32 33 34 34 35 61 36
    +000020 37 38 37 64 36 12 85 02 12 0b 63 6f 6e 74 65 6e
    +000030 74 3a 72 61 77 18 8a e3 8c c5 9d 24 22 ee 01 3c
    +[...]
    +000120 62 6f 64 79 3e 3c 2f 68 74 6d 6c 3e 0a
    +
    +% curl -H "Accept: application/octet-stream" http://localhost:8000/content/00012614f7d43df6418523445a6787d6/content:raw
    +
    +HTTP/1.1 200 OK
    +Content-Length: 238
    +Cache-Control: max-age=14400
    +X-Timestamp: 1244880122250
    +Content-Type: application/octet-stream
    +
    +[...]
    +     +
    +

    + + +

    Cell or Row Query (Multiple Values)

    + +

    +

    +GET /<table>/<row>
+  ( / ( <column> ( : <qualifier> )?
+      ( , <column> ( : <qualifier> )? )+ )?
+    ( / ( <start-timestamp> ',' )? <end-timestamp> )? )?
+  ( ?v= <num-versions> )?
+
    +

    +Retrieves one or more cells from a full row, or one or more specified columns +in the row, with optional filtering via timestamp, and an optional restriction +on the maximum number of versions to return. +Set Accept header to text/xml for XML reply. +Set Accept header to application/json for JSON reply. +Set Accept header to application/x-protobuf for protobufs. +Set Accept header to application/octet-stream for binary. +If not successful, returns appropriate HTTP error status code. +If successful, returns row results in the requested encoding. +

    +NOTE: If binary encoding is requested, only one cell can be returned, the +first to match the resource specification. The row, column, and timestamp +associated with the cell will be transmitted in X headers: X-Row, +X-Column, and X-Timestamp, respectively. Depending on the +precision of the resource specification, some of the X-headers may be elided +as redundant. +

    +Suffix Globbing +

    +Multiple value queries of a row can optionally append a suffix glob on the row +key. This is a restricted form of scanner which will return all values in all +rows that have keys which contain the supplied key on their left hand side, +for example: +

    +

    +    org.someorg.*
+        -> org.someorg.blog
+        -> org.someorg.home
+        -> org.someorg.www
+
    +

    +Examples: +

    +

    + +% curl -H "Accept: text/xml" http://localhost:8000/urls/https|ad.doubleclick.net|*
    +
    +HTTP/1.1 200 OK
    +Cache-Control: max-age=14400
    +Content-Type: text/xml
    +Transfer-Encoding: chunked
    +
    +<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
    +<CellSet><Row key="aHR0cHx3d3cudGVsZWdyYXBoLmNvLnVrfDgwfG5ld3N8d29ybGRuZXdzfG5
    +vcnRoYW1lcmljYXx1c2F8NTQ5MTI4NHxBcm5vbGQtU2Nod2FyemVuZWdnZXItdW52ZWlscy1wYXBlc
    +mxlc3MtY2xhc3Nyb29tcy1wbGFuLmh0bWw="><Cell timestamp="1244701257843" column="a
    +W5mbzpjcmF3bGVyLTEyNDQ3MDEyNTc4NDM=">eyJpcCI6IjIwOC41MS4xMzcuOSIsIm1pbWV0eXBlI
    +joidGV4dC9odG1sO2NoYXJzZXQ9SVNPLT
    +[...]
    +</Cell><Cell timestamp="1244701513390" column="aW5mbzp1cmw=">aHR0cDovL3d3dy50Z
    +WxlZ3JhcGguY28udWs6ODAvdGVsZWdyYXBoL3RlbXBsYXRlL3ZlcjEtMC90ZW1wbGF0ZXMvZnJhZ21
    +lbnRzL2NvbW1vbi90bWdsQnJhbmRDU1MuanNw</Cell></Row></CellSet>
    +
    +% curl -H "Accept: text/xml" http://localhost:8000/content/00012614f7d43df6418523445a6787d6
    +
    +HTTP/1.1 200 OK
    +Cache-Control: max-age=14400
    +Content-Type: text/xml
    +Content-Length: 1177
    +
    +<CellSet><Row key="MDAwMTI2MTRmN2Q0M2RmNjQxODUyMzQ0NWE2Nzg3ZDY="><Cell timesta
    +mp="1244880122250" column="Y29udGVudDpyYXc=">PCFET0NUWVBFIEhUTUwgUFVCTElDICItL
    +y9JRVRGLy9EVEQgSFRNTCAyLjAvL0VOIj4KPGh0bWw+PGhlYWQ+Cjx0aXRsZT4zMDEgTW92ZWQgUGV
    +ybWFuZW50bHk8L3RpdGxlPgo8L2hlYWQ+PGJvZHk+CjxoMT5Nb3ZlZCBQZXJtYW5lbnRseTwvaDE+C
    +jxwPlRoZSBkb2N1bWVudCBoYXMgbW92ZWQgPGEgaHJlZj0iaHR0cDovL3R3aXR0ZXIuY29tL2R1bmN
    +hbnJpbGV5Ij5oZXJlPC9hPi48L3A+CjwvYm9keT48L2h0bWw+Cg==</Cell><Cell timestamp="1
    +244880122250" column="aW5mbzpjcmF3bGVyLWh0dHB8d3d3LnR3aXR0ZXIuY29tfDgwfGR1bmNh
    +bnJpbGV5LTEyNDQ4ODAxMjIyNTA=">eyJpcCI6IjE2OC4xNDMuMTYyLjY4IiwibWltZXR5cGUiOiJ0
    +ZXh0L2h0bWw7IGNoYXJzZXQ9aXNvLTg4NTktMSIsInZpYSI6Imh0dHA6Ly93d3cuaW5xdWlzaXRyLm
    +NvbTo4MC8yNTkyNy90b3NoMC1hbmQtdGhlLWRlbWktbW9vcmUtbnNmdy1waWMvIn0=</Cell><Cell
    +timestamp="1244880122250" column="aW5mbzpsZW5ndGg=">MjM4</Cell><Cell timestamp
    +="1244880122250" column="aW5mbzptaW1ldHlwZQ==">dGV4dC9odG1sOyBjaGFyc2V0PWlzby0
    +4ODU5LTE=</Cell><Cell timestamp="1244880122250" column="dXJsOmh0dHB8d3d3LnR3aX
    +R0ZXIuY29tfDgwfGR1bmNhbnJpbGV5">aHR0cDovL3d3dy50d2l0dGVyLmNvbTo4MC9kdW5jYW5yaW
    +xleQ==</Cell></Row></CellSet>
    +
    +% curl -H "Accept: application/json" http://localhost:8000/content/00012614f7d43df6418523445a6787d6
    +
    +HTTP/1.1 200 OK
    +Cache-Control: max-age=14400
    +Content-Type: application/json
    +Transfer-Encoding: chunked
    +
    +{"Row":{"@key":"MDAwMTI2MTRmN2Q0M2RmNjQxODUyMzQ0NWE2Nzg3ZDY=","Cell":[{"@times
    +tamp":"1244880122250","@column":"Y29udGVudDpyYXc=","$":"PCFET0NUWVBFIEhUTUwgUF
    +VCTElDICItLy9JRVRGLy9EVEQgSFRNTCAyLjAvL0VOIj4KPGh0bWw+PGhlYWQ+Cjx0aXRsZT4zMDEg
    +TW92ZWQgUGVybWFuZW50bHk8L3RpdGxlPgo8L2hlYWQ+PGJvZHk+CjxoMT5Nb3ZlZCBQZXJtYW5lbn
    +RseTwvaDE+CjxwPlRoZSBkb2N1bWVudCBoYXMgbW92ZWQgPGEgaHJlZj0iaHR0cDovL3R3aXR0ZXIu
    +Y29tL2R1bmNhbnJpbGV5Ij5oZXJlPC9hPi48L3A+CjwvYm9keT48L2h0bWw+Cg=="},{"@timestam
    +p":"1244880122250","@column":"aW5mbzpjcmF3bGVyLWh0dHB8d3d3LnR3aXR0ZXIuY29tfDgw
    +fGR1bmNhbnJpbGV5LTEyNDQ4ODAxMjIyNTA=","$":"eyJpcCI6IjE2OC4xNDMuMTYyLjY4IiwibWl
    +tZXR5cGUiOiJ0ZXh0L2h0bWw7IGNoYXJzZXQ9aXNvLTg4NTktMSIsInZpYSI6Imh0dHA6Ly93d3cua
    +W5xdWlzaXRyLmNvbTo4MC8yNTkyNy90b3NoMC1hbmQtdGhlLWRlbWktbW9vcmUtbnNmdy1waWMvIn0
    +="},{"@timestamp":"1244880122250","@column":"aW5mbzpsZW5ndGg=","$":"MjM4"},{"@
    +timestamp":"1244880122250","@column":"aW5mbzptaW1ldHlwZQ==","$":"dGV4dC9odG1sO
    +yBjaGFyc2V0PWlzby04ODU5LTE="},{"@timestamp":"1244880122250","@column":"dXJsOmh
    +0dHB8d3d3LnR3aXR0ZXIuY29tfDgwfGR1bmNhbnJpbGV5","$":"aHR0cDovL3d3dy50d2l0dGVyLm
    +NvbTo4MC9kdW5jYW5yaWxleQ=="}]}}
    +     +

    +NOTE: The cell value is given in JSON encoding as the value associated with the key "$". +

    + +% curl -H "Accept: application/x-protobuf" http://localhost:8000/content/00012614f7d43df6418523445a6787d6
    +
    +HTTP/1.1 200 OK
    +Content-Length: 692
    +Cache-Control: max-age=14400
    +Content-Type: application/x-protobuf
    +
    +000000 0a b1 05 0a 20 30 30 30 31 32 36 31 34 66 37 64
    +000010 34 33 64 66 36 34 31 38 35 32 33 34 34 35 61 36
    +000020 37 38 37 64 36 12 85 02 12 0b 63 6f 6e 74 65 6e
    +000030 74 3a 72 61 77 18 8a e3 8c c5 9d 24 22 ee 01 3c
    +[...]
    +0002b0 69 6c 65 79
    +     +

    +

    + + +

    Cell Store (Single)

    + +

    +

    +PUT /<table>/<row>/<column>( : <qualifier> )? ( / <timestamp> )?
+
+POST /<table>/<row>/<column>( : <qualifier> )? ( / <timestamp> )?
+
    +

    +Stores cell data into the specified location. +If not successful, returns appropriate HTTP error status code. +If successful, returns HTTP 200 status. +Set Content-Type header to text/xml for XML encoding. +Set Content-Type header to application/x-protobuf for protobufs encoding. +Set Content-Type header to application/octet-stream for binary encoding. +When using binary encoding, optionally, set X-Timestamp header to the desired +timestamp. +

    +PUT and POST operations are equivalent here: Specified addresses without +existing data will create new values. Specified addresses with existing data +will create new versions, overwriting an existing version if all of { row, +column:qualifer, timestamp } match that of the existing value. +

    +See "Cell Query (Single Value)" section for encoding examples. +

    +Examples: +

    +

    + +% curl -H "Content-Type: text/xml" --data '[...]' http://localhost:8000/test/testrow/test:testcolumn
    +
    +HTTP/1.1 200 OK
    +Content-Length: 0
    +     +
    +

    + + +

    Cell Store (Multiple)

    + +

    +

    +PUT /<table>/<false-row-key>
+
+POST /<table>/<false-row-key>
+
    +

    +Use a false row key. Row, column, and timestamp values in supplied cells +override the specifications of the same on the path, allowing for posting of +multiple values to a table in batch. If not successful, returns appropriate +HTTP error status code. If successful, returns HTTP 200 status. +Set Content-Type to text/xml for XML encoding. +Set Content-Type header to application/x-protobuf for protobufs encoding. +Supply commit data in the PUT or POST body. +

    +PUT and POST operations are equivalent here: Specified addresses without +existing data will create new values. Specified addresses with existing data +will create new versions, overwriting an existing version if all of { row, +column:qualifer, timestamp } match that of the existing value. +

    +See "Cell or Row Query (Multiple Values)" for encoding examples. +

    + + +

    Row, Column, or Cell Delete

    + +

    +

    +DELETE /<table>/<row>
+  ( / ( <column> ( : <qualifier> )? 
+    ( / <timestamp> )? )?
+
    +

    +Deletes an entire row, a entire column family, or specific cell(s), depending +on how specific the data address. If not successful, returns appropriate HTTP +error status code. If successful, returns HTTP 200 status. +

    +NOTE: DELETE /<table> will not work. +Use DELETE /<table>/schema instead. +

    + + +

    Scanner Creation

    + +

    +

    +PUT /<table>/scanner
+
+POST /<table>/scanner
+
    +

    +Allocates a new table scanner. +If not successful, returns appropriate HTTP error status code. +If successful, returns HTTP 201 status (created) and the URI which should be +used to address the scanner, e.g. +

    +

    /<table>/scanner/112876541342014107c0fa92
    +

    +Set Content-Type to text/xml if supplying an XML scanner specification. +Set Content-Type to application/protobuf if supplying a protobufs +encoded specification. +

    +Examples: +

    +

    + +% curl -H "Content-Type: text/xml" -d '<Scanner batch="1"/>' http://localhost:8000/content/scanner
    +
    +HTTP/1.1 201 Created
    +Location: http://localhost:8000/content/scanner/12447063229213b1937
    +Content-Length: 0
    +     +
    +

    + + +

    Scanner Get Next

    + +

    +

    +GET /<table>/scanner/<scanner-id>
+
    +

    +Returns the values of the next cells found by the scanner, up to the configured batch amount. +Set Accept header to text/xml for XML encoding. +Set Accept header to application/x-protobuf for protobufs encoding. +Set Accept header to application/octet-stream for binary encoding. +If not successful, returns appropriate HTTP error status code. +If result is successful but the scanner is exhausted, returns HTTP 204 status (no content). +Otherwise, returns HTTP 200 status and row and cell data in the response body. +See examples from the "Cell or Row Query (Multiple Values)" section. +

    +NOTE: The binary encoding option returns only one cell regardless of the +batching parameter supplied during scanner creation. The row, column, and +timestamp associated with the cell are transmitted as X-headers: +X-Row, X-Column, and X-Timestamp respectively. +

    +Examples: +

    +

    + +% curl -H "Content-Type: text/xml" http://localhost:8000/content/scanner/12447063229213b1937
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: text/xml
    +Content-Length: 589
    +
    +<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
    +<CellSet><Row key="MDAyMDFjMTAwNjk4ZGNkYjU5MDQxNTVkZGQ3OGRlZTk="><Cell timesta
    +mp="1244701281234" column="Y29udGVudDpyYXc=">PCFET0NUWVBFIEhUTUwgUFVCTElDICItL
    +y9JRVRGLy9EVEQgSFRNTCAyLjAvL0VOIj4KPGh0bWw+PGhlYWQ+Cjx0aXRsZT40MDQgTm90IEZvdW5
    +kPC90aXRsZT4KPC9oZWFkPjxib2R5Pgo8aDE+Tm90IEZvdW5kPC9oMT4KPHA+VGhlIHJlcXVlc3RlZ
    +CBVUkwgL3JvYm90cy50eHQgd2FzIG5vdCBmb3VuZCBvbiB0aGlzIHNlcnZlci48L3A+Cjxocj4KPGF
    +kZHJlc3M+QXBhY2hlLzIuMi4zIChSZWQgSGF0KSBTZXJ2ZXIgYXQgd3gubWduZXR3b3JrLmNvbSBQb
    +3J0IDgwPC9hZGRyZXNzPgo8L2JvZHk+PC9odG1sPgo=</Cell></Row></CellSet>
    +
    +% curl -H "Content-Type: application/json" http://localhost:8000/content/scanner/12447063229213b1937
    +
    +HTTP/1.1 200 OK
    +Cache-Control: no-cache
    +Content-Type: application/json
    +Transfer-Encoding: chunked
    +
    +{"Row":{"@key":"MDAyMDFjMTAwNjk4ZGNkYjU5MDQxNTVkZGQ3OGRlZTk=","Cell":{"@timest
    +amp":"1244701281234","@column":"aW5mbzpjcmF3bGVyLWh0dHB8d3gubWduZXR3b3JrLmNvbX
    +w4MHxyb2JvdHMudHh0LTEyNDQ3MDEyODEyMzQ=","$":"eyJpcCI6IjE5OS4xOTMuMTAuMTAxIiwib
    +WltZXR5cGUiOiJ0ZXh0L2h0bWw7IGNoYXJzZXQ9aXNvLTg4NTktMSIsInZpYSI6Imh0dHA6Ly93eC5
    +tZ25ldHdvcmsuY29tOjgwL2pzL2N1cnJlbnRzaGFuZGxlci5qcyJ9"}}}
    +
    +% curl -H "Content-Type: application/x-protobuf" http://localhost:8000/content/scanner/12447063229213b1937
    +
    +HTTP/1.1 200 OK
    +Content-Length: 63
    +Cache-Control: no-cache
    +Content-Type: application/x-protobuf
    +
    +000000 0a 3d 0a 20 30 30 32 30 31 63 31 30 30 36 39 38
    +000010 64 63 64 62 35 39 30 34 31 35 35 64 64 64 37 38
    +000020 64 65 65 39 12 19 12 0b 69 6e 66 6f 3a 6c 65 6e
    +000030 67 74 68 18 d2 97 e9 ef 9c 24 22 03 32 39 30
    +
    +% curl -H "Content-Type: application/octet-stream" http://localhost:8000/content/scanner/12447063229213b1937
    +
    +HTTP/1.1 200 OK
    +Content-Length: 37
    +Cache-Control: no-cache
    +X-Column: dXJsOmh0dHB8d3gubWduZXR3b3JrLmNvbXw4MHxyb2JvdHMudHh0
    +X-Row: MDAyMDFjMTAwNjk4ZGNkYjU5MDQxNTVkZGQ3OGRlZTk=
    +X-Timestamp: 1244701281234
    +Content-Type: application/octet-stream
    +
    +000000 68 74 74 70 3a 2f 2f 77 78 2e 6d 67 6e 65 74 77
    +000010 6f 72 6b 2e 63 6f 6d 3a 38 30 2f 72 6f 62 6f 74
    +000020 73 2e 74 78 74
    +     +
    +

    + + +

    Scanner Deletion

    + +

    +

    +DELETE /<table>/scanner/<scanner-id>
+
    +

    +Deletes resources associated with the scanner. This is an optional action. +Scanners will expire after some globally configurable interval has elapsed +with no activity on the scanner. If not successful, returns appropriate HTTP +error status code. If successful, returns HTTP status 200. +

    +Examples: +

    +

    + +% telnet localhost 8000
    +DELETE http://localhost:8000/content/scanner/12447063229213b1937 HTTP/1.0
    +
    +HTTP/1.1 200 OK
    +Content-Length: 0
    +     +
    +

    + +

    + +

    XML Schema

    + +

    +

    +<schema targetNamespace="StargateSchema" elementFormDefault="qualified" 
+xmlns="http://www.w3.org/2001/XMLSchema" xmlns:tns="StargateSchema">
+
+    <element name="CellSet" type="tns:CellSet"></element>
+    
+    <complexType name="CellSet">
+      <sequence>
+        <element name="row" type="tns:Row" maxOccurs="unbounded" minOccurs="1"></element>
+      </sequence>
+    </complexType>
+
+    <complexType name="Row">
+      <sequence>
+        <element name="key" type="base64Binary"></element>
+        <element name="cell" type="tns:Cell" maxOccurs="unbounded" minOccurs="1"></element>
+      </sequence>
+    </complexType>
+
+    <complexType name="Cell">
+      <sequence>
+        <element name="value" maxOccurs="1" minOccurs="1"><simpleType><restriction base="base64Binary"></restriction></simpleType></element>
+      </sequence>
+      <attribute name="column" type="base64Binary" />
+      <attribute name="timestamp" type="int" />
+    </complexType>
+
+    <element name="Version" type="tns:Version"></element>
+    
+    <complexType name="Version">
+      <attribute name="Stargate" type="string"></attribute>
+      <attribute name="JVM" type="string"></attribute>
+      <attribute name="OS" type="string"></attribute>
+      <attribute name="Server" type="string"></attribute>
+      <attribute name="Jersey" type="string"></attribute>
+    </complexType>
+
+
+    <element name="TableList" type="tns:TableList"></element>
+    
+    <complexType name="TableList">
+      <sequence>
+        <element name="table" type="tns:Table" maxOccurs="unbounded" minOccurs="1"></element>
+      </sequence>
+    </complexType>
+
+    <complexType name="Table">
+      <sequence>
+        <element name="name" type="string"></element>
+      </sequence>
+    </complexType>
+
+    <element name="TableInfo" type="tns:TableInfo"></element>
+    
+    <complexType name="TableInfo">
+      <sequence>
+        <element name="region" type="tns:TableRegion" maxOccurs="unbounded" minOccurs="1"></element>
+      </sequence>
+      <attribute name="name" type="string"></attribute>
+    </complexType>
+
+    <complexType name="TableRegion">
+      <attribute name="name" type="string"></attribute>
+      <attribute name="id" type="int"></attribute>
+      <attribute name="startKey" type="base64Binary"></attribute>
+      <attribute name="endKey" type="base64Binary"></attribute>
+      <attribute name="location" type="string"></attribute>
+    </complexType>
+
+    <element name="TableSchema" type="tns:TableSchema"></element>
+    
+    <complexType name="TableSchema">
+      <sequence>
+        <element name="column" type="tns:ColumnSchema" maxOccurs="unbounded" minOccurs="1"></element>
+      </sequence>
+      <attribute name="name" type="string"></attribute>
+      <anyAttribute></anyAttribute>
+    </complexType>
+
+    <complexType name="ColumnSchema">
+      <attribute name="name" type="string"></attribute>
+      <anyAttribute></anyAttribute>
+    </complexType>
+
+    <element name="Scanner" type="tns:Scanner"></element>
+    
+    <complexType name="Scanner">
+      <attribute name="startRow" type="base64Binary"></attribute>
+      <attribute name="endRow" type="base64Binary"></attribute>
+      <attribute name="columns" type="base64Binary"></attribute>
+      <attribute name="batch" type="int"></attribute>
+      <attribute name="startTime" type="int"></attribute>
+      <attribute name="endTime" type="int"></attribute>
+    </complexType>
+
+    <element name="StorageClusterVersion"
+      type="tns:StorageClusterVersion">
+    </element>
+    
+    <complexType name="StorageClusterVersion">
+      <attribute name="version" type="string"></attribute>
+    </complexType>
+
+    <element name="StorageClusterStatus"
+      type="tns:StorageClusterStatus">
+    </element>
+    
+    <complexType name="StorageClusterStatus">
+      <sequence>
+        <element name="liveNode" type="tns:Node"
+          maxOccurs="unbounded" minOccurs="0">
+        </element>
+        <element name="deadNode" type="string" maxOccurs="unbounded"
+          minOccurs="0">
+        </element>
+      </sequence>
+      <attribute name="regions" type="int"></attribute>
+      <attribute name="requests" type="int"></attribute>
+      <attribute name="averageLoad" type="float"></attribute>
+    </complexType>
+
+    <complexType name="Node">
+      <sequence>
+        <element name="region" type="tns:Region" maxOccurs="unbounded" minOccurs="0"></element>
+      </sequence>
+      <attribute name="name" type="string"></attribute>
+      <attribute name="startCode" type="int"></attribute>
+      <attribute name="requests" type="int"></attribute>
+      <attribute name="heapSizeMB" type="int"></attribute>
+      <attribute name="maxHeapSizeMB" type="int"></attribute>
+    </complexType>
+
+    <complexType name="Region">
+      <attribute name="name" type="base64Binary"></attribute>
+      <attribute name="stores" type="int"></attribute>
+      <attribute name="storefiles" type="int"></attribute>
+      <attribute name="storefileSizeMB" type="int"></attribute>
+      <attribute name="memstoreSizeMB" type="int"></attribute>
+      <attribute name="storefileIndexSizeMB" type="int"></attribute>
+    </complexType>
+</schema>
+
    + +

    + +

    Protobufs Schema

    + +

    +

    +message Version {
+  optional string stargateVersion = 1;
+  optional string jvmVersion = 2;
+  optional string osVersion = 3;
+  optional string serverVersion = 4;
+  optional string jerseyVersion = 5;
+}
+
+message StorageClusterStatus {
+  message Region {
+    required bytes name = 1;
+    optional int32 stores = 2;
+    optional int32 storefiles = 3;
+    optional int32 storefileSizeMB = 4;
+    optional int32 memstoreSizeMB = 5;
+    optional int32 storefileIndexSizeMB = 6;
+  }
+  message Node {
+    required string name = 1;    // name:port
+    optional int64 startCode = 2;
+    optional int32 requests = 3;
+    optional int32 heapSizeMB = 4;
+    optional int32 maxHeapSizeMB = 5;
+    repeated Region regions = 6;
+  }
+  // node status
+  repeated Node liveNodes = 1;
+  repeated string deadNodes = 2;
+  // summary statistics
+  optional int32 regions = 3; 
+  optional int32 requests = 4; 
+  optional double averageLoad = 5;
+}
+
+message TableList {
+  repeated string name = 1;
+}
+
+message TableInfo {
+  required string name = 1;
+  message Region {
+    required string name = 1;
+    optional bytes startKey = 2;
+    optional bytes endKey = 3;
+    optional int64 id = 4;
+    optional string location = 5;
+  }
+  repeated Region regions = 2;
+}
+
+message TableSchema {
+  optional string name = 1;
+  message Attribute {
+    required string name = 1;
+    required string value = 2;
+  }  
+  repeated Attribute attrs = 2;
+  repeated ColumnSchema columns = 3;
+  // optional helpful encodings of commonly used attributes
+  optional bool inMemory = 4;
+  optional bool readOnly = 5;
+}
+
+message ColumnSchema {
+  optional string name = 1;
+  message Attribute {
+    required string name = 1;
+    required string value = 2;
+  }
+  repeated Attribute attrs = 2;
+  // optional helpful encodings of commonly used attributes
+  optional int32 ttl = 3;
+  optional int32 maxVersions = 4;
+  optional string compression = 5;
+}
+
+message Cell {
+  optional bytes row = 1;       // unused if Cell is in a CellSet
+  optional bytes column = 2;
+  optional int64 timestamp = 3;
+  optional bytes data = 4;
+}
+
+message CellSet {
+  message Row {
+    required bytes key = 1;
+    repeated Cell values = 2;
+  }
+  repeated Row rows = 1;
+}
+
+message Scanner {
+  optional bytes startRow = 1;
+  optional bytes endRow = 2;
+  repeated bytes columns = 3;
+  optional int32 batch = 4;
+  optional int64 startTime = 5;
+  optional int64 endTime = 6;
+}
+
    + + + diff --git a/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/CellMessage.java b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/CellMessage.java new file mode 100644 index 0000000..8d2e69f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/CellMessage.java @@ -0,0 +1,571 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: CellMessage.proto + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +public final class CellMessage { + private CellMessage() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public interface CellOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional bytes row = 1; + boolean hasRow(); + com.google.protobuf.ByteString getRow(); + + // optional bytes column = 2; + boolean hasColumn(); + com.google.protobuf.ByteString getColumn(); + + // optional int64 timestamp = 3; + boolean hasTimestamp(); + long getTimestamp(); + + // optional bytes data = 4; + boolean hasData(); + com.google.protobuf.ByteString getData(); + } + public static final class Cell extends + com.google.protobuf.GeneratedMessage + implements CellOrBuilder { + // Use Cell.newBuilder() to construct. + private Cell(Builder builder) { + super(builder); + } + private Cell(boolean noInit) {} + + private static final Cell defaultInstance; + public static Cell getDefaultInstance() { + return defaultInstance; + } + + public Cell getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Cell_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Cell_fieldAccessorTable; + } + + private int bitField0_; + // optional bytes row = 1; + public static final int ROW_FIELD_NUMBER = 1; + private com.google.protobuf.ByteString row_; + public boolean hasRow() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public com.google.protobuf.ByteString getRow() { + return row_; + } + + // optional bytes column = 2; + public static final int COLUMN_FIELD_NUMBER = 2; + private com.google.protobuf.ByteString column_; + public boolean hasColumn() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public com.google.protobuf.ByteString getColumn() { + return column_; + } + + // optional int64 timestamp = 3; + public static final int TIMESTAMP_FIELD_NUMBER = 3; + private long timestamp_; + public boolean hasTimestamp() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public long getTimestamp() { + return timestamp_; + } + + // optional bytes data = 4; + public static final int DATA_FIELD_NUMBER = 4; + private com.google.protobuf.ByteString data_; + public boolean hasData() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public com.google.protobuf.ByteString getData() { + return data_; + } + + private void initFields() { + row_ = com.google.protobuf.ByteString.EMPTY; + column_ = com.google.protobuf.ByteString.EMPTY; + timestamp_ = 0L; + data_ = com.google.protobuf.ByteString.EMPTY; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, row_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, column_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt64(3, timestamp_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeBytes(4, data_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, row_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(2, column_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(3, timestamp_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(4, data_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.CellOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Cell_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Cell_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + row_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + column_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + timestamp_ = 0L; + bitField0_ = (bitField0_ & ~0x00000004); + data_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000008); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell build() { + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell result = new org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.row_ = row_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.column_ = column_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.timestamp_ = timestamp_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.data_ = data_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.getDefaultInstance()) return this; + if (other.hasRow()) { + setRow(other.getRow()); + } + if (other.hasColumn()) { + setColumn(other.getColumn()); + } + if (other.hasTimestamp()) { + setTimestamp(other.getTimestamp()); + } + if (other.hasData()) { + setData(other.getData()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + row_ = input.readBytes(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + column_ = input.readBytes(); + break; + } + case 24: { + bitField0_ |= 0x00000004; + timestamp_ = input.readInt64(); + break; + } + case 34: { + bitField0_ |= 0x00000008; + data_ = input.readBytes(); + break; + } + } + } + } + + private int bitField0_; + + // optional bytes row = 1; + private com.google.protobuf.ByteString row_ = com.google.protobuf.ByteString.EMPTY; + public boolean hasRow() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public com.google.protobuf.ByteString getRow() { + return row_; + } + public Builder setRow(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + row_ = value; + onChanged(); + return this; + } + public Builder clearRow() { + bitField0_ = (bitField0_ & ~0x00000001); + row_ = getDefaultInstance().getRow(); + onChanged(); + return this; + } + + // optional bytes column = 2; + private com.google.protobuf.ByteString column_ = com.google.protobuf.ByteString.EMPTY; + public boolean hasColumn() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public com.google.protobuf.ByteString getColumn() { + return column_; + } + public Builder setColumn(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + column_ = value; + onChanged(); + return this; + } + public Builder clearColumn() { + bitField0_ = (bitField0_ & ~0x00000002); + column_ = getDefaultInstance().getColumn(); + onChanged(); + return this; + } + + // optional int64 timestamp = 3; + private long timestamp_ ; + public boolean hasTimestamp() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public long getTimestamp() { + return timestamp_; + } + public Builder setTimestamp(long value) { + bitField0_ |= 0x00000004; + timestamp_ = value; + onChanged(); + return this; + } + public Builder clearTimestamp() { + bitField0_ = (bitField0_ & ~0x00000004); + timestamp_ = 0L; + onChanged(); + return this; + } + + // optional bytes data = 4; + private com.google.protobuf.ByteString data_ = com.google.protobuf.ByteString.EMPTY; + public boolean hasData() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public com.google.protobuf.ByteString getData() { + return data_; + } + public Builder setData(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000008; + data_ = value; + onChanged(); + return this; + } + public Builder clearData() { + bitField0_ = (bitField0_ & ~0x00000008); + data_ = getDefaultInstance().getData(); + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.Cell) + } + + static { + defaultInstance = new Cell(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.Cell) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Cell_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Cell_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\021CellMessage.proto\022/org.apache.hadoop.h" + + "base.rest.protobuf.generated\"D\n\004Cell\022\013\n\003" + + "row\030\001 \001(\014\022\016\n\006column\030\002 \001(\014\022\021\n\ttimestamp\030\003" + + " \001(\003\022\014\n\004data\030\004 \001(\014" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Cell_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Cell_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Cell_descriptor, + new java.lang.String[] { "Row", "Column", "Timestamp", "Data", }, + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.class, + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.Builder.class); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/CellSetMessage.java b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/CellSetMessage.java new file mode 100644 index 0000000..fb5bc2c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/CellSetMessage.java @@ -0,0 +1,1255 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: CellSetMessage.proto + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +public final class CellSetMessage { + private CellSetMessage() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public interface CellSetOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.CellSet.Row rows = 1; + java.util.List + getRowsList(); + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row getRows(int index); + int getRowsCount(); + java.util.List + getRowsOrBuilderList(); + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.RowOrBuilder getRowsOrBuilder( + int index); + } + public static final class CellSet extends + com.google.protobuf.GeneratedMessage + implements CellSetOrBuilder { + // Use CellSet.newBuilder() to construct. + private CellSet(Builder builder) { + super(builder); + } + private CellSet(boolean noInit) {} + + private static final CellSet defaultInstance; + public static CellSet getDefaultInstance() { + return defaultInstance; + } + + public CellSet getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_fieldAccessorTable; + } + + public interface RowOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required bytes key = 1; + boolean hasKey(); + com.google.protobuf.ByteString getKey(); + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.Cell values = 2; + java.util.List + getValuesList(); + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell getValues(int index); + int getValuesCount(); + java.util.List + getValuesOrBuilderList(); + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.CellOrBuilder getValuesOrBuilder( + int index); + } + public static final class Row extends + com.google.protobuf.GeneratedMessage + implements RowOrBuilder { + // Use Row.newBuilder() to construct. + private Row(Builder builder) { + super(builder); + } + private Row(boolean noInit) {} + + private static final Row defaultInstance; + public static Row getDefaultInstance() { + return defaultInstance; + } + + public Row getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_Row_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_Row_fieldAccessorTable; + } + + private int bitField0_; + // required bytes key = 1; + public static final int KEY_FIELD_NUMBER = 1; + private com.google.protobuf.ByteString key_; + public boolean hasKey() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public com.google.protobuf.ByteString getKey() { + return key_; + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.Cell values = 2; + public static final int VALUES_FIELD_NUMBER = 2; + private java.util.List values_; + public java.util.List getValuesList() { + return values_; + } + public java.util.List + getValuesOrBuilderList() { + return values_; + } + public int getValuesCount() { + return values_.size(); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell getValues(int index) { + return values_.get(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.CellOrBuilder getValuesOrBuilder( + int index) { + return values_.get(index); + } + + private void initFields() { + key_ = com.google.protobuf.ByteString.EMPTY; + values_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasKey()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, key_); + } + for (int i = 0; i < values_.size(); i++) { + output.writeMessage(2, values_.get(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, key_); + } + for (int i = 0; i < values_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(2, values_.get(i)); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.RowOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_Row_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_Row_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getValuesFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + key_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + if (valuesBuilder_ == null) { + values_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + } else { + valuesBuilder_.clear(); + } + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row build() { + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row result = new org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.key_ = key_; + if (valuesBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002)) { + values_ = java.util.Collections.unmodifiableList(values_); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.values_ = values_; + } else { + result.values_ = valuesBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.getDefaultInstance()) return this; + if (other.hasKey()) { + setKey(other.getKey()); + } + if (valuesBuilder_ == null) { + if (!other.values_.isEmpty()) { + if (values_.isEmpty()) { + values_ = other.values_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureValuesIsMutable(); + values_.addAll(other.values_); + } + onChanged(); + } + } else { + if (!other.values_.isEmpty()) { + if (valuesBuilder_.isEmpty()) { + valuesBuilder_.dispose(); + valuesBuilder_ = null; + values_ = other.values_; + bitField0_ = (bitField0_ & ~0x00000002); + valuesBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getValuesFieldBuilder() : null; + } else { + valuesBuilder_.addAllMessages(other.values_); + } + } + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasKey()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + key_ = input.readBytes(); + break; + } + case 18: { + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.Builder subBuilder = org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addValues(subBuilder.buildPartial()); + break; + } + } + } + } + + private int bitField0_; + + // required bytes key = 1; + private com.google.protobuf.ByteString key_ = com.google.protobuf.ByteString.EMPTY; + public boolean hasKey() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public com.google.protobuf.ByteString getKey() { + return key_; + } + public Builder setKey(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + key_ = value; + onChanged(); + return this; + } + public Builder clearKey() { + bitField0_ = (bitField0_ & ~0x00000001); + key_ = getDefaultInstance().getKey(); + onChanged(); + return this; + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.Cell values = 2; + private java.util.List values_ = + java.util.Collections.emptyList(); + private void ensureValuesIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + values_ = new java.util.ArrayList(values_); + bitField0_ |= 0x00000002; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell, org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.CellOrBuilder> valuesBuilder_; + + public java.util.List getValuesList() { + if (valuesBuilder_ == null) { + return java.util.Collections.unmodifiableList(values_); + } else { + return valuesBuilder_.getMessageList(); + } + } + public int getValuesCount() { + if (valuesBuilder_ == null) { + return values_.size(); + } else { + return valuesBuilder_.getCount(); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell getValues(int index) { + if (valuesBuilder_ == null) { + return values_.get(index); + } else { + return valuesBuilder_.getMessage(index); + } + } + public Builder setValues( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell value) { + if (valuesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureValuesIsMutable(); + values_.set(index, value); + onChanged(); + } else { + valuesBuilder_.setMessage(index, value); + } + return this; + } + public Builder setValues( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.Builder builderForValue) { + if (valuesBuilder_ == null) { + ensureValuesIsMutable(); + values_.set(index, builderForValue.build()); + onChanged(); + } else { + valuesBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addValues(org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell value) { + if (valuesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureValuesIsMutable(); + values_.add(value); + onChanged(); + } else { + valuesBuilder_.addMessage(value); + } + return this; + } + public Builder addValues( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell value) { + if (valuesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureValuesIsMutable(); + values_.add(index, value); + onChanged(); + } else { + valuesBuilder_.addMessage(index, value); + } + return this; + } + public Builder addValues( + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.Builder builderForValue) { + if (valuesBuilder_ == null) { + ensureValuesIsMutable(); + values_.add(builderForValue.build()); + onChanged(); + } else { + valuesBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addValues( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.Builder builderForValue) { + if (valuesBuilder_ == null) { + ensureValuesIsMutable(); + values_.add(index, builderForValue.build()); + onChanged(); + } else { + valuesBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllValues( + java.lang.Iterable values) { + if (valuesBuilder_ == null) { + ensureValuesIsMutable(); + super.addAll(values, values_); + onChanged(); + } else { + valuesBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearValues() { + if (valuesBuilder_ == null) { + values_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + } else { + valuesBuilder_.clear(); + } + return this; + } + public Builder removeValues(int index) { + if (valuesBuilder_ == null) { + ensureValuesIsMutable(); + values_.remove(index); + onChanged(); + } else { + valuesBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.Builder getValuesBuilder( + int index) { + return getValuesFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.CellOrBuilder getValuesOrBuilder( + int index) { + if (valuesBuilder_ == null) { + return values_.get(index); } else { + return valuesBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getValuesOrBuilderList() { + if (valuesBuilder_ != null) { + return valuesBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(values_); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.Builder addValuesBuilder() { + return getValuesFieldBuilder().addBuilder( + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.getDefaultInstance()); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.Builder addValuesBuilder( + int index) { + return getValuesFieldBuilder().addBuilder( + index, org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.getDefaultInstance()); + } + public java.util.List + getValuesBuilderList() { + return getValuesFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell, org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.CellOrBuilder> + getValuesFieldBuilder() { + if (valuesBuilder_ == null) { + valuesBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell, org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.Cell.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.CellOrBuilder>( + values_, + ((bitField0_ & 0x00000002) == 0x00000002), + getParentForChildren(), + isClean()); + values_ = null; + } + return valuesBuilder_; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.CellSet.Row) + } + + static { + defaultInstance = new Row(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.CellSet.Row) + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.CellSet.Row rows = 1; + public static final int ROWS_FIELD_NUMBER = 1; + private java.util.List rows_; + public java.util.List getRowsList() { + return rows_; + } + public java.util.List + getRowsOrBuilderList() { + return rows_; + } + public int getRowsCount() { + return rows_.size(); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row getRows(int index) { + return rows_.get(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.RowOrBuilder getRowsOrBuilder( + int index) { + return rows_.get(index); + } + + private void initFields() { + rows_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + for (int i = 0; i < getRowsCount(); i++) { + if (!getRows(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + for (int i = 0; i < rows_.size(); i++) { + output.writeMessage(1, rows_.get(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < rows_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, rows_.get(i)); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSetOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getRowsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (rowsBuilder_ == null) { + rows_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + } else { + rowsBuilder_.clear(); + } + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet build() { + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet result = new org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet(this); + int from_bitField0_ = bitField0_; + if (rowsBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + rows_ = java.util.Collections.unmodifiableList(rows_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.rows_ = rows_; + } else { + result.rows_ = rowsBuilder_.build(); + } + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.getDefaultInstance()) return this; + if (rowsBuilder_ == null) { + if (!other.rows_.isEmpty()) { + if (rows_.isEmpty()) { + rows_ = other.rows_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureRowsIsMutable(); + rows_.addAll(other.rows_); + } + onChanged(); + } + } else { + if (!other.rows_.isEmpty()) { + if (rowsBuilder_.isEmpty()) { + rowsBuilder_.dispose(); + rowsBuilder_ = null; + rows_ = other.rows_; + bitField0_ = (bitField0_ & ~0x00000001); + rowsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getRowsFieldBuilder() : null; + } else { + rowsBuilder_.addAllMessages(other.rows_); + } + } + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + for (int i = 0; i < getRowsCount(); i++) { + if (!getRows(i).isInitialized()) { + + return false; + } + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.Builder subBuilder = org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addRows(subBuilder.buildPartial()); + break; + } + } + } + } + + private int bitField0_; + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.CellSet.Row rows = 1; + private java.util.List rows_ = + java.util.Collections.emptyList(); + private void ensureRowsIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + rows_ = new java.util.ArrayList(rows_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row, org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.RowOrBuilder> rowsBuilder_; + + public java.util.List getRowsList() { + if (rowsBuilder_ == null) { + return java.util.Collections.unmodifiableList(rows_); + } else { + return rowsBuilder_.getMessageList(); + } + } + public int getRowsCount() { + if (rowsBuilder_ == null) { + return rows_.size(); + } else { + return rowsBuilder_.getCount(); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row getRows(int index) { + if (rowsBuilder_ == null) { + return rows_.get(index); + } else { + return rowsBuilder_.getMessage(index); + } + } + public Builder setRows( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row value) { + if (rowsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRowsIsMutable(); + rows_.set(index, value); + onChanged(); + } else { + rowsBuilder_.setMessage(index, value); + } + return this; + } + public Builder setRows( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.Builder builderForValue) { + if (rowsBuilder_ == null) { + ensureRowsIsMutable(); + rows_.set(index, builderForValue.build()); + onChanged(); + } else { + rowsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addRows(org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row value) { + if (rowsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRowsIsMutable(); + rows_.add(value); + onChanged(); + } else { + rowsBuilder_.addMessage(value); + } + return this; + } + public Builder addRows( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row value) { + if (rowsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRowsIsMutable(); + rows_.add(index, value); + onChanged(); + } else { + rowsBuilder_.addMessage(index, value); + } + return this; + } + public Builder addRows( + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.Builder builderForValue) { + if (rowsBuilder_ == null) { + ensureRowsIsMutable(); + rows_.add(builderForValue.build()); + onChanged(); + } else { + rowsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addRows( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.Builder builderForValue) { + if (rowsBuilder_ == null) { + ensureRowsIsMutable(); + rows_.add(index, builderForValue.build()); + onChanged(); + } else { + rowsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllRows( + java.lang.Iterable values) { + if (rowsBuilder_ == null) { + ensureRowsIsMutable(); + super.addAll(values, rows_); + onChanged(); + } else { + rowsBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearRows() { + if (rowsBuilder_ == null) { + rows_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + rowsBuilder_.clear(); + } + return this; + } + public Builder removeRows(int index) { + if (rowsBuilder_ == null) { + ensureRowsIsMutable(); + rows_.remove(index); + onChanged(); + } else { + rowsBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.Builder getRowsBuilder( + int index) { + return getRowsFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.RowOrBuilder getRowsOrBuilder( + int index) { + if (rowsBuilder_ == null) { + return rows_.get(index); } else { + return rowsBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getRowsOrBuilderList() { + if (rowsBuilder_ != null) { + return rowsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(rows_); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.Builder addRowsBuilder() { + return getRowsFieldBuilder().addBuilder( + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.getDefaultInstance()); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.Builder addRowsBuilder( + int index) { + return getRowsFieldBuilder().addBuilder( + index, org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.getDefaultInstance()); + } + public java.util.List + getRowsBuilderList() { + return getRowsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row, org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.RowOrBuilder> + getRowsFieldBuilder() { + if (rowsBuilder_ == null) { + rowsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row, org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.RowOrBuilder>( + rows_, + ((bitField0_ & 0x00000001) == 0x00000001), + getParentForChildren(), + isClean()); + rows_ = null; + } + return rowsBuilder_; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.CellSet) + } + + static { + defaultInstance = new CellSet(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.CellSet) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_Row_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_Row_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\024CellSetMessage.proto\022/org.apache.hadoo" + + "p.hbase.rest.protobuf.generated\032\021CellMes" + + "sage.proto\"\260\001\n\007CellSet\022J\n\004rows\030\001 \003(\0132<.o" + + "rg.apache.hadoop.hbase.rest.protobuf.gen" + + "erated.CellSet.Row\032Y\n\003Row\022\013\n\003key\030\001 \002(\014\022E" + + "\n\006values\030\002 \003(\01325.org.apache.hadoop.hbase" + + ".rest.protobuf.generated.Cell" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_descriptor, + new java.lang.String[] { "Rows", }, + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.class, + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Builder.class); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_Row_descriptor = + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_descriptor.getNestedTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_Row_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_CellSet_Row_descriptor, + new java.lang.String[] { "Key", "Values", }, + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.class, + org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet.Row.Builder.class); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + org.apache.hadoop.hbase.rest.protobuf.generated.CellMessage.getDescriptor(), + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/ColumnSchemaMessage.java b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/ColumnSchemaMessage.java new file mode 100644 index 0000000..07561c2 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/ColumnSchemaMessage.java @@ -0,0 +1,1423 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: ColumnSchemaMessage.proto + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +public final class ColumnSchemaMessage { + private ColumnSchemaMessage() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public interface ColumnSchemaOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional string name = 1; + boolean hasName(); + String getName(); + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchema.Attribute attrs = 2; + java.util.List + getAttrsList(); + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute getAttrs(int index); + int getAttrsCount(); + java.util.List + getAttrsOrBuilderList(); + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.AttributeOrBuilder getAttrsOrBuilder( + int index); + + // optional int32 ttl = 3; + boolean hasTtl(); + int getTtl(); + + // optional int32 maxVersions = 4; + boolean hasMaxVersions(); + int getMaxVersions(); + + // optional string compression = 5; + boolean hasCompression(); + String getCompression(); + } + public static final class ColumnSchema extends + com.google.protobuf.GeneratedMessage + implements ColumnSchemaOrBuilder { + // Use ColumnSchema.newBuilder() to construct. + private ColumnSchema(Builder builder) { + super(builder); + } + private ColumnSchema(boolean noInit) {} + + private static final ColumnSchema defaultInstance; + public static ColumnSchema getDefaultInstance() { + return defaultInstance; + } + + public ColumnSchema getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_fieldAccessorTable; + } + + public interface AttributeOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required string name = 1; + boolean hasName(); + String getName(); + + // required string value = 2; + boolean hasValue(); + String getValue(); + } + public static final class Attribute extends + com.google.protobuf.GeneratedMessage + implements AttributeOrBuilder { + // Use Attribute.newBuilder() to construct. + private Attribute(Builder builder) { + super(builder); + } + private Attribute(boolean noInit) {} + + private static final Attribute defaultInstance; + public static Attribute getDefaultInstance() { + return defaultInstance; + } + + public Attribute getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_Attribute_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_Attribute_fieldAccessorTable; + } + + private int bitField0_; + // required string name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private java.lang.Object name_; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + name_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // required string value = 2; + public static final int VALUE_FIELD_NUMBER = 2; + private java.lang.Object value_; + public boolean hasValue() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getValue() { + java.lang.Object ref = value_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + value_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getValueBytes() { + java.lang.Object ref = value_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + value_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + name_ = ""; + value_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasName()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasValue()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, getValueBytes()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(2, getValueBytes()); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.AttributeOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_Attribute_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_Attribute_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + value_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute build() { + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute result = new org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.value_ = value_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.getDefaultInstance()) return this; + if (other.hasName()) { + setName(other.getName()); + } + if (other.hasValue()) { + setValue(other.getValue()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasName()) { + + return false; + } + if (!hasValue()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + name_ = input.readBytes(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + value_ = input.readBytes(); + break; + } + } + } + } + + private int bitField0_; + + // required string name = 1; + private java.lang.Object name_ = ""; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + name_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setName(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + void setName(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + } + + // required string value = 2; + private java.lang.Object value_ = ""; + public boolean hasValue() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getValue() { + java.lang.Object ref = value_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + value_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setValue(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + value_ = value; + onChanged(); + return this; + } + public Builder clearValue() { + bitField0_ = (bitField0_ & ~0x00000002); + value_ = getDefaultInstance().getValue(); + onChanged(); + return this; + } + void setValue(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000002; + value_ = value; + onChanged(); + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchema.Attribute) + } + + static { + defaultInstance = new Attribute(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchema.Attribute) + } + + private int bitField0_; + // optional string name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private java.lang.Object name_; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + name_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchema.Attribute attrs = 2; + public static final int ATTRS_FIELD_NUMBER = 2; + private java.util.List attrs_; + public java.util.List getAttrsList() { + return attrs_; + } + public java.util.List + getAttrsOrBuilderList() { + return attrs_; + } + public int getAttrsCount() { + return attrs_.size(); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute getAttrs(int index) { + return attrs_.get(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.AttributeOrBuilder getAttrsOrBuilder( + int index) { + return attrs_.get(index); + } + + // optional int32 ttl = 3; + public static final int TTL_FIELD_NUMBER = 3; + private int ttl_; + public boolean hasTtl() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public int getTtl() { + return ttl_; + } + + // optional int32 maxVersions = 4; + public static final int MAXVERSIONS_FIELD_NUMBER = 4; + private int maxVersions_; + public boolean hasMaxVersions() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public int getMaxVersions() { + return maxVersions_; + } + + // optional string compression = 5; + public static final int COMPRESSION_FIELD_NUMBER = 5; + private java.lang.Object compression_; + public boolean hasCompression() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public String getCompression() { + java.lang.Object ref = compression_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + compression_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getCompressionBytes() { + java.lang.Object ref = compression_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + compression_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + name_ = ""; + attrs_ = java.util.Collections.emptyList(); + ttl_ = 0; + maxVersions_ = 0; + compression_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + for (int i = 0; i < getAttrsCount(); i++) { + if (!getAttrs(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getNameBytes()); + } + for (int i = 0; i < attrs_.size(); i++) { + output.writeMessage(2, attrs_.get(i)); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt32(3, ttl_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt32(4, maxVersions_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeBytes(5, getCompressionBytes()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getNameBytes()); + } + for (int i = 0; i < attrs_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(2, attrs_.get(i)); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(3, ttl_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(4, maxVersions_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(5, getCompressionBytes()); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchemaOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getAttrsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + if (attrsBuilder_ == null) { + attrs_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + } else { + attrsBuilder_.clear(); + } + ttl_ = 0; + bitField0_ = (bitField0_ & ~0x00000004); + maxVersions_ = 0; + bitField0_ = (bitField0_ & ~0x00000008); + compression_ = ""; + bitField0_ = (bitField0_ & ~0x00000010); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema build() { + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema result = new org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + if (attrsBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002)) { + attrs_ = java.util.Collections.unmodifiableList(attrs_); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.attrs_ = attrs_; + } else { + result.attrs_ = attrsBuilder_.build(); + } + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000002; + } + result.ttl_ = ttl_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000004; + } + result.maxVersions_ = maxVersions_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000008; + } + result.compression_ = compression_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.getDefaultInstance()) return this; + if (other.hasName()) { + setName(other.getName()); + } + if (attrsBuilder_ == null) { + if (!other.attrs_.isEmpty()) { + if (attrs_.isEmpty()) { + attrs_ = other.attrs_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureAttrsIsMutable(); + attrs_.addAll(other.attrs_); + } + onChanged(); + } + } else { + if (!other.attrs_.isEmpty()) { + if (attrsBuilder_.isEmpty()) { + attrsBuilder_.dispose(); + attrsBuilder_ = null; + attrs_ = other.attrs_; + bitField0_ = (bitField0_ & ~0x00000002); + attrsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getAttrsFieldBuilder() : null; + } else { + attrsBuilder_.addAllMessages(other.attrs_); + } + } + } + if (other.hasTtl()) { + setTtl(other.getTtl()); + } + if (other.hasMaxVersions()) { + setMaxVersions(other.getMaxVersions()); + } + if (other.hasCompression()) { + setCompression(other.getCompression()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + for (int i = 0; i < getAttrsCount(); i++) { + if (!getAttrs(i).isInitialized()) { + + return false; + } + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + name_ = input.readBytes(); + break; + } + case 18: { + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.Builder subBuilder = org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addAttrs(subBuilder.buildPartial()); + break; + } + case 24: { + bitField0_ |= 0x00000004; + ttl_ = input.readInt32(); + break; + } + case 32: { + bitField0_ |= 0x00000008; + maxVersions_ = input.readInt32(); + break; + } + case 42: { + bitField0_ |= 0x00000010; + compression_ = input.readBytes(); + break; + } + } + } + } + + private int bitField0_; + + // optional string name = 1; + private java.lang.Object name_ = ""; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + name_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setName(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + void setName(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchema.Attribute attrs = 2; + private java.util.List attrs_ = + java.util.Collections.emptyList(); + private void ensureAttrsIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + attrs_ = new java.util.ArrayList(attrs_); + bitField0_ |= 0x00000002; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.AttributeOrBuilder> attrsBuilder_; + + public java.util.List getAttrsList() { + if (attrsBuilder_ == null) { + return java.util.Collections.unmodifiableList(attrs_); + } else { + return attrsBuilder_.getMessageList(); + } + } + public int getAttrsCount() { + if (attrsBuilder_ == null) { + return attrs_.size(); + } else { + return attrsBuilder_.getCount(); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute getAttrs(int index) { + if (attrsBuilder_ == null) { + return attrs_.get(index); + } else { + return attrsBuilder_.getMessage(index); + } + } + public Builder setAttrs( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute value) { + if (attrsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureAttrsIsMutable(); + attrs_.set(index, value); + onChanged(); + } else { + attrsBuilder_.setMessage(index, value); + } + return this; + } + public Builder setAttrs( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.Builder builderForValue) { + if (attrsBuilder_ == null) { + ensureAttrsIsMutable(); + attrs_.set(index, builderForValue.build()); + onChanged(); + } else { + attrsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAttrs(org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute value) { + if (attrsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureAttrsIsMutable(); + attrs_.add(value); + onChanged(); + } else { + attrsBuilder_.addMessage(value); + } + return this; + } + public Builder addAttrs( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute value) { + if (attrsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureAttrsIsMutable(); + attrs_.add(index, value); + onChanged(); + } else { + attrsBuilder_.addMessage(index, value); + } + return this; + } + public Builder addAttrs( + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.Builder builderForValue) { + if (attrsBuilder_ == null) { + ensureAttrsIsMutable(); + attrs_.add(builderForValue.build()); + onChanged(); + } else { + attrsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addAttrs( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.Builder builderForValue) { + if (attrsBuilder_ == null) { + ensureAttrsIsMutable(); + attrs_.add(index, builderForValue.build()); + onChanged(); + } else { + attrsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllAttrs( + java.lang.Iterable values) { + if (attrsBuilder_ == null) { + ensureAttrsIsMutable(); + super.addAll(values, attrs_); + onChanged(); + } else { + attrsBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearAttrs() { + if (attrsBuilder_ == null) { + attrs_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + } else { + attrsBuilder_.clear(); + } + return this; + } + public Builder removeAttrs(int index) { + if (attrsBuilder_ == null) { + ensureAttrsIsMutable(); + attrs_.remove(index); + onChanged(); + } else { + attrsBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.Builder getAttrsBuilder( + int index) { + return getAttrsFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.AttributeOrBuilder getAttrsOrBuilder( + int index) { + if (attrsBuilder_ == null) { + return attrs_.get(index); } else { + return attrsBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getAttrsOrBuilderList() { + if (attrsBuilder_ != null) { + return attrsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(attrs_); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.Builder addAttrsBuilder() { + return getAttrsFieldBuilder().addBuilder( + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.getDefaultInstance()); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.Builder addAttrsBuilder( + int index) { + return getAttrsFieldBuilder().addBuilder( + index, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.getDefaultInstance()); + } + public java.util.List + getAttrsBuilderList() { + return getAttrsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.AttributeOrBuilder> + getAttrsFieldBuilder() { + if (attrsBuilder_ == null) { + attrsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.AttributeOrBuilder>( + attrs_, + ((bitField0_ & 0x00000002) == 0x00000002), + getParentForChildren(), + isClean()); + attrs_ = null; + } + return attrsBuilder_; + } + + // optional int32 ttl = 3; + private int ttl_ ; + public boolean hasTtl() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public int getTtl() { + return ttl_; + } + public Builder setTtl(int value) { + bitField0_ |= 0x00000004; + ttl_ = value; + onChanged(); + return this; + } + public Builder clearTtl() { + bitField0_ = (bitField0_ & ~0x00000004); + ttl_ = 0; + onChanged(); + return this; + } + + // optional int32 maxVersions = 4; + private int maxVersions_ ; + public boolean hasMaxVersions() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getMaxVersions() { + return maxVersions_; + } + public Builder setMaxVersions(int value) { + bitField0_ |= 0x00000008; + maxVersions_ = value; + onChanged(); + return this; + } + public Builder clearMaxVersions() { + bitField0_ = (bitField0_ & ~0x00000008); + maxVersions_ = 0; + onChanged(); + return this; + } + + // optional string compression = 5; + private java.lang.Object compression_ = ""; + public boolean hasCompression() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public String getCompression() { + java.lang.Object ref = compression_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + compression_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setCompression(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000010; + compression_ = value; + onChanged(); + return this; + } + public Builder clearCompression() { + bitField0_ = (bitField0_ & ~0x00000010); + compression_ = getDefaultInstance().getCompression(); + onChanged(); + return this; + } + void setCompression(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000010; + compression_ = value; + onChanged(); + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchema) + } + + static { + defaultInstance = new ColumnSchema(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchema) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_Attribute_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_Attribute_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\031ColumnSchemaMessage.proto\022/org.apache." + + "hadoop.hbase.rest.protobuf.generated\"\325\001\n" + + "\014ColumnSchema\022\014\n\004name\030\001 \001(\t\022V\n\005attrs\030\002 \003" + + "(\0132G.org.apache.hadoop.hbase.rest.protob" + + "uf.generated.ColumnSchema.Attribute\022\013\n\003t" + + "tl\030\003 \001(\005\022\023\n\013maxVersions\030\004 \001(\005\022\023\n\013compres" + + "sion\030\005 \001(\t\032(\n\tAttribute\022\014\n\004name\030\001 \002(\t\022\r\n" + + "\005value\030\002 \002(\t" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_descriptor, + new java.lang.String[] { "Name", "Attrs", "Ttl", "MaxVersions", "Compression", }, + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.class, + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Builder.class); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_Attribute_descriptor = + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_descriptor.getNestedTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_Attribute_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_ColumnSchema_Attribute_descriptor, + new java.lang.String[] { "Name", "Value", }, + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.class, + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Attribute.Builder.class); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/ScannerMessage.java b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/ScannerMessage.java new file mode 100644 index 0000000..65d614c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/ScannerMessage.java @@ -0,0 +1,883 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: ScannerMessage.proto + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +public final class ScannerMessage { + private ScannerMessage() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public interface ScannerOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional bytes startRow = 1; + boolean hasStartRow(); + com.google.protobuf.ByteString getStartRow(); + + // optional bytes endRow = 2; + boolean hasEndRow(); + com.google.protobuf.ByteString getEndRow(); + + // repeated bytes columns = 3; + java.util.List getColumnsList(); + int getColumnsCount(); + com.google.protobuf.ByteString getColumns(int index); + + // optional int32 batch = 4; + boolean hasBatch(); + int getBatch(); + + // optional int64 startTime = 5; + boolean hasStartTime(); + long getStartTime(); + + // optional int64 endTime = 6; + boolean hasEndTime(); + long getEndTime(); + + // optional int32 maxVersions = 7; + boolean hasMaxVersions(); + int getMaxVersions(); + + // optional string filter = 8; + boolean hasFilter(); + String getFilter(); + } + public static final class Scanner extends + com.google.protobuf.GeneratedMessage + implements ScannerOrBuilder { + // Use Scanner.newBuilder() to construct. + private Scanner(Builder builder) { + super(builder); + } + private Scanner(boolean noInit) {} + + private static final Scanner defaultInstance; + public static Scanner getDefaultInstance() { + return defaultInstance; + } + + public Scanner getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Scanner_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Scanner_fieldAccessorTable; + } + + private int bitField0_; + // optional bytes startRow = 1; + public static final int STARTROW_FIELD_NUMBER = 1; + private com.google.protobuf.ByteString startRow_; + public boolean hasStartRow() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public com.google.protobuf.ByteString getStartRow() { + return startRow_; + } + + // optional bytes endRow = 2; + public static final int ENDROW_FIELD_NUMBER = 2; + private com.google.protobuf.ByteString endRow_; + public boolean hasEndRow() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public com.google.protobuf.ByteString getEndRow() { + return endRow_; + } + + // repeated bytes columns = 3; + public static final int COLUMNS_FIELD_NUMBER = 3; + private java.util.List columns_; + public java.util.List + getColumnsList() { + return columns_; + } + public int getColumnsCount() { + return columns_.size(); + } + public com.google.protobuf.ByteString getColumns(int index) { + return columns_.get(index); + } + + // optional int32 batch = 4; + public static final int BATCH_FIELD_NUMBER = 4; + private int batch_; + public boolean hasBatch() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public int getBatch() { + return batch_; + } + + // optional int64 startTime = 5; + public static final int STARTTIME_FIELD_NUMBER = 5; + private long startTime_; + public boolean hasStartTime() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public long getStartTime() { + return startTime_; + } + + // optional int64 endTime = 6; + public static final int ENDTIME_FIELD_NUMBER = 6; + private long endTime_; + public boolean hasEndTime() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public long getEndTime() { + return endTime_; + } + + // optional int32 maxVersions = 7; + public static final int MAXVERSIONS_FIELD_NUMBER = 7; + private int maxVersions_; + public boolean hasMaxVersions() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + public int getMaxVersions() { + return maxVersions_; + } + + // optional string filter = 8; + public static final int FILTER_FIELD_NUMBER = 8; + private java.lang.Object filter_; + public boolean hasFilter() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + public String getFilter() { + java.lang.Object ref = filter_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + filter_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getFilterBytes() { + java.lang.Object ref = filter_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + filter_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + startRow_ = com.google.protobuf.ByteString.EMPTY; + endRow_ = com.google.protobuf.ByteString.EMPTY; + columns_ = java.util.Collections.emptyList();; + batch_ = 0; + startTime_ = 0L; + endTime_ = 0L; + maxVersions_ = 0; + filter_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, startRow_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, endRow_); + } + for (int i = 0; i < columns_.size(); i++) { + output.writeBytes(3, columns_.get(i)); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt32(4, batch_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt64(5, startTime_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeInt64(6, endTime_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + output.writeInt32(7, maxVersions_); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeBytes(8, getFilterBytes()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, startRow_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(2, endRow_); + } + { + int dataSize = 0; + for (int i = 0; i < columns_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeBytesSizeNoTag(columns_.get(i)); + } + size += dataSize; + size += 1 * getColumnsList().size(); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(4, batch_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(5, startTime_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(6, endTime_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(7, maxVersions_); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(8, getFilterBytes()); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.ScannerOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Scanner_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Scanner_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + startRow_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + endRow_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + columns_ = java.util.Collections.emptyList();; + bitField0_ = (bitField0_ & ~0x00000004); + batch_ = 0; + bitField0_ = (bitField0_ & ~0x00000008); + startTime_ = 0L; + bitField0_ = (bitField0_ & ~0x00000010); + endTime_ = 0L; + bitField0_ = (bitField0_ & ~0x00000020); + maxVersions_ = 0; + bitField0_ = (bitField0_ & ~0x00000040); + filter_ = ""; + bitField0_ = (bitField0_ & ~0x00000080); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner build() { + org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner result = new org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.startRow_ = startRow_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.endRow_ = endRow_; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + columns_ = java.util.Collections.unmodifiableList(columns_); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.columns_ = columns_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000004; + } + result.batch_ = batch_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000008; + } + result.startTime_ = startTime_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000010; + } + result.endTime_ = endTime_; + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000020; + } + result.maxVersions_ = maxVersions_; + if (((from_bitField0_ & 0x00000080) == 0x00000080)) { + to_bitField0_ |= 0x00000040; + } + result.filter_ = filter_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner.getDefaultInstance()) return this; + if (other.hasStartRow()) { + setStartRow(other.getStartRow()); + } + if (other.hasEndRow()) { + setEndRow(other.getEndRow()); + } + if (!other.columns_.isEmpty()) { + if (columns_.isEmpty()) { + columns_ = other.columns_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureColumnsIsMutable(); + columns_.addAll(other.columns_); + } + onChanged(); + } + if (other.hasBatch()) { + setBatch(other.getBatch()); + } + if (other.hasStartTime()) { + setStartTime(other.getStartTime()); + } + if (other.hasEndTime()) { + setEndTime(other.getEndTime()); + } + if (other.hasMaxVersions()) { + setMaxVersions(other.getMaxVersions()); + } + if (other.hasFilter()) { + setFilter(other.getFilter()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + startRow_ = input.readBytes(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + endRow_ = input.readBytes(); + break; + } + case 26: { + ensureColumnsIsMutable(); + columns_.add(input.readBytes()); + break; + } + case 32: { + bitField0_ |= 0x00000008; + batch_ = input.readInt32(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + startTime_ = input.readInt64(); + break; + } + case 48: { + bitField0_ |= 0x00000020; + endTime_ = input.readInt64(); + break; + } + case 56: { + bitField0_ |= 0x00000040; + maxVersions_ = input.readInt32(); + break; + } + case 66: { + bitField0_ |= 0x00000080; + filter_ = input.readBytes(); + break; + } + } + } + } + + private int bitField0_; + + // optional bytes startRow = 1; + private com.google.protobuf.ByteString startRow_ = com.google.protobuf.ByteString.EMPTY; + public boolean hasStartRow() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public com.google.protobuf.ByteString getStartRow() { + return startRow_; + } + public Builder setStartRow(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + startRow_ = value; + onChanged(); + return this; + } + public Builder clearStartRow() { + bitField0_ = (bitField0_ & ~0x00000001); + startRow_ = getDefaultInstance().getStartRow(); + onChanged(); + return this; + } + + // optional bytes endRow = 2; + private com.google.protobuf.ByteString endRow_ = com.google.protobuf.ByteString.EMPTY; + public boolean hasEndRow() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public com.google.protobuf.ByteString getEndRow() { + return endRow_; + } + public Builder setEndRow(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + endRow_ = value; + onChanged(); + return this; + } + public Builder clearEndRow() { + bitField0_ = (bitField0_ & ~0x00000002); + endRow_ = getDefaultInstance().getEndRow(); + onChanged(); + return this; + } + + // repeated bytes columns = 3; + private java.util.List columns_ = java.util.Collections.emptyList();; + private void ensureColumnsIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + columns_ = new java.util.ArrayList(columns_); + bitField0_ |= 0x00000004; + } + } + public java.util.List + getColumnsList() { + return java.util.Collections.unmodifiableList(columns_); + } + public int getColumnsCount() { + return columns_.size(); + } + public com.google.protobuf.ByteString getColumns(int index) { + return columns_.get(index); + } + public Builder setColumns( + int index, com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureColumnsIsMutable(); + columns_.set(index, value); + onChanged(); + return this; + } + public Builder addColumns(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureColumnsIsMutable(); + columns_.add(value); + onChanged(); + return this; + } + public Builder addAllColumns( + java.lang.Iterable values) { + ensureColumnsIsMutable(); + super.addAll(values, columns_); + onChanged(); + return this; + } + public Builder clearColumns() { + columns_ = java.util.Collections.emptyList();; + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + return this; + } + + // optional int32 batch = 4; + private int batch_ ; + public boolean hasBatch() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getBatch() { + return batch_; + } + public Builder setBatch(int value) { + bitField0_ |= 0x00000008; + batch_ = value; + onChanged(); + return this; + } + public Builder clearBatch() { + bitField0_ = (bitField0_ & ~0x00000008); + batch_ = 0; + onChanged(); + return this; + } + + // optional int64 startTime = 5; + private long startTime_ ; + public boolean hasStartTime() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public long getStartTime() { + return startTime_; + } + public Builder setStartTime(long value) { + bitField0_ |= 0x00000010; + startTime_ = value; + onChanged(); + return this; + } + public Builder clearStartTime() { + bitField0_ = (bitField0_ & ~0x00000010); + startTime_ = 0L; + onChanged(); + return this; + } + + // optional int64 endTime = 6; + private long endTime_ ; + public boolean hasEndTime() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + public long getEndTime() { + return endTime_; + } + public Builder setEndTime(long value) { + bitField0_ |= 0x00000020; + endTime_ = value; + onChanged(); + return this; + } + public Builder clearEndTime() { + bitField0_ = (bitField0_ & ~0x00000020); + endTime_ = 0L; + onChanged(); + return this; + } + + // optional int32 maxVersions = 7; + private int maxVersions_ ; + public boolean hasMaxVersions() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + public int getMaxVersions() { + return maxVersions_; + } + public Builder setMaxVersions(int value) { + bitField0_ |= 0x00000040; + maxVersions_ = value; + onChanged(); + return this; + } + public Builder clearMaxVersions() { + bitField0_ = (bitField0_ & ~0x00000040); + maxVersions_ = 0; + onChanged(); + return this; + } + + // optional string filter = 8; + private java.lang.Object filter_ = ""; + public boolean hasFilter() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + public String getFilter() { + java.lang.Object ref = filter_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + filter_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setFilter(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000080; + filter_ = value; + onChanged(); + return this; + } + public Builder clearFilter() { + bitField0_ = (bitField0_ & ~0x00000080); + filter_ = getDefaultInstance().getFilter(); + onChanged(); + return this; + } + void setFilter(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000080; + filter_ = value; + onChanged(); + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.Scanner) + } + + static { + defaultInstance = new Scanner(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.Scanner) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Scanner_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Scanner_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\024ScannerMessage.proto\022/org.apache.hadoo" + + "p.hbase.rest.protobuf.generated\"\224\001\n\007Scan" + + "ner\022\020\n\010startRow\030\001 \001(\014\022\016\n\006endRow\030\002 \001(\014\022\017\n" + + "\007columns\030\003 \003(\014\022\r\n\005batch\030\004 \001(\005\022\021\n\tstartTi" + + "me\030\005 \001(\003\022\017\n\007endTime\030\006 \001(\003\022\023\n\013maxVersions" + + "\030\007 \001(\005\022\016\n\006filter\030\010 \001(\t" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Scanner_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Scanner_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Scanner_descriptor, + new java.lang.String[] { "StartRow", "EndRow", "Columns", "Batch", "StartTime", "EndTime", "MaxVersions", "Filter", }, + org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner.class, + org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner.Builder.class); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/StorageClusterStatusMessage.java b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/StorageClusterStatusMessage.java new file mode 100644 index 0000000..a6023b9 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/StorageClusterStatusMessage.java @@ -0,0 +1,2873 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: StorageClusterStatusMessage.proto + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +public final class StorageClusterStatusMessage { + private StorageClusterStatusMessage() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public interface StorageClusterStatusOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus.Node liveNodes = 1; + java.util.List + getLiveNodesList(); + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node getLiveNodes(int index); + int getLiveNodesCount(); + java.util.List + getLiveNodesOrBuilderList(); + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.NodeOrBuilder getLiveNodesOrBuilder( + int index); + + // repeated string deadNodes = 2; + java.util.List getDeadNodesList(); + int getDeadNodesCount(); + String getDeadNodes(int index); + + // optional int32 regions = 3; + boolean hasRegions(); + int getRegions(); + + // optional int32 requests = 4; + boolean hasRequests(); + int getRequests(); + + // optional double averageLoad = 5; + boolean hasAverageLoad(); + double getAverageLoad(); + } + public static final class StorageClusterStatus extends + com.google.protobuf.GeneratedMessage + implements StorageClusterStatusOrBuilder { + // Use StorageClusterStatus.newBuilder() to construct. + private StorageClusterStatus(Builder builder) { + super(builder); + } + private StorageClusterStatus(boolean noInit) {} + + private static final StorageClusterStatus defaultInstance; + public static StorageClusterStatus getDefaultInstance() { + return defaultInstance; + } + + public StorageClusterStatus getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_fieldAccessorTable; + } + + public interface RegionOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required bytes name = 1; + boolean hasName(); + com.google.protobuf.ByteString getName(); + + // optional int32 stores = 2; + boolean hasStores(); + int getStores(); + + // optional int32 storefiles = 3; + boolean hasStorefiles(); + int getStorefiles(); + + // optional int32 storefileSizeMB = 4; + boolean hasStorefileSizeMB(); + int getStorefileSizeMB(); + + // optional int32 memstoreSizeMB = 5; + boolean hasMemstoreSizeMB(); + int getMemstoreSizeMB(); + + // optional int32 storefileIndexSizeMB = 6; + boolean hasStorefileIndexSizeMB(); + int getStorefileIndexSizeMB(); + + // optional int64 readRequestsCount = 7; + boolean hasReadRequestsCount(); + long getReadRequestsCount(); + + // optional int64 writeRequestsCount = 8; + boolean hasWriteRequestsCount(); + long getWriteRequestsCount(); + + // optional int32 rootIndexSizeKB = 9; + boolean hasRootIndexSizeKB(); + int getRootIndexSizeKB(); + + // optional int32 totalStaticIndexSizeKB = 10; + boolean hasTotalStaticIndexSizeKB(); + int getTotalStaticIndexSizeKB(); + + // optional int32 totalStaticBloomSizeKB = 11; + boolean hasTotalStaticBloomSizeKB(); + int getTotalStaticBloomSizeKB(); + + // optional int64 totalCompactingKVs = 12; + boolean hasTotalCompactingKVs(); + long getTotalCompactingKVs(); + + // optional int64 currentCompactedKVs = 13; + boolean hasCurrentCompactedKVs(); + long getCurrentCompactedKVs(); + } + public static final class Region extends + com.google.protobuf.GeneratedMessage + implements RegionOrBuilder { + // Use Region.newBuilder() to construct. + private Region(Builder builder) { + super(builder); + } + private Region(boolean noInit) {} + + private static final Region defaultInstance; + public static Region getDefaultInstance() { + return defaultInstance; + } + + public Region getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Region_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Region_fieldAccessorTable; + } + + private int bitField0_; + // required bytes name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private com.google.protobuf.ByteString name_; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public com.google.protobuf.ByteString getName() { + return name_; + } + + // optional int32 stores = 2; + public static final int STORES_FIELD_NUMBER = 2; + private int stores_; + public boolean hasStores() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public int getStores() { + return stores_; + } + + // optional int32 storefiles = 3; + public static final int STOREFILES_FIELD_NUMBER = 3; + private int storefiles_; + public boolean hasStorefiles() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public int getStorefiles() { + return storefiles_; + } + + // optional int32 storefileSizeMB = 4; + public static final int STOREFILESIZEMB_FIELD_NUMBER = 4; + private int storefileSizeMB_; + public boolean hasStorefileSizeMB() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getStorefileSizeMB() { + return storefileSizeMB_; + } + + // optional int32 memstoreSizeMB = 5; + public static final int MEMSTORESIZEMB_FIELD_NUMBER = 5; + private int memstoreSizeMB_; + public boolean hasMemstoreSizeMB() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public int getMemstoreSizeMB() { + return memstoreSizeMB_; + } + + // optional int32 storefileIndexSizeMB = 6; + public static final int STOREFILEINDEXSIZEMB_FIELD_NUMBER = 6; + private int storefileIndexSizeMB_; + public boolean hasStorefileIndexSizeMB() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + public int getStorefileIndexSizeMB() { + return storefileIndexSizeMB_; + } + + // optional int64 readRequestsCount = 7; + public static final int READREQUESTSCOUNT_FIELD_NUMBER = 7; + private long readRequestsCount_; + public boolean hasReadRequestsCount() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + public long getReadRequestsCount() { + return readRequestsCount_; + } + + // optional int64 writeRequestsCount = 8; + public static final int WRITEREQUESTSCOUNT_FIELD_NUMBER = 8; + private long writeRequestsCount_; + public boolean hasWriteRequestsCount() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + public long getWriteRequestsCount() { + return writeRequestsCount_; + } + + // optional int32 rootIndexSizeKB = 9; + public static final int ROOTINDEXSIZEKB_FIELD_NUMBER = 9; + private int rootIndexSizeKB_; + public boolean hasRootIndexSizeKB() { + return ((bitField0_ & 0x00000100) == 0x00000100); + } + public int getRootIndexSizeKB() { + return rootIndexSizeKB_; + } + + // optional int32 totalStaticIndexSizeKB = 10; + public static final int TOTALSTATICINDEXSIZEKB_FIELD_NUMBER = 10; + private int totalStaticIndexSizeKB_; + public boolean hasTotalStaticIndexSizeKB() { + return ((bitField0_ & 0x00000200) == 0x00000200); + } + public int getTotalStaticIndexSizeKB() { + return totalStaticIndexSizeKB_; + } + + // optional int32 totalStaticBloomSizeKB = 11; + public static final int TOTALSTATICBLOOMSIZEKB_FIELD_NUMBER = 11; + private int totalStaticBloomSizeKB_; + public boolean hasTotalStaticBloomSizeKB() { + return ((bitField0_ & 0x00000400) == 0x00000400); + } + public int getTotalStaticBloomSizeKB() { + return totalStaticBloomSizeKB_; + } + + // optional int64 totalCompactingKVs = 12; + public static final int TOTALCOMPACTINGKVS_FIELD_NUMBER = 12; + private long totalCompactingKVs_; + public boolean hasTotalCompactingKVs() { + return ((bitField0_ & 0x00000800) == 0x00000800); + } + public long getTotalCompactingKVs() { + return totalCompactingKVs_; + } + + // optional int64 currentCompactedKVs = 13; + public static final int CURRENTCOMPACTEDKVS_FIELD_NUMBER = 13; + private long currentCompactedKVs_; + public boolean hasCurrentCompactedKVs() { + return ((bitField0_ & 0x00001000) == 0x00001000); + } + public long getCurrentCompactedKVs() { + return currentCompactedKVs_; + } + + private void initFields() { + name_ = com.google.protobuf.ByteString.EMPTY; + stores_ = 0; + storefiles_ = 0; + storefileSizeMB_ = 0; + memstoreSizeMB_ = 0; + storefileIndexSizeMB_ = 0; + readRequestsCount_ = 0L; + writeRequestsCount_ = 0L; + rootIndexSizeKB_ = 0; + totalStaticIndexSizeKB_ = 0; + totalStaticBloomSizeKB_ = 0; + totalCompactingKVs_ = 0L; + currentCompactedKVs_ = 0L; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasName()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, name_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt32(2, stores_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt32(3, storefiles_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt32(4, storefileSizeMB_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeInt32(5, memstoreSizeMB_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + output.writeInt32(6, storefileIndexSizeMB_); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeInt64(7, readRequestsCount_); + } + if (((bitField0_ & 0x00000080) == 0x00000080)) { + output.writeInt64(8, writeRequestsCount_); + } + if (((bitField0_ & 0x00000100) == 0x00000100)) { + output.writeInt32(9, rootIndexSizeKB_); + } + if (((bitField0_ & 0x00000200) == 0x00000200)) { + output.writeInt32(10, totalStaticIndexSizeKB_); + } + if (((bitField0_ & 0x00000400) == 0x00000400)) { + output.writeInt32(11, totalStaticBloomSizeKB_); + } + if (((bitField0_ & 0x00000800) == 0x00000800)) { + output.writeInt64(12, totalCompactingKVs_); + } + if (((bitField0_ & 0x00001000) == 0x00001000)) { + output.writeInt64(13, currentCompactedKVs_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, name_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(2, stores_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(3, storefiles_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(4, storefileSizeMB_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(5, memstoreSizeMB_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(6, storefileIndexSizeMB_); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(7, readRequestsCount_); + } + if (((bitField0_ & 0x00000080) == 0x00000080)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(8, writeRequestsCount_); + } + if (((bitField0_ & 0x00000100) == 0x00000100)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(9, rootIndexSizeKB_); + } + if (((bitField0_ & 0x00000200) == 0x00000200)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(10, totalStaticIndexSizeKB_); + } + if (((bitField0_ & 0x00000400) == 0x00000400)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(11, totalStaticBloomSizeKB_); + } + if (((bitField0_ & 0x00000800) == 0x00000800)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(12, totalCompactingKVs_); + } + if (((bitField0_ & 0x00001000) == 0x00001000)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(13, currentCompactedKVs_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.RegionOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Region_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Region_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + stores_ = 0; + bitField0_ = (bitField0_ & ~0x00000002); + storefiles_ = 0; + bitField0_ = (bitField0_ & ~0x00000004); + storefileSizeMB_ = 0; + bitField0_ = (bitField0_ & ~0x00000008); + memstoreSizeMB_ = 0; + bitField0_ = (bitField0_ & ~0x00000010); + storefileIndexSizeMB_ = 0; + bitField0_ = (bitField0_ & ~0x00000020); + readRequestsCount_ = 0L; + bitField0_ = (bitField0_ & ~0x00000040); + writeRequestsCount_ = 0L; + bitField0_ = (bitField0_ & ~0x00000080); + rootIndexSizeKB_ = 0; + bitField0_ = (bitField0_ & ~0x00000100); + totalStaticIndexSizeKB_ = 0; + bitField0_ = (bitField0_ & ~0x00000200); + totalStaticBloomSizeKB_ = 0; + bitField0_ = (bitField0_ & ~0x00000400); + totalCompactingKVs_ = 0L; + bitField0_ = (bitField0_ & ~0x00000800); + currentCompactedKVs_ = 0L; + bitField0_ = (bitField0_ & ~0x00001000); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region build() { + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region result = new org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.stores_ = stores_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.storefiles_ = storefiles_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.storefileSizeMB_ = storefileSizeMB_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.memstoreSizeMB_ = memstoreSizeMB_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000020; + } + result.storefileIndexSizeMB_ = storefileIndexSizeMB_; + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000040; + } + result.readRequestsCount_ = readRequestsCount_; + if (((from_bitField0_ & 0x00000080) == 0x00000080)) { + to_bitField0_ |= 0x00000080; + } + result.writeRequestsCount_ = writeRequestsCount_; + if (((from_bitField0_ & 0x00000100) == 0x00000100)) { + to_bitField0_ |= 0x00000100; + } + result.rootIndexSizeKB_ = rootIndexSizeKB_; + if (((from_bitField0_ & 0x00000200) == 0x00000200)) { + to_bitField0_ |= 0x00000200; + } + result.totalStaticIndexSizeKB_ = totalStaticIndexSizeKB_; + if (((from_bitField0_ & 0x00000400) == 0x00000400)) { + to_bitField0_ |= 0x00000400; + } + result.totalStaticBloomSizeKB_ = totalStaticBloomSizeKB_; + if (((from_bitField0_ & 0x00000800) == 0x00000800)) { + to_bitField0_ |= 0x00000800; + } + result.totalCompactingKVs_ = totalCompactingKVs_; + if (((from_bitField0_ & 0x00001000) == 0x00001000)) { + to_bitField0_ |= 0x00001000; + } + result.currentCompactedKVs_ = currentCompactedKVs_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.getDefaultInstance()) return this; + if (other.hasName()) { + setName(other.getName()); + } + if (other.hasStores()) { + setStores(other.getStores()); + } + if (other.hasStorefiles()) { + setStorefiles(other.getStorefiles()); + } + if (other.hasStorefileSizeMB()) { + setStorefileSizeMB(other.getStorefileSizeMB()); + } + if (other.hasMemstoreSizeMB()) { + setMemstoreSizeMB(other.getMemstoreSizeMB()); + } + if (other.hasStorefileIndexSizeMB()) { + setStorefileIndexSizeMB(other.getStorefileIndexSizeMB()); + } + if (other.hasReadRequestsCount()) { + setReadRequestsCount(other.getReadRequestsCount()); + } + if (other.hasWriteRequestsCount()) { + setWriteRequestsCount(other.getWriteRequestsCount()); + } + if (other.hasRootIndexSizeKB()) { + setRootIndexSizeKB(other.getRootIndexSizeKB()); + } + if (other.hasTotalStaticIndexSizeKB()) { + setTotalStaticIndexSizeKB(other.getTotalStaticIndexSizeKB()); + } + if (other.hasTotalStaticBloomSizeKB()) { + setTotalStaticBloomSizeKB(other.getTotalStaticBloomSizeKB()); + } + if (other.hasTotalCompactingKVs()) { + setTotalCompactingKVs(other.getTotalCompactingKVs()); + } + if (other.hasCurrentCompactedKVs()) { + setCurrentCompactedKVs(other.getCurrentCompactedKVs()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasName()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + name_ = input.readBytes(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + stores_ = input.readInt32(); + break; + } + case 24: { + bitField0_ |= 0x00000004; + storefiles_ = input.readInt32(); + break; + } + case 32: { + bitField0_ |= 0x00000008; + storefileSizeMB_ = input.readInt32(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + memstoreSizeMB_ = input.readInt32(); + break; + } + case 48: { + bitField0_ |= 0x00000020; + storefileIndexSizeMB_ = input.readInt32(); + break; + } + case 56: { + bitField0_ |= 0x00000040; + readRequestsCount_ = input.readInt64(); + break; + } + case 64: { + bitField0_ |= 0x00000080; + writeRequestsCount_ = input.readInt64(); + break; + } + case 72: { + bitField0_ |= 0x00000100; + rootIndexSizeKB_ = input.readInt32(); + break; + } + case 80: { + bitField0_ |= 0x00000200; + totalStaticIndexSizeKB_ = input.readInt32(); + break; + } + case 88: { + bitField0_ |= 0x00000400; + totalStaticBloomSizeKB_ = input.readInt32(); + break; + } + case 96: { + bitField0_ |= 0x00000800; + totalCompactingKVs_ = input.readInt64(); + break; + } + case 104: { + bitField0_ |= 0x00001000; + currentCompactedKVs_ = input.readInt64(); + break; + } + } + } + } + + private int bitField0_; + + // required bytes name = 1; + private com.google.protobuf.ByteString name_ = com.google.protobuf.ByteString.EMPTY; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public com.google.protobuf.ByteString getName() { + return name_; + } + public Builder setName(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + + // optional int32 stores = 2; + private int stores_ ; + public boolean hasStores() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public int getStores() { + return stores_; + } + public Builder setStores(int value) { + bitField0_ |= 0x00000002; + stores_ = value; + onChanged(); + return this; + } + public Builder clearStores() { + bitField0_ = (bitField0_ & ~0x00000002); + stores_ = 0; + onChanged(); + return this; + } + + // optional int32 storefiles = 3; + private int storefiles_ ; + public boolean hasStorefiles() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public int getStorefiles() { + return storefiles_; + } + public Builder setStorefiles(int value) { + bitField0_ |= 0x00000004; + storefiles_ = value; + onChanged(); + return this; + } + public Builder clearStorefiles() { + bitField0_ = (bitField0_ & ~0x00000004); + storefiles_ = 0; + onChanged(); + return this; + } + + // optional int32 storefileSizeMB = 4; + private int storefileSizeMB_ ; + public boolean hasStorefileSizeMB() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getStorefileSizeMB() { + return storefileSizeMB_; + } + public Builder setStorefileSizeMB(int value) { + bitField0_ |= 0x00000008; + storefileSizeMB_ = value; + onChanged(); + return this; + } + public Builder clearStorefileSizeMB() { + bitField0_ = (bitField0_ & ~0x00000008); + storefileSizeMB_ = 0; + onChanged(); + return this; + } + + // optional int32 memstoreSizeMB = 5; + private int memstoreSizeMB_ ; + public boolean hasMemstoreSizeMB() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public int getMemstoreSizeMB() { + return memstoreSizeMB_; + } + public Builder setMemstoreSizeMB(int value) { + bitField0_ |= 0x00000010; + memstoreSizeMB_ = value; + onChanged(); + return this; + } + public Builder clearMemstoreSizeMB() { + bitField0_ = (bitField0_ & ~0x00000010); + memstoreSizeMB_ = 0; + onChanged(); + return this; + } + + // optional int32 storefileIndexSizeMB = 6; + private int storefileIndexSizeMB_ ; + public boolean hasStorefileIndexSizeMB() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + public int getStorefileIndexSizeMB() { + return storefileIndexSizeMB_; + } + public Builder setStorefileIndexSizeMB(int value) { + bitField0_ |= 0x00000020; + storefileIndexSizeMB_ = value; + onChanged(); + return this; + } + public Builder clearStorefileIndexSizeMB() { + bitField0_ = (bitField0_ & ~0x00000020); + storefileIndexSizeMB_ = 0; + onChanged(); + return this; + } + + // optional int64 readRequestsCount = 7; + private long readRequestsCount_ ; + public boolean hasReadRequestsCount() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + public long getReadRequestsCount() { + return readRequestsCount_; + } + public Builder setReadRequestsCount(long value) { + bitField0_ |= 0x00000040; + readRequestsCount_ = value; + onChanged(); + return this; + } + public Builder clearReadRequestsCount() { + bitField0_ = (bitField0_ & ~0x00000040); + readRequestsCount_ = 0L; + onChanged(); + return this; + } + + // optional int64 writeRequestsCount = 8; + private long writeRequestsCount_ ; + public boolean hasWriteRequestsCount() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + public long getWriteRequestsCount() { + return writeRequestsCount_; + } + public Builder setWriteRequestsCount(long value) { + bitField0_ |= 0x00000080; + writeRequestsCount_ = value; + onChanged(); + return this; + } + public Builder clearWriteRequestsCount() { + bitField0_ = (bitField0_ & ~0x00000080); + writeRequestsCount_ = 0L; + onChanged(); + return this; + } + + // optional int32 rootIndexSizeKB = 9; + private int rootIndexSizeKB_ ; + public boolean hasRootIndexSizeKB() { + return ((bitField0_ & 0x00000100) == 0x00000100); + } + public int getRootIndexSizeKB() { + return rootIndexSizeKB_; + } + public Builder setRootIndexSizeKB(int value) { + bitField0_ |= 0x00000100; + rootIndexSizeKB_ = value; + onChanged(); + return this; + } + public Builder clearRootIndexSizeKB() { + bitField0_ = (bitField0_ & ~0x00000100); + rootIndexSizeKB_ = 0; + onChanged(); + return this; + } + + // optional int32 totalStaticIndexSizeKB = 10; + private int totalStaticIndexSizeKB_ ; + public boolean hasTotalStaticIndexSizeKB() { + return ((bitField0_ & 0x00000200) == 0x00000200); + } + public int getTotalStaticIndexSizeKB() { + return totalStaticIndexSizeKB_; + } + public Builder setTotalStaticIndexSizeKB(int value) { + bitField0_ |= 0x00000200; + totalStaticIndexSizeKB_ = value; + onChanged(); + return this; + } + public Builder clearTotalStaticIndexSizeKB() { + bitField0_ = (bitField0_ & ~0x00000200); + totalStaticIndexSizeKB_ = 0; + onChanged(); + return this; + } + + // optional int32 totalStaticBloomSizeKB = 11; + private int totalStaticBloomSizeKB_ ; + public boolean hasTotalStaticBloomSizeKB() { + return ((bitField0_ & 0x00000400) == 0x00000400); + } + public int getTotalStaticBloomSizeKB() { + return totalStaticBloomSizeKB_; + } + public Builder setTotalStaticBloomSizeKB(int value) { + bitField0_ |= 0x00000400; + totalStaticBloomSizeKB_ = value; + onChanged(); + return this; + } + public Builder clearTotalStaticBloomSizeKB() { + bitField0_ = (bitField0_ & ~0x00000400); + totalStaticBloomSizeKB_ = 0; + onChanged(); + return this; + } + + // optional int64 totalCompactingKVs = 12; + private long totalCompactingKVs_ ; + public boolean hasTotalCompactingKVs() { + return ((bitField0_ & 0x00000800) == 0x00000800); + } + public long getTotalCompactingKVs() { + return totalCompactingKVs_; + } + public Builder setTotalCompactingKVs(long value) { + bitField0_ |= 0x00000800; + totalCompactingKVs_ = value; + onChanged(); + return this; + } + public Builder clearTotalCompactingKVs() { + bitField0_ = (bitField0_ & ~0x00000800); + totalCompactingKVs_ = 0L; + onChanged(); + return this; + } + + // optional int64 currentCompactedKVs = 13; + private long currentCompactedKVs_ ; + public boolean hasCurrentCompactedKVs() { + return ((bitField0_ & 0x00001000) == 0x00001000); + } + public long getCurrentCompactedKVs() { + return currentCompactedKVs_; + } + public Builder setCurrentCompactedKVs(long value) { + bitField0_ |= 0x00001000; + currentCompactedKVs_ = value; + onChanged(); + return this; + } + public Builder clearCurrentCompactedKVs() { + bitField0_ = (bitField0_ & ~0x00001000); + currentCompactedKVs_ = 0L; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus.Region) + } + + static { + defaultInstance = new Region(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus.Region) + } + + public interface NodeOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required string name = 1; + boolean hasName(); + String getName(); + + // optional int64 startCode = 2; + boolean hasStartCode(); + long getStartCode(); + + // optional int32 requests = 3; + boolean hasRequests(); + int getRequests(); + + // optional int32 heapSizeMB = 4; + boolean hasHeapSizeMB(); + int getHeapSizeMB(); + + // optional int32 maxHeapSizeMB = 5; + boolean hasMaxHeapSizeMB(); + int getMaxHeapSizeMB(); + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus.Region regions = 6; + java.util.List + getRegionsList(); + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region getRegions(int index); + int getRegionsCount(); + java.util.List + getRegionsOrBuilderList(); + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.RegionOrBuilder getRegionsOrBuilder( + int index); + } + public static final class Node extends + com.google.protobuf.GeneratedMessage + implements NodeOrBuilder { + // Use Node.newBuilder() to construct. + private Node(Builder builder) { + super(builder); + } + private Node(boolean noInit) {} + + private static final Node defaultInstance; + public static Node getDefaultInstance() { + return defaultInstance; + } + + public Node getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Node_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Node_fieldAccessorTable; + } + + private int bitField0_; + // required string name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private java.lang.Object name_; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + name_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional int64 startCode = 2; + public static final int STARTCODE_FIELD_NUMBER = 2; + private long startCode_; + public boolean hasStartCode() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getStartCode() { + return startCode_; + } + + // optional int32 requests = 3; + public static final int REQUESTS_FIELD_NUMBER = 3; + private int requests_; + public boolean hasRequests() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public int getRequests() { + return requests_; + } + + // optional int32 heapSizeMB = 4; + public static final int HEAPSIZEMB_FIELD_NUMBER = 4; + private int heapSizeMB_; + public boolean hasHeapSizeMB() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getHeapSizeMB() { + return heapSizeMB_; + } + + // optional int32 maxHeapSizeMB = 5; + public static final int MAXHEAPSIZEMB_FIELD_NUMBER = 5; + private int maxHeapSizeMB_; + public boolean hasMaxHeapSizeMB() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public int getMaxHeapSizeMB() { + return maxHeapSizeMB_; + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus.Region regions = 6; + public static final int REGIONS_FIELD_NUMBER = 6; + private java.util.List regions_; + public java.util.List getRegionsList() { + return regions_; + } + public java.util.List + getRegionsOrBuilderList() { + return regions_; + } + public int getRegionsCount() { + return regions_.size(); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region getRegions(int index) { + return regions_.get(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.RegionOrBuilder getRegionsOrBuilder( + int index) { + return regions_.get(index); + } + + private void initFields() { + name_ = ""; + startCode_ = 0L; + requests_ = 0; + heapSizeMB_ = 0; + maxHeapSizeMB_ = 0; + regions_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasName()) { + memoizedIsInitialized = 0; + return false; + } + for (int i = 0; i < getRegionsCount(); i++) { + if (!getRegions(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt64(2, startCode_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt32(3, requests_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt32(4, heapSizeMB_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeInt32(5, maxHeapSizeMB_); + } + for (int i = 0; i < regions_.size(); i++) { + output.writeMessage(6, regions_.get(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(2, startCode_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(3, requests_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(4, heapSizeMB_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(5, maxHeapSizeMB_); + } + for (int i = 0; i < regions_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(6, regions_.get(i)); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.NodeOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Node_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Node_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getRegionsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + startCode_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + requests_ = 0; + bitField0_ = (bitField0_ & ~0x00000004); + heapSizeMB_ = 0; + bitField0_ = (bitField0_ & ~0x00000008); + maxHeapSizeMB_ = 0; + bitField0_ = (bitField0_ & ~0x00000010); + if (regionsBuilder_ == null) { + regions_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000020); + } else { + regionsBuilder_.clear(); + } + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node build() { + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node result = new org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.startCode_ = startCode_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.requests_ = requests_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.heapSizeMB_ = heapSizeMB_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.maxHeapSizeMB_ = maxHeapSizeMB_; + if (regionsBuilder_ == null) { + if (((bitField0_ & 0x00000020) == 0x00000020)) { + regions_ = java.util.Collections.unmodifiableList(regions_); + bitField0_ = (bitField0_ & ~0x00000020); + } + result.regions_ = regions_; + } else { + result.regions_ = regionsBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.getDefaultInstance()) return this; + if (other.hasName()) { + setName(other.getName()); + } + if (other.hasStartCode()) { + setStartCode(other.getStartCode()); + } + if (other.hasRequests()) { + setRequests(other.getRequests()); + } + if (other.hasHeapSizeMB()) { + setHeapSizeMB(other.getHeapSizeMB()); + } + if (other.hasMaxHeapSizeMB()) { + setMaxHeapSizeMB(other.getMaxHeapSizeMB()); + } + if (regionsBuilder_ == null) { + if (!other.regions_.isEmpty()) { + if (regions_.isEmpty()) { + regions_ = other.regions_; + bitField0_ = (bitField0_ & ~0x00000020); + } else { + ensureRegionsIsMutable(); + regions_.addAll(other.regions_); + } + onChanged(); + } + } else { + if (!other.regions_.isEmpty()) { + if (regionsBuilder_.isEmpty()) { + regionsBuilder_.dispose(); + regionsBuilder_ = null; + regions_ = other.regions_; + bitField0_ = (bitField0_ & ~0x00000020); + regionsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getRegionsFieldBuilder() : null; + } else { + regionsBuilder_.addAllMessages(other.regions_); + } + } + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasName()) { + + return false; + } + for (int i = 0; i < getRegionsCount(); i++) { + if (!getRegions(i).isInitialized()) { + + return false; + } + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + name_ = input.readBytes(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + startCode_ = input.readInt64(); + break; + } + case 24: { + bitField0_ |= 0x00000004; + requests_ = input.readInt32(); + break; + } + case 32: { + bitField0_ |= 0x00000008; + heapSizeMB_ = input.readInt32(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + maxHeapSizeMB_ = input.readInt32(); + break; + } + case 50: { + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.Builder subBuilder = org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addRegions(subBuilder.buildPartial()); + break; + } + } + } + } + + private int bitField0_; + + // required string name = 1; + private java.lang.Object name_ = ""; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + name_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setName(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + void setName(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + } + + // optional int64 startCode = 2; + private long startCode_ ; + public boolean hasStartCode() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getStartCode() { + return startCode_; + } + public Builder setStartCode(long value) { + bitField0_ |= 0x00000002; + startCode_ = value; + onChanged(); + return this; + } + public Builder clearStartCode() { + bitField0_ = (bitField0_ & ~0x00000002); + startCode_ = 0L; + onChanged(); + return this; + } + + // optional int32 requests = 3; + private int requests_ ; + public boolean hasRequests() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public int getRequests() { + return requests_; + } + public Builder setRequests(int value) { + bitField0_ |= 0x00000004; + requests_ = value; + onChanged(); + return this; + } + public Builder clearRequests() { + bitField0_ = (bitField0_ & ~0x00000004); + requests_ = 0; + onChanged(); + return this; + } + + // optional int32 heapSizeMB = 4; + private int heapSizeMB_ ; + public boolean hasHeapSizeMB() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getHeapSizeMB() { + return heapSizeMB_; + } + public Builder setHeapSizeMB(int value) { + bitField0_ |= 0x00000008; + heapSizeMB_ = value; + onChanged(); + return this; + } + public Builder clearHeapSizeMB() { + bitField0_ = (bitField0_ & ~0x00000008); + heapSizeMB_ = 0; + onChanged(); + return this; + } + + // optional int32 maxHeapSizeMB = 5; + private int maxHeapSizeMB_ ; + public boolean hasMaxHeapSizeMB() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public int getMaxHeapSizeMB() { + return maxHeapSizeMB_; + } + public Builder setMaxHeapSizeMB(int value) { + bitField0_ |= 0x00000010; + maxHeapSizeMB_ = value; + onChanged(); + return this; + } + public Builder clearMaxHeapSizeMB() { + bitField0_ = (bitField0_ & ~0x00000010); + maxHeapSizeMB_ = 0; + onChanged(); + return this; + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus.Region regions = 6; + private java.util.List regions_ = + java.util.Collections.emptyList(); + private void ensureRegionsIsMutable() { + if (!((bitField0_ & 0x00000020) == 0x00000020)) { + regions_ = new java.util.ArrayList(regions_); + bitField0_ |= 0x00000020; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.RegionOrBuilder> regionsBuilder_; + + public java.util.List getRegionsList() { + if (regionsBuilder_ == null) { + return java.util.Collections.unmodifiableList(regions_); + } else { + return regionsBuilder_.getMessageList(); + } + } + public int getRegionsCount() { + if (regionsBuilder_ == null) { + return regions_.size(); + } else { + return regionsBuilder_.getCount(); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region getRegions(int index) { + if (regionsBuilder_ == null) { + return regions_.get(index); + } else { + return regionsBuilder_.getMessage(index); + } + } + public Builder setRegions( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region value) { + if (regionsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRegionsIsMutable(); + regions_.set(index, value); + onChanged(); + } else { + regionsBuilder_.setMessage(index, value); + } + return this; + } + public Builder setRegions( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.Builder builderForValue) { + if (regionsBuilder_ == null) { + ensureRegionsIsMutable(); + regions_.set(index, builderForValue.build()); + onChanged(); + } else { + regionsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addRegions(org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region value) { + if (regionsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRegionsIsMutable(); + regions_.add(value); + onChanged(); + } else { + regionsBuilder_.addMessage(value); + } + return this; + } + public Builder addRegions( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region value) { + if (regionsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRegionsIsMutable(); + regions_.add(index, value); + onChanged(); + } else { + regionsBuilder_.addMessage(index, value); + } + return this; + } + public Builder addRegions( + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.Builder builderForValue) { + if (regionsBuilder_ == null) { + ensureRegionsIsMutable(); + regions_.add(builderForValue.build()); + onChanged(); + } else { + regionsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addRegions( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.Builder builderForValue) { + if (regionsBuilder_ == null) { + ensureRegionsIsMutable(); + regions_.add(index, builderForValue.build()); + onChanged(); + } else { + regionsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllRegions( + java.lang.Iterable values) { + if (regionsBuilder_ == null) { + ensureRegionsIsMutable(); + super.addAll(values, regions_); + onChanged(); + } else { + regionsBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearRegions() { + if (regionsBuilder_ == null) { + regions_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000020); + onChanged(); + } else { + regionsBuilder_.clear(); + } + return this; + } + public Builder removeRegions(int index) { + if (regionsBuilder_ == null) { + ensureRegionsIsMutable(); + regions_.remove(index); + onChanged(); + } else { + regionsBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.Builder getRegionsBuilder( + int index) { + return getRegionsFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.RegionOrBuilder getRegionsOrBuilder( + int index) { + if (regionsBuilder_ == null) { + return regions_.get(index); } else { + return regionsBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getRegionsOrBuilderList() { + if (regionsBuilder_ != null) { + return regionsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(regions_); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.Builder addRegionsBuilder() { + return getRegionsFieldBuilder().addBuilder( + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.getDefaultInstance()); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.Builder addRegionsBuilder( + int index) { + return getRegionsFieldBuilder().addBuilder( + index, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.getDefaultInstance()); + } + public java.util.List + getRegionsBuilderList() { + return getRegionsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.RegionOrBuilder> + getRegionsFieldBuilder() { + if (regionsBuilder_ == null) { + regionsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.RegionOrBuilder>( + regions_, + ((bitField0_ & 0x00000020) == 0x00000020), + getParentForChildren(), + isClean()); + regions_ = null; + } + return regionsBuilder_; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus.Node) + } + + static { + defaultInstance = new Node(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus.Node) + } + + private int bitField0_; + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus.Node liveNodes = 1; + public static final int LIVENODES_FIELD_NUMBER = 1; + private java.util.List liveNodes_; + public java.util.List getLiveNodesList() { + return liveNodes_; + } + public java.util.List + getLiveNodesOrBuilderList() { + return liveNodes_; + } + public int getLiveNodesCount() { + return liveNodes_.size(); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node getLiveNodes(int index) { + return liveNodes_.get(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.NodeOrBuilder getLiveNodesOrBuilder( + int index) { + return liveNodes_.get(index); + } + + // repeated string deadNodes = 2; + public static final int DEADNODES_FIELD_NUMBER = 2; + private com.google.protobuf.LazyStringList deadNodes_; + public java.util.List + getDeadNodesList() { + return deadNodes_; + } + public int getDeadNodesCount() { + return deadNodes_.size(); + } + public String getDeadNodes(int index) { + return deadNodes_.get(index); + } + + // optional int32 regions = 3; + public static final int REGIONS_FIELD_NUMBER = 3; + private int regions_; + public boolean hasRegions() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public int getRegions() { + return regions_; + } + + // optional int32 requests = 4; + public static final int REQUESTS_FIELD_NUMBER = 4; + private int requests_; + public boolean hasRequests() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public int getRequests() { + return requests_; + } + + // optional double averageLoad = 5; + public static final int AVERAGELOAD_FIELD_NUMBER = 5; + private double averageLoad_; + public boolean hasAverageLoad() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public double getAverageLoad() { + return averageLoad_; + } + + private void initFields() { + liveNodes_ = java.util.Collections.emptyList(); + deadNodes_ = com.google.protobuf.LazyStringArrayList.EMPTY; + regions_ = 0; + requests_ = 0; + averageLoad_ = 0D; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + for (int i = 0; i < getLiveNodesCount(); i++) { + if (!getLiveNodes(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + for (int i = 0; i < liveNodes_.size(); i++) { + output.writeMessage(1, liveNodes_.get(i)); + } + for (int i = 0; i < deadNodes_.size(); i++) { + output.writeBytes(2, deadNodes_.getByteString(i)); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt32(3, regions_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt32(4, requests_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeDouble(5, averageLoad_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < liveNodes_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, liveNodes_.get(i)); + } + { + int dataSize = 0; + for (int i = 0; i < deadNodes_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeBytesSizeNoTag(deadNodes_.getByteString(i)); + } + size += dataSize; + size += 1 * getDeadNodesList().size(); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(3, regions_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(4, requests_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeDoubleSize(5, averageLoad_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatusOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getLiveNodesFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (liveNodesBuilder_ == null) { + liveNodes_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + } else { + liveNodesBuilder_.clear(); + } + deadNodes_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + regions_ = 0; + bitField0_ = (bitField0_ & ~0x00000004); + requests_ = 0; + bitField0_ = (bitField0_ & ~0x00000008); + averageLoad_ = 0D; + bitField0_ = (bitField0_ & ~0x00000010); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus build() { + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus result = new org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (liveNodesBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + liveNodes_ = java.util.Collections.unmodifiableList(liveNodes_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.liveNodes_ = liveNodes_; + } else { + result.liveNodes_ = liveNodesBuilder_.build(); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + deadNodes_ = new com.google.protobuf.UnmodifiableLazyStringList( + deadNodes_); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.deadNodes_ = deadNodes_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000001; + } + result.regions_ = regions_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000002; + } + result.requests_ = requests_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000004; + } + result.averageLoad_ = averageLoad_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.getDefaultInstance()) return this; + if (liveNodesBuilder_ == null) { + if (!other.liveNodes_.isEmpty()) { + if (liveNodes_.isEmpty()) { + liveNodes_ = other.liveNodes_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureLiveNodesIsMutable(); + liveNodes_.addAll(other.liveNodes_); + } + onChanged(); + } + } else { + if (!other.liveNodes_.isEmpty()) { + if (liveNodesBuilder_.isEmpty()) { + liveNodesBuilder_.dispose(); + liveNodesBuilder_ = null; + liveNodes_ = other.liveNodes_; + bitField0_ = (bitField0_ & ~0x00000001); + liveNodesBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getLiveNodesFieldBuilder() : null; + } else { + liveNodesBuilder_.addAllMessages(other.liveNodes_); + } + } + } + if (!other.deadNodes_.isEmpty()) { + if (deadNodes_.isEmpty()) { + deadNodes_ = other.deadNodes_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureDeadNodesIsMutable(); + deadNodes_.addAll(other.deadNodes_); + } + onChanged(); + } + if (other.hasRegions()) { + setRegions(other.getRegions()); + } + if (other.hasRequests()) { + setRequests(other.getRequests()); + } + if (other.hasAverageLoad()) { + setAverageLoad(other.getAverageLoad()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + for (int i = 0; i < getLiveNodesCount(); i++) { + if (!getLiveNodes(i).isInitialized()) { + + return false; + } + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.Builder subBuilder = org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addLiveNodes(subBuilder.buildPartial()); + break; + } + case 18: { + ensureDeadNodesIsMutable(); + deadNodes_.add(input.readBytes()); + break; + } + case 24: { + bitField0_ |= 0x00000004; + regions_ = input.readInt32(); + break; + } + case 32: { + bitField0_ |= 0x00000008; + requests_ = input.readInt32(); + break; + } + case 41: { + bitField0_ |= 0x00000010; + averageLoad_ = input.readDouble(); + break; + } + } + } + } + + private int bitField0_; + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus.Node liveNodes = 1; + private java.util.List liveNodes_ = + java.util.Collections.emptyList(); + private void ensureLiveNodesIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + liveNodes_ = new java.util.ArrayList(liveNodes_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.NodeOrBuilder> liveNodesBuilder_; + + public java.util.List getLiveNodesList() { + if (liveNodesBuilder_ == null) { + return java.util.Collections.unmodifiableList(liveNodes_); + } else { + return liveNodesBuilder_.getMessageList(); + } + } + public int getLiveNodesCount() { + if (liveNodesBuilder_ == null) { + return liveNodes_.size(); + } else { + return liveNodesBuilder_.getCount(); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node getLiveNodes(int index) { + if (liveNodesBuilder_ == null) { + return liveNodes_.get(index); + } else { + return liveNodesBuilder_.getMessage(index); + } + } + public Builder setLiveNodes( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node value) { + if (liveNodesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureLiveNodesIsMutable(); + liveNodes_.set(index, value); + onChanged(); + } else { + liveNodesBuilder_.setMessage(index, value); + } + return this; + } + public Builder setLiveNodes( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.Builder builderForValue) { + if (liveNodesBuilder_ == null) { + ensureLiveNodesIsMutable(); + liveNodes_.set(index, builderForValue.build()); + onChanged(); + } else { + liveNodesBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addLiveNodes(org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node value) { + if (liveNodesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureLiveNodesIsMutable(); + liveNodes_.add(value); + onChanged(); + } else { + liveNodesBuilder_.addMessage(value); + } + return this; + } + public Builder addLiveNodes( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node value) { + if (liveNodesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureLiveNodesIsMutable(); + liveNodes_.add(index, value); + onChanged(); + } else { + liveNodesBuilder_.addMessage(index, value); + } + return this; + } + public Builder addLiveNodes( + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.Builder builderForValue) { + if (liveNodesBuilder_ == null) { + ensureLiveNodesIsMutable(); + liveNodes_.add(builderForValue.build()); + onChanged(); + } else { + liveNodesBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addLiveNodes( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.Builder builderForValue) { + if (liveNodesBuilder_ == null) { + ensureLiveNodesIsMutable(); + liveNodes_.add(index, builderForValue.build()); + onChanged(); + } else { + liveNodesBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllLiveNodes( + java.lang.Iterable values) { + if (liveNodesBuilder_ == null) { + ensureLiveNodesIsMutable(); + super.addAll(values, liveNodes_); + onChanged(); + } else { + liveNodesBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearLiveNodes() { + if (liveNodesBuilder_ == null) { + liveNodes_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + liveNodesBuilder_.clear(); + } + return this; + } + public Builder removeLiveNodes(int index) { + if (liveNodesBuilder_ == null) { + ensureLiveNodesIsMutable(); + liveNodes_.remove(index); + onChanged(); + } else { + liveNodesBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.Builder getLiveNodesBuilder( + int index) { + return getLiveNodesFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.NodeOrBuilder getLiveNodesOrBuilder( + int index) { + if (liveNodesBuilder_ == null) { + return liveNodes_.get(index); } else { + return liveNodesBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getLiveNodesOrBuilderList() { + if (liveNodesBuilder_ != null) { + return liveNodesBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(liveNodes_); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.Builder addLiveNodesBuilder() { + return getLiveNodesFieldBuilder().addBuilder( + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.getDefaultInstance()); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.Builder addLiveNodesBuilder( + int index) { + return getLiveNodesFieldBuilder().addBuilder( + index, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.getDefaultInstance()); + } + public java.util.List + getLiveNodesBuilderList() { + return getLiveNodesFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.NodeOrBuilder> + getLiveNodesFieldBuilder() { + if (liveNodesBuilder_ == null) { + liveNodesBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.NodeOrBuilder>( + liveNodes_, + ((bitField0_ & 0x00000001) == 0x00000001), + getParentForChildren(), + isClean()); + liveNodes_ = null; + } + return liveNodesBuilder_; + } + + // repeated string deadNodes = 2; + private com.google.protobuf.LazyStringList deadNodes_ = com.google.protobuf.LazyStringArrayList.EMPTY; + private void ensureDeadNodesIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + deadNodes_ = new com.google.protobuf.LazyStringArrayList(deadNodes_); + bitField0_ |= 0x00000002; + } + } + public java.util.List + getDeadNodesList() { + return java.util.Collections.unmodifiableList(deadNodes_); + } + public int getDeadNodesCount() { + return deadNodes_.size(); + } + public String getDeadNodes(int index) { + return deadNodes_.get(index); + } + public Builder setDeadNodes( + int index, String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureDeadNodesIsMutable(); + deadNodes_.set(index, value); + onChanged(); + return this; + } + public Builder addDeadNodes(String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureDeadNodesIsMutable(); + deadNodes_.add(value); + onChanged(); + return this; + } + public Builder addAllDeadNodes( + java.lang.Iterable values) { + ensureDeadNodesIsMutable(); + super.addAll(values, deadNodes_); + onChanged(); + return this; + } + public Builder clearDeadNodes() { + deadNodes_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + return this; + } + void addDeadNodes(com.google.protobuf.ByteString value) { + ensureDeadNodesIsMutable(); + deadNodes_.add(value); + onChanged(); + } + + // optional int32 regions = 3; + private int regions_ ; + public boolean hasRegions() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public int getRegions() { + return regions_; + } + public Builder setRegions(int value) { + bitField0_ |= 0x00000004; + regions_ = value; + onChanged(); + return this; + } + public Builder clearRegions() { + bitField0_ = (bitField0_ & ~0x00000004); + regions_ = 0; + onChanged(); + return this; + } + + // optional int32 requests = 4; + private int requests_ ; + public boolean hasRequests() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getRequests() { + return requests_; + } + public Builder setRequests(int value) { + bitField0_ |= 0x00000008; + requests_ = value; + onChanged(); + return this; + } + public Builder clearRequests() { + bitField0_ = (bitField0_ & ~0x00000008); + requests_ = 0; + onChanged(); + return this; + } + + // optional double averageLoad = 5; + private double averageLoad_ ; + public boolean hasAverageLoad() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public double getAverageLoad() { + return averageLoad_; + } + public Builder setAverageLoad(double value) { + bitField0_ |= 0x00000010; + averageLoad_ = value; + onChanged(); + return this; + } + public Builder clearAverageLoad() { + bitField0_ = (bitField0_ & ~0x00000010); + averageLoad_ = 0D; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus) + } + + static { + defaultInstance = new StorageClusterStatus(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatus) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Region_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Region_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Node_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Node_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n!StorageClusterStatusMessage.proto\022/org" + + ".apache.hadoop.hbase.rest.protobuf.gener" + + "ated\"\333\005\n\024StorageClusterStatus\022]\n\tliveNod" + + "es\030\001 \003(\0132J.org.apache.hadoop.hbase.rest." + + "protobuf.generated.StorageClusterStatus." + + "Node\022\021\n\tdeadNodes\030\002 \003(\t\022\017\n\007regions\030\003 \001(\005" + + "\022\020\n\010requests\030\004 \001(\005\022\023\n\013averageLoad\030\005 \001(\001\032" + + "\322\002\n\006Region\022\014\n\004name\030\001 \002(\014\022\016\n\006stores\030\002 \001(\005" + + "\022\022\n\nstorefiles\030\003 \001(\005\022\027\n\017storefileSizeMB\030" + + "\004 \001(\005\022\026\n\016memstoreSizeMB\030\005 \001(\005\022\034\n\024storefi", + "leIndexSizeMB\030\006 \001(\005\022\031\n\021readRequestsCount" + + "\030\007 \001(\003\022\032\n\022writeRequestsCount\030\010 \001(\003\022\027\n\017ro" + + "otIndexSizeKB\030\t \001(\005\022\036\n\026totalStaticIndexS" + + "izeKB\030\n \001(\005\022\036\n\026totalStaticBloomSizeKB\030\013 " + + "\001(\005\022\032\n\022totalCompactingKVs\030\014 \001(\003\022\033\n\023curre" + + "ntCompactedKVs\030\r \001(\003\032\303\001\n\004Node\022\014\n\004name\030\001 " + + "\002(\t\022\021\n\tstartCode\030\002 \001(\003\022\020\n\010requests\030\003 \001(\005" + + "\022\022\n\nheapSizeMB\030\004 \001(\005\022\025\n\rmaxHeapSizeMB\030\005 " + + "\001(\005\022]\n\007regions\030\006 \003(\0132L.org.apache.hadoop" + + ".hbase.rest.protobuf.generated.StorageCl", + "usterStatus.Region" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_descriptor, + new java.lang.String[] { "LiveNodes", "DeadNodes", "Regions", "Requests", "AverageLoad", }, + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.class, + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Builder.class); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Region_descriptor = + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_descriptor.getNestedTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Region_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Region_descriptor, + new java.lang.String[] { "Name", "Stores", "Storefiles", "StorefileSizeMB", "MemstoreSizeMB", "StorefileIndexSizeMB", "ReadRequestsCount", "WriteRequestsCount", "RootIndexSizeKB", "TotalStaticIndexSizeKB", "TotalStaticBloomSizeKB", "TotalCompactingKVs", "CurrentCompactedKVs", }, + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.class, + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Region.Builder.class); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Node_descriptor = + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_descriptor.getNestedTypes().get(1); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Node_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_StorageClusterStatus_Node_descriptor, + new java.lang.String[] { "Name", "StartCode", "Requests", "HeapSizeMB", "MaxHeapSizeMB", "Regions", }, + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.class, + org.apache.hadoop.hbase.rest.protobuf.generated.StorageClusterStatusMessage.StorageClusterStatus.Node.Builder.class); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/TableInfoMessage.java b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/TableInfoMessage.java new file mode 100644 index 0000000..7c11a89 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/TableInfoMessage.java @@ -0,0 +1,1391 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: TableInfoMessage.proto + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +public final class TableInfoMessage { + private TableInfoMessage() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public interface TableInfoOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required string name = 1; + boolean hasName(); + String getName(); + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.TableInfo.Region regions = 2; + java.util.List + getRegionsList(); + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region getRegions(int index); + int getRegionsCount(); + java.util.List + getRegionsOrBuilderList(); + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.RegionOrBuilder getRegionsOrBuilder( + int index); + } + public static final class TableInfo extends + com.google.protobuf.GeneratedMessage + implements TableInfoOrBuilder { + // Use TableInfo.newBuilder() to construct. + private TableInfo(Builder builder) { + super(builder); + } + private TableInfo(boolean noInit) {} + + private static final TableInfo defaultInstance; + public static TableInfo getDefaultInstance() { + return defaultInstance; + } + + public TableInfo getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_fieldAccessorTable; + } + + public interface RegionOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required string name = 1; + boolean hasName(); + String getName(); + + // optional bytes startKey = 2; + boolean hasStartKey(); + com.google.protobuf.ByteString getStartKey(); + + // optional bytes endKey = 3; + boolean hasEndKey(); + com.google.protobuf.ByteString getEndKey(); + + // optional int64 id = 4; + boolean hasId(); + long getId(); + + // optional string location = 5; + boolean hasLocation(); + String getLocation(); + } + public static final class Region extends + com.google.protobuf.GeneratedMessage + implements RegionOrBuilder { + // Use Region.newBuilder() to construct. + private Region(Builder builder) { + super(builder); + } + private Region(boolean noInit) {} + + private static final Region defaultInstance; + public static Region getDefaultInstance() { + return defaultInstance; + } + + public Region getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_Region_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_Region_fieldAccessorTable; + } + + private int bitField0_; + // required string name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private java.lang.Object name_; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + name_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional bytes startKey = 2; + public static final int STARTKEY_FIELD_NUMBER = 2; + private com.google.protobuf.ByteString startKey_; + public boolean hasStartKey() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public com.google.protobuf.ByteString getStartKey() { + return startKey_; + } + + // optional bytes endKey = 3; + public static final int ENDKEY_FIELD_NUMBER = 3; + private com.google.protobuf.ByteString endKey_; + public boolean hasEndKey() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public com.google.protobuf.ByteString getEndKey() { + return endKey_; + } + + // optional int64 id = 4; + public static final int ID_FIELD_NUMBER = 4; + private long id_; + public boolean hasId() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public long getId() { + return id_; + } + + // optional string location = 5; + public static final int LOCATION_FIELD_NUMBER = 5; + private java.lang.Object location_; + public boolean hasLocation() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public String getLocation() { + java.lang.Object ref = location_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + location_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getLocationBytes() { + java.lang.Object ref = location_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + location_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + name_ = ""; + startKey_ = com.google.protobuf.ByteString.EMPTY; + endKey_ = com.google.protobuf.ByteString.EMPTY; + id_ = 0L; + location_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasName()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, startKey_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeBytes(3, endKey_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt64(4, id_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeBytes(5, getLocationBytes()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(2, startKey_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(3, endKey_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(4, id_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(5, getLocationBytes()); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.RegionOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_Region_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_Region_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + startKey_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + endKey_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + id_ = 0L; + bitField0_ = (bitField0_ & ~0x00000008); + location_ = ""; + bitField0_ = (bitField0_ & ~0x00000010); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region build() { + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region result = new org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.startKey_ = startKey_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.endKey_ = endKey_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.id_ = id_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.location_ = location_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.getDefaultInstance()) return this; + if (other.hasName()) { + setName(other.getName()); + } + if (other.hasStartKey()) { + setStartKey(other.getStartKey()); + } + if (other.hasEndKey()) { + setEndKey(other.getEndKey()); + } + if (other.hasId()) { + setId(other.getId()); + } + if (other.hasLocation()) { + setLocation(other.getLocation()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasName()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + name_ = input.readBytes(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + startKey_ = input.readBytes(); + break; + } + case 26: { + bitField0_ |= 0x00000004; + endKey_ = input.readBytes(); + break; + } + case 32: { + bitField0_ |= 0x00000008; + id_ = input.readInt64(); + break; + } + case 42: { + bitField0_ |= 0x00000010; + location_ = input.readBytes(); + break; + } + } + } + } + + private int bitField0_; + + // required string name = 1; + private java.lang.Object name_ = ""; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + name_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setName(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + void setName(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + } + + // optional bytes startKey = 2; + private com.google.protobuf.ByteString startKey_ = com.google.protobuf.ByteString.EMPTY; + public boolean hasStartKey() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public com.google.protobuf.ByteString getStartKey() { + return startKey_; + } + public Builder setStartKey(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + startKey_ = value; + onChanged(); + return this; + } + public Builder clearStartKey() { + bitField0_ = (bitField0_ & ~0x00000002); + startKey_ = getDefaultInstance().getStartKey(); + onChanged(); + return this; + } + + // optional bytes endKey = 3; + private com.google.protobuf.ByteString endKey_ = com.google.protobuf.ByteString.EMPTY; + public boolean hasEndKey() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public com.google.protobuf.ByteString getEndKey() { + return endKey_; + } + public Builder setEndKey(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + endKey_ = value; + onChanged(); + return this; + } + public Builder clearEndKey() { + bitField0_ = (bitField0_ & ~0x00000004); + endKey_ = getDefaultInstance().getEndKey(); + onChanged(); + return this; + } + + // optional int64 id = 4; + private long id_ ; + public boolean hasId() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public long getId() { + return id_; + } + public Builder setId(long value) { + bitField0_ |= 0x00000008; + id_ = value; + onChanged(); + return this; + } + public Builder clearId() { + bitField0_ = (bitField0_ & ~0x00000008); + id_ = 0L; + onChanged(); + return this; + } + + // optional string location = 5; + private java.lang.Object location_ = ""; + public boolean hasLocation() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public String getLocation() { + java.lang.Object ref = location_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + location_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setLocation(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000010; + location_ = value; + onChanged(); + return this; + } + public Builder clearLocation() { + bitField0_ = (bitField0_ & ~0x00000010); + location_ = getDefaultInstance().getLocation(); + onChanged(); + return this; + } + void setLocation(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000010; + location_ = value; + onChanged(); + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.TableInfo.Region) + } + + static { + defaultInstance = new Region(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.TableInfo.Region) + } + + private int bitField0_; + // required string name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private java.lang.Object name_; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + name_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.TableInfo.Region regions = 2; + public static final int REGIONS_FIELD_NUMBER = 2; + private java.util.List regions_; + public java.util.List getRegionsList() { + return regions_; + } + public java.util.List + getRegionsOrBuilderList() { + return regions_; + } + public int getRegionsCount() { + return regions_.size(); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region getRegions(int index) { + return regions_.get(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.RegionOrBuilder getRegionsOrBuilder( + int index) { + return regions_.get(index); + } + + private void initFields() { + name_ = ""; + regions_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasName()) { + memoizedIsInitialized = 0; + return false; + } + for (int i = 0; i < getRegionsCount(); i++) { + if (!getRegions(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getNameBytes()); + } + for (int i = 0; i < regions_.size(); i++) { + output.writeMessage(2, regions_.get(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getNameBytes()); + } + for (int i = 0; i < regions_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(2, regions_.get(i)); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfoOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getRegionsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + if (regionsBuilder_ == null) { + regions_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + } else { + regionsBuilder_.clear(); + } + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo build() { + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo result = new org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + if (regionsBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002)) { + regions_ = java.util.Collections.unmodifiableList(regions_); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.regions_ = regions_; + } else { + result.regions_ = regionsBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.getDefaultInstance()) return this; + if (other.hasName()) { + setName(other.getName()); + } + if (regionsBuilder_ == null) { + if (!other.regions_.isEmpty()) { + if (regions_.isEmpty()) { + regions_ = other.regions_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureRegionsIsMutable(); + regions_.addAll(other.regions_); + } + onChanged(); + } + } else { + if (!other.regions_.isEmpty()) { + if (regionsBuilder_.isEmpty()) { + regionsBuilder_.dispose(); + regionsBuilder_ = null; + regions_ = other.regions_; + bitField0_ = (bitField0_ & ~0x00000002); + regionsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getRegionsFieldBuilder() : null; + } else { + regionsBuilder_.addAllMessages(other.regions_); + } + } + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasName()) { + + return false; + } + for (int i = 0; i < getRegionsCount(); i++) { + if (!getRegions(i).isInitialized()) { + + return false; + } + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + name_ = input.readBytes(); + break; + } + case 18: { + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.Builder subBuilder = org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addRegions(subBuilder.buildPartial()); + break; + } + } + } + } + + private int bitField0_; + + // required string name = 1; + private java.lang.Object name_ = ""; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + name_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setName(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + void setName(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.TableInfo.Region regions = 2; + private java.util.List regions_ = + java.util.Collections.emptyList(); + private void ensureRegionsIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + regions_ = new java.util.ArrayList(regions_); + bitField0_ |= 0x00000002; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region, org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.RegionOrBuilder> regionsBuilder_; + + public java.util.List getRegionsList() { + if (regionsBuilder_ == null) { + return java.util.Collections.unmodifiableList(regions_); + } else { + return regionsBuilder_.getMessageList(); + } + } + public int getRegionsCount() { + if (regionsBuilder_ == null) { + return regions_.size(); + } else { + return regionsBuilder_.getCount(); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region getRegions(int index) { + if (regionsBuilder_ == null) { + return regions_.get(index); + } else { + return regionsBuilder_.getMessage(index); + } + } + public Builder setRegions( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region value) { + if (regionsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRegionsIsMutable(); + regions_.set(index, value); + onChanged(); + } else { + regionsBuilder_.setMessage(index, value); + } + return this; + } + public Builder setRegions( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.Builder builderForValue) { + if (regionsBuilder_ == null) { + ensureRegionsIsMutable(); + regions_.set(index, builderForValue.build()); + onChanged(); + } else { + regionsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addRegions(org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region value) { + if (regionsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRegionsIsMutable(); + regions_.add(value); + onChanged(); + } else { + regionsBuilder_.addMessage(value); + } + return this; + } + public Builder addRegions( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region value) { + if (regionsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRegionsIsMutable(); + regions_.add(index, value); + onChanged(); + } else { + regionsBuilder_.addMessage(index, value); + } + return this; + } + public Builder addRegions( + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.Builder builderForValue) { + if (regionsBuilder_ == null) { + ensureRegionsIsMutable(); + regions_.add(builderForValue.build()); + onChanged(); + } else { + regionsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addRegions( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.Builder builderForValue) { + if (regionsBuilder_ == null) { + ensureRegionsIsMutable(); + regions_.add(index, builderForValue.build()); + onChanged(); + } else { + regionsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllRegions( + java.lang.Iterable values) { + if (regionsBuilder_ == null) { + ensureRegionsIsMutable(); + super.addAll(values, regions_); + onChanged(); + } else { + regionsBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearRegions() { + if (regionsBuilder_ == null) { + regions_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + } else { + regionsBuilder_.clear(); + } + return this; + } + public Builder removeRegions(int index) { + if (regionsBuilder_ == null) { + ensureRegionsIsMutable(); + regions_.remove(index); + onChanged(); + } else { + regionsBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.Builder getRegionsBuilder( + int index) { + return getRegionsFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.RegionOrBuilder getRegionsOrBuilder( + int index) { + if (regionsBuilder_ == null) { + return regions_.get(index); } else { + return regionsBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getRegionsOrBuilderList() { + if (regionsBuilder_ != null) { + return regionsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(regions_); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.Builder addRegionsBuilder() { + return getRegionsFieldBuilder().addBuilder( + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.getDefaultInstance()); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.Builder addRegionsBuilder( + int index) { + return getRegionsFieldBuilder().addBuilder( + index, org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.getDefaultInstance()); + } + public java.util.List + getRegionsBuilderList() { + return getRegionsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region, org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.RegionOrBuilder> + getRegionsFieldBuilder() { + if (regionsBuilder_ == null) { + regionsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region, org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.RegionOrBuilder>( + regions_, + ((bitField0_ & 0x00000002) == 0x00000002), + getParentForChildren(), + isClean()); + regions_ = null; + } + return regionsBuilder_; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.TableInfo) + } + + static { + defaultInstance = new TableInfo(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.TableInfo) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_Region_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_Region_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\026TableInfoMessage.proto\022/org.apache.had" + + "oop.hbase.rest.protobuf.generated\"\305\001\n\tTa" + + "bleInfo\022\014\n\004name\030\001 \002(\t\022R\n\007regions\030\002 \003(\0132A" + + ".org.apache.hadoop.hbase.rest.protobuf.g" + + "enerated.TableInfo.Region\032V\n\006Region\022\014\n\004n" + + "ame\030\001 \002(\t\022\020\n\010startKey\030\002 \001(\014\022\016\n\006endKey\030\003 " + + "\001(\014\022\n\n\002id\030\004 \001(\003\022\020\n\010location\030\005 \001(\t" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_descriptor, + new java.lang.String[] { "Name", "Regions", }, + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.class, + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Builder.class); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_Region_descriptor = + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_descriptor.getNestedTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_Region_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableInfo_Region_descriptor, + new java.lang.String[] { "Name", "StartKey", "EndKey", "Id", "Location", }, + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.class, + org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInfo.Region.Builder.class); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/TableListMessage.java b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/TableListMessage.java new file mode 100644 index 0000000..dcc60a1 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/TableListMessage.java @@ -0,0 +1,441 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: TableListMessage.proto + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +public final class TableListMessage { + private TableListMessage() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public interface TableListOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated string name = 1; + java.util.List getNameList(); + int getNameCount(); + String getName(int index); + } + public static final class TableList extends + com.google.protobuf.GeneratedMessage + implements TableListOrBuilder { + // Use TableList.newBuilder() to construct. + private TableList(Builder builder) { + super(builder); + } + private TableList(boolean noInit) {} + + private static final TableList defaultInstance; + public static TableList getDefaultInstance() { + return defaultInstance; + } + + public TableList getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableList_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableList_fieldAccessorTable; + } + + // repeated string name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private com.google.protobuf.LazyStringList name_; + public java.util.List + getNameList() { + return name_; + } + public int getNameCount() { + return name_.size(); + } + public String getName(int index) { + return name_.get(index); + } + + private void initFields() { + name_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + for (int i = 0; i < name_.size(); i++) { + output.writeBytes(1, name_.getByteString(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + { + int dataSize = 0; + for (int i = 0; i < name_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeBytesSizeNoTag(name_.getByteString(i)); + } + size += dataSize; + size += 1 * getNameList().size(); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableListOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableList_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableList_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList build() { + org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList result = new org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList(this); + int from_bitField0_ = bitField0_; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + name_ = new com.google.protobuf.UnmodifiableLazyStringList( + name_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.name_ = name_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList.getDefaultInstance()) return this; + if (!other.name_.isEmpty()) { + if (name_.isEmpty()) { + name_ = other.name_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureNameIsMutable(); + name_.addAll(other.name_); + } + onChanged(); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + ensureNameIsMutable(); + name_.add(input.readBytes()); + break; + } + } + } + } + + private int bitField0_; + + // repeated string name = 1; + private com.google.protobuf.LazyStringList name_ = com.google.protobuf.LazyStringArrayList.EMPTY; + private void ensureNameIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + name_ = new com.google.protobuf.LazyStringArrayList(name_); + bitField0_ |= 0x00000001; + } + } + public java.util.List + getNameList() { + return java.util.Collections.unmodifiableList(name_); + } + public int getNameCount() { + return name_.size(); + } + public String getName(int index) { + return name_.get(index); + } + public Builder setName( + int index, String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNameIsMutable(); + name_.set(index, value); + onChanged(); + return this; + } + public Builder addName(String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNameIsMutable(); + name_.add(value); + onChanged(); + return this; + } + public Builder addAllName( + java.lang.Iterable values) { + ensureNameIsMutable(); + super.addAll(values, name_); + onChanged(); + return this; + } + public Builder clearName() { + name_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + return this; + } + void addName(com.google.protobuf.ByteString value) { + ensureNameIsMutable(); + name_.add(value); + onChanged(); + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.TableList) + } + + static { + defaultInstance = new TableList(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.TableList) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableList_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableList_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\026TableListMessage.proto\022/org.apache.had" + + "oop.hbase.rest.protobuf.generated\"\031\n\tTab" + + "leList\022\014\n\004name\030\001 \003(\t" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableList_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableList_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableList_descriptor, + new java.lang.String[] { "Name", }, + org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList.class, + org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList.Builder.class); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/TableSchemaMessage.java b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/TableSchemaMessage.java new file mode 100644 index 0000000..b4aa349 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/TableSchemaMessage.java @@ -0,0 +1,1617 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: TableSchemaMessage.proto + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +public final class TableSchemaMessage { + private TableSchemaMessage() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public interface TableSchemaOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional string name = 1; + boolean hasName(); + String getName(); + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.TableSchema.Attribute attrs = 2; + java.util.List + getAttrsList(); + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute getAttrs(int index); + int getAttrsCount(); + java.util.List + getAttrsOrBuilderList(); + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.AttributeOrBuilder getAttrsOrBuilder( + int index); + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchema columns = 3; + java.util.List + getColumnsList(); + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema getColumns(int index); + int getColumnsCount(); + java.util.List + getColumnsOrBuilderList(); + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchemaOrBuilder getColumnsOrBuilder( + int index); + + // optional bool inMemory = 4; + boolean hasInMemory(); + boolean getInMemory(); + + // optional bool readOnly = 5; + boolean hasReadOnly(); + boolean getReadOnly(); + } + public static final class TableSchema extends + com.google.protobuf.GeneratedMessage + implements TableSchemaOrBuilder { + // Use TableSchema.newBuilder() to construct. + private TableSchema(Builder builder) { + super(builder); + } + private TableSchema(boolean noInit) {} + + private static final TableSchema defaultInstance; + public static TableSchema getDefaultInstance() { + return defaultInstance; + } + + public TableSchema getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_fieldAccessorTable; + } + + public interface AttributeOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required string name = 1; + boolean hasName(); + String getName(); + + // required string value = 2; + boolean hasValue(); + String getValue(); + } + public static final class Attribute extends + com.google.protobuf.GeneratedMessage + implements AttributeOrBuilder { + // Use Attribute.newBuilder() to construct. + private Attribute(Builder builder) { + super(builder); + } + private Attribute(boolean noInit) {} + + private static final Attribute defaultInstance; + public static Attribute getDefaultInstance() { + return defaultInstance; + } + + public Attribute getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_Attribute_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_Attribute_fieldAccessorTable; + } + + private int bitField0_; + // required string name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private java.lang.Object name_; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + name_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // required string value = 2; + public static final int VALUE_FIELD_NUMBER = 2; + private java.lang.Object value_; + public boolean hasValue() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getValue() { + java.lang.Object ref = value_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + value_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getValueBytes() { + java.lang.Object ref = value_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + value_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + name_ = ""; + value_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasName()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasValue()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, getValueBytes()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(2, getValueBytes()); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.AttributeOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_Attribute_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_Attribute_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + value_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute build() { + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute result = new org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.value_ = value_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.getDefaultInstance()) return this; + if (other.hasName()) { + setName(other.getName()); + } + if (other.hasValue()) { + setValue(other.getValue()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasName()) { + + return false; + } + if (!hasValue()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + name_ = input.readBytes(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + value_ = input.readBytes(); + break; + } + } + } + } + + private int bitField0_; + + // required string name = 1; + private java.lang.Object name_ = ""; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + name_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setName(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + void setName(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + } + + // required string value = 2; + private java.lang.Object value_ = ""; + public boolean hasValue() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getValue() { + java.lang.Object ref = value_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + value_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setValue(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + value_ = value; + onChanged(); + return this; + } + public Builder clearValue() { + bitField0_ = (bitField0_ & ~0x00000002); + value_ = getDefaultInstance().getValue(); + onChanged(); + return this; + } + void setValue(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000002; + value_ = value; + onChanged(); + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.TableSchema.Attribute) + } + + static { + defaultInstance = new Attribute(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.TableSchema.Attribute) + } + + private int bitField0_; + // optional string name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private java.lang.Object name_; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + name_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.TableSchema.Attribute attrs = 2; + public static final int ATTRS_FIELD_NUMBER = 2; + private java.util.List attrs_; + public java.util.List getAttrsList() { + return attrs_; + } + public java.util.List + getAttrsOrBuilderList() { + return attrs_; + } + public int getAttrsCount() { + return attrs_.size(); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute getAttrs(int index) { + return attrs_.get(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.AttributeOrBuilder getAttrsOrBuilder( + int index) { + return attrs_.get(index); + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchema columns = 3; + public static final int COLUMNS_FIELD_NUMBER = 3; + private java.util.List columns_; + public java.util.List getColumnsList() { + return columns_; + } + public java.util.List + getColumnsOrBuilderList() { + return columns_; + } + public int getColumnsCount() { + return columns_.size(); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema getColumns(int index) { + return columns_.get(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchemaOrBuilder getColumnsOrBuilder( + int index) { + return columns_.get(index); + } + + // optional bool inMemory = 4; + public static final int INMEMORY_FIELD_NUMBER = 4; + private boolean inMemory_; + public boolean hasInMemory() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public boolean getInMemory() { + return inMemory_; + } + + // optional bool readOnly = 5; + public static final int READONLY_FIELD_NUMBER = 5; + private boolean readOnly_; + public boolean hasReadOnly() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public boolean getReadOnly() { + return readOnly_; + } + + private void initFields() { + name_ = ""; + attrs_ = java.util.Collections.emptyList(); + columns_ = java.util.Collections.emptyList(); + inMemory_ = false; + readOnly_ = false; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + for (int i = 0; i < getAttrsCount(); i++) { + if (!getAttrs(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + for (int i = 0; i < getColumnsCount(); i++) { + if (!getColumns(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getNameBytes()); + } + for (int i = 0; i < attrs_.size(); i++) { + output.writeMessage(2, attrs_.get(i)); + } + for (int i = 0; i < columns_.size(); i++) { + output.writeMessage(3, columns_.get(i)); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBool(4, inMemory_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeBool(5, readOnly_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getNameBytes()); + } + for (int i = 0; i < attrs_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(2, attrs_.get(i)); + } + for (int i = 0; i < columns_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(3, columns_.get(i)); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(4, inMemory_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(5, readOnly_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchemaOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getAttrsFieldBuilder(); + getColumnsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + if (attrsBuilder_ == null) { + attrs_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + } else { + attrsBuilder_.clear(); + } + if (columnsBuilder_ == null) { + columns_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000004); + } else { + columnsBuilder_.clear(); + } + inMemory_ = false; + bitField0_ = (bitField0_ & ~0x00000008); + readOnly_ = false; + bitField0_ = (bitField0_ & ~0x00000010); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema build() { + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema result = new org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + if (attrsBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002)) { + attrs_ = java.util.Collections.unmodifiableList(attrs_); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.attrs_ = attrs_; + } else { + result.attrs_ = attrsBuilder_.build(); + } + if (columnsBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004)) { + columns_ = java.util.Collections.unmodifiableList(columns_); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.columns_ = columns_; + } else { + result.columns_ = columnsBuilder_.build(); + } + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000002; + } + result.inMemory_ = inMemory_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000004; + } + result.readOnly_ = readOnly_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.getDefaultInstance()) return this; + if (other.hasName()) { + setName(other.getName()); + } + if (attrsBuilder_ == null) { + if (!other.attrs_.isEmpty()) { + if (attrs_.isEmpty()) { + attrs_ = other.attrs_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureAttrsIsMutable(); + attrs_.addAll(other.attrs_); + } + onChanged(); + } + } else { + if (!other.attrs_.isEmpty()) { + if (attrsBuilder_.isEmpty()) { + attrsBuilder_.dispose(); + attrsBuilder_ = null; + attrs_ = other.attrs_; + bitField0_ = (bitField0_ & ~0x00000002); + attrsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getAttrsFieldBuilder() : null; + } else { + attrsBuilder_.addAllMessages(other.attrs_); + } + } + } + if (columnsBuilder_ == null) { + if (!other.columns_.isEmpty()) { + if (columns_.isEmpty()) { + columns_ = other.columns_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureColumnsIsMutable(); + columns_.addAll(other.columns_); + } + onChanged(); + } + } else { + if (!other.columns_.isEmpty()) { + if (columnsBuilder_.isEmpty()) { + columnsBuilder_.dispose(); + columnsBuilder_ = null; + columns_ = other.columns_; + bitField0_ = (bitField0_ & ~0x00000004); + columnsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getColumnsFieldBuilder() : null; + } else { + columnsBuilder_.addAllMessages(other.columns_); + } + } + } + if (other.hasInMemory()) { + setInMemory(other.getInMemory()); + } + if (other.hasReadOnly()) { + setReadOnly(other.getReadOnly()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + for (int i = 0; i < getAttrsCount(); i++) { + if (!getAttrs(i).isInitialized()) { + + return false; + } + } + for (int i = 0; i < getColumnsCount(); i++) { + if (!getColumns(i).isInitialized()) { + + return false; + } + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + name_ = input.readBytes(); + break; + } + case 18: { + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.Builder subBuilder = org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addAttrs(subBuilder.buildPartial()); + break; + } + case 26: { + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Builder subBuilder = org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addColumns(subBuilder.buildPartial()); + break; + } + case 32: { + bitField0_ |= 0x00000008; + inMemory_ = input.readBool(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + readOnly_ = input.readBool(); + break; + } + } + } + } + + private int bitField0_; + + // optional string name = 1; + private java.lang.Object name_ = ""; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + name_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setName(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + void setName(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.TableSchema.Attribute attrs = 2; + private java.util.List attrs_ = + java.util.Collections.emptyList(); + private void ensureAttrsIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + attrs_ = new java.util.ArrayList(attrs_); + bitField0_ |= 0x00000002; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute, org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.AttributeOrBuilder> attrsBuilder_; + + public java.util.List getAttrsList() { + if (attrsBuilder_ == null) { + return java.util.Collections.unmodifiableList(attrs_); + } else { + return attrsBuilder_.getMessageList(); + } + } + public int getAttrsCount() { + if (attrsBuilder_ == null) { + return attrs_.size(); + } else { + return attrsBuilder_.getCount(); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute getAttrs(int index) { + if (attrsBuilder_ == null) { + return attrs_.get(index); + } else { + return attrsBuilder_.getMessage(index); + } + } + public Builder setAttrs( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute value) { + if (attrsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureAttrsIsMutable(); + attrs_.set(index, value); + onChanged(); + } else { + attrsBuilder_.setMessage(index, value); + } + return this; + } + public Builder setAttrs( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.Builder builderForValue) { + if (attrsBuilder_ == null) { + ensureAttrsIsMutable(); + attrs_.set(index, builderForValue.build()); + onChanged(); + } else { + attrsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAttrs(org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute value) { + if (attrsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureAttrsIsMutable(); + attrs_.add(value); + onChanged(); + } else { + attrsBuilder_.addMessage(value); + } + return this; + } + public Builder addAttrs( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute value) { + if (attrsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureAttrsIsMutable(); + attrs_.add(index, value); + onChanged(); + } else { + attrsBuilder_.addMessage(index, value); + } + return this; + } + public Builder addAttrs( + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.Builder builderForValue) { + if (attrsBuilder_ == null) { + ensureAttrsIsMutable(); + attrs_.add(builderForValue.build()); + onChanged(); + } else { + attrsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addAttrs( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.Builder builderForValue) { + if (attrsBuilder_ == null) { + ensureAttrsIsMutable(); + attrs_.add(index, builderForValue.build()); + onChanged(); + } else { + attrsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllAttrs( + java.lang.Iterable values) { + if (attrsBuilder_ == null) { + ensureAttrsIsMutable(); + super.addAll(values, attrs_); + onChanged(); + } else { + attrsBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearAttrs() { + if (attrsBuilder_ == null) { + attrs_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + } else { + attrsBuilder_.clear(); + } + return this; + } + public Builder removeAttrs(int index) { + if (attrsBuilder_ == null) { + ensureAttrsIsMutable(); + attrs_.remove(index); + onChanged(); + } else { + attrsBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.Builder getAttrsBuilder( + int index) { + return getAttrsFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.AttributeOrBuilder getAttrsOrBuilder( + int index) { + if (attrsBuilder_ == null) { + return attrs_.get(index); } else { + return attrsBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getAttrsOrBuilderList() { + if (attrsBuilder_ != null) { + return attrsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(attrs_); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.Builder addAttrsBuilder() { + return getAttrsFieldBuilder().addBuilder( + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.getDefaultInstance()); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.Builder addAttrsBuilder( + int index) { + return getAttrsFieldBuilder().addBuilder( + index, org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.getDefaultInstance()); + } + public java.util.List + getAttrsBuilderList() { + return getAttrsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute, org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.AttributeOrBuilder> + getAttrsFieldBuilder() { + if (attrsBuilder_ == null) { + attrsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute, org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.AttributeOrBuilder>( + attrs_, + ((bitField0_ & 0x00000002) == 0x00000002), + getParentForChildren(), + isClean()); + attrs_ = null; + } + return attrsBuilder_; + } + + // repeated .org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchema columns = 3; + private java.util.List columns_ = + java.util.Collections.emptyList(); + private void ensureColumnsIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + columns_ = new java.util.ArrayList(columns_); + bitField0_ |= 0x00000004; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchemaOrBuilder> columnsBuilder_; + + public java.util.List getColumnsList() { + if (columnsBuilder_ == null) { + return java.util.Collections.unmodifiableList(columns_); + } else { + return columnsBuilder_.getMessageList(); + } + } + public int getColumnsCount() { + if (columnsBuilder_ == null) { + return columns_.size(); + } else { + return columnsBuilder_.getCount(); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema getColumns(int index) { + if (columnsBuilder_ == null) { + return columns_.get(index); + } else { + return columnsBuilder_.getMessage(index); + } + } + public Builder setColumns( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema value) { + if (columnsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureColumnsIsMutable(); + columns_.set(index, value); + onChanged(); + } else { + columnsBuilder_.setMessage(index, value); + } + return this; + } + public Builder setColumns( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Builder builderForValue) { + if (columnsBuilder_ == null) { + ensureColumnsIsMutable(); + columns_.set(index, builderForValue.build()); + onChanged(); + } else { + columnsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addColumns(org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema value) { + if (columnsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureColumnsIsMutable(); + columns_.add(value); + onChanged(); + } else { + columnsBuilder_.addMessage(value); + } + return this; + } + public Builder addColumns( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema value) { + if (columnsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureColumnsIsMutable(); + columns_.add(index, value); + onChanged(); + } else { + columnsBuilder_.addMessage(index, value); + } + return this; + } + public Builder addColumns( + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Builder builderForValue) { + if (columnsBuilder_ == null) { + ensureColumnsIsMutable(); + columns_.add(builderForValue.build()); + onChanged(); + } else { + columnsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addColumns( + int index, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Builder builderForValue) { + if (columnsBuilder_ == null) { + ensureColumnsIsMutable(); + columns_.add(index, builderForValue.build()); + onChanged(); + } else { + columnsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllColumns( + java.lang.Iterable values) { + if (columnsBuilder_ == null) { + ensureColumnsIsMutable(); + super.addAll(values, columns_); + onChanged(); + } else { + columnsBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearColumns() { + if (columnsBuilder_ == null) { + columns_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + } else { + columnsBuilder_.clear(); + } + return this; + } + public Builder removeColumns(int index) { + if (columnsBuilder_ == null) { + ensureColumnsIsMutable(); + columns_.remove(index); + onChanged(); + } else { + columnsBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Builder getColumnsBuilder( + int index) { + return getColumnsFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchemaOrBuilder getColumnsOrBuilder( + int index) { + if (columnsBuilder_ == null) { + return columns_.get(index); } else { + return columnsBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getColumnsOrBuilderList() { + if (columnsBuilder_ != null) { + return columnsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(columns_); + } + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Builder addColumnsBuilder() { + return getColumnsFieldBuilder().addBuilder( + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.getDefaultInstance()); + } + public org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Builder addColumnsBuilder( + int index) { + return getColumnsFieldBuilder().addBuilder( + index, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.getDefaultInstance()); + } + public java.util.List + getColumnsBuilderList() { + return getColumnsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchemaOrBuilder> + getColumnsFieldBuilder() { + if (columnsBuilder_ == null) { + columnsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchema.Builder, org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.ColumnSchemaOrBuilder>( + columns_, + ((bitField0_ & 0x00000004) == 0x00000004), + getParentForChildren(), + isClean()); + columns_ = null; + } + return columnsBuilder_; + } + + // optional bool inMemory = 4; + private boolean inMemory_ ; + public boolean hasInMemory() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public boolean getInMemory() { + return inMemory_; + } + public Builder setInMemory(boolean value) { + bitField0_ |= 0x00000008; + inMemory_ = value; + onChanged(); + return this; + } + public Builder clearInMemory() { + bitField0_ = (bitField0_ & ~0x00000008); + inMemory_ = false; + onChanged(); + return this; + } + + // optional bool readOnly = 5; + private boolean readOnly_ ; + public boolean hasReadOnly() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public boolean getReadOnly() { + return readOnly_; + } + public Builder setReadOnly(boolean value) { + bitField0_ |= 0x00000010; + readOnly_ = value; + onChanged(); + return this; + } + public Builder clearReadOnly() { + bitField0_ = (bitField0_ & ~0x00000010); + readOnly_ = false; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.TableSchema) + } + + static { + defaultInstance = new TableSchema(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.TableSchema) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_Attribute_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_Attribute_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\030TableSchemaMessage.proto\022/org.apache.h" + + "adoop.hbase.rest.protobuf.generated\032\031Col" + + "umnSchemaMessage.proto\"\220\002\n\013TableSchema\022\014" + + "\n\004name\030\001 \001(\t\022U\n\005attrs\030\002 \003(\0132F.org.apache" + + ".hadoop.hbase.rest.protobuf.generated.Ta" + + "bleSchema.Attribute\022N\n\007columns\030\003 \003(\0132=.o" + + "rg.apache.hadoop.hbase.rest.protobuf.gen" + + "erated.ColumnSchema\022\020\n\010inMemory\030\004 \001(\010\022\020\n" + + "\010readOnly\030\005 \001(\010\032(\n\tAttribute\022\014\n\004name\030\001 \002" + + "(\t\022\r\n\005value\030\002 \002(\t" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_descriptor, + new java.lang.String[] { "Name", "Attrs", "Columns", "InMemory", "ReadOnly", }, + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.class, + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Builder.class); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_Attribute_descriptor = + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_descriptor.getNestedTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_Attribute_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_TableSchema_Attribute_descriptor, + new java.lang.String[] { "Name", "Value", }, + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.class, + org.apache.hadoop.hbase.rest.protobuf.generated.TableSchemaMessage.TableSchema.Attribute.Builder.class); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + org.apache.hadoop.hbase.rest.protobuf.generated.ColumnSchemaMessage.getDescriptor(), + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/VersionMessage.java b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/VersionMessage.java new file mode 100644 index 0000000..1757922 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/protobuf/generated/VersionMessage.java @@ -0,0 +1,805 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: VersionMessage.proto + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +public final class VersionMessage { + private VersionMessage() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public interface VersionOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional string restVersion = 1; + boolean hasRestVersion(); + String getRestVersion(); + + // optional string jvmVersion = 2; + boolean hasJvmVersion(); + String getJvmVersion(); + + // optional string osVersion = 3; + boolean hasOsVersion(); + String getOsVersion(); + + // optional string serverVersion = 4; + boolean hasServerVersion(); + String getServerVersion(); + + // optional string jerseyVersion = 5; + boolean hasJerseyVersion(); + String getJerseyVersion(); + } + public static final class Version extends + com.google.protobuf.GeneratedMessage + implements VersionOrBuilder { + // Use Version.newBuilder() to construct. + private Version(Builder builder) { + super(builder); + } + private Version(boolean noInit) {} + + private static final Version defaultInstance; + public static Version getDefaultInstance() { + return defaultInstance; + } + + public Version getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Version_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Version_fieldAccessorTable; + } + + private int bitField0_; + // optional string restVersion = 1; + public static final int RESTVERSION_FIELD_NUMBER = 1; + private java.lang.Object restVersion_; + public boolean hasRestVersion() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getRestVersion() { + java.lang.Object ref = restVersion_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + restVersion_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getRestVersionBytes() { + java.lang.Object ref = restVersion_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + restVersion_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional string jvmVersion = 2; + public static final int JVMVERSION_FIELD_NUMBER = 2; + private java.lang.Object jvmVersion_; + public boolean hasJvmVersion() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getJvmVersion() { + java.lang.Object ref = jvmVersion_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + jvmVersion_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getJvmVersionBytes() { + java.lang.Object ref = jvmVersion_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + jvmVersion_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional string osVersion = 3; + public static final int OSVERSION_FIELD_NUMBER = 3; + private java.lang.Object osVersion_; + public boolean hasOsVersion() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public String getOsVersion() { + java.lang.Object ref = osVersion_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + osVersion_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getOsVersionBytes() { + java.lang.Object ref = osVersion_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + osVersion_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional string serverVersion = 4; + public static final int SERVERVERSION_FIELD_NUMBER = 4; + private java.lang.Object serverVersion_; + public boolean hasServerVersion() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public String getServerVersion() { + java.lang.Object ref = serverVersion_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + serverVersion_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getServerVersionBytes() { + java.lang.Object ref = serverVersion_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + serverVersion_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional string jerseyVersion = 5; + public static final int JERSEYVERSION_FIELD_NUMBER = 5; + private java.lang.Object jerseyVersion_; + public boolean hasJerseyVersion() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public String getJerseyVersion() { + java.lang.Object ref = jerseyVersion_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + jerseyVersion_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getJerseyVersionBytes() { + java.lang.Object ref = jerseyVersion_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + jerseyVersion_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + restVersion_ = ""; + jvmVersion_ = ""; + osVersion_ = ""; + serverVersion_ = ""; + jerseyVersion_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getRestVersionBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, getJvmVersionBytes()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeBytes(3, getOsVersionBytes()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeBytes(4, getServerVersionBytes()); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeBytes(5, getJerseyVersionBytes()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getRestVersionBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(2, getJvmVersionBytes()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(3, getOsVersionBytes()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(4, getServerVersionBytes()); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(5, getJerseyVersionBytes()); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.VersionOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Version_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Version_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + restVersion_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + jvmVersion_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + osVersion_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + serverVersion_ = ""; + bitField0_ = (bitField0_ & ~0x00000008); + jerseyVersion_ = ""; + bitField0_ = (bitField0_ & ~0x00000010); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version.getDescriptor(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version getDefaultInstanceForType() { + return org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version build() { + org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version buildPartial() { + org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version result = new org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.restVersion_ = restVersion_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.jvmVersion_ = jvmVersion_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.osVersion_ = osVersion_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.serverVersion_ = serverVersion_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.jerseyVersion_ = jerseyVersion_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version) { + return mergeFrom((org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version other) { + if (other == org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version.getDefaultInstance()) return this; + if (other.hasRestVersion()) { + setRestVersion(other.getRestVersion()); + } + if (other.hasJvmVersion()) { + setJvmVersion(other.getJvmVersion()); + } + if (other.hasOsVersion()) { + setOsVersion(other.getOsVersion()); + } + if (other.hasServerVersion()) { + setServerVersion(other.getServerVersion()); + } + if (other.hasJerseyVersion()) { + setJerseyVersion(other.getJerseyVersion()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + restVersion_ = input.readBytes(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + jvmVersion_ = input.readBytes(); + break; + } + case 26: { + bitField0_ |= 0x00000004; + osVersion_ = input.readBytes(); + break; + } + case 34: { + bitField0_ |= 0x00000008; + serverVersion_ = input.readBytes(); + break; + } + case 42: { + bitField0_ |= 0x00000010; + jerseyVersion_ = input.readBytes(); + break; + } + } + } + } + + private int bitField0_; + + // optional string restVersion = 1; + private java.lang.Object restVersion_ = ""; + public boolean hasRestVersion() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getRestVersion() { + java.lang.Object ref = restVersion_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + restVersion_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setRestVersion(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + restVersion_ = value; + onChanged(); + return this; + } + public Builder clearRestVersion() { + bitField0_ = (bitField0_ & ~0x00000001); + restVersion_ = getDefaultInstance().getRestVersion(); + onChanged(); + return this; + } + void setRestVersion(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + restVersion_ = value; + onChanged(); + } + + // optional string jvmVersion = 2; + private java.lang.Object jvmVersion_ = ""; + public boolean hasJvmVersion() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getJvmVersion() { + java.lang.Object ref = jvmVersion_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + jvmVersion_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setJvmVersion(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + jvmVersion_ = value; + onChanged(); + return this; + } + public Builder clearJvmVersion() { + bitField0_ = (bitField0_ & ~0x00000002); + jvmVersion_ = getDefaultInstance().getJvmVersion(); + onChanged(); + return this; + } + void setJvmVersion(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000002; + jvmVersion_ = value; + onChanged(); + } + + // optional string osVersion = 3; + private java.lang.Object osVersion_ = ""; + public boolean hasOsVersion() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public String getOsVersion() { + java.lang.Object ref = osVersion_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + osVersion_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setOsVersion(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + osVersion_ = value; + onChanged(); + return this; + } + public Builder clearOsVersion() { + bitField0_ = (bitField0_ & ~0x00000004); + osVersion_ = getDefaultInstance().getOsVersion(); + onChanged(); + return this; + } + void setOsVersion(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000004; + osVersion_ = value; + onChanged(); + } + + // optional string serverVersion = 4; + private java.lang.Object serverVersion_ = ""; + public boolean hasServerVersion() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public String getServerVersion() { + java.lang.Object ref = serverVersion_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + serverVersion_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setServerVersion(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000008; + serverVersion_ = value; + onChanged(); + return this; + } + public Builder clearServerVersion() { + bitField0_ = (bitField0_ & ~0x00000008); + serverVersion_ = getDefaultInstance().getServerVersion(); + onChanged(); + return this; + } + void setServerVersion(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000008; + serverVersion_ = value; + onChanged(); + } + + // optional string jerseyVersion = 5; + private java.lang.Object jerseyVersion_ = ""; + public boolean hasJerseyVersion() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public String getJerseyVersion() { + java.lang.Object ref = jerseyVersion_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + jerseyVersion_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setJerseyVersion(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000010; + jerseyVersion_ = value; + onChanged(); + return this; + } + public Builder clearJerseyVersion() { + bitField0_ = (bitField0_ & ~0x00000010); + jerseyVersion_ = getDefaultInstance().getJerseyVersion(); + onChanged(); + return this; + } + void setJerseyVersion(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000010; + jerseyVersion_ = value; + onChanged(); + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hbase.rest.protobuf.generated.Version) + } + + static { + defaultInstance = new Version(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hbase.rest.protobuf.generated.Version) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Version_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Version_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\024VersionMessage.proto\022/org.apache.hadoo" + + "p.hbase.rest.protobuf.generated\"s\n\007Versi" + + "on\022\023\n\013restVersion\030\001 \001(\t\022\022\n\njvmVersion\030\002 " + + "\001(\t\022\021\n\tosVersion\030\003 \001(\t\022\025\n\rserverVersion\030" + + "\004 \001(\t\022\025\n\rjerseyVersion\030\005 \001(\t" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Version_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Version_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hbase_rest_protobuf_generated_Version_descriptor, + new java.lang.String[] { "RestVersion", "JvmVersion", "OsVersion", "ServerVersion", "JerseyVersion", }, + org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version.class, + org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version.Builder.class); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/provider/JAXBContextResolver.java b/src/main/java/org/apache/hadoop/hbase/rest/provider/JAXBContextResolver.java new file mode 100644 index 0000000..0c2ab3d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/provider/JAXBContextResolver.java @@ -0,0 +1,88 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.provider; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import javax.ws.rs.ext.ContextResolver; +import javax.ws.rs.ext.Provider; +import javax.xml.bind.JAXBContext; + +import org.apache.hadoop.hbase.rest.model.CellModel; +import org.apache.hadoop.hbase.rest.model.CellSetModel; +import org.apache.hadoop.hbase.rest.model.ColumnSchemaModel; +import org.apache.hadoop.hbase.rest.model.RowModel; +import org.apache.hadoop.hbase.rest.model.ScannerModel; +import org.apache.hadoop.hbase.rest.model.StorageClusterStatusModel; +import org.apache.hadoop.hbase.rest.model.StorageClusterVersionModel; +import org.apache.hadoop.hbase.rest.model.TableInfoModel; +import org.apache.hadoop.hbase.rest.model.TableListModel; +import org.apache.hadoop.hbase.rest.model.TableModel; +import org.apache.hadoop.hbase.rest.model.TableRegionModel; +import org.apache.hadoop.hbase.rest.model.TableSchemaModel; +import org.apache.hadoop.hbase.rest.model.VersionModel; + +import com.sun.jersey.api.json.JSONConfiguration; +import com.sun.jersey.api.json.JSONJAXBContext; + +/** + * Plumbing for hooking up Jersey's JSON entity body encoding and decoding + * support to JAXB. Modify how the context is created (by using e.g. a + * different configuration builder) to control how JSON is processed and + * created. + */ +@Provider +public class JAXBContextResolver implements ContextResolver { + + private final JAXBContext context; + + private final Set> types; + + private final Class[] cTypes = { + CellModel.class, + CellSetModel.class, + ColumnSchemaModel.class, + RowModel.class, + ScannerModel.class, + StorageClusterStatusModel.class, + StorageClusterVersionModel.class, + TableInfoModel.class, + TableListModel.class, + TableModel.class, + TableRegionModel.class, + TableSchemaModel.class, + VersionModel.class + }; + + @SuppressWarnings("unchecked") + public JAXBContextResolver() throws Exception { + this.types = new HashSet(Arrays.asList(cTypes)); + this.context = new JSONJAXBContext(JSONConfiguration.natural().build(), + cTypes); + } + + @Override + public JAXBContext getContext(Class objectType) { + return (types.contains(objectType)) ? context : null; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/provider/consumer/ProtobufMessageBodyConsumer.java b/src/main/java/org/apache/hadoop/hbase/rest/provider/consumer/ProtobufMessageBodyConsumer.java new file mode 100644 index 0000000..daccb8d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/provider/consumer/ProtobufMessageBodyConsumer.java @@ -0,0 +1,87 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.provider.consumer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; + +import javax.ws.rs.Consumes; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.ext.MessageBodyReader; +import javax.ws.rs.ext.Provider; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.rest.Constants; +import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; + +/** + * Adapter for hooking up Jersey content processing dispatch to + * ProtobufMessageHandler interface capable handlers for decoding protobuf input. + */ +@Provider +@Consumes({Constants.MIMETYPE_PROTOBUF, Constants.MIMETYPE_PROTOBUF_IETF}) +public class ProtobufMessageBodyConsumer + implements MessageBodyReader { + private static final Log LOG = + LogFactory.getLog(ProtobufMessageBodyConsumer.class); + + @Override + public boolean isReadable(Class type, Type genericType, + Annotation[] annotations, MediaType mediaType) { + return ProtobufMessageHandler.class.isAssignableFrom(type); + } + + @Override + public ProtobufMessageHandler readFrom(Class type, Type genericType, + Annotation[] annotations, MediaType mediaType, + MultivaluedMap httpHeaders, InputStream inputStream) + throws IOException, WebApplicationException { + ProtobufMessageHandler obj = null; + try { + obj = type.newInstance(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + do { + read = inputStream.read(buffer, 0, buffer.length); + if (read > 0) { + baos.write(buffer, 0, read); + } + } while (read > 0); + if (LOG.isDebugEnabled()) { + LOG.debug(getClass() + ": read " + baos.size() + " bytes from " + + inputStream); + } + obj = obj.getObjectFromMessage(baos.toByteArray()); + } catch (InstantiationException e) { + throw new WebApplicationException(e); + } catch (IllegalAccessException e) { + throw new WebApplicationException(e); + } + return obj; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/PlainTextMessageBodyProducer.java b/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/PlainTextMessageBodyProducer.java new file mode 100644 index 0000000..092c695 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/PlainTextMessageBodyProducer.java @@ -0,0 +1,73 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.provider.producer; + +import java.io.IOException; +import java.io.OutputStream; +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; + +import javax.ws.rs.Produces; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.ext.MessageBodyWriter; +import javax.ws.rs.ext.Provider; + +import org.apache.hadoop.hbase.rest.Constants; + +/** + * An adapter between Jersey and Object.toString(). Hooks up plain text output + * to the Jersey content handling framework. + * Jersey will first call getSize() to learn the number of bytes that will be + * sent, then writeTo to perform the actual I/O. + */ +@Provider +@Produces(Constants.MIMETYPE_TEXT) +public class PlainTextMessageBodyProducer + implements MessageBodyWriter { + + private ThreadLocal buffer = new ThreadLocal(); + + @Override + public boolean isWriteable(Class arg0, Type arg1, Annotation[] arg2, + MediaType arg3) { + return true; + } + + @Override + public long getSize(Object object, Class type, Type genericType, + Annotation[] annotations, MediaType mediaType) { + byte[] bytes = object.toString().getBytes(); + buffer.set(bytes); + return bytes.length; + } + + @Override + public void writeTo(Object object, Class type, Type genericType, + Annotation[] annotations, MediaType mediaType, + MultivaluedMap httpHeaders, OutputStream outStream) + throws IOException, WebApplicationException { + byte[] bytes = buffer.get(); + outStream.write(bytes); + buffer.remove(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java b/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java new file mode 100644 index 0000000..0117604 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java @@ -0,0 +1,80 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.provider.producer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; + +import javax.ws.rs.Produces; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.ext.MessageBodyWriter; +import javax.ws.rs.ext.Provider; + +import org.apache.hadoop.hbase.rest.Constants; +import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; + +/** + * An adapter between Jersey and ProtobufMessageHandler implementors. Hooks up + * protobuf output producing methods to the Jersey content handling framework. + * Jersey will first call getSize() to learn the number of bytes that will be + * sent, then writeTo to perform the actual I/O. + */ +@Provider +@Produces({Constants.MIMETYPE_PROTOBUF, Constants.MIMETYPE_PROTOBUF_IETF}) +public class ProtobufMessageBodyProducer + implements MessageBodyWriter { + + private ThreadLocal buffer = new ThreadLocal(); + + @Override + public boolean isWriteable(Class type, Type genericType, + Annotation[] annotations, MediaType mediaType) { + return ProtobufMessageHandler.class.isAssignableFrom(type); + } + + @Override + public long getSize(ProtobufMessageHandler m, Class type, Type genericType, + Annotation[] annotations, MediaType mediaType) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + baos.write(m.createProtobufOutput()); + } catch (IOException e) { + return -1; + } + byte[] bytes = baos.toByteArray(); + buffer.set(bytes); + return bytes.length; + } + + public void writeTo(ProtobufMessageHandler m, Class type, Type genericType, + Annotation[] annotations, MediaType mediaType, + MultivaluedMap httpHeaders, OutputStream entityStream) + throws IOException, WebApplicationException { + byte[] bytes = buffer.get(); + entityStream.write(bytes); + buffer.remove(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/security/KerberosInfo.java b/src/main/java/org/apache/hadoop/hbase/security/KerberosInfo.java new file mode 100644 index 0000000..da21145 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/security/KerberosInfo.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Indicates Kerberos related information to be used for authorizing connections + * over a given RPC protocol interface. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface KerberosInfo { + /** Key for getting server's Kerberos principal name from Configuration */ + String serverPrincipal(); + String clientPrincipal() default ""; +} diff --git a/src/main/java/org/apache/hadoop/hbase/security/TokenInfo.java b/src/main/java/org/apache/hadoop/hbase/security/TokenInfo.java new file mode 100644 index 0000000..22a9a5f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/security/TokenInfo.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Indicates Token related information to be used in authorizing connections + * over a given RPC protocol interface. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface TokenInfo { + /** The type of Token.getKind() to be handled */ + String value(); +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/security/User.java b/src/main/java/org/apache/hadoop/hbase/security/User.java new file mode 100644 index 0000000..298e3ab --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/security/User.java @@ -0,0 +1,610 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security; + +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.util.Methods; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.UndeclaredThrowableException; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; + +import org.apache.commons.logging.Log; + +/** + * Wrapper to abstract out usage of user and group information in HBase. + * + *

    + * This class provides a common interface for interacting with user and group + * information across changing APIs in different versions of Hadoop. It only + * provides access to the common set of functionality in + * {@link org.apache.hadoop.security.UserGroupInformation} currently needed by + * HBase, but can be extended as needs change. + *

    + */ +public abstract class User { + public static final String HBASE_SECURITY_CONF_KEY = + "hbase.security.authentication"; + + /** + * Flag to differentiate between API-incompatible changes to + * {@link org.apache.hadoop.security.UserGroupInformation} between vanilla + * Hadoop 0.20.x and secure Hadoop 0.20+. + */ + private static boolean IS_SECURE_HADOOP = true; + static { + try { + UserGroupInformation.class.getMethod("isSecurityEnabled"); + } catch (NoSuchMethodException nsme) { + IS_SECURE_HADOOP = false; + } + } + private static Log LOG = LogFactory.getLog(User.class); + + protected UserGroupInformation ugi; + + public UserGroupInformation getUGI() { + return ugi; + } + + /** + * Returns the full user name. For Kerberos principals this will include + * the host and realm portions of the principal name. + * @return User full name. + */ + public String getName() { + return ugi.getUserName(); + } + + /** + * Returns the list of groups of which this user is a member. On secure + * Hadoop this returns the group information for the user as resolved on the + * server. For 0.20 based Hadoop, the group names are passed from the client. + */ + public String[] getGroupNames() { + return ugi.getGroupNames(); + } + + /** + * Returns the shortened version of the user name -- the portion that maps + * to an operating system user name. + * @return Short name + */ + public abstract String getShortName(); + + /** + * Executes the given action within the context of this user. + */ + public abstract T runAs(PrivilegedAction action); + + /** + * Executes the given action within the context of this user. + */ + public abstract T runAs(PrivilegedExceptionAction action) + throws IOException, InterruptedException; + + /** + * Requests an authentication token for this user and stores it in the + * user's credentials. + * + * @throws IOException + */ + public abstract void obtainAuthTokenForJob(Configuration conf, Job job) + throws IOException, InterruptedException; + + /** + * Requests an authentication token for this user and stores it in the + * user's credentials. + * + * @throws IOException + */ + public abstract void obtainAuthTokenForJob(JobConf job) + throws IOException, InterruptedException; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + return ugi.equals(((User) o).ugi); + } + + @Override + public int hashCode() { + return ugi.hashCode(); + } + + @Override + public String toString() { + return ugi.toString(); + } + + /** + * Returns the {@code User} instance within current execution context. + */ + public static User getCurrent() throws IOException { + User user; + if (IS_SECURE_HADOOP) { + user = new SecureHadoopUser(); + } else { + user = new HadoopUser(); + } + if (user.getUGI() == null) { + return null; + } + return user; + } + + /** + * Wraps an underlying {@code UserGroupInformation} instance. + * @param ugi The base Hadoop user + * @return User + */ + public static User create(UserGroupInformation ugi) { + if (ugi == null) { + return null; + } + + if (IS_SECURE_HADOOP) { + return new SecureHadoopUser(ugi); + } + return new HadoopUser(ugi); + } + + /** + * Generates a new {@code User} instance specifically for use in test code. + * @param name the full username + * @param groups the group names to which the test user will belong + * @return a new User instance + */ + public static User createUserForTesting(Configuration conf, + String name, String[] groups) { + if (IS_SECURE_HADOOP) { + return SecureHadoopUser.createUserForTesting(conf, name, groups); + } + return HadoopUser.createUserForTesting(conf, name, groups); + } + + /** + * Log in the current process using the given configuration keys for the + * credential file and login principal. + * + *

    This is only applicable when + * running on secure Hadoop -- see + * org.apache.hadoop.security.SecurityUtil#login(Configuration,String,String,String). + * On regular Hadoop (without security features), this will safely be ignored. + *

    + * + * @param conf The configuration data to use + * @param fileConfKey Property key used to configure path to the credential file + * @param principalConfKey Property key used to configure login principal + * @param localhost Current hostname to use in any credentials + * @throws IOException underlying exception from SecurityUtil.login() call + */ + public static void login(Configuration conf, String fileConfKey, + String principalConfKey, String localhost) throws IOException { + if (IS_SECURE_HADOOP) { + SecureHadoopUser.login(conf, fileConfKey, principalConfKey, localhost); + } else { + HadoopUser.login(conf, fileConfKey, principalConfKey, localhost); + } + } + + /** + * Returns whether or not Kerberos authentication is configured for Hadoop. + * For non-secure Hadoop, this always returns false. + * For secure Hadoop, it will return the value from + * {@code UserGroupInformation.isSecurityEnabled()}. + */ + public static boolean isSecurityEnabled() { + if (IS_SECURE_HADOOP) { + return SecureHadoopUser.isSecurityEnabled(); + } else { + return HadoopUser.isSecurityEnabled(); + } + } + + /** + * Returns whether or not secure authentication is enabled for HBase. Note that + * HBase security requires HDFS security to provide any guarantees, so this requires that + * both hbase.security.authentication and hadoop.security.authentication + * are set to kerberos. + */ + public static boolean isHBaseSecurityEnabled(Configuration conf) { + return "kerberos".equalsIgnoreCase(conf.get(HBASE_SECURITY_CONF_KEY)) && + "kerberos".equalsIgnoreCase( + conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION)); + } + + /* Concrete implementations */ + + /** + * Bridges {@link User} calls to invocations of the appropriate methods + * in {@link org.apache.hadoop.security.UserGroupInformation} in regular + * Hadoop 0.20 (ASF Hadoop and other versions without the backported security + * features). + */ + private static class HadoopUser extends User { + + private HadoopUser() { + try { + ugi = (UserGroupInformation) callStatic("getCurrentUGI"); + if (ugi == null) { + // Secure Hadoop UGI will perform an implicit login if the current + // user is null. Emulate the same behavior here for consistency + Configuration conf = HBaseConfiguration.create(); + ugi = (UserGroupInformation) callStatic("login", + new Class[]{ Configuration.class }, new Object[]{ conf }); + if (ugi != null) { + callStatic("setCurrentUser", + new Class[]{ UserGroupInformation.class }, new Object[]{ ugi }); + } + } + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected exception HadoopUser"); + } + } + + private HadoopUser(UserGroupInformation ugi) { + this.ugi = ugi; + } + + @Override + public String getShortName() { + return ugi != null ? ugi.getUserName() : null; + } + + @Override + public T runAs(PrivilegedAction action) { + T result = null; + UserGroupInformation previous = null; + try { + previous = (UserGroupInformation) callStatic("getCurrentUGI"); + try { + if (ugi != null) { + callStatic("setCurrentUser", new Class[]{UserGroupInformation.class}, + new Object[]{ugi}); + } + result = action.run(); + } finally { + callStatic("setCurrentUser", new Class[]{UserGroupInformation.class}, + new Object[]{previous}); + } + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected exception in runAs()"); + } + return result; + } + + @Override + public T runAs(PrivilegedExceptionAction action) + throws IOException, InterruptedException { + T result = null; + try { + UserGroupInformation previous = + (UserGroupInformation) callStatic("getCurrentUGI"); + try { + if (ugi != null) { + callStatic("setCurrentUGI", new Class[]{UserGroupInformation.class}, + new Object[]{ugi}); + } + result = action.run(); + } finally { + callStatic("setCurrentUGI", new Class[]{UserGroupInformation.class}, + new Object[]{previous}); + } + } catch (Exception e) { + if (e instanceof IOException) { + throw (IOException)e; + } else if (e instanceof InterruptedException) { + throw (InterruptedException)e; + } else if (e instanceof RuntimeException) { + throw (RuntimeException)e; + } else { + throw new UndeclaredThrowableException(e, "Unknown exception in runAs()"); + } + } + return result; + } + + @Override + public void obtainAuthTokenForJob(Configuration conf, Job job) + throws IOException, InterruptedException { + // this is a no-op. token creation is only supported for kerberos + // authenticated clients + } + + @Override + public void obtainAuthTokenForJob(JobConf job) + throws IOException, InterruptedException { + // this is a no-op. token creation is only supported for kerberos + // authenticated clients + } + + /** @see User#createUserForTesting(org.apache.hadoop.conf.Configuration, String, String[]) */ + public static User createUserForTesting(Configuration conf, + String name, String[] groups) { + try { + Class c = Class.forName("org.apache.hadoop.security.UnixUserGroupInformation"); + Constructor constructor = c.getConstructor(String.class, String[].class); + if (constructor == null) { + throw new NullPointerException( + ); + } + UserGroupInformation newUser = + (UserGroupInformation)constructor.newInstance(name, groups); + // set user in configuration -- hack for regular hadoop + conf.set("hadoop.job.ugi", newUser.toString()); + return new HadoopUser(newUser); + } catch (ClassNotFoundException cnfe) { + throw new RuntimeException( + "UnixUserGroupInformation not found, is this secure Hadoop?", cnfe); + } catch (NoSuchMethodException nsme) { + throw new RuntimeException( + "No valid constructor found for UnixUserGroupInformation!", nsme); + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected exception instantiating new UnixUserGroupInformation"); + } + } + + /** + * No-op since we're running on a version of Hadoop that doesn't support + * logins. + * @see User#login(org.apache.hadoop.conf.Configuration, String, String, String) + */ + public static void login(Configuration conf, String fileConfKey, + String principalConfKey, String localhost) throws IOException { + LOG.info("Skipping login, not running on secure Hadoop"); + } + + /** Always returns {@code false}. */ + public static boolean isSecurityEnabled() { + return false; + } + } + + /** + * Bridges {@code User} invocations to underlying calls to + * {@link org.apache.hadoop.security.UserGroupInformation} for secure Hadoop + * 0.20 and versions 0.21 and above. + */ + private static class SecureHadoopUser extends User { + private String shortName; + + private SecureHadoopUser() throws IOException { + try { + ugi = (UserGroupInformation) callStatic("getCurrentUser"); + } catch (IOException ioe) { + throw ioe; + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected exception getting current secure user"); + } + } + + private SecureHadoopUser(UserGroupInformation ugi) { + this.ugi = ugi; + } + + @Override + public String getShortName() { + if (shortName != null) return shortName; + + try { + shortName = (String)call(ugi, "getShortUserName", null, null); + return shortName; + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected error getting user short name"); + } + } + + @Override + public T runAs(PrivilegedAction action) { + try { + return (T) call(ugi, "doAs", new Class[]{PrivilegedAction.class}, + new Object[]{action}); + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected exception in runAs()"); + } + } + + @Override + public T runAs(PrivilegedExceptionAction action) + throws IOException, InterruptedException { + try { + return (T) call(ugi, "doAs", + new Class[]{PrivilegedExceptionAction.class}, + new Object[]{action}); + } catch (IOException ioe) { + throw ioe; + } catch (InterruptedException ie) { + throw ie; + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected exception in runAs(PrivilegedExceptionAction)"); + } + } + + @Override + public void obtainAuthTokenForJob(Configuration conf, Job job) + throws IOException, InterruptedException { + try { + Class c = Class.forName( + "org.apache.hadoop.hbase.security.token.TokenUtil"); + Methods.call(c, null, "obtainTokenForJob", + new Class[]{Configuration.class, UserGroupInformation.class, + Job.class}, + new Object[]{conf, ugi, job}); + } catch (ClassNotFoundException cnfe) { + throw new RuntimeException("Failure loading TokenUtil class, " + +"is secure RPC available?", cnfe); + } catch (IOException ioe) { + throw ioe; + } catch (InterruptedException ie) { + throw ie; + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected error calling TokenUtil.obtainAndCacheToken()"); + } + } + + @Override + public void obtainAuthTokenForJob(JobConf job) + throws IOException, InterruptedException { + try { + Class c = Class.forName( + "org.apache.hadoop.hbase.security.token.TokenUtil"); + Methods.call(c, null, "obtainTokenForJob", + new Class[]{JobConf.class, UserGroupInformation.class}, + new Object[]{job, ugi}); + } catch (ClassNotFoundException cnfe) { + throw new RuntimeException("Failure loading TokenUtil class, " + +"is secure RPC available?", cnfe); + } catch (IOException ioe) { + throw ioe; + } catch (InterruptedException ie) { + throw ie; + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected error calling TokenUtil.obtainAndCacheToken()"); + } + } + + /** @see User#createUserForTesting(org.apache.hadoop.conf.Configuration, String, String[]) */ + public static User createUserForTesting(Configuration conf, + String name, String[] groups) { + try { + return new SecureHadoopUser( + (UserGroupInformation)callStatic("createUserForTesting", + new Class[]{String.class, String[].class}, + new Object[]{name, groups}) + ); + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Error creating secure test user"); + } + } + + /** + * Obtain credentials for the current process using the configured + * Kerberos keytab file and principal. + * @see User#login(org.apache.hadoop.conf.Configuration, String, String, String) + * + * @param conf the Configuration to use + * @param fileConfKey Configuration property key used to store the path + * to the keytab file + * @param principalConfKey Configuration property key used to store the + * principal name to login as + * @param localhost the local hostname + */ + public static void login(Configuration conf, String fileConfKey, + String principalConfKey, String localhost) throws IOException { + if (isSecurityEnabled()) { + // check for SecurityUtil class + try { + Class c = Class.forName("org.apache.hadoop.security.SecurityUtil"); + Class[] types = new Class[]{ + Configuration.class, String.class, String.class, String.class }; + Object[] args = new Object[]{ + conf, fileConfKey, principalConfKey, localhost }; + Methods.call(c, null, "login", types, args); + } catch (ClassNotFoundException cnfe) { + throw new RuntimeException("Unable to login using " + + "org.apache.hadoop.security.SecurityUtil.login(). SecurityUtil class " + + "was not found! Is this a version of secure Hadoop?", cnfe); + } catch (IOException ioe) { + throw ioe; + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unhandled exception in User.login()"); + } + } + } + + /** + * Returns the result of {@code UserGroupInformation.isSecurityEnabled()}. + */ + public static boolean isSecurityEnabled() { + try { + return (Boolean)callStatic("isSecurityEnabled"); + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected exception calling UserGroupInformation.isSecurityEnabled()"); + } + } + } + + /* Reflection helper methods */ + private static Object callStatic(String methodName) throws Exception { + return call(null, methodName, null, null); + } + + private static Object callStatic(String methodName, Class[] types, + Object[] args) throws Exception { + return call(null, methodName, types, args); + } + + private static Object call(UserGroupInformation instance, String methodName, + Class[] types, Object[] args) throws Exception { + return Methods.call(UserGroupInformation.class, instance, methodName, types, + args); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/CopyRecoveredEditsTask.java b/src/main/java/org/apache/hadoop/hbase/snapshot/CopyRecoveredEditsTask.java new file mode 100644 index 0000000..88a2e66 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/CopyRecoveredEditsTask.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import java.io.IOException; +import java.util.NavigableSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.wal.HLog; + +/** + * Copy over each of the files in a region's recovered.edits directory to the region's snapshot + * directory. + *

    + * This is a serial operation over each of the files in the recovered.edits directory and also + * streams all the bytes to the client and then back to the filesystem, so the files being copied + * should be small or it will (a) suck up a lot of bandwidth, and (b) take a long time. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class CopyRecoveredEditsTask extends SnapshotTask { + + private static final Log LOG = LogFactory.getLog(CopyRecoveredEditsTask.class); + private final FileSystem fs; + private final Path regiondir; + private final Path outputDir; + + /** + * @param snapshot Snapshot being taken + * @param monitor error monitor for the snapshot + * @param fs {@link FileSystem} where the snapshot is being taken + * @param regionDir directory for the region to examine for edits + * @param snapshotRegionDir directory for the region in the snapshot + */ + public CopyRecoveredEditsTask(SnapshotDescription snapshot, ForeignExceptionDispatcher monitor, + FileSystem fs, Path regionDir, Path snapshotRegionDir) { + super(snapshot, monitor); + this.fs = fs; + this.regiondir = regionDir; + this.outputDir = HLog.getRegionDirRecoveredEditsDir(snapshotRegionDir); + } + + @Override + public Void call() throws IOException { + NavigableSet files = HLog.getSplitEditFilesSorted(this.fs, regiondir); + if (files == null || files.size() == 0) return null; + + // copy over each file. + // this is really inefficient (could be trivially parallelized), but is + // really simple to reason about. + for (Path source : files) { + // check to see if the file is zero length, in which case we can skip it + FileStatus stat = fs.getFileStatus(source); + if (stat.getLen() <= 0) continue; + + // its not zero length, so copy over the file + Path out = new Path(outputDir, source.getName()); + LOG.debug("Copying " + source + " to " + out); + FileUtil.copy(fs, source, fs, out, true, fs.getConf()); + + // check for errors to the running operation after each file + this.rethrowException(); + } + return null; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/CorruptedSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/CorruptedSnapshotException.java new file mode 100644 index 0000000..2e16c1b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/CorruptedSnapshotException.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; + + +/** + * Exception thrown when the found snapshot info from the filesystem is not valid + */ +@SuppressWarnings("serial") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class CorruptedSnapshotException extends HBaseSnapshotException { + + /** + * @param message message describing the exception + * @param e cause + */ + public CorruptedSnapshotException(String message, Exception e) { + super(message, e); + } + + /** + * Snapshot was corrupt for some reason + * @param message full description of the failure + * @param snapshot snapshot that was expected + */ + public CorruptedSnapshotException(String message, SnapshotDescription snapshot) { + super(message, snapshot); + } + + /** + * @param message message describing the exception + */ + public CorruptedSnapshotException(String message) { + super(message, (SnapshotDescription)null); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java new file mode 100644 index 0000000..2455831 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java @@ -0,0 +1,710 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.snapshot; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.io.HLogLink; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.snapshot.ExportSnapshotException; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +/** + * Export the specified snapshot to a given FileSystem. + * + * The .snapshot/name folder is copied to the destination cluster + * and then all the hfiles/hlogs are copied using a Map-Reduce Job in the .archive/ location. + * When everything is done, the second cluster can restore the snapshot. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public final class ExportSnapshot extends Configured implements Tool { + private static final Log LOG = LogFactory.getLog(ExportSnapshot.class); + + private static final String CONF_TMP_DIR = "hbase.tmp.dir"; + private static final String CONF_FILES_USER = "snapshot.export.files.attributes.user"; + private static final String CONF_FILES_GROUP = "snapshot.export.files.attributes.group"; + private static final String CONF_FILES_MODE = "snapshot.export.files.attributes.mode"; + private static final String CONF_CHECKSUM_VERIFY = "snapshot.export.checksum.verify"; + private static final String CONF_OUTPUT_ROOT = "snapshot.export.output.root"; + private static final String CONF_INPUT_ROOT = "snapshot.export.input.root"; + + private static final String INPUT_FOLDER_PREFIX = "export-files."; + + // Export Map-Reduce Counters, to keep track of the progress + public enum Counter { MISSING_FILES, COPY_FAILED, BYTES_EXPECTED, BYTES_COPIED }; + + private static class ExportMapper extends Mapper { + final static int REPORT_SIZE = 1 * 1024 * 1024; + final static int BUFFER_SIZE = 64 * 1024; + + private boolean verifyChecksum; + private String filesGroup; + private String filesUser; + private short filesMode; + + private FileSystem outputFs; + private Path outputArchive; + private Path outputRoot; + + private FileSystem inputFs; + private Path inputArchive; + private Path inputRoot; + + @Override + public void setup(Context context) { + Configuration conf = context.getConfiguration(); + verifyChecksum = conf.getBoolean(CONF_CHECKSUM_VERIFY, true); + + filesGroup = conf.get(CONF_FILES_GROUP); + filesUser = conf.get(CONF_FILES_USER); + filesMode = (short)conf.getInt(CONF_FILES_MODE, 0); + outputRoot = new Path(conf.get(CONF_OUTPUT_ROOT)); + inputRoot = new Path(conf.get(CONF_INPUT_ROOT)); + + inputArchive = new Path(inputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY); + outputArchive = new Path(outputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY); + + try { + inputFs = FileSystem.get(inputRoot.toUri(), conf); + } catch (IOException e) { + throw new RuntimeException("Could not get the input FileSystem with root=" + inputRoot, e); + } + + try { + outputFs = FileSystem.get(outputRoot.toUri(), conf); + } catch (IOException e) { + throw new RuntimeException("Could not get the output FileSystem with root="+ outputRoot, e); + } + } + + @Override + public void map(Text key, NullWritable value, Context context) + throws InterruptedException, IOException { + Path inputPath = new Path(key.toString()); + Path outputPath = getOutputPath(inputPath); + + LOG.info("copy file input=" + inputPath + " output=" + outputPath); + if (copyFile(context, inputPath, outputPath)) { + LOG.info("copy completed for input=" + inputPath + " output=" + outputPath); + } + } + + /** + * Returns the location where the inputPath will be copied. + * - hfiles are encoded as hfile links hfile-region-table + * - logs are encoded as serverName/logName + */ + private Path getOutputPath(final Path inputPath) throws IOException { + Path path; + if (HFileLink.isHFileLink(inputPath) || StoreFile.isReference(inputPath)) { + String family = inputPath.getParent().getName(); + String table = HFileLink.getReferencedTableName(inputPath.getName()); + String region = HFileLink.getReferencedRegionName(inputPath.getName()); + String hfile = HFileLink.getReferencedHFileName(inputPath.getName()); + path = new Path(table, new Path(region, new Path(family, hfile))); + } else if (isHLogLinkPath(inputPath)) { + String logName = inputPath.getName(); + path = new Path(new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME), logName); + } else { + path = inputPath; + } + return new Path(outputArchive, path); + } + + private boolean copyFile(final Context context, final Path inputPath, final Path outputPath) + throws IOException { + FSDataInputStream in = openSourceFile(inputPath); + if (in == null) { + context.getCounter(Counter.MISSING_FILES).increment(1); + return false; + } + + try { + // Verify if the input file exists + FileStatus inputStat = getFileStatus(inputFs, inputPath); + if (inputStat == null) return false; + + // Verify if the output file exists and is the same that we want to copy + if (outputFs.exists(outputPath)) { + FileStatus outputStat = outputFs.getFileStatus(outputPath); + if (sameFile(inputStat, outputStat)) { + LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file."); + return true; + } + } + + context.getCounter(Counter.BYTES_EXPECTED).increment(inputStat.getLen()); + + // Ensure that the output folder is there and copy the file + outputFs.mkdirs(outputPath.getParent()); + FSDataOutputStream out = outputFs.create(outputPath, true); + try { + if (!copyData(context, inputPath, in, outputPath, out, inputStat.getLen())) + return false; + } finally { + out.close(); + } + + // Preserve attributes + return preserveAttributes(outputPath, inputStat); + } finally { + in.close(); + } + } + + /** + * Preserve the files attribute selected by the user copying them from the source file + */ + private boolean preserveAttributes(final Path path, final FileStatus refStat) { + FileStatus stat; + try { + stat = outputFs.getFileStatus(path); + } catch (IOException e) { + LOG.warn("Unable to get the status for file=" + path); + return false; + } + + try { + if (filesMode > 0 && stat.getPermission().toShort() != filesMode) { + outputFs.setPermission(path, new FsPermission(filesMode)); + } else if (!stat.getPermission().equals(refStat.getPermission())) { + outputFs.setPermission(path, refStat.getPermission()); + } + } catch (IOException e) { + LOG.error("Unable to set the permission for file=" + path, e); + return false; + } + + try { + String user = (filesUser != null) ? filesUser : refStat.getOwner(); + String group = (filesGroup != null) ? filesGroup : refStat.getGroup(); + if (!(user.equals(stat.getOwner()) && group.equals(stat.getGroup()))) { + outputFs.setOwner(path, user, group); + } + } catch (IOException e) { + LOG.error("Unable to set the owner/group for file=" + path, e); + return false; + } + + return true; + } + + private boolean copyData(final Context context, + final Path inputPath, final FSDataInputStream in, + final Path outputPath, final FSDataOutputStream out, + final long inputFileSize) { + final String statusMessage = "copied %s/" + StringUtils.humanReadableInt(inputFileSize) + + " (%.3f%%) from " + inputPath + " to " + outputPath; + + try { + byte[] buffer = new byte[BUFFER_SIZE]; + long totalBytesWritten = 0; + int reportBytes = 0; + int bytesRead; + + while ((bytesRead = in.read(buffer)) > 0) { + out.write(buffer, 0, bytesRead); + totalBytesWritten += bytesRead; + reportBytes += bytesRead; + + if (reportBytes >= REPORT_SIZE) { + context.getCounter(Counter.BYTES_COPIED).increment(reportBytes); + context.setStatus(String.format(statusMessage, + StringUtils.humanReadableInt(totalBytesWritten), + reportBytes/(float)inputFileSize)); + reportBytes = 0; + } + } + + context.getCounter(Counter.BYTES_COPIED).increment(reportBytes); + context.setStatus(String.format(statusMessage, + StringUtils.humanReadableInt(totalBytesWritten), + reportBytes/(float)inputFileSize)); + + // Verify that the written size match + if (totalBytesWritten != inputFileSize) { + LOG.error("number of bytes copied not matching copied=" + totalBytesWritten + + " expected=" + inputFileSize + " for file=" + inputPath); + context.getCounter(Counter.COPY_FAILED).increment(1); + return false; + } + + return true; + } catch (IOException e) { + LOG.error("Error copying " + inputPath + " to " + outputPath, e); + context.getCounter(Counter.COPY_FAILED).increment(1); + return false; + } + } + + private FSDataInputStream openSourceFile(final Path path) { + try { + if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) { + return new HFileLink(inputRoot, inputArchive, path).open(inputFs); + } else if (isHLogLinkPath(path)) { + String serverName = path.getParent().getName(); + String logName = path.getName(); + return new HLogLink(inputRoot, serverName, logName).open(inputFs); + } + return inputFs.open(path); + } catch (IOException e) { + LOG.error("Unable to open source file=" + path, e); + return null; + } + } + + private FileStatus getFileStatus(final FileSystem fs, final Path path) { + try { + if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) { + HFileLink link = new HFileLink(inputRoot, inputArchive, path); + return link.getFileStatus(fs); + } else if (isHLogLinkPath(path)) { + String serverName = path.getParent().getName(); + String logName = path.getName(); + return new HLogLink(inputRoot, serverName, logName).getFileStatus(fs); + } + return fs.getFileStatus(path); + } catch (IOException e) { + LOG.warn("Unable to get the status for file=" + path); + return null; + } + } + + private FileChecksum getFileChecksum(final FileSystem fs, final Path path) { + try { + return fs.getFileChecksum(path); + } catch (IOException e) { + LOG.warn("Unable to get checksum for file=" + path, e); + return null; + } + } + + /** + * Check if the two files are equal by looking at the file length, + * and at the checksum (if user has specified the verifyChecksum flag). + */ + private boolean sameFile(final FileStatus inputStat, final FileStatus outputStat) { + // Not matching length + if (inputStat.getLen() != outputStat.getLen()) return false; + + // Mark files as equals, since user asked for no checksum verification + if (!verifyChecksum) return true; + + // If checksums are not available, files are not the same. + FileChecksum inChecksum = getFileChecksum(inputFs, inputStat.getPath()); + if (inChecksum == null) return false; + + FileChecksum outChecksum = getFileChecksum(outputFs, outputStat.getPath()); + if (outChecksum == null) return false; + + return inChecksum.equals(outChecksum); + } + + /** + * HLog files are encoded as serverName/logName + * and since all the other files should be in /hbase/table/..path.. + * we can rely on the depth, for now. + */ + private static boolean isHLogLinkPath(final Path path) { + return path.depth() == 2; + } + } + + /** + * Extract the list of files (HFiles/HLogs) to copy using Map-Reduce. + * @return list of files referenced by the snapshot (pair of path and size) + */ + private List> getSnapshotFiles(final FileSystem fs, final Path snapshotDir) + throws IOException { + SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); + + final List> files = new ArrayList>(); + final String table = snapshotDesc.getTable(); + final Configuration conf = getConf(); + + // Get snapshot files + SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir, + new SnapshotReferenceUtil.FileVisitor() { + public void storeFile (final String region, final String family, final String hfile) + throws IOException { + Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile)); + long size = new HFileLink(conf, path).getFileStatus(fs).getLen(); + files.add(new Pair(path, size)); + } + + public void recoveredEdits (final String region, final String logfile) + throws IOException { + // copied with the snapshot referenecs + } + + public void logFile (final String server, final String logfile) + throws IOException { + long size = new HLogLink(conf, server, logfile).getFileStatus(fs).getLen(); + files.add(new Pair(new Path(server, logfile), size)); + } + }); + + return files; + } + + /** + * Given a list of file paths and sizes, create around ngroups in as balanced a way as possible. + * The groups created will have similar amounts of bytes. + *

    + * The algorithm used is pretty straightforward; the file list is sorted by size, + * and then each group fetch the bigger file available, iterating through groups + * alternating the direction. + */ + static List> getBalancedSplits(final List> files, int ngroups) { + // Sort files by size, from small to big + Collections.sort(files, new Comparator>() { + public int compare(Pair a, Pair b) { + long r = a.getSecond() - b.getSecond(); + return (r < 0) ? -1 : ((r > 0) ? 1 : 0); + } + }); + + // create balanced groups + List> fileGroups = new LinkedList>(); + long[] sizeGroups = new long[ngroups]; + int hi = files.size() - 1; + int lo = 0; + + List group; + int dir = 1; + int g = 0; + + while (hi >= lo) { + if (g == fileGroups.size()) { + group = new LinkedList(); + fileGroups.add(group); + } else { + group = fileGroups.get(g); + } + + Pair fileInfo = files.get(hi--); + + // add the hi one + sizeGroups[g] += fileInfo.getSecond(); + group.add(fileInfo.getFirst()); + + // change direction when at the end or the beginning + g += dir; + if (g == ngroups) { + dir = -1; + g = ngroups - 1; + } else if (g < 0) { + dir = 1; + g = 0; + } + } + + if (LOG.isDebugEnabled()) { + for (int i = 0; i < sizeGroups.length; ++i) { + LOG.debug("export split=" + i + " size=" + StringUtils.humanReadableInt(sizeGroups[i])); + } + } + + return fileGroups; + } + + private static Path getInputFolderPath(final FileSystem fs, final Configuration conf) + throws IOException, InterruptedException { + String stagingName = "exportSnapshot-" + EnvironmentEdgeManager.currentTimeMillis(); + Path stagingDir = new Path(conf.get(CONF_TMP_DIR), stagingName); + fs.mkdirs(stagingDir); + return new Path(stagingDir, INPUT_FOLDER_PREFIX + + String.valueOf(EnvironmentEdgeManager.currentTimeMillis())); + } + + /** + * Create the input files, with the path to copy, for the MR job. + * Each input files contains n files, and each input file has a similar amount data to copy. + * The number of input files created are based on the number of mappers provided as argument + * and the number of the files to copy. + */ + private static Path[] createInputFiles(final Configuration conf, + final List> snapshotFiles, int mappers) + throws IOException, InterruptedException { + FileSystem fs = FileSystem.get(conf); + Path inputFolderPath = getInputFolderPath(fs, conf); + LOG.debug("Input folder location: " + inputFolderPath); + + List> splits = getBalancedSplits(snapshotFiles, mappers); + Path[] inputFiles = new Path[splits.size()]; + + Text key = new Text(); + for (int i = 0; i < inputFiles.length; i++) { + List files = splits.get(i); + inputFiles[i] = new Path(inputFolderPath, String.format("export-%d.seq", i)); + SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inputFiles[i], + Text.class, NullWritable.class); + LOG.debug("Input split: " + i); + try { + for (Path file: files) { + LOG.debug(file.toString()); + key.set(file.toString()); + writer.append(key, NullWritable.get()); + } + } finally { + writer.close(); + } + } + + return inputFiles; + } + + /** + * Run Map-Reduce Job to perform the files copy. + */ + private boolean runCopyJob(final Path inputRoot, final Path outputRoot, + final List> snapshotFiles, final boolean verifyChecksum, + final String filesUser, final String filesGroup, final int filesMode, + final int mappers) throws IOException, InterruptedException, ClassNotFoundException { + Configuration conf = getConf(); + if (filesGroup != null) conf.set(CONF_FILES_GROUP, filesGroup); + if (filesUser != null) conf.set(CONF_FILES_USER, filesUser); + conf.setInt(CONF_FILES_MODE, filesMode); + conf.setBoolean(CONF_CHECKSUM_VERIFY, verifyChecksum); + conf.set(CONF_OUTPUT_ROOT, outputRoot.toString()); + conf.set(CONF_INPUT_ROOT, inputRoot.toString()); + conf.setInt("mapreduce.job.maps", mappers); + + // job.setMapSpeculativeExecution(false) + conf.setBoolean("mapreduce.map.speculative", false); + conf.setBoolean("mapreduce.reduce.speculative", false); + conf.setBoolean("mapred.map.tasks.speculative.execution", false); + conf.setBoolean("mapred.reduce.tasks.speculative.execution", false); + + Job job = new Job(conf); + job.setJobName("ExportSnapshot"); + job.setJarByClass(ExportSnapshot.class); + job.setMapperClass(ExportMapper.class); + job.setInputFormatClass(SequenceFileInputFormat.class); + job.setOutputFormatClass(NullOutputFormat.class); + job.setNumReduceTasks(0); + for (Path path: createInputFiles(conf, snapshotFiles, mappers)) { + LOG.debug("Add Input Path=" + path); + SequenceFileInputFormat.addInputPath(job, path); + } + + return job.waitForCompletion(true); + } + + /** + * Execute the export snapshot by copying the snapshot metadata, hfiles and hlogs. + * @return 0 on success, and != 0 upon failure. + */ + @Override + public int run(String[] args) throws Exception { + boolean verifyChecksum = true; + String snapshotName = null; + String filesGroup = null; + String filesUser = null; + Path outputRoot = null; + int filesMode = 0; + int mappers = getConf().getInt("mapreduce.job.maps", 1); + + // Process command line args + for (int i = 0; i < args.length; i++) { + String cmd = args[i]; + try { + if (cmd.equals("-snapshot")) { + snapshotName = args[++i]; + } else if (cmd.equals("-copy-to")) { + outputRoot = new Path(args[++i]); + } else if (cmd.equals("-no-checksum-verify")) { + verifyChecksum = false; + } else if (cmd.equals("-mappers")) { + mappers = Integer.parseInt(args[++i]); + } else if (cmd.equals("-chuser")) { + filesUser = args[++i]; + } else if (cmd.equals("-chgroup")) { + filesGroup = args[++i]; + } else if (cmd.equals("-chmod")) { + filesMode = Integer.parseInt(args[++i], 8); + } else if (cmd.equals("-h") || cmd.equals("--help")) { + printUsageAndExit(); + } else { + System.err.println("UNEXPECTED: " + cmd); + printUsageAndExit(); + } + } catch (Exception e) { + printUsageAndExit(); + } + } + + // Check user options + if (snapshotName == null) { + System.err.println("Snapshot name not provided."); + printUsageAndExit(); + } + + if (outputRoot == null) { + System.err.println("Destination file-system not provided."); + printUsageAndExit(); + } + + Configuration conf = getConf(); + Path inputRoot = FSUtils.getRootDir(conf); + FileSystem inputFs = FileSystem.get(conf); + FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf); + + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot); + Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot); + Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot); + + // Check if the snapshot already exists + if (outputFs.exists(outputSnapshotDir)) { + System.err.println("The snapshot '" + snapshotName + + "' already exists in the destination: " + outputSnapshotDir); + return 1; + } + + // Check if the snapshot already in-progress + if (outputFs.exists(snapshotTmpDir)) { + System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress"); + System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, "); + System.err.println("consider removing " + snapshotTmpDir + " before retrying export"); + return 1; + } + + // Step 0 - Extract snapshot files to copy + final List> files = getSnapshotFiles(inputFs, snapshotDir); + + // Step 1 - Copy fs1:/.snapshot/ to fs2:/.snapshot/.tmp/ + // The snapshot references must be copied before the hfiles otherwise the cleaner + // will remove them because they are unreferenced. + try { + FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf); + } catch (IOException e) { + System.err.println("Failed to copy the snapshot directory: from=" + snapshotDir + + " to=" + snapshotTmpDir); + e.printStackTrace(System.err); + return 1; + } + + // Step 2 - Start MR Job to copy files + // The snapshot references must be copied before the files otherwise the files gets removed + // by the HFileArchiver, since they have no references. + try { + if (files.size() == 0) { + LOG.warn("There are 0 store file to be copied. There may be no data in the table."); + } else { + if (!runCopyJob(inputRoot, outputRoot, files, verifyChecksum, + filesUser, filesGroup, filesMode, mappers)) { + throw new ExportSnapshotException("Snapshot export failed!"); + } + } + + // Step 3 - Rename fs2:/.snapshot/.tmp/ fs2:/.snapshot/ + if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) { + System.err.println("Snapshot export failed!"); + System.err.println("Unable to rename snapshot directory from=" + + snapshotTmpDir + " to=" + outputSnapshotDir); + return 1; + } + + return 0; + } catch (Exception e) { + System.err.println("Snapshot export failed!"); + e.printStackTrace(System.err); + outputFs.delete(outputSnapshotDir, true); + return 1; + } + } + + // ExportSnapshot + private void printUsageAndExit() { + System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName()); + System.err.println(" where [options] are:"); + System.err.println(" -h|-help Show this help and exit."); + System.err.println(" -snapshot NAME Snapshot to restore."); + System.err.println(" -copy-to NAME Remote destination hdfs://"); + System.err.println(" -no-checksum-verify Do not verify checksum."); + System.err.println(" -chuser USERNAME Change the owner of the files to the specified one."); + System.err.println(" -chgroup GROUP Change the group of the files to the specified one."); + System.err.println(" -chmod MODE Change the permission of the files to the specified one."); + System.err.println(" -mappers Number of mappers to use during the copy (mapreduce.job.maps)."); + System.err.println(); + System.err.println("Examples:"); + System.err.println(" hbase " + getClass() + " \\"); + System.err.println(" -snapshot MySnapshot -copy-to hdfs:///srv2:8082/hbase \\"); + System.err.println(" -chuser MyUser -chgroup MyGroup -chmod 700 -mappers 16"); + System.exit(1); + } + + /** + * The guts of the {@link #main} method. + * Call this method to avoid the {@link #main(String[])} System.exit. + * @param args + * @return errCode + * @throws Exception + */ + static int innerMain(final Configuration conf, final String [] args) throws Exception { + return ToolRunner.run(conf, new ExportSnapshot(), args); + } + + public static void main(String[] args) throws Exception { + System.exit(innerMain(HBaseConfiguration.create(), args)); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotException.java new file mode 100644 index 0000000..76a83cd --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotException.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Thrown when a snapshot could not be exported due to an error during the operation. + */ +@InterfaceAudience.Public +@SuppressWarnings("serial") +public class ExportSnapshotException extends HBaseSnapshotException { + + /** + * @param msg message describing the exception + */ + public ExportSnapshotException(String msg) { + super(msg); + } + + /** + * @param message message describing the exception + * @param e cause + */ + public ExportSnapshotException(String message, Exception e) { + super(message, e); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java new file mode 100644 index 0000000..70a8842 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; + +/** + * General exception base class for when a snapshot fails + */ +@SuppressWarnings("serial") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class HBaseSnapshotException extends HBaseIOException { + + private SnapshotDescription description; + + /** + * Some exception happened for a snapshot and don't even know the snapshot that it was about + * @param msg Full description of the failure + */ + public HBaseSnapshotException(String msg) { + super(msg); + } + + /** + * Exception for the given snapshot that has no previous root cause + * @param msg reason why the snapshot failed + * @param desc description of the snapshot that is being failed + */ + public HBaseSnapshotException(String msg, SnapshotDescription desc) { + super(msg); + this.description = desc; + } + + /** + * Exception for the given snapshot due to another exception + * @param msg reason why the snapshot failed + * @param cause root cause of the failure + * @param desc description of the snapshot that is being failed + */ + public HBaseSnapshotException(String msg, Throwable cause, SnapshotDescription desc) { + super(msg, cause); + this.description = desc; + } + + /** + * Exception when the description of the snapshot cannot be determined, due to some root other + * root cause + * @param message description of what caused the failure + * @param e root cause + */ + public HBaseSnapshotException(String message, Exception e) { + super(message, e); + } + + public SnapshotDescription getSnapshotDescription() { + return this.description; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/HSnapshotDescription.java b/src/main/java/org/apache/hadoop/hbase/snapshot/HSnapshotDescription.java new file mode 100644 index 0000000..1e321b7 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/HSnapshotDescription.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import java.io.IOException; +import java.io.DataInput; +import java.io.DataOutput; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Writable version of the SnapshotDescription used by the rpc + */ +public class HSnapshotDescription implements Writable { + private SnapshotDescription proto; + + public HSnapshotDescription() { + } + + public HSnapshotDescription(final SnapshotDescription proto) { + assert proto != null : "proto must be non-null"; + this.proto = proto; + } + + public String getName() { + return this.proto.getName(); + } + + public SnapshotDescription getProto() { + return this.proto; + } + + public SnapshotDescription.Type getType() { + return this.proto.getType(); + } + + public String getTable() { + return this.proto.getTable(); + } + + public boolean hasTable() { + return this.proto.hasTable(); + } + + public long getCreationTime() { + return this.proto.getCreationTime(); + } + + public int getVersion() { + return this.proto.getVersion(); + } + + public String toString() { + if (this.proto != null) { + return this.proto.toString(); + } + return "(no snapshot)"; + } + + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof HSnapshotDescription)) { + return false; + } + SnapshotDescription oproto = ((HSnapshotDescription)obj).getProto(); + if (this.proto == oproto) { + return true; + } + if (this.proto == null && oproto != null) { + return false; + } + return this.proto.equals(oproto); + } + + // Writable + /** + * INTERNAL This method is a part of {@link Writable} interface + * and is used for de-serialization of the HTableDescriptor over RPC + */ + @Override + public void readFields(DataInput in) throws IOException { + byte[] data = Bytes.readByteArray(in); + if (data.length > 0) { + this.proto = SnapshotDescription.parseFrom(data); + } else { + this.proto = null; + } + } + + /** + * INTERNAL This method is a part of {@link Writable} interface + * and is used for serialization of the HTableDescriptor over RPC + */ + @Override + public void write(DataOutput out) throws IOException { + if (this.proto != null) { + Bytes.writeByteArray(out, this.proto.toByteArray()); + } else { + Bytes.writeByteArray(out, new byte[0]); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceRegionHFilesTask.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceRegionHFilesTask.java new file mode 100644 index 0000000..60d48d9 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceRegionHFilesTask.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.util.FSUtils; + +/** + * Reference all the hfiles in a region for a snapshot. + *

    + * Doesn't take into acccount if the hfiles are valid or not, just keeps track of what's in the + * region's directory. + */ +public class ReferenceRegionHFilesTask extends SnapshotTask { + + public static final Log LOG = LogFactory.getLog(ReferenceRegionHFilesTask.class); + private final Path regiondir; + private final FileSystem fs; + private final PathFilter fileFilter; + private final Path snapshotDir; + + /** + * Reference all the files in the given region directory + * @param snapshot snapshot for which to add references + * @param monitor to check/send error + * @param regionDir region directory to look for errors + * @param fs {@link FileSystem} where the snapshot/region live + * @param regionSnapshotDir directory in the snapshot to store region files + */ + public ReferenceRegionHFilesTask(final SnapshotDescription snapshot, + ForeignExceptionDispatcher monitor, Path regionDir, final FileSystem fs, Path regionSnapshotDir) { + super(snapshot, monitor); + this.regiondir = regionDir; + this.fs = fs; + + this.fileFilter = new PathFilter() { + @Override + public boolean accept(Path path) { + try { + return fs.isFile(path); + } catch (IOException e) { + LOG.error("Failed to reach fs to check file:" + path + ", marking as not file"); + ReferenceRegionHFilesTask.this.snapshotFailure("Failed to reach fs to check file status", + e); + return false; + } + } + }; + this.snapshotDir = regionSnapshotDir; + } + + @Override + public Void call() throws IOException { + FileStatus[] families = FSUtils.listStatus(fs, regiondir, new FSUtils.FamilyDirFilter(fs)); + + // if no families, then we are done again + if (families == null || families.length == 0) { + LOG.info("No families under region directory:" + regiondir + + ", not attempting to add references."); + return null; + } + + // snapshot directories to store the hfile reference + List snapshotFamilyDirs = TakeSnapshotUtils.getFamilySnapshotDirectories(snapshot, + snapshotDir, families); + + LOG.debug("Add hfile references to snapshot directories:" + snapshotFamilyDirs); + for (int i = 0; i < families.length; i++) { + FileStatus family = families[i]; + Path familyDir = family.getPath(); + // get all the hfiles in the family + FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir, fileFilter); + + // if no hfiles, then we are done with this family + if (hfiles == null || hfiles.length == 0) { + LOG.debug("Not hfiles found for family: " + familyDir + ", skipping."); + continue; + } + + // make the snapshot's family directory + Path snapshotFamilyDir = snapshotFamilyDirs.get(i); + fs.mkdirs(snapshotFamilyDir); + + // create a reference for each hfile + for (FileStatus hfile : hfiles) { + // references are 0-length files, relying on file name. + Path referenceFile = new Path(snapshotFamilyDir, hfile.getPath().getName()); + LOG.debug("Creating reference for:" + hfile.getPath() + " at " + referenceFile); + if (!fs.createNewFile(referenceFile)) { + throw new IOException("Failed to create reference file:" + referenceFile); + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Finished referencing hfiles, current region state:"); + FSUtils.logFileSystemState(fs, regiondir, LOG); + LOG.debug("and the snapshot directory:"); + FSUtils.logFileSystemState(fs, snapshotDir, LOG); + } + return null; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceServerWALsTask.java b/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceServerWALsTask.java new file mode 100644 index 0000000..9c987ab --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/ReferenceServerWALsTask.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.errorhandling.ForeignException; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.util.FSUtils; + +/** + * Reference all the WAL files under a server's WAL directory + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class ReferenceServerWALsTask extends SnapshotTask { + private static final Log LOG = LogFactory.getLog(ReferenceServerWALsTask.class); + private final FileSystem fs; + private final Configuration conf; + private final String serverName; + private Path logDir; + + /** + * @param snapshot snapshot being run + * @param failureListener listener to check for errors while running the operation and to + * propagate errors found while running the task + * @param logDir log directory for the server. Name of the directory is taken as the name of the + * server + * @param conf {@link Configuration} to extract filesystem information + * @param fs filesystem where the log files are stored and should be referenced + */ + public ReferenceServerWALsTask(SnapshotDescription snapshot, + ForeignExceptionDispatcher failureListener, final Path logDir, final Configuration conf, + final FileSystem fs) { + super(snapshot, failureListener); + this.fs = fs; + this.conf = conf; + this.serverName = logDir.getName(); + this.logDir = logDir; + } + + /** + * Create reference files (empty files with the same path and file name as original). + * @throws IOException exception from hdfs or network problems + * @throws ForeignException exception from an external procedure + */ + @Override + public Void call() throws IOException, ForeignException { + // TODO switch to using a single file to reference all required WAL files + + // Iterate through each of the log files and add a reference to it. + // assumes that all the files under the server's logs directory is a log + FileStatus[] serverLogs = FSUtils.listStatus(fs, logDir, null); + if (serverLogs == null) { + LOG.debug("No logs for server directory:" + logDir + ", done referencing files."); + return null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Adding references for WAL files:" + Arrays.toString(serverLogs)); + } + + for (FileStatus file : serverLogs) { + this.rethrowException(); + + // add the reference to the file. ex: hbase/.snapshots/.logs// + Path rootDir = FSUtils.getRootDir(conf); + Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(this.snapshot, rootDir); + Path snapshotLogDir = TakeSnapshotUtils.getSnapshotHLogsDir(snapshotDir, serverName); + // actually store the reference on disk (small file) + Path ref = new Path(snapshotLogDir, file.getPath().getName()); + if (!fs.createNewFile(ref)) { + if (!fs.exists(ref)) { + throw new IOException("Couldn't create reference for:" + file.getPath()); + } + } + LOG.debug("Completed WAL referencing for: " + file.getPath() + " to " + ref); + } + + LOG.debug("Successfully completed WAL referencing for ALL files"); + return null; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java new file mode 100644 index 0000000..ff40783 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; + +/** + * Thrown when a snapshot could not be restored due to a server-side error when restoring it. + */ +@SuppressWarnings("serial") +public class RestoreSnapshotException extends HBaseSnapshotException { + public RestoreSnapshotException(String msg, SnapshotDescription desc) { + super(msg, desc); + } + + public RestoreSnapshotException(String msg, Throwable cause, SnapshotDescription desc) { + super(msg, cause, desc); + } + + public RestoreSnapshotException(String msg) { + super(msg); + } + + public RestoreSnapshotException(String message, Exception e) { + super(message, e); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java new file mode 100644 index 0000000..5b92060 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -0,0 +1,599 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.snapshot; + +import java.io.InputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.backup.HFileArchiver; +import org.apache.hadoop.hbase.monitoring.MonitoredTask; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.FSVisitor; +import org.apache.hadoop.hbase.util.ModifyRegionUtils; +import org.apache.hadoop.io.IOUtils; + +/** + * Helper to Restore/Clone a Snapshot + * + *

    The helper assumes that a table is already created, and by calling restore() + * the content present in the snapshot will be restored as the new content of the table. + * + *

    Clone from Snapshot: If the target table is empty, the restore operation + * is just a "clone operation", where the only operations are: + *

      + *
    • for each region in the snapshot create a new region + * (note that the region will have a different name, since the encoding contains the table name) + *
    • for each file in the region create a new HFileLink to point to the original file. + *
    • restore the logs, if any + *
    + * + *

    Restore from Snapshot: + *

      + *
    • for each region in the table verify which are available in the snapshot and which are not + *
        + *
      • if the region is not present in the snapshot, remove it. + *
      • if the region is present in the snapshot + *
          + *
        • for each file in the table region verify which are available in the snapshot + *
            + *
          • if the hfile is not present in the snapshot, remove it + *
          • if the hfile is present, keep it (nothing to do) + *
          + *
        • for each file in the snapshot region but not in the table + *
            + *
          • create a new HFileLink that point to the original file + *
          + *
        + *
      + *
    • for each region in the snapshot not present in the current table state + *
        + *
      • create a new region and for each file in the region create a new HFileLink + * (This is the same as the clone operation) + *
      + *
    • restore the logs, if any + *
    + */ +@InterfaceAudience.Private +public class RestoreSnapshotHelper { + private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class); + + private final Map regionsMap = + new TreeMap(Bytes.BYTES_COMPARATOR); + + private final ForeignExceptionDispatcher monitor; + private final MonitoredTask status; + + private final SnapshotDescription snapshotDesc; + private final Path snapshotDir; + + private final HTableDescriptor tableDesc; + private final Path tableDir; + + private final Configuration conf; + private final FileSystem fs; + + public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs, + final SnapshotDescription snapshotDescription, final Path snapshotDir, + final HTableDescriptor tableDescriptor, final Path tableDir, + final ForeignExceptionDispatcher monitor, final MonitoredTask status) + { + this.fs = fs; + this.conf = conf; + this.snapshotDesc = snapshotDescription; + this.snapshotDir = snapshotDir; + this.tableDesc = tableDescriptor; + this.tableDir = tableDir; + this.monitor = monitor; + this.status = status; + } + + /** + * Restore the on-disk table to a specified snapshot state. + * @return the set of regions touched by the restore operation + */ + public RestoreMetaChanges restoreHdfsRegions() throws IOException { + LOG.debug("starting restore"); + Set snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir); + if (snapshotRegionNames == null) { + LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty"); + return null; + } + + RestoreMetaChanges metaChanges = new RestoreMetaChanges(); + + // Identify which region are still available and which not. + // NOTE: we rely upon the region name as: "table name, start key, end key" + List tableRegions = getTableRegions(); + if (tableRegions != null) { + monitor.rethrowException(); + for (HRegionInfo regionInfo: tableRegions) { + String regionName = regionInfo.getEncodedName(); + if (snapshotRegionNames.contains(regionName)) { + LOG.info("region to restore: " + regionName); + snapshotRegionNames.remove(regionName); + metaChanges.addRegionToRestore(regionInfo); + } else { + LOG.info("region to remove: " + regionName); + metaChanges.addRegionToRemove(regionInfo); + } + } + + // Restore regions using the snapshot data + monitor.rethrowException(); + status.setStatus("Restoring table regions..."); + restoreHdfsRegions(metaChanges.getRegionsToRestore()); + status.setStatus("Finished restoring all table regions."); + + // Remove regions from the current table + monitor.rethrowException(); + status.setStatus("Starting to delete excess regions from table"); + removeHdfsRegions(metaChanges.getRegionsToRemove()); + status.setStatus("Finished deleting excess regions from table."); + } + + // Regions to Add: present in the snapshot but not in the current table + if (snapshotRegionNames.size() > 0) { + List regionsToAdd = new LinkedList(); + + monitor.rethrowException(); + for (String regionName: snapshotRegionNames) { + LOG.info("region to add: " + regionName); + Path regionDir = new Path(snapshotDir, regionName); + regionsToAdd.add(HRegion.loadDotRegionInfoFileContent(fs, regionDir)); + } + + // Create new regions cloning from the snapshot + monitor.rethrowException(); + status.setStatus("Cloning regions..."); + HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd); + metaChanges.setNewRegions(clonedRegions); + status.setStatus("Finished cloning regions."); + } + + // Restore WALs + monitor.rethrowException(); + status.setStatus("Restoring WALs to table..."); + restoreWALs(); + status.setStatus("Finished restoring WALs to table."); + + return metaChanges; + } + + /** + * Describe the set of operations needed to update META after restore. + */ + public static class RestoreMetaChanges { + private List regionsToRestore = null; + private List regionsToRemove = null; + private List regionsToAdd = null; + + /** + * @return true if there're new regions + */ + public boolean hasRegionsToAdd() { + return this.regionsToAdd != null && this.regionsToAdd.size() > 0; + } + + /** + * Returns the list of new regions added during the on-disk restore. + * The caller is responsible to add the regions to META. + * e.g MetaEditor.addRegionsToMeta(...) + * @return the list of regions to add to META + */ + public List getRegionsToAdd() { + return this.regionsToAdd; + } + + /** + * @return true if there're regions to restore + */ + public boolean hasRegionsToRestore() { + return this.regionsToRestore != null && this.regionsToRestore.size() > 0; + } + + /** + * Returns the list of 'restored regions' during the on-disk restore. + * The caller is responsible to add the regions to META if not present. + * @return the list of regions restored + */ + public List getRegionsToRestore() { + return this.regionsToRestore; + } + + /** + * @return true if there're regions to remove + */ + public boolean hasRegionsToRemove() { + return this.regionsToRemove != null && this.regionsToRemove.size() > 0; + } + + /** + * Returns the list of regions removed during the on-disk restore. + * The caller is responsible to remove the regions from META. + * e.g. MetaEditor.deleteRegions(...) + * @return the list of regions to remove from META + */ + public List getRegionsToRemove() { + return this.regionsToRemove; + } + + void setNewRegions(final HRegionInfo[] hris) { + if (hris != null) { + regionsToAdd = Arrays.asList(hris); + } else { + regionsToAdd = null; + } + } + + void addRegionToRemove(final HRegionInfo hri) { + if (regionsToRemove == null) { + regionsToRemove = new LinkedList(); + } + regionsToRemove.add(hri); + } + + void addRegionToRestore(final HRegionInfo hri) { + if (regionsToRestore == null) { + regionsToRestore = new LinkedList(); + } + regionsToRestore.add(hri); + } + } + + /** + * Remove specified regions from the file-system, using the archiver. + */ + private void removeHdfsRegions(final List regions) throws IOException { + if (regions != null && regions.size() > 0) { + for (HRegionInfo hri: regions) { + HFileArchiver.archiveRegion(conf, fs, hri); + } + } + } + + /** + * Restore specified regions by restoring content to the snapshot state. + */ + private void restoreHdfsRegions(final List regions) throws IOException { + if (regions == null || regions.size() == 0) return; + for (HRegionInfo hri: regions) restoreRegion(hri); + } + + /** + * Restore region by removing files not in the snapshot + * and adding the missing ones from the snapshot. + */ + private void restoreRegion(HRegionInfo regionInfo) throws IOException { + Path snapshotRegionDir = new Path(snapshotDir, regionInfo.getEncodedName()); + Map> snapshotFiles = + SnapshotReferenceUtil.getRegionHFileReferences(fs, snapshotRegionDir); + Path regionDir = new Path(tableDir, regionInfo.getEncodedName()); + String tableName = tableDesc.getNameAsString(); + + // Restore families present in the table + for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { + byte[] family = Bytes.toBytes(familyDir.getName()); + Set familyFiles = getTableRegionFamilyFiles(familyDir); + List snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName()); + if (snapshotFamilyFiles != null) { + List hfilesToAdd = new LinkedList(); + for (String hfileName: snapshotFamilyFiles) { + if (familyFiles.contains(hfileName)) { + // HFile already present + familyFiles.remove(hfileName); + } else { + // HFile missing + hfilesToAdd.add(hfileName); + } + } + + // Restore Missing files + for (String hfileName: hfilesToAdd) { + LOG.trace("Adding HFileLink " + hfileName + + " to region=" + regionInfo.getEncodedName() + " table=" + tableName); + restoreStoreFile(familyDir, regionInfo, hfileName); + } + + // Remove hfiles not present in the snapshot + for (String hfileName: familyFiles) { + Path hfile = new Path(familyDir, hfileName); + LOG.trace("Removing hfile=" + hfile + + " from region=" + regionInfo.getEncodedName() + " table=" + tableName); + HFileArchiver.archiveStoreFile(fs, regionInfo, conf, tableDir, family, hfile); + } + } else { + // Family doesn't exists in the snapshot + LOG.trace("Removing family=" + Bytes.toString(family) + + " from region=" + regionInfo.getEncodedName() + " table=" + tableName); + HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, family); + fs.delete(familyDir, true); + } + } + + // Add families not present in the table + for (Map.Entry> familyEntry: snapshotFiles.entrySet()) { + Path familyDir = new Path(regionDir, familyEntry.getKey()); + if (!fs.mkdirs(familyDir)) { + throw new IOException("Unable to create familyDir=" + familyDir); + } + + for (String hfileName: familyEntry.getValue()) { + LOG.trace("Adding HFileLink " + hfileName + " to table=" + tableName); + restoreStoreFile(familyDir, regionInfo, hfileName); + } + } + } + + /** + * @return The set of files in the specified family directory. + */ + private Set getTableRegionFamilyFiles(final Path familyDir) throws IOException { + Set familyFiles = new HashSet(); + + FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir); + if (hfiles == null) return familyFiles; + + for (FileStatus hfileRef: hfiles) { + String hfileName = hfileRef.getPath().getName(); + familyFiles.add(hfileName); + } + + return familyFiles; + } + + /** + * Clone specified regions. For each region create a new region + * and create a HFileLink for each hfile. + */ + private HRegionInfo[] cloneHdfsRegions(final List regions) throws IOException { + if (regions == null || regions.size() == 0) return null; + + final Map snapshotRegions = + new HashMap(regions.size()); + + // clone region info (change embedded tableName with the new one) + HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()]; + for (int i = 0; i < clonedRegionsInfo.length; ++i) { + // clone the region info from the snapshot region info + HRegionInfo snapshotRegionInfo = regions.get(i); + clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo); + + // add the region name mapping between snapshot and cloned + String snapshotRegionName = snapshotRegionInfo.getEncodedName(); + String clonedRegionName = clonedRegionsInfo[i].getEncodedName(); + regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName)); + LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName); + + // Add mapping between cloned region name and snapshot region info + snapshotRegions.put(clonedRegionName, snapshotRegionInfo); + } + + // create the regions on disk + ModifyRegionUtils.createRegions(conf, tableDir.getParent(), + tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() { + public void fillRegion(final HRegion region) throws IOException { + cloneRegion(region, snapshotRegions.get(region.getRegionInfo().getEncodedName())); + } + }); + + return clonedRegionsInfo; + } + + /** + * Clone region directory content from the snapshot info. + * + * Each region is encoded with the table name, so the cloned region will have + * a different region name. + * + * Instead of copying the hfiles a HFileLink is created. + * + * @param region {@link HRegion} cloned + * @param snapshotRegionInfo + */ + private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo) + throws IOException { + final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName()); + final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName()); + final String tableName = tableDesc.getNameAsString(); + SnapshotReferenceUtil.visitRegionStoreFiles(fs, snapshotRegionDir, + new FSVisitor.StoreFileVisitor() { + public void storeFile (final String region, final String family, final String hfile) + throws IOException { + LOG.info("Adding HFileLink " + hfile + " to table=" + tableName); + Path familyDir = new Path(regionDir, family); + restoreStoreFile(familyDir, snapshotRegionInfo, hfile); + } + }); + } + + /** + * Create a new {@link HFileLink} to reference the store file. + *

    The store file in the snapshot can be a simple hfile, an HFileLink or a reference. + *

      + *
    • hfile: abc -> table=region-abc + *
    • reference: abc.1234 -> table=region-abc.1234 + *
    • hfilelink: table=region-hfile -> table=region-hfile + *
    + * @param familyDir destination directory for the store file + * @param regionInfo destination region info for the table + * @param hfileName store file name (can be a Reference, HFileLink or simple HFile) + */ + private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo, + final String hfileName) throws IOException { + if (HFileLink.isHFileLink(hfileName)) { + HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName); + } else if (StoreFile.isReference(hfileName)) { + restoreReferenceFile(familyDir, regionInfo, hfileName); + } else { + HFileLink.create(conf, fs, familyDir, regionInfo, hfileName); + } + } + + /** + * Create a new {@link Reference} as copy of the source one. + * 

    +   * The source table looks like:
+   *    1234/abc      (original file)
+   *    5678/abc.1234 (reference file)
+   *
+   * After the clone operation looks like:
+   *   wxyz/table=1234-abc
+   *   stuv/table=1234-abc.wxyz
+   *
+   * NOTE that the region name in the clone changes (md5 of regioninfo)
+   * and the reference should reflect that change.
+   *
    + * @param familyDir destination directory for the store file + * @param regionInfo destination region info for the table + * @param hfileName reference file name + */ + private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo, + final String hfileName) throws IOException { + // Extract the referred information (hfile name and parent region) + String tableName = snapshotDesc.getTable(); + Path refPath = StoreFile.getReferredToFile(new Path(new Path(new Path(tableName, + regionInfo.getEncodedName()), familyDir.getName()), hfileName)); + String snapshotRegionName = refPath.getParent().getParent().getName(); + String fileName = refPath.getName(); + + // The new reference should have the cloned region name as parent, if it is a clone. + String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName))); + if (clonedRegionName == null) clonedRegionName = snapshotRegionName; + + // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName + String refLink = fileName; + if (!HFileLink.isHFileLink(fileName)) { + refLink = HFileLink.createHFileLinkName(tableName, snapshotRegionName, fileName); + } + Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName); + + // Create the new reference + Path linkPath = new Path(familyDir, + HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), hfileName)); + InputStream in = new HFileLink(conf, linkPath).open(fs); + OutputStream out = fs.create(outPath); + IOUtils.copyBytes(in, out, conf); + } + + /** + * Create a new {@link HRegionInfo} from the snapshot region info. + * Keep the same startKey, endKey, regionId and split information but change + * the table name. + * + * @param snapshotRegionInfo Info for region to clone. + * @return the new HRegion instance + */ + public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) { + return new HRegionInfo(tableDesc.getName(), + snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(), + snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId()); + } + + /** + * Restore snapshot WALs. + * + * Global Snapshot keep a reference to region servers logs present during the snapshot. + * (/hbase/.snapshot/snapshotName/.logs/hostName/logName) + * + * Since each log contains different tables data, logs must be split to + * extract the table that we are interested in. + */ + private void restoreWALs() throws IOException { + final SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir, + Bytes.toBytes(snapshotDesc.getTable()), regionsMap); + try { + // Recover.Edits + SnapshotReferenceUtil.visitRecoveredEdits(fs, snapshotDir, + new FSVisitor.RecoveredEditsVisitor() { + public void recoveredEdits (final String region, final String logfile) throws IOException { + Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile); + logSplitter.splitRecoveredEdit(path); + } + }); + + // Region Server Logs + SnapshotReferenceUtil.visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() { + public void logFile (final String server, final String logfile) throws IOException { + logSplitter.splitLog(server, logfile); + } + }); + } finally { + logSplitter.close(); + } + } + + /** + * @return the set of the regions contained in the table + */ + private List getTableRegions() throws IOException { + LOG.debug("get table regions: " + tableDir); + FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs)); + if (regionDirs == null) return null; + + List regions = new LinkedList(); + for (FileStatus regionDir: regionDirs) { + HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir.getPath()); + regions.add(hri); + } + LOG.debug("found " + regions.size() + " regions for table=" + tableDesc.getNameAsString()); + return regions; + } + + /** + * Create a new table descriptor cloning the snapshot table schema. + * + * @param snapshotTableDescriptor + * @param tableName + * @return cloned table descriptor + * @throws IOException + */ + public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor, + final byte[] tableName) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) { + htd.addFamily(hcd); + } + return htd; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java new file mode 100644 index 0000000..69dc3d0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; + +/** + * Thrown when a snapshot could not be created due to a server-side error when taking the snapshot. + */ +@SuppressWarnings("serial") +public class SnapshotCreationException extends HBaseSnapshotException { + + /** + * Used internally by the RPC engine to pass the exception back to the client. + * @param msg error message to pass back + */ + public SnapshotCreationException(String msg) { + super(msg); + } + + /** + * Failure to create the specified snapshot + * @param msg reason why the snapshot couldn't be completed + * @param desc description of the snapshot attempted + */ + public SnapshotCreationException(String msg, SnapshotDescription desc) { + super(msg, desc); + } + + /** + * Failure to create the specified snapshot due to an external cause + * @param msg reason why the snapshot couldn't be completed + * @param cause root cause of the failure + * @param desc description of the snapshot attempted + */ + public SnapshotCreationException(String msg, Throwable cause, SnapshotDescription desc) { + super(msg, cause, desc); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java new file mode 100644 index 0000000..e048768 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java @@ -0,0 +1,360 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.FSUtils; + +/** + * Utility class to help manage {@link SnapshotDescription SnapshotDesriptions}. + *

    + * Snapshots are laid out on disk like this: + * + * 

    + * /hbase/.snapshots
+ *          /.tmp                <---- working directory
+ *          /[snapshot name]     <----- completed snapshot
+ *
    + * + * A completed snapshot named 'completed' then looks like (multiple regions, servers, files, etc. + * signified by '...' on the same directory depth). + * + * 
    + * /hbase/.snapshots/completed
+ *                   .snapshotinfo          <--- Description of the snapshot
+ *                   .tableinfo             <--- Copy of the tableinfo
+ *                    /.logs
+ *                        /[server_name]
+ *                            /... [log files]
+ *                         ...
+ *                   /[region name]           <---- All the region's information
+ *                   .regioninfo              <---- Copy of the HRegionInfo
+ *                      /[column family name]
+ *                          /[hfile name]     <--- name of the hfile in the real region
+ *                          ...
+ *                      ...
+ *                    ...
+ *
    + * + * Utility methods in this class are useful for getting the correct locations for different parts of + * the snapshot, as well as moving completed snapshots into place (see + * {@link #completeSnapshot}, and writing the + * {@link SnapshotDescription} to the working snapshot directory. + */ +public class SnapshotDescriptionUtils { + + /** + * Filter that only accepts completed snapshot directories + */ + public static class CompletedSnaphotDirectoriesFilter extends FSUtils.DirFilter { + + /** + * @param fs + */ + public CompletedSnaphotDirectoriesFilter(FileSystem fs) { + super(fs); + } + + @Override + public boolean accept(Path path) { + // only accept directories that aren't the tmp directory + if (super.accept(path)) { + return !path.getName().equals(SNAPSHOT_TMP_DIR_NAME); + } + return false; + } + + } + + private static final Log LOG = LogFactory.getLog(SnapshotDescriptionUtils.class); + /** + * Version of the fs layout for a snapshot. Future snapshots may have different file layouts, + * which we may need to read in differently. + */ + public static final int SNAPSHOT_LAYOUT_VERSION = 0; + + // snapshot directory constants + /** + * The file contains the snapshot basic information and it is under the directory of a snapshot. + */ + public static final String SNAPSHOTINFO_FILE = ".snapshotinfo"; + + /** Temporary directory under the snapshot directory to store in-progress snapshots */ + public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp"; + // snapshot operation values + /** Default value if no start time is specified */ + public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0; + + public static final String MASTER_SNAPSHOT_TIMEOUT_MILLIS = "hbase.snapshot.master.timeout.millis"; + + /** By default, wait 60 seconds for a snapshot to complete */ + public static final long DEFAULT_MAX_WAIT_TIME = 60000; + + private SnapshotDescriptionUtils() { + // private constructor for utility class + } + + /** + * Check to make sure that the description of the snapshot requested is valid + * @param snapshot description of the snapshot + * @throws IllegalArgumentException if the name of the snapshot or the name of the table to + * snapshot are not valid names. + */ + public static void assertSnapshotRequestIsValid(SnapshotDescription snapshot) + throws IllegalArgumentException { + // FIXME these method names is really bad - trunk will probably change + // .META. and -ROOT- snapshots are not allowed + if (HTableDescriptor.isMetaTable(Bytes.toBytes(snapshot.getTable()))) { + throw new IllegalArgumentException(".META. and -ROOT- snapshots are not allowed"); + } + // make sure the snapshot name is valid + HTableDescriptor.isLegalTableName(Bytes.toBytes(snapshot.getName())); + // make sure the table name is valid + HTableDescriptor.isLegalTableName(Bytes.toBytes(snapshot.getTable())); + } + + /** + * @param conf {@link Configuration} from which to check for the timeout + * @param type type of snapshot being taken + * @param defaultMaxWaitTime Default amount of time to wait, if none is in the configuration + * @return the max amount of time the master should wait for a snapshot to complete + */ + public static long getMaxMasterTimeout(Configuration conf, SnapshotDescription.Type type, + long defaultMaxWaitTime) { + String confKey; + switch (type) { + case DISABLED: + default: + confKey = MASTER_SNAPSHOT_TIMEOUT_MILLIS; + } + return conf.getLong(confKey, defaultMaxWaitTime); + } + + /** + * Get the snapshot root directory. All the snapshots are kept under this directory, i.e. + * ${hbase.rootdir}/.snapshot + * @param rootDir hbase root directory + * @return the base directory in which all snapshots are kept + */ + public static Path getSnapshotRootDir(final Path rootDir) { + return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME); + } + + /** + * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root + * directory and all the data files for a snapshot are kept under this directory. + * @param snapshot snapshot being taken + * @param rootDir hbase root directory + * @return the final directory for the completed snapshot + */ + public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) { + return getCompletedSnapshotDir(snapshot.getName(), rootDir); + } + + /** + * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root + * directory and all the data files for a snapshot are kept under this directory. + * @param snapshotName name of the snapshot being taken + * @param rootDir hbase root directory + * @return the final directory for the completed snapshot + */ + public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) { + return getCompletedSnapshotDir(getSnapshotsDir(rootDir), snapshotName); + } + + /** + * Get the general working directory for snapshots - where they are built, where they are + * temporarily copied on export, etc. + * @param rootDir root directory of the HBase installation + * @return Path to the snapshot tmp directory, relative to the passed root directory + */ + public static Path getWorkingSnapshotDir(final Path rootDir) { + return new Path(getSnapshotsDir(rootDir), SNAPSHOT_TMP_DIR_NAME); + } + + /** + * Get the directory to build a snapshot, before it is finalized + * @param snapshot snapshot that will be built + * @param rootDir root directory of the hbase installation + * @return {@link Path} where one can build a snapshot + */ + public static Path getWorkingSnapshotDir(SnapshotDescription snapshot, final Path rootDir) { + return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshot.getName()); + } + + /** + * Get the directory to build a snapshot, before it is finalized + * @param snapshotName name of the snapshot + * @param rootDir root directory of the hbase installation + * @return {@link Path} where one can build a snapshot + */ + public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir) { + return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshotName); + } + + /** + * Get the directory to store the snapshot instance + * @param snapshotsDir hbase-global directory for storing all snapshots + * @param snapshotName name of the snapshot to take + * @return + */ + private static final Path getCompletedSnapshotDir(final Path snapshotsDir, String snapshotName) { + return new Path(snapshotsDir, snapshotName); + } + + /** + * @param rootDir hbase root directory + * @return the directory for all completed snapshots; + */ + public static final Path getSnapshotsDir(Path rootDir) { + return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME); + } + + /** + * Convert the passed snapshot description into a 'full' snapshot description based on default + * parameters, if none have been supplied. This resolves any 'optional' parameters that aren't + * supplied to their default values. + * @param snapshot general snapshot descriptor + * @param conf Configuration to read configured snapshot defaults if snapshot is not complete + * @return a valid snapshot description + * @throws IllegalArgumentException if the {@link SnapshotDescription} is not a complete + * {@link SnapshotDescription}. + */ + public static SnapshotDescription validate(SnapshotDescription snapshot, Configuration conf) + throws IllegalArgumentException { + if (!snapshot.hasTable()) { + throw new IllegalArgumentException( + "Descriptor doesn't apply to a table, so we can't build it."); + } + + // set the creation time, if one hasn't been set + long time = snapshot.getCreationTime(); + if (time == SnapshotDescriptionUtils.NO_SNAPSHOT_START_TIME_SPECIFIED) { + time = EnvironmentEdgeManager.currentTimeMillis(); + LOG.debug("Creation time not specified, setting to:" + time + " (current time:" + + EnvironmentEdgeManager.currentTimeMillis() + ")."); + SnapshotDescription.Builder builder = snapshot.toBuilder(); + builder.setCreationTime(time); + snapshot = builder.build(); + } + return snapshot; + } + + /** + * Write the snapshot description into the working directory of a snapshot + * @param snapshot description of the snapshot being taken + * @param workingDir working directory of the snapshot + * @param fs {@link FileSystem} on which the snapshot should be taken + * @throws IOException if we can't reach the filesystem and the file cannot be cleaned up on + * failure + */ + public static void writeSnapshotInfo(SnapshotDescription snapshot, Path workingDir, FileSystem fs) + throws IOException { + FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(), + HConstants.DATA_FILE_UMASK_KEY); + Path snapshotInfo = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE); + try { + FSDataOutputStream out = FSUtils.create(fs, snapshotInfo, perms, true); + try { + snapshot.writeTo(out); + } finally { + out.close(); + } + } catch (IOException e) { + // if we get an exception, try to remove the snapshot info + if (!fs.delete(snapshotInfo, false)) { + String msg = "Couldn't delete snapshot info file: " + snapshotInfo; + LOG.error(msg); + throw new IOException(msg); + } + } + } + + /** + * Read in the {@link SnapshotDescription} stored for the snapshot in the passed directory + * @param fs filesystem where the snapshot was taken + * @param snapshotDir directory where the snapshot was stored + * @return the stored snapshot description + * @throws CorruptedSnapshotException if the snapshot cannot be read + */ + public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir) + throws CorruptedSnapshotException { + Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE); + try { + FSDataInputStream in = null; + try { + in = fs.open(snapshotInfo); + return SnapshotDescription.parseFrom(in); + } finally { + if (in != null) in.close(); + } + } catch (IOException e) { + throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e); + } + } + + /** + * Move the finished snapshot to its final, publicly visible directory - this marks the snapshot + * as 'complete'. + * @param snapshot description of the snapshot being tabken + * @param rootdir root directory of the hbase installation + * @param workingDir directory where the in progress snapshot was built + * @param fs {@link FileSystem} where the snapshot was built + * @throws SnapshotCreationException if the snapshot could not be moved + * @throws IOException the filesystem could not be reached + */ + public static void completeSnapshot(SnapshotDescription snapshot, Path rootdir, Path workingDir, + FileSystem fs) throws SnapshotCreationException, IOException { + Path finishedDir = getCompletedSnapshotDir(snapshot, rootdir); + LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to " + + finishedDir); + if (!fs.rename(workingDir, finishedDir)) { + throw new SnapshotCreationException("Failed to move working directory(" + workingDir + + ") to completed directory(" + finishedDir + ").", snapshot); + } + } + + /** + * Returns a single line (no \n) representation of snapshot metadata. Use this instead of + * {@link SnapshotDescription#toString()}. We don't replace SnapshotDescrpition's toString + * because it is auto-generated by protoc. + * @param ssd + * @return Single line string with a summary of the snapshot parameters + */ + public static String toString(SnapshotDescription ssd) { + if (ssd == null) { + return null; + } + return "{ ss=" + ssd.getName() + " table=" + ssd.getTable() + + " type=" + ssd.getType() + " }"; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java new file mode 100644 index 0000000..eb02ece --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; + + +/** + * Thrown when the server is looking for a snapshot but can't find the snapshot on the filesystem + */ +@SuppressWarnings("serial") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class SnapshotDoesNotExistException extends HBaseSnapshotException { + /** + * @param msg full description of the failure + */ + public SnapshotDoesNotExistException(String msg) { + super(msg); + } + + /** + * @param desc expected snapshot to find + */ + public SnapshotDoesNotExistException(SnapshotDescription desc) { + super("Snapshot '" + desc.getName() +"' doesn't exist on the filesystem", desc); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotExistsException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotExistsException.java new file mode 100644 index 0000000..2ce2d31 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotExistsException.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; + +/** + * Thrown when a snapshot exists but should not + */ +@SuppressWarnings("serial") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class SnapshotExistsException extends HBaseSnapshotException { + + /** + * Failure due to the snapshot already existing + * @param msg full description of the failure + * @param desc snapshot that was attempted + */ + public SnapshotExistsException(String msg, SnapshotDescription desc) { + super(msg, desc); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java new file mode 100644 index 0000000..7caad2b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java @@ -0,0 +1,478 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.snapshot; + +import java.io.IOException; +import java.io.FileNotFoundException; +import java.text.SimpleDateFormat; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.Date; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.io.HLogLink; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.FSTableDescriptors; + +/** + * Tool for dumping snapshot information. + *
      + *
    1. Table Descriptor + *
    2. Snapshot creation time, type, format version, ... + *
    3. List of hfiles and hlogs + *
    4. Stats about hfiles and logs sizes, percentage of shared with the source table, ... + *
    + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public final class SnapshotInfo extends Configured implements Tool { + private static final Log LOG = LogFactory.getLog(SnapshotInfo.class); + + /** + * Statistics about the snapshot + *
      + *
    1. How many store files and logs are in the archive + *
    2. How many store files and logs are shared with the table + *
    3. Total store files and logs size and shared amount + *
    + */ + public static class SnapshotStats { + /** Information about the file referenced by the snapshot */ + static class FileInfo { + private final boolean inArchive; + private final long size; + + FileInfo(final boolean inArchive, final long size) { + this.inArchive = inArchive; + this.size = size; + } + + /** @return true if the file is in the archive */ + public boolean inArchive() { + return this.inArchive; + } + + /** @return true if the file is missing */ + public boolean isMissing() { + return this.size < 0; + } + + /** @return the file size */ + public long getSize() { + return this.size; + } + } + + private int hfileArchiveCount = 0; + private int hfilesMissing = 0; + private int hfilesCount = 0; + private int logsMissing = 0; + private int logsCount = 0; + private long hfileArchiveSize = 0; + private long hfileSize = 0; + private long logSize = 0; + + private final SnapshotDescription snapshot; + private final Configuration conf; + private final FileSystem fs; + + SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot) + { + this.snapshot = snapshot; + this.conf = conf; + this.fs = fs; + } + + /** @return the snapshot descriptor */ + public SnapshotDescription getSnapshotDescription() { + return this.snapshot; + } + + /** @return true if the snapshot is corrupted */ + public boolean isSnapshotCorrupted() { + return hfilesMissing > 0 || logsMissing > 0; + } + + /** @return the number of available store files */ + public int getStoreFilesCount() { + return hfilesCount + hfileArchiveCount; + } + + /** @return the number of available store files in the archive */ + public int getArchivedStoreFilesCount() { + return hfileArchiveCount; + } + + /** @return the number of available log files */ + public int getLogsCount() { + return logsCount; + } + + /** @return the number of missing store files */ + public int getMissingStoreFilesCount() { + return hfilesMissing; + } + + /** @return the number of missing log files */ + public int getMissingLogsCount() { + return logsMissing; + } + + /** @return the total size of the store files referenced by the snapshot */ + public long getStoreFilesSize() { + return hfileSize + hfileArchiveSize; + } + + /** @return the total size of the store files shared */ + public long getSharedStoreFilesSize() { + return hfileSize; + } + + /** @return the total size of the store files in the archive */ + public long getArchivedStoreFileSize() { + return hfileArchiveSize; + } + + /** @return the percentage of the shared store files */ + public float getSharedStoreFilePercentage() { + return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100; + } + + /** @return the total log size */ + public long getLogsSize() { + return logSize; + } + + /** + * Add the specified store file to the stats + * @param region region encoded Name + * @param family family name + * @param hfile store file name + * @return the store file information + */ + FileInfo addStoreFile(final String region, final String family, final String hfile) + throws IOException { + String table = this.snapshot.getTable(); + Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile)); + HFileLink link = new HFileLink(conf, path); + boolean inArchive = false; + long size = -1; + try { + if ((inArchive = fs.exists(link.getArchivePath()))) { + size = fs.getFileStatus(link.getArchivePath()).getLen(); + hfileArchiveSize += size; + hfileArchiveCount++; + } else { + size = link.getFileStatus(fs).getLen(); + hfileSize += size; + hfilesCount++; + } + } catch (FileNotFoundException e) { + hfilesMissing++; + } + return new FileInfo(inArchive, size); + } + + /** + * Add the specified recovered.edits file to the stats + * @param region region encoded name + * @param logfile log file name + * @return the recovered.edits information + */ + FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException { + Path rootDir = FSUtils.getRootDir(conf); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); + Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile); + long size = fs.getFileStatus(path).getLen(); + logSize += size; + logsCount++; + return new FileInfo(true, size); + } + + /** + * Add the specified log file to the stats + * @param server server name + * @param logfile log file name + * @return the log information + */ + FileInfo addLogFile(final String server, final String logfile) throws IOException { + HLogLink logLink = new HLogLink(conf, server, logfile); + long size = -1; + try { + size = logLink.getFileStatus(fs).getLen(); + logSize += size; + logsCount++; + } catch (FileNotFoundException e) { + logsMissing++; + } + return new FileInfo(false, size); + } + } + + private FileSystem fs; + private Path rootDir; + + private HTableDescriptor snapshotTableDesc; + private SnapshotDescription snapshotDesc; + private Path snapshotDir; + + @Override + public int run(String[] args) throws IOException, InterruptedException { + String snapshotName = null; + boolean showSchema = false; + boolean showFiles = false; + boolean showStats = false; + + // Process command line args + for (int i = 0; i < args.length; i++) { + String cmd = args[i]; + try { + if (cmd.equals("-snapshot")) { + snapshotName = args[++i]; + } else if (cmd.equals("-files")) { + showFiles = true; + } else if (cmd.equals("-stats")) { + showStats = true; + } else if (cmd.equals("-schema")) { + showSchema = true; + } else if (cmd.equals("-h") || cmd.equals("--help")) { + printUsageAndExit(); + } else { + System.err.println("UNEXPECTED: " + cmd); + printUsageAndExit(); + } + } catch (Exception e) { + printUsageAndExit(); + } + } + + if (snapshotName == null) { + System.err.println("Missing snapshot name!"); + printUsageAndExit(); + return 1; + } + + Configuration conf = getConf(); + fs = FileSystem.get(conf); + rootDir = FSUtils.getRootDir(conf); + + // Load snapshot information + if (!loadSnapshotInfo(snapshotName)) { + System.err.println("Snapshot '" + snapshotName + "' not found!"); + return 1; + } + + printInfo(); + if (showSchema) printSchema(); + if (showFiles || showStats) printFiles(showFiles); + + return 0; + } + + /** + * Load snapshot info and table descriptor for the specified snapshot + * @param snapshotName name of the snapshot to load + * @return false if snapshot is not found + */ + private boolean loadSnapshotInfo(final String snapshotName) throws IOException { + snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + if (!fs.exists(snapshotDir)) { + LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir); + return false; + } + + snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); + snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir); + return true; + } + + /** + * Dump the {@link SnapshotDescription} + */ + private void printInfo() { + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + System.out.println("Snapshot Info"); + System.out.println("----------------------------------------"); + System.out.println(" Name: " + snapshotDesc.getName()); + System.out.println(" Type: " + snapshotDesc.getType()); + System.out.println(" Table: " + snapshotDesc.getTable()); + System.out.println(" Format: " + snapshotDesc.getVersion()); + System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime()))); + System.out.println(); + } + + /** + * Dump the {@link HTableDescriptor} + */ + private void printSchema() { + System.out.println("Table Descriptor"); + System.out.println("----------------------------------------"); + System.out.println(snapshotTableDesc.toString()); + System.out.println(); + } + + /** + * Collect the hfiles and logs statistics of the snapshot and + * dump the file list if requested and the collected information. + */ + private void printFiles(final boolean showFiles) throws IOException { + if (showFiles) { + System.out.println("Snapshot Files"); + System.out.println("----------------------------------------"); + } + + // Collect information about hfiles and logs in the snapshot + final String table = this.snapshotDesc.getTable(); + final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc); + SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir, + new SnapshotReferenceUtil.FileVisitor() { + public void storeFile (final String region, final String family, final String hfile) + throws IOException { + SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile); + + if (showFiles) { + System.out.printf("%8s %s/%s/%s/%s %s%n", + (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())), + table, region, family, hfile, + (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : "")); + } + } + + public void recoveredEdits (final String region, final String logfile) + throws IOException { + SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile); + + if (showFiles) { + System.out.printf("%8s recovered.edits %s on region %s%n", + StringUtils.humanReadableInt(info.getSize()), logfile, region); + } + } + + public void logFile (final String server, final String logfile) + throws IOException { + SnapshotStats.FileInfo info = stats.addLogFile(server, logfile); + + if (showFiles) { + System.out.printf("%8s log %s on server %s %s%n", + (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())), + logfile, server, + (info.isMissing() ? "(NOT FOUND)" : "")); + } + } + }); + + // Dump the stats + System.out.println(); + if (stats.isSnapshotCorrupted()) { + System.out.println("**************************************************************"); + System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n", + stats.getMissingStoreFilesCount(), stats.getMissingLogsCount()); + System.out.println("**************************************************************"); + } + + System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n", + stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(), + StringUtils.humanReadableInt(stats.getStoreFilesSize()), + stats.getSharedStoreFilePercentage(), + StringUtils.humanReadableInt(stats.getSharedStoreFilesSize()) + ); + System.out.printf("%d Logs, total size %s%n", + stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize())); + System.out.println(); + } + + private void printUsageAndExit() { + System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName()); + System.err.println(" where [options] are:"); + System.err.println(" -h|-help Show this help and exit."); + System.err.println(" -snapshot NAME Snapshot to examine."); + System.err.println(" -files Files and logs list."); + System.err.println(" -stats Files and logs stats."); + System.err.println(" -schema Describe the snapshotted table."); + System.err.println(); + System.err.println("Examples:"); + System.err.println(" hbase " + getClass() + " \\"); + System.err.println(" -snapshot MySnapshot -files"); + System.exit(1); + } + + /** + * Returns the snapshot stats + * @param conf the {@link Configuration} to use + * @param snapshot {@link SnapshotDescription} to get stats from + * @return the snapshot stats + */ + public static SnapshotStats getSnapshotStats(final Configuration conf, + final SnapshotDescription snapshot) throws IOException { + Path rootDir = FSUtils.getRootDir(conf); + FileSystem fs = FileSystem.get(conf); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); + final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot); + SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir, + new SnapshotReferenceUtil.FileVisitor() { + public void storeFile (final String region, final String family, final String hfile) + throws IOException { + stats.addStoreFile(region, family, hfile); + } + + public void recoveredEdits (final String region, final String logfile) throws IOException { + stats.addRecoveredEdits(region, logfile); + } + + public void logFile (final String server, final String logfile) throws IOException { + stats.addLogFile(server, logfile); + } + }); + return stats; + } + + /** + * The guts of the {@link #main} method. + * Call this method to avoid the {@link #main(String[])} System.exit. + * @param args + * @return errCode + * @throws Exception + */ + static int innerMain(final String [] args) throws Exception { + return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args); + } + + public static void main(String[] args) throws Exception { + System.exit(innerMain(args)); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotLogSplitter.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotLogSplitter.java new file mode 100644 index 0000000..788435c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotLogSplitter.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.snapshot; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.io.HLogLink; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * If the snapshot has references to one or more log files, + * those must be split (each log contains multiple tables and regions) + * and must be placed in the region/recovered.edits folder. + * (recovered.edits files will be played on region startup) + * + * In case of Restore: the log can just be split in the recovered.edits folder. + * In case of Clone: each entry in the log must be modified to use the new region name. + * (region names are encoded with: tableName, startKey, regionIdTimeStamp) + * + * We can't use the normal split code, because the HLogKey contains the + * table name and the region name, and in case of "clone from snapshot" + * region name and table name will be different and must be replaced in + * the recovered.edits. + */ +@InterfaceAudience.Private +class SnapshotLogSplitter implements Closeable { + static final Log LOG = LogFactory.getLog(SnapshotLogSplitter.class); + + private final class LogWriter implements Closeable { + private HLog.Writer writer; + private Path logFile; + private long seqId; + + public LogWriter(final Configuration conf, final FileSystem fs, + final Path logDir, long seqId) throws IOException { + logFile = new Path(logDir, logFileName(seqId, true)); + this.writer = HLog.createWriter(fs, logFile, conf); + this.seqId = seqId; + } + + public void close() throws IOException { + writer.close(); + + Path finalFile = new Path(logFile.getParent(), logFileName(seqId, false)); + LOG.debug("LogWriter tmpLogFile=" + logFile + " -> logFile=" + finalFile); + fs.rename(logFile, finalFile); + } + + public void append(final HLog.Entry entry) throws IOException { + writer.append(entry); + if (seqId < entry.getKey().getLogSeqNum()) { + seqId = entry.getKey().getLogSeqNum(); + } + } + + private String logFileName(long seqId, boolean temp) { + String fileName = String.format("%019d", seqId); + if (temp) fileName += HLog.RECOVERED_LOG_TMPFILE_SUFFIX; + return fileName; + } + } + + private final Map regionLogWriters = + new TreeMap(Bytes.BYTES_COMPARATOR); + + private final Map regionsMap; + private final Configuration conf; + private final byte[] snapshotTableName; + private final byte[] tableName; + private final Path tableDir; + private final FileSystem fs; + + /** + * @params tableName snapshot table name + * @params regionsMap maps original region names to the new ones. + */ + public SnapshotLogSplitter(final Configuration conf, final FileSystem fs, + final Path tableDir, final byte[] snapshotTableName, + final Map regionsMap) { + this.regionsMap = regionsMap; + this.snapshotTableName = snapshotTableName; + this.tableName = Bytes.toBytes(tableDir.getName()); + this.tableDir = tableDir; + this.conf = conf; + this.fs = fs; + } + + public void close() throws IOException { + for (LogWriter writer: regionLogWriters.values()) { + writer.close(); + } + } + + public void splitLog(final String serverName, final String logfile) throws IOException { + LOG.debug("Restore log=" + logfile + " server=" + serverName + + " for snapshotTable=" + Bytes.toString(snapshotTableName) + + " to table=" + Bytes.toString(tableName)); + splitLog(new HLogLink(conf, serverName, logfile).getAvailablePath(fs)); + } + + public void splitRecoveredEdit(final Path editPath) throws IOException { + LOG.debug("Restore recover.edits=" + editPath + + " for snapshotTable=" + Bytes.toString(snapshotTableName) + + " to table=" + Bytes.toString(tableName)); + splitLog(editPath); + } + + /** + * Split the snapshot HLog reference into regions recovered.edits. + * + * The HLogKey contains the table name and the region name, + * and they must be changed to the restored table names. + * + * @param logPath Snapshot HLog reference path + */ + public void splitLog(final Path logPath) throws IOException { + HLog.Reader log = HLog.getReader(fs, logPath, conf); + try { + HLog.Entry entry; + LogWriter writer = null; + byte[] regionName = null; + byte[] newRegionName = null; + while ((entry = log.next()) != null) { + HLogKey key = entry.getKey(); + + // We're interested only in the snapshot table that we're restoring + if (!Bytes.equals(key.getTablename(), snapshotTableName)) continue; + + // Writer for region. + if (!Bytes.equals(regionName, key.getEncodedRegionName())) { + regionName = key.getEncodedRegionName().clone(); + + // Get the new region name in case of clone, or use the original one + newRegionName = regionsMap.get(regionName); + if (newRegionName == null) newRegionName = regionName; + + writer = getOrCreateWriter(newRegionName, key.getLogSeqNum()); + LOG.debug("+ regionName=" + Bytes.toString(regionName)); + } + + // Append Entry + key = new HLogKey(newRegionName, tableName, + key.getLogSeqNum(), key.getWriteTime(), key.getClusterId()); + writer.append(new HLog.Entry(key, entry.getEdit())); + } + } catch (IOException e) { + LOG.warn("Something wrong during the log split", e); + } finally { + log.close(); + } + } + + /** + * Create a LogWriter for specified region if not already created. + */ + private LogWriter getOrCreateWriter(final byte[] regionName, long seqId) throws IOException { + LogWriter writer = regionLogWriters.get(regionName); + if (writer == null) { + Path regionDir = HRegion.getRegionDir(tableDir, Bytes.toString(regionName)); + Path dir = HLog.getRegionDirRecoveredEditsDir(regionDir); + fs.mkdirs(dir); + + writer = new LogWriter(conf, fs, dir, seqId); + regionLogWriters.put(regionName, writer); + } + return(writer); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java new file mode 100644 index 0000000..eee2057 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java @@ -0,0 +1,251 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.snapshot; + +import java.io.IOException; +import java.util.HashSet; +import java.util.TreeMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileStatus; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.FSVisitor; + +/** + * Utility methods for interacting with the snapshot referenced files. + */ +@InterfaceAudience.Private +public final class SnapshotReferenceUtil { + public interface FileVisitor extends FSVisitor.StoreFileVisitor, + FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor { + } + + private SnapshotReferenceUtil() { + // private constructor for utility class + } + + /** + * Get log directory for a server in a snapshot. + * + * @param snapshotDir directory where the specific snapshot is stored + * @param serverName name of the parent regionserver for the log files + * @return path to the log home directory for the archive files. + */ + public static Path getLogsDir(Path snapshotDir, String serverName) { + return new Path(snapshotDir, HLog.getHLogDirectoryName(serverName)); + } + + /** + * Get the snapshotted recovered.edits dir for the specified region. + * + * @param snapshotDir directory where the specific snapshot is stored + * @param regionName name of the region + * @return path to the recovered.edits directory for the specified region files. + */ + public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) { + return HLog.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName)); + } + + /** + * Get the snapshot recovered.edits file + * + * @param snapshotDir directory where the specific snapshot is stored + * @param regionName name of the region + * @param logfile name of the edit file + * @return full path of the log file for the specified region files. + */ + public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) { + return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile); + } + + /** + * Iterate over the snapshot store files, restored.edits and logs + * + * @param fs {@link FileSystem} + * @param snapshotDir {@link Path} to the Snapshot directory + * @param visitor callback object to get the referenced files + * @throws IOException if an error occurred while scanning the directory + */ + public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir, + final FileVisitor visitor) throws IOException { + visitTableStoreFiles(fs, snapshotDir, visitor); + visitRecoveredEdits(fs, snapshotDir, visitor); + visitLogFiles(fs, snapshotDir, visitor); + } + + /** + * Iterate over the snapshot store files + * + * @param fs {@link FileSystem} + * @param snapshotDir {@link Path} to the Snapshot directory + * @param visitor callback object to get the store files + * @throws IOException if an error occurred while scanning the directory + */ + public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir, + final FSVisitor.StoreFileVisitor visitor) throws IOException { + FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor); + } + + /** + * Iterate over the snapshot store files in the specified region + * + * @param fs {@link FileSystem} + * @param regionDir {@link Path} to the Snapshot region directory + * @param visitor callback object to get the store files + * @throws IOException if an error occurred while scanning the directory + */ + public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir, + final FSVisitor.StoreFileVisitor visitor) throws IOException { + FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor); + } + + /** + * Iterate over the snapshot recovered.edits + * + * @param fs {@link FileSystem} + * @param snapshotDir {@link Path} to the Snapshot directory + * @param visitor callback object to get the recovered.edits files + * @throws IOException if an error occurred while scanning the directory + */ + public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir, + final FSVisitor.RecoveredEditsVisitor visitor) throws IOException { + FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor); + } + + /** + * Iterate over the snapshot log files + * + * @param fs {@link FileSystem} + * @param snapshotDir {@link Path} to the Snapshot directory + * @param visitor callback object to get the log files + * @throws IOException if an error occurred while scanning the directory + */ + public static void visitLogFiles(final FileSystem fs, final Path snapshotDir, + final FSVisitor.LogFileVisitor visitor) throws IOException { + FSVisitor.visitLogFiles(fs, snapshotDir, visitor); + } + + /** + * Returns the set of region names available in the snapshot. + * + * @param fs {@link FileSystem} + * @param snapshotDir {@link Path} to the Snapshot directory + * @throws IOException if an error occurred while scanning the directory + * @return the set of the regions contained in the snapshot + */ + public static Set getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir) + throws IOException { + FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs)); + if (regionDirs == null) return null; + + Set regions = new HashSet(); + for (FileStatus regionDir: regionDirs) { + regions.add(regionDir.getPath().getName()); + } + return regions; + } + + /** + * Get the list of hfiles for the specified snapshot region. + * NOTE: The current implementation keeps one empty file per HFile in the region. + * The file name matches the one in the original table, and by reconstructing + * the path you can quickly jump to the referenced file. + * + * @param fs {@link FileSystem} + * @param snapshotRegionDir {@link Path} to the Snapshot region directory + * @return Map of hfiles per family, the key is the family name and values are hfile names + * @throws IOException if an error occurred while scanning the directory + */ + public static Map> getRegionHFileReferences(final FileSystem fs, + final Path snapshotRegionDir) throws IOException { + final Map> familyFiles = new TreeMap>(); + + visitRegionStoreFiles(fs, snapshotRegionDir, + new FSVisitor.StoreFileVisitor() { + public void storeFile (final String region, final String family, final String hfile) + throws IOException { + List hfiles = familyFiles.get(family); + if (hfiles == null) { + hfiles = new LinkedList(); + familyFiles.put(family, hfiles); + } + hfiles.add(hfile); + } + }); + + return familyFiles; + } + + /** + * Returns the store file names in the snapshot. + * + * @param fs {@link FileSystem} + * @param snapshotDir {@link Path} to the Snapshot directory + * @throws IOException if an error occurred while scanning the directory + * @return the names of hfiles in the specified snaphot + */ + public static Set getHFileNames(final FileSystem fs, final Path snapshotDir) + throws IOException { + final Set names = new HashSet(); + visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() { + public void storeFile (final String region, final String family, final String hfile) + throws IOException { + if (HFileLink.isHFileLink(hfile)) { + names.add(HFileLink.getReferencedHFileName(hfile)); + } else { + names.add(hfile); + } + } + }); + return names; + } + + /** + * Returns the log file names available in the snapshot. + * + * @param fs {@link FileSystem} + * @param snapshotDir {@link Path} to the Snapshot directory + * @throws IOException if an error occurred while scanning the directory + * @return the names of hlogs in the specified snaphot + */ + public static Set getHLogNames(final FileSystem fs, final Path snapshotDir) + throws IOException { + final Set names = new HashSet(); + visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() { + public void logFile (final String server, final String logfile) throws IOException { + names.add(logfile); + } + }); + return names; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotTask.java b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotTask.java new file mode 100644 index 0000000..ede2d85 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotTask.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import java.util.concurrent.Callable; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.errorhandling.ForeignException; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; + +/** + * General snapshot operation taken on a regionserver + */ +@InterfaceAudience.Private +public abstract class SnapshotTask implements ForeignExceptionSnare, Callable{ + + protected final SnapshotDescription snapshot; + protected final ForeignExceptionDispatcher errorMonitor; + + /** + * @param snapshot Description of the snapshot we are going to operate on + * @param monitor listener interested in failures to the snapshot caused by this operation + */ + public SnapshotTask(SnapshotDescription snapshot, ForeignExceptionDispatcher monitor) { + assert monitor != null : "ForeignExceptionDispatcher must not be null!"; + assert snapshot != null : "SnapshotDescription must not be null!"; + this.snapshot = snapshot; + this.errorMonitor = monitor; + } + + public void snapshotFailure(String message, Exception e) { + ForeignException ee = new ForeignException(message, e); + errorMonitor.receive(ee); + } + + @Override + public void rethrowException() throws ForeignException { + this.errorMonitor.rethrowException(); + } + + @Override + public boolean hasException() { + return this.errorMonitor.hasException(); + } + + @Override + public ForeignException getException() { + return this.errorMonitor.getException(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java b/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java new file mode 100644 index 0000000..9d431b4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; + +/** + * Copy the table info into the snapshot directory + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class TableInfoCopyTask extends SnapshotTask { + + public static final Log LOG = LogFactory.getLog(TableInfoCopyTask.class); + private final FileSystem fs; + private final Path rootDir; + + /** + * Copy the table info for the given table into the snapshot + * @param monitor listen for errors while running the snapshot + * @param snapshot snapshot for which we are copying the table info + * @param fs {@link FileSystem} where the tableinfo is stored (and where the copy will be written) + * @param rootDir root of the {@link FileSystem} where the tableinfo is stored + */ + public TableInfoCopyTask(ForeignExceptionDispatcher monitor, + SnapshotDescription snapshot, FileSystem fs, Path rootDir) { + super(snapshot, monitor); + this.rootDir = rootDir; + this.fs = fs; + } + + @Override + public Void call() throws Exception { + LOG.debug("Running table info copy."); + this.rethrowException(); + LOG.debug("Attempting to copy table info for snapshot:" + + SnapshotDescriptionUtils.toString(this.snapshot)); + // get the HTable descriptor + HTableDescriptor orig = FSTableDescriptors.getTableDescriptor(fs, rootDir, + Bytes.toBytes(this.snapshot.getTable())); + this.rethrowException(); + // write a copy of descriptor to the snapshot directory + Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); + FSTableDescriptors.createTableDescriptorForTableDirectory(fs, snapshotDir, orig, false); + LOG.debug("Finished copying tableinfo."); + return null; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/TablePartiallyOpenException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/TablePartiallyOpenException.java new file mode 100644 index 0000000..6b27be8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/TablePartiallyOpenException.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Thrown if a table should be online/offline but is partially open + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class TablePartiallyOpenException extends IOException { + private static final long serialVersionUID = 3571982660065058361L; + + public TablePartiallyOpenException() { + super(); + } + + /** + * @param s message + */ + public TablePartiallyOpenException(String s) { + super(s); + } + + /** + * @param tableName Name of table that is partial open + */ + public TablePartiallyOpenException(byte[] tableName) { + this(Bytes.toString(tableName)); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/TakeSnapshotUtils.java b/src/main/java/org/apache/hadoop/hbase/snapshot/TakeSnapshotUtils.java new file mode 100644 index 0000000..220e964 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/TakeSnapshotUtils.java @@ -0,0 +1,323 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener; +import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; + +/** + * Utilities for useful when taking a snapshot + */ +public class TakeSnapshotUtils { + + private static final Log LOG = LogFactory.getLog(TakeSnapshotUtils.class); + + private TakeSnapshotUtils() { + // private constructor for util class + } + + /** + * Get the per-region snapshot description location. + *

    + * Under the per-snapshot directory, specific files per-region are kept in a similar layout as per + * the current directory layout. + * @param desc description of the snapshot + * @param rootDir root directory for the hbase installation + * @param regionName encoded name of the region (see {@link HRegionInfo#encodeRegionName(byte[])}) + * @return path to the per-region directory for the snapshot + */ + public static Path getRegionSnapshotDirectory(SnapshotDescription desc, Path rootDir, + String regionName) { + Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); + return HRegion.getRegionDir(snapshotDir, regionName); + } + + /** + * Get the home directory for store-level snapshot files. + *

    + * Specific files per store are kept in a similar layout as per the current directory layout. + * @param regionDir snapshot directory for the parent region, not the standard region + * directory. See {@link #getRegionSnapshotDirectory} + * @param family name of the store to snapshot + * @return path to the snapshot home directory for the store/family + */ + public static Path getStoreSnapshotDirectory(Path regionDir, String family) { + return Store.getStoreHomedir(regionDir, Bytes.toBytes(family)); + } + + /** + * Get the snapshot directory for each family to be added to the the snapshot + * @param snapshot description of the snapshot being take + * @param snapshotRegionDir directory in the snapshot where the region directory information + * should be stored + * @param families families to be added (can be null) + * @return paths to the snapshot directory for each family, in the same order as the families + * passed in + */ + public static List getFamilySnapshotDirectories(SnapshotDescription snapshot, + Path snapshotRegionDir, FileStatus[] families) { + if (families == null || families.length == 0) return Collections.emptyList(); + + List familyDirs = new ArrayList(families.length); + for (FileStatus family : families) { + // build the reference directory name + familyDirs.add(getStoreSnapshotDirectory(snapshotRegionDir, family.getPath().getName())); + } + return familyDirs; + } + + /** + * Create a snapshot timer for the master which notifies the monitor when an error occurs + * @param snapshot snapshot to monitor + * @param conf configuration to use when getting the max snapshot life + * @param monitor monitor to notify when the snapshot life expires + * @return the timer to use update to signal the start and end of the snapshot + */ + public static TimeoutExceptionInjector getMasterTimerAndBindToMonitor(SnapshotDescription snapshot, + Configuration conf, ForeignExceptionListener monitor) { + long maxTime = SnapshotDescriptionUtils.getMaxMasterTimeout(conf, snapshot.getType(), + SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME); + return new TimeoutExceptionInjector(monitor, maxTime); + } + + /** + * Verify that all the expected logs got referenced + * @param fs filesystem where the logs live + * @param logsDir original logs directory + * @param serverNames names of the servers that involved in the snapshot + * @param snapshot description of the snapshot being taken + * @param snapshotLogDir directory for logs in the snapshot + * @throws IOException + */ + public static void verifyAllLogsGotReferenced(FileSystem fs, Path logsDir, + Set serverNames, SnapshotDescription snapshot, Path snapshotLogDir) + throws IOException { + assertTrue(snapshot, "Logs directory doesn't exist in snapshot", fs.exists(logsDir)); + // for each of the server log dirs, make sure it matches the main directory + Multimap snapshotLogs = getMapOfServersAndLogs(fs, snapshotLogDir, serverNames); + Multimap realLogs = getMapOfServersAndLogs(fs, logsDir, serverNames); + if (realLogs != null) { + assertNotNull(snapshot, "No server logs added to snapshot", snapshotLogs); + } else { + assertNull(snapshot, "Snapshotted server logs that don't exist", snapshotLogs); + } + + // check the number of servers + Set>> serverEntries = realLogs.asMap().entrySet(); + Set>> snapshotEntries = snapshotLogs.asMap().entrySet(); + assertEquals(snapshot, "Not the same number of snapshot and original server logs directories", + serverEntries.size(), snapshotEntries.size()); + + // verify we snapshotted each of the log files + for (Entry> serverLogs : serverEntries) { + // if the server is not the snapshot, skip checking its logs + if (!serverNames.contains(serverLogs.getKey())) continue; + Collection snapshotServerLogs = snapshotLogs.get(serverLogs.getKey()); + assertNotNull(snapshot, "Snapshots missing logs for server:" + serverLogs.getKey(), + snapshotServerLogs); + + // check each of the log files + assertEquals(snapshot, + "Didn't reference all the log files for server:" + serverLogs.getKey(), serverLogs + .getValue().size(), snapshotServerLogs.size()); + for (String log : serverLogs.getValue()) { + assertTrue(snapshot, "Snapshot logs didn't include " + log, + snapshotServerLogs.contains(log)); + } + } + } + + /** + * Verify one of a snapshot's region's recovered.edits, has been at the surface (file names, + * length), match the original directory. + * @param fs filesystem on which the snapshot had been taken + * @param rootDir full path to the root hbase directory + * @param regionInfo info for the region + * @param snapshot description of the snapshot that was taken + * @throws IOException if there is an unexpected error talking to the filesystem + */ + public static void verifyRecoveredEdits(FileSystem fs, Path rootDir, HRegionInfo regionInfo, + SnapshotDescription snapshot) throws IOException { + Path regionDir = HRegion.getRegionDir(rootDir, regionInfo); + Path editsDir = HLog.getRegionDirRecoveredEditsDir(regionDir); + Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(snapshot, rootDir, + regionInfo.getEncodedName()); + Path snapshotEditsDir = HLog.getRegionDirRecoveredEditsDir(snapshotRegionDir); + + FileStatus[] edits = FSUtils.listStatus(fs, editsDir); + FileStatus[] snapshotEdits = FSUtils.listStatus(fs, snapshotEditsDir); + if (edits == null) { + assertNull(snapshot, "Snapshot has edits but table doesn't", snapshotEdits); + return; + } + + assertNotNull(snapshot, "Table has edits, but snapshot doesn't", snapshotEdits); + + // check each of the files + assertEquals(snapshot, "Not same number of edits in snapshot as table", edits.length, + snapshotEdits.length); + + // make sure we have a file with the same name as the original + // it would be really expensive to verify the content matches the original + for (FileStatus edit : edits) { + for (FileStatus sEdit : snapshotEdits) { + if (sEdit.getPath().equals(edit.getPath())) { + assertEquals(snapshot, "Snapshot file" + sEdit.getPath() + + " length not equal to the original: " + edit.getPath(), edit.getLen(), + sEdit.getLen()); + break; + } + } + assertTrue(snapshot, "No edit in snapshot with name:" + edit.getPath(), false); + } + } + + private static void assertNull(SnapshotDescription snapshot, String msg, Object isNull) + throws CorruptedSnapshotException { + if (isNull != null) { + throw new CorruptedSnapshotException(msg + ", Expected " + isNull + " to be null.", snapshot); + } + } + + private static void assertNotNull(SnapshotDescription snapshot, String msg, Object notNull) + throws CorruptedSnapshotException { + if (notNull == null) { + throw new CorruptedSnapshotException(msg + ", Expected object to not be null, but was null.", + snapshot); + } + } + + private static void assertTrue(SnapshotDescription snapshot, String msg, boolean isTrue) + throws CorruptedSnapshotException { + if (!isTrue) { + throw new CorruptedSnapshotException(msg + ", Expected true, but was false", snapshot); + } + } + + /** + * Assert that the expect matches the gotten amount + * @param msg message to add the to exception + * @param expected + * @param gotten + * @throws CorruptedSnapshotException thrown if the two elements don't match + */ + private static void assertEquals(SnapshotDescription snapshot, String msg, int expected, + int gotten) throws CorruptedSnapshotException { + if (expected != gotten) { + throw new CorruptedSnapshotException(msg + ". Expected:" + expected + ", got:" + gotten, + snapshot); + } + } + + /** + * Assert that the expect matches the gotten amount + * @param msg message to add the to exception + * @param expected + * @param gotten + * @throws CorruptedSnapshotException thrown if the two elements don't match + */ + private static void assertEquals(SnapshotDescription snapshot, String msg, long expected, + long gotten) throws CorruptedSnapshotException { + if (expected != gotten) { + throw new CorruptedSnapshotException(msg + ". Expected:" + expected + ", got:" + gotten, + snapshot); + } + } + + /** + * @param logdir + * @param toInclude list of servers to include. If empty or null, returns all servers + * @return maps of servers to all their log files. If there is no log directory, returns + * null + */ + private static Multimap getMapOfServersAndLogs(FileSystem fs, Path logdir, + Collection toInclude) throws IOException { + // create a path filter based on the passed directories to include + PathFilter filter = toInclude == null || toInclude.size() == 0 ? null + : new MatchesDirectoryNames(toInclude); + + // get all the expected directories + FileStatus[] serverLogDirs = FSUtils.listStatus(fs, logdir, filter); + if (serverLogDirs == null) return null; + + // map those into a multimap of servername -> [log files] + Multimap map = HashMultimap.create(); + for (FileStatus server : serverLogDirs) { + FileStatus[] serverLogs = FSUtils.listStatus(fs, server.getPath(), null); + if (serverLogs == null) continue; + for (FileStatus log : serverLogs) { + map.put(server.getPath().getName(), log.getPath().getName()); + } + } + return map; + } + + /** + * Path filter that only accepts paths where that have a {@link Path#getName()} that is contained + * in the specified collection. + */ + private static class MatchesDirectoryNames implements PathFilter { + + Collection paths; + + public MatchesDirectoryNames(Collection dirNames) { + this.paths = dirNames; + } + + @Override + public boolean accept(Path path) { + return paths.contains(path.getName()); + } + } + + /** + * Get the log directory for a specific snapshot + * @param snapshotDir directory where the specific snapshot will be store + * @param serverName name of the parent regionserver for the log files + * @return path to the log home directory for the archive files. + */ + public static Path getSnapshotHLogsDir(Path snapshotDir, String serverName) { + return new Path(snapshotDir, HLog.getHLogDirectoryName(serverName)); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/snapshot/UnknownSnapshotException.java b/src/main/java/org/apache/hadoop/hbase/snapshot/UnknownSnapshotException.java new file mode 100644 index 0000000..a6b381f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/snapshot/UnknownSnapshotException.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Exception thrown when we get a request for a snapshot we don't recognize. + */ +@SuppressWarnings("serial") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class UnknownSnapshotException extends HBaseSnapshotException { + + /** + * @param msg full information about the failure + */ + public UnknownSnapshotException(String msg) { + super(msg); + } + + public UnknownSnapshotException(String msg, Exception e) { + super(msg, e); + } + +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/CallQueue.java b/src/main/java/org/apache/hadoop/hbase/thrift/CallQueue.java new file mode 100644 index 0000000..5bd47a8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/CallQueue.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.thrift; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * A BlockingQueue reports waiting time in queue and queue length to + * ThriftMetrics. + */ +public class CallQueue implements BlockingQueue { + private static Log LOG = LogFactory.getLog(CallQueue.class); + + private final BlockingQueue underlyingQueue; + private final ThriftMetrics metrics; + + public CallQueue(BlockingQueue underlyingQueue, + ThriftMetrics metrics) { + this.underlyingQueue = underlyingQueue; + this.metrics = metrics; + } + + private static long now() { + return System.nanoTime(); + } + + public static class Call implements Runnable { + final long startTime; + final Runnable underlyingRunnable; + + Call(Runnable underlyingRunnable) { + this.underlyingRunnable = underlyingRunnable; + this.startTime = now(); + } + + @Override + public void run() { + underlyingRunnable.run(); + } + + public long timeInQueue() { + return now() - startTime; + } + + @Override + public boolean equals(Object other) { + if (other instanceof Call) { + Call otherCall = (Call)(other); + return this.underlyingRunnable.equals(otherCall.underlyingRunnable); + } else if (other instanceof Runnable) { + return this.underlyingRunnable.equals(other); + } + return false; + } + + @Override + public int hashCode() { + return this.underlyingRunnable.hashCode(); + } + } + + @Override + public Runnable poll() { + Call result = underlyingQueue.poll(); + updateMetrics(result); + return result; + } + + private void updateMetrics(Call result) { + if (result == null) { + return; + } + metrics.incTimeInQueue(result.timeInQueue()); + metrics.setCallQueueLen(this.size()); + } + + @Override + public Runnable poll(long timeout, TimeUnit unit) throws InterruptedException { + Call result = underlyingQueue.poll(timeout, unit); + updateMetrics(result); + return result; + } + + @Override + public Runnable remove() { + Call result = underlyingQueue.remove(); + updateMetrics(result); + return result; + } + + @Override + public Runnable take() throws InterruptedException { + Call result = underlyingQueue.take(); + updateMetrics(result); + return result; + } + + @Override + public int drainTo(Collection destination) { + return drainTo(destination, Integer.MAX_VALUE); + } + + @Override + public int drainTo(Collection destination, + int maxElements) { + if (destination == this) { + throw new IllegalArgumentException( + "A BlockingQueue cannot drain to itself."); + } + List drained = new ArrayList(); + underlyingQueue.drainTo(drained, maxElements); + for (Call r : drained) { + updateMetrics(r); + } + destination.addAll(drained); + int sz = drained.size(); + LOG.info("Elements drained: " + sz); + return sz; + } + + + @Override + public boolean offer(Runnable element) { + return underlyingQueue.offer(new Call(element)); + } + + @Override + public boolean offer(Runnable element, long timeout, TimeUnit unit) + throws InterruptedException { + return underlyingQueue.offer(new Call(element), timeout, unit); + } + @Override + public void put(Runnable element) throws InterruptedException { + underlyingQueue.put(new Call(element)); + } + + @Override + public boolean add(Runnable element) { + return underlyingQueue.add(new Call(element)); + } + + @Override + public boolean addAll(Collection elements) { + int added = 0; + for (Runnable r : elements) { + added += underlyingQueue.add(new Call(r)) ? 1 : 0; + } + return added != 0; + } + + @Override + public Runnable element() { + return underlyingQueue.element(); + } + + @Override + public Runnable peek() { + return underlyingQueue.peek(); + } + + @Override + public void clear() { + underlyingQueue.clear(); + } + + @Override + public boolean containsAll(Collection elements) { + return underlyingQueue.containsAll(elements); + } + + @Override + public boolean isEmpty() { + return underlyingQueue.isEmpty(); + } + + @Override + public Iterator iterator() { + return new Iterator() { + final Iterator underlyingIterator = underlyingQueue.iterator(); + @Override + public Runnable next() { + return underlyingIterator.next(); + } + + @Override + public boolean hasNext() { + return underlyingIterator.hasNext(); + } + + @Override + public void remove() { + underlyingIterator.remove(); + } + }; + } + + @Override + public boolean removeAll(Collection elements) { + return underlyingQueue.removeAll(elements); + } + + @Override + public boolean retainAll(Collection elements) { + return underlyingQueue.retainAll(elements); + } + + @Override + public int size() { + return underlyingQueue.size(); + } + + @Override + public Object[] toArray() { + return underlyingQueue.toArray(); + } + + @Override + public T[] toArray(T[] array) { + return underlyingQueue.toArray(array); + } + + @Override + public boolean contains(Object element) { + return underlyingQueue.contains(element); + } + + @Override + public int remainingCapacity() { + return underlyingQueue.remainingCapacity(); + } + + @Override + public boolean remove(Object element) { + return underlyingQueue.remove(element); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/HThreadedSelectorServerArgs.java b/src/main/java/org/apache/hadoop/hbase/thrift/HThreadedSelectorServerArgs.java new file mode 100644 index 0000000..494f5a5 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/HThreadedSelectorServerArgs.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.thrift; + +import org.apache.hadoop.conf.Configuration; +import org.apache.thrift.server.TThreadedSelectorServer; +import org.apache.thrift.transport.TNonblockingServerTransport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A TThreadedSelectorServer.Args that reads hadoop configuration + */ +public class HThreadedSelectorServerArgs extends TThreadedSelectorServer.Args { + + private static final Logger LOG = + LoggerFactory.getLogger(TThreadedSelectorServer.class); + + /** + * Number of selector threads for reading and writing socket + */ + public static final String SELECTOR_THREADS_CONF_KEY = + "hbase.thrift.selector.threads"; + + /** + * Number fo threads for processing the thrift calls + */ + public static final String WORKER_THREADS_CONF_KEY = + "hbase.thrift.worker.threads"; + + /** + * Time to wait for server to stop gracefully + */ + public static final String STOP_TIMEOUT_CONF_KEY = + "hbase.thrift.stop.timeout.seconds"; + + /** + * Maximum number of accepted elements per selector + */ + public static final String ACCEPT_QUEUE_SIZE_PER_THREAD_CONF_KEY = + "hbase.thrift.accept.queue.size.per.selector"; + + /** + * The strategy for handling new accepted connections. + */ + public static final String ACCEPT_POLICY_CONF_KEY = + "hbase.thrift.accept.policy"; + + public HThreadedSelectorServerArgs( + TNonblockingServerTransport transport, Configuration conf) { + super(transport); + readConf(conf); + } + + private void readConf(Configuration conf) { + int selectorThreads = conf.getInt( + SELECTOR_THREADS_CONF_KEY, getSelectorThreads()); + int workerThreads = conf.getInt( + WORKER_THREADS_CONF_KEY, getWorkerThreads()); + int stopTimeoutVal = conf.getInt( + STOP_TIMEOUT_CONF_KEY, getStopTimeoutVal()); + int acceptQueueSizePerThread = conf.getInt( + ACCEPT_QUEUE_SIZE_PER_THREAD_CONF_KEY, getAcceptQueueSizePerThread()); + AcceptPolicy acceptPolicy = AcceptPolicy.valueOf(conf.get( + ACCEPT_POLICY_CONF_KEY, getAcceptPolicy().toString()).toUpperCase()); + + super.selectorThreads(selectorThreads) + .workerThreads(workerThreads) + .stopTimeoutVal(stopTimeoutVal) + .acceptQueueSizePerThread(acceptQueueSizePerThread) + .acceptPolicy(acceptPolicy); + + LOG.info("Read configuration selectorThreads:" + selectorThreads + + " workerThreads:" + workerThreads + + " stopTimeoutVal:" + stopTimeoutVal + "sec" + + " acceptQueueSizePerThread:" + acceptQueueSizePerThread + + " acceptPolicy:" + acceptPolicy); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/HbaseHandlerMetricsProxy.java b/src/main/java/org/apache/hadoop/hbase/thrift/HbaseHandlerMetricsProxy.java new file mode 100644 index 0000000..22fe431 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/HbaseHandlerMetricsProxy.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.thrift; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.thrift.generated.Hbase; + + +/** + * Converts a Hbase.Iface using InvocationHandler so that it reports process + * time of each call to ThriftMetrics. + */ +public class HbaseHandlerMetricsProxy implements InvocationHandler { + + public static final Log LOG = LogFactory.getLog( + HbaseHandlerMetricsProxy.class); + + private final Hbase.Iface handler; + private final ThriftMetrics metrics; + + public static Hbase.Iface newInstance(Hbase.Iface handler, + ThriftMetrics metrics, + Configuration conf) { + return (Hbase.Iface) Proxy.newProxyInstance( + handler.getClass().getClassLoader(), + new Class[]{Hbase.Iface.class}, + new HbaseHandlerMetricsProxy(handler, metrics, conf)); + } + + private HbaseHandlerMetricsProxy( + Hbase.Iface handler, ThriftMetrics metrics, Configuration conf) { + this.handler = handler; + this.metrics = metrics; + } + + @Override + public Object invoke(Object proxy, Method m, Object[] args) + throws Throwable { + Object result; + try { + long start = now(); + result = m.invoke(handler, args); + int processTime = (int)(now() - start); + metrics.incMethodTime(m.getName(), processTime); + } catch (InvocationTargetException e) { + throw e.getTargetException(); + } catch (Exception e) { + throw new RuntimeException( + "unexpected invocation exception: " + e.getMessage()); + } + return result; + } + + private static long now() { + return System.nanoTime(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescer.java b/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescer.java new file mode 100644 index 0000000..c0205ba --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescer.java @@ -0,0 +1,370 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.thrift; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.thrift.ThriftServerRunner.HBaseHandler; +import org.apache.hadoop.hbase.thrift.generated.TIncrement; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.metrics.util.MBeanUtil; +import org.apache.thrift.TException; + +/** + * This class will coalesce increments from a thift server if + * hbase.regionserver.thrift.coalesceIncrement is set to true. Turning this + * config to true will cause the thrift server to queue increments into an + * instance of this class. The thread pool associated with this class will drain + * the coalesced increments as the thread is able. This can cause data loss if the + * thrift server dies or is shut down before everything in the queue is drained. + * + */ +public class IncrementCoalescer implements IncrementCoalescerMBean { + + /** + * Used to identify a cell that will be incremented. + * + */ + static class FullyQualifiedRow { + private byte[] table; + private byte[] rowKey; + private byte[] family; + private byte[] qualifier; + + public FullyQualifiedRow(byte[] table, byte[] rowKey, byte[] fam, byte[] qual) { + super(); + this.table = table; + this.rowKey = rowKey; + this.family = fam; + this.qualifier = qual; + } + + public byte[] getTable() { + return table; + } + + public void setTable(byte[] table) { + this.table = table; + } + + public byte[] getRowKey() { + return rowKey; + } + + public void setRowKey(byte[] rowKey) { + this.rowKey = rowKey; + } + + public byte[] getFamily() { + return family; + } + + public void setFamily(byte[] fam) { + this.family = fam; + } + + public byte[] getQualifier() { + return qualifier; + } + + public void setQualifier(byte[] qual) { + this.qualifier = qual; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(family); + result = prime * result + Arrays.hashCode(qualifier); + result = prime * result + Arrays.hashCode(rowKey); + result = prime * result + Arrays.hashCode(table); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + FullyQualifiedRow other = (FullyQualifiedRow) obj; + if (!Arrays.equals(family, other.family)) return false; + if (!Arrays.equals(qualifier, other.qualifier)) return false; + if (!Arrays.equals(rowKey, other.rowKey)) return false; + if (!Arrays.equals(table, other.table)) return false; + return true; + } + + } + + static class DaemonThreadFactory implements ThreadFactory { + static final AtomicInteger poolNumber = new AtomicInteger(1); + final ThreadGroup group; + final AtomicInteger threadNumber = new AtomicInteger(1); + final String namePrefix; + + DaemonThreadFactory() { + SecurityManager s = System.getSecurityManager(); + group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup(); + namePrefix = "ICV-" + poolNumber.getAndIncrement() + "-thread-"; + } + + public Thread newThread(Runnable r) { + Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); + if (!t.isDaemon()) t.setDaemon(true); + if (t.getPriority() != Thread.NORM_PRIORITY) t.setPriority(Thread.NORM_PRIORITY); + return t; + } + } + + private final AtomicLong failedIncrements = new AtomicLong(); + private final AtomicLong successfulCoalescings = new AtomicLong(); + private final AtomicLong totalIncrements = new AtomicLong(); + private final ConcurrentMap countersMap = + new ConcurrentHashMap(100000, 0.75f, 1500); + private final ThreadPoolExecutor pool; + private final HBaseHandler handler; + + private int maxQueueSize = 500000; + private static final int CORE_POOL_SIZE = 1; + + protected final Log LOG = LogFactory.getLog(this.getClass().getName()); + + @SuppressWarnings("deprecation") + public IncrementCoalescer(HBaseHandler hand) { + this.handler = hand; + LinkedBlockingQueue queue = new LinkedBlockingQueue(); + pool = + new ThreadPoolExecutor(CORE_POOL_SIZE, CORE_POOL_SIZE, 50, TimeUnit.MILLISECONDS, queue, + new DaemonThreadFactory()); + + MBeanUtil.registerMBean("thrift", "Thrift", this); + } + + public boolean queueIncrement(TIncrement inc) throws TException { + if (!canQueue()) { + failedIncrements.incrementAndGet(); + return false; + } + return internalQueueTincrement(inc); + } + + public boolean queueIncrements(List incs) throws TException { + if (!canQueue()) { + failedIncrements.incrementAndGet(); + return false; + } + + for (TIncrement tinc : incs) { + internalQueueTincrement(tinc); + } + return true; + + } + + private boolean internalQueueTincrement(TIncrement inc) throws TException { + byte[][] famAndQf = KeyValue.parseColumn(inc.getColumn()); + if (famAndQf.length < 1) return false; + byte[] qual = famAndQf.length == 1 ? new byte[0] : famAndQf[1]; + + return internalQueueIncrement(inc.getTable(), inc.getRow(), famAndQf[0], qual, + inc.getAmmount()); + + } + + private boolean internalQueueIncrement(byte[] tableName, byte[] rowKey, byte[] fam, + byte[] qual, long ammount) throws TException { + int countersMapSize = countersMap.size(); + + + //Make sure that the number of threads is scaled. + dynamicallySetCoreSize(countersMapSize); + + totalIncrements.incrementAndGet(); + + FullyQualifiedRow key = new FullyQualifiedRow(tableName, rowKey, fam, qual); + + long currentAmount = ammount; + // Spin until able to insert the value back without collisions + while (true) { + Long value = countersMap.remove(key); + if (value == null) { + // There was nothing there, create a new value + value = new Long(currentAmount); + } else { + value += currentAmount; + successfulCoalescings.incrementAndGet(); + } + // Try to put the value, only if there was none + Long oldValue = countersMap.putIfAbsent(key, value); + if (oldValue == null) { + // We were able to put it in, we're done + break; + } + // Someone else was able to put a value in, so let's remember our + // current value (plus what we picked up) and retry to add it in + currentAmount = value; + } + + // We limit the size of the queue simply because all we need is a + // notification that something needs to be incremented. No need + // for millions of callables that mean the same thing. + if (pool.getQueue().size() <= 1000) { + // queue it up + Callable callable = createIncCallable(); + pool.submit(callable); + } + + return true; + } + + public boolean canQueue() { + return countersMap.size() < maxQueueSize; + } + + private Callable createIncCallable() { + return new Callable() { + @Override + public Integer call() throws Exception { + int failures = 0; + Set keys = countersMap.keySet(); + for (FullyQualifiedRow row : keys) { + Long counter = countersMap.remove(row); + if (counter == null) { + continue; + } + try { + HTable table = handler.getTable(row.getTable()); + if (failures > 2) { + throw new IOException("Auto-Fail rest of ICVs"); + } + table.incrementColumnValue(row.getRowKey(), row.getFamily(), row.getQualifier(), + counter); + } catch (IOException e) { + // log failure of increment + failures++; + LOG.error("FAILED_ICV: " + Bytes.toString(row.getTable()) + ", " + + Bytes.toStringBinary(row.getRowKey()) + ", " + + Bytes.toStringBinary(row.getFamily()) + ", " + + Bytes.toStringBinary(row.getQualifier()) + ", " + counter); + } + + } + return failures; + } + }; + } + + /** + * This method samples the incoming requests and, if selected, will check if + * the corePoolSize should be changed. + * @param countersMapSize + */ + private void dynamicallySetCoreSize(int countersMapSize) { + // Here we are using countersMapSize as a random number, meaning this + // could be a Random object + if (countersMapSize % 10 != 0) { + return; + } + double currentRatio = (double) countersMapSize / (double) maxQueueSize; + int newValue = 1; + if (currentRatio < 0.1) { + // it's 1 + } else if (currentRatio < 0.3) { + newValue = 2; + } else if (currentRatio < 0.5) { + newValue = 4; + } else if (currentRatio < 0.7) { + newValue = 8; + } else if (currentRatio < 0.9) { + newValue = 14; + } else { + newValue = 22; + } + if (pool.getCorePoolSize() != newValue) { + pool.setCorePoolSize(newValue); + } + } + + // MBean get/set methods + public int getQueueSize() { + return pool.getQueue().size(); + } + public int getMaxQueueSize() { + return this.maxQueueSize; + } + public void setMaxQueueSize(int newSize) { + this.maxQueueSize = newSize; + } + + public long getPoolCompletedTaskCount() { + return pool.getCompletedTaskCount(); + } + public long getPoolTaskCount() { + return pool.getTaskCount(); + } + public int getPoolLargestPoolSize() { + return pool.getLargestPoolSize(); + } + public int getCorePoolSize() { + return pool.getCorePoolSize(); + } + public void setCorePoolSize(int newCoreSize) { + pool.setCorePoolSize(newCoreSize); + } + public int getMaxPoolSize() { + return pool.getMaximumPoolSize(); + } + public void setMaxPoolSize(int newMaxSize) { + pool.setMaximumPoolSize(newMaxSize); + } + public long getFailedIncrements() { + return failedIncrements.get(); + } + + public long getSuccessfulCoalescings() { + return successfulCoalescings.get(); + } + + public long getTotalIncrements() { + return totalIncrements.get(); + } + + public long getCountersMapSize() { + return countersMap.size(); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescerMBean.java b/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescerMBean.java new file mode 100644 index 0000000..c378368 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescerMBean.java @@ -0,0 +1,50 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.thrift; + +public interface IncrementCoalescerMBean { + public int getQueueSize(); + + public int getMaxQueueSize(); + + public void setMaxQueueSize(int newSize); + + public long getPoolCompletedTaskCount(); + + public long getPoolTaskCount(); + + public int getPoolLargestPoolSize(); + + public int getCorePoolSize(); + + public void setCorePoolSize(int newCoreSize); + + public int getMaxPoolSize(); + + public void setMaxPoolSize(int newMaxSize); + + public long getFailedIncrements(); + + public long getSuccessfulCoalescings(); + + public long getTotalIncrements(); + + public long getCountersMapSize(); +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java b/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java new file mode 100644 index 0000000..c7e104b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.thrift; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.thrift.CallQueue.Call; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.server.TServer; +import org.apache.thrift.server.TThreadPoolServer; +import org.apache.thrift.transport.TServerTransport; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * A bounded thread pool server customized for HBase. + */ +public class TBoundedThreadPoolServer extends TServer { + + private static final String QUEUE_FULL_MSG = + "Queue is full, closing connection"; + + /** + * The "core size" of the thread pool. New threads are created on every + * connection until this many threads are created. + */ + public static final String MIN_WORKER_THREADS_CONF_KEY = + "hbase.thrift.minWorkerThreads"; + + /** + * This default core pool size should be enough for many test scenarios. We + * want to override this with a much larger number (e.g. at least 200) for a + * large-scale production setup. + */ + public static final int DEFAULT_MIN_WORKER_THREADS = 16; + + /** + * The maximum size of the thread pool. When the pending request queue + * overflows, new threads are created until their number reaches this number. + * After that, the server starts dropping connections. + */ + public static final String MAX_WORKER_THREADS_CONF_KEY = + "hbase.thrift.maxWorkerThreads"; + + public static final int DEFAULT_MAX_WORKER_THREADS = 1000; + + /** + * The maximum number of pending connections waiting in the queue. If there + * are no idle threads in the pool, the server queues requests. Only when + * the queue overflows, new threads are added, up to + * hbase.thrift.maxQueuedRequests threads. + */ + public static final String MAX_QUEUED_REQUESTS_CONF_KEY = + "hbase.thrift.maxQueuedRequests"; + + public static final int DEFAULT_MAX_QUEUED_REQUESTS = 1000; + + /** + * Default amount of time in seconds to keep a thread alive. Worker threads + * are stopped after being idle for this long. + */ + public static final String THREAD_KEEP_ALIVE_TIME_SEC_CONF_KEY = + "hbase.thrift.threadKeepAliveTimeSec"; + + private static final int DEFAULT_THREAD_KEEP_ALIVE_TIME_SEC = 60; + + /** + * Time to wait after interrupting all worker threads. This is after a clean + * shutdown has been attempted. + */ + public static final int TIME_TO_WAIT_AFTER_SHUTDOWN_MS = 5000; + + private static final Log LOG = LogFactory.getLog( + TBoundedThreadPoolServer.class.getName()); + + private final CallQueue callQueue; + + public static class Args extends TThreadPoolServer.Args { + int maxQueuedRequests; + int threadKeepAliveTimeSec; + + public Args(TServerTransport transport, Configuration conf) { + super(transport); + minWorkerThreads = conf.getInt(MIN_WORKER_THREADS_CONF_KEY, + DEFAULT_MIN_WORKER_THREADS); + maxWorkerThreads = conf.getInt(MAX_WORKER_THREADS_CONF_KEY, + DEFAULT_MAX_WORKER_THREADS); + maxQueuedRequests = conf.getInt(MAX_QUEUED_REQUESTS_CONF_KEY, + DEFAULT_MAX_QUEUED_REQUESTS); + threadKeepAliveTimeSec = conf.getInt(THREAD_KEEP_ALIVE_TIME_SEC_CONF_KEY, + DEFAULT_THREAD_KEEP_ALIVE_TIME_SEC); + } + + @Override + public String toString() { + return "min worker threads=" + minWorkerThreads + + ", max worker threads=" + maxWorkerThreads + + ", max queued requests=" + maxQueuedRequests; + } + } + + /** Executor service for handling client connections */ + private ExecutorService executorService; + + /** Flag for stopping the server */ + private volatile boolean stopped; + + private Args serverOptions; + + public TBoundedThreadPoolServer(Args options, ThriftMetrics metrics) { + super(options); + + if (options.maxQueuedRequests > 0) { + this.callQueue = new CallQueue( + new LinkedBlockingQueue(options.maxQueuedRequests), metrics); + } else { + this.callQueue = new CallQueue(new SynchronousQueue(), metrics); + } + + ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); + tfb.setDaemon(true); + tfb.setNameFormat("thrift-worker-%d"); + executorService = + new ThreadPoolExecutor(options.minWorkerThreads, + options.maxWorkerThreads, options.threadKeepAliveTimeSec, + TimeUnit.SECONDS, this.callQueue, tfb.build()); + serverOptions = options; + } + + public void serve() { + try { + serverTransport_.listen(); + } catch (TTransportException ttx) { + LOG.error("Error occurred during listening.", ttx); + return; + } + + Runtime.getRuntime().addShutdownHook( + new Thread(getClass().getSimpleName() + "-shutdown-hook") { + @Override + public void run() { + TBoundedThreadPoolServer.this.stop(); + } + }); + + stopped = false; + while (!stopped && !Thread.interrupted()) { + TTransport client = null; + try { + client = serverTransport_.accept(); + } catch (TTransportException ttx) { + if (!stopped) { + LOG.warn("Transport error when accepting message", ttx); + continue; + } else { + // The server has been stopped + break; + } + } + + ClientConnnection command = new ClientConnnection(client); + try { + executorService.execute(command); + } catch (RejectedExecutionException rex) { + if (client.getClass() == TSocket.class) { + // We expect the client to be TSocket. + LOG.warn(QUEUE_FULL_MSG + " from " + + ((TSocket) client).getSocket().getRemoteSocketAddress()); + } else { + LOG.warn(QUEUE_FULL_MSG, rex); + } + client.close(); + } + } + + shutdownServer(); + } + + /** + * Loop until {@link ExecutorService#awaitTermination} finally does return + * without an interrupted exception. If we don't do this, then we'll shut + * down prematurely. We want to let the executor service clear its task + * queue, closing client sockets appropriately. + */ + private void shutdownServer() { + executorService.shutdown(); + + long msLeftToWait = + serverOptions.stopTimeoutUnit.toMillis(serverOptions.stopTimeoutVal); + long timeMillis = System.currentTimeMillis(); + + LOG.info("Waiting for up to " + msLeftToWait + " ms to finish processing" + + " pending requests"); + boolean interrupted = false; + while (msLeftToWait >= 0) { + try { + executorService.awaitTermination(msLeftToWait, TimeUnit.MILLISECONDS); + break; + } catch (InterruptedException ix) { + long timePassed = System.currentTimeMillis() - timeMillis; + msLeftToWait -= timePassed; + timeMillis += timePassed; + interrupted = true; + } + } + + LOG.info("Interrupting all worker threads and waiting for " + + TIME_TO_WAIT_AFTER_SHUTDOWN_MS + " ms longer"); + + // This will interrupt all the threads, even those running a task. + executorService.shutdownNow(); + Threads.sleepWithoutInterrupt(TIME_TO_WAIT_AFTER_SHUTDOWN_MS); + + // Preserve the interrupted status. + if (interrupted) { + Thread.currentThread().interrupt(); + } + LOG.info("Thrift server shutdown complete"); + } + + @Override + public void stop() { + stopped = true; + serverTransport_.interrupt(); + } + + private class ClientConnnection implements Runnable { + + private TTransport client; + + /** + * Default constructor. + * + * @param client Transport to process + */ + private ClientConnnection(TTransport client) { + this.client = client; + } + + /** + * Loops on processing a client forever + */ + public void run() { + TProcessor processor = null; + TTransport inputTransport = null; + TTransport outputTransport = null; + TProtocol inputProtocol = null; + TProtocol outputProtocol = null; + try { + processor = processorFactory_.getProcessor(client); + inputTransport = inputTransportFactory_.getTransport(client); + outputTransport = outputTransportFactory_.getTransport(client); + inputProtocol = inputProtocolFactory_.getProtocol(inputTransport); + outputProtocol = outputProtocolFactory_.getProtocol(outputTransport); + // we check stopped_ first to make sure we're not supposed to be shutting + // down. this is necessary for graceful shutdown. + while (!stopped && processor.process(inputProtocol, outputProtocol)) {} + } catch (TTransportException ttx) { + // Assume the client died and continue silently + } catch (TException tx) { + LOG.error("Thrift error occurred during processing of message.", tx); + } catch (Exception x) { + LOG.error("Error occurred during processing of message.", x); + } + + if (inputTransport != null) { + inputTransport.close(); + } + + if (outputTransport != null) { + outputTransport.close(); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java new file mode 100644 index 0000000..f82d0c0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java @@ -0,0 +1,144 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.thrift; + +import java.lang.reflect.Method; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.thrift.generated.Hbase; +import org.apache.hadoop.metrics.MetricsContext; +import org.apache.hadoop.metrics.MetricsRecord; +import org.apache.hadoop.metrics.MetricsUtil; +import org.apache.hadoop.metrics.Updater; +import org.apache.hadoop.metrics.util.MetricsBase; +import org.apache.hadoop.metrics.util.MetricsIntValue; +import org.apache.hadoop.metrics.util.MetricsRegistry; +import org.apache.hadoop.metrics.util.MetricsTimeVaryingInt; +import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong; +import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate; + +/** + * This class is for maintaining the various statistics of thrift server + * and publishing them through the metrics interfaces. + */ +public class ThriftMetrics implements Updater { + public final static Log LOG = LogFactory.getLog(ThriftMetrics.class); + public final static String CONTEXT_NAME = "thriftserver"; + + private final MetricsContext context; + private final MetricsRecord metricsRecord; + private final MetricsRegistry registry = new MetricsRegistry(); + private final long slowResponseTime; + public static final String SLOW_RESPONSE_NANO_SEC = + "hbase.thrift.slow.response.nano.second"; + public static final long DEFAULT_SLOW_RESPONSE_NANO_SEC = 10 * 1000 * 1000; + + private final MetricsIntValue callQueueLen = + new MetricsIntValue("callQueueLen", registry); + private final MetricsTimeVaryingRate numRowKeysInBatchGet = + new MetricsTimeVaryingRate("numRowKeysInBatchGet", registry); + private final MetricsTimeVaryingRate numRowKeysInBatchMutate = + new MetricsTimeVaryingRate("numRowKeysInBatchMutate", registry); + private final MetricsTimeVaryingRate timeInQueue = + new MetricsTimeVaryingRate("timeInQueue", registry); + private MetricsTimeVaryingRate thriftCall = + new MetricsTimeVaryingRate("thriftCall", registry); + private MetricsTimeVaryingRate slowThriftCall = + new MetricsTimeVaryingRate("slowThriftCall", registry); + + public ThriftMetrics(int port, Configuration conf, Class iface) { + slowResponseTime = conf.getLong( + SLOW_RESPONSE_NANO_SEC, DEFAULT_SLOW_RESPONSE_NANO_SEC); + context = MetricsUtil.getContext(CONTEXT_NAME); + metricsRecord = MetricsUtil.createRecord(context, CONTEXT_NAME); + + metricsRecord.setTag("port", port + ""); + + LOG.info("Initializing RPC Metrics with port=" + port); + + context.registerUpdater(this); + + createMetricsForMethods(iface); + } + + public void incTimeInQueue(long time) { + timeInQueue.inc(time); + } + + public void setCallQueueLen(int len) { + callQueueLen.set(len); + } + + public void incNumRowKeysInBatchGet(int diff) { + numRowKeysInBatchGet.inc(diff); + } + + public void incNumRowKeysInBatchMutate(int diff) { + numRowKeysInBatchMutate.inc(diff); + } + + public void incMethodTime(String name, int time) { + MetricsTimeVaryingRate methodTimeMetrc = getMethodTimeMetrics(name); + if (methodTimeMetrc == null) { + LOG.warn( + "Got incMethodTime() request for method that doesnt exist: " + name); + return; // ignore methods that dont exist. + } + + // inc method specific processTime + methodTimeMetrc.inc(time); + + // inc general processTime + thriftCall.inc(time); + if (time > slowResponseTime) { + slowThriftCall.inc(time); + } + } + + private void createMetricsForMethods(Class iface) { + LOG.debug("Creating metrics for interface " + iface.toString()); + for (Method m : iface.getDeclaredMethods()) { + if (getMethodTimeMetrics(m.getName()) == null) + LOG.debug("Creating metrics for method:" + m.getName()); + createMethodTimeMetrics(m.getName()); + } + } + + private MetricsTimeVaryingRate getMethodTimeMetrics(String key) { + return (MetricsTimeVaryingRate) registry.get(key); + } + + private MetricsTimeVaryingRate createMethodTimeMetrics(String key) { + return new MetricsTimeVaryingRate(key, this.registry); + } + + /** + * Push the metrics to the monitoring subsystem on doUpdate() call. + */ + public void doUpdates(final MetricsContext context) { + // getMetricsList() and pushMetric() are thread safe methods + for (MetricsBase m : registry.getMetricsList()) { + m.pushMetric(metricsRecord); + } + metricsRecord.update(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java new file mode 100644 index 0000000..43da992 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java @@ -0,0 +1,242 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.thrift; + +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.thrift.ThriftServerRunner.ImplType; +import org.apache.hadoop.hbase.util.InfoServer; +import org.apache.hadoop.hbase.util.Strings; +import org.apache.hadoop.hbase.util.VersionInfo; +import org.apache.hadoop.net.DNS; +import org.apache.hadoop.util.Shell.ExitCodeException; + +/** + * ThriftServer- this class starts up a Thrift server which implements the + * Hbase API specified in the Hbase.thrift IDL file. The server runs in an + * independent process. + */ +public class ThriftServer { + + private static final Log LOG = LogFactory.getLog(ThriftServer.class); + + private static final String MIN_WORKERS_OPTION = "minWorkers"; + private static final String MAX_WORKERS_OPTION = "workers"; + private static final String MAX_QUEUE_SIZE_OPTION = "queue"; + private static final String KEEP_ALIVE_SEC_OPTION = "keepAliveSec"; + static final String BIND_OPTION = "bind"; + static final String COMPACT_OPTION = "compact"; + static final String FRAMED_OPTION = "framed"; + static final String PORT_OPTION = "port"; + + private static final String DEFAULT_BIND_ADDR = "0.0.0.0"; + private static final int DEFAULT_LISTEN_PORT = 9090; + + private Configuration conf; + ThriftServerRunner serverRunner; + + private InfoServer infoServer; + + // + // Main program and support routines + // + + public ThriftServer(Configuration conf) { + this.conf = HBaseConfiguration.create(conf); + } + + private static void printUsageAndExit(Options options, int exitCode) + throws ExitCodeException { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("Thrift", null, options, + "To start the Thrift server run 'bin/hbase-daemon.sh start thrift'\n" + + "To shutdown the thrift server run 'bin/hbase-daemon.sh stop " + + "thrift' or send a kill signal to the thrift server pid", + true); + throw new ExitCodeException(exitCode, ""); + } + + /** + * Start up or shuts down the Thrift server, depending on the arguments. + * @param args + */ + void doMain(final String[] args) throws Exception { + processOptions(args); + // login the server principal (if using secure Hadoop) + if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) { + String machineName = Strings.domainNamePointerToHostName( + DNS.getDefaultHost(conf.get("hbase.thrift.dns.interface", "default"), + conf.get("hbase.thrift.dns.nameserver", "default"))); + User.login(conf, "hbase.thrift.keytab.file", + "hbase.thrift.kerberos.principal", machineName); + } + serverRunner = new ThriftServerRunner(conf); + + // Put up info server. + int port = conf.getInt("hbase.thrift.info.port", 9095); + if (port >= 0) { + conf.setLong("startcode", System.currentTimeMillis()); + String a = conf.get("hbase.thrift.info.bindAddress", "0.0.0.0"); + infoServer = new InfoServer("thrift", a, port, false, conf); + infoServer.setAttribute("hbase.conf", conf); + infoServer.start(); + } + serverRunner.run(); + } + + /** + * Parse the command line options to set parameters the conf. + */ + private void processOptions(final String[] args) throws Exception { + Options options = new Options(); + options.addOption("b", BIND_OPTION, true, "Address to bind " + + "the Thrift server to. Not supported by the Nonblocking and " + + "HsHa server [default: " + DEFAULT_BIND_ADDR + "]"); + options.addOption("p", PORT_OPTION, true, "Port to bind to [default: " + + DEFAULT_LISTEN_PORT + "]"); + options.addOption("f", FRAMED_OPTION, false, "Use framed transport"); + options.addOption("c", COMPACT_OPTION, false, "Use the compact protocol"); + options.addOption("h", "help", false, "Print help information"); + options.addOption(null, "infoport", true, "Port for web UI"); + + options.addOption("m", MIN_WORKERS_OPTION, true, + "The minimum number of worker threads for " + + ImplType.THREAD_POOL.simpleClassName()); + + options.addOption("w", MAX_WORKERS_OPTION, true, + "The maximum number of worker threads for " + + ImplType.THREAD_POOL.simpleClassName()); + + options.addOption("q", MAX_QUEUE_SIZE_OPTION, true, + "The maximum number of queued requests in " + + ImplType.THREAD_POOL.simpleClassName()); + + options.addOption("k", KEEP_ALIVE_SEC_OPTION, true, + "The amount of time in secods to keep a thread alive when idle in " + + ImplType.THREAD_POOL.simpleClassName()); + + options.addOptionGroup(ImplType.createOptionGroup()); + + CommandLineParser parser = new PosixParser(); + CommandLine cmd = parser.parse(options, args); + + // This is so complicated to please both bin/hbase and bin/hbase-daemon. + // hbase-daemon provides "start" and "stop" arguments + // hbase should print the help if no argument is provided + List commandLine = Arrays.asList(args); + boolean stop = commandLine.contains("stop"); + boolean start = commandLine.contains("start"); + boolean invalidStartStop = (start && stop) || (!start && !stop); + if (cmd.hasOption("help") || invalidStartStop) { + if (invalidStartStop) { + LOG.error("Exactly one of 'start' and 'stop' has to be specified"); + } + printUsageAndExit(options, 1); + } + + // Get port to bind to + try { + int listenPort = Integer.parseInt(cmd.getOptionValue(PORT_OPTION, + String.valueOf(DEFAULT_LISTEN_PORT))); + conf.setInt(ThriftServerRunner.PORT_CONF_KEY, listenPort); + } catch (NumberFormatException e) { + LOG.error("Could not parse the value provided for the port option", e); + printUsageAndExit(options, -1); + } + + // check for user-defined info server port setting, if so override the conf + try { + if (cmd.hasOption("infoport")) { + String val = cmd.getOptionValue("infoport"); + conf.setInt("hbase.thrift.info.port", Integer.valueOf(val)); + LOG.debug("Web UI port set to " + val); + } + } catch (NumberFormatException e) { + LOG.error("Could not parse the value provided for the infoport option", e); + printUsageAndExit(options, -1); + } + + // Make optional changes to the configuration based on command-line options + optionToConf(cmd, MIN_WORKERS_OPTION, + conf, TBoundedThreadPoolServer.MIN_WORKER_THREADS_CONF_KEY); + optionToConf(cmd, MAX_WORKERS_OPTION, + conf, TBoundedThreadPoolServer.MAX_WORKER_THREADS_CONF_KEY); + optionToConf(cmd, MAX_QUEUE_SIZE_OPTION, + conf, TBoundedThreadPoolServer.MAX_QUEUED_REQUESTS_CONF_KEY); + optionToConf(cmd, KEEP_ALIVE_SEC_OPTION, + conf, TBoundedThreadPoolServer.THREAD_KEEP_ALIVE_TIME_SEC_CONF_KEY); + + // Set general thrift server options + conf.setBoolean( + ThriftServerRunner.COMPACT_CONF_KEY, cmd.hasOption(COMPACT_OPTION)); + conf.setBoolean( + ThriftServerRunner.FRAMED_CONF_KEY, cmd.hasOption(FRAMED_OPTION)); + if (cmd.hasOption(BIND_OPTION)) { + conf.set( + ThriftServerRunner.BIND_CONF_KEY, cmd.getOptionValue(BIND_OPTION)); + } + + ImplType.setServerImpl(cmd, conf); + } + + public void stop() { + if (this.infoServer != null) { + LOG.info("Stopping infoServer"); + try { + this.infoServer.stop(); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + serverRunner.shutdown(); + } + + private static void optionToConf(CommandLine cmd, String option, + Configuration conf, String destConfKey) { + if (cmd.hasOption(option)) { + String value = cmd.getOptionValue(option); + LOG.info("Set configuration key:" + destConfKey + " value:" + value); + conf.set(destConfKey, value); + } + } + + /** + * @param args + * @throws Exception + */ + public static void main(String [] args) throws Exception { + VersionInfo.logVersion(); + try { + new ThriftServer(HBaseConfiguration.create()).doMain(args); + } catch (ExitCodeException ex) { + System.exit(ex.getExitCode()); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java new file mode 100644 index 0000000..cf170e4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java @@ -0,0 +1,1451 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.thrift; + +import static org.apache.hadoop.hbase.util.Bytes.getBytes; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.OperationWithAttributes; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.ParseFilter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.WhileMatchFilter; +import org.apache.hadoop.hbase.thrift.CallQueue.Call; +import org.apache.hadoop.hbase.thrift.generated.AlreadyExists; +import org.apache.hadoop.hbase.thrift.generated.BatchMutation; +import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor; +import org.apache.hadoop.hbase.thrift.generated.Hbase; +import org.apache.hadoop.hbase.thrift.generated.IOError; +import org.apache.hadoop.hbase.thrift.generated.IllegalArgument; +import org.apache.hadoop.hbase.thrift.generated.Mutation; +import org.apache.hadoop.hbase.thrift.generated.TCell; +import org.apache.hadoop.hbase.thrift.generated.TIncrement; +import org.apache.hadoop.hbase.thrift.generated.TRegionInfo; +import org.apache.hadoop.hbase.thrift.generated.TRowResult; +import org.apache.hadoop.hbase.thrift.generated.TScan; +import org.apache.hadoop.hbase.util.Addressing; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.server.THsHaServer; +import org.apache.thrift.server.TNonblockingServer; +import org.apache.thrift.server.TServer; +import org.apache.thrift.server.TThreadedSelectorServer; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TNonblockingServerSocket; +import org.apache.thrift.transport.TNonblockingServerTransport; +import org.apache.thrift.transport.TServerSocket; +import org.apache.thrift.transport.TServerTransport; +import org.apache.thrift.transport.TTransportFactory; + +import com.google.common.base.Joiner; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * ThriftServerRunner - this class starts up a Thrift server which implements + * the Hbase API specified in the Hbase.thrift IDL file. + */ +public class ThriftServerRunner implements Runnable { + + private static final Log LOG = LogFactory.getLog(ThriftServerRunner.class); + + static final String SERVER_TYPE_CONF_KEY = + "hbase.regionserver.thrift.server.type"; + + static final String BIND_CONF_KEY = "hbase.regionserver.thrift.ipaddress"; + static final String COMPACT_CONF_KEY = "hbase.regionserver.thrift.compact"; + static final String FRAMED_CONF_KEY = "hbase.regionserver.thrift.framed"; + static final String PORT_CONF_KEY = "hbase.regionserver.thrift.port"; + static final String COALESCE_INC_KEY = "hbase.regionserver.thrift.coalesceIncrement"; + + private static final String DEFAULT_BIND_ADDR = "0.0.0.0"; + public static final int DEFAULT_LISTEN_PORT = 9090; + private final int listenPort; + + private Configuration conf; + volatile TServer tserver; + private final Hbase.Iface handler; + private final ThriftMetrics metrics; + + /** An enum of server implementation selections */ + enum ImplType { + HS_HA("hsha", true, THsHaServer.class, false), + NONBLOCKING("nonblocking", true, TNonblockingServer.class, false), + THREAD_POOL("threadpool", false, TBoundedThreadPoolServer.class, true), + THREADED_SELECTOR( + "threadedselector", true, TThreadedSelectorServer.class, false); + + public static final ImplType DEFAULT = THREAD_POOL; + + final String option; + final boolean isAlwaysFramed; + final Class serverClass; + final boolean canSpecifyBindIP; + + ImplType(String option, boolean isAlwaysFramed, + Class serverClass, boolean canSpecifyBindIP) { + this.option = option; + this.isAlwaysFramed = isAlwaysFramed; + this.serverClass = serverClass; + this.canSpecifyBindIP = canSpecifyBindIP; + } + + /** + * @return -option so we can get the list of options from + * {@link #values()} + */ + @Override + public String toString() { + return "-" + option; + } + + String getDescription() { + StringBuilder sb = new StringBuilder("Use the " + + serverClass.getSimpleName()); + if (isAlwaysFramed) { + sb.append(" This implies the framed transport."); + } + if (this == DEFAULT) { + sb.append("This is the default."); + } + return sb.toString(); + } + + static OptionGroup createOptionGroup() { + OptionGroup group = new OptionGroup(); + for (ImplType t : values()) { + group.addOption(new Option(t.option, t.getDescription())); + } + return group; + } + + static ImplType getServerImpl(Configuration conf) { + String confType = conf.get(SERVER_TYPE_CONF_KEY, THREAD_POOL.option); + for (ImplType t : values()) { + if (confType.equals(t.option)) { + return t; + } + } + throw new AssertionError("Unknown server ImplType.option:" + confType); + } + + static void setServerImpl(CommandLine cmd, Configuration conf) { + ImplType chosenType = null; + int numChosen = 0; + for (ImplType t : values()) { + if (cmd.hasOption(t.option)) { + chosenType = t; + ++numChosen; + } + } + if (numChosen < 1) { + LOG.info("Using default thrift server type"); + chosenType = DEFAULT; + } else if (numChosen > 1) { + throw new AssertionError("Exactly one option out of " + + Arrays.toString(values()) + " has to be specified"); + } + LOG.info("Using thrift server type " + chosenType.option); + conf.set(SERVER_TYPE_CONF_KEY, chosenType.option); + } + + public String simpleClassName() { + return serverClass.getSimpleName(); + } + + public static List serversThatCannotSpecifyBindIP() { + List l = new ArrayList(); + for (ImplType t : values()) { + if (!t.canSpecifyBindIP) { + l.add(t.simpleClassName()); + } + } + return l; + } + + } + + public ThriftServerRunner(Configuration conf) throws IOException { + this(conf, new ThriftServerRunner.HBaseHandler(conf)); + } + + public ThriftServerRunner(Configuration conf, HBaseHandler handler) { + this.conf = HBaseConfiguration.create(conf); + this.listenPort = conf.getInt(PORT_CONF_KEY, DEFAULT_LISTEN_PORT); + this.metrics = new ThriftMetrics(listenPort, conf, Hbase.Iface.class); + handler.initMetrics(metrics); + this.handler = HbaseHandlerMetricsProxy.newInstance(handler, metrics, conf); + } + + /* + * Runs the Thrift server + */ + @Override + public void run() { + try { + setupServer(); + tserver.serve(); + } catch (Exception e) { + LOG.fatal("Cannot run ThriftServer", e); + // Crash the process if the ThriftServer is not running + System.exit(-1); + } + } + + public void shutdown() { + if (tserver != null) { + tserver.stop(); + tserver = null; + } + } + + /** + * Setting up the thrift TServer + */ + private void setupServer() throws Exception { + // Construct correct ProtocolFactory + TProtocolFactory protocolFactory; + if (conf.getBoolean(COMPACT_CONF_KEY, false)) { + LOG.debug("Using compact protocol"); + protocolFactory = new TCompactProtocol.Factory(); + } else { + LOG.debug("Using binary protocol"); + protocolFactory = new TBinaryProtocol.Factory(); + } + + Hbase.Processor processor = + new Hbase.Processor(handler); + ImplType implType = ImplType.getServerImpl(conf); + + // Construct correct TransportFactory + TTransportFactory transportFactory; + if (conf.getBoolean(FRAMED_CONF_KEY, false) || implType.isAlwaysFramed) { + transportFactory = new TFramedTransport.Factory(); + LOG.debug("Using framed transport"); + } else { + transportFactory = new TTransportFactory(); + } + + if (conf.get(BIND_CONF_KEY) != null && !implType.canSpecifyBindIP) { + LOG.error("Server types " + Joiner.on(", ").join( + ImplType.serversThatCannotSpecifyBindIP()) + " don't support IP " + + "address binding at the moment. See " + + "https://issues.apache.org/jira/browse/HBASE-2155 for details."); + throw new RuntimeException( + "-" + BIND_CONF_KEY + " not supported with " + implType); + } + + if (implType == ImplType.HS_HA || implType == ImplType.NONBLOCKING || + implType == ImplType.THREADED_SELECTOR) { + + TNonblockingServerTransport serverTransport = + new TNonblockingServerSocket(listenPort); + + if (implType == ImplType.NONBLOCKING) { + TNonblockingServer.Args serverArgs = + new TNonblockingServer.Args(serverTransport); + serverArgs.processor(processor) + .transportFactory(transportFactory) + .protocolFactory(protocolFactory); + tserver = new TNonblockingServer(serverArgs); + } else if (implType == ImplType.HS_HA) { + THsHaServer.Args serverArgs = new THsHaServer.Args(serverTransport); + CallQueue callQueue = + new CallQueue(new LinkedBlockingQueue(), metrics); + ExecutorService executorService = createExecutor( + callQueue, serverArgs.getWorkerThreads()); + serverArgs.executorService(executorService) + .processor(processor) + .transportFactory(transportFactory) + .protocolFactory(protocolFactory); + tserver = new THsHaServer(serverArgs); + } else { // THREADED_SELECTOR + TThreadedSelectorServer.Args serverArgs = + new HThreadedSelectorServerArgs(serverTransport, conf); + CallQueue callQueue = + new CallQueue(new LinkedBlockingQueue(), metrics); + ExecutorService executorService = createExecutor( + callQueue, serverArgs.getWorkerThreads()); + serverArgs.executorService(executorService) + .processor(processor) + .transportFactory(transportFactory) + .protocolFactory(protocolFactory); + tserver = new TThreadedSelectorServer(serverArgs); + } + LOG.info("starting HBase " + implType.simpleClassName() + + " server on " + Integer.toString(listenPort)); + } else if (implType == ImplType.THREAD_POOL) { + // Thread pool server. Get the IP address to bind to. + InetAddress listenAddress = getBindAddress(conf); + + TServerTransport serverTransport = new TServerSocket( + new InetSocketAddress(listenAddress, listenPort)); + + TBoundedThreadPoolServer.Args serverArgs = + new TBoundedThreadPoolServer.Args(serverTransport, conf); + serverArgs.processor(processor) + .transportFactory(transportFactory) + .protocolFactory(protocolFactory); + LOG.info("starting " + ImplType.THREAD_POOL.simpleClassName() + " on " + + listenAddress + ":" + Integer.toString(listenPort) + + "; " + serverArgs); + TBoundedThreadPoolServer tserver = + new TBoundedThreadPoolServer(serverArgs, metrics); + this.tserver = tserver; + } else { + throw new AssertionError("Unsupported Thrift server implementation: " + + implType.simpleClassName()); + } + + // A sanity check that we instantiated the right type of server. + if (tserver.getClass() != implType.serverClass) { + throw new AssertionError("Expected to create Thrift server class " + + implType.serverClass.getName() + " but got " + + tserver.getClass().getName()); + } + + registerFilters(conf); + } + + ExecutorService createExecutor(BlockingQueue callQueue, + int workerThreads) { + ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); + tfb.setDaemon(true); + tfb.setNameFormat("thrift-worker-%d"); + return new ThreadPoolExecutor(workerThreads, workerThreads, + Long.MAX_VALUE, TimeUnit.SECONDS, callQueue, tfb.build()); + } + + private InetAddress getBindAddress(Configuration conf) + throws UnknownHostException { + String bindAddressStr = conf.get(BIND_CONF_KEY, DEFAULT_BIND_ADDR); + return InetAddress.getByName(bindAddressStr); + } + + /** + * The HBaseHandler is a glue object that connects Thrift RPC calls to the + * HBase client API primarily defined in the HBaseAdmin and HTable objects. + */ + public static class HBaseHandler implements Hbase.Iface { + protected Configuration conf; + protected HBaseAdmin admin = null; + protected final Log LOG = LogFactory.getLog(this.getClass().getName()); + + // nextScannerId and scannerMap are used to manage scanner state + protected int nextScannerId = 0; + protected HashMap scannerMap = null; + private ThriftMetrics metrics = null; + + private static ThreadLocal> threadLocalTables = + new ThreadLocal>() { + @Override + protected Map initialValue() { + return new TreeMap(); + } + }; + + IncrementCoalescer coalescer = null; + + /** + * Returns a list of all the column families for a given htable. + * + * @param table + * @return + * @throws IOException + */ + byte[][] getAllColumns(HTable table) throws IOException { + HColumnDescriptor[] cds = table.getTableDescriptor().getColumnFamilies(); + byte[][] columns = new byte[cds.length][]; + for (int i = 0; i < cds.length; i++) { + columns[i] = Bytes.add(cds[i].getName(), + KeyValue.COLUMN_FAMILY_DELIM_ARRAY); + } + return columns; + } + + /** + * Creates and returns an HTable instance from a given table name. + * + * @param tableName + * name of table + * @return HTable object + * @throws IOException + * @throws IOError + */ + public HTable getTable(final byte[] tableName) throws + IOException { + String table = new String(tableName); + Map tables = threadLocalTables.get(); + if (!tables.containsKey(table)) { + tables.put(table, new HTable(conf, tableName)); + } + return tables.get(table); + } + + public HTable getTable(final ByteBuffer tableName) throws IOException { + return getTable(getBytes(tableName)); + } + + /** + * Assigns a unique ID to the scanner and adds the mapping to an internal + * hash-map. + * + * @param scanner + * @return integer scanner id + */ + protected synchronized int addScanner(ResultScanner scanner) { + int id = nextScannerId++; + scannerMap.put(id, scanner); + return id; + } + + /** + * Returns the scanner associated with the specified ID. + * + * @param id + * @return a Scanner, or null if ID was invalid. + */ + protected synchronized ResultScanner getScanner(int id) { + return scannerMap.get(id); + } + + /** + * Removes the scanner associated with the specified ID from the internal + * id->scanner hash-map. + * + * @param id + * @return a Scanner, or null if ID was invalid. + */ + protected synchronized ResultScanner removeScanner(int id) { + return scannerMap.remove(id); + } + + /** + * Constructs an HBaseHandler object. + * @throws IOException + */ + protected HBaseHandler() + throws IOException { + this(HBaseConfiguration.create()); + } + + protected HBaseHandler(final Configuration c) throws IOException { + this.conf = c; + admin = new HBaseAdmin(conf); + scannerMap = new HashMap(); + this.coalescer = new IncrementCoalescer(this); + } + + @Override + public void enableTable(ByteBuffer tableName) throws IOError { + try{ + admin.enableTable(getBytes(tableName)); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public void disableTable(ByteBuffer tableName) throws IOError{ + try{ + admin.disableTable(getBytes(tableName)); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public boolean isTableEnabled(ByteBuffer tableName) throws IOError { + try { + return HTable.isTableEnabled(this.conf, getBytes(tableName)); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public void compact(ByteBuffer tableNameOrRegionName) throws IOError { + try{ + admin.compact(getBytes(tableNameOrRegionName)); + } catch (InterruptedException e) { + throw new IOError(e.getMessage()); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public void majorCompact(ByteBuffer tableNameOrRegionName) throws IOError { + try{ + admin.majorCompact(getBytes(tableNameOrRegionName)); + } catch (InterruptedException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public List getTableNames() throws IOError { + try { + HTableDescriptor[] tables = this.admin.listTables(); + ArrayList list = new ArrayList(tables.length); + for (int i = 0; i < tables.length; i++) { + list.add(ByteBuffer.wrap(tables[i].getName())); + } + return list; + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public List getTableRegions(ByteBuffer tableName) + throws IOError { + try { + HTable table = getTable(tableName); + Map regionLocations = + table.getRegionLocations(); + List results = new ArrayList(); + for (Map.Entry entry : + regionLocations.entrySet()) { + HRegionInfo info = entry.getKey(); + ServerName serverName = entry.getValue(); + TRegionInfo region = new TRegionInfo(); + region.serverName = ByteBuffer.wrap( + Bytes.toBytes(serverName.getHostname())); + region.port = serverName.getPort(); + region.startKey = ByteBuffer.wrap(info.getStartKey()); + region.endKey = ByteBuffer.wrap(info.getEndKey()); + region.id = info.getRegionId(); + region.name = ByteBuffer.wrap(info.getRegionName()); + region.version = info.getVersion(); + results.add(region); + } + return results; + } catch (TableNotFoundException e) { + // Return empty list for non-existing table + return Collections.emptyList(); + } catch (IOException e){ + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + /** + * Convert ByteBuffer to byte array. Note that this cannot be replaced by + * Bytes.toBytes(). + */ + public static byte[] toBytes(ByteBuffer bb) { + byte[] result = new byte[bb.remaining()]; + // Make a duplicate so the position doesn't change + ByteBuffer dup = bb.duplicate(); + dup.get(result, 0, result.length); + return result; + } + + @Deprecated + @Override + public List get( + ByteBuffer tableName, ByteBuffer row, ByteBuffer column, + Map attributes) + throws IOError { + byte [][] famAndQf = KeyValue.parseColumn(getBytes(column)); + if(famAndQf.length == 1) { + return get(tableName, row, famAndQf[0], new byte[0], attributes); + } + return get(tableName, row, famAndQf[0], famAndQf[1], attributes); + } + + protected List get(ByteBuffer tableName, + ByteBuffer row, + byte[] family, + byte[] qualifier, + Map attributes) throws IOError { + try { + HTable table = getTable(tableName); + Get get = new Get(getBytes(row)); + addAttributes(get, attributes); + if (qualifier == null || qualifier.length == 0) { + get.addFamily(family); + } else { + get.addColumn(family, qualifier); + } + Result result = table.get(get); + return ThriftUtilities.cellFromHBase(result.raw()); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Deprecated + @Override + public List getVer(ByteBuffer tableName, ByteBuffer row, + ByteBuffer column, int numVersions, + Map attributes) throws IOError { + byte [][] famAndQf = KeyValue.parseColumn(getBytes(column)); + if(famAndQf.length == 1) { + return getVer(tableName, row, famAndQf[0], + new byte[0], numVersions, attributes); + } + return getVer(tableName, row, + famAndQf[0], famAndQf[1], numVersions, attributes); + } + + public List getVer(ByteBuffer tableName, ByteBuffer row, + byte[] family, + byte[] qualifier, int numVersions, + Map attributes) throws IOError { + try { + HTable table = getTable(tableName); + Get get = new Get(getBytes(row)); + addAttributes(get, attributes); + get.addColumn(family, qualifier); + get.setMaxVersions(numVersions); + Result result = table.get(get); + return ThriftUtilities.cellFromHBase(result.raw()); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Deprecated + @Override + public List getVerTs(ByteBuffer tableName, + ByteBuffer row, + ByteBuffer column, + long timestamp, + int numVersions, + Map attributes) throws IOError { + byte [][] famAndQf = KeyValue.parseColumn(getBytes(column)); + if(famAndQf.length == 1) { + return getVerTs(tableName, row, famAndQf[0], new byte[0], timestamp, + numVersions, attributes); + } + return getVerTs(tableName, row, famAndQf[0], famAndQf[1], timestamp, + numVersions, attributes); + } + + protected List getVerTs(ByteBuffer tableName, + ByteBuffer row, byte [] family, + byte [] qualifier, long timestamp, int numVersions, + Map attributes) throws IOError { + try { + HTable table = getTable(tableName); + Get get = new Get(getBytes(row)); + addAttributes(get, attributes); + get.addColumn(family, qualifier); + get.setTimeRange(Long.MIN_VALUE, timestamp); + get.setMaxVersions(numVersions); + Result result = table.get(get); + return ThriftUtilities.cellFromHBase(result.raw()); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public List getRow(ByteBuffer tableName, ByteBuffer row, + Map attributes) throws IOError { + return getRowWithColumnsTs(tableName, row, null, + HConstants.LATEST_TIMESTAMP, + attributes); + } + + @Override + public List getRowWithColumns(ByteBuffer tableName, + ByteBuffer row, + List columns, + Map attributes) throws IOError { + return getRowWithColumnsTs(tableName, row, columns, + HConstants.LATEST_TIMESTAMP, + attributes); + } + + @Override + public List getRowTs(ByteBuffer tableName, ByteBuffer row, + long timestamp, Map attributes) throws IOError { + return getRowWithColumnsTs(tableName, row, null, + timestamp, attributes); + } + + @Override + public List getRowWithColumnsTs( + ByteBuffer tableName, ByteBuffer row, List columns, + long timestamp, Map attributes) throws IOError { + try { + HTable table = getTable(tableName); + if (columns == null) { + Get get = new Get(getBytes(row)); + addAttributes(get, attributes); + get.setTimeRange(Long.MIN_VALUE, timestamp); + Result result = table.get(get); + return ThriftUtilities.rowResultFromHBase(result); + } + Get get = new Get(getBytes(row)); + addAttributes(get, attributes); + for(ByteBuffer column : columns) { + byte [][] famAndQf = KeyValue.parseColumn(getBytes(column)); + if (famAndQf.length == 1) { + get.addFamily(famAndQf[0]); + } else { + get.addColumn(famAndQf[0], famAndQf[1]); + } + } + get.setTimeRange(Long.MIN_VALUE, timestamp); + Result result = table.get(get); + return ThriftUtilities.rowResultFromHBase(result); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public List getRows(ByteBuffer tableName, + List rows, + Map attributes) + throws IOError { + return getRowsWithColumnsTs(tableName, rows, null, + HConstants.LATEST_TIMESTAMP, + attributes); + } + + @Override + public List getRowsWithColumns(ByteBuffer tableName, + List rows, + List columns, + Map attributes) throws IOError { + return getRowsWithColumnsTs(tableName, rows, columns, + HConstants.LATEST_TIMESTAMP, + attributes); + } + + @Override + public List getRowsTs(ByteBuffer tableName, + List rows, + long timestamp, + Map attributes) throws IOError { + return getRowsWithColumnsTs(tableName, rows, null, + timestamp, attributes); + } + + @Override + public List getRowsWithColumnsTs(ByteBuffer tableName, + List rows, + List columns, long timestamp, + Map attributes) throws IOError { + try { + List gets = new ArrayList(rows.size()); + HTable table = getTable(tableName); + if (metrics != null) { + metrics.incNumRowKeysInBatchGet(rows.size()); + } + for (ByteBuffer row : rows) { + Get get = new Get(getBytes(row)); + addAttributes(get, attributes); + if (columns != null) { + + for(ByteBuffer column : columns) { + byte [][] famAndQf = KeyValue.parseColumn(getBytes(column)); + if (famAndQf.length == 1) { + get.addFamily(famAndQf[0]); + } else { + get.addColumn(famAndQf[0], famAndQf[1]); + } + } + } + get.setTimeRange(Long.MIN_VALUE, timestamp); + gets.add(get); + } + Result[] result = table.get(gets); + return ThriftUtilities.rowResultFromHBase(result); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public void deleteAll( + ByteBuffer tableName, ByteBuffer row, ByteBuffer column, + Map attributes) + throws IOError { + deleteAllTs(tableName, row, column, HConstants.LATEST_TIMESTAMP, + attributes); + } + + @Override + public void deleteAllTs(ByteBuffer tableName, + ByteBuffer row, + ByteBuffer column, + long timestamp, Map attributes) throws IOError { + try { + HTable table = getTable(tableName); + Delete delete = new Delete(getBytes(row)); + addAttributes(delete, attributes); + byte [][] famAndQf = KeyValue.parseColumn(getBytes(column)); + if (famAndQf.length == 1) { + delete.deleteFamily(famAndQf[0], timestamp); + } else { + delete.deleteColumns(famAndQf[0], famAndQf[1], timestamp); + } + table.delete(delete); + + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public void deleteAllRow( + ByteBuffer tableName, ByteBuffer row, + Map attributes) throws IOError { + deleteAllRowTs(tableName, row, HConstants.LATEST_TIMESTAMP, attributes); + } + + @Override + public void deleteAllRowTs( + ByteBuffer tableName, ByteBuffer row, long timestamp, + Map attributes) throws IOError { + try { + HTable table = getTable(tableName); + Delete delete = new Delete(getBytes(row), timestamp, null); + addAttributes(delete, attributes); + table.delete(delete); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public void createTable(ByteBuffer in_tableName, + List columnFamilies) throws IOError, + IllegalArgument, AlreadyExists { + byte [] tableName = getBytes(in_tableName); + try { + if (admin.tableExists(tableName)) { + throw new AlreadyExists("table name already in use"); + } + HTableDescriptor desc = new HTableDescriptor(tableName); + for (ColumnDescriptor col : columnFamilies) { + HColumnDescriptor colDesc = ThriftUtilities.colDescFromThrift(col); + desc.addFamily(colDesc); + } + admin.createTable(desc); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } catch (IllegalArgumentException e) { + LOG.warn(e.getMessage(), e); + throw new IllegalArgument(e.getMessage()); + } + } + + @Override + public void deleteTable(ByteBuffer in_tableName) throws IOError { + byte [] tableName = getBytes(in_tableName); + if (LOG.isDebugEnabled()) { + LOG.debug("deleteTable: table=" + Bytes.toString(tableName)); + } + try { + if (!admin.tableExists(tableName)) { + throw new IOException("table does not exist"); + } + admin.deleteTable(tableName); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public void mutateRow(ByteBuffer tableName, ByteBuffer row, + List mutations, Map attributes) + throws IOError, IllegalArgument { + mutateRowTs(tableName, row, mutations, HConstants.LATEST_TIMESTAMP, + attributes); + } + + @Override + public void mutateRowTs(ByteBuffer tableName, ByteBuffer row, + List mutations, long timestamp, + Map attributes) + throws IOError, IllegalArgument { + HTable table = null; + try { + table = getTable(tableName); + Put put = new Put(getBytes(row), timestamp, null); + addAttributes(put, attributes); + + Delete delete = new Delete(getBytes(row)); + addAttributes(delete, attributes); + if (metrics != null) { + metrics.incNumRowKeysInBatchMutate(mutations.size()); + } + + // I apologize for all this mess :) + for (Mutation m : mutations) { + byte[][] famAndQf = KeyValue.parseColumn(getBytes(m.column)); + if (m.isDelete) { + if (famAndQf.length == 1) { + delete.deleteFamily(famAndQf[0], timestamp); + } else { + delete.deleteColumns(famAndQf[0], famAndQf[1], timestamp); + } + delete.setWriteToWAL(m.writeToWAL); + } else { + if(famAndQf.length == 1) { + put.add(famAndQf[0], HConstants.EMPTY_BYTE_ARRAY, + m.value != null ? getBytes(m.value) + : HConstants.EMPTY_BYTE_ARRAY); + } else { + put.add(famAndQf[0], famAndQf[1], + m.value != null ? getBytes(m.value) + : HConstants.EMPTY_BYTE_ARRAY); + } + put.setWriteToWAL(m.writeToWAL); + } + } + if (!delete.isEmpty()) + table.delete(delete); + if (!put.isEmpty()) + table.put(put); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } catch (IllegalArgumentException e) { + LOG.warn(e.getMessage(), e); + throw new IllegalArgument(e.getMessage()); + } + } + + @Override + public void mutateRows(ByteBuffer tableName, List rowBatches, + Map attributes) + throws IOError, IllegalArgument, TException { + mutateRowsTs(tableName, rowBatches, HConstants.LATEST_TIMESTAMP, attributes); + } + + @Override + public void mutateRowsTs( + ByteBuffer tableName, List rowBatches, long timestamp, + Map attributes) + throws IOError, IllegalArgument, TException { + List puts = new ArrayList(); + List deletes = new ArrayList(); + + for (BatchMutation batch : rowBatches) { + byte[] row = getBytes(batch.row); + List mutations = batch.mutations; + Delete delete = new Delete(row); + addAttributes(delete, attributes); + Put put = new Put(row, timestamp, null); + addAttributes(put, attributes); + for (Mutation m : mutations) { + byte[][] famAndQf = KeyValue.parseColumn(getBytes(m.column)); + if (m.isDelete) { + // no qualifier, family only. + if (famAndQf.length == 1) { + delete.deleteFamily(famAndQf[0], timestamp); + } else { + delete.deleteColumns(famAndQf[0], famAndQf[1], timestamp); + } + delete.setWriteToWAL(m.writeToWAL); + } else { + if(famAndQf.length == 1) { + put.add(famAndQf[0], HConstants.EMPTY_BYTE_ARRAY, + m.value != null ? getBytes(m.value) + : HConstants.EMPTY_BYTE_ARRAY); + } else { + put.add(famAndQf[0], famAndQf[1], + m.value != null ? getBytes(m.value) + : HConstants.EMPTY_BYTE_ARRAY); + } + put.setWriteToWAL(m.writeToWAL); + } + } + if (!delete.isEmpty()) + deletes.add(delete); + if (!put.isEmpty()) + puts.add(put); + } + + HTable table = null; + try { + table = getTable(tableName); + if (!puts.isEmpty()) + table.put(puts); + if (!deletes.isEmpty()) table.delete(deletes); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } catch (IllegalArgumentException e) { + LOG.warn(e.getMessage(), e); + throw new IllegalArgument(e.getMessage()); + } + } + + @Deprecated + @Override + public long atomicIncrement( + ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long amount) + throws IOError, IllegalArgument, TException { + byte [][] famAndQf = KeyValue.parseColumn(getBytes(column)); + if(famAndQf.length == 1) { + return atomicIncrement(tableName, row, famAndQf[0], new byte[0], + amount); + } + return atomicIncrement(tableName, row, famAndQf[0], famAndQf[1], amount); + } + + protected long atomicIncrement(ByteBuffer tableName, ByteBuffer row, + byte [] family, byte [] qualifier, long amount) + throws IOError, IllegalArgument, TException { + HTable table; + try { + table = getTable(tableName); + return table.incrementColumnValue( + getBytes(row), family, qualifier, amount); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + public void scannerClose(int id) throws IOError, IllegalArgument { + LOG.debug("scannerClose: id=" + id); + ResultScanner scanner = getScanner(id); + if (scanner == null) { + String message = "scanner ID is invalid"; + LOG.warn(message); + throw new IllegalArgument("scanner ID is invalid"); + } + scanner.close(); + removeScanner(id); + } + + @Override + public List scannerGetList(int id,int nbRows) + throws IllegalArgument, IOError { + LOG.debug("scannerGetList: id=" + id); + ResultScanner scanner = getScanner(id); + if (null == scanner) { + String message = "scanner ID is invalid"; + LOG.warn(message); + throw new IllegalArgument("scanner ID is invalid"); + } + + Result [] results = null; + try { + results = scanner.next(nbRows); + if (null == results) { + return new ArrayList(); + } + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + return ThriftUtilities.rowResultFromHBase(results); + } + + @Override + public List scannerGet(int id) throws IllegalArgument, IOError { + return scannerGetList(id,1); + } + + public int scannerOpenWithScan(ByteBuffer tableName, TScan tScan, + Map attributes) + throws IOError { + try { + HTable table = getTable(tableName); + Scan scan = new Scan(); + addAttributes(scan, attributes); + if (tScan.isSetStartRow()) { + scan.setStartRow(tScan.getStartRow()); + } + if (tScan.isSetStopRow()) { + scan.setStopRow(tScan.getStopRow()); + } + if (tScan.isSetTimestamp()) { + scan.setTimeRange(Long.MIN_VALUE, tScan.getTimestamp()); + } + if (tScan.isSetCaching()) { + scan.setCaching(tScan.getCaching()); + } + if (tScan.isSetColumns() && tScan.getColumns().size() != 0) { + for(ByteBuffer column : tScan.getColumns()) { + byte [][] famQf = KeyValue.parseColumn(getBytes(column)); + if(famQf.length == 1) { + scan.addFamily(famQf[0]); + } else { + scan.addColumn(famQf[0], famQf[1]); + } + } + } + if (tScan.isSetFilterString()) { + ParseFilter parseFilter = new ParseFilter(); + scan.setFilter( + parseFilter.parseFilterString(tScan.getFilterString())); + } + return addScanner(table.getScanner(scan)); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public int scannerOpen(ByteBuffer tableName, ByteBuffer startRow, + List columns, + Map attributes) throws IOError { + try { + HTable table = getTable(tableName); + Scan scan = new Scan(getBytes(startRow)); + addAttributes(scan, attributes); + if(columns != null && columns.size() != 0) { + for(ByteBuffer column : columns) { + byte [][] famQf = KeyValue.parseColumn(getBytes(column)); + if(famQf.length == 1) { + scan.addFamily(famQf[0]); + } else { + scan.addColumn(famQf[0], famQf[1]); + } + } + } + return addScanner(table.getScanner(scan)); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public int scannerOpenWithStop(ByteBuffer tableName, ByteBuffer startRow, + ByteBuffer stopRow, List columns, + Map attributes) + throws IOError, TException { + try { + HTable table = getTable(tableName); + Scan scan = new Scan(getBytes(startRow), getBytes(stopRow)); + addAttributes(scan, attributes); + if(columns != null && columns.size() != 0) { + for(ByteBuffer column : columns) { + byte [][] famQf = KeyValue.parseColumn(getBytes(column)); + if(famQf.length == 1) { + scan.addFamily(famQf[0]); + } else { + scan.addColumn(famQf[0], famQf[1]); + } + } + } + return addScanner(table.getScanner(scan)); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public int scannerOpenWithPrefix(ByteBuffer tableName, + ByteBuffer startAndPrefix, + List columns, + Map attributes) + throws IOError, TException { + try { + HTable table = getTable(tableName); + Scan scan = new Scan(getBytes(startAndPrefix)); + addAttributes(scan, attributes); + Filter f = new WhileMatchFilter( + new PrefixFilter(getBytes(startAndPrefix))); + scan.setFilter(f); + if (columns != null && columns.size() != 0) { + for(ByteBuffer column : columns) { + byte [][] famQf = KeyValue.parseColumn(getBytes(column)); + if(famQf.length == 1) { + scan.addFamily(famQf[0]); + } else { + scan.addColumn(famQf[0], famQf[1]); + } + } + } + return addScanner(table.getScanner(scan)); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public int scannerOpenTs(ByteBuffer tableName, ByteBuffer startRow, + List columns, long timestamp, + Map attributes) throws IOError, TException { + try { + HTable table = getTable(tableName); + Scan scan = new Scan(getBytes(startRow)); + addAttributes(scan, attributes); + scan.setTimeRange(Long.MIN_VALUE, timestamp); + if (columns != null && columns.size() != 0) { + for (ByteBuffer column : columns) { + byte [][] famQf = KeyValue.parseColumn(getBytes(column)); + if(famQf.length == 1) { + scan.addFamily(famQf[0]); + } else { + scan.addColumn(famQf[0], famQf[1]); + } + } + } + return addScanner(table.getScanner(scan)); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public int scannerOpenWithStopTs(ByteBuffer tableName, ByteBuffer startRow, + ByteBuffer stopRow, List columns, long timestamp, + Map attributes) + throws IOError, TException { + try { + HTable table = getTable(tableName); + Scan scan = new Scan(getBytes(startRow), getBytes(stopRow)); + addAttributes(scan, attributes); + scan.setTimeRange(Long.MIN_VALUE, timestamp); + if (columns != null && columns.size() != 0) { + for (ByteBuffer column : columns) { + byte [][] famQf = KeyValue.parseColumn(getBytes(column)); + if(famQf.length == 1) { + scan.addFamily(famQf[0]); + } else { + scan.addColumn(famQf[0], famQf[1]); + } + } + } + scan.setTimeRange(Long.MIN_VALUE, timestamp); + return addScanner(table.getScanner(scan)); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public Map getColumnDescriptors( + ByteBuffer tableName) throws IOError, TException { + try { + TreeMap columns = + new TreeMap(); + + HTable table = getTable(tableName); + HTableDescriptor desc = table.getTableDescriptor(); + + for (HColumnDescriptor e : desc.getFamilies()) { + ColumnDescriptor col = ThriftUtilities.colDescFromHbase(e); + columns.put(col.name, col); + } + return columns; + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public List getRowOrBefore(ByteBuffer tableName, ByteBuffer row, + ByteBuffer family) throws IOError { + try { + HTable table = getTable(getBytes(tableName)); + Result result = table.getRowOrBefore(getBytes(row), getBytes(family)); + return ThriftUtilities.cellFromHBase(result.raw()); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public TRegionInfo getRegionInfo(ByteBuffer searchRow) throws IOError { + try { + HTable table = getTable(HConstants.META_TABLE_NAME); + byte[] row = toBytes(searchRow); + Result startRowResult = table.getRowOrBefore( + row, HConstants.CATALOG_FAMILY); + + if (startRowResult == null) { + throw new IOException("Cannot find row in .META., row=" + + Bytes.toString(searchRow.array())); + } + + // find region start and end keys + byte[] value = startRowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + if (value == null || value.length == 0) { + throw new IOException("HRegionInfo REGIONINFO was null or " + + " empty in Meta for row=" + + Bytes.toString(row)); + } + HRegionInfo regionInfo = Writables.getHRegionInfo(value); + TRegionInfo region = new TRegionInfo(); + region.setStartKey(regionInfo.getStartKey()); + region.setEndKey(regionInfo.getEndKey()); + region.id = regionInfo.getRegionId(); + region.setName(regionInfo.getRegionName()); + region.version = regionInfo.getVersion(); + + // find region assignment to server + value = startRowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER); + if (value != null && value.length > 0) { + String hostAndPort = Bytes.toString(value); + region.setServerName(Bytes.toBytes( + Addressing.parseHostname(hostAndPort))); + region.port = Addressing.parsePort(hostAndPort); + } + return region; + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + private void initMetrics(ThriftMetrics metrics) { + this.metrics = metrics; + } + + @Override + public void increment(TIncrement tincrement) throws IOError, TException { + + if (tincrement.getRow().length == 0 || tincrement.getTable().length == 0) { + throw new TException("Must supply a table and a row key; can't increment"); + } + + if (conf.getBoolean(COALESCE_INC_KEY, false)) { + this.coalescer.queueIncrement(tincrement); + return; + } + + try { + HTable table = getTable(tincrement.getTable()); + Increment inc = ThriftUtilities.incrementFromThrift(tincrement); + table.increment(inc); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + throw new IOError(e.getMessage()); + } + } + + @Override + public void incrementRows(List tincrements) throws IOError, TException { + if (conf.getBoolean(COALESCE_INC_KEY, false)) { + this.coalescer.queueIncrements(tincrements); + return; + } + for (TIncrement tinc : tincrements) { + increment(tinc); + } + } + } + + + + /** + * Adds all the attributes into the Operation object + */ + private static void addAttributes(OperationWithAttributes op, + Map attributes) { + if (attributes == null || attributes.size() == 0) { + return; + } + for (Map.Entry entry : attributes.entrySet()) { + String name = Bytes.toStringBinary(getBytes(entry.getKey())); + byte[] value = getBytes(entry.getValue()); + op.setAttribute(name, value); + } + } + + public static void registerFilters(Configuration conf) { + String[] filters = conf.getStrings("hbase.thrift.filters"); + if(filters != null) { + for(String filterClass: filters) { + String[] filterPart = filterClass.split(":"); + if(filterPart.length != 2) { + LOG.warn("Invalid filter specification " + filterClass + " - skipping"); + } else { + ParseFilter.registerFilter(filterPart[0], filterPart[1]); + } + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java new file mode 100644 index 0000000..8313aa8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.thrift; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeMap; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor; +import org.apache.hadoop.hbase.thrift.generated.IllegalArgument; +import org.apache.hadoop.hbase.thrift.generated.TCell; +import org.apache.hadoop.hbase.thrift.generated.TIncrement; +import org.apache.hadoop.hbase.thrift.generated.TRowResult; +import org.apache.hadoop.hbase.util.Bytes; + +public class ThriftUtilities { + + /** + * This utility method creates a new Hbase HColumnDescriptor object based on a + * Thrift ColumnDescriptor "struct". + * + * @param in + * Thrift ColumnDescriptor object + * @return HColumnDescriptor + * @throws IllegalArgument + */ + static public HColumnDescriptor colDescFromThrift(ColumnDescriptor in) + throws IllegalArgument { + Compression.Algorithm comp = + Compression.getCompressionAlgorithmByName(in.compression.toLowerCase()); + StoreFile.BloomType bt = + BloomType.valueOf(in.bloomFilterType); + + if (in.name == null || !in.name.hasRemaining()) { + throw new IllegalArgument("column name is empty"); + } + byte [] parsedName = KeyValue.parseColumn(Bytes.getBytes(in.name))[0]; + HColumnDescriptor col = new HColumnDescriptor(parsedName) + .setMaxVersions(in.maxVersions) + .setCompressionType(comp) + .setInMemory(in.inMemory) + .setBlockCacheEnabled(in.blockCacheEnabled) + .setTimeToLive(in.timeToLive) + .setBloomFilterType(bt); + return col; + } + + /** + * This utility method creates a new Thrift ColumnDescriptor "struct" based on + * an Hbase HColumnDescriptor object. + * + * @param in + * Hbase HColumnDescriptor object + * @return Thrift ColumnDescriptor + */ + static public ColumnDescriptor colDescFromHbase(HColumnDescriptor in) { + ColumnDescriptor col = new ColumnDescriptor(); + col.name = ByteBuffer.wrap(Bytes.add(in.getName(), KeyValue.COLUMN_FAMILY_DELIM_ARRAY)); + col.maxVersions = in.getMaxVersions(); + col.compression = in.getCompression().toString(); + col.inMemory = in.isInMemory(); + col.blockCacheEnabled = in.isBlockCacheEnabled(); + col.bloomFilterType = in.getBloomFilterType().toString(); + return col; + } + + /** + * This utility method creates a list of Thrift TCell "struct" based on + * an Hbase Cell object. The empty list is returned if the input is null. + * + * @param in + * Hbase Cell object + * @return Thrift TCell array + */ + static public List cellFromHBase(KeyValue in) { + List list = new ArrayList(1); + if (in != null) { + list.add(new TCell(ByteBuffer.wrap(in.getValue()), in.getTimestamp())); + } + return list; + } + + /** + * This utility method creates a list of Thrift TCell "struct" based on + * an Hbase Cell array. The empty list is returned if the input is null. + * @param in Hbase Cell array + * @return Thrift TCell array + */ + static public List cellFromHBase(KeyValue[] in) { + List list = null; + if (in != null) { + list = new ArrayList(in.length); + for (int i = 0; i < in.length; i++) { + list.add(new TCell(ByteBuffer.wrap(in[i].getValue()), in[i].getTimestamp())); + } + } else { + list = new ArrayList(0); + } + return list; + } + + /** + * This utility method creates a list of Thrift TRowResult "struct" based on + * an Hbase RowResult object. The empty list is returned if the input is + * null. + * + * @param in + * Hbase RowResult object + * @return Thrift TRowResult array + */ + static public List rowResultFromHBase(Result[] in) { + List results = new ArrayList(); + for ( Result result_ : in) { + if(result_ == null || result_.isEmpty()) { + continue; + } + TRowResult result = new TRowResult(); + result.row = ByteBuffer.wrap(result_.getRow()); + result.columns = new TreeMap(); + for(KeyValue kv : result_.raw()) { + result.columns.put( + ByteBuffer.wrap(KeyValue.makeColumn(kv.getFamily(), + kv.getQualifier())), + new TCell(ByteBuffer.wrap(kv.getValue()), kv.getTimestamp())); + } + results.add(result); + } + return results; + } + + static public List rowResultFromHBase(Result in) { + Result [] result = { in }; + return rowResultFromHBase(result); + } + + /** + * From a {@link TIncrement} create an {@link Increment}. + * @param tincrement the Thrift version of an increment + * @return an increment that the {@link TIncrement} represented. + */ + public static Increment incrementFromThrift(TIncrement tincrement) { + Increment inc = new Increment(tincrement.getRow()); + byte[][] famAndQf = KeyValue.parseColumn(tincrement.getColumn()); + if (famAndQf.length <1 ) return null; + byte[] qual = famAndQf.length == 1 ? new byte[0]: famAndQf[1]; + inc.addColumn(famAndQf[0], qual, tincrement.getAmmount()); + return inc; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/AlreadyExists.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/AlreadyExists.java new file mode 100644 index 0000000..a5b81f5 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/AlreadyExists.java @@ -0,0 +1,386 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An AlreadyExists exceptions signals that a table with the specified + * name already exists + */ +public class AlreadyExists extends Exception implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("AlreadyExists"); + + private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new AlreadyExistsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new AlreadyExistsTupleSchemeFactory()); + } + + public String message; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + MESSAGE((short)1, "message"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // MESSAGE + return MESSAGE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(AlreadyExists.class, metaDataMap); + } + + public AlreadyExists() { + } + + public AlreadyExists( + String message) + { + this(); + this.message = message; + } + + /** + * Performs a deep copy on other. + */ + public AlreadyExists(AlreadyExists other) { + if (other.isSetMessage()) { + this.message = other.message; + } + } + + public AlreadyExists deepCopy() { + return new AlreadyExists(this); + } + + @Override + public void clear() { + this.message = null; + } + + public String getMessage() { + return this.message; + } + + public AlreadyExists setMessage(String message) { + this.message = message; + return this; + } + + public void unsetMessage() { + this.message = null; + } + + /** Returns true if field message is set (has been assigned a value) and false otherwise */ + public boolean isSetMessage() { + return this.message != null; + } + + public void setMessageIsSet(boolean value) { + if (!value) { + this.message = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case MESSAGE: + if (value == null) { + unsetMessage(); + } else { + setMessage((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case MESSAGE: + return getMessage(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case MESSAGE: + return isSetMessage(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof AlreadyExists) + return this.equals((AlreadyExists)that); + return false; + } + + public boolean equals(AlreadyExists that) { + if (that == null) + return false; + + boolean this_present_message = true && this.isSetMessage(); + boolean that_present_message = true && that.isSetMessage(); + if (this_present_message || that_present_message) { + if (!(this_present_message && that_present_message)) + return false; + if (!this.message.equals(that.message)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(AlreadyExists other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + AlreadyExists typedOther = (AlreadyExists)other; + + lastComparison = Boolean.valueOf(isSetMessage()).compareTo(typedOther.isSetMessage()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMessage()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.message, typedOther.message); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("AlreadyExists("); + boolean first = true; + + sb.append("message:"); + if (this.message == null) { + sb.append("null"); + } else { + sb.append(this.message); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class AlreadyExistsStandardSchemeFactory implements SchemeFactory { + public AlreadyExistsStandardScheme getScheme() { + return new AlreadyExistsStandardScheme(); + } + } + + private static class AlreadyExistsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, AlreadyExists struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // MESSAGE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.message = iprot.readString(); + struct.setMessageIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, AlreadyExists struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.message != null) { + oprot.writeFieldBegin(MESSAGE_FIELD_DESC); + oprot.writeString(struct.message); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class AlreadyExistsTupleSchemeFactory implements SchemeFactory { + public AlreadyExistsTupleScheme getScheme() { + return new AlreadyExistsTupleScheme(); + } + } + + private static class AlreadyExistsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, AlreadyExists struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetMessage()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetMessage()) { + oprot.writeString(struct.message); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, AlreadyExists struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.message = iprot.readString(); + struct.setMessageIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/BatchMutation.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/BatchMutation.java new file mode 100644 index 0000000..d5df940 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/BatchMutation.java @@ -0,0 +1,549 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A BatchMutation object is used to apply a number of Mutations to a single row. + */ +public class BatchMutation implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("BatchMutation"); + + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField MUTATIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("mutations", org.apache.thrift.protocol.TType.LIST, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new BatchMutationStandardSchemeFactory()); + schemes.put(TupleScheme.class, new BatchMutationTupleSchemeFactory()); + } + + public ByteBuffer row; // required + public List mutations; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + ROW((short)1, "row"), + MUTATIONS((short)2, "mutations"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // ROW + return ROW; + case 2: // MUTATIONS + return MUTATIONS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.MUTATIONS, new org.apache.thrift.meta_data.FieldMetaData("mutations", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Mutation.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(BatchMutation.class, metaDataMap); + } + + public BatchMutation() { + } + + public BatchMutation( + ByteBuffer row, + List mutations) + { + this(); + this.row = row; + this.mutations = mutations; + } + + /** + * Performs a deep copy on other. + */ + public BatchMutation(BatchMutation other) { + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetMutations()) { + List __this__mutations = new ArrayList(); + for (Mutation other_element : other.mutations) { + __this__mutations.add(new Mutation(other_element)); + } + this.mutations = __this__mutations; + } + } + + public BatchMutation deepCopy() { + return new BatchMutation(this); + } + + @Override + public void clear() { + this.row = null; + this.mutations = null; + } + + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + public BatchMutation setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public BatchMutation setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getMutationsSize() { + return (this.mutations == null) ? 0 : this.mutations.size(); + } + + public java.util.Iterator getMutationsIterator() { + return (this.mutations == null) ? null : this.mutations.iterator(); + } + + public void addToMutations(Mutation elem) { + if (this.mutations == null) { + this.mutations = new ArrayList(); + } + this.mutations.add(elem); + } + + public List getMutations() { + return this.mutations; + } + + public BatchMutation setMutations(List mutations) { + this.mutations = mutations; + return this; + } + + public void unsetMutations() { + this.mutations = null; + } + + /** Returns true if field mutations is set (has been assigned a value) and false otherwise */ + public boolean isSetMutations() { + return this.mutations != null; + } + + public void setMutationsIsSet(boolean value) { + if (!value) { + this.mutations = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case MUTATIONS: + if (value == null) { + unsetMutations(); + } else { + setMutations((List)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case ROW: + return getRow(); + + case MUTATIONS: + return getMutations(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case ROW: + return isSetRow(); + case MUTATIONS: + return isSetMutations(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof BatchMutation) + return this.equals((BatchMutation)that); + return false; + } + + public boolean equals(BatchMutation that) { + if (that == null) + return false; + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_mutations = true && this.isSetMutations(); + boolean that_present_mutations = true && that.isSetMutations(); + if (this_present_mutations || that_present_mutations) { + if (!(this_present_mutations && that_present_mutations)) + return false; + if (!this.mutations.equals(that.mutations)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(BatchMutation other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + BatchMutation typedOther = (BatchMutation)other; + + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMutations()).compareTo(typedOther.isSetMutations()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMutations()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.mutations, typedOther.mutations); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("BatchMutation("); + boolean first = true; + + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("mutations:"); + if (this.mutations == null) { + sb.append("null"); + } else { + sb.append(this.mutations); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class BatchMutationStandardSchemeFactory implements SchemeFactory { + public BatchMutationStandardScheme getScheme() { + return new BatchMutationStandardScheme(); + } + } + + private static class BatchMutationStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, BatchMutation struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // MUTATIONS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list0 = iprot.readListBegin(); + struct.mutations = new ArrayList(_list0.size); + for (int _i1 = 0; _i1 < _list0.size; ++_i1) + { + Mutation _elem2; // optional + _elem2 = new Mutation(); + _elem2.read(iprot); + struct.mutations.add(_elem2); + } + iprot.readListEnd(); + } + struct.setMutationsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, BatchMutation struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.mutations != null) { + oprot.writeFieldBegin(MUTATIONS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.mutations.size())); + for (Mutation _iter3 : struct.mutations) + { + _iter3.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class BatchMutationTupleSchemeFactory implements SchemeFactory { + public BatchMutationTupleScheme getScheme() { + return new BatchMutationTupleScheme(); + } + } + + private static class BatchMutationTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, BatchMutation struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetRow()) { + optionals.set(0); + } + if (struct.isSetMutations()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetMutations()) { + { + oprot.writeI32(struct.mutations.size()); + for (Mutation _iter4 : struct.mutations) + { + _iter4.write(oprot); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, BatchMutation struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(1)) { + { + org.apache.thrift.protocol.TList _list5 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.mutations = new ArrayList(_list5.size); + for (int _i6 = 0; _i6 < _list5.size; ++_i6) + { + Mutation _elem7; // optional + _elem7 = new Mutation(); + _elem7.read(iprot); + struct.mutations.add(_elem7); + } + } + struct.setMutationsIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java new file mode 100644 index 0000000..4ce85e7 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java @@ -0,0 +1,1184 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An HColumnDescriptor contains information about a column family + * such as the number of versions, compression settings, etc. It is + * used as input when creating a table or adding a column. + */ +public class ColumnDescriptor implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ColumnDescriptor"); + + private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField MAX_VERSIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("maxVersions", org.apache.thrift.protocol.TType.I32, (short)2); + private static final org.apache.thrift.protocol.TField COMPRESSION_FIELD_DESC = new org.apache.thrift.protocol.TField("compression", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField IN_MEMORY_FIELD_DESC = new org.apache.thrift.protocol.TField("inMemory", org.apache.thrift.protocol.TType.BOOL, (short)4); + private static final org.apache.thrift.protocol.TField BLOOM_FILTER_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("bloomFilterType", org.apache.thrift.protocol.TType.STRING, (short)5); + private static final org.apache.thrift.protocol.TField BLOOM_FILTER_VECTOR_SIZE_FIELD_DESC = new org.apache.thrift.protocol.TField("bloomFilterVectorSize", org.apache.thrift.protocol.TType.I32, (short)6); + private static final org.apache.thrift.protocol.TField BLOOM_FILTER_NB_HASHES_FIELD_DESC = new org.apache.thrift.protocol.TField("bloomFilterNbHashes", org.apache.thrift.protocol.TType.I32, (short)7); + private static final org.apache.thrift.protocol.TField BLOCK_CACHE_ENABLED_FIELD_DESC = new org.apache.thrift.protocol.TField("blockCacheEnabled", org.apache.thrift.protocol.TType.BOOL, (short)8); + private static final org.apache.thrift.protocol.TField TIME_TO_LIVE_FIELD_DESC = new org.apache.thrift.protocol.TField("timeToLive", org.apache.thrift.protocol.TType.I32, (short)9); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new ColumnDescriptorStandardSchemeFactory()); + schemes.put(TupleScheme.class, new ColumnDescriptorTupleSchemeFactory()); + } + + public ByteBuffer name; // required + public int maxVersions; // required + public String compression; // required + public boolean inMemory; // required + public String bloomFilterType; // required + public int bloomFilterVectorSize; // required + public int bloomFilterNbHashes; // required + public boolean blockCacheEnabled; // required + public int timeToLive; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + NAME((short)1, "name"), + MAX_VERSIONS((short)2, "maxVersions"), + COMPRESSION((short)3, "compression"), + IN_MEMORY((short)4, "inMemory"), + BLOOM_FILTER_TYPE((short)5, "bloomFilterType"), + BLOOM_FILTER_VECTOR_SIZE((short)6, "bloomFilterVectorSize"), + BLOOM_FILTER_NB_HASHES((short)7, "bloomFilterNbHashes"), + BLOCK_CACHE_ENABLED((short)8, "blockCacheEnabled"), + TIME_TO_LIVE((short)9, "timeToLive"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // NAME + return NAME; + case 2: // MAX_VERSIONS + return MAX_VERSIONS; + case 3: // COMPRESSION + return COMPRESSION; + case 4: // IN_MEMORY + return IN_MEMORY; + case 5: // BLOOM_FILTER_TYPE + return BLOOM_FILTER_TYPE; + case 6: // BLOOM_FILTER_VECTOR_SIZE + return BLOOM_FILTER_VECTOR_SIZE; + case 7: // BLOOM_FILTER_NB_HASHES + return BLOOM_FILTER_NB_HASHES; + case 8: // BLOCK_CACHE_ENABLED + return BLOCK_CACHE_ENABLED; + case 9: // TIME_TO_LIVE + return TIME_TO_LIVE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __MAXVERSIONS_ISSET_ID = 0; + private static final int __INMEMORY_ISSET_ID = 1; + private static final int __BLOOMFILTERVECTORSIZE_ISSET_ID = 2; + private static final int __BLOOMFILTERNBHASHES_ISSET_ID = 3; + private static final int __BLOCKCACHEENABLED_ISSET_ID = 4; + private static final int __TIMETOLIVE_ISSET_ID = 5; + private BitSet __isset_bit_vector = new BitSet(6); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.MAX_VERSIONS, new org.apache.thrift.meta_data.FieldMetaData("maxVersions", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.COMPRESSION, new org.apache.thrift.meta_data.FieldMetaData("compression", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.IN_MEMORY, new org.apache.thrift.meta_data.FieldMetaData("inMemory", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.BLOOM_FILTER_TYPE, new org.apache.thrift.meta_data.FieldMetaData("bloomFilterType", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.BLOOM_FILTER_VECTOR_SIZE, new org.apache.thrift.meta_data.FieldMetaData("bloomFilterVectorSize", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.BLOOM_FILTER_NB_HASHES, new org.apache.thrift.meta_data.FieldMetaData("bloomFilterNbHashes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.BLOCK_CACHE_ENABLED, new org.apache.thrift.meta_data.FieldMetaData("blockCacheEnabled", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.TIME_TO_LIVE, new org.apache.thrift.meta_data.FieldMetaData("timeToLive", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ColumnDescriptor.class, metaDataMap); + } + + public ColumnDescriptor() { + this.maxVersions = 3; + + this.compression = "NONE"; + + this.inMemory = false; + + this.bloomFilterType = "NONE"; + + this.bloomFilterVectorSize = 0; + + this.bloomFilterNbHashes = 0; + + this.blockCacheEnabled = false; + + this.timeToLive = -1; + + } + + public ColumnDescriptor( + ByteBuffer name, + int maxVersions, + String compression, + boolean inMemory, + String bloomFilterType, + int bloomFilterVectorSize, + int bloomFilterNbHashes, + boolean blockCacheEnabled, + int timeToLive) + { + this(); + this.name = name; + this.maxVersions = maxVersions; + setMaxVersionsIsSet(true); + this.compression = compression; + this.inMemory = inMemory; + setInMemoryIsSet(true); + this.bloomFilterType = bloomFilterType; + this.bloomFilterVectorSize = bloomFilterVectorSize; + setBloomFilterVectorSizeIsSet(true); + this.bloomFilterNbHashes = bloomFilterNbHashes; + setBloomFilterNbHashesIsSet(true); + this.blockCacheEnabled = blockCacheEnabled; + setBlockCacheEnabledIsSet(true); + this.timeToLive = timeToLive; + setTimeToLiveIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public ColumnDescriptor(ColumnDescriptor other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetName()) { + this.name = other.name; + } + this.maxVersions = other.maxVersions; + if (other.isSetCompression()) { + this.compression = other.compression; + } + this.inMemory = other.inMemory; + if (other.isSetBloomFilterType()) { + this.bloomFilterType = other.bloomFilterType; + } + this.bloomFilterVectorSize = other.bloomFilterVectorSize; + this.bloomFilterNbHashes = other.bloomFilterNbHashes; + this.blockCacheEnabled = other.blockCacheEnabled; + this.timeToLive = other.timeToLive; + } + + public ColumnDescriptor deepCopy() { + return new ColumnDescriptor(this); + } + + @Override + public void clear() { + this.name = null; + this.maxVersions = 3; + + this.compression = "NONE"; + + this.inMemory = false; + + this.bloomFilterType = "NONE"; + + this.bloomFilterVectorSize = 0; + + this.bloomFilterNbHashes = 0; + + this.blockCacheEnabled = false; + + this.timeToLive = -1; + + } + + public byte[] getName() { + setName(org.apache.thrift.TBaseHelper.rightSize(name)); + return name == null ? null : name.array(); + } + + public ByteBuffer bufferForName() { + return name; + } + + public ColumnDescriptor setName(byte[] name) { + setName(name == null ? (ByteBuffer)null : ByteBuffer.wrap(name)); + return this; + } + + public ColumnDescriptor setName(ByteBuffer name) { + this.name = name; + return this; + } + + public void unsetName() { + this.name = null; + } + + /** Returns true if field name is set (has been assigned a value) and false otherwise */ + public boolean isSetName() { + return this.name != null; + } + + public void setNameIsSet(boolean value) { + if (!value) { + this.name = null; + } + } + + public int getMaxVersions() { + return this.maxVersions; + } + + public ColumnDescriptor setMaxVersions(int maxVersions) { + this.maxVersions = maxVersions; + setMaxVersionsIsSet(true); + return this; + } + + public void unsetMaxVersions() { + __isset_bit_vector.clear(__MAXVERSIONS_ISSET_ID); + } + + /** Returns true if field maxVersions is set (has been assigned a value) and false otherwise */ + public boolean isSetMaxVersions() { + return __isset_bit_vector.get(__MAXVERSIONS_ISSET_ID); + } + + public void setMaxVersionsIsSet(boolean value) { + __isset_bit_vector.set(__MAXVERSIONS_ISSET_ID, value); + } + + public String getCompression() { + return this.compression; + } + + public ColumnDescriptor setCompression(String compression) { + this.compression = compression; + return this; + } + + public void unsetCompression() { + this.compression = null; + } + + /** Returns true if field compression is set (has been assigned a value) and false otherwise */ + public boolean isSetCompression() { + return this.compression != null; + } + + public void setCompressionIsSet(boolean value) { + if (!value) { + this.compression = null; + } + } + + public boolean isInMemory() { + return this.inMemory; + } + + public ColumnDescriptor setInMemory(boolean inMemory) { + this.inMemory = inMemory; + setInMemoryIsSet(true); + return this; + } + + public void unsetInMemory() { + __isset_bit_vector.clear(__INMEMORY_ISSET_ID); + } + + /** Returns true if field inMemory is set (has been assigned a value) and false otherwise */ + public boolean isSetInMemory() { + return __isset_bit_vector.get(__INMEMORY_ISSET_ID); + } + + public void setInMemoryIsSet(boolean value) { + __isset_bit_vector.set(__INMEMORY_ISSET_ID, value); + } + + public String getBloomFilterType() { + return this.bloomFilterType; + } + + public ColumnDescriptor setBloomFilterType(String bloomFilterType) { + this.bloomFilterType = bloomFilterType; + return this; + } + + public void unsetBloomFilterType() { + this.bloomFilterType = null; + } + + /** Returns true if field bloomFilterType is set (has been assigned a value) and false otherwise */ + public boolean isSetBloomFilterType() { + return this.bloomFilterType != null; + } + + public void setBloomFilterTypeIsSet(boolean value) { + if (!value) { + this.bloomFilterType = null; + } + } + + public int getBloomFilterVectorSize() { + return this.bloomFilterVectorSize; + } + + public ColumnDescriptor setBloomFilterVectorSize(int bloomFilterVectorSize) { + this.bloomFilterVectorSize = bloomFilterVectorSize; + setBloomFilterVectorSizeIsSet(true); + return this; + } + + public void unsetBloomFilterVectorSize() { + __isset_bit_vector.clear(__BLOOMFILTERVECTORSIZE_ISSET_ID); + } + + /** Returns true if field bloomFilterVectorSize is set (has been assigned a value) and false otherwise */ + public boolean isSetBloomFilterVectorSize() { + return __isset_bit_vector.get(__BLOOMFILTERVECTORSIZE_ISSET_ID); + } + + public void setBloomFilterVectorSizeIsSet(boolean value) { + __isset_bit_vector.set(__BLOOMFILTERVECTORSIZE_ISSET_ID, value); + } + + public int getBloomFilterNbHashes() { + return this.bloomFilterNbHashes; + } + + public ColumnDescriptor setBloomFilterNbHashes(int bloomFilterNbHashes) { + this.bloomFilterNbHashes = bloomFilterNbHashes; + setBloomFilterNbHashesIsSet(true); + return this; + } + + public void unsetBloomFilterNbHashes() { + __isset_bit_vector.clear(__BLOOMFILTERNBHASHES_ISSET_ID); + } + + /** Returns true if field bloomFilterNbHashes is set (has been assigned a value) and false otherwise */ + public boolean isSetBloomFilterNbHashes() { + return __isset_bit_vector.get(__BLOOMFILTERNBHASHES_ISSET_ID); + } + + public void setBloomFilterNbHashesIsSet(boolean value) { + __isset_bit_vector.set(__BLOOMFILTERNBHASHES_ISSET_ID, value); + } + + public boolean isBlockCacheEnabled() { + return this.blockCacheEnabled; + } + + public ColumnDescriptor setBlockCacheEnabled(boolean blockCacheEnabled) { + this.blockCacheEnabled = blockCacheEnabled; + setBlockCacheEnabledIsSet(true); + return this; + } + + public void unsetBlockCacheEnabled() { + __isset_bit_vector.clear(__BLOCKCACHEENABLED_ISSET_ID); + } + + /** Returns true if field blockCacheEnabled is set (has been assigned a value) and false otherwise */ + public boolean isSetBlockCacheEnabled() { + return __isset_bit_vector.get(__BLOCKCACHEENABLED_ISSET_ID); + } + + public void setBlockCacheEnabledIsSet(boolean value) { + __isset_bit_vector.set(__BLOCKCACHEENABLED_ISSET_ID, value); + } + + public int getTimeToLive() { + return this.timeToLive; + } + + public ColumnDescriptor setTimeToLive(int timeToLive) { + this.timeToLive = timeToLive; + setTimeToLiveIsSet(true); + return this; + } + + public void unsetTimeToLive() { + __isset_bit_vector.clear(__TIMETOLIVE_ISSET_ID); + } + + /** Returns true if field timeToLive is set (has been assigned a value) and false otherwise */ + public boolean isSetTimeToLive() { + return __isset_bit_vector.get(__TIMETOLIVE_ISSET_ID); + } + + public void setTimeToLiveIsSet(boolean value) { + __isset_bit_vector.set(__TIMETOLIVE_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case NAME: + if (value == null) { + unsetName(); + } else { + setName((ByteBuffer)value); + } + break; + + case MAX_VERSIONS: + if (value == null) { + unsetMaxVersions(); + } else { + setMaxVersions((Integer)value); + } + break; + + case COMPRESSION: + if (value == null) { + unsetCompression(); + } else { + setCompression((String)value); + } + break; + + case IN_MEMORY: + if (value == null) { + unsetInMemory(); + } else { + setInMemory((Boolean)value); + } + break; + + case BLOOM_FILTER_TYPE: + if (value == null) { + unsetBloomFilterType(); + } else { + setBloomFilterType((String)value); + } + break; + + case BLOOM_FILTER_VECTOR_SIZE: + if (value == null) { + unsetBloomFilterVectorSize(); + } else { + setBloomFilterVectorSize((Integer)value); + } + break; + + case BLOOM_FILTER_NB_HASHES: + if (value == null) { + unsetBloomFilterNbHashes(); + } else { + setBloomFilterNbHashes((Integer)value); + } + break; + + case BLOCK_CACHE_ENABLED: + if (value == null) { + unsetBlockCacheEnabled(); + } else { + setBlockCacheEnabled((Boolean)value); + } + break; + + case TIME_TO_LIVE: + if (value == null) { + unsetTimeToLive(); + } else { + setTimeToLive((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case NAME: + return getName(); + + case MAX_VERSIONS: + return Integer.valueOf(getMaxVersions()); + + case COMPRESSION: + return getCompression(); + + case IN_MEMORY: + return Boolean.valueOf(isInMemory()); + + case BLOOM_FILTER_TYPE: + return getBloomFilterType(); + + case BLOOM_FILTER_VECTOR_SIZE: + return Integer.valueOf(getBloomFilterVectorSize()); + + case BLOOM_FILTER_NB_HASHES: + return Integer.valueOf(getBloomFilterNbHashes()); + + case BLOCK_CACHE_ENABLED: + return Boolean.valueOf(isBlockCacheEnabled()); + + case TIME_TO_LIVE: + return Integer.valueOf(getTimeToLive()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case NAME: + return isSetName(); + case MAX_VERSIONS: + return isSetMaxVersions(); + case COMPRESSION: + return isSetCompression(); + case IN_MEMORY: + return isSetInMemory(); + case BLOOM_FILTER_TYPE: + return isSetBloomFilterType(); + case BLOOM_FILTER_VECTOR_SIZE: + return isSetBloomFilterVectorSize(); + case BLOOM_FILTER_NB_HASHES: + return isSetBloomFilterNbHashes(); + case BLOCK_CACHE_ENABLED: + return isSetBlockCacheEnabled(); + case TIME_TO_LIVE: + return isSetTimeToLive(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof ColumnDescriptor) + return this.equals((ColumnDescriptor)that); + return false; + } + + public boolean equals(ColumnDescriptor that) { + if (that == null) + return false; + + boolean this_present_name = true && this.isSetName(); + boolean that_present_name = true && that.isSetName(); + if (this_present_name || that_present_name) { + if (!(this_present_name && that_present_name)) + return false; + if (!this.name.equals(that.name)) + return false; + } + + boolean this_present_maxVersions = true; + boolean that_present_maxVersions = true; + if (this_present_maxVersions || that_present_maxVersions) { + if (!(this_present_maxVersions && that_present_maxVersions)) + return false; + if (this.maxVersions != that.maxVersions) + return false; + } + + boolean this_present_compression = true && this.isSetCompression(); + boolean that_present_compression = true && that.isSetCompression(); + if (this_present_compression || that_present_compression) { + if (!(this_present_compression && that_present_compression)) + return false; + if (!this.compression.equals(that.compression)) + return false; + } + + boolean this_present_inMemory = true; + boolean that_present_inMemory = true; + if (this_present_inMemory || that_present_inMemory) { + if (!(this_present_inMemory && that_present_inMemory)) + return false; + if (this.inMemory != that.inMemory) + return false; + } + + boolean this_present_bloomFilterType = true && this.isSetBloomFilterType(); + boolean that_present_bloomFilterType = true && that.isSetBloomFilterType(); + if (this_present_bloomFilterType || that_present_bloomFilterType) { + if (!(this_present_bloomFilterType && that_present_bloomFilterType)) + return false; + if (!this.bloomFilterType.equals(that.bloomFilterType)) + return false; + } + + boolean this_present_bloomFilterVectorSize = true; + boolean that_present_bloomFilterVectorSize = true; + if (this_present_bloomFilterVectorSize || that_present_bloomFilterVectorSize) { + if (!(this_present_bloomFilterVectorSize && that_present_bloomFilterVectorSize)) + return false; + if (this.bloomFilterVectorSize != that.bloomFilterVectorSize) + return false; + } + + boolean this_present_bloomFilterNbHashes = true; + boolean that_present_bloomFilterNbHashes = true; + if (this_present_bloomFilterNbHashes || that_present_bloomFilterNbHashes) { + if (!(this_present_bloomFilterNbHashes && that_present_bloomFilterNbHashes)) + return false; + if (this.bloomFilterNbHashes != that.bloomFilterNbHashes) + return false; + } + + boolean this_present_blockCacheEnabled = true; + boolean that_present_blockCacheEnabled = true; + if (this_present_blockCacheEnabled || that_present_blockCacheEnabled) { + if (!(this_present_blockCacheEnabled && that_present_blockCacheEnabled)) + return false; + if (this.blockCacheEnabled != that.blockCacheEnabled) + return false; + } + + boolean this_present_timeToLive = true; + boolean that_present_timeToLive = true; + if (this_present_timeToLive || that_present_timeToLive) { + if (!(this_present_timeToLive && that_present_timeToLive)) + return false; + if (this.timeToLive != that.timeToLive) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(ColumnDescriptor other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + ColumnDescriptor typedOther = (ColumnDescriptor)other; + + lastComparison = Boolean.valueOf(isSetName()).compareTo(typedOther.isSetName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, typedOther.name); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMaxVersions()).compareTo(typedOther.isSetMaxVersions()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMaxVersions()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.maxVersions, typedOther.maxVersions); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetCompression()).compareTo(typedOther.isSetCompression()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCompression()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compression, typedOther.compression); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetInMemory()).compareTo(typedOther.isSetInMemory()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetInMemory()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.inMemory, typedOther.inMemory); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetBloomFilterType()).compareTo(typedOther.isSetBloomFilterType()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetBloomFilterType()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.bloomFilterType, typedOther.bloomFilterType); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetBloomFilterVectorSize()).compareTo(typedOther.isSetBloomFilterVectorSize()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetBloomFilterVectorSize()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.bloomFilterVectorSize, typedOther.bloomFilterVectorSize); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetBloomFilterNbHashes()).compareTo(typedOther.isSetBloomFilterNbHashes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetBloomFilterNbHashes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.bloomFilterNbHashes, typedOther.bloomFilterNbHashes); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetBlockCacheEnabled()).compareTo(typedOther.isSetBlockCacheEnabled()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetBlockCacheEnabled()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.blockCacheEnabled, typedOther.blockCacheEnabled); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimeToLive()).compareTo(typedOther.isSetTimeToLive()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimeToLive()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timeToLive, typedOther.timeToLive); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("ColumnDescriptor("); + boolean first = true; + + sb.append("name:"); + if (this.name == null) { + sb.append("null"); + } else { + sb.append(this.name); + } + first = false; + if (!first) sb.append(", "); + sb.append("maxVersions:"); + sb.append(this.maxVersions); + first = false; + if (!first) sb.append(", "); + sb.append("compression:"); + if (this.compression == null) { + sb.append("null"); + } else { + sb.append(this.compression); + } + first = false; + if (!first) sb.append(", "); + sb.append("inMemory:"); + sb.append(this.inMemory); + first = false; + if (!first) sb.append(", "); + sb.append("bloomFilterType:"); + if (this.bloomFilterType == null) { + sb.append("null"); + } else { + sb.append(this.bloomFilterType); + } + first = false; + if (!first) sb.append(", "); + sb.append("bloomFilterVectorSize:"); + sb.append(this.bloomFilterVectorSize); + first = false; + if (!first) sb.append(", "); + sb.append("bloomFilterNbHashes:"); + sb.append(this.bloomFilterNbHashes); + first = false; + if (!first) sb.append(", "); + sb.append("blockCacheEnabled:"); + sb.append(this.blockCacheEnabled); + first = false; + if (!first) sb.append(", "); + sb.append("timeToLive:"); + sb.append(this.timeToLive); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class ColumnDescriptorStandardSchemeFactory implements SchemeFactory { + public ColumnDescriptorStandardScheme getScheme() { + return new ColumnDescriptorStandardScheme(); + } + } + + private static class ColumnDescriptorStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, ColumnDescriptor struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.name = iprot.readBinary(); + struct.setNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // MAX_VERSIONS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.maxVersions = iprot.readI32(); + struct.setMaxVersionsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COMPRESSION + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.compression = iprot.readString(); + struct.setCompressionIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // IN_MEMORY + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.inMemory = iprot.readBool(); + struct.setInMemoryIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // BLOOM_FILTER_TYPE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.bloomFilterType = iprot.readString(); + struct.setBloomFilterTypeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 6: // BLOOM_FILTER_VECTOR_SIZE + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.bloomFilterVectorSize = iprot.readI32(); + struct.setBloomFilterVectorSizeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 7: // BLOOM_FILTER_NB_HASHES + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.bloomFilterNbHashes = iprot.readI32(); + struct.setBloomFilterNbHashesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 8: // BLOCK_CACHE_ENABLED + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.blockCacheEnabled = iprot.readBool(); + struct.setBlockCacheEnabledIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 9: // TIME_TO_LIVE + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.timeToLive = iprot.readI32(); + struct.setTimeToLiveIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, ColumnDescriptor struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.name != null) { + oprot.writeFieldBegin(NAME_FIELD_DESC); + oprot.writeBinary(struct.name); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(MAX_VERSIONS_FIELD_DESC); + oprot.writeI32(struct.maxVersions); + oprot.writeFieldEnd(); + if (struct.compression != null) { + oprot.writeFieldBegin(COMPRESSION_FIELD_DESC); + oprot.writeString(struct.compression); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(IN_MEMORY_FIELD_DESC); + oprot.writeBool(struct.inMemory); + oprot.writeFieldEnd(); + if (struct.bloomFilterType != null) { + oprot.writeFieldBegin(BLOOM_FILTER_TYPE_FIELD_DESC); + oprot.writeString(struct.bloomFilterType); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(BLOOM_FILTER_VECTOR_SIZE_FIELD_DESC); + oprot.writeI32(struct.bloomFilterVectorSize); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(BLOOM_FILTER_NB_HASHES_FIELD_DESC); + oprot.writeI32(struct.bloomFilterNbHashes); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(BLOCK_CACHE_ENABLED_FIELD_DESC); + oprot.writeBool(struct.blockCacheEnabled); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(TIME_TO_LIVE_FIELD_DESC); + oprot.writeI32(struct.timeToLive); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class ColumnDescriptorTupleSchemeFactory implements SchemeFactory { + public ColumnDescriptorTupleScheme getScheme() { + return new ColumnDescriptorTupleScheme(); + } + } + + private static class ColumnDescriptorTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, ColumnDescriptor struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetName()) { + optionals.set(0); + } + if (struct.isSetMaxVersions()) { + optionals.set(1); + } + if (struct.isSetCompression()) { + optionals.set(2); + } + if (struct.isSetInMemory()) { + optionals.set(3); + } + if (struct.isSetBloomFilterType()) { + optionals.set(4); + } + if (struct.isSetBloomFilterVectorSize()) { + optionals.set(5); + } + if (struct.isSetBloomFilterNbHashes()) { + optionals.set(6); + } + if (struct.isSetBlockCacheEnabled()) { + optionals.set(7); + } + if (struct.isSetTimeToLive()) { + optionals.set(8); + } + oprot.writeBitSet(optionals, 9); + if (struct.isSetName()) { + oprot.writeBinary(struct.name); + } + if (struct.isSetMaxVersions()) { + oprot.writeI32(struct.maxVersions); + } + if (struct.isSetCompression()) { + oprot.writeString(struct.compression); + } + if (struct.isSetInMemory()) { + oprot.writeBool(struct.inMemory); + } + if (struct.isSetBloomFilterType()) { + oprot.writeString(struct.bloomFilterType); + } + if (struct.isSetBloomFilterVectorSize()) { + oprot.writeI32(struct.bloomFilterVectorSize); + } + if (struct.isSetBloomFilterNbHashes()) { + oprot.writeI32(struct.bloomFilterNbHashes); + } + if (struct.isSetBlockCacheEnabled()) { + oprot.writeBool(struct.blockCacheEnabled); + } + if (struct.isSetTimeToLive()) { + oprot.writeI32(struct.timeToLive); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, ColumnDescriptor struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(9); + if (incoming.get(0)) { + struct.name = iprot.readBinary(); + struct.setNameIsSet(true); + } + if (incoming.get(1)) { + struct.maxVersions = iprot.readI32(); + struct.setMaxVersionsIsSet(true); + } + if (incoming.get(2)) { + struct.compression = iprot.readString(); + struct.setCompressionIsSet(true); + } + if (incoming.get(3)) { + struct.inMemory = iprot.readBool(); + struct.setInMemoryIsSet(true); + } + if (incoming.get(4)) { + struct.bloomFilterType = iprot.readString(); + struct.setBloomFilterTypeIsSet(true); + } + if (incoming.get(5)) { + struct.bloomFilterVectorSize = iprot.readI32(); + struct.setBloomFilterVectorSizeIsSet(true); + } + if (incoming.get(6)) { + struct.bloomFilterNbHashes = iprot.readI32(); + struct.setBloomFilterNbHashesIsSet(true); + } + if (incoming.get(7)) { + struct.blockCacheEnabled = iprot.readBool(); + struct.setBlockCacheEnabledIsSet(true); + } + if (incoming.get(8)) { + struct.timeToLive = iprot.readI32(); + struct.setTimeToLiveIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java new file mode 100644 index 0000000..e9ca236 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java @@ -0,0 +1,52969 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Hbase { + + public interface Iface { + + /** + * Brings a table on-line (enables it) + * + * @param tableName name of the table + */ + public void enableTable(ByteBuffer tableName) throws IOError, org.apache.thrift.TException; + + /** + * Disables a table (takes it off-line) If it is being served, the master + * will tell the servers to stop serving it. + * + * @param tableName name of the table + */ + public void disableTable(ByteBuffer tableName) throws IOError, org.apache.thrift.TException; + + /** + * @return true if table is on-line + * + * @param tableName name of the table to check + */ + public boolean isTableEnabled(ByteBuffer tableName) throws IOError, org.apache.thrift.TException; + + public void compact(ByteBuffer tableNameOrRegionName) throws IOError, org.apache.thrift.TException; + + public void majorCompact(ByteBuffer tableNameOrRegionName) throws IOError, org.apache.thrift.TException; + + /** + * List all the userspace tables. + * + * @return returns a list of names + */ + public List getTableNames() throws IOError, org.apache.thrift.TException; + + /** + * List all the column families assoicated with a table. + * + * @return list of column family descriptors + * + * @param tableName table name + */ + public Map getColumnDescriptors(ByteBuffer tableName) throws IOError, org.apache.thrift.TException; + + /** + * List the regions associated with a table. + * + * @return list of region descriptors + * + * @param tableName table name + */ + public List getTableRegions(ByteBuffer tableName) throws IOError, org.apache.thrift.TException; + + /** + * Create a table with the specified column families. The name + * field for each ColumnDescriptor must be set and must end in a + * colon (:). All other fields are optional and will get default + * values if not explicitly specified. + * + * @throws IllegalArgument if an input parameter is invalid + * + * @throws AlreadyExists if the table name already exists + * + * @param tableName name of table to create + * + * @param columnFamilies list of column family descriptors + */ + public void createTable(ByteBuffer tableName, List columnFamilies) throws IOError, IllegalArgument, AlreadyExists, org.apache.thrift.TException; + + /** + * Deletes a table + * + * @throws IOError if table doesn't exist on server or there was some other + * problem + * + * @param tableName name of table to delete + */ + public void deleteTable(ByteBuffer tableName) throws IOError, org.apache.thrift.TException; + + /** + * Get a single TCell for the specified table, row, and column at the + * latest timestamp. Returns an empty list if no such value exists. + * + * @return value for specified row/column + * + * @param tableName name of table + * + * @param row row key + * + * @param column column name + * + * @param attributes Get attributes + */ + public List get(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get the specified number of versions for the specified table, + * row, and column. + * + * @return list of cells for specified row/column + * + * @param tableName name of table + * + * @param row row key + * + * @param column column name + * + * @param numVersions number of versions to retrieve + * + * @param attributes Get attributes + */ + public List getVer(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, int numVersions, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get the specified number of versions for the specified table, + * row, and column. Only versions less than or equal to the specified + * timestamp will be returned. + * + * @return list of cells for specified row/column + * + * @param tableName name of table + * + * @param row row key + * + * @param column column name + * + * @param timestamp timestamp + * + * @param numVersions number of versions to retrieve + * + * @param attributes Get attributes + */ + public List getVerTs(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, int numVersions, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get all the data for the specified table and row at the latest + * timestamp. Returns an empty list if the row does not exist. + * + * @return TRowResult containing the row and map of columns to TCells + * + * @param tableName name of table + * + * @param row row key + * + * @param attributes Get attributes + */ + public List getRow(ByteBuffer tableName, ByteBuffer row, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get the specified columns for the specified table and row at the latest + * timestamp. Returns an empty list if the row does not exist. + * + * @return TRowResult containing the row and map of columns to TCells + * + * @param tableName name of table + * + * @param row row key + * + * @param columns List of columns to return, null for all columns + * + * @param attributes Get attributes + */ + public List getRowWithColumns(ByteBuffer tableName, ByteBuffer row, List columns, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get all the data for the specified table and row at the specified + * timestamp. Returns an empty list if the row does not exist. + * + * @return TRowResult containing the row and map of columns to TCells + * + * @param tableName name of the table + * + * @param row row key + * + * @param timestamp timestamp + * + * @param attributes Get attributes + */ + public List getRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get the specified columns for the specified table and row at the specified + * timestamp. Returns an empty list if the row does not exist. + * + * @return TRowResult containing the row and map of columns to TCells + * + * @param tableName name of table + * + * @param row row key + * + * @param columns List of columns to return, null for all columns + * + * @param timestamp + * @param attributes Get attributes + */ + public List getRowWithColumnsTs(ByteBuffer tableName, ByteBuffer row, List columns, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get all the data for the specified table and rows at the latest + * timestamp. Returns an empty list if no rows exist. + * + * @return TRowResult containing the rows and map of columns to TCells + * + * @param tableName name of table + * + * @param rows row keys + * + * @param attributes Get attributes + */ + public List getRows(ByteBuffer tableName, List rows, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get the specified columns for the specified table and rows at the latest + * timestamp. Returns an empty list if no rows exist. + * + * @return TRowResult containing the rows and map of columns to TCells + * + * @param tableName name of table + * + * @param rows row keys + * + * @param columns List of columns to return, null for all columns + * + * @param attributes Get attributes + */ + public List getRowsWithColumns(ByteBuffer tableName, List rows, List columns, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get all the data for the specified table and rows at the specified + * timestamp. Returns an empty list if no rows exist. + * + * @return TRowResult containing the rows and map of columns to TCells + * + * @param tableName name of the table + * + * @param rows row keys + * + * @param timestamp timestamp + * + * @param attributes Get attributes + */ + public List getRowsTs(ByteBuffer tableName, List rows, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get the specified columns for the specified table and rows at the specified + * timestamp. Returns an empty list if no rows exist. + * + * @return TRowResult containing the rows and map of columns to TCells + * + * @param tableName name of table + * + * @param rows row keys + * + * @param columns List of columns to return, null for all columns + * + * @param timestamp + * @param attributes Get attributes + */ + public List getRowsWithColumnsTs(ByteBuffer tableName, List rows, List columns, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Apply a series of mutations (updates/deletes) to a row in a + * single transaction. If an exception is thrown, then the + * transaction is aborted. Default current timestamp is used, and + * all entries will have an identical timestamp. + * + * @param tableName name of table + * + * @param row row key + * + * @param mutations list of mutation commands + * + * @param attributes Mutation attributes + */ + public void mutateRow(ByteBuffer tableName, ByteBuffer row, List mutations, Map attributes) throws IOError, IllegalArgument, org.apache.thrift.TException; + + /** + * Apply a series of mutations (updates/deletes) to a row in a + * single transaction. If an exception is thrown, then the + * transaction is aborted. The specified timestamp is used, and + * all entries will have an identical timestamp. + * + * @param tableName name of table + * + * @param row row key + * + * @param mutations list of mutation commands + * + * @param timestamp timestamp + * + * @param attributes Mutation attributes + */ + public void mutateRowTs(ByteBuffer tableName, ByteBuffer row, List mutations, long timestamp, Map attributes) throws IOError, IllegalArgument, org.apache.thrift.TException; + + /** + * Apply a series of batches (each a series of mutations on a single row) + * in a single transaction. If an exception is thrown, then the + * transaction is aborted. Default current timestamp is used, and + * all entries will have an identical timestamp. + * + * @param tableName name of table + * + * @param rowBatches list of row batches + * + * @param attributes Mutation attributes + */ + public void mutateRows(ByteBuffer tableName, List rowBatches, Map attributes) throws IOError, IllegalArgument, org.apache.thrift.TException; + + /** + * Apply a series of batches (each a series of mutations on a single row) + * in a single transaction. If an exception is thrown, then the + * transaction is aborted. The specified timestamp is used, and + * all entries will have an identical timestamp. + * + * @param tableName name of table + * + * @param rowBatches list of row batches + * + * @param timestamp timestamp + * + * @param attributes Mutation attributes + */ + public void mutateRowsTs(ByteBuffer tableName, List rowBatches, long timestamp, Map attributes) throws IOError, IllegalArgument, org.apache.thrift.TException; + + /** + * Atomically increment the column value specified. Returns the next value post increment. + * + * @param tableName name of table + * + * @param row row to increment + * + * @param column name of column + * + * @param value amount to increment by + */ + public long atomicIncrement(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long value) throws IOError, IllegalArgument, org.apache.thrift.TException; + + /** + * Delete all cells that match the passed row and column. + * + * @param tableName name of table + * + * @param row Row to update + * + * @param column name of column whose value is to be deleted + * + * @param attributes Delete attributes + */ + public void deleteAll(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Delete all cells that match the passed row and column and whose + * timestamp is equal-to or older than the passed timestamp. + * + * @param tableName name of table + * + * @param row Row to update + * + * @param column name of column whose value is to be deleted + * + * @param timestamp timestamp + * + * @param attributes Delete attributes + */ + public void deleteAllTs(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Completely delete the row's cells. + * + * @param tableName name of table + * + * @param row key of the row to be completely deleted. + * + * @param attributes Delete attributes + */ + public void deleteAllRow(ByteBuffer tableName, ByteBuffer row, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Increment a cell by the ammount. + * Increments can be applied async if hbase.regionserver.thrift.coalesceIncrement is set to true. + * False is the default. Turn to true if you need the extra performance and can accept some + * data loss if a thrift server dies with increments still in the queue. + * + * @param increment The single increment to apply + */ + public void increment(TIncrement increment) throws IOError, org.apache.thrift.TException; + + public void incrementRows(List increments) throws IOError, org.apache.thrift.TException; + + /** + * Completely delete the row's cells marked with a timestamp + * equal-to or older than the passed timestamp. + * + * @param tableName name of table + * + * @param row key of the row to be completely deleted. + * + * @param timestamp timestamp + * + * @param attributes Delete attributes + */ + public void deleteAllRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get a scanner on the current table, using the Scan instance + * for the scan parameters. + * + * @param tableName name of table + * + * @param scan Scan instance + * + * @param attributes Scan attributes + */ + public int scannerOpenWithScan(ByteBuffer tableName, TScan scan, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get a scanner on the current table starting at the specified row and + * ending at the last row in the table. Return the specified columns. + * + * @return scanner id to be used with other scanner procedures + * + * @param tableName name of table + * + * @param startRow Starting row in table to scan. + * Send "" (empty string) to start at the first row. + * + * @param columns columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + * + * @param attributes Scan attributes + */ + public int scannerOpen(ByteBuffer tableName, ByteBuffer startRow, List columns, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get a scanner on the current table starting and stopping at the + * specified rows. ending at the last row in the table. Return the + * specified columns. + * + * @return scanner id to be used with other scanner procedures + * + * @param tableName name of table + * + * @param startRow Starting row in table to scan. + * Send "" (empty string) to start at the first row. + * + * @param stopRow row to stop scanning on. This row is *not* included in the + * scanner's results + * + * @param columns columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + * + * @param attributes Scan attributes + */ + public int scannerOpenWithStop(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List columns, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Open a scanner for a given prefix. That is all rows will have the specified + * prefix. No other rows will be returned. + * + * @return scanner id to use with other scanner calls + * + * @param tableName name of table + * + * @param startAndPrefix the prefix (and thus start row) of the keys you want + * + * @param columns the columns you want returned + * + * @param attributes Scan attributes + */ + public int scannerOpenWithPrefix(ByteBuffer tableName, ByteBuffer startAndPrefix, List columns, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get a scanner on the current table starting at the specified row and + * ending at the last row in the table. Return the specified columns. + * Only values with the specified timestamp are returned. + * + * @return scanner id to be used with other scanner procedures + * + * @param tableName name of table + * + * @param startRow Starting row in table to scan. + * Send "" (empty string) to start at the first row. + * + * @param columns columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + * + * @param timestamp timestamp + * + * @param attributes Scan attributes + */ + public int scannerOpenTs(ByteBuffer tableName, ByteBuffer startRow, List columns, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Get a scanner on the current table starting and stopping at the + * specified rows. ending at the last row in the table. Return the + * specified columns. Only values with the specified timestamp are + * returned. + * + * @return scanner id to be used with other scanner procedures + * + * @param tableName name of table + * + * @param startRow Starting row in table to scan. + * Send "" (empty string) to start at the first row. + * + * @param stopRow row to stop scanning on. This row is *not* included in the + * scanner's results + * + * @param columns columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + * + * @param timestamp timestamp + * + * @param attributes Scan attributes + */ + public int scannerOpenWithStopTs(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List columns, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException; + + /** + * Returns the scanner's current row value and advances to the next + * row in the table. When there are no more rows in the table, or a key + * greater-than-or-equal-to the scanner's specified stopRow is reached, + * an empty list is returned. + * + * @return a TRowResult containing the current row and a map of the columns to TCells. + * + * @throws IllegalArgument if ScannerID is invalid + * + * @throws NotFound when the scanner reaches the end + * + * @param id id of a scanner returned by scannerOpen + */ + public List scannerGet(int id) throws IOError, IllegalArgument, org.apache.thrift.TException; + + /** + * Returns, starting at the scanner's current row value nbRows worth of + * rows and advances to the next row in the table. When there are no more + * rows in the table, or a key greater-than-or-equal-to the scanner's + * specified stopRow is reached, an empty list is returned. + * + * @return a TRowResult containing the current row and a map of the columns to TCells. + * + * @throws IllegalArgument if ScannerID is invalid + * + * @throws NotFound when the scanner reaches the end + * + * @param id id of a scanner returned by scannerOpen + * + * @param nbRows number of results to return + */ + public List scannerGetList(int id, int nbRows) throws IOError, IllegalArgument, org.apache.thrift.TException; + + /** + * Closes the server-state associated with an open scanner. + * + * @throws IllegalArgument if ScannerID is invalid + * + * @param id id of a scanner returned by scannerOpen + */ + public void scannerClose(int id) throws IOError, IllegalArgument, org.apache.thrift.TException; + + /** + * Get the row just before the specified one. + * + * @return value for specified row/column + * + * @param tableName name of table + * + * @param row row key + * + * @param family column name + */ + public List getRowOrBefore(ByteBuffer tableName, ByteBuffer row, ByteBuffer family) throws IOError, org.apache.thrift.TException; + + /** + * Get the regininfo for the specified row. It scans + * the metatable to find region's start and end keys. + * + * @return value for specified row/column + * + * @param row row key + */ + public TRegionInfo getRegionInfo(ByteBuffer row) throws IOError, org.apache.thrift.TException; + + } + + public interface AsyncIface { + + public void enableTable(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void disableTable(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void isTableEnabled(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void compact(ByteBuffer tableNameOrRegionName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void majorCompact(ByteBuffer tableNameOrRegionName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getTableNames(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getColumnDescriptors(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getTableRegions(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void createTable(ByteBuffer tableName, List columnFamilies, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void deleteTable(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void get(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getVer(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, int numVersions, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getVerTs(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, int numVersions, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getRow(ByteBuffer tableName, ByteBuffer row, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getRowWithColumns(ByteBuffer tableName, ByteBuffer row, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getRowWithColumnsTs(ByteBuffer tableName, ByteBuffer row, List columns, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getRows(ByteBuffer tableName, List rows, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getRowsWithColumns(ByteBuffer tableName, List rows, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getRowsTs(ByteBuffer tableName, List rows, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getRowsWithColumnsTs(ByteBuffer tableName, List rows, List columns, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void mutateRow(ByteBuffer tableName, ByteBuffer row, List mutations, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void mutateRowTs(ByteBuffer tableName, ByteBuffer row, List mutations, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void mutateRows(ByteBuffer tableName, List rowBatches, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void mutateRowsTs(ByteBuffer tableName, List rowBatches, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void atomicIncrement(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long value, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void deleteAll(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void deleteAllTs(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void deleteAllRow(ByteBuffer tableName, ByteBuffer row, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void increment(TIncrement increment, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void incrementRows(List increments, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void deleteAllRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void scannerOpenWithScan(ByteBuffer tableName, TScan scan, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void scannerOpen(ByteBuffer tableName, ByteBuffer startRow, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void scannerOpenWithStop(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void scannerOpenWithPrefix(ByteBuffer tableName, ByteBuffer startAndPrefix, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void scannerOpenTs(ByteBuffer tableName, ByteBuffer startRow, List columns, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void scannerOpenWithStopTs(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List columns, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void scannerGet(int id, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void scannerGetList(int id, int nbRows, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void scannerClose(int id, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getRowOrBefore(ByteBuffer tableName, ByteBuffer row, ByteBuffer family, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getRegionInfo(ByteBuffer row, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + } + + public static class Client extends org.apache.thrift.TServiceClient implements Iface { + public static class Factory implements org.apache.thrift.TServiceClientFactory { + public Factory() {} + public Client getClient(org.apache.thrift.protocol.TProtocol prot) { + return new Client(prot); + } + public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) { + return new Client(iprot, oprot); + } + } + + public Client(org.apache.thrift.protocol.TProtocol prot) + { + super(prot, prot); + } + + public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) { + super(iprot, oprot); + } + + public void enableTable(ByteBuffer tableName) throws IOError, org.apache.thrift.TException + { + send_enableTable(tableName); + recv_enableTable(); + } + + public void send_enableTable(ByteBuffer tableName) throws org.apache.thrift.TException + { + enableTable_args args = new enableTable_args(); + args.setTableName(tableName); + sendBase("enableTable", args); + } + + public void recv_enableTable() throws IOError, org.apache.thrift.TException + { + enableTable_result result = new enableTable_result(); + receiveBase(result, "enableTable"); + if (result.io != null) { + throw result.io; + } + return; + } + + public void disableTable(ByteBuffer tableName) throws IOError, org.apache.thrift.TException + { + send_disableTable(tableName); + recv_disableTable(); + } + + public void send_disableTable(ByteBuffer tableName) throws org.apache.thrift.TException + { + disableTable_args args = new disableTable_args(); + args.setTableName(tableName); + sendBase("disableTable", args); + } + + public void recv_disableTable() throws IOError, org.apache.thrift.TException + { + disableTable_result result = new disableTable_result(); + receiveBase(result, "disableTable"); + if (result.io != null) { + throw result.io; + } + return; + } + + public boolean isTableEnabled(ByteBuffer tableName) throws IOError, org.apache.thrift.TException + { + send_isTableEnabled(tableName); + return recv_isTableEnabled(); + } + + public void send_isTableEnabled(ByteBuffer tableName) throws org.apache.thrift.TException + { + isTableEnabled_args args = new isTableEnabled_args(); + args.setTableName(tableName); + sendBase("isTableEnabled", args); + } + + public boolean recv_isTableEnabled() throws IOError, org.apache.thrift.TException + { + isTableEnabled_result result = new isTableEnabled_result(); + receiveBase(result, "isTableEnabled"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "isTableEnabled failed: unknown result"); + } + + public void compact(ByteBuffer tableNameOrRegionName) throws IOError, org.apache.thrift.TException + { + send_compact(tableNameOrRegionName); + recv_compact(); + } + + public void send_compact(ByteBuffer tableNameOrRegionName) throws org.apache.thrift.TException + { + compact_args args = new compact_args(); + args.setTableNameOrRegionName(tableNameOrRegionName); + sendBase("compact", args); + } + + public void recv_compact() throws IOError, org.apache.thrift.TException + { + compact_result result = new compact_result(); + receiveBase(result, "compact"); + if (result.io != null) { + throw result.io; + } + return; + } + + public void majorCompact(ByteBuffer tableNameOrRegionName) throws IOError, org.apache.thrift.TException + { + send_majorCompact(tableNameOrRegionName); + recv_majorCompact(); + } + + public void send_majorCompact(ByteBuffer tableNameOrRegionName) throws org.apache.thrift.TException + { + majorCompact_args args = new majorCompact_args(); + args.setTableNameOrRegionName(tableNameOrRegionName); + sendBase("majorCompact", args); + } + + public void recv_majorCompact() throws IOError, org.apache.thrift.TException + { + majorCompact_result result = new majorCompact_result(); + receiveBase(result, "majorCompact"); + if (result.io != null) { + throw result.io; + } + return; + } + + public List getTableNames() throws IOError, org.apache.thrift.TException + { + send_getTableNames(); + return recv_getTableNames(); + } + + public void send_getTableNames() throws org.apache.thrift.TException + { + getTableNames_args args = new getTableNames_args(); + sendBase("getTableNames", args); + } + + public List recv_getTableNames() throws IOError, org.apache.thrift.TException + { + getTableNames_result result = new getTableNames_result(); + receiveBase(result, "getTableNames"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getTableNames failed: unknown result"); + } + + public Map getColumnDescriptors(ByteBuffer tableName) throws IOError, org.apache.thrift.TException + { + send_getColumnDescriptors(tableName); + return recv_getColumnDescriptors(); + } + + public void send_getColumnDescriptors(ByteBuffer tableName) throws org.apache.thrift.TException + { + getColumnDescriptors_args args = new getColumnDescriptors_args(); + args.setTableName(tableName); + sendBase("getColumnDescriptors", args); + } + + public Map recv_getColumnDescriptors() throws IOError, org.apache.thrift.TException + { + getColumnDescriptors_result result = new getColumnDescriptors_result(); + receiveBase(result, "getColumnDescriptors"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getColumnDescriptors failed: unknown result"); + } + + public List getTableRegions(ByteBuffer tableName) throws IOError, org.apache.thrift.TException + { + send_getTableRegions(tableName); + return recv_getTableRegions(); + } + + public void send_getTableRegions(ByteBuffer tableName) throws org.apache.thrift.TException + { + getTableRegions_args args = new getTableRegions_args(); + args.setTableName(tableName); + sendBase("getTableRegions", args); + } + + public List recv_getTableRegions() throws IOError, org.apache.thrift.TException + { + getTableRegions_result result = new getTableRegions_result(); + receiveBase(result, "getTableRegions"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getTableRegions failed: unknown result"); + } + + public void createTable(ByteBuffer tableName, List columnFamilies) throws IOError, IllegalArgument, AlreadyExists, org.apache.thrift.TException + { + send_createTable(tableName, columnFamilies); + recv_createTable(); + } + + public void send_createTable(ByteBuffer tableName, List columnFamilies) throws org.apache.thrift.TException + { + createTable_args args = new createTable_args(); + args.setTableName(tableName); + args.setColumnFamilies(columnFamilies); + sendBase("createTable", args); + } + + public void recv_createTable() throws IOError, IllegalArgument, AlreadyExists, org.apache.thrift.TException + { + createTable_result result = new createTable_result(); + receiveBase(result, "createTable"); + if (result.io != null) { + throw result.io; + } + if (result.ia != null) { + throw result.ia; + } + if (result.exist != null) { + throw result.exist; + } + return; + } + + public void deleteTable(ByteBuffer tableName) throws IOError, org.apache.thrift.TException + { + send_deleteTable(tableName); + recv_deleteTable(); + } + + public void send_deleteTable(ByteBuffer tableName) throws org.apache.thrift.TException + { + deleteTable_args args = new deleteTable_args(); + args.setTableName(tableName); + sendBase("deleteTable", args); + } + + public void recv_deleteTable() throws IOError, org.apache.thrift.TException + { + deleteTable_result result = new deleteTable_result(); + receiveBase(result, "deleteTable"); + if (result.io != null) { + throw result.io; + } + return; + } + + public List get(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, Map attributes) throws IOError, org.apache.thrift.TException + { + send_get(tableName, row, column, attributes); + return recv_get(); + } + + public void send_get(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, Map attributes) throws org.apache.thrift.TException + { + get_args args = new get_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumn(column); + args.setAttributes(attributes); + sendBase("get", args); + } + + public List recv_get() throws IOError, org.apache.thrift.TException + { + get_result result = new get_result(); + receiveBase(result, "get"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get failed: unknown result"); + } + + public List getVer(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, int numVersions, Map attributes) throws IOError, org.apache.thrift.TException + { + send_getVer(tableName, row, column, numVersions, attributes); + return recv_getVer(); + } + + public void send_getVer(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, int numVersions, Map attributes) throws org.apache.thrift.TException + { + getVer_args args = new getVer_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumn(column); + args.setNumVersions(numVersions); + args.setAttributes(attributes); + sendBase("getVer", args); + } + + public List recv_getVer() throws IOError, org.apache.thrift.TException + { + getVer_result result = new getVer_result(); + receiveBase(result, "getVer"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getVer failed: unknown result"); + } + + public List getVerTs(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, int numVersions, Map attributes) throws IOError, org.apache.thrift.TException + { + send_getVerTs(tableName, row, column, timestamp, numVersions, attributes); + return recv_getVerTs(); + } + + public void send_getVerTs(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, int numVersions, Map attributes) throws org.apache.thrift.TException + { + getVerTs_args args = new getVerTs_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumn(column); + args.setTimestamp(timestamp); + args.setNumVersions(numVersions); + args.setAttributes(attributes); + sendBase("getVerTs", args); + } + + public List recv_getVerTs() throws IOError, org.apache.thrift.TException + { + getVerTs_result result = new getVerTs_result(); + receiveBase(result, "getVerTs"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getVerTs failed: unknown result"); + } + + public List getRow(ByteBuffer tableName, ByteBuffer row, Map attributes) throws IOError, org.apache.thrift.TException + { + send_getRow(tableName, row, attributes); + return recv_getRow(); + } + + public void send_getRow(ByteBuffer tableName, ByteBuffer row, Map attributes) throws org.apache.thrift.TException + { + getRow_args args = new getRow_args(); + args.setTableName(tableName); + args.setRow(row); + args.setAttributes(attributes); + sendBase("getRow", args); + } + + public List recv_getRow() throws IOError, org.apache.thrift.TException + { + getRow_result result = new getRow_result(); + receiveBase(result, "getRow"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRow failed: unknown result"); + } + + public List getRowWithColumns(ByteBuffer tableName, ByteBuffer row, List columns, Map attributes) throws IOError, org.apache.thrift.TException + { + send_getRowWithColumns(tableName, row, columns, attributes); + return recv_getRowWithColumns(); + } + + public void send_getRowWithColumns(ByteBuffer tableName, ByteBuffer row, List columns, Map attributes) throws org.apache.thrift.TException + { + getRowWithColumns_args args = new getRowWithColumns_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumns(columns); + args.setAttributes(attributes); + sendBase("getRowWithColumns", args); + } + + public List recv_getRowWithColumns() throws IOError, org.apache.thrift.TException + { + getRowWithColumns_result result = new getRowWithColumns_result(); + receiveBase(result, "getRowWithColumns"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRowWithColumns failed: unknown result"); + } + + public List getRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException + { + send_getRowTs(tableName, row, timestamp, attributes); + return recv_getRowTs(); + } + + public void send_getRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map attributes) throws org.apache.thrift.TException + { + getRowTs_args args = new getRowTs_args(); + args.setTableName(tableName); + args.setRow(row); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + sendBase("getRowTs", args); + } + + public List recv_getRowTs() throws IOError, org.apache.thrift.TException + { + getRowTs_result result = new getRowTs_result(); + receiveBase(result, "getRowTs"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRowTs failed: unknown result"); + } + + public List getRowWithColumnsTs(ByteBuffer tableName, ByteBuffer row, List columns, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException + { + send_getRowWithColumnsTs(tableName, row, columns, timestamp, attributes); + return recv_getRowWithColumnsTs(); + } + + public void send_getRowWithColumnsTs(ByteBuffer tableName, ByteBuffer row, List columns, long timestamp, Map attributes) throws org.apache.thrift.TException + { + getRowWithColumnsTs_args args = new getRowWithColumnsTs_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumns(columns); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + sendBase("getRowWithColumnsTs", args); + } + + public List recv_getRowWithColumnsTs() throws IOError, org.apache.thrift.TException + { + getRowWithColumnsTs_result result = new getRowWithColumnsTs_result(); + receiveBase(result, "getRowWithColumnsTs"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRowWithColumnsTs failed: unknown result"); + } + + public List getRows(ByteBuffer tableName, List rows, Map attributes) throws IOError, org.apache.thrift.TException + { + send_getRows(tableName, rows, attributes); + return recv_getRows(); + } + + public void send_getRows(ByteBuffer tableName, List rows, Map attributes) throws org.apache.thrift.TException + { + getRows_args args = new getRows_args(); + args.setTableName(tableName); + args.setRows(rows); + args.setAttributes(attributes); + sendBase("getRows", args); + } + + public List recv_getRows() throws IOError, org.apache.thrift.TException + { + getRows_result result = new getRows_result(); + receiveBase(result, "getRows"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRows failed: unknown result"); + } + + public List getRowsWithColumns(ByteBuffer tableName, List rows, List columns, Map attributes) throws IOError, org.apache.thrift.TException + { + send_getRowsWithColumns(tableName, rows, columns, attributes); + return recv_getRowsWithColumns(); + } + + public void send_getRowsWithColumns(ByteBuffer tableName, List rows, List columns, Map attributes) throws org.apache.thrift.TException + { + getRowsWithColumns_args args = new getRowsWithColumns_args(); + args.setTableName(tableName); + args.setRows(rows); + args.setColumns(columns); + args.setAttributes(attributes); + sendBase("getRowsWithColumns", args); + } + + public List recv_getRowsWithColumns() throws IOError, org.apache.thrift.TException + { + getRowsWithColumns_result result = new getRowsWithColumns_result(); + receiveBase(result, "getRowsWithColumns"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRowsWithColumns failed: unknown result"); + } + + public List getRowsTs(ByteBuffer tableName, List rows, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException + { + send_getRowsTs(tableName, rows, timestamp, attributes); + return recv_getRowsTs(); + } + + public void send_getRowsTs(ByteBuffer tableName, List rows, long timestamp, Map attributes) throws org.apache.thrift.TException + { + getRowsTs_args args = new getRowsTs_args(); + args.setTableName(tableName); + args.setRows(rows); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + sendBase("getRowsTs", args); + } + + public List recv_getRowsTs() throws IOError, org.apache.thrift.TException + { + getRowsTs_result result = new getRowsTs_result(); + receiveBase(result, "getRowsTs"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRowsTs failed: unknown result"); + } + + public List getRowsWithColumnsTs(ByteBuffer tableName, List rows, List columns, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException + { + send_getRowsWithColumnsTs(tableName, rows, columns, timestamp, attributes); + return recv_getRowsWithColumnsTs(); + } + + public void send_getRowsWithColumnsTs(ByteBuffer tableName, List rows, List columns, long timestamp, Map attributes) throws org.apache.thrift.TException + { + getRowsWithColumnsTs_args args = new getRowsWithColumnsTs_args(); + args.setTableName(tableName); + args.setRows(rows); + args.setColumns(columns); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + sendBase("getRowsWithColumnsTs", args); + } + + public List recv_getRowsWithColumnsTs() throws IOError, org.apache.thrift.TException + { + getRowsWithColumnsTs_result result = new getRowsWithColumnsTs_result(); + receiveBase(result, "getRowsWithColumnsTs"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRowsWithColumnsTs failed: unknown result"); + } + + public void mutateRow(ByteBuffer tableName, ByteBuffer row, List mutations, Map attributes) throws IOError, IllegalArgument, org.apache.thrift.TException + { + send_mutateRow(tableName, row, mutations, attributes); + recv_mutateRow(); + } + + public void send_mutateRow(ByteBuffer tableName, ByteBuffer row, List mutations, Map attributes) throws org.apache.thrift.TException + { + mutateRow_args args = new mutateRow_args(); + args.setTableName(tableName); + args.setRow(row); + args.setMutations(mutations); + args.setAttributes(attributes); + sendBase("mutateRow", args); + } + + public void recv_mutateRow() throws IOError, IllegalArgument, org.apache.thrift.TException + { + mutateRow_result result = new mutateRow_result(); + receiveBase(result, "mutateRow"); + if (result.io != null) { + throw result.io; + } + if (result.ia != null) { + throw result.ia; + } + return; + } + + public void mutateRowTs(ByteBuffer tableName, ByteBuffer row, List mutations, long timestamp, Map attributes) throws IOError, IllegalArgument, org.apache.thrift.TException + { + send_mutateRowTs(tableName, row, mutations, timestamp, attributes); + recv_mutateRowTs(); + } + + public void send_mutateRowTs(ByteBuffer tableName, ByteBuffer row, List mutations, long timestamp, Map attributes) throws org.apache.thrift.TException + { + mutateRowTs_args args = new mutateRowTs_args(); + args.setTableName(tableName); + args.setRow(row); + args.setMutations(mutations); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + sendBase("mutateRowTs", args); + } + + public void recv_mutateRowTs() throws IOError, IllegalArgument, org.apache.thrift.TException + { + mutateRowTs_result result = new mutateRowTs_result(); + receiveBase(result, "mutateRowTs"); + if (result.io != null) { + throw result.io; + } + if (result.ia != null) { + throw result.ia; + } + return; + } + + public void mutateRows(ByteBuffer tableName, List rowBatches, Map attributes) throws IOError, IllegalArgument, org.apache.thrift.TException + { + send_mutateRows(tableName, rowBatches, attributes); + recv_mutateRows(); + } + + public void send_mutateRows(ByteBuffer tableName, List rowBatches, Map attributes) throws org.apache.thrift.TException + { + mutateRows_args args = new mutateRows_args(); + args.setTableName(tableName); + args.setRowBatches(rowBatches); + args.setAttributes(attributes); + sendBase("mutateRows", args); + } + + public void recv_mutateRows() throws IOError, IllegalArgument, org.apache.thrift.TException + { + mutateRows_result result = new mutateRows_result(); + receiveBase(result, "mutateRows"); + if (result.io != null) { + throw result.io; + } + if (result.ia != null) { + throw result.ia; + } + return; + } + + public void mutateRowsTs(ByteBuffer tableName, List rowBatches, long timestamp, Map attributes) throws IOError, IllegalArgument, org.apache.thrift.TException + { + send_mutateRowsTs(tableName, rowBatches, timestamp, attributes); + recv_mutateRowsTs(); + } + + public void send_mutateRowsTs(ByteBuffer tableName, List rowBatches, long timestamp, Map attributes) throws org.apache.thrift.TException + { + mutateRowsTs_args args = new mutateRowsTs_args(); + args.setTableName(tableName); + args.setRowBatches(rowBatches); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + sendBase("mutateRowsTs", args); + } + + public void recv_mutateRowsTs() throws IOError, IllegalArgument, org.apache.thrift.TException + { + mutateRowsTs_result result = new mutateRowsTs_result(); + receiveBase(result, "mutateRowsTs"); + if (result.io != null) { + throw result.io; + } + if (result.ia != null) { + throw result.ia; + } + return; + } + + public long atomicIncrement(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long value) throws IOError, IllegalArgument, org.apache.thrift.TException + { + send_atomicIncrement(tableName, row, column, value); + return recv_atomicIncrement(); + } + + public void send_atomicIncrement(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long value) throws org.apache.thrift.TException + { + atomicIncrement_args args = new atomicIncrement_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumn(column); + args.setValue(value); + sendBase("atomicIncrement", args); + } + + public long recv_atomicIncrement() throws IOError, IllegalArgument, org.apache.thrift.TException + { + atomicIncrement_result result = new atomicIncrement_result(); + receiveBase(result, "atomicIncrement"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + if (result.ia != null) { + throw result.ia; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "atomicIncrement failed: unknown result"); + } + + public void deleteAll(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, Map attributes) throws IOError, org.apache.thrift.TException + { + send_deleteAll(tableName, row, column, attributes); + recv_deleteAll(); + } + + public void send_deleteAll(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, Map attributes) throws org.apache.thrift.TException + { + deleteAll_args args = new deleteAll_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumn(column); + args.setAttributes(attributes); + sendBase("deleteAll", args); + } + + public void recv_deleteAll() throws IOError, org.apache.thrift.TException + { + deleteAll_result result = new deleteAll_result(); + receiveBase(result, "deleteAll"); + if (result.io != null) { + throw result.io; + } + return; + } + + public void deleteAllTs(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException + { + send_deleteAllTs(tableName, row, column, timestamp, attributes); + recv_deleteAllTs(); + } + + public void send_deleteAllTs(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, Map attributes) throws org.apache.thrift.TException + { + deleteAllTs_args args = new deleteAllTs_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumn(column); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + sendBase("deleteAllTs", args); + } + + public void recv_deleteAllTs() throws IOError, org.apache.thrift.TException + { + deleteAllTs_result result = new deleteAllTs_result(); + receiveBase(result, "deleteAllTs"); + if (result.io != null) { + throw result.io; + } + return; + } + + public void deleteAllRow(ByteBuffer tableName, ByteBuffer row, Map attributes) throws IOError, org.apache.thrift.TException + { + send_deleteAllRow(tableName, row, attributes); + recv_deleteAllRow(); + } + + public void send_deleteAllRow(ByteBuffer tableName, ByteBuffer row, Map attributes) throws org.apache.thrift.TException + { + deleteAllRow_args args = new deleteAllRow_args(); + args.setTableName(tableName); + args.setRow(row); + args.setAttributes(attributes); + sendBase("deleteAllRow", args); + } + + public void recv_deleteAllRow() throws IOError, org.apache.thrift.TException + { + deleteAllRow_result result = new deleteAllRow_result(); + receiveBase(result, "deleteAllRow"); + if (result.io != null) { + throw result.io; + } + return; + } + + public void increment(TIncrement increment) throws IOError, org.apache.thrift.TException + { + send_increment(increment); + recv_increment(); + } + + public void send_increment(TIncrement increment) throws org.apache.thrift.TException + { + increment_args args = new increment_args(); + args.setIncrement(increment); + sendBase("increment", args); + } + + public void recv_increment() throws IOError, org.apache.thrift.TException + { + increment_result result = new increment_result(); + receiveBase(result, "increment"); + if (result.io != null) { + throw result.io; + } + return; + } + + public void incrementRows(List increments) throws IOError, org.apache.thrift.TException + { + send_incrementRows(increments); + recv_incrementRows(); + } + + public void send_incrementRows(List increments) throws org.apache.thrift.TException + { + incrementRows_args args = new incrementRows_args(); + args.setIncrements(increments); + sendBase("incrementRows", args); + } + + public void recv_incrementRows() throws IOError, org.apache.thrift.TException + { + incrementRows_result result = new incrementRows_result(); + receiveBase(result, "incrementRows"); + if (result.io != null) { + throw result.io; + } + return; + } + + public void deleteAllRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException + { + send_deleteAllRowTs(tableName, row, timestamp, attributes); + recv_deleteAllRowTs(); + } + + public void send_deleteAllRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map attributes) throws org.apache.thrift.TException + { + deleteAllRowTs_args args = new deleteAllRowTs_args(); + args.setTableName(tableName); + args.setRow(row); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + sendBase("deleteAllRowTs", args); + } + + public void recv_deleteAllRowTs() throws IOError, org.apache.thrift.TException + { + deleteAllRowTs_result result = new deleteAllRowTs_result(); + receiveBase(result, "deleteAllRowTs"); + if (result.io != null) { + throw result.io; + } + return; + } + + public int scannerOpenWithScan(ByteBuffer tableName, TScan scan, Map attributes) throws IOError, org.apache.thrift.TException + { + send_scannerOpenWithScan(tableName, scan, attributes); + return recv_scannerOpenWithScan(); + } + + public void send_scannerOpenWithScan(ByteBuffer tableName, TScan scan, Map attributes) throws org.apache.thrift.TException + { + scannerOpenWithScan_args args = new scannerOpenWithScan_args(); + args.setTableName(tableName); + args.setScan(scan); + args.setAttributes(attributes); + sendBase("scannerOpenWithScan", args); + } + + public int recv_scannerOpenWithScan() throws IOError, org.apache.thrift.TException + { + scannerOpenWithScan_result result = new scannerOpenWithScan_result(); + receiveBase(result, "scannerOpenWithScan"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "scannerOpenWithScan failed: unknown result"); + } + + public int scannerOpen(ByteBuffer tableName, ByteBuffer startRow, List columns, Map attributes) throws IOError, org.apache.thrift.TException + { + send_scannerOpen(tableName, startRow, columns, attributes); + return recv_scannerOpen(); + } + + public void send_scannerOpen(ByteBuffer tableName, ByteBuffer startRow, List columns, Map attributes) throws org.apache.thrift.TException + { + scannerOpen_args args = new scannerOpen_args(); + args.setTableName(tableName); + args.setStartRow(startRow); + args.setColumns(columns); + args.setAttributes(attributes); + sendBase("scannerOpen", args); + } + + public int recv_scannerOpen() throws IOError, org.apache.thrift.TException + { + scannerOpen_result result = new scannerOpen_result(); + receiveBase(result, "scannerOpen"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "scannerOpen failed: unknown result"); + } + + public int scannerOpenWithStop(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List columns, Map attributes) throws IOError, org.apache.thrift.TException + { + send_scannerOpenWithStop(tableName, startRow, stopRow, columns, attributes); + return recv_scannerOpenWithStop(); + } + + public void send_scannerOpenWithStop(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List columns, Map attributes) throws org.apache.thrift.TException + { + scannerOpenWithStop_args args = new scannerOpenWithStop_args(); + args.setTableName(tableName); + args.setStartRow(startRow); + args.setStopRow(stopRow); + args.setColumns(columns); + args.setAttributes(attributes); + sendBase("scannerOpenWithStop", args); + } + + public int recv_scannerOpenWithStop() throws IOError, org.apache.thrift.TException + { + scannerOpenWithStop_result result = new scannerOpenWithStop_result(); + receiveBase(result, "scannerOpenWithStop"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "scannerOpenWithStop failed: unknown result"); + } + + public int scannerOpenWithPrefix(ByteBuffer tableName, ByteBuffer startAndPrefix, List columns, Map attributes) throws IOError, org.apache.thrift.TException + { + send_scannerOpenWithPrefix(tableName, startAndPrefix, columns, attributes); + return recv_scannerOpenWithPrefix(); + } + + public void send_scannerOpenWithPrefix(ByteBuffer tableName, ByteBuffer startAndPrefix, List columns, Map attributes) throws org.apache.thrift.TException + { + scannerOpenWithPrefix_args args = new scannerOpenWithPrefix_args(); + args.setTableName(tableName); + args.setStartAndPrefix(startAndPrefix); + args.setColumns(columns); + args.setAttributes(attributes); + sendBase("scannerOpenWithPrefix", args); + } + + public int recv_scannerOpenWithPrefix() throws IOError, org.apache.thrift.TException + { + scannerOpenWithPrefix_result result = new scannerOpenWithPrefix_result(); + receiveBase(result, "scannerOpenWithPrefix"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "scannerOpenWithPrefix failed: unknown result"); + } + + public int scannerOpenTs(ByteBuffer tableName, ByteBuffer startRow, List columns, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException + { + send_scannerOpenTs(tableName, startRow, columns, timestamp, attributes); + return recv_scannerOpenTs(); + } + + public void send_scannerOpenTs(ByteBuffer tableName, ByteBuffer startRow, List columns, long timestamp, Map attributes) throws org.apache.thrift.TException + { + scannerOpenTs_args args = new scannerOpenTs_args(); + args.setTableName(tableName); + args.setStartRow(startRow); + args.setColumns(columns); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + sendBase("scannerOpenTs", args); + } + + public int recv_scannerOpenTs() throws IOError, org.apache.thrift.TException + { + scannerOpenTs_result result = new scannerOpenTs_result(); + receiveBase(result, "scannerOpenTs"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "scannerOpenTs failed: unknown result"); + } + + public int scannerOpenWithStopTs(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List columns, long timestamp, Map attributes) throws IOError, org.apache.thrift.TException + { + send_scannerOpenWithStopTs(tableName, startRow, stopRow, columns, timestamp, attributes); + return recv_scannerOpenWithStopTs(); + } + + public void send_scannerOpenWithStopTs(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List columns, long timestamp, Map attributes) throws org.apache.thrift.TException + { + scannerOpenWithStopTs_args args = new scannerOpenWithStopTs_args(); + args.setTableName(tableName); + args.setStartRow(startRow); + args.setStopRow(stopRow); + args.setColumns(columns); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + sendBase("scannerOpenWithStopTs", args); + } + + public int recv_scannerOpenWithStopTs() throws IOError, org.apache.thrift.TException + { + scannerOpenWithStopTs_result result = new scannerOpenWithStopTs_result(); + receiveBase(result, "scannerOpenWithStopTs"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "scannerOpenWithStopTs failed: unknown result"); + } + + public List scannerGet(int id) throws IOError, IllegalArgument, org.apache.thrift.TException + { + send_scannerGet(id); + return recv_scannerGet(); + } + + public void send_scannerGet(int id) throws org.apache.thrift.TException + { + scannerGet_args args = new scannerGet_args(); + args.setId(id); + sendBase("scannerGet", args); + } + + public List recv_scannerGet() throws IOError, IllegalArgument, org.apache.thrift.TException + { + scannerGet_result result = new scannerGet_result(); + receiveBase(result, "scannerGet"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + if (result.ia != null) { + throw result.ia; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "scannerGet failed: unknown result"); + } + + public List scannerGetList(int id, int nbRows) throws IOError, IllegalArgument, org.apache.thrift.TException + { + send_scannerGetList(id, nbRows); + return recv_scannerGetList(); + } + + public void send_scannerGetList(int id, int nbRows) throws org.apache.thrift.TException + { + scannerGetList_args args = new scannerGetList_args(); + args.setId(id); + args.setNbRows(nbRows); + sendBase("scannerGetList", args); + } + + public List recv_scannerGetList() throws IOError, IllegalArgument, org.apache.thrift.TException + { + scannerGetList_result result = new scannerGetList_result(); + receiveBase(result, "scannerGetList"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + if (result.ia != null) { + throw result.ia; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "scannerGetList failed: unknown result"); + } + + public void scannerClose(int id) throws IOError, IllegalArgument, org.apache.thrift.TException + { + send_scannerClose(id); + recv_scannerClose(); + } + + public void send_scannerClose(int id) throws org.apache.thrift.TException + { + scannerClose_args args = new scannerClose_args(); + args.setId(id); + sendBase("scannerClose", args); + } + + public void recv_scannerClose() throws IOError, IllegalArgument, org.apache.thrift.TException + { + scannerClose_result result = new scannerClose_result(); + receiveBase(result, "scannerClose"); + if (result.io != null) { + throw result.io; + } + if (result.ia != null) { + throw result.ia; + } + return; + } + + public List getRowOrBefore(ByteBuffer tableName, ByteBuffer row, ByteBuffer family) throws IOError, org.apache.thrift.TException + { + send_getRowOrBefore(tableName, row, family); + return recv_getRowOrBefore(); + } + + public void send_getRowOrBefore(ByteBuffer tableName, ByteBuffer row, ByteBuffer family) throws org.apache.thrift.TException + { + getRowOrBefore_args args = new getRowOrBefore_args(); + args.setTableName(tableName); + args.setRow(row); + args.setFamily(family); + sendBase("getRowOrBefore", args); + } + + public List recv_getRowOrBefore() throws IOError, org.apache.thrift.TException + { + getRowOrBefore_result result = new getRowOrBefore_result(); + receiveBase(result, "getRowOrBefore"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRowOrBefore failed: unknown result"); + } + + public TRegionInfo getRegionInfo(ByteBuffer row) throws IOError, org.apache.thrift.TException + { + send_getRegionInfo(row); + return recv_getRegionInfo(); + } + + public void send_getRegionInfo(ByteBuffer row) throws org.apache.thrift.TException + { + getRegionInfo_args args = new getRegionInfo_args(); + args.setRow(row); + sendBase("getRegionInfo", args); + } + + public TRegionInfo recv_getRegionInfo() throws IOError, org.apache.thrift.TException + { + getRegionInfo_result result = new getRegionInfo_result(); + receiveBase(result, "getRegionInfo"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRegionInfo failed: unknown result"); + } + + } + public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface { + public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { + private org.apache.thrift.async.TAsyncClientManager clientManager; + private org.apache.thrift.protocol.TProtocolFactory protocolFactory; + public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) { + this.clientManager = clientManager; + this.protocolFactory = protocolFactory; + } + public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) { + return new AsyncClient(protocolFactory, clientManager, transport); + } + } + + public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) { + super(protocolFactory, clientManager, transport); + } + + public void enableTable(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + enableTable_call method_call = new enableTable_call(tableName, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class enableTable_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + public enableTable_call(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("enableTable", org.apache.thrift.protocol.TMessageType.CALL, 0)); + enableTable_args args = new enableTable_args(); + args.setTableName(tableName); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_enableTable(); + } + } + + public void disableTable(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + disableTable_call method_call = new disableTable_call(tableName, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class disableTable_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + public disableTable_call(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("disableTable", org.apache.thrift.protocol.TMessageType.CALL, 0)); + disableTable_args args = new disableTable_args(); + args.setTableName(tableName); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_disableTable(); + } + } + + public void isTableEnabled(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + isTableEnabled_call method_call = new isTableEnabled_call(tableName, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class isTableEnabled_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + public isTableEnabled_call(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("isTableEnabled", org.apache.thrift.protocol.TMessageType.CALL, 0)); + isTableEnabled_args args = new isTableEnabled_args(); + args.setTableName(tableName); + args.write(prot); + prot.writeMessageEnd(); + } + + public boolean getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_isTableEnabled(); + } + } + + public void compact(ByteBuffer tableNameOrRegionName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + compact_call method_call = new compact_call(tableNameOrRegionName, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class compact_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableNameOrRegionName; + public compact_call(ByteBuffer tableNameOrRegionName, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableNameOrRegionName = tableNameOrRegionName; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("compact", org.apache.thrift.protocol.TMessageType.CALL, 0)); + compact_args args = new compact_args(); + args.setTableNameOrRegionName(tableNameOrRegionName); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_compact(); + } + } + + public void majorCompact(ByteBuffer tableNameOrRegionName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + majorCompact_call method_call = new majorCompact_call(tableNameOrRegionName, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class majorCompact_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableNameOrRegionName; + public majorCompact_call(ByteBuffer tableNameOrRegionName, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableNameOrRegionName = tableNameOrRegionName; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("majorCompact", org.apache.thrift.protocol.TMessageType.CALL, 0)); + majorCompact_args args = new majorCompact_args(); + args.setTableNameOrRegionName(tableNameOrRegionName); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_majorCompact(); + } + } + + public void getTableNames(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getTableNames_call method_call = new getTableNames_call(resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getTableNames_call extends org.apache.thrift.async.TAsyncMethodCall { + public getTableNames_call(org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getTableNames", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getTableNames_args args = new getTableNames_args(); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getTableNames(); + } + } + + public void getColumnDescriptors(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getColumnDescriptors_call method_call = new getColumnDescriptors_call(tableName, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getColumnDescriptors_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + public getColumnDescriptors_call(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getColumnDescriptors", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getColumnDescriptors_args args = new getColumnDescriptors_args(); + args.setTableName(tableName); + args.write(prot); + prot.writeMessageEnd(); + } + + public Map getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getColumnDescriptors(); + } + } + + public void getTableRegions(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getTableRegions_call method_call = new getTableRegions_call(tableName, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getTableRegions_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + public getTableRegions_call(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getTableRegions", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getTableRegions_args args = new getTableRegions_args(); + args.setTableName(tableName); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getTableRegions(); + } + } + + public void createTable(ByteBuffer tableName, List columnFamilies, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + createTable_call method_call = new createTable_call(tableName, columnFamilies, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class createTable_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private List columnFamilies; + public createTable_call(ByteBuffer tableName, List columnFamilies, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.columnFamilies = columnFamilies; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("createTable", org.apache.thrift.protocol.TMessageType.CALL, 0)); + createTable_args args = new createTable_args(); + args.setTableName(tableName); + args.setColumnFamilies(columnFamilies); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, IllegalArgument, AlreadyExists, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_createTable(); + } + } + + public void deleteTable(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + deleteTable_call method_call = new deleteTable_call(tableName, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class deleteTable_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + public deleteTable_call(ByteBuffer tableName, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("deleteTable", org.apache.thrift.protocol.TMessageType.CALL, 0)); + deleteTable_args args = new deleteTable_args(); + args.setTableName(tableName); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_deleteTable(); + } + } + + public void get(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + get_call method_call = new get_call(tableName, row, column, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class get_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private ByteBuffer column; + private Map attributes; + public get_call(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.column = column; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get", org.apache.thrift.protocol.TMessageType.CALL, 0)); + get_args args = new get_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumn(column); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_get(); + } + } + + public void getVer(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, int numVersions, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getVer_call method_call = new getVer_call(tableName, row, column, numVersions, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getVer_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private ByteBuffer column; + private int numVersions; + private Map attributes; + public getVer_call(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, int numVersions, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.column = column; + this.numVersions = numVersions; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getVer", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getVer_args args = new getVer_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumn(column); + args.setNumVersions(numVersions); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getVer(); + } + } + + public void getVerTs(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, int numVersions, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getVerTs_call method_call = new getVerTs_call(tableName, row, column, timestamp, numVersions, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getVerTs_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private ByteBuffer column; + private long timestamp; + private int numVersions; + private Map attributes; + public getVerTs_call(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, int numVersions, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.column = column; + this.timestamp = timestamp; + this.numVersions = numVersions; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getVerTs", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getVerTs_args args = new getVerTs_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumn(column); + args.setTimestamp(timestamp); + args.setNumVersions(numVersions); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getVerTs(); + } + } + + public void getRow(ByteBuffer tableName, ByteBuffer row, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getRow_call method_call = new getRow_call(tableName, row, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getRow_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private Map attributes; + public getRow_call(ByteBuffer tableName, ByteBuffer row, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getRow", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getRow_args args = new getRow_args(); + args.setTableName(tableName); + args.setRow(row); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getRow(); + } + } + + public void getRowWithColumns(ByteBuffer tableName, ByteBuffer row, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getRowWithColumns_call method_call = new getRowWithColumns_call(tableName, row, columns, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getRowWithColumns_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private List columns; + private Map attributes; + public getRowWithColumns_call(ByteBuffer tableName, ByteBuffer row, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.columns = columns; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getRowWithColumns", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getRowWithColumns_args args = new getRowWithColumns_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumns(columns); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getRowWithColumns(); + } + } + + public void getRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getRowTs_call method_call = new getRowTs_call(tableName, row, timestamp, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getRowTs_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private long timestamp; + private Map attributes; + public getRowTs_call(ByteBuffer tableName, ByteBuffer row, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.timestamp = timestamp; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getRowTs", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getRowTs_args args = new getRowTs_args(); + args.setTableName(tableName); + args.setRow(row); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getRowTs(); + } + } + + public void getRowWithColumnsTs(ByteBuffer tableName, ByteBuffer row, List columns, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getRowWithColumnsTs_call method_call = new getRowWithColumnsTs_call(tableName, row, columns, timestamp, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getRowWithColumnsTs_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private List columns; + private long timestamp; + private Map attributes; + public getRowWithColumnsTs_call(ByteBuffer tableName, ByteBuffer row, List columns, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.columns = columns; + this.timestamp = timestamp; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getRowWithColumnsTs", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getRowWithColumnsTs_args args = new getRowWithColumnsTs_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumns(columns); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getRowWithColumnsTs(); + } + } + + public void getRows(ByteBuffer tableName, List rows, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getRows_call method_call = new getRows_call(tableName, rows, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getRows_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private List rows; + private Map attributes; + public getRows_call(ByteBuffer tableName, List rows, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.rows = rows; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getRows", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getRows_args args = new getRows_args(); + args.setTableName(tableName); + args.setRows(rows); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getRows(); + } + } + + public void getRowsWithColumns(ByteBuffer tableName, List rows, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getRowsWithColumns_call method_call = new getRowsWithColumns_call(tableName, rows, columns, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getRowsWithColumns_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private List rows; + private List columns; + private Map attributes; + public getRowsWithColumns_call(ByteBuffer tableName, List rows, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.rows = rows; + this.columns = columns; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getRowsWithColumns", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getRowsWithColumns_args args = new getRowsWithColumns_args(); + args.setTableName(tableName); + args.setRows(rows); + args.setColumns(columns); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getRowsWithColumns(); + } + } + + public void getRowsTs(ByteBuffer tableName, List rows, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getRowsTs_call method_call = new getRowsTs_call(tableName, rows, timestamp, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getRowsTs_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private List rows; + private long timestamp; + private Map attributes; + public getRowsTs_call(ByteBuffer tableName, List rows, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.rows = rows; + this.timestamp = timestamp; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getRowsTs", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getRowsTs_args args = new getRowsTs_args(); + args.setTableName(tableName); + args.setRows(rows); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getRowsTs(); + } + } + + public void getRowsWithColumnsTs(ByteBuffer tableName, List rows, List columns, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getRowsWithColumnsTs_call method_call = new getRowsWithColumnsTs_call(tableName, rows, columns, timestamp, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getRowsWithColumnsTs_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private List rows; + private List columns; + private long timestamp; + private Map attributes; + public getRowsWithColumnsTs_call(ByteBuffer tableName, List rows, List columns, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.rows = rows; + this.columns = columns; + this.timestamp = timestamp; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getRowsWithColumnsTs", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getRowsWithColumnsTs_args args = new getRowsWithColumnsTs_args(); + args.setTableName(tableName); + args.setRows(rows); + args.setColumns(columns); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getRowsWithColumnsTs(); + } + } + + public void mutateRow(ByteBuffer tableName, ByteBuffer row, List mutations, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + mutateRow_call method_call = new mutateRow_call(tableName, row, mutations, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class mutateRow_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private List mutations; + private Map attributes; + public mutateRow_call(ByteBuffer tableName, ByteBuffer row, List mutations, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.mutations = mutations; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("mutateRow", org.apache.thrift.protocol.TMessageType.CALL, 0)); + mutateRow_args args = new mutateRow_args(); + args.setTableName(tableName); + args.setRow(row); + args.setMutations(mutations); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, IllegalArgument, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_mutateRow(); + } + } + + public void mutateRowTs(ByteBuffer tableName, ByteBuffer row, List mutations, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + mutateRowTs_call method_call = new mutateRowTs_call(tableName, row, mutations, timestamp, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class mutateRowTs_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private List mutations; + private long timestamp; + private Map attributes; + public mutateRowTs_call(ByteBuffer tableName, ByteBuffer row, List mutations, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.mutations = mutations; + this.timestamp = timestamp; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("mutateRowTs", org.apache.thrift.protocol.TMessageType.CALL, 0)); + mutateRowTs_args args = new mutateRowTs_args(); + args.setTableName(tableName); + args.setRow(row); + args.setMutations(mutations); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, IllegalArgument, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_mutateRowTs(); + } + } + + public void mutateRows(ByteBuffer tableName, List rowBatches, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + mutateRows_call method_call = new mutateRows_call(tableName, rowBatches, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class mutateRows_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private List rowBatches; + private Map attributes; + public mutateRows_call(ByteBuffer tableName, List rowBatches, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.rowBatches = rowBatches; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("mutateRows", org.apache.thrift.protocol.TMessageType.CALL, 0)); + mutateRows_args args = new mutateRows_args(); + args.setTableName(tableName); + args.setRowBatches(rowBatches); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, IllegalArgument, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_mutateRows(); + } + } + + public void mutateRowsTs(ByteBuffer tableName, List rowBatches, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + mutateRowsTs_call method_call = new mutateRowsTs_call(tableName, rowBatches, timestamp, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class mutateRowsTs_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private List rowBatches; + private long timestamp; + private Map attributes; + public mutateRowsTs_call(ByteBuffer tableName, List rowBatches, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.rowBatches = rowBatches; + this.timestamp = timestamp; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("mutateRowsTs", org.apache.thrift.protocol.TMessageType.CALL, 0)); + mutateRowsTs_args args = new mutateRowsTs_args(); + args.setTableName(tableName); + args.setRowBatches(rowBatches); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, IllegalArgument, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_mutateRowsTs(); + } + } + + public void atomicIncrement(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long value, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + atomicIncrement_call method_call = new atomicIncrement_call(tableName, row, column, value, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class atomicIncrement_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private ByteBuffer column; + private long value; + public atomicIncrement_call(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long value, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.column = column; + this.value = value; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("atomicIncrement", org.apache.thrift.protocol.TMessageType.CALL, 0)); + atomicIncrement_args args = new atomicIncrement_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumn(column); + args.setValue(value); + args.write(prot); + prot.writeMessageEnd(); + } + + public long getResult() throws IOError, IllegalArgument, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_atomicIncrement(); + } + } + + public void deleteAll(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + deleteAll_call method_call = new deleteAll_call(tableName, row, column, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class deleteAll_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private ByteBuffer column; + private Map attributes; + public deleteAll_call(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.column = column; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("deleteAll", org.apache.thrift.protocol.TMessageType.CALL, 0)); + deleteAll_args args = new deleteAll_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumn(column); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_deleteAll(); + } + } + + public void deleteAllTs(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + deleteAllTs_call method_call = new deleteAllTs_call(tableName, row, column, timestamp, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class deleteAllTs_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private ByteBuffer column; + private long timestamp; + private Map attributes; + public deleteAllTs_call(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.column = column; + this.timestamp = timestamp; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("deleteAllTs", org.apache.thrift.protocol.TMessageType.CALL, 0)); + deleteAllTs_args args = new deleteAllTs_args(); + args.setTableName(tableName); + args.setRow(row); + args.setColumn(column); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_deleteAllTs(); + } + } + + public void deleteAllRow(ByteBuffer tableName, ByteBuffer row, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + deleteAllRow_call method_call = new deleteAllRow_call(tableName, row, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class deleteAllRow_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private Map attributes; + public deleteAllRow_call(ByteBuffer tableName, ByteBuffer row, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("deleteAllRow", org.apache.thrift.protocol.TMessageType.CALL, 0)); + deleteAllRow_args args = new deleteAllRow_args(); + args.setTableName(tableName); + args.setRow(row); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_deleteAllRow(); + } + } + + public void increment(TIncrement increment, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + increment_call method_call = new increment_call(increment, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class increment_call extends org.apache.thrift.async.TAsyncMethodCall { + private TIncrement increment; + public increment_call(TIncrement increment, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.increment = increment; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("increment", org.apache.thrift.protocol.TMessageType.CALL, 0)); + increment_args args = new increment_args(); + args.setIncrement(increment); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_increment(); + } + } + + public void incrementRows(List increments, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + incrementRows_call method_call = new incrementRows_call(increments, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class incrementRows_call extends org.apache.thrift.async.TAsyncMethodCall { + private List increments; + public incrementRows_call(List increments, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.increments = increments; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("incrementRows", org.apache.thrift.protocol.TMessageType.CALL, 0)); + incrementRows_args args = new incrementRows_args(); + args.setIncrements(increments); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_incrementRows(); + } + } + + public void deleteAllRowTs(ByteBuffer tableName, ByteBuffer row, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + deleteAllRowTs_call method_call = new deleteAllRowTs_call(tableName, row, timestamp, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class deleteAllRowTs_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private long timestamp; + private Map attributes; + public deleteAllRowTs_call(ByteBuffer tableName, ByteBuffer row, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.timestamp = timestamp; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("deleteAllRowTs", org.apache.thrift.protocol.TMessageType.CALL, 0)); + deleteAllRowTs_args args = new deleteAllRowTs_args(); + args.setTableName(tableName); + args.setRow(row); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_deleteAllRowTs(); + } + } + + public void scannerOpenWithScan(ByteBuffer tableName, TScan scan, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + scannerOpenWithScan_call method_call = new scannerOpenWithScan_call(tableName, scan, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class scannerOpenWithScan_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private TScan scan; + private Map attributes; + public scannerOpenWithScan_call(ByteBuffer tableName, TScan scan, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.scan = scan; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("scannerOpenWithScan", org.apache.thrift.protocol.TMessageType.CALL, 0)); + scannerOpenWithScan_args args = new scannerOpenWithScan_args(); + args.setTableName(tableName); + args.setScan(scan); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public int getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_scannerOpenWithScan(); + } + } + + public void scannerOpen(ByteBuffer tableName, ByteBuffer startRow, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + scannerOpen_call method_call = new scannerOpen_call(tableName, startRow, columns, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class scannerOpen_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer startRow; + private List columns; + private Map attributes; + public scannerOpen_call(ByteBuffer tableName, ByteBuffer startRow, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.startRow = startRow; + this.columns = columns; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("scannerOpen", org.apache.thrift.protocol.TMessageType.CALL, 0)); + scannerOpen_args args = new scannerOpen_args(); + args.setTableName(tableName); + args.setStartRow(startRow); + args.setColumns(columns); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public int getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_scannerOpen(); + } + } + + public void scannerOpenWithStop(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + scannerOpenWithStop_call method_call = new scannerOpenWithStop_call(tableName, startRow, stopRow, columns, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class scannerOpenWithStop_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer startRow; + private ByteBuffer stopRow; + private List columns; + private Map attributes; + public scannerOpenWithStop_call(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.startRow = startRow; + this.stopRow = stopRow; + this.columns = columns; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("scannerOpenWithStop", org.apache.thrift.protocol.TMessageType.CALL, 0)); + scannerOpenWithStop_args args = new scannerOpenWithStop_args(); + args.setTableName(tableName); + args.setStartRow(startRow); + args.setStopRow(stopRow); + args.setColumns(columns); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public int getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_scannerOpenWithStop(); + } + } + + public void scannerOpenWithPrefix(ByteBuffer tableName, ByteBuffer startAndPrefix, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + scannerOpenWithPrefix_call method_call = new scannerOpenWithPrefix_call(tableName, startAndPrefix, columns, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class scannerOpenWithPrefix_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer startAndPrefix; + private List columns; + private Map attributes; + public scannerOpenWithPrefix_call(ByteBuffer tableName, ByteBuffer startAndPrefix, List columns, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.startAndPrefix = startAndPrefix; + this.columns = columns; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("scannerOpenWithPrefix", org.apache.thrift.protocol.TMessageType.CALL, 0)); + scannerOpenWithPrefix_args args = new scannerOpenWithPrefix_args(); + args.setTableName(tableName); + args.setStartAndPrefix(startAndPrefix); + args.setColumns(columns); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public int getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_scannerOpenWithPrefix(); + } + } + + public void scannerOpenTs(ByteBuffer tableName, ByteBuffer startRow, List columns, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + scannerOpenTs_call method_call = new scannerOpenTs_call(tableName, startRow, columns, timestamp, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class scannerOpenTs_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer startRow; + private List columns; + private long timestamp; + private Map attributes; + public scannerOpenTs_call(ByteBuffer tableName, ByteBuffer startRow, List columns, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.startRow = startRow; + this.columns = columns; + this.timestamp = timestamp; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("scannerOpenTs", org.apache.thrift.protocol.TMessageType.CALL, 0)); + scannerOpenTs_args args = new scannerOpenTs_args(); + args.setTableName(tableName); + args.setStartRow(startRow); + args.setColumns(columns); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public int getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_scannerOpenTs(); + } + } + + public void scannerOpenWithStopTs(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List columns, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + scannerOpenWithStopTs_call method_call = new scannerOpenWithStopTs_call(tableName, startRow, stopRow, columns, timestamp, attributes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class scannerOpenWithStopTs_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer startRow; + private ByteBuffer stopRow; + private List columns; + private long timestamp; + private Map attributes; + public scannerOpenWithStopTs_call(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List columns, long timestamp, Map attributes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.startRow = startRow; + this.stopRow = stopRow; + this.columns = columns; + this.timestamp = timestamp; + this.attributes = attributes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("scannerOpenWithStopTs", org.apache.thrift.protocol.TMessageType.CALL, 0)); + scannerOpenWithStopTs_args args = new scannerOpenWithStopTs_args(); + args.setTableName(tableName); + args.setStartRow(startRow); + args.setStopRow(stopRow); + args.setColumns(columns); + args.setTimestamp(timestamp); + args.setAttributes(attributes); + args.write(prot); + prot.writeMessageEnd(); + } + + public int getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_scannerOpenWithStopTs(); + } + } + + public void scannerGet(int id, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + scannerGet_call method_call = new scannerGet_call(id, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class scannerGet_call extends org.apache.thrift.async.TAsyncMethodCall { + private int id; + public scannerGet_call(int id, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.id = id; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("scannerGet", org.apache.thrift.protocol.TMessageType.CALL, 0)); + scannerGet_args args = new scannerGet_args(); + args.setId(id); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, IllegalArgument, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_scannerGet(); + } + } + + public void scannerGetList(int id, int nbRows, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + scannerGetList_call method_call = new scannerGetList_call(id, nbRows, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class scannerGetList_call extends org.apache.thrift.async.TAsyncMethodCall { + private int id; + private int nbRows; + public scannerGetList_call(int id, int nbRows, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.id = id; + this.nbRows = nbRows; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("scannerGetList", org.apache.thrift.protocol.TMessageType.CALL, 0)); + scannerGetList_args args = new scannerGetList_args(); + args.setId(id); + args.setNbRows(nbRows); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, IllegalArgument, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_scannerGetList(); + } + } + + public void scannerClose(int id, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + scannerClose_call method_call = new scannerClose_call(id, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class scannerClose_call extends org.apache.thrift.async.TAsyncMethodCall { + private int id; + public scannerClose_call(int id, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.id = id; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("scannerClose", org.apache.thrift.protocol.TMessageType.CALL, 0)); + scannerClose_args args = new scannerClose_args(); + args.setId(id); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws IOError, IllegalArgument, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_scannerClose(); + } + } + + public void getRowOrBefore(ByteBuffer tableName, ByteBuffer row, ByteBuffer family, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getRowOrBefore_call method_call = new getRowOrBefore_call(tableName, row, family, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getRowOrBefore_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer tableName; + private ByteBuffer row; + private ByteBuffer family; + public getRowOrBefore_call(ByteBuffer tableName, ByteBuffer row, ByteBuffer family, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tableName = tableName; + this.row = row; + this.family = family; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getRowOrBefore", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getRowOrBefore_args args = new getRowOrBefore_args(); + args.setTableName(tableName); + args.setRow(row); + args.setFamily(family); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getRowOrBefore(); + } + } + + public void getRegionInfo(ByteBuffer row, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getRegionInfo_call method_call = new getRegionInfo_call(row, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getRegionInfo_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer row; + public getRegionInfo_call(ByteBuffer row, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.row = row; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getRegionInfo", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getRegionInfo_args args = new getRegionInfo_args(); + args.setRow(row); + args.write(prot); + prot.writeMessageEnd(); + } + + public TRegionInfo getResult() throws IOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getRegionInfo(); + } + } + + } + + public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName()); + public Processor(I iface) { + super(iface, getProcessMap(new HashMap>())); + } + + protected Processor(I iface, Map> processMap) { + super(iface, getProcessMap(processMap)); + } + + private static Map> getProcessMap(Map> processMap) { + processMap.put("enableTable", new enableTable()); + processMap.put("disableTable", new disableTable()); + processMap.put("isTableEnabled", new isTableEnabled()); + processMap.put("compact", new compact()); + processMap.put("majorCompact", new majorCompact()); + processMap.put("getTableNames", new getTableNames()); + processMap.put("getColumnDescriptors", new getColumnDescriptors()); + processMap.put("getTableRegions", new getTableRegions()); + processMap.put("createTable", new createTable()); + processMap.put("deleteTable", new deleteTable()); + processMap.put("get", new get()); + processMap.put("getVer", new getVer()); + processMap.put("getVerTs", new getVerTs()); + processMap.put("getRow", new getRow()); + processMap.put("getRowWithColumns", new getRowWithColumns()); + processMap.put("getRowTs", new getRowTs()); + processMap.put("getRowWithColumnsTs", new getRowWithColumnsTs()); + processMap.put("getRows", new getRows()); + processMap.put("getRowsWithColumns", new getRowsWithColumns()); + processMap.put("getRowsTs", new getRowsTs()); + processMap.put("getRowsWithColumnsTs", new getRowsWithColumnsTs()); + processMap.put("mutateRow", new mutateRow()); + processMap.put("mutateRowTs", new mutateRowTs()); + processMap.put("mutateRows", new mutateRows()); + processMap.put("mutateRowsTs", new mutateRowsTs()); + processMap.put("atomicIncrement", new atomicIncrement()); + processMap.put("deleteAll", new deleteAll()); + processMap.put("deleteAllTs", new deleteAllTs()); + processMap.put("deleteAllRow", new deleteAllRow()); + processMap.put("increment", new increment()); + processMap.put("incrementRows", new incrementRows()); + processMap.put("deleteAllRowTs", new deleteAllRowTs()); + processMap.put("scannerOpenWithScan", new scannerOpenWithScan()); + processMap.put("scannerOpen", new scannerOpen()); + processMap.put("scannerOpenWithStop", new scannerOpenWithStop()); + processMap.put("scannerOpenWithPrefix", new scannerOpenWithPrefix()); + processMap.put("scannerOpenTs", new scannerOpenTs()); + processMap.put("scannerOpenWithStopTs", new scannerOpenWithStopTs()); + processMap.put("scannerGet", new scannerGet()); + processMap.put("scannerGetList", new scannerGetList()); + processMap.put("scannerClose", new scannerClose()); + processMap.put("getRowOrBefore", new getRowOrBefore()); + processMap.put("getRegionInfo", new getRegionInfo()); + return processMap; + } + + private static class enableTable extends org.apache.thrift.ProcessFunction { + public enableTable() { + super("enableTable"); + } + + protected enableTable_args getEmptyArgsInstance() { + return new enableTable_args(); + } + + protected enableTable_result getResult(I iface, enableTable_args args) throws org.apache.thrift.TException { + enableTable_result result = new enableTable_result(); + try { + iface.enableTable(args.tableName); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class disableTable extends org.apache.thrift.ProcessFunction { + public disableTable() { + super("disableTable"); + } + + protected disableTable_args getEmptyArgsInstance() { + return new disableTable_args(); + } + + protected disableTable_result getResult(I iface, disableTable_args args) throws org.apache.thrift.TException { + disableTable_result result = new disableTable_result(); + try { + iface.disableTable(args.tableName); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class isTableEnabled extends org.apache.thrift.ProcessFunction { + public isTableEnabled() { + super("isTableEnabled"); + } + + protected isTableEnabled_args getEmptyArgsInstance() { + return new isTableEnabled_args(); + } + + protected isTableEnabled_result getResult(I iface, isTableEnabled_args args) throws org.apache.thrift.TException { + isTableEnabled_result result = new isTableEnabled_result(); + try { + result.success = iface.isTableEnabled(args.tableName); + result.setSuccessIsSet(true); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class compact extends org.apache.thrift.ProcessFunction { + public compact() { + super("compact"); + } + + protected compact_args getEmptyArgsInstance() { + return new compact_args(); + } + + protected compact_result getResult(I iface, compact_args args) throws org.apache.thrift.TException { + compact_result result = new compact_result(); + try { + iface.compact(args.tableNameOrRegionName); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class majorCompact extends org.apache.thrift.ProcessFunction { + public majorCompact() { + super("majorCompact"); + } + + protected majorCompact_args getEmptyArgsInstance() { + return new majorCompact_args(); + } + + protected majorCompact_result getResult(I iface, majorCompact_args args) throws org.apache.thrift.TException { + majorCompact_result result = new majorCompact_result(); + try { + iface.majorCompact(args.tableNameOrRegionName); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getTableNames extends org.apache.thrift.ProcessFunction { + public getTableNames() { + super("getTableNames"); + } + + protected getTableNames_args getEmptyArgsInstance() { + return new getTableNames_args(); + } + + protected getTableNames_result getResult(I iface, getTableNames_args args) throws org.apache.thrift.TException { + getTableNames_result result = new getTableNames_result(); + try { + result.success = iface.getTableNames(); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getColumnDescriptors extends org.apache.thrift.ProcessFunction { + public getColumnDescriptors() { + super("getColumnDescriptors"); + } + + protected getColumnDescriptors_args getEmptyArgsInstance() { + return new getColumnDescriptors_args(); + } + + protected getColumnDescriptors_result getResult(I iface, getColumnDescriptors_args args) throws org.apache.thrift.TException { + getColumnDescriptors_result result = new getColumnDescriptors_result(); + try { + result.success = iface.getColumnDescriptors(args.tableName); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getTableRegions extends org.apache.thrift.ProcessFunction { + public getTableRegions() { + super("getTableRegions"); + } + + protected getTableRegions_args getEmptyArgsInstance() { + return new getTableRegions_args(); + } + + protected getTableRegions_result getResult(I iface, getTableRegions_args args) throws org.apache.thrift.TException { + getTableRegions_result result = new getTableRegions_result(); + try { + result.success = iface.getTableRegions(args.tableName); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class createTable extends org.apache.thrift.ProcessFunction { + public createTable() { + super("createTable"); + } + + protected createTable_args getEmptyArgsInstance() { + return new createTable_args(); + } + + protected createTable_result getResult(I iface, createTable_args args) throws org.apache.thrift.TException { + createTable_result result = new createTable_result(); + try { + iface.createTable(args.tableName, args.columnFamilies); + } catch (IOError io) { + result.io = io; + } catch (IllegalArgument ia) { + result.ia = ia; + } catch (AlreadyExists exist) { + result.exist = exist; + } + return result; + } + } + + private static class deleteTable extends org.apache.thrift.ProcessFunction { + public deleteTable() { + super("deleteTable"); + } + + protected deleteTable_args getEmptyArgsInstance() { + return new deleteTable_args(); + } + + protected deleteTable_result getResult(I iface, deleteTable_args args) throws org.apache.thrift.TException { + deleteTable_result result = new deleteTable_result(); + try { + iface.deleteTable(args.tableName); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class get extends org.apache.thrift.ProcessFunction { + public get() { + super("get"); + } + + protected get_args getEmptyArgsInstance() { + return new get_args(); + } + + protected get_result getResult(I iface, get_args args) throws org.apache.thrift.TException { + get_result result = new get_result(); + try { + result.success = iface.get(args.tableName, args.row, args.column, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getVer extends org.apache.thrift.ProcessFunction { + public getVer() { + super("getVer"); + } + + protected getVer_args getEmptyArgsInstance() { + return new getVer_args(); + } + + protected getVer_result getResult(I iface, getVer_args args) throws org.apache.thrift.TException { + getVer_result result = new getVer_result(); + try { + result.success = iface.getVer(args.tableName, args.row, args.column, args.numVersions, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getVerTs extends org.apache.thrift.ProcessFunction { + public getVerTs() { + super("getVerTs"); + } + + protected getVerTs_args getEmptyArgsInstance() { + return new getVerTs_args(); + } + + protected getVerTs_result getResult(I iface, getVerTs_args args) throws org.apache.thrift.TException { + getVerTs_result result = new getVerTs_result(); + try { + result.success = iface.getVerTs(args.tableName, args.row, args.column, args.timestamp, args.numVersions, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getRow extends org.apache.thrift.ProcessFunction { + public getRow() { + super("getRow"); + } + + protected getRow_args getEmptyArgsInstance() { + return new getRow_args(); + } + + protected getRow_result getResult(I iface, getRow_args args) throws org.apache.thrift.TException { + getRow_result result = new getRow_result(); + try { + result.success = iface.getRow(args.tableName, args.row, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getRowWithColumns extends org.apache.thrift.ProcessFunction { + public getRowWithColumns() { + super("getRowWithColumns"); + } + + protected getRowWithColumns_args getEmptyArgsInstance() { + return new getRowWithColumns_args(); + } + + protected getRowWithColumns_result getResult(I iface, getRowWithColumns_args args) throws org.apache.thrift.TException { + getRowWithColumns_result result = new getRowWithColumns_result(); + try { + result.success = iface.getRowWithColumns(args.tableName, args.row, args.columns, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getRowTs extends org.apache.thrift.ProcessFunction { + public getRowTs() { + super("getRowTs"); + } + + protected getRowTs_args getEmptyArgsInstance() { + return new getRowTs_args(); + } + + protected getRowTs_result getResult(I iface, getRowTs_args args) throws org.apache.thrift.TException { + getRowTs_result result = new getRowTs_result(); + try { + result.success = iface.getRowTs(args.tableName, args.row, args.timestamp, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getRowWithColumnsTs extends org.apache.thrift.ProcessFunction { + public getRowWithColumnsTs() { + super("getRowWithColumnsTs"); + } + + protected getRowWithColumnsTs_args getEmptyArgsInstance() { + return new getRowWithColumnsTs_args(); + } + + protected getRowWithColumnsTs_result getResult(I iface, getRowWithColumnsTs_args args) throws org.apache.thrift.TException { + getRowWithColumnsTs_result result = new getRowWithColumnsTs_result(); + try { + result.success = iface.getRowWithColumnsTs(args.tableName, args.row, args.columns, args.timestamp, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getRows extends org.apache.thrift.ProcessFunction { + public getRows() { + super("getRows"); + } + + protected getRows_args getEmptyArgsInstance() { + return new getRows_args(); + } + + protected getRows_result getResult(I iface, getRows_args args) throws org.apache.thrift.TException { + getRows_result result = new getRows_result(); + try { + result.success = iface.getRows(args.tableName, args.rows, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getRowsWithColumns extends org.apache.thrift.ProcessFunction { + public getRowsWithColumns() { + super("getRowsWithColumns"); + } + + protected getRowsWithColumns_args getEmptyArgsInstance() { + return new getRowsWithColumns_args(); + } + + protected getRowsWithColumns_result getResult(I iface, getRowsWithColumns_args args) throws org.apache.thrift.TException { + getRowsWithColumns_result result = new getRowsWithColumns_result(); + try { + result.success = iface.getRowsWithColumns(args.tableName, args.rows, args.columns, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getRowsTs extends org.apache.thrift.ProcessFunction { + public getRowsTs() { + super("getRowsTs"); + } + + protected getRowsTs_args getEmptyArgsInstance() { + return new getRowsTs_args(); + } + + protected getRowsTs_result getResult(I iface, getRowsTs_args args) throws org.apache.thrift.TException { + getRowsTs_result result = new getRowsTs_result(); + try { + result.success = iface.getRowsTs(args.tableName, args.rows, args.timestamp, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getRowsWithColumnsTs extends org.apache.thrift.ProcessFunction { + public getRowsWithColumnsTs() { + super("getRowsWithColumnsTs"); + } + + protected getRowsWithColumnsTs_args getEmptyArgsInstance() { + return new getRowsWithColumnsTs_args(); + } + + protected getRowsWithColumnsTs_result getResult(I iface, getRowsWithColumnsTs_args args) throws org.apache.thrift.TException { + getRowsWithColumnsTs_result result = new getRowsWithColumnsTs_result(); + try { + result.success = iface.getRowsWithColumnsTs(args.tableName, args.rows, args.columns, args.timestamp, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class mutateRow extends org.apache.thrift.ProcessFunction { + public mutateRow() { + super("mutateRow"); + } + + protected mutateRow_args getEmptyArgsInstance() { + return new mutateRow_args(); + } + + protected mutateRow_result getResult(I iface, mutateRow_args args) throws org.apache.thrift.TException { + mutateRow_result result = new mutateRow_result(); + try { + iface.mutateRow(args.tableName, args.row, args.mutations, args.attributes); + } catch (IOError io) { + result.io = io; + } catch (IllegalArgument ia) { + result.ia = ia; + } + return result; + } + } + + private static class mutateRowTs extends org.apache.thrift.ProcessFunction { + public mutateRowTs() { + super("mutateRowTs"); + } + + protected mutateRowTs_args getEmptyArgsInstance() { + return new mutateRowTs_args(); + } + + protected mutateRowTs_result getResult(I iface, mutateRowTs_args args) throws org.apache.thrift.TException { + mutateRowTs_result result = new mutateRowTs_result(); + try { + iface.mutateRowTs(args.tableName, args.row, args.mutations, args.timestamp, args.attributes); + } catch (IOError io) { + result.io = io; + } catch (IllegalArgument ia) { + result.ia = ia; + } + return result; + } + } + + private static class mutateRows extends org.apache.thrift.ProcessFunction { + public mutateRows() { + super("mutateRows"); + } + + protected mutateRows_args getEmptyArgsInstance() { + return new mutateRows_args(); + } + + protected mutateRows_result getResult(I iface, mutateRows_args args) throws org.apache.thrift.TException { + mutateRows_result result = new mutateRows_result(); + try { + iface.mutateRows(args.tableName, args.rowBatches, args.attributes); + } catch (IOError io) { + result.io = io; + } catch (IllegalArgument ia) { + result.ia = ia; + } + return result; + } + } + + private static class mutateRowsTs extends org.apache.thrift.ProcessFunction { + public mutateRowsTs() { + super("mutateRowsTs"); + } + + protected mutateRowsTs_args getEmptyArgsInstance() { + return new mutateRowsTs_args(); + } + + protected mutateRowsTs_result getResult(I iface, mutateRowsTs_args args) throws org.apache.thrift.TException { + mutateRowsTs_result result = new mutateRowsTs_result(); + try { + iface.mutateRowsTs(args.tableName, args.rowBatches, args.timestamp, args.attributes); + } catch (IOError io) { + result.io = io; + } catch (IllegalArgument ia) { + result.ia = ia; + } + return result; + } + } + + private static class atomicIncrement extends org.apache.thrift.ProcessFunction { + public atomicIncrement() { + super("atomicIncrement"); + } + + protected atomicIncrement_args getEmptyArgsInstance() { + return new atomicIncrement_args(); + } + + protected atomicIncrement_result getResult(I iface, atomicIncrement_args args) throws org.apache.thrift.TException { + atomicIncrement_result result = new atomicIncrement_result(); + try { + result.success = iface.atomicIncrement(args.tableName, args.row, args.column, args.value); + result.setSuccessIsSet(true); + } catch (IOError io) { + result.io = io; + } catch (IllegalArgument ia) { + result.ia = ia; + } + return result; + } + } + + private static class deleteAll extends org.apache.thrift.ProcessFunction { + public deleteAll() { + super("deleteAll"); + } + + protected deleteAll_args getEmptyArgsInstance() { + return new deleteAll_args(); + } + + protected deleteAll_result getResult(I iface, deleteAll_args args) throws org.apache.thrift.TException { + deleteAll_result result = new deleteAll_result(); + try { + iface.deleteAll(args.tableName, args.row, args.column, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class deleteAllTs extends org.apache.thrift.ProcessFunction { + public deleteAllTs() { + super("deleteAllTs"); + } + + protected deleteAllTs_args getEmptyArgsInstance() { + return new deleteAllTs_args(); + } + + protected deleteAllTs_result getResult(I iface, deleteAllTs_args args) throws org.apache.thrift.TException { + deleteAllTs_result result = new deleteAllTs_result(); + try { + iface.deleteAllTs(args.tableName, args.row, args.column, args.timestamp, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class deleteAllRow extends org.apache.thrift.ProcessFunction { + public deleteAllRow() { + super("deleteAllRow"); + } + + protected deleteAllRow_args getEmptyArgsInstance() { + return new deleteAllRow_args(); + } + + protected deleteAllRow_result getResult(I iface, deleteAllRow_args args) throws org.apache.thrift.TException { + deleteAllRow_result result = new deleteAllRow_result(); + try { + iface.deleteAllRow(args.tableName, args.row, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class increment extends org.apache.thrift.ProcessFunction { + public increment() { + super("increment"); + } + + protected increment_args getEmptyArgsInstance() { + return new increment_args(); + } + + protected increment_result getResult(I iface, increment_args args) throws org.apache.thrift.TException { + increment_result result = new increment_result(); + try { + iface.increment(args.increment); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class incrementRows extends org.apache.thrift.ProcessFunction { + public incrementRows() { + super("incrementRows"); + } + + protected incrementRows_args getEmptyArgsInstance() { + return new incrementRows_args(); + } + + protected incrementRows_result getResult(I iface, incrementRows_args args) throws org.apache.thrift.TException { + incrementRows_result result = new incrementRows_result(); + try { + iface.incrementRows(args.increments); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class deleteAllRowTs extends org.apache.thrift.ProcessFunction { + public deleteAllRowTs() { + super("deleteAllRowTs"); + } + + protected deleteAllRowTs_args getEmptyArgsInstance() { + return new deleteAllRowTs_args(); + } + + protected deleteAllRowTs_result getResult(I iface, deleteAllRowTs_args args) throws org.apache.thrift.TException { + deleteAllRowTs_result result = new deleteAllRowTs_result(); + try { + iface.deleteAllRowTs(args.tableName, args.row, args.timestamp, args.attributes); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class scannerOpenWithScan extends org.apache.thrift.ProcessFunction { + public scannerOpenWithScan() { + super("scannerOpenWithScan"); + } + + protected scannerOpenWithScan_args getEmptyArgsInstance() { + return new scannerOpenWithScan_args(); + } + + protected scannerOpenWithScan_result getResult(I iface, scannerOpenWithScan_args args) throws org.apache.thrift.TException { + scannerOpenWithScan_result result = new scannerOpenWithScan_result(); + try { + result.success = iface.scannerOpenWithScan(args.tableName, args.scan, args.attributes); + result.setSuccessIsSet(true); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class scannerOpen extends org.apache.thrift.ProcessFunction { + public scannerOpen() { + super("scannerOpen"); + } + + protected scannerOpen_args getEmptyArgsInstance() { + return new scannerOpen_args(); + } + + protected scannerOpen_result getResult(I iface, scannerOpen_args args) throws org.apache.thrift.TException { + scannerOpen_result result = new scannerOpen_result(); + try { + result.success = iface.scannerOpen(args.tableName, args.startRow, args.columns, args.attributes); + result.setSuccessIsSet(true); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class scannerOpenWithStop extends org.apache.thrift.ProcessFunction { + public scannerOpenWithStop() { + super("scannerOpenWithStop"); + } + + protected scannerOpenWithStop_args getEmptyArgsInstance() { + return new scannerOpenWithStop_args(); + } + + protected scannerOpenWithStop_result getResult(I iface, scannerOpenWithStop_args args) throws org.apache.thrift.TException { + scannerOpenWithStop_result result = new scannerOpenWithStop_result(); + try { + result.success = iface.scannerOpenWithStop(args.tableName, args.startRow, args.stopRow, args.columns, args.attributes); + result.setSuccessIsSet(true); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class scannerOpenWithPrefix extends org.apache.thrift.ProcessFunction { + public scannerOpenWithPrefix() { + super("scannerOpenWithPrefix"); + } + + protected scannerOpenWithPrefix_args getEmptyArgsInstance() { + return new scannerOpenWithPrefix_args(); + } + + protected scannerOpenWithPrefix_result getResult(I iface, scannerOpenWithPrefix_args args) throws org.apache.thrift.TException { + scannerOpenWithPrefix_result result = new scannerOpenWithPrefix_result(); + try { + result.success = iface.scannerOpenWithPrefix(args.tableName, args.startAndPrefix, args.columns, args.attributes); + result.setSuccessIsSet(true); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class scannerOpenTs extends org.apache.thrift.ProcessFunction { + public scannerOpenTs() { + super("scannerOpenTs"); + } + + protected scannerOpenTs_args getEmptyArgsInstance() { + return new scannerOpenTs_args(); + } + + protected scannerOpenTs_result getResult(I iface, scannerOpenTs_args args) throws org.apache.thrift.TException { + scannerOpenTs_result result = new scannerOpenTs_result(); + try { + result.success = iface.scannerOpenTs(args.tableName, args.startRow, args.columns, args.timestamp, args.attributes); + result.setSuccessIsSet(true); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class scannerOpenWithStopTs extends org.apache.thrift.ProcessFunction { + public scannerOpenWithStopTs() { + super("scannerOpenWithStopTs"); + } + + protected scannerOpenWithStopTs_args getEmptyArgsInstance() { + return new scannerOpenWithStopTs_args(); + } + + protected scannerOpenWithStopTs_result getResult(I iface, scannerOpenWithStopTs_args args) throws org.apache.thrift.TException { + scannerOpenWithStopTs_result result = new scannerOpenWithStopTs_result(); + try { + result.success = iface.scannerOpenWithStopTs(args.tableName, args.startRow, args.stopRow, args.columns, args.timestamp, args.attributes); + result.setSuccessIsSet(true); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class scannerGet extends org.apache.thrift.ProcessFunction { + public scannerGet() { + super("scannerGet"); + } + + protected scannerGet_args getEmptyArgsInstance() { + return new scannerGet_args(); + } + + protected scannerGet_result getResult(I iface, scannerGet_args args) throws org.apache.thrift.TException { + scannerGet_result result = new scannerGet_result(); + try { + result.success = iface.scannerGet(args.id); + } catch (IOError io) { + result.io = io; + } catch (IllegalArgument ia) { + result.ia = ia; + } + return result; + } + } + + private static class scannerGetList extends org.apache.thrift.ProcessFunction { + public scannerGetList() { + super("scannerGetList"); + } + + protected scannerGetList_args getEmptyArgsInstance() { + return new scannerGetList_args(); + } + + protected scannerGetList_result getResult(I iface, scannerGetList_args args) throws org.apache.thrift.TException { + scannerGetList_result result = new scannerGetList_result(); + try { + result.success = iface.scannerGetList(args.id, args.nbRows); + } catch (IOError io) { + result.io = io; + } catch (IllegalArgument ia) { + result.ia = ia; + } + return result; + } + } + + private static class scannerClose extends org.apache.thrift.ProcessFunction { + public scannerClose() { + super("scannerClose"); + } + + protected scannerClose_args getEmptyArgsInstance() { + return new scannerClose_args(); + } + + protected scannerClose_result getResult(I iface, scannerClose_args args) throws org.apache.thrift.TException { + scannerClose_result result = new scannerClose_result(); + try { + iface.scannerClose(args.id); + } catch (IOError io) { + result.io = io; + } catch (IllegalArgument ia) { + result.ia = ia; + } + return result; + } + } + + private static class getRowOrBefore extends org.apache.thrift.ProcessFunction { + public getRowOrBefore() { + super("getRowOrBefore"); + } + + protected getRowOrBefore_args getEmptyArgsInstance() { + return new getRowOrBefore_args(); + } + + protected getRowOrBefore_result getResult(I iface, getRowOrBefore_args args) throws org.apache.thrift.TException { + getRowOrBefore_result result = new getRowOrBefore_result(); + try { + result.success = iface.getRowOrBefore(args.tableName, args.row, args.family); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + private static class getRegionInfo extends org.apache.thrift.ProcessFunction { + public getRegionInfo() { + super("getRegionInfo"); + } + + protected getRegionInfo_args getEmptyArgsInstance() { + return new getRegionInfo_args(); + } + + protected getRegionInfo_result getResult(I iface, getRegionInfo_args args) throws org.apache.thrift.TException { + getRegionInfo_result result = new getRegionInfo_result(); + try { + result.success = iface.getRegionInfo(args.row); + } catch (IOError io) { + result.io = io; + } + return result; + } + } + + } + + public static class enableTable_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("enableTable_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new enableTable_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new enableTable_argsTupleSchemeFactory()); + } + + /** + * name of the table + */ + public ByteBuffer tableName; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of the table + */ + TABLE_NAME((short)1, "tableName"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Bytes"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(enableTable_args.class, metaDataMap); + } + + public enableTable_args() { + } + + public enableTable_args( + ByteBuffer tableName) + { + this(); + this.tableName = tableName; + } + + /** + * Performs a deep copy on other. + */ + public enableTable_args(enableTable_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + } + + public enableTable_args deepCopy() { + return new enableTable_args(this); + } + + @Override + public void clear() { + this.tableName = null; + } + + /** + * name of the table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of the table + */ + public enableTable_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public enableTable_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof enableTable_args) + return this.equals((enableTable_args)that); + return false; + } + + public boolean equals(enableTable_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(enableTable_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + enableTable_args typedOther = (enableTable_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("enableTable_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class enableTable_argsStandardSchemeFactory implements SchemeFactory { + public enableTable_argsStandardScheme getScheme() { + return new enableTable_argsStandardScheme(); + } + } + + private static class enableTable_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, enableTable_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, enableTable_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class enableTable_argsTupleSchemeFactory implements SchemeFactory { + public enableTable_argsTupleScheme getScheme() { + return new enableTable_argsTupleScheme(); + } + } + + private static class enableTable_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, enableTable_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, enableTable_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + } + } + + } + + public static class enableTable_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("enableTable_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new enableTable_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new enableTable_resultTupleSchemeFactory()); + } + + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(enableTable_result.class, metaDataMap); + } + + public enableTable_result() { + } + + public enableTable_result( + IOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public enableTable_result(enableTable_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public enableTable_result deepCopy() { + return new enableTable_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public IOError getIo() { + return this.io; + } + + public enableTable_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof enableTable_result) + return this.equals((enableTable_result)that); + return false; + } + + public boolean equals(enableTable_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(enableTable_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + enableTable_result typedOther = (enableTable_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("enableTable_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class enableTable_resultStandardSchemeFactory implements SchemeFactory { + public enableTable_resultStandardScheme getScheme() { + return new enableTable_resultStandardScheme(); + } + } + + private static class enableTable_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, enableTable_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, enableTable_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class enableTable_resultTupleSchemeFactory implements SchemeFactory { + public enableTable_resultTupleScheme getScheme() { + return new enableTable_resultTupleScheme(); + } + } + + private static class enableTable_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, enableTable_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, enableTable_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class disableTable_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("disableTable_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new disableTable_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new disableTable_argsTupleSchemeFactory()); + } + + /** + * name of the table + */ + public ByteBuffer tableName; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of the table + */ + TABLE_NAME((short)1, "tableName"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Bytes"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(disableTable_args.class, metaDataMap); + } + + public disableTable_args() { + } + + public disableTable_args( + ByteBuffer tableName) + { + this(); + this.tableName = tableName; + } + + /** + * Performs a deep copy on other. + */ + public disableTable_args(disableTable_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + } + + public disableTable_args deepCopy() { + return new disableTable_args(this); + } + + @Override + public void clear() { + this.tableName = null; + } + + /** + * name of the table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of the table + */ + public disableTable_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public disableTable_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof disableTable_args) + return this.equals((disableTable_args)that); + return false; + } + + public boolean equals(disableTable_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(disableTable_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + disableTable_args typedOther = (disableTable_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("disableTable_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class disableTable_argsStandardSchemeFactory implements SchemeFactory { + public disableTable_argsStandardScheme getScheme() { + return new disableTable_argsStandardScheme(); + } + } + + private static class disableTable_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, disableTable_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, disableTable_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class disableTable_argsTupleSchemeFactory implements SchemeFactory { + public disableTable_argsTupleScheme getScheme() { + return new disableTable_argsTupleScheme(); + } + } + + private static class disableTable_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, disableTable_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, disableTable_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + } + } + + } + + public static class disableTable_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("disableTable_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new disableTable_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new disableTable_resultTupleSchemeFactory()); + } + + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(disableTable_result.class, metaDataMap); + } + + public disableTable_result() { + } + + public disableTable_result( + IOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public disableTable_result(disableTable_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public disableTable_result deepCopy() { + return new disableTable_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public IOError getIo() { + return this.io; + } + + public disableTable_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof disableTable_result) + return this.equals((disableTable_result)that); + return false; + } + + public boolean equals(disableTable_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(disableTable_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + disableTable_result typedOther = (disableTable_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("disableTable_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class disableTable_resultStandardSchemeFactory implements SchemeFactory { + public disableTable_resultStandardScheme getScheme() { + return new disableTable_resultStandardScheme(); + } + } + + private static class disableTable_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, disableTable_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, disableTable_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class disableTable_resultTupleSchemeFactory implements SchemeFactory { + public disableTable_resultTupleScheme getScheme() { + return new disableTable_resultTupleScheme(); + } + } + + private static class disableTable_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, disableTable_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, disableTable_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class isTableEnabled_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("isTableEnabled_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new isTableEnabled_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new isTableEnabled_argsTupleSchemeFactory()); + } + + /** + * name of the table to check + */ + public ByteBuffer tableName; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of the table to check + */ + TABLE_NAME((short)1, "tableName"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Bytes"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(isTableEnabled_args.class, metaDataMap); + } + + public isTableEnabled_args() { + } + + public isTableEnabled_args( + ByteBuffer tableName) + { + this(); + this.tableName = tableName; + } + + /** + * Performs a deep copy on other. + */ + public isTableEnabled_args(isTableEnabled_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + } + + public isTableEnabled_args deepCopy() { + return new isTableEnabled_args(this); + } + + @Override + public void clear() { + this.tableName = null; + } + + /** + * name of the table to check + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of the table to check + */ + public isTableEnabled_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public isTableEnabled_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof isTableEnabled_args) + return this.equals((isTableEnabled_args)that); + return false; + } + + public boolean equals(isTableEnabled_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(isTableEnabled_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + isTableEnabled_args typedOther = (isTableEnabled_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("isTableEnabled_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class isTableEnabled_argsStandardSchemeFactory implements SchemeFactory { + public isTableEnabled_argsStandardScheme getScheme() { + return new isTableEnabled_argsStandardScheme(); + } + } + + private static class isTableEnabled_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, isTableEnabled_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, isTableEnabled_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class isTableEnabled_argsTupleSchemeFactory implements SchemeFactory { + public isTableEnabled_argsTupleScheme getScheme() { + return new isTableEnabled_argsTupleScheme(); + } + } + + private static class isTableEnabled_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, isTableEnabled_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, isTableEnabled_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + } + } + + } + + public static class isTableEnabled_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("isTableEnabled_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.BOOL, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new isTableEnabled_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new isTableEnabled_resultTupleSchemeFactory()); + } + + public boolean success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(isTableEnabled_result.class, metaDataMap); + } + + public isTableEnabled_result() { + } + + public isTableEnabled_result( + boolean success, + IOError io) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public isTableEnabled_result(isTableEnabled_result other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.success = other.success; + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public isTableEnabled_result deepCopy() { + return new isTableEnabled_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = false; + this.io = null; + } + + public boolean isSuccess() { + return this.success; + } + + public isTableEnabled_result setSuccess(boolean success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bit_vector.clear(__SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return __isset_bit_vector.get(__SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bit_vector.set(__SUCCESS_ISSET_ID, value); + } + + public IOError getIo() { + return this.io; + } + + public isTableEnabled_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Boolean)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Boolean.valueOf(isSuccess()); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof isTableEnabled_result) + return this.equals((isTableEnabled_result)that); + return false; + } + + public boolean equals(isTableEnabled_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(isTableEnabled_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + isTableEnabled_result typedOther = (isTableEnabled_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("isTableEnabled_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class isTableEnabled_resultStandardSchemeFactory implements SchemeFactory { + public isTableEnabled_resultStandardScheme getScheme() { + return new isTableEnabled_resultStandardScheme(); + } + } + + private static class isTableEnabled_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, isTableEnabled_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, isTableEnabled_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeBool(struct.success); + oprot.writeFieldEnd(); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class isTableEnabled_resultTupleSchemeFactory implements SchemeFactory { + public isTableEnabled_resultTupleScheme getScheme() { + return new isTableEnabled_resultTupleScheme(); + } + } + + private static class isTableEnabled_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, isTableEnabled_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeBool(struct.success); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, isTableEnabled_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class compact_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("compact_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_OR_REGION_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableNameOrRegionName", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new compact_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new compact_argsTupleSchemeFactory()); + } + + public ByteBuffer tableNameOrRegionName; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TABLE_NAME_OR_REGION_NAME((short)1, "tableNameOrRegionName"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME_OR_REGION_NAME + return TABLE_NAME_OR_REGION_NAME; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME_OR_REGION_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableNameOrRegionName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Bytes"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(compact_args.class, metaDataMap); + } + + public compact_args() { + } + + public compact_args( + ByteBuffer tableNameOrRegionName) + { + this(); + this.tableNameOrRegionName = tableNameOrRegionName; + } + + /** + * Performs a deep copy on other. + */ + public compact_args(compact_args other) { + if (other.isSetTableNameOrRegionName()) { + this.tableNameOrRegionName = other.tableNameOrRegionName; + } + } + + public compact_args deepCopy() { + return new compact_args(this); + } + + @Override + public void clear() { + this.tableNameOrRegionName = null; + } + + public byte[] getTableNameOrRegionName() { + setTableNameOrRegionName(org.apache.thrift.TBaseHelper.rightSize(tableNameOrRegionName)); + return tableNameOrRegionName == null ? null : tableNameOrRegionName.array(); + } + + public ByteBuffer bufferForTableNameOrRegionName() { + return tableNameOrRegionName; + } + + public compact_args setTableNameOrRegionName(byte[] tableNameOrRegionName) { + setTableNameOrRegionName(tableNameOrRegionName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableNameOrRegionName)); + return this; + } + + public compact_args setTableNameOrRegionName(ByteBuffer tableNameOrRegionName) { + this.tableNameOrRegionName = tableNameOrRegionName; + return this; + } + + public void unsetTableNameOrRegionName() { + this.tableNameOrRegionName = null; + } + + /** Returns true if field tableNameOrRegionName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableNameOrRegionName() { + return this.tableNameOrRegionName != null; + } + + public void setTableNameOrRegionNameIsSet(boolean value) { + if (!value) { + this.tableNameOrRegionName = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME_OR_REGION_NAME: + if (value == null) { + unsetTableNameOrRegionName(); + } else { + setTableNameOrRegionName((ByteBuffer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME_OR_REGION_NAME: + return getTableNameOrRegionName(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME_OR_REGION_NAME: + return isSetTableNameOrRegionName(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof compact_args) + return this.equals((compact_args)that); + return false; + } + + public boolean equals(compact_args that) { + if (that == null) + return false; + + boolean this_present_tableNameOrRegionName = true && this.isSetTableNameOrRegionName(); + boolean that_present_tableNameOrRegionName = true && that.isSetTableNameOrRegionName(); + if (this_present_tableNameOrRegionName || that_present_tableNameOrRegionName) { + if (!(this_present_tableNameOrRegionName && that_present_tableNameOrRegionName)) + return false; + if (!this.tableNameOrRegionName.equals(that.tableNameOrRegionName)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(compact_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + compact_args typedOther = (compact_args)other; + + lastComparison = Boolean.valueOf(isSetTableNameOrRegionName()).compareTo(typedOther.isSetTableNameOrRegionName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableNameOrRegionName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableNameOrRegionName, typedOther.tableNameOrRegionName); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("compact_args("); + boolean first = true; + + sb.append("tableNameOrRegionName:"); + if (this.tableNameOrRegionName == null) { + sb.append("null"); + } else { + sb.append(this.tableNameOrRegionName); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class compact_argsStandardSchemeFactory implements SchemeFactory { + public compact_argsStandardScheme getScheme() { + return new compact_argsStandardScheme(); + } + } + + private static class compact_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, compact_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME_OR_REGION_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableNameOrRegionName = iprot.readBinary(); + struct.setTableNameOrRegionNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, compact_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableNameOrRegionName != null) { + oprot.writeFieldBegin(TABLE_NAME_OR_REGION_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableNameOrRegionName); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class compact_argsTupleSchemeFactory implements SchemeFactory { + public compact_argsTupleScheme getScheme() { + return new compact_argsTupleScheme(); + } + } + + private static class compact_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, compact_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableNameOrRegionName()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetTableNameOrRegionName()) { + oprot.writeBinary(struct.tableNameOrRegionName); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, compact_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.tableNameOrRegionName = iprot.readBinary(); + struct.setTableNameOrRegionNameIsSet(true); + } + } + } + + } + + public static class compact_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("compact_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new compact_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new compact_resultTupleSchemeFactory()); + } + + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(compact_result.class, metaDataMap); + } + + public compact_result() { + } + + public compact_result( + IOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public compact_result(compact_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public compact_result deepCopy() { + return new compact_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public IOError getIo() { + return this.io; + } + + public compact_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof compact_result) + return this.equals((compact_result)that); + return false; + } + + public boolean equals(compact_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(compact_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + compact_result typedOther = (compact_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("compact_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class compact_resultStandardSchemeFactory implements SchemeFactory { + public compact_resultStandardScheme getScheme() { + return new compact_resultStandardScheme(); + } + } + + private static class compact_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, compact_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, compact_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class compact_resultTupleSchemeFactory implements SchemeFactory { + public compact_resultTupleScheme getScheme() { + return new compact_resultTupleScheme(); + } + } + + private static class compact_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, compact_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, compact_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class majorCompact_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("majorCompact_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_OR_REGION_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableNameOrRegionName", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new majorCompact_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new majorCompact_argsTupleSchemeFactory()); + } + + public ByteBuffer tableNameOrRegionName; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TABLE_NAME_OR_REGION_NAME((short)1, "tableNameOrRegionName"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME_OR_REGION_NAME + return TABLE_NAME_OR_REGION_NAME; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME_OR_REGION_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableNameOrRegionName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Bytes"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(majorCompact_args.class, metaDataMap); + } + + public majorCompact_args() { + } + + public majorCompact_args( + ByteBuffer tableNameOrRegionName) + { + this(); + this.tableNameOrRegionName = tableNameOrRegionName; + } + + /** + * Performs a deep copy on other. + */ + public majorCompact_args(majorCompact_args other) { + if (other.isSetTableNameOrRegionName()) { + this.tableNameOrRegionName = other.tableNameOrRegionName; + } + } + + public majorCompact_args deepCopy() { + return new majorCompact_args(this); + } + + @Override + public void clear() { + this.tableNameOrRegionName = null; + } + + public byte[] getTableNameOrRegionName() { + setTableNameOrRegionName(org.apache.thrift.TBaseHelper.rightSize(tableNameOrRegionName)); + return tableNameOrRegionName == null ? null : tableNameOrRegionName.array(); + } + + public ByteBuffer bufferForTableNameOrRegionName() { + return tableNameOrRegionName; + } + + public majorCompact_args setTableNameOrRegionName(byte[] tableNameOrRegionName) { + setTableNameOrRegionName(tableNameOrRegionName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableNameOrRegionName)); + return this; + } + + public majorCompact_args setTableNameOrRegionName(ByteBuffer tableNameOrRegionName) { + this.tableNameOrRegionName = tableNameOrRegionName; + return this; + } + + public void unsetTableNameOrRegionName() { + this.tableNameOrRegionName = null; + } + + /** Returns true if field tableNameOrRegionName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableNameOrRegionName() { + return this.tableNameOrRegionName != null; + } + + public void setTableNameOrRegionNameIsSet(boolean value) { + if (!value) { + this.tableNameOrRegionName = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME_OR_REGION_NAME: + if (value == null) { + unsetTableNameOrRegionName(); + } else { + setTableNameOrRegionName((ByteBuffer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME_OR_REGION_NAME: + return getTableNameOrRegionName(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME_OR_REGION_NAME: + return isSetTableNameOrRegionName(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof majorCompact_args) + return this.equals((majorCompact_args)that); + return false; + } + + public boolean equals(majorCompact_args that) { + if (that == null) + return false; + + boolean this_present_tableNameOrRegionName = true && this.isSetTableNameOrRegionName(); + boolean that_present_tableNameOrRegionName = true && that.isSetTableNameOrRegionName(); + if (this_present_tableNameOrRegionName || that_present_tableNameOrRegionName) { + if (!(this_present_tableNameOrRegionName && that_present_tableNameOrRegionName)) + return false; + if (!this.tableNameOrRegionName.equals(that.tableNameOrRegionName)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(majorCompact_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + majorCompact_args typedOther = (majorCompact_args)other; + + lastComparison = Boolean.valueOf(isSetTableNameOrRegionName()).compareTo(typedOther.isSetTableNameOrRegionName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableNameOrRegionName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableNameOrRegionName, typedOther.tableNameOrRegionName); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("majorCompact_args("); + boolean first = true; + + sb.append("tableNameOrRegionName:"); + if (this.tableNameOrRegionName == null) { + sb.append("null"); + } else { + sb.append(this.tableNameOrRegionName); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class majorCompact_argsStandardSchemeFactory implements SchemeFactory { + public majorCompact_argsStandardScheme getScheme() { + return new majorCompact_argsStandardScheme(); + } + } + + private static class majorCompact_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, majorCompact_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME_OR_REGION_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableNameOrRegionName = iprot.readBinary(); + struct.setTableNameOrRegionNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, majorCompact_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableNameOrRegionName != null) { + oprot.writeFieldBegin(TABLE_NAME_OR_REGION_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableNameOrRegionName); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class majorCompact_argsTupleSchemeFactory implements SchemeFactory { + public majorCompact_argsTupleScheme getScheme() { + return new majorCompact_argsTupleScheme(); + } + } + + private static class majorCompact_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, majorCompact_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableNameOrRegionName()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetTableNameOrRegionName()) { + oprot.writeBinary(struct.tableNameOrRegionName); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, majorCompact_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.tableNameOrRegionName = iprot.readBinary(); + struct.setTableNameOrRegionNameIsSet(true); + } + } + } + + } + + public static class majorCompact_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("majorCompact_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new majorCompact_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new majorCompact_resultTupleSchemeFactory()); + } + + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(majorCompact_result.class, metaDataMap); + } + + public majorCompact_result() { + } + + public majorCompact_result( + IOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public majorCompact_result(majorCompact_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public majorCompact_result deepCopy() { + return new majorCompact_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public IOError getIo() { + return this.io; + } + + public majorCompact_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof majorCompact_result) + return this.equals((majorCompact_result)that); + return false; + } + + public boolean equals(majorCompact_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(majorCompact_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + majorCompact_result typedOther = (majorCompact_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("majorCompact_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class majorCompact_resultStandardSchemeFactory implements SchemeFactory { + public majorCompact_resultStandardScheme getScheme() { + return new majorCompact_resultStandardScheme(); + } + } + + private static class majorCompact_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, majorCompact_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, majorCompact_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class majorCompact_resultTupleSchemeFactory implements SchemeFactory { + public majorCompact_resultTupleScheme getScheme() { + return new majorCompact_resultTupleScheme(); + } + } + + private static class majorCompact_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, majorCompact_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, majorCompact_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getTableNames_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getTableNames_args"); + + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getTableNames_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getTableNames_argsTupleSchemeFactory()); + } + + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { +; + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getTableNames_args.class, metaDataMap); + } + + public getTableNames_args() { + } + + /** + * Performs a deep copy on other. + */ + public getTableNames_args(getTableNames_args other) { + } + + public getTableNames_args deepCopy() { + return new getTableNames_args(this); + } + + @Override + public void clear() { + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getTableNames_args) + return this.equals((getTableNames_args)that); + return false; + } + + public boolean equals(getTableNames_args that) { + if (that == null) + return false; + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getTableNames_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getTableNames_args typedOther = (getTableNames_args)other; + + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getTableNames_args("); + boolean first = true; + + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getTableNames_argsStandardSchemeFactory implements SchemeFactory { + public getTableNames_argsStandardScheme getScheme() { + return new getTableNames_argsStandardScheme(); + } + } + + private static class getTableNames_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getTableNames_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getTableNames_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getTableNames_argsTupleSchemeFactory implements SchemeFactory { + public getTableNames_argsTupleScheme getScheme() { + return new getTableNames_argsTupleScheme(); + } + } + + private static class getTableNames_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getTableNames_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getTableNames_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + } + } + + } + + public static class getTableNames_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getTableNames_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getTableNames_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getTableNames_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getTableNames_result.class, metaDataMap); + } + + public getTableNames_result() { + } + + public getTableNames_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getTableNames_result(getTableNames_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (ByteBuffer other_element : other.success) { + __this__success.add(other_element); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getTableNames_result deepCopy() { + return new getTableNames_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(ByteBuffer elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getTableNames_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getTableNames_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getTableNames_result) + return this.equals((getTableNames_result)that); + return false; + } + + public boolean equals(getTableNames_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getTableNames_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getTableNames_result typedOther = (getTableNames_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getTableNames_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getTableNames_resultStandardSchemeFactory implements SchemeFactory { + public getTableNames_resultStandardScheme getScheme() { + return new getTableNames_resultStandardScheme(); + } + } + + private static class getTableNames_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getTableNames_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list26 = iprot.readListBegin(); + struct.success = new ArrayList(_list26.size); + for (int _i27 = 0; _i27 < _list26.size; ++_i27) + { + ByteBuffer _elem28; // optional + _elem28 = iprot.readBinary(); + struct.success.add(_elem28); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getTableNames_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.success.size())); + for (ByteBuffer _iter29 : struct.success) + { + oprot.writeBinary(_iter29); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getTableNames_resultTupleSchemeFactory implements SchemeFactory { + public getTableNames_resultTupleScheme getScheme() { + return new getTableNames_resultTupleScheme(); + } + } + + private static class getTableNames_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getTableNames_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (ByteBuffer _iter30 : struct.success) + { + oprot.writeBinary(_iter30); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getTableNames_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list31 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.success = new ArrayList(_list31.size); + for (int _i32 = 0; _i32 < _list31.size; ++_i32) + { + ByteBuffer _elem33; // optional + _elem33 = iprot.readBinary(); + struct.success.add(_elem33); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getColumnDescriptors_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getColumnDescriptors_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getColumnDescriptors_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getColumnDescriptors_argsTupleSchemeFactory()); + } + + /** + * table name + */ + public ByteBuffer tableName; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * table name + */ + TABLE_NAME((short)1, "tableName"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getColumnDescriptors_args.class, metaDataMap); + } + + public getColumnDescriptors_args() { + } + + public getColumnDescriptors_args( + ByteBuffer tableName) + { + this(); + this.tableName = tableName; + } + + /** + * Performs a deep copy on other. + */ + public getColumnDescriptors_args(getColumnDescriptors_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + } + + public getColumnDescriptors_args deepCopy() { + return new getColumnDescriptors_args(this); + } + + @Override + public void clear() { + this.tableName = null; + } + + /** + * table name + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * table name + */ + public getColumnDescriptors_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getColumnDescriptors_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getColumnDescriptors_args) + return this.equals((getColumnDescriptors_args)that); + return false; + } + + public boolean equals(getColumnDescriptors_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getColumnDescriptors_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getColumnDescriptors_args typedOther = (getColumnDescriptors_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getColumnDescriptors_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getColumnDescriptors_argsStandardSchemeFactory implements SchemeFactory { + public getColumnDescriptors_argsStandardScheme getScheme() { + return new getColumnDescriptors_argsStandardScheme(); + } + } + + private static class getColumnDescriptors_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getColumnDescriptors_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getColumnDescriptors_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getColumnDescriptors_argsTupleSchemeFactory implements SchemeFactory { + public getColumnDescriptors_argsTupleScheme getScheme() { + return new getColumnDescriptors_argsTupleScheme(); + } + } + + private static class getColumnDescriptors_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getColumnDescriptors_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getColumnDescriptors_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + } + } + + } + + public static class getColumnDescriptors_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getColumnDescriptors_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.MAP, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getColumnDescriptors_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getColumnDescriptors_resultTupleSchemeFactory()); + } + + public Map success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnDescriptor.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getColumnDescriptors_result.class, metaDataMap); + } + + public getColumnDescriptors_result() { + } + + public getColumnDescriptors_result( + Map success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getColumnDescriptors_result(getColumnDescriptors_result other) { + if (other.isSetSuccess()) { + Map __this__success = new HashMap(); + for (Map.Entry other_element : other.success.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ColumnDescriptor other_element_value = other_element.getValue(); + + ByteBuffer __this__success_copy_key = other_element_key; + + ColumnDescriptor __this__success_copy_value = new ColumnDescriptor(other_element_value); + + __this__success.put(__this__success_copy_key, __this__success_copy_value); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getColumnDescriptors_result deepCopy() { + return new getColumnDescriptors_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public void putToSuccess(ByteBuffer key, ColumnDescriptor val) { + if (this.success == null) { + this.success = new HashMap(); + } + this.success.put(key, val); + } + + public Map getSuccess() { + return this.success; + } + + public getColumnDescriptors_result setSuccess(Map success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getColumnDescriptors_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Map)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getColumnDescriptors_result) + return this.equals((getColumnDescriptors_result)that); + return false; + } + + public boolean equals(getColumnDescriptors_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getColumnDescriptors_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getColumnDescriptors_result typedOther = (getColumnDescriptors_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getColumnDescriptors_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getColumnDescriptors_resultStandardSchemeFactory implements SchemeFactory { + public getColumnDescriptors_resultStandardScheme getScheme() { + return new getColumnDescriptors_resultStandardScheme(); + } + } + + private static class getColumnDescriptors_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getColumnDescriptors_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map34 = iprot.readMapBegin(); + struct.success = new HashMap(2*_map34.size); + for (int _i35 = 0; _i35 < _map34.size; ++_i35) + { + ByteBuffer _key36; // required + ColumnDescriptor _val37; // optional + _key36 = iprot.readBinary(); + _val37 = new ColumnDescriptor(); + _val37.read(iprot); + struct.success.put(_key36, _val37); + } + iprot.readMapEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getColumnDescriptors_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (Map.Entry _iter38 : struct.success.entrySet()) + { + oprot.writeBinary(_iter38.getKey()); + _iter38.getValue().write(oprot); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getColumnDescriptors_resultTupleSchemeFactory implements SchemeFactory { + public getColumnDescriptors_resultTupleScheme getScheme() { + return new getColumnDescriptors_resultTupleScheme(); + } + } + + private static class getColumnDescriptors_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getColumnDescriptors_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (Map.Entry _iter39 : struct.success.entrySet()) + { + oprot.writeBinary(_iter39.getKey()); + _iter39.getValue().write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getColumnDescriptors_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TMap _map40 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new HashMap(2*_map40.size); + for (int _i41 = 0; _i41 < _map40.size; ++_i41) + { + ByteBuffer _key42; // required + ColumnDescriptor _val43; // optional + _key42 = iprot.readBinary(); + _val43 = new ColumnDescriptor(); + _val43.read(iprot); + struct.success.put(_key42, _val43); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getTableRegions_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getTableRegions_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getTableRegions_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getTableRegions_argsTupleSchemeFactory()); + } + + /** + * table name + */ + public ByteBuffer tableName; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * table name + */ + TABLE_NAME((short)1, "tableName"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getTableRegions_args.class, metaDataMap); + } + + public getTableRegions_args() { + } + + public getTableRegions_args( + ByteBuffer tableName) + { + this(); + this.tableName = tableName; + } + + /** + * Performs a deep copy on other. + */ + public getTableRegions_args(getTableRegions_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + } + + public getTableRegions_args deepCopy() { + return new getTableRegions_args(this); + } + + @Override + public void clear() { + this.tableName = null; + } + + /** + * table name + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * table name + */ + public getTableRegions_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getTableRegions_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getTableRegions_args) + return this.equals((getTableRegions_args)that); + return false; + } + + public boolean equals(getTableRegions_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getTableRegions_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getTableRegions_args typedOther = (getTableRegions_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getTableRegions_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getTableRegions_argsStandardSchemeFactory implements SchemeFactory { + public getTableRegions_argsStandardScheme getScheme() { + return new getTableRegions_argsStandardScheme(); + } + } + + private static class getTableRegions_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getTableRegions_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getTableRegions_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getTableRegions_argsTupleSchemeFactory implements SchemeFactory { + public getTableRegions_argsTupleScheme getScheme() { + return new getTableRegions_argsTupleScheme(); + } + } + + private static class getTableRegions_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getTableRegions_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getTableRegions_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + } + } + + } + + public static class getTableRegions_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getTableRegions_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getTableRegions_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getTableRegions_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRegionInfo.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getTableRegions_result.class, metaDataMap); + } + + public getTableRegions_result() { + } + + public getTableRegions_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getTableRegions_result(getTableRegions_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TRegionInfo other_element : other.success) { + __this__success.add(new TRegionInfo(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getTableRegions_result deepCopy() { + return new getTableRegions_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TRegionInfo elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getTableRegions_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getTableRegions_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getTableRegions_result) + return this.equals((getTableRegions_result)that); + return false; + } + + public boolean equals(getTableRegions_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getTableRegions_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getTableRegions_result typedOther = (getTableRegions_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getTableRegions_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getTableRegions_resultStandardSchemeFactory implements SchemeFactory { + public getTableRegions_resultStandardScheme getScheme() { + return new getTableRegions_resultStandardScheme(); + } + } + + private static class getTableRegions_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getTableRegions_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list44 = iprot.readListBegin(); + struct.success = new ArrayList(_list44.size); + for (int _i45 = 0; _i45 < _list44.size; ++_i45) + { + TRegionInfo _elem46; // optional + _elem46 = new TRegionInfo(); + _elem46.read(iprot); + struct.success.add(_elem46); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getTableRegions_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TRegionInfo _iter47 : struct.success) + { + _iter47.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getTableRegions_resultTupleSchemeFactory implements SchemeFactory { + public getTableRegions_resultTupleScheme getScheme() { + return new getTableRegions_resultTupleScheme(); + } + } + + private static class getTableRegions_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getTableRegions_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TRegionInfo _iter48 : struct.success) + { + _iter48.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getTableRegions_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list49 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list49.size); + for (int _i50 = 0; _i50 < _list49.size; ++_i50) + { + TRegionInfo _elem51; // optional + _elem51 = new TRegionInfo(); + _elem51.read(iprot); + struct.success.add(_elem51); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class createTable_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("createTable_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField COLUMN_FAMILIES_FIELD_DESC = new org.apache.thrift.protocol.TField("columnFamilies", org.apache.thrift.protocol.TType.LIST, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new createTable_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new createTable_argsTupleSchemeFactory()); + } + + /** + * name of table to create + */ + public ByteBuffer tableName; // required + /** + * list of column family descriptors + */ + public List columnFamilies; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table to create + */ + TABLE_NAME((short)1, "tableName"), + /** + * list of column family descriptors + */ + COLUMN_FAMILIES((short)2, "columnFamilies"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // COLUMN_FAMILIES + return COLUMN_FAMILIES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMN_FAMILIES, new org.apache.thrift.meta_data.FieldMetaData("columnFamilies", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, ColumnDescriptor.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(createTable_args.class, metaDataMap); + } + + public createTable_args() { + } + + public createTable_args( + ByteBuffer tableName, + List columnFamilies) + { + this(); + this.tableName = tableName; + this.columnFamilies = columnFamilies; + } + + /** + * Performs a deep copy on other. + */ + public createTable_args(createTable_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetColumnFamilies()) { + List __this__columnFamilies = new ArrayList(); + for (ColumnDescriptor other_element : other.columnFamilies) { + __this__columnFamilies.add(new ColumnDescriptor(other_element)); + } + this.columnFamilies = __this__columnFamilies; + } + } + + public createTable_args deepCopy() { + return new createTable_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.columnFamilies = null; + } + + /** + * name of table to create + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table to create + */ + public createTable_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public createTable_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public int getColumnFamiliesSize() { + return (this.columnFamilies == null) ? 0 : this.columnFamilies.size(); + } + + public java.util.Iterator getColumnFamiliesIterator() { + return (this.columnFamilies == null) ? null : this.columnFamilies.iterator(); + } + + public void addToColumnFamilies(ColumnDescriptor elem) { + if (this.columnFamilies == null) { + this.columnFamilies = new ArrayList(); + } + this.columnFamilies.add(elem); + } + + /** + * list of column family descriptors + */ + public List getColumnFamilies() { + return this.columnFamilies; + } + + /** + * list of column family descriptors + */ + public createTable_args setColumnFamilies(List columnFamilies) { + this.columnFamilies = columnFamilies; + return this; + } + + public void unsetColumnFamilies() { + this.columnFamilies = null; + } + + /** Returns true if field columnFamilies is set (has been assigned a value) and false otherwise */ + public boolean isSetColumnFamilies() { + return this.columnFamilies != null; + } + + public void setColumnFamiliesIsSet(boolean value) { + if (!value) { + this.columnFamilies = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case COLUMN_FAMILIES: + if (value == null) { + unsetColumnFamilies(); + } else { + setColumnFamilies((List)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case COLUMN_FAMILIES: + return getColumnFamilies(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case COLUMN_FAMILIES: + return isSetColumnFamilies(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof createTable_args) + return this.equals((createTable_args)that); + return false; + } + + public boolean equals(createTable_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_columnFamilies = true && this.isSetColumnFamilies(); + boolean that_present_columnFamilies = true && that.isSetColumnFamilies(); + if (this_present_columnFamilies || that_present_columnFamilies) { + if (!(this_present_columnFamilies && that_present_columnFamilies)) + return false; + if (!this.columnFamilies.equals(that.columnFamilies)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(createTable_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + createTable_args typedOther = (createTable_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumnFamilies()).compareTo(typedOther.isSetColumnFamilies()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumnFamilies()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columnFamilies, typedOther.columnFamilies); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("createTable_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("columnFamilies:"); + if (this.columnFamilies == null) { + sb.append("null"); + } else { + sb.append(this.columnFamilies); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class createTable_argsStandardSchemeFactory implements SchemeFactory { + public createTable_argsStandardScheme getScheme() { + return new createTable_argsStandardScheme(); + } + } + + private static class createTable_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, createTable_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // COLUMN_FAMILIES + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list52 = iprot.readListBegin(); + struct.columnFamilies = new ArrayList(_list52.size); + for (int _i53 = 0; _i53 < _list52.size; ++_i53) + { + ColumnDescriptor _elem54; // optional + _elem54 = new ColumnDescriptor(); + _elem54.read(iprot); + struct.columnFamilies.add(_elem54); + } + iprot.readListEnd(); + } + struct.setColumnFamiliesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, createTable_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.columnFamilies != null) { + oprot.writeFieldBegin(COLUMN_FAMILIES_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columnFamilies.size())); + for (ColumnDescriptor _iter55 : struct.columnFamilies) + { + _iter55.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class createTable_argsTupleSchemeFactory implements SchemeFactory { + public createTable_argsTupleScheme getScheme() { + return new createTable_argsTupleScheme(); + } + } + + private static class createTable_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, createTable_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetColumnFamilies()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetColumnFamilies()) { + { + oprot.writeI32(struct.columnFamilies.size()); + for (ColumnDescriptor _iter56 : struct.columnFamilies) + { + _iter56.write(oprot); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, createTable_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + { + org.apache.thrift.protocol.TList _list57 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.columnFamilies = new ArrayList(_list57.size); + for (int _i58 = 0; _i58 < _list57.size; ++_i58) + { + ColumnDescriptor _elem59; // optional + _elem59 = new ColumnDescriptor(); + _elem59.read(iprot); + struct.columnFamilies.add(_elem59); + } + } + struct.setColumnFamiliesIsSet(true); + } + } + } + + } + + public static class createTable_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("createTable_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField IA_FIELD_DESC = new org.apache.thrift.protocol.TField("ia", org.apache.thrift.protocol.TType.STRUCT, (short)2); + private static final org.apache.thrift.protocol.TField EXIST_FIELD_DESC = new org.apache.thrift.protocol.TField("exist", org.apache.thrift.protocol.TType.STRUCT, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new createTable_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new createTable_resultTupleSchemeFactory()); + } + + public IOError io; // required + public IllegalArgument ia; // required + public AlreadyExists exist; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"), + IA((short)2, "ia"), + EXIST((short)3, "exist"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + case 2: // IA + return IA; + case 3: // EXIST + return EXIST; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + tmpMap.put(_Fields.IA, new org.apache.thrift.meta_data.FieldMetaData("ia", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + tmpMap.put(_Fields.EXIST, new org.apache.thrift.meta_data.FieldMetaData("exist", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(createTable_result.class, metaDataMap); + } + + public createTable_result() { + } + + public createTable_result( + IOError io, + IllegalArgument ia, + AlreadyExists exist) + { + this(); + this.io = io; + this.ia = ia; + this.exist = exist; + } + + /** + * Performs a deep copy on other. + */ + public createTable_result(createTable_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + if (other.isSetIa()) { + this.ia = new IllegalArgument(other.ia); + } + if (other.isSetExist()) { + this.exist = new AlreadyExists(other.exist); + } + } + + public createTable_result deepCopy() { + return new createTable_result(this); + } + + @Override + public void clear() { + this.io = null; + this.ia = null; + this.exist = null; + } + + public IOError getIo() { + return this.io; + } + + public createTable_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public IllegalArgument getIa() { + return this.ia; + } + + public createTable_result setIa(IllegalArgument ia) { + this.ia = ia; + return this; + } + + public void unsetIa() { + this.ia = null; + } + + /** Returns true if field ia is set (has been assigned a value) and false otherwise */ + public boolean isSetIa() { + return this.ia != null; + } + + public void setIaIsSet(boolean value) { + if (!value) { + this.ia = null; + } + } + + public AlreadyExists getExist() { + return this.exist; + } + + public createTable_result setExist(AlreadyExists exist) { + this.exist = exist; + return this; + } + + public void unsetExist() { + this.exist = null; + } + + /** Returns true if field exist is set (has been assigned a value) and false otherwise */ + public boolean isSetExist() { + return this.exist != null; + } + + public void setExistIsSet(boolean value) { + if (!value) { + this.exist = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + case IA: + if (value == null) { + unsetIa(); + } else { + setIa((IllegalArgument)value); + } + break; + + case EXIST: + if (value == null) { + unsetExist(); + } else { + setExist((AlreadyExists)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + case IA: + return getIa(); + + case EXIST: + return getExist(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + case IA: + return isSetIa(); + case EXIST: + return isSetExist(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof createTable_result) + return this.equals((createTable_result)that); + return false; + } + + public boolean equals(createTable_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + boolean this_present_ia = true && this.isSetIa(); + boolean that_present_ia = true && that.isSetIa(); + if (this_present_ia || that_present_ia) { + if (!(this_present_ia && that_present_ia)) + return false; + if (!this.ia.equals(that.ia)) + return false; + } + + boolean this_present_exist = true && this.isSetExist(); + boolean that_present_exist = true && that.isSetExist(); + if (this_present_exist || that_present_exist) { + if (!(this_present_exist && that_present_exist)) + return false; + if (!this.exist.equals(that.exist)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(createTable_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + createTable_result typedOther = (createTable_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIa()).compareTo(typedOther.isSetIa()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIa()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ia, typedOther.ia); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetExist()).compareTo(typedOther.isSetExist()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetExist()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.exist, typedOther.exist); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("createTable_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + if (!first) sb.append(", "); + sb.append("ia:"); + if (this.ia == null) { + sb.append("null"); + } else { + sb.append(this.ia); + } + first = false; + if (!first) sb.append(", "); + sb.append("exist:"); + if (this.exist == null) { + sb.append("null"); + } else { + sb.append(this.exist); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class createTable_resultStandardSchemeFactory implements SchemeFactory { + public createTable_resultStandardScheme getScheme() { + return new createTable_resultStandardScheme(); + } + } + + private static class createTable_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, createTable_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // IA + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // EXIST + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.exist = new AlreadyExists(); + struct.exist.read(iprot); + struct.setExistIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, createTable_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.ia != null) { + oprot.writeFieldBegin(IA_FIELD_DESC); + struct.ia.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.exist != null) { + oprot.writeFieldBegin(EXIST_FIELD_DESC); + struct.exist.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class createTable_resultTupleSchemeFactory implements SchemeFactory { + public createTable_resultTupleScheme getScheme() { + return new createTable_resultTupleScheme(); + } + } + + private static class createTable_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, createTable_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + if (struct.isSetIa()) { + optionals.set(1); + } + if (struct.isSetExist()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + if (struct.isSetIa()) { + struct.ia.write(oprot); + } + if (struct.isSetExist()) { + struct.exist.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, createTable_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + if (incoming.get(1)) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } + if (incoming.get(2)) { + struct.exist = new AlreadyExists(); + struct.exist.read(iprot); + struct.setExistIsSet(true); + } + } + } + + } + + public static class deleteTable_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteTable_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteTable_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteTable_argsTupleSchemeFactory()); + } + + /** + * name of table to delete + */ + public ByteBuffer tableName; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table to delete + */ + TABLE_NAME((short)1, "tableName"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteTable_args.class, metaDataMap); + } + + public deleteTable_args() { + } + + public deleteTable_args( + ByteBuffer tableName) + { + this(); + this.tableName = tableName; + } + + /** + * Performs a deep copy on other. + */ + public deleteTable_args(deleteTable_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + } + + public deleteTable_args deepCopy() { + return new deleteTable_args(this); + } + + @Override + public void clear() { + this.tableName = null; + } + + /** + * name of table to delete + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table to delete + */ + public deleteTable_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public deleteTable_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteTable_args) + return this.equals((deleteTable_args)that); + return false; + } + + public boolean equals(deleteTable_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteTable_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteTable_args typedOther = (deleteTable_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteTable_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteTable_argsStandardSchemeFactory implements SchemeFactory { + public deleteTable_argsStandardScheme getScheme() { + return new deleteTable_argsStandardScheme(); + } + } + + private static class deleteTable_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteTable_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteTable_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteTable_argsTupleSchemeFactory implements SchemeFactory { + public deleteTable_argsTupleScheme getScheme() { + return new deleteTable_argsTupleScheme(); + } + } + + private static class deleteTable_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteTable_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteTable_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + } + } + + } + + public static class deleteTable_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteTable_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteTable_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteTable_resultTupleSchemeFactory()); + } + + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteTable_result.class, metaDataMap); + } + + public deleteTable_result() { + } + + public deleteTable_result( + IOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public deleteTable_result(deleteTable_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public deleteTable_result deepCopy() { + return new deleteTable_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public IOError getIo() { + return this.io; + } + + public deleteTable_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteTable_result) + return this.equals((deleteTable_result)that); + return false; + } + + public boolean equals(deleteTable_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteTable_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteTable_result typedOther = (deleteTable_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteTable_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteTable_resultStandardSchemeFactory implements SchemeFactory { + public deleteTable_resultStandardScheme getScheme() { + return new deleteTable_resultStandardScheme(); + } + } + + private static class deleteTable_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteTable_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteTable_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteTable_resultTupleSchemeFactory implements SchemeFactory { + public deleteTable_resultTupleScheme getScheme() { + return new deleteTable_resultTupleScheme(); + } + } + + private static class deleteTable_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteTable_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteTable_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class get_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new get_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new get_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row key + */ + public ByteBuffer row; // required + /** + * column name + */ + public ByteBuffer column; // required + /** + * Get attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row key + */ + ROW((short)2, "row"), + /** + * column name + */ + COLUMN((short)3, "column"), + /** + * Get attributes + */ + ATTRIBUTES((short)4, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // COLUMN + return COLUMN; + case 4: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_args.class, metaDataMap); + } + + public get_args() { + } + + public get_args( + ByteBuffer tableName, + ByteBuffer row, + ByteBuffer column, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.column = column; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public get_args(get_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetColumn()) { + this.column = other.column; + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public get_args deepCopy() { + return new get_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.column = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public get_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public get_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * row key + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row key + */ + public get_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public get_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + /** + * column name + */ + public byte[] getColumn() { + setColumn(org.apache.thrift.TBaseHelper.rightSize(column)); + return column == null ? null : column.array(); + } + + public ByteBuffer bufferForColumn() { + return column; + } + + /** + * column name + */ + public get_args setColumn(byte[] column) { + setColumn(column == null ? (ByteBuffer)null : ByteBuffer.wrap(column)); + return this; + } + + public get_args setColumn(ByteBuffer column) { + this.column = column; + return this; + } + + public void unsetColumn() { + this.column = null; + } + + /** Returns true if field column is set (has been assigned a value) and false otherwise */ + public boolean isSetColumn() { + return this.column != null; + } + + public void setColumnIsSet(boolean value) { + if (!value) { + this.column = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Get attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Get attributes + */ + public get_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMN: + if (value == null) { + unsetColumn(); + } else { + setColumn((ByteBuffer)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case COLUMN: + return getColumn(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case COLUMN: + return isSetColumn(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof get_args) + return this.equals((get_args)that); + return false; + } + + public boolean equals(get_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_column = true && this.isSetColumn(); + boolean that_present_column = true && that.isSetColumn(); + if (this_present_column || that_present_column) { + if (!(this_present_column && that_present_column)) + return false; + if (!this.column.equals(that.column)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(get_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + get_args typedOther = (get_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumn()).compareTo(typedOther.isSetColumn()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumn()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, typedOther.column); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("get_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("column:"); + if (this.column == null) { + sb.append("null"); + } else { + sb.append(this.column); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class get_argsStandardSchemeFactory implements SchemeFactory { + public get_argsStandardScheme getScheme() { + return new get_argsStandardScheme(); + } + } + + private static class get_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, get_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMN + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map60 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map60.size); + for (int _i61 = 0; _i61 < _map60.size; ++_i61) + { + ByteBuffer _key62; // required + ByteBuffer _val63; // optional + _key62 = iprot.readBinary(); + _val63 = iprot.readBinary(); + struct.attributes.put(_key62, _val63); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, get_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.column != null) { + oprot.writeFieldBegin(COLUMN_FIELD_DESC); + oprot.writeBinary(struct.column); + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter64 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter64.getKey()); + oprot.writeBinary(_iter64.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class get_argsTupleSchemeFactory implements SchemeFactory { + public get_argsTupleScheme getScheme() { + return new get_argsTupleScheme(); + } + } + + private static class get_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, get_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetColumn()) { + optionals.set(2); + } + if (struct.isSetAttributes()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetColumn()) { + oprot.writeBinary(struct.column); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter65 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter65.getKey()); + oprot.writeBinary(_iter65.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, get_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TMap _map66 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map66.size); + for (int _i67 = 0; _i67 < _map66.size; ++_i67) + { + ByteBuffer _key68; // required + ByteBuffer _val69; // optional + _key68 = iprot.readBinary(); + _val69 = iprot.readBinary(); + struct.attributes.put(_key68, _val69); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class get_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new get_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new get_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCell.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_result.class, metaDataMap); + } + + public get_result() { + } + + public get_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public get_result(get_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TCell other_element : other.success) { + __this__success.add(new TCell(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public get_result deepCopy() { + return new get_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TCell elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public get_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public get_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof get_result) + return this.equals((get_result)that); + return false; + } + + public boolean equals(get_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(get_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + get_result typedOther = (get_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("get_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class get_resultStandardSchemeFactory implements SchemeFactory { + public get_resultStandardScheme getScheme() { + return new get_resultStandardScheme(); + } + } + + private static class get_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, get_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list70 = iprot.readListBegin(); + struct.success = new ArrayList(_list70.size); + for (int _i71 = 0; _i71 < _list70.size; ++_i71) + { + TCell _elem72; // optional + _elem72 = new TCell(); + _elem72.read(iprot); + struct.success.add(_elem72); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, get_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TCell _iter73 : struct.success) + { + _iter73.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class get_resultTupleSchemeFactory implements SchemeFactory { + public get_resultTupleScheme getScheme() { + return new get_resultTupleScheme(); + } + } + + private static class get_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, get_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TCell _iter74 : struct.success) + { + _iter74.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, get_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list75 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list75.size); + for (int _i76 = 0; _i76 < _list75.size; ++_i76) + { + TCell _elem77; // optional + _elem77 = new TCell(); + _elem77.read(iprot); + struct.success.add(_elem77); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getVer_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getVer_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField NUM_VERSIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("numVersions", org.apache.thrift.protocol.TType.I32, (short)4); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)5); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getVer_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getVer_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row key + */ + public ByteBuffer row; // required + /** + * column name + */ + public ByteBuffer column; // required + /** + * number of versions to retrieve + */ + public int numVersions; // required + /** + * Get attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row key + */ + ROW((short)2, "row"), + /** + * column name + */ + COLUMN((short)3, "column"), + /** + * number of versions to retrieve + */ + NUM_VERSIONS((short)4, "numVersions"), + /** + * Get attributes + */ + ATTRIBUTES((short)5, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // COLUMN + return COLUMN; + case 4: // NUM_VERSIONS + return NUM_VERSIONS; + case 5: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __NUMVERSIONS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.NUM_VERSIONS, new org.apache.thrift.meta_data.FieldMetaData("numVersions", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getVer_args.class, metaDataMap); + } + + public getVer_args() { + } + + public getVer_args( + ByteBuffer tableName, + ByteBuffer row, + ByteBuffer column, + int numVersions, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.column = column; + this.numVersions = numVersions; + setNumVersionsIsSet(true); + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public getVer_args(getVer_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetColumn()) { + this.column = other.column; + } + this.numVersions = other.numVersions; + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public getVer_args deepCopy() { + return new getVer_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.column = null; + setNumVersionsIsSet(false); + this.numVersions = 0; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public getVer_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getVer_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * row key + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row key + */ + public getVer_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public getVer_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + /** + * column name + */ + public byte[] getColumn() { + setColumn(org.apache.thrift.TBaseHelper.rightSize(column)); + return column == null ? null : column.array(); + } + + public ByteBuffer bufferForColumn() { + return column; + } + + /** + * column name + */ + public getVer_args setColumn(byte[] column) { + setColumn(column == null ? (ByteBuffer)null : ByteBuffer.wrap(column)); + return this; + } + + public getVer_args setColumn(ByteBuffer column) { + this.column = column; + return this; + } + + public void unsetColumn() { + this.column = null; + } + + /** Returns true if field column is set (has been assigned a value) and false otherwise */ + public boolean isSetColumn() { + return this.column != null; + } + + public void setColumnIsSet(boolean value) { + if (!value) { + this.column = null; + } + } + + /** + * number of versions to retrieve + */ + public int getNumVersions() { + return this.numVersions; + } + + /** + * number of versions to retrieve + */ + public getVer_args setNumVersions(int numVersions) { + this.numVersions = numVersions; + setNumVersionsIsSet(true); + return this; + } + + public void unsetNumVersions() { + __isset_bit_vector.clear(__NUMVERSIONS_ISSET_ID); + } + + /** Returns true if field numVersions is set (has been assigned a value) and false otherwise */ + public boolean isSetNumVersions() { + return __isset_bit_vector.get(__NUMVERSIONS_ISSET_ID); + } + + public void setNumVersionsIsSet(boolean value) { + __isset_bit_vector.set(__NUMVERSIONS_ISSET_ID, value); + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Get attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Get attributes + */ + public getVer_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMN: + if (value == null) { + unsetColumn(); + } else { + setColumn((ByteBuffer)value); + } + break; + + case NUM_VERSIONS: + if (value == null) { + unsetNumVersions(); + } else { + setNumVersions((Integer)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case COLUMN: + return getColumn(); + + case NUM_VERSIONS: + return Integer.valueOf(getNumVersions()); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case COLUMN: + return isSetColumn(); + case NUM_VERSIONS: + return isSetNumVersions(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getVer_args) + return this.equals((getVer_args)that); + return false; + } + + public boolean equals(getVer_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_column = true && this.isSetColumn(); + boolean that_present_column = true && that.isSetColumn(); + if (this_present_column || that_present_column) { + if (!(this_present_column && that_present_column)) + return false; + if (!this.column.equals(that.column)) + return false; + } + + boolean this_present_numVersions = true; + boolean that_present_numVersions = true; + if (this_present_numVersions || that_present_numVersions) { + if (!(this_present_numVersions && that_present_numVersions)) + return false; + if (this.numVersions != that.numVersions) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getVer_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getVer_args typedOther = (getVer_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumn()).compareTo(typedOther.isSetColumn()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumn()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, typedOther.column); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetNumVersions()).compareTo(typedOther.isSetNumVersions()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetNumVersions()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.numVersions, typedOther.numVersions); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getVer_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("column:"); + if (this.column == null) { + sb.append("null"); + } else { + sb.append(this.column); + } + first = false; + if (!first) sb.append(", "); + sb.append("numVersions:"); + sb.append(this.numVersions); + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getVer_argsStandardSchemeFactory implements SchemeFactory { + public getVer_argsStandardScheme getScheme() { + return new getVer_argsStandardScheme(); + } + } + + private static class getVer_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getVer_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMN + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // NUM_VERSIONS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.numVersions = iprot.readI32(); + struct.setNumVersionsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map78 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map78.size); + for (int _i79 = 0; _i79 < _map78.size; ++_i79) + { + ByteBuffer _key80; // required + ByteBuffer _val81; // optional + _key80 = iprot.readBinary(); + _val81 = iprot.readBinary(); + struct.attributes.put(_key80, _val81); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getVer_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.column != null) { + oprot.writeFieldBegin(COLUMN_FIELD_DESC); + oprot.writeBinary(struct.column); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(NUM_VERSIONS_FIELD_DESC); + oprot.writeI32(struct.numVersions); + oprot.writeFieldEnd(); + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter82 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter82.getKey()); + oprot.writeBinary(_iter82.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getVer_argsTupleSchemeFactory implements SchemeFactory { + public getVer_argsTupleScheme getScheme() { + return new getVer_argsTupleScheme(); + } + } + + private static class getVer_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getVer_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetColumn()) { + optionals.set(2); + } + if (struct.isSetNumVersions()) { + optionals.set(3); + } + if (struct.isSetAttributes()) { + optionals.set(4); + } + oprot.writeBitSet(optionals, 5); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetColumn()) { + oprot.writeBinary(struct.column); + } + if (struct.isSetNumVersions()) { + oprot.writeI32(struct.numVersions); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter83 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter83.getKey()); + oprot.writeBinary(_iter83.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getVer_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(5); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } + if (incoming.get(3)) { + struct.numVersions = iprot.readI32(); + struct.setNumVersionsIsSet(true); + } + if (incoming.get(4)) { + { + org.apache.thrift.protocol.TMap _map84 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map84.size); + for (int _i85 = 0; _i85 < _map84.size; ++_i85) + { + ByteBuffer _key86; // required + ByteBuffer _val87; // optional + _key86 = iprot.readBinary(); + _val87 = iprot.readBinary(); + struct.attributes.put(_key86, _val87); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class getVer_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getVer_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getVer_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getVer_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCell.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getVer_result.class, metaDataMap); + } + + public getVer_result() { + } + + public getVer_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getVer_result(getVer_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TCell other_element : other.success) { + __this__success.add(new TCell(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getVer_result deepCopy() { + return new getVer_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TCell elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getVer_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getVer_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getVer_result) + return this.equals((getVer_result)that); + return false; + } + + public boolean equals(getVer_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getVer_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getVer_result typedOther = (getVer_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getVer_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getVer_resultStandardSchemeFactory implements SchemeFactory { + public getVer_resultStandardScheme getScheme() { + return new getVer_resultStandardScheme(); + } + } + + private static class getVer_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getVer_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list88 = iprot.readListBegin(); + struct.success = new ArrayList(_list88.size); + for (int _i89 = 0; _i89 < _list88.size; ++_i89) + { + TCell _elem90; // optional + _elem90 = new TCell(); + _elem90.read(iprot); + struct.success.add(_elem90); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getVer_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TCell _iter91 : struct.success) + { + _iter91.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getVer_resultTupleSchemeFactory implements SchemeFactory { + public getVer_resultTupleScheme getScheme() { + return new getVer_resultTupleScheme(); + } + } + + private static class getVer_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getVer_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TCell _iter92 : struct.success) + { + _iter92.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getVer_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list93 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list93.size); + for (int _i94 = 0; _i94 < _list93.size; ++_i94) + { + TCell _elem95; // optional + _elem95 = new TCell(); + _elem95.read(iprot); + struct.success.add(_elem95); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getVerTs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getVerTs_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)4); + private static final org.apache.thrift.protocol.TField NUM_VERSIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("numVersions", org.apache.thrift.protocol.TType.I32, (short)5); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)6); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getVerTs_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getVerTs_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row key + */ + public ByteBuffer row; // required + /** + * column name + */ + public ByteBuffer column; // required + /** + * timestamp + */ + public long timestamp; // required + /** + * number of versions to retrieve + */ + public int numVersions; // required + /** + * Get attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row key + */ + ROW((short)2, "row"), + /** + * column name + */ + COLUMN((short)3, "column"), + /** + * timestamp + */ + TIMESTAMP((short)4, "timestamp"), + /** + * number of versions to retrieve + */ + NUM_VERSIONS((short)5, "numVersions"), + /** + * Get attributes + */ + ATTRIBUTES((short)6, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // COLUMN + return COLUMN; + case 4: // TIMESTAMP + return TIMESTAMP; + case 5: // NUM_VERSIONS + return NUM_VERSIONS; + case 6: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private static final int __NUMVERSIONS_ISSET_ID = 1; + private BitSet __isset_bit_vector = new BitSet(2); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.NUM_VERSIONS, new org.apache.thrift.meta_data.FieldMetaData("numVersions", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getVerTs_args.class, metaDataMap); + } + + public getVerTs_args() { + } + + public getVerTs_args( + ByteBuffer tableName, + ByteBuffer row, + ByteBuffer column, + long timestamp, + int numVersions, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.column = column; + this.timestamp = timestamp; + setTimestampIsSet(true); + this.numVersions = numVersions; + setNumVersionsIsSet(true); + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public getVerTs_args(getVerTs_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetColumn()) { + this.column = other.column; + } + this.timestamp = other.timestamp; + this.numVersions = other.numVersions; + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public getVerTs_args deepCopy() { + return new getVerTs_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.column = null; + setTimestampIsSet(false); + this.timestamp = 0; + setNumVersionsIsSet(false); + this.numVersions = 0; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public getVerTs_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getVerTs_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * row key + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row key + */ + public getVerTs_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public getVerTs_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + /** + * column name + */ + public byte[] getColumn() { + setColumn(org.apache.thrift.TBaseHelper.rightSize(column)); + return column == null ? null : column.array(); + } + + public ByteBuffer bufferForColumn() { + return column; + } + + /** + * column name + */ + public getVerTs_args setColumn(byte[] column) { + setColumn(column == null ? (ByteBuffer)null : ByteBuffer.wrap(column)); + return this; + } + + public getVerTs_args setColumn(ByteBuffer column) { + this.column = column; + return this; + } + + public void unsetColumn() { + this.column = null; + } + + /** Returns true if field column is set (has been assigned a value) and false otherwise */ + public boolean isSetColumn() { + return this.column != null; + } + + public void setColumnIsSet(boolean value) { + if (!value) { + this.column = null; + } + } + + /** + * timestamp + */ + public long getTimestamp() { + return this.timestamp; + } + + /** + * timestamp + */ + public getVerTs_args setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + /** + * number of versions to retrieve + */ + public int getNumVersions() { + return this.numVersions; + } + + /** + * number of versions to retrieve + */ + public getVerTs_args setNumVersions(int numVersions) { + this.numVersions = numVersions; + setNumVersionsIsSet(true); + return this; + } + + public void unsetNumVersions() { + __isset_bit_vector.clear(__NUMVERSIONS_ISSET_ID); + } + + /** Returns true if field numVersions is set (has been assigned a value) and false otherwise */ + public boolean isSetNumVersions() { + return __isset_bit_vector.get(__NUMVERSIONS_ISSET_ID); + } + + public void setNumVersionsIsSet(boolean value) { + __isset_bit_vector.set(__NUMVERSIONS_ISSET_ID, value); + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Get attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Get attributes + */ + public getVerTs_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMN: + if (value == null) { + unsetColumn(); + } else { + setColumn((ByteBuffer)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case NUM_VERSIONS: + if (value == null) { + unsetNumVersions(); + } else { + setNumVersions((Integer)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case COLUMN: + return getColumn(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case NUM_VERSIONS: + return Integer.valueOf(getNumVersions()); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case COLUMN: + return isSetColumn(); + case TIMESTAMP: + return isSetTimestamp(); + case NUM_VERSIONS: + return isSetNumVersions(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getVerTs_args) + return this.equals((getVerTs_args)that); + return false; + } + + public boolean equals(getVerTs_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_column = true && this.isSetColumn(); + boolean that_present_column = true && that.isSetColumn(); + if (this_present_column || that_present_column) { + if (!(this_present_column && that_present_column)) + return false; + if (!this.column.equals(that.column)) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_numVersions = true; + boolean that_present_numVersions = true; + if (this_present_numVersions || that_present_numVersions) { + if (!(this_present_numVersions && that_present_numVersions)) + return false; + if (this.numVersions != that.numVersions) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getVerTs_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getVerTs_args typedOther = (getVerTs_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumn()).compareTo(typedOther.isSetColumn()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumn()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, typedOther.column); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetNumVersions()).compareTo(typedOther.isSetNumVersions()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetNumVersions()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.numVersions, typedOther.numVersions); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getVerTs_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("column:"); + if (this.column == null) { + sb.append("null"); + } else { + sb.append(this.column); + } + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + if (!first) sb.append(", "); + sb.append("numVersions:"); + sb.append(this.numVersions); + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getVerTs_argsStandardSchemeFactory implements SchemeFactory { + public getVerTs_argsStandardScheme getScheme() { + return new getVerTs_argsStandardScheme(); + } + } + + private static class getVerTs_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getVerTs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMN + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // NUM_VERSIONS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.numVersions = iprot.readI32(); + struct.setNumVersionsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 6: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map96 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map96.size); + for (int _i97 = 0; _i97 < _map96.size; ++_i97) + { + ByteBuffer _key98; // required + ByteBuffer _val99; // optional + _key98 = iprot.readBinary(); + _val99 = iprot.readBinary(); + struct.attributes.put(_key98, _val99); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getVerTs_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.column != null) { + oprot.writeFieldBegin(COLUMN_FIELD_DESC); + oprot.writeBinary(struct.column); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(NUM_VERSIONS_FIELD_DESC); + oprot.writeI32(struct.numVersions); + oprot.writeFieldEnd(); + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter100 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter100.getKey()); + oprot.writeBinary(_iter100.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getVerTs_argsTupleSchemeFactory implements SchemeFactory { + public getVerTs_argsTupleScheme getScheme() { + return new getVerTs_argsTupleScheme(); + } + } + + private static class getVerTs_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getVerTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetColumn()) { + optionals.set(2); + } + if (struct.isSetTimestamp()) { + optionals.set(3); + } + if (struct.isSetNumVersions()) { + optionals.set(4); + } + if (struct.isSetAttributes()) { + optionals.set(5); + } + oprot.writeBitSet(optionals, 6); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetColumn()) { + oprot.writeBinary(struct.column); + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetNumVersions()) { + oprot.writeI32(struct.numVersions); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter101 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter101.getKey()); + oprot.writeBinary(_iter101.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getVerTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(6); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } + if (incoming.get(3)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(4)) { + struct.numVersions = iprot.readI32(); + struct.setNumVersionsIsSet(true); + } + if (incoming.get(5)) { + { + org.apache.thrift.protocol.TMap _map102 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map102.size); + for (int _i103 = 0; _i103 < _map102.size; ++_i103) + { + ByteBuffer _key104; // required + ByteBuffer _val105; // optional + _key104 = iprot.readBinary(); + _val105 = iprot.readBinary(); + struct.attributes.put(_key104, _val105); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class getVerTs_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getVerTs_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getVerTs_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getVerTs_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCell.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getVerTs_result.class, metaDataMap); + } + + public getVerTs_result() { + } + + public getVerTs_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getVerTs_result(getVerTs_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TCell other_element : other.success) { + __this__success.add(new TCell(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getVerTs_result deepCopy() { + return new getVerTs_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TCell elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getVerTs_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getVerTs_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getVerTs_result) + return this.equals((getVerTs_result)that); + return false; + } + + public boolean equals(getVerTs_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getVerTs_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getVerTs_result typedOther = (getVerTs_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getVerTs_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getVerTs_resultStandardSchemeFactory implements SchemeFactory { + public getVerTs_resultStandardScheme getScheme() { + return new getVerTs_resultStandardScheme(); + } + } + + private static class getVerTs_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getVerTs_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list106 = iprot.readListBegin(); + struct.success = new ArrayList(_list106.size); + for (int _i107 = 0; _i107 < _list106.size; ++_i107) + { + TCell _elem108; // optional + _elem108 = new TCell(); + _elem108.read(iprot); + struct.success.add(_elem108); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getVerTs_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TCell _iter109 : struct.success) + { + _iter109.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getVerTs_resultTupleSchemeFactory implements SchemeFactory { + public getVerTs_resultTupleScheme getScheme() { + return new getVerTs_resultTupleScheme(); + } + } + + private static class getVerTs_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getVerTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TCell _iter110 : struct.success) + { + _iter110.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getVerTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list111 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list111.size); + for (int _i112 = 0; _i112 < _list111.size; ++_i112) + { + TCell _elem113; // optional + _elem113 = new TCell(); + _elem113.read(iprot); + struct.success.add(_elem113); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getRow_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRow_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRow_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRow_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row key + */ + public ByteBuffer row; // required + /** + * Get attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row key + */ + ROW((short)2, "row"), + /** + * Get attributes + */ + ATTRIBUTES((short)3, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRow_args.class, metaDataMap); + } + + public getRow_args() { + } + + public getRow_args( + ByteBuffer tableName, + ByteBuffer row, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public getRow_args(getRow_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public getRow_args deepCopy() { + return new getRow_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public getRow_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getRow_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * row key + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row key + */ + public getRow_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public getRow_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Get attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Get attributes + */ + public getRow_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRow_args) + return this.equals((getRow_args)that); + return false; + } + + public boolean equals(getRow_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRow_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRow_args typedOther = (getRow_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRow_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRow_argsStandardSchemeFactory implements SchemeFactory { + public getRow_argsStandardScheme getScheme() { + return new getRow_argsStandardScheme(); + } + } + + private static class getRow_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRow_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map114 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map114.size); + for (int _i115 = 0; _i115 < _map114.size; ++_i115) + { + ByteBuffer _key116; // required + ByteBuffer _val117; // optional + _key116 = iprot.readBinary(); + _val117 = iprot.readBinary(); + struct.attributes.put(_key116, _val117); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRow_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter118 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter118.getKey()); + oprot.writeBinary(_iter118.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRow_argsTupleSchemeFactory implements SchemeFactory { + public getRow_argsTupleScheme getScheme() { + return new getRow_argsTupleScheme(); + } + } + + private static class getRow_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRow_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetAttributes()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter119 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter119.getKey()); + oprot.writeBinary(_iter119.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRow_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TMap _map120 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map120.size); + for (int _i121 = 0; _i121 < _map120.size; ++_i121) + { + ByteBuffer _key122; // required + ByteBuffer _val123; // optional + _key122 = iprot.readBinary(); + _val123 = iprot.readBinary(); + struct.attributes.put(_key122, _val123); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class getRow_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRow_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRow_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRow_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowResult.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRow_result.class, metaDataMap); + } + + public getRow_result() { + } + + public getRow_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getRow_result(getRow_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TRowResult other_element : other.success) { + __this__success.add(new TRowResult(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getRow_result deepCopy() { + return new getRow_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TRowResult elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getRow_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getRow_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRow_result) + return this.equals((getRow_result)that); + return false; + } + + public boolean equals(getRow_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRow_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRow_result typedOther = (getRow_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRow_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRow_resultStandardSchemeFactory implements SchemeFactory { + public getRow_resultStandardScheme getScheme() { + return new getRow_resultStandardScheme(); + } + } + + private static class getRow_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRow_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list124 = iprot.readListBegin(); + struct.success = new ArrayList(_list124.size); + for (int _i125 = 0; _i125 < _list124.size; ++_i125) + { + TRowResult _elem126; // optional + _elem126 = new TRowResult(); + _elem126.read(iprot); + struct.success.add(_elem126); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRow_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TRowResult _iter127 : struct.success) + { + _iter127.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRow_resultTupleSchemeFactory implements SchemeFactory { + public getRow_resultTupleScheme getScheme() { + return new getRow_resultTupleScheme(); + } + } + + private static class getRow_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRow_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TRowResult _iter128 : struct.success) + { + _iter128.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRow_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list129 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list129.size); + for (int _i130 = 0; _i130 < _list129.size; ++_i130) + { + TRowResult _elem131; // optional + _elem131 = new TRowResult(); + _elem131.read(iprot); + struct.success.add(_elem131); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getRowWithColumns_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowWithColumns_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowWithColumns_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowWithColumns_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row key + */ + public ByteBuffer row; // required + /** + * List of columns to return, null for all columns + */ + public List columns; // required + /** + * Get attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row key + */ + ROW((short)2, "row"), + /** + * List of columns to return, null for all columns + */ + COLUMNS((short)3, "columns"), + /** + * Get attributes + */ + ATTRIBUTES((short)4, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // COLUMNS + return COLUMNS; + case 4: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowWithColumns_args.class, metaDataMap); + } + + public getRowWithColumns_args() { + } + + public getRowWithColumns_args( + ByteBuffer tableName, + ByteBuffer row, + List columns, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.columns = columns; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public getRowWithColumns_args(getRowWithColumns_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (ByteBuffer other_element : other.columns) { + __this__columns.add(other_element); + } + this.columns = __this__columns; + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public getRowWithColumns_args deepCopy() { + return new getRowWithColumns_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.columns = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public getRowWithColumns_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getRowWithColumns_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * row key + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row key + */ + public getRowWithColumns_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public getRowWithColumns_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(ByteBuffer elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + /** + * List of columns to return, null for all columns + */ + public List getColumns() { + return this.columns; + } + + /** + * List of columns to return, null for all columns + */ + public getRowWithColumns_args setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Get attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Get attributes + */ + public getRowWithColumns_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case COLUMNS: + return getColumns(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case COLUMNS: + return isSetColumns(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowWithColumns_args) + return this.equals((getRowWithColumns_args)that); + return false; + } + + public boolean equals(getRowWithColumns_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowWithColumns_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowWithColumns_args typedOther = (getRowWithColumns_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowWithColumns_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowWithColumns_argsStandardSchemeFactory implements SchemeFactory { + public getRowWithColumns_argsStandardScheme getScheme() { + return new getRowWithColumns_argsStandardScheme(); + } + } + + private static class getRowWithColumns_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowWithColumns_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list132 = iprot.readListBegin(); + struct.columns = new ArrayList(_list132.size); + for (int _i133 = 0; _i133 < _list132.size; ++_i133) + { + ByteBuffer _elem134; // optional + _elem134 = iprot.readBinary(); + struct.columns.add(_elem134); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map135 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map135.size); + for (int _i136 = 0; _i136 < _map135.size; ++_i136) + { + ByteBuffer _key137; // required + ByteBuffer _val138; // optional + _key137 = iprot.readBinary(); + _val138 = iprot.readBinary(); + struct.attributes.put(_key137, _val138); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowWithColumns_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size())); + for (ByteBuffer _iter139 : struct.columns) + { + oprot.writeBinary(_iter139); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter140 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter140.getKey()); + oprot.writeBinary(_iter140.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowWithColumns_argsTupleSchemeFactory implements SchemeFactory { + public getRowWithColumns_argsTupleScheme getScheme() { + return new getRowWithColumns_argsTupleScheme(); + } + } + + private static class getRowWithColumns_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowWithColumns_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetColumns()) { + optionals.set(2); + } + if (struct.isSetAttributes()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (ByteBuffer _iter141 : struct.columns) + { + oprot.writeBinary(_iter141); + } + } + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter142 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter142.getKey()); + oprot.writeBinary(_iter142.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowWithColumns_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list143 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.columns = new ArrayList(_list143.size); + for (int _i144 = 0; _i144 < _list143.size; ++_i144) + { + ByteBuffer _elem145; // optional + _elem145 = iprot.readBinary(); + struct.columns.add(_elem145); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TMap _map146 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map146.size); + for (int _i147 = 0; _i147 < _map146.size; ++_i147) + { + ByteBuffer _key148; // required + ByteBuffer _val149; // optional + _key148 = iprot.readBinary(); + _val149 = iprot.readBinary(); + struct.attributes.put(_key148, _val149); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class getRowWithColumns_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowWithColumns_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowWithColumns_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowWithColumns_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowResult.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowWithColumns_result.class, metaDataMap); + } + + public getRowWithColumns_result() { + } + + public getRowWithColumns_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getRowWithColumns_result(getRowWithColumns_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TRowResult other_element : other.success) { + __this__success.add(new TRowResult(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getRowWithColumns_result deepCopy() { + return new getRowWithColumns_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TRowResult elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getRowWithColumns_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getRowWithColumns_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowWithColumns_result) + return this.equals((getRowWithColumns_result)that); + return false; + } + + public boolean equals(getRowWithColumns_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowWithColumns_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowWithColumns_result typedOther = (getRowWithColumns_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowWithColumns_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowWithColumns_resultStandardSchemeFactory implements SchemeFactory { + public getRowWithColumns_resultStandardScheme getScheme() { + return new getRowWithColumns_resultStandardScheme(); + } + } + + private static class getRowWithColumns_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowWithColumns_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list150 = iprot.readListBegin(); + struct.success = new ArrayList(_list150.size); + for (int _i151 = 0; _i151 < _list150.size; ++_i151) + { + TRowResult _elem152; // optional + _elem152 = new TRowResult(); + _elem152.read(iprot); + struct.success.add(_elem152); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowWithColumns_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TRowResult _iter153 : struct.success) + { + _iter153.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowWithColumns_resultTupleSchemeFactory implements SchemeFactory { + public getRowWithColumns_resultTupleScheme getScheme() { + return new getRowWithColumns_resultTupleScheme(); + } + } + + private static class getRowWithColumns_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowWithColumns_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TRowResult _iter154 : struct.success) + { + _iter154.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowWithColumns_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list155 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list155.size); + for (int _i156 = 0; _i156 < _list155.size; ++_i156) + { + TRowResult _elem157; // optional + _elem157 = new TRowResult(); + _elem157.read(iprot); + struct.success.add(_elem157); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getRowTs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowTs_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowTs_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowTs_argsTupleSchemeFactory()); + } + + /** + * name of the table + */ + public ByteBuffer tableName; // required + /** + * row key + */ + public ByteBuffer row; // required + /** + * timestamp + */ + public long timestamp; // required + /** + * Get attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of the table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row key + */ + ROW((short)2, "row"), + /** + * timestamp + */ + TIMESTAMP((short)3, "timestamp"), + /** + * Get attributes + */ + ATTRIBUTES((short)4, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // TIMESTAMP + return TIMESTAMP; + case 4: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowTs_args.class, metaDataMap); + } + + public getRowTs_args() { + } + + public getRowTs_args( + ByteBuffer tableName, + ByteBuffer row, + long timestamp, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.timestamp = timestamp; + setTimestampIsSet(true); + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public getRowTs_args(getRowTs_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + this.timestamp = other.timestamp; + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public getRowTs_args deepCopy() { + return new getRowTs_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.attributes = null; + } + + /** + * name of the table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of the table + */ + public getRowTs_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getRowTs_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * row key + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row key + */ + public getRowTs_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public getRowTs_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + /** + * timestamp + */ + public long getTimestamp() { + return this.timestamp; + } + + /** + * timestamp + */ + public getRowTs_args setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Get attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Get attributes + */ + public getRowTs_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case TIMESTAMP: + return isSetTimestamp(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowTs_args) + return this.equals((getRowTs_args)that); + return false; + } + + public boolean equals(getRowTs_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowTs_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowTs_args typedOther = (getRowTs_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowTs_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowTs_argsStandardSchemeFactory implements SchemeFactory { + public getRowTs_argsStandardScheme getScheme() { + return new getRowTs_argsStandardScheme(); + } + } + + private static class getRowTs_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowTs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map158 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map158.size); + for (int _i159 = 0; _i159 < _map158.size; ++_i159) + { + ByteBuffer _key160; // required + ByteBuffer _val161; // optional + _key160 = iprot.readBinary(); + _val161 = iprot.readBinary(); + struct.attributes.put(_key160, _val161); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowTs_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter162 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter162.getKey()); + oprot.writeBinary(_iter162.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowTs_argsTupleSchemeFactory implements SchemeFactory { + public getRowTs_argsTupleScheme getScheme() { + return new getRowTs_argsTupleScheme(); + } + } + + private static class getRowTs_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetTimestamp()) { + optionals.set(2); + } + if (struct.isSetAttributes()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter163 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter163.getKey()); + oprot.writeBinary(_iter163.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TMap _map164 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map164.size); + for (int _i165 = 0; _i165 < _map164.size; ++_i165) + { + ByteBuffer _key166; // required + ByteBuffer _val167; // optional + _key166 = iprot.readBinary(); + _val167 = iprot.readBinary(); + struct.attributes.put(_key166, _val167); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class getRowTs_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowTs_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowTs_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowTs_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowResult.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowTs_result.class, metaDataMap); + } + + public getRowTs_result() { + } + + public getRowTs_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getRowTs_result(getRowTs_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TRowResult other_element : other.success) { + __this__success.add(new TRowResult(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getRowTs_result deepCopy() { + return new getRowTs_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TRowResult elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getRowTs_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getRowTs_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowTs_result) + return this.equals((getRowTs_result)that); + return false; + } + + public boolean equals(getRowTs_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowTs_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowTs_result typedOther = (getRowTs_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowTs_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowTs_resultStandardSchemeFactory implements SchemeFactory { + public getRowTs_resultStandardScheme getScheme() { + return new getRowTs_resultStandardScheme(); + } + } + + private static class getRowTs_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowTs_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list168 = iprot.readListBegin(); + struct.success = new ArrayList(_list168.size); + for (int _i169 = 0; _i169 < _list168.size; ++_i169) + { + TRowResult _elem170; // optional + _elem170 = new TRowResult(); + _elem170.read(iprot); + struct.success.add(_elem170); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowTs_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TRowResult _iter171 : struct.success) + { + _iter171.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowTs_resultTupleSchemeFactory implements SchemeFactory { + public getRowTs_resultTupleScheme getScheme() { + return new getRowTs_resultTupleScheme(); + } + } + + private static class getRowTs_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TRowResult _iter172 : struct.success) + { + _iter172.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list173 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list173.size); + for (int _i174 = 0; _i174 < _list173.size; ++_i174) + { + TRowResult _elem175; // optional + _elem175 = new TRowResult(); + _elem175.read(iprot); + struct.success.add(_elem175); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getRowWithColumnsTs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowWithColumnsTs_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)4); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)5); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowWithColumnsTs_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowWithColumnsTs_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row key + */ + public ByteBuffer row; // required + /** + * List of columns to return, null for all columns + */ + public List columns; // required + public long timestamp; // required + /** + * Get attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row key + */ + ROW((short)2, "row"), + /** + * List of columns to return, null for all columns + */ + COLUMNS((short)3, "columns"), + TIMESTAMP((short)4, "timestamp"), + /** + * Get attributes + */ + ATTRIBUTES((short)5, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // COLUMNS + return COLUMNS; + case 4: // TIMESTAMP + return TIMESTAMP; + case 5: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowWithColumnsTs_args.class, metaDataMap); + } + + public getRowWithColumnsTs_args() { + } + + public getRowWithColumnsTs_args( + ByteBuffer tableName, + ByteBuffer row, + List columns, + long timestamp, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.columns = columns; + this.timestamp = timestamp; + setTimestampIsSet(true); + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public getRowWithColumnsTs_args(getRowWithColumnsTs_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (ByteBuffer other_element : other.columns) { + __this__columns.add(other_element); + } + this.columns = __this__columns; + } + this.timestamp = other.timestamp; + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public getRowWithColumnsTs_args deepCopy() { + return new getRowWithColumnsTs_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.columns = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public getRowWithColumnsTs_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getRowWithColumnsTs_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * row key + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row key + */ + public getRowWithColumnsTs_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public getRowWithColumnsTs_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(ByteBuffer elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + /** + * List of columns to return, null for all columns + */ + public List getColumns() { + return this.columns; + } + + /** + * List of columns to return, null for all columns + */ + public getRowWithColumnsTs_args setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public long getTimestamp() { + return this.timestamp; + } + + public getRowWithColumnsTs_args setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Get attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Get attributes + */ + public getRowWithColumnsTs_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case COLUMNS: + return getColumns(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case COLUMNS: + return isSetColumns(); + case TIMESTAMP: + return isSetTimestamp(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowWithColumnsTs_args) + return this.equals((getRowWithColumnsTs_args)that); + return false; + } + + public boolean equals(getRowWithColumnsTs_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowWithColumnsTs_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowWithColumnsTs_args typedOther = (getRowWithColumnsTs_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowWithColumnsTs_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowWithColumnsTs_argsStandardSchemeFactory implements SchemeFactory { + public getRowWithColumnsTs_argsStandardScheme getScheme() { + return new getRowWithColumnsTs_argsStandardScheme(); + } + } + + private static class getRowWithColumnsTs_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowWithColumnsTs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list176 = iprot.readListBegin(); + struct.columns = new ArrayList(_list176.size); + for (int _i177 = 0; _i177 < _list176.size; ++_i177) + { + ByteBuffer _elem178; // optional + _elem178 = iprot.readBinary(); + struct.columns.add(_elem178); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map179 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map179.size); + for (int _i180 = 0; _i180 < _map179.size; ++_i180) + { + ByteBuffer _key181; // required + ByteBuffer _val182; // optional + _key181 = iprot.readBinary(); + _val182 = iprot.readBinary(); + struct.attributes.put(_key181, _val182); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowWithColumnsTs_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size())); + for (ByteBuffer _iter183 : struct.columns) + { + oprot.writeBinary(_iter183); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter184 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter184.getKey()); + oprot.writeBinary(_iter184.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowWithColumnsTs_argsTupleSchemeFactory implements SchemeFactory { + public getRowWithColumnsTs_argsTupleScheme getScheme() { + return new getRowWithColumnsTs_argsTupleScheme(); + } + } + + private static class getRowWithColumnsTs_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowWithColumnsTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetColumns()) { + optionals.set(2); + } + if (struct.isSetTimestamp()) { + optionals.set(3); + } + if (struct.isSetAttributes()) { + optionals.set(4); + } + oprot.writeBitSet(optionals, 5); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (ByteBuffer _iter185 : struct.columns) + { + oprot.writeBinary(_iter185); + } + } + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter186 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter186.getKey()); + oprot.writeBinary(_iter186.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowWithColumnsTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(5); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list187 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.columns = new ArrayList(_list187.size); + for (int _i188 = 0; _i188 < _list187.size; ++_i188) + { + ByteBuffer _elem189; // optional + _elem189 = iprot.readBinary(); + struct.columns.add(_elem189); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(3)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(4)) { + { + org.apache.thrift.protocol.TMap _map190 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map190.size); + for (int _i191 = 0; _i191 < _map190.size; ++_i191) + { + ByteBuffer _key192; // required + ByteBuffer _val193; // optional + _key192 = iprot.readBinary(); + _val193 = iprot.readBinary(); + struct.attributes.put(_key192, _val193); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class getRowWithColumnsTs_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowWithColumnsTs_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowWithColumnsTs_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowWithColumnsTs_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowResult.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowWithColumnsTs_result.class, metaDataMap); + } + + public getRowWithColumnsTs_result() { + } + + public getRowWithColumnsTs_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getRowWithColumnsTs_result(getRowWithColumnsTs_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TRowResult other_element : other.success) { + __this__success.add(new TRowResult(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getRowWithColumnsTs_result deepCopy() { + return new getRowWithColumnsTs_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TRowResult elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getRowWithColumnsTs_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getRowWithColumnsTs_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowWithColumnsTs_result) + return this.equals((getRowWithColumnsTs_result)that); + return false; + } + + public boolean equals(getRowWithColumnsTs_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowWithColumnsTs_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowWithColumnsTs_result typedOther = (getRowWithColumnsTs_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowWithColumnsTs_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowWithColumnsTs_resultStandardSchemeFactory implements SchemeFactory { + public getRowWithColumnsTs_resultStandardScheme getScheme() { + return new getRowWithColumnsTs_resultStandardScheme(); + } + } + + private static class getRowWithColumnsTs_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowWithColumnsTs_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list194 = iprot.readListBegin(); + struct.success = new ArrayList(_list194.size); + for (int _i195 = 0; _i195 < _list194.size; ++_i195) + { + TRowResult _elem196; // optional + _elem196 = new TRowResult(); + _elem196.read(iprot); + struct.success.add(_elem196); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowWithColumnsTs_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TRowResult _iter197 : struct.success) + { + _iter197.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowWithColumnsTs_resultTupleSchemeFactory implements SchemeFactory { + public getRowWithColumnsTs_resultTupleScheme getScheme() { + return new getRowWithColumnsTs_resultTupleScheme(); + } + } + + private static class getRowWithColumnsTs_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowWithColumnsTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TRowResult _iter198 : struct.success) + { + _iter198.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowWithColumnsTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list199 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list199.size); + for (int _i200 = 0; _i200 < _list199.size; ++_i200) + { + TRowResult _elem201; // optional + _elem201 = new TRowResult(); + _elem201.read(iprot); + struct.success.add(_elem201); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getRows_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRows_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("rows", org.apache.thrift.protocol.TType.LIST, (short)2); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRows_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRows_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row keys + */ + public List rows; // required + /** + * Get attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row keys + */ + ROWS((short)2, "rows"), + /** + * Get attributes + */ + ATTRIBUTES((short)3, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROWS + return ROWS; + case 3: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROWS, new org.apache.thrift.meta_data.FieldMetaData("rows", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRows_args.class, metaDataMap); + } + + public getRows_args() { + } + + public getRows_args( + ByteBuffer tableName, + List rows, + Map attributes) + { + this(); + this.tableName = tableName; + this.rows = rows; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public getRows_args(getRows_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRows()) { + List __this__rows = new ArrayList(); + for (ByteBuffer other_element : other.rows) { + __this__rows.add(other_element); + } + this.rows = __this__rows; + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public getRows_args deepCopy() { + return new getRows_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.rows = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public getRows_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getRows_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public int getRowsSize() { + return (this.rows == null) ? 0 : this.rows.size(); + } + + public java.util.Iterator getRowsIterator() { + return (this.rows == null) ? null : this.rows.iterator(); + } + + public void addToRows(ByteBuffer elem) { + if (this.rows == null) { + this.rows = new ArrayList(); + } + this.rows.add(elem); + } + + /** + * row keys + */ + public List getRows() { + return this.rows; + } + + /** + * row keys + */ + public getRows_args setRows(List rows) { + this.rows = rows; + return this; + } + + public void unsetRows() { + this.rows = null; + } + + /** Returns true if field rows is set (has been assigned a value) and false otherwise */ + public boolean isSetRows() { + return this.rows != null; + } + + public void setRowsIsSet(boolean value) { + if (!value) { + this.rows = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Get attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Get attributes + */ + public getRows_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROWS: + if (value == null) { + unsetRows(); + } else { + setRows((List)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROWS: + return getRows(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROWS: + return isSetRows(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRows_args) + return this.equals((getRows_args)that); + return false; + } + + public boolean equals(getRows_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_rows = true && this.isSetRows(); + boolean that_present_rows = true && that.isSetRows(); + if (this_present_rows || that_present_rows) { + if (!(this_present_rows && that_present_rows)) + return false; + if (!this.rows.equals(that.rows)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRows_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRows_args typedOther = (getRows_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRows()).compareTo(typedOther.isSetRows()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRows()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rows, typedOther.rows); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRows_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("rows:"); + if (this.rows == null) { + sb.append("null"); + } else { + sb.append(this.rows); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRows_argsStandardSchemeFactory implements SchemeFactory { + public getRows_argsStandardScheme getScheme() { + return new getRows_argsStandardScheme(); + } + } + + private static class getRows_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRows_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROWS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list202 = iprot.readListBegin(); + struct.rows = new ArrayList(_list202.size); + for (int _i203 = 0; _i203 < _list202.size; ++_i203) + { + ByteBuffer _elem204; // optional + _elem204 = iprot.readBinary(); + struct.rows.add(_elem204); + } + iprot.readListEnd(); + } + struct.setRowsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map205 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map205.size); + for (int _i206 = 0; _i206 < _map205.size; ++_i206) + { + ByteBuffer _key207; // required + ByteBuffer _val208; // optional + _key207 = iprot.readBinary(); + _val208 = iprot.readBinary(); + struct.attributes.put(_key207, _val208); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRows_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.rows != null) { + oprot.writeFieldBegin(ROWS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.rows.size())); + for (ByteBuffer _iter209 : struct.rows) + { + oprot.writeBinary(_iter209); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter210 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter210.getKey()); + oprot.writeBinary(_iter210.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRows_argsTupleSchemeFactory implements SchemeFactory { + public getRows_argsTupleScheme getScheme() { + return new getRows_argsTupleScheme(); + } + } + + private static class getRows_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRows_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRows()) { + optionals.set(1); + } + if (struct.isSetAttributes()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRows()) { + { + oprot.writeI32(struct.rows.size()); + for (ByteBuffer _iter211 : struct.rows) + { + oprot.writeBinary(_iter211); + } + } + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter212 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter212.getKey()); + oprot.writeBinary(_iter212.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRows_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + { + org.apache.thrift.protocol.TList _list213 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.rows = new ArrayList(_list213.size); + for (int _i214 = 0; _i214 < _list213.size; ++_i214) + { + ByteBuffer _elem215; // optional + _elem215 = iprot.readBinary(); + struct.rows.add(_elem215); + } + } + struct.setRowsIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TMap _map216 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map216.size); + for (int _i217 = 0; _i217 < _map216.size; ++_i217) + { + ByteBuffer _key218; // required + ByteBuffer _val219; // optional + _key218 = iprot.readBinary(); + _val219 = iprot.readBinary(); + struct.attributes.put(_key218, _val219); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class getRows_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRows_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRows_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRows_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowResult.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRows_result.class, metaDataMap); + } + + public getRows_result() { + } + + public getRows_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getRows_result(getRows_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TRowResult other_element : other.success) { + __this__success.add(new TRowResult(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getRows_result deepCopy() { + return new getRows_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TRowResult elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getRows_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getRows_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRows_result) + return this.equals((getRows_result)that); + return false; + } + + public boolean equals(getRows_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRows_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRows_result typedOther = (getRows_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRows_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRows_resultStandardSchemeFactory implements SchemeFactory { + public getRows_resultStandardScheme getScheme() { + return new getRows_resultStandardScheme(); + } + } + + private static class getRows_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRows_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list220 = iprot.readListBegin(); + struct.success = new ArrayList(_list220.size); + for (int _i221 = 0; _i221 < _list220.size; ++_i221) + { + TRowResult _elem222; // optional + _elem222 = new TRowResult(); + _elem222.read(iprot); + struct.success.add(_elem222); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRows_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TRowResult _iter223 : struct.success) + { + _iter223.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRows_resultTupleSchemeFactory implements SchemeFactory { + public getRows_resultTupleScheme getScheme() { + return new getRows_resultTupleScheme(); + } + } + + private static class getRows_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRows_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TRowResult _iter224 : struct.success) + { + _iter224.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRows_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list225 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list225.size); + for (int _i226 = 0; _i226 < _list225.size; ++_i226) + { + TRowResult _elem227; // optional + _elem227 = new TRowResult(); + _elem227.read(iprot); + struct.success.add(_elem227); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getRowsWithColumns_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowsWithColumns_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("rows", org.apache.thrift.protocol.TType.LIST, (short)2); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowsWithColumns_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowsWithColumns_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row keys + */ + public List rows; // required + /** + * List of columns to return, null for all columns + */ + public List columns; // required + /** + * Get attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row keys + */ + ROWS((short)2, "rows"), + /** + * List of columns to return, null for all columns + */ + COLUMNS((short)3, "columns"), + /** + * Get attributes + */ + ATTRIBUTES((short)4, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROWS + return ROWS; + case 3: // COLUMNS + return COLUMNS; + case 4: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROWS, new org.apache.thrift.meta_data.FieldMetaData("rows", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowsWithColumns_args.class, metaDataMap); + } + + public getRowsWithColumns_args() { + } + + public getRowsWithColumns_args( + ByteBuffer tableName, + List rows, + List columns, + Map attributes) + { + this(); + this.tableName = tableName; + this.rows = rows; + this.columns = columns; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public getRowsWithColumns_args(getRowsWithColumns_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRows()) { + List __this__rows = new ArrayList(); + for (ByteBuffer other_element : other.rows) { + __this__rows.add(other_element); + } + this.rows = __this__rows; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (ByteBuffer other_element : other.columns) { + __this__columns.add(other_element); + } + this.columns = __this__columns; + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public getRowsWithColumns_args deepCopy() { + return new getRowsWithColumns_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.rows = null; + this.columns = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public getRowsWithColumns_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getRowsWithColumns_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public int getRowsSize() { + return (this.rows == null) ? 0 : this.rows.size(); + } + + public java.util.Iterator getRowsIterator() { + return (this.rows == null) ? null : this.rows.iterator(); + } + + public void addToRows(ByteBuffer elem) { + if (this.rows == null) { + this.rows = new ArrayList(); + } + this.rows.add(elem); + } + + /** + * row keys + */ + public List getRows() { + return this.rows; + } + + /** + * row keys + */ + public getRowsWithColumns_args setRows(List rows) { + this.rows = rows; + return this; + } + + public void unsetRows() { + this.rows = null; + } + + /** Returns true if field rows is set (has been assigned a value) and false otherwise */ + public boolean isSetRows() { + return this.rows != null; + } + + public void setRowsIsSet(boolean value) { + if (!value) { + this.rows = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(ByteBuffer elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + /** + * List of columns to return, null for all columns + */ + public List getColumns() { + return this.columns; + } + + /** + * List of columns to return, null for all columns + */ + public getRowsWithColumns_args setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Get attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Get attributes + */ + public getRowsWithColumns_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROWS: + if (value == null) { + unsetRows(); + } else { + setRows((List)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROWS: + return getRows(); + + case COLUMNS: + return getColumns(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROWS: + return isSetRows(); + case COLUMNS: + return isSetColumns(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowsWithColumns_args) + return this.equals((getRowsWithColumns_args)that); + return false; + } + + public boolean equals(getRowsWithColumns_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_rows = true && this.isSetRows(); + boolean that_present_rows = true && that.isSetRows(); + if (this_present_rows || that_present_rows) { + if (!(this_present_rows && that_present_rows)) + return false; + if (!this.rows.equals(that.rows)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowsWithColumns_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowsWithColumns_args typedOther = (getRowsWithColumns_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRows()).compareTo(typedOther.isSetRows()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRows()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rows, typedOther.rows); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowsWithColumns_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("rows:"); + if (this.rows == null) { + sb.append("null"); + } else { + sb.append(this.rows); + } + first = false; + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowsWithColumns_argsStandardSchemeFactory implements SchemeFactory { + public getRowsWithColumns_argsStandardScheme getScheme() { + return new getRowsWithColumns_argsStandardScheme(); + } + } + + private static class getRowsWithColumns_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumns_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROWS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list228 = iprot.readListBegin(); + struct.rows = new ArrayList(_list228.size); + for (int _i229 = 0; _i229 < _list228.size; ++_i229) + { + ByteBuffer _elem230; // optional + _elem230 = iprot.readBinary(); + struct.rows.add(_elem230); + } + iprot.readListEnd(); + } + struct.setRowsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list231 = iprot.readListBegin(); + struct.columns = new ArrayList(_list231.size); + for (int _i232 = 0; _i232 < _list231.size; ++_i232) + { + ByteBuffer _elem233; // optional + _elem233 = iprot.readBinary(); + struct.columns.add(_elem233); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map234 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map234.size); + for (int _i235 = 0; _i235 < _map234.size; ++_i235) + { + ByteBuffer _key236; // required + ByteBuffer _val237; // optional + _key236 = iprot.readBinary(); + _val237 = iprot.readBinary(); + struct.attributes.put(_key236, _val237); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsWithColumns_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.rows != null) { + oprot.writeFieldBegin(ROWS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.rows.size())); + for (ByteBuffer _iter238 : struct.rows) + { + oprot.writeBinary(_iter238); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size())); + for (ByteBuffer _iter239 : struct.columns) + { + oprot.writeBinary(_iter239); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter240 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter240.getKey()); + oprot.writeBinary(_iter240.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowsWithColumns_argsTupleSchemeFactory implements SchemeFactory { + public getRowsWithColumns_argsTupleScheme getScheme() { + return new getRowsWithColumns_argsTupleScheme(); + } + } + + private static class getRowsWithColumns_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumns_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRows()) { + optionals.set(1); + } + if (struct.isSetColumns()) { + optionals.set(2); + } + if (struct.isSetAttributes()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRows()) { + { + oprot.writeI32(struct.rows.size()); + for (ByteBuffer _iter241 : struct.rows) + { + oprot.writeBinary(_iter241); + } + } + } + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (ByteBuffer _iter242 : struct.columns) + { + oprot.writeBinary(_iter242); + } + } + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter243 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter243.getKey()); + oprot.writeBinary(_iter243.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumns_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + { + org.apache.thrift.protocol.TList _list244 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.rows = new ArrayList(_list244.size); + for (int _i245 = 0; _i245 < _list244.size; ++_i245) + { + ByteBuffer _elem246; // optional + _elem246 = iprot.readBinary(); + struct.rows.add(_elem246); + } + } + struct.setRowsIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list247 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.columns = new ArrayList(_list247.size); + for (int _i248 = 0; _i248 < _list247.size; ++_i248) + { + ByteBuffer _elem249; // optional + _elem249 = iprot.readBinary(); + struct.columns.add(_elem249); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TMap _map250 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map250.size); + for (int _i251 = 0; _i251 < _map250.size; ++_i251) + { + ByteBuffer _key252; // required + ByteBuffer _val253; // optional + _key252 = iprot.readBinary(); + _val253 = iprot.readBinary(); + struct.attributes.put(_key252, _val253); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class getRowsWithColumns_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowsWithColumns_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowsWithColumns_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowsWithColumns_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowResult.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowsWithColumns_result.class, metaDataMap); + } + + public getRowsWithColumns_result() { + } + + public getRowsWithColumns_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getRowsWithColumns_result(getRowsWithColumns_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TRowResult other_element : other.success) { + __this__success.add(new TRowResult(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getRowsWithColumns_result deepCopy() { + return new getRowsWithColumns_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TRowResult elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getRowsWithColumns_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getRowsWithColumns_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowsWithColumns_result) + return this.equals((getRowsWithColumns_result)that); + return false; + } + + public boolean equals(getRowsWithColumns_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowsWithColumns_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowsWithColumns_result typedOther = (getRowsWithColumns_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowsWithColumns_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowsWithColumns_resultStandardSchemeFactory implements SchemeFactory { + public getRowsWithColumns_resultStandardScheme getScheme() { + return new getRowsWithColumns_resultStandardScheme(); + } + } + + private static class getRowsWithColumns_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumns_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list254 = iprot.readListBegin(); + struct.success = new ArrayList(_list254.size); + for (int _i255 = 0; _i255 < _list254.size; ++_i255) + { + TRowResult _elem256; // optional + _elem256 = new TRowResult(); + _elem256.read(iprot); + struct.success.add(_elem256); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsWithColumns_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TRowResult _iter257 : struct.success) + { + _iter257.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowsWithColumns_resultTupleSchemeFactory implements SchemeFactory { + public getRowsWithColumns_resultTupleScheme getScheme() { + return new getRowsWithColumns_resultTupleScheme(); + } + } + + private static class getRowsWithColumns_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumns_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TRowResult _iter258 : struct.success) + { + _iter258.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumns_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list259 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list259.size); + for (int _i260 = 0; _i260 < _list259.size; ++_i260) + { + TRowResult _elem261; // optional + _elem261 = new TRowResult(); + _elem261.read(iprot); + struct.success.add(_elem261); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getRowsTs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowsTs_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("rows", org.apache.thrift.protocol.TType.LIST, (short)2); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowsTs_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowsTs_argsTupleSchemeFactory()); + } + + /** + * name of the table + */ + public ByteBuffer tableName; // required + /** + * row keys + */ + public List rows; // required + /** + * timestamp + */ + public long timestamp; // required + /** + * Get attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of the table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row keys + */ + ROWS((short)2, "rows"), + /** + * timestamp + */ + TIMESTAMP((short)3, "timestamp"), + /** + * Get attributes + */ + ATTRIBUTES((short)4, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROWS + return ROWS; + case 3: // TIMESTAMP + return TIMESTAMP; + case 4: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROWS, new org.apache.thrift.meta_data.FieldMetaData("rows", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowsTs_args.class, metaDataMap); + } + + public getRowsTs_args() { + } + + public getRowsTs_args( + ByteBuffer tableName, + List rows, + long timestamp, + Map attributes) + { + this(); + this.tableName = tableName; + this.rows = rows; + this.timestamp = timestamp; + setTimestampIsSet(true); + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public getRowsTs_args(getRowsTs_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRows()) { + List __this__rows = new ArrayList(); + for (ByteBuffer other_element : other.rows) { + __this__rows.add(other_element); + } + this.rows = __this__rows; + } + this.timestamp = other.timestamp; + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public getRowsTs_args deepCopy() { + return new getRowsTs_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.rows = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.attributes = null; + } + + /** + * name of the table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of the table + */ + public getRowsTs_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getRowsTs_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public int getRowsSize() { + return (this.rows == null) ? 0 : this.rows.size(); + } + + public java.util.Iterator getRowsIterator() { + return (this.rows == null) ? null : this.rows.iterator(); + } + + public void addToRows(ByteBuffer elem) { + if (this.rows == null) { + this.rows = new ArrayList(); + } + this.rows.add(elem); + } + + /** + * row keys + */ + public List getRows() { + return this.rows; + } + + /** + * row keys + */ + public getRowsTs_args setRows(List rows) { + this.rows = rows; + return this; + } + + public void unsetRows() { + this.rows = null; + } + + /** Returns true if field rows is set (has been assigned a value) and false otherwise */ + public boolean isSetRows() { + return this.rows != null; + } + + public void setRowsIsSet(boolean value) { + if (!value) { + this.rows = null; + } + } + + /** + * timestamp + */ + public long getTimestamp() { + return this.timestamp; + } + + /** + * timestamp + */ + public getRowsTs_args setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Get attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Get attributes + */ + public getRowsTs_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROWS: + if (value == null) { + unsetRows(); + } else { + setRows((List)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROWS: + return getRows(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROWS: + return isSetRows(); + case TIMESTAMP: + return isSetTimestamp(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowsTs_args) + return this.equals((getRowsTs_args)that); + return false; + } + + public boolean equals(getRowsTs_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_rows = true && this.isSetRows(); + boolean that_present_rows = true && that.isSetRows(); + if (this_present_rows || that_present_rows) { + if (!(this_present_rows && that_present_rows)) + return false; + if (!this.rows.equals(that.rows)) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowsTs_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowsTs_args typedOther = (getRowsTs_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRows()).compareTo(typedOther.isSetRows()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRows()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rows, typedOther.rows); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowsTs_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("rows:"); + if (this.rows == null) { + sb.append("null"); + } else { + sb.append(this.rows); + } + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowsTs_argsStandardSchemeFactory implements SchemeFactory { + public getRowsTs_argsStandardScheme getScheme() { + return new getRowsTs_argsStandardScheme(); + } + } + + private static class getRowsTs_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsTs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROWS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list262 = iprot.readListBegin(); + struct.rows = new ArrayList(_list262.size); + for (int _i263 = 0; _i263 < _list262.size; ++_i263) + { + ByteBuffer _elem264; // optional + _elem264 = iprot.readBinary(); + struct.rows.add(_elem264); + } + iprot.readListEnd(); + } + struct.setRowsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map265 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map265.size); + for (int _i266 = 0; _i266 < _map265.size; ++_i266) + { + ByteBuffer _key267; // required + ByteBuffer _val268; // optional + _key267 = iprot.readBinary(); + _val268 = iprot.readBinary(); + struct.attributes.put(_key267, _val268); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsTs_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.rows != null) { + oprot.writeFieldBegin(ROWS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.rows.size())); + for (ByteBuffer _iter269 : struct.rows) + { + oprot.writeBinary(_iter269); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter270 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter270.getKey()); + oprot.writeBinary(_iter270.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowsTs_argsTupleSchemeFactory implements SchemeFactory { + public getRowsTs_argsTupleScheme getScheme() { + return new getRowsTs_argsTupleScheme(); + } + } + + private static class getRowsTs_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowsTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRows()) { + optionals.set(1); + } + if (struct.isSetTimestamp()) { + optionals.set(2); + } + if (struct.isSetAttributes()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRows()) { + { + oprot.writeI32(struct.rows.size()); + for (ByteBuffer _iter271 : struct.rows) + { + oprot.writeBinary(_iter271); + } + } + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter272 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter272.getKey()); + oprot.writeBinary(_iter272.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowsTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + { + org.apache.thrift.protocol.TList _list273 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.rows = new ArrayList(_list273.size); + for (int _i274 = 0; _i274 < _list273.size; ++_i274) + { + ByteBuffer _elem275; // optional + _elem275 = iprot.readBinary(); + struct.rows.add(_elem275); + } + } + struct.setRowsIsSet(true); + } + if (incoming.get(2)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TMap _map276 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map276.size); + for (int _i277 = 0; _i277 < _map276.size; ++_i277) + { + ByteBuffer _key278; // required + ByteBuffer _val279; // optional + _key278 = iprot.readBinary(); + _val279 = iprot.readBinary(); + struct.attributes.put(_key278, _val279); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class getRowsTs_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowsTs_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowsTs_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowsTs_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowResult.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowsTs_result.class, metaDataMap); + } + + public getRowsTs_result() { + } + + public getRowsTs_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getRowsTs_result(getRowsTs_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TRowResult other_element : other.success) { + __this__success.add(new TRowResult(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getRowsTs_result deepCopy() { + return new getRowsTs_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TRowResult elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getRowsTs_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getRowsTs_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowsTs_result) + return this.equals((getRowsTs_result)that); + return false; + } + + public boolean equals(getRowsTs_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowsTs_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowsTs_result typedOther = (getRowsTs_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowsTs_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowsTs_resultStandardSchemeFactory implements SchemeFactory { + public getRowsTs_resultStandardScheme getScheme() { + return new getRowsTs_resultStandardScheme(); + } + } + + private static class getRowsTs_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsTs_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list280 = iprot.readListBegin(); + struct.success = new ArrayList(_list280.size); + for (int _i281 = 0; _i281 < _list280.size; ++_i281) + { + TRowResult _elem282; // optional + _elem282 = new TRowResult(); + _elem282.read(iprot); + struct.success.add(_elem282); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsTs_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TRowResult _iter283 : struct.success) + { + _iter283.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowsTs_resultTupleSchemeFactory implements SchemeFactory { + public getRowsTs_resultTupleScheme getScheme() { + return new getRowsTs_resultTupleScheme(); + } + } + + private static class getRowsTs_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowsTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TRowResult _iter284 : struct.success) + { + _iter284.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowsTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list285 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list285.size); + for (int _i286 = 0; _i286 < _list285.size; ++_i286) + { + TRowResult _elem287; // optional + _elem287 = new TRowResult(); + _elem287.read(iprot); + struct.success.add(_elem287); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getRowsWithColumnsTs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowsWithColumnsTs_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("rows", org.apache.thrift.protocol.TType.LIST, (short)2); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)4); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)5); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowsWithColumnsTs_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowsWithColumnsTs_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row keys + */ + public List rows; // required + /** + * List of columns to return, null for all columns + */ + public List columns; // required + public long timestamp; // required + /** + * Get attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row keys + */ + ROWS((short)2, "rows"), + /** + * List of columns to return, null for all columns + */ + COLUMNS((short)3, "columns"), + TIMESTAMP((short)4, "timestamp"), + /** + * Get attributes + */ + ATTRIBUTES((short)5, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROWS + return ROWS; + case 3: // COLUMNS + return COLUMNS; + case 4: // TIMESTAMP + return TIMESTAMP; + case 5: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROWS, new org.apache.thrift.meta_data.FieldMetaData("rows", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowsWithColumnsTs_args.class, metaDataMap); + } + + public getRowsWithColumnsTs_args() { + } + + public getRowsWithColumnsTs_args( + ByteBuffer tableName, + List rows, + List columns, + long timestamp, + Map attributes) + { + this(); + this.tableName = tableName; + this.rows = rows; + this.columns = columns; + this.timestamp = timestamp; + setTimestampIsSet(true); + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public getRowsWithColumnsTs_args(getRowsWithColumnsTs_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRows()) { + List __this__rows = new ArrayList(); + for (ByteBuffer other_element : other.rows) { + __this__rows.add(other_element); + } + this.rows = __this__rows; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (ByteBuffer other_element : other.columns) { + __this__columns.add(other_element); + } + this.columns = __this__columns; + } + this.timestamp = other.timestamp; + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public getRowsWithColumnsTs_args deepCopy() { + return new getRowsWithColumnsTs_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.rows = null; + this.columns = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public getRowsWithColumnsTs_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getRowsWithColumnsTs_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public int getRowsSize() { + return (this.rows == null) ? 0 : this.rows.size(); + } + + public java.util.Iterator getRowsIterator() { + return (this.rows == null) ? null : this.rows.iterator(); + } + + public void addToRows(ByteBuffer elem) { + if (this.rows == null) { + this.rows = new ArrayList(); + } + this.rows.add(elem); + } + + /** + * row keys + */ + public List getRows() { + return this.rows; + } + + /** + * row keys + */ + public getRowsWithColumnsTs_args setRows(List rows) { + this.rows = rows; + return this; + } + + public void unsetRows() { + this.rows = null; + } + + /** Returns true if field rows is set (has been assigned a value) and false otherwise */ + public boolean isSetRows() { + return this.rows != null; + } + + public void setRowsIsSet(boolean value) { + if (!value) { + this.rows = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(ByteBuffer elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + /** + * List of columns to return, null for all columns + */ + public List getColumns() { + return this.columns; + } + + /** + * List of columns to return, null for all columns + */ + public getRowsWithColumnsTs_args setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public long getTimestamp() { + return this.timestamp; + } + + public getRowsWithColumnsTs_args setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Get attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Get attributes + */ + public getRowsWithColumnsTs_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROWS: + if (value == null) { + unsetRows(); + } else { + setRows((List)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROWS: + return getRows(); + + case COLUMNS: + return getColumns(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROWS: + return isSetRows(); + case COLUMNS: + return isSetColumns(); + case TIMESTAMP: + return isSetTimestamp(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowsWithColumnsTs_args) + return this.equals((getRowsWithColumnsTs_args)that); + return false; + } + + public boolean equals(getRowsWithColumnsTs_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_rows = true && this.isSetRows(); + boolean that_present_rows = true && that.isSetRows(); + if (this_present_rows || that_present_rows) { + if (!(this_present_rows && that_present_rows)) + return false; + if (!this.rows.equals(that.rows)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowsWithColumnsTs_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowsWithColumnsTs_args typedOther = (getRowsWithColumnsTs_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRows()).compareTo(typedOther.isSetRows()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRows()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rows, typedOther.rows); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowsWithColumnsTs_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("rows:"); + if (this.rows == null) { + sb.append("null"); + } else { + sb.append(this.rows); + } + first = false; + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowsWithColumnsTs_argsStandardSchemeFactory implements SchemeFactory { + public getRowsWithColumnsTs_argsStandardScheme getScheme() { + return new getRowsWithColumnsTs_argsStandardScheme(); + } + } + + private static class getRowsWithColumnsTs_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumnsTs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROWS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list288 = iprot.readListBegin(); + struct.rows = new ArrayList(_list288.size); + for (int _i289 = 0; _i289 < _list288.size; ++_i289) + { + ByteBuffer _elem290; // optional + _elem290 = iprot.readBinary(); + struct.rows.add(_elem290); + } + iprot.readListEnd(); + } + struct.setRowsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list291 = iprot.readListBegin(); + struct.columns = new ArrayList(_list291.size); + for (int _i292 = 0; _i292 < _list291.size; ++_i292) + { + ByteBuffer _elem293; // optional + _elem293 = iprot.readBinary(); + struct.columns.add(_elem293); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map294 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map294.size); + for (int _i295 = 0; _i295 < _map294.size; ++_i295) + { + ByteBuffer _key296; // required + ByteBuffer _val297; // optional + _key296 = iprot.readBinary(); + _val297 = iprot.readBinary(); + struct.attributes.put(_key296, _val297); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsWithColumnsTs_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.rows != null) { + oprot.writeFieldBegin(ROWS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.rows.size())); + for (ByteBuffer _iter298 : struct.rows) + { + oprot.writeBinary(_iter298); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size())); + for (ByteBuffer _iter299 : struct.columns) + { + oprot.writeBinary(_iter299); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter300 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter300.getKey()); + oprot.writeBinary(_iter300.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowsWithColumnsTs_argsTupleSchemeFactory implements SchemeFactory { + public getRowsWithColumnsTs_argsTupleScheme getScheme() { + return new getRowsWithColumnsTs_argsTupleScheme(); + } + } + + private static class getRowsWithColumnsTs_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumnsTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRows()) { + optionals.set(1); + } + if (struct.isSetColumns()) { + optionals.set(2); + } + if (struct.isSetTimestamp()) { + optionals.set(3); + } + if (struct.isSetAttributes()) { + optionals.set(4); + } + oprot.writeBitSet(optionals, 5); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRows()) { + { + oprot.writeI32(struct.rows.size()); + for (ByteBuffer _iter301 : struct.rows) + { + oprot.writeBinary(_iter301); + } + } + } + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (ByteBuffer _iter302 : struct.columns) + { + oprot.writeBinary(_iter302); + } + } + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter303 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter303.getKey()); + oprot.writeBinary(_iter303.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumnsTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(5); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + { + org.apache.thrift.protocol.TList _list304 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.rows = new ArrayList(_list304.size); + for (int _i305 = 0; _i305 < _list304.size; ++_i305) + { + ByteBuffer _elem306; // optional + _elem306 = iprot.readBinary(); + struct.rows.add(_elem306); + } + } + struct.setRowsIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list307 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.columns = new ArrayList(_list307.size); + for (int _i308 = 0; _i308 < _list307.size; ++_i308) + { + ByteBuffer _elem309; // optional + _elem309 = iprot.readBinary(); + struct.columns.add(_elem309); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(3)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(4)) { + { + org.apache.thrift.protocol.TMap _map310 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map310.size); + for (int _i311 = 0; _i311 < _map310.size; ++_i311) + { + ByteBuffer _key312; // required + ByteBuffer _val313; // optional + _key312 = iprot.readBinary(); + _val313 = iprot.readBinary(); + struct.attributes.put(_key312, _val313); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class getRowsWithColumnsTs_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowsWithColumnsTs_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowsWithColumnsTs_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowsWithColumnsTs_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowResult.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowsWithColumnsTs_result.class, metaDataMap); + } + + public getRowsWithColumnsTs_result() { + } + + public getRowsWithColumnsTs_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getRowsWithColumnsTs_result(getRowsWithColumnsTs_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TRowResult other_element : other.success) { + __this__success.add(new TRowResult(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getRowsWithColumnsTs_result deepCopy() { + return new getRowsWithColumnsTs_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TRowResult elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getRowsWithColumnsTs_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getRowsWithColumnsTs_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowsWithColumnsTs_result) + return this.equals((getRowsWithColumnsTs_result)that); + return false; + } + + public boolean equals(getRowsWithColumnsTs_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowsWithColumnsTs_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowsWithColumnsTs_result typedOther = (getRowsWithColumnsTs_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowsWithColumnsTs_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowsWithColumnsTs_resultStandardSchemeFactory implements SchemeFactory { + public getRowsWithColumnsTs_resultStandardScheme getScheme() { + return new getRowsWithColumnsTs_resultStandardScheme(); + } + } + + private static class getRowsWithColumnsTs_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowsWithColumnsTs_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list314 = iprot.readListBegin(); + struct.success = new ArrayList(_list314.size); + for (int _i315 = 0; _i315 < _list314.size; ++_i315) + { + TRowResult _elem316; // optional + _elem316 = new TRowResult(); + _elem316.read(iprot); + struct.success.add(_elem316); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowsWithColumnsTs_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TRowResult _iter317 : struct.success) + { + _iter317.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowsWithColumnsTs_resultTupleSchemeFactory implements SchemeFactory { + public getRowsWithColumnsTs_resultTupleScheme getScheme() { + return new getRowsWithColumnsTs_resultTupleScheme(); + } + } + + private static class getRowsWithColumnsTs_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumnsTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TRowResult _iter318 : struct.success) + { + _iter318.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowsWithColumnsTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list319 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list319.size); + for (int _i320 = 0; _i320 < _list319.size; ++_i320) + { + TRowResult _elem321; // optional + _elem321 = new TRowResult(); + _elem321.read(iprot); + struct.success.add(_elem321); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class mutateRow_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("mutateRow_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField MUTATIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("mutations", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new mutateRow_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new mutateRow_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row key + */ + public ByteBuffer row; // required + /** + * list of mutation commands + */ + public List mutations; // required + /** + * Mutation attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row key + */ + ROW((short)2, "row"), + /** + * list of mutation commands + */ + MUTATIONS((short)3, "mutations"), + /** + * Mutation attributes + */ + ATTRIBUTES((short)4, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // MUTATIONS + return MUTATIONS; + case 4: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.MUTATIONS, new org.apache.thrift.meta_data.FieldMetaData("mutations", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Mutation.class)))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(mutateRow_args.class, metaDataMap); + } + + public mutateRow_args() { + } + + public mutateRow_args( + ByteBuffer tableName, + ByteBuffer row, + List mutations, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.mutations = mutations; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public mutateRow_args(mutateRow_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetMutations()) { + List __this__mutations = new ArrayList(); + for (Mutation other_element : other.mutations) { + __this__mutations.add(new Mutation(other_element)); + } + this.mutations = __this__mutations; + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public mutateRow_args deepCopy() { + return new mutateRow_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.mutations = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public mutateRow_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public mutateRow_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * row key + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row key + */ + public mutateRow_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public mutateRow_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getMutationsSize() { + return (this.mutations == null) ? 0 : this.mutations.size(); + } + + public java.util.Iterator getMutationsIterator() { + return (this.mutations == null) ? null : this.mutations.iterator(); + } + + public void addToMutations(Mutation elem) { + if (this.mutations == null) { + this.mutations = new ArrayList(); + } + this.mutations.add(elem); + } + + /** + * list of mutation commands + */ + public List getMutations() { + return this.mutations; + } + + /** + * list of mutation commands + */ + public mutateRow_args setMutations(List mutations) { + this.mutations = mutations; + return this; + } + + public void unsetMutations() { + this.mutations = null; + } + + /** Returns true if field mutations is set (has been assigned a value) and false otherwise */ + public boolean isSetMutations() { + return this.mutations != null; + } + + public void setMutationsIsSet(boolean value) { + if (!value) { + this.mutations = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Mutation attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Mutation attributes + */ + public mutateRow_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case MUTATIONS: + if (value == null) { + unsetMutations(); + } else { + setMutations((List)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case MUTATIONS: + return getMutations(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case MUTATIONS: + return isSetMutations(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof mutateRow_args) + return this.equals((mutateRow_args)that); + return false; + } + + public boolean equals(mutateRow_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_mutations = true && this.isSetMutations(); + boolean that_present_mutations = true && that.isSetMutations(); + if (this_present_mutations || that_present_mutations) { + if (!(this_present_mutations && that_present_mutations)) + return false; + if (!this.mutations.equals(that.mutations)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(mutateRow_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + mutateRow_args typedOther = (mutateRow_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMutations()).compareTo(typedOther.isSetMutations()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMutations()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.mutations, typedOther.mutations); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("mutateRow_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("mutations:"); + if (this.mutations == null) { + sb.append("null"); + } else { + sb.append(this.mutations); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class mutateRow_argsStandardSchemeFactory implements SchemeFactory { + public mutateRow_argsStandardScheme getScheme() { + return new mutateRow_argsStandardScheme(); + } + } + + private static class mutateRow_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRow_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // MUTATIONS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list322 = iprot.readListBegin(); + struct.mutations = new ArrayList(_list322.size); + for (int _i323 = 0; _i323 < _list322.size; ++_i323) + { + Mutation _elem324; // optional + _elem324 = new Mutation(); + _elem324.read(iprot); + struct.mutations.add(_elem324); + } + iprot.readListEnd(); + } + struct.setMutationsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map325 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map325.size); + for (int _i326 = 0; _i326 < _map325.size; ++_i326) + { + ByteBuffer _key327; // required + ByteBuffer _val328; // optional + _key327 = iprot.readBinary(); + _val328 = iprot.readBinary(); + struct.attributes.put(_key327, _val328); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRow_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.mutations != null) { + oprot.writeFieldBegin(MUTATIONS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.mutations.size())); + for (Mutation _iter329 : struct.mutations) + { + _iter329.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter330 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter330.getKey()); + oprot.writeBinary(_iter330.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class mutateRow_argsTupleSchemeFactory implements SchemeFactory { + public mutateRow_argsTupleScheme getScheme() { + return new mutateRow_argsTupleScheme(); + } + } + + private static class mutateRow_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, mutateRow_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetMutations()) { + optionals.set(2); + } + if (struct.isSetAttributes()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetMutations()) { + { + oprot.writeI32(struct.mutations.size()); + for (Mutation _iter331 : struct.mutations) + { + _iter331.write(oprot); + } + } + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter332 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter332.getKey()); + oprot.writeBinary(_iter332.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, mutateRow_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list333 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.mutations = new ArrayList(_list333.size); + for (int _i334 = 0; _i334 < _list333.size; ++_i334) + { + Mutation _elem335; // optional + _elem335 = new Mutation(); + _elem335.read(iprot); + struct.mutations.add(_elem335); + } + } + struct.setMutationsIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TMap _map336 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map336.size); + for (int _i337 = 0; _i337 < _map336.size; ++_i337) + { + ByteBuffer _key338; // required + ByteBuffer _val339; // optional + _key338 = iprot.readBinary(); + _val339 = iprot.readBinary(); + struct.attributes.put(_key338, _val339); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class mutateRow_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("mutateRow_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField IA_FIELD_DESC = new org.apache.thrift.protocol.TField("ia", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new mutateRow_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new mutateRow_resultTupleSchemeFactory()); + } + + public IOError io; // required + public IllegalArgument ia; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"), + IA((short)2, "ia"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + case 2: // IA + return IA; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + tmpMap.put(_Fields.IA, new org.apache.thrift.meta_data.FieldMetaData("ia", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(mutateRow_result.class, metaDataMap); + } + + public mutateRow_result() { + } + + public mutateRow_result( + IOError io, + IllegalArgument ia) + { + this(); + this.io = io; + this.ia = ia; + } + + /** + * Performs a deep copy on other. + */ + public mutateRow_result(mutateRow_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + if (other.isSetIa()) { + this.ia = new IllegalArgument(other.ia); + } + } + + public mutateRow_result deepCopy() { + return new mutateRow_result(this); + } + + @Override + public void clear() { + this.io = null; + this.ia = null; + } + + public IOError getIo() { + return this.io; + } + + public mutateRow_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public IllegalArgument getIa() { + return this.ia; + } + + public mutateRow_result setIa(IllegalArgument ia) { + this.ia = ia; + return this; + } + + public void unsetIa() { + this.ia = null; + } + + /** Returns true if field ia is set (has been assigned a value) and false otherwise */ + public boolean isSetIa() { + return this.ia != null; + } + + public void setIaIsSet(boolean value) { + if (!value) { + this.ia = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + case IA: + if (value == null) { + unsetIa(); + } else { + setIa((IllegalArgument)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + case IA: + return getIa(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + case IA: + return isSetIa(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof mutateRow_result) + return this.equals((mutateRow_result)that); + return false; + } + + public boolean equals(mutateRow_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + boolean this_present_ia = true && this.isSetIa(); + boolean that_present_ia = true && that.isSetIa(); + if (this_present_ia || that_present_ia) { + if (!(this_present_ia && that_present_ia)) + return false; + if (!this.ia.equals(that.ia)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(mutateRow_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + mutateRow_result typedOther = (mutateRow_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIa()).compareTo(typedOther.isSetIa()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIa()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ia, typedOther.ia); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("mutateRow_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + if (!first) sb.append(", "); + sb.append("ia:"); + if (this.ia == null) { + sb.append("null"); + } else { + sb.append(this.ia); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class mutateRow_resultStandardSchemeFactory implements SchemeFactory { + public mutateRow_resultStandardScheme getScheme() { + return new mutateRow_resultStandardScheme(); + } + } + + private static class mutateRow_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRow_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // IA + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRow_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.ia != null) { + oprot.writeFieldBegin(IA_FIELD_DESC); + struct.ia.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class mutateRow_resultTupleSchemeFactory implements SchemeFactory { + public mutateRow_resultTupleScheme getScheme() { + return new mutateRow_resultTupleScheme(); + } + } + + private static class mutateRow_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, mutateRow_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + if (struct.isSetIa()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + if (struct.isSetIa()) { + struct.ia.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, mutateRow_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + if (incoming.get(1)) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } + } + } + + } + + public static class mutateRowTs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("mutateRowTs_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField MUTATIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("mutations", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)4); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)5); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new mutateRowTs_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new mutateRowTs_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row key + */ + public ByteBuffer row; // required + /** + * list of mutation commands + */ + public List mutations; // required + /** + * timestamp + */ + public long timestamp; // required + /** + * Mutation attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row key + */ + ROW((short)2, "row"), + /** + * list of mutation commands + */ + MUTATIONS((short)3, "mutations"), + /** + * timestamp + */ + TIMESTAMP((short)4, "timestamp"), + /** + * Mutation attributes + */ + ATTRIBUTES((short)5, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // MUTATIONS + return MUTATIONS; + case 4: // TIMESTAMP + return TIMESTAMP; + case 5: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.MUTATIONS, new org.apache.thrift.meta_data.FieldMetaData("mutations", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Mutation.class)))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(mutateRowTs_args.class, metaDataMap); + } + + public mutateRowTs_args() { + } + + public mutateRowTs_args( + ByteBuffer tableName, + ByteBuffer row, + List mutations, + long timestamp, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.mutations = mutations; + this.timestamp = timestamp; + setTimestampIsSet(true); + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public mutateRowTs_args(mutateRowTs_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetMutations()) { + List __this__mutations = new ArrayList(); + for (Mutation other_element : other.mutations) { + __this__mutations.add(new Mutation(other_element)); + } + this.mutations = __this__mutations; + } + this.timestamp = other.timestamp; + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public mutateRowTs_args deepCopy() { + return new mutateRowTs_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.mutations = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public mutateRowTs_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public mutateRowTs_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * row key + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row key + */ + public mutateRowTs_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public mutateRowTs_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getMutationsSize() { + return (this.mutations == null) ? 0 : this.mutations.size(); + } + + public java.util.Iterator getMutationsIterator() { + return (this.mutations == null) ? null : this.mutations.iterator(); + } + + public void addToMutations(Mutation elem) { + if (this.mutations == null) { + this.mutations = new ArrayList(); + } + this.mutations.add(elem); + } + + /** + * list of mutation commands + */ + public List getMutations() { + return this.mutations; + } + + /** + * list of mutation commands + */ + public mutateRowTs_args setMutations(List mutations) { + this.mutations = mutations; + return this; + } + + public void unsetMutations() { + this.mutations = null; + } + + /** Returns true if field mutations is set (has been assigned a value) and false otherwise */ + public boolean isSetMutations() { + return this.mutations != null; + } + + public void setMutationsIsSet(boolean value) { + if (!value) { + this.mutations = null; + } + } + + /** + * timestamp + */ + public long getTimestamp() { + return this.timestamp; + } + + /** + * timestamp + */ + public mutateRowTs_args setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Mutation attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Mutation attributes + */ + public mutateRowTs_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case MUTATIONS: + if (value == null) { + unsetMutations(); + } else { + setMutations((List)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case MUTATIONS: + return getMutations(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case MUTATIONS: + return isSetMutations(); + case TIMESTAMP: + return isSetTimestamp(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof mutateRowTs_args) + return this.equals((mutateRowTs_args)that); + return false; + } + + public boolean equals(mutateRowTs_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_mutations = true && this.isSetMutations(); + boolean that_present_mutations = true && that.isSetMutations(); + if (this_present_mutations || that_present_mutations) { + if (!(this_present_mutations && that_present_mutations)) + return false; + if (!this.mutations.equals(that.mutations)) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(mutateRowTs_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + mutateRowTs_args typedOther = (mutateRowTs_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMutations()).compareTo(typedOther.isSetMutations()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMutations()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.mutations, typedOther.mutations); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("mutateRowTs_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("mutations:"); + if (this.mutations == null) { + sb.append("null"); + } else { + sb.append(this.mutations); + } + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class mutateRowTs_argsStandardSchemeFactory implements SchemeFactory { + public mutateRowTs_argsStandardScheme getScheme() { + return new mutateRowTs_argsStandardScheme(); + } + } + + private static class mutateRowTs_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRowTs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // MUTATIONS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list340 = iprot.readListBegin(); + struct.mutations = new ArrayList(_list340.size); + for (int _i341 = 0; _i341 < _list340.size; ++_i341) + { + Mutation _elem342; // optional + _elem342 = new Mutation(); + _elem342.read(iprot); + struct.mutations.add(_elem342); + } + iprot.readListEnd(); + } + struct.setMutationsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map343 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map343.size); + for (int _i344 = 0; _i344 < _map343.size; ++_i344) + { + ByteBuffer _key345; // required + ByteBuffer _val346; // optional + _key345 = iprot.readBinary(); + _val346 = iprot.readBinary(); + struct.attributes.put(_key345, _val346); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRowTs_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.mutations != null) { + oprot.writeFieldBegin(MUTATIONS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.mutations.size())); + for (Mutation _iter347 : struct.mutations) + { + _iter347.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter348 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter348.getKey()); + oprot.writeBinary(_iter348.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class mutateRowTs_argsTupleSchemeFactory implements SchemeFactory { + public mutateRowTs_argsTupleScheme getScheme() { + return new mutateRowTs_argsTupleScheme(); + } + } + + private static class mutateRowTs_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, mutateRowTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetMutations()) { + optionals.set(2); + } + if (struct.isSetTimestamp()) { + optionals.set(3); + } + if (struct.isSetAttributes()) { + optionals.set(4); + } + oprot.writeBitSet(optionals, 5); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetMutations()) { + { + oprot.writeI32(struct.mutations.size()); + for (Mutation _iter349 : struct.mutations) + { + _iter349.write(oprot); + } + } + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter350 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter350.getKey()); + oprot.writeBinary(_iter350.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, mutateRowTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(5); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list351 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.mutations = new ArrayList(_list351.size); + for (int _i352 = 0; _i352 < _list351.size; ++_i352) + { + Mutation _elem353; // optional + _elem353 = new Mutation(); + _elem353.read(iprot); + struct.mutations.add(_elem353); + } + } + struct.setMutationsIsSet(true); + } + if (incoming.get(3)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(4)) { + { + org.apache.thrift.protocol.TMap _map354 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map354.size); + for (int _i355 = 0; _i355 < _map354.size; ++_i355) + { + ByteBuffer _key356; // required + ByteBuffer _val357; // optional + _key356 = iprot.readBinary(); + _val357 = iprot.readBinary(); + struct.attributes.put(_key356, _val357); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class mutateRowTs_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("mutateRowTs_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField IA_FIELD_DESC = new org.apache.thrift.protocol.TField("ia", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new mutateRowTs_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new mutateRowTs_resultTupleSchemeFactory()); + } + + public IOError io; // required + public IllegalArgument ia; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"), + IA((short)2, "ia"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + case 2: // IA + return IA; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + tmpMap.put(_Fields.IA, new org.apache.thrift.meta_data.FieldMetaData("ia", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(mutateRowTs_result.class, metaDataMap); + } + + public mutateRowTs_result() { + } + + public mutateRowTs_result( + IOError io, + IllegalArgument ia) + { + this(); + this.io = io; + this.ia = ia; + } + + /** + * Performs a deep copy on other. + */ + public mutateRowTs_result(mutateRowTs_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + if (other.isSetIa()) { + this.ia = new IllegalArgument(other.ia); + } + } + + public mutateRowTs_result deepCopy() { + return new mutateRowTs_result(this); + } + + @Override + public void clear() { + this.io = null; + this.ia = null; + } + + public IOError getIo() { + return this.io; + } + + public mutateRowTs_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public IllegalArgument getIa() { + return this.ia; + } + + public mutateRowTs_result setIa(IllegalArgument ia) { + this.ia = ia; + return this; + } + + public void unsetIa() { + this.ia = null; + } + + /** Returns true if field ia is set (has been assigned a value) and false otherwise */ + public boolean isSetIa() { + return this.ia != null; + } + + public void setIaIsSet(boolean value) { + if (!value) { + this.ia = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + case IA: + if (value == null) { + unsetIa(); + } else { + setIa((IllegalArgument)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + case IA: + return getIa(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + case IA: + return isSetIa(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof mutateRowTs_result) + return this.equals((mutateRowTs_result)that); + return false; + } + + public boolean equals(mutateRowTs_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + boolean this_present_ia = true && this.isSetIa(); + boolean that_present_ia = true && that.isSetIa(); + if (this_present_ia || that_present_ia) { + if (!(this_present_ia && that_present_ia)) + return false; + if (!this.ia.equals(that.ia)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(mutateRowTs_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + mutateRowTs_result typedOther = (mutateRowTs_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIa()).compareTo(typedOther.isSetIa()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIa()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ia, typedOther.ia); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("mutateRowTs_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + if (!first) sb.append(", "); + sb.append("ia:"); + if (this.ia == null) { + sb.append("null"); + } else { + sb.append(this.ia); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class mutateRowTs_resultStandardSchemeFactory implements SchemeFactory { + public mutateRowTs_resultStandardScheme getScheme() { + return new mutateRowTs_resultStandardScheme(); + } + } + + private static class mutateRowTs_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRowTs_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // IA + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRowTs_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.ia != null) { + oprot.writeFieldBegin(IA_FIELD_DESC); + struct.ia.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class mutateRowTs_resultTupleSchemeFactory implements SchemeFactory { + public mutateRowTs_resultTupleScheme getScheme() { + return new mutateRowTs_resultTupleScheme(); + } + } + + private static class mutateRowTs_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, mutateRowTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + if (struct.isSetIa()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + if (struct.isSetIa()) { + struct.ia.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, mutateRowTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + if (incoming.get(1)) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } + } + } + + } + + public static class mutateRows_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("mutateRows_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_BATCHES_FIELD_DESC = new org.apache.thrift.protocol.TField("rowBatches", org.apache.thrift.protocol.TType.LIST, (short)2); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new mutateRows_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new mutateRows_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * list of row batches + */ + public List rowBatches; // required + /** + * Mutation attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * list of row batches + */ + ROW_BATCHES((short)2, "rowBatches"), + /** + * Mutation attributes + */ + ATTRIBUTES((short)3, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW_BATCHES + return ROW_BATCHES; + case 3: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW_BATCHES, new org.apache.thrift.meta_data.FieldMetaData("rowBatches", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, BatchMutation.class)))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(mutateRows_args.class, metaDataMap); + } + + public mutateRows_args() { + } + + public mutateRows_args( + ByteBuffer tableName, + List rowBatches, + Map attributes) + { + this(); + this.tableName = tableName; + this.rowBatches = rowBatches; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public mutateRows_args(mutateRows_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRowBatches()) { + List __this__rowBatches = new ArrayList(); + for (BatchMutation other_element : other.rowBatches) { + __this__rowBatches.add(new BatchMutation(other_element)); + } + this.rowBatches = __this__rowBatches; + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public mutateRows_args deepCopy() { + return new mutateRows_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.rowBatches = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public mutateRows_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public mutateRows_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public int getRowBatchesSize() { + return (this.rowBatches == null) ? 0 : this.rowBatches.size(); + } + + public java.util.Iterator getRowBatchesIterator() { + return (this.rowBatches == null) ? null : this.rowBatches.iterator(); + } + + public void addToRowBatches(BatchMutation elem) { + if (this.rowBatches == null) { + this.rowBatches = new ArrayList(); + } + this.rowBatches.add(elem); + } + + /** + * list of row batches + */ + public List getRowBatches() { + return this.rowBatches; + } + + /** + * list of row batches + */ + public mutateRows_args setRowBatches(List rowBatches) { + this.rowBatches = rowBatches; + return this; + } + + public void unsetRowBatches() { + this.rowBatches = null; + } + + /** Returns true if field rowBatches is set (has been assigned a value) and false otherwise */ + public boolean isSetRowBatches() { + return this.rowBatches != null; + } + + public void setRowBatchesIsSet(boolean value) { + if (!value) { + this.rowBatches = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Mutation attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Mutation attributes + */ + public mutateRows_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW_BATCHES: + if (value == null) { + unsetRowBatches(); + } else { + setRowBatches((List)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW_BATCHES: + return getRowBatches(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW_BATCHES: + return isSetRowBatches(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof mutateRows_args) + return this.equals((mutateRows_args)that); + return false; + } + + public boolean equals(mutateRows_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_rowBatches = true && this.isSetRowBatches(); + boolean that_present_rowBatches = true && that.isSetRowBatches(); + if (this_present_rowBatches || that_present_rowBatches) { + if (!(this_present_rowBatches && that_present_rowBatches)) + return false; + if (!this.rowBatches.equals(that.rowBatches)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(mutateRows_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + mutateRows_args typedOther = (mutateRows_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRowBatches()).compareTo(typedOther.isSetRowBatches()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRowBatches()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rowBatches, typedOther.rowBatches); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("mutateRows_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("rowBatches:"); + if (this.rowBatches == null) { + sb.append("null"); + } else { + sb.append(this.rowBatches); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class mutateRows_argsStandardSchemeFactory implements SchemeFactory { + public mutateRows_argsStandardScheme getScheme() { + return new mutateRows_argsStandardScheme(); + } + } + + private static class mutateRows_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRows_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW_BATCHES + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list358 = iprot.readListBegin(); + struct.rowBatches = new ArrayList(_list358.size); + for (int _i359 = 0; _i359 < _list358.size; ++_i359) + { + BatchMutation _elem360; // optional + _elem360 = new BatchMutation(); + _elem360.read(iprot); + struct.rowBatches.add(_elem360); + } + iprot.readListEnd(); + } + struct.setRowBatchesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map361 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map361.size); + for (int _i362 = 0; _i362 < _map361.size; ++_i362) + { + ByteBuffer _key363; // required + ByteBuffer _val364; // optional + _key363 = iprot.readBinary(); + _val364 = iprot.readBinary(); + struct.attributes.put(_key363, _val364); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRows_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.rowBatches != null) { + oprot.writeFieldBegin(ROW_BATCHES_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.rowBatches.size())); + for (BatchMutation _iter365 : struct.rowBatches) + { + _iter365.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter366 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter366.getKey()); + oprot.writeBinary(_iter366.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class mutateRows_argsTupleSchemeFactory implements SchemeFactory { + public mutateRows_argsTupleScheme getScheme() { + return new mutateRows_argsTupleScheme(); + } + } + + private static class mutateRows_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, mutateRows_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRowBatches()) { + optionals.set(1); + } + if (struct.isSetAttributes()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRowBatches()) { + { + oprot.writeI32(struct.rowBatches.size()); + for (BatchMutation _iter367 : struct.rowBatches) + { + _iter367.write(oprot); + } + } + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter368 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter368.getKey()); + oprot.writeBinary(_iter368.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, mutateRows_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + { + org.apache.thrift.protocol.TList _list369 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.rowBatches = new ArrayList(_list369.size); + for (int _i370 = 0; _i370 < _list369.size; ++_i370) + { + BatchMutation _elem371; // optional + _elem371 = new BatchMutation(); + _elem371.read(iprot); + struct.rowBatches.add(_elem371); + } + } + struct.setRowBatchesIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TMap _map372 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map372.size); + for (int _i373 = 0; _i373 < _map372.size; ++_i373) + { + ByteBuffer _key374; // required + ByteBuffer _val375; // optional + _key374 = iprot.readBinary(); + _val375 = iprot.readBinary(); + struct.attributes.put(_key374, _val375); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class mutateRows_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("mutateRows_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField IA_FIELD_DESC = new org.apache.thrift.protocol.TField("ia", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new mutateRows_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new mutateRows_resultTupleSchemeFactory()); + } + + public IOError io; // required + public IllegalArgument ia; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"), + IA((short)2, "ia"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + case 2: // IA + return IA; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + tmpMap.put(_Fields.IA, new org.apache.thrift.meta_data.FieldMetaData("ia", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(mutateRows_result.class, metaDataMap); + } + + public mutateRows_result() { + } + + public mutateRows_result( + IOError io, + IllegalArgument ia) + { + this(); + this.io = io; + this.ia = ia; + } + + /** + * Performs a deep copy on other. + */ + public mutateRows_result(mutateRows_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + if (other.isSetIa()) { + this.ia = new IllegalArgument(other.ia); + } + } + + public mutateRows_result deepCopy() { + return new mutateRows_result(this); + } + + @Override + public void clear() { + this.io = null; + this.ia = null; + } + + public IOError getIo() { + return this.io; + } + + public mutateRows_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public IllegalArgument getIa() { + return this.ia; + } + + public mutateRows_result setIa(IllegalArgument ia) { + this.ia = ia; + return this; + } + + public void unsetIa() { + this.ia = null; + } + + /** Returns true if field ia is set (has been assigned a value) and false otherwise */ + public boolean isSetIa() { + return this.ia != null; + } + + public void setIaIsSet(boolean value) { + if (!value) { + this.ia = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + case IA: + if (value == null) { + unsetIa(); + } else { + setIa((IllegalArgument)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + case IA: + return getIa(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + case IA: + return isSetIa(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof mutateRows_result) + return this.equals((mutateRows_result)that); + return false; + } + + public boolean equals(mutateRows_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + boolean this_present_ia = true && this.isSetIa(); + boolean that_present_ia = true && that.isSetIa(); + if (this_present_ia || that_present_ia) { + if (!(this_present_ia && that_present_ia)) + return false; + if (!this.ia.equals(that.ia)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(mutateRows_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + mutateRows_result typedOther = (mutateRows_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIa()).compareTo(typedOther.isSetIa()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIa()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ia, typedOther.ia); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("mutateRows_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + if (!first) sb.append(", "); + sb.append("ia:"); + if (this.ia == null) { + sb.append("null"); + } else { + sb.append(this.ia); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class mutateRows_resultStandardSchemeFactory implements SchemeFactory { + public mutateRows_resultStandardScheme getScheme() { + return new mutateRows_resultStandardScheme(); + } + } + + private static class mutateRows_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRows_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // IA + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRows_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.ia != null) { + oprot.writeFieldBegin(IA_FIELD_DESC); + struct.ia.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class mutateRows_resultTupleSchemeFactory implements SchemeFactory { + public mutateRows_resultTupleScheme getScheme() { + return new mutateRows_resultTupleScheme(); + } + } + + private static class mutateRows_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, mutateRows_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + if (struct.isSetIa()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + if (struct.isSetIa()) { + struct.ia.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, mutateRows_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + if (incoming.get(1)) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } + } + } + + } + + public static class mutateRowsTs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("mutateRowsTs_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_BATCHES_FIELD_DESC = new org.apache.thrift.protocol.TField("rowBatches", org.apache.thrift.protocol.TType.LIST, (short)2); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new mutateRowsTs_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new mutateRowsTs_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * list of row batches + */ + public List rowBatches; // required + /** + * timestamp + */ + public long timestamp; // required + /** + * Mutation attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * list of row batches + */ + ROW_BATCHES((short)2, "rowBatches"), + /** + * timestamp + */ + TIMESTAMP((short)3, "timestamp"), + /** + * Mutation attributes + */ + ATTRIBUTES((short)4, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW_BATCHES + return ROW_BATCHES; + case 3: // TIMESTAMP + return TIMESTAMP; + case 4: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW_BATCHES, new org.apache.thrift.meta_data.FieldMetaData("rowBatches", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, BatchMutation.class)))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(mutateRowsTs_args.class, metaDataMap); + } + + public mutateRowsTs_args() { + } + + public mutateRowsTs_args( + ByteBuffer tableName, + List rowBatches, + long timestamp, + Map attributes) + { + this(); + this.tableName = tableName; + this.rowBatches = rowBatches; + this.timestamp = timestamp; + setTimestampIsSet(true); + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public mutateRowsTs_args(mutateRowsTs_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRowBatches()) { + List __this__rowBatches = new ArrayList(); + for (BatchMutation other_element : other.rowBatches) { + __this__rowBatches.add(new BatchMutation(other_element)); + } + this.rowBatches = __this__rowBatches; + } + this.timestamp = other.timestamp; + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public mutateRowsTs_args deepCopy() { + return new mutateRowsTs_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.rowBatches = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public mutateRowsTs_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public mutateRowsTs_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public int getRowBatchesSize() { + return (this.rowBatches == null) ? 0 : this.rowBatches.size(); + } + + public java.util.Iterator getRowBatchesIterator() { + return (this.rowBatches == null) ? null : this.rowBatches.iterator(); + } + + public void addToRowBatches(BatchMutation elem) { + if (this.rowBatches == null) { + this.rowBatches = new ArrayList(); + } + this.rowBatches.add(elem); + } + + /** + * list of row batches + */ + public List getRowBatches() { + return this.rowBatches; + } + + /** + * list of row batches + */ + public mutateRowsTs_args setRowBatches(List rowBatches) { + this.rowBatches = rowBatches; + return this; + } + + public void unsetRowBatches() { + this.rowBatches = null; + } + + /** Returns true if field rowBatches is set (has been assigned a value) and false otherwise */ + public boolean isSetRowBatches() { + return this.rowBatches != null; + } + + public void setRowBatchesIsSet(boolean value) { + if (!value) { + this.rowBatches = null; + } + } + + /** + * timestamp + */ + public long getTimestamp() { + return this.timestamp; + } + + /** + * timestamp + */ + public mutateRowsTs_args setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Mutation attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Mutation attributes + */ + public mutateRowsTs_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW_BATCHES: + if (value == null) { + unsetRowBatches(); + } else { + setRowBatches((List)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW_BATCHES: + return getRowBatches(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW_BATCHES: + return isSetRowBatches(); + case TIMESTAMP: + return isSetTimestamp(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof mutateRowsTs_args) + return this.equals((mutateRowsTs_args)that); + return false; + } + + public boolean equals(mutateRowsTs_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_rowBatches = true && this.isSetRowBatches(); + boolean that_present_rowBatches = true && that.isSetRowBatches(); + if (this_present_rowBatches || that_present_rowBatches) { + if (!(this_present_rowBatches && that_present_rowBatches)) + return false; + if (!this.rowBatches.equals(that.rowBatches)) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(mutateRowsTs_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + mutateRowsTs_args typedOther = (mutateRowsTs_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRowBatches()).compareTo(typedOther.isSetRowBatches()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRowBatches()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rowBatches, typedOther.rowBatches); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("mutateRowsTs_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("rowBatches:"); + if (this.rowBatches == null) { + sb.append("null"); + } else { + sb.append(this.rowBatches); + } + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class mutateRowsTs_argsStandardSchemeFactory implements SchemeFactory { + public mutateRowsTs_argsStandardScheme getScheme() { + return new mutateRowsTs_argsStandardScheme(); + } + } + + private static class mutateRowsTs_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRowsTs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW_BATCHES + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list376 = iprot.readListBegin(); + struct.rowBatches = new ArrayList(_list376.size); + for (int _i377 = 0; _i377 < _list376.size; ++_i377) + { + BatchMutation _elem378; // optional + _elem378 = new BatchMutation(); + _elem378.read(iprot); + struct.rowBatches.add(_elem378); + } + iprot.readListEnd(); + } + struct.setRowBatchesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map379 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map379.size); + for (int _i380 = 0; _i380 < _map379.size; ++_i380) + { + ByteBuffer _key381; // required + ByteBuffer _val382; // optional + _key381 = iprot.readBinary(); + _val382 = iprot.readBinary(); + struct.attributes.put(_key381, _val382); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRowsTs_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.rowBatches != null) { + oprot.writeFieldBegin(ROW_BATCHES_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.rowBatches.size())); + for (BatchMutation _iter383 : struct.rowBatches) + { + _iter383.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter384 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter384.getKey()); + oprot.writeBinary(_iter384.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class mutateRowsTs_argsTupleSchemeFactory implements SchemeFactory { + public mutateRowsTs_argsTupleScheme getScheme() { + return new mutateRowsTs_argsTupleScheme(); + } + } + + private static class mutateRowsTs_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, mutateRowsTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRowBatches()) { + optionals.set(1); + } + if (struct.isSetTimestamp()) { + optionals.set(2); + } + if (struct.isSetAttributes()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRowBatches()) { + { + oprot.writeI32(struct.rowBatches.size()); + for (BatchMutation _iter385 : struct.rowBatches) + { + _iter385.write(oprot); + } + } + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter386 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter386.getKey()); + oprot.writeBinary(_iter386.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, mutateRowsTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + { + org.apache.thrift.protocol.TList _list387 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.rowBatches = new ArrayList(_list387.size); + for (int _i388 = 0; _i388 < _list387.size; ++_i388) + { + BatchMutation _elem389; // optional + _elem389 = new BatchMutation(); + _elem389.read(iprot); + struct.rowBatches.add(_elem389); + } + } + struct.setRowBatchesIsSet(true); + } + if (incoming.get(2)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TMap _map390 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map390.size); + for (int _i391 = 0; _i391 < _map390.size; ++_i391) + { + ByteBuffer _key392; // required + ByteBuffer _val393; // optional + _key392 = iprot.readBinary(); + _val393 = iprot.readBinary(); + struct.attributes.put(_key392, _val393); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class mutateRowsTs_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("mutateRowsTs_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField IA_FIELD_DESC = new org.apache.thrift.protocol.TField("ia", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new mutateRowsTs_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new mutateRowsTs_resultTupleSchemeFactory()); + } + + public IOError io; // required + public IllegalArgument ia; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"), + IA((short)2, "ia"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + case 2: // IA + return IA; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + tmpMap.put(_Fields.IA, new org.apache.thrift.meta_data.FieldMetaData("ia", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(mutateRowsTs_result.class, metaDataMap); + } + + public mutateRowsTs_result() { + } + + public mutateRowsTs_result( + IOError io, + IllegalArgument ia) + { + this(); + this.io = io; + this.ia = ia; + } + + /** + * Performs a deep copy on other. + */ + public mutateRowsTs_result(mutateRowsTs_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + if (other.isSetIa()) { + this.ia = new IllegalArgument(other.ia); + } + } + + public mutateRowsTs_result deepCopy() { + return new mutateRowsTs_result(this); + } + + @Override + public void clear() { + this.io = null; + this.ia = null; + } + + public IOError getIo() { + return this.io; + } + + public mutateRowsTs_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public IllegalArgument getIa() { + return this.ia; + } + + public mutateRowsTs_result setIa(IllegalArgument ia) { + this.ia = ia; + return this; + } + + public void unsetIa() { + this.ia = null; + } + + /** Returns true if field ia is set (has been assigned a value) and false otherwise */ + public boolean isSetIa() { + return this.ia != null; + } + + public void setIaIsSet(boolean value) { + if (!value) { + this.ia = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + case IA: + if (value == null) { + unsetIa(); + } else { + setIa((IllegalArgument)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + case IA: + return getIa(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + case IA: + return isSetIa(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof mutateRowsTs_result) + return this.equals((mutateRowsTs_result)that); + return false; + } + + public boolean equals(mutateRowsTs_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + boolean this_present_ia = true && this.isSetIa(); + boolean that_present_ia = true && that.isSetIa(); + if (this_present_ia || that_present_ia) { + if (!(this_present_ia && that_present_ia)) + return false; + if (!this.ia.equals(that.ia)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(mutateRowsTs_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + mutateRowsTs_result typedOther = (mutateRowsTs_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIa()).compareTo(typedOther.isSetIa()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIa()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ia, typedOther.ia); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("mutateRowsTs_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + if (!first) sb.append(", "); + sb.append("ia:"); + if (this.ia == null) { + sb.append("null"); + } else { + sb.append(this.ia); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class mutateRowsTs_resultStandardSchemeFactory implements SchemeFactory { + public mutateRowsTs_resultStandardScheme getScheme() { + return new mutateRowsTs_resultStandardScheme(); + } + } + + private static class mutateRowsTs_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, mutateRowsTs_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // IA + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, mutateRowsTs_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.ia != null) { + oprot.writeFieldBegin(IA_FIELD_DESC); + struct.ia.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class mutateRowsTs_resultTupleSchemeFactory implements SchemeFactory { + public mutateRowsTs_resultTupleScheme getScheme() { + return new mutateRowsTs_resultTupleScheme(); + } + } + + private static class mutateRowsTs_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, mutateRowsTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + if (struct.isSetIa()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + if (struct.isSetIa()) { + struct.ia.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, mutateRowsTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + if (incoming.get(1)) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } + } + } + + } + + public static class atomicIncrement_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("atomicIncrement_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.I64, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new atomicIncrement_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new atomicIncrement_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row to increment + */ + public ByteBuffer row; // required + /** + * name of column + */ + public ByteBuffer column; // required + /** + * amount to increment by + */ + public long value; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row to increment + */ + ROW((short)2, "row"), + /** + * name of column + */ + COLUMN((short)3, "column"), + /** + * amount to increment by + */ + VALUE((short)4, "value"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // COLUMN + return COLUMN; + case 4: // VALUE + return VALUE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __VALUE_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(atomicIncrement_args.class, metaDataMap); + } + + public atomicIncrement_args() { + } + + public atomicIncrement_args( + ByteBuffer tableName, + ByteBuffer row, + ByteBuffer column, + long value) + { + this(); + this.tableName = tableName; + this.row = row; + this.column = column; + this.value = value; + setValueIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public atomicIncrement_args(atomicIncrement_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetColumn()) { + this.column = other.column; + } + this.value = other.value; + } + + public atomicIncrement_args deepCopy() { + return new atomicIncrement_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.column = null; + setValueIsSet(false); + this.value = 0; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public atomicIncrement_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public atomicIncrement_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * row to increment + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row to increment + */ + public atomicIncrement_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public atomicIncrement_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + /** + * name of column + */ + public byte[] getColumn() { + setColumn(org.apache.thrift.TBaseHelper.rightSize(column)); + return column == null ? null : column.array(); + } + + public ByteBuffer bufferForColumn() { + return column; + } + + /** + * name of column + */ + public atomicIncrement_args setColumn(byte[] column) { + setColumn(column == null ? (ByteBuffer)null : ByteBuffer.wrap(column)); + return this; + } + + public atomicIncrement_args setColumn(ByteBuffer column) { + this.column = column; + return this; + } + + public void unsetColumn() { + this.column = null; + } + + /** Returns true if field column is set (has been assigned a value) and false otherwise */ + public boolean isSetColumn() { + return this.column != null; + } + + public void setColumnIsSet(boolean value) { + if (!value) { + this.column = null; + } + } + + /** + * amount to increment by + */ + public long getValue() { + return this.value; + } + + /** + * amount to increment by + */ + public atomicIncrement_args setValue(long value) { + this.value = value; + setValueIsSet(true); + return this; + } + + public void unsetValue() { + __isset_bit_vector.clear(__VALUE_ISSET_ID); + } + + /** Returns true if field value is set (has been assigned a value) and false otherwise */ + public boolean isSetValue() { + return __isset_bit_vector.get(__VALUE_ISSET_ID); + } + + public void setValueIsSet(boolean value) { + __isset_bit_vector.set(__VALUE_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMN: + if (value == null) { + unsetColumn(); + } else { + setColumn((ByteBuffer)value); + } + break; + + case VALUE: + if (value == null) { + unsetValue(); + } else { + setValue((Long)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case COLUMN: + return getColumn(); + + case VALUE: + return Long.valueOf(getValue()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case COLUMN: + return isSetColumn(); + case VALUE: + return isSetValue(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof atomicIncrement_args) + return this.equals((atomicIncrement_args)that); + return false; + } + + public boolean equals(atomicIncrement_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_column = true && this.isSetColumn(); + boolean that_present_column = true && that.isSetColumn(); + if (this_present_column || that_present_column) { + if (!(this_present_column && that_present_column)) + return false; + if (!this.column.equals(that.column)) + return false; + } + + boolean this_present_value = true; + boolean that_present_value = true; + if (this_present_value || that_present_value) { + if (!(this_present_value && that_present_value)) + return false; + if (this.value != that.value) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(atomicIncrement_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + atomicIncrement_args typedOther = (atomicIncrement_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumn()).compareTo(typedOther.isSetColumn()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumn()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, typedOther.column); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("atomicIncrement_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("column:"); + if (this.column == null) { + sb.append("null"); + } else { + sb.append(this.column); + } + first = false; + if (!first) sb.append(", "); + sb.append("value:"); + sb.append(this.value); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class atomicIncrement_argsStandardSchemeFactory implements SchemeFactory { + public atomicIncrement_argsStandardScheme getScheme() { + return new atomicIncrement_argsStandardScheme(); + } + } + + private static class atomicIncrement_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, atomicIncrement_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMN + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.value = iprot.readI64(); + struct.setValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, atomicIncrement_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.column != null) { + oprot.writeFieldBegin(COLUMN_FIELD_DESC); + oprot.writeBinary(struct.column); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(VALUE_FIELD_DESC); + oprot.writeI64(struct.value); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class atomicIncrement_argsTupleSchemeFactory implements SchemeFactory { + public atomicIncrement_argsTupleScheme getScheme() { + return new atomicIncrement_argsTupleScheme(); + } + } + + private static class atomicIncrement_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, atomicIncrement_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetColumn()) { + optionals.set(2); + } + if (struct.isSetValue()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetColumn()) { + oprot.writeBinary(struct.column); + } + if (struct.isSetValue()) { + oprot.writeI64(struct.value); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, atomicIncrement_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } + if (incoming.get(3)) { + struct.value = iprot.readI64(); + struct.setValueIsSet(true); + } + } + } + + } + + public static class atomicIncrement_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("atomicIncrement_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.I64, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField IA_FIELD_DESC = new org.apache.thrift.protocol.TField("ia", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new atomicIncrement_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new atomicIncrement_resultTupleSchemeFactory()); + } + + public long success; // required + public IOError io; // required + public IllegalArgument ia; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"), + IA((short)2, "ia"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + case 2: // IA + return IA; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + tmpMap.put(_Fields.IA, new org.apache.thrift.meta_data.FieldMetaData("ia", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(atomicIncrement_result.class, metaDataMap); + } + + public atomicIncrement_result() { + } + + public atomicIncrement_result( + long success, + IOError io, + IllegalArgument ia) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.io = io; + this.ia = ia; + } + + /** + * Performs a deep copy on other. + */ + public atomicIncrement_result(atomicIncrement_result other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.success = other.success; + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + if (other.isSetIa()) { + this.ia = new IllegalArgument(other.ia); + } + } + + public atomicIncrement_result deepCopy() { + return new atomicIncrement_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = 0; + this.io = null; + this.ia = null; + } + + public long getSuccess() { + return this.success; + } + + public atomicIncrement_result setSuccess(long success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bit_vector.clear(__SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return __isset_bit_vector.get(__SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bit_vector.set(__SUCCESS_ISSET_ID, value); + } + + public IOError getIo() { + return this.io; + } + + public atomicIncrement_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public IllegalArgument getIa() { + return this.ia; + } + + public atomicIncrement_result setIa(IllegalArgument ia) { + this.ia = ia; + return this; + } + + public void unsetIa() { + this.ia = null; + } + + /** Returns true if field ia is set (has been assigned a value) and false otherwise */ + public boolean isSetIa() { + return this.ia != null; + } + + public void setIaIsSet(boolean value) { + if (!value) { + this.ia = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Long)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + case IA: + if (value == null) { + unsetIa(); + } else { + setIa((IllegalArgument)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Long.valueOf(getSuccess()); + + case IO: + return getIo(); + + case IA: + return getIa(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + case IA: + return isSetIa(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof atomicIncrement_result) + return this.equals((atomicIncrement_result)that); + return false; + } + + public boolean equals(atomicIncrement_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + boolean this_present_ia = true && this.isSetIa(); + boolean that_present_ia = true && that.isSetIa(); + if (this_present_ia || that_present_ia) { + if (!(this_present_ia && that_present_ia)) + return false; + if (!this.ia.equals(that.ia)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(atomicIncrement_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + atomicIncrement_result typedOther = (atomicIncrement_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIa()).compareTo(typedOther.isSetIa()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIa()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ia, typedOther.ia); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("atomicIncrement_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + if (!first) sb.append(", "); + sb.append("ia:"); + if (this.ia == null) { + sb.append("null"); + } else { + sb.append(this.ia); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class atomicIncrement_resultStandardSchemeFactory implements SchemeFactory { + public atomicIncrement_resultStandardScheme getScheme() { + return new atomicIncrement_resultStandardScheme(); + } + } + + private static class atomicIncrement_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, atomicIncrement_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.success = iprot.readI64(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // IA + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, atomicIncrement_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeI64(struct.success); + oprot.writeFieldEnd(); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.ia != null) { + oprot.writeFieldBegin(IA_FIELD_DESC); + struct.ia.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class atomicIncrement_resultTupleSchemeFactory implements SchemeFactory { + public atomicIncrement_resultTupleScheme getScheme() { + return new atomicIncrement_resultTupleScheme(); + } + } + + private static class atomicIncrement_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, atomicIncrement_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + if (struct.isSetIa()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetSuccess()) { + oprot.writeI64(struct.success); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + if (struct.isSetIa()) { + struct.ia.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, atomicIncrement_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.success = iprot.readI64(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + if (incoming.get(2)) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } + } + } + + } + + public static class deleteAll_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteAll_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteAll_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteAll_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * Row to update + */ + public ByteBuffer row; // required + /** + * name of column whose value is to be deleted + */ + public ByteBuffer column; // required + /** + * Delete attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * Row to update + */ + ROW((short)2, "row"), + /** + * name of column whose value is to be deleted + */ + COLUMN((short)3, "column"), + /** + * Delete attributes + */ + ATTRIBUTES((short)4, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // COLUMN + return COLUMN; + case 4: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteAll_args.class, metaDataMap); + } + + public deleteAll_args() { + } + + public deleteAll_args( + ByteBuffer tableName, + ByteBuffer row, + ByteBuffer column, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.column = column; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public deleteAll_args(deleteAll_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetColumn()) { + this.column = other.column; + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public deleteAll_args deepCopy() { + return new deleteAll_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.column = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public deleteAll_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public deleteAll_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * Row to update + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * Row to update + */ + public deleteAll_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public deleteAll_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + /** + * name of column whose value is to be deleted + */ + public byte[] getColumn() { + setColumn(org.apache.thrift.TBaseHelper.rightSize(column)); + return column == null ? null : column.array(); + } + + public ByteBuffer bufferForColumn() { + return column; + } + + /** + * name of column whose value is to be deleted + */ + public deleteAll_args setColumn(byte[] column) { + setColumn(column == null ? (ByteBuffer)null : ByteBuffer.wrap(column)); + return this; + } + + public deleteAll_args setColumn(ByteBuffer column) { + this.column = column; + return this; + } + + public void unsetColumn() { + this.column = null; + } + + /** Returns true if field column is set (has been assigned a value) and false otherwise */ + public boolean isSetColumn() { + return this.column != null; + } + + public void setColumnIsSet(boolean value) { + if (!value) { + this.column = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Delete attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Delete attributes + */ + public deleteAll_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMN: + if (value == null) { + unsetColumn(); + } else { + setColumn((ByteBuffer)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case COLUMN: + return getColumn(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case COLUMN: + return isSetColumn(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteAll_args) + return this.equals((deleteAll_args)that); + return false; + } + + public boolean equals(deleteAll_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_column = true && this.isSetColumn(); + boolean that_present_column = true && that.isSetColumn(); + if (this_present_column || that_present_column) { + if (!(this_present_column && that_present_column)) + return false; + if (!this.column.equals(that.column)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteAll_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteAll_args typedOther = (deleteAll_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumn()).compareTo(typedOther.isSetColumn()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumn()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, typedOther.column); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteAll_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("column:"); + if (this.column == null) { + sb.append("null"); + } else { + sb.append(this.column); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteAll_argsStandardSchemeFactory implements SchemeFactory { + public deleteAll_argsStandardScheme getScheme() { + return new deleteAll_argsStandardScheme(); + } + } + + private static class deleteAll_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAll_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMN + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map394 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map394.size); + for (int _i395 = 0; _i395 < _map394.size; ++_i395) + { + ByteBuffer _key396; // required + ByteBuffer _val397; // optional + _key396 = iprot.readBinary(); + _val397 = iprot.readBinary(); + struct.attributes.put(_key396, _val397); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAll_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.column != null) { + oprot.writeFieldBegin(COLUMN_FIELD_DESC); + oprot.writeBinary(struct.column); + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter398 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter398.getKey()); + oprot.writeBinary(_iter398.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteAll_argsTupleSchemeFactory implements SchemeFactory { + public deleteAll_argsTupleScheme getScheme() { + return new deleteAll_argsTupleScheme(); + } + } + + private static class deleteAll_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteAll_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetColumn()) { + optionals.set(2); + } + if (struct.isSetAttributes()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetColumn()) { + oprot.writeBinary(struct.column); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter399 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter399.getKey()); + oprot.writeBinary(_iter399.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteAll_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TMap _map400 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map400.size); + for (int _i401 = 0; _i401 < _map400.size; ++_i401) + { + ByteBuffer _key402; // required + ByteBuffer _val403; // optional + _key402 = iprot.readBinary(); + _val403 = iprot.readBinary(); + struct.attributes.put(_key402, _val403); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class deleteAll_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteAll_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteAll_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteAll_resultTupleSchemeFactory()); + } + + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteAll_result.class, metaDataMap); + } + + public deleteAll_result() { + } + + public deleteAll_result( + IOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public deleteAll_result(deleteAll_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public deleteAll_result deepCopy() { + return new deleteAll_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public IOError getIo() { + return this.io; + } + + public deleteAll_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteAll_result) + return this.equals((deleteAll_result)that); + return false; + } + + public boolean equals(deleteAll_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteAll_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteAll_result typedOther = (deleteAll_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteAll_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteAll_resultStandardSchemeFactory implements SchemeFactory { + public deleteAll_resultStandardScheme getScheme() { + return new deleteAll_resultStandardScheme(); + } + } + + private static class deleteAll_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAll_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAll_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteAll_resultTupleSchemeFactory implements SchemeFactory { + public deleteAll_resultTupleScheme getScheme() { + return new deleteAll_resultTupleScheme(); + } + } + + private static class deleteAll_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteAll_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteAll_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class deleteAllTs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteAllTs_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)4); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)5); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteAllTs_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteAllTs_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * Row to update + */ + public ByteBuffer row; // required + /** + * name of column whose value is to be deleted + */ + public ByteBuffer column; // required + /** + * timestamp + */ + public long timestamp; // required + /** + * Delete attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * Row to update + */ + ROW((short)2, "row"), + /** + * name of column whose value is to be deleted + */ + COLUMN((short)3, "column"), + /** + * timestamp + */ + TIMESTAMP((short)4, "timestamp"), + /** + * Delete attributes + */ + ATTRIBUTES((short)5, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // COLUMN + return COLUMN; + case 4: // TIMESTAMP + return TIMESTAMP; + case 5: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteAllTs_args.class, metaDataMap); + } + + public deleteAllTs_args() { + } + + public deleteAllTs_args( + ByteBuffer tableName, + ByteBuffer row, + ByteBuffer column, + long timestamp, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.column = column; + this.timestamp = timestamp; + setTimestampIsSet(true); + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public deleteAllTs_args(deleteAllTs_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetColumn()) { + this.column = other.column; + } + this.timestamp = other.timestamp; + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public deleteAllTs_args deepCopy() { + return new deleteAllTs_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.column = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public deleteAllTs_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public deleteAllTs_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * Row to update + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * Row to update + */ + public deleteAllTs_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public deleteAllTs_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + /** + * name of column whose value is to be deleted + */ + public byte[] getColumn() { + setColumn(org.apache.thrift.TBaseHelper.rightSize(column)); + return column == null ? null : column.array(); + } + + public ByteBuffer bufferForColumn() { + return column; + } + + /** + * name of column whose value is to be deleted + */ + public deleteAllTs_args setColumn(byte[] column) { + setColumn(column == null ? (ByteBuffer)null : ByteBuffer.wrap(column)); + return this; + } + + public deleteAllTs_args setColumn(ByteBuffer column) { + this.column = column; + return this; + } + + public void unsetColumn() { + this.column = null; + } + + /** Returns true if field column is set (has been assigned a value) and false otherwise */ + public boolean isSetColumn() { + return this.column != null; + } + + public void setColumnIsSet(boolean value) { + if (!value) { + this.column = null; + } + } + + /** + * timestamp + */ + public long getTimestamp() { + return this.timestamp; + } + + /** + * timestamp + */ + public deleteAllTs_args setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Delete attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Delete attributes + */ + public deleteAllTs_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMN: + if (value == null) { + unsetColumn(); + } else { + setColumn((ByteBuffer)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case COLUMN: + return getColumn(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case COLUMN: + return isSetColumn(); + case TIMESTAMP: + return isSetTimestamp(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteAllTs_args) + return this.equals((deleteAllTs_args)that); + return false; + } + + public boolean equals(deleteAllTs_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_column = true && this.isSetColumn(); + boolean that_present_column = true && that.isSetColumn(); + if (this_present_column || that_present_column) { + if (!(this_present_column && that_present_column)) + return false; + if (!this.column.equals(that.column)) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteAllTs_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteAllTs_args typedOther = (deleteAllTs_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumn()).compareTo(typedOther.isSetColumn()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumn()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, typedOther.column); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteAllTs_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("column:"); + if (this.column == null) { + sb.append("null"); + } else { + sb.append(this.column); + } + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteAllTs_argsStandardSchemeFactory implements SchemeFactory { + public deleteAllTs_argsStandardScheme getScheme() { + return new deleteAllTs_argsStandardScheme(); + } + } + + private static class deleteAllTs_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAllTs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMN + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map404 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map404.size); + for (int _i405 = 0; _i405 < _map404.size; ++_i405) + { + ByteBuffer _key406; // required + ByteBuffer _val407; // optional + _key406 = iprot.readBinary(); + _val407 = iprot.readBinary(); + struct.attributes.put(_key406, _val407); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAllTs_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.column != null) { + oprot.writeFieldBegin(COLUMN_FIELD_DESC); + oprot.writeBinary(struct.column); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter408 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter408.getKey()); + oprot.writeBinary(_iter408.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteAllTs_argsTupleSchemeFactory implements SchemeFactory { + public deleteAllTs_argsTupleScheme getScheme() { + return new deleteAllTs_argsTupleScheme(); + } + } + + private static class deleteAllTs_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteAllTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetColumn()) { + optionals.set(2); + } + if (struct.isSetTimestamp()) { + optionals.set(3); + } + if (struct.isSetAttributes()) { + optionals.set(4); + } + oprot.writeBitSet(optionals, 5); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetColumn()) { + oprot.writeBinary(struct.column); + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter409 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter409.getKey()); + oprot.writeBinary(_iter409.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(5); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } + if (incoming.get(3)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(4)) { + { + org.apache.thrift.protocol.TMap _map410 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map410.size); + for (int _i411 = 0; _i411 < _map410.size; ++_i411) + { + ByteBuffer _key412; // required + ByteBuffer _val413; // optional + _key412 = iprot.readBinary(); + _val413 = iprot.readBinary(); + struct.attributes.put(_key412, _val413); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class deleteAllTs_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteAllTs_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteAllTs_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteAllTs_resultTupleSchemeFactory()); + } + + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteAllTs_result.class, metaDataMap); + } + + public deleteAllTs_result() { + } + + public deleteAllTs_result( + IOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public deleteAllTs_result(deleteAllTs_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public deleteAllTs_result deepCopy() { + return new deleteAllTs_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public IOError getIo() { + return this.io; + } + + public deleteAllTs_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteAllTs_result) + return this.equals((deleteAllTs_result)that); + return false; + } + + public boolean equals(deleteAllTs_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteAllTs_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteAllTs_result typedOther = (deleteAllTs_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteAllTs_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteAllTs_resultStandardSchemeFactory implements SchemeFactory { + public deleteAllTs_resultStandardScheme getScheme() { + return new deleteAllTs_resultStandardScheme(); + } + } + + private static class deleteAllTs_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAllTs_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAllTs_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteAllTs_resultTupleSchemeFactory implements SchemeFactory { + public deleteAllTs_resultTupleScheme getScheme() { + return new deleteAllTs_resultTupleScheme(); + } + } + + private static class deleteAllTs_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteAllTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class deleteAllRow_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteAllRow_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteAllRow_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteAllRow_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * key of the row to be completely deleted. + */ + public ByteBuffer row; // required + /** + * Delete attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * key of the row to be completely deleted. + */ + ROW((short)2, "row"), + /** + * Delete attributes + */ + ATTRIBUTES((short)3, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteAllRow_args.class, metaDataMap); + } + + public deleteAllRow_args() { + } + + public deleteAllRow_args( + ByteBuffer tableName, + ByteBuffer row, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public deleteAllRow_args(deleteAllRow_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public deleteAllRow_args deepCopy() { + return new deleteAllRow_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public deleteAllRow_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public deleteAllRow_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * key of the row to be completely deleted. + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * key of the row to be completely deleted. + */ + public deleteAllRow_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public deleteAllRow_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Delete attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Delete attributes + */ + public deleteAllRow_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteAllRow_args) + return this.equals((deleteAllRow_args)that); + return false; + } + + public boolean equals(deleteAllRow_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteAllRow_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteAllRow_args typedOther = (deleteAllRow_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteAllRow_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteAllRow_argsStandardSchemeFactory implements SchemeFactory { + public deleteAllRow_argsStandardScheme getScheme() { + return new deleteAllRow_argsStandardScheme(); + } + } + + private static class deleteAllRow_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAllRow_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map414 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map414.size); + for (int _i415 = 0; _i415 < _map414.size; ++_i415) + { + ByteBuffer _key416; // required + ByteBuffer _val417; // optional + _key416 = iprot.readBinary(); + _val417 = iprot.readBinary(); + struct.attributes.put(_key416, _val417); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAllRow_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter418 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter418.getKey()); + oprot.writeBinary(_iter418.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteAllRow_argsTupleSchemeFactory implements SchemeFactory { + public deleteAllRow_argsTupleScheme getScheme() { + return new deleteAllRow_argsTupleScheme(); + } + } + + private static class deleteAllRow_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteAllRow_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetAttributes()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter419 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter419.getKey()); + oprot.writeBinary(_iter419.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllRow_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TMap _map420 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map420.size); + for (int _i421 = 0; _i421 < _map420.size; ++_i421) + { + ByteBuffer _key422; // required + ByteBuffer _val423; // optional + _key422 = iprot.readBinary(); + _val423 = iprot.readBinary(); + struct.attributes.put(_key422, _val423); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class deleteAllRow_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteAllRow_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteAllRow_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteAllRow_resultTupleSchemeFactory()); + } + + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteAllRow_result.class, metaDataMap); + } + + public deleteAllRow_result() { + } + + public deleteAllRow_result( + IOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public deleteAllRow_result(deleteAllRow_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public deleteAllRow_result deepCopy() { + return new deleteAllRow_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public IOError getIo() { + return this.io; + } + + public deleteAllRow_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteAllRow_result) + return this.equals((deleteAllRow_result)that); + return false; + } + + public boolean equals(deleteAllRow_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteAllRow_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteAllRow_result typedOther = (deleteAllRow_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteAllRow_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteAllRow_resultStandardSchemeFactory implements SchemeFactory { + public deleteAllRow_resultStandardScheme getScheme() { + return new deleteAllRow_resultStandardScheme(); + } + } + + private static class deleteAllRow_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAllRow_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAllRow_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteAllRow_resultTupleSchemeFactory implements SchemeFactory { + public deleteAllRow_resultTupleScheme getScheme() { + return new deleteAllRow_resultTupleScheme(); + } + } + + private static class deleteAllRow_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteAllRow_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllRow_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class increment_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("increment_args"); + + private static final org.apache.thrift.protocol.TField INCREMENT_FIELD_DESC = new org.apache.thrift.protocol.TField("increment", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new increment_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new increment_argsTupleSchemeFactory()); + } + + /** + * The single increment to apply + */ + public TIncrement increment; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * The single increment to apply + */ + INCREMENT((short)1, "increment"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // INCREMENT + return INCREMENT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.INCREMENT, new org.apache.thrift.meta_data.FieldMetaData("increment", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TIncrement.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(increment_args.class, metaDataMap); + } + + public increment_args() { + } + + public increment_args( + TIncrement increment) + { + this(); + this.increment = increment; + } + + /** + * Performs a deep copy on other. + */ + public increment_args(increment_args other) { + if (other.isSetIncrement()) { + this.increment = new TIncrement(other.increment); + } + } + + public increment_args deepCopy() { + return new increment_args(this); + } + + @Override + public void clear() { + this.increment = null; + } + + /** + * The single increment to apply + */ + public TIncrement getIncrement() { + return this.increment; + } + + /** + * The single increment to apply + */ + public increment_args setIncrement(TIncrement increment) { + this.increment = increment; + return this; + } + + public void unsetIncrement() { + this.increment = null; + } + + /** Returns true if field increment is set (has been assigned a value) and false otherwise */ + public boolean isSetIncrement() { + return this.increment != null; + } + + public void setIncrementIsSet(boolean value) { + if (!value) { + this.increment = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case INCREMENT: + if (value == null) { + unsetIncrement(); + } else { + setIncrement((TIncrement)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case INCREMENT: + return getIncrement(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case INCREMENT: + return isSetIncrement(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof increment_args) + return this.equals((increment_args)that); + return false; + } + + public boolean equals(increment_args that) { + if (that == null) + return false; + + boolean this_present_increment = true && this.isSetIncrement(); + boolean that_present_increment = true && that.isSetIncrement(); + if (this_present_increment || that_present_increment) { + if (!(this_present_increment && that_present_increment)) + return false; + if (!this.increment.equals(that.increment)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(increment_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + increment_args typedOther = (increment_args)other; + + lastComparison = Boolean.valueOf(isSetIncrement()).compareTo(typedOther.isSetIncrement()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIncrement()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.increment, typedOther.increment); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("increment_args("); + boolean first = true; + + sb.append("increment:"); + if (this.increment == null) { + sb.append("null"); + } else { + sb.append(this.increment); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class increment_argsStandardSchemeFactory implements SchemeFactory { + public increment_argsStandardScheme getScheme() { + return new increment_argsStandardScheme(); + } + } + + private static class increment_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, increment_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // INCREMENT + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.increment = new TIncrement(); + struct.increment.read(iprot); + struct.setIncrementIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, increment_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.increment != null) { + oprot.writeFieldBegin(INCREMENT_FIELD_DESC); + struct.increment.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class increment_argsTupleSchemeFactory implements SchemeFactory { + public increment_argsTupleScheme getScheme() { + return new increment_argsTupleScheme(); + } + } + + private static class increment_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, increment_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIncrement()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIncrement()) { + struct.increment.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, increment_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.increment = new TIncrement(); + struct.increment.read(iprot); + struct.setIncrementIsSet(true); + } + } + } + + } + + public static class increment_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("increment_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new increment_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new increment_resultTupleSchemeFactory()); + } + + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(increment_result.class, metaDataMap); + } + + public increment_result() { + } + + public increment_result( + IOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public increment_result(increment_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public increment_result deepCopy() { + return new increment_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public IOError getIo() { + return this.io; + } + + public increment_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof increment_result) + return this.equals((increment_result)that); + return false; + } + + public boolean equals(increment_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(increment_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + increment_result typedOther = (increment_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("increment_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class increment_resultStandardSchemeFactory implements SchemeFactory { + public increment_resultStandardScheme getScheme() { + return new increment_resultStandardScheme(); + } + } + + private static class increment_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, increment_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, increment_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class increment_resultTupleSchemeFactory implements SchemeFactory { + public increment_resultTupleScheme getScheme() { + return new increment_resultTupleScheme(); + } + } + + private static class increment_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, increment_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, increment_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class incrementRows_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("incrementRows_args"); + + private static final org.apache.thrift.protocol.TField INCREMENTS_FIELD_DESC = new org.apache.thrift.protocol.TField("increments", org.apache.thrift.protocol.TType.LIST, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new incrementRows_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new incrementRows_argsTupleSchemeFactory()); + } + + /** + * The list of increments + */ + public List increments; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * The list of increments + */ + INCREMENTS((short)1, "increments"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // INCREMENTS + return INCREMENTS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.INCREMENTS, new org.apache.thrift.meta_data.FieldMetaData("increments", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TIncrement.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(incrementRows_args.class, metaDataMap); + } + + public incrementRows_args() { + } + + public incrementRows_args( + List increments) + { + this(); + this.increments = increments; + } + + /** + * Performs a deep copy on other. + */ + public incrementRows_args(incrementRows_args other) { + if (other.isSetIncrements()) { + List __this__increments = new ArrayList(); + for (TIncrement other_element : other.increments) { + __this__increments.add(new TIncrement(other_element)); + } + this.increments = __this__increments; + } + } + + public incrementRows_args deepCopy() { + return new incrementRows_args(this); + } + + @Override + public void clear() { + this.increments = null; + } + + public int getIncrementsSize() { + return (this.increments == null) ? 0 : this.increments.size(); + } + + public java.util.Iterator getIncrementsIterator() { + return (this.increments == null) ? null : this.increments.iterator(); + } + + public void addToIncrements(TIncrement elem) { + if (this.increments == null) { + this.increments = new ArrayList(); + } + this.increments.add(elem); + } + + /** + * The list of increments + */ + public List getIncrements() { + return this.increments; + } + + /** + * The list of increments + */ + public incrementRows_args setIncrements(List increments) { + this.increments = increments; + return this; + } + + public void unsetIncrements() { + this.increments = null; + } + + /** Returns true if field increments is set (has been assigned a value) and false otherwise */ + public boolean isSetIncrements() { + return this.increments != null; + } + + public void setIncrementsIsSet(boolean value) { + if (!value) { + this.increments = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case INCREMENTS: + if (value == null) { + unsetIncrements(); + } else { + setIncrements((List)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case INCREMENTS: + return getIncrements(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case INCREMENTS: + return isSetIncrements(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof incrementRows_args) + return this.equals((incrementRows_args)that); + return false; + } + + public boolean equals(incrementRows_args that) { + if (that == null) + return false; + + boolean this_present_increments = true && this.isSetIncrements(); + boolean that_present_increments = true && that.isSetIncrements(); + if (this_present_increments || that_present_increments) { + if (!(this_present_increments && that_present_increments)) + return false; + if (!this.increments.equals(that.increments)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(incrementRows_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + incrementRows_args typedOther = (incrementRows_args)other; + + lastComparison = Boolean.valueOf(isSetIncrements()).compareTo(typedOther.isSetIncrements()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIncrements()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.increments, typedOther.increments); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("incrementRows_args("); + boolean first = true; + + sb.append("increments:"); + if (this.increments == null) { + sb.append("null"); + } else { + sb.append(this.increments); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class incrementRows_argsStandardSchemeFactory implements SchemeFactory { + public incrementRows_argsStandardScheme getScheme() { + return new incrementRows_argsStandardScheme(); + } + } + + private static class incrementRows_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, incrementRows_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // INCREMENTS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list424 = iprot.readListBegin(); + struct.increments = new ArrayList(_list424.size); + for (int _i425 = 0; _i425 < _list424.size; ++_i425) + { + TIncrement _elem426; // optional + _elem426 = new TIncrement(); + _elem426.read(iprot); + struct.increments.add(_elem426); + } + iprot.readListEnd(); + } + struct.setIncrementsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, incrementRows_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.increments != null) { + oprot.writeFieldBegin(INCREMENTS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.increments.size())); + for (TIncrement _iter427 : struct.increments) + { + _iter427.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class incrementRows_argsTupleSchemeFactory implements SchemeFactory { + public incrementRows_argsTupleScheme getScheme() { + return new incrementRows_argsTupleScheme(); + } + } + + private static class incrementRows_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, incrementRows_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIncrements()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIncrements()) { + { + oprot.writeI32(struct.increments.size()); + for (TIncrement _iter428 : struct.increments) + { + _iter428.write(oprot); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, incrementRows_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list429 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.increments = new ArrayList(_list429.size); + for (int _i430 = 0; _i430 < _list429.size; ++_i430) + { + TIncrement _elem431; // optional + _elem431 = new TIncrement(); + _elem431.read(iprot); + struct.increments.add(_elem431); + } + } + struct.setIncrementsIsSet(true); + } + } + } + + } + + public static class incrementRows_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("incrementRows_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new incrementRows_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new incrementRows_resultTupleSchemeFactory()); + } + + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(incrementRows_result.class, metaDataMap); + } + + public incrementRows_result() { + } + + public incrementRows_result( + IOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public incrementRows_result(incrementRows_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public incrementRows_result deepCopy() { + return new incrementRows_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public IOError getIo() { + return this.io; + } + + public incrementRows_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof incrementRows_result) + return this.equals((incrementRows_result)that); + return false; + } + + public boolean equals(incrementRows_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(incrementRows_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + incrementRows_result typedOther = (incrementRows_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("incrementRows_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class incrementRows_resultStandardSchemeFactory implements SchemeFactory { + public incrementRows_resultStandardScheme getScheme() { + return new incrementRows_resultStandardScheme(); + } + } + + private static class incrementRows_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, incrementRows_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, incrementRows_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class incrementRows_resultTupleSchemeFactory implements SchemeFactory { + public incrementRows_resultTupleScheme getScheme() { + return new incrementRows_resultTupleScheme(); + } + } + + private static class incrementRows_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, incrementRows_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, incrementRows_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class deleteAllRowTs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteAllRowTs_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteAllRowTs_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteAllRowTs_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * key of the row to be completely deleted. + */ + public ByteBuffer row; // required + /** + * timestamp + */ + public long timestamp; // required + /** + * Delete attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * key of the row to be completely deleted. + */ + ROW((short)2, "row"), + /** + * timestamp + */ + TIMESTAMP((short)3, "timestamp"), + /** + * Delete attributes + */ + ATTRIBUTES((short)4, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // TIMESTAMP + return TIMESTAMP; + case 4: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteAllRowTs_args.class, metaDataMap); + } + + public deleteAllRowTs_args() { + } + + public deleteAllRowTs_args( + ByteBuffer tableName, + ByteBuffer row, + long timestamp, + Map attributes) + { + this(); + this.tableName = tableName; + this.row = row; + this.timestamp = timestamp; + setTimestampIsSet(true); + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public deleteAllRowTs_args(deleteAllRowTs_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + this.timestamp = other.timestamp; + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public deleteAllRowTs_args deepCopy() { + return new deleteAllRowTs_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public deleteAllRowTs_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public deleteAllRowTs_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * key of the row to be completely deleted. + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * key of the row to be completely deleted. + */ + public deleteAllRowTs_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public deleteAllRowTs_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + /** + * timestamp + */ + public long getTimestamp() { + return this.timestamp; + } + + /** + * timestamp + */ + public deleteAllRowTs_args setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Delete attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Delete attributes + */ + public deleteAllRowTs_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case TIMESTAMP: + return isSetTimestamp(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteAllRowTs_args) + return this.equals((deleteAllRowTs_args)that); + return false; + } + + public boolean equals(deleteAllRowTs_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteAllRowTs_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteAllRowTs_args typedOther = (deleteAllRowTs_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteAllRowTs_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteAllRowTs_argsStandardSchemeFactory implements SchemeFactory { + public deleteAllRowTs_argsStandardScheme getScheme() { + return new deleteAllRowTs_argsStandardScheme(); + } + } + + private static class deleteAllRowTs_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAllRowTs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map432 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map432.size); + for (int _i433 = 0; _i433 < _map432.size; ++_i433) + { + ByteBuffer _key434; // required + ByteBuffer _val435; // optional + _key434 = iprot.readBinary(); + _val435 = iprot.readBinary(); + struct.attributes.put(_key434, _val435); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAllRowTs_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter436 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter436.getKey()); + oprot.writeBinary(_iter436.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteAllRowTs_argsTupleSchemeFactory implements SchemeFactory { + public deleteAllRowTs_argsTupleScheme getScheme() { + return new deleteAllRowTs_argsTupleScheme(); + } + } + + private static class deleteAllRowTs_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteAllRowTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetTimestamp()) { + optionals.set(2); + } + if (struct.isSetAttributes()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter437 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter437.getKey()); + oprot.writeBinary(_iter437.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllRowTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TMap _map438 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map438.size); + for (int _i439 = 0; _i439 < _map438.size; ++_i439) + { + ByteBuffer _key440; // required + ByteBuffer _val441; // optional + _key440 = iprot.readBinary(); + _val441 = iprot.readBinary(); + struct.attributes.put(_key440, _val441); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class deleteAllRowTs_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteAllRowTs_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteAllRowTs_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteAllRowTs_resultTupleSchemeFactory()); + } + + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteAllRowTs_result.class, metaDataMap); + } + + public deleteAllRowTs_result() { + } + + public deleteAllRowTs_result( + IOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public deleteAllRowTs_result(deleteAllRowTs_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public deleteAllRowTs_result deepCopy() { + return new deleteAllRowTs_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public IOError getIo() { + return this.io; + } + + public deleteAllRowTs_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteAllRowTs_result) + return this.equals((deleteAllRowTs_result)that); + return false; + } + + public boolean equals(deleteAllRowTs_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteAllRowTs_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteAllRowTs_result typedOther = (deleteAllRowTs_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteAllRowTs_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteAllRowTs_resultStandardSchemeFactory implements SchemeFactory { + public deleteAllRowTs_resultStandardScheme getScheme() { + return new deleteAllRowTs_resultStandardScheme(); + } + } + + private static class deleteAllRowTs_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteAllRowTs_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteAllRowTs_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteAllRowTs_resultTupleSchemeFactory implements SchemeFactory { + public deleteAllRowTs_resultTupleScheme getScheme() { + return new deleteAllRowTs_resultTupleScheme(); + } + } + + private static class deleteAllRowTs_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteAllRowTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteAllRowTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class scannerOpenWithScan_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerOpenWithScan_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField SCAN_FIELD_DESC = new org.apache.thrift.protocol.TField("scan", org.apache.thrift.protocol.TType.STRUCT, (short)2); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerOpenWithScan_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerOpenWithScan_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * Scan instance + */ + public TScan scan; // required + /** + * Scan attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * Scan instance + */ + SCAN((short)2, "scan"), + /** + * Scan attributes + */ + ATTRIBUTES((short)3, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // SCAN + return SCAN; + case 3: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.SCAN, new org.apache.thrift.meta_data.FieldMetaData("scan", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TScan.class))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerOpenWithScan_args.class, metaDataMap); + } + + public scannerOpenWithScan_args() { + } + + public scannerOpenWithScan_args( + ByteBuffer tableName, + TScan scan, + Map attributes) + { + this(); + this.tableName = tableName; + this.scan = scan; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public scannerOpenWithScan_args(scannerOpenWithScan_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetScan()) { + this.scan = new TScan(other.scan); + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public scannerOpenWithScan_args deepCopy() { + return new scannerOpenWithScan_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.scan = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public scannerOpenWithScan_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public scannerOpenWithScan_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * Scan instance + */ + public TScan getScan() { + return this.scan; + } + + /** + * Scan instance + */ + public scannerOpenWithScan_args setScan(TScan scan) { + this.scan = scan; + return this; + } + + public void unsetScan() { + this.scan = null; + } + + /** Returns true if field scan is set (has been assigned a value) and false otherwise */ + public boolean isSetScan() { + return this.scan != null; + } + + public void setScanIsSet(boolean value) { + if (!value) { + this.scan = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Scan attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Scan attributes + */ + public scannerOpenWithScan_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case SCAN: + if (value == null) { + unsetScan(); + } else { + setScan((TScan)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case SCAN: + return getScan(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case SCAN: + return isSetScan(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerOpenWithScan_args) + return this.equals((scannerOpenWithScan_args)that); + return false; + } + + public boolean equals(scannerOpenWithScan_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_scan = true && this.isSetScan(); + boolean that_present_scan = true && that.isSetScan(); + if (this_present_scan || that_present_scan) { + if (!(this_present_scan && that_present_scan)) + return false; + if (!this.scan.equals(that.scan)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerOpenWithScan_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerOpenWithScan_args typedOther = (scannerOpenWithScan_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetScan()).compareTo(typedOther.isSetScan()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetScan()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.scan, typedOther.scan); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerOpenWithScan_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("scan:"); + if (this.scan == null) { + sb.append("null"); + } else { + sb.append(this.scan); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerOpenWithScan_argsStandardSchemeFactory implements SchemeFactory { + public scannerOpenWithScan_argsStandardScheme getScheme() { + return new scannerOpenWithScan_argsStandardScheme(); + } + } + + private static class scannerOpenWithScan_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithScan_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // SCAN + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.scan = new TScan(); + struct.scan.read(iprot); + struct.setScanIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map442 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map442.size); + for (int _i443 = 0; _i443 < _map442.size; ++_i443) + { + ByteBuffer _key444; // required + ByteBuffer _val445; // optional + _key444 = iprot.readBinary(); + _val445 = iprot.readBinary(); + struct.attributes.put(_key444, _val445); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithScan_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.scan != null) { + oprot.writeFieldBegin(SCAN_FIELD_DESC); + struct.scan.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter446 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter446.getKey()); + oprot.writeBinary(_iter446.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerOpenWithScan_argsTupleSchemeFactory implements SchemeFactory { + public scannerOpenWithScan_argsTupleScheme getScheme() { + return new scannerOpenWithScan_argsTupleScheme(); + } + } + + private static class scannerOpenWithScan_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithScan_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetScan()) { + optionals.set(1); + } + if (struct.isSetAttributes()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetScan()) { + struct.scan.write(oprot); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter447 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter447.getKey()); + oprot.writeBinary(_iter447.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithScan_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.scan = new TScan(); + struct.scan.read(iprot); + struct.setScanIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TMap _map448 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map448.size); + for (int _i449 = 0; _i449 < _map448.size; ++_i449) + { + ByteBuffer _key450; // required + ByteBuffer _val451; // optional + _key450 = iprot.readBinary(); + _val451 = iprot.readBinary(); + struct.attributes.put(_key450, _val451); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class scannerOpenWithScan_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerOpenWithScan_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.I32, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerOpenWithScan_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerOpenWithScan_resultTupleSchemeFactory()); + } + + public int success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32 , "ScannerID"))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerOpenWithScan_result.class, metaDataMap); + } + + public scannerOpenWithScan_result() { + } + + public scannerOpenWithScan_result( + int success, + IOError io) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public scannerOpenWithScan_result(scannerOpenWithScan_result other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.success = other.success; + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public scannerOpenWithScan_result deepCopy() { + return new scannerOpenWithScan_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = 0; + this.io = null; + } + + public int getSuccess() { + return this.success; + } + + public scannerOpenWithScan_result setSuccess(int success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bit_vector.clear(__SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return __isset_bit_vector.get(__SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bit_vector.set(__SUCCESS_ISSET_ID, value); + } + + public IOError getIo() { + return this.io; + } + + public scannerOpenWithScan_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Integer)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Integer.valueOf(getSuccess()); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerOpenWithScan_result) + return this.equals((scannerOpenWithScan_result)that); + return false; + } + + public boolean equals(scannerOpenWithScan_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerOpenWithScan_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerOpenWithScan_result typedOther = (scannerOpenWithScan_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerOpenWithScan_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerOpenWithScan_resultStandardSchemeFactory implements SchemeFactory { + public scannerOpenWithScan_resultStandardScheme getScheme() { + return new scannerOpenWithScan_resultStandardScheme(); + } + } + + private static class scannerOpenWithScan_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithScan_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithScan_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeI32(struct.success); + oprot.writeFieldEnd(); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerOpenWithScan_resultTupleSchemeFactory implements SchemeFactory { + public scannerOpenWithScan_resultTupleScheme getScheme() { + return new scannerOpenWithScan_resultTupleScheme(); + } + } + + private static class scannerOpenWithScan_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithScan_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeI32(struct.success); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithScan_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class scannerOpen_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerOpen_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField START_ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("startRow", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerOpen_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerOpen_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + public ByteBuffer startRow; // required + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + public List columns; // required + /** + * Scan attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + START_ROW((short)2, "startRow"), + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + COLUMNS((short)3, "columns"), + /** + * Scan attributes + */ + ATTRIBUTES((short)4, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // START_ROW + return START_ROW; + case 3: // COLUMNS + return COLUMNS; + case 4: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.START_ROW, new org.apache.thrift.meta_data.FieldMetaData("startRow", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerOpen_args.class, metaDataMap); + } + + public scannerOpen_args() { + } + + public scannerOpen_args( + ByteBuffer tableName, + ByteBuffer startRow, + List columns, + Map attributes) + { + this(); + this.tableName = tableName; + this.startRow = startRow; + this.columns = columns; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public scannerOpen_args(scannerOpen_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetStartRow()) { + this.startRow = other.startRow; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (ByteBuffer other_element : other.columns) { + __this__columns.add(other_element); + } + this.columns = __this__columns; + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public scannerOpen_args deepCopy() { + return new scannerOpen_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.startRow = null; + this.columns = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public scannerOpen_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public scannerOpen_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + public byte[] getStartRow() { + setStartRow(org.apache.thrift.TBaseHelper.rightSize(startRow)); + return startRow == null ? null : startRow.array(); + } + + public ByteBuffer bufferForStartRow() { + return startRow; + } + + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + public scannerOpen_args setStartRow(byte[] startRow) { + setStartRow(startRow == null ? (ByteBuffer)null : ByteBuffer.wrap(startRow)); + return this; + } + + public scannerOpen_args setStartRow(ByteBuffer startRow) { + this.startRow = startRow; + return this; + } + + public void unsetStartRow() { + this.startRow = null; + } + + /** Returns true if field startRow is set (has been assigned a value) and false otherwise */ + public boolean isSetStartRow() { + return this.startRow != null; + } + + public void setStartRowIsSet(boolean value) { + if (!value) { + this.startRow = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(ByteBuffer elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + public List getColumns() { + return this.columns; + } + + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + public scannerOpen_args setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Scan attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Scan attributes + */ + public scannerOpen_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case START_ROW: + if (value == null) { + unsetStartRow(); + } else { + setStartRow((ByteBuffer)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case START_ROW: + return getStartRow(); + + case COLUMNS: + return getColumns(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case START_ROW: + return isSetStartRow(); + case COLUMNS: + return isSetColumns(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerOpen_args) + return this.equals((scannerOpen_args)that); + return false; + } + + public boolean equals(scannerOpen_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_startRow = true && this.isSetStartRow(); + boolean that_present_startRow = true && that.isSetStartRow(); + if (this_present_startRow || that_present_startRow) { + if (!(this_present_startRow && that_present_startRow)) + return false; + if (!this.startRow.equals(that.startRow)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerOpen_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerOpen_args typedOther = (scannerOpen_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetStartRow()).compareTo(typedOther.isSetStartRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStartRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.startRow, typedOther.startRow); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerOpen_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("startRow:"); + if (this.startRow == null) { + sb.append("null"); + } else { + sb.append(this.startRow); + } + first = false; + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerOpen_argsStandardSchemeFactory implements SchemeFactory { + public scannerOpen_argsStandardScheme getScheme() { + return new scannerOpen_argsStandardScheme(); + } + } + + private static class scannerOpen_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpen_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // START_ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.startRow = iprot.readBinary(); + struct.setStartRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list452 = iprot.readListBegin(); + struct.columns = new ArrayList(_list452.size); + for (int _i453 = 0; _i453 < _list452.size; ++_i453) + { + ByteBuffer _elem454; // optional + _elem454 = iprot.readBinary(); + struct.columns.add(_elem454); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map455 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map455.size); + for (int _i456 = 0; _i456 < _map455.size; ++_i456) + { + ByteBuffer _key457; // required + ByteBuffer _val458; // optional + _key457 = iprot.readBinary(); + _val458 = iprot.readBinary(); + struct.attributes.put(_key457, _val458); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpen_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.startRow != null) { + oprot.writeFieldBegin(START_ROW_FIELD_DESC); + oprot.writeBinary(struct.startRow); + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size())); + for (ByteBuffer _iter459 : struct.columns) + { + oprot.writeBinary(_iter459); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter460 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter460.getKey()); + oprot.writeBinary(_iter460.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerOpen_argsTupleSchemeFactory implements SchemeFactory { + public scannerOpen_argsTupleScheme getScheme() { + return new scannerOpen_argsTupleScheme(); + } + } + + private static class scannerOpen_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpen_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetStartRow()) { + optionals.set(1); + } + if (struct.isSetColumns()) { + optionals.set(2); + } + if (struct.isSetAttributes()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetStartRow()) { + oprot.writeBinary(struct.startRow); + } + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (ByteBuffer _iter461 : struct.columns) + { + oprot.writeBinary(_iter461); + } + } + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter462 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter462.getKey()); + oprot.writeBinary(_iter462.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpen_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.startRow = iprot.readBinary(); + struct.setStartRowIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list463 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.columns = new ArrayList(_list463.size); + for (int _i464 = 0; _i464 < _list463.size; ++_i464) + { + ByteBuffer _elem465; // optional + _elem465 = iprot.readBinary(); + struct.columns.add(_elem465); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TMap _map466 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map466.size); + for (int _i467 = 0; _i467 < _map466.size; ++_i467) + { + ByteBuffer _key468; // required + ByteBuffer _val469; // optional + _key468 = iprot.readBinary(); + _val469 = iprot.readBinary(); + struct.attributes.put(_key468, _val469); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class scannerOpen_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerOpen_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.I32, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerOpen_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerOpen_resultTupleSchemeFactory()); + } + + public int success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32 , "ScannerID"))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerOpen_result.class, metaDataMap); + } + + public scannerOpen_result() { + } + + public scannerOpen_result( + int success, + IOError io) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public scannerOpen_result(scannerOpen_result other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.success = other.success; + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public scannerOpen_result deepCopy() { + return new scannerOpen_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = 0; + this.io = null; + } + + public int getSuccess() { + return this.success; + } + + public scannerOpen_result setSuccess(int success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bit_vector.clear(__SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return __isset_bit_vector.get(__SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bit_vector.set(__SUCCESS_ISSET_ID, value); + } + + public IOError getIo() { + return this.io; + } + + public scannerOpen_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Integer)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Integer.valueOf(getSuccess()); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerOpen_result) + return this.equals((scannerOpen_result)that); + return false; + } + + public boolean equals(scannerOpen_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerOpen_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerOpen_result typedOther = (scannerOpen_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerOpen_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerOpen_resultStandardSchemeFactory implements SchemeFactory { + public scannerOpen_resultStandardScheme getScheme() { + return new scannerOpen_resultStandardScheme(); + } + } + + private static class scannerOpen_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpen_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpen_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeI32(struct.success); + oprot.writeFieldEnd(); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerOpen_resultTupleSchemeFactory implements SchemeFactory { + public scannerOpen_resultTupleScheme getScheme() { + return new scannerOpen_resultTupleScheme(); + } + } + + private static class scannerOpen_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpen_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeI32(struct.success); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpen_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class scannerOpenWithStop_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerOpenWithStop_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField START_ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("startRow", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField STOP_ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("stopRow", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)4); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)5); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerOpenWithStop_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerOpenWithStop_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + public ByteBuffer startRow; // required + /** + * row to stop scanning on. This row is *not* included in the + * scanner's results + */ + public ByteBuffer stopRow; // required + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + public List columns; // required + /** + * Scan attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + START_ROW((short)2, "startRow"), + /** + * row to stop scanning on. This row is *not* included in the + * scanner's results + */ + STOP_ROW((short)3, "stopRow"), + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + COLUMNS((short)4, "columns"), + /** + * Scan attributes + */ + ATTRIBUTES((short)5, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // START_ROW + return START_ROW; + case 3: // STOP_ROW + return STOP_ROW; + case 4: // COLUMNS + return COLUMNS; + case 5: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.START_ROW, new org.apache.thrift.meta_data.FieldMetaData("startRow", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.STOP_ROW, new org.apache.thrift.meta_data.FieldMetaData("stopRow", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerOpenWithStop_args.class, metaDataMap); + } + + public scannerOpenWithStop_args() { + } + + public scannerOpenWithStop_args( + ByteBuffer tableName, + ByteBuffer startRow, + ByteBuffer stopRow, + List columns, + Map attributes) + { + this(); + this.tableName = tableName; + this.startRow = startRow; + this.stopRow = stopRow; + this.columns = columns; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public scannerOpenWithStop_args(scannerOpenWithStop_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetStartRow()) { + this.startRow = other.startRow; + } + if (other.isSetStopRow()) { + this.stopRow = other.stopRow; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (ByteBuffer other_element : other.columns) { + __this__columns.add(other_element); + } + this.columns = __this__columns; + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public scannerOpenWithStop_args deepCopy() { + return new scannerOpenWithStop_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.startRow = null; + this.stopRow = null; + this.columns = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public scannerOpenWithStop_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public scannerOpenWithStop_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + public byte[] getStartRow() { + setStartRow(org.apache.thrift.TBaseHelper.rightSize(startRow)); + return startRow == null ? null : startRow.array(); + } + + public ByteBuffer bufferForStartRow() { + return startRow; + } + + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + public scannerOpenWithStop_args setStartRow(byte[] startRow) { + setStartRow(startRow == null ? (ByteBuffer)null : ByteBuffer.wrap(startRow)); + return this; + } + + public scannerOpenWithStop_args setStartRow(ByteBuffer startRow) { + this.startRow = startRow; + return this; + } + + public void unsetStartRow() { + this.startRow = null; + } + + /** Returns true if field startRow is set (has been assigned a value) and false otherwise */ + public boolean isSetStartRow() { + return this.startRow != null; + } + + public void setStartRowIsSet(boolean value) { + if (!value) { + this.startRow = null; + } + } + + /** + * row to stop scanning on. This row is *not* included in the + * scanner's results + */ + public byte[] getStopRow() { + setStopRow(org.apache.thrift.TBaseHelper.rightSize(stopRow)); + return stopRow == null ? null : stopRow.array(); + } + + public ByteBuffer bufferForStopRow() { + return stopRow; + } + + /** + * row to stop scanning on. This row is *not* included in the + * scanner's results + */ + public scannerOpenWithStop_args setStopRow(byte[] stopRow) { + setStopRow(stopRow == null ? (ByteBuffer)null : ByteBuffer.wrap(stopRow)); + return this; + } + + public scannerOpenWithStop_args setStopRow(ByteBuffer stopRow) { + this.stopRow = stopRow; + return this; + } + + public void unsetStopRow() { + this.stopRow = null; + } + + /** Returns true if field stopRow is set (has been assigned a value) and false otherwise */ + public boolean isSetStopRow() { + return this.stopRow != null; + } + + public void setStopRowIsSet(boolean value) { + if (!value) { + this.stopRow = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(ByteBuffer elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + public List getColumns() { + return this.columns; + } + + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + public scannerOpenWithStop_args setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Scan attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Scan attributes + */ + public scannerOpenWithStop_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case START_ROW: + if (value == null) { + unsetStartRow(); + } else { + setStartRow((ByteBuffer)value); + } + break; + + case STOP_ROW: + if (value == null) { + unsetStopRow(); + } else { + setStopRow((ByteBuffer)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case START_ROW: + return getStartRow(); + + case STOP_ROW: + return getStopRow(); + + case COLUMNS: + return getColumns(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case START_ROW: + return isSetStartRow(); + case STOP_ROW: + return isSetStopRow(); + case COLUMNS: + return isSetColumns(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerOpenWithStop_args) + return this.equals((scannerOpenWithStop_args)that); + return false; + } + + public boolean equals(scannerOpenWithStop_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_startRow = true && this.isSetStartRow(); + boolean that_present_startRow = true && that.isSetStartRow(); + if (this_present_startRow || that_present_startRow) { + if (!(this_present_startRow && that_present_startRow)) + return false; + if (!this.startRow.equals(that.startRow)) + return false; + } + + boolean this_present_stopRow = true && this.isSetStopRow(); + boolean that_present_stopRow = true && that.isSetStopRow(); + if (this_present_stopRow || that_present_stopRow) { + if (!(this_present_stopRow && that_present_stopRow)) + return false; + if (!this.stopRow.equals(that.stopRow)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerOpenWithStop_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerOpenWithStop_args typedOther = (scannerOpenWithStop_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetStartRow()).compareTo(typedOther.isSetStartRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStartRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.startRow, typedOther.startRow); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetStopRow()).compareTo(typedOther.isSetStopRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStopRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.stopRow, typedOther.stopRow); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerOpenWithStop_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("startRow:"); + if (this.startRow == null) { + sb.append("null"); + } else { + sb.append(this.startRow); + } + first = false; + if (!first) sb.append(", "); + sb.append("stopRow:"); + if (this.stopRow == null) { + sb.append("null"); + } else { + sb.append(this.stopRow); + } + first = false; + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerOpenWithStop_argsStandardSchemeFactory implements SchemeFactory { + public scannerOpenWithStop_argsStandardScheme getScheme() { + return new scannerOpenWithStop_argsStandardScheme(); + } + } + + private static class scannerOpenWithStop_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithStop_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // START_ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.startRow = iprot.readBinary(); + struct.setStartRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // STOP_ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.stopRow = iprot.readBinary(); + struct.setStopRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list470 = iprot.readListBegin(); + struct.columns = new ArrayList(_list470.size); + for (int _i471 = 0; _i471 < _list470.size; ++_i471) + { + ByteBuffer _elem472; // optional + _elem472 = iprot.readBinary(); + struct.columns.add(_elem472); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map473 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map473.size); + for (int _i474 = 0; _i474 < _map473.size; ++_i474) + { + ByteBuffer _key475; // required + ByteBuffer _val476; // optional + _key475 = iprot.readBinary(); + _val476 = iprot.readBinary(); + struct.attributes.put(_key475, _val476); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithStop_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.startRow != null) { + oprot.writeFieldBegin(START_ROW_FIELD_DESC); + oprot.writeBinary(struct.startRow); + oprot.writeFieldEnd(); + } + if (struct.stopRow != null) { + oprot.writeFieldBegin(STOP_ROW_FIELD_DESC); + oprot.writeBinary(struct.stopRow); + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size())); + for (ByteBuffer _iter477 : struct.columns) + { + oprot.writeBinary(_iter477); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter478 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter478.getKey()); + oprot.writeBinary(_iter478.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerOpenWithStop_argsTupleSchemeFactory implements SchemeFactory { + public scannerOpenWithStop_argsTupleScheme getScheme() { + return new scannerOpenWithStop_argsTupleScheme(); + } + } + + private static class scannerOpenWithStop_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStop_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetStartRow()) { + optionals.set(1); + } + if (struct.isSetStopRow()) { + optionals.set(2); + } + if (struct.isSetColumns()) { + optionals.set(3); + } + if (struct.isSetAttributes()) { + optionals.set(4); + } + oprot.writeBitSet(optionals, 5); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetStartRow()) { + oprot.writeBinary(struct.startRow); + } + if (struct.isSetStopRow()) { + oprot.writeBinary(struct.stopRow); + } + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (ByteBuffer _iter479 : struct.columns) + { + oprot.writeBinary(_iter479); + } + } + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter480 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter480.getKey()); + oprot.writeBinary(_iter480.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStop_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(5); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.startRow = iprot.readBinary(); + struct.setStartRowIsSet(true); + } + if (incoming.get(2)) { + struct.stopRow = iprot.readBinary(); + struct.setStopRowIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TList _list481 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.columns = new ArrayList(_list481.size); + for (int _i482 = 0; _i482 < _list481.size; ++_i482) + { + ByteBuffer _elem483; // optional + _elem483 = iprot.readBinary(); + struct.columns.add(_elem483); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(4)) { + { + org.apache.thrift.protocol.TMap _map484 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map484.size); + for (int _i485 = 0; _i485 < _map484.size; ++_i485) + { + ByteBuffer _key486; // required + ByteBuffer _val487; // optional + _key486 = iprot.readBinary(); + _val487 = iprot.readBinary(); + struct.attributes.put(_key486, _val487); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class scannerOpenWithStop_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerOpenWithStop_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.I32, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerOpenWithStop_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerOpenWithStop_resultTupleSchemeFactory()); + } + + public int success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32 , "ScannerID"))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerOpenWithStop_result.class, metaDataMap); + } + + public scannerOpenWithStop_result() { + } + + public scannerOpenWithStop_result( + int success, + IOError io) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public scannerOpenWithStop_result(scannerOpenWithStop_result other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.success = other.success; + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public scannerOpenWithStop_result deepCopy() { + return new scannerOpenWithStop_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = 0; + this.io = null; + } + + public int getSuccess() { + return this.success; + } + + public scannerOpenWithStop_result setSuccess(int success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bit_vector.clear(__SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return __isset_bit_vector.get(__SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bit_vector.set(__SUCCESS_ISSET_ID, value); + } + + public IOError getIo() { + return this.io; + } + + public scannerOpenWithStop_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Integer)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Integer.valueOf(getSuccess()); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerOpenWithStop_result) + return this.equals((scannerOpenWithStop_result)that); + return false; + } + + public boolean equals(scannerOpenWithStop_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerOpenWithStop_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerOpenWithStop_result typedOther = (scannerOpenWithStop_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerOpenWithStop_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerOpenWithStop_resultStandardSchemeFactory implements SchemeFactory { + public scannerOpenWithStop_resultStandardScheme getScheme() { + return new scannerOpenWithStop_resultStandardScheme(); + } + } + + private static class scannerOpenWithStop_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithStop_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithStop_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeI32(struct.success); + oprot.writeFieldEnd(); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerOpenWithStop_resultTupleSchemeFactory implements SchemeFactory { + public scannerOpenWithStop_resultTupleScheme getScheme() { + return new scannerOpenWithStop_resultTupleScheme(); + } + } + + private static class scannerOpenWithStop_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStop_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeI32(struct.success); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStop_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class scannerOpenWithPrefix_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerOpenWithPrefix_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField START_AND_PREFIX_FIELD_DESC = new org.apache.thrift.protocol.TField("startAndPrefix", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerOpenWithPrefix_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerOpenWithPrefix_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * the prefix (and thus start row) of the keys you want + */ + public ByteBuffer startAndPrefix; // required + /** + * the columns you want returned + */ + public List columns; // required + /** + * Scan attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * the prefix (and thus start row) of the keys you want + */ + START_AND_PREFIX((short)2, "startAndPrefix"), + /** + * the columns you want returned + */ + COLUMNS((short)3, "columns"), + /** + * Scan attributes + */ + ATTRIBUTES((short)4, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // START_AND_PREFIX + return START_AND_PREFIX; + case 3: // COLUMNS + return COLUMNS; + case 4: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.START_AND_PREFIX, new org.apache.thrift.meta_data.FieldMetaData("startAndPrefix", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerOpenWithPrefix_args.class, metaDataMap); + } + + public scannerOpenWithPrefix_args() { + } + + public scannerOpenWithPrefix_args( + ByteBuffer tableName, + ByteBuffer startAndPrefix, + List columns, + Map attributes) + { + this(); + this.tableName = tableName; + this.startAndPrefix = startAndPrefix; + this.columns = columns; + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public scannerOpenWithPrefix_args(scannerOpenWithPrefix_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetStartAndPrefix()) { + this.startAndPrefix = other.startAndPrefix; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (ByteBuffer other_element : other.columns) { + __this__columns.add(other_element); + } + this.columns = __this__columns; + } + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public scannerOpenWithPrefix_args deepCopy() { + return new scannerOpenWithPrefix_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.startAndPrefix = null; + this.columns = null; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public scannerOpenWithPrefix_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public scannerOpenWithPrefix_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * the prefix (and thus start row) of the keys you want + */ + public byte[] getStartAndPrefix() { + setStartAndPrefix(org.apache.thrift.TBaseHelper.rightSize(startAndPrefix)); + return startAndPrefix == null ? null : startAndPrefix.array(); + } + + public ByteBuffer bufferForStartAndPrefix() { + return startAndPrefix; + } + + /** + * the prefix (and thus start row) of the keys you want + */ + public scannerOpenWithPrefix_args setStartAndPrefix(byte[] startAndPrefix) { + setStartAndPrefix(startAndPrefix == null ? (ByteBuffer)null : ByteBuffer.wrap(startAndPrefix)); + return this; + } + + public scannerOpenWithPrefix_args setStartAndPrefix(ByteBuffer startAndPrefix) { + this.startAndPrefix = startAndPrefix; + return this; + } + + public void unsetStartAndPrefix() { + this.startAndPrefix = null; + } + + /** Returns true if field startAndPrefix is set (has been assigned a value) and false otherwise */ + public boolean isSetStartAndPrefix() { + return this.startAndPrefix != null; + } + + public void setStartAndPrefixIsSet(boolean value) { + if (!value) { + this.startAndPrefix = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(ByteBuffer elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + /** + * the columns you want returned + */ + public List getColumns() { + return this.columns; + } + + /** + * the columns you want returned + */ + public scannerOpenWithPrefix_args setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Scan attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Scan attributes + */ + public scannerOpenWithPrefix_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case START_AND_PREFIX: + if (value == null) { + unsetStartAndPrefix(); + } else { + setStartAndPrefix((ByteBuffer)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case START_AND_PREFIX: + return getStartAndPrefix(); + + case COLUMNS: + return getColumns(); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case START_AND_PREFIX: + return isSetStartAndPrefix(); + case COLUMNS: + return isSetColumns(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerOpenWithPrefix_args) + return this.equals((scannerOpenWithPrefix_args)that); + return false; + } + + public boolean equals(scannerOpenWithPrefix_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_startAndPrefix = true && this.isSetStartAndPrefix(); + boolean that_present_startAndPrefix = true && that.isSetStartAndPrefix(); + if (this_present_startAndPrefix || that_present_startAndPrefix) { + if (!(this_present_startAndPrefix && that_present_startAndPrefix)) + return false; + if (!this.startAndPrefix.equals(that.startAndPrefix)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerOpenWithPrefix_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerOpenWithPrefix_args typedOther = (scannerOpenWithPrefix_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetStartAndPrefix()).compareTo(typedOther.isSetStartAndPrefix()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStartAndPrefix()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.startAndPrefix, typedOther.startAndPrefix); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerOpenWithPrefix_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("startAndPrefix:"); + if (this.startAndPrefix == null) { + sb.append("null"); + } else { + sb.append(this.startAndPrefix); + } + first = false; + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerOpenWithPrefix_argsStandardSchemeFactory implements SchemeFactory { + public scannerOpenWithPrefix_argsStandardScheme getScheme() { + return new scannerOpenWithPrefix_argsStandardScheme(); + } + } + + private static class scannerOpenWithPrefix_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithPrefix_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // START_AND_PREFIX + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.startAndPrefix = iprot.readBinary(); + struct.setStartAndPrefixIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list488 = iprot.readListBegin(); + struct.columns = new ArrayList(_list488.size); + for (int _i489 = 0; _i489 < _list488.size; ++_i489) + { + ByteBuffer _elem490; // optional + _elem490 = iprot.readBinary(); + struct.columns.add(_elem490); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map491 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map491.size); + for (int _i492 = 0; _i492 < _map491.size; ++_i492) + { + ByteBuffer _key493; // required + ByteBuffer _val494; // optional + _key493 = iprot.readBinary(); + _val494 = iprot.readBinary(); + struct.attributes.put(_key493, _val494); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithPrefix_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.startAndPrefix != null) { + oprot.writeFieldBegin(START_AND_PREFIX_FIELD_DESC); + oprot.writeBinary(struct.startAndPrefix); + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size())); + for (ByteBuffer _iter495 : struct.columns) + { + oprot.writeBinary(_iter495); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter496 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter496.getKey()); + oprot.writeBinary(_iter496.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerOpenWithPrefix_argsTupleSchemeFactory implements SchemeFactory { + public scannerOpenWithPrefix_argsTupleScheme getScheme() { + return new scannerOpenWithPrefix_argsTupleScheme(); + } + } + + private static class scannerOpenWithPrefix_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithPrefix_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetStartAndPrefix()) { + optionals.set(1); + } + if (struct.isSetColumns()) { + optionals.set(2); + } + if (struct.isSetAttributes()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetStartAndPrefix()) { + oprot.writeBinary(struct.startAndPrefix); + } + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (ByteBuffer _iter497 : struct.columns) + { + oprot.writeBinary(_iter497); + } + } + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter498 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter498.getKey()); + oprot.writeBinary(_iter498.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithPrefix_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.startAndPrefix = iprot.readBinary(); + struct.setStartAndPrefixIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list499 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.columns = new ArrayList(_list499.size); + for (int _i500 = 0; _i500 < _list499.size; ++_i500) + { + ByteBuffer _elem501; // optional + _elem501 = iprot.readBinary(); + struct.columns.add(_elem501); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TMap _map502 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map502.size); + for (int _i503 = 0; _i503 < _map502.size; ++_i503) + { + ByteBuffer _key504; // required + ByteBuffer _val505; // optional + _key504 = iprot.readBinary(); + _val505 = iprot.readBinary(); + struct.attributes.put(_key504, _val505); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class scannerOpenWithPrefix_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerOpenWithPrefix_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.I32, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerOpenWithPrefix_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerOpenWithPrefix_resultTupleSchemeFactory()); + } + + public int success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32 , "ScannerID"))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerOpenWithPrefix_result.class, metaDataMap); + } + + public scannerOpenWithPrefix_result() { + } + + public scannerOpenWithPrefix_result( + int success, + IOError io) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public scannerOpenWithPrefix_result(scannerOpenWithPrefix_result other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.success = other.success; + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public scannerOpenWithPrefix_result deepCopy() { + return new scannerOpenWithPrefix_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = 0; + this.io = null; + } + + public int getSuccess() { + return this.success; + } + + public scannerOpenWithPrefix_result setSuccess(int success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bit_vector.clear(__SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return __isset_bit_vector.get(__SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bit_vector.set(__SUCCESS_ISSET_ID, value); + } + + public IOError getIo() { + return this.io; + } + + public scannerOpenWithPrefix_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Integer)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Integer.valueOf(getSuccess()); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerOpenWithPrefix_result) + return this.equals((scannerOpenWithPrefix_result)that); + return false; + } + + public boolean equals(scannerOpenWithPrefix_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerOpenWithPrefix_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerOpenWithPrefix_result typedOther = (scannerOpenWithPrefix_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerOpenWithPrefix_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerOpenWithPrefix_resultStandardSchemeFactory implements SchemeFactory { + public scannerOpenWithPrefix_resultStandardScheme getScheme() { + return new scannerOpenWithPrefix_resultStandardScheme(); + } + } + + private static class scannerOpenWithPrefix_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithPrefix_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithPrefix_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeI32(struct.success); + oprot.writeFieldEnd(); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerOpenWithPrefix_resultTupleSchemeFactory implements SchemeFactory { + public scannerOpenWithPrefix_resultTupleScheme getScheme() { + return new scannerOpenWithPrefix_resultTupleScheme(); + } + } + + private static class scannerOpenWithPrefix_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithPrefix_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeI32(struct.success); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithPrefix_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class scannerOpenTs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerOpenTs_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField START_ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("startRow", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)4); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)5); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerOpenTs_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerOpenTs_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + public ByteBuffer startRow; // required + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + public List columns; // required + /** + * timestamp + */ + public long timestamp; // required + /** + * Scan attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + START_ROW((short)2, "startRow"), + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + COLUMNS((short)3, "columns"), + /** + * timestamp + */ + TIMESTAMP((short)4, "timestamp"), + /** + * Scan attributes + */ + ATTRIBUTES((short)5, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // START_ROW + return START_ROW; + case 3: // COLUMNS + return COLUMNS; + case 4: // TIMESTAMP + return TIMESTAMP; + case 5: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.START_ROW, new org.apache.thrift.meta_data.FieldMetaData("startRow", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerOpenTs_args.class, metaDataMap); + } + + public scannerOpenTs_args() { + } + + public scannerOpenTs_args( + ByteBuffer tableName, + ByteBuffer startRow, + List columns, + long timestamp, + Map attributes) + { + this(); + this.tableName = tableName; + this.startRow = startRow; + this.columns = columns; + this.timestamp = timestamp; + setTimestampIsSet(true); + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public scannerOpenTs_args(scannerOpenTs_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetStartRow()) { + this.startRow = other.startRow; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (ByteBuffer other_element : other.columns) { + __this__columns.add(other_element); + } + this.columns = __this__columns; + } + this.timestamp = other.timestamp; + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public scannerOpenTs_args deepCopy() { + return new scannerOpenTs_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.startRow = null; + this.columns = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public scannerOpenTs_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public scannerOpenTs_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + public byte[] getStartRow() { + setStartRow(org.apache.thrift.TBaseHelper.rightSize(startRow)); + return startRow == null ? null : startRow.array(); + } + + public ByteBuffer bufferForStartRow() { + return startRow; + } + + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + public scannerOpenTs_args setStartRow(byte[] startRow) { + setStartRow(startRow == null ? (ByteBuffer)null : ByteBuffer.wrap(startRow)); + return this; + } + + public scannerOpenTs_args setStartRow(ByteBuffer startRow) { + this.startRow = startRow; + return this; + } + + public void unsetStartRow() { + this.startRow = null; + } + + /** Returns true if field startRow is set (has been assigned a value) and false otherwise */ + public boolean isSetStartRow() { + return this.startRow != null; + } + + public void setStartRowIsSet(boolean value) { + if (!value) { + this.startRow = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(ByteBuffer elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + public List getColumns() { + return this.columns; + } + + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + public scannerOpenTs_args setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + /** + * timestamp + */ + public long getTimestamp() { + return this.timestamp; + } + + /** + * timestamp + */ + public scannerOpenTs_args setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Scan attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Scan attributes + */ + public scannerOpenTs_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case START_ROW: + if (value == null) { + unsetStartRow(); + } else { + setStartRow((ByteBuffer)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case START_ROW: + return getStartRow(); + + case COLUMNS: + return getColumns(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case START_ROW: + return isSetStartRow(); + case COLUMNS: + return isSetColumns(); + case TIMESTAMP: + return isSetTimestamp(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerOpenTs_args) + return this.equals((scannerOpenTs_args)that); + return false; + } + + public boolean equals(scannerOpenTs_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_startRow = true && this.isSetStartRow(); + boolean that_present_startRow = true && that.isSetStartRow(); + if (this_present_startRow || that_present_startRow) { + if (!(this_present_startRow && that_present_startRow)) + return false; + if (!this.startRow.equals(that.startRow)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerOpenTs_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerOpenTs_args typedOther = (scannerOpenTs_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetStartRow()).compareTo(typedOther.isSetStartRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStartRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.startRow, typedOther.startRow); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerOpenTs_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("startRow:"); + if (this.startRow == null) { + sb.append("null"); + } else { + sb.append(this.startRow); + } + first = false; + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerOpenTs_argsStandardSchemeFactory implements SchemeFactory { + public scannerOpenTs_argsStandardScheme getScheme() { + return new scannerOpenTs_argsStandardScheme(); + } + } + + private static class scannerOpenTs_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenTs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // START_ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.startRow = iprot.readBinary(); + struct.setStartRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list506 = iprot.readListBegin(); + struct.columns = new ArrayList(_list506.size); + for (int _i507 = 0; _i507 < _list506.size; ++_i507) + { + ByteBuffer _elem508; // optional + _elem508 = iprot.readBinary(); + struct.columns.add(_elem508); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map509 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map509.size); + for (int _i510 = 0; _i510 < _map509.size; ++_i510) + { + ByteBuffer _key511; // required + ByteBuffer _val512; // optional + _key511 = iprot.readBinary(); + _val512 = iprot.readBinary(); + struct.attributes.put(_key511, _val512); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenTs_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.startRow != null) { + oprot.writeFieldBegin(START_ROW_FIELD_DESC); + oprot.writeBinary(struct.startRow); + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size())); + for (ByteBuffer _iter513 : struct.columns) + { + oprot.writeBinary(_iter513); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter514 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter514.getKey()); + oprot.writeBinary(_iter514.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerOpenTs_argsTupleSchemeFactory implements SchemeFactory { + public scannerOpenTs_argsTupleScheme getScheme() { + return new scannerOpenTs_argsTupleScheme(); + } + } + + private static class scannerOpenTs_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetStartRow()) { + optionals.set(1); + } + if (struct.isSetColumns()) { + optionals.set(2); + } + if (struct.isSetTimestamp()) { + optionals.set(3); + } + if (struct.isSetAttributes()) { + optionals.set(4); + } + oprot.writeBitSet(optionals, 5); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetStartRow()) { + oprot.writeBinary(struct.startRow); + } + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (ByteBuffer _iter515 : struct.columns) + { + oprot.writeBinary(_iter515); + } + } + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter516 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter516.getKey()); + oprot.writeBinary(_iter516.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(5); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.startRow = iprot.readBinary(); + struct.setStartRowIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list517 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.columns = new ArrayList(_list517.size); + for (int _i518 = 0; _i518 < _list517.size; ++_i518) + { + ByteBuffer _elem519; // optional + _elem519 = iprot.readBinary(); + struct.columns.add(_elem519); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(3)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(4)) { + { + org.apache.thrift.protocol.TMap _map520 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map520.size); + for (int _i521 = 0; _i521 < _map520.size; ++_i521) + { + ByteBuffer _key522; // required + ByteBuffer _val523; // optional + _key522 = iprot.readBinary(); + _val523 = iprot.readBinary(); + struct.attributes.put(_key522, _val523); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class scannerOpenTs_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerOpenTs_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.I32, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerOpenTs_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerOpenTs_resultTupleSchemeFactory()); + } + + public int success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32 , "ScannerID"))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerOpenTs_result.class, metaDataMap); + } + + public scannerOpenTs_result() { + } + + public scannerOpenTs_result( + int success, + IOError io) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public scannerOpenTs_result(scannerOpenTs_result other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.success = other.success; + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public scannerOpenTs_result deepCopy() { + return new scannerOpenTs_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = 0; + this.io = null; + } + + public int getSuccess() { + return this.success; + } + + public scannerOpenTs_result setSuccess(int success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bit_vector.clear(__SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return __isset_bit_vector.get(__SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bit_vector.set(__SUCCESS_ISSET_ID, value); + } + + public IOError getIo() { + return this.io; + } + + public scannerOpenTs_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Integer)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Integer.valueOf(getSuccess()); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerOpenTs_result) + return this.equals((scannerOpenTs_result)that); + return false; + } + + public boolean equals(scannerOpenTs_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerOpenTs_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerOpenTs_result typedOther = (scannerOpenTs_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerOpenTs_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerOpenTs_resultStandardSchemeFactory implements SchemeFactory { + public scannerOpenTs_resultStandardScheme getScheme() { + return new scannerOpenTs_resultStandardScheme(); + } + } + + private static class scannerOpenTs_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenTs_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenTs_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeI32(struct.success); + oprot.writeFieldEnd(); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerOpenTs_resultTupleSchemeFactory implements SchemeFactory { + public scannerOpenTs_resultTupleScheme getScheme() { + return new scannerOpenTs_resultTupleScheme(); + } + } + + private static class scannerOpenTs_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeI32(struct.success); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class scannerOpenWithStopTs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerOpenWithStopTs_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField START_ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("startRow", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField STOP_ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("stopRow", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)4); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)5); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)6); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerOpenWithStopTs_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerOpenWithStopTs_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + public ByteBuffer startRow; // required + /** + * row to stop scanning on. This row is *not* included in the + * scanner's results + */ + public ByteBuffer stopRow; // required + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + public List columns; // required + /** + * timestamp + */ + public long timestamp; // required + /** + * Scan attributes + */ + public Map attributes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + START_ROW((short)2, "startRow"), + /** + * row to stop scanning on. This row is *not* included in the + * scanner's results + */ + STOP_ROW((short)3, "stopRow"), + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + COLUMNS((short)4, "columns"), + /** + * timestamp + */ + TIMESTAMP((short)5, "timestamp"), + /** + * Scan attributes + */ + ATTRIBUTES((short)6, "attributes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // START_ROW + return START_ROW; + case 3: // STOP_ROW + return STOP_ROW; + case 4: // COLUMNS + return COLUMNS; + case 5: // TIMESTAMP + return TIMESTAMP; + case 6: // ATTRIBUTES + return ATTRIBUTES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.START_ROW, new org.apache.thrift.meta_data.FieldMetaData("startRow", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.STOP_ROW, new org.apache.thrift.meta_data.FieldMetaData("stopRow", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerOpenWithStopTs_args.class, metaDataMap); + } + + public scannerOpenWithStopTs_args() { + } + + public scannerOpenWithStopTs_args( + ByteBuffer tableName, + ByteBuffer startRow, + ByteBuffer stopRow, + List columns, + long timestamp, + Map attributes) + { + this(); + this.tableName = tableName; + this.startRow = startRow; + this.stopRow = stopRow; + this.columns = columns; + this.timestamp = timestamp; + setTimestampIsSet(true); + this.attributes = attributes; + } + + /** + * Performs a deep copy on other. + */ + public scannerOpenWithStopTs_args(scannerOpenWithStopTs_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetStartRow()) { + this.startRow = other.startRow; + } + if (other.isSetStopRow()) { + this.stopRow = other.stopRow; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (ByteBuffer other_element : other.columns) { + __this__columns.add(other_element); + } + this.columns = __this__columns; + } + this.timestamp = other.timestamp; + if (other.isSetAttributes()) { + Map __this__attributes = new HashMap(); + for (Map.Entry other_element : other.attributes.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + ByteBuffer other_element_value = other_element.getValue(); + + ByteBuffer __this__attributes_copy_key = other_element_key; + + ByteBuffer __this__attributes_copy_value = other_element_value; + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + } + + public scannerOpenWithStopTs_args deepCopy() { + return new scannerOpenWithStopTs_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.startRow = null; + this.stopRow = null; + this.columns = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.attributes = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public scannerOpenWithStopTs_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public scannerOpenWithStopTs_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + public byte[] getStartRow() { + setStartRow(org.apache.thrift.TBaseHelper.rightSize(startRow)); + return startRow == null ? null : startRow.array(); + } + + public ByteBuffer bufferForStartRow() { + return startRow; + } + + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + public scannerOpenWithStopTs_args setStartRow(byte[] startRow) { + setStartRow(startRow == null ? (ByteBuffer)null : ByteBuffer.wrap(startRow)); + return this; + } + + public scannerOpenWithStopTs_args setStartRow(ByteBuffer startRow) { + this.startRow = startRow; + return this; + } + + public void unsetStartRow() { + this.startRow = null; + } + + /** Returns true if field startRow is set (has been assigned a value) and false otherwise */ + public boolean isSetStartRow() { + return this.startRow != null; + } + + public void setStartRowIsSet(boolean value) { + if (!value) { + this.startRow = null; + } + } + + /** + * row to stop scanning on. This row is *not* included in the + * scanner's results + */ + public byte[] getStopRow() { + setStopRow(org.apache.thrift.TBaseHelper.rightSize(stopRow)); + return stopRow == null ? null : stopRow.array(); + } + + public ByteBuffer bufferForStopRow() { + return stopRow; + } + + /** + * row to stop scanning on. This row is *not* included in the + * scanner's results + */ + public scannerOpenWithStopTs_args setStopRow(byte[] stopRow) { + setStopRow(stopRow == null ? (ByteBuffer)null : ByteBuffer.wrap(stopRow)); + return this; + } + + public scannerOpenWithStopTs_args setStopRow(ByteBuffer stopRow) { + this.stopRow = stopRow; + return this; + } + + public void unsetStopRow() { + this.stopRow = null; + } + + /** Returns true if field stopRow is set (has been assigned a value) and false otherwise */ + public boolean isSetStopRow() { + return this.stopRow != null; + } + + public void setStopRowIsSet(boolean value) { + if (!value) { + this.stopRow = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(ByteBuffer elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + public List getColumns() { + return this.columns; + } + + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + public scannerOpenWithStopTs_args setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + /** + * timestamp + */ + public long getTimestamp() { + return this.timestamp; + } + + /** + * timestamp + */ + public scannerOpenWithStopTs_args setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(ByteBuffer key, ByteBuffer val) { + if (this.attributes == null) { + this.attributes = new HashMap(); + } + this.attributes.put(key, val); + } + + /** + * Scan attributes + */ + public Map getAttributes() { + return this.attributes; + } + + /** + * Scan attributes + */ + public scannerOpenWithStopTs_args setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case START_ROW: + if (value == null) { + unsetStartRow(); + } else { + setStartRow((ByteBuffer)value); + } + break; + + case STOP_ROW: + if (value == null) { + unsetStopRow(); + } else { + setStopRow((ByteBuffer)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case START_ROW: + return getStartRow(); + + case STOP_ROW: + return getStopRow(); + + case COLUMNS: + return getColumns(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case ATTRIBUTES: + return getAttributes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case START_ROW: + return isSetStartRow(); + case STOP_ROW: + return isSetStopRow(); + case COLUMNS: + return isSetColumns(); + case TIMESTAMP: + return isSetTimestamp(); + case ATTRIBUTES: + return isSetAttributes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerOpenWithStopTs_args) + return this.equals((scannerOpenWithStopTs_args)that); + return false; + } + + public boolean equals(scannerOpenWithStopTs_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_startRow = true && this.isSetStartRow(); + boolean that_present_startRow = true && that.isSetStartRow(); + if (this_present_startRow || that_present_startRow) { + if (!(this_present_startRow && that_present_startRow)) + return false; + if (!this.startRow.equals(that.startRow)) + return false; + } + + boolean this_present_stopRow = true && this.isSetStopRow(); + boolean that_present_stopRow = true && that.isSetStopRow(); + if (this_present_stopRow || that_present_stopRow) { + if (!(this_present_stopRow && that_present_stopRow)) + return false; + if (!this.stopRow.equals(that.stopRow)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerOpenWithStopTs_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerOpenWithStopTs_args typedOther = (scannerOpenWithStopTs_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetStartRow()).compareTo(typedOther.isSetStartRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStartRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.startRow, typedOther.startRow); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetStopRow()).compareTo(typedOther.isSetStopRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStopRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.stopRow, typedOther.stopRow); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerOpenWithStopTs_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("startRow:"); + if (this.startRow == null) { + sb.append("null"); + } else { + sb.append(this.startRow); + } + first = false; + if (!first) sb.append(", "); + sb.append("stopRow:"); + if (this.stopRow == null) { + sb.append("null"); + } else { + sb.append(this.stopRow); + } + first = false; + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerOpenWithStopTs_argsStandardSchemeFactory implements SchemeFactory { + public scannerOpenWithStopTs_argsStandardScheme getScheme() { + return new scannerOpenWithStopTs_argsStandardScheme(); + } + } + + private static class scannerOpenWithStopTs_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithStopTs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // START_ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.startRow = iprot.readBinary(); + struct.setStartRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // STOP_ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.stopRow = iprot.readBinary(); + struct.setStopRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list524 = iprot.readListBegin(); + struct.columns = new ArrayList(_list524.size); + for (int _i525 = 0; _i525 < _list524.size; ++_i525) + { + ByteBuffer _elem526; // optional + _elem526 = iprot.readBinary(); + struct.columns.add(_elem526); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 6: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map527 = iprot.readMapBegin(); + struct.attributes = new HashMap(2*_map527.size); + for (int _i528 = 0; _i528 < _map527.size; ++_i528) + { + ByteBuffer _key529; // required + ByteBuffer _val530; // optional + _key529 = iprot.readBinary(); + _val530 = iprot.readBinary(); + struct.attributes.put(_key529, _val530); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithStopTs_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.startRow != null) { + oprot.writeFieldBegin(START_ROW_FIELD_DESC); + oprot.writeBinary(struct.startRow); + oprot.writeFieldEnd(); + } + if (struct.stopRow != null) { + oprot.writeFieldBegin(STOP_ROW_FIELD_DESC); + oprot.writeBinary(struct.stopRow); + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size())); + for (ByteBuffer _iter531 : struct.columns) + { + oprot.writeBinary(_iter531); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.attributes.size())); + for (Map.Entry _iter532 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter532.getKey()); + oprot.writeBinary(_iter532.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerOpenWithStopTs_argsTupleSchemeFactory implements SchemeFactory { + public scannerOpenWithStopTs_argsTupleScheme getScheme() { + return new scannerOpenWithStopTs_argsTupleScheme(); + } + } + + private static class scannerOpenWithStopTs_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStopTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetStartRow()) { + optionals.set(1); + } + if (struct.isSetStopRow()) { + optionals.set(2); + } + if (struct.isSetColumns()) { + optionals.set(3); + } + if (struct.isSetTimestamp()) { + optionals.set(4); + } + if (struct.isSetAttributes()) { + optionals.set(5); + } + oprot.writeBitSet(optionals, 6); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetStartRow()) { + oprot.writeBinary(struct.startRow); + } + if (struct.isSetStopRow()) { + oprot.writeBinary(struct.stopRow); + } + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (ByteBuffer _iter533 : struct.columns) + { + oprot.writeBinary(_iter533); + } + } + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry _iter534 : struct.attributes.entrySet()) + { + oprot.writeBinary(_iter534.getKey()); + oprot.writeBinary(_iter534.getValue()); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStopTs_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(6); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.startRow = iprot.readBinary(); + struct.setStartRowIsSet(true); + } + if (incoming.get(2)) { + struct.stopRow = iprot.readBinary(); + struct.setStopRowIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TList _list535 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.columns = new ArrayList(_list535.size); + for (int _i536 = 0; _i536 < _list535.size; ++_i536) + { + ByteBuffer _elem537; // optional + _elem537 = iprot.readBinary(); + struct.columns.add(_elem537); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(4)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(5)) { + { + org.apache.thrift.protocol.TMap _map538 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.attributes = new HashMap(2*_map538.size); + for (int _i539 = 0; _i539 < _map538.size; ++_i539) + { + ByteBuffer _key540; // required + ByteBuffer _val541; // optional + _key540 = iprot.readBinary(); + _val541 = iprot.readBinary(); + struct.attributes.put(_key540, _val541); + } + } + struct.setAttributesIsSet(true); + } + } + } + + } + + public static class scannerOpenWithStopTs_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerOpenWithStopTs_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.I32, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerOpenWithStopTs_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerOpenWithStopTs_resultTupleSchemeFactory()); + } + + public int success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32 , "ScannerID"))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerOpenWithStopTs_result.class, metaDataMap); + } + + public scannerOpenWithStopTs_result() { + } + + public scannerOpenWithStopTs_result( + int success, + IOError io) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public scannerOpenWithStopTs_result(scannerOpenWithStopTs_result other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.success = other.success; + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public scannerOpenWithStopTs_result deepCopy() { + return new scannerOpenWithStopTs_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = 0; + this.io = null; + } + + public int getSuccess() { + return this.success; + } + + public scannerOpenWithStopTs_result setSuccess(int success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bit_vector.clear(__SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return __isset_bit_vector.get(__SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bit_vector.set(__SUCCESS_ISSET_ID, value); + } + + public IOError getIo() { + return this.io; + } + + public scannerOpenWithStopTs_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Integer)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Integer.valueOf(getSuccess()); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerOpenWithStopTs_result) + return this.equals((scannerOpenWithStopTs_result)that); + return false; + } + + public boolean equals(scannerOpenWithStopTs_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerOpenWithStopTs_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerOpenWithStopTs_result typedOther = (scannerOpenWithStopTs_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerOpenWithStopTs_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerOpenWithStopTs_resultStandardSchemeFactory implements SchemeFactory { + public scannerOpenWithStopTs_resultStandardScheme getScheme() { + return new scannerOpenWithStopTs_resultStandardScheme(); + } + } + + private static class scannerOpenWithStopTs_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerOpenWithStopTs_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerOpenWithStopTs_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeI32(struct.success); + oprot.writeFieldEnd(); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerOpenWithStopTs_resultTupleSchemeFactory implements SchemeFactory { + public scannerOpenWithStopTs_resultTupleScheme getScheme() { + return new scannerOpenWithStopTs_resultTupleScheme(); + } + } + + private static class scannerOpenWithStopTs_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStopTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeI32(struct.success); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerOpenWithStopTs_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class scannerGet_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerGet_args"); + + private static final org.apache.thrift.protocol.TField ID_FIELD_DESC = new org.apache.thrift.protocol.TField("id", org.apache.thrift.protocol.TType.I32, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerGet_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerGet_argsTupleSchemeFactory()); + } + + /** + * id of a scanner returned by scannerOpen + */ + public int id; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * id of a scanner returned by scannerOpen + */ + ID((short)1, "id"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // ID + return ID; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __ID_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.ID, new org.apache.thrift.meta_data.FieldMetaData("id", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32 , "ScannerID"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerGet_args.class, metaDataMap); + } + + public scannerGet_args() { + } + + public scannerGet_args( + int id) + { + this(); + this.id = id; + setIdIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public scannerGet_args(scannerGet_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.id = other.id; + } + + public scannerGet_args deepCopy() { + return new scannerGet_args(this); + } + + @Override + public void clear() { + setIdIsSet(false); + this.id = 0; + } + + /** + * id of a scanner returned by scannerOpen + */ + public int getId() { + return this.id; + } + + /** + * id of a scanner returned by scannerOpen + */ + public scannerGet_args setId(int id) { + this.id = id; + setIdIsSet(true); + return this; + } + + public void unsetId() { + __isset_bit_vector.clear(__ID_ISSET_ID); + } + + /** Returns true if field id is set (has been assigned a value) and false otherwise */ + public boolean isSetId() { + return __isset_bit_vector.get(__ID_ISSET_ID); + } + + public void setIdIsSet(boolean value) { + __isset_bit_vector.set(__ID_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case ID: + if (value == null) { + unsetId(); + } else { + setId((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case ID: + return Integer.valueOf(getId()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case ID: + return isSetId(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerGet_args) + return this.equals((scannerGet_args)that); + return false; + } + + public boolean equals(scannerGet_args that) { + if (that == null) + return false; + + boolean this_present_id = true; + boolean that_present_id = true; + if (this_present_id || that_present_id) { + if (!(this_present_id && that_present_id)) + return false; + if (this.id != that.id) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerGet_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerGet_args typedOther = (scannerGet_args)other; + + lastComparison = Boolean.valueOf(isSetId()).compareTo(typedOther.isSetId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.id, typedOther.id); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerGet_args("); + boolean first = true; + + sb.append("id:"); + sb.append(this.id); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerGet_argsStandardSchemeFactory implements SchemeFactory { + public scannerGet_argsStandardScheme getScheme() { + return new scannerGet_argsStandardScheme(); + } + } + + private static class scannerGet_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerGet_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // ID + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.id = iprot.readI32(); + struct.setIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerGet_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(ID_FIELD_DESC); + oprot.writeI32(struct.id); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerGet_argsTupleSchemeFactory implements SchemeFactory { + public scannerGet_argsTupleScheme getScheme() { + return new scannerGet_argsTupleScheme(); + } + } + + private static class scannerGet_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerGet_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetId()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetId()) { + oprot.writeI32(struct.id); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerGet_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.id = iprot.readI32(); + struct.setIdIsSet(true); + } + } + } + + } + + public static class scannerGet_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerGet_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField IA_FIELD_DESC = new org.apache.thrift.protocol.TField("ia", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerGet_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerGet_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + public IllegalArgument ia; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"), + IA((short)2, "ia"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + case 2: // IA + return IA; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowResult.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + tmpMap.put(_Fields.IA, new org.apache.thrift.meta_data.FieldMetaData("ia", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerGet_result.class, metaDataMap); + } + + public scannerGet_result() { + } + + public scannerGet_result( + List success, + IOError io, + IllegalArgument ia) + { + this(); + this.success = success; + this.io = io; + this.ia = ia; + } + + /** + * Performs a deep copy on other. + */ + public scannerGet_result(scannerGet_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TRowResult other_element : other.success) { + __this__success.add(new TRowResult(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + if (other.isSetIa()) { + this.ia = new IllegalArgument(other.ia); + } + } + + public scannerGet_result deepCopy() { + return new scannerGet_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + this.ia = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TRowResult elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public scannerGet_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public scannerGet_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public IllegalArgument getIa() { + return this.ia; + } + + public scannerGet_result setIa(IllegalArgument ia) { + this.ia = ia; + return this; + } + + public void unsetIa() { + this.ia = null; + } + + /** Returns true if field ia is set (has been assigned a value) and false otherwise */ + public boolean isSetIa() { + return this.ia != null; + } + + public void setIaIsSet(boolean value) { + if (!value) { + this.ia = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + case IA: + if (value == null) { + unsetIa(); + } else { + setIa((IllegalArgument)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + case IA: + return getIa(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + case IA: + return isSetIa(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerGet_result) + return this.equals((scannerGet_result)that); + return false; + } + + public boolean equals(scannerGet_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + boolean this_present_ia = true && this.isSetIa(); + boolean that_present_ia = true && that.isSetIa(); + if (this_present_ia || that_present_ia) { + if (!(this_present_ia && that_present_ia)) + return false; + if (!this.ia.equals(that.ia)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerGet_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerGet_result typedOther = (scannerGet_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIa()).compareTo(typedOther.isSetIa()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIa()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ia, typedOther.ia); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerGet_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + if (!first) sb.append(", "); + sb.append("ia:"); + if (this.ia == null) { + sb.append("null"); + } else { + sb.append(this.ia); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerGet_resultStandardSchemeFactory implements SchemeFactory { + public scannerGet_resultStandardScheme getScheme() { + return new scannerGet_resultStandardScheme(); + } + } + + private static class scannerGet_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerGet_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list542 = iprot.readListBegin(); + struct.success = new ArrayList(_list542.size); + for (int _i543 = 0; _i543 < _list542.size; ++_i543) + { + TRowResult _elem544; // optional + _elem544 = new TRowResult(); + _elem544.read(iprot); + struct.success.add(_elem544); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // IA + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerGet_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TRowResult _iter545 : struct.success) + { + _iter545.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.ia != null) { + oprot.writeFieldBegin(IA_FIELD_DESC); + struct.ia.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerGet_resultTupleSchemeFactory implements SchemeFactory { + public scannerGet_resultTupleScheme getScheme() { + return new scannerGet_resultTupleScheme(); + } + } + + private static class scannerGet_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerGet_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + if (struct.isSetIa()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TRowResult _iter546 : struct.success) + { + _iter546.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + if (struct.isSetIa()) { + struct.ia.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerGet_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list547 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list547.size); + for (int _i548 = 0; _i548 < _list547.size; ++_i548) + { + TRowResult _elem549; // optional + _elem549 = new TRowResult(); + _elem549.read(iprot); + struct.success.add(_elem549); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + if (incoming.get(2)) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } + } + } + + } + + public static class scannerGetList_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerGetList_args"); + + private static final org.apache.thrift.protocol.TField ID_FIELD_DESC = new org.apache.thrift.protocol.TField("id", org.apache.thrift.protocol.TType.I32, (short)1); + private static final org.apache.thrift.protocol.TField NB_ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("nbRows", org.apache.thrift.protocol.TType.I32, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerGetList_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerGetList_argsTupleSchemeFactory()); + } + + /** + * id of a scanner returned by scannerOpen + */ + public int id; // required + /** + * number of results to return + */ + public int nbRows; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * id of a scanner returned by scannerOpen + */ + ID((short)1, "id"), + /** + * number of results to return + */ + NB_ROWS((short)2, "nbRows"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // ID + return ID; + case 2: // NB_ROWS + return NB_ROWS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __ID_ISSET_ID = 0; + private static final int __NBROWS_ISSET_ID = 1; + private BitSet __isset_bit_vector = new BitSet(2); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.ID, new org.apache.thrift.meta_data.FieldMetaData("id", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32 , "ScannerID"))); + tmpMap.put(_Fields.NB_ROWS, new org.apache.thrift.meta_data.FieldMetaData("nbRows", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerGetList_args.class, metaDataMap); + } + + public scannerGetList_args() { + } + + public scannerGetList_args( + int id, + int nbRows) + { + this(); + this.id = id; + setIdIsSet(true); + this.nbRows = nbRows; + setNbRowsIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public scannerGetList_args(scannerGetList_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.id = other.id; + this.nbRows = other.nbRows; + } + + public scannerGetList_args deepCopy() { + return new scannerGetList_args(this); + } + + @Override + public void clear() { + setIdIsSet(false); + this.id = 0; + setNbRowsIsSet(false); + this.nbRows = 0; + } + + /** + * id of a scanner returned by scannerOpen + */ + public int getId() { + return this.id; + } + + /** + * id of a scanner returned by scannerOpen + */ + public scannerGetList_args setId(int id) { + this.id = id; + setIdIsSet(true); + return this; + } + + public void unsetId() { + __isset_bit_vector.clear(__ID_ISSET_ID); + } + + /** Returns true if field id is set (has been assigned a value) and false otherwise */ + public boolean isSetId() { + return __isset_bit_vector.get(__ID_ISSET_ID); + } + + public void setIdIsSet(boolean value) { + __isset_bit_vector.set(__ID_ISSET_ID, value); + } + + /** + * number of results to return + */ + public int getNbRows() { + return this.nbRows; + } + + /** + * number of results to return + */ + public scannerGetList_args setNbRows(int nbRows) { + this.nbRows = nbRows; + setNbRowsIsSet(true); + return this; + } + + public void unsetNbRows() { + __isset_bit_vector.clear(__NBROWS_ISSET_ID); + } + + /** Returns true if field nbRows is set (has been assigned a value) and false otherwise */ + public boolean isSetNbRows() { + return __isset_bit_vector.get(__NBROWS_ISSET_ID); + } + + public void setNbRowsIsSet(boolean value) { + __isset_bit_vector.set(__NBROWS_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case ID: + if (value == null) { + unsetId(); + } else { + setId((Integer)value); + } + break; + + case NB_ROWS: + if (value == null) { + unsetNbRows(); + } else { + setNbRows((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case ID: + return Integer.valueOf(getId()); + + case NB_ROWS: + return Integer.valueOf(getNbRows()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case ID: + return isSetId(); + case NB_ROWS: + return isSetNbRows(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerGetList_args) + return this.equals((scannerGetList_args)that); + return false; + } + + public boolean equals(scannerGetList_args that) { + if (that == null) + return false; + + boolean this_present_id = true; + boolean that_present_id = true; + if (this_present_id || that_present_id) { + if (!(this_present_id && that_present_id)) + return false; + if (this.id != that.id) + return false; + } + + boolean this_present_nbRows = true; + boolean that_present_nbRows = true; + if (this_present_nbRows || that_present_nbRows) { + if (!(this_present_nbRows && that_present_nbRows)) + return false; + if (this.nbRows != that.nbRows) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerGetList_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerGetList_args typedOther = (scannerGetList_args)other; + + lastComparison = Boolean.valueOf(isSetId()).compareTo(typedOther.isSetId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.id, typedOther.id); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetNbRows()).compareTo(typedOther.isSetNbRows()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetNbRows()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.nbRows, typedOther.nbRows); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerGetList_args("); + boolean first = true; + + sb.append("id:"); + sb.append(this.id); + first = false; + if (!first) sb.append(", "); + sb.append("nbRows:"); + sb.append(this.nbRows); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerGetList_argsStandardSchemeFactory implements SchemeFactory { + public scannerGetList_argsStandardScheme getScheme() { + return new scannerGetList_argsStandardScheme(); + } + } + + private static class scannerGetList_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerGetList_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // ID + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.id = iprot.readI32(); + struct.setIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // NB_ROWS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.nbRows = iprot.readI32(); + struct.setNbRowsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerGetList_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(ID_FIELD_DESC); + oprot.writeI32(struct.id); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(NB_ROWS_FIELD_DESC); + oprot.writeI32(struct.nbRows); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerGetList_argsTupleSchemeFactory implements SchemeFactory { + public scannerGetList_argsTupleScheme getScheme() { + return new scannerGetList_argsTupleScheme(); + } + } + + private static class scannerGetList_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerGetList_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetId()) { + optionals.set(0); + } + if (struct.isSetNbRows()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetId()) { + oprot.writeI32(struct.id); + } + if (struct.isSetNbRows()) { + oprot.writeI32(struct.nbRows); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerGetList_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.id = iprot.readI32(); + struct.setIdIsSet(true); + } + if (incoming.get(1)) { + struct.nbRows = iprot.readI32(); + struct.setNbRowsIsSet(true); + } + } + } + + } + + public static class scannerGetList_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerGetList_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField IA_FIELD_DESC = new org.apache.thrift.protocol.TField("ia", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerGetList_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerGetList_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + public IllegalArgument ia; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"), + IA((short)2, "ia"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + case 2: // IA + return IA; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRowResult.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + tmpMap.put(_Fields.IA, new org.apache.thrift.meta_data.FieldMetaData("ia", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerGetList_result.class, metaDataMap); + } + + public scannerGetList_result() { + } + + public scannerGetList_result( + List success, + IOError io, + IllegalArgument ia) + { + this(); + this.success = success; + this.io = io; + this.ia = ia; + } + + /** + * Performs a deep copy on other. + */ + public scannerGetList_result(scannerGetList_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TRowResult other_element : other.success) { + __this__success.add(new TRowResult(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + if (other.isSetIa()) { + this.ia = new IllegalArgument(other.ia); + } + } + + public scannerGetList_result deepCopy() { + return new scannerGetList_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + this.ia = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TRowResult elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public scannerGetList_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public scannerGetList_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public IllegalArgument getIa() { + return this.ia; + } + + public scannerGetList_result setIa(IllegalArgument ia) { + this.ia = ia; + return this; + } + + public void unsetIa() { + this.ia = null; + } + + /** Returns true if field ia is set (has been assigned a value) and false otherwise */ + public boolean isSetIa() { + return this.ia != null; + } + + public void setIaIsSet(boolean value) { + if (!value) { + this.ia = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + case IA: + if (value == null) { + unsetIa(); + } else { + setIa((IllegalArgument)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + case IA: + return getIa(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + case IA: + return isSetIa(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerGetList_result) + return this.equals((scannerGetList_result)that); + return false; + } + + public boolean equals(scannerGetList_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + boolean this_present_ia = true && this.isSetIa(); + boolean that_present_ia = true && that.isSetIa(); + if (this_present_ia || that_present_ia) { + if (!(this_present_ia && that_present_ia)) + return false; + if (!this.ia.equals(that.ia)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerGetList_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerGetList_result typedOther = (scannerGetList_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIa()).compareTo(typedOther.isSetIa()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIa()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ia, typedOther.ia); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerGetList_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + if (!first) sb.append(", "); + sb.append("ia:"); + if (this.ia == null) { + sb.append("null"); + } else { + sb.append(this.ia); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerGetList_resultStandardSchemeFactory implements SchemeFactory { + public scannerGetList_resultStandardScheme getScheme() { + return new scannerGetList_resultStandardScheme(); + } + } + + private static class scannerGetList_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerGetList_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list550 = iprot.readListBegin(); + struct.success = new ArrayList(_list550.size); + for (int _i551 = 0; _i551 < _list550.size; ++_i551) + { + TRowResult _elem552; // optional + _elem552 = new TRowResult(); + _elem552.read(iprot); + struct.success.add(_elem552); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // IA + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerGetList_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TRowResult _iter553 : struct.success) + { + _iter553.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.ia != null) { + oprot.writeFieldBegin(IA_FIELD_DESC); + struct.ia.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerGetList_resultTupleSchemeFactory implements SchemeFactory { + public scannerGetList_resultTupleScheme getScheme() { + return new scannerGetList_resultTupleScheme(); + } + } + + private static class scannerGetList_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerGetList_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + if (struct.isSetIa()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TRowResult _iter554 : struct.success) + { + _iter554.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + if (struct.isSetIa()) { + struct.ia.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerGetList_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list555 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list555.size); + for (int _i556 = 0; _i556 < _list555.size; ++_i556) + { + TRowResult _elem557; // optional + _elem557 = new TRowResult(); + _elem557.read(iprot); + struct.success.add(_elem557); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + if (incoming.get(2)) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } + } + } + + } + + public static class scannerClose_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerClose_args"); + + private static final org.apache.thrift.protocol.TField ID_FIELD_DESC = new org.apache.thrift.protocol.TField("id", org.apache.thrift.protocol.TType.I32, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerClose_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerClose_argsTupleSchemeFactory()); + } + + /** + * id of a scanner returned by scannerOpen + */ + public int id; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * id of a scanner returned by scannerOpen + */ + ID((short)1, "id"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // ID + return ID; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __ID_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.ID, new org.apache.thrift.meta_data.FieldMetaData("id", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32 , "ScannerID"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerClose_args.class, metaDataMap); + } + + public scannerClose_args() { + } + + public scannerClose_args( + int id) + { + this(); + this.id = id; + setIdIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public scannerClose_args(scannerClose_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.id = other.id; + } + + public scannerClose_args deepCopy() { + return new scannerClose_args(this); + } + + @Override + public void clear() { + setIdIsSet(false); + this.id = 0; + } + + /** + * id of a scanner returned by scannerOpen + */ + public int getId() { + return this.id; + } + + /** + * id of a scanner returned by scannerOpen + */ + public scannerClose_args setId(int id) { + this.id = id; + setIdIsSet(true); + return this; + } + + public void unsetId() { + __isset_bit_vector.clear(__ID_ISSET_ID); + } + + /** Returns true if field id is set (has been assigned a value) and false otherwise */ + public boolean isSetId() { + return __isset_bit_vector.get(__ID_ISSET_ID); + } + + public void setIdIsSet(boolean value) { + __isset_bit_vector.set(__ID_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case ID: + if (value == null) { + unsetId(); + } else { + setId((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case ID: + return Integer.valueOf(getId()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case ID: + return isSetId(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerClose_args) + return this.equals((scannerClose_args)that); + return false; + } + + public boolean equals(scannerClose_args that) { + if (that == null) + return false; + + boolean this_present_id = true; + boolean that_present_id = true; + if (this_present_id || that_present_id) { + if (!(this_present_id && that_present_id)) + return false; + if (this.id != that.id) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerClose_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerClose_args typedOther = (scannerClose_args)other; + + lastComparison = Boolean.valueOf(isSetId()).compareTo(typedOther.isSetId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.id, typedOther.id); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerClose_args("); + boolean first = true; + + sb.append("id:"); + sb.append(this.id); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerClose_argsStandardSchemeFactory implements SchemeFactory { + public scannerClose_argsStandardScheme getScheme() { + return new scannerClose_argsStandardScheme(); + } + } + + private static class scannerClose_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerClose_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // ID + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.id = iprot.readI32(); + struct.setIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerClose_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(ID_FIELD_DESC); + oprot.writeI32(struct.id); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerClose_argsTupleSchemeFactory implements SchemeFactory { + public scannerClose_argsTupleScheme getScheme() { + return new scannerClose_argsTupleScheme(); + } + } + + private static class scannerClose_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerClose_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetId()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetId()) { + oprot.writeI32(struct.id); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerClose_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.id = iprot.readI32(); + struct.setIdIsSet(true); + } + } + } + + } + + public static class scannerClose_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("scannerClose_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField IA_FIELD_DESC = new org.apache.thrift.protocol.TField("ia", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new scannerClose_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new scannerClose_resultTupleSchemeFactory()); + } + + public IOError io; // required + public IllegalArgument ia; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"), + IA((short)2, "ia"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + case 2: // IA + return IA; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + tmpMap.put(_Fields.IA, new org.apache.thrift.meta_data.FieldMetaData("ia", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(scannerClose_result.class, metaDataMap); + } + + public scannerClose_result() { + } + + public scannerClose_result( + IOError io, + IllegalArgument ia) + { + this(); + this.io = io; + this.ia = ia; + } + + /** + * Performs a deep copy on other. + */ + public scannerClose_result(scannerClose_result other) { + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + if (other.isSetIa()) { + this.ia = new IllegalArgument(other.ia); + } + } + + public scannerClose_result deepCopy() { + return new scannerClose_result(this); + } + + @Override + public void clear() { + this.io = null; + this.ia = null; + } + + public IOError getIo() { + return this.io; + } + + public scannerClose_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public IllegalArgument getIa() { + return this.ia; + } + + public scannerClose_result setIa(IllegalArgument ia) { + this.ia = ia; + return this; + } + + public void unsetIa() { + this.ia = null; + } + + /** Returns true if field ia is set (has been assigned a value) and false otherwise */ + public boolean isSetIa() { + return this.ia != null; + } + + public void setIaIsSet(boolean value) { + if (!value) { + this.ia = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + case IA: + if (value == null) { + unsetIa(); + } else { + setIa((IllegalArgument)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + case IA: + return getIa(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + case IA: + return isSetIa(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof scannerClose_result) + return this.equals((scannerClose_result)that); + return false; + } + + public boolean equals(scannerClose_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + boolean this_present_ia = true && this.isSetIa(); + boolean that_present_ia = true && that.isSetIa(); + if (this_present_ia || that_present_ia) { + if (!(this_present_ia && that_present_ia)) + return false; + if (!this.ia.equals(that.ia)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(scannerClose_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + scannerClose_result typedOther = (scannerClose_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIa()).compareTo(typedOther.isSetIa()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIa()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ia, typedOther.ia); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("scannerClose_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + if (!first) sb.append(", "); + sb.append("ia:"); + if (this.ia == null) { + sb.append("null"); + } else { + sb.append(this.ia); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class scannerClose_resultStandardSchemeFactory implements SchemeFactory { + public scannerClose_resultStandardScheme getScheme() { + return new scannerClose_resultStandardScheme(); + } + } + + private static class scannerClose_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, scannerClose_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // IA + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, scannerClose_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.ia != null) { + oprot.writeFieldBegin(IA_FIELD_DESC); + struct.ia.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class scannerClose_resultTupleSchemeFactory implements SchemeFactory { + public scannerClose_resultTupleScheme getScheme() { + return new scannerClose_resultTupleScheme(); + } + } + + private static class scannerClose_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, scannerClose_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + if (struct.isSetIa()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + if (struct.isSetIa()) { + struct.ia.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, scannerClose_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + if (incoming.get(1)) { + struct.ia = new IllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } + } + } + + } + + public static class getRowOrBefore_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowOrBefore_args"); + + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField FAMILY_FIELD_DESC = new org.apache.thrift.protocol.TField("family", org.apache.thrift.protocol.TType.STRING, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowOrBefore_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowOrBefore_argsTupleSchemeFactory()); + } + + /** + * name of table + */ + public ByteBuffer tableName; // required + /** + * row key + */ + public ByteBuffer row; // required + /** + * column name + */ + public ByteBuffer family; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * name of table + */ + TABLE_NAME((short)1, "tableName"), + /** + * row key + */ + ROW((short)2, "row"), + /** + * column name + */ + FAMILY((short)3, "family"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_NAME + return TABLE_NAME; + case 2: // ROW + return ROW; + case 3: // FAMILY + return FAMILY; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.FAMILY, new org.apache.thrift.meta_data.FieldMetaData("family", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowOrBefore_args.class, metaDataMap); + } + + public getRowOrBefore_args() { + } + + public getRowOrBefore_args( + ByteBuffer tableName, + ByteBuffer row, + ByteBuffer family) + { + this(); + this.tableName = tableName; + this.row = row; + this.family = family; + } + + /** + * Performs a deep copy on other. + */ + public getRowOrBefore_args(getRowOrBefore_args other) { + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetFamily()) { + this.family = other.family; + } + } + + public getRowOrBefore_args deepCopy() { + return new getRowOrBefore_args(this); + } + + @Override + public void clear() { + this.tableName = null; + this.row = null; + this.family = null; + } + + /** + * name of table + */ + public byte[] getTableName() { + setTableName(org.apache.thrift.TBaseHelper.rightSize(tableName)); + return tableName == null ? null : tableName.array(); + } + + public ByteBuffer bufferForTableName() { + return tableName; + } + + /** + * name of table + */ + public getRowOrBefore_args setTableName(byte[] tableName) { + setTableName(tableName == null ? (ByteBuffer)null : ByteBuffer.wrap(tableName)); + return this; + } + + public getRowOrBefore_args setTableName(ByteBuffer tableName) { + this.tableName = tableName; + return this; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + /** + * row key + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row key + */ + public getRowOrBefore_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public getRowOrBefore_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + /** + * column name + */ + public byte[] getFamily() { + setFamily(org.apache.thrift.TBaseHelper.rightSize(family)); + return family == null ? null : family.array(); + } + + public ByteBuffer bufferForFamily() { + return family; + } + + /** + * column name + */ + public getRowOrBefore_args setFamily(byte[] family) { + setFamily(family == null ? (ByteBuffer)null : ByteBuffer.wrap(family)); + return this; + } + + public getRowOrBefore_args setFamily(ByteBuffer family) { + this.family = family; + return this; + } + + public void unsetFamily() { + this.family = null; + } + + /** Returns true if field family is set (has been assigned a value) and false otherwise */ + public boolean isSetFamily() { + return this.family != null; + } + + public void setFamilyIsSet(boolean value) { + if (!value) { + this.family = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case FAMILY: + if (value == null) { + unsetFamily(); + } else { + setFamily((ByteBuffer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_NAME: + return getTableName(); + + case ROW: + return getRow(); + + case FAMILY: + return getFamily(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_NAME: + return isSetTableName(); + case ROW: + return isSetRow(); + case FAMILY: + return isSetFamily(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowOrBefore_args) + return this.equals((getRowOrBefore_args)that); + return false; + } + + public boolean equals(getRowOrBefore_args that) { + if (that == null) + return false; + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_family = true && this.isSetFamily(); + boolean that_present_family = true && that.isSetFamily(); + if (this_present_family || that_present_family) { + if (!(this_present_family && that_present_family)) + return false; + if (!this.family.equals(that.family)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowOrBefore_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowOrBefore_args typedOther = (getRowOrBefore_args)other; + + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(typedOther.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, typedOther.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetFamily()).compareTo(typedOther.isSetFamily()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetFamily()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.family, typedOther.family); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowOrBefore_args("); + boolean first = true; + + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("family:"); + if (this.family == null) { + sb.append("null"); + } else { + sb.append(this.family); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowOrBefore_argsStandardSchemeFactory implements SchemeFactory { + public getRowOrBefore_argsStandardScheme getScheme() { + return new getRowOrBefore_argsStandardScheme(); + } + } + + private static class getRowOrBefore_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowOrBefore_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // FAMILY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.family = iprot.readBinary(); + struct.setFamilyIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowOrBefore_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeBinary(struct.tableName); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.family != null) { + oprot.writeFieldBegin(FAMILY_FIELD_DESC); + oprot.writeBinary(struct.family); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowOrBefore_argsTupleSchemeFactory implements SchemeFactory { + public getRowOrBefore_argsTupleScheme getScheme() { + return new getRowOrBefore_argsTupleScheme(); + } + } + + private static class getRowOrBefore_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowOrBefore_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTableName()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetFamily()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetTableName()) { + oprot.writeBinary(struct.tableName); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetFamily()) { + oprot.writeBinary(struct.family); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowOrBefore_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.tableName = iprot.readBinary(); + struct.setTableNameIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + struct.family = iprot.readBinary(); + struct.setFamilyIsSet(true); + } + } + } + + } + + public static class getRowOrBefore_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRowOrBefore_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRowOrBefore_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRowOrBefore_resultTupleSchemeFactory()); + } + + public List success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCell.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRowOrBefore_result.class, metaDataMap); + } + + public getRowOrBefore_result() { + } + + public getRowOrBefore_result( + List success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getRowOrBefore_result(getRowOrBefore_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TCell other_element : other.success) { + __this__success.add(new TCell(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getRowOrBefore_result deepCopy() { + return new getRowOrBefore_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TCell elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getRowOrBefore_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getRowOrBefore_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRowOrBefore_result) + return this.equals((getRowOrBefore_result)that); + return false; + } + + public boolean equals(getRowOrBefore_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRowOrBefore_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRowOrBefore_result typedOther = (getRowOrBefore_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRowOrBefore_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRowOrBefore_resultStandardSchemeFactory implements SchemeFactory { + public getRowOrBefore_resultStandardScheme getScheme() { + return new getRowOrBefore_resultStandardScheme(); + } + } + + private static class getRowOrBefore_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRowOrBefore_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list558 = iprot.readListBegin(); + struct.success = new ArrayList(_list558.size); + for (int _i559 = 0; _i559 < _list558.size; ++_i559) + { + TCell _elem560; // optional + _elem560 = new TCell(); + _elem560.read(iprot); + struct.success.add(_elem560); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRowOrBefore_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TCell _iter561 : struct.success) + { + _iter561.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRowOrBefore_resultTupleSchemeFactory implements SchemeFactory { + public getRowOrBefore_resultTupleScheme getScheme() { + return new getRowOrBefore_resultTupleScheme(); + } + } + + private static class getRowOrBefore_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRowOrBefore_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TCell _iter562 : struct.success) + { + _iter562.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRowOrBefore_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list563 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list563.size); + for (int _i564 = 0; _i564 < _list563.size; ++_i564) + { + TCell _elem565; // optional + _elem565 = new TCell(); + _elem565.read(iprot); + struct.success.add(_elem565); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getRegionInfo_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRegionInfo_args"); + + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRegionInfo_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRegionInfo_argsTupleSchemeFactory()); + } + + /** + * row key + */ + public ByteBuffer row; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * row key + */ + ROW((short)1, "row"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // ROW + return ROW; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRegionInfo_args.class, metaDataMap); + } + + public getRegionInfo_args() { + } + + public getRegionInfo_args( + ByteBuffer row) + { + this(); + this.row = row; + } + + /** + * Performs a deep copy on other. + */ + public getRegionInfo_args(getRegionInfo_args other) { + if (other.isSetRow()) { + this.row = other.row; + } + } + + public getRegionInfo_args deepCopy() { + return new getRegionInfo_args(this); + } + + @Override + public void clear() { + this.row = null; + } + + /** + * row key + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row key + */ + public getRegionInfo_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public getRegionInfo_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case ROW: + return getRow(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case ROW: + return isSetRow(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRegionInfo_args) + return this.equals((getRegionInfo_args)that); + return false; + } + + public boolean equals(getRegionInfo_args that) { + if (that == null) + return false; + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRegionInfo_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRegionInfo_args typedOther = (getRegionInfo_args)other; + + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRegionInfo_args("); + boolean first = true; + + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRegionInfo_argsStandardSchemeFactory implements SchemeFactory { + public getRegionInfo_argsStandardScheme getScheme() { + return new getRegionInfo_argsStandardScheme(); + } + } + + private static class getRegionInfo_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRegionInfo_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRegionInfo_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRegionInfo_argsTupleSchemeFactory implements SchemeFactory { + public getRegionInfo_argsTupleScheme getScheme() { + return new getRegionInfo_argsTupleScheme(); + } + } + + private static class getRegionInfo_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRegionInfo_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetRow()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRegionInfo_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + } + } + + } + + public static class getRegionInfo_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRegionInfo_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getRegionInfo_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getRegionInfo_resultTupleSchemeFactory()); + } + + public TRegionInfo success; // required + public IOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TRegionInfo.class))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRegionInfo_result.class, metaDataMap); + } + + public getRegionInfo_result() { + } + + public getRegionInfo_result( + TRegionInfo success, + IOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getRegionInfo_result(getRegionInfo_result other) { + if (other.isSetSuccess()) { + this.success = new TRegionInfo(other.success); + } + if (other.isSetIo()) { + this.io = new IOError(other.io); + } + } + + public getRegionInfo_result deepCopy() { + return new getRegionInfo_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public TRegionInfo getSuccess() { + return this.success; + } + + public getRegionInfo_result setSuccess(TRegionInfo success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public IOError getIo() { + return this.io; + } + + public getRegionInfo_result setIo(IOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((TRegionInfo)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((IOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getRegionInfo_result) + return this.equals((getRegionInfo_result)that); + return false; + } + + public boolean equals(getRegionInfo_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getRegionInfo_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getRegionInfo_result typedOther = (getRegionInfo_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getRegionInfo_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getRegionInfo_resultStandardSchemeFactory implements SchemeFactory { + public getRegionInfo_resultStandardScheme getScheme() { + return new getRegionInfo_resultStandardScheme(); + } + } + + private static class getRegionInfo_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getRegionInfo_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.success = new TRegionInfo(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getRegionInfo_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + struct.success.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getRegionInfo_resultTupleSchemeFactory implements SchemeFactory { + public getRegionInfo_resultTupleScheme getScheme() { + return new getRegionInfo_resultTupleScheme(); + } + } + + private static class getRegionInfo_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getRegionInfo_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + struct.success.write(oprot); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getRegionInfo_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = new TRegionInfo(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new IOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/IOError.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/IOError.java new file mode 100644 index 0000000..11e31e3 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/IOError.java @@ -0,0 +1,387 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An IOError exception signals that an error occurred communicating + * to the Hbase master or an Hbase region server. Also used to return + * more general Hbase error conditions. + */ +public class IOError extends Exception implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("IOError"); + + private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new IOErrorStandardSchemeFactory()); + schemes.put(TupleScheme.class, new IOErrorTupleSchemeFactory()); + } + + public String message; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + MESSAGE((short)1, "message"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // MESSAGE + return MESSAGE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(IOError.class, metaDataMap); + } + + public IOError() { + } + + public IOError( + String message) + { + this(); + this.message = message; + } + + /** + * Performs a deep copy on other. + */ + public IOError(IOError other) { + if (other.isSetMessage()) { + this.message = other.message; + } + } + + public IOError deepCopy() { + return new IOError(this); + } + + @Override + public void clear() { + this.message = null; + } + + public String getMessage() { + return this.message; + } + + public IOError setMessage(String message) { + this.message = message; + return this; + } + + public void unsetMessage() { + this.message = null; + } + + /** Returns true if field message is set (has been assigned a value) and false otherwise */ + public boolean isSetMessage() { + return this.message != null; + } + + public void setMessageIsSet(boolean value) { + if (!value) { + this.message = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case MESSAGE: + if (value == null) { + unsetMessage(); + } else { + setMessage((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case MESSAGE: + return getMessage(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case MESSAGE: + return isSetMessage(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof IOError) + return this.equals((IOError)that); + return false; + } + + public boolean equals(IOError that) { + if (that == null) + return false; + + boolean this_present_message = true && this.isSetMessage(); + boolean that_present_message = true && that.isSetMessage(); + if (this_present_message || that_present_message) { + if (!(this_present_message && that_present_message)) + return false; + if (!this.message.equals(that.message)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(IOError other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + IOError typedOther = (IOError)other; + + lastComparison = Boolean.valueOf(isSetMessage()).compareTo(typedOther.isSetMessage()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMessage()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.message, typedOther.message); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("IOError("); + boolean first = true; + + sb.append("message:"); + if (this.message == null) { + sb.append("null"); + } else { + sb.append(this.message); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class IOErrorStandardSchemeFactory implements SchemeFactory { + public IOErrorStandardScheme getScheme() { + return new IOErrorStandardScheme(); + } + } + + private static class IOErrorStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, IOError struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // MESSAGE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.message = iprot.readString(); + struct.setMessageIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, IOError struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.message != null) { + oprot.writeFieldBegin(MESSAGE_FIELD_DESC); + oprot.writeString(struct.message); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class IOErrorTupleSchemeFactory implements SchemeFactory { + public IOErrorTupleScheme getScheme() { + return new IOErrorTupleScheme(); + } + } + + private static class IOErrorTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, IOError struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetMessage()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetMessage()) { + oprot.writeString(struct.message); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, IOError struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.message = iprot.readString(); + struct.setMessageIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/IllegalArgument.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/IllegalArgument.java new file mode 100644 index 0000000..ede215f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/IllegalArgument.java @@ -0,0 +1,386 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An IllegalArgument exception indicates an illegal or invalid + * argument was passed into a procedure. + */ +public class IllegalArgument extends Exception implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("IllegalArgument"); + + private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new IllegalArgumentStandardSchemeFactory()); + schemes.put(TupleScheme.class, new IllegalArgumentTupleSchemeFactory()); + } + + public String message; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + MESSAGE((short)1, "message"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // MESSAGE + return MESSAGE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(IllegalArgument.class, metaDataMap); + } + + public IllegalArgument() { + } + + public IllegalArgument( + String message) + { + this(); + this.message = message; + } + + /** + * Performs a deep copy on other. + */ + public IllegalArgument(IllegalArgument other) { + if (other.isSetMessage()) { + this.message = other.message; + } + } + + public IllegalArgument deepCopy() { + return new IllegalArgument(this); + } + + @Override + public void clear() { + this.message = null; + } + + public String getMessage() { + return this.message; + } + + public IllegalArgument setMessage(String message) { + this.message = message; + return this; + } + + public void unsetMessage() { + this.message = null; + } + + /** Returns true if field message is set (has been assigned a value) and false otherwise */ + public boolean isSetMessage() { + return this.message != null; + } + + public void setMessageIsSet(boolean value) { + if (!value) { + this.message = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case MESSAGE: + if (value == null) { + unsetMessage(); + } else { + setMessage((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case MESSAGE: + return getMessage(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case MESSAGE: + return isSetMessage(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof IllegalArgument) + return this.equals((IllegalArgument)that); + return false; + } + + public boolean equals(IllegalArgument that) { + if (that == null) + return false; + + boolean this_present_message = true && this.isSetMessage(); + boolean that_present_message = true && that.isSetMessage(); + if (this_present_message || that_present_message) { + if (!(this_present_message && that_present_message)) + return false; + if (!this.message.equals(that.message)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(IllegalArgument other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + IllegalArgument typedOther = (IllegalArgument)other; + + lastComparison = Boolean.valueOf(isSetMessage()).compareTo(typedOther.isSetMessage()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMessage()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.message, typedOther.message); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("IllegalArgument("); + boolean first = true; + + sb.append("message:"); + if (this.message == null) { + sb.append("null"); + } else { + sb.append(this.message); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class IllegalArgumentStandardSchemeFactory implements SchemeFactory { + public IllegalArgumentStandardScheme getScheme() { + return new IllegalArgumentStandardScheme(); + } + } + + private static class IllegalArgumentStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, IllegalArgument struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // MESSAGE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.message = iprot.readString(); + struct.setMessageIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, IllegalArgument struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.message != null) { + oprot.writeFieldBegin(MESSAGE_FIELD_DESC); + oprot.writeString(struct.message); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class IllegalArgumentTupleSchemeFactory implements SchemeFactory { + public IllegalArgumentTupleScheme getScheme() { + return new IllegalArgumentTupleScheme(); + } + } + + private static class IllegalArgumentTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, IllegalArgument struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetMessage()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetMessage()) { + oprot.writeString(struct.message); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, IllegalArgument struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.message = iprot.readString(); + struct.setMessageIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/Mutation.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/Mutation.java new file mode 100644 index 0000000..ef1817f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/Mutation.java @@ -0,0 +1,702 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A Mutation object is used to either update or delete a column-value. + */ +public class Mutation implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("Mutation"); + + private static final org.apache.thrift.protocol.TField IS_DELETE_FIELD_DESC = new org.apache.thrift.protocol.TField("isDelete", org.apache.thrift.protocol.TType.BOOL, (short)1); + private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField WRITE_TO_WAL_FIELD_DESC = new org.apache.thrift.protocol.TField("writeToWAL", org.apache.thrift.protocol.TType.BOOL, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new MutationStandardSchemeFactory()); + schemes.put(TupleScheme.class, new MutationTupleSchemeFactory()); + } + + public boolean isDelete; // required + public ByteBuffer column; // required + public ByteBuffer value; // required + public boolean writeToWAL; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IS_DELETE((short)1, "isDelete"), + COLUMN((short)2, "column"), + VALUE((short)3, "value"), + WRITE_TO_WAL((short)4, "writeToWAL"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IS_DELETE + return IS_DELETE; + case 2: // COLUMN + return COLUMN; + case 3: // VALUE + return VALUE; + case 4: // WRITE_TO_WAL + return WRITE_TO_WAL; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __ISDELETE_ISSET_ID = 0; + private static final int __WRITETOWAL_ISSET_ID = 1; + private BitSet __isset_bit_vector = new BitSet(2); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IS_DELETE, new org.apache.thrift.meta_data.FieldMetaData("isDelete", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.WRITE_TO_WAL, new org.apache.thrift.meta_data.FieldMetaData("writeToWAL", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Mutation.class, metaDataMap); + } + + public Mutation() { + this.isDelete = false; + + this.writeToWAL = true; + + } + + public Mutation( + boolean isDelete, + ByteBuffer column, + ByteBuffer value, + boolean writeToWAL) + { + this(); + this.isDelete = isDelete; + setIsDeleteIsSet(true); + this.column = column; + this.value = value; + this.writeToWAL = writeToWAL; + setWriteToWALIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public Mutation(Mutation other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.isDelete = other.isDelete; + if (other.isSetColumn()) { + this.column = other.column; + } + if (other.isSetValue()) { + this.value = other.value; + } + this.writeToWAL = other.writeToWAL; + } + + public Mutation deepCopy() { + return new Mutation(this); + } + + @Override + public void clear() { + this.isDelete = false; + + this.column = null; + this.value = null; + this.writeToWAL = true; + + } + + public boolean isIsDelete() { + return this.isDelete; + } + + public Mutation setIsDelete(boolean isDelete) { + this.isDelete = isDelete; + setIsDeleteIsSet(true); + return this; + } + + public void unsetIsDelete() { + __isset_bit_vector.clear(__ISDELETE_ISSET_ID); + } + + /** Returns true if field isDelete is set (has been assigned a value) and false otherwise */ + public boolean isSetIsDelete() { + return __isset_bit_vector.get(__ISDELETE_ISSET_ID); + } + + public void setIsDeleteIsSet(boolean value) { + __isset_bit_vector.set(__ISDELETE_ISSET_ID, value); + } + + public byte[] getColumn() { + setColumn(org.apache.thrift.TBaseHelper.rightSize(column)); + return column == null ? null : column.array(); + } + + public ByteBuffer bufferForColumn() { + return column; + } + + public Mutation setColumn(byte[] column) { + setColumn(column == null ? (ByteBuffer)null : ByteBuffer.wrap(column)); + return this; + } + + public Mutation setColumn(ByteBuffer column) { + this.column = column; + return this; + } + + public void unsetColumn() { + this.column = null; + } + + /** Returns true if field column is set (has been assigned a value) and false otherwise */ + public boolean isSetColumn() { + return this.column != null; + } + + public void setColumnIsSet(boolean value) { + if (!value) { + this.column = null; + } + } + + public byte[] getValue() { + setValue(org.apache.thrift.TBaseHelper.rightSize(value)); + return value == null ? null : value.array(); + } + + public ByteBuffer bufferForValue() { + return value; + } + + public Mutation setValue(byte[] value) { + setValue(value == null ? (ByteBuffer)null : ByteBuffer.wrap(value)); + return this; + } + + public Mutation setValue(ByteBuffer value) { + this.value = value; + return this; + } + + public void unsetValue() { + this.value = null; + } + + /** Returns true if field value is set (has been assigned a value) and false otherwise */ + public boolean isSetValue() { + return this.value != null; + } + + public void setValueIsSet(boolean value) { + if (!value) { + this.value = null; + } + } + + public boolean isWriteToWAL() { + return this.writeToWAL; + } + + public Mutation setWriteToWAL(boolean writeToWAL) { + this.writeToWAL = writeToWAL; + setWriteToWALIsSet(true); + return this; + } + + public void unsetWriteToWAL() { + __isset_bit_vector.clear(__WRITETOWAL_ISSET_ID); + } + + /** Returns true if field writeToWAL is set (has been assigned a value) and false otherwise */ + public boolean isSetWriteToWAL() { + return __isset_bit_vector.get(__WRITETOWAL_ISSET_ID); + } + + public void setWriteToWALIsSet(boolean value) { + __isset_bit_vector.set(__WRITETOWAL_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IS_DELETE: + if (value == null) { + unsetIsDelete(); + } else { + setIsDelete((Boolean)value); + } + break; + + case COLUMN: + if (value == null) { + unsetColumn(); + } else { + setColumn((ByteBuffer)value); + } + break; + + case VALUE: + if (value == null) { + unsetValue(); + } else { + setValue((ByteBuffer)value); + } + break; + + case WRITE_TO_WAL: + if (value == null) { + unsetWriteToWAL(); + } else { + setWriteToWAL((Boolean)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IS_DELETE: + return Boolean.valueOf(isIsDelete()); + + case COLUMN: + return getColumn(); + + case VALUE: + return getValue(); + + case WRITE_TO_WAL: + return Boolean.valueOf(isWriteToWAL()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IS_DELETE: + return isSetIsDelete(); + case COLUMN: + return isSetColumn(); + case VALUE: + return isSetValue(); + case WRITE_TO_WAL: + return isSetWriteToWAL(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof Mutation) + return this.equals((Mutation)that); + return false; + } + + public boolean equals(Mutation that) { + if (that == null) + return false; + + boolean this_present_isDelete = true; + boolean that_present_isDelete = true; + if (this_present_isDelete || that_present_isDelete) { + if (!(this_present_isDelete && that_present_isDelete)) + return false; + if (this.isDelete != that.isDelete) + return false; + } + + boolean this_present_column = true && this.isSetColumn(); + boolean that_present_column = true && that.isSetColumn(); + if (this_present_column || that_present_column) { + if (!(this_present_column && that_present_column)) + return false; + if (!this.column.equals(that.column)) + return false; + } + + boolean this_present_value = true && this.isSetValue(); + boolean that_present_value = true && that.isSetValue(); + if (this_present_value || that_present_value) { + if (!(this_present_value && that_present_value)) + return false; + if (!this.value.equals(that.value)) + return false; + } + + boolean this_present_writeToWAL = true; + boolean that_present_writeToWAL = true; + if (this_present_writeToWAL || that_present_writeToWAL) { + if (!(this_present_writeToWAL && that_present_writeToWAL)) + return false; + if (this.writeToWAL != that.writeToWAL) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(Mutation other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + Mutation typedOther = (Mutation)other; + + lastComparison = Boolean.valueOf(isSetIsDelete()).compareTo(typedOther.isSetIsDelete()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIsDelete()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.isDelete, typedOther.isDelete); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumn()).compareTo(typedOther.isSetColumn()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumn()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, typedOther.column); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetWriteToWAL()).compareTo(typedOther.isSetWriteToWAL()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetWriteToWAL()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.writeToWAL, typedOther.writeToWAL); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("Mutation("); + boolean first = true; + + sb.append("isDelete:"); + sb.append(this.isDelete); + first = false; + if (!first) sb.append(", "); + sb.append("column:"); + if (this.column == null) { + sb.append("null"); + } else { + sb.append(this.column); + } + first = false; + if (!first) sb.append(", "); + sb.append("value:"); + if (this.value == null) { + sb.append("null"); + } else { + sb.append(this.value); + } + first = false; + if (!first) sb.append(", "); + sb.append("writeToWAL:"); + sb.append(this.writeToWAL); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class MutationStandardSchemeFactory implements SchemeFactory { + public MutationStandardScheme getScheme() { + return new MutationStandardScheme(); + } + } + + private static class MutationStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, Mutation struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IS_DELETE + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.isDelete = iprot.readBool(); + struct.setIsDeleteIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // COLUMN + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.value = iprot.readBinary(); + struct.setValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // WRITE_TO_WAL + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.writeToWAL = iprot.readBool(); + struct.setWriteToWALIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, Mutation struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(IS_DELETE_FIELD_DESC); + oprot.writeBool(struct.isDelete); + oprot.writeFieldEnd(); + if (struct.column != null) { + oprot.writeFieldBegin(COLUMN_FIELD_DESC); + oprot.writeBinary(struct.column); + oprot.writeFieldEnd(); + } + if (struct.value != null) { + oprot.writeFieldBegin(VALUE_FIELD_DESC); + oprot.writeBinary(struct.value); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(WRITE_TO_WAL_FIELD_DESC); + oprot.writeBool(struct.writeToWAL); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class MutationTupleSchemeFactory implements SchemeFactory { + public MutationTupleScheme getScheme() { + return new MutationTupleScheme(); + } + } + + private static class MutationTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, Mutation struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIsDelete()) { + optionals.set(0); + } + if (struct.isSetColumn()) { + optionals.set(1); + } + if (struct.isSetValue()) { + optionals.set(2); + } + if (struct.isSetWriteToWAL()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetIsDelete()) { + oprot.writeBool(struct.isDelete); + } + if (struct.isSetColumn()) { + oprot.writeBinary(struct.column); + } + if (struct.isSetValue()) { + oprot.writeBinary(struct.value); + } + if (struct.isSetWriteToWAL()) { + oprot.writeBool(struct.writeToWAL); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, Mutation struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.isDelete = iprot.readBool(); + struct.setIsDeleteIsSet(true); + } + if (incoming.get(1)) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } + if (incoming.get(2)) { + struct.value = iprot.readBinary(); + struct.setValueIsSet(true); + } + if (incoming.get(3)) { + struct.writeToWAL = iprot.readBool(); + struct.setWriteToWALIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TCell.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TCell.java new file mode 100644 index 0000000..6ee8ca7 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TCell.java @@ -0,0 +1,497 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * TCell - Used to transport a cell value (byte[]) and the timestamp it was + * stored with together as a result for get and getRow methods. This promotes + * the timestamp of a cell to a first-class value, making it easy to take + * note of temporal data. Cell is used all the way from HStore up to HTable. + */ +public class TCell implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCell"); + + private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TCellStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TCellTupleSchemeFactory()); + } + + public ByteBuffer value; // required + public long timestamp; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + VALUE((short)1, "value"), + TIMESTAMP((short)2, "timestamp"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // VALUE + return VALUE; + case 2: // TIMESTAMP + return TIMESTAMP; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Bytes"))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCell.class, metaDataMap); + } + + public TCell() { + } + + public TCell( + ByteBuffer value, + long timestamp) + { + this(); + this.value = value; + this.timestamp = timestamp; + setTimestampIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public TCell(TCell other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetValue()) { + this.value = other.value; + } + this.timestamp = other.timestamp; + } + + public TCell deepCopy() { + return new TCell(this); + } + + @Override + public void clear() { + this.value = null; + setTimestampIsSet(false); + this.timestamp = 0; + } + + public byte[] getValue() { + setValue(org.apache.thrift.TBaseHelper.rightSize(value)); + return value == null ? null : value.array(); + } + + public ByteBuffer bufferForValue() { + return value; + } + + public TCell setValue(byte[] value) { + setValue(value == null ? (ByteBuffer)null : ByteBuffer.wrap(value)); + return this; + } + + public TCell setValue(ByteBuffer value) { + this.value = value; + return this; + } + + public void unsetValue() { + this.value = null; + } + + /** Returns true if field value is set (has been assigned a value) and false otherwise */ + public boolean isSetValue() { + return this.value != null; + } + + public void setValueIsSet(boolean value) { + if (!value) { + this.value = null; + } + } + + public long getTimestamp() { + return this.timestamp; + } + + public TCell setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case VALUE: + if (value == null) { + unsetValue(); + } else { + setValue((ByteBuffer)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case VALUE: + return getValue(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case VALUE: + return isSetValue(); + case TIMESTAMP: + return isSetTimestamp(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TCell) + return this.equals((TCell)that); + return false; + } + + public boolean equals(TCell that) { + if (that == null) + return false; + + boolean this_present_value = true && this.isSetValue(); + boolean that_present_value = true && that.isSetValue(); + if (this_present_value || that_present_value) { + if (!(this_present_value && that_present_value)) + return false; + if (!this.value.equals(that.value)) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TCell other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TCell typedOther = (TCell)other; + + lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TCell("); + boolean first = true; + + sb.append("value:"); + if (this.value == null) { + sb.append("null"); + } else { + sb.append(this.value); + } + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TCellStandardSchemeFactory implements SchemeFactory { + public TCellStandardScheme getScheme() { + return new TCellStandardScheme(); + } + } + + private static class TCellStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TCell struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.value = iprot.readBinary(); + struct.setValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TCell struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.value != null) { + oprot.writeFieldBegin(VALUE_FIELD_DESC); + oprot.writeBinary(struct.value); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TCellTupleSchemeFactory implements SchemeFactory { + public TCellTupleScheme getScheme() { + return new TCellTupleScheme(); + } + } + + private static class TCellTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TCell struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetValue()) { + optionals.set(0); + } + if (struct.isSetTimestamp()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetValue()) { + oprot.writeBinary(struct.value); + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TCell struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.value = iprot.readBinary(); + struct.setValueIsSet(true); + } + if (incoming.get(1)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java new file mode 100644 index 0000000..6d24aa0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java @@ -0,0 +1,715 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * For increments that are not incrementColumnValue + * equivalents. + */ +public class TIncrement implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TIncrement"); + + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMN_FIELD_DESC = new org.apache.thrift.protocol.TField("column", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField AMMOUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("ammount", org.apache.thrift.protocol.TType.I64, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TIncrementStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TIncrementTupleSchemeFactory()); + } + + public ByteBuffer table; // required + public ByteBuffer row; // required + public ByteBuffer column; // required + public long ammount; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TABLE((short)1, "table"), + ROW((short)2, "row"), + COLUMN((short)3, "column"), + AMMOUNT((short)4, "ammount"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE + return TABLE; + case 2: // ROW + return ROW; + case 3: // COLUMN + return COLUMN; + case 4: // AMMOUNT + return AMMOUNT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __AMMOUNT_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMN, new org.apache.thrift.meta_data.FieldMetaData("column", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.AMMOUNT, new org.apache.thrift.meta_data.FieldMetaData("ammount", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TIncrement.class, metaDataMap); + } + + public TIncrement() { + } + + public TIncrement( + ByteBuffer table, + ByteBuffer row, + ByteBuffer column, + long ammount) + { + this(); + this.table = table; + this.row = row; + this.column = column; + this.ammount = ammount; + setAmmountIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public TIncrement(TIncrement other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetTable()) { + this.table = other.table; + } + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetColumn()) { + this.column = other.column; + } + this.ammount = other.ammount; + } + + public TIncrement deepCopy() { + return new TIncrement(this); + } + + @Override + public void clear() { + this.table = null; + this.row = null; + this.column = null; + setAmmountIsSet(false); + this.ammount = 0; + } + + public byte[] getTable() { + setTable(org.apache.thrift.TBaseHelper.rightSize(table)); + return table == null ? null : table.array(); + } + + public ByteBuffer bufferForTable() { + return table; + } + + public TIncrement setTable(byte[] table) { + setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table)); + return this; + } + + public TIncrement setTable(ByteBuffer table) { + this.table = table; + return this; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + public TIncrement setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public TIncrement setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public byte[] getColumn() { + setColumn(org.apache.thrift.TBaseHelper.rightSize(column)); + return column == null ? null : column.array(); + } + + public ByteBuffer bufferForColumn() { + return column; + } + + public TIncrement setColumn(byte[] column) { + setColumn(column == null ? (ByteBuffer)null : ByteBuffer.wrap(column)); + return this; + } + + public TIncrement setColumn(ByteBuffer column) { + this.column = column; + return this; + } + + public void unsetColumn() { + this.column = null; + } + + /** Returns true if field column is set (has been assigned a value) and false otherwise */ + public boolean isSetColumn() { + return this.column != null; + } + + public void setColumnIsSet(boolean value) { + if (!value) { + this.column = null; + } + } + + public long getAmmount() { + return this.ammount; + } + + public TIncrement setAmmount(long ammount) { + this.ammount = ammount; + setAmmountIsSet(true); + return this; + } + + public void unsetAmmount() { + __isset_bit_vector.clear(__AMMOUNT_ISSET_ID); + } + + /** Returns true if field ammount is set (has been assigned a value) and false otherwise */ + public boolean isSetAmmount() { + return __isset_bit_vector.get(__AMMOUNT_ISSET_ID); + } + + public void setAmmountIsSet(boolean value) { + __isset_bit_vector.set(__AMMOUNT_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMN: + if (value == null) { + unsetColumn(); + } else { + setColumn((ByteBuffer)value); + } + break; + + case AMMOUNT: + if (value == null) { + unsetAmmount(); + } else { + setAmmount((Long)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE: + return getTable(); + + case ROW: + return getRow(); + + case COLUMN: + return getColumn(); + + case AMMOUNT: + return Long.valueOf(getAmmount()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE: + return isSetTable(); + case ROW: + return isSetRow(); + case COLUMN: + return isSetColumn(); + case AMMOUNT: + return isSetAmmount(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TIncrement) + return this.equals((TIncrement)that); + return false; + } + + public boolean equals(TIncrement that) { + if (that == null) + return false; + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_column = true && this.isSetColumn(); + boolean that_present_column = true && that.isSetColumn(); + if (this_present_column || that_present_column) { + if (!(this_present_column && that_present_column)) + return false; + if (!this.column.equals(that.column)) + return false; + } + + boolean this_present_ammount = true; + boolean that_present_ammount = true; + if (this_present_ammount || that_present_ammount) { + if (!(this_present_ammount && that_present_ammount)) + return false; + if (this.ammount != that.ammount) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TIncrement other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TIncrement typedOther = (TIncrement)other; + + lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumn()).compareTo(typedOther.isSetColumn()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumn()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.column, typedOther.column); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAmmount()).compareTo(typedOther.isSetAmmount()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAmmount()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ammount, typedOther.ammount); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TIncrement("); + boolean first = true; + + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + sb.append(this.table); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("column:"); + if (this.column == null) { + sb.append("null"); + } else { + sb.append(this.column); + } + first = false; + if (!first) sb.append(", "); + sb.append("ammount:"); + sb.append(this.ammount); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TIncrementStandardSchemeFactory implements SchemeFactory { + public TIncrementStandardScheme getScheme() { + return new TIncrementStandardScheme(); + } + } + + private static class TIncrementStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TIncrement struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMN + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // AMMOUNT + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.ammount = iprot.readI64(); + struct.setAmmountIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TIncrement struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.table != null) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeBinary(struct.table); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.column != null) { + oprot.writeFieldBegin(COLUMN_FIELD_DESC); + oprot.writeBinary(struct.column); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(AMMOUNT_FIELD_DESC); + oprot.writeI64(struct.ammount); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TIncrementTupleSchemeFactory implements SchemeFactory { + public TIncrementTupleScheme getScheme() { + return new TIncrementTupleScheme(); + } + } + + private static class TIncrementTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TIncrement struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetTable()) { + optionals.set(0); + } + if (struct.isSetRow()) { + optionals.set(1); + } + if (struct.isSetColumn()) { + optionals.set(2); + } + if (struct.isSetAmmount()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTable()) { + oprot.writeBinary(struct.table); + } + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetColumn()) { + oprot.writeBinary(struct.column); + } + if (struct.isSetAmmount()) { + oprot.writeI64(struct.ammount); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TIncrement struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } + if (incoming.get(1)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(2)) { + struct.column = iprot.readBinary(); + struct.setColumnIsSet(true); + } + if (incoming.get(3)) { + struct.ammount = iprot.readI64(); + struct.setAmmountIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRegionInfo.java new file mode 100644 index 0000000..ed251e8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRegionInfo.java @@ -0,0 +1,1012 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A TRegionInfo contains information about an HTable region. + */ +public class TRegionInfo implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRegionInfo"); + + private static final org.apache.thrift.protocol.TField START_KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("startKey", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField END_KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("endKey", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField ID_FIELD_DESC = new org.apache.thrift.protocol.TField("id", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)4); + private static final org.apache.thrift.protocol.TField VERSION_FIELD_DESC = new org.apache.thrift.protocol.TField("version", org.apache.thrift.protocol.TType.BYTE, (short)5); + private static final org.apache.thrift.protocol.TField SERVER_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("serverName", org.apache.thrift.protocol.TType.STRING, (short)6); + private static final org.apache.thrift.protocol.TField PORT_FIELD_DESC = new org.apache.thrift.protocol.TField("port", org.apache.thrift.protocol.TType.I32, (short)7); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TRegionInfoStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TRegionInfoTupleSchemeFactory()); + } + + public ByteBuffer startKey; // required + public ByteBuffer endKey; // required + public long id; // required + public ByteBuffer name; // required + public byte version; // required + public ByteBuffer serverName; // required + public int port; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + START_KEY((short)1, "startKey"), + END_KEY((short)2, "endKey"), + ID((short)3, "id"), + NAME((short)4, "name"), + VERSION((short)5, "version"), + SERVER_NAME((short)6, "serverName"), + PORT((short)7, "port"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // START_KEY + return START_KEY; + case 2: // END_KEY + return END_KEY; + case 3: // ID + return ID; + case 4: // NAME + return NAME; + case 5: // VERSION + return VERSION; + case 6: // SERVER_NAME + return SERVER_NAME; + case 7: // PORT + return PORT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __ID_ISSET_ID = 0; + private static final int __VERSION_ISSET_ID = 1; + private static final int __PORT_ISSET_ID = 2; + private BitSet __isset_bit_vector = new BitSet(3); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.START_KEY, new org.apache.thrift.meta_data.FieldMetaData("startKey", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.END_KEY, new org.apache.thrift.meta_data.FieldMetaData("endKey", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.ID, new org.apache.thrift.meta_data.FieldMetaData("id", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.VERSION, new org.apache.thrift.meta_data.FieldMetaData("version", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BYTE))); + tmpMap.put(_Fields.SERVER_NAME, new org.apache.thrift.meta_data.FieldMetaData("serverName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.PORT, new org.apache.thrift.meta_data.FieldMetaData("port", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TRegionInfo.class, metaDataMap); + } + + public TRegionInfo() { + } + + public TRegionInfo( + ByteBuffer startKey, + ByteBuffer endKey, + long id, + ByteBuffer name, + byte version, + ByteBuffer serverName, + int port) + { + this(); + this.startKey = startKey; + this.endKey = endKey; + this.id = id; + setIdIsSet(true); + this.name = name; + this.version = version; + setVersionIsSet(true); + this.serverName = serverName; + this.port = port; + setPortIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public TRegionInfo(TRegionInfo other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetStartKey()) { + this.startKey = other.startKey; + } + if (other.isSetEndKey()) { + this.endKey = other.endKey; + } + this.id = other.id; + if (other.isSetName()) { + this.name = other.name; + } + this.version = other.version; + if (other.isSetServerName()) { + this.serverName = other.serverName; + } + this.port = other.port; + } + + public TRegionInfo deepCopy() { + return new TRegionInfo(this); + } + + @Override + public void clear() { + this.startKey = null; + this.endKey = null; + setIdIsSet(false); + this.id = 0; + this.name = null; + setVersionIsSet(false); + this.version = 0; + this.serverName = null; + setPortIsSet(false); + this.port = 0; + } + + public byte[] getStartKey() { + setStartKey(org.apache.thrift.TBaseHelper.rightSize(startKey)); + return startKey == null ? null : startKey.array(); + } + + public ByteBuffer bufferForStartKey() { + return startKey; + } + + public TRegionInfo setStartKey(byte[] startKey) { + setStartKey(startKey == null ? (ByteBuffer)null : ByteBuffer.wrap(startKey)); + return this; + } + + public TRegionInfo setStartKey(ByteBuffer startKey) { + this.startKey = startKey; + return this; + } + + public void unsetStartKey() { + this.startKey = null; + } + + /** Returns true if field startKey is set (has been assigned a value) and false otherwise */ + public boolean isSetStartKey() { + return this.startKey != null; + } + + public void setStartKeyIsSet(boolean value) { + if (!value) { + this.startKey = null; + } + } + + public byte[] getEndKey() { + setEndKey(org.apache.thrift.TBaseHelper.rightSize(endKey)); + return endKey == null ? null : endKey.array(); + } + + public ByteBuffer bufferForEndKey() { + return endKey; + } + + public TRegionInfo setEndKey(byte[] endKey) { + setEndKey(endKey == null ? (ByteBuffer)null : ByteBuffer.wrap(endKey)); + return this; + } + + public TRegionInfo setEndKey(ByteBuffer endKey) { + this.endKey = endKey; + return this; + } + + public void unsetEndKey() { + this.endKey = null; + } + + /** Returns true if field endKey is set (has been assigned a value) and false otherwise */ + public boolean isSetEndKey() { + return this.endKey != null; + } + + public void setEndKeyIsSet(boolean value) { + if (!value) { + this.endKey = null; + } + } + + public long getId() { + return this.id; + } + + public TRegionInfo setId(long id) { + this.id = id; + setIdIsSet(true); + return this; + } + + public void unsetId() { + __isset_bit_vector.clear(__ID_ISSET_ID); + } + + /** Returns true if field id is set (has been assigned a value) and false otherwise */ + public boolean isSetId() { + return __isset_bit_vector.get(__ID_ISSET_ID); + } + + public void setIdIsSet(boolean value) { + __isset_bit_vector.set(__ID_ISSET_ID, value); + } + + public byte[] getName() { + setName(org.apache.thrift.TBaseHelper.rightSize(name)); + return name == null ? null : name.array(); + } + + public ByteBuffer bufferForName() { + return name; + } + + public TRegionInfo setName(byte[] name) { + setName(name == null ? (ByteBuffer)null : ByteBuffer.wrap(name)); + return this; + } + + public TRegionInfo setName(ByteBuffer name) { + this.name = name; + return this; + } + + public void unsetName() { + this.name = null; + } + + /** Returns true if field name is set (has been assigned a value) and false otherwise */ + public boolean isSetName() { + return this.name != null; + } + + public void setNameIsSet(boolean value) { + if (!value) { + this.name = null; + } + } + + public byte getVersion() { + return this.version; + } + + public TRegionInfo setVersion(byte version) { + this.version = version; + setVersionIsSet(true); + return this; + } + + public void unsetVersion() { + __isset_bit_vector.clear(__VERSION_ISSET_ID); + } + + /** Returns true if field version is set (has been assigned a value) and false otherwise */ + public boolean isSetVersion() { + return __isset_bit_vector.get(__VERSION_ISSET_ID); + } + + public void setVersionIsSet(boolean value) { + __isset_bit_vector.set(__VERSION_ISSET_ID, value); + } + + public byte[] getServerName() { + setServerName(org.apache.thrift.TBaseHelper.rightSize(serverName)); + return serverName == null ? null : serverName.array(); + } + + public ByteBuffer bufferForServerName() { + return serverName; + } + + public TRegionInfo setServerName(byte[] serverName) { + setServerName(serverName == null ? (ByteBuffer)null : ByteBuffer.wrap(serverName)); + return this; + } + + public TRegionInfo setServerName(ByteBuffer serverName) { + this.serverName = serverName; + return this; + } + + public void unsetServerName() { + this.serverName = null; + } + + /** Returns true if field serverName is set (has been assigned a value) and false otherwise */ + public boolean isSetServerName() { + return this.serverName != null; + } + + public void setServerNameIsSet(boolean value) { + if (!value) { + this.serverName = null; + } + } + + public int getPort() { + return this.port; + } + + public TRegionInfo setPort(int port) { + this.port = port; + setPortIsSet(true); + return this; + } + + public void unsetPort() { + __isset_bit_vector.clear(__PORT_ISSET_ID); + } + + /** Returns true if field port is set (has been assigned a value) and false otherwise */ + public boolean isSetPort() { + return __isset_bit_vector.get(__PORT_ISSET_ID); + } + + public void setPortIsSet(boolean value) { + __isset_bit_vector.set(__PORT_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case START_KEY: + if (value == null) { + unsetStartKey(); + } else { + setStartKey((ByteBuffer)value); + } + break; + + case END_KEY: + if (value == null) { + unsetEndKey(); + } else { + setEndKey((ByteBuffer)value); + } + break; + + case ID: + if (value == null) { + unsetId(); + } else { + setId((Long)value); + } + break; + + case NAME: + if (value == null) { + unsetName(); + } else { + setName((ByteBuffer)value); + } + break; + + case VERSION: + if (value == null) { + unsetVersion(); + } else { + setVersion((Byte)value); + } + break; + + case SERVER_NAME: + if (value == null) { + unsetServerName(); + } else { + setServerName((ByteBuffer)value); + } + break; + + case PORT: + if (value == null) { + unsetPort(); + } else { + setPort((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case START_KEY: + return getStartKey(); + + case END_KEY: + return getEndKey(); + + case ID: + return Long.valueOf(getId()); + + case NAME: + return getName(); + + case VERSION: + return Byte.valueOf(getVersion()); + + case SERVER_NAME: + return getServerName(); + + case PORT: + return Integer.valueOf(getPort()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case START_KEY: + return isSetStartKey(); + case END_KEY: + return isSetEndKey(); + case ID: + return isSetId(); + case NAME: + return isSetName(); + case VERSION: + return isSetVersion(); + case SERVER_NAME: + return isSetServerName(); + case PORT: + return isSetPort(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TRegionInfo) + return this.equals((TRegionInfo)that); + return false; + } + + public boolean equals(TRegionInfo that) { + if (that == null) + return false; + + boolean this_present_startKey = true && this.isSetStartKey(); + boolean that_present_startKey = true && that.isSetStartKey(); + if (this_present_startKey || that_present_startKey) { + if (!(this_present_startKey && that_present_startKey)) + return false; + if (!this.startKey.equals(that.startKey)) + return false; + } + + boolean this_present_endKey = true && this.isSetEndKey(); + boolean that_present_endKey = true && that.isSetEndKey(); + if (this_present_endKey || that_present_endKey) { + if (!(this_present_endKey && that_present_endKey)) + return false; + if (!this.endKey.equals(that.endKey)) + return false; + } + + boolean this_present_id = true; + boolean that_present_id = true; + if (this_present_id || that_present_id) { + if (!(this_present_id && that_present_id)) + return false; + if (this.id != that.id) + return false; + } + + boolean this_present_name = true && this.isSetName(); + boolean that_present_name = true && that.isSetName(); + if (this_present_name || that_present_name) { + if (!(this_present_name && that_present_name)) + return false; + if (!this.name.equals(that.name)) + return false; + } + + boolean this_present_version = true; + boolean that_present_version = true; + if (this_present_version || that_present_version) { + if (!(this_present_version && that_present_version)) + return false; + if (this.version != that.version) + return false; + } + + boolean this_present_serverName = true && this.isSetServerName(); + boolean that_present_serverName = true && that.isSetServerName(); + if (this_present_serverName || that_present_serverName) { + if (!(this_present_serverName && that_present_serverName)) + return false; + if (!this.serverName.equals(that.serverName)) + return false; + } + + boolean this_present_port = true; + boolean that_present_port = true; + if (this_present_port || that_present_port) { + if (!(this_present_port && that_present_port)) + return false; + if (this.port != that.port) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TRegionInfo other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TRegionInfo typedOther = (TRegionInfo)other; + + lastComparison = Boolean.valueOf(isSetStartKey()).compareTo(typedOther.isSetStartKey()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStartKey()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.startKey, typedOther.startKey); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetEndKey()).compareTo(typedOther.isSetEndKey()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetEndKey()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.endKey, typedOther.endKey); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetId()).compareTo(typedOther.isSetId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.id, typedOther.id); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetName()).compareTo(typedOther.isSetName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, typedOther.name); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetVersion()).compareTo(typedOther.isSetVersion()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetVersion()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.version, typedOther.version); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetServerName()).compareTo(typedOther.isSetServerName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetServerName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.serverName, typedOther.serverName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetPort()).compareTo(typedOther.isSetPort()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetPort()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.port, typedOther.port); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TRegionInfo("); + boolean first = true; + + sb.append("startKey:"); + if (this.startKey == null) { + sb.append("null"); + } else { + sb.append(this.startKey); + } + first = false; + if (!first) sb.append(", "); + sb.append("endKey:"); + if (this.endKey == null) { + sb.append("null"); + } else { + sb.append(this.endKey); + } + first = false; + if (!first) sb.append(", "); + sb.append("id:"); + sb.append(this.id); + first = false; + if (!first) sb.append(", "); + sb.append("name:"); + if (this.name == null) { + sb.append("null"); + } else { + sb.append(this.name); + } + first = false; + if (!first) sb.append(", "); + sb.append("version:"); + sb.append(this.version); + first = false; + if (!first) sb.append(", "); + sb.append("serverName:"); + if (this.serverName == null) { + sb.append("null"); + } else { + sb.append(this.serverName); + } + first = false; + if (!first) sb.append(", "); + sb.append("port:"); + sb.append(this.port); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TRegionInfoStandardSchemeFactory implements SchemeFactory { + public TRegionInfoStandardScheme getScheme() { + return new TRegionInfoStandardScheme(); + } + } + + private static class TRegionInfoStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TRegionInfo struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // START_KEY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.startKey = iprot.readBinary(); + struct.setStartKeyIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // END_KEY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.endKey = iprot.readBinary(); + struct.setEndKeyIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // ID + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.id = iprot.readI64(); + struct.setIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.name = iprot.readBinary(); + struct.setNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // VERSION + if (schemeField.type == org.apache.thrift.protocol.TType.BYTE) { + struct.version = iprot.readByte(); + struct.setVersionIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 6: // SERVER_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.serverName = iprot.readBinary(); + struct.setServerNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 7: // PORT + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.port = iprot.readI32(); + struct.setPortIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TRegionInfo struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.startKey != null) { + oprot.writeFieldBegin(START_KEY_FIELD_DESC); + oprot.writeBinary(struct.startKey); + oprot.writeFieldEnd(); + } + if (struct.endKey != null) { + oprot.writeFieldBegin(END_KEY_FIELD_DESC); + oprot.writeBinary(struct.endKey); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(ID_FIELD_DESC); + oprot.writeI64(struct.id); + oprot.writeFieldEnd(); + if (struct.name != null) { + oprot.writeFieldBegin(NAME_FIELD_DESC); + oprot.writeBinary(struct.name); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(VERSION_FIELD_DESC); + oprot.writeByte(struct.version); + oprot.writeFieldEnd(); + if (struct.serverName != null) { + oprot.writeFieldBegin(SERVER_NAME_FIELD_DESC); + oprot.writeBinary(struct.serverName); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(PORT_FIELD_DESC); + oprot.writeI32(struct.port); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TRegionInfoTupleSchemeFactory implements SchemeFactory { + public TRegionInfoTupleScheme getScheme() { + return new TRegionInfoTupleScheme(); + } + } + + private static class TRegionInfoTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TRegionInfo struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetStartKey()) { + optionals.set(0); + } + if (struct.isSetEndKey()) { + optionals.set(1); + } + if (struct.isSetId()) { + optionals.set(2); + } + if (struct.isSetName()) { + optionals.set(3); + } + if (struct.isSetVersion()) { + optionals.set(4); + } + if (struct.isSetServerName()) { + optionals.set(5); + } + if (struct.isSetPort()) { + optionals.set(6); + } + oprot.writeBitSet(optionals, 7); + if (struct.isSetStartKey()) { + oprot.writeBinary(struct.startKey); + } + if (struct.isSetEndKey()) { + oprot.writeBinary(struct.endKey); + } + if (struct.isSetId()) { + oprot.writeI64(struct.id); + } + if (struct.isSetName()) { + oprot.writeBinary(struct.name); + } + if (struct.isSetVersion()) { + oprot.writeByte(struct.version); + } + if (struct.isSetServerName()) { + oprot.writeBinary(struct.serverName); + } + if (struct.isSetPort()) { + oprot.writeI32(struct.port); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TRegionInfo struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(7); + if (incoming.get(0)) { + struct.startKey = iprot.readBinary(); + struct.setStartKeyIsSet(true); + } + if (incoming.get(1)) { + struct.endKey = iprot.readBinary(); + struct.setEndKeyIsSet(true); + } + if (incoming.get(2)) { + struct.id = iprot.readI64(); + struct.setIdIsSet(true); + } + if (incoming.get(3)) { + struct.name = iprot.readBinary(); + struct.setNameIsSet(true); + } + if (incoming.get(4)) { + struct.version = iprot.readByte(); + struct.setVersionIsSet(true); + } + if (incoming.get(5)) { + struct.serverName = iprot.readBinary(); + struct.setServerNameIsSet(true); + } + if (incoming.get(6)) { + struct.port = iprot.readI32(); + struct.setPortIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java new file mode 100644 index 0000000..4477289 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java @@ -0,0 +1,560 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Holds row name and then a map of columns to cells. + */ +public class TRowResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRowResult"); + + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.MAP, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TRowResultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TRowResultTupleSchemeFactory()); + } + + public ByteBuffer row; // required + public Map columns; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + ROW((short)1, "row"), + COLUMNS((short)2, "columns"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // ROW + return ROW; + case 2: // COLUMNS + return COLUMNS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"), + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCell.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TRowResult.class, metaDataMap); + } + + public TRowResult() { + } + + public TRowResult( + ByteBuffer row, + Map columns) + { + this(); + this.row = row; + this.columns = columns; + } + + /** + * Performs a deep copy on other. + */ + public TRowResult(TRowResult other) { + if (other.isSetRow()) { + this.row = other.row; + } + if (other.isSetColumns()) { + Map __this__columns = new HashMap(); + for (Map.Entry other_element : other.columns.entrySet()) { + + ByteBuffer other_element_key = other_element.getKey(); + TCell other_element_value = other_element.getValue(); + + ByteBuffer __this__columns_copy_key = other_element_key; + + TCell __this__columns_copy_value = new TCell(other_element_value); + + __this__columns.put(__this__columns_copy_key, __this__columns_copy_value); + } + this.columns = __this__columns; + } + } + + public TRowResult deepCopy() { + return new TRowResult(this); + } + + @Override + public void clear() { + this.row = null; + this.columns = null; + } + + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + public TRowResult setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public TRowResult setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public void putToColumns(ByteBuffer key, TCell val) { + if (this.columns == null) { + this.columns = new HashMap(); + } + this.columns.put(key, val); + } + + public Map getColumns() { + return this.columns; + } + + public TRowResult setColumns(Map columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case ROW: + return getRow(); + + case COLUMNS: + return getColumns(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case ROW: + return isSetRow(); + case COLUMNS: + return isSetColumns(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TRowResult) + return this.equals((TRowResult)that); + return false; + } + + public boolean equals(TRowResult that) { + if (that == null) + return false; + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TRowResult other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TRowResult typedOther = (TRowResult)other; + + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TRowResult("); + boolean first = true; + + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + sb.append(this.row); + } + first = false; + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TRowResultStandardSchemeFactory implements SchemeFactory { + public TRowResultStandardScheme getScheme() { + return new TRowResultStandardScheme(); + } + } + + private static class TRowResultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TRowResult struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map8 = iprot.readMapBegin(); + struct.columns = new HashMap(2*_map8.size); + for (int _i9 = 0; _i9 < _map8.size; ++_i9) + { + ByteBuffer _key10; // required + TCell _val11; // optional + _key10 = iprot.readBinary(); + _val11 = new TCell(); + _val11.read(iprot); + struct.columns.put(_key10, _val11); + } + iprot.readMapEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TRowResult struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.columns.size())); + for (Map.Entry _iter12 : struct.columns.entrySet()) + { + oprot.writeBinary(_iter12.getKey()); + _iter12.getValue().write(oprot); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TRowResultTupleSchemeFactory implements SchemeFactory { + public TRowResultTupleScheme getScheme() { + return new TRowResultTupleScheme(); + } + } + + private static class TRowResultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TRowResult struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetRow()) { + optionals.set(0); + } + if (struct.isSetColumns()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (Map.Entry _iter13 : struct.columns.entrySet()) + { + oprot.writeBinary(_iter13.getKey()); + _iter13.getValue().write(oprot); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TRowResult struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + if (incoming.get(1)) { + { + org.apache.thrift.protocol.TMap _map14 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.columns = new HashMap(2*_map14.size); + for (int _i15 = 0; _i15 < _map14.size; ++_i15) + { + ByteBuffer _key16; // required + TCell _val17; // required + _key16 = iprot.readBinary(); + _val17 = new TCell(); + _val17.read(iprot); + struct.columns.put(_key16, _val17); + } + } + struct.setColumnsIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java new file mode 100644 index 0000000..f7cc05d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java @@ -0,0 +1,966 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A Scan object is used to specify scanner parameters when opening a scanner. + */ +public class TScan implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TScan"); + + private static final org.apache.thrift.protocol.TField START_ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("startRow", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField STOP_ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("stopRow", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)4); + private static final org.apache.thrift.protocol.TField CACHING_FIELD_DESC = new org.apache.thrift.protocol.TField("caching", org.apache.thrift.protocol.TType.I32, (short)5); + private static final org.apache.thrift.protocol.TField FILTER_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("filterString", org.apache.thrift.protocol.TType.STRING, (short)6); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TScanStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TScanTupleSchemeFactory()); + } + + public ByteBuffer startRow; // optional + public ByteBuffer stopRow; // optional + public long timestamp; // optional + public List columns; // optional + public int caching; // optional + public ByteBuffer filterString; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + START_ROW((short)1, "startRow"), + STOP_ROW((short)2, "stopRow"), + TIMESTAMP((short)3, "timestamp"), + COLUMNS((short)4, "columns"), + CACHING((short)5, "caching"), + FILTER_STRING((short)6, "filterString"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // START_ROW + return START_ROW; + case 2: // STOP_ROW + return STOP_ROW; + case 3: // TIMESTAMP + return TIMESTAMP; + case 4: // COLUMNS + return COLUMNS; + case 5: // CACHING + return CACHING; + case 6: // FILTER_STRING + return FILTER_STRING; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private static final int __CACHING_ISSET_ID = 1; + private BitSet __isset_bit_vector = new BitSet(2); + private _Fields optionals[] = {_Fields.START_ROW,_Fields.STOP_ROW,_Fields.TIMESTAMP,_Fields.COLUMNS,_Fields.CACHING,_Fields.FILTER_STRING}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.START_ROW, new org.apache.thrift.meta_data.FieldMetaData("startRow", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.STOP_ROW, new org.apache.thrift.meta_data.FieldMetaData("stopRow", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text")))); + tmpMap.put(_Fields.CACHING, new org.apache.thrift.meta_data.FieldMetaData("caching", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.FILTER_STRING, new org.apache.thrift.meta_data.FieldMetaData("filterString", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , "Text"))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TScan.class, metaDataMap); + } + + public TScan() { + } + + /** + * Performs a deep copy on other. + */ + public TScan(TScan other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetStartRow()) { + this.startRow = other.startRow; + } + if (other.isSetStopRow()) { + this.stopRow = other.stopRow; + } + this.timestamp = other.timestamp; + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (ByteBuffer other_element : other.columns) { + __this__columns.add(other_element); + } + this.columns = __this__columns; + } + this.caching = other.caching; + if (other.isSetFilterString()) { + this.filterString = other.filterString; + } + } + + public TScan deepCopy() { + return new TScan(this); + } + + @Override + public void clear() { + this.startRow = null; + this.stopRow = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.columns = null; + setCachingIsSet(false); + this.caching = 0; + this.filterString = null; + } + + public byte[] getStartRow() { + setStartRow(org.apache.thrift.TBaseHelper.rightSize(startRow)); + return startRow == null ? null : startRow.array(); + } + + public ByteBuffer bufferForStartRow() { + return startRow; + } + + public TScan setStartRow(byte[] startRow) { + setStartRow(startRow == null ? (ByteBuffer)null : ByteBuffer.wrap(startRow)); + return this; + } + + public TScan setStartRow(ByteBuffer startRow) { + this.startRow = startRow; + return this; + } + + public void unsetStartRow() { + this.startRow = null; + } + + /** Returns true if field startRow is set (has been assigned a value) and false otherwise */ + public boolean isSetStartRow() { + return this.startRow != null; + } + + public void setStartRowIsSet(boolean value) { + if (!value) { + this.startRow = null; + } + } + + public byte[] getStopRow() { + setStopRow(org.apache.thrift.TBaseHelper.rightSize(stopRow)); + return stopRow == null ? null : stopRow.array(); + } + + public ByteBuffer bufferForStopRow() { + return stopRow; + } + + public TScan setStopRow(byte[] stopRow) { + setStopRow(stopRow == null ? (ByteBuffer)null : ByteBuffer.wrap(stopRow)); + return this; + } + + public TScan setStopRow(ByteBuffer stopRow) { + this.stopRow = stopRow; + return this; + } + + public void unsetStopRow() { + this.stopRow = null; + } + + /** Returns true if field stopRow is set (has been assigned a value) and false otherwise */ + public boolean isSetStopRow() { + return this.stopRow != null; + } + + public void setStopRowIsSet(boolean value) { + if (!value) { + this.stopRow = null; + } + } + + public long getTimestamp() { + return this.timestamp; + } + + public TScan setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(ByteBuffer elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + public List getColumns() { + return this.columns; + } + + public TScan setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public int getCaching() { + return this.caching; + } + + public TScan setCaching(int caching) { + this.caching = caching; + setCachingIsSet(true); + return this; + } + + public void unsetCaching() { + __isset_bit_vector.clear(__CACHING_ISSET_ID); + } + + /** Returns true if field caching is set (has been assigned a value) and false otherwise */ + public boolean isSetCaching() { + return __isset_bit_vector.get(__CACHING_ISSET_ID); + } + + public void setCachingIsSet(boolean value) { + __isset_bit_vector.set(__CACHING_ISSET_ID, value); + } + + public byte[] getFilterString() { + setFilterString(org.apache.thrift.TBaseHelper.rightSize(filterString)); + return filterString == null ? null : filterString.array(); + } + + public ByteBuffer bufferForFilterString() { + return filterString; + } + + public TScan setFilterString(byte[] filterString) { + setFilterString(filterString == null ? (ByteBuffer)null : ByteBuffer.wrap(filterString)); + return this; + } + + public TScan setFilterString(ByteBuffer filterString) { + this.filterString = filterString; + return this; + } + + public void unsetFilterString() { + this.filterString = null; + } + + /** Returns true if field filterString is set (has been assigned a value) and false otherwise */ + public boolean isSetFilterString() { + return this.filterString != null; + } + + public void setFilterStringIsSet(boolean value) { + if (!value) { + this.filterString = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case START_ROW: + if (value == null) { + unsetStartRow(); + } else { + setStartRow((ByteBuffer)value); + } + break; + + case STOP_ROW: + if (value == null) { + unsetStopRow(); + } else { + setStopRow((ByteBuffer)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case CACHING: + if (value == null) { + unsetCaching(); + } else { + setCaching((Integer)value); + } + break; + + case FILTER_STRING: + if (value == null) { + unsetFilterString(); + } else { + setFilterString((ByteBuffer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case START_ROW: + return getStartRow(); + + case STOP_ROW: + return getStopRow(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case COLUMNS: + return getColumns(); + + case CACHING: + return Integer.valueOf(getCaching()); + + case FILTER_STRING: + return getFilterString(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case START_ROW: + return isSetStartRow(); + case STOP_ROW: + return isSetStopRow(); + case TIMESTAMP: + return isSetTimestamp(); + case COLUMNS: + return isSetColumns(); + case CACHING: + return isSetCaching(); + case FILTER_STRING: + return isSetFilterString(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TScan) + return this.equals((TScan)that); + return false; + } + + public boolean equals(TScan that) { + if (that == null) + return false; + + boolean this_present_startRow = true && this.isSetStartRow(); + boolean that_present_startRow = true && that.isSetStartRow(); + if (this_present_startRow || that_present_startRow) { + if (!(this_present_startRow && that_present_startRow)) + return false; + if (!this.startRow.equals(that.startRow)) + return false; + } + + boolean this_present_stopRow = true && this.isSetStopRow(); + boolean that_present_stopRow = true && that.isSetStopRow(); + if (this_present_stopRow || that_present_stopRow) { + if (!(this_present_stopRow && that_present_stopRow)) + return false; + if (!this.stopRow.equals(that.stopRow)) + return false; + } + + boolean this_present_timestamp = true && this.isSetTimestamp(); + boolean that_present_timestamp = true && that.isSetTimestamp(); + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_caching = true && this.isSetCaching(); + boolean that_present_caching = true && that.isSetCaching(); + if (this_present_caching || that_present_caching) { + if (!(this_present_caching && that_present_caching)) + return false; + if (this.caching != that.caching) + return false; + } + + boolean this_present_filterString = true && this.isSetFilterString(); + boolean that_present_filterString = true && that.isSetFilterString(); + if (this_present_filterString || that_present_filterString) { + if (!(this_present_filterString && that_present_filterString)) + return false; + if (!this.filterString.equals(that.filterString)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TScan other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TScan typedOther = (TScan)other; + + lastComparison = Boolean.valueOf(isSetStartRow()).compareTo(typedOther.isSetStartRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStartRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.startRow, typedOther.startRow); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetStopRow()).compareTo(typedOther.isSetStopRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStopRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.stopRow, typedOther.stopRow); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetCaching()).compareTo(typedOther.isSetCaching()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCaching()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.caching, typedOther.caching); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetFilterString()).compareTo(typedOther.isSetFilterString()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetFilterString()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.filterString, typedOther.filterString); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TScan("); + boolean first = true; + + if (isSetStartRow()) { + sb.append("startRow:"); + if (this.startRow == null) { + sb.append("null"); + } else { + sb.append(this.startRow); + } + first = false; + } + if (isSetStopRow()) { + if (!first) sb.append(", "); + sb.append("stopRow:"); + if (this.stopRow == null) { + sb.append("null"); + } else { + sb.append(this.stopRow); + } + first = false; + } + if (isSetTimestamp()) { + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + } + if (isSetColumns()) { + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + } + if (isSetCaching()) { + if (!first) sb.append(", "); + sb.append("caching:"); + sb.append(this.caching); + first = false; + } + if (isSetFilterString()) { + if (!first) sb.append(", "); + sb.append("filterString:"); + if (this.filterString == null) { + sb.append("null"); + } else { + sb.append(this.filterString); + } + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TScanStandardSchemeFactory implements SchemeFactory { + public TScanStandardScheme getScheme() { + return new TScanStandardScheme(); + } + } + + private static class TScanStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TScan struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // START_ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.startRow = iprot.readBinary(); + struct.setStartRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // STOP_ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.stopRow = iprot.readBinary(); + struct.setStopRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list18 = iprot.readListBegin(); + struct.columns = new ArrayList(_list18.size); + for (int _i19 = 0; _i19 < _list18.size; ++_i19) + { + ByteBuffer _elem20; // optional + _elem20 = iprot.readBinary(); + struct.columns.add(_elem20); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // CACHING + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.caching = iprot.readI32(); + struct.setCachingIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 6: // FILTER_STRING + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.filterString = iprot.readBinary(); + struct.setFilterStringIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TScan struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.startRow != null) { + if (struct.isSetStartRow()) { + oprot.writeFieldBegin(START_ROW_FIELD_DESC); + oprot.writeBinary(struct.startRow); + oprot.writeFieldEnd(); + } + } + if (struct.stopRow != null) { + if (struct.isSetStopRow()) { + oprot.writeFieldBegin(STOP_ROW_FIELD_DESC); + oprot.writeBinary(struct.stopRow); + oprot.writeFieldEnd(); + } + } + if (struct.isSetTimestamp()) { + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + if (struct.isSetColumns()) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.columns.size())); + for (ByteBuffer _iter21 : struct.columns) + { + oprot.writeBinary(_iter21); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.isSetCaching()) { + oprot.writeFieldBegin(CACHING_FIELD_DESC); + oprot.writeI32(struct.caching); + oprot.writeFieldEnd(); + } + if (struct.filterString != null) { + if (struct.isSetFilterString()) { + oprot.writeFieldBegin(FILTER_STRING_FIELD_DESC); + oprot.writeBinary(struct.filterString); + oprot.writeFieldEnd(); + } + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TScanTupleSchemeFactory implements SchemeFactory { + public TScanTupleScheme getScheme() { + return new TScanTupleScheme(); + } + } + + private static class TScanTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetStartRow()) { + optionals.set(0); + } + if (struct.isSetStopRow()) { + optionals.set(1); + } + if (struct.isSetTimestamp()) { + optionals.set(2); + } + if (struct.isSetColumns()) { + optionals.set(3); + } + if (struct.isSetCaching()) { + optionals.set(4); + } + if (struct.isSetFilterString()) { + optionals.set(5); + } + oprot.writeBitSet(optionals, 6); + if (struct.isSetStartRow()) { + oprot.writeBinary(struct.startRow); + } + if (struct.isSetStopRow()) { + oprot.writeBinary(struct.stopRow); + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (ByteBuffer _iter22 : struct.columns) + { + oprot.writeBinary(_iter22); + } + } + } + if (struct.isSetCaching()) { + oprot.writeI32(struct.caching); + } + if (struct.isSetFilterString()) { + oprot.writeBinary(struct.filterString); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(6); + if (incoming.get(0)) { + struct.startRow = iprot.readBinary(); + struct.setStartRowIsSet(true); + } + if (incoming.get(1)) { + struct.stopRow = iprot.readBinary(); + struct.setStopRowIsSet(true); + } + if (incoming.get(2)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TList _list23 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.columns = new ArrayList(_list23.size); + for (int _i24 = 0; _i24 < _list23.size; ++_i24) + { + ByteBuffer _elem25; // optional + _elem25 = iprot.readBinary(); + struct.columns.add(_elem25); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(4)) { + struct.caching = iprot.readI32(); + struct.setCachingIsSet(true); + } + if (incoming.get(5)) { + struct.filterString = iprot.readBinary(); + struct.setFilterStringIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java new file mode 100644 index 0000000..2a8863f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java @@ -0,0 +1,343 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.thrift2; + +import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.deleteFromThrift; +import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.deletesFromHBase; +import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.deletesFromThrift; +import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.getFromThrift; +import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.getsFromThrift; +import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.incrementFromThrift; +import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.putFromThrift; +import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.putsFromThrift; +import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.resultFromHBase; +import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.resultsFromHBase; +import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.scanFromThrift; + +import java.io.IOException; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.HTablePool; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.thrift.ThriftMetrics; +import org.apache.hadoop.hbase.thrift2.generated.TDelete; +import org.apache.hadoop.hbase.thrift2.generated.TGet; +import org.apache.hadoop.hbase.thrift2.generated.THBaseService; +import org.apache.hadoop.hbase.thrift2.generated.TIOError; +import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument; +import org.apache.hadoop.hbase.thrift2.generated.TIncrement; +import org.apache.hadoop.hbase.thrift2.generated.TPut; +import org.apache.hadoop.hbase.thrift2.generated.TResult; +import org.apache.hadoop.hbase.thrift2.generated.TScan; +import org.apache.thrift.TException; + +/** + * This class is a glue object that connects Thrift RPC calls to the HBase client API primarily defined in the + * HTableInterface. + */ +public class ThriftHBaseServiceHandler implements THBaseService.Iface { + + // TODO: Size of pool configuraple + private final HTablePool htablePool; + private static final Log LOG = LogFactory.getLog(ThriftHBaseServiceHandler.class); + + // nextScannerId and scannerMap are used to manage scanner state + // TODO: Cleanup thread for Scanners, Scanner id wrap + private final AtomicInteger nextScannerId = new AtomicInteger(0); + private final Map scannerMap = new ConcurrentHashMap(); + + public static THBaseService.Iface newInstance( + Configuration conf, ThriftMetrics metrics) { + THBaseService.Iface handler = new ThriftHBaseServiceHandler(conf); + return (THBaseService.Iface) Proxy.newProxyInstance( + handler.getClass().getClassLoader(), + new Class[]{THBaseService.Iface.class}, + new THBaseServiceMetricsProxy(handler, metrics)); + } + + private static class THBaseServiceMetricsProxy implements InvocationHandler { + private final THBaseService.Iface handler; + private final ThriftMetrics metrics; + + private THBaseServiceMetricsProxy( + THBaseService.Iface handler, ThriftMetrics metrics) { + this.handler = handler; + this.metrics = metrics; + } + + @Override + public Object invoke(Object proxy, Method m, Object[] args) + throws Throwable { + Object result; + try { + long start = now(); + result = m.invoke(handler, args); + int processTime = (int)(now() - start); + metrics.incMethodTime(m.getName(), processTime); + } catch (InvocationTargetException e) { + throw e.getTargetException(); + } catch (Exception e) { + throw new RuntimeException( + "unexpected invocation exception: " + e.getMessage()); + } + return result; + } + } + + private static long now() { + return System.nanoTime(); + } + + ThriftHBaseServiceHandler(Configuration conf) { + htablePool = new HTablePool(conf, Integer.MAX_VALUE); + } + + private HTableInterface getTable(byte[] tableName) { + return htablePool.getTable(tableName); + } + + private void closeTable(HTableInterface table) throws TIOError { + try { + table.close(); + } catch (IOException e) { + throw getTIOError(e); + } + } + + private TIOError getTIOError(IOException e) { + TIOError err = new TIOError(); + err.setMessage(e.getMessage()); + return err; + } + + /** + * Assigns a unique ID to the scanner and adds the mapping to an internal HashMap. + * + * @param scanner to add + * @return Id for this Scanner + */ + private int addScanner(ResultScanner scanner) { + int id = nextScannerId.getAndIncrement(); + scannerMap.put(id, scanner); + return id; + } + + /** + * Returns the Scanner associated with the specified Id. + * + * @param id of the Scanner to get + * @return a Scanner, or null if the Id is invalid + */ + private ResultScanner getScanner(int id) { + return scannerMap.get(id); + } + + /** + * Removes the scanner associated with the specified ID from the internal HashMap. + * + * @param id of the Scanner to remove + * @return the removed Scanner, or null if the Id is invalid + */ + protected ResultScanner removeScanner(int id) { + return scannerMap.remove(id); + } + + @Override + public boolean exists(ByteBuffer table, TGet get) throws TIOError, TException { + HTableInterface htable = getTable(table.array()); + try { + return htable.exists(getFromThrift(get)); + } catch (IOException e) { + throw getTIOError(e); + } finally { + closeTable(htable); + } + } + + @Override + public TResult get(ByteBuffer table, TGet get) throws TIOError, TException { + HTableInterface htable = getTable(table.array()); + try { + return resultFromHBase(htable.get(getFromThrift(get))); + } catch (IOException e) { + throw getTIOError(e); + } finally { + closeTable(htable); + } + } + + @Override + public List getMultiple(ByteBuffer table, List gets) throws TIOError, TException { + HTableInterface htable = getTable(table.array()); + try { + return resultsFromHBase(htable.get(getsFromThrift(gets))); + } catch (IOException e) { + throw getTIOError(e); + } finally { + closeTable(htable); + } + } + + @Override + public void put(ByteBuffer table, TPut put) throws TIOError, TException { + HTableInterface htable = getTable(table.array()); + try { + htable.put(putFromThrift(put)); + } catch (IOException e) { + throw getTIOError(e); + } finally { + closeTable(htable); + } + } + + @Override + public boolean checkAndPut(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TPut put) + throws TIOError, TException { + HTableInterface htable = getTable(table.array()); + try { + return htable.checkAndPut(row.array(), family.array(), qualifier.array(), (value == null) ? null : value.array(), putFromThrift(put)); + } catch (IOException e) { + throw getTIOError(e); + } finally { + closeTable(htable); + } + } + + @Override + public void putMultiple(ByteBuffer table, List puts) throws TIOError, TException { + HTableInterface htable = getTable(table.array()); + try { + htable.put(putsFromThrift(puts)); + } catch (IOException e) { + throw getTIOError(e); + } finally { + closeTable(htable); + } + } + + @Override + public void deleteSingle(ByteBuffer table, TDelete deleteSingle) throws TIOError, TException { + HTableInterface htable = getTable(table.array()); + try { + htable.delete(deleteFromThrift(deleteSingle)); + } catch (IOException e) { + throw getTIOError(e); + } finally { + closeTable(htable); + } + } + + @Override + public List deleteMultiple(ByteBuffer table, List deletes) throws TIOError, TException { + HTableInterface htable = getTable(table.array()); + List tempDeletes = deletesFromThrift(deletes); + try { + htable.delete(tempDeletes); + } catch (IOException e) { + throw getTIOError(e); + } finally { + closeTable(htable); + } + return deletesFromHBase(tempDeletes); + } + + @Override + public boolean checkAndDelete(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, + TDelete deleteSingle) throws TIOError, TException { + HTableInterface htable = getTable(table.array()); + + try { + if (value == null) { + return htable.checkAndDelete(row.array(), family.array(), qualifier.array(), null, deleteFromThrift(deleteSingle)); + } else { + return htable.checkAndDelete(row.array(), family.array(), qualifier.array(), value.array(), deleteFromThrift(deleteSingle)); + } + } catch (IOException e) { + throw getTIOError(e); + } finally { + closeTable(htable); + } + } + + @Override + public TResult increment(ByteBuffer table, TIncrement increment) throws TIOError, TException { + HTableInterface htable = getTable(table.array()); + try { + return resultFromHBase(htable.increment(incrementFromThrift(increment))); + } catch (IOException e) { + throw getTIOError(e); + } finally { + closeTable(htable); + } + } + + @Override + public int openScanner(ByteBuffer table, TScan scan) throws TIOError, TException { + HTableInterface htable = getTable(table.array()); + ResultScanner resultScanner = null; + try { + resultScanner = htable.getScanner(scanFromThrift(scan)); + } catch (IOException e) { + throw getTIOError(e); + } finally { + closeTable(htable); + } + return addScanner(resultScanner); + } + + @Override + public List getScannerRows(int scannerId, int numRows) throws TIOError, TIllegalArgument, TException { + ResultScanner scanner = getScanner(scannerId); + if (scanner == null) { + TIllegalArgument ex = new TIllegalArgument(); + ex.setMessage("Invalid scanner Id"); + throw ex; + } + + try { + return resultsFromHBase(scanner.next(numRows)); + } catch (IOException e) { + throw getTIOError(e); + } + } + + @Override + public void closeScanner(int scannerId) throws TIOError, TIllegalArgument, TException { + if (removeScanner(scannerId) == null) { + TIllegalArgument ex = new TIllegalArgument(); + ex.setMessage("Invalid scanner Id"); + throw ex; + } + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java new file mode 100644 index 0000000..1b97c63 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java @@ -0,0 +1,300 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.thrift2; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.thrift.CallQueue; +import org.apache.hadoop.hbase.thrift.CallQueue.Call; +import org.apache.hadoop.hbase.thrift.ThriftMetrics; +import org.apache.hadoop.hbase.thrift2.generated.THBaseService; +import org.apache.hadoop.hbase.util.InfoServer; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.server.THsHaServer; +import org.apache.thrift.server.TNonblockingServer; +import org.apache.thrift.server.TServer; +import org.apache.thrift.server.TThreadPoolServer; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TNonblockingServerSocket; +import org.apache.thrift.transport.TNonblockingServerTransport; +import org.apache.thrift.transport.TServerSocket; +import org.apache.thrift.transport.TServerTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.transport.TTransportFactory; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * ThriftServer - this class starts up a Thrift server which implements the HBase API specified in the + * HbaseClient.thrift IDL file. + */ +@SuppressWarnings({ "rawtypes", "unchecked" }) +public class ThriftServer { + private static final Log log = LogFactory.getLog(ThriftServer.class); + + public static final String DEFAULT_LISTEN_PORT = "9090"; + + public ThriftServer() { + } + + private static void printUsage() { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("Thrift", null, getOptions(), + "To start the Thrift server run 'bin/hbase-daemon.sh start thrift2'\n" + + "To shutdown the thrift server run 'bin/hbase-daemon.sh stop thrift2' or" + + " send a kill signal to the thrift server pid", + true); + } + + private static Options getOptions() { + Options options = new Options(); + options.addOption("b", "bind", true, + "Address to bind the Thrift server to. Not supported by the Nonblocking and HsHa server [default: 0.0.0.0]"); + options.addOption("p", "port", true, "Port to bind to [default: " + DEFAULT_LISTEN_PORT + "]"); + options.addOption("f", "framed", false, "Use framed transport"); + options.addOption("c", "compact", false, "Use the compact protocol"); + options.addOption("h", "help", false, "Print help information"); + options.addOption(null, "infoport", true, "Port for web UI"); + + OptionGroup servers = new OptionGroup(); + servers.addOption( + new Option("nonblocking", false, "Use the TNonblockingServer. This implies the framed transport.")); + servers.addOption(new Option("hsha", false, "Use the THsHaServer. This implies the framed transport.")); + servers.addOption(new Option("threadpool", false, "Use the TThreadPoolServer. This is the default.")); + options.addOptionGroup(servers); + return options; + } + + private static CommandLine parseArguments(Options options, String[] args) throws ParseException { + CommandLineParser parser = new PosixParser(); + return parser.parse(options, args); + } + + private static TProtocolFactory getTProtocolFactory(boolean isCompact) { + if (isCompact) { + log.debug("Using compact protocol"); + return new TCompactProtocol.Factory(); + } else { + log.debug("Using binary protocol"); + return new TBinaryProtocol.Factory(); + } + } + + private static TTransportFactory getTTransportFactory(boolean framed) { + if (framed) { + log.debug("Using framed transport"); + return new TFramedTransport.Factory(); + } else { + return new TTransportFactory(); + } + } + + /* + * If bindValue is null, we don't bind. + */ + private static InetSocketAddress bindToPort(String bindValue, int listenPort) + throws UnknownHostException { + try { + if (bindValue == null) { + return new InetSocketAddress(listenPort); + } else { + return new InetSocketAddress(InetAddress.getByName(bindValue), listenPort); + } + } catch (UnknownHostException e) { + throw new RuntimeException("Could not bind to provided ip address", e); + } + } + + private static TServer getTNonBlockingServer(TProtocolFactory protocolFactory, THBaseService.Processor processor, + TTransportFactory transportFactory, InetSocketAddress inetSocketAddress) throws TTransportException { + TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress); + log.info("starting HBase Nonblocking Thrift server on " + inetSocketAddress.toString()); + TNonblockingServer.Args serverArgs = new TNonblockingServer.Args(serverTransport); + serverArgs.processor(processor); + serverArgs.transportFactory(transportFactory); + serverArgs.protocolFactory(protocolFactory); + return new TNonblockingServer(serverArgs); + } + + private static TServer getTHsHaServer(TProtocolFactory protocolFactory, + THBaseService.Processor processor, TTransportFactory transportFactory, + InetSocketAddress inetSocketAddress, ThriftMetrics metrics) + throws TTransportException { + TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress); + log.info("starting HBase HsHA Thrift server on " + inetSocketAddress.toString()); + THsHaServer.Args serverArgs = new THsHaServer.Args(serverTransport); + ExecutorService executorService = createExecutor( + serverArgs.getWorkerThreads(), metrics); + serverArgs.executorService(executorService); + serverArgs.processor(processor); + serverArgs.transportFactory(transportFactory); + serverArgs.protocolFactory(protocolFactory); + return new THsHaServer(serverArgs); + } + + private static ExecutorService createExecutor( + int workerThreads, ThriftMetrics metrics) { + CallQueue callQueue = new CallQueue( + new LinkedBlockingQueue(), metrics); + ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); + tfb.setDaemon(true); + tfb.setNameFormat("thrift2-worker-%d"); + return new ThreadPoolExecutor(workerThreads, workerThreads, + Long.MAX_VALUE, TimeUnit.SECONDS, callQueue, tfb.build()); + } + + private static TServer getTThreadPoolServer(TProtocolFactory protocolFactory, THBaseService.Processor processor, + TTransportFactory transportFactory, InetSocketAddress inetSocketAddress) throws TTransportException { + TServerTransport serverTransport = new TServerSocket(inetSocketAddress); + log.info("starting HBase ThreadPool Thrift server on " + inetSocketAddress.toString()); + TThreadPoolServer.Args serverArgs = new TThreadPoolServer.Args(serverTransport); + serverArgs.processor(processor); + serverArgs.transportFactory(transportFactory); + serverArgs.protocolFactory(protocolFactory); + return new TThreadPoolServer(serverArgs); + } + + /** + * Start up the Thrift2 server. + * + * @param args + */ + public static void main(String[] args) throws Exception { + TServer server = null; + Options options = getOptions(); + try { + CommandLine cmd = parseArguments(options, args); + + /** + * This is to please both bin/hbase and bin/hbase-daemon. hbase-daemon provides "start" and "stop" arguments hbase + * should print the help if no argument is provided + */ + List argList = cmd.getArgList(); + if (cmd.hasOption("help") || !argList.contains("start") || argList.contains("stop")) { + printUsage(); + System.exit(1); + } + + // Get port to bind to + int listenPort = 0; + try { + listenPort = Integer.parseInt(cmd.getOptionValue("port", DEFAULT_LISTEN_PORT)); + } catch (NumberFormatException e) { + throw new RuntimeException("Could not parse the value provided for the port option", e); + } + + boolean nonblocking = cmd.hasOption("nonblocking"); + boolean hsha = cmd.hasOption("hsha"); + + Configuration conf = HBaseConfiguration.create(); + ThriftMetrics metrics = new ThriftMetrics( + listenPort, conf, THBaseService.Iface.class); + + String implType = "threadpool"; + if (nonblocking) { + implType = "nonblocking"; + } else if (hsha) { + implType = "hsha"; + } + + conf.set("hbase.regionserver.thrift.server.type", implType); + conf.setInt("hbase.regionserver.thrift.port", listenPort); + + // Construct correct ProtocolFactory + boolean compact = cmd.hasOption("compact"); + TProtocolFactory protocolFactory = getTProtocolFactory(compact); + THBaseService.Iface handler = + ThriftHBaseServiceHandler.newInstance(conf, metrics); + THBaseService.Processor processor = new THBaseService.Processor(handler); + conf.setBoolean("hbase.regionserver.thrift.compact", compact); + + boolean framed = cmd.hasOption("framed") || nonblocking || hsha; + TTransportFactory transportFactory = getTTransportFactory(framed); + conf.setBoolean("hbase.regionserver.thrift.framed", framed); + + // TODO: Remove once HBASE-2155 is resolved + if (cmd.hasOption("bind") && (nonblocking || hsha)) { + log.error("The Nonblocking and HsHaServer servers don't support IP address binding at the moment." + + " See https://issues.apache.org/jira/browse/HBASE-2155 for details."); + printUsage(); + System.exit(1); + } + + // check for user-defined info server port setting, if so override the conf + try { + if (cmd.hasOption("infoport")) { + String val = cmd.getOptionValue("infoport"); + conf.setInt("hbase.thrift.info.port", Integer.valueOf(val)); + log.debug("Web UI port set to " + val); + } + } catch (NumberFormatException e) { + log.error("Could not parse the value provided for the infoport option", e); + printUsage(); + System.exit(1); + } + + // Put up info server. + int port = conf.getInt("hbase.thrift.info.port", 9095); + if (port >= 0) { + conf.setLong("startcode", System.currentTimeMillis()); + String a = conf.get("hbase.thrift.info.bindAddress", "0.0.0.0"); + InfoServer infoServer = new InfoServer("thrift", a, port, false, conf); + infoServer.setAttribute("hbase.conf", conf); + infoServer.start(); + } + + InetSocketAddress inetSocketAddress = bindToPort(cmd.getOptionValue("bind"), listenPort); + + if (nonblocking) { + server = getTNonBlockingServer(protocolFactory, processor, transportFactory, inetSocketAddress); + } else if (hsha) { + server = getTHsHaServer(protocolFactory, processor, transportFactory, inetSocketAddress, metrics); + } else { + server = getTThreadPoolServer(protocolFactory, processor, transportFactory, inetSocketAddress); + } + } catch (Exception e) { + log.error(e.getMessage(), e); + printUsage(); + System.exit(1); + } + server.serve(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java new file mode 100644 index 0000000..b63ed74 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java @@ -0,0 +1,336 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.thrift2; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.thrift2.generated.*; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; + +public class ThriftUtilities { + + private ThriftUtilities() { + throw new UnsupportedOperationException("Can't initialize class"); + } + + /** + * Creates a {@link Get} (HBase) from a {@link TGet} (Thrift). + * + * This ignores any timestamps set on {@link TColumn} objects. + * + * @param in the TGet to convert + * + * @return Get object + * + * @throws IOException if an invalid time range or max version parameter is given + */ + public static Get getFromThrift(TGet in) throws IOException { + Get out = new Get(in.getRow()); + + // Timestamp overwrites time range if both are set + if (in.isSetTimestamp()) { + out.setTimeStamp(in.getTimestamp()); + } else if (in.isSetTimeRange()) { + out.setTimeRange(in.getTimeRange().getMinStamp(), in.getTimeRange().getMaxStamp()); + } + + if (in.isSetMaxVersions()) { + out.setMaxVersions(in.getMaxVersions()); + } + + if (!in.isSetColumns()) { + return out; + } + + for (TColumn column : in.getColumns()) { + if (column.isSetQualifier()) { + out.addColumn(column.getFamily(), column.getQualifier()); + } else { + out.addFamily(column.getFamily()); + } + } + + return out; + } + + /** + * Converts multiple {@link TGet}s (Thrift) into a list of {@link Get}s (HBase). + * + * @param in list of TGets to convert + * + * @return list of Get objects + * + * @throws IOException if an invalid time range or max version parameter is given + * @see #getFromThrift(TGet) + */ + public static List getsFromThrift(List in) throws IOException { + List out = new ArrayList(in.size()); + for (TGet get : in) { + out.add(getFromThrift(get)); + } + return out; + } + + /** + * Creates a {@link TResult} (Thrift) from a {@link Result} (HBase). + * + * @param in the Result to convert + * + * @return converted result, returns an empty result if the input is null + */ + public static TResult resultFromHBase(Result in) { + KeyValue[] raw = in.raw(); + TResult out = new TResult(); + byte[] row = in.getRow(); + if (row != null) { + out.setRow(in.getRow()); + } + List columnValues = new ArrayList(); + for (KeyValue kv : raw) { + TColumnValue col = new TColumnValue(); + col.setFamily(kv.getFamily()); + col.setQualifier(kv.getQualifier()); + col.setTimestamp(kv.getTimestamp()); + col.setValue(kv.getValue()); + columnValues.add(col); + } + out.setColumnValues(columnValues); + return out; + } + + /** + * Converts multiple {@link Result}s (HBase) into a list of {@link TResult}s (Thrift). + * + * @param in array of Results to convert + * + * @return list of converted TResults + * + * @see #resultFromHBase(Result) + */ + public static List resultsFromHBase(Result[] in) { + List out = new ArrayList(in.length); + for (Result result : in) { + out.add(resultFromHBase(result)); + } + return out; + } + + /** + * Creates a {@link Put} (HBase) from a {@link TPut} (Thrift) + * + * @param in the TPut to convert + * + * @return converted Put + */ + public static Put putFromThrift(TPut in) { + Put out; + + if (in.isSetTimestamp()) { + out = new Put(in.getRow(), in.getTimestamp(), null); + } else { + out = new Put(in.getRow()); + } + + out.setWriteToWAL(in.isWriteToWal()); + + for (TColumnValue columnValue : in.getColumnValues()) { + if (columnValue.isSetTimestamp()) { + out.add(columnValue.getFamily(), columnValue.getQualifier(), columnValue.getTimestamp(), + columnValue.getValue()); + } else { + out.add(columnValue.getFamily(), columnValue.getQualifier(), columnValue.getValue()); + } + } + + return out; + } + + /** + * Converts multiple {@link TPut}s (Thrift) into a list of {@link Put}s (HBase). + * + * @param in list of TPuts to convert + * + * @return list of converted Puts + * + * @see #putFromThrift(TPut) + */ + public static List putsFromThrift(List in) { + List out = new ArrayList(in.size()); + for (TPut put : in) { + out.add(putFromThrift(put)); + } + return out; + } + + /** + * Creates a {@link Delete} (HBase) from a {@link TDelete} (Thrift). + * + * @param in the TDelete to convert + * + * @return converted Delete + */ + public static Delete deleteFromThrift(TDelete in) { + Delete out; + + if (in.isSetColumns()) { + out = new Delete(in.getRow()); + for (TColumn column : in.getColumns()) { + if (column.isSetQualifier()) { + if (column.isSetTimestamp()) { + if (in.isSetDeleteType() && + in.getDeleteType().equals(TDeleteType.DELETE_COLUMNS)) + out.deleteColumns(column.getFamily(), column.getQualifier(), column.getTimestamp()); + else + out.deleteColumn(column.getFamily(), column.getQualifier(), column.getTimestamp()); + } else { + if (in.isSetDeleteType() && + in.getDeleteType().equals(TDeleteType.DELETE_COLUMNS)) + out.deleteColumns(column.getFamily(), column.getQualifier()); + else + out.deleteColumn(column.getFamily(), column.getQualifier()); + } + + } else { + if (column.isSetTimestamp()) { + out.deleteFamily(column.getFamily(), column.getTimestamp()); + } else { + out.deleteFamily(column.getFamily()); + } + } + } + } else { + if (in.isSetTimestamp()) { + out = new Delete(in.getRow(), in.getTimestamp(), null); + } else { + out = new Delete(in.getRow()); + } + } + out.setWriteToWAL(in.isWriteToWal()); + return out; + } + + /** + * Converts multiple {@link TDelete}s (Thrift) into a list of {@link Delete}s (HBase). + * + * @param in list of TDeletes to convert + * + * @return list of converted Deletes + * + * @see #deleteFromThrift(TDelete) + */ + + public static List deletesFromThrift(List in) { + List out = new ArrayList(in.size()); + for (TDelete delete : in) { + out.add(deleteFromThrift(delete)); + } + return out; + } + + public static TDelete deleteFromHBase(Delete in) { + TDelete out = new TDelete(ByteBuffer.wrap(in.getRow())); + + List columns = new ArrayList(); + long rowTimestamp = in.getTimeStamp(); + if (rowTimestamp != HConstants.LATEST_TIMESTAMP) { + out.setTimestamp(rowTimestamp); + } + + // Map> + for (Map.Entry> familyEntry : in.getFamilyMap().entrySet()) { + TColumn column = new TColumn(ByteBuffer.wrap(familyEntry.getKey())); + for (KeyValue keyValue : familyEntry.getValue()) { + byte[] family = keyValue.getFamily(); + byte[] qualifier = keyValue.getQualifier(); + long timestamp = keyValue.getTimestamp(); + if (family != null) { + column.setFamily(family); + } + if (qualifier != null) { + column.setQualifier(qualifier); + } + if (timestamp != HConstants.LATEST_TIMESTAMP) { + column.setTimestamp(keyValue.getTimestamp()); + } + } + columns.add(column); + } + out.setColumns(columns); + + return out; + } + + public static List deletesFromHBase(List in) { + List out = new ArrayList(in.size()); + for (Delete delete : in) { + if (delete == null) { + out.add(null); + } else { + out.add(deleteFromHBase(delete)); + } + } + return out; + } + + public static Scan scanFromThrift(TScan in) throws IOException { + Scan out = new Scan(); + + if (in.isSetStartRow()) + out.setStartRow(in.getStartRow()); + if (in.isSetStopRow()) + out.setStopRow(in.getStopRow()); + if (in.isSetCaching()) + out.setCaching(in.getCaching()); + if (in.isSetMaxVersions()) { + out.setMaxVersions(in.getMaxVersions()); + } + + if (in.isSetColumns()) { + for (TColumn column : in.getColumns()) { + if (column.isSetQualifier()) { + out.addColumn(column.getFamily(), column.getQualifier()); + } else { + out.addFamily(column.getFamily()); + } + } + } + + TTimeRange timeRange = in.getTimeRange(); + if (timeRange != null && + timeRange.isSetMinStamp() && timeRange.isSetMaxStamp()) { + out.setTimeRange(timeRange.getMinStamp(), timeRange.getMaxStamp()); + } + + return out; + } + + public static Increment incrementFromThrift(TIncrement in) throws IOException { + Increment out = new Increment(in.getRow()); + for (TColumnIncrement column : in.getColumns()) { + out.addColumn(column.getFamily(), column.getQualifier(), column.getAmount()); + } + out.setWriteToWAL(in.isWriteToWal()); + return out; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumn.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumn.java new file mode 100644 index 0000000..ba11ece --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumn.java @@ -0,0 +1,608 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Addresses a single cell or multiple cells + * in a HBase table by column family and optionally + * a column qualifier and timestamp + */ +public class TColumn implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumn"); + + private static final org.apache.thrift.protocol.TField FAMILY_FIELD_DESC = new org.apache.thrift.protocol.TField("family", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField QUALIFIER_FIELD_DESC = new org.apache.thrift.protocol.TField("qualifier", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TColumnStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TColumnTupleSchemeFactory()); + } + + public ByteBuffer family; // required + public ByteBuffer qualifier; // optional + public long timestamp; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + FAMILY((short)1, "family"), + QUALIFIER((short)2, "qualifier"), + TIMESTAMP((short)3, "timestamp"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // FAMILY + return FAMILY; + case 2: // QUALIFIER + return QUALIFIER; + case 3: // TIMESTAMP + return TIMESTAMP; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + private _Fields optionals[] = {_Fields.QUALIFIER,_Fields.TIMESTAMP}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.FAMILY, new org.apache.thrift.meta_data.FieldMetaData("family", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.QUALIFIER, new org.apache.thrift.meta_data.FieldMetaData("qualifier", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TColumn.class, metaDataMap); + } + + public TColumn() { + } + + public TColumn( + ByteBuffer family) + { + this(); + this.family = family; + } + + /** + * Performs a deep copy on other. + */ + public TColumn(TColumn other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetFamily()) { + this.family = org.apache.thrift.TBaseHelper.copyBinary(other.family); +; + } + if (other.isSetQualifier()) { + this.qualifier = org.apache.thrift.TBaseHelper.copyBinary(other.qualifier); +; + } + this.timestamp = other.timestamp; + } + + public TColumn deepCopy() { + return new TColumn(this); + } + + @Override + public void clear() { + this.family = null; + this.qualifier = null; + setTimestampIsSet(false); + this.timestamp = 0; + } + + public byte[] getFamily() { + setFamily(org.apache.thrift.TBaseHelper.rightSize(family)); + return family == null ? null : family.array(); + } + + public ByteBuffer bufferForFamily() { + return family; + } + + public TColumn setFamily(byte[] family) { + setFamily(family == null ? (ByteBuffer)null : ByteBuffer.wrap(family)); + return this; + } + + public TColumn setFamily(ByteBuffer family) { + this.family = family; + return this; + } + + public void unsetFamily() { + this.family = null; + } + + /** Returns true if field family is set (has been assigned a value) and false otherwise */ + public boolean isSetFamily() { + return this.family != null; + } + + public void setFamilyIsSet(boolean value) { + if (!value) { + this.family = null; + } + } + + public byte[] getQualifier() { + setQualifier(org.apache.thrift.TBaseHelper.rightSize(qualifier)); + return qualifier == null ? null : qualifier.array(); + } + + public ByteBuffer bufferForQualifier() { + return qualifier; + } + + public TColumn setQualifier(byte[] qualifier) { + setQualifier(qualifier == null ? (ByteBuffer)null : ByteBuffer.wrap(qualifier)); + return this; + } + + public TColumn setQualifier(ByteBuffer qualifier) { + this.qualifier = qualifier; + return this; + } + + public void unsetQualifier() { + this.qualifier = null; + } + + /** Returns true if field qualifier is set (has been assigned a value) and false otherwise */ + public boolean isSetQualifier() { + return this.qualifier != null; + } + + public void setQualifierIsSet(boolean value) { + if (!value) { + this.qualifier = null; + } + } + + public long getTimestamp() { + return this.timestamp; + } + + public TColumn setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case FAMILY: + if (value == null) { + unsetFamily(); + } else { + setFamily((ByteBuffer)value); + } + break; + + case QUALIFIER: + if (value == null) { + unsetQualifier(); + } else { + setQualifier((ByteBuffer)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case FAMILY: + return getFamily(); + + case QUALIFIER: + return getQualifier(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case FAMILY: + return isSetFamily(); + case QUALIFIER: + return isSetQualifier(); + case TIMESTAMP: + return isSetTimestamp(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TColumn) + return this.equals((TColumn)that); + return false; + } + + public boolean equals(TColumn that) { + if (that == null) + return false; + + boolean this_present_family = true && this.isSetFamily(); + boolean that_present_family = true && that.isSetFamily(); + if (this_present_family || that_present_family) { + if (!(this_present_family && that_present_family)) + return false; + if (!this.family.equals(that.family)) + return false; + } + + boolean this_present_qualifier = true && this.isSetQualifier(); + boolean that_present_qualifier = true && that.isSetQualifier(); + if (this_present_qualifier || that_present_qualifier) { + if (!(this_present_qualifier && that_present_qualifier)) + return false; + if (!this.qualifier.equals(that.qualifier)) + return false; + } + + boolean this_present_timestamp = true && this.isSetTimestamp(); + boolean that_present_timestamp = true && that.isSetTimestamp(); + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TColumn other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TColumn typedOther = (TColumn)other; + + lastComparison = Boolean.valueOf(isSetFamily()).compareTo(typedOther.isSetFamily()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetFamily()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.family, typedOther.family); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetQualifier()).compareTo(typedOther.isSetQualifier()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetQualifier()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.qualifier, typedOther.qualifier); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TColumn("); + boolean first = true; + + sb.append("family:"); + if (this.family == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.family, sb); + } + first = false; + if (isSetQualifier()) { + if (!first) sb.append(", "); + sb.append("qualifier:"); + if (this.qualifier == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.qualifier, sb); + } + first = false; + } + if (isSetTimestamp()) { + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (family == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'family' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TColumnStandardSchemeFactory implements SchemeFactory { + public TColumnStandardScheme getScheme() { + return new TColumnStandardScheme(); + } + } + + private static class TColumnStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TColumn struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // FAMILY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.family = iprot.readBinary(); + struct.setFamilyIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // QUALIFIER + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.qualifier = iprot.readBinary(); + struct.setQualifierIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TColumn struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.family != null) { + oprot.writeFieldBegin(FAMILY_FIELD_DESC); + oprot.writeBinary(struct.family); + oprot.writeFieldEnd(); + } + if (struct.qualifier != null) { + if (struct.isSetQualifier()) { + oprot.writeFieldBegin(QUALIFIER_FIELD_DESC); + oprot.writeBinary(struct.qualifier); + oprot.writeFieldEnd(); + } + } + if (struct.isSetTimestamp()) { + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TColumnTupleSchemeFactory implements SchemeFactory { + public TColumnTupleScheme getScheme() { + return new TColumnTupleScheme(); + } + } + + private static class TColumnTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TColumn struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.family); + BitSet optionals = new BitSet(); + if (struct.isSetQualifier()) { + optionals.set(0); + } + if (struct.isSetTimestamp()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetQualifier()) { + oprot.writeBinary(struct.qualifier); + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TColumn struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.family = iprot.readBinary(); + struct.setFamilyIsSet(true); + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.qualifier = iprot.readBinary(); + struct.setQualifierIsSet(true); + } + if (incoming.get(1)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnIncrement.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnIncrement.java new file mode 100644 index 0000000..6362250 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnIncrement.java @@ -0,0 +1,602 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Represents a single cell and the amount to increment it by + */ +public class TColumnIncrement implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumnIncrement"); + + private static final org.apache.thrift.protocol.TField FAMILY_FIELD_DESC = new org.apache.thrift.protocol.TField("family", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField QUALIFIER_FIELD_DESC = new org.apache.thrift.protocol.TField("qualifier", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField AMOUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("amount", org.apache.thrift.protocol.TType.I64, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TColumnIncrementStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TColumnIncrementTupleSchemeFactory()); + } + + public ByteBuffer family; // required + public ByteBuffer qualifier; // required + public long amount; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + FAMILY((short)1, "family"), + QUALIFIER((short)2, "qualifier"), + AMOUNT((short)3, "amount"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // FAMILY + return FAMILY; + case 2: // QUALIFIER + return QUALIFIER; + case 3: // AMOUNT + return AMOUNT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __AMOUNT_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + private _Fields optionals[] = {_Fields.AMOUNT}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.FAMILY, new org.apache.thrift.meta_data.FieldMetaData("family", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.QUALIFIER, new org.apache.thrift.meta_data.FieldMetaData("qualifier", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.AMOUNT, new org.apache.thrift.meta_data.FieldMetaData("amount", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TColumnIncrement.class, metaDataMap); + } + + public TColumnIncrement() { + this.amount = 1L; + + } + + public TColumnIncrement( + ByteBuffer family, + ByteBuffer qualifier) + { + this(); + this.family = family; + this.qualifier = qualifier; + } + + /** + * Performs a deep copy on other. + */ + public TColumnIncrement(TColumnIncrement other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetFamily()) { + this.family = org.apache.thrift.TBaseHelper.copyBinary(other.family); +; + } + if (other.isSetQualifier()) { + this.qualifier = org.apache.thrift.TBaseHelper.copyBinary(other.qualifier); +; + } + this.amount = other.amount; + } + + public TColumnIncrement deepCopy() { + return new TColumnIncrement(this); + } + + @Override + public void clear() { + this.family = null; + this.qualifier = null; + this.amount = 1L; + + } + + public byte[] getFamily() { + setFamily(org.apache.thrift.TBaseHelper.rightSize(family)); + return family == null ? null : family.array(); + } + + public ByteBuffer bufferForFamily() { + return family; + } + + public TColumnIncrement setFamily(byte[] family) { + setFamily(family == null ? (ByteBuffer)null : ByteBuffer.wrap(family)); + return this; + } + + public TColumnIncrement setFamily(ByteBuffer family) { + this.family = family; + return this; + } + + public void unsetFamily() { + this.family = null; + } + + /** Returns true if field family is set (has been assigned a value) and false otherwise */ + public boolean isSetFamily() { + return this.family != null; + } + + public void setFamilyIsSet(boolean value) { + if (!value) { + this.family = null; + } + } + + public byte[] getQualifier() { + setQualifier(org.apache.thrift.TBaseHelper.rightSize(qualifier)); + return qualifier == null ? null : qualifier.array(); + } + + public ByteBuffer bufferForQualifier() { + return qualifier; + } + + public TColumnIncrement setQualifier(byte[] qualifier) { + setQualifier(qualifier == null ? (ByteBuffer)null : ByteBuffer.wrap(qualifier)); + return this; + } + + public TColumnIncrement setQualifier(ByteBuffer qualifier) { + this.qualifier = qualifier; + return this; + } + + public void unsetQualifier() { + this.qualifier = null; + } + + /** Returns true if field qualifier is set (has been assigned a value) and false otherwise */ + public boolean isSetQualifier() { + return this.qualifier != null; + } + + public void setQualifierIsSet(boolean value) { + if (!value) { + this.qualifier = null; + } + } + + public long getAmount() { + return this.amount; + } + + public TColumnIncrement setAmount(long amount) { + this.amount = amount; + setAmountIsSet(true); + return this; + } + + public void unsetAmount() { + __isset_bit_vector.clear(__AMOUNT_ISSET_ID); + } + + /** Returns true if field amount is set (has been assigned a value) and false otherwise */ + public boolean isSetAmount() { + return __isset_bit_vector.get(__AMOUNT_ISSET_ID); + } + + public void setAmountIsSet(boolean value) { + __isset_bit_vector.set(__AMOUNT_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case FAMILY: + if (value == null) { + unsetFamily(); + } else { + setFamily((ByteBuffer)value); + } + break; + + case QUALIFIER: + if (value == null) { + unsetQualifier(); + } else { + setQualifier((ByteBuffer)value); + } + break; + + case AMOUNT: + if (value == null) { + unsetAmount(); + } else { + setAmount((Long)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case FAMILY: + return getFamily(); + + case QUALIFIER: + return getQualifier(); + + case AMOUNT: + return Long.valueOf(getAmount()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case FAMILY: + return isSetFamily(); + case QUALIFIER: + return isSetQualifier(); + case AMOUNT: + return isSetAmount(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TColumnIncrement) + return this.equals((TColumnIncrement)that); + return false; + } + + public boolean equals(TColumnIncrement that) { + if (that == null) + return false; + + boolean this_present_family = true && this.isSetFamily(); + boolean that_present_family = true && that.isSetFamily(); + if (this_present_family || that_present_family) { + if (!(this_present_family && that_present_family)) + return false; + if (!this.family.equals(that.family)) + return false; + } + + boolean this_present_qualifier = true && this.isSetQualifier(); + boolean that_present_qualifier = true && that.isSetQualifier(); + if (this_present_qualifier || that_present_qualifier) { + if (!(this_present_qualifier && that_present_qualifier)) + return false; + if (!this.qualifier.equals(that.qualifier)) + return false; + } + + boolean this_present_amount = true && this.isSetAmount(); + boolean that_present_amount = true && that.isSetAmount(); + if (this_present_amount || that_present_amount) { + if (!(this_present_amount && that_present_amount)) + return false; + if (this.amount != that.amount) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TColumnIncrement other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TColumnIncrement typedOther = (TColumnIncrement)other; + + lastComparison = Boolean.valueOf(isSetFamily()).compareTo(typedOther.isSetFamily()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetFamily()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.family, typedOther.family); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetQualifier()).compareTo(typedOther.isSetQualifier()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetQualifier()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.qualifier, typedOther.qualifier); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAmount()).compareTo(typedOther.isSetAmount()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAmount()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.amount, typedOther.amount); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TColumnIncrement("); + boolean first = true; + + sb.append("family:"); + if (this.family == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.family, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("qualifier:"); + if (this.qualifier == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.qualifier, sb); + } + first = false; + if (isSetAmount()) { + if (!first) sb.append(", "); + sb.append("amount:"); + sb.append(this.amount); + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (family == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'family' was not present! Struct: " + toString()); + } + if (qualifier == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'qualifier' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TColumnIncrementStandardSchemeFactory implements SchemeFactory { + public TColumnIncrementStandardScheme getScheme() { + return new TColumnIncrementStandardScheme(); + } + } + + private static class TColumnIncrementStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TColumnIncrement struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // FAMILY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.family = iprot.readBinary(); + struct.setFamilyIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // QUALIFIER + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.qualifier = iprot.readBinary(); + struct.setQualifierIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // AMOUNT + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.amount = iprot.readI64(); + struct.setAmountIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TColumnIncrement struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.family != null) { + oprot.writeFieldBegin(FAMILY_FIELD_DESC); + oprot.writeBinary(struct.family); + oprot.writeFieldEnd(); + } + if (struct.qualifier != null) { + oprot.writeFieldBegin(QUALIFIER_FIELD_DESC); + oprot.writeBinary(struct.qualifier); + oprot.writeFieldEnd(); + } + if (struct.isSetAmount()) { + oprot.writeFieldBegin(AMOUNT_FIELD_DESC); + oprot.writeI64(struct.amount); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TColumnIncrementTupleSchemeFactory implements SchemeFactory { + public TColumnIncrementTupleScheme getScheme() { + return new TColumnIncrementTupleScheme(); + } + } + + private static class TColumnIncrementTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TColumnIncrement struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.family); + oprot.writeBinary(struct.qualifier); + BitSet optionals = new BitSet(); + if (struct.isSetAmount()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetAmount()) { + oprot.writeI64(struct.amount); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TColumnIncrement struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.family = iprot.readBinary(); + struct.setFamilyIsSet(true); + struct.qualifier = iprot.readBinary(); + struct.setQualifierIsSet(true); + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.amount = iprot.readI64(); + struct.setAmountIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnValue.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnValue.java new file mode 100644 index 0000000..39e5d2a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnValue.java @@ -0,0 +1,707 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Represents a single cell and its value. + */ +public class TColumnValue implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumnValue"); + + private static final org.apache.thrift.protocol.TField FAMILY_FIELD_DESC = new org.apache.thrift.protocol.TField("family", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField QUALIFIER_FIELD_DESC = new org.apache.thrift.protocol.TField("qualifier", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TColumnValueStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TColumnValueTupleSchemeFactory()); + } + + public ByteBuffer family; // required + public ByteBuffer qualifier; // required + public ByteBuffer value; // required + public long timestamp; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + FAMILY((short)1, "family"), + QUALIFIER((short)2, "qualifier"), + VALUE((short)3, "value"), + TIMESTAMP((short)4, "timestamp"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // FAMILY + return FAMILY; + case 2: // QUALIFIER + return QUALIFIER; + case 3: // VALUE + return VALUE; + case 4: // TIMESTAMP + return TIMESTAMP; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + private _Fields optionals[] = {_Fields.TIMESTAMP}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.FAMILY, new org.apache.thrift.meta_data.FieldMetaData("family", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.QUALIFIER, new org.apache.thrift.meta_data.FieldMetaData("qualifier", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TColumnValue.class, metaDataMap); + } + + public TColumnValue() { + } + + public TColumnValue( + ByteBuffer family, + ByteBuffer qualifier, + ByteBuffer value) + { + this(); + this.family = family; + this.qualifier = qualifier; + this.value = value; + } + + /** + * Performs a deep copy on other. + */ + public TColumnValue(TColumnValue other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetFamily()) { + this.family = org.apache.thrift.TBaseHelper.copyBinary(other.family); +; + } + if (other.isSetQualifier()) { + this.qualifier = org.apache.thrift.TBaseHelper.copyBinary(other.qualifier); +; + } + if (other.isSetValue()) { + this.value = org.apache.thrift.TBaseHelper.copyBinary(other.value); +; + } + this.timestamp = other.timestamp; + } + + public TColumnValue deepCopy() { + return new TColumnValue(this); + } + + @Override + public void clear() { + this.family = null; + this.qualifier = null; + this.value = null; + setTimestampIsSet(false); + this.timestamp = 0; + } + + public byte[] getFamily() { + setFamily(org.apache.thrift.TBaseHelper.rightSize(family)); + return family == null ? null : family.array(); + } + + public ByteBuffer bufferForFamily() { + return family; + } + + public TColumnValue setFamily(byte[] family) { + setFamily(family == null ? (ByteBuffer)null : ByteBuffer.wrap(family)); + return this; + } + + public TColumnValue setFamily(ByteBuffer family) { + this.family = family; + return this; + } + + public void unsetFamily() { + this.family = null; + } + + /** Returns true if field family is set (has been assigned a value) and false otherwise */ + public boolean isSetFamily() { + return this.family != null; + } + + public void setFamilyIsSet(boolean value) { + if (!value) { + this.family = null; + } + } + + public byte[] getQualifier() { + setQualifier(org.apache.thrift.TBaseHelper.rightSize(qualifier)); + return qualifier == null ? null : qualifier.array(); + } + + public ByteBuffer bufferForQualifier() { + return qualifier; + } + + public TColumnValue setQualifier(byte[] qualifier) { + setQualifier(qualifier == null ? (ByteBuffer)null : ByteBuffer.wrap(qualifier)); + return this; + } + + public TColumnValue setQualifier(ByteBuffer qualifier) { + this.qualifier = qualifier; + return this; + } + + public void unsetQualifier() { + this.qualifier = null; + } + + /** Returns true if field qualifier is set (has been assigned a value) and false otherwise */ + public boolean isSetQualifier() { + return this.qualifier != null; + } + + public void setQualifierIsSet(boolean value) { + if (!value) { + this.qualifier = null; + } + } + + public byte[] getValue() { + setValue(org.apache.thrift.TBaseHelper.rightSize(value)); + return value == null ? null : value.array(); + } + + public ByteBuffer bufferForValue() { + return value; + } + + public TColumnValue setValue(byte[] value) { + setValue(value == null ? (ByteBuffer)null : ByteBuffer.wrap(value)); + return this; + } + + public TColumnValue setValue(ByteBuffer value) { + this.value = value; + return this; + } + + public void unsetValue() { + this.value = null; + } + + /** Returns true if field value is set (has been assigned a value) and false otherwise */ + public boolean isSetValue() { + return this.value != null; + } + + public void setValueIsSet(boolean value) { + if (!value) { + this.value = null; + } + } + + public long getTimestamp() { + return this.timestamp; + } + + public TColumnValue setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case FAMILY: + if (value == null) { + unsetFamily(); + } else { + setFamily((ByteBuffer)value); + } + break; + + case QUALIFIER: + if (value == null) { + unsetQualifier(); + } else { + setQualifier((ByteBuffer)value); + } + break; + + case VALUE: + if (value == null) { + unsetValue(); + } else { + setValue((ByteBuffer)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case FAMILY: + return getFamily(); + + case QUALIFIER: + return getQualifier(); + + case VALUE: + return getValue(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case FAMILY: + return isSetFamily(); + case QUALIFIER: + return isSetQualifier(); + case VALUE: + return isSetValue(); + case TIMESTAMP: + return isSetTimestamp(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TColumnValue) + return this.equals((TColumnValue)that); + return false; + } + + public boolean equals(TColumnValue that) { + if (that == null) + return false; + + boolean this_present_family = true && this.isSetFamily(); + boolean that_present_family = true && that.isSetFamily(); + if (this_present_family || that_present_family) { + if (!(this_present_family && that_present_family)) + return false; + if (!this.family.equals(that.family)) + return false; + } + + boolean this_present_qualifier = true && this.isSetQualifier(); + boolean that_present_qualifier = true && that.isSetQualifier(); + if (this_present_qualifier || that_present_qualifier) { + if (!(this_present_qualifier && that_present_qualifier)) + return false; + if (!this.qualifier.equals(that.qualifier)) + return false; + } + + boolean this_present_value = true && this.isSetValue(); + boolean that_present_value = true && that.isSetValue(); + if (this_present_value || that_present_value) { + if (!(this_present_value && that_present_value)) + return false; + if (!this.value.equals(that.value)) + return false; + } + + boolean this_present_timestamp = true && this.isSetTimestamp(); + boolean that_present_timestamp = true && that.isSetTimestamp(); + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TColumnValue other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TColumnValue typedOther = (TColumnValue)other; + + lastComparison = Boolean.valueOf(isSetFamily()).compareTo(typedOther.isSetFamily()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetFamily()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.family, typedOther.family); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetQualifier()).compareTo(typedOther.isSetQualifier()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetQualifier()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.qualifier, typedOther.qualifier); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TColumnValue("); + boolean first = true; + + sb.append("family:"); + if (this.family == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.family, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("qualifier:"); + if (this.qualifier == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.qualifier, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("value:"); + if (this.value == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.value, sb); + } + first = false; + if (isSetTimestamp()) { + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (family == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'family' was not present! Struct: " + toString()); + } + if (qualifier == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'qualifier' was not present! Struct: " + toString()); + } + if (value == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'value' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TColumnValueStandardSchemeFactory implements SchemeFactory { + public TColumnValueStandardScheme getScheme() { + return new TColumnValueStandardScheme(); + } + } + + private static class TColumnValueStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TColumnValue struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // FAMILY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.family = iprot.readBinary(); + struct.setFamilyIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // QUALIFIER + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.qualifier = iprot.readBinary(); + struct.setQualifierIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.value = iprot.readBinary(); + struct.setValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TColumnValue struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.family != null) { + oprot.writeFieldBegin(FAMILY_FIELD_DESC); + oprot.writeBinary(struct.family); + oprot.writeFieldEnd(); + } + if (struct.qualifier != null) { + oprot.writeFieldBegin(QUALIFIER_FIELD_DESC); + oprot.writeBinary(struct.qualifier); + oprot.writeFieldEnd(); + } + if (struct.value != null) { + oprot.writeFieldBegin(VALUE_FIELD_DESC); + oprot.writeBinary(struct.value); + oprot.writeFieldEnd(); + } + if (struct.isSetTimestamp()) { + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TColumnValueTupleSchemeFactory implements SchemeFactory { + public TColumnValueTupleScheme getScheme() { + return new TColumnValueTupleScheme(); + } + } + + private static class TColumnValueTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TColumnValue struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.family); + oprot.writeBinary(struct.qualifier); + oprot.writeBinary(struct.value); + BitSet optionals = new BitSet(); + if (struct.isSetTimestamp()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TColumnValue struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.family = iprot.readBinary(); + struct.setFamilyIsSet(true); + struct.qualifier = iprot.readBinary(); + struct.setQualifierIsSet(true); + struct.value = iprot.readBinary(); + struct.setValueIsSet(true); + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java new file mode 100644 index 0000000..a732d7a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java @@ -0,0 +1,887 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Used to perform Delete operations on a single row. + * + * The scope can be further narrowed down by specifying a list of + * columns or column families as TColumns. + * + * Specifying only a family in a TColumn will delete the whole family. + * If a timestamp is specified all versions with a timestamp less than + * or equal to this will be deleted. If no timestamp is specified the + * current time will be used. + * + * Specifying a family and a column qualifier in a TColumn will delete only + * this qualifier. If a timestamp is specified only versions equal + * to this timestamp will be deleted. If no timestamp is specified the + * most recent version will be deleted. To delete all previous versions, + * specify the DELETE_COLUMNS TDeleteType. + * + * The top level timestamp is only used if a complete row should be deleted + * (i.e. no columns are passed) and if it is specified it works the same way + * as if you had added a TColumn for every column family and this timestamp + * (i.e. all versions older than or equal in all column families will be deleted) + * + */ +public class TDelete implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TDelete"); + + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)2); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField DELETE_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("deleteType", org.apache.thrift.protocol.TType.I32, (short)4); + private static final org.apache.thrift.protocol.TField WRITE_TO_WAL_FIELD_DESC = new org.apache.thrift.protocol.TField("writeToWal", org.apache.thrift.protocol.TType.BOOL, (short)5); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TDeleteStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TDeleteTupleSchemeFactory()); + } + + public ByteBuffer row; // required + public List columns; // optional + public long timestamp; // optional + /** + * + * @see TDeleteType + */ + public TDeleteType deleteType; // optional + public boolean writeToWal; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + ROW((short)1, "row"), + COLUMNS((short)2, "columns"), + TIMESTAMP((short)3, "timestamp"), + /** + * + * @see TDeleteType + */ + DELETE_TYPE((short)4, "deleteType"), + WRITE_TO_WAL((short)5, "writeToWal"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // ROW + return ROW; + case 2: // COLUMNS + return COLUMNS; + case 3: // TIMESTAMP + return TIMESTAMP; + case 4: // DELETE_TYPE + return DELETE_TYPE; + case 5: // WRITE_TO_WAL + return WRITE_TO_WAL; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private static final int __WRITETOWAL_ISSET_ID = 1; + private BitSet __isset_bit_vector = new BitSet(2); + private _Fields optionals[] = {_Fields.COLUMNS,_Fields.TIMESTAMP,_Fields.DELETE_TYPE,_Fields.WRITE_TO_WAL}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumn.class)))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.DELETE_TYPE, new org.apache.thrift.meta_data.FieldMetaData("deleteType", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TDeleteType.class))); + tmpMap.put(_Fields.WRITE_TO_WAL, new org.apache.thrift.meta_data.FieldMetaData("writeToWal", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TDelete.class, metaDataMap); + } + + public TDelete() { + this.deleteType = org.apache.hadoop.hbase.thrift2.generated.TDeleteType.DELETE_COLUMNS; + + this.writeToWal = true; + + } + + public TDelete( + ByteBuffer row) + { + this(); + this.row = row; + } + + /** + * Performs a deep copy on other. + */ + public TDelete(TDelete other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetRow()) { + this.row = org.apache.thrift.TBaseHelper.copyBinary(other.row); +; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (TColumn other_element : other.columns) { + __this__columns.add(new TColumn(other_element)); + } + this.columns = __this__columns; + } + this.timestamp = other.timestamp; + if (other.isSetDeleteType()) { + this.deleteType = other.deleteType; + } + this.writeToWal = other.writeToWal; + } + + public TDelete deepCopy() { + return new TDelete(this); + } + + @Override + public void clear() { + this.row = null; + this.columns = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.deleteType = org.apache.hadoop.hbase.thrift2.generated.TDeleteType.DELETE_COLUMNS; + + this.writeToWal = true; + + } + + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + public TDelete setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public TDelete setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(TColumn elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + public List getColumns() { + return this.columns; + } + + public TDelete setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public long getTimestamp() { + return this.timestamp; + } + + public TDelete setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + /** + * + * @see TDeleteType + */ + public TDeleteType getDeleteType() { + return this.deleteType; + } + + /** + * + * @see TDeleteType + */ + public TDelete setDeleteType(TDeleteType deleteType) { + this.deleteType = deleteType; + return this; + } + + public void unsetDeleteType() { + this.deleteType = null; + } + + /** Returns true if field deleteType is set (has been assigned a value) and false otherwise */ + public boolean isSetDeleteType() { + return this.deleteType != null; + } + + public void setDeleteTypeIsSet(boolean value) { + if (!value) { + this.deleteType = null; + } + } + + public boolean isWriteToWal() { + return this.writeToWal; + } + + public TDelete setWriteToWal(boolean writeToWal) { + this.writeToWal = writeToWal; + setWriteToWalIsSet(true); + return this; + } + + public void unsetWriteToWal() { + __isset_bit_vector.clear(__WRITETOWAL_ISSET_ID); + } + + /** Returns true if field writeToWal is set (has been assigned a value) and false otherwise */ + public boolean isSetWriteToWal() { + return __isset_bit_vector.get(__WRITETOWAL_ISSET_ID); + } + + public void setWriteToWalIsSet(boolean value) { + __isset_bit_vector.set(__WRITETOWAL_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case DELETE_TYPE: + if (value == null) { + unsetDeleteType(); + } else { + setDeleteType((TDeleteType)value); + } + break; + + case WRITE_TO_WAL: + if (value == null) { + unsetWriteToWal(); + } else { + setWriteToWal((Boolean)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case ROW: + return getRow(); + + case COLUMNS: + return getColumns(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case DELETE_TYPE: + return getDeleteType(); + + case WRITE_TO_WAL: + return Boolean.valueOf(isWriteToWal()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case ROW: + return isSetRow(); + case COLUMNS: + return isSetColumns(); + case TIMESTAMP: + return isSetTimestamp(); + case DELETE_TYPE: + return isSetDeleteType(); + case WRITE_TO_WAL: + return isSetWriteToWal(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TDelete) + return this.equals((TDelete)that); + return false; + } + + public boolean equals(TDelete that) { + if (that == null) + return false; + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_timestamp = true && this.isSetTimestamp(); + boolean that_present_timestamp = true && that.isSetTimestamp(); + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_deleteType = true && this.isSetDeleteType(); + boolean that_present_deleteType = true && that.isSetDeleteType(); + if (this_present_deleteType || that_present_deleteType) { + if (!(this_present_deleteType && that_present_deleteType)) + return false; + if (!this.deleteType.equals(that.deleteType)) + return false; + } + + boolean this_present_writeToWal = true && this.isSetWriteToWal(); + boolean that_present_writeToWal = true && that.isSetWriteToWal(); + if (this_present_writeToWal || that_present_writeToWal) { + if (!(this_present_writeToWal && that_present_writeToWal)) + return false; + if (this.writeToWal != that.writeToWal) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TDelete other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TDelete typedOther = (TDelete)other; + + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetDeleteType()).compareTo(typedOther.isSetDeleteType()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetDeleteType()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.deleteType, typedOther.deleteType); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetWriteToWal()).compareTo(typedOther.isSetWriteToWal()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetWriteToWal()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.writeToWal, typedOther.writeToWal); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TDelete("); + boolean first = true; + + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.row, sb); + } + first = false; + if (isSetColumns()) { + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + } + if (isSetTimestamp()) { + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + } + if (isSetDeleteType()) { + if (!first) sb.append(", "); + sb.append("deleteType:"); + if (this.deleteType == null) { + sb.append("null"); + } else { + sb.append(this.deleteType); + } + first = false; + } + if (isSetWriteToWal()) { + if (!first) sb.append(", "); + sb.append("writeToWal:"); + sb.append(this.writeToWal); + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (row == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'row' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TDeleteStandardSchemeFactory implements SchemeFactory { + public TDeleteStandardScheme getScheme() { + return new TDeleteStandardScheme(); + } + } + + private static class TDeleteStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TDelete struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list24 = iprot.readListBegin(); + struct.columns = new ArrayList(_list24.size); + for (int _i25 = 0; _i25 < _list24.size; ++_i25) + { + TColumn _elem26; // optional + _elem26 = new TColumn(); + _elem26.read(iprot); + struct.columns.add(_elem26); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // DELETE_TYPE + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.deleteType = TDeleteType.findByValue(iprot.readI32()); + struct.setDeleteTypeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // WRITE_TO_WAL + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.writeToWal = iprot.readBool(); + struct.setWriteToWalIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TDelete struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + if (struct.isSetColumns()) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size())); + for (TColumn _iter27 : struct.columns) + { + _iter27.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.isSetTimestamp()) { + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + } + if (struct.deleteType != null) { + if (struct.isSetDeleteType()) { + oprot.writeFieldBegin(DELETE_TYPE_FIELD_DESC); + oprot.writeI32(struct.deleteType.getValue()); + oprot.writeFieldEnd(); + } + } + if (struct.isSetWriteToWal()) { + oprot.writeFieldBegin(WRITE_TO_WAL_FIELD_DESC); + oprot.writeBool(struct.writeToWal); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TDeleteTupleSchemeFactory implements SchemeFactory { + public TDeleteTupleScheme getScheme() { + return new TDeleteTupleScheme(); + } + } + + private static class TDeleteTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TDelete struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.row); + BitSet optionals = new BitSet(); + if (struct.isSetColumns()) { + optionals.set(0); + } + if (struct.isSetTimestamp()) { + optionals.set(1); + } + if (struct.isSetDeleteType()) { + optionals.set(2); + } + if (struct.isSetWriteToWal()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (TColumn _iter28 : struct.columns) + { + _iter28.write(oprot); + } + } + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetDeleteType()) { + oprot.writeI32(struct.deleteType.getValue()); + } + if (struct.isSetWriteToWal()) { + oprot.writeBool(struct.writeToWal); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TDelete struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list29 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.columns = new ArrayList(_list29.size); + for (int _i30 = 0; _i30 < _list29.size; ++_i30) + { + TColumn _elem31; // optional + _elem31 = new TColumn(); + _elem31.read(iprot); + struct.columns.add(_elem31); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(1)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(2)) { + struct.deleteType = TDeleteType.findByValue(iprot.readI32()); + struct.setDeleteTypeIsSet(true); + } + if (incoming.get(3)) { + struct.writeToWal = iprot.readBool(); + struct.setWriteToWalIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDeleteType.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDeleteType.java new file mode 100644 index 0000000..e5d77a6 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDeleteType.java @@ -0,0 +1,50 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + + +import java.util.Map; +import java.util.HashMap; +import org.apache.thrift.TEnum; + +/** + * Specify type of delete: + * - DELETE_COLUMN means exactly one version will be removed, + * - DELETE_COLUMNS means previous versions will also be removed. + */ +public enum TDeleteType implements org.apache.thrift.TEnum { + DELETE_COLUMN(0), + DELETE_COLUMNS(1); + + private final int value; + + private TDeleteType(int value) { + this.value = value; + } + + /** + * Get the integer value of this enum value, as defined in the Thrift IDL. + */ + public int getValue() { + return value; + } + + /** + * Find a the enum type by its integer value, as defined in the Thrift IDL. + * @return null if the value is not found. + */ + public static TDeleteType findByValue(int value) { + switch (value) { + case 0: + return DELETE_COLUMN; + case 1: + return DELETE_COLUMNS; + default: + return null; + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java new file mode 100644 index 0000000..14e6d51 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java @@ -0,0 +1,860 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Used to perform Get operations on a single row. + * + * The scope can be further narrowed down by specifying a list of + * columns or column families. + * + * To get everything for a row, instantiate a Get object with just the row to get. + * To further define the scope of what to get you can add a timestamp or time range + * with an optional maximum number of versions to return. + * + * If you specify a time range and a timestamp the range is ignored. + * Timestamps on TColumns are ignored. + * + * TODO: Filter, Locks + */ +public class TGet implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGet"); + + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)2); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField TIME_RANGE_FIELD_DESC = new org.apache.thrift.protocol.TField("timeRange", org.apache.thrift.protocol.TType.STRUCT, (short)4); + private static final org.apache.thrift.protocol.TField MAX_VERSIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("maxVersions", org.apache.thrift.protocol.TType.I32, (short)5); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TGetStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TGetTupleSchemeFactory()); + } + + public ByteBuffer row; // required + public List columns; // optional + public long timestamp; // optional + public TTimeRange timeRange; // optional + public int maxVersions; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + ROW((short)1, "row"), + COLUMNS((short)2, "columns"), + TIMESTAMP((short)3, "timestamp"), + TIME_RANGE((short)4, "timeRange"), + MAX_VERSIONS((short)5, "maxVersions"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // ROW + return ROW; + case 2: // COLUMNS + return COLUMNS; + case 3: // TIMESTAMP + return TIMESTAMP; + case 4: // TIME_RANGE + return TIME_RANGE; + case 5: // MAX_VERSIONS + return MAX_VERSIONS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private static final int __MAXVERSIONS_ISSET_ID = 1; + private BitSet __isset_bit_vector = new BitSet(2); + private _Fields optionals[] = {_Fields.COLUMNS,_Fields.TIMESTAMP,_Fields.TIME_RANGE,_Fields.MAX_VERSIONS}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumn.class)))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.TIME_RANGE, new org.apache.thrift.meta_data.FieldMetaData("timeRange", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTimeRange.class))); + tmpMap.put(_Fields.MAX_VERSIONS, new org.apache.thrift.meta_data.FieldMetaData("maxVersions", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGet.class, metaDataMap); + } + + public TGet() { + } + + public TGet( + ByteBuffer row) + { + this(); + this.row = row; + } + + /** + * Performs a deep copy on other. + */ + public TGet(TGet other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetRow()) { + this.row = org.apache.thrift.TBaseHelper.copyBinary(other.row); +; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (TColumn other_element : other.columns) { + __this__columns.add(new TColumn(other_element)); + } + this.columns = __this__columns; + } + this.timestamp = other.timestamp; + if (other.isSetTimeRange()) { + this.timeRange = new TTimeRange(other.timeRange); + } + this.maxVersions = other.maxVersions; + } + + public TGet deepCopy() { + return new TGet(this); + } + + @Override + public void clear() { + this.row = null; + this.columns = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.timeRange = null; + setMaxVersionsIsSet(false); + this.maxVersions = 0; + } + + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + public TGet setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public TGet setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(TColumn elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + public List getColumns() { + return this.columns; + } + + public TGet setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public long getTimestamp() { + return this.timestamp; + } + + public TGet setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public TTimeRange getTimeRange() { + return this.timeRange; + } + + public TGet setTimeRange(TTimeRange timeRange) { + this.timeRange = timeRange; + return this; + } + + public void unsetTimeRange() { + this.timeRange = null; + } + + /** Returns true if field timeRange is set (has been assigned a value) and false otherwise */ + public boolean isSetTimeRange() { + return this.timeRange != null; + } + + public void setTimeRangeIsSet(boolean value) { + if (!value) { + this.timeRange = null; + } + } + + public int getMaxVersions() { + return this.maxVersions; + } + + public TGet setMaxVersions(int maxVersions) { + this.maxVersions = maxVersions; + setMaxVersionsIsSet(true); + return this; + } + + public void unsetMaxVersions() { + __isset_bit_vector.clear(__MAXVERSIONS_ISSET_ID); + } + + /** Returns true if field maxVersions is set (has been assigned a value) and false otherwise */ + public boolean isSetMaxVersions() { + return __isset_bit_vector.get(__MAXVERSIONS_ISSET_ID); + } + + public void setMaxVersionsIsSet(boolean value) { + __isset_bit_vector.set(__MAXVERSIONS_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case TIME_RANGE: + if (value == null) { + unsetTimeRange(); + } else { + setTimeRange((TTimeRange)value); + } + break; + + case MAX_VERSIONS: + if (value == null) { + unsetMaxVersions(); + } else { + setMaxVersions((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case ROW: + return getRow(); + + case COLUMNS: + return getColumns(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case TIME_RANGE: + return getTimeRange(); + + case MAX_VERSIONS: + return Integer.valueOf(getMaxVersions()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case ROW: + return isSetRow(); + case COLUMNS: + return isSetColumns(); + case TIMESTAMP: + return isSetTimestamp(); + case TIME_RANGE: + return isSetTimeRange(); + case MAX_VERSIONS: + return isSetMaxVersions(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TGet) + return this.equals((TGet)that); + return false; + } + + public boolean equals(TGet that) { + if (that == null) + return false; + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_timestamp = true && this.isSetTimestamp(); + boolean that_present_timestamp = true && that.isSetTimestamp(); + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_timeRange = true && this.isSetTimeRange(); + boolean that_present_timeRange = true && that.isSetTimeRange(); + if (this_present_timeRange || that_present_timeRange) { + if (!(this_present_timeRange && that_present_timeRange)) + return false; + if (!this.timeRange.equals(that.timeRange)) + return false; + } + + boolean this_present_maxVersions = true && this.isSetMaxVersions(); + boolean that_present_maxVersions = true && that.isSetMaxVersions(); + if (this_present_maxVersions || that_present_maxVersions) { + if (!(this_present_maxVersions && that_present_maxVersions)) + return false; + if (this.maxVersions != that.maxVersions) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TGet other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TGet typedOther = (TGet)other; + + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimeRange()).compareTo(typedOther.isSetTimeRange()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimeRange()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timeRange, typedOther.timeRange); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMaxVersions()).compareTo(typedOther.isSetMaxVersions()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMaxVersions()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.maxVersions, typedOther.maxVersions); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TGet("); + boolean first = true; + + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.row, sb); + } + first = false; + if (isSetColumns()) { + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + } + if (isSetTimestamp()) { + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + } + if (isSetTimeRange()) { + if (!first) sb.append(", "); + sb.append("timeRange:"); + if (this.timeRange == null) { + sb.append("null"); + } else { + sb.append(this.timeRange); + } + first = false; + } + if (isSetMaxVersions()) { + if (!first) sb.append(", "); + sb.append("maxVersions:"); + sb.append(this.maxVersions); + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (row == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'row' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TGetStandardSchemeFactory implements SchemeFactory { + public TGetStandardScheme getScheme() { + return new TGetStandardScheme(); + } + } + + private static class TGetStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TGet struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list8 = iprot.readListBegin(); + struct.columns = new ArrayList(_list8.size); + for (int _i9 = 0; _i9 < _list8.size; ++_i9) + { + TColumn _elem10; // optional + _elem10 = new TColumn(); + _elem10.read(iprot); + struct.columns.add(_elem10); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // TIME_RANGE + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.timeRange = new TTimeRange(); + struct.timeRange.read(iprot); + struct.setTimeRangeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // MAX_VERSIONS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.maxVersions = iprot.readI32(); + struct.setMaxVersionsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TGet struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + if (struct.isSetColumns()) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size())); + for (TColumn _iter11 : struct.columns) + { + _iter11.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.isSetTimestamp()) { + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + } + if (struct.timeRange != null) { + if (struct.isSetTimeRange()) { + oprot.writeFieldBegin(TIME_RANGE_FIELD_DESC); + struct.timeRange.write(oprot); + oprot.writeFieldEnd(); + } + } + if (struct.isSetMaxVersions()) { + oprot.writeFieldBegin(MAX_VERSIONS_FIELD_DESC); + oprot.writeI32(struct.maxVersions); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TGetTupleSchemeFactory implements SchemeFactory { + public TGetTupleScheme getScheme() { + return new TGetTupleScheme(); + } + } + + private static class TGetTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TGet struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.row); + BitSet optionals = new BitSet(); + if (struct.isSetColumns()) { + optionals.set(0); + } + if (struct.isSetTimestamp()) { + optionals.set(1); + } + if (struct.isSetTimeRange()) { + optionals.set(2); + } + if (struct.isSetMaxVersions()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (TColumn _iter12 : struct.columns) + { + _iter12.write(oprot); + } + } + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetTimeRange()) { + struct.timeRange.write(oprot); + } + if (struct.isSetMaxVersions()) { + oprot.writeI32(struct.maxVersions); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TGet struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list13 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.columns = new ArrayList(_list13.size); + for (int _i14 = 0; _i14 < _list13.size; ++_i14) + { + TColumn _elem15; // optional + _elem15 = new TColumn(); + _elem15.read(iprot); + struct.columns.add(_elem15); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(1)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(2)) { + struct.timeRange = new TTimeRange(); + struct.timeRange.read(iprot); + struct.setTimeRangeIsSet(true); + } + if (incoming.get(3)) { + struct.maxVersions = iprot.readI32(); + struct.setMaxVersionsIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java new file mode 100644 index 0000000..5628998 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java @@ -0,0 +1,14533 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class THBaseService { + + public interface Iface { + + /** + * Test for the existence of columns in the table, as specified in the TGet. + * + * @return true if the specified TGet matches one or more keys, false if not + * + * @param table the table to check on + * + * @param get the TGet to check for + */ + public boolean exists(ByteBuffer table, TGet get) throws TIOError, org.apache.thrift.TException; + + /** + * Method for getting data from a row. + * + * If the row cannot be found an empty Result is returned. + * This can be checked by the empty field of the TResult + * + * @return the result + * + * @param table the table to get from + * + * @param get the TGet to fetch + */ + public TResult get(ByteBuffer table, TGet get) throws TIOError, org.apache.thrift.TException; + + /** + * Method for getting multiple rows. + * + * If a row cannot be found there will be a null + * value in the result list for that TGet at the + * same position. + * + * So the Results are in the same order as the TGets. + * + * @param table the table to get from + * + * @param gets a list of TGets to fetch, the Result list + * will have the Results at corresponding positions + * or null if there was an error + */ + public List getMultiple(ByteBuffer table, List gets) throws TIOError, org.apache.thrift.TException; + + /** + * Commit a TPut to a table. + * + * @param table the table to put data in + * + * @param put the TPut to put + */ + public void put(ByteBuffer table, TPut put) throws TIOError, org.apache.thrift.TException; + + /** + * Atomically checks if a row/family/qualifier value matches the expected + * value. If it does, it adds the TPut. + * + * @return true if the new put was executed, false otherwise + * + * @param table to check in and put to + * + * @param row row to check + * + * @param family column family to check + * + * @param qualifier column qualifier to check + * + * @param value the expected value, if not provided the + * check is for the non-existence of the + * column in question + * + * @param put the TPut to put if the check succeeds + */ + public boolean checkAndPut(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TPut put) throws TIOError, org.apache.thrift.TException; + + /** + * Commit a List of Puts to the table. + * + * @param table the table to put data in + * + * @param puts a list of TPuts to commit + */ + public void putMultiple(ByteBuffer table, List puts) throws TIOError, org.apache.thrift.TException; + + /** + * Deletes as specified by the TDelete. + * + * Note: "delete" is a reserved keyword and cannot be used in Thrift + * thus the inconsistent naming scheme from the other functions. + * + * @param table the table to delete from + * + * @param deleteSingle the TDelete to delete + */ + public void deleteSingle(ByteBuffer table, TDelete deleteSingle) throws TIOError, org.apache.thrift.TException; + + /** + * Bulk commit a List of TDeletes to the table. + * + * This returns a list of TDeletes that were not + * executed. So if everything succeeds you'll + * receive an empty list. + * + * @param table the table to delete from + * + * @param deletes list of TDeletes to delete + */ + public List deleteMultiple(ByteBuffer table, List deletes) throws TIOError, org.apache.thrift.TException; + + /** + * Atomically checks if a row/family/qualifier value matches the expected + * value. If it does, it adds the delete. + * + * @return true if the new delete was executed, false otherwise + * + * @param table to check in and delete from + * + * @param row row to check + * + * @param family column family to check + * + * @param qualifier column qualifier to check + * + * @param value the expected value, if not provided the + * check is for the non-existence of the + * column in question + * + * @param deleteSingle the TDelete to execute if the check succeeds + */ + public boolean checkAndDelete(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TDelete deleteSingle) throws TIOError, org.apache.thrift.TException; + + public TResult increment(ByteBuffer table, TIncrement increment) throws TIOError, org.apache.thrift.TException; + + /** + * Get a Scanner for the provided TScan object. + * + * @return Scanner Id to be used with other scanner procedures + * + * @param table the table to get the Scanner for + * + * @param scan the scan object to get a Scanner for + */ + public int openScanner(ByteBuffer table, TScan scan) throws TIOError, org.apache.thrift.TException; + + /** + * Grabs multiple rows from a Scanner. + * + * @return Between zero and numRows TResults + * + * @param scannerId the Id of the Scanner to return rows from. This is an Id returned from the openScanner function. + * + * @param numRows number of rows to return + */ + public List getScannerRows(int scannerId, int numRows) throws TIOError, TIllegalArgument, org.apache.thrift.TException; + + /** + * Closes the scanner. Should be called if you need to close + * the Scanner before all results are read. + * + * Exhausted scanners are closed automatically. + * + * @param scannerId the Id of the Scanner to close * + */ + public void closeScanner(int scannerId) throws TIOError, TIllegalArgument, org.apache.thrift.TException; + + } + + public interface AsyncIface { + + public void exists(ByteBuffer table, TGet get, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void get(ByteBuffer table, TGet get, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getMultiple(ByteBuffer table, List gets, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void put(ByteBuffer table, TPut put, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void checkAndPut(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TPut put, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void putMultiple(ByteBuffer table, List puts, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void deleteSingle(ByteBuffer table, TDelete deleteSingle, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void deleteMultiple(ByteBuffer table, List deletes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void checkAndDelete(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TDelete deleteSingle, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void increment(ByteBuffer table, TIncrement increment, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void openScanner(ByteBuffer table, TScan scan, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void getScannerRows(int scannerId, int numRows, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void closeScanner(int scannerId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + } + + public static class Client extends org.apache.thrift.TServiceClient implements Iface { + public static class Factory implements org.apache.thrift.TServiceClientFactory { + public Factory() {} + public Client getClient(org.apache.thrift.protocol.TProtocol prot) { + return new Client(prot); + } + public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) { + return new Client(iprot, oprot); + } + } + + public Client(org.apache.thrift.protocol.TProtocol prot) + { + super(prot, prot); + } + + public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) { + super(iprot, oprot); + } + + public boolean exists(ByteBuffer table, TGet get) throws TIOError, org.apache.thrift.TException + { + send_exists(table, get); + return recv_exists(); + } + + public void send_exists(ByteBuffer table, TGet get) throws org.apache.thrift.TException + { + exists_args args = new exists_args(); + args.setTable(table); + args.setGet(get); + sendBase("exists", args); + } + + public boolean recv_exists() throws TIOError, org.apache.thrift.TException + { + exists_result result = new exists_result(); + receiveBase(result, "exists"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "exists failed: unknown result"); + } + + public TResult get(ByteBuffer table, TGet get) throws TIOError, org.apache.thrift.TException + { + send_get(table, get); + return recv_get(); + } + + public void send_get(ByteBuffer table, TGet get) throws org.apache.thrift.TException + { + get_args args = new get_args(); + args.setTable(table); + args.setGet(get); + sendBase("get", args); + } + + public TResult recv_get() throws TIOError, org.apache.thrift.TException + { + get_result result = new get_result(); + receiveBase(result, "get"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get failed: unknown result"); + } + + public List getMultiple(ByteBuffer table, List gets) throws TIOError, org.apache.thrift.TException + { + send_getMultiple(table, gets); + return recv_getMultiple(); + } + + public void send_getMultiple(ByteBuffer table, List gets) throws org.apache.thrift.TException + { + getMultiple_args args = new getMultiple_args(); + args.setTable(table); + args.setGets(gets); + sendBase("getMultiple", args); + } + + public List recv_getMultiple() throws TIOError, org.apache.thrift.TException + { + getMultiple_result result = new getMultiple_result(); + receiveBase(result, "getMultiple"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getMultiple failed: unknown result"); + } + + public void put(ByteBuffer table, TPut put) throws TIOError, org.apache.thrift.TException + { + send_put(table, put); + recv_put(); + } + + public void send_put(ByteBuffer table, TPut put) throws org.apache.thrift.TException + { + put_args args = new put_args(); + args.setTable(table); + args.setPut(put); + sendBase("put", args); + } + + public void recv_put() throws TIOError, org.apache.thrift.TException + { + put_result result = new put_result(); + receiveBase(result, "put"); + if (result.io != null) { + throw result.io; + } + return; + } + + public boolean checkAndPut(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TPut put) throws TIOError, org.apache.thrift.TException + { + send_checkAndPut(table, row, family, qualifier, value, put); + return recv_checkAndPut(); + } + + public void send_checkAndPut(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TPut put) throws org.apache.thrift.TException + { + checkAndPut_args args = new checkAndPut_args(); + args.setTable(table); + args.setRow(row); + args.setFamily(family); + args.setQualifier(qualifier); + args.setValue(value); + args.setPut(put); + sendBase("checkAndPut", args); + } + + public boolean recv_checkAndPut() throws TIOError, org.apache.thrift.TException + { + checkAndPut_result result = new checkAndPut_result(); + receiveBase(result, "checkAndPut"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "checkAndPut failed: unknown result"); + } + + public void putMultiple(ByteBuffer table, List puts) throws TIOError, org.apache.thrift.TException + { + send_putMultiple(table, puts); + recv_putMultiple(); + } + + public void send_putMultiple(ByteBuffer table, List puts) throws org.apache.thrift.TException + { + putMultiple_args args = new putMultiple_args(); + args.setTable(table); + args.setPuts(puts); + sendBase("putMultiple", args); + } + + public void recv_putMultiple() throws TIOError, org.apache.thrift.TException + { + putMultiple_result result = new putMultiple_result(); + receiveBase(result, "putMultiple"); + if (result.io != null) { + throw result.io; + } + return; + } + + public void deleteSingle(ByteBuffer table, TDelete deleteSingle) throws TIOError, org.apache.thrift.TException + { + send_deleteSingle(table, deleteSingle); + recv_deleteSingle(); + } + + public void send_deleteSingle(ByteBuffer table, TDelete deleteSingle) throws org.apache.thrift.TException + { + deleteSingle_args args = new deleteSingle_args(); + args.setTable(table); + args.setDeleteSingle(deleteSingle); + sendBase("deleteSingle", args); + } + + public void recv_deleteSingle() throws TIOError, org.apache.thrift.TException + { + deleteSingle_result result = new deleteSingle_result(); + receiveBase(result, "deleteSingle"); + if (result.io != null) { + throw result.io; + } + return; + } + + public List deleteMultiple(ByteBuffer table, List deletes) throws TIOError, org.apache.thrift.TException + { + send_deleteMultiple(table, deletes); + return recv_deleteMultiple(); + } + + public void send_deleteMultiple(ByteBuffer table, List deletes) throws org.apache.thrift.TException + { + deleteMultiple_args args = new deleteMultiple_args(); + args.setTable(table); + args.setDeletes(deletes); + sendBase("deleteMultiple", args); + } + + public List recv_deleteMultiple() throws TIOError, org.apache.thrift.TException + { + deleteMultiple_result result = new deleteMultiple_result(); + receiveBase(result, "deleteMultiple"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "deleteMultiple failed: unknown result"); + } + + public boolean checkAndDelete(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TDelete deleteSingle) throws TIOError, org.apache.thrift.TException + { + send_checkAndDelete(table, row, family, qualifier, value, deleteSingle); + return recv_checkAndDelete(); + } + + public void send_checkAndDelete(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TDelete deleteSingle) throws org.apache.thrift.TException + { + checkAndDelete_args args = new checkAndDelete_args(); + args.setTable(table); + args.setRow(row); + args.setFamily(family); + args.setQualifier(qualifier); + args.setValue(value); + args.setDeleteSingle(deleteSingle); + sendBase("checkAndDelete", args); + } + + public boolean recv_checkAndDelete() throws TIOError, org.apache.thrift.TException + { + checkAndDelete_result result = new checkAndDelete_result(); + receiveBase(result, "checkAndDelete"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "checkAndDelete failed: unknown result"); + } + + public TResult increment(ByteBuffer table, TIncrement increment) throws TIOError, org.apache.thrift.TException + { + send_increment(table, increment); + return recv_increment(); + } + + public void send_increment(ByteBuffer table, TIncrement increment) throws org.apache.thrift.TException + { + increment_args args = new increment_args(); + args.setTable(table); + args.setIncrement(increment); + sendBase("increment", args); + } + + public TResult recv_increment() throws TIOError, org.apache.thrift.TException + { + increment_result result = new increment_result(); + receiveBase(result, "increment"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "increment failed: unknown result"); + } + + public int openScanner(ByteBuffer table, TScan scan) throws TIOError, org.apache.thrift.TException + { + send_openScanner(table, scan); + return recv_openScanner(); + } + + public void send_openScanner(ByteBuffer table, TScan scan) throws org.apache.thrift.TException + { + openScanner_args args = new openScanner_args(); + args.setTable(table); + args.setScan(scan); + sendBase("openScanner", args); + } + + public int recv_openScanner() throws TIOError, org.apache.thrift.TException + { + openScanner_result result = new openScanner_result(); + receiveBase(result, "openScanner"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "openScanner failed: unknown result"); + } + + public List getScannerRows(int scannerId, int numRows) throws TIOError, TIllegalArgument, org.apache.thrift.TException + { + send_getScannerRows(scannerId, numRows); + return recv_getScannerRows(); + } + + public void send_getScannerRows(int scannerId, int numRows) throws org.apache.thrift.TException + { + getScannerRows_args args = new getScannerRows_args(); + args.setScannerId(scannerId); + args.setNumRows(numRows); + sendBase("getScannerRows", args); + } + + public List recv_getScannerRows() throws TIOError, TIllegalArgument, org.apache.thrift.TException + { + getScannerRows_result result = new getScannerRows_result(); + receiveBase(result, "getScannerRows"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.io != null) { + throw result.io; + } + if (result.ia != null) { + throw result.ia; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getScannerRows failed: unknown result"); + } + + public void closeScanner(int scannerId) throws TIOError, TIllegalArgument, org.apache.thrift.TException + { + send_closeScanner(scannerId); + recv_closeScanner(); + } + + public void send_closeScanner(int scannerId) throws org.apache.thrift.TException + { + closeScanner_args args = new closeScanner_args(); + args.setScannerId(scannerId); + sendBase("closeScanner", args); + } + + public void recv_closeScanner() throws TIOError, TIllegalArgument, org.apache.thrift.TException + { + closeScanner_result result = new closeScanner_result(); + receiveBase(result, "closeScanner"); + if (result.io != null) { + throw result.io; + } + if (result.ia != null) { + throw result.ia; + } + return; + } + + } + public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface { + public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { + private org.apache.thrift.async.TAsyncClientManager clientManager; + private org.apache.thrift.protocol.TProtocolFactory protocolFactory; + public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) { + this.clientManager = clientManager; + this.protocolFactory = protocolFactory; + } + public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) { + return new AsyncClient(protocolFactory, clientManager, transport); + } + } + + public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) { + super(protocolFactory, clientManager, transport); + } + + public void exists(ByteBuffer table, TGet get, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + exists_call method_call = new exists_call(table, get, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class exists_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer table; + private TGet get; + public exists_call(ByteBuffer table, TGet get, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.table = table; + this.get = get; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("exists", org.apache.thrift.protocol.TMessageType.CALL, 0)); + exists_args args = new exists_args(); + args.setTable(table); + args.setGet(get); + args.write(prot); + prot.writeMessageEnd(); + } + + public boolean getResult() throws TIOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_exists(); + } + } + + public void get(ByteBuffer table, TGet get, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + get_call method_call = new get_call(table, get, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class get_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer table; + private TGet get; + public get_call(ByteBuffer table, TGet get, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.table = table; + this.get = get; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get", org.apache.thrift.protocol.TMessageType.CALL, 0)); + get_args args = new get_args(); + args.setTable(table); + args.setGet(get); + args.write(prot); + prot.writeMessageEnd(); + } + + public TResult getResult() throws TIOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_get(); + } + } + + public void getMultiple(ByteBuffer table, List gets, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getMultiple_call method_call = new getMultiple_call(table, gets, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getMultiple_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer table; + private List gets; + public getMultiple_call(ByteBuffer table, List gets, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.table = table; + this.gets = gets; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getMultiple", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getMultiple_args args = new getMultiple_args(); + args.setTable(table); + args.setGets(gets); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws TIOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getMultiple(); + } + } + + public void put(ByteBuffer table, TPut put, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + put_call method_call = new put_call(table, put, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class put_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer table; + private TPut put; + public put_call(ByteBuffer table, TPut put, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.table = table; + this.put = put; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("put", org.apache.thrift.protocol.TMessageType.CALL, 0)); + put_args args = new put_args(); + args.setTable(table); + args.setPut(put); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws TIOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_put(); + } + } + + public void checkAndPut(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TPut put, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + checkAndPut_call method_call = new checkAndPut_call(table, row, family, qualifier, value, put, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class checkAndPut_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer table; + private ByteBuffer row; + private ByteBuffer family; + private ByteBuffer qualifier; + private ByteBuffer value; + private TPut put; + public checkAndPut_call(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TPut put, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.table = table; + this.row = row; + this.family = family; + this.qualifier = qualifier; + this.value = value; + this.put = put; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("checkAndPut", org.apache.thrift.protocol.TMessageType.CALL, 0)); + checkAndPut_args args = new checkAndPut_args(); + args.setTable(table); + args.setRow(row); + args.setFamily(family); + args.setQualifier(qualifier); + args.setValue(value); + args.setPut(put); + args.write(prot); + prot.writeMessageEnd(); + } + + public boolean getResult() throws TIOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_checkAndPut(); + } + } + + public void putMultiple(ByteBuffer table, List puts, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + putMultiple_call method_call = new putMultiple_call(table, puts, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class putMultiple_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer table; + private List puts; + public putMultiple_call(ByteBuffer table, List puts, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.table = table; + this.puts = puts; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("putMultiple", org.apache.thrift.protocol.TMessageType.CALL, 0)); + putMultiple_args args = new putMultiple_args(); + args.setTable(table); + args.setPuts(puts); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws TIOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_putMultiple(); + } + } + + public void deleteSingle(ByteBuffer table, TDelete deleteSingle, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + deleteSingle_call method_call = new deleteSingle_call(table, deleteSingle, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class deleteSingle_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer table; + private TDelete deleteSingle; + public deleteSingle_call(ByteBuffer table, TDelete deleteSingle, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.table = table; + this.deleteSingle = deleteSingle; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("deleteSingle", org.apache.thrift.protocol.TMessageType.CALL, 0)); + deleteSingle_args args = new deleteSingle_args(); + args.setTable(table); + args.setDeleteSingle(deleteSingle); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws TIOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_deleteSingle(); + } + } + + public void deleteMultiple(ByteBuffer table, List deletes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + deleteMultiple_call method_call = new deleteMultiple_call(table, deletes, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class deleteMultiple_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer table; + private List deletes; + public deleteMultiple_call(ByteBuffer table, List deletes, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.table = table; + this.deletes = deletes; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("deleteMultiple", org.apache.thrift.protocol.TMessageType.CALL, 0)); + deleteMultiple_args args = new deleteMultiple_args(); + args.setTable(table); + args.setDeletes(deletes); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws TIOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_deleteMultiple(); + } + } + + public void checkAndDelete(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TDelete deleteSingle, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + checkAndDelete_call method_call = new checkAndDelete_call(table, row, family, qualifier, value, deleteSingle, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class checkAndDelete_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer table; + private ByteBuffer row; + private ByteBuffer family; + private ByteBuffer qualifier; + private ByteBuffer value; + private TDelete deleteSingle; + public checkAndDelete_call(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, ByteBuffer value, TDelete deleteSingle, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.table = table; + this.row = row; + this.family = family; + this.qualifier = qualifier; + this.value = value; + this.deleteSingle = deleteSingle; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("checkAndDelete", org.apache.thrift.protocol.TMessageType.CALL, 0)); + checkAndDelete_args args = new checkAndDelete_args(); + args.setTable(table); + args.setRow(row); + args.setFamily(family); + args.setQualifier(qualifier); + args.setValue(value); + args.setDeleteSingle(deleteSingle); + args.write(prot); + prot.writeMessageEnd(); + } + + public boolean getResult() throws TIOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_checkAndDelete(); + } + } + + public void increment(ByteBuffer table, TIncrement increment, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + increment_call method_call = new increment_call(table, increment, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class increment_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer table; + private TIncrement increment; + public increment_call(ByteBuffer table, TIncrement increment, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.table = table; + this.increment = increment; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("increment", org.apache.thrift.protocol.TMessageType.CALL, 0)); + increment_args args = new increment_args(); + args.setTable(table); + args.setIncrement(increment); + args.write(prot); + prot.writeMessageEnd(); + } + + public TResult getResult() throws TIOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_increment(); + } + } + + public void openScanner(ByteBuffer table, TScan scan, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + openScanner_call method_call = new openScanner_call(table, scan, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class openScanner_call extends org.apache.thrift.async.TAsyncMethodCall { + private ByteBuffer table; + private TScan scan; + public openScanner_call(ByteBuffer table, TScan scan, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.table = table; + this.scan = scan; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("openScanner", org.apache.thrift.protocol.TMessageType.CALL, 0)); + openScanner_args args = new openScanner_args(); + args.setTable(table); + args.setScan(scan); + args.write(prot); + prot.writeMessageEnd(); + } + + public int getResult() throws TIOError, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_openScanner(); + } + } + + public void getScannerRows(int scannerId, int numRows, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getScannerRows_call method_call = new getScannerRows_call(scannerId, numRows, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getScannerRows_call extends org.apache.thrift.async.TAsyncMethodCall { + private int scannerId; + private int numRows; + public getScannerRows_call(int scannerId, int numRows, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.scannerId = scannerId; + this.numRows = numRows; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getScannerRows", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getScannerRows_args args = new getScannerRows_args(); + args.setScannerId(scannerId); + args.setNumRows(numRows); + args.write(prot); + prot.writeMessageEnd(); + } + + public List getResult() throws TIOError, TIllegalArgument, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getScannerRows(); + } + } + + public void closeScanner(int scannerId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + closeScanner_call method_call = new closeScanner_call(scannerId, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class closeScanner_call extends org.apache.thrift.async.TAsyncMethodCall { + private int scannerId; + public closeScanner_call(int scannerId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.scannerId = scannerId; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("closeScanner", org.apache.thrift.protocol.TMessageType.CALL, 0)); + closeScanner_args args = new closeScanner_args(); + args.setScannerId(scannerId); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws TIOError, TIllegalArgument, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_closeScanner(); + } + } + + } + + public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName()); + public Processor(I iface) { + super(iface, getProcessMap(new HashMap>())); + } + + protected Processor(I iface, Map> processMap) { + super(iface, getProcessMap(processMap)); + } + + private static Map> getProcessMap(Map> processMap) { + processMap.put("exists", new exists()); + processMap.put("get", new get()); + processMap.put("getMultiple", new getMultiple()); + processMap.put("put", new put()); + processMap.put("checkAndPut", new checkAndPut()); + processMap.put("putMultiple", new putMultiple()); + processMap.put("deleteSingle", new deleteSingle()); + processMap.put("deleteMultiple", new deleteMultiple()); + processMap.put("checkAndDelete", new checkAndDelete()); + processMap.put("increment", new increment()); + processMap.put("openScanner", new openScanner()); + processMap.put("getScannerRows", new getScannerRows()); + processMap.put("closeScanner", new closeScanner()); + return processMap; + } + + private static class exists extends org.apache.thrift.ProcessFunction { + public exists() { + super("exists"); + } + + protected exists_args getEmptyArgsInstance() { + return new exists_args(); + } + + protected exists_result getResult(I iface, exists_args args) throws org.apache.thrift.TException { + exists_result result = new exists_result(); + try { + result.success = iface.exists(args.table, args.get); + result.setSuccessIsSet(true); + } catch (TIOError io) { + result.io = io; + } + return result; + } + } + + private static class get extends org.apache.thrift.ProcessFunction { + public get() { + super("get"); + } + + protected get_args getEmptyArgsInstance() { + return new get_args(); + } + + protected get_result getResult(I iface, get_args args) throws org.apache.thrift.TException { + get_result result = new get_result(); + try { + result.success = iface.get(args.table, args.get); + } catch (TIOError io) { + result.io = io; + } + return result; + } + } + + private static class getMultiple extends org.apache.thrift.ProcessFunction { + public getMultiple() { + super("getMultiple"); + } + + protected getMultiple_args getEmptyArgsInstance() { + return new getMultiple_args(); + } + + protected getMultiple_result getResult(I iface, getMultiple_args args) throws org.apache.thrift.TException { + getMultiple_result result = new getMultiple_result(); + try { + result.success = iface.getMultiple(args.table, args.gets); + } catch (TIOError io) { + result.io = io; + } + return result; + } + } + + private static class put extends org.apache.thrift.ProcessFunction { + public put() { + super("put"); + } + + protected put_args getEmptyArgsInstance() { + return new put_args(); + } + + protected put_result getResult(I iface, put_args args) throws org.apache.thrift.TException { + put_result result = new put_result(); + try { + iface.put(args.table, args.put); + } catch (TIOError io) { + result.io = io; + } + return result; + } + } + + private static class checkAndPut extends org.apache.thrift.ProcessFunction { + public checkAndPut() { + super("checkAndPut"); + } + + protected checkAndPut_args getEmptyArgsInstance() { + return new checkAndPut_args(); + } + + protected checkAndPut_result getResult(I iface, checkAndPut_args args) throws org.apache.thrift.TException { + checkAndPut_result result = new checkAndPut_result(); + try { + result.success = iface.checkAndPut(args.table, args.row, args.family, args.qualifier, args.value, args.put); + result.setSuccessIsSet(true); + } catch (TIOError io) { + result.io = io; + } + return result; + } + } + + private static class putMultiple extends org.apache.thrift.ProcessFunction { + public putMultiple() { + super("putMultiple"); + } + + protected putMultiple_args getEmptyArgsInstance() { + return new putMultiple_args(); + } + + protected putMultiple_result getResult(I iface, putMultiple_args args) throws org.apache.thrift.TException { + putMultiple_result result = new putMultiple_result(); + try { + iface.putMultiple(args.table, args.puts); + } catch (TIOError io) { + result.io = io; + } + return result; + } + } + + private static class deleteSingle extends org.apache.thrift.ProcessFunction { + public deleteSingle() { + super("deleteSingle"); + } + + protected deleteSingle_args getEmptyArgsInstance() { + return new deleteSingle_args(); + } + + protected deleteSingle_result getResult(I iface, deleteSingle_args args) throws org.apache.thrift.TException { + deleteSingle_result result = new deleteSingle_result(); + try { + iface.deleteSingle(args.table, args.deleteSingle); + } catch (TIOError io) { + result.io = io; + } + return result; + } + } + + private static class deleteMultiple extends org.apache.thrift.ProcessFunction { + public deleteMultiple() { + super("deleteMultiple"); + } + + protected deleteMultiple_args getEmptyArgsInstance() { + return new deleteMultiple_args(); + } + + protected deleteMultiple_result getResult(I iface, deleteMultiple_args args) throws org.apache.thrift.TException { + deleteMultiple_result result = new deleteMultiple_result(); + try { + result.success = iface.deleteMultiple(args.table, args.deletes); + } catch (TIOError io) { + result.io = io; + } + return result; + } + } + + private static class checkAndDelete extends org.apache.thrift.ProcessFunction { + public checkAndDelete() { + super("checkAndDelete"); + } + + protected checkAndDelete_args getEmptyArgsInstance() { + return new checkAndDelete_args(); + } + + protected checkAndDelete_result getResult(I iface, checkAndDelete_args args) throws org.apache.thrift.TException { + checkAndDelete_result result = new checkAndDelete_result(); + try { + result.success = iface.checkAndDelete(args.table, args.row, args.family, args.qualifier, args.value, args.deleteSingle); + result.setSuccessIsSet(true); + } catch (TIOError io) { + result.io = io; + } + return result; + } + } + + private static class increment extends org.apache.thrift.ProcessFunction { + public increment() { + super("increment"); + } + + protected increment_args getEmptyArgsInstance() { + return new increment_args(); + } + + protected increment_result getResult(I iface, increment_args args) throws org.apache.thrift.TException { + increment_result result = new increment_result(); + try { + result.success = iface.increment(args.table, args.increment); + } catch (TIOError io) { + result.io = io; + } + return result; + } + } + + private static class openScanner extends org.apache.thrift.ProcessFunction { + public openScanner() { + super("openScanner"); + } + + protected openScanner_args getEmptyArgsInstance() { + return new openScanner_args(); + } + + protected openScanner_result getResult(I iface, openScanner_args args) throws org.apache.thrift.TException { + openScanner_result result = new openScanner_result(); + try { + result.success = iface.openScanner(args.table, args.scan); + result.setSuccessIsSet(true); + } catch (TIOError io) { + result.io = io; + } + return result; + } + } + + private static class getScannerRows extends org.apache.thrift.ProcessFunction { + public getScannerRows() { + super("getScannerRows"); + } + + protected getScannerRows_args getEmptyArgsInstance() { + return new getScannerRows_args(); + } + + protected getScannerRows_result getResult(I iface, getScannerRows_args args) throws org.apache.thrift.TException { + getScannerRows_result result = new getScannerRows_result(); + try { + result.success = iface.getScannerRows(args.scannerId, args.numRows); + } catch (TIOError io) { + result.io = io; + } catch (TIllegalArgument ia) { + result.ia = ia; + } + return result; + } + } + + private static class closeScanner extends org.apache.thrift.ProcessFunction { + public closeScanner() { + super("closeScanner"); + } + + protected closeScanner_args getEmptyArgsInstance() { + return new closeScanner_args(); + } + + protected closeScanner_result getResult(I iface, closeScanner_args args) throws org.apache.thrift.TException { + closeScanner_result result = new closeScanner_result(); + try { + iface.closeScanner(args.scannerId); + } catch (TIOError io) { + result.io = io; + } catch (TIllegalArgument ia) { + result.ia = ia; + } + return result; + } + } + + } + + public static class exists_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("exists_args"); + + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField GET_FIELD_DESC = new org.apache.thrift.protocol.TField("get", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new exists_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new exists_argsTupleSchemeFactory()); + } + + /** + * the table to check on + */ + public ByteBuffer table; // required + /** + * the TGet to check for + */ + public TGet get; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * the table to check on + */ + TABLE((short)1, "table"), + /** + * the TGet to check for + */ + GET((short)2, "get"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE + return TABLE; + case 2: // GET + return GET; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.GET, new org.apache.thrift.meta_data.FieldMetaData("get", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGet.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(exists_args.class, metaDataMap); + } + + public exists_args() { + } + + public exists_args( + ByteBuffer table, + TGet get) + { + this(); + this.table = table; + this.get = get; + } + + /** + * Performs a deep copy on other. + */ + public exists_args(exists_args other) { + if (other.isSetTable()) { + this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table); +; + } + if (other.isSetGet()) { + this.get = new TGet(other.get); + } + } + + public exists_args deepCopy() { + return new exists_args(this); + } + + @Override + public void clear() { + this.table = null; + this.get = null; + } + + /** + * the table to check on + */ + public byte[] getTable() { + setTable(org.apache.thrift.TBaseHelper.rightSize(table)); + return table == null ? null : table.array(); + } + + public ByteBuffer bufferForTable() { + return table; + } + + /** + * the table to check on + */ + public exists_args setTable(byte[] table) { + setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table)); + return this; + } + + public exists_args setTable(ByteBuffer table) { + this.table = table; + return this; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + /** + * the TGet to check for + */ + public TGet getGet() { + return this.get; + } + + /** + * the TGet to check for + */ + public exists_args setGet(TGet get) { + this.get = get; + return this; + } + + public void unsetGet() { + this.get = null; + } + + /** Returns true if field get is set (has been assigned a value) and false otherwise */ + public boolean isSetGet() { + return this.get != null; + } + + public void setGetIsSet(boolean value) { + if (!value) { + this.get = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((ByteBuffer)value); + } + break; + + case GET: + if (value == null) { + unsetGet(); + } else { + setGet((TGet)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE: + return getTable(); + + case GET: + return getGet(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE: + return isSetTable(); + case GET: + return isSetGet(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof exists_args) + return this.equals((exists_args)that); + return false; + } + + public boolean equals(exists_args that) { + if (that == null) + return false; + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_get = true && this.isSetGet(); + boolean that_present_get = true && that.isSetGet(); + if (this_present_get || that_present_get) { + if (!(this_present_get && that_present_get)) + return false; + if (!this.get.equals(that.get)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(exists_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + exists_args typedOther = (exists_args)other; + + lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetGet()).compareTo(typedOther.isSetGet()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetGet()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.get, typedOther.get); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("exists_args("); + boolean first = true; + + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.table, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("get:"); + if (this.get == null) { + sb.append("null"); + } else { + sb.append(this.get); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (table == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString()); + } + if (get == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'get' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class exists_argsStandardSchemeFactory implements SchemeFactory { + public exists_argsStandardScheme getScheme() { + return new exists_argsStandardScheme(); + } + } + + private static class exists_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, exists_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // GET + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.get = new TGet(); + struct.get.read(iprot); + struct.setGetIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, exists_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.table != null) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeBinary(struct.table); + oprot.writeFieldEnd(); + } + if (struct.get != null) { + oprot.writeFieldBegin(GET_FIELD_DESC); + struct.get.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class exists_argsTupleSchemeFactory implements SchemeFactory { + public exists_argsTupleScheme getScheme() { + return new exists_argsTupleScheme(); + } + } + + private static class exists_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, exists_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.table); + struct.get.write(oprot); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, exists_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + struct.get = new TGet(); + struct.get.read(iprot); + struct.setGetIsSet(true); + } + } + + } + + public static class exists_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("exists_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.BOOL, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new exists_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new exists_resultTupleSchemeFactory()); + } + + public boolean success; // required + public TIOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(exists_result.class, metaDataMap); + } + + public exists_result() { + } + + public exists_result( + boolean success, + TIOError io) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public exists_result(exists_result other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.success = other.success; + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + } + + public exists_result deepCopy() { + return new exists_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = false; + this.io = null; + } + + public boolean isSuccess() { + return this.success; + } + + public exists_result setSuccess(boolean success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bit_vector.clear(__SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return __isset_bit_vector.get(__SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bit_vector.set(__SUCCESS_ISSET_ID, value); + } + + public TIOError getIo() { + return this.io; + } + + public exists_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Boolean)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Boolean.valueOf(isSuccess()); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof exists_result) + return this.equals((exists_result)that); + return false; + } + + public boolean equals(exists_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(exists_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + exists_result typedOther = (exists_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("exists_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class exists_resultStandardSchemeFactory implements SchemeFactory { + public exists_resultStandardScheme getScheme() { + return new exists_resultStandardScheme(); + } + } + + private static class exists_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, exists_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, exists_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeBool(struct.success); + oprot.writeFieldEnd(); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class exists_resultTupleSchemeFactory implements SchemeFactory { + public exists_resultTupleScheme getScheme() { + return new exists_resultTupleScheme(); + } + } + + private static class exists_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, exists_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeBool(struct.success); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, exists_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class get_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_args"); + + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField GET_FIELD_DESC = new org.apache.thrift.protocol.TField("get", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new get_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new get_argsTupleSchemeFactory()); + } + + /** + * the table to get from + */ + public ByteBuffer table; // required + /** + * the TGet to fetch + */ + public TGet get; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * the table to get from + */ + TABLE((short)1, "table"), + /** + * the TGet to fetch + */ + GET((short)2, "get"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE + return TABLE; + case 2: // GET + return GET; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.GET, new org.apache.thrift.meta_data.FieldMetaData("get", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGet.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_args.class, metaDataMap); + } + + public get_args() { + } + + public get_args( + ByteBuffer table, + TGet get) + { + this(); + this.table = table; + this.get = get; + } + + /** + * Performs a deep copy on other. + */ + public get_args(get_args other) { + if (other.isSetTable()) { + this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table); +; + } + if (other.isSetGet()) { + this.get = new TGet(other.get); + } + } + + public get_args deepCopy() { + return new get_args(this); + } + + @Override + public void clear() { + this.table = null; + this.get = null; + } + + /** + * the table to get from + */ + public byte[] getTable() { + setTable(org.apache.thrift.TBaseHelper.rightSize(table)); + return table == null ? null : table.array(); + } + + public ByteBuffer bufferForTable() { + return table; + } + + /** + * the table to get from + */ + public get_args setTable(byte[] table) { + setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table)); + return this; + } + + public get_args setTable(ByteBuffer table) { + this.table = table; + return this; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + /** + * the TGet to fetch + */ + public TGet getGet() { + return this.get; + } + + /** + * the TGet to fetch + */ + public get_args setGet(TGet get) { + this.get = get; + return this; + } + + public void unsetGet() { + this.get = null; + } + + /** Returns true if field get is set (has been assigned a value) and false otherwise */ + public boolean isSetGet() { + return this.get != null; + } + + public void setGetIsSet(boolean value) { + if (!value) { + this.get = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((ByteBuffer)value); + } + break; + + case GET: + if (value == null) { + unsetGet(); + } else { + setGet((TGet)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE: + return getTable(); + + case GET: + return getGet(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE: + return isSetTable(); + case GET: + return isSetGet(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof get_args) + return this.equals((get_args)that); + return false; + } + + public boolean equals(get_args that) { + if (that == null) + return false; + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_get = true && this.isSetGet(); + boolean that_present_get = true && that.isSetGet(); + if (this_present_get || that_present_get) { + if (!(this_present_get && that_present_get)) + return false; + if (!this.get.equals(that.get)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(get_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + get_args typedOther = (get_args)other; + + lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetGet()).compareTo(typedOther.isSetGet()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetGet()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.get, typedOther.get); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("get_args("); + boolean first = true; + + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.table, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("get:"); + if (this.get == null) { + sb.append("null"); + } else { + sb.append(this.get); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (table == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString()); + } + if (get == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'get' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class get_argsStandardSchemeFactory implements SchemeFactory { + public get_argsStandardScheme getScheme() { + return new get_argsStandardScheme(); + } + } + + private static class get_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, get_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // GET + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.get = new TGet(); + struct.get.read(iprot); + struct.setGetIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, get_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.table != null) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeBinary(struct.table); + oprot.writeFieldEnd(); + } + if (struct.get != null) { + oprot.writeFieldBegin(GET_FIELD_DESC); + struct.get.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class get_argsTupleSchemeFactory implements SchemeFactory { + public get_argsTupleScheme getScheme() { + return new get_argsTupleScheme(); + } + } + + private static class get_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, get_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.table); + struct.get.write(oprot); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, get_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + struct.get = new TGet(); + struct.get.read(iprot); + struct.setGetIsSet(true); + } + } + + } + + public static class get_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new get_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new get_resultTupleSchemeFactory()); + } + + public TResult success; // required + public TIOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TResult.class))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_result.class, metaDataMap); + } + + public get_result() { + } + + public get_result( + TResult success, + TIOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public get_result(get_result other) { + if (other.isSetSuccess()) { + this.success = new TResult(other.success); + } + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + } + + public get_result deepCopy() { + return new get_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public TResult getSuccess() { + return this.success; + } + + public get_result setSuccess(TResult success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public TIOError getIo() { + return this.io; + } + + public get_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((TResult)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof get_result) + return this.equals((get_result)that); + return false; + } + + public boolean equals(get_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(get_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + get_result typedOther = (get_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("get_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class get_resultStandardSchemeFactory implements SchemeFactory { + public get_resultStandardScheme getScheme() { + return new get_resultStandardScheme(); + } + } + + private static class get_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, get_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.success = new TResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, get_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + struct.success.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class get_resultTupleSchemeFactory implements SchemeFactory { + public get_resultTupleScheme getScheme() { + return new get_resultTupleScheme(); + } + } + + private static class get_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, get_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + struct.success.write(oprot); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, get_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = new TResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getMultiple_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getMultiple_args"); + + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField GETS_FIELD_DESC = new org.apache.thrift.protocol.TField("gets", org.apache.thrift.protocol.TType.LIST, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getMultiple_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getMultiple_argsTupleSchemeFactory()); + } + + /** + * the table to get from + */ + public ByteBuffer table; // required + /** + * a list of TGets to fetch, the Result list + * will have the Results at corresponding positions + * or null if there was an error + */ + public List gets; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * the table to get from + */ + TABLE((short)1, "table"), + /** + * a list of TGets to fetch, the Result list + * will have the Results at corresponding positions + * or null if there was an error + */ + GETS((short)2, "gets"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE + return TABLE; + case 2: // GETS + return GETS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.GETS, new org.apache.thrift.meta_data.FieldMetaData("gets", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TGet.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getMultiple_args.class, metaDataMap); + } + + public getMultiple_args() { + } + + public getMultiple_args( + ByteBuffer table, + List gets) + { + this(); + this.table = table; + this.gets = gets; + } + + /** + * Performs a deep copy on other. + */ + public getMultiple_args(getMultiple_args other) { + if (other.isSetTable()) { + this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table); +; + } + if (other.isSetGets()) { + List __this__gets = new ArrayList(); + for (TGet other_element : other.gets) { + __this__gets.add(new TGet(other_element)); + } + this.gets = __this__gets; + } + } + + public getMultiple_args deepCopy() { + return new getMultiple_args(this); + } + + @Override + public void clear() { + this.table = null; + this.gets = null; + } + + /** + * the table to get from + */ + public byte[] getTable() { + setTable(org.apache.thrift.TBaseHelper.rightSize(table)); + return table == null ? null : table.array(); + } + + public ByteBuffer bufferForTable() { + return table; + } + + /** + * the table to get from + */ + public getMultiple_args setTable(byte[] table) { + setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table)); + return this; + } + + public getMultiple_args setTable(ByteBuffer table) { + this.table = table; + return this; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + public int getGetsSize() { + return (this.gets == null) ? 0 : this.gets.size(); + } + + public java.util.Iterator getGetsIterator() { + return (this.gets == null) ? null : this.gets.iterator(); + } + + public void addToGets(TGet elem) { + if (this.gets == null) { + this.gets = new ArrayList(); + } + this.gets.add(elem); + } + + /** + * a list of TGets to fetch, the Result list + * will have the Results at corresponding positions + * or null if there was an error + */ + public List getGets() { + return this.gets; + } + + /** + * a list of TGets to fetch, the Result list + * will have the Results at corresponding positions + * or null if there was an error + */ + public getMultiple_args setGets(List gets) { + this.gets = gets; + return this; + } + + public void unsetGets() { + this.gets = null; + } + + /** Returns true if field gets is set (has been assigned a value) and false otherwise */ + public boolean isSetGets() { + return this.gets != null; + } + + public void setGetsIsSet(boolean value) { + if (!value) { + this.gets = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((ByteBuffer)value); + } + break; + + case GETS: + if (value == null) { + unsetGets(); + } else { + setGets((List)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE: + return getTable(); + + case GETS: + return getGets(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE: + return isSetTable(); + case GETS: + return isSetGets(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getMultiple_args) + return this.equals((getMultiple_args)that); + return false; + } + + public boolean equals(getMultiple_args that) { + if (that == null) + return false; + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_gets = true && this.isSetGets(); + boolean that_present_gets = true && that.isSetGets(); + if (this_present_gets || that_present_gets) { + if (!(this_present_gets && that_present_gets)) + return false; + if (!this.gets.equals(that.gets)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getMultiple_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getMultiple_args typedOther = (getMultiple_args)other; + + lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetGets()).compareTo(typedOther.isSetGets()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetGets()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.gets, typedOther.gets); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getMultiple_args("); + boolean first = true; + + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.table, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("gets:"); + if (this.gets == null) { + sb.append("null"); + } else { + sb.append(this.gets); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (table == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString()); + } + if (gets == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'gets' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getMultiple_argsStandardSchemeFactory implements SchemeFactory { + public getMultiple_argsStandardScheme getScheme() { + return new getMultiple_argsStandardScheme(); + } + } + + private static class getMultiple_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getMultiple_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // GETS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list48 = iprot.readListBegin(); + struct.gets = new ArrayList(_list48.size); + for (int _i49 = 0; _i49 < _list48.size; ++_i49) + { + TGet _elem50; // optional + _elem50 = new TGet(); + _elem50.read(iprot); + struct.gets.add(_elem50); + } + iprot.readListEnd(); + } + struct.setGetsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getMultiple_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.table != null) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeBinary(struct.table); + oprot.writeFieldEnd(); + } + if (struct.gets != null) { + oprot.writeFieldBegin(GETS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.gets.size())); + for (TGet _iter51 : struct.gets) + { + _iter51.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getMultiple_argsTupleSchemeFactory implements SchemeFactory { + public getMultiple_argsTupleScheme getScheme() { + return new getMultiple_argsTupleScheme(); + } + } + + private static class getMultiple_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getMultiple_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.table); + { + oprot.writeI32(struct.gets.size()); + for (TGet _iter52 : struct.gets) + { + _iter52.write(oprot); + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getMultiple_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + { + org.apache.thrift.protocol.TList _list53 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.gets = new ArrayList(_list53.size); + for (int _i54 = 0; _i54 < _list53.size; ++_i54) + { + TGet _elem55; // optional + _elem55 = new TGet(); + _elem55.read(iprot); + struct.gets.add(_elem55); + } + } + struct.setGetsIsSet(true); + } + } + + } + + public static class getMultiple_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getMultiple_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getMultiple_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getMultiple_resultTupleSchemeFactory()); + } + + public List success; // required + public TIOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TResult.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getMultiple_result.class, metaDataMap); + } + + public getMultiple_result() { + } + + public getMultiple_result( + List success, + TIOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public getMultiple_result(getMultiple_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TResult other_element : other.success) { + __this__success.add(new TResult(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + } + + public getMultiple_result deepCopy() { + return new getMultiple_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TResult elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getMultiple_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public TIOError getIo() { + return this.io; + } + + public getMultiple_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getMultiple_result) + return this.equals((getMultiple_result)that); + return false; + } + + public boolean equals(getMultiple_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getMultiple_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getMultiple_result typedOther = (getMultiple_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getMultiple_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getMultiple_resultStandardSchemeFactory implements SchemeFactory { + public getMultiple_resultStandardScheme getScheme() { + return new getMultiple_resultStandardScheme(); + } + } + + private static class getMultiple_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getMultiple_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list56 = iprot.readListBegin(); + struct.success = new ArrayList(_list56.size); + for (int _i57 = 0; _i57 < _list56.size; ++_i57) + { + TResult _elem58; // optional + _elem58 = new TResult(); + _elem58.read(iprot); + struct.success.add(_elem58); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getMultiple_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TResult _iter59 : struct.success) + { + _iter59.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getMultiple_resultTupleSchemeFactory implements SchemeFactory { + public getMultiple_resultTupleScheme getScheme() { + return new getMultiple_resultTupleScheme(); + } + } + + private static class getMultiple_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getMultiple_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TResult _iter60 : struct.success) + { + _iter60.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getMultiple_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list61 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list61.size); + for (int _i62 = 0; _i62 < _list61.size; ++_i62) + { + TResult _elem63; // optional + _elem63 = new TResult(); + _elem63.read(iprot); + struct.success.add(_elem63); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class put_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("put_args"); + + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField PUT_FIELD_DESC = new org.apache.thrift.protocol.TField("put", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new put_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new put_argsTupleSchemeFactory()); + } + + /** + * the table to put data in + */ + public ByteBuffer table; // required + /** + * the TPut to put + */ + public TPut put; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * the table to put data in + */ + TABLE((short)1, "table"), + /** + * the TPut to put + */ + PUT((short)2, "put"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE + return TABLE; + case 2: // PUT + return PUT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.PUT, new org.apache.thrift.meta_data.FieldMetaData("put", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TPut.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(put_args.class, metaDataMap); + } + + public put_args() { + } + + public put_args( + ByteBuffer table, + TPut put) + { + this(); + this.table = table; + this.put = put; + } + + /** + * Performs a deep copy on other. + */ + public put_args(put_args other) { + if (other.isSetTable()) { + this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table); +; + } + if (other.isSetPut()) { + this.put = new TPut(other.put); + } + } + + public put_args deepCopy() { + return new put_args(this); + } + + @Override + public void clear() { + this.table = null; + this.put = null; + } + + /** + * the table to put data in + */ + public byte[] getTable() { + setTable(org.apache.thrift.TBaseHelper.rightSize(table)); + return table == null ? null : table.array(); + } + + public ByteBuffer bufferForTable() { + return table; + } + + /** + * the table to put data in + */ + public put_args setTable(byte[] table) { + setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table)); + return this; + } + + public put_args setTable(ByteBuffer table) { + this.table = table; + return this; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + /** + * the TPut to put + */ + public TPut getPut() { + return this.put; + } + + /** + * the TPut to put + */ + public put_args setPut(TPut put) { + this.put = put; + return this; + } + + public void unsetPut() { + this.put = null; + } + + /** Returns true if field put is set (has been assigned a value) and false otherwise */ + public boolean isSetPut() { + return this.put != null; + } + + public void setPutIsSet(boolean value) { + if (!value) { + this.put = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((ByteBuffer)value); + } + break; + + case PUT: + if (value == null) { + unsetPut(); + } else { + setPut((TPut)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE: + return getTable(); + + case PUT: + return getPut(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE: + return isSetTable(); + case PUT: + return isSetPut(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof put_args) + return this.equals((put_args)that); + return false; + } + + public boolean equals(put_args that) { + if (that == null) + return false; + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_put = true && this.isSetPut(); + boolean that_present_put = true && that.isSetPut(); + if (this_present_put || that_present_put) { + if (!(this_present_put && that_present_put)) + return false; + if (!this.put.equals(that.put)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(put_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + put_args typedOther = (put_args)other; + + lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetPut()).compareTo(typedOther.isSetPut()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetPut()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.put, typedOther.put); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("put_args("); + boolean first = true; + + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.table, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("put:"); + if (this.put == null) { + sb.append("null"); + } else { + sb.append(this.put); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (table == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString()); + } + if (put == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'put' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class put_argsStandardSchemeFactory implements SchemeFactory { + public put_argsStandardScheme getScheme() { + return new put_argsStandardScheme(); + } + } + + private static class put_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, put_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // PUT + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.put = new TPut(); + struct.put.read(iprot); + struct.setPutIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, put_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.table != null) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeBinary(struct.table); + oprot.writeFieldEnd(); + } + if (struct.put != null) { + oprot.writeFieldBegin(PUT_FIELD_DESC); + struct.put.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class put_argsTupleSchemeFactory implements SchemeFactory { + public put_argsTupleScheme getScheme() { + return new put_argsTupleScheme(); + } + } + + private static class put_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, put_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.table); + struct.put.write(oprot); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, put_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + struct.put = new TPut(); + struct.put.read(iprot); + struct.setPutIsSet(true); + } + } + + } + + public static class put_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("put_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new put_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new put_resultTupleSchemeFactory()); + } + + public TIOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(put_result.class, metaDataMap); + } + + public put_result() { + } + + public put_result( + TIOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public put_result(put_result other) { + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + } + + public put_result deepCopy() { + return new put_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public TIOError getIo() { + return this.io; + } + + public put_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof put_result) + return this.equals((put_result)that); + return false; + } + + public boolean equals(put_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(put_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + put_result typedOther = (put_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("put_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class put_resultStandardSchemeFactory implements SchemeFactory { + public put_resultStandardScheme getScheme() { + return new put_resultStandardScheme(); + } + } + + private static class put_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, put_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, put_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class put_resultTupleSchemeFactory implements SchemeFactory { + public put_resultTupleScheme getScheme() { + return new put_resultTupleScheme(); + } + } + + private static class put_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, put_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, put_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class checkAndPut_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("checkAndPut_args"); + + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField FAMILY_FIELD_DESC = new org.apache.thrift.protocol.TField("family", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField QUALIFIER_FIELD_DESC = new org.apache.thrift.protocol.TField("qualifier", org.apache.thrift.protocol.TType.STRING, (short)4); + private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)5); + private static final org.apache.thrift.protocol.TField PUT_FIELD_DESC = new org.apache.thrift.protocol.TField("put", org.apache.thrift.protocol.TType.STRUCT, (short)6); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new checkAndPut_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new checkAndPut_argsTupleSchemeFactory()); + } + + /** + * to check in and put to + */ + public ByteBuffer table; // required + /** + * row to check + */ + public ByteBuffer row; // required + /** + * column family to check + */ + public ByteBuffer family; // required + /** + * column qualifier to check + */ + public ByteBuffer qualifier; // required + /** + * the expected value, if not provided the + * check is for the non-existence of the + * column in question + */ + public ByteBuffer value; // required + /** + * the TPut to put if the check succeeds + */ + public TPut put; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * to check in and put to + */ + TABLE((short)1, "table"), + /** + * row to check + */ + ROW((short)2, "row"), + /** + * column family to check + */ + FAMILY((short)3, "family"), + /** + * column qualifier to check + */ + QUALIFIER((short)4, "qualifier"), + /** + * the expected value, if not provided the + * check is for the non-existence of the + * column in question + */ + VALUE((short)5, "value"), + /** + * the TPut to put if the check succeeds + */ + PUT((short)6, "put"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE + return TABLE; + case 2: // ROW + return ROW; + case 3: // FAMILY + return FAMILY; + case 4: // QUALIFIER + return QUALIFIER; + case 5: // VALUE + return VALUE; + case 6: // PUT + return PUT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.FAMILY, new org.apache.thrift.meta_data.FieldMetaData("family", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.QUALIFIER, new org.apache.thrift.meta_data.FieldMetaData("qualifier", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.PUT, new org.apache.thrift.meta_data.FieldMetaData("put", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TPut.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(checkAndPut_args.class, metaDataMap); + } + + public checkAndPut_args() { + } + + public checkAndPut_args( + ByteBuffer table, + ByteBuffer row, + ByteBuffer family, + ByteBuffer qualifier, + ByteBuffer value, + TPut put) + { + this(); + this.table = table; + this.row = row; + this.family = family; + this.qualifier = qualifier; + this.value = value; + this.put = put; + } + + /** + * Performs a deep copy on other. + */ + public checkAndPut_args(checkAndPut_args other) { + if (other.isSetTable()) { + this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table); +; + } + if (other.isSetRow()) { + this.row = org.apache.thrift.TBaseHelper.copyBinary(other.row); +; + } + if (other.isSetFamily()) { + this.family = org.apache.thrift.TBaseHelper.copyBinary(other.family); +; + } + if (other.isSetQualifier()) { + this.qualifier = org.apache.thrift.TBaseHelper.copyBinary(other.qualifier); +; + } + if (other.isSetValue()) { + this.value = org.apache.thrift.TBaseHelper.copyBinary(other.value); +; + } + if (other.isSetPut()) { + this.put = new TPut(other.put); + } + } + + public checkAndPut_args deepCopy() { + return new checkAndPut_args(this); + } + + @Override + public void clear() { + this.table = null; + this.row = null; + this.family = null; + this.qualifier = null; + this.value = null; + this.put = null; + } + + /** + * to check in and put to + */ + public byte[] getTable() { + setTable(org.apache.thrift.TBaseHelper.rightSize(table)); + return table == null ? null : table.array(); + } + + public ByteBuffer bufferForTable() { + return table; + } + + /** + * to check in and put to + */ + public checkAndPut_args setTable(byte[] table) { + setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table)); + return this; + } + + public checkAndPut_args setTable(ByteBuffer table) { + this.table = table; + return this; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + /** + * row to check + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row to check + */ + public checkAndPut_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public checkAndPut_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + /** + * column family to check + */ + public byte[] getFamily() { + setFamily(org.apache.thrift.TBaseHelper.rightSize(family)); + return family == null ? null : family.array(); + } + + public ByteBuffer bufferForFamily() { + return family; + } + + /** + * column family to check + */ + public checkAndPut_args setFamily(byte[] family) { + setFamily(family == null ? (ByteBuffer)null : ByteBuffer.wrap(family)); + return this; + } + + public checkAndPut_args setFamily(ByteBuffer family) { + this.family = family; + return this; + } + + public void unsetFamily() { + this.family = null; + } + + /** Returns true if field family is set (has been assigned a value) and false otherwise */ + public boolean isSetFamily() { + return this.family != null; + } + + public void setFamilyIsSet(boolean value) { + if (!value) { + this.family = null; + } + } + + /** + * column qualifier to check + */ + public byte[] getQualifier() { + setQualifier(org.apache.thrift.TBaseHelper.rightSize(qualifier)); + return qualifier == null ? null : qualifier.array(); + } + + public ByteBuffer bufferForQualifier() { + return qualifier; + } + + /** + * column qualifier to check + */ + public checkAndPut_args setQualifier(byte[] qualifier) { + setQualifier(qualifier == null ? (ByteBuffer)null : ByteBuffer.wrap(qualifier)); + return this; + } + + public checkAndPut_args setQualifier(ByteBuffer qualifier) { + this.qualifier = qualifier; + return this; + } + + public void unsetQualifier() { + this.qualifier = null; + } + + /** Returns true if field qualifier is set (has been assigned a value) and false otherwise */ + public boolean isSetQualifier() { + return this.qualifier != null; + } + + public void setQualifierIsSet(boolean value) { + if (!value) { + this.qualifier = null; + } + } + + /** + * the expected value, if not provided the + * check is for the non-existence of the + * column in question + */ + public byte[] getValue() { + setValue(org.apache.thrift.TBaseHelper.rightSize(value)); + return value == null ? null : value.array(); + } + + public ByteBuffer bufferForValue() { + return value; + } + + /** + * the expected value, if not provided the + * check is for the non-existence of the + * column in question + */ + public checkAndPut_args setValue(byte[] value) { + setValue(value == null ? (ByteBuffer)null : ByteBuffer.wrap(value)); + return this; + } + + public checkAndPut_args setValue(ByteBuffer value) { + this.value = value; + return this; + } + + public void unsetValue() { + this.value = null; + } + + /** Returns true if field value is set (has been assigned a value) and false otherwise */ + public boolean isSetValue() { + return this.value != null; + } + + public void setValueIsSet(boolean value) { + if (!value) { + this.value = null; + } + } + + /** + * the TPut to put if the check succeeds + */ + public TPut getPut() { + return this.put; + } + + /** + * the TPut to put if the check succeeds + */ + public checkAndPut_args setPut(TPut put) { + this.put = put; + return this; + } + + public void unsetPut() { + this.put = null; + } + + /** Returns true if field put is set (has been assigned a value) and false otherwise */ + public boolean isSetPut() { + return this.put != null; + } + + public void setPutIsSet(boolean value) { + if (!value) { + this.put = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case FAMILY: + if (value == null) { + unsetFamily(); + } else { + setFamily((ByteBuffer)value); + } + break; + + case QUALIFIER: + if (value == null) { + unsetQualifier(); + } else { + setQualifier((ByteBuffer)value); + } + break; + + case VALUE: + if (value == null) { + unsetValue(); + } else { + setValue((ByteBuffer)value); + } + break; + + case PUT: + if (value == null) { + unsetPut(); + } else { + setPut((TPut)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE: + return getTable(); + + case ROW: + return getRow(); + + case FAMILY: + return getFamily(); + + case QUALIFIER: + return getQualifier(); + + case VALUE: + return getValue(); + + case PUT: + return getPut(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE: + return isSetTable(); + case ROW: + return isSetRow(); + case FAMILY: + return isSetFamily(); + case QUALIFIER: + return isSetQualifier(); + case VALUE: + return isSetValue(); + case PUT: + return isSetPut(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof checkAndPut_args) + return this.equals((checkAndPut_args)that); + return false; + } + + public boolean equals(checkAndPut_args that) { + if (that == null) + return false; + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_family = true && this.isSetFamily(); + boolean that_present_family = true && that.isSetFamily(); + if (this_present_family || that_present_family) { + if (!(this_present_family && that_present_family)) + return false; + if (!this.family.equals(that.family)) + return false; + } + + boolean this_present_qualifier = true && this.isSetQualifier(); + boolean that_present_qualifier = true && that.isSetQualifier(); + if (this_present_qualifier || that_present_qualifier) { + if (!(this_present_qualifier && that_present_qualifier)) + return false; + if (!this.qualifier.equals(that.qualifier)) + return false; + } + + boolean this_present_value = true && this.isSetValue(); + boolean that_present_value = true && that.isSetValue(); + if (this_present_value || that_present_value) { + if (!(this_present_value && that_present_value)) + return false; + if (!this.value.equals(that.value)) + return false; + } + + boolean this_present_put = true && this.isSetPut(); + boolean that_present_put = true && that.isSetPut(); + if (this_present_put || that_present_put) { + if (!(this_present_put && that_present_put)) + return false; + if (!this.put.equals(that.put)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(checkAndPut_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + checkAndPut_args typedOther = (checkAndPut_args)other; + + lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetFamily()).compareTo(typedOther.isSetFamily()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetFamily()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.family, typedOther.family); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetQualifier()).compareTo(typedOther.isSetQualifier()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetQualifier()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.qualifier, typedOther.qualifier); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetPut()).compareTo(typedOther.isSetPut()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetPut()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.put, typedOther.put); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("checkAndPut_args("); + boolean first = true; + + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.table, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.row, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("family:"); + if (this.family == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.family, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("qualifier:"); + if (this.qualifier == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.qualifier, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("value:"); + if (this.value == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.value, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("put:"); + if (this.put == null) { + sb.append("null"); + } else { + sb.append(this.put); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (table == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString()); + } + if (row == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'row' was not present! Struct: " + toString()); + } + if (family == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'family' was not present! Struct: " + toString()); + } + if (qualifier == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'qualifier' was not present! Struct: " + toString()); + } + if (put == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'put' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class checkAndPut_argsStandardSchemeFactory implements SchemeFactory { + public checkAndPut_argsStandardScheme getScheme() { + return new checkAndPut_argsStandardScheme(); + } + } + + private static class checkAndPut_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, checkAndPut_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // FAMILY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.family = iprot.readBinary(); + struct.setFamilyIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // QUALIFIER + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.qualifier = iprot.readBinary(); + struct.setQualifierIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.value = iprot.readBinary(); + struct.setValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 6: // PUT + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.put = new TPut(); + struct.put.read(iprot); + struct.setPutIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, checkAndPut_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.table != null) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeBinary(struct.table); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.family != null) { + oprot.writeFieldBegin(FAMILY_FIELD_DESC); + oprot.writeBinary(struct.family); + oprot.writeFieldEnd(); + } + if (struct.qualifier != null) { + oprot.writeFieldBegin(QUALIFIER_FIELD_DESC); + oprot.writeBinary(struct.qualifier); + oprot.writeFieldEnd(); + } + if (struct.value != null) { + oprot.writeFieldBegin(VALUE_FIELD_DESC); + oprot.writeBinary(struct.value); + oprot.writeFieldEnd(); + } + if (struct.put != null) { + oprot.writeFieldBegin(PUT_FIELD_DESC); + struct.put.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class checkAndPut_argsTupleSchemeFactory implements SchemeFactory { + public checkAndPut_argsTupleScheme getScheme() { + return new checkAndPut_argsTupleScheme(); + } + } + + private static class checkAndPut_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, checkAndPut_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.table); + oprot.writeBinary(struct.row); + oprot.writeBinary(struct.family); + oprot.writeBinary(struct.qualifier); + struct.put.write(oprot); + BitSet optionals = new BitSet(); + if (struct.isSetValue()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetValue()) { + oprot.writeBinary(struct.value); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, checkAndPut_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + struct.family = iprot.readBinary(); + struct.setFamilyIsSet(true); + struct.qualifier = iprot.readBinary(); + struct.setQualifierIsSet(true); + struct.put = new TPut(); + struct.put.read(iprot); + struct.setPutIsSet(true); + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.value = iprot.readBinary(); + struct.setValueIsSet(true); + } + } + } + + } + + public static class checkAndPut_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("checkAndPut_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.BOOL, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new checkAndPut_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new checkAndPut_resultTupleSchemeFactory()); + } + + public boolean success; // required + public TIOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(checkAndPut_result.class, metaDataMap); + } + + public checkAndPut_result() { + } + + public checkAndPut_result( + boolean success, + TIOError io) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public checkAndPut_result(checkAndPut_result other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.success = other.success; + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + } + + public checkAndPut_result deepCopy() { + return new checkAndPut_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = false; + this.io = null; + } + + public boolean isSuccess() { + return this.success; + } + + public checkAndPut_result setSuccess(boolean success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bit_vector.clear(__SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return __isset_bit_vector.get(__SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bit_vector.set(__SUCCESS_ISSET_ID, value); + } + + public TIOError getIo() { + return this.io; + } + + public checkAndPut_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Boolean)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Boolean.valueOf(isSuccess()); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof checkAndPut_result) + return this.equals((checkAndPut_result)that); + return false; + } + + public boolean equals(checkAndPut_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(checkAndPut_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + checkAndPut_result typedOther = (checkAndPut_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("checkAndPut_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class checkAndPut_resultStandardSchemeFactory implements SchemeFactory { + public checkAndPut_resultStandardScheme getScheme() { + return new checkAndPut_resultStandardScheme(); + } + } + + private static class checkAndPut_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, checkAndPut_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, checkAndPut_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeBool(struct.success); + oprot.writeFieldEnd(); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class checkAndPut_resultTupleSchemeFactory implements SchemeFactory { + public checkAndPut_resultTupleScheme getScheme() { + return new checkAndPut_resultTupleScheme(); + } + } + + private static class checkAndPut_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, checkAndPut_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeBool(struct.success); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, checkAndPut_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class putMultiple_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("putMultiple_args"); + + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField PUTS_FIELD_DESC = new org.apache.thrift.protocol.TField("puts", org.apache.thrift.protocol.TType.LIST, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new putMultiple_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new putMultiple_argsTupleSchemeFactory()); + } + + /** + * the table to put data in + */ + public ByteBuffer table; // required + /** + * a list of TPuts to commit + */ + public List puts; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * the table to put data in + */ + TABLE((short)1, "table"), + /** + * a list of TPuts to commit + */ + PUTS((short)2, "puts"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE + return TABLE; + case 2: // PUTS + return PUTS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.PUTS, new org.apache.thrift.meta_data.FieldMetaData("puts", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TPut.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(putMultiple_args.class, metaDataMap); + } + + public putMultiple_args() { + } + + public putMultiple_args( + ByteBuffer table, + List puts) + { + this(); + this.table = table; + this.puts = puts; + } + + /** + * Performs a deep copy on other. + */ + public putMultiple_args(putMultiple_args other) { + if (other.isSetTable()) { + this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table); +; + } + if (other.isSetPuts()) { + List __this__puts = new ArrayList(); + for (TPut other_element : other.puts) { + __this__puts.add(new TPut(other_element)); + } + this.puts = __this__puts; + } + } + + public putMultiple_args deepCopy() { + return new putMultiple_args(this); + } + + @Override + public void clear() { + this.table = null; + this.puts = null; + } + + /** + * the table to put data in + */ + public byte[] getTable() { + setTable(org.apache.thrift.TBaseHelper.rightSize(table)); + return table == null ? null : table.array(); + } + + public ByteBuffer bufferForTable() { + return table; + } + + /** + * the table to put data in + */ + public putMultiple_args setTable(byte[] table) { + setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table)); + return this; + } + + public putMultiple_args setTable(ByteBuffer table) { + this.table = table; + return this; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + public int getPutsSize() { + return (this.puts == null) ? 0 : this.puts.size(); + } + + public java.util.Iterator getPutsIterator() { + return (this.puts == null) ? null : this.puts.iterator(); + } + + public void addToPuts(TPut elem) { + if (this.puts == null) { + this.puts = new ArrayList(); + } + this.puts.add(elem); + } + + /** + * a list of TPuts to commit + */ + public List getPuts() { + return this.puts; + } + + /** + * a list of TPuts to commit + */ + public putMultiple_args setPuts(List puts) { + this.puts = puts; + return this; + } + + public void unsetPuts() { + this.puts = null; + } + + /** Returns true if field puts is set (has been assigned a value) and false otherwise */ + public boolean isSetPuts() { + return this.puts != null; + } + + public void setPutsIsSet(boolean value) { + if (!value) { + this.puts = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((ByteBuffer)value); + } + break; + + case PUTS: + if (value == null) { + unsetPuts(); + } else { + setPuts((List)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE: + return getTable(); + + case PUTS: + return getPuts(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE: + return isSetTable(); + case PUTS: + return isSetPuts(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof putMultiple_args) + return this.equals((putMultiple_args)that); + return false; + } + + public boolean equals(putMultiple_args that) { + if (that == null) + return false; + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_puts = true && this.isSetPuts(); + boolean that_present_puts = true && that.isSetPuts(); + if (this_present_puts || that_present_puts) { + if (!(this_present_puts && that_present_puts)) + return false; + if (!this.puts.equals(that.puts)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(putMultiple_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + putMultiple_args typedOther = (putMultiple_args)other; + + lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetPuts()).compareTo(typedOther.isSetPuts()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetPuts()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.puts, typedOther.puts); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("putMultiple_args("); + boolean first = true; + + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.table, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("puts:"); + if (this.puts == null) { + sb.append("null"); + } else { + sb.append(this.puts); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (table == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString()); + } + if (puts == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'puts' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class putMultiple_argsStandardSchemeFactory implements SchemeFactory { + public putMultiple_argsStandardScheme getScheme() { + return new putMultiple_argsStandardScheme(); + } + } + + private static class putMultiple_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, putMultiple_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // PUTS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list64 = iprot.readListBegin(); + struct.puts = new ArrayList(_list64.size); + for (int _i65 = 0; _i65 < _list64.size; ++_i65) + { + TPut _elem66; // optional + _elem66 = new TPut(); + _elem66.read(iprot); + struct.puts.add(_elem66); + } + iprot.readListEnd(); + } + struct.setPutsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, putMultiple_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.table != null) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeBinary(struct.table); + oprot.writeFieldEnd(); + } + if (struct.puts != null) { + oprot.writeFieldBegin(PUTS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.puts.size())); + for (TPut _iter67 : struct.puts) + { + _iter67.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class putMultiple_argsTupleSchemeFactory implements SchemeFactory { + public putMultiple_argsTupleScheme getScheme() { + return new putMultiple_argsTupleScheme(); + } + } + + private static class putMultiple_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, putMultiple_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.table); + { + oprot.writeI32(struct.puts.size()); + for (TPut _iter68 : struct.puts) + { + _iter68.write(oprot); + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, putMultiple_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + { + org.apache.thrift.protocol.TList _list69 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.puts = new ArrayList(_list69.size); + for (int _i70 = 0; _i70 < _list69.size; ++_i70) + { + TPut _elem71; // optional + _elem71 = new TPut(); + _elem71.read(iprot); + struct.puts.add(_elem71); + } + } + struct.setPutsIsSet(true); + } + } + + } + + public static class putMultiple_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("putMultiple_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new putMultiple_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new putMultiple_resultTupleSchemeFactory()); + } + + public TIOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(putMultiple_result.class, metaDataMap); + } + + public putMultiple_result() { + } + + public putMultiple_result( + TIOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public putMultiple_result(putMultiple_result other) { + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + } + + public putMultiple_result deepCopy() { + return new putMultiple_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public TIOError getIo() { + return this.io; + } + + public putMultiple_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof putMultiple_result) + return this.equals((putMultiple_result)that); + return false; + } + + public boolean equals(putMultiple_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(putMultiple_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + putMultiple_result typedOther = (putMultiple_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("putMultiple_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class putMultiple_resultStandardSchemeFactory implements SchemeFactory { + public putMultiple_resultStandardScheme getScheme() { + return new putMultiple_resultStandardScheme(); + } + } + + private static class putMultiple_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, putMultiple_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, putMultiple_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class putMultiple_resultTupleSchemeFactory implements SchemeFactory { + public putMultiple_resultTupleScheme getScheme() { + return new putMultiple_resultTupleScheme(); + } + } + + private static class putMultiple_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, putMultiple_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, putMultiple_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class deleteSingle_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteSingle_args"); + + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField DELETE_SINGLE_FIELD_DESC = new org.apache.thrift.protocol.TField("deleteSingle", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteSingle_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteSingle_argsTupleSchemeFactory()); + } + + /** + * the table to delete from + */ + public ByteBuffer table; // required + /** + * the TDelete to delete + */ + public TDelete deleteSingle; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * the table to delete from + */ + TABLE((short)1, "table"), + /** + * the TDelete to delete + */ + DELETE_SINGLE((short)2, "deleteSingle"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE + return TABLE; + case 2: // DELETE_SINGLE + return DELETE_SINGLE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.DELETE_SINGLE, new org.apache.thrift.meta_data.FieldMetaData("deleteSingle", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TDelete.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteSingle_args.class, metaDataMap); + } + + public deleteSingle_args() { + } + + public deleteSingle_args( + ByteBuffer table, + TDelete deleteSingle) + { + this(); + this.table = table; + this.deleteSingle = deleteSingle; + } + + /** + * Performs a deep copy on other. + */ + public deleteSingle_args(deleteSingle_args other) { + if (other.isSetTable()) { + this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table); +; + } + if (other.isSetDeleteSingle()) { + this.deleteSingle = new TDelete(other.deleteSingle); + } + } + + public deleteSingle_args deepCopy() { + return new deleteSingle_args(this); + } + + @Override + public void clear() { + this.table = null; + this.deleteSingle = null; + } + + /** + * the table to delete from + */ + public byte[] getTable() { + setTable(org.apache.thrift.TBaseHelper.rightSize(table)); + return table == null ? null : table.array(); + } + + public ByteBuffer bufferForTable() { + return table; + } + + /** + * the table to delete from + */ + public deleteSingle_args setTable(byte[] table) { + setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table)); + return this; + } + + public deleteSingle_args setTable(ByteBuffer table) { + this.table = table; + return this; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + /** + * the TDelete to delete + */ + public TDelete getDeleteSingle() { + return this.deleteSingle; + } + + /** + * the TDelete to delete + */ + public deleteSingle_args setDeleteSingle(TDelete deleteSingle) { + this.deleteSingle = deleteSingle; + return this; + } + + public void unsetDeleteSingle() { + this.deleteSingle = null; + } + + /** Returns true if field deleteSingle is set (has been assigned a value) and false otherwise */ + public boolean isSetDeleteSingle() { + return this.deleteSingle != null; + } + + public void setDeleteSingleIsSet(boolean value) { + if (!value) { + this.deleteSingle = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((ByteBuffer)value); + } + break; + + case DELETE_SINGLE: + if (value == null) { + unsetDeleteSingle(); + } else { + setDeleteSingle((TDelete)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE: + return getTable(); + + case DELETE_SINGLE: + return getDeleteSingle(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE: + return isSetTable(); + case DELETE_SINGLE: + return isSetDeleteSingle(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteSingle_args) + return this.equals((deleteSingle_args)that); + return false; + } + + public boolean equals(deleteSingle_args that) { + if (that == null) + return false; + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_deleteSingle = true && this.isSetDeleteSingle(); + boolean that_present_deleteSingle = true && that.isSetDeleteSingle(); + if (this_present_deleteSingle || that_present_deleteSingle) { + if (!(this_present_deleteSingle && that_present_deleteSingle)) + return false; + if (!this.deleteSingle.equals(that.deleteSingle)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteSingle_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteSingle_args typedOther = (deleteSingle_args)other; + + lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetDeleteSingle()).compareTo(typedOther.isSetDeleteSingle()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetDeleteSingle()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.deleteSingle, typedOther.deleteSingle); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteSingle_args("); + boolean first = true; + + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.table, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("deleteSingle:"); + if (this.deleteSingle == null) { + sb.append("null"); + } else { + sb.append(this.deleteSingle); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (table == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString()); + } + if (deleteSingle == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'deleteSingle' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteSingle_argsStandardSchemeFactory implements SchemeFactory { + public deleteSingle_argsStandardScheme getScheme() { + return new deleteSingle_argsStandardScheme(); + } + } + + private static class deleteSingle_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteSingle_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // DELETE_SINGLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.deleteSingle = new TDelete(); + struct.deleteSingle.read(iprot); + struct.setDeleteSingleIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteSingle_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.table != null) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeBinary(struct.table); + oprot.writeFieldEnd(); + } + if (struct.deleteSingle != null) { + oprot.writeFieldBegin(DELETE_SINGLE_FIELD_DESC); + struct.deleteSingle.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteSingle_argsTupleSchemeFactory implements SchemeFactory { + public deleteSingle_argsTupleScheme getScheme() { + return new deleteSingle_argsTupleScheme(); + } + } + + private static class deleteSingle_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteSingle_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.table); + struct.deleteSingle.write(oprot); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteSingle_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + struct.deleteSingle = new TDelete(); + struct.deleteSingle.read(iprot); + struct.setDeleteSingleIsSet(true); + } + } + + } + + public static class deleteSingle_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteSingle_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteSingle_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteSingle_resultTupleSchemeFactory()); + } + + public TIOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteSingle_result.class, metaDataMap); + } + + public deleteSingle_result() { + } + + public deleteSingle_result( + TIOError io) + { + this(); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public deleteSingle_result(deleteSingle_result other) { + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + } + + public deleteSingle_result deepCopy() { + return new deleteSingle_result(this); + } + + @Override + public void clear() { + this.io = null; + } + + public TIOError getIo() { + return this.io; + } + + public deleteSingle_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteSingle_result) + return this.equals((deleteSingle_result)that); + return false; + } + + public boolean equals(deleteSingle_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteSingle_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteSingle_result typedOther = (deleteSingle_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteSingle_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteSingle_resultStandardSchemeFactory implements SchemeFactory { + public deleteSingle_resultStandardScheme getScheme() { + return new deleteSingle_resultStandardScheme(); + } + } + + private static class deleteSingle_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteSingle_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteSingle_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteSingle_resultTupleSchemeFactory implements SchemeFactory { + public deleteSingle_resultTupleScheme getScheme() { + return new deleteSingle_resultTupleScheme(); + } + } + + private static class deleteSingle_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteSingle_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteSingle_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class deleteMultiple_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteMultiple_args"); + + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField DELETES_FIELD_DESC = new org.apache.thrift.protocol.TField("deletes", org.apache.thrift.protocol.TType.LIST, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteMultiple_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteMultiple_argsTupleSchemeFactory()); + } + + /** + * the table to delete from + */ + public ByteBuffer table; // required + /** + * list of TDeletes to delete + */ + public List deletes; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * the table to delete from + */ + TABLE((short)1, "table"), + /** + * list of TDeletes to delete + */ + DELETES((short)2, "deletes"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE + return TABLE; + case 2: // DELETES + return DELETES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.DELETES, new org.apache.thrift.meta_data.FieldMetaData("deletes", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TDelete.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteMultiple_args.class, metaDataMap); + } + + public deleteMultiple_args() { + } + + public deleteMultiple_args( + ByteBuffer table, + List deletes) + { + this(); + this.table = table; + this.deletes = deletes; + } + + /** + * Performs a deep copy on other. + */ + public deleteMultiple_args(deleteMultiple_args other) { + if (other.isSetTable()) { + this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table); +; + } + if (other.isSetDeletes()) { + List __this__deletes = new ArrayList(); + for (TDelete other_element : other.deletes) { + __this__deletes.add(new TDelete(other_element)); + } + this.deletes = __this__deletes; + } + } + + public deleteMultiple_args deepCopy() { + return new deleteMultiple_args(this); + } + + @Override + public void clear() { + this.table = null; + this.deletes = null; + } + + /** + * the table to delete from + */ + public byte[] getTable() { + setTable(org.apache.thrift.TBaseHelper.rightSize(table)); + return table == null ? null : table.array(); + } + + public ByteBuffer bufferForTable() { + return table; + } + + /** + * the table to delete from + */ + public deleteMultiple_args setTable(byte[] table) { + setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table)); + return this; + } + + public deleteMultiple_args setTable(ByteBuffer table) { + this.table = table; + return this; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + public int getDeletesSize() { + return (this.deletes == null) ? 0 : this.deletes.size(); + } + + public java.util.Iterator getDeletesIterator() { + return (this.deletes == null) ? null : this.deletes.iterator(); + } + + public void addToDeletes(TDelete elem) { + if (this.deletes == null) { + this.deletes = new ArrayList(); + } + this.deletes.add(elem); + } + + /** + * list of TDeletes to delete + */ + public List getDeletes() { + return this.deletes; + } + + /** + * list of TDeletes to delete + */ + public deleteMultiple_args setDeletes(List deletes) { + this.deletes = deletes; + return this; + } + + public void unsetDeletes() { + this.deletes = null; + } + + /** Returns true if field deletes is set (has been assigned a value) and false otherwise */ + public boolean isSetDeletes() { + return this.deletes != null; + } + + public void setDeletesIsSet(boolean value) { + if (!value) { + this.deletes = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((ByteBuffer)value); + } + break; + + case DELETES: + if (value == null) { + unsetDeletes(); + } else { + setDeletes((List)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE: + return getTable(); + + case DELETES: + return getDeletes(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE: + return isSetTable(); + case DELETES: + return isSetDeletes(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteMultiple_args) + return this.equals((deleteMultiple_args)that); + return false; + } + + public boolean equals(deleteMultiple_args that) { + if (that == null) + return false; + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_deletes = true && this.isSetDeletes(); + boolean that_present_deletes = true && that.isSetDeletes(); + if (this_present_deletes || that_present_deletes) { + if (!(this_present_deletes && that_present_deletes)) + return false; + if (!this.deletes.equals(that.deletes)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteMultiple_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteMultiple_args typedOther = (deleteMultiple_args)other; + + lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetDeletes()).compareTo(typedOther.isSetDeletes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetDeletes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.deletes, typedOther.deletes); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteMultiple_args("); + boolean first = true; + + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.table, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("deletes:"); + if (this.deletes == null) { + sb.append("null"); + } else { + sb.append(this.deletes); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (table == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString()); + } + if (deletes == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'deletes' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteMultiple_argsStandardSchemeFactory implements SchemeFactory { + public deleteMultiple_argsStandardScheme getScheme() { + return new deleteMultiple_argsStandardScheme(); + } + } + + private static class deleteMultiple_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteMultiple_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // DELETES + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list72 = iprot.readListBegin(); + struct.deletes = new ArrayList(_list72.size); + for (int _i73 = 0; _i73 < _list72.size; ++_i73) + { + TDelete _elem74; // optional + _elem74 = new TDelete(); + _elem74.read(iprot); + struct.deletes.add(_elem74); + } + iprot.readListEnd(); + } + struct.setDeletesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteMultiple_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.table != null) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeBinary(struct.table); + oprot.writeFieldEnd(); + } + if (struct.deletes != null) { + oprot.writeFieldBegin(DELETES_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.deletes.size())); + for (TDelete _iter75 : struct.deletes) + { + _iter75.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteMultiple_argsTupleSchemeFactory implements SchemeFactory { + public deleteMultiple_argsTupleScheme getScheme() { + return new deleteMultiple_argsTupleScheme(); + } + } + + private static class deleteMultiple_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.table); + { + oprot.writeI32(struct.deletes.size()); + for (TDelete _iter76 : struct.deletes) + { + _iter76.write(oprot); + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + { + org.apache.thrift.protocol.TList _list77 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.deletes = new ArrayList(_list77.size); + for (int _i78 = 0; _i78 < _list77.size; ++_i78) + { + TDelete _elem79; // optional + _elem79 = new TDelete(); + _elem79.read(iprot); + struct.deletes.add(_elem79); + } + } + struct.setDeletesIsSet(true); + } + } + + } + + public static class deleteMultiple_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("deleteMultiple_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new deleteMultiple_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new deleteMultiple_resultTupleSchemeFactory()); + } + + public List success; // required + public TIOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TDelete.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(deleteMultiple_result.class, metaDataMap); + } + + public deleteMultiple_result() { + } + + public deleteMultiple_result( + List success, + TIOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public deleteMultiple_result(deleteMultiple_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TDelete other_element : other.success) { + __this__success.add(new TDelete(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + } + + public deleteMultiple_result deepCopy() { + return new deleteMultiple_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TDelete elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public deleteMultiple_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public TIOError getIo() { + return this.io; + } + + public deleteMultiple_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof deleteMultiple_result) + return this.equals((deleteMultiple_result)that); + return false; + } + + public boolean equals(deleteMultiple_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(deleteMultiple_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + deleteMultiple_result typedOther = (deleteMultiple_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("deleteMultiple_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class deleteMultiple_resultStandardSchemeFactory implements SchemeFactory { + public deleteMultiple_resultStandardScheme getScheme() { + return new deleteMultiple_resultStandardScheme(); + } + } + + private static class deleteMultiple_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, deleteMultiple_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list80 = iprot.readListBegin(); + struct.success = new ArrayList(_list80.size); + for (int _i81 = 0; _i81 < _list80.size; ++_i81) + { + TDelete _elem82; // optional + _elem82 = new TDelete(); + _elem82.read(iprot); + struct.success.add(_elem82); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, deleteMultiple_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TDelete _iter83 : struct.success) + { + _iter83.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class deleteMultiple_resultTupleSchemeFactory implements SchemeFactory { + public deleteMultiple_resultTupleScheme getScheme() { + return new deleteMultiple_resultTupleScheme(); + } + } + + private static class deleteMultiple_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TDelete _iter84 : struct.success) + { + _iter84.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, deleteMultiple_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list85 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list85.size); + for (int _i86 = 0; _i86 < _list85.size; ++_i86) + { + TDelete _elem87; // optional + _elem87 = new TDelete(); + _elem87.read(iprot); + struct.success.add(_elem87); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class checkAndDelete_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("checkAndDelete_args"); + + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField FAMILY_FIELD_DESC = new org.apache.thrift.protocol.TField("family", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField QUALIFIER_FIELD_DESC = new org.apache.thrift.protocol.TField("qualifier", org.apache.thrift.protocol.TType.STRING, (short)4); + private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)5); + private static final org.apache.thrift.protocol.TField DELETE_SINGLE_FIELD_DESC = new org.apache.thrift.protocol.TField("deleteSingle", org.apache.thrift.protocol.TType.STRUCT, (short)6); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new checkAndDelete_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new checkAndDelete_argsTupleSchemeFactory()); + } + + /** + * to check in and delete from + */ + public ByteBuffer table; // required + /** + * row to check + */ + public ByteBuffer row; // required + /** + * column family to check + */ + public ByteBuffer family; // required + /** + * column qualifier to check + */ + public ByteBuffer qualifier; // required + /** + * the expected value, if not provided the + * check is for the non-existence of the + * column in question + */ + public ByteBuffer value; // required + /** + * the TDelete to execute if the check succeeds + */ + public TDelete deleteSingle; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * to check in and delete from + */ + TABLE((short)1, "table"), + /** + * row to check + */ + ROW((short)2, "row"), + /** + * column family to check + */ + FAMILY((short)3, "family"), + /** + * column qualifier to check + */ + QUALIFIER((short)4, "qualifier"), + /** + * the expected value, if not provided the + * check is for the non-existence of the + * column in question + */ + VALUE((short)5, "value"), + /** + * the TDelete to execute if the check succeeds + */ + DELETE_SINGLE((short)6, "deleteSingle"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE + return TABLE; + case 2: // ROW + return ROW; + case 3: // FAMILY + return FAMILY; + case 4: // QUALIFIER + return QUALIFIER; + case 5: // VALUE + return VALUE; + case 6: // DELETE_SINGLE + return DELETE_SINGLE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.FAMILY, new org.apache.thrift.meta_data.FieldMetaData("family", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.QUALIFIER, new org.apache.thrift.meta_data.FieldMetaData("qualifier", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.DELETE_SINGLE, new org.apache.thrift.meta_data.FieldMetaData("deleteSingle", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TDelete.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(checkAndDelete_args.class, metaDataMap); + } + + public checkAndDelete_args() { + } + + public checkAndDelete_args( + ByteBuffer table, + ByteBuffer row, + ByteBuffer family, + ByteBuffer qualifier, + ByteBuffer value, + TDelete deleteSingle) + { + this(); + this.table = table; + this.row = row; + this.family = family; + this.qualifier = qualifier; + this.value = value; + this.deleteSingle = deleteSingle; + } + + /** + * Performs a deep copy on other. + */ + public checkAndDelete_args(checkAndDelete_args other) { + if (other.isSetTable()) { + this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table); +; + } + if (other.isSetRow()) { + this.row = org.apache.thrift.TBaseHelper.copyBinary(other.row); +; + } + if (other.isSetFamily()) { + this.family = org.apache.thrift.TBaseHelper.copyBinary(other.family); +; + } + if (other.isSetQualifier()) { + this.qualifier = org.apache.thrift.TBaseHelper.copyBinary(other.qualifier); +; + } + if (other.isSetValue()) { + this.value = org.apache.thrift.TBaseHelper.copyBinary(other.value); +; + } + if (other.isSetDeleteSingle()) { + this.deleteSingle = new TDelete(other.deleteSingle); + } + } + + public checkAndDelete_args deepCopy() { + return new checkAndDelete_args(this); + } + + @Override + public void clear() { + this.table = null; + this.row = null; + this.family = null; + this.qualifier = null; + this.value = null; + this.deleteSingle = null; + } + + /** + * to check in and delete from + */ + public byte[] getTable() { + setTable(org.apache.thrift.TBaseHelper.rightSize(table)); + return table == null ? null : table.array(); + } + + public ByteBuffer bufferForTable() { + return table; + } + + /** + * to check in and delete from + */ + public checkAndDelete_args setTable(byte[] table) { + setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table)); + return this; + } + + public checkAndDelete_args setTable(ByteBuffer table) { + this.table = table; + return this; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + /** + * row to check + */ + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + /** + * row to check + */ + public checkAndDelete_args setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public checkAndDelete_args setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + /** + * column family to check + */ + public byte[] getFamily() { + setFamily(org.apache.thrift.TBaseHelper.rightSize(family)); + return family == null ? null : family.array(); + } + + public ByteBuffer bufferForFamily() { + return family; + } + + /** + * column family to check + */ + public checkAndDelete_args setFamily(byte[] family) { + setFamily(family == null ? (ByteBuffer)null : ByteBuffer.wrap(family)); + return this; + } + + public checkAndDelete_args setFamily(ByteBuffer family) { + this.family = family; + return this; + } + + public void unsetFamily() { + this.family = null; + } + + /** Returns true if field family is set (has been assigned a value) and false otherwise */ + public boolean isSetFamily() { + return this.family != null; + } + + public void setFamilyIsSet(boolean value) { + if (!value) { + this.family = null; + } + } + + /** + * column qualifier to check + */ + public byte[] getQualifier() { + setQualifier(org.apache.thrift.TBaseHelper.rightSize(qualifier)); + return qualifier == null ? null : qualifier.array(); + } + + public ByteBuffer bufferForQualifier() { + return qualifier; + } + + /** + * column qualifier to check + */ + public checkAndDelete_args setQualifier(byte[] qualifier) { + setQualifier(qualifier == null ? (ByteBuffer)null : ByteBuffer.wrap(qualifier)); + return this; + } + + public checkAndDelete_args setQualifier(ByteBuffer qualifier) { + this.qualifier = qualifier; + return this; + } + + public void unsetQualifier() { + this.qualifier = null; + } + + /** Returns true if field qualifier is set (has been assigned a value) and false otherwise */ + public boolean isSetQualifier() { + return this.qualifier != null; + } + + public void setQualifierIsSet(boolean value) { + if (!value) { + this.qualifier = null; + } + } + + /** + * the expected value, if not provided the + * check is for the non-existence of the + * column in question + */ + public byte[] getValue() { + setValue(org.apache.thrift.TBaseHelper.rightSize(value)); + return value == null ? null : value.array(); + } + + public ByteBuffer bufferForValue() { + return value; + } + + /** + * the expected value, if not provided the + * check is for the non-existence of the + * column in question + */ + public checkAndDelete_args setValue(byte[] value) { + setValue(value == null ? (ByteBuffer)null : ByteBuffer.wrap(value)); + return this; + } + + public checkAndDelete_args setValue(ByteBuffer value) { + this.value = value; + return this; + } + + public void unsetValue() { + this.value = null; + } + + /** Returns true if field value is set (has been assigned a value) and false otherwise */ + public boolean isSetValue() { + return this.value != null; + } + + public void setValueIsSet(boolean value) { + if (!value) { + this.value = null; + } + } + + /** + * the TDelete to execute if the check succeeds + */ + public TDelete getDeleteSingle() { + return this.deleteSingle; + } + + /** + * the TDelete to execute if the check succeeds + */ + public checkAndDelete_args setDeleteSingle(TDelete deleteSingle) { + this.deleteSingle = deleteSingle; + return this; + } + + public void unsetDeleteSingle() { + this.deleteSingle = null; + } + + /** Returns true if field deleteSingle is set (has been assigned a value) and false otherwise */ + public boolean isSetDeleteSingle() { + return this.deleteSingle != null; + } + + public void setDeleteSingleIsSet(boolean value) { + if (!value) { + this.deleteSingle = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((ByteBuffer)value); + } + break; + + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case FAMILY: + if (value == null) { + unsetFamily(); + } else { + setFamily((ByteBuffer)value); + } + break; + + case QUALIFIER: + if (value == null) { + unsetQualifier(); + } else { + setQualifier((ByteBuffer)value); + } + break; + + case VALUE: + if (value == null) { + unsetValue(); + } else { + setValue((ByteBuffer)value); + } + break; + + case DELETE_SINGLE: + if (value == null) { + unsetDeleteSingle(); + } else { + setDeleteSingle((TDelete)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE: + return getTable(); + + case ROW: + return getRow(); + + case FAMILY: + return getFamily(); + + case QUALIFIER: + return getQualifier(); + + case VALUE: + return getValue(); + + case DELETE_SINGLE: + return getDeleteSingle(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE: + return isSetTable(); + case ROW: + return isSetRow(); + case FAMILY: + return isSetFamily(); + case QUALIFIER: + return isSetQualifier(); + case VALUE: + return isSetValue(); + case DELETE_SINGLE: + return isSetDeleteSingle(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof checkAndDelete_args) + return this.equals((checkAndDelete_args)that); + return false; + } + + public boolean equals(checkAndDelete_args that) { + if (that == null) + return false; + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_family = true && this.isSetFamily(); + boolean that_present_family = true && that.isSetFamily(); + if (this_present_family || that_present_family) { + if (!(this_present_family && that_present_family)) + return false; + if (!this.family.equals(that.family)) + return false; + } + + boolean this_present_qualifier = true && this.isSetQualifier(); + boolean that_present_qualifier = true && that.isSetQualifier(); + if (this_present_qualifier || that_present_qualifier) { + if (!(this_present_qualifier && that_present_qualifier)) + return false; + if (!this.qualifier.equals(that.qualifier)) + return false; + } + + boolean this_present_value = true && this.isSetValue(); + boolean that_present_value = true && that.isSetValue(); + if (this_present_value || that_present_value) { + if (!(this_present_value && that_present_value)) + return false; + if (!this.value.equals(that.value)) + return false; + } + + boolean this_present_deleteSingle = true && this.isSetDeleteSingle(); + boolean that_present_deleteSingle = true && that.isSetDeleteSingle(); + if (this_present_deleteSingle || that_present_deleteSingle) { + if (!(this_present_deleteSingle && that_present_deleteSingle)) + return false; + if (!this.deleteSingle.equals(that.deleteSingle)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(checkAndDelete_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + checkAndDelete_args typedOther = (checkAndDelete_args)other; + + lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetFamily()).compareTo(typedOther.isSetFamily()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetFamily()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.family, typedOther.family); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetQualifier()).compareTo(typedOther.isSetQualifier()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetQualifier()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.qualifier, typedOther.qualifier); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetDeleteSingle()).compareTo(typedOther.isSetDeleteSingle()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetDeleteSingle()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.deleteSingle, typedOther.deleteSingle); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("checkAndDelete_args("); + boolean first = true; + + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.table, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.row, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("family:"); + if (this.family == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.family, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("qualifier:"); + if (this.qualifier == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.qualifier, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("value:"); + if (this.value == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.value, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("deleteSingle:"); + if (this.deleteSingle == null) { + sb.append("null"); + } else { + sb.append(this.deleteSingle); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (table == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString()); + } + if (row == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'row' was not present! Struct: " + toString()); + } + if (family == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'family' was not present! Struct: " + toString()); + } + if (qualifier == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'qualifier' was not present! Struct: " + toString()); + } + if (deleteSingle == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'deleteSingle' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class checkAndDelete_argsStandardSchemeFactory implements SchemeFactory { + public checkAndDelete_argsStandardScheme getScheme() { + return new checkAndDelete_argsStandardScheme(); + } + } + + private static class checkAndDelete_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, checkAndDelete_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // FAMILY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.family = iprot.readBinary(); + struct.setFamilyIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // QUALIFIER + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.qualifier = iprot.readBinary(); + struct.setQualifierIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.value = iprot.readBinary(); + struct.setValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 6: // DELETE_SINGLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.deleteSingle = new TDelete(); + struct.deleteSingle.read(iprot); + struct.setDeleteSingleIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, checkAndDelete_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.table != null) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeBinary(struct.table); + oprot.writeFieldEnd(); + } + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.family != null) { + oprot.writeFieldBegin(FAMILY_FIELD_DESC); + oprot.writeBinary(struct.family); + oprot.writeFieldEnd(); + } + if (struct.qualifier != null) { + oprot.writeFieldBegin(QUALIFIER_FIELD_DESC); + oprot.writeBinary(struct.qualifier); + oprot.writeFieldEnd(); + } + if (struct.value != null) { + oprot.writeFieldBegin(VALUE_FIELD_DESC); + oprot.writeBinary(struct.value); + oprot.writeFieldEnd(); + } + if (struct.deleteSingle != null) { + oprot.writeFieldBegin(DELETE_SINGLE_FIELD_DESC); + struct.deleteSingle.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class checkAndDelete_argsTupleSchemeFactory implements SchemeFactory { + public checkAndDelete_argsTupleScheme getScheme() { + return new checkAndDelete_argsTupleScheme(); + } + } + + private static class checkAndDelete_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, checkAndDelete_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.table); + oprot.writeBinary(struct.row); + oprot.writeBinary(struct.family); + oprot.writeBinary(struct.qualifier); + struct.deleteSingle.write(oprot); + BitSet optionals = new BitSet(); + if (struct.isSetValue()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetValue()) { + oprot.writeBinary(struct.value); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, checkAndDelete_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + struct.family = iprot.readBinary(); + struct.setFamilyIsSet(true); + struct.qualifier = iprot.readBinary(); + struct.setQualifierIsSet(true); + struct.deleteSingle = new TDelete(); + struct.deleteSingle.read(iprot); + struct.setDeleteSingleIsSet(true); + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.value = iprot.readBinary(); + struct.setValueIsSet(true); + } + } + } + + } + + public static class checkAndDelete_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("checkAndDelete_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.BOOL, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new checkAndDelete_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new checkAndDelete_resultTupleSchemeFactory()); + } + + public boolean success; // required + public TIOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(checkAndDelete_result.class, metaDataMap); + } + + public checkAndDelete_result() { + } + + public checkAndDelete_result( + boolean success, + TIOError io) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public checkAndDelete_result(checkAndDelete_result other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.success = other.success; + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + } + + public checkAndDelete_result deepCopy() { + return new checkAndDelete_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = false; + this.io = null; + } + + public boolean isSuccess() { + return this.success; + } + + public checkAndDelete_result setSuccess(boolean success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bit_vector.clear(__SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return __isset_bit_vector.get(__SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bit_vector.set(__SUCCESS_ISSET_ID, value); + } + + public TIOError getIo() { + return this.io; + } + + public checkAndDelete_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Boolean)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Boolean.valueOf(isSuccess()); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof checkAndDelete_result) + return this.equals((checkAndDelete_result)that); + return false; + } + + public boolean equals(checkAndDelete_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(checkAndDelete_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + checkAndDelete_result typedOther = (checkAndDelete_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("checkAndDelete_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class checkAndDelete_resultStandardSchemeFactory implements SchemeFactory { + public checkAndDelete_resultStandardScheme getScheme() { + return new checkAndDelete_resultStandardScheme(); + } + } + + private static class checkAndDelete_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, checkAndDelete_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, checkAndDelete_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeBool(struct.success); + oprot.writeFieldEnd(); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class checkAndDelete_resultTupleSchemeFactory implements SchemeFactory { + public checkAndDelete_resultTupleScheme getScheme() { + return new checkAndDelete_resultTupleScheme(); + } + } + + private static class checkAndDelete_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, checkAndDelete_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeBool(struct.success); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, checkAndDelete_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class increment_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("increment_args"); + + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField INCREMENT_FIELD_DESC = new org.apache.thrift.protocol.TField("increment", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new increment_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new increment_argsTupleSchemeFactory()); + } + + /** + * the table to increment the value on + */ + public ByteBuffer table; // required + /** + * the TIncrement to increment + */ + public TIncrement increment; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * the table to increment the value on + */ + TABLE((short)1, "table"), + /** + * the TIncrement to increment + */ + INCREMENT((short)2, "increment"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE + return TABLE; + case 2: // INCREMENT + return INCREMENT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.INCREMENT, new org.apache.thrift.meta_data.FieldMetaData("increment", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TIncrement.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(increment_args.class, metaDataMap); + } + + public increment_args() { + } + + public increment_args( + ByteBuffer table, + TIncrement increment) + { + this(); + this.table = table; + this.increment = increment; + } + + /** + * Performs a deep copy on other. + */ + public increment_args(increment_args other) { + if (other.isSetTable()) { + this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table); +; + } + if (other.isSetIncrement()) { + this.increment = new TIncrement(other.increment); + } + } + + public increment_args deepCopy() { + return new increment_args(this); + } + + @Override + public void clear() { + this.table = null; + this.increment = null; + } + + /** + * the table to increment the value on + */ + public byte[] getTable() { + setTable(org.apache.thrift.TBaseHelper.rightSize(table)); + return table == null ? null : table.array(); + } + + public ByteBuffer bufferForTable() { + return table; + } + + /** + * the table to increment the value on + */ + public increment_args setTable(byte[] table) { + setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table)); + return this; + } + + public increment_args setTable(ByteBuffer table) { + this.table = table; + return this; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + /** + * the TIncrement to increment + */ + public TIncrement getIncrement() { + return this.increment; + } + + /** + * the TIncrement to increment + */ + public increment_args setIncrement(TIncrement increment) { + this.increment = increment; + return this; + } + + public void unsetIncrement() { + this.increment = null; + } + + /** Returns true if field increment is set (has been assigned a value) and false otherwise */ + public boolean isSetIncrement() { + return this.increment != null; + } + + public void setIncrementIsSet(boolean value) { + if (!value) { + this.increment = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((ByteBuffer)value); + } + break; + + case INCREMENT: + if (value == null) { + unsetIncrement(); + } else { + setIncrement((TIncrement)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE: + return getTable(); + + case INCREMENT: + return getIncrement(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE: + return isSetTable(); + case INCREMENT: + return isSetIncrement(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof increment_args) + return this.equals((increment_args)that); + return false; + } + + public boolean equals(increment_args that) { + if (that == null) + return false; + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_increment = true && this.isSetIncrement(); + boolean that_present_increment = true && that.isSetIncrement(); + if (this_present_increment || that_present_increment) { + if (!(this_present_increment && that_present_increment)) + return false; + if (!this.increment.equals(that.increment)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(increment_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + increment_args typedOther = (increment_args)other; + + lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIncrement()).compareTo(typedOther.isSetIncrement()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIncrement()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.increment, typedOther.increment); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("increment_args("); + boolean first = true; + + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.table, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("increment:"); + if (this.increment == null) { + sb.append("null"); + } else { + sb.append(this.increment); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (table == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString()); + } + if (increment == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'increment' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class increment_argsStandardSchemeFactory implements SchemeFactory { + public increment_argsStandardScheme getScheme() { + return new increment_argsStandardScheme(); + } + } + + private static class increment_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, increment_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // INCREMENT + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.increment = new TIncrement(); + struct.increment.read(iprot); + struct.setIncrementIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, increment_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.table != null) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeBinary(struct.table); + oprot.writeFieldEnd(); + } + if (struct.increment != null) { + oprot.writeFieldBegin(INCREMENT_FIELD_DESC); + struct.increment.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class increment_argsTupleSchemeFactory implements SchemeFactory { + public increment_argsTupleScheme getScheme() { + return new increment_argsTupleScheme(); + } + } + + private static class increment_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, increment_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.table); + struct.increment.write(oprot); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, increment_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + struct.increment = new TIncrement(); + struct.increment.read(iprot); + struct.setIncrementIsSet(true); + } + } + + } + + public static class increment_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("increment_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new increment_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new increment_resultTupleSchemeFactory()); + } + + public TResult success; // required + public TIOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TResult.class))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(increment_result.class, metaDataMap); + } + + public increment_result() { + } + + public increment_result( + TResult success, + TIOError io) + { + this(); + this.success = success; + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public increment_result(increment_result other) { + if (other.isSetSuccess()) { + this.success = new TResult(other.success); + } + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + } + + public increment_result deepCopy() { + return new increment_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + } + + public TResult getSuccess() { + return this.success; + } + + public increment_result setSuccess(TResult success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public TIOError getIo() { + return this.io; + } + + public increment_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((TResult)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof increment_result) + return this.equals((increment_result)that); + return false; + } + + public boolean equals(increment_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(increment_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + increment_result typedOther = (increment_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("increment_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class increment_resultStandardSchemeFactory implements SchemeFactory { + public increment_resultStandardScheme getScheme() { + return new increment_resultStandardScheme(); + } + } + + private static class increment_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, increment_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.success = new TResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, increment_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + struct.success.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class increment_resultTupleSchemeFactory implements SchemeFactory { + public increment_resultTupleScheme getScheme() { + return new increment_resultTupleScheme(); + } + } + + private static class increment_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, increment_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + struct.success.write(oprot); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, increment_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = new TResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class openScanner_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("openScanner_args"); + + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField SCAN_FIELD_DESC = new org.apache.thrift.protocol.TField("scan", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new openScanner_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new openScanner_argsTupleSchemeFactory()); + } + + /** + * the table to get the Scanner for + */ + public ByteBuffer table; // required + /** + * the scan object to get a Scanner for + */ + public TScan scan; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * the table to get the Scanner for + */ + TABLE((short)1, "table"), + /** + * the scan object to get a Scanner for + */ + SCAN((short)2, "scan"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE + return TABLE; + case 2: // SCAN + return SCAN; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.SCAN, new org.apache.thrift.meta_data.FieldMetaData("scan", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TScan.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(openScanner_args.class, metaDataMap); + } + + public openScanner_args() { + } + + public openScanner_args( + ByteBuffer table, + TScan scan) + { + this(); + this.table = table; + this.scan = scan; + } + + /** + * Performs a deep copy on other. + */ + public openScanner_args(openScanner_args other) { + if (other.isSetTable()) { + this.table = org.apache.thrift.TBaseHelper.copyBinary(other.table); +; + } + if (other.isSetScan()) { + this.scan = new TScan(other.scan); + } + } + + public openScanner_args deepCopy() { + return new openScanner_args(this); + } + + @Override + public void clear() { + this.table = null; + this.scan = null; + } + + /** + * the table to get the Scanner for + */ + public byte[] getTable() { + setTable(org.apache.thrift.TBaseHelper.rightSize(table)); + return table == null ? null : table.array(); + } + + public ByteBuffer bufferForTable() { + return table; + } + + /** + * the table to get the Scanner for + */ + public openScanner_args setTable(byte[] table) { + setTable(table == null ? (ByteBuffer)null : ByteBuffer.wrap(table)); + return this; + } + + public openScanner_args setTable(ByteBuffer table) { + this.table = table; + return this; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + /** + * the scan object to get a Scanner for + */ + public TScan getScan() { + return this.scan; + } + + /** + * the scan object to get a Scanner for + */ + public openScanner_args setScan(TScan scan) { + this.scan = scan; + return this; + } + + public void unsetScan() { + this.scan = null; + } + + /** Returns true if field scan is set (has been assigned a value) and false otherwise */ + public boolean isSetScan() { + return this.scan != null; + } + + public void setScanIsSet(boolean value) { + if (!value) { + this.scan = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((ByteBuffer)value); + } + break; + + case SCAN: + if (value == null) { + unsetScan(); + } else { + setScan((TScan)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE: + return getTable(); + + case SCAN: + return getScan(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE: + return isSetTable(); + case SCAN: + return isSetScan(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof openScanner_args) + return this.equals((openScanner_args)that); + return false; + } + + public boolean equals(openScanner_args that) { + if (that == null) + return false; + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_scan = true && this.isSetScan(); + boolean that_present_scan = true && that.isSetScan(); + if (this_present_scan || that_present_scan) { + if (!(this_present_scan && that_present_scan)) + return false; + if (!this.scan.equals(that.scan)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(openScanner_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + openScanner_args typedOther = (openScanner_args)other; + + lastComparison = Boolean.valueOf(isSetTable()).compareTo(typedOther.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, typedOther.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetScan()).compareTo(typedOther.isSetScan()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetScan()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.scan, typedOther.scan); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("openScanner_args("); + boolean first = true; + + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.table, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("scan:"); + if (this.scan == null) { + sb.append("null"); + } else { + sb.append(this.scan); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (table == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'table' was not present! Struct: " + toString()); + } + if (scan == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'scan' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class openScanner_argsStandardSchemeFactory implements SchemeFactory { + public openScanner_argsStandardScheme getScheme() { + return new openScanner_argsStandardScheme(); + } + } + + private static class openScanner_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, openScanner_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // SCAN + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.scan = new TScan(); + struct.scan.read(iprot); + struct.setScanIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, openScanner_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.table != null) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeBinary(struct.table); + oprot.writeFieldEnd(); + } + if (struct.scan != null) { + oprot.writeFieldBegin(SCAN_FIELD_DESC); + struct.scan.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class openScanner_argsTupleSchemeFactory implements SchemeFactory { + public openScanner_argsTupleScheme getScheme() { + return new openScanner_argsTupleScheme(); + } + } + + private static class openScanner_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, openScanner_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.table); + struct.scan.write(oprot); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, openScanner_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.table = iprot.readBinary(); + struct.setTableIsSet(true); + struct.scan = new TScan(); + struct.scan.read(iprot); + struct.setScanIsSet(true); + } + } + + } + + public static class openScanner_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("openScanner_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.I32, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new openScanner_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new openScanner_resultTupleSchemeFactory()); + } + + public int success; // required + public TIOError io; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(openScanner_result.class, metaDataMap); + } + + public openScanner_result() { + } + + public openScanner_result( + int success, + TIOError io) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.io = io; + } + + /** + * Performs a deep copy on other. + */ + public openScanner_result(openScanner_result other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.success = other.success; + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + } + + public openScanner_result deepCopy() { + return new openScanner_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = 0; + this.io = null; + } + + public int getSuccess() { + return this.success; + } + + public openScanner_result setSuccess(int success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bit_vector.clear(__SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return __isset_bit_vector.get(__SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bit_vector.set(__SUCCESS_ISSET_ID, value); + } + + public TIOError getIo() { + return this.io; + } + + public openScanner_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Integer)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Integer.valueOf(getSuccess()); + + case IO: + return getIo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof openScanner_result) + return this.equals((openScanner_result)that); + return false; + } + + public boolean equals(openScanner_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(openScanner_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + openScanner_result typedOther = (openScanner_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("openScanner_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class openScanner_resultStandardSchemeFactory implements SchemeFactory { + public openScanner_resultStandardScheme getScheme() { + return new openScanner_resultStandardScheme(); + } + } + + private static class openScanner_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, openScanner_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, openScanner_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeI32(struct.success); + oprot.writeFieldEnd(); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class openScanner_resultTupleSchemeFactory implements SchemeFactory { + public openScanner_resultTupleScheme getScheme() { + return new openScanner_resultTupleScheme(); + } + } + + private static class openScanner_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, openScanner_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeI32(struct.success); + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, openScanner_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readI32(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + } + } + + } + + public static class getScannerRows_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getScannerRows_args"); + + private static final org.apache.thrift.protocol.TField SCANNER_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("scannerId", org.apache.thrift.protocol.TType.I32, (short)1); + private static final org.apache.thrift.protocol.TField NUM_ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("numRows", org.apache.thrift.protocol.TType.I32, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getScannerRows_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getScannerRows_argsTupleSchemeFactory()); + } + + /** + * the Id of the Scanner to return rows from. This is an Id returned from the openScanner function. + */ + public int scannerId; // required + /** + * number of rows to return + */ + public int numRows; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * the Id of the Scanner to return rows from. This is an Id returned from the openScanner function. + */ + SCANNER_ID((short)1, "scannerId"), + /** + * number of rows to return + */ + NUM_ROWS((short)2, "numRows"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // SCANNER_ID + return SCANNER_ID; + case 2: // NUM_ROWS + return NUM_ROWS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SCANNERID_ISSET_ID = 0; + private static final int __NUMROWS_ISSET_ID = 1; + private BitSet __isset_bit_vector = new BitSet(2); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SCANNER_ID, new org.apache.thrift.meta_data.FieldMetaData("scannerId", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.NUM_ROWS, new org.apache.thrift.meta_data.FieldMetaData("numRows", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getScannerRows_args.class, metaDataMap); + } + + public getScannerRows_args() { + this.numRows = 1; + + } + + public getScannerRows_args( + int scannerId, + int numRows) + { + this(); + this.scannerId = scannerId; + setScannerIdIsSet(true); + this.numRows = numRows; + setNumRowsIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public getScannerRows_args(getScannerRows_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.scannerId = other.scannerId; + this.numRows = other.numRows; + } + + public getScannerRows_args deepCopy() { + return new getScannerRows_args(this); + } + + @Override + public void clear() { + setScannerIdIsSet(false); + this.scannerId = 0; + this.numRows = 1; + + } + + /** + * the Id of the Scanner to return rows from. This is an Id returned from the openScanner function. + */ + public int getScannerId() { + return this.scannerId; + } + + /** + * the Id of the Scanner to return rows from. This is an Id returned from the openScanner function. + */ + public getScannerRows_args setScannerId(int scannerId) { + this.scannerId = scannerId; + setScannerIdIsSet(true); + return this; + } + + public void unsetScannerId() { + __isset_bit_vector.clear(__SCANNERID_ISSET_ID); + } + + /** Returns true if field scannerId is set (has been assigned a value) and false otherwise */ + public boolean isSetScannerId() { + return __isset_bit_vector.get(__SCANNERID_ISSET_ID); + } + + public void setScannerIdIsSet(boolean value) { + __isset_bit_vector.set(__SCANNERID_ISSET_ID, value); + } + + /** + * number of rows to return + */ + public int getNumRows() { + return this.numRows; + } + + /** + * number of rows to return + */ + public getScannerRows_args setNumRows(int numRows) { + this.numRows = numRows; + setNumRowsIsSet(true); + return this; + } + + public void unsetNumRows() { + __isset_bit_vector.clear(__NUMROWS_ISSET_ID); + } + + /** Returns true if field numRows is set (has been assigned a value) and false otherwise */ + public boolean isSetNumRows() { + return __isset_bit_vector.get(__NUMROWS_ISSET_ID); + } + + public void setNumRowsIsSet(boolean value) { + __isset_bit_vector.set(__NUMROWS_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SCANNER_ID: + if (value == null) { + unsetScannerId(); + } else { + setScannerId((Integer)value); + } + break; + + case NUM_ROWS: + if (value == null) { + unsetNumRows(); + } else { + setNumRows((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SCANNER_ID: + return Integer.valueOf(getScannerId()); + + case NUM_ROWS: + return Integer.valueOf(getNumRows()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SCANNER_ID: + return isSetScannerId(); + case NUM_ROWS: + return isSetNumRows(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getScannerRows_args) + return this.equals((getScannerRows_args)that); + return false; + } + + public boolean equals(getScannerRows_args that) { + if (that == null) + return false; + + boolean this_present_scannerId = true; + boolean that_present_scannerId = true; + if (this_present_scannerId || that_present_scannerId) { + if (!(this_present_scannerId && that_present_scannerId)) + return false; + if (this.scannerId != that.scannerId) + return false; + } + + boolean this_present_numRows = true; + boolean that_present_numRows = true; + if (this_present_numRows || that_present_numRows) { + if (!(this_present_numRows && that_present_numRows)) + return false; + if (this.numRows != that.numRows) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getScannerRows_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getScannerRows_args typedOther = (getScannerRows_args)other; + + lastComparison = Boolean.valueOf(isSetScannerId()).compareTo(typedOther.isSetScannerId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetScannerId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.scannerId, typedOther.scannerId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetNumRows()).compareTo(typedOther.isSetNumRows()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetNumRows()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.numRows, typedOther.numRows); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getScannerRows_args("); + boolean first = true; + + sb.append("scannerId:"); + sb.append(this.scannerId); + first = false; + if (!first) sb.append(", "); + sb.append("numRows:"); + sb.append(this.numRows); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // alas, we cannot check 'scannerId' because it's a primitive and you chose the non-beans generator. + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getScannerRows_argsStandardSchemeFactory implements SchemeFactory { + public getScannerRows_argsStandardScheme getScheme() { + return new getScannerRows_argsStandardScheme(); + } + } + + private static class getScannerRows_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getScannerRows_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // SCANNER_ID + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.scannerId = iprot.readI32(); + struct.setScannerIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // NUM_ROWS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.numRows = iprot.readI32(); + struct.setNumRowsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + if (!struct.isSetScannerId()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'scannerId' was not found in serialized data! Struct: " + toString()); + } + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getScannerRows_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SCANNER_ID_FIELD_DESC); + oprot.writeI32(struct.scannerId); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(NUM_ROWS_FIELD_DESC); + oprot.writeI32(struct.numRows); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getScannerRows_argsTupleSchemeFactory implements SchemeFactory { + public getScannerRows_argsTupleScheme getScheme() { + return new getScannerRows_argsTupleScheme(); + } + } + + private static class getScannerRows_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getScannerRows_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeI32(struct.scannerId); + BitSet optionals = new BitSet(); + if (struct.isSetNumRows()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetNumRows()) { + oprot.writeI32(struct.numRows); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getScannerRows_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.scannerId = iprot.readI32(); + struct.setScannerIdIsSet(true); + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.numRows = iprot.readI32(); + struct.setNumRowsIsSet(true); + } + } + } + + } + + public static class getScannerRows_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getScannerRows_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.LIST, (short)0); + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField IA_FIELD_DESC = new org.apache.thrift.protocol.TField("ia", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getScannerRows_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getScannerRows_resultTupleSchemeFactory()); + } + + public List success; // required + public TIOError io; // required + /** + * if the scannerId is invalid + */ + public TIllegalArgument ia; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + IO((short)1, "io"), + /** + * if the scannerId is invalid + */ + IA((short)2, "ia"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // IO + return IO; + case 2: // IA + return IA; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TResult.class)))); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + tmpMap.put(_Fields.IA, new org.apache.thrift.meta_data.FieldMetaData("ia", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getScannerRows_result.class, metaDataMap); + } + + public getScannerRows_result() { + } + + public getScannerRows_result( + List success, + TIOError io, + TIllegalArgument ia) + { + this(); + this.success = success; + this.io = io; + this.ia = ia; + } + + /** + * Performs a deep copy on other. + */ + public getScannerRows_result(getScannerRows_result other) { + if (other.isSetSuccess()) { + List __this__success = new ArrayList(); + for (TResult other_element : other.success) { + __this__success.add(new TResult(other_element)); + } + this.success = __this__success; + } + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + if (other.isSetIa()) { + this.ia = new TIllegalArgument(other.ia); + } + } + + public getScannerRows_result deepCopy() { + return new getScannerRows_result(this); + } + + @Override + public void clear() { + this.success = null; + this.io = null; + this.ia = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(TResult elem) { + if (this.success == null) { + this.success = new ArrayList(); + } + this.success.add(elem); + } + + public List getSuccess() { + return this.success; + } + + public getScannerRows_result setSuccess(List success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public TIOError getIo() { + return this.io; + } + + public getScannerRows_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + /** + * if the scannerId is invalid + */ + public TIllegalArgument getIa() { + return this.ia; + } + + /** + * if the scannerId is invalid + */ + public getScannerRows_result setIa(TIllegalArgument ia) { + this.ia = ia; + return this; + } + + public void unsetIa() { + this.ia = null; + } + + /** Returns true if field ia is set (has been assigned a value) and false otherwise */ + public boolean isSetIa() { + return this.ia != null; + } + + public void setIaIsSet(boolean value) { + if (!value) { + this.ia = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((List)value); + } + break; + + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + case IA: + if (value == null) { + unsetIa(); + } else { + setIa((TIllegalArgument)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case IO: + return getIo(); + + case IA: + return getIa(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case IO: + return isSetIo(); + case IA: + return isSetIa(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getScannerRows_result) + return this.equals((getScannerRows_result)that); + return false; + } + + public boolean equals(getScannerRows_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + boolean this_present_ia = true && this.isSetIa(); + boolean that_present_ia = true && that.isSetIa(); + if (this_present_ia || that_present_ia) { + if (!(this_present_ia && that_present_ia)) + return false; + if (!this.ia.equals(that.ia)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getScannerRows_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getScannerRows_result typedOther = (getScannerRows_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIa()).compareTo(typedOther.isSetIa()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIa()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ia, typedOther.ia); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getScannerRows_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + if (!first) sb.append(", "); + sb.append("ia:"); + if (this.ia == null) { + sb.append("null"); + } else { + sb.append(this.ia); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getScannerRows_resultStandardSchemeFactory implements SchemeFactory { + public getScannerRows_resultStandardScheme getScheme() { + return new getScannerRows_resultStandardScheme(); + } + } + + private static class getScannerRows_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getScannerRows_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list88 = iprot.readListBegin(); + struct.success = new ArrayList(_list88.size); + for (int _i89 = 0; _i89 < _list88.size; ++_i89) + { + TResult _elem90; // optional + _elem90 = new TResult(); + _elem90.read(iprot); + struct.success.add(_elem90); + } + iprot.readListEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // IA + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.ia = new TIllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getScannerRows_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); + for (TResult _iter91 : struct.success) + { + _iter91.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.ia != null) { + oprot.writeFieldBegin(IA_FIELD_DESC); + struct.ia.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getScannerRows_resultTupleSchemeFactory implements SchemeFactory { + public getScannerRows_resultTupleScheme getScheme() { + return new getScannerRows_resultTupleScheme(); + } + } + + private static class getScannerRows_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getScannerRows_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetIo()) { + optionals.set(1); + } + if (struct.isSetIa()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (TResult _iter92 : struct.success) + { + _iter92.write(oprot); + } + } + } + if (struct.isSetIo()) { + struct.io.write(oprot); + } + if (struct.isSetIa()) { + struct.ia.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getScannerRows_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TList _list93 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new ArrayList(_list93.size); + for (int _i94 = 0; _i94 < _list93.size; ++_i94) + { + TResult _elem95; // optional + _elem95 = new TResult(); + _elem95.read(iprot); + struct.success.add(_elem95); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + if (incoming.get(2)) { + struct.ia = new TIllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } + } + } + + } + + public static class closeScanner_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("closeScanner_args"); + + private static final org.apache.thrift.protocol.TField SCANNER_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("scannerId", org.apache.thrift.protocol.TType.I32, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new closeScanner_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new closeScanner_argsTupleSchemeFactory()); + } + + /** + * the Id of the Scanner to close * + */ + public int scannerId; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * the Id of the Scanner to close * + */ + SCANNER_ID((short)1, "scannerId"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // SCANNER_ID + return SCANNER_ID; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SCANNERID_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SCANNER_ID, new org.apache.thrift.meta_data.FieldMetaData("scannerId", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(closeScanner_args.class, metaDataMap); + } + + public closeScanner_args() { + } + + public closeScanner_args( + int scannerId) + { + this(); + this.scannerId = scannerId; + setScannerIdIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public closeScanner_args(closeScanner_args other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.scannerId = other.scannerId; + } + + public closeScanner_args deepCopy() { + return new closeScanner_args(this); + } + + @Override + public void clear() { + setScannerIdIsSet(false); + this.scannerId = 0; + } + + /** + * the Id of the Scanner to close * + */ + public int getScannerId() { + return this.scannerId; + } + + /** + * the Id of the Scanner to close * + */ + public closeScanner_args setScannerId(int scannerId) { + this.scannerId = scannerId; + setScannerIdIsSet(true); + return this; + } + + public void unsetScannerId() { + __isset_bit_vector.clear(__SCANNERID_ISSET_ID); + } + + /** Returns true if field scannerId is set (has been assigned a value) and false otherwise */ + public boolean isSetScannerId() { + return __isset_bit_vector.get(__SCANNERID_ISSET_ID); + } + + public void setScannerIdIsSet(boolean value) { + __isset_bit_vector.set(__SCANNERID_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SCANNER_ID: + if (value == null) { + unsetScannerId(); + } else { + setScannerId((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SCANNER_ID: + return Integer.valueOf(getScannerId()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SCANNER_ID: + return isSetScannerId(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof closeScanner_args) + return this.equals((closeScanner_args)that); + return false; + } + + public boolean equals(closeScanner_args that) { + if (that == null) + return false; + + boolean this_present_scannerId = true; + boolean that_present_scannerId = true; + if (this_present_scannerId || that_present_scannerId) { + if (!(this_present_scannerId && that_present_scannerId)) + return false; + if (this.scannerId != that.scannerId) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(closeScanner_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + closeScanner_args typedOther = (closeScanner_args)other; + + lastComparison = Boolean.valueOf(isSetScannerId()).compareTo(typedOther.isSetScannerId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetScannerId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.scannerId, typedOther.scannerId); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("closeScanner_args("); + boolean first = true; + + sb.append("scannerId:"); + sb.append(this.scannerId); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // alas, we cannot check 'scannerId' because it's a primitive and you chose the non-beans generator. + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class closeScanner_argsStandardSchemeFactory implements SchemeFactory { + public closeScanner_argsStandardScheme getScheme() { + return new closeScanner_argsStandardScheme(); + } + } + + private static class closeScanner_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, closeScanner_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // SCANNER_ID + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.scannerId = iprot.readI32(); + struct.setScannerIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + if (!struct.isSetScannerId()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'scannerId' was not found in serialized data! Struct: " + toString()); + } + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, closeScanner_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SCANNER_ID_FIELD_DESC); + oprot.writeI32(struct.scannerId); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class closeScanner_argsTupleSchemeFactory implements SchemeFactory { + public closeScanner_argsTupleScheme getScheme() { + return new closeScanner_argsTupleScheme(); + } + } + + private static class closeScanner_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, closeScanner_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeI32(struct.scannerId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, closeScanner_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.scannerId = iprot.readI32(); + struct.setScannerIdIsSet(true); + } + } + + } + + public static class closeScanner_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("closeScanner_result"); + + private static final org.apache.thrift.protocol.TField IO_FIELD_DESC = new org.apache.thrift.protocol.TField("io", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField IA_FIELD_DESC = new org.apache.thrift.protocol.TField("ia", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new closeScanner_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new closeScanner_resultTupleSchemeFactory()); + } + + public TIOError io; // required + /** + * if the scannerId is invalid + */ + public TIllegalArgument ia; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + IO((short)1, "io"), + /** + * if the scannerId is invalid + */ + IA((short)2, "ia"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // IO + return IO; + case 2: // IA + return IA; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.IO, new org.apache.thrift.meta_data.FieldMetaData("io", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + tmpMap.put(_Fields.IA, new org.apache.thrift.meta_data.FieldMetaData("ia", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(closeScanner_result.class, metaDataMap); + } + + public closeScanner_result() { + } + + public closeScanner_result( + TIOError io, + TIllegalArgument ia) + { + this(); + this.io = io; + this.ia = ia; + } + + /** + * Performs a deep copy on other. + */ + public closeScanner_result(closeScanner_result other) { + if (other.isSetIo()) { + this.io = new TIOError(other.io); + } + if (other.isSetIa()) { + this.ia = new TIllegalArgument(other.ia); + } + } + + public closeScanner_result deepCopy() { + return new closeScanner_result(this); + } + + @Override + public void clear() { + this.io = null; + this.ia = null; + } + + public TIOError getIo() { + return this.io; + } + + public closeScanner_result setIo(TIOError io) { + this.io = io; + return this; + } + + public void unsetIo() { + this.io = null; + } + + /** Returns true if field io is set (has been assigned a value) and false otherwise */ + public boolean isSetIo() { + return this.io != null; + } + + public void setIoIsSet(boolean value) { + if (!value) { + this.io = null; + } + } + + /** + * if the scannerId is invalid + */ + public TIllegalArgument getIa() { + return this.ia; + } + + /** + * if the scannerId is invalid + */ + public closeScanner_result setIa(TIllegalArgument ia) { + this.ia = ia; + return this; + } + + public void unsetIa() { + this.ia = null; + } + + /** Returns true if field ia is set (has been assigned a value) and false otherwise */ + public boolean isSetIa() { + return this.ia != null; + } + + public void setIaIsSet(boolean value) { + if (!value) { + this.ia = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case IO: + if (value == null) { + unsetIo(); + } else { + setIo((TIOError)value); + } + break; + + case IA: + if (value == null) { + unsetIa(); + } else { + setIa((TIllegalArgument)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case IO: + return getIo(); + + case IA: + return getIa(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case IO: + return isSetIo(); + case IA: + return isSetIa(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof closeScanner_result) + return this.equals((closeScanner_result)that); + return false; + } + + public boolean equals(closeScanner_result that) { + if (that == null) + return false; + + boolean this_present_io = true && this.isSetIo(); + boolean that_present_io = true && that.isSetIo(); + if (this_present_io || that_present_io) { + if (!(this_present_io && that_present_io)) + return false; + if (!this.io.equals(that.io)) + return false; + } + + boolean this_present_ia = true && this.isSetIa(); + boolean that_present_ia = true && that.isSetIa(); + if (this_present_ia || that_present_ia) { + if (!(this_present_ia && that_present_ia)) + return false; + if (!this.ia.equals(that.ia)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(closeScanner_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + closeScanner_result typedOther = (closeScanner_result)other; + + lastComparison = Boolean.valueOf(isSetIo()).compareTo(typedOther.isSetIo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.io, typedOther.io); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetIa()).compareTo(typedOther.isSetIa()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetIa()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ia, typedOther.ia); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("closeScanner_result("); + boolean first = true; + + sb.append("io:"); + if (this.io == null) { + sb.append("null"); + } else { + sb.append(this.io); + } + first = false; + if (!first) sb.append(", "); + sb.append("ia:"); + if (this.ia == null) { + sb.append("null"); + } else { + sb.append(this.ia); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class closeScanner_resultStandardSchemeFactory implements SchemeFactory { + public closeScanner_resultStandardScheme getScheme() { + return new closeScanner_resultStandardScheme(); + } + } + + private static class closeScanner_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, closeScanner_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // IO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // IA + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.ia = new TIllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, closeScanner_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.io != null) { + oprot.writeFieldBegin(IO_FIELD_DESC); + struct.io.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.ia != null) { + oprot.writeFieldBegin(IA_FIELD_DESC); + struct.ia.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class closeScanner_resultTupleSchemeFactory implements SchemeFactory { + public closeScanner_resultTupleScheme getScheme() { + return new closeScanner_resultTupleScheme(); + } + } + + private static class closeScanner_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, closeScanner_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetIo()) { + optionals.set(0); + } + if (struct.isSetIa()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetIo()) { + struct.io.write(oprot); + } + if (struct.isSetIa()) { + struct.ia.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, closeScanner_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.io = new TIOError(); + struct.io.read(iprot); + struct.setIoIsSet(true); + } + if (incoming.get(1)) { + struct.ia = new TIllegalArgument(); + struct.ia.read(iprot); + struct.setIaIsSet(true); + } + } + } + + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIOError.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIOError.java new file mode 100644 index 0000000..19f7546 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIOError.java @@ -0,0 +1,385 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A TIOError exception signals that an error occurred communicating + * to the HBase master or a HBase region server. Also used to return + * more general HBase error conditions. + */ +public class TIOError extends Exception implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TIOError"); + + private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TIOErrorStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TIOErrorTupleSchemeFactory()); + } + + public String message; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + MESSAGE((short)1, "message"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // MESSAGE + return MESSAGE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private _Fields optionals[] = {_Fields.MESSAGE}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TIOError.class, metaDataMap); + } + + public TIOError() { + } + + /** + * Performs a deep copy on other. + */ + public TIOError(TIOError other) { + if (other.isSetMessage()) { + this.message = other.message; + } + } + + public TIOError deepCopy() { + return new TIOError(this); + } + + @Override + public void clear() { + this.message = null; + } + + public String getMessage() { + return this.message; + } + + public TIOError setMessage(String message) { + this.message = message; + return this; + } + + public void unsetMessage() { + this.message = null; + } + + /** Returns true if field message is set (has been assigned a value) and false otherwise */ + public boolean isSetMessage() { + return this.message != null; + } + + public void setMessageIsSet(boolean value) { + if (!value) { + this.message = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case MESSAGE: + if (value == null) { + unsetMessage(); + } else { + setMessage((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case MESSAGE: + return getMessage(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case MESSAGE: + return isSetMessage(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TIOError) + return this.equals((TIOError)that); + return false; + } + + public boolean equals(TIOError that) { + if (that == null) + return false; + + boolean this_present_message = true && this.isSetMessage(); + boolean that_present_message = true && that.isSetMessage(); + if (this_present_message || that_present_message) { + if (!(this_present_message && that_present_message)) + return false; + if (!this.message.equals(that.message)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TIOError other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TIOError typedOther = (TIOError)other; + + lastComparison = Boolean.valueOf(isSetMessage()).compareTo(typedOther.isSetMessage()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMessage()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.message, typedOther.message); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TIOError("); + boolean first = true; + + if (isSetMessage()) { + sb.append("message:"); + if (this.message == null) { + sb.append("null"); + } else { + sb.append(this.message); + } + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TIOErrorStandardSchemeFactory implements SchemeFactory { + public TIOErrorStandardScheme getScheme() { + return new TIOErrorStandardScheme(); + } + } + + private static class TIOErrorStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TIOError struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // MESSAGE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.message = iprot.readString(); + struct.setMessageIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TIOError struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.message != null) { + if (struct.isSetMessage()) { + oprot.writeFieldBegin(MESSAGE_FIELD_DESC); + oprot.writeString(struct.message); + oprot.writeFieldEnd(); + } + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TIOErrorTupleSchemeFactory implements SchemeFactory { + public TIOErrorTupleScheme getScheme() { + return new TIOErrorTupleScheme(); + } + } + + private static class TIOErrorTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TIOError struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetMessage()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetMessage()) { + oprot.writeString(struct.message); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TIOError struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.message = iprot.readString(); + struct.setMessageIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIllegalArgument.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIllegalArgument.java new file mode 100644 index 0000000..b5395a0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIllegalArgument.java @@ -0,0 +1,384 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A TIllegalArgument exception indicates an illegal or invalid + * argument was passed into a procedure. + */ +public class TIllegalArgument extends Exception implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TIllegalArgument"); + + private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TIllegalArgumentStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TIllegalArgumentTupleSchemeFactory()); + } + + public String message; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + MESSAGE((short)1, "message"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // MESSAGE + return MESSAGE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private _Fields optionals[] = {_Fields.MESSAGE}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TIllegalArgument.class, metaDataMap); + } + + public TIllegalArgument() { + } + + /** + * Performs a deep copy on other. + */ + public TIllegalArgument(TIllegalArgument other) { + if (other.isSetMessage()) { + this.message = other.message; + } + } + + public TIllegalArgument deepCopy() { + return new TIllegalArgument(this); + } + + @Override + public void clear() { + this.message = null; + } + + public String getMessage() { + return this.message; + } + + public TIllegalArgument setMessage(String message) { + this.message = message; + return this; + } + + public void unsetMessage() { + this.message = null; + } + + /** Returns true if field message is set (has been assigned a value) and false otherwise */ + public boolean isSetMessage() { + return this.message != null; + } + + public void setMessageIsSet(boolean value) { + if (!value) { + this.message = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case MESSAGE: + if (value == null) { + unsetMessage(); + } else { + setMessage((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case MESSAGE: + return getMessage(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case MESSAGE: + return isSetMessage(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TIllegalArgument) + return this.equals((TIllegalArgument)that); + return false; + } + + public boolean equals(TIllegalArgument that) { + if (that == null) + return false; + + boolean this_present_message = true && this.isSetMessage(); + boolean that_present_message = true && that.isSetMessage(); + if (this_present_message || that_present_message) { + if (!(this_present_message && that_present_message)) + return false; + if (!this.message.equals(that.message)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TIllegalArgument other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TIllegalArgument typedOther = (TIllegalArgument)other; + + lastComparison = Boolean.valueOf(isSetMessage()).compareTo(typedOther.isSetMessage()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMessage()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.message, typedOther.message); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TIllegalArgument("); + boolean first = true; + + if (isSetMessage()) { + sb.append("message:"); + if (this.message == null) { + sb.append("null"); + } else { + sb.append(this.message); + } + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TIllegalArgumentStandardSchemeFactory implements SchemeFactory { + public TIllegalArgumentStandardScheme getScheme() { + return new TIllegalArgumentStandardScheme(); + } + } + + private static class TIllegalArgumentStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TIllegalArgument struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // MESSAGE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.message = iprot.readString(); + struct.setMessageIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TIllegalArgument struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.message != null) { + if (struct.isSetMessage()) { + oprot.writeFieldBegin(MESSAGE_FIELD_DESC); + oprot.writeString(struct.message); + oprot.writeFieldEnd(); + } + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TIllegalArgumentTupleSchemeFactory implements SchemeFactory { + public TIllegalArgumentTupleScheme getScheme() { + return new TIllegalArgumentTupleScheme(); + } + } + + private static class TIllegalArgumentTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TIllegalArgument struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetMessage()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetMessage()) { + oprot.writeString(struct.message); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TIllegalArgument struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.message = iprot.readString(); + struct.setMessageIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java new file mode 100644 index 0000000..6b23b99 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java @@ -0,0 +1,648 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Used to perform Increment operations for a single row. + * + * You can specify if this Increment should be written + * to the write-ahead Log (WAL) or not. It defaults to true. + */ +public class TIncrement implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TIncrement"); + + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)2); + private static final org.apache.thrift.protocol.TField WRITE_TO_WAL_FIELD_DESC = new org.apache.thrift.protocol.TField("writeToWal", org.apache.thrift.protocol.TType.BOOL, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TIncrementStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TIncrementTupleSchemeFactory()); + } + + public ByteBuffer row; // required + public List columns; // required + public boolean writeToWal; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + ROW((short)1, "row"), + COLUMNS((short)2, "columns"), + WRITE_TO_WAL((short)3, "writeToWal"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // ROW + return ROW; + case 2: // COLUMNS + return COLUMNS; + case 3: // WRITE_TO_WAL + return WRITE_TO_WAL; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __WRITETOWAL_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); + private _Fields optionals[] = {_Fields.WRITE_TO_WAL}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumnIncrement.class)))); + tmpMap.put(_Fields.WRITE_TO_WAL, new org.apache.thrift.meta_data.FieldMetaData("writeToWal", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TIncrement.class, metaDataMap); + } + + public TIncrement() { + this.writeToWal = true; + + } + + public TIncrement( + ByteBuffer row, + List columns) + { + this(); + this.row = row; + this.columns = columns; + } + + /** + * Performs a deep copy on other. + */ + public TIncrement(TIncrement other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetRow()) { + this.row = org.apache.thrift.TBaseHelper.copyBinary(other.row); +; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (TColumnIncrement other_element : other.columns) { + __this__columns.add(new TColumnIncrement(other_element)); + } + this.columns = __this__columns; + } + this.writeToWal = other.writeToWal; + } + + public TIncrement deepCopy() { + return new TIncrement(this); + } + + @Override + public void clear() { + this.row = null; + this.columns = null; + this.writeToWal = true; + + } + + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + public TIncrement setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public TIncrement setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(TColumnIncrement elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + public List getColumns() { + return this.columns; + } + + public TIncrement setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public boolean isWriteToWal() { + return this.writeToWal; + } + + public TIncrement setWriteToWal(boolean writeToWal) { + this.writeToWal = writeToWal; + setWriteToWalIsSet(true); + return this; + } + + public void unsetWriteToWal() { + __isset_bit_vector.clear(__WRITETOWAL_ISSET_ID); + } + + /** Returns true if field writeToWal is set (has been assigned a value) and false otherwise */ + public boolean isSetWriteToWal() { + return __isset_bit_vector.get(__WRITETOWAL_ISSET_ID); + } + + public void setWriteToWalIsSet(boolean value) { + __isset_bit_vector.set(__WRITETOWAL_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case WRITE_TO_WAL: + if (value == null) { + unsetWriteToWal(); + } else { + setWriteToWal((Boolean)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case ROW: + return getRow(); + + case COLUMNS: + return getColumns(); + + case WRITE_TO_WAL: + return Boolean.valueOf(isWriteToWal()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case ROW: + return isSetRow(); + case COLUMNS: + return isSetColumns(); + case WRITE_TO_WAL: + return isSetWriteToWal(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TIncrement) + return this.equals((TIncrement)that); + return false; + } + + public boolean equals(TIncrement that) { + if (that == null) + return false; + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_writeToWal = true && this.isSetWriteToWal(); + boolean that_present_writeToWal = true && that.isSetWriteToWal(); + if (this_present_writeToWal || that_present_writeToWal) { + if (!(this_present_writeToWal && that_present_writeToWal)) + return false; + if (this.writeToWal != that.writeToWal) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TIncrement other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TIncrement typedOther = (TIncrement)other; + + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetWriteToWal()).compareTo(typedOther.isSetWriteToWal()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetWriteToWal()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.writeToWal, typedOther.writeToWal); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TIncrement("); + boolean first = true; + + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.row, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + if (isSetWriteToWal()) { + if (!first) sb.append(", "); + sb.append("writeToWal:"); + sb.append(this.writeToWal); + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (row == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'row' was not present! Struct: " + toString()); + } + if (columns == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'columns' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TIncrementStandardSchemeFactory implements SchemeFactory { + public TIncrementStandardScheme getScheme() { + return new TIncrementStandardScheme(); + } + } + + private static class TIncrementStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TIncrement struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list32 = iprot.readListBegin(); + struct.columns = new ArrayList(_list32.size); + for (int _i33 = 0; _i33 < _list32.size; ++_i33) + { + TColumnIncrement _elem34; // optional + _elem34 = new TColumnIncrement(); + _elem34.read(iprot); + struct.columns.add(_elem34); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // WRITE_TO_WAL + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.writeToWal = iprot.readBool(); + struct.setWriteToWalIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TIncrement struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.columns != null) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size())); + for (TColumnIncrement _iter35 : struct.columns) + { + _iter35.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.isSetWriteToWal()) { + oprot.writeFieldBegin(WRITE_TO_WAL_FIELD_DESC); + oprot.writeBool(struct.writeToWal); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TIncrementTupleSchemeFactory implements SchemeFactory { + public TIncrementTupleScheme getScheme() { + return new TIncrementTupleScheme(); + } + } + + private static class TIncrementTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TIncrement struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.row); + { + oprot.writeI32(struct.columns.size()); + for (TColumnIncrement _iter36 : struct.columns) + { + _iter36.write(oprot); + } + } + BitSet optionals = new BitSet(); + if (struct.isSetWriteToWal()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetWriteToWal()) { + oprot.writeBool(struct.writeToWal); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TIncrement struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + { + org.apache.thrift.protocol.TList _list37 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.columns = new ArrayList(_list37.size); + for (int _i38 = 0; _i38 < _list37.size; ++_i38) + { + TColumnIncrement _elem39; // optional + _elem39 = new TColumnIncrement(); + _elem39.read(iprot); + struct.columns.add(_elem39); + } + } + struct.setColumnsIsSet(true); + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.writeToWal = iprot.readBool(); + struct.setWriteToWalIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java new file mode 100644 index 0000000..a934824 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java @@ -0,0 +1,748 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Used to perform Put operations for a single row. + * + * Add column values to this object and they'll be added. + * You can provide a default timestamp if the column values + * don't have one. If you don't provide a default timestamp + * the current time is inserted. + * + * You can also specify if this Put should be written + * to the write-ahead Log (WAL) or not. It defaults to true. + */ +public class TPut implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TPut"); + + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField COLUMN_VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("columnValues", org.apache.thrift.protocol.TType.LIST, (short)2); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField WRITE_TO_WAL_FIELD_DESC = new org.apache.thrift.protocol.TField("writeToWal", org.apache.thrift.protocol.TType.BOOL, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TPutStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TPutTupleSchemeFactory()); + } + + public ByteBuffer row; // required + public List columnValues; // required + public long timestamp; // optional + public boolean writeToWal; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + ROW((short)1, "row"), + COLUMN_VALUES((short)2, "columnValues"), + TIMESTAMP((short)3, "timestamp"), + WRITE_TO_WAL((short)4, "writeToWal"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // ROW + return ROW; + case 2: // COLUMN_VALUES + return COLUMN_VALUES; + case 3: // TIMESTAMP + return TIMESTAMP; + case 4: // WRITE_TO_WAL + return WRITE_TO_WAL; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIMESTAMP_ISSET_ID = 0; + private static final int __WRITETOWAL_ISSET_ID = 1; + private BitSet __isset_bit_vector = new BitSet(2); + private _Fields optionals[] = {_Fields.TIMESTAMP,_Fields.WRITE_TO_WAL}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.COLUMN_VALUES, new org.apache.thrift.meta_data.FieldMetaData("columnValues", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumnValue.class)))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.WRITE_TO_WAL, new org.apache.thrift.meta_data.FieldMetaData("writeToWal", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TPut.class, metaDataMap); + } + + public TPut() { + this.writeToWal = true; + + } + + public TPut( + ByteBuffer row, + List columnValues) + { + this(); + this.row = row; + this.columnValues = columnValues; + } + + /** + * Performs a deep copy on other. + */ + public TPut(TPut other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetRow()) { + this.row = org.apache.thrift.TBaseHelper.copyBinary(other.row); +; + } + if (other.isSetColumnValues()) { + List __this__columnValues = new ArrayList(); + for (TColumnValue other_element : other.columnValues) { + __this__columnValues.add(new TColumnValue(other_element)); + } + this.columnValues = __this__columnValues; + } + this.timestamp = other.timestamp; + this.writeToWal = other.writeToWal; + } + + public TPut deepCopy() { + return new TPut(this); + } + + @Override + public void clear() { + this.row = null; + this.columnValues = null; + setTimestampIsSet(false); + this.timestamp = 0; + this.writeToWal = true; + + } + + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + public TPut setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public TPut setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getColumnValuesSize() { + return (this.columnValues == null) ? 0 : this.columnValues.size(); + } + + public java.util.Iterator getColumnValuesIterator() { + return (this.columnValues == null) ? null : this.columnValues.iterator(); + } + + public void addToColumnValues(TColumnValue elem) { + if (this.columnValues == null) { + this.columnValues = new ArrayList(); + } + this.columnValues.add(elem); + } + + public List getColumnValues() { + return this.columnValues; + } + + public TPut setColumnValues(List columnValues) { + this.columnValues = columnValues; + return this; + } + + public void unsetColumnValues() { + this.columnValues = null; + } + + /** Returns true if field columnValues is set (has been assigned a value) and false otherwise */ + public boolean isSetColumnValues() { + return this.columnValues != null; + } + + public void setColumnValuesIsSet(boolean value) { + if (!value) { + this.columnValues = null; + } + } + + public long getTimestamp() { + return this.timestamp; + } + + public TPut setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); + } + + public boolean isWriteToWal() { + return this.writeToWal; + } + + public TPut setWriteToWal(boolean writeToWal) { + this.writeToWal = writeToWal; + setWriteToWalIsSet(true); + return this; + } + + public void unsetWriteToWal() { + __isset_bit_vector.clear(__WRITETOWAL_ISSET_ID); + } + + /** Returns true if field writeToWal is set (has been assigned a value) and false otherwise */ + public boolean isSetWriteToWal() { + return __isset_bit_vector.get(__WRITETOWAL_ISSET_ID); + } + + public void setWriteToWalIsSet(boolean value) { + __isset_bit_vector.set(__WRITETOWAL_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMN_VALUES: + if (value == null) { + unsetColumnValues(); + } else { + setColumnValues((List)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((Long)value); + } + break; + + case WRITE_TO_WAL: + if (value == null) { + unsetWriteToWal(); + } else { + setWriteToWal((Boolean)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case ROW: + return getRow(); + + case COLUMN_VALUES: + return getColumnValues(); + + case TIMESTAMP: + return Long.valueOf(getTimestamp()); + + case WRITE_TO_WAL: + return Boolean.valueOf(isWriteToWal()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case ROW: + return isSetRow(); + case COLUMN_VALUES: + return isSetColumnValues(); + case TIMESTAMP: + return isSetTimestamp(); + case WRITE_TO_WAL: + return isSetWriteToWal(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TPut) + return this.equals((TPut)that); + return false; + } + + public boolean equals(TPut that) { + if (that == null) + return false; + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_columnValues = true && this.isSetColumnValues(); + boolean that_present_columnValues = true && that.isSetColumnValues(); + if (this_present_columnValues || that_present_columnValues) { + if (!(this_present_columnValues && that_present_columnValues)) + return false; + if (!this.columnValues.equals(that.columnValues)) + return false; + } + + boolean this_present_timestamp = true && this.isSetTimestamp(); + boolean that_present_timestamp = true && that.isSetTimestamp(); + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + boolean this_present_writeToWal = true && this.isSetWriteToWal(); + boolean that_present_writeToWal = true && that.isSetWriteToWal(); + if (this_present_writeToWal || that_present_writeToWal) { + if (!(this_present_writeToWal && that_present_writeToWal)) + return false; + if (this.writeToWal != that.writeToWal) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TPut other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TPut typedOther = (TPut)other; + + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumnValues()).compareTo(typedOther.isSetColumnValues()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumnValues()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columnValues, typedOther.columnValues); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetWriteToWal()).compareTo(typedOther.isSetWriteToWal()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetWriteToWal()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.writeToWal, typedOther.writeToWal); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TPut("); + boolean first = true; + + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.row, sb); + } + first = false; + if (!first) sb.append(", "); + sb.append("columnValues:"); + if (this.columnValues == null) { + sb.append("null"); + } else { + sb.append(this.columnValues); + } + first = false; + if (isSetTimestamp()) { + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + } + if (isSetWriteToWal()) { + if (!first) sb.append(", "); + sb.append("writeToWal:"); + sb.append(this.writeToWal); + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (row == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'row' was not present! Struct: " + toString()); + } + if (columnValues == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'columnValues' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TPutStandardSchemeFactory implements SchemeFactory { + public TPutStandardScheme getScheme() { + return new TPutStandardScheme(); + } + } + + private static class TPutStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TPut struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // COLUMN_VALUES + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list16 = iprot.readListBegin(); + struct.columnValues = new ArrayList(_list16.size); + for (int _i17 = 0; _i17 < _list16.size; ++_i17) + { + TColumnValue _elem18; // optional + _elem18 = new TColumnValue(); + _elem18.read(iprot); + struct.columnValues.add(_elem18); + } + iprot.readListEnd(); + } + struct.setColumnValuesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // WRITE_TO_WAL + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.writeToWal = iprot.readBool(); + struct.setWriteToWalIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TPut struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.row != null) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + if (struct.columnValues != null) { + oprot.writeFieldBegin(COLUMN_VALUES_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columnValues.size())); + for (TColumnValue _iter19 : struct.columnValues) + { + _iter19.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.isSetTimestamp()) { + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + } + if (struct.isSetWriteToWal()) { + oprot.writeFieldBegin(WRITE_TO_WAL_FIELD_DESC); + oprot.writeBool(struct.writeToWal); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TPutTupleSchemeFactory implements SchemeFactory { + public TPutTupleScheme getScheme() { + return new TPutTupleScheme(); + } + } + + private static class TPutTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TPut struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeBinary(struct.row); + { + oprot.writeI32(struct.columnValues.size()); + for (TColumnValue _iter20 : struct.columnValues) + { + _iter20.write(oprot); + } + } + BitSet optionals = new BitSet(); + if (struct.isSetTimestamp()) { + optionals.set(0); + } + if (struct.isSetWriteToWal()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + if (struct.isSetWriteToWal()) { + oprot.writeBool(struct.writeToWal); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TPut struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + { + org.apache.thrift.protocol.TList _list21 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.columnValues = new ArrayList(_list21.size); + for (int _i22 = 0; _i22 < _list21.size; ++_i22) + { + TColumnValue _elem23; // optional + _elem23 = new TColumnValue(); + _elem23.read(iprot); + struct.columnValues.add(_elem23); + } + } + struct.setColumnValuesIsSet(true); + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + if (incoming.get(1)) { + struct.writeToWal = iprot.readBool(); + struct.setWriteToWalIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java new file mode 100644 index 0000000..f0fe2a0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java @@ -0,0 +1,549 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * if no Result is found, row and columnValues will not be set. + */ +public class TResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TResult"); + + private static final org.apache.thrift.protocol.TField ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("row", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField COLUMN_VALUES_FIELD_DESC = new org.apache.thrift.protocol.TField("columnValues", org.apache.thrift.protocol.TType.LIST, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TResultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TResultTupleSchemeFactory()); + } + + public ByteBuffer row; // optional + public List columnValues; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + ROW((short)1, "row"), + COLUMN_VALUES((short)2, "columnValues"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // ROW + return ROW; + case 2: // COLUMN_VALUES + return COLUMN_VALUES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private _Fields optionals[] = {_Fields.ROW}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.ROW, new org.apache.thrift.meta_data.FieldMetaData("row", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.COLUMN_VALUES, new org.apache.thrift.meta_data.FieldMetaData("columnValues", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumnValue.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TResult.class, metaDataMap); + } + + public TResult() { + } + + public TResult( + List columnValues) + { + this(); + this.columnValues = columnValues; + } + + /** + * Performs a deep copy on other. + */ + public TResult(TResult other) { + if (other.isSetRow()) { + this.row = org.apache.thrift.TBaseHelper.copyBinary(other.row); +; + } + if (other.isSetColumnValues()) { + List __this__columnValues = new ArrayList(); + for (TColumnValue other_element : other.columnValues) { + __this__columnValues.add(new TColumnValue(other_element)); + } + this.columnValues = __this__columnValues; + } + } + + public TResult deepCopy() { + return new TResult(this); + } + + @Override + public void clear() { + this.row = null; + this.columnValues = null; + } + + public byte[] getRow() { + setRow(org.apache.thrift.TBaseHelper.rightSize(row)); + return row == null ? null : row.array(); + } + + public ByteBuffer bufferForRow() { + return row; + } + + public TResult setRow(byte[] row) { + setRow(row == null ? (ByteBuffer)null : ByteBuffer.wrap(row)); + return this; + } + + public TResult setRow(ByteBuffer row) { + this.row = row; + return this; + } + + public void unsetRow() { + this.row = null; + } + + /** Returns true if field row is set (has been assigned a value) and false otherwise */ + public boolean isSetRow() { + return this.row != null; + } + + public void setRowIsSet(boolean value) { + if (!value) { + this.row = null; + } + } + + public int getColumnValuesSize() { + return (this.columnValues == null) ? 0 : this.columnValues.size(); + } + + public java.util.Iterator getColumnValuesIterator() { + return (this.columnValues == null) ? null : this.columnValues.iterator(); + } + + public void addToColumnValues(TColumnValue elem) { + if (this.columnValues == null) { + this.columnValues = new ArrayList(); + } + this.columnValues.add(elem); + } + + public List getColumnValues() { + return this.columnValues; + } + + public TResult setColumnValues(List columnValues) { + this.columnValues = columnValues; + return this; + } + + public void unsetColumnValues() { + this.columnValues = null; + } + + /** Returns true if field columnValues is set (has been assigned a value) and false otherwise */ + public boolean isSetColumnValues() { + return this.columnValues != null; + } + + public void setColumnValuesIsSet(boolean value) { + if (!value) { + this.columnValues = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case ROW: + if (value == null) { + unsetRow(); + } else { + setRow((ByteBuffer)value); + } + break; + + case COLUMN_VALUES: + if (value == null) { + unsetColumnValues(); + } else { + setColumnValues((List)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case ROW: + return getRow(); + + case COLUMN_VALUES: + return getColumnValues(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case ROW: + return isSetRow(); + case COLUMN_VALUES: + return isSetColumnValues(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TResult) + return this.equals((TResult)that); + return false; + } + + public boolean equals(TResult that) { + if (that == null) + return false; + + boolean this_present_row = true && this.isSetRow(); + boolean that_present_row = true && that.isSetRow(); + if (this_present_row || that_present_row) { + if (!(this_present_row && that_present_row)) + return false; + if (!this.row.equals(that.row)) + return false; + } + + boolean this_present_columnValues = true && this.isSetColumnValues(); + boolean that_present_columnValues = true && that.isSetColumnValues(); + if (this_present_columnValues || that_present_columnValues) { + if (!(this_present_columnValues && that_present_columnValues)) + return false; + if (!this.columnValues.equals(that.columnValues)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TResult other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TResult typedOther = (TResult)other; + + lastComparison = Boolean.valueOf(isSetRow()).compareTo(typedOther.isSetRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.row, typedOther.row); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumnValues()).compareTo(typedOther.isSetColumnValues()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumnValues()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columnValues, typedOther.columnValues); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TResult("); + boolean first = true; + + if (isSetRow()) { + sb.append("row:"); + if (this.row == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.row, sb); + } + first = false; + } + if (!first) sb.append(", "); + sb.append("columnValues:"); + if (this.columnValues == null) { + sb.append("null"); + } else { + sb.append(this.columnValues); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (columnValues == null) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'columnValues' was not present! Struct: " + toString()); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TResultStandardSchemeFactory implements SchemeFactory { + public TResultStandardScheme getScheme() { + return new TResultStandardScheme(); + } + } + + private static class TResultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TResult struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // COLUMN_VALUES + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list0 = iprot.readListBegin(); + struct.columnValues = new ArrayList(_list0.size); + for (int _i1 = 0; _i1 < _list0.size; ++_i1) + { + TColumnValue _elem2; // optional + _elem2 = new TColumnValue(); + _elem2.read(iprot); + struct.columnValues.add(_elem2); + } + iprot.readListEnd(); + } + struct.setColumnValuesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TResult struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.row != null) { + if (struct.isSetRow()) { + oprot.writeFieldBegin(ROW_FIELD_DESC); + oprot.writeBinary(struct.row); + oprot.writeFieldEnd(); + } + } + if (struct.columnValues != null) { + oprot.writeFieldBegin(COLUMN_VALUES_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columnValues.size())); + for (TColumnValue _iter3 : struct.columnValues) + { + _iter3.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TResultTupleSchemeFactory implements SchemeFactory { + public TResultTupleScheme getScheme() { + return new TResultTupleScheme(); + } + } + + private static class TResultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TResult struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + { + oprot.writeI32(struct.columnValues.size()); + for (TColumnValue _iter4 : struct.columnValues) + { + _iter4.write(oprot); + } + } + BitSet optionals = new BitSet(); + if (struct.isSetRow()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetRow()) { + oprot.writeBinary(struct.row); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TResult struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + { + org.apache.thrift.protocol.TList _list5 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.columnValues = new ArrayList(_list5.size); + for (int _i6 = 0; _i6 < _list5.size; ++_i6) + { + TColumnValue _elem7; // optional + _elem7 = new TColumnValue(); + _elem7.read(iprot); + struct.columnValues.add(_elem7); + } + } + struct.setColumnValuesIsSet(true); + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.row = iprot.readBinary(); + struct.setRowIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java new file mode 100644 index 0000000..cbcdf91 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java @@ -0,0 +1,965 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Any timestamps in the columns are ignored, use timeRange to select by timestamp. + * Max versions defaults to 1. + */ +public class TScan implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TScan"); + + private static final org.apache.thrift.protocol.TField START_ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("startRow", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField STOP_ROW_FIELD_DESC = new org.apache.thrift.protocol.TField("stopRow", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField COLUMNS_FIELD_DESC = new org.apache.thrift.protocol.TField("columns", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField CACHING_FIELD_DESC = new org.apache.thrift.protocol.TField("caching", org.apache.thrift.protocol.TType.I32, (short)4); + private static final org.apache.thrift.protocol.TField MAX_VERSIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("maxVersions", org.apache.thrift.protocol.TType.I32, (short)5); + private static final org.apache.thrift.protocol.TField TIME_RANGE_FIELD_DESC = new org.apache.thrift.protocol.TField("timeRange", org.apache.thrift.protocol.TType.STRUCT, (short)6); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TScanStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TScanTupleSchemeFactory()); + } + + public ByteBuffer startRow; // optional + public ByteBuffer stopRow; // optional + public List columns; // optional + public int caching; // optional + public int maxVersions; // optional + public TTimeRange timeRange; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + START_ROW((short)1, "startRow"), + STOP_ROW((short)2, "stopRow"), + COLUMNS((short)3, "columns"), + CACHING((short)4, "caching"), + MAX_VERSIONS((short)5, "maxVersions"), + TIME_RANGE((short)6, "timeRange"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // START_ROW + return START_ROW; + case 2: // STOP_ROW + return STOP_ROW; + case 3: // COLUMNS + return COLUMNS; + case 4: // CACHING + return CACHING; + case 5: // MAX_VERSIONS + return MAX_VERSIONS; + case 6: // TIME_RANGE + return TIME_RANGE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __CACHING_ISSET_ID = 0; + private static final int __MAXVERSIONS_ISSET_ID = 1; + private BitSet __isset_bit_vector = new BitSet(2); + private _Fields optionals[] = {_Fields.START_ROW,_Fields.STOP_ROW,_Fields.COLUMNS,_Fields.CACHING,_Fields.MAX_VERSIONS,_Fields.TIME_RANGE}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.START_ROW, new org.apache.thrift.meta_data.FieldMetaData("startRow", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.STOP_ROW, new org.apache.thrift.meta_data.FieldMetaData("stopRow", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.COLUMNS, new org.apache.thrift.meta_data.FieldMetaData("columns", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TColumn.class)))); + tmpMap.put(_Fields.CACHING, new org.apache.thrift.meta_data.FieldMetaData("caching", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.MAX_VERSIONS, new org.apache.thrift.meta_data.FieldMetaData("maxVersions", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.TIME_RANGE, new org.apache.thrift.meta_data.FieldMetaData("timeRange", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTimeRange.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TScan.class, metaDataMap); + } + + public TScan() { + this.maxVersions = 1; + + } + + /** + * Performs a deep copy on other. + */ + public TScan(TScan other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + if (other.isSetStartRow()) { + this.startRow = org.apache.thrift.TBaseHelper.copyBinary(other.startRow); +; + } + if (other.isSetStopRow()) { + this.stopRow = org.apache.thrift.TBaseHelper.copyBinary(other.stopRow); +; + } + if (other.isSetColumns()) { + List __this__columns = new ArrayList(); + for (TColumn other_element : other.columns) { + __this__columns.add(new TColumn(other_element)); + } + this.columns = __this__columns; + } + this.caching = other.caching; + this.maxVersions = other.maxVersions; + if (other.isSetTimeRange()) { + this.timeRange = new TTimeRange(other.timeRange); + } + } + + public TScan deepCopy() { + return new TScan(this); + } + + @Override + public void clear() { + this.startRow = null; + this.stopRow = null; + this.columns = null; + setCachingIsSet(false); + this.caching = 0; + this.maxVersions = 1; + + this.timeRange = null; + } + + public byte[] getStartRow() { + setStartRow(org.apache.thrift.TBaseHelper.rightSize(startRow)); + return startRow == null ? null : startRow.array(); + } + + public ByteBuffer bufferForStartRow() { + return startRow; + } + + public TScan setStartRow(byte[] startRow) { + setStartRow(startRow == null ? (ByteBuffer)null : ByteBuffer.wrap(startRow)); + return this; + } + + public TScan setStartRow(ByteBuffer startRow) { + this.startRow = startRow; + return this; + } + + public void unsetStartRow() { + this.startRow = null; + } + + /** Returns true if field startRow is set (has been assigned a value) and false otherwise */ + public boolean isSetStartRow() { + return this.startRow != null; + } + + public void setStartRowIsSet(boolean value) { + if (!value) { + this.startRow = null; + } + } + + public byte[] getStopRow() { + setStopRow(org.apache.thrift.TBaseHelper.rightSize(stopRow)); + return stopRow == null ? null : stopRow.array(); + } + + public ByteBuffer bufferForStopRow() { + return stopRow; + } + + public TScan setStopRow(byte[] stopRow) { + setStopRow(stopRow == null ? (ByteBuffer)null : ByteBuffer.wrap(stopRow)); + return this; + } + + public TScan setStopRow(ByteBuffer stopRow) { + this.stopRow = stopRow; + return this; + } + + public void unsetStopRow() { + this.stopRow = null; + } + + /** Returns true if field stopRow is set (has been assigned a value) and false otherwise */ + public boolean isSetStopRow() { + return this.stopRow != null; + } + + public void setStopRowIsSet(boolean value) { + if (!value) { + this.stopRow = null; + } + } + + public int getColumnsSize() { + return (this.columns == null) ? 0 : this.columns.size(); + } + + public java.util.Iterator getColumnsIterator() { + return (this.columns == null) ? null : this.columns.iterator(); + } + + public void addToColumns(TColumn elem) { + if (this.columns == null) { + this.columns = new ArrayList(); + } + this.columns.add(elem); + } + + public List getColumns() { + return this.columns; + } + + public TScan setColumns(List columns) { + this.columns = columns; + return this; + } + + public void unsetColumns() { + this.columns = null; + } + + /** Returns true if field columns is set (has been assigned a value) and false otherwise */ + public boolean isSetColumns() { + return this.columns != null; + } + + public void setColumnsIsSet(boolean value) { + if (!value) { + this.columns = null; + } + } + + public int getCaching() { + return this.caching; + } + + public TScan setCaching(int caching) { + this.caching = caching; + setCachingIsSet(true); + return this; + } + + public void unsetCaching() { + __isset_bit_vector.clear(__CACHING_ISSET_ID); + } + + /** Returns true if field caching is set (has been assigned a value) and false otherwise */ + public boolean isSetCaching() { + return __isset_bit_vector.get(__CACHING_ISSET_ID); + } + + public void setCachingIsSet(boolean value) { + __isset_bit_vector.set(__CACHING_ISSET_ID, value); + } + + public int getMaxVersions() { + return this.maxVersions; + } + + public TScan setMaxVersions(int maxVersions) { + this.maxVersions = maxVersions; + setMaxVersionsIsSet(true); + return this; + } + + public void unsetMaxVersions() { + __isset_bit_vector.clear(__MAXVERSIONS_ISSET_ID); + } + + /** Returns true if field maxVersions is set (has been assigned a value) and false otherwise */ + public boolean isSetMaxVersions() { + return __isset_bit_vector.get(__MAXVERSIONS_ISSET_ID); + } + + public void setMaxVersionsIsSet(boolean value) { + __isset_bit_vector.set(__MAXVERSIONS_ISSET_ID, value); + } + + public TTimeRange getTimeRange() { + return this.timeRange; + } + + public TScan setTimeRange(TTimeRange timeRange) { + this.timeRange = timeRange; + return this; + } + + public void unsetTimeRange() { + this.timeRange = null; + } + + /** Returns true if field timeRange is set (has been assigned a value) and false otherwise */ + public boolean isSetTimeRange() { + return this.timeRange != null; + } + + public void setTimeRangeIsSet(boolean value) { + if (!value) { + this.timeRange = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case START_ROW: + if (value == null) { + unsetStartRow(); + } else { + setStartRow((ByteBuffer)value); + } + break; + + case STOP_ROW: + if (value == null) { + unsetStopRow(); + } else { + setStopRow((ByteBuffer)value); + } + break; + + case COLUMNS: + if (value == null) { + unsetColumns(); + } else { + setColumns((List)value); + } + break; + + case CACHING: + if (value == null) { + unsetCaching(); + } else { + setCaching((Integer)value); + } + break; + + case MAX_VERSIONS: + if (value == null) { + unsetMaxVersions(); + } else { + setMaxVersions((Integer)value); + } + break; + + case TIME_RANGE: + if (value == null) { + unsetTimeRange(); + } else { + setTimeRange((TTimeRange)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case START_ROW: + return getStartRow(); + + case STOP_ROW: + return getStopRow(); + + case COLUMNS: + return getColumns(); + + case CACHING: + return Integer.valueOf(getCaching()); + + case MAX_VERSIONS: + return Integer.valueOf(getMaxVersions()); + + case TIME_RANGE: + return getTimeRange(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case START_ROW: + return isSetStartRow(); + case STOP_ROW: + return isSetStopRow(); + case COLUMNS: + return isSetColumns(); + case CACHING: + return isSetCaching(); + case MAX_VERSIONS: + return isSetMaxVersions(); + case TIME_RANGE: + return isSetTimeRange(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TScan) + return this.equals((TScan)that); + return false; + } + + public boolean equals(TScan that) { + if (that == null) + return false; + + boolean this_present_startRow = true && this.isSetStartRow(); + boolean that_present_startRow = true && that.isSetStartRow(); + if (this_present_startRow || that_present_startRow) { + if (!(this_present_startRow && that_present_startRow)) + return false; + if (!this.startRow.equals(that.startRow)) + return false; + } + + boolean this_present_stopRow = true && this.isSetStopRow(); + boolean that_present_stopRow = true && that.isSetStopRow(); + if (this_present_stopRow || that_present_stopRow) { + if (!(this_present_stopRow && that_present_stopRow)) + return false; + if (!this.stopRow.equals(that.stopRow)) + return false; + } + + boolean this_present_columns = true && this.isSetColumns(); + boolean that_present_columns = true && that.isSetColumns(); + if (this_present_columns || that_present_columns) { + if (!(this_present_columns && that_present_columns)) + return false; + if (!this.columns.equals(that.columns)) + return false; + } + + boolean this_present_caching = true && this.isSetCaching(); + boolean that_present_caching = true && that.isSetCaching(); + if (this_present_caching || that_present_caching) { + if (!(this_present_caching && that_present_caching)) + return false; + if (this.caching != that.caching) + return false; + } + + boolean this_present_maxVersions = true && this.isSetMaxVersions(); + boolean that_present_maxVersions = true && that.isSetMaxVersions(); + if (this_present_maxVersions || that_present_maxVersions) { + if (!(this_present_maxVersions && that_present_maxVersions)) + return false; + if (this.maxVersions != that.maxVersions) + return false; + } + + boolean this_present_timeRange = true && this.isSetTimeRange(); + boolean that_present_timeRange = true && that.isSetTimeRange(); + if (this_present_timeRange || that_present_timeRange) { + if (!(this_present_timeRange && that_present_timeRange)) + return false; + if (!this.timeRange.equals(that.timeRange)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TScan other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TScan typedOther = (TScan)other; + + lastComparison = Boolean.valueOf(isSetStartRow()).compareTo(typedOther.isSetStartRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStartRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.startRow, typedOther.startRow); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetStopRow()).compareTo(typedOther.isSetStopRow()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetStopRow()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.stopRow, typedOther.stopRow); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetColumns()).compareTo(typedOther.isSetColumns()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetColumns()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.columns, typedOther.columns); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetCaching()).compareTo(typedOther.isSetCaching()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCaching()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.caching, typedOther.caching); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMaxVersions()).compareTo(typedOther.isSetMaxVersions()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMaxVersions()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.maxVersions, typedOther.maxVersions); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTimeRange()).compareTo(typedOther.isSetTimeRange()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimeRange()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timeRange, typedOther.timeRange); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TScan("); + boolean first = true; + + if (isSetStartRow()) { + sb.append("startRow:"); + if (this.startRow == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.startRow, sb); + } + first = false; + } + if (isSetStopRow()) { + if (!first) sb.append(", "); + sb.append("stopRow:"); + if (this.stopRow == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.stopRow, sb); + } + first = false; + } + if (isSetColumns()) { + if (!first) sb.append(", "); + sb.append("columns:"); + if (this.columns == null) { + sb.append("null"); + } else { + sb.append(this.columns); + } + first = false; + } + if (isSetCaching()) { + if (!first) sb.append(", "); + sb.append("caching:"); + sb.append(this.caching); + first = false; + } + if (isSetMaxVersions()) { + if (!first) sb.append(", "); + sb.append("maxVersions:"); + sb.append(this.maxVersions); + first = false; + } + if (isSetTimeRange()) { + if (!first) sb.append(", "); + sb.append("timeRange:"); + if (this.timeRange == null) { + sb.append("null"); + } else { + sb.append(this.timeRange); + } + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TScanStandardSchemeFactory implements SchemeFactory { + public TScanStandardScheme getScheme() { + return new TScanStandardScheme(); + } + } + + private static class TScanStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TScan struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // START_ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.startRow = iprot.readBinary(); + struct.setStartRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // STOP_ROW + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.stopRow = iprot.readBinary(); + struct.setStopRowIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // COLUMNS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list40 = iprot.readListBegin(); + struct.columns = new ArrayList(_list40.size); + for (int _i41 = 0; _i41 < _list40.size; ++_i41) + { + TColumn _elem42; // optional + _elem42 = new TColumn(); + _elem42.read(iprot); + struct.columns.add(_elem42); + } + iprot.readListEnd(); + } + struct.setColumnsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // CACHING + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.caching = iprot.readI32(); + struct.setCachingIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // MAX_VERSIONS + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.maxVersions = iprot.readI32(); + struct.setMaxVersionsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 6: // TIME_RANGE + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.timeRange = new TTimeRange(); + struct.timeRange.read(iprot); + struct.setTimeRangeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TScan struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.startRow != null) { + if (struct.isSetStartRow()) { + oprot.writeFieldBegin(START_ROW_FIELD_DESC); + oprot.writeBinary(struct.startRow); + oprot.writeFieldEnd(); + } + } + if (struct.stopRow != null) { + if (struct.isSetStopRow()) { + oprot.writeFieldBegin(STOP_ROW_FIELD_DESC); + oprot.writeBinary(struct.stopRow); + oprot.writeFieldEnd(); + } + } + if (struct.columns != null) { + if (struct.isSetColumns()) { + oprot.writeFieldBegin(COLUMNS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size())); + for (TColumn _iter43 : struct.columns) + { + _iter43.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.isSetCaching()) { + oprot.writeFieldBegin(CACHING_FIELD_DESC); + oprot.writeI32(struct.caching); + oprot.writeFieldEnd(); + } + if (struct.isSetMaxVersions()) { + oprot.writeFieldBegin(MAX_VERSIONS_FIELD_DESC); + oprot.writeI32(struct.maxVersions); + oprot.writeFieldEnd(); + } + if (struct.timeRange != null) { + if (struct.isSetTimeRange()) { + oprot.writeFieldBegin(TIME_RANGE_FIELD_DESC); + struct.timeRange.write(oprot); + oprot.writeFieldEnd(); + } + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TScanTupleSchemeFactory implements SchemeFactory { + public TScanTupleScheme getScheme() { + return new TScanTupleScheme(); + } + } + + private static class TScanTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetStartRow()) { + optionals.set(0); + } + if (struct.isSetStopRow()) { + optionals.set(1); + } + if (struct.isSetColumns()) { + optionals.set(2); + } + if (struct.isSetCaching()) { + optionals.set(3); + } + if (struct.isSetMaxVersions()) { + optionals.set(4); + } + if (struct.isSetTimeRange()) { + optionals.set(5); + } + oprot.writeBitSet(optionals, 6); + if (struct.isSetStartRow()) { + oprot.writeBinary(struct.startRow); + } + if (struct.isSetStopRow()) { + oprot.writeBinary(struct.stopRow); + } + if (struct.isSetColumns()) { + { + oprot.writeI32(struct.columns.size()); + for (TColumn _iter44 : struct.columns) + { + _iter44.write(oprot); + } + } + } + if (struct.isSetCaching()) { + oprot.writeI32(struct.caching); + } + if (struct.isSetMaxVersions()) { + oprot.writeI32(struct.maxVersions); + } + if (struct.isSetTimeRange()) { + struct.timeRange.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(6); + if (incoming.get(0)) { + struct.startRow = iprot.readBinary(); + struct.setStartRowIsSet(true); + } + if (incoming.get(1)) { + struct.stopRow = iprot.readBinary(); + struct.setStopRowIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list45 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.columns = new ArrayList(_list45.size); + for (int _i46 = 0; _i46 < _list45.size; ++_i46) + { + TColumn _elem47; // optional + _elem47 = new TColumn(); + _elem47.read(iprot); + struct.columns.add(_elem47); + } + } + struct.setColumnsIsSet(true); + } + if (incoming.get(3)) { + struct.caching = iprot.readI32(); + struct.setCachingIsSet(true); + } + if (incoming.get(4)) { + struct.maxVersions = iprot.readI32(); + struct.setMaxVersionsIsSet(true); + } + if (incoming.get(5)) { + struct.timeRange = new TTimeRange(); + struct.timeRange.read(iprot); + struct.setTimeRangeIsSet(true); + } + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTimeRange.java b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTimeRange.java new file mode 100644 index 0000000..98058ea --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTimeRange.java @@ -0,0 +1,466 @@ +/** + * Autogenerated by Thrift Compiler (0.8.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TTimeRange implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTimeRange"); + + private static final org.apache.thrift.protocol.TField MIN_STAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("minStamp", org.apache.thrift.protocol.TType.I64, (short)1); + private static final org.apache.thrift.protocol.TField MAX_STAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("maxStamp", org.apache.thrift.protocol.TType.I64, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TTimeRangeStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TTimeRangeTupleSchemeFactory()); + } + + public long minStamp; // required + public long maxStamp; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + MIN_STAMP((short)1, "minStamp"), + MAX_STAMP((short)2, "maxStamp"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // MIN_STAMP + return MIN_STAMP; + case 2: // MAX_STAMP + return MAX_STAMP; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __MINSTAMP_ISSET_ID = 0; + private static final int __MAXSTAMP_ISSET_ID = 1; + private BitSet __isset_bit_vector = new BitSet(2); + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.MIN_STAMP, new org.apache.thrift.meta_data.FieldMetaData("minStamp", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.MAX_STAMP, new org.apache.thrift.meta_data.FieldMetaData("maxStamp", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTimeRange.class, metaDataMap); + } + + public TTimeRange() { + } + + public TTimeRange( + long minStamp, + long maxStamp) + { + this(); + this.minStamp = minStamp; + setMinStampIsSet(true); + this.maxStamp = maxStamp; + setMaxStampIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public TTimeRange(TTimeRange other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); + this.minStamp = other.minStamp; + this.maxStamp = other.maxStamp; + } + + public TTimeRange deepCopy() { + return new TTimeRange(this); + } + + @Override + public void clear() { + setMinStampIsSet(false); + this.minStamp = 0; + setMaxStampIsSet(false); + this.maxStamp = 0; + } + + public long getMinStamp() { + return this.minStamp; + } + + public TTimeRange setMinStamp(long minStamp) { + this.minStamp = minStamp; + setMinStampIsSet(true); + return this; + } + + public void unsetMinStamp() { + __isset_bit_vector.clear(__MINSTAMP_ISSET_ID); + } + + /** Returns true if field minStamp is set (has been assigned a value) and false otherwise */ + public boolean isSetMinStamp() { + return __isset_bit_vector.get(__MINSTAMP_ISSET_ID); + } + + public void setMinStampIsSet(boolean value) { + __isset_bit_vector.set(__MINSTAMP_ISSET_ID, value); + } + + public long getMaxStamp() { + return this.maxStamp; + } + + public TTimeRange setMaxStamp(long maxStamp) { + this.maxStamp = maxStamp; + setMaxStampIsSet(true); + return this; + } + + public void unsetMaxStamp() { + __isset_bit_vector.clear(__MAXSTAMP_ISSET_ID); + } + + /** Returns true if field maxStamp is set (has been assigned a value) and false otherwise */ + public boolean isSetMaxStamp() { + return __isset_bit_vector.get(__MAXSTAMP_ISSET_ID); + } + + public void setMaxStampIsSet(boolean value) { + __isset_bit_vector.set(__MAXSTAMP_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case MIN_STAMP: + if (value == null) { + unsetMinStamp(); + } else { + setMinStamp((Long)value); + } + break; + + case MAX_STAMP: + if (value == null) { + unsetMaxStamp(); + } else { + setMaxStamp((Long)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case MIN_STAMP: + return Long.valueOf(getMinStamp()); + + case MAX_STAMP: + return Long.valueOf(getMaxStamp()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case MIN_STAMP: + return isSetMinStamp(); + case MAX_STAMP: + return isSetMaxStamp(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TTimeRange) + return this.equals((TTimeRange)that); + return false; + } + + public boolean equals(TTimeRange that) { + if (that == null) + return false; + + boolean this_present_minStamp = true; + boolean that_present_minStamp = true; + if (this_present_minStamp || that_present_minStamp) { + if (!(this_present_minStamp && that_present_minStamp)) + return false; + if (this.minStamp != that.minStamp) + return false; + } + + boolean this_present_maxStamp = true; + boolean that_present_maxStamp = true; + if (this_present_maxStamp || that_present_maxStamp) { + if (!(this_present_maxStamp && that_present_maxStamp)) + return false; + if (this.maxStamp != that.maxStamp) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(TTimeRange other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TTimeRange typedOther = (TTimeRange)other; + + lastComparison = Boolean.valueOf(isSetMinStamp()).compareTo(typedOther.isSetMinStamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMinStamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.minStamp, typedOther.minStamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMaxStamp()).compareTo(typedOther.isSetMaxStamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMaxStamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.maxStamp, typedOther.maxStamp); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TTimeRange("); + boolean first = true; + + sb.append("minStamp:"); + sb.append(this.minStamp); + first = false; + if (!first) sb.append(", "); + sb.append("maxStamp:"); + sb.append(this.maxStamp); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // alas, we cannot check 'minStamp' because it's a primitive and you chose the non-beans generator. + // alas, we cannot check 'maxStamp' because it's a primitive and you chose the non-beans generator. + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TTimeRangeStandardSchemeFactory implements SchemeFactory { + public TTimeRangeStandardScheme getScheme() { + return new TTimeRangeStandardScheme(); + } + } + + private static class TTimeRangeStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TTimeRange struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // MIN_STAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.minStamp = iprot.readI64(); + struct.setMinStampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // MAX_STAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.maxStamp = iprot.readI64(); + struct.setMaxStampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + if (!struct.isSetMinStamp()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'minStamp' was not found in serialized data! Struct: " + toString()); + } + if (!struct.isSetMaxStamp()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'maxStamp' was not found in serialized data! Struct: " + toString()); + } + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TTimeRange struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(MIN_STAMP_FIELD_DESC); + oprot.writeI64(struct.minStamp); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(MAX_STAMP_FIELD_DESC); + oprot.writeI64(struct.maxStamp); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TTimeRangeTupleSchemeFactory implements SchemeFactory { + public TTimeRangeTupleScheme getScheme() { + return new TTimeRangeTupleScheme(); + } + } + + private static class TTimeRangeTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TTimeRange struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeI64(struct.minStamp); + oprot.writeI64(struct.maxStamp); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TTimeRange struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.minStamp = iprot.readI64(); + struct.setMinStampIsSet(true); + struct.maxStamp = iprot.readI64(); + struct.setMaxStampIsSet(true); + } + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/thrift2/package.html b/src/main/java/org/apache/hadoop/hbase/thrift2/package.html new file mode 100644 index 0000000..3d25e4d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/thrift2/package.html @@ -0,0 +1,107 @@ + + + + + + + +Provides an HBase Thrift +service. + +This package contains a Thrift interface definition file for an HBase RPC +service and a Java server implementation. + +There are currently 2 thrift server implementations in HBase, the packages: + +

      +
    • org.apache.hadoop.hbase.thrift: This may one day be marked as depreceated.
      • +
    • org.apache.hadoop.hbase.thrift2: i.e. this package. This is intended to closely match to the HTable interface and + to one day supercede the older thrift (the old thrift mimics an API HBase no longer has).
      • +
    + +

    What is Thrift?

    + + +

    "Thrift is a software framework for scalable cross-language services +development. It combines a software stack with a code generation engine to +build services that work efficiently and seamlessly between C++, Java, Python, +PHP, Ruby, Erlang, Perl, Haskell, C#, Cocoa, JavaScript, Node.js, Smalltalk, +and OCaml. Originally developed at Facebook, Thrift was open sourced in April +2007 and entered the Apache Incubator in May, 2008". +From http://thrift.apache.org/

    + +

    Description

    + +

    The HBase API is defined in the +file hbase.thrift. A server-side implementation of the API is in +org.apache.hadoop.hbase.thrift2.ThriftHBaseServiceHandler with the +server boiler plate in org.apache.hadoop.hbase.thrift2.ThriftServer. +The generated interfaces, types, and RPC utility files are checked into SVN under the +org.apache.hadoop.hbase.thrift2.generated directory. +

    +

    To stop, use: +

    +  ./bin/hbase-daemon.sh stop thrift
+
    + +These are the command line arguments the Thrift server understands in addition to start and stop: +
    +
    -b, --bind
    +
    Address to bind the Thrift server to. Not supported by the Nonblocking and HsHa server [default: 0.0.0.0]
    + +
    -p, --port
    +
    Port to bind to [default: 9090]
    + +
    -f, --framed
    +
    Use framed transport (implied when using one of the non-blocking servers)
    + +
    -c, --compact
    +
    Use the compact protocol [default: binary protocol]
    + +
    -h, --help
    +
    Displays usage information for the Thrift server
    + +
    -threadpool
    +
    Use the TThreadPoolServer. This is the default.
    + +
    -hsha
    +
    Use the THsHaServer. This implies the framed transport.
    + +
    -nonblocking
    +
    Use the TNonblockingServer. This implies the framed transport.
    +
    + +

    Details

    + +

    HBase currently uses version 0.8.0 of Apache Thrift.

    +

    The files were generated by running the commands under the hbase checkout dir: +

    +  thrift -strict --gen java:hashcode ./src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
+  # Move the generated files into place their expected location under hbase
+  mv gen-java/org/apache/hadoop/hbase/thrift2/generated/* src/main/java/org/apache/hadoop/hbase/thrift2/generated/
+  # Remove the gen-java file made by thrift
+  rm -rf gen-java
+
    + +

    The 'thrift' binary is the Thrift compiler, and it is distributed separately from HBase +in a Thrift release. Additionally, specific language runtime libraries are a +part of a Thrift release. A version of the Java runtime is included in HBase via maven. +

    + + + diff --git a/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/src/main/java/org/apache/hadoop/hbase/tool/Canary.java new file mode 100644 index 0000000..0cf5569 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/tool/Canary.java @@ -0,0 +1,253 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.tool; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +import org.apache.hadoop.conf.Configuration; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableNotFoundException; + +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.HBaseAdmin; + +/** + * HBase Canary Tool, that that can be used to do + * "canary monitoring" of a running HBase cluster. + * + * Foreach region tries to get one row per column family + * and outputs some information about failure or latency. + */ +public final class Canary implements Tool { + // Sink interface used by the canary to outputs information + public interface Sink { + public void publishReadFailure(HRegionInfo region); + public void publishReadFailure(HRegionInfo region, HColumnDescriptor column); + public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime); + } + + // Simple implementation of canary sink that allows to plot on + // file or standard output timings or failures. + public static class StdOutSink implements Sink { + @Override + public void publishReadFailure(HRegionInfo region) { + LOG.error(String.format("read from region %s failed", region.getRegionNameAsString())); + } + + @Override + public void publishReadFailure(HRegionInfo region, HColumnDescriptor column) { + LOG.error(String.format("read from region %s column family %s failed", + region.getRegionNameAsString(), column.getNameAsString())); + } + + @Override + public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime) { + LOG.info(String.format("read from region %s column family %s in %dms", + region.getRegionNameAsString(), column.getNameAsString(), msTime)); + } + } + + private static final long DEFAULT_INTERVAL = 6000; + + private static final Log LOG = LogFactory.getLog(Canary.class); + + private Configuration conf = null; + private HBaseAdmin admin = null; + private long interval = 0; + private Sink sink = null; + + public Canary() { + this(new StdOutSink()); + } + + public Canary(Sink sink) { + this.sink = sink; + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public int run(String[] args) throws Exception { + int tables_index = -1; + + // Process command line args + for (int i = 0; i < args.length; i++) { + String cmd = args[i]; + + if (cmd.startsWith("-")) { + if (tables_index >= 0) { + // command line args must be in the form: [opts] [table 1 [table 2 ...]] + System.err.println("Invalid command line options"); + printUsageAndExit(); + } + + if (cmd.equals("-help")) { + // user asked for help, print the help and quit. + printUsageAndExit(); + } else if (cmd.equals("-daemon") && interval == 0) { + // user asked for daemon mode, set a default interval between checks + interval = DEFAULT_INTERVAL; + } else if (cmd.equals("-interval")) { + // user has specified an interval for canary breaths (-interval N) + i++; + + if (i == args.length) { + System.err.println("-interval needs a numeric value argument."); + printUsageAndExit(); + } + + try { + interval = Long.parseLong(args[i]) * 1000; + } catch (NumberFormatException e) { + System.err.println("-interval needs a numeric value argument."); + printUsageAndExit(); + } + } else { + // no options match + System.err.println(cmd + " options is invalid."); + printUsageAndExit(); + } + } else if (tables_index < 0) { + // keep track of first table name specified by the user + tables_index = i; + } + } + + // initialize HBase conf and admin + if (conf == null) conf = HBaseConfiguration.create(); + admin = new HBaseAdmin(conf); + + // lets the canary monitor the cluster + do { + if (admin.isAborted()) { + LOG.error("HBaseAdmin aborted"); + return(1); + } + + if (tables_index >= 0) { + for (int i = tables_index; i < args.length; i++) { + sniff(args[i]); + } + } else { + sniff(); + } + + Thread.sleep(interval); + } while (interval > 0); + + return(0); + } + + private void printUsageAndExit() { + System.err.printf("Usage: bin/hbase %s [opts] [table 1 [table 2...]]\n", getClass().getName()); + System.err.println(" where [opts] are:"); + System.err.println(" -help Show this help and exit."); + System.err.println(" -daemon Continuous check at defined intervals."); + System.err.println(" -interval Interval between checks (sec)"); + System.exit(1); + } + + /* + * canary entry point to monitor all the tables. + */ + private void sniff() throws Exception { + for (HTableDescriptor table : admin.listTables()) { + sniff(table); + } + } + + /* + * canary entry point to monitor specified table. + */ + private void sniff(String tableName) throws Exception { + if (admin.isTableAvailable(tableName)) { + sniff(admin.getTableDescriptor(tableName.getBytes())); + } else { + LOG.warn(String.format("Table %s is not available", tableName)); + } + } + + /* + * Loops over regions that owns this table, + * and output some information abouts the state. + */ + private void sniff(HTableDescriptor tableDesc) throws Exception { + HTable table = null; + + try { + table = new HTable(admin.getConfiguration(), tableDesc.getName()); + } catch (TableNotFoundException e) { + return; + } + + for (HRegionInfo region : admin.getTableRegions(tableDesc.getName())) { + try { + sniffRegion(region, table); + } catch (Exception e) { + sink.publishReadFailure(region); + } + } + } + + /* + * For each column family of the region tries to get one row + * and outputs the latency, or the failure. + */ + private void sniffRegion(HRegionInfo region, HTable table) throws Exception { + HTableDescriptor tableDesc = table.getTableDescriptor(); + for (HColumnDescriptor column : tableDesc.getColumnFamilies()) { + Get get = new Get(region.getStartKey()); + get.addFamily(column.getName()); + + try { + long startTime = System.currentTimeMillis(); + table.get(get); + long time = System.currentTimeMillis() - startTime; + + sink.publishReadTiming(region, column, time); + } catch (Exception e) { + sink.publishReadFailure(region, column); + } + } + } + + public static void main(String[] args) throws Exception { + int exitCode = ToolRunner.run(new Canary(), args); + System.exit(exitCode); + } +} + diff --git a/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java b/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java new file mode 100644 index 0000000..814dd74 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.cli.BasicParser; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +/** + * Common base class used for HBase command-line tools. Simplifies workflow and + * command-line argument parsing. + */ +public abstract class AbstractHBaseTool implements Tool { + + private static final int EXIT_SUCCESS = 0; + private static final int EXIT_FAILURE = 1; + + private static final String HELP_OPTION = "help"; + + private static final Log LOG = LogFactory.getLog(AbstractHBaseTool.class); + + private final Options options = new Options(); + + protected Configuration conf = null; + + private static final Set requiredOptions = new TreeSet(); + + /** + * Override this to add command-line options using {@link #addOptWithArg} + * and similar methods. + */ + protected abstract void addOptions(); + + /** + * This method is called to process the options after they have been parsed. + */ + protected abstract void processOptions(CommandLine cmd); + + /** The "main function" of the tool */ + protected abstract int doWork() throws Exception; + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public final int run(String[] args) throws Exception { + if (conf == null) { + LOG.error("Tool configuration is not initialized"); + throw new NullPointerException("conf"); + } + + CommandLine cmd; + try { + // parse the command line arguments + cmd = parseArgs(args); + } catch (ParseException e) { + LOG.error("Error when parsing command-line arguemnts", e); + printUsage(); + return EXIT_FAILURE; + } + + if (cmd.hasOption(HELP_OPTION) || !sanityCheckOptions(cmd)) { + printUsage(); + return EXIT_FAILURE; + } + + processOptions(cmd); + + int ret = EXIT_FAILURE; + try { + ret = doWork(); + } catch (Exception e) { + LOG.error("Error running command-line tool", e); + return EXIT_FAILURE; + } + return ret; + } + + private boolean sanityCheckOptions(CommandLine cmd) { + boolean success = true; + for (String reqOpt : requiredOptions) { + if (!cmd.hasOption(reqOpt)) { + LOG.error("Required option -" + reqOpt + " is missing"); + success = false; + } + } + return success; + } + + private CommandLine parseArgs(String[] args) throws ParseException { + options.addOption(HELP_OPTION, false, "Show usage"); + addOptions(); + CommandLineParser parser = new BasicParser(); + return parser.parse(options, args); + } + + private void printUsage() { + HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.setWidth(80); + String usageHeader = "Options:"; + String usageFooter = ""; + String usageStr = "bin/hbase " + getClass().getName() + " "; + + helpFormatter.printHelp(usageStr, usageHeader, options, + usageFooter); + } + + protected void addRequiredOptWithArg(String opt, String description) { + requiredOptions.add(opt); + addOptWithArg(opt, description); + } + + protected void addOptNoArg(String opt, String description) { + options.addOption(opt, false, description); + } + + protected void addOptWithArg(String opt, String description) { + options.addOption(opt, true, description); + } + + /** + * Parse a number and enforce a range. + */ + public static long parseLong(String s, long minValue, long maxValue) { + long l = Long.parseLong(s); + if (l < minValue || l > maxValue) { + throw new IllegalArgumentException("The value " + l + + " is out of range [" + minValue + ", " + maxValue + "]"); + } + return l; + } + + public static int parseInt(String s, int minValue, int maxValue) { + return (int) parseLong(s, minValue, maxValue); + } + + /** Call this from the concrete tool class's main function. */ + protected void doStaticMain(String args[]) { + int ret; + try { + ret = ToolRunner.run(HBaseConfiguration.create(), this, args); + } catch (Exception ex) { + LOG.error("Error running command-line tool", ex); + ret = EXIT_FAILURE; + } + System.exit(ret); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/Addressing.java b/src/main/java/org/apache/hadoop/hbase/util/Addressing.java new file mode 100644 index 0000000..05c51ea --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Addressing.java @@ -0,0 +1,76 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.net.InetSocketAddress; + +/** + * Utility for network addresses, resolving and naming. + */ +public class Addressing { + public static final String VALID_PORT_REGEX = "[\\d]+"; + public static final String HOSTNAME_PORT_SEPARATOR = ":"; + + /** + * @param hostAndPort Formatted as <hostname> ':' <port> + * @return An InetSocketInstance + */ + public static InetSocketAddress createInetSocketAddressFromHostAndPortStr( + final String hostAndPort) { + return new InetSocketAddress(parseHostname(hostAndPort), parsePort(hostAndPort)); + } + + /** + * @param hostname Server hostname + * @param port Server port + * @return Returns a concatenation of hostname and + * port in following + * form: <hostname> ':' <port>. For example, if hostname + * is example.org and port is 1234, this method will return + * example.org:1234 + */ + public static String createHostAndPortStr(final String hostname, final int port) { + return hostname + HOSTNAME_PORT_SEPARATOR + port; + } + + /** + * @param hostAndPort Formatted as <hostname> ':' <port> + * @return The hostname portion of hostAndPort + */ + public static String parseHostname(final String hostAndPort) { + int colonIndex = hostAndPort.lastIndexOf(HOSTNAME_PORT_SEPARATOR); + if (colonIndex < 0) { + throw new IllegalArgumentException("Not a host:port pair: " + hostAndPort); + } + return hostAndPort.substring(0, colonIndex); + } + + /** + * @param hostAndPort Formatted as <hostname> ':' <port> + * @return The port portion of hostAndPort + */ + public static int parsePort(final String hostAndPort) { + int colonIndex = hostAndPort.lastIndexOf(HOSTNAME_PORT_SEPARATOR); + if (colonIndex < 0) { + throw new IllegalArgumentException("Not a host:port pair: " + hostAndPort); + } + return Integer.parseInt(hostAndPort.substring(colonIndex + 1)); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/util/Base64.java b/src/main/java/org/apache/hadoop/hbase/util/Base64.java new file mode 100644 index 0000000..892f808 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Base64.java @@ -0,0 +1,1643 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FilterInputStream; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.io.Serializable; +import java.io.UnsupportedEncodingException; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +/** + * Encodes and decodes to and from Base64 notation. + * + *

    + * Homepage: http://iharder.net/base64. + *

    + * + *

    + * Change Log: + *

    + *
      + *
    • v2.2.1 - Fixed bug using URL_SAFE and ORDERED encodings. Fixed bug + * when using very small files (~< 40 bytes).
      • + *
    • v2.2 - Added some helper methods for encoding/decoding directly from + * one file to the next. Also added a main() method to support command + * line encoding/decoding from one file to the next. Also added these + * Base64 dialects: + *
        + *
      1. The default is RFC3548 format.
        2. + *
      2. Using Base64.URLSAFE generates URL and file name friendly format as + * described in Section 4 of RFC3548. + * http://www.faqs.org/rfcs/rfc3548.html
        3. + *
      3. Using Base64.ORDERED generates URL and file name friendly format + * that preserves lexical ordering as described in + * http://www.faqs.org/qa/rfcc-1940.html
        4. + *
      + *

      + * Special thanks to Jim Kellerman at + * http://www.powerset.com/ for contributing the new Base64 dialects. + *

      • + * + *
    • v2.1 - Cleaned up javadoc comments and unused variables and methods. + * Added some convenience methods for reading and writing to and from files. + *
      • + *
    • v2.0.2 - Now specifies UTF-8 encoding in places where the code fails on + * systems with other encodings (like EBCDIC).
      • + *
    • v2.0.1 - Fixed an error when decoding a single byte, that is, when the + * encoded data was a single byte.
      • + *
    • v2.0 - I got rid of methods that used booleans to set options. Now + * everything is more consolidated and cleaner. The code now detects when + * data that's being decoded is gzip-compressed and will decompress it + * automatically. Generally things are cleaner. You'll probably have to + * change some method calls that you were making to support the new options + * format (ints that you "OR" together).
      • + *
    • v1.5.1 - Fixed bug when decompressing and decoding to a byte[] using + * decode( String s, boolean gzipCompressed ). Added the ability to + * "suspend" encoding in the Output Stream so you can turn on and off the + * encoding if you need to embed base64 data in an otherwise "normal" stream + * (like an XML file).
      • + *
    • v1.5 - Output stream pases on flush() command but doesn't do anything + * itself. This helps when using GZIP streams. Added the ability to + * GZip-compress objects before encoding them.
      • + *
    • v1.4 - Added helper methods to read/write files.
      • + *
    • v1.3.6 - Fixed OutputStream.flush() so that 'position' is reset.
      • + *
    • v1.3.5 - Added flag to turn on and off line breaks. Fixed bug in input + * stream where last buffer being read, if not completely full, was not + * returned.
      • + *
    • v1.3.4 - Fixed when "improperly padded stream" error was thrown at the + * wrong time.
      • + *
    • v1.3.3 - Fixed I/O streams which were totally messed up.
      • + *
    + * + *

    + * I am placing this code in the Public Domain. Do with it as you will. This + * software comes with no guarantees or warranties but with plenty of + * well-wishing instead! + *

    + * Please visit http://iharder.net/base64 + * periodically to check for updates or to contribute improvements. + *

    + * author: Robert Harder, rob@iharder.net + *
    + * version: 2.2.1 + */ +public class Base64 { + + /* ******** P U B L I C F I E L D S ******** */ + + /** No options specified. Value is zero. */ + public final static int NO_OPTIONS = 0; + + /** Specify encoding. */ + public final static int ENCODE = 1; + + /** Specify decoding. */ + public final static int DECODE = 0; + + /** Specify that data should be gzip-compressed. */ + public final static int GZIP = 2; + + /** Don't break lines when encoding (violates strict Base64 specification) */ + public final static int DONT_BREAK_LINES = 8; + + /** + * Encode using Base64-like encoding that is URL and Filename safe as + * described in Section 4 of RFC3548: + * + * http://www.faqs.org/rfcs/rfc3548.html. + * It is important to note that data encoded this way is not + * officially valid Base64, or at the very least should not be called Base64 + * without also specifying that is was encoded using the URL and + * Filename safe dialect. + */ + public final static int URL_SAFE = 16; + + /** + * Encode using the special "ordered" dialect of Base64 described here: + * + * http://www.faqs.org/qa/rfcc-1940.html. + */ + public final static int ORDERED = 32; + + /* ******** P R I V A T E F I E L D S ******** */ + + private static final Log LOG = LogFactory.getLog(Base64.class); + + /** Maximum line length (76) of Base64 output. */ + private final static int MAX_LINE_LENGTH = 76; + + /** The equals sign (=) as a byte. */ + private final static byte EQUALS_SIGN = (byte) '='; + + /** The new line character (\n) as a byte. */ + private final static byte NEW_LINE = (byte) '\n'; + + /** Preferred encoding. */ + private final static String PREFERRED_ENCODING = "UTF-8"; + + private final static byte WHITE_SPACE_ENC = -5; // Indicates white space + private final static byte EQUALS_SIGN_ENC = -1; // Indicates equals sign + + /* ******** S T A N D A R D B A S E 6 4 A L P H A B E T ******** */ + + /** The 64 valid Base64 values. */ + + /* + * Host platform may be something funny like EBCDIC, so we hardcode these + * values. + */ + private final static byte[] _STANDARD_ALPHABET = { (byte) 'A', (byte) 'B', + (byte) 'C', (byte) 'D', (byte) 'E', (byte) 'F', (byte) 'G', (byte) 'H', + (byte) 'I', (byte) 'J', (byte) 'K', (byte) 'L', (byte) 'M', (byte) 'N', + (byte) 'O', (byte) 'P', (byte) 'Q', (byte) 'R', (byte) 'S', (byte) 'T', + (byte) 'U', (byte) 'V', (byte) 'W', (byte) 'X', (byte) 'Y', (byte) 'Z', + (byte) 'a', (byte) 'b', (byte) 'c', (byte) 'd', (byte) 'e', (byte) 'f', + (byte) 'g', (byte) 'h', (byte) 'i', (byte) 'j', (byte) 'k', (byte) 'l', + (byte) 'm', (byte) 'n', (byte) 'o', (byte) 'p', (byte) 'q', (byte) 'r', + (byte) 's', (byte) 't', (byte) 'u', (byte) 'v', (byte) 'w', (byte) 'x', + (byte) 'y', (byte) 'z', (byte) '0', (byte) '1', (byte) '2', (byte) '3', + (byte) '4', (byte) '5', (byte) '6', (byte) '7', (byte) '8', (byte) '9', + (byte) '+', (byte) '/' + }; + + /** + * Translates a Base64 value to either its 6-bit reconstruction value or a + * negative number indicating some other meaning. + */ + private final static byte[] _STANDARD_DECODABET = { + -9, -9, -9, -9, -9, -9, -9, -9, -9, // Decimal 0 - 8 + -5, -5, // Whitespace: Tab, Newline + -9, -9, // Decimal 11 - 12 + -5, // Whitespace: Return + -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, // Decimal 14 - 26 + -9, -9, -9, -9, -9, // Decimal 27 - 31 + -5, // Whitespace: Space + -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, // Decimal 33 - 42 + 62, // Plus sign at decimal 43 + -9, -9, -9, // Decimal 44 - 46 + 63, // Slash at decimal 47 + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, // Numbers zero - nine + -9, -9, -9, // Decimal 58 - 60 + -1, // Equals sign at decimal 61 + -9, -9, -9, // Decimal 62 - 64 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, // Letters 'A' - 'N' + 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, // Letters 'O' - 'Z' + -9, -9, -9, -9, -9, -9, // Decimal 91 - 96 + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, // Letters 'a' - 'm' + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, // Letters 'n' -'z' + -9, -9, -9, -9 // Decimal 123 - 126 + }; + + /* ******** U R L S A F E B A S E 6 4 A L P H A B E T ******** */ + + /** + * Used in the URL and Filename safe dialect described in Section 4 of RFC3548 + * + * http://www.faqs.org/rfcs/rfc3548.html. + * Notice that the last two bytes become "hyphen" and "underscore" instead of + * "plus" and "slash." + */ + private final static byte[] _URL_SAFE_ALPHABET = { (byte) 'A', (byte) 'B', + (byte) 'C', (byte) 'D', (byte) 'E', (byte) 'F', (byte) 'G', (byte) 'H', + (byte) 'I', (byte) 'J', (byte) 'K', (byte) 'L', (byte) 'M', (byte) 'N', + (byte) 'O', (byte) 'P', (byte) 'Q', (byte) 'R', (byte) 'S', (byte) 'T', + (byte) 'U', (byte) 'V', (byte) 'W', (byte) 'X', (byte) 'Y', (byte) 'Z', + (byte) 'a', (byte) 'b', (byte) 'c', (byte) 'd', (byte) 'e', (byte) 'f', + (byte) 'g', (byte) 'h', (byte) 'i', (byte) 'j', (byte) 'k', (byte) 'l', + (byte) 'm', (byte) 'n', (byte) 'o', (byte) 'p', (byte) 'q', (byte) 'r', + (byte) 's', (byte) 't', (byte) 'u', (byte) 'v', (byte) 'w', (byte) 'x', + (byte) 'y', (byte) 'z', (byte) '0', (byte) '1', (byte) '2', (byte) '3', + (byte) '4', (byte) '5', (byte) '6', (byte) '7', (byte) '8', (byte) '9', + (byte) '-', (byte) '_' + }; + + /** + * Used in decoding URL and Filename safe dialects of Base64. + */ + private final static byte[] _URL_SAFE_DECODABET = { + -9, -9, -9, -9, -9, -9, -9, -9, -9, // Decimal 0 - 8 + -5, -5, // Whitespace: Tab, Newline + -9, -9, // Decimal 11 - 12 + -5, // Whitespace: Return + -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, // Decimal 14 - 26 + -9, -9, -9, -9, -9, // Decimal 27 - 31 + -5, // Whitespace: Space + -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, // Decimal 33 - 42 + -9, // Plus sign at 43 + -9, // Decimal 44 + 62, // Minus sign at 45 + -9, // Decimal 46 + -9, // Slash at 47 + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, // Numbers 0 - 9 + -9, -9, -9, // Decimal 58 - 60 + -1, // Equals sign at 61 + -9, -9, -9, // Decimal 62 - 64 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, // Letters 'A' - 'N' + 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, // Letters 'O' - 'Z' + -9, -9, -9, -9, // Decimal 91 - 94 + 63, // Underscore at 95 + -9, // Decimal 96 + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, // Letters 'a' - 'm' + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, // Letters 'n' - 'z' + -9, -9, -9, -9 // Decimal 123 - 126 + }; + + /* ******** O R D E R E D B A S E 6 4 A L P H A B E T ******** */ + + /** + * In addition to being URL and file name friendly, this encoding preserves + * the sort order of encoded values. Whatever is input, be it string or + * just an array of bytes, when you use this encoding, the encoded value sorts + * exactly the same as the input value. It is described in the RFC change + * request: + * http://www.faqs.org/qa/rfcc-1940.html. + * + * It replaces "plus" and "slash" with "hyphen" and "underscore" and + * rearranges the alphabet so that the characters are in their natural sort + * order. + */ + private final static byte[] _ORDERED_ALPHABET = { (byte) '-', (byte) '0', + (byte) '1', (byte) '2', (byte) '3', (byte) '4', (byte) '5', (byte) '6', + (byte) '7', (byte) '8', (byte) '9', (byte) 'A', (byte) 'B', (byte) 'C', + (byte) 'D', (byte) 'E', (byte) 'F', (byte) 'G', (byte) 'H', (byte) 'I', + (byte) 'J', (byte) 'K', (byte) 'L', (byte) 'M', (byte) 'N', (byte) 'O', + (byte) 'P', (byte) 'Q', (byte) 'R', (byte) 'S', (byte) 'T', (byte) 'U', + (byte) 'V', (byte) 'W', (byte) 'X', (byte) 'Y', (byte) 'Z', (byte) '_', + (byte) 'a', (byte) 'b', (byte) 'c', (byte) 'd', (byte) 'e', (byte) 'f', + (byte) 'g', (byte) 'h', (byte) 'i', (byte) 'j', (byte) 'k', (byte) 'l', + (byte) 'm', (byte) 'n', (byte) 'o', (byte) 'p', (byte) 'q', (byte) 'r', + (byte) 's', (byte) 't', (byte) 'u', (byte) 'v', (byte) 'w', (byte) 'x', + (byte) 'y', (byte) 'z' + }; + + /** + * Used in decoding the "ordered" dialect of Base64. + */ + private final static byte[] _ORDERED_DECODABET = { + -9, -9, -9, -9, -9, -9, -9, -9, -9, // Decimal 0 - 8 + -5, -5, // Whitespace: Tab, Newline + -9, -9, // Decimal 11 - 12 + -5, // Whitespace: Return + -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, // Decimal 14 - 26 + -9, -9, -9, -9, -9, // Decimal 27 - 31 + -5, // Whitespace: Space + -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, // Decimal 33 - 42 + -9, // Plus sign at 43 + -9, // Decimal 44 + 0, // Minus sign at 45 + -9, // Decimal 46 + -9, // Slash at decimal 47 + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, // Numbers 0 - 9 + -9, -9, -9, // Decimal 58 - 60 + -1, // Equals sign at 61 + -9, -9, -9, // Decimal 62 - 64 + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, // Letters 'A' - 'M' + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, // Letters 'N' - 'Z' + -9, -9, -9, -9, // Decimal 91 - 94 + 37, // Underscore at 95 + -9, // Decimal 96 + 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, // Letters 'a' - 'm' + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, // Letters 'n' - 'z' + -9, -9, -9, -9 // Decimal 123 - 126 + }; + + /* ******** D E T E R M I N E W H I C H A L H A B E T ******** */ + + /** + * Returns one of the _SOMETHING_ALPHABET byte arrays depending on the options + * specified. It's possible, though silly, to specify ORDERED and URLSAFE in + * which case one of them will be picked, though there is no guarantee as to + * which one will be picked. + * + * @param options URL_SAFE or ORDERED + * @return alphabet array to use + */ + protected static byte[] getAlphabet(int options) { + if ((options & URL_SAFE) == URL_SAFE) { + return _URL_SAFE_ALPHABET; + + } else if ((options & ORDERED) == ORDERED) { + return _ORDERED_ALPHABET; + + } else { + return _STANDARD_ALPHABET; + } + } // end getAlphabet + + /** + * Returns one of the _SOMETHING_DECODABET byte arrays depending on the + * options specified. It's possible, though silly, to specify ORDERED and + * URL_SAFE in which case one of them will be picked, though there is no + * guarantee as to which one will be picked. + * @param options URL_SAFE or ORDERED + * @return alphabet array to use + */ + protected static byte[] getDecodabet(int options) { + if ((options & URL_SAFE) == URL_SAFE) { + return _URL_SAFE_DECODABET; + + } else if ((options & ORDERED) == ORDERED) { + return _ORDERED_DECODABET; + + } else { + return _STANDARD_DECODABET; + } + } // end getDecodabet + + /** Defeats instantiation. */ + private Base64() {} + + /** + * Main program. Used for testing. + * + * Encodes or decodes two files from the command line + * + * @param args command arguments + */ + public static void main(String[] args) { + if (args.length < 3) { + usage("Not enough arguments."); + + } else { + String flag = args[0]; + String infile = args[1]; + String outfile = args[2]; + if (flag.equals("-e")) { // encode + encodeFileToFile(infile, outfile); + + } else if (flag.equals("-d")) { // decode + decodeFileToFile(infile, outfile); + + } else { + usage("Unknown flag: " + flag); + } + } + } // end main + + /** + * Prints command line usage. + * + * @param msg A message to include with usage info. + */ + private static void usage(String msg) { + System.err.println(msg); + System.err.println("Usage: java Base64 -e|-d inputfile outputfile"); + } // end usage + + /* ******** E N C O D I N G M E T H O D S ******** */ + + /** + * Encodes up to the first three bytes of array threeBytes and + * returns a four-byte array in Base64 notation. The actual number of + * significant bytes in your array is given by numSigBytes. The + * array threeBytes needs only be as big as numSigBytes. + * Code can reuse a byte array by passing a four-byte array as b4. + * + * @param b4 A reusable byte array to reduce array instantiation + * @param threeBytes the array to convert + * @param numSigBytes the number of significant bytes in your array + * @param options options for get alphabet + * @return four byte array in Base64 notation. + * @since 1.5.1 + */ + protected static byte[] encode3to4(byte[] b4, byte[] threeBytes, + int numSigBytes, int options) { + encode3to4(threeBytes, 0, numSigBytes, b4, 0, options); + return b4; + } // end encode3to4 + + /** + * Encodes up to three bytes of the array source and writes the + * resulting four Base64 bytes to destination. The source and + * destination arrays can be manipulated anywhere along their length by + * specifying srcOffset and destOffset. This method + * does not check to make sure your arrays are large enough to accomodate + * srcOffset + 3 for the source array or + * destOffset + 4 for the destination array. The + * actual number of significant bytes in your array is given by + * numSigBytes. + *

    + * This is the lowest level of the encoding methods with all possible + * parameters. + * + * @param source the array to convert + * @param srcOffset the index where conversion begins + * @param numSigBytes the number of significant bytes in your array + * @param destination the array to hold the conversion + * @param destOffset the index where output will be put + * @param options options for get alphabet + * @return the destination array + * @since 1.3 + */ + protected static byte[] encode3to4(byte[] source, int srcOffset, + int numSigBytes, byte[] destination, int destOffset, int options) { + byte[] ALPHABET = getAlphabet(options); + + // 1 2 3 + // 01234567890123456789012345678901 Bit position + // --------000000001111111122222222 Array position from threeBytes + // --------| || || || | Six bit groups to index ALPHABET + // >>18 >>12 >> 6 >> 0 Right shift necessary + // 0x3f 0x3f 0x3f Additional AND + + // Create buffer with zero-padding if there are only one or two + // significant bytes passed in the array. + // We have to shift left 24 in order to flush out the 1's that appear + // when Java treats a value as negative that is cast from a byte to an int. + int inBuff = + (numSigBytes > 0 ? ((source[srcOffset] << 24) >>> 8) : 0) + | (numSigBytes > 1 ? ((source[srcOffset + 1] << 24) >>> 16) : 0) + | (numSigBytes > 2 ? ((source[srcOffset + 2] << 24) >>> 24) : 0); + + switch (numSigBytes) { + case 3: + destination[destOffset] = ALPHABET[(inBuff >>> 18)]; + destination[destOffset + 1] = ALPHABET[(inBuff >>> 12) & 0x3f]; + destination[destOffset + 2] = ALPHABET[(inBuff >>> 6) & 0x3f]; + destination[destOffset + 3] = ALPHABET[(inBuff) & 0x3f]; + return destination; + + case 2: + destination[destOffset] = ALPHABET[(inBuff >>> 18)]; + destination[destOffset + 1] = ALPHABET[(inBuff >>> 12) & 0x3f]; + destination[destOffset + 2] = ALPHABET[(inBuff >>> 6) & 0x3f]; + destination[destOffset + 3] = EQUALS_SIGN; + return destination; + + case 1: + destination[destOffset] = ALPHABET[(inBuff >>> 18)]; + destination[destOffset + 1] = ALPHABET[(inBuff >>> 12) & 0x3f]; + destination[destOffset + 2] = EQUALS_SIGN; + destination[destOffset + 3] = EQUALS_SIGN; + return destination; + + default: + return destination; + } // end switch + } // end encode3to4 + + /** + * Serializes an object and returns the Base64-encoded version of that + * serialized object. If the object cannot be serialized or there is another + * error, the method will return null. The object is not + * GZip-compressed before being encoded. + * + * @param serializableObject The object to encode + * @return The Base64-encoded object + * @since 1.4 + */ + public static String encodeObject(Serializable serializableObject) { + return encodeObject(serializableObject, NO_OPTIONS); + } // end encodeObject + + /** + * Serializes an object and returns the Base64-encoded version of that + * serialized object. If the object cannot be serialized or there is another + * error, the method will return null. + *

    + * Valid options: + *

      + *
    • GZIP: gzip-compresses object before encoding it.
      • + *
    • DONT_BREAK_LINES: don't break lines at 76 characters. Note: + * Technically, this makes your encoding non-compliant.
      • + *
    + *

    + * Example: encodeObject( myObj, Base64.GZIP ) or + *

    + * Example: + * encodeObject( myObj, Base64.GZIP | Base64.DONT_BREAK_LINES ) + * + * @param serializableObject The object to encode + * @param options Specified options + * @see Base64#GZIP + * @see Base64#DONT_BREAK_LINES + * @return The Base64-encoded object + * @since 2.0 + */ + @SuppressWarnings({"ConstantConditions"}) + public static String encodeObject(Serializable serializableObject, + int options) { + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + OutputStream b64os = null; + ObjectOutputStream oos = null; + try { + // ObjectOutputStream -> (GZIP) -> Base64 -> ByteArrayOutputStream + b64os = new Base64OutputStream(baos, ENCODE | options); + + oos = ((options & GZIP) == GZIP) ? + new ObjectOutputStream(new GZIPOutputStream(b64os)) : + new ObjectOutputStream(b64os); + + oos.writeObject(serializableObject); + return new String(baos.toByteArray(), PREFERRED_ENCODING); + + } catch (UnsupportedEncodingException uue) { + return new String(baos.toByteArray()); + + } catch (IOException e) { + LOG.error("error encoding object", e); + return null; + + } finally { + if (oos != null) { + try { + oos.close(); + } catch (Exception e) { + LOG.error("error closing ObjectOutputStream", e); + } + } + if (b64os != null) { + try { + b64os.close(); + } catch (Exception e) { + LOG.error("error closing Base64OutputStream", e); + } + } + try { + baos.close(); + } catch (Exception e) { + LOG.error("error closing ByteArrayOutputStream", e); + } + } // end finally + } // end encode + + /** + * Encodes a byte array into Base64 notation. Does not GZip-compress data. + * + * @param source The data to convert + * @return encoded byte array + * @since 1.4 + */ + public static String encodeBytes(byte[] source) { + return encodeBytes(source, 0, source.length, NO_OPTIONS); + } // end encodeBytes + + /** + * Encodes a byte array into Base64 notation. + *

    + * Valid options: + *

      + *
    • GZIP: gzip-compresses object before encoding it.
      • + *
    • DONT_BREAK_LINES: don't break lines at 76 characters. Note: + * Technically, this makes your encoding non-compliant.
      • + *
    + * + *

    + * Example: encodeBytes( myData, Base64.GZIP ) or + *

    + * Example: + * encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES ) + * + * @param source The data to convert + * @param options Specified options + * @see Base64#GZIP + * @see Base64#DONT_BREAK_LINES + * @see Base64#URL_SAFE + * @see Base64#ORDERED + * @return encoded byte array + * @since 2.0 + */ + public static String encodeBytes(byte[] source, int options) { + return encodeBytes(source, 0, source.length, options); + } // end encodeBytes + + /** + * Encodes a byte array into Base64 notation. Does not GZip-compress data. + * + * @param source The data to convert + * @param off Offset in array where conversion should begin + * @param len Length of data to convert + * @return encoded byte array + * @since 1.4 + */ + public static String encodeBytes(byte[] source, int off, int len) { + return encodeBytes(source, off, len, NO_OPTIONS); + } // end encodeBytes + + /** + * Encodes a byte array into Base64 notation. + *

    + * Valid options: + *

      + *
    • GZIP: gzip-compresses object before encoding it.
      • + *
    • DONT_BREAK_LINES: don't break lines at 76 characters. Note: + * Technically, this makes your encoding non-compliant.
      • + *
    + * + *

    + * Example: encodeBytes( myData, Base64.GZIP ) or + *

    + * Example: + * encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES ) + * + * @param source The data to convert + * @param off Offset in array where conversion should begin + * @param len Length of data to convert + * @param options Specified options + * @see Base64#GZIP + * @see Base64#DONT_BREAK_LINES + * @see Base64#URL_SAFE + * @see Base64#ORDERED + * @return encoded byte array + * @since 2.0 + */ + public static String encodeBytes(byte[] source, int off, int len, int options) { + if ((options & GZIP) == GZIP) { // Compress? + // GZip -> Base64 -> ByteArray + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream gzos = null; + + try { + gzos = + new GZIPOutputStream(new Base64OutputStream(baos, ENCODE | options)); + + gzos.write(source, off, len); + gzos.close(); + gzos = null; + return new String(baos.toByteArray(), PREFERRED_ENCODING); + + } catch (UnsupportedEncodingException uue) { + return new String(baos.toByteArray()); + + } catch (IOException e) { + LOG.error("error encoding byte array", e); + return null; + + } finally { + if (gzos != null) { + try { + gzos.close(); + } catch (Exception e) { + LOG.error("error closing GZIPOutputStream", e); + } + } + try { + baos.close(); + } catch (Exception e) { + LOG.error("error closing ByteArrayOutputStream", e); + } + } // end finally + + } // end Compress + + // Don't compress. Better not to use streams at all then. + + boolean breakLines = ((options & DONT_BREAK_LINES) == 0); + + int len43 = len * 4 / 3; + byte[] outBuff = + new byte[(len43) // Main 4:3 + + ((len % 3) > 0 ? 4 : 0) // padding + + (breakLines ? (len43 / MAX_LINE_LENGTH) : 0)]; // New lines + int d = 0; + int e = 0; + int len2 = len - 2; + int lineLength = 0; + for (; d < len2; d += 3, e += 4) { + encode3to4(source, d + off, 3, outBuff, e, options); + + lineLength += 4; + if (breakLines && lineLength == MAX_LINE_LENGTH) { + outBuff[e + 4] = NEW_LINE; + e++; + lineLength = 0; + } // end if: end of line + } // end for: each piece of array + + if (d < len) { + encode3to4(source, d + off, len - d, outBuff, e, options); + e += 4; + } // end if: some padding needed + + // Return value according to relevant encoding. + try { + return new String(outBuff, 0, e, PREFERRED_ENCODING); + + } catch (UnsupportedEncodingException uue) { + return new String(outBuff, 0, e); + } + } // end encodeBytes + + /* ******** D E C O D I N G M E T H O D S ******** */ + + /** + * Decodes four bytes from array source and writes the resulting + * bytes (up to three of them) to destination. The source and + * destination arrays can be manipulated anywhere along their length by + * specifying srcOffset and destOffset. This method + * does not check to make sure your arrays are large enough to accomodate + * srcOffset + 4 for the source array or + * destOffset + 3 for the destination array. This + * method returns the actual number of bytes that were converted from the + * Base64 encoding. + *

    + * This is the lowest level of the decoding methods with all possible + * parameters. + *

    + * + * @param source the array to convert + * @param srcOffset the index where conversion begins + * @param destination the array to hold the conversion + * @param destOffset the index where output will be put + * @param options options for getDecoabet + * @see Base64#URL_SAFE + * @see Base64#ORDERED + * @return the number of decoded bytes converted + * @since 1.3 + */ + @SuppressWarnings({"ConstantConditions"}) + protected static int decode4to3(byte[] source, int srcOffset, + byte[] destination, int destOffset, int options) { + byte[] DECODABET = getDecodabet(options); + + if (source[srcOffset + 2] == EQUALS_SIGN) { // Example: Dk== + // Two ways to do the same thing. Don't know which way I like best. + // int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) + // | ( ( DECODABET[ source[ srcOffset + 1] ] << 24 ) >>> 12 ); + int outBuff = + ((DECODABET[source[srcOffset]] & 0xFF) << 18) + | ((DECODABET[source[srcOffset + 1]] & 0xFF) << 12); + + destination[destOffset] = (byte) (outBuff >>> 16); + return 1; + + } else if (source[srcOffset + 3] == EQUALS_SIGN) { // Example: DkL= + // Two ways to do the same thing. Don't know which way I like best. + // int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) + // | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 ) + // | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 ); + int outBuff = + ((DECODABET[source[srcOffset]] & 0xFF) << 18) + | ((DECODABET[source[srcOffset + 1]] & 0xFF) << 12) + | ((DECODABET[source[srcOffset + 2]] & 0xFF) << 6); + + destination[destOffset] = (byte) (outBuff >>> 16); + destination[destOffset + 1] = (byte) (outBuff >>> 8); + return 2; + + } else { // Example: DkLE + try { + // Two ways to do the same thing. Don't know which way I like best. + // int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) + // | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 ) + // | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 ) + // | ( ( DECODABET[ source[ srcOffset + 3 ] ] << 24 ) >>> 24 ); + int outBuff = + ((DECODABET[source[srcOffset]] & 0xFF) << 18) + | ((DECODABET[source[srcOffset + 1]] & 0xFF) << 12) + | ((DECODABET[source[srcOffset + 2]] & 0xFF) << 6) + | ((DECODABET[source[srcOffset + 3]] & 0xFF)); + + destination[destOffset] = (byte) (outBuff >> 16); + destination[destOffset + 1] = (byte) (outBuff >> 8); + destination[destOffset + 2] = (byte) (outBuff); + + return 3; + + } catch (Exception e) { + LOG.error("error decoding bytes at " + source[srcOffset] + ": " + + (DECODABET[source[srcOffset]]) + ", " + source[srcOffset + 1] + + ": " + (DECODABET[source[srcOffset + 1]]) + ", " + + source[srcOffset + 2] + ": " + (DECODABET[source[srcOffset + 2]]) + + ", " + source[srcOffset + 3] + ": " + + (DECODABET[source[srcOffset + 3]]), e); + return -1; + } // end catch + } + } // end decodeToBytes + + /** + * Very low-level access to decoding ASCII characters in the form of a byte + * array. Does not support automatically gunzipping or any other "fancy" + * features. + * + * @param source The Base64 encoded data + * @param off The offset of where to begin decoding + * @param len The length of characters to decode + * @param options options for getDecodabet + * @see Base64#URL_SAFE + * @see Base64#ORDERED + * @return decoded data + * @since 1.3 + */ + public static byte[] decode(byte[] source, int off, int len, int options) { + byte[] DECODABET = getDecodabet(options); + + int len34 = len * 3 / 4; + byte[] outBuff = new byte[len34]; // Upper limit on size of output + int outBuffPosn = 0; + + byte[] b4 = new byte[4]; + int b4Posn = 0; + int i; + byte sbiCrop; + byte sbiDecode; + for (i = off; i < off + len; i++) { + sbiCrop = (byte) (source[i] & 0x7f); // Only the low seven bits + sbiDecode = DECODABET[sbiCrop]; + + if (sbiDecode >= WHITE_SPACE_ENC) { // Whitespace, Equals or better + if (sbiDecode >= EQUALS_SIGN_ENC) { // Equals or better + b4[b4Posn++] = sbiCrop; + if (b4Posn > 3) { + outBuffPosn += decode4to3(b4, 0, outBuff, outBuffPosn, options); + b4Posn = 0; + + // If that was the equals sign, break out of 'for' loop + if (sbiCrop == EQUALS_SIGN) + break; + } // end if: quartet built + } // end if: equals sign or better + } else { + LOG.error("Bad Base64 input character at " + i + ": " + source[i] + + "(decimal)"); + return null; + } // end else: + } // each input character + + byte[] out = new byte[outBuffPosn]; + System.arraycopy(outBuff, 0, out, 0, outBuffPosn); + return out; + } // end decode + + /** + * Decodes data from Base64 notation, automatically detecting gzip-compressed + * data and decompressing it. + * + * @param s the string to decode + * @return the decoded data + * @since 1.4 + */ + public static byte[] decode(String s) { + return decode(s, NO_OPTIONS); + } + + /** + * Decodes data from Base64 notation, automatically detecting gzip-compressed + * data and decompressing it. + * + * @param s the string to decode + * @param options options for decode + * @see Base64#URL_SAFE + * @see Base64#ORDERED + * @return the decoded data + * @since 1.4 + */ + public static byte[] decode(String s, int options) { + byte[] bytes; + try { + bytes = s.getBytes(PREFERRED_ENCODING); + + } catch (UnsupportedEncodingException uee) { + bytes = s.getBytes(); + } // end catch + + // Decode + + bytes = decode(bytes, 0, bytes.length, options); + + // Check to see if it's gzip-compressed + // GZIP Magic Two-Byte Number: 0x8b1f (35615) + + if (bytes != null && bytes.length >= 4) { + int head = (bytes[0] & 0xff) | ((bytes[1] << 8) & 0xff00); + if (GZIPInputStream.GZIP_MAGIC == head) { + GZIPInputStream gzis = null; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + gzis = new GZIPInputStream(new ByteArrayInputStream(bytes)); + + byte[] buffer = new byte[2048]; + for (int length; (length = gzis.read(buffer)) >= 0; ) { + baos.write(buffer, 0, length); + } // end while: reading input + + // No error? Get new bytes. + bytes = baos.toByteArray(); + + } catch (IOException e) { + // Just return originally-decoded bytes + + } finally { + try { + baos.close(); + } catch (Exception e) { + LOG.error("error closing ByteArrayOutputStream", e); + } + if (gzis != null) { + try { + gzis.close(); + } catch (Exception e) { + LOG.error("error closing GZIPInputStream", e); + } + } + } // end finally + } // end if: gzipped + } // end if: bytes.length >= 2 + + return bytes; + } // end decode + + /** + * Attempts to decode Base64 data and deserialize a Java Object within. + * Returns null if there was an error. + * + * @param encodedObject The Base64 data to decode + * @return The decoded and deserialized object + * @since 1.5 + */ + public static Object decodeToObject(String encodedObject) { + // Decode and gunzip if necessary + byte[] objBytes = decode(encodedObject); + + Object obj = null; + ObjectInputStream ois = null; + try { + ois = new ObjectInputStream(new ByteArrayInputStream(objBytes)); + obj = ois.readObject(); + + } catch (IOException e) { + LOG.error("error decoding object", e); + + } catch (ClassNotFoundException e) { + LOG.error("error decoding object", e); + + } finally { + if (ois != null) { + try { + ois.close(); + } catch (Exception e) { + LOG.error("error closing ObjectInputStream", e); + } + } + } // end finally + + return obj; + } // end decodeObject + + /** + * Convenience method for encoding data to a file. + * + * @param dataToEncode byte array of data to encode in base64 form + * @param filename Filename for saving encoded data + * @return true if successful, false otherwise + * + * @since 2.1 + */ + public static boolean encodeToFile(byte[] dataToEncode, String filename) { + boolean success = false; + Base64OutputStream bos = null; + try { + bos = new Base64OutputStream(new FileOutputStream(filename), ENCODE); + bos.write(dataToEncode); + success = true; + + } catch (IOException e) { + LOG.error("error encoding file: " + filename, e); + success = false; + + } finally { + if (bos != null) { + try { + bos.close(); + } catch (Exception e) { + LOG.error("error closing Base64OutputStream", e); + } + } + } // end finally + + return success; + } // end encodeToFile + + /** + * Convenience method for decoding data to a file. + * + * @param dataToDecode Base64-encoded data as a string + * @param filename Filename for saving decoded data + * @return true if successful, false otherwise + * + * @since 2.1 + */ + public static boolean decodeToFile(String dataToDecode, String filename) { + boolean success = false; + Base64OutputStream bos = null; + try { + bos = new Base64OutputStream(new FileOutputStream(filename), DECODE); + bos.write(dataToDecode.getBytes(PREFERRED_ENCODING)); + success = true; + + } catch (IOException e) { + LOG.error("error decoding to file: " + filename, e); + success = false; + + } finally { + if (bos != null) { + try { + bos.close(); + } catch (Exception e) { + LOG.error("error closing Base64OutputStream", e); + } + } + } // end finally + + return success; + } // end decodeToFile + + /** + * Convenience method for reading a base64-encoded file and decoding it. + * + * @param filename Filename for reading encoded data + * @return decoded byte array or null if unsuccessful + * + * @since 2.1 + */ + public static byte[] decodeFromFile(String filename) { + byte[] decodedData = null; + Base64InputStream bis = null; + try { + File file = new File(filename); + byte[] buffer; + + // Check the size of file + if (file.length() > Integer.MAX_VALUE) { + LOG.fatal("File is too big for this convenience method (" + + file.length() + " bytes)."); + return null; + } // end if: file too big for int index + + buffer = new byte[(int) file.length()]; + + // Open a stream + + bis = new Base64InputStream(new BufferedInputStream( + new FileInputStream(file)), DECODE); + + // Read until done + + int length = 0; + for (int numBytes; (numBytes = bis.read(buffer, length, 4096)) >= 0; ) { + length += numBytes; + } + + // Save in a variable to return + + decodedData = new byte[length]; + System.arraycopy(buffer, 0, decodedData, 0, length); + + } catch (IOException e) { + LOG.error("Error decoding from file " + filename, e); + + } finally { + if (bis != null) { + try { + bis.close(); + } catch (Exception e) { + LOG.error("error closing Base64InputStream", e); + } + } + } // end finally + + return decodedData; + } // end decodeFromFile + + /** + * Convenience method for reading a binary file and base64-encoding it. + * + * @param filename Filename for reading binary data + * @return base64-encoded string or null if unsuccessful + * + * @since 2.1 + */ + public static String encodeFromFile(String filename) { + String encodedData = null; + Base64InputStream bis = null; + try { + File file = new File(filename); + + // Need max() for math on small files (v2.2.1) + + byte[] buffer = new byte[Math.max((int) (file.length() * 1.4), 40)]; + + // Open a stream + + bis = new Base64InputStream(new BufferedInputStream( + new FileInputStream(file)), ENCODE); + + // Read until done + int length = 0; + for (int numBytes; (numBytes = bis.read(buffer, length, 4096)) >= 0; ) { + length += numBytes; + } + + // Save in a variable to return + + encodedData = new String(buffer, 0, length, PREFERRED_ENCODING); + + } catch (IOException e) { + LOG.error("Error encoding from file " + filename, e); + + } finally { + if (bis != null) { + try { + bis.close(); + } catch (Exception e) { + LOG.error("error closing Base64InputStream", e); + } + } + } // end finally + + return encodedData; + } // end encodeFromFile + + /** + * Reads infile and encodes it to outfile. + * + * @param infile Input file + * @param outfile Output file + * @since 2.2 + */ + public static void encodeFileToFile(String infile, String outfile) { + String encoded = encodeFromFile(infile); + OutputStream out = null; + try { + out = new BufferedOutputStream(new FileOutputStream(outfile)); + out.write(encoded.getBytes("US-ASCII")); // Strict, 7-bit output. + + } catch (IOException e) { + LOG.error("error encoding from file " + infile + " to " + outfile, e); + + } finally { + if (out != null) { + try { + out.close(); + } catch (Exception e) { + LOG.error("error closing " + outfile, e); + } + } + } // end finally + } // end encodeFileToFile + + /** + * Reads infile and decodes it to outfile. + * + * @param infile Input file + * @param outfile Output file + * @since 2.2 + */ + public static void decodeFileToFile(String infile, String outfile) { + byte[] decoded = decodeFromFile(infile); + OutputStream out = null; + try { + out = new BufferedOutputStream(new FileOutputStream(outfile)); + out.write(decoded); + + } catch (IOException e) { + LOG.error("error decoding from file " + infile + " to " + outfile, e); + + } finally { + if (out != null) { + try { + out.close(); + } catch (Exception e) { + LOG.error("error closing " + outfile, e); + } + } + } // end finally + } // end decodeFileToFile + + /* ******** I N N E R C L A S S I N P U T S T R E A M ******** */ + + /** + * A {@link Base64.Base64InputStream} will read data from another + * InputStream, given in the constructor, and + * encode/decode to/from Base64 notation on the fly. + * + * @see Base64 + * @since 1.3 + */ + public static class Base64InputStream extends FilterInputStream { + private boolean encode; // Encoding or decoding + private int position; // Current position in the buffer + private byte[] buffer; // Buffer holding converted data + private int bufferLength; // Length of buffer (3 or 4) + private int numSigBytes; // Meaningful bytes in the buffer + private int lineLength; + private boolean breakLines; // Break lines at < 80 characters + private int options; // Record options + private byte[] decodabet; // Local copy avoids method calls + + /** + * Constructs a {@link Base64InputStream} in DECODE mode. + * + * @param in the InputStream from which to read data. + * @since 1.3 + */ + public Base64InputStream(InputStream in) { + this(in, DECODE); + } // end constructor + + /** + * Constructs a {@link Base64.Base64InputStream} in either ENCODE or DECODE mode. + *

    + * Valid options: + * + * 

    +     *   ENCODE or DECODE: Encode or Decode as data is read.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     (only meaningful when encoding)
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     *
    + * + *

    + * Example: new Base64.Base64InputStream( in, Base64.DECODE ) + * + * + * @param in the InputStream from which to read data. + * @param options Specified options + * @see Base64#ENCODE + * @see Base64#DECODE + * @see Base64#DONT_BREAK_LINES + * @since 2.0 + */ + public Base64InputStream(InputStream in, int options) { + super(in); + this.breakLines = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES; + this.encode = (options & ENCODE) == ENCODE; + this.bufferLength = encode ? 4 : 3; + this.buffer = new byte[bufferLength]; + this.position = -1; + this.lineLength = 0; + this.options = options; // Record for later, mostly to determine which + // alphabet to use + this.decodabet = getDecodabet(options); + } // end constructor + + /** + * Reads enough of the input stream to convert to/from Base64 and returns + * the next byte. + * + * @return next byte + * @since 1.3 + */ + @Override + public int read() throws IOException { + // Do we need to get data? + if (position < 0) { + if (encode) { + byte[] b3 = new byte[3]; + int numBinaryBytes = 0; + for (int i = 0; i < 3; i++) { + try { + int b = in.read(); + + // If end of stream, b is -1. + if (b >= 0) { + b3[i] = (byte) b; + numBinaryBytes++; + } // end if: not end of stream + + } catch (IOException e) { + // Only a problem if we got no data at all. + if (i == 0) + throw e; + + } // end catch + } // end for: each needed input byte + + if (numBinaryBytes > 0) { + encode3to4(b3, 0, numBinaryBytes, buffer, 0, options); + position = 0; + numSigBytes = 4; + + } else { + return -1; + } // end else + + } else { + byte[] b4 = new byte[4]; + int i; + for (i = 0; i < 4; i++) { + // Read four "meaningful" bytes: + int b; + do { + b = in.read(); + } while (b >= 0 && decodabet[b & 0x7f] <= WHITE_SPACE_ENC); + + if (b < 0) { + break; // Reads a -1 if end of stream + } + + b4[i] = (byte) b; + } // end for: each needed input byte + + if (i == 4) { + numSigBytes = decode4to3(b4, 0, buffer, 0, options); + position = 0; + + } else if (i == 0) { + return -1; + + } else { + // Must have broken out from above. + throw new IOException("Improperly padded Base64 input."); + } // end + } // end else: decode + } // end else: get data + + // Got data? + if (position >= 0) { + // End of relevant data? + if ( /* !encode && */position >= numSigBytes) { + return -1; + } + + if (encode && breakLines && lineLength >= MAX_LINE_LENGTH) { + lineLength = 0; + return '\n'; + + } + lineLength++; // This isn't important when decoding + // but throwing an extra "if" seems + // just as wasteful. + + int b = buffer[position++]; + + if (position >= bufferLength) + position = -1; + + return b & 0xFF; // This is how you "cast" a byte that's + // intended to be unsigned. + + } + + // When JDK1.4 is more accepted, use an assertion here. + throw new IOException("Error in Base64 code reading stream."); + + } // end read + + /** + * Calls {@link #read()} repeatedly until the end of stream is reached or + * len bytes are read. Returns number of bytes read into array + * or -1 if end of stream is encountered. + * + * @param dest array to hold values + * @param off offset for array + * @param len max number of bytes to read into array + * @return bytes read into array or -1 if end of stream is encountered. + * @since 1.3 + */ + @Override + public int read(byte[] dest, int off, int len) throws IOException { + int i; + int b; + for (i = 0; i < len; i++) { + b = read(); + if (b >= 0) { + dest[off + i] = (byte) b; + } else if (i == 0) { + return -1; + } else { + break; // Out of 'for' loop + } + } // end for: each byte read + return i; + } // end read + + } // end inner class InputStream + + /* ******** I N N E R C L A S S O U T P U T S T R E A M ******** */ + + /** + * A {@link Base64.Base64OutputStream} will write data to another + * OutputStream, given in the constructor, and + * encode/decode to/from Base64 notation on the fly. + * + * @see Base64 + * @since 1.3 + */ + public static class Base64OutputStream extends FilterOutputStream { + private boolean encode; + private int position; + private byte[] buffer; + private int bufferLength; + private int lineLength; + private boolean breakLines; + private byte[] b4; // Scratch used in a few places + private boolean suspendEncoding; + private int options; // Record for later + private byte[] decodabet; // Local copy avoids method calls + + /** + * Constructs a {@link Base64OutputStream} in ENCODE mode. + * + * @param out the OutputStream to which data will be written. + * @since 1.3 + */ + public Base64OutputStream(OutputStream out) { + this(out, ENCODE); + } // end constructor + + /** + * Constructs a {@link Base64OutputStream} in either ENCODE or DECODE mode. + *

    + * Valid options: + * + *

      + *
    • ENCODE or DECODE: Encode or Decode as data is read.
      • + *
    • DONT_BREAK_LINES: don't break lines at 76 characters (only + * meaningful when encoding) Note: Technically, this makes your + * encoding non-compliant.
      • + *
    + * + *

    + * Example: new Base64.Base64OutputStream( out, Base64.ENCODE ) + * + * @param out the OutputStream to which data will be written. + * @param options Specified options. + * @see Base64#ENCODE + * @see Base64#DECODE + * @see Base64#DONT_BREAK_LINES + * @since 1.3 + */ + public Base64OutputStream(OutputStream out, int options) { + super(out); + this.breakLines = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES; + this.encode = (options & ENCODE) == ENCODE; + this.bufferLength = encode ? 3 : 4; + this.buffer = new byte[bufferLength]; + this.position = 0; + this.lineLength = 0; + this.suspendEncoding = false; + this.b4 = new byte[4]; + this.options = options; + this.decodabet = getDecodabet(options); + } // end constructor + + /** + * Writes the byte to the output stream after converting to/from Base64 + * notation. When encoding, bytes are buffered three at a time before the + * output stream actually gets a write() call. When decoding, bytes are + * buffered four at a time. + * + * @param theByte the byte to write + * @since 1.3 + */ + @Override + public void write(int theByte) throws IOException { + // Encoding suspended? + if (suspendEncoding) { + super.out.write(theByte); + return; + } // end if: supsended + + // Encode? + if (encode) { + buffer[position++] = (byte) theByte; + if (position >= bufferLength) { // Enough to encode. + out.write(encode3to4(b4, buffer, bufferLength, options)); + lineLength += 4; + if (breakLines && lineLength >= MAX_LINE_LENGTH) { + out.write(NEW_LINE); + lineLength = 0; + } // end if: end of line + + position = 0; + } // end if: enough to output + + } else { + // Meaningful Base64 character? + if (decodabet[theByte & 0x7f] > WHITE_SPACE_ENC) { + buffer[position++] = (byte) theByte; + if (position >= bufferLength) { // Enough to output. + int len = decode4to3(buffer, 0, b4, 0, options); + out.write(b4, 0, len); + position = 0; + } // end if: enough to output + + } else if (decodabet[theByte & 0x7f] != WHITE_SPACE_ENC) { + throw new IOException("Invalid character in Base64 data."); + } // end else: not white space either + } // end else: decoding + } // end write + + /** + * Calls {@link #write(int)} repeatedly until len bytes are + * written. + * + * @param theBytes array from which to read bytes + * @param off offset for array + * @param len max number of bytes to read into array + * @since 1.3 + */ + @Override + public void write(byte[] theBytes, int off, int len) throws IOException { + // Encoding suspended? + if (suspendEncoding) { + super.out.write(theBytes, off, len); + return; + } // end if: supsended + + for (int i = 0; i < len; i++) { + write(theBytes[off + i]); + } // end for: each byte written + + } // end write + + /** + * Method added by PHIL. [Thanks, PHIL. -Rob] This pads the buffer without + * closing the stream. + * + * @throws IOException e + */ + public void flushBase64() throws IOException { + if (position > 0) { + if (encode) { + out.write(encode3to4(b4, buffer, position, options)); + position = 0; + + } else { + throw new IOException("Base64 input not properly padded."); + } // end else: decoding + } // end if: buffer partially full + + } // end flush + + /** + * Flushes and closes (I think, in the superclass) the stream. + * + * @since 1.3 + */ + @Override + public void close() throws IOException { + // 1. Ensure that pending characters are written + flushBase64(); + + // 2. Actually close the stream + // Base class both flushes and closes. + super.close(); + + buffer = null; + out = null; + } // end close + + /** + * Suspends encoding of the stream. May be helpful if you need to embed a + * piece of base640-encoded data in a stream. + * + * @throws IOException e + * @since 1.5.1 + */ + public void suspendEncoding() throws IOException { + flushBase64(); + this.suspendEncoding = true; + } // end suspendEncoding + + /** + * Resumes encoding of the stream. May be helpful if you need to embed a + * piece of base640-encoded data in a stream. + * + * @since 1.5.1 + */ + public void resumeEncoding() { + this.suspendEncoding = false; + } // end resumeEncoding + + } // end inner class OutputStream + +} // end class Base64 diff --git a/src/main/java/org/apache/hadoop/hbase/util/BloomFilter.java b/src/main/java/org/apache/hadoop/hbase/util/BloomFilter.java new file mode 100644 index 0000000..7dd2f68 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/BloomFilter.java @@ -0,0 +1,61 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.nio.ByteBuffer; + +/** + * Defines the general behavior of a bloom filter. + * + *

    + * The Bloom filter is a data structure that was introduced in 1970 and that + * has been adopted by the networking research community in the past decade + * thanks to the bandwidth efficiencies that it offers for the transmission of + * set membership information between networked hosts. A sender encodes the + * information into a bit vector, the Bloom filter, that is more compact than a + * conventional representation. Computation and space costs for construction + * are linear in the number of elements. The receiver uses the filter to test + * whether various elements are members of the set. Though the filter will + * occasionally return a false positive, it will never return a false negative. + * When creating the filter, the sender can choose its desired point in a + * trade-off between the false positive rate and the size. + * + * @see BloomFilterWriter for the ability to add elements to a Bloom filter + */ +public interface BloomFilter extends BloomFilterBase { + + /** + * Check if the specified key is contained in the bloom filter. + * + * @param buf data to check for existence of + * @param offset offset into the data + * @param length length of the data + * @param bloom bloom filter data to search. This can be null if auto-loading + * is supported. + * @return true if matched by bloom, false if not + */ + boolean contains(byte [] buf, int offset, int length, ByteBuffer bloom); + + /** + * @return true if this Bloom filter can automatically load its data + * and thus allows a null byte buffer to be passed to contains() + */ + boolean supportsAutoLoading(); +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/util/BloomFilterBase.java b/src/main/java/org/apache/hadoop/hbase/util/BloomFilterBase.java new file mode 100644 index 0000000..ab218b7 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/BloomFilterBase.java @@ -0,0 +1,56 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.io.RawComparator; + +/** + * Common methods Bloom filter methods required at read and write time. + */ +public interface BloomFilterBase { + + /** + * @return The number of keys added to the bloom + */ + long getKeyCount(); + + /** + * @return The max number of keys that can be inserted + * to maintain the desired error rate + */ + long getMaxKeys(); + + /** + * @return Size of the bloom, in bytes + */ + long getByteSize(); + + /** + * Create a key for a row-column Bloom filter. + */ + byte[] createBloomKey(byte[] rowBuf, int rowOffset, int rowLen, + byte[] qualBuf, int qualOffset, int qualLen); + + /** + * @return Bloom key comparator + */ + RawComparator getComparator(); + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java b/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java new file mode 100644 index 0000000..418bd16 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java @@ -0,0 +1,266 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.DataInput; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; + +/** + * Handles Bloom filter initialization based on configuration and serialized + * metadata in the reader and writer of {@link StoreFile}. + */ +public final class BloomFilterFactory { + + private static final Log LOG = + LogFactory.getLog(BloomFilterFactory.class.getName()); + + /** This class should not be instantiated. */ + private BloomFilterFactory() {} + + /** + * Specifies the target error rate to use when selecting the number of keys + * per Bloom filter. + */ + public static final String IO_STOREFILE_BLOOM_ERROR_RATE = + "io.storefile.bloom.error.rate"; + + /** + * Maximum folding factor allowed. The Bloom filter will be shrunk by + * the factor of up to 2 ** this times if we oversize it initially. + */ + public static final String IO_STOREFILE_BLOOM_MAX_FOLD = + "io.storefile.bloom.max.fold"; + + /** + * For default (single-block) Bloom filters this specifies the maximum number + * of keys. + */ + public static final String IO_STOREFILE_BLOOM_MAX_KEYS = + "io.storefile.bloom.max.keys"; + + /** Master switch to enable Bloom filters */ + public static final String IO_STOREFILE_BLOOM_ENABLED = + "io.storefile.bloom.enabled"; + + /** Master switch to enable Delete Family Bloom filters */ + public static final String IO_STOREFILE_DELETEFAMILY_BLOOM_ENABLED = + "io.storefile.delete.family.bloom.enabled"; + + /** + * Target Bloom block size. Bloom filter blocks of approximately this size + * are interleaved with data blocks. + */ + public static final String IO_STOREFILE_BLOOM_BLOCK_SIZE = + "io.storefile.bloom.block.size"; + + /** Maximum number of times a Bloom filter can be "folded" if oversized */ + private static final int MAX_ALLOWED_FOLD_FACTOR = 7; + + /** + * Instantiates the correct Bloom filter class based on the version provided + * in the meta block data. + * + * @param meta the byte array holding the Bloom filter's metadata, including + * version information + * @param reader the {@link HFile} reader to use to lazily load Bloom filter + * blocks + * @return an instance of the correct type of Bloom filter + * @throws IllegalArgumentException + */ + public static BloomFilter + createFromMeta(DataInput meta, HFile.Reader reader) + throws IllegalArgumentException, IOException { + int version = meta.readInt(); + switch (version) { + case ByteBloomFilter.VERSION: + // This is only possible in a version 1 HFile. We are ignoring the + // passed comparator because raw byte comparators are always used + // in version 1 Bloom filters. + return new ByteBloomFilter(meta); + + case CompoundBloomFilterBase.VERSION: + return new CompoundBloomFilter(meta, reader); + + default: + throw new IllegalArgumentException( + "Bad bloom filter format version " + version + ); + } + } + + /** + * @return true if general Bloom (Row or RowCol) filters are enabled in the + * given configuration + */ + public static boolean isGeneralBloomEnabled(Configuration conf) { + return conf.getBoolean(IO_STOREFILE_BLOOM_ENABLED, true); + } + + /** + * @return true if Delete Family Bloom filters are enabled in the given configuration + */ + public static boolean isDeleteFamilyBloomEnabled(Configuration conf) { + return conf.getBoolean(IO_STOREFILE_DELETEFAMILY_BLOOM_ENABLED, true); + } + + /** + * @return the Bloom filter error rate in the given configuration + */ + public static float getErrorRate(Configuration conf) { + return conf.getFloat(IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01); + } + + /** + * @return the value for Bloom filter max fold in the given configuration + */ + public static int getMaxFold(Configuration conf) { + return conf.getInt(IO_STOREFILE_BLOOM_MAX_FOLD, MAX_ALLOWED_FOLD_FACTOR); + } + + /** @return the compound Bloom filter block size from the configuration */ + public static int getBloomBlockSize(Configuration conf) { + return conf.getInt(IO_STOREFILE_BLOOM_BLOCK_SIZE, 128 * 1024); + } + + /** + * @return max key for the Bloom filter from the configuration + */ + public static int getMaxKeys(Configuration conf) { + return conf.getInt(IO_STOREFILE_BLOOM_MAX_KEYS, 128 * 1000 * 1000); + } + + /** + * Creates a new general (Row or RowCol) Bloom filter at the time of + * {@link org.apache.hadoop.hbase.regionserver.StoreFile} writing. + * + * @param conf + * @param cacheConf + * @param bloomType + * @param maxKeys an estimate of the number of keys we expect to insert. + * Irrelevant if compound Bloom filters are enabled. + * @param writer the HFile writer + * @return the new Bloom filter, or null in case Bloom filters are disabled + * or when failed to create one. + */ + public static BloomFilterWriter createGeneralBloomAtWrite(Configuration conf, + CacheConfig cacheConf, BloomType bloomType, int maxKeys, + HFile.Writer writer) { + if (!isGeneralBloomEnabled(conf)) { + LOG.trace("Bloom filters are disabled by configuration for " + + writer.getPath() + + (conf == null ? " (configuration is null)" : "")); + return null; + } else if (bloomType == BloomType.NONE) { + LOG.trace("Bloom filter is turned off for the column family"); + return null; + } + + float err = getErrorRate(conf); + + // In case of row/column Bloom filter lookups, each lookup is an OR if two + // separate lookups. Therefore, if each lookup's false positive rate is p, + // the resulting false positive rate is err = 1 - (1 - p)^2, and + // p = 1 - sqrt(1 - err). + if (bloomType == BloomType.ROWCOL) { + err = (float) (1 - Math.sqrt(1 - err)); + } + + int maxFold = conf.getInt(IO_STOREFILE_BLOOM_MAX_FOLD, + MAX_ALLOWED_FOLD_FACTOR); + + // Do we support compound bloom filters? + if (HFile.getFormatVersion(conf) > HFile.MIN_FORMAT_VERSION) { + // In case of compound Bloom filters we ignore the maxKeys hint. + CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter( + getBloomBlockSize(conf), err, Hash.getHashType(conf), maxFold, + cacheConf.shouldCacheBloomsOnWrite(), bloomType == BloomType.ROWCOL + ? KeyValue.KEY_COMPARATOR : Bytes.BYTES_RAWCOMPARATOR); + writer.addInlineBlockWriter(bloomWriter); + return bloomWriter; + } else { + // A single-block Bloom filter. Only used when testing HFile format + // version 1. + int tooBig = conf.getInt(IO_STOREFILE_BLOOM_MAX_KEYS, + 128 * 1000 * 1000); + + if (maxKeys <= 0) { + LOG.warn("Invalid maximum number of keys specified: " + maxKeys + + ", not using Bloom filter"); + return null; + } else if (maxKeys < tooBig) { + BloomFilterWriter bloom = new ByteBloomFilter((int) maxKeys, err, + Hash.getHashType(conf), maxFold); + bloom.allocBloom(); + return bloom; + } else { + LOG.debug("Skipping bloom filter because max keysize too large: " + + maxKeys); + } + } + return null; + } + + /** + * Creates a new Delete Family Bloom filter at the time of + * {@link org.apache.hadoop.hbase.regionserver.StoreFile} writing. + * @param conf + * @param cacheConf + * @param maxKeys an estimate of the number of keys we expect to insert. + * Irrelevant if compound Bloom filters are enabled. + * @param writer the HFile writer + * @return the new Bloom filter, or null in case Bloom filters are disabled + * or when failed to create one. + */ + public static BloomFilterWriter createDeleteBloomAtWrite(Configuration conf, + CacheConfig cacheConf, int maxKeys, HFile.Writer writer) { + if (!isDeleteFamilyBloomEnabled(conf)) { + LOG.info("Delete Bloom filters are disabled by configuration for " + + writer.getPath() + + (conf == null ? " (configuration is null)" : "")); + return null; + } + + float err = getErrorRate(conf); + + if (HFile.getFormatVersion(conf) > HFile.MIN_FORMAT_VERSION) { + int maxFold = getMaxFold(conf); + // In case of compound Bloom filters we ignore the maxKeys hint. + CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter( + getBloomBlockSize(conf), err, Hash.getHashType(conf), + maxFold, + cacheConf.shouldCacheBloomsOnWrite(), Bytes.BYTES_RAWCOMPARATOR); + writer.addInlineBlockWriter(bloomWriter); + return bloomWriter; + } else { + LOG.info("Delete Family Bloom filter is not supported in HFile V1"); + return null; + } + } +}; diff --git a/src/main/java/org/apache/hadoop/hbase/util/BloomFilterWriter.java b/src/main/java/org/apache/hadoop/hbase/util/BloomFilterWriter.java new file mode 100644 index 0000000..46691fb --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/BloomFilterWriter.java @@ -0,0 +1,61 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.io.Writable; + +/** + * Specifies methods needed to add elements to a Bloom filter and serialize the + * resulting Bloom filter as a sequence of bytes. + */ +public interface BloomFilterWriter extends BloomFilterBase { + + /** Allocate memory for the bloom filter data. */ + void allocBloom(); + + /** Compact the Bloom filter before writing metadata & data to disk. */ + void compactBloom(); + + /** + * Get a writable interface into bloom filter meta data. + * + * @return a writable instance that can be later written to a stream + */ + Writable getMetaWriter(); + + /** + * Get a writable interface into bloom filter data (the actual Bloom bits). + * Not used for compound Bloom filters. + * + * @return a writable instance that can be later written to a stream + */ + Writable getDataWriter(); + + /** + * Add the specified binary to the bloom filter. + * + * @param buf data to be added to the bloom + * @param offset offset into the data to be added + * @param len length of the data to be added + */ + void add(byte[] buf, int offset, int len); + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java b/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java new file mode 100644 index 0000000..7f666ac --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java @@ -0,0 +1,654 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.io.RawComparator; +import org.apache.hadoop.io.Writable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.text.NumberFormat; +import java.util.Random; + +/** + * Implements a Bloom filter, as defined by Bloom in 1970. + *

    + * The Bloom filter is a data structure that was introduced in 1970 and that has + * been adopted by the networking research community in the past decade thanks + * to the bandwidth efficiencies that it offers for the transmission of set + * membership information between networked hosts. A sender encodes the + * information into a bit vector, the Bloom filter, that is more compact than a + * conventional representation. Computation and space costs for construction are + * linear in the number of elements. The receiver uses the filter to test + * whether various elements are members of the set. Though the filter will + * occasionally return a false positive, it will never return a false negative. + * When creating the filter, the sender can choose its desired point in a + * trade-off between the false positive rate and the size. + * + *

    + * Originally inspired by European Commission + * One-Lab Project 034819. + * + * Bloom filters are very sensitive to the number of elements inserted into + * them. For HBase, the number of entries depends on the size of the data stored + * in the column. Currently the default region size is 256MB, so entry count ~= + * 256MB / (average value size for column). Despite this rule of thumb, there is + * no efficient way to calculate the entry count after compactions. Therefore, + * it is often easier to use a dynamic bloom filter that will add extra space + * instead of allowing the error rate to grow. + * + * ( http://www.eecs.harvard.edu/~michaelm/NEWWORK/postscripts/BloomFilterSurvey + * .pdf ) + * + * m denotes the number of bits in the Bloom filter (bitSize) n denotes the + * number of elements inserted into the Bloom filter (maxKeys) k represents the + * number of hash functions used (nbHash) e represents the desired false + * positive rate for the bloom (err) + * + * If we fix the error rate (e) and know the number of entries, then the optimal + * bloom size m = -(n * ln(err) / (ln(2)^2) ~= n * ln(err) / ln(0.6185) + * + * The probability of false positives is minimized when k = m/n ln(2). + * + * @see BloomFilter The general behavior of a filter + * + * @see + * Space/Time Trade-Offs in Hash Coding with Allowable Errors + */ +public class ByteBloomFilter implements BloomFilter, BloomFilterWriter { + + /** Current file format version */ + public static final int VERSION = 1; + + /** Bytes (B) in the array. This actually has to fit into an int. */ + protected long byteSize; + /** Number of hash functions */ + protected int hashCount; + /** Hash type */ + protected final int hashType; + /** Hash Function */ + protected final Hash hash; + /** Keys currently in the bloom */ + protected int keyCount; + /** Max Keys expected for the bloom */ + protected int maxKeys; + /** Bloom bits */ + protected ByteBuffer bloom; + + /** Record separator for the Bloom filter statistics human-readable string */ + public static final String STATS_RECORD_SEP = "; "; + + /** + * Used in computing the optimal Bloom filter size. This approximately equals + * 0.480453. + */ + public static final double LOG2_SQUARED = Math.log(2) * Math.log(2); + + /** + * A random number generator to use for "fake lookups" when testing to + * estimate the ideal false positive rate. + */ + private static Random randomGeneratorForTest; + + /** Bit-value lookup array to prevent doing the same work over and over */ + private static final byte [] bitvals = { + (byte) 0x01, + (byte) 0x02, + (byte) 0x04, + (byte) 0x08, + (byte) 0x10, + (byte) 0x20, + (byte) 0x40, + (byte) 0x80 + }; + + /** + * Loads bloom filter meta data from file input. + * @param meta stored bloom meta data + * @throws IllegalArgumentException meta data is invalid + */ + public ByteBloomFilter(DataInput meta) + throws IOException, IllegalArgumentException { + this.byteSize = meta.readInt(); + this.hashCount = meta.readInt(); + this.hashType = meta.readInt(); + this.keyCount = meta.readInt(); + this.maxKeys = this.keyCount; + + this.hash = Hash.getInstance(this.hashType); + if (hash == null) { + throw new IllegalArgumentException("Invalid hash type: " + hashType); + } + sanityCheck(); + } + + /** + * @param maxKeys + * @param errorRate + * @return the number of bits for a Bloom filter than can hold the given + * number of keys and provide the given error rate, assuming that the + * optimal number of hash functions is used and it does not have to + * be an integer. + */ + public static long computeBitSize(long maxKeys, double errorRate) { + return (long) Math.ceil(maxKeys * (-Math.log(errorRate) / LOG2_SQUARED)); + } + + /** + * The maximum number of keys we can put into a Bloom filter of a certain + * size to maintain the given error rate, assuming the number of hash + * functions is chosen optimally and does not even have to be an integer + * (hence the "ideal" in the function name). + * + * @param bitSize + * @param errorRate + * @return maximum number of keys that can be inserted into the Bloom filter + * @see #computeMaxKeys(long, double, int) for a more precise estimate + */ + public static long idealMaxKeys(long bitSize, double errorRate) { + // The reason we need to use floor here is that otherwise we might put + // more keys in a Bloom filter than is allowed by the target error rate. + return (long) (bitSize * (LOG2_SQUARED / -Math.log(errorRate))); + } + + /** + * The maximum number of keys we can put into a Bloom filter of a certain + * size to get the given error rate, with the given number of hash functions. + * + * @param bitSize + * @param errorRate + * @param hashCount + * @return the maximum number of keys that can be inserted in a Bloom filter + * to maintain the target error rate, if the number of hash functions + * is provided. + */ + public static long computeMaxKeys(long bitSize, double errorRate, + int hashCount) { + return (long) (-bitSize * 1.0 / hashCount * + Math.log(1 - Math.exp(Math.log(errorRate) / hashCount))); + } + + /** + * Computes the error rate for this Bloom filter, taking into account the + * actual number of hash functions and keys inserted. The return value of + * this function changes as a Bloom filter is being populated. Used for + * reporting the actual error rate of compound Bloom filters when writing + * them out. + * + * @return error rate for this particular Bloom filter + */ + public double actualErrorRate() { + return actualErrorRate(keyCount, byteSize * 8, hashCount); + } + + /** + * Computes the actual error rate for the given number of elements, number + * of bits, and number of hash functions. Taken directly from the + * Wikipedia Bloom filter article. + * + * @param maxKeys + * @param bitSize + * @param functionCount + * @return the actual error rate + */ + public static double actualErrorRate(long maxKeys, long bitSize, + int functionCount) { + return Math.exp(Math.log(1 - Math.exp(-functionCount * maxKeys * 1.0 + / bitSize)) * functionCount); + } + + /** + * Increases the given byte size of a Bloom filter until it can be folded by + * the given factor. + * + * @param bitSize + * @param foldFactor + * @return Foldable byte size + */ + public static int computeFoldableByteSize(long bitSize, int foldFactor) { + long byteSizeLong = (bitSize + 7) / 8; + int mask = (1 << foldFactor) - 1; + if ((mask & byteSizeLong) != 0) { + byteSizeLong >>= foldFactor; + ++byteSizeLong; + byteSizeLong <<= foldFactor; + } + if (byteSizeLong > Integer.MAX_VALUE) { + throw new IllegalArgumentException("byteSize=" + byteSizeLong + " too " + + "large for bitSize=" + bitSize + ", foldFactor=" + foldFactor); + } + return (int) byteSizeLong; + } + + private static int optimalFunctionCount(int maxKeys, long bitSize) { + return (int) Math.ceil(Math.log(2) * (bitSize / maxKeys)); + } + + /** Private constructor used by other constructors. */ + private ByteBloomFilter(int hashType) { + this.hashType = hashType; + this.hash = Hash.getInstance(hashType); + } + + /** + * Determines & initializes bloom filter meta data from user config. Call + * {@link #allocBloom()} to allocate bloom filter data. + * + * @param maxKeys Maximum expected number of keys that will be stored in this + * bloom + * @param errorRate Desired false positive error rate. Lower rate = more + * storage required + * @param hashType Type of hash function to use + * @param foldFactor When finished adding entries, you may be able to 'fold' + * this bloom to save space. Tradeoff potentially excess bytes in + * bloom for ability to fold if keyCount is exponentially greater + * than maxKeys. + * @throws IllegalArgumentException + */ + public ByteBloomFilter(int maxKeys, double errorRate, int hashType, + int foldFactor) throws IllegalArgumentException { + this(hashType); + + long bitSize = computeBitSize(maxKeys, errorRate); + hashCount = optimalFunctionCount(maxKeys, bitSize); + this.maxKeys = maxKeys; + + // increase byteSize so folding is possible + byteSize = computeFoldableByteSize(bitSize, foldFactor); + + sanityCheck(); + } + + /** + * Creates a Bloom filter of the given size. + * + * @param byteSizeHint the desired number of bytes for the Bloom filter bit + * array. Will be increased so that folding is possible. + * @param errorRate target false positive rate of the Bloom filter + * @param hashType Bloom filter hash function type + * @param foldFactor + * @return the new Bloom filter of the desired size + */ + public static ByteBloomFilter createBySize(int byteSizeHint, + double errorRate, int hashType, int foldFactor) { + ByteBloomFilter bbf = new ByteBloomFilter(hashType); + + bbf.byteSize = computeFoldableByteSize(byteSizeHint * 8, foldFactor); + long bitSize = bbf.byteSize * 8; + bbf.maxKeys = (int) idealMaxKeys(bitSize, errorRate); + bbf.hashCount = optimalFunctionCount(bbf.maxKeys, bitSize); + + // Adjust max keys to bring error rate closer to what was requested, + // because byteSize was adjusted to allow for folding, and hashCount was + // rounded. + bbf.maxKeys = (int) computeMaxKeys(bitSize, errorRate, bbf.hashCount); + + return bbf; + } + + /** + * Creates another similar Bloom filter. Does not copy the actual bits, and + * sets the new filter's key count to zero. + * + * @return a Bloom filter with the same configuration as this + */ + public ByteBloomFilter createAnother() { + ByteBloomFilter bbf = new ByteBloomFilter(hashType); + bbf.byteSize = byteSize; + bbf.hashCount = hashCount; + bbf.maxKeys = maxKeys; + return bbf; + } + + @Override + public void allocBloom() { + if (this.bloom != null) { + throw new IllegalArgumentException("can only create bloom once."); + } + this.bloom = ByteBuffer.allocate((int)this.byteSize); + assert this.bloom.hasArray(); + } + + void sanityCheck() throws IllegalArgumentException { + if(0 >= this.byteSize || this.byteSize > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Invalid byteSize: " + this.byteSize); + } + + if(this.hashCount <= 0) { + throw new IllegalArgumentException("Hash function count must be > 0"); + } + + if (this.hash == null) { + throw new IllegalArgumentException("hashType must be known"); + } + + if (this.keyCount < 0) { + throw new IllegalArgumentException("must have positive keyCount"); + } + } + + void bloomCheck(ByteBuffer bloom) throws IllegalArgumentException { + if (this.byteSize != bloom.limit()) { + throw new IllegalArgumentException( + "Configured bloom length should match actual length"); + } + } + + public void add(byte [] buf) { + add(buf, 0, buf.length); + } + + @Override + public void add(byte [] buf, int offset, int len) { + /* + * For faster hashing, use combinatorial generation + * http://www.eecs.harvard.edu/~kirsch/pubs/bbbf/esa06.pdf + */ + int hash1 = this.hash.hash(buf, offset, len, 0); + int hash2 = this.hash.hash(buf, offset, len, hash1); + + for (int i = 0; i < this.hashCount; i++) { + long hashLoc = Math.abs((hash1 + i * hash2) % (this.byteSize * 8)); + set(hashLoc); + } + + ++this.keyCount; + } + + /** Should only be used in tests */ + boolean contains(byte [] buf) { + return contains(buf, 0, buf.length, this.bloom); + } + + /** Should only be used in tests */ + boolean contains(byte [] buf, int offset, int length) { + return contains(buf, offset, length, bloom); + } + + /** Should only be used in tests */ + boolean contains(byte[] buf, ByteBuffer bloom) { + return contains(buf, 0, buf.length, bloom); + } + + @Override + public boolean contains(byte[] buf, int offset, int length, + ByteBuffer theBloom) { + if (theBloom == null) { + // In a version 1 HFile Bloom filter data is stored in a separate meta + // block which is loaded on demand, but in version 2 it is pre-loaded. + // We want to use the same API in both cases. + theBloom = bloom; + } + + if (theBloom.limit() != byteSize) { + throw new IllegalArgumentException("Bloom does not match expected size:" + + " theBloom.limit()=" + theBloom.limit() + ", byteSize=" + byteSize); + } + + return contains(buf, offset, length, theBloom.array(), + theBloom.arrayOffset(), (int) byteSize, hash, hashCount); + } + + public static boolean contains(byte[] buf, int offset, int length, + byte[] bloomArray, int bloomOffset, int bloomSize, Hash hash, + int hashCount) { + + int hash1 = hash.hash(buf, offset, length, 0); + int hash2 = hash.hash(buf, offset, length, hash1); + int bloomBitSize = bloomSize << 3; + + if (randomGeneratorForTest == null) { + // Production mode. + int compositeHash = hash1; + for (int i = 0; i < hashCount; i++) { + int hashLoc = Math.abs(compositeHash % bloomBitSize); + compositeHash += hash2; + if (!get(hashLoc, bloomArray, bloomOffset)) { + return false; + } + } + } else { + // Test mode with "fake lookups" to estimate "ideal false positive rate". + for (int i = 0; i < hashCount; i++) { + int hashLoc = randomGeneratorForTest.nextInt(bloomBitSize); + if (!get(hashLoc, bloomArray, bloomOffset)){ + return false; + } + } + } + return true; + } + + //--------------------------------------------------------------------------- + /** Private helpers */ + + /** + * Set the bit at the specified index to 1. + * + * @param pos index of bit + */ + void set(long pos) { + int bytePos = (int)(pos / 8); + int bitPos = (int)(pos % 8); + byte curByte = bloom.get(bytePos); + curByte |= bitvals[bitPos]; + bloom.put(bytePos, curByte); + } + + /** + * Check if bit at specified index is 1. + * + * @param pos index of bit + * @return true if bit at specified index is 1, false if 0. + */ + static boolean get(int pos, byte[] bloomArray, int bloomOffset) { + int bytePos = pos >> 3; //pos / 8 + int bitPos = pos & 0x7; //pos % 8 + byte curByte = bloomArray[bloomOffset + bytePos]; + curByte &= bitvals[bitPos]; + return (curByte != 0); + } + + @Override + public long getKeyCount() { + return keyCount; + } + + @Override + public long getMaxKeys() { + return maxKeys; + } + + @Override + public long getByteSize() { + return byteSize; + } + + public int getHashType() { + return hashType; + } + + @Override + public void compactBloom() { + // see if the actual size is exponentially smaller than expected. + if (this.keyCount > 0 && this.bloom.hasArray()) { + int pieces = 1; + int newByteSize = (int)this.byteSize; + int newMaxKeys = this.maxKeys; + + // while exponentially smaller & folding is lossless + while ( (newByteSize & 1) == 0 && newMaxKeys > (this.keyCount<<1) ) { + pieces <<= 1; + newByteSize >>= 1; + newMaxKeys >>= 1; + } + + // if we should fold these into pieces + if (pieces > 1) { + byte[] array = this.bloom.array(); + int start = this.bloom.arrayOffset(); + int end = start + newByteSize; + int off = end; + for(int p = 1; p < pieces; ++p) { + for(int pos = start; pos < end; ++pos) { + array[pos] |= array[off++]; + } + } + // folding done, only use a subset of this array + this.bloom.rewind(); + this.bloom.limit(newByteSize); + this.bloom = this.bloom.slice(); + this.byteSize = newByteSize; + this.maxKeys = newMaxKeys; + } + } + } + + + //--------------------------------------------------------------------------- + + /** + * Writes just the bloom filter to the output array + * @param out OutputStream to place bloom + * @throws IOException Error writing bloom array + */ + public void writeBloom(final DataOutput out) throws IOException { + if (!this.bloom.hasArray()) { + throw new IOException("Only writes ByteBuffer with underlying array."); + } + out.write(bloom.array(), bloom.arrayOffset(), bloom.limit()); + } + + @Override + public Writable getMetaWriter() { + return new MetaWriter(); + } + + @Override + public Writable getDataWriter() { + return new DataWriter(); + } + + private class MetaWriter implements Writable { + protected MetaWriter() {} + @Override + public void readFields(DataInput arg0) throws IOException { + throw new IOException("Cant read with this class."); + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(VERSION); + out.writeInt((int) byteSize); + out.writeInt(hashCount); + out.writeInt(hashType); + out.writeInt(keyCount); + } + } + + private class DataWriter implements Writable { + protected DataWriter() {} + @Override + public void readFields(DataInput arg0) throws IOException { + throw new IOException("Cant read with this class."); + } + + @Override + public void write(DataOutput out) throws IOException { + writeBloom(out); + } + } + + public int getHashCount() { + return hashCount; + } + + @Override + public boolean supportsAutoLoading() { + return bloom != null; + } + + public static void setFakeLookupMode(boolean enabled) { + if (enabled) { + randomGeneratorForTest = new Random(283742987L); + } else { + randomGeneratorForTest = null; + } + } + + /** + * {@inheritDoc} + * Just concatenate row and column by default. May return the original row + * buffer if the column qualifier is empty. + */ + @Override + public byte[] createBloomKey(byte[] rowBuf, int rowOffset, int rowLen, + byte[] qualBuf, int qualOffset, int qualLen) { + // Optimize the frequent case when only the row is provided. + if (qualLen <= 0 && rowOffset == 0 && rowLen == rowBuf.length) + return rowBuf; + + byte [] result = new byte[rowLen + qualLen]; + System.arraycopy(rowBuf, rowOffset, result, 0, rowLen); + if (qualLen > 0) + System.arraycopy(qualBuf, qualOffset, result, rowLen, qualLen); + return result; + } + + @Override + public RawComparator getComparator() { + return Bytes.BYTES_RAWCOMPARATOR; + } + + /** + * A human-readable string with statistics for the given Bloom filter. + * + * @param bloomFilter the Bloom filter to output statistics for; + * @return a string consisting of "<key>: <value>" parts + * separated by {@link #STATS_RECORD_SEP}. + */ + public static String formatStats(BloomFilterBase bloomFilter) { + StringBuilder sb = new StringBuilder(); + long k = bloomFilter.getKeyCount(); + long m = bloomFilter.getMaxKeys(); + + sb.append("BloomSize: " + bloomFilter.getByteSize() + STATS_RECORD_SEP); + sb.append("No of Keys in bloom: " + k + STATS_RECORD_SEP); + sb.append("Max Keys for bloom: " + m); + if (m > 0) { + sb.append(STATS_RECORD_SEP + "Percentage filled: " + + NumberFormat.getPercentInstance().format(k * 1.0 / m)); + } + return sb.toString(); + } + + @Override + public String toString() { + return formatStats(this) + STATS_RECORD_SEP + "Actual error rate: " + + String.format("%.8f", actualErrorRate()); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/ByteBufferOutputStream.java b/src/main/java/org/apache/hadoop/hbase/util/ByteBufferOutputStream.java new file mode 100644 index 0000000..599c15b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/ByteBufferOutputStream.java @@ -0,0 +1,132 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; + +/** + * Not thread safe! + */ +public class ByteBufferOutputStream extends OutputStream { + + protected ByteBuffer buf; + + public ByteBufferOutputStream(int capacity) { + this(capacity, false); + } + + public ByteBufferOutputStream(int capacity, boolean useDirectByteBuffer) { + if (useDirectByteBuffer) { + buf = ByteBuffer.allocateDirect(capacity); + } else { + buf = ByteBuffer.allocate(capacity); + } + } + + public int size() { + return buf.position(); + } + + /** + * This flips the underlying BB so be sure to use it _last_! + * @return ByteBuffer + */ + public ByteBuffer getByteBuffer() { + buf.flip(); + return buf; + } + + private void checkSizeAndGrow(int extra) { + if ( (buf.position() + extra) > buf.limit()) { + // size calculation is complex, because we could overflow negative, + // and/or not allocate enough space. this fixes that. + int newSize = (int)Math.min((((long)buf.capacity()) * 2), + (long)(Integer.MAX_VALUE)); + newSize = Math.max(newSize, buf.position() + extra); + + ByteBuffer newBuf = ByteBuffer.allocate(newSize); + buf.flip(); + newBuf.put(buf); + buf = newBuf; + } + } + + // OutputStream + @Override + public void write(int b) throws IOException { + checkSizeAndGrow(Bytes.SIZEOF_BYTE); + + buf.put((byte)b); + } + + /** + * Writes the complete contents of this byte buffer output stream to + * the specified output stream argument. + * + * @param out the output stream to which to write the data. + * @exception IOException if an I/O error occurs. + */ + public synchronized void writeTo(OutputStream out) throws IOException { + WritableByteChannel channel = Channels.newChannel(out); + ByteBuffer bb = buf.duplicate(); + bb.flip(); + channel.write(bb); + } + + @Override + public void write(byte[] b) throws IOException { + checkSizeAndGrow(b.length); + + buf.put(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + checkSizeAndGrow(len); + + buf.put(b, off, len); + } + + @Override + public void flush() throws IOException { + // noop + } + + @Override + public void close() throws IOException { + // noop again. heh + } + + public byte[] toByteArray(int offset, int length) { + ByteBuffer bb = buf.duplicate(); + bb.flip(); + + byte[] chunk = new byte[length]; + + bb.position(offset); + bb.get(chunk, 0, length); + return chunk; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java b/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java new file mode 100644 index 0000000..453e145 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java @@ -0,0 +1,439 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +import org.apache.hadoop.hbase.io.encoding. + EncoderBufferTooSmallException; +import org.apache.hadoop.io.WritableUtils; + +/** + * Utility functions for working with byte buffers, such as reading/writing + * variable-length long numbers. + */ +public final class ByteBufferUtils { + + // "Compressed integer" serialization helper constants. + private final static int VALUE_MASK = 0x7f; + private final static int NEXT_BIT_SHIFT = 7; + private final static int NEXT_BIT_MASK = 1 << 7; + + private ByteBufferUtils() { + } + + /** + * Similar to {@link WritableUtils#writeVLong(java.io.DataOutput, long)}, + * but writes to a {@link ByteBuffer}. + */ + public static void writeVLong(ByteBuffer out, long i) { + if (i >= -112 && i <= 127) { + out.put((byte) i); + return; + } + + int len = -112; + if (i < 0) { + i ^= -1L; // take one's complement + len = -120; + } + + long tmp = i; + while (tmp != 0) { + tmp = tmp >> 8; + len--; + } + + out.put((byte) len); + + len = (len < -120) ? -(len + 120) : -(len + 112); + + for (int idx = len; idx != 0; idx--) { + int shiftbits = (idx - 1) * 8; + long mask = 0xFFL << shiftbits; + out.put((byte) ((i & mask) >> shiftbits)); + } + } + + /** + * Similar to {@link WritableUtils#readVLong(DataInput)} but reads from a + * {@link ByteBuffer}. + */ + public static long readVLong(ByteBuffer in) { + byte firstByte = in.get(); + int len = WritableUtils.decodeVIntSize(firstByte); + if (len == 1) { + return firstByte; + } + long i = 0; + for (int idx = 0; idx < len-1; idx++) { + byte b = in.get(); + i = i << 8; + i = i | (b & 0xFF); + } + return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i); + } + + + /** + * Put in buffer integer using 7 bit encoding. For each written byte: + * 7 bits are used to store value + * 1 bit is used to indicate whether there is next bit. + * @param value Int to be compressed. + * @param out Where to put compressed data + * @return Number of bytes written. + * @throws IOException on stream error + */ + public static int putCompressedInt(OutputStream out, final int value) + throws IOException { + int i = 0; + int tmpvalue = value; + do { + byte b = (byte) (tmpvalue & VALUE_MASK); + tmpvalue >>>= NEXT_BIT_SHIFT; + if (tmpvalue != 0) { + b |= (byte) NEXT_BIT_MASK; + } + out.write(b); + i++; + } while (tmpvalue != 0); + return i; + } + + /** + * Put in output stream 32 bit integer (Big Endian byte order). + * @param out Where to put integer. + * @param value Value of integer. + * @throws IOException On stream error. + */ + public static void putInt(OutputStream out, final int value) + throws IOException { + for (int i = Bytes.SIZEOF_INT - 1; i >= 0; --i) { + out.write((byte) (value >>> (i * 8))); + } + } + + /** + * Copy the data to the output stream and update position in buffer. + * @param out the stream to write bytes to + * @param in the buffer to read bytes from + * @param length the number of bytes to copy + */ + public static void moveBufferToStream(OutputStream out, ByteBuffer in, + int length) throws IOException { + copyBufferToStream(out, in, in.position(), length); + skip(in, length); + } + + /** + * Copy data from a buffer to an output stream. Does not update the position + * in the buffer. + * @param out the stream to write bytes to + * @param in the buffer to read bytes from + * @param offset the offset in the buffer (from the buffer's array offset) + * to start copying bytes from + * @param length the number of bytes to copy + */ + public static void copyBufferToStream(OutputStream out, ByteBuffer in, + int offset, int length) throws IOException { + if (in.hasArray()) { + out.write(in.array(), in.arrayOffset() + offset, + length); + } else { + for (int i = 0; i < length; ++i) { + out.write(in.get(offset + i)); + } + } + } + + public static int putLong(OutputStream out, final long value, + final int fitInBytes) throws IOException { + long tmpValue = value; + for (int i = 0; i < fitInBytes; ++i) { + out.write((byte) (tmpValue & 0xff)); + tmpValue >>>= 8; + } + return fitInBytes; + } + + /** + * Check how many bytes are required to store value. + * @param value Value which size will be tested. + * @return How many bytes are required to store value. + */ + public static int longFitsIn(final long value) { + if (value < 0) { + return 8; + } + + if (value < (1l << 4 * 8)) { + // no more than 4 bytes + if (value < (1l << 2 * 8)) { + if (value < (1l << 1 * 8)) { + return 1; + } + return 2; + } + if (value < (1l << 3 * 8)) { + return 3; + } + return 4; + } + // more than 4 bytes + if (value < (1l << 6 * 8)) { + if (value < (1l << 5 * 8)) { + return 5; + } + return 6; + } + if (value < (1l << 7 * 8)) { + return 7; + } + return 8; + } + + /** + * Check how many bytes is required to store value. + * @param value Value which size will be tested. + * @return How many bytes are required to store value. + */ + public static int intFitsIn(final int value) { + if (value < 0) { + return 4; + } + + if (value < (1 << 2 * 8)) { + if (value < (1 << 1 * 8)) { + return 1; + } + return 2; + } + if (value <= (1 << 3 * 8)) { + return 3; + } + return 4; + } + + /** + * Read integer from stream coded in 7 bits and increment position. + * @return the integer that has been read + * @throws IOException + */ + public static int readCompressedInt(InputStream input) + throws IOException { + int result = 0; + int i = 0; + byte b; + do { + b = (byte) input.read(); + result += (b & VALUE_MASK) << (NEXT_BIT_SHIFT * i); + i++; + if (i > Bytes.SIZEOF_INT + 1) { + throw new IllegalStateException( + "Corrupted compressed int (too long: " + (i + 1) + " bytes)"); + } + } while (0 != (b & NEXT_BIT_MASK)); + return result; + } + + /** + * Read integer from buffer coded in 7 bits and increment position. + * @return Read integer. + */ + public static int readCompressedInt(ByteBuffer buffer) { + byte b = buffer.get(); + if ((b & NEXT_BIT_MASK) != 0) { + return (b & VALUE_MASK) + (readCompressedInt(buffer) << NEXT_BIT_SHIFT); + } + return b & VALUE_MASK; + } + + /** + * Read long which was written to fitInBytes bytes and increment position. + * @param fitInBytes In how many bytes given long is stored. + * @return The value of parsed long. + * @throws IOException + */ + public static long readLong(InputStream in, final int fitInBytes) + throws IOException { + long tmpLong = 0; + for (int i = 0; i < fitInBytes; ++i) { + tmpLong |= (in.read() & 0xffl) << (8 * i); + } + return tmpLong; + } + + /** + * Read long which was written to fitInBytes bytes and increment position. + * @param fitInBytes In how many bytes given long is stored. + * @return The value of parsed long. + */ + public static long readLong(ByteBuffer in, final int fitInBytes) { + long tmpLength = 0; + for (int i = 0; i < fitInBytes; ++i) { + tmpLength |= (in.get() & 0xffl) << (8l * i); + } + return tmpLength; + } + + /** + * Asserts that there is at least the given amount of unfilled space + * remaining in the given buffer. + * @param out typically, the buffer we are writing to + * @param length the required space in the buffer + * @throws EncoderBufferTooSmallException If there are no enough bytes. + */ + public static void ensureSpace(ByteBuffer out, int length) + throws EncoderBufferTooSmallException { + if (out.position() + length > out.limit()) { + throw new EncoderBufferTooSmallException( + "Buffer position=" + out.position() + + ", buffer limit=" + out.limit() + + ", length to be written=" + length); + } + } + + /** + * Copy the given number of bytes from the given stream and put it at the + * current position of the given buffer, updating the position in the buffer. + * @param out the buffer to write data to + * @param in the stream to read data from + * @param length the number of bytes to read/write + */ + public static void copyFromStreamToBuffer(ByteBuffer out, + DataInputStream in, int length) throws IOException { + if (out.hasArray()) { + in.readFully(out.array(), out.position() + out.arrayOffset(), + length); + skip(out, length); + } else { + for (int i = 0; i < length; ++i) { + out.put(in.readByte()); + } + } + } + + /** + * Copy from one buffer to another from given offset + * @param out destination buffer + * @param in source buffer + * @param sourceOffset offset in the source buffer + * @param length how many bytes to copy + */ + public static void copyFromBufferToBuffer(ByteBuffer out, + ByteBuffer in, int sourceOffset, int length) { + if (in.hasArray() && out.hasArray()) { + System.arraycopy(in.array(), sourceOffset + in.arrayOffset(), + out.array(), out.position() + + out.arrayOffset(), length); + skip(out, length); + } else { + for (int i = 0; i < length; ++i) { + out.put(in.get(sourceOffset + i)); + } + } + } + + /** + * Find length of common prefix of two parts in the buffer + * @param buffer Where parts are located. + * @param offsetLeft Offset of the first part. + * @param offsetRight Offset of the second part. + * @param limit Maximal length of common prefix. + * @return Length of prefix. + */ + public static int findCommonPrefix(ByteBuffer buffer, int offsetLeft, + int offsetRight, int limit) { + int prefix = 0; + + for (; prefix < limit; ++prefix) { + if (buffer.get(offsetLeft + prefix) != buffer.get(offsetRight + prefix)) { + break; + } + } + + return prefix; + } + + /** + * Find length of common prefix in two arrays. + * @param left Array to be compared. + * @param leftOffset Offset in left array. + * @param leftLength Length of left array. + * @param right Array to be compared. + * @param rightArray Offset in right array. + * @param rightLength Length of right array. + */ + public static int findCommonPrefix( + byte[] left, int leftOffset, int leftLength, + byte[] right, int rightOffset, int rightLength) { + int length = Math.min(leftLength, rightLength); + int result = 0; + + while (result < length && + left[leftOffset + result] == right[rightOffset + result]) { + result++; + } + + return result; + } + + /** + * Check whether two parts in the same buffer are equal. + * @param buffer In which buffer there are parts + * @param offsetLeft Beginning of first part. + * @param lengthLeft Length of the first part. + * @param offsetRight Beginning of the second part. + * @param lengthRight Length of the second part. + * @return + */ + public static boolean arePartsEqual(ByteBuffer buffer, + int offsetLeft, int lengthLeft, + int offsetRight, int lengthRight) { + if (lengthLeft != lengthRight) { + return false; + } + + if (buffer.hasArray()) { + return 0 == Bytes.compareTo( + buffer.array(), buffer.arrayOffset() + offsetLeft, lengthLeft, + buffer.array(), buffer.arrayOffset() + offsetRight, lengthRight); + } + + for (int i = 0; i < lengthRight; ++i) { + if (buffer.get(offsetLeft + i) != buffer.get(offsetRight + i)) { + return false; + } + } + return true; + } + + /** + * Increment position in buffer. + * @param buffer In this buffer. + * @param length By that many bytes. + */ + public static void skip(ByteBuffer buffer, int length) { + buffer.position(buffer.position() + length); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java new file mode 100644 index 0000000..6a41308 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Bytes.java @@ -0,0 +1,1670 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.lang.reflect.Field; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Comparator; +import java.util.Iterator; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.io.RawComparator; +import org.apache.hadoop.io.WritableComparator; +import org.apache.hadoop.io.WritableUtils; + +import sun.misc.Unsafe; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Utility class that handles byte arrays, conversions to/from other types, + * comparisons, hash code generation, manufacturing keys for HashMaps or + * HashSets, etc. + */ +public class Bytes { + + private static final Log LOG = LogFactory.getLog(Bytes.class); + + /** + * Size of boolean in bytes + */ + public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE; + + /** + * Size of byte in bytes + */ + public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN; + + /** + * Size of char in bytes + */ + public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE; + + /** + * Size of double in bytes + */ + public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE; + + /** + * Size of float in bytes + */ + public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE; + + /** + * Size of int in bytes + */ + public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE; + + /** + * Size of long in bytes + */ + public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE; + + /** + * Size of short in bytes + */ + public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE; + + + /** + * Estimate of size cost to pay beyond payload in jvm for instance of byte []. + * Estimate based on study of jhat and jprofiler numbers. + */ + // JHat says BU is 56 bytes. + // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?) + public static final int ESTIMATED_HEAP_TAX = 16; + + /** + * Byte array comparator class. + */ + public static class ByteArrayComparator implements RawComparator { + /** + * Constructor + */ + public ByteArrayComparator() { + super(); + } + public int compare(byte [] left, byte [] right) { + return compareTo(left, right); + } + public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) { + return LexicographicalComparerHolder.BEST_COMPARER. + compareTo(b1, s1, l1, b2, s2, l2); + } + } + + /** + * A {@link ByteArrayComparator} that treats the empty array as the largest value. + * This is useful for comparing row end keys for regions. + */ + // TODO: unfortunately, HBase uses byte[0] as both start and end keys for region + // boundaries. Thus semantically, we should treat empty byte array as the smallest value + // while comparing row keys, start keys etc; but as the largest value for comparing + // region boundaries for endKeys. + public static class RowEndKeyComparator extends ByteArrayComparator { + @Override + public int compare(byte[] left, byte[] right) { + return compare(left, 0, left.length, right, 0, right.length); + } + @Override + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + if (b1 == b2 && s1 == s2 && l1 == l2) { + return 0; + } + if (l1 == 0) { + return l2; //0 or positive + } + if (l2 == 0) { + return -1; + } + return super.compare(b1, s1, l1, b2, s2, l2); + } + } + + /** + * Pass this to TreeMaps where byte [] are keys. + */ + public static Comparator BYTES_COMPARATOR = + new ByteArrayComparator(); + + /** + * Use comparing byte arrays, byte-by-byte + */ + public static RawComparator BYTES_RAWCOMPARATOR = + new ByteArrayComparator(); + + /** + * Read byte-array written with a WritableableUtils.vint prefix. + * @param in Input to read from. + * @return byte array read off in + * @throws IOException e + */ + public static byte [] readByteArray(final DataInput in) + throws IOException { + int len = WritableUtils.readVInt(in); + if (len < 0) { + throw new NegativeArraySizeException(Integer.toString(len)); + } + byte [] result = new byte[len]; + in.readFully(result, 0, len); + return result; + } + + /** + * Read byte-array written with a WritableableUtils.vint prefix. + * IOException is converted to a RuntimeException. + * @param in Input to read from. + * @return byte array read off in + */ + public static byte [] readByteArrayThrowsRuntime(final DataInput in) { + try { + return readByteArray(in); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Write byte-array with a WritableableUtils.vint prefix. + * @param out output stream to be written to + * @param b array to write + * @throws IOException e + */ + public static void writeByteArray(final DataOutput out, final byte [] b) + throws IOException { + if(b == null) { + WritableUtils.writeVInt(out, 0); + } else { + writeByteArray(out, b, 0, b.length); + } + } + + /** + * Write byte-array to out with a vint length prefix. + * @param out output stream + * @param b array + * @param offset offset into array + * @param length length past offset + * @throws IOException e + */ + public static void writeByteArray(final DataOutput out, final byte [] b, + final int offset, final int length) + throws IOException { + WritableUtils.writeVInt(out, length); + out.write(b, offset, length); + } + + /** + * Write byte-array from src to tgt with a vint length prefix. + * @param tgt target array + * @param tgtOffset offset into target array + * @param src source array + * @param srcOffset source offset + * @param srcLength source length + * @return New offset in src array. + */ + public static int writeByteArray(final byte [] tgt, final int tgtOffset, + final byte [] src, final int srcOffset, final int srcLength) { + byte [] vint = vintToBytes(srcLength); + System.arraycopy(vint, 0, tgt, tgtOffset, vint.length); + int offset = tgtOffset + vint.length; + System.arraycopy(src, srcOffset, tgt, offset, srcLength); + return offset + srcLength; + } + + /** + * Put bytes at the specified byte array position. + * @param tgtBytes the byte array + * @param tgtOffset position in the array + * @param srcBytes array to write out + * @param srcOffset source offset + * @param srcLength source length + * @return incremented offset + */ + public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes, + int srcOffset, int srcLength) { + System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength); + return tgtOffset + srcLength; + } + + /** + * Write a single byte out to the specified byte array position. + * @param bytes the byte array + * @param offset position in the array + * @param b byte to write out + * @return incremented offset + */ + public static int putByte(byte[] bytes, int offset, byte b) { + bytes[offset] = b; + return offset + 1; + } + + /** + * Returns a new byte array, copied from the passed ByteBuffer. + * @param bb A ByteBuffer + * @return the byte array + */ + public static byte[] toBytes(ByteBuffer bb) { + int length = bb.limit(); + byte [] result = new byte[length]; + System.arraycopy(bb.array(), bb.arrayOffset(), result, 0, length); + return result; + } + + /** + * @param b Presumed UTF-8 encoded byte array. + * @return String made from b + */ + public static String toString(final byte [] b) { + if (b == null) { + return null; + } + return toString(b, 0, b.length); + } + + /** + * Joins two byte arrays together using a separator. + * @param b1 The first byte array. + * @param sep The separator to use. + * @param b2 The second byte array. + */ + public static String toString(final byte [] b1, + String sep, + final byte [] b2) { + return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length); + } + + /** + * This method will convert utf8 encoded bytes into a string. If + * an UnsupportedEncodingException occurs, this method will eat it + * and return null instead. + * + * @param b Presumed UTF-8 encoded byte array. + * @param off offset into array + * @param len length of utf-8 sequence + * @return String made from b or null + */ + public static String toString(final byte [] b, int off, int len) { + if (b == null) { + return null; + } + if (len == 0) { + return ""; + } + try { + return new String(b, off, len, HConstants.UTF8_ENCODING); + } catch (UnsupportedEncodingException e) { + LOG.error("UTF-8 not supported?", e); + return null; + } + } + + /** + * Write a printable representation of a byte array. + * + * @param b byte array + * @return string + * @see #toStringBinary(byte[], int, int) + */ + public static String toStringBinary(final byte [] b) { + if (b == null) + return "null"; + return toStringBinary(b, 0, b.length); + } + + /** + * Converts the given byte buffer, from its array offset to its limit, to + * a string. The position and the mark are ignored. + * + * @param buf a byte buffer + * @return a string representation of the buffer's binary contents + */ + public static String toStringBinary(ByteBuffer buf) { + if (buf == null) + return "null"; + return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit()); + } + + /** + * Write a printable representation of a byte array. Non-printable + * characters are hex escaped in the format \\x%02X, eg: + * \x00 \x05 etc + * + * @param b array to write out + * @param off offset to start at + * @param len length to write + * @return string output + */ + public static String toStringBinary(final byte [] b, int off, int len) { + StringBuilder result = new StringBuilder(); + for (int i = off; i < off + len ; ++i ) { + int ch = b[i] & 0xFF; + if ( (ch >= '0' && ch <= '9') + || (ch >= 'A' && ch <= 'Z') + || (ch >= 'a' && ch <= 'z') + || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) { + result.append((char)ch); + } else { + result.append(String.format("\\x%02X", ch)); + } + } + return result.toString(); + } + + private static boolean isHexDigit(char c) { + return + (c >= 'A' && c <= 'F') || + (c >= '0' && c <= '9'); + } + + /** + * Takes a ASCII digit in the range A-F0-9 and returns + * the corresponding integer/ordinal value. + * @param ch The hex digit. + * @return The converted hex value as a byte. + */ + public static byte toBinaryFromHex(byte ch) { + if ( ch >= 'A' && ch <= 'F' ) + return (byte) ((byte)10 + (byte) (ch - 'A')); + // else + return (byte) (ch - '0'); + } + + public static byte [] toBytesBinary(String in) { + // this may be bigger than we need, but let's be safe. + byte [] b = new byte[in.length()]; + int size = 0; + for (int i = 0; i < in.length(); ++i) { + char ch = in.charAt(i); + if (ch == '\\' && in.length() > i+1 && in.charAt(i+1) == 'x') { + // ok, take next 2 hex digits. + char hd1 = in.charAt(i+2); + char hd2 = in.charAt(i+3); + + // they need to be A-F0-9: + if (!isHexDigit(hd1) || + !isHexDigit(hd2)) { + // bogus escape code, ignore: + continue; + } + // turn hex ASCII digit -> number + byte d = (byte) ((toBinaryFromHex((byte)hd1) << 4) + toBinaryFromHex((byte)hd2)); + + b[size++] = d; + i += 3; // skip 3 + } else { + b[size++] = (byte) ch; + } + } + // resize: + byte [] b2 = new byte[size]; + System.arraycopy(b, 0, b2, 0, size); + return b2; + } + + /** + * Converts a string to a UTF-8 byte array. + * @param s string + * @return the byte array + */ + public static byte[] toBytes(String s) { + try { + return s.getBytes(HConstants.UTF8_ENCODING); + } catch (UnsupportedEncodingException e) { + LOG.error("UTF-8 not supported?", e); + return null; + } + } + + /** + * Convert a boolean to a byte array. True becomes -1 + * and false becomes 0. + * + * @param b value + * @return b encoded in a byte array. + */ + public static byte [] toBytes(final boolean b) { + return new byte[] { b ? (byte) -1 : (byte) 0 }; + } + + /** + * Reverses {@link #toBytes(boolean)} + * @param b array + * @return True or false. + */ + public static boolean toBoolean(final byte [] b) { + if (b.length != 1) { + throw new IllegalArgumentException("Array has wrong size: " + b.length); + } + return b[0] != (byte) 0; + } + + /** + * Convert a long value to a byte array using big-endian. + * + * @param val value to convert + * @return the byte array + */ + public static byte[] toBytes(long val) { + byte [] b = new byte[8]; + for (int i = 7; i > 0; i--) { + b[i] = (byte) val; + val >>>= 8; + } + b[0] = (byte) val; + return b; + } + + /** + * Converts a byte array to a long value. Reverses + * {@link #toBytes(long)} + * @param bytes array + * @return the long value + */ + public static long toLong(byte[] bytes) { + return toLong(bytes, 0, SIZEOF_LONG); + } + + /** + * Converts a byte array to a long value. Assumes there will be + * {@link #SIZEOF_LONG} bytes available. + * + * @param bytes bytes + * @param offset offset + * @return the long value + */ + public static long toLong(byte[] bytes, int offset) { + return toLong(bytes, offset, SIZEOF_LONG); + } + + /** + * Converts a byte array to a long value. + * + * @param bytes array of bytes + * @param offset offset into array + * @param length length of data (must be {@link #SIZEOF_LONG}) + * @return the long value + * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or + * if there's not enough room in the array at the offset indicated. + */ + public static long toLong(byte[] bytes, int offset, final int length) { + if (length != SIZEOF_LONG || offset + length > bytes.length) { + throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG); + } + long l = 0; + for(int i = offset; i < offset + length; i++) { + l <<= 8; + l ^= bytes[i] & 0xFF; + } + return l; + } + + private static IllegalArgumentException + explainWrongLengthOrOffset(final byte[] bytes, + final int offset, + final int length, + final int expectedLength) { + String reason; + if (length != expectedLength) { + reason = "Wrong length: " + length + ", expected " + expectedLength; + } else { + reason = "offset (" + offset + ") + length (" + length + ") exceed the" + + " capacity of the array: " + bytes.length; + } + return new IllegalArgumentException(reason); + } + + /** + * Put a long value out to the specified byte array position. + * @param bytes the byte array + * @param offset position in the array + * @param val long to write out + * @return incremented offset + * @throws IllegalArgumentException if the byte array given doesn't have + * enough room at the offset specified. + */ + public static int putLong(byte[] bytes, int offset, long val) { + if (bytes.length - offset < SIZEOF_LONG) { + throw new IllegalArgumentException("Not enough room to put a long at" + + " offset " + offset + " in a " + bytes.length + " byte array"); + } + for(int i = offset + 7; i > offset; i--) { + bytes[i] = (byte) val; + val >>>= 8; + } + bytes[offset] = (byte) val; + return offset + SIZEOF_LONG; + } + + /** + * Presumes float encoded as IEEE 754 floating-point "single format" + * @param bytes byte array + * @return Float made from passed byte array. + */ + public static float toFloat(byte [] bytes) { + return toFloat(bytes, 0); + } + + /** + * Presumes float encoded as IEEE 754 floating-point "single format" + * @param bytes array to convert + * @param offset offset into array + * @return Float made from passed byte array. + */ + public static float toFloat(byte [] bytes, int offset) { + return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT)); + } + + /** + * @param bytes byte array + * @param offset offset to write to + * @param f float value + * @return New offset in bytes + */ + public static int putFloat(byte [] bytes, int offset, float f) { + return putInt(bytes, offset, Float.floatToRawIntBits(f)); + } + + /** + * @param f float value + * @return the float represented as byte [] + */ + public static byte [] toBytes(final float f) { + // Encode it as int + return Bytes.toBytes(Float.floatToRawIntBits(f)); + } + + /** + * @param bytes byte array + * @return Return double made from passed bytes. + */ + public static double toDouble(final byte [] bytes) { + return toDouble(bytes, 0); + } + + /** + * @param bytes byte array + * @param offset offset where double is + * @return Return double made from passed bytes. + */ + public static double toDouble(final byte [] bytes, final int offset) { + return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG)); + } + + /** + * @param bytes byte array + * @param offset offset to write to + * @param d value + * @return New offset into array bytes + */ + public static int putDouble(byte [] bytes, int offset, double d) { + return putLong(bytes, offset, Double.doubleToLongBits(d)); + } + + /** + * Serialize a double as the IEEE 754 double format output. The resultant + * array will be 8 bytes long. + * + * @param d value + * @return the double represented as byte [] + */ + public static byte [] toBytes(final double d) { + // Encode it as a long + return Bytes.toBytes(Double.doubleToRawLongBits(d)); + } + + /** + * Convert an int value to a byte array + * @param val value + * @return the byte array + */ + public static byte[] toBytes(int val) { + byte [] b = new byte[4]; + for(int i = 3; i > 0; i--) { + b[i] = (byte) val; + val >>>= 8; + } + b[0] = (byte) val; + return b; + } + + /** + * Converts a byte array to an int value + * @param bytes byte array + * @return the int value + */ + public static int toInt(byte[] bytes) { + return toInt(bytes, 0, SIZEOF_INT); + } + + /** + * Converts a byte array to an int value + * @param bytes byte array + * @param offset offset into array + * @return the int value + */ + public static int toInt(byte[] bytes, int offset) { + return toInt(bytes, offset, SIZEOF_INT); + } + + /** + * Converts a byte array to an int value + * @param bytes byte array + * @param offset offset into array + * @param length length of int (has to be {@link #SIZEOF_INT}) + * @return the int value + * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or + * if there's not enough room in the array at the offset indicated. + */ + public static int toInt(byte[] bytes, int offset, final int length) { + if (length != SIZEOF_INT || offset + length > bytes.length) { + throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT); + } + int n = 0; + for(int i = offset; i < (offset + length); i++) { + n <<= 8; + n ^= bytes[i] & 0xFF; + } + return n; + } + + /** + * Put an int value out to the specified byte array position. + * @param bytes the byte array + * @param offset position in the array + * @param val int to write out + * @return incremented offset + * @throws IllegalArgumentException if the byte array given doesn't have + * enough room at the offset specified. + */ + public static int putInt(byte[] bytes, int offset, int val) { + if (bytes.length - offset < SIZEOF_INT) { + throw new IllegalArgumentException("Not enough room to put an int at" + + " offset " + offset + " in a " + bytes.length + " byte array"); + } + for(int i= offset + 3; i > offset; i--) { + bytes[i] = (byte) val; + val >>>= 8; + } + bytes[offset] = (byte) val; + return offset + SIZEOF_INT; + } + + /** + * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long. + * @param val value + * @return the byte array + */ + public static byte[] toBytes(short val) { + byte[] b = new byte[SIZEOF_SHORT]; + b[1] = (byte) val; + val >>= 8; + b[0] = (byte) val; + return b; + } + + /** + * Converts a byte array to a short value + * @param bytes byte array + * @return the short value + */ + public static short toShort(byte[] bytes) { + return toShort(bytes, 0, SIZEOF_SHORT); + } + + /** + * Converts a byte array to a short value + * @param bytes byte array + * @param offset offset into array + * @return the short value + */ + public static short toShort(byte[] bytes, int offset) { + return toShort(bytes, offset, SIZEOF_SHORT); + } + + /** + * Converts a byte array to a short value + * @param bytes byte array + * @param offset offset into array + * @param length length, has to be {@link #SIZEOF_SHORT} + * @return the short value + * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT} + * or if there's not enough room in the array at the offset indicated. + */ + public static short toShort(byte[] bytes, int offset, final int length) { + if (length != SIZEOF_SHORT || offset + length > bytes.length) { + throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT); + } + short n = 0; + n ^= bytes[offset] & 0xFF; + n <<= 8; + n ^= bytes[offset+1] & 0xFF; + return n; + } + + /** + * This method will get a sequence of bytes from pos -> limit, + * but will restore pos after. + * @param buf + * @return byte array + */ + public static byte[] getBytes(ByteBuffer buf) { + int savedPos = buf.position(); + byte [] newBytes = new byte[buf.remaining()]; + buf.get(newBytes); + buf.position(savedPos); + return newBytes; + } + + /** + * Put a short value out to the specified byte array position. + * @param bytes the byte array + * @param offset position in the array + * @param val short to write out + * @return incremented offset + * @throws IllegalArgumentException if the byte array given doesn't have + * enough room at the offset specified. + */ + public static int putShort(byte[] bytes, int offset, short val) { + if (bytes.length - offset < SIZEOF_SHORT) { + throw new IllegalArgumentException("Not enough room to put a short at" + + " offset " + offset + " in a " + bytes.length + " byte array"); + } + bytes[offset+1] = (byte) val; + val >>= 8; + bytes[offset] = (byte) val; + return offset + SIZEOF_SHORT; + } + + /** + * Convert a BigDecimal value to a byte array + * + * @param val + * @return the byte array + */ + public static byte[] toBytes(BigDecimal val) { + byte[] valueBytes = val.unscaledValue().toByteArray(); + byte[] result = new byte[valueBytes.length + SIZEOF_INT]; + int offset = putInt(result, 0, val.scale()); + putBytes(result, offset, valueBytes, 0, valueBytes.length); + return result; + } + + + /** + * Converts a byte array to a BigDecimal + * + * @param bytes + * @return the char value + */ + public static BigDecimal toBigDecimal(byte[] bytes) { + return toBigDecimal(bytes, 0, bytes.length); + } + + /** + * Converts a byte array to a BigDecimal value + * + * @param bytes + * @param offset + * @param length + * @return the char value + */ + public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) { + if (bytes == null || length < SIZEOF_INT + 1 || + (offset + length > bytes.length)) { + return null; + } + + int scale = toInt(bytes, offset); + byte[] tcBytes = new byte[length - SIZEOF_INT]; + System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT); + return new BigDecimal(new BigInteger(tcBytes), scale); + } + + /** + * Put a BigDecimal value out to the specified byte array position. + * + * @param bytes the byte array + * @param offset position in the array + * @param val BigDecimal to write out + * @return incremented offset + */ + public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) { + if (bytes == null) { + return offset; + } + + byte[] valueBytes = val.unscaledValue().toByteArray(); + byte[] result = new byte[valueBytes.length + SIZEOF_INT]; + offset = putInt(result, offset, val.scale()); + return putBytes(result, offset, valueBytes, 0, valueBytes.length); + } + + /** + * @param vint Integer to make a vint of. + * @return Vint as bytes array. + */ + public static byte [] vintToBytes(final long vint) { + long i = vint; + int size = WritableUtils.getVIntSize(i); + byte [] result = new byte[size]; + int offset = 0; + if (i >= -112 && i <= 127) { + result[offset] = (byte) i; + return result; + } + + int len = -112; + if (i < 0) { + i ^= -1L; // take one's complement' + len = -120; + } + + long tmp = i; + while (tmp != 0) { + tmp = tmp >> 8; + len--; + } + + result[offset++] = (byte) len; + + len = (len < -120) ? -(len + 120) : -(len + 112); + + for (int idx = len; idx != 0; idx--) { + int shiftbits = (idx - 1) * 8; + long mask = 0xFFL << shiftbits; + result[offset++] = (byte)((i & mask) >> shiftbits); + } + return result; + } + + /** + * @param buffer buffer to convert + * @return vint bytes as an integer. + */ + public static long bytesToVint(final byte [] buffer) { + int offset = 0; + byte firstByte = buffer[offset++]; + int len = WritableUtils.decodeVIntSize(firstByte); + if (len == 1) { + return firstByte; + } + long i = 0; + for (int idx = 0; idx < len-1; idx++) { + byte b = buffer[offset++]; + i = i << 8; + i = i | (b & 0xFF); + } + return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i); + } + + /** + * Reads a zero-compressed encoded long from input stream and returns it. + * @param buffer Binary array + * @param offset Offset into array at which vint begins. + * @throws java.io.IOException e + * @return deserialized long from stream. + */ + public static long readVLong(final byte [] buffer, final int offset) + throws IOException { + byte firstByte = buffer[offset]; + int len = WritableUtils.decodeVIntSize(firstByte); + if (len == 1) { + return firstByte; + } + long i = 0; + for (int idx = 0; idx < len-1; idx++) { + byte b = buffer[offset + 1 + idx]; + i = i << 8; + i = i | (b & 0xFF); + } + return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i); + } + + /** + * @param left left operand + * @param right right operand + * @return 0 if equal, < 0 if left is less than right, etc. + */ + public static int compareTo(final byte [] left, final byte [] right) { + return LexicographicalComparerHolder.BEST_COMPARER. + compareTo(left, 0, left.length, right, 0, right.length); + } + + /** + * Lexicographically compare two arrays. + * + * @param buffer1 left operand + * @param buffer2 right operand + * @param offset1 Where to start comparing in the left buffer + * @param offset2 Where to start comparing in the right buffer + * @param length1 How much to compare from the left buffer + * @param length2 How much to compare from the right buffer + * @return 0 if equal, < 0 if left is less than right, etc. + */ + public static int compareTo(byte[] buffer1, int offset1, int length1, + byte[] buffer2, int offset2, int length2) { + return LexicographicalComparerHolder.BEST_COMPARER. + compareTo(buffer1, offset1, length1, buffer2, offset2, length2); + } + + interface Comparer { + abstract public int compareTo(T buffer1, int offset1, int length1, + T buffer2, int offset2, int length2); + } + + @VisibleForTesting + static Comparer lexicographicalComparerJavaImpl() { + return LexicographicalComparerHolder.PureJavaComparer.INSTANCE; + } + + /** + * Provides a lexicographical comparer implementation; either a Java + * implementation or a faster implementation based on {@link Unsafe}. + * + *

    Uses reflection to gracefully fall back to the Java implementation if + * {@code Unsafe} isn't available. + */ + @VisibleForTesting + static class LexicographicalComparerHolder { + static final String UNSAFE_COMPARER_NAME = + LexicographicalComparerHolder.class.getName() + "$UnsafeComparer"; + + static final Comparer BEST_COMPARER = getBestComparer(); + /** + * Returns the Unsafe-using Comparer, or falls back to the pure-Java + * implementation if unable to do so. + */ + static Comparer getBestComparer() { + try { + Class theClass = Class.forName(UNSAFE_COMPARER_NAME); + + // yes, UnsafeComparer does implement Comparer + @SuppressWarnings("unchecked") + Comparer comparer = + (Comparer) theClass.getEnumConstants()[0]; + return comparer; + } catch (Throwable t) { // ensure we really catch *everything* + return lexicographicalComparerJavaImpl(); + } + } + + enum PureJavaComparer implements Comparer { + INSTANCE; + + @Override + public int compareTo(byte[] buffer1, int offset1, int length1, + byte[] buffer2, int offset2, int length2) { + // Short circuit equal case + if (buffer1 == buffer2 && + offset1 == offset2 && + length1 == length2) { + return 0; + } + // Bring WritableComparator code local + int end1 = offset1 + length1; + int end2 = offset2 + length2; + for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) { + int a = (buffer1[i] & 0xff); + int b = (buffer2[j] & 0xff); + if (a != b) { + return a - b; + } + } + return length1 - length2; + } + } + + @VisibleForTesting + enum UnsafeComparer implements Comparer { + INSTANCE; + + static final Unsafe theUnsafe; + + /** The offset to the first element in a byte array. */ + static final int BYTE_ARRAY_BASE_OFFSET; + + static { + theUnsafe = (Unsafe) AccessController.doPrivileged( + new PrivilegedAction() { + @Override + public Object run() { + try { + Field f = Unsafe.class.getDeclaredField("theUnsafe"); + f.setAccessible(true); + return f.get(null); + } catch (NoSuchFieldException e) { + // It doesn't matter what we throw; + // it's swallowed in getBestComparer(). + throw new Error(); + } catch (IllegalAccessException e) { + throw new Error(); + } + } + }); + + BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class); + + // sanity check - this should never fail + if (theUnsafe.arrayIndexScale(byte[].class) != 1) { + throw new AssertionError(); + } + } + + static final boolean littleEndian = + ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN); + + /** + * Returns true if x1 is less than x2, when both values are treated as + * unsigned. + */ + static boolean lessThanUnsigned(long x1, long x2) { + return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE); + } + + /** + * Lexicographically compare two arrays. + * + * @param buffer1 left operand + * @param buffer2 right operand + * @param offset1 Where to start comparing in the left buffer + * @param offset2 Where to start comparing in the right buffer + * @param length1 How much to compare from the left buffer + * @param length2 How much to compare from the right buffer + * @return 0 if equal, < 0 if left is less than right, etc. + */ + @Override + public int compareTo(byte[] buffer1, int offset1, int length1, + byte[] buffer2, int offset2, int length2) { + // Short circuit equal case + if (buffer1 == buffer2 && + offset1 == offset2 && + length1 == length2) { + return 0; + } + int minLength = Math.min(length1, length2); + int minWords = minLength / SIZEOF_LONG; + int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET; + int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET; + + /* + * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a + * time is no slower than comparing 4 bytes at a time even on 32-bit. + * On the other hand, it is substantially faster on 64-bit. + */ + for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) { + long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i); + long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i); + long diff = lw ^ rw; + + if (diff != 0) { + if (!littleEndian) { + return lessThanUnsigned(lw, rw) ? -1 : 1; + } + + // Use binary search + int n = 0; + int y; + int x = (int) diff; + if (x == 0) { + x = (int) (diff >>> 32); + n = 32; + } + + y = x << 16; + if (y == 0) { + n += 16; + } else { + x = y; + } + + y = x << 8; + if (y == 0) { + n += 8; + } + return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL)); + } + } + + // The epilogue to cover the last (minLength % 8) elements. + for (int i = minWords * SIZEOF_LONG; i < minLength; i++) { + int a = (buffer1[offset1 + i] & 0xff); + int b = (buffer2[offset2 + i] & 0xff); + if (a != b) { + return a - b; + } + } + return length1 - length2; + } + } + } + + /** + * @param left left operand + * @param right right operand + * @return True if equal + */ + public static boolean equals(final byte [] left, final byte [] right) { + // Could use Arrays.equals? + //noinspection SimplifiableConditionalExpression + if (left == right) return true; + if (left == null || right == null) return false; + if (left.length != right.length) return false; + if (left.length == 0) return true; + + // Since we're often comparing adjacent sorted data, + // it's usual to have equal arrays except for the very last byte + // so check that first + if (left[left.length - 1] != right[right.length - 1]) return false; + + return compareTo(left, right) == 0; + } + + public static boolean equals(final byte[] left, int leftOffset, int leftLen, + final byte[] right, int rightOffset, int rightLen) { + // short circuit case + if (left == right && + leftOffset == rightOffset && + leftLen == rightLen) { + return true; + } + // different lengths fast check + if (leftLen != rightLen) { + return false; + } + if (leftLen == 0) { + return true; + } + + // Since we're often comparing adjacent sorted data, + // it's usual to have equal arrays except for the very last byte + // so check that first + if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false; + + return LexicographicalComparerHolder.BEST_COMPARER. + compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0; + } + + + /** + * Return true if the byte array on the right is a prefix of the byte + * array on the left. + */ + public static boolean startsWith(byte[] bytes, byte[] prefix) { + return bytes != null && prefix != null && + bytes.length >= prefix.length && + LexicographicalComparerHolder.BEST_COMPARER. + compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0; + } + + /** + * @param b bytes to hash + * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the + * passed in array. This method is what {@link org.apache.hadoop.io.Text} and + * {@link ImmutableBytesWritable} use calculating hash code. + */ + public static int hashCode(final byte [] b) { + return hashCode(b, b.length); + } + + /** + * @param b value + * @param length length of the value + * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the + * passed in array. This method is what {@link org.apache.hadoop.io.Text} and + * {@link ImmutableBytesWritable} use calculating hash code. + */ + public static int hashCode(final byte [] b, final int length) { + return WritableComparator.hashBytes(b, length); + } + + /** + * @param b bytes to hash + * @return A hash of b as an Integer that can be used as key in + * Maps. + */ + public static Integer mapKey(final byte [] b) { + return hashCode(b); + } + + /** + * @param b bytes to hash + * @param length length to hash + * @return A hash of b as an Integer that can be used as key in + * Maps. + */ + public static Integer mapKey(final byte [] b, final int length) { + return hashCode(b, length); + } + + /** + * @param a lower half + * @param b upper half + * @return New array that has a in lower half and b in upper half. + */ + public static byte [] add(final byte [] a, final byte [] b) { + return add(a, b, HConstants.EMPTY_BYTE_ARRAY); + } + + /** + * @param a first third + * @param b second third + * @param c third third + * @return New array made from a, b and c + */ + public static byte [] add(final byte [] a, final byte [] b, final byte [] c) { + byte [] result = new byte[a.length + b.length + c.length]; + System.arraycopy(a, 0, result, 0, a.length); + System.arraycopy(b, 0, result, a.length, b.length); + System.arraycopy(c, 0, result, a.length + b.length, c.length); + return result; + } + + /** + * @param a array + * @param length amount of bytes to grab + * @return First length bytes from a + */ + public static byte [] head(final byte [] a, final int length) { + if (a.length < length) { + return null; + } + byte [] result = new byte[length]; + System.arraycopy(a, 0, result, 0, length); + return result; + } + + /** + * @param a array + * @param length amount of bytes to snarf + * @return Last length bytes from a + */ + public static byte [] tail(final byte [] a, final int length) { + if (a.length < length) { + return null; + } + byte [] result = new byte[length]; + System.arraycopy(a, a.length - length, result, 0, length); + return result; + } + + /** + * @param a array + * @param length new array size + * @return Value in a plus length prepended 0 bytes + */ + public static byte [] padHead(final byte [] a, final int length) { + byte [] padding = new byte[length]; + for (int i = 0; i < length; i++) { + padding[i] = 0; + } + return add(padding,a); + } + + /** + * @param a array + * @param length new array size + * @return Value in a plus length appended 0 bytes + */ + public static byte [] padTail(final byte [] a, final int length) { + byte [] padding = new byte[length]; + for (int i = 0; i < length; i++) { + padding[i] = 0; + } + return add(a,padding); + } + + /** + * Split passed range. Expensive operation relatively. Uses BigInteger math. + * Useful splitting ranges for MapReduce jobs. + * @param a Beginning of range + * @param b End of range + * @param num Number of times to split range. Pass 1 if you want to split + * the range in two; i.e. one split. + * @return Array of dividing values + */ + public static byte [][] split(final byte [] a, final byte [] b, final int num) { + return split(a, b, false, num); + } + + /** + * Split passed range. Expensive operation relatively. Uses BigInteger math. + * Useful splitting ranges for MapReduce jobs. + * @param a Beginning of range + * @param b End of range + * @param inclusive Whether the end of range is prefix-inclusive or is + * considered an exclusive boundary. Automatic splits are generally exclusive + * and manual splits with an explicit range utilize an inclusive end of range. + * @param num Number of times to split range. Pass 1 if you want to split + * the range in two; i.e. one split. + * @return Array of dividing values + */ + public static byte[][] split(final byte[] a, final byte[] b, + boolean inclusive, final int num) { + byte[][] ret = new byte[num + 2][]; + int i = 0; + Iterable iter = iterateOnSplits(a, b, inclusive, num); + if (iter == null) + return null; + for (byte[] elem : iter) { + ret[i++] = elem; + } + return ret; + } + + /** + * Iterate over keys within the passed range, splitting at an [a,b) boundary. + */ + public static Iterable iterateOnSplits(final byte[] a, + final byte[] b, final int num) + { + return iterateOnSplits(a, b, false, num); + } + + /** + * Iterate over keys within the passed range. + */ + public static Iterable iterateOnSplits( + final byte[] a, final byte[]b, boolean inclusive, final int num) + { + byte [] aPadded; + byte [] bPadded; + if (a.length < b.length) { + aPadded = padTail(a, b.length - a.length); + bPadded = b; + } else if (b.length < a.length) { + aPadded = a; + bPadded = padTail(b, a.length - b.length); + } else { + aPadded = a; + bPadded = b; + } + if (compareTo(aPadded,bPadded) >= 0) { + throw new IllegalArgumentException("b <= a"); + } + if (num <= 0) { + throw new IllegalArgumentException("num cannot be < 0"); + } + byte [] prependHeader = {1, 0}; + final BigInteger startBI = new BigInteger(add(prependHeader, aPadded)); + final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded)); + BigInteger diffBI = stopBI.subtract(startBI); + if (inclusive) { + diffBI = diffBI.add(BigInteger.ONE); + } + final BigInteger splitsBI = BigInteger.valueOf(num + 1); + if(diffBI.compareTo(splitsBI) < 0) { + return null; + } + final BigInteger intervalBI; + try { + intervalBI = diffBI.divide(splitsBI); + } catch(Exception e) { + LOG.error("Exception caught during division", e); + return null; + } + + final Iterator iterator = new Iterator() { + private int i = -1; + + @Override + public boolean hasNext() { + return i < num+1; + } + + @Override + public byte[] next() { + i++; + if (i == 0) return a; + if (i == num + 1) return b; + + BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i))); + byte [] padded = curBI.toByteArray(); + if (padded[1] == 0) + padded = tail(padded, padded.length - 2); + else + padded = tail(padded, padded.length - 1); + return padded; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + }; + + return new Iterable() { + @Override + public Iterator iterator() { + return iterator; + } + }; + } + + /** + * @param bytes array to hash + * @param offset offset to start from + * @param length length to hash + * */ + public static int hashCode(byte[] bytes, int offset, int length) { + int hash = 1; + for (int i = offset; i < offset + length; i++) + hash = (31 * hash) + (int) bytes[i]; + return hash; + } + + /** + * @param t operands + * @return Array of byte arrays made from passed array of Text + */ + public static byte [][] toByteArrays(final String [] t) { + byte [][] result = new byte[t.length][]; + for (int i = 0; i < t.length; i++) { + result[i] = Bytes.toBytes(t[i]); + } + return result; + } + + /** + * @param column operand + * @return A byte array of a byte array where first and only entry is + * column + */ + public static byte [][] toByteArrays(final String column) { + return toByteArrays(toBytes(column)); + } + + /** + * @param column operand + * @return A byte array of a byte array where first and only entry is + * column + */ + public static byte [][] toByteArrays(final byte [] column) { + byte [][] result = new byte[1][]; + result[0] = column; + return result; + } + + /** + * Binary search for keys in indexes. + * + * @param arr array of byte arrays to search for + * @param key the key you want to find + * @param offset the offset in the key you want to find + * @param length the length of the key + * @param comparator a comparator to compare. + * @return zero-based index of the key, if the key is present in the array. + * Otherwise, a value -(i + 1) such that the key is between arr[i - + * 1] and arr[i] non-inclusively, where i is in [0, i], if we define + * arr[-1] = -Inf and arr[N] = Inf for an N-element array. The above + * means that this function can return 2N + 1 different values + * ranging from -(N + 1) to N - 1. + */ + public static int binarySearch(byte [][]arr, byte []key, int offset, + int length, RawComparator comparator) { + int low = 0; + int high = arr.length - 1; + + while (low <= high) { + int mid = (low+high) >>> 1; + // we have to compare in this order, because the comparator order + // has special logic when the 'left side' is a special key. + int cmp = comparator.compare(key, offset, length, + arr[mid], 0, arr[mid].length); + // key lives above the midpoint + if (cmp > 0) + low = mid + 1; + // key lives below the midpoint + else if (cmp < 0) + high = mid - 1; + // BAM. how often does this really happen? + else + return mid; + } + return - (low+1); + } + + /** + * Bytewise binary increment/deincrement of long contained in byte array + * on given amount. + * + * @param value - array of bytes containing long (length <= SIZEOF_LONG) + * @param amount value will be incremented on (deincremented if negative) + * @return array of bytes containing incremented long (length == SIZEOF_LONG) + */ + public static byte [] incrementBytes(byte[] value, long amount) + { + byte[] val = value; + if (val.length < SIZEOF_LONG) { + // Hopefully this doesn't happen too often. + byte [] newvalue; + if (val[0] < 0) { + newvalue = new byte[]{-1, -1, -1, -1, -1, -1, -1, -1}; + } else { + newvalue = new byte[SIZEOF_LONG]; + } + System.arraycopy(val, 0, newvalue, newvalue.length - val.length, + val.length); + val = newvalue; + } else if (val.length > SIZEOF_LONG) { + throw new IllegalArgumentException("Increment Bytes - value too big: " + + val.length); + } + if(amount == 0) return val; + if(val[0] < 0){ + return binaryIncrementNeg(val, amount); + } + return binaryIncrementPos(val, amount); + } + + /* increment/deincrement for positive value */ + private static byte [] binaryIncrementPos(byte [] value, long amount) { + long amo = amount; + int sign = 1; + if (amount < 0) { + amo = -amount; + sign = -1; + } + for(int i=0;i> 8); + int val = value[value.length-i-1] & 0x0ff; + int total = val + cur; + if(total > 255) { + amo += sign; + total %= 256; + } else if (total < 0) { + amo -= sign; + } + value[value.length-i-1] = (byte)total; + if (amo == 0) return value; + } + return value; + } + + /* increment/deincrement for negative value */ + private static byte [] binaryIncrementNeg(byte [] value, long amount) { + long amo = amount; + int sign = 1; + if (amount < 0) { + amo = -amount; + sign = -1; + } + for(int i=0;i> 8); + int val = ((~value[value.length-i-1]) & 0x0ff) + 1; + int total = cur - val; + if(total >= 0) { + amo += sign; + } else if (total < -256) { + amo -= sign; + total %= 256; + } + value[value.length-i-1] = (byte)total; + if (amo == 0) return value; + } + return value; + } + + /** + * Writes a string as a fixed-size field, padded with zeros. + */ + public static void writeStringFixedSize(final DataOutput out, String s, + int size) throws IOException { + byte[] b = toBytes(s); + if (b.length > size) { + throw new IOException("Trying to write " + b.length + " bytes (" + + toStringBinary(b) + ") into a field of length " + size); + } + + out.writeBytes(s); + for (int i = 0; i < size - s.length(); ++i) + out.writeByte(0); + } + + /** + * Reads a fixed-size field and interprets it as a string padded with zeros. + */ + public static String readStringFixedSize(final DataInput in, int size) + throws IOException { + byte[] b = new byte[size]; + in.readFully(b); + int n = b.length; + while (n > 0 && b[n - 1] == 0) + --n; + + return toString(b, 0, n); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/CancelableProgressable.java b/src/main/java/org/apache/hadoop/hbase/util/CancelableProgressable.java new file mode 100644 index 0000000..6d7c389 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/CancelableProgressable.java @@ -0,0 +1,37 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +/** + * Similar interface as {@link org.apache.hadoop.util.Progressable} but returns + * a boolean to support canceling the operation. + *

    + * Used for doing updating of OPENING znode during log replay on region open. + */ +public interface CancelableProgressable { + + /** + * Report progress. Returns true if operations should continue, false if the + * operation should be canceled and rolled back. + * @return whether to continue (true) or cancel (false) the operation + */ + public boolean progress(); + +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java b/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java new file mode 100644 index 0000000..d61238b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java @@ -0,0 +1,99 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.lang.ClassNotFoundException; +import java.util.zip.Checksum; +import java.lang.reflect.Constructor; + +/** + * Utility class that is used to generate a Checksum object. + * The Checksum implementation is pluggable and an application + * can specify their own class that implements their own + * Checksum algorithm. + */ +public class ChecksumFactory { + + static private final Class[] EMPTY_ARRAY = new Class[]{}; + + /** + * Create a new instance of a Checksum object. + * @return The newly created Checksum object + */ + static public Checksum newInstance(String className) throws IOException { + try { + Class clazz = getClassByName(className); + return (Checksum)newInstance(clazz); + } catch (ClassNotFoundException e) { + throw new IOException(e); + } + } + + /** + * Returns a Constructor that can be used to create a Checksum object. + * @return The Constructor that can be used to create a + * new Checksum object. + * @param theClass classname for which an constructor is created + * @return a new Constructor object + */ + static public Constructor newConstructor(String className) + throws IOException { + try { + Class clazz = getClassByName(className); + Constructor ctor = clazz.getDeclaredConstructor(EMPTY_ARRAY); + ctor.setAccessible(true); + return ctor; + } catch (ClassNotFoundException e) { + throw new IOException(e); + } catch (java.lang.NoSuchMethodException e) { + throw new IOException(e); + } + } + + /** Create an object for the given class and initialize it from conf + * + * @param theClass class of which an object is created + * @return a new object + */ + static private T newInstance(Class theClass) { + T result; + try { + Constructor ctor = theClass.getDeclaredConstructor(EMPTY_ARRAY); + ctor.setAccessible(true); + result = ctor.newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + return result; + } + + /** + * Load a class by name. + * @param name the class name. + * @return the class object. + * @throws ClassNotFoundException if the class is not found. + */ + static private Class getClassByName(String name) + throws ClassNotFoundException { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + return Class.forName(name, true, classLoader); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java b/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java new file mode 100644 index 0000000..885625b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.util.zip.Checksum; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Checksum types. The Checksum type is a one byte number + * that stores a representation of the checksum algorithm + * used to encode a hfile. The ordinal of these cannot + * change or else you risk breaking all existing HFiles out there. + */ +public enum ChecksumType { + + NULL((byte)0) { + @Override + public String getName() { + return "NULL"; + } + @Override + public void initialize() { + // do nothing + } + @Override + public Checksum getChecksumObject() throws IOException { + return null; // checksums not used + } + }, + + CRC32((byte)1) { + private volatile Constructor ctor; + + @Override + public String getName() { + return "CRC32"; + } + + @Override + public void initialize() { + final String PURECRC32 = "org.apache.hadoop.util.PureJavaCrc32"; + final String JDKCRC = "java.util.zip.CRC32"; + LOG = LogFactory.getLog(ChecksumType.class); + + // check if hadoop library is available + try { + ctor = ChecksumFactory.newConstructor(PURECRC32); + LOG.info("Checksum using " + PURECRC32); + } catch (Exception e) { + LOG.trace(PURECRC32 + " not available."); + } + try { + // The default checksum class name is java.util.zip.CRC32. + // This is available on all JVMs. + if (ctor == null) { + ctor = ChecksumFactory.newConstructor(JDKCRC); + LOG.info("Checksum can use " + JDKCRC); + } + } catch (Exception e) { + LOG.trace(JDKCRC + " not available."); + } + } + + @Override + public Checksum getChecksumObject() throws IOException { + if (ctor == null) { + throw new IOException("Bad constructor for " + getName()); + } + try { + return (Checksum)ctor.newInstance(); + } catch (Exception e) { + throw new IOException(e); + } + } + }, + + CRC32C((byte)2) { + private transient Constructor ctor; + + @Override + public String getName() { + return "CRC32C"; + } + + @Override + public void initialize() { + final String PURECRC32C = "org.apache.hadoop.util.PureJavaCrc32C"; + LOG = LogFactory.getLog(ChecksumType.class); + try { + ctor = ChecksumFactory.newConstructor(PURECRC32C); + LOG.info("Checksum can use " + PURECRC32C); + } catch (Exception e) { + LOG.trace(PURECRC32C + " not available."); + } + } + + @Override + public Checksum getChecksumObject() throws IOException { + if (ctor == null) { + throw new IOException("Bad constructor for " + getName()); + } + try { + return (Checksum)ctor.newInstance(); + } catch (Exception e) { + throw new IOException(e); + } + } + }; + + private final byte code; + protected Log LOG; + + /** initializes the relevant checksum class object */ + abstract void initialize(); + + /** returns the name of this checksum type */ + public abstract String getName(); + + private ChecksumType(final byte c) { + this.code = c; + initialize(); + } + + /** returns a object that can be used to generate/validate checksums */ + public abstract Checksum getChecksumObject() throws IOException; + + public byte getCode() { + return this.code; + } + + /** + * Cannot rely on enum ordinals . They change if item is removed or moved. + * Do our own codes. + * @param b + * @return Type associated with passed code. + */ + public static ChecksumType codeToType(final byte b) { + for (ChecksumType t : ChecksumType.values()) { + if (t.getCode() == b) { + return t; + } + } + throw new RuntimeException("Unknown checksum type code " + b); + } + + /** + * Map a checksum name to a specific type. + * Do our own names. + * @param b + * @return Type associated with passed code. + */ + public static ChecksumType nameToType(final String name) { + for (ChecksumType t : ChecksumType.values()) { + if (t.getName().equals(name)) { + return t; + } + } + throw new RuntimeException("Unknown checksum type name " + name); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java b/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java new file mode 100644 index 0000000..0c470ad --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.net.URL; +import java.net.URLClassLoader; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.hadoop.classification.InterfaceAudience; + +import com.google.common.base.Preconditions; + +/** + * Base class loader that defines couple shared constants used + * by sub-classes. It also defined method getClassLoadingLock for parallel + * class loading and JDK 1.6 support. This method (getClassLoadingLock) + * is similar to the same method in the base class Java ClassLoader + * introduced in JDK 1.7, but not in JDK 1.6. + */ +@InterfaceAudience.Private +public class ClassLoaderBase extends URLClassLoader { + + // Maps class name to the corresponding lock object + private final ConcurrentHashMap parallelLockMap + = new ConcurrentHashMap(); + + protected static final String DEFAULT_LOCAL_DIR = "/tmp/hbase-local-dir"; + protected static final String LOCAL_DIR_KEY = "hbase.local.dir"; + + /** + * Parent class loader. + */ + protected final ClassLoader parent; + + /** + * Creates a DynamicClassLoader that can load classes dynamically + * from jar files under a specific folder. + * + * @param parent the parent ClassLoader to set. + */ + public ClassLoaderBase(final ClassLoader parent) { + super(new URL[]{}, parent); + Preconditions.checkNotNull(parent, "No parent classloader!"); + this.parent = parent; + } + + /** + * Returns the lock object for class loading operations. + */ + protected Object getClassLoadingLock(String className) { + Object lock = parallelLockMap.get(className); + if (lock != null) { + return lock; + } + + Object newLock = new Object(); + lock = parallelLockMap.putIfAbsent(className, newLock); + if (lock == null) { + lock = newLock; + } + return lock; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java b/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java new file mode 100644 index 0000000..4d6cdfb --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java @@ -0,0 +1,312 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + +/** + * Class for determining the "size" of a class, an attempt to calculate the + * actual bytes that an object of this class will occupy in memory + * + * The core of this class is taken from the Derby project + */ +public class ClassSize { + static final Log LOG = LogFactory.getLog(ClassSize.class); + + private static int nrOfRefsPerObj = 2; + + /** Array overhead */ + public static final int ARRAY; + + /** Overhead for ArrayList(0) */ + public static final int ARRAYLIST; + + /** Overhead for ByteBuffer */ + public static final int BYTE_BUFFER; + + /** Overhead for an Integer */ + public static final int INTEGER; + + /** Overhead for entry in map */ + public static final int MAP_ENTRY; + + /** Object overhead is minimum 2 * reference size (8 bytes on 64-bit) */ + public static final int OBJECT; + + /** Reference size is 8 bytes on 64-bit, 4 bytes on 32-bit */ + public static final int REFERENCE; + + /** String overhead */ + public static final int STRING; + + /** Overhead for TreeMap */ + public static final int TREEMAP; + + /** Overhead for ConcurrentHashMap */ + public static final int CONCURRENT_HASHMAP; + + /** Overhead for ConcurrentHashMap.Entry */ + public static final int CONCURRENT_HASHMAP_ENTRY; + + /** Overhead for ConcurrentHashMap.Segment */ + public static final int CONCURRENT_HASHMAP_SEGMENT; + + /** Overhead for ConcurrentSkipListMap */ + public static final int CONCURRENT_SKIPLISTMAP; + + /** Overhead for ConcurrentSkipListMap Entry */ + public static final int CONCURRENT_SKIPLISTMAP_ENTRY; + + /** Overhead for ReentrantReadWriteLock */ + public static final int REENTRANT_LOCK; + + /** Overhead for AtomicLong */ + public static final int ATOMIC_LONG; + + /** Overhead for AtomicInteger */ + public static final int ATOMIC_INTEGER; + + /** Overhead for AtomicBoolean */ + public static final int ATOMIC_BOOLEAN; + + /** Overhead for CopyOnWriteArraySet */ + public static final int COPYONWRITE_ARRAYSET; + + /** Overhead for CopyOnWriteArrayList */ + public static final int COPYONWRITE_ARRAYLIST; + + /* Are we running on jdk7? */ + private static final boolean JDK7; + static { + final String version = System.getProperty("java.version"); + // Verify String looks like this: 1.6.0_29 + if (!version.matches("\\d\\.\\d\\..*")) { + throw new RuntimeException("Unexpected version format: " + version); + } + // Convert char to int + int major = (int) (version.charAt(0) - '0'); + int minor = (int) (version.charAt(2) - '0'); + JDK7 = major == 1 && minor == 7; + } + + /** + * Method for reading the arc settings and setting overheads according + * to 32-bit or 64-bit architecture. + */ + static { + //Default value is set to 8, covering the case when arcModel is unknown + if (is32BitJVM()) { + REFERENCE = 4; + } else { + REFERENCE = 8; + } + + OBJECT = 2 * REFERENCE; + + ARRAY = align(3 * REFERENCE); + + ARRAYLIST = align(OBJECT + align(REFERENCE) + align(ARRAY) + + (2 * Bytes.SIZEOF_INT)); + + //noinspection PointlessArithmeticExpression + BYTE_BUFFER = align(OBJECT + align(REFERENCE) + align(ARRAY) + + (5 * Bytes.SIZEOF_INT) + + (3 * Bytes.SIZEOF_BOOLEAN) + Bytes.SIZEOF_LONG); + + INTEGER = align(OBJECT + Bytes.SIZEOF_INT); + + MAP_ENTRY = align(OBJECT + 5 * REFERENCE + Bytes.SIZEOF_BOOLEAN); + + TREEMAP = align(OBJECT + (2 * Bytes.SIZEOF_INT) + align(7 * REFERENCE)); + + STRING = align(OBJECT + ARRAY + REFERENCE + ((JDK7? 2: 3) * Bytes.SIZEOF_INT)); + + CONCURRENT_HASHMAP = align(((JDK7? 3: 2) * Bytes.SIZEOF_INT) + ARRAY + + (6 * REFERENCE) + OBJECT); + + CONCURRENT_HASHMAP_ENTRY = align(REFERENCE + OBJECT + (3 * REFERENCE) + + (2 * Bytes.SIZEOF_INT)); + + CONCURRENT_HASHMAP_SEGMENT = align(REFERENCE + OBJECT + + (3 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_FLOAT + ARRAY); + + CONCURRENT_SKIPLISTMAP = align(Bytes.SIZEOF_INT + OBJECT + (8 * REFERENCE)); + + CONCURRENT_SKIPLISTMAP_ENTRY = align( + align(OBJECT + (3 * REFERENCE)) + /* one node per entry */ + align((OBJECT + (3 * REFERENCE))/2)); /* one index per two entries */ + + REENTRANT_LOCK = align(OBJECT + (3 * REFERENCE)); + + ATOMIC_LONG = align(OBJECT + Bytes.SIZEOF_LONG); + + ATOMIC_INTEGER = align(OBJECT + Bytes.SIZEOF_INT); + + ATOMIC_BOOLEAN = align(OBJECT + Bytes.SIZEOF_BOOLEAN); + + COPYONWRITE_ARRAYSET = align(OBJECT + REFERENCE); + + COPYONWRITE_ARRAYLIST = align(OBJECT + (2 * REFERENCE) + ARRAY); + } + + /** + * The estimate of the size of a class instance depends on whether the JVM + * uses 32 or 64 bit addresses, that is it depends on the size of an object + * reference. It is a linear function of the size of a reference, e.g. + * 24 + 5*r where r is the size of a reference (usually 4 or 8 bytes). + * + * This method returns the coefficients of the linear function, e.g. {24, 5} + * in the above example. + * + * @param cl A class whose instance size is to be estimated + * @param debug debug flag + * @return an array of 3 integers. The first integer is the size of the + * primitives, the second the number of arrays and the third the number of + * references. + */ + @SuppressWarnings("unchecked") + private static int [] getSizeCoefficients(Class cl, boolean debug) { + int primitives = 0; + int arrays = 0; + //The number of references that a new object takes + int references = nrOfRefsPerObj; + int index = 0; + + for ( ; null != cl; cl = cl.getSuperclass()) { + Field[] field = cl.getDeclaredFields(); + if (null != field) { + for (Field aField : field) { + if (Modifier.isStatic(aField.getModifiers())) continue; + Class fieldClass = aField.getType(); + if (fieldClass.isArray()) { + arrays++; + references++; + } else if (!fieldClass.isPrimitive()) { + references++; + } else {// Is simple primitive + String name = fieldClass.getName(); + + if (name.equals("int") || name.equals("I")) + primitives += Bytes.SIZEOF_INT; + else if (name.equals("long") || name.equals("J")) + primitives += Bytes.SIZEOF_LONG; + else if (name.equals("boolean") || name.equals("Z")) + primitives += Bytes.SIZEOF_BOOLEAN; + else if (name.equals("short") || name.equals("S")) + primitives += Bytes.SIZEOF_SHORT; + else if (name.equals("byte") || name.equals("B")) + primitives += Bytes.SIZEOF_BYTE; + else if (name.equals("char") || name.equals("C")) + primitives += Bytes.SIZEOF_CHAR; + else if (name.equals("float") || name.equals("F")) + primitives += Bytes.SIZEOF_FLOAT; + else if (name.equals("double") || name.equals("D")) + primitives += Bytes.SIZEOF_DOUBLE; + } + if (debug) { + if (LOG.isDebugEnabled()) { + // Write out region name as string and its encoded name. + LOG.debug("" + index + " " + aField.getName() + " " + aField.getType()); + } + } + index++; + } + } + } + return new int [] {primitives, arrays, references}; + } + + /** + * Estimate the static space taken up by a class instance given the + * coefficients returned by getSizeCoefficients. + * + * @param coeff the coefficients + * + * @param debug debug flag + * @return the size estimate, in bytes + */ + private static long estimateBaseFromCoefficients(int [] coeff, boolean debug) { + long prealign_size = coeff[0] + align(coeff[1] * ARRAY) + coeff[2] * REFERENCE; + + // Round up to a multiple of 8 + long size = align(prealign_size); + if(debug) { + if (LOG.isDebugEnabled()) { + // Write out region name as string and its encoded name. + LOG.debug("Primitives=" + coeff[0] + ", arrays=" + coeff[1] + + ", references(includes " + nrOfRefsPerObj + + " for object overhead)=" + coeff[2] + ", refSize " + REFERENCE + + ", size=" + size + ", prealign_size=" + prealign_size); + } + } + return size; + } + + /** + * Estimate the static space taken up by the fields of a class. This includes + * the space taken up by by references (the pointer) but not by the referenced + * object. So the estimated size of an array field does not depend on the size + * of the array. Similarly the size of an object (reference) field does not + * depend on the object. + * + * @param cl class + * @param debug debug flag + * @return the size estimate in bytes. + */ + @SuppressWarnings("unchecked") + public static long estimateBase(Class cl, boolean debug) { + return estimateBaseFromCoefficients( getSizeCoefficients(cl, debug), debug); + } + + /** + * Aligns a number to 8. + * @param num number to align to 8 + * @return smallest number >= input that is a multiple of 8 + */ + public static int align(int num) { + return (int)(align((long)num)); + } + + /** + * Aligns a number to 8. + * @param num number to align to 8 + * @return smallest number >= input that is a multiple of 8 + */ + public static long align(long num) { + //The 7 comes from that the alignSize is 8 which is the number of bytes + //stored and sent together + return ((num + 7) >> 3) << 3; + } + + /** + * Determines if we are running in a 32-bit JVM. Some unit tests need to + * know this too. + */ + public static boolean is32BitJVM() { + return System.getProperty("sun.arch.data.model").equals("32"); + } + +} + diff --git a/src/main/java/org/apache/hadoop/hbase/util/Classes.java b/src/main/java/org/apache/hadoop/hbase/util/Classes.java new file mode 100644 index 0000000..beca9c2 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Classes.java @@ -0,0 +1,135 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.io.WritableFactories; + +/** + * Utilities for class manipulation. + */ +public class Classes { + + /** + * Dynamic class loader to load filter/comparators + */ + private final static ClassLoader CLASS_LOADER; + + static { + ClassLoader parent = Classes.class.getClassLoader(); + Configuration conf = HBaseConfiguration.create(); + CLASS_LOADER = new DynamicClassLoader(conf, parent); + } + + /** + * Equivalent of {@link Class#forName(String)} which also returns classes for + * primitives like boolean, etc. + * + * @param className + * The name of the class to retrieve. Can be either a normal class or + * a primitive class. + * @return The class specified by className + * @throws ClassNotFoundException + * If the requested class can not be found. + */ + public static Class extendedForName(String className) + throws ClassNotFoundException { + Class valueType; + if (className.equals("boolean")) { + valueType = boolean.class; + } else if (className.equals("byte")) { + valueType = byte.class; + } else if (className.equals("short")) { + valueType = short.class; + } else if (className.equals("int")) { + valueType = int.class; + } else if (className.equals("long")) { + valueType = long.class; + } else if (className.equals("float")) { + valueType = float.class; + } else if (className.equals("double")) { + valueType = double.class; + } else if (className.equals("char")) { + valueType = char.class; + } else { + valueType = Class.forName(className); + } + return valueType; + } + + @SuppressWarnings("rawtypes") + public static String stringify(Class[] classes) { + StringBuilder buf = new StringBuilder(); + if (classes != null) { + for (Class c : classes) { + if (buf.length() > 0) { + buf.append(","); + } + buf.append(c.getName()); + } + } else { + buf.append("NULL"); + } + return buf.toString(); + } + + /** + * Used to dynamically load a filter class, and create a Writable filter. + * This filter class most likely extends Configurable. + * + * @param className the filter class name. + * @return a filter + */ + @SuppressWarnings("unchecked") + public static Filter createWritableForName(String className) { + try { + Class clazz = + (Class) Class.forName(className, true, CLASS_LOADER); + return (Filter)WritableFactories.newInstance(clazz, new Configuration()); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Can't find class " + className); + } + } + + /** + * This method is almost the same as #createWritableForName, except + * that this one doesn't expect the filter class to extends Configurable. + * + * @param className the filter class name. + * @return a filter + */ + @SuppressWarnings("unchecked") + public static Filter createForName(String className) { + try { + Class clazz = + (Class)Class.forName(className, true, CLASS_LOADER); + return (Filter)clazz.newInstance(); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Can't find class " + className); + } catch (InstantiationException e) { + throw new RuntimeException("Couldn't instantiate " + className, e); + } catch (IllegalAccessException e) { + throw new RuntimeException("No access to " + className, e); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java b/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java new file mode 100644 index 0000000..25cec2a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java @@ -0,0 +1,129 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.SortedSet; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner; + +/** + * Utility scanner that wraps a sortable collection and serves + * as a KeyValueScanner. + */ +public class CollectionBackedScanner extends NonLazyKeyValueScanner { + final private Iterable data; + final KeyValue.KVComparator comparator; + private Iterator iter; + private KeyValue current; + + public CollectionBackedScanner(SortedSet set) { + this(set, KeyValue.COMPARATOR); + } + + public CollectionBackedScanner(SortedSet set, + KeyValue.KVComparator comparator) { + this.comparator = comparator; + data = set; + init(); + } + + public CollectionBackedScanner(List list) { + this(list, KeyValue.COMPARATOR); + } + + public CollectionBackedScanner(List list, + KeyValue.KVComparator comparator) { + Collections.sort(list, comparator); + this.comparator = comparator; + data = list; + init(); + } + + public CollectionBackedScanner(KeyValue.KVComparator comparator, + KeyValue... array) { + this.comparator = comparator; + + List tmp = new ArrayList(array.length); + for( int i = 0; i < array.length ; ++i) { + tmp.add(array[i]); + } + Collections.sort(tmp, comparator); + data = tmp; + init(); + } + + private void init() { + iter = data.iterator(); + if(iter.hasNext()){ + current = iter.next(); + } + } + + @Override + public KeyValue peek() { + return current; + } + + @Override + public KeyValue next() { + KeyValue oldCurrent = current; + if(iter.hasNext()){ + current = iter.next(); + } else { + current = null; + } + return oldCurrent; + } + + @Override + public boolean seek(KeyValue seekKv) { + // restart iterator + iter = data.iterator(); + return reseek(seekKv); + } + + @Override + public boolean reseek(KeyValue seekKv) { + while(iter.hasNext()){ + KeyValue next = iter.next(); + int ret = comparator.compare(next, seekKv); + if(ret >= 0){ + current = next; + return true; + } + } + return false; + } + + @Override + public long getSequenceID() { + return 0; + } + + @Override + public void close() { + // do nothing + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java new file mode 100644 index 0000000..6c4ddcf --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java @@ -0,0 +1,177 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.hadoop.hbase.io.hfile.BlockType; +import org.apache.hadoop.hbase.io.hfile.FixedFileTrailer; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileBlock; +import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex; +import org.apache.hadoop.io.RawComparator; + +/** + * A Bloom filter implementation built on top of {@link ByteBloomFilter}, + * encapsulating a set of fixed-size Bloom filters written out at the time of + * {@link org.apache.hadoop.hbase.io.hfile.HFile} generation into the data + * block stream, and loaded on demand at query time. This class only provides + * reading capabilities. + */ +public class CompoundBloomFilter extends CompoundBloomFilterBase + implements BloomFilter { + + /** Used to load chunks on demand */ + private HFile.Reader reader; + + private HFileBlockIndex.BlockIndexReader index; + + private int hashCount; + private Hash hash; + + private long[] numQueriesPerChunk; + private long[] numPositivesPerChunk; + + /** + * De-serialization for compound Bloom filter metadata. Must be consistent + * with what {@link CompoundBloomFilterWriter} does. + * + * @param meta serialized Bloom filter metadata without any magic blocks + * @throws IOException + */ + public CompoundBloomFilter(DataInput meta, HFile.Reader reader) + throws IOException { + this.reader = reader; + + totalByteSize = meta.readLong(); + hashCount = meta.readInt(); + hashType = meta.readInt(); + totalKeyCount = meta.readLong(); + totalMaxKeys = meta.readLong(); + numChunks = meta.readInt(); + comparator = FixedFileTrailer.createComparator( + Bytes.toString(Bytes.readByteArray(meta))); + + hash = Hash.getInstance(hashType); + if (hash == null) { + throw new IllegalArgumentException("Invalid hash type: " + hashType); + } + + index = new HFileBlockIndex.BlockIndexReader(comparator, 1); + index.readRootIndex(meta, numChunks); + } + + @Override + public boolean contains(byte[] key, int keyOffset, int keyLength, + ByteBuffer bloom) { + // We try to store the result in this variable so we can update stats for + // testing, but when an error happens, we log a message and return. + boolean result; + + int block = index.rootBlockContainingKey(key, keyOffset, keyLength); + if (block < 0) { + result = false; // This key is not in the file. + } else { + HFileBlock bloomBlock; + try { + // We cache the block and use a positional read. + bloomBlock = reader.readBlock(index.getRootBlockOffset(block), + index.getRootBlockDataSize(block), true, true, false, + BlockType.BLOOM_CHUNK); + } catch (IOException ex) { + // The Bloom filter is broken, turn it off. + throw new IllegalArgumentException( + "Failed to load Bloom block for key " + + Bytes.toStringBinary(key, keyOffset, keyLength), ex); + } + + ByteBuffer bloomBuf = bloomBlock.getBufferReadOnly(); + result = ByteBloomFilter.contains(key, keyOffset, keyLength, + bloomBuf.array(), bloomBuf.arrayOffset() + bloomBlock.headerSize(), + bloomBlock.getUncompressedSizeWithoutHeader(), hash, hashCount); + } + + if (numQueriesPerChunk != null && block >= 0) { + // Update statistics. Only used in unit tests. + ++numQueriesPerChunk[block]; + if (result) + ++numPositivesPerChunk[block]; + } + + return result; + } + + public boolean supportsAutoLoading() { + return true; + } + + public int getNumChunks() { + return numChunks; + } + + @Override + public RawComparator getComparator() { + return comparator; + } + + public void enableTestingStats() { + numQueriesPerChunk = new long[numChunks]; + numPositivesPerChunk = new long[numChunks]; + } + + public String formatTestingStats() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < numChunks; ++i) { + sb.append("chunk #"); + sb.append(i); + sb.append(": queries="); + sb.append(numQueriesPerChunk[i]); + sb.append(", positives="); + sb.append(numPositivesPerChunk[i]); + sb.append(", positiveRatio="); + sb.append(numPositivesPerChunk[i] * 1.0 / numQueriesPerChunk[i]); + sb.append(";\n"); + } + return sb.toString(); + } + + public long getNumQueriesForTesting(int chunk) { + return numQueriesPerChunk[chunk]; + } + + public long getNumPositivesForTesting(int chunk) { + return numPositivesPerChunk[chunk]; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(ByteBloomFilter.formatStats(this)); + sb.append(ByteBloomFilter.STATS_RECORD_SEP + + "Number of chunks: " + numChunks); + sb.append(ByteBloomFilter.STATS_RECORD_SEP + + "Comparator: " + comparator.getClass().getSimpleName()); + return sb.toString(); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterBase.java b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterBase.java new file mode 100644 index 0000000..a096087 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterBase.java @@ -0,0 +1,95 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.io.RawComparator; + +public class CompoundBloomFilterBase implements BloomFilterBase { + + /** + * At read time, the total number of chunks. At write time, the number of + * chunks created so far. The first chunk has an ID of 0, and the current + * chunk has the ID of numChunks - 1. + */ + protected int numChunks; + + /** + * The Bloom filter version. There used to be a DynamicByteBloomFilter which + * had version 2. + */ + public static final int VERSION = 3; + + /** Target error rate for configuring the filter and for information */ + protected float errorRate; + + /** The total number of keys in all chunks */ + protected long totalKeyCount; + protected long totalByteSize; + protected long totalMaxKeys; + + /** Hash function type to use, as defined in {@link Hash} */ + protected int hashType; + + /** Comparator used to compare Bloom filter keys */ + protected RawComparator comparator; + + @Override + public long getMaxKeys() { + return totalMaxKeys; + } + + @Override + public long getKeyCount() { + return totalKeyCount; + } + + @Override + public long getByteSize() { + return totalByteSize; + } + + private static final byte[] DUMMY = new byte[0]; + + /** + * Prepare an ordered pair of row and qualifier to be compared using + * KeyValue.KeyComparator. This is only used for row-column Bloom + * filters. + */ + @Override + public byte[] createBloomKey(byte[] row, int roffset, int rlength, + byte[] qualifier, int qoffset, int qlength) { + if (qualifier == null) + qualifier = DUMMY; + + // Make sure this does not specify a timestamp so that the default maximum + // (most recent) timestamp is used. + KeyValue kv = KeyValue.createFirstOnRow(row, roffset, rlength, DUMMY, 0, 0, + qualifier, qoffset, qlength); + return kv.getKey(); + } + + @Override + public RawComparator getComparator() { + return comparator; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterWriter.java b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterWriter.java new file mode 100644 index 0000000..8571d9c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterWriter.java @@ -0,0 +1,277 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.Queue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.io.hfile.BlockType; +import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex; +import org.apache.hadoop.hbase.io.hfile.InlineBlockWriter; +import org.apache.hadoop.io.RawComparator; +import org.apache.hadoop.io.Writable; + +/** + * Adds methods required for writing a compound Bloom filter to the data + * section of an {@link org.apache.hadoop.hbase.io.hfile.HFile} to the + * {@link CompoundBloomFilter} class. + */ +public class CompoundBloomFilterWriter extends CompoundBloomFilterBase + implements BloomFilterWriter, InlineBlockWriter { + + protected static final Log LOG = + LogFactory.getLog(CompoundBloomFilterWriter.class); + + /** The current chunk being written to */ + private ByteBloomFilter chunk; + + /** Previous chunk, so that we can create another similar chunk */ + private ByteBloomFilter prevChunk; + + /** Maximum fold factor */ + private int maxFold; + + /** The size of individual Bloom filter chunks to create */ + private int chunkByteSize; + + /** A Bloom filter chunk enqueued for writing */ + private static class ReadyChunk { + int chunkId; + byte[] firstKey; + ByteBloomFilter chunk; + } + + private Queue readyChunks = new LinkedList(); + + /** The first key in the current Bloom filter chunk. */ + private byte[] firstKeyInChunk = null; + + private HFileBlockIndex.BlockIndexWriter bloomBlockIndexWriter = + new HFileBlockIndex.BlockIndexWriter(); + + /** Whether to cache-on-write compound Bloom filter chunks */ + private boolean cacheOnWrite; + + /** + * @param chunkByteSizeHint + * each chunk's size in bytes. The real chunk size might be different + * as required by the fold factor. + * @param errorRate + * target false positive rate + * @param hashType + * hash function type to use + * @param maxFold + * maximum degree of folding allowed + */ + public CompoundBloomFilterWriter(int chunkByteSizeHint, float errorRate, + int hashType, int maxFold, boolean cacheOnWrite, + RawComparator comparator) { + chunkByteSize = ByteBloomFilter.computeFoldableByteSize( + chunkByteSizeHint * 8, maxFold); + + this.errorRate = errorRate; + this.hashType = hashType; + this.maxFold = maxFold; + this.cacheOnWrite = cacheOnWrite; + this.comparator = comparator; + } + + @Override + public boolean shouldWriteBlock(boolean closing) { + enqueueReadyChunk(closing); + return !readyChunks.isEmpty(); + } + + /** + * Enqueue the current chunk if it is ready to be written out. + * + * @param closing true if we are closing the file, so we do not expect new + * keys to show up + */ + private void enqueueReadyChunk(boolean closing) { + if (chunk == null || + (chunk.getKeyCount() < chunk.getMaxKeys() && !closing)) { + return; + } + + if (firstKeyInChunk == null) { + throw new NullPointerException("Trying to enqueue a chunk, " + + "but first key is null: closing=" + closing + ", keyCount=" + + chunk.getKeyCount() + ", maxKeys=" + chunk.getMaxKeys()); + } + + ReadyChunk readyChunk = new ReadyChunk(); + readyChunk.chunkId = numChunks - 1; + readyChunk.chunk = chunk; + readyChunk.firstKey = firstKeyInChunk; + readyChunks.add(readyChunk); + + long prevMaxKeys = chunk.getMaxKeys(); + long prevByteSize = chunk.getByteSize(); + + chunk.compactBloom(); + + if (LOG.isDebugEnabled() && prevByteSize != chunk.getByteSize()) { + LOG.debug("Compacted Bloom chunk #" + readyChunk.chunkId + " from [" + + prevMaxKeys + " max keys, " + prevByteSize + " bytes] to [" + + chunk.getMaxKeys() + " max keys, " + chunk.getByteSize() + + " bytes]"); + } + + totalMaxKeys += chunk.getMaxKeys(); + totalByteSize += chunk.getByteSize(); + + firstKeyInChunk = null; + prevChunk = chunk; + chunk = null; + } + + /** + * Adds a Bloom filter key. This key must be greater than the previous key, + * as defined by the comparator this compound Bloom filter is configured + * with. For efficiency, key monotonicity is not checked here. See + * {@link org.apache.hadoop.hbase.regionserver.StoreFile.Writer#append( + * org.apache.hadoop.hbase.KeyValue)} for the details of deduplication. + */ + @Override + public void add(byte[] bloomKey, int keyOffset, int keyLength) { + if (bloomKey == null) + throw new NullPointerException(); + + enqueueReadyChunk(false); + + if (chunk == null) { + if (firstKeyInChunk != null) { + throw new IllegalStateException("First key in chunk already set: " + + Bytes.toStringBinary(firstKeyInChunk)); + } + firstKeyInChunk = Arrays.copyOfRange(bloomKey, keyOffset, keyOffset + + keyLength); + + if (prevChunk == null) { + // First chunk + chunk = ByteBloomFilter.createBySize(chunkByteSize, errorRate, + hashType, maxFold); + } else { + // Use the same parameters as the last chunk, but a new array and + // a zero key count. + chunk = prevChunk.createAnother(); + } + + if (chunk.getKeyCount() != 0) { + throw new IllegalStateException("keyCount=" + chunk.getKeyCount() + + " > 0"); + } + + chunk.allocBloom(); + ++numChunks; + } + + chunk.add(bloomKey, keyOffset, keyLength); + ++totalKeyCount; + } + + @Override + public void writeInlineBlock(DataOutput out) throws IOException { + // We don't remove the chunk from the queue here, because we might need it + // again for cache-on-write. + ReadyChunk readyChunk = readyChunks.peek(); + + ByteBloomFilter readyChunkBloom = readyChunk.chunk; + readyChunkBloom.getDataWriter().write(out); + } + + @Override + public void blockWritten(long offset, int onDiskSize, int uncompressedSize) { + ReadyChunk readyChunk = readyChunks.remove(); + bloomBlockIndexWriter.addEntry(readyChunk.firstKey, offset, onDiskSize); + } + + @Override + public BlockType getInlineBlockType() { + return BlockType.BLOOM_CHUNK; + } + + private class MetaWriter implements Writable { + protected MetaWriter() {} + + @Override + public void readFields(DataInput in) throws IOException { + throw new IOException("Cant read with this class."); + } + + /** + * This is modeled after {@link ByteBloomFilter.MetaWriter} for simplicity, + * although the two metadata formats do not have to be consistent. This + * does have to be consistent with how {@link + * CompoundBloomFilter#CompoundBloomFilter(DataInput, + * org.apache.hadoop.hbase.io.hfile.HFile.Reader)} reads fields. + */ + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(VERSION); + + out.writeLong(getByteSize()); + out.writeInt(prevChunk.getHashCount()); + out.writeInt(prevChunk.getHashType()); + out.writeLong(getKeyCount()); + out.writeLong(getMaxKeys()); + + // Fields that don't have equivalents in ByteBloomFilter. + out.writeInt(numChunks); + Bytes.writeByteArray(out, + Bytes.toBytes(comparator.getClass().getName())); + + // Write a single-level index without compression or block header. + bloomBlockIndexWriter.writeSingleLevelIndex(out, "Bloom filter"); + } + } + + @Override + public Writable getMetaWriter() { + return new MetaWriter(); + } + + @Override + public void compactBloom() { + } + + @Override + public void allocBloom() { + // Nothing happens here. All allocation happens on demand. + } + + @Override + public Writable getDataWriter() { + return null; + } + + @Override + public boolean cacheOnWrite() { + return cacheOnWrite; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java b/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java new file mode 100644 index 0000000..f37d262 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java @@ -0,0 +1,144 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.io.compress.Compressor; + +/** + * Compression validation test. Checks compression is working. Be sure to run + * on every node in your cluster. + */ +public class CompressionTest { + static final Log LOG = LogFactory.getLog(CompressionTest.class); + + public static boolean testCompression(String codec) { + codec = codec.toLowerCase(); + + Compression.Algorithm a; + + try { + a = Compression.getCompressionAlgorithmByName(codec); + } catch (IllegalArgumentException e) { + LOG.warn("Codec type: " + codec + " is not known"); + return false; + } + + try { + testCompression(a); + return true; + } catch (IOException ignored) { + LOG.warn("Can't instantiate codec: " + codec, ignored); + return false; + } + } + + private final static Boolean[] compressionTestResults + = new Boolean[Compression.Algorithm.values().length]; + static { + for (int i = 0 ; i < compressionTestResults.length ; ++i) { + compressionTestResults[i] = null; + } + } + + public static void testCompression(Compression.Algorithm algo) + throws IOException { + if (compressionTestResults[algo.ordinal()] != null) { + if (compressionTestResults[algo.ordinal()]) { + return ; // already passed test, dont do it again. + } else { + // failed. + throw new IOException("Compression algorithm '" + algo.getName() + "'" + + " previously failed test."); + } + } + + Configuration conf = HBaseConfiguration.create(); + try { + Compressor c = algo.getCompressor(); + algo.returnCompressor(c); + compressionTestResults[algo.ordinal()] = true; // passes + } catch (Throwable t) { + compressionTestResults[algo.ordinal()] = false; // failure + throw new IOException(t); + } + } + + protected static Path path = new Path(".hfile-comp-test"); + + public static void usage() { + System.err.println( + "Usage: CompressionTest none|gz|lzo|snappy\n" + + "\n" + + "For example:\n" + + " hbase " + CompressionTest.class + " file:///tmp/testfile gz\n"); + System.exit(1); + } + + public static void doSmokeTest(FileSystem fs, Path path, String codec) + throws Exception { + Configuration conf = HBaseConfiguration.create(); + HFile.Writer writer = HFile.getWriterFactoryNoCache(conf) + .withPath(fs, path) + .withCompression(codec) + .create(); + writer.append(Bytes.toBytes("testkey"), Bytes.toBytes("testval")); + writer.appendFileInfo(Bytes.toBytes("infokey"), Bytes.toBytes("infoval")); + writer.close(); + + HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf)); + reader.loadFileInfo(); + byte[] key = reader.getFirstKey(); + boolean rc = Bytes.toString(key).equals("testkey"); + reader.close(); + + if (!rc) { + throw new Exception("Read back incorrect result: " + + Bytes.toStringBinary(key)); + } + } + + public static void main(String[] args) throws Exception { + if (args.length != 2) { + usage(); + System.exit(1); + } + + Configuration conf = new Configuration(); + Path path = new Path(args[0]); + FileSystem fs = path.getFileSystem(conf); + try { + doSmokeTest(fs, path, args[1]); + } finally { + fs.delete(path, false); + } + System.out.println("SUCCESS"); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java b/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java new file mode 100644 index 0000000..ee8ca3b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java @@ -0,0 +1,336 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URL; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Collection; +import java.util.Enumeration; +import java.util.concurrent.ConcurrentMap; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; + +import com.google.common.collect.MapMaker; + +/** + * ClassLoader used to load classes for Coprocessor instances. + *

    + * This ClassLoader always tries to load classes from the specified coprocessor + * jar first actually using URLClassLoader logic before delegating to the parent + * ClassLoader, thus avoiding dependency conflicts between HBase's classpath and + * classes in the coprocessor jar. + *

    + * Certain classes are exempt from being loaded by this ClassLoader because it + * would prevent them from being cast to the equivalent classes in the region + * server. For example, the Coprocessor interface needs to be loaded by the + * region server's ClassLoader to prevent a ClassCastException when casting the + * coprocessor implementation. + *

    + * A HDFS path can be used to specify the coprocessor jar. In this case, the jar + * will be copied to local at first under some folder under ${hbase.local.dir}/jars/tmp/. + * The local copy will be removed automatically when the HBase server instance is + * stopped. + *

    + * This ClassLoader also handles resource loading. In most cases this + * ClassLoader will attempt to load resources from the coprocessor jar first + * before delegating to the parent. However, like in class loading, + * some resources need to be handled differently. For all of the Hadoop + * default configurations (e.g. hbase-default.xml) we will check the parent + * ClassLoader first to prevent issues such as failing the HBase default + * configuration version check. + */ +@InterfaceAudience.Private +public class CoprocessorClassLoader extends ClassLoaderBase { + private static final Log LOG = LogFactory.getLog(CoprocessorClassLoader.class); + + // A temporary place ${hbase.local.dir}/jars/tmp/ to store the local + // copy of the jar file and the libraries contained in the jar. + private static final String TMP_JARS_DIR = File.separator + + "jars" + File.separator + "tmp" + File.separator; + + /** + * External class loaders cache keyed by external jar path. + * ClassLoader instance is stored as a weak-reference + * to allow GC'ing when it is not used + * (@see HBASE-7205) + */ + private static final ConcurrentMap classLoadersCache = + new MapMaker().concurrencyLevel(3).weakValues().makeMap(); + + /** + * If the class being loaded starts with any of these strings, we will skip + * trying to load it from the coprocessor jar and instead delegate + * directly to the parent ClassLoader. + */ + private static final String[] CLASS_PREFIX_EXEMPTIONS = new String[] { + // Java standard library: + "com.sun.", + "launcher.", + "java.", + "javax.", + "org.ietf", + "org.omg", + "org.w3c", + "org.xml", + "sunw.", + // logging + "org.apache.commons.logging", + "org.apache.log4j", + "com.hadoop", + // Hadoop/HBase/ZK: + "org.apache.hadoop", + "org.apache.zookeeper", + }; + + /** + * If the resource being loaded matches any of these patterns, we will first + * attempt to load the resource with the parent ClassLoader. Only if the + * resource is not found by the parent do we attempt to load it from the coprocessor jar. + */ + private static final Pattern[] RESOURCE_LOAD_PARENT_FIRST_PATTERNS = + new Pattern[] { + Pattern.compile("^[^-]+-default\\.xml$") + }; + + /** + * Creates a JarClassLoader that loads classes from the given paths. + */ + private CoprocessorClassLoader(ClassLoader parent) { + super(parent); + } + + private void init(Path path, String pathPrefix, + Configuration conf) throws IOException { + if (path == null) { + throw new IOException("The jar path is null"); + } + if (!path.toString().endsWith(".jar")) { + throw new IOException(path.toString() + ": not a jar file?"); + } + + // Copy the jar to the local filesystem + String parentDirPath = + conf.get(LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + TMP_JARS_DIR; + File parentDir = new File(parentDirPath); + if (!parentDir.mkdirs() && !parentDir.isDirectory()) { + throw new RuntimeException("Failed to create local dir " + parentDir.getPath() + + ", CoprocessorClassLoader failed to init"); + } + + FileSystem fs = path.getFileSystem(conf); + File dst = new File(parentDir, "." + pathPrefix + "." + + path.getName() + "." + System.currentTimeMillis() + ".jar"); + fs.copyToLocalFile(path, new Path(dst.toString())); + dst.deleteOnExit(); + + addURL(dst.getCanonicalFile().toURI().toURL()); + + JarFile jarFile = new JarFile(dst.toString()); + try { + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + if (entry.getName().matches("[/]?lib/[^/]+\\.jar")) { + File file = new File(parentDir, "." + pathPrefix + "." + path.getName() + + "." + System.currentTimeMillis() + "." + entry.getName().substring(5)); + IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true); + file.deleteOnExit(); + addURL(file.toURI().toURL()); + } + } + } finally { + jarFile.close(); + } + } + + // This method is used in unit test + public static CoprocessorClassLoader getIfCached(final Path path) { + if (path == null) return null; // No class loader for null path + return classLoadersCache.get(path); + } + + // This method is used in unit test + public static Collection getAllCached() { + return classLoadersCache.values(); + } + + // This method is used in unit test + public static void clearCache() { + classLoadersCache.clear(); + } + + /** + * Get a CoprocessorClassLoader for a coprocessor jar path from cache. + * If not in cache, create one. + * + * @param path the path to the coprocessor jar file to load classes from + * @param parent the parent class loader for exempted classes + * @param pathPrefix a prefix used in temp path name to store the jar file locally + * @param conf the configuration used to create the class loader, if needed + * @return a CoprocessorClassLoader for the coprocessor jar path + * @throws IOException + */ + public static CoprocessorClassLoader getClassLoader(final Path path, + final ClassLoader parent, final String pathPrefix, + final Configuration conf) throws IOException { + CoprocessorClassLoader cl = getIfCached(path); + if (cl != null){ + LOG.debug("Found classloader "+ cl + "for "+ path.toString()); + return cl; + } + + cl = AccessController.doPrivileged(new PrivilegedAction() { + @Override + public CoprocessorClassLoader run() { + return new CoprocessorClassLoader(parent); + } + }); + + cl.init(path, pathPrefix, conf); + + // Cache class loader as a weak value, will be GC'ed when no reference left + CoprocessorClassLoader prev = classLoadersCache.putIfAbsent(path, cl); + if (prev != null) { + // Lost update race, use already added class loader + cl = prev; + } + return cl; + } + + @Override + public Class loadClass(String name) + throws ClassNotFoundException { + // Delegate to the parent immediately if this class is exempt + if (isClassExempt(name)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping exempt class " + name + + " - delegating directly to parent"); + } + return parent.loadClass(name); + } + + synchronized (getClassLoadingLock(name)) { + // Check whether the class has already been loaded: + Class clasz = findLoadedClass(name); + if (clasz != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Class " + name + " already loaded"); + } + } + else { + try { + // Try to find this class using the URLs passed to this ClassLoader + if (LOG.isDebugEnabled()) { + LOG.debug("Finding class: " + name); + } + clasz = findClass(name); + } catch (ClassNotFoundException e) { + // Class not found using this ClassLoader, so delegate to parent + if (LOG.isDebugEnabled()) { + LOG.debug("Class " + name + " not found - delegating to parent"); + } + try { + clasz = parent.loadClass(name); + } catch (ClassNotFoundException e2) { + // Class not found in this ClassLoader or in the parent ClassLoader + // Log some debug output before re-throwing ClassNotFoundException + if (LOG.isDebugEnabled()) { + LOG.debug("Class " + name + " not found in parent loader"); + } + throw e2; + } + } + } + return clasz; + } + } + + @Override + public URL getResource(String name) { + URL resource = null; + boolean parentLoaded = false; + + // Delegate to the parent first if necessary + if (loadResourceUsingParentFirst(name)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Checking parent first for resource " + name); + } + resource = super.getResource(name); + parentLoaded = true; + } + + if (resource == null) { + synchronized (getClassLoadingLock(name)) { + // Try to find the resource in this jar + resource = findResource(name); + if ((resource == null) && !parentLoaded) { + // Not found in this jar and we haven't attempted to load + // the resource in the parent yet; fall back to the parent + resource = super.getResource(name); + } + } + } + return resource; + } + + /** + * Determines whether the given class should be exempt from being loaded + * by this ClassLoader. + * @param name the name of the class to test. + * @return true if the class should *not* be loaded by this ClassLoader; + * false otherwise. + */ + protected boolean isClassExempt(String name) { + for (String exemptPrefix : CLASS_PREFIX_EXEMPTIONS) { + if (name.startsWith(exemptPrefix)) { + return true; + } + } + return false; + } + + /** + * Determines whether we should attempt to load the given resource using the + * parent first before attempting to load the resource using this ClassLoader. + * @param name the name of the resource to test. + * @return true if we should attempt to load the resource using the parent + * first; false if we should attempt to load the resource using this + * ClassLoader first. + */ + protected boolean loadResourceUsingParentFirst(String name) { + for (Pattern resourcePattern : RESOURCE_LOAD_PARENT_FIRST_PATTERNS) { + if (resourcePattern.matcher(name).matches()) { + return true; + } + } + return false; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/DefaultEnvironmentEdge.java b/src/main/java/org/apache/hadoop/hbase/util/DefaultEnvironmentEdge.java new file mode 100644 index 0000000..66f9192 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/DefaultEnvironmentEdge.java @@ -0,0 +1,37 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +/** + * Default implementation of an environment edge. + */ +public class DefaultEnvironmentEdge implements EnvironmentEdge { + + + /** + * {@inheritDoc} + *

    + * This implementation returns {@link System#currentTimeMillis()} + */ + @Override + public long currentTimeMillis() { + return System.currentTimeMillis(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java b/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java new file mode 100644 index 0000000..f9081f2 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java @@ -0,0 +1,95 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.util.List; + +import com.google.common.base.Preconditions; + +public class DirectMemoryUtils { + /** + * @return the setting of -XX:MaxDirectMemorySize as a long. Returns 0 if + * -XX:MaxDirectMemorySize is not set. + */ + + public static long getDirectMemorySize() { + RuntimeMXBean RuntimemxBean = ManagementFactory.getRuntimeMXBean(); + List arguments = RuntimemxBean.getInputArguments(); + long multiplier = 1; //for the byte case. + for (String s : arguments) { + if (s.contains("-XX:MaxDirectMemorySize=")) { + String memSize = s.toLowerCase() + .replace("-xx:maxdirectmemorysize=", "").trim(); + + if (memSize.contains("k")) { + multiplier = 1024; + } + + else if (memSize.contains("m")) { + multiplier = 1048576; + } + + else if (memSize.contains("g")) { + multiplier = 1073741824; + } + memSize = memSize.replaceAll("[^\\d]", ""); + + long retValue = Long.parseLong(memSize); + return retValue * multiplier; + } + + } + return 0; + } + + /** + * DirectByteBuffers are garbage collected by using a phantom reference and a + * reference queue. Every once a while, the JVM checks the reference queue and + * cleans the DirectByteBuffers. However, as this doesn't happen + * immediately after discarding all references to a DirectByteBuffer, it's + * easy to OutOfMemoryError yourself using DirectByteBuffers. This function + * explicitly calls the Cleaner method of a DirectByteBuffer. + * + * @param toBeDestroyed + * The DirectByteBuffer that will be "cleaned". Utilizes reflection. + * + */ + public static void destroyDirectByteBuffer(ByteBuffer toBeDestroyed) + throws IllegalArgumentException, IllegalAccessException, + InvocationTargetException, SecurityException, NoSuchMethodException { + + Preconditions.checkArgument(toBeDestroyed.isDirect(), + "toBeDestroyed isn't direct!"); + + Method cleanerMethod = toBeDestroyed.getClass().getMethod("cleaner"); + cleanerMethod.setAccessible(true); + Object cleaner = cleanerMethod.invoke(toBeDestroyed); + Method cleanMethod = cleaner.getClass().getMethod("clean"); + cleanMethod.setAccessible(true); + cleanMethod.invoke(cleaner); + + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java b/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java new file mode 100644 index 0000000..b8a221f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +/** + * This is a class loader that can load classes dynamically from new + * jar files under a configured folder. The paths to the jar files are + * converted to URLs, and URLClassLoader logic is actually used to load + * classes. This class loader always uses its parent class loader + * to load a class at first. Only if its parent class loader + * can not load a class, we will try to load it using the logic here. + *

    + * The configured folder can be a HDFS path. In this case, the jar files + * under that folder will be copied to local at first under ${hbase.local.dir}/jars/. + * The local copy will be updated if the remote copy is updated, according to its + * last modified timestamp. + *

    + * We can't unload a class already loaded. So we will use the existing + * jar files we already know to load any class which can't be loaded + * using the parent class loader. If we still can't load the class from + * the existing jar files, we will check if any new jar file is added, + * if so, we will load the new jar file and try to load the class again. + * If still failed, a class not found exception will be thrown. + *

    + * Be careful in uploading new jar files and make sure all classes + * are consistent, otherwise, we may not be able to load your + * classes properly. + */ +@InterfaceAudience.Private +public class DynamicClassLoader extends ClassLoaderBase { + private static final Log LOG = + LogFactory.getLog(DynamicClassLoader.class); + + // Dynamic jars are put under ${hbase.local.dir}/jars/ + private static final String DYNAMIC_JARS_DIR = File.separator + + "jars" + File.separator; + + private static final String DYNAMIC_JARS_DIR_KEY = "hbase.dynamic.jars.dir"; + + private File localDir; + + // FileSystem of the remote path, set only if remoteDir != null + private FileSystem remoteDirFs; + private Path remoteDir; + + // Last modified time of local jars + private HashMap jarModifiedTime; + + /** + * Creates a DynamicClassLoader that can load classes dynamically + * from jar files under a specific folder. + * + * @param conf the configuration for the cluster. + * @param parent the parent ClassLoader to set. + */ + public DynamicClassLoader( + final Configuration conf, final ClassLoader parent) { + super(parent); + + jarModifiedTime = new HashMap(); + String localDirPath = conf.get( + LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + DYNAMIC_JARS_DIR; + localDir = new File(localDirPath); + if (!localDir.mkdirs() && !localDir.isDirectory()) { + throw new RuntimeException("Failed to create local dir " + localDir.getPath() + + ", DynamicClassLoader failed to init"); + } + + String remotePath = conf.get(DYNAMIC_JARS_DIR_KEY); + if (remotePath == null || remotePath.equals(localDirPath)) { + remoteDir = null; // ignore if it is the same as the local path + } else { + remoteDir = new Path(remotePath); + try { + remoteDirFs = remoteDir.getFileSystem(conf); + } catch (IOException ioe) { + LOG.warn("Failed to identify the fs of dir " + + remoteDir + ", ignored", ioe); + remoteDir = null; + } + } + } + + @Override + public Class loadClass(String name) + throws ClassNotFoundException { + try { + return parent.loadClass(name); + } catch (ClassNotFoundException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Class " + name + " not found - using dynamical class loader"); + } + + synchronized (getClassLoadingLock(name)) { + // Check whether the class has already been loaded: + Class clasz = findLoadedClass(name); + if (clasz != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Class " + name + " already loaded"); + } + } + else { + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Finding class: " + name); + } + clasz = findClass(name); + } catch (ClassNotFoundException cnfe) { + // Load new jar files if any + if (LOG.isDebugEnabled()) { + LOG.debug("Loading new jar files, if any"); + } + loadNewJars(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Finding class again: " + name); + } + clasz = findClass(name); + } + } + return clasz; + } + } + } + + private synchronized void loadNewJars() { + // Refresh local jar file lists + for (File file: localDir.listFiles()) { + String fileName = file.getName(); + if (jarModifiedTime.containsKey(fileName)) { + continue; + } + if (file.isFile() && fileName.endsWith(".jar")) { + jarModifiedTime.put(fileName, Long.valueOf(file.lastModified())); + try { + URL url = file.toURI().toURL(); + addURL(url); + } catch (MalformedURLException mue) { + // This should not happen, just log it + LOG.warn("Failed to load new jar " + fileName, mue); + } + } + } + + // Check remote files + FileStatus[] statuses = null; + if (remoteDir != null) { + try { + statuses = remoteDirFs.listStatus(remoteDir); + } catch (IOException ioe) { + LOG.warn("Failed to check remote dir status " + remoteDir, ioe); + } + } + if (statuses == null || statuses.length == 0) { + return; // no remote files at all + } + + for (FileStatus status: statuses) { + if (status.isDir()) continue; // No recursive lookup + Path path = status.getPath(); + String fileName = path.getName(); + if (!fileName.endsWith(".jar")) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignored non-jar file " + fileName); + } + continue; // Ignore non-jar files + } + Long cachedLastModificationTime = jarModifiedTime.get(fileName); + if (cachedLastModificationTime != null) { + long lastModified = status.getModificationTime(); + if (lastModified < cachedLastModificationTime.longValue()) { + // There could be some race, for example, someone uploads + // a new one right in the middle the old one is copied to + // local. We can check the size as well. But it is still + // not guaranteed. This should be rare. Most likely, + // we already have the latest one. + // If you are unlucky to hit this race issue, you have + // to touch the remote jar to update its last modified time + continue; + } + } + try { + // Copy it to local + File dst = new File(localDir, fileName); + remoteDirFs.copyToLocalFile(path, new Path(dst.getPath())); + jarModifiedTime.put(fileName, Long.valueOf(dst.lastModified())); + URL url = dst.toURI().toURL(); + addURL(url); + } catch (IOException ioe) { + LOG.warn("Failed to load new jar " + fileName, ioe); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdge.java b/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdge.java new file mode 100644 index 0000000..16e65d3 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdge.java @@ -0,0 +1,36 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +/** + * Has some basic interaction with the environment. Alternate implementations + * can be used where required (eg in tests). + * + * @see EnvironmentEdgeManager + */ +public interface EnvironmentEdge { + + /** + * Returns the currentTimeMillis. + * + * @return currentTimeMillis. + */ + long currentTimeMillis(); +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java b/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java new file mode 100644 index 0000000..ce4b82b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManager.java @@ -0,0 +1,75 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +/** + * Manages a singleton instance of the environment edge. This class shall + * implement static versions of the interface {@link EnvironmentEdge}, then + * defer to the delegate on invocation. + */ +public class EnvironmentEdgeManager { + private static volatile EnvironmentEdge delegate = new DefaultEnvironmentEdge(); + + private EnvironmentEdgeManager() { + + } + + /** + * Retrieves the singleton instance of the {@link EnvironmentEdge} that is + * being managed. + * + * @return the edge. + */ + public static EnvironmentEdge getDelegate() { + return delegate; + } + + /** + * Resets the managed instance to the default instance: {@link + * DefaultEnvironmentEdge}. + */ + public static void reset() { + injectEdge(new DefaultEnvironmentEdge()); + } + + /** + * Injects the given edge such that it becomes the managed entity. If null is + * passed to this method, the default type is assigned to the delegate. + * + * @param edge the new edge. + */ + public static void injectEdge(EnvironmentEdge edge) { + if (edge == null) { + reset(); + } else { + delegate = edge; + } + } + + /** + * Defers to the delegate and calls the + * {@link EnvironmentEdge#currentTimeMillis()} method. + * + * @return current time in millis according to the delegate. + */ + public static long currentTimeMillis() { + return getDelegate().currentTimeMillis(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java new file mode 100644 index 0000000..0f1ea14 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.lang.reflect.InvocationTargetException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.RemoteExceptionHandler; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; +import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException; + + +/** + * Implementation for hdfs + */ +public class FSHDFSUtils extends FSUtils{ + private static final Log LOG = LogFactory.getLog(FSHDFSUtils.class); + + /** + * Lease timeout constant, sourced from HDFS upstream. + * The upstream constant is defined in a private interface, so we + * can't reuse for compatibility reasons. + * NOTE: On versions earlier than Hadoop 0.23, the constant is in + * o.a.h.hdfs.protocol.FSConstants, while for 0.23 and above it is + * in o.a.h.hdfs.protocol.HdfsConstants cause of HDFS-1620. + */ + public static final long LEASE_SOFTLIMIT_PERIOD = 60 * 1000; + + public static final String TEST_TRIGGER_DFS_APPEND = "hbase.test.trigger.dfs.append"; + + @Override + public void recoverFileLease(final FileSystem fs, final Path p, Configuration conf) + throws IOException{ + if (!isAppendSupported(conf)) { + LOG.warn("Running on HDFS without append enabled may result in data loss"); + return; + } + // lease recovery not needed for local file system case. + // currently, local file system doesn't implement append either. + if (!(fs instanceof DistributedFileSystem)) { + return; + } + LOG.info("Recovering file " + p); + long startWaiting = System.currentTimeMillis(); + + // Trying recovery + boolean recovered = false; + long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 300000); + // conf parameter passed from unit test, indicating whether fs.append() should be triggered + boolean triggerAppend = conf.getBoolean(TEST_TRIGGER_DFS_APPEND, false); + // retrying lease recovery may preempt pending lease recovery; default to waiting for 4 seconds + // after calling recoverLease + int waitingPeriod = conf.getInt("hbase.lease.recovery.waiting.period", 4000); + Exception ex = null; + while (!recovered) { + try { + try { + DistributedFileSystem dfs = (DistributedFileSystem) fs; + if (triggerAppend) throw new IOException(); + try { + recovered = (Boolean) DistributedFileSystem.class.getMethod( + "recoverLease", new Class[] { Path.class }).invoke(dfs, p); + if (!recovered) LOG.debug("recoverLease returned false"); + } catch (InvocationTargetException ite) { + // function was properly called, but threw it's own exception + throw (IOException) ite.getCause(); + } + } catch (Exception e) { + LOG.debug("Failed fs.recoverLease invocation, " + e.toString() + + ", trying fs.append instead"); + ex = e; + } + if (ex != null || System.currentTimeMillis() - startWaiting > recoveryTimeout) { + LOG.debug("trying fs.append for " + p + " with " + ex); + ex = null; // assume the following append() call would succeed + FSDataOutputStream out = fs.append(p); + out.close(); + recovered = true; + LOG.debug("fs.append passed"); + } + if (recovered) break; + } catch (IOException e) { + e = RemoteExceptionHandler.checkIOException(e); + if (e instanceof AlreadyBeingCreatedException) { + // We expect that we'll get this message while the lease is still + // within its soft limit, but if we get it past that, it means + // that the RS is holding onto the file even though it lost its + // znode. We could potentially abort after some time here. + long waitedFor = System.currentTimeMillis() - startWaiting; + if (waitedFor > LEASE_SOFTLIMIT_PERIOD) { + LOG.warn("Waited " + waitedFor + "ms for lease recovery on " + p + + ":" + e.getMessage()); + } + } else if (e instanceof LeaseExpiredException && + e.getMessage().contains("File does not exist")) { + // This exception comes out instead of FNFE, fix it + throw new FileNotFoundException( + "The given HLog wasn't found at " + p.toString()); + } else { + throw new IOException("Failed to open " + p + " for append", e); + } + } + try { + Thread.sleep(waitingPeriod); + } catch (InterruptedException ie) { + InterruptedIOException iioe = new InterruptedIOException(); + iioe.initCause(ie); + throw iioe; + } + // we keep original behavior without retrying lease recovery + break; + } + LOG.info("Finished lease recovery attempt for " + p); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSMapRUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSMapRUtils.java new file mode 100644 index 0000000..e70b0d4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/FSMapRUtils.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * MapR implementation. + */ +public class FSMapRUtils extends FSUtils { + private static final Log LOG = LogFactory.getLog(FSMapRUtils.class); + + public void recoverFileLease(final FileSystem fs, final Path p, + Configuration conf) throws IOException { + LOG.info("Recovering file " + p.toString() + + " by changing permission to readonly"); + FsPermission roPerm = new FsPermission((short) 0444); + fs.setPermission(p, roPerm); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java new file mode 100644 index 0000000..6b4c64b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java @@ -0,0 +1,632 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.HBaseFileSystem; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableDescriptors; +import org.apache.hadoop.hbase.TableInfoMissingException; + + +/** + * Implementation of {@link TableDescriptors} that reads descriptors from the + * passed filesystem. It expects descriptors to be in a file under the + * table's directory in FS. Can be read-only -- i.e. does not modify + * the filesystem or can be read and write. + * + *

    Also has utility for keeping up the table descriptors tableinfo file. + * The table schema file is kept under the table directory in the filesystem. + * It has a {@link #TABLEINFO_NAME} prefix and then a suffix that is the + * edit sequenceid: e.g. .tableinfo.0000000003. This sequenceid + * is always increasing. It starts at zero. The table schema file with the + * highest sequenceid has the most recent schema edit. Usually there is one file + * only, the most recent but there may be short periods where there are more + * than one file. Old files are eventually cleaned. Presumption is that there + * will not be lots of concurrent clients making table schema edits. If so, + * the below needs a bit of a reworking and perhaps some supporting api in hdfs. + */ +public class FSTableDescriptors implements TableDescriptors { + private static final Log LOG = LogFactory.getLog(FSTableDescriptors.class); + private final FileSystem fs; + private final Path rootdir; + private final boolean fsreadonly; + long cachehits = 0; + long invocations = 0; + + /** The file name used to store HTD in HDFS */ + public static final String TABLEINFO_NAME = ".tableinfo"; + + // This cache does not age out the old stuff. Thinking is that the amount + // of data we keep up in here is so small, no need to do occasional purge. + // TODO. + private final Map cache = + new ConcurrentHashMap(); + + /** + * Data structure to hold modification time and table descriptor. + */ + static class TableDescriptorModtime { + private final HTableDescriptor descriptor; + private final long modtime; + + TableDescriptorModtime(final long modtime, final HTableDescriptor htd) { + this.descriptor = htd; + this.modtime = modtime; + } + + long getModtime() { + return this.modtime; + } + + HTableDescriptor getTableDescriptor() { + return this.descriptor; + } + } + + public FSTableDescriptors(final FileSystem fs, final Path rootdir) { + this(fs, rootdir, false); + } + + /** + * @param fs + * @param rootdir + * @param fsreadOnly True if we are read-only when it comes to filesystem + * operations; i.e. on remove, we do not do delete in fs. + */ + public FSTableDescriptors(final FileSystem fs, final Path rootdir, + final boolean fsreadOnly) { + super(); + this.fs = fs; + this.rootdir = rootdir; + this.fsreadonly = fsreadOnly; + } + + /* (non-Javadoc) + * @see org.apache.hadoop.hbase.TableDescriptors#getHTableDescriptor(java.lang.String) + */ + @Override + public HTableDescriptor get(final byte [] tablename) + throws IOException { + return get(Bytes.toString(tablename)); + } + + /* (non-Javadoc) + * @see org.apache.hadoop.hbase.TableDescriptors#getTableDescriptor(byte[]) + */ + @Override + public HTableDescriptor get(final String tablename) + throws IOException { + invocations++; + if (HTableDescriptor.ROOT_TABLEDESC.getNameAsString().equals(tablename)) { + cachehits++; + return HTableDescriptor.ROOT_TABLEDESC; + } + if (HTableDescriptor.META_TABLEDESC.getNameAsString().equals(tablename)) { + cachehits++; + return HTableDescriptor.META_TABLEDESC; + } + // .META. and -ROOT- is already handled. If some one tries to get the descriptor for + // .logs, .oldlogs or .corrupt throw an exception. + if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(tablename)) { + throw new IOException("No descriptor found for table = " + tablename); + } + + // Look in cache of descriptors. + TableDescriptorModtime cachedtdm = this.cache.get(tablename); + + if (cachedtdm != null) { + // Check mod time has not changed (this is trip to NN). + if (getTableInfoModtime(this.fs, this.rootdir, tablename) <= cachedtdm.getModtime()) { + cachehits++; + return cachedtdm.getTableDescriptor(); + } + } + + TableDescriptorModtime tdmt = null; + try { + tdmt = getTableDescriptorModtime(this.fs, this.rootdir, tablename); + } catch (NullPointerException e) { + LOG.debug("Exception during readTableDecriptor. Current table name = " + + tablename, e); + } catch (IOException ioe) { + LOG.debug("Exception during readTableDecriptor. Current table name = " + + tablename, ioe); + } + + if (tdmt == null) { + LOG.warn("The following folder is in HBase's root directory and " + + "doesn't contain a table descriptor, " + + "do consider deleting it: " + tablename); + } else { + this.cache.put(tablename, tdmt); + } + return tdmt == null ? null : tdmt.getTableDescriptor(); + } + + /* (non-Javadoc) + * @see org.apache.hadoop.hbase.TableDescriptors#getTableDescriptors(org.apache.hadoop.fs.FileSystem, org.apache.hadoop.fs.Path) + */ + @Override + public Map getAll() + throws IOException { + Map htds = new TreeMap(); + List tableDirs = FSUtils.getTableDirs(fs, rootdir); + for (Path d: tableDirs) { + HTableDescriptor htd = null; + try { + + htd = get(d.getName()); + } catch (FileNotFoundException fnfe) { + // inability of retrieving one HTD shouldn't stop getting the remaining + LOG.warn("Trouble retrieving htd", fnfe); + } + if (htd == null) continue; + htds.put(d.getName(), htd); + } + return htds; + } + + @Override + public void add(HTableDescriptor htd) throws IOException { + if (Bytes.equals(HConstants.ROOT_TABLE_NAME, htd.getName())) { + throw new NotImplementedException(); + } + if (Bytes.equals(HConstants.META_TABLE_NAME, htd.getName())) { + throw new NotImplementedException(); + } + if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(htd.getNameAsString())) { + throw new NotImplementedException(); + } + if (!this.fsreadonly) updateHTableDescriptor(this.fs, this.rootdir, htd); + long modtime = getTableInfoModtime(this.fs, this.rootdir, htd.getNameAsString()); + this.cache.put(htd.getNameAsString(), new TableDescriptorModtime(modtime, htd)); + } + + @Override + public HTableDescriptor remove(final String tablename) + throws IOException { + if (!this.fsreadonly) { + Path tabledir = FSUtils.getTablePath(this.rootdir, tablename); + if (this.fs.exists(tabledir)) { + if (!HBaseFileSystem.deleteDirFromFileSystem(fs, tabledir)) { + throw new IOException("Failed delete of " + tabledir.toString()); + } + } + } + TableDescriptorModtime tdm = this.cache.remove(tablename); + return tdm == null ? null : tdm.getTableDescriptor(); + } + + /** + * Checks if .tableinfo exists for given table + * + * @param fs file system + * @param rootdir root directory of HBase installation + * @param tableName name of table + * @return true if exists + * @throws IOException + */ + public static boolean isTableInfoExists(FileSystem fs, Path rootdir, + String tableName) throws IOException { + FileStatus status = getTableInfoPath(fs, rootdir, tableName); + return status == null? false: fs.exists(status.getPath()); + } + + private static FileStatus getTableInfoPath(final FileSystem fs, + final Path rootdir, final String tableName) + throws IOException { + Path tabledir = FSUtils.getTablePath(rootdir, tableName); + return getTableInfoPath(fs, tabledir); + } + + /** + * Looks under the table directory in the filesystem for files with a + * {@link #TABLEINFO_NAME} prefix. Returns reference to the 'latest' instance. + * @param fs + * @param tabledir + * @return The 'current' tableinfo file. + * @throws IOException + */ + public static FileStatus getTableInfoPath(final FileSystem fs, + final Path tabledir) + throws IOException { + FileStatus [] status = FSUtils.listStatus(fs, tabledir, new PathFilter() { + @Override + public boolean accept(Path p) { + // Accept any file that starts with TABLEINFO_NAME + return p.getName().startsWith(TABLEINFO_NAME); + } + }); + if (status == null || status.length < 1) return null; + Arrays.sort(status, new FileStatusFileNameComparator()); + if (status.length > 1) { + // Clean away old versions of .tableinfo + for (int i = 1; i < status.length; i++) { + Path p = status[i].getPath(); + // Clean up old versions + if (!HBaseFileSystem.deleteFileFromFileSystem(fs, p)) { + LOG.warn("Failed cleanup of " + status); + } else { + LOG.debug("Cleaned up old tableinfo file " + p); + } + } + } + return status[0]; + } + + /** + * Compare {@link FileStatus} instances by {@link Path#getName()}. + * Returns in reverse order. + */ + static class FileStatusFileNameComparator + implements Comparator { + @Override + public int compare(FileStatus left, FileStatus right) { + return -left.compareTo(right); + } + } + + /** + * Width of the sequenceid that is a suffix on a tableinfo file. + */ + static final int WIDTH_OF_SEQUENCE_ID = 10; + + /* + * @param number Number to use as suffix. + * @return Returns zero-prefixed 5-byte wide decimal version of passed + * number (Does absolute in case number is negative). + */ + static String formatTableInfoSequenceId(final int number) { + byte [] b = new byte[WIDTH_OF_SEQUENCE_ID]; + int d = Math.abs(number); + for (int i = b.length - 1; i >= 0; i--) { + b[i] = (byte)((d % 10) + '0'); + d /= 10; + } + return Bytes.toString(b); + } + + /** + * Regex to eat up sequenceid suffix on a .tableinfo file. + * Use regex because may encounter oldstyle .tableinfos where there is no + * sequenceid on the end. + */ + private static final Pattern SUFFIX = + Pattern.compile(TABLEINFO_NAME + "(\\.([0-9]{" + WIDTH_OF_SEQUENCE_ID + "}))?$"); + + + /** + * @param p Path to a .tableinfo file. + * @return The current editid or 0 if none found. + */ + static int getTableInfoSequenceid(final Path p) { + if (p == null) return 0; + Matcher m = SUFFIX.matcher(p.getName()); + if (!m.matches()) throw new IllegalArgumentException(p.toString()); + String suffix = m.group(2); + if (suffix == null || suffix.length() <= 0) return 0; + return Integer.parseInt(m.group(2)); + } + + /** + * @param tabledir + * @param sequenceid + * @return Name of tableinfo file. + */ + static Path getTableInfoFileName(final Path tabledir, final int sequenceid) { + return new Path(tabledir, + TABLEINFO_NAME + "." + formatTableInfoSequenceId(sequenceid)); + } + + /** + * @param fs + * @param rootdir + * @param tableName + * @return Modification time for the table {@link #TABLEINFO_NAME} file + * or 0 if no tableinfo file found. + * @throws IOException + */ + static long getTableInfoModtime(final FileSystem fs, final Path rootdir, + final String tableName) + throws IOException { + FileStatus status = getTableInfoPath(fs, rootdir, tableName); + return status == null? 0: status.getModificationTime(); + } + + /** + * Get HTD from HDFS. + * @param fs + * @param hbaseRootDir + * @param tableName + * @return Descriptor or null if none found. + * @throws IOException + */ + public static HTableDescriptor getTableDescriptor(FileSystem fs, + Path hbaseRootDir, byte[] tableName) + throws IOException { + HTableDescriptor htd = null; + try { + TableDescriptorModtime tdmt = + getTableDescriptorModtime(fs, hbaseRootDir, Bytes.toString(tableName)); + htd = tdmt == null ? null : tdmt.getTableDescriptor(); + } catch (NullPointerException e) { + LOG.debug("Exception during readTableDecriptor. Current table name = " + + Bytes.toString(tableName), e); + } + return htd; + } + + static HTableDescriptor getTableDescriptor(FileSystem fs, + Path hbaseRootDir, String tableName) throws NullPointerException, IOException { + TableDescriptorModtime tdmt = getTableDescriptorModtime(fs, hbaseRootDir, tableName); + return tdmt == null ? null : tdmt.getTableDescriptor(); + } + + static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs, + Path hbaseRootDir, String tableName) throws NullPointerException, IOException{ + // ignore both -ROOT- and .META. tables + if (Bytes.compareTo(Bytes.toBytes(tableName), HConstants.ROOT_TABLE_NAME) == 0 + || Bytes.compareTo(Bytes.toBytes(tableName), HConstants.META_TABLE_NAME) == 0) { + return null; + } + return getTableDescriptorModtime(fs, FSUtils.getTablePath(hbaseRootDir, tableName)); + } + + static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs, Path tableDir) + throws NullPointerException, IOException { + if (tableDir == null) throw new NullPointerException(); + FileStatus status = getTableInfoPath(fs, tableDir); + if (status == null) { + throw new TableInfoMissingException("No .tableinfo file under " + + tableDir.toUri()); + } + FSDataInputStream fsDataInputStream = fs.open(status.getPath()); + HTableDescriptor hTableDescriptor = null; + try { + hTableDescriptor = new HTableDescriptor(); + hTableDescriptor.readFields(fsDataInputStream); + } finally { + fsDataInputStream.close(); + } + return new TableDescriptorModtime(status.getModificationTime(), hTableDescriptor); + } + + public static HTableDescriptor getTableDescriptor(FileSystem fs, Path tableDir) + throws IOException, NullPointerException { + TableDescriptorModtime tdmt = getTableDescriptorModtime(fs, tableDir); + return tdmt == null? null: tdmt.getTableDescriptor(); + } + + + /** + * Update table descriptor + * @param fs + * @param conf + * @param hTableDescriptor + * @return New tableinfo or null if we failed update. + * @throws IOException Thrown if failed update. + */ + static Path updateHTableDescriptor(FileSystem fs, Path rootdir, + HTableDescriptor hTableDescriptor) + throws IOException { + Path tableDir = FSUtils.getTablePath(rootdir, hTableDescriptor.getName()); + Path p = writeTableDescriptor(fs, hTableDescriptor, tableDir, + getTableInfoPath(fs, tableDir)); + if (p == null) throw new IOException("Failed update"); + LOG.info("Updated tableinfo=" + p); + return p; + } + + /** + * Deletes a table's directory from the file system if exists. Used in unit + * tests. + */ + public static void deleteTableDescriptorIfExists(String tableName, + Configuration conf) throws IOException { + FileSystem fs = FSUtils.getCurrentFileSystem(conf); + FileStatus status = getTableInfoPath(fs, FSUtils.getRootDir(conf), tableName); + // The below deleteDirectory works for either file or directory. + if (status != null && fs.exists(status.getPath())) { + FSUtils.deleteDirectory(fs, status.getPath()); + } + } + + /** + * @param fs + * @param hTableDescriptor + * @param tableDir + * @param status + * @return Descriptor file or null if we failed write. + * @throws IOException + */ + private static Path writeTableDescriptor(final FileSystem fs, + final HTableDescriptor hTableDescriptor, final Path tableDir, + final FileStatus status) + throws IOException { + // Get temporary dir into which we'll first write a file to avoid + // half-written file phenomeon. + Path tmpTableDir = new Path(tableDir, ".tmp"); + // What is current sequenceid? We read the current sequenceid from + // the current file. After we read it, another thread could come in and + // compete with us writing out next version of file. The below retries + // should help in this case some but its hard to do guarantees in face of + // concurrent schema edits. + int currentSequenceid = + status == null? 0: getTableInfoSequenceid(status.getPath()); + int sequenceid = currentSequenceid; + // Put arbitrary upperbound on how often we retry + int retries = 10; + int retrymax = currentSequenceid + retries; + Path tableInfoPath = null; + do { + sequenceid += 1; + Path p = getTableInfoFileName(tmpTableDir, sequenceid); + if (fs.exists(p)) { + LOG.debug(p + " exists; retrying up to " + retries + " times"); + continue; + } + try { + writeHTD(fs, p, hTableDescriptor); + tableInfoPath = getTableInfoFileName(tableDir, sequenceid); + if (!HBaseFileSystem.renameDirForFileSystem(fs, p, tableInfoPath)) { + throw new IOException("Failed rename of " + p + " to " + tableInfoPath); + } + } catch (IOException ioe) { + // Presume clash of names or something; go around again. + LOG.debug("Failed write and/or rename; retrying", ioe); + if (!FSUtils.deleteDirectory(fs, p)) { + LOG.warn("Failed cleanup of " + p); + } + tableInfoPath = null; + continue; + } + // Cleanup old schema file. + if (status != null) { + if (!FSUtils.deleteDirectory(fs, status.getPath())) { + LOG.warn("Failed delete of " + status.getPath() + "; continuing"); + } + } + break; + } while (sequenceid < retrymax); + return tableInfoPath; + } + + private static void writeHTD(final FileSystem fs, final Path p, + final HTableDescriptor htd) + throws IOException { + FSDataOutputStream out = HBaseFileSystem.createPathOnFileSystem(fs, p, false); + try { + htd.write(out); + out.write('\n'); + out.write('\n'); + out.write(Bytes.toBytes(htd.toString())); + } finally { + out.close(); + } + } + + /** + * Create new HTableDescriptor in HDFS. Happens when we are creating table. + * + * @param htableDescriptor + * @param conf + */ + public static boolean createTableDescriptor(final HTableDescriptor htableDescriptor, + Configuration conf) + throws IOException { + return createTableDescriptor(htableDescriptor, conf, false); + } + + /** + * Create new HTableDescriptor in HDFS. Happens when we are creating table. If + * forceCreation is true then even if previous table descriptor is present it + * will be overwritten + * + * @param htableDescriptor + * @param conf + * @param forceCreation True if we are to overwrite existing file. + */ + static boolean createTableDescriptor(final HTableDescriptor htableDescriptor, + final Configuration conf, boolean forceCreation) + throws IOException { + FileSystem fs = FSUtils.getCurrentFileSystem(conf); + return createTableDescriptor(fs, FSUtils.getRootDir(conf), htableDescriptor, + forceCreation); + } + + /** + * Create new HTableDescriptor in HDFS. Happens when we are creating table. + * Used by tests. + * @param fs + * @param htableDescriptor + * @param rootdir + */ + public static boolean createTableDescriptor(FileSystem fs, Path rootdir, + HTableDescriptor htableDescriptor) + throws IOException { + return createTableDescriptor(fs, rootdir, htableDescriptor, false); + } + + /** + * Create new HTableDescriptor in HDFS. Happens when we are creating table. If + * forceCreation is true then even if previous table descriptor is present it + * will be overwritten + * + * @param fs + * @param htableDescriptor + * @param rootdir + * @param forceCreation + * @return True if we successfully created file. + */ + public static boolean createTableDescriptor(FileSystem fs, Path rootdir, + HTableDescriptor htableDescriptor, boolean forceCreation) + throws IOException { + Path tabledir = FSUtils.getTablePath(rootdir, htableDescriptor.getNameAsString()); + return createTableDescriptorForTableDirectory(fs, tabledir, htableDescriptor, forceCreation); + } + + /** + * Create a new HTableDescriptor in HDFS in the specified table directory. Happens when we create + * a new table or snapshot a table. + * @param fs filesystem where the descriptor should be written + * @param tabledir directory under which we should write the file + * @param htableDescriptor description of the table to write + * @param forceCreation if true,then even if previous table descriptor is present it will + * be overwritten + * @return true if the we successfully created the file, false if the file + * already exists and we weren't forcing the descriptor creation. + * @throws IOException if a filesystem error occurs + */ + public static boolean createTableDescriptorForTableDirectory(FileSystem fs, Path tabledir, + HTableDescriptor htableDescriptor, boolean forceCreation) throws IOException { + FileStatus status = getTableInfoPath(fs, tabledir); + if (status != null) { + LOG.info("Current tableInfoPath = " + status.getPath()); + if (!forceCreation) { + if (fs.exists(status.getPath()) && status.getLen() > 0) { + LOG.info("TableInfo already exists.. Skipping creation"); + return false; + } + } + } + Path p = writeTableDescriptor(fs, htableDescriptor, tabledir, status); + return p != null; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java new file mode 100644 index 0000000..1bc3eea --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -0,0 +1,1387 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.HBaseFileSystem; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HDFSBlocksDistribution; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.RemoteExceptionHandler; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; + +/** + * Utility methods for interacting with the underlying file system. + */ +public abstract class FSUtils { + private static final Log LOG = LogFactory.getLog(FSUtils.class); + + /** Full access permissions (starting point for a umask) */ + private static final String FULL_RWX_PERMISSIONS = "777"; + + protected FSUtils() { + super(); + } + + public static FSUtils getInstance(FileSystem fs, Configuration conf) { + String scheme = fs.getUri().getScheme(); + if (scheme == null) { + LOG.warn("Could not find scheme for uri " + + fs.getUri() + ", default to hdfs"); + scheme = "hdfs"; + } + Class fsUtilsClass = conf.getClass("hbase.fsutil." + + scheme + ".impl", FSHDFSUtils.class); // Default to HDFS impl + FSUtils fsUtils = (FSUtils)ReflectionUtils.newInstance(fsUtilsClass, conf); + return fsUtils; + } + + /** + * Delete if exists. + * @param fs filesystem object + * @param dir directory to delete + * @return True if deleted dir + * @throws IOException e + */ + public static boolean deleteDirectory(final FileSystem fs, final Path dir) + throws IOException { + return fs.exists(dir) && fs.delete(dir, true); + } + + /** + * Check if directory exists. If it does not, create it. + * @param fs filesystem object + * @param dir path to check + * @return Path + * @throws IOException e + */ + public Path checkdir(final FileSystem fs, final Path dir) throws IOException { + if (!fs.exists(dir)) { + HBaseFileSystem.makeDirOnFileSystem(fs, dir); + } + return dir; + } + + /** + * Create the specified file on the filesystem. By default, this will: + *

      + *
    1. overwrite the file if it exists
      2. + *
    2. apply the umask in the configuration (if it is enabled)
      3. + *
    3. use the fs configured buffer size (or {@value DEFAULT_BUFFER_SIZE} if + * not set)
      4. + *
    4. use the default replication
      5. + *
    5. use the default block size
      6. + *
    6. not track progress
      7. + *
    + * + * @param fs {@link FileSystem} on which to write the file + * @param path {@link Path} to the file to write + * @return output stream to the created file + * @throws IOException if the file cannot be created + */ + public static FSDataOutputStream create(FileSystem fs, Path path, + FsPermission perm) throws IOException { + return create(fs, path, perm, true); + } + + /** + * Create the specified file on the filesystem. By default, this will: + *
      + *
    1. apply the umask in the configuration (if it is enabled)
      2. + *
    2. use the fs configured buffer size (or {@value DEFAULT_BUFFER_SIZE} if + * not set)
      3. + *
    3. use the default replication
      4. + *
    4. use the default block size
      5. + *
    5. not track progress
      6. + *
    + * + * @param fs {@link FileSystem} on which to write the file + * @param path {@link Path} to the file to write + * @param perm + * @param overwrite Whether or not the created file should be overwritten. + * @return output stream to the created file + * @throws IOException if the file cannot be created + */ + public static FSDataOutputStream create(FileSystem fs, Path path, FsPermission perm, + boolean overwrite) throws IOException { + LOG.debug("Creating file=" + path + " with permission=" + perm); + return HBaseFileSystem.createPathWithPermsOnFileSystem(fs, path, perm, overwrite); + } + + /** + * Get the file permissions specified in the configuration, if they are + * enabled. + * + * @param fs filesystem that the file will be created on. + * @param conf configuration to read for determining if permissions are + * enabled and which to use + * @param permssionConfKey property key in the configuration to use when + * finding the permission + * @return the permission to use when creating a new file on the fs. If + * special permissions are not specified in the configuration, then + * the default permissions on the the fs will be returned. + */ + public static FsPermission getFilePermissions(final FileSystem fs, + final Configuration conf, final String permssionConfKey) { + boolean enablePermissions = conf.getBoolean( + HConstants.ENABLE_DATA_FILE_UMASK, false); + + if (enablePermissions) { + try { + FsPermission perm = new FsPermission(FULL_RWX_PERMISSIONS); + // make sure that we have a mask, if not, go default. + String mask = conf.get(permssionConfKey); + if (mask == null) + return FsPermission.getDefault(); + // appy the umask + FsPermission umask = new FsPermission(mask); + return perm.applyUMask(umask); + } catch (IllegalArgumentException e) { + LOG.warn( + "Incorrect umask attempted to be created: " + + conf.get(permssionConfKey) + + ", using default file permissions.", e); + return FsPermission.getDefault(); + } + } + return FsPermission.getDefault(); + } + + /** + * Checks to see if the specified file system is available + * + * @param fs filesystem + * @throws IOException e + */ + public static void checkFileSystemAvailable(final FileSystem fs) + throws IOException { + if (!(fs instanceof DistributedFileSystem)) { + return; + } + IOException exception = null; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + if (dfs.exists(new Path("/"))) { + return; + } + } catch (IOException e) { + exception = RemoteExceptionHandler.checkIOException(e); + } + try { + fs.close(); + } catch (Exception e) { + LOG.error("file system close failed: ", e); + } + IOException io = new IOException("File system is not available"); + io.initCause(exception); + throw io; + } + + /** + * We use reflection because {@link DistributedFileSystem#setSafeMode( + * FSConstants.SafeModeAction action, boolean isChecked)} is not in hadoop 1.1 + * + * @param dfs + * @return whether we're in safe mode + * @throws IOException + */ + private static boolean isInSafeMode(DistributedFileSystem dfs) throws IOException { + boolean inSafeMode = false; + try { + Method m = DistributedFileSystem.class.getMethod("setSafeMode", new Class []{ + org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.class, boolean.class}); + inSafeMode = (Boolean) m.invoke(dfs, + org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET, true); + } catch (Exception e) { + if (e instanceof IOException) throw (IOException) e; + + // Check whether dfs is on safemode. + inSafeMode = dfs.setSafeMode( + org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET); + } + return inSafeMode; + } + + /** + * Check whether dfs is in safemode. + * @param conf + * @throws IOException + */ + public static void checkDfsSafeMode(final Configuration conf) + throws IOException { + boolean isInSafeMode = false; + FileSystem fs = FileSystem.get(conf); + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem dfs = (DistributedFileSystem)fs; + isInSafeMode = isInSafeMode(dfs); + } + if (isInSafeMode) { + throw new IOException("File system is in safemode, it can't be written now"); + } + } + + /** + * Verifies current version of file system + * + * @param fs filesystem object + * @param rootdir root hbase directory + * @return null if no version file exists, version string otherwise. + * @throws IOException e + */ + public static String getVersion(FileSystem fs, Path rootdir) + throws IOException { + Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME); + String version = null; + if (fs.exists(versionFile)) { + FSDataInputStream s = + fs.open(versionFile); + try { + version = DataInputStream.readUTF(s); + } catch (EOFException eof) { + LOG.warn("Version file was empty, odd, will try to set it."); + } finally { + s.close(); + } + } + return version; + } + + /** + * Verifies current version of file system + * + * @param fs file system + * @param rootdir root directory of HBase installation + * @param message if true, issues a message on System.out + * + * @throws IOException e + */ + public static void checkVersion(FileSystem fs, Path rootdir, + boolean message) throws IOException { + checkVersion(fs, rootdir, message, 0, + HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS); + } + + /** + * Verifies current version of file system + * + * @param fs file system + * @param rootdir root directory of HBase installation + * @param message if true, issues a message on System.out + * @param wait wait interval + * @param retries number of times to retry + * + * @throws IOException e + */ + public static void checkVersion(FileSystem fs, Path rootdir, + boolean message, int wait, int retries) throws IOException { + String version = getVersion(fs, rootdir); + + if (version == null) { + if (!rootRegionExists(fs, rootdir)) { + // rootDir is empty (no version file and no root region) + // just create new version file (HBASE-1195) + FSUtils.setVersion(fs, rootdir, wait, retries); + return; + } + } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0) + return; + + // version is deprecated require migration + // Output on stdout so user sees it in terminal. + String msg = "HBase file layout needs to be upgraded." + + " You have version " + version + + " and I want version " + HConstants.FILE_SYSTEM_VERSION + + ". Is your hbase.rootdir valid? If so, you may need to run " + + "'hbase hbck -fixVersionFile'."; + if (message) { + System.out.println("WARNING! " + msg); + } + throw new FileSystemVersionException(msg); + } + + /** + * Sets version of file system + * + * @param fs filesystem object + * @param rootdir hbase root + * @throws IOException e + */ + public static void setVersion(FileSystem fs, Path rootdir) + throws IOException { + setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0, + HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS); + } + + /** + * Sets version of file system + * + * @param fs filesystem object + * @param rootdir hbase root + * @param wait time to wait for retry + * @param retries number of times to retry before failing + * @throws IOException e + */ + public static void setVersion(FileSystem fs, Path rootdir, int wait, int retries) + throws IOException { + setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, wait, retries); + } + + /** + * Return the number of bytes that large input files should be optimally + * be split into to minimize i/o time. + * + * use reflection to search for getDefaultBlockSize(Path f) + * if the method doesn't exist, fall back to using getDefaultBlockSize() + * + * @param fs filesystem object + * @return the default block size for the path's filesystem + * @throws IOException e + */ + public static long getDefaultBlockSize(final FileSystem fs, final Path path) throws IOException { + Method m = null; + Class cls = fs.getClass(); + try { + m = cls.getMethod("getDefaultBlockSize", new Class[] { Path.class }); + } catch (NoSuchMethodException e) { + LOG.info("FileSystem doesn't support getDefaultBlockSize"); + } catch (SecurityException e) { + LOG.info("Doesn't have access to getDefaultBlockSize on FileSystems", e); + m = null; // could happen on setAccessible() + } + if (m == null) { + return fs.getDefaultBlockSize(); + } else { + try { + Object ret = m.invoke(fs, path); + return ((Long)ret).longValue(); + } catch (Exception e) { + throw new IOException(e); + } + } + } + + /* + * Get the default replication. + * + * use reflection to search for getDefaultReplication(Path f) + * if the method doesn't exist, fall back to using getDefaultReplication() + * + * @param fs filesystem object + * @param f path of file + * @return default replication for the path's filesystem + * @throws IOException e + */ + public static short getDefaultReplication(final FileSystem fs, final Path path) throws IOException { + Method m = null; + Class cls = fs.getClass(); + try { + m = cls.getMethod("getDefaultReplication", new Class[] { Path.class }); + } catch (NoSuchMethodException e) { + LOG.info("FileSystem doesn't support getDefaultReplication"); + } catch (SecurityException e) { + LOG.info("Doesn't have access to getDefaultReplication on FileSystems", e); + m = null; // could happen on setAccessible() + } + if (m == null) { + return fs.getDefaultReplication(); + } else { + try { + Object ret = m.invoke(fs, path); + return ((Number)ret).shortValue(); + } catch (Exception e) { + throw new IOException(e); + } + } + } + + /** + * Returns the default buffer size to use during writes. + * + * The size of the buffer should probably be a multiple of hardware + * page size (4096 on Intel x86), and it determines how much data is + * buffered during read and write operations. + * + * @param fs filesystem object + * @return default buffer size to use during writes + */ + public static int getDefaultBufferSize(final FileSystem fs) { + return fs.getConf().getInt("io.file.buffer.size", 4096); + } + + /** + * Sets version of file system + * + * @param fs filesystem object + * @param rootdir hbase root directory + * @param version version to set + * @param wait time to wait for retry + * @param retries number of times to retry before throwing an IOException + * @throws IOException e + */ + public static void setVersion(FileSystem fs, Path rootdir, String version, + int wait, int retries) throws IOException { + Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME); + while (true) { + try { + FSDataOutputStream s = fs.create(versionFile); + s.writeUTF(version); + LOG.debug("Created version file at " + rootdir.toString() + + " set its version at:" + version); + s.close(); + return; + } catch (IOException e) { + if (retries > 0) { + LOG.warn("Unable to create version file at " + rootdir.toString() + + ", retrying: " + e.getMessage()); + fs.delete(versionFile, false); + try { + if (wait > 0) { + Thread.sleep(wait); + } + } catch (InterruptedException ex) { + // ignore + } + retries--; + } else { + throw e; + } + } + } + } + + /** + * Checks that a cluster ID file exists in the HBase root directory + * @param fs the root directory FileSystem + * @param rootdir the HBase root directory in HDFS + * @param wait how long to wait between retries + * @return true if the file exists, otherwise false + * @throws IOException if checking the FileSystem fails + */ + public static boolean checkClusterIdExists(FileSystem fs, Path rootdir, + int wait) throws IOException { + while (true) { + try { + Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME); + return fs.exists(filePath); + } catch (IOException ioe) { + if (wait > 0) { + LOG.warn("Unable to check cluster ID file in " + rootdir.toString() + + ", retrying in "+wait+"msec: "+StringUtils.stringifyException(ioe)); + try { + Thread.sleep(wait); + } catch (InterruptedException ie) { + Thread.interrupted(); + break; + } + } else { + throw ioe; + } + } + } + return false; + } + + /** + * Returns the value of the unique cluster ID stored for this HBase instance. + * @param fs the root directory FileSystem + * @param rootdir the path to the HBase root directory + * @return the unique cluster identifier + * @throws IOException if reading the cluster ID file fails + */ + public static String getClusterId(FileSystem fs, Path rootdir) + throws IOException { + Path idPath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME); + String clusterId = null; + if (fs.exists(idPath)) { + FSDataInputStream in = fs.open(idPath); + try { + clusterId = in.readUTF(); + } catch (EOFException eof) { + LOG.warn("Cluster ID file "+idPath.toString()+" was empty"); + } finally{ + in.close(); + } + } else { + LOG.warn("Cluster ID file does not exist at " + idPath.toString()); + } + return clusterId; + } + + /** + * Writes a new unique identifier for this cluster to the "hbase.id" file + * in the HBase root directory + * @param fs the root directory FileSystem + * @param rootdir the path to the HBase root directory + * @param clusterId the unique identifier to store + * @param wait how long (in milliseconds) to wait between retries + * @throws IOException if writing to the FileSystem fails and no wait value + */ + public static void setClusterId(FileSystem fs, Path rootdir, String clusterId, + int wait) throws IOException { + while (true) { + try { + Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME); + FSDataOutputStream s = fs.create(filePath); + s.writeUTF(clusterId); + s.close(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created cluster ID file at " + filePath.toString() + + " with ID: " + clusterId); + } + return; + } catch (IOException ioe) { + if (wait > 0) { + LOG.warn("Unable to create cluster ID file in " + rootdir.toString() + + ", retrying in "+wait+"msec: "+StringUtils.stringifyException(ioe)); + try { + Thread.sleep(wait); + } catch (InterruptedException ie) { + Thread.interrupted(); + break; + } + } else { + throw ioe; + } + } + } + } + + /** + * Verifies root directory path is a valid URI with a scheme + * + * @param root root directory path + * @return Passed root argument. + * @throws IOException if not a valid URI with a scheme + */ + public static Path validateRootPath(Path root) throws IOException { + try { + URI rootURI = new URI(root.toString()); + String scheme = rootURI.getScheme(); + if (scheme == null) { + throw new IOException("Root directory does not have a scheme"); + } + return root; + } catch (URISyntaxException e) { + IOException io = new IOException("Root directory path is not a valid " + + "URI -- check your " + HConstants.HBASE_DIR + " configuration"); + io.initCause(e); + throw io; + } + } + + /** + * If DFS, check safe mode and if so, wait until we clear it. + * @param conf configuration + * @param wait Sleep between retries + * @throws IOException e + */ + public static void waitOnSafeMode(final Configuration conf, + final long wait) + throws IOException { + FileSystem fs = FileSystem.get(conf); + if (!(fs instanceof DistributedFileSystem)) return; + DistributedFileSystem dfs = (DistributedFileSystem)fs; + // Make sure dfs is not in safe mode + while (isInSafeMode(dfs)) { + LOG.info("Waiting for dfs to exit safe mode..."); + try { + Thread.sleep(wait); + } catch (InterruptedException e) { + //continue + } + } + } + + /** + * Return the 'path' component of a Path. In Hadoop, Path is an URI. This + * method returns the 'path' component of a Path's URI: e.g. If a Path is + * hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir, + * this method returns /hbase_trunk/TestTable/compaction.dir. + * This method is useful if you want to print out a Path without qualifying + * Filesystem instance. + * @param p Filesystem Path whose 'path' component we are to return. + * @return Path portion of the Filesystem + */ + public static String getPath(Path p) { + return p.toUri().getPath(); + } + + /** + * @param c configuration + * @return Path to hbase root directory: i.e. hbase.rootdir from + * configuration as a qualified Path. + * @throws IOException e + */ + public static Path getRootDir(final Configuration c) throws IOException { + Path p = new Path(c.get(HConstants.HBASE_DIR)); + FileSystem fs = p.getFileSystem(c); + return p.makeQualified(fs); + } + + public static void setRootDir(final Configuration c, final Path root) throws IOException { + c.set(HConstants.HBASE_DIR, root.toString()); + } + + /** + * Checks if root region exists + * + * @param fs file system + * @param rootdir root directory of HBase installation + * @return true if exists + * @throws IOException e + */ + public static boolean rootRegionExists(FileSystem fs, Path rootdir) + throws IOException { + Path rootRegionDir = + HRegion.getRegionDir(rootdir, HRegionInfo.ROOT_REGIONINFO); + return fs.exists(rootRegionDir); + } + + /** + * Compute HDFS blocks distribution of a given file, or a portion of the file + * @param fs file system + * @param status file status of the file + * @param start start position of the portion + * @param length length of the portion + * @return The HDFS blocks distribution + */ + static public HDFSBlocksDistribution computeHDFSBlocksDistribution( + final FileSystem fs, FileStatus status, long start, long length) + throws IOException { + HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution(); + BlockLocation [] blockLocations = + fs.getFileBlockLocations(status, start, length); + for(BlockLocation bl : blockLocations) { + String [] hosts = bl.getHosts(); + long len = bl.getLength(); + blocksDistribution.addHostsAndBlockWeight(hosts, len); + } + + return blocksDistribution; + } + + + + /** + * Runs through the hbase rootdir and checks all stores have only + * one file in them -- that is, they've been major compacted. Looks + * at root and meta tables too. + * @param fs filesystem + * @param hbaseRootDir hbase root directory + * @return True if this hbase install is major compacted. + * @throws IOException e + */ + public static boolean isMajorCompacted(final FileSystem fs, + final Path hbaseRootDir) + throws IOException { + // Presumes any directory under hbase.rootdir is a table. + FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs)); + for (FileStatus tableDir : tableDirs) { + // Skip the .log directory. All others should be tables. Inside a table, + // there are compaction.dir directories to skip. Otherwise, all else + // should be regions. Then in each region, should only be family + // directories. Under each of these, should be one file only. + Path d = tableDir.getPath(); + if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) { + continue; + } + FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs)); + for (FileStatus regionDir : regionDirs) { + Path dd = regionDir.getPath(); + if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) { + continue; + } + // Else its a region name. Now look in region for families. + FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs)); + for (FileStatus familyDir : familyDirs) { + Path family = familyDir.getPath(); + // Now in family make sure only one file. + FileStatus[] familyStatus = fs.listStatus(family); + if (familyStatus.length > 1) { + LOG.debug(family.toString() + " has " + familyStatus.length + + " files."); + return false; + } + } + } + } + return true; + } + + // TODO move this method OUT of FSUtils. No dependencies to HMaster + /** + * Returns the total overall fragmentation percentage. Includes .META. and + * -ROOT- as well. + * + * @param master The master defining the HBase root and file system. + * @return A map for each table and its percentage. + * @throws IOException When scanning the directory fails. + */ + public static int getTotalTableFragmentation(final HMaster master) + throws IOException { + Map map = getTableFragmentation(master); + return map != null && map.size() > 0 ? map.get("-TOTAL-") : -1; + } + + /** + * Runs through the HBase rootdir and checks how many stores for each table + * have more than one file in them. Checks -ROOT- and .META. too. The total + * percentage across all tables is stored under the special key "-TOTAL-". + * + * @param master The master defining the HBase root and file system. + * @return A map for each table and its percentage. + * @throws IOException When scanning the directory fails. + */ + public static Map getTableFragmentation( + final HMaster master) + throws IOException { + Path path = getRootDir(master.getConfiguration()); + // since HMaster.getFileSystem() is package private + FileSystem fs = path.getFileSystem(master.getConfiguration()); + return getTableFragmentation(fs, path); + } + + /** + * Runs through the HBase rootdir and checks how many stores for each table + * have more than one file in them. Checks -ROOT- and .META. too. The total + * percentage across all tables is stored under the special key "-TOTAL-". + * + * @param fs The file system to use. + * @param hbaseRootDir The root directory to scan. + * @return A map for each table and its percentage. + * @throws IOException When scanning the directory fails. + */ + public static Map getTableFragmentation( + final FileSystem fs, final Path hbaseRootDir) + throws IOException { + Map frags = new HashMap(); + int cfCountTotal = 0; + int cfFragTotal = 0; + DirFilter df = new DirFilter(fs); + // presumes any directory under hbase.rootdir is a table + FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df); + for (FileStatus tableDir : tableDirs) { + // Skip the .log directory. All others should be tables. Inside a table, + // there are compaction.dir directories to skip. Otherwise, all else + // should be regions. Then in each region, should only be family + // directories. Under each of these, should be one file only. + Path d = tableDir.getPath(); + if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) { + continue; + } + int cfCount = 0; + int cfFrag = 0; + FileStatus[] regionDirs = fs.listStatus(d, df); + for (FileStatus regionDir : regionDirs) { + Path dd = regionDir.getPath(); + if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) { + continue; + } + // else its a region name, now look in region for families + FileStatus[] familyDirs = fs.listStatus(dd, df); + for (FileStatus familyDir : familyDirs) { + cfCount++; + cfCountTotal++; + Path family = familyDir.getPath(); + // now in family make sure only one file + FileStatus[] familyStatus = fs.listStatus(family); + if (familyStatus.length > 1) { + cfFrag++; + cfFragTotal++; + } + } + } + // compute percentage per table and store in result list + frags.put(d.getName(), Math.round((float) cfFrag / cfCount * 100)); + } + // set overall percentage for all tables + frags.put("-TOTAL-", Math.round((float) cfFragTotal / cfCountTotal * 100)); + return frags; + } + + /** + * Expects to find -ROOT- directory. + * @param fs filesystem + * @param hbaseRootDir hbase root directory + * @return True if this a pre020 layout. + * @throws IOException e + */ + public static boolean isPre020FileLayout(final FileSystem fs, + final Path hbaseRootDir) + throws IOException { + Path mapfiles = new Path(new Path(new Path(new Path(hbaseRootDir, "-ROOT-"), + "70236052"), "info"), "mapfiles"); + return fs.exists(mapfiles); + } + + /** + * Runs through the hbase rootdir and checks all stores have only + * one file in them -- that is, they've been major compacted. Looks + * at root and meta tables too. This version differs from + * {@link #isMajorCompacted(FileSystem, Path)} in that it expects a + * pre-0.20.0 hbase layout on the filesystem. Used migrating. + * @param fs filesystem + * @param hbaseRootDir hbase root directory + * @return True if this hbase install is major compacted. + * @throws IOException e + */ + public static boolean isMajorCompactedPre020(final FileSystem fs, + final Path hbaseRootDir) + throws IOException { + // Presumes any directory under hbase.rootdir is a table. + FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs)); + for (FileStatus tableDir : tableDirs) { + // Inside a table, there are compaction.dir directories to skip. + // Otherwise, all else should be regions. Then in each region, should + // only be family directories. Under each of these, should be a mapfile + // and info directory and in these only one file. + Path d = tableDir.getPath(); + if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) { + continue; + } + FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs)); + for (FileStatus regionDir : regionDirs) { + Path dd = regionDir.getPath(); + if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) { + continue; + } + // Else its a region name. Now look in region for families. + FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs)); + for (FileStatus familyDir : familyDirs) { + Path family = familyDir.getPath(); + FileStatus[] infoAndMapfile = fs.listStatus(family); + // Assert that only info and mapfile in family dir. + if (infoAndMapfile.length != 0 && infoAndMapfile.length != 2) { + LOG.debug(family.toString() + + " has more than just info and mapfile: " + infoAndMapfile.length); + return false; + } + // Make sure directory named info or mapfile. + for (int ll = 0; ll < 2; ll++) { + if (infoAndMapfile[ll].getPath().getName().equals("info") || + infoAndMapfile[ll].getPath().getName().equals("mapfiles")) + continue; + LOG.debug("Unexpected directory name: " + + infoAndMapfile[ll].getPath()); + return false; + } + // Now in family, there are 'mapfile' and 'info' subdirs. Just + // look in the 'mapfile' subdir. + FileStatus[] familyStatus = + fs.listStatus(new Path(family, "mapfiles")); + if (familyStatus.length > 1) { + LOG.debug(family.toString() + " has " + familyStatus.length + + " files."); + return false; + } + } + } + } + return true; + } + + /** + * A {@link PathFilter} that returns only regular files. + */ + static class FileFilter implements PathFilter { + private final FileSystem fs; + + public FileFilter(final FileSystem fs) { + this.fs = fs; + } + + @Override + public boolean accept(Path p) { + try { + return fs.isFile(p); + } catch (IOException e) { + LOG.debug("unable to verify if path=" + p + " is a regular file", e); + return false; + } + } + } + + /** + * A {@link PathFilter} that returns directories. + */ + public static class DirFilter implements PathFilter { + private final FileSystem fs; + + public DirFilter(final FileSystem fs) { + this.fs = fs; + } + + @Override + public boolean accept(Path p) { + boolean isValid = false; + try { + if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(p)) { + isValid = false; + } else { + isValid = this.fs.getFileStatus(p).isDir(); + } + } catch (IOException e) { + e.printStackTrace(); + } + return isValid; + } + } + + /** + * Heuristic to determine whether is safe or not to open a file for append + * Looks both for dfs.support.append and use reflection to search + * for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush() + * @param conf + * @return True if append support + */ + public static boolean isAppendSupported(final Configuration conf) { + boolean append = conf.getBoolean("dfs.support.append", false); + if (append) { + try { + // TODO: The implementation that comes back when we do a createWriter + // may not be using SequenceFile so the below is not a definitive test. + // Will do for now (hdfs-200). + SequenceFile.Writer.class.getMethod("syncFs", new Class []{}); + append = true; + } catch (SecurityException e) { + } catch (NoSuchMethodException e) { + append = false; + } + } + if (!append) { + // Look for the 0.21, 0.22, new-style append evidence. + try { + FSDataOutputStream.class.getMethod("hflush", new Class []{}); + append = true; + } catch (NoSuchMethodException e) { + append = false; + } + } + return append; + } + + /** + * @param conf + * @return True if this filesystem whose scheme is 'hdfs'. + * @throws IOException + */ + public static boolean isHDFS(final Configuration conf) throws IOException { + FileSystem fs = FileSystem.get(conf); + String scheme = fs.getUri().getScheme(); + return scheme.equalsIgnoreCase("hdfs"); + } + + /** + * Recover file lease. Used when a file might be suspect + * to be had been left open by another process. + * @param fs FileSystem handle + * @param p Path of file to recover lease + * @param conf Configuration handle + * @throws IOException + */ + public abstract void recoverFileLease(final FileSystem fs, final Path p, + Configuration conf) throws IOException; + + /** + * @param fs + * @param rootdir + * @return All the table directories under rootdir. Ignore non table hbase folders such as + * .logs, .oldlogs, .corrupt, .META., and -ROOT- folders. + * @throws IOException + */ + public static List getTableDirs(final FileSystem fs, final Path rootdir) + throws IOException { + // presumes any directory under hbase.rootdir is a table + FileStatus [] dirs = fs.listStatus(rootdir, new DirFilter(fs)); + List tabledirs = new ArrayList(dirs.length); + for (FileStatus dir: dirs) { + Path p = dir.getPath(); + String tableName = p.getName(); + if (!HConstants.HBASE_NON_USER_TABLE_DIRS.contains(tableName)) { + tabledirs.add(p); + } + } + return tabledirs; + } + + public static Path getTablePath(Path rootdir, byte [] tableName) { + return getTablePath(rootdir, Bytes.toString(tableName)); + } + + public static Path getTablePath(Path rootdir, final String tableName) { + return new Path(rootdir, tableName); + } + + /** + * Filter for all dirs that don't start with '.' + */ + public static class RegionDirFilter implements PathFilter { + // This pattern will accept 0.90+ style hex region dirs and older numeric region dir names. + final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$"); + final FileSystem fs; + + public RegionDirFilter(FileSystem fs) { + this.fs = fs; + } + + @Override + public boolean accept(Path rd) { + if (!regionDirPattern.matcher(rd.getName()).matches()) { + return false; + } + + try { + return fs.getFileStatus(rd).isDir(); + } catch (IOException ioe) { + // Maybe the file was moved or the fs was disconnected. + LOG.warn("Skipping file " + rd +" due to IOException", ioe); + return false; + } + } + } + + /** + * Given a particular table dir, return all the regiondirs inside it, excluding files such as + * .tableinfo + * @param fs A file system for the Path + * @param tableDir Path to a specific table directory / + * @return List of paths to valid region directories in table dir. + * @throws IOException + */ + public static List getRegionDirs(final FileSystem fs, final Path tableDir) throws IOException { + // assumes we are in a table dir. + FileStatus[] rds = fs.listStatus(tableDir, new RegionDirFilter(fs)); + List regionDirs = new ArrayList(rds.length); + for (FileStatus rdfs: rds) { + Path rdPath = rdfs.getPath(); + regionDirs.add(rdPath); + } + return regionDirs; + } + + /** + * Filter for all dirs that are legal column family names. This is generally used for colfam + * dirs ///. + */ + public static class FamilyDirFilter implements PathFilter { + final FileSystem fs; + + public FamilyDirFilter(FileSystem fs) { + this.fs = fs; + } + + @Override + public boolean accept(Path rd) { + try { + // throws IAE if invalid + HColumnDescriptor.isLegalFamilyName(Bytes.toBytes(rd.getName())); + } catch (IllegalArgumentException iae) { + // path name is an invalid family name and thus is excluded. + return false; + } + + try { + return fs.getFileStatus(rd).isDir(); + } catch (IOException ioe) { + // Maybe the file was moved or the fs was disconnected. + LOG.warn("Skipping file " + rd +" due to IOException", ioe); + return false; + } + } + } + + /** + * Given a particular region dir, return all the familydirs inside it + * + * @param fs A file system for the Path + * @param regionDir Path to a specific region directory + * @return List of paths to valid family directories in region dir. + * @throws IOException + */ + public static List getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException { + // assumes we are in a region dir. + FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs)); + List familyDirs = new ArrayList(fds.length); + for (FileStatus fdfs: fds) { + Path fdPath = fdfs.getPath(); + familyDirs.add(fdPath); + } + return familyDirs; + } + + /** + * Filter for HFiles that excludes reference files. + */ + public static class HFileFilter implements PathFilter { + // This pattern will accept 0.90+ style hex hfies files but reject reference files + final public static Pattern hfilePattern = Pattern.compile("^([0-9a-f]+)$"); + + final FileSystem fs; + + public HFileFilter(FileSystem fs) { + this.fs = fs; + } + + @Override + public boolean accept(Path rd) { + if (!hfilePattern.matcher(rd.getName()).matches()) { + return false; + } + + try { + // only files + return !fs.getFileStatus(rd).isDir(); + } catch (IOException ioe) { + // Maybe the file was moved or the fs was disconnected. + LOG.warn("Skipping file " + rd +" due to IOException", ioe); + return false; + } + } + } + + /** + * @param conf + * @return Returns the filesystem of the hbase rootdir. + * @throws IOException + */ + public static FileSystem getCurrentFileSystem(Configuration conf) + throws IOException { + return getRootDir(conf).getFileSystem(conf); + } + + /** + * Runs through the HBase rootdir and creates a reverse lookup map for + * table StoreFile names to the full Path. + *
    + * Example...
    + * Key = 3944417774205889744
    + * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744 + * + * @param fs The file system to use. + * @param hbaseRootDir The root directory to scan. + * @return Map keyed by StoreFile name with a value of the full Path. + * @throws IOException When scanning the directory fails. + */ + public static Map getTableStoreFilePathMap( + final FileSystem fs, final Path hbaseRootDir) + throws IOException { + Map map = new HashMap(); + + // if this method looks similar to 'getTableFragmentation' that is because + // it was borrowed from it. + + DirFilter df = new DirFilter(fs); + // presumes any directory under hbase.rootdir is a table + FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df); + for (FileStatus tableDir : tableDirs) { + // Skip the .log and other non-table directories. All others should be tables. + // Inside a table, there are compaction.dir directories to skip. Otherwise, all else + // should be regions. + Path d = tableDir.getPath(); + if (HConstants.HBASE_NON_TABLE_DIRS.contains(d.getName())) { + continue; + } + FileStatus[] regionDirs = fs.listStatus(d, df); + for (FileStatus regionDir : regionDirs) { + Path dd = regionDir.getPath(); + if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) { + continue; + } + // else its a region name, now look in region for families + FileStatus[] familyDirs = fs.listStatus(dd, df); + for (FileStatus familyDir : familyDirs) { + Path family = familyDir.getPath(); + // now in family, iterate over the StoreFiles and + // put in map + FileStatus[] familyStatus = fs.listStatus(family); + for (FileStatus sfStatus : familyStatus) { + Path sf = sfStatus.getPath(); + map.put( sf.getName(), sf); + } + + } + } + } + return map; + } + + /** + * Calls fs.listStatus() and treats FileNotFoundException as non-fatal + * This accommodates differences between hadoop versions + * + * @param fs file system + * @param dir directory + * @param filter path filter + * @return null if tabledir doesn't exist, otherwise FileStatus array + */ + public static FileStatus [] listStatus(final FileSystem fs, + final Path dir, final PathFilter filter) throws IOException { + FileStatus [] status = null; + try { + status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter); + } catch (FileNotFoundException fnfe) { + // if directory doesn't exist, return null + LOG.debug(dir + " doesn't exist"); + } + if (status == null || status.length < 1) return null; + return status; + } + + /** + * Calls fs.listStatus() and treats FileNotFoundException as non-fatal + * This would accommodates differences between hadoop versions + * + * @param fs file system + * @param dir directory + * @return null if tabledir doesn't exist, otherwise FileStatus array + */ + public static FileStatus[] listStatus(final FileSystem fs, final Path dir) throws IOException { + return listStatus(fs, dir, null); + } + + /** + * Calls fs.delete() and returns the value returned by the fs.delete() + * + * @param fs + * @param path + * @param recursive + * @return + * @throws IOException + */ + public static boolean delete(final FileSystem fs, final Path path, final boolean recursive) + throws IOException { + return fs.delete(path, recursive); + } + + /** + * Throw an exception if an action is not permitted by a user on a file. + * + * @param user + * the user + * @param file + * the file + * @param action + * the action + */ + public static void checkAccess(User user, FileStatus file, + FsAction action) throws AccessControlException { + // See HBASE-7814. UserGroupInformation from hadoop 0.20.x may not support getShortName(). + String username = user.getShortName(); + if (username.equals(file.getOwner())) { + if (file.getPermission().getUserAction().implies(action)) { + return; + } + } else if (contains(user.getGroupNames(), file.getGroup())) { + if (file.getPermission().getGroupAction().implies(action)) { + return; + } + } else if (file.getPermission().getOtherAction().implies(action)) { + return; + } + throw new AccessControlException("Permission denied:" + " action=" + action + + " path=" + file.getPath() + " user=" + username); + } + + private static boolean contains(String[] groups, String user) { + for (String group : groups) { + if (group.equals(user)) { + return true; + } + } + return false; + } + + /** + * Calls fs.exists(). Checks if the specified path exists + * + * @param fs + * @param path + * @return + * @throws IOException + */ + public static boolean isExists(final FileSystem fs, final Path path) throws IOException { + return fs.exists(path); + } + + /** + * Log the current state of the filesystem from a certain root directory + * @param fs filesystem to investigate + * @param root root file/directory to start logging from + * @param LOG log to output information + * @throws IOException if an unexpected exception occurs + */ + public static void logFileSystemState(final FileSystem fs, final Path root, Log LOG) + throws IOException { + LOG.debug("Current file system:"); + logFSTree(LOG, fs, root, "|-"); + } + + /** + * Recursive helper to log the state of the FS + * @see #logFileSystemState(FileSystem, Path, Log) + */ + private static void logFSTree(Log LOG, final FileSystem fs, final Path root, String prefix) + throws IOException { + FileStatus[] files = FSUtils.listStatus(fs, root, null); + if (files == null) return; + + for (FileStatus file : files) { + if (file.isDir()) { + LOG.debug(prefix + file.getPath().getName() + "/"); + logFSTree(LOG, fs, file.getPath(), prefix + "---"); + } else { + LOG.debug(prefix + file.getPath().getName()); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java b/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java new file mode 100644 index 0000000..a945c03 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java @@ -0,0 +1,194 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.util.NavigableSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.FSUtils; + +/** + * Utility methods for interacting with the hbase.root file system. + */ +@InterfaceAudience.Private +public final class FSVisitor { + private static final Log LOG = LogFactory.getLog(FSVisitor.class); + + public interface StoreFileVisitor { + void storeFile(final String region, final String family, final String hfileName) + throws IOException; + } + + public interface RecoveredEditsVisitor { + void recoveredEdits (final String region, final String logfile) + throws IOException; + } + + public interface LogFileVisitor { + void logFile (final String server, final String logfile) + throws IOException; + } + + private FSVisitor() { + // private constructor for utility class + } + + /** + * Iterate over the table store files + * + * @param fs {@link FileSystem} + * @param tableDir {@link Path} to the table directory + * @param visitor callback object to get the store files + * @throws IOException if an error occurred while scanning the directory + */ + public static void visitTableStoreFiles(final FileSystem fs, final Path tableDir, + final StoreFileVisitor visitor) throws IOException { + FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs)); + if (regions == null) { + LOG.info("No regions under directory:" + tableDir); + return; + } + + for (FileStatus region: regions) { + visitRegionStoreFiles(fs, region.getPath(), visitor); + } + } + + /** + * Iterate over the region store files + * + * @param fs {@link FileSystem} + * @param regionDir {@link Path} to the region directory + * @param visitor callback object to get the store files + * @throws IOException if an error occurred while scanning the directory + */ + public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir, + final StoreFileVisitor visitor) throws IOException { + FileStatus[] families = FSUtils.listStatus(fs, regionDir, new FSUtils.FamilyDirFilter(fs)); + if (families == null) { + LOG.info("No families under region directory:" + regionDir); + return; + } + + PathFilter fileFilter = new FSUtils.FileFilter(fs); + for (FileStatus family: families) { + Path familyDir = family.getPath(); + String familyName = familyDir.getName(); + + // get all the storeFiles in the family + FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir, fileFilter); + if (storeFiles == null) { + LOG.debug("No hfiles found for family: " + familyDir + ", skipping."); + continue; + } + + for (FileStatus hfile: storeFiles) { + Path hfilePath = hfile.getPath(); + visitor.storeFile(regionDir.getName(), familyName, hfilePath.getName()); + } + } + } + + /** + * Iterate over each region in the table and inform about recovered.edits + * + * @param fs {@link FileSystem} + * @param tableDir {@link Path} to the table directory + * @param visitor callback object to get the recovered.edits files + * @throws IOException if an error occurred while scanning the directory + */ + public static void visitTableRecoveredEdits(final FileSystem fs, final Path tableDir, + final FSVisitor.RecoveredEditsVisitor visitor) throws IOException { + FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs)); + if (regions == null) { + LOG.info("No regions under directory:" + tableDir); + return; + } + + for (FileStatus region: regions) { + visitRegionRecoveredEdits(fs, region.getPath(), visitor); + } + } + + /** + * Iterate over recovered.edits of the specified region + * + * @param fs {@link FileSystem} + * @param regionDir {@link Path} to the Region directory + * @param visitor callback object to get the recovered.edits files + * @throws IOException if an error occurred while scanning the directory + */ + public static void visitRegionRecoveredEdits(final FileSystem fs, final Path regionDir, + final FSVisitor.RecoveredEditsVisitor visitor) throws IOException { + NavigableSet files = HLog.getSplitEditFilesSorted(fs, regionDir); + if (files == null || files.size() == 0) return; + + for (Path source: files) { + // check to see if the file is zero length, in which case we can skip it + FileStatus stat = fs.getFileStatus(source); + if (stat.getLen() <= 0) continue; + + visitor.recoveredEdits(regionDir.getName(), source.getName()); + } + } + + /** + * Iterate over hbase log files + * + * @param fs {@link FileSystem} + * @param rootDir {@link Path} to the HBase root folder + * @param visitor callback object to get the log files + * @throws IOException if an error occurred while scanning the directory + */ + public static void visitLogFiles(final FileSystem fs, final Path rootDir, + final LogFileVisitor visitor) throws IOException { + Path logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); + FileStatus[] logServerDirs = FSUtils.listStatus(fs, logsDir); + if (logServerDirs == null) { + LOG.info("No logs under directory:" + logsDir); + return; + } + + for (FileStatus serverLogs: logServerDirs) { + String serverName = serverLogs.getPath().getName(); + + FileStatus[] hlogs = FSUtils.listStatus(fs, serverLogs.getPath()); + if (hlogs == null) { + LOG.debug("No hfiles found for server: " + serverName + ", skipping."); + continue; + } + + for (FileStatus hlogRef: hlogs) { + visitor.logFile(serverName, hlogRef.getPath().getName()); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/FileSystemVersionException.java b/src/main/java/org/apache/hadoop/hbase/util/FileSystemVersionException.java new file mode 100644 index 0000000..5235121 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/FileSystemVersionException.java @@ -0,0 +1,39 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.IOException; + +/** Thrown when the file system needs to be upgraded */ +public class FileSystemVersionException extends IOException { + private static final long serialVersionUID = 1004053363L; + + /** default constructor */ + public FileSystemVersionException() { + super(); + } + + /** @param s message */ + public FileSystemVersionException(String s) { + super(s); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/GetJavaProperty.java b/src/main/java/org/apache/hadoop/hbase/util/GetJavaProperty.java new file mode 100644 index 0000000..361334d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/GetJavaProperty.java @@ -0,0 +1,37 @@ +/** + * Copyright 2012 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +/** + * A generic way for querying Java properties. + */ +public class GetJavaProperty { + public static void main(String args[]) { + if (args.length == 0) { + for (Object prop: System.getProperties().keySet()) { + System.out.println(prop + "=" + System.getProperty((String)prop, "")); + } + } else { + for (String prop: args) { + System.out.println(System.getProperty(prop, "")); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseConfTool.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseConfTool.java new file mode 100644 index 0000000..225f92c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseConfTool.java @@ -0,0 +1,41 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; + +/** + * Tool that prints out a configuration. + * Pass the configuration key on the command-line. + */ +public class HBaseConfTool { + public static void main(String args[]) { + if (args.length < 1) { + System.err.println("Usage: HBaseConfTool "); + System.exit(1); + return; + } + + Configuration conf = HBaseConfiguration.create(); + System.out.println(conf.get(args[0])); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java new file mode 100644 index 0000000..e7604fa --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -0,0 +1,3741 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.MetaScanner; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.RowMutations; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; +import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; +import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler; +import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl; +import org.apache.hadoop.hbase.zookeeper.RootRegionTracker; +import org.apache.hadoop.hbase.zookeeper.ZKTable; +import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.zookeeper.KeeperException; +import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.TreeMultimap; + +/** + * HBaseFsck (hbck) is a tool for checking and repairing region consistency and + * table integrity problems in a corrupted HBase. + *

    + * Region consistency checks verify that .META., region deployment on region + * servers and the state of data in HDFS (.regioninfo files) all are in + * accordance. + *

    + * Table integrity checks verify that all possible row keys resolve to exactly + * one region of a table. This means there are no individual degenerate + * or backwards regions; no holes between regions; and that there are no + * overlapping regions. + *

    + * The general repair strategy works in two phases: + *

      + *
    1. Repair Table Integrity on HDFS. (merge or fabricate regions) + *
    2. Repair Region Consistency with .META. and assignments + *
    + *

    + * For table integrity repairs, the tables' region directories are scanned + * for .regioninfo files. Each table's integrity is then verified. If there + * are any orphan regions (regions with no .regioninfo files) or holes, new + * regions are fabricated. Backwards regions are sidelined as well as empty + * degenerate (endkey==startkey) regions. If there are any overlapping regions, + * a new region is created and all data is merged into the new region. + *

    + * Table integrity repairs deal solely with HDFS and could potentially be done + * offline -- the hbase region servers or master do not need to be running. + * This phase can eventually be used to completely reconstruct the META table in + * an offline fashion. + *

    + * Region consistency requires three conditions -- 1) valid .regioninfo file + * present in an HDFS region dir, 2) valid row with .regioninfo data in META, + * and 3) a region is deployed only at the regionserver that was assigned to + * with proper state in the master. + *

    + * Region consistency repairs require hbase to be online so that hbck can + * contact the HBase master and region servers. The hbck#connect() method must + * first be called successfully. Much of the region consistency information + * is transient and less risky to repair. + *

    + * If hbck is run from the command line, there are a handful of arguments that + * can be used to limit the kinds of repairs hbck will do. See the code in + * {@link #printUsageAndExit()} for more details. + */ +public class HBaseFsck extends Configured implements Tool { + public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute + public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000; + private static final int MAX_NUM_THREADS = 50; // #threads to contact regions + private static boolean rsSupportsOffline = true; + private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2; + private static final int DEFAULT_MAX_MERGE = 5; + private static final String TO_BE_LOADED = "to_be_loaded"; + + /********************** + * Internal resources + **********************/ + private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName()); + private ClusterStatus status; + private HConnection connection; + private HBaseAdmin admin; + private HTable meta; + protected ExecutorService executor; // threads to retrieve data from regionservers + private long startMillis = System.currentTimeMillis(); + private HFileCorruptionChecker hfcc; + private int retcode = 0; + + /*********** + * Options + ***********/ + private static boolean details = false; // do we display the full report + private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older + private boolean fixAssignments = false; // fix assignment errors? + private boolean fixMeta = false; // fix meta errors? + private boolean checkHdfs = true; // load and check fs consistency? + private boolean fixHdfsHoles = false; // fix fs holes? + private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky) + private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo) + private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo) + private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs + private boolean fixSplitParents = false; // fix lingering split parents + private boolean fixReferenceFiles = false; // fix lingering reference store file + + // limit checking/fixes to listed tables, if empty attempt to check/fix all + // -ROOT- and .META. are always checked + private Set tablesIncluded = new HashSet(); + private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge + private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline + private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions + private Path sidelineDir = null; + + private boolean rerun = false; // if we tried to fix something, rerun hbck + private static boolean summary = false; // if we want to print less output + private boolean checkMetaOnly = false; + private boolean ignorePreCheckPermission = false; // if pre-check permission + + /********* + * State + *********/ + final private ErrorReporter errors; + int fixes = 0; + + /** + * This map contains the state of all hbck items. It maps from encoded region + * name to HbckInfo structure. The information contained in HbckInfo is used + * to detect and correct consistency (hdfs/meta/deployment) problems. + */ + private TreeMap regionInfoMap = new TreeMap(); + private TreeSet disabledTables = + new TreeSet(Bytes.BYTES_COMPARATOR); + // Empty regioninfo qualifiers in .META. + private Set emptyRegionInfoQualifiers = new HashSet(); + + /** + * This map from Tablename -> TableInfo contains the structures necessary to + * detect table consistency problems (holes, dupes, overlaps). It is sorted + * to prevent dupes. + * + * If tablesIncluded is empty, this map contains all tables. + * Otherwise, it contains only meta tables and tables in tablesIncluded, + * unless checkMetaOnly is specified, in which case, it contains only + * the meta tables (.META. and -ROOT-). + */ + private SortedMap tablesInfo = new ConcurrentSkipListMap(); + + /** + * When initially looking at HDFS, we attempt to find any orphaned data. + */ + private List orphanHdfsDirs = Collections.synchronizedList(new ArrayList()); + + private Map> orphanTableDirs = new HashMap>(); + + /** + * Constructor + * + * @param conf Configuration object + * @throws MasterNotRunningException if the master is not running + * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper + */ + public HBaseFsck(Configuration conf) throws MasterNotRunningException, + ZooKeeperConnectionException, IOException, ClassNotFoundException { + super(conf); + errors = getErrorReporter(conf); + + int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS); + executor = new ScheduledThreadPoolExecutor(numThreads); + } + + /** + * Constructor + * + * @param conf + * Configuration object + * @throws MasterNotRunningException + * if the master is not running + * @throws ZooKeeperConnectionException + * if unable to connect to ZooKeeper + */ + public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException, + ZooKeeperConnectionException, IOException, ClassNotFoundException { + super(conf); + errors = getErrorReporter(getConf()); + this.executor = exec; + } + + /** + * To repair region consistency, one must call connect() in order to repair + * online state. + */ + public void connect() throws IOException { + admin = new HBaseAdmin(getConf()); + meta = new HTable(getConf(), HConstants.META_TABLE_NAME); + status = admin.getMaster().getClusterStatus(); + connection = admin.getConnection(); + } + + /** + * Get deployed regions according to the region servers. + */ + private void loadDeployedRegions() throws IOException, InterruptedException { + // From the master, get a list of all known live region servers + Collection regionServers = status.getServers(); + errors.print("Number of live region servers: " + regionServers.size()); + if (details) { + for (ServerName rsinfo: regionServers) { + errors.print(" " + rsinfo.getServerName()); + } + } + + // From the master, get a list of all dead region servers + Collection deadRegionServers = status.getDeadServerNames(); + errors.print("Number of dead region servers: " + deadRegionServers.size()); + if (details) { + for (ServerName name: deadRegionServers) { + errors.print(" " + name); + } + } + + // Print the current master name and state + errors.print("Master: " + status.getMaster()); + + // Print the list of all backup masters + Collection backupMasters = status.getBackupMasters(); + errors.print("Number of backup masters: " + backupMasters.size()); + if (details) { + for (ServerName name: backupMasters) { + errors.print(" " + name); + } + } + + // Determine what's deployed + processRegionServers(regionServers); + } + + /** + * Clear the current state of hbck. + */ + private void clearState() { + // Make sure regionInfo is empty before starting + fixes = 0; + regionInfoMap.clear(); + emptyRegionInfoQualifiers.clear(); + disabledTables.clear(); + errors.clear(); + tablesInfo.clear(); + orphanHdfsDirs.clear(); + } + + /** + * This repair method analyzes hbase data in hdfs and repairs it to satisfy + * the table integrity rules. HBase doesn't need to be online for this + * operation to work. + */ + public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException { + // Initial pass to fix orphans. + if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles() + || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) { + LOG.info("Loading regioninfos HDFS"); + // if nothing is happening this should always complete in two iterations. + int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3); + int curIter = 0; + do { + clearState(); // clears hbck state and reset fixes to 0 and. + // repair what's on HDFS + restoreHdfsIntegrity(); + curIter++;// limit the number of iterations. + } while (fixes > 0 && curIter <= maxIterations); + + // Repairs should be done in the first iteration and verification in the second. + // If there are more than 2 passes, something funny has happened. + if (curIter > 2) { + if (curIter == maxIterations) { + LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. " + + "Tables integrity may not be fully repaired!"); + } else { + LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations"); + } + } + } + } + + /** + * This repair method requires the cluster to be online since it contacts + * region servers and the masters. It makes each region's state in HDFS, in + * .META., and deployments consistent. + * + * @return If > 0 , number of errors detected, if < 0 there was an unrecoverable + * error. If 0, we have a clean hbase. + */ + public int onlineConsistencyRepair() throws IOException, KeeperException, + InterruptedException { + clearState(); + + LOG.info("Loading regionsinfo from the .META. table"); + boolean success = loadMetaEntries(); + if (!success) return -1; + + // Check if .META. is found only once and in the right place + if (!checkMetaRegion()) { + // Will remove later if we can fix it + errors.reportError("Encountered fatal error. Exiting..."); + return -2; + } + + // get a list of all tables that have not changed recently. + if (!checkMetaOnly) { + reportTablesInFlux(); + } + + // get regions according to what is online on each RegionServer + loadDeployedRegions(); + + // load regiondirs and regioninfos from HDFS + if (shouldCheckHdfs()) { + loadHdfsRegionDirs(); + loadHdfsRegionInfos(); + } + + // Empty cells in .META.? + reportEmptyMetaCells(); + + // Get disabled tables from ZooKeeper + loadDisabledTables(); + + // fix the orphan tables + fixOrphanTables(); + + // Check and fix consistency + checkAndFixConsistency(); + + // Check integrity (does not fix) + checkIntegrity(); + return errors.getErrorList().size(); + } + + /** + * Contacts the master and prints out cluster-wide information + * @return 0 on success, non-zero on failure + */ + public int onlineHbck() throws IOException, KeeperException, InterruptedException { + // print hbase server version + errors.print("Version: " + status.getHBaseVersion()); + offlineHdfsIntegrityRepair(); + + // turn the balancer off + boolean oldBalancer = admin.setBalancerRunning(false, true); + try { + onlineConsistencyRepair(); + } + finally { + admin.setBalancerRunning(oldBalancer, false); + } + + offlineReferenceFileRepair(); + + // Print table summary + printTableSummary(tablesInfo); + return errors.summarize(); + } + + /** + * Iterates through the list of all orphan/invalid regiondirs. + */ + private void adoptHdfsOrphans(Collection orphanHdfsDirs) throws IOException { + for (HbckInfo hi : orphanHdfsDirs) { + LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir()); + adoptHdfsOrphan(hi); + } + } + + /** + * Orphaned regions are regions without a .regioninfo file in them. We "adopt" + * these orphans by creating a new region, and moving the column families, + * recovered edits, HLogs, into the new region dir. We determine the region + * startkey and endkeys by looking at all of the hfiles inside the column + * families to identify the min and max keys. The resulting region will + * likely violate table integrity but will be dealt with by merging + * overlapping regions. + */ + private void adoptHdfsOrphan(HbckInfo hi) throws IOException { + Path p = hi.getHdfsRegionDir(); + FileSystem fs = p.getFileSystem(getConf()); + FileStatus[] dirs = fs.listStatus(p); + if (dirs == null) { + LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " + + p + ". This dir could probably be deleted."); + return ; + } + + String tableName = Bytes.toString(hi.getTableName()); + TableInfo tableInfo = tablesInfo.get(tableName); + Preconditions.checkNotNull("Table " + tableName + "' not present!", tableInfo); + HTableDescriptor template = tableInfo.getHTD(); + + // find min and max key values + Pair orphanRegionRange = null; + for (FileStatus cf : dirs) { + String cfName= cf.getPath().getName(); + // TODO Figure out what the special dirs are + if (cfName.startsWith(".") || cfName.equals("splitlog")) continue; + + FileStatus[] hfiles = fs.listStatus(cf.getPath()); + for (FileStatus hfile : hfiles) { + byte[] start, end; + HFile.Reader hf = null; + try { + CacheConfig cacheConf = new CacheConfig(getConf()); + hf = HFile.createReader(fs, hfile.getPath(), cacheConf); + hf.loadFileInfo(); + KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey()); + start = startKv.getRow(); + KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey()); + end = endKv.getRow(); + } catch (IOException ioe) { + LOG.warn("Problem reading orphan file " + hfile + ", skipping"); + continue; + } catch (NullPointerException ioe) { + LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping"); + continue; + } finally { + if (hf != null) { + hf.close(); + } + } + + // expand the range to include the range of all hfiles + if (orphanRegionRange == null) { + // first range + orphanRegionRange = new Pair(start, end); + } else { + // TODO add test + + // expand range only if the hfile is wider. + if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) { + orphanRegionRange.setFirst(start); + } + if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) { + orphanRegionRange.setSecond(end); + } + } + } + } + if (orphanRegionRange == null) { + LOG.warn("No data in dir " + p + ", sidelining data"); + fixes++; + sidelineRegionDir(fs, hi); + return; + } + LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " + + Bytes.toString(orphanRegionRange.getSecond()) + ")"); + + // create new region on hdfs. move data into place. + HRegionInfo hri = new HRegionInfo(template.getName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond()); + LOG.info("Creating new region : " + hri); + HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template); + Path target = region.getRegionDir(); + + // rename all the data to new region + mergeRegionDirs(target, hi); + fixes++; + } + + /** + * This method determines if there are table integrity errors in HDFS. If + * there are errors and the appropriate "fix" options are enabled, the method + * will first correct orphan regions making them into legit regiondirs, and + * then reload to merge potentially overlapping regions. + * + * @return number of table integrity errors found + */ + private int restoreHdfsIntegrity() throws IOException, InterruptedException { + // Determine what's on HDFS + LOG.info("Loading HBase regioninfo from HDFS..."); + loadHdfsRegionDirs(); // populating regioninfo table. + + int errs = errors.getErrorList().size(); + // First time just get suggestions. + tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs. + checkHdfsIntegrity(false, false); + + if (errors.getErrorList().size() == errs) { + LOG.info("No integrity errors. We are done with this phase. Glorious."); + return 0; + } + + if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) { + adoptHdfsOrphans(orphanHdfsDirs); + // TODO optimize by incrementally adding instead of reloading. + } + + // Make sure there are no holes now. + if (shouldFixHdfsHoles()) { + clearState(); // this also resets # fixes. + loadHdfsRegionDirs(); + tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs. + tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false); + } + + // Now we fix overlaps + if (shouldFixHdfsOverlaps()) { + // second pass we fix overlaps. + clearState(); // this also resets # fixes. + loadHdfsRegionDirs(); + tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs. + tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps()); + } + + return errors.getErrorList().size(); + } + + /** + * Scan all the store file names to find any lingering reference files, + * which refer to some none-exiting files. If "fix" option is enabled, + * any lingering reference file will be sidelined if found. + *

    + * Lingering reference file prevents a region from opening. It has to + * be fixed before a cluster can start properly. + */ + private void offlineReferenceFileRepair() throws IOException { + Configuration conf = getConf(); + Path hbaseRoot = FSUtils.getRootDir(conf); + FileSystem fs = hbaseRoot.getFileSystem(conf); + Map allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot); + for (Path path: allFiles.values()) { + boolean isReference = false; + try { + isReference = StoreFile.isReference(path); + } catch (Throwable t) { + // Ignore. Some files may not be store files at all. + // For example, files under .oldlogs folder in .META. + // Warning message is already logged by + // StoreFile#isReference. + } + if (!isReference) continue; + + Path referredToFile = StoreFile.getReferredToFile(path); + if (fs.exists(referredToFile)) continue; // good, expected + + // Found a lingering reference file + errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE, + "Found lingering reference file " + path); + if (!shouldFixReferenceFiles()) continue; + + // Now, trying to fix it since requested + boolean success = false; + String pathStr = path.toString(); + + // A reference file path should be like + // ${hbase.rootdir}/table_name/region_id/family_name/referred_file.region_name + // Up 3 directories to get the table folder. + // So the file will be sidelined to a similar folder structure. + int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR); + for (int i = 0; index > 0 && i < 3; i++) { + index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index); + } + if (index > 0) { + Path rootDir = getSidelineDir(); + Path dst = new Path(rootDir, pathStr.substring(index)); + fs.mkdirs(dst.getParent()); + LOG.info("Trying to sildeline reference file" + + path + " to " + dst); + setShouldRerun(); + + success = fs.rename(path, dst); + } + if (!success) { + LOG.error("Failed to sideline reference file " + path); + } + } + } + + /** + * TODO -- need to add tests for this. + */ + private void reportEmptyMetaCells() { + errors.print("Number of empty REGIONINFO_QUALIFIER rows in .META.: " + + emptyRegionInfoQualifiers.size()); + if (details) { + for (Result r: emptyRegionInfoQualifiers) { + errors.print(" " + r); + } + } + } + + /** + * TODO -- need to add tests for this. + */ + private void reportTablesInFlux() { + AtomicInteger numSkipped = new AtomicInteger(0); + HTableDescriptor[] allTables = getTables(numSkipped); + errors.print("Number of Tables: " + allTables.length); + if (details) { + if (numSkipped.get() > 0) { + errors.detail("Number of Tables in flux: " + numSkipped.get()); + } + for (HTableDescriptor td : allTables) { + String tableName = td.getNameAsString(); + errors.detail(" Table: " + tableName + "\t" + + (td.isReadOnly() ? "ro" : "rw") + "\t" + + (td.isRootRegion() ? "ROOT" : + (td.isMetaRegion() ? "META" : " ")) + "\t" + + " families: " + td.getFamilies().size()); + } + } + } + + public ErrorReporter getErrors() { + return errors; + } + + /** + * Read the .regioninfo file from the file system. If there is no + * .regioninfo, add it to the orphan hdfs region list. + */ + private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException { + Path regionDir = hbi.getHdfsRegionDir(); + if (regionDir == null) { + LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry); + return; + } + + if (hbi.hdfsEntry.hri != null) { + // already loaded data + return; + } + + Path regioninfo = new Path(regionDir, HRegion.REGIONINFO_FILE); + FileSystem fs = regioninfo.getFileSystem(getConf()); + + FSDataInputStream in = fs.open(regioninfo); + HRegionInfo hri = new HRegionInfo(); + hri.readFields(in); + in.close(); + LOG.debug("HRegionInfo read: " + hri.toString()); + hbi.hdfsEntry.hri = hri; + } + + /** + * Exception thrown when a integrity repair operation fails in an + * unresolvable way. + */ + public static class RegionRepairException extends IOException { + private static final long serialVersionUID = 1L; + final IOException ioe; + public RegionRepairException(String s, IOException ioe) { + super(s); + this.ioe = ioe; + } + } + + /** + * Populate hbi's from regionInfos loaded from file system. + */ + private SortedMap loadHdfsRegionInfos() throws IOException, InterruptedException { + tablesInfo.clear(); // regenerating the data + // generate region split structure + Collection hbckInfos = regionInfoMap.values(); + + // Parallelized read of .regioninfo files. + List hbis = new ArrayList(hbckInfos.size()); + List> hbiFutures; + + for (HbckInfo hbi : hbckInfos) { + WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors); + hbis.add(work); + } + + // Submit and wait for completion + hbiFutures = executor.invokeAll(hbis); + + for(int i=0; i f = hbiFutures.get(i); + try { + f.get(); + } catch(ExecutionException e) { + LOG.warn("Failed to read .regioninfo file for region " + + work.hbi.getRegionNameAsString(), e.getCause()); + } + } + + // serialized table info gathering. + for (HbckInfo hbi: hbckInfos) { + + if (hbi.getHdfsHRI() == null) { + // was an orphan + continue; + } + + + // get table name from hdfs, populate various HBaseFsck tables. + String tableName = Bytes.toString(hbi.getTableName()); + if (tableName == null) { + // There was an entry in META not in the HDFS? + LOG.warn("tableName was null for: " + hbi); + continue; + } + + TableInfo modTInfo = tablesInfo.get(tableName); + if (modTInfo == null) { + // only executed once per table. + modTInfo = new TableInfo(tableName); + Path hbaseRoot = FSUtils.getRootDir(getConf()); + tablesInfo.put(tableName, modTInfo); + try { + HTableDescriptor htd = + FSTableDescriptors.getTableDescriptor(hbaseRoot.getFileSystem(getConf()), + hbaseRoot, tableName); + modTInfo.htds.add(htd); + } catch (IOException ioe) { + if (!orphanTableDirs.containsKey(tableName)) { + LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe); + //should only report once for each table + errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE, + "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName); + Set columns = new HashSet(); + orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi)); + } + } + } + if (!hbi.isSkipChecks()) { + modTInfo.addRegionInfo(hbi); + } + } + + return tablesInfo; + } + + /** + * To get the column family list according to the column family dirs + * @param columns + * @param hbi + * @return + * @throws IOException + */ + private Set getColumnFamilyList(Set columns, HbckInfo hbi) throws IOException { + Path regionDir = hbi.getHdfsRegionDir(); + FileSystem fs = regionDir.getFileSystem(getConf()); + FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs)); + for (FileStatus subdir : subDirs) { + String columnfamily = subdir.getPath().getName(); + columns.add(columnfamily); + } + return columns; + } + + /** + * To fabricate a .tableinfo file with following contents
    + * 1. the correct tablename
    + * 2. the correct colfamily list
    + * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}
    + * @param tableName + * @throws IOException + */ + private boolean fabricateTableInfo(String tableName, Set columns) throws IOException { + if (columns ==null || columns.isEmpty()) return false; + HTableDescriptor htd = new HTableDescriptor(tableName); + for (String columnfamimly : columns) { + htd.addFamily(new HColumnDescriptor(columnfamimly)); + } + FSTableDescriptors.createTableDescriptor(htd, getConf(), true); + return true; + } + + /** + * To fix orphan table by creating a .tableinfo file under tableDir
    + * 1. if TableInfo is cached, to recover the .tableinfo accordingly
    + * 2. else create a default .tableinfo file with following items
    + *  2.1 the correct tablename
    + *  2.2 the correct colfamily list
    + *  2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}
    + * @throws IOException + */ + public void fixOrphanTables() throws IOException { + if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) { + + Path hbaseRoot = FSUtils.getRootDir(getConf()); + List tmpList = new ArrayList(); + tmpList.addAll(orphanTableDirs.keySet()); + HTableDescriptor[] htds = getHTableDescriptors(tmpList); + Iterator>> iter = orphanTableDirs.entrySet().iterator(); + int j = 0; + int numFailedCase = 0; + while (iter.hasNext()) { + Entry> entry = (Entry>) iter.next(); + String tableName = entry.getKey(); + LOG.info("Trying to fix orphan table error: " + tableName); + if (j < htds.length) { + if (tableName.equals(Bytes.toString(htds[j].getName()))) { + HTableDescriptor htd = htds[j]; + LOG.info("fixing orphan table: " + tableName + " from cache"); + FSTableDescriptors.createTableDescriptor( + hbaseRoot.getFileSystem(getConf()), hbaseRoot, htd, true); + j++; + iter.remove(); + } + } else { + if (fabricateTableInfo(tableName, entry.getValue())) { + LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file"); + LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName); + iter.remove(); + } else { + LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information"); + numFailedCase++; + } + } + fixes++; + } + + if (orphanTableDirs.isEmpty()) { + // all orphanTableDirs are luckily recovered + // re-run doFsck after recovering the .tableinfo file + setShouldRerun(); + LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed"); + } else if (numFailedCase > 0) { + LOG.error("Failed to fix " + numFailedCase + + " OrphanTables with default .tableinfo files"); + } + + } + //cleanup the list + orphanTableDirs.clear(); + + } + + /** + * This borrows code from MasterFileSystem.bootstrap() + * + * @return an open .META. HRegion + */ + private HRegion createNewRootAndMeta() throws IOException { + Path rootdir = new Path(getConf().get(HConstants.HBASE_DIR)); + Configuration c = getConf(); + HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO); + MasterFileSystem.setInfoFamilyCachingForRoot(false); + HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO); + MasterFileSystem.setInfoFamilyCachingForMeta(false); + HRegion root = HRegion.createHRegion(rootHRI, rootdir, c, + HTableDescriptor.ROOT_TABLEDESC); + HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, + HTableDescriptor.META_TABLEDESC); + MasterFileSystem.setInfoFamilyCachingForRoot(true); + MasterFileSystem.setInfoFamilyCachingForMeta(true); + + // Add first region from the META table to the ROOT region. + HRegion.addRegionToMETA(root, meta); + root.close(); + root.getLog().closeAndDelete(); + return meta; + } + + /** + * Generate set of puts to add to new meta. This expects the tables to be + * clean with no overlaps or holes. If there are any problems it returns null. + * + * @return An array list of puts to do in bulk, null if tables have problems + */ + private ArrayList generatePuts(SortedMap tablesInfo) throws IOException { + ArrayList puts = new ArrayList(); + boolean hasProblems = false; + for (Entry e : tablesInfo.entrySet()) { + String name = e.getKey(); + + // skip "-ROOT-" and ".META." + if (Bytes.compareTo(Bytes.toBytes(name), HConstants.ROOT_TABLE_NAME) == 0 + || Bytes.compareTo(Bytes.toBytes(name), HConstants.META_TABLE_NAME) == 0) { + continue; + } + + TableInfo ti = e.getValue(); + for (Entry> spl : ti.sc.getStarts().asMap() + .entrySet()) { + Collection his = spl.getValue(); + int sz = his.size(); + if (sz != 1) { + // problem + LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey()) + + " had " + sz + " regions instead of exactly 1." ); + hasProblems = true; + continue; + } + + // add the row directly to meta. + HbckInfo hi = his.iterator().next(); + HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry; + Put p = new Put(hri.getRegionName()); + p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + puts.add(p); + } + } + return hasProblems ? null : puts; + } + + /** + * Suggest fixes for each table + */ + private void suggestFixes(SortedMap tablesInfo) throws IOException { + for (TableInfo tInfo : tablesInfo.values()) { + TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors); + tInfo.checkRegionChain(handler); + } + } + + /** + * Rebuilds meta from information in hdfs/fs. Depends on configuration + * settings passed into hbck constructor to point to a particular fs/dir. + * + * @param fix flag that determines if method should attempt to fix holes + * @return true if successful, false if attempt failed. + */ + public boolean rebuildMeta(boolean fix) throws IOException, + InterruptedException { + + // TODO check to make sure hbase is offline. (or at least the table + // currently being worked on is off line) + + // Determine what's on HDFS + LOG.info("Loading HBase regioninfo from HDFS..."); + loadHdfsRegionDirs(); // populating regioninfo table. + + int errs = errors.getErrorList().size(); + tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs. + checkHdfsIntegrity(false, false); + + // make sure ok. + if (errors.getErrorList().size() != errs) { + // While in error state, iterate until no more fixes possible + while(true) { + fixes = 0; + suggestFixes(tablesInfo); + errors.clear(); + loadHdfsRegionInfos(); // update tableInfos based on region info in fs. + checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps()); + + int errCount = errors.getErrorList().size(); + + if (fixes == 0) { + if (errCount > 0) { + return false; // failed to fix problems. + } else { + break; // no fixes and no problems? drop out and fix stuff! + } + } + } + } + + // we can rebuild, move old root and meta out of the way and start + LOG.info("HDFS regioninfo's seems good. Sidelining old .META."); + Path backupDir = sidelineOldRootAndMeta(); + + LOG.info("Creating new .META."); + HRegion meta = createNewRootAndMeta(); + + // populate meta + List puts = generatePuts(tablesInfo); + if (puts == null) { + LOG.fatal("Problem encountered when creating new .META. entries. " + + "You may need to restore the previously sidelined -ROOT- and .META."); + return false; + } + meta.put(puts.toArray(new Put[0])); + meta.close(); + meta.getLog().closeAndDelete(); + LOG.info("Success! .META. table rebuilt."); + LOG.info("Old -ROOT- and .META. are moved into " + backupDir); + return true; + } + + private SortedMap checkHdfsIntegrity(boolean fixHoles, + boolean fixOverlaps) throws IOException { + LOG.info("Checking HBase region split map from HDFS data..."); + for (TableInfo tInfo : tablesInfo.values()) { + TableIntegrityErrorHandler handler; + if (fixHoles || fixOverlaps) { + handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(), + fixHoles, fixOverlaps); + } else { + handler = tInfo.new IntegrityFixSuggester(tInfo, errors); + } + if (!tInfo.checkRegionChain(handler)) { + // should dump info as well. + errors.report("Found inconsistency in table " + tInfo.getName()); + } + } + return tablesInfo; + } + + private Path getSidelineDir() throws IOException { + if (sidelineDir == null) { + Path hbaseDir = FSUtils.getRootDir(getConf()); + Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME); + sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-" + + startMillis); + } + return sidelineDir; + } + + /** + * Sideline a region dir (instead of deleting it) + */ + Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException { + return sidelineRegionDir(fs, null, hi); + } + + /** + * Sideline a region dir (instead of deleting it) + * + * @param parentDir if specified, the region will be sidelined to + * folder like .../parentDir//. The purpose + * is to group together similar regions sidelined, for example, those + * regions should be bulk loaded back later on. If null, it is ignored. + */ + Path sidelineRegionDir(FileSystem fs, + String parentDir, HbckInfo hi) throws IOException { + String tableName = Bytes.toString(hi.getTableName()); + Path regionDir = hi.getHdfsRegionDir(); + + if (!fs.exists(regionDir)) { + LOG.warn("No previous " + regionDir + " exists. Continuing."); + return null; + } + + Path rootDir = getSidelineDir(); + if (parentDir != null) { + rootDir = new Path(rootDir, parentDir); + } + Path sidelineTableDir= new Path(rootDir, tableName); + Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName()); + fs.mkdirs(sidelineRegionDir); + boolean success = false; + FileStatus[] cfs = fs.listStatus(regionDir); + if (cfs == null) { + LOG.info("Region dir is empty: " + regionDir); + } else { + for (FileStatus cf : cfs) { + Path src = cf.getPath(); + Path dst = new Path(sidelineRegionDir, src.getName()); + if (fs.isFile(src)) { + // simple file + success = fs.rename(src, dst); + if (!success) { + String msg = "Unable to rename file " + src + " to " + dst; + LOG.error(msg); + throw new IOException(msg); + } + continue; + } + + // is a directory. + fs.mkdirs(dst); + + LOG.info("Sidelining files from " + src + " into containing region " + dst); + // FileSystem.rename is inconsistent with directories -- if the + // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir, + // it moves the src into the dst dir resulting in (foo/a/b). If + // the dst does not exist, and the src a dir, src becomes dst. (foo/b) + FileStatus[] hfiles = fs.listStatus(src); + if (hfiles != null && hfiles.length > 0) { + for (FileStatus hfile : hfiles) { + success = fs.rename(hfile.getPath(), dst); + if (!success) { + String msg = "Unable to rename file " + src + " to " + dst; + LOG.error(msg); + throw new IOException(msg); + } + } + } + LOG.debug("Sideline directory contents:"); + debugLsr(sidelineRegionDir); + } + } + + LOG.info("Removing old region dir: " + regionDir); + success = fs.delete(regionDir, true); + if (!success) { + String msg = "Unable to delete dir " + regionDir; + LOG.error(msg); + throw new IOException(msg); + } + return sidelineRegionDir; + } + + /** + * Side line an entire table. + */ + void sidelineTable(FileSystem fs, byte[] table, Path hbaseDir, + Path backupHbaseDir) throws IOException { + String tableName = Bytes.toString(table); + Path tableDir = new Path(hbaseDir, tableName); + if (fs.exists(tableDir)) { + Path backupTableDir= new Path(backupHbaseDir, tableName); + boolean success = fs.rename(tableDir, backupTableDir); + if (!success) { + throw new IOException("Failed to move " + tableName + " from " + + tableDir.getName() + " to " + backupTableDir.getName()); + } + } else { + LOG.info("No previous " + tableName + " exists. Continuing."); + } + } + + /** + * @return Path to backup of original directory + */ + Path sidelineOldRootAndMeta() throws IOException { + // put current -ROOT- and .META. aside. + Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR)); + FileSystem fs = hbaseDir.getFileSystem(getConf()); + Path backupDir = getSidelineDir(); + fs.mkdirs(backupDir); + + sidelineTable(fs, HConstants.ROOT_TABLE_NAME, hbaseDir, backupDir); + try { + sidelineTable(fs, HConstants.META_TABLE_NAME, hbaseDir, backupDir); + } catch (IOException e) { + LOG.error("Attempt to sideline meta failed, attempt to revert...", e); + try { + // move it back. + sidelineTable(fs, HConstants.ROOT_TABLE_NAME, backupDir, hbaseDir); + LOG.warn("... revert succeed. -ROOT- and .META. still in " + + "original state."); + } catch (IOException ioe) { + LOG.fatal("... failed to sideline root and meta and failed to restore " + + "prevoius state. Currently in inconsistent state. To restore " + + "try to rename -ROOT- in " + backupDir.getName() + " to " + + hbaseDir.getName() + ".", ioe); + } + throw e; // throw original exception + } + return backupDir; + } + + /** + * Load the list of disabled tables in ZK into local set. + * @throws ZooKeeperConnectionException + * @throws IOException + */ + private void loadDisabledTables() + throws ZooKeeperConnectionException, IOException { + HConnectionManager.execute(new HConnectable(getConf()) { + @Override + public Void connect(HConnection connection) throws IOException { + ZooKeeperWatcher zkw = connection.getZooKeeperWatcher(); + try { + for (Entry> e : ZKTableReadOnly.getDisabledOrDisablingTables(zkw) + .entrySet()) { + for (String tableName : e.getValue()) { + disabledTables.add(Bytes.toBytes(tableName)); + } + } + } catch (KeeperException ke) { + throw new IOException(ke); + } + return null; + } + }); + } + + /** + * Check if the specified region's table is disabled. + */ + private boolean isTableDisabled(HRegionInfo regionInfo) { + return disabledTables.contains(regionInfo.getTableName()); + } + + /** + * Scan HDFS for all regions, recording their information into + * regionInfoMap + */ + public void loadHdfsRegionDirs() throws IOException, InterruptedException { + Path rootDir = new Path(getConf().get(HConstants.HBASE_DIR)); + FileSystem fs = rootDir.getFileSystem(getConf()); + + // list all tables from HDFS + List tableDirs = Lists.newArrayList(); + + boolean foundVersionFile = false; + FileStatus[] files = fs.listStatus(rootDir); + for (FileStatus file : files) { + String dirName = file.getPath().getName(); + if (dirName.equals(HConstants.VERSION_FILE_NAME)) { + foundVersionFile = true; + } else { + if ((!checkMetaOnly && isTableIncluded(dirName)) || + dirName.equals("-ROOT-") || + dirName.equals(".META.")) { + tableDirs.add(file); + } + } + } + + // verify that version file exists + if (!foundVersionFile) { + errors.reportError(ERROR_CODE.NO_VERSION_FILE, + "Version file does not exist in root dir " + rootDir); + if (shouldFixVersionFile()) { + LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME + + " file."); + setShouldRerun(); + FSUtils.setVersion(fs, rootDir, getConf().getInt( + HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt( + HConstants.VERSION_FILE_WRITE_ATTEMPTS, + HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS)); + } + } + + // level 1: /* + List dirs = new ArrayList(tableDirs.size()); + List> dirsFutures; + + for (FileStatus tableDir : tableDirs) { + LOG.debug("Loading region dirs from " +tableDir.getPath()); + dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir)); + } + + // Invoke and wait for Callables to complete + dirsFutures = executor.invokeAll(dirs); + + for(Future f: dirsFutures) { + try { + f.get(); + } catch(ExecutionException e) { + LOG.warn("Could not load region dir " , e.getCause()); + } + } + } + + /** + * Record the location of the ROOT region as found in ZooKeeper, + * as if it were in a META table. This is so that we can check + * deployment of ROOT. + */ + private boolean recordRootRegion() throws IOException { + HRegionLocation rootLocation = connection.locateRegion( + HConstants.ROOT_TABLE_NAME, HConstants.EMPTY_START_ROW); + + // Check if Root region is valid and existing + if (rootLocation == null || rootLocation.getRegionInfo() == null || + rootLocation.getHostname() == null) { + errors.reportError(ERROR_CODE.NULL_ROOT_REGION, + "Root Region or some of its attributes are null."); + return false; + } + ServerName sn; + try { + sn = getRootRegionServerName(); + } catch (InterruptedException e) { + throw new IOException("Interrupted", e); + } + MetaEntry m = + new MetaEntry(rootLocation.getRegionInfo(), sn, System.currentTimeMillis()); + HbckInfo hbInfo = new HbckInfo(m); + regionInfoMap.put(rootLocation.getRegionInfo().getEncodedName(), hbInfo); + return true; + } + + private ServerName getRootRegionServerName() + throws IOException, InterruptedException { + RootRegionTracker rootRegionTracker = + new RootRegionTracker(this.connection.getZooKeeperWatcher(), new Abortable() { + @Override + public void abort(String why, Throwable e) { + LOG.error(why, e); + System.exit(1); + } + @Override + public boolean isAborted(){ + return false; + } + + }); + rootRegionTracker.start(); + ServerName sn = null; + try { + sn = rootRegionTracker.getRootRegionLocation(); + } finally { + rootRegionTracker.stop(); + } + return sn; + } + + /** + * Contacts each regionserver and fetches metadata about regions. + * @param regionServerList - the list of region servers to connect to + * @throws IOException if a remote or network exception occurs + */ + void processRegionServers(Collection regionServerList) + throws IOException, InterruptedException { + + List workItems = new ArrayList(regionServerList.size()); + List> workFutures; + + // loop to contact each region server in parallel + for (ServerName rsinfo: regionServerList) { + workItems.add(new WorkItemRegion(this, rsinfo, errors, connection)); + } + + workFutures = executor.invokeAll(workItems); + + for(int i=0; i f = workFutures.get(i); + try { + f.get(); + } catch(ExecutionException e) { + LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(), + e.getCause()); + } + } + } + + /** + * Check consistency of all regions that have been found in previous phases. + */ + private void checkAndFixConsistency() + throws IOException, KeeperException, InterruptedException { + for (java.util.Map.Entry e: regionInfoMap.entrySet()) { + checkRegionConsistency(e.getKey(), e.getValue()); + } + } + + private void preCheckPermission() throws IOException, AccessControlException { + if (shouldIgnorePreCheckPermission()) { + return; + } + + Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR)); + FileSystem fs = hbaseDir.getFileSystem(getConf()); + User user = User.getCurrent(); + FileStatus[] files = fs.listStatus(hbaseDir); + for (FileStatus file : files) { + try { + FSUtils.checkAccess(user, file, FsAction.WRITE); + } catch (AccessControlException ace) { + LOG.warn("Got AccessControlException when preCheckPermission ", ace); + errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + user.getShortName() + + " does not have write perms to " + file.getPath() + + ". Please rerun hbck as hdfs user " + file.getOwner()); + throw new AccessControlException(ace); + } + } + } + + /** + * Deletes region from meta table + */ + private void deleteMetaRegion(HbckInfo hi) throws IOException { + Delete d = new Delete(hi.metaEntry.getRegionName()); + meta.delete(d); + meta.flushCommits(); + LOG.info("Deleted " + hi.metaEntry.getRegionNameAsString() + " from META" ); + } + + /** + * Reset the split parent region info in meta table + */ + private void resetSplitParent(HbckInfo hi) throws IOException { + RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName()); + Delete d = new Delete(hi.metaEntry.getRegionName()); + d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER); + d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER); + mutations.add(d); + + Put p = new Put(hi.metaEntry.getRegionName()); + HRegionInfo hri = new HRegionInfo(hi.metaEntry); + hri.setOffline(false); + hri.setSplit(false); + p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + mutations.add(p); + + meta.mutateRow(mutations); + meta.flushCommits(); + LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" ); + } + + /** + * This backwards-compatibility wrapper for permanently offlining a region + * that should not be alive. If the region server does not support the + * "offline" method, it will use the closest unassign method instead. This + * will basically work until one attempts to disable or delete the affected + * table. The problem has to do with in-memory only master state, so + * restarting the HMaster or failing over to another should fix this. + */ + private void offline(byte[] regionName) throws IOException { + String regionString = Bytes.toStringBinary(regionName); + if (!rsSupportsOffline) { + LOG.warn("Using unassign region " + regionString + + " instead of using offline method, you should" + + " restart HMaster after these repairs"); + admin.unassign(regionName, true); + return; + } + + // first time we assume the rs's supports #offline. + try { + LOG.info("Offlining region " + regionString); + admin.getMaster().offline(regionName); + } catch (IOException ioe) { + String notFoundMsg = "java.lang.NoSuchMethodException: " + + "org.apache.hadoop.hbase.master.HMaster.offline([B)"; + if (ioe.getMessage().contains(notFoundMsg)) { + LOG.warn("Using unassign region " + regionString + + " instead of using offline method, you should" + + " restart HMaster after these repairs"); + rsSupportsOffline = false; // in the future just use unassign + admin.unassign(regionName, true); + return; + } + throw ioe; + } + } + + private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException { + for (OnlineEntry rse : hi.deployedEntries) { + LOG.debug("Undeploy region " + rse.hri + " from " + rse.hsa); + try { + HBaseFsckRepair.closeRegionSilentlyAndWait(admin, rse.hsa, rse.hri); + offline(rse.hri.getRegionName()); + } catch (IOException ioe) { + LOG.warn("Got exception when attempting to offline region " + + Bytes.toString(rse.hri.getRegionName()), ioe); + } + } + } + + /** + * Attempts to undeploy a region from a region server based in information in + * META. Any operations that modify the file system should make sure that + * its corresponding region is not deployed to prevent data races. + * + * A separate call is required to update the master in-memory region state + * kept in the AssignementManager. Because disable uses this state instead of + * that found in META, we can't seem to cleanly disable/delete tables that + * have been hbck fixed. When used on a version of HBase that does not have + * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master + * restart or failover may be required. + */ + @SuppressWarnings("deprecation") + private void closeRegion(HbckInfo hi) throws IOException, InterruptedException { + if (hi.metaEntry == null && hi.hdfsEntry == null) { + undeployRegions(hi); + return; + } + + // get assignment info and hregioninfo from meta. + Get get = new Get(hi.getRegionName()); + get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); + get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER); + Result r = meta.get(get); + byte[] value = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); + byte[] startcodeBytes = r.getValue(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER); + if (value == null || startcodeBytes == null) { + errors.reportError("Unable to close region " + + hi.getRegionNameAsString() + " because meta does not " + + "have handle to reach it."); + return; + } + long startcode = Bytes.toLong(startcodeBytes); + + ServerName hsa = new ServerName(Bytes.toString(value), startcode); + byte[] hriVal = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + HRegionInfo hri= Writables.getHRegionInfoOrNull(hriVal); + if (hri == null) { + LOG.warn("Unable to close region " + hi.getRegionNameAsString() + + " because META had invalid or missing " + + HConstants.CATALOG_FAMILY_STR + ":" + + Bytes.toString(HConstants.REGIONINFO_QUALIFIER) + + " qualifier value."); + return; + } + + // close the region -- close files and remove assignment + HBaseFsckRepair.closeRegionSilentlyAndWait(admin, hsa, hri); + } + + private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException, + KeeperException, InterruptedException { + // If we are trying to fix the errors + if (shouldFixAssignments()) { + errors.print(msg); + undeployRegions(hbi); + setShouldRerun(); + HRegionInfo hri = hbi.getHdfsHRI(); + if (hri == null) { + hri = hbi.metaEntry; + } + HBaseFsckRepair.fixUnassigned(admin, hri); + HBaseFsckRepair.waitUntilAssigned(admin, hri); + } + } + + /** + * Check a single region for consistency and correct deployment. + */ + private void checkRegionConsistency(final String key, final HbckInfo hbi) + throws IOException, KeeperException, InterruptedException { + String descriptiveName = hbi.toString(); + + boolean inMeta = hbi.metaEntry != null; + // In case not checking HDFS, assume the region is on HDFS + boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null; + boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null; + boolean isDeployed = !hbi.deployedOn.isEmpty(); + boolean isMultiplyDeployed = hbi.deployedOn.size() > 1; + boolean deploymentMatchesMeta = + hasMetaAssignment && isDeployed && !isMultiplyDeployed && + hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0)); + boolean splitParent = + (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline(); + boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry); + boolean recentlyModified = inHdfs && + hbi.getModTime() + timelag > System.currentTimeMillis(); + + // ========== First the healthy cases ============= + if (hbi.containsOnlyHdfsEdits()) { + return; + } + if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) { + return; + } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) { + LOG.info("Region " + descriptiveName + " is in META, and in a disabled " + + "tabled that is not deployed"); + return; + } else if (recentlyModified) { + LOG.warn("Region " + descriptiveName + " was recently modified -- skipping"); + return; + } + // ========== Cases where the region is not in META ============= + else if (!inMeta && !inHdfs && !isDeployed) { + // We shouldn't have record of this region at all then! + assert false : "Entry for region with no data"; + } else if (!inMeta && !inHdfs && isDeployed) { + errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region " + + descriptiveName + ", key=" + key + ", not on HDFS or in META but " + + "deployed on " + Joiner.on(", ").join(hbi.deployedOn)); + if (shouldFixAssignments()) { + undeployRegions(hbi); + } + + } else if (!inMeta && inHdfs && !isDeployed) { + errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region " + + descriptiveName + " on HDFS, but not listed in META " + + "or deployed on any region server"); + // restore region consistency of an adopted orphan + if (shouldFixMeta()) { + if (!hbi.isHdfsRegioninfoPresent()) { + LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired" + + " in table integrity repair phase if -fixHdfsOrphans was" + + " used."); + return; + } + + LOG.info("Patching .META. with .regioninfo: " + hbi.getHdfsHRI()); + HBaseFsckRepair.fixMetaHoleOnline(getConf(), hbi.getHdfsHRI()); + + tryAssignmentRepair(hbi, "Trying to reassign region..."); + } + + } else if (!inMeta && inHdfs && isDeployed) { + errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName + + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn)); + debugLsr(hbi.getHdfsRegionDir()); + if (shouldFixMeta()) { + if (!hbi.isHdfsRegioninfoPresent()) { + LOG.error("This should have been repaired in table integrity repair phase"); + return; + } + + LOG.info("Patching .META. with with .regioninfo: " + hbi.getHdfsHRI()); + HBaseFsckRepair.fixMetaHoleOnline(getConf(), hbi.getHdfsHRI()); + + tryAssignmentRepair(hbi, "Trying to fix unassigned region..."); + } + + // ========== Cases where the region is in META ============= + } else if (inMeta && inHdfs && !isDeployed && splitParent) { + // check whether this is an actual error, or just transient state where parent + // is not cleaned + if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) { + // check that split daughters are there + HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName()); + HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName()); + if (infoA != null && infoB != null) { + // we already processed or will process daughters. Move on, nothing to see here. + hbi.setSkipChecks(true); + return; + } + } + errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region " + + descriptiveName + " is a split parent in META, in HDFS, " + + "and not deployed on any region server. This could be transient."); + if (shouldFixSplitParents()) { + setShouldRerun(); + resetSplitParent(hbi); + } + } else if (inMeta && !inHdfs && !isDeployed) { + errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region " + + descriptiveName + " found in META, but not in HDFS " + + "or deployed on any region server."); + if (shouldFixMeta()) { + deleteMetaRegion(hbi); + } + } else if (inMeta && !inHdfs && isDeployed) { + errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName + + " found in META, but not in HDFS, " + + "and deployed on " + Joiner.on(", ").join(hbi.deployedOn)); + // We treat HDFS as ground truth. Any information in meta is transient + // and equivalent data can be regenerated. So, lets unassign and remove + // these problems from META. + if (shouldFixAssignments()) { + errors.print("Trying to fix unassigned region..."); + closeRegion(hbi);// Close region will cause RS to abort. + } + if (shouldFixMeta()) { + // wait for it to complete + deleteMetaRegion(hbi); + } + } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) { + errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName + + " not deployed on any region server."); + tryAssignmentRepair(hbi, "Trying to fix unassigned region..."); + } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) { + errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED, + "Region " + descriptiveName + " should not be deployed according " + + "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn)); + if (shouldFixAssignments()) { + errors.print("Trying to close the region " + descriptiveName); + setShouldRerun(); + HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn); + } + } else if (inMeta && inHdfs && isMultiplyDeployed) { + errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName + + " is listed in META on region server " + hbi.metaEntry.regionServer + + " but is multiply assigned to region servers " + + Joiner.on(", ").join(hbi.deployedOn)); + // If we are trying to fix the errors + if (shouldFixAssignments()) { + errors.print("Trying to fix assignment error..."); + setShouldRerun(); + HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn); + } + } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) { + errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region " + + descriptiveName + " listed in META on region server " + + hbi.metaEntry.regionServer + " but found on region server " + + hbi.deployedOn.get(0)); + // If we are trying to fix the errors + if (shouldFixAssignments()) { + errors.print("Trying to fix assignment error..."); + setShouldRerun(); + HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn); + HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI()); + } + } else { + errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName + + " is in an unforeseen state:" + + " inMeta=" + inMeta + + " inHdfs=" + inHdfs + + " isDeployed=" + isDeployed + + " isMultiplyDeployed=" + isMultiplyDeployed + + " deploymentMatchesMeta=" + deploymentMatchesMeta + + " shouldBeDeployed=" + shouldBeDeployed); + } + } + + /** + * Checks tables integrity. Goes over all regions and scans the tables. + * Collects all the pieces for each table and checks if there are missing, + * repeated or overlapping ones. + * @throws IOException + */ + SortedMap checkIntegrity() throws IOException { + tablesInfo = new TreeMap (); + List noHDFSRegionInfos = new ArrayList(); + LOG.debug("There are " + regionInfoMap.size() + " region info entries"); + for (HbckInfo hbi : regionInfoMap.values()) { + // Check only valid, working regions + if (hbi.metaEntry == null) { + // this assumes that consistency check has run loadMetaEntry + noHDFSRegionInfos.add(hbi); + Path p = hbi.getHdfsRegionDir(); + if (p == null) { + errors.report("No regioninfo in Meta or HDFS. " + hbi); + } + + // TODO test. + continue; + } + if (hbi.metaEntry.regionServer == null) { + errors.detail("Skipping region because no region server: " + hbi); + continue; + } + if (hbi.metaEntry.isOffline()) { + errors.detail("Skipping region because it is offline: " + hbi); + continue; + } + if (hbi.containsOnlyHdfsEdits()) { + errors.detail("Skipping region because it only contains edits" + hbi); + continue; + } + + // Missing regionDir or over-deployment is checked elsewhere. Include + // these cases in modTInfo, so we can evaluate those regions as part of + // the region chain in META + //if (hbi.foundRegionDir == null) continue; + //if (hbi.deployedOn.size() != 1) continue; + if (hbi.deployedOn.size() == 0) continue; + + // We should be safe here + String tableName = hbi.metaEntry.getTableNameAsString(); + TableInfo modTInfo = tablesInfo.get(tableName); + if (modTInfo == null) { + modTInfo = new TableInfo(tableName); + } + for (ServerName server : hbi.deployedOn) { + modTInfo.addServer(server); + } + + if (!hbi.isSkipChecks()) { + modTInfo.addRegionInfo(hbi); + } + + tablesInfo.put(tableName, modTInfo); + } + + for (TableInfo tInfo : tablesInfo.values()) { + TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors); + if (!tInfo.checkRegionChain(handler)) { + errors.report("Found inconsistency in table " + tInfo.getName()); + } + } + return tablesInfo; + } + + /** + * Merge hdfs data by moving from contained HbckInfo into targetRegionDir. + * @return number of file move fixes done to merge regions. + */ + public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException { + int fileMoves = 0; + + LOG.debug("Contained region dir after close and pause"); + debugLsr(contained.getHdfsRegionDir()); + + // rename the contained into the container. + FileSystem fs = targetRegionDir.getFileSystem(getConf()); + FileStatus[] dirs = fs.listStatus(contained.getHdfsRegionDir()); + + if (dirs == null) { + if (!fs.exists(contained.getHdfsRegionDir())) { + LOG.warn("HDFS region dir " + contained.getHdfsRegionDir() + " already sidelined."); + } else { + sidelineRegionDir(fs, contained); + } + return fileMoves; + } + + for (FileStatus cf : dirs) { + Path src = cf.getPath(); + Path dst = new Path(targetRegionDir, src.getName()); + + if (src.getName().equals(HRegion.REGIONINFO_FILE)) { + // do not copy the old .regioninfo file. + continue; + } + + if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) { + // do not copy the .oldlogs files + continue; + } + + LOG.info("Moving files from " + src + " into containing region " + dst); + // FileSystem.rename is inconsistent with directories -- if the + // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir, + // it moves the src into the dst dir resulting in (foo/a/b). If + // the dst does not exist, and the src a dir, src becomes dst. (foo/b) + for (FileStatus hfile : fs.listStatus(src)) { + boolean success = fs.rename(hfile.getPath(), dst); + if (success) { + fileMoves++; + } + } + LOG.debug("Sideline directory contents:"); + debugLsr(targetRegionDir); + } + + // if all success. + sidelineRegionDir(fs, contained); + LOG.info("Sidelined region dir "+ contained.getHdfsRegionDir() + " into " + + getSidelineDir()); + debugLsr(contained.getHdfsRegionDir()); + + return fileMoves; + } + + /** + * Maintain information about a particular table. + */ + public class TableInfo { + String tableName; + TreeSet deployedOn; + + // backwards regions + final List backwards = new ArrayList(); + + // sidelined big overlapped regions + final Map sidelinedRegions = new HashMap(); + + // region split calculator + final RegionSplitCalculator sc = new RegionSplitCalculator(cmp); + + // Histogram of different HTableDescriptors found. Ideally there is only one! + final Set htds = new HashSet(); + + // key = start split, values = set of splits in problem group + final Multimap overlapGroups = + TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp); + + TableInfo(String name) { + this.tableName = name; + deployedOn = new TreeSet (); + } + + /** + * @return descriptor common to all regions. null if are none or multiple! + */ + private HTableDescriptor getHTD() { + if (htds.size() == 1) { + return (HTableDescriptor)htds.toArray()[0]; + } else { + LOG.error("None/Multiple table descriptors found for table '" + + tableName + "' regions: " + htds); + } + return null; + } + + public void addRegionInfo(HbckInfo hir) { + if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) { + // end key is absolute end key, just add it. + sc.add(hir); + return; + } + + // if not the absolute end key, check for cycle + if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) { + errors.reportError( + ERROR_CODE.REGION_CYCLE, + String.format("The endkey for this region comes before the " + + "startkey, startkey=%s, endkey=%s", + Bytes.toStringBinary(hir.getStartKey()), + Bytes.toStringBinary(hir.getEndKey())), this, hir); + backwards.add(hir); + return; + } + + // main case, add to split calculator + sc.add(hir); + } + + public void addServer(ServerName server) { + this.deployedOn.add(server); + } + + public String getName() { + return tableName; + } + + public int getNumRegions() { + return sc.getStarts().size() + backwards.size(); + } + + private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl { + ErrorReporter errors; + + IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) { + this.errors = errors; + setTableInfo(ti); + } + + @Override + public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{ + errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY, + "First region should start with an empty key. You need to " + + " create a new region and regioninfo in HDFS to plug the hole.", + getTableInfo(), hi); + } + + @Override + public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException { + errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY, + "Last region should end with an empty key. You need to " + + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo()); + } + + @Override + public void handleDegenerateRegion(HbckInfo hi) throws IOException{ + errors.reportError(ERROR_CODE.DEGENERATE_REGION, + "Region has the same start and end key.", getTableInfo(), hi); + } + + @Override + public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{ + byte[] key = r1.getStartKey(); + // dup start key + errors.reportError(ERROR_CODE.DUPE_STARTKEYS, + "Multiple regions have the same startkey: " + + Bytes.toStringBinary(key), getTableInfo(), r1); + errors.reportError(ERROR_CODE.DUPE_STARTKEYS, + "Multiple regions have the same startkey: " + + Bytes.toStringBinary(key), getTableInfo(), r2); + } + + @Override + public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{ + errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN, + "There is an overlap in the region chain.", + getTableInfo(), hi1, hi2); + } + + @Override + public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{ + errors.reportError( + ERROR_CODE.HOLE_IN_REGION_CHAIN, + "There is a hole in the region chain between " + + Bytes.toStringBinary(holeStart) + " and " + + Bytes.toStringBinary(holeStop) + + ". You need to create a new .regioninfo and region " + + "dir in hdfs to plug the hole."); + } + }; + + /** + * This handler fixes integrity errors from hdfs information. There are + * basically three classes of integrity problems 1) holes, 2) overlaps, and + * 3) invalid regions. + * + * This class overrides methods that fix holes and the overlap group case. + * Individual cases of particular overlaps are handled by the general + * overlap group merge repair case. + * + * If hbase is online, this forces regions offline before doing merge + * operations. + */ + private class HDFSIntegrityFixer extends IntegrityFixSuggester { + Configuration conf; + + boolean fixOverlaps = true; + + HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf, + boolean fixHoles, boolean fixOverlaps) { + super(ti, errors); + this.conf = conf; + this.fixOverlaps = fixOverlaps; + // TODO properly use fixHoles + } + + /** + * This is a special case hole -- when the first region of a table is + * missing from META, HBase doesn't acknowledge the existance of the + * table. + */ + public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException { + errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY, + "First region should start with an empty key. Creating a new " + + "region and regioninfo in HDFS to plug the hole.", + getTableInfo(), next); + HTableDescriptor htd = getTableInfo().getHTD(); + // from special EMPTY_START_ROW to next region's startKey + HRegionInfo newRegion = new HRegionInfo(htd.getName(), + HConstants.EMPTY_START_ROW, next.getStartKey()); + + // TODO test + HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); + LOG.info("Table region start key was not empty. Created new empty region: " + + newRegion + " " +region); + fixes++; + } + + public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException { + errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY, + "Last region should end with an empty key. Creating a new " + + "region and regioninfo in HDFS to plug the hole.", getTableInfo()); + HTableDescriptor htd = getTableInfo().getHTD(); + // from curEndKey to EMPTY_START_ROW + HRegionInfo newRegion = new HRegionInfo(htd.getName(), curEndKey, + HConstants.EMPTY_START_ROW); + + HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); + LOG.info("Table region end key was not empty. Created new empty region: " + newRegion + + " " + region); + fixes++; + } + + /** + * There is a hole in the hdfs regions that violates the table integrity + * rules. Create a new empty region that patches the hole. + */ + public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException { + errors.reportError( + ERROR_CODE.HOLE_IN_REGION_CHAIN, + "There is a hole in the region chain between " + + Bytes.toStringBinary(holeStartKey) + " and " + + Bytes.toStringBinary(holeStopKey) + + ". Creating a new regioninfo and region " + + "dir in hdfs to plug the hole."); + HTableDescriptor htd = getTableInfo().getHTD(); + HRegionInfo newRegion = new HRegionInfo(htd.getName(), holeStartKey, holeStopKey); + HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); + LOG.info("Plugged hold by creating new empty region: "+ newRegion + " " +region); + fixes++; + } + + /** + * This takes set of overlapping regions and merges them into a single + * region. This covers cases like degenerate regions, shared start key, + * general overlaps, duplicate ranges, and partial overlapping regions. + * + * Cases: + * - Clean regions that overlap + * - Only .oldlogs regions (can't find start/stop range, or figure out) + */ + @Override + public void handleOverlapGroup(Collection overlap) + throws IOException { + Preconditions.checkNotNull(overlap); + Preconditions.checkArgument(overlap.size() >0); + + if (!this.fixOverlaps) { + LOG.warn("Not attempting to repair overlaps."); + return; + } + + if (overlap.size() > maxMerge) { + LOG.warn("Overlap group has " + overlap.size() + " overlapping " + + "regions which is greater than " + maxMerge + ", the max number of regions to merge"); + if (sidelineBigOverlaps) { + // we only sideline big overlapped groups that exceeds the max number of regions to merge + sidelineBigOverlaps(overlap); + } + return; + } + + mergeOverlaps(overlap); + } + + void mergeOverlaps(Collection overlap) + throws IOException { + LOG.info("== Merging regions into one region: " + + Joiner.on(",").join(overlap)); + // get the min / max range and close all concerned regions + Pair range = null; + for (HbckInfo hi : overlap) { + if (range == null) { + range = new Pair(hi.getStartKey(), hi.getEndKey()); + } else { + if (RegionSplitCalculator.BYTES_COMPARATOR + .compare(hi.getStartKey(), range.getFirst()) < 0) { + range.setFirst(hi.getStartKey()); + } + if (RegionSplitCalculator.BYTES_COMPARATOR + .compare(hi.getEndKey(), range.getSecond()) > 0) { + range.setSecond(hi.getEndKey()); + } + } + // need to close files so delete can happen. + LOG.debug("Closing region before moving data around: " + hi); + LOG.debug("Contained region dir before close"); + debugLsr(hi.getHdfsRegionDir()); + try { + LOG.info("Closing region: " + hi); + closeRegion(hi); + } catch (IOException ioe) { + LOG.warn("Was unable to close region " + hi + + ". Just continuing... ", ioe); + } catch (InterruptedException e) { + LOG.warn("Was unable to close region " + hi + + ". Just continuing... ", e); + } + + try { + LOG.info("Offlining region: " + hi); + offline(hi.getRegionName()); + } catch (IOException ioe) { + LOG.warn("Unable to offline region from master: " + hi + + ". Just continuing... ", ioe); + } + } + + // create new empty container region. + HTableDescriptor htd = getTableInfo().getHTD(); + // from start key to end Key + HRegionInfo newRegion = new HRegionInfo(htd.getName(), range.getFirst(), + range.getSecond()); + HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); + LOG.info("Created new empty container region: " + + newRegion + " to contain regions: " + Joiner.on(",").join(overlap)); + debugLsr(region.getRegionDir()); + + // all target regions are closed, should be able to safely cleanup. + boolean didFix= false; + Path target = region.getRegionDir(); + for (HbckInfo contained : overlap) { + LOG.info("Merging " + contained + " into " + target ); + int merges = mergeRegionDirs(target, contained); + if (merges > 0) { + didFix = true; + } + } + if (didFix) { + fixes++; + } + } + + /** + * Sideline some regions in a big overlap group so that it + * will have fewer regions, and it is easier to merge them later on. + * + * @param bigOverlap the overlapped group with regions more than maxMerge + * @throws IOException + */ + void sidelineBigOverlaps( + Collection bigOverlap) throws IOException { + int overlapsToSideline = bigOverlap.size() - maxMerge; + if (overlapsToSideline > maxOverlapsToSideline) { + overlapsToSideline = maxOverlapsToSideline; + } + List regionsToSideline = + RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline); + FileSystem fs = FileSystem.get(conf); + for (HbckInfo regionToSideline: regionsToSideline) { + try { + LOG.info("Closing region: " + regionToSideline); + closeRegion(regionToSideline); + } catch (IOException ioe) { + LOG.warn("Was unable to close region " + regionToSideline + + ". Just continuing... ", ioe); + } catch (InterruptedException e) { + LOG.warn("Was unable to close region " + regionToSideline + + ". Just continuing... ", e); + } + + try { + LOG.info("Offlining region: " + regionToSideline); + offline(regionToSideline.getRegionName()); + } catch (IOException ioe) { + LOG.warn("Unable to offline region from master: " + regionToSideline + + ". Just continuing... ", ioe); + } + + LOG.info("Before sideline big overlapped region: " + regionToSideline.toString()); + Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline); + if (sidelineRegionDir != null) { + sidelinedRegions.put(sidelineRegionDir, regionToSideline); + LOG.info("After sidelined big overlapped region: " + + regionToSideline.getRegionNameAsString() + + " to " + sidelineRegionDir.toString()); + fixes++; + } + } + } + } + + /** + * Check the region chain (from META) of this table. We are looking for + * holes, overlaps, and cycles. + * @return false if there are errors + * @throws IOException + */ + public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException { + // When table is disabled no need to check for the region chain. Some of the regions + // accidently if deployed, this below code might report some issues like missing start + // or end regions or region hole in chain and may try to fix which is unwanted. + if (disabledTables.contains(this.tableName.getBytes())) { + return true; + } + int originalErrorsCount = errors.getErrorList().size(); + Multimap regions = sc.calcCoverage(); + SortedSet splits = sc.getSplits(); + + byte[] prevKey = null; + byte[] problemKey = null; + for (byte[] key : splits) { + Collection ranges = regions.get(key); + if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) { + for (HbckInfo rng : ranges) { + handler.handleRegionStartKeyNotEmpty(rng); + } + } + + // check for degenerate ranges + for (HbckInfo rng : ranges) { + // special endkey case converts '' to null + byte[] endKey = rng.getEndKey(); + endKey = (endKey.length == 0) ? null : endKey; + if (Bytes.equals(rng.getStartKey(),endKey)) { + handler.handleDegenerateRegion(rng); + } + } + + if (ranges.size() == 1) { + // this split key is ok -- no overlap, not a hole. + if (problemKey != null) { + LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key)); + } + problemKey = null; // fell through, no more problem. + } else if (ranges.size() > 1) { + // set the new problem key group name, if already have problem key, just + // keep using it. + if (problemKey == null) { + // only for overlap regions. + LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key)); + problemKey = key; + } + overlapGroups.putAll(problemKey, ranges); + + // record errors + ArrayList subRange = new ArrayList(ranges); + // this dumb and n^2 but this shouldn't happen often + for (HbckInfo r1 : ranges) { + subRange.remove(r1); + for (HbckInfo r2 : subRange) { + if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) { + handler.handleDuplicateStartKeys(r1,r2); + } else { + // overlap + handler.handleOverlapInRegionChain(r1, r2); + } + } + } + + } else if (ranges.size() == 0) { + if (problemKey != null) { + LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key)); + } + problemKey = null; + + byte[] holeStopKey = sc.getSplits().higher(key); + // if higher key is null we reached the top. + if (holeStopKey != null) { + // hole + handler.handleHoleInRegionChain(key, holeStopKey); + } + } + prevKey = key; + } + + // When the last region of a table is proper and having an empty end key, 'prevKey' + // will be null. + if (prevKey != null) { + handler.handleRegionEndKeyNotEmpty(prevKey); + } + + for (Collection overlap : overlapGroups.asMap().values()) { + handler.handleOverlapGroup(overlap); + } + + if (details) { + // do full region split map dump + errors.print("---- Table '" + this.tableName + + "': region split map"); + dump(splits, regions); + errors.print("---- Table '" + this.tableName + + "': overlap groups"); + dumpOverlapProblems(overlapGroups); + errors.print("There are " + overlapGroups.keySet().size() + + " overlap groups with " + overlapGroups.size() + + " overlapping regions"); + } + if (!sidelinedRegions.isEmpty()) { + LOG.warn("Sidelined big overlapped regions, please bulk load them!"); + errors.print("---- Table '" + this.tableName + + "': sidelined big overlapped regions"); + dumpSidelinedRegions(sidelinedRegions); + } + return errors.getErrorList().size() == originalErrorsCount; + } + + /** + * This dumps data in a visually reasonable way for visual debugging + * + * @param splits + * @param regions + */ + void dump(SortedSet splits, Multimap regions) { + // we display this way because the last end key should be displayed as well. + StringBuilder sb = new StringBuilder(); + for (byte[] k : splits) { + sb.setLength(0); // clear out existing buffer, if any. + sb.append(Bytes.toStringBinary(k) + ":\t"); + for (HbckInfo r : regions.get(k)) { + sb.append("[ "+ r.toString() + ", " + + Bytes.toStringBinary(r.getEndKey())+ "]\t"); + } + errors.print(sb.toString()); + } + } + } + + public void dumpOverlapProblems(Multimap regions) { + // we display this way because the last end key should be displayed as + // well. + for (byte[] k : regions.keySet()) { + errors.print(Bytes.toStringBinary(k) + ":"); + for (HbckInfo r : regions.get(k)) { + errors.print("[ " + r.toString() + ", " + + Bytes.toStringBinary(r.getEndKey()) + "]"); + } + errors.print("----"); + } + } + + public void dumpSidelinedRegions(Map regions) { + for (Map.Entry entry: regions.entrySet()) { + String tableName = Bytes.toStringBinary(entry.getValue().getTableName()); + Path path = entry.getKey(); + errors.print("This sidelined region dir should be bulk loaded: " + + path.toString()); + errors.print("Bulk load command looks like: " + + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles " + + path.toUri().getPath() + " "+ tableName); + } + } + + public Multimap getOverlapGroups( + String table) { + TableInfo ti = tablesInfo.get(table); + return ti.overlapGroups; + } + + /** + * Return a list of user-space table names whose metadata have not been + * modified in the last few milliseconds specified by timelag + * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER, + * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last + * milliseconds specified by timelag, then the table is a candidate to be returned. + * @return tables that have not been modified recently + * @throws IOException if an error is encountered + */ + HTableDescriptor[] getTables(AtomicInteger numSkipped) { + List tableNames = new ArrayList(); + long now = System.currentTimeMillis(); + + for (HbckInfo hbi : regionInfoMap.values()) { + MetaEntry info = hbi.metaEntry; + + // if the start key is zero, then we have found the first region of a table. + // pick only those tables that were not modified in the last few milliseconds. + if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) { + if (info.modTime + timelag < now) { + tableNames.add(info.getTableNameAsString()); + } else { + numSkipped.incrementAndGet(); // one more in-flux table + } + } + } + return getHTableDescriptors(tableNames); + } + + HTableDescriptor[] getHTableDescriptors(List tableNames) { + HTableDescriptor[] htd = new HTableDescriptor[0]; + try { + LOG.info("getHTableDescriptors == tableNames => " + tableNames); + htd = new HBaseAdmin(getConf()).getTableDescriptors(tableNames); + } catch (IOException e) { + LOG.debug("Exception getting table descriptors", e); + } + return htd; + } + + + /** + * Gets the entry in regionInfo corresponding to the the given encoded + * region name. If the region has not been seen yet, a new entry is added + * and returned. + */ + private synchronized HbckInfo getOrCreateInfo(String name) { + HbckInfo hbi = regionInfoMap.get(name); + if (hbi == null) { + hbi = new HbckInfo(null); + regionInfoMap.put(name, hbi); + } + return hbi; + } + + /** + * Check values in regionInfo for .META. + * Check if zero or more than one regions with META are found. + * If there are inconsistencies (i.e. zero or more than one regions + * pretend to be holding the .META.) try to fix that and report an error. + * @throws IOException from HBaseFsckRepair functions + * @throws KeeperException + * @throws InterruptedException + */ + boolean checkMetaRegion() + throws IOException, KeeperException, InterruptedException { + List metaRegions = Lists.newArrayList(); + for (HbckInfo value : regionInfoMap.values()) { + if (value.metaEntry.isMetaRegion()) { + metaRegions.add(value); + } + } + + // If something is wrong + if (metaRegions.size() != 1) { + HRegionLocation rootLocation = connection.locateRegion( + HConstants.ROOT_TABLE_NAME, HConstants.EMPTY_START_ROW); + HbckInfo root = + regionInfoMap.get(rootLocation.getRegionInfo().getEncodedName()); + + // If there is no region holding .META. + if (metaRegions.size() == 0) { + errors.reportError(ERROR_CODE.NO_META_REGION, ".META. is not found on any region."); + if (shouldFixAssignments()) { + errors.print("Trying to fix a problem with .META..."); + setShouldRerun(); + // try to fix it (treat it as unassigned region) + HBaseFsckRepair.fixUnassigned(admin, root.metaEntry); + HBaseFsckRepair.waitUntilAssigned(admin, root.getHdfsHRI()); + } + } + // If there are more than one regions pretending to hold the .META. + else if (metaRegions.size() > 1) { + errors.reportError(ERROR_CODE.MULTI_META_REGION, ".META. is found on more than one region."); + if (shouldFixAssignments()) { + errors.print("Trying to fix a problem with .META..."); + setShouldRerun(); + // try fix it (treat is a dupe assignment) + List deployedOn = Lists.newArrayList(); + for (HbckInfo mRegion : metaRegions) { + deployedOn.add(mRegion.metaEntry.regionServer); + } + HBaseFsckRepair.fixMultiAssignment(admin, root.metaEntry, deployedOn); + } + } + // rerun hbck with hopefully fixed META + return false; + } + // no errors, so continue normally + return true; + } + + /** + * Scan .META. and -ROOT-, adding all regions found to the regionInfo map. + * @throws IOException if an error is encountered + */ + boolean loadMetaEntries() throws IOException { + + // get a list of all regions from the master. This involves + // scanning the META table + if (!recordRootRegion()) { + // Will remove later if we can fix it + errors.reportError("Fatal error: unable to get root region location. Exiting..."); + return false; + } + + MetaScannerVisitor visitor = new MetaScannerVisitorBase() { + int countRecord = 1; + + // comparator to sort KeyValues with latest modtime + final Comparator comp = new Comparator() { + public int compare(KeyValue k1, KeyValue k2) { + return (int)(k1.getTimestamp() - k2.getTimestamp()); + } + }; + + public boolean processRow(Result result) throws IOException { + try { + + // record the latest modification of this META record + long ts = Collections.max(result.list(), comp).getTimestamp(); + Pair pair = MetaReader.parseCatalogResult(result); + if (pair == null || pair.getFirst() == null) { + emptyRegionInfoQualifiers.add(result); + return true; + } + ServerName sn = null; + if (pair.getSecond() != null) { + sn = pair.getSecond(); + } + HRegionInfo hri = pair.getFirst(); + if (!(isTableIncluded(hri.getTableNameAsString()) + || hri.isMetaRegion() || hri.isRootRegion())) { + return true; + } + PairOfSameType daughters = MetaReader.getDaughterRegions(result); + MetaEntry m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond()); + HbckInfo hbInfo = new HbckInfo(m); + HbckInfo previous = regionInfoMap.put(hri.getEncodedName(), hbInfo); + if (previous != null) { + throw new IOException("Two entries in META are same " + previous); + } + + // show proof of progress to the user, once for every 100 records. + if (countRecord % 100 == 0) { + errors.progress(); + } + countRecord++; + return true; + } catch (RuntimeException e) { + LOG.error("Result=" + result); + throw e; + } + } + }; + + // Scan -ROOT- to pick up META regions + MetaScanner.metaScan(getConf(), visitor, null, null, + Integer.MAX_VALUE, HConstants.ROOT_TABLE_NAME); + + if (!checkMetaOnly) { + // Scan .META. to pick up user regions + MetaScanner.metaScan(getConf(), visitor); + } + + errors.print(""); + return true; + } + + /** + * Stores the regioninfo entries scanned from META + */ + static class MetaEntry extends HRegionInfo { + ServerName regionServer; // server hosting this region + long modTime; // timestamp of most recent modification metadata + HRegionInfo splitA, splitB; //split daughters + + public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) { + this(rinfo, regionServer, modTime, null, null); + } + + public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime, + HRegionInfo splitA, HRegionInfo splitB) { + super(rinfo); + this.regionServer = regionServer; + this.modTime = modTime; + this.splitA = splitA; + this.splitB = splitB; + } + + public boolean equals(Object o) { + boolean superEq = super.equals(o); + if (!superEq) { + return superEq; + } + + MetaEntry me = (MetaEntry) o; + if (!regionServer.equals(me.regionServer)) { + return false; + } + return (modTime == me.modTime); + } + } + + /** + * Stores the regioninfo entries from HDFS + */ + static class HdfsEntry { + HRegionInfo hri; + Path hdfsRegionDir = null; + long hdfsRegionDirModTime = 0; + boolean hdfsRegioninfoFilePresent = false; + boolean hdfsOnlyEdits = false; + } + + /** + * Stores the regioninfo retrieved from Online region servers. + */ + static class OnlineEntry { + HRegionInfo hri; + ServerName hsa; + + public String toString() { + return hsa.toString() + ";" + hri.getRegionNameAsString(); + } + } + + /** + * Maintain information about a particular region. It gathers information + * from three places -- HDFS, META, and region servers. + */ + public static class HbckInfo implements KeyRange { + private MetaEntry metaEntry = null; // info in META + private HdfsEntry hdfsEntry = null; // info in HDFS + private List deployedEntries = Lists.newArrayList(); // on Region Server + private List deployedOn = Lists.newArrayList(); // info on RS's + private boolean skipChecks = false; // whether to skip further checks to this region info. + + HbckInfo(MetaEntry metaEntry) { + this.metaEntry = metaEntry; + } + + public synchronized void addServer(HRegionInfo hri, ServerName server) { + OnlineEntry rse = new OnlineEntry() ; + rse.hri = hri; + rse.hsa = server; + this.deployedEntries.add(rse); + this.deployedOn.add(server); + } + + public synchronized String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{ meta => "); + sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null"); + sb.append( ", hdfs => " + getHdfsRegionDir()); + sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries)); + sb.append(" }"); + return sb.toString(); + } + + @Override + public byte[] getStartKey() { + if (this.metaEntry != null) { + return this.metaEntry.getStartKey(); + } else if (this.hdfsEntry != null) { + return this.hdfsEntry.hri.getStartKey(); + } else { + LOG.error("Entry " + this + " has no meta or hdfs region start key."); + return null; + } + } + + @Override + public byte[] getEndKey() { + if (this.metaEntry != null) { + return this.metaEntry.getEndKey(); + } else if (this.hdfsEntry != null) { + return this.hdfsEntry.hri.getEndKey(); + } else { + LOG.error("Entry " + this + " has no meta or hdfs region start key."); + return null; + } + } + + public byte[] getTableName() { + if (this.metaEntry != null) { + return this.metaEntry.getTableName(); + } else if (this.hdfsEntry != null) { + // we are only guaranteed to have a path and not an HRI for hdfsEntry, + // so we get the name from the Path + Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent(); + return Bytes.toBytes(tableDir.getName()); + } else { + // Currently no code exercises this path, but we could add one for + // getting table name from OnlineEntry + return null; + } + } + + public String getRegionNameAsString() { + if (metaEntry != null) { + return metaEntry.getRegionNameAsString(); + } else if (hdfsEntry != null) { + if (hdfsEntry.hri != null) { + return hdfsEntry.hri.getRegionNameAsString(); + } + } + return null; + } + + public byte[] getRegionName() { + if (metaEntry != null) { + return metaEntry.getRegionName(); + } else if (hdfsEntry != null) { + return hdfsEntry.hri.getRegionName(); + } else { + return null; + } + } + + Path getHdfsRegionDir() { + if (hdfsEntry == null) { + return null; + } + return hdfsEntry.hdfsRegionDir; + } + + boolean containsOnlyHdfsEdits() { + if (hdfsEntry == null) { + return false; + } + return hdfsEntry.hdfsOnlyEdits; + } + + boolean isHdfsRegioninfoPresent() { + if (hdfsEntry == null) { + return false; + } + return hdfsEntry.hdfsRegioninfoFilePresent; + } + + long getModTime() { + if (hdfsEntry == null) { + return 0; + } + return hdfsEntry.hdfsRegionDirModTime; + } + + HRegionInfo getHdfsHRI() { + if (hdfsEntry == null) { + return null; + } + return hdfsEntry.hri; + } + + public void setSkipChecks(boolean skipChecks) { + this.skipChecks = skipChecks; + } + + public boolean isSkipChecks() { + return skipChecks; + } + } + + final static Comparator cmp = new Comparator() { + @Override + public int compare(HbckInfo l, HbckInfo r) { + if (l == r) { + // same instance + return 0; + } + + int tableCompare = RegionSplitCalculator.BYTES_COMPARATOR.compare( + l.getTableName(), r.getTableName()); + if (tableCompare != 0) { + return tableCompare; + } + + int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare( + l.getStartKey(), r.getStartKey()); + if (startComparison != 0) { + return startComparison; + } + + // Special case for absolute endkey + byte[] endKey = r.getEndKey(); + endKey = (endKey.length == 0) ? null : endKey; + byte[] endKey2 = l.getEndKey(); + endKey2 = (endKey2.length == 0) ? null : endKey2; + int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare( + endKey2, endKey); + + if (endComparison != 0) { + return endComparison; + } + + // use regionId as tiebreaker. + // Null is considered after all possible values so make it bigger. + if (l.hdfsEntry == null && r.hdfsEntry == null) { + return 0; + } + if (l.hdfsEntry == null && r.hdfsEntry != null) { + return 1; + } + // l.hdfsEntry must not be null + if (r.hdfsEntry == null) { + return -1; + } + // both l.hdfsEntry and r.hdfsEntry must not be null. + return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId()); + } + }; + + /** + * Prints summary of all tables found on the system. + */ + private void printTableSummary(SortedMap tablesInfo) { + StringBuilder sb = new StringBuilder(); + errors.print("Summary:"); + for (TableInfo tInfo : tablesInfo.values()) { + if (errors.tableHasErrors(tInfo)) { + errors.print("Table " + tInfo.getName() + " is inconsistent."); + } else { + errors.print(" " + tInfo.getName() + " is okay."); + } + errors.print(" Number of regions: " + tInfo.getNumRegions()); + sb.setLength(0); // clear out existing buffer, if any. + sb.append(" Deployed on: "); + for (ServerName server : tInfo.deployedOn) { + sb.append(" " + server.toString()); + } + errors.print(sb.toString()); + } + } + + static ErrorReporter getErrorReporter( + final Configuration conf) throws ClassNotFoundException { + Class reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class); + return (ErrorReporter)ReflectionUtils.newInstance(reporter, conf); + } + + public interface ErrorReporter { + public static enum ERROR_CODE { + UNKNOWN, NO_META_REGION, NULL_ROOT_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META, + NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED, + MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE, + FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS, + HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION, + ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE, + WRONG_USAGE + } + public void clear(); + public void report(String message); + public void reportError(String message); + public void reportError(ERROR_CODE errorCode, String message); + public void reportError(ERROR_CODE errorCode, String message, TableInfo table); + public void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info); + public void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info1, HbckInfo info2); + public int summarize(); + public void detail(String details); + public ArrayList getErrorList(); + public void progress(); + public void print(String message); + public void resetErrors(); + public boolean tableHasErrors(TableInfo table); + } + + static class PrintingErrorReporter implements ErrorReporter { + public int errorCount = 0; + private int showProgress; + + Set errorTables = new HashSet(); + + // for use by unit tests to verify which errors were discovered + private ArrayList errorList = new ArrayList(); + + public void clear() { + errorTables.clear(); + errorList.clear(); + errorCount = 0; + } + + public synchronized void reportError(ERROR_CODE errorCode, String message) { + if (errorCode == ERROR_CODE.WRONG_USAGE) { + System.err.println(message); + return; + } + + errorList.add(errorCode); + if (!summary) { + System.out.println("ERROR: " + message); + } + errorCount++; + showProgress = 0; + } + + public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) { + errorTables.add(table); + reportError(errorCode, message); + } + + public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table, + HbckInfo info) { + errorTables.add(table); + String reference = "(region " + info.getRegionNameAsString() + ")"; + reportError(errorCode, reference + " " + message); + } + + public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table, + HbckInfo info1, HbckInfo info2) { + errorTables.add(table); + String reference = "(regions " + info1.getRegionNameAsString() + + " and " + info2.getRegionNameAsString() + ")"; + reportError(errorCode, reference + " " + message); + } + + public synchronized void reportError(String message) { + reportError(ERROR_CODE.UNKNOWN, message); + } + + /** + * Report error information, but do not increment the error count. Intended for cases + * where the actual error would have been reported previously. + * @param message + */ + public synchronized void report(String message) { + if (! summary) { + System.out.println("ERROR: " + message); + } + showProgress = 0; + } + + public synchronized int summarize() { + System.out.println(Integer.toString(errorCount) + + " inconsistencies detected."); + if (errorCount == 0) { + System.out.println("Status: OK"); + return 0; + } else { + System.out.println("Status: INCONSISTENT"); + return -1; + } + } + + public ArrayList getErrorList() { + return errorList; + } + + public synchronized void print(String message) { + if (!summary) { + System.out.println(message); + } + } + + @Override + public boolean tableHasErrors(TableInfo table) { + return errorTables.contains(table); + } + + @Override + public void resetErrors() { + errorCount = 0; + } + + public synchronized void detail(String message) { + if (details) { + System.out.println(message); + } + showProgress = 0; + } + + public synchronized void progress() { + if (showProgress++ == 10) { + if (!summary) { + System.out.print("."); + } + showProgress = 0; + } + } + } + + /** + * Contact a region server and get all information from it + */ + static class WorkItemRegion implements Callable { + private HBaseFsck hbck; + private ServerName rsinfo; + private ErrorReporter errors; + private HConnection connection; + + WorkItemRegion(HBaseFsck hbck, ServerName info, + ErrorReporter errors, HConnection connection) { + this.hbck = hbck; + this.rsinfo = info; + this.errors = errors; + this.connection = connection; + } + + @Override + public synchronized Void call() throws IOException { + errors.progress(); + try { + HRegionInterface server = + connection.getHRegionConnection(rsinfo.getHostname(), rsinfo.getPort()); + + // list all online regions from this region server + List regions = server.getOnlineRegions(); + regions = filterRegions(regions); + if (details) { + errors.detail("RegionServer: " + rsinfo.getServerName() + + " number of regions: " + regions.size()); + for (HRegionInfo rinfo: regions) { + errors.detail(" " + rinfo.getRegionNameAsString() + + " id: " + rinfo.getRegionId() + + " encoded_name: " + rinfo.getEncodedName() + + " start: " + Bytes.toStringBinary(rinfo.getStartKey()) + + " end: " + Bytes.toStringBinary(rinfo.getEndKey())); + } + } + + // check to see if the existence of this region matches the region in META + for (HRegionInfo r:regions) { + HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName()); + hbi.addServer(r, rsinfo); + } + } catch (IOException e) { // unable to connect to the region server. + errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() + + " Unable to fetch region information. " + e); + throw e; + } + return null; + } + + private List filterRegions(List regions) { + List ret = Lists.newArrayList(); + for (HRegionInfo hri : regions) { + if (hri.isMetaTable() || (!hbck.checkMetaOnly + && hbck.isTableIncluded(hri.getTableNameAsString()))) { + ret.add(hri); + } + } + return ret; + } + } + + /** + * Contact hdfs and get all information about specified table directory into + * regioninfo list. + */ + static class WorkItemHdfsDir implements Callable { + private HBaseFsck hbck; + private FileStatus tableDir; + private ErrorReporter errors; + private FileSystem fs; + + WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors, + FileStatus status) { + this.hbck = hbck; + this.fs = fs; + this.tableDir = status; + this.errors = errors; + } + + @Override + public synchronized Void call() throws IOException { + try { + String tableName = tableDir.getPath().getName(); + // ignore hidden files + if (tableName.startsWith(".") && + !tableName.equals( Bytes.toString(HConstants.META_TABLE_NAME))) { + return null; + } + // level 2: /
    /* + FileStatus[] regionDirs = fs.listStatus(tableDir.getPath()); + for (FileStatus regionDir : regionDirs) { + String encodedName = regionDir.getPath().getName(); + // ignore directories that aren't hexadecimal + if (!encodedName.toLowerCase().matches("[0-9a-f]+")) { + continue; + } + + LOG.debug("Loading region info from hdfs:"+ regionDir.getPath()); + HbckInfo hbi = hbck.getOrCreateInfo(encodedName); + HdfsEntry he = new HdfsEntry(); + synchronized (hbi) { + if (hbi.getHdfsRegionDir() != null) { + errors.print("Directory " + encodedName + " duplicate??" + + hbi.getHdfsRegionDir()); + } + + he.hdfsRegionDir = regionDir.getPath(); + he.hdfsRegionDirModTime = regionDir.getModificationTime(); + Path regioninfoFile = new Path(he.hdfsRegionDir, HRegion.REGIONINFO_FILE); + he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile); + // we add to orphan list when we attempt to read .regioninfo + + // Set a flag if this region contains only edits + // This is special case if a region is left after split + he.hdfsOnlyEdits = true; + FileStatus[] subDirs = fs.listStatus(regionDir.getPath()); + Path ePath = HLog.getRegionDirRecoveredEditsDir(regionDir.getPath()); + for (FileStatus subDir : subDirs) { + String sdName = subDir.getPath().getName(); + if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) { + he.hdfsOnlyEdits = false; + break; + } + } + hbi.hdfsEntry = he; + } + } + } catch (IOException e) { + // unable to connect to the region server. + errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: " + + tableDir.getPath().getName() + + " Unable to fetch region information. " + e); + throw e; + } + return null; + } + } + + /** + * Contact hdfs and get all information about specified table directory into + * regioninfo list. + */ + static class WorkItemHdfsRegionInfo implements Callable { + private HbckInfo hbi; + private HBaseFsck hbck; + private ErrorReporter errors; + + WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) { + this.hbi = hbi; + this.hbck = hbck; + this.errors = errors; + } + + @Override + public synchronized Void call() throws IOException { + // only load entries that haven't been loaded yet. + if (hbi.getHdfsHRI() == null) { + try { + hbck.loadHdfsRegioninfo(hbi); + } catch (IOException ioe) { + String msg = "Orphan region in HDFS: Unable to load .regioninfo from table " + + Bytes.toString(hbi.getTableName()) + " in hdfs dir " + + hbi.getHdfsRegionDir() + + "! It may be an invalid format or version file. Treating as " + + "an orphaned regiondir."; + errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg); + try { + hbck.debugLsr(hbi.getHdfsRegionDir()); + } catch (IOException ioe2) { + LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2); + throw ioe2; + } + hbck.orphanHdfsDirs.add(hbi); + throw ioe; + } + } + return null; + } + }; + + /** + * Display the full report from fsck. This displays all live and dead region + * servers, and all known regions. + */ + public void setDisplayFullReport() { + details = true; + } + + /** + * Set summary mode. + * Print only summary of the tables and status (OK or INCONSISTENT) + */ + void setSummary() { + summary = true; + } + + /** + * Set META check mode. + * Print only info about META table deployment/state + */ + void setCheckMetaOnly() { + checkMetaOnly = true; + } + + /** + * Check if we should rerun fsck again. This checks if we've tried to + * fix something and we should rerun fsck tool again. + * Display the full report from fsck. This displays all live and dead + * region servers, and all known regions. + */ + void setShouldRerun() { + rerun = true; + } + + boolean shouldRerun() { + return rerun; + } + + /** + * Fix inconsistencies found by fsck. This should try to fix errors (if any) + * found by fsck utility. + */ + public void setFixAssignments(boolean shouldFix) { + fixAssignments = shouldFix; + } + + boolean shouldFixAssignments() { + return fixAssignments; + } + + public void setFixMeta(boolean shouldFix) { + fixMeta = shouldFix; + } + + boolean shouldFixMeta() { + return fixMeta; + } + + public void setCheckHdfs(boolean checking) { + checkHdfs = checking; + } + + boolean shouldCheckHdfs() { + return checkHdfs; + } + + public void setFixHdfsHoles(boolean shouldFix) { + fixHdfsHoles = shouldFix; + } + + boolean shouldFixHdfsHoles() { + return fixHdfsHoles; + } + + public void setFixTableOrphans(boolean shouldFix) { + fixTableOrphans = shouldFix; + } + + boolean shouldFixTableOrphans() { + return fixTableOrphans; + } + + public void setFixHdfsOverlaps(boolean shouldFix) { + fixHdfsOverlaps = shouldFix; + } + + boolean shouldFixHdfsOverlaps() { + return fixHdfsOverlaps; + } + + public void setFixHdfsOrphans(boolean shouldFix) { + fixHdfsOrphans = shouldFix; + } + + boolean shouldFixHdfsOrphans() { + return fixHdfsOrphans; + } + + public void setFixVersionFile(boolean shouldFix) { + fixVersionFile = shouldFix; + } + + public boolean shouldFixVersionFile() { + return fixVersionFile; + } + + public void setSidelineBigOverlaps(boolean sbo) { + this.sidelineBigOverlaps = sbo; + } + + public boolean shouldSidelineBigOverlaps() { + return sidelineBigOverlaps; + } + + public void setFixSplitParents(boolean shouldFix) { + fixSplitParents = shouldFix; + } + + boolean shouldFixSplitParents() { + return fixSplitParents; + } + + public void setFixReferenceFiles(boolean shouldFix) { + fixReferenceFiles = shouldFix; + } + + boolean shouldFixReferenceFiles() { + return fixReferenceFiles; + } + + public boolean shouldIgnorePreCheckPermission() { + return ignorePreCheckPermission; + } + + public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) { + this.ignorePreCheckPermission = ignorePreCheckPermission; + } + + /** + * @param mm maximum number of regions to merge into a single region. + */ + public void setMaxMerge(int mm) { + this.maxMerge = mm; + } + + public int getMaxMerge() { + return maxMerge; + } + + public void setMaxOverlapsToSideline(int mo) { + this.maxOverlapsToSideline = mo; + } + + public int getMaxOverlapsToSideline() { + return maxOverlapsToSideline; + } + + /** + * Only check/fix tables specified by the list, + * Empty list means all tables are included. + */ + boolean isTableIncluded(String table) { + return (tablesIncluded.size() == 0) || tablesIncluded.contains(table); + } + + public void includeTable(String table) { + tablesIncluded.add(table); + } + + Set getIncludedTables() { + return new HashSet(tablesIncluded); + } + + /** + * We are interested in only those tables that have not changed their state in + * META during the last few seconds specified by hbase.admin.fsck.timelag + * @param seconds - the time in seconds + */ + public void setTimeLag(long seconds) { + timelag = seconds * 1000; // convert to milliseconds + } + + /** + * + * @param sidelineDir - HDFS path to sideline data + */ + public void setSidelineDir(String sidelineDir) { + this.sidelineDir = new Path(sidelineDir); + } + + protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException { + return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles); + } + + public HFileCorruptionChecker getHFilecorruptionChecker() { + return hfcc; + } + + public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) { + this.hfcc = hfcc; + } + + public void setRetCode(int code) { + this.retcode = code; + } + + public int getRetCode() { + return retcode; + } + + protected HBaseFsck printUsageAndExit() { + StringWriter sw = new StringWriter(2048); + PrintWriter out = new PrintWriter(sw); + out.println("Usage: fsck [opts] {only tables}"); + out.println(" where [opts] are:"); + out.println(" -help Display help options (this)"); + out.println(" -details Display full report of all regions."); + out.println(" -timelag Process only regions that " + + " have not experienced any metadata updates in the last " + + " seconds."); + out.println(" -sleepBeforeRerun Sleep this many seconds" + + " before checking if the fix worked if run with -fix"); + out.println(" -summary Print only summary of the tables and status."); + out.println(" -metaonly Only check the state of ROOT and META tables."); + out.println(" -sidelineDir HDFS path to backup existing meta and root."); + + out.println(""); + out.println(" Metadata Repair options: (expert features, use with caution!)"); + out.println(" -fix Try to fix region assignments. This is for backwards compatiblity"); + out.println(" -fixAssignments Try to fix region assignments. Replaces the old -fix"); + out.println(" -fixMeta Try to fix meta problems. This assumes HDFS region info is good."); + out.println(" -noHdfsChecking Don't load/check region info from HDFS." + + " Assumes META region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap"); + out.println(" -fixHdfsHoles Try to fix region holes in hdfs."); + out.println(" -fixHdfsOrphans Try to fix region dirs with no .regioninfo file in hdfs"); + out.println(" -fixTableOrphans Try to fix table dirs with no .tableinfo file in hdfs (online mode only)"); + out.println(" -fixHdfsOverlaps Try to fix region overlaps in hdfs."); + out.println(" -fixVersionFile Try to fix missing hbase.version file in hdfs."); + out.println(" -maxMerge When fixing region overlaps, allow at most regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)"); + out.println(" -sidelineBigOverlaps When fixing region overlaps, allow to sideline big overlaps"); + out.println(" -maxOverlapsToSideline When fixing region overlaps, allow at most regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)"); + out.println(" -fixSplitParents Try to force offline split parents to be online."); + out.println(" -ignorePreCheckPermission ignore filesystem permission pre-check"); + out.println(" -fixReferenceFiles Try to offline lingering reference store files"); + + out.println(""); + out.println(" Datafile Repair options: (expert features, use with caution!)"); + out.println(" -checkCorruptHFiles Check all Hfiles by opening them to make sure they are valid"); + out.println(" -sidelineCorruptHfiles Quarantine corrupted HFiles. implies -checkCorruptHfiles"); + + out.println(""); + out.println(" Metadata Repair shortcuts"); + out.println(" -repair Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " + + "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles"); + out.println(" -repairHoles Shortcut for -fixAssignments -fixMeta -fixHdfsHoles"); + + out.flush(); + errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString()); + + setRetCode(-2); + return this; + } + + /** + * Main program + * + * @param args + * @throws Exception + */ + public static void main(String[] args) throws Exception { + // create a fsck object + Configuration conf = HBaseConfiguration.create(); + Path hbasedir = new Path(conf.get(HConstants.HBASE_DIR)); + URI defaultFs = hbasedir.getFileSystem(conf).getUri(); + conf.set("fs.defaultFS", defaultFs.toString()); // for hadoop 0.21+ + conf.set("fs.default.name", defaultFs.toString()); // for hadoop 0.20 + + int ret = ToolRunner.run(new HBaseFsck(conf), args); + System.exit(ret); + } + + @Override + public int run(String[] args) throws Exception { + exec(executor, args); + return getRetCode(); + } + + public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException, + InterruptedException { + long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN; + + boolean checkCorruptHFiles = false; + boolean sidelineCorruptHFiles = false; + + // Process command-line args. + for (int i = 0; i < args.length; i++) { + String cmd = args[i]; + if (cmd.equals("-help") || cmd.equals("-h")) { + return printUsageAndExit(); + } else if (cmd.equals("-details")) { + setDisplayFullReport(); + } else if (cmd.equals("-timelag")) { + if (i == args.length - 1) { + errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value."); + return printUsageAndExit(); + } + try { + long timelag = Long.parseLong(args[i+1]); + setTimeLag(timelag); + } catch (NumberFormatException e) { + errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value."); + return printUsageAndExit(); + } + i++; + } else if (cmd.equals("-sleepBeforeRerun")) { + if (i == args.length - 1) { + errors.reportError(ERROR_CODE.WRONG_USAGE, + "HBaseFsck: -sleepBeforeRerun needs a value."); + return printUsageAndExit(); + } + try { + sleepBeforeRerun = Long.parseLong(args[i+1]); + } catch (NumberFormatException e) { + errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value."); + return printUsageAndExit(); + } + i++; + } else if (cmd.equals("-sidelineDir")) { + if (i == args.length - 1) { + errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value."); + return printUsageAndExit(); + } + i++; + setSidelineDir(args[i]); + } else if (cmd.equals("-fix")) { + errors.reportError(ERROR_CODE.WRONG_USAGE, + "This option is deprecated, please use -fixAssignments instead."); + setFixAssignments(true); + } else if (cmd.equals("-fixAssignments")) { + setFixAssignments(true); + } else if (cmd.equals("-fixMeta")) { + setFixMeta(true); + } else if (cmd.equals("-noHdfsChecking")) { + setCheckHdfs(false); + } else if (cmd.equals("-fixHdfsHoles")) { + setFixHdfsHoles(true); + } else if (cmd.equals("-fixHdfsOrphans")) { + setFixHdfsOrphans(true); + } else if (cmd.equals("-fixTableOrphans")) { + setFixTableOrphans(true); + } else if (cmd.equals("-fixHdfsOverlaps")) { + setFixHdfsOverlaps(true); + } else if (cmd.equals("-fixVersionFile")) { + setFixVersionFile(true); + } else if (cmd.equals("-sidelineBigOverlaps")) { + setSidelineBigOverlaps(true); + } else if (cmd.equals("-fixSplitParents")) { + setFixSplitParents(true); + } else if (cmd.equals("-ignorePreCheckPermission")) { + setIgnorePreCheckPermission(true); + } else if (cmd.equals("-checkCorruptHFiles")) { + checkCorruptHFiles = true; + } else if (cmd.equals("-sidelineCorruptHFiles")) { + sidelineCorruptHFiles = true; + } else if (cmd.equals("-fixReferenceFiles")) { + setFixReferenceFiles(true); + } else if (cmd.equals("-repair")) { + // this attempts to merge overlapping hdfs regions, needs testing + // under load + setFixHdfsHoles(true); + setFixHdfsOrphans(true); + setFixMeta(true); + setFixAssignments(true); + setFixHdfsOverlaps(true); + setFixVersionFile(true); + setSidelineBigOverlaps(true); + setFixSplitParents(false); + setCheckHdfs(true); + setFixReferenceFiles(true); + } else if (cmd.equals("-repairHoles")) { + // this will make all missing hdfs regions available but may lose data + setFixHdfsHoles(true); + setFixHdfsOrphans(false); + setFixMeta(true); + setFixAssignments(true); + setFixHdfsOverlaps(false); + setSidelineBigOverlaps(false); + setFixSplitParents(false); + setCheckHdfs(true); + } else if (cmd.equals("-maxOverlapsToSideline")) { + if (i == args.length - 1) { + errors.reportError(ERROR_CODE.WRONG_USAGE, + "-maxOverlapsToSideline needs a numeric value argument."); + return printUsageAndExit(); + } + try { + int maxOverlapsToSideline = Integer.parseInt(args[i+1]); + setMaxOverlapsToSideline(maxOverlapsToSideline); + } catch (NumberFormatException e) { + errors.reportError(ERROR_CODE.WRONG_USAGE, + "-maxOverlapsToSideline needs a numeric value argument."); + return printUsageAndExit(); + } + i++; + } else if (cmd.equals("-maxMerge")) { + if (i == args.length - 1) { + errors.reportError(ERROR_CODE.WRONG_USAGE, + "-maxMerge needs a numeric value argument."); + return printUsageAndExit(); + } + try { + int maxMerge = Integer.parseInt(args[i+1]); + setMaxMerge(maxMerge); + } catch (NumberFormatException e) { + errors.reportError(ERROR_CODE.WRONG_USAGE, + "-maxMerge needs a numeric value argument."); + return printUsageAndExit(); + } + i++; + } else if (cmd.equals("-summary")) { + setSummary(); + } else if (cmd.equals("-metaonly")) { + setCheckMetaOnly(); + } else if (cmd.startsWith("-")) { + errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd); + return printUsageAndExit(); + } else { + includeTable(cmd); + errors.print("Allow checking/fixes for table: " + cmd); + } + } + + // pre-check current user has FS write permission or not + try { + preCheckPermission(); + } catch (AccessControlException ace) { + Runtime.getRuntime().exit(-1); + } catch (IOException ioe) { + Runtime.getRuntime().exit(-1); + } + + // do the real work of hbck + connect(); + + // if corrupt file mode is on, first fix them since they may be opened later + if (checkCorruptHFiles || sidelineCorruptHFiles) { + LOG.info("Checking all hfiles for corruption"); + HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles); + setHFileCorruptionChecker(hfcc); // so we can get result + Collection tables = getIncludedTables(); + Collection tableDirs = new ArrayList(); + Path rootdir = FSUtils.getRootDir(getConf()); + if (tables.size() > 0) { + for (String t : tables) { + tableDirs.add(FSUtils.getTablePath(rootdir, t)); + } + } else { + tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir); + } + hfcc.checkTables(tableDirs); + hfcc.report(errors); + } + + // check and fix table integrity, region consistency. + int code = onlineHbck(); + setRetCode(code); + // If we have changed the HBase state it is better to run hbck again + // to see if we haven't broken something else in the process. + // We run it only once more because otherwise we can easily fall into + // an infinite loop. + if (shouldRerun()) { + try { + LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix..."); + Thread.sleep(sleepBeforeRerun); + } catch (InterruptedException ie) { + return this; + } + // Just report + setFixAssignments(false); + setFixMeta(false); + setFixHdfsHoles(false); + setFixHdfsOverlaps(false); + setFixVersionFile(false); + setFixTableOrphans(false); + errors.resetErrors(); + code = onlineHbck(); + setRetCode(code); + } + return this; + } + + /** + * ls -r for debugging purposes + */ + void debugLsr(Path p) throws IOException { + debugLsr(getConf(), p, errors); + } + + /** + * ls -r for debugging purposes + */ + public static void debugLsr(Configuration conf, + Path p) throws IOException { + debugLsr(conf, p, new PrintingErrorReporter()); + } + + /** + * ls -r for debugging purposes + */ + public static void debugLsr(Configuration conf, + Path p, ErrorReporter errors) throws IOException { + if (!LOG.isDebugEnabled() || p == null) { + return; + } + FileSystem fs = p.getFileSystem(conf); + + if (!fs.exists(p)) { + // nothing + return; + } + errors.print(p.toString()); + + if (fs.isFile(p)) { + return; + } + + if (fs.getFileStatus(p).isDir()) { + FileStatus[] fss= fs.listStatus(p); + for (FileStatus status : fss) { + debugLsr(conf, status.getPath(), errors); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java new file mode 100644 index 0000000..81ad866 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java @@ -0,0 +1,196 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.zookeeper.KeeperException; + +/** + * This class contains helper methods that repair parts of hbase's filesystem + * contents. + */ +public class HBaseFsckRepair { + public static final Log LOG = LogFactory.getLog(HBaseFsckRepair.class); + + /** + * Fix multiple assignment by doing silent closes on each RS hosting the region + * and then force ZK unassigned node to OFFLINE to trigger assignment by + * master. + * + * @param admin HBase admin used to undeploy + * @param region Region to undeploy + * @param servers list of Servers to undeploy from + */ + public static void fixMultiAssignment(HBaseAdmin admin, HRegionInfo region, + List servers) + throws IOException, KeeperException, InterruptedException { + HRegionInfo actualRegion = new HRegionInfo(region); + + // Close region on the servers silently + for(ServerName server : servers) { + closeRegionSilentlyAndWait(admin, server, actualRegion); + } + + // Force ZK node to OFFLINE so master assigns + forceOfflineInZK(admin, actualRegion); + } + + /** + * Fix unassigned by creating/transition the unassigned ZK node for this + * region to OFFLINE state with a special flag to tell the master that this is + * a forced operation by HBCK. + * + * This assumes that info is in META. + * + * @param conf + * @param region + * @throws IOException + * @throws KeeperException + */ + public static void fixUnassigned(HBaseAdmin admin, HRegionInfo region) + throws IOException, KeeperException { + HRegionInfo actualRegion = new HRegionInfo(region); + + // Force ZK node to OFFLINE so master assigns + forceOfflineInZK(admin, actualRegion); + } + + /** + * In 0.90, this forces an HRI offline by setting the RegionTransitionData + * in ZK to have HBCK_CODE_NAME as the server. This is a special case in + * the AssignmentManager that attempts an assign call by the master. + * + * @see org.apache.hadoop.hbase.master.AssignementManager#handleHBCK + * + * This doesn't seem to work properly in the updated version of 0.92+'s hbck + * so we use assign to force the region into transition. This has the + * side-effect of requiring a HRegionInfo that considers regionId (timestamp) + * in comparators that is addressed by HBASE-5563. + */ + private static void forceOfflineInZK(HBaseAdmin admin, final HRegionInfo region) + throws ZooKeeperConnectionException, KeeperException, IOException { + admin.assign(region.getRegionName()); + } + + /* + * Should we check all assignments or just not in RIT? + */ + public static void waitUntilAssigned(HBaseAdmin admin, + HRegionInfo region) throws IOException, InterruptedException { + long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000); + long expiration = timeout + System.currentTimeMillis(); + while (System.currentTimeMillis() < expiration) { + try { + Map rits= + admin.getClusterStatus().getRegionsInTransition(); + + if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) { + // yay! no longer RIT + return; + } + // still in rit + LOG.info("Region still in transition, waiting for " + + "it to become assigned: " + region); + } catch (IOException e) { + LOG.warn("Exception when waiting for region to become assigned," + + " retrying", e); + } + Thread.sleep(1000); + } + throw new IOException("Region " + region + " failed to move out of " + + "transition within timeout " + timeout + "ms"); + } + + /** + * Contacts a region server and waits up to hbase.hbck.close.timeout ms + * (default 120s) to close the region. This bypasses the active hmaster. + */ + public static void closeRegionSilentlyAndWait(HBaseAdmin admin, + ServerName server, HRegionInfo region) throws IOException, InterruptedException { + HConnection connection = admin.getConnection(); + HRegionInterface rs = connection.getHRegionConnection(server.getHostname(), + server.getPort()); + rs.closeRegion(region, false); + long timeout = admin.getConfiguration() + .getLong("hbase.hbck.close.timeout", 120000); + long expiration = timeout + System.currentTimeMillis(); + while (System.currentTimeMillis() < expiration) { + try { + HRegionInfo rsRegion = rs.getRegionInfo(region.getRegionName()); + if (rsRegion == null) + return; + } catch (IOException ioe) { + return; + } + Thread.sleep(1000); + } + throw new IOException("Region " + region + " failed to close within" + + " timeout " + timeout); + } + + /** + * Puts the specified HRegionInfo into META. + */ + public static void fixMetaHoleOnline(Configuration conf, + HRegionInfo hri) throws IOException { + Put p = new Put(hri.getRegionName()); + p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + HTable meta = new HTable(conf, HConstants.META_TABLE_NAME); + meta.put(p); + meta.close(); + } + + /** + * Creates, flushes, and closes a new region. + */ + public static HRegion createHDFSRegionDir(Configuration conf, + HRegionInfo hri, HTableDescriptor htd) throws IOException { + // Create HRegion + Path root = FSUtils.getRootDir(conf); + HRegion region = HRegion.createHRegion(hri, root, conf, htd); + HLog hlog = region.getLog(); + + // Close the new region to flush to disk. Close log file too. + region.close(); + hlog.closeAndDelete(); + return region; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java b/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java new file mode 100644 index 0000000..92a2408 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java @@ -0,0 +1,182 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.Store; + +/** + * Helper class for all utilities related to archival/retrieval of HFiles + */ +public class HFileArchiveUtil { + + private HFileArchiveUtil() { + // non-external instantiation - util class + } + + /** + * Get the directory to archive a store directory + * @param conf {@link Configuration} to read for the archive directory name + * @param tableName table name under which the store currently lives + * @param regionName region encoded name under which the store currently lives + * @param family name of the family in the store + * @return {@link Path} to the directory to archive the given store or + * null if it should not be archived + */ + public static Path getStoreArchivePath(final Configuration conf, final String tableName, + final String regionName, final String familyName) throws IOException { + Path tableArchiveDir = getTableArchivePath(conf, tableName); + return Store.getStoreHomedir(tableArchiveDir, regionName, familyName); + } + + /** + * Get the directory to archive a store directory + * @param conf {@link Configuration} to read for the archive directory name + * @param region parent region information under which the store currently + * lives + * @param family name of the family in the store + * @return {@link Path} to the directory to archive the given store or + * null if it should not be archived + */ + public static Path getStoreArchivePath(Configuration conf, HRegion region, byte [] family){ + return getStoreArchivePath(conf, region.getRegionInfo(), region.getTableDir(), family); + } + + /** + * Get the directory to archive a store directory + * @param conf {@link Configuration} to read for the archive directory name. Can be null. + * @param region parent region information under which the store currently lives + * @param tabledir directory for the table under which the store currently lives + * @param family name of the family in the store + * @return {@link Path} to the directory to archive the given store or null if it should + * not be archived + */ + public static Path getStoreArchivePath(Configuration conf, HRegionInfo region, Path tabledir, + byte[] family) { + Path tableArchiveDir = getTableArchivePath(tabledir); + return Store.getStoreHomedir(tableArchiveDir, + HRegionInfo.encodeRegionName(region.getRegionName()), family); + } + + /** + * Get the archive directory for a given region under the specified table + * @param conf {@link Configuration} to read the archive directory from. Can be null + * @param tabledir the original table directory. Cannot be null. + * @param regiondir the path to the region directory. Cannot be null. + * @return {@link Path} to the directory to archive the given region, or null if it + * should not be archived + */ + public static Path getRegionArchiveDir(Configuration conf, Path tabledir, Path regiondir) { + // get the archive directory for a table + Path archiveDir = getTableArchivePath(tabledir); + + // then add on the region path under the archive + String encodedRegionName = regiondir.getName(); + return HRegion.getRegionDir(archiveDir, encodedRegionName); + } + + /** + * Get the archive directory for a given region under the specified table + * @param rootdir {@link Path} to the root directory where hbase files are stored (for building + * the archive path) + * @param tabledir the original table directory. Cannot be null. + * @param regiondir the path to the region directory. Cannot be null. + * @return {@link Path} to the directory to archive the given region, or null if it + * should not be archived + */ + public static Path getRegionArchiveDir(Path rootdir, Path tabledir, Path regiondir) { + // get the archive directory for a table + Path archiveDir = getTableArchivePath(rootdir, tabledir.getName()); + + // then add on the region path under the archive + String encodedRegionName = regiondir.getName(); + return HRegion.getRegionDir(archiveDir, encodedRegionName); + } + + /** + * Get the path to the table archive directory based on the configured archive directory. + *

    + * Get the path to the table's archive directory. + *

    + * Generally of the form: /hbase/.archive/[tablename] + * @param tabledir directory of the table to be archived. Cannot be null. + * @return {@link Path} to the archive directory for the table + */ + public static Path getTableArchivePath(Path tabledir) { + Path root = tabledir.getParent(); + return getTableArchivePath(root, tabledir.getName()); + } + + /** + * Get the path to the table archive directory based on the configured archive directory. + *

    + * Get the path to the table's archive directory. + *

    + * Generally of the form: /hbase/.archive/[tablename] + * @param rootdir {@link Path} to the root directory where hbase files are stored (for building + * the archive path) + * @param tableName Name of the table to be archived. Cannot be null. + * @return {@link Path} to the archive directory for the table + */ + public static Path getTableArchivePath(final Path rootdir, final String tableName) { + return new Path(getArchivePath(rootdir), tableName); + } + + /** + * Get the path to the table archive directory based on the configured archive directory. + *

    + * Assumed that the table should already be archived. + * @param conf {@link Configuration} to read the archive directory property. Can be null + * @param tableName Name of the table to be archived. Cannot be null. + * @return {@link Path} to the archive directory for the table + */ + public static Path getTableArchivePath(final Configuration conf, final String tableName) + throws IOException { + return new Path(getArchivePath(conf), tableName); + } + + /** + * Get the full path to the archive directory on the configured {@link FileSystem} + * @param conf to look for archive directory name and root directory. Cannot be null. Notes for + * testing: requires a FileSystem root directory to be specified. + * @return the full {@link Path} to the archive directory, as defined by the configuration + * @throws IOException if an unexpected error occurs + */ + public static Path getArchivePath(Configuration conf) throws IOException { + return getArchivePath(FSUtils.getRootDir(conf)); + } + + /** + * Get the full path to the archive directory on the configured {@link FileSystem} + * @param rootdir {@link Path} to the root directory where hbase files are stored (for building + * the archive path) + * @return the full {@link Path} to the archive directory, as defined by the configuration + */ + private static Path getArchivePath(final Path rootdir) { + return new Path(rootdir, HConstants.HFILE_ARCHIVE_DIRECTORY); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/HMerge.java b/src/main/java/org/apache/hadoop/hbase/util/HMerge.java new file mode 100644 index 0000000..a685aa7 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/HMerge.java @@ -0,0 +1,438 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.RemoteExceptionHandler; +import org.apache.hadoop.hbase.TableNotDisabledException; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.wal.HLog; + +/** + * A non-instantiable class that has a static method capable of compacting + * a table by merging adjacent regions. + */ +class HMerge { + // TODO: Where is this class used? How does it relate to Merge in same package? + static final Log LOG = LogFactory.getLog(HMerge.class); + static final Random rand = new Random(); + + /* + * Not instantiable + */ + private HMerge() { + super(); + } + + /** + * Scans the table and merges two adjacent regions if they are small. This + * only happens when a lot of rows are deleted. + * + * When merging the META region, the HBase instance must be offline. + * When merging a normal table, the HBase instance must be online, but the + * table must be disabled. + * + * @param conf - configuration object for HBase + * @param fs - FileSystem where regions reside + * @param tableName - Table to be compacted + * @throws IOException + */ + public static void merge(Configuration conf, FileSystem fs, + final byte [] tableName) + throws IOException { + merge(conf, fs, tableName, true); + } + + /** + * Scans the table and merges two adjacent regions if they are small. This + * only happens when a lot of rows are deleted. + * + * When merging the META region, the HBase instance must be offline. + * When merging a normal table, the HBase instance must be online, but the + * table must be disabled. + * + * @param conf - configuration object for HBase + * @param fs - FileSystem where regions reside + * @param tableName - Table to be compacted + * @param testMasterRunning True if we are to verify master is down before + * running merge + * @throws IOException + */ + public static void merge(Configuration conf, FileSystem fs, + final byte [] tableName, final boolean testMasterRunning) + throws IOException { + boolean masterIsRunning = false; + if (testMasterRunning) { + masterIsRunning = HConnectionManager + .execute(new HConnectable(conf) { + @Override + public Boolean connect(HConnection connection) throws IOException { + return connection.isMasterRunning(); + } + }); + } + if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) { + if (masterIsRunning) { + throw new IllegalStateException( + "Can not compact META table if instance is on-line"); + } + new OfflineMerger(conf, fs).process(); + } else { + if(!masterIsRunning) { + throw new IllegalStateException( + "HBase instance must be running to merge a normal table"); + } + HBaseAdmin admin = new HBaseAdmin(conf); + if (!admin.isTableDisabled(tableName)) { + throw new TableNotDisabledException(tableName); + } + new OnlineMerger(conf, fs, tableName).process(); + } + } + + private static abstract class Merger { + protected final Configuration conf; + protected final FileSystem fs; + protected final Path tabledir; + protected final HTableDescriptor htd; + protected final HLog hlog; + private final long maxFilesize; + + + protected Merger(Configuration conf, FileSystem fs, final byte [] tableName) + throws IOException { + this.conf = conf; + this.fs = fs; + this.maxFilesize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, + HConstants.DEFAULT_MAX_FILE_SIZE); + + this.tabledir = new Path( + fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR))), + Bytes.toString(tableName) + ); + this.htd = FSTableDescriptors.getTableDescriptor(this.fs, this.tabledir); + Path logdir = new Path(tabledir, "merge_" + System.currentTimeMillis() + + HConstants.HREGION_LOGDIR_NAME); + Path oldLogDir = new Path(tabledir, HConstants.HREGION_OLDLOGDIR_NAME); + this.hlog = new HLog(fs, logdir, oldLogDir, conf); + } + + void process() throws IOException { + try { + for (HRegionInfo[] regionsToMerge = next(); + regionsToMerge != null; + regionsToMerge = next()) { + if (!merge(regionsToMerge)) { + return; + } + } + } finally { + try { + hlog.closeAndDelete(); + + } catch(IOException e) { + LOG.error(e); + } + } + } + + protected boolean merge(final HRegionInfo[] info) throws IOException { + if (info.length < 2) { + LOG.info("only one region - nothing to merge"); + return false; + } + + HRegion currentRegion = null; + long currentSize = 0; + HRegion nextRegion = null; + long nextSize = 0; + for (int i = 0; i < info.length - 1; i++) { + if (currentRegion == null) { + currentRegion = HRegion.newHRegion(tabledir, hlog, fs, conf, info[i], + this.htd, null); + currentRegion.initialize(); + currentSize = currentRegion.getLargestHStoreSize(); + } + nextRegion = HRegion.newHRegion(tabledir, hlog, fs, conf, info[i + 1], + this.htd, null); + nextRegion.initialize(); + nextSize = nextRegion.getLargestHStoreSize(); + + if ((currentSize + nextSize) <= (maxFilesize / 2)) { + // We merge two adjacent regions if their total size is less than + // one half of the desired maximum size + LOG.info("Merging regions " + currentRegion.getRegionNameAsString() + + " and " + nextRegion.getRegionNameAsString()); + HRegion mergedRegion = + HRegion.mergeAdjacent(currentRegion, nextRegion); + updateMeta(currentRegion.getRegionName(), nextRegion.getRegionName(), + mergedRegion); + break; + } + LOG.info("not merging regions " + Bytes.toStringBinary(currentRegion.getRegionName()) + + " and " + Bytes.toStringBinary(nextRegion.getRegionName())); + currentRegion.close(); + currentRegion = nextRegion; + currentSize = nextSize; + } + if(currentRegion != null) { + currentRegion.close(); + } + return true; + } + + protected abstract HRegionInfo[] next() throws IOException; + + protected abstract void updateMeta(final byte [] oldRegion1, + final byte [] oldRegion2, HRegion newRegion) + throws IOException; + + } + + /** Instantiated to compact a normal user table */ + private static class OnlineMerger extends Merger { + private final byte [] tableName; + private final HTable table; + private final ResultScanner metaScanner; + private HRegionInfo latestRegion; + + OnlineMerger(Configuration conf, FileSystem fs, + final byte [] tableName) + throws IOException { + super(conf, fs, tableName); + this.tableName = tableName; + this.table = new HTable(conf, HConstants.META_TABLE_NAME); + this.metaScanner = table.getScanner(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + this.latestRegion = null; + } + + private HRegionInfo nextRegion() throws IOException { + try { + Result results = getMetaRow(); + if (results == null) { + return null; + } + byte[] regionInfoValue = results.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + if (regionInfoValue == null || regionInfoValue.length == 0) { + throw new NoSuchElementException("meta region entry missing " + + Bytes.toString(HConstants.CATALOG_FAMILY) + ":" + + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)); + } + HRegionInfo region = Writables.getHRegionInfo(regionInfoValue); + if (!Bytes.equals(region.getTableName(), this.tableName)) { + return null; + } + return region; + } catch (IOException e) { + e = RemoteExceptionHandler.checkIOException(e); + LOG.error("meta scanner error", e); + metaScanner.close(); + throw e; + } + } + + /* + * Check current row has a HRegionInfo. Skip to next row if HRI is empty. + * @return A Map of the row content else null if we are off the end. + * @throws IOException + */ + private Result getMetaRow() throws IOException { + Result currentRow = metaScanner.next(); + boolean foundResult = false; + while (currentRow != null) { + LOG.info("Row: <" + Bytes.toStringBinary(currentRow.getRow()) + ">"); + byte[] regionInfoValue = currentRow.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + if (regionInfoValue == null || regionInfoValue.length == 0) { + currentRow = metaScanner.next(); + continue; + } + foundResult = true; + break; + } + return foundResult ? currentRow : null; + } + + @Override + protected HRegionInfo[] next() throws IOException { + List regions = new ArrayList(); + if(latestRegion == null) { + latestRegion = nextRegion(); + } + if(latestRegion != null) { + regions.add(latestRegion); + } + latestRegion = nextRegion(); + if(latestRegion != null) { + regions.add(latestRegion); + } + return regions.toArray(new HRegionInfo[regions.size()]); + } + + @Override + protected void updateMeta(final byte [] oldRegion1, + final byte [] oldRegion2, + HRegion newRegion) + throws IOException { + byte[][] regionsToDelete = {oldRegion1, oldRegion2}; + for (int r = 0; r < regionsToDelete.length; r++) { + if(Bytes.equals(regionsToDelete[r], latestRegion.getRegionName())) { + latestRegion = null; + } + Delete delete = new Delete(regionsToDelete[r]); + table.delete(delete); + if(LOG.isDebugEnabled()) { + LOG.debug("updated columns in row: " + Bytes.toStringBinary(regionsToDelete[r])); + } + } + newRegion.getRegionInfo().setOffline(true); + + Put put = new Put(newRegion.getRegionName()); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(newRegion.getRegionInfo())); + table.put(put); + + if(LOG.isDebugEnabled()) { + LOG.debug("updated columns in row: " + + Bytes.toStringBinary(newRegion.getRegionName())); + } + } + } + + /** Instantiated to compact the meta region */ + private static class OfflineMerger extends Merger { + private final List metaRegions = new ArrayList(); + private final HRegion root; + + OfflineMerger(Configuration conf, FileSystem fs) + throws IOException { + super(conf, fs, HConstants.META_TABLE_NAME); + + Path rootTableDir = HTableDescriptor.getTableDir( + fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR))), + HConstants.ROOT_TABLE_NAME); + + // Scan root region to find all the meta regions + + root = HRegion.newHRegion(rootTableDir, hlog, fs, conf, + HRegionInfo.ROOT_REGIONINFO, HTableDescriptor.ROOT_TABLEDESC, null); + root.initialize(); + + Scan scan = new Scan(); + scan.addColumn(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + InternalScanner rootScanner = + root.getScanner(scan); + + try { + List results = new ArrayList(); + boolean hasMore; + do { + hasMore = rootScanner.next(results); + for(KeyValue kv: results) { + HRegionInfo info = Writables.getHRegionInfoOrNull(kv.getValue()); + if (info != null) { + metaRegions.add(info); + } + } + } while (hasMore); + } finally { + rootScanner.close(); + try { + root.close(); + + } catch(IOException e) { + LOG.error(e); + } + } + } + + @Override + protected HRegionInfo[] next() { + HRegionInfo[] results = null; + if (metaRegions.size() > 0) { + results = metaRegions.toArray(new HRegionInfo[metaRegions.size()]); + metaRegions.clear(); + } + return results; + } + + @Override + protected void updateMeta(final byte [] oldRegion1, + final byte [] oldRegion2, HRegion newRegion) + throws IOException { + byte[][] regionsToDelete = {oldRegion1, oldRegion2}; + for(int r = 0; r < regionsToDelete.length; r++) { + Delete delete = new Delete(regionsToDelete[r]); + delete.deleteColumns(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + delete.deleteColumns(HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER); + delete.deleteColumns(HConstants.CATALOG_FAMILY, + HConstants.STARTCODE_QUALIFIER); + delete.deleteColumns(HConstants.CATALOG_FAMILY, + HConstants.SPLITA_QUALIFIER); + delete.deleteColumns(HConstants.CATALOG_FAMILY, + HConstants.SPLITB_QUALIFIER); + root.delete(delete, null, true); + + if(LOG.isDebugEnabled()) { + LOG.debug("updated columns in row: " + Bytes.toStringBinary(regionsToDelete[r])); + } + } + HRegionInfo newInfo = newRegion.getRegionInfo(); + newInfo.setOffline(true); + Put put = new Put(newRegion.getRegionName()); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(newInfo)); + root.put(put); + if(LOG.isDebugEnabled()) { + LOG.debug("updated columns in row: " + Bytes.toStringBinary(newRegion.getRegionName())); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/HasThread.java b/src/main/java/org/apache/hadoop/hbase/util/HasThread.java new file mode 100644 index 0000000..076604f --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/HasThread.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.lang.Thread.UncaughtExceptionHandler; + +/** + * Abstract class which contains a Thread and delegates the common Thread + * methods to that instance. + * + * The purpose of this class is to workaround Sun JVM bug #6915621, in which + * something internal to the JDK uses Thread.currentThread() as a monitor + * lock. This can produce deadlocks like HBASE-4367, HBASE-4101, etc. + */ +public abstract class HasThread implements Runnable { + private final Thread thread; + + public HasThread() { + this.thread = new Thread(this); + } + + public HasThread(String name) { + this.thread = new Thread(this, name); + } + + public Thread getThread() { + return thread; + } + + public abstract void run(); + + //// Begin delegation to Thread + + public final String getName() { + return thread.getName(); + } + + public void interrupt() { + thread.interrupt(); + } + + public final boolean isAlive() { + return thread.isAlive(); + } + + public boolean isInterrupted() { + return thread.isInterrupted(); + } + + public final void setDaemon(boolean on) { + thread.setDaemon(on); + } + + public final void setName(String name) { + thread.setName(name); + } + + public final void setPriority(int newPriority) { + thread.setPriority(newPriority); + } + + public void setUncaughtExceptionHandler(UncaughtExceptionHandler eh) { + thread.setUncaughtExceptionHandler(eh); + } + + public void start() { + thread.start(); + } + + public final void join() throws InterruptedException { + thread.join(); + } + + public final void join(long millis, int nanos) throws InterruptedException { + thread.join(millis, nanos); + } + + public final void join(long millis) throws InterruptedException { + thread.join(millis); + } + //// End delegation to Thread +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/Hash.java b/src/main/java/org/apache/hadoop/hbase/util/Hash.java new file mode 100644 index 0000000..8a3dcf5 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Hash.java @@ -0,0 +1,134 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.conf.Configuration; + +/** + * This class represents a common API for hashing functions. + */ +public abstract class Hash { + /** Constant to denote invalid hash type. */ + public static final int INVALID_HASH = -1; + /** Constant to denote {@link JenkinsHash}. */ + public static final int JENKINS_HASH = 0; + /** Constant to denote {@link MurmurHash}. */ + public static final int MURMUR_HASH = 1; + + /** + * This utility method converts String representation of hash function name + * to a symbolic constant. Currently two function types are supported, + * "jenkins" and "murmur". + * @param name hash function name + * @return one of the predefined constants + */ + public static int parseHashType(String name) { + if ("jenkins".equalsIgnoreCase(name)) { + return JENKINS_HASH; + } else if ("murmur".equalsIgnoreCase(name)) { + return MURMUR_HASH; + } else { + return INVALID_HASH; + } + } + + /** + * This utility method converts the name of the configured + * hash type to a symbolic constant. + * @param conf configuration + * @return one of the predefined constants + */ + public static int getHashType(Configuration conf) { + String name = conf.get("hbase.hash.type", "murmur"); + return parseHashType(name); + } + + /** + * Get a singleton instance of hash function of a given type. + * @param type predefined hash type + * @return hash function instance, or null if type is invalid + */ + public static Hash getInstance(int type) { + switch(type) { + case JENKINS_HASH: + return JenkinsHash.getInstance(); + case MURMUR_HASH: + return MurmurHash.getInstance(); + default: + return null; + } + } + + /** + * Get a singleton instance of hash function of a type + * defined in the configuration. + * @param conf current configuration + * @return defined hash type, or null if type is invalid + */ + public static Hash getInstance(Configuration conf) { + int type = getHashType(conf); + return getInstance(type); + } + + /** + * Calculate a hash using all bytes from the input argument, and + * a seed of -1. + * @param bytes input bytes + * @return hash value + */ + public int hash(byte[] bytes) { + return hash(bytes, bytes.length, -1); + } + + /** + * Calculate a hash using all bytes from the input argument, + * and a provided seed value. + * @param bytes input bytes + * @param initval seed value + * @return hash value + */ + public int hash(byte[] bytes, int initval) { + return hash(bytes, 0, bytes.length, initval); + } + + /** + * Calculate a hash using bytes from 0 to length, and + * the provided seed value + * @param bytes input bytes + * @param length length of the valid bytes after offset to consider + * @param initval seed value + * @return hash value + */ + public int hash(byte[] bytes, int length, int initval) { + return hash(bytes, 0, length, initval); + } + + /** + * Calculate a hash using bytes from offset to offset + + * length, and the provided seed value. + * @param bytes input bytes + * @param offset the offset into the array to start consideration + * @param length length of the valid bytes after offset to consider + * @param initval seed value + * @return hash value + */ + public abstract int hash(byte[] bytes, int offset, int length, int initval); +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/HashedBytes.java b/src/main/java/org/apache/hadoop/hbase/util/HashedBytes.java new file mode 100644 index 0000000..e6471cb --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/HashedBytes.java @@ -0,0 +1,61 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.Arrays; + +/** + * This class encapsulates a byte array and overrides hashCode and equals so + * that it's identity is based on the data rather than the array instance. + */ +public class HashedBytes { + + private final byte[] bytes; + private final int hashCode; + + public HashedBytes(byte[] bytes) { + this.bytes = bytes; + hashCode = Bytes.hashCode(bytes); + } + + public byte[] getBytes() { + return bytes; + } + + @Override + public int hashCode() { + return hashCode; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; + HashedBytes other = (HashedBytes) obj; + return Arrays.equals(bytes, other.bytes); + } + + @Override + public String toString() { + return Bytes.toStringBinary(bytes); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/util/IdLock.java b/src/main/java/org/apache/hadoop/hbase/util/IdLock.java new file mode 100644 index 0000000..e9202dd --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/IdLock.java @@ -0,0 +1,120 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Allows multiple concurrent clients to lock on a numeric id with a minimal + * memory overhead. The intended usage is as follows: + * + * 

    + * IdLock.Entry lockEntry = idLock.getLockEntry(id);
+ * try {
+ *   // User code.
+ * } finally {
+ *   idLock.releaseLockEntry(lockEntry);
+ * }
    + */ +public class IdLock { + + /** An entry returned to the client as a lock object */ + public static class Entry { + private final long id; + private int numWaiters; + private boolean isLocked = true; + + private Entry(long id) { + this.id = id; + } + + public String toString() { + return "id=" + id + ", numWaiter=" + numWaiters + ", isLocked=" + + isLocked; + } + } + + private ConcurrentMap map = + new ConcurrentHashMap(); + + /** + * Blocks until the lock corresponding to the given id is acquired. + * + * @param id an arbitrary number to lock on + * @return an "entry" to pass to {@link #releaseLockEntry(Entry)} to release + * the lock + * @throws IOException if interrupted + */ + public Entry getLockEntry(long id) throws IOException { + Entry entry = new Entry(id); + Entry existing; + while ((existing = map.putIfAbsent(entry.id, entry)) != null) { + synchronized (existing) { + if (existing.isLocked) { + ++existing.numWaiters; // Add ourselves to waiters. + while (existing.isLocked) { + try { + existing.wait(); + } catch (InterruptedException e) { + --existing.numWaiters; // Remove ourselves from waiters. + throw new InterruptedIOException( + "Interrupted waiting to acquire sparse lock"); + } + } + + --existing.numWaiters; // Remove ourselves from waiters. + existing.isLocked = true; + return existing; + } + // If the entry is not locked, it might already be deleted from the + // map, so we cannot return it. We need to get our entry into the map + // or get someone else's locked entry. + } + } + return entry; + } + + /** + * Must be called in a finally block to decrease the internal counter and + * remove the monitor object for the given id if the caller is the last + * client. + * + * @param entry the return value of {@link #getLockEntry(long)} + */ + public void releaseLockEntry(Entry entry) { + synchronized (entry) { + entry.isLocked = false; + if (entry.numWaiters > 0) { + entry.notify(); + } else { + map.remove(entry.id); + } + } + } + + /** For testing */ + void assertMapEmpty() { + assert map.size() == 0; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/IncrementingEnvironmentEdge.java b/src/main/java/org/apache/hadoop/hbase/util/IncrementingEnvironmentEdge.java new file mode 100644 index 0000000..e105b77 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/IncrementingEnvironmentEdge.java @@ -0,0 +1,39 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +/** + * Uses an incrementing algorithm instead of the default. + */ +public class IncrementingEnvironmentEdge implements EnvironmentEdge { + + private long timeIncrement = 1; + + /** + * {@inheritDoc} + *

    + * This method increments a known value for the current time each time this + * method is called. The first value is 1. + */ + @Override + public synchronized long currentTimeMillis() { + return timeIncrement++; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java b/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java new file mode 100644 index 0000000..a1698da --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/InfoServer.java @@ -0,0 +1,132 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.URL; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.http.HttpServer; +import org.mortbay.jetty.handler.ContextHandlerCollection; +import org.mortbay.jetty.servlet.Context; +import org.mortbay.jetty.servlet.DefaultServlet; + +/** + * Create a Jetty embedded server to answer http requests. The primary goal + * is to serve up status information for the server. + * There are three contexts: + * "/stacks/" -> points to stack trace + * "/static/" -> points to common static files (src/hbase-webapps/static) + * "/" -> the jsp server code from (src/hbase-webapps/) + */ +public class InfoServer extends HttpServer { + private final Configuration config; + + /** + * Create a status server on the given port. + * The jsp scripts are taken from src/hbase-webapps/name. + * @param name The name of the server + * @param bindAddress address to bind to + * @param port The port to use on the server + * @param findPort whether the server should start at the given port and + * increment by 1 until it finds a free port. + * @throws IOException e + */ + public InfoServer(String name, String bindAddress, int port, boolean findPort, + final Configuration c) + throws IOException { + super(name, bindAddress, port, findPort, c); + this.config = c; + fixupLogsServletLocation(); + } + + /** + * Fixup where the logs app points, make it point at hbase logs rather than + * hadoop logs. + */ + private void fixupLogsServletLocation() { + // Must be same as up in hadoop. + final String logsContextPath = "/logs"; + // Now, put my logs in place of hadoops... disable old one first. + Context oldLogsContext = null; + for (Map.Entry e : defaultContexts.entrySet()) { + if (e.getKey().getContextPath().equals(logsContextPath)) { + oldLogsContext = e.getKey(); + break; + } + } + if (oldLogsContext != null) { + this.defaultContexts.put(oldLogsContext, Boolean.FALSE); + } + // Now do my logs. + // Set up the context for "/logs/" if "hbase.log.dir" property is defined. + String logDir = System.getProperty("hbase.log.dir"); + if (logDir != null) { + // This is a little presumptious but seems to work. + Context logContext = + new Context((ContextHandlerCollection)this.webServer.getHandler(), + logsContextPath); + logContext.setResourceBase(logDir); + logContext.addServlet(DefaultServlet.class, "/"); + defaultContexts.put(logContext, true); + } + } + + /** + * Get the pathname to the webapps files. + * @param appName eg "secondary" or "datanode" + * @return the pathname as a URL + * @throws FileNotFoundException if 'webapps' directory cannot be found on CLASSPATH. + */ + protected String getWebAppsPath(String appName) throws FileNotFoundException { + // Copied from the super-class. + String resourceName = "hbase-webapps/" + appName; + URL url = getClass().getClassLoader().getResource(resourceName); + if (url == null) + throw new FileNotFoundException(resourceName + " not found in CLASSPATH"); + String urlString = url.toString(); + return urlString.substring(0, urlString.lastIndexOf('/')); + } + + /** + * Get the pathname to the path files. + * @return the pathname as a URL + */ + protected String getWebAppsPath() throws IOException { + // Hack: webapps is not a unique enough element to find in CLASSPATH + // We'll more than likely find the hadoop webapps dir. So, instead + // look for the 'master' webapp in the webapps subdir. That should + // get us the hbase context. Presumption is that place where the + // master webapp resides is where we want this InfoServer picking up + // web applications. + final String master = "master"; + String p = getWebAppsPath(master); + // Now strip master off the end if it is present + if(p.endsWith(master)) { + return p.substring(0, p.lastIndexOf(master)); + } + return p; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java b/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java new file mode 100644 index 0000000..b079f2e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java @@ -0,0 +1,267 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.ShutdownHook; + +/** + * Utility used running a cluster all in the one JVM. + */ +public class JVMClusterUtil { + private static final Log LOG = LogFactory.getLog(JVMClusterUtil.class); + + /** + * Datastructure to hold RegionServer Thread and RegionServer instance + */ + public static class RegionServerThread extends Thread { + private final HRegionServer regionServer; + + public RegionServerThread(final HRegionServer r, final int index) { + super(r, "RegionServer:" + index + ";" + r.getServerName()); + this.regionServer = r; + } + + /** @return the region server */ + public HRegionServer getRegionServer() { + return this.regionServer; + } + + /** + * Block until the region server has come online, indicating it is ready + * to be used. + */ + public void waitForServerOnline() { + // The server is marked online after the init method completes inside of + // the HRS#run method. HRS#init can fail for whatever region. In those + // cases, we'll jump out of the run without setting online flag. Check + // stopRequested so we don't wait here a flag that will never be flipped. + regionServer.waitForServerOnline(); + } + } + + /** + * Creates a {@link RegionServerThread}. + * Call 'start' on the returned thread to make it run. + * @param c Configuration to use. + * @param hrsc Class to create. + * @param index Used distinguishing the object returned. + * @throws IOException + * @return Region server added. + */ + public static JVMClusterUtil.RegionServerThread createRegionServerThread( + final Configuration c, final Class hrsc, + final int index) + throws IOException { + HRegionServer server; + try { + server = hrsc.getConstructor(Configuration.class).newInstance(c); + } catch (InvocationTargetException ite) { + Throwable target = ite.getTargetException(); + throw new RuntimeException("Failed construction of RegionServer: " + + hrsc.toString() + ((target.getCause() != null)? + target.getCause().getMessage(): ""), target); + } catch (Exception e) { + IOException ioe = new IOException(); + ioe.initCause(e); + throw ioe; + } + return new JVMClusterUtil.RegionServerThread(server, index); + } + + + /** + * Datastructure to hold Master Thread and Master instance + */ + public static class MasterThread extends Thread { + private final HMaster master; + + public MasterThread(final HMaster m, final int index) { + super(m, "Master:" + index + ";" + m.getServerName()); + this.master = m; + } + + /** @return the master */ + public HMaster getMaster() { + return this.master; + } + } + + /** + * Creates a {@link MasterThread}. + * Call 'start' on the returned thread to make it run. + * @param c Configuration to use. + * @param hmc Class to create. + * @param index Used distinguishing the object returned. + * @throws IOException + * @return Master added. + */ + public static JVMClusterUtil.MasterThread createMasterThread( + final Configuration c, final Class hmc, + final int index) + throws IOException { + HMaster server; + try { + server = hmc.getConstructor(Configuration.class).newInstance(c); + } catch (InvocationTargetException ite) { + Throwable target = ite.getTargetException(); + throw new RuntimeException("Failed construction of Master: " + + hmc.toString() + ((target.getCause() != null)? + target.getCause().getMessage(): ""), target); + } catch (Exception e) { + IOException ioe = new IOException(); + ioe.initCause(e); + throw ioe; + } + return new JVMClusterUtil.MasterThread(server, index); + } + + private static JVMClusterUtil.MasterThread findActiveMaster( + List masters) { + for (JVMClusterUtil.MasterThread t : masters) { + if (t.master.isActiveMaster()) { + return t; + } + } + + return null; + } + + /** + * Start the cluster. Waits until there is a primary master initialized + * and returns its address. + * @param masters + * @param regionservers + * @return Address to use contacting primary master. + */ + public static String startup(final List masters, + final List regionservers) throws IOException { + + if (masters == null || masters.isEmpty()) { + return null; + } + + for (JVMClusterUtil.MasterThread t : masters) { + t.start(); + } + + // Wait for an active master + // having an active master before starting the region threads allows + // then to succeed on their connection to master + long startTime = System.currentTimeMillis(); + while (findActiveMaster(masters) == null) { + try { + Thread.sleep(100); + } catch (InterruptedException ignored) { + } + if (System.currentTimeMillis() > startTime + 30000) { + throw new RuntimeException("Master not active after 30 seconds"); + } + } + + if (regionservers != null) { + for (JVMClusterUtil.RegionServerThread t: regionservers) { + HRegionServer hrs = t.getRegionServer(); + ShutdownHook.install(hrs.getConfiguration(), FileSystem.get(hrs + .getConfiguration()), hrs, t); + t.start(); + } + } + + // Wait for an active master to be initialized (implies being master) + // with this, when we return the cluster is complete + startTime = System.currentTimeMillis(); + while (true) { + JVMClusterUtil.MasterThread t = findActiveMaster(masters); + if (t != null && t.master.isInitialized()) { + return t.master.getServerName().toString(); + } + if (System.currentTimeMillis() > startTime + 200000) { + throw new RuntimeException("Master not initialized after 200 seconds"); + } + try { + Thread.sleep(100); + } catch (InterruptedException ignored) { + // Keep waiting + } + } + } + + /** + * @param masters + * @param regionservers + */ + public static void shutdown(final List masters, + final List regionservers) { + LOG.debug("Shutting down HBase Cluster"); + if (masters != null) { + // Do backups first. + JVMClusterUtil.MasterThread activeMaster = null; + for (JVMClusterUtil.MasterThread t : masters) { + if (!t.master.isActiveMaster()) { + t.master.stopMaster(); + } else { + activeMaster = t; + } + } + // Do active after. + if (activeMaster != null) activeMaster.master.shutdown(); + } + // regionServerThreads can never be null because they are initialized when + // the class is constructed. + for(RegionServerThread t: regionservers) { + if (t.isAlive()) { + try { + t.getRegionServer().stop("Shutdown requested"); + t.join(); + } catch (InterruptedException e) { + // continue + } + } + } + if (masters != null) { + for (JVMClusterUtil.MasterThread t : masters) { + while (t.master.isAlive()) { + try { + // The below has been replaced to debug sometime hangs on end of + // tests. + // this.master.join(): + Threads.threadDumpingIsAlive(t.master.getThread()); + } catch(InterruptedException e) { + // continue + } + } + } + } + LOG.info("Shutdown of " + + ((masters != null) ? masters.size() : "0") + " master(s) and " + + ((regionservers != null) ? regionservers.size() : "0") + + " regionserver(s) complete"); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/JenkinsHash.java b/src/main/java/org/apache/hadoop/hbase/util/JenkinsHash.java new file mode 100644 index 0000000..f4f61d0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/JenkinsHash.java @@ -0,0 +1,257 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import static java.lang.Integer.rotateLeft; + +import java.io.FileInputStream; +import java.io.IOException; + +/** + * Produces 32-bit hash for hash table lookup. + * + * 

    lookup3.c, by Bob Jenkins, May 2006, Public Domain.
+ *
+ * You can use this free for any purpose.  It's in the public domain.
+ * It has no warranty.
+ *
    + * + * @see lookup3.c + * @see Hash Functions (and how this + * function compares to others such as CRC, MD?, etc + * @see Has update on the + * Dr. Dobbs Article + */ +public class JenkinsHash extends Hash { + private static final int BYTE_MASK = 0xff; + + private static JenkinsHash _instance = new JenkinsHash(); + + public static Hash getInstance() { + return _instance; + } + + /** + * taken from hashlittle() -- hash a variable-length key into a 32-bit value + * + * @param key the key (the unaligned variable-length array of bytes) + * @param nbytes number of bytes to include in hash + * @param initval can be any integer value + * @return a 32-bit value. Every bit of the key affects every bit of the + * return value. Two keys differing by one or two bits will have totally + * different hash values. + * + *

    The best hash table sizes are powers of 2. There is no need to do mod + * a prime (mod is sooo slow!). If you need less than 32 bits, use a bitmask. + * For example, if you need only 10 bits, do + * h = (h & hashmask(10)); + * In which case, the hash table should have hashsize(10) elements. + * + *

    If you are hashing n strings byte[][] k, do it like this: + * for (int i = 0, h = 0; i < n; ++i) h = hash( k[i], h); + * + *

    By Bob Jenkins, 2006. bob_jenkins@burtleburtle.net. You may use this + * code any way you wish, private, educational, or commercial. It's free. + * + *

    Use for hash table lookup, or anything where one collision in 2^^32 is + * acceptable. Do NOT use for cryptographic purposes. + */ + @Override + @SuppressWarnings("fallthrough") + public int hash(byte[] key, int off, int nbytes, int initval) { + int length = nbytes; + int a, b, c; + a = b = c = 0xdeadbeef + length + initval; + int offset = off; + for (; length > 12; offset += 12, length -= 12) { + a += (key[offset] & BYTE_MASK); + a += ((key[offset + 1] & BYTE_MASK) << 8); + a += ((key[offset + 2] & BYTE_MASK) << 16); + a += ((key[offset + 3] & BYTE_MASK) << 24); + b += (key[offset + 4] & BYTE_MASK); + b += ((key[offset + 5] & BYTE_MASK) << 8); + b += ((key[offset + 6] & BYTE_MASK) << 16); + b += ((key[offset + 7] & BYTE_MASK) << 24); + c += (key[offset + 8] & BYTE_MASK); + c += ((key[offset + 9] & BYTE_MASK) << 8); + c += ((key[offset + 10] & BYTE_MASK) << 16); + c += ((key[offset + 11] & BYTE_MASK) << 24); + + /* + * mix -- mix 3 32-bit values reversibly. + * This is reversible, so any information in (a,b,c) before mix() is + * still in (a,b,c) after mix(). + * + * If four pairs of (a,b,c) inputs are run through mix(), or through + * mix() in reverse, there are at least 32 bits of the output that + * are sometimes the same for one pair and different for another pair. + * + * This was tested for: + * - pairs that differed by one bit, by two bits, in any combination + * of top bits of (a,b,c), or in any combination of bottom bits of + * (a,b,c). + * - "differ" is defined as +, -, ^, or ~^. For + and -, I transformed + * the output delta to a Gray code (a^(a>>1)) so a string of 1's (as + * is commonly produced by subtraction) look like a single 1-bit + * difference. + * - the base values were pseudorandom, all zero but one bit set, or + * all zero plus a counter that starts at zero. + * + * Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that + * satisfy this are + * 4 6 8 16 19 4 + * 9 15 3 18 27 15 + * 14 9 3 7 17 3 + * Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing for + * "differ" defined as + with a one-bit base and a two-bit delta. I + * used http://burtleburtle.net/bob/hash/avalanche.html to choose + * the operations, constants, and arrangements of the variables. + * + * This does not achieve avalanche. There are input bits of (a,b,c) + * that fail to affect some output bits of (a,b,c), especially of a. + * The most thoroughly mixed value is c, but it doesn't really even + * achieve avalanche in c. + * + * This allows some parallelism. Read-after-writes are good at doubling + * the number of bits affected, so the goal of mixing pulls in the + * opposite direction as the goal of parallelism. I did what I could. + * Rotates seem to cost as much as shifts on every machine I could lay + * my hands on, and rotates are much kinder to the top and bottom bits, + * so I used rotates. + * + * #define mix(a,b,c) \ + * { \ + * a -= c; a ^= rot(c, 4); c += b; \ + * b -= a; b ^= rot(a, 6); a += c; \ + * c -= b; c ^= rot(b, 8); b += a; \ + * a -= c; a ^= rot(c,16); c += b; \ + * b -= a; b ^= rot(a,19); a += c; \ + * c -= b; c ^= rot(b, 4); b += a; \ + * } + * + * mix(a,b,c); + */ + a -= c; a ^= rotateLeft(c, 4); c += b; + b -= a; b ^= rotateLeft(a, 6); a += c; + c -= b; c ^= rotateLeft(b, 8); b += a; + a -= c; a ^= rotateLeft(c, 16); c += b; + b -= a; b ^= rotateLeft(a, 19); a += c; + c -= b; c ^= rotateLeft(b, 4); b += a; + } + + //-------------------------------- last block: affect all 32 bits of (c) + switch (length) { // all the case statements fall through + case 12: + c += ((key[offset + 11] & BYTE_MASK) << 24); + case 11: + c += ((key[offset + 10] & BYTE_MASK) << 16); + case 10: + c += ((key[offset + 9] & BYTE_MASK) << 8); + case 9: + c += (key[offset + 8] & BYTE_MASK); + case 8: + b += ((key[offset + 7] & BYTE_MASK) << 24); + case 7: + b += ((key[offset + 6] & BYTE_MASK) << 16); + case 6: + b += ((key[offset + 5] & BYTE_MASK) << 8); + case 5: + b += (key[offset + 4] & BYTE_MASK); + case 4: + a += ((key[offset + 3] & BYTE_MASK) << 24); + case 3: + a += ((key[offset + 2] & BYTE_MASK) << 16); + case 2: + a += ((key[offset + 1] & BYTE_MASK) << 8); + case 1: + //noinspection PointlessArithmeticExpression + a += (key[offset + 0] & BYTE_MASK); + break; + case 0: + return c; + } + /* + * final -- final mixing of 3 32-bit values (a,b,c) into c + * + * Pairs of (a,b,c) values differing in only a few bits will usually + * produce values of c that look totally different. This was tested for + * - pairs that differed by one bit, by two bits, in any combination + * of top bits of (a,b,c), or in any combination of bottom bits of + * (a,b,c). + * + * - "differ" is defined as +, -, ^, or ~^. For + and -, I transformed + * the output delta to a Gray code (a^(a>>1)) so a string of 1's (as + * is commonly produced by subtraction) look like a single 1-bit + * difference. + * + * - the base values were pseudorandom, all zero but one bit set, or + * all zero plus a counter that starts at zero. + * + * These constants passed: + * 14 11 25 16 4 14 24 + * 12 14 25 16 4 14 24 + * and these came close: + * 4 8 15 26 3 22 24 + * 10 8 15 26 3 22 24 + * 11 8 15 26 3 22 24 + * + * #define final(a,b,c) \ + * { + * c ^= b; c -= rot(b,14); \ + * a ^= c; a -= rot(c,11); \ + * b ^= a; b -= rot(a,25); \ + * c ^= b; c -= rot(b,16); \ + * a ^= c; a -= rot(c,4); \ + * b ^= a; b -= rot(a,14); \ + * c ^= b; c -= rot(b,24); \ + * } + * + */ + c ^= b; c -= rotateLeft(b, 14); + a ^= c; a -= rotateLeft(c, 11); + b ^= a; b -= rotateLeft(a, 25); + c ^= b; c -= rotateLeft(b, 16); + a ^= c; a -= rotateLeft(c, 4); + b ^= a; b -= rotateLeft(a, 14); + c ^= b; c -= rotateLeft(b, 24); + return c; + } + + /** + * Compute the hash of the specified file + * @param args name of file to compute hash of. + * @throws IOException e + */ + public static void main(String[] args) throws IOException { + if (args.length != 1) { + System.err.println("Usage: JenkinsHash filename"); + System.exit(-1); + } + FileInputStream in = new FileInputStream(args[0]); + byte[] bytes = new byte[512]; + int value = 0; + JenkinsHash hash = new JenkinsHash(); + for (int length = in.read(bytes); length > 0; length = in.read(bytes)) { + value = hash.hash(bytes, length, value); + } + System.out.println(Math.abs(value)); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/JvmVersion.java b/src/main/java/org/apache/hadoop/hbase/util/JvmVersion.java new file mode 100644 index 0000000..b7eb7e5 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/JvmVersion.java @@ -0,0 +1,43 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.HashSet; +import java.util.Set; + +/** + * Certain JVM versions are known to be unstable with HBase. This + * class has a utility function to determine whether the current JVM + * is known to be unstable. + */ +public abstract class JvmVersion { + private static Set BAD_JVM_VERSIONS = new HashSet(); + static { + BAD_JVM_VERSIONS.add("1.6.0_18"); + } + + /** + * Return true if the current JVM is known to be unstable. + */ + public static boolean isBadJvmVersion() { + String version = System.getProperty("java.version"); + return version != null && BAD_JVM_VERSIONS.contains(version); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/KeyRange.java b/src/main/java/org/apache/hadoop/hbase/util/KeyRange.java new file mode 100644 index 0000000..958edef --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/KeyRange.java @@ -0,0 +1,29 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +/** + * A key range use in split coverage. + */ +public interface KeyRange { + abstract byte[] getStartKey(); + + abstract byte[] getEndKey(); +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/Keying.java b/src/main/java/org/apache/hadoop/hbase/util/Keying.java new file mode 100644 index 0000000..2e3d027 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Keying.java @@ -0,0 +1,116 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.StringTokenizer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utility creating hbase friendly keys. + * Use fabricating row names or column qualifiers. + *

    TODO: Add createSchemeless key, a key that doesn't care if scheme is + * http or https. + * @see Bytes#split(byte[], byte[], int) + */ +public class Keying { + private static final String SCHEME = "r:"; + private static final Pattern URI_RE_PARSER = + Pattern.compile("^([^:/?#]+://(?:[^/?#@]+@)?)([^:/?#]+)(.*)$"); + + /** + * Makes a key out of passed URI for use as row name or column qualifier. + * + * This method runs transforms on the passed URI so it sits better + * as a key (or portion-of-a-key) in hbase. The host portion of + * the URI authority is reversed so subdomains sort under their parent + * domain. The returned String is an opaque URI of an artificial + * r: scheme to prevent the result being considered an URI of + * the original scheme. Here is an example of the transform: The url + * http://lucene.apache.org/index.html?query=something#middle is + * returned as + * r:http://org.apache.lucene/index.html?query=something#middle + * The transforms are reversible. No transform is done if passed URI is + * not hierarchical. + * + *

    If authority userinfo is present, will mess up the sort + * (until we do more work).

    + * + * @param u URL to transform. + * @return An opaque URI of artificial 'r' scheme with host portion of URI + * authority reversed (if present). + * @see #keyToUri(String) + * @see RFC2396 + */ + public static String createKey(final String u) { + if (u.startsWith(SCHEME)) { + throw new IllegalArgumentException("Starts with " + SCHEME); + } + Matcher m = getMatcher(u); + if (m == null || !m.matches()) { + // If no match, return original String. + return u; + } + return SCHEME + m.group(1) + reverseHostname(m.group(2)) + m.group(3); + } + + /** + * Reverse the {@link #createKey(String)} transform. + * + * @param s URI made by {@link #createKey(String)}. + * @return 'Restored' URI made by reversing the {@link #createKey(String)} + * transform. + */ + public static String keyToUri(final String s) { + if (!s.startsWith(SCHEME)) { + return s; + } + Matcher m = getMatcher(s.substring(SCHEME.length())); + if (m == null || !m.matches()) { + // If no match, return original String. + return s; + } + return m.group(1) + reverseHostname(m.group(2)) + m.group(3); + } + + private static Matcher getMatcher(final String u) { + if (u == null || u.length() <= 0) { + return null; + } + return URI_RE_PARSER.matcher(u); + } + + private static String reverseHostname(final String hostname) { + if (hostname == null) { + return ""; + } + StringBuilder sb = new StringBuilder(hostname.length()); + for (StringTokenizer st = new StringTokenizer(hostname, ".", false); + st.hasMoreElements();) { + Object next = st.nextElement(); + if (sb.length() > 0) { + sb.insert(0, "."); + } + sb.insert(0, next); + } + return sb.toString(); + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/MD5Hash.java b/src/main/java/org/apache/hadoop/hbase/util/MD5Hash.java new file mode 100644 index 0000000..b2998c9 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/MD5Hash.java @@ -0,0 +1,67 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Utility class for MD5 + * MD5 hash produces a 128-bit digest. + */ +public class MD5Hash { + private static final Log LOG = LogFactory.getLog(MD5Hash.class); + + /** + * Given a byte array, returns in MD5 hash as a hex string. + * @param key + * @return SHA1 hash as a 32 character hex string. + */ + public static String getMD5AsHex(byte[] key) { + return getMD5AsHex(key, 0, key.length); + } + + /** + * Given a byte array, returns its MD5 hash as a hex string. + * Only "length" number of bytes starting at "offset" within the + * byte array are used. + * + * @param key the key to hash (variable length byte array) + * @param offset + * @param length + * @return MD5 hash as a 32 character hex string. + */ + public static String getMD5AsHex(byte[] key, int offset, int length) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(key, offset, length); + byte[] digest = md.digest(); + return new String(Hex.encodeHex(digest)); + } catch (NoSuchAlgorithmException e) { + // this should never happen unless the JDK is messed up. + throw new RuntimeException("Error computing MD5 hash", e); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java b/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java new file mode 100644 index 0000000..a56926a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java @@ -0,0 +1,43 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +/** + * An environment edge that uses a manually set value. This is useful for testing events that are supposed to + * happen in the same millisecond. + */ +public class ManualEnvironmentEdge implements EnvironmentEdge { + + // Sometimes 0 ts might have a special value, so lets start with 1 + protected long value = 1L; + + public void setValue(long newValue) { + value = newValue; + } + + public void incValue(long addedValue) { + value += addedValue; + } + + @Override + public long currentTimeMillis() { + return this.value; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/Merge.java b/src/main/java/org/apache/hadoop/hbase/util/Merge.java new file mode 100644 index 0000000..1ed4225 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Merge.java @@ -0,0 +1,394 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.io.WritableComparator; +import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +import java.io.IOException; +import java.util.List; + +/** + * Utility that can merge any two regions in the same table: adjacent, + * overlapping or disjoint. + */ +public class Merge extends Configured implements Tool { + static final Log LOG = LogFactory.getLog(Merge.class); + private Path rootdir; + private volatile MetaUtils utils; + private byte [] tableName; // Name of table + private volatile byte [] region1; // Name of region 1 + private volatile byte [] region2; // Name of region 2 + private volatile boolean isMetaTable; + private volatile HRegionInfo mergeInfo; + + /** default constructor */ + public Merge() { + super(); + } + + /** + * @param conf configuration + */ + public Merge(Configuration conf) { + this.mergeInfo = null; + setConf(conf); + } + + public int run(String[] args) throws Exception { + if (parseArgs(args) != 0) { + return -1; + } + + // Verify file system is up. + FileSystem fs = FileSystem.get(getConf()); // get DFS handle + LOG.info("Verifying that file system is available..."); + try { + FSUtils.checkFileSystemAvailable(fs); + } catch (IOException e) { + LOG.fatal("File system is not available", e); + return -1; + } + + // Verify HBase is down + LOG.info("Verifying that HBase is not running..."); + try { + HBaseAdmin.checkHBaseAvailable(getConf()); + LOG.fatal("HBase cluster must be off-line."); + return -1; + } catch (ZooKeeperConnectionException zkce) { + // If no zk, presume no master. + } catch (MasterNotRunningException e) { + // Expected. Ignore. + } + + // Initialize MetaUtils and and get the root of the HBase installation + + this.utils = new MetaUtils(getConf()); + this.rootdir = FSUtils.getRootDir(getConf()); + try { + if (isMetaTable) { + mergeTwoMetaRegions(); + } else { + mergeTwoRegions(); + } + return 0; + } catch (Exception e) { + LOG.fatal("Merge failed", e); + utils.scanMetaRegion(HRegionInfo.FIRST_META_REGIONINFO, + new MetaUtils.ScannerListener() { + public boolean processRow(HRegionInfo info) { + System.err.println(info.toString()); + return true; + } + } + ); + + return -1; + + } finally { + if (this.utils != null) { + this.utils.shutdown(); + } + } + } + + /** @return HRegionInfo for merge result */ + HRegionInfo getMergedHRegionInfo() { + return this.mergeInfo; + } + + /* + * Merge two meta regions. This is unlikely to be needed soon as we have only + * seend the meta table split once and that was with 64MB regions. With 256MB + * regions, it will be some time before someone has enough data in HBase to + * split the meta region and even less likely that a merge of two meta + * regions will be needed, but it is included for completeness. + */ + private void mergeTwoMetaRegions() throws IOException { + HRegion rootRegion = utils.getRootRegion(); + Get get = new Get(region1); + get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + List cells1 = rootRegion.get(get, null).list(); + HRegionInfo info1 = Writables.getHRegionInfo((cells1 == null)? null: cells1.get(0).getValue()); + + get = new Get(region2); + get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + List cells2 = rootRegion.get(get, null).list(); + HRegionInfo info2 = Writables.getHRegionInfo((cells2 == null)? null: cells2.get(0).getValue()); + HRegion merged = merge(HTableDescriptor.META_TABLEDESC, info1, rootRegion, info2, rootRegion); + LOG.info("Adding " + merged.getRegionInfo() + " to " + + rootRegion.getRegionInfo()); + HRegion.addRegionToMETA(rootRegion, merged); + merged.close(); + } + + private static class MetaScannerListener + implements MetaUtils.ScannerListener { + private final byte [] region1; + private final byte [] region2; + private HRegionInfo meta1 = null; + private HRegionInfo meta2 = null; + + MetaScannerListener(final byte [] region1, final byte [] region2) { + this.region1 = region1; + this.region2 = region2; + } + + public boolean processRow(HRegionInfo info) { + if (meta1 == null && HRegion.rowIsInRange(info, region1)) { + meta1 = info; + } + if (region2 != null && meta2 == null && + HRegion.rowIsInRange(info, region2)) { + meta2 = info; + } + return meta1 == null || (region2 != null && meta2 == null); + } + + HRegionInfo getMeta1() { + return meta1; + } + + HRegionInfo getMeta2() { + return meta2; + } + } + + /* + * Merges two regions from a user table. + */ + private void mergeTwoRegions() throws IOException { + LOG.info("Merging regions " + Bytes.toStringBinary(this.region1) + " and " + + Bytes.toStringBinary(this.region2) + " in table " + Bytes.toString(this.tableName)); + // Scan the root region for all the meta regions that contain the regions + // we're merging. + MetaScannerListener listener = new MetaScannerListener(region1, region2); + this.utils.scanRootRegion(listener); + HRegionInfo meta1 = listener.getMeta1(); + if (meta1 == null) { + throw new IOException("Could not find meta region for " + Bytes.toStringBinary(region1)); + } + HRegionInfo meta2 = listener.getMeta2(); + if (meta2 == null) { + throw new IOException("Could not find meta region for " + Bytes.toStringBinary(region2)); + } + LOG.info("Found meta for region1 " + Bytes.toStringBinary(meta1.getRegionName()) + + ", meta for region2 " + Bytes.toStringBinary(meta2.getRegionName())); + HRegion metaRegion1 = this.utils.getMetaRegion(meta1); + Get get = new Get(region1); + get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + List cells1 = metaRegion1.get(get, null).list(); + HRegionInfo info1 = + Writables.getHRegionInfo((cells1 == null)? null: cells1.get(0).getValue()); + if (info1 == null) { + throw new NullPointerException("info1 is null using key " + + Bytes.toStringBinary(region1) + " in " + meta1); + } + + HRegion metaRegion2; + if (Bytes.equals(meta1.getRegionName(), meta2.getRegionName())) { + metaRegion2 = metaRegion1; + } else { + metaRegion2 = utils.getMetaRegion(meta2); + } + get = new Get(region2); + get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + List cells2 = metaRegion2.get(get, null).list(); + HRegionInfo info2 = Writables.getHRegionInfo((cells2 == null)? null: cells2.get(0).getValue()); + if (info2 == null) { + throw new NullPointerException("info2 is null using key " + meta2); + } + HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(FileSystem.get(getConf()), + this.rootdir, this.tableName); + HRegion merged = merge(htd, info1, metaRegion1, info2, metaRegion2); + + // Now find the meta region which will contain the newly merged region + + listener = new MetaScannerListener(merged.getRegionName(), null); + utils.scanRootRegion(listener); + HRegionInfo mergedInfo = listener.getMeta1(); + if (mergedInfo == null) { + throw new IOException("Could not find meta region for " + + Bytes.toStringBinary(merged.getRegionName())); + } + HRegion mergeMeta; + if (Bytes.equals(mergedInfo.getRegionName(), meta1.getRegionName())) { + mergeMeta = metaRegion1; + } else if (Bytes.equals(mergedInfo.getRegionName(), meta2.getRegionName())) { + mergeMeta = metaRegion2; + } else { + mergeMeta = utils.getMetaRegion(mergedInfo); + } + LOG.info("Adding " + merged.getRegionInfo() + " to " + + mergeMeta.getRegionInfo()); + + HRegion.addRegionToMETA(mergeMeta, merged); + merged.close(); + } + + /* + * Actually merge two regions and update their info in the meta region(s) + * If the meta is split, meta1 may be different from meta2. (and we may have + * to scan the meta if the resulting merged region does not go in either) + * Returns HRegion object for newly merged region + */ + private HRegion merge(final HTableDescriptor htd, HRegionInfo info1, + HRegion meta1, HRegionInfo info2, HRegion meta2) + throws IOException { + if (info1 == null) { + throw new IOException("Could not find " + Bytes.toStringBinary(region1) + " in " + + Bytes.toStringBinary(meta1.getRegionName())); + } + if (info2 == null) { + throw new IOException("Cound not find " + Bytes.toStringBinary(region2) + " in " + + Bytes.toStringBinary(meta2.getRegionName())); + } + HRegion merged = null; + HLog log = utils.getLog(); + HRegion r1 = HRegion.openHRegion(info1, htd, log, getConf()); + try { + HRegion r2 = HRegion.openHRegion(info2, htd, log, getConf()); + try { + merged = HRegion.merge(r1, r2); + } finally { + if (!r2.isClosed()) { + r2.close(); + } + } + } finally { + if (!r1.isClosed()) { + r1.close(); + } + } + + // Remove the old regions from meta. + // HRegion.merge has already deleted their files + + removeRegionFromMeta(meta1, info1); + removeRegionFromMeta(meta2, info2); + + this.mergeInfo = merged.getRegionInfo(); + return merged; + } + + /* + * Removes a region's meta information from the passed meta + * region. + * + * @param meta META HRegion to be updated + * @param regioninfo HRegionInfo of region to remove from meta + * + * @throws IOException + */ + private void removeRegionFromMeta(HRegion meta, HRegionInfo regioninfo) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing region: " + regioninfo + " from " + meta); + } + + Delete delete = new Delete(regioninfo.getRegionName(), + System.currentTimeMillis(), null); + meta.delete(delete, null, true); + } + + /* + * Adds a region's meta information from the passed meta + * region. + * + * @param metainfo META HRegionInfo to be updated + * @param region HRegion to add to meta + * + * @throws IOException + */ + private int parseArgs(String[] args) throws IOException { + GenericOptionsParser parser = + new GenericOptionsParser(getConf(), args); + + String[] remainingArgs = parser.getRemainingArgs(); + if (remainingArgs.length != 3) { + usage(); + return -1; + } + tableName = Bytes.toBytes(remainingArgs[0]); + isMetaTable = Bytes.compareTo(tableName, HConstants.META_TABLE_NAME) == 0; + + region1 = Bytes.toBytesBinary(remainingArgs[1]); + region2 = Bytes.toBytesBinary(remainingArgs[2]); + int status = 0; + if (notInTable(tableName, region1) || notInTable(tableName, region2)) { + status = -1; + } else if (Bytes.equals(region1, region2)) { + LOG.error("Can't merge a region with itself"); + status = -1; + } + return status; + } + + private boolean notInTable(final byte [] tn, final byte [] rn) { + if (WritableComparator.compareBytes(tn, 0, tn.length, rn, 0, tn.length) != 0) { + LOG.error("Region " + Bytes.toStringBinary(rn) + " does not belong to table " + + Bytes.toString(tn)); + return true; + } + return false; + } + + private void usage() { + System.err + .println("For hadoop 0.20, Usage: bin/hbase org.apache.hadoop.hbase.util.Merge " + + "[-Dfs.default.name=hdfs://nn:port] \n"); + System.err + .println("For hadoop 0.21+, Usage: bin/hbase org.apache.hadoop.hbase.util.Merge " + + "[-Dfs.defaultFS=hdfs://nn:port] \n"); + } + + public static void main(String[] args) { + int status; + try { + status = ToolRunner.run(HBaseConfiguration.create(), new Merge(), args); + } catch (Exception e) { + LOG.error("exiting due to error", e); + status = -1; + } + System.exit(status); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java b/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java new file mode 100644 index 0000000..af8d734 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java @@ -0,0 +1,404 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.wal.HLog; + +/** + * Contains utility methods for manipulating HBase meta tables. + * Be sure to call {@link #shutdown()} when done with this class so it closes + * resources opened during meta processing (ROOT, META, etc.). Be careful + * how you use this class. If used during migrations, be careful when using + * this class to check whether migration is needed. + */ +public class MetaUtils { + private static final Log LOG = LogFactory.getLog(MetaUtils.class); + private final Configuration conf; + private FileSystem fs; + private HLog log; + private HRegion rootRegion; + private Map metaRegions = Collections.synchronizedSortedMap( + new TreeMap(Bytes.BYTES_COMPARATOR)); + + /** Default constructor + * @throws IOException e + */ + public MetaUtils() throws IOException { + this(HBaseConfiguration.create()); + } + + /** + * @param conf Configuration + * @throws IOException e + */ + public MetaUtils(Configuration conf) throws IOException { + this.conf = conf; + conf.setInt("hbase.client.retries.number", 1); + this.rootRegion = null; + initialize(); + } + + /** + * Verifies that DFS is available and that HBase is off-line. + * @throws IOException e + */ + private void initialize() throws IOException { + this.fs = FileSystem.get(this.conf); + } + + /** + * @return the HLog + * @throws IOException e + */ + public synchronized HLog getLog() throws IOException { + if (this.log == null) { + Path logdir = new Path(this.fs.getHomeDirectory(), + HConstants.HREGION_LOGDIR_NAME + "_" + System.currentTimeMillis()); + Path oldLogDir = new Path(this.fs.getHomeDirectory(), + HConstants.HREGION_OLDLOGDIR_NAME); + this.log = new HLog(this.fs, logdir, oldLogDir, this.conf); + } + return this.log; + } + + /** + * @return HRegion for root region + * @throws IOException e + */ + public HRegion getRootRegion() throws IOException { + if (this.rootRegion == null) { + openRootRegion(); + } + return this.rootRegion; + } + + /** + * Open or return cached opened meta region + * + * @param metaInfo HRegionInfo for meta region + * @return meta HRegion + * @throws IOException e + */ + public HRegion getMetaRegion(HRegionInfo metaInfo) throws IOException { + HRegion meta = metaRegions.get(metaInfo.getRegionName()); + if (meta == null) { + meta = openMetaRegion(metaInfo); + LOG.info("OPENING META " + meta.toString()); + this.metaRegions.put(metaInfo.getRegionName(), meta); + } + return meta; + } + + /** + * Closes catalog regions if open. Also closes and deletes the HLog. You + * must call this method if you want to persist changes made during a + * MetaUtils edit session. + */ + public void shutdown() { + if (this.rootRegion != null) { + try { + this.rootRegion.close(); + } catch (IOException e) { + LOG.error("closing root region", e); + } finally { + this.rootRegion = null; + } + } + try { + for (HRegion r: metaRegions.values()) { + LOG.info("CLOSING META " + r.toString()); + r.close(); + } + } catch (IOException e) { + LOG.error("closing meta region", e); + } finally { + metaRegions.clear(); + } + try { + if (this.log != null) { + this.log.rollWriter(); + this.log.closeAndDelete(); + } + } catch (IOException e) { + LOG.error("closing HLog", e); + } finally { + this.log = null; + } + } + + /** + * Used by scanRootRegion and scanMetaRegion to call back the caller so it + * can process the data for a row. + */ + public interface ScannerListener { + /** + * Callback so client of scanner can process row contents + * + * @param info HRegionInfo for row + * @return false to terminate the scan + * @throws IOException e + */ + public boolean processRow(HRegionInfo info) throws IOException; + } + + /** + * Scans the root region. For every meta region found, calls the listener with + * the HRegionInfo of the meta region. + * + * @param listener method to be called for each meta region found + * @throws IOException e + */ + public void scanRootRegion(ScannerListener listener) throws IOException { + // Open root region so we can scan it + if (this.rootRegion == null) { + openRootRegion(); + } + scanMetaRegion(this.rootRegion, listener); + } + + /** + * Scan the passed in metaregion m invoking the passed + * listener per row found. + * @param r region + * @param listener scanner listener + * @throws IOException e + */ + public void scanMetaRegion(final HRegion r, final ScannerListener listener) + throws IOException { + Scan scan = new Scan(); + scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + InternalScanner s = r.getScanner(scan); + try { + List results = new ArrayList(); + boolean hasNext = true; + do { + hasNext = s.next(results); + HRegionInfo info = null; + for (KeyValue kv: results) { + info = Writables.getHRegionInfoOrNull(kv.getValue()); + if (info == null) { + LOG.warn("Region info is null for row " + + Bytes.toStringBinary(kv.getRow()) + " in table " + + r.getTableDesc().getNameAsString()); + } + continue; + } + if (!listener.processRow(info)) { + break; + } + results.clear(); + } while (hasNext); + } finally { + s.close(); + } + } + + /** + * Scans a meta region. For every region found, calls the listener with + * the HRegionInfo of the region. + * TODO: Use Visitor rather than Listener pattern. Allow multiple Visitors. + * Use this everywhere we scan meta regions: e.g. in metascanners, in close + * handling, etc. Have it pass in the whole row, not just HRegionInfo. + *

    Use for reading meta only. Does not close region when done. + * Use {@link #getMetaRegion(HRegionInfo)} instead if writing. Adds + * meta region to list that will get a close on {@link #shutdown()}. + * + * @param metaRegionInfo HRegionInfo for meta region + * @param listener method to be called for each meta region found + * @throws IOException e + */ + public void scanMetaRegion(HRegionInfo metaRegionInfo, + ScannerListener listener) + throws IOException { + // Open meta region so we can scan it + HRegion metaRegion = openMetaRegion(metaRegionInfo); + scanMetaRegion(metaRegion, listener); + } + + private synchronized HRegion openRootRegion() throws IOException { + if (this.rootRegion != null) { + return this.rootRegion; + } + this.rootRegion = HRegion.openHRegion(HRegionInfo.ROOT_REGIONINFO, + HTableDescriptor.ROOT_TABLEDESC, getLog(), + this.conf); + this.rootRegion.compactStores(); + return this.rootRegion; + } + + private HRegion openMetaRegion(HRegionInfo metaInfo) throws IOException { + HRegion meta = HRegion.openHRegion(metaInfo, HTableDescriptor.META_TABLEDESC, + getLog(), this.conf); + meta.compactStores(); + return meta; + } + + /** + * Set a single region on/offline. + * This is a tool to repair tables that have offlined tables in their midst. + * Can happen on occasion. Use at your own risk. Call from a bit of java + * or jython script. This method is 'expensive' in that it creates a + * {@link HTable} instance per invocation to go against .META. + * @param c A configuration that has its hbase.master + * properly set. + * @param row Row in the catalog .META. table whose HRegionInfo's offline + * status we want to change. + * @param onlineOffline Pass true to OFFLINE the region. + * @throws IOException e + */ + public static void changeOnlineStatus (final Configuration c, + final byte [] row, final boolean onlineOffline) + throws IOException { + HTable t = new HTable(c, HConstants.META_TABLE_NAME); + Get get = new Get(row); + get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + Result res = t.get(get); + KeyValue [] kvs = res.raw(); + if(kvs.length <= 0) { + throw new IOException("no information for row " + Bytes.toString(row)); + } + byte [] value = kvs[0].getValue(); + if (value == null) { + throw new IOException("no information for row " + Bytes.toString(row)); + } + HRegionInfo info = Writables.getHRegionInfo(value); + Put put = new Put(row); + info.setOffline(onlineOffline); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(info)); + t.put(put); + + Delete delete = new Delete(row); + delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); + delete.deleteColumns(HConstants.CATALOG_FAMILY, + HConstants.STARTCODE_QUALIFIER); + + t.delete(delete); + } + + /** + * Update COL_REGIONINFO in meta region r with HRegionInfo hri + * + * @param r region + * @param hri region info + * @throws IOException e + */ + public void updateMETARegionInfo(HRegion r, final HRegionInfo hri) + throws IOException { + if (LOG.isDebugEnabled()) { + Get get = new Get(hri.getRegionName()); + get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + Result res = r.get(get, null); + KeyValue [] kvs = res.raw(); + if(kvs.length <= 0) { + return; + } + byte [] value = kvs[0].getValue(); + if (value == null) { + return; + } + HRegionInfo h = Writables.getHRegionInfoOrNull(value); + + LOG.debug("Old " + Bytes.toString(HConstants.CATALOG_FAMILY) + ":" + + Bytes.toString(HConstants.REGIONINFO_QUALIFIER) + " for " + + hri.toString() + " in " + r.toString() + " is: " + h.toString()); + } + + Put put = new Put(hri.getRegionName()); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + r.put(put); + + if (LOG.isDebugEnabled()) { + Get get = new Get(hri.getRegionName()); + get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + Result res = r.get(get, null); + KeyValue [] kvs = res.raw(); + if(kvs.length <= 0) { + return; + } + byte [] value = kvs[0].getValue(); + if (value == null) { + return; + } + HRegionInfo h = Writables.getHRegionInfoOrNull(value); + LOG.debug("New " + Bytes.toString(HConstants.CATALOG_FAMILY) + ":" + + Bytes.toString(HConstants.REGIONINFO_QUALIFIER) + " for " + + hri.toString() + " in " + r.toString() + " is: " + h.toString()); + } + } + + /** + * @return List of {@link HRegionInfo} rows found in the ROOT or META + * catalog table. + * @param tableName Name of table to go looking for. + * @throws IOException e + * @see #getMetaRegion(HRegionInfo) + */ + public List getMETARows(final byte [] tableName) + throws IOException { + final List result = new ArrayList(); + // If passed table name is META, then return the root region. + if (Bytes.equals(HConstants.META_TABLE_NAME, tableName)) { + result.add(openRootRegion().getRegionInfo()); + return result; + } + // Return all meta regions that contain the passed tablename. + scanRootRegion(new ScannerListener() { + private final Log SL_LOG = LogFactory.getLog(this.getClass()); + + public boolean processRow(HRegionInfo info) throws IOException { + SL_LOG.debug("Testing " + info); + if (Bytes.equals(info.getTableName(), + HConstants.META_TABLE_NAME)) { + result.add(info); + return false; + } + return true; + }}); + return result; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/Methods.java b/src/main/java/org/apache/hadoop/hbase/util/Methods.java new file mode 100644 index 0000000..a6f4835 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Methods.java @@ -0,0 +1,65 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.UndeclaredThrowableException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class Methods { + private static Log LOG = LogFactory.getLog(Methods.class); + + public static Object call(Class clazz, T instance, String methodName, + Class[] types, Object[] args) throws Exception { + try { + Method m = clazz.getMethod(methodName, types); + return m.invoke(instance, args); + } catch (IllegalArgumentException arge) { + LOG.fatal("Constructed invalid call. class="+clazz.getName()+ + " method=" + methodName + " types=" + Classes.stringify(types), arge); + throw arge; + } catch (NoSuchMethodException nsme) { + throw new IllegalArgumentException( + "Can't find method "+methodName+" in "+clazz.getName()+"!", nsme); + } catch (InvocationTargetException ite) { + // unwrap the underlying exception and rethrow + if (ite.getTargetException() != null) { + if (ite.getTargetException() instanceof Exception) { + throw (Exception)ite.getTargetException(); + } else if (ite.getTargetException() instanceof Error) { + throw (Error)ite.getTargetException(); + } + } + throw new UndeclaredThrowableException(ite, + "Unknown exception invoking "+clazz.getName()+"."+methodName+"()"); + } catch (IllegalAccessException iae) { + throw new IllegalArgumentException( + "Denied access calling "+clazz.getName()+"."+methodName+"()", iae); + } catch (SecurityException se) { + LOG.fatal("SecurityException calling method. class="+clazz.getName()+ + " method=" + methodName + " types=" + Classes.stringify(types), se); + throw se; + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java new file mode 100644 index 0000000..ba72227 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java @@ -0,0 +1,176 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.backup.HFileArchiver; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.catalog.MetaEditor; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.regionserver.HRegion; + +/** + * Utility methods for interacting with the regions. + */ +@InterfaceAudience.Private +public abstract class ModifyRegionUtils { + private static final Log LOG = LogFactory.getLog(ModifyRegionUtils.class); + + private ModifyRegionUtils() { + } + + public interface RegionFillTask { + public void fillRegion(final HRegion region) throws IOException; + } + + /** + * Create new set of regions on the specified file-system. + * NOTE: that you should add the regions to .META. after this operation. + * + * @param conf {@link Configuration} + * @param rootDir Root directory for HBase instance + * @param hTableDescriptor description of the table + * @param newRegions {@link HRegionInfo} that describes the regions to create + * @throws IOException + */ + public static List createRegions(final Configuration conf, final Path rootDir, + final HTableDescriptor hTableDescriptor, final HRegionInfo[] newRegions) throws IOException { + return createRegions(conf, rootDir, hTableDescriptor, newRegions, null); + } + + /** + * Create new set of regions on the specified file-system. + * NOTE: that you should add the regions to .META. after this operation. + * + * @param conf {@link Configuration} + * @param rootDir Root directory for HBase instance + * @param hTableDescriptor description of the table + * @param newRegions {@link HRegionInfo} that describes the regions to create + * @param task {@link RegionFillTask} custom code to populate region after creation + * @throws IOException + */ + public static List createRegions(final Configuration conf, final Path rootDir, + final HTableDescriptor hTableDescriptor, final HRegionInfo[] newRegions, + final RegionFillTask task) throws IOException { + if (newRegions == null) return null; + int regionNumber = newRegions.length; + ThreadPoolExecutor regionOpenAndInitThreadPool = getRegionOpenAndInitThreadPool(conf, + "RegionOpenAndInitThread-" + hTableDescriptor.getNameAsString(), regionNumber); + CompletionService completionService = new ExecutorCompletionService( + regionOpenAndInitThreadPool); + List regionInfos = new ArrayList(); + for (final HRegionInfo newRegion : newRegions) { + completionService.submit(new Callable() { + public HRegionInfo call() throws IOException { + // 1. Create HRegion + HRegion region = HRegion.createHRegion(newRegion, + rootDir, conf, hTableDescriptor, null, + false, true); + try { + // 2. Custom user code to interact with the created region + if (task != null) { + task.fillRegion(region); + } + } finally { + // 3. Close the new region to flush to disk. Close log file too. + region.close(); + } + return region.getRegionInfo(); + } + }); + } + try { + // 4. wait for all regions to finish creation + for (int i = 0; i < regionNumber; i++) { + Future future = completionService.take(); + HRegionInfo regionInfo = future.get(); + regionInfos.add(regionInfo); + } + } catch (InterruptedException e) { + LOG.error("Caught " + e + " during region creation"); + throw new InterruptedIOException(e.getMessage()); + } catch (ExecutionException e) { + throw new IOException(e); + } finally { + regionOpenAndInitThreadPool.shutdownNow(); + } + return regionInfos; + } + + /* + * used by createRegions() to get the thread pool executor based on the + * "hbase.hregion.open.and.init.threads.max" property. + */ + static ThreadPoolExecutor getRegionOpenAndInitThreadPool(final Configuration conf, + final String threadNamePrefix, int regionNumber) { + int maxThreads = Math.min(regionNumber, conf.getInt( + "hbase.hregion.open.and.init.threads.max", 10)); + ThreadPoolExecutor regionOpenAndInitThreadPool = Threads + .getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS, + new ThreadFactory() { + private int count = 1; + + public Thread newThread(Runnable r) { + Thread t = new Thread(r, threadNamePrefix + "-" + count++); + return t; + } + }); + return regionOpenAndInitThreadPool; + } + + /** + * Trigger immediate assignment of the regions in round-robin fashion + * + * @param assignmentManager + * @param regions + */ + public static void assignRegions(final AssignmentManager assignmentManager, + final List regions) throws IOException { + try { + assignmentManager.assignUserRegionsToOnlineServers(regions); + } catch (InterruptedException ie) { + LOG.error("Caught " + ie + " during round-robin assignment"); + throw new InterruptedIOException(ie.getMessage()); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/MurmurHash.java b/src/main/java/org/apache/hadoop/hbase/util/MurmurHash.java new file mode 100644 index 0000000..085bf1e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/MurmurHash.java @@ -0,0 +1,88 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +/** + * This is a very fast, non-cryptographic hash suitable for general hash-based + * lookup. See http://murmurhash.googlepages.com/ for more details. + * + *

    The C version of MurmurHash 2.0 found at that site was ported + * to Java by Andrzej Bialecki (ab at getopt org).

    + */ +public class MurmurHash extends Hash { + private static MurmurHash _instance = new MurmurHash(); + + public static Hash getInstance() { + return _instance; + } + + @Override + public int hash(byte[] data, int offset, int length, int seed) { + int m = 0x5bd1e995; + int r = 24; + + int h = seed ^ length; + + int len_4 = length >> 2; + + for (int i = 0; i < len_4; i++) { + int i_4 = (i << 2) + offset; + int k = data[i_4 + 3]; + k = k << 8; + k = k | (data[i_4 + 2] & 0xff); + k = k << 8; + k = k | (data[i_4 + 1] & 0xff); + k = k << 8; + //noinspection PointlessArithmeticExpression + k = k | (data[i_4 + 0] & 0xff); + k *= m; + k ^= k >>> r; + k *= m; + h *= m; + h ^= k; + } + + // avoid calculating modulo + int len_m = len_4 << 2; + int left = length - len_m; + int i_m = len_m + offset; + + if (left != 0) { + if (left >= 3) { + h ^= data[i_m + 2] << 16; + } + if (left >= 2) { + h ^= data[i_m + 1] << 8; + } + if (left >= 1) { + h ^= data[i_m]; + } + + h *= m; + } + + h ^= h >>> 13; + h *= m; + h ^= h >>> 15; + + return h; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/Objects.java b/src/main/java/org/apache/hadoop/hbase/util/Objects.java new file mode 100644 index 0000000..088b54a --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Objects.java @@ -0,0 +1,189 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Action; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.MultiAction; +import org.apache.hadoop.hbase.client.Put; + +/** + * Utility methods for interacting with object instances. + */ +public class Objects { + private static class QuantityMap extends HashMap { + public void increment(String type, int count) { + Quantity q = get(type); + if (q == null) { + q = new Quantity(); + q.what = type; + put(type, q); + } + q.increment(count); + } + + public void stat(String type, int value) { + Quantity q = get(type); + if (q == null) { + q = new Stat(); + q.what = type; + put(type, q); + } + q.increment(value); + } + } + + private static class Quantity { + int count; + String what; + + public void increment(int amount) { + count += amount; + } + + public void appendToString(StringBuilder out) { + if (out.length() > 0) out.append(", "); + + out.append(count).append(" ").append(what); + if (count != 1 && !what.endsWith("s")) { + out.append("s"); + } + } + } + + private static class Stat extends Quantity { + int min; + int max; + long total; + + public void increment(int amount) { + if (count == 0) { + min = max = amount; + } else { + min = Math.min(min, amount); + max = Math.max(max, amount); + } + total += amount; + count++; + } + + public void appendToString(StringBuilder out) { + super.appendToString(out); + + out.append(" [ "); + if (count > 0) { + out.append("min=").append(min) + .append(" max=").append(max) + .append(" avg=").append((int)(total/count)); + } else { + out.append("none"); + } + out.append(" ]"); + } + } + + private static class QuantityComparator implements Comparator { + @Override + public int compare(Quantity q1, Quantity q2) { + if (q1.count < q2.count) { + return -1; + } else if (q1.count > q2.count) { + return 1; + } + return 0; + } + } + + /** + * Attempts to construct a text description of the given object, by + * introspecting known classes and building a description of size. + * @param obj + * @return Description + */ + public static String describeQuantity(Object obj) { + StringBuilder str = new StringBuilder(); + QuantityMap quantities = new QuantityMap(); + quantify(obj, quantities); + List totals = new ArrayList(quantities.values()); + Collections.sort(totals, new QuantityComparator()); + for (Quantity q : totals) { + q.appendToString(str); + } + return str.toString(); + } + + public static void quantify(Object obj, QuantityMap quantities) { + if (obj == null) { + return; + } + + if (obj.getClass().isArray()) { + Class type = obj.getClass().getComponentType(); + int length = Array.getLength(obj); + if (type.isPrimitive()) { + quantities.increment(type.getSimpleName(), length); + } else { + for (int i=0; i keyValues : ((Put)obj).getFamilyMap().values()) { + for (KeyValue kv : keyValues) { + quantities.stat("values", kv.getValueLength()); + } + } + } else if (obj instanceof Delete) { + quantities.increment("Delete", 1); + for (List kvs : ((Delete)obj).getFamilyMap().values()) { + quantities.increment("KeyValue", kvs.size()); + } + } else if (obj instanceof Increment) { + quantities.increment("Increment", 1); + quantities.increment("KeyValue", ((Increment)obj).numColumns()); + } else if (obj instanceof Get) { + quantities.increment("Get", 1); + } else { + String type = obj.getClass().getSimpleName(); + quantities.increment(type, 1); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/Pair.java b/src/main/java/org/apache/hadoop/hbase/util/Pair.java new file mode 100644 index 0000000..9b4a567 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Pair.java @@ -0,0 +1,131 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.Serializable; + +/** + * A generic class for pairs. + * @param + * @param + */ +public class Pair implements Serializable +{ + private static final long serialVersionUID = -3986244606585552569L; + protected T1 first = null; + protected T2 second = null; + + /** + * Default constructor. + */ + public Pair() + { + } + + /** + * Constructor + * @param a operand + * @param b operand + */ + public Pair(T1 a, T2 b) + { + this.first = a; + this.second = b; + } + + /** + * Constructs a new pair, inferring the type via the passed arguments + * @param type for first + * @param type for second + * @param a first element + * @param b second element + * @return a new pair containing the passed arguments + */ + public static Pair newPair(T1 a, T2 b) { + return new Pair(a, b); + } + + /** + * Replace the first element of the pair. + * @param a operand + */ + public void setFirst(T1 a) + { + this.first = a; + } + + /** + * Replace the second element of the pair. + * @param b operand + */ + public void setSecond(T2 b) + { + this.second = b; + } + + /** + * Return the first element stored in the pair. + * @return T1 + */ + public T1 getFirst() + { + return first; + } + + /** + * Return the second element stored in the pair. + * @return T2 + */ + public T2 getSecond() + { + return second; + } + + private static boolean equals(Object x, Object y) + { + return (x == null && y == null) || (x != null && x.equals(y)); + } + + @Override + @SuppressWarnings("unchecked") + public boolean equals(Object other) + { + return other instanceof Pair && equals(first, ((Pair)other).first) && + equals(second, ((Pair)other).second); + } + + @Override + public int hashCode() + { + if (first == null) + return (second == null) ? 0 : second.hashCode() + 1; + else if (second == null) + return first.hashCode() + 2; + else + return first.hashCode() * 17 + second.hashCode(); + } + + @Override + public String toString() + { + return "{" + getFirst() + "," + getSecond() + "}"; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/util/PairOfSameType.java b/src/main/java/org/apache/hadoop/hbase/util/PairOfSameType.java new file mode 100644 index 0000000..ddee30d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/PairOfSameType.java @@ -0,0 +1,112 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.util.Iterator; + +import org.apache.commons.lang.NotImplementedException; + +/** + * A generic, immutable class for pairs of objects both of type T. + * @param + * @see Pair if Types differ. + */ +public class PairOfSameType implements Iterable { + private final T first; + private final T second; + + /** + * Constructor + * @param a operand + * @param b operand + */ + public PairOfSameType(T a, T b) { + this.first = a; + this.second = b; + } + + /** + * Return the first element stored in the pair. + * @return T + */ + public T getFirst() { + return first; + } + + /** + * Return the second element stored in the pair. + * @return T + */ + public T getSecond() { + return second; + } + + private static boolean equals(Object x, Object y) { + return (x == null && y == null) || (x != null && x.equals(y)); + } + + @Override + @SuppressWarnings("unchecked") + public boolean equals(Object other) { + return other instanceof PairOfSameType && + equals(first, ((PairOfSameType)other).first) && + equals(second, ((PairOfSameType)other).second); + } + + @Override + public int hashCode() { + if (first == null) + return (second == null) ? 0 : second.hashCode() + 1; + else if (second == null) + return first.hashCode() + 2; + else + return first.hashCode() * 17 + second.hashCode(); + } + + @Override + public String toString() { + return "{" + getFirst() + "," + getSecond() + "}"; + } + + @Override + public Iterator iterator() { + return new Iterator() { + private int returned = 0; + + @Override + public boolean hasNext() { + return this.returned < 2; + } + + @Override + public T next() { + if (++this.returned == 1) return getFirst(); + else if (this.returned == 2) return getSecond(); + else throw new IllegalAccessError("this.returned=" + this.returned); + } + + @Override + public void remove() { + throw new NotImplementedException(); + } + }; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java b/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java new file mode 100644 index 0000000..1956e6b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java @@ -0,0 +1,447 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * + * The PoolMap maps a key to a collection of values, the elements + * of which are managed by a pool. In effect, that collection acts as a shared + * pool of resources, access to which is closely controlled as per the semantics + * of the pool. + * + *

    + * In case the size of the pool is set to a non-zero positive number, that is + * used to cap the number of resources that a pool may contain for any given + * key. A size of {@link Integer#MAX_VALUE} is interpreted as an unbounded pool. + *

    + * + * @param + * the type of the key to the resource + * @param + * the type of the resource being pooled + */ +public class PoolMap implements Map { + private PoolType poolType; + + private int poolMaxSize; + + private Map> pools = new ConcurrentHashMap>(); + + public PoolMap(PoolType poolType) { + this.poolType = poolType; + } + + public PoolMap(PoolType poolType, int poolMaxSize) { + this.poolType = poolType; + this.poolMaxSize = poolMaxSize; + } + + @Override + public V get(Object key) { + Pool pool = pools.get(key); + return pool != null ? pool.get() : null; + } + + @Override + public V put(K key, V value) { + Pool pool = pools.get(key); + if (pool == null) { + pools.put(key, pool = createPool()); + } + return pool != null ? pool.put(value) : null; + } + + @SuppressWarnings("unchecked") + @Override + public V remove(Object key) { + Pool pool = pools.remove(key); + if (pool != null) { + remove((K) key, pool.get()); + } + return null; + } + + public boolean remove(K key, V value) { + Pool pool = pools.get(key); + boolean res = false; + if (pool != null) { + res = pool.remove(value); + if (res && pool.size() == 0) { + pools.remove(key); + } + } + return res; + } + + @Override + public Collection values() { + Collection values = new ArrayList(); + for (Pool pool : pools.values()) { + Collection poolValues = pool.values(); + if (poolValues != null) { + values.addAll(poolValues); + } + } + return values; + } + + public Collection values(K key) { + Collection values = new ArrayList(); + Pool pool = pools.get(key); + if (pool != null) { + Collection poolValues = pool.values(); + if (poolValues != null) { + values.addAll(poolValues); + } + } + return values; + } + + + @Override + public boolean isEmpty() { + return pools.isEmpty(); + } + + @Override + public int size() { + return pools.size(); + } + + public int size(K key) { + Pool pool = pools.get(key); + return pool != null ? pool.size() : 0; + } + + @Override + public boolean containsKey(Object key) { + return pools.containsKey(key); + } + + @Override + public boolean containsValue(Object value) { + if (value == null) { + return false; + } + for (Pool pool : pools.values()) { + if (value.equals(pool.get())) { + return true; + } + } + return false; + } + + @Override + public void putAll(Map map) { + for (Map.Entry entry : map.entrySet()) { + put(entry.getKey(), entry.getValue()); + } + } + + @Override + public void clear() { + for (Pool pool : pools.values()) { + pool.clear(); + } + pools.clear(); + } + + @Override + public Set keySet() { + return pools.keySet(); + } + + @Override + public Set> entrySet() { + Set> entries = new HashSet>(); + for (Map.Entry> poolEntry : pools.entrySet()) { + final K poolKey = poolEntry.getKey(); + final Pool pool = poolEntry.getValue(); + for (final V poolValue : pool.values()) { + if (pool != null) { + entries.add(new Map.Entry() { + @Override + public K getKey() { + return poolKey; + } + + @Override + public V getValue() { + return poolValue; + } + + @Override + public V setValue(V value) { + return pool.put(value); + } + }); + } + } + } + return null; + } + + protected interface Pool { + public R get(); + + public R put(R resource); + + public boolean remove(R resource); + + public void clear(); + + public Collection values(); + + public int size(); + } + + public enum PoolType { + Reusable, ThreadLocal, RoundRobin; + + public static PoolType valueOf(String poolTypeName, + PoolType defaultPoolType, PoolType... allowedPoolTypes) { + PoolType poolType = PoolType.fuzzyMatch(poolTypeName); + if (poolType != null) { + boolean allowedType = false; + if (poolType.equals(defaultPoolType)) { + allowedType = true; + } else { + if (allowedPoolTypes != null) { + for (PoolType allowedPoolType : allowedPoolTypes) { + if (poolType.equals(allowedPoolType)) { + allowedType = true; + break; + } + } + } + } + if (!allowedType) { + poolType = null; + } + } + return (poolType != null) ? poolType : defaultPoolType; + } + + public static String fuzzyNormalize(String name) { + return name != null ? name.replaceAll("-", "").trim().toLowerCase() : ""; + } + + public static PoolType fuzzyMatch(String name) { + for (PoolType poolType : values()) { + if (fuzzyNormalize(name).equals(fuzzyNormalize(poolType.name()))) { + return poolType; + } + } + return null; + } + } + + protected Pool createPool() { + switch (poolType) { + case Reusable: + return new ReusablePool(poolMaxSize); + case RoundRobin: + return new RoundRobinPool(poolMaxSize); + case ThreadLocal: + return new ThreadLocalPool(); + } + return null; + } + + /** + * The ReusablePool represents a {@link PoolMap.Pool} that builds + * on the {@link LinkedList} class. It essentially allows resources to be + * checked out, at which point it is removed from this pool. When the resource + * is no longer required, it should be returned to the pool in order to be + * reused. + * + *

    + * If {@link #maxSize} is set to {@link Integer#MAX_VALUE}, then the size of + * the pool is unbounded. Otherwise, it caps the number of consumers that can + * check out a resource from this pool to the (non-zero positive) value + * specified in {@link #maxSize}. + *

    + * + * @param + * the type of the resource + */ + @SuppressWarnings("serial") + public class ReusablePool extends ConcurrentLinkedQueue implements Pool { + private int maxSize; + + public ReusablePool(int maxSize) { + this.maxSize = maxSize; + + } + + @Override + public R get() { + return poll(); + } + + @Override + public R put(R resource) { + if (size() < maxSize) { + add(resource); + } + return null; + } + + @Override + public Collection values() { + return this; + } + } + + /** + * The RoundRobinPool represents a {@link PoolMap.Pool}, which + * stores its resources in an {@link ArrayList}. It load-balances access to + * its resources by returning a different resource every time a given key is + * looked up. + * + *

    + * If {@link #maxSize} is set to {@link Integer#MAX_VALUE}, then the size of + * the pool is unbounded. Otherwise, it caps the number of resources in this + * pool to the (non-zero positive) value specified in {@link #maxSize}. + *

    + * + * @param + * the type of the resource + * + */ + @SuppressWarnings("serial") + class RoundRobinPool extends CopyOnWriteArrayList implements Pool { + private int maxSize; + private int nextResource = 0; + + public RoundRobinPool(int maxSize) { + this.maxSize = maxSize; + } + + @Override + public R put(R resource) { + if (size() < maxSize) { + add(resource); + } + return null; + } + + @Override + public R get() { + if (size() < maxSize) { + return null; + } + nextResource %= size(); + R resource = get(nextResource++); + return resource; + } + + @Override + public Collection values() { + return this; + } + + } + + /** + * The ThreadLocalPool represents a {@link PoolMap.Pool} that + * builds on the {@link ThreadLocal} class. It essentially binds the resource + * to the thread from which it is accessed. + * + *

    + * Note that the size of the pool is essentially bounded by the number of threads + * that add resources to this pool. + *

    + * + * @param + * the type of the resource + */ + static class ThreadLocalPool extends ThreadLocal implements Pool { + private static final Map, AtomicInteger> poolSizes = new HashMap, AtomicInteger>(); + + public ThreadLocalPool() { + } + + @Override + public R put(R resource) { + R previousResource = get(); + if (previousResource == null) { + AtomicInteger poolSize = poolSizes.get(this); + if (poolSize == null) { + poolSizes.put(this, poolSize = new AtomicInteger(0)); + } + poolSize.incrementAndGet(); + } + this.set(resource); + return previousResource; + } + + @Override + public void remove() { + super.remove(); + AtomicInteger poolSize = poolSizes.get(this); + if (poolSize != null) { + poolSize.decrementAndGet(); + } + } + + @Override + public int size() { + AtomicInteger poolSize = poolSizes.get(this); + return poolSize != null ? poolSize.get() : 0; + } + + @Override + public boolean remove(R resource) { + R previousResource = super.get(); + if (resource != null && resource.equals(previousResource)) { + remove(); + return true; + } else { + return false; + } + } + + @Override + public void clear() { + super.remove(); + } + + @Override + public Collection values() { + List values = new ArrayList(); + values.add(get()); + return values; + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/ProtoUtil.java b/src/main/java/org/apache/hadoop/hbase/util/ProtoUtil.java new file mode 100644 index 0000000..9212967 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/ProtoUtil.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.DataInput; +import java.io.IOException; + +public abstract class ProtoUtil { + + /** + * Read a variable length integer in the same format that ProtoBufs encodes. + * @param in the input stream to read from + * @return the integer + * @throws IOException if it is malformed or EOF. + */ + public static int readRawVarint32(DataInput in) throws IOException { + byte tmp = in.readByte(); + if (tmp >= 0) { + return tmp; + } + int result = tmp & 0x7f; + if ((tmp = in.readByte()) >= 0) { + result |= tmp << 7; + } else { + result |= (tmp & 0x7f) << 7; + if ((tmp = in.readByte()) >= 0) { + result |= tmp << 14; + } else { + result |= (tmp & 0x7f) << 14; + if ((tmp = in.readByte()) >= 0) { + result |= tmp << 21; + } else { + result |= (tmp & 0x7f) << 21; + result |= (tmp = in.readByte()) << 28; + if (tmp < 0) { + // Discard upper 32 bits. + for (int i = 0; i < 5; i++) { + if (in.readByte() >= 0) { + return result; + } + } + throw new IOException("Malformed varint"); + } + } + } + } + return result; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java b/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java new file mode 100644 index 0000000..449e2b0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java @@ -0,0 +1,236 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Map.Entry; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.TreeMultimap; + +/** + * This is a generic region split calculator. It requires Ranges that provide + * start, end, and a comparator. It works in two phases -- the first adds ranges + * and rejects backwards ranges. Then one calls calcRegions to generate the + * multimap that has a start split key as a key and possibly multiple Ranges as + * members. + * + * To traverse, one normally would get the split set, and iterate through the + * calcRegions. Normal regions would have only one entry, holes would have zero, + * and any overlaps would have multiple entries. + * + * The interface is a bit cumbersome currently but is exposed this way so that + * clients can choose how to iterate through the region splits. + * + * @param + */ +public class RegionSplitCalculator { + final static Log LOG = LogFactory.getLog(RegionSplitCalculator.class); + + private final Comparator rangeCmp; + /** + * This contains a sorted set of all the possible split points + * + * Invariant: once populated this has 0 entries if empty or at most n+1 values + * where n == number of added ranges. + */ + private final TreeSet splits = new TreeSet(BYTES_COMPARATOR); + + /** + * This is a map from start key to regions with the same start key. + * + * Invariant: This always have n values in total + */ + private final Multimap starts = ArrayListMultimap.create(); + + /** + * SPECIAL CASE + */ + private final static byte[] ENDKEY = null; + + public RegionSplitCalculator(Comparator cmp) { + rangeCmp = cmp; + } + + public final static Comparator BYTES_COMPARATOR = new ByteArrayComparator() { + @Override + public int compare(byte[] l, byte[] r) { + if (l == null && r == null) + return 0; + if (l == null) + return 1; + if (r == null) + return -1; + return super.compare(l, r); + } + }; + + /** + * SPECIAL CASE wrapper for empty end key + * + * @return ENDKEY if end key is empty, else normal endkey. + */ + private static byte[] specialEndKey(R range) { + byte[] end = range.getEndKey(); + if (end.length == 0) { + return ENDKEY; + } + return end; + } + + /** + * Adds an edge to the split calculator + * + * @return true if is included, false if backwards/invalid + */ + public boolean add(R range) { + byte[] start = range.getStartKey(); + byte[] end = specialEndKey(range); + + if (end != ENDKEY && Bytes.compareTo(start, end) > 0) { + // don't allow backwards edges + LOG.debug("attempted to add backwards edge: " + Bytes.toString(start) + + " " + Bytes.toString(end)); + return false; + } + + splits.add(start); + splits.add(end); + starts.put(start, range); + return true; + } + + /** + * Generates a coverage multimap from split key to Regions that start with the + * split key. + * + * @return coverage multimap + */ + public Multimap calcCoverage() { + // This needs to be sorted to force the use of the comparator on the values, + // otherwise byte array comparison isn't used + Multimap regions = TreeMultimap.create(BYTES_COMPARATOR, + rangeCmp); + + // march through all splits from the start points + for (Entry> start : starts.asMap().entrySet()) { + byte[] key = start.getKey(); + for (R r : start.getValue()) { + regions.put(key, r); + + for (byte[] coveredSplit : splits.subSet(r.getStartKey(), + specialEndKey(r))) { + regions.put(coveredSplit, r); + } + } + } + return regions; + } + + public TreeSet getSplits() { + return splits; + } + + public Multimap getStarts() { + return starts; + } + + /** + * Find specified number of top ranges in a big overlap group. + * It could return less if there are not that many top ranges. + * Once these top ranges are excluded, the big overlap group will + * be broken into ranges with no overlapping, or smaller overlapped + * groups, and most likely some holes. + * + * @param bigOverlap a list of ranges that overlap with each other + * @param count the max number of ranges to find + * @return a list of ranges that overlap with most others + */ + public static List + findBigRanges(Collection bigOverlap, int count) { + List bigRanges = new ArrayList(); + + // The key is the count of overlaps, + // The value is a list of ranges that have that many overlaps + TreeMap> overlapRangeMap = new TreeMap>(); + for (R r: bigOverlap) { + // Calculates the # of overlaps for each region + // and populates rangeOverlapMap + byte[] startKey = r.getStartKey(); + byte[] endKey = specialEndKey(r); + + int overlappedRegions = 0; + for (R rr: bigOverlap) { + byte[] start = rr.getStartKey(); + byte[] end = specialEndKey(rr); + + if (BYTES_COMPARATOR.compare(startKey, end) < 0 + && BYTES_COMPARATOR.compare(endKey, start) > 0) { + overlappedRegions++; + } + } + + // One region always overlaps with itself, + // so overlappedRegions should be more than 1 + // for actual overlaps. + if (overlappedRegions > 1) { + Integer key = Integer.valueOf(overlappedRegions); + List ranges = overlapRangeMap.get(key); + if (ranges == null) { + ranges = new ArrayList(); + overlapRangeMap.put(key, ranges); + } + ranges.add(r); + } + } + int toBeAdded = count; + for (Integer key: overlapRangeMap.descendingKeySet()) { + List chunk = overlapRangeMap.get(key); + int chunkSize = chunk.size(); + if (chunkSize <= toBeAdded) { + bigRanges.addAll(chunk); + toBeAdded -= chunkSize; + if (toBeAdded > 0) continue; + } else { + // Try to use the middle chunk in case the overlapping is + // chained, for example: [a, c), [b, e), [d, g), [f h)... + // In such a case, sideline the middle chunk will break + // the group efficiently. + int start = (chunkSize - toBeAdded)/2; + int end = start + toBeAdded; + for (int i = start; i < end; i++) { + bigRanges.add(chunk.get(i)); + } + } + break; + } + return bigRanges; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java new file mode 100644 index 0000000..619a715 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java @@ -0,0 +1,1022 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.NoServerForRegionException; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +/** + * The {@link RegionSplitter} class provides several utilities to help in the + * administration lifecycle for developers who choose to manually split regions + * instead of having HBase handle that automatically. The most useful utilities + * are: + *

    + *

      + *
    • Create a table with a specified number of pre-split regions + *
    • Execute a rolling split of all regions on an existing table + *
    + *

    + * Both operations can be safely done on a live server. + *

    + * Question: How do I turn off automatic splitting?
    + * Answer: Automatic splitting is determined by the configuration value + * HConstants.HREGION_MAX_FILESIZE. It is not recommended that you set this + * to Long.MAX_VALUE in case you forget about manual splits. A suggested setting + * is 100GB, which would result in > 1hr major compactions if reached. + *

    + * Question: Why did the original authors decide to manually split?
    + * Answer: Specific workload characteristics of our use case allowed us + * to benefit from a manual split system. + *

    + *

      + *
    • Data (~1k) that would grow instead of being replaced + *
    • Data growth was roughly uniform across all regions + *
    • OLTP workload. Data loss is a big deal. + *
    + *

    + * Question: Why is manual splitting good for this workload?
    + * Answer: Although automated splitting is not a bad option, there are + * benefits to manual splitting. + *

    + *

      + *
    • With growing amounts of data, splits will continually be needed. Since + * you always know exactly what regions you have, long-term debugging and + * profiling is much easier with manual splits. It is hard to trace the logs to + * understand region level problems if it keeps splitting and getting renamed. + *
    • Data offlining bugs + unknown number of split regions == oh crap! If an + * HLog or StoreFile was mistakenly unprocessed by HBase due to a weird bug and + * you notice it a day or so later, you can be assured that the regions + * specified in these files are the same as the current regions and you have + * less headaches trying to restore/replay your data. + *
    • You can finely tune your compaction algorithm. With roughly uniform data + * growth, it's easy to cause split / compaction storms as the regions all + * roughly hit the same data size at the same time. With manual splits, you can + * let staggered, time-based major compactions spread out your network IO load. + *
    + *

    + * Question: What's the optimal number of pre-split regions to create?
    + * Answer: Mileage will vary depending upon your application. + *

    + * The short answer for our application is that we started with 10 pre-split + * regions / server and watched our data growth over time. It's better to err on + * the side of too little regions and rolling split later. + *

    + * The more complicated answer is that this depends upon the largest storefile + * in your region. With a growing data size, this will get larger over time. You + * want the largest region to be just big enough that the {@link Store} compact + * selection algorithm only compacts it due to a timed major. If you don't, your + * cluster can be prone to compaction storms as the algorithm decides to run + * major compactions on a large series of regions all at once. Note that + * compaction storms are due to the uniform data growth, not the manual split + * decision. + *

    + * If you pre-split your regions too thin, you can increase the major compaction + * interval by configuring HConstants.MAJOR_COMPACTION_PERIOD. If your data size + * grows too large, use this script to perform a network IO safe rolling split + * of all regions. + */ +public class RegionSplitter { + static final Log LOG = LogFactory.getLog(RegionSplitter.class); + + /** + * A generic interface for the RegionSplitter code to use for all it's + * functionality. Note that the original authors of this code use + * {@link HexStringSplit} to partition their table and set it as default, but + * provided this for your custom algorithm. To use, create a new derived class + * from this interface and call {@link RegionSplitter#createPresplitTable} or + * {@link RegionSplitter#rollingSplit(String, String, Configuration)} with the + * argument splitClassName giving the name of your class. + */ + public static interface SplitAlgorithm { + /** + * Split a pre-existing region into 2 regions. + * + * @param start + * first row (inclusive) + * @param end + * last row (exclusive) + * @return the split row to use + */ + byte[] split(byte[] start, byte[] end); + + /** + * Split an entire table. + * + * @param numRegions + * number of regions to split the table into + * + * @throws RuntimeException + * user input is validated at this time. may throw a runtime + * exception in response to a parse failure + * @return array of split keys for the initial regions of the table. The + * length of the returned array should be numRegions-1. + */ + byte[][] split(int numRegions); + + /** + * In HBase, the first row is represented by an empty byte array. This might + * cause problems with your split algorithm or row printing. All your APIs + * will be passed firstRow() instead of empty array. + * + * @return your representation of your first row + */ + byte[] firstRow(); + + /** + * In HBase, the last row is represented by an empty byte array. This might + * cause problems with your split algorithm or row printing. All your APIs + * will be passed firstRow() instead of empty array. + * + * @return your representation of your last row + */ + byte[] lastRow(); + + /** + * In HBase, the last row is represented by an empty byte array. Set this + * value to help the split code understand how to evenly divide the first + * region. + * + * @param userInput + * raw user input (may throw RuntimeException on parse failure) + */ + void setFirstRow(String userInput); + + /** + * In HBase, the last row is represented by an empty byte array. Set this + * value to help the split code understand how to evenly divide the last + * region. Note that this last row is inclusive for all rows sharing the + * same prefix. + * + * @param userInput + * raw user input (may throw RuntimeException on parse failure) + */ + void setLastRow(String userInput); + + /** + * @param input + * user or file input for row + * @return byte array representation of this row for HBase + */ + byte[] strToRow(String input); + + /** + * @param row + * byte array representing a row in HBase + * @return String to use for debug & file printing + */ + String rowToStr(byte[] row); + + /** + * @return the separator character to use when storing / printing the row + */ + String separator(); + } + + /** + * The main function for the RegionSplitter application. Common uses: + *

    + *

      + *
    • create a table named 'myTable' with 60 pre-split regions containing 2 + * column families 'test' & 'rs', assuming the keys are hex-encoded ASCII: + *
        + *
      • bin/hbase org.apache.hadoop.hbase.util.RegionSplitter -c 60 -f test:rs + * myTable HexStringSplit + *
      + *
    • perform a rolling split of 'myTable' (i.e. 60 => 120 regions), # 2 + * outstanding splits at a time, assuming keys are uniformly distributed + * bytes: + *
        + *
      • bin/hbase org.apache.hadoop.hbase.util.RegionSplitter -r -o 2 myTable + * UniformSplit + *
      + *
    + * + * There are two SplitAlgorithms built into RegionSplitter, HexStringSplit + * and UniformSplit. These are different strategies for choosing region + * boundaries. See their source code for details. + * + * @param args + * Usage: RegionSplitter <TABLE> <SPLITALGORITHM> + * <-c <# regions> -f <family:family:...> | -r + * [-o <# outstanding splits>]> + * [-D <conf.param=value>] + * @throws IOException + * HBase IO problem + * @throws InterruptedException + * user requested exit + * @throws ParseException + * problem parsing user input + */ + @SuppressWarnings("static-access") + public static void main(String[] args) throws IOException, + InterruptedException, ParseException { + Configuration conf = HBaseConfiguration.create(); + + // parse user input + Options opt = new Options(); + opt.addOption(OptionBuilder.withArgName("property=value").hasArg() + .withDescription("Override HBase Configuration Settings").create("D")); + opt.addOption(OptionBuilder.withArgName("region count").hasArg() + .withDescription( + "Create a new table with a pre-split number of regions") + .create("c")); + opt.addOption(OptionBuilder.withArgName("family:family:...").hasArg() + .withDescription( + "Column Families to create with new table. Required with -c") + .create("f")); + opt.addOption("h", false, "Print this usage help"); + opt.addOption("r", false, "Perform a rolling split of an existing region"); + opt.addOption(OptionBuilder.withArgName("count").hasArg().withDescription( + "Max outstanding splits that have unfinished major compactions") + .create("o")); + opt.addOption(null, "firstrow", true, + "First Row in Table for Split Algorithm"); + opt.addOption(null, "lastrow", true, + "Last Row in Table for Split Algorithm"); + opt.addOption(null, "risky", false, + "Skip verification steps to complete quickly." + + "STRONGLY DISCOURAGED for production systems. "); + CommandLine cmd = new GnuParser().parse(opt, args); + + if (cmd.hasOption("D")) { + for (String confOpt : cmd.getOptionValues("D")) { + String[] kv = confOpt.split("=", 2); + if (kv.length == 2) { + conf.set(kv[0], kv[1]); + LOG.debug("-D configuration override: " + kv[0] + "=" + kv[1]); + } else { + throw new ParseException("-D option format invalid: " + confOpt); + } + } + } + + if (cmd.hasOption("risky")) { + conf.setBoolean("split.verify", false); + } + + boolean createTable = cmd.hasOption("c") && cmd.hasOption("f"); + boolean rollingSplit = cmd.hasOption("r"); + boolean oneOperOnly = createTable ^ rollingSplit; + + if (2 != cmd.getArgList().size() || !oneOperOnly || cmd.hasOption("h")) { + new HelpFormatter().printHelp("RegionSplitter
    \n"+ + "SPLITALGORITHM is a java class name of a class implementing " + + "SplitAlgorithm, or one of the special strings HexStringSplit " + + "or UniformSplit, which are built-in split algorithms. " + + "HexStringSplit treats keys as hexadecimal ASCII, and " + + "UniformSplit treats keys as arbitrary bytes.", opt); + return; + } + String tableName = cmd.getArgs()[0]; + String splitClass = cmd.getArgs()[1]; + SplitAlgorithm splitAlgo = newSplitAlgoInstance(conf, splitClass); + + if (cmd.hasOption("firstrow")) { + splitAlgo.setFirstRow(cmd.getOptionValue("firstrow")); + } + if (cmd.hasOption("lastrow")) { + splitAlgo.setLastRow(cmd.getOptionValue("lastrow")); + } + + if (createTable) { + conf.set("split.count", cmd.getOptionValue("c")); + createPresplitTable(tableName, splitAlgo, cmd.getOptionValue("f").split(":"), conf); + } + + if (rollingSplit) { + if (cmd.hasOption("o")) { + conf.set("split.outstanding", cmd.getOptionValue("o")); + } + rollingSplit(tableName, splitAlgo, conf); + } + } + + static void createPresplitTable(String tableName, SplitAlgorithm splitAlgo, + String[] columnFamilies, Configuration conf) throws IOException, + InterruptedException { + final int splitCount = conf.getInt("split.count", 0); + Preconditions.checkArgument(splitCount > 1, "Split count must be > 1"); + + Preconditions.checkArgument(columnFamilies.length > 0, + "Must specify at least one column family. "); + LOG.debug("Creating table " + tableName + " with " + columnFamilies.length + + " column families. Presplitting to " + splitCount + " regions"); + + HTableDescriptor desc = new HTableDescriptor(tableName); + for (String cf : columnFamilies) { + desc.addFamily(new HColumnDescriptor(Bytes.toBytes(cf))); + } + HBaseAdmin admin = new HBaseAdmin(conf); + Preconditions.checkArgument(!admin.tableExists(tableName), + "Table already exists: " + tableName); + admin.createTable(desc, splitAlgo.split(splitCount)); + LOG.debug("Table created! Waiting for regions to show online in META..."); + + if (!conf.getBoolean("split.verify", true)) { + // NOTE: createTable is synchronous on the table, but not on the regions + HTable table = new HTable(conf, tableName); + int onlineRegions = 0; + while (onlineRegions < splitCount) { + onlineRegions = table.getRegionsInfo().size(); + LOG.debug(onlineRegions + " of " + splitCount + " regions online..."); + if (onlineRegions < splitCount) { + Thread.sleep(10 * 1000); // sleep + } + } + table.close(); + } + + LOG.debug("Finished creating table with " + splitCount + " regions"); + } + + static void rollingSplit(String tableName, SplitAlgorithm splitAlgo, + Configuration conf) throws IOException, InterruptedException { + final int minOS = conf.getInt("split.outstanding", 2); + + HTable table = new HTable(conf, tableName); + + // max outstanding splits. default == 50% of servers + final int MAX_OUTSTANDING = + Math.max(table.getConnection().getCurrentNrHRS() / 2, minOS); + + Path hbDir = new Path(conf.get(HConstants.HBASE_DIR)); + Path tableDir = HTableDescriptor.getTableDir(hbDir, table.getTableName()); + Path splitFile = new Path(tableDir, "_balancedSplit"); + FileSystem fs = FileSystem.get(conf); + + // get a list of daughter regions to create + LinkedList> tmpRegionSet = getSplits(table, splitAlgo); + LinkedList> outstanding = Lists.newLinkedList(); + int splitCount = 0; + final int origCount = tmpRegionSet.size(); + + // all splits must compact & we have 1 compact thread, so 2 split + // requests to the same RS can stall the outstanding split queue. + // To fix, group the regions into an RS pool and round-robin through it + LOG.debug("Bucketing regions by regionserver..."); + TreeMap>> daughterRegions = + Maps.newTreeMap(); + for (Pair dr : tmpRegionSet) { + String rsLocation = table.getRegionLocation(dr.getSecond()). + getHostnamePort(); + if (!daughterRegions.containsKey(rsLocation)) { + LinkedList> entry = Lists.newLinkedList(); + daughterRegions.put(rsLocation, entry); + } + daughterRegions.get(rsLocation).add(dr); + } + LOG.debug("Done with bucketing. Split time!"); + long startTime = System.currentTimeMillis(); + + // open the split file and modify it as splits finish + FSDataInputStream tmpIn = fs.open(splitFile); + byte[] rawData = new byte[tmpIn.available()]; + tmpIn.readFully(rawData); + tmpIn.close(); + FSDataOutputStream splitOut = fs.create(splitFile); + splitOut.write(rawData); + + try { + // *** split code *** + while (!daughterRegions.isEmpty()) { + LOG.debug(daughterRegions.size() + " RS have regions to splt."); + + // Get RegionServer : region count mapping + final TreeMap rsSizes = Maps.newTreeMap(); + Map regionsInfo = table.getRegionLocations(); + for (ServerName rs : regionsInfo.values()) { + if (rsSizes.containsKey(rs)) { + rsSizes.put(rs, rsSizes.get(rs) + 1); + } else { + rsSizes.put(rs, 1); + } + } + + // sort the RS by the number of regions they have + List serversLeft = Lists.newArrayList(daughterRegions .keySet()); + Collections.sort(serversLeft, new Comparator() { + public int compare(String o1, String o2) { + return rsSizes.get(o1).compareTo(rsSizes.get(o2)); + } + }); + + // round-robin through the RS list. Choose the lightest-loaded servers + // first to keep the master from load-balancing regions as we split. + for (String rsLoc : serversLeft) { + Pair dr = null; + + // find a region in the RS list that hasn't been moved + LOG.debug("Finding a region on " + rsLoc); + LinkedList> regionList = daughterRegions + .get(rsLoc); + while (!regionList.isEmpty()) { + dr = regionList.pop(); + + // get current region info + byte[] split = dr.getSecond(); + HRegionLocation regionLoc = table.getRegionLocation(split); + + // if this region moved locations + String newRs = regionLoc.getHostnamePort(); + if (newRs.compareTo(rsLoc) != 0) { + LOG.debug("Region with " + splitAlgo.rowToStr(split) + + " moved to " + newRs + ". Relocating..."); + // relocate it, don't use it right now + if (!daughterRegions.containsKey(newRs)) { + LinkedList> entry = Lists.newLinkedList(); + daughterRegions.put(newRs, entry); + } + daughterRegions.get(newRs).add(dr); + dr = null; + continue; + } + + // make sure this region wasn't already split + byte[] sk = regionLoc.getRegionInfo().getStartKey(); + if (sk.length != 0) { + if (Bytes.equals(split, sk)) { + LOG.debug("Region already split on " + + splitAlgo.rowToStr(split) + ". Skipping this region..."); + ++splitCount; + dr = null; + continue; + } + byte[] start = dr.getFirst(); + Preconditions.checkArgument(Bytes.equals(start, sk), splitAlgo + .rowToStr(start) + " != " + splitAlgo.rowToStr(sk)); + } + + // passed all checks! found a good region + break; + } + if (regionList.isEmpty()) { + daughterRegions.remove(rsLoc); + } + if (dr == null) + continue; + + // we have a good region, time to split! + byte[] split = dr.getSecond(); + LOG.debug("Splitting at " + splitAlgo.rowToStr(split)); + HBaseAdmin admin = new HBaseAdmin(table.getConfiguration()); + admin.split(table.getTableName(), split); + + LinkedList> finished = Lists.newLinkedList(); + if (conf.getBoolean("split.verify", true)) { + // we need to verify and rate-limit our splits + outstanding.addLast(dr); + // with too many outstanding splits, wait for some to finish + while (outstanding.size() >= MAX_OUTSTANDING) { + finished = splitScan(outstanding, table, splitAlgo); + if (finished.isEmpty()) { + Thread.sleep(30 * 1000); + } else { + outstanding.removeAll(finished); + } + } + } else { + finished.add(dr); + } + + // mark each finished region as successfully split. + for (Pair region : finished) { + splitOut.writeChars("- " + splitAlgo.rowToStr(region.getFirst()) + + " " + splitAlgo.rowToStr(region.getSecond()) + "\n"); + splitCount++; + if (splitCount % 10 == 0) { + long tDiff = (System.currentTimeMillis() - startTime) + / splitCount; + LOG.debug("STATUS UPDATE: " + splitCount + " / " + origCount + + ". Avg Time / Split = " + + org.apache.hadoop.util.StringUtils.formatTime(tDiff)); + } + } + } + } + if (conf.getBoolean("split.verify", true)) { + while (!outstanding.isEmpty()) { + LinkedList> finished = splitScan(outstanding, + table, splitAlgo); + if (finished.isEmpty()) { + Thread.sleep(30 * 1000); + } else { + outstanding.removeAll(finished); + for (Pair region : finished) { + splitOut.writeChars("- " + splitAlgo.rowToStr(region.getFirst()) + + " " + splitAlgo.rowToStr(region.getSecond()) + "\n"); + } + } + } + } + LOG.debug("All regions have been successfully split!"); + } finally { + long tDiff = System.currentTimeMillis() - startTime; + LOG.debug("TOTAL TIME = " + + org.apache.hadoop.util.StringUtils.formatTime(tDiff)); + LOG.debug("Splits = " + splitCount); + LOG.debug("Avg Time / Split = " + + org.apache.hadoop.util.StringUtils.formatTime(tDiff / splitCount)); + + splitOut.close(); + if (table != null){ + table.close(); + } + } + fs.delete(splitFile, false); + } + + /** + * @throws IOException if the specified SplitAlgorithm class couldn't be + * instantiated + */ + public static SplitAlgorithm newSplitAlgoInstance(Configuration conf, + String splitClassName) throws IOException { + Class splitClass; + + // For split algorithms builtin to RegionSplitter, the user can specify + // their simple class name instead of a fully qualified class name. + if(splitClassName.equals(HexStringSplit.class.getSimpleName())) { + splitClass = HexStringSplit.class; + } else if (splitClassName.equals(UniformSplit.class.getSimpleName())) { + splitClass = UniformSplit.class; + } else { + try { + splitClass = conf.getClassByName(splitClassName); + } catch (ClassNotFoundException e) { + throw new IOException("Couldn't load split class " + splitClassName, e); + } + if(splitClass == null) { + throw new IOException("Failed loading split class " + splitClassName); + } + if(!SplitAlgorithm.class.isAssignableFrom(splitClass)) { + throw new IOException( + "Specified split class doesn't implement SplitAlgorithm"); + } + } + try { + return splitClass.asSubclass(SplitAlgorithm.class).newInstance(); + } catch (Exception e) { + throw new IOException("Problem loading split algorithm: ", e); + } + } + + static LinkedList> splitScan( + LinkedList> regionList, HTable table, + SplitAlgorithm splitAlgo) + throws IOException, InterruptedException { + LinkedList> finished = Lists.newLinkedList(); + LinkedList> logicalSplitting = Lists.newLinkedList(); + LinkedList> physicalSplitting = Lists.newLinkedList(); + + // get table info + Path hbDir = new Path(table.getConfiguration().get(HConstants.HBASE_DIR)); + Path tableDir = HTableDescriptor.getTableDir(hbDir, table.getTableName()); + Path splitFile = new Path(tableDir, "_balancedSplit"); + FileSystem fs = tableDir.getFileSystem(table.getConfiguration()); + + // clear the cache to forcibly refresh region information + table.clearRegionCache(); + + // for every region that hasn't been verified as a finished split + for (Pair region : regionList) { + byte[] start = region.getFirst(); + byte[] split = region.getSecond(); + + // see if the new split daughter region has come online + try { + HRegionInfo dri = table.getRegionLocation(split).getRegionInfo(); + if (dri.isOffline() || !Bytes.equals(dri.getStartKey(), split)) { + logicalSplitting.add(region); + continue; + } + } catch (NoServerForRegionException nsfre) { + // NSFRE will occur if the old META entry has no server assigned + LOG.info(nsfre); + logicalSplitting.add(region); + continue; + } + + try { + // when a daughter region is opened, a compaction is triggered + // wait until compaction completes for both daughter regions + LinkedList check = Lists.newLinkedList(); + check.add(table.getRegionLocation(start).getRegionInfo()); + check.add(table.getRegionLocation(split).getRegionInfo()); + for (HRegionInfo hri : check.toArray(new HRegionInfo[] {})) { + boolean refFound = false; + byte[] sk = hri.getStartKey(); + if (sk.length == 0) + sk = splitAlgo.firstRow(); + String startKey = splitAlgo.rowToStr(sk); + HTableDescriptor htd = table.getTableDescriptor(); + // check every Column Family for that region + for (HColumnDescriptor c : htd.getFamilies()) { + Path cfDir = Store.getStoreHomedir(tableDir, hri.getEncodedName(), + c.getName()); + if (fs.exists(cfDir)) { + for (FileStatus file : fs.listStatus(cfDir)) { + refFound |= StoreFile.isReference(file.getPath()); + if (refFound) + break; + } + } + if (refFound) + break; + } + // compaction is completed when all reference files are gone + if (!refFound) { + check.remove(hri); + } + } + if (check.isEmpty()) { + finished.add(region); + } else { + physicalSplitting.add(region); + } + } catch (NoServerForRegionException nsfre) { + LOG.debug("No Server Exception thrown for: " + + splitAlgo.rowToStr(start)); + physicalSplitting.add(region); + table.clearRegionCache(); + } + } + + LOG.debug("Split Scan: " + finished.size() + " finished / " + + logicalSplitting.size() + " split wait / " + + physicalSplitting.size() + " reference wait"); + + return finished; + } + + static LinkedList> getSplits(HTable table, + SplitAlgorithm splitAlgo) throws IOException { + Path hbDir = new Path(table.getConfiguration().get(HConstants.HBASE_DIR)); + Path tableDir = HTableDescriptor.getTableDir(hbDir, table.getTableName()); + Path splitFile = new Path(tableDir, "_balancedSplit"); + FileSystem fs = tableDir.getFileSystem(table.getConfiguration()); + + // using strings because (new byte[]{0}).equals(new byte[]{0}) == false + Set> daughterRegions = Sets.newHashSet(); + + // does a split file exist? + if (!fs.exists(splitFile)) { + // NO = fresh start. calculate splits to make + LOG.debug("No _balancedSplit file. Calculating splits..."); + + // query meta for all regions in the table + Set> rows = Sets.newHashSet(); + Pair tmp = table.getStartEndKeys(); + Preconditions.checkArgument( + tmp.getFirst().length == tmp.getSecond().length, + "Start and End rows should be equivalent"); + for (int i = 0; i < tmp.getFirst().length; ++i) { + byte[] start = tmp.getFirst()[i], end = tmp.getSecond()[i]; + if (start.length == 0) + start = splitAlgo.firstRow(); + if (end.length == 0) + end = splitAlgo.lastRow(); + rows.add(Pair.newPair(start, end)); + } + LOG.debug("Table " + Bytes.toString(table.getTableName()) + " has " + + rows.size() + " regions that will be split."); + + // prepare the split file + Path tmpFile = new Path(tableDir, "_balancedSplit_prepare"); + FSDataOutputStream tmpOut = fs.create(tmpFile); + + // calculate all the splits == [daughterRegions] = [(start, splitPoint)] + for (Pair r : rows) { + byte[] splitPoint = splitAlgo.split(r.getFirst(), r.getSecond()); + String startStr = splitAlgo.rowToStr(r.getFirst()); + String splitStr = splitAlgo.rowToStr(splitPoint); + daughterRegions.add(Pair.newPair(startStr, splitStr)); + LOG.debug("Will Split [" + startStr + " , " + + splitAlgo.rowToStr(r.getSecond()) + ") at " + splitStr); + tmpOut.writeChars("+ " + startStr + splitAlgo.separator() + splitStr + + "\n"); + } + tmpOut.close(); + fs.rename(tmpFile, splitFile); + } else { + LOG.debug("_balancedSplit file found. Replay log to restore state..."); + FSUtils.getInstance(fs, table.getConfiguration()) + .recoverFileLease(fs, splitFile, table.getConfiguration()); + + // parse split file and process remaining splits + FSDataInputStream tmpIn = fs.open(splitFile); + StringBuilder sb = new StringBuilder(tmpIn.available()); + while (tmpIn.available() > 0) { + sb.append(tmpIn.readChar()); + } + tmpIn.close(); + for (String line : sb.toString().split("\n")) { + String[] cmd = line.split(splitAlgo.separator()); + Preconditions.checkArgument(3 == cmd.length); + byte[] start = splitAlgo.strToRow(cmd[1]); + String startStr = splitAlgo.rowToStr(start); + byte[] splitPoint = splitAlgo.strToRow(cmd[2]); + String splitStr = splitAlgo.rowToStr(splitPoint); + Pair r = Pair.newPair(startStr, splitStr); + if (cmd[0].equals("+")) { + LOG.debug("Adding: " + r); + daughterRegions.add(r); + } else { + LOG.debug("Removing: " + r); + Preconditions.checkArgument(cmd[0].equals("-"), + "Unknown option: " + cmd[0]); + Preconditions.checkState(daughterRegions.contains(r), + "Missing row: " + r); + daughterRegions.remove(r); + } + } + LOG.debug("Done reading. " + daughterRegions.size() + " regions left."); + } + LinkedList> ret = Lists.newLinkedList(); + for (Pair r : daughterRegions) { + ret.add(Pair.newPair(splitAlgo.strToRow(r.getFirst()), splitAlgo + .strToRow(r.getSecond()))); + } + return ret; + } + + /** + * HexStringSplit is a well-known {@link SplitAlgorithm} for choosing region + * boundaries. The format of a HexStringSplit region boundary is the ASCII + * representation of an MD5 checksum, or any other uniformly distributed + * hexadecimal value. Row are hex-encoded long values in the range + * "00000000" => "FFFFFFFF" and are left-padded with zeros to keep the + * same order lexicographically as if they were binary. + * + * Since this split algorithm uses hex strings as keys, it is easy to read & + * write in the shell but takes up more space and may be non-intuitive. + */ + public static class HexStringSplit implements SplitAlgorithm { + final static String DEFAULT_MIN_HEX = "00000000"; + final static String DEFAULT_MAX_HEX = "FFFFFFFF"; + + String firstRow = DEFAULT_MIN_HEX; + BigInteger firstRowInt = BigInteger.ZERO; + String lastRow = DEFAULT_MAX_HEX; + BigInteger lastRowInt = new BigInteger(lastRow, 16); + int rowComparisonLength = lastRow.length(); + + public byte[] split(byte[] start, byte[] end) { + BigInteger s = convertToBigInteger(start); + BigInteger e = convertToBigInteger(end); + Preconditions.checkArgument(!e.equals(BigInteger.ZERO)); + return convertToByte(split2(s, e)); + } + + public byte[][] split(int n) { + Preconditions.checkArgument(lastRowInt.compareTo(firstRowInt) > 0, + "last row (%s) is configured less than first row (%s)", lastRow, + firstRow); + // +1 to range because the last row is inclusive + BigInteger range = lastRowInt.subtract(firstRowInt).add(BigInteger.ONE); + Preconditions.checkState(range.compareTo(BigInteger.valueOf(n)) >= 0, + "split granularity (%s) is greater than the range (%s)", n, range); + + BigInteger[] splits = new BigInteger[n - 1]; + BigInteger sizeOfEachSplit = range.divide(BigInteger.valueOf(n)); + for (int i = 1; i < n; i++) { + // NOTE: this means the last region gets all the slop. + // This is not a big deal if we're assuming n << MAXHEX + splits[i - 1] = firstRowInt.add(sizeOfEachSplit.multiply(BigInteger + .valueOf(i))); + } + return convertToBytes(splits); + } + + public byte[] firstRow() { + return convertToByte(firstRowInt); + } + + public byte[] lastRow() { + return convertToByte(lastRowInt); + } + + public void setFirstRow(String userInput) { + firstRow = userInput; + firstRowInt = new BigInteger(firstRow, 16); + } + + public void setLastRow(String userInput) { + lastRow = userInput; + lastRowInt = new BigInteger(lastRow, 16); + // Precondition: lastRow > firstRow, so last's length is the greater + rowComparisonLength = lastRow.length(); + } + + public byte[] strToRow(String in) { + return convertToByte(new BigInteger(in, 16)); + } + + public String rowToStr(byte[] row) { + return Bytes.toStringBinary(row); + } + + public String separator() { + return " "; + } + + /** + * Divide 2 numbers in half (for split algorithm) + * + * @param a number #1 + * @param b number #2 + * @return the midpoint of the 2 numbers + */ + public BigInteger split2(BigInteger a, BigInteger b) { + return a.add(b).divide(BigInteger.valueOf(2)).abs(); + } + + /** + * Returns an array of bytes corresponding to an array of BigIntegers + * + * @param bigIntegers numbers to convert + * @return bytes corresponding to the bigIntegers + */ + public byte[][] convertToBytes(BigInteger[] bigIntegers) { + byte[][] returnBytes = new byte[bigIntegers.length][]; + for (int i = 0; i < bigIntegers.length; i++) { + returnBytes[i] = convertToByte(bigIntegers[i]); + } + return returnBytes; + } + + /** + * Returns the bytes corresponding to the BigInteger + * + * @param bigInteger number to convert + * @param pad padding length + * @return byte corresponding to input BigInteger + */ + public static byte[] convertToByte(BigInteger bigInteger, int pad) { + String bigIntegerString = bigInteger.toString(16); + bigIntegerString = StringUtils.leftPad(bigIntegerString, pad, '0'); + return Bytes.toBytes(bigIntegerString); + } + + /** + * Returns the bytes corresponding to the BigInteger + * + * @param bigInteger number to convert + * @return corresponding bytes + */ + public byte[] convertToByte(BigInteger bigInteger) { + return convertToByte(bigInteger, rowComparisonLength); + } + + /** + * Returns the BigInteger represented by the byte array + * + * @param row byte array representing row + * @return the corresponding BigInteger + */ + public BigInteger convertToBigInteger(byte[] row) { + return (row.length > 0) ? new BigInteger(Bytes.toString(row), 16) + : BigInteger.ZERO; + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + " [" + rowToStr(firstRow()) + + "," + rowToStr(lastRow()) + "]"; + } + } + + /** + * A SplitAlgorithm that divides the space of possible keys evenly. Useful + * when the keys are approximately uniform random bytes (e.g. hashes). Rows + * are raw byte values in the range 00 => FF and are right-padded with + * zeros to keep the same memcmp() order. This is the natural algorithm to use + * for a byte[] environment and saves space, but is not necessarily the + * easiest for readability. + */ + public static class UniformSplit implements SplitAlgorithm { + static final byte xFF = (byte) 0xFF; + byte[] firstRowBytes = ArrayUtils.EMPTY_BYTE_ARRAY; + byte[] lastRowBytes = + new byte[] {xFF, xFF, xFF, xFF, xFF, xFF, xFF, xFF}; + public byte[] split(byte[] start, byte[] end) { + return Bytes.split(start, end, 1)[1]; + } + + @Override + public byte[][] split(int numRegions) { + Preconditions.checkArgument( + Bytes.compareTo(lastRowBytes, firstRowBytes) > 0, + "last row (%s) is configured less than first row (%s)", + Bytes.toStringBinary(lastRowBytes), + Bytes.toStringBinary(firstRowBytes)); + + byte[][] splits = Bytes.split(firstRowBytes, lastRowBytes, true, + numRegions - 1); + Preconditions.checkState(splits != null, + "Could not split region with given user input: " + this); + + // remove endpoints, which are included in the splits list + return Arrays.copyOfRange(splits, 1, splits.length - 1); + } + + @Override + public byte[] firstRow() { + return firstRowBytes; + } + + @Override + public byte[] lastRow() { + return lastRowBytes; + } + + @Override + public void setFirstRow(String userInput) { + firstRowBytes = Bytes.toBytesBinary(userInput); + } + + @Override + public void setLastRow(String userInput) { + lastRowBytes = Bytes.toBytesBinary(userInput); + } + + @Override + public byte[] strToRow(String input) { + return Bytes.toBytesBinary(input); + } + + @Override + public String rowToStr(byte[] row) { + return Bytes.toStringBinary(row); + } + + @Override + public String separator() { + return ","; + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + " [" + rowToStr(firstRow()) + + "," + rowToStr(lastRow()) + "]"; + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java b/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java new file mode 100644 index 0000000..15e741d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java @@ -0,0 +1,68 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class RetryCounter { + private static final Log LOG = LogFactory.getLog(RetryCounter.class); + private final int maxRetries; + private int retriesRemaining; + private final int retryIntervalMillis; + private final TimeUnit timeUnit; + + public RetryCounter(int maxRetries, + int retryIntervalMillis, TimeUnit timeUnit) { + this.maxRetries = maxRetries; + this.retriesRemaining = maxRetries; + this.retryIntervalMillis = retryIntervalMillis; + this.timeUnit = timeUnit; + } + + public int getMaxRetries() { + return maxRetries; + } + + /** + * Sleep for a exponentially back off time + * @throws InterruptedException + */ + public void sleepUntilNextRetry() throws InterruptedException { + int attempts = getAttemptTimes(); + long sleepTime = (long) (retryIntervalMillis * Math.pow(2, attempts)); + LOG.info("Sleeping " + sleepTime + "ms before retry #" + attempts + "..."); + timeUnit.sleep(sleepTime); + } + + public boolean shouldRetry() { + return retriesRemaining > 0; + } + + public void useRetry() { + retriesRemaining--; + } + + public int getAttemptTimes() { + return maxRetries-retriesRemaining+1; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java b/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java new file mode 100644 index 0000000..445234e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java @@ -0,0 +1,38 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.concurrent.TimeUnit; + +public class RetryCounterFactory { + private final int maxRetries; + private final int retryIntervalMillis; + + public RetryCounterFactory(int maxRetries, int retryIntervalMillis) { + this.maxRetries = maxRetries; + this.retryIntervalMillis = retryIntervalMillis; + } + + public RetryCounter create() { + return new RetryCounter( + maxRetries, retryIntervalMillis, TimeUnit.MILLISECONDS + ); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java b/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java new file mode 100644 index 0000000..b2f3770 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java @@ -0,0 +1,82 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.lang.management.RuntimeMXBean; +import java.lang.management.ManagementFactory; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +/** + * Base class for command lines that start up various HBase daemons. + */ +public abstract class ServerCommandLine extends Configured implements Tool { + private static final Log LOG = LogFactory.getLog(ServerCommandLine.class); + + /** + * Implementing subclasses should return a usage string to print out. + */ + protected abstract String getUsage(); + + /** + * Print usage information for this command line. + * + * @param message if not null, print this message before the usage info. + */ + protected void usage(String message) { + if (message != null) { + System.err.println(message); + System.err.println(""); + } + + System.err.println(getUsage()); + } + + /** + * Log information about the currently running JVM. + */ + public static void logJVMInfo() { + // Print out vm stats before starting up. + RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean(); + if (runtime != null) { + LOG.info("vmName=" + runtime.getVmName() + ", vmVendor=" + + runtime.getVmVendor() + ", vmVersion=" + runtime.getVmVersion()); + LOG.info("vmInputArguments=" + runtime.getInputArguments()); + } + } + + /** + * Parse and run the given command line. This may exit the JVM if + * a nonzero exit code is returned from run(). + */ + public void doMain(String args[]) throws Exception { + int ret = ToolRunner.run( + HBaseConfiguration.create(), this, args); + if (ret != 0) { + System.exit(ret); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/ShutdownHookManager.java b/src/main/java/org/apache/hadoop/hbase/util/ShutdownHookManager.java new file mode 100644 index 0000000..ebf308c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/ShutdownHookManager.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; + +/** + * This class provides ShutdownHookManager shims for HBase to interact with the Hadoop 1.0.x and the + * Hadoop 2.0+ series. + * + * NOTE: No testing done against 0.22.x, or 0.21.x. + */ +abstract public class ShutdownHookManager { + private static ShutdownHookManager instance; + + static Class shutdownHookManagerClass = null; + static { + try { + // This class exists in hadoop 2.0+ but not in Hadoop 20.x/1.x + shutdownHookManagerClass = Class.forName("org.apache.hadoop.util.ShutdownHookManager"); + instance = new ShutdownHookManagerV2(); + } catch (Exception e) { + instance = new ShutdownHookManagerV1(); + } + } + + abstract public void addShutdownHook(Thread shutdownHook, int priority); + + abstract public boolean removeShutdownHook(Runnable shutdownHook); + + public static void affixShutdownHook(Thread shutdownHook, int priority) { + instance.addShutdownHook(shutdownHook, priority); + } + + public static boolean deleteShutdownHook(Runnable shutdownHook) { + return instance.removeShutdownHook(shutdownHook); + } + + private static class ShutdownHookManagerV1 extends ShutdownHookManager { + // priority is ignored in hadoop versions earlier than 2.0 + public void addShutdownHook(Thread shutdownHookThread, int priority) { + Runtime.getRuntime().addShutdownHook(shutdownHookThread); + } + + public boolean removeShutdownHook(Runnable shutdownHook) { + Thread shutdownHookThread = null; + if (!(shutdownHook instanceof Thread)) { + shutdownHookThread = new Thread(shutdownHook); + } else shutdownHookThread = (Thread) shutdownHook; + + return Runtime.getRuntime().removeShutdownHook(shutdownHookThread); + } + }; + + private static class ShutdownHookManagerV2 extends ShutdownHookManager { + public void addShutdownHook(Thread shutdownHookThread, int priority) { + try { + Methods.call(shutdownHookManagerClass, + Methods.call(shutdownHookManagerClass, null, "get", null, null), + "addShutdownHook", + new Class[] { Runnable.class, int.class }, + new Object[] { shutdownHookThread, priority }); + } catch (Exception ex) { + throw new RuntimeException("we could not use ShutdownHookManager.addShutdownHook", ex); + } + } + + public boolean removeShutdownHook(Runnable shutdownHook) { + try { + return (Boolean) + Methods.call(shutdownHookManagerClass, + Methods.call(shutdownHookManagerClass, null, "get", null, null), + "removeShutdownHook", + new Class[] { Runnable.class }, + new Object[] { shutdownHook }); + } catch (Exception ex) { + throw new RuntimeException("we could not use ShutdownHookManager", ex); + } + } + }; + +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/SizeBasedThrottler.java b/src/main/java/org/apache/hadoop/hbase/util/SizeBasedThrottler.java new file mode 100644 index 0000000..af57887 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/SizeBasedThrottler.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Utility class that can be used to implement + * queues with limited capacity (in terms of memory). + * It maintains internal counter and provides + * two operations: increase and decrease. + * Increase blocks until internal counter is lower than + * given threshold and then increases internal counter. + * Decrease decreases internal counter and wakes up + * waiting threads if counter is lower than threshold. + * + * This implementation allows you to set the value of internal + * counter to be greater than threshold. It happens + * when internal counter is lower than threshold and + * increase method is called with parameter 'delta' big enough + * so that sum of delta and internal counter is greater than + * threshold. This is not a bug, this is a feature. + * It solves some problems: + * - thread calling increase with big parameter will not be + * starved by other threads calling increase with small + * arguments. + * - thread calling increase with argument greater than + * threshold won't deadlock. This is useful when throttling + * queues - you can submit object that is bigger than limit. + * + * This implementation introduces small costs in terms of + * synchronization (no synchronization in most cases at all), but is + * vulnerable to races. For details see documentation of + * increase method. + */ +public class SizeBasedThrottler { + + private final long threshold; + private final AtomicLong currentSize; + + /** + * Creates SizeBoundary with provided threshold + * + * @param threshold threshold used by instance + */ + public SizeBasedThrottler(long threshold) { + if (threshold <= 0) { + throw new IllegalArgumentException("Treshold must be greater than 0"); + } + this.threshold = threshold; + this.currentSize = new AtomicLong(0); + } + + /** + * Blocks until internal counter is lower than threshold + * and then increases value of internal counter. + * + * THIS METHOD IS VULNERABLE TO RACES. + * It may happen that increment operation will + * succeed immediately, even if it should block. This happens when + * at least two threads call increase at the some moment. The decision + * whether to block is made at the beginning, without synchronization. + * If value of currentSize is lower than threshold at that time, call + * will succeed immediately. It is possible, that 2 threads will make + * decision not to block, even if one of them should block. + * + * @param delta increase internal counter by this value + * @return new value of internal counter + * @throws InterruptedException when interrupted during waiting + */ + public synchronized long increase(long delta) throws InterruptedException{ + if (currentSize.get() >= threshold) { + synchronized (this) { + while (currentSize.get() >= threshold) { + wait(); + } + } + } + + return currentSize.addAndGet(delta); + } + + + /** + * Decreases value of internal counter. Wakes up waiting threads if required. + * + * @param delta decrease internal counter by this value + * @return new value of internal counter + */ + public synchronized long decrease(long delta) { + final long newSize = currentSize.addAndGet(-delta); + + if (newSize < threshold && newSize + delta >= threshold) { + synchronized (this) { + notifyAll(); + } + } + + return newSize; + } + + /** + * + * @return current value of internal counter + */ + public synchronized long getCurrentValue(){ + return currentSize.get(); + } + + /** + * @return threshold + */ + public long getThreshold(){ + return threshold; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java b/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java new file mode 100644 index 0000000..a8d7d68 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java @@ -0,0 +1,114 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Stoppable; + +/** + * Sleeper for current thread. + * Sleeps for passed period. Also checks passed boolean and if interrupted, + * will return if the flag is set (rather than go back to sleep until its + * sleep time is up). + */ +public class Sleeper { + private final Log LOG = LogFactory.getLog(this.getClass().getName()); + private final int period; + private final Stoppable stopper; + private static final long MINIMAL_DELTA_FOR_LOGGING = 10000; + + private final Object sleepLock = new Object(); + private boolean triggerWake = false; + + /** + * @param sleep sleep time in milliseconds + * @param stopper When {@link Stoppable#isStopped()} is true, this thread will + * cleanup and exit cleanly. + */ + public Sleeper(final int sleep, final Stoppable stopper) { + this.period = sleep; + this.stopper = stopper; + } + + /** + * Sleep for period. + */ + public void sleep() { + sleep(System.currentTimeMillis()); + } + + /** + * If currently asleep, stops sleeping; if not asleep, will skip the next + * sleep cycle. + */ + public void skipSleepCycle() { + synchronized (sleepLock) { + triggerWake = true; + sleepLock.notifyAll(); + } + } + + /** + * Sleep for period adjusted by passed startTime + * @param startTime Time some task started previous to now. Time to sleep + * will be docked current time minus passed startTime. + */ + public void sleep(final long startTime) { + if (this.stopper.isStopped()) { + return; + } + long now = System.currentTimeMillis(); + long waitTime = this.period - (now - startTime); + if (waitTime > this.period) { + LOG.warn("Calculated wait time > " + this.period + + "; setting to this.period: " + System.currentTimeMillis() + ", " + + startTime); + waitTime = this.period; + } + while (waitTime > 0) { + long woke = -1; + try { + synchronized (sleepLock) { + if (triggerWake) break; + sleepLock.wait(waitTime); + } + woke = System.currentTimeMillis(); + long slept = woke - now; + if (slept - this.period > MINIMAL_DELTA_FOR_LOGGING) { + LOG.warn("We slept " + slept + "ms instead of " + this.period + + "ms, this is likely due to a long " + + "garbage collecting pause and it's usually bad, see " + + "http://hbase.apache.org/book.html#trouble.rs.runtime.zkexpired"); + } + } catch(InterruptedException iex) { + // We we interrupted because we're meant to stop? If not, just + // continue ignoring the interruption + if (this.stopper.isStopped()) { + return; + } + } + // Recalculate waitTime. + woke = (woke == -1)? System.currentTimeMillis(): woke; + waitTime = this.period - (woke - startTime); + } + triggerWake = false; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java b/src/main/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java new file mode 100644 index 0000000..7229e14 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java @@ -0,0 +1,285 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.lang.ref.Reference; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.SoftReference; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * A SortedMap implementation that uses Soft Reference values + * internally to make it play well with the GC when in a low-memory + * situation. Use as a cache where you also need SortedMap functionality. + * + * @param key class + * @param value class + */ +public class SoftValueSortedMap implements SortedMap { + private final SortedMap> internalMap; + private final ReferenceQueue rq = new ReferenceQueue(); + private Object sync; + + /** Constructor */ + public SoftValueSortedMap() { + this(new TreeMap>()); + } + + /** + * Constructor + * @param c comparator + */ + public SoftValueSortedMap(final Comparator c) { + this(new TreeMap>(c)); + } + + /** Internal constructor + * @param original object to wrap and synchronize on + */ + private SoftValueSortedMap(SortedMap> original) { + this(original, original); + } + + /** Internal constructor + * For headMap, tailMap, and subMap support + * @param original object to wrap + * @param sync object to synchronize on + */ + private SoftValueSortedMap(SortedMap> original, Object sync) { + this.internalMap = original; + this.sync = sync; + } + + /** + * Checks soft references and cleans any that have been placed on + * ReferenceQueue. Call if get/put etc. are not called regularly. + * Internally these call checkReferences on each access. + * @return How many references cleared. + */ + @SuppressWarnings("unchecked") + private int checkReferences() { + int i = 0; + for (Reference ref; (ref = this.rq.poll()) != null;) { + i++; + this.internalMap.remove(((SoftValue)ref).key); + } + return i; + } + + public V put(K key, V value) { + synchronized(sync) { + checkReferences(); + SoftValue oldValue = this.internalMap.put(key, + new SoftValue(key, value, this.rq)); + return oldValue == null ? null : oldValue.get(); + } + } + + @Override + public void putAll(Map m) { + throw new RuntimeException("Not implemented"); + } + + public V get(Object key) { + synchronized(sync) { + checkReferences(); + SoftValue value = this.internalMap.get(key); + if (value == null) { + return null; + } + if (value.get() == null) { + this.internalMap.remove(key); + return null; + } + return value.get(); + } + } + + public V remove(Object key) { + synchronized(sync) { + checkReferences(); + SoftValue value = this.internalMap.remove(key); + return value == null ? null : value.get(); + } + } + + public boolean containsKey(Object key) { + synchronized(sync) { + checkReferences(); + return this.internalMap.containsKey(key); + } + } + + public boolean containsValue(Object value) { + throw new UnsupportedOperationException("Don't support containsValue!"); + } + + public K firstKey() { + synchronized(sync) { + checkReferences(); + return internalMap.firstKey(); + } + } + + public K lastKey() { + synchronized(sync) { + checkReferences(); + return internalMap.lastKey(); + } + } + + public SoftValueSortedMap headMap(K key) { + synchronized(sync) { + checkReferences(); + return new SoftValueSortedMap(this.internalMap.headMap(key), sync); + } + } + + public SoftValueSortedMap tailMap(K key) { + synchronized(sync) { + checkReferences(); + return new SoftValueSortedMap(this.internalMap.tailMap(key), sync); + } + } + + public SoftValueSortedMap subMap(K fromKey, K toKey) { + synchronized(sync) { + checkReferences(); + return new SoftValueSortedMap(this.internalMap.subMap(fromKey, + toKey), sync); + } + } + + /* + * retrieves the value associated with the greatest key strictly less than + * the given key, or null if there is no such key + * @param key the key we're interested in + */ + public synchronized V lowerValueByKey(K key) { + synchronized(sync) { + checkReferences(); + + Map.Entry> entry = + ((NavigableMap>) this.internalMap).lowerEntry(key); + if (entry==null) { + return null; + } + SoftValue value=entry.getValue(); + if (value==null) { + return null; + } + if (value.get() == null) { + this.internalMap.remove(key); + return null; + } + return value.get(); + } + } + + public boolean isEmpty() { + synchronized(sync) { + checkReferences(); + return this.internalMap.isEmpty(); + } + } + + public int size() { + synchronized(sync) { + checkReferences(); + return this.internalMap.size(); + } + } + + public void clear() { + synchronized(sync) { + checkReferences(); + this.internalMap.clear(); + } + } + + public Set keySet() { + synchronized(sync) { + checkReferences(); + // this is not correct as per SortedMap contract (keySet should be + // modifiable) + // needed here so that another thread cannot modify the keyset + // without locking + return Collections.unmodifiableSet(this.internalMap.keySet()); + } + } + + public Comparator comparator() { + return this.internalMap.comparator(); + } + + public Set> entrySet() { + synchronized(sync) { + checkReferences(); + // this is not correct as per SortedMap contract (entrySet should be + // backed by map) + Set> realEntries = new LinkedHashSet>(); + for (Map.Entry> entry : this.internalMap.entrySet()) { + realEntries.add(entry.getValue()); + } + return realEntries; + } + } + + public Collection values() { + synchronized(sync) { + checkReferences(); + ArrayList hardValues = new ArrayList(); + for (SoftValue softValue : this.internalMap.values()) { + hardValues.add(softValue.get()); + } + return hardValues; + } + } + + private static class SoftValue extends SoftReference implements Map.Entry { + final K key; + + SoftValue(K key, V value, ReferenceQueue q) { + super(value, q); + this.key = key; + } + + public K getKey() { + return this.key; + } + + public V getValue() { + return get(); + } + + public V setValue(V value) { + throw new RuntimeException("Not implemented"); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/SortedCopyOnWriteSet.java b/src/main/java/org/apache/hadoop/hbase/util/SortedCopyOnWriteSet.java new file mode 100644 index 0000000..f04f306 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/SortedCopyOnWriteSet.java @@ -0,0 +1,175 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.SortedSet; +import java.util.TreeSet; + +/** + * Simple {@link java.util.SortedSet} implementation that uses an internal + * {@link java.util.TreeSet} to provide ordering. All mutation operations + * create a new copy of the TreeSet instance, so are very + * expensive. This class is only intended for use on small, very rarely + * written collections that expect highly concurrent reads. Read operations + * are performed on a reference to the internal TreeSet at the + * time of invocation, so will not see any mutations to the collection during + * their operation. + * + *

    Note that due to the use of a {@link java.util.TreeSet} internally, + * a {@link java.util.Comparator} instance must be provided, or collection + * elements must implement {@link java.lang.Comparable}. + *

    + * @param A class implementing {@link java.lang.Comparable} or able to be + * compared by a provided comparator. + */ +public class SortedCopyOnWriteSet implements SortedSet { + private SortedSet internalSet; + + public SortedCopyOnWriteSet() { + this.internalSet = new TreeSet(); + } + + public SortedCopyOnWriteSet(Collection c) { + this.internalSet = new TreeSet(c); + } + + public SortedCopyOnWriteSet(Comparator comparator) { + this.internalSet = new TreeSet(comparator); + } + + @Override + public int size() { + return internalSet.size(); + } + + @Override + public boolean isEmpty() { + return internalSet.isEmpty(); + } + + @Override + public boolean contains(Object o) { + return internalSet.contains(o); + } + + @Override + public Iterator iterator() { + return internalSet.iterator(); + } + + @Override + public Object[] toArray() { + return internalSet.toArray(); + } + + @Override + public T[] toArray(T[] a) { + return internalSet.toArray(a); + } + + @Override + public synchronized boolean add(E e) { + SortedSet newSet = new TreeSet(internalSet); + boolean added = newSet.add(e); + internalSet = newSet; + return added; + } + + @Override + public synchronized boolean remove(Object o) { + SortedSet newSet = new TreeSet(internalSet); + boolean removed = newSet.remove(o); + internalSet = newSet; + return removed; + } + + @Override + public boolean containsAll(Collection c) { + return internalSet.containsAll(c); + } + + @Override + public synchronized boolean addAll(Collection c) { + SortedSet newSet = new TreeSet(internalSet); + boolean changed = newSet.addAll(c); + internalSet = newSet; + return changed; + } + + @Override + public synchronized boolean retainAll(Collection c) { + SortedSet newSet = new TreeSet(internalSet); + boolean changed = newSet.retainAll(c); + internalSet = newSet; + return changed; + } + + @Override + public synchronized boolean removeAll(Collection c) { + SortedSet newSet = new TreeSet(internalSet); + boolean changed = newSet.removeAll(c); + internalSet = newSet; + return changed; + } + + @Override + public synchronized void clear() { + Comparator comparator = internalSet.comparator(); + if (comparator != null) { + internalSet = new TreeSet(comparator); + } else { + internalSet = new TreeSet(); + } + } + + @Override + public Comparator comparator() { + return internalSet.comparator(); + } + + @Override + public SortedSet subSet(E fromElement, E toElement) { + return internalSet.subSet(fromElement, toElement); + } + + @Override + public SortedSet headSet(E toElement) { + return internalSet.headSet(toElement); + } + + @Override + public SortedSet tailSet(E fromElement) { + return internalSet.tailSet(fromElement); + } + + @Override + public E first() { + return internalSet.first(); + } + + @Override + public E last() { + return internalSet.last(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/Strings.java b/src/main/java/org/apache/hadoop/hbase/util/Strings.java new file mode 100644 index 0000000..5e98c59 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Strings.java @@ -0,0 +1,75 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +/** + * Utility for Strings. + */ +public class Strings { + public final static String DEFAULT_SEPARATOR = "="; + public final static String DEFAULT_KEYVALUE_SEPARATOR = ", "; + + /** + * Append to a StringBuilder a key/value. + * Uses default separators. + * @param sb StringBuilder to use + * @param key Key to append. + * @param value Value to append. + * @return Passed sb populated with key/value. + */ + public static StringBuilder appendKeyValue(final StringBuilder sb, + final String key, final Object value) { + return appendKeyValue(sb, key, value, DEFAULT_SEPARATOR, + DEFAULT_KEYVALUE_SEPARATOR); + } + + /** + * Append to a StringBuilder a key/value. + * Uses default separators. + * @param sb StringBuilder to use + * @param key Key to append. + * @param value Value to append. + * @param separator Value to use between key and value. + * @param keyValueSeparator Value to use between key/value sets. + * @return Passed sb populated with key/value. + */ + public static StringBuilder appendKeyValue(final StringBuilder sb, + final String key, final Object value, final String separator, + final String keyValueSeparator) { + if (sb.length() > 0) { + sb.append(keyValueSeparator); + } + return sb.append(key).append(separator).append(value); + } + + /** + * Given a PTR string generated via reverse DNS lookup, return everything + * except the trailing period. Example for host.example.com., return + * host.example.com + * @param dnPtr a domain name pointer (PTR) string. + * @return Sanitized hostname with last period stripped off. + * + */ + public static String domainNamePointerToHostName(String dnPtr) { + if (dnPtr == null) + return null; + return dnPtr.endsWith(".") ? dnPtr.substring(0, dnPtr.length()-1) : dnPtr; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/Threads.java b/src/main/java/org/apache/hadoop/hbase/util/Threads.java new file mode 100644 index 0000000..a5a4d13 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Threads.java @@ -0,0 +1,232 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.PrintWriter; +import java.lang.Thread.UncaughtExceptionHandler; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * Thread Utility + */ +public class Threads { + protected static final Log LOG = LogFactory.getLog(Threads.class); + private static final AtomicInteger poolNumber = new AtomicInteger(1); + + /** + * Utility method that sets name, daemon status and starts passed thread. + * @param t thread to run + * @return Returns the passed Thread t. + */ + public static Thread setDaemonThreadRunning(final Thread t) { + return setDaemonThreadRunning(t, t.getName()); + } + + /** + * Utility method that sets name, daemon status and starts passed thread. + * @param t thread to frob + * @param name new name + * @return Returns the passed Thread t. + */ + public static Thread setDaemonThreadRunning(final Thread t, + final String name) { + return setDaemonThreadRunning(t, name, null); + } + + /** + * Utility method that sets name, daemon status and starts passed thread. + * @param t thread to frob + * @param name new name + * @param handler A handler to set on the thread. Pass null if want to + * use default handler. + * @return Returns the passed Thread t. + */ + public static Thread setDaemonThreadRunning(final Thread t, + final String name, final UncaughtExceptionHandler handler) { + t.setName(name); + if (handler != null) { + t.setUncaughtExceptionHandler(handler); + } + t.setDaemon(true); + t.start(); + return t; + } + + /** + * Shutdown passed thread using isAlive and join. + * @param t Thread to shutdown + */ + public static void shutdown(final Thread t) { + shutdown(t, 0); + } + + /** + * Shutdown passed thread using isAlive and join. + * @param joinwait Pass 0 if we're to wait forever. + * @param t Thread to shutdown + */ + public static void shutdown(final Thread t, final long joinwait) { + if (t == null) return; + while (t.isAlive()) { + try { + t.join(joinwait); + } catch (InterruptedException e) { + LOG.warn(t.getName() + "; joinwait=" + joinwait, e); + } + } + } + + + /** + * @param t Waits on the passed thread to die dumping a threaddump every + * minute while its up. + * @throws InterruptedException + */ + public static void threadDumpingIsAlive(final Thread t) + throws InterruptedException { + if (t == null) { + return; + } + + while (t.isAlive()) { + t.join(60 * 1000); + if (t.isAlive()) { + ReflectionUtils.printThreadInfo(new PrintWriter(System.out), + "Automatic Stack Trace every 60 seconds waiting on " + + t.getName()); + } + } + } + + /** + * @param millis How long to sleep for in milliseconds. + */ + public static void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + /** + * Sleeps for the given amount of time even if interrupted. Preserves + * the interrupt status. + * @param msToWait the amount of time to sleep in milliseconds + */ + public static void sleepWithoutInterrupt(final long msToWait) { + long timeMillis = System.currentTimeMillis(); + long endTime = timeMillis + msToWait; + boolean interrupted = false; + while (timeMillis < endTime) { + try { + Thread.sleep(endTime - timeMillis); + } catch (InterruptedException ex) { + interrupted = true; + } + timeMillis = System.currentTimeMillis(); + } + + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + /** + * Create a new CachedThreadPool with a bounded number as the maximum + * thread size in the pool. + * + * @param maxCachedThread the maximum thread could be created in the pool + * @param timeout the maximum time to wait + * @param unit the time unit of the timeout argument + * @param threadFactory the factory to use when creating new threads + * @return threadPoolExecutor the cachedThreadPool with a bounded number + * as the maximum thread size in the pool. + */ + public static ThreadPoolExecutor getBoundedCachedThreadPool( + int maxCachedThread, long timeout, TimeUnit unit, + ThreadFactory threadFactory) { + ThreadPoolExecutor boundedCachedThreadPool = + new ThreadPoolExecutor(maxCachedThread, maxCachedThread, timeout, + TimeUnit.SECONDS, new LinkedBlockingQueue(), threadFactory); + // allow the core pool threads timeout and terminate + boundedCachedThreadPool.allowCoreThreadTimeOut(true); + return boundedCachedThreadPool; + } + + + /** + * Returns a {@link java.util.concurrent.ThreadFactory} that names each + * created thread uniquely, with a common prefix. + * + * @param prefix The prefix of every created Thread's name + * @return a {@link java.util.concurrent.ThreadFactory} that names threads + */ + public static ThreadFactory getNamedThreadFactory(final String prefix) { + SecurityManager s = System.getSecurityManager(); + final ThreadGroup threadGroup = (s != null) ? s.getThreadGroup() : Thread.currentThread() + .getThreadGroup(); + + return new ThreadFactory() { + final AtomicInteger threadNumber = new AtomicInteger(1); + private final int poolNumber = Threads.poolNumber.getAndIncrement(); + final ThreadGroup group = threadGroup; + + @Override + public Thread newThread(Runnable r) { + final String name = prefix + "pool-" + poolNumber + "-thread-" + + threadNumber.getAndIncrement(); + return new Thread(group, r, name); + } + }; + } + + + /** + * Get a named {@link ThreadFactory} that just builds daemon threads + * @param prefix name prefix for all threads created from the factory + * @return a thread factory that creates named, daemon threads + */ + public static ThreadFactory newDaemonThreadFactory(final String prefix) { + final ThreadFactory namedFactory = getNamedThreadFactory(prefix); + return new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread t = namedFactory.newThread(r); + if (!t.isDaemon()) { + t.setDaemon(true); + } + if (t.getPriority() != Thread.NORM_PRIORITY) { + t.setPriority(Thread.NORM_PRIORITY); + } + return t; + } + + }; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java b/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java new file mode 100644 index 0000000..b516e16 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java @@ -0,0 +1,115 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.commons.logging.LogFactory; +import java.io.PrintWriter; + +import org.apache.hadoop.hbase.VersionAnnotation; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.commons.logging.Log; + +/** + * This class finds the package info for hbase and the VersionAnnotation + * information. Taken from hadoop. Only name of annotation is different. + */ +public class VersionInfo { + private static final Log LOG = LogFactory.getLog(VersionInfo.class.getName()); + private static Package myPackage; + private static VersionAnnotation version; + + static { + myPackage = VersionAnnotation.class.getPackage(); + version = myPackage.getAnnotation(VersionAnnotation.class); + } + + /** + * Get the meta-data for the hbase package. + * @return package + */ + static Package getPackage() { + return myPackage; + } + + /** + * Get the hbase version. + * @return the hbase version string, eg. "0.6.3-dev" + */ + public static String getVersion() { + return version != null ? version.version() : "Unknown"; + } + + /** + * Get the subversion revision number for the root directory + * @return the revision number, eg. "451451" + */ + public static String getRevision() { + return version != null ? version.revision() : "Unknown"; + } + + /** + * The date that hbase was compiled. + * @return the compilation date in unix date format + */ + public static String getDate() { + return version != null ? version.date() : "Unknown"; + } + + /** + * The user that compiled hbase. + * @return the username of the user + */ + public static String getUser() { + return version != null ? version.user() : "Unknown"; + } + + /** + * Get the subversion URL for the root hbase directory. + * @return the url + */ + public static String getUrl() { + return version != null ? version.url() : "Unknown"; + } + + static String[] versionReport() { + return new String[] { + "HBase " + getVersion(), + "Subversion " + getUrl() + " -r " + getRevision(), + "Compiled by " + getUser() + " on " + getDate() + }; + } + + public static void writeTo(PrintWriter out) { + for (String line : versionReport()) { + out.println(line); + } + } + + public static void logVersion() { + for (String line : versionReport()) { + LOG.info(line); + } + } + + public static void main(String[] args) { + logVersion(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/Writables.java b/src/main/java/org/apache/hadoop/hbase/util/Writables.java new file mode 100644 index 0000000..1719e84 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/Writables.java @@ -0,0 +1,219 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.migration.HRegionInfo090x; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.Writable; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Utility class with methods for manipulating Writable objects + */ +public class Writables { + /** + * @param w writable + * @return The bytes of w gotten by running its + * {@link Writable#write(java.io.DataOutput)} method. + * @throws IOException e + * @see #getWritable(byte[], Writable) + */ + public static byte [] getBytes(final Writable w) throws IOException { + if (w == null) { + throw new IllegalArgumentException("Writable cannot be null"); + } + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(byteStream); + try { + w.write(out); + out.close(); + out = null; + return byteStream.toByteArray(); + } finally { + if (out != null) { + out.close(); + } + } + } + + /** + * Put a bunch of Writables as bytes all into the one byte array. + * @param ws writable + * @return The bytes of w gotten by running its + * {@link Writable#write(java.io.DataOutput)} method. + * @throws IOException e + * @see #getHRegionInfos(byte[], int, int) + */ + public static byte [] getBytes(final Writable... ws) throws IOException { + List bytes = new ArrayList(); + int size = 0; + for (Writable w: ws) { + byte [] b = getBytes(w); + size += b.length; + bytes.add(b); + } + byte [] result = new byte[size]; + int offset = 0; + for (byte [] b: bytes) { + System.arraycopy(b, 0, result, offset, b.length); + offset += b.length; + } + return result; + } + + /** + * Set bytes into the passed Writable by calling its + * {@link Writable#readFields(java.io.DataInput)}. + * @param bytes serialized bytes + * @param w An empty Writable (usually made by calling the null-arg + * constructor). + * @return The passed Writable after its readFields has been called fed + * by the passed bytes array or IllegalArgumentException + * if passed null or an empty bytes array. + * @throws IOException e + * @throws IllegalArgumentException + */ + public static Writable getWritable(final byte [] bytes, final Writable w) + throws IOException { + return getWritable(bytes, 0, bytes.length, w); + } + + /** + * Set bytes into the passed Writable by calling its + * {@link Writable#readFields(java.io.DataInput)}. + * @param bytes serialized bytes + * @param offset offset into array + * @param length length of data + * @param w An empty Writable (usually made by calling the null-arg + * constructor). + * @return The passed Writable after its readFields has been called fed + * by the passed bytes array or IllegalArgumentException + * if passed null or an empty bytes array. + * @throws IOException e + * @throws IllegalArgumentException + */ + public static Writable getWritable(final byte [] bytes, final int offset, + final int length, final Writable w) + throws IOException { + if (bytes == null || length <=0) { + throw new IllegalArgumentException("Can't build a writable with empty " + + "bytes array"); + } + if (w == null) { + throw new IllegalArgumentException("Writable cannot be null"); + } + DataInputBuffer in = new DataInputBuffer(); + try { + in.reset(bytes, offset, length); + w.readFields(in); + return w; + } finally { + in.close(); + } + } + + /** + * @param bytes serialized bytes + * @return A HRegionInfo instance built out of passed bytes. + * @throws IOException e + */ + public static HRegionInfo getHRegionInfo(final byte [] bytes) + throws IOException { + return (HRegionInfo)getWritable(bytes, new HRegionInfo()); + } + + /** + * @param bytes serialized bytes + * @return All the hregioninfos that are in the byte array. Keeps reading + * till we hit the end. + * @throws IOException e + */ + public static List getHRegionInfos(final byte [] bytes, + final int offset, final int length) + throws IOException { + if (bytes == null) { + throw new IllegalArgumentException("Can't build a writable with empty " + + "bytes array"); + } + DataInputBuffer in = new DataInputBuffer(); + List hris = new ArrayList(); + try { + in.reset(bytes, offset, length); + while (in.available() > 0) { + HRegionInfo hri = new HRegionInfo(); + hri.readFields(in); + hris.add(hri); + } + } finally { + in.close(); + } + return hris; + } + + /** + * @param bytes serialized bytes + * @return A HRegionInfo instance built out of passed bytes + * or null if passed bytes are null or an empty array. + * @throws IOException e + */ + public static HRegionInfo getHRegionInfoOrNull(final byte [] bytes) + throws IOException { + return (bytes == null || bytes.length <= 0)? + null : getHRegionInfo(bytes); + } + + /** + * Copy one Writable to another. Copies bytes using data streams. + * @param src Source Writable + * @param tgt Target Writable + * @return The target Writable. + * @throws IOException e + */ + public static Writable copyWritable(final Writable src, final Writable tgt) + throws IOException { + return copyWritable(getBytes(src), tgt); + } + + /** + * Copy one Writable to another. Copies bytes using data streams. + * @param bytes Source Writable + * @param tgt Target Writable + * @return The target Writable. + * @throws IOException e + */ + public static Writable copyWritable(final byte [] bytes, final Writable tgt) + throws IOException { + DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytes)); + try { + tgt.readFields(dis); + } finally { + dis.close(); + } + return tgt; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java new file mode 100644 index 0000000..0ac44d8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java @@ -0,0 +1,369 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util.hbck; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.CorruptHFileException; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.util.FSUtils.FamilyDirFilter; +import org.apache.hadoop.hbase.util.FSUtils.HFileFilter; +import org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter; +import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter; + +/** + * This class marches through all of the region's hfiles and verifies that + * they are all valid files. One just needs to instantiate the class, use + * checkTables(List) and then retrieve the corrupted hfiles (and + * quarantined files if in quarantining mode) + * + * The implementation currently parallelizes at the regionDir level. + */ +public class HFileCorruptionChecker { + private static final Log LOG = LogFactory.getLog(HFileCorruptionChecker.class); + + final Configuration conf; + final FileSystem fs; + final CacheConfig cacheConf; + final ExecutorService executor; + final Set corrupted = new ConcurrentSkipListSet(); + final Set failures = new ConcurrentSkipListSet(); + final Set quarantined = new ConcurrentSkipListSet(); + final Set missing = new ConcurrentSkipListSet(); + final boolean inQuarantineMode; + final AtomicInteger hfilesChecked = new AtomicInteger(); + + public HFileCorruptionChecker(Configuration conf, ExecutorService executor, + boolean quarantine) throws IOException { + this.conf = conf; + this.fs = FileSystem.get(conf); + this.cacheConf = new CacheConfig(conf); + this.executor = executor; + this.inQuarantineMode = quarantine; + } + + /** + * Checks a path to see if it is a valid hfile. + * + * @param p + * full Path to an HFile + * @throws IOException + * This is a connectivity related exception + */ + protected void checkHFile(Path p) throws IOException { + HFile.Reader r = null; + try { + r = HFile.createReader(fs, p, cacheConf); + } catch (CorruptHFileException che) { + LOG.warn("Found corrupt HFile " + p, che); + corrupted.add(p); + if (inQuarantineMode) { + Path dest = createQuarantinePath(p); + LOG.warn("Quarantining corrupt HFile " + p + " into " + dest); + boolean success = fs.mkdirs(dest.getParent()); + success = success ? fs.rename(p, dest): false; + if (!success) { + failures.add(p); + } else { + quarantined.add(dest); + } + } + return; + } catch (FileNotFoundException fnfe) { + LOG.warn("HFile " + p + " was missing. Likely removed due to compaction/split?"); + missing.add(p); + } finally { + hfilesChecked.addAndGet(1); + if (r != null) { + r.close(true); + } + } + } + + /** + * Given a path, generates a new path to where we move a corrupted hfile (bad + * trailer, no trailer). + * + * @param hFile + * Path to a corrupt hfile (assumes that it is HBASE_DIR/ table + * /region/cf/file) + * @return path to where corrupted files are stored. This should be + * HBASE_DIR/.corrupt/table/region/cf/file. + */ + Path createQuarantinePath(Path hFile) { + // extract the normal dirs structure + Path cfDir = hFile.getParent(); + Path regionDir = cfDir.getParent(); + Path tableDir = regionDir.getParent(); + + // build up the corrupted dirs strcture + Path corruptBaseDir = new Path(conf.get(HConstants.HBASE_DIR), conf.get( + "hbase.hfile.quarantine.dir", HConstants.CORRUPT_DIR_NAME)); + Path corruptTableDir = new Path(corruptBaseDir, tableDir.getName()); + Path corruptRegionDir = new Path(corruptTableDir, regionDir.getName()); + Path corruptFamilyDir = new Path(corruptRegionDir, cfDir.getName()); + Path corruptHfile = new Path(corruptFamilyDir, hFile.getName()); + return corruptHfile; + } + + /** + * Check all files in a column family dir. + * + * @param cfDir + * column family directory + * @throws IOException + */ + protected void checkColFamDir(Path cfDir) throws IOException { + FileStatus[] hfs = null; + try { + hfs = fs.listStatus(cfDir, new HFileFilter(fs)); // use same filter as scanner. + } catch (FileNotFoundException fnfe) { + // Hadoop 0.23+ listStatus semantics throws an exception if the path does not exist. + LOG.warn("Colfam Directory " + cfDir + + " does not exist. Likely due to concurrent split/compaction. Skipping."); + missing.add(cfDir); + return; + } + + // Hadoop 1.0 listStatus does not throw an exception if the path does not exist. + if (hfs.length == 0 && !fs.exists(cfDir)) { + LOG.warn("Colfam Directory " + cfDir + + " does not exist. Likely due to concurrent split/compaction. Skipping."); + missing.add(cfDir); + return; + } + for (FileStatus hfFs : hfs) { + Path hf = hfFs.getPath(); + checkHFile(hf); + } + } + + /** + * Check all column families in a region dir. + * + * @param regionDir + * region directory + * @throws IOException + */ + protected void checkRegionDir(Path regionDir) throws IOException { + FileStatus[] cfs = null; + try { + cfs = fs.listStatus(regionDir, new FamilyDirFilter(fs)); + } catch (FileNotFoundException fnfe) { + // Hadoop 0.23+ listStatus semantics throws an exception if the path does not exist. + LOG.warn("Region Directory " + regionDir + + " does not exist. Likely due to concurrent split/compaction. Skipping."); + missing.add(regionDir); + return; + } + + // Hadoop 1.0 listStatus does not throw an exception if the path does not exist. + if (cfs.length == 0 && !fs.exists(regionDir)) { + LOG.warn("Region Directory " + regionDir + + " does not exist. Likely due to concurrent split/compaction. Skipping."); + missing.add(regionDir); + return; + } + + for (FileStatus cfFs : cfs) { + Path cfDir = cfFs.getPath(); + checkColFamDir(cfDir); + } + } + + /** + * Check all the regiondirs in the specified tableDir + * + * @param tableDir + * path to a table + * @throws IOException + */ + void checkTableDir(Path tableDir) throws IOException { + FileStatus[] rds = fs.listStatus(tableDir, new RegionDirFilter(fs)); + if (rds.length == 0 && !fs.exists(tableDir)) { + // interestingly listStatus does not throw an exception if the path does not exist. + LOG.warn("Table Directory " + tableDir + + " does not exist. Likely due to concurrent delete. Skipping."); + missing.add(tableDir); + return; + } + + // Parallelize check at the region dir level + List rdcs = new ArrayList(); + List> rdFutures; + + for (FileStatus rdFs : rds) { + Path rdDir = rdFs.getPath(); + RegionDirChecker work = new RegionDirChecker(rdDir); + rdcs.add(work); + } + + // Submit and wait for completion + try { + rdFutures = executor.invokeAll(rdcs); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + LOG.warn("Region dirs checking interrupted!", ie); + return; + } + + for (int i = 0; i < rdFutures.size(); i++) { + Future f = rdFutures.get(i); + try { + f.get(); + } catch (ExecutionException e) { + LOG.warn("Failed to quaratine an HFile in regiondir " + + rdcs.get(i).regionDir, e.getCause()); + // rethrow IOExceptions + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); + } + + // rethrow RuntimeExceptions + if (e.getCause() instanceof RuntimeException) { + throw (RuntimeException) e.getCause(); + } + + // this should never happen + LOG.error("Unexpected exception encountered", e); + return; // bailing out. + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + LOG.warn("Region dirs check interrupted!", ie); + // bailing out + return; + } + } + } + + /** + * An individual work item for parallelized regiondir processing. This is + * intentionally an inner class so it can use the shared error sets and fs. + */ + private class RegionDirChecker implements Callable { + final Path regionDir; + + RegionDirChecker(Path regionDir) { + this.regionDir = regionDir; + } + + @Override + public Void call() throws IOException { + checkRegionDir(regionDir); + return null; + } + } + + /** + * Check the specified table dirs for bad hfiles. + */ + public void checkTables(Collection tables) throws IOException { + for (Path t : tables) { + checkTableDir(t); + } + } + + /** + * @return the set of check failure file paths after checkTables is called. + */ + public Collection getFailures() { + return new HashSet(failures); + } + + /** + * @return the set of corrupted file paths after checkTables is called. + */ + public Collection getCorrupted() { + return new HashSet(corrupted); + } + + /** + * @return number of hfiles checked in the last HfileCorruptionChecker run + */ + public int getHFilesChecked() { + return hfilesChecked.get(); + } + + /** + * @return the set of successfully quarantined paths after checkTables is called. + */ + public Collection getQuarantined() { + return new HashSet(quarantined); + } + + /** + * @return the set of paths that were missing. Likely due to deletion/moves from + * compaction or flushes. + */ + public Collection getMissing() { + return new HashSet(missing); + } + + /** + * Print a human readable summary of hfile quarantining operations. + * @param out + */ + public void report(ErrorReporter out) { + out.print("Checked " + hfilesChecked.get() + " hfile for corruption"); + out.print(" HFiles corrupted: " + corrupted.size()); + if (inQuarantineMode) { + out.print(" HFiles successfully quarantined: " + quarantined.size()); + for (Path sq : quarantined) { + out.print(" " + sq); + } + out.print(" HFiles failed quarantine: " + failures.size()); + for (Path fq : failures) { + out.print(" " + fq); + } + } + out.print(" HFiles moved while checking: " + missing.size()); + for (Path mq : missing) { + out.print(" " + mq); + } + + String initialState = (corrupted.size() == 0) ? "OK" : "CORRUPTED"; + String fixedState = (corrupted.size() == quarantined.size()) ? "OK" + : "CORRUPTED"; + + if (inQuarantineMode) { + out.print("Summary: " + initialState + " => " + fixedState); + } else { + out.print("Summary: " + initialState); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java new file mode 100644 index 0000000..5e91e2e --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util.hbck; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.util.HBaseFsck; +import org.apache.hadoop.io.MultipleIOException; + +/** + * This code is used to rebuild meta off line from file system data. If there + * are any problem detected, it will fail suggesting actions for the user to do + * to "fix" problems. If it succeeds, it will backup the previous .META. and + * -ROOT- dirs and write new tables in place. + * + * This is an advanced feature, so is only exposed for use if explicitly + * mentioned. + * + * hbase org.apache.hadoop.hbase.util.hbck.OfflineMetaRepair ... + */ +public class OfflineMetaRepair { + private static final Log LOG = LogFactory.getLog(OfflineMetaRepair.class.getName()); + + protected static void printUsageAndExit() { + StringBuilder sb = new StringBuilder(); + sb.append("Usage: OfflineMetaRepair [opts]\n"). + append(" where [opts] are:\n"). + append(" -details Display full report of all regions.\n"). + append(" -base Base Hbase Data directory.\n"). + append(" -sidelineDir HDFS path to backup existing meta and root.\n"). + append(" -fix Auto fix as many problems as possible.\n"). + append(" -fixHoles Auto fix as region holes."); + System.err.println(sb.toString()); + Runtime.getRuntime().exit(-2); + } + + /** + * Main program + * + * @param args + * @throws Exception + */ + public static void main(String[] args) throws Exception { + + // create a fsck object + Configuration conf = HBaseConfiguration.create(); + // Cover both bases, the old way of setting default fs and the new. + // We're supposed to run on 0.20 and 0.21 anyways. + conf.set("fs.defaultFS", conf.get(HConstants.HBASE_DIR)); + conf.set("fs.default.name", conf.get(HConstants.HBASE_DIR)); + HBaseFsck fsck = new HBaseFsck(conf); + boolean fixHoles = false; + + // Process command-line args. + for (int i = 0; i < args.length; i++) { + String cmd = args[i]; + if (cmd.equals("-details")) { + fsck.setDisplayFullReport(); + } else if (cmd.equals("-base")) { + if (i == args.length - 1) { + System.err.println("OfflineMetaRepair: -base needs an HDFS path."); + printUsageAndExit(); + } + // update hbase root dir to user-specified base + i++; + String path = args[i]; + conf.set(HConstants.HBASE_DIR, path); + conf.set("fs.defaultFS", conf.get(HConstants.HBASE_DIR)); + conf.set("fs.default.name", conf.get(HConstants.HBASE_DIR)); + } else if (cmd.equals("-sidelineDir")) { + if (i == args.length - 1) { + System.err.println("OfflineMetaRepair: -sidelineDir needs an HDFS path."); + printUsageAndExit(); + } + // set the hbck sideline dir to user-specified one + i++; + fsck.setSidelineDir(args[i]); + } else if (cmd.equals("-fixHoles")) { + fixHoles = true; + } else if (cmd.equals("-fix")) { + // make all fix options true + fixHoles = true; + } else { + String str = "Unknown command line option : " + cmd; + LOG.info(str); + System.out.println(str); + printUsageAndExit(); + } + } + + // Fsck doesn't shutdown and and doesn't provide a way to shutdown its + // threads cleanly, so we do a System.exit. + boolean success = false; + try { + success = fsck.rebuildMeta(fixHoles); + } catch (MultipleIOException mioes) { + for (IOException ioe : mioes.getExceptions()) { + LOG.error("Bailed out due to:", ioe); + } + } catch (Exception e) { + LOG.error("Bailed out due to: ", e); + } finally { + System.exit(success ? 0 : 1); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java new file mode 100644 index 0000000..4310bf8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util.hbck; + +import java.io.IOException; +import java.util.Collection; + +import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo; +import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo; + +/** + * This interface provides callbacks for handling particular table integrity + * invariant violations. This could probably be boiled down to handling holes + * and handling overlaps but currently preserves the older more specific error + * condition codes. + */ +public interface TableIntegrityErrorHandler { + + TableInfo getTableInfo(); + + /** + * Set the TableInfo used by all HRegionInfos fabricated by other callbacks + */ + void setTableInfo(TableInfo ti); + + /** + * Callback for handling case where a Table has a first region that does not + * have an empty start key. + * + * @param hi An HbckInfo of the second region in a table. This should have + * a non-empty startkey, and can be used to fabricate a first region that + * has an empty start key. + */ + void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException; + + /** + * Callback for handling case where a Table has a last region that does not + * have an empty end key. + * + * @param curEndKey The end key of the current last region. There should be a new region + * with start key as this and an empty end key. + */ + void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException; + + /** + * Callback for handling a region that has the same start and end key. + * + * @param hi An HbckInfo for a degenerate key. + */ + void handleDegenerateRegion(HbckInfo hi) throws IOException; + + /** + * Callback for handling two regions that have the same start key. This is + * a specific case of a region overlap. + * @param hi1 one of the overlapping HbckInfo + * @param hi2 the other overlapping HbckInfo + */ + void handleDuplicateStartKeys(HbckInfo hi1, HbckInfo hi2) throws IOException; + + /** + * Callback for handling two reigons that overlap in some arbitrary way. + * This is a specific case of region overlap, and called for each possible + * pair. If two regions have the same start key, the handleDuplicateStartKeys + * method is called. + * @param hi1 one of the overlapping HbckInfo + * @param hi2 the other overlapping HbckInfo + */ + void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) + throws IOException; + + /** + * Callback for handling a region hole between two keys. + * @param holeStartKey key at the beginning of the region hole + * @param holeEndKey key at the end of the region hole + + */ + void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeEndKey) + throws IOException; + + /** + * Callback for handling an group of regions that overlap. + * @param overlap Collection of overlapping regions. + */ + void handleOverlapGroup(Collection overlap) throws IOException; +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandlerImpl.java b/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandlerImpl.java new file mode 100644 index 0000000..b142b0c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandlerImpl.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util.hbck; + +import java.io.IOException; +import java.util.Collection; + +import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo; +import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo; + +/** + * Simple implementation of TableIntegrityErrorHandler. Can be used as a base + * class. + */ +abstract public class TableIntegrityErrorHandlerImpl implements + TableIntegrityErrorHandler { + TableInfo ti; + + /** + * {@inheritDoc} + */ + @Override + public TableInfo getTableInfo() { + return ti; + } + + /** + * {@inheritDoc} + */ + @Override + public void setTableInfo(TableInfo ti2) { + this.ti = ti2; + } + + /** + * {@inheritDoc} + */ + @Override + public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException { + } + + /** + * {@inheritDoc} + */ + @Override + public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException { + } + + /** + * {@inheritDoc} + */ + @Override + public void handleDegenerateRegion(HbckInfo hi) throws IOException { + } + + /** + * {@inheritDoc} + */ + @Override + public void handleDuplicateStartKeys(HbckInfo hi1, HbckInfo hi2) + throws IOException { + } + + /** + * {@inheritDoc} + */ + @Override + public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) + throws IOException { + } + + /** + * {@inheritDoc} + */ + @Override + public void handleHoleInRegionChain(byte[] holeStart, byte[] holeEnd) + throws IOException { + } + + /** + * {@inheritDoc} + */ + @Override + public void handleOverlapGroup(Collection overlap) + throws IOException { + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterId.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterId.java new file mode 100644 index 0000000..0b1b647 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterId.java @@ -0,0 +1,74 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.zookeeper; + +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.zookeeper.KeeperException; + +/** + * Publishes and synchronizes a unique identifier specific to a given HBase + * cluster. The stored identifier is read from the file system by the active + * master on startup, and is subsequently available to all watchers (including + * clients). + */ +public class ClusterId { + private ZooKeeperWatcher watcher; + private Abortable abortable; + private String id; + + public ClusterId(ZooKeeperWatcher watcher, Abortable abortable) { + this.watcher = watcher; + this.abortable = abortable; + } + + public boolean hasId() { + return getId() != null; + } + + public String getId() { + try { + if (id == null) { + id = readClusterIdZNode(watcher); + } + } catch (KeeperException ke) { + abortable.abort("Unexpected exception from ZooKeeper reading cluster ID", + ke); + } + return id; + } + + public static String readClusterIdZNode(ZooKeeperWatcher watcher) + throws KeeperException { + if (ZKUtil.checkExists(watcher, watcher.clusterIdZNode) != -1) { + byte[] data = ZKUtil.getData(watcher, watcher.clusterIdZNode); + if (data != null) { + return Bytes.toString(data); + } + } + return null; + } + + public static void setClusterId(ZooKeeperWatcher watcher, String id) + throws KeeperException { + ZKUtil.createSetData(watcher, watcher.clusterIdZNode, Bytes.toBytes(id)); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterStatusTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterStatusTracker.java new file mode 100644 index 0000000..7e1a952 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterStatusTracker.java @@ -0,0 +1,86 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.zookeeper.KeeperException; + +/** + * Tracker on cluster settings up in zookeeper. + * This is not related to {@link ClusterStatus}. That class is a data structure + * that holds snapshot of current view on cluster. This class is about tracking + * cluster attributes up in zookeeper. + * + */ +public class ClusterStatusTracker extends ZooKeeperNodeTracker { + private static final Log LOG = LogFactory.getLog(ClusterStatusTracker.class); + + /** + * Creates a cluster status tracker. + * + *

    After construction, use {@link #start} to kick off tracking. + * + * @param watcher + * @param abortable + */ + public ClusterStatusTracker(ZooKeeperWatcher watcher, Abortable abortable) { + super(watcher, watcher.clusterStateZNode, abortable); + } + + /** + * Checks if cluster is up. + * @return true if root region location is available, false if not + */ + public boolean isClusterUp() { + return super.getData(false) != null; + } + + /** + * Sets the cluster as up. + * @throws KeeperException unexpected zk exception + */ + public void setClusterUp() + throws KeeperException { + byte [] upData = Bytes.toBytes(new java.util.Date().toString()); + try { + ZKUtil.createAndWatch(watcher, watcher.clusterStateZNode, upData); + } catch(KeeperException.NodeExistsException nee) { + ZKUtil.setData(watcher, watcher.clusterStateZNode, upData); + } + } + + /** + * Sets the cluster as down by deleting the znode. + * @throws KeeperException unexpected zk exception + */ + public void setClusterDown() + throws KeeperException { + try { + ZKUtil.deleteNode(watcher, watcher.clusterStateZNode); + } catch(KeeperException.NoNodeException nne) { + LOG.warn("Attempted to set cluster as down but already down, cluster " + + "state node (" + watcher.clusterStateZNode + ") not found"); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/DrainingServerTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/DrainingServerTracker.java new file mode 100644 index 0000000..e90edf9 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/DrainingServerTracker.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.io.IOException; +import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.zookeeper.KeeperException; + +/** + * Tracks the list of draining region servers via ZK. + * + *

    This class is responsible for watching for changes to the draining + * servers list. It handles adds/deletes in the draining RS list and + * watches each node. + * + *

    If an RS gets deleted from draining list, we call + * {@link ServerManager#removeServerFromDrainList(ServerName)} + * + *

    If an RS gets added to the draining list, we add a watcher to it and call + * {@link ServerManager#addServerToDrainList(ServerName)} + * + */ +public class DrainingServerTracker extends ZooKeeperListener { + private static final Log LOG = LogFactory.getLog(DrainingServerTracker.class); + + private ServerManager serverManager; + private NavigableSet drainingServers = new TreeSet(); + private Abortable abortable; + + public DrainingServerTracker(ZooKeeperWatcher watcher, + Abortable abortable, ServerManager serverManager) { + super(watcher); + this.abortable = abortable; + this.serverManager = serverManager; + } + + /** + * Starts the tracking of draining RegionServers. + * + *

    All Draining RSs will be tracked after this method is called. + * + * @throws KeeperException + */ + public void start() throws KeeperException, IOException { + watcher.registerListener(this); + List servers = + ZKUtil.listChildrenAndWatchThem(watcher, watcher.drainingZNode); + add(servers); + } + + private void add(final List servers) throws IOException { + synchronized(this.drainingServers) { + this.drainingServers.clear(); + for (String n: servers) { + final ServerName sn = new ServerName(ZKUtil.getNodeName(n)); + this.drainingServers.add(sn); + this.serverManager.addServerToDrainList(sn); + LOG.info("Draining RS node created, adding to list [" + + sn + "]"); + + } + } + } + + private void remove(final ServerName sn) { + synchronized(this.drainingServers) { + this.drainingServers.remove(sn); + this.serverManager.removeServerFromDrainList(sn); + } + } + + @Override + public void nodeDeleted(final String path) { + if(path.startsWith(watcher.drainingZNode)) { + final ServerName sn = new ServerName(ZKUtil.getNodeName(path)); + LOG.info("Draining RS node deleted, removing from list [" + + sn + "]"); + remove(sn); + } + } + + @Override + public void nodeChildrenChanged(final String path) { + if(path.equals(watcher.drainingZNode)) { + try { + final List newNodes = + ZKUtil.listChildrenAndWatchThem(watcher, watcher.drainingZNode); + add(newNodes); + } catch (KeeperException e) { + abortable.abort("Unexpected zk exception getting RS nodes", e); + } catch (IOException e) { + abortable.abort("Unexpected zk exception getting RS nodes", e); + } + } + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java new file mode 100644 index 0000000..43a9e97 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java @@ -0,0 +1,157 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.Properties; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.util.Strings; +import org.apache.hadoop.net.DNS; +import org.apache.hadoop.util.StringUtils; +import org.apache.zookeeper.server.ServerConfig; +import org.apache.zookeeper.server.ZooKeeperServerMain; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; +import org.apache.zookeeper.server.quorum.QuorumPeerMain; + +/** + * HBase's version of ZooKeeper's QuorumPeer. When HBase is set to manage + * ZooKeeper, this class is used to start up QuorumPeer instances. By doing + * things in here rather than directly calling to ZooKeeper, we have more + * control over the process. This class uses {@link ZKConfig} to parse the + * zoo.cfg and inject variables from HBase's site.xml configuration in. + */ +public class HQuorumPeer { + + /** + * Parse ZooKeeper configuration from HBase XML config and run a QuorumPeer. + * @param args String[] of command line arguments. Not used. + */ + public static void main(String[] args) { + Configuration conf = HBaseConfiguration.create(); + try { + Properties zkProperties = ZKConfig.makeZKProps(conf); + writeMyID(zkProperties); + QuorumPeerConfig zkConfig = new QuorumPeerConfig(); + zkConfig.parseProperties(zkProperties); + + // login the zookeeper server principal (if using security) + ZKUtil.loginServer(conf, "hbase.zookeeper.server.keytab.file", + "hbase.zookeeper.server.kerberos.principal", + zkConfig.getClientPortAddress().getHostName()); + + runZKServer(zkConfig); + } catch (Exception e) { + e.printStackTrace(); + System.exit(-1); + } + } + + private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException, IOException { + if (zkConfig.isDistributed()) { + QuorumPeerMain qp = new QuorumPeerMain(); + qp.runFromConfig(zkConfig); + } else { + ZooKeeperServerMain zk = new ZooKeeperServerMain(); + ServerConfig serverConfig = new ServerConfig(); + serverConfig.readFrom(zkConfig); + zk.runFromConfig(serverConfig); + } + } + + private static boolean addressIsLocalHost(String address) { + return address.equals("localhost") || address.equals("127.0.0.1"); + } + + static void writeMyID(Properties properties) throws IOException { + long myId = -1; + + Configuration conf = HBaseConfiguration.create(); + String myAddress = Strings.domainNamePointerToHostName(DNS.getDefaultHost( + conf.get("hbase.zookeeper.dns.interface","default"), + conf.get("hbase.zookeeper.dns.nameserver","default"))); + + List ips = new ArrayList(); + + // Add what could be the best (configured) match + ips.add(myAddress.contains(".") ? + myAddress : + StringUtils.simpleHostname(myAddress)); + + // For all nics get all hostnames and IPs + Enumeration nics = NetworkInterface.getNetworkInterfaces(); + while(nics.hasMoreElements()) { + Enumeration rawAdrs = + ((NetworkInterface)nics.nextElement()).getInetAddresses(); + while(rawAdrs.hasMoreElements()) { + InetAddress inet = (InetAddress) rawAdrs.nextElement(); + ips.add(StringUtils.simpleHostname(inet.getHostName())); + ips.add(inet.getHostAddress()); + } + } + + for (Entry entry : properties.entrySet()) { + String key = entry.getKey().toString().trim(); + String value = entry.getValue().toString().trim(); + if (key.startsWith("server.")) { + int dot = key.indexOf('.'); + long id = Long.parseLong(key.substring(dot + 1)); + String[] parts = value.split(":"); + String address = parts[0]; + if (addressIsLocalHost(address) || ips.contains(address)) { + myId = id; + break; + } + } + } + + // Set the max session timeout from the provided client-side timeout + properties.setProperty("maxSessionTimeout", + conf.get("zookeeper.session.timeout", "180000")); + + if (myId == -1) { + throw new IOException("Could not find my address: " + myAddress + + " in list of ZooKeeper quorum servers"); + } + + String dataDirStr = properties.get("dataDir").toString().trim(); + File dataDir = new File(dataDirStr); + if (!dataDir.isDirectory()) { + if (!dataDir.mkdirs()) { + throw new IOException("Unable to create data dir " + dataDir); + } + } + + File myIdFile = new File(dataDir, "myid"); + PrintWriter w = new PrintWriter(myIdFile); + w.println(myId); + w.close(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java new file mode 100644 index 0000000..4da6f96 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java @@ -0,0 +1,47 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HRegionInfo; + +/** + * Tracks the unassigned zookeeper node used by the META table. + *

    + * If META is already assigned when instantiating this class, you will not + * receive any notification for that assignment. You will receive a + * notification after META has been successfully assigned to a new location. + */ +public class MetaNodeTracker extends ZooKeeperNodeTracker { + /** + * Creates a meta node tracker. + * @param watcher + * @param abortable + */ + public MetaNodeTracker(final ZooKeeperWatcher watcher, final Abortable abortable) { + super(watcher, ZKUtil.joinZNode(watcher.assignmentZNode, + HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()), abortable); + } + + @Override + public void nodeDeleted(String path) { + super.nodeDeleted(path); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java new file mode 100644 index 0000000..2551e68 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java @@ -0,0 +1,381 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.Reader; +import java.net.BindException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.zookeeper.server.NIOServerCnxnFactory; +import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.server.persistence.FileTxnLog; + +/** + * TODO: Most of the code in this class is ripped from ZooKeeper tests. Instead + * of redoing it, we should contribute updates to their code which let us more + * easily access testing helper objects. + */ +public class MiniZooKeeperCluster { + private static final Log LOG = LogFactory.getLog(MiniZooKeeperCluster.class); + + private static final int TICK_TIME = 2000; + private static final int CONNECTION_TIMEOUT = 30000; + + private boolean started; + + /** The default port. If zero, we use a random port. */ + private int defaultClientPort = 0; + + private int clientPort; + + private List standaloneServerFactoryList; + private List zooKeeperServers; + private List clientPortList; + + private int activeZKServerIndex; + private int tickTime = 0; + + private Configuration configuration; + + public MiniZooKeeperCluster() { + this(new Configuration()); + } + + public MiniZooKeeperCluster(Configuration configuration) { + this.started = false; + this.configuration = configuration; + activeZKServerIndex = -1; + zooKeeperServers = new ArrayList(); + clientPortList = new ArrayList(); + standaloneServerFactoryList = new ArrayList(); + } + + public void setDefaultClientPort(int clientPort) { + if (clientPort <= 0) { + throw new IllegalArgumentException("Invalid default ZK client port: " + + clientPort); + } + this.defaultClientPort = clientPort; + } + + /** + * Selects a ZK client port. Returns the default port if specified. + * Otherwise, returns a random port. The random port is selected from the + * range between 49152 to 65535. These ports cannot be registered with IANA + * and are intended for dynamic allocation (see http://bit.ly/dynports). + */ + private int selectClientPort() { + if (defaultClientPort > 0) { + return defaultClientPort; + } + return 0xc000 + new Random().nextInt(0x3f00); + } + + public void setTickTime(int tickTime) { + this.tickTime = tickTime; + } + + public int getBackupZooKeeperServerNum() { + return zooKeeperServers.size()-1; + } + + public int getZooKeeperServerNum() { + return zooKeeperServers.size(); + } + + // / XXX: From o.a.zk.t.ClientBase + private static void setupTestEnv() { + // during the tests we run with 100K prealloc in the logs. + // on windows systems prealloc of 64M was seen to take ~15seconds + // resulting in test failure (client timeout on first session). + // set env and directly in order to handle static init/gc issues + System.setProperty("zookeeper.preAllocSize", "100"); + FileTxnLog.setPreallocSize(100 * 1024); + } + + public int startup(File baseDir) throws IOException, InterruptedException { + return startup(baseDir,1); + } + + /** + * @param baseDir + * @param numZooKeeperServers + * @return ClientPort server bound to. + * @throws IOException + * @throws InterruptedException + */ + public int startup(File baseDir, int numZooKeeperServers) throws IOException, + InterruptedException { + if (numZooKeeperServers <= 0) + return -1; + + setupTestEnv(); + shutdown(); + + int tentativePort = selectClientPort(); + + // running all the ZK servers + for (int i = 0; i < numZooKeeperServers; i++) { + File dir = new File(baseDir, "zookeeper_"+i).getAbsoluteFile(); + recreateDir(dir); + int tickTimeToUse; + if (this.tickTime > 0) { + tickTimeToUse = this.tickTime; + } else { + tickTimeToUse = TICK_TIME; + } + ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse); + NIOServerCnxnFactory standaloneServerFactory; + while (true) { + try { + standaloneServerFactory = new NIOServerCnxnFactory(); + standaloneServerFactory.configure( + new InetSocketAddress(tentativePort), + configuration.getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, + 1000)); + } catch (BindException e) { + LOG.debug("Failed binding ZK Server to client port: " + + tentativePort); + // This port is already in use, try to use another. + tentativePort++; + continue; + } + break; + } + + // Start up this ZK server + standaloneServerFactory.startup(server); + if (!waitForServerUp(tentativePort, CONNECTION_TIMEOUT)) { + throw new IOException("Waiting for startup of standalone server"); + } + + // We have selected this port as a client port. + clientPortList.add(tentativePort); + standaloneServerFactoryList.add(standaloneServerFactory); + zooKeeperServers.add(server); + } + + // set the first one to be active ZK; Others are backups + activeZKServerIndex = 0; + started = true; + clientPort = clientPortList.get(activeZKServerIndex); + LOG.info("Started MiniZK Cluster and connect 1 ZK server " + + "on client port: " + clientPort); + return clientPort; + } + + private void recreateDir(File dir) throws IOException { + if (dir.exists()) { + FileUtil.fullyDelete(dir); + } + try { + dir.mkdirs(); + } catch (SecurityException e) { + throw new IOException("creating dir: " + dir, e); + } + } + + /** + * @throws IOException + */ + public void shutdown() throws IOException { + if (!started) { + return; + } + // shut down all the zk servers + for (int i = 0; i < standaloneServerFactoryList.size(); i++) { + NIOServerCnxnFactory standaloneServerFactory = + standaloneServerFactoryList.get(i); + int clientPort = clientPortList.get(i); + + standaloneServerFactory.shutdown(); + if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) { + throw new IOException("Waiting for shutdown of standalone server"); + } + } + + // clear everything + started = false; + activeZKServerIndex = 0; + standaloneServerFactoryList.clear(); + clientPortList.clear(); + zooKeeperServers.clear(); + + LOG.info("Shutdown MiniZK cluster with all ZK servers"); + } + + /**@return clientPort return clientPort if there is another ZK backup can run + * when killing the current active; return -1, if there is no backups. + * @throws IOException + * @throws InterruptedException + */ + public int killCurrentActiveZooKeeperServer() throws IOException, + InterruptedException { + if (!started || activeZKServerIndex < 0 ) { + return -1; + } + + // Shutdown the current active one + NIOServerCnxnFactory standaloneServerFactory = + standaloneServerFactoryList.get(activeZKServerIndex); + int clientPort = clientPortList.get(activeZKServerIndex); + + standaloneServerFactory.shutdown(); + if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) { + throw new IOException("Waiting for shutdown of standalone server"); + } + + // remove the current active zk server + standaloneServerFactoryList.remove(activeZKServerIndex); + clientPortList.remove(activeZKServerIndex); + zooKeeperServers.remove(activeZKServerIndex); + LOG.info("Kill the current active ZK servers in the cluster " + + "on client port: " + clientPort); + + if (standaloneServerFactoryList.size() == 0) { + // there is no backup servers; + return -1; + } + clientPort = clientPortList.get(activeZKServerIndex); + LOG.info("Activate a backup zk server in the cluster " + + "on client port: " + clientPort); + // return the next back zk server's port + return clientPort; + } + + /** + * Kill one back up ZK servers + * @throws IOException + * @throws InterruptedException + */ + public void killOneBackupZooKeeperServer() throws IOException, + InterruptedException { + if (!started || activeZKServerIndex < 0 || + standaloneServerFactoryList.size() <= 1) { + return ; + } + + int backupZKServerIndex = activeZKServerIndex+1; + // Shutdown the current active one + NIOServerCnxnFactory standaloneServerFactory = + standaloneServerFactoryList.get(backupZKServerIndex); + int clientPort = clientPortList.get(backupZKServerIndex); + + standaloneServerFactory.shutdown(); + if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) { + throw new IOException("Waiting for shutdown of standalone server"); + } + + // remove this backup zk server + standaloneServerFactoryList.remove(backupZKServerIndex); + clientPortList.remove(backupZKServerIndex); + zooKeeperServers.remove(backupZKServerIndex); + LOG.info("Kill one backup ZK servers in the cluster " + + "on client port: " + clientPort); + } + + // XXX: From o.a.zk.t.ClientBase + private static boolean waitForServerDown(int port, long timeout) { + long start = System.currentTimeMillis(); + while (true) { + try { + Socket sock = new Socket("localhost", port); + try { + OutputStream outstream = sock.getOutputStream(); + outstream.write("stat".getBytes()); + outstream.flush(); + } finally { + sock.close(); + } + } catch (IOException e) { + return true; + } + + if (System.currentTimeMillis() > start + timeout) { + break; + } + try { + Thread.sleep(250); + } catch (InterruptedException e) { + // ignore + } + } + return false; + } + + // XXX: From o.a.zk.t.ClientBase + private static boolean waitForServerUp(int port, long timeout) { + long start = System.currentTimeMillis(); + while (true) { + try { + Socket sock = new Socket("localhost", port); + BufferedReader reader = null; + try { + OutputStream outstream = sock.getOutputStream(); + outstream.write("stat".getBytes()); + outstream.flush(); + + Reader isr = new InputStreamReader(sock.getInputStream()); + reader = new BufferedReader(isr); + String line = reader.readLine(); + if (line != null && line.startsWith("Zookeeper version:")) { + return true; + } + } finally { + sock.close(); + if (reader != null) { + reader.close(); + } + } + } catch (IOException e) { + // ignore as this is expected + LOG.info("server localhost:" + port + " not up " + e); + } + + if (System.currentTimeMillis() > start + timeout) { + break; + } + try { + Thread.sleep(250); + } catch (InterruptedException e) { + // ignore + } + } + return false; + } + + public int getClientPort() { + return clientPort; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java new file mode 100644 index 0000000..ba5db2d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java @@ -0,0 +1,650 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.RetryCounter; +import org.apache.hadoop.hbase.util.RetryCounterFactory; +import org.apache.zookeeper.AsyncCallback; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.Op; +import org.apache.zookeeper.OpResult; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.ZooKeeper.States; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.apache.zookeeper.proto.CreateRequest; +import org.apache.zookeeper.proto.SetDataRequest; + +/** + * A zookeeper that can handle 'recoverable' errors. + * To handle recoverable errors, developers need to realize that there are two + * classes of requests: idempotent and non-idempotent requests. Read requests + * and unconditional sets and deletes are examples of idempotent requests, they + * can be reissued with the same results. + * (Although, the delete may throw a NoNodeException on reissue its effect on + * the ZooKeeper state is the same.) Non-idempotent requests need special + * handling, application and library writers need to keep in mind that they may + * need to encode information in the data or name of znodes to detect + * retries. A simple example is a create that uses a sequence flag. + * If a process issues a create("/x-", ..., SEQUENCE) and gets a connection + * loss exception, that process will reissue another + * create("/x-", ..., SEQUENCE) and get back x-111. When the process does a + * getChildren("/"), it sees x-1,x-30,x-109,x-110,x-111, now it could be + * that x-109 was the result of the previous create, so the process actually + * owns both x-109 and x-111. An easy way around this is to use "x-process id-" + * when doing the create. If the process is using an id of 352, before reissuing + * the create it will do a getChildren("/") and see "x-222-1", "x-542-30", + * "x-352-109", x-333-110". The process will know that the original create + * succeeded an the znode it created is "x-352-109". + * @see "http://wiki.apache.org/hadoop/ZooKeeper/ErrorHandling" + */ +public class RecoverableZooKeeper { + private static final Log LOG = LogFactory.getLog(RecoverableZooKeeper.class); + // the actual ZooKeeper client instance + private volatile ZooKeeper zk; + private final RetryCounterFactory retryCounterFactory; + // An identifier of this process in the cluster + private final String identifier; + private final byte[] id; + private Watcher watcher; + private int sessionTimeout; + private String quorumServers; + + // The metadata attached to each piece of data has the + // format: + // 1-byte constant + // 4-byte big-endian integer (length of next field) + // identifier corresponding uniquely to this process + // It is prepended to the data supplied by the user. + + // the magic number is to be backward compatible + private static final byte MAGIC =(byte) 0XFF; + private static final int MAGIC_SIZE = Bytes.SIZEOF_BYTE; + private static final int ID_LENGTH_OFFSET = MAGIC_SIZE; + private static final int ID_LENGTH_SIZE = Bytes.SIZEOF_INT; + + public RecoverableZooKeeper(String quorumServers, int sessionTimeout, + Watcher watcher, int maxRetries, int retryIntervalMillis) + throws IOException { + this.zk = new ZooKeeper(quorumServers, sessionTimeout, watcher); + this.retryCounterFactory = + new RetryCounterFactory(maxRetries, retryIntervalMillis); + + // the identifier = processID@hostName + this.identifier = ManagementFactory.getRuntimeMXBean().getName(); + LOG.info("The identifier of this process is " + identifier); + this.id = Bytes.toBytes(identifier); + this.watcher = watcher; + this.sessionTimeout = sessionTimeout; + this.quorumServers = quorumServers; + } + + public void reconnectAfterExpiration() + throws IOException, InterruptedException { + LOG.info("Closing dead ZooKeeper connection, session" + + " was: 0x"+Long.toHexString(zk.getSessionId())); + zk.close(); + this.zk = new ZooKeeper(this.quorumServers, + this.sessionTimeout, this.watcher); + LOG.info("Recreated a ZooKeeper, session" + + " is: 0x"+Long.toHexString(zk.getSessionId())); + } + + /** + * delete is an idempotent operation. Retry before throwing exception. + * This function will not throw NoNodeException if the path does not + * exist. + */ + public void delete(String path, int version) + throws InterruptedException, KeeperException { + RetryCounter retryCounter = retryCounterFactory.create(); + boolean isRetry = false; // False for first attempt, true for all retries. + while (true) { + try { + zk.delete(path, version); + return; + } catch (KeeperException e) { + switch (e.code()) { + case NONODE: + if (isRetry) { + LOG.info("Node " + path + " already deleted. Assuming that a " + + "previous attempt succeeded."); + return; + } + LOG.warn("Node " + path + " already deleted, and this is not a " + + "retry"); + throw e; + + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + retryOrThrow(retryCounter, e, "delete"); + break; + + default: + throw e; + } + } + retryCounter.sleepUntilNextRetry(); + retryCounter.useRetry(); + isRetry = true; + } + } + + /** + * exists is an idempotent operation. Retry before throwing exception + * @return A Stat instance + */ + public Stat exists(String path, Watcher watcher) + throws KeeperException, InterruptedException { + RetryCounter retryCounter = retryCounterFactory.create(); + while (true) { + try { + return zk.exists(path, watcher); + } catch (KeeperException e) { + switch (e.code()) { + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + retryOrThrow(retryCounter, e, "exists"); + break; + + default: + throw e; + } + } + retryCounter.sleepUntilNextRetry(); + retryCounter.useRetry(); + } + } + + /** + * exists is an idempotent operation. Retry before throwing exception + * @return A Stat instance + */ + public Stat exists(String path, boolean watch) + throws KeeperException, InterruptedException { + RetryCounter retryCounter = retryCounterFactory.create(); + while (true) { + try { + return zk.exists(path, watch); + } catch (KeeperException e) { + switch (e.code()) { + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + retryOrThrow(retryCounter, e, "exists"); + break; + + default: + throw e; + } + } + retryCounter.sleepUntilNextRetry(); + retryCounter.useRetry(); + } + } + + private void retryOrThrow(RetryCounter retryCounter, KeeperException e, + String opName) throws KeeperException { + LOG.warn("Possibly transient ZooKeeper exception: " + e); + if (!retryCounter.shouldRetry()) { + LOG.error("ZooKeeper " + opName + " failed after " + + retryCounter.getMaxRetries() + " retries"); + throw e; + } + } + + /** + * getChildren is an idempotent operation. Retry before throwing exception + * @return List of children znodes + */ + public List getChildren(String path, Watcher watcher) + throws KeeperException, InterruptedException { + RetryCounter retryCounter = retryCounterFactory.create(); + while (true) { + try { + return zk.getChildren(path, watcher); + } catch (KeeperException e) { + switch (e.code()) { + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + retryOrThrow(retryCounter, e, "getChildren"); + break; + + default: + throw e; + } + } + retryCounter.sleepUntilNextRetry(); + retryCounter.useRetry(); + } + } + + /** + * getChildren is an idempotent operation. Retry before throwing exception + * @return List of children znodes + */ + public List getChildren(String path, boolean watch) + throws KeeperException, InterruptedException { + RetryCounter retryCounter = retryCounterFactory.create(); + while (true) { + try { + return zk.getChildren(path, watch); + } catch (KeeperException e) { + switch (e.code()) { + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + retryOrThrow(retryCounter, e, "getChildren"); + break; + + default: + throw e; + } + } + retryCounter.sleepUntilNextRetry(); + retryCounter.useRetry(); + } + } + + /** + * getData is an idempotent operation. Retry before throwing exception + * @return Data + */ + public byte[] getData(String path, Watcher watcher, Stat stat) + throws KeeperException, InterruptedException { + RetryCounter retryCounter = retryCounterFactory.create(); + while (true) { + try { + byte[] revData = zk.getData(path, watcher, stat); + return this.removeMetaData(revData); + } catch (KeeperException e) { + switch (e.code()) { + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + retryOrThrow(retryCounter, e, "getData"); + break; + + default: + throw e; + } + } + retryCounter.sleepUntilNextRetry(); + retryCounter.useRetry(); + } + } + + /** + * getData is an idemnpotent operation. Retry before throwing exception + * @return Data + */ + public byte[] getData(String path, boolean watch, Stat stat) + throws KeeperException, InterruptedException { + RetryCounter retryCounter = retryCounterFactory.create(); + while (true) { + try { + byte[] revData = zk.getData(path, watch, stat); + return this.removeMetaData(revData); + } catch (KeeperException e) { + switch (e.code()) { + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + retryOrThrow(retryCounter, e, "getData"); + break; + + default: + throw e; + } + } + retryCounter.sleepUntilNextRetry(); + retryCounter.useRetry(); + } + } + + /** + * setData is NOT an idempotent operation. Retry may cause BadVersion Exception + * Adding an identifier field into the data to check whether + * badversion is caused by the result of previous correctly setData + * @return Stat instance + */ + public Stat setData(String path, byte[] data, int version) + throws KeeperException, InterruptedException { + RetryCounter retryCounter = retryCounterFactory.create(); + byte[] newData = appendMetaData(data); + while (true) { + try { + return zk.setData(path, newData, version); + } catch (KeeperException e) { + switch (e.code()) { + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + retryOrThrow(retryCounter, e, "setData"); + break; + case BADVERSION: + // try to verify whether the previous setData success or not + try{ + Stat stat = new Stat(); + byte[] revData = zk.getData(path, false, stat); + if (Bytes.equals(revData, newData)) { + // the bad version is caused by previous successful setData + return stat; + } + } catch(KeeperException keeperException){ + // the ZK is not reliable at this moment. just throwing exception + throw keeperException; + } + + // throw other exceptions and verified bad version exceptions + default: + throw e; + } + } + retryCounter.sleepUntilNextRetry(); + retryCounter.useRetry(); + } + } + + /** + *

    + * NONSEQUENTIAL create is idempotent operation. + * Retry before throwing exceptions. + * But this function will not throw the NodeExist exception back to the + * application. + *

    + *

    + * But SEQUENTIAL is NOT idempotent operation. It is necessary to add + * identifier to the path to verify, whether the previous one is successful + * or not. + *

    + * + * @return Path + */ + public String create(String path, byte[] data, List acl, + CreateMode createMode) + throws KeeperException, InterruptedException { + byte[] newData = appendMetaData(data); + switch (createMode) { + case EPHEMERAL: + case PERSISTENT: + return createNonSequential(path, newData, acl, createMode); + + case EPHEMERAL_SEQUENTIAL: + case PERSISTENT_SEQUENTIAL: + return createSequential(path, newData, acl, createMode); + + default: + throw new IllegalArgumentException("Unrecognized CreateMode: " + + createMode); + } + } + + private String createNonSequential(String path, byte[] data, List acl, + CreateMode createMode) throws KeeperException, InterruptedException { + RetryCounter retryCounter = retryCounterFactory.create(); + boolean isRetry = false; // False for first attempt, true for all retries. + while (true) { + try { + return zk.create(path, data, acl, createMode); + } catch (KeeperException e) { + switch (e.code()) { + case NODEEXISTS: + if (isRetry) { + // If the connection was lost, there is still a possibility that + // we have successfully created the node at our previous attempt, + // so we read the node and compare. + byte[] currentData = zk.getData(path, false, null); + if (currentData != null && + Bytes.compareTo(currentData, data) == 0) { + // We successfully created a non-sequential node + return path; + } + LOG.error("Node " + path + " already exists with " + + Bytes.toStringBinary(currentData) + ", could not write " + + Bytes.toStringBinary(data)); + throw e; + } + LOG.info("Node " + path + " already exists and this is not a " + + "retry"); + throw e; + + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + retryOrThrow(retryCounter, e, "create"); + break; + + default: + throw e; + } + } + retryCounter.sleepUntilNextRetry(); + retryCounter.useRetry(); + isRetry = true; + } + } + + private String createSequential(String path, byte[] data, + List acl, CreateMode createMode) + throws KeeperException, InterruptedException { + RetryCounter retryCounter = retryCounterFactory.create(); + boolean first = true; + String newPath = path+this.identifier; + while (true) { + try { + if (!first) { + // Check if we succeeded on a previous attempt + String previousResult = findPreviousSequentialNode(newPath); + if (previousResult != null) { + return previousResult; + } + } + first = false; + return zk.create(newPath, data, acl, createMode); + } catch (KeeperException e) { + switch (e.code()) { + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + retryOrThrow(retryCounter, e, "create"); + break; + + default: + throw e; + } + } + retryCounter.sleepUntilNextRetry(); + retryCounter.useRetry(); + } + } + + /** + * Convert Iterable of {@link ZKOp} we got into the ZooKeeper.Op + * instances to actually pass to multi (need to do this in order to appendMetaData). + */ + private Iterable prepareZKMulti(Iterable ops) + throws UnsupportedOperationException { + if(ops == null) return null; + + List preparedOps = new LinkedList(); + for (Op op : ops) { + if (op.getType() == ZooDefs.OpCode.create) { + CreateRequest create = (CreateRequest)op.toRequestRecord(); + preparedOps.add(Op.create(create.getPath(), appendMetaData(create.getData()), + create.getAcl(), create.getFlags())); + } else if (op.getType() == ZooDefs.OpCode.delete) { + // no need to appendMetaData for delete + preparedOps.add(op); + } else if (op.getType() == ZooDefs.OpCode.setData) { + SetDataRequest setData = (SetDataRequest)op.toRequestRecord(); + preparedOps.add(Op.setData(setData.getPath(), appendMetaData(setData.getData()), + setData.getVersion())); + } else { + throw new UnsupportedOperationException("Unexpected ZKOp type: " + op.getClass().getName()); + } + } + return preparedOps; + } + + /** + * Run multiple operations in a transactional manner. Retry before throwing exception + */ + public List multi(Iterable ops) + throws KeeperException, InterruptedException { + RetryCounter retryCounter = retryCounterFactory.create(); + Iterable multiOps = prepareZKMulti(ops); + while (true) { + try { + return zk.multi(multiOps); + } catch (KeeperException e) { + switch (e.code()) { + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + retryOrThrow(retryCounter, e, "multi"); + break; + + default: + throw e; + } + } + retryCounter.sleepUntilNextRetry(); + retryCounter.useRetry(); + } + } + + private String findPreviousSequentialNode(String path) + throws KeeperException, InterruptedException { + int lastSlashIdx = path.lastIndexOf('/'); + assert(lastSlashIdx != -1); + String parent = path.substring(0, lastSlashIdx); + String nodePrefix = path.substring(lastSlashIdx+1); + + List nodes = zk.getChildren(parent, false); + List matching = filterByPrefix(nodes, nodePrefix); + for (String node : matching) { + String nodePath = parent + "/" + node; + Stat stat = zk.exists(nodePath, false); + if (stat != null) { + return nodePath; + } + } + return null; + } + + public byte[] removeMetaData(byte[] data) { + if(data == null || data.length == 0) { + return data; + } + // check the magic data; to be backward compatible + byte magic = data[0]; + if(magic != MAGIC) { + return data; + } + + int idLength = Bytes.toInt(data, ID_LENGTH_OFFSET); + int dataLength = data.length-MAGIC_SIZE-ID_LENGTH_SIZE-idLength; + int dataOffset = MAGIC_SIZE+ID_LENGTH_SIZE+idLength; + + byte[] newData = new byte[dataLength]; + System.arraycopy(data, dataOffset, newData, 0, dataLength); + + return newData; + + } + + private byte[] appendMetaData(byte[] data) { + if(data == null || data.length == 0){ + return data; + } + + byte[] newData = new byte[MAGIC_SIZE+ID_LENGTH_SIZE+id.length+data.length]; + int pos = 0; + pos = Bytes.putByte(newData, pos, MAGIC); + pos = Bytes.putInt(newData, pos, id.length); + pos = Bytes.putBytes(newData, pos, id, 0, id.length); + pos = Bytes.putBytes(newData, pos, data, 0, data.length); + + return newData; + } + + public long getSessionId() { + return zk.getSessionId(); + } + + public void close() throws InterruptedException { + zk.close(); + } + + public States getState() { + return zk.getState(); + } + + public ZooKeeper getZooKeeper() { + return zk; + } + + public byte[] getSessionPasswd() { + return zk.getSessionPasswd(); + } + + public void sync(String path, AsyncCallback.VoidCallback cb, Object ctx) { + this.zk.sync(path, null, null); + } + + /** + * Filters the given node list by the given prefixes. + * This method is all-inclusive--if any element in the node list starts + * with any of the given prefixes, then it is included in the result. + * + * @param nodes the nodes to filter + * @param prefixes the prefixes to include in the result + * @return list of every element that starts with one of the prefixes + */ + private static List filterByPrefix(List nodes, + String... prefixes) { + List lockChildren = new ArrayList(); + for (String child : nodes){ + for (String prefix : prefixes){ + if (child.startsWith(prefix)){ + lockChildren.add(child); + break; + } + } + } + return lockChildren; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java new file mode 100644 index 0000000..b3b2130 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java @@ -0,0 +1,130 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.zookeeper.KeeperException; + +/** + * Tracks the online region servers via ZK. + * + *

    Handling of new RSs checking in is done via RPC. This class + * is only responsible for watching for expired nodes. It handles + * listening for changes in the RS node list and watching each node. + * + *

    If an RS node gets deleted, this automatically handles calling of + * {@link ServerManager#expireServer(ServerName)} + */ +public class RegionServerTracker extends ZooKeeperListener { + private static final Log LOG = LogFactory.getLog(RegionServerTracker.class); + private NavigableSet regionServers = new TreeSet(); + private ServerManager serverManager; + private Abortable abortable; + + public RegionServerTracker(ZooKeeperWatcher watcher, + Abortable abortable, ServerManager serverManager) { + super(watcher); + this.abortable = abortable; + this.serverManager = serverManager; + } + + /** + * Starts the tracking of online RegionServers. + * + *

    All RSs will be tracked after this method is called. + * + * @throws KeeperException + * @throws IOException + */ + public void start() throws KeeperException, IOException { + watcher.registerListener(this); + List servers = + ZKUtil.listChildrenAndWatchThem(watcher, watcher.rsZNode); + add(servers); + } + + private void add(final List servers) throws IOException { + synchronized(this.regionServers) { + this.regionServers.clear(); + for (String n: servers) { + ServerName sn = ServerName.parseServerName(ZKUtil.getNodeName(n)); + this.regionServers.add(sn); + } + } + } + + private void remove(final ServerName sn) { + synchronized(this.regionServers) { + this.regionServers.remove(sn); + } + } + + @Override + public void nodeDeleted(String path) { + if (path.startsWith(watcher.rsZNode)) { + String serverName = ZKUtil.getNodeName(path); + LOG.info("RegionServer ephemeral node deleted, processing expiration [" + + serverName + "]"); + ServerName sn = ServerName.parseServerName(serverName); + if (!serverManager.isServerOnline(sn)) { + LOG.warn(serverName.toString() + " is not online or isn't known to the master."+ + "The latter could be caused by a DNS misconfiguration."); + return; + } + remove(sn); + this.serverManager.expireServer(sn); + } + } + + @Override + public void nodeChildrenChanged(String path) { + if (path.equals(watcher.rsZNode)) { + try { + List servers = + ZKUtil.listChildrenAndWatchThem(watcher, watcher.rsZNode); + add(servers); + } catch (IOException e) { + abortable.abort("Unexpected zk exception getting RS nodes", e); + } catch (KeeperException e) { + abortable.abort("Unexpected zk exception getting RS nodes", e); + } + } + } + + /** + * Gets the online servers. + * @return list of online servers + */ + public List getOnlineServers() { + synchronized (this.regionServers) { + return new ArrayList(this.regionServers); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java new file mode 100644 index 0000000..48a8b3d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java @@ -0,0 +1,107 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.catalog.RootLocationEditor; +import org.apache.hadoop.hbase.util.Addressing; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.zookeeper.KeeperException; + +/** + * Tracks the root region server location node in zookeeper. + * Root region location is set by {@link RootLocationEditor} usually called + * out of RegionServerServices. + * This class has a watcher on the root location and notices changes. + */ +public class RootRegionTracker extends ZooKeeperNodeTracker { + /** + * Creates a root region location tracker. + * + *

    After construction, use {@link #start} to kick off tracking. + * + * @param watcher + * @param abortable + */ + public RootRegionTracker(ZooKeeperWatcher watcher, Abortable abortable) { + super(watcher, watcher.rootServerZNode, abortable); + } + + /** + * Checks if the root region location is available. + * @return true if root region location is available, false if not + */ + public boolean isLocationAvailable() { + return super.getData(true) != null; + } + + /** + * Gets the root region location, if available. Null if not. Does not block. + * @return server name + * @throws InterruptedException + */ + public ServerName getRootRegionLocation() throws InterruptedException { + return dataToServerName(super.getData(true)); + } + + /** + * Gets the root region location, if available, and waits for up to the + * specified timeout if not immediately available. + * Given the zookeeper notification could be delayed, we will try to + * get the latest data. + * @param timeout maximum time to wait, in millis + * @return server name for server hosting root region formatted as per + * {@link ServerName}, or null if none available + * @throws InterruptedException if interrupted while waiting + */ + public ServerName waitRootRegionLocation(long timeout) + throws InterruptedException { + if (false == checkIfBaseNodeAvailable()) { + String errorMsg = "Check the value configured in 'zookeeper.znode.parent'. " + + "There could be a mismatch with the one configured in the master."; + LOG.error(errorMsg); + throw new IllegalArgumentException(errorMsg); + } + return dataToServerName(super.blockUntilAvailable(timeout, true)); + } + + /* + * @param data + * @return Returns null if data is null else converts passed data + * to a ServerName instance. + */ + private static ServerName dataToServerName(final byte [] data) { + // The str returned could be old style -- pre hbase-1502 -- which was + // hostname and port seperated by a colon rather than hostname, port and + // startcode delimited by a ','. + if (data == null || data.length <= 0) return null; + String str = Bytes.toString(data); + int index = str.indexOf(ServerName.SERVERNAME_SEPARATOR); + if (index != -1) { + // Presume its ServerName.toString() format. + return ServerName.parseServerName(str); + } + // Presume it a hostname:port format. + String hostname = Addressing.parseHostname(str); + int port = Addressing.parsePort(str); + return new ServerName(hostname, port, -1L); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java new file mode 100644 index 0000000..f14b026 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java @@ -0,0 +1,1020 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.executor.RegionTransitionData; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.zookeeper.AsyncCallback; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.Code; +import org.apache.zookeeper.KeeperException.NoNodeException; +import org.apache.zookeeper.KeeperException.NodeExistsException; +import org.apache.zookeeper.data.Stat; + +/** + * Utility class for doing region assignment in ZooKeeper. This class extends + * stuff done in {@link ZKUtil} to cover specific assignment operations. + *

    + * Contains only static methods and constants. + *

    + * Used by both the Master and RegionServer. + *

    + * All valid transitions outlined below: + *

    + * MASTER + *

      + *
    1. + * Master creates an unassigned node as OFFLINE. + * - Cluster startup and table enabling. + *
      2. + *
    2. + * Master forces an existing unassigned node to OFFLINE. + * - RegionServer failure. + * - Allows transitions from all states to OFFLINE. + *
      3. + *
    3. + * Master deletes an unassigned node that was in a OPENED state. + * - Normal region transitions. Besides cluster startup, no other deletions + * of unassigned nodes is allowed. + *
      4. + *
    4. + * Master deletes all unassigned nodes regardless of state. + * - Cluster startup before any assignment happens. + *
      5. + *
    + *

    + * REGIONSERVER + *

      + *
    1. + * RegionServer creates an unassigned node as CLOSING. + * - All region closes will do this in response to a CLOSE RPC from Master. + * - A node can never be transitioned to CLOSING, only created. + *
      2. + *
    2. + * RegionServer transitions an unassigned node from CLOSING to CLOSED. + * - Normal region closes. CAS operation. + *
      3. + *
    3. + * RegionServer transitions an unassigned node from OFFLINE to OPENING. + * - All region opens will do this in response to an OPEN RPC from the Master. + * - Normal region opens. CAS operation. + *
      4. + *
    4. + * RegionServer transitions an unassigned node from OPENING to OPENED. + * - Normal region opens. CAS operation. + *
      5. + *
    + */ +public class ZKAssign { + private static final Log LOG = LogFactory.getLog(ZKAssign.class); + + /** + * Gets the full path node name for the unassigned node for the specified + * region. + * @param zkw zk reference + * @param regionName region name + * @return full path node name + */ + public static String getNodeName(ZooKeeperWatcher zkw, String regionName) { + return ZKUtil.joinZNode(zkw.assignmentZNode, regionName); + } + + /** + * Gets the region name from the full path node name of an unassigned node. + * @param path full zk path + * @return region name + */ + public static String getRegionName(ZooKeeperWatcher zkw, String path) { + return path.substring(zkw.assignmentZNode.length()+1); + } + + // Master methods + + /** + * Creates a new unassigned node in the OFFLINE state for the specified region. + * + *

    Does not transition nodes from other states. If a node already exists + * for this region, a {@link NodeExistsException} will be thrown. + * + *

    Sets a watcher on the unassigned region node if the method is successful. + * + *

    This method should only be used during cluster startup and the enabling + * of a table. + * + * @param zkw zk reference + * @param region region to be created as offline + * @param serverName server event originates from + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NodeExistsException if node already exists + */ + public static void createNodeOffline(ZooKeeperWatcher zkw, HRegionInfo region, + ServerName serverName) + throws KeeperException, KeeperException.NodeExistsException { + createNodeOffline(zkw, region, serverName, EventType.M_ZK_REGION_OFFLINE); + } + + public static void createNodeOffline(ZooKeeperWatcher zkw, HRegionInfo region, + ServerName serverName, final EventType event) + throws KeeperException, KeeperException.NodeExistsException { + LOG.debug(zkw.prefix("Creating unassigned node for " + + region.getEncodedName() + " in OFFLINE state")); + RegionTransitionData data = new RegionTransitionData(event, + region.getRegionName(), serverName); + String node = getNodeName(zkw, region.getEncodedName()); + ZKUtil.createAndWatch(zkw, node, data.getBytes()); + } + + /** + * Creates an unassigned node in the OFFLINE state for the specified region. + *

    + * Runs asynchronously. Depends on no pre-existing znode. + * + *

    Sets a watcher on the unassigned region node. + * + * @param zkw zk reference + * @param region region to be created as offline + * @param serverName server event originates from + * @param cb + * @param ctx + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NodeExistsException if node already exists + */ + public static void asyncCreateNodeOffline(ZooKeeperWatcher zkw, + HRegionInfo region, ServerName serverName, + final AsyncCallback.StringCallback cb, final Object ctx) + throws KeeperException { + LOG.debug(zkw.prefix("Async create of unassigned node for " + + region.getEncodedName() + " with OFFLINE state")); + RegionTransitionData data = new RegionTransitionData( + EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName); + String node = getNodeName(zkw, region.getEncodedName()); + ZKUtil.asyncCreate(zkw, node, data.getBytes(), cb, ctx); + } + + /** + * Forces an existing unassigned node to the OFFLINE state for the specified + * region. + * + *

    Does not create a new node. If a node does not already exist for this + * region, a {@link NoNodeException} will be thrown. + * + *

    Sets a watcher on the unassigned region node if the method is + * successful. + * + *

    This method should only be used during recovery of regionserver failure. + * + * @param zkw zk reference + * @param region region to be forced as offline + * @param serverName server event originates from + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NoNodeException if node does not exist + */ + public static void forceNodeOffline(ZooKeeperWatcher zkw, HRegionInfo region, + ServerName serverName) + throws KeeperException, KeeperException.NoNodeException { + LOG.debug(zkw.prefix("Forcing existing unassigned node for " + + region.getEncodedName() + " to OFFLINE state")); + RegionTransitionData data = new RegionTransitionData( + EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName); + String node = getNodeName(zkw, region.getEncodedName()); + ZKUtil.setData(zkw, node, data.getBytes()); + } + + /** + * Creates or force updates an unassigned node to the OFFLINE state for the + * specified region. + *

    + * Attempts to create the node but if it exists will force it to transition to + * and OFFLINE state. + * + *

    Sets a watcher on the unassigned region node if the method is + * successful. + * + *

    This method should be used when assigning a region. + * + * @param zkw zk reference + * @param region region to be created as offline + * @param serverName server event originates from + * @return the version of the znode created in OFFLINE state, -1 if + * unsuccessful. + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NodeExistsException if node already exists + */ + public static int createOrForceNodeOffline(ZooKeeperWatcher zkw, + HRegionInfo region, ServerName serverName) throws KeeperException { + return createOrForceNodeOffline(zkw, region, serverName, false, true); + } + + /** + * Creates or force updates an unassigned node to the OFFLINE state for the + * specified region. + *

    + * Attempts to create the node but if it exists will force it to transition to + * and OFFLINE state. + *

    + * Sets a watcher on the unassigned region node if the method is successful. + * + *

    + * This method should be used when assigning a region. + * + * @param zkw + * zk reference + * @param region + * region to be created as offline + * @param serverName + * server event originates from + * @param hijack + * - true if to be hijacked and reassigned, false otherwise + * @param allowCreation + * - true if the node has to be created newly, false otherwise + * @throws KeeperException + * if unexpected zookeeper exception + * @return the version of the znode created in OFFLINE state, -1 if + * unsuccessful. + * @throws KeeperException.NodeExistsException + * if node already exists + */ + public static int createOrForceNodeOffline(ZooKeeperWatcher zkw, + HRegionInfo region, ServerName serverName, + boolean hijack, boolean allowCreation) + throws KeeperException { + LOG.debug(zkw.prefix("Creating (or updating) unassigned node for " + + region.getEncodedName() + " with OFFLINE state")); + RegionTransitionData data = new RegionTransitionData( + EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName); + String node = getNodeName(zkw, region.getEncodedName()); + Stat stat = new Stat(); + zkw.sync(node); + int version = ZKUtil.checkExists(zkw, node); + if (version == -1) { + // While trying to transit a node to OFFLINE that was in previously in + // OPENING state but before it could transit to OFFLINE state if RS had + // opened the region then the Master deletes the assigned region znode. + // In that case the znode will not exist. So we should not + // create the znode again which will lead to double assignment. + if (hijack && !allowCreation) { + return -1; + } + return ZKUtil.createAndWatch(zkw, node, data.getBytes()); + } else { + RegionTransitionData curDataInZNode = ZKAssign.getDataNoWatch(zkw, region + .getEncodedName(), stat); + // Do not move the node to OFFLINE if znode is in any of the following + // state. + // Because these are already executed states. + if (hijack && null != curDataInZNode) { + EventType eventType = curDataInZNode.getEventType(); + if (eventType.equals(EventType.M_ZK_REGION_CLOSING) + || eventType.equals(EventType.RS_ZK_REGION_CLOSED) + || eventType.equals(EventType.RS_ZK_REGION_OPENED)) { + return -1; + } + } + + boolean setData = false; + try { + setData = ZKUtil.setData(zkw, node, data.getBytes(), version); + // Setdata throws KeeperException which aborts the Master. So we are + // catching it here. + // If just before setting the znode to OFFLINE if the RS has made any + // change to the + // znode state then we need to return -1. + } catch (KeeperException kpe) { + LOG.info("Version mismatch while setting the node to OFFLINE state."); + return -1; + } + if (!setData) { + return -1; + } else { + // We successfully forced to OFFLINE, reset watch and handle if + // the state changed in between our set and the watch + RegionTransitionData curData = + ZKAssign.getData(zkw, region.getEncodedName()); + if (curData.getEventType() != data.getEventType()) { + // state changed, need to process + return -1; + } + } + } + return stat.getVersion() + 1; + } + + /** + * Deletes an existing unassigned node that is in the OPENED state for the + * specified region. + * + *

    If a node does not already exist for this region, a + * {@link NoNodeException} will be thrown. + * + *

    No watcher is set whether this succeeds or not. + * + *

    Returns false if the node was not in the proper state but did exist. + * + *

    This method is used during normal region transitions when a region + * finishes successfully opening. This is the Master acknowledging completion + * of the specified regions transition. + * + * @param zkw zk reference + * @param regionName opened region to be deleted from zk + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NoNodeException if node does not exist + */ + public static boolean deleteOpenedNode(ZooKeeperWatcher zkw, + String regionName) + throws KeeperException, KeeperException.NoNodeException { + return deleteNode(zkw, regionName, EventType.RS_ZK_REGION_OPENED); + } + + /** + * Deletes an existing unassigned node that is in the OFFLINE state for the + * specified region. + * + *

    If a node does not already exist for this region, a + * {@link NoNodeException} will be thrown. + * + *

    No watcher is set whether this succeeds or not. + * + *

    Returns false if the node was not in the proper state but did exist. + * + *

    This method is used during master failover when the regions on an RS + * that has died are all set to OFFLINE before being processed. + * + * @param zkw zk reference + * @param regionName closed region to be deleted from zk + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NoNodeException if node does not exist + */ + public static boolean deleteOfflineNode(ZooKeeperWatcher zkw, + String regionName) + throws KeeperException, KeeperException.NoNodeException { + return deleteNode(zkw, regionName, EventType.M_ZK_REGION_OFFLINE); + } + + /** + * Deletes an existing unassigned node that is in the CLOSED state for the + * specified region. + * + *

    If a node does not already exist for this region, a + * {@link NoNodeException} will be thrown. + * + *

    No watcher is set whether this succeeds or not. + * + *

    Returns false if the node was not in the proper state but did exist. + * + *

    This method is used during table disables when a region finishes + * successfully closing. This is the Master acknowledging completion + * of the specified regions transition to being closed. + * + * @param zkw zk reference + * @param regionName closed region to be deleted from zk + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NoNodeException if node does not exist + */ + public static boolean deleteClosedNode(ZooKeeperWatcher zkw, + String regionName) + throws KeeperException, KeeperException.NoNodeException { + return deleteNode(zkw, regionName, EventType.RS_ZK_REGION_CLOSED); + } + + /** + * Deletes an existing unassigned node that is in the CLOSING state for the + * specified region. + * + *

    If a node does not already exist for this region, a + * {@link NoNodeException} will be thrown. + * + *

    No watcher is set whether this succeeds or not. + * + *

    Returns false if the node was not in the proper state but did exist. + * + *

    This method is used during table disables when a region finishes + * successfully closing. This is the Master acknowledging completion + * of the specified regions transition to being closed. + * + * @param zkw zk reference + * @param region closing region to be deleted from zk + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NoNodeException if node does not exist + */ + public static boolean deleteClosingNode(ZooKeeperWatcher zkw, + HRegionInfo region) + throws KeeperException, KeeperException.NoNodeException { + String regionName = region.getEncodedName(); + return deleteNode(zkw, regionName, EventType.M_ZK_REGION_CLOSING); + } + + /** + * Deletes an existing unassigned node that is in the specified state for the + * specified region. + * + *

    If a node does not already exist for this region, a + * {@link NoNodeException} will be thrown. + * + *

    No watcher is set whether this succeeds or not. + * + *

    Returns false if the node was not in the proper state but did exist. + * + *

    This method is used when a region finishes opening/closing. + * The Master acknowledges completion + * of the specified regions transition to being closed/opened. + * + * @param zkw zk reference + * @param regionName region to be deleted from zk + * @param expectedState state region must be in for delete to complete + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NoNodeException if node does not exist + */ + public static boolean deleteNode(ZooKeeperWatcher zkw, String regionName, + EventType expectedState) + throws KeeperException, KeeperException.NoNodeException { + return deleteNode(zkw, regionName, expectedState, -1); + } + + /** + * Deletes an existing unassigned node that is in the specified state for the + * specified region. + * + *

    If a node does not already exist for this region, a + * {@link NoNodeException} will be thrown. + * + *

    No watcher is set whether this succeeds or not. + * + *

    Returns false if the node was not in the proper state but did exist. + * + *

    This method is used when a region finishes opening/closing. + * The Master acknowledges completion + * of the specified regions transition to being closed/opened. + * + * @param zkw zk reference + * @param regionName region to be deleted from zk + * @param expectedState state region must be in for delete to complete + * @param expectedVersion of the znode that is to be deleted. + * If expectedVersion need not be compared while deleting the znode + * pass -1 + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NoNodeException if node does not exist + */ + public static boolean deleteNode(ZooKeeperWatcher zkw, String regionName, + EventType expectedState, int expectedVersion) + throws KeeperException, KeeperException.NoNodeException { + LOG.debug(zkw.prefix("Deleting existing unassigned " + + "node for " + regionName + " that is in expected state " + expectedState)); + String node = getNodeName(zkw, regionName); + zkw.sync(node); + Stat stat = new Stat(); + byte [] bytes = ZKUtil.getDataNoWatch(zkw, node, stat); + if (bytes == null) { + // If it came back null, node does not exist. + throw KeeperException.create(Code.NONODE); + } + RegionTransitionData data = RegionTransitionData.fromBytes(bytes); + if (!data.getEventType().equals(expectedState)) { + LOG.warn(zkw.prefix("Attempting to delete unassigned " + + "node " + regionName + " in " + expectedState + + " state but node is in " + data.getEventType() + " state")); + return false; + } + if (expectedVersion != -1 + && stat.getVersion() != expectedVersion) { + LOG.warn("The node " + regionName + " we are trying to delete is not" + + " the expected one. Got a version mismatch"); + return false; + } + if(!ZKUtil.deleteNode(zkw, node, stat.getVersion())) { + LOG.warn(zkw.prefix("Attempting to delete " + + "unassigned node " + regionName + " in " + expectedState + + " state but after verifying state, we got a version mismatch")); + return false; + } + LOG.debug(zkw.prefix("Successfully deleted unassigned node for region " + + regionName + " in expected state " + expectedState)); + return true; + } + + /** + * Deletes all unassigned nodes regardless of their state. + * + *

    No watchers are set. + * + *

    This method is used by the Master during cluster startup to clear out + * any existing state from other cluster runs. + * + * @param zkw zk reference + * @throws KeeperException if unexpected zookeeper exception + */ + public static void deleteAllNodes(ZooKeeperWatcher zkw) + throws KeeperException { + LOG.debug(zkw.prefix("Deleting any existing unassigned nodes")); + ZKUtil.deleteChildrenRecursively(zkw, zkw.assignmentZNode); + } + + // RegionServer methods + + /** + * Creates a new unassigned node in the CLOSING state for the specified + * region. + * + *

    Does not transition nodes from any states. If a node already exists + * for this region, a {@link NodeExistsException} will be thrown. + * + *

    If creation is successful, returns the version number of the CLOSING + * node created. + * + *

    Does not set any watches. + * + *

    This method should only be used by a RegionServer when initiating a + * close of a region after receiving a CLOSE RPC from the Master. + * + * @param zkw zk reference + * @param region region to be created as closing + * @param serverName server event originates from + * @return version of node after transition, -1 if unsuccessful transition + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NodeExistsException if node already exists + */ + public static int createNodeClosing(ZooKeeperWatcher zkw, HRegionInfo region, + ServerName serverName) + throws KeeperException, KeeperException.NodeExistsException { + LOG.debug(zkw.prefix("Creating unassigned node for " + + region.getEncodedName() + " in a CLOSING state")); + + RegionTransitionData data = new RegionTransitionData( + EventType.M_ZK_REGION_CLOSING, region.getRegionName(), serverName); + + String node = getNodeName(zkw, region.getEncodedName()); + return ZKUtil.createAndWatch(zkw, node, data.getBytes()); + } + + /** + * Transitions an existing unassigned node for the specified region which is + * currently in the CLOSING state to be in the CLOSED state. + * + *

    Does not transition nodes from other states. If for some reason the + * node could not be transitioned, the method returns -1. If the transition + * is successful, the version of the node after transition is returned. + * + *

    This method can fail and return false for three different reasons: + *

    • Unassigned node for this region does not exist
      • + *
    • Unassigned node for this region is not in CLOSING state
      • + *
    • After verifying CLOSING state, update fails because of wrong version + * (someone else already transitioned the node)
      • + *
    + * + *

    Does not set any watches. + * + *

    This method should only be used by a RegionServer when initiating a + * close of a region after receiving a CLOSE RPC from the Master. + * + * @param zkw zk reference + * @param region region to be transitioned to closed + * @param serverName server event originates from + * @return version of node after transition, -1 if unsuccessful transition + * @throws KeeperException if unexpected zookeeper exception + */ + public static int transitionNodeClosed(ZooKeeperWatcher zkw, + HRegionInfo region, ServerName serverName, int expectedVersion) + throws KeeperException { + return transitionNode(zkw, region, serverName, + EventType.M_ZK_REGION_CLOSING, + EventType.RS_ZK_REGION_CLOSED, expectedVersion); + } + + /** + * Transitions an existing unassigned node for the specified region which is + * currently in the OFFLINE state to be in the OPENING state. + * + *

    Does not transition nodes from other states. If for some reason the + * node could not be transitioned, the method returns -1. If the transition + * is successful, the version of the node written as OPENING is returned. + * + *

    This method can fail and return -1 for three different reasons: + *

    • Unassigned node for this region does not exist
      • + *
    • Unassigned node for this region is not in OFFLINE state
      • + *
    • After verifying OFFLINE state, update fails because of wrong version + * (someone else already transitioned the node)
      • + *
    + * + *

    Does not set any watches. + * + *

    This method should only be used by a RegionServer when initiating an + * open of a region after receiving an OPEN RPC from the Master. + * + * @param zkw zk reference + * @param region region to be transitioned to opening + * @param serverName server event originates from + * @return version of node after transition, -1 if unsuccessful transition + * @throws KeeperException if unexpected zookeeper exception + */ + public static int transitionNodeOpening(ZooKeeperWatcher zkw, + HRegionInfo region, ServerName serverName) + throws KeeperException { + return transitionNodeOpening(zkw, region, serverName, + EventType.M_ZK_REGION_OFFLINE); + } + + public static int transitionNodeOpening(ZooKeeperWatcher zkw, + HRegionInfo region, ServerName serverName, final EventType beginState) + throws KeeperException { + return transitionNode(zkw, region, serverName, beginState, + EventType.RS_ZK_REGION_OPENING, -1); + } + + /** + * Retransitions an existing unassigned node for the specified region which is + * currently in the OPENING state to be in the OPENING state. + * + *

    Does not transition nodes from other states. If for some reason the + * node could not be transitioned, the method returns -1. If the transition + * is successful, the version of the node rewritten as OPENING is returned. + * + *

    This method can fail and return -1 for three different reasons: + *

    • Unassigned node for this region does not exist
      • + *
    • Unassigned node for this region is not in OPENING state
      • + *
    • After verifying OPENING state, update fails because of wrong version + * (someone else already transitioned the node)
      • + *
    + * + *

    Does not set any watches. + * + *

    This method should only be used by a RegionServer when initiating an + * open of a region after receiving an OPEN RPC from the Master. + * + * @param zkw zk reference + * @param region region to be transitioned to opening + * @param serverName server event originates from + * @return version of node after transition, -1 if unsuccessful transition + * @throws KeeperException if unexpected zookeeper exception + */ + public static int retransitionNodeOpening(ZooKeeperWatcher zkw, + HRegionInfo region, ServerName serverName, int expectedVersion) + throws KeeperException { + return transitionNode(zkw, region, serverName, + EventType.RS_ZK_REGION_OPENING, + EventType.RS_ZK_REGION_OPENING, expectedVersion); + } + + /** + * Transitions an existing unassigned node for the specified region which is + * currently in the OPENING state to be in the OPENED state. + * + *

    Does not transition nodes from other states. If for some reason the + * node could not be transitioned, the method returns -1. If the transition + * is successful, the version of the node after transition is returned. + * + *

    This method can fail and return false for three different reasons: + *

    • Unassigned node for this region does not exist
      • + *
    • Unassigned node for this region is not in OPENING state
      • + *
    • After verifying OPENING state, update fails because of wrong version + * (this should never actually happen since an RS only does this transition + * following a transition to OPENING. if two RS are conflicting, one would + * fail the original transition to OPENING and not this transition)
      • + *
    + * + *

    Does not set any watches. + * + *

    This method should only be used by a RegionServer when completing the + * open of a region. + * + * @param zkw zk reference + * @param region region to be transitioned to opened + * @param serverName server event originates from + * @return version of node after transition, -1 if unsuccessful transition + * @throws KeeperException if unexpected zookeeper exception + */ + public static int transitionNodeOpened(ZooKeeperWatcher zkw, + HRegionInfo region, ServerName serverName, int expectedVersion) + throws KeeperException { + return transitionNode(zkw, region, serverName, + EventType.RS_ZK_REGION_OPENING, + EventType.RS_ZK_REGION_OPENED, expectedVersion); + } + + /** + * Method that actually performs unassigned node transitions. + * + *

    Attempts to transition the unassigned node for the specified region + * from the expected state to the state in the specified transition data. + * + *

    Method first reads existing data and verifies it is in the expected + * state. If the node does not exist or the node is not in the expected + * state, the method returns -1. If the transition is successful, the + * version number of the node following the transition is returned. + * + *

    If the read state is what is expected, it attempts to write the new + * state and data into the node. When doing this, it includes the expected + * version (determined when the existing state was verified) to ensure that + * only one transition is successful. If there is a version mismatch, the + * method returns -1. + * + *

    If the write is successful, no watch is set and the method returns true. + * + * @param zkw zk reference + * @param region region to be transitioned to opened + * @param serverName server event originates from + * @param endState state to transition node to if all checks pass + * @param beginState state the node must currently be in to do transition + * @param expectedVersion expected version of data before modification, or -1 + * @return version of node after transition, -1 if unsuccessful transition + * @throws KeeperException if unexpected zookeeper exception + */ + public static int transitionNode(ZooKeeperWatcher zkw, HRegionInfo region, + ServerName serverName, EventType beginState, EventType endState, + int expectedVersion) + throws KeeperException { + return transitionNode(zkw, region, serverName, beginState, endState, + expectedVersion, null); + } + + public static int transitionNode(ZooKeeperWatcher zkw, HRegionInfo region, + ServerName serverName, EventType beginState, EventType endState, + int expectedVersion, final byte [] payload) + throws KeeperException { + String encoded = region.getEncodedName(); + if(LOG.isDebugEnabled()) { + LOG.debug(zkw.prefix("Attempting to transition node " + + HRegionInfo.prettyPrint(encoded) + + " from " + beginState.toString() + " to " + endState.toString())); + } + + String node = getNodeName(zkw, encoded); + zkw.sync(node); + + // Read existing data of the node + Stat stat = new Stat(); + byte [] existingBytes = ZKUtil.getDataNoWatch(zkw, node, stat); + if (existingBytes == null) { + // Node no longer exists. Return -1. It means unsuccessful transition. + return -1; + } + RegionTransitionData existingData = + RegionTransitionData.fromBytes(existingBytes); + + // Verify it is the expected version + if(expectedVersion != -1 && stat.getVersion() != expectedVersion) { + LOG.warn(zkw.prefix("Attempt to transition the " + + "unassigned node for " + encoded + + " from " + beginState + " to " + endState + " failed, " + + "the node existed but was version " + stat.getVersion() + + " not the expected version " + expectedVersion)); + return -1; + } else if (beginState.equals(EventType.M_ZK_REGION_OFFLINE) + && endState.equals(EventType.RS_ZK_REGION_OPENING) + && expectedVersion == -1 && stat.getVersion() != 0) { + // the below check ensures that double assignment doesnot happen. + // When the node is created for the first time then the expected version + // that is passed will be -1 and the version in znode will be 0. + // In all other cases the version in znode will be > 0. + LOG.warn(zkw.prefix("Attempt to transition the " + "unassigned node for " + + encoded + " from " + beginState + " to " + endState + " failed, " + + "the node existed but was version " + stat.getVersion() + + " not the expected version " + expectedVersion)); + return -1; + } + + // Verify it is in expected state + if(!existingData.getEventType().equals(beginState)) { + LOG.warn(zkw.prefix("Attempt to transition the " + + "unassigned node for " + encoded + + " from " + beginState + " to " + endState + " failed, " + + "the node existed but was in the state " + existingData.getEventType() + + " set by the server " + serverName)); + return -1; + } + + // Write new data, ensuring data has not changed since we last read it + try { + RegionTransitionData data = new RegionTransitionData(endState, + region.getRegionName(), serverName, payload); + if(!ZKUtil.setData(zkw, node, data.getBytes(), stat.getVersion())) { + LOG.warn(zkw.prefix("Attempt to transition the " + + "unassigned node for " + encoded + + " from " + beginState + " to " + endState + " failed, " + + "the node existed and was in the expected state but then when " + + "setting data we got a version mismatch")); + return -1; + } + if(LOG.isDebugEnabled()) { + LOG.debug(zkw.prefix("Successfully transitioned node " + encoded + + " from " + beginState + " to " + endState)); + } + return stat.getVersion() + 1; + } catch (KeeperException.NoNodeException nne) { + LOG.warn(zkw.prefix("Attempt to transition the " + + "unassigned node for " + encoded + + " from " + beginState + " to " + endState + " failed, " + + "the node existed and was in the expected state but then when " + + "setting data it no longer existed")); + return -1; + } + } + + /** + * Gets the current data in the unassigned node for the specified region name + * or fully-qualified path. + * + *

    Returns null if the region does not currently have a node. + * + *

    Sets a watch on the node if the node exists. + * + * @param zkw zk reference + * @param pathOrRegionName fully-specified path or region name + * @return data for the unassigned node + * @throws KeeperException if unexpected zookeeper exception + */ + public static RegionTransitionData getData(ZooKeeperWatcher zkw, + String pathOrRegionName) + throws KeeperException { + String node = pathOrRegionName.startsWith("/") ? + pathOrRegionName : getNodeName(zkw, pathOrRegionName); + byte [] data = ZKUtil.getDataAndWatch(zkw, node); + if(data == null) { + return null; + } + return RegionTransitionData.fromBytes(data); + } + + /** + * Gets the current data in the unassigned node for the specified region name + * or fully-qualified path. + * + *

    Returns null if the region does not currently have a node. + * + *

    Sets a watch on the node if the node exists. + * + * @param zkw zk reference + * @param pathOrRegionName fully-specified path or region name + * @param stat object to populate the version. + * @return data for the unassigned node + * @throws KeeperException if unexpected zookeeper exception + */ + public static RegionTransitionData getDataAndWatch(ZooKeeperWatcher zkw, + String pathOrRegionName, Stat stat) + throws KeeperException { + String node = pathOrRegionName.startsWith("/") ? + pathOrRegionName : getNodeName(zkw, pathOrRegionName); + byte [] data = ZKUtil.getDataAndWatch(zkw, node, stat); + if(data == null) { + return null; + } + return RegionTransitionData.fromBytes(data); + } + + /** + * Gets the current data in the unassigned node for the specified region name + * or fully-qualified path. + * + *

    Returns null if the region does not currently have a node. + * + *

    Does not set a watch. + * + * @param zkw zk reference + * @param pathOrRegionName fully-specified path or region name + * @param stat object to store node info into on getData call + * @return data for the unassigned node or null if node does not exist + * @throws KeeperException if unexpected zookeeper exception + */ + public static RegionTransitionData getDataNoWatch(ZooKeeperWatcher zkw, + String pathOrRegionName, Stat stat) + throws KeeperException { + String node = pathOrRegionName.startsWith("/") ? + pathOrRegionName : getNodeName(zkw, pathOrRegionName); + byte [] data = ZKUtil.getDataNoWatch(zkw, node, stat); + if (data == null) { + return null; + } + return RegionTransitionData.fromBytes(data); + } + + /** + * Get the version of the specified znode + * @param zkw zk reference + * @param region region's info + * @return the version of the znode, -1 if it doesn't exist + * @throws KeeperException + */ + public static int getVersion(ZooKeeperWatcher zkw, HRegionInfo region) + throws KeeperException { + String znode = getNodeName(zkw, region.getEncodedName()); + return ZKUtil.checkExists(zkw, znode); + } + + /** + * Delete the assignment node regardless of its current state. + *

    + * Fail silent even if the node does not exist at all. + * @param watcher + * @param regionInfo + * @throws KeeperException + */ + public static void deleteNodeFailSilent(ZooKeeperWatcher watcher, + HRegionInfo regionInfo) + throws KeeperException { + String node = getNodeName(watcher, regionInfo.getEncodedName()); + ZKUtil.deleteNodeFailSilent(watcher, node); + } + + /** + * Blocks until there are no node in regions in transition. + *

    + * Used in testing only. + * @param zkw zk reference + * @throws KeeperException + * @throws InterruptedException + */ + public static void blockUntilNoRIT(ZooKeeperWatcher zkw) + throws KeeperException, InterruptedException { + while (ZKUtil.nodeHasChildren(zkw, zkw.assignmentZNode)) { + List znodes = + ZKUtil.listChildrenAndWatchForNewChildren(zkw, zkw.assignmentZNode); + if (znodes != null && !znodes.isEmpty()) { + for (String znode : znodes) { + LOG.debug("ZK RIT -> " + znode); + } + } + Thread.sleep(100); + } + } + + /** + * Blocks until there is at least one node in regions in transition. + *

    + * Used in testing only. + * @param zkw zk reference + * @throws KeeperException + * @throws InterruptedException + */ + public static void blockUntilRIT(ZooKeeperWatcher zkw) + throws KeeperException, InterruptedException { + while (!ZKUtil.nodeHasChildren(zkw, zkw.assignmentZNode)) { + List znodes = + ZKUtil.listChildrenAndWatchForNewChildren(zkw, zkw.assignmentZNode); + if (znodes == null || znodes.isEmpty()) { + LOG.debug("No RIT in ZK"); + } + Thread.sleep(100); + } + } + + /** + * Verifies that the specified region is in the specified state in ZooKeeper. + *

    + * Returns true if region is in transition and in the specified state in + * ZooKeeper. Returns false if the region does not exist in ZK or is in + * a different state. + *

    + * Method synchronizes() with ZK so will yield an up-to-date result but is + * a slow read. + * @param zkw + * @param region + * @param expectedState + * @return true if region exists and is in expected state + */ + public static boolean verifyRegionState(ZooKeeperWatcher zkw, + HRegionInfo region, EventType expectedState) + throws KeeperException { + String encoded = region.getEncodedName(); + + String node = getNodeName(zkw, encoded); + zkw.sync(node); + + // Read existing data of the node + byte [] existingBytes = null; + try { + existingBytes = ZKUtil.getDataAndWatch(zkw, node); + } catch (KeeperException.NoNodeException nne) { + return false; + } catch (KeeperException e) { + throw e; + } + if (existingBytes == null) return false; + RegionTransitionData existingData = + RegionTransitionData.fromBytes(existingBytes); + if (existingData.getEventType() == expectedState){ + return true; + } + return false; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java new file mode 100644 index 0000000..4cc9c01 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java @@ -0,0 +1,247 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.util.StringUtils; + +/** + * Utility methods for reading, parsing, and building zookeeper configuration. + */ +public class ZKConfig { + private static final Log LOG = LogFactory.getLog(ZKConfig.class); + + private static final String VARIABLE_START = "${"; + private static final int VARIABLE_START_LENGTH = VARIABLE_START.length(); + private static final String VARIABLE_END = "}"; + private static final int VARIABLE_END_LENGTH = VARIABLE_END.length(); + + /** + * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg. + * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse + * the corresponding config options from the HBase XML configs and generate + * the appropriate ZooKeeper properties. + * @param conf Configuration to read from. + * @return Properties holding mappings representing ZooKeeper zoo.cfg file. + */ + public static Properties makeZKProps(Configuration conf) { + // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read + // it and grab its configuration properties. + ClassLoader cl = HQuorumPeer.class.getClassLoader(); + final InputStream inputStream = + cl.getResourceAsStream(HConstants.ZOOKEEPER_CONFIG_NAME); + if (inputStream != null) { + try { + return parseZooCfg(conf, inputStream); + } catch (IOException e) { + LOG.warn("Cannot read " + HConstants.ZOOKEEPER_CONFIG_NAME + + ", loading from XML files", e); + } + } + + // Otherwise, use the configuration options from HBase's XML files. + Properties zkProperties = new Properties(); + + // Directly map all of the hbase.zookeeper.property.KEY properties. + for (Entry entry : conf) { + String key = entry.getKey(); + if (key.startsWith(HConstants.ZK_CFG_PROPERTY_PREFIX)) { + String zkKey = key.substring(HConstants.ZK_CFG_PROPERTY_PREFIX_LEN); + String value = entry.getValue(); + // If the value has variables substitutions, need to do a get. + if (value.contains(VARIABLE_START)) { + value = conf.get(key); + } + zkProperties.put(zkKey, value); + } + } + + // If clientPort is not set, assign the default. + if (zkProperties.getProperty(HConstants.CLIENT_PORT_STR) == null) { + zkProperties.put(HConstants.CLIENT_PORT_STR, + HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT); + } + + // Create the server.X properties. + int peerPort = conf.getInt("hbase.zookeeper.peerport", 2888); + int leaderPort = conf.getInt("hbase.zookeeper.leaderport", 3888); + + final String[] serverHosts = conf.getStrings(HConstants.ZOOKEEPER_QUORUM, + HConstants.LOCALHOST); + for (int i = 0; i < serverHosts.length; ++i) { + String serverHost = serverHosts[i]; + String address = serverHost + ":" + peerPort + ":" + leaderPort; + String key = "server." + i; + zkProperties.put(key, address); + } + + return zkProperties; + } + + /** + * Parse ZooKeeper's zoo.cfg, injecting HBase Configuration variables in. + * This method is used for testing so we can pass our own InputStream. + * @param conf HBaseConfiguration to use for injecting variables. + * @param inputStream InputStream to read from. + * @return Properties parsed from config stream with variables substituted. + * @throws IOException if anything goes wrong parsing config + */ + public static Properties parseZooCfg(Configuration conf, + InputStream inputStream) throws IOException { + Properties properties = new Properties(); + try { + properties.load(inputStream); + } catch (IOException e) { + final String msg = "fail to read properties from " + + HConstants.ZOOKEEPER_CONFIG_NAME; + LOG.fatal(msg); + throw new IOException(msg, e); + } + for (Entry entry : properties.entrySet()) { + String value = entry.getValue().toString().trim(); + String key = entry.getKey().toString().trim(); + StringBuilder newValue = new StringBuilder(); + int varStart = value.indexOf(VARIABLE_START); + int varEnd = 0; + while (varStart != -1) { + varEnd = value.indexOf(VARIABLE_END, varStart); + if (varEnd == -1) { + String msg = "variable at " + varStart + " has no end marker"; + LOG.fatal(msg); + throw new IOException(msg); + } + String variable = value.substring(varStart + VARIABLE_START_LENGTH, varEnd); + + String substituteValue = System.getProperty(variable); + if (substituteValue == null) { + substituteValue = conf.get(variable); + } + if (substituteValue == null) { + String msg = "variable " + variable + " not set in system property " + + "or hbase configs"; + LOG.fatal(msg); + throw new IOException(msg); + } + + newValue.append(substituteValue); + + varEnd += VARIABLE_END_LENGTH; + varStart = value.indexOf(VARIABLE_START, varEnd); + } + // Special case for 'hbase.cluster.distributed' property being 'true' + if (key.startsWith("server.")) { + boolean mode = conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, HConstants.DEFAULT_CLUSTER_DISTRIBUTED); + if (mode == HConstants.CLUSTER_IS_DISTRIBUTED && value.startsWith(HConstants.LOCALHOST)) { + String msg = "The server in zoo.cfg cannot be set to localhost " + + "in a fully-distributed setup because it won't be reachable. " + + "See \"Getting Started\" for more information."; + LOG.fatal(msg); + throw new IOException(msg); + } + } + newValue.append(value.substring(varEnd)); + properties.setProperty(key, newValue.toString()); + } + return properties; + } + + /** + * Return the ZK Quorum servers string given zk properties returned by + * makeZKProps + * @param properties + * @return Quorum servers String + */ + public static String getZKQuorumServersString(Properties properties) { + String clientPort = null; + List servers = new ArrayList(); + + // The clientPort option may come after the server.X hosts, so we need to + // grab everything and then create the final host:port comma separated list. + boolean anyValid = false; + for (Entry property : properties.entrySet()) { + String key = property.getKey().toString().trim(); + String value = property.getValue().toString().trim(); + if (key.equals("clientPort")) { + clientPort = value; + } + else if (key.startsWith("server.")) { + String host = value.substring(0, value.indexOf(':')); + servers.add(host); + try { + //noinspection ResultOfMethodCallIgnored + InetAddress.getByName(host); + anyValid = true; + } catch (UnknownHostException e) { + LOG.warn(StringUtils.stringifyException(e)); + } + } + } + + if (!anyValid) { + LOG.error("no valid quorum servers found in " + HConstants.ZOOKEEPER_CONFIG_NAME); + return null; + } + + if (clientPort == null) { + LOG.error("no clientPort found in " + HConstants.ZOOKEEPER_CONFIG_NAME); + return null; + } + + if (servers.isEmpty()) { + LOG.fatal("No server.X lines found in conf/zoo.cfg. HBase must have a " + + "ZooKeeper cluster configured for its operation."); + return null; + } + + StringBuilder hostPortBuilder = new StringBuilder(); + for (int i = 0; i < servers.size(); ++i) { + String host = servers.get(i); + if (i > 0) { + hostPortBuilder.append(','); + } + hostPortBuilder.append(host); + hostPortBuilder.append(':'); + hostPortBuilder.append(clientPort); + } + + return hostPortBuilder.toString(); + } + + /** + * Return the ZK Quorum servers string given the specified configuration. + * @param conf + * @return Quorum servers + */ + public static String getZKQuorumServersString(Configuration conf) { + return getZKQuorumServersString(makeZKProps(conf)); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java new file mode 100644 index 0000000..f201799 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.zookeeper; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.zookeeper.KeeperException; + +/** + * Handles coordination of a single "leader" instance among many possible + * candidates. The first {@link ZKLeaderManager} to successfully create + * the given znode becomes the leader, allowing the instance to continue + * with whatever processing must be protected. Other {@link ZKLeaderManager} + * instances will wait to be notified of changes to the leader znode. + * If the current master instance fails, the ephemeral leader znode will + * be removed, and all waiting instances will be notified, with the race + * to claim the leader znode beginning all over again. + */ +public class ZKLeaderManager extends ZooKeeperListener { + private static Log LOG = LogFactory.getLog(ZKLeaderManager.class); + + private final AtomicBoolean leaderExists = new AtomicBoolean(); + private String leaderZNode; + private byte[] nodeId; + private Stoppable candidate; + + public ZKLeaderManager(ZooKeeperWatcher watcher, String leaderZNode, + byte[] identifier, Stoppable candidate) { + super(watcher); + this.leaderZNode = leaderZNode; + this.nodeId = identifier; + this.candidate = candidate; + } + + public void start() { + try { + watcher.registerListener(this); + String parent = ZKUtil.getParent(leaderZNode); + if (ZKUtil.checkExists(watcher, parent) < 0) { + ZKUtil.createWithParents(watcher, parent); + } + } catch (KeeperException ke) { + watcher.abort("Unhandled zk exception when starting", ke); + candidate.stop("Unhandled zk exception starting up: "+ke.getMessage()); + } + } + + @Override + public void nodeCreated(String path) { + if (leaderZNode.equals(path) && !candidate.isStopped()) { + handleLeaderChange(); + } + } + + @Override + public void nodeDeleted(String path) { + if (leaderZNode.equals(path) && !candidate.isStopped()) { + handleLeaderChange(); + } + } + + private void handleLeaderChange() { + try { + synchronized(leaderExists) { + if (ZKUtil.watchAndCheckExists(watcher, leaderZNode)) { + LOG.info("Found new leader for znode: "+leaderZNode); + leaderExists.set(true); + } else { + LOG.info("Leader change, but no new leader found"); + leaderExists.set(false); + leaderExists.notifyAll(); + } + } + } catch (KeeperException ke) { + watcher.abort("ZooKeeper error checking for leader znode", ke); + candidate.stop("ZooKeeper error checking for leader: "+ke.getMessage()); + } + } + + /** + * Blocks until this instance has claimed the leader ZNode in ZooKeeper + */ + public void waitToBecomeLeader() { + while (!candidate.isStopped()) { + try { + if (ZKUtil.createEphemeralNodeAndWatch(watcher, leaderZNode, nodeId)) { + // claimed the leader znode + leaderExists.set(true); + if (LOG.isDebugEnabled()) { + LOG.debug("Claimed the leader znode as '"+ + Bytes.toStringBinary(nodeId)+"'"); + } + return; + } + + // if claiming the node failed, there should be another existing node + byte[] currentId = ZKUtil.getDataAndWatch(watcher, leaderZNode); + if (currentId != null && Bytes.equals(currentId, nodeId)) { + // claimed with our ID, but we didn't grab it, possibly restarted? + LOG.info("Found existing leader with our ID ("+ + Bytes.toStringBinary(nodeId)+"), removing"); + ZKUtil.deleteNode(watcher, leaderZNode); + leaderExists.set(false); + } else { + LOG.info("Found existing leader with ID: "+Bytes.toStringBinary(nodeId)); + leaderExists.set(true); + } + } catch (KeeperException ke) { + watcher.abort("Unexpected error from ZK, stopping candidate", ke); + candidate.stop("Unexpected error from ZK: "+ke.getMessage()); + return; + } + + // wait for next chance + synchronized(leaderExists) { + while (leaderExists.get() && !candidate.isStopped()) { + try { + leaderExists.wait(); + } catch (InterruptedException ie) { + LOG.debug("Interrupted waiting on leader", ie); + } + } + } + } + } + + /** + * Removes the leader znode, if it is currently claimed by this instance. + */ + public void stepDownAsLeader() { + try { + synchronized(leaderExists) { + if (!leaderExists.get()) { + return; + } + byte[] leaderId = ZKUtil.getData(watcher, leaderZNode); + if (leaderId != null && Bytes.equals(nodeId, leaderId)) { + LOG.info("Stepping down as leader"); + ZKUtil.deleteNodeFailSilent(watcher, leaderZNode); + leaderExists.set(false); + } else { + LOG.info("Not current leader, no need to step down"); + } + } + } catch (KeeperException ke) { + watcher.abort("Unhandled zookeeper exception removing leader node", ke); + candidate.stop("Unhandled zookeeper exception removing leader node: " + + ke.getMessage()); + } + } + + public boolean hasLeader() { + return leaderExists.get(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java new file mode 100644 index 0000000..500bd3c --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java @@ -0,0 +1,54 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.zookeeper; + +import java.util.Properties; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; + +/** + * Tool for reading ZooKeeper servers from HBase XML configuration and producing + * a line-by-line list for use by bash scripts. + */ +public class ZKServerTool { + /** + * Run the tool. + * @param args Command line arguments. + */ + public static void main(String args[]) { + Configuration conf = HBaseConfiguration.create(); + // Note that we do not simply grab the property + // HConstants.ZOOKEEPER_QUORUM from the HBaseConfiguration because the + // user may be using a zoo.cfg file. + Properties zkProps = ZKConfig.makeZKProps(conf); + for (Entry entry : zkProps.entrySet()) { + String key = entry.getKey().toString().trim(); + String value = entry.getValue().toString().trim(); + if (key.startsWith("server.")) { + String[] parts = value.split(":"); + String host = parts[0]; + System.out.println("ZK host:" + host); + } + } + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java new file mode 100644 index 0000000..8e9aa49 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java @@ -0,0 +1,259 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.lang.reflect.Field; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseFileSystem; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.master.SplitLogManager; +import org.apache.hadoop.hbase.regionserver.SplitLogWorker; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Common methods and attributes used by {@link SplitLogManager} and + * {@link SplitLogWorker} + */ +public class ZKSplitLog { + private static final Log LOG = LogFactory.getLog(ZKSplitLog.class); + + public static final int DEFAULT_TIMEOUT = 300000; // 5 mins + public static final int DEFAULT_ZK_RETRIES = 3; + public static final int DEFAULT_MAX_RESUBMIT = 3; + public static final int DEFAULT_UNASSIGNED_TIMEOUT = (3 * 60 * 1000); //3 min + + /** + * Gets the full path node name for the log file being split. + * This method will url encode the filename. + * @param zkw zk reference + * @param filename log file name (only the basename) + */ + public static String getEncodedNodeName(ZooKeeperWatcher zkw, + String filename) { + return ZKUtil.joinZNode(zkw.splitLogZNode, encode(filename)); + } + + public static String getFileName(String node) { + String basename = node.substring(node.lastIndexOf('/') + 1); + return decode(basename); + } + + + public static String encode(String s) { + try { + return URLEncoder.encode(s, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("URLENCODER doesn't support UTF-8"); + } + } + + public static String decode(String s) { + try { + return URLDecoder.decode(s, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("URLDecoder doesn't support UTF-8"); + } + } + + public static String getRescanNode(ZooKeeperWatcher zkw) { + return ZKUtil.joinZNode(zkw.splitLogZNode, "RESCAN"); + } + + public static boolean isRescanNode(ZooKeeperWatcher zkw, String path) { + String prefix = getRescanNode(zkw); + if (path.length() <= prefix.length()) { + return false; + } + for (int i = 0; i < prefix.length(); i++) { + if (prefix.charAt(i) != path.charAt(i)) { + return false; + } + } + return true; + } + + public static boolean isTaskPath(ZooKeeperWatcher zkw, String path) { + String dirname = path.substring(0, path.lastIndexOf('/')); + return dirname.equals(zkw.splitLogZNode); + } + + public static enum TaskState { + TASK_UNASSIGNED("unassigned"), + TASK_OWNED("owned"), + TASK_RESIGNED("resigned"), + TASK_DONE("done"), + TASK_ERR("err"); + + private final byte[] state; + private TaskState(String s) { + state = s.getBytes(); + } + + public byte[] get(String serverName) { + return (Bytes.add(state, " ".getBytes(), serverName.getBytes())); + } + + public String getWriterName(byte[] data) { + String str = Bytes.toString(data); + return str.substring(str.indexOf(' ') + 1); + } + + + /** + * @param s + * @return True if {@link #state} is a prefix of s. False otherwise. + */ + public boolean equals(byte[] s) { + if (s.length < state.length) { + return (false); + } + for (int i = 0; i < state.length; i++) { + if (state[i] != s[i]) { + return (false); + } + } + return (true); + } + + public boolean equals(byte[] s, String serverName) { + return (Arrays.equals(s, get(serverName))); + } + @Override + public String toString() { + return new String(state); + } + } + + public static Path getSplitLogDir(Path rootdir, String tmpname) { + return new Path(new Path(rootdir, HConstants.SPLIT_LOGDIR_NAME), tmpname); + } + + public static String getSplitLogDirTmpComponent(String worker, String file) { + return (worker + "_" + ZKSplitLog.encode(file)); + } + + public static void markCorrupted(Path rootdir, String logFileName, + FileSystem fs) { + Path file = new Path(getSplitLogDir(rootdir, logFileName), "corrupt"); + try { + HBaseFileSystem.createNewFileOnFileSystem(fs, file); + } catch (IOException e) { + LOG.warn("Could not flag a log file as corrupted. Failed to create " + + file, e); + } + } + + public static boolean isCorrupted(Path rootdir, String logFileName, + FileSystem fs) throws IOException { + Path file = new Path(getSplitLogDir(rootdir, logFileName), "corrupt"); + boolean isCorrupt; + isCorrupt = fs.exists(file); + return isCorrupt; + } + + + public static class Counters { + //SplitLogManager counters + public static AtomicLong tot_mgr_log_split_batch_start = new AtomicLong(0); + public static AtomicLong tot_mgr_log_split_batch_success = + new AtomicLong(0); + public static AtomicLong tot_mgr_log_split_batch_err = new AtomicLong(0); + public static AtomicLong tot_mgr_new_unexpected_hlogs = new AtomicLong(0); + public static AtomicLong tot_mgr_log_split_start = new AtomicLong(0); + public static AtomicLong tot_mgr_log_split_success = new AtomicLong(0); + public static AtomicLong tot_mgr_log_split_err = new AtomicLong(0); + public static AtomicLong tot_mgr_node_create_queued = new AtomicLong(0); + public static AtomicLong tot_mgr_node_create_result = new AtomicLong(0); + public static AtomicLong tot_mgr_node_already_exists = new AtomicLong(0); + public static AtomicLong tot_mgr_node_create_err = new AtomicLong(0); + public static AtomicLong tot_mgr_node_create_retry = new AtomicLong(0); + public static AtomicLong tot_mgr_get_data_queued = new AtomicLong(0); + public static AtomicLong tot_mgr_get_data_result = new AtomicLong(0); + public static AtomicLong tot_mgr_get_data_nonode = new AtomicLong(0); + public static AtomicLong tot_mgr_get_data_err = new AtomicLong(0); + public static AtomicLong tot_mgr_get_data_retry = new AtomicLong(0); + public static AtomicLong tot_mgr_node_delete_queued = new AtomicLong(0); + public static AtomicLong tot_mgr_node_delete_result = new AtomicLong(0); + public static AtomicLong tot_mgr_node_delete_err = new AtomicLong(0); + public static AtomicLong tot_mgr_resubmit = new AtomicLong(0); + public static AtomicLong tot_mgr_resubmit_failed = new AtomicLong(0); + public static AtomicLong tot_mgr_null_data = new AtomicLong(0); + public static AtomicLong tot_mgr_orphan_task_acquired = new AtomicLong(0); + public static AtomicLong tot_mgr_wait_for_zk_delete = new AtomicLong(0); + public static AtomicLong tot_mgr_unacquired_orphan_done = new AtomicLong(0); + public static AtomicLong tot_mgr_resubmit_threshold_reached = + new AtomicLong(0); + public static AtomicLong tot_mgr_missing_state_in_delete = + new AtomicLong(0); + public static AtomicLong tot_mgr_heartbeat = new AtomicLong(0); + public static AtomicLong tot_mgr_rescan = new AtomicLong(0); + public static AtomicLong tot_mgr_rescan_deleted = new AtomicLong(0); + public static AtomicLong tot_mgr_task_deleted = new AtomicLong(0); + public static AtomicLong tot_mgr_resubmit_unassigned = new AtomicLong(0); + public static AtomicLong tot_mgr_relist_logdir = new AtomicLong(0); + public static AtomicLong tot_mgr_resubmit_dead_server_task = + new AtomicLong(0); + + + + // SplitLogWorker counters + public static AtomicLong tot_wkr_failed_to_grab_task_no_data = + new AtomicLong(0); + public static AtomicLong tot_wkr_failed_to_grab_task_exception = + new AtomicLong(0); + public static AtomicLong tot_wkr_failed_to_grab_task_owned = + new AtomicLong(0); + public static AtomicLong tot_wkr_failed_to_grab_task_lost_race = + new AtomicLong(0); + public static AtomicLong tot_wkr_task_acquired = new AtomicLong(0); + public static AtomicLong tot_wkr_task_resigned = new AtomicLong(0); + public static AtomicLong tot_wkr_task_done = new AtomicLong(0); + public static AtomicLong tot_wkr_task_err = new AtomicLong(0); + public static AtomicLong tot_wkr_task_heartbeat = new AtomicLong(0); + public static AtomicLong tot_wkr_task_acquired_rescan = new AtomicLong(0); + public static AtomicLong tot_wkr_get_data_queued = new AtomicLong(0); + public static AtomicLong tot_wkr_get_data_result = new AtomicLong(0); + public static AtomicLong tot_wkr_get_data_retry = new AtomicLong(0); + public static AtomicLong tot_wkr_preempt_task = new AtomicLong(0); + public static AtomicLong tot_wkr_task_heartbeat_failed = new AtomicLong(0); + public static AtomicLong tot_wkr_final_transistion_failed = + new AtomicLong(0); + + public static void resetCounters() throws Exception { + Class cl = (new Counters()).getClass(); + Field[] flds = cl.getDeclaredFields(); + for (Field fld : flds) { + ((AtomicLong)fld.get(null)).set(0); + } + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java new file mode 100644 index 0000000..6aa5027 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java @@ -0,0 +1,378 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp; +import org.apache.zookeeper.KeeperException; + +/** + * Helper class for table state tracking for use by {@link AssignmentManager}. + * Reads, caches and sets state up in zookeeper. If multiple read/write + * clients, will make for confusion. Read-only clients other than + * AssignmentManager interested in learning table state can use the + * read-only utility methods in {@link ZKTableReadOnly}. + * + *

    To save on trips to the zookeeper ensemble, internally we cache table + * state. + */ +public class ZKTable { + // A znode will exist under the table directory if it is in any of the + // following states: {@link TableState#ENABLING} , {@link TableState#DISABLING}, + // or {@link TableState#DISABLED}. If {@link TableState#ENABLED}, there will + // be no entry for a table in zk. Thats how it currently works. + + private static final Log LOG = LogFactory.getLog(ZKTable.class); + private final ZooKeeperWatcher watcher; + + /** + * Cache of what we found in zookeeper so we don't have to go to zk ensemble + * for every query. Synchronize access rather than use concurrent Map because + * synchronization needs to span query of zk. + */ + private final Map cache = + new HashMap(); + + // TODO: Make it so always a table znode. Put table schema here as well as table state. + // Have watcher on table znode so all are notified of state or schema change. + /** + * States a Table can be in. + * Compatibility note: ENABLED does not exist in 0.92 releases. In 0.92, the absence of + * the znode indicates the table is enabled. + */ + public static enum TableState { + ENABLED, + DISABLED, + DISABLING, + ENABLING + }; + + public ZKTable(final ZooKeeperWatcher zkw) throws KeeperException { + super(); + this.watcher = zkw; + populateTableStates(); + } + + /** + * Gets a list of all the tables set as disabled in zookeeper. + * @throws KeeperException + */ + private void populateTableStates() + throws KeeperException { + synchronized (this.cache) { + List children = + ZKUtil.listChildrenNoWatch(this.watcher, this.watcher.masterTableZNode); + if (children == null) return; + for (String child: children) { + TableState state = getTableState(this.watcher, child); + if (state != null) this.cache.put(child, state); + } + } + } + + /** + * @param zkw + * @param child + * @return Null or {@link TableState} found in znode. + * @throws KeeperException + */ + private static TableState getTableState(final ZooKeeperWatcher zkw, + final String child) + throws KeeperException { + return ZKTableReadOnly.getTableState(zkw, zkw.masterTableZNode, child); + } + + /** + * Sets the specified table as DISABLED in zookeeper. Fails silently if the + * table is already disabled in zookeeper. Sets no watches. + * @param tableName + * @throws KeeperException unexpected zookeeper exception + */ + public void setDisabledTable(String tableName) + throws KeeperException { + synchronized (this.cache) { + if (!isDisablingOrDisabledTable(tableName)) { + LOG.warn("Moving table " + tableName + " state to disabled but was " + + "not first in disabling state: " + this.cache.get(tableName)); + } + setTableState(tableName, TableState.DISABLED); + } + } + + /** + * Sets the specified table as DISABLING in zookeeper. Fails silently if the + * table is already disabled in zookeeper. Sets no watches. + * @param tableName + * @throws KeeperException unexpected zookeeper exception + */ + public void setDisablingTable(final String tableName) + throws KeeperException { + synchronized (this.cache) { + if (!isEnabledOrDisablingTable(tableName)) { + LOG.warn("Moving table " + tableName + " state to disabling but was " + + "not first in enabled state: " + this.cache.get(tableName)); + } + setTableState(tableName, TableState.DISABLING); + } + } + + /** + * Sets the specified table as ENABLING in zookeeper. Fails silently if the + * table is already disabled in zookeeper. Sets no watches. + * @param tableName + * @throws KeeperException unexpected zookeeper exception + */ + public void setEnablingTable(final String tableName) + throws KeeperException { + synchronized (this.cache) { + if (!isDisabledOrEnablingTable(tableName)) { + LOG.warn("Moving table " + tableName + " state to enabling but was " + + "not first in disabled state: " + this.cache.get(tableName)); + } + setTableState(tableName, TableState.ENABLING); + } + } + + /** + * Sets the specified table as ENABLING in zookeeper atomically + * If the table is already in ENABLING state, no operation is performed + * @param tableName + * @return if the operation succeeds or not + * @throws KeeperException unexpected zookeeper exception + */ + public boolean checkAndSetEnablingTable(final String tableName) + throws KeeperException { + synchronized (this.cache) { + if (isEnablingTable(tableName)) { + return false; + } + setTableState(tableName, TableState.ENABLING); + return true; + } + } + + /** + * If the table is found in ENABLING state the inmemory state is removed. + * This helps in cases where CreateTable is to be retried by the client incase of failures. + * If deleteZNode is true - the znode is also deleted + * @param tableName + * @param deleteZNode + * @throws KeeperException + */ + public void removeEnablingTable(final String tableName, boolean deleteZNode) + throws KeeperException { + synchronized (this.cache) { + if (isEnablingTable(tableName)) { + this.cache.remove(tableName); + if (deleteZNode) { + ZKUtil.deleteNodeFailSilent(this.watcher, + ZKUtil.joinZNode(this.watcher.masterTableZNode, tableName)); + } + } + + } + } + + /** + * Sets the specified table as ENABLING in zookeeper atomically + * If the table isn't in DISABLED state, no operation is performed + * @param tableName + * @return if the operation succeeds or not + * @throws KeeperException unexpected zookeeper exception + */ + public boolean checkDisabledAndSetEnablingTable(final String tableName) + throws KeeperException { + synchronized (this.cache) { + if (!isDisabledTable(tableName)) { + return false; + } + setTableState(tableName, TableState.ENABLING); + return true; + } + } + + /** + * Sets the specified table as DISABLING in zookeeper atomically + * If the table isn't in ENABLED state, no operation is performed + * @param tableName + * @return if the operation succeeds or not + * @throws KeeperException unexpected zookeeper exception + */ + public boolean checkEnabledAndSetDisablingTable(final String tableName) + throws KeeperException { + synchronized (this.cache) { + if (this.cache.get(tableName) != null && !isEnabledTable(tableName)) { + return false; + } + setTableState(tableName, TableState.DISABLING); + return true; + } + } + + private void setTableState(final String tableName, final TableState state) + throws KeeperException { + String znode = ZKUtil.joinZNode(this.watcher.masterTableZNode, tableName); + if (ZKUtil.checkExists(this.watcher, znode) == -1) { + ZKUtil.createAndFailSilent(this.watcher, znode); + } + String znode92 = ZKUtil.joinZNode(this.watcher.masterTableZNode92, tableName); + boolean settingToEnabled = (state == TableState.ENABLED); + // 0.92 format znode differs in that it is deleted to represent ENABLED, + // so only create if we are not setting to enabled. + if (!settingToEnabled) { + if (ZKUtil.checkExists(this.watcher, znode92) == -1) { + ZKUtil.createAndFailSilent(this.watcher, znode92); + } + } + synchronized (this.cache) { + List ops = new LinkedList(); + if (settingToEnabled) { + ops.add(ZKUtilOp.deleteNodeFailSilent(znode92)); + } + else { + ops.add(ZKUtilOp.setData(znode92, Bytes.toBytes(state.toString()))); + } + // If not running multi-update either because of configuration or failure, + // set the current format znode after the 0.92 format znode. + // This is so in the case of failure, the AssignmentManager is guaranteed to + // see the state was not applied, since it uses the current format znode internally. + ops.add(ZKUtilOp.setData(znode, Bytes.toBytes(state.toString()))); + ZKUtil.multiOrSequential(this.watcher, ops, true); + this.cache.put(tableName, state); + } + } + + public boolean isDisabledTable(final String tableName) { + return isTableState(tableName, TableState.DISABLED); + } + + public boolean isDisablingTable(final String tableName) { + return isTableState(tableName, TableState.DISABLING); + } + + public boolean isEnablingTable(final String tableName) { + return isTableState(tableName, TableState.ENABLING); + } + + public boolean isEnabledTable(String tableName) { + return isTableState(tableName, TableState.ENABLED); + } + + public boolean isDisablingOrDisabledTable(final String tableName) { + synchronized (this.cache) { + return isDisablingTable(tableName) || isDisabledTable(tableName); + } + } + + public boolean isEnabledOrDisablingTable(final String tableName) { + synchronized (this.cache) { + return isEnabledTable(tableName) || isDisablingTable(tableName); + } + } + + public boolean isDisabledOrEnablingTable(final String tableName) { + synchronized (this.cache) { + return isDisabledTable(tableName) || isEnablingTable(tableName); + } + } + + private boolean isTableState(final String tableName, final TableState state) { + synchronized (this.cache) { + TableState currentState = this.cache.get(tableName); + return ZKTableReadOnly.isTableState(currentState, state); + } + } + + /** + * Deletes the table in zookeeper. Fails silently if the + * table is not currently disabled in zookeeper. Sets no watches. + * @param tableName + * @throws KeeperException unexpected zookeeper exception + */ + public void setDeletedTable(final String tableName) + throws KeeperException { + synchronized (this.cache) { + List ops = new LinkedList(); + ops.add(ZKUtilOp.deleteNodeFailSilent( + ZKUtil.joinZNode(this.watcher.masterTableZNode92, tableName))); + // If not running multi-update either because of configuration or failure, + // delete the current format znode after the 0.92 format znode. This is so in the case of + // failure, the AssignmentManager is guaranteed to see the table was not deleted, since it + // uses the current format znode internally. + ops.add(ZKUtilOp.deleteNodeFailSilent( + ZKUtil.joinZNode(this.watcher.masterTableZNode, tableName))); + ZKUtil.multiOrSequential(this.watcher, ops, true); + if (this.cache.remove(tableName) == null) { + LOG.warn("Moving table " + tableName + " state to deleted but was " + + "already deleted"); + } + } + } + + /** + * Sets the ENABLED state in the cache and creates or force updates a node to + * ENABLED state for the specified table + * + * @param tableName + * @throws KeeperException + */ + public void setEnabledTable(final String tableName) throws KeeperException { + setTableState(tableName, TableState.ENABLED); + } + + /** + * check if table is present . + * + * @param tableName + * @return true if the table is present + */ + public boolean isTablePresent(final String tableName) { + synchronized (this.cache) { + TableState state = this.cache.get(tableName); + return !(state == null); + } + } + + /** + * Gets a list of all the tables set as disabled in zookeeper. + * @return Set of disabled tables, empty Set if none + */ + public Set getDisabledTables() { + Set disabledTables = new HashSet(); + synchronized (this.cache) { + Set tables = this.cache.keySet(); + for (String table: tables) { + if (isDisabledTable(table)) disabledTables.add(table); + } + } + return disabledTables; + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java new file mode 100644 index 0000000..f9fe97b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java @@ -0,0 +1,205 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.Map; + +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKTable; +import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState; +import org.apache.zookeeper.KeeperException; + +/** + * Non-instantiable class that provides helper functions for + * clients other than {@link AssignmentManager} for reading the + * state of a table in ZK. + * + *

    Does not cache state like {@link ZKTable}, actually reads from ZK each call. + */ +public class ZKTableReadOnly { + + private ZKTableReadOnly() {} + + /** + * Go to zookeeper and see if state of table is {@link TableState#DISABLED}. + * This method does not use cache as {@link #isDisabledTable(String)} does. + * This method is for clients other than {@link AssignmentManager} + * @param zkw + * @param tableName + * @return True if table is enabled. + * @throws KeeperException + */ + public static boolean isDisabledTable(final ZooKeeperWatcher zkw, + final String tableName) + throws KeeperException { + TableState state = getTableState(zkw, tableName); + return isTableState(TableState.DISABLED, state); + } + + /** + * Go to zookeeper and see if state of table is {@link TableState#ENABLED}. + * @param zkw + * @param tableName + * @return True if table is enabled. + * @throws KeeperException + */ + public static boolean isEnabledTable(final ZooKeeperWatcher zkw, + final String tableName) throws KeeperException { + TableState state = getTableState(zkw, tableName); + // If a table is ENABLED then we are removing table state znode in 0.92 + // but in 0.94 we keep it in ENABLED state. + return state == null || state == TableState.ENABLED; + } + + /** + * Go to zookeeper and see if state of table is {@link TableState#DISABLING} + * of {@link TableState#DISABLED}. + * @param zkw + * @param tableName + * @return True if table is enabled. + * @throws KeeperException + */ + public static boolean isDisablingOrDisabledTable(final ZooKeeperWatcher zkw, + final String tableName) throws KeeperException { + TableState state = getTableState(zkw, tableName); + return isTableState(TableState.DISABLING, state) || + isTableState(TableState.DISABLED, state); + } + + /** + * Gets a list of all the tables set as disabled in zookeeper. + * @return Set of disabled tables, empty Set if none + * @throws KeeperException + */ + public static Set getDisabledTables(ZooKeeperWatcher zkw) + throws KeeperException { + Set disabledTables = new HashSet(); + List children = + ZKUtil.listChildrenNoWatch(zkw, zkw.clientTableZNode); + for (String child: children) { + TableState state = getTableState(zkw, child); + if (state == TableState.DISABLED) disabledTables.add(child); + } + return disabledTables; + } + + /** + * Gets a list of all the tables set as disabled in zookeeper. + * @return Map of disabled tables, empty MAP if none + * @throws KeeperException + */ + public static Map> getDisabledOrDisablingTables(ZooKeeperWatcher zkw) + throws KeeperException { + Map> disabledTables = new HashMap>(); + List children = ZKUtil.listChildrenNoWatch(zkw, zkw.clientTableZNode); + for (String child : children) { + TableState state = getTableState(zkw, child); + if (state == TableState.DISABLED) { + Set tables = disabledTables.get(state); + if (tables == null) { + tables = new HashSet(); + disabledTables.put(TableState.DISABLED, tables); + } + tables.add(child); + continue; + } + if (state == TableState.DISABLING) { + Set tables = disabledTables.get(state); + if (tables == null) { + tables = new HashSet(); + disabledTables.put(TableState.DISABLING, tables); + } + tables.add(child); + continue; + } + } + return disabledTables; + } + + /** + * Gets a list of all the tables set as enabled or enabling in zookeeper. + * @return Map of enabled tables, empty map if none + * @throws KeeperException + */ + public static Map> getEnabledOrEnablingTables(ZooKeeperWatcher zkw) + throws KeeperException { + Map> enabledTables = new HashMap>(); + List children = ZKUtil.listChildrenNoWatch(zkw, zkw.clientTableZNode); + for (String child : children) { + TableState state = getTableState(zkw, child); + if (state == TableState.ENABLED) { + Set tables = enabledTables.get(state); + if (tables == null) { + tables = new HashSet(); + enabledTables.put(TableState.ENABLED, tables); + } + tables.add(child); + continue; + } + if (state == TableState.ENABLING) { + Set tables = enabledTables.get(state); + if (tables == null) { + tables = new HashSet(); + enabledTables.put(TableState.ENABLING, tables); + } + tables.add(child); + continue; + } + } + return enabledTables; + } + + static boolean isTableState(final TableState expectedState, + final TableState currentState) { + return currentState != null && currentState.equals(expectedState); + } + + /** + * Read the TableState from ZooKeeper + * @throws KeeperException + */ + static TableState getTableState(final ZooKeeperWatcher zkw, + final String child) throws KeeperException { + return getTableState(zkw, zkw.clientTableZNode, child); + } + + /** + * @deprecated Only for 0.92/0.94 compatibility. Use getTableState(zkw, child) instead. + */ + static TableState getTableState(final ZooKeeperWatcher zkw, + final String parent, final String child) throws KeeperException { + String znode = ZKUtil.joinZNode(parent, child); + byte [] data = ZKUtil.getData(zkw, znode); + if (data == null || data.length <= 0) { + return null; + } + String str = Bytes.toString(data); + try { + return TableState.valueOf(str); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(str); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java new file mode 100644 index 0000000..a91d441 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java @@ -0,0 +1,1678 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.net.InetSocketAddress; +import java.net.InetAddress; +import java.net.Socket; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; +import java.util.HashMap; +import java.util.Map; + +import javax.security.auth.login.LoginException; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.authentication.util.KerberosUtil; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.EmptyWatcher; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.executor.RegionTransitionData; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent; +import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent; +import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData; +import org.apache.zookeeper.AsyncCallback; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.NoNodeException; +import org.apache.zookeeper.Op; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.client.ZooKeeperSaslClient; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.apache.zookeeper.server.ZooKeeperSaslServer; +import org.apache.zookeeper.proto.CreateRequest; +import org.apache.zookeeper.proto.DeleteRequest; +import org.apache.zookeeper.proto.SetDataRequest; + +/** + * Internal HBase utility class for ZooKeeper. + * + *

    Contains only static methods and constants. + * + *

    Methods all throw {@link KeeperException} if there is an unexpected + * zookeeper exception, so callers of these methods must handle appropriately. + * If ZK is required for the operation, the server will need to be aborted. + */ +public class ZKUtil { + private static final Log LOG = LogFactory.getLog(ZKUtil.class); + + // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved. + private static final char ZNODE_PATH_SEPARATOR = '/'; + private static int zkDumpConnectionTimeOut; + + /** + * Creates a new connection to ZooKeeper, pulling settings and ensemble config + * from the specified configuration object using methods from {@link ZKConfig}. + * + * Sets the connection status monitoring watcher to the specified watcher. + * + * @param conf configuration to pull ensemble and other settings from + * @param watcher watcher to monitor connection changes + * @return connection to zookeeper + * @throws IOException if unable to connect to zk or config problem + */ + public static RecoverableZooKeeper connect(Configuration conf, Watcher watcher) + throws IOException { + Properties properties = ZKConfig.makeZKProps(conf); + String ensemble = ZKConfig.getZKQuorumServersString(properties); + return connect(conf, ensemble, watcher); + } + + public static RecoverableZooKeeper connect(Configuration conf, String ensemble, + Watcher watcher) + throws IOException { + return connect(conf, ensemble, watcher, ""); + } + + public static RecoverableZooKeeper connect(Configuration conf, String ensemble, + Watcher watcher, final String descriptor) + throws IOException { + if(ensemble == null) { + throw new IOException("Unable to determine ZooKeeper ensemble"); + } + int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT, + HConstants.DEFAULT_ZK_SESSION_TIMEOUT); + LOG.debug(descriptor + " opening connection to ZooKeeper with ensemble (" + + ensemble + ")"); + int retry = conf.getInt("zookeeper.recovery.retry", 3); + int retryIntervalMillis = + conf.getInt("zookeeper.recovery.retry.intervalmill", 1000); + zkDumpConnectionTimeOut = conf.getInt("zookeeper.dump.connection.timeout", + 1000); + return new RecoverableZooKeeper(ensemble, timeout, watcher, + retry, retryIntervalMillis); + } + + /** + * Log in the current zookeeper server process using the given configuration + * keys for the credential file and login principal. + * + *

    This is only applicable when running on secure hbase + * On regular HBase (without security features), this will safely be ignored. + *

    + * + * @param conf The configuration data to use + * @param keytabFileKey Property key used to configure the path to the credential file + * @param userNameKey Property key used to configure the login principal + * @param hostname Current hostname to use in any credentials + * @throws IOException underlying exception from SecurityUtil.login() call + */ + public static void loginServer(Configuration conf, String keytabFileKey, + String userNameKey, String hostname) throws IOException { + login(conf, keytabFileKey, userNameKey, hostname, + ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY, + JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME); + } + + /** + * Log in the current zookeeper client using the given configuration + * keys for the credential file and login principal. + * + *

    This is only applicable when running on secure hbase + * On regular HBase (without security features), this will safely be ignored. + *

    + * + * @param conf The configuration data to use + * @param keytabFileKey Property key used to configure the path to the credential file + * @param userNameKey Property key used to configure the login principal + * @param hostname Current hostname to use in any credentials + * @throws IOException underlying exception from SecurityUtil.login() call + */ + public static void loginClient(Configuration conf, String keytabFileKey, + String userNameKey, String hostname) throws IOException { + login(conf, keytabFileKey, userNameKey, hostname, + ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY, + JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME); + } + + /** + * Log in the current process using the given configuration keys for the + * credential file and login principal. + * + *

    This is only applicable when running on secure hbase + * On regular HBase (without security features), this will safely be ignored. + *

    + * + * @param conf The configuration data to use + * @param keytabFileKey Property key used to configure the path to the credential file + * @param userNameKey Property key used to configure the login principal + * @param hostname Current hostname to use in any credentials + * @param loginContextProperty property name to expose the entry name + * @param loginContextName jaas entry name + * @throws IOException underlying exception from SecurityUtil.login() call + */ + private static void login(Configuration conf, String keytabFileKey, + String userNameKey, String hostname, + String loginContextProperty, String loginContextName) + throws IOException { + if (!isSecureZooKeeper(conf)) + return; + + // User has specified a jaas.conf, keep this one as the good one. + // HBASE_OPTS="-Djava.security.auth.login.config=jaas.conf" + if (System.getProperty("java.security.auth.login.config") != null) + return; + + // No keytab specified, no auth + String keytabFilename = conf.get(keytabFileKey); + if (keytabFilename == null) { + LOG.warn("no keytab specified for: " + keytabFileKey); + return; + } + + String principalConfig = conf.get(userNameKey, System.getProperty("user.name")); + String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname); + + // Initialize the "jaas.conf" for keyTab/principal, + // If keyTab is not specified use the Ticket Cache. + // and set the zookeeper login context name. + JaasConfiguration jaasConf = new JaasConfiguration(loginContextName, + principalName, keytabFilename); + javax.security.auth.login.Configuration.setConfiguration(jaasConf); + System.setProperty(loginContextProperty, loginContextName); + } + + /** + * A JAAS configuration that defines the login modules that we want to use for login. + */ + private static class JaasConfiguration extends javax.security.auth.login.Configuration { + private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME = + "zookeeper-server-keytab-kerberos"; + private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME = + "zookeeper-client-keytab-kerberos"; + + private static final Map BASIC_JAAS_OPTIONS = + new HashMap(); + static { + String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG"); + if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) { + BASIC_JAAS_OPTIONS.put("debug", "true"); + } + } + + private static final Map KEYTAB_KERBEROS_OPTIONS = + new HashMap(); + static { + KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true"); + KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true"); + KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true"); + KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS); + } + + private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN = + new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(), + LoginModuleControlFlag.REQUIRED, + KEYTAB_KERBEROS_OPTIONS); + + private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF = + new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN}; + + private javax.security.auth.login.Configuration baseConfig; + private final String loginContextName; + private final boolean useTicketCache; + private final String keytabFile; + private final String principal; + + public JaasConfiguration(String loginContextName, String principal) { + this(loginContextName, principal, null, true); + } + + public JaasConfiguration(String loginContextName, String principal, String keytabFile) { + this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0); + } + + private JaasConfiguration(String loginContextName, String principal, + String keytabFile, boolean useTicketCache) { + try { + this.baseConfig = javax.security.auth.login.Configuration.getConfiguration(); + } catch (SecurityException e) { + this.baseConfig = null; + } + this.loginContextName = loginContextName; + this.useTicketCache = useTicketCache; + this.keytabFile = keytabFile; + this.principal = principal; + LOG.info("JaasConfiguration loginContextName=" + loginContextName + + " principal=" + principal + " useTicketCache=" + useTicketCache + + " keytabFile=" + keytabFile); + } + + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String appName) { + if (loginContextName.equals(appName)) { + if (!useTicketCache) { + KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile); + KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true"); + } + KEYTAB_KERBEROS_OPTIONS.put("principal", principal); + KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false"); + return KEYTAB_KERBEROS_CONF; + } + if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName); + return(null); + } + } + + // + // Helper methods + // + + /** + * Join the prefix znode name with the suffix znode name to generate a proper + * full znode name. + * + * Assumes prefix does not end with slash and suffix does not begin with it. + * + * @param prefix beginning of znode name + * @param suffix ending of znode name + * @return result of properly joining prefix with suffix + */ + public static String joinZNode(String prefix, String suffix) { + return prefix + ZNODE_PATH_SEPARATOR + suffix; + } + + /** + * Returns the full path of the immediate parent of the specified node. + * @param node path to get parent of + * @return parent of path, null if passed the root node or an invalid node + */ + public static String getParent(String node) { + int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR); + return idx <= 0 ? null : node.substring(0, idx); + } + + /** + * Get the name of the current node from the specified fully-qualified path. + * @param path fully-qualified path + * @return name of the current node + */ + public static String getNodeName(String path) { + return path.substring(path.lastIndexOf("/")+1); + } + + /** + * Get the key to the ZK ensemble for this configuration without + * adding a name at the end + * @param conf Configuration to use to build the key + * @return ensemble key without a name + */ + public static String getZooKeeperClusterKey(Configuration conf) { + return getZooKeeperClusterKey(conf, null); + } + + /** + * Get the key to the ZK ensemble for this configuration and append + * a name at the end + * @param conf Configuration to use to build the key + * @param name Name that should be appended at the end if not empty or null + * @return ensemble key with a name (if any) + */ + public static String getZooKeeperClusterKey(Configuration conf, String name) { + String ensemble = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll( + "[\\t\\n\\x0B\\f\\r]", "")); + StringBuilder builder = new StringBuilder(ensemble); + builder.append(":"); + builder.append(conf.get(HConstants.ZOOKEEPER_CLIENT_PORT)); + builder.append(":"); + builder.append(conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT)); + if (name != null && !name.isEmpty()) { + builder.append(","); + builder.append(name); + } + return builder.toString(); + } + + /** + * Apply the settings in the given key to the given configuration, this is + * used to communicate with distant clusters + * @param conf configuration object to configure + * @param key string that contains the 3 required configuratins + * @throws IOException + */ + public static void applyClusterKeyToConf(Configuration conf, String key) + throws IOException{ + String[] parts = transformClusterKey(key); + conf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]); + conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, parts[1]); + conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[2]); + } + + /** + * Separate the given key into the three configurations it should contain: + * hbase.zookeeper.quorum, hbase.zookeeper.client.port + * and zookeeper.znode.parent + * @param key + * @return the three configuration in the described order + * @throws IOException + */ + public static String[] transformClusterKey(String key) throws IOException { + String[] parts = key.split(":"); + if (parts.length != 3) { + throw new IOException("Cluster key invalid, the format should be:" + + HConstants.ZOOKEEPER_QUORUM + ":hbase.zookeeper.client.port:" + + HConstants.ZOOKEEPER_ZNODE_PARENT); + } + return parts; + } + + // + // Existence checks and watches + // + + /** + * Watch the specified znode for delete/create/change events. The watcher is + * set whether or not the node exists. If the node already exists, the method + * returns true. If the node does not exist, the method returns false. + * + * @param zkw zk reference + * @param znode path of node to watch + * @return true if znode exists, false if does not exist or error + * @throws KeeperException if unexpected zookeeper exception + */ + public static boolean watchAndCheckExists(ZooKeeperWatcher zkw, String znode) + throws KeeperException { + try { + Stat s = zkw.getRecoverableZooKeeper().exists(znode, zkw); + boolean exists = s != null ? true : false; + if (exists) { + LOG.debug(zkw.prefix("Set watcher on existing znode " + znode)); + } else { + LOG.debug(zkw.prefix(znode+" does not exist. Watcher is set.")); + } + return exists; + } catch (KeeperException e) { + LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e); + zkw.keeperException(e); + return false; + } catch (InterruptedException e) { + LOG.warn(zkw.prefix("Unable to set watcher on znode " + znode), e); + zkw.interruptedException(e); + return false; + } + } + + /** + * Check if the specified node exists. Sets no watches. + * + * @param zkw zk reference + * @param znode path of node to watch + * @return version of the node if it exists, -1 if does not exist + * @throws KeeperException if unexpected zookeeper exception + */ + public static int checkExists(ZooKeeperWatcher zkw, String znode) + throws KeeperException { + try { + Stat s = zkw.getRecoverableZooKeeper().exists(znode, null); + return s != null ? s.getVersion() : -1; + } catch (KeeperException e) { + LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e); + zkw.keeperException(e); + return -1; + } catch (InterruptedException e) { + LOG.warn(zkw.prefix("Unable to set watcher on znode (" + znode + ")"), e); + zkw.interruptedException(e); + return -1; + } + } + + // + // Znode listings + // + + /** + * Lists the children znodes of the specified znode. Also sets a watch on + * the specified znode which will capture a NodeDeleted event on the specified + * znode as well as NodeChildrenChanged if any children of the specified znode + * are created or deleted. + * + * Returns null if the specified node does not exist. Otherwise returns a + * list of children of the specified node. If the node exists but it has no + * children, an empty list will be returned. + * + * @param zkw zk reference + * @param znode path of node to list and watch children of + * @return list of children of the specified node, an empty list if the node + * exists but has no children, and null if the node does not exist + * @throws KeeperException if unexpected zookeeper exception + */ + public static List listChildrenAndWatchForNewChildren( + ZooKeeperWatcher zkw, String znode) + throws KeeperException { + try { + List children = zkw.getRecoverableZooKeeper().getChildren(znode, zkw); + return children; + } catch(KeeperException.NoNodeException ke) { + LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " + + "because node does not exist (not an error)")); + return null; + } catch (KeeperException e) { + LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e); + zkw.keeperException(e); + return null; + } catch (InterruptedException e) { + LOG.warn(zkw.prefix("Unable to list children of znode " + znode + " "), e); + zkw.interruptedException(e); + return null; + } + } + + /** + * List all the children of the specified znode, setting a watch for children + * changes and also setting a watch on every individual child in order to get + * the NodeCreated and NodeDeleted events. + * @param zkw zookeeper reference + * @param znode node to get children of and watch + * @return list of znode names, null if the node doesn't exist + * @throws KeeperException + */ + public static List listChildrenAndWatchThem(ZooKeeperWatcher zkw, + String znode) throws KeeperException { + List children = listChildrenAndWatchForNewChildren(zkw, znode); + if (children == null) { + return null; + } + for (String child : children) { + watchAndCheckExists(zkw, joinZNode(znode, child)); + } + return children; + } + + /** + * Lists the children of the specified znode without setting any watches. + * + * Used to list the currently online regionservers and their addresses. + * + * Sets no watches at all, this method is best effort. + * + * Returns an empty list if the node has no children. Returns null if the + * parent node itself does not exist. + * + * @param zkw zookeeper reference + * @param znode node to get children of as addresses + * @return list of data of children of specified znode, empty if no children, + * null if parent does not exist + * @throws KeeperException if unexpected zookeeper exception + */ + public static List listChildrenNoWatch( + ZooKeeperWatcher zkw, String znode) + throws KeeperException { + List children = null; + try { + // List the children without watching + children = zkw.getRecoverableZooKeeper().getChildren(znode, null); + } catch(KeeperException.NoNodeException nne) { + return null; + } catch(InterruptedException ie) { + zkw.interruptedException(ie); + } + return children; + } + + /** + * Simple class to hold a node path and node data. + */ + public static class NodeAndData { + private String node; + private byte [] data; + public NodeAndData(String node, byte [] data) { + this.node = node; + this.data = data; + } + public String getNode() { + return node; + } + public byte [] getData() { + return data; + } + @Override + public String toString() { + return node + " (" + RegionTransitionData.fromBytes(data) + ")"; + } + public boolean isEmpty() { + return (data.length == 0); + } + } + + /** + * Checks if the specified znode has any children. Sets no watches. + * + * Returns true if the node exists and has children. Returns false if the + * node does not exist or if the node does not have any children. + * + * Used during master initialization to determine if the master is a + * failed-over-to master or the first master during initial cluster startup. + * If the directory for regionserver ephemeral nodes is empty then this is + * a cluster startup, if not then it is not cluster startup. + * + * @param zkw zk reference + * @param znode path of node to check for children of + * @return true if node has children, false if not or node does not exist + * @throws KeeperException if unexpected zookeeper exception + */ + public static boolean nodeHasChildren(ZooKeeperWatcher zkw, String znode) + throws KeeperException { + try { + return !zkw.getRecoverableZooKeeper().getChildren(znode, null).isEmpty(); + } catch(KeeperException.NoNodeException ke) { + LOG.debug(zkw.prefix("Unable to list children of znode " + znode + " " + + "because node does not exist (not an error)")); + return false; + } catch (KeeperException e) { + LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e); + zkw.keeperException(e); + return false; + } catch (InterruptedException e) { + LOG.warn(zkw.prefix("Unable to list children of znode " + znode), e); + zkw.interruptedException(e); + return false; + } + } + + /** + * Get the number of children of the specified node. + * + * If the node does not exist or has no children, returns 0. + * + * Sets no watches at all. + * + * @param zkw zk reference + * @param znode path of node to count children of + * @return number of children of specified node, 0 if none or parent does not + * exist + * @throws KeeperException if unexpected zookeeper exception + */ + public static int getNumberOfChildren(ZooKeeperWatcher zkw, String znode) + throws KeeperException { + try { + Stat stat = zkw.getRecoverableZooKeeper().exists(znode, null); + return stat == null ? 0 : stat.getNumChildren(); + } catch(KeeperException e) { + LOG.warn(zkw.prefix("Unable to get children of node " + znode)); + zkw.keeperException(e); + } catch(InterruptedException e) { + zkw.interruptedException(e); + } + return 0; + } + + // + // Data retrieval + // + + /** + * Get znode data. Does not set a watcher. + * @return ZNode data + */ + public static byte [] getData(ZooKeeperWatcher zkw, String znode) + throws KeeperException { + try { + byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, null); + logRetrievedMsg(zkw, znode, data, false); + return data; + } catch (KeeperException.NoNodeException e) { + LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " + + "because node does not exist (not an error)")); + return null; + } catch (KeeperException e) { + LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e); + zkw.keeperException(e); + return null; + } catch (InterruptedException e) { + LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e); + zkw.interruptedException(e); + return null; + } + } + + /** + * Get the data at the specified znode and set a watch. + * + * Returns the data and sets a watch if the node exists. Returns null and no + * watch is set if the node does not exist or there is an exception. + * + * @param zkw zk reference + * @param znode path of node + * @return data of the specified znode, or null + * @throws KeeperException if unexpected zookeeper exception + */ + public static byte [] getDataAndWatch(ZooKeeperWatcher zkw, String znode) + throws KeeperException { + return getDataInternal(zkw, znode, null, true); + } + + /** + * Get the data at the specified znode and set a watch. + * + * Returns the data and sets a watch if the node exists. Returns null and no + * watch is set if the node does not exist or there is an exception. + * + * @param zkw zk reference + * @param znode path of node + * @param stat object to populate the version of the znode + * @return data of the specified znode, or null + * @throws KeeperException if unexpected zookeeper exception + */ + public static byte[] getDataAndWatch(ZooKeeperWatcher zkw, String znode, + Stat stat) throws KeeperException { + return getDataInternal(zkw, znode, stat, true); + } + + private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat, + boolean watcherSet) + throws KeeperException { + try { + byte [] data = zkw.getRecoverableZooKeeper().getData(znode, zkw, stat); + logRetrievedMsg(zkw, znode, data, watcherSet); + return data; + } catch (KeeperException.NoNodeException e) { + LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " + + "because node does not exist (not an error)")); + return null; + } catch (KeeperException e) { + LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e); + zkw.keeperException(e); + return null; + } catch (InterruptedException e) { + LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e); + zkw.interruptedException(e); + return null; + } + } + + /** + * Get the data at the specified znode without setting a watch. + * + * Returns the data if the node exists. Returns null if the node does not + * exist. + * + * Sets the stats of the node in the passed Stat object. Pass a null stat if + * not interested. + * + * @param zkw zk reference + * @param znode path of node + * @param stat node status to get if node exists + * @return data of the specified znode, or null if node does not exist + * @throws KeeperException if unexpected zookeeper exception + */ + public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode, + Stat stat) + throws KeeperException { + try { + byte [] data = zkw.getRecoverableZooKeeper().getData(znode, null, stat); + logRetrievedMsg(zkw, znode, data, false); + return data; + } catch (KeeperException.NoNodeException e) { + LOG.debug(zkw.prefix("Unable to get data of znode " + znode + " " + + "because node does not exist (not necessarily an error)")); + return null; + } catch (KeeperException e) { + LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e); + zkw.keeperException(e); + return null; + } catch (InterruptedException e) { + LOG.warn(zkw.prefix("Unable to get data of znode " + znode), e); + zkw.interruptedException(e); + return null; + } + } + + /** + * Returns the date of child znodes of the specified znode. Also sets a watch on + * the specified znode which will capture a NodeDeleted event on the specified + * znode as well as NodeChildrenChanged if any children of the specified znode + * are created or deleted. + * + * Returns null if the specified node does not exist. Otherwise returns a + * list of children of the specified node. If the node exists but it has no + * children, an empty list will be returned. + * + * @param zkw zk reference + * @param baseNode path of node to list and watch children of + * @return list of data of children of the specified node, an empty list if the node + * exists but has no children, and null if the node does not exist + * @throws KeeperException if unexpected zookeeper exception + */ + public static List getChildDataAndWatchForNewChildren( + ZooKeeperWatcher zkw, String baseNode) throws KeeperException { + List nodes = + ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode); + List newNodes = new ArrayList(); + if (nodes != null) { + for (String node : nodes) { + String nodePath = ZKUtil.joinZNode(baseNode, node); + byte[] data = ZKUtil.getDataAndWatch(zkw, nodePath); + newNodes.add(new NodeAndData(nodePath, data)); + } + } + return newNodes; + } + + /** + * Update the data of an existing node with the expected version to have the + * specified data. + * + * Throws an exception if there is a version mismatch or some other problem. + * + * Sets no watches under any conditions. + * + * @param zkw zk reference + * @param znode + * @param data + * @param expectedVersion + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.BadVersionException if version mismatch + */ + public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode, + byte [] data, int expectedVersion) + throws KeeperException { + try { + zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion); + } catch(InterruptedException ie) { + zkw.interruptedException(ie); + } + } + + // + // Data setting + // + + /** + * Sets the data of the existing znode to be the specified data. Ensures that + * the current data has the specified expected version. + * + *

    If the node does not exist, a {@link NoNodeException} will be thrown. + * + *

    If their is a version mismatch, method returns null. + * + *

    No watches are set but setting data will trigger other watchers of this + * node. + * + *

    If there is another problem, a KeeperException will be thrown. + * + * @param zkw zk reference + * @param znode path of node + * @param data data to set for node + * @param expectedVersion version expected when setting data + * @return true if data set, false if version mismatch + * @throws KeeperException if unexpected zookeeper exception + */ + public static boolean setData(ZooKeeperWatcher zkw, String znode, + byte [] data, int expectedVersion) + throws KeeperException, KeeperException.NoNodeException { + try { + return zkw.getRecoverableZooKeeper().setData(znode, data, expectedVersion) != null; + } catch (InterruptedException e) { + zkw.interruptedException(e); + return false; + } + } + + /** + * Set data into node creating node if it doesn't yet exist. + * Does not set watch. + * + * @param zkw zk reference + * @param znode path of node + * @param data data to set for node + * @throws KeeperException + */ + public static void createSetData(final ZooKeeperWatcher zkw, final String znode, + final byte [] data) + throws KeeperException { + if (checkExists(zkw, znode) == -1) { + ZKUtil.createWithParents(zkw, znode, data); + } else { + ZKUtil.setData(zkw, znode, data); + } + } + + /** + * Sets the data of the existing znode to be the specified data. The node + * must exist but no checks are done on the existing data or version. + * + *

    If the node does not exist, a {@link NoNodeException} will be thrown. + * + *

    No watches are set but setting data will trigger other watchers of this + * node. + * + *

    If there is another problem, a KeeperException will be thrown. + * + * @param zkw zk reference + * @param znode path of node + * @param data data to set for node + * @throws KeeperException if unexpected zookeeper exception + */ + public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data) + throws KeeperException, KeeperException.NoNodeException { + setData(zkw, (SetData)ZKUtilOp.setData(znode, data)); + } + + private static void setData(ZooKeeperWatcher zkw, SetData setData) + throws KeeperException, KeeperException.NoNodeException { + SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord(); + setData(zkw, sd.getPath(), sd.getData(), sd.getVersion()); + } + + /** + * Returns whether or not secure authentication is enabled + * (whether hbase.security.authentication is set to + * kerberos. + */ + public static boolean isSecureZooKeeper(Configuration conf) { + // hbase shell need to use: + // -Djava.security.auth.login.config=user-jaas.conf + // since each user has a different jaas.conf + if (System.getProperty("java.security.auth.login.config") != null) + return true; + + // Master & RSs uses hbase.zookeeper.client.* + return("kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")) && + conf.get("hbase.zookeeper.client.keytab.file") != null); + } + + private static ArrayList createACL(ZooKeeperWatcher zkw, String node) { + if (isSecureZooKeeper(zkw.getConfiguration())) { + // Certain znodes are accessed directly by the client, + // so they must be readable by non-authenticated clients + if ((node.equals(zkw.baseZNode) == true) || + (node.equals(zkw.rootServerZNode) == true) || + (node.equals(zkw.masterAddressZNode) == true) || + (node.equals(zkw.clusterIdZNode) == true) || + (node.equals(zkw.rsZNode) == true) || + (node.equals(zkw.backupMasterAddressesZNode) == true) || + (node.startsWith(zkw.masterTableZNode) == true) || + (node.startsWith(zkw.masterTableZNode92) == true)) { + return ZooKeeperWatcher.CREATOR_ALL_AND_WORLD_READABLE; + } + return Ids.CREATOR_ALL_ACL; + } else { + return Ids.OPEN_ACL_UNSAFE; + } + } + + // + // Node creation + // + + /** + * + * Set the specified znode to be an ephemeral node carrying the specified + * data. + * + * If the node is created successfully, a watcher is also set on the node. + * + * If the node is not created successfully because it already exists, this + * method will also set a watcher on the node. + * + * If there is another problem, a KeeperException will be thrown. + * + * @param zkw zk reference + * @param znode path of node + * @param data data of node + * @return true if node created, false if not, watch set in both cases + * @throws KeeperException if unexpected zookeeper exception + */ + public static boolean createEphemeralNodeAndWatch(ZooKeeperWatcher zkw, + String znode, byte [] data) + throws KeeperException { + try { + zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode), + CreateMode.EPHEMERAL); + } catch (KeeperException.NodeExistsException nee) { + if(!watchAndCheckExists(zkw, znode)) { + // It did exist but now it doesn't, try again + return createEphemeralNodeAndWatch(zkw, znode, data); + } + return false; + } catch (InterruptedException e) { + LOG.info("Interrupted", e); + Thread.currentThread().interrupt(); + } + return true; + } + + /** + * Creates the specified znode to be a persistent node carrying the specified + * data. + * + * Returns true if the node was successfully created, false if the node + * already existed. + * + * If the node is created successfully, a watcher is also set on the node. + * + * If the node is not created successfully because it already exists, this + * method will also set a watcher on the node but return false. + * + * If there is another problem, a KeeperException will be thrown. + * + * @param zkw zk reference + * @param znode path of node + * @param data data of node + * @return true if node created, false if not, watch set in both cases + * @throws KeeperException if unexpected zookeeper exception + */ + public static boolean createNodeIfNotExistsAndWatch( + ZooKeeperWatcher zkw, String znode, byte [] data) + throws KeeperException { + try { + zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode), + CreateMode.PERSISTENT); + } catch (KeeperException.NodeExistsException nee) { + try { + zkw.getRecoverableZooKeeper().exists(znode, zkw); + } catch (InterruptedException e) { + zkw.interruptedException(e); + return false; + } + return false; + } catch (InterruptedException e) { + zkw.interruptedException(e); + return false; + } + return true; + } + + /** + * Creates the specified node with the specified data and watches it. + * + *

    Throws an exception if the node already exists. + * + *

    The node created is persistent and open access. + * + *

    Returns the version number of the created node if successful. + * + * @param zkw zk reference + * @param znode path of node to create + * @param data data of node to create + * @return version of node created + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NodeExistsException if node already exists + */ + public static int createAndWatch(ZooKeeperWatcher zkw, + String znode, byte [] data) + throws KeeperException, KeeperException.NodeExistsException { + try { + zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode), + CreateMode.PERSISTENT); + return zkw.getRecoverableZooKeeper().exists(znode, zkw).getVersion(); + } catch (InterruptedException e) { + zkw.interruptedException(e); + return -1; + } + } + + /** + * Async creates the specified node with the specified data. + * + *

    Throws an exception if the node already exists. + * + *

    The node created is persistent and open access. + * + * @param zkw zk reference + * @param znode path of node to create + * @param data data of node to create + * @param cb + * @param ctx + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NodeExistsException if node already exists + */ + public static void asyncCreate(ZooKeeperWatcher zkw, + String znode, byte [] data, final AsyncCallback.StringCallback cb, + final Object ctx) { + zkw.getRecoverableZooKeeper().getZooKeeper().create(znode, data, + createACL(zkw, znode), CreateMode.PERSISTENT, cb, ctx); + } + + /** + * Creates the specified node, iff the node does not exist. Does not set a + * watch and fails silently if the node already exists. + * + * The node created is persistent and open access. + * + * @param zkw zk reference + * @param znode path of node + * @throws KeeperException if unexpected zookeeper exception + */ + public static void createAndFailSilent(ZooKeeperWatcher zkw, + String znode) throws KeeperException { + createAndFailSilent(zkw, znode, new byte[0]); + } + + /** + * Creates the specified node containing specified data, iff the node does not exist. Does + * not set a watch and fails silently if the node already exists. + * + * The node created is persistent and open access. + * + * @param zkw zk reference + * @param znode path of node + * @param data a byte array data to store in the znode + * @throws KeeperException if unexpected zookeeper exception + */ + public static void createAndFailSilent(ZooKeeperWatcher zkw, + String znode, byte[] data) throws KeeperException { + createAndFailSilent(zkw, + (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, data)); + } + + private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs) + throws KeeperException { + CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord(); + String znode = create.getPath(); + try { + RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper(); + if (zk.exists(znode, false) == null) { + zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags())); + } + } catch(KeeperException.NodeExistsException nee) { + } catch(KeeperException.NoAuthException nee){ + try { + if (null == zkw.getRecoverableZooKeeper().exists(znode, false)) { + // If we failed to create the file and it does not already exist. + throw(nee); + } + } catch (InterruptedException ie) { + zkw.interruptedException(ie); + } + + } catch(InterruptedException ie) { + zkw.interruptedException(ie); + } + } + + /** + * Creates the specified node and all parent nodes required for it to exist. + * + * No watches are set and no errors are thrown if the node already exists. + * + * The nodes created are persistent and open access. + * + * @param zkw zk reference + * @param znode path of node + * @throws KeeperException if unexpected zookeeper exception + */ + public static void createWithParents(ZooKeeperWatcher zkw, String znode) + throws KeeperException { + createWithParents(zkw, znode, new byte[0]); + } + + /** + * Creates the specified node and all parent nodes required for it to exist. The creation of + * parent znodes is not atomic with the leafe znode creation but the data is written atomically + * when the leaf node is created. + * + * No watches are set and no errors are thrown if the node already exists. + * + * The nodes created are persistent and open access. + * + * @param zkw zk reference + * @param znode path of node + * @throws KeeperException if unexpected zookeeper exception + */ + public static void createWithParents(ZooKeeperWatcher zkw, String znode, byte[] data) + throws KeeperException { + try { + if(znode == null) { + return; + } + zkw.getRecoverableZooKeeper().create(znode, data, createACL(zkw, znode), + CreateMode.PERSISTENT); + } catch(KeeperException.NodeExistsException nee) { + return; + } catch(KeeperException.NoNodeException nne) { + createWithParents(zkw, getParent(znode)); + createWithParents(zkw, znode, data); + } catch(InterruptedException ie) { + zkw.interruptedException(ie); + } + } + + // + // Deletes + // + + /** + * Delete the specified node. Sets no watches. Throws all exceptions. + */ + public static void deleteNode(ZooKeeperWatcher zkw, String node) + throws KeeperException { + deleteNode(zkw, node, -1); + } + + /** + * Delete the specified node with the specified version. Sets no watches. + * Throws all exceptions. + */ + public static boolean deleteNode(ZooKeeperWatcher zkw, String node, + int version) + throws KeeperException { + try { + zkw.getRecoverableZooKeeper().delete(node, version); + return true; + } catch(KeeperException.BadVersionException bve) { + return false; + } catch(InterruptedException ie) { + zkw.interruptedException(ie); + return false; + } + } + + /** + * Deletes the specified node. Fails silent if the node does not exist. + * @param zkw + * @param node + * @throws KeeperException + */ + public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node) + throws KeeperException { + deleteNodeFailSilent(zkw, + (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node)); + } + + private static void deleteNodeFailSilent(ZooKeeperWatcher zkw, + DeleteNodeFailSilent dnfs) throws KeeperException { + DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord(); + try { + zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion()); + } catch(KeeperException.NoNodeException nne) { + } catch(InterruptedException ie) { + zkw.interruptedException(ie); + } + } + + /** + * Delete the specified node and all of it's children. + *

    + * If the node does not exist, just returns. + *

    + * Sets no watches. Throws all exceptions besides dealing with deletion of + * children. + */ + public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node) + throws KeeperException { + try { + List children = ZKUtil.listChildrenNoWatch(zkw, node); + // the node is already deleted, so we just finish + if (children == null) return; + + if(!children.isEmpty()) { + for(String child : children) { + deleteNodeRecursively(zkw, joinZNode(node, child)); + } + } + zkw.getRecoverableZooKeeper().delete(node, -1); + } catch(InterruptedException ie) { + zkw.interruptedException(ie); + } + } + + /** + * Delete all the children of the specified node but not the node itself. + * + * Sets no watches. Throws all exceptions besides dealing with deletion of + * children. + */ + public static void deleteChildrenRecursively(ZooKeeperWatcher zkw, String node) + throws KeeperException { + List children = ZKUtil.listChildrenNoWatch(zkw, node); + if (children == null || children.isEmpty()) return; + for(String child : children) { + deleteNodeRecursively(zkw, joinZNode(node, child)); + } + } + + /** + * Represents an action taken by ZKUtil, e.g. createAndFailSilent. + * These actions are higher-level than {@link ZKOp} actions, which represent + * individual actions in the ZooKeeper API, like create. + */ + public abstract static class ZKUtilOp { + private String path; + + private ZKUtilOp(String path) { + this.path = path; + } + + /** + * @return a createAndFailSilent ZKUtilOp + */ + public static ZKUtilOp createAndFailSilent(String path, byte[] data) { + return new CreateAndFailSilent(path, data); + } + + /** + * @return a deleteNodeFailSilent ZKUtilOP + */ + public static ZKUtilOp deleteNodeFailSilent(String path) { + return new DeleteNodeFailSilent(path); + } + + /** + * @return a setData ZKUtilOp + */ + public static ZKUtilOp setData(String path, byte [] data) { + return new SetData(path, data); + } + + /** + * @return path to znode where the ZKOp will occur + */ + public String getPath() { + return path; + } + + /** + * ZKUtilOp representing createAndFailSilent in ZooKeeper + * (attempt to create node, ignore error if already exists) + */ + public static class CreateAndFailSilent extends ZKUtilOp { + private byte [] data; + + private CreateAndFailSilent(String path, byte [] data) { + super(path); + this.data = data; + } + + public byte[] getData() { + return data; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof CreateAndFailSilent)) return false; + + CreateAndFailSilent op = (CreateAndFailSilent) o; + return getPath().equals(op.getPath()) && Arrays.equals(data, op.data); + } + } + + /** + * ZKUtilOp representing deleteNodeFailSilent in ZooKeeper + * (attempt to delete node, ignore error if node doesn't exist) + */ + public static class DeleteNodeFailSilent extends ZKUtilOp { + private DeleteNodeFailSilent(String path) { + super(path); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DeleteNodeFailSilent)) return false; + + return super.equals(o); + } + } + + /** + * @return ZKUtilOp representing setData in ZooKeeper + */ + public static class SetData extends ZKUtilOp { + private byte [] data; + + private SetData(String path, byte [] data) { + super(path); + this.data = data; + } + + public byte[] getData() { + return data; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof SetData)) return false; + + SetData op = (SetData) o; + return getPath().equals(op.getPath()) && Arrays.equals(data, op.data); + } + } + } + + /** + * Convert from ZKUtilOp to ZKOp + */ + private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op) + throws UnsupportedOperationException { + if(op == null) return null; + + if (op instanceof CreateAndFailSilent) { + CreateAndFailSilent cafs = (CreateAndFailSilent)op; + return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()), + CreateMode.PERSISTENT); + } else if (op instanceof DeleteNodeFailSilent) { + DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op; + return Op.delete(dnfs.getPath(), -1); + } else if (op instanceof SetData) { + SetData sd = (SetData)op; + return Op.setData(sd.getPath(), sd.getData(), -1); + } else { + throw new UnsupportedOperationException("Unexpected ZKUtilOp type: " + + op.getClass().getName()); + } + } + + /** + * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality. + * Otherwise, run the list of operations sequentially. + * + * If all of the following are true: + * - runSequentialOnMultiFailure is true + * - hbase.zookeeper.useMulti is true + * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*) + * Then: + * - we retry the operations one-by-one (sequentially) + * + * Note *: an example is receiving a NodeExistsException from a "create" call. Without multi, + * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who + * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught). + * This will cause all operations in the multi to fail, however, because + * the NodeExistsException that zk.create throws will fail the multi transaction. + * In this case, if the previous conditions hold, the commands are run sequentially, which should + * result in the correct final state, but means that the operations will not run atomically. + * + * @throws KeeperException + */ + public static void multiOrSequential(ZooKeeperWatcher zkw, List ops, + boolean runSequentialOnMultiFailure) throws KeeperException { + if (ops == null) return; + boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false); + + if (useMulti) { + List zkOps = new LinkedList(); + for (ZKUtilOp op : ops) { + zkOps.add(toZooKeeperOp(zkw, op)); + } + try { + zkw.getRecoverableZooKeeper().multi(zkOps); + } catch (KeeperException ke) { + switch (ke.code()) { + case NODEEXISTS: + case NONODE: + case BADVERSION: + case NOAUTH: + // if we get an exception that could be solved by running sequentially + // (and the client asked us to), then break out and run sequentially + if (runSequentialOnMultiFailure) { + LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "." + + " Attempting to run operations sequentially because" + + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + "."); + processSequentially(zkw, ops); + break; + } + default: + throw ke; + } + } catch (InterruptedException ie) { + zkw.interruptedException(ie); + } + } else { + // run sequentially + processSequentially(zkw, ops); + } + } + + private static void processSequentially(ZooKeeperWatcher zkw, List ops) + throws KeeperException, NoNodeException { + for (ZKUtilOp op : ops) { + if (op instanceof CreateAndFailSilent) { + createAndFailSilent(zkw, (CreateAndFailSilent) op); + } else if (op instanceof DeleteNodeFailSilent) { + deleteNodeFailSilent(zkw, (DeleteNodeFailSilent) op); + } else if (op instanceof SetData) { + setData(zkw, (SetData) op); + } else { + throw new UnsupportedOperationException("Unexpected ZKUtilOp type: " + + op.getClass().getName()); + } + } + } + + // + // ZooKeeper cluster information + // + + /** @return String dump of everything in ZooKeeper. */ + public static String dump(ZooKeeperWatcher zkw) { + StringBuilder sb = new StringBuilder(); + try { + sb.append("HBase is rooted at ").append(zkw.baseZNode); + sb.append("\nActive master address: ").append( + ServerName.parseVersionedServerName(getData(zkw, zkw.masterAddressZNode))); + sb.append("\nBackup master addresses:"); + for (String child : listChildrenNoWatch(zkw, + zkw.backupMasterAddressesZNode)) { + sb.append("\n ").append(child); + } + sb.append("\nRegion server holding ROOT: ").append( + Bytes.toStringBinary(getData(zkw, zkw.rootServerZNode))); + sb.append("\nRegion servers:"); + for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) { + sb.append("\n ").append(child); + } + try { + getReplicationZnodesDump(zkw, sb); + } catch (KeeperException ke) { + LOG.warn("Couldn't get the replication znode dump." + ke.getStackTrace()); + } + sb.append("\nQuorum Server Statistics:"); + String[] servers = zkw.getQuorum().split(","); + for (String server : servers) { + sb.append("\n ").append(server); + try { + String[] stat = getServerStats(server, ZKUtil.zkDumpConnectionTimeOut); + + if (stat == null) { + sb.append("[Error] invalid quorum server: " + server); + break; + } + + for (String s : stat) { + sb.append("\n ").append(s); + } + } catch (Exception e) { + sb.append("\n ERROR: ").append(e.getMessage()); + } + } + } catch (KeeperException ke) { + sb.append("\nFATAL ZooKeeper Exception!\n"); + sb.append("\n" + ke.getMessage()); + } + return sb.toString(); + } + + private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb) + throws KeeperException { + String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication", + "replication"); + String replicationZnode = joinZNode(zkw.baseZNode, replicationZNodeName); + if (ZKUtil.checkExists(zkw, replicationZnode) == -1) + return; + // do a ls -r on this znode + List stack = new LinkedList(); + stack.add(replicationZnode); + do { + String znodeToProcess = stack.remove(stack.size() - 1); + sb.append("\n").append(znodeToProcess).append(": ") + .append(Bytes.toString(ZKUtil.getData(zkw, znodeToProcess))); + for (String zNodeChild : ZKUtil.listChildrenNoWatch(zkw, znodeToProcess)) { + stack.add(ZKUtil.joinZNode(znodeToProcess, zNodeChild)); + } + } while (stack.size() > 0); + } + /** + * Gets the statistics from the given server. + * + * @param server The server to get the statistics from. + * @param timeout The socket timeout to use. + * @return The array of response strings. + * @throws IOException When the socket communication fails. + */ + public static String[] getServerStats(String server, int timeout) + throws IOException { + String[] sp = server.split(":"); + if (sp == null || sp.length == 0) { + return null; + } + + String host = sp[0]; + int port = sp.length > 1 ? Integer.parseInt(sp[1]) + : HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT; + + Socket socket = new Socket(); + InetSocketAddress sockAddr = new InetSocketAddress(host, port); + socket.connect(sockAddr, timeout); + + socket.setSoTimeout(timeout); + PrintWriter out = new PrintWriter(socket.getOutputStream(), true); + BufferedReader in = new BufferedReader(new InputStreamReader( + socket.getInputStream())); + out.println("stat"); + out.flush(); + ArrayList res = new ArrayList(); + while (true) { + String line = in.readLine(); + if (line != null) { + res.add(line); + } else { + break; + } + } + socket.close(); + return res.toArray(new String[res.size()]); + } + + private static void logRetrievedMsg(final ZooKeeperWatcher zkw, + final String znode, final byte [] data, final boolean watcherSet) { + if (!LOG.isDebugEnabled()) return; + LOG.debug(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) + + " byte(s) of data from znode " + znode + + (watcherSet? " and set watcher; ": "; data=") + + (data == null? "null": data.length == 0? "empty": ( + znode.startsWith(zkw.assignmentZNode) ? + RegionTransitionData.fromBytes(data).toString() + : StringUtils.abbreviate(Bytes.toStringBinary(data), 32))))); + } + + /** + * Waits for HBase installation's base (parent) znode to become available. + * @throws IOException on ZK errors + */ + public static void waitForBaseZNode(Configuration conf) throws IOException { + LOG.info("Waiting until the base znode is available"); + String parentZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, + HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); + ZooKeeper zk = new ZooKeeper(ZKConfig.getZKQuorumServersString(conf), + conf.getInt(HConstants.ZK_SESSION_TIMEOUT, + HConstants.DEFAULT_ZK_SESSION_TIMEOUT), EmptyWatcher.instance); + + final int maxTimeMs = 10000; + final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS; + + KeeperException keeperEx = null; + try { + try { + for (int attempt = 0; attempt < maxNumAttempts; ++attempt) { + try { + if (zk.exists(parentZNode, false) != null) { + LOG.info("Parent znode exists: " + parentZNode); + keeperEx = null; + break; + } + } catch (KeeperException e) { + keeperEx = e; + } + Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS); + } + } finally { + zk.close(); + } + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + + if (keeperEx != null) { + throw new IOException(keeperEx); + } + } + + /** + * Recursively print the current state of ZK (non-transactional) + * @param root name of the root directory in zk to print + * @throws KeeperException + */ + public static void logZKTree(ZooKeeperWatcher zkw, String root) { + if (!LOG.isDebugEnabled()) return; + LOG.debug("Current zk system:"); + String prefix = "|-"; + LOG.debug(prefix + root); + try { + logZKTree(zkw, root, prefix); + } catch (KeeperException e) { + throw new RuntimeException(e); + } + } + + /** + * Helper method to print the current state of the ZK tree. + * @see #logZKTree(ZooKeeperWatcher, String) + * @throws KeeperException if an unexpected exception occurs + */ + protected static void logZKTree(ZooKeeperWatcher zkw, String root, String prefix) throws KeeperException { + List children = ZKUtil.listChildrenNoWatch(zkw, root); + if (children == null) return; + for (String child : children) { + LOG.debug(prefix + child); + String node = ZKUtil.joinZNode(root.equals("/") ? "" : root, child); + logZKTree(zkw, node, prefix + "---"); + } + } + +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java new file mode 100644 index 0000000..bfd1ce8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java @@ -0,0 +1,78 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + + +/** + * Base class for internal listeners of ZooKeeper events. + * + * The {@link ZooKeeperWatcher} for a process will execute the appropriate + * methods of implementations of this class. In order to receive events from + * the watcher, every listener must register itself via {@link ZooKeeperWatcher#registerListener}. + * + * Subclasses need only override those methods in which they are interested. + * + * Note that the watcher will be blocked when invoking methods in listeners so + * they must not be long-running. + */ +public abstract class ZooKeeperListener { + + // Reference to the zk watcher which also contains configuration and constants + protected ZooKeeperWatcher watcher; + + /** + * Construct a ZooKeeper event listener. + */ + public ZooKeeperListener(ZooKeeperWatcher watcher) { + this.watcher = watcher; + } + + /** + * Called when a new node has been created. + * @param path full path of the new node + */ + public void nodeCreated(String path) { + // no-op + } + + /** + * Called when a node has been deleted + * @param path full path of the deleted node + */ + public void nodeDeleted(String path) { + // no-op + } + + /** + * Called when an existing node has changed data. + * @param path full path of the updated node + */ + public void nodeDataChanged(String path) { + // no-op + } + + /** + * Called when an existing node has a child node added or removed. + * @param path full path of the node whose children have changed + */ + public void nodeChildrenChanged(String path) { + // no-op + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperMainServerArg.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperMainServerArg.java new file mode 100644 index 0000000..e6f7f16 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperMainServerArg.java @@ -0,0 +1,78 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.zookeeper; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; + +/** + * Tool for reading a ZooKeeper server from HBase XML configuration producing + * the '-server host:port' argument to pass ZooKeeperMain. This program + * emits either '-server HOST:PORT" where HOST is one of the zk ensemble + * members plus zk client port OR it emits '' if no zk servers found (Yes, + * it emits '-server' too). + */ +public class ZooKeeperMainServerArg { + public String parse(final Configuration c) { + // Note that we do not simply grab the property + // HConstants.ZOOKEEPER_QUORUM from the HBaseConfiguration because the + // user may be using a zoo.cfg file. + Properties zkProps = ZKConfig.makeZKProps(c); + String host = null; + String clientPort = null; + List hosts = new ArrayList(); + for (Entry entry: zkProps.entrySet()) { + String key = entry.getKey().toString().trim(); + String value = entry.getValue().toString().trim(); + if (key.startsWith("server.")) { + String[] parts = value.split(":"); + hosts.add(parts[0]); + } else if (key.endsWith("clientPort")) { + clientPort = value; + } + } + if (hosts.isEmpty() || clientPort == null) + return null; + for (int i = 0; i < hosts.size(); i++) { + if (i > 0) + host += "," + hosts.get(i); + else + host = hosts.get(i); + } + return host != null ? host + ":" + clientPort : null; + } + + /** + * Run the tool. + * @param args Command line arguments. First arg is path to zookeepers file. + */ + public static void main(String args[]) { + Configuration conf = HBaseConfiguration.create(); + String hostport = new ZooKeeperMainServerArg().parse(conf); + System.out.println((hostport == null || hostport.length() == 0)? "": + "-server " + hostport); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java new file mode 100644 index 0000000..4365f78 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java @@ -0,0 +1,219 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; +import org.apache.zookeeper.KeeperException; + +/** + * Tracks the availability and value of a single ZooKeeper node. + * + *

    Utilizes the {@link ZooKeeperListener} interface to get the necessary + * ZooKeeper events related to the node. + * + *

    This is the base class used by trackers in both the Master and + * RegionServers. + */ +public abstract class ZooKeeperNodeTracker extends ZooKeeperListener { + + static final Log LOG = LogFactory.getLog(ZooKeeperNodeTracker.class); + /** Path of node being tracked */ + protected final String node; + + /** Data of the node being tracked */ + private byte [] data; + + /** Used to abort if a fatal error occurs */ + protected final Abortable abortable; + + private volatile boolean stopped = false; + + /** + * Constructs a new ZK node tracker. + * + *

    After construction, use {@link #start} to kick off tracking. + * + * @param watcher + * @param node + * @param abortable + */ + public ZooKeeperNodeTracker(ZooKeeperWatcher watcher, String node, + Abortable abortable) { + super(watcher); + this.node = node; + this.abortable = abortable; + this.data = null; + } + + /** + * Starts the tracking of the node in ZooKeeper. + * + *

    Use {@link #blockUntilAvailable()} to block until the node is available + * or {@link #getData(boolean)} to get the data of the node if it is available. + */ + public synchronized void start() { + this.watcher.registerListener(this); + try { + if(ZKUtil.watchAndCheckExists(watcher, node)) { + byte [] data = ZKUtil.getDataAndWatch(watcher, node); + if(data != null) { + this.data = data; + } else { + // It existed but now does not, try again to ensure a watch is set + LOG.debug("Try starting again because there is no data from " + node); + start(); + } + } + } catch (KeeperException e) { + abortable.abort("Unexpected exception during initialization, aborting", e); + } + } + + public synchronized void stop() { + this.stopped = true; + notifyAll(); + } + + /** + * Gets the data of the node, blocking until the node is available. + * + * @return data of the node + * @throws InterruptedException if the waiting thread is interrupted + */ + public synchronized byte [] blockUntilAvailable() + throws InterruptedException { + return blockUntilAvailable(0, false); + } + + /** + * Gets the data of the node, blocking until the node is available or the + * specified timeout has elapsed. + * + * @param timeout maximum time to wait for the node data to be available, + * n milliseconds. Pass 0 for no timeout. + * @return data of the node + * @throws InterruptedException if the waiting thread is interrupted + */ + public synchronized byte [] blockUntilAvailable(long timeout, boolean refresh) + throws InterruptedException { + if (timeout < 0) throw new IllegalArgumentException(); + boolean notimeout = timeout == 0; + long startTime = System.currentTimeMillis(); + long remaining = timeout; + if (refresh) { + try { + this.data = ZKUtil.getDataAndWatch(watcher, node); + } catch(KeeperException e) { + abortable.abort("Unexpected exception handling blockUntilAvailable", e); + } + } + while (!this.stopped && (notimeout || remaining > 0) && this.data == null) { + // We expect a notification; but we wait with a + // a timeout to lower the impact of a race condition if any + wait(100); + remaining = timeout - (System.currentTimeMillis() - startTime); + } + return this.data; + } + + /** + * Gets the data of the node. + * + *

    If the node is currently available, the most up-to-date known version of + * the data is returned. If the node is not currently available, null is + * returned. + * @param refresh whether to refresh the data by calling ZK directly. + * @return data of the node, null if unavailable + */ + public synchronized byte [] getData(boolean refresh) { + if (refresh) { + try { + this.data = ZKUtil.getDataAndWatch(watcher, node); + } catch(KeeperException e) { + abortable.abort("Unexpected exception handling getData", e); + } + } + return this.data; + } + + public String getNode() { + return this.node; + } + + @Override + public synchronized void nodeCreated(String path) { + if (!path.equals(node)) return; + try { + byte [] data = ZKUtil.getDataAndWatch(watcher, node); + if (data != null) { + this.data = data; + notifyAll(); + } else { + nodeDeleted(path); + } + } catch(KeeperException e) { + abortable.abort("Unexpected exception handling nodeCreated event", e); + } + } + + @Override + public synchronized void nodeDeleted(String path) { + if(path.equals(node)) { + try { + if(ZKUtil.watchAndCheckExists(watcher, node)) { + nodeCreated(path); + } else { + this.data = null; + } + } catch(KeeperException e) { + abortable.abort("Unexpected exception handling nodeDeleted event", e); + } + } + } + + @Override + public synchronized void nodeDataChanged(String path) { + if(path.equals(node)) { + nodeCreated(path); + } + } + + /** + * Checks if the baseznode set as per the property 'zookeeper.znode.parent' + * exists. + * @return true if baseznode exists. + * false if doesnot exists. + */ + public boolean checkIfBaseNodeAvailable() { + try { + if (ZKUtil.checkExists(watcher, watcher.baseZNode) == -1) { + return false; + } + } catch (KeeperException e) { + abortable + .abort( + "Exception while checking if basenode exists.", + e); + } + return true; + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java new file mode 100644 index 0000000..e023692 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java @@ -0,0 +1,467 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; + +/** + * Acts as the single ZooKeeper Watcher. One instance of this is instantiated + * for each Master, RegionServer, and client process. + * + *

    This is the only class that implements {@link Watcher}. Other internal + * classes which need to be notified of ZooKeeper events must register with + * the local instance of this watcher via {@link #registerListener}. + * + *

    This class also holds and manages the connection to ZooKeeper. Code to + * deal with connection related events and exceptions are handled here. + */ +public class ZooKeeperWatcher implements Watcher, Abortable { + private static final Log LOG = LogFactory.getLog(ZooKeeperWatcher.class); + + // Identifier for this watcher (for logging only). It is made of the prefix + // passed on construction and the zookeeper sessionid. + private String identifier; + + // zookeeper quorum + private String quorum; + + // zookeeper connection + private RecoverableZooKeeper recoverableZooKeeper; + + // abortable in case of zk failure + private Abortable abortable; + + // listeners to be notified + private final List listeners = + new CopyOnWriteArrayList(); + + // set of unassigned nodes watched + private Set unassignedNodes = new HashSet(); + + // node names + + // base znode for this cluster + public String baseZNode; + // znode containing location of server hosting root region + public String rootServerZNode; + // znode containing ephemeral nodes of the regionservers + public String rsZNode; + // znode containing ephemeral nodes of the draining regionservers + public String drainingZNode; + // znode of currently active master + public String masterAddressZNode; + // znode of this master in backup master directory, if not the active master + public String backupMasterAddressesZNode; + // znode containing the current cluster state + public String clusterStateZNode; + // znode used for region transitioning and assignment + public String assignmentZNode; + // znode that the master uses for reading/writing the table disabling/enabling states + public String masterTableZNode; + // znode where the client reads table enabling/disabling states. + public String clientTableZNode; + // znode where the master writes table disabling/enabling states in the format expected + // by 0.92.0/0.92.1 clients for backwards compatibility. See HBASE-6710 for details. + public String masterTableZNode92; + // znode containing the unique cluster ID + public String clusterIdZNode; + // znode used for log splitting work assignment + public String splitLogZNode; + + // Certain ZooKeeper nodes need to be world-readable + public static final ArrayList CREATOR_ALL_AND_WORLD_READABLE = + new ArrayList() { { + add(new ACL(ZooDefs.Perms.READ,ZooDefs.Ids.ANYONE_ID_UNSAFE)); + add(new ACL(ZooDefs.Perms.ALL,ZooDefs.Ids.AUTH_IDS)); + }}; + + private final Configuration conf; + + private final Exception constructorCaller; + + /** + * Instantiate a ZooKeeper connection and watcher. + * @param descriptor Descriptive string that is added to zookeeper sessionid + * and used as identifier for this instance. + * @throws IOException + * @throws ZooKeeperConnectionException + */ + public ZooKeeperWatcher(Configuration conf, String descriptor, + Abortable abortable) throws ZooKeeperConnectionException, IOException { + this(conf, descriptor, abortable, false); + } + /** + * Instantiate a ZooKeeper connection and watcher. + * @param descriptor Descriptive string that is added to zookeeper sessionid + * and used as identifier for this instance. + * @throws IOException + * @throws ZooKeeperConnectionException + */ + public ZooKeeperWatcher(Configuration conf, String descriptor, + Abortable abortable, boolean canCreateBaseZNode) + throws IOException, ZooKeeperConnectionException { + this.conf = conf; + // Capture a stack trace now. Will print it out later if problem so we can + // distingush amongst the myriad ZKWs. + try { + throw new Exception("ZKW CONSTRUCTOR STACK TRACE FOR DEBUGGING"); + } catch (Exception e) { + this.constructorCaller = e; + } + this.quorum = ZKConfig.getZKQuorumServersString(conf); + // Identifier will get the sessionid appended later below down when we + // handle the syncconnect event. + this.identifier = descriptor; + this.abortable = abortable; + setNodeNames(conf); + this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, this, descriptor); + if (canCreateBaseZNode) { + createBaseZNodes(); + } + } + + private void createBaseZNodes() throws ZooKeeperConnectionException { + try { + // Create all the necessary "directories" of znodes + ZKUtil.createAndFailSilent(this, baseZNode); + ZKUtil.createAndFailSilent(this, assignmentZNode); + ZKUtil.createAndFailSilent(this, rsZNode); + ZKUtil.createAndFailSilent(this, drainingZNode); + ZKUtil.createAndFailSilent(this, masterTableZNode); + ZKUtil.createAndFailSilent(this, masterTableZNode92); + ZKUtil.createAndFailSilent(this, splitLogZNode); + ZKUtil.createAndFailSilent(this, backupMasterAddressesZNode); + } catch (KeeperException e) { + throw new ZooKeeperConnectionException( + prefix("Unexpected KeeperException creating base node"), e); + } + } + + private boolean isFinishedRetryingRecoverable(final long finished) { + return System.currentTimeMillis() < finished; + } + + @Override + public String toString() { + return this.identifier; + } + + /** + * Adds this instance's identifier as a prefix to the passed str + * @param str String to amend. + * @return A new string with this instance's identifier as prefix: e.g. + * if passed 'hello world', the returned string could be + */ + public String prefix(final String str) { + return this.toString() + " " + str; + } + + /** + * Set the local variable node names using the specified configuration. + */ + private void setNodeNames(Configuration conf) { + baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, + HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); + rootServerZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.rootserver", "root-region-server")); + rsZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.rs", "rs")); + drainingZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.draining.rs", "draining")); + masterAddressZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.master", "master")); + backupMasterAddressesZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.backup.masters", "backup-masters")); + clusterStateZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.state", "shutdown")); + assignmentZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.unassigned", "unassigned")); + String tableZNodeDefault = "table"; + masterTableZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.masterTableEnableDisable", tableZNodeDefault)); + clientTableZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.clientTableEnableDisable", tableZNodeDefault)); + masterTableZNode92 = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.masterTableEnableDisable92", "table92")); + clusterIdZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.clusterId", "hbaseid")); + splitLogZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.splitlog", HConstants.SPLIT_LOGDIR_NAME)); + } + + /** + * Register the specified listener to receive ZooKeeper events. + * @param listener + */ + public void registerListener(ZooKeeperListener listener) { + listeners.add(listener); + } + + /** + * Register the specified listener to receive ZooKeeper events and add it as + * the first in the list of current listeners. + * @param listener + */ + public void registerListenerFirst(ZooKeeperListener listener) { + listeners.add(0, listener); + } + + /** + * Clean all existing listeners + */ + public void unregisterAllListeners() { + listeners.clear(); + } + + /** + * Get a copy of current registered listeners + */ + public List getListeners() { + return new ArrayList(listeners); + } + + /** + * @return The number of currently registered listeners + */ + public int getNumberOfListeners() { + return listeners.size(); + } + + /** + * Get the connection to ZooKeeper. + * @return connection reference to zookeeper + */ + public RecoverableZooKeeper getRecoverableZooKeeper() { + return recoverableZooKeeper; + } + + public void reconnectAfterExpiration() throws IOException, InterruptedException { + recoverableZooKeeper.reconnectAfterExpiration(); + } + + /** + * Get the quorum address of this instance. + * @return quorum string of this zookeeper connection instance + */ + public String getQuorum() { + return quorum; + } + + /** + * Method called from ZooKeeper for events and connection status. + *

    + * Valid events are passed along to listeners. Connection status changes + * are dealt with locally. + */ + @Override + public void process(WatchedEvent event) { + LOG.debug(prefix("Received ZooKeeper Event, " + + "type=" + event.getType() + ", " + + "state=" + event.getState() + ", " + + "path=" + event.getPath())); + + switch(event.getType()) { + + // If event type is NONE, this is a connection status change + case None: { + connectionEvent(event); + break; + } + + // Otherwise pass along to the listeners + + case NodeCreated: { + for(ZooKeeperListener listener : listeners) { + listener.nodeCreated(event.getPath()); + } + break; + } + + case NodeDeleted: { + for(ZooKeeperListener listener : listeners) { + listener.nodeDeleted(event.getPath()); + } + break; + } + + case NodeDataChanged: { + for(ZooKeeperListener listener : listeners) { + listener.nodeDataChanged(event.getPath()); + } + break; + } + + case NodeChildrenChanged: { + for(ZooKeeperListener listener : listeners) { + listener.nodeChildrenChanged(event.getPath()); + } + break; + } + } + } + + // Connection management + + /** + * Called when there is a connection-related event via the Watcher callback. + *

    + * If Disconnected or Expired, this should shutdown the cluster. But, since + * we send a KeeperException.SessionExpiredException along with the abort + * call, it's possible for the Abortable to catch it and try to create a new + * session with ZooKeeper. This is what the client does in HCM. + *

    + * @param event + */ + private void connectionEvent(WatchedEvent event) { + switch(event.getState()) { + case SyncConnected: + // Now, this callback can be invoked before the this.zookeeper is set. + // Wait a little while. + long finished = System.currentTimeMillis() + + this.conf.getLong("hbase.zookeeper.watcher.sync.connected.wait", 2000); + while (System.currentTimeMillis() < finished) { + Threads.sleep(1); + if (this.recoverableZooKeeper != null) break; + } + if (this.recoverableZooKeeper == null) { + LOG.error("ZK is null on connection event -- see stack trace " + + "for the stack trace when constructor was called on this zkw", + this.constructorCaller); + throw new NullPointerException("ZK is null"); + } + this.identifier = this.identifier + "-0x" + + Long.toHexString(this.recoverableZooKeeper.getSessionId()); + // Update our identifier. Otherwise ignore. + LOG.debug(this.identifier + " connected"); + break; + + // Abort the server if Disconnected or Expired + case Disconnected: + LOG.debug(prefix("Received Disconnected from ZooKeeper, ignoring")); + break; + + case Expired: + String msg = prefix(this.identifier + " received expired from " + + "ZooKeeper, aborting"); + // TODO: One thought is to add call to ZooKeeperListener so say, + // ZooKeeperNodeTracker can zero out its data values. + if (this.abortable != null) this.abortable.abort(msg, + new KeeperException.SessionExpiredException()); + break; + } + } + + /** + * Forces a synchronization of this ZooKeeper client connection. + *

    + * Executing this method before running other methods will ensure that the + * subsequent operations are up-to-date and consistent as of the time that + * the sync is complete. + *

    + * This is used for compareAndSwap type operations where we need to read the + * data of an existing node and delete or transition that node, utilizing the + * previously read version and data. We want to ensure that the version read + * is up-to-date from when we begin the operation. + */ + public void sync(String path) { + this.recoverableZooKeeper.sync(path, null, null); + } + + /** + * Handles KeeperExceptions in client calls. + *

    + * This may be temporary but for now this gives one place to deal with these. + *

    + * TODO: Currently this method rethrows the exception to let the caller handle + *

    + * @param ke + * @throws KeeperException + */ + public void keeperException(KeeperException ke) + throws KeeperException { + LOG.error(prefix("Received unexpected KeeperException, re-throwing exception"), ke); + throw ke; + } + + /** + * Handles InterruptedExceptions in client calls. + *

    + * This may be temporary but for now this gives one place to deal with these. + *

    + * TODO: Currently, this method does nothing. + * Is this ever expected to happen? Do we abort or can we let it run? + * Maybe this should be logged as WARN? It shouldn't happen? + *

    + * @param ie + */ + public void interruptedException(InterruptedException ie) { + LOG.debug(prefix("Received InterruptedException, doing nothing here"), ie); + // At least preserver interrupt. + Thread.currentThread().interrupt(); + // no-op + } + + /** + * Close the connection to ZooKeeper. + * @throws InterruptedException + */ + public void close() { + try { + if (recoverableZooKeeper != null) { + recoverableZooKeeper.close(); +// super.close(); + } + } catch (InterruptedException e) { + } + } + + public Configuration getConfiguration() { + return conf; + } + + @Override + public void abort(String why, Throwable e) { + this.abortable.abort(why, e); + } + + @Override + public boolean isAborted() { + return this.abortable.isAborted(); + } +} diff --git a/src/main/javadoc/org/apache/hadoop/hbase/io/hfile/package.html b/src/main/javadoc/org/apache/hadoop/hbase/io/hfile/package.html new file mode 100644 index 0000000..fa9244f --- /dev/null +++ b/src/main/javadoc/org/apache/hadoop/hbase/io/hfile/package.html @@ -0,0 +1,25 @@ + + + + + + + +Provides the hbase data+index+metadata file. + + diff --git a/src/main/javadoc/org/apache/hadoop/hbase/ipc/package.html b/src/main/javadoc/org/apache/hadoop/hbase/ipc/package.html new file mode 100644 index 0000000..0e01bdc --- /dev/null +++ b/src/main/javadoc/org/apache/hadoop/hbase/ipc/package.html @@ -0,0 +1,24 @@ + + + + + +Tools to help define network clients and servers. +This is the hadoop copied local so can fix bugs and make hbase-specific optimizations. + + diff --git a/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html b/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html new file mode 100644 index 0000000..d61becb --- /dev/null +++ b/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html @@ -0,0 +1,163 @@ + + + + + + + +

    Multi Cluster Replication

    +This package provides replication between HBase clusters. +

    + +

    Table Of Contents

    +
      +
    1. Status
      2. +
    2. Requirements
      3. +
    3. Deployment
      4. +
    4. Verifying Replicated Data
      5. +
    + +

    + +

    Status

    + +

    +This package is experimental quality software and is only meant to be a base +for future developments. The current implementation offers the following +features: + +

      +
    1. Master/Slave replication.
      2. +
    2. Master/Master replication.
      3. +
    3. Cyclic replication.
      4. +
    4. Replication of scoped families in user tables.
      5. +
    5. Start/stop replication stream.
      6. +
    6. Supports clusters of different sizes.
      7. +
    7. Handling of partitions longer than 10 minutes.
      8. +
    8. Ability to add/remove slave clusters at runtime.
      9. +
    9. MapReduce job to compare tables on two clusters
      10. +
    +Please report bugs on the project's Jira when found. +

    + +

    Requirements

    + +

    + +Before trying out replication, make sure to review the following requirements: + +

      +
    1. Zookeeper should be handled by yourself, not by HBase, and should + always be available during the deployment.
      2. +
    2. All machines from both clusters should be able to reach every + other machine since replication goes from any region server to any + other one on the slave cluster. That also includes the + Zookeeper clusters.
      3. +
    3. Both clusters should have the same HBase and Hadoop major revision. + For example, having 0.90.1 on the master and 0.90.0 on the slave is + correct but not 0.90.1 and 0.89.20100725.
      4. +
    4. Every table that contains families that are scoped for replication + should exist on every cluster with the exact same name, same for those + replicated families.
      5. +
    5. For multiple slaves, Master/Master, or cyclic replication version + 0.92 or greater is needed.
      6. +
    + +

    + +

    Deployment

    + +

    + +The following steps describe how to enable replication from a cluster +to another. +

      +
    1. Edit ${HBASE_HOME}/conf/hbase-site.xml on both cluster to add + the following configurations: + 
      +<property>
+  <name>hbase.replication</name>
+  <value>true</value>
+</property>
      + deploy the files, and then restart HBase if it was running. +
      2. +
    2. Run the following command in the master's shell while it's running + 
      add_peer
      + This will show you the help to setup the replication stream between + both clusters. If both clusters use the same Zookeeper cluster, you have + to use a different zookeeper.znode.parent since they can't + write in the same folder. +
      3. +
    3. + Once you have a peer, you need to enable replication on your column families. + One way to do it is to alter the table and to set the scope like this: + 
      +      disable 'your_table'
+      alter 'your_table', {NAME => 'family_name', REPLICATION_SCOPE => '1'}
+      enable 'your_table'
+
      + Currently, a scope of 0 (default) means that it won't be replicated and a + scope of 1 means it's going to be. In the future, different scope can be + used for routing policies. +
      4. +
    4. To list all configured peers run the following command in the master's + shell + 
      list_peers
      (as of version 0.92) +
      5. +
    + +You can confirm that your setup works by looking at any region server's log +on the master cluster and look for the following lines; + +
    +Considering 1 rs, with ratio 0.1
+Getting 1 rs from peer cluster # 0
+Choosing peer 10.10.1.49:62020
    + +In this case it indicates that 1 region server from the slave cluster +was chosen for replication.

    + +Should you want to stop the replication while the clusters are running, open +the shell on the master cluster and issue this command: +
    +hbase(main):001:0> stop_replication
    + +Replication of already queued edits will still happen after you +issued that command but new entries won't be. To start it back, simply replace +"false" with "true" in the command. + +

    + + +

    Verifying Replicated Data

    + +

    +Verifying the replicated data on two clusters is easy to do in the shell when +looking only at a few rows, but doing a systematic comparison requires more +computing power. This is why the VerifyReplication MR job was created, it has +to be run on the master cluster and needs to be provided with a peer id (the +one provided when establishing a replication stream) and a table name. Other +options let you specify a time range and specific families. This job's short +name is "verifyrep" and needs to be provided when pointing "hadoop jar" to the +hbase jar. +

    + + + diff --git a/src/main/javadoc/org/apache/hadoop/hbase/thrift/doc-files/Hbase.html b/src/main/javadoc/org/apache/hadoop/hbase/thrift/doc-files/Hbase.html new file mode 100644 index 0000000..5cee1da --- /dev/null +++ b/src/main/javadoc/org/apache/hadoop/hbase/thrift/doc-files/Hbase.html @@ -0,0 +1,627 @@ + + + +Thrift module: Hbase +

    Thrift module: Hbase

    +
    + + + + +
    ModuleServicesData typesConstants
    HbaseHbase
    + +    		AlreadyExists
    +BatchMutation
    +Bytes
    +ColumnDescriptor
    +IOError
    +IllegalArgument
    +Mutation
    +ScannerID
    +TCell
    +TRegionInfo
    +TRowResult
    +Text
    +
    +

    Type declarations

    +

    Typedef: Text

    +

    Base type: string

    +
    +

    Typedef: Bytes

    +

    Base type: string

    +
    +

    Typedef: ScannerID

    +

    Base type: i32

    +
    +

    Data structures

    +

    Struct: TCell

    + + + +
    FieldTypeDescriptionRequiredDefault value
    valueBytesyes
    timestampi64yes

    TCell - Used to transport a cell value (byte[]) and the timestamp it was +stored with together as a result for get and getRow methods. This promotes +the timestamp of a cell to a first-class value, making it easy to take +note of temporal data. Cell is used all the way from HStore up to HTable. +

    Struct: ColumnDescriptor

    + + + + + + + + + + +
    FieldTypeDescriptionRequiredDefault value
    nameTextyes
    maxVersionsi32yes3
    compressionstringyes"NONE"
    inMemoryboolyes0
    bloomFilterTypestringyes"NONE"
    bloomFilterVectorSizei32yes0
    bloomFilterNbHashesi32yes0
    blockCacheEnabledboolyes0
    timeToLivei32yes-1

    An HColumnDescriptor contains information about a column family +such as the number of versions, compression settings, etc. It is +used as input when creating a table or adding a column. +

    Struct: TRegionInfo

    + + + + + + +
    FieldTypeDescriptionRequiredDefault value
    startKeyTextyes
    endKeyTextyes
    idi64yes
    nameTextyes
    versionbyteyes

    A TRegionInfo contains information about an HTable region. +

    Struct: Mutation

    + + + + +
    FieldTypeDescriptionRequiredDefault value
    isDeleteboolyes0
    columnTextyes
    valueTextyes

    A Mutation object is used to either update or delete a column-value. +

    Struct: BatchMutation

    + + + +
    FieldTypeDescriptionRequiredDefault value
    rowTextyes
    mutationslist<Mutation>yes

    A BatchMutation object is used to apply a number of Mutations to a single row. +

    Struct: TRowResult

    + + + +
    FieldTypeDescriptionRequiredDefault value
    rowTextyes
    columnsmap<Text, TCell>yes

    Holds row name and then a map of columns to cells. +

    Exception: IOError

    + + +
    FieldTypeDescriptionRequiredDefault value
    messagestringyes

    An IOError exception signals that an error occurred communicating +to the Hbase master or an Hbase region server. Also used to return +more general Hbase error conditions. +

    Exception: IllegalArgument

    + + +
    FieldTypeDescriptionRequiredDefault value
    messagestringyes

    An IllegalArgument exception indicates an illegal or invalid +argument was passed into a procedure. +

    Exception: AlreadyExists

    + + +
    FieldTypeDescriptionRequiredDefault value
    messagestringyes

    An AlreadyExists exceptions signals that a table with the specified +name already exists +

    Services

    +

    Service: Hbase

    +

    Function: Hbase.enableTable

    +
    void enableTable(Bytes tableName)
+    throws IOError
+
    Brings a table on-line (enables it) +

    Parameters
    + +
    NameDescription
    tableNamename of the table +

    +

    Function: Hbase.disableTable

    +
    void disableTable(Bytes tableName)
+    throws IOError
+
    Disables a table (takes it off-line) If it is being served, the master +will tell the servers to stop serving it. +

    Parameters
    + +
    NameDescription
    tableNamename of the table +

    +

    Function: Hbase.isTableEnabled

    +
    bool isTableEnabled(Bytes tableName)
+    throws IOError
+
    @return true if table is on-line +

    Parameters
    + +
    NameDescription
    tableNamename of the table to check +

    +

    Function: Hbase.compact

    +
    void compact(Bytes tableNameOrRegionName)
+    throws IOError
+

    Parameters
    + +
    NameDescription
    tableNameOrRegionName

    +

    Function: Hbase.majorCompact

    +
    void majorCompact(Bytes tableNameOrRegionName)
+    throws IOError
+

    Parameters
    + +
    NameDescription
    tableNameOrRegionName

    +

    Function: Hbase.getTableNames

    +
    list<Text> getTableNames()
+    throws IOError
+
    List all the userspace tables. +

    +@return returns a list of names +

    Function: Hbase.getColumnDescriptors

    +
    map<Text, ColumnDescriptor> getColumnDescriptors(Text tableName)
+    throws IOError
+
    List all the column families assoicated with a table. +

    +@return list of column family descriptors +

    Parameters
    + +
    NameDescription
    tableNametable name +

    +

    Function: Hbase.getTableRegions

    +
    list<TRegionInfo> getTableRegions(Text tableName)
+    throws IOError
+
    List the regions associated with a table. +

    +@return list of region descriptors +

    Parameters
    + +
    NameDescription
    tableNametable name +

    +

    Function: Hbase.createTable

    +
    void createTable(Text tableName,
+                 list<ColumnDescriptor> columnFamilies)
+    throws IOError, IllegalArgument, AlreadyExists
+
    Create a table with the specified column families. The name +field for each ColumnDescriptor must be set and must end in a +colon (:). All other fields are optional and will get default +values if not explicitly specified. +

    +@throws IllegalArgument if an input parameter is invalid +

    +@throws AlreadyExists if the table name already exists +

    Parameters
    + +
    NameDescription
    tableNamename of table to create +
    columnFamilieslist of column family descriptors +

    +

    Function: Hbase.deleteTable

    +
    void deleteTable(Text tableName)
+    throws IOError
+
    Deletes a table +

    +@throws IOError if table doesn't exist on server or there was some other +problem +

    Parameters
    + +
    NameDescription
    tableNamename of table to delete +

    +

    Function: Hbase.get

    +
    list<TCell> get(Text tableName,
+                Text row,
+                Text column)
+    throws IOError
+
    Get a single TCell for the specified table, row, and column at the +latest timestamp. Returns an empty list if no such value exists. +

    +@return value for specified row/column +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowrow key +
    columncolumn name +

    +

    Function: Hbase.getVer

    +
    list<TCell> getVer(Text tableName,
+                   Text row,
+                   Text column,
+                   i32 numVersions)
+    throws IOError
+
    Get the specified number of versions for the specified table, +row, and column. +

    +@return list of cells for specified row/column +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowrow key +
    columncolumn name +
    numVersionsnumber of versions to retrieve +

    +

    Function: Hbase.getVerTs

    +
    list<TCell> getVerTs(Text tableName,
+                     Text row,
+                     Text column,
+                     i64 timestamp,
+                     i32 numVersions)
+    throws IOError
+
    Get the specified number of versions for the specified table, +row, and column. Only versions less than or equal to the specified +timestamp will be returned. +

    +@return list of cells for specified row/column +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowrow key +
    columncolumn name +
    timestamptimestamp +
    numVersionsnumber of versions to retrieve +

    +

    Function: Hbase.getRow

    +
    list<TRowResult> getRow(Text tableName,
+                        Text row)
+    throws IOError
+
    Get all the data for the specified table and row at the latest +timestamp. Returns an empty list if the row does not exist. +

    +@return TRowResult containing the row and map of columns to TCells +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowrow key +

    +

    Function: Hbase.getRowWithColumns

    +
    list<TRowResult> getRowWithColumns(Text tableName,
+                                   Text row,
+                                   list<Text> columns)
+    throws IOError
+
    Get the specified columns for the specified table and row at the latest +timestamp. Returns an empty list if the row does not exist. +

    +@return TRowResult containing the row and map of columns to TCells +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowrow key +
    columnsList of columns to return, null for all columns +

    +

    Function: Hbase.getRowTs

    +
    list<TRowResult> getRowTs(Text tableName,
+                          Text row,
+                          i64 timestamp)
+    throws IOError
+
    Get all the data for the specified table and row at the specified +timestamp. Returns an empty list if the row does not exist. +

    +@return TRowResult containing the row and map of columns to TCells +

    Parameters
    + +
    NameDescription
    tableNamename of the table +
    rowrow key +
    timestamptimestamp +

    +

    Function: Hbase.getRowWithColumnsTs

    +
    list<TRowResult> getRowWithColumnsTs(Text tableName,
+                                     Text row,
+                                     list<Text> columns,
+                                     i64 timestamp)
+    throws IOError
+
    Get the specified columns for the specified table and row at the specified +timestamp. Returns an empty list if the row does not exist. +

    +@return TRowResult containing the row and map of columns to TCells +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowrow key +
    columnsList of columns to return, null for all columns +
    timestamp

    +

    Function: Hbase.mutateRow

    +
    void mutateRow(Text tableName,
+               Text row,
+               list<Mutation> mutations)
+    throws IOError, IllegalArgument
+
    Apply a series of mutations (updates/deletes) to a row in a +single transaction. If an exception is thrown, then the +transaction is aborted. Default current timestamp is used, and +all entries will have an identical timestamp. +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowrow key +
    mutationslist of mutation commands +

    +

    Function: Hbase.mutateRowTs

    +
    void mutateRowTs(Text tableName,
+                 Text row,
+                 list<Mutation> mutations,
+                 i64 timestamp)
+    throws IOError, IllegalArgument
+
    Apply a series of mutations (updates/deletes) to a row in a +single transaction. If an exception is thrown, then the +transaction is aborted. The specified timestamp is used, and +all entries will have an identical timestamp. +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowrow key +
    mutationslist of mutation commands +
    timestamptimestamp +

    +

    Function: Hbase.mutateRows

    +
    void mutateRows(Text tableName,
+                list<BatchMutation> rowBatches)
+    throws IOError, IllegalArgument
+
    Apply a series of batches (each a series of mutations on a single row) +in a single transaction. If an exception is thrown, then the +transaction is aborted. Default current timestamp is used, and +all entries will have an identical timestamp. +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowBatcheslist of row batches +

    +

    Function: Hbase.mutateRowsTs

    +
    void mutateRowsTs(Text tableName,
+                  list<BatchMutation> rowBatches,
+                  i64 timestamp)
+    throws IOError, IllegalArgument
+
    Apply a series of batches (each a series of mutations on a single row) +in a single transaction. If an exception is thrown, then the +transaction is aborted. The specified timestamp is used, and +all entries will have an identical timestamp. +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowBatcheslist of row batches +
    timestamptimestamp +

    +

    Function: Hbase.atomicIncrement

    +
    i64 atomicIncrement(Text tableName,
+                    Text row,
+                    Text column,
+                    i64 value)
+    throws IOError, IllegalArgument
+
    Atomically increment the column value specified. Returns the next value post increment. +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowrow to increment +
    columnname of column +
    valueamount to increment by +

    +

    Function: Hbase.deleteAll

    +
    void deleteAll(Text tableName,
+               Text row,
+               Text column)
+    throws IOError
+
    Delete all cells that match the passed row and column. +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowRow to update +
    columnname of column whose value is to be deleted +

    +

    Function: Hbase.deleteAllTs

    +
    void deleteAllTs(Text tableName,
+                 Text row,
+                 Text column,
+                 i64 timestamp)
+    throws IOError
+
    Delete all cells that match the passed row and column and whose +timestamp is equal-to or older than the passed timestamp. +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowRow to update +
    columnname of column whose value is to be deleted +
    timestamptimestamp +

    +

    Function: Hbase.deleteAllRow

    +
    void deleteAllRow(Text tableName,
+                  Text row)
+    throws IOError
+
    Completely delete the row's cells. +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowkey of the row to be completely deleted. +

    +

    Function: Hbase.deleteAllRowTs

    +
    void deleteAllRowTs(Text tableName,
+                    Text row,
+                    i64 timestamp)
+    throws IOError
+
    Completely delete the row's cells marked with a timestamp +equal-to or older than the passed timestamp. +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    rowkey of the row to be completely deleted. +
    timestamptimestamp +

    +

    Function: Hbase.scannerOpen

    +
    ScannerID scannerOpen(Text tableName,
+                      Text startRow,
+                      list<Text> columns)
+    throws IOError
+
    Get a scanner on the current table starting at the specified row and +ending at the last row in the table. Return the specified columns. +

    +@return scanner id to be used with other scanner procedures +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    startRowStarting row in table to scan. +Send "" (empty string) to start at the first row. +
    columnscolumns to scan. If column name is a column family, all +columns of the specified column family are returned. It's also possible +to pass a regex in the column qualifier. +

    +

    Function: Hbase.scannerOpenWithStop

    +
    ScannerID scannerOpenWithStop(Text tableName,
+                              Text startRow,
+                              Text stopRow,
+                              list<Text> columns)
+    throws IOError
+
    Get a scanner on the current table starting and stopping at the +specified rows. ending at the last row in the table. Return the +specified columns. +

    +@return scanner id to be used with other scanner procedures +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    startRowStarting row in table to scan. +Send "" (empty string) to start at the first row. +
    stopRowrow to stop scanning on. This row is *not* included in the +scanner's results +
    columnscolumns to scan. If column name is a column family, all +columns of the specified column family are returned. It's also possible +to pass a regex in the column qualifier. +

    +

    Function: Hbase.scannerOpenWithPrefix

    +
    ScannerID scannerOpenWithPrefix(Text tableName,
+                                Text startAndPrefix,
+                                list<Text> columns)
+    throws IOError
+
    Open a scanner for a given prefix. That is all rows will have the specified +prefix. No other rows will be returned. +

    +@return scanner id to use with other scanner calls +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    startAndPrefixthe prefix (and thus start row) of the keys you want +
    columnsthe columns you want returned +

    +

    Function: Hbase.scannerOpenTs

    +
    ScannerID scannerOpenTs(Text tableName,
+                        Text startRow,
+                        list<Text> columns,
+                        i64 timestamp)
+    throws IOError
+
    Get a scanner on the current table starting at the specified row and +ending at the last row in the table. Return the specified columns. +Only values with the specified timestamp are returned. +

    +@return scanner id to be used with other scanner procedures +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    startRowStarting row in table to scan. +Send "" (empty string) to start at the first row. +
    columnscolumns to scan. If column name is a column family, all +columns of the specified column family are returned. It's also possible +to pass a regex in the column qualifier. +
    timestamptimestamp +

    +

    Function: Hbase.scannerOpenWithStopTs

    +
    ScannerID scannerOpenWithStopTs(Text tableName,
+                                Text startRow,
+                                Text stopRow,
+                                list<Text> columns,
+                                i64 timestamp)
+    throws IOError
+
    Get a scanner on the current table starting and stopping at the +specified rows. ending at the last row in the table. Return the +specified columns. Only values with the specified timestamp are +returned. +

    +@return scanner id to be used with other scanner procedures +

    Parameters
    + +
    NameDescription
    tableNamename of table +
    startRowStarting row in table to scan. +Send "" (empty string) to start at the first row. +
    stopRowrow to stop scanning on. This row is *not* included in the +scanner's results +
    columnscolumns to scan. If column name is a column family, all +columns of the specified column family are returned. It's also possible +to pass a regex in the column qualifier. +
    timestamptimestamp +

    +

    Function: Hbase.scannerGet

    +
    list<TRowResult> scannerGet(ScannerID id)
+    throws IOError, IllegalArgument
+
    Returns the scanner's current row value and advances to the next +row in the table. When there are no more rows in the table, or a key +greater-than-or-equal-to the scanner's specified stopRow is reached, +an empty list is returned. +

    +@return a TRowResult containing the current row and a map of the columns to TCells. +

    +@throws IllegalArgument if ScannerID is invalid +

    +@throws NotFound when the scanner reaches the end +

    Parameters
    + +
    NameDescription
    idid of a scanner returned by scannerOpen +

    +

    Function: Hbase.scannerGetList

    +
    list<TRowResult> scannerGetList(ScannerID id,
+                                i32 nbRows)
+    throws IOError, IllegalArgument
+
    Returns, starting at the scanner's current row value nbRows worth of +rows and advances to the next row in the table. When there are no more +rows in the table, or a key greater-than-or-equal-to the scanner's +specified stopRow is reached, an empty list is returned. +

    +@return a TRowResult containing the current row and a map of the columns to TCells. +

    +@throws IllegalArgument if ScannerID is invalid +

    +@throws NotFound when the scanner reaches the end +

    Parameters
    + +
    NameDescription
    idid of a scanner returned by scannerOpen +
    nbRowsnumber of results to return +

    +

    Function: Hbase.scannerClose

    +
    void scannerClose(ScannerID id)
+    throws IOError, IllegalArgument
+
    Closes the server-state associated with an open scanner. +

    +@throws IllegalArgument if ScannerID is invalid +

    Parameters
    + +
    NameDescription
    idid of a scanner returned by scannerOpen +

    +

    diff --git a/src/main/javadoc/org/apache/hadoop/hbase/thrift/doc-files/index.html b/src/main/javadoc/org/apache/hadoop/hbase/thrift/doc-files/index.html new file mode 100644 index 0000000..2f37fe1 --- /dev/null +++ b/src/main/javadoc/org/apache/hadoop/hbase/thrift/doc-files/index.html @@ -0,0 +1,79 @@ + + + +All Thrift declarations +

    All Thrift declarations

    + + + + + +
    ModuleServicesData typesConstants
    HbaseHbase
    + +    		AlreadyExists
    +BatchMutation
    +Bytes
    +ColumnDescriptor
    +IOError
    +IllegalArgument
    +Mutation
    +ScannerID
    +TCell
    +TRegionInfo
    +TRowResult
    +Text
    +
    + diff --git a/src/main/javadoc/org/apache/hadoop/hbase/thrift/doc-files/style.css b/src/main/javadoc/org/apache/hadoop/hbase/thrift/doc-files/style.css new file mode 100644 index 0000000..d494718 --- /dev/null +++ b/src/main/javadoc/org/apache/hadoop/hbase/thrift/doc-files/style.css @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* Auto-generated CSS for generated Thrift docs */ +body { font-family: Tahoma, sans-serif; } +pre { background-color: #dddddd; padding: 6px; } +h3,h4 { padding-top: 0px; margin-top: 0px; } +div.definition { border: 1px solid gray; margin: 10px; padding: 10px; } +div.extends { margin: -0.5em 0 1em 5em } +table { border: 1px solid grey; border-collapse: collapse; } +td { border: 1px solid grey; padding: 1px 6px; vertical-align: top; } +th { border: 1px solid black; background-color: #bbbbbb; + text-align: left; padding: 1px 6px; } diff --git a/src/main/javadoc/org/apache/hadoop/hbase/thrift/package.html b/src/main/javadoc/org/apache/hadoop/hbase/thrift/package.html new file mode 100644 index 0000000..b82a98a --- /dev/null +++ b/src/main/javadoc/org/apache/hadoop/hbase/thrift/package.html @@ -0,0 +1,109 @@ + + + + + + + +Provides an HBase Thrift +service. + +This directory contains a Thrift interface definition file for an HBase RPC +service and a Java server implementation. + + +

    What is Thrift?

    +

    "Thrift is a software framework for scalable cross-language services development. +It combines a software stack with a code generation engine to build services +that work efficiently and seamlessly between C++, Java, Python, PHP, Ruby, +Erlang, Perl, Haskell, C#, Cocoa, Smalltalk, and OCaml."
    + +

    Description

    + +

    Important note: We tried to deprecate this Thrift interface and replace it +with the Interface defined over in the thrift2 package only this package will not die. +Folks keep adding to it and fixing it up so its around for another while until someone +takes command and drives this package out of existence replacing it w/ an Interface that +better matches the hbase API (this package was modelled on old HBase API long since dropped). +

    + +

    The {@link org.apache.hadoop.hbase.thrift.generated.Hbase.Iface HBase API} is defined in the +file Hbase.thrift (Click the former to see the +thrift generated documentation of thrift interface). A server-side implementation of the API is in +{@link org.apache.hadoop.hbase.thrift.ThriftServer}. The generated interfaces, +types, and RPC utility files reside in the +{@link org.apache.hadoop.hbase.thrift.generated} package. +

    + +

    To start ThriftServer, use: +

    +  ./bin/hbase-daemon.sh start thrift
+
    + +

    To stop, use: +

    +  ./bin/hbase-daemon.sh stop thrift
+
    + +These are the command line arguments the Thrift server understands in addition to start and stop: +
    +
    -b, --bind
    +
    Address to bind the Thrift server to. Not supported by the Nonblocking and HsHa server [default: 0.0.0.0]
    + +
    -p, --port
    +
    Port to bind to [default: 9090]
    + +
    -f, --framed
    +
    Use framed transport (implied when using one of the non-blocking servers)
    + +
    -c, --compact
    +
    Use the compact protocol [default: binary protocol]
    + +
    -h, --help
    +
    Displays usage information for the Thrift server
    + +
    -threadpool
    +
    Use the TThreadPoolServer. This is the default.
    + +
    -hsha
    +
    Use the THsHaServer. This implies the framed transport.
    + +
    -nonblocking
    +
    Use the TNonblockingServer. This implies the framed transport.
    +
    + +

    Details

    + +

    HBase currently uses version 0.8.0 of Apache Thrift.

    + +

    The files were generated by running the commands under the hbase checkout dir: +

    +  thrift -strict --gen java:hashcode ./src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
+  # Move the generated files into place their expected location under hbase
+  mv gen-java/org/apache/hadoop/hbase/thrift/generated/* src/main/java/org/apache/hadoop/hbase/thrift/generated/
+  # Remove the gen-java file made by thrift
+  rm -rf gen-java
+
    + +

    The 'thrift' binary is the Thrift compiler, and it is distributed as a part +of the Thrift package. Additionally, specific language runtime libraries are a +part of the Thrift package. A version of the Java runtime is included in HBase +via Maven.

    + + + diff --git a/src/main/javadoc/overview.html b/src/main/javadoc/overview.html new file mode 100644 index 0000000..e79d715 --- /dev/null +++ b/src/main/javadoc/overview.html @@ -0,0 +1,57 @@ + + + + + + + HBase + + +HBase is a scalable, distributed database built on Hadoop Core. + +

    Table of Contents

    + + + +

    Getting Started

    +

    See the Getting Started +section of the HBase Book. +

    + +

    Example API Usage

    +

    For sample Java code, see org.apache.hadoop.hbase.client documentation.

    + +

    If your client is NOT Java, consider the Thrift or REST libraries.

    + +

    Related Documentation

    + + + + diff --git a/src/main/protobuf/ErrorHandling.proto b/src/main/protobuf/ErrorHandling.proto new file mode 100644 index 0000000..53c4165 --- /dev/null +++ b/src/main/protobuf/ErrorHandling.proto @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are used for error handling + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "ErrorHandlingProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +/** + * Protobuf version of a java.lang.StackTraceElement + * so we can serialize exceptions. + */ +message StackTraceElementMessage { + optional string declaringClass = 1; + optional string methodName = 2; + optional string fileName = 3; + optional int32 lineNumber = 4; +} + +/** + * Cause of a remote failure for a generic exception. Contains + * all the information for a generic exception as well as + * optional info about the error for generic info passing + * (which should be another protobuffed class). + */ +message GenericExceptionMessage { + optional string className = 1; + optional string message = 2; + optional bytes errorInfo = 3; + repeated StackTraceElementMessage trace = 4; +} + +/** + * Exception sent across the wire when a remote task needs + * to notify other tasks that it failed and why + */ +message ForeignExceptionMessage { + optional string source = 1; + optional GenericExceptionMessage genericException = 2; + +} diff --git a/src/main/protobuf/hbase.proto b/src/main/protobuf/hbase.proto new file mode 100644 index 0000000..93d4232 --- /dev/null +++ b/src/main/protobuf/hbase.proto @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are shared throughout HBase + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "HBaseProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +/** + * Description of the snapshot to take + */ +message SnapshotDescription { + required string name = 1; + optional string table = 2; // not needed for delete, but checked for in taking snapshot + optional int64 creationTime = 3 [default = 0]; + enum Type { + DISABLED = 0; + FLUSH = 1; + } + optional Type type = 4 [default = FLUSH]; + optional int32 version = 5; +} diff --git a/src/main/python/hbase/merge_conf.py b/src/main/python/hbase/merge_conf.py new file mode 100644 index 0000000..764d98a --- /dev/null +++ b/src/main/python/hbase/merge_conf.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +Merges Hadoop/HBase configuration files in the given order, so that options +specified in later configuration files override those specified in earlier +files. +''' + +import os +import re +import sys +import textwrap + +from optparse import OptionParser +from xml.dom.minidom import parse, getDOMImplementation + + +class MergeConfTool: + ''' + Merges the given set of Hadoop/HBase configuration files, with later files + overriding earlier ones. + ''' + + INDENT = ' ' * 2 + + # Description text is inside configuration, property, and description tags. + DESC_INDENT = INDENT * 3 + + def main(self): + '''The main entry point for the configuration merge tool.''' + self.parse_options() + self.merge() + + def parse_options(self): + '''Parses command-line options.''' + parser = OptionParser(usage='%prog -o ') + parser.add_option('-o', '--output_file', + help='Destination configuration file') + opts, input_files = parser.parse_args() + if not opts.output_file: + self.fatal('--output_file is not specified') + if not input_files: + self.fatal('No input files specified') + for f_path in input_files: + if not os.path.isfile(f_path): + self.fatal('Input file %s does not exist' % f_path) + self.input_files = input_files + self.output_file = opts.output_file + + def merge(self): + '''Merges input configuration files into the output file.''' + values = {} # Conf key to values + source_files = {} # Conf key to the file name where the value came from + descriptions = {} # Conf key to description (optional) + + # Read input files in the given order and update configuration maps + for f_path in self.input_files: + self.current_file = f_path + f_basename = os.path.basename(f_path) + f_dom = parse(f_path) + for property in f_dom.getElementsByTagName('property'): + self.current_property = property + name = self.element_text('name') + value = self.element_text('value') + values[name] = value + source_files[name] = f_basename + + if property.getElementsByTagName('description'): + descriptions[name] = self.element_text('description') + + # Create the output configuration file + dom_impl = getDOMImplementation() + self.merged_conf = dom_impl.createDocument(None, 'configuration', None) + for k in sorted(values.keys()): + new_property = self.merged_conf.createElement('property') + c = self.merged_conf.createComment('from ' + source_files[k]) + new_property.appendChild(c) + self.append_text_child(new_property, 'name', k) + self.append_text_child(new_property, 'value', values[k]) + + description = descriptions.get(k, None) + if description: + description = ' '.join(description.strip().split()) + textwrap_kwargs = {} + if sys.version_info >= (2, 6): + textwrap_kwargs = dict(break_on_hyphens=False) + description = ('\n' + self.DESC_INDENT).join( + textwrap.wrap(description, 80 - len(self.DESC_INDENT), + break_long_words=False, **textwrap_kwargs)) + self.append_text_child(new_property, 'description', description) + self.merged_conf.documentElement.appendChild(new_property) + + pretty_conf = self.merged_conf.toprettyxml(indent=self.INDENT) + + # Remove space before and after names and values. This way we don't have + # to worry about leading and trailing whitespace creeping in. + pretty_conf = re.sub(r'(?<=)\s*', '', pretty_conf) + pretty_conf = re.sub(r'(?<=)\s*', '', pretty_conf) + pretty_conf = re.sub(r'\s*(?=)', '', pretty_conf) + pretty_conf = re.sub(r'\s*(?=)', '', pretty_conf) + + out_f = open(self.output_file, 'w') + try: + out_f.write(pretty_conf) + finally: + out_f.close() + + def element_text(self, tag_name): + return self.whole_text(self.only_element(tag_name)) + + def fatal(self, msg): + print >> sys.stderr, msg + sys.exit(1) + + def only_element(self, tag_name): + l = self.current_property.getElementsByTagName(tag_name) + if len(l) != 1: + self.fatal('Invalid property in %s, only one ' + '"%s" element expected: %s' % (self.current_file, tag_name, + self.current_property.toxml())) + return l[0] + + def whole_text(self, element): + if len(element.childNodes) > 1: + self.fatal('No more than one child expected in %s: %s' % ( + self.current_file, element.toxml())) + if len(element.childNodes) == 1: + return element.childNodes[0].wholeText.strip() + return '' + + def append_text_child(self, property_element, tag_name, value): + element = self.merged_conf.createElement(tag_name) + element.appendChild(self.merged_conf.createTextNode(value)) + property_element.appendChild(element) + + +if __name__ == '__main__': + MergeConfTool().main() + diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml new file mode 100644 index 0000000..2c402a8 --- /dev/null +++ b/src/main/resources/hbase-default.xml @@ -0,0 +1,975 @@ + + + + + + hbase.rootdir + file:///tmp/hbase-${user.name}/hbase + The directory shared by region servers and into + which HBase persists. The URL should be 'fully-qualified' + to include the filesystem scheme. For example, to specify the + HDFS directory '/hbase' where the HDFS instance's namenode is + running at namenode.example.org on port 9000, set this value to: + hdfs://namenode.example.org:9000/hbase. By default HBase writes + into /tmp. Change this configuration else all data will be lost + on machine restart. + + + + hbase.master.port + 60000 + The port the HBase Master should bind to. + + + hbase.cluster.distributed + false + The mode the cluster will be in. Possible values are + false for standalone mode and true for distributed mode. If + false, startup will run all HBase and ZooKeeper daemons together + in the one JVM. + + + + hbase.tmp.dir + ${java.io.tmpdir}/hbase-${user.name} + Temporary directory on the local filesystem. + Change this setting to point to a location more permanent + than '/tmp' (The '/tmp' directory is often cleared on + machine restart). + + + + hbase.local.dir + ${hbase.tmp.dir}/local/ + Directory on the local filesystem to be used + as a local storage. + + + + hbase.master.info.port + 60010 + The port for the HBase Master web UI. + Set to -1 if you do not want a UI instance run. + + + + hbase.master.info.bindAddress + 0.0.0.0 + The bind address for the HBase Master web UI + + + + hbase.client.write.buffer + 2097152 + Default size of the HTable clien write buffer in bytes. + A bigger buffer takes more memory -- on both the client and server + side since server instantiates the passed write buffer to process + it -- but a larger buffer size reduces the number of RPCs made. + For an estimate of server-side memory-used, evaluate + hbase.client.write.buffer * hbase.regionserver.handler.count + + + + hbase.regionserver.port + 60020 + The port the HBase RegionServer binds to. + + + + hbase.regionserver.info.port + 60030 + The port for the HBase RegionServer web UI + Set to -1 if you do not want the RegionServer UI to run. + + + + hbase.regionserver.info.port.auto + false + Whether or not the Master or RegionServer + UI should search for a port to bind to. Enables automatic port + search if hbase.regionserver.info.port is already in use. + Useful for testing, turned off by default. + + + + hbase.regionserver.info.bindAddress + 0.0.0.0 + The address for the HBase RegionServer web UI + + + + hbase.regionserver.class + org.apache.hadoop.hbase.ipc.HRegionInterface + The RegionServer interface to use. + Used by the client opening proxy to remote region server. + + + + hbase.client.pause + 1000 + General client pause value. Used mostly as value to wait + before running a retry of a failed get, region lookup, etc. + + + hbase.client.retries.number + 10 + Maximum retries. Used as maximum for all retryable + operations such as fetching of the root region from root region + server, getting a cell's value, starting a row update, etc. + Default: 10. + + + + hbase.bulkload.retries.number + 0 + Maximum retries. This is maximum number of iterations + to atomic bulk loads are attempted in the face of splitting operations + 0 means never give up. Default: 0. + + + + hbase.client.scanner.caching + 1 + Number of rows that will be fetched when calling next + on a scanner if it is not served from (local, client) memory. Higher + caching values will enable faster scanners but will eat up more memory + and some calls of next may take longer and longer times when the cache is empty. + Do not set this value such that the time between invocations is greater + than the scanner timeout; i.e. hbase.regionserver.lease.period + + + + hbase.client.keyvalue.maxsize + 10485760 + Specifies the combined maximum allowed size of a KeyValue + instance. This is to set an upper boundary for a single entry saved in a + storage file. Since they cannot be split it helps avoiding that a region + cannot be split any further because the data is too large. It seems wise + to set this to a fraction of the maximum region size. Setting it to zero + or less disables the check. + + + + hbase.regionserver.lease.period + 60000 + HRegion server lease period in milliseconds. Default is + 60 seconds. Clients must report in within this period else they are + considered dead. + + + hbase.regionserver.handler.count + 10 + Count of RPC Listener instances spun up on RegionServers. + Same property is used by the Master for count of master handlers. + Default is 10. + + + + hbase.regionserver.msginterval + 3000 + Interval between messages from the RegionServer to Master + in milliseconds. + + + + hbase.regionserver.optionallogflushinterval + 1000 + Sync the HLog to the HDFS after this interval if it has not + accumulated enough entries to trigger a sync. Default 1 second. Units: + milliseconds. + + + + hbase.regionserver.regionSplitLimit + 2147483647 + Limit for the number of regions after which no more region + splitting should take place. This is not a hard limit for the number of + regions but acts as a guideline for the regionserver to stop splitting after + a certain limit. Default is set to MAX_INT; i.e. do not block splitting. + + + + hbase.regionserver.logroll.period + 3600000 + Period at which we will roll the commit log regardless + of how many edits it has. + + + hbase.regionserver.logroll.errors.tolerated + 2 + The number of consecutive WAL close errors we will allow + before triggering a server abort. A setting of 0 will cause the + region server to abort if closing the current WAL writer fails during + log rolling. Even a small value (2 or 3) will allow a region server + to ride over transient HDFS errors. + + + hbase.regionserver.hlog.reader.impl + org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogReader + The HLog file reader implementation. + + + hbase.regionserver.hlog.writer.impl + org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter + The HLog file writer implementation. + + + hbase.regionserver.separate.hlog.for.meta + false + + + hbase.regionserver.nbreservationblocks + 4 + The number of resevoir blocks of memory release on + OOME so we can cleanup properly before server shutdown. + + + + hbase.zookeeper.dns.interface + default + The name of the Network Interface from which a ZooKeeper server + should report its IP address. + + + + hbase.zookeeper.dns.nameserver + default + The host name or IP address of the name server (DNS) + which a ZooKeeper server should use to determine the host name used by the + master for communication and display purposes. + + + + hbase.regionserver.dns.interface + default + The name of the Network Interface from which a region server + should report its IP address. + + + + hbase.regionserver.dns.nameserver + default + The host name or IP address of the name server (DNS) + which a region server should use to determine the host name used by the + master for communication and display purposes. + + + + hbase.master.dns.interface + default + The name of the Network Interface from which a master + should report its IP address. + + + + hbase.master.dns.nameserver + default + The host name or IP address of the name server (DNS) + which a master should use to determine the host name used + for communication and display purposes. + + + + hbase.balancer.period + + 300000 + Period at which the region balancer runs in the Master. + + + + hbase.regions.slop + 0.2 + Rebalance if any regionserver has average + (average * slop) regions. + Default is 20% slop. + + + + hbase.master.logcleaner.ttl + 600000 + Maximum time a HLog can stay in the .oldlogdir directory, + after which it will be cleaned by a Master thread. + + + + hbase.master.logcleaner.plugins + org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner + A comma-separated list of LogCleanerDelegate invoked by + the LogsCleaner service. These WAL/HLog cleaners are called in order, + so put the HLog cleaner that prunes the most HLog files in front. To + implement your own LogCleanerDelegate, just put it in HBase's classpath + and add the fully qualified class name here. Always add the above + default log cleaners in the list. + + + + hbase.regionserver.global.memstore.upperLimit + 0.4 + Maximum size of all memstores in a region server before new + updates are blocked and flushes are forced. Defaults to 40% of heap + + + + hbase.regionserver.global.memstore.lowerLimit + 0.35 + When memstores are being forced to flush to make room in + memory, keep flushing until we hit this mark. Defaults to 35% of heap. + This value equal to hbase.regionserver.global.memstore.upperLimit causes + the minimum possible flushing to occur when updates are blocked due to + memstore limiting. + + + + hbase.server.thread.wakefrequency + 10000 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as log roller. + + + + hbase.server.versionfile.writeattempts + 3 + + How many time to retry attempting to write a version file + before just aborting. Each attempt is seperated by the + hbase.server.thread.wakefrequency milliseconds. + + + + hbase.regionserver.optionalcacheflushinterval + 3600000 + + Maximum amount of time an edit lives in memory before being automatically flushed. + Default 1 hour. Set it to 0 to disable automatic flushing. + + + + hbase.hregion.memstore.flush.size + 134217728 + + Memstore will be flushed to disk if size of the memstore + exceeds this number of bytes. Value is checked by a thread that runs + every hbase.server.thread.wakefrequency. + + + + hbase.hregion.preclose.flush.size + 5242880 + + If the memstores in a region are this size or larger when we go + to close, run a "pre-flush" to clear out memstores before we put up + the region closed flag and take the region offline. On close, + a flush is run under the close flag to empty memory. During + this time the region is offline and we are not taking on any writes. + If the memstore content is large, this flush could take a long time to + complete. The preflush is meant to clean out the bulk of the memstore + before putting up the close flag and taking the region offline so the + flush that runs under the close flag has little to do. + + + + hbase.hregion.memstore.block.multiplier + 2 + + Block updates if memstore has hbase.hregion.block.memstore + time hbase.hregion.flush.size bytes. Useful preventing + runaway memstore during spikes in update traffic. Without an + upper-bound, memstore fills such that when it flushes the + resultant flush files take a long time to compact or split, or + worse, we OOME. + + + + hbase.hregion.memstore.mslab.enabled + true + + Enables the MemStore-Local Allocation Buffer, + a feature which works to prevent heap fragmentation under + heavy write loads. This can reduce the frequency of stop-the-world + GC pauses on large heaps. + + + + hbase.hregion.max.filesize + 10737418240 + + Maximum HStoreFile size. If any one of a column families' HStoreFiles has + grown to exceed this value, the hosting HRegion is split in two. + Default: 10G. + + + + hbase.hstore.compactionThreshold + 3 + + If more than this number of HStoreFiles in any one HStore + (one HStoreFile is written per flush of memstore) then a compaction + is run to rewrite all HStoreFiles files as one. Larger numbers + put off compaction but when it runs, it takes longer to complete. + + + + hbase.hstore.blockingStoreFiles + 7 + + If more than this number of StoreFiles in any one Store + (one StoreFile is written per flush of MemStore) then updates are + blocked for this HRegion until a compaction is completed, or + until hbase.hstore.blockingWaitTime has been exceeded. + + + + hbase.hstore.blockingWaitTime + 90000 + + The time an HRegion will block updates for after hitting the StoreFile + limit defined by hbase.hstore.blockingStoreFiles. + After this time has elapsed, the HRegion will stop blocking updates even + if a compaction has not been completed. Default: 90 seconds. + + + + hbase.hstore.compaction.max + 10 + Max number of HStoreFiles to compact per 'minor' compaction. + + + + hbase.hregion.majorcompaction + 86400000 + The time (in miliseconds) between 'major' compactions of all + HStoreFiles in a region. Default: 1 day. + Set to 0 to disable automated major compactions. + + + + hbase.mapreduce.hfileoutputformat.blocksize + 65536 + The mapreduce HFileOutputFormat writes storefiles/hfiles. + This is the minimum hfile blocksize to emit. Usually in hbase, writing + hfiles, the blocksize is gotten from the table schema (HColumnDescriptor) + but in the mapreduce outputformat context, we don't have access to the + schema so get blocksize from Configuration. The smaller you make + the blocksize, the bigger your index and the less you fetch on a + random-access. Set the blocksize down if you have small cells and want + faster random-access of individual cells. + + + + hfile.block.cache.size + 0.25 + + Percentage of maximum heap (-Xmx setting) to allocate to block cache + used by HFile/StoreFile. Default of 0.25 means allocate 25%. + Set to 0 to disable but it's not recommended. + + + + hbase.hash.type + murmur + The hashing algorithm for use in HashFunction. Two values are + supported now: murmur (MurmurHash) and jenkins (JenkinsHash). + Used by bloom filters. + + + + hfile.block.index.cacheonwrite + false + + This allows to put non-root multi-level index blocks into the block + cache at the time the index is being written. + + + + hbase.regionserver.checksum.verify + false + + Allow hbase to do checksums rather than using hdfs checksums. This is a backwards + incompatible change. + + + + hfile.index.block.max.size + 131072 + + When the size of a leaf-level, intermediate-level, or root-level + index block in a multi-level block index grows to this size, the + block is written out and a new block is started. + + + + hfile.format.version + 2 + + The HFile format version to use for new files. Set this to 1 to test + backwards-compatibility. The default value of this option should be + consistent with FixedFileTrailer.MAX_VERSION. + + + + io.storefile.bloom.block.size + 131072 + + The size in bytes of a single block ("chunk") of a compound Bloom + filter. This size is approximate, because Bloom blocks can only be + inserted at data block boundaries, and the number of keys per data + block varies. + + + + io.storefile.bloom.cacheonwrite + false + + Enables cache-on-write for inline blocks of a compound Bloom filter. + + + + hbase.rs.cacheblocksonwrite + false + + Whether an HFile block should be added to the block cache when the + block is finished. + + + + hbase.rpc.engine + org.apache.hadoop.hbase.ipc.WritableRpcEngine + Implementation of org.apache.hadoop.hbase.ipc.RpcEngine to be + used for client / server RPC call marshalling. + + + + + + hbase.master.keytab.file + + Full path to the kerberos keytab file to use for logging in + the configured HMaster server principal. + + + + hbase.master.kerberos.principal + + Ex. "hbase/_HOST@EXAMPLE.COM". The kerberos principal name + that should be used to run the HMaster process. The principal name should + be in the form: user/hostname@DOMAIN. If "_HOST" is used as the hostname + portion, it will be replaced with the actual hostname of the running + instance. + + + + hbase.regionserver.keytab.file + + Full path to the kerberos keytab file to use for logging in + the configured HRegionServer server principal. + + + + hbase.regionserver.kerberos.principal + + Ex. "hbase/_HOST@EXAMPLE.COM". The kerberos principal name + that should be used to run the HRegionServer process. The principal name + should be in the form: user/hostname@DOMAIN. If "_HOST" is used as the + hostname portion, it will be replaced with the actual hostname of the + running instance. An entry for this principal must exist in the file + specified in hbase.regionserver.keytab.file + + + + + + hadoop.policy.file + hbase-policy.xml + The policy configuration file used by RPC servers to make + authorization decisions on client requests. Only used when HBase + security is enabled. + + + + hbase.superuser + + List of users or groups (comma-separated), who are allowed + full privileges, regardless of stored ACLs, across the cluster. + Only used when HBase security is enabled. + + + + hbase.auth.key.update.interval + 86400000 + The update interval for master key for authentication tokens + in servers in milliseconds. Only used when HBase security is enabled. + + + + hbase.auth.token.max.lifetime + 604800000 + The maximum lifetime in milliseconds after which an + authentication token expires. Only used when HBase security is enabled. + + + + + zookeeper.session.timeout + 180000 + ZooKeeper session timeout. + HBase passes this to the zk quorum as suggested maximum time for a + session (This setting becomes zookeeper's 'maxSessionTimeout'). See + http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions + "The client sends a requested timeout, the server responds with the + timeout that it can give the client. " In milliseconds. + + + + zookeeper.znode.parent + /hbase + Root ZNode for HBase in ZooKeeper. All of HBase's ZooKeeper + files that are configured with a relative path will go under this node. + By default, all of HBase's ZooKeeper file path are configured with a + relative path, so they will all go under this directory unless changed. + + + + zookeeper.znode.rootserver + root-region-server + Path to ZNode holding root region location. This is written by + the master and read by clients and region servers. If a relative path is + given, the parent folder will be ${zookeeper.znode.parent}. By default, + this means the root location is stored at /hbase/root-region-server. + + + + + zookeeper.znode.acl.parent + acl + Root ZNode for access control lists. + + + + hbase.coprocessor.region.classes + + A comma-separated list of Coprocessors that are loaded by + default on all tables. For any override coprocessor method, these classes + will be called in order. After implementing your own Coprocessor, just put + it in HBase's classpath and add the fully qualified class name here. + A coprocessor can also be loaded on demand by setting HTableDescriptor. + + + + + hbase.coprocessor.master.classes + + A comma-separated list of + org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are + loaded by default on the active HMaster process. For any implemented + coprocessor methods, the listed classes will be called in order. After + implementing your own MasterObserver, just put it in HBase's classpath + and add the fully qualified class name here. + + + + + + hbase.zookeeper.quorum + localhost + Comma separated list of servers in the ZooKeeper Quorum. + For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com". + By default this is set to localhost for local and pseudo-distributed modes + of operation. For a fully-distributed setup, this should be set to a full + list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh + this is the list of servers which we will start/stop ZooKeeper on. + + + + hbase.zookeeper.peerport + 2888 + Port used by ZooKeeper peers to talk to each other. + See http://hadoop.apache.org/zookeeper/docs/r3.1.1/zookeeperStarted.html#sc_RunningReplicatedZooKeeper + for more information. + + + + hbase.zookeeper.leaderport + 3888 + Port used by ZooKeeper for leader election. + See http://hadoop.apache.org/zookeeper/docs/r3.1.1/zookeeperStarted.html#sc_RunningReplicatedZooKeeper + for more information. + + + + hbase.zookeeper.useMulti + false + Instructs HBase to make use of ZooKeeper's multi-update functionality. + This allows certain ZooKeeper operations to complete more quickly and prevents some issues + with rare ZooKeeper failure scenarios (see the release note of HBASE-6710 for an example). + IMPORTANT: only set this to true if all ZooKeeper servers in the cluster are on version 3.4+ + and will not be downgraded. ZooKeeper versions before 3.4 do not support multi-update and will + not fail gracefully if multi-update is invoked (see ZOOKEEPER-1495). + + + + + + + hbase.zookeeper.property.initLimit + 10 + Property from ZooKeeper's config zoo.cfg. + The number of ticks that the initial synchronization phase can take. + + + + hbase.zookeeper.property.syncLimit + 5 + Property from ZooKeeper's config zoo.cfg. + The number of ticks that can pass between sending a request and getting an + acknowledgment. + + + + hbase.zookeeper.property.dataDir + ${hbase.tmp.dir}/zookeeper + Property from ZooKeeper's config zoo.cfg. + The directory where the snapshot is stored. + + + + hbase.zookeeper.property.clientPort + 2181 + Property from ZooKeeper's config zoo.cfg. + The port at which the clients will connect. + + + + hbase.zookeeper.property.maxClientCnxns + 300 + Property from ZooKeeper's config zoo.cfg. + Limit on number of concurrent connections (at the socket level) that a + single client, identified by IP address, may make to a single member of + the ZooKeeper ensemble. Set high to avoid zk connection issues running + standalone and pseudo-distributed. + + + + + hbase.rest.port + 8080 + The port for the HBase REST server. + + + hbase.rest.readonly + false + + Defines the mode the REST server will be started in. Possible values are: + false: All HTTP methods are permitted - GET/PUT/POST/DELETE. + true: Only the GET method is permitted. + + + + + hbase.defaults.for.version + @@@VERSION@@@ + + This defaults file was compiled for version @@@VERSION@@@. This variable is used + to make sure that a user doesn't have an old version of hbase-default.xml on the + classpath. + + + + hbase.defaults.for.version.skip + false + + Set to true to skip the 'hbase.defaults.for.version' check. + Setting this to true can be useful in contexts other than + the other side of a maven generation; i.e. running in an + ide. You'll want to set this boolean to true to avoid + seeing the RuntimException complaint: "hbase-default.xml file + seems to be for and old version of HBase (@@@VERSION@@@), this + version is X.X.X-SNAPSHOT" + + + + hbase.coprocessor.abortonerror + false + + Set to true to cause the hosting server (master or regionserver) to + abort if a coprocessor throws a Throwable object that is not IOException or + a subclass of IOException. Setting it to true might be useful in development + environments where one wants to terminate the server as soon as possible to + simplify coprocessor failure analysis. + + + + hbase.online.schema.update.enable + false + + Set true to enable online schema changes. This is an experimental feature. + There are known issues modifying table schemas at the same time a region + split is happening so your table needs to be quiescent or else you have to + be running with splits disabled. + + + + dfs.support.append + true + Does HDFS allow appends to files? + This is an hdfs config. set in here so the hdfs client will do append support. + You must ensure that this config. is true serverside too when running hbase + (You will have to restart your cluster after setting it). + + + + hbase.thrift.minWorkerThreads + 16 + + The "core size" of the thread pool. New threads are created on every + connection until this many threads are created. + + + + hbase.thrift.maxWorkerThreads + 1000 + + The maximum size of the thread pool. When the pending request queue + overflows, new threads are created until their number reaches this number. + After that, the server starts dropping connections. + + + + hbase.thrift.maxQueuedRequests + 1000 + + The maximum number of pending Thrift connections waiting in the queue. If + there are no idle threads in the pool, the server queues requests. Only + when the queue overflows, new threads are added, up to + hbase.thrift.maxQueuedRequests threads. + + + + hbase.offheapcache.percentage + 0 + + The amount of off heap space to be allocated towards the experimental + off heap cache. If you desire the cache to be disabled, simply set this + value to 0. + + + + hbase.data.umask.enable + false + Enable, if true, that file permissions should be assigned + to the files written by the regionserver + + + + hbase.data.umask + 000 + File permissions that should be used to write data + files when hbase.data.umask.enable is true + + + + + hbase.metrics.showTableName + true + Whether to include the prefix "tbl.tablename" in per-column family metrics. + If true, for each metric M, per-cf metrics will be reported for tbl.T.cf.CF.M, if false, + per-cf metrics will be aggregated by column-family across tables, and reported for cf.CF.M. + In both cases, the aggregated metric M across tables and cfs will be reported. + + + + hbase.use.secondary.index + false + Enable this property when you are using secondary index. + + + + hbase.index.half.storefile.reader.class + org.apache.hadoop.hbase.index.io.IndexHalfStoreFileReader + + Do not change this. It is used internally when secondary + index is enabled + + + + hbase.index.loadbalancer.class + + org.apache.hadoop.hbase.index.SecIndexLoadBalancer + + Do not change this. It is used internally when secondary + index is enabled. + + + + hbase.table.archive.directory + .archive + Per-table directory name under which to backup files for a + table. Files are moved to the same directories as they would be under the + table directory, but instead are just one level lower (under + table/.archive/... rather than table/...). Currently only applies to HFiles. + + + hbase.master.hfilecleaner.plugins + org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner + A comma-separated list of HFileCleanerDelegate invoked by + the HFileCleaner service. These HFiles cleaners are called in order, + so put the cleaner that prunes the most files in front. To + implement your own HFileCleanerDelegate, just put it in HBase's classpath + and add the fully qualified class name here. Always add the above + default log cleaners in the list as they will be overwritten in hbase-site.xml. + + + + hbase.rest.threads.max + 100 + + The maximum number of threads of the REST server thread pool. + Threads in the pool are reused to process REST requests. This + controls the maximum number of requests processed concurrently. + It may help to control the memory used by the REST server to + avoid OOM issues. If the thread pool is full, incoming requests + will be queued up and wait for some free threads. The default + is 100. + + + + hbase.rest.threads.min + 2 + + The minimum number of threads of the REST server thread pool. + The thread pool always has at least these number of threads so + the REST server is ready to serve incoming requests. The default + is 2. + + + diff --git a/src/main/resources/hbase-webapps/master/index.html b/src/main/resources/hbase-webapps/master/index.html new file mode 100644 index 0000000..0f05414 --- /dev/null +++ b/src/main/resources/hbase-webapps/master/index.html @@ -0,0 +1,20 @@ + + diff --git a/src/main/resources/hbase-webapps/master/master.jsp b/src/main/resources/hbase-webapps/master/master.jsp new file mode 100644 index 0000000..6651c93 --- /dev/null +++ b/src/main/resources/hbase-webapps/master/master.jsp @@ -0,0 +1,20 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> + diff --git a/src/main/resources/hbase-webapps/master/snapshot.jsp b/src/main/resources/hbase-webapps/master/snapshot.jsp new file mode 100644 index 0000000..ab007e9 --- /dev/null +++ b/src/main/resources/hbase-webapps/master/snapshot.jsp @@ -0,0 +1,189 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="java.util.Date" + import="java.util.HashMap" + import="org.apache.hadoop.conf.Configuration" + import="org.apache.hadoop.hbase.client.HBaseAdmin" + import="org.apache.hadoop.hbase.client.HConnectionManager" + import="org.apache.hadoop.hbase.HRegionInfo" + import="org.apache.hadoop.hbase.master.HMaster" + import="org.apache.hadoop.hbase.util.Bytes" + import="org.apache.hadoop.hbase.util.FSUtils" + import="org.apache.hadoop.hbase.protobuf.ProtobufUtil" + import="org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription" + import="org.apache.hadoop.hbase.snapshot.SnapshotInfo" + import="org.apache.hadoop.util.StringUtils" + import="java.util.List" + import="java.util.Map" + import="org.apache.hadoop.hbase.HConstants"%><% + HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER); + Configuration conf = master.getConfiguration(); + HBaseAdmin hbadmin = new HBaseAdmin(conf); + boolean readOnly = conf.getBoolean("hbase.master.ui.readonly", false); + String snapshotName = request.getParameter("name"); + SnapshotDescription snapshot = null; + SnapshotInfo.SnapshotStats stats = null; + for (SnapshotDescription snapshotDesc: hbadmin.listSnapshots()) { + if (snapshotName.equals(snapshotDesc.getName())) { + snapshot = snapshotDesc; + stats = SnapshotInfo.getSnapshotStats(conf, snapshot); + break; + } + } + + String action = request.getParameter("action"); + String cloneName = request.getParameter("cloneName"); + boolean isActionResultPage = (!readOnly && action != null); +%> + + + + + + +<% if (isActionResultPage) { %> + HBase Master: <%= master.getServerName() %> + +<% } else { %> + Snapshot: <%= snapshotName %> +<% } %> + + + +<% if (isActionResultPage) { %> +

    Snapshot action request...

    +<% + if (action.equals("restore")) { + hbadmin.restoreSnapshot(snapshotName); + %> Restore Snapshot request accepted. <% + } else if (action.equals("clone")) { + if (cloneName != null && cloneName.length() > 0) { + hbadmin.cloneSnapshot(snapshotName, cloneName); + %> Clone from Snapshot request accepted. <% + } else { + %> Clone from Snapshot request failed, No table name specified. <% + } + } +%> +

    Go Back, or wait for the redirect. + +<% } else if (snapshot == null) { %> +

    Snapshot "<%= snapshotName %>" does not exists

    + +
    +

    Go Back, or wait for the redirect. +<% } else { %> +

    Snapshot: <%= snapshotName %>

    + +
    +

    Snapshot Attributes

    + + + + + + + + + + + + + + <% if (stats.isSnapshotCorrupted()) { %> + + <% } else { %> + + <% } %> + +
    TableCreation TimeTypeFormat VersionState
    <%= snapshot.getTable() %><%= new Date(snapshot.getCreationTime()) %><%= snapshot.getType() %><%= snapshot.getVersion() %>CORRUPTEDok
    +

    + <%= stats.getStoreFilesCount() %> HFiles (<%= stats.getArchivedStoreFilesCount() %> in archive), + total size <%= StringUtils.humanReadableInt(stats.getStoreFilesSize()) %> + (<%= stats.getSharedStoreFilePercentage() %>% + <%= StringUtils.humanReadableInt(stats.getSharedStoreFilesSize()) %> shared with the source + table) +

    +

    + <%= stats.getLogsCount() %> Logs, total size + <%= StringUtils.humanReadableInt(stats.getLogsSize()) %> +

    + <% if (stats.isSnapshotCorrupted()) { %> +

    CORRUPTED Snapshot

    +

    + <%= stats.getMissingStoreFilesCount() %> hfile(s) and + <%= stats.getMissingLogsCount() %> log(s) missing. +

    + <% } %> +<% + } // end else + +HConnectionManager.deleteConnection(hbadmin.getConfiguration()); +%> + + +<% if (!readOnly && action == null && snapshot != null) { %> +

    +Actions: +

    +

    + + + + + + + + + + + + + + + + + + + + + + +
    +  New Table Name (clone): + This action will create a new table by cloning the snapshot content. + There are no copies of data involved. + And writing on the newly created table will not influence the snapshot data. +
     
    +   Restore a specified snapshot. + The restore will replace the content of the original table, + bringing back the content to the snapshot state. + The table must be disabled.
    +
    +

    + +<% } %> + + diff --git a/src/main/resources/hbase-webapps/master/table.jsp b/src/main/resources/hbase-webapps/master/table.jsp new file mode 100644 index 0000000..a2e2010 --- /dev/null +++ b/src/main/resources/hbase-webapps/master/table.jsp @@ -0,0 +1,285 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="java.util.HashMap" + import="org.apache.hadoop.io.Writable" + import="org.apache.hadoop.conf.Configuration" + import="org.apache.hadoop.hbase.client.HTable" + import="org.apache.hadoop.hbase.client.HBaseAdmin" + import="org.apache.hadoop.hbase.client.HConnectionManager" + import="org.apache.hadoop.hbase.HRegionInfo" + import="org.apache.hadoop.hbase.ServerName" + import="org.apache.hadoop.hbase.HServerAddress" + import="org.apache.hadoop.hbase.ServerName" + import="org.apache.hadoop.hbase.HServerInfo" + import="org.apache.hadoop.hbase.HServerLoad" + import="org.apache.hadoop.hbase.HServerLoad.RegionLoad" + import="org.apache.hadoop.hbase.io.ImmutableBytesWritable" + import="org.apache.hadoop.hbase.master.HMaster" + import="org.apache.hadoop.hbase.util.Bytes" + import="org.apache.hadoop.hbase.util.FSUtils" + import="java.util.Map" + import="org.apache.hadoop.hbase.HConstants"%><% + HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER); + Configuration conf = master.getConfiguration(); + HBaseAdmin hbadmin = new HBaseAdmin(conf); + String tableName = request.getParameter("name"); + HTable table = new HTable(conf, tableName); + String tableHeader = "

    Table Regions

    "; + ServerName rl = master.getCatalogTracker().getRootLocation(); + boolean showFragmentation = conf.getBoolean("hbase.master.ui.fragmentation.enabled", false); + boolean readOnly = conf.getBoolean("hbase.master.ui.readonly", false); + Map frags = null; + if (showFragmentation) { + frags = FSUtils.getTableFragmentation(master); + } + // HARDCODED FOR NOW TODO: FIX GET FROM ZK + // This port might be wrong if RS actually ended up using something else. + int infoPort = conf.getInt("hbase.regionserver.info.port", 60030); +%> + + + + + +<% + String action = request.getParameter("action"); + String key = request.getParameter("key"); + if ( !readOnly && action != null ) { +%> + + + + + + +

    Table action request accepted

    +

    +<% + if (action.equals("split")) { + if (key != null && key.length() > 0) { + hbadmin.split(key); + } else { + hbadmin.split(tableName); + } + + %> Split request accepted. <% + } else if (action.equals("compact")) { + if (key != null && key.length() > 0) { + hbadmin.compact(key); + } else { + hbadmin.compact(tableName); + } + %> Compact request accepted. <% + } +%> +

    Go Back, or wait for the redirect. + + +<% +} else { +%> + +Table: <%= tableName %> + + + + +

    Table: <%= tableName %>

    + +
    +<% + if(tableName.equals(Bytes.toString(HConstants.ROOT_TABLE_NAME))) { +%> +<%= tableHeader %> +<% + String url = "http://" + rl.getHostname() + ":" + infoPort + "/"; +%> +     + + + + + + +
    NameRegion ServerStart KeyEnd KeyRequests
    <%= tableName %><%= rl.getHostname() %>:<%= rl.getPort() %>--
    +<% + } else if(tableName.equals(Bytes.toString(HConstants.META_TABLE_NAME))) { +%> +<%= tableHeader %> +<% + // NOTE: Presumes one meta region only. + HRegionInfo meta = HRegionInfo.FIRST_META_REGIONINFO; + ServerName metaLocation = master.getCatalogTracker().waitForMeta(1); + for (int i = 0; i < 1; i++) { + String url = "http://" + metaLocation.getHostname() + ":" + infoPort + "/"; +%> + + <%= meta.getRegionNameAsString() %> + <%= metaLocation.getHostname().toString() + ":" + infoPort %> + -<%= Bytes.toString(meta.getStartKey()) %><%= Bytes.toString(meta.getEndKey()) %> + +<% } %> + +<%} else { + try { %> +

    Table Attributes

    + + + + + + + + + + + + + + + +<% if (showFragmentation) { %> + + + + + +<% } %> +
    Attribute NameValueDescription
    Enabled<%= hbadmin.isTableEnabled(table.getTableName()) %>Is the table enabled
    Compaction<%= hbadmin.getCompactionState(table.getTableName()) %>Is the table compacting
    Fragmentation<%= frags.get(tableName) != null ? frags.get(tableName).intValue() + "%" : "n/a" %>How fragmented is the table. After a major compaction it is 0%.
    +<% + Map regDistribution = new HashMap(); + Map regions = table.getRegionLocations(); + if(regions != null && regions.size() > 0) { %> +<%= tableHeader %> +<% + for (Map.Entry hriEntry : regions.entrySet()) { + HRegionInfo regionInfo = hriEntry.getKey(); + ServerName addr = hriEntry.getValue(); + long req = 0; + + String urlRegionServer = null; + + if (addr != null) { + HServerLoad sl = master.getServerManager().getLoad(addr); + if (sl != null) { + Map map = sl.getRegionsLoad(); + if (map.containsKey(regionInfo.getRegionName())) { + req = map.get(regionInfo.getRegionName()).getRequestsCount(); + } + // This port might be wrong if RS actually ended up using something else. + urlRegionServer = + "http://" + addr.getHostname().toString() + ":" + infoPort + "/"; + Integer i = regDistribution.get(urlRegionServer); + if (null == i) i = new Integer(0); + regDistribution.put(urlRegionServer, i+1); + } + } +%> + + <%= Bytes.toStringBinary(regionInfo.getRegionName())%> + <% + if (urlRegionServer != null) { + %> + + <%= addr.getHostname().toString() + ":" + infoPort %> + + <% + } else { + %> + not deployed + <% + } + %> + <%= Bytes.toStringBinary(regionInfo.getStartKey())%> + <%= Bytes.toStringBinary(regionInfo.getEndKey())%> + <%= req%> + +<% } %> + +

    Regions by Region Server

    + +<% + for (Map.Entry rdEntry : regDistribution.entrySet()) { +%> + + + + +<% } %> +
    Region ServerRegion Count
    <%= rdEntry.getKey()%><%= rdEntry.getValue()%>
    +<% } +} catch(Exception ex) { + ex.printStackTrace(System.err); +} +} // end else + +HConnectionManager.deleteConnection(hbadmin.getConfiguration()); +%> + + +<% if (!readOnly) { %> +

    +Actions: +

    +

    + + + + + + + + + + + + + + + + + + + + + + +
    +  Region Key (optional):This action will force a compaction of all + regions of the table, or, if a key is supplied, only the region containing the + given key.
     
    +  Region Key (optional):This action will force a split of all eligible + regions of the table, or, if a key is supplied, only the region containing the + given key. An eligible region is one that does not contain any references to + other regions. Split requests for noneligible regions will be ignored.
    +
    +

    +<% } %> +<% +} +%> + + + diff --git a/src/main/resources/hbase-webapps/master/tablesDetailed.jsp b/src/main/resources/hbase-webapps/master/tablesDetailed.jsp new file mode 100644 index 0000000..ecab99d --- /dev/null +++ b/src/main/resources/hbase-webapps/master/tablesDetailed.jsp @@ -0,0 +1,62 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="java.util.*" + import="org.apache.hadoop.util.StringUtils" + import="org.apache.hadoop.conf.Configuration" + import="org.apache.hadoop.hbase.master.HMaster" + import="org.apache.hadoop.hbase.client.HBaseAdmin" + import="org.apache.hadoop.hbase.HTableDescriptor" %><% + HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER); + Configuration conf = master.getConfiguration(); +%> + + + + +HBase Master: <%= master.getServerName()%>%> + + + + +

    User Tables

    +<% HTableDescriptor[] tables = new HBaseAdmin(conf).listTables(); + if(tables != null && tables.length > 0) { %> + + + + + +<% for(HTableDescriptor htDesc : tables ) { %> + + + + +<% } %> + +

    <%= tables.length %> table(s) in set.

    +
    TableDescription
    <%= htDesc.getNameAsString() %><%= htDesc.toString() %>
    +<% } %> + + diff --git a/src/main/resources/hbase-webapps/master/zk.jsp b/src/main/resources/hbase-webapps/master/zk.jsp new file mode 100644 index 0000000..c205d7f --- /dev/null +++ b/src/main/resources/hbase-webapps/master/zk.jsp @@ -0,0 +1,57 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="java.io.IOException" + import="org.apache.hadoop.conf.Configuration" + import="org.apache.hadoop.hbase.client.HBaseAdmin" + import="org.apache.hadoop.hbase.client.HConnection" + import="org.apache.hadoop.hbase.client.HConnectionManager" + import="org.apache.hadoop.hbase.HRegionInfo" + import="org.apache.hadoop.hbase.zookeeper.ZKUtil" + import="org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher" + import="org.apache.hadoop.hbase.HBaseConfiguration" + import="org.apache.hadoop.hbase.master.HMaster" + import="org.apache.hadoop.hbase.HConstants"%><% + HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER); + ZooKeeperWatcher watcher = master.getZooKeeperWatcher(); +%> + + + + + +ZooKeeper Dump + + + + +

    ZooKeeper Dump

    + +
    +
    +<%= ZKUtil.dump(watcher) %>
+
    + + + diff --git a/src/main/resources/hbase-webapps/regionserver/index.html b/src/main/resources/hbase-webapps/regionserver/index.html new file mode 100644 index 0000000..47fc1b9 --- /dev/null +++ b/src/main/resources/hbase-webapps/regionserver/index.html @@ -0,0 +1,20 @@ + + diff --git a/src/main/resources/hbase-webapps/regionserver/regionserver.jsp b/src/main/resources/hbase-webapps/regionserver/regionserver.jsp new file mode 100644 index 0000000..ee63075 --- /dev/null +++ b/src/main/resources/hbase-webapps/regionserver/regionserver.jsp @@ -0,0 +1,20 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> + diff --git a/src/main/resources/hbase-webapps/rest/index.html b/src/main/resources/hbase-webapps/rest/index.html new file mode 100644 index 0000000..e4084b7 --- /dev/null +++ b/src/main/resources/hbase-webapps/rest/index.html @@ -0,0 +1,20 @@ + + diff --git a/src/main/resources/hbase-webapps/rest/rest.jsp b/src/main/resources/hbase-webapps/rest/rest.jsp new file mode 100644 index 0000000..ba9856c --- /dev/null +++ b/src/main/resources/hbase-webapps/rest/rest.jsp @@ -0,0 +1,74 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="org.apache.hadoop.conf.Configuration" + import="org.apache.hadoop.hbase.HBaseConfiguration" + import="org.apache.hadoop.hbase.util.VersionInfo" + import="java.util.Date" +%> + +<% +Configuration conf = (Configuration)getServletContext().getAttribute("hbase.conf"); +long startcode = conf.getLong("startcode", System.currentTimeMillis()); +String listenPort = conf.get("hbase.rest.port", "8080"); +String serverInfo = listenPort + "," + String.valueOf(startcode); +%> + + + + + +HBase REST Server + + + + + +

    RESTServer: <%= serverInfo %>

    + +
    + +

    Attributes

    + ++++ + + + +
    Attribute NameValueDescription
    HBase Version<%= VersionInfo.getVersion() %>, r<%= VersionInfo.getRevision() %>HBase version and revision
    HBase Compiled<%= VersionInfo.getDate() %>, <%= VersionInfo.getUser() %>When HBase version was compiled and by whom
    REST Server Start Time<%= new Date(startcode) %>Date stamp of when this REST server was started
    + +
    +Apache HBase Wiki on REST + + + diff --git a/src/main/resources/hbase-webapps/static/favicon.ico b/src/main/resources/hbase-webapps/static/favicon.ico new file mode 100644 index 0000000..6e4d0f7 Binary files /dev/null and b/src/main/resources/hbase-webapps/static/favicon.ico differ diff --git a/src/main/resources/hbase-webapps/static/hbase.css b/src/main/resources/hbase-webapps/static/hbase.css new file mode 100644 index 0000000..1fbdba1 --- /dev/null +++ b/src/main/resources/hbase-webapps/static/hbase.css @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +h1, h2, h3 { color: DarkSlateBlue } +table { border: thin solid DodgerBlue } +tr { border: thin solid DodgerBlue } +td { border: thin solid DodgerBlue } +th { border: thin solid DodgerBlue } +#logo {float: right;} +#logo img {border: none;} +#page_title {} +#tasks_menu { + padding: 10px; +} + +div.warning { + border: 1px solid #666; + background-color: #fcc; + font-size: 110%; + font-weight: bold; +} + +td.undeployed-region { + background-color: #faa; +} + +tr.task-monitor-COMPLETE td { + background-color: #afa; +} + +tr.task-monitor-ABORTED td { + background-color: #ffa; +} + +tr.task-monitor-WAITING td { + background-color: #ccc; + font-style: italic; +} ++ diff --git a/src/main/resources/hbase-webapps/static/hbase_logo.png b/src/main/resources/hbase-webapps/static/hbase_logo.png new file mode 100644 index 0000000..615b0a8 Binary files /dev/null and b/src/main/resources/hbase-webapps/static/hbase_logo.png differ diff --git a/src/main/resources/hbase-webapps/static/hbase_logo_med.gif b/src/main/resources/hbase-webapps/static/hbase_logo_med.gif new file mode 100644 index 0000000..36d3e3c Binary files /dev/null and b/src/main/resources/hbase-webapps/static/hbase_logo_med.gif differ diff --git a/src/main/resources/hbase-webapps/thrift/index.html b/src/main/resources/hbase-webapps/thrift/index.html new file mode 100644 index 0000000..9925269 --- /dev/null +++ b/src/main/resources/hbase-webapps/thrift/index.html @@ -0,0 +1,20 @@ + + diff --git a/src/main/resources/hbase-webapps/thrift/thrift.jsp b/src/main/resources/hbase-webapps/thrift/thrift.jsp new file mode 100644 index 0000000..eee9940 --- /dev/null +++ b/src/main/resources/hbase-webapps/thrift/thrift.jsp @@ -0,0 +1,80 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="org.apache.hadoop.conf.Configuration" + import="org.apache.hadoop.hbase.HBaseConfiguration" + import="org.apache.hadoop.hbase.util.VersionInfo" + import="java.util.Date" +%> + +<% +Configuration conf = (Configuration)getServletContext().getAttribute("hbase.conf"); +long startcode = conf.getLong("startcode", System.currentTimeMillis()); +String listenPort = conf.get("hbase.regionserver.thrift.port", "9090"); +String serverInfo = listenPort + "," + String.valueOf(startcode); +String implType = conf.get("hbase.regionserver.thrift.server.type", "threadpool"); +String compact = conf.get("hbase.regionserver.thrift.compact", "false"); +String framed = conf.get("hbase.regionserver.thrift.framed", "false"); +%> + + + + + +HBase Thrift Server + + + + + +

    ThriftServer: <%= serverInfo %>

    + +
    + +

    Attributes

    + ++++ + + + + + + +
    Attribute NameValueDescription
    HBase Version<%= VersionInfo.getVersion() %>, r<%= VersionInfo.getRevision() %>HBase version and revision
    HBase Compiled<%= VersionInfo.getDate() %>, <%= VersionInfo.getUser() %>When HBase version was compiled and by whom
    Thrift Server Start Time<%= new Date(startcode) %>Date stamp of when this Thrift server was started
    Thrift Impl Type<%= implType %>Thrift RPC engine implementation type chosen by this Thrift server
    Compact Protocol<%= compact %>Thrift RPC engine uses compact protocol
    Framed Transport<%= framed %>Thrift RPC engine uses framed transport
    + +
    +Apache HBase Wiki on Thrift + + + diff --git a/src/main/resources/org/apache/hadoop/hbase/mapred/RowCounter_Counters.properties b/src/main/resources/org/apache/hadoop/hbase/mapred/RowCounter_Counters.properties new file mode 100644 index 0000000..661e56d --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/mapred/RowCounter_Counters.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ResourceBundle properties file for RowCounter MR job + +CounterGroupName= RowCounter + +ROWS.name= Rows diff --git a/src/main/resources/org/apache/hadoop/hbase/mapreduce/RowCounter_Counters.properties b/src/main/resources/org/apache/hadoop/hbase/mapreduce/RowCounter_Counters.properties new file mode 100644 index 0000000..661e56d --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/mapreduce/RowCounter_Counters.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ResourceBundle properties file for RowCounter MR job + +CounterGroupName= RowCounter + +ROWS.name= Rows diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd b/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd new file mode 100644 index 0000000..c2df60d --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd @@ -0,0 +1,178 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellMessage.proto new file mode 100644 index 0000000..a7bfe83 --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellMessage.proto @@ -0,0 +1,26 @@ +// Copyright 2010 The Apache Software Foundation +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +message Cell { + optional bytes row = 1; // unused if Cell is in a CellSet + optional bytes column = 2; + optional int64 timestamp = 3; + optional bytes data = 4; +} diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellSetMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellSetMessage.proto new file mode 100644 index 0000000..dfdf125 --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/CellSetMessage.proto @@ -0,0 +1,29 @@ +// Copyright 2010 The Apache Software Foundation +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import "CellMessage.proto"; + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +message CellSet { + message Row { + required bytes key = 1; + repeated Cell values = 2; + } + repeated Row rows = 1; +} diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ColumnSchemaMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ColumnSchemaMessage.proto new file mode 100644 index 0000000..0a9a9af --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ColumnSchemaMessage.proto @@ -0,0 +1,32 @@ +// Copyright 2010 The Apache Software Foundation +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +message ColumnSchema { + optional string name = 1; + message Attribute { + required string name = 1; + required string value = 2; + } + repeated Attribute attrs = 2; + // optional helpful encodings of commonly used attributes + optional int32 ttl = 3; + optional int32 maxVersions = 4; + optional string compression = 5; +} diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ScannerMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ScannerMessage.proto new file mode 100644 index 0000000..6ef3191 --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/ScannerMessage.proto @@ -0,0 +1,30 @@ +// Copyright 2010 The Apache Software Foundation +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +message Scanner { + optional bytes startRow = 1; + optional bytes endRow = 2; + repeated bytes columns = 3; + optional int32 batch = 4; + optional int64 startTime = 5; + optional int64 endTime = 6; + optional int32 maxVersions = 7; + optional string filter = 8; +} diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/StorageClusterStatusMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/StorageClusterStatusMessage.proto new file mode 100644 index 0000000..46e275d --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/StorageClusterStatusMessage.proto @@ -0,0 +1,52 @@ +// Copyright 2010 The Apache Software Foundation +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +message StorageClusterStatus { + message Region { + required bytes name = 1; + optional int32 stores = 2; + optional int32 storefiles = 3; + optional int32 storefileSizeMB = 4; + optional int32 memstoreSizeMB = 5; + optional int32 storefileIndexSizeMB = 6; + optional int64 readRequestsCount = 7; + optional int64 writeRequestsCount = 8; + optional int32 rootIndexSizeKB = 9; + optional int32 totalStaticIndexSizeKB = 10; + optional int32 totalStaticBloomSizeKB = 11; + optional int64 totalCompactingKVs = 12; + optional int64 currentCompactedKVs = 13; + } + message Node { + required string name = 1; // name:port + optional int64 startCode = 2; + optional int32 requests = 3; + optional int32 heapSizeMB = 4; + optional int32 maxHeapSizeMB = 5; + repeated Region regions = 6; + } + // node status + repeated Node liveNodes = 1; + repeated string deadNodes = 2; + // summary statistics + optional int32 regions = 3; + optional int32 requests = 4; + optional double averageLoad = 5; +} diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableInfoMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableInfoMessage.proto new file mode 100644 index 0000000..5dd9120 --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableInfoMessage.proto @@ -0,0 +1,31 @@ +// Copyright 2010 The Apache Software Foundation +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +message TableInfo { + required string name = 1; + message Region { + required string name = 1; + optional bytes startKey = 2; + optional bytes endKey = 3; + optional int64 id = 4; + optional string location = 5; + } + repeated Region regions = 2; +} diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableListMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableListMessage.proto new file mode 100644 index 0000000..2ce4d25 --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableListMessage.proto @@ -0,0 +1,23 @@ +// Copyright 2010 The Apache Software Foundation +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +message TableList { + repeated string name = 1; +} diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableSchemaMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableSchemaMessage.proto new file mode 100644 index 0000000..d817722 --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/TableSchemaMessage.proto @@ -0,0 +1,34 @@ +// Copyright 2010 The Apache Software Foundation +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import "ColumnSchemaMessage.proto"; + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +message TableSchema { + optional string name = 1; + message Attribute { + required string name = 1; + required string value = 2; + } + repeated Attribute attrs = 2; + repeated ColumnSchema columns = 3; + // optional helpful encodings of commonly used attributes + optional bool inMemory = 4; + optional bool readOnly = 5; +} diff --git a/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/VersionMessage.proto b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/VersionMessage.proto new file mode 100644 index 0000000..2404a2e --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/rest/protobuf/VersionMessage.proto @@ -0,0 +1,27 @@ +// Copyright 2010 The Apache Software Foundation +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.hadoop.hbase.rest.protobuf.generated; + +message Version { + optional string restVersion = 1; + optional string jvmVersion = 2; + optional string osVersion = 3; + optional string serverVersion = 4; + optional string jerseyVersion = 5; +} diff --git a/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift b/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift new file mode 100644 index 0000000..4b81b1e --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift @@ -0,0 +1,914 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ---------------------------------------------------------------- +// Hbase.thrift +// +// This is a Thrift interface definition file for the Hbase service. +// Target language libraries for C++, Java, Ruby, PHP, (and more) are +// generated by running this file through the Thrift compiler with the +// appropriate flags. The Thrift compiler binary and runtime +// libraries for various languages are available +// from the Apache Incubator (http://incubator.apache.org/thrift/) +// +// See the package.html file for information on the version of Thrift +// used to generate the *.java files checked into the Hbase project. +// ---------------------------------------------------------------- + +namespace java org.apache.hadoop.hbase.thrift.generated +namespace cpp apache.hadoop.hbase.thrift +namespace rb Apache.Hadoop.Hbase.Thrift +namespace py hbase +namespace perl Hbase + +// +// Types +// + +// NOTE: all variables with the Text type are assumed to be correctly +// formatted UTF-8 strings. This is a programming language and locale +// dependent property that the client application is repsonsible for +// maintaining. If strings with an invalid encoding are sent, an +// IOError will be thrown. + +typedef binary Text +typedef binary Bytes +typedef i32 ScannerID + +/** + * TCell - Used to transport a cell value (byte[]) and the timestamp it was + * stored with together as a result for get and getRow methods. This promotes + * the timestamp of a cell to a first-class value, making it easy to take + * note of temporal data. Cell is used all the way from HStore up to HTable. + */ +struct TCell{ + 1:Bytes value, + 2:i64 timestamp +} + +/** + * An HColumnDescriptor contains information about a column family + * such as the number of versions, compression settings, etc. It is + * used as input when creating a table or adding a column. + */ +struct ColumnDescriptor { + 1:Text name, + 2:i32 maxVersions = 3, + 3:string compression = "NONE", + 4:bool inMemory = 0, + 5:string bloomFilterType = "NONE", + 6:i32 bloomFilterVectorSize = 0, + 7:i32 bloomFilterNbHashes = 0, + 8:bool blockCacheEnabled = 0, + 9:i32 timeToLive = -1 +} + +/** + * A TRegionInfo contains information about an HTable region. + */ +struct TRegionInfo { + 1:Text startKey, + 2:Text endKey, + 3:i64 id, + 4:Text name, + 5:byte version, + 6:Text serverName, + 7:i32 port +} + +/** + * A Mutation object is used to either update or delete a column-value. + */ +struct Mutation { + 1:bool isDelete = 0, + 2:Text column, + 3:Text value, + 4:bool writeToWAL = 1 +} + + +/** + * A BatchMutation object is used to apply a number of Mutations to a single row. + */ +struct BatchMutation { + 1:Text row, + 2:list mutations +} + +/** + * For increments that are not incrementColumnValue + * equivalents. + */ +struct TIncrement { + 1:Text table, + 2:Text row, + 3:Text column, + 4:i64 ammount +} + +/** + * Holds row name and then a map of columns to cells. + */ +struct TRowResult { + 1:Text row, + 2:map columns +} + +/** + * A Scan object is used to specify scanner parameters when opening a scanner. + */ +struct TScan { + 1:optional Text startRow, + 2:optional Text stopRow, + 3:optional i64 timestamp, + 4:optional list columns, + 5:optional i32 caching, + 6:optional Text filterString +} + +// +// Exceptions +// +/** + * An IOError exception signals that an error occurred communicating + * to the Hbase master or an Hbase region server. Also used to return + * more general Hbase error conditions. + */ +exception IOError { + 1:string message +} + +/** + * An IllegalArgument exception indicates an illegal or invalid + * argument was passed into a procedure. + */ +exception IllegalArgument { + 1:string message +} + +/** + * An AlreadyExists exceptions signals that a table with the specified + * name already exists + */ +exception AlreadyExists { + 1:string message +} + +// +// Service +// + +service Hbase { + /** + * Brings a table on-line (enables it) + */ + void enableTable( + /** name of the table */ + 1:Bytes tableName + ) throws (1:IOError io) + + /** + * Disables a table (takes it off-line) If it is being served, the master + * will tell the servers to stop serving it. + */ + void disableTable( + /** name of the table */ + 1:Bytes tableName + ) throws (1:IOError io) + + /** + * @return true if table is on-line + */ + bool isTableEnabled( + /** name of the table to check */ + 1:Bytes tableName + ) throws (1:IOError io) + + void compact(1:Bytes tableNameOrRegionName) + throws (1:IOError io) + + void majorCompact(1:Bytes tableNameOrRegionName) + throws (1:IOError io) + + /** + * List all the userspace tables. + * + * @return returns a list of names + */ + list getTableNames() + throws (1:IOError io) + + /** + * List all the column families assoicated with a table. + * + * @return list of column family descriptors + */ + map getColumnDescriptors ( + /** table name */ + 1:Text tableName + ) throws (1:IOError io) + + /** + * List the regions associated with a table. + * + * @return list of region descriptors + */ + list getTableRegions( + /** table name */ + 1:Text tableName) + throws (1:IOError io) + + /** + * Create a table with the specified column families. The name + * field for each ColumnDescriptor must be set and must end in a + * colon (:). All other fields are optional and will get default + * values if not explicitly specified. + * + * @throws IllegalArgument if an input parameter is invalid + * + * @throws AlreadyExists if the table name already exists + */ + void createTable( + /** name of table to create */ + 1:Text tableName, + + /** list of column family descriptors */ + 2:list columnFamilies + ) throws (1:IOError io, 2:IllegalArgument ia, 3:AlreadyExists exist) + + /** + * Deletes a table + * + * @throws IOError if table doesn't exist on server or there was some other + * problem + */ + void deleteTable( + /** name of table to delete */ + 1:Text tableName + ) throws (1:IOError io) + + /** + * Get a single TCell for the specified table, row, and column at the + * latest timestamp. Returns an empty list if no such value exists. + * + * @return value for specified row/column + */ + list get( + /** name of table */ + 1:Text tableName, + + /** row key */ + 2:Text row, + + /** column name */ + 3:Text column, + + /** Get attributes */ + 4:map attributes + ) throws (1:IOError io) + + /** + * Get the specified number of versions for the specified table, + * row, and column. + * + * @return list of cells for specified row/column + */ + list getVer( + /** name of table */ + 1:Text tableName, + + /** row key */ + 2:Text row, + + /** column name */ + 3:Text column, + + /** number of versions to retrieve */ + 4:i32 numVersions, + + /** Get attributes */ + 5:map attributes + ) throws (1:IOError io) + + /** + * Get the specified number of versions for the specified table, + * row, and column. Only versions less than or equal to the specified + * timestamp will be returned. + * + * @return list of cells for specified row/column + */ + list getVerTs( + /** name of table */ + 1:Text tableName, + + /** row key */ + 2:Text row, + + /** column name */ + 3:Text column, + + /** timestamp */ + 4:i64 timestamp, + + /** number of versions to retrieve */ + 5:i32 numVersions, + + /** Get attributes */ + 6:map attributes + ) throws (1:IOError io) + + /** + * Get all the data for the specified table and row at the latest + * timestamp. Returns an empty list if the row does not exist. + * + * @return TRowResult containing the row and map of columns to TCells + */ + list getRow( + /** name of table */ + 1:Text tableName, + + /** row key */ + 2:Text row, + + /** Get attributes */ + 3:map attributes + ) throws (1:IOError io) + + /** + * Get the specified columns for the specified table and row at the latest + * timestamp. Returns an empty list if the row does not exist. + * + * @return TRowResult containing the row and map of columns to TCells + */ + list getRowWithColumns( + /** name of table */ + 1:Text tableName, + + /** row key */ + 2:Text row, + + /** List of columns to return, null for all columns */ + 3:list columns, + + /** Get attributes */ + 4:map attributes + ) throws (1:IOError io) + + /** + * Get all the data for the specified table and row at the specified + * timestamp. Returns an empty list if the row does not exist. + * + * @return TRowResult containing the row and map of columns to TCells + */ + list getRowTs( + /** name of the table */ + 1:Text tableName, + + /** row key */ + 2:Text row, + + /** timestamp */ + 3:i64 timestamp, + + /** Get attributes */ + 4:map attributes + ) throws (1:IOError io) + + /** + * Get the specified columns for the specified table and row at the specified + * timestamp. Returns an empty list if the row does not exist. + * + * @return TRowResult containing the row and map of columns to TCells + */ + list getRowWithColumnsTs( + /** name of table */ + 1:Text tableName, + + /** row key */ + 2:Text row, + + /** List of columns to return, null for all columns */ + 3:list columns, + 4:i64 timestamp, + + /** Get attributes */ + 5:map attributes + ) throws (1:IOError io) + + /** + * Get all the data for the specified table and rows at the latest + * timestamp. Returns an empty list if no rows exist. + * + * @return TRowResult containing the rows and map of columns to TCells + */ + list getRows( + /** name of table */ + 1:Text tableName, + + /** row keys */ + 2:list rows + + /** Get attributes */ + 3:map attributes + ) throws (1:IOError io) + + /** + * Get the specified columns for the specified table and rows at the latest + * timestamp. Returns an empty list if no rows exist. + * + * @return TRowResult containing the rows and map of columns to TCells + */ + list getRowsWithColumns( + /** name of table */ + 1:Text tableName, + + /** row keys */ + 2:list rows, + + /** List of columns to return, null for all columns */ + 3:list columns, + + /** Get attributes */ + 4:map attributes + ) throws (1:IOError io) + + /** + * Get all the data for the specified table and rows at the specified + * timestamp. Returns an empty list if no rows exist. + * + * @return TRowResult containing the rows and map of columns to TCells + */ + list getRowsTs( + /** name of the table */ + 1:Text tableName, + + /** row keys */ + 2:list rows + + /** timestamp */ + 3:i64 timestamp, + + /** Get attributes */ + 4:map attributes + ) throws (1:IOError io) + + /** + * Get the specified columns for the specified table and rows at the specified + * timestamp. Returns an empty list if no rows exist. + * + * @return TRowResult containing the rows and map of columns to TCells + */ + list getRowsWithColumnsTs( + /** name of table */ + 1:Text tableName, + + /** row keys */ + 2:list rows + + /** List of columns to return, null for all columns */ + 3:list columns, + 4:i64 timestamp, + + /** Get attributes */ + 5:map attributes + ) throws (1:IOError io) + + /** + * Apply a series of mutations (updates/deletes) to a row in a + * single transaction. If an exception is thrown, then the + * transaction is aborted. Default current timestamp is used, and + * all entries will have an identical timestamp. + */ + void mutateRow( + /** name of table */ + 1:Text tableName, + + /** row key */ + 2:Text row, + + /** list of mutation commands */ + 3:list mutations, + + /** Mutation attributes */ + 4:map attributes + ) throws (1:IOError io, 2:IllegalArgument ia) + + /** + * Apply a series of mutations (updates/deletes) to a row in a + * single transaction. If an exception is thrown, then the + * transaction is aborted. The specified timestamp is used, and + * all entries will have an identical timestamp. + */ + void mutateRowTs( + /** name of table */ + 1:Text tableName, + + /** row key */ + 2:Text row, + + /** list of mutation commands */ + 3:list mutations, + + /** timestamp */ + 4:i64 timestamp, + + /** Mutation attributes */ + 5:map attributes + ) throws (1:IOError io, 2:IllegalArgument ia) + + /** + * Apply a series of batches (each a series of mutations on a single row) + * in a single transaction. If an exception is thrown, then the + * transaction is aborted. Default current timestamp is used, and + * all entries will have an identical timestamp. + */ + void mutateRows( + /** name of table */ + 1:Text tableName, + + /** list of row batches */ + 2:list rowBatches, + + /** Mutation attributes */ + 3:map attributes + ) throws (1:IOError io, 2:IllegalArgument ia) + + /** + * Apply a series of batches (each a series of mutations on a single row) + * in a single transaction. If an exception is thrown, then the + * transaction is aborted. The specified timestamp is used, and + * all entries will have an identical timestamp. + */ + void mutateRowsTs( + /** name of table */ + 1:Text tableName, + + /** list of row batches */ + 2:list rowBatches, + + /** timestamp */ + 3:i64 timestamp, + + /** Mutation attributes */ + 4:map attributes + ) throws (1:IOError io, 2:IllegalArgument ia) + + /** + * Atomically increment the column value specified. Returns the next value post increment. + */ + i64 atomicIncrement( + /** name of table */ + 1:Text tableName, + + /** row to increment */ + 2:Text row, + + /** name of column */ + 3:Text column, + + /** amount to increment by */ + 4:i64 value + ) throws (1:IOError io, 2:IllegalArgument ia) + + /** + * Delete all cells that match the passed row and column. + */ + void deleteAll( + /** name of table */ + 1:Text tableName, + + /** Row to update */ + 2:Text row, + + /** name of column whose value is to be deleted */ + 3:Text column, + + /** Delete attributes */ + 4:map attributes + ) throws (1:IOError io) + + /** + * Delete all cells that match the passed row and column and whose + * timestamp is equal-to or older than the passed timestamp. + */ + void deleteAllTs( + /** name of table */ + 1:Text tableName, + + /** Row to update */ + 2:Text row, + + /** name of column whose value is to be deleted */ + 3:Text column, + + /** timestamp */ + 4:i64 timestamp, + + /** Delete attributes */ + 5:map attributes + ) throws (1:IOError io) + + /** + * Completely delete the row's cells. + */ + void deleteAllRow( + /** name of table */ + 1:Text tableName, + + /** key of the row to be completely deleted. */ + 2:Text row, + + /** Delete attributes */ + 3:map attributes + ) throws (1:IOError io) + + /** + * Increment a cell by the ammount. + * Increments can be applied async if hbase.regionserver.thrift.coalesceIncrement is set to true. + * False is the default. Turn to true if you need the extra performance and can accept some + * data loss if a thrift server dies with increments still in the queue. + */ + void increment( + /** The single increment to apply */ + 1:TIncrement increment + ) throws (1:IOError io) + + + void incrementRows( + /** The list of increments */ + 1:list increments + ) throws (1:IOError io) + + /** + * Completely delete the row's cells marked with a timestamp + * equal-to or older than the passed timestamp. + */ + void deleteAllRowTs( + /** name of table */ + 1:Text tableName, + + /** key of the row to be completely deleted. */ + 2:Text row, + + /** timestamp */ + 3:i64 timestamp, + + /** Delete attributes */ + 4:map attributes + ) throws (1:IOError io) + + /** + * Get a scanner on the current table, using the Scan instance + * for the scan parameters. + */ + ScannerID scannerOpenWithScan( + /** name of table */ + 1:Text tableName, + + /** Scan instance */ + 2:TScan scan, + + /** Scan attributes */ + 3:map attributes + ) throws (1:IOError io) + + /** + * Get a scanner on the current table starting at the specified row and + * ending at the last row in the table. Return the specified columns. + * + * @return scanner id to be used with other scanner procedures + */ + ScannerID scannerOpen( + /** name of table */ + 1:Text tableName, + + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + 2:Text startRow, + + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + 3:list columns, + + /** Scan attributes */ + 4:map attributes + ) throws (1:IOError io) + + /** + * Get a scanner on the current table starting and stopping at the + * specified rows. ending at the last row in the table. Return the + * specified columns. + * + * @return scanner id to be used with other scanner procedures + */ + ScannerID scannerOpenWithStop( + /** name of table */ + 1:Text tableName, + + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + 2:Text startRow, + + /** + * row to stop scanning on. This row is *not* included in the + * scanner's results + */ + 3:Text stopRow, + + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + 4:list columns, + + /** Scan attributes */ + 5:map attributes + ) throws (1:IOError io) + + /** + * Open a scanner for a given prefix. That is all rows will have the specified + * prefix. No other rows will be returned. + * + * @return scanner id to use with other scanner calls + */ + ScannerID scannerOpenWithPrefix( + /** name of table */ + 1:Text tableName, + + /** the prefix (and thus start row) of the keys you want */ + 2:Text startAndPrefix, + + /** the columns you want returned */ + 3:list columns, + + /** Scan attributes */ + 4:map attributes + ) throws (1:IOError io) + + /** + * Get a scanner on the current table starting at the specified row and + * ending at the last row in the table. Return the specified columns. + * Only values with the specified timestamp are returned. + * + * @return scanner id to be used with other scanner procedures + */ + ScannerID scannerOpenTs( + /** name of table */ + 1:Text tableName, + + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + 2:Text startRow, + + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + 3:list columns, + + /** timestamp */ + 4:i64 timestamp, + + /** Scan attributes */ + 5:map attributes + ) throws (1:IOError io) + + /** + * Get a scanner on the current table starting and stopping at the + * specified rows. ending at the last row in the table. Return the + * specified columns. Only values with the specified timestamp are + * returned. + * + * @return scanner id to be used with other scanner procedures + */ + ScannerID scannerOpenWithStopTs( + /** name of table */ + 1:Text tableName, + + /** + * Starting row in table to scan. + * Send "" (empty string) to start at the first row. + */ + 2:Text startRow, + + /** + * row to stop scanning on. This row is *not* included in the + * scanner's results + */ + 3:Text stopRow, + + /** + * columns to scan. If column name is a column family, all + * columns of the specified column family are returned. It's also possible + * to pass a regex in the column qualifier. + */ + 4:list columns, + + /** timestamp */ + 5:i64 timestamp, + + /** Scan attributes */ + 6:map attributes + ) throws (1:IOError io) + + /** + * Returns the scanner's current row value and advances to the next + * row in the table. When there are no more rows in the table, or a key + * greater-than-or-equal-to the scanner's specified stopRow is reached, + * an empty list is returned. + * + * @return a TRowResult containing the current row and a map of the columns to TCells. + * + * @throws IllegalArgument if ScannerID is invalid + * + * @throws NotFound when the scanner reaches the end + */ + list scannerGet( + /** id of a scanner returned by scannerOpen */ + 1:ScannerID id + ) throws (1:IOError io, 2:IllegalArgument ia) + + /** + * Returns, starting at the scanner's current row value nbRows worth of + * rows and advances to the next row in the table. When there are no more + * rows in the table, or a key greater-than-or-equal-to the scanner's + * specified stopRow is reached, an empty list is returned. + * + * @return a TRowResult containing the current row and a map of the columns to TCells. + * + * @throws IllegalArgument if ScannerID is invalid + * + * @throws NotFound when the scanner reaches the end + */ + list scannerGetList( + /** id of a scanner returned by scannerOpen */ + 1:ScannerID id, + + /** number of results to return */ + 2:i32 nbRows + ) throws (1:IOError io, 2:IllegalArgument ia) + + /** + * Closes the server-state associated with an open scanner. + * + * @throws IllegalArgument if ScannerID is invalid + */ + void scannerClose( + /** id of a scanner returned by scannerOpen */ + 1:ScannerID id + ) throws (1:IOError io, 2:IllegalArgument ia) + + /** + * Get the row just before the specified one. + * + * @return value for specified row/column + */ + list getRowOrBefore( + /** name of table */ + 1:Text tableName, + + /** row key */ + 2:Text row, + + /** column name */ + 3:Text family + ) throws (1:IOError io) + + /** + * Get the regininfo for the specified row. It scans + * the metatable to find region's start and end keys. + * + * @return value for specified row/column + */ + TRegionInfo getRegionInfo( + /** row key */ + 1:Text row, + + ) throws (1:IOError io) +} diff --git a/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift b/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift new file mode 100644 index 0000000..5bb0f51 --- /dev/null +++ b/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift @@ -0,0 +1,412 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// NOTE: The "required" and "optional" keywords for the service methods are purely for documentation + +namespace java org.apache.hadoop.hbase.thrift2.generated +namespace cpp apache.hadoop.hbase.thrift2 +namespace rb Apache.Hadoop.Hbase.Thrift2 +namespace py hbase +namespace perl Hbase + +struct TTimeRange { + 1: required i64 minStamp, + 2: required i64 maxStamp +} + +/** + * Addresses a single cell or multiple cells + * in a HBase table by column family and optionally + * a column qualifier and timestamp + */ +struct TColumn { + 1: required binary family, + 2: optional binary qualifier, + 3: optional i64 timestamp +} + +/** + * Represents a single cell and its value. + */ +struct TColumnValue { + 1: required binary family, + 2: required binary qualifier, + 3: required binary value, + 4: optional i64 timestamp +} + +/** + * Represents a single cell and the amount to increment it by + */ +struct TColumnIncrement { + 1: required binary family, + 2: required binary qualifier, + 3: optional i64 amount = 1 +} + +/** + * if no Result is found, row and columnValues will not be set. + */ +struct TResult { + 1: optional binary row, + 2: required list columnValues +} + +/** + * Specify type of delete: + * - DELETE_COLUMN means exactly one version will be removed, + * - DELETE_COLUMNS means previous versions will also be removed. + */ +enum TDeleteType { + DELETE_COLUMN = 0, + DELETE_COLUMNS = 1 +} + +/** + * Used to perform Get operations on a single row. + * + * The scope can be further narrowed down by specifying a list of + * columns or column families. + * + * To get everything for a row, instantiate a Get object with just the row to get. + * To further define the scope of what to get you can add a timestamp or time range + * with an optional maximum number of versions to return. + * + * If you specify a time range and a timestamp the range is ignored. + * Timestamps on TColumns are ignored. + * + * TODO: Filter, Locks + */ +struct TGet { + 1: required binary row, + 2: optional list columns, + + 3: optional i64 timestamp, + 4: optional TTimeRange timeRange, + + 5: optional i32 maxVersions, +} + +/** + * Used to perform Put operations for a single row. + * + * Add column values to this object and they'll be added. + * You can provide a default timestamp if the column values + * don't have one. If you don't provide a default timestamp + * the current time is inserted. + * + * You can also specify if this Put should be written + * to the write-ahead Log (WAL) or not. It defaults to true. + */ +struct TPut { + 1: required binary row, + 2: required list columnValues + 3: optional i64 timestamp, + 4: optional bool writeToWal = 1 +} + +/** + * Used to perform Delete operations on a single row. + * + * The scope can be further narrowed down by specifying a list of + * columns or column families as TColumns. + * + * Specifying only a family in a TColumn will delete the whole family. + * If a timestamp is specified all versions with a timestamp less than + * or equal to this will be deleted. If no timestamp is specified the + * current time will be used. + * + * Specifying a family and a column qualifier in a TColumn will delete only + * this qualifier. If a timestamp is specified only versions equal + * to this timestamp will be deleted. If no timestamp is specified the + * most recent version will be deleted. To delete all previous versions, + * specify the DELETE_COLUMNS TDeleteType. + * + * The top level timestamp is only used if a complete row should be deleted + * (i.e. no columns are passed) and if it is specified it works the same way + * as if you had added a TColumn for every column family and this timestamp + * (i.e. all versions older than or equal in all column families will be deleted) + * + */ +struct TDelete { + 1: required binary row, + 2: optional list columns, + 3: optional i64 timestamp, + 4: optional TDeleteType deleteType = 1, + 5: optional bool writeToWal = 1 +} + +/** + * Used to perform Increment operations for a single row. + * + * You can specify if this Increment should be written + * to the write-ahead Log (WAL) or not. It defaults to true. + */ +struct TIncrement { + 1: required binary row, + 2: required list columns, + 3: optional bool writeToWal = 1 +} + +/** + * Any timestamps in the columns are ignored, use timeRange to select by timestamp. + * Max versions defaults to 1. + */ +struct TScan { + 1: optional binary startRow, + 2: optional binary stopRow, + 3: optional list columns + 4: optional i32 caching, + 5: optional i32 maxVersions=1, + 6: optional TTimeRange timeRange, +} + +// +// Exceptions +// + +/** + * A TIOError exception signals that an error occurred communicating + * to the HBase master or a HBase region server. Also used to return + * more general HBase error conditions. + */ +exception TIOError { + 1: optional string message +} + +/** + * A TIllegalArgument exception indicates an illegal or invalid + * argument was passed into a procedure. + */ +exception TIllegalArgument { + 1: optional string message +} + +service THBaseService { + + /** + * Test for the existence of columns in the table, as specified in the TGet. + * + * @return true if the specified TGet matches one or more keys, false if not + */ + bool exists( + /** the table to check on */ + 1: required binary table, + + /** the TGet to check for */ + 2: required TGet get + ) throws (1:TIOError io) + + /** + * Method for getting data from a row. + * + * If the row cannot be found an empty Result is returned. + * This can be checked by the empty field of the TResult + * + * @return the result + */ + TResult get( + /** the table to get from */ + 1: required binary table, + + /** the TGet to fetch */ + 2: required TGet get + ) throws (1: TIOError io) + + /** + * Method for getting multiple rows. + * + * If a row cannot be found there will be a null + * value in the result list for that TGet at the + * same position. + * + * So the Results are in the same order as the TGets. + */ + list getMultiple( + /** the table to get from */ + 1: required binary table, + + /** a list of TGets to fetch, the Result list + will have the Results at corresponding positions + or null if there was an error */ + 2: required list gets + ) throws (1: TIOError io) + + /** + * Commit a TPut to a table. + */ + void put( + /** the table to put data in */ + 1: required binary table, + + /** the TPut to put */ + 2: required TPut put + ) throws (1: TIOError io) + + /** + * Atomically checks if a row/family/qualifier value matches the expected + * value. If it does, it adds the TPut. + * + * @return true if the new put was executed, false otherwise + */ + bool checkAndPut( + /** to check in and put to */ + 1: required binary table, + + /** row to check */ + 2: required binary row, + + /** column family to check */ + 3: required binary family, + + /** column qualifier to check */ + 4: required binary qualifier, + + /** the expected value, if not provided the + check is for the non-existence of the + column in question */ + 5: binary value, + + /** the TPut to put if the check succeeds */ + 6: required TPut put + ) throws (1: TIOError io) + + /** + * Commit a List of Puts to the table. + */ + void putMultiple( + /** the table to put data in */ + 1: required binary table, + + /** a list of TPuts to commit */ + 2: required list puts + ) throws (1: TIOError io) + + /** + * Deletes as specified by the TDelete. + * + * Note: "delete" is a reserved keyword and cannot be used in Thrift + * thus the inconsistent naming scheme from the other functions. + */ + void deleteSingle( + /** the table to delete from */ + 1: required binary table, + + /** the TDelete to delete */ + 2: required TDelete deleteSingle + ) throws (1: TIOError io) + + /** + * Bulk commit a List of TDeletes to the table. + * + * This returns a list of TDeletes that were not + * executed. So if everything succeeds you'll + * receive an empty list. + */ + list deleteMultiple( + /** the table to delete from */ + 1: required binary table, + + /** list of TDeletes to delete */ + 2: required list deletes + ) throws (1: TIOError io) + + /** + * Atomically checks if a row/family/qualifier value matches the expected + * value. If it does, it adds the delete. + * + * @return true if the new delete was executed, false otherwise + */ + bool checkAndDelete( + /** to check in and delete from */ + 1: required binary table, + + /** row to check */ + 2: required binary row, + + /** column family to check */ + 3: required binary family, + + /** column qualifier to check */ + 4: required binary qualifier, + + /** the expected value, if not provided the + check is for the non-existence of the + column in question */ + 5: binary value, + + /** the TDelete to execute if the check succeeds */ + 6: required TDelete deleteSingle + ) throws (1: TIOError io) + + TResult increment( + /** the table to increment the value on */ + 1: required binary table, + + /** the TIncrement to increment */ + 2: required TIncrement increment + ) throws (1: TIOError io) + + /** + * Get a Scanner for the provided TScan object. + * + * @return Scanner Id to be used with other scanner procedures + */ + i32 openScanner( + /** the table to get the Scanner for */ + 1: required binary table, + + /** the scan object to get a Scanner for */ + 2: required TScan scan, + ) throws (1: TIOError io) + + /** + * Grabs multiple rows from a Scanner. + * + * @return Between zero and numRows TResults + */ + list getScannerRows( + /** the Id of the Scanner to return rows from. This is an Id returned from the openScanner function. */ + 1: required i32 scannerId, + + /** number of rows to return */ + 2: i32 numRows = 1 + ) throws ( + 1: TIOError io, + + /** if the scannerId is invalid */ + 2: TIllegalArgument ia + ) + + /** + * Closes the scanner. Should be called if you need to close + * the Scanner before all results are read. + * + * Exhausted scanners are closed automatically. + */ + void closeScanner( + /** the Id of the Scanner to close **/ + 1: required i32 scannerId + ) throws ( + 1: TIOError io, + + /** if the scannerId is invalid */ + 2: TIllegalArgument ia + ) + +} diff --git a/src/main/ruby/hbase.rb b/src/main/ruby/hbase.rb new file mode 100644 index 0000000..4d97cd0 --- /dev/null +++ b/src/main/ruby/hbase.rb @@ -0,0 +1,80 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# HBase ruby classes. +# Has wrapper classes for org.apache.hadoop.hbase.client.HBaseAdmin +# and for org.apache.hadoop.hbase.client.HTable. Classes take +# Formatters on construction and outputs any results using +# Formatter methods. These classes are only really for use by +# the hirb.rb HBase Shell script; they don't make much sense elsewhere. +# For example, the exists method on Admin class prints to the formatter +# whether the table exists and returns nil regardless. +include Java + +include_class('java.lang.Integer') {|package,name| "J#{name}" } +include_class('java.lang.Long') {|package,name| "J#{name}" } +include_class('java.lang.Boolean') {|package,name| "J#{name}" } + +module HBaseConstants + COLUMN = "COLUMN" + COLUMNS = "COLUMNS" + TIMESTAMP = "TIMESTAMP" + TIMERANGE = "TIMERANGE" + NAME = org.apache.hadoop.hbase.HConstants::NAME + VERSIONS = org.apache.hadoop.hbase.HConstants::VERSIONS + IN_MEMORY = org.apache.hadoop.hbase.HConstants::IN_MEMORY + CONFIG = org.apache.hadoop.hbase.HConstants::CONFIG + STOPROW = "STOPROW" + STARTROW = "STARTROW" + ENDROW = STOPROW + RAW = "RAW" + LIMIT = "LIMIT" + METHOD = "METHOD" + MAXLENGTH = "MAXLENGTH" + CACHE_BLOCKS = "CACHE_BLOCKS" + REPLICATION_SCOPE = "REPLICATION_SCOPE" + INTERVAL = 'INTERVAL' + CACHE = 'CACHE' + FILTER = 'FILTER' + SPLITS = 'SPLITS' + SPLITS_FILE = 'SPLITS_FILE' + SPLITALGO = 'SPLITALGO' + NUMREGIONS = 'NUMREGIONS' + + # Load constants from hbase java API + def self.promote_constants(constants) + # The constants to import are all in uppercase + constants.each do |c| + next if c =~ /DEFAULT_.*/ || c != c.upcase + next if eval("defined?(#{c})") + eval("#{c} = '#{c}'") + end + end + + promote_constants(org.apache.hadoop.hbase.HColumnDescriptor.constants) + promote_constants(org.apache.hadoop.hbase.HTableDescriptor.constants) +end + +# Include classes definition +require 'hbase/hbase' +require 'hbase/admin' +require 'hbase/table' +require 'hbase/replication_admin' +require 'hbase/security' diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb new file mode 100644 index 0000000..dfad892 --- /dev/null +++ b/src/main/ruby/hbase/admin.rb @@ -0,0 +1,701 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java +java_import org.apache.hadoop.hbase.util.Pair +java_import org.apache.hadoop.hbase.util.RegionSplitter + +# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin + +module Hbase + class Admin + include HBaseConstants + + def initialize(configuration, formatter) + @admin = org.apache.hadoop.hbase.client.HBaseAdmin.new(configuration) + connection = @admin.getConnection() + @conf = configuration + @zk_wrapper = connection.getZooKeeperWatcher() + zk = @zk_wrapper.getRecoverableZooKeeper().getZooKeeper() + @zk_main = org.apache.zookeeper.ZooKeeperMain.new(zk) + @formatter = formatter + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of tables in hbase + def list + @admin.listTables.map { |t| t.getNameAsString } + end + + #---------------------------------------------------------------------------------------------- + # Requests a table or region flush + def flush(table_or_region_name) + @admin.flush(table_or_region_name) + end + + #---------------------------------------------------------------------------------------------- + # Requests a table or region or column family compaction + def compact(table_or_region_name, family = nil) + if family == nil + @admin.compact(table_or_region_name) + else + # We are compacting a column family within a region. + @admin.compact(table_or_region_name, family) + end + end + + #---------------------------------------------------------------------------------------------- + # Requests a table or region or column family major compaction + def major_compact(table_or_region_name, family = nil) + if family == nil + @admin.majorCompact(table_or_region_name) + else + # We are major compacting a column family within a region or table. + @admin.majorCompact(table_or_region_name, family) + end + end + + #---------------------------------------------------------------------------------------------- + # Requests a regionserver's HLog roll + def hlog_roll(server_name) + @admin.rollHLogWriter(server_name) + end + + #---------------------------------------------------------------------------------------------- + # Requests a table or region split + def split(table_or_region_name, split_point) + if split_point == nil + @admin.split(table_or_region_name) + else + @admin.split(table_or_region_name, split_point) + end + end + + #---------------------------------------------------------------------------------------------- + # Requests a cluster balance + # Returns true if balancer ran + def balancer() + @admin.balancer() + end + + #---------------------------------------------------------------------------------------------- + # Enable/disable balancer + # Returns previous balancer switch setting. + def balance_switch(enableDisable) + @admin.setBalancerRunning( + java.lang.Boolean::valueOf(enableDisable), java.lang.Boolean::valueOf(false)) + end + + #---------------------------------------------------------------------------------------------- + # Enables a table + def enable(table_name) + tableExists(table_name) + return if enabled?(table_name) + @admin.enableTable(table_name) + end + + #---------------------------------------------------------------------------------------------- + # Enables all tables matching the given regex + def enable_all(regex) + regex = regex.to_s + @admin.enableTables(regex) + end + + #---------------------------------------------------------------------------------------------- + # Disables a table + def disable(table_name) + tableExists(table_name) + return if disabled?(table_name) + @admin.disableTable(table_name) + end + + #---------------------------------------------------------------------------------------------- + # Disables all tables matching the given regex + def disable_all(regex) + regex = regex.to_s + @admin.disableTables(regex).map { |t| t.getNameAsString } + end + + #--------------------------------------------------------------------------------------------- + # Throw exception if table doesn't exist + def tableExists(table_name) + raise ArgumentError, "Table #{table_name} does not exist.'" unless exists?(table_name) + end + + #---------------------------------------------------------------------------------------------- + # Is table disabled? + def disabled?(table_name) + @admin.isTableDisabled(table_name) + end + + #---------------------------------------------------------------------------------------------- + # Drops a table + def drop(table_name) + tableExists(table_name) + raise ArgumentError, "Table #{table_name} is enabled. Disable it first.'" if enabled?(table_name) + + @admin.deleteTable(table_name) + end + + #---------------------------------------------------------------------------------------------- + # Drops a table + def drop_all(regex) + regex = regex.to_s + failed = @admin.deleteTables(regex).map { |t| t.getNameAsString } + return failed + end + + #---------------------------------------------------------------------------------------------- + # Returns ZooKeeper status dump + def zk_dump + org.apache.hadoop.hbase.zookeeper.ZKUtil::dump(@zk_wrapper) + end + + #---------------------------------------------------------------------------------------------- + # Creates a table + def create(table_name, *args) + # Fail if table name is not a string + raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) + + # Flatten params array + args = args.flatten.compact + + # Fail if no column families defined + raise(ArgumentError, "Table must have at least one column family") if args.empty? + + # Start defining the table + htd = org.apache.hadoop.hbase.HTableDescriptor.new(table_name) + splits = nil + # Args are either columns or splits, add them to the table definition + # TODO: add table options support + args.each do |arg| + unless arg.kind_of?(String) || arg.kind_of?(Hash) + raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") + end + + if arg.kind_of?(String) + # the arg is a string, default action is to add a column to the table + htd.addFamily(hcd(arg, htd)) + else + # arg is a hash. 4 possibilities: + if (arg.has_key?(SPLITS) or arg.has_key?(SPLITS_FILE)) + if arg.has_key?(SPLITS_FILE) + unless File.exist?(arg[SPLITS_FILE]) + raise(ArgumentError, "Splits file #{arg[SPLITS_FILE]} doesn't exist") + end + arg[SPLITS] = [] + File.foreach(arg[SPLITS_FILE]) do |line| + arg[SPLITS].push(line.strip()) + end + end + + splits = Java::byte[][arg[SPLITS].size].new + idx = 0 + arg[SPLITS].each do |split| + splits[idx] = split.to_java_bytes + idx = idx + 1 + end + elsif (arg.has_key?(NUMREGIONS) or arg.has_key?(SPLITALGO)) + # (1) deprecated region pre-split API + raise(ArgumentError, "Column family configuration should be specified in a separate clause") if arg.has_key?(NAME) + raise(ArgumentError, "Number of regions must be specified") unless arg.has_key?(NUMREGIONS) + raise(ArgumentError, "Split algorithm must be specified") unless arg.has_key?(SPLITALGO) + raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1 + num_regions = arg[NUMREGIONS] + split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg[SPLITALGO]) + splits = split_algo.split(JInteger.valueOf(num_regions)) + elsif (method = arg.delete(METHOD)) + # (2) table_att modification + raise(ArgumentError, "table_att is currently the only supported method") unless method == 'table_att' + raise(ArgumentError, "NUMREGIONS & SPLITALGO must both be specified") unless arg.has_key?(NUMREGIONS) == arg.has_key?(split_algo) + htd.setMaxFileSize(JLong.valueOf(arg[MAX_FILESIZE])) if arg[MAX_FILESIZE] + htd.setReadOnly(JBoolean.valueOf(arg[READONLY])) if arg[READONLY] + htd.setMemStoreFlushSize(JLong.valueOf(arg[MEMSTORE_FLUSHSIZE])) if arg[MEMSTORE_FLUSHSIZE] + htd.setDeferredLogFlush(JBoolean.valueOf(arg[DEFERRED_LOG_FLUSH])) if arg[DEFERRED_LOG_FLUSH] + htd.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT]) if arg[COMPRESSION_COMPACT] + if arg[NUMREGIONS] + raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1 + num_regions = arg[NUMREGIONS] + split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg[SPLITALGO]) + splits = split_algo.split(JInteger.valueOf(num_regions)) + end + if arg[CONFIG] + raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash) + for k,v in arg[CONFIG] + v = v.to_s unless v.nil? + htd.setValue(k, v) + end + end + else + # (3) column family spec + descriptor = hcd(arg, htd) + htd.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT]) if arg[COMPRESSION_COMPACT] + htd.addFamily(hcd(arg, htd)) + end + end + end + + if splits.nil? + # Perform the create table call + @admin.createTable(htd) + else + # Perform the create table call + @admin.createTable(htd, splits) + end + end + + #---------------------------------------------------------------------------------------------- + # Closes a region. + # If server name is nil, we presume region_name is full region name (HRegionInfo.getRegionName). + # If server name is not nil, we presume it is the region's encoded name (HRegionInfo.getEncodedName) + def close_region(region_name, server) + if (server == nil || !closeEncodedRegion?(region_name, server)) + @admin.closeRegion(region_name, server) + end + end + + #---------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------- + # Assign a region + def assign(region_name) + @admin.assign(region_name.to_java_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Unassign a region + def unassign(region_name, force) + @admin.unassign(region_name.to_java_bytes, java.lang.Boolean::valueOf(force)) + end + + #---------------------------------------------------------------------------------------------- + # Move a region + def move(encoded_region_name, server = nil) + @admin.move(encoded_region_name.to_java_bytes, server ? server.to_java_bytes: nil) + end + + #---------------------------------------------------------------------------------------------- + # Returns table's structure description + def describe(table_name) + tables = @admin.listTables.to_a + tables << org.apache.hadoop.hbase.HTableDescriptor::META_TABLEDESC + tables << org.apache.hadoop.hbase.HTableDescriptor::ROOT_TABLEDESC + + tables.each do |t| + # Found the table + return t.to_s if t.getNameAsString == table_name + end + + raise(ArgumentError, "Failed to find table named #{table_name}") + end + + #---------------------------------------------------------------------------------------------- + # Truncates table (deletes all records by recreating the table) + def truncate(table_name, conf = @conf) + raise ArgumentError, "Truncate on index table is not supported." unless !(table_name.end_with?("_idx")) + raise ArgumentError, "Table #{table_name} is not enabled. Enable it first.'" unless enabled?(table_name) + + h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name) + is_index_table_exist = @admin.tableExists(org.apache.hadoop.hbase.index.util.IndexUtils.getIndexTableName(table_name)) + + if is_index_table_exist + table_description = org.apache.hadoop.hbase.index.util.IndexUtils.readIndexedHTableDescriptor(table_name, conf) + else + table_description = h_table.getTableDescriptor() + end + + yield 'Disabling table...' if block_given? + @admin.disableTable(table_name) + + yield 'Dropping table...' if block_given? + @admin.deleteTable(table_name) + + yield 'Creating table...' if block_given? + @admin.createTable(table_description) + end + + #---------------------------------------------------------------------------------------------- + # Truncates table while maintaing region boundaries (deletes all records by recreating the table) + def truncate_preserve(table_name, conf = @conf) + raise ArgumentError, "Truncate on index table is not supported." unless !(table_name.end_with?("_idx")) + raise ArgumentError, "Table #{table_name} is not enabled. Enable it first.'" unless enabled?(table_name) + + h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name) + splitKeys = h_table.getSplitKeys() + is_index_table_exist = @admin.tableExists(org.apache.hadoop.hbase.index.util.IndexUtils.getIndexTableName(table_name)) + + if is_index_table_exist + table_description = org.apache.hadoop.hbase.index.util.IndexUtils.readIndexedHTableDescriptor(table_name, conf) + else + table_description = h_table.getTableDescriptor() + end + + yield 'Disabling table...' if block_given? + @admin.disableTable(table_name) + + yield 'Dropping table...' if block_given? + @admin.deleteTable(table_name) + + yield 'Creating table with region boundaries...' if block_given? + @admin.createTable(table_description, splitKeys) + end + + #---------------------------------------------------------------------------------------------- + + # Check the status of alter command (number of regions reopened) + def alter_status(table_name) + # Table name should be a string + raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) + + # Table should exist + raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) + + status = Pair.new() + begin + status = @admin.getAlterStatus(table_name.to_java_bytes) + if status.getSecond() != 0 + puts "#{status.getSecond() - status.getFirst()}/#{status.getSecond()} regions updated." + else + puts "All regions updated." + end + sleep 1 + end while status != nil && status.getFirst() != 0 + puts "Done." + end + + #---------------------------------------------------------------------------------------------- + # Change table structure or table options + def alter(table_name, wait = true, *args) + # Table name should be a string + raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) + + # Table should exist + raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) + + # There should be at least one argument + raise(ArgumentError, "There should be at least one argument but the table name") if args.empty? + + # Get table descriptor + htd = @admin.getTableDescriptor(table_name.to_java_bytes) + + # Process all args + args.each do |arg| + # Normalize args to support column name only alter specs + arg = { NAME => arg } if arg.kind_of?(String) + + # Normalize args to support shortcut delete syntax + arg = { METHOD => 'delete', NAME => arg['delete'] } if arg['delete'] + + # No method parameter, try to use the args as a column definition + unless method = arg.delete(METHOD) + # Note that we handle owner here, and also below (see (2)) as part of the "METHOD => 'table_att'" table attributes. + # In other words, if OWNER is specified, then METHOD is set to table_att. + # alter 'tablename', {OWNER => 'username'} (that is, METHOD => 'table_att' is not specified). + if arg[OWNER] + htd.setOwnerString(arg[OWNER]) + @admin.modifyTable(table_name.to_java_bytes, htd) + return + end + + descriptor = hcd(arg, htd) + + if arg[COMPRESSION_COMPACT] + descriptor.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT]) + end + column_name = descriptor.getNameAsString + + # If column already exist, then try to alter it. Create otherwise. + if htd.hasFamily(column_name.to_java_bytes) + @admin.modifyColumn(table_name, descriptor) + if wait == true + puts "Updating all regions with the new schema..." + alter_status(table_name) + end + else + @admin.addColumn(table_name, descriptor) + if wait == true + puts "Updating all regions with the new schema..." + alter_status(table_name) + end + end + next + end + + # Delete column family + if method == "delete" + raise(ArgumentError, "NAME parameter missing for delete method") unless arg[NAME] + @admin.deleteColumn(table_name, arg[NAME]) + if wait == true + puts "Updating all regions with the new schema..." + alter_status(table_name) + end + next + end + + # Change table attributes + if method == "table_att" + htd.setMaxFileSize(JLong.valueOf(arg[MAX_FILESIZE])) if arg[MAX_FILESIZE] + htd.setReadOnly(JBoolean.valueOf(arg[READONLY])) if arg[READONLY] + htd.setMemStoreFlushSize(JLong.valueOf(arg[MEMSTORE_FLUSHSIZE])) if arg[MEMSTORE_FLUSHSIZE] + htd.setDeferredLogFlush(JBoolean.valueOf(arg[DEFERRED_LOG_FLUSH])) if arg[DEFERRED_LOG_FLUSH] + # (2) Here, we handle the alternate syntax of ownership setting, where method => 'table_att' is specified. + htd.setOwnerString(arg[OWNER]) if arg[OWNER] + + # set a coprocessor attribute + if arg.kind_of?(Hash) + arg.each do |key, value| + k = String.new(key) # prepare to strip + k.strip! + + if (k =~ /coprocessor/i) + # validate coprocessor specs + v = String.new(value) + v.strip! + if !(v =~ /^([^\|]*)\|([^\|]+)\|[\s]*([\d]*)[\s]*(\|.*)?$/) + raise ArgumentError, "Coprocessor value doesn't match spec: #{v}" + end + + # generate a coprocessor ordinal by checking max id of existing cps + maxId = 0 + htd.getValues().each do |k1, v1| + attrName = org.apache.hadoop.hbase.util.Bytes.toString(k1.get()) + # a cp key is coprocessor$(\d) + if (attrName =~ /coprocessor\$(\d+)/i) + ids = attrName.scan(/coprocessor\$(\d+)/i) + maxId = ids[0][0].to_i if ids[0][0].to_i > maxId + end + end + maxId += 1 + htd.setValue(k + "\$" + maxId.to_s, value) + end + end + end + + if arg[CONFIG] + raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash) + for k,v in arg[CONFIG] + v = v.to_s unless v.nil? + htd.setValue(k, v) + end + end + @admin.modifyTable(table_name.to_java_bytes, htd) + if wait == true + puts "Updating all regions with the new schema..." + alter_status(table_name) + end + next + end + + # Unset table attributes + if method == "table_att_unset" + if arg.kind_of?(Hash) + if (!arg[NAME]) + next + end + if (htd.getValue(arg[NAME]) == nil) + raise ArgumentError, "Can not find attribute: #{arg[NAME]}" + end + htd.remove(arg[NAME].to_java_bytes) + @admin.modifyTable(table_name.to_java_bytes, htd) + if wait == true + puts "Updating all regions with the new schema..." + alter_status(table_name) + end + end + next + end + + # Unknown method + raise ArgumentError, "Unknown method: #{method}" + end + end + + def status(format) + status = @admin.getClusterStatus() + if format == "detailed" + puts("version %s" % [ status.getHBaseVersion() ]) + # Put regions in transition first because usually empty + puts("%d regionsInTransition" % status.getRegionsInTransition().size()) + for k, v in status.getRegionsInTransition() + puts(" %s" % [v]) + end + master_coprocs = java.util.Arrays.toString(@admin.getMasterCoprocessors()) + if master_coprocs != nil + puts("master coprocessors: %s" % master_coprocs) + end + puts("%d live servers" % [ status.getServersSize() ]) + for server in status.getServers() + puts(" %s:%d %d" % \ + [ server.getHostname(), server.getPort(), server.getStartcode() ]) + puts(" %s" % [ status.getLoad(server).toString() ]) + for name, region in status.getLoad(server).getRegionsLoad() + puts(" %s" % [ region.getNameAsString() ]) + puts(" %s" % [ region.toString() ]) + end + end + puts("%d dead servers" % [ status.getDeadServers() ]) + for server in status.getDeadServerNames() + puts(" %s" % [ server ]) + end + elsif format == "simple" + load = 0 + regions = 0 + puts("%d live servers" % [ status.getServersSize() ]) + for server in status.getServers() + puts(" %s:%d %d" % \ + [ server.getHostname(), server.getPort(), server.getStartcode() ]) + puts(" %s" % [ status.getLoad(server).toString() ]) + load += status.getLoad(server).getNumberOfRequests() + regions += status.getLoad(server).getNumberOfRegions() + end + puts("%d dead servers" % [ status.getDeadServers() ]) + for server in status.getDeadServerNames() + puts(" %s" % [ server ]) + end + puts("Aggregate load: %d, regions: %d" % [ load , regions ] ) + else + puts "#{status.getServersSize} servers, #{status.getDeadServers} dead, #{'%.4f' % status.getAverageLoad} average load" + end + end + + #---------------------------------------------------------------------------------------------- + # + # Helper methods + # + + # Does table exist? + def exists?(table_name) + @admin.tableExists(table_name) + end + + #---------------------------------------------------------------------------------------------- + # Is table enabled + def enabled?(table_name) + @admin.isTableEnabled(table_name) + end + + #---------------------------------------------------------------------------------------------- + #Is supplied region name is encoded region name + def closeEncodedRegion?(region_name, server) + @admin.closeRegionWithEncodedRegionName(region_name, server) + end + + #---------------------------------------------------------------------------------------------- + # Return a new HColumnDescriptor made of passed args + def hcd(arg, htd) + # String arg, single parameter constructor + return org.apache.hadoop.hbase.HColumnDescriptor.new(arg) if arg.kind_of?(String) + + raise(ArgumentError, "Column family #{arg} must have a name") unless name = arg[NAME] + + family = htd.getFamily(name.to_java_bytes) + # create it if it's a new family + family ||= org.apache.hadoop.hbase.HColumnDescriptor.new(name.to_java_bytes) + + family.setBlockCacheEnabled(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE) + family.setScope(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE) + family.setInMemory(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY) + family.setTimeToLive(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::TTL])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::TTL) + family.setDataBlockEncoding(org.apache.hadoop.hbase.io.encoding.DataBlockEncoding.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING) + family.setEncodeOnDisk(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::ENCODE_ON_DISK])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::ENCODE_ON_DISK) + family.setBlocksize(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE) + family.setMaxVersions(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS) + family.setMinVersions(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS) + family.setKeepDeletedCells(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS) + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER) + bloomtype = arg[org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER].upcase + unless org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.include?(bloomtype) + raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" ")) + else + family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.valueOf(bloomtype)) + end + end + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION) + compression = arg[org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION].upcase + unless org.apache.hadoop.hbase.io.hfile.Compression::Algorithm.constants.include?(compression) + raise(ArgumentError, "Compression #{compression} is not supported. Use one of " + org.apache.hadoop.hbase.io.hfile.Compression::Algorithm.constants.join(" ")) + else + family.setCompressionType(org.apache.hadoop.hbase.io.hfile.Compression::Algorithm.valueOf(compression)) + end + end + + if arg[CONFIG] + raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash) + for k,v in arg[CONFIG] + v = v.to_s unless v.nil? + family.setValue(k, v) + end + end + return family + end + + #---------------------------------------------------------------------------------------------- + # Enables/disables a region by name + def online(region_name, on_off) + # Open meta table + meta = org.apache.hadoop.hbase.client.HTable.new(org.apache.hadoop.hbase.HConstants::META_TABLE_NAME) + + # Read region info + # FIXME: fail gracefully if can't find the region + region_bytes = region_name.to_java_bytes + g = org.apache.hadoop.hbase.client.Get.new(region_bytes) + g.addColumn(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER) + hri_bytes = meta.get(g).value + + # Change region status + hri = org.apache.hadoop.hbase.util.Writables.getWritable(hri_bytes, org.apache.hadoop.hbase.HRegionInfo.new) + hri.setOffline(on_off) + + # Write it back + put = org.apache.hadoop.hbase.client.Put.new(region_bytes) + put.add(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER, org.apache.hadoop.hbase.util.Writables.getBytes(hri)) + meta.put(put) + end + + #---------------------------------------------------------------------------------------------- + # Take a snapshot of specified table + def snapshot(table, snapshot_name) + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Restore specified snapshot + def restore_snapshot(snapshot_name) + @admin.restoreSnapshot(snapshot_name.to_java_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Create a new table by cloning the snapshot content + def clone_snapshot(snapshot_name, table) + @admin.cloneSnapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Delete specified snapshot + def delete_snapshot(snapshot_name) + @admin.deleteSnapshot(snapshot_name.to_java_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of snapshots + def list_snapshot + @admin.listSnapshots + end + end +end diff --git a/src/main/ruby/hbase/hbase.rb b/src/main/ruby/hbase/hbase.rb new file mode 100644 index 0000000..2c37840 --- /dev/null +++ b/src/main/ruby/hbase/hbase.rb @@ -0,0 +1,60 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +require 'hbase/admin' +require 'hbase/table' +require 'hbase/security' + +module Hbase + class Hbase + attr_accessor :configuration + + def initialize(config = nil) + # Create configuration + if config + self.configuration = config + else + self.configuration = org.apache.hadoop.hbase.HBaseConfiguration.create + # Turn off retries in hbase and ipc. Human doesn't want to wait on N retries. + configuration.setInt("hbase.client.retries.number", 7) + configuration.setInt("ipc.client.connect.max.retries", 3) + end + end + + def admin(formatter) + ::Hbase::Admin.new(configuration, formatter) + end + + # Create new one each time + def table(table, formatter) + ::Hbase::Table.new(configuration, table, formatter) + end + + def replication_admin(formatter) + ::Hbase::RepAdmin.new(configuration, formatter) + end + + def security_admin(formatter) + ::Hbase::SecurityAdmin.new(configuration, formatter) + end + end +end diff --git a/src/main/ruby/hbase/replication_admin.rb b/src/main/ruby/hbase/replication_admin.rb new file mode 100644 index 0000000..f694f5f --- /dev/null +++ b/src/main/ruby/hbase/replication_admin.rb @@ -0,0 +1,82 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin + +module Hbase + class RepAdmin + include HBaseConstants + + def initialize(configuration, formatter) + @replication_admin = org.apache.hadoop.hbase.client.replication.ReplicationAdmin.new(configuration) + @formatter = formatter + end + + #---------------------------------------------------------------------------------------------- + # Add a new peer cluster to replicate to + def add_peer(id, cluster_key) + @replication_admin.addPeer(id, cluster_key) + end + + #---------------------------------------------------------------------------------------------- + # Remove a peer cluster, stops the replication + def remove_peer(id) + @replication_admin.removePeer(id) + end + + #---------------------------------------------------------------------------------------------- + # List all peer clusters + def list_peers + @replication_admin.listPeers + end + + #---------------------------------------------------------------------------------------------- + # Get peer cluster state + def get_peer_state(id) + @replication_admin.getPeerState(id) + end + + #---------------------------------------------------------------------------------------------- + # Restart the replication stream to the specified peer + def enable_peer(id) + @replication_admin.enablePeer(id) + end + + #---------------------------------------------------------------------------------------------- + # Stop the replication stream to the specified peer + def disable_peer(id) + @replication_admin.disablePeer(id) + end + + #---------------------------------------------------------------------------------------------- + # Restart the replication, in an unknown state + def start_replication + @replication_admin.setReplicating(true) + end + + #---------------------------------------------------------------------------------------------- + # Kill switch for replication, stops all its features + def stop_replication + @replication_admin.setReplicating(false) + end + end +end diff --git a/src/main/ruby/hbase/security.rb b/src/main/ruby/hbase/security.rb new file mode 100644 index 0000000..5041ec4 --- /dev/null +++ b/src/main/ruby/hbase/security.rb @@ -0,0 +1,188 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin + +module Hbase + class SecurityAdmin + include HBaseConstants + + def initialize(configuration, formatter) + @config = configuration + @admin = org.apache.hadoop.hbase.client.HBaseAdmin.new(configuration) + @formatter = formatter + end + + #---------------------------------------------------------------------------------------------- + def grant(user, permissions, table_name=nil, family=nil, qualifier=nil) + security_available? + + # TODO: need to validate user name + + # Verify that the specified permission is valid + if (permissions == nil || permissions.length == 0) + raise(ArgumentError, "Invalid permission: no actions associated with user") + end + + if (table_name != nil) + # Table should exist + raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) + + htd = @admin.getTableDescriptor(table_name.to_java_bytes) + + if (family != nil) + raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes) + end + + # invoke cp endpoint to perform access controlse + fambytes = family.to_java_bytes if (family != nil) + qualbytes = qualifier.to_java_bytes if (qualifier != nil) + user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new( + user.to_java_bytes, table_name.to_java_bytes, + fambytes, qualbytes, permissions.to_java_bytes) + else + user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new( + user.to_java_bytes, permissions.to_java_bytes) + end + + meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, + org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) + protocol = meta_table.coprocessorProxy( + org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class, + org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) + begin + protocol.grant(user_permission) + rescue java.io.IOException => e + if !(e.message.include? "java.lang.NoSuchMethodException") + raise e + end + + # Server has not the new API, try the old one + if (table_name == nil) + raise "Global permissions not supported by HBase Server" + end + + tp = org.apache.hadoop.hbase.security.access.TablePermission.new(table_name.to_java_bytes, fambytes, qualbytes, permissions.to_java_bytes) + protocol.grant(user.to_java_bytes, tp) + end + end + + #---------------------------------------------------------------------------------------------- + def revoke(user, table_name=nil, family=nil, qualifier=nil) + security_available? + + # TODO: need to validate user name + + if (table_name != nil) + # Table should exist + raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) + + htd = @admin.getTableDescriptor(table_name.to_java_bytes) + + if (family != nil) + raise(ArgumentError, "Can't find family: #{family}") unless htd.hasFamily(family.to_java_bytes) + end + + # invoke cp endpoint to perform access control + fambytes = family.to_java_bytes if (family != nil) + qualbytes = qualifier.to_java_bytes if (qualifier != nil) + user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new( + user.to_java_bytes, table_name.to_java_bytes, + fambytes, qualbytes, "".to_java_bytes) + else + user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new( + user.to_java_bytes, "".to_java_bytes) + end + + meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, + org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) + protocol = meta_table.coprocessorProxy( + org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class, + org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) + begin + protocol.revoke(user_permission) + rescue java.io.IOException => e + if !(e.message.include? "java.lang.NoSuchMethodException") + raise e + end + + # Server has not the new API, try the old one + if (table_name == nil) + raise "Global permissions not supported by HBase Server" + end + + tp = org.apache.hadoop.hbase.security.access.TablePermission.new(table_name.to_java_bytes, fambytes, qualbytes, "".to_java_bytes) + protocol.revoke(user.to_java_bytes, tp) + end + end + + #---------------------------------------------------------------------------------------------- + def user_permission(table_name=nil) + security_available? + + if (table_name != nil) + raise(ArgumentError, "Can't find table: #{table_name}") unless exists?(table_name) + end + + meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, + org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) + protocol = meta_table.coprocessorProxy( + org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class, + org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) + perms = protocol.getUserPermissions(table_name != nil ? table_name.to_java_bytes : nil) + + res = {} + count = 0 + perms.each do |value| + user_name = String.from_java_bytes(value.getUser) + table = (value.getTable != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getTable) : '' + family = (value.getFamily != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getFamily) : '' + qualifier = (value.getQualifier != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getQualifier) : '' + + action = org.apache.hadoop.hbase.security.access.Permission.new value.getActions + + if block_given? + yield(user_name, "#{table},#{family},#{qualifier}: #{action.to_s}") + else + res[user_name] ||= {} + res[user_name][family + ":" +qualifier] = action + end + count += 1 + end + + return ((block_given?) ? count : res) + end + + # Does table exist? + def exists?(table_name) + @admin.tableExists(table_name) + end + + # Make sure that security classes are available + def security_available?() + begin + org.apache.hadoop.hbase.security.access.AccessControllerProtocol + rescue NameError + raise(ArgumentError, "DISABLED: Security features are not available in this build of HBase") + end + end + + end +end diff --git a/src/main/ruby/hbase/table.rb b/src/main/ruby/hbase/table.rb new file mode 100644 index 0000000..ed41132 --- /dev/null +++ b/src/main/ruby/hbase/table.rb @@ -0,0 +1,360 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +# Wrapper for org.apache.hadoop.hbase.client.HTable + +module Hbase + class Table + include HBaseConstants + attr_reader :table + + def initialize(configuration, table_name, formatter) + @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name) + end + + #---------------------------------------------------------------------------------------------- + # Put a cell 'value' at specified table/row/column + def put(row, column, value, timestamp = nil) + p = org.apache.hadoop.hbase.client.Put.new(row.to_s.to_java_bytes) + family, qualifier = parse_column_name(column) + if timestamp + p.add(family, qualifier, timestamp, value.to_s.to_java_bytes) + else + p.add(family, qualifier, value.to_s.to_java_bytes) + end + @table.put(p) + end + + #---------------------------------------------------------------------------------------------- + # Delete a cell + def delete(row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) + deleteall(row, column, timestamp) + end + + #---------------------------------------------------------------------------------------------- + # Delete a row + def deleteall(row, column = nil, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) + d = org.apache.hadoop.hbase.client.Delete.new(row.to_s.to_java_bytes, timestamp, nil) + if column + family, qualifier = parse_column_name(column) + d.deleteColumns(family, qualifier, timestamp) + end + @table.delete(d) + end + + #---------------------------------------------------------------------------------------------- + # Increment a counter atomically + def incr(row, column, value = nil) + value ||= 1 + family, qualifier = parse_column_name(column) + @table.incrementColumnValue(row.to_s.to_java_bytes, family, qualifier, value) + end + + #---------------------------------------------------------------------------------------------- + # Count rows in a table + def count(interval = 1000, caching_rows = 10) + # We can safely set scanner caching with the first key only filter + scan = org.apache.hadoop.hbase.client.Scan.new + scan.cache_blocks = false + scan.caching = caching_rows + scan.setFilter(org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter.new) + + # Run the scanner + scanner = @table.getScanner(scan) + count = 0 + iter = scanner.iterator + + # Iterate results + while iter.hasNext + row = iter.next + count += 1 + next unless (block_given? && count % interval == 0) + # Allow command modules to visualize counting process + yield(count, + org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow)) + end + + # Return the counter + return count + end + + #---------------------------------------------------------------------------------------------- + # Get from table + def get(row, *args) + get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) + maxlength = -1 + + # Normalize args + args = args.first if args.first.kind_of?(Hash) + if args.kind_of?(String) || args.kind_of?(Array) + columns = [ args ].flatten.compact + args = { COLUMNS => columns } + end + + # + # Parse arguments + # + unless args.kind_of?(Hash) + raise ArgumentError, "Failed parse of of #{args.inspect}, #{args.class}" + end + + # Get maxlength parameter if passed + maxlength = args.delete(MAXLENGTH) if args[MAXLENGTH] + filter = args.delete(FILTER) if args[FILTER] + + unless args.empty? + columns = args[COLUMN] || args[COLUMNS] + if args[VERSIONS] + vers = args[VERSIONS] + else + vers = 1 + end + if columns + # Normalize types, convert string to an array of strings + columns = [ columns ] if columns.is_a?(String) + + # At this point it is either an array or some unsupported stuff + unless columns.kind_of?(Array) + raise ArgumentError, "Failed parse column argument type #{args.inspect}, #{args.class}" + end + + # Get each column name and add it to the filter + columns.each do |column| + family, qualifier = parse_column_name(column.to_s) + if qualifier + get.addColumn(family, qualifier) + else + get.addFamily(family) + end + end + + # Additional params + get.setMaxVersions(vers) + get.setTimeStamp(args[TIMESTAMP]) if args[TIMESTAMP] + get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] + else + # May have passed TIMESTAMP and row only; wants all columns from ts. + unless ts = args[TIMESTAMP] || tr = args[TIMERANGE] + raise ArgumentError, "Failed parse of #{args.inspect}, #{args.class}" + end + + get.setMaxVersions(vers) + # Set the timestamp/timerange + get.setTimeStamp(ts.to_i) if args[TIMESTAMP] + get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] + end + end + + unless filter.class == String + get.setFilter(filter) + else + get.setFilter(org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter)) + end + + # Call hbase for the results + result = @table.get(get) + return nil if result.isEmpty + + # Print out results. Result can be Cell or RowResult. + res = {} + result.list.each do |kv| + family = String.from_java_bytes(kv.getFamily) + qualifier = org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getQualifier) + + column = "#{family}:#{qualifier}" + value = to_string(column, kv, maxlength) + + if block_given? + yield(column, value) + else + res[column] = value + end + end + + # If block given, we've yielded all the results, otherwise just return them + return ((block_given?) ? nil : res) + end + + #---------------------------------------------------------------------------------------------- + # Fetches and decodes a counter value from hbase + def get_counter(row, column) + family, qualifier = parse_column_name(column.to_s) + # Format get request + get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) + get.addColumn(family, qualifier) + get.setMaxVersions(1) + + # Call hbase + result = @table.get(get) + return nil if result.isEmpty + + # Fetch cell value + cell = result.list[0] + org.apache.hadoop.hbase.util.Bytes::toLong(cell.getValue) + end + + #---------------------------------------------------------------------------------------------- + # Scans whole table or a range of keys and returns rows matching specific criterias + def scan(args = {}) + unless args.kind_of?(Hash) + raise ArgumentError, "Arguments should be a hash. Failed to parse #{args.inspect}, #{args.class}" + end + + limit = args.delete("LIMIT") || -1 + maxlength = args.delete("MAXLENGTH") || -1 + + if args.any? + filter = args["FILTER"] + startrow = args["STARTROW"] || '' + stoprow = args["STOPROW"] + timestamp = args["TIMESTAMP"] + columns = args["COLUMNS"] || args["COLUMN"] || [] + cache_blocks = args["CACHE_BLOCKS"] || true + cache = args["CACHE"] || 0 + versions = args["VERSIONS"] || 1 + timerange = args[TIMERANGE] + raw = args["RAW"] || false + + # Normalize column names + columns = [columns] if columns.class == String + unless columns.kind_of?(Array) + raise ArgumentError.new("COLUMNS must be specified as a String or an Array") + end + + scan = if stoprow + org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes, stoprow.to_java_bytes) + else + org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes) + end + + columns.each do |c| + family, qualifier = parse_column_name(c.to_s) + if qualifier + scan.addColumn(family, qualifier) + else + scan.addFamily(family) + end + end + + unless filter.class == String + scan.setFilter(filter) + else + scan.setFilter(org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter)) + end + + scan.setTimeStamp(timestamp) if timestamp + scan.setCacheBlocks(cache_blocks) + scan.setCaching(cache) if cache > 0 + scan.setMaxVersions(versions) if versions > 1 + scan.setTimeRange(timerange[0], timerange[1]) if timerange + scan.setRaw(raw) + else + scan = org.apache.hadoop.hbase.client.Scan.new + end + + # Start the scanner + scanner = @table.getScanner(scan) + count = 0 + res = {} + iter = scanner.iterator + + # Iterate results + while iter.hasNext + if limit > 0 && count >= limit + break + end + + row = iter.next + key = org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow) + + row.list.each do |kv| + family = String.from_java_bytes(kv.getFamily) + qualifier = org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getQualifier) + + column = "#{family}:#{qualifier}" + cell = to_string(column, kv, maxlength) + + if block_given? + yield(key, "column=#{column}, #{cell}") + else + res[key] ||= {} + res[key][column] = cell + end + end + + # One more row processed + count += 1 + end + + return ((block_given?) ? count : res) + end + + #---------------------------------------------------------------------------------------- + # Helper methods + + # Returns a list of column names in the table + def get_all_columns + @table.table_descriptor.getFamilies.map do |family| + "#{family.getNameAsString}:" + end + end + + # Checks if current table is one of the 'meta' tables + def is_meta_table? + tn = @table.table_name + org.apache.hadoop.hbase.util.Bytes.equals(tn, org.apache.hadoop.hbase.HConstants::META_TABLE_NAME) || org.apache.hadoop.hbase.util.Bytes.equals(tn, org.apache.hadoop.hbase.HConstants::ROOT_TABLE_NAME) + end + + # Returns family and (when has it) qualifier for a column name + def parse_column_name(column) + split = org.apache.hadoop.hbase.KeyValue.parseColumn(column.to_java_bytes) + return split[0], (split.length > 1) ? split[1] : nil + end + + # Make a String of the passed kv + # Intercept cells whose format we know such as the info:regioninfo in .META. + def to_string(column, kv, maxlength = -1) + if is_meta_table? + if column == 'info:regioninfo' or column == 'info:splitA' or column == 'info:splitB' + hri = org.apache.hadoop.hbase.util.Writables.getHRegionInfoOrNull(kv.getValue) + return "timestamp=%d, value=%s" % [kv.getTimestamp, hri.toString] + end + if column == 'info:serverstartcode' + if kv.getValue.length > 0 + str_val = org.apache.hadoop.hbase.util.Bytes.toLong(kv.getValue) + else + str_val = org.apache.hadoop.hbase.util.Bytes.toStringBinary(kv.getValue) + end + return "timestamp=%d, value=%s" % [kv.getTimestamp, str_val] + end + end + + if kv.isDelete + val = "timestamp=#{kv.getTimestamp}, type=#{org.apache.hadoop.hbase.KeyValue::Type::codeToType(kv.getType)}" + else + val = "timestamp=#{kv.getTimestamp}, value=#{org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getValue)}" + end + (maxlength != -1) ? val[0, maxlength] : val + end + + end +end diff --git a/src/main/ruby/irb/hirb.rb b/src/main/ruby/irb/hirb.rb new file mode 100644 index 0000000..584f700 --- /dev/null +++ b/src/main/ruby/irb/hirb.rb @@ -0,0 +1,60 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require 'rbconfig' + +module IRB + WINDOZE = Config::CONFIG['host_os'] =~ /mswin|mingw/ + + # Subclass of IRB so can intercept methods + class HIRB < Irb + def initialize + # This is ugly. Our 'help' method above provokes the following message + # on irb construction: 'irb: warn: can't alias help from irb_help.' + # Below, we reset the output so its pointed at /dev/null during irb + # construction just so this message does not come out after we emit + # the banner. Other attempts at playing with the hash of methods + # down in IRB didn't seem to work. I think the worst thing that can + # happen is the shell exiting because of failed IRB construction with + # no error (though we're not blanking STDERR) + begin + # Map the '/dev/null' according to the runing platform + # Under Windows platform the 'dev/null' is not fully compliant with unix, + # and the 'NUL' object need to be use instead. + devnull = "/dev/null" + devnull = "NUL" if WINDOZE + f = File.open(devnull, "w") + $stdout = f + super + ensure + f.close() + $stdout = STDOUT + end + end + + def output_value + # Suppress output if last_value is 'nil' + # Otherwise, when user types help, get ugly 'nil' + # after all output. + if @context.last_value != nil + super + end + end + end +end diff --git a/src/main/ruby/shell.rb b/src/main/ruby/shell.rb new file mode 100644 index 0000000..8d831bc --- /dev/null +++ b/src/main/ruby/shell.rb @@ -0,0 +1,315 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Shell commands module +module Shell + @@commands = {} + def self.commands + @@commands + end + + @@command_groups = {} + def self.command_groups + @@command_groups + end + + def self.load_command(name, group) + return if commands[name] + + # Register command in the group + raise ArgumentError, "Unknown group: #{group}" unless command_groups[group] + command_groups[group][:commands] << name + + # Load command + begin + require "shell/commands/#{name}" + klass_name = name.to_s.gsub(/(?:^|_)(.)/) { $1.upcase } # camelize + commands[name] = eval("Commands::#{klass_name}") + rescue => e + raise "Can't load hbase shell command: #{name}. Error: #{e}\n#{e.backtrace.join("\n")}" + end + end + + def self.load_command_group(group, opts) + raise ArgumentError, "No :commands for group #{group}" unless opts[:commands] + + command_groups[group] = { + :commands => [], + :command_names => opts[:commands], + :full_name => opts[:full_name] || group, + :comment => opts[:comment] + } + + opts[:commands].each do |command| + load_command(command, group) + end + end + + #---------------------------------------------------------------------- + class Shell + attr_accessor :hbase + attr_accessor :formatter + + @debug = false + attr_accessor :debug + + def initialize(hbase, formatter) + self.hbase = hbase + self.formatter = formatter + end + + def hbase_admin + @hbase_admin ||= hbase.admin(formatter) + end + + def hbase_table(name) + hbase.table(name, formatter) + end + + def hbase_replication_admin + @hbase_replication_admin ||= hbase.replication_admin(formatter) + end + + def hbase_security_admin + @hbase_security_admin ||= hbase.security_admin(formatter) + end + + def export_commands(where) + ::Shell.commands.keys.each do |cmd| + where.send :instance_eval, <<-EOF + def #{cmd}(*args) + @shell.command('#{cmd}', *args) + puts + end + EOF + end + end + + def command_instance(command) + ::Shell.commands[command.to_s].new(self) + end + + def command(command, *args) + command_instance(command).command_safe(self.debug, *args) + end + + def print_banner + puts "HBase Shell; enter 'help' for list of supported commands." + puts 'Type "exit" to leave the HBase Shell' + print 'Version ' + command('version') + puts + end + + def help_multi_command(command) + puts "Command: #{command}" + puts command_instance(command).help + puts + return nil + end + + def help_command(command) + puts command_instance(command).help + return nil + end + + def help_group(group_name) + group = ::Shell.command_groups[group_name.to_s] + group[:commands].sort.each { |cmd| help_multi_command(cmd) } + if group[:comment] + puts '-' * 80 + puts + puts group[:comment] + puts + end + return nil + end + + def help(command = nil) + if command + return help_command(command) if ::Shell.commands[command.to_s] + return help_group(command) if ::Shell.command_groups[command.to_s] + puts "ERROR: Invalid command or command group name: #{command}" + puts + end + + puts help_header + puts + puts 'COMMAND GROUPS:' + ::Shell.command_groups.each do |name, group| + puts " Group name: " + name + puts " Commands: " + group[:command_names].sort.join(', ') + puts + end + unless command + puts 'SHELL USAGE:' + help_footer + end + return nil + end + + def help_header + return "HBase Shell, version #{org.apache.hadoop.hbase.util.VersionInfo.getVersion()}, " + + "r#{org.apache.hadoop.hbase.util.VersionInfo.getRevision()}, " + + "#{org.apache.hadoop.hbase.util.VersionInfo.getDate()}" + "\n" + + "Type 'help \"COMMAND\"', (e.g. 'help \"get\"' -- the quotes are necessary) for help on a specific command.\n" + + "Commands are grouped. Type 'help \"COMMAND_GROUP\"', (e.g. 'help \"general\"') for help on a command group." + end + + def help_footer + puts <<-HERE +Quote all names in HBase Shell such as table and column names. Commas delimit +command parameters. Type after entering a command to run it. +Dictionaries of configuration used in the creation and alteration of tables are +Ruby Hashes. They look like this: + + {'key1' => 'value1', 'key2' => 'value2', ...} + +and are opened and closed with curley-braces. Key/values are delimited by the +'=>' character combination. Usually keys are predefined constants such as +NAME, VERSIONS, COMPRESSION, etc. Constants do not need to be quoted. Type +'Object.constants' to see a (messy) list of all constants in the environment. + +If you are using binary keys or values and need to enter them in the shell, use +double-quote'd hexadecimal representation. For example: + + hbase> get 't1', "key\\x03\\x3f\\xcd" + hbase> get 't1', "key\\003\\023\\011" + hbase> put 't1', "test\\xef\\xff", 'f1:', "\\x01\\x33\\x40" + +The HBase shell is the (J)Ruby IRB with the above HBase-specific commands added. +For more on the HBase Shell, see http://hbase.apache.org/docs/current/book.html + HERE + end + end +end + +# Load commands base class +require 'shell/commands' + +# Load all commands +Shell.load_command_group( + 'general', + :full_name => 'GENERAL HBASE SHELL COMMANDS', + :commands => %w[ + status + version + whoami + ] +) + +Shell.load_command_group( + 'ddl', + :full_name => 'TABLES MANAGEMENT COMMANDS', + :commands => %w[ + alter + create + describe + disable + disable_all + is_disabled + drop + drop_all + enable + enable_all + is_enabled + exists + list + show_filters + alter_status + alter_async + ] +) + +Shell.load_command_group( + 'dml', + :full_name => 'DATA MANIPULATION COMMANDS', + :commands => %w[ + count + delete + deleteall + get + get_counter + incr + put + scan + truncate + truncate_preserve + ] +) + +Shell.load_command_group( + 'tools', + :full_name => 'HBASE SURGERY TOOLS', + :comment => "WARNING: Above commands are for 'experts'-only as misuse can damage an install", + :commands => %w[ + assign + balancer + balance_switch + close_region + compact + flush + major_compact + move + split + unassign + zk_dump + hlog_roll + ] +) + +Shell.load_command_group( + 'replication', + :full_name => 'CLUSTER REPLICATION TOOLS', + :comment => "In order to use these tools, hbase.replication must be true. enabling/disabling is currently unsupported", + :commands => %w[ + add_peer + remove_peer + list_peers + enable_peer + disable_peer + start_replication + stop_replication + ] +) + +Shell.load_command_group( + 'snapshot', + :full_name => 'CLUSTER SNAPSHOT TOOLS', + :commands => %w[ + snapshot + clone_snapshot + restore_snapshot + delete_snapshot + list_snapshots + ] +) + +Shell.load_command_group( + 'security', + :full_name => 'SECURITY TOOLS', + :comment => "NOTE: Above commands are only applicable if running with the AccessController coprocessor", + :commands => %w[ + grant + revoke + user_permission + ] +) + diff --git a/src/main/ruby/shell/commands.rb b/src/main/ruby/shell/commands.rb new file mode 100644 index 0000000..af6df33 --- /dev/null +++ b/src/main/ruby/shell/commands.rb @@ -0,0 +1,85 @@ +# +# Copyright 2009 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Command + attr_accessor :shell + + def initialize(shell) + self.shell = shell + end + + def command_safe(debug, *args) + translate_hbase_exceptions(*args) { command(*args) } + rescue => e + puts + puts "ERROR: #{e}" + puts "Backtrace: #{e.backtrace.join("\n ")}" if debug + puts + puts "Here is some help for this command:" + puts help + puts + ensure + return nil + end + + def admin + shell.hbase_admin + end + + def table(name) + shell.hbase_table(name) + end + + def replication_admin + shell.hbase_replication_admin + end + + def security_admin + shell.hbase_security_admin + end + + #---------------------------------------------------------------------- + + def formatter + shell.formatter + end + + def format_simple_command + now = Time.now + yield + formatter.header + formatter.footer(now) + end + + def translate_hbase_exceptions(*args) + yield + rescue org.apache.hadoop.hbase.TableNotFoundException + raise "Unknown table #{args.first}!" + rescue org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException + valid_cols = table(args.first).get_all_columns.map { |c| c + '*' } + raise "Unknown column family! Valid column names: #{valid_cols.join(", ")}" + rescue org.apache.hadoop.hbase.TableExistsException + raise "Table already exists: #{args.first}!" + end + end + end +end diff --git a/src/main/ruby/shell/commands/add_peer.rb b/src/main/ruby/shell/commands/add_peer.rb new file mode 100644 index 0000000..7669fb7 --- /dev/null +++ b/src/main/ruby/shell/commands/add_peer.rb @@ -0,0 +1,44 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AddPeer< Command + def help + return <<-EOF +Add a peer cluster to replicate to, the id must be a short and +the cluster key is composed like this: +hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent +This gives a full path for HBase to connect to another cluster. +Examples: + + hbase> add_peer '1', "server1.cie.com:2181:/hbase" + hbase> add_peer '2', "zk1,zk2,zk3:2182:/hbase-prod" +EOF + end + + def command(id, cluster_key) + format_simple_command do + replication_admin.add_peer(id, cluster_key) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/alter.rb b/src/main/ruby/shell/commands/alter.rb new file mode 100644 index 0000000..3be8f0a --- /dev/null +++ b/src/main/ruby/shell/commands/alter.rb @@ -0,0 +1,90 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Alter < Command + def help + return <<-EOF +Alter column family schema; pass table name and a dictionary +specifying new column family schema. Dictionaries are described +on the main help command output. Dictionary must include name +of column family to alter. For example, + +To change or add the 'f1' column family in table 't1' from defaults +to instead keep a maximum of 5 cell VERSIONS, do: + + hbase> alter 't1', NAME => 'f1', VERSIONS => 5 + +To delete the 'f1' column family in table 't1', do: + + hbase> alter 't1', NAME => 'f1', METHOD => 'delete' + +or a shorter version: + + hbase> alter 't1', 'delete' => 'f1' + +You can also change the column family config by set attribute CONFIG like this: + hbase> alter 'test', NAME=>'f', CONFIG => {'hbase.hstore.compaction.min' => '5'} + +You can also change table-scope attributes like MAX_FILESIZE +MEMSTORE_FLUSHSIZE, READONLY, and DEFERRED_LOG_FLUSH. + +For example, to change the max size of a family to 128MB, do: + + hbase> alter 't1', METHOD => 'table_att', MAX_FILESIZE => '134217728' + +You can also change the table-scope by set attribute CONFIG like this: + hbase> alter 'test', METHOD=>'table_att', CONFIG => {'hbase.hstore.compaction.min' => '5'} + +You can add a table coprocessor by setting a table coprocessor attribute: + + hbase> alter 't1', METHOD => 'table_att', + 'coprocessor'=>'hdfs:///foo.jar|com.foo.FooRegionObserver|1001|arg1=1,arg2=2' + +Since you can have multiple coprocessors configured for a table, a +sequence number will be automatically appended to the attribute name +to uniquely identify it. + +The coprocessor attribute must match the pattern below in order for +the framework to understand how to load the coprocessor classes: + + [coprocessor jar file location] | class name | [priority] | [arguments] + +You can also remove a table-scope attribute: + + hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'MAX_FILESIZE' + + hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'coprocessor$1' + +There could be more than one alteration in one command: + + hbase> alter 't1', {NAME => 'f1'}, {NAME => 'f2', METHOD => 'delete'} +EOF + end + + def command(table, *args) + format_simple_command do + admin.alter(table, true, *args) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/alter_async.rb b/src/main/ruby/shell/commands/alter_async.rb new file mode 100644 index 0000000..01dfd99 --- /dev/null +++ b/src/main/ruby/shell/commands/alter_async.rb @@ -0,0 +1,66 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AlterAsync < Command + def help + return <<-EOF +Alter column family schema, does not wait for all regions to receive the +schema changes. Pass table name and a dictionary specifying new column +family schema. Dictionaries are described on the main help command output. +Dictionary must include name of column family to alter. For example, + +To change or add the 'f1' column family in table 't1' from defaults +to instead keep a maximum of 5 cell VERSIONS, do: + + hbase> alter_async 't1', NAME => 'f1', VERSIONS => 5 + +To delete the 'f1' column family in table 't1', do: + + hbase> alter_async 't1', NAME => 'f1', METHOD => 'delete' + +or a shorter version: + + hbase> alter_async 't1', 'delete' => 'f1' + +You can also change table-scope attributes like MAX_FILESIZE +MEMSTORE_FLUSHSIZE, READONLY, and DEFERRED_LOG_FLUSH. + +For example, to change the max size of a family to 128MB, do: + + hbase> alter 't1', METHOD => 'table_att', MAX_FILESIZE => '134217728' + +There could be more than one alteration in one command: + + hbase> alter 't1', {NAME => 'f1'}, {NAME => 'f2', METHOD => 'delete'} + +To check if all the regions have been updated, use alter_status +EOF + end + + def command(table, *args) + format_simple_command do + admin.alter(table, false, *args) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/alter_status.rb b/src/main/ruby/shell/commands/alter_status.rb new file mode 100644 index 0000000..cc79e11 --- /dev/null +++ b/src/main/ruby/shell/commands/alter_status.rb @@ -0,0 +1,38 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AlterStatus < Command + def help + return <<-EOF +Get the status of the alter command. Indicates the number of regions of the +table that have received the updated schema +Pass table name. + +hbase> alter_status 't1' +EOF + end + def command(table) + admin.alter_status(table) + end + end + end +end diff --git a/src/main/ruby/shell/commands/assign.rb b/src/main/ruby/shell/commands/assign.rb new file mode 100644 index 0000000..4c83d3c --- /dev/null +++ b/src/main/ruby/shell/commands/assign.rb @@ -0,0 +1,39 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Assign < Command + def help + return <<-EOF +Assign a region.Use with caution.If region already assigned, +this command will just go ahead and reassign +the region anyways. For experts only. +EOF + end + + def command(region_name) + format_simple_command do + admin.assign(region_name) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/balance_switch.rb b/src/main/ruby/shell/commands/balance_switch.rb new file mode 100644 index 0000000..0eac765 --- /dev/null +++ b/src/main/ruby/shell/commands/balance_switch.rb @@ -0,0 +1,43 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class BalanceSwitch < Command + def help + return <<-EOF +Enable/Disable balancer. Returns previous balancer state. +Examples: + + hbase> balance_switch true + hbase> balance_switch false +EOF + end + + def command(enableDisable) + format_simple_command do + formatter.row([ + admin.balance_switch(enableDisable)? "true" : "false" + ]) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/balancer.rb b/src/main/ruby/shell/commands/balancer.rb new file mode 100644 index 0000000..05ecd3a --- /dev/null +++ b/src/main/ruby/shell/commands/balancer.rb @@ -0,0 +1,41 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Balancer < Command + def help + return <<-EOF +Trigger the cluster balancer. Returns true if balancer ran and was able to +tell the region servers to unassign all the regions to balance (the re-assignment itself is async). +Otherwise false (Will not run if regions in transition). +EOF + end + + def command() + format_simple_command do + formatter.row([ + admin.balancer()? "true": "false" + ]) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/clone_snapshot.rb b/src/main/ruby/shell/commands/clone_snapshot.rb new file mode 100644 index 0000000..43d240f --- /dev/null +++ b/src/main/ruby/shell/commands/clone_snapshot.rb @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CloneSnapshot < Command + def help + return <<-EOF +Create a new table by cloning the snapshot content. +There're no copies of data involved. +And writing on the newly created table will not influence the snapshot data. + +Examples: + hbase> clone_snapshot 'snapshotName', 'tableName' +EOF + end + + def command(snapshot_name, table) + format_simple_command do + admin.clone_snapshot(snapshot_name, table) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/close_region.rb b/src/main/ruby/shell/commands/close_region.rb new file mode 100644 index 0000000..f1010ac --- /dev/null +++ b/src/main/ruby/shell/commands/close_region.rb @@ -0,0 +1,58 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CloseRegion < Command + def help + return <<-EOF +Close a single region. Ask the master to close a region out on the cluster +or if 'SERVER_NAME' is supplied, ask the designated hosting regionserver to +close the region directly. Closing a region, the master expects 'REGIONNAME' +to be a fully qualified region name. When asking the hosting regionserver to +directly close a region, you pass the regions' encoded name only. A region +name looks like this: + + TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. + +The trailing period is part of the regionserver name. A region's encoded name +is the hash at the end of a region name; e.g. 527db22f95c8a9e0116f0cc13c680396 +(without the period). A 'SERVER_NAME' is its host, port plus startcode. For +example: host187.example.com,60020,1289493121758 (find servername in master ui +or when you do detailed status in shell). This command will end up running +close on the region hosting regionserver. The close is done without the +master's involvement (It will not know of the close). Once closed, region will +stay closed. Use assign to reopen/reassign. Use unassign or move to assign +the region elsewhere on cluster. Use with caution. For experts only. +Examples: + + hbase> close_region 'REGIONNAME' + hbase> close_region 'REGIONNAME', 'SERVER_NAME' +EOF + end + + def command(region_name, server = nil) + format_simple_command do + admin.close_region(region_name, server) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/compact.rb b/src/main/ruby/shell/commands/compact.rb new file mode 100644 index 0000000..fa0b82e --- /dev/null +++ b/src/main/ruby/shell/commands/compact.rb @@ -0,0 +1,48 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Compact < Command + def help + return <<-EOF + Compact all regions in passed table or pass a region row + to compact an individual region. You can also compact a single column + family within a region. + Examples: + Compact all regions in a table: + hbase> compact 't1' + Compact an entire region: + hbase> compact 'r1' + Compact only a column family within a region: + hbase> compact 'r1', 'c1' + Compact a column family within a table: + hbase> compact 't1', 'c1' + EOF + end + + def command(table_or_region_name, family = nil) + format_simple_command do + admin.compact(table_or_region_name, family) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/count.rb b/src/main/ruby/shell/commands/count.rb new file mode 100644 index 0000000..6596441 --- /dev/null +++ b/src/main/ruby/shell/commands/count.rb @@ -0,0 +1,61 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Count < Command + def help + return <<-EOF +Count the number of rows in a table. This operation may take a LONG +time (Run '$HADOOP_HOME/bin/hadoop jar hbase.jar rowcount' to run a +counting mapreduce job). Current count is shown every 1000 rows by +default. Count interval may be optionally specified. Scan caching +is enabled on count scans by default. Default cache size is 10 rows. +If your rows are small in size, you may want to increase this +parameter. Examples: + + hbase> count 't1' + hbase> count 't1', INTERVAL => 100000 + hbase> count 't1', CACHE => 1000 + hbase> count 't1', INTERVAL => 10, CACHE => 1000 +EOF + end + + def command(table, params = {}) + # If the second parameter is an integer, then it is the old command syntax + params = { 'INTERVAL' => params } if params.kind_of?(Fixnum) + + # Merge params with defaults + params = { + 'INTERVAL' => 1000, + 'CACHE' => 10 + }.merge(params) + + # Call the counter method + now = Time.now + formatter.header + count = table(table).count(params['INTERVAL'].to_i, params['CACHE'].to_i) do |cnt, row| + formatter.row([ "Current count: #{cnt}, row: #{row}" ]) + end + formatter.footer(now, count) + end + end + end +end diff --git a/src/main/ruby/shell/commands/create.rb b/src/main/ruby/shell/commands/create.rb new file mode 100644 index 0000000..14c1b0f --- /dev/null +++ b/src/main/ruby/shell/commands/create.rb @@ -0,0 +1,51 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Create < Command + def help + return <<-EOF +Create table; pass table name, a dictionary of specifications per +column family, and optionally a dictionary of table configuration. +Dictionaries are described below in the GENERAL NOTES section. +Examples: + + hbase> create 't1', {NAME => 'f1', VERSIONS => 5} + hbase> create 't1', {NAME => 'f1'}, {NAME => 'f2'}, {NAME => 'f3'} + hbase> # The above in shorthand would be the following: + hbase> create 't1', 'f1', 'f2', 'f3' + hbase> create 't1', {NAME => 'f1', VERSIONS => 1, TTL => 2592000, BLOCKCACHE => true} + hbase> create 't1', 'f1', {SPLITS => ['10', '20', '30', '40']} + hbase> create 't1', 'f1', {SPLITS_FILE => 'splits.txt'} + hbase> # Optionally pre-split the table into NUMREGIONS, using + hbase> # SPLITALGO ("HexStringSplit", "UniformSplit" or classname) + hbase> create 't1', 'f1', {NUMREGIONS => 15, SPLITALGO => 'HexStringSplit'} +EOF + end + + def command(table, *args) + format_simple_command do + admin.create(table, *args) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/delete.rb b/src/main/ruby/shell/commands/delete.rb new file mode 100644 index 0000000..12bc405 --- /dev/null +++ b/src/main/ruby/shell/commands/delete.rb @@ -0,0 +1,43 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Delete < Command + def help + return <<-EOF +Put a delete cell value at specified table/row/column and optionally +timestamp coordinates. Deletes must match the deleted cell's +coordinates exactly. When scanning, a delete cell suppresses older +versions. To delete a cell from 't1' at row 'r1' under column 'c1' +marked with the time 'ts1', do: + + hbase> delete 't1', 'r1', 'c1', ts1 +EOF + end + + def command(table, row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) + format_simple_command do + table(table).delete(row, column, timestamp) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/delete_snapshot.rb b/src/main/ruby/shell/commands/delete_snapshot.rb new file mode 100644 index 0000000..b8c3791 --- /dev/null +++ b/src/main/ruby/shell/commands/delete_snapshot.rb @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DeleteSnapshot < Command + def help + return <<-EOF +Delete a specified snapshot. Examples: + + hbase> delete_snapshot 'snapshotName', +EOF + end + + def command(snapshot_name) + format_simple_command do + admin.delete_snapshot(snapshot_name) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/deleteall.rb b/src/main/ruby/shell/commands/deleteall.rb new file mode 100644 index 0000000..5731b60 --- /dev/null +++ b/src/main/ruby/shell/commands/deleteall.rb @@ -0,0 +1,42 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Deleteall < Command + def help + return <<-EOF +Delete all cells in a given row; pass a table name, row, and optionally +a column and timestamp. Examples: + + hbase> deleteall 't1', 'r1' + hbase> deleteall 't1', 'r1', 'c1' + hbase> deleteall 't1', 'r1', 'c1', ts1 +EOF + end + + def command(table, row, column = nil, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) + format_simple_command do + table(table).deleteall(row, column, timestamp) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/describe.rb b/src/main/ruby/shell/commands/describe.rb new file mode 100644 index 0000000..0f35507 --- /dev/null +++ b/src/main/ruby/shell/commands/describe.rb @@ -0,0 +1,42 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Describe < Command + def help + return <<-EOF +Describe the named table. For example: + hbase> describe 't1' +EOF + end + + def command(table) + now = Time.now + + desc = admin.describe(table) + + formatter.header([ "DESCRIPTION", "ENABLED" ], [ 64 ]) + formatter.row([ desc, admin.enabled?(table).to_s ], true, [ 64 ]) + formatter.footer(now) + end + end + end +end diff --git a/src/main/ruby/shell/commands/disable.rb b/src/main/ruby/shell/commands/disable.rb new file mode 100644 index 0000000..34c5f9c --- /dev/null +++ b/src/main/ruby/shell/commands/disable.rb @@ -0,0 +1,37 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Disable < Command + def help + return <<-EOF +Start disable of named table: e.g. "hbase> disable 't1'" +EOF + end + + def command(table) + format_simple_command do + admin.disable(table) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/disable_all.rb b/src/main/ruby/shell/commands/disable_all.rb new file mode 100644 index 0000000..b7c9cf5 --- /dev/null +++ b/src/main/ruby/shell/commands/disable_all.rb @@ -0,0 +1,50 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DisableAll < Command + def help + return <<-EOF +Disable all of tables matching the given regex: + +hbase> disable_all 't.*' +EOF + end + + def command(regex) + regex = /^#{regex}$/ unless regex.is_a?(Regexp) + list = admin.list.grep(regex) + count = list.size + list.each do |table| + formatter.row([ table ]) + end + puts "\nDisable the above #{count} tables (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No tables matched the regex #{regex.to_s}" if count == 0 + return unless answer =~ /y.*/i + failed = admin.disable_all(regex) + puts "#{count - failed.size} tables successfully disabled" + puts "#{failed.size} tables not disabled due to an exception: #{failed.join ','}" unless failed.size == 0 + end + end + end +end diff --git a/src/main/ruby/shell/commands/disable_peer.rb b/src/main/ruby/shell/commands/disable_peer.rb new file mode 100644 index 0000000..da9941a --- /dev/null +++ b/src/main/ruby/shell/commands/disable_peer.rb @@ -0,0 +1,42 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DisablePeer< Command + def help + return <<-EOF +Stops the replication stream to the specified cluster, but still +keeps track of new edits to replicate. + +Examples: + + hbase> disable_peer '1' +EOF + end + + def command(id) + format_simple_command do + replication_admin.disable_peer(id) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/drop.rb b/src/main/ruby/shell/commands/drop.rb new file mode 100644 index 0000000..40831d2 --- /dev/null +++ b/src/main/ruby/shell/commands/drop.rb @@ -0,0 +1,37 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Drop < Command + def help + return <<-EOF +Drop the named table. Table must first be disabled: e.g. "hbase> drop 't1'" +EOF + end + + def command(table) + format_simple_command do + admin.drop(table) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/drop_all.rb b/src/main/ruby/shell/commands/drop_all.rb new file mode 100644 index 0000000..dcbefa3 --- /dev/null +++ b/src/main/ruby/shell/commands/drop_all.rb @@ -0,0 +1,50 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DropAll < Command + def help + return <<-EOF +Drop all of the tables matching the given regex: + +hbase> drop_all 't.*' +EOF + end + + def command(regex) + regex = /^#{regex}$/ unless regex.is_a?(Regexp) + list = admin.list.grep(regex) + count = list.size + list.each do |table| + formatter.row([ table ]) + end + puts "\nDrop the above #{count} tables (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No tables matched the regex #{regex.to_s}" if count == 0 + return unless answer =~ /y.*/i + failed = admin.drop_all(regex) + puts "#{count - failed.size} tables successfully dropped" + puts "#{failed.size} tables not dropped due to an exception: #{failed.join ','}" unless failed.size == 0 + end + end + end +end diff --git a/src/main/ruby/shell/commands/enable.rb b/src/main/ruby/shell/commands/enable.rb new file mode 100644 index 0000000..a0dc340 --- /dev/null +++ b/src/main/ruby/shell/commands/enable.rb @@ -0,0 +1,37 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Enable < Command + def help + return <<-EOF +Start enable of named table: e.g. "hbase> enable 't1'" +EOF + end + + def command(table) + format_simple_command do + admin.enable(table) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/enable_all.rb b/src/main/ruby/shell/commands/enable_all.rb new file mode 100644 index 0000000..ee01637 --- /dev/null +++ b/src/main/ruby/shell/commands/enable_all.rb @@ -0,0 +1,50 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class EnableAll < Command + def help + return <<-EOF +Enable all of the tables matching the given regex: + +hbase> enable_all 't.*' +EOF + end + + def command(regex) + regex = /^#{regex}$/ unless regex.is_a?(Regexp) + list = admin.list.grep(regex) + count = list.size + list.each do |table| + formatter.row([ table ]) + end + puts "\nEnable the above #{count} tables (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No tables matched the regex #{regex.to_s}" if count == 0 + return unless answer =~ /y.*/i + failed = admin.enable_all(regex) + puts "#{count - failed.size} tables successfully enabled" + puts "#{failed.size} tables not enabled due to an exception: #{failed.join ','}" unless failed.size == 0 + end + end + end +end diff --git a/src/main/ruby/shell/commands/enable_peer.rb b/src/main/ruby/shell/commands/enable_peer.rb new file mode 100644 index 0000000..de45909 --- /dev/null +++ b/src/main/ruby/shell/commands/enable_peer.rb @@ -0,0 +1,42 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class EnablePeer< Command + def help + return <<-EOF +Restarts the replication to the specified peer cluster, +continuing from where it was disabled. + +Examples: + + hbase> enable_peer '1' +EOF + end + + def command(id) + format_simple_command do + replication_admin.enable_peer(id) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/exists.rb b/src/main/ruby/shell/commands/exists.rb new file mode 100644 index 0000000..f35f197 --- /dev/null +++ b/src/main/ruby/shell/commands/exists.rb @@ -0,0 +1,39 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Exists < Command + def help + return <<-EOF +Does the named table exist? e.g. "hbase> exists 't1'" +EOF + end + + def command(table) + format_simple_command do + formatter.row([ + "Table #{table} " + (admin.exists?(table.to_s) ? "does exist" : "does not exist") + ]) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/flush.rb b/src/main/ruby/shell/commands/flush.rb new file mode 100644 index 0000000..ba59766 --- /dev/null +++ b/src/main/ruby/shell/commands/flush.rb @@ -0,0 +1,41 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Flush < Command + def help + return <<-EOF +Flush all regions in passed table or pass a region row to +flush an individual region. For example: + + hbase> flush 'TABLENAME' + hbase> flush 'REGIONNAME' +EOF + end + + def command(table_or_region_name) + format_simple_command do + admin.flush(table_or_region_name) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/get.rb b/src/main/ruby/shell/commands/get.rb new file mode 100644 index 0000000..8456d58 --- /dev/null +++ b/src/main/ruby/shell/commands/get.rb @@ -0,0 +1,55 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Get < Command + def help + return <<-EOF +Get row or cell contents; pass table name, row, and optionally +a dictionary of column(s), timestamp, timerange and versions. Examples: + + hbase> get 't1', 'r1' + hbase> get 't1', 'r1', {TIMERANGE => [ts1, ts2]} + hbase> get 't1', 'r1', {COLUMN => 'c1'} + hbase> get 't1', 'r1', {COLUMN => ['c1', 'c2', 'c3']} + hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1} + hbase> get 't1', 'r1', {COLUMN => 'c1', TIMERANGE => [ts1, ts2], VERSIONS => 4} + hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1, VERSIONS => 4} + hbase> get 't1', 'r1', {FILTER => "ValueFilter(=, 'binary:abc')"} + hbase> get 't1', 'r1', 'c1' + hbase> get 't1', 'r1', 'c1', 'c2' + hbase> get 't1', 'r1', ['c1', 'c2'] +EOF + end + + def command(table, row, *args) + now = Time.now + formatter.header(["COLUMN", "CELL"]) + + table(table).get(row, *args) do |column, value| + formatter.row([ column, value ]) + end + + formatter.footer(now) + end + end + end +end diff --git a/src/main/ruby/shell/commands/get_counter.rb b/src/main/ruby/shell/commands/get_counter.rb new file mode 100644 index 0000000..3cbe226 --- /dev/null +++ b/src/main/ruby/shell/commands/get_counter.rb @@ -0,0 +1,43 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetCounter < Command + def help + return <<-EOF +Return a counter cell value at specified table/row/column coordinates. +A cell cell should be managed with atomic increment function oh HBase +and the data should be binary encoded. Example: + + hbase> get_counter 't1', 'r1', 'c1' +EOF + end + + def command(table, row, column, value = nil) + if cnt = table(table).get_counter(row, column) + puts "COUNTER VALUE = #{cnt}" + else + puts "No counter found at specified coordinates" + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/grant.rb b/src/main/ruby/shell/commands/grant.rb new file mode 100644 index 0000000..7bdc959 --- /dev/null +++ b/src/main/ruby/shell/commands/grant.rb @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Grant < Command + def help + return <<-EOF +Grant users specific rights. +Syntax : grant [ [ []] + +permissions is either zero or more letters from the set "RWXCA". +READ('R'), WRITE('W'), EXEC('X'), CREATE('C'), ADMIN('A') + +For example: + + hbase> grant 'bobsmith', 'RWXCA' + hbase> grant 'bobsmith', 'RW', 't1', 'f1', 'col1' +EOF + end + + def command(user, rights, table_name=nil, family=nil, qualifier=nil) + format_simple_command do + security_admin.grant(user, rights, table_name, family, qualifier) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/hlog_roll.rb b/src/main/ruby/shell/commands/hlog_roll.rb new file mode 100644 index 0000000..02c7c59 --- /dev/null +++ b/src/main/ruby/shell/commands/hlog_roll.rb @@ -0,0 +1,40 @@ +# +# Copyright 2011 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +module Shell + module Commands + class HlogRoll < Command + def help + return <<-EOF +Roll the log writer. That is, start writing log messages to a new file. +The name of the regionserver should be given as the parameter. A +'server_name' is the host, port plus startcode of a regionserver. For +example: host187.example.com,60020,1289493121758 (find servername in +master ui or when you do detailed status in shell) +EOF + end + + def command(server_name) + format_simple_command do + admin.hlog_roll(server_name) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/incr.rb b/src/main/ruby/shell/commands/incr.rb new file mode 100644 index 0000000..38a2fc5 --- /dev/null +++ b/src/main/ruby/shell/commands/incr.rb @@ -0,0 +1,42 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Incr < Command + def help + return <<-EOF +Increments a cell 'value' at specified table/row/column coordinates. +To increment a cell value in table 't1' at row 'r1' under column +'c1' by 1 (can be omitted) or 10 do: + + hbase> incr 't1', 'r1', 'c1' + hbase> incr 't1', 'r1', 'c1', 1 + hbase> incr 't1', 'r1', 'c1', 10 +EOF + end + + def command(table, row, column, value = nil) + cnt = table(table).incr(row, column, value) + puts "COUNTER VALUE = #{cnt}" + end + end + end +end diff --git a/src/main/ruby/shell/commands/is_disabled.rb b/src/main/ruby/shell/commands/is_disabled.rb new file mode 100644 index 0000000..9d3c7ee --- /dev/null +++ b/src/main/ruby/shell/commands/is_disabled.rb @@ -0,0 +1,39 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class IsDisabled < Command + def help + return <<-EOF +Is named table disabled?: e.g. "hbase> is_disabled 't1'" +EOF + end + + def command(table) + format_simple_command do + formatter.row([ + admin.disabled?(table)? "true" : "false" + ]) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/is_enabled.rb b/src/main/ruby/shell/commands/is_enabled.rb new file mode 100644 index 0000000..96b2b15 --- /dev/null +++ b/src/main/ruby/shell/commands/is_enabled.rb @@ -0,0 +1,39 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class IsEnabled < Command + def help + return <<-EOF +Is named table enabled?: e.g. "hbase> is_enabled 't1'" +EOF + end + + def command(table) + format_simple_command do + formatter.row([ + admin.enabled?(table)? "true" : "false" + ]) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/list.rb b/src/main/ruby/shell/commands/list.rb new file mode 100644 index 0000000..592fb5e --- /dev/null +++ b/src/main/ruby/shell/commands/list.rb @@ -0,0 +1,48 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class List < Command + def help + return <<-EOF +List all tables in hbase. Optional regular expression parameter could +be used to filter the output. Examples: + + hbase> list + hbase> list 'abc.*' +EOF + end + + def command(regex = ".*") + now = Time.now + formatter.header([ "TABLE" ]) + + regex = /#{regex}/ unless regex.is_a?(Regexp) + list = admin.list.grep(regex) + list.each do |table| + formatter.row([ table ]) + end + + formatter.footer(now, list.size) + end + end + end +end diff --git a/src/main/ruby/shell/commands/list_peers.rb b/src/main/ruby/shell/commands/list_peers.rb new file mode 100644 index 0000000..2f57592 --- /dev/null +++ b/src/main/ruby/shell/commands/list_peers.rb @@ -0,0 +1,47 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListPeers< Command + def help + return <<-EOF +List all replication peer clusters. + + hbase> list_peers +EOF + end + + def command() + now = Time.now + peers = replication_admin.list_peers + + formatter.header(["PEER_ID", "CLUSTER_KEY", "STATE"]) + + peers.entrySet().each do |e| + state = replication_admin.get_peer_state(e.key) + formatter.row([ e.key, e.value, state ]) + end + + formatter.footer(now) + end + end + end +end diff --git a/src/main/ruby/shell/commands/list_snapshots.rb b/src/main/ruby/shell/commands/list_snapshots.rb new file mode 100644 index 0000000..9513518 --- /dev/null +++ b/src/main/ruby/shell/commands/list_snapshots.rb @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'time' + +module Shell + module Commands + class ListSnapshots < Command + def help + return <<-EOF +List all snapshots taken (by printing the names and relative information). +Optional regular expression parameter could be used to filter the output +by snapshot name. + +Examples: + hbase> list_snapshots + hbase> list_snapshots 'abc.*' +EOF + end + + def command(regex = ".*") + now = Time.now + formatter.header([ "SNAPSHOT", "TABLE + CREATION TIME"]) + + regex = /#{regex}/ unless regex.is_a?(Regexp) + list = admin.list_snapshot.select {|s| regex.match(s.getName)} + list.each do |snapshot| + creation_time = Time.at(snapshot.getCreationTime() / 1000).to_s + formatter.row([ snapshot.getName, snapshot.getTable + " (" + creation_time + ")" ]) + end + + formatter.footer(now, list.size) + return list.map { |s| s.getName() } + end + end + end +end diff --git a/src/main/ruby/shell/commands/major_compact.rb b/src/main/ruby/shell/commands/major_compact.rb new file mode 100644 index 0000000..325344b --- /dev/null +++ b/src/main/ruby/shell/commands/major_compact.rb @@ -0,0 +1,49 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class MajorCompact < Command + def help + return <<-EOF + Run major compaction on passed table or pass a region row + to major compact an individual region. To compact a single + column family within a region specify the region name + followed by the column family name. + Examples: + Compact all regions in a table: + hbase> major_compact 't1' + Compact an entire region: + hbase> major_compact 'r1' + Compact a single column family within a region: + hbase> major_compact 'r1', 'c1' + Compact a single column family within a table: + hbase> major_compact 't1', 'c1' + EOF + end + + def command(table_or_region_name, family = nil) + format_simple_command do + admin.major_compact(table_or_region_name, family) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/move.rb b/src/main/ruby/shell/commands/move.rb new file mode 100644 index 0000000..0e3db8f --- /dev/null +++ b/src/main/ruby/shell/commands/move.rb @@ -0,0 +1,48 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Move < Command + def help + return <<-EOF +Move a region. Optionally specify target regionserver else we choose one +at random. NOTE: You pass the encoded region name, not the region name so +this command is a little different to the others. The encoded region name +is the hash suffix on region names: e.g. if the region name were +TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. then +the encoded region name portion is 527db22f95c8a9e0116f0cc13c680396 +A server name is its host, port plus startcode. For example: +host187.example.com,60020,1289493121758 +Examples: + + hbase> move 'ENCODED_REGIONNAME' + hbase> move 'ENCODED_REGIONNAME', 'SERVER_NAME' +EOF + end + + def command(encoded_region_name, server_name = nil) + format_simple_command do + admin.move(encoded_region_name, server_name) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/put.rb b/src/main/ruby/shell/commands/put.rb new file mode 100644 index 0000000..dde0433 --- /dev/null +++ b/src/main/ruby/shell/commands/put.rb @@ -0,0 +1,41 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Put < Command + def help + return <<-EOF +Put a cell 'value' at specified table/row/column and optionally +timestamp coordinates. To put a cell value into table 't1' at +row 'r1' under column 'c1' marked with the time 'ts1', do: + + hbase> put 't1', 'r1', 'c1', 'value', ts1 +EOF + end + + def command(table, row, column, value, timestamp = nil) + format_simple_command do + table(table).put(row, column, value, timestamp) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/remove_peer.rb b/src/main/ruby/shell/commands/remove_peer.rb new file mode 100644 index 0000000..034434a --- /dev/null +++ b/src/main/ruby/shell/commands/remove_peer.rb @@ -0,0 +1,40 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RemovePeer< Command + def help + return <<-EOF +Stops the specified replication stream and deletes all the meta +information kept about it. Examples: + + hbase> remove_peer '1' +EOF + end + + def command(id) + format_simple_command do + replication_admin.remove_peer(id) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/restore_snapshot.rb b/src/main/ruby/shell/commands/restore_snapshot.rb new file mode 100644 index 0000000..4d53171 --- /dev/null +++ b/src/main/ruby/shell/commands/restore_snapshot.rb @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RestoreSnapshot < Command + def help + return <<-EOF +Restore a specified snapshot. +The restore will replace the content of the original table, +bringing back the content to the snapshot state. +The table must be disabled. + +Examples: + hbase> restore_snapshot 'snapshotName' +EOF + end + + def command(snapshot_name) + format_simple_command do + admin.restore_snapshot(snapshot_name) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/revoke.rb b/src/main/ruby/shell/commands/revoke.rb new file mode 100644 index 0000000..721519e --- /dev/null +++ b/src/main/ruby/shell/commands/revoke.rb @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Revoke < Command + def help + return <<-EOF +Revoke a user's access rights. +Syntax : revoke [
    [ []] +For example: + + hbase> revoke 'bobsmith' + hbase> revoke 'bobsmith', 't1', 'f1', 'col1' +EOF + end + + def command(user, table_name=nil, family=nil, qualifier=nil) + format_simple_command do + security_admin.revoke(user, table_name, family, qualifier) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/scan.rb b/src/main/ruby/shell/commands/scan.rb new file mode 100644 index 0000000..518e628 --- /dev/null +++ b/src/main/ruby/shell/commands/scan.rb @@ -0,0 +1,76 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Scan < Command + def help + return <<-EOF +Scan a table; pass table name and optionally a dictionary of scanner +specifications. Scanner specifications may include one or more of: +TIMERANGE, FILTER, LIMIT, STARTROW, STOPROW, TIMESTAMP, MAXLENGTH, +or COLUMNS, CACHE + +If no columns are specified, all columns will be scanned. +To scan all members of a column family, leave the qualifier empty as in +'col_family:'. + +The filter can be specified in two ways: +1. Using a filterString - more information on this is available in the +Filter Language document attached to the HBASE-4176 JIRA +2. Using the entire package name of the filter. + +Some examples: + + hbase> scan '.META.' + hbase> scan '.META.', {COLUMNS => 'info:regioninfo'} + hbase> scan 't1', {COLUMNS => ['c1', 'c2'], LIMIT => 10, STARTROW => 'xyz'} + hbase> scan 't1', {COLUMNS => 'c1', TIMERANGE => [1303668804, 1303668904]} + hbase> scan 't1', {FILTER => "(PrefixFilter ('row2') AND (QualifierFilter (>=, 'binary:xyz'))) AND (TimestampsFilter ( 123, 456))"} + hbase> scan 't1', {FILTER => org.apache.hadoop.hbase.filter.ColumnPaginationFilter.new(1, 0)} + +For experts, there is an additional option -- CACHE_BLOCKS -- which +switches block caching for the scanner on (true) or off (false). By +default it is enabled. Examples: + + hbase> scan 't1', {COLUMNS => ['c1', 'c2'], CACHE_BLOCKS => false} + +Also for experts, there is an advanced option -- RAW -- which instructs the +scanner to return all cells (including delete markers and uncollected deleted +cells). This option cannot be combined with requesting specific COLUMNS. +Disabled by default. Example: + + hbase> scan 't1', {RAW => true, VERSIONS => 10} +EOF + end + + def command(table, args = {}) + now = Time.now + formatter.header(["ROW", "COLUMN+CELL"]) + + count = table(table).scan(args) do |row, cells| + formatter.row([ row, cells ]) + end + + formatter.footer(now, count) + end + end + end +end diff --git a/src/main/ruby/shell/commands/show_filters.rb b/src/main/ruby/shell/commands/show_filters.rb new file mode 100644 index 0000000..1581db5 --- /dev/null +++ b/src/main/ruby/shell/commands/show_filters.rb @@ -0,0 +1,55 @@ +# +# Copyright 2011 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +java_import org.apache.hadoop.hbase.filter.ParseFilter + +module Shell + module Commands + class ShowFilters < Command + def help + return <<-EOF +Show all the filters in hbase. Example: + hbase> show_filters + + Documentation on filters mentioned below can be found at: https://our.intern.facebook.com/intern/wiki/index.php/HBase/Filter_Language + ColumnPrefixFilter + TimestampsFilter + PageFilter + ..... + KeyOnlyFilter +EOF + end + + def command( ) + now = Time.now + formatter.row(["Documentation on filters mentioned below can " + + "be found at: https://our.intern.facebook.com/intern/" + + "wiki/index.php/HBase/Filter_Language"]) + + parseFilter = ParseFilter.new + supportedFilters = parseFilter.getSupportedFilters + + supportedFilters.each do |filter| + formatter.row([filter]) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/snapshot.rb b/src/main/ruby/shell/commands/snapshot.rb new file mode 100644 index 0000000..1c4ecfe --- /dev/null +++ b/src/main/ruby/shell/commands/snapshot.rb @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Snapshot < Command + def help + return <<-EOF +Take a snapshot of specified table. Examples: + + hbase> snapshot 'sourceTable', 'snapshotName' +EOF + end + + def command(table, snapshot_name) + format_simple_command do + admin.snapshot(table, snapshot_name) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/split.rb b/src/main/ruby/shell/commands/split.rb new file mode 100644 index 0000000..47e21f8 --- /dev/null +++ b/src/main/ruby/shell/commands/split.rb @@ -0,0 +1,43 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Split < Command + def help + return <<-EOF +Split entire table or pass a region to split individual region. With the +second parameter, you can specify an explicit split key for the region. +Examples: + split 'tableName' + split 'regionName' # format: 'tableName,startKey,id' + split 'tableName', 'splitKey' + split 'regionName', 'splitKey' +EOF + end + + def command(table_or_region_name, split_point = nil) + format_simple_command do + admin.split(table_or_region_name, split_point) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/start_replication.rb b/src/main/ruby/shell/commands/start_replication.rb new file mode 100644 index 0000000..5d1cd1b --- /dev/null +++ b/src/main/ruby/shell/commands/start_replication.rb @@ -0,0 +1,43 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class StartReplication < Command + def help + return <<-EOF +Restarts all the replication features. The state in which each +stream starts in is undetermined. +WARNING: +start/stop replication is only meant to be used in critical load situations. +Examples: + + hbase> start_replication +EOF + end + + def command + format_simple_command do + replication_admin.start_replication + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/status.rb b/src/main/ruby/shell/commands/status.rb new file mode 100644 index 0000000..4b22acb --- /dev/null +++ b/src/main/ruby/shell/commands/status.rb @@ -0,0 +1,41 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Status < Command + def help + return <<-EOF +Show cluster status. Can be 'summary', 'simple', or 'detailed'. The +default is 'summary'. Examples: + + hbase> status + hbase> status 'simple' + hbase> status 'summary' + hbase> status 'detailed' +EOF + end + + def command(format = 'summary') + admin.status(format) + end + end + end +end diff --git a/src/main/ruby/shell/commands/stop_replication.rb b/src/main/ruby/shell/commands/stop_replication.rb new file mode 100644 index 0000000..f5074d7 --- /dev/null +++ b/src/main/ruby/shell/commands/stop_replication.rb @@ -0,0 +1,43 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class StopReplication < Command + def help + return <<-EOF +Stops all the replication features. The state in which each +stream stops in is undetermined. +WARNING: +start/stop replication is only meant to be used in critical load situations. +Examples: + + hbase> stop_replication +EOF + end + + def command + format_simple_command do + replication_admin.stop_replication + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/truncate.rb b/src/main/ruby/shell/commands/truncate.rb new file mode 100644 index 0000000..a24e167 --- /dev/null +++ b/src/main/ruby/shell/commands/truncate.rb @@ -0,0 +1,39 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Truncate < Command + def help + return <<-EOF + Disables, drops and recreates the specified table. +EOF + end + + def command(table) + format_simple_command do + puts "Truncating '#{table}' table (it may take a while):" + admin.truncate(table) { |log| puts " - #{log}" } + end + end + + end + end +end diff --git a/src/main/ruby/shell/commands/truncate_preserve.rb b/src/main/ruby/shell/commands/truncate_preserve.rb new file mode 100644 index 0000000..1692edf --- /dev/null +++ b/src/main/ruby/shell/commands/truncate_preserve.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class TruncatePreserve < Command + def help + return <<-EOF + Disables, drops and recreates the specified table while still maintaing the previous region boundaries. +EOF + end + + def command(table) + format_simple_command do + puts "Truncating '#{table}' table (it may take a while):" + admin.truncate_preserve(table) { |log| puts " - #{log}" } + end + end + + end + end +end \ No newline at end of file diff --git a/src/main/ruby/shell/commands/unassign.rb b/src/main/ruby/shell/commands/unassign.rb new file mode 100644 index 0000000..75242ba --- /dev/null +++ b/src/main/ruby/shell/commands/unassign.rb @@ -0,0 +1,44 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Unassign < Command + def help + return <<-EOF +Unassign a region. Unassign will close region in current location and then +reopen it again. Pass 'true' to force the unassignment ('force' will clear +all in-memory state in master before the reassign. If results in +double assignment use hbck -fix to resolve. To be used by experts). +Use with caution. For expert use only. Examples: + + hbase> unassign 'REGIONNAME' + hbase> unassign 'REGIONNAME', true +EOF + end + + def command(region_name, force = 'false') + format_simple_command do + admin.unassign(region_name, force) + end + end + end + end +end diff --git a/src/main/ruby/shell/commands/user_permission.rb b/src/main/ruby/shell/commands/user_permission.rb new file mode 100644 index 0000000..ad4a7b0 --- /dev/null +++ b/src/main/ruby/shell/commands/user_permission.rb @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class UserPermission < Command + def help + return <<-EOF +Show all permissions for the particular user. +Syntax : user_permission
    +For example: + + hbase> user_permission + hbase> user_permission 'table1' +EOF + end + + def command(table=nil) + #format_simple_command do + #admin.user_permission(table) + now = Time.now + formatter.header(["User", "Table,Family,Qualifier:Permission"]) + + count = security_admin.user_permission(table) do |user, permission| + formatter.row([ user, permission]) + end + + formatter.footer(now, count) + end + end + end +end diff --git a/src/main/ruby/shell/commands/version.rb b/src/main/ruby/shell/commands/version.rb new file mode 100644 index 0000000..372b0dc --- /dev/null +++ b/src/main/ruby/shell/commands/version.rb @@ -0,0 +1,38 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Version < Command + def help + return <<-EOF +Output this HBase version +EOF + end + + def command + # Output version. + puts "#{org.apache.hadoop.hbase.util.VersionInfo.getVersion()}, " + + "r#{org.apache.hadoop.hbase.util.VersionInfo.getRevision()}, " + + "#{org.apache.hadoop.hbase.util.VersionInfo.getDate()}" + end + end + end +end diff --git a/src/main/ruby/shell/commands/whoami.rb b/src/main/ruby/shell/commands/whoami.rb new file mode 100644 index 0000000..040ad7e --- /dev/null +++ b/src/main/ruby/shell/commands/whoami.rb @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Whoami < Command + def help + return <<-EOF +Show the current hbase user. +Syntax : whoami +For example: + + hbase> whoami +EOF + end + + def command() + puts "#{org.apache.hadoop.hbase.security.User.getCurrent().toString()}" + end + end + end +end diff --git a/src/main/ruby/shell/commands/zk_dump.rb b/src/main/ruby/shell/commands/zk_dump.rb new file mode 100644 index 0000000..bb23962 --- /dev/null +++ b/src/main/ruby/shell/commands/zk_dump.rb @@ -0,0 +1,35 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ZkDump < Command + def help + return <<-EOF +Dump status of HBase cluster as seen by ZooKeeper. +EOF + end + + def command + puts admin.zk_dump + end + end + end +end diff --git a/src/main/ruby/shell/formatter.rb b/src/main/ruby/shell/formatter.rb new file mode 100644 index 0000000..ea17882 --- /dev/null +++ b/src/main/ruby/shell/formatter.rb @@ -0,0 +1,171 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Results formatter +module Shell + module Formatter + # Base abstract class for results formatting. + class Base + attr_reader :row_count + + def is_valid_io?(obj) + obj.instance_of?(IO) || obj == Kernel + end + + def refresh_width() + if $stdout.tty? + @max_width = Java::jline.Terminal.getTerminal().getTerminalWidth() + else + @max_width = 0 + end + end + + # Takes an output stream and a print width. + def initialize(opts = {}) + options = { + :output_stream => Kernel, + }.merge(opts) + + @out = options[:output_stream] + refresh_width + @row_count = 0 + + # raise an error if the stream is not valid + raise(TypeError, "Type #{@out.class} of parameter #{@out} is not IO") unless is_valid_io?(@out) + end + + def header(args = [], widths = []) + refresh_width + row(args, false, widths) if args.length > 0 + @row_count = 0 + end + + # Output a row. + # Inset is whether or not to offset row by a space. + def row(args = [], inset = true, widths = []) + # Print out nothing + return if !args || args.empty? + + # Print a string + if args.is_a?(String) + output(args) + @out.puts + return + end + + # TODO: Look at the type. Is it RowResult? + if args.length == 1 + splits = split(@max_width, dump(args[0])) + for l in splits + output(@max_width, l) + @out.puts + end + elsif args.length == 2 + if @max_width == 0 + col1width = col2width = 0 + else + col1width = (not widths or widths.length == 0) ? @max_width / 4 : @max_width * widths[0] / 100 + col2width = (not widths or widths.length < 2) ? @max_width - col1width - 2 : @max_width * widths[1] / 100 - 2 + end + splits1 = split(col1width, dump(args[0])) + splits2 = split(col2width, dump(args[1])) + biggest = (splits2.length > splits1.length)? splits2.length: splits1.length + index = 0 + while index < biggest + # Inset by one space if inset is set. + @out.print(" ") if inset + output(col1width, splits1[index]) + # Add extra space so second column lines up w/ second column output + @out.print(" ") unless inset + @out.print(" ") + output(col2width, splits2[index]) + index += 1 + @out.puts + end + else + # Print a space to set off multi-column rows + print ' ' + first = true + for e in args + @out.print " " unless first + first = false + @out.print e + end + puts + end + @row_count += 1 + end + + def split(width, str) + if width == 0 + return [str] + end + result = [] + index = 0 + while index < str.length do + result << str.slice(index, width) + index += width + end + result + end + + def dump(str) + return if str.instance_of?(Fixnum) + # Remove double-quotes added by 'dump'. + return str + end + + def output(str) + output(@max_width, str) + end + + def output(width, str) + if str == nil + str = '' + end + if not width or width == str.length + @out.print(str) + else + @out.printf('%-*s', width, str) + end + end + + def footer(start_time = nil, row_count = nil) + return unless start_time + row_count ||= @row_count + # Only output elapsed time and row count if startTime passed + @out.puts("%d row(s) in %.4f seconds" % [row_count, Time.now - start_time]) + end + end + + + class Console < Base + end + + class XHTMLFormatter < Base + # http://www.germane-software.com/software/rexml/doc/classes/REXML/Document.html + # http://www.crummy.com/writing/RubyCookbook/test_results/75942.html + end + + class JSON < Base + end + end +end + diff --git a/src/main/xslt/configuration_to_docbook_section.xsl b/src/main/xslt/configuration_to_docbook_section.xsl new file mode 100644 index 0000000..95f7fd5 --- /dev/null +++ b/src/main/xslt/configuration_to_docbook_section.xsl @@ -0,0 +1,68 @@ + + + + + +
    +HBase Default Configuration + + + + +HBase Default Configuration + +The documentation below is generated using the default hbase configuration file, +hbase-default.xml, as source. + + + + + + + + + + + + + + Default: + + + + + + + +
    +     +     diff --git a/src/packages/build.xml b/src/packages/build.xml new file mode 100644 index 0000000..e17295f --- /dev/null +++ b/src/packages/build.xml @@ -0,0 +1,149 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/packages/conf-pseudo/hbase-site.xml b/src/packages/conf-pseudo/hbase-site.xml new file mode 100644 index 0000000..b4e8de6 --- /dev/null +++ b/src/packages/conf-pseudo/hbase-site.xml @@ -0,0 +1,68 @@ + + + + + + hbase.rootdir + hdfs://localhost:9000/hbase + + + hbase.cluster.distributed + true + The mode the cluster will be in. Possible values are + false: standalone and pseudo-distributed setups with managed Zookeeper + true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh) + + + + hbase.zookeeper.property.clientPort + 2181 + Property from ZooKeeper's config zoo.cfg. + The port at which the clients will connect. + + + + dfs.replication + 1 + The replication count for HLog and HFile storage. Should not be greater than HDFS datanode count. + + + + hbase.zookeeper.quorum + localhost + Comma separated list of servers in the ZooKeeper Quorum. + For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com". + By default this is set to localhost for local and pseudo-distributed modes + of operation. For a fully-distributed setup, this should be set to a full + list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh + this is the list of servers which we will start/stop ZooKeeper on. + + + + hbase.zookeeper.property.dataDir + /var/lib/zookeeper/data + Property from ZooKeeper's config zoo.cfg. + The directory where the snapshot is stored. + + + diff --git a/src/packages/deb/conf-pseudo.control/conffile b/src/packages/deb/conf-pseudo.control/conffile new file mode 100644 index 0000000..5e838d7 --- /dev/null +++ b/src/packages/deb/conf-pseudo.control/conffile @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +/etc/hbase/hbase-site.xml diff --git a/src/packages/deb/conf-pseudo.control/control b/src/packages/deb/conf-pseudo.control/control new file mode 100644 index 0000000..f31c218 --- /dev/null +++ b/src/packages/deb/conf-pseudo.control/control @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Package: hbase-conf-pseudo +Version: @version@ +Section: misc +Priority: optional +Architecture: all +Depends: openjdk-6-jre-headless, hadoop, zookeeper, hbase +Maintainer: Apache Software Foundation +Description: HBase pseudo cluster configuration for single node cluster testing. +Distribution: development diff --git a/src/packages/deb/conf-pseudo.control/postinst b/src/packages/deb/conf-pseudo.control/postinst new file mode 100644 index 0000000..7c736fb --- /dev/null +++ b/src/packages/deb/conf-pseudo.control/postinst @@ -0,0 +1,26 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +/etc/init.d/hadoop-namenode start 2>/dev/null >/dev/null +/etc/init.d/hadoop-datanode start 2>/dev/null >/dev/null +su - hdfs -c "hadoop fs -mkdir /hbase" 2>/dev/null >/dev/null +su - hdfs -c "hadoop fs -chown hbase /hbase" 2>/dev/null >/dev/null +/etc/init.d/hbase-master start 2>/dev/null >/dev/null +/etc/init.d/hbase-regionserver start 2>/dev/null >/dev/null +ln -sf ../init.d/hbase-master /etc/rc2.d/S94hbase-master +ln -sf ../init.d/hbase-regionserver /etc/rc6.d/S94hbase-regionserver + diff --git a/src/packages/deb/conf-pseudo.control/prerm b/src/packages/deb/conf-pseudo.control/prerm new file mode 100644 index 0000000..fc48dfd --- /dev/null +++ b/src/packages/deb/conf-pseudo.control/prerm @@ -0,0 +1,21 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +/etc/init.d/hbase-regionserver stop 2>/dev/null >/dev/null +/etc/init.d/hbase-master stop 2>/dev/null >/dev/null +rm -f /etc/rc2.d/S94hbase-master +rm -f /etc/rc2.d/S94hbase-regionserver diff --git a/src/packages/deb/hbase.control/conffile b/src/packages/deb/hbase.control/conffile new file mode 100644 index 0000000..445d095 --- /dev/null +++ b/src/packages/deb/hbase.control/conffile @@ -0,0 +1,5 @@ +/etc/hbase/hadoop-metrics.properties +/etc/hbase/hbase-env.sh +/etc/hbase/hbase-site.xml +/etc/hbase/log4j.properties +/etc/hbase/regionservers diff --git a/src/packages/deb/hbase.control/control b/src/packages/deb/hbase.control/control new file mode 100644 index 0000000..badf17e --- /dev/null +++ b/src/packages/deb/hbase.control/control @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Package: hbase +Version: @version@ +Section: misc +Priority: optional +Architecture: all +Depends: openjdk-6-jre-headless, hadoop, zookeeper +Maintainer: Apache Software Foundation +Description: HBase is the Hadoop database. Use it when you need random, realtime read/write access to your Big Data. This project's goal is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters of commodity hardware. +Distribution: development diff --git a/src/packages/deb/hbase.control/postinst b/src/packages/deb/hbase.control/postinst new file mode 100644 index 0000000..a77fe4f --- /dev/null +++ b/src/packages/deb/hbase.control/postinst @@ -0,0 +1,24 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bash /usr/share/hbase/sbin/update-hbase-env.sh \ + --prefix=/usr \ + --bin-dir=/usr/bin \ + --conf-dir=/etc/hbase \ + --log-dir=/var/log/hbase \ + --pid-dir=/var/run/hbase + diff --git a/src/packages/deb/hbase.control/postrm b/src/packages/deb/hbase.control/postrm new file mode 100644 index 0000000..25f73a4 --- /dev/null +++ b/src/packages/deb/hbase.control/postrm @@ -0,0 +1,20 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +/usr/sbin/userdel hbase 2> /dev/null >/dev/null +exit 0 + diff --git a/src/packages/deb/hbase.control/preinst b/src/packages/deb/hbase.control/preinst new file mode 100644 index 0000000..497bbab --- /dev/null +++ b/src/packages/deb/hbase.control/preinst @@ -0,0 +1,21 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hadoop + +/usr/sbin/useradd --comment "HBase" --shell /bin/bash -M -r --groups hadoop --home /usr/share/hbase hbase 2> /dev/null || : + diff --git a/src/packages/deb/hbase.control/prerm b/src/packages/deb/hbase.control/prerm new file mode 100644 index 0000000..55486e6 --- /dev/null +++ b/src/packages/deb/hbase.control/prerm @@ -0,0 +1,27 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +/etc/init.d/hbase-master stop 2>/dev/null >/dev/null +/etc/init.d/hbase-regionserver stop 2>/dev/null >/dev/null +bash /usr/share/hbase/sbin/update-hbase-env.sh \ + --prefix=/usr \ + --bin-dir=/usr/bin \ + --conf-dir=/etc/hbase \ + --log-dir=/var/log/hbase \ + --pid-dir=/var/run/hbase \ + --uninstal + diff --git a/src/packages/deb/init.d/hbase-master b/src/packages/deb/init.d/hbase-master new file mode 100644 index 0000000..55eea9b --- /dev/null +++ b/src/packages/deb/init.d/hbase-master @@ -0,0 +1,142 @@ +#! /bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### BEGIN INIT INFO +# Provides: hbase-master +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: +# Short-Description: Apache HBase Master +### END INIT INFO + +set -e + +# /etc/init.d/hbase-master: start and stop the Apache HBase Master daemon + +test -x /usr/bin/hbase || exit 0 +( /usr/bin/hbase 2>&1 | grep -q hbase ) 2>/dev/null || exit 0 + +umask 022 + +if test -f /etc/default/hbase-env.sh; then + . /etc/default/hbase-env.sh +fi + +. /lib/lsb/init-functions + +# Are we running from init? +run_by_init() { + ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ] +} + +check_for_no_start() { + # forget it if we're trying to start, and /etc/hbase/hbase-master_not_to_be_run exists + if [ -e /etc/hbase/hbase-master_not_to_be_run ]; then + if [ "$1" = log_end_msg ]; then + log_end_msg 0 + fi + if ! run_by_init; then + log_action_msg "Apache HBase Master server not in use (/etc/hbase/hbase-master_not_to_be_run)" + fi + exit 0 + fi +} + +check_privsep_dir() { + # Create the PrivSep empty dir if necessary + if [ ! -d ${HBASE_PID_DIR} ]; then + mkdir -p ${HBASE_PID_DIR} + chown root:hadoop ${HBASE_PID_DIR} + chmod 0775 ${HBASE_PID_DIR} + fi +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin" + +case "$1" in + start) + check_privsep_dir + check_for_no_start + log_daemon_msg "Starting Apache HBase Master server" "hbase-master" + if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-master.pid -c hbase -x ${HBASE_HOME}/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start master; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + stop) + log_daemon_msg "Stopping Apache HBase Master server" "hbase-master" + if start-stop-daemon --stop --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-master.pid; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + restart) + check_privsep_dir + log_daemon_msg "Restarting Apache HBase Master server" "hbase-master" + start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HBASE_PID_DIR}/hbase-hbase-master.pid + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-master.pid -c hbase -x ${HBASE_HOME}/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start master; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + try-restart) + check_privsep_dir + log_daemon_msg "Restarting Apache HBase Master server" "hbase-master" + set +e + start-stop-daemon --stop --quiet --retry 30 --pidfile ${HBASE_PID_DIR}/hbase-hbase-master.pid + RET="$?" + set -e + case $RET in + 0) + # old daemon stopped + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-master.pid -c hbase -x ${HBASE_HOME}/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start master; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + 1) + # daemon not running + log_progress_msg "(not running)" + log_end_msg 0 + ;; + *) + # failed to stop + log_progress_msg "(failed to stop)" + log_end_msg 1 + ;; + esac + ;; + + status) + status_of_proc -p ${HBASE_PID_DIR}/hbase-hbase-master.pid ${JAVA_HOME}/bin/java hbase-master && exit 0 || exit $? + ;; + + *) + log_action_msg "Usage: /etc/init.d/hbase-master {start|stop|restart|try-restart|status}" + exit 1 +esac + +exit 0 diff --git a/src/packages/deb/init.d/hbase-regionserver b/src/packages/deb/init.d/hbase-regionserver new file mode 100644 index 0000000..0aaa507 --- /dev/null +++ b/src/packages/deb/init.d/hbase-regionserver @@ -0,0 +1,142 @@ +#! /bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### BEGIN INIT INFO +# Provides: hbase-regionserver +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: +# Short-Description: Apache HBase Region Server +### END INIT INFO + +set -e + +# /etc/init.d/hbase-regionserver: start and stop the Apache HBase Region Server daemon + +test -x /usr/bin/hbase || exit 0 +( /usr/bin/hbase 2>&1 | grep -q hbase ) 2>/dev/null || exit 0 + +umask 022 + +if test -f /etc/default/hbase-env.sh; then + . /etc/default/hbase-env.sh +fi + +. /lib/lsb/init-functions + +# Are we running from init? +run_by_init() { + ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ] +} + +check_for_no_start() { + # forget it if we're trying to start, and /etc/hbase/hbase-regionserver_not_to_be_run exists + if [ -e /etc/hbase/hbase-regionserver_not_to_be_run ]; then + if [ "$1" = log_end_msg ]; then + log_end_msg 0 + fi + if ! run_by_init; then + log_action_msg "Apache HBase Region Server server not in use (/etc/hbase/hbase-regionserver_not_to_be_run)" + fi + exit 0 + fi +} + +check_privsep_dir() { + # Create the PrivSep empty dir if necessary + if [ ! -d ${HBASE_PID_DIR} ]; then + mkdir -p ${HBASE_PID_DIR} + chown root:hadoop ${HBASE_PID_DIR} + chmod 0775 ${HBASE_PID_DIR} + fi +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin" + +case "$1" in + start) + check_privsep_dir + check_for_no_start + log_daemon_msg "Starting Apache HBase Region Server server" "hbase-regionserver" + if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid -c hbase -x ${HBASE_HOME}/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start regionserver; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + stop) + log_daemon_msg "Stopping Apache HBase Region Server server" "hbase-regionserver" + if start-stop-daemon --stop --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + restart) + check_privsep_dir + log_daemon_msg "Restarting Apache HBase Region Server server" "hbase-regionserver" + start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid -c hbase -x ${HBASE_HOME}/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start regionserver; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + try-restart) + check_privsep_dir + log_daemon_msg "Restarting Apache HBase Region Server server" "hbase-regionserver" + set +e + start-stop-daemon --stop --quiet --retry 30 --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid + RET="$?" + set -e + case $RET in + 0) + # old daemon stopped + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid -c hbase -x ${HBASE_HOME}/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start regionserver; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + 1) + # daemon not running + log_progress_msg "(not running)" + log_end_msg 0 + ;; + *) + # failed to stop + log_progress_msg "(failed to stop)" + log_end_msg 1 + ;; + esac + ;; + + status) + status_of_proc -p ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid ${JAVA_HOME}/bin/java hbase-regionserver && exit 0 || exit $? + ;; + + *) + log_action_msg "Usage: /etc/init.d/hbase-regionserver {start|stop|restart|try-restart|status}" + exit 1 +esac + +exit 0 diff --git a/src/packages/rpm/init.d/hbase-master b/src/packages/rpm/init.d/hbase-master new file mode 100644 index 0000000..67752da --- /dev/null +++ b/src/packages/rpm/init.d/hbase-master @@ -0,0 +1,84 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Starts a HBase master +# +# chkconfig: 2345 95 10 +# description: HBase master + +source /etc/rc.d/init.d/functions +source /etc/default/hbase-env.sh + +RETVAL=0 +PIDFILE="${HBASE_PID_DIR}/hbase-hbase-master.pid" +desc="HBase master daemon" + +start() { + echo -n $"Starting $desc (hbase-master): " + daemon --user hbase ${HBASE_HOME}/bin/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hbase-master + return $RETVAL +} + +stop() { + echo -n $"Stopping $desc (hbase-master): " + daemon --user hbase ${HBASE_HOME}/bin/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master + RETVAL=$? + sleep 5 + echo + [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hbase-master $PIDFILE +} + +restart() { + stop + start +} + +checkstatus(){ + status -p $PIDFILE ${JAVA_HOME}/bin/java + RETVAL=$? +} + +condrestart(){ + [ -e /var/lock/subsys/hbase-master ] && restart || : +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + checkstatus + ;; + restart) + restart + ;; + condrestart) + condrestart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart}" + exit 1 +esac + +exit $RETVAL diff --git a/src/packages/rpm/init.d/hbase-regionserver b/src/packages/rpm/init.d/hbase-regionserver new file mode 100644 index 0000000..186ea21 --- /dev/null +++ b/src/packages/rpm/init.d/hbase-regionserver @@ -0,0 +1,84 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Starts a HBase region server +# +# chkconfig: 2345 96 10 +# description: HBase region server + +source /etc/rc.d/init.d/functions +source /etc/default/hbase-env.sh + +RETVAL=0 +PIDFILE="${HBASE_PID_DIR}/hbase-hbase-regionserver.pid" +desc="HBase regionserver daemon" + +start() { + echo -n $"Starting $desc (hbase-regionserver): " + daemon --user hbase ${HBASE_HOME}/bin/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start regionserver + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hbase-regionserver + return $RETVAL +} + +stop() { + echo -n $"Stopping $desc (hbase-regionserver): " + daemon --user hbase ${HBASE_HOME}/bin/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop regionserver + RETVAL=$? + sleep 5 + echo + [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hbase-regionserver $PIDFILE +} + +restart() { + stop + start +} + +checkstatus(){ + status -p $PIDFILE ${JAVA_HOME}/bin/java + RETVAL=$? +} + +condrestart(){ + [ -e /var/lock/subsys/hbase-regionserver ] && restart || : +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + checkstatus + ;; + restart) + restart + ;; + condrestart) + condrestart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart}" + exit 1 +esac + +exit $RETVAL diff --git a/src/packages/rpm/spec/conf-pseudo.spec b/src/packages/rpm/spec/conf-pseudo.spec new file mode 100644 index 0000000..b8350d2 --- /dev/null +++ b/src/packages/rpm/spec/conf-pseudo.spec @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# RPM Spec file for HBase version @version@ +# + +%define name hbase-conf-pseudo +%define version @version@ +%define release @package.release@ + +# Installation Locations +%define _source @package.name@ +%define _final_name @final.name@ +%define _prefix @package.prefix@ +%define _bin_dir %{_prefix}/bin +%define _conf_dir @package.conf.dir@ +%define _include_dir %{_prefix}/include +%define _lib_dir %{_prefix}/lib +%define _lib64_dir %{_prefix}/lib64 +%define _libexec_dir %{_prefix}/libexec +%define _log_dir @package.log.dir@ +%define _man_dir %{_prefix}/man +%define _pid_dir @package.pid.dir@ +%define _sbin_dir %{_prefix}/sbin +%define _share_dir %{_prefix}/share/hbase +%define _src_dir %{_prefix}/src +%define _var_dir %{_prefix}/var/lib + +# Build time settings +%define _build_dir @package.build.dir@ +%define _final_name @final.name@ +%define debug_package %{nil} + +Summary: Default HBase configuration templates +License: Apache License, Version 2.0 +URL: http://hbase.apache.org/ +Vendor: Apache Software Foundation +Group: Development/Libraries +Name: %{name} +Version: %{version} +Release: %{release} +Source0: %{_source} +Prefix: %{_prefix} +Prefix: %{_conf_dir} +Prefix: %{_log_dir} +Prefix: %{_pid_dir} +Buildroot: %{_build_dir} +Requires: hbase == %{version}, sh-utils, textutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service, jdk >= 1.6 +AutoReqProv: no +Provides: hbase-conf-pseudo + +%description +Installation of this RPM will setup your machine to run in pseudo-distributed mode where each HBase daemon runs in a separate Java process. + +%prep +%setup -n %{_final_name} + +%build +if [ -d ${RPM_BUILD_DIR}%{_conf_dir} ]; then + rm -rf ${RPM_BUILD_DIR}%{_conf_dir} +fi + +mkdir -p ${RPM_BUILD_DIR}%{_conf_dir} +mkdir -p ${RPM_BUILD_DIR}%{_share_dir}/src/packages/conf-pseudo +cp -f ${RPM_BUILD_DIR}/%{_final_name}/src/packages/conf-pseudo/hbase-site.xml ${RPM_BUILD_DIR}%{_share_dir}/src/packages/conf-pseudo/hbase-site.xml +rm -rf ${RPM_BUILD_DIR}/%{_final_name} + +%preun +/sbin/chkconfig --del hbase-master +/sbin/chkconfig --del hbase-regionserver +/etc/init.d/hbase-master stop 2>/dev/null >/dev/null +/etc/init.d/hbase-regionserver stop 2>/dev/null >/dev/null +exit 0 + +%post +cp -f ${RPM_INSTALL_PREFIX0}/share/hbase/src/packages/conf-pseudo/*.xml ${RPM_INSTALL_PREFIX1} 2>/dev/null >/dev/null +/etc/init.d/hadoop-namenode start 2>/dev/null >/dev/null +/etc/init.d/hadoop-datanode start 2>/dev/null >/dev/null +su - hdfs -c "hadoop fs -mkdir /hbase" 2>/dev/null >/dev/null +su - hdfs -c "hadoop fs -chown hbase /hbase" 2>/dev/null >/dev/null +/etc/init.d/hbase-master start 2>/dev/null >/dev/null +/etc/init.d/hbase-regionserver start 2>/dev/null >/dev/null +/sbin/chkconfig hbase-master --add +/sbin/chkconfig hbase-regionserver --add +/sbin/chkconfig hbase-master on +/sbin/chkconfig hbase-regionserver on + +%files +%defattr(-,root,root) +%config %{_share_dir}/src/packages/conf-pseudo/hbase-site.xml diff --git a/src/packages/rpm/spec/hbase.spec b/src/packages/rpm/spec/hbase.spec new file mode 100644 index 0000000..1af874d --- /dev/null +++ b/src/packages/rpm/spec/hbase.spec @@ -0,0 +1,135 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# RPM Spec file for HBase version @version@ +# + +%define name hbase +%define version @version@ +%define release @package.release@ + +# Installation Locations +%define _source @package.name@ +%define _final_name @final.name@ +%define _prefix @package.prefix@ +%define _bin_dir %{_prefix}/bin +%define _conf_dir @package.conf.dir@ +%define _include_dir %{_prefix}/include +%define _lib_dir %{_prefix}/lib +%define _lib64_dir %{_prefix}/lib64 +%define _libexec_dir %{_prefix}/libexec +%define _log_dir @package.log.dir@ +%define _man_dir %{_prefix}/man +%define _pid_dir @package.pid.dir@ +%define _sbin_dir %{_prefix}/sbin +%define _share_dir %{_prefix}/share/hbase +%define _src_dir %{_prefix}/src +%define _var_dir %{_prefix}/var/lib + +# Build time settings +%define _build_dir @package.build.dir@ +%define _final_name @final.name@ +%define debug_package %{nil} + +Summary: Default HBase configuration templates +License: Apache License, Version 2.0 +URL: http://hbase.apache.org/ +Vendor: Apache Software Foundation +Group: Development/Libraries +Name: %{name} +Version: %{version} +Release: %{release} +Source0: %{_source} +Prefix: %{_prefix} +Prefix: %{_conf_dir} +Prefix: %{_log_dir} +Prefix: %{_pid_dir} +Buildroot: %{_build_dir} +Requires: sh-utils, textutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service, jdk >= 1.6, hadoop +AutoReqProv: no +Provides: hbase + +%description +Installation of this RPM will setup your machine to run in pseudo-distributed mode where each HBase daemon runs in a separate Java process. + +%prep +%setup -n %{_final_name} + +%build +if [ -d ${RPM_BUILD_DIR}%{_prefix} ]; then + rm -rf ${RPM_BUILD_DIR}%{_prefix} +fi + +if [ -d ${RPM_BUILD_DIR}%{_log_dir} ]; then + rm -rf ${RPM_BUILD_DIR}%{_log_dir} +fi + +if [ -d ${RPM_BUILD_DIR}%{_conf_dir} ]; then + rm -rf ${RPM_BUILD_DIR}%{_conf_dir} +fi + +if [ -d ${RPM_BUILD_DIR}%{_pid_dir} ]; then + rm -rf ${RPM_BUILD_DIR}%{_pid_dir} +fi + +mkdir -p ${RPM_BUILD_DIR}%{_conf_dir} +mkdir -p ${RPM_BUILD_DIR}%{_log_dir} +mkdir -p ${RPM_BUILD_DIR}%{_conf_dir} +mkdir -p ${RPM_BUILD_DIR}%{_pid_dir} +mkdir -p ${RPM_BUILD_DIR}%{_share_dir} +mkdir -p ${RPM_BUILD_DIR}%{_share_dir}/sbin +mkdir -p ${RPM_BUILD_DIR}/etc/rc.d/init.d + +cp ${RPM_BUILD_DIR}/%{_final_name}/src/packages/update-hbase-env.sh ${RPM_BUILD_DIR}%{_share_dir}/sbin/update-hbase-env.sh +cp ${RPM_BUILD_DIR}/%{_final_name}/src/packages/rpm/init.d/hbase-master ${RPM_BUILD_DIR}%{_share_dir}/sbin/hbase-master +cp ${RPM_BUILD_DIR}/%{_final_name}/src/packages/rpm/init.d/hbase-regionserver ${RPM_BUILD_DIR}%{_share_dir}/sbin/hbase-regionserver +chmod 0755 ${RPM_BUILD_DIR}%{_share_dir}/sbin/* +rm -f ${RPM_BUILD_DIR}/%{_final_name}/*.txt +rm -f ${RPM_BUILD_DIR}/%{_final_name}/pom.xml +mv -f ${RPM_BUILD_DIR}/%{_final_name}/conf/* ${RPM_BUILD_DIR}%{_conf_dir} +rmdir ${RPM_BUILD_DIR}/%{_final_name}/conf +rm -rf ${RPM_BUILD_DIR}/%{_final_name}/src +mv -f ${RPM_BUILD_DIR}/%{_final_name}/* ${RPM_BUILD_DIR}%{_share_dir} + +%install +cp -Rp ${RPM_BUILD_DIR} ${RPM_BUILD_ROOT} + +%preun +${RPM_INSTALL_PREFIX0}/share/hbase/sbin/update-hbase-env.sh \ + --prefix=${RPM_INSTALL_PREFIX0} \ + --bin-dir=${RPM_INSTALL_PREFIX0}/bin \ + --conf-dir=${RPM_INSTALL_PREFIX1} \ + --log-dir=${RPM_INSTALL_PREFIX2} \ + --pid-dir=${RPM_INSTALL_PREFIX3} \ + --uninstall + +%pre +getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hadoop + +/usr/sbin/useradd --comment "HBase" --shell /bin/bash -M -r --groups hadoop --home %{_share_dir} hbase 2> /dev/null || : + +%post +${RPM_INSTALL_PREFIX0}/share/hbase/sbin/update-hbase-env.sh \ + --prefix=${RPM_INSTALL_PREFIX0} \ + --bin-dir=${RPM_INSTALL_PREFIX0}/bin \ + --conf-dir=${RPM_INSTALL_PREFIX1} \ + --log-dir=${RPM_INSTALL_PREFIX2} \ + --pid-dir=${RPM_INSTALL_PREFIX3} + +%files +%defattr(-,root,root) +%{_prefix} +%config %{_conf_dir} diff --git a/src/packages/update-hbase-env.sh b/src/packages/update-hbase-env.sh new file mode 100644 index 0000000..45276b2 --- /dev/null +++ b/src/packages/update-hbase-env.sh @@ -0,0 +1,193 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script configures hbase-env.sh and symlinkis directories for +# relocating RPM locations. + +usage() { + echo " +usage: $0 + Required parameters: + --prefix=PREFIX path to install into + + Optional parameters: + --arch=i386 OS Architecture + --bin-dir=PREFIX/bin Executable directory + --conf-dir=/etc/hbase Configuration directory + --log-dir=/var/log/hbase Log directory + --pid-dir=/var/run PID file location + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'arch:' \ + -l 'prefix:' \ + -l 'bin-dir:' \ + -l 'conf-dir:' \ + -l 'lib-dir:' \ + -l 'log-dir:' \ + -l 'pid-dir:' \ + -l 'uninstall' \ + -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "${OPTS}" +while true ; do + case "$1" in + --arch) + ARCH=$2 ; shift 2 + ;; + --prefix) + PREFIX=$2 ; shift 2 + ;; + --bin-dir) + BIN_DIR=$2 ; shift 2 + ;; + --log-dir) + LOG_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --conf-dir) + CONF_DIR=$2 ; shift 2 + ;; + --pid-dir) + PID_DIR=$2 ; shift 2 + ;; + --uninstall) + UNINSTALL=1; shift + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +ARCH=${ARCH:-i386} +BIN_DIR=${BIN_DIR:-$PREFIX/share/hbase/bin} +CONF_DIR=${CONF_DIR:-$PREFIX/conf} +LIB_DIR=${LIB_DIR:-$PREFIX/lib} +LOG_DIR=${LOG_DIR:-$PREFIX/var/log} +PID_DIR=${PID_DIR:-$PREFIX/var/run} +UNINSTALL=${UNINSTALL:-0} + +if [ "${ARCH}" != "i386" ]; then + LIB_DIR=${LIB_DIR}64 +fi + +[ -f /etc/default/hadoop-env.sh ] && . /etc/default/hadoop-env.sh +[ -f /etc/default/zookeeper-env.sh ] && . /etc/default/zookeeper-env.sh + +if [ "${UNINSTALL}" -eq "1" ]; then + # Remove symlinks + if [ "${BIN_DIR}" != "${PREFIX}/share/hbase/bin" ]; then + for var in `ls ${PREFIX}/share/hbase/bin`; do + rm -f ${BIN_DIR}/${var} + done + fi + if [ -f /etc/default/hbase-env.sh ]; then + rm -f /etc/default/hbase-env.sh + fi + if [ "${CONF_DIR}" != "${PREFIX}/share/hbase/conf" ]; then + rm -f ${PREFIX}/share/hbase/conf + fi + + rm -f ${PREFIX}/share/hbase/sbin/hbase-master + rm -f ${PREFIX}/share/hbase/sbin/hbase-regionserver + rm -f /etc/init.d/hbase-master + rm -f /etc/init.d/hbase-regionserver + +else + # Create symlinks + if [ "${BIN_DIR}" != "${PREFIX}/share/hbase/bin" ]; then + for var in `ls ${PREFIX}/share/hbase/bin`; do + ln -sf ${PREFIX}/share/hbase/bin/${var} ${BIN_DIR}/${var} + done + fi + if [ "${CONF_DIR}" != "${PREFIX}/share/hbase/conf" ]; then + ln -sf ${CONF_DIR} ${PREFIX}/share/hbase/conf + fi + + chmod 755 ${PREFIX}/share/hbase/sbin/* + + ln -sf ${PREFIX}/share/hbase/sbin/hbase-master /etc/init.d/hbase-master + ln -sf ${PREFIX}/share/hbase/sbin/hbase-regionserver /etc/init.d/hbase-regionserver + + ln -sf ${CONF_DIR}/hbase-env.sh /etc/default/hbase-env.sh + ln -sf ${CONF_DIR}/hbase-env.sh /etc/profile.d/hbase-env.sh + + if [ -n "${HADOOP_HOME}" -a -d "${HADOOP_HOME}" ]; then + HADOOP_JARS=`ls ${HADOOP_HOME}/*.jar | tr '\n' ':'` + fi + + if [ -n "${ZOOKEEPER_HOME}" -a -d "${ZOOKEEPER_HOME}/share/zookeeper" ]; then + ZOOKEEPER_JARS=`ls ${ZOOKEEPER_HOME}/share/zookeeper/*.jar | tr '\n' ':'` + fi + + mkdir -p ${PID_DIR} + mkdir -p ${LOG_DIR} + chown hbase ${PID_DIR} + chown hbase ${LOG_DIR} + + TFILE="/tmp/$(basename $0).$$.tmp" + grep -v "^export HBASE_HOME" ${CONF_DIR}/hbase-env.sh | \ + grep -v "^export HBASE_CONF_DIR" | \ + grep -v "^export HBASE_CLASSPATH" | \ + grep -v "^export HBASE_MANAGES_ZK" | \ + grep -v "^export HBASE_IDENT_STRING" | \ + grep -v "^export HBASE_PID_DIR" | \ + grep -v "^export HBASE_LOG_DIR" | \ + grep -v "^export JAVA_HOME" > ${TFILE} + if [ -z "${JAVA_HOME}" ]; then + if [ -e /etc/lsb-release ]; then + JAVA_HOME=`update-alternatives --config java | grep java | cut -f2 -d':' | cut -f2 -d' ' | sed -e 's/\/bin\/java//'` + else + JAVA_HOME=/usr/java/default + fi + fi + if [ "${JAVA_HOME}xxx" != "xxx" ]; then + echo "export JAVA_HOME=${JAVA_HOME}" >> ${TFILE} + fi + echo "export HBASE_IDENT_STRING=\`whoami\`" >> ${TFILE} + echo "export HBASE_HOME=${PREFIX}/share/hbase" >> ${TFILE} + echo "export HBASE_CONF_DIR=${CONF_DIR}" >> ${TFILE} + echo "export HBASE_CLASSPATH=${CONF_DIR}:${HADOOP_CONF_DIR}:${HADOOP_JARS}:${ZOOKEEPER_JARS}" >> ${TFILE} + echo "export HBASE_MANAGES_ZK=false" >> ${TFILE} + echo "export HBASE_PID_DIR=${PID_DIR}" >> ${TFILE} + echo "export HBASE_LOG_DIR=${LOG_DIR}" >> ${TFILE} + cp ${TFILE} ${CONF_DIR}/hbase-env.sh + rm -f ${TFILE} +fi diff --git a/src/saveVersion.sh b/src/saveVersion.sh new file mode 100644 index 0000000..baae4e2 --- /dev/null +++ b/src/saveVersion.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +# This file is used to generate the annotation of package info that +# records the user, url, revision and timestamp. + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +unset LANG +unset LC_CTYPE +version=$1 +outputDirectory=$2 +user=`whoami` +date=`date` +cwd=`pwd` +if [ -d .svn ]; then + revision=`svn info | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p'` + url=`svn info | sed -n -e 's/URL: \(.*\)/\1/p'` +elif [ -d .git ]; then + revision=`git log -1 --pretty=format:"%H"` + hostname=`hostname` + url="git://${hostname}${cwd}" +else + revision="Unknown" + url="file://$cwd" +fi +mkdir -p "$outputDirectory/org/apache/hadoop/hbase" +cat >"$outputDirectory/org/apache/hadoop/hbase/package-info.java" < + + + + + 2012-04-14 + + Apache HBase + + + Apache HBase software is the Hadoop database. Think of it as a distributed, scalable, big data store. + Use Apache HBase software when you need random, realtime read/write access to your Big Data. This project's goal is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters of commodity hardware. HBase is an open-source, distributed, versioned, column-oriented store modeled after Google's Bigtable: A Distributed Storage System for Structured Data by Chang et al. Just as Bigtable leverages the distributed data storage provided by the Google File System, HBase provides Bigtable-like capabilities on top of Hadoop and HDFS. + + + + Java + + + + Apache hbase 0.92.1 + 2012-03-19 + 0.92.1 + + + + + + + + + + + Apache HBase PMC + + + + + diff --git a/src/site/resources/images/architecture.gif b/src/site/resources/images/architecture.gif new file mode 100644 index 0000000..8d84a23 Binary files /dev/null and b/src/site/resources/images/architecture.gif differ diff --git a/src/site/resources/images/big_h_logo.png b/src/site/resources/images/big_h_logo.png new file mode 100644 index 0000000..5256094 Binary files /dev/null and b/src/site/resources/images/big_h_logo.png differ diff --git a/src/site/resources/images/big_h_logo.svg b/src/site/resources/images/big_h_logo.svg new file mode 100644 index 0000000..ab24198 --- /dev/null +++ b/src/site/resources/images/big_h_logo.svg @@ -0,0 +1,139 @@ + + + +image/svg+xml + + + + + + + + +APACHE + \ No newline at end of file diff --git a/src/site/resources/images/favicon.ico b/src/site/resources/images/favicon.ico new file mode 100644 index 0000000..6e4d0f7 Binary files /dev/null and b/src/site/resources/images/favicon.ico differ diff --git a/src/site/resources/images/hadoop-logo.jpg b/src/site/resources/images/hadoop-logo.jpg new file mode 100644 index 0000000..809525d Binary files /dev/null and b/src/site/resources/images/hadoop-logo.jpg differ diff --git a/src/site/resources/images/hbase_logo.png b/src/site/resources/images/hbase_logo.png new file mode 100644 index 0000000..e962ce0 Binary files /dev/null and b/src/site/resources/images/hbase_logo.png differ diff --git a/src/site/resources/images/hbase_logo.svg b/src/site/resources/images/hbase_logo.svg new file mode 100644 index 0000000..2cc26d9 --- /dev/null +++ b/src/site/resources/images/hbase_logo.svg @@ -0,0 +1,78 @@ + + + +image/svg+xml + + + + + + + \ No newline at end of file diff --git a/src/site/resources/images/hfile.png b/src/site/resources/images/hfile.png new file mode 100644 index 0000000..5762970 Binary files /dev/null and b/src/site/resources/images/hfile.png differ diff --git a/src/site/resources/images/hfilev2.png b/src/site/resources/images/hfilev2.png new file mode 100644 index 0000000..54cc0cf Binary files /dev/null and b/src/site/resources/images/hfilev2.png differ diff --git a/src/site/resources/images/replication_overview.png b/src/site/resources/images/replication_overview.png new file mode 100644 index 0000000..47d7b4c Binary files /dev/null and b/src/site/resources/images/replication_overview.png differ diff --git a/src/site/site.vm b/src/site/site.vm new file mode 100644 index 0000000..f651010 --- /dev/null +++ b/src/site/site.vm @@ -0,0 +1,545 @@ + +#* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*# + +#macro ( link $href $name $target $img $position $alt $border $width $height ) + #set ( $linkTitle = ' title="' + $name + '"' ) + #if( $target ) + #set ( $linkTarget = ' target="' + $target + '"' ) + #else + #set ( $linkTarget = "" ) + #end + #if ( ( $href.toLowerCase().startsWith("http") || $href.toLowerCase().startsWith("https") ) ) + #set ( $linkClass = ' class="externalLink"' ) + #else + #set ( $linkClass = "" ) + #end + #if ( $img ) + #if ( $position == "left" ) + #image($img $alt $border $width $height)$name + #else + $name #image($img $alt $border $width $height) + #end + #else + $name + #end +#end +## +#macro ( image $img $alt $border $width $height ) + #if( $img ) + #if ( ! ( $img.toLowerCase().startsWith("http") || $img.toLowerCase().startsWith("https") ) ) + #set ( $imgSrc = $PathTool.calculateLink( $img, $relativePath ) ) + #set ( $imgSrc = $imgSrc.replaceAll( "\\", "/" ) ) + #set ( $imgSrc = ' src="' + $imgSrc + '"' ) + #else + #set ( $imgSrc = ' src="' + $img + '"' ) + #end + #if( $alt ) + #set ( $imgAlt = ' alt="' + $alt + '"' ) + #else + #set ( $imgAlt = ' alt=""' ) + #end + #if( $border ) + #set ( $imgBorder = ' border="' + $border + '"' ) + #else + #set ( $imgBorder = "" ) + #end + #if( $width ) + #set ( $imgWidth = ' width="' + $width + '"' ) + #else + #set ( $imgWidth = "" ) + #end + #if( $height ) + #set ( $imgHeight = ' height="' + $height + '"' ) + #else + #set ( $imgHeight = "" ) + #end + + #end +#end +#macro ( banner $banner $id ) + #if ( $banner ) + #if( $banner.href ) + + #else + + #end + #end +#end +## +#macro ( links $links ) + #set ( $counter = 0 ) + #foreach( $item in $links ) + #set ( $counter = $counter + 1 ) + #set ( $currentItemHref = $PathTool.calculateLink( $item.href, $relativePath ) ) + #set ( $currentItemHref = $currentItemHref.replaceAll( "\\", "/" ) ) + #link( $currentItemHref $item.name $item.target $item.img $item.position $item.alt $item.border $item.width $item.height ) + #if ( $links.size() > $counter ) + | + #end + #end +#end +## +#macro ( breadcrumbs $breadcrumbs ) + #set ( $counter = 0 ) + #foreach( $item in $breadcrumbs ) + #set ( $counter = $counter + 1 ) + #set ( $currentItemHref = $PathTool.calculateLink( $item.href, $relativePath ) ) + #set ( $currentItemHref = $currentItemHref.replaceAll( "\\", "/" ) ) +## + #if ( $currentItemHref == $alignedFileName || $currentItemHref == "" ) + $item.name + #else + #link( $currentItemHref $item.name $item.target $item.img $item.position $item.alt $item.border $item.width $item.height ) + #end + #if ( $breadcrumbs.size() > $counter ) + > + #end + #end +#end +## +#macro ( displayTree $display $item ) + #if ( $item && $item.items && $item.items.size() > 0 ) + #foreach( $subitem in $item.items ) + #set ( $subitemHref = $PathTool.calculateLink( $subitem.href, $relativePath ) ) + #set ( $subitemHref = $subitemHref.replaceAll( "\\", "/" ) ) + #if ( $alignedFileName == $subitemHref ) + #set ( $display = true ) + #end +## + #displayTree( $display $subitem ) + #end + #end +#end +## +#macro ( menuItem $item ) + #set ( $collapse = "none" ) + #set ( $currentItemHref = $PathTool.calculateLink( $item.href, $relativePath ) ) + #set ( $currentItemHref = $currentItemHref.replaceAll( "\\", "/" ) ) +## + #if ( $item && $item.items && $item.items.size() > 0 ) + #if ( $item.collapse == false ) + #set ( $collapse = "expanded" ) + #else + ## By default collapsed + #set ( $collapse = "collapsed" ) + #end +## + #set ( $display = false ) + #displayTree( $display $item ) +## + #if ( $alignedFileName == $currentItemHref || $display ) + #set ( $collapse = "expanded" ) + #end + #end +
  • + #if ( $item.img ) + #if ( $item.position == "left" ) + #if ( $alignedFileName == $currentItemHref ) + #image($item.img $item.alt $item.border $item.width $item.height) $item.name + #else + #link($currentItemHref $item.name $item.target $item.img $item.position $item.alt $item.border $item.width $item.height) + #end + #else + #if ( $alignedFileName == $currentItemHref ) + $item.name #image($item.img $item.alt $item.border $item.width $item.height) + #else + #link($currentItemHref $item.name $item.target $item.img $item.position $item.alt $item.border $item.width $item.height) + #end + #end + #else + #if ( $alignedFileName == $currentItemHref ) + $item.name + #else + #link( $currentItemHref $item.name $item.target $item.img $item.position $item.alt $item.border $item.width $item.height ) + #end + #end + #if ( $item && $item.items && $item.items.size() > 0 ) + #if ( $collapse == "expanded" ) +
      + #foreach( $subitem in $item.items ) + #menuItem( $subitem ) + #end +
    + #end + #end +
    • +#end +## +#macro ( mainMenu $menus ) + #foreach( $menu in $menus ) + #if ( $menu.name ) + #if ( $menu.img ) + #if( $menu.position ) + #set ( $position = $menu.position ) + #else + #set ( $position = "left" ) + #end +## + #if ( ! ( $menu.img.toLowerCase().startsWith("http") || $menu.img.toLowerCase().startsWith("https") ) ) + #set ( $src = $PathTool.calculateLink( $menu.img, $relativePath ) ) + #set ( $src = $src.replaceAll( "\\", "/" ) ) + #set ( $src = ' src="' + $src + '"' ) + #else + #set ( $src = ' src="' + $menu.img + '"' ) + #end +## + #if( $menu.alt ) + #set ( $alt = ' alt="' + $menu.alt + '"' ) + #else + #set ( $alt = ' alt="' + $menu.name + '"' ) + #end +## + #if( $menu.border ) + #set ( $border = ' border="' + $menu.border + '"' ) + #else + #set ( $border = ' border="0"' ) + #end +## + #if( $menu.width ) + #set ( $width = ' width="' + $menu.width + '"' ) + #else + #set ( $width = "" ) + #end + #if( $menu.height ) + #set ( $height = ' height="' + $menu.height + '"' ) + #else + #set ( $height = "" ) + #end +## + #set ( $img = '" ) +## + #if ( $position == "left" ) +
    $img $menu.name
    + #else +
    $menu.name $img
    + #end + #else +
    $menu.name
    + #end + #end + #if ( $menu.items && $menu.items.size() > 0 ) +
      + #foreach( $item in $menu.items ) + #menuItem( $item ) + #end +
    + #end + #end +#end +## +#macro ( copyright ) + #if ( $project ) + #if ( ${project.organization} && ${project.organization.name} ) + #set ( $period = "" ) + #else + #set ( $period = "." ) + #end +## + #set ( $currentYear = ${currentDate.year} + 1900 ) +## + #if ( ${project.inceptionYear} && ( ${project.inceptionYear} != ${currentYear.toString()} ) ) + ${project.inceptionYear}-${currentYear}${period} + #else + ${currentYear}${period} + #end +## + #if ( ${project.organization} ) + #if ( ${project.organization.name} && ${project.organization.url} ) + ${project.organization.name}. + #elseif ( ${project.organization.name} ) + ${project.organization.name}. + #end + #end + #end +#end +## +#macro ( publishDate $position $publishDate $version ) + #if ( $publishDate && $publishDate.format ) + #set ( $format = $publishDate.format ) + #else + #set ( $format = "yyyy-MM-dd" ) + #end +## + $dateFormat.applyPattern( $format ) +## + #set ( $dateToday = $dateFormat.format( $currentDate ) ) +## + #if ( $publishDate && $publishDate.position ) + #set ( $datePosition = $publishDate.position ) + #else + #set ( $datePosition = "left" ) + #end +## + #if ( $version ) + #if ( $version.position ) + #set ( $versionPosition = $version.position ) + #else + #set ( $versionPosition = "left" ) + #end + #else + #set ( $version = "" ) + #set ( $versionPosition = "left" ) + #end +## + #set ( $breadcrumbs = $decoration.body.breadcrumbs ) + #set ( $links = $decoration.body.links ) + + #if ( $datePosition.equalsIgnoreCase( "right" ) && $links && $links.size() > 0 ) + #set ( $prefix = " |" ) + #else + #set ( $prefix = "" ) + #end +## + #if ( $datePosition.equalsIgnoreCase( $position ) ) + #if ( ( $datePosition.equalsIgnoreCase( "right" ) ) || ( $datePosition.equalsIgnoreCase( "bottom" ) ) ) + $prefix $i18n.getString( "site-renderer", $locale, "template.lastpublished" ): $dateToday + #if ( $versionPosition.equalsIgnoreCase( $position ) ) +  | $i18n.getString( "site-renderer", $locale, "template.version" ): ${project.version} + #end + #elseif ( ( $datePosition.equalsIgnoreCase( "navigation-bottom" ) ) || ( $datePosition.equalsIgnoreCase( "navigation-top" ) ) ) +
    + $i18n.getString( "site-renderer", $locale, "template.lastpublished" ): $dateToday + #if ( $versionPosition.equalsIgnoreCase( $position ) ) +  | $i18n.getString( "site-renderer", $locale, "template.version" ): ${project.version} + #end +
    + #elseif ( $datePosition.equalsIgnoreCase("left") ) +
    + $i18n.getString( "site-renderer", $locale, "template.lastpublished" ): $dateToday + #if ( $versionPosition.equalsIgnoreCase( $position ) ) +  | $i18n.getString( "site-renderer", $locale, "template.version" ): ${project.version} + #end + #if ( $breadcrumbs && $breadcrumbs.size() > 0 ) + | #breadcrumbs( $breadcrumbs ) + #end +
    + #end + #elseif ( $versionPosition.equalsIgnoreCase( $position ) ) + #if ( ( $versionPosition.equalsIgnoreCase( "right" ) ) || ( $versionPosition.equalsIgnoreCase( "bottom" ) ) ) + $prefix $i18n.getString( "site-renderer", $locale, "template.version" ): ${project.version} + #elseif ( ( $versionPosition.equalsIgnoreCase( "navigation-bottom" ) ) || ( $versionPosition.equalsIgnoreCase( "navigation-top" ) ) ) +
    + $i18n.getString( "site-renderer", $locale, "template.version" ): ${project.version} +
    + #elseif ( $versionPosition.equalsIgnoreCase("left") ) +
    + $i18n.getString( "site-renderer", $locale, "template.version" ): ${project.version} + #if ( $breadcrumbs && $breadcrumbs.size() > 0 ) + | #breadcrumbs( $breadcrumbs ) + #end +
    + #end + #elseif ( $position.equalsIgnoreCase( "left" ) ) + #if ( $breadcrumbs && $breadcrumbs.size() > 0 ) +
    + #breadcrumbs( $breadcrumbs ) +
    + #end + #end +#end +## +#macro ( poweredByLogo $poweredBy ) + #if( $poweredBy ) + #foreach ($item in $poweredBy) + #if( $item.href ) + #set ( $href = $PathTool.calculateLink( $item.href, $relativePath ) ) + #set ( $href = $href.replaceAll( "\\", "/" ) ) + #else + #set ( $href="http://maven.apache.org/" ) + #end +## + #if( $item.name ) + #set ( $name = $item.name ) + #else + #set ( $name = $i18n.getString( "site-renderer", $locale, "template.builtby" ) ) + #set ( $name = "${name} Maven" ) + #end +## + #if( $item.img ) + #set ( $img = $item.img ) + #else + #set ( $img = "images/logos/maven-feather.png" ) + #end +## + #if ( ! ( $img.toLowerCase().startsWith("http") || $img.toLowerCase().startsWith("https") ) ) + #set ( $img = $PathTool.calculateLink( $img, $relativePath ) ) + #set ( $img = $src.replaceAll( "\\", "/" ) ) + #end +## + #if( $item.alt ) + #set ( $alt = ' alt="' + $item.alt + '"' ) + #else + #set ( $alt = ' alt="' + $name + '"' ) + #end +## + #if( $item.border ) + #set ( $border = ' border="' + $item.border + '"' ) + #else + #set ( $border = "" ) + #end +## + #if( $item.width ) + #set ( $width = ' width="' + $item.width + '"' ) + #else + #set ( $width = "" ) + #end + #if( $item.height ) + #set ( $height = ' height="' + $item.height + '"' ) + #else + #set ( $height = "" ) + #end +## + + + + #end + #if( $poweredBy.isEmpty() ) + + $i18n.getString( + + #end + #else + + $i18n.getString( + + #end +#end +## + + + + $title + + + +#foreach( $author in $authors ) + +#end +#if ( $dateCreation ) + +#end +#if ( $dateRevision ) + +#end +#if ( $locale ) + +#end + #if ( $decoration.body.head ) + #foreach( $item in $decoration.body.head.getChildren() ) + ## Workaround for DOXIA-150 due to a non-desired behaviour in p-u + ## @see org.codehaus.plexus.util.xml.Xpp3Dom#toString() + ## @see org.codehaus.plexus.util.xml.Xpp3Dom#toUnescapedString() + #set ( $documentHeader = "" ) + #set ( $documentHeader = $documentHeader.replaceAll( "\\", "" ) ) + #if ( $item.name == "script" ) + $StringUtils.replace( $item.toUnescapedString(), $documentHeader, "" ) + #else + $StringUtils.replace( $item.toString(), $documentHeader, "" ) + #end + #end + #end + ## $headContent + + + + + + +
    + +
    +
    +
    + $bodyContent +
    +
    +
    +
    +
    + + + diff --git a/src/site/site.xml b/src/site/site.xml new file mode 100644 index 0000000..a6b7579 --- /dev/null +++ b/src/site/site.xml @@ -0,0 +1,77 @@ + + + + + + + Apache HBase + images/hbase_logo.png + http://hbase.apache.org/ + + + Version 0.94 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.maven.skins + maven-stylus-skin + + diff --git a/src/site/xdoc/acid-semantics.xml b/src/site/xdoc/acid-semantics.xml new file mode 100644 index 0000000..81f7f04 --- /dev/null +++ b/src/site/xdoc/acid-semantics.xml @@ -0,0 +1,231 @@ + + + + + + + + + Apache HBase (TM) ACID Properties + + + + +
    +

    Apache HBase (TM) is not an ACID compliant database. However, it does guarantee certain specific + properties.

    +

    This specification enumerates the ACID properties of HBase.

    +
    +
    +

    For the sake of common vocabulary, we define the following terms:

    +
    +
    Atomicity
    +
    an operation is atomic if it either completes entirely or not at all
    + +
    Consistency
    +
    + all actions cause the table to transition from one valid state directly to another + (eg a row will not disappear during an update, etc) +
    + +
    Isolation
    +
    + an operation is isolated if it appears to complete independently of any other concurrent transaction +
    + +
    Durability
    +
    any update that reports "successful" to the client will not be lost
    + +
    Visibility
    +
    an update is considered visible if any subsequent read will see the update as having been committed
    +
    +

    + The terms must and may are used as specified by RFC 2119. + In short, the word "must" implies that, if some case exists where the statement + is not true, it is a bug. The word "may" implies that, even if the guarantee + is provided in a current release, users should not rely on it. +

    +
    +
    +
      +
    • Read APIs +
        +
      • get
        • +
      • scan
        • +
      +
      • +
    • Write APIs
      • +
        +
      • put
        • +
      • batch put
        • +
      • delete
        • +
      +
    • Combination (read-modify-write) APIs
      • +
        +
      • incrementColumnValue
        • +
      • checkAndPut
        • +
      +
    +
    + +
    + +
    + +
      +
    1. All mutations are atomic within a row. Any put will either wholely succeed or wholely fail.[3]
      2. +
        +
      1. An operation that returns a "success" code has completely succeeded.
        2. +
      2. An operation that returns a "failure" code has completely failed.
        3. +
      3. An operation that times out may have succeeded and may have failed. However, + it will not have partially succeeded or failed.
        4. +
      +
    2. This is true even if the mutation crosses multiple column families within a row.
      3. +
    3. APIs that mutate several rows will _not_ be atomic across the multiple rows. + For example, a multiput that operates on rows 'a','b', and 'c' may return having + mutated some but not all of the rows. In such cases, these APIs will return a list + of success codes, each of which may be succeeded, failed, or timed out as described above.
      4. +
    4. The checkAndPut API happens atomically like the typical compareAndSet (CAS) operation + found in many hardware architectures.
      5. +
    5. The order of mutations is seen to happen in a well-defined order for each row, with no + interleaving. For example, if one writer issues the mutation "a=1,b=1,c=1" and + another writer issues the mutation "a=2,b=2,c=2", the row must either + be "a=1,b=1,c=1" or "a=2,b=2,c=2" and must not be something + like "a=1,b=2,c=1".
      6. +
        +
      1. Please note that this is not true _across rows_ for multirow batch mutations.
        2. +
      +
    +
    +
    +
      +
    1. All rows returned via any access API will consist of a complete row that existed at + some point in the table's history.
      2. +
    2. This is true across column families - i.e a get of a full row that occurs concurrent + with some mutations 1,2,3,4,5 will return a complete row that existed at some point in time + between mutation i and i+1 for some i between 1 and 5.
      3. +
    3. The state of a row will only move forward through the history of edits to it.
      4. +
    + +
    +

    + A scan is not a consistent view of a table. Scans do + not exhibit snapshot isolation. +

    +

    + Rather, scans have the following properties: +

    + +
      +
    1. + Any row returned by the scan will be a consistent view (i.e. that version + of the complete row existed at some point in time) [1] +
      2. +
    2. + A scan will always reflect a view of the data at least as new as + the beginning of the scan. This satisfies the visibility guarantees + enumerated below.
      3. +
        +
      1. For example, if client A writes data X and then communicates via a side + channel to client B, any scans started by client B will contain data at least + as new as X.
        2. +
      2. A scan _must_ reflect all mutations committed prior to the construction + of the scanner, and _may_ reflect some mutations committed subsequent to the + construction of the scanner.
        3. +
      3. Scans must include all data written prior to the scan (except in + the case where data is subsequently mutated, in which case it _may_ reflect + the mutation)
        4. +
      +
    +

    + Those familiar with relational databases will recognize this isolation level as "read committed". +

    +

    + Please note that the guarantees listed above regarding scanner consistency + are referring to "transaction commit time", not the "timestamp" + field of each cell. That is to say, a scanner started at time t may see edits + with a timestamp value greater than t, if those edits were committed with a + "forward dated" timestamp before the scanner was constructed. +

    +
    +
    +
    +
      +
    1. When a client receives a "success" response for any mutation, that + mutation is immediately visible to both that client and any client with whom it + later communicates through side channels. [3]
      2. +
    2. A row must never exhibit so-called "time-travel" properties. That + is to say, if a series of mutations moves a row sequentially through a series of + states, any sequence of concurrent reads will return a subsequence of those states.
      3. +
        +
      1. For example, if a row's cells are mutated using the "incrementColumnValue" + API, a client must never see the value of any cell decrease.
        2. +
      2. This is true regardless of which read API is used to read back the mutation.
        3. +
      +
    3. Any version of a cell that has been returned to a read operation is guaranteed to + be durably stored.
      4. +
    + +
    +
    +
      +
    1. All visible data is also durable data. That is to say, a read will never return + data that has not been made durable on disk[2]
      2. +
    2. Any operation that returns a "success" code (eg does not throw an exception) + will be made durable.[3]
      3. +
    3. Any operation that returns a "failure" code will not be made durable + (subject to the Atomicity guarantees above)
      4. +
    4. All reasonable failure scenarios will not affect any of the guarantees of this document.
      5. + +
    +
    +
    +

    All of the above guarantees must be possible within Apache HBase. For users who would like to trade + off some guarantees for performance, HBase may offer several tuning options. For example:

    +
      +
    • Visibility may be tuned on a per-read basis to allow stale reads or time travel.
      • +
    • Durability may be tuned to only flush data to disk on a periodic basis
      • +
    +
    +
    +
    +

    + For more information, see the client architecture or data model sections in the Apache HBase Reference Guide. +

    +
    + +
    +

    [1] A consistent view is not guaranteed intra-row scanning -- i.e. fetching a portion of + a row in one RPC then going back to fetch another portion of the row in a subsequent RPC. + Intra-row scanning happens when you set a limit on how many values to return per Scan#next + (See Scan#setBatch(int)). +

    + +

    [2] In the context of Apache HBase, "durably on disk" implies an hflush() call on the transaction + log. This does not actually imply an fsync() to magnetic media, but rather just that the data has been + written to the OS cache on all replicas of the log. In the case of a full datacenter power loss, it is + possible that the edits are not truly durable.

    +

    [3] Puts will either wholely succeed or wholely fail, provided that they are actually sent + to the RegionServer. If the writebuffer is used, Puts will not be sent until the writebuffer is filled + or it is explicitly flushed.

    + +
    + + +     diff --git a/src/site/xdoc/bulk-loads.xml b/src/site/xdoc/bulk-loads.xml new file mode 100644 index 0000000..f4d3d75 --- /dev/null +++ b/src/site/xdoc/bulk-loads.xml @@ -0,0 +1,30 @@ + + + + + + Bulk Loads in Apache HBase (TM) + + + +

    This page has been retired. The contents have been moved to the + Bulk Loading section + in the Reference Guide. +

    + +     diff --git a/src/site/xdoc/cygwin.xml b/src/site/xdoc/cygwin.xml new file mode 100644 index 0000000..40e0f72 --- /dev/null +++ b/src/site/xdoc/cygwin.xml @@ -0,0 +1,241 @@ + + + + + Installing Apache HBase (TM) on Windows using Cygwin + + + +
    +

    Apache HBase (TM) is a distributed, column-oriented store, modeled after Google's BigTable. Apache HBase is built on top of Hadoop for its MapReduce and distributed file system implementation. All these projects are open-source and part of the Apache Software Foundation.

    + +

    As being distributed, large scale platforms, the Hadoop and HBase projects mainly focus on *nix environments for production installations. However, being developed in Java, both projects are fully portable across platforms and, hence, also to the Windows operating system. For ease of development the projects rely on Cygwin to have a *nix-like environment on Windows to run the shell scripts.

    +
    +
    +

    This document explains the intricacies of running Apache HBase on Windows using Cygwin as an all-in-one single-node installation for testing and development. The HBase Overview and QuickStart guides on the other hand go a long way in explaning how to setup HBase in more complex deployment scenario's.

    +
    + +
    +

    For running Apache HBase on Windows, 3 technologies are required: Java, Cygwin and SSH. The following paragraphs detail the installation of each of the aforementioned technologies.

    +
    +

    HBase depends on the Java Platform, Standard Edition, 6 Release. So the target system has to be provided with at least the Java Runtime Environment (JRE); however if the system will also be used for development, the Jave Development Kit (JDK) is preferred. You can download the latest versions for both from Sun's download page. Installation is a simple GUI wizard that guides you through the process.

    +
    +
    +

    Cygwin is probably the oddest technology in this solution stack. It provides a dynamic link library that emulates most of a *nix environment on Windows. On top of that a whole bunch of the most common *nix tools are supplied. Combined, the DLL with the tools form a very *nix-alike environment on Windows.

    + +

    For installation, Cygwin provides the setup.exe utility that tracks the versions of all installed components on the target system and provides the mechanism for installing or updating everything from the mirror sites of Cygwin.

    + +

    To support installation, the setup.exe utility uses 2 directories on the target system. The Root directory for Cygwin (defaults to C:\cygwin) which will become / within the eventual Cygwin installation; and the Local Package directory (e.g. C:\cygsetup that is the cache where setup.exe stores the packages before they are installed. The cache must not be the same folder as the Cygwin root.

    + +

    Perform following steps to install Cygwin, which are elaboratly detailed in the 2nd chapter of the Cygwin User's Guide:

    + +
      +
    1. Make sure you have Administrator privileges on the target system.
      2. +
    2. Choose and create you Root and Local Package directories. A good suggestion is to use C:\cygwin\root and C:\cygwin\setup folders.
      3. +
    3. Download the setup.exe utility and save it to the Local Package directory.
      4. +
    4. Run the setup.exe utility, +
        +
      1. Choose the Install from Internet option,
        2. +
      2. Choose your Root and Local Package folders
        3. +
      3. and select an appropriate mirror.
        4. +
      4. Don't select any additional packages yet, as we only want to install Cygwin for now.
        5. +
      5. Wait for download and install
        6. +
      6. Finish the installation
        7. +
      +
      5. +
    5. Optionally, you can now also add a shortcut to your Start menu pointing to the setup.exe utility in the Local Package folder.
      6. +
    6. Add CYGWIN_HOME system-wide environment variable that points to your Root directory.
      7. +
    7. Add %CYGWIN_HOME%\bin to the end of your PATH environment variable.
      8. +
    8. Reboot the sytem after making changes to the environment variables otherwise the OS will not be able to find the Cygwin utilities.
      9. +
    9. Test your installation by running your freshly created shortcuts or the Cygwin.bat command in the Root folder. You should end up in a terminal window that is running a Bash shell. Test the shell by issuing following commands: +
        +
      1. cd / should take you to thr Root directory in Cygwin;
        2. +
      2. the LS commands that should list all files and folders in the current directory.
        3. +
      3. Use the exit command to end the terminal.
        4. +
      +
      10. +
    10. When needed, to uninstall Cygwin you can simply delete the Root and Local Package directory, and the shortcuts that were created during installation.
      11. +
    +
    +
    +

    HBase (and Hadoop) rely on SSH for interprocess/-node communication and launching remote commands. SSH will be provisioned on the target system via Cygwin, which supports running Cygwin programs as Windows services!

    + +
      +
    1. Rerun the setup.exe utility.
      2. +
    2. Leave all parameters as is, skipping through the wizard using the Next button until the Select Packages panel is shown.
      3. +
    3. Maximize the window and click the View button to toggle to the list view, which is ordered alfabetically on Package, making it easier to find the packages we'll need.
      4. +
    4. Select the following packages by clicking the status word (normally Skip) so it's marked for installation. Use the Next button to download and install the packages. +
        +
      1. OpenSSH
        2. +
      2. tcp_wrappers
        3. +
      3. diffutils
        4. +
      4. zlib
        5. +
      +
      5. +
    5. Wait for the install to complete and finish the installation.
      6. +
    +
    +
    +

    Download the latest release of Apache HBase from the website. As the Apache HBase distributable is just a zipped archive, installation is as simple as unpacking the archive so it ends up in its final installation directory. Notice that HBase has to be installed in Cygwin and a good directory suggestion is to use /usr/local/ (or [Root directory]\usr\local in Windows slang). You should end up with a /usr/local/hbase-<version> installation in Cygwin.

    + +This finishes installation. We go on with the configuration. +
    +
    +
    +

    There are 3 parts left to configure: Java, SSH and HBase itself. Following paragraphs explain eacht topic in detail.

    +
    +

    One important thing to remember in shell scripting in general (i.e. *nix and Windows) is that managing, manipulating and assembling path names that contains spaces can be very hard, due to the need to escape and quote those characters and strings. So we try to stay away from spaces in path names. *nix environments can help us out here very easily by using symbolic links.

    + +
      +
    1. Create a link in /usr/local to the Java home directory by using the following command and substituting the name of your chosen Java environment: +
      LN -s /cygdrive/c/Program\ Files/Java/<jre name> /usr/local/<jre name>
      +
      2. +
    2. Test your java installation by changing directories to your Java folder CD /usr/local/<jre name> and issueing the command ./bin/java -version. This should output your version of the chosen JRE.
      3. +
    +
    +
    +SSH +

    Configuring SSH is quite elaborate, but primarily a question of launching it by default as a Windows service.

    + +
      +
    1. On Windows Vista and above make sure you run the Cygwin shell with elevated privileges, by right-clicking on the shortcut an using Run as Administrator.
      2. +
    2. First of all, we have to make sure the rights on some crucial files are correct. Use the commands underneath. You can verify all rights by using the LS -L command on the different files. Also, notice the auto-completion feature in the shell using <TAB> is extremely handy in these situations. +
        +
      1. chmod +r /etc/passwd to make the passwords file readable for all
        2. +
      2. chmod u+w /etc/passwd to make the passwords file writable for the owner
        3. +
      3. chmod +r /etc/group to make the groups file readable for all
        4. +
      +
        +
      1. chmod u+w /etc/group to make the groups file writable for the owner
        2. +
      +
        +
      1. chmod 755 /var to make the var folder writable to owner and readable and executable to all
        2. +
      +
      3. +
    3. Edit the /etc/hosts.allow file using your favorite editor (why not VI in the shell!) and make sure the following two lines are in there before the PARANOID line: +
        +
      1. ALL : localhost 127.0.0.1/32 : allow
        2. +
      2. ALL : [::1]/128 : allow
        3. +
      +
      4. +
    4. Next we have to configure SSH by using the script ssh-host-config +
        +
      1. If this script asks to overwrite an existing /etc/ssh_config, answer yes.
        2. +
      2. If this script asks to overwrite an existing /etc/sshd_config, answer yes.
        3. +
      3. If this script asks to use privilege separation, answer yes.
        4. +
      4. If this script asks to install sshd as a service, answer yes. Make sure you started your shell as Adminstrator!
        5. +
      5. If this script asks for the CYGWIN value, just <enter> as the default is ntsec.
        6. +
      6. If this script asks to create the sshd account, answer yes.
        7. +
      7. If this script asks to use a different user name as service account, answer no as the default will suffice.
        8. +
      8. If this script asks to create the cyg_server account, answer yes. Enter a password for the account.
        9. +
      +
      5. +
    5. Start the SSH service using net start sshd or cygrunsrv --start sshd. Notice that cygrunsrv is the utility that make the process run as a Windows service. Confirm that you see a message stating that the CYGWIN sshd service was started succesfully.
      6. +
    6. Harmonize Windows and Cygwin user account by using the commands: +
        +
      1. mkpasswd -cl > /etc/passwd
        2. +
      2. mkgroup --local > /etc/group
        3. +
      +
      7. +
    7. Test the installation of SSH: +
        +
      1. Open a new Cygwin terminal
        2. +
      2. Use the command whoami to verify your userID
        3. +
      3. Issue an ssh localhost to connect to the system itself +
          +
        1. Answer yes when presented with the server's fingerprint
          2. +
        2. Issue your password when prompted
          3. +
        3. test a few commands in the remote session
          4. +
        4. The exit command should take you back to your first shell in Cygwin
          5. +
        +
        4. +
      4. Exit should terminate the Cygwin shell.
        5. +
      +
      8. +
    +
    +
    +If all previous configurations are working properly, we just need some tinkering at the HBase config files to properly resolve on Windows/Cygwin. All files and paths referenced here start from the HBase [installation directory] as working directory. +
      +
    1. HBase uses the ./conf/hbase-env.sh to configure its dependencies on the runtime environment. Copy and uncomment following lines just underneath their original, change them to fit your environemnt. They should read something like: +
        +
      1. export JAVA_HOME=/usr/local/<jre name>
        2. +
      2. export HBASE_IDENT_STRING=$HOSTNAME as this most likely does not inlcude spaces.
        3. +
      +
      2. +
    2. HBase uses the ./conf/hbase-default.xml file for configuration. Some properties do not resolve to existing directories because the JVM runs on Windows. This is the major issue to keep in mind when working with Cygwin: within the shell all paths are *nix-alike, hence relative to the root /. However, every parameter that is to be consumed within the windows processes themself, need to be Windows settings, hence C:\-alike. Change following propeties in the configuration file, adjusting paths where necessary to conform with your own installation: +
        +
      1. hbase.rootdir must read e.g. file:///C:/cygwin/root/tmp/hbase/data
        2. +
      2. hbase.tmp.dir must read C:/cygwin/root/tmp/hbase/tmp
        3. +
      3. hbase.zookeeper.quorum must read 127.0.0.1 because for some reason localhost doesn't seem to resolve properly on Cygwin.
        4. +
      +
      3. +
    3. Make sure the configured hbase.rootdir and hbase.tmp.dir directories exist and have the proper rights set up e.g. by issuing a chmod 777 on them.
      4. +
    +
    +
    +
    +Testing +

    +This should conclude the installation and configuration of Apache HBase on Windows using Cygwin. So it's time to test it. +

      +
    1. Start a Cygwin terminal, if you haven't already.
      2. +
    2. Change directory to HBase installation using CD /usr/local/hbase-<version>, preferably using auto-completion.
      3. +
    3. Start HBase using the command ./bin/start-hbase.sh +
        +
      1. When prompted to accept the SSH fingerprint, answer yes.
        2. +
      2. When prompted, provide your password. Maybe multiple times.
        3. +
      3. When the command completes, the HBase server should have started.
        4. +
      4. However, to be absolutely certain, check the logs in the ./logs directory for any exceptions.
        5. +
      +
      4. +
    4. Next we start the HBase shell using the command ./bin/hbase shell
      5. +
    5. We run some simple test commands +
        +
      1. Create a simple table using command create 'test', 'data'
        2. +
      2. Verify the table exists using the command list
        3. +
      3. Insert data into the table using e.g. +
        put 'test', 'row1', 'data:1', 'value1'
+put 'test', 'row2', 'data:2', 'value2'
+put 'test', 'row3', 'data:3', 'value3'
        +
        4. +
      4. List all rows in the table using the command scan 'test' that should list all the rows previously inserted. Notice how 3 new columns where added without changing the schema!
        5. +
      5. Finally we get rid of the table by issuing disable 'test' followed by drop 'test' and verified by list which should give an empty listing.
        6. +
      +
      6. +
    6. Leave the shell by exit
      7. +
    7. To stop the HBase server issue the ./bin/stop-hbase.sh command. And wait for it to complete!!! Killing the process might corrupt your data on disk.
      8. +
    8. In case of problems, +
        +
      1. verify the HBase logs in the ./logs directory.
        2. +
      2. Try to fix the problem
        3. +
      3. Get help on the forums or IRC (#hbase@freenode.net). People are very active and keen to help out!
        4. +
      4. Stopr, restart and retest the server.
        5. +
      +
      9. +
    +

    +
    + +
    +

    +Now your HBase server is running, start coding and build that next killer app on this particular, but scalable datastore! +

    +
    + +     diff --git a/src/site/xdoc/index.xml b/src/site/xdoc/index.xml new file mode 100644 index 0000000..b4c1f81 --- /dev/null +++ b/src/site/xdoc/index.xml @@ -0,0 +1,82 @@ + + + + + Apache HBase™ Home + + + + +
    +

    Apache HBase™ is the Hadoop database, a distributed, scalable, big data store. +

    +

    When Would I Use Apache HBase?

    +

    + Use Apache HBase when you need random, realtime read/write access to your Big Data. + This project's goal is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters of commodity hardware. +Apache HBase is an open-source, distributed, versioned, column-oriented store modeled after Google's Bigtable: A Distributed Storage System for Structured Data by Chang et al. + Just as Bigtable leverages the distributed data storage provided by the Google File System, Apache HBase provides Bigtable-like capabilities on top of Hadoop and HDFS. +

    +

    Features

    +

    +

      +
    • Linear and modular scalability. +
      • +
    • Strictly consistent reads and writes. +
      • +
    • Automatic and configurable sharding of tables +
      • +
    • Automatic failover support between RegionServers. +
      • +
    • Convenient base classes for backing Hadoop MapReduce jobs with Apache HBase tables. +
      • +
    • Easy to use Java API for client access. +
      • +
    • Block cache and Bloom Filters for real-time queries. +
      • +
    • Query predicate push down via server side Filters +
      • +
    • Thrift gateway and a REST-ful Web service that supports XML, Protobuf, and binary data encoding options +
      • +
    • Extensible jruby-based (JIRB) shell +
      • +
    • Support for exporting metrics via the Hadoop metrics subsystem to files or Ganglia; or via JMX +
      • +
    +

    +

    Where Can I Get More Information?

    +

    See the Architecture Overview, the Apache HBase Reference Guide FAQ, + and the other documentation links on the left! +

    +
    +
    +

    June 13th, 2013 HBaseCon2013 in San Francisco. Submit an Abstract!

    +

    April 11th, 2013 HBase Meetup at AdRoll in San Francisco

    +

    February 28th, 2013 HBase Meetup at Intel Mission Campus

    +

    February 19th, 2013 Developers PowWow at HortonWorks' new digs

    +

    January 23rd, 2013 HBase Meetup at WibiData World HQ!

    +

    December 4th, 2012 0.96 Bug Squashing and Testing Hackathon at Cloudera, SF.

    +

    October 29th, 2012 HBase User Group Meetup at Wize Commerce in San Mateo.

    +

    October 25th, 2012 Strata/Hadoop World HBase Meetup. in NYC

    +

    September 11th, 2012 Contributor's Pow-Wow at HortonWorks HQ.

    + +

    Old News

    +
    + + +     diff --git a/src/site/xdoc/metrics.xml b/src/site/xdoc/metrics.xml new file mode 100644 index 0000000..47902b8 --- /dev/null +++ b/src/site/xdoc/metrics.xml @@ -0,0 +1,146 @@ + + + + + + Apache HBase (TM) Metrics + + + + +
    +

    + Apache HBase (TM) emits Hadoop metrics. +

    +
    +
    +

    First read up on Hadoop metrics. + If you are using ganglia, the GangliaMetrics + wiki page is useful read.

    +

    To have HBase emit metrics, edit $HBASE_HOME/conf/hadoop-metrics.properties + and enable metric 'contexts' per plugin. As of this writing, hadoop supports + file and ganglia plugins. + Yes, the hbase metrics files is named hadoop-metrics rather than + hbase-metrics because currently at least the hadoop metrics system has the + properties filename hardcoded. Per metrics context, + comment out the NullContext and enable one or more plugins instead. +

    +

    + If you enable the hbase context, on regionservers you'll see total requests since last + metric emission, count of regions and storefiles as well as a count of memstore size. + On the master, you'll see a count of the cluster's requests. +

    +

    + Enabling the rpc context is good if you are interested in seeing + metrics on each hbase rpc method invocation (counts and time taken). +

    +

    + The jvm context is + useful for long-term stats on running hbase jvms -- memory used, thread counts, etc. + As of this writing, if more than one jvm is running emitting metrics, at least + in ganglia, the stats are aggregated rather than reported per instance. +

    +
    + +
    +

    + In addition to the standard output contexts supported by the Hadoop + metrics package, you can also export HBase metrics via Java Management + Extensions (JMX). This will allow viewing HBase stats in JConsole or + any other JMX client. +

    +
    +

    + To enable JMX support in HBase, first edit + $HBASE_HOME/conf/hadoop-metrics.properties to support + metrics refreshing. (If you've running 0.94.1 and above, or have already configured + hadoop-metrics.properties for another output context, + you can skip this step). +

    + +# Configuration of the "hbase" context for null +hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread +hbase.period=60 + +# Configuration of the "jvm" context for null +jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread +jvm.period=60 + +# Configuration of the "rpc" context for null +rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread +rpc.period=60 + +
    +
    +

    + For remote access, you will need to configure JMX remote passwords + and access profiles. Create the files: +

    +
    +
    $HBASE_HOME/conf/jmxremote.passwd (set permissions + to 600)
    +
    + +monitorRole monitorpass +controlRole controlpass + +
    + +
    $HBASE_HOME/conf/jmxremote.access
    +
    + +monitorRole readonly +controlRole readwrite + +
    +
    +
    +
    +

    + Finally, edit the $HBASE_HOME/conf/hbase-env.sh + script to add JMX support: +

    +
    +
    $HBASE_HOME/conf/hbase-env.sh
    +
    +

    Add the lines:

    + +HBASE_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false" +HBASE_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.password.file=$HBASE_HOME/conf/jmxremote.passwd" +HBASE_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.access.file=$HBASE_HOME/conf/jmxremote.access" + +export HBASE_MASTER_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=10101" +export HBASE_REGIONSERVER_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=10102" + +
    +
    +

    + After restarting the processes you want to monitor, you should now be + able to run JConsole (included with the JDK since JDK 5.0) to view + the statistics via JMX. HBase MBeans are exported under the + hadoop domain in JMX. +

    +
    +
    +

    + For more information on understanding HBase metrics, see the metrics section in the Apache HBase Reference Guide. +

    +
    +
    + +     diff --git a/src/site/xdoc/old_news.xml b/src/site/xdoc/old_news.xml new file mode 100644 index 0000000..1dc3eae --- /dev/null +++ b/src/site/xdoc/old_news.xml @@ -0,0 +1,69 @@ + + + + + + + + + Old Apache HBase (TM) News + + + +
    +

    August 8th, 2012 Apache HBase 0.94.1 is available for download

    +

    June 15th, 2012 Birds-of-a-feather in San Jose, day after Hadoop Summit

    +

    May 23rd, 2012 HackConAthon in Palo Alto

    +

    May 22nd, 2012 HBaseCon2012 in San Francisco

    +

    March 27th, 2012 Meetup @ StumbleUpon in San Francisco

    + +

    January 19th, 2012 Meetup @ EBay

    +

    January 23rd, 2012 Apache HBase 0.92.0 released. Download it!

    +

    December 23rd, 2011 Apache HBase 0.90.5 released. Download it!

    +

    November 29th, 2011 Developer Pow-Wow in SF at Salesforce HQ

    +

    November 7th, 2011 HBase Meetup in NYC (6PM) at the AppNexus office

    +

    August 22nd, 2011 HBase Hackathon (11AM) and Meetup (6PM) at FB in PA

    +

    June 30th, 2011 HBase Contributor Day, the day after the Hadoop Summit hosted by Y!

    +

    June 8th, 2011 HBase Hackathon in Berlin to coincide with Berlin Buzzwords

    +

    May 19th, 2011 Apache HBase 0.90.3 released. Download it!

    +

    April 12th, 2011 Apache HBase 0.90.2 released. Download it!

    +

    March 21st, HBase 0.92 Hackathon at StumbleUpon, SF

    +

    February 22nd, HUG12: February HBase User Group at StumbleUpon SF

    +

    December 13th, HBase Hackathon: Coprocessor Edition

    +

    November 19th, Hadoop HUG in London is all about Apache HBase

    +

    November 15-19th, Devoxx features HBase Training and multiple HBase presentations

    +

    October 12th, HBase-related presentations by core contributors and users at Hadoop World 2010

    +

    October 11th, HUG-NYC: HBase User Group NYC Edition (Night before Hadoop World)

    +

    June 30th, Apache HBase Contributor Workshop (Day after Hadoop Summit)

    +

    May 10th, 2010: Apache HBase graduates from Hadoop sub-project to Apache Top Level Project

    +

    Signup for HBase User Group Meeting, HUG10 hosted by Trend Micro, April 19th, 2010

    + +

    HBase User Group Meeting, HUG9 hosted by Mozilla, March 10th, 2010

    +

    Sign up for the HBase User Group Meeting, HUG8, January 27th, 2010 at StumbleUpon in SF

    +

    September 8th, 2010: Apache HBase 0.20.0 is faster, stronger, slimmer, and sweeter tasting than any previous Apache HBase release. Get it off the Releases page.

    +

    ApacheCon in Oakland: November 2-6th, 2009: + The Apache Foundation will be celebrating its 10th anniversary in beautiful Oakland by the Bay. Lots of good talks and meetups including an HBase presentation by a couple of the lads.

    +

    HBase at Hadoop World in NYC: October 2nd, 2009: A few of us will be talking on Practical HBase out east at Hadoop World: NYC.

    +

    HUG7 and HBase Hackathon: August 7th-9th, 2009 at StumbleUpon in SF: Sign up for the HBase User Group Meeting, HUG7 or for the Hackathon or for both (all are welcome!).

    +

    June, 2009 -- HBase at HadoopSummit2009 and at NOSQL: See the presentations

    +

    March 3rd, 2009 -- HUG6: HBase User Group 6

    +

    January 30th, 2009 -- LA Hbackathon:HBase January Hackathon Los Angeles at Streamy in Manhattan Beach

    +
    + +     diff --git a/src/site/xdoc/pseudo-distributed.xml b/src/site/xdoc/pseudo-distributed.xml new file mode 100644 index 0000000..a1b38ba --- /dev/null +++ b/src/site/xdoc/pseudo-distributed.xml @@ -0,0 +1,38 @@ + + + + + + + + +Running Apache HBase (TM) in pseudo-distributed mode + + + + +

    This page has been retired. The contents have been moved to the + Distributed Operation: Pseudo- and Fully-distributed modes section + in the Reference Guide. +

    + + + +     + diff --git a/src/site/xdoc/replication.xml b/src/site/xdoc/replication.xml new file mode 100644 index 0000000..727ca26 --- /dev/null +++ b/src/site/xdoc/replication.xml @@ -0,0 +1,551 @@ + + + + + + + + + Apache HBase (TM) Replication + + + +
    +

    + The replication feature of Apache HBase (TM) provides a way to copy data between HBase deployments. It + can serve as a disaster recovery solution and can contribute to provide + higher availability at the HBase layer. It can also serve more practically; + for example, as a way to easily copy edits from a web-facing cluster to a "MapReduce" + cluster which will process old and new data and ship back the results + automatically. +

    +

    + The basic architecture pattern used for Apache HBase replication is (HBase cluster) master-push; + it is much easier to keep track of what’s currently being replicated since + each region server has its own write-ahead-log (aka WAL or HLog), just like + other well known solutions like MySQL master/slave replication where + there’s only one bin log to keep track of. One master cluster can + replicate to any number of slave clusters, and each region server will + participate to replicate their own stream of edits. For more information + on the different properties of master/slave replication and other types + of replication, please consult + How Google Serves Data From Multiple Datacenters. +

    +

    + The replication is done asynchronously, meaning that the clusters can + be geographically distant, the links between them can be offline for + some time, and rows inserted on the master cluster won’t be + available at the same time on the slave clusters (eventual consistency). +

    +

    + The replication format used in this design is conceptually the same as + + MySQL’s statement-based replication . Instead of SQL statements, whole + WALEdits (consisting of multiple cell inserts coming from the clients' + Put and Delete) are replicated in order to maintain atomicity. +

    +

    + The HLogs from each region server are the basis of HBase replication, + and must be kept in HDFS as long as they are needed to replicate data + to any slave cluster. Each RS reads from the oldest log it needs to + replicate and keeps the current position inside ZooKeeper to simplify + failure recovery. That position can be different for every slave + cluster, same for the queue of HLogs to process. +

    +

    + The clusters participating in replication can be of asymmetric sizes + and the master cluster will do its “best effort” to balance the stream + of replication on the slave clusters by relying on randomization. +

    +

    + As of version 0.92, Apache HBase supports master/master and cyclic + replication as well as replication to multiple slaves. +

    + +
    +
    +

    + The guide on enabling and using cluster replication is contained + in the API documentation shipped with your Apache HBase distribution. +

    +

    + The most up-to-date documentation is + + available at this address. +

    +
    +
    +

    + The following sections describe the life of a single edit going from a + client that communicates with a master cluster all the way to a single + slave cluster. +

    +
    +

    + The client uses an API that sends a Put, Delete or ICV to a region + server. The key values are transformed into a WALEdit by the region + server and is inspected by the replication code that, for each family + that is scoped for replication, adds the scope to the edit. The edit + is appended to the current WAL and is then applied to its MemStore. +

    +

    + In a separate thread, the edit is read from the log (as part of a batch) + and only the KVs that are replicable are kept (that is, that they are part + of a family scoped GLOBAL in the family's schema, non-catalog so not + .META. or -ROOT-, and did not originate in the target slave cluster - in + case of cyclic replication). +

    +

    + The edit is then tagged with the master's cluster UUID. + When the buffer is filled, or the reader hits the end of the file, + the buffer is sent to a random region server on the slave cluster. +

    +

    + Synchronously, the region server that receives the edits reads them + sequentially and separates each of them into buffers, one per table. + Once all edits are read, each buffer is flushed using the normal HBase + client (HTables managed by a HTablePool). This is done in order to + leverage parallel insertion (MultiPut). + The master's cluster UUID is retained in the edits applied at the + slave cluster in order to allow cyclic replication. +

    +

    + Back in the master cluster's region server, the offset for the current + WAL that's being replicated is registered in ZooKeeper. +

    +
    +
    +

    + The edit is inserted in the same way. +

    +

    + In the separate thread, the region server reads, filters and buffers + the log edits the same way as during normal processing. The slave + region server that's contacted doesn't answer to the RPC, so the master + region server will sleep and retry up to a configured number of times. + If the slave RS still isn't available, the master cluster RS will select a + new subset of RS to replicate to and will retry sending the buffer of + edits. +

    +

    + In the mean time, the WALs will be rolled and stored in a queue in + ZooKeeper. Logs that are archived by their region server (archiving is + basically moving a log from the region server's logs directory to a + central logs archive directory) will update their paths in the in-memory + queue of the replicating thread. +

    +

    + When the slave cluster is finally available, the buffer will be applied + the same way as during normal processing. The master cluster RS will then + replicate the backlog of logs. +

    +
    +
    +
    +

    + This section describes in depth how each of replication's internal + features operate. +

    +
    +

    + HBase replication maintains all of its state in Zookeeper. By default, this state is + contained in the base znode: +

    + 
    +                /hbase/replication
+
    +

    + There are three major child znodes in the base replication znode: +

      +
    • State znode: /hbase/replication/state
      • +
    • Peers znode: /hbase/replication/peers
      • +
    • RS znode: /hbase/replication/rs
      • +
    +

    +
    +

    + The state znode indicates whether or not replication is enabled on the cluster + corresponding to this zookeeper quorum. It does not have any child znodes and simply + contains a boolean value. This value is initialized on startup based on the + hbase.replication config parameter in the hbase-site.xml file. The status + value is read/maintained by the ReplicationZookeeper.ReplicationStatusTracker + class. It is also cached locally using an AtomicBoolean in the ReplicationZookeeper + class. This value can be changed on a live cluster using the stop_replication + command available through the hbase shell. +

    + 
    +                /hbase/replication/state [VALUE: true]
+
    +
    +
    +

    + The peers znode contains a list of all peer replication clusters and the + current replication state of those clusters. It has one child peer znode + for each peer cluster. The peer znode is named with the cluster id provided + by the user in the HBase shell. The value of the peer znode contains + the peers cluster key provided by the user in the HBase Shell. The cluster key + contains a list of zookeeper nodes in the clusters quorum, the client port for the + zookeeper quorum, and the base znode for HBase + (i.e. “zk1.host.com,zk2.host.com,zk3.host.com:2181:/hbase”). +

    + 
    +                /hbase/replication/peers
+                    /1 [Value: zk1.host.com,zk2.host.com,zk3.host.com:2181:/hbase]
+                    /2 [Value: zk5.host.com,zk6.host.com,zk7.host.com:2181:/hbase]
+
    +

    + Each of these peer znodes has a child znode that indicates whether or not + replication is enabled on that peer cluster. These peer-state znodes do not + have child znodes and simply contain a boolean value (i.e. ENABLED or DISABLED). + This value is read/maintained by the ReplicationPeer.PeerStateTracker class. + It is also cached locally using an AtomicBoolean in the ReplicationPeer class. +

    + 
    +                /hbase/replication/peers
+                    /1/peer-state [Value: ENABLED]
+                    /2/peer-state [Value: DISABLED]
+
    +
    +
    +

    + The rs znode contains a list of all outstanding HLog files in the cluster + that need to be replicated. The list is divided into a set of queues organized by + region server and the peer cluster the region server is shipping the HLogs to. The + rs znode has one child znode for each region server in the cluster. The child + znode name is simply the regionserver name (a concatenation of the region server’s + hostname, client port and start code). These region servers could either be dead or alive. +

    + 
    +                /hbase/replication/rs
+                    /hostname.example.org,6020,1234
+                    /hostname2.example.org,6020,2856
+
    +

    + Within each region server znode, the region server maintains a set of HLog replication + queues. Each region server has one queue for every peer cluster it replicates to. + These queues are represented by child znodes named using the cluster id of the peer + cluster they represent (see the peer znode section). +

    + 
    +                /hbase/replication/rs
+                    /hostname.example.org,6020,1234
+                        /1
+                        /2
+
    +

    + Each queue has one child znode for every HLog that still needs to be replicated. + The value of these HLog child znodes is the latest position that has been replicated. + This position is updated every time a HLog entry is replicated. +

    + 
    +                /hbase/replication/rs
+                    /hostname.example.org,6020,1234
+                        /1
+                            23522342.23422 [VALUE: 254]
+                            12340993.22342 [VALUE: 0]
+
    +
    +
    +
    +
    +

    + All of the base znode names are configurable through parameters: +

    +
    + + + + + + + + + + + + + + + + + + + + + + + + +
    ParameterDefault Value
    zookeeper.znode.parent/hbase
    zookeeper.znode.replicationreplication
    zookeeper.znode.replication.peerspeers
    zookeeper.znode.replication.peers.statepeer-state
    zookeeper.znode.replication.rsrs
    +

    + The default replication znode structure looks like the following: +

    + 
    +                /hbase/replication/state
+                /hbase/replication/peers/{peerId}/peer-state
+                /hbase/replication/rs
+
    + +
    +
      +
    • hbase.replication (Default: false) - Controls whether replication is enabled + or disabled for the cluster.
      • +
    • replication.sleep.before.failover (Default: 2000) - The amount of time a failover + worker waits before attempting to replicate a dead region server’s HLog queues.
      • +
    • replication.executor.workers (Default: 1) - The number of dead region servers + one region server should attempt to failover simultaneously.
      • +
    +
    + +
    +

    + When a master cluster RS initiates a replication source to a slave cluster, + it first connects to the slave's ZooKeeper ensemble using the provided + cluster key (that key is composed of the value of hbase.zookeeper.quorum, + zookeeper.znode.parent and hbase.zookeeper.property.clientPort). It + then scans the "rs" directory to discover all the available sinks + (region servers that are accepting incoming streams of edits to replicate) + and will randomly choose a subset of them using a configured + ratio (which has a default value of 10%). For example, if a slave + cluster has 150 machines, 15 will be chosen as potential recipient for + edits that this master cluster RS will be sending. Since this is done by all + master cluster RSs, the probability that all slave RSs are used is very high, + and this method works for clusters of any size. For example, a master cluster + of 10 machines replicating to a slave cluster of 5 machines with a ratio + of 10% means that the master cluster RSs will choose one machine each + at random, thus the chance of overlapping and full usage of the slave + cluster is higher. +

    +
    +
    +

    + Every master cluster RS has its own znode in the replication znodes hierarchy. + It contains one znode per peer cluster (if 5 slave clusters, 5 znodes + are created), and each of these contain a queue + of HLogs to process. Each of these queues will track the HLogs created + by that RS, but they can differ in size. For example, if one slave + cluster becomes unavailable for some time then the HLogs should not be deleted, + thus they need to stay in the queue (while the others are processed). + See the section named "Region server failover" for an example. +

    +

    + When a source is instantiated, it contains the current HLog that the + region server is writing to. During log rolling, the new file is added + to the queue of each slave cluster's znode just before it's made available. + This ensures that all the sources are aware that a new log exists + before HLog is able to append edits into it, but this operations is + now more expensive. + The queue items are discarded when the replication thread cannot read + more entries from a file (because it reached the end of the last block) + and that there are other files in the queue. + This means that if a source is up-to-date and replicates from the log + that the region server writes to, reading up to the "end" of the + current file won't delete the item in the queue. +

    +

    + When a log is archived (because it's not used anymore or because there's + too many of them per hbase.regionserver.maxlogs typically because insertion + rate is faster than region flushing), it will notify the source threads that the path + for that log changed. If the a particular source was already done with + it, it will just ignore the message. If it's in the queue, the path + will be updated in memory. If the log is currently being replicated, + the change will be done atomically so that the reader doesn't try to + open the file when it's already moved. Also, moving a file is a NameNode + operation so, if the reader is currently reading the log, it won't + generate any exception. +

    +
    +
    +

    + By default, a source will try to read from a log file and ship log + entries as fast as possible to a sink. This is first limited by the + filtering of log entries; only KeyValues that are scoped GLOBAL and + that don't belong to catalog tables will be retained. A second limit + is imposed on the total size of the list of edits to replicate per slave, + which by default is 64MB. This means that a master cluster RS with 3 slaves + will use at most 192MB to store data to replicate. This doesn't account + the data filtered that wasn't garbage collected. +

    +

    + Once the maximum size of edits was buffered or the reader hits the end + of the log file, the source thread will stop reading and will choose + at random a sink to replicate to (from the list that was generated by + keeping only a subset of slave RSs). It will directly issue a RPC to + the chosen machine and will wait for the method to return. If it's + successful, the source will determine if the current file is emptied + or if it should continue to read from it. If the former, it will delete + the znode in the queue. If the latter, it will register the new offset + in the log's znode. If the RPC threw an exception, the source will retry + 10 times until trying to find a different sink. +

    +
    +
    +

    + If replication isn't enabled, the master's logs cleaning thread will + delete old logs using a configured TTL. This doesn't work well with + replication since archived logs passed their TTL may still be in a + queue. Thus, the default behavior is augmented so that if a log is + passed its TTL, the cleaning thread will lookup every queue until it + finds the log (while caching the ones it finds). If it's not found, + the log will be deleted. The next time it has to look for a log, + it will first use its cache. +

    +
    +
    +

    + As long as region servers don't fail, keeping track of the logs in ZK + doesn't add any value. Unfortunately, they do fail, so since ZooKeeper + is highly available we can count on it and its semantics to help us + managing the transfer of the queues. +

    +

    + All the master cluster RSs keep a watcher on every other one of them to be + notified when one dies (just like the master does). When it happens, + they all race to create a znode called "lock" inside the dead RS' znode + that contains its queues. The one that creates it successfully will + proceed by transferring all the queues to its own znode (one by one + since ZK doesn't support the rename operation) and will delete all the + old ones when it's done. The recovered queues' znodes will be named + with the id of the slave cluster appended with the name of the dead + server. +

    +

    + Once that is done, the master cluster RS will create one new source thread per + copied queue, and each of them will follow the read/filter/ship pattern. + The main difference is that those queues will never have new data since + they don't belong to their new region server, which means that when + the reader hits the end of the last log, the queue's znode will be + deleted and the master cluster RS will close that replication source. +

    +

    + For example, consider a master cluster with 3 region servers that's + replicating to a single slave with id '2'. The following hierarchy + represents what the znodes layout could be at some point in time. We + can see the RSs' znodes all contain a "peers" znode that contains a + single queue. The znode names in the queues represent the actual file + names on HDFS in the form "address,port.timestamp". +

    + 
    +/hbase/replication/rs/
+                      1.1.1.1,60020,123456780/
+                          2/
+                              1.1.1.1,60020.1234  (Contains a position)
+                              1.1.1.1,60020.1265
+                      1.1.1.2,60020,123456790/
+                          2/
+                              1.1.1.2,60020.1214  (Contains a position)
+                              1.1.1.2,60020.1248
+                              1.1.1.2,60020.1312
+                      1.1.1.3,60020,    123456630/
+                          2/
+                              1.1.1.3,60020.1280  (Contains a position)
+
    +

    + Now let's say that 1.1.1.2 loses its ZK session. The survivors will race + to create a lock, and for some reasons 1.1.1.3 wins. It will then start + transferring all the queues to its local peers znode by appending the + name of the dead server. Right before 1.1.1.3 is able to clean up the + old znodes, the layout will look like the following: +

    + 
    +/hbase/replication/rs/
+                      1.1.1.1,60020,123456780/
+                          2/
+                              1.1.1.1,60020.1234  (Contains a position)
+                              1.1.1.1,60020.1265
+                      1.1.1.2,60020,123456790/
+                          lock
+                          2/
+                              1.1.1.2,60020.1214  (Contains a position)
+                              1.1.1.2,60020.1248
+                              1.1.1.2,60020.1312
+                      1.1.1.3,60020,123456630/
+                          2/
+                              1.1.1.3,60020.1280  (Contains a position)
+
+                          2-1.1.1.2,60020,123456790/
+                              1.1.1.2,60020.1214  (Contains a position)
+                              1.1.1.2,60020.1248
+                              1.1.1.2,60020.1312
+
    +

    + Some time later, but before 1.1.1.3 is able to finish replicating the + last HLog from 1.1.1.2, let's say that it dies too (also some new logs + were created in the normal queues). The last RS will then try to lock + 1.1.1.3's znode and will begin transferring all the queues. The new + layout will be: +

    + 
    +/hbase/replication/rs/
+                      1.1.1.1,60020,123456780/
+                          2/
+                              1.1.1.1,60020.1378  (Contains a position)
+
+                          2-1.1.1.3,60020,123456630/
+                              1.1.1.3,60020.1325  (Contains a position)
+                              1.1.1.3,60020.1401
+
+                          2-1.1.1.2,60020,123456790-1.1.1.3,60020,123456630/
+                              1.1.1.2,60020.1312  (Contains a position)
+                      1.1.1.3,60020,123456630/
+                          lock
+                          2/
+                              1.1.1.3,60020.1325  (Contains a position)
+                              1.1.1.3,60020.1401
+
+                          2-1.1.1.2,60020,123456790/
+                              1.1.1.2,60020.1312  (Contains a position)
+
    +
    + +
    +
    +

    + Yes, this is for much later. +

    +
    +
    +

    + You can use the HBase-provided utility called CopyTable from the package + org.apache.hadoop.hbase.mapreduce in order to have a discp-like tool to + bulk copy data. +

    +
    +
    +

    + Yes, this behavior would help a lot but it's not currently available + in HBase (BatchUpdate had that, but it was lost in the new API). +

    +
    +
    +
    +

    + Here's a list of all the jiras that relate to major issues or missing + features in the replication implementation. +

    +
      +
    1. + HBASE-2611, basically if a region server dies while recovering the + queues of another dead RS, we will miss the data from the queues + that weren't copied. +
      2. +
    +
    + + diff --git a/src/site/xdoc/resources.xml b/src/site/xdoc/resources.xml new file mode 100644 index 0000000..214fc79 --- /dev/null +++ b/src/site/xdoc/resources.xml @@ -0,0 +1,37 @@ + + + + + Other Apache HBase (TM) Resources + + + +
    +
    +
    +

    HBase: The Definitive Guide Random Access to Your Planet-Size Data by Lars George. Publisher: O'Reilly Media, Released: August 2011, Pages: 556.

    +
    +
    +

    HBase In Action By Nick Dimiduk and Amandeep Khurana. Publisher: Manning, MEAP Began: January 2012, Softbound print: Fall 2012, Pages: 350.

    +
    +
    +

    HBase Administration Cookbook by Yifeng Jiang. Publisher: PACKT Publishing, Release: Expected August 2012, Pages: 335.

    +
    +
    +
    + +     diff --git a/src/site/xdoc/sponsors.xml b/src/site/xdoc/sponsors.xml new file mode 100644 index 0000000..639e0c3 --- /dev/null +++ b/src/site/xdoc/sponsors.xml @@ -0,0 +1,44 @@ + + + + + Apache HBase™ Sponsors + + + +
    +

    First off, thanks to all who sponsor + our parent, the Apache Software Foundation. +

    +

    The below companies have been gracious enough to provide their commerical tool offerings free of charge to the Apache HBase™ project. +

    +

    +
    +
    +

    To contribute to the Apache Software Foundation, a good idea in our opinion, see the ASF Sponsorship page. +

    +
    + +     diff --git a/src/test/data/hbase-4388-root.dir.tgz b/src/test/data/hbase-4388-root.dir.tgz new file mode 100644 index 0000000..da2244e Binary files /dev/null and b/src/test/data/hbase-4388-root.dir.tgz differ diff --git a/src/test/java/org/apache/hadoop/hbase/ClassFinder.java b/src/test/java/org/apache/hadoop/hbase/ClassFinder.java new file mode 100644 index 0000000..2461534 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/ClassFinder.java @@ -0,0 +1,232 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.IOException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.net.URL; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.jar.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * A class that finds a set of classes that are locally accessible + * (from .class or .jar files), and satisfy the conditions that are + * imposed by name and class filters provided by the user. + */ +public class ClassFinder { + private static final Log LOG = LogFactory.getLog(ClassFinder.class); + private static String CLASS_EXT = ".class"; + + private FileNameFilter fileNameFilter; + private ClassFilter classFilter; + private FileFilter fileFilter; + + public static interface FileNameFilter { + public boolean isCandidateFile(String fileName, String absFilePath); + }; + + public static interface ClassFilter { + public boolean isCandidateClass(Class c); + }; + + public ClassFinder(FileNameFilter fileNameFilter, ClassFilter classFilter) { + this.classFilter = classFilter; + this.fileNameFilter = fileNameFilter; + this.fileFilter = new FileFilterWithName(fileNameFilter); + } + + /** + * Finds the classes in current package (of ClassFinder) and nested packages. + * @param proceedOnExceptions whether to ignore exceptions encountered for + * individual jars/files/classes, and proceed looking for others. + */ + public Set> findClasses(boolean proceedOnExceptions) + throws ClassNotFoundException, IOException, LinkageError { + return findClasses(this.getClass().getPackage().getName(), proceedOnExceptions); + } + + /** + * Finds the classes in a package and nested packages. + * @param packageName package names + * @param proceedOnExceptions whether to ignore exceptions encountered for + * individual jars/files/classes, and proceed looking for others. + */ + public Set> findClasses(String packageName, boolean proceedOnExceptions) + throws ClassNotFoundException, IOException, LinkageError { + final String path = packageName.replace('.', '/'); + final Pattern jarResourceRe = Pattern.compile("^file:(.+\\.jar)!/" + path + "$"); + + Enumeration resources = ClassLoader.getSystemClassLoader().getResources(path); + List dirs = new ArrayList(); + List jars = new ArrayList(); + + while (resources.hasMoreElements()) { + URL resource = resources.nextElement(); + String resourcePath = resource.getFile(); + Matcher matcher = jarResourceRe.matcher(resourcePath); + if (matcher.find()) { + jars.add(matcher.group(1)); + } else { + dirs.add(new File(resource.getFile())); + } + } + + Set> classes = new HashSet>(); + for (File directory : dirs) { + classes.addAll(findClassesFromFiles(directory, packageName, proceedOnExceptions)); + } + for (String jarFileName : jars) { + classes.addAll(findClassesFromJar(jarFileName, packageName, proceedOnExceptions)); + } + return classes; + } + + private Set> findClassesFromJar(String jarFileName, + String packageName, boolean proceedOnExceptions) + throws IOException, ClassNotFoundException, LinkageError { + JarInputStream jarFile = null; + try { + jarFile = new JarInputStream(new FileInputStream(jarFileName)); + } catch (IOException ioEx) { + if (!proceedOnExceptions) { + throw ioEx; + } + LOG.error("Failed to look for classes in " + jarFileName + ": " + ioEx); + } + + Set> classes = new HashSet>(); + JarEntry entry = null; + while (true) { + try { + entry = jarFile.getNextJarEntry(); + } catch (IOException ioEx) { + if (!proceedOnExceptions) { + throw ioEx; + } + LOG.error("Failed to get next entry from " + jarFileName + ": " + ioEx); + break; + } + if (entry == null) { + break; // loop termination condition + } + + String className = entry.getName(); + if (!className.endsWith(CLASS_EXT)) { + continue; + } + int ix = className.lastIndexOf('/'); + String fileName = (ix >= 0) ? className.substring(ix + 1) : className; + if (!this.fileNameFilter.isCandidateFile(fileName, className)) { + continue; + } + className = className + .substring(0, className.length() - CLASS_EXT.length()).replace('/', '.'); + if (!className.startsWith(packageName)) { + continue; + } + Class c = makeClass(className, proceedOnExceptions); + if (c != null) { + if (!classes.add(c)) { + LOG.error("Ignoring duplicate class " + className); + } + } + } + return classes; + } + + private Set> findClassesFromFiles(File baseDirectory, String packageName, + boolean proceedOnExceptions) throws ClassNotFoundException, LinkageError { + Set> classes = new HashSet>(); + if (!baseDirectory.exists()) { + LOG.error("Failed to find " + baseDirectory.getAbsolutePath()); + return classes; + } + + File[] files = baseDirectory.listFiles(this.fileFilter); + if (files == null) { + LOG.error("Failed to get files from " + baseDirectory.getAbsolutePath()); + return classes; + } + + for (File file : files) { + final String fileName = file.getName(); + if (file.isDirectory()) { + classes.addAll(findClassesFromFiles(file, packageName + "." + fileName, + proceedOnExceptions)); + } else { + String className = packageName + '.' + + fileName.substring(0, fileName.length() - CLASS_EXT.length()); + Class c = makeClass(className, proceedOnExceptions); + if (c != null) { + if (!classes.add(c)) { + LOG.error("Ignoring duplicate class " + className); + } + } + } + } + return classes; + } + + private Class makeClass(String className, boolean proceedOnExceptions) + throws ClassNotFoundException, LinkageError { + try { + Class c = Class.forName(className, false, this.getClass().getClassLoader()); + return classFilter.isCandidateClass(c) ? c : null; + } catch (ClassNotFoundException classNotFoundEx) { + if (!proceedOnExceptions) { + throw classNotFoundEx; + } + LOG.error("Failed to instantiate or check " + className + ": " + classNotFoundEx); + } catch (LinkageError linkageEx) { + if (!proceedOnExceptions) { + throw linkageEx; + } + LOG.error("Failed to instantiate or check " + className + ": " + linkageEx); + } + return null; + } + + private class FileFilterWithName implements FileFilter { + private FileNameFilter nameFilter; + + public FileFilterWithName(FileNameFilter nameFilter) { + this.nameFilter = nameFilter; + } + + @Override + public boolean accept(File file) { + return file.isDirectory() + || (file.getName().endsWith(CLASS_EXT) + && nameFilter.isCandidateFile(file.getName(), file.getAbsolutePath())); + } + }; +}; diff --git a/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java b/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java new file mode 100644 index 0000000..f40aa79 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.regex.Pattern; + +import org.apache.hadoop.hbase.ClassFinder.ClassFilter; +import org.apache.hadoop.hbase.ClassFinder.FileNameFilter; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runners.Suite; + +/** + * ClassFinder that is pre-configured with filters that will only allow test classes. + * The name is strange because a logical name would start with "Test" and be confusing. + */ +public class ClassTestFinder extends ClassFinder { + + public ClassTestFinder() { + super(new TestFileNameFilter(), new TestClassFilter()); + } + + public ClassTestFinder(Class category) { + super(new TestFileNameFilter(), new TestClassFilter(category)); + } + + public static Class[] getCategoryAnnotations(Class c) { + Category category = c.getAnnotation(Category.class); + if (category != null) { + return category.value(); + } + return new Class[0]; + } + + public static class TestFileNameFilter implements FileNameFilter { + private static final Pattern hadoopCompactRe = + Pattern.compile("hbase-hadoop\\d?-compat"); + + @Override + public boolean isCandidateFile(String fileName, String absFilePath) { + boolean isTestFile = fileName.startsWith("Test") + || fileName.startsWith("IntegrationTest"); + return isTestFile && !hadoopCompactRe.matcher(absFilePath).find(); + } + }; + + /* + * A class is considered as a test class if: + * - it's not Abstract AND + * - one or more of its methods is annotated with org.junit.Test OR + * - the class is annotated with Suite.SuiteClasses + * */ + public static class TestClassFilter implements ClassFilter { + private Class categoryAnnotation = null; + public TestClassFilter(Class categoryAnnotation) { + this.categoryAnnotation = categoryAnnotation; + } + + public TestClassFilter() { + this(null); + } + + @Override + public boolean isCandidateClass(Class c) { + return isTestClass(c) && isCategorizedClass(c); + } + + private boolean isTestClass(Class c) { + if (Modifier.isAbstract(c.getModifiers())) { + return false; + } + + if (c.getAnnotation(Suite.SuiteClasses.class) != null) { + return true; + } + + for (Method met : c.getMethods()) { + if (met.getAnnotation(Test.class) != null) { + return true; + } + } + + return false; + } + + private boolean isCategorizedClass(Class c) { + if (this.categoryAnnotation == null) { + return true; + } + for (Class cc : getCategoryAnnotations(c)) { + if (cc.equals(this.categoryAnnotation)) { + return true; + } + } + return false; + } + }; +}; diff --git a/src/test/java/org/apache/hadoop/hbase/ClusterManager.java b/src/test/java/org/apache/hadoop/hbase/ClusterManager.java new file mode 100644 index 0000000..5a5b010 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/ClusterManager.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configured; + + +/** + * ClusterManager is an api to manage servers in a distributed environment. It provides services + * for starting / stopping / killing Hadoop/HBase daemons. Concrete implementations provide actual + * functionality for carrying out deployment-specific tasks. + */ +@InterfaceAudience.Private +public abstract class ClusterManager extends Configured { + protected static final Log LOG = LogFactory.getLog(ClusterManager.class); + + private static final String SIGKILL = "SIGKILL"; + private static final String SIGSTOP = "SIGSTOP"; + private static final String SIGCONT = "SIGCONT"; + + public ClusterManager() { + } + + /** + * Type of the service daemon + */ + public static enum ServiceType { + HADOOP_NAMENODE("namenode"), + HADOOP_DATANODE("datanode"), + HADOOP_JOBTRACKER("jobtracker"), + HADOOP_TASKTRACKER("tasktracker"), + HBASE_MASTER("master"), + HBASE_REGIONSERVER("regionserver"); + + private String name; + + ServiceType(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return getName(); + } + } + + /** + * Start the service on the given host + */ + public abstract void start(ServiceType service, String hostname) throws IOException; + + /** + * Stop the service on the given host + */ + public abstract void stop(ServiceType service, String hostname) throws IOException; + + /** + * Restart the service on the given host + */ + public abstract void restart(ServiceType service, String hostname) throws IOException; + + /** + * Send the given posix signal to the service + */ + public abstract void signal(ServiceType service, String signal, + String hostname) throws IOException; + + /** + * Kill the service running on given host + */ + public void kill(ServiceType service, String hostname) throws IOException { + signal(service, SIGKILL, hostname); + } + + /** + * Suspend the service running on given host + */ + public void suspend(ServiceType service, String hostname) throws IOException { + signal(service, SIGSTOP, hostname); + } + + /** + * Resume the services running on given hosts + */ + public void resume(ServiceType service, String hostname) throws IOException { + signal(service, SIGCONT, hostname); + } + + /** + * Returns whether the service is running on the remote host. This only checks whether the + * service still has a pid. + */ + public abstract boolean isRunning(ServiceType service, String hostname) throws IOException; + + /* TODO: further API ideas: + * + * //return services running on host: + * ServiceType[] getRunningServicesOnHost(String hostname); + * + * //return which services can be run on host (for example, to query whether hmaster can run on this host) + * ServiceType[] getRunnableServicesOnHost(String hostname); + * + * //return which hosts can run this service + * String[] getRunnableHostsForService(ServiceType service); + */ + +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java new file mode 100644 index 0000000..53d4ce3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java @@ -0,0 +1,270 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.util.HashMap; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClusterManager.ServiceType; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.ipc.HMasterInterface; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.util.Threads; + +import com.google.common.collect.Sets; + +/** + * Manages the interactions with an already deployed distributed cluster (as opposed to + * a pseudo-distributed, or mini/local cluster). This is used by integration and system tests. + */ +@InterfaceAudience.Private +public class DistributedHBaseCluster extends HBaseCluster { + + private HBaseAdmin admin; + + private ClusterManager clusterManager; + + public DistributedHBaseCluster(Configuration conf, ClusterManager clusterManager) + throws IOException { + super(conf); + this.clusterManager = clusterManager; + this.admin = new HBaseAdmin(conf); + this.initialClusterStatus = getClusterStatus(); + } + + public void setClusterManager(ClusterManager clusterManager) { + this.clusterManager = clusterManager; + } + + public ClusterManager getClusterManager() { + return clusterManager; + } + + /** + * Returns a ClusterStatus for this HBase cluster + * @throws IOException + */ + @Override + public ClusterStatus getClusterStatus() throws IOException { + return admin.getClusterStatus(); + } + + @Override + public ClusterStatus getInitialClusterStatus() throws IOException { + return initialClusterStatus; + } + + @Override + public void close() throws IOException { + if (this.admin != null) { + admin.close(); + } + } + + @Override + public void startRegionServer(String hostname) throws IOException { + LOG.info("Starting RS on: " + hostname); + clusterManager.start(ServiceType.HBASE_REGIONSERVER, hostname); + } + + @Override + public void killRegionServer(ServerName serverName) throws IOException { + LOG.info("Aborting RS: " + serverName.getServerName()); + clusterManager.kill(ServiceType.HBASE_REGIONSERVER, serverName.getHostname()); + } + + @Override + public void stopRegionServer(ServerName serverName) throws IOException { + LOG.info("Stopping RS: " + serverName.getServerName()); + clusterManager.stop(ServiceType.HBASE_REGIONSERVER, serverName.getHostname()); + } + + @Override + public void waitForRegionServerToStop(ServerName serverName, long timeout) throws IOException { + waitForServiceToStop(ServiceType.HBASE_REGIONSERVER, serverName, timeout); + } + + private void waitForServiceToStop(ServiceType service, ServerName serverName, long timeout) + throws IOException { + LOG.info("Waiting service:" + service + " to stop: " + serverName.getServerName()); + long start = System.currentTimeMillis(); + + while ((System.currentTimeMillis() - start) < timeout) { + if (!clusterManager.isRunning(service, serverName.getHostname())) { + return; + } + Threads.sleep(1000); + } + throw new IOException("did timeout waiting for service to stop:" + serverName); + } + + @Override + public HMasterInterface getMasterAdmin() throws IOException { + HConnection conn = HConnectionManager.getConnection(conf); + return conn.getMaster(); + } + + @Override + public void startMaster(String hostname) throws IOException { + LOG.info("Starting Master on: " + hostname); + clusterManager.start(ServiceType.HBASE_MASTER, hostname); + } + + @Override + public void killMaster(ServerName serverName) throws IOException { + LOG.info("Aborting Master: " + serverName.getServerName()); + clusterManager.kill(ServiceType.HBASE_MASTER, serverName.getHostname()); + } + + @Override + public void stopMaster(ServerName serverName) throws IOException { + LOG.info("Stopping Master: " + serverName.getServerName()); + clusterManager.stop(ServiceType.HBASE_MASTER, serverName.getHostname()); + } + + @Override + public void waitForMasterToStop(ServerName serverName, long timeout) throws IOException { + waitForServiceToStop(ServiceType.HBASE_MASTER, serverName, timeout); + } + + @Override + public boolean waitForActiveAndReadyMaster(long timeout) throws IOException { + long start = System.currentTimeMillis(); + while (System.currentTimeMillis() - start < timeout) { + try { + getMasterAdmin(); + return true; + } catch (MasterNotRunningException m) { + LOG.warn("Master not started yet " + m); + } catch (ZooKeeperConnectionException e) { + LOG.warn("Failed to connect to ZK " + e); + } + Threads.sleep(1000); + } + return false; + } + + @Override + public ServerName getServerHoldingRegion(byte[] regionName) throws IOException { + HConnection connection = admin.getConnection(); + HRegionLocation regionLoc = connection.locateRegion(regionName); + if (regionLoc == null) { + return null; + } + + org.apache.hadoop.hbase.HServerInfo sn + = connection.getHRegionConnection(regionLoc.getHostname(), regionLoc.getPort()).getHServerInfo(); + + return new ServerName(sn.getServerAddress().getHostname(), sn.getServerAddress().getPort(), sn.getStartCode()); + } + + @Override + public void waitUntilShutDown() { + //Simply wait for a few seconds for now (after issuing serverManager.kill + throw new RuntimeException("Not implemented yet"); + } + + @Override + public void shutdown() throws IOException { + //not sure we want this + throw new RuntimeException("Not implemented yet"); + } + + @Override + public boolean isDistributedCluster() { + return true; + } + + @Override + public void restoreClusterStatus(ClusterStatus initial) throws IOException { + //TODO: caution: not tested throughly + ClusterStatus current = getClusterStatus(); + + //restore masters + + //check whether current master has changed + if (!ServerName.isSameHostnameAndPort(initial.getMaster(), current.getMaster())) { + //master has changed, we would like to undo this. + //1. Kill the current backups + //2. Stop current master + //3. Start a master at the initial hostname (if not already running as backup) + //4. Start backup masters + boolean foundOldMaster = false; + for (ServerName currentBackup : current.getBackupMasters()) { + if (!ServerName.isSameHostnameAndPort(currentBackup, initial.getMaster())) { + stopMaster(currentBackup); + } else { + foundOldMaster = true; + } + } + stopMaster(current.getMaster()); + if (foundOldMaster) { //if initial master is not running as a backup + startMaster(initial.getMaster().getHostname()); + } + waitForActiveAndReadyMaster(); //wait so that active master takes over + + //start backup masters + for (ServerName backup : initial.getBackupMasters()) { + //these are not started in backup mode, but we should already have an active master + startMaster(backup.getHostname()); + } + } else { + //current master has not changed, match up backup masters + HashMap initialBackups = new HashMap(); + HashMap currentBackups = new HashMap(); + + for (ServerName server : initial.getBackupMasters()) { + initialBackups.put(server.getHostname(), server); + } + for (ServerName server : current.getBackupMasters()) { + currentBackups.put(server.getHostname(), server); + } + + for (String hostname : Sets.difference(initialBackups.keySet(), currentBackups.keySet())) { + startMaster(hostname); + } + + for (String hostname : Sets.difference(currentBackups.keySet(), initialBackups.keySet())) { + stopMaster(currentBackups.get(hostname)); + } + } + + //restore region servers + HashMap initialServers = new HashMap(); + HashMap currentServers = new HashMap(); + + for (ServerName server : initial.getServers()) { + initialServers.put(server.getHostname(), server); + } + for (ServerName server : current.getServers()) { + currentServers.put(server.getHostname(), server); + } + + for (String hostname : Sets.difference(initialServers.keySet(), currentServers.keySet())) { + startRegionServer(hostname); + } + + for (String hostname : Sets.difference(currentServers.keySet(), initialServers.keySet())) { + stopRegionServer(currentServers.get(hostname)); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java new file mode 100644 index 0000000..4a5e394 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java @@ -0,0 +1,264 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.Closeable; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.ipc.HMasterInterface; +import org.apache.hadoop.hbase.ipc.HRegionInterface; + +/** + * This class defines methods that can help with managing HBase clusters + * from unit tests and system tests. There are 3 types of cluster deployments: + *
      + *
    • MiniHBaseCluster: each server is run in the same JVM in separate threads, + * used by unit tests
      • + *
    • DistributedHBaseCluster: the cluster is pre-deployed, system and integration tests can + * interact with the cluster.
      • + *
    • ProcessBasedLocalHBaseCluster: each server is deployed locally but in separate + * JVMs.
      • + *
    + *

    + * HBaseCluster unifies the way tests interact with the cluster, so that the same test can + * be run against a mini-cluster during unit test execution, or a distributed cluster having + * tens/hundreds of nodes during execution of integration tests. + * + *

    + * HBaseCluster exposes client-side public interfaces to tests, so that tests does not assume + * running in a particular mode. Not all the tests are suitable to be run on an actual cluster, + * and some tests will still need to mock stuff and introspect internal state. For those use + * cases from unit tests, or if more control is needed, you can use the subclasses directly. + * In that sense, this class does not abstract away every interface that + * MiniHBaseCluster or DistributedHBaseCluster provide. + */ +@InterfaceAudience.Private +public abstract class HBaseCluster implements Closeable, Configurable { + static final Log LOG = LogFactory.getLog(HBaseCluster.class.getName()); + protected Configuration conf; + + /** the status of the cluster before we begin */ + protected ClusterStatus initialClusterStatus; + + /** + * Construct an HBaseCluster + * @param conf Configuration to be used for cluster + */ + public HBaseCluster(Configuration conf) { + setConf(conf); + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return conf; + } + + /** + * Returns a ClusterStatus for this HBase cluster. + * @see #getInitialClusterStatus() + */ + public abstract ClusterStatus getClusterStatus() throws IOException; + + /** + * Returns a ClusterStatus for this HBase cluster as observed at the + * starting of the HBaseCluster + */ + public ClusterStatus getInitialClusterStatus() throws IOException { + return initialClusterStatus; + } + + /** + * Returns an {@link HmasterInterface} to the active master + */ + public abstract HMasterInterface getMasterAdmin() + throws IOException; + + /** + * Starts a new region server on the given hostname or if this is a mini/local cluster, + * starts a region server locally. + * @param hostname the hostname to start the regionserver on + * @throws IOException if something goes wrong + */ + public abstract void startRegionServer(String hostname) throws IOException; + + /** + * Kills the region server process if this is a distributed cluster, otherwise + * this causes the region server to exit doing basic clean up only. + * @throws IOException if something goes wrong + */ + public abstract void killRegionServer(ServerName serverName) throws IOException; + + /** + * Stops the given region server, by attempting a gradual stop. + * @return whether the operation finished with success + * @throws IOException if something goes wrong + */ + public abstract void stopRegionServer(ServerName serverName) throws IOException; + + /** + * Wait for the specified region server to join the cluster + * @return whether the operation finished with success + * @throws IOException if something goes wrong or timeout occurs + */ + public void waitForRegionServerToStart(String hostname, long timeout) + throws IOException { + long start = System.currentTimeMillis(); + while ((System.currentTimeMillis() - start) < timeout) { + for (ServerName server : getClusterStatus().getServers()) { + if (server.getHostname().equals(hostname)) { + return; + } + } + Threads.sleep(100); + } + throw new IOException("did timeout waiting for region server to start:" + hostname); + } + + /** + * Wait for the specified region server to stop the thread / process. + * @return whether the operation finished with success + * @throws IOException if something goes wrong or timeout occurs + */ + public abstract void waitForRegionServerToStop(ServerName serverName, long timeout) + throws IOException; + + /** + * Starts a new master on the given hostname or if this is a mini/local cluster, + * starts a master locally. + * @param hostname the hostname to start the master on + * @return whether the operation finished with success + * @throws IOException if something goes wrong + */ + public abstract void startMaster(String hostname) throws IOException; + + /** + * Kills the master process if this is a distributed cluster, otherwise, + * this causes master to exit doing basic clean up only. + * @throws IOException if something goes wrong + */ + public abstract void killMaster(ServerName serverName) throws IOException; + + /** + * Stops the given master, by attempting a gradual stop. + * @throws IOException if something goes wrong + */ + public abstract void stopMaster(ServerName serverName) throws IOException; + + /** + * Wait for the specified master to stop the thread / process. + * @throws IOException if something goes wrong or timeout occurs + */ + public abstract void waitForMasterToStop(ServerName serverName, long timeout) + throws IOException; + + /** + * Blocks until there is an active master and that master has completed + * initialization. + * + * @return true if an active master becomes available. false if there are no + * masters left. + * @throws IOException if something goes wrong or timeout occurs + */ + public boolean waitForActiveAndReadyMaster() + throws IOException { + return waitForActiveAndReadyMaster(Long.MAX_VALUE); + } + + /** + * Blocks until there is an active master and that master has completed + * initialization. + * @param timeout the timeout limit in ms + * @return true if an active master becomes available. false if there are no + * masters left. + */ + public abstract boolean waitForActiveAndReadyMaster(long timeout) + throws IOException; + + /** + * Wait for HBase Cluster to shut down. + */ + public abstract void waitUntilShutDown() throws IOException; + + /** + * Shut down the HBase cluster + */ + public abstract void shutdown() throws IOException; + + /** + * Restores the cluster to it's initial state if this is a real cluster, + * otherwise does nothing. + */ + public void restoreInitialStatus() throws IOException { + restoreClusterStatus(getInitialClusterStatus()); + } + + /** + * Restores the cluster to given state if this is a real cluster, + * otherwise does nothing. + */ + public void restoreClusterStatus(ClusterStatus desiredStatus) throws IOException { + } + + /** + * Get the ServerName of region server serving ROOT region + */ + public ServerName getServerHoldingRoot() throws IOException { + return getServerHoldingRegion(HRegionInfo.ROOT_REGIONINFO.getRegionName()); + } + + /** + * Get the ServerName of region server serving the first META region + */ + public ServerName getServerHoldingMeta() throws IOException { + return getServerHoldingRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); + } + + /** + * Get the ServerName of region server serving the specified region + * @param regionName Name of the region in bytes + * @return ServerName that hosts the region or null + */ + public abstract ServerName getServerHoldingRegion(byte[] regionName) throws IOException; + + /** + * @return whether we are interacting with a distributed cluster as opposed to an + * in-process mini/local cluster. + */ + public boolean isDistributedCluster() { + return false; + } + + /** + * Closes all the resources held open for this cluster. Note that this call does not shutdown + * the cluster. + * @see #shutdown() + */ + @Override + public abstract void close() throws IOException; +} diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java b/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java new file mode 100644 index 0000000..bc3b3fd --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java @@ -0,0 +1,222 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.File; +import java.io.IOException; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClusterManager.CommandProvider.Operation; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.util.Shell; + +/** + * A default cluster manager for HBase. Uses SSH, and hbase shell scripts + * to manage the cluster. Assumes Unix-like commands are available like 'ps', + * 'kill', etc. Also assumes the user running the test has enough "power" to start & stop + * servers on the remote machines (for example, the test user could be the same user as the + * user the daemon isrunning as) + */ +@InterfaceAudience.Private +public class HBaseClusterManager extends ClusterManager { + private String sshUserName; + private String sshOptions; + + /** + * The command format that is used to execute the remote command. Arguments: + * 1 SSH options, 2 user name , 3 "@" if username is set, 4 host, 5 original command. + */ + private static final String DEFAULT_TUNNEL_CMD = "/usr/bin/ssh %1$s %2$s%3$s%4$s \"%5$s\""; + private String tunnelCmd; + + @Override + public void setConf(Configuration conf) { + super.setConf(conf); + if (conf == null) { + // Configured gets passed null before real conf. Why? I don't know. + return; + } + sshUserName = conf.get("hbase.it.clustermanager.ssh.user", ""); + String extraSshOptions = conf.get("hbase.it.clustermanager.ssh.opts", ""); + sshOptions = System.getenv("HBASE_SSH_OPTS"); + if (!extraSshOptions.isEmpty()) { + sshOptions = StringUtils.join(new Object[] { sshOptions, extraSshOptions }, " "); + } + sshOptions = (sshOptions == null) ? "" : sshOptions; + tunnelCmd = conf.get("hbase.it.clustermanager.ssh.cmd", DEFAULT_TUNNEL_CMD); + LOG.info("Running with SSH user [" + sshUserName + "] and options [" + sshOptions + "]"); + } + + /** + * Executes commands over SSH + */ + protected class RemoteShell extends Shell.ShellCommandExecutor { + private String hostname; + + public RemoteShell(String hostname, String[] execString, File dir, Map env, + long timeout) { + super(execString, dir, env, timeout); + this.hostname = hostname; + } + + public RemoteShell(String hostname, String[] execString, File dir, Map env) { + super(execString, dir, env); + this.hostname = hostname; + } + + public RemoteShell(String hostname, String[] execString, File dir) { + super(execString, dir); + this.hostname = hostname; + } + + public RemoteShell(String hostname, String[] execString) { + super(execString); + this.hostname = hostname; + } + + public String[] getExecString() { + String at = sshUserName.isEmpty() ? "" : "@"; + String remoteCmd = StringUtils.join(super.getExecString(), " "); + String cmd = String.format(tunnelCmd, sshOptions, sshUserName, at, hostname, remoteCmd); + LOG.info("Executing full command [" + cmd + "]"); + return new String[] { "/usr/bin/env", "bash", "-c", cmd }; + } + + @Override + public void execute() throws IOException { + super.execute(); + } + } + + /** + * Provides command strings for services to be executed by Shell. CommandProviders are + * pluggable, and different deployments(windows, bigtop, etc) can be managed by + * plugging-in custom CommandProvider's or ClusterManager's. + */ + static abstract class CommandProvider { + + enum Operation { + START, STOP, RESTART + } + + public abstract String getCommand(ServiceType service, Operation op); + + public String isRunningCommand(ServiceType service) { + return findPidCommand(service); + } + + protected String findPidCommand(ServiceType service) { + String servicePathFilter = ""; + if (service == ServiceType.HBASE_MASTER || service == ServiceType.HBASE_REGIONSERVER) { + servicePathFilter = " | grep hbase"; + } + return String.format("ps ux | grep %s %s | grep -v grep | tr -s ' ' | cut -d ' ' -f2", + service, servicePathFilter); + } + + public String signalCommand(ServiceType service, String signal) { + return String.format("%s | xargs kill -s %s", findPidCommand(service), signal); + } + } + + /** + * CommandProvider to manage the service using bin/hbase-* scripts + */ + static class HBaseShellCommandProvider extends CommandProvider { + private String getHBaseHome() { + return System.getenv("HBASE_HOME"); + } + + private String getConfig() { + String confDir = System.getenv("HBASE_CONF_DIR"); + if (confDir != null) { + return String.format("--config %s", confDir); + } + return ""; + } + + @Override + public String getCommand(ServiceType service, Operation op) { + return String.format("%s/bin/hbase-daemon.sh %s %s %s", getHBaseHome(), getConfig(), + op.toString().toLowerCase(), service); + } + } + + public HBaseClusterManager() { + super(); + } + + protected CommandProvider getCommandProvider(ServiceType service) { + //TODO: make it pluggable, or auto-detect the best command provider, should work with + //hadoop daemons as well + return new HBaseShellCommandProvider(); + } + + /** + * Execute the given command on the host using SSH + * @return pair of exit code and command output + * @throws IOException if something goes wrong. + */ + private Pair exec(String hostname, String... cmd) throws IOException { + LOG.info("Executing remote command: " + StringUtils.join(cmd, " ") + " , hostname:" + hostname); + + RemoteShell shell = new RemoteShell(hostname, cmd); + shell.execute(); + + LOG.info("Executed remote command, exit code:" + shell.getExitCode() + + " , output:" + shell.getOutput()); + + return new Pair(shell.getExitCode(), shell.getOutput()); + } + + private void exec(String hostname, ServiceType service, Operation op) throws IOException { + exec(hostname, getCommandProvider(service).getCommand(service, op)); + } + + @Override + public void start(ServiceType service, String hostname) throws IOException { + exec(hostname, service, Operation.START); + } + + @Override + public void stop(ServiceType service, String hostname) throws IOException { + exec(hostname, service, Operation.STOP); + } + + @Override + public void restart(ServiceType service, String hostname) throws IOException { + exec(hostname, service, Operation.RESTART); + } + + @Override + public void signal(ServiceType service, String signal, String hostname) throws IOException { + exec(hostname, getCommandProvider(service).signalCommand(service, signal)); + } + + @Override + public boolean isRunning(ServiceType service, String hostname) throws IOException { + String ret = exec(hostname, getCommandProvider(service).isRunningCommand(service)) + .getSecond(); + return ret.length() > 0; + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java new file mode 100644 index 0000000..cbbf737 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java @@ -0,0 +1,747 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.File; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.Iterator; +import java.util.List; +import java.util.NavigableMap; + +import junit.framework.AssertionFailedError; +import junit.framework.TestCase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.util.*; +import org.apache.hadoop.hdfs.MiniDFSCluster; + +/** + * Abstract HBase test class. Initializes a few things that can come in handly + * like an HBaseConfiguration and filesystem. + * @deprecated Write junit4 unit tests using {@link HBaseTestingUtility} + */ +public abstract class HBaseTestCase extends TestCase { + private static final Log LOG = LogFactory.getLog(HBaseTestCase.class); + + /** configuration parameter name for test directory + * @deprecated see HBaseTestingUtility#TEST_DIRECTORY_KEY + **/ + private static final String TEST_DIRECTORY_KEY = "test.build.data"; + +/* + protected final static byte [] fam1 = Bytes.toBytes("colfamily1"); + protected final static byte [] fam2 = Bytes.toBytes("colfamily2"); + protected final static byte [] fam3 = Bytes.toBytes("colfamily3"); +*/ + protected final static byte [] fam1 = Bytes.toBytes("colfamily11"); + protected final static byte [] fam2 = Bytes.toBytes("colfamily21"); + protected final static byte [] fam3 = Bytes.toBytes("colfamily31"); + + protected static final byte [][] COLUMNS = {fam1, fam2, fam3}; + + private boolean localfs = false; + protected static Path testDir = null; + protected FileSystem fs = null; + protected HRegion root = null; + protected HRegion meta = null; + protected static final char FIRST_CHAR = 'a'; + protected static final char LAST_CHAR = 'z'; + protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|"; + protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR}; + protected String START_KEY; + protected static final int MAXVERSIONS = 3; + + static { + initialize(); + } + + public volatile Configuration conf; + + /** constructor */ + public HBaseTestCase() { + super(); + init(); + } + + /** + * @param name + */ + public HBaseTestCase(String name) { + super(name); + init(); + } + + private void init() { + conf = HBaseConfiguration.create(); + try { + START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_ENCODING); + } catch (UnsupportedEncodingException e) { + LOG.fatal("error during initialization", e); + fail(); + } + } + + /** + * Note that this method must be called after the mini hdfs cluster has + * started or we end up with a local file system. + */ + @Override + protected void setUp() throws Exception { + super.setUp(); + localfs = + (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0); + + if (fs == null) { + this.fs = FileSystem.get(conf); + } + try { + if (localfs) { + this.testDir = getUnitTestdir(getName()); + if (fs.exists(testDir)) { + fs.delete(testDir, true); + } + } else { + this.testDir = + this.fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR))); + } + } catch (Exception e) { + LOG.fatal("error during setup", e); + throw e; + } + } + + @Override + protected void tearDown() throws Exception { + try { + if (localfs) { + if (this.fs.exists(testDir)) { + this.fs.delete(testDir, true); + } + } + } catch (Exception e) { + LOG.fatal("error during tear down", e); + } + super.tearDown(); + } + + /** + * @see HBaseTestingUtility#getBaseTestDir + * @param testName + * @return directory to use for this test + */ + protected Path getUnitTestdir(String testName) { + return new Path( + System.getProperty( + HBaseTestingUtility.BASE_TEST_DIRECTORY_KEY, + HBaseTestingUtility.DEFAULT_BASE_TEST_DIRECTORY + ), + testName + ); + } + + /** + * You must call close on the returned region and then close on the log file + * it created. Do {@link HRegion#close()} followed by {@link HRegion#getLog()} + * and on it call close. + * @param desc + * @param startKey + * @param endKey + * @return An {@link HRegion} + * @throws IOException + */ + public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey, + byte [] endKey) + throws IOException { + return createNewHRegion(desc, startKey, endKey, this.conf); + } + + public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey, + byte [] endKey, Configuration conf) + throws IOException { + FileSystem filesystem = FileSystem.get(conf); + HRegionInfo hri = new HRegionInfo(desc.getName(), startKey, endKey); + return HRegion.createHRegion(hri, testDir, conf, desc); + } + + protected HRegion openClosedRegion(final HRegion closedRegion) + throws IOException { + HRegion r = new HRegion(closedRegion); + r.initialize(); + return r; + } + + /** + * Create a table of name name with {@link COLUMNS} for + * families. + * @param name Name to give table. + * @return Column descriptor. + */ + protected HTableDescriptor createTableDescriptor(final String name) { + return createTableDescriptor(name, MAXVERSIONS); + } + + /** + * Create a table of name name with {@link COLUMNS} for + * families. + * @param name Name to give table. + * @param versions How many versions to allow per column. + * @return Column descriptor. + */ + protected HTableDescriptor createTableDescriptor(final String name, + final int versions) { + return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS, + versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED); + } + + /** + * Create a table of name name with {@link COLUMNS} for + * families. + * @param name Name to give table. + * @param versions How many versions to allow per column. + * @return Column descriptor. + */ + protected HTableDescriptor createTableDescriptor(final String name, + final int minVersions, final int versions, final int ttl, boolean keepDeleted) { + HTableDescriptor htd = new HTableDescriptor(name); + for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) { + htd.addFamily(new HColumnDescriptor(cfName) + .setMinVersions(minVersions) + .setMaxVersions(versions) + .setKeepDeletedCells(keepDeleted) + .setBlockCacheEnabled(false) + .setTimeToLive(ttl) + ); + } + return htd; + } + + /** + * Add content to region r on the passed column + * column. + * Adds data of the from 'aaa', 'aab', etc where key and value are the same. + * @param r + * @param columnFamily + * @param column + * @throws IOException + * @return count of what we added. + */ + public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column) + throws IOException { + byte [] startKey = r.getRegionInfo().getStartKey(); + byte [] endKey = r.getRegionInfo().getEndKey(); + byte [] startKeyBytes = startKey; + if (startKeyBytes == null || startKeyBytes.length == 0) { + startKeyBytes = START_KEY_BYTES; + } + return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column), + startKeyBytes, endKey, -1); + } + + /** + * Add content to region r on the passed column + * column. + * Adds data of the from 'aaa', 'aab', etc where key and value are the same. + * @param r + * @param columnFamily + * @throws IOException + * @return count of what we added. + */ + protected static long addContent(final HRegion r, final byte [] columnFamily) + throws IOException { + return addContent(r, columnFamily, null); + } + + /** + * Add content to region r on the passed column + * column. + * Adds data of the from 'aaa', 'aab', etc where key and value are the same. + * @param updater An instance of {@link Incommon}. + * @param columnFamily + * @throws IOException + * @return count of what we added. + */ + protected static long addContent(final Incommon updater, + final String columnFamily) throws IOException { + return addContent(updater, columnFamily, START_KEY_BYTES, null); + } + + protected static long addContent(final Incommon updater, final String family, + final String column) throws IOException { + return addContent(updater, family, column, START_KEY_BYTES, null); + } + + /** + * Add content to region r on the passed column + * column. + * Adds data of the from 'aaa', 'aab', etc where key and value are the same. + * @param updater An instance of {@link Incommon}. + * @param columnFamily + * @param startKeyBytes Where to start the rows inserted + * @param endKey Where to stop inserting rows. + * @return count of what we added. + * @throws IOException + */ + protected static long addContent(final Incommon updater, final String columnFamily, + final byte [] startKeyBytes, final byte [] endKey) + throws IOException { + return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1); + } + + protected static long addContent(final Incommon updater, final String family, + final String column, final byte [] startKeyBytes, + final byte [] endKey) throws IOException { + return addContent(updater, family, column, startKeyBytes, endKey, -1); + } + + /** + * Add content to region r on the passed column + * column. + * Adds data of the from 'aaa', 'aab', etc where key and value are the same. + * @param updater An instance of {@link Incommon}. + * @param column + * @param startKeyBytes Where to start the rows inserted + * @param endKey Where to stop inserting rows. + * @param ts Timestamp to write the content with. + * @return count of what we added. + * @throws IOException + */ + protected static long addContent(final Incommon updater, + final String columnFamily, + final String column, + final byte [] startKeyBytes, final byte [] endKey, final long ts) + throws IOException { + long count = 0; + // Add rows of three characters. The first character starts with the + // 'a' character and runs up to 'z'. Per first character, we run the + // second character over same range. And same for the third so rows + // (and values) look like this: 'aaa', 'aab', 'aac', etc. + char secondCharStart = (char)startKeyBytes[1]; + char thirdCharStart = (char)startKeyBytes[2]; + EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) { + for (char d = secondCharStart; d <= LAST_CHAR; d++) { + for (char e = thirdCharStart; e <= LAST_CHAR; e++) { + byte [] t = new byte [] {(byte)c, (byte)d, (byte)e}; + if (endKey != null && endKey.length > 0 + && Bytes.compareTo(endKey, t) <= 0) { + break EXIT; + } + try { + Put put; + if(ts != -1) { + put = new Put(t, ts, null); + } else { + put = new Put(t); + } + try { + StringBuilder sb = new StringBuilder(); + if (column != null && column.contains(":")) { + sb.append(column); + } else { + if (columnFamily != null) { + sb.append(columnFamily); + if (!columnFamily.endsWith(":")) { + sb.append(":"); + } + if (column != null) { + sb.append(column); + } + } + } + byte[][] split = + KeyValue.parseColumn(Bytes.toBytes(sb.toString())); + if(split.length == 1) { + put.add(split[0], new byte[0], t); + } else { + put.add(split[0], split[1], t); + } + updater.put(put); + count++; + } catch (RuntimeException ex) { + ex.printStackTrace(); + throw ex; + } catch (IOException ex) { + ex.printStackTrace(); + throw ex; + } + } catch (RuntimeException ex) { + ex.printStackTrace(); + throw ex; + } catch (IOException ex) { + ex.printStackTrace(); + throw ex; + } + } + // Set start character back to FIRST_CHAR after we've done first loop. + thirdCharStart = FIRST_CHAR; + } + secondCharStart = FIRST_CHAR; + } + return count; + } + + /** + * Implementors can flushcache. + */ + public static interface FlushCache { + /** + * @throws IOException + */ + public void flushcache() throws IOException; + } + + /** + * Interface used by tests so can do common operations against an HTable + * or an HRegion. + * + * TOOD: Come up w/ a better name for this interface. + */ + public static interface Incommon { + /** + * + * @param delete + * @param lockid + * @param writeToWAL + * @throws IOException + */ + public void delete(Delete delete, Integer lockid, boolean writeToWAL) + throws IOException; + + /** + * @param put + * @throws IOException + */ + public void put(Put put) throws IOException; + + public Result get(Get get) throws IOException; + + /** + * @param family + * @param qualifiers + * @param firstRow + * @param ts + * @return scanner for specified columns, first row and timestamp + * @throws IOException + */ + public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers, + byte [] firstRow, long ts) + throws IOException; + } + + /** + * A class that makes a {@link Incommon} out of a {@link HRegion} + */ + public static class HRegionIncommon implements Incommon, FlushCache { + final HRegion region; + + /** + * @param HRegion + */ + public HRegionIncommon(final HRegion HRegion) { + this.region = HRegion; + } + + public void put(Put put) throws IOException { + region.put(put); + } + + public void delete(Delete delete, Integer lockid, boolean writeToWAL) + throws IOException { + this.region.delete(delete, lockid, writeToWAL); + } + + public Result get(Get get) throws IOException { + return region.get(get, null); + } + + public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers, + byte [] firstRow, long ts) + throws IOException { + Scan scan = new Scan(firstRow); + if(qualifiers == null || qualifiers.length == 0) { + scan.addFamily(family); + } else { + for(int i=0; i { + public boolean next(List values) + throws IOException; + + public void close() throws IOException; + } + + public static class ClientScannerIncommon implements ScannerIncommon { + ResultScanner scanner; + public ClientScannerIncommon(ResultScanner scanner) { + this.scanner = scanner; + } + + public boolean next(List values) + throws IOException { + Result results = scanner.next(); + if (results == null) { + return false; + } + values.clear(); + values.addAll(results.list()); + return true; + } + + public void close() throws IOException { + scanner.close(); + } + + @SuppressWarnings("unchecked") + public Iterator iterator() { + return scanner.iterator(); + } + } + + public static class InternalScannerIncommon implements ScannerIncommon { + InternalScanner scanner; + + public InternalScannerIncommon(InternalScanner scanner) { + this.scanner = scanner; + } + + public boolean next(List results) + throws IOException { + return scanner.next(results); + } + + public void close() throws IOException { + scanner.close(); + } + + public Iterator iterator() { + throw new UnsupportedOperationException(); + } + } + +// protected void assertCellEquals(final HRegion region, final byte [] row, +// final byte [] column, final long timestamp, final String value) +// throws IOException { +// Map result = region.getFull(row, null, timestamp, 1, null); +// Cell cell_value = result.get(column); +// if (value == null) { +// assertEquals(Bytes.toString(column) + " at timestamp " + timestamp, null, +// cell_value); +// } else { +// if (cell_value == null) { +// fail(Bytes.toString(column) + " at timestamp " + timestamp + +// "\" was expected to be \"" + value + " but was null"); +// } +// if (cell_value != null) { +// assertEquals(Bytes.toString(column) + " at timestamp " +// + timestamp, value, new String(cell_value.getValue())); +// } +// } +// } + + protected void assertResultEquals(final HRegion region, final byte [] row, + final byte [] family, final byte [] qualifier, final long timestamp, + final byte [] value) + throws IOException { + Get get = new Get(row); + get.setTimeStamp(timestamp); + Result res = region.get(get, null); + NavigableMap>> map = + res.getMap(); + byte [] res_value = map.get(family).get(qualifier).get(timestamp); + + if (value == null) { + assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) + + " at timestamp " + timestamp, null, res_value); + } else { + if (res_value == null) { + fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) + + " at timestamp " + timestamp + "\" was expected to be \"" + + Bytes.toStringBinary(value) + " but was null"); + } + if (res_value != null) { + assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) + + " at timestamp " + + timestamp, value, new String(res_value)); + } + } + } + + /** + * Initializes parameters used in the test environment: + * + * Sets the configuration parameter TEST_DIRECTORY_KEY if not already set. + * Sets the boolean debugging if "DEBUGGING" is set in the environment. + * If debugging is enabled, reconfigures logging so that the root log level is + * set to WARN and the logging level for the package is set to DEBUG. + */ + public static void initialize() { + if (System.getProperty(TEST_DIRECTORY_KEY) == null) { + System.setProperty(TEST_DIRECTORY_KEY, new File( + "build/hbase/test").getAbsolutePath()); + } + } + + /** + * Common method to close down a MiniDFSCluster and the associated file system + * + * @param cluster + */ + public static void shutdownDfs(MiniDFSCluster cluster) { + if (cluster != null) { + LOG.info("Shutting down Mini DFS "); + try { + cluster.shutdown(); + } catch (Exception e) { + /// Can get a java.lang.reflect.UndeclaredThrowableException thrown + // here because of an InterruptedException. Don't let exceptions in + // here be cause of test failure. + } + try { + FileSystem fs = cluster.getFileSystem(); + if (fs != null) { + LOG.info("Shutting down FileSystem"); + fs.close(); + } + FileSystem.closeAll(); + } catch (IOException e) { + LOG.error("error closing file system", e); + } + } + } + + /** + * You must call {@link #closeRootAndMeta()} when done after calling this + * method. It does cleanup. + * @throws IOException + */ + protected void createRootAndMetaRegions() throws IOException { + root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir, + conf, HTableDescriptor.ROOT_TABLEDESC); + meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir, + conf, HTableDescriptor.META_TABLEDESC); + HRegion.addRegionToMETA(root, meta); + } + + protected void closeRootAndMeta() throws IOException { + if (meta != null) { + meta.close(); + meta.getLog().closeAndDelete(); + } + if (root != null) { + root.close(); + root.getLog().closeAndDelete(); + } + } + + public static void assertByteEquals(byte[] expected, + byte[] actual) { + if (Bytes.compareTo(expected, actual) != 0) { + throw new AssertionFailedError("expected:<" + + Bytes.toString(expected) + "> but was:<" + + Bytes.toString(actual) + ">"); + } + } + + public static void assertEquals(byte[] expected, + byte[] actual) { + if (Bytes.compareTo(expected, actual) != 0) { + throw new AssertionFailedError("expected:<" + + Bytes.toStringBinary(expected) + "> but was:<" + + Bytes.toStringBinary(actual) + ">"); + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java new file mode 100644 index 0000000..3a76d89 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -0,0 +1,2317 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.UnknownHostException; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Random; +import java.util.UUID; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Jdk14Logger; +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.ChecksumUtil; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; +import org.apache.hadoop.hbase.util.RegionSplitter; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKConfig; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MiniMRCluster; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.KeeperException.NodeExistsException; +import org.apache.zookeeper.ZooKeeper; + +/** + * Facility for testing HBase. Replacement for + * old HBaseTestCase and HBaseClusterTestCase functionality. + * Create an instance and keep it around testing HBase. This class is + * meant to be your one-stop shop for anything you might need testing. Manages + * one cluster at a time only. Managed cluster can be an in-process + * {@link MiniHBaseCluster}, or a deployed cluster of type {@link DistributedHBaseCluster}. + * Not all methods work with the real cluster. + * Depends on log4j being on classpath and + * hbase-site.xml for logging and test-run configuration. It does not set + * logging levels nor make changes to configuration parameters. + */ +public class HBaseTestingUtility { + private static final Log LOG = LogFactory.getLog(HBaseTestingUtility.class); + private Configuration conf; + private MiniZooKeeperCluster zkCluster = null; + + /** + * The default number of regions per regionserver when creating a pre-split + * table. + */ + private static int DEFAULT_REGIONS_PER_SERVER = 5; + + /** + * Set if we were passed a zkCluster. If so, we won't shutdown zk as + * part of general shutdown. + */ + private boolean passedZkCluster = false; + private MiniDFSCluster dfsCluster = null; + + private HBaseCluster hbaseCluster = null; + private MiniMRCluster mrCluster = null; + + // Directory where we put the data for this instance of HBaseTestingUtility + private File dataTestDir = null; + + // Directory (usually a subdirectory of dataTestDir) used by the dfs cluster + // if any + private File clusterTestDir = null; + + /** + * System property key to get test directory value. + * Name is as it is because mini dfs has hard-codings to put test data here. + * It should NOT be used directly in HBase, as it's a property used in + * mini dfs. + * @deprecated can be used only with mini dfs + */ + private static final String TEST_DIRECTORY_KEY = "test.build.data"; + + /** + * System property key to get base test directory value + */ + public static final String BASE_TEST_DIRECTORY_KEY = + "test.build.data.basedirectory"; + + /** + * Default base directory for test output. + */ + public static final String DEFAULT_BASE_TEST_DIRECTORY = "target/test-data"; + + /** Compression algorithms to use in parameterized JUnit 4 tests */ + public static final List COMPRESSION_ALGORITHMS_PARAMETERIZED = + Arrays.asList(new Object[][] { + { Compression.Algorithm.NONE }, + { Compression.Algorithm.GZ } + }); + + /** This is for unit tests parameterized with a single boolean. */ + public static final List BOOLEAN_PARAMETERIZED = + Arrays.asList(new Object[][] { + { new Boolean(false) }, + { new Boolean(true) } + }); + + /** Compression algorithms to use in testing */ + public static final Compression.Algorithm[] COMPRESSION_ALGORITHMS ={ + Compression.Algorithm.NONE, Compression.Algorithm.GZ + }; + + /** + * Create all combinations of Bloom filters and compression algorithms for + * testing. + */ + private static List bloomAndCompressionCombinations() { + List configurations = new ArrayList(); + for (Compression.Algorithm comprAlgo : + HBaseTestingUtility.COMPRESSION_ALGORITHMS) { + for (StoreFile.BloomType bloomType : StoreFile.BloomType.values()) { + configurations.add(new Object[] { comprAlgo, bloomType }); + } + } + return Collections.unmodifiableList(configurations); + } + + public static final Collection BLOOM_AND_COMPRESSION_COMBINATIONS = + bloomAndCompressionCombinations(); + + public HBaseTestingUtility() { + this(HBaseConfiguration.create()); + } + + public HBaseTestingUtility(Configuration conf) { + this.conf = conf; + + // a hbase checksum verification failure will cause unit tests to fail + ChecksumUtil.generateExceptionForChecksumFailureForTest(true); + setHDFSClientRetryProperty(); + } + + private void setHDFSClientRetryProperty() { + this.conf.setInt("hdfs.client.retries.number", 1); + HBaseFileSystem.setRetryCounts(conf); + } + + /** + * Returns this classes's instance of {@link Configuration}. Be careful how + * you use the returned Configuration since {@link HConnection} instances + * can be shared. The Map of HConnections is keyed by the Configuration. If + * say, a Connection was being used against a cluster that had been shutdown, + * see {@link #shutdownMiniCluster()}, then the Connection will no longer + * be wholesome. Rather than use the return direct, its usually best to + * make a copy and use that. Do + * Configuration c = new Configuration(INSTANCE.getConfiguration()); + * @return Instance of Configuration. + */ + public Configuration getConfiguration() { + return this.conf; + } + + public void setHBaseCluster(HBaseCluster hbaseCluster) { + this.hbaseCluster = hbaseCluster; + } + + /** + * @return Where to write test data on local filesystem; usually + * {@link #DEFAULT_BASE_TEST_DIRECTORY} + * Should not be used by the unit tests, hence its's private. + * Unit test will use a subdirectory of this directory. + * @see #setupDataTestDir() + * @see #getTestFileSystem() + */ + private Path getBaseTestDir() { + String PathName = System.getProperty( + BASE_TEST_DIRECTORY_KEY, DEFAULT_BASE_TEST_DIRECTORY); + + return new Path(PathName); + } + + /** + * @return Where to write test data on local filesystem, specific to + * the test. Useful for tests that do not use a cluster. + * Creates it if it does not exist already. + * @see #getTestFileSystem() + */ + public Path getDataTestDir() { + if (dataTestDir == null){ + setupDataTestDir(); + } + return new Path(dataTestDir.getAbsolutePath()); + } + + /** + * @return Where the DFS cluster will write data on the local subsystem. + * Creates it if it does not exist already. + * @see #getTestFileSystem() + */ + public Path getClusterTestDir() { + if (clusterTestDir == null){ + setupClusterTestDir(); + } + return new Path(clusterTestDir.getAbsolutePath()); + } + + /** + * @param subdirName + * @return Path to a subdirectory named subdirName under + * {@link #getDataTestDir()}. + * Does *NOT* create it if it does not exist. + */ + public Path getDataTestDir(final String subdirName) { + return new Path(getDataTestDir(), subdirName); + } + + /** + * Home our data in a dir under {@link #DEFAULT_BASE_TEST_DIRECTORY}. + * Give it a random name so can have many concurrent tests running if + * we need to. It needs to amend the {@link #TEST_DIRECTORY_KEY} + * System property, as it's what minidfscluster bases + * it data dir on. Moding a System property is not the way to do concurrent + * instances -- another instance could grab the temporary + * value unintentionally -- but not anything can do about it at moment; + * single instance only is how the minidfscluster works. + * + * We also create the underlying directory for + * hadoop.log.dir, mapred.local.dir and hadoop.tmp.dir, and set the values + * in the conf, and as a system property for hadoop.tmp.dir + * + * @return The calculated data test build directory. + */ + private void setupDataTestDir() { + if (dataTestDir != null) { + LOG.warn("Data test dir already setup in " + + dataTestDir.getAbsolutePath()); + return; + } + + String randomStr = UUID.randomUUID().toString(); + Path testPath= new Path(getBaseTestDir(), randomStr); + + dataTestDir = new File(testPath.toString()).getAbsoluteFile(); + dataTestDir.deleteOnExit(); + + createSubDirAndSystemProperty( + "hadoop.log.dir", + testPath, "hadoop-log-dir"); + + // This is defaulted in core-default.xml to /tmp/hadoop-${user.name}, but + // we want our own value to ensure uniqueness on the same machine + createSubDirAndSystemProperty( + "hadoop.tmp.dir", + testPath, "hadoop-tmp-dir"); + + // Read and modified in org.apache.hadoop.mapred.MiniMRCluster + createSubDir( + "mapred.local.dir", + testPath, "mapred-local-dir"); + + createSubDirAndSystemProperty( + "mapred.working.dir", + testPath, "mapred-working-dir"); + + createSubDir( + "hbase.local.dir", + testPath, "hbase-local-dir"); + } + + private void createSubDir(String propertyName, Path parent, String subDirName){ + Path newPath= new Path(parent, subDirName); + File newDir = new File(newPath.toString()).getAbsoluteFile(); + newDir.deleteOnExit(); + conf.set(propertyName, newDir.getAbsolutePath()); + } + + private void createSubDirAndSystemProperty( + String propertyName, Path parent, String subDirName){ + + String sysValue = System.getProperty(propertyName); + + if (sysValue != null) { + // There is already a value set. So we do nothing but hope + // that there will be no conflicts + LOG.info("System.getProperty(\""+propertyName+"\") already set to: "+ + sysValue + " so I do NOT create it in "+dataTestDir.getAbsolutePath()); + String confValue = conf.get(propertyName); + if (confValue != null && !confValue.endsWith(sysValue)){ + LOG.warn( + propertyName + " property value differs in configuration and system: "+ + "Configuration="+confValue+" while System="+sysValue+ + " Erasing configuration value by system value." + ); + } + conf.set(propertyName, sysValue); + } else { + // Ok, it's not set, so we create it as a subdirectory + createSubDir(propertyName, parent, subDirName); + System.setProperty(propertyName, conf.get(propertyName)); + } + } + + /** + * Creates a directory for the DFS cluster, under the test data + */ + private void setupClusterTestDir() { + if (clusterTestDir != null) { + LOG.warn("Cluster test dir already setup in " + + clusterTestDir.getAbsolutePath()); + return; + } + + // Using randomUUID ensures that multiple clusters can be launched by + // a same test, if it stops & starts them + Path testDir = getDataTestDir("dfscluster_" + UUID.randomUUID().toString()); + clusterTestDir = new File(testDir.toString()).getAbsoluteFile(); + // Have it cleaned up on exit + clusterTestDir.deleteOnExit(); + } + + /** + * @throws IOException If a cluster -- zk, dfs, or hbase -- already running. + */ + public void isRunningCluster() throws IOException { + if (dfsCluster == null) return; + throw new IOException("Cluster already running at " + + this.clusterTestDir); + } + + /** + * Start a minidfscluster. + * @param servers How many DNs to start. + * @throws Exception + * @see {@link #shutdownMiniDFSCluster()} + * @return The mini dfs cluster created. + */ + public MiniDFSCluster startMiniDFSCluster(int servers) throws Exception { + return startMiniDFSCluster(servers, null); + } + + /** + * Start a minidfscluster. + * This is useful if you want to run datanode on distinct hosts for things + * like HDFS block location verification. + * If you start MiniDFSCluster without host names, all instances of the + * datanodes will have the same host name. + * @param hosts hostnames DNs to run on. + * @throws Exception + * @see {@link #shutdownMiniDFSCluster()} + * @return The mini dfs cluster created. + */ + public MiniDFSCluster startMiniDFSCluster(final String hosts[]) + throws Exception { + if ( hosts != null && hosts.length != 0) { + return startMiniDFSCluster(hosts.length, hosts); + } else { + return startMiniDFSCluster(1, null); + } + } + + /** + * Start a minidfscluster. + * Can only create one. + * @param servers How many DNs to start. + * @param hosts hostnames DNs to run on. + * @throws Exception + * @see {@link #shutdownMiniDFSCluster()} + * @return The mini dfs cluster created. + */ + public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[]) + throws Exception { + + // Check that there is not already a cluster running + isRunningCluster(); + + // Initialize the local directory used by the MiniDFS + if (clusterTestDir == null) { + setupClusterTestDir(); + } + + // We have to set this property as it is used by MiniCluster + System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.toString()); + + // Some tests also do this: + // System.getProperty("test.cache.data", "build/test/cache"); + // It's also deprecated + System.setProperty("test.cache.data", this.clusterTestDir.toString()); + + // Ok, now we can start + this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true, + true, null, null, hosts, null); + + // Set this just-started cluster as our filesystem. + FileSystem fs = this.dfsCluster.getFileSystem(); + this.conf.set("fs.defaultFS", fs.getUri().toString()); + // Do old style too just to be safe. + this.conf.set("fs.default.name", fs.getUri().toString()); + + // Wait for the cluster to be totally up + this.dfsCluster.waitClusterUp(); + + return this.dfsCluster; + } + + /** + * Shuts down instance created by call to {@link #startMiniDFSCluster(int)} + * or does nothing. + * @throws Exception + */ + public void shutdownMiniDFSCluster() throws Exception { + if (this.dfsCluster != null) { + // The below throws an exception per dn, AsynchronousCloseException. + this.dfsCluster.shutdown(); + dfsCluster = null; + } + + } + + /** + * Call this if you only want a zk cluster. + * @see #startMiniZKCluster() if you want zk + dfs + hbase mini cluster. + * @throws Exception + * @see #shutdownMiniZKCluster() + * @return zk cluster started. + */ + public MiniZooKeeperCluster startMiniZKCluster() throws Exception { + return startMiniZKCluster(1); + } + + /** + * Call this if you only want a zk cluster. + * @param zooKeeperServerNum + * @see #startMiniZKCluster() if you want zk + dfs + hbase mini cluster. + * @throws Exception + * @see #shutdownMiniZKCluster() + * @return zk cluster started. + */ + public MiniZooKeeperCluster startMiniZKCluster(int zooKeeperServerNum) + throws Exception { + File zkClusterFile = new File(getClusterTestDir().toString()); + return startMiniZKCluster(zkClusterFile, zooKeeperServerNum); + } + + private MiniZooKeeperCluster startMiniZKCluster(final File dir) + throws Exception { + return startMiniZKCluster(dir,1); + } + + private MiniZooKeeperCluster startMiniZKCluster(final File dir, + int zooKeeperServerNum) + throws Exception { + if (this.zkCluster != null) { + throw new IOException("Cluster already running at " + dir); + } + this.passedZkCluster = false; + this.zkCluster = new MiniZooKeeperCluster(this.getConfiguration()); + int clientPort = this.zkCluster.startup(dir,zooKeeperServerNum); + this.conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, + Integer.toString(clientPort)); + return this.zkCluster; + } + + /** + * Shuts down zk cluster created by call to {@link #startMiniZKCluster(File)} + * or does nothing. + * @throws IOException + * @see #startMiniZKCluster() + */ + public void shutdownMiniZKCluster() throws IOException { + if (this.zkCluster != null) { + this.zkCluster.shutdown(); + this.zkCluster = null; + } + } + + /** + * Start up a minicluster of hbase, dfs, and zookeeper. + * @throws Exception + * @return Mini hbase cluster instance created. + * @see {@link #shutdownMiniDFSCluster()} + */ + public MiniHBaseCluster startMiniCluster() throws Exception { + return startMiniCluster(1, 1); + } + + /** + * Start up a minicluster of hbase, optionally dfs, and zookeeper. + * Modifies Configuration. Homes the cluster data directory under a random + * subdirectory in a directory under System property test.build.data. + * Directory is cleaned up on exit. + * @param numSlaves Number of slaves to start up. We'll start this many + * datanodes and regionservers. If numSlaves is > 1, then make sure + * hbase.regionserver.info.port is -1 (i.e. no ui per regionserver) otherwise + * bind errors. + * @throws Exception + * @see {@link #shutdownMiniCluster()} + * @return Mini hbase cluster instance created. + */ + public MiniHBaseCluster startMiniCluster(final int numSlaves) + throws Exception { + return startMiniCluster(1, numSlaves); + } + + + /** + * start minicluster + * @throws Exception + * @see {@link #shutdownMiniCluster()} + * @return Mini hbase cluster instance created. + */ + public MiniHBaseCluster startMiniCluster(final int numMasters, + final int numSlaves) + throws Exception { + return startMiniCluster(numMasters, numSlaves, null); + } + + + /** + * Start up a minicluster of hbase, optionally dfs, and zookeeper. + * Modifies Configuration. Homes the cluster data directory under a random + * subdirectory in a directory under System property test.build.data. + * Directory is cleaned up on exit. + * @param numMasters Number of masters to start up. We'll start this many + * hbase masters. If numMasters > 1, you can find the active/primary master + * with {@link MiniHBaseCluster#getMaster()}. + * @param numSlaves Number of slaves to start up. We'll start this many + * regionservers. If dataNodeHosts == null, this also indicates the number of + * datanodes to start. If dataNodeHosts != null, the number of datanodes is + * based on dataNodeHosts.length. + * If numSlaves is > 1, then make sure + * hbase.regionserver.info.port is -1 (i.e. no ui per regionserver) otherwise + * bind errors. + * @param dataNodeHosts hostnames DNs to run on. + * This is useful if you want to run datanode on distinct hosts for things + * like HDFS block location verification. + * If you start MiniDFSCluster without host names, + * all instances of the datanodes will have the same host name. + * @throws Exception + * @see {@link #shutdownMiniCluster()} + * @return Mini hbase cluster instance created. + */ + public MiniHBaseCluster startMiniCluster(final int numMasters, + final int numSlaves, final String[] dataNodeHosts) + throws Exception { + int numDataNodes = numSlaves; + if ( dataNodeHosts != null && dataNodeHosts.length != 0) { + numDataNodes = dataNodeHosts.length; + } + + LOG.info("Starting up minicluster with " + numMasters + " master(s) and " + + numSlaves + " regionserver(s) and " + numDataNodes + " datanode(s)"); + + // If we already put up a cluster, fail. + isRunningCluster(); + + // Bring up mini dfs cluster. This spews a bunch of warnings about missing + // scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'. + startMiniDFSCluster(numDataNodes, dataNodeHosts); + + // Start up a zk cluster. + if (this.zkCluster == null) { + startMiniZKCluster(clusterTestDir); + } + + // Start the MiniHBaseCluster + return startMiniHBaseCluster(numMasters, numSlaves); + } + + /** + * Starts up mini hbase cluster. Usually used after call to + * {@link #startMiniCluster(int, int)} when doing stepped startup of clusters. + * Usually you won't want this. You'll usually want {@link #startMiniCluster()}. + * @param numMasters + * @param numSlaves + * @return Reference to the hbase mini hbase cluster. + * @throws IOException + * @throws InterruptedException + * @see {@link #startMiniCluster()} + */ + public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, + final int numSlaves) + throws IOException, InterruptedException { + // Now do the mini hbase cluster. Set the hbase.rootdir in config. + createRootDir(); + + // These settings will make the server waits until this exact number of + // regions servers are connected. + if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) { + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, numSlaves); + } + if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) { + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, numSlaves); + } + + Configuration c = new Configuration(this.conf); + this.hbaseCluster = new MiniHBaseCluster(c, numMasters, numSlaves); + // Don't leave here till we've done a successful scan of the .META. + HTable t = new HTable(c, HConstants.META_TABLE_NAME); + ResultScanner s = t.getScanner(new Scan()); + while (s.next() != null) { + continue; + } + s.close(); + t.close(); + + getHBaseAdmin(); // create immediately the hbaseAdmin + LOG.info("Minicluster is up"); + return (MiniHBaseCluster)this.hbaseCluster; + } + + /** + * Starts the hbase cluster up again after shutting it down previously in a + * test. Use this if you want to keep dfs/zk up and just stop/start hbase. + * @param servers number of region servers + * @throws IOException + */ + public void restartHBaseCluster(int servers) throws IOException, InterruptedException { + this.hbaseCluster = new MiniHBaseCluster(this.conf, servers); + // Don't leave here till we've done a successful scan of the .META. + HTable t = new HTable(new Configuration(this.conf), HConstants.META_TABLE_NAME); + ResultScanner s = t.getScanner(new Scan()); + while (s.next() != null) { + // do nothing + } + LOG.info("HBase has been restarted"); + s.close(); + t.close(); + } + + /** + * @return Current mini hbase cluster. Only has something in it after a call + * to {@link #startMiniCluster()}. + * @see #startMiniCluster() + */ + public MiniHBaseCluster getMiniHBaseCluster() { + if (this.hbaseCluster instanceof MiniHBaseCluster) { + return (MiniHBaseCluster)this.hbaseCluster; + } + throw new RuntimeException(hbaseCluster + " not an instance of " + + MiniHBaseCluster.class.getName()); + } + + /** + * Stops mini hbase, zk, and hdfs clusters. + * @throws IOException + * @see {@link #startMiniCluster(int)} + */ + public void shutdownMiniCluster() throws Exception { + LOG.info("Shutting down minicluster"); + shutdownMiniHBaseCluster(); + if (!this.passedZkCluster){ + shutdownMiniZKCluster(); + } + shutdownMiniDFSCluster(); + + // Clean up our directory. + if (this.clusterTestDir != null && this.clusterTestDir.exists()) { + // Need to use deleteDirectory because File.delete required dir is empty. + if (!FSUtils.deleteDirectory(FileSystem.getLocal(this.conf), + new Path(this.clusterTestDir.toString()))) { + LOG.warn("Failed delete of " + this.clusterTestDir.toString()); + } + this.clusterTestDir = null; + } + LOG.info("Minicluster is down"); + } + + /** + * Shutdown HBase mini cluster. Does not shutdown zk or dfs if running. + * @throws IOException + */ + public void shutdownMiniHBaseCluster() throws IOException { + if (hbaseAdmin != null) { + hbaseAdmin.close(); + hbaseAdmin = null; + } + // unset the configuration for MIN and MAX RS to start + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1); + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1); + if (this.hbaseCluster != null) { + this.hbaseCluster.shutdown(); + // Wait till hbase is down before going on to shutdown zk. + this.hbaseCluster.waitUntilShutDown(); + this.hbaseCluster = null; + } + } + + /** + * Returns the path to the default root dir the minicluster uses. + * Note: this does not cause the root dir to be created. + * @return Fully qualified path for the default hbase root dir + * @throws IOException + */ + public Path getDefaultRootDirPath() throws IOException { + FileSystem fs = FileSystem.get(this.conf); + return new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase"); + } + + /** + * Creates an hbase rootdir in user home directory. Also creates hbase + * version file. Normally you won't make use of this method. Root hbasedir + * is created for you as part of mini cluster startup. You'd only use this + * method if you were doing manual operation. + * @return Fully qualified path to hbase root dir + * @throws IOException + */ + public Path createRootDir() throws IOException { + FileSystem fs = FileSystem.get(this.conf); + Path hbaseRootdir = getDefaultRootDirPath(); + this.conf.set(HConstants.HBASE_DIR, hbaseRootdir.toString()); + fs.mkdirs(hbaseRootdir); + FSUtils.setVersion(fs, hbaseRootdir); + return hbaseRootdir; + } + + /** + * Flushes all caches in the mini hbase cluster + * @throws IOException + */ + public void flush() throws IOException { + getMiniHBaseCluster().flushcache(); + } + + /** + * Flushes all caches in the mini hbase cluster + * @throws IOException + */ + public void flush(byte [] tableName) throws IOException { + getMiniHBaseCluster().flushcache(tableName); + } + + /** + * Compact all regions in the mini hbase cluster + * @throws IOException + */ + public void compact(boolean major) throws IOException { + getMiniHBaseCluster().compact(major); + } + + /** + * Compact all of a table's reagion in the mini hbase cluster + * @throws IOException + */ + public void compact(byte [] tableName, boolean major) throws IOException { + getMiniHBaseCluster().compact(tableName, major); + } + + + /** + * Create a table. + * @param tableName + * @param family + * @return An HTable instance for the created table. + * @throws IOException + */ + public HTable createTable(byte[] tableName, byte[] family) + throws IOException{ + return createTable(tableName, new byte[][]{family}); + } + + /** + * Create a table. + * @param tableName + * @param families + * @return An HTable instance for the created table. + * @throws IOException + */ + public HTable createTable(byte[] tableName, byte[][] families) + throws IOException { + return createTable(tableName, families, + new Configuration(getConfiguration())); + } + + public HTable createTable(byte[] tableName, byte[][] families, + int numVersions, byte[] startKey, byte[] endKey, int numRegions) + throws IOException{ + HTableDescriptor desc = new HTableDescriptor(tableName); + for (byte[] family : families) { + HColumnDescriptor hcd = new HColumnDescriptor(family) + .setMaxVersions(numVersions); + desc.addFamily(hcd); + } + getHBaseAdmin().createTable(desc, startKey, endKey, numRegions); + return new HTable(getConfiguration(), tableName); + } + + /** + * Create a table. + * @param tableName + * @param families + * @param c Configuration to use + * @return An HTable instance for the created table. + * @throws IOException + */ + public HTable createTable(byte[] tableName, byte[][] families, + final Configuration c) + throws IOException { + HTableDescriptor desc = new HTableDescriptor(tableName); + for(byte[] family : families) { + desc.addFamily(new HColumnDescriptor(family)); + } + getHBaseAdmin().createTable(desc); + return new HTable(c, tableName); + } + + /** + * Create a table. + * @param tableName + * @param families + * @param c Configuration to use + * @param numVersions + * @return An HTable instance for the created table. + * @throws IOException + */ + public HTable createTable(byte[] tableName, byte[][] families, + final Configuration c, int numVersions) + throws IOException { + HTableDescriptor desc = new HTableDescriptor(tableName); + for(byte[] family : families) { + HColumnDescriptor hcd = new HColumnDescriptor(family) + .setMaxVersions(numVersions); + desc.addFamily(hcd); + } + getHBaseAdmin().createTable(desc); + return new HTable(c, tableName); + } + + /** + * Create a table. + * @param tableName + * @param family + * @param numVersions + * @return An HTable instance for the created table. + * @throws IOException + */ + public HTable createTable(byte[] tableName, byte[] family, int numVersions) + throws IOException { + return createTable(tableName, new byte[][]{family}, numVersions); + } + + /** + * Create a table. + * @param tableName + * @param families + * @param numVersions + * @return An HTable instance for the created table. + * @throws IOException + */ + public HTable createTable(byte[] tableName, byte[][] families, + int numVersions) + throws IOException { + HTableDescriptor desc = new HTableDescriptor(tableName); + for (byte[] family : families) { + HColumnDescriptor hcd = new HColumnDescriptor(family) + .setMaxVersions(numVersions); + desc.addFamily(hcd); + } + getHBaseAdmin().createTable(desc); + return new HTable(new Configuration(getConfiguration()), tableName); + } + + /** + * Create a table. + * @param tableName + * @param families + * @param numVersions + * @return An HTable instance for the created table. + * @throws IOException + */ + public HTable createTable(byte[] tableName, byte[][] families, + int numVersions, int blockSize) throws IOException { + HTableDescriptor desc = new HTableDescriptor(tableName); + for (byte[] family : families) { + HColumnDescriptor hcd = new HColumnDescriptor(family) + .setMaxVersions(numVersions) + .setBlocksize(blockSize); + desc.addFamily(hcd); + } + getHBaseAdmin().createTable(desc); + return new HTable(new Configuration(getConfiguration()), tableName); + } + + /** + * Create a table. + * @param tableName + * @param families + * @param numVersions + * @return An HTable instance for the created table. + * @throws IOException + */ + public HTable createTable(byte[] tableName, byte[][] families, + int[] numVersions) + throws IOException { + HTableDescriptor desc = new HTableDescriptor(tableName); + int i = 0; + for (byte[] family : families) { + HColumnDescriptor hcd = new HColumnDescriptor(family) + .setMaxVersions(numVersions[i]); + desc.addFamily(hcd); + i++; + } + getHBaseAdmin().createTable(desc); + return new HTable(new Configuration(getConfiguration()), tableName); + } + + /** + * Drop an existing table + * @param tableName existing table + */ + public void deleteTable(byte[] tableName) throws IOException { + try { + getHBaseAdmin().disableTable(tableName); + } catch (TableNotEnabledException e) { + LOG.debug("Table: " + Bytes.toString(tableName) + " already disabled, so just deleting it."); + } + getHBaseAdmin().deleteTable(tableName); + } + + /** + * Provide an existing table name to truncate + * @param tableName existing table + * @return HTable to that new table + * @throws IOException + */ + public HTable truncateTable(byte [] tableName) throws IOException { + HTable table = new HTable(getConfiguration(), tableName); + Scan scan = new Scan(); + ResultScanner resScan = table.getScanner(scan); + for(Result res : resScan) { + Delete del = new Delete(res.getRow()); + table.delete(del); + } + resScan = table.getScanner(scan); + resScan.close(); + return table; + } + + /** + * Load table with rows from 'aaa' to 'zzz'. + * @param t Table + * @param f Family + * @return Count of rows loaded. + * @throws IOException + */ + public int loadTable(final HTable t, final byte[] f) throws IOException { + t.setAutoFlush(false); + byte[] k = new byte[3]; + int rowCount = 0; + for (byte b1 = 'a'; b1 <= 'z'; b1++) { + for (byte b2 = 'a'; b2 <= 'z'; b2++) { + for (byte b3 = 'a'; b3 <= 'z'; b3++) { + k[0] = b1; + k[1] = b2; + k[2] = b3; + Put put = new Put(k); + put.add(f, null, k); + t.put(put); + rowCount++; + } + } + } + t.flushCommits(); + return rowCount; + } + + /** + * Load table of multiple column families with rows from 'aaa' to 'zzz'. + * @param t Table + * @param f Array of Families to load + * @return Count of rows loaded. + * @throws IOException + */ + public int loadTable(final HTable t, final byte[][] f) throws IOException { + t.setAutoFlush(false); + byte[] k = new byte[3]; + int rowCount = 0; + for (byte b1 = 'a'; b1 <= 'z'; b1++) { + for (byte b2 = 'a'; b2 <= 'z'; b2++) { + for (byte b3 = 'a'; b3 <= 'z'; b3++) { + k[0] = b1; + k[1] = b2; + k[2] = b3; + Put put = new Put(k); + for (int i = 0; i < f.length; i++) { + put.add(f[i], null, k); + } + t.put(put); + rowCount++; + } + } + } + t.flushCommits(); + return rowCount; + } + + /** + * Load region with rows from 'aaa' to 'zzz'. + * @param r Region + * @param f Family + * @return Count of rows loaded. + * @throws IOException + */ + public int loadRegion(final HRegion r, final byte[] f) + throws IOException { + return loadRegion(r, f, false); + } + + /** + * Load region with rows from 'aaa' to 'zzz'. + * @param r Region + * @param f Family + * @param flush flush the cache if true + * @return Count of rows loaded. + * @throws IOException + */ + public int loadRegion(final HRegion r, final byte[] f, final boolean flush) + throws IOException { + byte[] k = new byte[3]; + int rowCount = 0; + for (byte b1 = 'a'; b1 <= 'z'; b1++) { + for (byte b2 = 'a'; b2 <= 'z'; b2++) { + for (byte b3 = 'a'; b3 <= 'z'; b3++) { + k[0] = b1; + k[1] = b2; + k[2] = b3; + Put put = new Put(k); + put.add(f, null, k); + if (r.getLog() == null) put.setWriteToWAL(false); + r.put(put); + rowCount++; + } + } + if (flush) { + r.flushcache(); + } + } + return rowCount; + } + + /** + * Return the number of rows in the given table. + */ + public int countRows(final HTable table) throws IOException { + Scan scan = new Scan(); + ResultScanner results = table.getScanner(scan); + int count = 0; + for (@SuppressWarnings("unused") Result res : results) { + count++; + } + results.close(); + return count; + } + + public int countRows(final HTable table, final byte[]... families) throws IOException { + Scan scan = new Scan(); + for (byte[] family: families) { + scan.addFamily(family); + } + ResultScanner results = table.getScanner(scan); + int count = 0; + for (@SuppressWarnings("unused") Result res : results) { + count++; + } + results.close(); + return count; + } + + /** + * Return an md5 digest of the entire contents of a table. + */ + public String checksumRows(final HTable table) throws Exception { + Scan scan = new Scan(); + ResultScanner results = table.getScanner(scan); + MessageDigest digest = MessageDigest.getInstance("MD5"); + for (Result res : results) { + digest.update(res.getRow()); + } + results.close(); + return digest.toString(); + } + + /** + * Creates many regions names "aaa" to "zzz". + * + * @param table The table to use for the data. + * @param columnFamily The family to insert the data into. + * @return count of regions created. + * @throws IOException When creating the regions fails. + */ + public int createMultiRegions(HTable table, byte[] columnFamily) + throws IOException { + return createMultiRegions(table, columnFamily, true); + } + + public static final byte[][] KEYS = { + HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes("bbb"), + Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"), + Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"), + Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"), + Bytes.toBytes("lll"), Bytes.toBytes("mmm"), Bytes.toBytes("nnn"), + Bytes.toBytes("ooo"), Bytes.toBytes("ppp"), Bytes.toBytes("qqq"), + Bytes.toBytes("rrr"), Bytes.toBytes("sss"), Bytes.toBytes("ttt"), + Bytes.toBytes("uuu"), Bytes.toBytes("vvv"), Bytes.toBytes("www"), + Bytes.toBytes("xxx"), Bytes.toBytes("yyy") + }; + + public static final byte[][] KEYS_FOR_HBA_CREATE_TABLE = { + Bytes.toBytes("bbb"), + Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"), + Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"), + Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"), + Bytes.toBytes("lll"), Bytes.toBytes("mmm"), Bytes.toBytes("nnn"), + Bytes.toBytes("ooo"), Bytes.toBytes("ppp"), Bytes.toBytes("qqq"), + Bytes.toBytes("rrr"), Bytes.toBytes("sss"), Bytes.toBytes("ttt"), + Bytes.toBytes("uuu"), Bytes.toBytes("vvv"), Bytes.toBytes("www"), + Bytes.toBytes("xxx"), Bytes.toBytes("yyy"), Bytes.toBytes("zzz") + }; + + + /** + * Creates many regions names "aaa" to "zzz". + * + * @param table The table to use for the data. + * @param columnFamily The family to insert the data into. + * @param cleanupFS True if a previous region should be remove from the FS + * @return count of regions created. + * @throws IOException When creating the regions fails. + */ + public int createMultiRegions(HTable table, byte[] columnFamily, boolean cleanupFS) + throws IOException { + return createMultiRegions(getConfiguration(), table, columnFamily, KEYS, cleanupFS); + } + + /** + * Creates the specified number of regions in the specified table. + * @param c + * @param table + * @param family + * @param numRegions + * @return + * @throws IOException + */ + public int createMultiRegions(final Configuration c, final HTable table, + final byte [] family, int numRegions) + throws IOException { + if (numRegions < 3) throw new IOException("Must create at least 3 regions"); + byte [] startKey = Bytes.toBytes("aaaaa"); + byte [] endKey = Bytes.toBytes("zzzzz"); + byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3); + byte [][] regionStartKeys = new byte[splitKeys.length+1][]; + for (int i=0;i,,123456789" row with an empty start + // and end key. Adding the custom regions below adds those blindly, + // including the new start region from empty to "bbb". lg + List rows = getMetaTableRows(htd.getName()); + String regionToDeleteInFS = table + .getRegionsInRange(Bytes.toBytes(""), Bytes.toBytes("")).get(0) + .getRegionInfo().getEncodedName(); + List newRegions = new ArrayList(startKeys.length); + // add custom ones + int count = 0; + for (int i = 0; i < startKeys.length; i++) { + int j = (i + 1) % startKeys.length; + HRegionInfo hri = new HRegionInfo(table.getTableName(), + startKeys[i], startKeys[j]); + Put put = new Put(hri.getRegionName()); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + meta.put(put); + LOG.info("createMultiRegions: inserted " + hri.toString()); + newRegions.add(hri); + count++; + } + // see comment above, remove "old" (or previous) single region + for (byte[] row : rows) { + LOG.info("createMultiRegions: deleting meta row -> " + + Bytes.toStringBinary(row)); + meta.delete(new Delete(row)); + } + if (cleanupFS) { + // see HBASE-7417 - this confused TestReplication + // remove the "old" region from FS + Path tableDir = new Path(getDefaultRootDirPath().toString() + + System.getProperty("file.separator") + htd.getNameAsString() + + System.getProperty("file.separator") + regionToDeleteInFS); + getDFSCluster().getFileSystem().delete(tableDir); + } + // flush cache of regions + HConnection conn = table.getConnection(); + conn.clearRegionCache(); + // assign all the new regions IF table is enabled. + HBaseAdmin admin = getHBaseAdmin(); + if (admin.isTableEnabled(table.getTableName())) { + for(HRegionInfo hri : newRegions) { + admin.assign(hri.getRegionName()); + } + } + + meta.close(); + + return count; + } + + /** + * Create rows in META for regions of the specified table with the specified + * start keys. The first startKey should be a 0 length byte array if you + * want to form a proper range of regions. + * @param conf + * @param htd + * @param startKeys + * @return list of region info for regions added to meta + * @throws IOException + */ + public List createMultiRegionsInMeta(final Configuration conf, + final HTableDescriptor htd, byte [][] startKeys) + throws IOException { + HTable meta = new HTable(conf, HConstants.META_TABLE_NAME); + Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR); + List newRegions = new ArrayList(startKeys.length); + // add custom ones + for (int i = 0; i < startKeys.length; i++) { + int j = (i + 1) % startKeys.length; + HRegionInfo hri = new HRegionInfo(htd.getName(), startKeys[i], + startKeys[j]); + Put put = new Put(hri.getRegionName()); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + meta.put(put); + LOG.info("createMultiRegionsInMeta: inserted " + hri.toString()); + newRegions.add(hri); + } + + meta.close(); + return newRegions; + } + + /** + * Returns all rows from the .META. table. + * + * @throws IOException When reading the rows fails. + */ + public List getMetaTableRows() throws IOException { + // TODO: Redo using MetaReader class + HTable t = new HTable(new Configuration(this.conf), HConstants.META_TABLE_NAME); + List rows = new ArrayList(); + ResultScanner s = t.getScanner(new Scan()); + for (Result result : s) { + LOG.info("getMetaTableRows: row -> " + + Bytes.toStringBinary(result.getRow())); + rows.add(result.getRow()); + } + s.close(); + t.close(); + return rows; + } + + /** + * Returns all rows from the .META. table for a given user table + * + * @throws IOException When reading the rows fails. + */ + public List getMetaTableRows(byte[] tableName) throws IOException { + // TODO: Redo using MetaReader. + HTable t = new HTable(new Configuration(this.conf), HConstants.META_TABLE_NAME); + List rows = new ArrayList(); + ResultScanner s = t.getScanner(new Scan()); + for (Result result : s) { + byte[] val = result.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + if (val == null) { + LOG.error("No region info for row " + Bytes.toString(result.getRow())); + // TODO figure out what to do for this new hosed case. + continue; + } + HRegionInfo info = Writables.getHRegionInfo(val); + if (Bytes.compareTo(info.getTableName(), tableName) == 0) { + LOG.info("getMetaTableRows: row -> " + + Bytes.toStringBinary(result.getRow()) + info); + rows.add(result.getRow()); + } + } + s.close(); + t.close(); + return rows; + } + + /** + * Tool to get the reference to the region server object that holds the + * region of the specified user table. + * It first searches for the meta rows that contain the region of the + * specified table, then gets the index of that RS, and finally retrieves + * the RS's reference. + * @param tableName user table to lookup in .META. + * @return region server that holds it, null if the row doesn't exist + * @throws IOException + */ + public HRegionServer getRSForFirstRegionInTable(byte[] tableName) + throws IOException { + List metaRows = getMetaTableRows(tableName); + if (metaRows == null || metaRows.isEmpty()) { + return null; + } + LOG.debug("Found " + metaRows.size() + " rows for table " + + Bytes.toString(tableName)); + byte [] firstrow = metaRows.get(0); + LOG.debug("FirstRow=" + Bytes.toString(firstrow)); + int index = getMiniHBaseCluster().getServerWith(firstrow); + return getMiniHBaseCluster().getRegionServerThreads().get(index).getRegionServer(); + } + + /** + * Starts a MiniMRCluster with a default number of + * TaskTracker's. + * + * @throws IOException When starting the cluster fails. + */ + public void startMiniMapReduceCluster() throws IOException { + startMiniMapReduceCluster(2); + } + + /** + * Starts a MiniMRCluster. + * + * @param servers The number of TaskTracker's to start. + * @throws IOException When starting the cluster fails. + */ + public void startMiniMapReduceCluster(final int servers) throws IOException { + LOG.info("Starting mini mapreduce cluster..."); + // These are needed for the new and improved Map/Reduce framework + Configuration c = getConfiguration(); + String logDir = c.get("hadoop.log.dir"); + String tmpDir = c.get("hadoop.tmp.dir"); + if (logDir == null) { + logDir = tmpDir; + } + System.setProperty("hadoop.log.dir", logDir); + c.set("mapred.output.dir", tmpDir); + + // Tests were failing because this process used 6GB of virtual memory and was getting killed. + // we up the VM usable so that processes don't get killed. + conf.setFloat("yarn.nodemanager.vmem-pmem-ratio", 8.0f); + + mrCluster = new MiniMRCluster(servers, + FileSystem.get(conf).getUri().toString(), 1); + LOG.info("Mini mapreduce cluster started"); + JobConf mrClusterJobConf = mrCluster.createJobConf(); + c.set("mapred.job.tracker", mrClusterJobConf.get("mapred.job.tracker")); + /* this for mrv2 support */ + conf.set("mapreduce.framework.name", "yarn"); + String rmAdress = mrClusterJobConf.get("yarn.resourcemanager.address"); + if (rmAdress != null) { + conf.set("yarn.resourcemanager.address", rmAdress); + } + String schedulerAdress = + mrClusterJobConf.get("yarn.resourcemanager.scheduler.address"); + if (schedulerAdress != null) { + conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress); + } + } + + /** + * Stops the previously started MiniMRCluster. + */ + public void shutdownMiniMapReduceCluster() { + LOG.info("Stopping mini mapreduce cluster..."); + if (mrCluster != null) { + mrCluster.shutdown(); + mrCluster = null; + } + // Restore configuration to point to local jobtracker + conf.set("mapred.job.tracker", "local"); + LOG.info("Mini mapreduce cluster stopped"); + } + + /** + * Switches the logger for the given class to DEBUG level. + * + * @param clazz The class for which to switch to debug logging. + */ + public void enableDebug(Class clazz) { + Log l = LogFactory.getLog(clazz); + if (l instanceof Log4JLogger) { + ((Log4JLogger) l).getLogger().setLevel(org.apache.log4j.Level.DEBUG); + } else if (l instanceof Jdk14Logger) { + ((Jdk14Logger) l).getLogger().setLevel(java.util.logging.Level.ALL); + } + } + + /** + * Expire the Master's session + * @throws Exception + */ + public void expireMasterSession() throws Exception { + HMaster master = getMiniHBaseCluster().getMaster(); + expireSession(master.getZooKeeper(), false); + } + + /** + * Expire a region server's session + * @param index which RS + * @throws Exception + */ + public void expireRegionServerSession(int index) throws Exception { + HRegionServer rs = getMiniHBaseCluster().getRegionServer(index); + expireSession(rs.getZooKeeper(), false); + decrementMinRegionServerCount(); + } + + private void decrementMinRegionServerCount() { + // decrement the count for this.conf, for newly spwaned master + // this.hbaseCluster shares this configuration too + decrementMinRegionServerCount(getConfiguration()); + + // each master thread keeps a copy of configuration + for (MasterThread master : getHBaseCluster().getMasterThreads()) { + decrementMinRegionServerCount(master.getMaster().getConfiguration()); + } + } + + private void decrementMinRegionServerCount(Configuration conf) { + int currentCount = conf.getInt( + ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1); + if (currentCount != -1) { + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, + Math.max(currentCount - 1, 1)); + } + } + + /** + * Expire a ZooKeeper session as recommended in ZooKeeper documentation + * http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A4 + * There are issues when doing this: + * [1] http://www.mail-archive.com/dev@zookeeper.apache.org/msg01942.html + * [2] https://issues.apache.org/jira/browse/ZOOKEEPER-1105 + * + * @param nodeZK - the ZK to make expiry + * @param checkStatus - true to check if the we can create a HTable with the + * current configuration. + */ + public void expireSession(ZooKeeperWatcher nodeZK, boolean checkStatus) + throws Exception { + Configuration c = new Configuration(this.conf); + String quorumServers = ZKConfig.getZKQuorumServersString(c); + int sessionTimeout = 500; + ZooKeeper zk = nodeZK.getRecoverableZooKeeper().getZooKeeper(); + byte[] password = zk.getSessionPasswd(); + long sessionID = zk.getSessionId(); + + // Expiry seems to be asynchronous (see comment from P. Hunt in [1]), + // so we create a first watcher to be sure that the + // event was sent. We expect that if our watcher receives the event + // other watchers on the same machine will get is as well. + // When we ask to close the connection, ZK does not close it before + // we receive all the events, so don't have to capture the event, just + // closing the connection should be enough. + ZooKeeper monitor = new ZooKeeper(quorumServers, + 1000, new org.apache.zookeeper.Watcher(){ + @Override + public void process(WatchedEvent watchedEvent) { + LOG.info("Monitor ZKW received event="+watchedEvent); + } + } , sessionID, password); + + // Making it expire + ZooKeeper newZK = new ZooKeeper(quorumServers, + sessionTimeout, EmptyWatcher.instance, sessionID, password); + newZK.close(); + LOG.info("ZK Closed Session 0x" + Long.toHexString(sessionID)); + + // Now closing & waiting to be sure that the clients get it. + monitor.close(); + + if (checkStatus) { + new HTable(new Configuration(conf), HConstants.META_TABLE_NAME).close(); + } + } + + /** + * Get the Mini HBase cluster. + * + * @return hbase cluster + * @see #getHBaseClusterInterface() + */ + public MiniHBaseCluster getHBaseCluster() { + return getMiniHBaseCluster(); + } + + /** + * Returns the HBaseCluster instance. + *

    Returned object can be any of the subclasses of HBaseCluster, and the + * tests referring this should not assume that the cluster is a mini cluster or a + * distributed one. If the test only works on a mini cluster, then specific + * method {@link #getMiniHBaseCluster()} can be used instead w/o the + * need to type-cast. + */ + public HBaseCluster getHBaseClusterInterface() { + //implementation note: we should rename this method as #getHBaseCluster(), + //but this would require refactoring 90+ calls. + return hbaseCluster; + } + + /** + * Returns a HBaseAdmin instance. + * This instance is shared between HBaseTestingUtility intance users. + * Don't close it, it will be closed automatically when the + * cluster shutdowns + * + * @return The HBaseAdmin instance. + * @throws IOException + */ + public synchronized HBaseAdmin getHBaseAdmin() + throws IOException { + if (hbaseAdmin == null){ + hbaseAdmin = new HBaseAdmin(new Configuration(getConfiguration())); + } + return hbaseAdmin; + } + private HBaseAdmin hbaseAdmin = null; + + /** + * Closes the named region. + * + * @param regionName The region to close. + * @throws IOException + */ + public void closeRegion(String regionName) throws IOException { + closeRegion(Bytes.toBytes(regionName)); + } + + /** + * Closes the named region. + * + * @param regionName The region to close. + * @throws IOException + */ + public void closeRegion(byte[] regionName) throws IOException { + getHBaseAdmin().closeRegion(regionName, null); + } + + /** + * Closes the region containing the given row. + * + * @param row The row to find the containing region. + * @param table The table to find the region. + * @throws IOException + */ + public void closeRegionByRow(String row, HTable table) throws IOException { + closeRegionByRow(Bytes.toBytes(row), table); + } + + /** + * Closes the region containing the given row. + * + * @param row The row to find the containing region. + * @param table The table to find the region. + * @throws IOException + */ + public void closeRegionByRow(byte[] row, HTable table) throws IOException { + HRegionLocation hrl = table.getRegionLocation(row); + closeRegion(hrl.getRegionInfo().getRegionName()); + } + + public MiniZooKeeperCluster getZkCluster() { + return zkCluster; + } + + public void setZkCluster(MiniZooKeeperCluster zkCluster) { + this.passedZkCluster = true; + this.zkCluster = zkCluster; + conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkCluster.getClientPort()); + } + + public MiniDFSCluster getDFSCluster() { + return dfsCluster; + } + + public void setDFSCluster(MiniDFSCluster cluster) throws IOException { + if (dfsCluster != null && dfsCluster.isClusterUp()) { + throw new IOException("DFSCluster is already running! Shut it down first."); + } + this.dfsCluster = cluster; + } + + public FileSystem getTestFileSystem() throws IOException { + return HFileSystem.get(conf); + } + + /** + * @return True if we removed the test dir + * @throws IOException + */ + public boolean cleanupTestDir() throws IOException { + if (dataTestDir == null ){ + return false; + } else { + boolean ret = deleteDir(getDataTestDir()); + dataTestDir = null; + return ret; + } + } + + /** + * @param subdir Test subdir name. + * @return True if we removed the test dir + * @throws IOException + */ + public boolean cleanupTestDir(final String subdir) throws IOException { + if (dataTestDir == null){ + return false; + } + return deleteDir(getDataTestDir(subdir)); + } + + /** + * @param dir Directory to delete + * @return True if we deleted it. + * @throws IOException + */ + public boolean deleteDir(final Path dir) throws IOException { + FileSystem fs = getTestFileSystem(); + if (fs.exists(dir)) { + return fs.delete(getDataTestDir(), true); + } + return false; + } + + public void waitTableAvailable(byte[] table, long timeoutMillis) + throws InterruptedException, IOException { + long startWait = System.currentTimeMillis(); + while (!getHBaseAdmin().isTableAvailable(table)) { + assertTrue("Timed out waiting for table to become available " + + Bytes.toStringBinary(table), + System.currentTimeMillis() - startWait < timeoutMillis); + Thread.sleep(200); + } + } + + public void waitTableEnabled(byte[] table, long timeoutMillis) + throws InterruptedException, IOException { + long startWait = System.currentTimeMillis(); + while (!getHBaseAdmin().isTableAvailable(table) && + !getHBaseAdmin().isTableEnabled(table)) { + assertTrue("Timed out waiting for table to become available and enabled " + + Bytes.toStringBinary(table), + System.currentTimeMillis() - startWait < timeoutMillis); + Thread.sleep(200); + } + } + + /** + * Make sure that at least the specified number of region servers + * are running + * @param num minimum number of region servers that should be running + * @return true if we started some servers + * @throws IOException + */ + public boolean ensureSomeRegionServersAvailable(final int num) + throws IOException { + boolean startedServer = false; + MiniHBaseCluster hbaseCluster = getMiniHBaseCluster(); + for (int i=hbaseCluster.getLiveRegionServerThreads().size(); i + * 2010-06-15 11:52:28,511 WARN [DataStreamer for file /hbase/.logs/hlog.1276627923013 block blk_928005470262850423_1021] hdfs.DFSClient$DFSOutputStream(2657): Error Recovery for block blk_928005470262850423_1021 failed because recovery from primary datanode 127.0.0.1:53683 failed 4 times. Pipeline was 127.0.0.1:53687, 127.0.0.1:53683. Will retry... + * + * @param stream A DFSClient.DFSOutputStream. + * @param max + * @throws NoSuchFieldException + * @throws SecurityException + * @throws IllegalAccessException + * @throws IllegalArgumentException + */ + public static void setMaxRecoveryErrorCount(final OutputStream stream, + final int max) { + try { + Class [] clazzes = DFSClient.class.getDeclaredClasses(); + for (Class clazz: clazzes) { + String className = clazz.getSimpleName(); + if (className.equals("DFSOutputStream")) { + if (clazz.isInstance(stream)) { + Field maxRecoveryErrorCountField = + stream.getClass().getDeclaredField("maxRecoveryErrorCount"); + maxRecoveryErrorCountField.setAccessible(true); + maxRecoveryErrorCountField.setInt(stream, max); + break; + } + } + } + } catch (Exception e) { + LOG.info("Could not set max recovery field", e); + } + } + + /** + * Wait until all regions for a table in .META. have a non-empty + * info:server, up to 60 seconds. This means all regions have been deployed, + * master has been informed and updated .META. with the regions deployed + * server. + * @param tableName the table name + * @throws IOException + */ + public void waitUntilAllRegionsAssigned(final byte[] tableName) throws IOException { + waitUntilAllRegionsAssigned(tableName, 60000); + } + + /** + * Wait until all regions for a table in .META. have a non-empty + * info:server, or until timeout. This means all regions have been + * deployed, master has been informed and updated .META. with the regions + * deployed server. + * @param tableName the table name + * @param timeout timeout, in milliseconds + * @throws IOException + */ + public void waitUntilAllRegionsAssigned(final byte[] tableName, final long timeout) + throws IOException { + long deadline = System.currentTimeMillis() + timeout; + HTable meta = new HTable(getConfiguration(), HConstants.META_TABLE_NAME); + try { + while (true) { + boolean allRegionsAssigned = true; + Scan scan = new Scan(); + scan.addFamily(HConstants.CATALOG_FAMILY); + ResultScanner s = meta.getScanner(scan); + try { + Result r; + while ((r = s.next()) != null) { + byte [] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + HRegionInfo info = Writables.getHRegionInfoOrNull(b); + if (info != null && Bytes.equals(info.getTableName(), tableName)) { + b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); + allRegionsAssigned &= (b != null); + } + } + } finally { + s.close(); + } + if (allRegionsAssigned) { + return; + } + long now = System.currentTimeMillis(); + if (now > deadline) { + throw new IOException("Timeout waiting for all regions of " + + Bytes.toStringBinary(tableName) + " to be assigned"); + } + try { + Thread.sleep(deadline - now < 200 ? deadline - now : 200); + } catch (InterruptedException e) { + throw new IOException(e); + } + } + } finally { + meta.close(); + } + } + + /** + * Do a small get/scan against one store. This is required because store + * has no actual methods of querying itself, and relies on StoreScanner. + */ + public static List getFromStoreFile(Store store, + Get get) throws IOException { + MultiVersionConsistencyControl.resetThreadReadPoint(); + Scan scan = new Scan(get); + InternalScanner scanner = (InternalScanner) store.getScanner(scan, + scan.getFamilyMap().get(store.getFamily().getName())); + + List result = new ArrayList(); + scanner.next(result); + if (!result.isEmpty()) { + // verify that we are on the row we want: + KeyValue kv = result.get(0); + if (!Bytes.equals(kv.getRow(), get.getRow())) { + result.clear(); + } + } + scanner.close(); + return result; + } + + /** + * Do a small get/scan against one store. This is required because store + * has no actual methods of querying itself, and relies on StoreScanner. + */ + public static List getFromStoreFile(Store store, + byte [] row, + NavigableSet columns + ) throws IOException { + Get get = new Get(row); + Map> s = get.getFamilyMap(); + s.put(store.getFamily().getName(), columns); + + return getFromStoreFile(store,get); + } + + /** + * Gets a ZooKeeperWatcher. + * @param TEST_UTIL + */ + public static ZooKeeperWatcher getZooKeeperWatcher( + HBaseTestingUtility TEST_UTIL) throws ZooKeeperConnectionException, + IOException { + ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + "unittest", new Abortable() { + boolean aborted = false; + + @Override + public void abort(String why, Throwable e) { + aborted = true; + throw new RuntimeException("Fatal ZK error, why=" + why, e); + } + + @Override + public boolean isAborted() { + return aborted; + } + }); + return zkw; + } + + /** + * Creates a znode with OPENED state. + * @param TEST_UTIL + * @param region + * @param serverName + * @return + * @throws IOException + * @throws ZooKeeperConnectionException + * @throws KeeperException + * @throws NodeExistsException + */ + public static ZooKeeperWatcher createAndForceNodeToOpenedState( + HBaseTestingUtility TEST_UTIL, HRegion region, + ServerName serverName) throws ZooKeeperConnectionException, + IOException, KeeperException, NodeExistsException { + ZooKeeperWatcher zkw = getZooKeeperWatcher(TEST_UTIL); + ZKAssign.createNodeOffline(zkw, region.getRegionInfo(), serverName); + int version = ZKAssign.transitionNodeOpening(zkw, region + .getRegionInfo(), serverName); + ZKAssign.transitionNodeOpened(zkw, region.getRegionInfo(), serverName, + version); + return zkw; + } + + public static void assertKVListsEqual(String additionalMsg, + final List expected, + final List actual) { + final int eLen = expected.size(); + final int aLen = actual.size(); + final int minLen = Math.min(eLen, aLen); + + int i; + for (i = 0; i < minLen + && KeyValue.COMPARATOR.compare(expected.get(i), actual.get(i)) == 0; + ++i) {} + + if (additionalMsg == null) { + additionalMsg = ""; + } + if (!additionalMsg.isEmpty()) { + additionalMsg = ". " + additionalMsg; + } + + if (eLen != aLen || i != minLen) { + throw new AssertionError( + "Expected and actual KV arrays differ at position " + i + ": " + + safeGetAsStr(expected, i) + " (length " + eLen +") vs. " + + safeGetAsStr(actual, i) + " (length " + aLen + ")" + additionalMsg); + } + } + + private static String safeGetAsStr(List lst, int i) { + if (0 <= i && i < lst.size()) { + return lst.get(i).toString(); + } else { + return ""; + } + } + + public String getClusterKey() { + return conf.get(HConstants.ZOOKEEPER_QUORUM) + ":" + + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":" + + conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, + HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); + } + + /** Creates a random table with the given parameters */ + public HTable createRandomTable(String tableName, + final Collection families, + final int maxVersions, + final int numColsPerRow, + final int numFlushes, + final int numRegions, + final int numRowsPerFlush) + throws IOException, InterruptedException { + + LOG.info("\n\nCreating random table " + tableName + " with " + numRegions + + " regions, " + numFlushes + " storefiles per region, " + + numRowsPerFlush + " rows per flush, maxVersions=" + maxVersions + + "\n"); + + final Random rand = new Random(tableName.hashCode() * 17L + 12938197137L); + final int numCF = families.size(); + final byte[][] cfBytes = new byte[numCF][]; + final byte[] tableNameBytes = Bytes.toBytes(tableName); + + { + int cfIndex = 0; + for (String cf : families) { + cfBytes[cfIndex++] = Bytes.toBytes(cf); + } + } + + final int actualStartKey = 0; + final int actualEndKey = Integer.MAX_VALUE; + final int keysPerRegion = (actualEndKey - actualStartKey) / numRegions; + final int splitStartKey = actualStartKey + keysPerRegion; + final int splitEndKey = actualEndKey - keysPerRegion; + final String keyFormat = "%08x"; + final HTable table = createTable(tableNameBytes, cfBytes, + maxVersions, + Bytes.toBytes(String.format(keyFormat, splitStartKey)), + Bytes.toBytes(String.format(keyFormat, splitEndKey)), + numRegions); + if (hbaseCluster != null) { + getMiniHBaseCluster().flushcache(HConstants.META_TABLE_NAME); + } + + for (int iFlush = 0; iFlush < numFlushes; ++iFlush) { + for (int iRow = 0; iRow < numRowsPerFlush; ++iRow) { + final byte[] row = Bytes.toBytes(String.format(keyFormat, + actualStartKey + rand.nextInt(actualEndKey - actualStartKey))); + + Put put = new Put(row); + Delete del = new Delete(row); + for (int iCol = 0; iCol < numColsPerRow; ++iCol) { + final byte[] cf = cfBytes[rand.nextInt(numCF)]; + final long ts = rand.nextInt(); + final byte[] qual = Bytes.toBytes("col" + iCol); + if (rand.nextBoolean()) { + final byte[] value = Bytes.toBytes("value_for_row_" + iRow + + "_cf_" + Bytes.toStringBinary(cf) + "_col_" + iCol + "_ts_" + + ts + "_random_" + rand.nextLong()); + put.add(cf, qual, ts, value); + } else if (rand.nextDouble() < 0.8) { + del.deleteColumn(cf, qual, ts); + } else { + del.deleteColumns(cf, qual, ts); + } + } + + if (!put.isEmpty()) { + table.put(put); + } + + if (!del.isEmpty()) { + table.delete(del); + } + } + LOG.info("Initiating flush #" + iFlush + " for table " + tableName); + table.flushCommits(); + if (hbaseCluster != null) { + getMiniHBaseCluster().flushcache(tableNameBytes); + } + } + + return table; + } + + private static final int MIN_RANDOM_PORT = 0xc000; + private static final int MAX_RANDOM_PORT = 0xfffe; + + /** + * Returns a random port. These ports cannot be registered with IANA and are + * intended for dynamic allocation (see http://bit.ly/dynports). + */ + public static int randomPort() { + return MIN_RANDOM_PORT + + new Random().nextInt(MAX_RANDOM_PORT - MIN_RANDOM_PORT); + } + + public static int randomFreePort() { + int port = 0; + do { + port = randomPort(); + try { + ServerSocket sock = new ServerSocket(port); + sock.close(); + } catch (IOException ex) { + port = 0; + } + } while (port == 0); + return port; + } + + public static void waitForHostPort(String host, int port) + throws IOException { + final int maxTimeMs = 10000; + final int maxNumAttempts = maxTimeMs / HConstants.SOCKET_RETRY_WAIT_MS; + IOException savedException = null; + LOG.info("Waiting for server at " + host + ":" + port); + for (int attempt = 0; attempt < maxNumAttempts; ++attempt) { + try { + Socket sock = new Socket(InetAddress.getByName(host), port); + sock.close(); + savedException = null; + LOG.info("Server at " + host + ":" + port + " is available"); + break; + } catch (UnknownHostException e) { + throw new IOException("Failed to look up " + host, e); + } catch (IOException e) { + savedException = e; + } + Threads.sleepWithoutInterrupt(HConstants.SOCKET_RETRY_WAIT_MS); + } + + if (savedException != null) { + throw savedException; + } + } + + /** + * Creates a pre-split table for load testing. If the table already exists, + * logs a warning and continues. + * @return the number of regions the table was split into + */ + public static int createPreSplitLoadTestTable(Configuration conf, + byte[] tableName, byte[] columnFamily, Algorithm compression, + DataBlockEncoding dataBlockEncoding) throws IOException { + HTableDescriptor desc = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor(columnFamily); + hcd.setDataBlockEncoding(dataBlockEncoding); + hcd.setCompressionType(compression); + return createPreSplitLoadTestTable(conf, desc, hcd); + } + + /** + * Creates a pre-split table for load testing. If the table already exists, + * logs a warning and continues. + * @return the number of regions the table was split into + */ + public static int createPreSplitLoadTestTable(Configuration conf, + HTableDescriptor desc, HColumnDescriptor hcd) throws IOException { + if (!desc.hasFamily(hcd.getName())) { + desc.addFamily(hcd); + } + + int totalNumberOfRegions = 0; + HBaseAdmin admin = new HBaseAdmin(conf); + try { + // create a table a pre-splits regions. + // The number of splits is set as: + // region servers * regions per region server). + int numberOfServers = admin.getClusterStatus().getServers().size(); + if (numberOfServers == 0) { + throw new IllegalStateException("No live regionservers"); + } + + totalNumberOfRegions = numberOfServers * DEFAULT_REGIONS_PER_SERVER; + LOG.info("Number of live regionservers: " + numberOfServers + ", " + + "pre-splitting table into " + totalNumberOfRegions + " regions " + + "(default regions per server: " + DEFAULT_REGIONS_PER_SERVER + ")"); + + byte[][] splits = new RegionSplitter.HexStringSplit().split( + totalNumberOfRegions); + + admin.createTable(desc, splits); + admin.close(); + } catch (MasterNotRunningException e) { + LOG.error("Master not running", e); + throw new IOException(e); + } catch (TableExistsException e) { + LOG.warn("Table " + Bytes.toStringBinary(desc.getName()) + + " already exists, continuing"); + } finally { + admin.close(); + } + return totalNumberOfRegions; + } + + public static int getMetaRSPort(Configuration conf) throws IOException { + HTable table = new HTable(conf, HConstants.META_TABLE_NAME); + HRegionLocation hloc = table.getRegionLocation(Bytes.toBytes("")); + table.close(); + return hloc.getPort(); + } + + public HRegion createTestRegion(String tableName, HColumnDescriptor hcd) + throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(hcd); + HRegionInfo info = + new HRegionInfo(Bytes.toBytes(tableName), null, null, false); + HRegion region = + HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), htd); + return region; + } + + /** + * Create region split keys between startkey and endKey + * + * @param startKey + * @param endKey + * @param numRegions the number of regions to be created. it has to be greater than 3. + * @return + */ + public byte[][] getRegionSplitStartKeys(byte[] startKey, byte[] endKey, int numRegions){ + assertTrue(numRegions>3); + byte [][] tmpSplitKeys = Bytes.split(startKey, endKey, numRegions - 3); + byte [][] result = new byte[tmpSplitKeys.length+1][]; + for (int i=0;i generateColumnDescriptors() { + return generateColumnDescriptors(""); + } + + /** + * Create a set of column descriptors with the combination of compression, + * encoding, bloom codecs available. + * @param prefix family names prefix + * @return the list of column descriptors + */ + public static List generateColumnDescriptors(final String prefix) { + List htds = new ArrayList(); + long familyId = 0; + for (Compression.Algorithm compressionType: getSupportedCompressionAlgorithms()) { + for (DataBlockEncoding encodingType: DataBlockEncoding.values()) { + for (StoreFile.BloomType bloomType: StoreFile.BloomType.values()) { + String name = String.format("%s-cf-!@#&-%d!@#", prefix, familyId); + HColumnDescriptor htd = new HColumnDescriptor(name); + htd.setCompressionType(compressionType); + htd.setDataBlockEncoding(encodingType); + htd.setBloomFilterType(bloomType); + htds.add(htd); + familyId++; + } + } + } + return htds; + } + + /** + * Get supported compression algorithms. + * @return supported compression algorithms. + */ + public static Compression.Algorithm[] getSupportedCompressionAlgorithms() { + String[] allAlgos = HFile.getSupportedCompressionAlgorithms(); + List supportedAlgos = new ArrayList(); + for (String algoName : allAlgos) { + try { + Compression.Algorithm algo = Compression.getCompressionAlgorithmByName(algoName); + algo.getCompressor(); + supportedAlgos.add(algo); + } catch (Throwable t) { + // this algo is not available + } + } + return supportedAlgos.toArray(new Compression.Algorithm[0]); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java new file mode 100644 index 0000000..a3b7ed9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java @@ -0,0 +1,370 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.math.random.RandomData; +import org.apache.commons.math.random.RandomDataImpl; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.util.Bytes; + +/** + *

    + * This class runs performance benchmarks for {@link HFile}. + *

    + */ +public class HFilePerformanceEvaluation { + + private static final int ROW_LENGTH = 10; + private static final int ROW_COUNT = 1000000; + private static final int RFILE_BLOCKSIZE = 8 * 1024; + + static final Log LOG = + LogFactory.getLog(HFilePerformanceEvaluation.class.getName()); + + static byte [] format(final int i) { + String v = Integer.toString(i); + return Bytes.toBytes("0000000000".substring(v.length()) + v); + } + + static ImmutableBytesWritable format(final int i, ImmutableBytesWritable w) { + w.set(format(i)); + return w; + } + + private void runBenchmarks() throws Exception { + final Configuration conf = new Configuration(); + final FileSystem fs = FileSystem.get(conf); + final Path mf = fs.makeQualified(new Path("performanceevaluation.mapfile")); + if (fs.exists(mf)) { + fs.delete(mf, true); + } + + runBenchmark(new SequentialWriteBenchmark(conf, fs, mf, ROW_COUNT), + ROW_COUNT); + PerformanceEvaluationCommons.concurrentReads(new Runnable() { + public void run() { + try { + runBenchmark(new UniformRandomSmallScan(conf, fs, mf, ROW_COUNT), + ROW_COUNT); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + PerformanceEvaluationCommons.concurrentReads(new Runnable() { + public void run() { + try { + runBenchmark(new UniformRandomReadBenchmark(conf, fs, mf, ROW_COUNT), + ROW_COUNT); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + PerformanceEvaluationCommons.concurrentReads(new Runnable() { + public void run() { + try { + runBenchmark(new GaussianRandomReadBenchmark(conf, fs, mf, ROW_COUNT), + ROW_COUNT); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + PerformanceEvaluationCommons.concurrentReads(new Runnable() { + public void run() { + try { + runBenchmark(new SequentialReadBenchmark(conf, fs, mf, ROW_COUNT), + ROW_COUNT); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + + } + + protected void runBenchmark(RowOrientedBenchmark benchmark, int rowCount) + throws Exception { + LOG.info("Running " + benchmark.getClass().getSimpleName() + " for " + + rowCount + " rows."); + long elapsedTime = benchmark.run(); + LOG.info("Running " + benchmark.getClass().getSimpleName() + " for " + + rowCount + " rows took " + elapsedTime + "ms."); + } + + static abstract class RowOrientedBenchmark { + + protected final Configuration conf; + protected final FileSystem fs; + protected final Path mf; + protected final int totalRows; + + public RowOrientedBenchmark(Configuration conf, FileSystem fs, Path mf, + int totalRows) { + this.conf = conf; + this.fs = fs; + this.mf = mf; + this.totalRows = totalRows; + } + + void setUp() throws Exception { + // do nothing + } + + abstract void doRow(int i) throws Exception; + + protected int getReportingPeriod() { + return this.totalRows / 10; + } + + void tearDown() throws Exception { + // do nothing + } + + /** + * Run benchmark + * @return elapsed time. + * @throws Exception + */ + long run() throws Exception { + long elapsedTime; + setUp(); + long startTime = System.currentTimeMillis(); + try { + for (int i = 0; i < totalRows; i++) { + if (i > 0 && i % getReportingPeriod() == 0) { + LOG.info("Processed " + i + " rows."); + } + doRow(i); + } + elapsedTime = System.currentTimeMillis() - startTime; + } finally { + tearDown(); + } + return elapsedTime; + } + + } + + static class SequentialWriteBenchmark extends RowOrientedBenchmark { + protected HFile.Writer writer; + private Random random = new Random(); + private byte[] bytes = new byte[ROW_LENGTH]; + + public SequentialWriteBenchmark(Configuration conf, FileSystem fs, Path mf, + int totalRows) { + super(conf, fs, mf, totalRows); + } + + @Override + void setUp() throws Exception { + writer = + HFile.getWriterFactoryNoCache(conf) + .withPath(fs, mf) + .withBlockSize(RFILE_BLOCKSIZE) + .create(); + } + + @Override + void doRow(int i) throws Exception { + writer.append(format(i), generateValue()); + } + + private byte[] generateValue() { + random.nextBytes(bytes); + return bytes; + } + + @Override + protected int getReportingPeriod() { + return this.totalRows; // don't report progress + } + + @Override + void tearDown() throws Exception { + writer.close(); + } + + } + + static abstract class ReadBenchmark extends RowOrientedBenchmark { + + protected HFile.Reader reader; + + public ReadBenchmark(Configuration conf, FileSystem fs, Path mf, + int totalRows) { + super(conf, fs, mf, totalRows); + } + + @Override + void setUp() throws Exception { + reader = HFile.createReader(this.fs, this.mf, new CacheConfig(this.conf)); + this.reader.loadFileInfo(); + } + + @Override + void tearDown() throws Exception { + reader.close(); + } + + } + + static class SequentialReadBenchmark extends ReadBenchmark { + private HFileScanner scanner; + + public SequentialReadBenchmark(Configuration conf, FileSystem fs, + Path mf, int totalRows) { + super(conf, fs, mf, totalRows); + } + + @Override + void setUp() throws Exception { + super.setUp(); + this.scanner = this.reader.getScanner(false, false); + this.scanner.seekTo(); + } + + @Override + void doRow(int i) throws Exception { + if (this.scanner.next()) { + ByteBuffer k = this.scanner.getKey(); + PerformanceEvaluationCommons.assertKey(format(i + 1), k); + ByteBuffer v = scanner.getValue(); + PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); + } + } + + @Override + protected int getReportingPeriod() { + return this.totalRows; // don't report progress + } + + } + + static class UniformRandomReadBenchmark extends ReadBenchmark { + + private Random random = new Random(); + + public UniformRandomReadBenchmark(Configuration conf, FileSystem fs, + Path mf, int totalRows) { + super(conf, fs, mf, totalRows); + } + + @Override + void doRow(int i) throws Exception { + HFileScanner scanner = this.reader.getScanner(false, true); + byte [] b = getRandomRow(); + scanner.seekTo(b); + ByteBuffer k = scanner.getKey(); + PerformanceEvaluationCommons.assertKey(b, k); + ByteBuffer v = scanner.getValue(); + PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); + } + + private byte [] getRandomRow() { + return format(random.nextInt(totalRows)); + } + } + + static class UniformRandomSmallScan extends ReadBenchmark { + private Random random = new Random(); + + public UniformRandomSmallScan(Configuration conf, FileSystem fs, + Path mf, int totalRows) { + super(conf, fs, mf, totalRows/10); + } + + @Override + void doRow(int i) throws Exception { + HFileScanner scanner = this.reader.getScanner(false, false); + byte [] b = getRandomRow(); + if (scanner.seekTo(b) != 0) { + System.out.println("Nonexistent row: " + new String(b)); + return; + } + ByteBuffer k = scanner.getKey(); + PerformanceEvaluationCommons.assertKey(b, k); + // System.out.println("Found row: " + new String(b)); + for (int ii = 0; ii < 30; ii++) { + if (!scanner.next()) { + System.out.println("NOTHING FOLLOWS"); + } + ByteBuffer v = scanner.getValue(); + PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); + } + } + + private byte [] getRandomRow() { + return format(random.nextInt(totalRows)); + } + } + + static class GaussianRandomReadBenchmark extends ReadBenchmark { + + private RandomData randomData = new RandomDataImpl(); + + public GaussianRandomReadBenchmark(Configuration conf, FileSystem fs, + Path mf, int totalRows) { + super(conf, fs, mf, totalRows); + } + + @Override + void doRow(int i) throws Exception { + HFileScanner scanner = this.reader.getScanner(false, true); + scanner.seekTo(getGaussianRandomRowBytes()); + for (int ii = 0; ii < 30; ii++) { + if (!scanner.next()) { + System.out.println("NOTHING FOLLOWS"); + } + scanner.getKey(); + scanner.getValue(); + } + } + + private byte [] getGaussianRandomRowBytes() { + int r = (int) randomData.nextGaussian((double)totalRows / 2.0, + (double)totalRows / 10.0); + return format(r); + } + } + + /** + * @param args + * @throws Exception + * @throws IOException + */ + public static void main(String[] args) throws Exception { + new HFilePerformanceEvaluation().runBenchmarks(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/HServerLoad092.java b/src/test/java/org/apache/hadoop/hbase/HServerLoad092.java new file mode 100644 index 0000000..203dcd2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/HServerLoad092.java @@ -0,0 +1,693 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Strings; +import org.apache.hadoop.io.VersionedWritable; +import org.apache.hadoop.io.WritableComparable; + +/** + * This class is used to export current state of load on a RegionServer. + * This is the version of HServerLoad that we had in 0.92. + */ +public class HServerLoad092 extends VersionedWritable +implements WritableComparable { + private static final byte VERSION = 2; + // Empty load instance. + public static final HServerLoad092 EMPTY_HSERVERLOAD = new HServerLoad092(); + + /** Number of requests per second since last report. + */ + // TODO: Instead build this up out of region counters. + private int numberOfRequests = 0; + + /** Total Number of requests from the start of the region server. + */ + private int totalNumberOfRequests = 0; + + /** the amount of used heap, in MB */ + private int usedHeapMB = 0; + + /** the maximum allowable size of the heap, in MB */ + private int maxHeapMB = 0; + + // Regionserver-level coprocessors, e.g., WALObserver implementations. + // Region-level coprocessors, on the other hand, are stored inside RegionLoad + // objects. + private Set coprocessors = + new TreeSet(); + + /** + * HBASE-4070: Improve region server metrics to report loaded coprocessors. + * + * @return Returns the set of all coprocessors on this + * regionserver, where this set is the union of the + * regionserver-level coprocessors on one hand, and all of the region-level + * coprocessors, on the other. + * + * We must iterate through all regions loaded on this regionserver to + * obtain all of the region-level coprocessors. + */ + public String[] getCoprocessors() { + TreeSet returnValue = new TreeSet(coprocessors); + for (Map.Entry rls: getRegionsLoad().entrySet()) { + for (String coprocessor: rls.getValue().getCoprocessors()) { + returnValue.add(coprocessor); + } + } + return returnValue.toArray(new String[0]); + } + + /** per-region load metrics */ + private Map regionLoad = + new TreeMap(Bytes.BYTES_COMPARATOR); + + /** @return the object version number */ + public byte getVersion() { + return VERSION; + } + + /** + * Encapsulates per-region loading metrics. + */ + public static class RegionLoad extends VersionedWritable { + private static final byte VERSION = 1; + + /** @return the object version number */ + public byte getVersion() { + return VERSION; + } + + /** the region name */ + private byte[] name; + /** the number of stores for the region */ + private int stores; + /** the number of storefiles for the region */ + private int storefiles; + /** the total size of the store files for the region, uncompressed, in MB */ + private int storeUncompressedSizeMB; + /** the current total size of the store files for the region, in MB */ + private int storefileSizeMB; + /** the current size of the memstore for the region, in MB */ + private int memstoreSizeMB; + + /** + * The current total size of root-level store file indexes for the region, + * in MB. The same as {@link #rootIndexSizeKB} but in MB. + */ + private int storefileIndexSizeMB; + /** the current total read requests made to region */ + private int readRequestsCount; + /** the current total write requests made to region */ + private int writeRequestsCount; + /** the total compacting key values in currently running compaction */ + private long totalCompactingKVs; + /** the completed count of key values in currently running compaction */ + private long currentCompactedKVs; + + /** The current total size of root-level indexes for the region, in KB. */ + private int rootIndexSizeKB; + + /** The total size of all index blocks, not just the root level, in KB. */ + private int totalStaticIndexSizeKB; + + /** + * The total size of all Bloom filter blocks, not just loaded into the + * block cache, in KB. + */ + private int totalStaticBloomSizeKB; + + // Region-level coprocessors. + Set coprocessors = + new TreeSet(); + + /** + * Constructor, for Writable + */ + public RegionLoad() { + super(); + } + + /** + * @param name + * @param stores + * @param storefiles + * @param storeUncompressedSizeMB + * @param storefileSizeMB + * @param memstoreSizeMB + * @param storefileIndexSizeMB + * @param readRequestsCount + * @param writeRequestsCount + * @param totalCompactingKVs + * @param currentCompactedKVs + * @param coprocessors + */ + public RegionLoad(final byte[] name, final int stores, + final int storefiles, final int storeUncompressedSizeMB, + final int storefileSizeMB, + final int memstoreSizeMB, final int storefileIndexSizeMB, + final int rootIndexSizeKB, final int totalStaticIndexSizeKB, + final int totalStaticBloomSizeKB, + final int readRequestsCount, final int writeRequestsCount, + final long totalCompactingKVs, final long currentCompactedKVs, + final Set coprocessors) { + this.name = name; + this.stores = stores; + this.storefiles = storefiles; + this.storeUncompressedSizeMB = storeUncompressedSizeMB; + this.storefileSizeMB = storefileSizeMB; + this.memstoreSizeMB = memstoreSizeMB; + this.storefileIndexSizeMB = storefileIndexSizeMB; + this.rootIndexSizeKB = rootIndexSizeKB; + this.totalStaticIndexSizeKB = totalStaticIndexSizeKB; + this.totalStaticBloomSizeKB = totalStaticBloomSizeKB; + this.readRequestsCount = readRequestsCount; + this.writeRequestsCount = writeRequestsCount; + this.totalCompactingKVs = totalCompactingKVs; + this.currentCompactedKVs = currentCompactedKVs; + this.coprocessors = coprocessors; + } + + // Getters + private String[] getCoprocessors() { + return coprocessors.toArray(new String[0]); + } + + /** + * @return the region name + */ + public byte[] getName() { + return name; + } + + /** + * @return the region name as a string + */ + public String getNameAsString() { + return Bytes.toString(name); + } + + /** + * @return the number of stores + */ + public int getStores() { + return stores; + } + + /** + * @return the number of storefiles + */ + public int getStorefiles() { + return storefiles; + } + + /** + * @return the total size of the storefiles, in MB + */ + public int getStorefileSizeMB() { + return storefileSizeMB; + } + + /** + * @return the memstore size, in MB + */ + public int getMemStoreSizeMB() { + return memstoreSizeMB; + } + + /** + * @return the approximate size of storefile indexes on the heap, in MB + */ + public int getStorefileIndexSizeMB() { + return storefileIndexSizeMB; + } + + /** + * @return the number of requests made to region + */ + public long getRequestsCount() { + return readRequestsCount + writeRequestsCount; + } + + /** + * @return the number of read requests made to region + */ + public long getReadRequestsCount() { + return readRequestsCount; + } + + /** + * @return the number of read requests made to region + */ + public long getWriteRequestsCount() { + return writeRequestsCount; + } + + /** + * @return the total number of kvs in current compaction + */ + public long getTotalCompactingKVs() { + return totalCompactingKVs; + } + + /** + * @return the number of already compacted kvs in current compaction + */ + public long getCurrentCompactedKVs() { + return currentCompactedKVs; + } + + // Setters + + /** + * @param name the region name + */ + public void setName(byte[] name) { + this.name = name; + } + + /** + * @param stores the number of stores + */ + public void setStores(int stores) { + this.stores = stores; + } + + /** + * @param storefiles the number of storefiles + */ + public void setStorefiles(int storefiles) { + this.storefiles = storefiles; + } + + /** + * @param memstoreSizeMB the memstore size, in MB + */ + public void setMemStoreSizeMB(int memstoreSizeMB) { + this.memstoreSizeMB = memstoreSizeMB; + } + + /** + * @param storefileIndexSizeMB the approximate size of storefile indexes + * on the heap, in MB + */ + public void setStorefileIndexSizeMB(int storefileIndexSizeMB) { + this.storefileIndexSizeMB = storefileIndexSizeMB; + } + + /** + * @param requestsCount the number of read requests to region + */ + public void setReadRequestsCount(int requestsCount) { + this.readRequestsCount = requestsCount; + } + + /** + * @param requestsCount the number of write requests to region + */ + public void setWriteRequestsCount(int requestsCount) { + this.writeRequestsCount = requestsCount; + } + + /** + * @param totalCompactingKVs the number of kvs total in current compaction + */ + public void setTotalCompactingKVs(long totalCompactingKVs) { + this.totalCompactingKVs = totalCompactingKVs; + } + + /** + * @param currentCompactedKVs the number of kvs already compacted in + * current compaction + */ + public void setCurrentCompactedKVs(long currentCompactedKVs) { + this.currentCompactedKVs = currentCompactedKVs; + } + + // Writable + public void readFields(DataInput in) throws IOException { + super.readFields(in); + int version = in.readByte(); + if (version > VERSION) throw new IOException("Version mismatch; " + version); + int namelen = in.readInt(); + this.name = new byte[namelen]; + in.readFully(this.name); + this.stores = in.readInt(); + this.storefiles = in.readInt(); + this.storeUncompressedSizeMB = in.readInt(); + this.storefileSizeMB = in.readInt(); + this.memstoreSizeMB = in.readInt(); + this.storefileIndexSizeMB = in.readInt(); + this.readRequestsCount = in.readInt(); + this.writeRequestsCount = in.readInt(); + this.rootIndexSizeKB = in.readInt(); + this.totalStaticIndexSizeKB = in.readInt(); + this.totalStaticBloomSizeKB = in.readInt(); + this.totalCompactingKVs = in.readLong(); + this.currentCompactedKVs = in.readLong(); + int coprocessorsSize = in.readInt(); + coprocessors = new TreeSet(); + for (int i = 0; i < coprocessorsSize; i++) { + coprocessors.add(in.readUTF()); + } + } + + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeByte(VERSION); + out.writeInt(name.length); + out.write(name); + out.writeInt(stores); + out.writeInt(storefiles); + out.writeInt(storeUncompressedSizeMB); + out.writeInt(storefileSizeMB); + out.writeInt(memstoreSizeMB); + out.writeInt(storefileIndexSizeMB); + out.writeInt(readRequestsCount); + out.writeInt(writeRequestsCount); + out.writeInt(rootIndexSizeKB); + out.writeInt(totalStaticIndexSizeKB); + out.writeInt(totalStaticBloomSizeKB); + out.writeLong(totalCompactingKVs); + out.writeLong(currentCompactedKVs); + out.writeInt(coprocessors.size()); + for (String coprocessor: coprocessors) { + out.writeUTF(coprocessor); + } + } + + /** + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "numberOfStores", + Integer.valueOf(this.stores)); + sb = Strings.appendKeyValue(sb, "numberOfStorefiles", + Integer.valueOf(this.storefiles)); + sb = Strings.appendKeyValue(sb, "storefileUncompressedSizeMB", + Integer.valueOf(this.storeUncompressedSizeMB)); + sb = Strings.appendKeyValue(sb, "storefileSizeMB", + Integer.valueOf(this.storefileSizeMB)); + if (this.storeUncompressedSizeMB != 0) { + sb = Strings.appendKeyValue(sb, "compressionRatio", + String.format("%.4f", (float)this.storefileSizeMB/ + (float)this.storeUncompressedSizeMB)); + } + sb = Strings.appendKeyValue(sb, "memstoreSizeMB", + Integer.valueOf(this.memstoreSizeMB)); + sb = Strings.appendKeyValue(sb, "storefileIndexSizeMB", + Integer.valueOf(this.storefileIndexSizeMB)); + sb = Strings.appendKeyValue(sb, "readRequestsCount", + Long.valueOf(this.readRequestsCount)); + sb = Strings.appendKeyValue(sb, "writeRequestsCount", + Long.valueOf(this.writeRequestsCount)); + sb = Strings.appendKeyValue(sb, "rootIndexSizeKB", + Integer.valueOf(this.rootIndexSizeKB)); + sb = Strings.appendKeyValue(sb, "totalStaticIndexSizeKB", + Integer.valueOf(this.totalStaticIndexSizeKB)); + sb = Strings.appendKeyValue(sb, "totalStaticBloomSizeKB", + Integer.valueOf(this.totalStaticBloomSizeKB)); + sb = Strings.appendKeyValue(sb, "totalCompactingKVs", + Long.valueOf(this.totalCompactingKVs)); + sb = Strings.appendKeyValue(sb, "currentCompactedKVs", + Long.valueOf(this.currentCompactedKVs)); + float compactionProgressPct = Float.NaN; + if( this.totalCompactingKVs > 0 ) { + compactionProgressPct = Float.valueOf( + this.currentCompactedKVs / this.totalCompactingKVs); + } + sb = Strings.appendKeyValue(sb, "compactionProgressPct", + compactionProgressPct); + String coprocessors = Arrays.toString(getCoprocessors()); + if (coprocessors != null) { + sb = Strings.appendKeyValue(sb, "coprocessors", + Arrays.toString(getCoprocessors())); + } + return sb.toString(); + } + } + + /* + * TODO: Other metrics that might be considered when the master is actually + * doing load balancing instead of merely trying to decide where to assign + * a region: + *
      + *
    • # of CPUs, heap size (to determine the "class" of machine). For + * now, we consider them to be homogeneous.
      • + *
    • #requests per region (Map<{String|HRegionInfo}, Integer>)
      • + *
    • #compactions and/or #splits (churn)
      • + *
    • server death rate (maybe there is something wrong with this server)
      • + *
    + */ + + /** default constructor (used by Writable) */ + public HServerLoad092() { + super(); + } + + /** + * Constructor + * @param numberOfRequests + * @param usedHeapMB + * @param maxHeapMB + * @param coprocessors : coprocessors loaded at the regionserver-level + */ + public HServerLoad092(final int totalNumberOfRequests, + final int numberOfRequests, final int usedHeapMB, final int maxHeapMB, + final Map regionLoad, + final Set coprocessors) { + this.numberOfRequests = numberOfRequests; + this.usedHeapMB = usedHeapMB; + this.maxHeapMB = maxHeapMB; + this.regionLoad = regionLoad; + this.totalNumberOfRequests = totalNumberOfRequests; + this.coprocessors = coprocessors; + } + + /** + * Constructor + * @param hsl the template HServerLoad + */ + public HServerLoad092(final HServerLoad092 hsl) { + this(hsl.totalNumberOfRequests, hsl.numberOfRequests, hsl.usedHeapMB, + hsl.maxHeapMB, hsl.getRegionsLoad(), hsl.coprocessors); + for (Map.Entry e : hsl.regionLoad.entrySet()) { + this.regionLoad.put(e.getKey(), e.getValue()); + } + } + + /** + * Originally, this method factored in the effect of requests going to the + * server as well. However, this does not interact very well with the current + * region rebalancing code, which only factors number of regions. For the + * interim, until we can figure out how to make rebalancing use all the info + * available, we're just going to make load purely the number of regions. + * + * @return load factor for this server + */ + public int getLoad() { + // int load = numberOfRequests == 0 ? 1 : numberOfRequests; + // load *= numberOfRegions == 0 ? 1 : numberOfRegions; + // return load; + return this.regionLoad.size(); + } + + /** + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return toString(1); + } + + /** + * Returns toString() with the number of requests divided by the message + * interval in seconds + * @param msgInterval + * @return The load as a String + */ + public String toString(int msgInterval) { + int numberOfRegions = this.regionLoad.size(); + StringBuilder sb = new StringBuilder(); + sb = Strings.appendKeyValue(sb, "requestsPerSecond", + Integer.valueOf(numberOfRequests/msgInterval)); + sb = Strings.appendKeyValue(sb, "numberOfOnlineRegions", + Integer.valueOf(numberOfRegions)); + sb = Strings.appendKeyValue(sb, "usedHeapMB", + Integer.valueOf(this.usedHeapMB)); + sb = Strings.appendKeyValue(sb, "maxHeapMB", Integer.valueOf(maxHeapMB)); + return sb.toString(); + } + + /** + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null) { + return false; + } + if (getClass() != o.getClass()) { + return false; + } + return compareTo((HServerLoad092)o) == 0; + } + + // Getters + + /** + * @return the numberOfRegions + */ + public int getNumberOfRegions() { + return this.regionLoad.size(); + } + + /** + * @return the numberOfRequests per second. + */ + public int getNumberOfRequests() { + return numberOfRequests; + } + + /** + * @return the numberOfRequests + */ + public int getTotalNumberOfRequests() { + return totalNumberOfRequests; + } + + /** + * @return the amount of heap in use, in MB + */ + public int getUsedHeapMB() { + return usedHeapMB; + } + + /** + * @return the maximum allowable heap size, in MB + */ + public int getMaxHeapMB() { + return maxHeapMB; + } + + /** + * @return region load metrics + */ + public Map getRegionsLoad() { + return Collections.unmodifiableMap(regionLoad); + } + + /** + * @return Count of storefiles on this regionserver + */ + public int getStorefiles() { + int count = 0; + for (RegionLoad info: regionLoad.values()) + count += info.getStorefiles(); + return count; + } + + /** + * @return Total size of store files in MB + */ + public int getStorefileSizeInMB() { + int count = 0; + for (RegionLoad info: regionLoad.values()) + count += info.getStorefileSizeMB(); + return count; + } + + /** + * @return Size of memstores in MB + */ + public int getMemStoreSizeInMB() { + int count = 0; + for (RegionLoad info: regionLoad.values()) + count += info.getMemStoreSizeMB(); + return count; + } + + /** + * @return Size of store file indexes in MB + */ + public int getStorefileIndexSizeInMB() { + int count = 0; + for (RegionLoad info: regionLoad.values()) + count += info.getStorefileIndexSizeMB(); + return count; + } + + // Writable + + public void readFields(DataInput in) throws IOException { + super.readFields(in); + int version = in.readByte(); + if (version > VERSION) throw new IOException("Version mismatch; " + version); + numberOfRequests = in.readInt(); + usedHeapMB = in.readInt(); + maxHeapMB = in.readInt(); + int numberOfRegions = in.readInt(); + for (int i = 0; i < numberOfRegions; i++) { + RegionLoad rl = new RegionLoad(); + rl.readFields(in); + regionLoad.put(rl.getName(), rl); + } + totalNumberOfRequests = in.readInt(); + int coprocessorsSize = in.readInt(); + for(int i = 0; i < coprocessorsSize; i++) { + coprocessors.add(in.readUTF()); + } + } + + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeByte(VERSION); + out.writeInt(numberOfRequests); + out.writeInt(usedHeapMB); + out.writeInt(maxHeapMB); + out.writeInt(this.regionLoad.size()); + for (RegionLoad rl: regionLoad.values()) + rl.write(out); + out.writeInt(totalNumberOfRequests); + out.writeInt(coprocessors.size()); + for (String coprocessor: coprocessors) { + out.writeUTF(coprocessor); + } + } + + // Comparable + + public int compareTo(HServerLoad092 o) { + return this.getLoad() - o.getLoad(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java b/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java new file mode 100644 index 0000000..9c603ae --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import junit.framework.Assert; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.LoadTestTool; + +/** + * A base class for tests that do something with the cluster while running + * {@link LoadTestTool} to write and verify some data. + */ +public abstract class IngestIntegrationTestBase { + private static String tableName = null; + + /** A soft limit on how long we should run */ + private static final String RUN_TIME_KEY = "hbase.%s.runtime"; + + protected static final Log LOG = LogFactory.getLog(IngestIntegrationTestBase.class); + protected IntegrationTestingUtility util; + protected HBaseCluster cluster; + private LoadTestTool loadTool; + + protected void setUp(int numSlavesBase) throws Exception { + tableName = this.getClass().getSimpleName(); + util = new IntegrationTestingUtility(); + LOG.info("Initializing cluster with " + numSlavesBase + " servers"); + util.initializeCluster(numSlavesBase); + LOG.info("Done initializing cluster"); + cluster = util.getHBaseClusterInterface(); + deleteTableIfNecessary(); + loadTool = new LoadTestTool(); + loadTool.setConf(util.getConfiguration()); + // Initialize load test tool before we start breaking things; + // LoadTestTool init, even when it is a no-op, is very fragile. + int ret = loadTool.run(new String[] { "-tn", tableName, "-init_only" }); + Assert.assertEquals("Failed to initialize LoadTestTool", 0, ret); + } + + protected void tearDown() throws Exception { + LOG.info("Restoring the cluster"); + util.restoreCluster(); + LOG.info("Done restoring the cluster"); + } + + private void deleteTableIfNecessary() throws IOException { + if (util.getHBaseAdmin().tableExists(tableName)) { + util.deleteTable(Bytes.toBytes(tableName)); + } + } + + protected void runIngestTest(long defaultRunTime, int keysPerServerPerIter, + int colsPerKey, int recordSize, int writeThreads) throws Exception { + LOG.info("Running ingest"); + LOG.info("Cluster size:" + util.getHBaseClusterInterface().getClusterStatus().getServersSize()); + + long start = System.currentTimeMillis(); + String runtimeKey = String.format(RUN_TIME_KEY, this.getClass().getSimpleName()); + long runtime = util.getConfiguration().getLong(runtimeKey, defaultRunTime); + long startKey = 0; + + long numKeys = getNumKeys(keysPerServerPerIter); + while (System.currentTimeMillis() - start < 0.9 * runtime) { + LOG.info("Intended run time: " + (runtime/60000) + " min, left:" + + ((runtime - (System.currentTimeMillis() - start))/60000) + " min"); + + int ret = loadTool.run(new String[] { + "-tn", tableName, + "-write", String.format("%d:%d:%d", colsPerKey, recordSize, writeThreads), + "-start_key", String.valueOf(startKey), + "-num_keys", String.valueOf(numKeys), + "-skip_init" + }); + if (0 != ret) { + String errorMsg = "Load failed with error code " + ret; + LOG.error(errorMsg); + Assert.fail(errorMsg); + } + + ret = loadTool.run(new String[] { + "-tn", tableName, + "-read", "100:20", + "-start_key", String.valueOf(startKey), + "-num_keys", String.valueOf(numKeys), + "-skip_init" + }); + if (0 != ret) { + String errorMsg = "Verification failed with error code " + ret; + LOG.error(errorMsg); + Assert.fail(errorMsg); + } + startKey += numKeys; + } + } + + /** Estimates a data size based on the cluster size */ + private long getNumKeys(int keysPerServer) + throws IOException { + int numRegionServers = cluster.getClusterStatus().getServersSize(); + return keysPerServer * numRegionServers; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java new file mode 100644 index 0000000..57415a1 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.util.ChaosMonkey; +import org.apache.hadoop.hbase.util.ChaosMonkey.BatchRestartRs; +import org.apache.hadoop.hbase.util.ChaosMonkey.RestartActiveMaster; +import org.apache.hadoop.hbase.util.ChaosMonkey.RestartRandomRs; +import org.apache.hadoop.hbase.util.ChaosMonkey.RestartRsHoldingMeta; +import org.apache.hadoop.hbase.util.ChaosMonkey.RestartRsHoldingRoot; +import org.apache.hadoop.hbase.util.ChaosMonkey.RollingBatchRestartRs; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * A system test which does large data ingestion and verify using {@link LoadTestTool}. + * It performs a set of actions deterministically using ChaosMonkey, then starts killing + * things randomly. You can configure how long should the load test run by using + * "hbase.IntegrationTestDataIngestSlowDeterministic.runtime" configuration parameter. + */ +@Category(IntegrationTests.class) +public class IntegrationTestDataIngestSlowDeterministic extends IngestIntegrationTestBase { + private static final int SERVER_COUNT = 3; // number of slaves for the smallest cluster + private static final long DEFAULT_RUN_TIME = 30 * 60 * 1000; + private static final long CHAOS_EVERY_MS = 150 * 1000; // Chaos every 2.5 minutes. + + private ChaosMonkey monkey; + + @Before + public void setUp() throws Exception { + super.setUp(SERVER_COUNT); + ChaosMonkey.Action[] actions = new ChaosMonkey.Action[] { + new RestartRandomRs(60000), + new BatchRestartRs(5000, 0.5f), + new RestartActiveMaster(5000), + new RollingBatchRestartRs(5000, 1.0f), + new RestartRsHoldingMeta(35000), + new RestartRsHoldingRoot(35000) + }; + monkey = new ChaosMonkey(util, new ChaosMonkey.CompositeSequentialPolicy( + new ChaosMonkey.DoActionsOncePolicy(CHAOS_EVERY_MS, actions), + new ChaosMonkey.PeriodicRandomActionPolicy(CHAOS_EVERY_MS, actions))); + monkey.start(); + } + + @After + public void tearDown() throws Exception { + if (monkey != null) { + monkey.stop("tearDown"); + monkey.waitForStop(); + } + super.tearDown(); + } + + @Test + public void testDataIngest() throws Exception { + runIngestTest(DEFAULT_RUN_TIME, 2500, 10, 100, 5); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java new file mode 100644 index 0000000..72891b1 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.util.ChaosMonkey; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * A system test which does large data ingestion and verify using {@link LoadTestTool}, + * while killing the region servers and the master(s) randomly. You can configure how long + * should the load test run by using "hbase.IntegrationTestDataIngestWithChaosMonkey.runtime" + * configuration parameter. + */ +@Category(IntegrationTests.class) +public class IntegrationTestDataIngestWithChaosMonkey extends IngestIntegrationTestBase { + + private static final int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster + + // run for 5 min by default + private static final long DEFAULT_RUN_TIME = 5 * 60 * 1000; + + private ChaosMonkey monkey; + + @Before + public void setUp() throws Exception { + super.setUp(NUM_SLAVES_BASE); + monkey = new ChaosMonkey(util, ChaosMonkey.EVERY_MINUTE_RANDOM_ACTION_POLICY); + monkey.start(); + } + + @After + public void tearDown() throws Exception { + if (monkey != null) { + monkey.stop("test has finished, that's why"); + monkey.waitForStop(); + } + super.tearDown(); + } + + @Test + public void testDataIngest() throws Exception { + runIngestTest(DEFAULT_RUN_TIME, 2500, 10, 100, 20); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestingUtility.java new file mode 100644 index 0000000..45cbc01 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestingUtility.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; + +import java.io.IOException; + +/** + * Facility for integration/system tests. This extends {@link HBaseTestingUtility} + * and adds-in the functionality needed by integration and system tests. This class understands + * distributed and pseudo-distributed/local cluster deployments, and abstracts those from the tests + * in this module. + *

    + * IntegrationTestingUtility is constructed and used by the integration tests, but the tests + * themselves should not assume a particular deployment. They can rely on the methods in this + * class and HBaseCluster. Before the testing begins, the test should initialize the cluster by + * calling {@link #initializeCluster(int)}. + *

    + * The cluster that is used defaults to a mini cluster, but it can be forced to use a distributed + * cluster by calling {@link #setUseDistributedCluster(Configuration)}. This method is invoked by + * test drivers (maven, IntegrationTestsDriver, etc) before initializing the cluster + * via {@link #initializeCluster(int)}. Individual tests should not directly call + * {@link #setUseDistributedCluster(Configuration)}. + */ +public class IntegrationTestingUtility extends HBaseTestingUtility { + + public IntegrationTestingUtility() { + this(HBaseConfiguration.create()); + } + + public IntegrationTestingUtility(Configuration conf) { + super(conf); + } + + /** + * Configuration that controls whether this utility assumes a running/deployed cluster. + * This is different than "hbase.cluster.distributed" since that parameter indicates whether the + * cluster is in an actual distributed environment, while this shows that there is a + * deployed (distributed or pseudo-distributed) cluster running, and we do not need to + * start a mini-cluster for tests. + */ + public static final String IS_DISTRIBUTED_CLUSTER = "hbase.test.cluster.distributed"; + + /** + * Initializes the state of the cluster. It starts a new in-process mini cluster, OR + * if we are given an already deployed distributed cluster it initializes the state. + * @param numSlaves Number of slaves to start up if we are booting a mini cluster. Otherwise + * we check whether this many nodes are available and throw an exception if not. + */ + public void initializeCluster(int numSlaves) throws Exception { + if (isDistributedCluster()) { + createDistributedHBaseCluster(); + checkNodeCount(numSlaves); + } else { + startMiniCluster(numSlaves); + } + } + + /** + * Checks whether we have more than numSlaves nodes. Throws an + * exception otherwise. + */ + public void checkNodeCount(int numSlaves) throws Exception { + HBaseCluster cluster = getHBaseClusterInterface(); + if (cluster.getClusterStatus().getServers().size() < numSlaves) { + throw new Exception("Cluster does not have enough nodes:" + numSlaves); + } + } + + /** + * Restores the cluster to the initial state if it is a distributed cluster, otherwise, shutdowns the + * mini cluster. + */ + public void restoreCluster() throws IOException { + if (isDistributedCluster()) { + getHBaseClusterInterface().restoreInitialStatus(); + } else { + getMiniHBaseCluster().shutdown(); + } + } + + /** + * Sets the configuration property to use a distributed cluster for the integration tests. Test drivers + * should use this to enforce cluster deployment. + */ + public static void setUseDistributedCluster(Configuration conf) { + conf.setBoolean(IS_DISTRIBUTED_CLUSTER, true); + System.setProperty(IS_DISTRIBUTED_CLUSTER, "true"); + } + + /** + * @return whether we are interacting with a distributed cluster as opposed to and in-process mini + * cluster or a local cluster. + * @see IntegrationTestingUtility#setUseDistributedCluster(Configuration) + */ + private boolean isDistributedCluster() { + Configuration conf = getConfiguration(); + boolean isDistributedCluster = false; + isDistributedCluster = Boolean.parseBoolean(System.getProperty(IS_DISTRIBUTED_CLUSTER, "false")); + if (!isDistributedCluster) { + isDistributedCluster = conf.getBoolean(IS_DISTRIBUTED_CLUSTER, false); + } + return isDistributedCluster; + } + + private void createDistributedHBaseCluster() throws IOException { + Configuration conf = getConfiguration(); + Class clusterManagerClass = conf.getClass( + HConstants.HBASE_CLUSTER_MANAGER_CLASS, HBaseClusterManager.class, + ClusterManager.class); + ClusterManager clusterManager = ReflectionUtils.newInstance( + clusterManagerClass, conf); + setHBaseCluster(new DistributedHBaseCluster(conf, clusterManager)); + getHBaseAdmin(); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTests.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTests.java new file mode 100644 index 0000000..d429e24 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTests.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +/** + * Tag a test as 'integration/system' test, meaning that the test class has the following + * characteristics:

      + *
    • Possibly takes hours to complete
      • + *
    • Can be run on a mini cluster or an actual cluster
      • + *
    • Can make changes to the given cluster (starting stopping daemons, etc)
      • + *
    • Should not be run in parallel of other integration tests
      • + *
    + * + * Integration / System tests should have a class name starting with "IntegrationTest", and + * should be annotated with @Category(IntegrationTests.class). Integration tests can be run + * using the IntegrationTestsDriver class or from mvn verify. + * + * @see SmallTests + * @see MediumTests + * @see LargeTests + */ +public interface IntegrationTests { +} diff --git a/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java b/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java new file mode 100644 index 0000000..a6f3760 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +import org.apache.commons.cli.CommandLine; +import org.apache.hadoop.hbase.util.AbstractHBaseTool; +import org.apache.hadoop.util.ToolRunner; +import org.junit.internal.TextListener; +import org.junit.runner.JUnitCore; +import org.junit.runner.Result; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * This class drives the Integration test suite execution. Executes all + * tests having @Category(IntegrationTests.class) annotation against an + * already deployed distributed cluster. + */ +public class IntegrationTestsDriver extends AbstractHBaseTool { + private static final String TESTS_ARG = "test"; + private static final Log LOG = LogFactory.getLog(IntegrationTestsDriver.class); + private IntegrationTestFilter intTestFilter = new IntegrationTestFilter(); + + public static void main(String[] args) throws Exception { + int ret = ToolRunner.run(new IntegrationTestsDriver(), args); + System.exit(ret); + } + + private class IntegrationTestFilter extends ClassTestFinder.TestClassFilter { + private Pattern testFilterRe = Pattern.compile(".*"); + public IntegrationTestFilter() { + super(IntegrationTests.class); + } + + public void setPattern(String pattern) { + testFilterRe = Pattern.compile(pattern); + } + + @Override + public boolean isCandidateClass(Class c) { + return super.isCandidateClass(c) && testFilterRe.matcher(c.getName()).find(); + } + } + + @Override + protected void addOptions() { + addOptWithArg(TESTS_ARG, "a Java regular expression to filter tests on"); + } + + @Override + protected void processOptions(CommandLine cmd) { + String testFilterString = cmd.getOptionValue(TESTS_ARG, null); + if (testFilterString != null) { + intTestFilter.setPattern(testFilterString); + } + } + + /** + * Returns test classes annotated with @Category(IntegrationTests.class), + * according to the filter specific on the command line (if any). + */ + private Class[] findIntegrationTestClasses() + throws ClassNotFoundException, LinkageError, IOException { + ClassTestFinder.TestFileNameFilter nameFilter = new ClassTestFinder.TestFileNameFilter(); + ClassFinder classFinder = new ClassFinder(nameFilter, intTestFilter); + Set> classes = classFinder.findClasses(true); + return classes.toArray(new Class[classes.size()]); + } + + + @Override + protected int doWork() throws Exception { + //this is called from the command line, so we should set to use the distributed cluster + IntegrationTestingUtility.setUseDistributedCluster(conf); + Class[] classes = findIntegrationTestClasses(); + LOG.info("Found " + classes.length + " integration tests to run"); + + JUnitCore junit = new JUnitCore(); + junit.addListener(new TextListener(System.out)); + Result result = junit.run(classes); + + return result.wasSuccessful() ? 0 : 1; + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/KeyValueTestUtil.java b/src/test/java/org/apache/hadoop/hbase/KeyValueTestUtil.java new file mode 100644 index 0000000..36d768a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/KeyValueTestUtil.java @@ -0,0 +1,54 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.util.Bytes; + +public class KeyValueTestUtil { + + public static KeyValue create( + String row, + String family, + String qualifier, + long timestamp, + String value) + { + return create(row, family, qualifier, timestamp, KeyValue.Type.Put, value); + } + + public static KeyValue create( + String row, + String family, + String qualifier, + long timestamp, + KeyValue.Type type, + String value) + { + return new KeyValue( + Bytes.toBytes(row), + Bytes.toBytes(family), + Bytes.toBytes(qualifier), + timestamp, + type, + Bytes.toBytes(value) + ); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/LargeTests.java b/src/test/java/org/apache/hadoop/hbase/LargeTests.java new file mode 100644 index 0000000..da9abda --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/LargeTests.java @@ -0,0 +1,39 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +/** + * Tag a test as 'large', meaning that the test class has the following + * characteristics: + * - executed in an isolated JVM. Tests can however be executed in different + * JVM on the same machine simultaneously. + * - will not have to be executed by the developer before submitting a bug + * - ideally, last less than 2 minutes to help parallelization + * + * It the worst case compared to small or medium, use it only for tests that + * you cannot put in the other categories + * + * @see SmallTests + * @see MediumTests + * @see IntegrationTests + */ +public interface LargeTests { +} diff --git a/src/test/java/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java new file mode 100644 index 0000000..7635949 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java @@ -0,0 +1,349 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.math.random.RandomData; +import org.apache.commons.math.random.RandomDataImpl; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.io.MapFile; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableComparable; + +/** + *

    + * This class runs performance benchmarks for {@link MapFile}. + *

    + */ +public class MapFilePerformanceEvaluation { + protected final Configuration conf; + private static final int ROW_LENGTH = 10; + private static final int ROW_COUNT = 100000; + + static final Log LOG = + LogFactory.getLog(MapFilePerformanceEvaluation.class.getName()); + + /** + * @param c + */ + public MapFilePerformanceEvaluation(final Configuration c) { + super(); + this.conf = c; + } + + static ImmutableBytesWritable format(final int i, ImmutableBytesWritable w) { + String v = Integer.toString(i); + w.set(Bytes.toBytes("0000000000".substring(v.length()) + v)); + return w; + } + + private void runBenchmarks() throws Exception { + final FileSystem fs = FileSystem.get(this.conf); + final Path mf = fs.makeQualified(new Path("performanceevaluation.mapfile")); + if (fs.exists(mf)) { + fs.delete(mf, true); + } + runBenchmark(new SequentialWriteBenchmark(conf, fs, mf, ROW_COUNT), + ROW_COUNT); + + PerformanceEvaluationCommons.concurrentReads(new Runnable() { + public void run() { + try { + runBenchmark(new UniformRandomSmallScan(conf, fs, mf, ROW_COUNT), + ROW_COUNT); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + PerformanceEvaluationCommons.concurrentReads(new Runnable() { + public void run() { + try { + runBenchmark(new UniformRandomReadBenchmark(conf, fs, mf, ROW_COUNT), + ROW_COUNT); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + PerformanceEvaluationCommons.concurrentReads(new Runnable() { + public void run() { + try { + runBenchmark(new GaussianRandomReadBenchmark(conf, fs, mf, ROW_COUNT), + ROW_COUNT); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + PerformanceEvaluationCommons.concurrentReads(new Runnable() { + public void run() { + try { + runBenchmark(new SequentialReadBenchmark(conf, fs, mf, ROW_COUNT), + ROW_COUNT); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + } + + protected void runBenchmark(RowOrientedBenchmark benchmark, int rowCount) + throws Exception { + LOG.info("Running " + benchmark.getClass().getSimpleName() + " for " + + rowCount + " rows."); + long elapsedTime = benchmark.run(); + LOG.info("Running " + benchmark.getClass().getSimpleName() + " for " + + rowCount + " rows took " + elapsedTime + "ms."); + } + + static abstract class RowOrientedBenchmark { + + protected final Configuration conf; + protected final FileSystem fs; + protected final Path mf; + protected final int totalRows; + + public RowOrientedBenchmark(Configuration conf, FileSystem fs, Path mf, + int totalRows) { + this.conf = conf; + this.fs = fs; + this.mf = mf; + this.totalRows = totalRows; + } + + void setUp() throws Exception { + // do nothing + } + + abstract void doRow(int i) throws Exception; + + protected int getReportingPeriod() { + return this.totalRows / 10; + } + + void tearDown() throws Exception { + // do nothing + } + + /** + * Run benchmark + * @return elapsed time. + * @throws Exception + */ + long run() throws Exception { + long elapsedTime; + setUp(); + long startTime = System.currentTimeMillis(); + try { + for (int i = 0; i < totalRows; i++) { + if (i > 0 && i % getReportingPeriod() == 0) { + LOG.info("Processed " + i + " rows."); + } + doRow(i); + } + elapsedTime = System.currentTimeMillis() - startTime; + } finally { + tearDown(); + } + return elapsedTime; + } + + } + + static class SequentialWriteBenchmark extends RowOrientedBenchmark { + + protected MapFile.Writer writer; + private Random random = new Random(); + private byte[] bytes = new byte[ROW_LENGTH]; + private ImmutableBytesWritable key = new ImmutableBytesWritable(); + private ImmutableBytesWritable value = new ImmutableBytesWritable(); + + public SequentialWriteBenchmark(Configuration conf, FileSystem fs, Path mf, + int totalRows) { + super(conf, fs, mf, totalRows); + } + + @Override + void setUp() throws Exception { + writer = new MapFile.Writer(conf, fs, mf.toString(), + ImmutableBytesWritable.class, ImmutableBytesWritable.class); + } + + @Override + void doRow(int i) throws Exception { + value.set(generateValue()); + writer.append(format(i, key), value); + } + + private byte[] generateValue() { + random.nextBytes(bytes); + return bytes; + } + + @Override + protected int getReportingPeriod() { + return this.totalRows; // don't report progress + } + + @Override + void tearDown() throws Exception { + writer.close(); + } + + } + + static abstract class ReadBenchmark extends RowOrientedBenchmark { + ImmutableBytesWritable key = new ImmutableBytesWritable(); + ImmutableBytesWritable value = new ImmutableBytesWritable(); + + protected MapFile.Reader reader; + + public ReadBenchmark(Configuration conf, FileSystem fs, Path mf, + int totalRows) { + super(conf, fs, mf, totalRows); + } + + @Override + void setUp() throws Exception { + reader = new MapFile.Reader(fs, mf.toString(), conf); + } + + @Override + void tearDown() throws Exception { + reader.close(); + } + + } + + static class SequentialReadBenchmark extends ReadBenchmark { + ImmutableBytesWritable verify = new ImmutableBytesWritable(); + + public SequentialReadBenchmark(Configuration conf, FileSystem fs, + Path mf, int totalRows) { + super(conf, fs, mf, totalRows); + } + + @Override + void doRow(int i) throws Exception { + this.reader.next(key, value); + PerformanceEvaluationCommons.assertKey(this.key.get(), + format(i, this.verify).get()); + PerformanceEvaluationCommons.assertValueSize(ROW_LENGTH, value.getSize()); + } + + @Override + protected int getReportingPeriod() { + return this.totalRows; // don't report progress + } + + } + + static class UniformRandomReadBenchmark extends ReadBenchmark { + + private Random random = new Random(); + + public UniformRandomReadBenchmark(Configuration conf, FileSystem fs, + Path mf, int totalRows) { + super(conf, fs, mf, totalRows); + } + + @Override + void doRow(int i) throws Exception { + ImmutableBytesWritable k = getRandomRow(); + ImmutableBytesWritable r = (ImmutableBytesWritable)reader.get(k, value); + PerformanceEvaluationCommons.assertValueSize(r.getSize(), ROW_LENGTH); + } + + private ImmutableBytesWritable getRandomRow() { + return format(random.nextInt(totalRows), key); + } + + } + + static class UniformRandomSmallScan extends ReadBenchmark { + private Random random = new Random(); + + public UniformRandomSmallScan(Configuration conf, FileSystem fs, + Path mf, int totalRows) { + super(conf, fs, mf, totalRows/10); + } + + @Override + void doRow(int i) throws Exception { + ImmutableBytesWritable ibw = getRandomRow(); + WritableComparable wc = this.reader.getClosest(ibw, this.value); + if (wc == null) { + throw new NullPointerException(); + } + PerformanceEvaluationCommons.assertKey(ibw.get(), + ((ImmutableBytesWritable)wc).get()); + // TODO: Verify we're getting right values. + for (int ii = 0; ii < 29; ii++) { + this.reader.next(this.key, this.value); + PerformanceEvaluationCommons.assertValueSize(this.value.getSize(), ROW_LENGTH); + } + } + + private ImmutableBytesWritable getRandomRow() { + return format(random.nextInt(totalRows), key); + } + } + + static class GaussianRandomReadBenchmark extends ReadBenchmark { + private RandomData randomData = new RandomDataImpl(); + + public GaussianRandomReadBenchmark(Configuration conf, FileSystem fs, + Path mf, int totalRows) { + super(conf, fs, mf, totalRows); + } + + @Override + void doRow(int i) throws Exception { + ImmutableBytesWritable k = getGaussianRandomRow(); + ImmutableBytesWritable r = (ImmutableBytesWritable)reader.get(k, value); + PerformanceEvaluationCommons.assertValueSize(r.getSize(), ROW_LENGTH); + } + + private ImmutableBytesWritable getGaussianRandomRow() { + int r = (int) randomData.nextGaussian((double)totalRows / 2.0, + (double)totalRows / 10.0); + return format(r, key); + } + + } + + /** + * @param args + * @throws Exception + * @throws IOException + */ + public static void main(String[] args) throws Exception { + new MapFilePerformanceEvaluation(HBaseConfiguration.create()). + runBenchmarks(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/MediumTests.java b/src/test/java/org/apache/hadoop/hbase/MediumTests.java new file mode 100644 index 0000000..24a44b9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/MediumTests.java @@ -0,0 +1,38 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +/** + * Tag a test as 'Medium', meaning that the test class has the following + * characteristics: + * - executed in an isolated JVM. Tests can however be executed in different + * JVM on the same machine simultaneously. + * - will have to be executed by the developer before submitting a bug + * - ideally, last less than 1 minutes to help parallelization + * + * Use it for tests that cannot be tagged as 'Small'. + * + * @see SmallTests + * @see LargeTests + * @see IntegrationTests + */ +public interface MediumTests { +} diff --git a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java new file mode 100644 index 0000000..109d94e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java @@ -0,0 +1,680 @@ +/** + * Copyright 2008 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.ipc.HMasterInterface; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.io.MapWritable; + +/** + * This class creates a single process HBase cluster. + * each server. The master uses the 'default' FileSystem. The RegionServers, + * if we are running on DistributedFilesystem, create a FileSystem instance + * each and will close down their instance on the way out. + */ +public class MiniHBaseCluster extends HBaseCluster { + static final Log LOG = LogFactory.getLog(MiniHBaseCluster.class.getName()); + public LocalHBaseCluster hbaseCluster; + private static int index; + + /** + * Start a MiniHBaseCluster. + * @param conf Configuration to be used for cluster + * @param numRegionServers initial number of region servers to start. + * @throws IOException + */ + public MiniHBaseCluster(Configuration conf, int numRegionServers) + throws IOException, InterruptedException { + this(conf, 1, numRegionServers); + } + + /** + * Start a MiniHBaseCluster. + * @param conf Configuration to be used for cluster + * @param numMasters initial number of masters to start. + * @param numRegionServers initial number of region servers to start. + * @throws IOException + */ + public MiniHBaseCluster(Configuration conf, int numMasters, + int numRegionServers) + throws IOException, InterruptedException { + this(conf, numMasters, numRegionServers, null, null); + } + + public MiniHBaseCluster(Configuration conf, int numMasters, int numRegionServers, + Class masterClass, + Class regionserverClass) + throws IOException, InterruptedException { + super(conf); + conf.set(HConstants.MASTER_PORT, "0"); + init(numMasters, numRegionServers, masterClass, regionserverClass); + this.initialClusterStatus = getClusterStatus(); + } + + public Configuration getConfiguration() { + return this.conf; + } + + /** + * Subclass so can get at protected methods (none at moment). Also, creates + * a FileSystem instance per instantiation. Adds a shutdown own FileSystem + * on the way out. Shuts down own Filesystem only, not All filesystems as + * the FileSystem system exit hook does. + */ + public static class MiniHBaseClusterRegionServer extends HRegionServer { + private Thread shutdownThread = null; + private User user = null; + public static boolean TEST_SKIP_CLOSE = false; + + public MiniHBaseClusterRegionServer(Configuration conf) + throws IOException, InterruptedException { + super(conf); + this.user = User.getCurrent(); + } + + /* + * @param c + * @param currentfs We return this if we did not make a new one. + * @param uniqueName Same name used to help identify the created fs. + * @return A new fs instance if we are up on DistributeFileSystem. + * @throws IOException + */ + + @Override + protected void handleReportForDutyResponse(MapWritable c) throws IOException { + super.handleReportForDutyResponse(c); + // Run this thread to shutdown our filesystem on way out. + this.shutdownThread = new SingleFileSystemShutdownThread(getFileSystem()); + } + + @Override + public void run() { + try { + this.user.runAs(new PrivilegedAction(){ + public Object run() { + runRegionServer(); + return null; + } + }); + } catch (Throwable t) { + LOG.error("Exception in run", t); + } finally { + // Run this on the way out. + if (this.shutdownThread != null) { + this.shutdownThread.start(); + Threads.shutdown(this.shutdownThread, 30000); + } + } + } + + private void runRegionServer() { + super.run(); + } + + @Override + public void kill() { + super.kill(); + } + + public void abort(final String reason, final Throwable cause) { + this.user.runAs(new PrivilegedAction() { + public Object run() { + abortRegionServer(reason, cause); + return null; + } + }); + } + + private void abortRegionServer(String reason, Throwable cause) { + super.abort(reason, cause); + } + } + + /** + * Alternate shutdown hook. + * Just shuts down the passed fs, not all as default filesystem hook does. + */ + static class SingleFileSystemShutdownThread extends Thread { + private final FileSystem fs; + SingleFileSystemShutdownThread(final FileSystem fs) { + super("Shutdown of " + fs); + this.fs = fs; + } + @Override + public void run() { + try { + LOG.info("Hook closing fs=" + this.fs); + this.fs.close(); + } catch (NullPointerException npe) { + LOG.debug("Need to fix these: " + npe.toString()); + } catch (IOException e) { + LOG.warn("Running hook", e); + } + } + } + + private void init(final int nMasterNodes, final int nRegionNodes, + Class masterClass, + Class regionserverClass) + throws IOException, InterruptedException { + try { + if (masterClass == null){ + masterClass = HMaster.class; + } + if (regionserverClass == null){ + regionserverClass = MiniHBaseCluster.MiniHBaseClusterRegionServer.class; + } + + // start up a LocalHBaseCluster + hbaseCluster = new LocalHBaseCluster(conf, nMasterNodes, 0, + masterClass, regionserverClass); + + // manually add the regionservers as other users + for (int i=0; i mts; + long start = System.currentTimeMillis(); + while (!(mts = getMasterThreads()).isEmpty() + && (System.currentTimeMillis() - start) < timeout) { + for (JVMClusterUtil.MasterThread mt : mts) { + ServerManager serverManager = mt.getMaster().getServerManager(); + if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized() + && !serverManager.areDeadServersInProgress()) { + return true; + } + } + + Threads.sleep(100); + } + return false; + } + + /** + * @return List of master threads. + */ + public List getMasterThreads() { + return this.hbaseCluster.getMasters(); + } + + /** + * @return List of live master threads (skips the aborted and the killed) + */ + public List getLiveMasterThreads() { + return this.hbaseCluster.getLiveMasters(); + } + + /** + * Wait for Mini HBase Cluster to shut down. + */ + public void join() { + this.hbaseCluster.join(); + } + + /** + * Shut down the mini HBase cluster + * @throws IOException + */ + public void shutdown() throws IOException { + if (this.hbaseCluster != null) { + this.hbaseCluster.shutdown(); + } + HConnectionManager.deleteAllConnections(); + } + + @Override + public void close() throws IOException { + } + + @Override + public ClusterStatus getClusterStatus() throws IOException { + HMaster master = getMaster(); + return master == null ? null : master.getClusterStatus(); + } + + /** + * Call flushCache on all regions on all participating regionservers. + * @throws IOException + */ + public void flushcache() throws IOException { + for (JVMClusterUtil.RegionServerThread t: + this.hbaseCluster.getRegionServers()) { + for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) { + r.flushcache(); + } + } + } + + /** + * Call flushCache on all regions of the specified table. + * @throws IOException + */ + public void flushcache(byte [] tableName) throws IOException { + for (JVMClusterUtil.RegionServerThread t: + this.hbaseCluster.getRegionServers()) { + for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) { + if(Bytes.equals(r.getTableDesc().getName(), tableName)) { + r.flushcache(); + } + } + } + } + + /** + * Call flushCache on all regions on all participating regionservers. + * @throws IOException + */ + public void compact(boolean major) throws IOException { + for (JVMClusterUtil.RegionServerThread t: + this.hbaseCluster.getRegionServers()) { + for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) { + r.compactStores(major); + } + } + } + + /** + * Call flushCache on all regions of the specified table. + * @throws IOException + */ + public void compact(byte [] tableName, boolean major) throws IOException { + for (JVMClusterUtil.RegionServerThread t: + this.hbaseCluster.getRegionServers()) { + for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) { + if(Bytes.equals(r.getTableDesc().getName(), tableName)) { + r.compactStores(major); + } + } + } + } + + /** + * @return List of region server threads. + */ + public List getRegionServerThreads() { + return this.hbaseCluster.getRegionServers(); + } + + /** + * @return List of live region server threads (skips the aborted and the killed) + */ + public List getLiveRegionServerThreads() { + return this.hbaseCluster.getLiveRegionServers(); + } + + /** + * Grab a numbered region server of your choice. + * @param serverNumber + * @return region server + */ + public HRegionServer getRegionServer(int serverNumber) { + return hbaseCluster.getRegionServer(serverNumber); + } + + public List getRegions(byte[] tableName) { + List ret = new ArrayList(); + for (JVMClusterUtil.RegionServerThread rst : getRegionServerThreads()) { + HRegionServer hrs = rst.getRegionServer(); + for (HRegion region : hrs.getOnlineRegionsLocalContext()) { + if (Bytes.equals(region.getTableDesc().getName(), tableName)) { + ret.add(region); + } + } + } + return ret; + } + + /** + * @return Index into List of {@link MiniHBaseCluster#getRegionServerThreads()} + * of HRS carrying regionName. Returns -1 if none found. + */ + public int getServerWithMeta() { + return getServerWith(HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); + } + + /** + * Get the location of the specified region + * @param regionName Name of the region in bytes + * @return Index into List of {@link MiniHBaseCluster#getRegionServerThreads()} + * of HRS carrying .META.. Returns -1 if none found. + */ + public int getServerWith(byte[] regionName) { + int index = -1; + int count = 0; + for (JVMClusterUtil.RegionServerThread rst: getRegionServerThreads()) { + HRegionServer hrs = rst.getRegionServer(); + HRegion metaRegion = + hrs.getOnlineRegion(regionName); + if (metaRegion != null) { + index = count; + break; + } + count++; + } + return index; + } + + @Override + public ServerName getServerHoldingRegion(byte[] regionName) throws IOException { + int index = getServerWith(regionName); + if (index < 0) { + return null; + } + return getRegionServer(index).getServerName(); + } + + /** + * Counts the total numbers of regions being served by the currently online + * region servers by asking each how many regions they have. Does not look + * at META at all. Count includes catalog tables. + * @return number of regions being served by all region servers + */ + public long countServedRegions() { + long count = 0; + for (JVMClusterUtil.RegionServerThread rst : getLiveRegionServerThreads()) { + count += rst.getRegionServer().getNumberOfOnlineRegions(); + } + return count; + } + + @Override + public void waitUntilShutDown() { + this.hbaseCluster.join(); + } + + protected int getRegionServerIndex(ServerName serverName) { + //we have a small number of region servers, this should be fine for now. + List servers = getRegionServerThreads(); + for (int i=0; i < servers.size(); i++) { + if (servers.get(i).getRegionServer().getServerName().equals(serverName)) { + return i; + } + } + return -1; + } + + protected int getMasterIndex(ServerName serverName) { + List masters = getMasterThreads(); + for (int i = 0; i < masters.size(); i++) { + if (masters.get(i).getMaster().getServerName().equals(serverName)) { + return i; + } + } + return -1; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java b/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java new file mode 100644 index 0000000..f9a00dd --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java @@ -0,0 +1,151 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.Set; +import java.util.HashSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; + +public abstract class MultithreadedTestUtil { + + public static final Log LOG = + LogFactory.getLog(MultithreadedTestUtil.class); + + public static class TestContext { + private final Configuration conf; + private Throwable err = null; + private boolean stopped = false; + private int threadDoneCount = 0; + private Set testThreads = new HashSet(); + + public TestContext(Configuration configuration) { + this.conf = configuration; + } + + protected Configuration getConf() { + return conf; + } + + public synchronized boolean shouldRun() { + return !stopped && err == null; + } + + public void addThread(TestThread t) { + testThreads.add(t); + } + + public void startThreads() { + for (TestThread t : testThreads) { + t.start(); + } + } + + public void waitFor(long millis) throws Exception { + long endTime = System.currentTimeMillis() + millis; + while (!stopped) { + long left = endTime - System.currentTimeMillis(); + if (left <= 0) break; + synchronized (this) { + checkException(); + wait(left); + } + } + } + private synchronized void checkException() throws Exception { + if (err != null) { + throw new RuntimeException("Deferred", err); + } + } + + public synchronized void threadFailed(Throwable t) { + if (err == null) err = t; + LOG.error("Failed!", err); + notify(); + } + + public synchronized void threadDone() { + threadDoneCount++; + } + + public void setStopFlag(boolean s) throws Exception { + synchronized (this) { + stopped = s; + } + } + + public void stop() throws Exception { + synchronized (this) { + stopped = true; + } + for (TestThread t : testThreads) { + t.join(); + } + checkException(); + } + } + + /** + * A thread that can be added to a test context, and properly + * passes exceptions through. + */ + public static abstract class TestThread extends Thread { + protected final TestContext ctx; + protected boolean stopped; + + public TestThread(TestContext ctx) { + this.ctx = ctx; + } + + public void run() { + try { + doWork(); + } catch (Throwable t) { + ctx.threadFailed(t); + } + ctx.threadDone(); + } + + public abstract void doWork() throws Exception; + + protected void stopTestThread() { + this.stopped = true; + } + } + + /** + * A test thread that performs a repeating operation. + */ + public static abstract class RepeatingTestThread extends TestThread { + public RepeatingTestThread(TestContext ctx) { + super(ctx); + } + + public final void doWork() throws Exception { + while (ctx.shouldRun() && !stopped) { + doAnAction(); + } + } + + public abstract void doAnAction() throws Exception; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java new file mode 100644 index 0000000..16c7653 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -0,0 +1,1337 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.PrintStream; +import java.io.File; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; +import java.util.Arrays; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.lang.reflect.Constructor; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.filter.WhileMatchFilter; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.Hash; +import org.apache.hadoop.hbase.util.MurmurHash; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer; +import org.apache.hadoop.util.LineReader; + +/** + * Script used evaluating HBase performance and scalability. Runs a HBase + * client that steps through one of a set of hardcoded tests or 'experiments' + * (e.g. a random reads test, a random writes test, etc.). Pass on the + * command-line which test to run and how many clients are participating in + * this experiment. Run java PerformanceEvaluation --help to + * obtain usage. + * + *

    This class sets up and runs the evaluation programs described in + * Section 7, Performance Evaluation, of the Bigtable + * paper, pages 8-10. + * + *

    If number of clients > 1, we start up a MapReduce job. Each map task + * runs an individual client. Each client does about 1GB of data. + */ +public class PerformanceEvaluation { + protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName()); + + private static final int ROW_LENGTH = 1000; + private static final int ONE_GB = 1024 * 1024 * 1000; + private static final int ROWS_PER_GB = ONE_GB / ROW_LENGTH; + + public static final byte[] TABLE_NAME = Bytes.toBytes("TestTable"); + public static final byte[] FAMILY_NAME = Bytes.toBytes("info"); + public static final byte[] QUALIFIER_NAME = Bytes.toBytes("data"); + + protected static final HTableDescriptor TABLE_DESCRIPTOR; + static { + TABLE_DESCRIPTOR = new HTableDescriptor(TABLE_NAME); + TABLE_DESCRIPTOR.addFamily(new HColumnDescriptor(FAMILY_NAME)); + } + + protected Map commands = new TreeMap(); + + volatile Configuration conf; + private boolean miniCluster = false; + private boolean nomapred = false; + private int N = 1; + private int R = ROWS_PER_GB; + private boolean flushCommits = true; + private boolean writeToWAL = true; + private int presplitRegions = 0; + + private static final Path PERF_EVAL_DIR = new Path("performance_evaluation"); + /** + * Regex to parse lines in input file passed to mapreduce task. + */ + public static final Pattern LINE_PATTERN = + Pattern.compile("startRow=(\\d+),\\s+" + + "perClientRunRows=(\\d+),\\s+" + + "totalRows=(\\d+),\\s+" + + "clients=(\\d+),\\s+" + + "flushCommits=(\\w+),\\s+" + + "writeToWAL=(\\w+)"); + + /** + * Enum for map metrics. Keep it out here rather than inside in the Map + * inner-class so we can find associated properties. + */ + protected static enum Counter { + /** elapsed time */ + ELAPSED_TIME, + /** number of rows */ + ROWS} + + + /** + * Constructor + * @param c Configuration object + */ + public PerformanceEvaluation(final Configuration c) { + this.conf = c; + + addCommandDescriptor(RandomReadTest.class, "randomRead", + "Run random read test"); + addCommandDescriptor(RandomSeekScanTest.class, "randomSeekScan", + "Run random seek and scan 100 test"); + addCommandDescriptor(RandomScanWithRange10Test.class, "scanRange10", + "Run random seek scan with both start and stop row (max 10 rows)"); + addCommandDescriptor(RandomScanWithRange100Test.class, "scanRange100", + "Run random seek scan with both start and stop row (max 100 rows)"); + addCommandDescriptor(RandomScanWithRange1000Test.class, "scanRange1000", + "Run random seek scan with both start and stop row (max 1000 rows)"); + addCommandDescriptor(RandomScanWithRange10000Test.class, "scanRange10000", + "Run random seek scan with both start and stop row (max 10000 rows)"); + addCommandDescriptor(RandomWriteTest.class, "randomWrite", + "Run random write test"); + addCommandDescriptor(SequentialReadTest.class, "sequentialRead", + "Run sequential read test"); + addCommandDescriptor(SequentialWriteTest.class, "sequentialWrite", + "Run sequential write test"); + addCommandDescriptor(ScanTest.class, "scan", + "Run scan test (read every row)"); + addCommandDescriptor(FilteredScanTest.class, "filterScan", + "Run scan test using a filter to find a specific row based on it's value (make sure to use --rows=20)"); + } + + protected void addCommandDescriptor(Class cmdClass, + String name, String description) { + CmdDescriptor cmdDescriptor = + new CmdDescriptor(cmdClass, name, description); + commands.put(name, cmdDescriptor); + } + + /** + * Implementations can have their status set. + */ + static interface Status { + /** + * Sets status + * @param msg status message + * @throws IOException + */ + void setStatus(final String msg) throws IOException; + } + + /** + * This class works as the InputSplit of Performance Evaluation + * MapReduce InputFormat, and the Record Value of RecordReader. + * Each map task will only read one record from a PeInputSplit, + * the record value is the PeInputSplit itself. + */ + public static class PeInputSplit extends InputSplit implements Writable { + private int startRow = 0; + private int rows = 0; + private int totalRows = 0; + private int clients = 0; + private boolean flushCommits = false; + private boolean writeToWAL = true; + + public PeInputSplit() { + this.startRow = 0; + this.rows = 0; + this.totalRows = 0; + this.clients = 0; + this.flushCommits = false; + this.writeToWAL = true; + } + + public PeInputSplit(int startRow, int rows, int totalRows, int clients, + boolean flushCommits, boolean writeToWAL) { + this.startRow = startRow; + this.rows = rows; + this.totalRows = totalRows; + this.clients = clients; + this.flushCommits = flushCommits; + this.writeToWAL = writeToWAL; + } + + @Override + public void readFields(DataInput in) throws IOException { + this.startRow = in.readInt(); + this.rows = in.readInt(); + this.totalRows = in.readInt(); + this.clients = in.readInt(); + this.flushCommits = in.readBoolean(); + this.writeToWAL = in.readBoolean(); + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(startRow); + out.writeInt(rows); + out.writeInt(totalRows); + out.writeInt(clients); + out.writeBoolean(flushCommits); + out.writeBoolean(writeToWAL); + } + + @Override + public long getLength() throws IOException, InterruptedException { + return 0; + } + + @Override + public String[] getLocations() throws IOException, InterruptedException { + return new String[0]; + } + + public int getStartRow() { + return startRow; + } + + public int getRows() { + return rows; + } + + public int getTotalRows() { + return totalRows; + } + + public int getClients() { + return clients; + } + + public boolean isFlushCommits() { + return flushCommits; + } + + public boolean isWriteToWAL() { + return writeToWAL; + } + } + + /** + * InputFormat of Performance Evaluation MapReduce job. + * It extends from FileInputFormat, want to use it's methods such as setInputPaths(). + */ + public static class PeInputFormat extends FileInputFormat { + + @Override + public List getSplits(JobContext job) throws IOException { + // generate splits + List splitList = new ArrayList(); + + for (FileStatus file: listStatus(job)) { + Path path = file.getPath(); + FileSystem fs = path.getFileSystem(job.getConfiguration()); + FSDataInputStream fileIn = fs.open(path); + LineReader in = new LineReader(fileIn, job.getConfiguration()); + int lineLen = 0; + while(true) { + Text lineText = new Text(); + lineLen = in.readLine(lineText); + if(lineLen <= 0) { + break; + } + Matcher m = LINE_PATTERN.matcher(lineText.toString()); + if((m != null) && m.matches()) { + int startRow = Integer.parseInt(m.group(1)); + int rows = Integer.parseInt(m.group(2)); + int totalRows = Integer.parseInt(m.group(3)); + int clients = Integer.parseInt(m.group(4)); + boolean flushCommits = Boolean.parseBoolean(m.group(5)); + boolean writeToWAL = Boolean.parseBoolean(m.group(6)); + + LOG.debug("split["+ splitList.size() + "] " + + " startRow=" + startRow + + " rows=" + rows + + " totalRows=" + totalRows + + " clients=" + clients + + " flushCommits=" + flushCommits + + " writeToWAL=" + writeToWAL); + + PeInputSplit newSplit = + new PeInputSplit(startRow, rows, totalRows, clients, + flushCommits, writeToWAL); + splitList.add(newSplit); + } + } + in.close(); + } + + LOG.info("Total # of splits: " + splitList.size()); + return splitList; + } + + @Override + public RecordReader createRecordReader(InputSplit split, + TaskAttemptContext context) { + return new PeRecordReader(); + } + + public static class PeRecordReader extends RecordReader { + private boolean readOver = false; + private PeInputSplit split = null; + private NullWritable key = null; + private PeInputSplit value = null; + + @Override + public void initialize(InputSplit split, TaskAttemptContext context) + throws IOException, InterruptedException { + this.readOver = false; + this.split = (PeInputSplit)split; + } + + @Override + public boolean nextKeyValue() throws IOException, InterruptedException { + if(readOver) { + return false; + } + + key = NullWritable.get(); + value = (PeInputSplit)split; + + readOver = true; + return true; + } + + @Override + public NullWritable getCurrentKey() throws IOException, InterruptedException { + return key; + } + + @Override + public PeInputSplit getCurrentValue() throws IOException, InterruptedException { + return value; + } + + @Override + public float getProgress() throws IOException, InterruptedException { + if(readOver) { + return 1.0f; + } else { + return 0.0f; + } + } + + @Override + public void close() throws IOException { + // do nothing + } + } + } + + /** + * MapReduce job that runs a performance evaluation client in each map task. + */ + public static class EvaluationMapTask + extends Mapper { + + /** configuration parameter name that contains the command */ + public final static String CMD_KEY = "EvaluationMapTask.command"; + /** configuration parameter name that contains the PE impl */ + public static final String PE_KEY = "EvaluationMapTask.performanceEvalImpl"; + + private Class cmd; + private PerformanceEvaluation pe; + + @Override + protected void setup(Context context) throws IOException, InterruptedException { + this.cmd = forName(context.getConfiguration().get(CMD_KEY), Test.class); + + // this is required so that extensions of PE are instantiated within the + // map reduce task... + Class peClass = + forName(context.getConfiguration().get(PE_KEY), PerformanceEvaluation.class); + try { + this.pe = peClass.getConstructor(Configuration.class) + .newInstance(context.getConfiguration()); + } catch (Exception e) { + throw new IllegalStateException("Could not instantiate PE instance", e); + } + } + + private Class forName(String className, Class type) { + Class clazz = null; + try { + clazz = Class.forName(className).asSubclass(type); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Could not find class for name: " + className, e); + } + return clazz; + } + + protected void map(NullWritable key, PeInputSplit value, final Context context) + throws IOException, InterruptedException { + + Status status = new Status() { + public void setStatus(String msg) { + context.setStatus(msg); + } + }; + + // Evaluation task + long elapsedTime = this.pe.runOneClient(this.cmd, value.getStartRow(), + value.getRows(), value.getTotalRows(), + value.isFlushCommits(), value.isWriteToWAL(), + status); + // Collect how much time the thing took. Report as map output and + // to the ELAPSED_TIME counter. + context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime); + context.getCounter(Counter.ROWS).increment(value.rows); + context.write(new LongWritable(value.startRow), new LongWritable(elapsedTime)); + context.progress(); + } + } + + /* + * If table does not already exist, create. + * @param c Client to use checking. + * @return True if we created the table. + * @throws IOException + */ + private boolean checkTable(HBaseAdmin admin) throws IOException { + HTableDescriptor tableDescriptor = getTableDescriptor(); + if (this.presplitRegions > 0) { + // presplit requested + if (admin.tableExists(tableDescriptor.getName())) { + admin.disableTable(tableDescriptor.getName()); + admin.deleteTable(tableDescriptor.getName()); + } + + byte[][] splits = getSplits(); + for (int i=0; i < splits.length; i++) { + LOG.debug(" split " + i + ": " + Bytes.toStringBinary(splits[i])); + } + admin.createTable(tableDescriptor, splits); + LOG.info ("Table created with " + this.presplitRegions + " splits"); + } + else { + boolean tableExists = admin.tableExists(tableDescriptor.getName()); + if (!tableExists) { + admin.createTable(tableDescriptor); + LOG.info("Table " + tableDescriptor + " created"); + } + } + boolean tableExists = admin.tableExists(tableDescriptor.getName()); + return tableExists; + } + + protected HTableDescriptor getTableDescriptor() { + return TABLE_DESCRIPTOR; + } + + /** + * generates splits based on total number of rows and specified split regions + * + * @return splits : array of byte [] + */ + protected byte[][] getSplits() { + if (this.presplitRegions == 0) + return new byte [0][]; + + byte[][] splits = new byte[this.presplitRegions][]; + int jump = this.R / this.presplitRegions; + for (int i=0; i cmd) + throws IOException, InterruptedException, ClassNotFoundException { + checkTable(new HBaseAdmin(conf)); + if (this.nomapred) { + doMultipleClients(cmd); + } else { + doMapReduce(cmd); + } + } + + /* + * Run all clients in this vm each to its own thread. + * @param cmd Command to run. + * @throws IOException + */ + private void doMultipleClients(final Class cmd) throws IOException { + final List threads = new ArrayList(this.N); + final int perClientRows = R/N; + for (int i = 0; i < this.N; i++) { + Thread t = new Thread (Integer.toString(i)) { + @Override + public void run() { + super.run(); + PerformanceEvaluation pe = new PerformanceEvaluation(conf); + int index = Integer.parseInt(getName()); + try { + long elapsedTime = pe.runOneClient(cmd, index * perClientRows, + perClientRows, R, + flushCommits, writeToWAL, new Status() { + public void setStatus(final String msg) throws IOException { + LOG.info("client-" + getName() + " " + msg); + } + }); + LOG.info("Finished " + getName() + " in " + elapsedTime + + "ms writing " + perClientRows + " rows"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + threads.add(t); + } + for (Thread t: threads) { + t.start(); + } + for (Thread t: threads) { + while(t.isAlive()) { + try { + t.join(); + } catch (InterruptedException e) { + LOG.debug("Interrupted, continuing" + e.toString()); + } + } + } + } + + /* + * Run a mapreduce job. Run as many maps as asked-for clients. + * Before we start up the job, write out an input file with instruction + * per client regards which row they are to start on. + * @param cmd Command to run. + * @throws IOException + */ + private void doMapReduce(final Class cmd) throws IOException, + InterruptedException, ClassNotFoundException { + Path inputDir = writeInputFile(this.conf); + this.conf.set(EvaluationMapTask.CMD_KEY, cmd.getName()); + this.conf.set(EvaluationMapTask.PE_KEY, getClass().getName()); + Job job = new Job(this.conf); + job.setJarByClass(PerformanceEvaluation.class); + job.setJobName("HBase Performance Evaluation"); + + job.setInputFormatClass(PeInputFormat.class); + PeInputFormat.setInputPaths(job, inputDir); + + job.setOutputKeyClass(LongWritable.class); + job.setOutputValueClass(LongWritable.class); + + job.setMapperClass(EvaluationMapTask.class); + job.setReducerClass(LongSumReducer.class); + + job.setNumReduceTasks(1); + + job.setOutputFormatClass(TextOutputFormat.class); + TextOutputFormat.setOutputPath(job, new Path(inputDir,"outputs")); + + TableMapReduceUtil.addDependencyJars(job); + // Add a Class from the hbase.jar so it gets registered too. + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + org.apache.hadoop.hbase.util.Bytes.class); + + TableMapReduceUtil.initCredentials(job); + + job.waitForCompletion(true); + } + + /* + * Write input file of offsets-per-client for the mapreduce job. + * @param c Configuration + * @return Directory that contains file written. + * @throws IOException + */ + private Path writeInputFile(final Configuration c) throws IOException { + FileSystem fs = FileSystem.get(c); + if (!fs.exists(PERF_EVAL_DIR)) { + fs.mkdirs(PERF_EVAL_DIR); + } + SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss"); + Path subdir = new Path(PERF_EVAL_DIR, formatter.format(new Date())); + fs.mkdirs(subdir); + Path inputFile = new Path(subdir, "input.txt"); + PrintStream out = new PrintStream(fs.create(inputFile)); + // Make input random. + Map m = new TreeMap(); + Hash h = MurmurHash.getInstance(); + int perClientRows = (this.R / this.N); + try { + for (int i = 0; i < 10; i++) { + for (int j = 0; j < N; j++) { + String s = "startRow=" + ((j * perClientRows) + (i * (perClientRows/10))) + + ", perClientRunRows=" + (perClientRows / 10) + + ", totalRows=" + this.R + + ", clients=" + this.N + + ", flushCommits=" + this.flushCommits + + ", writeToWAL=" + this.writeToWAL; + int hash = h.hash(Bytes.toBytes(s)); + m.put(hash, s); + } + } + for (Map.Entry e: m.entrySet()) { + out.println(e.getValue()); + } + } finally { + out.close(); + } + return subdir; + } + + /** + * Describes a command. + */ + static class CmdDescriptor { + private Class cmdClass; + private String name; + private String description; + + CmdDescriptor(Class cmdClass, String name, String description) { + this.cmdClass = cmdClass; + this.name = name; + this.description = description; + } + + public Class getCmdClass() { + return cmdClass; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + } + + /** + * Wraps up options passed to {@link org.apache.hadoop.hbase.PerformanceEvaluation.Test + * tests}. This makes the reflection logic a little easier to understand... + */ + static class TestOptions { + private int startRow; + private int perClientRunRows; + private int totalRows; + private byte[] tableName; + private boolean flushCommits; + private boolean writeToWAL = true; + + TestOptions() { + } + + TestOptions(int startRow, int perClientRunRows, int totalRows, byte[] tableName, boolean flushCommits, boolean writeToWAL) { + this.startRow = startRow; + this.perClientRunRows = perClientRunRows; + this.totalRows = totalRows; + this.tableName = tableName; + this.flushCommits = flushCommits; + this.writeToWAL = writeToWAL; + } + + public int getStartRow() { + return startRow; + } + + public int getPerClientRunRows() { + return perClientRunRows; + } + + public int getTotalRows() { + return totalRows; + } + + public byte[] getTableName() { + return tableName; + } + + public boolean isFlushCommits() { + return flushCommits; + } + + public boolean isWriteToWAL() { + return writeToWAL; + } + } + + /* + * A test. + * Subclass to particularize what happens per row. + */ + static abstract class Test { + // Below is make it so when Tests are all running in the one + // jvm, that they each have a differently seeded Random. + private static final Random randomSeed = + new Random(System.currentTimeMillis()); + private static long nextRandomSeed() { + return randomSeed.nextLong(); + } + protected final Random rand = new Random(nextRandomSeed()); + + protected final int startRow; + protected final int perClientRunRows; + protected final int totalRows; + private final Status status; + protected byte[] tableName; + protected HBaseAdmin admin; + protected HTable table; + protected volatile Configuration conf; + protected boolean flushCommits; + protected boolean writeToWAL; + + /** + * Note that all subclasses of this class must provide a public contructor + * that has the exact same list of arguments. + */ + Test(final Configuration conf, final TestOptions options, final Status status) { + super(); + this.startRow = options.getStartRow(); + this.perClientRunRows = options.getPerClientRunRows(); + this.totalRows = options.getTotalRows(); + this.status = status; + this.tableName = options.getTableName(); + this.table = null; + this.conf = conf; + this.flushCommits = options.isFlushCommits(); + this.writeToWAL = options.isWriteToWAL(); + } + + private String generateStatus(final int sr, final int i, final int lr) { + return sr + "/" + i + "/" + lr; + } + + protected int getReportingPeriod() { + int period = this.perClientRunRows / 10; + return period == 0? this.perClientRunRows: period; + } + + void testSetup() throws IOException { + this.admin = new HBaseAdmin(conf); + this.table = new HTable(conf, tableName); + this.table.setAutoFlush(false); + this.table.setScannerCaching(30); + } + + void testTakedown() throws IOException { + if (flushCommits) { + this.table.flushCommits(); + } + table.close(); + } + + /* + * Run test + * @return Elapsed time. + * @throws IOException + */ + long test() throws IOException { + long elapsedTime; + testSetup(); + long startTime = System.currentTimeMillis(); + try { + testTimed(); + elapsedTime = System.currentTimeMillis() - startTime; + } finally { + testTakedown(); + } + return elapsedTime; + } + + /** + * Provides an extension point for tests that don't want a per row invocation. + */ + void testTimed() throws IOException { + int lastRow = this.startRow + this.perClientRunRows; + // Report on completion of 1/10th of total. + for (int i = this.startRow; i < lastRow; i++) { + testRow(i); + if (status != null && i > 0 && (i % getReportingPeriod()) == 0) { + status.setStatus(generateStatus(this.startRow, i, lastRow)); + } + } + } + + /* + * Test for individual row. + * @param i Row index. + */ + void testRow(final int i) throws IOException { + } + } + + @SuppressWarnings("unused") + static class RandomSeekScanTest extends Test { + RandomSeekScanTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testRow(final int i) throws IOException { + Scan scan = new Scan(getRandomRow(this.rand, this.totalRows)); + scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); + scan.setFilter(new WhileMatchFilter(new PageFilter(120))); + ResultScanner s = this.table.getScanner(scan); + //int count = 0; + for (Result rr = null; (rr = s.next()) != null;) { + // LOG.info("" + count++ + " " + rr.toString()); + } + s.close(); + } + + @Override + protected int getReportingPeriod() { + int period = this.perClientRunRows / 100; + return period == 0? this.perClientRunRows: period; + } + + } + + @SuppressWarnings("unused") + static abstract class RandomScanWithRangeTest extends Test { + RandomScanWithRangeTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testRow(final int i) throws IOException { + Pair startAndStopRow = getStartAndStopRow(); + Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond()); + scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); + ResultScanner s = this.table.getScanner(scan); + int count = 0; + for (Result rr = null; (rr = s.next()) != null;) { + count++; + } + + if (i % 100 == 0) { + LOG.info(String.format("Scan for key range %s - %s returned %s rows", + Bytes.toString(startAndStopRow.getFirst()), + Bytes.toString(startAndStopRow.getSecond()), count)); + } + + s.close(); + } + + protected abstract Pair getStartAndStopRow(); + + protected Pair generateStartAndStopRows(int maxRange) { + int start = this.rand.nextInt(Integer.MAX_VALUE) % totalRows; + int stop = start + maxRange; + return new Pair(format(start), format(stop)); + } + + @Override + protected int getReportingPeriod() { + int period = this.perClientRunRows / 100; + return period == 0? this.perClientRunRows: period; + } + } + + static class RandomScanWithRange10Test extends RandomScanWithRangeTest { + RandomScanWithRange10Test(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + protected Pair getStartAndStopRow() { + return generateStartAndStopRows(10); + } + } + + static class RandomScanWithRange100Test extends RandomScanWithRangeTest { + RandomScanWithRange100Test(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + protected Pair getStartAndStopRow() { + return generateStartAndStopRows(100); + } + } + + static class RandomScanWithRange1000Test extends RandomScanWithRangeTest { + RandomScanWithRange1000Test(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + protected Pair getStartAndStopRow() { + return generateStartAndStopRows(1000); + } + } + + static class RandomScanWithRange10000Test extends RandomScanWithRangeTest { + RandomScanWithRange10000Test(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + protected Pair getStartAndStopRow() { + return generateStartAndStopRows(10000); + } + } + + static class RandomReadTest extends Test { + RandomReadTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testRow(final int i) throws IOException { + Get get = new Get(getRandomRow(this.rand, this.totalRows)); + get.addColumn(FAMILY_NAME, QUALIFIER_NAME); + this.table.get(get); + } + + @Override + protected int getReportingPeriod() { + int period = this.perClientRunRows / 100; + return period == 0? this.perClientRunRows: period; + } + + } + + static class RandomWriteTest extends Test { + RandomWriteTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testRow(final int i) throws IOException { + byte [] row = getRandomRow(this.rand, this.totalRows); + Put put = new Put(row); + byte[] value = generateValue(this.rand); + put.add(FAMILY_NAME, QUALIFIER_NAME, value); + put.setWriteToWAL(writeToWAL); + table.put(put); + } + } + + static class ScanTest extends Test { + private ResultScanner testScanner; + + ScanTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testSetup() throws IOException { + super.testSetup(); + } + + @Override + void testTakedown() throws IOException { + if (this.testScanner != null) { + this.testScanner.close(); + } + super.testTakedown(); + } + + + @Override + void testRow(final int i) throws IOException { + if (this.testScanner == null) { + Scan scan = new Scan(format(this.startRow)); + scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); + this.testScanner = table.getScanner(scan); + } + testScanner.next(); + } + + } + + static class SequentialReadTest extends Test { + SequentialReadTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testRow(final int i) throws IOException { + Get get = new Get(format(i)); + get.addColumn(FAMILY_NAME, QUALIFIER_NAME); + table.get(get); + } + + } + + static class SequentialWriteTest extends Test { + SequentialWriteTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testRow(final int i) throws IOException { + Put put = new Put(format(i)); + byte[] value = generateValue(this.rand); + put.add(FAMILY_NAME, QUALIFIER_NAME, value); + put.setWriteToWAL(writeToWAL); + table.put(put); + } + + } + + static class FilteredScanTest extends Test { + protected static final Log LOG = LogFactory.getLog(FilteredScanTest.class.getName()); + + FilteredScanTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testRow(int i) throws IOException { + byte[] value = generateValue(this.rand); + Scan scan = constructScan(value); + ResultScanner scanner = null; + try { + scanner = this.table.getScanner(scan); + while (scanner.next() != null) { + } + } finally { + if (scanner != null) scanner.close(); + } + } + + protected Scan constructScan(byte[] valuePrefix) throws IOException { + Filter filter = new SingleColumnValueFilter( + FAMILY_NAME, QUALIFIER_NAME, CompareFilter.CompareOp.EQUAL, + new BinaryComparator(valuePrefix) + ); + Scan scan = new Scan(); + scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); + scan.setFilter(filter); + return scan; + } + } + + /* + * Format passed integer. + * @param number + * @return Returns zero-prefixed 10-byte wide decimal version of passed + * number (Does absolute in case number is negative). + */ + public static byte [] format(final int number) { + byte [] b = new byte[10]; + int d = Math.abs(number); + for (int i = b.length - 1; i >= 0; i--) { + b[i] = (byte)((d % 10) + '0'); + d /= 10; + } + return b; + } + + /* + * This method takes some time and is done inline uploading data. For + * example, doing the mapfile test, generation of the key and value + * consumes about 30% of CPU time. + * @return Generated random value to insert into a table cell. + */ + public static byte[] generateValue(final Random r) { + byte [] b = new byte [ROW_LENGTH]; + r.nextBytes(b); + return b; + } + + static byte [] getRandomRow(final Random random, final int totalRows) { + return format(random.nextInt(Integer.MAX_VALUE) % totalRows); + } + + long runOneClient(final Class cmd, final int startRow, + final int perClientRunRows, final int totalRows, + boolean flushCommits, boolean writeToWAL, + final Status status) + throws IOException { + status.setStatus("Start " + cmd + " at offset " + startRow + " for " + + perClientRunRows + " rows"); + long totalElapsedTime = 0; + + Test t = null; + TestOptions options = new TestOptions(startRow, perClientRunRows, + totalRows, getTableDescriptor().getName(), flushCommits, writeToWAL); + try { + Constructor constructor = cmd.getDeclaredConstructor( + Configuration.class, TestOptions.class, Status.class); + t = constructor.newInstance(this.conf, options, status); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException("Invalid command class: " + + cmd.getName() + ". It does not provide a constructor as described by" + + "the javadoc comment. Available constructors are: " + + Arrays.toString(cmd.getConstructors())); + } catch (Exception e) { + throw new IllegalStateException("Failed to construct command class", e); + } + totalElapsedTime = t.test(); + + status.setStatus("Finished " + cmd + " in " + totalElapsedTime + + "ms at offset " + startRow + " for " + perClientRunRows + " rows"); + return totalElapsedTime; + } + + private void runNIsOne(final Class cmd) { + Status status = new Status() { + public void setStatus(String msg) throws IOException { + LOG.info(msg); + } + }; + + HBaseAdmin admin = null; + try { + admin = new HBaseAdmin(this.conf); + checkTable(admin); + runOneClient(cmd, 0, this.R, this.R, this.flushCommits, this.writeToWAL, + status); + } catch (Exception e) { + LOG.error("Failed", e); + } + } + + private void runTest(final Class cmd) throws IOException, + InterruptedException, ClassNotFoundException { + MiniHBaseCluster hbaseMiniCluster = null; + MiniDFSCluster dfsCluster = null; + MiniZooKeeperCluster zooKeeperCluster = null; + if (this.miniCluster) { + dfsCluster = new MiniDFSCluster(conf, 2, true, (String[])null); + zooKeeperCluster = new MiniZooKeeperCluster(); + int zooKeeperPort = zooKeeperCluster.startup(new File(System.getProperty("java.io.tmpdir"))); + + // mangle the conf so that the fs parameter points to the minidfs we + // just started up + FileSystem fs = dfsCluster.getFileSystem(); + conf.set("fs.default.name", fs.getUri().toString()); + conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(zooKeeperPort)); + Path parentdir = fs.getHomeDirectory(); + conf.set(HConstants.HBASE_DIR, parentdir.toString()); + fs.mkdirs(parentdir); + FSUtils.setVersion(fs, parentdir); + hbaseMiniCluster = new MiniHBaseCluster(this.conf, N); + } + + try { + if (N == 1) { + // If there is only one client and one HRegionServer, we assume nothing + // has been set up at all. + runNIsOne(cmd); + } else { + // Else, run + runNIsMoreThanOne(cmd); + } + } finally { + if(this.miniCluster) { + if (hbaseMiniCluster != null) hbaseMiniCluster.shutdown(); + if (zooKeeperCluster != null) zooKeeperCluster.shutdown(); + HBaseTestCase.shutdownDfs(dfsCluster); + } + } + } + + protected void printUsage() { + printUsage(null); + } + + protected void printUsage(final String message) { + if (message != null && message.length() > 0) { + System.err.println(message); + } + System.err.println("Usage: java " + this.getClass().getName() + " \\"); + System.err.println(" [--miniCluster] [--nomapred] [--rows=ROWS] "); + System.err.println(); + System.err.println("Options:"); + System.err.println(" miniCluster Run the test on an HBaseMiniCluster"); + System.err.println(" nomapred Run multiple clients using threads " + + "(rather than use mapreduce)"); + System.err.println(" rows Rows each client runs. Default: One million"); + System.err.println(" flushCommits Used to determine if the test should flush the table. Default: false"); + System.err.println(" writeToWAL Set writeToWAL on puts. Default: True"); + System.err.println(" presplit Create presplit table. Recommended for accurate perf analysis (see guide). Default: disabled"); + System.err.println(); + System.err.println("Command:"); + for (CmdDescriptor command : commands.values()) { + System.err.println(String.format(" %-15s %s", command.getName(), command.getDescription())); + } + System.err.println(); + System.err.println("Args:"); + System.err.println(" nclients Integer. Required. Total number of " + + "clients (and HRegionServers)"); + System.err.println(" running: 1 <= value <= 500"); + System.err.println("Examples:"); + System.err.println(" To run a single evaluation client:"); + System.err.println(" $ bin/hbase " + this.getClass().getName() + + " sequentialWrite 1"); + } + + private void getArgs(final int start, final String[] args) { + if(start + 1 > args.length) { + throw new IllegalArgumentException("must supply the number of clients"); + } + N = Integer.parseInt(args[start]); + if (N < 1) { + throw new IllegalArgumentException("Number of clients must be > 1"); + } + // Set total number of rows to write. + this.R = this.R * N; + } + + public int doCommandLine(final String[] args) { + // Process command-line args. TODO: Better cmd-line processing + // (but hopefully something not as painful as cli options). + int errCode = -1; + if (args.length < 1) { + printUsage(); + return errCode; + } + + try { + for (int i = 0; i < args.length; i++) { + String cmd = args[i]; + if (cmd.equals("-h") || cmd.startsWith("--h")) { + printUsage(); + errCode = 0; + break; + } + + final String miniClusterArgKey = "--miniCluster"; + if (cmd.startsWith(miniClusterArgKey)) { + this.miniCluster = true; + continue; + } + + final String nmr = "--nomapred"; + if (cmd.startsWith(nmr)) { + this.nomapred = true; + continue; + } + + final String rows = "--rows="; + if (cmd.startsWith(rows)) { + this.R = Integer.parseInt(cmd.substring(rows.length())); + continue; + } + + final String flushCommits = "--flushCommits="; + if (cmd.startsWith(flushCommits)) { + this.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length())); + continue; + } + + final String writeToWAL = "--writeToWAL="; + if (cmd.startsWith(writeToWAL)) { + this.writeToWAL = Boolean.parseBoolean(cmd.substring(writeToWAL.length())); + continue; + } + + final String presplit = "--presplit="; + if (cmd.startsWith(presplit)) { + this.presplitRegions = Integer.parseInt(cmd.substring(presplit.length())); + continue; + } + + Class cmdClass = determineCommandClass(cmd); + if (cmdClass != null) { + getArgs(i + 1, args); + runTest(cmdClass); + errCode = 0; + break; + } + + printUsage(); + break; + } + } catch (Exception e) { + e.printStackTrace(); + } + + return errCode; + } + + private Class determineCommandClass(String cmd) { + CmdDescriptor descriptor = commands.get(cmd); + return descriptor != null ? descriptor.getCmdClass() : null; + } + + /** + * @param args + */ + public static void main(final String[] args) { + Configuration c = HBaseConfiguration.create(); + System.exit(new PerformanceEvaluation(c).doCommandLine(args)); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java new file mode 100644 index 0000000..eac7207 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java @@ -0,0 +1,78 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + + +/** + * Code shared by PE tests. + */ +public class PerformanceEvaluationCommons { + static final Log LOG = + LogFactory.getLog(PerformanceEvaluationCommons.class.getName()); + + public static void assertValueSize(final int expectedSize, final int got) { + if (got != expectedSize) { + throw new AssertionError("Expected " + expectedSize + " but got " + got); + } + } + + public static void assertKey(final byte [] expected, final ByteBuffer got) { + byte [] b = new byte[got.limit()]; + got.get(b, 0, got.limit()); + assertKey(expected, b); + } + + public static void assertKey(final byte [] expected, final byte [] got) { + if (!org.apache.hadoop.hbase.util.Bytes.equals(expected, got)) { + throw new AssertionError("Expected " + + org.apache.hadoop.hbase.util.Bytes.toString(expected) + + " but got " + org.apache.hadoop.hbase.util.Bytes.toString(got)); + } + } + + public static void concurrentReads(final Runnable r) { + final int count = 1; + long now = System.currentTimeMillis(); + List threads = new ArrayList(count); + for (int i = 0; i < count; i++) { + Thread t = new Thread(r); + t.setName("" + i); + threads.add(t); + } + for (Thread t: threads) { + t.start(); + } + for (Thread t: threads) { + try { + t.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + LOG.info("Test took " + (System.currentTimeMillis() - now)); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java b/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java new file mode 100644 index 0000000..62a5144 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java @@ -0,0 +1,221 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import com.sun.management.UnixOperatingSystemMXBean; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.client.HConnectionTestingUtility; + +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; +import java.util.*; + + +/** + * Check the resources used: + * - threads + * - file descriptor + */ +public class ResourceChecker { + private static final Log LOG = LogFactory.getLog(ResourceChecker.class); + + enum Phase { + INITIAL, INTERMEDIATE, END + } + private static Set initialThreadNames = new HashSet(); + + /** + * On unix, we know how to get the number of open file descriptor + */ + private static class ResourceAnalyzer { + private static final OperatingSystemMXBean osStats; + private static final UnixOperatingSystemMXBean unixOsStats; + + public long getThreadsCount(Phase phase) { + Map stackTraces = Thread.getAllStackTraces(); + if (phase == Phase.INITIAL) { + for (Thread t : stackTraces.keySet()) { + initialThreadNames.add(t.getName()); + } + } + return stackTraces.size(); + } + + public long getOpenFileDescriptorCount() { + if (unixOsStats == null) { + return 0; + } else { + return unixOsStats.getOpenFileDescriptorCount(); + } + } + + public long getMaxFileDescriptorCount() { + if (unixOsStats == null) { + return 0; + } else { + return unixOsStats.getMaxFileDescriptorCount(); + } + } + + public long getConnectionCount(){ + return HConnectionTestingUtility.getConnectionCount(); + } + + static { + osStats = + ManagementFactory.getOperatingSystemMXBean(); + if (osStats instanceof UnixOperatingSystemMXBean) { + unixOsStats = (UnixOperatingSystemMXBean) osStats; + } else { + unixOsStats = null; + } + } + } + + private static final ResourceAnalyzer rc = new ResourceAnalyzer(); + + /** + * Maximum we set for the thread. Will get a warning in logs + * if we go other this limit + */ + private static final long MAX_THREADS_COUNT = 500; + + /** + * Maximum we set for the thread. Will get a warning in logs + * if we go other this limit + */ + private static final long MAX_FILE_HANDLES_COUNT = 1024; + + + private long initialThreadsCount; + private long initialFileHandlesCount; + private long initialConnectionCount; + + + public boolean checkThreads(String tagLine) { + boolean isOk = true; + long threadCount = rc.getThreadsCount(Phase.INTERMEDIATE); + + if (threadCount > MAX_THREADS_COUNT) { + LOG.error( + tagLine + ": too many threads used. We use " + + threadCount + " our max is " + MAX_THREADS_COUNT); + isOk = false; + } + return isOk; + } + + public boolean check(String tagLine) { + + boolean isOk = checkThreads(tagLine); + if (!checkFileHandles(tagLine)) isOk = false; + + return isOk; + } + + public ResourceChecker(String tagLine) { + init(tagLine); + } + + public final void init(String tagLine) { + if (rc.getMaxFileDescriptorCount() < MAX_FILE_HANDLES_COUNT) { + LOG.error( + "Bad configuration: the operating systems file handles maximum is " + + rc.getMaxFileDescriptorCount() + " our is " + MAX_FILE_HANDLES_COUNT); + } + + logInfo(Phase.INITIAL, tagLine); + + initialThreadsCount = rc.getThreadsCount(Phase.INITIAL); + initialFileHandlesCount = rc.getOpenFileDescriptorCount(); + initialConnectionCount= rc.getConnectionCount(); + + check(tagLine); + } + + public void logInfo(Phase phase, String tagLine) { + long threadCount = rc.getThreadsCount(phase); + LOG.info( + tagLine + ": " + + threadCount + " threads" + + (initialThreadsCount > 0 ? + " (was " + initialThreadsCount + "), " : ", ") + + rc.getOpenFileDescriptorCount() + " file descriptors" + + (initialFileHandlesCount > 0 ? + " (was " + initialFileHandlesCount + "). " : " ") + + rc.getConnectionCount() + " connections" + + (initialConnectionCount > 0 ? + " (was " + initialConnectionCount + "), " : ", ") + + (initialThreadsCount > 0 && threadCount > initialThreadsCount ? + " -thread leak?- " : "") + + (initialFileHandlesCount > 0 && + rc.getOpenFileDescriptorCount() > initialFileHandlesCount ? + " -file handle leak?- " : "") + + (initialConnectionCount > 0 && + rc.getConnectionCount() > initialConnectionCount ? + " -connection leak?- " : "" ) + ); + if (phase == Phase.END) { + Map stackTraces = Thread.getAllStackTraces(); + if (stackTraces.size() > initialThreadNames.size()) { + for (Thread t : stackTraces.keySet()) { + if (!initialThreadNames.contains(t.getName())) { + LOG.info(tagLine + ": potentially hanging thread - " + t.getName()); + StackTraceElement[] stackElements = stackTraces.get(t); + for (StackTraceElement ele : stackElements) { + LOG.info("\t" + ele); + } + } + } + } + } + } + + + public boolean checkFileHandles(String tagLine) { + boolean isOk = true; + + if (rc.getOpenFileDescriptorCount() > MAX_FILE_HANDLES_COUNT) { + LOG.error( + tagLine + ": too many file handles used. We use " + + rc.getOpenFileDescriptorCount() + " our max is " + + MAX_FILE_HANDLES_COUNT); + isOk = false; + } + + return isOk; + } + + /** + * Helper function: print the threads + */ + public static void printThreads(){ + Set threads = Thread.getAllStackTraces().keySet(); + System.out.println("name; state; isDameon; isAlive; isInterrupted"); + for (Thread t: threads){ + System.out.println( + t.getName()+";"+t.getState()+";"+t.isDaemon()+";"+t.isAlive()+ + ";"+t.isInterrupted() + ); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java b/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java new file mode 100644 index 0000000..56638c9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java @@ -0,0 +1,88 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + + +/** + * Class that implements a JUnit rule to be called before and after each + * test method to check the resources used: + * - file descriptors + * - threads + * @see ResourceChecker + */ +public class ResourceCheckerJUnitRule extends org.junit.rules.TestWatcher { + private ResourceChecker cu; + private boolean endDone; + + /** + * To be called before the test methods + * @param testName + */ + private void start(String testName) { + cu = new ResourceChecker("before "+testName); + endDone = false; + } + + /** + * To be called after the test methods + * @param testName + */ + private void end(String testName) { + if (!endDone) { + endDone = true; + cu.logInfo(ResourceChecker.Phase.END, "after " + testName); + cu.check("after "+testName); + } + } + + /** + * Get the test name from the JUnit Description + * @param description + * @return the string for the short test name + */ + private String descriptionToShortTestName( + org.junit.runner.Description description) { + final int toRemove = "org.apache.hadoop.hbase.".length(); + return description.getTestClass().getName().substring(toRemove) + + "#" + description.getMethodName(); + } + + @Override + protected void succeeded(org.junit.runner.Description description) { + end(descriptionToShortTestName(description)); + } + + @Override + protected void failed(java.lang.Throwable e, org.junit.runner.Description description) { + end(descriptionToShortTestName(description)); + } + + @Override + protected void starting(org.junit.runner.Description description) { + start(descriptionToShortTestName(description)); + } + + @Override + protected void finished(org.junit.runner.Description description) { + end(descriptionToShortTestName(description)); + } +} + diff --git a/src/test/java/org/apache/hadoop/hbase/SmallTests.java b/src/test/java/org/apache/hadoop/hbase/SmallTests.java new file mode 100644 index 0000000..eba6a94 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/SmallTests.java @@ -0,0 +1,35 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +/** + * Tag a test as 'small', meaning that the test class has the following + * characteristics: + * - can be run simultaneously with other small tests in the same JVM + * - ideally, last less than 15 seconds + * - does not use a cluster + * + * @see MediumTests + * @see LargeTests + * @see IntegrationTests + */ +public interface SmallTests { +} diff --git a/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java new file mode 100644 index 0000000..227d243 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java @@ -0,0 +1,380 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread; +import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + +/** + * Test case that uses multiple threads to read and write multifamily rows + * into a table, verifying that reads never see partially-complete writes. + * + * This can run as a junit test, or with a main() function which runs against + * a real cluster (eg for testing with failures, region movement, etc) + */ +@Category(MediumTests.class) +public class TestAcidGuarantees implements Tool { + protected static final Log LOG = LogFactory.getLog(TestAcidGuarantees.class); + public static final byte [] TABLE_NAME = Bytes.toBytes("TestAcidGuarantees"); + public static final byte [] FAMILY_A = Bytes.toBytes("A"); + public static final byte [] FAMILY_B = Bytes.toBytes("B"); + public static final byte [] FAMILY_C = Bytes.toBytes("C"); + public static final byte [] QUALIFIER_NAME = Bytes.toBytes("data"); + + public static final byte[][] FAMILIES = new byte[][] { + FAMILY_A, FAMILY_B, FAMILY_C }; + + private HBaseTestingUtility util; + + public static int NUM_COLS_TO_CHECK = 50; + + // when run as main + private Configuration conf; + + private void createTableIfMissing() + throws IOException { + try { + util.createTable(TABLE_NAME, FAMILIES); + } catch (TableExistsException tee) { + } + } + + public TestAcidGuarantees() { + // Set small flush size for minicluster so we exercise reseeking scanners + Configuration conf = HBaseConfiguration.create(); + conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128*1024)); + // prevent aggressive region split + conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, + ConstantSizeRegionSplitPolicy.class.getName()); + util = new HBaseTestingUtility(conf); + } + + /** + * Thread that does random full-row writes into a table. + */ + public static class AtomicityWriter extends RepeatingTestThread { + Random rand = new Random(); + byte data[] = new byte[10]; + byte targetRows[][]; + byte targetFamilies[][]; + HTable table; + AtomicLong numWritten = new AtomicLong(); + + public AtomicityWriter(TestContext ctx, byte targetRows[][], + byte targetFamilies[][]) throws IOException { + super(ctx); + this.targetRows = targetRows; + this.targetFamilies = targetFamilies; + table = new HTable(ctx.getConf(), TABLE_NAME); + } + public void doAnAction() throws Exception { + // Pick a random row to write into + byte[] targetRow = targetRows[rand.nextInt(targetRows.length)]; + Put p = new Put(targetRow); + rand.nextBytes(data); + + for (byte[] family : targetFamilies) { + for (int i = 0; i < NUM_COLS_TO_CHECK; i++) { + byte qualifier[] = Bytes.toBytes("col" + i); + p.add(family, qualifier, data); + } + } + table.put(p); + numWritten.getAndIncrement(); + } + } + + /** + * Thread that does single-row reads in a table, looking for partially + * completed rows. + */ + public static class AtomicGetReader extends RepeatingTestThread { + byte targetRow[]; + byte targetFamilies[][]; + HTable table; + int numVerified = 0; + AtomicLong numRead = new AtomicLong(); + + public AtomicGetReader(TestContext ctx, byte targetRow[], + byte targetFamilies[][]) throws IOException { + super(ctx); + this.targetRow = targetRow; + this.targetFamilies = targetFamilies; + table = new HTable(ctx.getConf(), TABLE_NAME); + } + + public void doAnAction() throws Exception { + Get g = new Get(targetRow); + Result res = table.get(g); + byte[] gotValue = null; + if (res.getRow() == null) { + // Trying to verify but we didn't find the row - the writing + // thread probably just hasn't started writing yet, so we can + // ignore this action + return; + } + + for (byte[] family : targetFamilies) { + for (int i = 0; i < NUM_COLS_TO_CHECK; i++) { + byte qualifier[] = Bytes.toBytes("col" + i); + byte thisValue[] = res.getValue(family, qualifier); + if (gotValue != null && !Bytes.equals(gotValue, thisValue)) { + gotFailure(gotValue, res); + } + numVerified++; + gotValue = thisValue; + } + } + numRead.getAndIncrement(); + } + + private void gotFailure(byte[] expected, Result res) { + StringBuilder msg = new StringBuilder(); + msg.append("Failed after ").append(numVerified).append("!"); + msg.append("Expected=").append(Bytes.toStringBinary(expected)); + msg.append("Got:\n"); + for (KeyValue kv : res.list()) { + msg.append(kv.toString()); + msg.append(" val= "); + msg.append(Bytes.toStringBinary(kv.getValue())); + msg.append("\n"); + } + throw new RuntimeException(msg.toString()); + } + } + + /** + * Thread that does full scans of the table looking for any partially completed + * rows. + */ + public static class AtomicScanReader extends RepeatingTestThread { + byte targetFamilies[][]; + HTable table; + AtomicLong numScans = new AtomicLong(); + AtomicLong numRowsScanned = new AtomicLong(); + + public AtomicScanReader(TestContext ctx, + byte targetFamilies[][]) throws IOException { + super(ctx); + this.targetFamilies = targetFamilies; + table = new HTable(ctx.getConf(), TABLE_NAME); + } + + public void doAnAction() throws Exception { + Scan s = new Scan(); + for (byte[] family : targetFamilies) { + s.addFamily(family); + } + ResultScanner scanner = table.getScanner(s); + + for (Result res : scanner) { + byte[] gotValue = null; + + for (byte[] family : targetFamilies) { + for (int i = 0; i < NUM_COLS_TO_CHECK; i++) { + byte qualifier[] = Bytes.toBytes("col" + i); + byte thisValue[] = res.getValue(family, qualifier); + if (gotValue != null && !Bytes.equals(gotValue, thisValue)) { + gotFailure(gotValue, res); + } + gotValue = thisValue; + } + } + numRowsScanned.getAndIncrement(); + } + numScans.getAndIncrement(); + } + + private void gotFailure(byte[] expected, Result res) { + StringBuilder msg = new StringBuilder(); + msg.append("Failed after ").append(numRowsScanned).append("!"); + msg.append("Expected=").append(Bytes.toStringBinary(expected)); + msg.append("Got:\n"); + for (KeyValue kv : res.list()) { + msg.append(kv.toString()); + msg.append(" val= "); + msg.append(Bytes.toStringBinary(kv.getValue())); + msg.append("\n"); + } + throw new RuntimeException(msg.toString()); + } + } + + + public void runTestAtomicity(long millisToRun, + int numWriters, + int numGetters, + int numScanners, + int numUniqueRows) throws Exception { + createTableIfMissing(); + TestContext ctx = new TestContext(util.getConfiguration()); + + byte rows[][] = new byte[numUniqueRows][]; + for (int i = 0; i < numUniqueRows; i++) { + rows[i] = Bytes.toBytes("test_row_" + i); + } + + List writers = Lists.newArrayList(); + for (int i = 0; i < numWriters; i++) { + AtomicityWriter writer = new AtomicityWriter( + ctx, rows, FAMILIES); + writers.add(writer); + ctx.addThread(writer); + } + // Add a flusher + ctx.addThread(new RepeatingTestThread(ctx) { + HBaseAdmin admin = new HBaseAdmin(util.getConfiguration()); + public void doAnAction() throws Exception { + admin.flush(TABLE_NAME); + } + }); + + List getters = Lists.newArrayList(); + for (int i = 0; i < numGetters; i++) { + AtomicGetReader getter = new AtomicGetReader( + ctx, rows[i % numUniqueRows], FAMILIES); + getters.add(getter); + ctx.addThread(getter); + } + + List scanners = Lists.newArrayList(); + for (int i = 0; i < numScanners; i++) { + AtomicScanReader scanner = new AtomicScanReader(ctx, FAMILIES); + scanners.add(scanner); + ctx.addThread(scanner); + } + + ctx.startThreads(); + ctx.waitFor(millisToRun); + ctx.stop(); + + LOG.info("Finished test. Writers:"); + for (AtomicityWriter writer : writers) { + LOG.info(" wrote " + writer.numWritten.get()); + } + LOG.info("Readers:"); + for (AtomicGetReader reader : getters) { + LOG.info(" read " + reader.numRead.get()); + } + LOG.info("Scanners:"); + for (AtomicScanReader scanner : scanners) { + LOG.info(" scanned " + scanner.numScans.get()); + LOG.info(" verified " + scanner.numRowsScanned.get() + " rows"); + } + } + + @Test + public void testGetAtomicity() throws Exception { + util.startMiniCluster(1); + try { + runTestAtomicity(20000, 5, 5, 0, 3); + } finally { + util.shutdownMiniCluster(); + } + } + + @Test + public void testScanAtomicity() throws Exception { + util.startMiniCluster(1); + try { + runTestAtomicity(20000, 5, 0, 5, 3); + } finally { + util.shutdownMiniCluster(); + } + } + + @Test + public void testMixedAtomicity() throws Exception { + util.startMiniCluster(1); + try { + runTestAtomicity(20000, 5, 2, 2, 3); + } finally { + util.shutdownMiniCluster(); + } + } + + //////////////////////////////////////////////////////////////////////////// + // Tool interface + //////////////////////////////////////////////////////////////////////////// + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void setConf(Configuration c) { + this.conf = c; + this.util = new HBaseTestingUtility(c); + } + + @Override + public int run(String[] arg0) throws Exception { + Configuration c = getConf(); + int millis = c.getInt("millis", 5000); + int numWriters = c.getInt("numWriters", 50); + int numGetters = c.getInt("numGetters", 2); + int numScanners = c.getInt("numScanners", 2); + int numUniqueRows = c.getInt("numUniqueRows", 3); + runTestAtomicity(millis, numWriters, numGetters, numScanners, numUniqueRows); + return 0; + } + + public static void main(String args[]) throws Exception { + Configuration c = HBaseConfiguration.create(); + int status; + try { + TestAcidGuarantees test = new TestAcidGuarantees(); + status = ToolRunner.run(c, test, args); + } catch (Exception e) { + LOG.error("Exiting due to error", e); + status = -1; + } + System.exit(status); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java b/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java new file mode 100644 index 0000000..b5413bc --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import static junit.framework.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.net.URL; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.regex.Pattern; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runners.Suite; + + +/** + * Checks tests are categorized. + */ +@Category(SmallTests.class) +public class TestCheckTestClasses { + /** + * Throws an assertion if we find a test class without category (small/medium/large/integration). + * List all the test classes without category in the assertion message. + */ + @Test + public void checkClasses() throws Exception { + List> badClasses = new java.util.ArrayList>(); + ClassTestFinder classFinder = new ClassTestFinder(); + for (Class c : classFinder.findClasses(false)) { + if (ClassTestFinder.getCategoryAnnotations(c).length == 0) { + badClasses.add(c); + } + } + assertTrue("There are " + badClasses.size() + " test classes without category: " + + badClasses, badClasses.isEmpty()); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/TestClassFinder.java b/src/test/java/org/apache/hadoop/hbase/TestClassFinder.java new file mode 100644 index 0000000..3bd8e65 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestClassFinder.java @@ -0,0 +1,345 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; +import java.util.jar.*; +import javax.tools.*; + +import org.apache.hadoop.hbase.SmallTests; + +import org.junit.experimental.categories.Category; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.commons.io.FileUtils; + +@Category(SmallTests.class) +public class TestClassFinder { + private static final HBaseTestingUtility testUtil = new HBaseTestingUtility(); + private static final String BASEPKG = "tfcpkg"; + // Use unique jar/class/package names in each test case with the help + // of these global counters; we are mucking with ClassLoader in this test + // and we don't want individual test cases to conflict via it. + private static AtomicLong testCounter = new AtomicLong(0); + private static AtomicLong jarCounter = new AtomicLong(0); + + private static String basePath = null; + + // Default name/class filters for testing. + private static final ClassFinder.FileNameFilter trueNameFilter = + new ClassFinder.FileNameFilter() { + @Override + public boolean isCandidateFile(String fileName, String absFilePath) { + return true; + } + }; + private static final ClassFinder.ClassFilter trueClassFilter = + new ClassFinder.ClassFilter() { + @Override + public boolean isCandidateClass(Class c) { + return true; + } + }; + + @BeforeClass + public static void createTestDir() throws IOException { + basePath = testUtil.getDataTestDir(TestClassFinder.class.getSimpleName()).toString(); + if (!basePath.endsWith("/")) { + basePath += "/"; + } + // Make sure we get a brand new directory. + File testDir = new File(basePath); + if (testDir.exists()) { + deleteTestDir(); + } + assertTrue(testDir.mkdirs()); + } + + @AfterClass + public static void deleteTestDir() throws IOException { + testUtil.cleanupTestDir(TestClassFinder.class.getSimpleName()); + } + + @Test + public void testClassFinderCanFindClassesInJars() throws Exception { + long counter = testCounter.incrementAndGet(); + FileAndPath c1 = compileTestClass(counter, "", "c1"); + FileAndPath c2 = compileTestClass(counter, ".nested", "c2"); + FileAndPath c3 = compileTestClass(counter, "", "c3"); + packageAndLoadJar(c1, c3); + packageAndLoadJar(c2); + + ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter); + Set> allClasses = allClassesFinder.findClasses( + makePackageName("", counter), false); + assertEquals(3, allClasses.size()); + } + + @Test + public void testClassFinderHandlesConflicts() throws Exception { + long counter = testCounter.incrementAndGet(); + FileAndPath c1 = compileTestClass(counter, "", "c1"); + FileAndPath c2 = compileTestClass(counter, "", "c2"); + packageAndLoadJar(c1, c2); + packageAndLoadJar(c1); + + ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter); + Set> allClasses = allClassesFinder.findClasses( + makePackageName("", counter), false); + assertEquals(2, allClasses.size()); + } + + @Test + public void testClassFinderHandlesNestedPackages() throws Exception { + final String NESTED = ".nested"; + final String CLASSNAME1 = "c2"; + final String CLASSNAME2 = "c3"; + long counter = testCounter.incrementAndGet(); + FileAndPath c1 = compileTestClass(counter, "", "c1"); + FileAndPath c2 = compileTestClass(counter, NESTED, CLASSNAME1); + FileAndPath c3 = compileTestClass(counter, NESTED, CLASSNAME2); + packageAndLoadJar(c1, c2); + packageAndLoadJar(c3); + + ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter); + Set> nestedClasses = allClassesFinder.findClasses( + makePackageName(NESTED, counter), false); + assertEquals(2, nestedClasses.size()); + Class nestedClass1 = makeClass(NESTED, CLASSNAME1, counter); + assertTrue(nestedClasses.contains(nestedClass1)); + Class nestedClass2 = makeClass(NESTED, CLASSNAME2, counter); + assertTrue(nestedClasses.contains(nestedClass2)); + } + + @Test + public void testClassFinderFiltersByNameInJar() throws Exception { + final String CLASSNAME = "c1"; + final String CLASSNAMEEXCPREFIX = "c2"; + long counter = testCounter.incrementAndGet(); + FileAndPath c1 = compileTestClass(counter, "", CLASSNAME); + FileAndPath c2 = compileTestClass(counter, "", CLASSNAMEEXCPREFIX + "1"); + FileAndPath c3 = compileTestClass(counter, "", CLASSNAMEEXCPREFIX + "2"); + packageAndLoadJar(c1, c2, c3); + + ClassFinder.FileNameFilter notExcNameFilter = new ClassFinder.FileNameFilter() { + @Override + public boolean isCandidateFile(String fileName, String absFilePath) { + return !fileName.startsWith(CLASSNAMEEXCPREFIX); + } + }; + ClassFinder incClassesFinder = new ClassFinder(notExcNameFilter, trueClassFilter); + Set> incClasses = incClassesFinder.findClasses( + makePackageName("", counter), false); + assertEquals(1, incClasses.size()); + Class incClass = makeClass("", CLASSNAME, counter); + assertTrue(incClasses.contains(incClass)); + } + + @Test + public void testClassFinderFiltersByClassInJar() throws Exception { + final String CLASSNAME = "c1"; + final String CLASSNAMEEXCPREFIX = "c2"; + long counter = testCounter.incrementAndGet(); + FileAndPath c1 = compileTestClass(counter, "", CLASSNAME); + FileAndPath c2 = compileTestClass(counter, "", CLASSNAMEEXCPREFIX + "1"); + FileAndPath c3 = compileTestClass(counter, "", CLASSNAMEEXCPREFIX + "2"); + packageAndLoadJar(c1, c2, c3); + + final ClassFinder.ClassFilter notExcClassFilter = new ClassFinder.ClassFilter() { + @Override + public boolean isCandidateClass(Class c) { + return !c.getSimpleName().startsWith(CLASSNAMEEXCPREFIX); + } + }; + ClassFinder incClassesFinder = new ClassFinder(trueNameFilter, notExcClassFilter); + Set> incClasses = incClassesFinder.findClasses( + makePackageName("", counter), false); + assertEquals(1, incClasses.size()); + Class incClass = makeClass("", CLASSNAME, counter); + assertTrue(incClasses.contains(incClass)); + } + + @Test + public void testClassFinderCanFindClassesInDirs() throws Exception { + // Well, technically, we are not guaranteed that the classes will + // be in dirs, but during normal build they would be. + ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter); + Set> allClasses = allClassesFinder.findClasses( + this.getClass().getPackage().getName(), false); + assertTrue(allClasses.contains(this.getClass())); + assertTrue(allClasses.contains(ClassFinder.class)); + } + + @Test + public void testClassFinderFiltersByNameInDirs() throws Exception { + final String thisName = this.getClass().getSimpleName(); + ClassFinder.FileNameFilter notThisFilter = new ClassFinder.FileNameFilter() { + @Override + public boolean isCandidateFile(String fileName, String absFilePath) { + return !fileName.equals(thisName + ".class"); + } + }; + String thisPackage = this.getClass().getPackage().getName(); + ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter); + Set> allClasses = allClassesFinder.findClasses(thisPackage, false); + ClassFinder notThisClassFinder = new ClassFinder(notThisFilter, trueClassFilter); + Set> notAllClasses = notThisClassFinder.findClasses(thisPackage, false); + assertFalse(notAllClasses.contains(this.getClass())); + assertEquals(allClasses.size() - 1, notAllClasses.size()); + } + + @Test + public void testClassFinderFiltersByClassInDirs() throws Exception { + ClassFinder.ClassFilter notThisFilter = new ClassFinder.ClassFilter() { + @Override + public boolean isCandidateClass(Class c) { + return c != TestClassFinder.class; + } + }; + String thisPackage = this.getClass().getPackage().getName(); + ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter); + Set> allClasses = allClassesFinder.findClasses(thisPackage, false); + ClassFinder notThisClassFinder = new ClassFinder(trueNameFilter, notThisFilter); + Set> notAllClasses = notThisClassFinder.findClasses(thisPackage, false); + assertFalse(notAllClasses.contains(this.getClass())); + assertEquals(allClasses.size() - 1, notAllClasses.size()); + } + + @Test + public void testClassFinderDefaultsToOwnPackage() throws Exception { + // Correct handling of nested packages is tested elsewhere, so here we just assume + // pkgClasses is the correct answer that we don't have to check. + ClassFinder allClassesFinder = new ClassFinder(trueNameFilter, trueClassFilter); + Set> pkgClasses = allClassesFinder.findClasses( + ClassFinder.class.getPackage().getName(), false); + Set> defaultClasses = allClassesFinder.findClasses(false); + assertArrayEquals(pkgClasses.toArray(), defaultClasses.toArray()); + } + + private static class FileAndPath { + String path; + File file; + public FileAndPath(String path, File file) { + this.file = file; + this.path = path; + } + } + + private static Class makeClass(String nestedPkgSuffix, + String className, long counter) throws ClassNotFoundException { + return Class.forName( + makePackageName(nestedPkgSuffix, counter) + "." + className + counter); + } + + private static String makePackageName(String nestedSuffix, long counter) { + return BASEPKG + counter + nestedSuffix; + } + + /** + * Compiles the test class with bogus code into a .class file. + * Unfortunately it's very tedious. + * @param counter Unique test counter. + * @param packageNameSuffix Package name suffix (e.g. ".suffix") for nesting, or "". + * @return The resulting .class file and the location in jar it is supposed to go to. + */ + private static FileAndPath compileTestClass(long counter, + String packageNameSuffix, String classNamePrefix) throws Exception { + classNamePrefix = classNamePrefix + counter; + String packageName = makePackageName(packageNameSuffix, counter); + String javaPath = basePath + classNamePrefix + ".java"; + String classPath = basePath + classNamePrefix + ".class"; + PrintStream source = new PrintStream(javaPath); + source.println("package " + packageName + ";"); + source.println("public class " + classNamePrefix + + " { public static void main(String[] args) { } };"); + source.close(); + JavaCompiler jc = ToolProvider.getSystemJavaCompiler(); + int result = jc.run(null, null, null, javaPath); + assertEquals(0, result); + File classFile = new File(classPath); + assertTrue(classFile.exists()); + return new FileAndPath(packageName.replace('.', '/') + '/', classFile); + } + + /** + * Makes a jar out of some class files. Unfortunately it's very tedious. + * @param filesInJar Files created via compileTestClass. + */ + private static void packageAndLoadJar(FileAndPath... filesInJar) throws Exception { + // First, write the bogus jar file. + String path = basePath + "jar" + jarCounter.incrementAndGet() + ".jar"; + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + FileOutputStream fos = new FileOutputStream(path); + JarOutputStream jarOutputStream = new JarOutputStream(fos, manifest); + // Directory entries for all packages have to be added explicitly for + // resources to be findable via ClassLoader. Directory entries must end + // with "/"; the initial one is expected to, also. + Set pathsInJar = new HashSet(); + for (FileAndPath fileAndPath : filesInJar) { + String pathToAdd = fileAndPath.path; + while (pathsInJar.add(pathToAdd)) { + int ix = pathToAdd.lastIndexOf('/', pathToAdd.length() - 2); + if (ix < 0) { + break; + } + pathToAdd = pathToAdd.substring(0, ix); + } + } + for (String pathInJar : pathsInJar) { + jarOutputStream.putNextEntry(new JarEntry(pathInJar)); + jarOutputStream.closeEntry(); + } + for (FileAndPath fileAndPath : filesInJar) { + File file = fileAndPath.file; + jarOutputStream.putNextEntry( + new JarEntry(fileAndPath.path + file.getName())); + byte[] allBytes = new byte[(int)file.length()]; + FileInputStream fis = new FileInputStream(file); + fis.read(allBytes); + fis.close(); + jarOutputStream.write(allBytes); + jarOutputStream.closeEntry(); + } + jarOutputStream.close(); + fos.close(); + + // Add the file to classpath. + File jarFile = new File(path); + assertTrue(jarFile.exists()); + URLClassLoader urlClassLoader = (URLClassLoader)ClassLoader.getSystemClassLoader(); + Method method = URLClassLoader.class + .getDeclaredMethod("addURL", new Class[] { URL.class }); + method.setAccessible(true); + method.invoke(urlClassLoader, new Object[] { jarFile.toURI().toURL() }); + } +}; diff --git a/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java b/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java new file mode 100644 index 0000000..b150d13 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestClusterBootOrder.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests the boot order indifference between regionserver and master + */ +@Category(MediumTests.class) +public class TestClusterBootOrder { + + private static final long SLEEP_INTERVAL = 1000; + private static final long SLEEP_TIME = 4000; + + private HBaseTestingUtility testUtil; + private LocalHBaseCluster cluster; + private RegionServerThread rs; + private MasterThread master; + + @Before + public void setUp() throws Exception { + testUtil = new HBaseTestingUtility(); + testUtil.startMiniDFSCluster(1); + testUtil.startMiniZKCluster(1); + testUtil.createRootDir(); //manually setup hbase dir to point to minidfscluster + cluster = new LocalHBaseCluster(testUtil.getConfiguration(), 0, 0); + } + + @After + public void tearDown() throws Exception { + cluster.shutdown(); + cluster.join(); + testUtil.shutdownMiniZKCluster(); + testUtil.shutdownMiniDFSCluster(); + } + + private void startRegionServer() throws Exception { + rs = cluster.addRegionServer(); + rs.start(); + + for (int i=0; i * SLEEP_INTERVAL < SLEEP_TIME ;i++) { + //we cannot block on wait for rs at this point , since master is not up. + Thread.sleep(SLEEP_INTERVAL); + assertTrue(rs.isAlive()); + } + } + + private void startMaster() throws Exception { + master = cluster.addMaster(); + master.start(); + + for (int i=0; i * SLEEP_INTERVAL < SLEEP_TIME ;i++) { + Thread.sleep(SLEEP_INTERVAL); + assertTrue(master.isAlive()); + } + } + + private void waitForClusterOnline() { + while (true) { + if (master.getMaster().isInitialized()) { + break; + } + try { + Thread.sleep(100); + } catch (InterruptedException ignored) { + // Keep waiting + } + } + rs.waitForServerOnline(); + } + + /** + * Tests launching the cluster by first starting regionserver, and then the master + * to ensure that it does not matter which is started first. + */ + @Test + public void testBootRegionServerFirst() throws Exception { + startRegionServer(); + startMaster(); + waitForClusterOnline(); + } + + /** + * Tests launching the cluster by first starting master, and then the regionserver + * to ensure that it does not matter which is started first. + */ + @Test + public void testBootMasterFirst() throws Exception { + startMaster(); + startRegionServer(); + waitForClusterOnline(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/TestCompare.java b/src/test/java/org/apache/hadoop/hbase/TestCompare.java new file mode 100644 index 0000000..a9e2509 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestCompare.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import junit.framework.TestCase; + +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +/** + * Test comparing HBase objects. + */ +@Category(SmallTests.class) +public class TestCompare extends TestCase { + + /** + * Sort of HRegionInfo. + */ + public void testHRegionInfo() { + HRegionInfo a = new HRegionInfo(Bytes.toBytes("a"), null, null); + HRegionInfo b = new HRegionInfo(Bytes.toBytes("b"), null, null); + assertTrue(a.compareTo(b) != 0); + HTableDescriptor t = new HTableDescriptor("t"); + byte [] midway = Bytes.toBytes("midway"); + a = new HRegionInfo(t.getName(), null, midway); + b = new HRegionInfo(t.getName(), midway, null); + assertTrue(a.compareTo(b) < 0); + assertTrue(b.compareTo(a) > 0); + assertEquals(a, a); + assertTrue(a.compareTo(a) == 0); + a = new HRegionInfo(t.getName(), Bytes.toBytes("a"), Bytes.toBytes("d")); + b = new HRegionInfo(t.getName(), Bytes.toBytes("e"), Bytes.toBytes("g")); + assertTrue(a.compareTo(b) < 0); + a = new HRegionInfo(t.getName(), Bytes.toBytes("aaaa"), Bytes.toBytes("dddd")); + b = new HRegionInfo(t.getName(), Bytes.toBytes("e"), Bytes.toBytes("g")); + assertTrue(a.compareTo(b) < 0); + a = new HRegionInfo(t.getName(), Bytes.toBytes("aaaa"), Bytes.toBytes("dddd")); + b = new HRegionInfo(t.getName(), Bytes.toBytes("aaaa"), Bytes.toBytes("eeee")); + assertTrue(a.compareTo(b) < 0); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java b/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java new file mode 100644 index 0000000..4679cf2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java @@ -0,0 +1,221 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + + +import java.io.IOException; +import java.util.List; + +import junit.framework.Assert; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test the draining servers feature. + * @see HBASE-4298 + */ +@Category(MediumTests.class) +public class TestDrainingServer { + private static final Log LOG = LogFactory.getLog(TestDrainingServer.class); + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + private static final byte [] TABLENAME = Bytes.toBytes("t"); + private static final byte [] FAMILY = Bytes.toBytes("f"); + private static final int COUNT_OF_REGIONS = HBaseTestingUtility.KEYS.length; + private static final int NB_SLAVES = 5; + + /** + * Spin up a cluster with a bunch of regions on it. + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(NB_SLAVES); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL); + HTableDescriptor htd = new HTableDescriptor(TABLENAME); + htd.addFamily(new HColumnDescriptor(FAMILY)); + TEST_UTIL.createMultiRegionsInMeta(TEST_UTIL.getConfiguration(), htd, + HBaseTestingUtility.KEYS); + // Make a mark for the table in the filesystem. + FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); + FSTableDescriptors. + createTableDescriptor(fs, FSUtils.getRootDir(TEST_UTIL.getConfiguration()), htd); + // Assign out the regions we just created. + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster(); + admin.disableTable(TABLENAME); + admin.enableTable(TABLENAME); + boolean ready = false; + while (!ready) { + ZKAssign.blockUntilNoRIT(zkw); + // Assert that every regionserver has some regions on it, else invoke the balancer. + ready = true; + for (int i = 0; i < NB_SLAVES; i++) { + HRegionServer hrs = cluster.getRegionServer(i); + if (hrs.getOnlineRegions().isEmpty()) { + ready = false; + break; + } + } + if (!ready) { + admin.balancer(); + Thread.sleep(100); + } + } + } + + private static HRegionServer setDrainingServer(final HRegionServer hrs) + throws KeeperException { + LOG.info("Making " + hrs.getServerName() + " the draining server; " + + "it has " + hrs.getNumberOfOnlineRegions() + " online regions"); + ZooKeeperWatcher zkw = hrs.getZooKeeper(); + String hrsDrainingZnode = + ZKUtil.joinZNode(zkw.drainingZNode, hrs.getServerName().toString()); + ZKUtil.createWithParents(zkw, hrsDrainingZnode); + return hrs; + } + + private static HRegionServer unsetDrainingServer(final HRegionServer hrs) + throws KeeperException { + ZooKeeperWatcher zkw = hrs.getZooKeeper(); + String hrsDrainingZnode = + ZKUtil.joinZNode(zkw.drainingZNode, hrs.getServerName().toString()); + ZKUtil.deleteNode(zkw, hrsDrainingZnode); + return hrs; + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Test adding server to draining servers and then move regions off it. + * Make sure that no regions are moved back to the draining server. + * @throws IOException + * @throws KeeperException + */ + @Test // (timeout=30000) + public void testDrainingServerOffloading() + throws IOException, KeeperException { + // I need master in the below. + HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); + HRegionInfo hriToMoveBack = null; + // Set first server as draining server. + HRegionServer drainingServer = + setDrainingServer(TEST_UTIL.getMiniHBaseCluster().getRegionServer(0)); + try { + final int regionsOnDrainingServer = + drainingServer.getNumberOfOnlineRegions(); + Assert.assertTrue(regionsOnDrainingServer > 0); + List hris = drainingServer.getOnlineRegions(); + for (HRegionInfo hri : hris) { + // Pass null and AssignmentManager will chose a random server BUT it + // should exclude draining servers. + master.move(hri.getEncodedNameAsBytes(), null); + // Save off region to move back. + hriToMoveBack = hri; + } + // Wait for regions to come back on line again. + waitForAllRegionsOnline(); + Assert.assertEquals(0, drainingServer.getNumberOfOnlineRegions()); + } finally { + unsetDrainingServer(drainingServer); + } + // Now we've unset the draining server, we should be able to move a region + // to what was the draining server. + master.move(hriToMoveBack.getEncodedNameAsBytes(), + Bytes.toBytes(drainingServer.getServerName().toString())); + // Wait for regions to come back on line again. + waitForAllRegionsOnline(); + Assert.assertEquals(1, drainingServer.getNumberOfOnlineRegions()); + } + + /** + * Test that draining servers are ignored even after killing regionserver(s). + * Verify that the draining server is not given any of the dead servers regions. + * @throws KeeperException + * @throws IOException + */ + @Test (timeout=30000) + public void testDrainingServerWithAbort() throws KeeperException, IOException { + // Add first server to draining servers up in zk. + HRegionServer drainingServer = + setDrainingServer(TEST_UTIL.getMiniHBaseCluster().getRegionServer(0)); + try { + final int regionsOnDrainingServer = + drainingServer.getNumberOfOnlineRegions(); + Assert.assertTrue(regionsOnDrainingServer > 0); + // Kill a few regionservers. + int aborted = 0; + final int numberToAbort = 2; + for (int i = 1; i < TEST_UTIL.getMiniHBaseCluster().countServedRegions(); i++) { + HRegionServer hrs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i); + if (hrs.getServerName().equals(drainingServer.getServerName())) continue; + hrs.abort("Aborting"); + aborted++; + if (aborted >= numberToAbort) break; + } + // Wait for regions to come back on line again. + waitForAllRegionsOnline(); + // Assert the draining server still has the same number of regions. + Assert.assertEquals(regionsOnDrainingServer, + drainingServer.getNumberOfOnlineRegions()); + } finally { + unsetDrainingServer(drainingServer); + } + } + + private void waitForAllRegionsOnline() { + while (TEST_UTIL.getMiniHBaseCluster().getMaster(). + getAssignmentManager().isRegionsInTransition()) { + Threads.sleep(10); + } + // Wait for regions to come back on line again. + while (!isAllRegionsOnline()) { + Threads.sleep(10); + } + } + + private boolean isAllRegionsOnline() { + return TEST_UTIL.getMiniHBaseCluster().countServedRegions() == + (COUNT_OF_REGIONS + 2 /*catalog regions*/); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestFSTableDescriptorForceCreation.java b/src/test/java/org/apache/hadoop/hbase/TestFSTableDescriptorForceCreation.java new file mode 100644 index 0000000..6bc7c32 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestFSTableDescriptorForceCreation.java @@ -0,0 +1,79 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.junit.*; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestFSTableDescriptorForceCreation { + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @Test + public void testShouldCreateNewTableDescriptorIfForcefulCreationIsFalse() + throws IOException { + final String name = "newTable2"; + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + Path rootdir = new Path(UTIL.getDataTestDir(), name); + HTableDescriptor htd = new HTableDescriptor(name); + + assertTrue("Should create new table descriptor", + FSTableDescriptors.createTableDescriptor(fs, rootdir, htd, false)); + } + + @Test + public void testShouldNotCreateTheSameTableDescriptorIfForcefulCreationIsFalse() + throws IOException { + final String name = "testAlreadyExists"; + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + // Cleanup old tests if any detrius laying around. + Path rootdir = new Path(UTIL.getDataTestDir(), name); + TableDescriptors htds = new FSTableDescriptors(fs, rootdir); + HTableDescriptor htd = new HTableDescriptor(name); + htds.add(htd); + assertFalse("Should not create new table descriptor", + FSTableDescriptors.createTableDescriptor(fs, rootdir, htd, false)); + } + + @Test + public void testShouldAllowForcefulCreationOfAlreadyExistingTableDescriptor() + throws Exception { + final String name = "createNewTableNew2"; + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + Path rootdir = new Path(UTIL.getDataTestDir(), name); + HTableDescriptor htd = new HTableDescriptor(name); + FSTableDescriptors.createTableDescriptor(fs, rootdir, htd, false); + assertTrue("Should create new table descriptor", + FSTableDescriptors.createTableDescriptor(fs, rootdir, htd, true)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java b/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java new file mode 100644 index 0000000..e725d43 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java @@ -0,0 +1,132 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestFullLogReconstruction { + + private final static HBaseTestingUtility + TEST_UTIL = new HBaseTestingUtility(); + + private final static byte[] TABLE_NAME = Bytes.toBytes("tabletest"); + private final static byte[] FAMILY = Bytes.toBytes("family"); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration c = TEST_UTIL.getConfiguration(); + c.setBoolean("dfs.support.append", true); + // quicker heartbeat interval for faster DN death notification + c.setInt("heartbeat.recheck.interval", 5000); + c.setInt("dfs.heartbeat.interval", 1); + c.setInt("dfs.socket.timeout", 5000); + // faster failover with cluster.shutdown();fs.close() idiom + c.setInt("ipc.client.connect.max.retries", 1); + c.setInt("dfs.client.block.recovery.retries", 1); + TEST_UTIL.startMiniCluster(2); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + } + + /** + * @throws java.lang.Exception + */ + @After + public void tearDown() throws Exception { + } + + /** + * Test the whole reconstruction loop. Build a table with regions aaa to zzz + * and load every one of them multiple times with the same date and do a flush + * at some point. Kill one of the region servers and scan the table. We should + * see all the rows. + * @throws Exception + */ + @Test (timeout=300000) + public void testReconstruction() throws Exception { + + HTable table = TEST_UTIL.createTable(TABLE_NAME, FAMILY); + + TEST_UTIL.createMultiRegions(table, Bytes.toBytes("family")); + + // Load up the table with simple rows and count them + int initialCount = TEST_UTIL.loadTable(table, FAMILY); + Scan scan = new Scan(); + ResultScanner results = table.getScanner(scan); + int count = 0; + for (Result res : results) { + count++; + } + results.close(); + + assertEquals(initialCount, count); + + for(int i = 0; i < 4; i++) { + TEST_UTIL.loadTable(table, FAMILY); + } + + TEST_UTIL.expireRegionServerSession(0); + scan = new Scan(); + results = table.getScanner(scan); + int newCount = 0; + for (Result res : results) { + newCount++; + } + assertEquals(count, newCount); + results.close(); + table.close(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java b/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java new file mode 100644 index 0000000..ad77e0a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestGlobalMemStoreSize.java @@ -0,0 +1,182 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional infomation + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.List; +import java.util.ArrayList; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test HBASE-3694 whether the GlobalMemStoreSize is the same as the summary + * of all the online region's MemStoreSize + */ +@Category(MediumTests.class) +public class TestGlobalMemStoreSize { + private final Log LOG = LogFactory.getLog(this.getClass().getName()); + private static int regionServerNum = 4; + private static int regionNum = 16; + // total region num = region num + root and meta regions + private static int totalRegionNum = regionNum+2; + + private HBaseTestingUtility TEST_UTIL; + private MiniHBaseCluster cluster; + + /** + * Test the global mem store size in the region server is equal to sum of each + * region's mem store size + * @throws Exception + */ + @Test + public void testGlobalMemStore() throws Exception { + // Start the cluster + LOG.info("Starting cluster"); + Configuration conf = HBaseConfiguration.create(); + conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000); + conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 5000); + TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(1, regionServerNum); + cluster = TEST_UTIL.getHBaseCluster(); + LOG.info("Waiting for active/ready master"); + cluster.waitForActiveAndReadyMaster(); + + // Create a table with regions + byte [] table = Bytes.toBytes("TestGlobalMemStoreSize"); + byte [] family = Bytes.toBytes("family"); + LOG.info("Creating table with " + regionNum + " regions"); + HTable ht = TEST_UTIL.createTable(table, family); + int numRegions = TEST_UTIL.createMultiRegions(conf, ht, family, + regionNum); + assertEquals(regionNum,numRegions); + waitForAllRegionsAssigned(); + + for (HRegionServer server : getOnlineRegionServers()) { + long globalMemStoreSize = 0; + for (HRegionInfo regionInfo : server.getOnlineRegions()) { + globalMemStoreSize += + server.getFromOnlineRegions(regionInfo.getEncodedName()). + getMemstoreSize().get(); + } + assertEquals(server.getRegionServerAccounting().getGlobalMemstoreSize(), + globalMemStoreSize); + } + + // check the global memstore size after flush + int i = 0; + for (HRegionServer server : getOnlineRegionServers()) { + LOG.info("Starting flushes on " + server.getServerName() + + ", size=" + server.getRegionServerAccounting().getGlobalMemstoreSize()); + + for (HRegionInfo regionInfo : server.getOnlineRegions()) { + HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName()); + flush(r, server); + } + LOG.info("Post flush on " + server.getServerName()); + long now = System.currentTimeMillis(); + long timeout = now + 1000; + while(server.getRegionServerAccounting().getGlobalMemstoreSize() != 0 && + timeout < System.currentTimeMillis()) { + Threads.sleep(10); + } + long size = server.getRegionServerAccounting().getGlobalMemstoreSize(); + if (size > 0) { + // If size > 0, see if its because the meta region got edits while + // our test was running.... + for (HRegionInfo regionInfo : server.getOnlineRegions()) { + HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName()); + long l = r.getMemstoreSize().longValue(); + if (l > 0) { + // Only meta could have edits at this stage. Give it another flush + // clear them. + assertTrue(regionInfo.isMetaRegion()); + LOG.info(r.toString() + " " + l + ", reflushing"); + r.flushcache(); + } + } + } + size = server.getRegionServerAccounting().getGlobalMemstoreSize(); + assertEquals("Server=" + server.getServerName() + ", i=" + i++, 0, size); + } + + ht.close(); + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Flush and log stats on flush + * @param r + * @param server + * @throws IOException + */ + private void flush(final HRegion r, final HRegionServer server) + throws IOException { + LOG.info("Flush " + r.toString() + " on " + server.getServerName() + + ", " + r.flushcache() + ", size=" + + server.getRegionServerAccounting().getGlobalMemstoreSize()); + } + + /** figure out how many regions are currently being served. */ + private int getRegionCount() throws IOException { + int total = 0; + for (HRegionServer server : getOnlineRegionServers()) { + total += server.getOnlineRegions().size(); + } + return total; + } + + private List getOnlineRegionServers() { + List list = new ArrayList(); + for (JVMClusterUtil.RegionServerThread rst : + cluster.getRegionServerThreads()) { + if (rst.getRegionServer().isOnline()) { + list.add(rst.getRegionServer()); + } + } + return list; + } + + /** + * Wait until all the regions are assigned. + */ + private void waitForAllRegionsAssigned() throws IOException { + while (getRegionCount() < totalRegionNum) { + LOG.debug("Waiting for there to be "+totalRegionNum+" regions, but there are " + getRegionCount() + " right now."); + try { + Thread.sleep(100); + } catch (InterruptedException e) {} + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java b/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java new file mode 100644 index 0000000..67602c5 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestHBaseFileSystem.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.net.URI; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Progressable; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestHBaseFileSystem { + public static final Log LOG = LogFactory.getLog(TestHBaseFileSystem.class); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static Configuration conf; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf = TEST_UTIL.getConfiguration(); + conf.setBoolean("dfs.support.append", true); + // The below config supported by 0.20-append and CDH3b2 + conf.setInt("dfs.client.block.recovery.retries", 2); + TEST_UTIL.startMiniDFSCluster(3); + Path hbaseRootDir = + TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase")); + LOG.info("hbase.rootdir=" + hbaseRootDir); + conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString()); + conf.setInt("hdfs.client.retries.number", 10); + HBaseFileSystem.setRetryCounts(conf); + } + + + @Test + public void testNonIdempotentOpsWithRetries() throws IOException { + LOG.info("testNonIdempotentOpsWithRetries"); + + Path rootDir = new Path(TestHBaseFileSystem.conf.get(HConstants.HBASE_DIR)); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + // Create a Region + assertTrue(HBaseFileSystem.createPathOnFileSystem(fs, rootDir, + true) != null); + + boolean result = HBaseFileSystem.makeDirOnFileSystem(new MockFileSystemForCreate(), + new Path("/a")); + assertTrue("Couldn't create the directory", result); + + try { + HBaseFileSystem.createPathOnFileSystem(new MockFileSystemForCreate(), + new Path("/A"), false); + assertTrue(false);// control should not come here. + } catch (Exception e) { + LOG.info(e); + } + + result = HBaseFileSystem.renameDirForFileSystem(new MockFileSystem(), new Path("/a"), + new Path("/b")); + assertTrue("Couldn't rename the directory", result); + + result = HBaseFileSystem.deleteDirFromFileSystem(new MockFileSystem(), + new Path("/a")); + + assertTrue("Couldn't delete the directory", result); + fs.delete(rootDir, true); + } + + static class MockFileSystemForCreate extends MockFileSystem { + @Override + public boolean exists(Path path) { + if ("/A".equals(path.toString())) return true; + return false; + } + } + + /** + * a mock fs which throws exception for first 3 times, and then process the call (returns the + * excepted result). + */ + static class MockFileSystem extends FileSystem { + int retryCount; + final static int successRetryCount = 3; + + public MockFileSystem() { + retryCount = 0; + } + + @Override + public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2) throws IOException { + throw new IOException(""); + } + + @Override + public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3, + short arg4, long arg5, Progressable arg6) throws IOException { + LOG.debug("Create, " + retryCount); + if (retryCount++ < successRetryCount) throw new IOException("Something bad happen"); + return null; + } + + @Override + public boolean delete(Path arg0) throws IOException { + if (retryCount++ < successRetryCount) throw new IOException("Something bad happen"); + return true; + } + + @Override + public boolean delete(Path arg0, boolean arg1) throws IOException { + if (retryCount++ < successRetryCount) throw new IOException("Something bad happen"); + return true; + } + + @Override + public FileStatus getFileStatus(Path arg0) throws IOException { + FileStatus fs = new FileStatus(); + return fs; + } + + @Override + public boolean exists(Path path) { + return true; + } + + @Override + public URI getUri() { + throw new RuntimeException("Something bad happen"); + } + + @Override + public Path getWorkingDirectory() { + throw new RuntimeException("Something bad happen"); + } + + @Override + public FileStatus[] listStatus(Path arg0) throws IOException { + throw new IOException("Something bad happen"); + } + + @Override + public boolean mkdirs(Path arg0, FsPermission arg1) throws IOException { + LOG.debug("mkdirs, " + retryCount); + if (retryCount++ < successRetryCount) throw new IOException("Something bad happen"); + return true; + } + + @Override + public FSDataInputStream open(Path arg0, int arg1) throws IOException { + throw new IOException("Something bad happen"); + } + + @Override + public boolean rename(Path arg0, Path arg1) throws IOException { + LOG.debug("rename, " + retryCount); + if (retryCount++ < successRetryCount) throw new IOException("Something bad happen"); + return true; + } + + @Override + public void setWorkingDirectory(Path arg0) { + throw new RuntimeException("Something bad happen"); + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java new file mode 100644 index 0000000..3ed5d52 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java @@ -0,0 +1,238 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test our testing utility class + */ +@Category(LargeTests.class) +public class TestHBaseTestingUtility { + private final Log LOG = LogFactory.getLog(this.getClass()); + + /** + * Basic sanity test that spins up multiple HDFS and HBase clusters that share + * the same ZK ensemble. We then create the same table in both and make sure + * that what we insert in one place doesn't end up in the other. + * @throws Exception + */ + @Test (timeout=180000) + public void testMultiClusters() throws Exception { + // Create three clusters + + // Cluster 1. + HBaseTestingUtility htu1 = new HBaseTestingUtility(); + // Set a different zk path for each cluster + htu1.getConfiguration().set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); + htu1.startMiniZKCluster(); + + // Cluster 2 + HBaseTestingUtility htu2 = new HBaseTestingUtility(); + htu2.getConfiguration().set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); + htu2.getConfiguration().set(HConstants.ZOOKEEPER_CLIENT_PORT, + htu1.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT, "-1")); + htu2.setZkCluster(htu1.getZkCluster()); + + // Cluster 3; seed it with the conf from htu1 so we pickup the 'right' + // zk cluster config; it is set back into the config. as part of the + // start of minizkcluster. + HBaseTestingUtility htu3 = new HBaseTestingUtility(); + htu3.getConfiguration().set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3"); + htu3.getConfiguration().set(HConstants.ZOOKEEPER_CLIENT_PORT, + htu1.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT, "-1")); + htu3.setZkCluster(htu1.getZkCluster()); + + try { + htu1.startMiniCluster(); + htu2.startMiniCluster(); + htu3.startMiniCluster(); + + final byte[] TABLE_NAME = Bytes.toBytes("test"); + final byte[] FAM_NAME = Bytes.toBytes("fam"); + final byte[] ROW = Bytes.toBytes("row"); + final byte[] QUAL_NAME = Bytes.toBytes("qual"); + final byte[] VALUE = Bytes.toBytes("value"); + + HTable table1 = htu1.createTable(TABLE_NAME, FAM_NAME); + HTable table2 = htu2.createTable(TABLE_NAME, FAM_NAME); + + Put put = new Put(ROW); + put.add(FAM_NAME, QUAL_NAME, VALUE); + table1.put(put); + + Get get = new Get(ROW); + get.addColumn(FAM_NAME, QUAL_NAME); + Result res = table1.get(get); + assertEquals(1, res.size()); + + res = table2.get(get); + assertEquals(0, res.size()); + + table1.close(); + table2.close(); + + } finally { + htu3.shutdownMiniCluster(); + htu2.shutdownMiniCluster(); + htu1.shutdownMiniCluster(); + } + } + + @Test public void testMiniCluster() throws Exception { + HBaseTestingUtility hbt = new HBaseTestingUtility(); + + MiniHBaseCluster cluster = hbt.startMiniCluster(); + try { + assertEquals(1, cluster.getLiveRegionServerThreads().size()); + } finally { + hbt.shutdownMiniCluster(); + } + } + + /** + * Test that we can start and stop multiple time a cluster + * with the same HBaseTestingUtility. + */ + @Test public void testMultipleStartStop() throws Exception{ + HBaseTestingUtility htu1 = new HBaseTestingUtility(); + Path foo = new Path("foo"); + + htu1.startMiniCluster(); + htu1.getDFSCluster().getFileSystem().create(foo); + assertTrue( htu1.getDFSCluster().getFileSystem().exists(foo)); + htu1.shutdownMiniCluster(); + + htu1.startMiniCluster(); + assertFalse( htu1.getDFSCluster().getFileSystem().exists(foo)); + htu1.getDFSCluster().getFileSystem().create(foo); + assertTrue( htu1.getDFSCluster().getFileSystem().exists(foo)); + htu1.shutdownMiniCluster(); + } + + + @Test public void testMiniZooKeeper() throws Exception { + HBaseTestingUtility hbt = new HBaseTestingUtility(); + MiniZooKeeperCluster cluster1 = hbt.startMiniZKCluster(); + try { + assertEquals(0, cluster1.getBackupZooKeeperServerNum()); + assertTrue((cluster1.killCurrentActiveZooKeeperServer() == -1)); + } finally { + hbt.shutdownMiniZKCluster(); + } + + // set up zookeeper cluster with 5 zk servers + MiniZooKeeperCluster cluster2 = hbt.startMiniZKCluster(5); + int defaultClientPort = 21818; + cluster2.setDefaultClientPort(defaultClientPort); + try { + assertEquals(4, cluster2.getBackupZooKeeperServerNum()); + + // killing the current active zk server + assertTrue((cluster2.killCurrentActiveZooKeeperServer() >= defaultClientPort)); + assertTrue((cluster2.killCurrentActiveZooKeeperServer() >= defaultClientPort)); + assertEquals(2, cluster2.getBackupZooKeeperServerNum()); + assertEquals(3, cluster2.getZooKeeperServerNum()); + + // killing the backup zk servers + cluster2.killOneBackupZooKeeperServer(); + cluster2.killOneBackupZooKeeperServer(); + assertEquals(0, cluster2.getBackupZooKeeperServerNum()); + assertEquals(1, cluster2.getZooKeeperServerNum()); + + // killing the last zk server + assertTrue((cluster2.killCurrentActiveZooKeeperServer() == -1)); + // this should do nothing. + cluster2.killOneBackupZooKeeperServer(); + assertEquals(-1, cluster2.getBackupZooKeeperServerNum()); + assertEquals(0, cluster2.getZooKeeperServerNum()); + } finally { + hbt.shutdownMiniZKCluster(); + } + } + + @Test public void testMiniDFSCluster() throws Exception { + HBaseTestingUtility hbt = new HBaseTestingUtility(); + MiniDFSCluster cluster = hbt.startMiniDFSCluster(1); + FileSystem dfs = cluster.getFileSystem(); + Path dir = new Path("dir"); + Path qualifiedDir = dfs.makeQualified(dir); + LOG.info("dir=" + dir + ", qualifiedDir=" + qualifiedDir); + assertFalse(dfs.exists(qualifiedDir)); + assertTrue(dfs.mkdirs(qualifiedDir)); + assertTrue(dfs.delete(qualifiedDir, true)); + hbt.shutdownMiniCluster(); + } + + @Test public void testSetupClusterTestBuildDir() throws Exception { + HBaseTestingUtility hbt = new HBaseTestingUtility(); + Path testdir = hbt.getClusterTestDir(); + LOG.info("uuid-subdir=" + testdir); + FileSystem fs = hbt.getTestFileSystem(); + + assertFalse(fs.exists(testdir)); + + hbt.startMiniDFSCluster(1); + assertTrue(fs.exists(testdir)); + + hbt.shutdownMiniCluster(); + assertFalse(fs.exists(testdir)); + + } + + @Test public void testTestDir() throws Exception { + HBaseTestingUtility hbt = new HBaseTestingUtility(); + Path testdir = hbt.getDataTestDir(); + LOG.info("testdir=" + testdir); + FileSystem fs = hbt.getTestFileSystem(); + assertTrue(!fs.exists(testdir)); + assertTrue(fs.mkdirs(testdir)); + assertTrue(hbt.cleanupTestDir()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java b/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java new file mode 100644 index 0000000..ea69406 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java @@ -0,0 +1,69 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.HashMap; +import java.util.Map; + +import static junit.framework.Assert.assertEquals; + +@Category(SmallTests.class) +public class TestHDFSBlocksDistribution { + @Test + public void testAddHostsAndBlockWeight() throws Exception { + HDFSBlocksDistribution distribution = new HDFSBlocksDistribution(); + distribution.addHostsAndBlockWeight(null, 100); + assertEquals("Expecting no hosts weights", 0, distribution.getHostAndWeights().size()); + distribution.addHostsAndBlockWeight(new String[0], 100); + assertEquals("Expecting no hosts weights", 0, distribution.getHostAndWeights().size()); + distribution.addHostsAndBlockWeight(new String[] {"test"}, 101); + assertEquals("Should be one host", 1, distribution.getHostAndWeights().size()); + distribution.addHostsAndBlockWeight(new String[] {"test"}, 202); + assertEquals("Should be one host", 1, distribution.getHostAndWeights().size()); + assertEquals("test host should have weight 303", 303, + distribution.getHostAndWeights().get("test").getWeight()); + distribution.addHostsAndBlockWeight(new String[] {"testTwo"}, 222); + assertEquals("Should be two hosts", 2, distribution.getHostAndWeights().size()); + assertEquals("Total weight should be 525", 525, distribution.getUniqueBlocksTotalWeight()); + } + + public class MockHDFSBlocksDistribution extends HDFSBlocksDistribution { + public Map getHostAndWeights() { + HashMap map = new HashMap(); + map.put("test", new HostAndWeight(null, 100)); + return map; + } + + } + + @Test + public void testAdd() throws Exception { + HDFSBlocksDistribution distribution = new HDFSBlocksDistribution(); + distribution.add(new MockHDFSBlocksDistribution()); + assertEquals("Expecting no hosts weights", 0, distribution.getHostAndWeights().size()); + distribution.addHostsAndBlockWeight(new String[]{"test"}, 10); + assertEquals("Should be one host", 1, distribution.getHostAndWeights().size()); + distribution.add(new MockHDFSBlocksDistribution()); + assertEquals("Should be one host", 1, distribution.getHostAndWeights().size()); + assertEquals("Total weight should be 10", 10, distribution.getUniqueBlocksTotalWeight()); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/TestHRegionLocation.java b/src/test/java/org/apache/hadoop/hbase/TestHRegionLocation.java new file mode 100644 index 0000000..a9b8f94 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestHRegionLocation.java @@ -0,0 +1,86 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestHRegionLocation { + /** + * HRegionLocations are equal if they have the same 'location' -- i.e. host and + * port -- even if they are carrying different regions. Verify that is indeed + * the case. + */ + @Test + public void testHashAndEqualsCode() { + ServerName hsa1 = new ServerName("localhost", 1234, -1L); + HRegionLocation hrl1 = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, + hsa1.getHostname(), hsa1.getPort()); + HRegionLocation hrl2 = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, + hsa1.getHostname(), hsa1.getPort()); + assertEquals(hrl1.hashCode(), hrl2.hashCode()); + assertTrue(hrl1.equals(hrl2)); + HRegionLocation hrl3 = new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, + hsa1.getHostname(), hsa1.getPort()); + assertNotSame(hrl1, hrl3); + // They are equal because they have same location even though they are + // carrying different regions. + assertTrue(hrl1.equals(hrl3)); + ServerName hsa2 = new ServerName("localhost", 12345, -1L); + HRegionLocation hrl4 = new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, + hsa2.getHostname(), hsa2.getPort()); + // These have same HRI but different locations so should be different. + assertFalse(hrl3.equals(hrl4)); + } + + @Test + public void testToString() { + ServerName hsa1 = new ServerName("localhost", 1234, -1L); + HRegionLocation hrl1 = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, + hsa1.getHostname(), hsa1.getPort()); + System.out.println(hrl1.toString()); + } + + @Test + public void testCompareTo() { + ServerName hsa1 = new ServerName("localhost", 1234, -1L); + HRegionLocation hsl1 = + new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, hsa1.getHostname(), hsa1.getPort()); + ServerName hsa2 = new ServerName("localhost", 1235, -1L); + HRegionLocation hsl2 = + new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, hsa2.getHostname(), hsa2.getPort()); + assertTrue(hsl1.compareTo(hsl1) == 0); + assertTrue(hsl2.compareTo(hsl2) == 0); + int compare1 = hsl1.compareTo(hsl2); + int compare2 = hsl2.compareTo(hsl1); + assertTrue((compare1 > 0)? compare2 < 0: compare2 > 0); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestHServerAddress.java b/src/test/java/org/apache/hadoop/hbase/TestHServerAddress.java new file mode 100644 index 0000000..1a69337 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestHServerAddress.java @@ -0,0 +1,90 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.hbase.util.Writables; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests for {@link HServerAddress} + */ +@Category(SmallTests.class) +public class TestHServerAddress { + @Test + public void testHashCode() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerAddress hsa2 = new HServerAddress("localhost", 1234); + assertEquals(hsa1.hashCode(), hsa2.hashCode()); + HServerAddress hsa3 = new HServerAddress("localhost", 1235); + assertNotSame(hsa1.hashCode(), hsa3.hashCode()); + } + + @Test + public void testHServerAddress() { + new HServerAddress(); + } + + @Test + public void testHServerAddressInetSocketAddress() { + HServerAddress hsa1 = + new HServerAddress(new InetSocketAddress("localhost", 1234)); + System.out.println(hsa1.toString()); + } + + @Test + public void testHServerAddressString() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerAddress hsa2 = + new HServerAddress(new InetSocketAddress("localhost", 1234)); + assertTrue(hsa1.equals(hsa2)); + } + + @Test + public void testHServerAddressHServerAddress() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerAddress hsa2 = new HServerAddress(hsa1); + assertEquals(hsa1, hsa2); + } + + @Test + public void testReadFields() throws IOException { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerAddress hsa2 = new HServerAddress("localhost", 1235); + byte [] bytes = Writables.getBytes(hsa1); + HServerAddress deserialized = + (HServerAddress)Writables.getWritable(bytes, new HServerAddress()); + assertEquals(hsa1, deserialized); + bytes = Writables.getBytes(hsa2); + deserialized = + (HServerAddress)Writables.getWritable(bytes, new HServerAddress()); + assertNotSame(hsa1, deserialized); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestHServerInfo.java b/src/test/java/org/apache/hadoop/hbase/TestHServerInfo.java new file mode 100644 index 0000000..4780221 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestHServerInfo.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.*; + +import java.io.IOException; + +import org.apache.hadoop.hbase.util.Writables; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestHServerInfo { + + @Test + public void testHashCodeAndEquals() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerInfo hsi1 = new HServerInfo(hsa1, 1L, 5678); + HServerInfo hsi2 = new HServerInfo(hsa1, 1L, 5678); + HServerInfo hsi3 = new HServerInfo(hsa1, 2L, 5678); + HServerInfo hsi4 = new HServerInfo(hsa1, 1L, 5677); + HServerAddress hsa2 = new HServerAddress("localhost", 1235); + HServerInfo hsi5 = new HServerInfo(hsa2, 1L, 5678); + assertEquals(hsi1.hashCode(), hsi2.hashCode()); + assertTrue(hsi1.equals(hsi2)); + assertNotSame(hsi1.hashCode(), hsi3.hashCode()); + assertFalse(hsi1.equals(hsi3)); + assertNotSame(hsi1.hashCode(), hsi4.hashCode()); + assertFalse(hsi1.equals(hsi4)); + assertNotSame(hsi1.hashCode(), hsi5.hashCode()); + assertFalse(hsi1.equals(hsi5)); + } + + @Test + public void testHServerInfoHServerInfo() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerInfo hsi1 = new HServerInfo(hsa1, 1L, 5678); + HServerInfo hsi2 = new HServerInfo(hsi1); + assertEquals(hsi1, hsi2); + } + + @Test + public void testGetServerAddress() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerInfo hsi1 = new HServerInfo(hsa1, 1L, 5678); + assertEquals(hsi1.getServerAddress(), hsa1); + } + + @Test + public void testToString() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerInfo hsi1 = new HServerInfo(hsa1, 1L, 5678); + System.out.println(hsi1.toString()); + } + + @Test + public void testReadFields() throws IOException { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerInfo hsi1 = new HServerInfo(hsa1, 1L, 5678); + HServerAddress hsa2 = new HServerAddress("localhost", 1235); + HServerInfo hsi2 = new HServerInfo(hsa2, 1L, 5678); + byte [] bytes = Writables.getBytes(hsi1); + HServerInfo deserialized = + (HServerInfo)Writables.getWritable(bytes, new HServerInfo()); + assertEquals(hsi1, deserialized); + bytes = Writables.getBytes(hsi2); + deserialized = (HServerInfo)Writables.getWritable(bytes, new HServerInfo()); + assertNotSame(hsa1, deserialized); + } + + @Test + public void testCompareTo() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerInfo hsi1 = new HServerInfo(hsa1, 1L, 5678); + HServerAddress hsa2 = new HServerAddress("localhost", 1235); + HServerInfo hsi2 = new HServerInfo(hsa2, 1L, 5678); + assertTrue(hsi1.compareTo(hsi1) == 0); + assertTrue(hsi2.compareTo(hsi2) == 0); + int compare1 = hsi1.compareTo(hsi2); + int compare2 = hsi2.compareTo(hsi1); + assertTrue((compare1 > 0)? compare2 < 0: compare2 > 0); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java b/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java new file mode 100644 index 0000000..586b7af --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.SampleRegionWALObserver; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test setting values in the descriptor + */ +@Category(SmallTests.class) +public class TestHTableDescriptor { + final static Log LOG = LogFactory.getLog(TestHTableDescriptor.class); + + /** + * Test cps in the table description + * @throws Exception + */ + @Test + public void testGetSetRemoveCP() throws Exception { + HTableDescriptor desc = new HTableDescriptor("table"); + // simple CP + String className = BaseRegionObserver.class.getName(); + // add and check that it is present + desc.addCoprocessor(className); + assertTrue(desc.hasCoprocessor(className)); + // remove it and check that it is gone + desc.removeCoprocessor(className); + assertFalse(desc.hasCoprocessor(className)); + } + + /** + * Test cps in the table description + * @throws Exception + */ + @Test + public void testSetListRemoveCP() throws Exception { + HTableDescriptor desc = new HTableDescriptor("testGetSetRemoveCP"); + // simple CP + String className1 = BaseRegionObserver.class.getName(); + String className2 = SampleRegionWALObserver.class.getName(); + // Check that any coprocessor is present. + assertTrue(desc.getCoprocessors().size() == 0); + + // Add the 1 coprocessor and check if present. + desc.addCoprocessor(className1); + assertTrue(desc.getCoprocessors().size() == 1); + assertTrue(desc.getCoprocessors().contains(className1)); + + // Add the 2nd coprocessor and check if present. + // remove it and check that it is gone + desc.addCoprocessor(className2); + assertTrue(desc.getCoprocessors().size() == 2); + assertTrue(desc.getCoprocessors().contains(className2)); + + // Remove one and check + desc.removeCoprocessor(className1); + assertTrue(desc.getCoprocessors().size() == 1); + assertFalse(desc.getCoprocessors().contains(className1)); + assertTrue(desc.getCoprocessors().contains(className2)); + + // Remove the last and check + desc.removeCoprocessor(className2); + assertTrue(desc.getCoprocessors().size() == 0); + assertFalse(desc.getCoprocessors().contains(className1)); + assertFalse(desc.getCoprocessors().contains(className2)); + } + + /** + * Test that we add and remove strings from settings properly. + * @throws Exception + */ + @Test + public void testRemoveString() throws Exception { + HTableDescriptor desc = new HTableDescriptor("table"); + String key = "Some"; + String value = "value"; + desc.setValue(key, value); + assertEquals(value, desc.getValue(key)); + desc.remove(key); + assertEquals(null, desc.getValue(key)); + } + + /** + * Test default value handling for maxFileSize + */ + @Test + public void testGetMaxFileSize() { + HTableDescriptor desc = new HTableDescriptor("table"); + assertEquals(-1, desc.getMaxFileSize()); + desc.setMaxFileSize(1111L); + assertEquals(1111L, desc.getMaxFileSize()); + } + + /** + * Test default value handling for memStoreFlushSize + */ + @Test + public void testGetMemStoreFlushSize() { + HTableDescriptor desc = new HTableDescriptor("table"); + assertEquals(-1, desc.getMemStoreFlushSize()); + desc.setMemStoreFlushSize(1111L); + assertEquals(1111L, desc.getMemStoreFlushSize()); + } + + String legalTableNames[] = { "foo", "with-dash_under.dot", "_under_start_ok", }; + String illegalTableNames[] = { ".dot_start_illegal", "-dash_start_illegal", "spaces not ok" }; + + @Test + public void testLegalHTableNames() { + for (String tn : legalTableNames) { + HTableDescriptor.isLegalTableName(Bytes.toBytes(tn)); + } + } + + @Test + public void testIllegalHTableNames() { + for (String tn : illegalTableNames) { + try { + HTableDescriptor.isLegalTableName(Bytes.toBytes(tn)); + fail("invalid tablename " + tn + " should have failed"); + } catch (Exception e) { + // expected + } + } + } + + @Test + public void testLegalHTableNamesRegex() { + for (String tn : legalTableNames) { + LOG.info("Testing: '" + tn + "'"); + assertTrue(Pattern.matches(HTableDescriptor.VALID_USER_TABLE_REGEX, tn)); + } + } + + @Test + public void testIllegalHTableNamesRegex() { + for (String tn : illegalTableNames) { + LOG.info("Testing: '" + tn + "'"); + assertFalse(Pattern.matches(HTableDescriptor.VALID_USER_TABLE_REGEX, tn)); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java b/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java new file mode 100644 index 0000000..f7c90d4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java @@ -0,0 +1,149 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.net.URL; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertTrue; + +/** + * Testing, info servers are disabled. This test enables then and checks that + * they serve pages. + */ +@Category(MediumTests.class) +public class TestInfoServers { + static final Log LOG = LogFactory.getLog(TestInfoServers.class); + private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void beforeClass() throws Exception { + // The info servers do not run in tests by default. + // Set them to ephemeral ports so they will start + UTIL.getConfiguration().setInt("hbase.master.info.port", 0); + UTIL.getConfiguration().setInt("hbase.regionserver.info.port", 0); + UTIL.getConfiguration().setBoolean("hbase.master.ui.readonly", true); + UTIL.startMiniCluster(); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + /** + * @throws Exception + */ + @Test + public void testInfoServersRedirect() throws Exception { + // give the cluster time to start up + new HTable(UTIL.getConfiguration(), ".META.").close(); + int port = UTIL.getHBaseCluster().getMaster().getInfoServer().getPort(); + assertContainsContent(new URL("http://localhost:" + port + + "/index.html"), "master-status"); + port = UTIL.getHBaseCluster().getRegionServerThreads().get(0).getRegionServer(). + getInfoServer().getPort(); + assertContainsContent(new URL("http://localhost:" + port + + "/index.html"), "rs-status"); + } + + /** + * Test that the status pages in the minicluster load properly. + * + * This is somewhat a duplicate of TestRSStatusServlet and + * TestMasterStatusServlet, but those are true unit tests + * whereas this uses a cluster. + */ + @Test + public void testInfoServersStatusPages() throws Exception { + // give the cluster time to start up + new HTable(UTIL.getConfiguration(), ".META.").close(); + int port = UTIL.getHBaseCluster().getMaster().getInfoServer().getPort(); + assertContainsContent(new URL("http://localhost:" + port + + "/master-status"), "META"); + port = UTIL.getHBaseCluster().getRegionServerThreads().get(0).getRegionServer(). + getInfoServer().getPort(); + assertContainsContent(new URL("http://localhost:" + port + + "/rs-status"), "META"); + } + + @Test + public void testMasterServerReadOnly() throws Exception { + String sTableName = "testMasterServerReadOnly"; + byte[] tableName = Bytes.toBytes(sTableName); + byte[] cf = Bytes.toBytes("d"); + UTIL.createTable(tableName, cf); + new HTable(UTIL.getConfiguration(), tableName).close(); + int port = UTIL.getHBaseCluster().getMaster().getInfoServer().getPort(); + assertDoesNotContainContent( + new URL("http://localhost:" + port + "/table.jsp?name=" + sTableName + "&action=split&key="), + "Table action request accepted"); + assertDoesNotContainContent( + new URL("http://localhost:" + port + "/table.jsp?name=" + sTableName), + "Actions:"); + } + + private void assertContainsContent(final URL u, final String expected) + throws IOException { + LOG.info("Testing " + u.toString() + " has " + expected); + String content = getUrlContent(u); + assertTrue("expected=" + expected + ", content=" + content, + content.contains(expected)); + } + + + + private void assertDoesNotContainContent(final URL u, final String expected) + throws IOException { + LOG.info("Testing " + u.toString() + " has " + expected); + String content = getUrlContent(u); + assertTrue("Does Not Contain =" + expected + ", content=" + content, + !content.contains(expected)); + } + + private String getUrlContent(URL u) throws IOException { + java.net.URLConnection c = u.openConnection(); + c.connect(); + StringBuilder sb = new StringBuilder(); + BufferedInputStream bis = new BufferedInputStream(c.getInputStream()); + byte [] bytes = new byte[1024]; + for (int read = -1; (read = bis.read(bytes)) != -1;) { + sb.append(new String(bytes, 0, read)); + } + bis.close(); + String content = sb.toString(); + return content; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java new file mode 100644 index 0000000..fe3dfeb --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java @@ -0,0 +1,510 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.util.Set; +import java.util.TreeSet; + +import junit.framework.TestCase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue.KVComparator; +import org.apache.hadoop.hbase.KeyValue.MetaComparator; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestKeyValue extends TestCase { + private final Log LOG = LogFactory.getLog(this.getClass().getName()); + + public void testColumnCompare() throws Exception { + final byte [] a = Bytes.toBytes("aaa"); + byte [] family1 = Bytes.toBytes("abc"); + byte [] qualifier1 = Bytes.toBytes("def"); + byte [] family2 = Bytes.toBytes("abcd"); + byte [] qualifier2 = Bytes.toBytes("ef"); + + KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, Type.Put, a); + assertFalse(aaa.matchingColumn(family2, qualifier2)); + assertTrue(aaa.matchingColumn(family1, qualifier1)); + aaa = new KeyValue(a, family2, qualifier2, 0L, Type.Put, a); + assertFalse(aaa.matchingColumn(family1, qualifier1)); + assertTrue(aaa.matchingColumn(family2,qualifier2)); + byte [] nullQualifier = new byte[0]; + aaa = new KeyValue(a, family1, nullQualifier, 0L, Type.Put, a); + assertTrue(aaa.matchingColumn(family1,null)); + assertFalse(aaa.matchingColumn(family2,qualifier2)); + } + + /** + * Test a corner case when the family qualifier is a prefix of the + * column qualifier. + */ + public void testColumnCompare_prefix() throws Exception { + final byte [] a = Bytes.toBytes("aaa"); + byte [] family1 = Bytes.toBytes("abc"); + byte [] qualifier1 = Bytes.toBytes("def"); + byte [] family2 = Bytes.toBytes("ab"); + byte [] qualifier2 = Bytes.toBytes("def"); + + KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, Type.Put, a); + assertFalse(aaa.matchingColumn(family2, qualifier2)); + } + + public void testBasics() throws Exception { + LOG.info("LOWKEY: " + KeyValue.LOWESTKEY.toString()); + check(Bytes.toBytes(getName()), + Bytes.toBytes(getName()), Bytes.toBytes(getName()), 1, + Bytes.toBytes(getName())); + // Test empty value and empty column -- both should work. (not empty fam) + check(Bytes.toBytes(getName()), Bytes.toBytes(getName()), null, 1, null); + check(HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes(getName()), null, 1, null); + } + + private void check(final byte [] row, final byte [] family, byte [] qualifier, + final long timestamp, final byte [] value) { + KeyValue kv = new KeyValue(row, family, qualifier, timestamp, value); + assertTrue(Bytes.compareTo(kv.getRow(), row) == 0); + assertTrue(kv.matchingColumn(family, qualifier)); + // Call toString to make sure it works. + LOG.info(kv.toString()); + } + + public void testPlainCompare() throws Exception { + final byte [] a = Bytes.toBytes("aaa"); + final byte [] b = Bytes.toBytes("bbb"); + final byte [] fam = Bytes.toBytes("col"); + final byte [] qf = Bytes.toBytes("umn"); +// final byte [] column = Bytes.toBytes("col:umn"); + KeyValue aaa = new KeyValue(a, fam, qf, a); + KeyValue bbb = new KeyValue(b, fam, qf, b); + byte [] keyabb = aaa.getKey(); + byte [] keybbb = bbb.getKey(); + assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) < 0); + assertTrue(KeyValue.KEY_COMPARATOR.compare(keyabb, 0, keyabb.length, keybbb, + 0, keybbb.length) < 0); + assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) > 0); + assertTrue(KeyValue.KEY_COMPARATOR.compare(keybbb, 0, keybbb.length, keyabb, + 0, keyabb.length) > 0); + // Compare breaks if passed same ByteBuffer as both left and right arguments. + assertTrue(KeyValue.COMPARATOR.compare(bbb, bbb) == 0); + assertTrue(KeyValue.KEY_COMPARATOR.compare(keybbb, 0, keybbb.length, keybbb, + 0, keybbb.length) == 0); + assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0); + assertTrue(KeyValue.KEY_COMPARATOR.compare(keyabb, 0, keyabb.length, keyabb, + 0, keyabb.length) == 0); + // Do compare with different timestamps. + aaa = new KeyValue(a, fam, qf, 1, a); + bbb = new KeyValue(a, fam, qf, 2, a); + assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) > 0); + assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) < 0); + assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0); + // Do compare with different types. Higher numbered types -- Delete + // should sort ahead of lower numbers; i.e. Put + aaa = new KeyValue(a, fam, qf, 1, KeyValue.Type.Delete, a); + bbb = new KeyValue(a, fam, qf, 1, a); + assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) < 0); + assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) > 0); + assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0); + } + + public void testMoreComparisons() throws Exception { + // Root compares + long now = System.currentTimeMillis(); + KeyValue a = new KeyValue(Bytes.toBytes(".META.,,99999999999999"), now); + KeyValue b = new KeyValue(Bytes.toBytes(".META.,,1"), now); + KVComparator c = new KeyValue.RootComparator(); + assertTrue(c.compare(b, a) < 0); + KeyValue aa = new KeyValue(Bytes.toBytes(".META.,,1"), now); + KeyValue bb = new KeyValue(Bytes.toBytes(".META.,,1"), + Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1235943454602L, + (byte[])null); + assertTrue(c.compare(aa, bb) < 0); + + // Meta compares + KeyValue aaa = new KeyValue( + Bytes.toBytes("TestScanMultipleVersions,row_0500,1236020145502"), now); + KeyValue bbb = new KeyValue( + Bytes.toBytes("TestScanMultipleVersions,,99999999999999"), now); + c = new KeyValue.MetaComparator(); + assertTrue(c.compare(bbb, aaa) < 0); + + KeyValue aaaa = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,1236023996656"), + Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236024396271L, + (byte[])null); + assertTrue(c.compare(aaaa, bbb) < 0); + + KeyValue x = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"), + Bytes.toBytes("info"), Bytes.toBytes(""), 9223372036854775807L, + (byte[])null); + KeyValue y = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"), + Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236034574912L, + (byte[])null); + assertTrue(c.compare(x, y) < 0); + comparisons(new KeyValue.MetaComparator()); + comparisons(new KeyValue.KVComparator()); + metacomparisons(new KeyValue.RootComparator()); + metacomparisons(new KeyValue.MetaComparator()); + } + + public void testBadMetaCompareSingleDelim() { + MetaComparator c = new KeyValue.MetaComparator(); + long now = System.currentTimeMillis(); + // meta keys values are not quite right. A users can enter illegal values + // from shell when scanning meta. + KeyValue a = new KeyValue(Bytes.toBytes("table,a1"), now); + KeyValue b = new KeyValue(Bytes.toBytes("table,a2"), now); + try { + c.compare(a, b); + } catch (IllegalArgumentException iae) { + assertEquals(".META. key must have two ',' delimiters and have the following" + + " format: ',,'", iae.getMessage()); + return; + } + fail("Expected IllegalArgumentException"); + } + + public void testMetaComparatorTableKeysWithCommaOk() { + MetaComparator c = new KeyValue.MetaComparator(); + long now = System.currentTimeMillis(); + // meta keys values are not quite right. A users can enter illegal values + // from shell when scanning meta. + KeyValue a = new KeyValue(Bytes.toBytes("table,key,with,commas1,1234"), now); + KeyValue b = new KeyValue(Bytes.toBytes("table,key,with,commas2,0123"), now); + assertTrue(c.compare(a, b) < 0); + } + + /** + * Tests cases where rows keys have characters below the ','. + * See HBASE-832 + * @throws IOException + */ + public void testKeyValueBorderCases() throws IOException { + // % sorts before , so if we don't do special comparator, rowB would + // come before rowA. + KeyValue rowA = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/,1234"), + Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null); + KeyValue rowB = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/%20,99999"), + Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null); + assertTrue(KeyValue.META_COMPARATOR.compare(rowA, rowB) < 0); + + rowA = new KeyValue(Bytes.toBytes("testtable,,1234"), Bytes.toBytes("fam"), + Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null); + rowB = new KeyValue(Bytes.toBytes("testtable,$www.hbase.org/,99999"), + Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null); + assertTrue(KeyValue.META_COMPARATOR.compare(rowA, rowB) < 0); + + rowA = new KeyValue(Bytes.toBytes(".META.,testtable,www.hbase.org/,1234,4321"), + Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null); + rowB = new KeyValue(Bytes.toBytes(".META.,testtable,www.hbase.org/%20,99999,99999"), + Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null); + assertTrue(KeyValue.ROOT_COMPARATOR.compare(rowA, rowB) < 0); + } + + private void metacomparisons(final KeyValue.MetaComparator c) { + long now = System.currentTimeMillis(); + assertTrue(c.compare(new KeyValue(Bytes.toBytes(".META.,a,,0,1"), now), + new KeyValue(Bytes.toBytes(".META.,a,,0,1"), now)) == 0); + KeyValue a = new KeyValue(Bytes.toBytes(".META.,a,,0,1"), now); + KeyValue b = new KeyValue(Bytes.toBytes(".META.,a,,0,2"), now); + assertTrue(c.compare(a, b) < 0); + assertTrue(c.compare(new KeyValue(Bytes.toBytes(".META.,a,,0,2"), now), + new KeyValue(Bytes.toBytes(".META.,a,,0,1"), now)) > 0); + } + + private void comparisons(final KeyValue.KVComparator c) { + long now = System.currentTimeMillis(); + assertTrue(c.compare(new KeyValue(Bytes.toBytes(".META.,,1"), now), + new KeyValue(Bytes.toBytes(".META.,,1"), now)) == 0); + assertTrue(c.compare(new KeyValue(Bytes.toBytes(".META.,,1"), now), + new KeyValue(Bytes.toBytes(".META.,,2"), now)) < 0); + assertTrue(c.compare(new KeyValue(Bytes.toBytes(".META.,,2"), now), + new KeyValue(Bytes.toBytes(".META.,,1"), now)) > 0); + } + + public void testBinaryKeys() throws Exception { + Set set = new TreeSet(KeyValue.COMPARATOR); + final byte [] fam = Bytes.toBytes("col"); + final byte [] qf = Bytes.toBytes("umn"); + final byte [] nb = new byte[0]; + KeyValue [] keys = {new KeyValue(Bytes.toBytes("aaaaa,\u0000\u0000,2"), fam, qf, 2, nb), + new KeyValue(Bytes.toBytes("aaaaa,\u0001,3"), fam, qf, 3, nb), + new KeyValue(Bytes.toBytes("aaaaa,,1"), fam, qf, 1, nb), + new KeyValue(Bytes.toBytes("aaaaa,\u1000,5"), fam, qf, 5, nb), + new KeyValue(Bytes.toBytes("aaaaa,a,4"), fam, qf, 4, nb), + new KeyValue(Bytes.toBytes("a,a,0"), fam, qf, 0, nb), + }; + // Add to set with bad comparator + for (int i = 0; i < keys.length; i++) { + set.add(keys[i]); + } + // This will output the keys incorrectly. + boolean assertion = false; + int count = 0; + try { + for (KeyValue k: set) { + assertTrue(count++ == k.getTimestamp()); + } + } catch (junit.framework.AssertionFailedError e) { + // Expected + assertion = true; + } + assertTrue(assertion); + // Make set with good comparator + set = new TreeSet(new KeyValue.MetaComparator()); + for (int i = 0; i < keys.length; i++) { + set.add(keys[i]); + } + count = 0; + for (KeyValue k: set) { + assertTrue(count++ == k.getTimestamp()); + } + // Make up -ROOT- table keys. + KeyValue [] rootKeys = { + new KeyValue(Bytes.toBytes(".META.,aaaaa,\u0000\u0000,0,2"), fam, qf, 2, nb), + new KeyValue(Bytes.toBytes(".META.,aaaaa,\u0001,0,3"), fam, qf, 3, nb), + new KeyValue(Bytes.toBytes(".META.,aaaaa,,0,1"), fam, qf, 1, nb), + new KeyValue(Bytes.toBytes(".META.,aaaaa,\u1000,0,5"), fam, qf, 5, nb), + new KeyValue(Bytes.toBytes(".META.,aaaaa,a,0,4"), fam, qf, 4, nb), + new KeyValue(Bytes.toBytes(".META.,,0"), fam, qf, 0, nb), + }; + // This will output the keys incorrectly. + set = new TreeSet(new KeyValue.MetaComparator()); + // Add to set with bad comparator + for (int i = 0; i < keys.length; i++) { + set.add(rootKeys[i]); + } + assertion = false; + count = 0; + try { + for (KeyValue k: set) { + assertTrue(count++ == k.getTimestamp()); + } + } catch (junit.framework.AssertionFailedError e) { + // Expected + assertion = true; + } + // Now with right comparator + set = new TreeSet(new KeyValue.RootComparator()); + // Add to set with bad comparator + for (int i = 0; i < keys.length; i++) { + set.add(rootKeys[i]); + } + count = 0; + for (KeyValue k: set) { + assertTrue(count++ == k.getTimestamp()); + } + } + + public void testStackedUpKeyValue() { + // Test multiple KeyValues in a single blob. + + // TODO actually write this test! + + } + + private final byte[] rowA = Bytes.toBytes("rowA"); + private final byte[] rowB = Bytes.toBytes("rowB"); + + private final byte[] family = Bytes.toBytes("family"); + private final byte[] qualA = Bytes.toBytes("qfA"); + + private void assertKVLess(KeyValue.KVComparator c, + KeyValue less, + KeyValue greater) { + int cmp = c.compare(less,greater); + assertTrue(cmp < 0); + cmp = c.compare(greater,less); + assertTrue(cmp > 0); + } + + private void assertKVLessWithoutRow(KeyValue.KeyComparator c, int common, KeyValue less, + KeyValue greater) { + int cmp = c.compareIgnoringPrefix(common, less.getBuffer(), less.getOffset() + + KeyValue.ROW_OFFSET, less.getKeyLength(), greater.getBuffer(), + greater.getOffset() + KeyValue.ROW_OFFSET, greater.getKeyLength()); + assertTrue(cmp < 0); + cmp = c.compareIgnoringPrefix(common, greater.getBuffer(), greater.getOffset() + + KeyValue.ROW_OFFSET, greater.getKeyLength(), less.getBuffer(), + less.getOffset() + KeyValue.ROW_OFFSET, less.getKeyLength()); + assertTrue(cmp > 0); + } + + public void testCompareWithoutRow() { + final KeyValue.KeyComparator c = KeyValue.KEY_COMPARATOR; + byte[] row = Bytes.toBytes("row"); + + byte[] fa = Bytes.toBytes("fa"); + byte[] fami = Bytes.toBytes("fami"); + byte[] fami1 = Bytes.toBytes("fami1"); + + byte[] qual0 = Bytes.toBytes(""); + byte[] qual1 = Bytes.toBytes("qf1"); + byte[] qual2 = Bytes.toBytes("qf2"); + long ts = 1; + + // 'fa:' + KeyValue kv_0 = new KeyValue(row, fa, qual0, ts, Type.Put); + // 'fami:' + KeyValue kv0_0 = new KeyValue(row, fami, qual0, ts, Type.Put); + // 'fami:qf1' + KeyValue kv0_1 = new KeyValue(row, fami, qual1, ts, Type.Put); + // 'fami:qf2' + KeyValue kv0_2 = new KeyValue(row, fami, qual2, ts, Type.Put); + // 'fami1:' + KeyValue kv1_0 = new KeyValue(row, fami1, qual0, ts, Type.Put); + + // 'fami:qf1' < 'fami:qf2' + assertKVLessWithoutRow(c, 0, kv0_1, kv0_2); + // 'fami:qf1' < 'fami1:' + assertKVLessWithoutRow(c, 0, kv0_1, kv1_0); + + // Test comparison by skipping the same prefix bytes. + /*** + * KeyValue Format and commonLength: + * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|.... + * ------------------|-------commonLength--------|-------------- + */ + int commonLength = KeyValue.ROW_LENGTH_SIZE + KeyValue.FAMILY_LENGTH_SIZE + + row.length; + // 'fa:' < 'fami:'. They have commonPrefix + 2 same prefix bytes. + assertKVLessWithoutRow(c, commonLength + 2, kv_0, kv0_0); + // 'fami:' < 'fami:qf1'. They have commonPrefix + 4 same prefix bytes. + assertKVLessWithoutRow(c, commonLength + 4, kv0_0, kv0_1); + // 'fami:qf1' < 'fami1:'. They have commonPrefix + 4 same prefix bytes. + assertKVLessWithoutRow(c, commonLength + 4, kv0_1, kv1_0); + // 'fami:qf1' < 'fami:qf2'. They have commonPrefix + 6 same prefix bytes. + assertKVLessWithoutRow(c, commonLength + 6, kv0_1, kv0_2); + } + + public void testFirstLastOnRow() { + final KVComparator c = KeyValue.COMPARATOR; + long ts = 1; + + // These are listed in sort order (ie: every one should be less + // than the one on the next line). + final KeyValue firstOnRowA = KeyValue.createFirstOnRow(rowA); + final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, Type.Put); + final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, Type.Put); + + final KeyValue lastOnRowA = KeyValue.createLastOnRow(rowA); + final KeyValue firstOnRowB = KeyValue.createFirstOnRow(rowB); + final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, Type.Put); + + assertKVLess(c, firstOnRowA, firstOnRowB); + assertKVLess(c, firstOnRowA, kvA_1); + assertKVLess(c, firstOnRowA, kvA_2); + assertKVLess(c, kvA_1, kvA_2); + assertKVLess(c, kvA_2, firstOnRowB); + assertKVLess(c, kvA_1, firstOnRowB); + + assertKVLess(c, lastOnRowA, firstOnRowB); + assertKVLess(c, firstOnRowB, kvB); + assertKVLess(c, lastOnRowA, kvB); + + assertKVLess(c, kvA_2, lastOnRowA); + assertKVLess(c, kvA_1, lastOnRowA); + assertKVLess(c, firstOnRowA, lastOnRowA); + } + + public void testCreateKeyOnly() throws Exception { + long ts = 1; + byte [] value = Bytes.toBytes("a real value"); + byte [] evalue = new byte[0]; // empty value + + for (byte[] val : new byte[][]{value, evalue}) { + for (boolean useLen : new boolean[]{false,true}) { + KeyValue kv1 = new KeyValue(rowA, family, qualA, ts, val); + KeyValue kv1ko = kv1.createKeyOnly(useLen); + // keys are still the same + assertTrue(kv1.equals(kv1ko)); + // but values are not + assertTrue(kv1ko.getValue().length == (useLen?Bytes.SIZEOF_INT:0)); + if (useLen) { + assertEquals(kv1.getValueLength(), Bytes.toInt(kv1ko.getValue())); + } + } + } + } + + public void testCreateKeyValueFromKey() { + KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), + Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("myValue")); + int initialPadding = 10; + int endingPadding = 20; + int keyLen = kv.getKeyLength(); + byte[] tmpArr = new byte[initialPadding + endingPadding + keyLen]; + System.arraycopy(kv.getBuffer(), kv.getKeyOffset(), tmpArr, + initialPadding, keyLen); + KeyValue kvFromKey = KeyValue.createKeyValueFromKey(tmpArr, initialPadding, + keyLen); + assertEquals(keyLen, kvFromKey.getKeyLength()); + assertEquals(KeyValue.ROW_OFFSET + keyLen, kvFromKey.getBuffer().length); + System.err.println("kv=" + kv); + System.err.println("kvFromKey=" + kvFromKey); + assertEquals(kvFromKey.toString(), + kv.toString().replaceAll("=[0-9]+", "=0")); + } + + /** + * The row cache is cleared and re-read for the new value + * + * @throws IOException + */ + public void testReadFields() throws IOException { + KeyValue kv1 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("cf1"), + Bytes.toBytes("qualifier1"), 12345L, Bytes.toBytes("value1")); + kv1.getRow(); // set row cache of kv1 + KeyValue kv2 = new KeyValue(Bytes.toBytes("row2"), Bytes.toBytes("cf2"), + Bytes.toBytes("qualifier2"), 12345L, Bytes.toBytes("value2")); + kv1.readFields(new DataInputStream(new ByteArrayInputStream(WritableUtils + .toByteArray(kv2)))); + // check equality + assertEquals(kv1, kv2); + // check cache state (getRow() return the cached value if the cache is set) + assertTrue(Bytes.equals(kv1.getRow(), kv2.getRow())); + } + + /** + * Tests that getTimestamp() does always return the proper timestamp, even after updating it. + * See HBASE-6265. + */ + public void testGetTimestamp() { + KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), + Bytes.toBytes("myQualifier"), HConstants.LATEST_TIMESTAMP, + Bytes.toBytes("myValue")); + long time1 = kv.getTimestamp(); + kv.updateLatestStamp(Bytes.toBytes(12345L)); + long time2 = kv.getTimestamp(); + assertEquals(HConstants.LATEST_TIMESTAMP, time1); + assertEquals(12345L, time2); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java new file mode 100644 index 0000000..8b1c018 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.zookeeper.KeeperException; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestLocalHBaseCluster { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + /** + * Check that we can start a local HBase cluster specifying a custom master + * and regionserver class and then cast back to those classes; also that + * the cluster will launch and terminate cleanly. See HBASE-6011. + */ + @Test + public void testLocalHBaseCluster() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDataTestDir("hbase.rootdir"). + makeQualified(TEST_UTIL.getTestFileSystem().getUri(), + TEST_UTIL.getTestFileSystem().getWorkingDirectory()).toString()); + MiniZooKeeperCluster zkCluster = TEST_UTIL.startMiniZKCluster(); + conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(zkCluster.getClientPort())); + LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 1, MyHMaster.class, + MyHRegionServer.class); + // Can we cast back to our master class? + try { + ((MyHMaster)cluster.getMaster(0)).setZKCluster(zkCluster); + } catch (ClassCastException e) { + fail("Could not cast master to our class"); + } + // Can we cast back to our regionserver class? + try { + ((MyHRegionServer)cluster.getRegionServer(0)).echo(42); + } catch (ClassCastException e) { + fail("Could not cast regionserver to our class"); + } + // Does the cluster start successfully? + try { + cluster.startup(); + waitForClusterUp(conf); + } catch (IOException e) { + fail("LocalHBaseCluster did not start successfully"); + } finally { + cluster.shutdown(); + } + } + + private void waitForClusterUp(Configuration conf) throws IOException { + HTable t = new HTable(conf, HConstants.META_TABLE_NAME); + ResultScanner s = t.getScanner(new Scan()); + while (s.next() != null) { + continue; + } + s.close(); + t.close(); + } + + /** + * A private master class similar to that used by HMasterCommandLine when + * running in local mode. + */ + public static class MyHMaster extends HMaster { + private MiniZooKeeperCluster zkcluster = null; + + public MyHMaster(Configuration conf) throws IOException, KeeperException, + InterruptedException { + super(conf); + } + + @Override + public void run() { + super.run(); + if (this.zkcluster != null) { + try { + this.zkcluster.shutdown(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + void setZKCluster(final MiniZooKeeperCluster zkcluster) { + this.zkcluster = zkcluster; + } + } + + /** + * A private regionserver class with a dummy method for testing casts + */ + public static class MyHRegionServer extends HRegionServer { + + public MyHRegionServer(Configuration conf) throws IOException, + InterruptedException { + super(conf); + } + + public int echo(int val) { + return val; + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java b/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java new file mode 100644 index 0000000..cf2b5cd --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java @@ -0,0 +1,333 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Map; +import java.util.NavigableMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestCase.FlushCache; +import org.apache.hadoop.hbase.HBaseTestCase.HTableIncommon; +import org.apache.hadoop.hbase.HBaseTestCase.Incommon; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Port of old TestScanMultipleVersions, TestTimestamp and TestGetRowVersions + * from old testing framework to {@link HBaseTestingUtility}. + */ +@Category(MediumTests.class) +public class TestMultiVersions { + private static final Log LOG = LogFactory.getLog(TestMultiVersions.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private HBaseAdmin admin; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + UTIL.startMiniCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Before + public void before() + throws MasterNotRunningException, ZooKeeperConnectionException { + this.admin = new HBaseAdmin(UTIL.getConfiguration()); + } + + @After + public void after() throws IOException { + this.admin.close(); + } + + /** + * Tests user specifiable time stamps putting, getting and scanning. Also + * tests same in presence of deletes. Test cores are written so can be + * run against an HRegion and against an HTable: i.e. both local and remote. + * + *

    Port of old TestTimestamp test to here so can better utilize the spun + * up cluster running more than a single test per spin up. Keep old tests' + * crazyness. + */ + @Test + public void testTimestamps() throws Exception { + HTableDescriptor desc = new HTableDescriptor("testTimestamps"); + desc.addFamily(new HColumnDescriptor(TimestampTestBase.FAMILY_NAME)); + this.admin.createTable(desc); + HTable table = new HTable(UTIL.getConfiguration(), desc.getName()); + // TODO: Remove these deprecated classes or pull them in here if this is + // only test using them. + Incommon incommon = new HTableIncommon(table); + TimestampTestBase.doTestDelete(incommon, new FlushCache() { + public void flushcache() throws IOException { + UTIL.getHBaseCluster().flushcache(); + } + }); + + // Perhaps drop and readd the table between tests so the former does + // not pollute this latter? Or put into separate tests. + TimestampTestBase.doTestTimestampScanning(incommon, new FlushCache() { + public void flushcache() throws IOException { + UTIL.getMiniHBaseCluster().flushcache(); + } + }); + + table.close(); + } + + /** + * Verifies versions across a cluster restart. + * Port of old TestGetRowVersions test to here so can better utilize the spun + * up cluster running more than a single test per spin up. Keep old tests' + * crazyness. + */ + @Test + public void testGetRowVersions() throws Exception { + final String tableName = "testGetRowVersions"; + final byte [] contents = Bytes.toBytes("contents"); + final byte [] row = Bytes.toBytes("row"); + final byte [] value1 = Bytes.toBytes("value1"); + final byte [] value2 = Bytes.toBytes("value2"); + final long timestamp1 = 100L; + final long timestamp2 = 200L; + final HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(contents)); + this.admin.createTable(desc); + Put put = new Put(row, timestamp1, null); + put.add(contents, contents, value1); + HTable table = new HTable(UTIL.getConfiguration(), tableName); + table.put(put); + // Shut down and restart the HBase cluster + table.close(); + UTIL.shutdownMiniHBaseCluster(); + LOG.debug("HBase cluster shut down -- restarting"); + UTIL.startMiniHBaseCluster(1, 1); + // Make a new connection. Use new Configuration instance because old one + // is tied to an HConnection that has since gone stale. + table = new HTable(new Configuration(UTIL.getConfiguration()), tableName); + // Overwrite previous value + put = new Put(row, timestamp2, null); + put.add(contents, contents, value2); + table.put(put); + // Now verify that getRow(row, column, latest) works + Get get = new Get(row); + // Should get one version by default + Result r = table.get(get); + assertNotNull(r); + assertFalse(r.isEmpty()); + assertTrue(r.size() == 1); + byte [] value = r.getValue(contents, contents); + assertTrue(value.length != 0); + assertTrue(Bytes.equals(value, value2)); + // Now check getRow with multiple versions + get = new Get(row); + get.setMaxVersions(); + r = table.get(get); + assertTrue(r.size() == 2); + value = r.getValue(contents, contents); + assertTrue(value.length != 0); + assertTrue(Bytes.equals(value, value2)); + NavigableMap>> map = + r.getMap(); + NavigableMap> familyMap = + map.get(contents); + NavigableMap versionMap = familyMap.get(contents); + assertTrue(versionMap.size() == 2); + assertTrue(Bytes.equals(value1, versionMap.get(timestamp1))); + assertTrue(Bytes.equals(value2, versionMap.get(timestamp2))); + table.close(); + } + + /** + * Port of old TestScanMultipleVersions test here so can better utilize the + * spun up cluster running more than just a single test. Keep old tests + * crazyness. + * + *

    Tests five cases of scans and timestamps. + * @throws Exception + */ + @Test + public void testScanMultipleVersions() throws Exception { + final byte [] tableName = Bytes.toBytes("testScanMultipleVersions"); + final HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + final byte [][] rows = new byte[][] { + Bytes.toBytes("row_0200"), + Bytes.toBytes("row_0800") + }; + final byte [][] splitRows = new byte[][] {Bytes.toBytes("row_0500")}; + final long [] timestamp = new long[] {100L, 1000L}; + this.admin.createTable(desc, splitRows); + HTable table = new HTable(UTIL.getConfiguration(), tableName); + // Assert we got the region layout wanted. + NavigableMap locations = table.getRegionLocations(); + assertEquals(2, locations.size()); + int index = 0; + for (Map.Entry e: locations.entrySet()) { + HRegionInfo hri = e.getKey(); + if (index == 0) { + assertTrue(Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())); + assertTrue(Bytes.equals(hri.getEndKey(), splitRows[0])); + } else if (index == 1) { + assertTrue(Bytes.equals(splitRows[0], hri.getStartKey())); + assertTrue(Bytes.equals(hri.getEndKey(), HConstants.EMPTY_END_ROW)); + } + index++; + } + // Insert data + for (int i = 0; i < locations.size(); i++) { + for (int j = 0; j < timestamp.length; j++) { + Put put = new Put(rows[i], timestamp[j], null); + put.add(HConstants.CATALOG_FAMILY, null, timestamp[j], + Bytes.toBytes(timestamp[j])); + table.put(put); + } + } + // There are 5 cases we have to test. Each is described below. + for (int i = 0; i < rows.length; i++) { + for (int j = 0; j < timestamp.length; j++) { + Get get = new Get(rows[i]); + get.addFamily(HConstants.CATALOG_FAMILY); + get.setTimeStamp(timestamp[j]); + Result result = table.get(get); + int cellCount = 0; + for(@SuppressWarnings("unused")KeyValue kv : result.list()) { + cellCount++; + } + assertTrue(cellCount == 1); + } + table.close(); + } + + // Case 1: scan with LATEST_TIMESTAMP. Should get two rows + int count = 0; + Scan scan = new Scan(); + scan.addFamily(HConstants.CATALOG_FAMILY); + ResultScanner s = table.getScanner(scan); + try { + for (Result rr = null; (rr = s.next()) != null;) { + System.out.println(rr.toString()); + count += 1; + } + assertEquals("Number of rows should be 2", 2, count); + } finally { + s.close(); + } + + // Case 2: Scan with a timestamp greater than most recent timestamp + // (in this case > 1000 and < LATEST_TIMESTAMP. Should get 2 rows. + + count = 0; + scan = new Scan(); + scan.setTimeRange(1000L, Long.MAX_VALUE); + scan.addFamily(HConstants.CATALOG_FAMILY); + + s = table.getScanner(scan); + try { + while (s.next() != null) { + count += 1; + } + assertEquals("Number of rows should be 2", 2, count); + } finally { + s.close(); + } + + // Case 3: scan with timestamp equal to most recent timestamp + // (in this case == 1000. Should get 2 rows. + + count = 0; + scan = new Scan(); + scan.setTimeStamp(1000L); + scan.addFamily(HConstants.CATALOG_FAMILY); + + s = table.getScanner(scan); + try { + while (s.next() != null) { + count += 1; + } + assertEquals("Number of rows should be 2", 2, count); + } finally { + s.close(); + } + + // Case 4: scan with timestamp greater than first timestamp but less than + // second timestamp (100 < timestamp < 1000). Should get 2 rows. + + count = 0; + scan = new Scan(); + scan.setTimeRange(100L, 1000L); + scan.addFamily(HConstants.CATALOG_FAMILY); + + s = table.getScanner(scan); + try { + while (s.next() != null) { + count += 1; + } + assertEquals("Number of rows should be 2", 2, count); + } finally { + s.close(); + } + + // Case 5: scan with timestamp equal to first timestamp (100) + // Should get 2 rows. + + count = 0; + scan = new Scan(); + scan.setTimeStamp(100L); + scan.addFamily(HConstants.CATALOG_FAMILY); + + s = table.getScanner(scan); + try { + while (s.next() != null) { + count += 1; + } + assertEquals("Number of rows should be 2", 2, count); + } finally { + s.close(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestNodeHealthCheckChore.java b/src/test/java/org/apache/hadoop/hbase/TestNodeHealthCheckChore.java new file mode 100644 index 0000000..9ae974e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestNodeHealthCheckChore.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.HealthChecker.HealthCheckerExitStatus; +import org.junit.After; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestNodeHealthCheckChore { + + private static final Log LOG = LogFactory.getLog(TestNodeHealthCheckChore.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private File healthScriptFile; + + + @After + public void cleanUp() throws IOException { + UTIL.cleanupTestDir(); + } + + @Test + public void testHealthChecker() throws Exception { + Configuration config = getConfForNodeHealthScript(); + config.addResource(healthScriptFile.getName()); + String location = healthScriptFile.getAbsolutePath(); + long timeout = config.getLong(HConstants.HEALTH_SCRIPT_TIMEOUT, 200); + + HealthChecker checker = new HealthChecker(); + checker.init(location, timeout); + + String normalScript = "echo \"I am all fine\""; + createScript(normalScript, true); + HealthReport report = checker.checkHealth(); + assertEquals(HealthCheckerExitStatus.SUCCESS, report.getStatus()); + + LOG.info("Health Status:" + checker); + + String errorScript = "echo ERROR\n echo \"Node not healthy\""; + createScript(errorScript, true); + report = checker.checkHealth(); + assertEquals(HealthCheckerExitStatus.FAILED, report.getStatus()); + LOG.info("Health Status:" + report.getHealthReport()); + + String timeOutScript = "sleep 4\n echo\"I am fine\""; + createScript(timeOutScript, true); + report = checker.checkHealth(); + assertEquals(HealthCheckerExitStatus.TIMED_OUT, report.getStatus()); + LOG.info("Health Status:" + report.getHealthReport()); + + healthScriptFile.delete(); + } + + @Test + public void testNodeHealthChore() throws Exception{ + Stoppable stop = new StoppableImplementation(); + Configuration conf = getConfForNodeHealthScript(); + String errorScript = "echo ERROR\n echo \"Node not healthy\""; + createScript(errorScript, true); + HealthCheckChore rsChore = new HealthCheckChore(100, stop, conf); + //Default threshold is three. + rsChore.chore(); + rsChore.chore(); + assertFalse("Stoppable must not be stopped.", stop.isStopped()); + rsChore.chore(); + assertTrue("Stoppable must have been stopped.", stop.isStopped()); + } + + private void createScript(String scriptStr, boolean setExecutable) + throws Exception { + healthScriptFile.createNewFile(); + PrintWriter pw = new PrintWriter(new FileOutputStream(healthScriptFile)); + pw.println(scriptStr); + pw.flush(); + pw.close(); + healthScriptFile.setExecutable(setExecutable); + } + + private Configuration getConfForNodeHealthScript() { + Configuration conf = UTIL.getConfiguration(); + File tempDir = new File(UTIL.getDataTestDir().toString()); + tempDir.mkdirs(); + healthScriptFile = new File(tempDir.getAbsolutePath(), "HealthScript.sh"); + conf.set(HConstants.HEALTH_SCRIPT_LOC, + healthScriptFile.getAbsolutePath()); + conf.setLong(HConstants.HEALTH_FAILURE_THRESHOLD, 3); + conf.setLong(HConstants.HEALTH_SCRIPT_TIMEOUT, 200); + return conf; + } + + /** + * Simple helper class that just keeps track of whether or not its stopped. + */ + private static class StoppableImplementation implements Stoppable { + private volatile boolean stop = false; + + @Override + public void stop(String why) { + this.stop = true; + } + + @Override + public boolean isStopped() { + return this.stop; + } + + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java b/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java new file mode 100644 index 0000000..ec04f4d --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java @@ -0,0 +1,235 @@ +/** + * Copyright 2008 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test whether region rebalancing works. (HBASE-71) + */ +@Category(LargeTests.class) +public class TestRegionRebalancing { + final Log LOG = LogFactory.getLog(this.getClass().getName()); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + HTable table; + HTableDescriptor desc; + private static final byte [] FAMILY_NAME = Bytes.toBytes("col"); + + @BeforeClass + public static void beforeClass() throws Exception { + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Before + public void before() { + this.desc = new HTableDescriptor("test"); + this.desc.addFamily(new HColumnDescriptor(FAMILY_NAME)); + } + + /** + * For HBASE-71. Try a few different configurations of starting and stopping + * region servers to see if the assignment or regions is pretty balanced. + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testRebalanceOnRegionServerNumberChange() + throws IOException, InterruptedException { + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + admin.createTable(this.desc, Arrays.copyOfRange(HBaseTestingUtility.KEYS, + 1, HBaseTestingUtility.KEYS.length)); + this.table = new HTable(UTIL.getConfiguration(), this.desc.getName()); + CatalogTracker ct = new CatalogTracker(UTIL.getConfiguration()); + ct.start(); + try { + MetaReader.fullScanMetaAndPrint(ct); + } finally { + ct.stop(); + } + assertEquals("Test table should have right number of regions", + HBaseTestingUtility.KEYS.length, + this.table.getStartKeys().length); + + // verify that the region assignments are balanced to start out + assertRegionsAreBalanced(); + + // add a region server - total of 2 + LOG.info("Started second server=" + + UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName()); + UTIL.getHBaseCluster().getMaster().balance(); + assertRegionsAreBalanced(); + + // add a region server - total of 3 + LOG.info("Started third server=" + + UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName()); + UTIL.getHBaseCluster().getMaster().balance(); + assertRegionsAreBalanced(); + + // kill a region server - total of 2 + LOG.info("Stopped third server=" + UTIL.getHBaseCluster().stopRegionServer(2, false)); + UTIL.getHBaseCluster().waitOnRegionServer(2); + UTIL.getHBaseCluster().getMaster().balance(); + assertRegionsAreBalanced(); + + // start two more region servers - total of 4 + LOG.info("Readding third server=" + + UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName()); + LOG.info("Added fourth server=" + + UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName()); + UTIL.getHBaseCluster().getMaster().balance(); + assertRegionsAreBalanced(); + + for (int i = 0; i < 6; i++){ + LOG.info("Adding " + (i + 5) + "th region server"); + UTIL.getHBaseCluster().startRegionServer(); + } + UTIL.getHBaseCluster().getMaster().balance(); + assertRegionsAreBalanced(); + table.close(); + } + + /** figure out how many regions are currently being served. */ + private int getRegionCount() throws IOException { + int total = 0; + for (HRegionServer server : getOnlineRegionServers()) { + total += server.getOnlineRegions().size(); + } + return total; + } + + /** + * Determine if regions are balanced. Figure out the total, divide by the + * number of online servers, then test if each server is +/- 1 of average + * rounded up. + */ + private void assertRegionsAreBalanced() throws IOException { + // TODO: Fix this test. Old balancer used to run with 'slop'. New + // balancer does not. + boolean success = false; + float slop = (float)UTIL.getConfiguration().getFloat("hbase.regions.slop", 0.1f); + if (slop <= 0) slop = 1; + + for (int i = 0; i < 5; i++) { + success = true; + // make sure all the regions are reassigned before we test balance + waitForAllRegionsAssigned(); + + int regionCount = getRegionCount(); + List servers = getOnlineRegionServers(); + double avg = UTIL.getHBaseCluster().getMaster().getAverageLoad(); + int avgLoadPlusSlop = (int)Math.ceil(avg * (1 + slop)); + int avgLoadMinusSlop = (int)Math.floor(avg * (1 - slop)) - 1; + LOG.debug("There are " + servers.size() + " servers and " + regionCount + + " regions. Load Average: " + avg + " low border: " + avgLoadMinusSlop + + ", up border: " + avgLoadPlusSlop + "; attempt: " + i); + + for (HRegionServer server : servers) { + int serverLoad = server.getOnlineRegions().size(); + LOG.debug(server.getServerName() + " Avg: " + avg + " actual: " + serverLoad); + if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop + && serverLoad >= avgLoadMinusSlop)) { + for (HRegionInfo hri : server.getOnlineRegions()) { + if (hri.isMetaRegion() || hri.isRootRegion()) serverLoad--; + // LOG.debug(hri.getRegionNameAsString()); + } + if (!(serverLoad <= avgLoadPlusSlop && serverLoad >= avgLoadMinusSlop)) { + LOG.debug(server.getServerName() + " Isn't balanced!!! Avg: " + avg + + " actual: " + serverLoad + " slop: " + slop); + success = false; + break; + } + } + } + + if (!success) { + // one or more servers are not balanced. sleep a little to give it a + // chance to catch up. then, go back to the retry loop. + try { + Thread.sleep(10000); + } catch (InterruptedException e) {} + + UTIL.getHBaseCluster().getMaster().balance(); + continue; + } + + // if we get here, all servers were balanced, so we should just return. + return; + } + // if we get here, we tried 5 times and never got to short circuit out of + // the retry loop, so this is a failure. + fail("After 5 attempts, region assignments were not balanced."); + } + + private List getOnlineRegionServers() { + List list = new ArrayList(); + for (JVMClusterUtil.RegionServerThread rst : + UTIL.getHBaseCluster().getRegionServerThreads()) { + if (rst.getRegionServer().isOnline()) { + list.add(rst.getRegionServer()); + } + } + return list; + } + + /** + * Wait until all the regions are assigned. + */ + private void waitForAllRegionsAssigned() throws IOException { + int totalRegions = HBaseTestingUtility.KEYS.length+2; + while (getRegionCount() < totalRegions) { + // while (!cluster.getMaster().allRegionsAssigned()) { + LOG.debug("Waiting for there to be "+ totalRegions +" regions, but there are " + getRegionCount() + " right now."); + try { + Thread.sleep(200); + } catch (InterruptedException e) {} + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestSerialization.java b/src/test/java/org/apache/hadoop/hbase/TestSerialization.java new file mode 100644 index 0000000..50cb9d4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestSerialization.java @@ -0,0 +1,595 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + + +import static org.junit.Assert.*; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.hadoop.hbase.HServerLoad092.RegionLoad; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.RowLock; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.io.HbaseMapWritable; +import org.apache.hadoop.hbase.io.TimeRange; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.io.DataInputBuffer; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test HBase Writables serializations + */ +@Category(SmallTests.class) +public class TestSerialization { + @Test + public void testHServerLoadVersioning() throws IOException { + Set cps = new HashSet(0); + Map regions = new TreeMap(Bytes.BYTES_COMPARATOR); + regions.put(HConstants.META_TABLE_NAME, + new HServerLoad092.RegionLoad(HConstants.META_TABLE_NAME, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, cps)); + HServerLoad092 hsl092 = new HServerLoad092(0, 0, 0, 0, regions, cps); + byte [] hsl092bytes = Writables.getBytes(hsl092); + HServerLoad hsl = (HServerLoad)Writables.getWritable(hsl092bytes, new HServerLoad()); + // TO BE CONTINUED + } + + @Test public void testCompareFilter() throws Exception { + Filter f = new RowFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + byte [] bytes = Writables.getBytes(f); + Filter ff = (Filter)Writables.getWritable(bytes, new RowFilter()); + assertNotNull(ff); + } + + @Test public void testKeyValue() throws Exception { + final String name = "testKeyValue"; + byte [] row = Bytes.toBytes(name); + byte [] family = Bytes.toBytes(name); + byte [] qualifier = Bytes.toBytes(name); + KeyValue original = new KeyValue(row, family, qualifier); + byte [] bytes = Writables.getBytes(original); + KeyValue newone = (KeyValue)Writables.getWritable(bytes, new KeyValue()); + assertTrue(KeyValue.COMPARATOR.compare(original, newone) == 0); + } + + @SuppressWarnings("unchecked") + @Test public void testHbaseMapWritable() throws Exception { + HbaseMapWritable hmw = + new HbaseMapWritable(); + hmw.put("key".getBytes(), "value".getBytes()); + byte [] bytes = Writables.getBytes(hmw); + hmw = (HbaseMapWritable) + Writables.getWritable(bytes, new HbaseMapWritable()); + assertTrue(hmw.size() == 1); + assertTrue(Bytes.equals("value".getBytes(), hmw.get("key".getBytes()))); + } + + + @Test public void testTableDescriptor() throws Exception { + final String name = "testTableDescriptor"; + HTableDescriptor htd = createTableDescriptor(name); + byte [] mb = Writables.getBytes(htd); + HTableDescriptor deserializedHtd = + (HTableDescriptor)Writables.getWritable(mb, new HTableDescriptor()); + assertEquals(htd.getNameAsString(), deserializedHtd.getNameAsString()); + } + + /** + * Test RegionInfo serialization + * @throws Exception + */ + @Test public void testRegionInfo() throws Exception { + HRegionInfo hri = createRandomRegion("testRegionInfo"); + byte [] hrib = Writables.getBytes(hri); + HRegionInfo deserializedHri = + (HRegionInfo)Writables.getWritable(hrib, new HRegionInfo()); + assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName()); + //assertEquals(hri.getTableDesc().getFamilies().size(), + // deserializedHri.getTableDesc().getFamilies().size()); + } + + @Test public void testRegionInfos() throws Exception { + HRegionInfo hri = createRandomRegion("testRegionInfos"); + byte [] hrib = Writables.getBytes(hri); + byte [] triple = new byte [3 * hrib.length]; + System.arraycopy(hrib, 0, triple, 0, hrib.length); + System.arraycopy(hrib, 0, triple, hrib.length, hrib.length); + System.arraycopy(hrib, 0, triple, hrib.length * 2, hrib.length); + List regions = Writables.getHRegionInfos(triple, 0, triple.length); + assertTrue(regions.size() == 3); + assertTrue(regions.get(0).equals(regions.get(1))); + assertTrue(regions.get(0).equals(regions.get(2))); + } + + private HRegionInfo createRandomRegion(final String name) { + HTableDescriptor htd = new HTableDescriptor(name); + String [] families = new String [] {"info", "anchor"}; + for (int i = 0; i < families.length; i++) { + htd.addFamily(new HColumnDescriptor(families[i])); + } + return new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW, + HConstants.EMPTY_END_ROW); + } + + @Test public void testPut() throws Exception{ + byte[] row = "row".getBytes(); + byte[] fam = "fam".getBytes(); + byte[] qf1 = "qf1".getBytes(); + byte[] qf2 = "qf2".getBytes(); + byte[] qf3 = "qf3".getBytes(); + byte[] qf4 = "qf4".getBytes(); + byte[] qf5 = "qf5".getBytes(); + byte[] qf6 = "qf6".getBytes(); + byte[] qf7 = "qf7".getBytes(); + byte[] qf8 = "qf8".getBytes(); + + long ts = System.currentTimeMillis(); + byte[] val = "val".getBytes(); + + Put put = new Put(row); + put.setWriteToWAL(false); + put.add(fam, qf1, ts, val); + put.add(fam, qf2, ts, val); + put.add(fam, qf3, ts, val); + put.add(fam, qf4, ts, val); + put.add(fam, qf5, ts, val); + put.add(fam, qf6, ts, val); + put.add(fam, qf7, ts, val); + put.add(fam, qf8, ts, val); + + byte[] sb = Writables.getBytes(put); + Put desPut = (Put)Writables.getWritable(sb, new Put()); + + //Timing test +// long start = System.nanoTime(); +// desPut = (Put)Writables.getWritable(sb, new Put()); +// long stop = System.nanoTime(); +// System.out.println("timer " +(stop-start)); + + assertTrue(Bytes.equals(put.getRow(), desPut.getRow())); + List list = null; + List desList = null; + for(Map.Entry> entry : put.getFamilyMap().entrySet()){ + assertTrue(desPut.getFamilyMap().containsKey(entry.getKey())); + list = entry.getValue(); + desList = desPut.getFamilyMap().get(entry.getKey()); + for(int i=0; i list = null; + List desList = null; + for(Map.Entry> entry : put.getFamilyMap().entrySet()){ + assertTrue(desPut.getFamilyMap().containsKey(entry.getKey())); + list = entry.getValue(); + desList = desPut.getFamilyMap().get(entry.getKey()); + for(int i=0; i list = null; + List desList = null; + for(Map.Entry> entry : + delete.getFamilyMap().entrySet()){ + assertTrue(desDelete.getFamilyMap().containsKey(entry.getKey())); + list = entry.getValue(); + desList = desDelete.getFamilyMap().get(entry.getKey()); + for(int i=0; i set = null; + Set desSet = null; + + for(Map.Entry> entry : + get.getFamilyMap().entrySet()){ + assertTrue(desGet.getFamilyMap().containsKey(entry.getKey())); + set = entry.getValue(); + desSet = desGet.getFamilyMap().get(entry.getKey()); + for(byte [] qualifier : set){ + assertTrue(desSet.contains(qualifier)); + } + } + + assertEquals(get.getLockId(), desGet.getLockId()); + assertEquals(get.getMaxVersions(), desGet.getMaxVersions()); + TimeRange tr = get.getTimeRange(); + TimeRange desTr = desGet.getTimeRange(); + assertEquals(tr.getMax(), desTr.getMax()); + assertEquals(tr.getMin(), desTr.getMin()); + } + + + @Test public void testScan() throws Exception { + + byte[] startRow = "startRow".getBytes(); + byte[] stopRow = "stopRow".getBytes(); + byte[] fam = "fam".getBytes(); + byte[] qf1 = "qf1".getBytes(); + + long ts = System.currentTimeMillis(); + int maxVersions = 2; + + Scan scan = new Scan(startRow, stopRow); + scan.addColumn(fam, qf1); + scan.setTimeRange(ts, ts+1); + scan.setMaxVersions(maxVersions); + + byte[] sb = Writables.getBytes(scan); + Scan desScan = (Scan)Writables.getWritable(sb, new Scan()); + + assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow())); + assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow())); + assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks()); + Set set = null; + Set desSet = null; + + for(Map.Entry> entry : + scan.getFamilyMap().entrySet()){ + assertTrue(desScan.getFamilyMap().containsKey(entry.getKey())); + set = entry.getValue(); + desSet = desScan.getFamilyMap().get(entry.getKey()); + for(byte[] column : set){ + assertTrue(desSet.contains(column)); + } + + // Test filters are serialized properly. + scan = new Scan(startRow); + final String name = "testScan"; + byte [] prefix = Bytes.toBytes(name); + scan.setFilter(new PrefixFilter(prefix)); + sb = Writables.getBytes(scan); + desScan = (Scan)Writables.getWritable(sb, new Scan()); + Filter f = desScan.getFilter(); + assertTrue(f instanceof PrefixFilter); + } + + assertEquals(scan.getMaxVersions(), desScan.getMaxVersions()); + TimeRange tr = scan.getTimeRange(); + TimeRange desTr = desScan.getTimeRange(); + assertEquals(tr.getMax(), desTr.getMax()); + assertEquals(tr.getMin(), desTr.getMin()); + } + + @Test public void testResultEmpty() throws Exception { + List keys = new ArrayList(); + Result r = new Result(keys); + assertTrue(r.isEmpty()); + byte [] rb = Writables.getBytes(r); + Result deserializedR = (Result)Writables.getWritable(rb, new Result()); + assertTrue(deserializedR.isEmpty()); + } + + + @Test public void testResult() throws Exception { + byte [] rowA = Bytes.toBytes("rowA"); + byte [] famA = Bytes.toBytes("famA"); + byte [] qfA = Bytes.toBytes("qfA"); + byte [] valueA = Bytes.toBytes("valueA"); + + byte [] rowB = Bytes.toBytes("rowB"); + byte [] famB = Bytes.toBytes("famB"); + byte [] qfB = Bytes.toBytes("qfB"); + byte [] valueB = Bytes.toBytes("valueB"); + + KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA); + KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB); + + Result result = new Result(new KeyValue[]{kvA, kvB}); + + byte [] rb = Writables.getBytes(result); + Result deResult = (Result)Writables.getWritable(rb, new Result()); + + assertTrue("results are not equivalent, first key mismatch", + result.raw()[0].equals(deResult.raw()[0])); + + assertTrue("results are not equivalent, second key mismatch", + result.raw()[1].equals(deResult.raw()[1])); + + // Test empty Result + Result r = new Result(); + byte [] b = Writables.getBytes(r); + Result deserialized = (Result)Writables.getWritable(b, new Result()); + assertEquals(r.size(), deserialized.size()); + } + + @Test public void testResultDynamicBuild() throws Exception { + byte [] rowA = Bytes.toBytes("rowA"); + byte [] famA = Bytes.toBytes("famA"); + byte [] qfA = Bytes.toBytes("qfA"); + byte [] valueA = Bytes.toBytes("valueA"); + + byte [] rowB = Bytes.toBytes("rowB"); + byte [] famB = Bytes.toBytes("famB"); + byte [] qfB = Bytes.toBytes("qfB"); + byte [] valueB = Bytes.toBytes("valueB"); + + KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA); + KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB); + + Result result = new Result(new KeyValue[]{kvA, kvB}); + + byte [] rb = Writables.getBytes(result); + + + // Call getRow() first + Result deResult = (Result)Writables.getWritable(rb, new Result()); + byte [] row = deResult.getRow(); + assertTrue(Bytes.equals(row, rowA)); + + // Call sorted() first + deResult = (Result)Writables.getWritable(rb, new Result()); + assertTrue("results are not equivalent, first key mismatch", + result.raw()[0].equals(deResult.raw()[0])); + assertTrue("results are not equivalent, second key mismatch", + result.raw()[1].equals(deResult.raw()[1])); + + // Call raw() first + deResult = (Result)Writables.getWritable(rb, new Result()); + assertTrue("results are not equivalent, first key mismatch", + result.raw()[0].equals(deResult.raw()[0])); + assertTrue("results are not equivalent, second key mismatch", + result.raw()[1].equals(deResult.raw()[1])); + + + } + + @Test public void testResultArray() throws Exception { + byte [] rowA = Bytes.toBytes("rowA"); + byte [] famA = Bytes.toBytes("famA"); + byte [] qfA = Bytes.toBytes("qfA"); + byte [] valueA = Bytes.toBytes("valueA"); + + byte [] rowB = Bytes.toBytes("rowB"); + byte [] famB = Bytes.toBytes("famB"); + byte [] qfB = Bytes.toBytes("qfB"); + byte [] valueB = Bytes.toBytes("valueB"); + + KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA); + KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB); + + + Result result1 = new Result(new KeyValue[]{kvA, kvB}); + Result result2 = new Result(new KeyValue[]{kvB}); + Result result3 = new Result(new KeyValue[]{kvB}); + + Result [] results = new Result [] {result1, result2, result3}; + + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(byteStream); + Result.writeArray(out, results); + + byte [] rb = byteStream.toByteArray(); + + DataInputBuffer in = new DataInputBuffer(); + in.reset(rb, 0, rb.length); + + Result [] deResults = Result.readArray(in); + + assertTrue(results.length == deResults.length); + + for(int i=0;i keys = new ArrayList(); + Result r = new Result(keys); + Result [] results = new Result [] {r}; + + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(byteStream); + + Result.writeArray(out, results); + + results = null; + + byteStream = new ByteArrayOutputStream(); + out = new DataOutputStream(byteStream); + Result.writeArray(out, results); + + byte [] rb = byteStream.toByteArray(); + + DataInputBuffer in = new DataInputBuffer(); + in.reset(rb, 0, rb.length); + + Result [] deResults = Result.readArray(in); + + assertTrue(deResults.length == 0); + + results = new Result[0]; + + byteStream = new ByteArrayOutputStream(); + out = new DataOutputStream(byteStream); + Result.writeArray(out, results); + + rb = byteStream.toByteArray(); + + in = new DataInputBuffer(); + in.reset(rb, 0, rb.length); + + deResults = Result.readArray(in); + + assertTrue(deResults.length == 0); + + } + + @Test public void testTimeRange() throws Exception{ + TimeRange tr = new TimeRange(0,5); + byte [] mb = Writables.getBytes(tr); + TimeRange deserializedTr = + (TimeRange)Writables.getWritable(mb, new TimeRange()); + + assertEquals(tr.getMax(), deserializedTr.getMax()); + assertEquals(tr.getMin(), deserializedTr.getMin()); + + } + + @Test public void testKeyValue2() throws Exception { + final String name = "testKeyValue2"; + byte[] row = name.getBytes(); + byte[] fam = "fam".getBytes(); + byte[] qf = "qf".getBytes(); + long ts = System.currentTimeMillis(); + byte[] val = "val".getBytes(); + + KeyValue kv = new KeyValue(row, fam, qf, ts, val); + + byte [] mb = Writables.getBytes(kv); + KeyValue deserializedKv = + (KeyValue)Writables.getWritable(mb, new KeyValue()); + assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer())); + assertEquals(kv.getOffset(), deserializedKv.getOffset()); + assertEquals(kv.getLength(), deserializedKv.getLength()); + } + + protected static final int MAXVERSIONS = 3; + protected final static byte [] fam1 = Bytes.toBytes("colfamily1"); + protected final static byte [] fam2 = Bytes.toBytes("colfamily2"); + protected final static byte [] fam3 = Bytes.toBytes("colfamily3"); + protected static final byte [][] COLUMNS = {fam1, fam2, fam3}; + + /** + * Create a table of name name with {@link COLUMNS} for + * families. + * @param name Name to give table. + * @return Column descriptor. + */ + protected HTableDescriptor createTableDescriptor(final String name) { + return createTableDescriptor(name, MAXVERSIONS); + } + + /** + * Create a table of name name with {@link COLUMNS} for + * families. + * @param name Name to give table. + * @param versions How many versions to allow per column. + * @return Column descriptor. + */ + protected HTableDescriptor createTableDescriptor(final String name, + final int versions) { + HTableDescriptor htd = new HTableDescriptor(name); + htd.addFamily(new HColumnDescriptor(fam1) + .setMaxVersions(versions) + .setBlockCacheEnabled(false) + ); + htd.addFamily(new HColumnDescriptor(fam2) + .setMaxVersions(versions) + .setBlockCacheEnabled(false) + ); + htd.addFamily(new HColumnDescriptor(fam3) + .setMaxVersions(versions) + .setBlockCacheEnabled(false) + ); + return htd; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestServerName.java b/src/test/java/org/apache/hadoop/hbase/TestServerName.java new file mode 100644 index 0000000..eaa60c4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestServerName.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +import java.util.regex.Pattern; + +import org.apache.hadoop.hbase.util.Addressing; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestServerName { + @Test + public void testRegexPatterns() { + assertTrue(Pattern.matches(Addressing.VALID_PORT_REGEX, "123")); + assertFalse(Pattern.matches(Addressing.VALID_PORT_REGEX, "")); + assertTrue(ServerName.SERVERNAME_PATTERN.matcher("www1.example.org,1234,567").matches()); + ServerName.parseServerName("a.b.c,58102,1319771740322"); + ServerName.parseServerName("192.168.1.199,58102,1319771740322"); + ServerName.parseServerName("a.b.c:58102"); + ServerName.parseServerName("192.168.1.199:58102"); + } + + @Test public void testParseOfBytes() { + final String snStr = "www.example.org,1234,5678"; + ServerName sn = new ServerName(snStr); + byte [] versionedBytes = sn.getVersionedBytes(); + assertEquals(snStr, ServerName.parseVersionedServerName(versionedBytes).toString()); + final String hostnamePortStr = "www.example.org:1234"; + byte [] bytes = Bytes.toBytes(hostnamePortStr); + String expecting = + hostnamePortStr.replace(":", ServerName.SERVERNAME_SEPARATOR) + + ServerName.SERVERNAME_SEPARATOR + ServerName.NON_STARTCODE; + assertEquals(expecting, ServerName.parseVersionedServerName(bytes).toString()); + } + + @Test + public void testServerName() { + ServerName sn = new ServerName("www.example.org", 1234, 5678); + ServerName sn2 = new ServerName("www.example.org", 1234, 5678); + ServerName sn3 = new ServerName("www.example.org", 1234, 56789); + assertTrue(sn.equals(sn2)); + assertFalse(sn.equals(sn3)); + assertEquals(sn.hashCode(), sn2.hashCode()); + assertNotSame(sn.hashCode(), sn3.hashCode()); + assertEquals(sn.toString(), + ServerName.getServerName("www.example.org", 1234, 5678)); + assertEquals(sn.toString(), + ServerName.getServerName("www.example.org:1234", 5678)); + assertEquals(sn.toString(), + "www.example.org" + ServerName.SERVERNAME_SEPARATOR + + "1234" + ServerName.SERVERNAME_SEPARATOR + "5678"); + } + + @Test + public void getServerStartcodeFromServerName() { + ServerName sn = new ServerName("www.example.org", 1234, 5678); + assertEquals(5678, + ServerName.getServerStartcodeFromServerName(sn.toString())); + assertNotSame(5677, + ServerName.getServerStartcodeFromServerName(sn.toString())); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java new file mode 100644 index 0000000..1ee0500 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java @@ -0,0 +1,405 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.ZKConfig; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.ZooKeeper.States; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestZooKeeper { + private final Log LOG = LogFactory.getLog(this.getClass()); + + private final static HBaseTestingUtility + TEST_UTIL = new HBaseTestingUtility(); + private static HConnection persistentConnection; + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // create a connection *before* the cluster is started, to validate that the + // connection's ZK trackers are initialized on demand + persistentConnection = HConnectionManager.createConnection(TEST_UTIL.getConfiguration()); + + // Test we can first start the ZK cluster by itself + TEST_UTIL.startMiniZKCluster(); + TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); + TEST_UTIL.startMiniCluster(2); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + persistentConnection.close(); + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + TEST_UTIL.ensureSomeRegionServersAvailable(2); + } + + private ZooKeeperWatcher getZooKeeperWatcher(HConnection c) throws + NoSuchMethodException, InvocationTargetException, IllegalAccessException { + + Method getterZK = c.getClass().getMethod("getKeepAliveZooKeeperWatcher"); + getterZK.setAccessible(true); + + return (ZooKeeperWatcher) getterZK.invoke(c); + } + + /** + * See HBASE-1232 and http://wiki.apache.org/hadoop/ZooKeeper/FAQ#4. + * @throws IOException + * @throws InterruptedException + */ + // fails frequently, disabled for now, see HBASE-6406 + // @Test + public void testClientSessionExpired() + throws Exception { + LOG.info("testClientSessionExpired"); + Configuration c = new Configuration(TEST_UTIL.getConfiguration()); + new HTable(c, HConstants.META_TABLE_NAME).close(); + HConnection connection = HConnectionManager.getConnection(c); + ZooKeeperWatcher connectionZK = connection.getZooKeeperWatcher(); + TEST_UTIL.expireSession(connectionZK, false); + + // provoke session expiration by doing something with ZK + ZKUtil.dump(connectionZK); + + // Check that the old ZK connection is closed, means we did expire + System.err.println("ZooKeeper should have timed out"); + LOG.info("state=" + connectionZK.getRecoverableZooKeeper().getState()); + Assert.assertTrue(connectionZK.getRecoverableZooKeeper().getState(). + equals(States.CLOSED)); + + // Check that the client recovered + ZooKeeperWatcher newConnectionZK = connection.getZooKeeperWatcher(); + States state = newConnectionZK.getRecoverableZooKeeper().getState(); + LOG.info("state=" + state); + Assert.assertTrue(state.equals(States.CONNECTED) || state.equals(States.CONNECTING)); + } + + @Test + public void testRegionServerSessionExpired() throws Exception { + LOG.info("Starting testRegionServerSessionExpired"); + int metaIndex = TEST_UTIL.getMiniHBaseCluster().getServerWithMeta(); + TEST_UTIL.expireRegionServerSession(metaIndex); + testSanity(); + } + + // @Test Disabled because seems to make no sense expiring master session + // and then trying to create table (down in testSanity); on master side + // it will fail because the master's session has expired -- St.Ack 07/24/2012 + public void testMasterSessionExpired() throws Exception { + LOG.info("Starting testMasterSessionExpired"); + TEST_UTIL.expireMasterSession(); + testSanity(); + } + + /** + * Make sure we can use the cluster + * @throws Exception + */ + private void testSanity() throws Exception { + String tableName = "test"+System.currentTimeMillis(); + HBaseAdmin admin = new HBaseAdmin(new Configuration(TEST_UTIL.getConfiguration())); + testAdminSanity(admin, tableName); + HTable table = new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName); + testTableSanity(table, tableName); + } + + private void testSanity(HConnection conn, ExecutorService pool) throws Exception { + String tableName = "test"+System.currentTimeMillis(); + HBaseAdmin admin = new HBaseAdmin(persistentConnection); + testAdminSanity(admin, tableName); + HTable table = new HTable(Bytes.toBytes(tableName), persistentConnection, pool); + testTableSanity(table, tableName); + + } + private void testAdminSanity(HBaseAdmin admin, String tableName) throws Exception { + HTableDescriptor desc = new HTableDescriptor(tableName); + HColumnDescriptor family = new HColumnDescriptor("fam"); + desc.addFamily(family); + LOG.info("Creating table " + tableName); + admin.createTable(desc); + } + + private void testTableSanity(HTable table, String tableName) throws Exception { + Put put = new Put(Bytes.toBytes("testrow")); + put.add(Bytes.toBytes("fam"), Bytes.toBytes("col"), Bytes.toBytes("testdata")); + LOG.info("Putting table " + tableName); + table.put(put); + table.close(); + } + + @Test + public void testMultipleZK() { + try { + HTable localMeta = + new HTable(new Configuration(TEST_UTIL.getConfiguration()), HConstants.META_TABLE_NAME); + Configuration otherConf = new Configuration(TEST_UTIL.getConfiguration()); + otherConf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1"); + HTable ipMeta = new HTable(otherConf, HConstants.META_TABLE_NAME); + + // dummy, just to open the connection + localMeta.exists(new Get(HConstants.LAST_ROW)); + ipMeta.exists(new Get(HConstants.LAST_ROW)); + + // make sure they aren't the same + assertFalse(HConnectionManager.getConnection(localMeta.getConfiguration()).getZooKeeperWatcher() + == HConnectionManager.getConnection(otherConf).getZooKeeperWatcher()); + assertFalse(HConnectionManager.getConnection(localMeta.getConfiguration()) + .getZooKeeperWatcher().getQuorum().equals(HConnectionManager + .getConnection(otherConf).getZooKeeperWatcher().getQuorum())); + localMeta.close(); + ipMeta.close(); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + /** + * Create a znode with data + * @throws Exception + */ + @Test + public void testCreateWithParents() throws Exception { + ZooKeeperWatcher zkw = + new ZooKeeperWatcher(new Configuration(TEST_UTIL.getConfiguration()), + TestZooKeeper.class.getName(), null); + byte[] expectedData = new byte[] { 1, 2, 3 }; + ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4/testCreateWithParents", expectedData); + byte[] data = ZKUtil.getData(zkw, "/l1/l2/l3/l4/testCreateWithParents"); + assertEquals(Bytes.equals(expectedData, data), true); + ZKUtil.deleteNodeRecursively(zkw, "/l1"); + + ZKUtil.createWithParents(zkw, "/testCreateWithParents", expectedData); + data = ZKUtil.getData(zkw, "/testCreateWithParents"); + assertEquals(Bytes.equals(expectedData, data), true); + ZKUtil.deleteNodeRecursively(zkw, "/testCreateWithParents"); + } + + /** + * Create a bunch of znodes in a hierarchy, try deleting one that has childs + * (it will fail), then delete it recursively, then delete the last znode + * @throws Exception + */ + @Test + public void testZNodeDeletes() throws Exception { + ZooKeeperWatcher zkw = new ZooKeeperWatcher( + new Configuration(TEST_UTIL.getConfiguration()), + TestZooKeeper.class.getName(), null); + ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4"); + try { + ZKUtil.deleteNode(zkw, "/l1/l2"); + fail("We should not be able to delete if znode has childs"); + } catch (KeeperException ex) { + assertNotNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null)); + } + ZKUtil.deleteNodeRecursively(zkw, "/l1/l2"); + // make sure it really is deleted + assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null)); + + // do the same delete again and make sure it doesn't crash + ZKUtil.deleteNodeRecursively(zkw, "/l1/l2"); + + ZKUtil.deleteNode(zkw, "/l1"); + assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2", null)); + } + + @Test + public void testClusterKey() throws Exception { + testKey("server", "2181", "hbase"); + testKey("server1,server2,server3", "2181", "hbase"); + try { + ZKUtil.transformClusterKey("2181:hbase"); + } catch (IOException ex) { + // OK + } + } + + /** + * Test with a connection that existed before the cluster was started + */ + @Test + public void testPersistentConnection() throws Exception { + ExecutorService pool = new ThreadPoolExecutor(1, 10, 10, TimeUnit.SECONDS, + new SynchronousQueue()); + testSanity(persistentConnection, pool); + } + + private void testKey(String ensemble, String port, String znode) + throws IOException { + Configuration conf = new Configuration(); + String key = ensemble+":"+port+":"+znode; + String[] parts = ZKUtil.transformClusterKey(key); + assertEquals(ensemble, parts[0]); + assertEquals(port, parts[1]); + assertEquals(znode, parts[2]); + ZKUtil.applyClusterKeyToConf(conf, key); + assertEquals(parts[0], conf.get(HConstants.ZOOKEEPER_QUORUM)); + assertEquals(parts[1], conf.get(HConstants.ZOOKEEPER_CLIENT_PORT)); + assertEquals(parts[2], conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT)); + String reconstructedKey = ZKUtil.getZooKeeperClusterKey(conf); + assertEquals(key, reconstructedKey); + } + + /** + * A test for HBASE-3238 + * @throws IOException A connection attempt to zk failed + * @throws InterruptedException One of the non ZKUtil actions was interrupted + * @throws KeeperException Any of the zookeeper connections had a + * KeeperException + */ + @Test + public void testCreateSilentIsReallySilent() throws InterruptedException, + KeeperException, IOException { + Configuration c = TEST_UTIL.getConfiguration(); + + String aclZnode = "/aclRoot"; + String quorumServers = ZKConfig.getZKQuorumServersString(c); + int sessionTimeout = 5 * 1000; // 5 seconds + ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance); + zk.addAuthInfo("digest", "hbase:rox".getBytes()); + + // Assumes the root of the ZooKeeper space is writable as it creates a node + // wherever the cluster home is defined. + ZooKeeperWatcher zk2 = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + "testMasterAddressManagerFromZK", null); + + // I set this acl after the attempted creation of the cluster home node. + // Add retries in case of retryable zk exceptions. + while (true) { + try { + zk.setACL("/", ZooDefs.Ids.CREATOR_ALL_ACL, -1); + break; + } catch (KeeperException e) { + switch (e.code()) { + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + LOG.warn("Possibly transient ZooKeeper exception: " + e); + Threads.sleep(100); + break; + default: + throw e; + } + } + } + + while (true) { + try { + zk.create(aclZnode, null, ZooDefs.Ids.CREATOR_ALL_ACL, CreateMode.PERSISTENT); + break; + } catch (KeeperException e) { + switch (e.code()) { + case CONNECTIONLOSS: + case SESSIONEXPIRED: + case OPERATIONTIMEOUT: + LOG.warn("Possibly transient ZooKeeper exception: " + e); + Threads.sleep(100); + break; + default: + throw e; + } + } + } + zk.close(); + ZKUtil.createAndFailSilent(zk2, aclZnode); + } + + /** + * Test should not fail with NPE when getChildDataAndWatchForNewChildren + * invoked with wrongNode + */ + @Test + public void testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE() + throws Exception { + ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + "testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE", null); + ZKUtil.getChildDataAndWatchForNewChildren(zkw, "/wrongNode"); + } + + /** + * Master recovery when the znode already exists. Internally, this + * test differs from {@link #testMasterSessionExpired} because here + * the master znode will exist in ZK. + */ + @Test(timeout=60000) + public void testMasterZKSessionRecoveryFailure() throws Exception { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + HMaster m = cluster.getMaster(); + m.abort("Test recovery from zk session expired", + new KeeperException.SessionExpiredException()); + assertFalse(m.isStopped()); + testSanity(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/TimestampTestBase.java b/src/test/java/org/apache/hadoop/hbase/TimestampTestBase.java new file mode 100644 index 0000000..633d7a9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TimestampTestBase.java @@ -0,0 +1,276 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Tests user specifiable time stamps putting, getting and scanning. Also + * tests same in presence of deletes. Test cores are written so can be + * run against an HRegion and against an HTable: i.e. both local and remote. + */ +public class TimestampTestBase extends HBaseTestCase { + private static final long T0 = 10L; + private static final long T1 = 100L; + private static final long T2 = 200L; + + public static final byte [] FAMILY_NAME = Bytes.toBytes("colfamily11"); + private static final byte [] QUALIFIER_NAME = Bytes.toBytes("contents"); + + private static final byte [] ROW = Bytes.toBytes("row"); + + /* + * Run test that delete works according to description in hadoop-1784. + * @param incommon + * @param flusher + * @throws IOException + */ + public static void doTestDelete(final Incommon incommon, FlushCache flusher) + throws IOException { + // Add values at various timestamps (Values are timestampes as bytes). + put(incommon, T0); + put(incommon, T1); + put(incommon, T2); + put(incommon); + // Verify that returned versions match passed timestamps. + assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1}); + + // If I delete w/o specifying a timestamp, this means I'm deleting the + // latest. + delete(incommon); + // Verify that I get back T2 through T1 -- that the latest version has + // been deleted. + assertVersions(incommon, new long [] {T2, T1, T0}); + + // Flush everything out to disk and then retry + flusher.flushcache(); + assertVersions(incommon, new long [] {T2, T1, T0}); + + // Now add, back a latest so I can test remove other than the latest. + put(incommon); + assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1}); + delete(incommon, T2); + assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0}); + // Flush everything out to disk and then retry + flusher.flushcache(); + assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0}); + + // Now try deleting all from T2 back inclusive (We first need to add T2 + // back into the mix and to make things a little interesting, delete and + // then readd T1. + put(incommon, T2); + delete(incommon, T1); + put(incommon, T1); + + Delete delete = new Delete(ROW); + delete.deleteColumns(FAMILY_NAME, QUALIFIER_NAME, T2); + incommon.delete(delete, null, true); + + // Should only be current value in set. Assert this is so + assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP); + + // Flush everything out to disk and then redo above tests + flusher.flushcache(); + assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP); + } + + private static void assertOnlyLatest(final Incommon incommon, + final long currentTime) + throws IOException { + Get get = null; + get = new Get(ROW); + get.addColumn(FAMILY_NAME, QUALIFIER_NAME); + get.setMaxVersions(3); + Result result = incommon.get(get); + assertEquals(1, result.size()); + long time = Bytes.toLong(result.raw()[0].getValue()); + assertEquals(time, currentTime); + } + + /* + * Assert that returned versions match passed in timestamps and that results + * are returned in the right order. Assert that values when converted to + * longs match the corresponding passed timestamp. + * @param r + * @param tss + * @throws IOException + */ + public static void assertVersions(final Incommon incommon, final long [] tss) + throws IOException { + // Assert that 'latest' is what we expect. + Get get = null; + get = new Get(ROW); + get.addColumn(FAMILY_NAME, QUALIFIER_NAME); + Result r = incommon.get(get); + byte [] bytes = r.getValue(FAMILY_NAME, QUALIFIER_NAME); + long t = Bytes.toLong(bytes); + assertEquals(tss[0], t); + + // Now assert that if we ask for multiple versions, that they come out in + // order. + get = new Get(ROW); + get.addColumn(FAMILY_NAME, QUALIFIER_NAME); + get.setMaxVersions(tss.length); + Result result = incommon.get(get); + KeyValue [] kvs = result.raw(); + assertEquals(kvs.length, tss.length); + for(int i=0;ivalue = +// new TreeMap(Bytes.BYTES_COMPARATOR); +// while (scanner.next(key, value)) { +// assertTrue(key.getTimestamp() <= ts); +// // Content matches the key or HConstants.LATEST_TIMESTAMP. +// // (Key does not match content if we 'put' with LATEST_TIMESTAMP). +// long l = Bytes.toLong(value.get(COLUMN).getValue()); +// assertTrue(key.getTimestamp() == l || +// HConstants.LATEST_TIMESTAMP == l); +// count++; +// value.clear(); +// } + } finally { + scanner.close(); + } + return count; + } + + public static void put(final Incommon loader, final long ts) + throws IOException { + put(loader, Bytes.toBytes(ts), ts); + } + + public static void put(final Incommon loader) + throws IOException { + long ts = HConstants.LATEST_TIMESTAMP; + put(loader, Bytes.toBytes(ts), ts); + } + + /* + * Put values. + * @param loader + * @param bytes + * @param ts + * @throws IOException + */ + public static void put(final Incommon loader, final byte [] bytes, + final long ts) + throws IOException { + Put put = new Put(ROW, ts, null); + put.setWriteToWAL(false); + put.add(FAMILY_NAME, QUALIFIER_NAME, bytes); + loader.put(put); + } + + public static void delete(final Incommon loader) throws IOException { + delete(loader, null); + } + + public static void delete(final Incommon loader, final byte [] column) + throws IOException { + delete(loader, column, HConstants.LATEST_TIMESTAMP); + } + + public static void delete(final Incommon loader, final long ts) + throws IOException { + delete(loader, null, ts); + } + + public static void delete(final Incommon loader, final byte [] column, + final long ts) + throws IOException { + Delete delete = ts == HConstants.LATEST_TIMESTAMP? + new Delete(ROW): new Delete(ROW, ts, null); + delete.deleteColumn(FAMILY_NAME, QUALIFIER_NAME, ts); + loader.delete(delete, null, true); + } + + public static Result get(final Incommon loader) throws IOException { + return loader.get(new Get(ROW)); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java b/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java new file mode 100644 index 0000000..8915f6f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.avro; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericArray; +import org.apache.avro.generic.GenericData; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.avro.generated.AColumn; +import org.apache.hadoop.hbase.avro.generated.AColumnValue; +import org.apache.hadoop.hbase.avro.generated.AFamilyDescriptor; +import org.apache.hadoop.hbase.avro.generated.AGet; +import org.apache.hadoop.hbase.avro.generated.APut; +import org.apache.hadoop.hbase.avro.generated.ATableDescriptor; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Unit testing for AvroServer.HBaseImpl, a part of the + * org.apache.hadoop.hbase.avro package. + */ +@Category(MediumTests.class) +public class TestAvroServer { + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + // Static names for tables, columns, rows, and values + // TODO(hammer): Better style to define these in test method? + private static ByteBuffer tableAname = ByteBuffer.wrap(Bytes.toBytes("tableA")); + private static ByteBuffer tableBname = ByteBuffer.wrap(Bytes.toBytes("tableB")); + private static ByteBuffer familyAname = ByteBuffer.wrap(Bytes.toBytes("FamilyA")); + private static ByteBuffer qualifierAname = ByteBuffer.wrap(Bytes.toBytes("QualifierA")); + private static ByteBuffer rowAname = ByteBuffer.wrap(Bytes.toBytes("RowA")); + private static ByteBuffer valueA = ByteBuffer.wrap(Bytes.toBytes("ValueA")); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + // Nothing to do. + } + + /** + * @throws java.lang.Exception + */ + @After + public void tearDown() throws Exception { + // Nothing to do. + } + + /** + * Tests for creating, enabling, disabling, modifying, and deleting tables. + * + * @throws Exception + */ + @Test (timeout=300000) + public void testTableAdminAndMetadata() throws Exception { + AvroServer.HBaseImpl impl = + new AvroServer.HBaseImpl(TEST_UTIL.getConfiguration()); + + assertEquals(impl.listTables().size(), 0); + + ATableDescriptor tableA = new ATableDescriptor(); + tableA.name = tableAname; + impl.createTable(tableA); + assertEquals(impl.listTables().size(), 1); + assertTrue(impl.isTableEnabled(tableAname)); + assertTrue(impl.tableExists(tableAname)); + + ATableDescriptor tableB = new ATableDescriptor(); + tableB.name = tableBname; + impl.createTable(tableB); + assertEquals(impl.listTables().size(), 2); + + impl.disableTable(tableBname); + assertFalse(impl.isTableEnabled(tableBname)); + + impl.deleteTable(tableBname); + assertEquals(impl.listTables().size(), 1); + + impl.disableTable(tableAname); + assertFalse(impl.isTableEnabled(tableAname)); + + long oldMaxFileSize = impl.describeTable(tableAname).maxFileSize; + tableA.maxFileSize = 123456L; + impl.modifyTable(tableAname, tableA); + + // It can take a while for the change to take effect. Wait here a while. + while(impl.describeTable(tableAname).maxFileSize == oldMaxFileSize) { + Threads.sleep(100); + } + + assertTrue(impl.describeTable(tableAname).maxFileSize == 123456L); + assertEquals(123456L, (long) impl.describeTable(tableAname).maxFileSize); +/* DISABLED FOR NOW TILL WE HAVE BETTER DISABLE/ENABLE + impl.enableTable(tableAname); + assertTrue(impl.isTableEnabled(tableAname)); + + impl.disableTable(tableAname); + */ + impl.deleteTable(tableAname); + } + + /** + * Tests for creating, modifying, and deleting column families. + * + * @throws Exception + */ + @Test + public void testFamilyAdminAndMetadata() throws Exception { + AvroServer.HBaseImpl impl = + new AvroServer.HBaseImpl(TEST_UTIL.getConfiguration()); + + ATableDescriptor tableA = new ATableDescriptor(); + tableA.name = tableAname; + AFamilyDescriptor familyA = new AFamilyDescriptor(); + familyA.name = familyAname; + Schema familyArraySchema = Schema.createArray(AFamilyDescriptor.SCHEMA$); + GenericArray families = new GenericData.Array(1, familyArraySchema); + families.add(familyA); + tableA.families = families; + impl.createTable(tableA); + assertEquals(impl.describeTable(tableAname).families.size(), 1); + + impl.disableTable(tableAname); + assertFalse(impl.isTableEnabled(tableAname)); + + familyA.maxVersions = 123456; + impl.modifyFamily(tableAname, familyAname, familyA); + assertEquals((int) impl.describeFamily(tableAname, familyAname).maxVersions, 123456); + + impl.deleteFamily(tableAname, familyAname); + assertEquals(impl.describeTable(tableAname).families.size(), 0); + + impl.deleteTable(tableAname); + } + + /** + * Tests for adding, reading, and deleting data. + * + * @throws Exception + */ + @Test + public void testDML() throws Exception { + AvroServer.HBaseImpl impl = + new AvroServer.HBaseImpl(TEST_UTIL.getConfiguration()); + + ATableDescriptor tableA = new ATableDescriptor(); + tableA.name = tableAname; + AFamilyDescriptor familyA = new AFamilyDescriptor(); + familyA.name = familyAname; + Schema familyArraySchema = Schema.createArray(AFamilyDescriptor.SCHEMA$); + GenericArray families = new GenericData.Array(1, familyArraySchema); + families.add(familyA); + tableA.families = families; + impl.createTable(tableA); + assertEquals(impl.describeTable(tableAname).families.size(), 1); + + AGet getA = new AGet(); + getA.row = rowAname; + Schema columnsSchema = Schema.createArray(AColumn.SCHEMA$); + GenericArray columns = new GenericData.Array(1, columnsSchema); + AColumn column = new AColumn(); + column.family = familyAname; + column.qualifier = qualifierAname; + columns.add(column); + getA.columns = columns; + + assertFalse(impl.exists(tableAname, getA)); + + APut putA = new APut(); + putA.row = rowAname; + Schema columnValuesSchema = Schema.createArray(AColumnValue.SCHEMA$); + GenericArray columnValues = new GenericData.Array(1, columnValuesSchema); + AColumnValue acv = new AColumnValue(); + acv.family = familyAname; + acv.qualifier = qualifierAname; + acv.value = valueA; + columnValues.add(acv); + putA.columnValues = columnValues; + + impl.put(tableAname, putA); + assertTrue(impl.exists(tableAname, getA)); + + assertEquals(impl.get(tableAname, getA).entries.size(), 1); + + impl.disableTable(tableAname); + impl.deleteTable(tableAname); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/avro/TestAvroUtil.java b/src/test/java/org/apache/hadoop/hbase/avro/TestAvroUtil.java new file mode 100644 index 0000000..80c8591 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/avro/TestAvroUtil.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.avro; + + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.avro.generated.AResult; +import org.apache.hadoop.hbase.client.Result; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +@Category(SmallTests.class) +public class TestAvroUtil { + + + @Test + public void testGetEmpty() { + Result result = Mockito.mock(Result.class); + Mockito.when(result.getRow()).thenReturn(null); + //Get on a row, that does not exist, returns a result, + //whose row is null. + AResult aresult = AvroUtil.resultToAResult(result); + Assert.assertNotNull(aresult); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java new file mode 100644 index 0000000..7e28f6b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java @@ -0,0 +1,439 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.backup.HFileArchiver; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; +import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; +import org.apache.hadoop.hbase.util.StoppableImplementation; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up + * a region + */ +@Category(MediumTests.class) +public class TestHFileArchiving { + + private static final Log LOG = LogFactory.getLog(TestHFileArchiving.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final byte[] TEST_FAM = Bytes.toBytes("fam"); + + /** + * Setup the config for the cluster + */ + @BeforeClass + public static void setupCluster() throws Exception { + setupConf(UTIL.getConfiguration()); + UTIL.startMiniCluster(); + + // We don't want the cleaner to remove files. The tests do that. + UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().interrupt(); + } + + private static void setupConf(Configuration conf) { + // disable the ui + conf.setInt("hbase.regionsever.info.port", -1); + // drop the memstore size so we get flushes + conf.setInt("hbase.hregion.memstore.flush.size", 25000); + // disable major compactions + conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0); + + // prevent aggressive region split + conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, + ConstantSizeRegionSplitPolicy.class.getName()); + } + + @After + public void tearDown() throws Exception { + // cleanup the archive directory + try { + clearArchiveDirectory(); + } catch (IOException e) { + Assert.fail("Failure to delete archive directory:" + e.getMessage()); + } + } + + @AfterClass + public static void cleanupTest() throws Exception { + try { + UTIL.shutdownMiniCluster(); + } catch (Exception e) { + // NOOP; + } + } + + @Test + public void testRemovesRegionDirOnArchive() throws Exception { + byte[] TABLE_NAME = Bytes.toBytes("testRemovesRegionDirOnArchive"); + UTIL.createTable(TABLE_NAME, TEST_FAM); + + final HBaseAdmin admin = UTIL.getHBaseAdmin(); + + // get the current store files for the region + List servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); + // make sure we only have 1 region serving this table + assertEquals(1, servingRegions.size()); + HRegion region = servingRegions.get(0); + + // and load the table + UTIL.loadRegion(region, TEST_FAM); + + // shutdown the table so we can manipulate the files + admin.disableTable(TABLE_NAME); + + FileSystem fs = UTIL.getTestFileSystem(); + + // now attempt to depose the region + Path regionDir = HRegion.getRegionDir(region.getTableDir().getParent(), region.getRegionInfo()); + + HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo()); + + // check for the existence of the archive directory and some files in it + Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region); + assertTrue(fs.exists(archiveDir)); + + // check to make sure the store directory was copied + FileStatus[] stores = fs.listStatus(archiveDir); + assertTrue(stores.length == 1); + + // make sure we archived the store files + FileStatus[] storeFiles = fs.listStatus(stores[0].getPath()); + assertTrue(storeFiles.length > 0); + + // then ensure the region's directory isn't present + assertFalse(fs.exists(regionDir)); + + UTIL.deleteTable(TABLE_NAME); + } + + /** + * Test that the region directory is removed when we archive a region without store files, but + * still has hidden files. + * @throws Exception + */ + @Test + public void testDeleteRegionWithNoStoreFiles() throws Exception { + byte[] TABLE_NAME = Bytes.toBytes("testDeleteRegionWithNoStoreFiles"); + UTIL.createTable(TABLE_NAME, TEST_FAM); + + // get the current store files for the region + List servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); + // make sure we only have 1 region serving this table + assertEquals(1, servingRegions.size()); + HRegion region = servingRegions.get(0); + + FileSystem fs = region.getFilesystem(); + + // make sure there are some files in the regiondir + Path rootDir = FSUtils.getRootDir(fs.getConf()); + Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo()); + FileStatus[] regionFiles = FSUtils.listStatus(fs, regionDir, null); + Assert.assertNotNull("No files in the region directory", regionFiles); + if (LOG.isDebugEnabled()) { + List files = new ArrayList(); + for (FileStatus file : regionFiles) { + files.add(file.getPath()); + } + LOG.debug("Current files:" + files); + } + // delete the visible folders so we just have hidden files/folders + final PathFilter dirFilter = new FSUtils.DirFilter(fs); + PathFilter nonHidden = new PathFilter() { + @Override + public boolean accept(Path file) { + return dirFilter.accept(file) && !file.getName().toString().startsWith("."); + } + }; + FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden); + for (FileStatus store : storeDirs) { + LOG.debug("Deleting store for test"); + fs.delete(store.getPath(), true); + } + + // then archive the region + HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo()); + + // and check to make sure the region directoy got deleted + assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir)); + + UTIL.deleteTable(TABLE_NAME); + } + + @Test + public void testArchiveOnTableDelete() throws Exception { + byte[] TABLE_NAME = Bytes.toBytes("testArchiveOnTableDelete"); + UTIL.createTable(TABLE_NAME, TEST_FAM); + + List servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); + // make sure we only have 1 region serving this table + assertEquals(1, servingRegions.size()); + HRegion region = servingRegions.get(0); + + // get the parent RS and monitor + HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME); + FileSystem fs = hrs.getFileSystem(); + + // put some data on the region + LOG.debug("-------Loading table"); + UTIL.loadRegion(region, TEST_FAM); + + // get the hfiles in the region + List regions = hrs.getOnlineRegions(TABLE_NAME); + assertEquals("More that 1 region for test table.", 1, regions.size()); + + region = regions.get(0); + // wait for all the compactions to complete + region.waitForFlushesAndCompactions(); + + // disable table to prevent new updates + UTIL.getHBaseAdmin().disableTable(TABLE_NAME); + LOG.debug("Disabled table"); + + // remove all the files from the archive to get a fair comparison + clearArchiveDirectory(); + + // then get the current store files + Path regionDir = region.getRegionDir(); + List storeFiles = getRegionStoreFiles(fs, regionDir); + + // then delete the table so the hfiles get archived + UTIL.deleteTable(TABLE_NAME); + + // then get the files in the archive directory. + Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration()); + List archivedFiles = getAllFileNames(fs, archiveDir); + Collections.sort(storeFiles); + Collections.sort(archivedFiles); + + LOG.debug("Store files:"); + for (int i = 0; i < storeFiles.size(); i++) { + LOG.debug(i + " - " + storeFiles.get(i)); + } + LOG.debug("Archive files:"); + for (int i = 0; i < archivedFiles.size(); i++) { + LOG.debug(i + " - " + archivedFiles.get(i)); + } + + assertTrue("Archived files are missing some of the store files!", + archivedFiles.containsAll(storeFiles)); + } + + /** + * Test that the store files are archived when a column family is removed. + * @throws Exception + */ + @Test + public void testArchiveOnTableFamilyDelete() throws Exception { + byte[] TABLE_NAME = Bytes.toBytes("testArchiveOnTableFamilyDelete"); + UTIL.createTable(TABLE_NAME, TEST_FAM); + + List servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); + // make sure we only have 1 region serving this table + assertEquals(1, servingRegions.size()); + HRegion region = servingRegions.get(0); + + // get the parent RS and monitor + HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME); + FileSystem fs = hrs.getFileSystem(); + + // put some data on the region + LOG.debug("-------Loading table"); + UTIL.loadRegion(region, TEST_FAM); + + // get the hfiles in the region + List regions = hrs.getOnlineRegions(TABLE_NAME); + assertEquals("More that 1 region for test table.", 1, regions.size()); + + region = regions.get(0); + // wait for all the compactions to complete + region.waitForFlushesAndCompactions(); + + // disable table to prevent new updates + UTIL.getHBaseAdmin().disableTable(TABLE_NAME); + LOG.debug("Disabled table"); + + // remove all the files from the archive to get a fair comparison + clearArchiveDirectory(); + + // then get the current store files + Path regionDir = region.getRegionDir(); + List storeFiles = getRegionStoreFiles(fs, regionDir); + + // then delete the table so the hfiles get archived + UTIL.getHBaseAdmin().deleteColumn(TABLE_NAME, TEST_FAM); + + // then get the files in the archive directory. + Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration()); + List archivedFiles = getAllFileNames(fs, archiveDir); + Collections.sort(storeFiles); + Collections.sort(archivedFiles); + + LOG.debug("Store files:"); + for (int i = 0; i < storeFiles.size(); i++) { + LOG.debug(i + " - " + storeFiles.get(i)); + } + LOG.debug("Archive files:"); + for (int i = 0; i < archivedFiles.size(); i++) { + LOG.debug(i + " - " + archivedFiles.get(i)); + } + + assertTrue("Archived files are missing some of the store files!", + archivedFiles.containsAll(storeFiles)); + + UTIL.deleteTable(TABLE_NAME); + } + + /** + * Test HFileArchiver.resolveAndArchive() race condition HBASE-7643 + */ + @Test + public void testCleaningRace() throws Exception { + final long TEST_TIME = 20 * 1000; + + Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration(); + Path rootDir = UTIL.getDataTestDir("testCleaningRace"); + FileSystem fs = UTIL.getTestFileSystem(); + + Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); + Path regionDir = new Path("table", "abcdef"); + Path familyDir = new Path(regionDir, "cf"); + + Path sourceRegionDir = new Path(rootDir, regionDir); + fs.mkdirs(sourceRegionDir); + + Stoppable stoppable = new StoppableImplementation(); + + // The cleaner should be looping without long pauses to reproduce the race condition. + HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir); + try { + cleaner.start(); + + // Keep creating/archiving new files while the cleaner is running in the other thread + long startTime = System.currentTimeMillis(); + for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) { + Path file = new Path(familyDir, String.valueOf(fid)); + Path sourceFile = new Path(rootDir, file); + Path archiveFile = new Path(archiveDir, file); + + fs.createNewFile(sourceFile); + + try { + // Try to archive the file + HFileArchiver.archiveRegion(fs, rootDir, + sourceRegionDir.getParent(), sourceRegionDir); + + // The archiver succeded, the file is no longer in the original location + // but it's in the archive location. + LOG.debug("hfile=" + fid + " should be in the archive"); + assertTrue(fs.exists(archiveFile)); + assertFalse(fs.exists(sourceFile)); + } catch (IOException e) { + // The archiver is unable to archive the file. Probably HBASE-7643 race condition. + // in this case, the file should not be archived, and we should have the file + // in the original location. + LOG.debug("hfile=" + fid + " should be in the source location"); + assertFalse(fs.exists(archiveFile)); + assertTrue(fs.exists(sourceFile)); + + // Avoid to have this file in the next run + fs.delete(sourceFile, false); + } + } + } finally { + stoppable.stop("test end"); + cleaner.join(); + fs.delete(rootDir, true); + } + } + + private void clearArchiveDirectory() throws IOException { + UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true); + } + + /** + * Get the names of all the files below the given directory + * @param fs + * @param archiveDir + * @return + * @throws IOException + */ + private List getAllFileNames(final FileSystem fs, Path archiveDir) throws IOException { + FileStatus[] files = FSUtils.listStatus(fs, archiveDir, null); + return recurseOnFiles(fs, files, new ArrayList()); + } + + /** Recursively lookup all the file names under the file[] array **/ + private List recurseOnFiles(FileSystem fs, FileStatus[] files, List fileNames) + throws IOException { + if (files == null || files.length == 0) return fileNames; + + for (FileStatus file : files) { + if (file.isDir()) { + recurseOnFiles(fs, FSUtils.listStatus(fs, file.getPath(), null), fileNames); + } else fileNames.add(file.getPath().getName()); + } + return fileNames; + } + + private List getRegionStoreFiles(final FileSystem fs, final Path regionDir) + throws IOException { + List storeFiles = getAllFileNames(fs, regionDir); + // remove all the non-storefile named files for the region + for (int i = 0; i < storeFiles.size(); i++) { + String file = storeFiles.get(i); + if (file.contains(HRegion.REGIONINFO_FILE) || file.contains("hlog")) { + storeFiles.remove(i--); + } + } + storeFiles.remove(HRegion.REGIONINFO_FILE); + return storeFiles; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java new file mode 100644 index 0000000..5071a89 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java @@ -0,0 +1,548 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.catalog; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.net.ConnectException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import junit.framework.Assert; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HConnectionTestingUtility; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.RetriesExhaustedException; +import org.apache.hadoop.hbase.client.ServerCallable; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.util.Progressable; +import org.apache.zookeeper.KeeperException; +import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test {@link CatalogTracker} + */ +@Category(MediumTests.class) +public class TestCatalogTracker { + private static final Log LOG = LogFactory.getLog(TestCatalogTracker.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final ServerName SN = + new ServerName("example.org", 1234, System.currentTimeMillis()); + private ZooKeeperWatcher watcher; + private Abortable abortable; + + @BeforeClass public static void beforeClass() throws Exception { + // Set this down so tests run quicker + UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3); + UTIL.startMiniZKCluster(); + } + + @AfterClass public static void afterClass() throws IOException { + UTIL.getZkCluster().shutdown(); + } + + @Before public void before() throws IOException { + this.abortable = new Abortable() { + @Override + public void abort(String why, Throwable e) { + LOG.info(why, e); + } + + @Override + public boolean isAborted() { + return false; + } + }; + this.watcher = new ZooKeeperWatcher(UTIL.getConfiguration(), + this.getClass().getSimpleName(), this.abortable, true); + } + + @After public void after() { + this.watcher.close(); + } + + private CatalogTracker constructAndStartCatalogTracker(final HConnection c) + throws IOException, InterruptedException { + CatalogTracker ct = new CatalogTracker(this.watcher, UTIL.getConfiguration(), + c, this.abortable); + ct.start(); + return ct; + } + + /** + * Test that we get notification if .META. moves. + * @throws IOException + * @throws InterruptedException + * @throws KeeperException + */ + @Test public void testThatIfMETAMovesWeAreNotified() + throws IOException, InterruptedException, KeeperException { + HConnection connection = Mockito.mock(HConnection.class); + constructAndStartCatalogTracker(connection); + try { + RootLocationEditor.setRootLocation(this.watcher, + new ServerName("example.com", 1234, System.currentTimeMillis())); + } finally { + // Clean out root location or later tests will be confused... they presume + // start fresh in zk. + RootLocationEditor.deleteRootLocation(this.watcher); + } + } + + /** + * Test interruptable while blocking wait on root and meta. + * @throws IOException + * @throws InterruptedException + */ + @Test public void testInterruptWaitOnMetaAndRoot() + throws IOException, InterruptedException { + HRegionInterface implementation = Mockito.mock(HRegionInterface.class); + HConnection connection = mockConnection(implementation); + try { + final CatalogTracker ct = constructAndStartCatalogTracker(connection); + ServerName hsa = ct.getRootLocation(); + Assert.assertNull(hsa); + ServerName meta = ct.getMetaLocation(); + Assert.assertNull(meta); + Thread t = new Thread() { + @Override + public void run() { + try { + ct.waitForMeta(); + } catch (InterruptedException e) { + throw new RuntimeException("Interrupted", e); + } + } + }; + t.start(); + while (!t.isAlive()) + Threads.sleep(1); + Threads.sleep(1); + assertTrue(t.isAlive()); + ct.stop(); + // Join the thread... should exit shortly. + t.join(); + } finally { + HConnectionManager.deleteConnection(UTIL.getConfiguration()); + } + } + + /** + * Test for HBASE-4288. Throw an IOE when trying to verify meta region and + * prove it doesn't cause master shutdown. + * @see HBASE-4288 + * @throws IOException + * @throws InterruptedException + * @throws KeeperException + */ + @Test + public void testServerNotRunningIOException() + throws IOException, InterruptedException, KeeperException { + // Mock an HRegionInterface. + final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); + HConnection connection = mockConnection(implementation); + try { + // If a 'getRegionInfo' is called on mocked HRegionInterface, throw IOE + // the first time. 'Succeed' the second time we are called. + Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())). + thenThrow(new IOException("Server not running, aborting")). + thenReturn(new HRegionInfo()); + + // After we encounter the above 'Server not running', we should catch the + // IOE and go into retrying for the meta mode. We'll do gets on -ROOT- to + // get new meta location. Return something so this 'get' succeeds + // (here we mock up getRegionServerWithRetries, the wrapper around + // the actual get). + + // TODO: Refactor. This method has been moved out of HConnection. + // It works for now but has been deprecated. + Mockito.when(connection.getRegionServerWithRetries((ServerCallable)Mockito.any())). + thenReturn(getMetaTableRowResult()); + + // Now start up the catalogtracker with our doctored Connection. + final CatalogTracker ct = constructAndStartCatalogTracker(connection); + try { + // Set a location for root and meta. + RootLocationEditor.setRootLocation(this.watcher, SN); + ct.setMetaLocation(SN); + // Call the method that HBASE-4288 calls. It will try and verify the + // meta location and will fail on first attempt then go into a long wait. + // So, do this in a thread and then reset meta location to break it out + // of its wait after a bit of time. + final AtomicBoolean metaSet = new AtomicBoolean(false); + final CountDownLatch latch = new CountDownLatch(1); + Thread t = new Thread() { + @Override + public void run() { + try { + latch.countDown(); + metaSet.set(ct.waitForMeta(100000) != null); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }; + t.start(); + latch.await(); + Threads.sleep(1); + // Now reset the meta as though it were redeployed. + ct.setMetaLocation(SN); + t.join(); + Assert.assertTrue(metaSet.get()); + } finally { + // Clean out root and meta locations or later tests will be confused... + // they presume start fresh in zk. + ct.resetMetaLocation(); + RootLocationEditor.deleteRootLocation(this.watcher); + } + } finally { + // Clear out our doctored connection or could mess up subsequent tests. + HConnectionManager.deleteConnection(UTIL.getConfiguration()); + } + } + + private void testVerifyMetaRegionLocationWithException(Exception ex) + throws IOException, InterruptedException, KeeperException { + // Mock an HRegionInterface. + final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); + HConnection connection = mockConnection(implementation); + try { + // If a 'get' is called on mocked interface, throw connection refused. + Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())). + thenThrow(ex); + // Now start up the catalogtracker with our doctored Connection. + final CatalogTracker ct = constructAndStartCatalogTracker(connection); + try { + RootLocationEditor.setRootLocation(this.watcher, SN); + long timeout = UTIL.getConfiguration(). + getLong("hbase.catalog.verification.timeout", 1000); + Assert.assertFalse(ct.verifyMetaRegionLocation(timeout)); + } finally { + // Clean out root location or later tests will be confused... they + // presume start fresh in zk. + RootLocationEditor.deleteRootLocation(this.watcher); + } + } finally { + // Clear out our doctored connection or could mess up subsequent tests. + HConnectionManager.deleteConnection(UTIL.getConfiguration()); + } + } + + /** + * Test we survive a connection refused {@link ConnectException} + * @throws IOException + * @throws InterruptedException + * @throws KeeperException + */ + @Test + public void testGetMetaServerConnectionFails() + throws IOException, InterruptedException, KeeperException { + testVerifyMetaRegionLocationWithException(new ConnectException("Connection refused")); + } + + /** + * Test that verifyMetaRegionLocation properly handles getting a + * ServerNotRunningException. See HBASE-4470. + * Note this doesn't check the exact exception thrown in the + * HBASE-4470 as there it is thrown from getHConnection() and + * here it is thrown from get() -- but those are both called + * from the same function anyway, and this way is less invasive than + * throwing from getHConnection would be. + * + * @throws IOException + * @throws InterruptedException + * @throws KeeperException + */ + @Test + public void testVerifyMetaRegionServerNotRunning() + throws IOException, InterruptedException, KeeperException { + testVerifyMetaRegionLocationWithException(new ServerNotRunningYetException("mock")); + } + + /** + * Test get of root region fails properly if nothing to connect to. + * @throws IOException + * @throws InterruptedException + * @throws KeeperException + */ + @Test + public void testVerifyRootRegionLocationFails() + throws IOException, InterruptedException, KeeperException { + HConnection connection = Mockito.mock(HConnection.class); + ConnectException connectException = + new ConnectException("Connection refused"); + final HRegionInterface implementation = + Mockito.mock(HRegionInterface.class); + Mockito.when(implementation.getRegionInfo((byte [])Mockito.any())). + thenThrow(connectException); + Mockito.when(connection.getHRegionConnection(Mockito.anyString(), + Mockito.anyInt(), Mockito.anyBoolean())). + thenReturn(implementation); + final CatalogTracker ct = constructAndStartCatalogTracker(connection); + try { + RootLocationEditor.setRootLocation(this.watcher, + new ServerName("example.com", 1234, System.currentTimeMillis())); + Assert.assertFalse(ct.verifyRootRegionLocation(100)); + } finally { + // Clean out root location or later tests will be confused... they presume + // start fresh in zk. + RootLocationEditor.deleteRootLocation(this.watcher); + } + } + + @Test (expected = NotAllMetaRegionsOnlineException.class) + public void testTimeoutWaitForRoot() + throws IOException, InterruptedException { + HConnection connection = Mockito.mock(HConnection.class); + final CatalogTracker ct = constructAndStartCatalogTracker(connection); + ct.waitForRoot(100); + } + + @Test (expected = RetriesExhaustedException.class) + public void testTimeoutWaitForMeta() + throws IOException, InterruptedException { + HConnection connection = + HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration()); + try { + final CatalogTracker ct = constructAndStartCatalogTracker(connection); + ct.waitForMeta(100); + } finally { + HConnectionManager.deleteConnection(UTIL.getConfiguration()); + } + } + + /** + * Test waiting on root w/ no timeout specified. + * @throws IOException + * @throws InterruptedException + * @throws KeeperException + */ + @Test public void testNoTimeoutWaitForRoot() + throws IOException, InterruptedException, KeeperException { + HConnection connection = Mockito.mock(HConnection.class); + final CatalogTracker ct = constructAndStartCatalogTracker(connection); + ServerName hsa = ct.getRootLocation(); + Assert.assertNull(hsa); + + // Now test waiting on root location getting set. + Thread t = new WaitOnMetaThread(ct); + startWaitAliveThenWaitItLives(t, 1000); + // Set a root location. + hsa = setRootLocation(); + // Join the thread... should exit shortly. + t.join(); + // Now root is available. + Assert.assertTrue(ct.getRootLocation().equals(hsa)); + } + + private ServerName setRootLocation() throws KeeperException { + RootLocationEditor.setRootLocation(this.watcher, SN); + return SN; + } + + /** + * Test waiting on meta w/ no timeout specified. + * @throws Exception + */ + @Ignore // Can't make it work reliably on all platforms; mockito gets confused + // Throwing: org.mockito.exceptions.misusing.WrongTypeOfReturnValue: + // Result cannot be returned by locateRegion() + // If you plug locateRegion, it then throws for incCounter, and if you plug + // that ... and so one. + @Test public void testNoTimeoutWaitForMeta() + throws Exception { + // Mock an HConnection and a HRegionInterface implementation. Have the + // HConnection return the HRI. Have the HRI return a few mocked up responses + // to make our test work. + // Mock an HRegionInterface. + final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); + HConnection connection = mockConnection(implementation); + try { + // Now the ct is up... set into the mocks some answers that make it look + // like things have been getting assigned. Make it so we'll return a + // location (no matter what the Get is). Same for getHRegionInfo -- always + // just return the meta region. + final Result result = getMetaTableRowResult(); + + // TODO: Refactor. This method has been moved out of HConnection. + // It works for now but has been deprecated. + Mockito.when(connection.getRegionServerWithRetries((ServerCallable)Mockito.any())). + thenReturn(result); + Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())). + thenReturn(HRegionInfo.FIRST_META_REGIONINFO); + final CatalogTracker ct = constructAndStartCatalogTracker(connection); + ServerName hsa = ct.getMetaLocation(); + Assert.assertNull(hsa); + + // Now test waiting on meta location getting set. + Thread t = new WaitOnMetaThread(ct) { + @Override + void doWaiting() throws InterruptedException { + this.ct.waitForMeta(); + } + }; + startWaitAliveThenWaitItLives(t, 1000); + + // This should trigger wake up of meta wait (Its the removal of the meta + // region unassigned node that triggers catalogtrackers that a meta has + // been assigned). + String node = ct.getMetaNodeTracker().getNode(); + ZKUtil.createAndFailSilent(this.watcher, node); + MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN); + ZKUtil.deleteNode(this.watcher, node); + // Go get the new meta location. waitForMeta gets and verifies meta. + Assert.assertTrue(ct.waitForMeta(10000).equals(SN)); + // Join the thread... should exit shortly. + t.join(); + // Now meta is available. + Assert.assertTrue(ct.waitForMeta(10000).equals(SN)); + } finally { + HConnectionManager.deleteConnection(UTIL.getConfiguration()); + } + } + + /** + * @param implementation An {@link HRegionInterface} instance; you'll likely + * want to pass a mocked HRS; can be null. + * @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when + * {@link HConnection#getConfiguration()} is called, a 'location' when + * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called, + * and that returns the passed {@link HRegionInterface} instance when + * {@link HConnection#getHRegionConnection(String, int)} + * is called (Be sure call + * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)} + * when done with this mocked Connection. + * @throws IOException + */ + private HConnection mockConnection(final HRegionInterface implementation) + throws IOException { + HConnection connection = + HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration()); + Mockito.doNothing().when(connection).close(); + // Make it so we return any old location when asked. + final HRegionLocation anyLocation = + new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(), + SN.getPort()); + Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(), + (byte[]) Mockito.any(), Mockito.anyBoolean())). + thenReturn(anyLocation); + Mockito.when(connection.locateRegion((byte[]) Mockito.any(), + (byte[]) Mockito.any())). + thenReturn(anyLocation); + if (implementation != null) { + // If a call to getHRegionConnection, return this implementation. + Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())). + thenReturn(implementation); + } + return connection; + } + + /** + * @return A mocked up Result that fakes a Get on a row in the + * .META. table. + * @throws IOException + */ + private Result getMetaTableRowResult() throws IOException { + List kvs = new ArrayList(); + kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, + HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO))); + kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, + HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, + Bytes.toBytes(SN.getHostAndPort()))); + kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, + HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, + Bytes.toBytes(SN.getStartcode()))); + return new Result(kvs); + } + + private void startWaitAliveThenWaitItLives(final Thread t, final int ms) { + t.start(); + while(!t.isAlive()) { + // Wait + } + // Wait one second. + Threads.sleep(ms); + Assert.assertTrue("Assert " + t.getName() + " still waiting", t.isAlive()); + } + + class CountingProgressable implements Progressable { + final AtomicInteger counter = new AtomicInteger(0); + @Override + public void progress() { + this.counter.incrementAndGet(); + } + } + + /** + * Wait on META. + * Default is wait on -ROOT-. + */ + class WaitOnMetaThread extends Thread { + final CatalogTracker ct; + + WaitOnMetaThread(final CatalogTracker ct) { + super("WaitOnMeta"); + this.ct = ct; + } + + @Override + public void run() { + try { + doWaiting(); + } catch (InterruptedException e) { + throw new RuntimeException("Failed wait", e); + } + LOG.info("Exiting " + getName()); + } + + void doWaiting() throws InterruptedException { + this.ct.waitForRoot(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java new file mode 100644 index 0000000..1105ec9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java @@ -0,0 +1,305 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.catalog; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test {@link MetaReader}, {@link MetaEditor}, and {@link RootLocationEditor}. + */ +@Category(MediumTests.class) +public class TestMetaReaderEditor { + private static final Log LOG = LogFactory.getLog(TestMetaReaderEditor.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static ZooKeeperWatcher zkw; + private static CatalogTracker ct; + private final static Abortable ABORTABLE = new Abortable() { + private final AtomicBoolean abort = new AtomicBoolean(false); + + @Override + public void abort(String why, Throwable e) { + LOG.info(why, e); + abort.set(true); + } + + @Override + public boolean isAborted() { + return abort.get(); + } + + }; + + @BeforeClass public static void beforeClass() throws Exception { + UTIL.startMiniCluster(3); + + Configuration c = new Configuration(UTIL.getConfiguration()); + // Tests to 4 retries every 5 seconds. Make it try every 1 second so more + // responsive. 1 second is default as is ten retries. + c.setLong("hbase.client.pause", 1000); + c.setInt("hbase.client.retries.number", 10); + zkw = new ZooKeeperWatcher(c, "TestMetaReaderEditor", ABORTABLE); + ct = new CatalogTracker(zkw, c, ABORTABLE); + ct.start(); + } + + @AfterClass public static void afterClass() throws Exception { + ABORTABLE.abort("test ending", null); + ct.stop(); + UTIL.shutdownMiniCluster(); + } + + /** + * Does {@link MetaReader#getRegion(CatalogTracker, byte[])} and a write + * against .META. while its hosted server is restarted to prove our retrying + * works. + * @throws IOException + * @throws InterruptedException + */ + @Test public void testRetrying() + throws IOException, InterruptedException { + final String name = "testRetrying"; + LOG.info("Started " + name); + final byte [] nameBytes = Bytes.toBytes(name); + HTable t = UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY); + int regionCount = UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY); + // Test it works getting a region from just made user table. + final List regions = + testGettingTableRegions(this.ct, nameBytes, regionCount); + MetaTask reader = new MetaTask(this.ct, "reader") { + @Override + void metaTask() throws Throwable { + testGetRegion(this.ct, regions.get(0)); + LOG.info("Read " + regions.get(0).getEncodedName()); + } + }; + MetaTask writer = new MetaTask(this.ct, "writer") { + @Override + void metaTask() throws Throwable { + MetaEditor.addRegionToMeta(this.ct, regions.get(0)); + LOG.info("Wrote " + regions.get(0).getEncodedName()); + } + }; + reader.start(); + writer.start(); + + // We're gonna check how it takes. If it takes too long, we will consider + // it as a fail. We can't put that in the @Test tag as we want to close + // the threads nicely + final long timeOut = 180000; + long startTime = System.currentTimeMillis(); + + try { + // Make sure reader and writer are working. + assertTrue(reader.isProgressing()); + assertTrue(writer.isProgressing()); + + // Kill server hosting meta -- twice . See if our reader/writer ride over the + // meta moves. They'll need to retry. + for (int i = 0; i < 2; i++) { + LOG.info("Restart=" + i); + UTIL.ensureSomeRegionServersAvailable(2); + int index = -1; + do { + index = UTIL.getMiniHBaseCluster().getServerWithMeta(); + }while (index == -1 && + startTime + timeOut < System.currentTimeMillis()); + + if (index != -1){ + UTIL.getMiniHBaseCluster().abortRegionServer(index); + UTIL.getMiniHBaseCluster().waitOnRegionServer(index); + } + } + + assertTrue("reader: "+reader.toString(), reader.isProgressing()); + assertTrue("writer: "+writer.toString(), writer.isProgressing()); + } catch (IOException e) { + throw e; + } finally { + reader.stop = true; + writer.stop = true; + reader.join(); + writer.join(); + t.close(); + } + long exeTime = System.currentTimeMillis() - startTime; + assertTrue("Timeout: test took " + exeTime / 1000 + " sec", exeTime < timeOut); + } + + /** + * Thread that runs a MetaReader/MetaEditor task until asked stop. + */ + abstract static class MetaTask extends Thread { + boolean stop = false; + int count = 0; + Throwable t = null; + final CatalogTracker ct; + + MetaTask(final CatalogTracker ct, final String name) { + super(name); + this.ct = ct; + } + + @Override + public void run() { + try { + while(!this.stop) { + LOG.info("Before " + this.getName()+ ", count=" + this.count); + metaTask(); + this.count += 1; + LOG.info("After " + this.getName() + ", count=" + this.count); + Thread.sleep(100); + } + } catch (Throwable t) { + LOG.info(this.getName() + " failed", t); + this.t = t; + } + } + + boolean isProgressing() throws InterruptedException { + int currentCount = this.count; + while(currentCount == this.count) { + if (!isAlive()) return false; + if (this.t != null) return false; + Thread.sleep(10); + } + return true; + } + + @Override + public String toString() { + return "count=" + this.count + ", t=" + + (this.t == null? "null": this.t.toString()); + } + + abstract void metaTask() throws Throwable; + } + + @Test public void testGetRegionsCatalogTables() + throws IOException, InterruptedException { + List regions = + MetaReader.getTableRegions(ct, HConstants.META_TABLE_NAME); + assertTrue(regions.size() >= 1); + assertTrue(MetaReader.getTableRegionsAndLocations(ct, + Bytes.toString(HConstants.META_TABLE_NAME)).size() >= 1); + assertTrue(MetaReader.getTableRegionsAndLocations(ct, + Bytes.toString(HConstants.ROOT_TABLE_NAME)).size() == 1); + } + + @Test public void testTableExists() throws IOException { + final String name = "testTableExists"; + final byte [] nameBytes = Bytes.toBytes(name); + assertFalse(MetaReader.tableExists(ct, name)); + UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY); + assertTrue(MetaReader.tableExists(ct, name)); + HBaseAdmin admin = UTIL.getHBaseAdmin(); + admin.disableTable(name); + admin.deleteTable(name); + assertFalse(MetaReader.tableExists(ct, name)); + assertTrue(MetaReader.tableExists(ct, + Bytes.toString(HConstants.META_TABLE_NAME))); + assertTrue(MetaReader.tableExists(ct, + Bytes.toString(HConstants.ROOT_TABLE_NAME))); + } + + @Test public void testGetRegion() throws IOException, InterruptedException { + final String name = "testGetRegion"; + LOG.info("Started " + name); + // Test get on non-existent region. + Pair pair = + MetaReader.getRegion(ct, Bytes.toBytes("nonexistent-region")); + assertNull(pair); + // Test it works getting a region from meta/root. + pair = + MetaReader.getRegion(ct, HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); + assertEquals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(), + pair.getFirst().getEncodedName()); + LOG.info("Finished " + name); + } + + // Test for the optimization made in HBASE-3650 + @Test public void testScanMetaForTable() + throws IOException, InterruptedException { + final String name = "testScanMetaForTable"; + LOG.info("Started " + name); + + /** Create 2 tables + - testScanMetaForTable + - testScanMetaForTablf + **/ + + UTIL.createTable(Bytes.toBytes(name), HConstants.CATALOG_FAMILY); + // name that is +1 greater than the first one (e+1=f) + byte[] greaterName = Bytes.toBytes("testScanMetaForTablf"); + UTIL.createTable(greaterName, HConstants.CATALOG_FAMILY); + + // Now make sure we only get the regions from 1 of the tables at a time + + assertEquals(1, MetaReader.getTableRegions(ct, Bytes.toBytes(name)).size()); + assertEquals(1, MetaReader.getTableRegions(ct, greaterName).size()); + } + + private static List testGettingTableRegions(final CatalogTracker ct, + final byte [] nameBytes, final int regionCount) + throws IOException, InterruptedException { + List regions = MetaReader.getTableRegions(ct, nameBytes); + assertEquals(regionCount, regions.size()); + Pair pair = + MetaReader.getRegion(ct, regions.get(0).getRegionName()); + assertEquals(regions.get(0).getEncodedName(), + pair.getFirst().getEncodedName()); + return regions; + } + + private static void testGetRegion(final CatalogTracker ct, + final HRegionInfo region) + throws IOException, InterruptedException { + Pair pair = + MetaReader.getRegion(ct, region.getRegionName()); + assertEquals(region.getEncodedName(), + pair.getFirst().getEncodedName()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java new file mode 100644 index 0000000..222b847 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.catalog; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HConnectionTestingUtility; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test MetaReader/Editor but without spinning up a cluster. + * We mock regionserver back and forth (we do spin up a zk cluster). + */ +@Category(MediumTests.class) +public class TestMetaReaderEditorNoCluster { + private static final Log LOG = LogFactory.getLog(TestMetaReaderEditorNoCluster.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final Abortable ABORTABLE = new Abortable() { + boolean aborted = false; + @Override + public void abort(String why, Throwable e) { + LOG.info(why, e); + this.aborted = true; + throw new RuntimeException(e); + } + @Override + public boolean isAborted() { + return this.aborted; + } + }; + + @Before + public void before() throws Exception { + UTIL.startMiniZKCluster(); + } + + @After + public void after() throws IOException { + UTIL.shutdownMiniZKCluster(); + } + + /** + * Test that MetaReader will ride over server throwing + * "Server not running" IOEs. + * @see https://issues.apache.org/jira/browse/HBASE-3446 + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testRideOverServerNotRunning() throws IOException, InterruptedException { + // Need a zk watcher. + ZooKeeperWatcher zkw = new ZooKeeperWatcher(UTIL.getConfiguration(), + this.getClass().getSimpleName(), ABORTABLE, true); + // This is a servername we use in a few places below. + ServerName sn = new ServerName("example.com", 1234, System.currentTimeMillis()); + + HConnection connection = null; + CatalogTracker ct = null; + try { + // Mock an HRegionInterface. Our mock implementation will fail a few + // times when we go to open a scanner. + final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); + // When openScanner called throw IOE 'Server not running' a few times + // before we return a scanner id. Whats WEIRD is that these + // exceptions do not show in the log because they are caught and only + // printed if we FAIL. We eventually succeed after retry so these don't + // show. We will know if they happened or not because we will ask + // mockito at the end of this test to verify that openscanner was indeed + // called the wanted number of times. + final long scannerid = 123L; + Mockito.when(implementation.openScanner((byte [])Mockito.any(), + (Scan)Mockito.any())). + thenThrow(new IOException("Server not running (1 of 3)")). + thenThrow(new IOException("Server not running (2 of 3)")). + thenThrow(new IOException("Server not running (3 of 3)")). + thenReturn(scannerid); + // Make it so a verifiable answer comes back when next is called. Return + // the verifiable answer and then a null so we stop scanning. Our + // verifiable answer is something that looks like a row in META with + // a server and startcode that is that of the above defined servername. + List kvs = new ArrayList(); + final byte [] rowToVerify = Bytes.toBytes("rowToVerify"); + kvs.add(new KeyValue(rowToVerify, + HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO))); + kvs.add(new KeyValue(rowToVerify, + HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, + Bytes.toBytes(sn.getHostAndPort()))); + kvs.add(new KeyValue(rowToVerify, + HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, + Bytes.toBytes(sn.getStartcode()))); + final Result [] result = new Result [] {new Result(kvs)}; + Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())). + thenReturn(result). + thenReturn(null); + + // Associate a spied-upon HConnection with UTIL.getConfiguration. Need + // to shove this in here first so it gets picked up all over; e.g. by + // HTable. + connection = HConnectionTestingUtility.getSpiedConnection(UTIL.getConfiguration()); + // Fix the location lookup so it 'works' though no network. First + // make an 'any location' object. + final HRegionLocation anyLocation = + new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn.getHostname(), + sn.getPort()); + // Return the any location object when locateRegion is called in HTable + // constructor and when its called by ServerCallable (it uses getRegionLocation). + // The ugly format below comes of 'Important gotcha on spying real objects!' from + // http://mockito.googlecode.com/svn/branches/1.6/javadoc/org/mockito/Mockito.html + Mockito.doReturn(anyLocation). + when(connection).locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any()); + Mockito.doReturn(anyLocation). + when(connection).getRegionLocation((byte[]) Mockito.any(), + (byte[]) Mockito.any(), Mockito.anyBoolean()); + + // Now shove our HRI implementation into the spied-upon connection. + Mockito.doReturn(implementation). + when(connection).getHRegionConnection(Mockito.anyString(), Mockito.anyInt()); + + // Now start up the catalogtracker with our doctored Connection. + ct = new CatalogTracker(zkw, null, connection, ABORTABLE); + ct.start(); + // Scan meta for user tables and verify we got back expected answer. + NavigableMap hris = MetaReader.getServerUserRegions(ct, sn); + assertTrue(hris.size() == 1); + assertTrue(hris.firstEntry().getKey().equals(HRegionInfo.FIRST_META_REGIONINFO)); + assertTrue(Bytes.equals(rowToVerify, hris.firstEntry().getValue().getRow())); + // Finally verify that openscanner was called four times -- three times + // with exception and then on 4th attempt we succeed. + Mockito.verify(implementation, Mockito.times(4)). + openScanner((byte [])Mockito.any(), (Scan)Mockito.any()); + } finally { + if (ct != null) ct.stop(); + HConnectionManager.deleteConnection(UTIL.getConfiguration()); + zkw.close(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java new file mode 100644 index 0000000..e34d8bc --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation; +import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionKey; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.mockito.Mockito; + +/** + * {@link HConnection} testing utility. + */ +public class HConnectionTestingUtility { + /* + * Not part of {@link HBaseTestingUtility} because this class is not + * in same package as {@link HConnection}. Would have to reveal ugly + * {@link HConnectionManager} innards to HBaseTestingUtility to give it access. + */ + /** + * Get a Mocked {@link HConnection} that goes with the passed conf + * configuration instance. Minimally the mock will return + * conf when {@link HConnection#getConfiguration()} is invoked. + * Be sure to shutdown the connection when done by calling + * {@link HConnectionManager#deleteConnection(Configuration, boolean)} else it + * will stick around; this is probably not what you want. + * @param conf configuration + * @return HConnection object for conf + * @throws ZooKeeperConnectionException + */ + public static HConnection getMockedConnection(final Configuration conf) + throws ZooKeeperConnectionException { + HConnectionKey connectionKey = new HConnectionKey(conf); + synchronized (HConnectionManager.HBASE_INSTANCES) { + HConnectionImplementation connection = + HConnectionManager.HBASE_INSTANCES.get(connectionKey); + if (connection == null) { + connection = Mockito.mock(HConnectionImplementation.class); + Mockito.when(connection.getConfiguration()).thenReturn(conf); + HConnectionManager.HBASE_INSTANCES.put(connectionKey, connection); + } + return connection; + } + } + + /** + * Calls {@link #getMockedConnection(Configuration)} and then mocks a few + * more of the popular {@link HConnection} methods so they do 'normal' + * operation (see return doc below for list). Be sure to shutdown the + * connection when done by calling + * {@link HConnectionManager#deleteConnection(Configuration, boolean)} else it + * will stick around; this is probably not what you want. + * @param implementation An {@link HRegionInterface} instance; you'll likely + * want to pass a mocked HRS; can be null. + * + * @param conf Configuration to use + * @param implementation An HRegionInterface; can be null but is usually + * itself a mock. + * @param sn ServerName to include in the region location returned by this + * implementation + * @param hri HRegionInfo to include in the location returned when + * getRegionLocation is called on the mocked connection + * @return Mock up a connection that returns a {@link Configuration} when + * {@link HConnection#getConfiguration()} is called, a 'location' when + * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called, + * and that returns the passed {@link HRegionInterface} instance when + * {@link HConnection#getHRegionConnection(String, int)} + * is called (Be sure call + * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration, boolean)} + * when done with this mocked Connection. + * @throws IOException + */ + public static HConnection getMockedConnectionAndDecorate(final Configuration conf, + final HRegionInterface implementation, final ServerName sn, final HRegionInfo hri) + throws IOException { + HConnection c = HConnectionTestingUtility.getMockedConnection(conf); + Mockito.doNothing().when(c).close(); + // Make it so we return a particular location when asked. + final HRegionLocation loc = new HRegionLocation(hri, sn.getHostname(), sn.getPort()); + Mockito.when(c.getRegionLocation((byte[]) Mockito.any(), + (byte[]) Mockito.any(), Mockito.anyBoolean())). + thenReturn(loc); + Mockito.when(c.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any())). + thenReturn(loc); + if (implementation != null) { + // If a call to getHRegionConnection, return this implementation. + Mockito.when(c.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())). + thenReturn(implementation); + } + return c; + } + + /** + * Get a Mockito spied-upon {@link HConnection} that goes with the passed + * conf configuration instance. + * Be sure to shutdown the connection when done by calling + * {@link HConnectionManager#deleteConnection(Configuration, boolean)} else it + * will stick around; this is probably not what you want. + * @param conf configuration + * @return HConnection object for conf + * @throws ZooKeeperConnectionException + * @see http://mockito.googlecode.com/svn/branches/1.6/javadoc/org/mockito/Mockito.html#spy(T) + */ + public static HConnection getSpiedConnection(final Configuration conf) + throws ZooKeeperConnectionException { + HConnectionKey connectionKey = new HConnectionKey(conf); + synchronized (HConnectionManager.HBASE_INSTANCES) { + HConnectionImplementation connection = + HConnectionManager.HBASE_INSTANCES.get(connectionKey); + if (connection == null) { + connection = Mockito.spy(new HConnectionImplementation(conf, true)); + HConnectionManager.HBASE_INSTANCES.put(connectionKey, connection); + } + return connection; + } + } + + /** + * @return Count of extant connection instances + */ + public static int getConnectionCount() { + synchronized (HConnectionManager.HBASE_INSTANCES) { + return HConnectionManager.HBASE_INSTANCES.size(); + } + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java new file mode 100644 index 0000000..6c596d3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java @@ -0,0 +1,1653 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.executor.EventHandler; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.executor.ExecutorService; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.wal.HLogUtilsForTests; +import org.apache.hadoop.hbase.InvalidFamilyOperationException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.*; +import org.junit.experimental.categories.Category; + + +/** + * Class to test HBaseAdmin. + * Spins up the minicluster once at test start and then takes it down afterward. + * Add any testing of HBaseAdmin functionality here. + */ +@Category(LargeTests.class) +public class TestAdmin { + final Log LOG = LogFactory.getLog(getClass()); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private HBaseAdmin admin; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true); + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); + TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6); + TEST_UTIL.getConfiguration().setBoolean( + "hbase.master.enabletable.roundrobin", true); + TEST_UTIL.startMiniCluster(3); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + this.admin = TEST_UTIL.getHBaseAdmin(); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testSplitFlushCompactUnknownTable() throws InterruptedException { + final String unknowntable = "fubar"; + Exception exception = null; + try { + this.admin.compact(unknowntable); + } catch (IOException e) { + exception = e; + } + assertTrue(exception instanceof TableNotFoundException); + + exception = null; + try { + this.admin.flush(unknowntable); + } catch (IOException e) { + exception = e; + } + assertTrue(exception instanceof TableNotFoundException); + + exception = null; + try { + this.admin.split(unknowntable); + } catch (IOException e) { + exception = e; + } + assertTrue(exception instanceof TableNotFoundException); + } + + @Test + public void testDeleteEditUnknownColumnFamilyAndOrTable() throws IOException { + // Test we get exception if we try to + final String nonexistent = "nonexistent"; + HColumnDescriptor nonexistentHcd = new HColumnDescriptor(nonexistent); + Exception exception = null; + try { + this.admin.addColumn(nonexistent, nonexistentHcd); + } catch (IOException e) { + exception = e; + } + assertTrue(exception instanceof TableNotFoundException); + + exception = null; + try { + this.admin.deleteTable(nonexistent); + } catch (IOException e) { + exception = e; + } + assertTrue(exception instanceof TableNotFoundException); + + exception = null; + try { + this.admin.deleteColumn(nonexistent, nonexistent); + } catch (IOException e) { + exception = e; + } + assertTrue(exception instanceof TableNotFoundException); + + exception = null; + try { + this.admin.disableTable(nonexistent); + } catch (IOException e) { + exception = e; + } + assertTrue(exception instanceof TableNotFoundException); + + exception = null; + try { + this.admin.enableTable(nonexistent); + } catch (IOException e) { + exception = e; + } + assertTrue(exception instanceof TableNotFoundException); + + exception = null; + try { + this.admin.modifyColumn(nonexistent, nonexistentHcd); + } catch (IOException e) { + exception = e; + } + assertTrue(exception instanceof TableNotFoundException); + + exception = null; + try { + HTableDescriptor htd = new HTableDescriptor(nonexistent); + this.admin.modifyTable(htd.getName(), htd); + } catch (IOException e) { + exception = e; + } + assertTrue(exception instanceof TableNotFoundException); + + // Now make it so at least the table exists and then do tests against a + // nonexistent column family -- see if we get right exceptions. + final String tableName = "t"; + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor("cf")); + this.admin.createTable(htd); + try { + exception = null; + try { + this.admin.deleteColumn(htd.getName(), nonexistentHcd.getName()); + } catch (IOException e) { + exception = e; + } + assertTrue(exception instanceof InvalidFamilyOperationException); + + exception = null; + try { + this.admin.modifyColumn(htd.getName(), nonexistentHcd); + } catch (IOException e) { + exception = e; + } + assertTrue(exception instanceof InvalidFamilyOperationException); + } finally { + this.admin.disableTable(tableName); + this.admin.deleteTable(tableName); + } + } + + @Test + public void testDisableAndEnableTable() throws IOException { + final byte [] row = Bytes.toBytes("row"); + final byte [] qualifier = Bytes.toBytes("qualifier"); + final byte [] value = Bytes.toBytes("value"); + final byte [] table = Bytes.toBytes("testDisableAndEnableTable"); + HTable ht = TEST_UTIL.createTable(table, HConstants.CATALOG_FAMILY); + Put put = new Put(row); + put.add(HConstants.CATALOG_FAMILY, qualifier, value); + ht.put(put); + Get get = new Get(row); + get.addColumn(HConstants.CATALOG_FAMILY, qualifier); + ht.get(get); + + this.admin.disableTable(table); + assertTrue("Table must be disabled.", TEST_UTIL.getHBaseCluster() + .getMaster().getAssignmentManager().getZKTable().isDisabledTable( + Bytes.toString(table))); + + // Test that table is disabled + get = new Get(row); + get.addColumn(HConstants.CATALOG_FAMILY, qualifier); + boolean ok = false; + try { + ht.get(get); + } catch (DoNotRetryIOException e) { + ok = true; + } + assertTrue(ok); + this.admin.enableTable(table); + assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster() + .getMaster().getAssignmentManager().getZKTable().isEnabledTable( + Bytes.toString(table))); + + // Test that table is enabled + try { + ht.get(get); + } catch (RetriesExhaustedException e) { + ok = false; + } + assertTrue(ok); + ht.close(); + } + + @Test + public void testIsEnabledOnUnknownTable() throws Exception { + try { + admin.isTableEnabled(Bytes.toBytes("unkownTable")); + fail("Test should fail if isTableEnabled called on unknown table."); + } catch (IOException e) { + } + } + + @Test + public void testDisableAndEnableTables() throws IOException { + final byte [] row = Bytes.toBytes("row"); + final byte [] qualifier = Bytes.toBytes("qualifier"); + final byte [] value = Bytes.toBytes("value"); + final byte [] table1 = Bytes.toBytes("testDisableAndEnableTable1"); + final byte [] table2 = Bytes.toBytes("testDisableAndEnableTable2"); + HTable ht1 = TEST_UTIL.createTable(table1, HConstants.CATALOG_FAMILY); + HTable ht2 = TEST_UTIL.createTable(table2, HConstants.CATALOG_FAMILY); + Put put = new Put(row); + put.add(HConstants.CATALOG_FAMILY, qualifier, value); + ht1.put(put); + ht2.put(put); + Get get = new Get(row); + get.addColumn(HConstants.CATALOG_FAMILY, qualifier); + ht1.get(get); + ht2.get(get); + + this.admin.disableTables("testDisableAndEnableTable.*"); + + // Test that tables are disabled + get = new Get(row); + get.addColumn(HConstants.CATALOG_FAMILY, qualifier); + boolean ok = false; + try { + ht1.get(get); + ht2.get(get); + } catch (DoNotRetryIOException e) { + ok = true; + } + + assertTrue(ok); + this.admin.enableTables("testDisableAndEnableTable.*"); + + // Test that tables are enabled + try { + ht1.get(get); + } catch (IOException e) { + ok = false; + } + try { + ht2.get(get); + } catch (IOException e) { + ok = false; + } + assertTrue(ok); + + ht1.close(); + ht2.close(); + } + + @Test + public void testCreateTable() throws IOException { + HTableDescriptor [] tables = admin.listTables(); + int numTables = tables.length; + TEST_UTIL.createTable(Bytes.toBytes("testCreateTable"), + HConstants.CATALOG_FAMILY).close(); + tables = this.admin.listTables(); + assertEquals(numTables + 1, tables.length); + assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster() + .getMaster().getAssignmentManager().getZKTable().isEnabledTable( + "testCreateTable")); + } + + @Test + public void testGetTableDescriptor() throws IOException { + HColumnDescriptor fam1 = new HColumnDescriptor("fam1"); + HColumnDescriptor fam2 = new HColumnDescriptor("fam2"); + HColumnDescriptor fam3 = new HColumnDescriptor("fam3"); + HTableDescriptor htd = new HTableDescriptor("myTestTable"); + htd.addFamily(fam1); + htd.addFamily(fam2); + htd.addFamily(fam3); + this.admin.createTable(htd); + HTable table = new HTable(TEST_UTIL.getConfiguration(), "myTestTable"); + HTableDescriptor confirmedHtd = table.getTableDescriptor(); + assertEquals(htd.compareTo(confirmedHtd), 0); + table.close(); + } + + @Test + public void testHColumnValidName() { + boolean exceptionThrown = false; + try { + HColumnDescriptor fam1 = new HColumnDescriptor("\\test\\abc"); + } catch(IllegalArgumentException iae) { + exceptionThrown = true; + assertTrue(exceptionThrown); + } + } + /** + * Verify schema modification takes. + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testOnlineChangeTableSchema() throws IOException, InterruptedException { + final byte [] tableName = Bytes.toBytes("changeTableSchemaOnline"); + TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration().setBoolean( + "hbase.online.schema.update.enable", true); + HTableDescriptor [] tables = admin.listTables(); + int numTables = tables.length; + TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); + tables = this.admin.listTables(); + assertEquals(numTables + 1, tables.length); + + // FIRST, do htabledescriptor changes. + HTableDescriptor htd = this.admin.getTableDescriptor(tableName); + // Make a copy and assert copy is good. + HTableDescriptor copy = new HTableDescriptor(htd); + assertTrue(htd.equals(copy)); + // Now amend the copy. Introduce differences. + long newFlushSize = htd.getMemStoreFlushSize() / 2; + if (newFlushSize <=0) { + newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2; + } + copy.setMemStoreFlushSize(newFlushSize); + final String key = "anyoldkey"; + assertTrue(htd.getValue(key) == null); + copy.setValue(key, key); + boolean expectedException = false; + try { + modifyTable(tableName, copy); + } catch (TableNotDisabledException re) { + expectedException = true; + } + assertFalse(expectedException); + HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName); + assertFalse(htd.equals(modifiedHtd)); + assertTrue(copy.equals(modifiedHtd)); + assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize()); + assertEquals(key, modifiedHtd.getValue(key)); + + // Now work on column family changes. + htd = this.admin.getTableDescriptor(tableName); + int countOfFamilies = modifiedHtd.getFamilies().size(); + assertTrue(countOfFamilies > 0); + HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next(); + int maxversions = hcd.getMaxVersions(); + final int newMaxVersions = maxversions + 1; + hcd.setMaxVersions(newMaxVersions); + final byte [] hcdName = hcd.getName(); + expectedException = false; + try { + this.admin.modifyColumn(tableName, hcd); + } catch (TableNotDisabledException re) { + expectedException = true; + } + assertFalse(expectedException); + modifiedHtd = this.admin.getTableDescriptor(tableName); + HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName); + assertEquals(newMaxVersions, modifiedHcd.getMaxVersions()); + + // Try adding a column + assertFalse(this.admin.isTableDisabled(tableName)); + final String xtracolName = "xtracol"; + htd = this.admin.getTableDescriptor(tableName); + HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName); + xtracol.setValue(xtracolName, xtracolName); + expectedException = false; + try { + this.admin.addColumn(tableName, xtracol); + } catch (TableNotDisabledException re) { + expectedException = true; + } + // Add column should work even if the table is enabled + assertFalse(expectedException); + modifiedHtd = this.admin.getTableDescriptor(tableName); + hcd = modifiedHtd.getFamily(xtracol.getName()); + assertTrue(hcd != null); + assertTrue(hcd.getValue(xtracolName).equals(xtracolName)); + + // Delete the just-added column. + this.admin.deleteColumn(tableName, xtracol.getName()); + modifiedHtd = this.admin.getTableDescriptor(tableName); + hcd = modifiedHtd.getFamily(xtracol.getName()); + assertTrue(hcd == null); + + // Delete the table + this.admin.disableTable(tableName); + this.admin.deleteTable(tableName); + this.admin.listTables(); + assertFalse(this.admin.tableExists(tableName)); + } + + @Test + public void testShouldFailOnlineSchemaUpdateIfOnlineSchemaIsNotEnabled() + throws Exception { + final byte[] tableName = Bytes.toBytes("changeTableSchemaOnlineFailure"); + TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration().setBoolean( + "hbase.online.schema.update.enable", false); + HTableDescriptor[] tables = admin.listTables(); + int numTables = tables.length; + TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); + tables = this.admin.listTables(); + assertEquals(numTables + 1, tables.length); + + // FIRST, do htabledescriptor changes. + HTableDescriptor htd = this.admin.getTableDescriptor(tableName); + // Make a copy and assert copy is good. + HTableDescriptor copy = new HTableDescriptor(htd); + assertTrue(htd.equals(copy)); + // Now amend the copy. Introduce differences. + long newFlushSize = htd.getMemStoreFlushSize() / 2; + if (newFlushSize <=0) { + newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2; + } + copy.setMemStoreFlushSize(newFlushSize); + final String key = "anyoldkey"; + assertTrue(htd.getValue(key) == null); + copy.setValue(key, key); + boolean expectedException = false; + try { + modifyTable(tableName, copy); + } catch (TableNotDisabledException re) { + expectedException = true; + } + assertTrue("Online schema update should not happen.", expectedException); + } + + /** + * Modify table is async so wait on completion of the table operation in master. + * @param tableName + * @param htd + * @throws IOException + */ + private void modifyTable(final byte [] tableName, final HTableDescriptor htd) + throws IOException { + MasterServices services = TEST_UTIL.getMiniHBaseCluster().getMaster(); + ExecutorService executor = services.getExecutorService(); + AtomicBoolean done = new AtomicBoolean(false); + executor.registerListener(EventType.C_M_MODIFY_TABLE, new DoneListener(done)); + this.admin.modifyTable(tableName, htd); + while (!done.get()) { + synchronized (done) { + try { + done.wait(100); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + executor.unregisterListener(EventType.C_M_MODIFY_TABLE); + } + + /** + * Listens for when an event is done in Master. + */ + static class DoneListener implements EventHandler.EventHandlerListener { + private final AtomicBoolean done; + + DoneListener(final AtomicBoolean done) { + super(); + this.done = done; + } + + @Override + public void afterProcess(EventHandler event) { + this.done.set(true); + synchronized (this.done) { + // Wake anyone waiting on this value to change. + this.done.notifyAll(); + } + } + + @Override + public void beforeProcess(EventHandler event) { + // continue + } + } + + protected void verifyRoundRobinDistribution(HTable ht, int expectedRegions) throws IOException { + int numRS = ht.getConnection().getCurrentNrHRS(); + Map regions = ht.getRegionsInfo(); + Map> server2Regions = new HashMap>(); + for (Map.Entry entry : regions.entrySet()) { + HServerAddress server = entry.getValue(); + List regs = server2Regions.get(server); + if (regs == null) { + regs = new ArrayList(); + server2Regions.put(server, regs); + } + regs.add(entry.getKey()); + } + float average = (float) expectedRegions/numRS; + int min = (int)Math.floor(average); + int max = (int)Math.ceil(average); + for (List regionList : server2Regions.values()) { + assertTrue(regionList.size() == min || regionList.size() == max); + } + } + + @Test + public void testCreateTableNumberOfRegions() throws IOException, InterruptedException { + byte[] tableName = Bytes.toBytes("testCreateTableNumberOfRegions"); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc); + HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); + Map regions = ht.getRegionLocations(); + assertEquals("Table should have only 1 region", 1, regions.size()); + ht.close(); + + byte[] TABLE_2 = Bytes.add(tableName, Bytes.toBytes("_2")); + desc = new HTableDescriptor(TABLE_2); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, new byte[][] { new byte[] { 42 } }); + HTable ht2 = new HTable(TEST_UTIL.getConfiguration(), TABLE_2); + regions = ht2.getRegionLocations(); + assertEquals("Table should have only 2 region", 2, regions.size()); + ht2.close(); + + byte[] TABLE_3 = Bytes.add(tableName, Bytes.toBytes("_3")); + desc = new HTableDescriptor(TABLE_3); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, "a".getBytes(), "z".getBytes(), 3); + HTable ht3 = new HTable(TEST_UTIL.getConfiguration(), TABLE_3); + regions = ht3.getRegionLocations(); + assertEquals("Table should have only 3 region", 3, regions.size()); + ht3.close(); + + byte[] TABLE_4 = Bytes.add(tableName, Bytes.toBytes("_4")); + desc = new HTableDescriptor(TABLE_4); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + try { + admin.createTable(desc, "a".getBytes(), "z".getBytes(), 2); + fail("Should not be able to create a table with only 2 regions using this API."); + } catch (IllegalArgumentException eae) { + // Expected + } + + byte[] TABLE_5 = Bytes.add(tableName, Bytes.toBytes("_5")); + desc = new HTableDescriptor(TABLE_5); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, new byte[] { 1 }, new byte[] { 127 }, 16); + HTable ht5 = new HTable(TEST_UTIL.getConfiguration(), TABLE_5); + regions = ht5.getRegionLocations(); + assertEquals("Table should have 16 region", 16, regions.size()); + ht5.close(); + } + + @Test + public void testCreateTableWithRegions() throws IOException, InterruptedException { + + byte[] tableName = Bytes.toBytes("testCreateTableWithRegions"); + + byte [][] splitKeys = { + new byte [] { 1, 1, 1 }, + new byte [] { 2, 2, 2 }, + new byte [] { 3, 3, 3 }, + new byte [] { 4, 4, 4 }, + new byte [] { 5, 5, 5 }, + new byte [] { 6, 6, 6 }, + new byte [] { 7, 7, 7 }, + new byte [] { 8, 8, 8 }, + new byte [] { 9, 9, 9 }, + }; + int expectedRegions = splitKeys.length + 1; + + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, splitKeys); + + HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); + Map regions = ht.getRegionsInfo(); + assertEquals("Tried to create " + expectedRegions + " regions " + + "but only found " + regions.size(), + expectedRegions, regions.size()); + System.err.println("Found " + regions.size() + " regions"); + + Iterator hris = regions.keySet().iterator(); + HRegionInfo hri = hris.next(); + assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0); + assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0])); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0])); + assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1])); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1])); + assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2])); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2])); + assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3])); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3])); + assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4])); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4])); + assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5])); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5])); + assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6])); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6])); + assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7])); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7])); + assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8])); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8])); + assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0); + + verifyRoundRobinDistribution(ht, expectedRegions); + ht.close(); + + // Now test using start/end with a number of regions + + // Use 80 bit numbers to make sure we aren't limited + byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; + byte [] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; + + // Splitting into 10 regions, we expect (null,1) ... (9, null) + // with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle + + expectedRegions = 10; + + byte [] TABLE_2 = Bytes.add(tableName, Bytes.toBytes("_2")); + + desc = new HTableDescriptor(TABLE_2); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + admin.createTable(desc, startKey, endKey, expectedRegions); + + HTable ht2 = new HTable(TEST_UTIL.getConfiguration(), TABLE_2); + regions = ht2.getRegionsInfo(); + assertEquals("Tried to create " + expectedRegions + " regions " + + "but only found " + regions.size(), + expectedRegions, regions.size()); + System.err.println("Found " + regions.size() + " regions"); + + hris = regions.keySet().iterator(); + hri = hris.next(); + assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0); + assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {1,1,1,1,1,1,1,1,1,1})); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {1,1,1,1,1,1,1,1,1,1})); + assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {2,2,2,2,2,2,2,2,2,2})); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {2,2,2,2,2,2,2,2,2,2})); + assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {3,3,3,3,3,3,3,3,3,3})); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {3,3,3,3,3,3,3,3,3,3})); + assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {4,4,4,4,4,4,4,4,4,4})); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {4,4,4,4,4,4,4,4,4,4})); + assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {5,5,5,5,5,5,5,5,5,5})); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {5,5,5,5,5,5,5,5,5,5})); + assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {6,6,6,6,6,6,6,6,6,6})); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {6,6,6,6,6,6,6,6,6,6})); + assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {7,7,7,7,7,7,7,7,7,7})); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {7,7,7,7,7,7,7,7,7,7})); + assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {8,8,8,8,8,8,8,8,8,8})); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {8,8,8,8,8,8,8,8,8,8})); + assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {9,9,9,9,9,9,9,9,9,9})); + hri = hris.next(); + assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {9,9,9,9,9,9,9,9,9,9})); + assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0); + + verifyRoundRobinDistribution(ht2, expectedRegions); + ht2.close(); + + // Try once more with something that divides into something infinite + + startKey = new byte [] { 0, 0, 0, 0, 0, 0 }; + endKey = new byte [] { 1, 0, 0, 0, 0, 0 }; + + expectedRegions = 5; + + byte [] TABLE_3 = Bytes.add(tableName, Bytes.toBytes("_3")); + + desc = new HTableDescriptor(TABLE_3); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + admin.createTable(desc, startKey, endKey, expectedRegions); + + + HTable ht3 = new HTable(TEST_UTIL.getConfiguration(), TABLE_3); + regions = ht3.getRegionsInfo(); + assertEquals("Tried to create " + expectedRegions + " regions " + + "but only found " + regions.size(), + expectedRegions, regions.size()); + System.err.println("Found " + regions.size() + " regions"); + + verifyRoundRobinDistribution(ht3, expectedRegions); + ht3.close(); + + + // Try an invalid case where there are duplicate split keys + splitKeys = new byte [][] { + new byte [] { 1, 1, 1 }, + new byte [] { 2, 2, 2 }, + new byte [] { 3, 3, 3 }, + new byte [] { 2, 2, 2 } + }; + + byte [] TABLE_4 = Bytes.add(tableName, Bytes.toBytes("_4")); + desc = new HTableDescriptor(TABLE_4); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + HBaseAdmin ladmin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + try { + ladmin.createTable(desc, splitKeys); + assertTrue("Should not be able to create this table because of " + + "duplicate split keys", false); + } catch(IllegalArgumentException iae) { + // Expected + } + ladmin.close(); + } + + + @Test + public void testCreateTableWithOnlyEmptyStartRow() throws IOException { + byte[] tableName = Bytes.toBytes("testCreateTableWithOnlyEmptyStartRow"); + byte[][] splitKeys = new byte[1][]; + splitKeys[0] = HConstants.EMPTY_BYTE_ARRAY; + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor("col")); + try { + admin.createTable(desc, splitKeys); + fail("Test case should fail as empty split key is passed."); + } catch (IllegalArgumentException e) { + } + } + + @Test + public void testCreateTableWithEmptyRowInTheSplitKeys() throws IOException { + byte[] tableName = Bytes + .toBytes("testCreateTableWithEmptyRowInTheSplitKeys"); + byte[][] splitKeys = new byte[3][]; + splitKeys[0] = "region1".getBytes(); + splitKeys[1] = HConstants.EMPTY_BYTE_ARRAY; + splitKeys[2] = "region2".getBytes(); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor("col")); + try { + admin.createTable(desc, splitKeys); + fail("Test case should fail as empty split key is passed."); + } catch (IllegalArgumentException e) { + } + } + + @Test + public void testTableExist() throws IOException { + final byte [] table = Bytes.toBytes("testTableExist"); + boolean exist = false; + exist = this.admin.tableExists(table); + assertEquals(false, exist); + TEST_UTIL.createTable(table, HConstants.CATALOG_FAMILY); + exist = this.admin.tableExists(table); + assertEquals(true, exist); + } + + /** + * Tests forcing split from client and having scanners successfully ride over split. + * @throws Exception + * @throws IOException + */ + @Test + public void testForceSplit() throws Exception { + byte[][] familyNames = new byte[][] { Bytes.toBytes("cf") }; + int[] rowCounts = new int[] { 6000 }; + int numVersions = HColumnDescriptor.DEFAULT_VERSIONS; + int blockSize = 256; + splitTest(null, familyNames, rowCounts, numVersions, blockSize); + + byte[] splitKey = Bytes.toBytes(3500); + splitTest(splitKey, familyNames, rowCounts, numVersions, blockSize); + } + + /** + * Test round-robin assignment on enableTable. + * + * @throws IOException + */ + @Test + public void testEnableTableRoundRobinAssignment() throws IOException { + byte[] tableName = Bytes.toBytes("testEnableTableAssignment"); + byte[][] splitKeys = { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 }, + new byte[] { 3, 3, 3 }, new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 }, + new byte[] { 6, 6, 6 }, new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 }, + new byte[] { 9, 9, 9 } }; + int expectedRegions = splitKeys.length + 1; + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, splitKeys); + HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); + Map regions = ht.getRegionsInfo(); + assertEquals("Tried to create " + expectedRegions + " regions " + + "but only found " + regions.size(), expectedRegions, regions.size()); + // Disable table. + admin.disableTable(tableName); + // Enable table, use round-robin assignment to assign regions. + admin.enableTable(tableName); + + // Check the assignment. + HTable metaTable = new HTable(TEST_UTIL.getConfiguration(), + HConstants.META_TABLE_NAME); + List regionInfos = admin.getTableRegions(tableName); + Map serverMap = new HashMap(); + for (int i = 0, j = regionInfos.size(); i < j; i++) { + HRegionInfo hri = regionInfos.get(i); + Get get = new Get(hri.getRegionName()); + Result result = metaTable.get(get); + String server = Bytes.toString(result.getValue(HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER)); + Integer regioncount = serverMap.get(server); + if (regioncount == null) { + regioncount = 0; + } + regioncount++; + serverMap.put(server, regioncount); + } + List> entryList = new ArrayList>( + serverMap.entrySet()); + Collections.sort(entryList, new Comparator>() { + public int compare(Map.Entry oa, + Map.Entry ob) { + return (oa.getValue() - ob.getValue()); + } + }); + assertTrue(entryList.size() == 3); + assertTrue((entryList.get(2).getValue() - entryList.get(0).getValue()) < 2); + } + + /** + * Multi-family scenario. Tests forcing split from client and + * having scanners successfully ride over split. + * @throws Exception + * @throws IOException + */ + @Test + public void testForceSplitMultiFamily() throws Exception { + int numVersions = HColumnDescriptor.DEFAULT_VERSIONS; + + // use small HFile block size so that we can have lots of blocks in HFile + // Otherwise, if there is only one block, + // HFileBlockIndex.midKey()'s value == startKey + int blockSize = 256; + byte[][] familyNames = new byte[][] { Bytes.toBytes("cf1"), + Bytes.toBytes("cf2") }; + + // one of the column families isn't splittable + int[] rowCounts = new int[] { 6000, 1 }; + splitTest(null, familyNames, rowCounts, numVersions, blockSize); + + rowCounts = new int[] { 1, 6000 }; + splitTest(null, familyNames, rowCounts, numVersions, blockSize); + + // one column family has much smaller data than the other + // the split key should be based on the largest column family + rowCounts = new int[] { 6000, 300 }; + splitTest(null, familyNames, rowCounts, numVersions, blockSize); + + rowCounts = new int[] { 300, 6000 }; + splitTest(null, familyNames, rowCounts, numVersions, blockSize); + + } + + void splitTest(byte[] splitPoint, byte[][] familyNames, int[] rowCounts, + int numVersions, int blockSize) throws Exception { + byte [] tableName = Bytes.toBytes("testForceSplit"); + assertFalse(admin.tableExists(tableName)); + final HTable table = TEST_UTIL.createTable(tableName, familyNames, + numVersions, blockSize); + int rowCount = 0; + byte[] q = new byte[0]; + + // insert rows into column families. The number of rows that have values + // in a specific column family is decided by rowCounts[familyIndex] + for (int index = 0; index < familyNames.length; index++) { + ArrayList puts = new ArrayList(rowCounts[index]); + for (int i = 0; i < rowCounts[index]; i++) { + byte[] k = Bytes.toBytes(i); + Put put = new Put(k); + put.add(familyNames[index], q, k); + puts.add(put); + } + table.put(puts); + + if ( rowCount < rowCounts[index] ) { + rowCount = rowCounts[index]; + } + } + + // get the initial layout (should just be one region) + Map m = table.getRegionsInfo(); + System.out.println("Initial regions (" + m.size() + "): " + m); + assertTrue(m.size() == 1); + + // Verify row count + Scan scan = new Scan(); + ResultScanner scanner = table.getScanner(scan); + int rows = 0; + for(@SuppressWarnings("unused") Result result : scanner) { + rows++; + } + scanner.close(); + assertEquals(rowCount, rows); + + // Have an outstanding scan going on to make sure we can scan over splits. + scan = new Scan(); + scanner = table.getScanner(scan); + // Scan first row so we are into first region before split happens. + scanner.next(); + + final AtomicInteger count = new AtomicInteger(0); + Thread t = new Thread("CheckForSplit") { + public void run() { + for (int i = 0; i < 20; i++) { + try { + sleep(1000); + } catch (InterruptedException e) { + continue; + } + // check again table = new HTable(conf, tableName); + Map regions = null; + try { + regions = table.getRegionsInfo(); + } catch (IOException e) { + e.printStackTrace(); + } + if (regions == null) continue; + count.set(regions.size()); + if (count.get() >= 2) break; + LOG.debug("Cycle waiting on split"); + } + } + }; + t.start(); + // Split the table + this.admin.split(tableName, splitPoint); + t.join(); + + // Verify row count + rows = 1; // We counted one row above. + for (@SuppressWarnings("unused") Result result : scanner) { + rows++; + if (rows > rowCount) { + scanner.close(); + assertTrue("Scanned more than expected (" + rowCount + ")", false); + } + } + scanner.close(); + assertEquals(rowCount, rows); + + Map regions = null; + try { + regions = table.getRegionsInfo(); + } catch (IOException e) { + e.printStackTrace(); + } + assertEquals(2, regions.size()); + HRegionInfo[] r = regions.keySet().toArray(new HRegionInfo[0]); + if (splitPoint != null) { + // make sure the split point matches our explicit configuration + assertEquals(Bytes.toString(splitPoint), + Bytes.toString(r[0].getEndKey())); + assertEquals(Bytes.toString(splitPoint), + Bytes.toString(r[1].getStartKey())); + LOG.debug("Properly split on " + Bytes.toString(splitPoint)); + } else { + if (familyNames.length > 1) { + int splitKey = Bytes.toInt(r[0].getEndKey()); + // check if splitKey is based on the largest column family + // in terms of it store size + int deltaForLargestFamily = Math.abs(rowCount/2 - splitKey); + for (int index = 0; index < familyNames.length; index++) { + int delta = Math.abs(rowCounts[index]/2 - splitKey); + assertTrue(delta >= deltaForLargestFamily); + } + } + } + TEST_UTIL.deleteTable(tableName); + table.close(); + } + + /** + * HADOOP-2156 + * @throws IOException + */ + @Test (expected=IllegalArgumentException.class) + public void testEmptyHHTableDescriptor() throws IOException { + this.admin.createTable(new HTableDescriptor()); + } + + @Test (expected=IllegalArgumentException.class) + public void testInvalidHColumnDescriptor() throws IOException { + new HColumnDescriptor("/cfamily/name"); + } + + @Test(timeout=36000) + public void testEnableDisableAddColumnDeleteColumn() throws Exception { + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL); + byte [] tableName = Bytes.toBytes("testMasterAdmin"); + TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); + while (!ZKTableReadOnly.isEnabledTable(zkw, "testMasterAdmin")) { + Thread.sleep(10); + } + this.admin.disableTable(tableName); + try { + new HTable(TEST_UTIL.getConfiguration(), tableName); + } catch (DoNotRetryIOException e) { + //expected + } + + this.admin.addColumn(tableName, new HColumnDescriptor("col2")); + this.admin.enableTable(tableName); + try { + this.admin.deleteColumn(tableName, Bytes.toBytes("col2")); + } catch (TableNotDisabledException e) { + LOG.info(e); + } + this.admin.disableTable(tableName); + this.admin.deleteTable(tableName); + } + + @Test + public void testCreateBadTables() throws IOException { + String msg = null; + try { + this.admin.createTable(HTableDescriptor.ROOT_TABLEDESC); + } catch (IllegalArgumentException e) { + msg = e.toString(); + } + assertTrue("Unexcepted exception message " + msg, msg != null && + msg.startsWith(IllegalArgumentException.class.getName()) && + msg.contains(HTableDescriptor.ROOT_TABLEDESC.getNameAsString())); + msg = null; + try { + this.admin.createTable(HTableDescriptor.META_TABLEDESC); + } catch(IllegalArgumentException e) { + msg = e.toString(); + } + assertTrue("Unexcepted exception message " + msg, msg != null && + msg.startsWith(IllegalArgumentException.class.getName()) && + msg.contains(HTableDescriptor.META_TABLEDESC.getNameAsString())); + + // Now try and do concurrent creation with a bunch of threads. + final HTableDescriptor threadDesc = + new HTableDescriptor("threaded_testCreateBadTables"); + threadDesc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + int count = 10; + Thread [] threads = new Thread [count]; + final AtomicInteger successes = new AtomicInteger(0); + final AtomicInteger failures = new AtomicInteger(0); + final HBaseAdmin localAdmin = this.admin; + for (int i = 0; i < count; i++) { + threads[i] = new Thread(Integer.toString(i)) { + @Override + public void run() { + try { + localAdmin.createTable(threadDesc); + successes.incrementAndGet(); + } catch (TableExistsException e) { + failures.incrementAndGet(); + } catch (IOException e) { + throw new RuntimeException("Failed threaded create" + getName(), e); + } + } + }; + } + for (int i = 0; i < count; i++) { + threads[i].start(); + } + for (int i = 0; i < count; i++) { + while(threads[i].isAlive()) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + // continue + } + } + } + // All threads are now dead. Count up how many tables were created and + // how many failed w/ appropriate exception. + assertEquals(1, successes.get()); + assertEquals(count - 1, failures.get()); + } + + /** + * Test for hadoop-1581 'HBASE: Unopenable tablename bug'. + * @throws Exception + */ + @Test + public void testTableNameClash() throws Exception { + String name = "testTableNameClash"; + admin.createTable(new HTableDescriptor(name + "SOMEUPPERCASE")); + admin.createTable(new HTableDescriptor(name)); + // Before fix, below would fail throwing a NoServerForRegionException. + new HTable(TEST_UTIL.getConfiguration(), name).close(); + } + + /*** + * HMaster.createTable used to be kind of synchronous call + * Thus creating of table with lots of regions can cause RPC timeout + * After the fix to make createTable truly async, RPC timeout shouldn't be an + * issue anymore + * @throws Exception + */ + @Test + public void testCreateTableRPCTimeOut() throws Exception { + String name = "testCreateTableRPCTimeOut"; + int oldTimeout = TEST_UTIL.getConfiguration(). + getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT); + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500); + try { + int expectedRegions = 100; + // Use 80 bit numbers to make sure we aren't limited + byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; + byte [] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; + HBaseAdmin hbaseadmin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + hbaseadmin.createTable(new HTableDescriptor(name), startKey, endKey, + expectedRegions); + hbaseadmin.close(); + } finally { + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, oldTimeout); + } + } + + /** + * Test read only tables + * @throws Exception + */ + @Test + public void testReadOnlyTable() throws Exception { + byte [] name = Bytes.toBytes("testReadOnlyTable"); + HTable table = TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY); + byte[] value = Bytes.toBytes("somedata"); + // This used to use an empty row... That must have been a bug + Put put = new Put(value); + put.add(HConstants.CATALOG_FAMILY, HConstants.CATALOG_FAMILY, value); + table.put(put); + table.close(); + } + + /** + * Test that user table names can contain '-' and '.' so long as they do not + * start with same. HBASE-771 + * @throws IOException + */ + @Test + public void testTableNames() throws IOException { + byte[][] illegalNames = new byte[][] { + Bytes.toBytes("-bad"), + Bytes.toBytes(".bad"), + HConstants.ROOT_TABLE_NAME, + HConstants.META_TABLE_NAME + }; + for (int i = 0; i < illegalNames.length; i++) { + try { + new HTableDescriptor(illegalNames[i]); + throw new IOException("Did not detect '" + + Bytes.toString(illegalNames[i]) + "' as an illegal user table name"); + } catch (IllegalArgumentException e) { + // expected + } + } + byte[] legalName = Bytes.toBytes("g-oo.d"); + try { + new HTableDescriptor(legalName); + } catch (IllegalArgumentException e) { + throw new IOException("Legal user table name: '" + + Bytes.toString(legalName) + "' caused IllegalArgumentException: " + + e.getMessage()); + } + } + + /** + * For HADOOP-2579 + * @throws IOException + */ + @Test (expected=TableExistsException.class) + public void testTableExistsExceptionWithATable() throws IOException { + final byte [] name = Bytes.toBytes("testTableExistsExceptionWithATable"); + TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY).close(); + TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY); + } + + /** + * Can't disable a table if the table isn't in enabled state + * @throws IOException + */ + @Test (expected=TableNotEnabledException.class) + public void testTableNotEnabledExceptionWithATable() throws IOException { + final byte [] name = Bytes.toBytes( + "testTableNotEnabledExceptionWithATable"); + TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY).close(); + this.admin.disableTable(name); + this.admin.disableTable(name); + } + + /** + * Can't enable a table if the table isn't in disabled state + * @throws IOException + */ + @Test (expected=TableNotDisabledException.class) + public void testTableNotDisabledExceptionWithATable() throws IOException { + final byte [] name = Bytes.toBytes( + "testTableNotDisabledExceptionWithATable"); + HTable t = TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY); + try { + this.admin.enableTable(name); + }finally { + t.close(); + } + } + + /** + * For HADOOP-2579 + * @throws IOException + */ + @Test (expected=TableNotFoundException.class) + public void testTableNotFoundExceptionWithoutAnyTables() throws IOException { + new HTable(TEST_UTIL.getConfiguration(), + "testTableNotFoundExceptionWithoutAnyTables"); + } + @Test + public void testShouldCloseTheRegionBasedOnTheEncodedRegionName() + throws Exception { + byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion"); + createTableWithDefaultConf(TABLENAME); + + HRegionInfo info = null; + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); + List onlineRegions = rs.getOnlineRegions(); + for (HRegionInfo regionInfo : onlineRegions) { + if (!regionInfo.isMetaTable()) { + info = regionInfo; + admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), rs + .getServerName().getServerName()); + } + } + Thread.sleep(1000); + onlineRegions = rs.getOnlineRegions(); + assertFalse("The region should not be present in online regions list.", + onlineRegions.contains(info)); + } + + @Test + public void testCloseRegionIfInvalidRegionNameIsPassed() throws Exception { + byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion1"); + createTableWithDefaultConf(TABLENAME); + + HRegionInfo info = null; + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); + List onlineRegions = rs.getOnlineRegions(); + for (HRegionInfo regionInfo : onlineRegions) { + if (!regionInfo.isMetaTable()) { + if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion1")) { + info = regionInfo; + admin.closeRegionWithEncodedRegionName("sample", rs.getServerName() + .getServerName()); + } + } + } + onlineRegions = rs.getOnlineRegions(); + assertTrue("The region should be present in online regions list.", + onlineRegions.contains(info)); + } + + @Test + public void testCloseRegionThatFetchesTheHRIFromMeta() throws Exception { + byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion2"); + createTableWithDefaultConf(TABLENAME); + + HRegionInfo info = null; + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); + List onlineRegions = rs.getOnlineRegions(); + for (HRegionInfo regionInfo : onlineRegions) { + if (!regionInfo.isMetaTable()) { + + if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion2")) { + info = regionInfo; + admin.closeRegion(regionInfo.getRegionNameAsString(), rs + .getServerName().getServerName()); + } + } + } + + boolean isInList = rs.getOnlineRegions().contains(info); + long timeout = System.currentTimeMillis() + 2000; + while ((System.currentTimeMillis() < timeout) && (isInList)) { + Thread.sleep(100); + isInList = rs.getOnlineRegions().contains(info); + } + + assertFalse("The region should not be present in online regions list.", + isInList); + } + + @Test + public void testCloseRegionWhenServerNameIsNull() throws Exception { + byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion3"); + createTableWithDefaultConf(TABLENAME); + + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); + + try { + List onlineRegions = rs.getOnlineRegions(); + for (HRegionInfo regionInfo : onlineRegions) { + if (!regionInfo.isMetaTable()) { + if (regionInfo.getRegionNameAsString() + .contains("TestHBACloseRegion3")) { + admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), + null); + } + } + } + fail("The test should throw exception if the servername passed is null."); + } catch (IllegalArgumentException e) { + } + } + + + @Test + public void testCloseRegionWhenServerNameIsEmpty() throws Exception { + byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegionWhenServerNameIsEmpty"); + createTableWithDefaultConf(TABLENAME); + + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); + + try { + List onlineRegions = rs.getOnlineRegions(); + for (HRegionInfo regionInfo : onlineRegions) { + if (!regionInfo.isMetaTable()) { + if (regionInfo.getRegionNameAsString() + .contains("TestHBACloseRegionWhenServerNameIsEmpty")) { + admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), + " "); + } + } + } + fail("The test should throw exception if the servername passed is empty."); + } catch (IllegalArgumentException e) { + } + } + + @Test + public void testCloseRegionWhenEncodedRegionNameIsNotGiven() throws Exception { + byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion4"); + createTableWithDefaultConf(TABLENAME); + + HRegionInfo info = null; + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); + + List onlineRegions = rs.getOnlineRegions(); + for (HRegionInfo regionInfo : onlineRegions) { + if (!regionInfo.isMetaTable()) { + if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion4")) { + info = regionInfo; + admin.closeRegionWithEncodedRegionName(regionInfo + .getRegionNameAsString(), rs.getServerName().getServerName()); + } + } + } + onlineRegions = rs.getOnlineRegions(); + assertTrue("The region should be present in online regions list.", + onlineRegions.contains(info)); + } + + private HBaseAdmin createTable(byte[] TABLENAME) throws IOException { + + Configuration config = TEST_UTIL.getConfiguration(); + HBaseAdmin admin = new HBaseAdmin(config); + + HTableDescriptor htd = new HTableDescriptor(TABLENAME); + HColumnDescriptor hcd = new HColumnDescriptor("value"); + + htd.addFamily(hcd); + admin.createTable(htd, null); + return admin; + } + + private void createTableWithDefaultConf(byte[] TABLENAME) throws IOException { + HTableDescriptor htd = new HTableDescriptor(TABLENAME); + HColumnDescriptor hcd = new HColumnDescriptor("value"); + htd.addFamily(hcd); + + admin.createTable(htd, null); + } + + /** + * For HBASE-2556 + * @throws IOException + */ + @Test + public void testGetTableRegions() throws IOException { + + byte[] tableName = Bytes.toBytes("testGetTableRegions"); + + int expectedRegions = 10; + + // Use 80 bit numbers to make sure we aren't limited + byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; + byte [] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; + + + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, startKey, endKey, expectedRegions); + + List RegionInfos = admin.getTableRegions(tableName); + + assertEquals("Tried to create " + expectedRegions + " regions " + + "but only found " + RegionInfos.size(), + expectedRegions, RegionInfos.size()); + + } + + @Test + public void testHLogRollWriting() throws Exception { + setUpforLogRolling(); + String className = this.getClass().getName(); + StringBuilder v = new StringBuilder(className); + while (v.length() < 1000) { + v.append(className); + } + byte[] value = Bytes.toBytes(v.toString()); + HRegionServer regionServer = startAndWriteData("TestLogRolling", value); + LOG.info("after writing there are " + + HLogUtilsForTests.getNumLogFiles(regionServer.getWAL()) + " log files"); + + // flush all regions + + List regions = new ArrayList(regionServer + .getOnlineRegionsLocalContext()); + for (HRegion r : regions) { + r.flushcache(); + } + admin.rollHLogWriter(regionServer.getServerName().getServerName()); + int count = HLogUtilsForTests.getNumLogFiles(regionServer.getWAL()); + LOG.info("after flushing all regions and rolling logs there are " + + count + " log files"); + assertTrue(("actual count: " + count), count <= 2); + } + + private void setUpforLogRolling() { + // Force a region split after every 768KB + TEST_UTIL.getConfiguration().setLong(HConstants.HREGION_MAX_FILESIZE, + 768L * 1024L); + + // We roll the log after every 32 writes + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.maxlogentries", 32); + + TEST_UTIL.getConfiguration().setInt( + "hbase.regionserver.logroll.errors.tolerated", 2); + TEST_UTIL.getConfiguration().setInt("ipc.ping.interval", 10 * 1000); + TEST_UTIL.getConfiguration().setInt("ipc.socket.timeout", 10 * 1000); + TEST_UTIL.getConfiguration().setInt("hbase.rpc.timeout", 10 * 1000); + + // For less frequently updated regions flush after every 2 flushes + TEST_UTIL.getConfiguration().setInt( + "hbase.hregion.memstore.optionalflushcount", 2); + + // We flush the cache after every 8192 bytes + TEST_UTIL.getConfiguration().setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, + 8192); + + // Increase the amount of time between client retries + TEST_UTIL.getConfiguration().setLong("hbase.client.pause", 10 * 1000); + + // Reduce thread wake frequency so that other threads can get + // a chance to run. + TEST_UTIL.getConfiguration().setInt(HConstants.THREAD_WAKE_FREQUENCY, + 2 * 1000); + + /**** configuration for testLogRollOnDatanodeDeath ****/ + // make sure log.hflush() calls syncFs() to open a pipeline + TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); + // lower the namenode & datanode heartbeat so the namenode + // quickly detects datanode failures + TEST_UTIL.getConfiguration().setInt("heartbeat.recheck.interval", 5000); + TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1); + // the namenode might still try to choose the recently-dead datanode + // for a pipeline, so try to a new pipeline multiple times + TEST_UTIL.getConfiguration().setInt("dfs.client.block.write.retries", 30); + TEST_UTIL.getConfiguration().setInt( + "hbase.regionserver.hlog.tolerable.lowreplication", 2); + TEST_UTIL.getConfiguration().setInt( + "hbase.regionserver.hlog.lowreplication.rolllimit", 3); + } + + private HRegionServer startAndWriteData(String tableName, byte[] value) + throws IOException { + // When the META table can be opened, the region servers are running + new HTable( + TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME).close(); + HRegionServer regionServer = TEST_UTIL.getHBaseCluster() + .getRegionServerThreads().get(0).getRegionServer(); + + // Create the test table and open it + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc); + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + + regionServer = TEST_UTIL.getRSForFirstRegionInTable(Bytes + .toBytes(tableName)); + for (int i = 1; i <= 256; i++) { // 256 writes should cause 8 log rolls + Put put = new Put(Bytes.toBytes("row" + String.format("%1$04d", i))); + put.add(HConstants.CATALOG_FAMILY, null, value); + table.put(put); + if (i % 32 == 0) { + // After every 32 writes sleep to let the log roller run + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + // continue + } + } + } + + table.close(); + return regionServer; + } + + /** + * HBASE-4417 checkHBaseAvailable() doesn't close zk connections + */ + @Test + public void testCheckHBaseAvailableClosesConnection() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + + int initialCount = HConnectionTestingUtility.getConnectionCount(); + HBaseAdmin.checkHBaseAvailable(conf); + int finalCount = HConnectionTestingUtility.getConnectionCount(); + + Assert.assertEquals(initialCount, finalCount) ; + } + + @Test + public void testDisableCatalogTable() throws Exception { + try { + this.admin.disableTable(".META."); + fail("Expected to throw IllegalArgumentException"); + } catch (IllegalArgumentException e) { + } + // Before the fix for HBASE-6146, the below table creation was failing as the META table + // actually getting disabled by the disableTable() call. + HTableDescriptor htd = new HTableDescriptor("testDisableCatalogTable".getBytes()); + HColumnDescriptor hcd = new HColumnDescriptor("cf1".getBytes()); + htd.addFamily(hcd); + TEST_UTIL.getHBaseAdmin().createTable(htd); + } + + @Test + public void testGetRegion() throws Exception { + final String name = "testGetRegion"; + LOG.info("Started " + name); + final byte [] nameBytes = Bytes.toBytes(name); + HTable t = TEST_UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY); + TEST_UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY); + CatalogTracker ct = new CatalogTracker(TEST_UTIL.getConfiguration()); + ct.start(); + try { + HRegionLocation regionLocation = t.getRegionLocation("mmm"); + HRegionInfo region = regionLocation.getRegionInfo(); + byte[] regionName = region.getRegionName(); + Pair pair = admin.getRegion(regionName, ct); + assertTrue(Bytes.equals(regionName, pair.getFirst().getRegionName())); + pair = admin.getRegion(region.getEncodedNameAsBytes(), ct); + assertTrue(Bytes.equals(regionName, pair.getFirst().getRegionName())); + } finally { + ct.stop(); + } + } + + @Test + public void testRootTableSplit() throws Exception { + Scan s = new Scan(); + HTable rootTable = new HTable(TEST_UTIL.getConfiguration(), HConstants.ROOT_TABLE_NAME); + ResultScanner scanner = rootTable.getScanner(s); + Result metaEntry = scanner.next(); + this.admin.split(HConstants.ROOT_TABLE_NAME, metaEntry.getRow()); + Thread.sleep(1000); + List regions = TEST_UTIL.getMiniHBaseCluster().getRegions(HConstants.ROOT_TABLE_NAME); + assertEquals("ROOT region should not be splitted.",1, regions.size()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java b/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java new file mode 100644 index 0000000..e071b76 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java @@ -0,0 +1,201 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestAttributes { + @Test + public void testAttributesSerialization() throws IOException { + Put put = new Put(); + put.setAttribute("attribute1", Bytes.toBytes("value1")); + put.setAttribute("attribute2", Bytes.toBytes("value2")); + put.setAttribute("attribute3", Bytes.toBytes("value3")); + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DataOutput out = new DataOutputStream(byteArrayOutputStream); + put.write(out); + + Put put2 = new Put(); + Assert.assertTrue(put2.getAttributesMap().isEmpty()); + + put2.readFields(new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))); + + Assert.assertNull(put2.getAttribute("absent")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), put2.getAttribute("attribute1"))); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), put2.getAttribute("attribute2"))); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value3"), put2.getAttribute("attribute3"))); + Assert.assertEquals(3, put2.getAttributesMap().size()); + } + + @Test + public void testPutAttributes() { + Put put = new Put(); + Assert.assertTrue(put.getAttributesMap().isEmpty()); + Assert.assertNull(put.getAttribute("absent")); + + put.setAttribute("absent", null); + Assert.assertTrue(put.getAttributesMap().isEmpty()); + Assert.assertNull(put.getAttribute("absent")); + + // adding attribute + put.setAttribute("attribute1", Bytes.toBytes("value1")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), put.getAttribute("attribute1"))); + Assert.assertEquals(1, put.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), put.getAttributesMap().get("attribute1"))); + + // overriding attribute value + put.setAttribute("attribute1", Bytes.toBytes("value12")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), put.getAttribute("attribute1"))); + Assert.assertEquals(1, put.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), put.getAttributesMap().get("attribute1"))); + + // adding another attribute + put.setAttribute("attribute2", Bytes.toBytes("value2")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), put.getAttribute("attribute2"))); + Assert.assertEquals(2, put.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), put.getAttributesMap().get("attribute2"))); + + // removing attribute + put.setAttribute("attribute2", null); + Assert.assertNull(put.getAttribute("attribute2")); + Assert.assertEquals(1, put.getAttributesMap().size()); + Assert.assertNull(put.getAttributesMap().get("attribute2")); + + // removing non-existed attribute + put.setAttribute("attribute2", null); + Assert.assertNull(put.getAttribute("attribute2")); + Assert.assertEquals(1, put.getAttributesMap().size()); + Assert.assertNull(put.getAttributesMap().get("attribute2")); + + // removing another attribute + put.setAttribute("attribute1", null); + Assert.assertNull(put.getAttribute("attribute1")); + Assert.assertTrue(put.getAttributesMap().isEmpty()); + Assert.assertNull(put.getAttributesMap().get("attribute1")); + } + + + @Test + public void testDeleteAttributes() { + Delete del = new Delete(); + Assert.assertTrue(del.getAttributesMap().isEmpty()); + Assert.assertNull(del.getAttribute("absent")); + + del.setAttribute("absent", null); + Assert.assertTrue(del.getAttributesMap().isEmpty()); + Assert.assertNull(del.getAttribute("absent")); + + // adding attribute + del.setAttribute("attribute1", Bytes.toBytes("value1")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), del.getAttribute("attribute1"))); + Assert.assertEquals(1, del.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), del.getAttributesMap().get("attribute1"))); + + // overriding attribute value + del.setAttribute("attribute1", Bytes.toBytes("value12")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), del.getAttribute("attribute1"))); + Assert.assertEquals(1, del.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), del.getAttributesMap().get("attribute1"))); + + // adding another attribute + del.setAttribute("attribute2", Bytes.toBytes("value2")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), del.getAttribute("attribute2"))); + Assert.assertEquals(2, del.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), del.getAttributesMap().get("attribute2"))); + + // removing attribute + del.setAttribute("attribute2", null); + Assert.assertNull(del.getAttribute("attribute2")); + Assert.assertEquals(1, del.getAttributesMap().size()); + Assert.assertNull(del.getAttributesMap().get("attribute2")); + + // removing non-existed attribute + del.setAttribute("attribute2", null); + Assert.assertNull(del.getAttribute("attribute2")); + Assert.assertEquals(1, del.getAttributesMap().size()); + Assert.assertNull(del.getAttributesMap().get("attribute2")); + + // removing another attribute + del.setAttribute("attribute1", null); + Assert.assertNull(del.getAttribute("attribute1")); + Assert.assertTrue(del.getAttributesMap().isEmpty()); + Assert.assertNull(del.getAttributesMap().get("attribute1")); + } + + @Test + public void testGetId() { + Get get = new Get(); + Assert.assertNull("Make sure id is null if unset", get.toMap().get("id")); + get.setId("myId"); + Assert.assertEquals("myId", get.toMap().get("id")); + } + + @Test + public void testAppendId() { + Append append = new Append(); + Assert.assertNull("Make sure id is null if unset", append.toMap().get("id")); + append.setId("myId"); + Assert.assertEquals("myId", append.toMap().get("id")); + } + + @Test + public void testDeleteId() { + Delete delete = new Delete(); + Assert.assertNull("Make sure id is null if unset", delete.toMap().get("id")); + delete.setId("myId"); + Assert.assertEquals("myId", delete.toMap().get("id")); + } + + @Test + public void testPutId() { + Put put = new Put(); + Assert.assertNull("Make sure id is null if unset", put.toMap().get("id")); + put.setId("myId"); + Assert.assertEquals("myId", put.toMap().get("id")); + } + + @Test + public void testScanId() { + Scan scan = new Scan(); + Assert.assertNull("Make sure id is null if unset", scan.toMap().get("id")); + scan.setId("myId"); + Assert.assertEquals("myId", scan.toMap().get("id")); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java new file mode 100644 index 0000000..66a8b3d --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java @@ -0,0 +1,269 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.MD5Hash; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test clone snapshots from the client + */ +@Category(LargeTests.class) +public class TestCloneSnapshotFromClient { + final Log LOG = LogFactory.getLog(getClass()); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private final byte[] FAMILY = Bytes.toBytes("cf"); + + private byte[] emptySnapshot; + private byte[] snapshotName0; + private byte[] snapshotName1; + private byte[] snapshotName2; + private int snapshot0Rows; + private int snapshot1Rows; + private byte[] tableName; + private HBaseAdmin admin; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true); + TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10); + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); + TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6); + TEST_UTIL.getConfiguration().setBoolean( + "hbase.master.enabletable.roundrobin", true); + TEST_UTIL.startMiniCluster(3); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Initialize the tests with a table filled with some data + * and two snapshots (snapshotName0, snapshotName1) of different states. + * The tableName, snapshotNames and the number of rows in the snapshot are initialized. + */ + @Before + public void setup() throws Exception { + this.admin = TEST_UTIL.getHBaseAdmin(); + + long tid = System.currentTimeMillis(); + tableName = Bytes.toBytes("testtb-" + tid); + emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid); + snapshotName0 = Bytes.toBytes("snaptb0-" + tid); + snapshotName1 = Bytes.toBytes("snaptb1-" + tid); + snapshotName2 = Bytes.toBytes("snaptb2-" + tid); + + // create Table and disable it + createTable(tableName, FAMILY); + admin.disableTable(tableName); + + // take an empty snapshot + admin.snapshot(emptySnapshot, tableName); + + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + try { + // enable table and insert data + admin.enableTable(tableName); + loadData(table, 500, FAMILY); + snapshot0Rows = TEST_UTIL.countRows(table); + admin.disableTable(tableName); + + // take a snapshot + admin.snapshot(snapshotName0, tableName); + + // enable table and insert more data + admin.enableTable(tableName); + loadData(table, 500, FAMILY); + snapshot1Rows = TEST_UTIL.countRows(table); + admin.disableTable(tableName); + + // take a snapshot of the updated table + admin.snapshot(snapshotName1, tableName); + + // re-enable table + admin.enableTable(tableName); + } finally { + table.close(); + } + } + + @After + public void tearDown() throws Exception { + if (admin.tableExists(tableName)) { + TEST_UTIL.deleteTable(tableName); + } + admin.deleteSnapshot(snapshotName0); + admin.deleteSnapshot(snapshotName1); + + // Ensure the archiver to be empty + MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); + mfs.getFileSystem().delete( + new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true); + } + + @Test(expected=SnapshotDoesNotExistException.class) + public void testCloneNonExistentSnapshot() throws IOException, InterruptedException { + String snapshotName = "random-snapshot-" + System.currentTimeMillis(); + String tableName = "random-table-" + System.currentTimeMillis(); + admin.cloneSnapshot(snapshotName, tableName); + } + + @Test + public void testCloneSnapshot() throws IOException, InterruptedException { + byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis()); + testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows); + testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows); + testCloneSnapshot(clonedTableName, emptySnapshot, 0); + } + + private void testCloneSnapshot(final byte[] tableName, final byte[] snapshotName, + int snapshotRows) throws IOException, InterruptedException { + // create a new table from snapshot + admin.cloneSnapshot(snapshotName, tableName); + verifyRowCount(tableName, snapshotRows); + + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + + /** + * Verify that tables created from the snapshot are still alive after source table deletion. + */ + @Test + public void testCloneLinksAfterDelete() throws IOException, InterruptedException { + // Clone a table from the first snapshot + byte[] clonedTableName = Bytes.toBytes("clonedtb1-" + System.currentTimeMillis()); + admin.cloneSnapshot(snapshotName0, clonedTableName); + verifyRowCount(clonedTableName, snapshot0Rows); + + // Take a snapshot of this cloned table. + admin.disableTable(clonedTableName); + admin.snapshot(snapshotName2, clonedTableName); + + // Clone the snapshot of the cloned table + byte[] clonedTableName2 = Bytes.toBytes("clonedtb2-" + System.currentTimeMillis()); + admin.cloneSnapshot(snapshotName2, clonedTableName2); + verifyRowCount(clonedTableName2, snapshot0Rows); + admin.disableTable(clonedTableName2); + + // Remove the original table + admin.disableTable(tableName); + admin.deleteTable(tableName); + waitCleanerRun(); + + // Verify the first cloned table + admin.enableTable(clonedTableName); + verifyRowCount(clonedTableName, snapshot0Rows); + + // Verify the second cloned table + admin.enableTable(clonedTableName2); + verifyRowCount(clonedTableName2, snapshot0Rows); + admin.disableTable(clonedTableName2); + + // Delete the first cloned table + admin.disableTable(clonedTableName); + admin.deleteTable(clonedTableName); + waitCleanerRun(); + + // Verify the second cloned table + admin.enableTable(clonedTableName2); + verifyRowCount(clonedTableName2, snapshot0Rows); + + // Clone a new table from cloned + byte[] clonedTableName3 = Bytes.toBytes("clonedtb3-" + System.currentTimeMillis()); + admin.cloneSnapshot(snapshotName2, clonedTableName3); + verifyRowCount(clonedTableName3, snapshot0Rows); + + // Delete the cloned tables + admin.disableTable(clonedTableName2); + admin.deleteTable(clonedTableName2); + admin.disableTable(clonedTableName3); + admin.deleteTable(clonedTableName3); + admin.deleteSnapshot(snapshotName2); + } + + // ========================================================================== + // Helpers + // ========================================================================== + private void createTable(final byte[] tableName, final byte[]... families) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + for (byte[] family: families) { + HColumnDescriptor hcd = new HColumnDescriptor(family); + htd.addFamily(hcd); + } + byte[][] splitKeys = new byte[16][]; + byte[] hex = Bytes.toBytes("0123456789abcdef"); + for (int i = 0; i < 16; ++i) { + splitKeys[i] = new byte[] { hex[i] }; + } + admin.createTable(htd, splitKeys); + } + + public void loadData(final HTable table, int rows, byte[]... families) throws IOException { + byte[] qualifier = Bytes.toBytes("q"); + table.setAutoFlush(false); + while (rows-- > 0) { + byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows)); + byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value)); + Put put = new Put(key); + put.setWriteToWAL(false); + for (byte[] family: families) { + put.add(family, qualifier, value); + } + table.put(put); + } + table.flushCommits(); + } + + private void waitCleanerRun() throws InterruptedException { + TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting(); + } + + private void verifyRowCount(final byte[] tableName, long expectedRows) throws IOException { + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + assertEquals(expectedRows, TEST_UTIL.countRows(table)); + table.close(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestConnectionUtils.java b/src/test/java/org/apache/hadoop/hbase/client/TestConnectionUtils.java new file mode 100644 index 0000000..4120ec3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestConnectionUtils.java @@ -0,0 +1,56 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.Set; +import java.util.TreeSet; + +import static org.junit.Assert.assertTrue; + +@Category(SmallTests.class) +public class TestConnectionUtils { + + @Test + public void testRetryTimeJitter() { + long[] retries = new long[200]; + long baseTime = 1000000; //Larger number than reality to help test randomness. + long maxTimeExpected = (long) (baseTime * 1.01f); + for (int i = 0; i < retries.length; i++) { + retries[i] = ConnectionUtils.getPauseTime(baseTime, 0); + } + + Set retyTimeSet = new TreeSet(); + for (long l : retries) { + /*make sure that there is some jitter but only 1%*/ + assertTrue(l >= baseTime); + assertTrue(l <= maxTimeExpected); + // Add the long to the set + retyTimeSet.add(l); + } + + //Make sure that most are unique. some overlap will happen + assertTrue(retyTimeSet.size() > (retries.length * 0.80)); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java new file mode 100644 index 0000000..aa0a7f2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -0,0 +1,4867 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.metrics.ScanMetrics; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; +import org.apache.hadoop.hbase.coprocessor.MultiRowMutationProtocol; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.KeyOnlyFilter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.filter.RegexStringComparator; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.filter.WhileMatchFilter; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.io.DataInputBuffer; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Run tests that use the HBase clients; {@link HTable} and {@link HTablePool}. + * Sets up the HBase mini cluster once at start and runs through all client tests. + * Each creates a table named for the method and does its stuff against that. + */ +@Category(LargeTests.class) +@SuppressWarnings ("deprecation") +public class TestFromClientSide { + final Log LOG = LogFactory.getLog(getClass()); + protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static byte [] ROW = Bytes.toBytes("testRow"); + private static byte [] FAMILY = Bytes.toBytes("testFamily"); + private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); + private static byte [] VALUE = Bytes.toBytes("testValue"); + protected static int SLAVES = 3; + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + MultiRowMutationEndpoint.class.getName()); + // We need more than one region server in this test + TEST_UTIL.startMiniCluster(SLAVES); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + // Nothing to do. + } + + /** + * @throws java.lang.Exception + */ + @After + public void tearDown() throws Exception { + // Nothing to do. + } + + /** + * Basic client side validation of HBASE-4536 + */ + @Test + public void testKeepDeletedCells() throws Exception { + final byte[] TABLENAME = Bytes.toBytes("testKeepDeletesCells"); + final byte[] FAMILY = Bytes.toBytes("family"); + final byte[] C0 = Bytes.toBytes("c0"); + + final byte[] T1 = Bytes.toBytes("T1"); + final byte[] T2 = Bytes.toBytes("T2"); + final byte[] T3 = Bytes.toBytes("T3"); + HColumnDescriptor hcd = new HColumnDescriptor(FAMILY) + .setKeepDeletedCells(true); + + HTableDescriptor desc = new HTableDescriptor(TABLENAME); + desc.addFamily(hcd); + TEST_UTIL.getHBaseAdmin().createTable(desc); + Configuration c = TEST_UTIL.getConfiguration(); + HTable h = new HTable(c, TABLENAME); + + long ts = System.currentTimeMillis(); + Put p = new Put(T1, ts); + p.add(FAMILY, C0, T1); + h.put(p); + p = new Put(T1, ts+2); + p.add(FAMILY, C0, T2); + h.put(p); + p = new Put(T1, ts+4); + p.add(FAMILY, C0, T3); + h.put(p); + + Delete d = new Delete(T1, ts+3, null); + h.delete(d); + + d = new Delete(T1, ts+3, null); + d.deleteColumns(FAMILY, C0, ts+3); + h.delete(d); + + Get g = new Get(T1); + // does *not* include the delete + g.setTimeRange(0, ts+3); + Result r = h.get(g); + assertArrayEquals(T2, r.getValue(FAMILY, C0)); + + Scan s = new Scan(T1); + s.setTimeRange(0, ts+3); + s.setMaxVersions(); + ResultScanner scanner = h.getScanner(s); + KeyValue[] kvs = scanner.next().raw(); + assertArrayEquals(T2, kvs[0].getValue()); + assertArrayEquals(T1, kvs[1].getValue()); + scanner.close(); + + s = new Scan(T1); + s.setRaw(true); + s.setMaxVersions(); + scanner = h.getScanner(s); + kvs = scanner.next().raw(); + assertTrue(kvs[0].isDeleteFamily()); + assertArrayEquals(T3, kvs[1].getValue()); + assertTrue(kvs[2].isDelete()); + assertArrayEquals(T2, kvs[3].getValue()); + assertArrayEquals(T1, kvs[4].getValue()); + scanner.close(); + h.close(); + } + + /** + * HBASE-2468 use case 1 and 2: region info de/serialization + */ + @Test + public void testRegionCacheDeSerialization() throws Exception { + // 1. test serialization. + LOG.info("Starting testRegionCacheDeSerialization"); + final byte[] TABLENAME = Bytes.toBytes("testCachePrewarm2"); + final byte[] FAMILY = Bytes.toBytes("family"); + Configuration conf = TEST_UTIL.getConfiguration(); + TEST_UTIL.createTable(TABLENAME, FAMILY); + + // Set up test table: + // Create table: + HTable table = new HTable(conf, TABLENAME); + + // Create multiple regions for this table + TEST_UTIL.createMultiRegions(table, FAMILY); + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + while (scanner.next() != null) continue; + + Path tempPath = new Path(TEST_UTIL.getDataTestDir(), "regions.dat"); + + final String tempFileName = tempPath.toString(); + + FileOutputStream fos = new FileOutputStream(tempFileName); + DataOutputStream dos = new DataOutputStream(fos); + + // serialize the region info and output to a local file. + table.serializeRegionInfo(dos); + dos.flush(); + dos.close(); + + // read a local file and deserialize the region info from it. + FileInputStream fis = new FileInputStream(tempFileName); + DataInputStream dis = new DataInputStream(fis); + + Map deserRegions = + table.deserializeRegionInfo(dis); + dis.close(); + + // regions obtained from meta scanner. + Map loadedRegions = + table.getRegionsInfo(); + + // set the deserialized regions to the global cache. + table.getConnection().clearRegionCache(); + + table.getConnection().prewarmRegionCache(table.getTableName(), + deserRegions); + + // verify whether the 2 maps are identical or not. + assertEquals("Number of cached region is incorrect", + HConnectionManager.getCachedRegionCount(conf, TABLENAME), + loadedRegions.size()); + + // verify each region is prefetched or not. + for (Map.Entry e: loadedRegions.entrySet()) { + HRegionInfo hri = e.getKey(); + assertTrue(HConnectionManager.isRegionCached(conf, + hri.getTableName(), hri.getStartKey())); + } + + // delete the temp file + File f = new java.io.File(tempFileName); + f.delete(); + LOG.info("Finishing testRegionCacheDeSerialization"); + } + + /** + * HBASE-2468 use case 3: + */ + @Test + public void testRegionCachePreWarm() throws Exception { + LOG.info("Starting testRegionCachePreWarm"); + final byte [] TABLENAME = Bytes.toBytes("testCachePrewarm"); + Configuration conf = TEST_UTIL.getConfiguration(); + + // Set up test table: + // Create table: + TEST_UTIL.createTable(TABLENAME, FAMILY); + + // disable region cache for the table. + HTable.setRegionCachePrefetch(conf, TABLENAME, false); + assertFalse("The table is disabled for region cache prefetch", + HTable.getRegionCachePrefetch(conf, TABLENAME)); + + HTable table = new HTable(conf, TABLENAME); + + // create many regions for the table. + TEST_UTIL.createMultiRegions(table, FAMILY); + // This count effectively waits until the regions have been + // fully assigned + TEST_UTIL.countRows(table); + table.getConnection().clearRegionCache(); + assertEquals("Clearing cache should have 0 cached ", 0, + HConnectionManager.getCachedRegionCount(conf, TABLENAME)); + + // A Get is suppose to do a region lookup request + Get g = new Get(Bytes.toBytes("aaa")); + table.get(g); + + // only one region should be cached if the cache prefetch is disabled. + assertEquals("Number of cached region is incorrect ", 1, + HConnectionManager.getCachedRegionCount(conf, TABLENAME)); + + // now we enable cached prefetch. + HTable.setRegionCachePrefetch(conf, TABLENAME, true); + assertTrue("The table is enabled for region cache prefetch", + HTable.getRegionCachePrefetch(conf, TABLENAME)); + + HTable.setRegionCachePrefetch(conf, TABLENAME, false); + assertFalse("The table is disabled for region cache prefetch", + HTable.getRegionCachePrefetch(conf, TABLENAME)); + + HTable.setRegionCachePrefetch(conf, TABLENAME, true); + assertTrue("The table is enabled for region cache prefetch", + HTable.getRegionCachePrefetch(conf, TABLENAME)); + + table.getConnection().clearRegionCache(); + + assertEquals("Number of cached region is incorrect ", 0, + HConnectionManager.getCachedRegionCount(conf, TABLENAME)); + + // if there is a cache miss, some additional regions should be prefetched. + Get g2 = new Get(Bytes.toBytes("bbb")); + table.get(g2); + + // Get the configured number of cache read-ahead regions. + int prefetchRegionNumber = conf.getInt("hbase.client.prefetch.limit", 10); + + // the total number of cached regions == region('aaa") + prefeched regions. + LOG.info("Testing how many regions cached"); + assertEquals("Number of cached region is incorrect ", prefetchRegionNumber, + HConnectionManager.getCachedRegionCount(conf, TABLENAME)); + + table.getConnection().clearRegionCache(); + + Get g3 = new Get(Bytes.toBytes("abc")); + table.get(g3); + assertEquals("Number of cached region is incorrect ", prefetchRegionNumber, + HConnectionManager.getCachedRegionCount(conf, TABLENAME)); + + LOG.info("Finishing testRegionCachePreWarm"); + } + + + /** + * Verifies that getConfiguration returns the same Configuration object used + * to create the HTable instance. + */ + @Test + public void testGetConfiguration() throws Exception { + byte[] TABLE = Bytes.toBytes("testGetConfiguration"); + byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") }; + Configuration conf = TEST_UTIL.getConfiguration(); + HTable table = TEST_UTIL.createTable(TABLE, FAMILIES, conf); + assertSame(conf, table.getConfiguration()); + } + + /** + * Test from client side of an involved filter against a multi family that + * involves deletes. + * + * @throws Exception + */ + @Test + public void testWeirdCacheBehaviour() throws Exception { + byte [] TABLE = Bytes.toBytes("testWeirdCacheBehaviour"); + byte [][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), + Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"), + Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") }; + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES); + String value = "this is the value"; + String value2 = "this is some other value"; + String keyPrefix1 = UUID.randomUUID().toString(); + String keyPrefix2 = UUID.randomUUID().toString(); + String keyPrefix3 = UUID.randomUUID().toString(); + putRows(ht, 3, value, keyPrefix1); + putRows(ht, 3, value, keyPrefix2); + putRows(ht, 3, value, keyPrefix3); + ht.flushCommits(); + putRows(ht, 3, value2, keyPrefix1); + putRows(ht, 3, value2, keyPrefix2); + putRows(ht, 3, value2, keyPrefix3); + HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE); + System.out.println("Checking values for key: " + keyPrefix1); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix1, value2, table)); + System.out.println("Checking values for key: " + keyPrefix2); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix2, value2, table)); + System.out.println("Checking values for key: " + keyPrefix3); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix3, value2, table)); + deleteColumns(ht, value2, keyPrefix1); + deleteColumns(ht, value2, keyPrefix2); + deleteColumns(ht, value2, keyPrefix3); + System.out.println("Starting important checks....."); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, + 0, getNumberOfRows(keyPrefix1, value2, table)); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, + 0, getNumberOfRows(keyPrefix2, value2, table)); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, + 0, getNumberOfRows(keyPrefix3, value2, table)); + ht.setScannerCaching(0); + assertEquals("Got back incorrect number of rows from scan", 0, + getNumberOfRows(keyPrefix1, value2, table)); ht.setScannerCaching(100); + assertEquals("Got back incorrect number of rows from scan", 0, + getNumberOfRows(keyPrefix2, value2, table)); + } + + private void deleteColumns(HTable ht, String value, String keyPrefix) + throws IOException { + ResultScanner scanner = buildScanner(keyPrefix, value, ht); + Iterator it = scanner.iterator(); + int count = 0; + while (it.hasNext()) { + Result result = it.next(); + Delete delete = new Delete(result.getRow()); + delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2")); + ht.delete(delete); + count++; + } + assertEquals("Did not perform correct number of deletes", 3, count); + } + + private int getNumberOfRows(String keyPrefix, String value, HTable ht) + throws Exception { + ResultScanner resultScanner = buildScanner(keyPrefix, value, ht); + Iterator scanner = resultScanner.iterator(); + int numberOfResults = 0; + while (scanner.hasNext()) { + Result result = scanner.next(); + System.out.println("Got back key: " + Bytes.toString(result.getRow())); + for (KeyValue kv : result.raw()) { + System.out.println("kv=" + kv.toString() + ", " + + Bytes.toString(kv.getValue())); + } + numberOfResults++; + } + return numberOfResults; + } + + private ResultScanner buildScanner(String keyPrefix, String value, HTable ht) + throws IOException { + // OurFilterList allFilters = new OurFilterList(); + FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */); + allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); + SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes + .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes + .toBytes(value)); + filter.setFilterIfMissing(true); + allFilters.addFilter(filter); + + // allFilters.addFilter(new + // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"), + // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value))); + + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("trans-blob")); + scan.addFamily(Bytes.toBytes("trans-type")); + scan.addFamily(Bytes.toBytes("trans-date")); + scan.addFamily(Bytes.toBytes("trans-tags")); + scan.addFamily(Bytes.toBytes("trans-group")); + scan.setFilter(allFilters); + + return ht.getScanner(scan); + } + + private void putRows(HTable ht, int numRows, String value, String key) + throws IOException { + for (int i = 0; i < numRows; i++) { + String row = key + "_" + UUID.randomUUID().toString(); + System.out.println(String.format("Saving row: %s, with value %s", row, + value)); + Put put = new Put(Bytes.toBytes(row)); + put.setWriteToWAL(false); + put.add(Bytes.toBytes("trans-blob"), null, Bytes + .toBytes("value for blob")); + put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement")); + put.add(Bytes.toBytes("trans-date"), null, Bytes + .toBytes("20090921010101999")); + put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes + .toBytes(value)); + put.add(Bytes.toBytes("trans-group"), null, Bytes + .toBytes("adhocTransactionGroupId")); + ht.put(put); + } + } + + /** + * Test filters when multiple regions. It does counts. Needs eye-balling of + * logs to ensure that we're not scanning more regions that we're supposed to. + * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package. + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testFilterAcrossMultipleRegions() + throws IOException, InterruptedException { + byte [] name = Bytes.toBytes("testFilterAcrossMutlipleRegions"); + HTable t = TEST_UTIL.createTable(name, FAMILY); + int rowCount = TEST_UTIL.loadTable(t, FAMILY); + assertRowCount(t, rowCount); + // Split the table. Should split on a reasonable key; 'lqj' + Map regions = splitTable(t); + assertRowCount(t, rowCount); + // Get end key of first region. + byte [] endKey = regions.keySet().iterator().next().getEndKey(); + // Count rows with a filter that stops us before passed 'endKey'. + // Should be count of rows in first region. + int endKeyCount = countRows(t, createScanWithRowFilter(endKey)); + assertTrue(endKeyCount < rowCount); + + // How do I know I did not got to second region? Thats tough. Can't really + // do that in client-side region test. I verified by tracing in debugger. + // I changed the messages that come out when set to DEBUG so should see + // when scanner is done. Says "Finished with scanning..." with region name. + // Check that its finished in right region. + + // New test. Make it so scan goes into next region by one and then two. + // Make sure count comes out right. + byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)}; + int plusOneCount = countRows(t, createScanWithRowFilter(key)); + assertEquals(endKeyCount + 1, plusOneCount); + key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)}; + int plusTwoCount = countRows(t, createScanWithRowFilter(key)); + assertEquals(endKeyCount + 2, plusTwoCount); + + // New test. Make it so I scan one less than endkey. + key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)}; + int minusOneCount = countRows(t, createScanWithRowFilter(key)); + assertEquals(endKeyCount - 1, minusOneCount); + // For above test... study logs. Make sure we do "Finished with scanning.." + // in first region and that we do not fall into the next region. + + key = new byte [] {'a', 'a', 'a'}; + int countBBB = countRows(t, + createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL)); + assertEquals(1, countBBB); + + int countGreater = countRows(t, createScanWithRowFilter(endKey, null, + CompareFilter.CompareOp.GREATER_OR_EQUAL)); + // Because started at start of table. + assertEquals(0, countGreater); + countGreater = countRows(t, createScanWithRowFilter(endKey, endKey, + CompareFilter.CompareOp.GREATER_OR_EQUAL)); + assertEquals(rowCount - endKeyCount, countGreater); + } + + /* + * @param key + * @return Scan with RowFilter that does LESS than passed key. + */ + private Scan createScanWithRowFilter(final byte [] key) { + return createScanWithRowFilter(key, null, CompareFilter.CompareOp.LESS); + } + + /* + * @param key + * @param op + * @param startRow + * @return Scan with RowFilter that does CompareOp op on passed key. + */ + private Scan createScanWithRowFilter(final byte [] key, + final byte [] startRow, CompareFilter.CompareOp op) { + // Make sure key is of some substance... non-null and > than first key. + assertTrue(key != null && key.length > 0 && + Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0); + LOG.info("Key=" + Bytes.toString(key)); + Scan s = startRow == null? new Scan(): new Scan(startRow); + Filter f = new RowFilter(op, new BinaryComparator(key)); + f = new WhileMatchFilter(f); + s.setFilter(f); + return s; + } + + /* + * @param t + * @param s + * @return Count of rows in table. + * @throws IOException + */ + private int countRows(final HTable t, final Scan s) + throws IOException { + // Assert all rows in table. + ResultScanner scanner = t.getScanner(s); + int count = 0; + for (Result result: scanner) { + count++; + assertTrue(result.size() > 0); + // LOG.info("Count=" + count + ", row=" + Bytes.toString(result.getRow())); + } + return count; + } + + private void assertRowCount(final HTable t, final int expected) + throws IOException { + assertEquals(expected, countRows(t, new Scan())); + } + + /* + * Split table into multiple regions. + * @param t Table to split. + * @return Map of regions to servers. + * @throws IOException + */ + private Map splitTable(final HTable t) + throws IOException, InterruptedException { + // Split this table in two. + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + admin.split(t.getTableName()); + Map regions = waitOnSplit(t); + assertTrue(regions.size() > 1); + return regions; + } + + /* + * Wait on table split. May return because we waited long enough on the split + * and it didn't happen. Caller should check. + * @param t + * @return Map of table regions; caller needs to check table actually split. + */ + private Map waitOnSplit(final HTable t) + throws IOException { + Map regions = t.getRegionsInfo(); + int originalCount = regions.size(); + for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) { + Thread.currentThread(); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + regions = t.getRegionsInfo(); + if (regions.size() > originalCount) break; + } + return regions; + } + + @Test + public void testSuperSimple() throws Exception { + byte [] TABLE = Bytes.toBytes("testSuperSimple"); + HTable ht = TEST_UTIL.createTable(TABLE, FAMILY); + Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, VALUE); + ht.put(put); + Scan scan = new Scan(); + scan.addColumn(FAMILY, TABLE); + ResultScanner scanner = ht.getScanner(scan); + Result result = scanner.next(); + assertTrue("Expected null result", result == null); + scanner.close(); + } + + @Test + public void testMaxKeyValueSize() throws Exception { + byte [] TABLE = Bytes.toBytes("testMaxKeyValueSize"); + Configuration conf = TEST_UTIL.getConfiguration(); + String oldMaxSize = conf.get("hbase.client.keyvalue.maxsize"); + HTable ht = TEST_UTIL.createTable(TABLE, FAMILY); + byte[] value = new byte[4 * 1024 * 1024]; + Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, value); + ht.put(put); + try { + conf.setInt("hbase.client.keyvalue.maxsize", 2 * 1024 * 1024); + TABLE = Bytes.toBytes("testMaxKeyValueSize2"); + ht = TEST_UTIL.createTable(TABLE, FAMILY); + put = new Put(ROW); + put.add(FAMILY, QUALIFIER, value); + ht.put(put); + fail("Inserting a too large KeyValue worked, should throw exception"); + } catch(Exception e) {} + conf.set("hbase.client.keyvalue.maxsize", oldMaxSize); + } + + @Test + public void testFilters() throws Exception { + byte [] TABLE = Bytes.toBytes("testFilters"); + HTable ht = TEST_UTIL.createTable(TABLE, FAMILY); + byte [][] ROWS = makeN(ROW, 10); + byte [][] QUALIFIERS = { + Bytes.toBytes("col0--"), Bytes.toBytes("col1--"), + Bytes.toBytes("col2--"), Bytes.toBytes("col3--"), + Bytes.toBytes("col4--"), Bytes.toBytes("col5--"), + Bytes.toBytes("col6--"), Bytes.toBytes("col7--"), + Bytes.toBytes("col8--"), Bytes.toBytes("col9--") + }; + for(int i=0;i<10;i++) { + Put put = new Put(ROWS[i]); + put.setWriteToWAL(false); + put.add(FAMILY, QUALIFIERS[i], VALUE); + ht.put(put); + } + Scan scan = new Scan(); + scan.addFamily(FAMILY); + Filter filter = new QualifierFilter(CompareOp.EQUAL, + new RegexStringComparator("col[1-5]")); + scan.setFilter(filter); + ResultScanner scanner = ht.getScanner(scan); + int expectedIndex = 1; + for(Result result : ht.getScanner(scan)) { + assertEquals(result.size(), 1); + assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[expectedIndex])); + assertTrue(Bytes.equals(result.raw()[0].getQualifier(), + QUALIFIERS[expectedIndex])); + expectedIndex++; + } + assertEquals(expectedIndex, 6); + scanner.close(); + } + + @Test + public void testKeyOnlyFilter() throws Exception { + byte [] TABLE = Bytes.toBytes("testKeyOnlyFilter"); + HTable ht = TEST_UTIL.createTable(TABLE, FAMILY); + byte [][] ROWS = makeN(ROW, 10); + byte [][] QUALIFIERS = { + Bytes.toBytes("col0--"), Bytes.toBytes("col1--"), + Bytes.toBytes("col2--"), Bytes.toBytes("col3--"), + Bytes.toBytes("col4--"), Bytes.toBytes("col5--"), + Bytes.toBytes("col6--"), Bytes.toBytes("col7--"), + Bytes.toBytes("col8--"), Bytes.toBytes("col9--") + }; + for(int i=0;i<10;i++) { + Put put = new Put(ROWS[i]); + put.setWriteToWAL(false); + put.add(FAMILY, QUALIFIERS[i], VALUE); + ht.put(put); + } + Scan scan = new Scan(); + scan.addFamily(FAMILY); + Filter filter = new KeyOnlyFilter(true); + scan.setFilter(filter); + ResultScanner scanner = ht.getScanner(scan); + int count = 0; + for(Result result : ht.getScanner(scan)) { + assertEquals(result.size(), 1); + assertEquals(result.raw()[0].getValueLength(), Bytes.SIZEOF_INT); + assertEquals(Bytes.toInt(result.raw()[0].getValue()), VALUE.length); + count++; + } + assertEquals(count, 10); + scanner.close(); + } + + /** + * Test simple table and non-existent row cases. + */ + @Test + public void testSimpleMissing() throws Exception { + byte [] TABLE = Bytes.toBytes("testSimpleMissing"); + HTable ht = TEST_UTIL.createTable(TABLE, FAMILY); + byte [][] ROWS = makeN(ROW, 4); + + // Try to get a row on an empty table + Get get = new Get(ROWS[0]); + Result result = ht.get(get); + assertEmptyResult(result); + + get = new Get(ROWS[0]); + get.addFamily(FAMILY); + result = ht.get(get); + assertEmptyResult(result); + + get = new Get(ROWS[0]); + get.addColumn(FAMILY, QUALIFIER); + result = ht.get(get); + assertEmptyResult(result); + + Scan scan = new Scan(); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + + scan = new Scan(ROWS[0]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + scan = new Scan(ROWS[0],ROWS[1]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + scan = new Scan(); + scan.addFamily(FAMILY); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + scan = new Scan(); + scan.addColumn(FAMILY, QUALIFIER); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + // Insert a row + + Put put = new Put(ROWS[2]); + put.add(FAMILY, QUALIFIER, VALUE); + ht.put(put); + + // Try to get empty rows around it + + get = new Get(ROWS[1]); + result = ht.get(get); + assertEmptyResult(result); + + get = new Get(ROWS[0]); + get.addFamily(FAMILY); + result = ht.get(get); + assertEmptyResult(result); + + get = new Get(ROWS[3]); + get.addColumn(FAMILY, QUALIFIER); + result = ht.get(get); + assertEmptyResult(result); + + // Try to scan empty rows around it + + scan = new Scan(ROWS[3]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + scan = new Scan(ROWS[0],ROWS[2]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + // Make sure we can actually get the row + + get = new Get(ROWS[2]); + result = ht.get(get); + assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); + + get = new Get(ROWS[2]); + get.addFamily(FAMILY); + result = ht.get(get); + assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); + + get = new Get(ROWS[2]); + get.addColumn(FAMILY, QUALIFIER); + result = ht.get(get); + assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); + + // Make sure we can scan the row + + scan = new Scan(); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); + + scan = new Scan(ROWS[0],ROWS[3]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); + + scan = new Scan(ROWS[2],ROWS[3]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); + } + + /** + * Test basic puts, gets, scans, and deletes for a single row + * in a multiple family table. + */ + @Test + public void testSingleRowMultipleFamily() throws Exception { + byte [] TABLE = Bytes.toBytes("testSingleRowMultipleFamily"); + byte [][] ROWS = makeN(ROW, 3); + byte [][] FAMILIES = makeNAscii(FAMILY, 10); + byte [][] QUALIFIERS = makeN(QUALIFIER, 10); + byte [][] VALUES = makeN(VALUE, 10); + + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES); + + Get get; + Scan scan; + Delete delete; + Put put; + Result result; + + //////////////////////////////////////////////////////////////////////////// + // Insert one column to one family + //////////////////////////////////////////////////////////////////////////// + + put = new Put(ROWS[0]); + put.add(FAMILIES[4], QUALIFIERS[0], VALUES[0]); + ht.put(put); + + // Get the single column + getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0); + + // Scan the single column + scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0); + + // Get empty results around inserted column + getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0); + + // Scan empty results around inserted column + scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0); + + //////////////////////////////////////////////////////////////////////////// + // Flush memstore and run same tests from storefiles + //////////////////////////////////////////////////////////////////////////// + + TEST_UTIL.flush(); + + // Redo get and scan tests from storefile + getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0); + scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0); + getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0); + scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0); + + //////////////////////////////////////////////////////////////////////////// + // Now, Test reading from memstore and storefiles at once + //////////////////////////////////////////////////////////////////////////// + + // Insert multiple columns to two other families + put = new Put(ROWS[0]); + put.add(FAMILIES[2], QUALIFIERS[2], VALUES[2]); + put.add(FAMILIES[2], QUALIFIERS[4], VALUES[4]); + put.add(FAMILIES[4], QUALIFIERS[4], VALUES[4]); + put.add(FAMILIES[6], QUALIFIERS[6], VALUES[6]); + put.add(FAMILIES[6], QUALIFIERS[7], VALUES[7]); + put.add(FAMILIES[7], QUALIFIERS[7], VALUES[7]); + put.add(FAMILIES[9], QUALIFIERS[0], VALUES[0]); + ht.put(put); + + // Get multiple columns across multiple families and get empties around it + singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES); + + // Scan multiple columns across multiple families and scan empties around it + singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES); + + //////////////////////////////////////////////////////////////////////////// + // Flush the table again + //////////////////////////////////////////////////////////////////////////// + + TEST_UTIL.flush(); + + // Redo tests again + singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES); + singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES); + + // Insert more data to memstore + put = new Put(ROWS[0]); + put.add(FAMILIES[6], QUALIFIERS[5], VALUES[5]); + put.add(FAMILIES[6], QUALIFIERS[8], VALUES[8]); + put.add(FAMILIES[6], QUALIFIERS[9], VALUES[9]); + put.add(FAMILIES[4], QUALIFIERS[3], VALUES[3]); + ht.put(put); + + //////////////////////////////////////////////////////////////////////////// + // Delete a storefile column + //////////////////////////////////////////////////////////////////////////// + delete = new Delete(ROWS[0]); + delete.deleteColumns(FAMILIES[6], QUALIFIERS[7]); + ht.delete(delete); + + // Try to get deleted column + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[6], QUALIFIERS[7]); + result = ht.get(get); + assertEmptyResult(result); + + // Try to scan deleted column + scan = new Scan(); + scan.addColumn(FAMILIES[6], QUALIFIERS[7]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + // Make sure we can still get a column before it and after it + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[6], QUALIFIERS[6]); + result = ht.get(get); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]); + + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[6], QUALIFIERS[8]); + result = ht.get(get); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]); + + // Make sure we can still scan a column before it and after it + scan = new Scan(); + scan.addColumn(FAMILIES[6], QUALIFIERS[6]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]); + + scan = new Scan(); + scan.addColumn(FAMILIES[6], QUALIFIERS[8]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]); + + //////////////////////////////////////////////////////////////////////////// + // Delete a memstore column + //////////////////////////////////////////////////////////////////////////// + delete = new Delete(ROWS[0]); + delete.deleteColumns(FAMILIES[6], QUALIFIERS[8]); + ht.delete(delete); + + // Try to get deleted column + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[6], QUALIFIERS[8]); + result = ht.get(get); + assertEmptyResult(result); + + // Try to scan deleted column + scan = new Scan(); + scan.addColumn(FAMILIES[6], QUALIFIERS[8]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + // Make sure we can still get a column before it and after it + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[6], QUALIFIERS[6]); + result = ht.get(get); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]); + + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[6], QUALIFIERS[9]); + result = ht.get(get); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]); + + // Make sure we can still scan a column before it and after it + scan = new Scan(); + scan.addColumn(FAMILIES[6], QUALIFIERS[6]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]); + + scan = new Scan(); + scan.addColumn(FAMILIES[6], QUALIFIERS[9]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]); + + //////////////////////////////////////////////////////////////////////////// + // Delete joint storefile/memstore family + //////////////////////////////////////////////////////////////////////////// + + delete = new Delete(ROWS[0]); + delete.deleteFamily(FAMILIES[4]); + ht.delete(delete); + + // Try to get storefile column in deleted family + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[4], QUALIFIERS[4]); + result = ht.get(get); + assertEmptyResult(result); + + // Try to get memstore column in deleted family + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[4], QUALIFIERS[3]); + result = ht.get(get); + assertEmptyResult(result); + + // Try to get deleted family + get = new Get(ROWS[0]); + get.addFamily(FAMILIES[4]); + result = ht.get(get); + assertEmptyResult(result); + + // Try to scan storefile column in deleted family + scan = new Scan(); + scan.addColumn(FAMILIES[4], QUALIFIERS[4]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + // Try to scan memstore column in deleted family + scan = new Scan(); + scan.addColumn(FAMILIES[4], QUALIFIERS[3]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + // Try to scan deleted family + scan = new Scan(); + scan.addFamily(FAMILIES[4]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + // Make sure we can still get another family + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[2], QUALIFIERS[2]); + result = ht.get(get); + assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]); + + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[6], QUALIFIERS[9]); + result = ht.get(get); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]); + + // Make sure we can still scan another family + scan = new Scan(); + scan.addColumn(FAMILIES[6], QUALIFIERS[6]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]); + + scan = new Scan(); + scan.addColumn(FAMILIES[6], QUALIFIERS[9]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]); + + //////////////////////////////////////////////////////////////////////////// + // Flush everything and rerun delete tests + //////////////////////////////////////////////////////////////////////////// + + TEST_UTIL.flush(); + + // Try to get storefile column in deleted family + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[4], QUALIFIERS[4]); + result = ht.get(get); + assertEmptyResult(result); + + // Try to get memstore column in deleted family + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[4], QUALIFIERS[3]); + result = ht.get(get); + assertEmptyResult(result); + + // Try to get deleted family + get = new Get(ROWS[0]); + get.addFamily(FAMILIES[4]); + result = ht.get(get); + assertEmptyResult(result); + + // Try to scan storefile column in deleted family + scan = new Scan(); + scan.addColumn(FAMILIES[4], QUALIFIERS[4]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + // Try to scan memstore column in deleted family + scan = new Scan(); + scan.addColumn(FAMILIES[4], QUALIFIERS[3]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + // Try to scan deleted family + scan = new Scan(); + scan.addFamily(FAMILIES[4]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + // Make sure we can still get another family + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[2], QUALIFIERS[2]); + result = ht.get(get); + assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]); + + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[6], QUALIFIERS[9]); + result = ht.get(get); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]); + + // Make sure we can still scan another family + scan = new Scan(); + scan.addColumn(FAMILIES[6], QUALIFIERS[6]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]); + + scan = new Scan(); + scan.addColumn(FAMILIES[6], QUALIFIERS[9]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]); + + } + + @Test + public void testNull() throws Exception { + byte [] TABLE = Bytes.toBytes("testNull"); + + // Null table name (should NOT work) + try { + TEST_UTIL.createTable(null, FAMILY); + fail("Creating a table with null name passed, should have failed"); + } catch(Exception e) {} + + // Null family (should NOT work) + try { + TEST_UTIL.createTable(TABLE, (byte[])null); + fail("Creating a table with a null family passed, should fail"); + } catch(Exception e) {} + + HTable ht = TEST_UTIL.createTable(TABLE, FAMILY); + + // Null row (should NOT work) + try { + Put put = new Put((byte[])null); + put.add(FAMILY, QUALIFIER, VALUE); + ht.put(put); + fail("Inserting a null row worked, should throw exception"); + } catch(Exception e) {} + + // Null qualifier (should work) + { + Put put = new Put(ROW); + put.add(FAMILY, null, VALUE); + ht.put(put); + + getTestNull(ht, ROW, FAMILY, VALUE); + + scanTestNull(ht, ROW, FAMILY, VALUE); + + Delete delete = new Delete(ROW); + delete.deleteColumns(FAMILY, null); + ht.delete(delete); + + Get get = new Get(ROW); + Result result = ht.get(get); + assertEmptyResult(result); + } + + // Use a new table + byte [] TABLE2 = Bytes.toBytes("testNull2"); + ht = TEST_UTIL.createTable(TABLE2, FAMILY); + + // Empty qualifier, byte[0] instead of null (should work) + try { + Put put = new Put(ROW); + put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE); + ht.put(put); + + getTestNull(ht, ROW, FAMILY, VALUE); + + scanTestNull(ht, ROW, FAMILY, VALUE); + + // Flush and try again + + TEST_UTIL.flush(); + + getTestNull(ht, ROW, FAMILY, VALUE); + + scanTestNull(ht, ROW, FAMILY, VALUE); + + Delete delete = new Delete(ROW); + delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY); + ht.delete(delete); + + Get get = new Get(ROW); + Result result = ht.get(get); + assertEmptyResult(result); + + } catch(Exception e) { + throw new IOException("Using a row with null qualifier threw exception, should "); + } + + // Null value + try { + Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, null); + ht.put(put); + + Get get = new Get(ROW); + get.addColumn(FAMILY, QUALIFIER); + Result result = ht.get(get); + assertSingleResult(result, ROW, FAMILY, QUALIFIER, null); + + Scan scan = new Scan(); + scan.addColumn(FAMILY, QUALIFIER); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROW, FAMILY, QUALIFIER, null); + + Delete delete = new Delete(ROW); + delete.deleteColumns(FAMILY, QUALIFIER); + ht.delete(delete); + + get = new Get(ROW); + result = ht.get(get); + assertEmptyResult(result); + + } catch(Exception e) { + throw new IOException("Null values should be allowed, but threw exception"); + } + } + + @Test + public void testVersions() throws Exception { + byte [] TABLE = Bytes.toBytes("testVersions"); + + long [] STAMPS = makeStamps(20); + byte [][] VALUES = makeNAscii(VALUE, 20); + + HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10); + + // Insert 4 versions of same column + Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); + put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); + put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); + put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); + ht.put(put); + + // Verify we can get each one properly + getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); + getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); + getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); + getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); + scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); + scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); + scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); + scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); + + // Verify we don't accidentally get others + getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); + getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); + getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); + scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); + scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); + scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); + + // Ensure maxVersions in query is respected + Get get = new Get(ROW); + get.addColumn(FAMILY, QUALIFIER); + get.setMaxVersions(2); + Result result = ht.get(get); + assertNResult(result, ROW, FAMILY, QUALIFIER, + new long [] {STAMPS[4], STAMPS[5]}, + new byte[][] {VALUES[4], VALUES[5]}, + 0, 1); + + Scan scan = new Scan(ROW); + scan.addColumn(FAMILY, QUALIFIER); + scan.setMaxVersions(2); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILY, QUALIFIER, + new long [] {STAMPS[4], STAMPS[5]}, + new byte[][] {VALUES[4], VALUES[5]}, + 0, 1); + + // Flush and redo + + TEST_UTIL.flush(); + + // Verify we can get each one properly + getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); + getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); + getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); + getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); + scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); + scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); + scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); + scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); + + // Verify we don't accidentally get others + getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); + getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); + getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); + scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); + scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); + scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); + + // Ensure maxVersions in query is respected + get = new Get(ROW); + get.addColumn(FAMILY, QUALIFIER); + get.setMaxVersions(2); + result = ht.get(get); + assertNResult(result, ROW, FAMILY, QUALIFIER, + new long [] {STAMPS[4], STAMPS[5]}, + new byte[][] {VALUES[4], VALUES[5]}, + 0, 1); + + scan = new Scan(ROW); + scan.addColumn(FAMILY, QUALIFIER); + scan.setMaxVersions(2); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILY, QUALIFIER, + new long [] {STAMPS[4], STAMPS[5]}, + new byte[][] {VALUES[4], VALUES[5]}, + 0, 1); + + + // Add some memstore and retest + + // Insert 4 more versions of same column and a dupe + put = new Put(ROW); + put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); + put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]); + put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]); + put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]); + ht.put(put); + + // Ensure maxVersions in query is respected + get = new Get(ROW); + get.addColumn(FAMILY, QUALIFIER); + get.setMaxVersions(); + result = ht.get(get); + assertNResult(result, ROW, FAMILY, QUALIFIER, + new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]}, + 0, 7); + + scan = new Scan(ROW); + scan.addColumn(FAMILY, QUALIFIER); + scan.setMaxVersions(); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILY, QUALIFIER, + new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]}, + 0, 7); + + get = new Get(ROW); + get.setMaxVersions(); + result = ht.get(get); + assertNResult(result, ROW, FAMILY, QUALIFIER, + new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]}, + 0, 7); + + scan = new Scan(ROW); + scan.setMaxVersions(); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILY, QUALIFIER, + new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]}, + 0, 7); + + // Verify we can get each one properly + getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); + getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); + getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); + getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]); + scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); + scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); + scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); + scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]); + + // Verify we don't accidentally get others + getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); + getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]); + scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); + scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]); + + // Ensure maxVersions of table is respected + + TEST_UTIL.flush(); + + // Insert 4 more versions of same column and a dupe + put = new Put(ROW); + put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]); + put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]); + put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]); + put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]); + ht.put(put); + + get = new Get(ROW); + get.addColumn(FAMILY, QUALIFIER); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertNResult(result, ROW, FAMILY, QUALIFIER, + new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]}, + new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]}, + 0, 9); + + scan = new Scan(ROW); + scan.addColumn(FAMILY, QUALIFIER); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILY, QUALIFIER, + new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]}, + new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]}, + 0, 9); + + // Delete a version in the memstore and a version in a storefile + Delete delete = new Delete(ROW); + delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]); + delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]); + ht.delete(delete); + + // Test that it's gone + get = new Get(ROW); + get.addColumn(FAMILY, QUALIFIER); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertNResult(result, ROW, FAMILY, QUALIFIER, + new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]}, + 0, 9); + + scan = new Scan(ROW); + scan.addColumn(FAMILY, QUALIFIER); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILY, QUALIFIER, + new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]}, + 0, 9); + + } + + @Test + public void testVersionLimits() throws Exception { + byte [] TABLE = Bytes.toBytes("testVersionLimits"); + byte [][] FAMILIES = makeNAscii(FAMILY, 3); + int [] LIMITS = {1,3,5}; + long [] STAMPS = makeStamps(10); + byte [][] VALUES = makeNAscii(VALUE, 10); + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, LIMITS); + + // Insert limit + 1 on each family + Put put = new Put(ROW); + put.add(FAMILIES[0], QUALIFIER, STAMPS[0], VALUES[0]); + put.add(FAMILIES[0], QUALIFIER, STAMPS[1], VALUES[1]); + put.add(FAMILIES[1], QUALIFIER, STAMPS[0], VALUES[0]); + put.add(FAMILIES[1], QUALIFIER, STAMPS[1], VALUES[1]); + put.add(FAMILIES[1], QUALIFIER, STAMPS[2], VALUES[2]); + put.add(FAMILIES[1], QUALIFIER, STAMPS[3], VALUES[3]); + put.add(FAMILIES[2], QUALIFIER, STAMPS[0], VALUES[0]); + put.add(FAMILIES[2], QUALIFIER, STAMPS[1], VALUES[1]); + put.add(FAMILIES[2], QUALIFIER, STAMPS[2], VALUES[2]); + put.add(FAMILIES[2], QUALIFIER, STAMPS[3], VALUES[3]); + put.add(FAMILIES[2], QUALIFIER, STAMPS[4], VALUES[4]); + put.add(FAMILIES[2], QUALIFIER, STAMPS[5], VALUES[5]); + put.add(FAMILIES[2], QUALIFIER, STAMPS[6], VALUES[6]); + ht.put(put); + + // Verify we only get the right number out of each + + // Family0 + + Get get = new Get(ROW); + get.addColumn(FAMILIES[0], QUALIFIER); + get.setMaxVersions(Integer.MAX_VALUE); + Result result = ht.get(get); + assertNResult(result, ROW, FAMILIES[0], QUALIFIER, + new long [] {STAMPS[1]}, + new byte[][] {VALUES[1]}, + 0, 0); + + get = new Get(ROW); + get.addFamily(FAMILIES[0]); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertNResult(result, ROW, FAMILIES[0], QUALIFIER, + new long [] {STAMPS[1]}, + new byte[][] {VALUES[1]}, + 0, 0); + + Scan scan = new Scan(ROW); + scan.addColumn(FAMILIES[0], QUALIFIER); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILIES[0], QUALIFIER, + new long [] {STAMPS[1]}, + new byte[][] {VALUES[1]}, + 0, 0); + + scan = new Scan(ROW); + scan.addFamily(FAMILIES[0]); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILIES[0], QUALIFIER, + new long [] {STAMPS[1]}, + new byte[][] {VALUES[1]}, + 0, 0); + + // Family1 + + get = new Get(ROW); + get.addColumn(FAMILIES[1], QUALIFIER); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertNResult(result, ROW, FAMILIES[1], QUALIFIER, + new long [] {STAMPS[1], STAMPS[2], STAMPS[3]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3]}, + 0, 2); + + get = new Get(ROW); + get.addFamily(FAMILIES[1]); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertNResult(result, ROW, FAMILIES[1], QUALIFIER, + new long [] {STAMPS[1], STAMPS[2], STAMPS[3]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3]}, + 0, 2); + + scan = new Scan(ROW); + scan.addColumn(FAMILIES[1], QUALIFIER); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILIES[1], QUALIFIER, + new long [] {STAMPS[1], STAMPS[2], STAMPS[3]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3]}, + 0, 2); + + scan = new Scan(ROW); + scan.addFamily(FAMILIES[1]); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILIES[1], QUALIFIER, + new long [] {STAMPS[1], STAMPS[2], STAMPS[3]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3]}, + 0, 2); + + // Family2 + + get = new Get(ROW); + get.addColumn(FAMILIES[2], QUALIFIER); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertNResult(result, ROW, FAMILIES[2], QUALIFIER, + new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]}, + new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]}, + 0, 4); + + get = new Get(ROW); + get.addFamily(FAMILIES[2]); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertNResult(result, ROW, FAMILIES[2], QUALIFIER, + new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]}, + new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]}, + 0, 4); + + scan = new Scan(ROW); + scan.addColumn(FAMILIES[2], QUALIFIER); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILIES[2], QUALIFIER, + new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]}, + new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]}, + 0, 4); + + scan = new Scan(ROW); + scan.addFamily(FAMILIES[2]); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILIES[2], QUALIFIER, + new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]}, + new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]}, + 0, 4); + + // Try all families + + get = new Get(ROW); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertTrue("Expected 9 keys but received " + result.size(), + result.size() == 9); + + get = new Get(ROW); + get.addFamily(FAMILIES[0]); + get.addFamily(FAMILIES[1]); + get.addFamily(FAMILIES[2]); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertTrue("Expected 9 keys but received " + result.size(), + result.size() == 9); + + get = new Get(ROW); + get.addColumn(FAMILIES[0], QUALIFIER); + get.addColumn(FAMILIES[1], QUALIFIER); + get.addColumn(FAMILIES[2], QUALIFIER); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertTrue("Expected 9 keys but received " + result.size(), + result.size() == 9); + + scan = new Scan(ROW); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertTrue("Expected 9 keys but received " + result.size(), + result.size() == 9); + + scan = new Scan(ROW); + scan.setMaxVersions(Integer.MAX_VALUE); + scan.addFamily(FAMILIES[0]); + scan.addFamily(FAMILIES[1]); + scan.addFamily(FAMILIES[2]); + result = getSingleScanResult(ht, scan); + assertTrue("Expected 9 keys but received " + result.size(), + result.size() == 9); + + scan = new Scan(ROW); + scan.setMaxVersions(Integer.MAX_VALUE); + scan.addColumn(FAMILIES[0], QUALIFIER); + scan.addColumn(FAMILIES[1], QUALIFIER); + scan.addColumn(FAMILIES[2], QUALIFIER); + result = getSingleScanResult(ht, scan); + assertTrue("Expected 9 keys but received " + result.size(), + result.size() == 9); + + } + + @Test + public void testDeletes() throws Exception { + byte [] TABLE = Bytes.toBytes("testDeletes"); + + byte [][] ROWS = makeNAscii(ROW, 6); + byte [][] FAMILIES = makeNAscii(FAMILY, 3); + byte [][] VALUES = makeN(VALUE, 5); + long [] ts = {1000, 2000, 3000, 4000, 5000}; + + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES); + + Put put = new Put(ROW); + put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); + put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]); + ht.put(put); + + Delete delete = new Delete(ROW); + delete.deleteFamily(FAMILIES[0], ts[0]); + ht.delete(delete); + + Get get = new Get(ROW); + get.addFamily(FAMILIES[0]); + get.setMaxVersions(Integer.MAX_VALUE); + Result result = ht.get(get); + assertNResult(result, ROW, FAMILIES[0], QUALIFIER, + new long [] {ts[1]}, + new byte[][] {VALUES[1]}, + 0, 0); + + Scan scan = new Scan(ROW); + scan.addFamily(FAMILIES[0]); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILIES[0], QUALIFIER, + new long [] {ts[1]}, + new byte[][] {VALUES[1]}, + 0, 0); + + // Test delete latest version + put = new Put(ROW); + put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); + put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]); + put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]); + put.add(FAMILIES[0], null, ts[4], VALUES[4]); + put.add(FAMILIES[0], null, ts[2], VALUES[2]); + put.add(FAMILIES[0], null, ts[3], VALUES[3]); + ht.put(put); + + delete = new Delete(ROW); + delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4] + ht.delete(delete); + + get = new Get(ROW); + get.addColumn(FAMILIES[0], QUALIFIER); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertNResult(result, ROW, FAMILIES[0], QUALIFIER, + new long [] {ts[1], ts[2], ts[3]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3]}, + 0, 2); + + scan = new Scan(ROW); + scan.addColumn(FAMILIES[0], QUALIFIER); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILIES[0], QUALIFIER, + new long [] {ts[1], ts[2], ts[3]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3]}, + 0, 2); + + // Test for HBASE-1847 + delete = new Delete(ROW); + delete.deleteColumn(FAMILIES[0], null); + ht.delete(delete); + + // Cleanup null qualifier + delete = new Delete(ROW); + delete.deleteColumns(FAMILIES[0], null); + ht.delete(delete); + + // Expected client behavior might be that you can re-put deleted values + // But alas, this is not to be. We can't put them back in either case. + + put = new Put(ROW); + put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000 + put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000 + ht.put(put); + + + // It used to be due to the internal implementation of Get, that + // the Get() call would return ts[4] UNLIKE the Scan below. With + // the switch to using Scan for Get this is no longer the case. + get = new Get(ROW); + get.addFamily(FAMILIES[0]); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertNResult(result, ROW, FAMILIES[0], QUALIFIER, + new long [] {ts[1], ts[2], ts[3]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3]}, + 0, 2); + + // The Scanner returns the previous values, the expected-naive-unexpected behavior + + scan = new Scan(ROW); + scan.addFamily(FAMILIES[0]); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROW, FAMILIES[0], QUALIFIER, + new long [] {ts[1], ts[2], ts[3]}, + new byte[][] {VALUES[1], VALUES[2], VALUES[3]}, + 0, 2); + + // Test deleting an entire family from one row but not the other various ways + + put = new Put(ROWS[0]); + put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]); + put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]); + put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]); + put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]); + ht.put(put); + + put = new Put(ROWS[1]); + put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]); + put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]); + put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]); + put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]); + ht.put(put); + + put = new Put(ROWS[2]); + put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]); + put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]); + put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]); + put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]); + ht.put(put); + + // Assert that above went in. + get = new Get(ROWS[2]); + get.addFamily(FAMILIES[1]); + get.addFamily(FAMILIES[2]); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertTrue("Expected 4 key but received " + result.size() + ": " + result, + result.size() == 4); + + delete = new Delete(ROWS[0]); + delete.deleteFamily(FAMILIES[2]); + ht.delete(delete); + + delete = new Delete(ROWS[1]); + delete.deleteColumns(FAMILIES[1], QUALIFIER); + ht.delete(delete); + + delete = new Delete(ROWS[2]); + delete.deleteColumn(FAMILIES[1], QUALIFIER); + delete.deleteColumn(FAMILIES[1], QUALIFIER); + delete.deleteColumn(FAMILIES[2], QUALIFIER); + ht.delete(delete); + + get = new Get(ROWS[0]); + get.addFamily(FAMILIES[1]); + get.addFamily(FAMILIES[2]); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertTrue("Expected 2 keys but received " + result.size(), + result.size() == 2); + assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, + new long [] {ts[0], ts[1]}, + new byte[][] {VALUES[0], VALUES[1]}, + 0, 1); + + scan = new Scan(ROWS[0]); + scan.addFamily(FAMILIES[1]); + scan.addFamily(FAMILIES[2]); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertTrue("Expected 2 keys but received " + result.size(), + result.size() == 2); + assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, + new long [] {ts[0], ts[1]}, + new byte[][] {VALUES[0], VALUES[1]}, + 0, 1); + + get = new Get(ROWS[1]); + get.addFamily(FAMILIES[1]); + get.addFamily(FAMILIES[2]); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertTrue("Expected 2 keys but received " + result.size(), + result.size() == 2); + + scan = new Scan(ROWS[1]); + scan.addFamily(FAMILIES[1]); + scan.addFamily(FAMILIES[2]); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertTrue("Expected 2 keys but received " + result.size(), + result.size() == 2); + + get = new Get(ROWS[2]); + get.addFamily(FAMILIES[1]); + get.addFamily(FAMILIES[2]); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertEquals(1, result.size()); + assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER, + new long [] {ts[2]}, + new byte[][] {VALUES[2]}, + 0, 0); + + scan = new Scan(ROWS[2]); + scan.addFamily(FAMILIES[1]); + scan.addFamily(FAMILIES[2]); + scan.setMaxVersions(Integer.MAX_VALUE); + result = getSingleScanResult(ht, scan); + assertEquals(1, result.size()); + assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER, + new long [] {ts[2]}, + new byte[][] {VALUES[2]}, + 0, 0); + + // Test if we delete the family first in one row (HBASE-1541) + + delete = new Delete(ROWS[3]); + delete.deleteFamily(FAMILIES[1]); + ht.delete(delete); + + put = new Put(ROWS[3]); + put.add(FAMILIES[2], QUALIFIER, VALUES[0]); + ht.put(put); + + put = new Put(ROWS[4]); + put.add(FAMILIES[1], QUALIFIER, VALUES[1]); + put.add(FAMILIES[2], QUALIFIER, VALUES[2]); + ht.put(put); + + get = new Get(ROWS[3]); + get.addFamily(FAMILIES[1]); + get.addFamily(FAMILIES[2]); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertTrue("Expected 1 key but received " + result.size(), + result.size() == 1); + + get = new Get(ROWS[4]); + get.addFamily(FAMILIES[1]); + get.addFamily(FAMILIES[2]); + get.setMaxVersions(Integer.MAX_VALUE); + result = ht.get(get); + assertTrue("Expected 2 keys but received " + result.size(), + result.size() == 2); + + scan = new Scan(ROWS[3]); + scan.addFamily(FAMILIES[1]); + scan.addFamily(FAMILIES[2]); + scan.setMaxVersions(Integer.MAX_VALUE); + ResultScanner scanner = ht.getScanner(scan); + result = scanner.next(); + assertTrue("Expected 1 key but received " + result.size(), + result.size() == 1); + assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[3])); + assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[0])); + result = scanner.next(); + assertTrue("Expected 2 keys but received " + result.size(), + result.size() == 2); + assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[4])); + assertTrue(Bytes.equals(result.raw()[1].getRow(), ROWS[4])); + assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[1])); + assertTrue(Bytes.equals(result.raw()[1].getValue(), VALUES[2])); + scanner.close(); + + // Add test of bulk deleting. + for (int i = 0; i < 10; i++) { + byte [] bytes = Bytes.toBytes(i); + put = new Put(bytes); + put.setWriteToWAL(false); + put.add(FAMILIES[0], QUALIFIER, bytes); + ht.put(put); + } + for (int i = 0; i < 10; i++) { + byte [] bytes = Bytes.toBytes(i); + get = new Get(bytes); + get.addFamily(FAMILIES[0]); + result = ht.get(get); + assertTrue(result.size() == 1); + } + ArrayList deletes = new ArrayList(); + for (int i = 0; i < 10; i++) { + byte [] bytes = Bytes.toBytes(i); + delete = new Delete(bytes); + delete.deleteFamily(FAMILIES[0]); + deletes.add(delete); + } + ht.delete(deletes); + for (int i = 0; i < 10; i++) { + byte [] bytes = Bytes.toBytes(i); + get = new Get(bytes); + get.addFamily(FAMILIES[0]); + result = ht.get(get); + assertTrue(result.size() == 0); + } + } + + /* + * Baseline "scalability" test. + * + * Tests one hundred families, one million columns, one million versions + */ + @Ignore @Test + public void testMillions() throws Exception { + + // 100 families + + // millions of columns + + // millions of versions + + } + + @Ignore @Test + public void testMultipleRegionsAndBatchPuts() throws Exception { + // Two family table + + // Insert lots of rows + + // Insert to the same row with batched puts + + // Insert to multiple rows with batched puts + + // Split the table + + // Get row from first region + + // Get row from second region + + // Scan all rows + + // Insert to multiple regions with batched puts + + // Get row from first region + + // Get row from second region + + // Scan all rows + + + } + + @Ignore @Test + public void testMultipleRowMultipleFamily() throws Exception { + + } + + // + // JIRA Testers + // + + /** + * HBASE-867 + * If millions of columns in a column family, hbase scanner won't come up + * + * Test will create numRows rows, each with numColsPerRow columns + * (1 version each), and attempt to scan them all. + * + * To test at scale, up numColsPerRow to the millions + * (have not gotten that to work running as junit though) + */ + @Test + public void testJiraTest867() throws Exception { + int numRows = 10; + int numColsPerRow = 2000; + + byte [] TABLE = Bytes.toBytes("testJiraTest867"); + + byte [][] ROWS = makeN(ROW, numRows); + byte [][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow); + + HTable ht = TEST_UTIL.createTable(TABLE, FAMILY); + + // Insert rows + + for(int i=0;i some timestamp + */ + @Test + public void testJiraTest1182() throws Exception { + + byte [] TABLE = Bytes.toBytes("testJiraTest1182"); + byte [][] VALUES = makeNAscii(VALUE, 7); + long [] STAMPS = makeStamps(7); + + HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10); + + // Insert lots versions + + Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); + put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); + put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); + put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); + put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); + put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); + ht.put(put); + + getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); + getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5); + getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); + + scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); + scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5); + scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); + + // Try same from storefile + TEST_UTIL.flush(); + + getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); + getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5); + getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); + + scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); + scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5); + scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); + } + + /** + * HBASE-52 + * Add a means of scanning over all versions + */ + @Test + public void testJiraTest52() throws Exception { + byte [] TABLE = Bytes.toBytes("testJiraTest52"); + byte [][] VALUES = makeNAscii(VALUE, 7); + long [] STAMPS = makeStamps(7); + + HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10); + + // Insert lots versions + + Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); + put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); + put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); + put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); + put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); + put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); + ht.put(put); + + getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); + + scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); + + // Try same from storefile + TEST_UTIL.flush(); + + getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); + + scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); + } + + // + // Bulk Testers + // + + private void getVersionRangeAndVerifyGreaterThan(HTable ht, byte [] row, + byte [] family, byte [] qualifier, long [] stamps, byte [][] values, + int start, int end) + throws IOException { + Get get = new Get(row); + get.addColumn(family, qualifier); + get.setMaxVersions(Integer.MAX_VALUE); + get.setTimeRange(stamps[start+1], Long.MAX_VALUE); + Result result = ht.get(get); + assertNResult(result, row, family, qualifier, stamps, values, start+1, end); + } + + private void getVersionRangeAndVerify(HTable ht, byte [] row, byte [] family, + byte [] qualifier, long [] stamps, byte [][] values, int start, int end) + throws IOException { + Get get = new Get(row); + get.addColumn(family, qualifier); + get.setMaxVersions(Integer.MAX_VALUE); + get.setTimeRange(stamps[start], stamps[end]+1); + Result result = ht.get(get); + assertNResult(result, row, family, qualifier, stamps, values, start, end); + } + + private void getAllVersionsAndVerify(HTable ht, byte [] row, byte [] family, + byte [] qualifier, long [] stamps, byte [][] values, int start, int end) + throws IOException { + Get get = new Get(row); + get.addColumn(family, qualifier); + get.setMaxVersions(Integer.MAX_VALUE); + Result result = ht.get(get); + assertNResult(result, row, family, qualifier, stamps, values, start, end); + } + + private void scanVersionRangeAndVerifyGreaterThan(HTable ht, byte [] row, + byte [] family, byte [] qualifier, long [] stamps, byte [][] values, + int start, int end) + throws IOException { + Scan scan = new Scan(row); + scan.addColumn(family, qualifier); + scan.setMaxVersions(Integer.MAX_VALUE); + scan.setTimeRange(stamps[start+1], Long.MAX_VALUE); + Result result = getSingleScanResult(ht, scan); + assertNResult(result, row, family, qualifier, stamps, values, start+1, end); + } + + private void scanVersionRangeAndVerify(HTable ht, byte [] row, byte [] family, + byte [] qualifier, long [] stamps, byte [][] values, int start, int end) + throws IOException { + Scan scan = new Scan(row); + scan.addColumn(family, qualifier); + scan.setMaxVersions(Integer.MAX_VALUE); + scan.setTimeRange(stamps[start], stamps[end]+1); + Result result = getSingleScanResult(ht, scan); + assertNResult(result, row, family, qualifier, stamps, values, start, end); + } + + private void scanAllVersionsAndVerify(HTable ht, byte [] row, byte [] family, + byte [] qualifier, long [] stamps, byte [][] values, int start, int end) + throws IOException { + Scan scan = new Scan(row); + scan.addColumn(family, qualifier); + scan.setMaxVersions(Integer.MAX_VALUE); + Result result = getSingleScanResult(ht, scan); + assertNResult(result, row, family, qualifier, stamps, values, start, end); + } + + private void getVersionAndVerify(HTable ht, byte [] row, byte [] family, + byte [] qualifier, long stamp, byte [] value) + throws Exception { + Get get = new Get(row); + get.addColumn(family, qualifier); + get.setTimeStamp(stamp); + get.setMaxVersions(Integer.MAX_VALUE); + Result result = ht.get(get); + assertSingleResult(result, row, family, qualifier, stamp, value); + } + + private void getVersionAndVerifyMissing(HTable ht, byte [] row, byte [] family, + byte [] qualifier, long stamp) + throws Exception { + Get get = new Get(row); + get.addColumn(family, qualifier); + get.setTimeStamp(stamp); + get.setMaxVersions(Integer.MAX_VALUE); + Result result = ht.get(get); + assertEmptyResult(result); + } + + private void scanVersionAndVerify(HTable ht, byte [] row, byte [] family, + byte [] qualifier, long stamp, byte [] value) + throws Exception { + Scan scan = new Scan(row); + scan.addColumn(family, qualifier); + scan.setTimeStamp(stamp); + scan.setMaxVersions(Integer.MAX_VALUE); + Result result = getSingleScanResult(ht, scan); + assertSingleResult(result, row, family, qualifier, stamp, value); + } + + private void scanVersionAndVerifyMissing(HTable ht, byte [] row, + byte [] family, byte [] qualifier, long stamp) + throws Exception { + Scan scan = new Scan(row); + scan.addColumn(family, qualifier); + scan.setTimeStamp(stamp); + scan.setMaxVersions(Integer.MAX_VALUE); + Result result = getSingleScanResult(ht, scan); + assertNullResult(result); + } + + private void getTestNull(HTable ht, byte [] row, byte [] family, + byte [] value) + throws Exception { + + Get get = new Get(row); + get.addColumn(family, null); + Result result = ht.get(get); + assertSingleResult(result, row, family, null, value); + + get = new Get(row); + get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY); + result = ht.get(get); + assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); + + get = new Get(row); + get.addFamily(family); + result = ht.get(get); + assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); + + get = new Get(row); + result = ht.get(get); + assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); + + } + + private void scanTestNull(HTable ht, byte [] row, byte [] family, + byte [] value) + throws Exception { + + Scan scan = new Scan(); + scan.addColumn(family, null); + Result result = getSingleScanResult(ht, scan); + assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); + + scan = new Scan(); + scan.addColumn(family, HConstants.EMPTY_BYTE_ARRAY); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); + + scan = new Scan(); + scan.addFamily(family); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); + + scan = new Scan(); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); + + } + + private void singleRowGetTest(HTable ht, byte [][] ROWS, byte [][] FAMILIES, + byte [][] QUALIFIERS, byte [][] VALUES) + throws Exception { + + // Single column from memstore + Get get = new Get(ROWS[0]); + get.addColumn(FAMILIES[4], QUALIFIERS[0]); + Result result = ht.get(get); + assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]); + + // Single column from storefile + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[2], QUALIFIERS[2]); + result = ht.get(get); + assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]); + + // Single column from storefile, family match + get = new Get(ROWS[0]); + get.addFamily(FAMILIES[7]); + result = ht.get(get); + assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]); + + // Two columns, one from memstore one from storefile, same family, + // wildcard match + get = new Get(ROWS[0]); + get.addFamily(FAMILIES[4]); + result = ht.get(get); + assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0], + FAMILIES[4], QUALIFIERS[4], VALUES[4]); + + // Two columns, one from memstore one from storefile, same family, + // explicit match + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[4], QUALIFIERS[0]); + get.addColumn(FAMILIES[4], QUALIFIERS[4]); + result = ht.get(get); + assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0], + FAMILIES[4], QUALIFIERS[4], VALUES[4]); + + // Three column, one from memstore two from storefile, different families, + // wildcard match + get = new Get(ROWS[0]); + get.addFamily(FAMILIES[4]); + get.addFamily(FAMILIES[7]); + result = ht.get(get); + assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, + new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} }); + + // Multiple columns from everywhere storefile, many family, wildcard + get = new Get(ROWS[0]); + get.addFamily(FAMILIES[2]); + get.addFamily(FAMILIES[4]); + get.addFamily(FAMILIES[6]); + get.addFamily(FAMILIES[7]); + result = ht.get(get); + assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, + new int [][] { + {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7} + }); + + // Multiple columns from everywhere storefile, many family, wildcard + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[2], QUALIFIERS[2]); + get.addColumn(FAMILIES[2], QUALIFIERS[4]); + get.addColumn(FAMILIES[4], QUALIFIERS[0]); + get.addColumn(FAMILIES[4], QUALIFIERS[4]); + get.addColumn(FAMILIES[6], QUALIFIERS[6]); + get.addColumn(FAMILIES[6], QUALIFIERS[7]); + get.addColumn(FAMILIES[7], QUALIFIERS[7]); + get.addColumn(FAMILIES[7], QUALIFIERS[8]); + result = ht.get(get); + assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, + new int [][] { + {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7} + }); + + // Everything + get = new Get(ROWS[0]); + result = ht.get(get); + assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, + new int [][] { + {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0} + }); + + // Get around inserted columns + + get = new Get(ROWS[1]); + result = ht.get(get); + assertEmptyResult(result); + + get = new Get(ROWS[0]); + get.addColumn(FAMILIES[4], QUALIFIERS[3]); + get.addColumn(FAMILIES[2], QUALIFIERS[3]); + result = ht.get(get); + assertEmptyResult(result); + + } + + private void singleRowScanTest(HTable ht, byte [][] ROWS, byte [][] FAMILIES, + byte [][] QUALIFIERS, byte [][] VALUES) + throws Exception { + + // Single column from memstore + Scan scan = new Scan(); + scan.addColumn(FAMILIES[4], QUALIFIERS[0]); + Result result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]); + + // Single column from storefile + scan = new Scan(); + scan.addColumn(FAMILIES[2], QUALIFIERS[2]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]); + + // Single column from storefile, family match + scan = new Scan(); + scan.addFamily(FAMILIES[7]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]); + + // Two columns, one from memstore one from storefile, same family, + // wildcard match + scan = new Scan(); + scan.addFamily(FAMILIES[4]); + result = getSingleScanResult(ht, scan); + assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0], + FAMILIES[4], QUALIFIERS[4], VALUES[4]); + + // Two columns, one from memstore one from storefile, same family, + // explicit match + scan = new Scan(); + scan.addColumn(FAMILIES[4], QUALIFIERS[0]); + scan.addColumn(FAMILIES[4], QUALIFIERS[4]); + result = getSingleScanResult(ht, scan); + assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0], + FAMILIES[4], QUALIFIERS[4], VALUES[4]); + + // Three column, one from memstore two from storefile, different families, + // wildcard match + scan = new Scan(); + scan.addFamily(FAMILIES[4]); + scan.addFamily(FAMILIES[7]); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, + new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} }); + + // Multiple columns from everywhere storefile, many family, wildcard + scan = new Scan(); + scan.addFamily(FAMILIES[2]); + scan.addFamily(FAMILIES[4]); + scan.addFamily(FAMILIES[6]); + scan.addFamily(FAMILIES[7]); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, + new int [][] { + {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7} + }); + + // Multiple columns from everywhere storefile, many family, wildcard + scan = new Scan(); + scan.addColumn(FAMILIES[2], QUALIFIERS[2]); + scan.addColumn(FAMILIES[2], QUALIFIERS[4]); + scan.addColumn(FAMILIES[4], QUALIFIERS[0]); + scan.addColumn(FAMILIES[4], QUALIFIERS[4]); + scan.addColumn(FAMILIES[6], QUALIFIERS[6]); + scan.addColumn(FAMILIES[6], QUALIFIERS[7]); + scan.addColumn(FAMILIES[7], QUALIFIERS[7]); + scan.addColumn(FAMILIES[7], QUALIFIERS[8]); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, + new int [][] { + {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7} + }); + + // Everything + scan = new Scan(); + result = getSingleScanResult(ht, scan); + assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, + new int [][] { + {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0} + }); + + // Scan around inserted columns + + scan = new Scan(ROWS[1]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + scan = new Scan(); + scan.addColumn(FAMILIES[4], QUALIFIERS[3]); + scan.addColumn(FAMILIES[2], QUALIFIERS[3]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + } + + /** + * Verify a single column using gets. + * Expects family and qualifier arrays to be valid for at least + * the range: idx-2 < idx < idx+2 + */ + private void getVerifySingleColumn(HTable ht, + byte [][] ROWS, int ROWIDX, + byte [][] FAMILIES, int FAMILYIDX, + byte [][] QUALIFIERS, int QUALIFIERIDX, + byte [][] VALUES, int VALUEIDX) + throws Exception { + + Get get = new Get(ROWS[ROWIDX]); + Result result = ht.get(get); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + get = new Get(ROWS[ROWIDX]); + get.addFamily(FAMILIES[FAMILYIDX]); + result = ht.get(get); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + get = new Get(ROWS[ROWIDX]); + get.addFamily(FAMILIES[FAMILYIDX-2]); + get.addFamily(FAMILIES[FAMILYIDX]); + get.addFamily(FAMILIES[FAMILYIDX+2]); + result = ht.get(get); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + get = new Get(ROWS[ROWIDX]); + get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[0]); + result = ht.get(get); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + get = new Get(ROWS[ROWIDX]); + get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[1]); + get.addFamily(FAMILIES[FAMILYIDX]); + result = ht.get(get); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + get = new Get(ROWS[ROWIDX]); + get.addFamily(FAMILIES[FAMILYIDX]); + get.addColumn(FAMILIES[FAMILYIDX+1], QUALIFIERS[1]); + get.addColumn(FAMILIES[FAMILYIDX-2], QUALIFIERS[1]); + get.addFamily(FAMILIES[FAMILYIDX-1]); + get.addFamily(FAMILIES[FAMILYIDX+2]); + result = ht.get(get); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + } + + + /** + * Verify a single column using scanners. + * Expects family and qualifier arrays to be valid for at least + * the range: idx-2 to idx+2 + * Expects row array to be valid for at least idx to idx+2 + */ + private void scanVerifySingleColumn(HTable ht, + byte [][] ROWS, int ROWIDX, + byte [][] FAMILIES, int FAMILYIDX, + byte [][] QUALIFIERS, int QUALIFIERIDX, + byte [][] VALUES, int VALUEIDX) + throws Exception { + + Scan scan = new Scan(); + Result result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + scan = new Scan(ROWS[ROWIDX]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + scan = new Scan(ROWS[ROWIDX], ROWS[ROWIDX+1]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX+1]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + scan = new Scan(); + scan.addFamily(FAMILIES[FAMILYIDX]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + scan = new Scan(); + scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + scan = new Scan(); + scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]); + scan.addFamily(FAMILIES[FAMILYIDX]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + scan = new Scan(); + scan.addColumn(FAMILIES[FAMILYIDX-1], QUALIFIERS[QUALIFIERIDX+1]); + scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]); + scan.addFamily(FAMILIES[FAMILYIDX+1]); + result = getSingleScanResult(ht, scan); + assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], + QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); + + } + + /** + * Verify we do not read any values by accident around a single column + * Same requirements as getVerifySingleColumn + */ + private void getVerifySingleEmpty(HTable ht, + byte [][] ROWS, int ROWIDX, + byte [][] FAMILIES, int FAMILYIDX, + byte [][] QUALIFIERS, int QUALIFIERIDX) + throws Exception { + + Get get = new Get(ROWS[ROWIDX]); + get.addFamily(FAMILIES[4]); + get.addColumn(FAMILIES[4], QUALIFIERS[1]); + Result result = ht.get(get); + assertEmptyResult(result); + + get = new Get(ROWS[ROWIDX]); + get.addFamily(FAMILIES[4]); + get.addColumn(FAMILIES[4], QUALIFIERS[2]); + result = ht.get(get); + assertEmptyResult(result); + + get = new Get(ROWS[ROWIDX]); + get.addFamily(FAMILIES[3]); + get.addColumn(FAMILIES[4], QUALIFIERS[2]); + get.addFamily(FAMILIES[5]); + result = ht.get(get); + assertEmptyResult(result); + + get = new Get(ROWS[ROWIDX+1]); + result = ht.get(get); + assertEmptyResult(result); + + } + + private void scanVerifySingleEmpty(HTable ht, + byte [][] ROWS, int ROWIDX, + byte [][] FAMILIES, int FAMILYIDX, + byte [][] QUALIFIERS, int QUALIFIERIDX) + throws Exception { + + Scan scan = new Scan(ROWS[ROWIDX+1]); + Result result = getSingleScanResult(ht, scan); + assertNullResult(result); + + scan = new Scan(ROWS[ROWIDX+1],ROWS[ROWIDX+2]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + scan = new Scan(); + scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]); + scan.addFamily(FAMILIES[FAMILYIDX-1]); + result = getSingleScanResult(ht, scan); + assertNullResult(result); + + } + + // + // Verifiers + // + + private void assertKey(KeyValue key, byte [] row, byte [] family, + byte [] qualifier, byte [] value) + throws Exception { + assertTrue("Expected row [" + Bytes.toString(row) + "] " + + "Got row [" + Bytes.toString(key.getRow()) +"]", + equals(row, key.getRow())); + assertTrue("Expected family [" + Bytes.toString(family) + "] " + + "Got family [" + Bytes.toString(key.getFamily()) + "]", + equals(family, key.getFamily())); + assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " + + "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]", + equals(qualifier, key.getQualifier())); + assertTrue("Expected value [" + Bytes.toString(value) + "] " + + "Got value [" + Bytes.toString(key.getValue()) + "]", + equals(value, key.getValue())); + } + + private void assertIncrementKey(KeyValue key, byte [] row, byte [] family, + byte [] qualifier, long value) + throws Exception { + assertTrue("Expected row [" + Bytes.toString(row) + "] " + + "Got row [" + Bytes.toString(key.getRow()) +"]", + equals(row, key.getRow())); + assertTrue("Expected family [" + Bytes.toString(family) + "] " + + "Got family [" + Bytes.toString(key.getFamily()) + "]", + equals(family, key.getFamily())); + assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " + + "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]", + equals(qualifier, key.getQualifier())); + assertTrue("Expected value [" + value + "] " + + "Got value [" + Bytes.toLong(key.getValue()) + "]", + Bytes.toLong(key.getValue()) == value); + } + + private void assertNumKeys(Result result, int n) throws Exception { + assertTrue("Expected " + n + " keys but got " + result.size(), + result.size() == n); + } + + private void assertNResult(Result result, byte [] row, + byte [][] families, byte [][] qualifiers, byte [][] values, + int [][] idxs) + throws Exception { + assertTrue("Expected row [" + Bytes.toString(row) + "] " + + "Got row [" + Bytes.toString(result.getRow()) +"]", + equals(row, result.getRow())); + assertTrue("Expected " + idxs.length + " keys but result contains " + + result.size(), result.size() == idxs.length); + + KeyValue [] keys = result.raw(); + + for(int i=0;i 256) { + return makeNBig(base, n); + } + byte [][] ret = new byte[n][]; + for(int i=0;i 256) { + return makeNBig(base, n); + } + byte [][] ret = new byte[n][]; + for(int i=0;i> 8); + ret[i] = Bytes.add(base, new byte[]{(byte)byteB,(byte)byteA}); + } + return ret; + } + + private long [] makeStamps(int n) { + long [] stamps = new long[n]; + for(int i=0;i navigableMap = + result.getMap().get(FAMILY).get(qualifier); + assertEquals("AAA", Bytes.toString(navigableMap.get(1L))); + assertEquals("BBB", Bytes.toString(navigableMap.get(2L))); + + // Update the value at timestamp 1 + put = new Put(row); + put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC")); + hTable.put(put); + + // Update the value at timestamp 2 + put = new Put(row); + put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD")); + hTable.put(put); + + // Check that the values at timestamp 2 and 1 got updated + result = hTable.get(get); + navigableMap = result.getMap().get(FAMILY).get(qualifier); + assertEquals("CCC", Bytes.toString(navigableMap.get(1L))); + assertEquals("DDD", Bytes.toString(navigableMap.get(2L))); + } + + @Test + public void testUpdatesWithMajorCompaction() throws Exception { + + String tableName = "testUpdatesWithMajorCompaction"; + byte [] TABLE = Bytes.toBytes(tableName); + HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10); + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + + // Write a column with values at timestamp 1, 2 and 3 + byte[] row = Bytes.toBytes("row2"); + byte[] qualifier = Bytes.toBytes("myCol"); + Put put = new Put(row); + put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA")); + hTable.put(put); + + put = new Put(row); + put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB")); + hTable.put(put); + + put = new Put(row); + put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE")); + hTable.put(put); + + Get get = new Get(row); + get.addColumn(FAMILY, qualifier); + get.setMaxVersions(); + + // Check that the column indeed has the right values at timestamps 1 and + // 2 + Result result = hTable.get(get); + NavigableMap navigableMap = + result.getMap().get(FAMILY).get(qualifier); + assertEquals("AAA", Bytes.toString(navigableMap.get(1L))); + assertEquals("BBB", Bytes.toString(navigableMap.get(2L))); + + // Trigger a major compaction + admin.flush(tableName); + admin.majorCompact(tableName); + Thread.sleep(6000); + + // Update the value at timestamp 1 + put = new Put(row); + put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC")); + hTable.put(put); + + // Update the value at timestamp 2 + put = new Put(row); + put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD")); + hTable.put(put); + + // Trigger a major compaction + admin.flush(tableName); + admin.majorCompact(tableName); + Thread.sleep(6000); + + // Check that the values at timestamp 2 and 1 got updated + result = hTable.get(get); + navigableMap = result.getMap().get(FAMILY).get(qualifier); + assertEquals("CCC", Bytes.toString(navigableMap.get(1L))); + assertEquals("DDD", Bytes.toString(navigableMap.get(2L))); + } + + @Test + public void testMajorCompactionBetweenTwoUpdates() throws Exception { + + String tableName = "testMajorCompactionBetweenTwoUpdates"; + byte [] TABLE = Bytes.toBytes(tableName); + HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10); + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + + // Write a column with values at timestamp 1, 2 and 3 + byte[] row = Bytes.toBytes("row3"); + byte[] qualifier = Bytes.toBytes("myCol"); + Put put = new Put(row); + put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA")); + hTable.put(put); + + put = new Put(row); + put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB")); + hTable.put(put); + + put = new Put(row); + put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE")); + hTable.put(put); + + Get get = new Get(row); + get.addColumn(FAMILY, qualifier); + get.setMaxVersions(); + + // Check that the column indeed has the right values at timestamps 1 and + // 2 + Result result = hTable.get(get); + NavigableMap navigableMap = + result.getMap().get(FAMILY).get(qualifier); + assertEquals("AAA", Bytes.toString(navigableMap.get(1L))); + assertEquals("BBB", Bytes.toString(navigableMap.get(2L))); + + // Trigger a major compaction + admin.flush(tableName); + admin.majorCompact(tableName); + Thread.sleep(6000); + + // Update the value at timestamp 1 + put = new Put(row); + put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC")); + hTable.put(put); + + // Trigger a major compaction + admin.flush(tableName); + admin.majorCompact(tableName); + Thread.sleep(6000); + + // Update the value at timestamp 2 + put = new Put(row); + put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD")); + hTable.put(put); + + // Trigger a major compaction + admin.flush(tableName); + admin.majorCompact(tableName); + Thread.sleep(6000); + + // Check that the values at timestamp 2 and 1 got updated + result = hTable.get(get); + navigableMap = result.getMap().get(FAMILY).get(qualifier); + + assertEquals("CCC", Bytes.toString(navigableMap.get(1L))); + assertEquals("DDD", Bytes.toString(navigableMap.get(2L))); + } + + @Test + public void testGet_EmptyTable() throws IOException { + HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_EmptyTable"), FAMILY); + Get get = new Get(ROW); + get.addFamily(FAMILY); + Result r = table.get(get); + assertTrue(r.isEmpty()); + } + + @Test + public void testGet_NonExistentRow() throws IOException { + HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_NonExistentRow"), FAMILY); + Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, VALUE); + table.put(put); + LOG.info("Row put"); + + Get get = new Get(ROW); + get.addFamily(FAMILY); + Result r = table.get(get); + assertFalse(r.isEmpty()); + System.out.println("Row retrieved successfully"); + + byte [] missingrow = Bytes.toBytes("missingrow"); + get = new Get(missingrow); + get.addFamily(FAMILY); + r = table.get(get); + assertTrue(r.isEmpty()); + LOG.info("Row missing as it should be"); + } + + @Test + public void testPut() throws IOException { + final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents"); + final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam"); + final byte [] row1 = Bytes.toBytes("row1"); + final byte [] row2 = Bytes.toBytes("row2"); + final byte [] value = Bytes.toBytes("abcd"); + HTable table = TEST_UTIL.createTable(Bytes.toBytes("testPut"), + new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY}); + Put put = new Put(row1); + put.add(CONTENTS_FAMILY, null, value); + table.put(put); + + put = new Put(row2); + put.add(CONTENTS_FAMILY, null, value); + + assertEquals(put.size(), 1); + assertEquals(put.getFamilyMap().get(CONTENTS_FAMILY).size(), 1); + + KeyValue kv = put.getFamilyMap().get(CONTENTS_FAMILY).get(0); + + assertTrue(Bytes.equals(kv.getFamily(), CONTENTS_FAMILY)); + // will it return null or an empty byte array? + assertTrue(Bytes.equals(kv.getQualifier(), new byte[0])); + + assertTrue(Bytes.equals(kv.getValue(), value)); + + table.put(put); + + Scan scan = new Scan(); + scan.addColumn(CONTENTS_FAMILY, null); + ResultScanner scanner = table.getScanner(scan); + for (Result r : scanner) { + for(KeyValue key : r.raw()) { + System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString()); + } + } + } + + @Test + public void testPutNoCF() throws IOException { + final byte[] BAD_FAM = Bytes.toBytes("BAD_CF"); + final byte[] VAL = Bytes.toBytes(100); + HTable table = TEST_UTIL.createTable(Bytes.toBytes("testPutNoCF"), new byte[][]{FAMILY}); + + boolean caughtNSCFE = false; + + try { + Put p = new Put(ROW); + p.add(BAD_FAM, QUALIFIER, VAL); + table.put(p); + } catch (RetriesExhaustedWithDetailsException e) { + caughtNSCFE = e.getCause(0) instanceof NoSuchColumnFamilyException; + } + assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE); + + } + + @Test + public void testRowsPut() throws IOException { + final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents"); + final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam"); + final int NB_BATCH_ROWS = 10; + final byte[] value = Bytes.toBytes("abcd"); + HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPut"), + new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY }); + ArrayList rowsUpdate = new ArrayList(); + for (int i = 0; i < NB_BATCH_ROWS; i++) { + byte[] row = Bytes.toBytes("row" + i); + Put put = new Put(row); + put.setWriteToWAL(false); + put.add(CONTENTS_FAMILY, null, value); + rowsUpdate.add(put); + } + table.put(rowsUpdate); + Scan scan = new Scan(); + scan.addFamily(CONTENTS_FAMILY); + ResultScanner scanner = table.getScanner(scan); + int nbRows = 0; + for (@SuppressWarnings("unused") + Result row : scanner) + nbRows++; + assertEquals(NB_BATCH_ROWS, nbRows); + } + + @Test + public void testRowsPutBufferedOneFlush() throws IOException { + final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents"); + final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam"); + final byte [] value = Bytes.toBytes("abcd"); + final int NB_BATCH_ROWS = 10; + HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedOneFlush"), + new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY}); + table.setAutoFlush(false); + ArrayList rowsUpdate = new ArrayList(); + for (int i = 0; i < NB_BATCH_ROWS * 10; i++) { + byte[] row = Bytes.toBytes("row" + i); + Put put = new Put(row); + put.setWriteToWAL(false); + put.add(CONTENTS_FAMILY, null, value); + rowsUpdate.add(put); + } + table.put(rowsUpdate); + + Scan scan = new Scan(); + scan.addFamily(CONTENTS_FAMILY); + ResultScanner scanner = table.getScanner(scan); + int nbRows = 0; + for (@SuppressWarnings("unused") + Result row : scanner) + nbRows++; + assertEquals(0, nbRows); + scanner.close(); + + table.flushCommits(); + + scan = new Scan(); + scan.addFamily(CONTENTS_FAMILY); + scanner = table.getScanner(scan); + nbRows = 0; + for (@SuppressWarnings("unused") + Result row : scanner) + nbRows++; + assertEquals(NB_BATCH_ROWS * 10, nbRows); + } + + @Test + public void testRowsPutBufferedManyManyFlushes() throws IOException { + final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents"); + final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam"); + final byte[] value = Bytes.toBytes("abcd"); + final int NB_BATCH_ROWS = 10; + HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedManyManyFlushes"), + new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY }); + table.setAutoFlush(false); + table.setWriteBufferSize(10); + ArrayList rowsUpdate = new ArrayList(); + for (int i = 0; i < NB_BATCH_ROWS * 10; i++) { + byte[] row = Bytes.toBytes("row" + i); + Put put = new Put(row); + put.setWriteToWAL(false); + put.add(CONTENTS_FAMILY, null, value); + rowsUpdate.add(put); + } + table.put(rowsUpdate); + + table.flushCommits(); + + Scan scan = new Scan(); + scan.addFamily(CONTENTS_FAMILY); + ResultScanner scanner = table.getScanner(scan); + int nbRows = 0; + for (@SuppressWarnings("unused") + Result row : scanner) + nbRows++; + assertEquals(NB_BATCH_ROWS * 10, nbRows); + } + + @Test + public void testAddKeyValue() throws IOException { + final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents"); + final byte[] value = Bytes.toBytes("abcd"); + final byte[] row1 = Bytes.toBytes("row1"); + final byte[] row2 = Bytes.toBytes("row2"); + byte[] qualifier = Bytes.toBytes("qf1"); + Put put = new Put(row1); + + // Adding KeyValue with the same row + KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value); + boolean ok = true; + try { + put.add(kv); + } catch (IOException e) { + ok = false; + } + assertEquals(true, ok); + + // Adding KeyValue with the different row + kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value); + ok = false; + try { + put.add(kv); + } catch (IOException e) { + ok = true; + } + assertEquals(true, ok); + } + + /** + * test for HBASE-737 + * @throws IOException + */ + @Test + public void testHBase737 () throws IOException { + final byte [] FAM1 = Bytes.toBytes("fam1"); + final byte [] FAM2 = Bytes.toBytes("fam2"); + // Open table + HTable table = TEST_UTIL.createTable(Bytes.toBytes("testHBase737"), + new byte [][] {FAM1, FAM2}); + // Insert some values + Put put = new Put(ROW); + put.add(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg")); + table.put(put); + try { + Thread.sleep(1000); + } catch (InterruptedException i) { + //ignore + } + + put = new Put(ROW); + put.add(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456")); + table.put(put); + + try { + Thread.sleep(1000); + } catch (InterruptedException i) { + //ignore + } + + put = new Put(ROW); + put.add(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop")); + table.put(put); + + long times[] = new long[3]; + + // First scan the memstore + + Scan scan = new Scan(); + scan.addFamily(FAM1); + scan.addFamily(FAM2); + ResultScanner s = table.getScanner(scan); + try { + int index = 0; + Result r = null; + while ((r = s.next()) != null) { + for(KeyValue key : r.raw()) { + times[index++] = key.getTimestamp(); + } + } + } finally { + s.close(); + } + for (int i = 0; i < times.length - 1; i++) { + for (int j = i + 1; j < times.length; j++) { + assertTrue(times[j] > times[i]); + } + } + + // Flush data to disk and try again + TEST_UTIL.flush(); + + // Reset times + for(int i=0;i times[i]); + } + } + } + + @Test + public void testListTables() throws IOException, InterruptedException { + byte [] t1 = Bytes.toBytes("testListTables1"); + byte [] t2 = Bytes.toBytes("testListTables2"); + byte [] t3 = Bytes.toBytes("testListTables3"); + byte [][] tables = new byte[][] { t1, t2, t3 }; + for (int i = 0; i < tables.length; i++) { + TEST_UTIL.createTable(tables[i], FAMILY); + } + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + HTableDescriptor[] ts = admin.listTables(); + HashSet result = new HashSet(ts.length); + for (int i = 0; i < ts.length; i++) { + result.add(ts[i]); + } + int size = result.size(); + assertTrue(size >= tables.length); + for (int i = 0; i < tables.length && i < size; i++) { + boolean found = false; + for (int j = 0; j < ts.length; j++) { + if (Bytes.equals(ts[j].getName(), tables[i])) { + found = true; + break; + } + } + assertTrue("Not found: " + Bytes.toString(tables[i]), found); + } + } + + /** + * creates an HTable for tableName using an unmanaged HConnection. + * + * @param tableName - table to create + * @return the created HTable object + * @throws IOException + */ + HTable createUnmangedHConnectionHTable(final byte [] tableName) throws IOException { + TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY); + HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration()); + ExecutorService pool = new ThreadPoolExecutor(1, Integer.MAX_VALUE, + 60, TimeUnit.SECONDS, + new SynchronousQueue(), + Threads.newDaemonThreadFactory("test-from-client-table")); + ((ThreadPoolExecutor)pool).allowCoreThreadTimeOut(true); + return new HTable(tableName, conn, pool); + } + + /** + * simple test that just executes parts of the client + * API that accept a pre-created HConnction instance + * + * @throws IOException + */ + @Test + public void testUnmanagedHConnection() throws IOException { + final byte[] tableName = Bytes.toBytes("testUnmanagedHConnection"); + HTable t = createUnmangedHConnectionHTable(tableName); + HBaseAdmin ha = new HBaseAdmin(t.getConnection()); + assertTrue(ha.tableExists(tableName)); + assertTrue(t.get(new Get(ROW)).isEmpty()); + } + + /** + * test of that unmanaged HConnections are able to reconnect + * properly (see HBASE-5058) + * + * @throws Exception + */ + @Test + public void testUnmanagedHConnectionReconnect() throws Exception { + final byte[] tableName = Bytes.toBytes("testUnmanagedHConnectionReconnect"); + HTable t = createUnmangedHConnectionHTable(tableName); + HConnection conn = t.getConnection(); + HBaseAdmin ha = new HBaseAdmin(conn); + assertTrue(ha.tableExists(tableName)); + assertTrue(t.get(new Get(ROW)).isEmpty()); + + // stop the master + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + cluster.stopMaster(0, false); + cluster.waitOnMaster(0); + + // start up a new master + cluster.startMaster(); + assertTrue(cluster.waitForActiveAndReadyMaster()); + + // test that the same unmanaged connection works with a new + // HBaseAdmin and can connect to the new master; + HBaseAdmin newAdmin = new HBaseAdmin(conn); + assertTrue(newAdmin.tableExists(tableName)); + assert(newAdmin.getClusterStatus().getServersSize() == SLAVES); + } + + @Test + public void testMiscHTableStuff() throws IOException { + final byte[] tableAname = Bytes.toBytes("testMiscHTableStuffA"); + final byte[] tableBname = Bytes.toBytes("testMiscHTableStuffB"); + final byte[] attrName = Bytes.toBytes("TESTATTR"); + final byte[] attrValue = Bytes.toBytes("somevalue"); + byte[] value = Bytes.toBytes("value"); + + HTable a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY); + HTable b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY); + Put put = new Put(ROW); + put.add(HConstants.CATALOG_FAMILY, null, value); + a.put(put); + + // open a new connection to A and a connection to b + HTable newA = new HTable(TEST_UTIL.getConfiguration(), tableAname); + + // copy data from A to B + Scan scan = new Scan(); + scan.addFamily(HConstants.CATALOG_FAMILY); + ResultScanner s = newA.getScanner(scan); + try { + for (Result r : s) { + put = new Put(r.getRow()); + put.setWriteToWAL(false); + for (KeyValue kv : r.raw()) { + put.add(kv); + } + b.put(put); + } + } finally { + s.close(); + } + + // Opening a new connection to A will cause the tables to be reloaded + HTable anotherA = new HTable(TEST_UTIL.getConfiguration(), tableAname); + Get get = new Get(ROW); + get.addFamily(HConstants.CATALOG_FAMILY); + anotherA.get(get); + + // We can still access A through newA because it has the table information + // cached. And if it needs to recalibrate, that will cause the information + // to be reloaded. + + // Test user metadata + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + // make a modifiable descriptor + HTableDescriptor desc = new HTableDescriptor(a.getTableDescriptor()); + // offline the table + admin.disableTable(tableAname); + // add a user attribute to HTD + desc.setValue(attrName, attrValue); + // add a user attribute to HCD + for (HColumnDescriptor c : desc.getFamilies()) + c.setValue(attrName, attrValue); + // update metadata for all regions of this table + admin.modifyTable(tableAname, desc); + // enable the table + admin.enableTable(tableAname); + + // Test that attribute changes were applied + desc = a.getTableDescriptor(); + assertTrue("wrong table descriptor returned", + Bytes.compareTo(desc.getName(), tableAname) == 0); + // check HTD attribute + value = desc.getValue(attrName); + assertFalse("missing HTD attribute value", value == null); + assertFalse("HTD attribute value is incorrect", + Bytes.compareTo(value, attrValue) != 0); + // check HCD attribute + for (HColumnDescriptor c : desc.getFamilies()) { + value = c.getValue(attrName); + assertFalse("missing HCD attribute value", value == null); + assertFalse("HCD attribute value is incorrect", + Bytes.compareTo(value, attrValue) != 0); + } + } + + @Test + public void testGetClosestRowBefore() throws IOException { + final byte [] tableAname = Bytes.toBytes("testGetClosestRowBefore"); + final byte [] row = Bytes.toBytes("row"); + + + byte[] firstRow = Bytes.toBytes("ro"); + byte[] beforeFirstRow = Bytes.toBytes("rn"); + byte[] beforeSecondRow = Bytes.toBytes("rov"); + + HTable table = TEST_UTIL.createTable(tableAname, + new byte [][] {HConstants.CATALOG_FAMILY, Bytes.toBytes("info2")}); + Put put = new Put(firstRow); + Put put2 = new Put(row); + byte[] zero = new byte[]{0}; + byte[] one = new byte[]{1}; + + put.add(HConstants.CATALOG_FAMILY, null, zero); + put2.add(HConstants.CATALOG_FAMILY, null, one); + + table.put(put); + table.put(put2); + + Result result = null; + + // Test before first that null is returned + result = table.getRowOrBefore(beforeFirstRow, HConstants.CATALOG_FAMILY); + assertTrue(result == null); + + // Test at first that first is returned + result = table.getRowOrBefore(firstRow, HConstants.CATALOG_FAMILY); + assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null)); + assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), zero)); + + // Test in between first and second that first is returned + result = table.getRowOrBefore(beforeSecondRow, HConstants.CATALOG_FAMILY); + assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null)); + assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), zero)); + + // Test at second make sure second is returned + result = table.getRowOrBefore(row, HConstants.CATALOG_FAMILY); + assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null)); + assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one)); + + // Test after second, make sure second is returned + result = table.getRowOrBefore(Bytes.add(row,one), HConstants.CATALOG_FAMILY); + assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null)); + assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one)); + } + + /** + * For HBASE-2156 + * @throws Exception + */ + @Test + public void testScanVariableReuse() throws Exception { + Scan scan = new Scan(); + scan.addFamily(FAMILY); + scan.addColumn(FAMILY, ROW); + + assertTrue(scan.getFamilyMap().get(FAMILY).size() == 1); + + scan = new Scan(); + scan.addFamily(FAMILY); + + assertTrue(scan.getFamilyMap().get(FAMILY) == null); + assertTrue(scan.getFamilyMap().containsKey(FAMILY)); + } + + @Test + public void testMultiRowMutation() throws Exception { + LOG.info("Starting testMultiRowMutation"); + final byte [] TABLENAME = Bytes.toBytes("testMultiRowMutation"); + final byte [] ROW1 = Bytes.toBytes("testRow1"); + + HTable t = TEST_UTIL.createTable(TABLENAME, FAMILY); + List mrm = new ArrayList(); + Put p = new Put(ROW); + p.add(FAMILY, QUALIFIER, VALUE); + mrm.add(p); + p = new Put(ROW1); + p.add(FAMILY, QUALIFIER, VALUE); + mrm.add(p); + MultiRowMutationProtocol mr = t.coprocessorProxy( + MultiRowMutationProtocol.class, ROW); + mr.mutateRows(mrm); + Get g = new Get(ROW); + Result r = t.get(g); + assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER))); + g = new Get(ROW1); + r = t.get(g); + assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER))); + } + + @Test + public void testRowMutation() throws Exception { + LOG.info("Starting testRowMutation"); + final byte [] TABLENAME = Bytes.toBytes("testRowMutation"); + HTable t = TEST_UTIL.createTable(TABLENAME, FAMILY); + byte [][] QUALIFIERS = new byte [][] { + Bytes.toBytes("a"), Bytes.toBytes("b") + }; + RowMutations arm = new RowMutations(ROW); + Put p = new Put(ROW); + p.add(FAMILY, QUALIFIERS[0], VALUE); + arm.add(p); + t.mutateRow(arm); + + Get g = new Get(ROW); + Result r = t.get(g); + assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0]))); + + arm = new RowMutations(ROW); + p = new Put(ROW); + p.add(FAMILY, QUALIFIERS[1], VALUE); + arm.add(p); + Delete d = new Delete(ROW); + d.deleteColumns(FAMILY, QUALIFIERS[0]); + arm.add(d); + t.batch(Arrays.asList((Row)arm)); + r = t.get(g); + assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1]))); + assertNull(r.getValue(FAMILY, QUALIFIERS[0])); + } + + @Test + public void testAppend() throws Exception { + LOG.info("Starting testAppend"); + final byte [] TABLENAME = Bytes.toBytes("testAppend"); + HTable t = TEST_UTIL.createTable(TABLENAME, FAMILY); + byte[] v1 = Bytes.toBytes("42"); + byte[] v2 = Bytes.toBytes("23"); + byte [][] QUALIFIERS = new byte [][] { + Bytes.toBytes("a"), Bytes.toBytes("b") + }; + Append a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[0], v1); + a.add(FAMILY, QUALIFIERS[1], v2); + a.setReturnResults(false); + assertNullResult(t.append(a)); + + a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[0], v2); + a.add(FAMILY, QUALIFIERS[1], v1); + Result r = t.append(a); + assertEquals(0, Bytes.compareTo(Bytes.add(v1,v2), r.getValue(FAMILY, QUALIFIERS[0]))); + assertEquals(0, Bytes.compareTo(Bytes.add(v2,v1), r.getValue(FAMILY, QUALIFIERS[1]))); + } + + @Test + public void testIncrementWithDeletes() throws Exception { + LOG.info("Starting testIncrementWithDeletes"); + final byte [] TABLENAME = Bytes.toBytes("testIncrementWithDeletes"); + HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY); + final byte[] COLUMN = Bytes.toBytes("column"); + + ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5); + TEST_UTIL.flush(TABLENAME); + + Delete del = new Delete(ROW); + ht.delete(del); + + ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5); + + Get get = new Get(ROW); + Result r = ht.get(get); + assertEquals(1, r.size()); + assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN))); + } + + @Test + public void testIncrementingInvalidValue() throws Exception { + LOG.info("Starting testIncrementingInvalidValue"); + final byte [] TABLENAME = Bytes.toBytes("testIncrementingInvalidValue"); + HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY); + final byte[] COLUMN = Bytes.toBytes("column"); + Put p = new Put(ROW); + // write an integer here (not a Long) + p.add(FAMILY, COLUMN, Bytes.toBytes(5)); + ht.put(p); + try { + ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5); + fail("Should have thrown DoNotRetryIOException"); + } catch (DoNotRetryIOException iox) { + // success + } + Increment inc = new Increment(ROW); + inc.addColumn(FAMILY, COLUMN, 5); + try { + ht.increment(inc); + fail("Should have thrown DoNotRetryIOException"); + } catch (DoNotRetryIOException iox) { + // success + } + } + + + + @Test + public void testIncrement() throws Exception { + LOG.info("Starting testIncrement"); + final byte [] TABLENAME = Bytes.toBytes("testIncrement"); + HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY); + + byte [][] ROWS = new byte [][] { + Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"), + Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"), + Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i") + }; + byte [][] QUALIFIERS = new byte [][] { + Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"), + Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"), + Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i") + }; + + // Do some simple single-column increments + + // First with old API + ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[0], 1); + ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[1], 2); + ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[2], 3); + ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[3], 4); + + // Now increment things incremented with old and do some new + Increment inc = new Increment(ROW); + inc.addColumn(FAMILY, QUALIFIERS[1], 1); + inc.addColumn(FAMILY, QUALIFIERS[3], 1); + inc.addColumn(FAMILY, QUALIFIERS[4], 1); + ht.increment(inc); + + // Verify expected results + Result r = ht.get(new Get(ROW)); + KeyValue [] kvs = r.raw(); + assertEquals(5, kvs.length); + assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1); + assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 3); + assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 3); + assertIncrementKey(kvs[3], ROW, FAMILY, QUALIFIERS[3], 5); + assertIncrementKey(kvs[4], ROW, FAMILY, QUALIFIERS[4], 1); + + // Now try multiple columns by different amounts + inc = new Increment(ROWS[0]); + for (int i=0;i queue = new SynchronousQueue(); + List tasks = new ArrayList(5); + for (int i = 0; i < 5; i++) { + tasks.add(new Runnable() { + public void run() { + try { + // The thread blocks here until we decide to let it go + queue.take(); + } catch (InterruptedException ie) { } + } + }); + } + // First, add two tasks and make sure the pool size follows + pool.submit(tasks.get(0)); + assertEquals(1, pool.getPoolSize()); + pool.submit(tasks.get(1)); + assertEquals(2, pool.getPoolSize()); + + // Next, terminate those tasks and then make sure the pool is still the + // same size + queue.put(new Object()); + queue.put(new Object()); + assertEquals(2, pool.getPoolSize()); + + //ensure that ThreadPoolExecutor knows that tasks are finished. + while (pool.getCompletedTaskCount() < 2) { + Threads.sleep(1); + } + + // Now let's simulate adding a RS meaning that we'll go up to three + // concurrent threads. The pool should not grow larger than three. + pool.submit(tasks.get(2)); + pool.submit(tasks.get(3)); + pool.submit(tasks.get(4)); + assertEquals(3, pool.getPoolSize()); + queue.put(new Object()); + queue.put(new Object()); + queue.put(new Object()); + } + + @Test + public void testClientPoolRoundRobin() throws IOException { + final byte[] tableName = Bytes.toBytes("testClientPoolRoundRobin"); + + int poolSize = 3; + int numVersions = poolSize * 2; + Configuration conf = TEST_UTIL.getConfiguration(); + conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin"); + conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize); + + HTable table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, + conf, Integer.MAX_VALUE); + table.setAutoFlush(true); + Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, VALUE); + + Get get = new Get(ROW); + get.addColumn(FAMILY, QUALIFIER); + get.setMaxVersions(); + + for (int versions = 1; versions <= numVersions; versions++) { + table.put(put); + + Result result = table.get(get); + NavigableMap navigableMap = result.getMap().get(FAMILY) + .get(QUALIFIER); + + assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER + + " did not match " + versions, versions, navigableMap.size()); + for (Map.Entry entry : navigableMap.entrySet()) { + assertTrue("The value at time " + entry.getKey() + + " did not match what was put", + Bytes.equals(VALUE, entry.getValue())); + } + } + } + + @Test + public void testClientPoolThreadLocal() throws IOException { + final byte[] tableName = Bytes.toBytes("testClientPoolThreadLocal"); + + int poolSize = Integer.MAX_VALUE; + int numVersions = 3; + Configuration conf = TEST_UTIL.getConfiguration(); + conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local"); + conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize); + + final HTable table = TEST_UTIL.createTable(tableName, + new byte[][] { FAMILY }, conf); + table.setAutoFlush(true); + final Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, VALUE); + + final Get get = new Get(ROW); + get.addColumn(FAMILY, QUALIFIER); + get.setMaxVersions(); + + for (int versions = 1; versions <= numVersions; versions++) { + table.put(put); + + Result result = table.get(get); + NavigableMap navigableMap = result.getMap().get(FAMILY) + .get(QUALIFIER); + + assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER + + " did not match " + versions, versions, navigableMap.size()); + for (Map.Entry entry : navigableMap.entrySet()) { + assertTrue("The value at time " + entry.getKey() + + " did not match what was put", + Bytes.equals(VALUE, entry.getValue())); + } + } + + final Object waitLock = new Object(); + ExecutorService executorService = Executors.newFixedThreadPool(numVersions); + final AtomicReference error = new AtomicReference(null); + for (int versions = numVersions; versions < numVersions * 2; versions++) { + final int versionsCopy = versions; + executorService.submit(new Callable() { + @Override + public Void call() { + try { + table.put(put); + + Result result = table.get(get); + NavigableMap navigableMap = result.getMap() + .get(FAMILY).get(QUALIFIER); + + assertEquals("The number of versions of '" + FAMILY + ":" + + QUALIFIER + " did not match " + versionsCopy, versionsCopy, + navigableMap.size()); + for (Map.Entry entry : navigableMap.entrySet()) { + assertTrue("The value at time " + entry.getKey() + + " did not match what was put", + Bytes.equals(VALUE, entry.getValue())); + } + synchronized (waitLock) { + waitLock.wait(); + } + } catch (Exception e) { + } catch (AssertionError e) { + // the error happens in a thread, it won't fail the test, + // need to pass it to the caller for proper handling. + error.set(e); + LOG.error(e); + } + + return null; + } + }); + } + synchronized (waitLock) { + waitLock.notifyAll(); + } + executorService.shutdownNow(); + assertNull(error.get()); + } + + @Test + public void testCheckAndPut() throws IOException { + final byte [] anotherrow = Bytes.toBytes("anotherrow"); + final byte [] value2 = Bytes.toBytes("abcd"); + + HTable table = TEST_UTIL.createTable(Bytes.toBytes("testCheckAndPut"), + new byte [][] {FAMILY}); + Put put1 = new Put(ROW); + put1.add(FAMILY, QUALIFIER, VALUE); + + // row doesn't exist, so using non-null value should be considered "not match". + boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put1); + assertEquals(ok, false); + + // row doesn't exist, so using "null" to check for existence should be considered "match". + ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1); + assertEquals(ok, true); + + // row now exists, so using "null" to check for existence should be considered "not match". + ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1); + assertEquals(ok, false); + + Put put2 = new Put(ROW); + put2.add(FAMILY, QUALIFIER, value2); + + // row now exists, use the matching value to check + ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put2); + assertEquals(ok, true); + + Put put3 = new Put(anotherrow); + put3.add(FAMILY, QUALIFIER, VALUE); + + // try to do CheckAndPut on different rows + try { + ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, value2, put3); + fail("trying to check and modify different rows should have failed."); + } catch(Exception e) {} + + } + + /** + * Test ScanMetrics + * @throws Exception + */ + @Test + @SuppressWarnings ("unused") + public void testScanMetrics() throws Exception { + byte [] TABLENAME = Bytes.toBytes("testScanMetrics"); + + Configuration conf = TEST_UTIL.getConfiguration(); + TEST_UTIL.createTable(TABLENAME, FAMILY); + + // Set up test table: + // Create table: + HTable ht = new HTable(conf, TABLENAME); + + // Create multiple regions for this table + int numOfRegions = TEST_UTIL.createMultiRegions(ht, FAMILY); + // Create 3 rows in the table, with rowkeys starting with "z*" so that + // scan are forced to hit all the regions. + Put put1 = new Put(Bytes.toBytes("z1")); + put1.add(FAMILY, QUALIFIER, VALUE); + Put put2 = new Put(Bytes.toBytes("z2")); + put2.add(FAMILY, QUALIFIER, VALUE); + Put put3 = new Put(Bytes.toBytes("z3")); + put3.add(FAMILY, QUALIFIER, VALUE); + ht.put(Arrays.asList(put1, put2, put3)); + + Scan scan1 = new Scan(); + int numRecords = 0; + for(Result result : ht.getScanner(scan1)) { + numRecords++; + } + LOG.info("test data has " + numRecords + " records."); + + // by default, scan metrics collection is turned off + assertEquals(null, scan1.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA)); + + // turn on scan metrics + Scan scan = new Scan(); + scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE)); + ResultScanner scanner = ht.getScanner(scan); + // per HBASE-5717, this should still collect even if you don't run all the way to + // the end of the scanner. So this is asking for 2 of the 3 rows we inserted. + for (Result result : scanner.next(numRecords - 1)) { + } + scanner.close(); + + ScanMetrics scanMetrics = getScanMetrics(scan); + assertEquals("Did not access all the regions in the table", numOfRegions, + scanMetrics.countOfRegions.getCurrentIntervalValue()); + + // now, test that the metrics are still collected even if you don't call close, but do + // run past the end of all the records + Scan scanWithoutClose = new Scan(); + scanWithoutClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE)); + ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose); + for (Result result : scannerWithoutClose.next(numRecords + 1)) { + } + ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose); + assertEquals("Did not access all the regions in the table", numOfRegions, + scanMetricsWithoutClose.countOfRegions.getCurrentIntervalValue()); + + // finally, test that the metrics are collected correctly if you both run past all the records, + // AND close the scanner + Scan scanWithClose = new Scan(); + scanWithClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE)); + ResultScanner scannerWithClose = ht.getScanner(scanWithClose); + for (Result result : scannerWithClose.next(numRecords + 1)) { + } + scannerWithClose.close(); + ScanMetrics scanMetricsWithClose = getScanMetrics(scanWithClose); + assertEquals("Did not access all the regions in the table", numOfRegions, + scanMetricsWithClose.countOfRegions.getCurrentIntervalValue()); + } + + private ScanMetrics getScanMetrics(Scan scan) throws Exception { + byte[] serializedMetrics = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA); + assertTrue("Serialized metrics were not found.", serializedMetrics != null); + + DataInputBuffer in = new DataInputBuffer(); + in.reset(serializedMetrics, 0, serializedMetrics.length); + ScanMetrics scanMetrics = new ScanMetrics(); + scanMetrics.readFields(in); + return scanMetrics; + } + + /** + * Tests that cache on write works all the way up from the client-side. + * + * Performs inserts, flushes, and compactions, verifying changes in the block + * cache along the way. + * + * @throws Exception + */ + @Test + public void testCacheOnWriteEvictOnClose() throws Exception { + byte [] tableName = Bytes.toBytes("testCOWEOCfromClient"); + byte [] data = Bytes.toBytes("data"); + HTable table = TEST_UTIL.createTable(tableName, new byte [][] {FAMILY}); + // get the block cache and region + String regionName = table.getRegionLocations().firstKey().getEncodedName(); + HRegion region = TEST_UTIL.getRSForFirstRegionInTable( + tableName).getFromOnlineRegions(regionName); + Store store = region.getStores().values().iterator().next(); + CacheConfig cacheConf = store.getCacheConfig(); + cacheConf.setCacheDataOnWrite(true); + cacheConf.setEvictOnClose(true); + BlockCache cache = cacheConf.getBlockCache(); + + // establish baseline stats + long startBlockCount = cache.getBlockCount(); + long startBlockHits = cache.getStats().getHitCount(); + long startBlockMiss = cache.getStats().getMissCount(); + + // wait till baseline is stable, (minimal 500 ms) + for (int i = 0; i < 5; i++) { + Thread.sleep(100); + if (startBlockCount != cache.getBlockCount() + || startBlockHits != cache.getStats().getHitCount() + || startBlockMiss != cache.getStats().getMissCount()) { + startBlockCount = cache.getBlockCount(); + startBlockHits = cache.getStats().getHitCount(); + startBlockMiss = cache.getStats().getMissCount(); + i = -1; + } + } + + // insert data + Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, data); + table.put(put); + assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data)); + // data was in memstore so don't expect any changes + assertEquals(startBlockCount, cache.getBlockCount()); + assertEquals(startBlockHits, cache.getStats().getHitCount()); + assertEquals(startBlockMiss, cache.getStats().getMissCount()); + // flush the data + System.out.println("Flushing cache"); + region.flushcache(); + // expect one more block in cache, no change in hits/misses + long expectedBlockCount = startBlockCount + 1; + long expectedBlockHits = startBlockHits; + long expectedBlockMiss = startBlockMiss; + assertEquals(expectedBlockCount, cache.getBlockCount()); + assertEquals(expectedBlockHits, cache.getStats().getHitCount()); + assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); + // read the data and expect same blocks, one new hit, no misses + assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data)); + assertEquals(expectedBlockCount, cache.getBlockCount()); + assertEquals(++expectedBlockHits, cache.getStats().getHitCount()); + assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); + // insert a second column, read the row, no new blocks, one new hit + byte [] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER); + byte [] data2 = Bytes.add(data, data); + put = new Put(ROW); + put.add(FAMILY, QUALIFIER2, data2); + table.put(put); + Result r = table.get(new Get(ROW)); + assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data)); + assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2)); + assertEquals(expectedBlockCount, cache.getBlockCount()); + assertEquals(++expectedBlockHits, cache.getStats().getHitCount()); + assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); + // flush, one new block + System.out.println("Flushing cache"); + region.flushcache(); + assertEquals(++expectedBlockCount, cache.getBlockCount()); + assertEquals(expectedBlockHits, cache.getStats().getHitCount()); + assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); + // compact, net minus two blocks, two hits, no misses + System.out.println("Compacting"); + assertEquals(2, store.getNumberOfStoreFiles()); + store.triggerMajorCompaction(); + region.compactStores(); + waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max + assertEquals(1, store.getNumberOfStoreFiles()); + expectedBlockCount -= 2; // evicted two blocks, cached none + assertEquals(expectedBlockCount, cache.getBlockCount()); + expectedBlockHits += 2; + assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); + assertEquals(expectedBlockHits, cache.getStats().getHitCount()); + // read the row, this should be a cache miss because we don't cache data + // blocks on compaction + r = table.get(new Get(ROW)); + assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data)); + assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2)); + expectedBlockCount += 1; // cached one data block + assertEquals(expectedBlockCount, cache.getBlockCount()); + assertEquals(expectedBlockHits, cache.getStats().getHitCount()); + assertEquals(++expectedBlockMiss, cache.getStats().getMissCount()); + } + + private void waitForStoreFileCount(Store store, int count, int timeout) + throws InterruptedException { + long start = System.currentTimeMillis(); + while (start + timeout > System.currentTimeMillis() && + store.getNumberOfStoreFiles() != count) { + Thread.sleep(100); + } + System.out.println("start=" + start + ", now=" + + System.currentTimeMillis() + ", cur=" + store.getNumberOfStoreFiles()); + assertEquals(count, store.getNumberOfStoreFiles()); + } + + @Test + /** + * Tests the non cached version of getRegionLocation by moving a region. + */ + public void testNonCachedGetRegionLocation() throws Exception { + // Test Initialization. + String tableName = "testNonCachedGetRegionLocation"; + byte [] TABLE = Bytes.toBytes(tableName); + byte [] family1 = Bytes.toBytes("f1"); + byte [] family2 = Bytes.toBytes("f2"); + HTable table = TEST_UTIL.createTable(TABLE, new byte[][] {family1, family2}, 10); + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + Map regionsMap = table.getRegionLocations(); + assertEquals(1, regionsMap.size()); + HRegionInfo regionInfo = regionsMap.keySet().iterator().next(); + ServerName addrBefore = regionsMap.get(regionInfo); + // Verify region location before move. + HServerAddress addrCache = + table.getRegionLocation(regionInfo.getStartKey(), false).getServerAddress(); + HServerAddress addrNoCache = + table.getRegionLocation(regionInfo.getStartKey(), + true).getServerAddress(); + + assertEquals(addrBefore.getPort(), addrCache.getPort()); + assertEquals(addrBefore.getPort(), addrNoCache.getPort()); + + ServerName addrAfter = null; + // Now move the region to a different server. + for (int i = 0; i < SLAVES; i++) { + HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i); + ServerName addr = regionServer.getServerName(); + if (addr.getPort() != addrBefore.getPort()) { + admin.move(regionInfo.getEncodedNameAsBytes(), + Bytes.toBytes(addr.toString())); + // Wait for the region to move. + Thread.sleep(5000); + addrAfter = addr; + break; + } + } + + // Verify the region was moved. + addrCache = + table.getRegionLocation(regionInfo.getStartKey(), false).getServerAddress(); + addrNoCache = + table.getRegionLocation(regionInfo.getStartKey(), + true).getServerAddress(); + assertNotNull(addrAfter); + assertTrue(addrAfter.getPort() != addrCache.getPort()); + assertEquals(addrAfter.getPort(), addrNoCache.getPort()); + } + + @Test + /** + * Tests getRegionsInRange by creating some regions over which a range of + * keys spans; then changing the key range. + */ + public void testGetRegionsInRange() throws Exception { + // Test Initialization. + byte [] startKey = Bytes.toBytes("ddc"); + byte [] endKey = Bytes.toBytes("mmm"); + byte [] TABLE = Bytes.toBytes("testGetRegionsInRange"); + HTable table = TEST_UTIL.createTable(TABLE, new byte[][] {FAMILY}, 10); + int numOfRegions = TEST_UTIL.createMultiRegions(table, FAMILY); + assertEquals(25, numOfRegions); + + // Get the regions in this range + List regionsList = table.getRegionsInRange(startKey, + endKey); + assertEquals(10, regionsList.size()); + + // Change the start key + startKey = Bytes.toBytes("fff"); + regionsList = table.getRegionsInRange(startKey, endKey); + assertEquals(7, regionsList.size()); + + // Change the end key + endKey = Bytes.toBytes("nnn"); + regionsList = table.getRegionsInRange(startKey, endKey); + assertEquals(8, regionsList.size()); + + // Empty start key + regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW, endKey); + assertEquals(13, regionsList.size()); + + // Empty end key + regionsList = table.getRegionsInRange(startKey, HConstants.EMPTY_END_ROW); + assertEquals(20, regionsList.size()); + + // Both start and end keys empty + regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW, + HConstants.EMPTY_END_ROW); + assertEquals(25, regionsList.size()); + + // Change the end key to somewhere in the last block + endKey = Bytes.toBytes("yyz"); + regionsList = table.getRegionsInRange(startKey, endKey); + assertEquals(20, regionsList.size()); + + // Change the start key to somewhere in the first block + startKey = Bytes.toBytes("aac"); + regionsList = table.getRegionsInRange(startKey, endKey); + assertEquals(25, regionsList.size()); + + // Make start and end key the same + startKey = endKey = Bytes.toBytes("ccc"); + regionsList = table.getRegionsInRange(startKey, endKey); + assertEquals(1, regionsList.size()); + } + + @Test + public void testJira6912() throws Exception { + byte [] TABLE = Bytes.toBytes("testJira6912"); + HTable foo = TEST_UTIL.createTable(TABLE, new byte[][] {FAMILY}, 10); + + List puts = new ArrayList(); + for (int i=0;i !=100; i++){ + Put put = new Put(Bytes.toBytes(i)); + put.add(FAMILY, FAMILY, Bytes.toBytes(i)); + puts.add(put); + } + foo.put(puts); + // If i comment this out it works + TEST_UTIL.flush(); + + Scan scan = new Scan(); + scan.setStartRow(Bytes.toBytes(1)); + scan.setStopRow(Bytes.toBytes(3)); + scan.addColumn(FAMILY, FAMILY); + scan.setFilter(new RowFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.toBytes(1)))); + + ResultScanner scanner = foo.getScanner(scan); + Result[] bar = scanner.next(100); + assertEquals(1, bar.length); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java new file mode 100644 index 0000000..7dd60de --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java @@ -0,0 +1,260 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + +@Category(LargeTests.class) +public class TestFromClientSide3 { + final Log LOG = LogFactory.getLog(getClass()); + private final static HBaseTestingUtility TEST_UTIL + = new HBaseTestingUtility(); + private static byte[] ROW = Bytes.toBytes("testRow"); + private static byte[] FAMILY = Bytes.toBytes("testFamily"); + private static byte[] QUALIFIER = Bytes.toBytes("testQualifier"); + private static byte[] VALUE = Bytes.toBytes("testValue"); + private static Random random = new Random(); + private static int SLAVES = 3; + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean( + "hbase.online.schema.update.enable", true); + TEST_UTIL.startMiniCluster(SLAVES); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + // Nothing to do. + } + + /** + * @throws java.lang.Exception + */ + @After + public void tearDown() throws Exception { + // Nothing to do. + } + + private void randomCFPuts(HTable table, byte[] row, byte[] family, int nPuts) + throws Exception { + Put put = new Put(row); + for (int i = 0; i < nPuts; i++) { + byte[] qualifier = Bytes.toBytes(random.nextInt()); + byte[] value = Bytes.toBytes(random.nextInt()); + put.add(family, qualifier, value); + } + table.put(put); + } + + private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table, + byte[] row, byte[] family, int nFlushes, int nPuts) throws Exception { + + // connection needed for poll-wait + HConnection conn = HConnectionManager.getConnection(TEST_UTIL + .getConfiguration()); + HRegionLocation loc = table.getRegionLocation(row, true); + HRegionInterface server = conn.getHRegionConnection(loc.getHostname(), loc + .getPort()); + byte[] regName = loc.getRegionInfo().getRegionName(); + + for (int i = 0; i < nFlushes; i++) { + randomCFPuts(table, row, family, nPuts); + int sfCount = server.getStoreFileList(regName, FAMILY).size(); + + // TODO: replace this api with a synchronous flush after HBASE-2949 + admin.flush(table.getTableName()); + + // synchronously poll wait for a new storefile to appear (flush happened) + while (server.getStoreFileList(regName, FAMILY).size() == sfCount) { + Thread.sleep(40); + } + } + } + + // override the config settings at the CF level and ensure priority + @Test(timeout = 60000) + public void testAdvancedConfigOverride() throws Exception { + /* + * Overall idea: (1) create 3 store files and issue a compaction. config's + * compaction.min == 3, so should work. (2) Increase the compaction.min + * toggle in the HTD to 5 and modify table. If we use the HTD value instead + * of the default config value, adding 3 files and issuing a compaction + * SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2 + * and modify table. The CF schema should override the Table schema and now + * cause a minor compaction. + */ + TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3); + + String tableName = "testAdvancedConfigOverride"; + byte[] TABLE = Bytes.toBytes(tableName); + HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10); + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + HConnection connection = HConnectionManager.getConnection(TEST_UTIL + .getConfiguration()); + + // Create 3 store files. + byte[] row = Bytes.toBytes(random.nextInt()); + performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100); + + // Verify we have multiple store files. + HRegionLocation loc = hTable.getRegionLocation(row, true); + byte[] regionName = loc.getRegionInfo().getRegionName(); + HRegionInterface server = connection.getHRegionConnection( + loc.getHostname(), loc.getPort()); + assertTrue(server.getStoreFileList(regionName, FAMILY).size() > 1); + + // Issue a compaction request + admin.compact(TABLE); + + // poll wait for the compactions to happen + for (int i = 0; i < 10 * 1000 / 40; ++i) { + // The number of store files after compaction should be lesser. + loc = hTable.getRegionLocation(row, true); + if (!loc.getRegionInfo().isOffline()) { + regionName = loc.getRegionInfo().getRegionName(); + server = connection.getHRegionConnection(loc.getHostname(), loc + .getPort()); + if (server.getStoreFileList(regionName, FAMILY).size() <= 1) { + break; + } + } + Thread.sleep(40); + } + // verify the compactions took place and that we didn't just time out + assertTrue(server.getStoreFileList(regionName, FAMILY).size() <= 1); + + // change the compaction.min config option for this table to 5 + LOG.info("hbase.hstore.compaction.min should now be 5"); + HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor()); + htd.setValue("hbase.hstore.compaction.min", String.valueOf(5)); + admin.modifyTable(TABLE, htd); + Pair st; + while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) { + LOG.debug(st.getFirst() + " regions left to update"); + Thread.sleep(40); + } + LOG.info("alter status finished"); + + // Create 3 more store files. + performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10); + + // Issue a compaction request + admin.compact(TABLE); + + // This time, the compaction request should not happen + Thread.sleep(10 * 1000); + int sfCount = 0; + loc = hTable.getRegionLocation(row, true); + regionName = loc.getRegionInfo().getRegionName(); + server = connection.getHRegionConnection(loc.getHostname(), loc.getPort()); + sfCount = server.getStoreFileList(regionName, FAMILY).size(); + assertTrue(sfCount > 1); + + // change an individual CF's config option to 2 & online schema update + LOG.info("hbase.hstore.compaction.min should now be 2"); + HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); + hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2)); + htd.addFamily(hcd); + admin.modifyTable(TABLE, htd); + while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) { + LOG.debug(st.getFirst() + " regions left to update"); + Thread.sleep(40); + } + LOG.info("alter status finished"); + + // Issue a compaction request + admin.compact(TABLE); + + // poll wait for the compactions to happen + for (int i = 0; i < 10 * 1000 / 40; ++i) { + loc = hTable.getRegionLocation(row, true); + regionName = loc.getRegionInfo().getRegionName(); + try { + server = connection.getHRegionConnection(loc.getHostname(), loc + .getPort()); + if (server.getStoreFileList(regionName, FAMILY).size() < sfCount) { + break; + } + } catch (Exception e) { + LOG.debug("Waiting for region to come online: " + regionName); + } + Thread.sleep(40); + } + // verify the compaction took place and that we didn't just time out + assertTrue(server.getStoreFileList(regionName, FAMILY).size() < sfCount); + + // Finally, ensure that we can remove a custom config value after we made it + LOG.info("Removing CF config value"); + LOG.info("hbase.hstore.compaction.min should now be 5"); + hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); + hcd.setValue("hbase.hstore.compaction.min", null); + htd.addFamily(hcd); + admin.modifyTable(TABLE, htd); + while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) { + LOG.debug(st.getFirst() + " regions left to update"); + Thread.sleep(40); + } + LOG.info("alter status finished"); + assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue( + "hbase.hstore.compaction.min")); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideWithCoprocessor.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideWithCoprocessor.java new file mode 100644 index 0000000..7b313dc --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideWithCoprocessor.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; +import org.apache.hadoop.hbase.regionserver.NoOpScanPolicyObserver; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +/** + * Test all client operations with a coprocessor that + * just implements the default flush/compact/scan policy + */ +@Category(LargeTests.class) +public class TestFromClientSideWithCoprocessor extends TestFromClientSide { + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + MultiRowMutationEndpoint.class.getName(), NoOpScanPolicyObserver.class.getName()); + // We need more than one region server in this test + TEST_UTIL.startMiniCluster(SLAVES); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestGet.java b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java new file mode 100644 index 0000000..46c3ff3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestGet.java @@ -0,0 +1,175 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Base64; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.io.ByteStreams; + +// TODO: cover more test cases +@Category(SmallTests.class) +public class TestGet { + + private static final String WRITABLE_GET = + "AgD//////////wAAAAEBD3Rlc3QuTW9ja0ZpbHRlcgEAAAAAAAAAAH//////////AQAAAAAAAAAA"; + + private static final String MOCK_FILTER_JAR = + "UEsDBBQACAgIACmBi0IAAAAAAAAAAAAAAAAJAAQATUVUQS1JTkYv/soAAAMAUEsHCAAAAAACAAAA" + + "AAAAAFBLAwQUAAgICAApgYtCAAAAAAAAAAAAAAAAFAAAAE1FVEEtSU5GL01BTklGRVNULk1G803M" + + "y0xLLS7RDUstKs7Mz7NSMNQz4OVyLkpNLElN0XWqBAmY6xnEG1gqaPgXJSbnpCo45xcV5BcllgCV" + + "a/Jy8XIBAFBLBwgxyqRbQwAAAEQAAABQSwMECgAACAAAbICLQgAAAAAAAAAAAAAAAAUAAAB0ZXN0" + + "L1BLAwQUAAgICAAcgItCAAAAAAAAAAAAAAAAFQAAAHRlc3QvTW9ja0ZpbHRlci5jbGFzc41Qy07C" + + "QBS9A4VKBZGHoO7cgQvHmLjCuPBBQlJloWE/tCMdLZ1mOlV/y5WJCz/AjzLeDqCRYOIs7uuce87N" + + "fHy+vQPAEezakCNQ1TzR9Ep6D30Raq5ssAh0pZpQFjMv4DRgvpQxDcYs4fTOcOiMeoYTAsUTEQl9" + + "SiDf6Y4IWOfS5w7koVSGAhTRwBURv06nY65u2TjEjborPRaOmBJZPx9aOhAJgZq7dE+PgKM48/uC" + + "hz4SWh33nj0yKiS9YJoNojjVvczYuXz2eKyFjBIb6gQaC9pg+I2gDVOTQwRXiBAoPCmh8Zb2b49h" + + "qhcmzVUAet/IVHkcL8bt6s/xBxkb9gA/B7KXxwo/BaONHcVMMBf2X2HtBYscOBiLZliCdYzlGQFz" + + "BTOBDagiaxNrC7uakTk2m4guS1SMRGsGziWyqgFN47xlsH+K1f4UaxuxbcPf+QJQSwcI8UIYqlEB" + + "AABeAgAAUEsBAhQAFAAICAgAKYGLQgAAAAACAAAAAAAAAAkABAAAAAAAAAAAAAAAAAAAAE1FVEEt" + + "SU5GL/7KAABQSwECFAAUAAgICAApgYtCMcqkW0MAAABEAAAAFAAAAAAAAAAAAAAAAAA9AAAATUVU" + + "QS1JTkYvTUFOSUZFU1QuTUZQSwECCgAKAAAIAABsgItCAAAAAAAAAAAAAAAABQAAAAAAAAAAAAAA" + + "AADCAAAAdGVzdC9QSwECFAAUAAgICAAcgItC8UIYqlEBAABeAgAAFQAAAAAAAAAAAAAAAADlAAAA" + + "dGVzdC9Nb2NrRmlsdGVyLmNsYXNzUEsFBgAAAAAEAAQA8wAAAHkCAAAAAA=="; + + @Test + public void testAttributesSerialization() throws IOException { + Get get = new Get(); + get.setAttribute("attribute1", Bytes.toBytes("value1")); + get.setAttribute("attribute2", Bytes.toBytes("value2")); + get.setAttribute("attribute3", Bytes.toBytes("value3")); + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DataOutput out = new DataOutputStream(byteArrayOutputStream); + get.write(out); + + Get get2 = new Get(); + Assert.assertTrue(get2.getAttributesMap().isEmpty()); + + get2.readFields(new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))); + + Assert.assertNull(get2.getAttribute("absent")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), get2.getAttribute("attribute1"))); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), get2.getAttribute("attribute2"))); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value3"), get2.getAttribute("attribute3"))); + Assert.assertEquals(3, get2.getAttributesMap().size()); + } + + @Test + public void testGetAttributes() { + Get get = new Get(); + Assert.assertTrue(get.getAttributesMap().isEmpty()); + Assert.assertNull(get.getAttribute("absent")); + + get.setAttribute("absent", null); + Assert.assertTrue(get.getAttributesMap().isEmpty()); + Assert.assertNull(get.getAttribute("absent")); + + // adding attribute + get.setAttribute("attribute1", Bytes.toBytes("value1")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), get.getAttribute("attribute1"))); + Assert.assertEquals(1, get.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), get.getAttributesMap().get("attribute1"))); + + // overriding attribute value + get.setAttribute("attribute1", Bytes.toBytes("value12")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), get.getAttribute("attribute1"))); + Assert.assertEquals(1, get.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), get.getAttributesMap().get("attribute1"))); + + // adding another attribute + get.setAttribute("attribute2", Bytes.toBytes("value2")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), get.getAttribute("attribute2"))); + Assert.assertEquals(2, get.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), get.getAttributesMap().get("attribute2"))); + + // removing attribute + get.setAttribute("attribute2", null); + Assert.assertNull(get.getAttribute("attribute2")); + Assert.assertEquals(1, get.getAttributesMap().size()); + Assert.assertNull(get.getAttributesMap().get("attribute2")); + + // removing non-existed attribute + get.setAttribute("attribute2", null); + Assert.assertNull(get.getAttribute("attribute2")); + Assert.assertEquals(1, get.getAttributesMap().size()); + Assert.assertNull(get.getAttributesMap().get("attribute2")); + + // removing another attribute + get.setAttribute("attribute1", null); + Assert.assertNull(get.getAttribute("attribute1")); + Assert.assertTrue(get.getAttributesMap().isEmpty()); + Assert.assertNull(get.getAttributesMap().get("attribute1")); + } + + @Test + public void testDynamicFilter() throws Exception { + DataInput dis = ByteStreams.newDataInput(Base64.decode(WRITABLE_GET)); + Get get = new Get(); + try { + get.readFields(dis); + fail("Should not be able to load the filter class"); + } catch (RuntimeException re) { + String msg = re.getMessage(); + Assert.assertTrue(msg != null + && msg.contains("Can't find class test.MockFilter")); + } + + Configuration conf = HBaseConfiguration.create(); + String localPath = conf.get("hbase.local.dir") + + File.separator + "jars" + File.separator; + File jarFile = new File(localPath, "MockFilter.jar"); + jarFile.deleteOnExit(); + + FileOutputStream fos = new FileOutputStream(jarFile); + fos.write(Base64.decode(MOCK_FILTER_JAR)); + fos.close(); + + dis = ByteStreams.newDataInput(Base64.decode(WRITABLE_GET)); + get.readFields(dis); + Assert.assertEquals("test.MockFilter", + get.getFilter().getClass().getName()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java new file mode 100644 index 0000000..98a99e7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java @@ -0,0 +1,395 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation; +import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionKey; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * This class is for testing HCM features + */ +@Category(MediumTests.class) +public class TestHCM { + private static final Log LOG = LogFactory.getLog(TestHCM.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final byte[] TABLE_NAME = Bytes.toBytes("test"); + private static final byte[] TABLE_NAME1 = Bytes.toBytes("test1"); + private static final byte[] TABLE_NAME2 = Bytes.toBytes("test2"); + private static final byte[] FAM_NAM = Bytes.toBytes("f"); + private static final byte[] ROW = Bytes.toBytes("bbb"); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(1); + } + + @AfterClass public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws InterruptedException + * @throws IllegalAccessException + * @throws NoSuchFieldException + * @throws ZooKeeperConnectionException + * @throws IllegalArgumentException + * @throws SecurityException + * @see https://issues.apache.org/jira/browse/HBASE-2925 + */ + // Disabling. Of course this test will OOME using new Configuration each time + // St.Ack 20110428 + // @Test + public void testManyNewConnectionsDoesnotOOME() + throws SecurityException, IllegalArgumentException, + ZooKeeperConnectionException, NoSuchFieldException, IllegalAccessException, + InterruptedException { + createNewConfigurations(); + } + + private static Random _randy = new Random(); + + public static void createNewConfigurations() throws SecurityException, + IllegalArgumentException, NoSuchFieldException, + IllegalAccessException, InterruptedException, ZooKeeperConnectionException { + HConnection last = null; + for (int i = 0; i <= (HConnectionManager.MAX_CACHED_HBASE_INSTANCES * 2); i++) { + // set random key to differentiate the connection from previous ones + Configuration configuration = HBaseConfiguration.create(); + configuration.set("somekey", String.valueOf(_randy.nextInt())); + System.out.println("Hash Code: " + configuration.hashCode()); + HConnection connection = HConnectionManager.getConnection(configuration); + if (last != null) { + if (last == connection) { + System.out.println("!! Got same connection for once !!"); + } + } + // change the configuration once, and the cached connection is lost forever: + // the hashtable holding the cache won't be able to find its own keys + // to remove them, so the LRU strategy does not work. + configuration.set("someotherkey", String.valueOf(_randy.nextInt())); + last = connection; + LOG.info("Cache Size: " + getHConnectionManagerCacheSize()); + Thread.sleep(100); + } + Assert.assertEquals(1, + getHConnectionManagerCacheSize()); + } + + private static int getHConnectionManagerCacheSize(){ + return HConnectionTestingUtility.getConnectionCount(); + } + + @Test + public void abortingHConnectionRemovesItselfFromHCM() throws Exception { + // Save off current HConnections + Map oldHBaseInstances = + new HashMap(); + oldHBaseInstances.putAll(HConnectionManager.HBASE_INSTANCES); + + HConnectionManager.HBASE_INSTANCES.clear(); + + try { + HConnection connection = HConnectionManager.getConnection(TEST_UTIL.getConfiguration()); + connection.abort("test abortingHConnectionRemovesItselfFromHCM", new Exception( + "test abortingHConnectionRemovesItselfFromHCM")); + Assert.assertNotSame(connection, + HConnectionManager.getConnection(TEST_UTIL.getConfiguration())); + } finally { + // Put original HConnections back + HConnectionManager.HBASE_INSTANCES.clear(); + HConnectionManager.HBASE_INSTANCES.putAll(oldHBaseInstances); + } + } + + /** + * Test that when we delete a location using the first row of a region + * that we really delete it. + * @throws Exception + */ + @Test + public void testRegionCaching() throws Exception{ + HTable table = TEST_UTIL.createTable(TABLE_NAME, FAM_NAM); + TEST_UTIL.createMultiRegions(table, FAM_NAM); + Put put = new Put(ROW); + put.add(FAM_NAM, ROW, ROW); + table.put(put); + HConnectionManager.HConnectionImplementation conn = + (HConnectionManager.HConnectionImplementation)table.getConnection(); + assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW)); + conn.deleteCachedLocation(TABLE_NAME, ROW); + HRegionLocation rl = conn.getCachedLocation(TABLE_NAME, ROW); + assertNull("What is this location?? " + rl, rl); + table.close(); + } + + /** + * Test that Connection or Pool are not closed when managed externally + * @throws Exception + */ + @Test + public void testConnectionManagement() throws Exception{ + TEST_UTIL.createTable(TABLE_NAME1, FAM_NAM); + HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration()); + ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10, + 60, TimeUnit.SECONDS, + new SynchronousQueue(), + Threads.newDaemonThreadFactory("test-hcm-table")); + + HTable table = new HTable(TABLE_NAME1, conn, pool); + table.close(); + assertFalse(conn.isClosed()); + assertFalse(pool.isShutdown()); + table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME1, pool); + table.close(); + assertFalse(pool.isShutdown()); + conn.close(); + pool.shutdownNow(); + } + + /** + * Make sure that {@link Configuration} instances that are essentially the + * same map to the same {@link HConnection} instance. + */ + @Test + public void testConnectionSameness() throws Exception { + HConnection previousConnection = null; + for (int i = 0; i < 2; i++) { + // set random key to differentiate the connection from previous ones + Configuration configuration = TEST_UTIL.getConfiguration(); + configuration.set("some_key", String.valueOf(_randy.nextInt())); + LOG.info("The hash code of the current configuration is: " + + configuration.hashCode()); + HConnection currentConnection = HConnectionManager + .getConnection(configuration); + if (previousConnection != null) { + assertTrue( + "Did not get the same connection even though its key didn't change", + previousConnection == currentConnection); + } + previousConnection = currentConnection; + // change the configuration, so that it is no longer reachable from the + // client's perspective. However, since its part of the LRU doubly linked + // list, it will eventually get thrown out, at which time it should also + // close the corresponding {@link HConnection}. + configuration.set("other_key", String.valueOf(_randy.nextInt())); + } + } + + /** + * Makes sure that there is no leaking of + * {@link HConnectionManager.TableServers} in the {@link HConnectionManager} + * class. + */ + @Test + public void testConnectionUniqueness() throws Exception { + int zkmaxconnections = TEST_UTIL.getConfiguration(). + getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, + HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS); + // Test up to a max that is < the maximum number of zk connections. If we + // go above zk connections, we just fall into cycle where we are failing + // to set up a session and test runs for a long time. + int maxConnections = Math.min(zkmaxconnections - 1, 20); + List connections = new ArrayList(maxConnections); + HConnection previousConnection = null; + try { + for (int i = 0; i < maxConnections; i++) { + // set random key to differentiate the connection from previous ones + Configuration configuration = new Configuration(TEST_UTIL.getConfiguration()); + configuration.set("some_key", String.valueOf(_randy.nextInt())); + configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID, + String.valueOf(_randy.nextInt())); + LOG.info("The hash code of the current configuration is: " + + configuration.hashCode()); + HConnection currentConnection = + HConnectionManager.getConnection(configuration); + if (previousConnection != null) { + assertTrue("Got the same connection even though its key changed!", + previousConnection != currentConnection); + } + // change the configuration, so that it is no longer reachable from the + // client's perspective. However, since its part of the LRU doubly linked + // list, it will eventually get thrown out, at which time it should also + // close the corresponding {@link HConnection}. + configuration.set("other_key", String.valueOf(_randy.nextInt())); + + previousConnection = currentConnection; + LOG.info("The current HConnectionManager#HBASE_INSTANCES cache size is: " + + getHConnectionManagerCacheSize()); + Thread.sleep(50); + connections.add(currentConnection); + } + } finally { + for (HConnection c: connections) { + // Clean up connections made so we don't interfere w/ subsequent tests. + HConnectionManager.deleteConnection(c.getConfiguration()); + } + } + } + + @Test + public void testClosing() throws Exception { + Configuration configuration = + new Configuration(TEST_UTIL.getConfiguration()); + configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID, + String.valueOf(_randy.nextInt())); + + HConnection c1 = HConnectionManager.createConnection(configuration); + // We create two connections with the same key. + HConnection c2 = HConnectionManager.createConnection(configuration); + + HConnection c3 = HConnectionManager.getConnection(configuration); + HConnection c4 = HConnectionManager.getConnection(configuration); + assertTrue(c3 == c4); + + c1.close(); + assertTrue(c1.isClosed()); + assertFalse(c2.isClosed()); + assertFalse(c3.isClosed()); + + c3.close(); + // still a reference left + assertFalse(c3.isClosed()); + c3.close(); + assertTrue(c3.isClosed()); + // c3 was removed from the cache + HConnection c5 = HConnectionManager.getConnection(configuration); + assertTrue(c5 != c3); + + assertFalse(c2.isClosed()); + c2.close(); + assertTrue(c2.isClosed()); + c5.close(); + assertTrue(c5.isClosed()); + } + + /** + * Trivial test to verify that nobody messes with + * {@link HConnectionManager#createConnection(Configuration)} + */ + @Test + public void testCreateConnection() throws Exception { + Configuration configuration = TEST_UTIL.getConfiguration(); + HConnection c1 = HConnectionManager.createConnection(configuration); + HConnection c2 = HConnectionManager.createConnection(configuration); + // created from the same configuration, yet they are different + assertTrue(c1 != c2); + assertTrue(c1.getConfiguration() == c2.getConfiguration()); + // make sure these were not cached + HConnection c3 = HConnectionManager.getConnection(configuration); + assertTrue(c1 != c3); + assertTrue(c2 != c3); + } + + /** + * Tests that a closed connection does not have a live zookeeper + * @throws Exception + */ + @Test + public void testDeleteForZKConnLeak() throws Exception { + TEST_UTIL.createTable(TABLE_NAME2, FAM_NAM); + final Configuration config = TEST_UTIL.getConfiguration(); + + ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10, + 5, TimeUnit.SECONDS, + new SynchronousQueue(), + Threads.newDaemonThreadFactory("test-hcm-delete")); + + pool.submit(new Runnable() { + @Override + public void run() { + while (!Thread.interrupted()) { + try { + HConnection conn = HConnectionManager.getConnection(config); + HConnectionManager.deleteStaleConnection(conn); + } catch (Exception e) { + } + } + } + }); + + // use connection multiple times + for (int i = 0; i < 10; i++) { + HConnection c1 = null; + try { + c1 = HConnectionManager.getConnection(config); + HTable table = new HTable(TABLE_NAME2, c1, pool); + table.close(); + } catch (Exception e) { + } finally { + if (c1 != null) { + if (c1.isClosed()) { + // cannot use getZooKeeper as method instantiates watcher if null + Field zkwField = c1.getClass().getDeclaredField("zooKeeper"); + zkwField.setAccessible(true); + Object watcher = zkwField.get(c1); + + if (watcher != null) { + if (((ZooKeeperWatcher)watcher).getRecoverableZooKeeper().getState().isAlive()) { + pool.shutdownNow(); + fail("Live zookeeper in closed connection"); + } + } + } + c1.close(); + } + } + } + + pool.shutdownNow(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHConnection.java b/src/test/java/org/apache/hadoop/hbase/client/TestHConnection.java new file mode 100644 index 0000000..470616e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestHConnection.java @@ -0,0 +1,163 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.MultithreadedTestUtil; +import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Various tests of using HConnection + */ +@Category(MediumTests.class) +public class TestHConnection { + final Log LOG = LogFactory.getLog(getClass()); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final long MILLIS_TO_WAIT_FOR_RACE = 1000; + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Thread that periodically aborts the connection + */ + class AbortThread extends RepeatingTestThread { + final HConnection connection; + + public AbortThread(MultithreadedTestUtil.TestContext ctx, HConnection connection) { + super(ctx); + this.connection = connection; + } + + @Override + public void doAnAction() throws IOException { + connection.abort("test session expired", new KeeperException.SessionExpiredException()); + } + }; + + class HConnectionRaceTester extends HConnectionImplementation { + public HConnectionRaceTester(Configuration configuration, boolean managed) throws ZooKeeperConnectionException { + super(configuration, managed); + } + + /** + * Sleep after calling getZookeeperWatcher to attempt to trigger a race condition. + * If the HConnection retrieves the ZooKeeperWatcher but does not cache the value, + * by the time the new watcher is used, it could be null if the connection was aborted. + * This sleep will increase the time between when the watcher is retrieved and when it is used. + */ + public ZooKeeperWatcher getZooKeeperWatcher() throws ZooKeeperConnectionException { + ZooKeeperWatcher zkw = super.getZooKeeperWatcher(); + try { + Thread.sleep(10); + } catch (InterruptedException ie) { + // Ignore + } + return zkw; + } + } + + /** + * Test that a connection that is aborted while calling isTableDisabled doesn't NPE + */ + @Test + public void testTableDisabledRace() throws Exception { + final HConnection connection = new HConnectionRaceTester(TEST_UTIL.getConfiguration(), true); + MultithreadedTestUtil.TestContext ctx = + new MultithreadedTestUtil.TestContext(TEST_UTIL.getConfiguration()); + RepeatingTestThread disabledChecker = new RepeatingTestThread(ctx) { + @Override + public void doAnAction() throws IOException { + try { + connection.isTableDisabled(Bytes.toBytes("tableToCheck")); + } catch (IOException ioe) { + // Ignore. ZK can legitimately fail, only care if we get a NullPointerException + } + } + }; + AbortThread abortThread = new AbortThread(ctx, connection); + + ctx.addThread(disabledChecker); + ctx.addThread(abortThread); + ctx.startThreads(); + ctx.waitFor(MILLIS_TO_WAIT_FOR_RACE); + ctx.stop(); + } + + /** + * Test that a connection that is aborted while calling getCurrentNrNRS doesn't NPE + */ + @Test + public void testGetCurrentNrHRSRace() throws Exception { + final HConnection connection = new HConnectionRaceTester(TEST_UTIL.getConfiguration(), true); + MultithreadedTestUtil.TestContext ctx = + new MultithreadedTestUtil.TestContext(TEST_UTIL.getConfiguration()); + RepeatingTestThread getCurrentNrHRSCaller = new RepeatingTestThread(ctx) { + @Override + public void doAnAction() throws IOException { + try { + connection.getCurrentNrHRS(); + } catch (IOException ioe) { + // Ignore. ZK can legitimately fail, only care if we get a NullPointerException + } + } + }; + AbortThread abortThread = new AbortThread(ctx, connection); + + ctx.addThread(getCurrentNrHRSCaller); + ctx.addThread(abortThread); + ctx.startThreads(); + ctx.waitFor(MILLIS_TO_WAIT_FOR_RACE); + ctx.stop(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java b/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java new file mode 100644 index 0000000..74f10e5 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestHTablePool.java @@ -0,0 +1,342 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; + +import junit.framework.Assert; +import junit.framework.TestCase; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.PoolMap.PoolType; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * Tests HTablePool. + */ +@RunWith(Suite.class) +@Suite.SuiteClasses({TestHTablePool.TestHTableReusablePool.class, TestHTablePool.TestHTableThreadLocalPool.class}) +@Category(MediumTests.class) +public class TestHTablePool { + private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static byte[] TABLENAME = Bytes.toBytes("TestHTablePool"); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(1); + TEST_UTIL.createTable(TABLENAME, HConstants.CATALOG_FAMILY); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + public abstract static class TestHTablePoolType extends TestCase { + protected abstract PoolType getPoolType(); + + @Test + public void testTableWithStringName() throws Exception { + HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(), + Integer.MAX_VALUE, getPoolType()); + String tableName = Bytes.toString(TABLENAME); + + // Request a table from an empty pool + HTableInterface table = pool.getTable(tableName); + Assert.assertNotNull(table); + + // Close table (returns table to the pool) + table.close(); + + // Request a table of the same name + HTableInterface sameTable = pool.getTable(tableName); + Assert.assertSame( + ((HTablePool.PooledHTable) table).getWrappedTable(), + ((HTablePool.PooledHTable) sameTable).getWrappedTable()); + } + + @Test + public void testTableWithByteArrayName() throws IOException { + HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(), + Integer.MAX_VALUE, getPoolType()); + + // Request a table from an empty pool + HTableInterface table = pool.getTable(TABLENAME); + Assert.assertNotNull(table); + + // Close table (returns table to the pool) + table.close(); + + // Request a table of the same name + HTableInterface sameTable = pool.getTable(TABLENAME); + Assert.assertSame( + ((HTablePool.PooledHTable) table).getWrappedTable(), + ((HTablePool.PooledHTable) sameTable).getWrappedTable()); + } + + @Test + public void testTablesWithDifferentNames() throws IOException { + HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(), + Integer.MAX_VALUE, getPoolType()); + // We add the class to the table name as the HBase cluster is reused + // during the tests: this gives naming unicity. + byte[] otherTable = Bytes.toBytes( + "OtherTable_" + getClass().getSimpleName() + ); + TEST_UTIL.createTable(otherTable, HConstants.CATALOG_FAMILY); + + // Request a table from an empty pool + HTableInterface table1 = pool.getTable(TABLENAME); + HTableInterface table2 = pool.getTable(otherTable); + Assert.assertNotNull(table2); + + // Close tables (returns tables to the pool) + table1.close(); + table2.close(); + + // Request tables of the same names + HTableInterface sameTable1 = pool.getTable(TABLENAME); + HTableInterface sameTable2 = pool.getTable(otherTable); + Assert.assertSame( + ((HTablePool.PooledHTable) table1).getWrappedTable(), + ((HTablePool.PooledHTable) sameTable1).getWrappedTable()); + Assert.assertSame( + ((HTablePool.PooledHTable) table2).getWrappedTable(), + ((HTablePool.PooledHTable) sameTable2).getWrappedTable()); + } + @Test + public void testProxyImplementationReturned() { + HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(), + Integer.MAX_VALUE); + String tableName = Bytes.toString(TABLENAME);// Request a table from + // an + // empty pool + HTableInterface table = pool.getTable(tableName); + + // Test if proxy implementation is returned + Assert.assertTrue(table instanceof HTablePool.PooledHTable); + } + + @Test + public void testDeprecatedUsagePattern() throws IOException { + HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(), + Integer.MAX_VALUE); + String tableName = Bytes.toString(TABLENAME);// Request a table from + // an + // empty pool + + // get table will return proxy implementation + HTableInterface table = pool.getTable(tableName); + + // put back the proxy implementation instead of closing it + pool.putTable(table); + + // Request a table of the same name + HTableInterface sameTable = pool.getTable(tableName); + + // test no proxy over proxy created + Assert.assertSame(((HTablePool.PooledHTable) table).getWrappedTable(), + ((HTablePool.PooledHTable) sameTable).getWrappedTable()); + } + + @Test + public void testReturnDifferentTable() throws IOException { + HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(), + Integer.MAX_VALUE); + String tableName = Bytes.toString(TABLENAME);// Request a table from + // an + // empty pool + + // get table will return proxy implementation + final HTableInterface table = pool.getTable(tableName); + HTableInterface alienTable = new HTable(TEST_UTIL.getConfiguration(), + TABLENAME) { + // implementation doesn't matter as long the table is not from + // pool + }; + try { + // put the wrong table in pool + pool.putTable(alienTable); + Assert.fail("alien table accepted in pool"); + } catch (IllegalArgumentException e) { + Assert.assertTrue("alien table rejected", true); + } + } + } + + @Category(MediumTests.class) + public static class TestHTableReusablePool extends TestHTablePoolType { + @Override + protected PoolType getPoolType() { + return PoolType.Reusable; + } + + @Test + public void testTableWithMaxSize() throws Exception { + HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(), 2, + getPoolType()); + + // Request tables from an empty pool + HTableInterface table1 = pool.getTable(TABLENAME); + HTableInterface table2 = pool.getTable(TABLENAME); + HTableInterface table3 = pool.getTable(TABLENAME); + + // Close tables (returns tables to the pool) + table1.close(); + table2.close(); + // The pool should reject this one since it is already full + table3.close(); + + // Request tables of the same name + HTableInterface sameTable1 = pool.getTable(TABLENAME); + HTableInterface sameTable2 = pool.getTable(TABLENAME); + HTableInterface sameTable3 = pool.getTable(TABLENAME); + Assert.assertSame( + ((HTablePool.PooledHTable) table1).getWrappedTable(), + ((HTablePool.PooledHTable) sameTable1).getWrappedTable()); + Assert.assertSame( + ((HTablePool.PooledHTable) table2).getWrappedTable(), + ((HTablePool.PooledHTable) sameTable2).getWrappedTable()); + Assert.assertNotSame( + ((HTablePool.PooledHTable) table3).getWrappedTable(), + ((HTablePool.PooledHTable) sameTable3).getWrappedTable()); + } + + @Test + public void testCloseTablePool() throws IOException { + HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(), 4, + getPoolType()); + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + + if (admin.tableExists(TABLENAME)) { + admin.disableTable(TABLENAME); + admin.deleteTable(TABLENAME); + } + + HTableDescriptor tableDescriptor = new HTableDescriptor(TABLENAME); + tableDescriptor.addFamily(new HColumnDescriptor("randomFamily")); + admin.createTable(tableDescriptor); + + // Request tables from an empty pool + HTableInterface[] tables = new HTableInterface[4]; + for (int i = 0; i < 4; ++i) { + tables[i] = pool.getTable(TABLENAME); + } + + pool.closeTablePool(TABLENAME); + + for (int i = 0; i < 4; ++i) { + tables[i].close(); + } + + Assert.assertEquals(4, + pool.getCurrentPoolSize(Bytes.toString(TABLENAME))); + + pool.closeTablePool(TABLENAME); + + Assert.assertEquals(0, + pool.getCurrentPoolSize(Bytes.toString(TABLENAME))); + } + } + + @Category(MediumTests.class) + public static class TestHTableThreadLocalPool extends TestHTablePoolType { + @Override + protected PoolType getPoolType() { + return PoolType.ThreadLocal; + } + + @Test + public void testTableWithMaxSize() throws Exception { + HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(), 2, + getPoolType()); + + // Request tables from an empty pool + HTableInterface table1 = pool.getTable(TABLENAME); + HTableInterface table2 = pool.getTable(TABLENAME); + HTableInterface table3 = pool.getTable(TABLENAME); + + // Close tables (returns tables to the pool) + table1.close(); + table2.close(); + // The pool should not reject this one since the number of threads + // <= 2 + table3.close(); + + // Request tables of the same name + HTableInterface sameTable1 = pool.getTable(TABLENAME); + HTableInterface sameTable2 = pool.getTable(TABLENAME); + HTableInterface sameTable3 = pool.getTable(TABLENAME); + Assert.assertSame( + ((HTablePool.PooledHTable) table3).getWrappedTable(), + ((HTablePool.PooledHTable) sameTable1).getWrappedTable()); + Assert.assertSame( + ((HTablePool.PooledHTable) table3).getWrappedTable(), + ((HTablePool.PooledHTable) sameTable2).getWrappedTable()); + Assert.assertSame( + ((HTablePool.PooledHTable) table3).getWrappedTable(), + ((HTablePool.PooledHTable) sameTable3).getWrappedTable()); + } + + @Test + public void testCloseTablePool() throws IOException { + HTablePool pool = new HTablePool(TEST_UTIL.getConfiguration(), 4, + getPoolType()); + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + + if (admin.tableExists(TABLENAME)) { + admin.disableTable(TABLENAME); + admin.deleteTable(TABLENAME); + } + + HTableDescriptor tableDescriptor = new HTableDescriptor(TABLENAME); + tableDescriptor.addFamily(new HColumnDescriptor("randomFamily")); + admin.createTable(tableDescriptor); + + // Request tables from an empty pool + HTableInterface[] tables = new HTableInterface[4]; + for (int i = 0; i < 4; ++i) { + tables[i] = pool.getTable(TABLENAME); + } + + pool.closeTablePool(TABLENAME); + + for (int i = 0; i < 4; ++i) { + tables[i].close(); + } + + Assert.assertEquals(1, + pool.getCurrentPoolSize(Bytes.toString(TABLENAME))); + + pool.closeTablePool(TABLENAME); + + Assert.assertEquals(0, + pool.getCurrentPoolSize(Bytes.toString(TABLENAME))); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHTableUtil.java b/src/test/java/org/apache/hadoop/hbase/client/TestHTableUtil.java new file mode 100644 index 0000000..24f878e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestHTableUtil.java @@ -0,0 +1,135 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * This class provides tests for the {@link HTableUtil} class + * + */ +@Category(MediumTests.class) +public class TestHTableUtil { + final Log LOG = LogFactory.getLog(getClass()); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static byte [] ROW = Bytes.toBytes("testRow"); + private static byte [] FAMILY = Bytes.toBytes("testFamily"); + private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); + private static byte [] VALUE = Bytes.toBytes("testValue"); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * + * @throws Exception + */ + @Test + public void testBucketPut() throws Exception { + byte [] TABLE = Bytes.toBytes("testBucketPut"); + HTable ht = TEST_UTIL.createTable(TABLE, FAMILY); + ht.setAutoFlush( false ); + + List puts = new ArrayList(); + puts.add( createPut("row1") ); + puts.add( createPut("row2") ); + puts.add( createPut("row3") ); + puts.add( createPut("row4") ); + + HTableUtil.bucketRsPut( ht, puts ); + + Scan scan = new Scan(); + scan.addColumn(FAMILY, QUALIFIER); + int count = 0; + for(Result result : ht.getScanner(scan)) { + count++; + } + LOG.info("bucket put count=" + count); + assertEquals(count, puts.size()); + ht.close(); + } + + private Put createPut(String row) { + Put put = new Put( Bytes.toBytes(row)); + put.add(FAMILY, QUALIFIER, VALUE); + return put; + } + + /** + * + * @throws Exception + */ + @Test + public void testBucketBatch() throws Exception { + byte [] TABLE = Bytes.toBytes("testBucketBatch"); + HTable ht = TEST_UTIL.createTable(TABLE, FAMILY); + + List rows = new ArrayList(); + rows.add( createPut("row1") ); + rows.add( createPut("row2") ); + rows.add( createPut("row3") ); + rows.add( createPut("row4") ); + + HTableUtil.bucketRsBatch( ht, rows ); + + Scan scan = new Scan(); + scan.addColumn(FAMILY, QUALIFIER); + + int count = 0; + for(Result result : ht.getScanner(scan)) { + count++; + } + LOG.info("bucket batch count=" + count); + assertEquals(count, rows.size()); + ht.close(); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java new file mode 100644 index 0000000..6eff27c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java @@ -0,0 +1,367 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +import junit.framework.Assert; +import junit.framework.AssertionFailedError; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.FsShell; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.migration.HRegionInfo090x; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test migration that removes HTableDescriptor from HRegionInfo moving the + * meta version from no version to {@link MetaReader#META_VERSION}. + */ +@Category(MediumTests.class) +public class TestMetaMigrationRemovingHTD { + static final Log LOG = LogFactory.getLog(TestMetaMigrationRemovingHTD.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static String TESTTABLE = "TestTable"; + private final static int ROWCOUNT = 100; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Start up our mini cluster on top of an 0.90 root.dir that has data from + // a 0.90 hbase run -- it has a table with 100 rows in it -- and see if + // we can migrate from 0.90. + TEST_UTIL.startMiniZKCluster(); + TEST_UTIL.startMiniDFSCluster(1); + Path testdir = TEST_UTIL.getDataTestDir("TestMetaMigrationRemovingHTD"); + // Untar our test dir. + File untar = untar(new File(testdir.toString())); + // Now copy the untar up into hdfs so when we start hbase, we'll run from it. + Configuration conf = TEST_UTIL.getConfiguration(); + FsShell shell = new FsShell(conf); + FileSystem fs = FileSystem.get(conf); + // find where hbase will root itself, so we can copy filesystem there + Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath(); + if (!fs.isDirectory(hbaseRootDir.getParent())) { + // mkdir at first + fs.mkdirs(hbaseRootDir.getParent()); + } + doFsCommand(shell, + new String [] {"-put", untar.toURI().toString(), hbaseRootDir.toString()}); + // See whats in minihdfs. + doFsCommand(shell, new String [] {"-lsr", "/"}); + TEST_UTIL.startMiniHBaseCluster(1, 1); + // Assert we are running against the copied-up filesystem. The copied-up + // rootdir should have had a table named 'TestTable' in it. Assert it + // present. + HTable t = new HTable(TEST_UTIL.getConfiguration(), TESTTABLE); + ResultScanner scanner = t.getScanner(new Scan()); + int count = 0; + while (scanner.next() != null) { + count++; + } + // Assert that we find all 100 rows that are in the data we loaded. If + // so then we must have migrated it from 0.90 to 0.92. + Assert.assertEquals(ROWCOUNT, count); + scanner.close(); + t.close(); + } + + private static File untar(final File testdir) throws IOException { + // Find the src data under src/test/data + final String datafile = "hbase-4388-root.dir"; + String srcTarFile = + System.getProperty("project.build.testSourceDirectory", "src/test") + + File.separator + "data" + File.separator + datafile + ".tgz"; + File homedir = new File(testdir.toString()); + File tgtUntarDir = new File(homedir, datafile); + if (tgtUntarDir.exists()) { + if (!FileUtil.fullyDelete(tgtUntarDir)) { + throw new IOException("Failed delete of " + tgtUntarDir.toString()); + } + } + LOG.info("Untarring " + srcTarFile + " into " + homedir.toString()); + FileUtil.unTar(new File(srcTarFile), homedir); + Assert.assertTrue(tgtUntarDir.exists()); + return tgtUntarDir; + } + + private static void doFsCommand(final FsShell shell, final String [] args) + throws Exception { + // Run the 'put' command. + int errcode = shell.run(args); + if (errcode != 0) throw new IOException("Failed put; errcode=" + errcode); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testMetaUpdatedFlagInROOT() throws Exception { + boolean metaUpdated = MetaMigrationRemovingHTD. + isMetaHRIUpdated(TEST_UTIL.getMiniHBaseCluster().getMaster()); + assertEquals(true, metaUpdated); + } + + @Test + public void testMetaMigration() throws Exception { + LOG.info("Starting testMetaWithLegacyHRI"); + final byte [] FAMILY = Bytes.toBytes("family"); + HTableDescriptor htd = new HTableDescriptor("testMetaMigration"); + HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); + htd.addFamily(hcd); + Configuration conf = TEST_UTIL.getConfiguration(); + createMultiRegionsWithLegacyHRI(conf, htd, FAMILY, + new byte[][]{ + HConstants.EMPTY_START_ROW, + Bytes.toBytes("region_a"), + Bytes.toBytes("region_b")}); + CatalogTracker ct = + TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker(); + // Erase the current version of root meta for this test. + undoVersionInMeta(); + MetaReader.fullScanMetaAndPrint(ct); + LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI"); + + Set htds = + MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo( + TEST_UTIL.getHBaseCluster().getMaster()); + MetaReader.fullScanMetaAndPrint(ct); + // Should be one entry only and it should be for the table we just added. + assertEquals(1, htds.size()); + assertTrue(htds.contains(htd)); + // Assert that the flag in ROOT is updated to reflect the correct status + boolean metaUpdated = + MetaMigrationRemovingHTD.isMetaHRIUpdated( + TEST_UTIL.getMiniHBaseCluster().getMaster()); + assertEquals(true, metaUpdated); + } + + /** + * This test assumes a master crash/failure during the meta migration process + * and attempts to continue the meta migration process when a new master takes over. + * When a master dies during the meta migration we will have some rows of + * META.CatalogFamily updated with new HRI, (i.e HRI with out HTD) and some + * still hanging with legacy HRI. (i.e HRI with HTD). When the backup master/ or + * fresh start of master attempts the migration it will encouter some rows of META + * already updated with new HRI and some still legacy. This test will simulate this + * scenario and validates that the migration process can safely skip the updated + * rows and migrate any pending rows at startup. + * @throws Exception + */ + @Test + public void testMasterCrashDuringMetaMigration() throws Exception { + final byte[] FAMILY = Bytes.toBytes("family"); + HTableDescriptor htd = new HTableDescriptor("testMasterCrashDuringMetaMigration"); + HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); + htd.addFamily(hcd); + Configuration conf = TEST_UTIL.getConfiguration(); + // Create 10 New regions. + createMultiRegionsWithNewHRI(conf, htd, FAMILY, 10); + // Create 10 Legacy regions. + createMultiRegionsWithLegacyHRI(conf, htd, FAMILY, 10); + CatalogTracker ct = + TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker(); + // Erase the current version of root meta for this test. + undoVersionInMeta(); + MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(ct); + //MetaReader.fullScanMetaAndPrint(ct); + LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI"); + + Set htds = + MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo( + TEST_UTIL.getHBaseCluster().getMaster()); + assertEquals(1, htds.size()); + assertTrue(htds.contains(htd)); + // Assert that the flag in ROOT is updated to reflect the correct status + boolean metaUpdated = MetaMigrationRemovingHTD. + isMetaHRIUpdated(TEST_UTIL.getMiniHBaseCluster().getMaster()); + assertEquals(true, metaUpdated); + LOG.info("END testMetaWithLegacyHRI"); + } + + private void undoVersionInMeta() throws IOException { + Delete d = new Delete(HRegionInfo.ROOT_REGIONINFO.getRegionName()); + // Erase the current version of root meta for this test. + d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER); + HTable rootTable = + new HTable(TEST_UTIL.getConfiguration(), HConstants.ROOT_TABLE_NAME); + try { + rootTable.delete(d); + } finally { + rootTable.close(); + } + } + + public static void assertEquals(int expected, int actual) { + if (expected != actual) { + throw new AssertionFailedError("expected:<" + + expected + "> but was:<" + + actual + ">"); + } + } + + public static void assertEquals(boolean expected, boolean actual) { + if (expected != actual) { + throw new AssertionFailedError("expected:<" + + expected + "> but was:<" + + actual + ">"); + } + } + + + /** + * @param c + * @param htd + * @param family + * @param numRegions + * @return + * @throws IOException + * @deprecated Just for testing migration of meta from 0.90 to 0.92... will be + * removed thereafter + */ + public int createMultiRegionsWithLegacyHRI(final Configuration c, + final HTableDescriptor htd, final byte [] family, int numRegions) + throws IOException { + if (numRegions < 3) throw new IOException("Must create at least 3 regions"); + byte [] startKey = Bytes.toBytes("aaaaa"); + byte [] endKey = Bytes.toBytes("zzzzz"); + byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3); + byte [][] regionStartKeys = new byte[splitKeys.length+1][]; + for (int i=0;i newRegions + = new ArrayList(startKeys.length); + int count = 0; + for (int i = 0; i < startKeys.length; i++) { + int j = (i + 1) % startKeys.length; + HRegionInfo090x hri = new HRegionInfo090x(htd, + startKeys[i], startKeys[j]); + Put put = new Put(hri.getRegionName()); + put.setWriteToWAL(false); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + meta.put(put); + LOG.info("createMultiRegions: PUT inserted " + hri.toString()); + + newRegions.add(hri); + count++; + } + meta.close(); + return count; + } + + int createMultiRegionsWithNewHRI(final Configuration c, + final HTableDescriptor htd, final byte [] family, int numRegions) + throws IOException { + if (numRegions < 3) throw new IOException("Must create at least 3 regions"); + byte [] startKey = Bytes.toBytes("aaaaa"); + byte [] endKey = Bytes.toBytes("zzzzz"); + byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3); + byte [][] regionStartKeys = new byte[splitKeys.length+1][]; + for (int i=0;i newRegions + = new ArrayList(startKeys.length); + int count = 0; + for (int i = 0; i < startKeys.length; i++) { + int j = (i + 1) % startKeys.length; + HRegionInfo hri = new HRegionInfo(htd.getName(), + startKeys[i], startKeys[j]); + Put put = new Put(hri.getRegionName()); + put.setWriteToWAL(false); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + meta.put(put); + LOG.info("createMultiRegions: PUT inserted " + hri.toString()); + + newRegions.add(hri); + count++; + } + meta.close(); + return count; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java b/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java new file mode 100644 index 0000000..914d708 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java @@ -0,0 +1,250 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.math.BigDecimal; +import java.util.List; +import java.util.NavigableMap; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.catalog.MetaEditor; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.StoppableImplementation; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.util.StringUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +@Category(MediumTests.class) +public class TestMetaScanner { + final Log LOG = LogFactory.getLog(getClass()); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + public void setUp() throws Exception { + TEST_UTIL.startMiniCluster(1); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testMetaScanner() throws Exception { + LOG.info("Starting testMetaScanner"); + setUp(); + final byte[] TABLENAME = Bytes.toBytes("testMetaScanner"); + final byte[] FAMILY = Bytes.toBytes("family"); + TEST_UTIL.createTable(TABLENAME, FAMILY); + Configuration conf = TEST_UTIL.getConfiguration(); + HTable table = new HTable(conf, TABLENAME); + TEST_UTIL.createMultiRegions(conf, table, FAMILY, + new byte[][]{ + HConstants.EMPTY_START_ROW, + Bytes.toBytes("region_a"), + Bytes.toBytes("region_b")}); + // Make sure all the regions are deployed + TEST_UTIL.countRows(table); + + MetaScanner.MetaScannerVisitor visitor = + mock(MetaScanner.MetaScannerVisitor.class); + doReturn(true).when(visitor).processRow((Result)anyObject()); + + // Scanning the entire table should give us three rows + MetaScanner.metaScan(conf, visitor, TABLENAME); + verify(visitor, times(3)).processRow((Result)anyObject()); + + // Scanning the table with a specified empty start row should also + // give us three META rows + reset(visitor); + doReturn(true).when(visitor).processRow((Result)anyObject()); + MetaScanner.metaScan(conf, visitor, TABLENAME, HConstants.EMPTY_BYTE_ARRAY, 1000); + verify(visitor, times(3)).processRow((Result)anyObject()); + + // Scanning the table starting in the middle should give us two rows: + // region_a and region_b + reset(visitor); + doReturn(true).when(visitor).processRow((Result)anyObject()); + MetaScanner.metaScan(conf, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1000); + verify(visitor, times(2)).processRow((Result)anyObject()); + + // Scanning with a limit of 1 should only give us one row + reset(visitor); + doReturn(true).when(visitor).processRow((Result)anyObject()); + MetaScanner.metaScan(conf, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1); + verify(visitor, times(1)).processRow((Result)anyObject()); + table.close(); + } + + @Test + public void testConcurrentMetaScannerAndCatalogJanitor() throws Throwable { + /* TEST PLAN: start with only one region in a table. Have a splitter + * thread and metascanner threads that continously scan the meta table for regions. + * CatalogJanitor from master will run frequently to clean things up + */ + TEST_UTIL.getConfiguration().setLong("hbase.catalogjanitor.interval", 500); + setUp(); + + final long runtime = 30 * 1000; //30 sec + LOG.info("Starting testConcurrentMetaScannerAndCatalogJanitor"); + final byte[] TABLENAME = Bytes.toBytes("testConcurrentMetaScannerAndCatalogJanitor"); + final byte[] FAMILY = Bytes.toBytes("family"); + TEST_UTIL.createTable(TABLENAME, FAMILY); + final CatalogTracker catalogTracker = mock(CatalogTracker.class); + when(catalogTracker.getConnection()).thenReturn(TEST_UTIL.getHBaseAdmin().getConnection()); + + class RegionMetaSplitter extends StoppableImplementation implements Runnable { + Random random = new Random(); + Throwable ex = null; + @Override + public void run() { + while (!isStopped()) { + try { + List regions = MetaScanner.listAllRegions( + TEST_UTIL.getConfiguration(), false); + + //select a random region + HRegionInfo parent = regions.get(random.nextInt(regions.size())); + if (parent == null || !Bytes.equals(TABLENAME, parent.getTableName())) { + continue; + } + + long startKey = 0, endKey = Long.MAX_VALUE; + byte[] start = parent.getStartKey(); + byte[] end = parent.getEndKey(); + if (!Bytes.equals(HConstants.EMPTY_START_ROW, parent.getStartKey())) { + startKey = Bytes.toLong(parent.getStartKey()); + } + if (!Bytes.equals(HConstants.EMPTY_END_ROW, parent.getEndKey())) { + endKey = Bytes.toLong(parent.getEndKey()); + } + if (startKey == endKey) { + continue; + } + + long midKey = BigDecimal.valueOf(startKey).add(BigDecimal.valueOf(endKey)) + .divideToIntegralValue(BigDecimal.valueOf(2)).longValue(); + + HRegionInfo splita = new HRegionInfo(TABLENAME, + start, + Bytes.toBytes(midKey)); + HRegionInfo splitb = new HRegionInfo(TABLENAME, + Bytes.toBytes(midKey), + end); + + MetaEditor.offlineParentInMeta(catalogTracker, parent, splita, splitb); + Threads.sleep(100); + MetaEditor.addDaughter(catalogTracker, splitb, null); + MetaEditor.addDaughter(catalogTracker, splita, null); + + Threads.sleep(random.nextInt(200)); + } catch (Throwable e) { + ex = e; + Assert.fail(StringUtils.stringifyException(e)); + } + } + } + void rethrowExceptionIfAny() throws Throwable { + if (ex != null) { throw ex; } + } + } + + class MetaScannerVerifier extends StoppableImplementation implements Runnable { + Random random = new Random(); + Throwable ex = null; + @Override + public void run() { + while(!isStopped()) { + try { + NavigableMap regions = + MetaScanner.allTableRegions(TEST_UTIL.getConfiguration(), TABLENAME, false); + + LOG.info("-------"); + byte[] lastEndKey = HConstants.EMPTY_START_ROW; + for (HRegionInfo hri: regions.navigableKeySet()) { + long startKey = 0, endKey = Long.MAX_VALUE; + if (!Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())) { + startKey = Bytes.toLong(hri.getStartKey()); + } + if (!Bytes.equals(HConstants.EMPTY_END_ROW, hri.getEndKey())) { + endKey = Bytes.toLong(hri.getEndKey()); + } + LOG.info("start:" + startKey + " end:" + endKey + " hri:" + hri); + Assert.assertTrue(Bytes.equals(lastEndKey, hri.getStartKey())); + lastEndKey = hri.getEndKey(); + } + Assert.assertTrue(Bytes.equals(lastEndKey, HConstants.EMPTY_END_ROW)); + LOG.info("-------"); + Threads.sleep(10 + random.nextInt(50)); + } catch (Throwable e) { + ex = e; + Assert.fail(StringUtils.stringifyException(e)); + } + } + } + void rethrowExceptionIfAny() throws Throwable { + if (ex != null) { throw ex; } + } + } + + RegionMetaSplitter regionMetaSplitter = new RegionMetaSplitter(); + MetaScannerVerifier metaScannerVerifier = new MetaScannerVerifier(); + + Thread regionMetaSplitterThread = new Thread(regionMetaSplitter); + Thread metaScannerVerifierThread = new Thread(metaScannerVerifier); + + regionMetaSplitterThread.start(); + metaScannerVerifierThread.start(); + + Threads.sleep(runtime); + + regionMetaSplitter.stop("test finished"); + metaScannerVerifier.stop("test finished"); + + regionMetaSplitterThread.join(); + metaScannerVerifierThread.join(); + + regionMetaSplitter.rethrowExceptionIfAny(); + metaScannerVerifier.rethrowExceptionIfAny(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java b/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java new file mode 100644 index 0000000..ae5ff1f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java @@ -0,0 +1,574 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +@Category(MediumTests.class) +public class TestMultiParallel { + private static final Log LOG = LogFactory.getLog(TestMultiParallel.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final byte[] VALUE = Bytes.toBytes("value"); + private static final byte[] QUALIFIER = Bytes.toBytes("qual"); + private static final String FAMILY = "family"; + private static final String TEST_TABLE = "multi_test_table"; + private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY); + private static final byte[] ONE_ROW = Bytes.toBytes("xxx"); + private static final byte [][] KEYS = makeKeys(); + + private static final int slaves = 2; // also used for testing HTable pool size + + @BeforeClass public static void beforeClass() throws Exception { + UTIL.startMiniCluster(slaves); + HTable t = UTIL.createTable(Bytes.toBytes(TEST_TABLE), Bytes.toBytes(FAMILY)); + UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY)); + UTIL.waitTableAvailable(Bytes.toBytes(TEST_TABLE), 15 * 1000); + t.close(); + } + + @AfterClass public static void afterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Before public void before() throws IOException { + LOG.info("before"); + if (UTIL.ensureSomeRegionServersAvailable(slaves)) { + // Distribute regions + UTIL.getMiniHBaseCluster().getMaster().balance(); + } + LOG.info("before done"); + } + + private static byte[][] makeKeys() { + byte [][] starterKeys = HBaseTestingUtility.KEYS; + // Create a "non-uniform" test set with the following characteristics: + // a) Unequal number of keys per region + + // Don't use integer as a multiple, so that we have a number of keys that is + // not a multiple of the number of regions + int numKeys = (int) ((float) starterKeys.length * 10.33F); + + List keys = new ArrayList(); + for (int i = 0; i < numKeys; i++) { + int kIdx = i % starterKeys.length; + byte[] k = starterKeys[kIdx]; + byte[] cp = new byte[k.length + 1]; + System.arraycopy(k, 0, cp, 0, k.length); + cp[k.length] = new Integer(i % 256).byteValue(); + keys.add(cp); + } + + // b) Same duplicate keys (showing multiple Gets/Puts to the same row, which + // should work) + // c) keys are not in sorted order (within a region), to ensure that the + // sorting code and index mapping doesn't break the functionality + for (int i = 0; i < 100; i++) { + int kIdx = i % starterKeys.length; + byte[] k = starterKeys[kIdx]; + byte[] cp = new byte[k.length + 1]; + System.arraycopy(k, 0, cp, 0, k.length); + cp[k.length] = new Integer(i % 256).byteValue(); + keys.add(cp); + } + return keys.toArray(new byte [][] {new byte [] {}}); + } + + + /** + * This is for testing the active number of threads that were used while + * doing a batch operation. It inserts one row per region via the batch + * operation, and then checks the number of active threads. + * For HBASE-3553 + * @throws IOException + * @throws InterruptedException + * @throws NoSuchFieldException + * @throws SecurityException + */ + @Test(timeout=300000) + public void testActiveThreadsCount() throws Exception{ + HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + List puts = constructPutRequests(); // creates a Put for every region + table.batch(puts); + Field poolField = table.getClass().getDeclaredField("pool"); + poolField.setAccessible(true); + ThreadPoolExecutor tExecutor = (ThreadPoolExecutor) poolField.get(table); + assertEquals(slaves, tExecutor.getLargestPoolSize()); + table.close(); + } + + @Test(timeout=300000) + public void testBatchWithGet() throws Exception { + LOG.info("test=testBatchWithGet"); + HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + + // load test data + List puts = constructPutRequests(); + table.batch(puts); + + // create a list of gets and run it + List gets = new ArrayList(); + for (byte[] k : KEYS) { + Get get = new Get(k); + get.addColumn(BYTES_FAMILY, QUALIFIER); + gets.add(get); + } + Result[] multiRes = new Result[gets.size()]; + table.batch(gets, multiRes); + + // Same gets using individual call API + List singleRes = new ArrayList(); + for (Row get : gets) { + singleRes.add(table.get((Get) get)); + } + + // Compare results + Assert.assertEquals(singleRes.size(), multiRes.length); + for (int i = 0; i < singleRes.size(); i++) { + Assert.assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER)); + KeyValue[] singleKvs = singleRes.get(i).raw(); + KeyValue[] multiKvs = multiRes[i].raw(); + for (int j = 0; j < singleKvs.length; j++) { + Assert.assertEquals(singleKvs[j], multiKvs[j]); + Assert.assertEquals(0, Bytes.compareTo(singleKvs[j].getValue(), multiKvs[j] + .getValue())); + } + } + table.close(); + } + + @Test + public void testBadFam() throws Exception { + LOG.info("test=testBadFam"); + HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + + List actions = new ArrayList(); + Put p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("bad_family"), Bytes.toBytes("qual"), Bytes.toBytes("value")); + actions.add(p); + p = new Put(Bytes.toBytes("row2")); + p.add(BYTES_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value")); + actions.add(p); + + // row1 and row2 should be in the same region. + + Object [] r = new Object[actions.size()]; + try { + table.batch(actions, r); + fail(); + } catch (RetriesExhaustedWithDetailsException ex) { + LOG.debug(ex); + // good! + assertFalse(ex.mayHaveClusterIssues()); + } + assertEquals(2, r.length); + assertTrue(r[0] instanceof Throwable); + assertTrue(r[1] instanceof Result); + table.close(); + } + + /** + * Only run one Multi test with a forced RegionServer abort. Otherwise, the + * unit tests will take an unnecessarily long time to run. + * + * @throws Exception + */ + @Test (timeout=300000) + public void testFlushCommitsWithAbort() throws Exception { + LOG.info("test=testFlushCommitsWithAbort"); + doTestFlushCommits(true); + } + + @Test (timeout=300000) + public void testFlushCommitsNoAbort() throws Exception { + LOG.info("test=testFlushCommitsNoAbort"); + doTestFlushCommits(false); + } + + /** + * Set table auto flush to false and test flushing commits + * @param doAbort true if abort one regionserver in the testing + * @throws Exception + */ + private void doTestFlushCommits(boolean doAbort) throws Exception { + // Load the data + LOG.info("get new table"); + HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + table.setAutoFlush(false); + table.setWriteBufferSize(10 * 1024 * 1024); + + LOG.info("constructPutRequests"); + List puts = constructPutRequests(); + for (Row put : puts) { + table.put((Put) put); + } + LOG.info("puts"); + table.flushCommits(); + int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads() + .size(); + assert liveRScount > 0; + JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster() + .getLiveRegionServerThreads().get(0); + if (doAbort) { + LOG.info("Aborted=" + UTIL.getMiniHBaseCluster().abortRegionServer(0)); + + // If we waiting for no regions being online after we abort the server, we + // could ensure the master has re-assigned the regions on killed server + // after putting keys successfully, it means the server we abort is dead + // and detected by matser + while (liveRS.getRegionServer().getNumberOfOnlineRegions() != 0) { + Thread.sleep(100); + } + while (UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() == liveRScount) { + Thread.sleep(100); + } + + // try putting more keys after the abort. same key/qual... just validating + // no exceptions thrown + puts = constructPutRequests(); + for (Row put : puts) { + table.put((Put) put); + } + + table.flushCommits(); + } + + LOG.info("validating loaded data"); + validateLoadedData(table); + + // Validate server and region count + List liveRSs = + UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + int count = 0; + for (JVMClusterUtil.RegionServerThread t: liveRSs) { + count++; + LOG.info("Count=" + count + ", Alive=" + t.getRegionServer()); + } + LOG.info("Count=" + count); + Assert.assertEquals("Server count=" + count + ", abort=" + doAbort, + (doAbort ? (liveRScount - 1) : liveRScount), count); + for (JVMClusterUtil.RegionServerThread t: liveRSs) { + int regions = t.getRegionServer().getOnlineRegions().size(); + Assert.assertTrue("Count of regions=" + regions, regions > 10); + } + table.close(); + LOG.info("done"); + } + + @Test (timeout=300000) + public void testBatchWithPut() throws Exception { + LOG.info("test=testBatchWithPut"); + HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + + // put multiple rows using a batch + List puts = constructPutRequests(); + + Object[] results = table.batch(puts); + validateSizeAndEmpty(results, KEYS.length); + + if (true) { + int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads() + .size(); + assert liveRScount > 0; + JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster() + .getLiveRegionServerThreads().get(0); + liveRS.getRegionServer().abort("Aborting for tests", + new Exception("testBatchWithPut")); + + puts = constructPutRequests(); + results = table.batch(puts); + validateSizeAndEmpty(results, KEYS.length); + } + + validateLoadedData(table); + table.close(); + } + + @Test(timeout=300000) + public void testBatchWithDelete() throws Exception { + LOG.info("test=testBatchWithDelete"); + HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + + // Load some data + List puts = constructPutRequests(); + Object[] results = table.batch(puts); + validateSizeAndEmpty(results, KEYS.length); + + // Deletes + List deletes = new ArrayList(); + for (int i = 0; i < KEYS.length; i++) { + Delete delete = new Delete(KEYS[i]); + delete.deleteFamily(BYTES_FAMILY); + deletes.add(delete); + } + results = table.batch(deletes); + validateSizeAndEmpty(results, KEYS.length); + + // Get to make sure ... + for (byte[] k : KEYS) { + Get get = new Get(k); + get.addColumn(BYTES_FAMILY, QUALIFIER); + Assert.assertFalse(table.exists(get)); + } + table.close(); + } + + @Test(timeout=300000) + public void testHTableDeleteWithList() throws Exception { + LOG.info("test=testHTableDeleteWithList"); + HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + + // Load some data + List puts = constructPutRequests(); + Object[] results = table.batch(puts); + validateSizeAndEmpty(results, KEYS.length); + + // Deletes + ArrayList deletes = new ArrayList(); + for (int i = 0; i < KEYS.length; i++) { + Delete delete = new Delete(KEYS[i]); + delete.deleteFamily(BYTES_FAMILY); + deletes.add(delete); + } + table.delete(deletes); + Assert.assertTrue(deletes.isEmpty()); + + // Get to make sure ... + for (byte[] k : KEYS) { + Get get = new Get(k); + get.addColumn(BYTES_FAMILY, QUALIFIER); + Assert.assertFalse(table.exists(get)); + } + table.close(); + } + + @Test(timeout=300000) + public void testBatchWithManyColsInOneRowGetAndPut() throws Exception { + LOG.info("test=testBatchWithManyColsInOneRowGetAndPut"); + HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + + List puts = new ArrayList(); + for (int i = 0; i < 100; i++) { + Put put = new Put(ONE_ROW); + byte[] qual = Bytes.toBytes("column" + i); + put.add(BYTES_FAMILY, qual, VALUE); + puts.add(put); + } + Object[] results = table.batch(puts); + + // validate + validateSizeAndEmpty(results, 100); + + // get the data back and validate that it is correct + List gets = new ArrayList(); + for (int i = 0; i < 100; i++) { + Get get = new Get(ONE_ROW); + byte[] qual = Bytes.toBytes("column" + i); + get.addColumn(BYTES_FAMILY, qual); + gets.add(get); + } + + Object[] multiRes = table.batch(gets); + + int idx = 0; + for (Object r : multiRes) { + byte[] qual = Bytes.toBytes("column" + idx); + validateResult(r, qual, VALUE); + idx++; + } + table.close(); + } + + @Test(timeout=300000) + public void testBatchWithIncrementAndAppend() throws Exception { + LOG.info("test=testBatchWithIncrementAndAppend"); + final byte[] QUAL1 = Bytes.toBytes("qual1"); + final byte[] QUAL2 = Bytes.toBytes("qual2"); + final byte[] QUAL3 = Bytes.toBytes("qual3"); + final byte[] QUAL4 = Bytes.toBytes("qual4"); + HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + Delete d = new Delete(ONE_ROW); + table.delete(d); + Put put = new Put(ONE_ROW); + put.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("abc")); + put.add(BYTES_FAMILY, QUAL2, Bytes.toBytes(1L)); + table.put(put); + + Increment inc = new Increment(ONE_ROW); + inc.addColumn(BYTES_FAMILY, QUAL2, 1); + inc.addColumn(BYTES_FAMILY, QUAL3, 1); + + Append a = new Append(ONE_ROW); + a.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("def")); + a.add(BYTES_FAMILY, QUAL4, Bytes.toBytes("xyz")); + List actions = new ArrayList(); + actions.add(inc); + actions.add(a); + + Object[] multiRes = table.batch(actions); + validateResult(multiRes[1], QUAL1, Bytes.toBytes("abcdef")); + validateResult(multiRes[1], QUAL4, Bytes.toBytes("xyz")); + validateResult(multiRes[0], QUAL2, Bytes.toBytes(2L)); + validateResult(multiRes[0], QUAL3, Bytes.toBytes(1L)); + table.close(); + } + + @Test(timeout=300000) + public void testBatchWithMixedActions() throws Exception { + LOG.info("test=testBatchWithMixedActions"); + HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + + // Load some data to start + Object[] results = table.batch(constructPutRequests()); + validateSizeAndEmpty(results, KEYS.length); + + // Batch: get, get, put(new col), delete, get, get of put, get of deleted, + // put + List actions = new ArrayList(); + + byte[] qual2 = Bytes.toBytes("qual2"); + byte[] val2 = Bytes.toBytes("putvalue2"); + + // 0 get + Get get = new Get(KEYS[10]); + get.addColumn(BYTES_FAMILY, QUALIFIER); + actions.add(get); + + // 1 get + get = new Get(KEYS[11]); + get.addColumn(BYTES_FAMILY, QUALIFIER); + actions.add(get); + + // 2 put of new column + Put put = new Put(KEYS[10]); + put.add(BYTES_FAMILY, qual2, val2); + actions.add(put); + + // 3 delete + Delete delete = new Delete(KEYS[20]); + delete.deleteFamily(BYTES_FAMILY); + actions.add(delete); + + // 4 get + get = new Get(KEYS[30]); + get.addColumn(BYTES_FAMILY, QUALIFIER); + actions.add(get); + + // There used to be a 'get' of a previous put here, but removed + // since this API really cannot guarantee order in terms of mixed + // get/puts. + + // 5 put of new column + put = new Put(KEYS[40]); + put.add(BYTES_FAMILY, qual2, val2); + actions.add(put); + + results = table.batch(actions); + + // Validation + + validateResult(results[0]); + validateResult(results[1]); + validateEmpty(results[2]); + validateEmpty(results[3]); + validateResult(results[4]); + validateEmpty(results[5]); + + // validate last put, externally from the batch + get = new Get(KEYS[40]); + get.addColumn(BYTES_FAMILY, qual2); + Result r = table.get(get); + validateResult(r, qual2, val2); + + table.close(); + } + + // // Helper methods //// + + private void validateResult(Object r) { + validateResult(r, QUALIFIER, VALUE); + } + + private void validateResult(Object r1, byte[] qual, byte[] val) { + // TODO provide nice assert here or something. + Result r = (Result)r1; + Assert.assertTrue(r.containsColumn(BYTES_FAMILY, qual)); + Assert.assertEquals(0, Bytes.compareTo(val, r.getValue(BYTES_FAMILY, qual))); + } + + private List constructPutRequests() { + List puts = new ArrayList(); + for (byte[] k : KEYS) { + Put put = new Put(k); + put.add(BYTES_FAMILY, QUALIFIER, VALUE); + puts.add(put); + } + return puts; + } + + private void validateLoadedData(HTable table) throws IOException { + // get the data back and validate that it is correct + for (byte[] k : KEYS) { + Get get = new Get(k); + get.addColumn(BYTES_FAMILY, QUALIFIER); + Result r = table.get(get); + Assert.assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER)); + Assert.assertEquals(0, Bytes.compareTo(VALUE, r + .getValue(BYTES_FAMILY, QUALIFIER))); + } + } + + private void validateEmpty(Object r1) { + Result result = (Result)r1; + Assert.assertTrue(result != null); + Assert.assertTrue(result.getRow() == null); + Assert.assertEquals(0, result.raw().length); + } + + private void validateSizeAndEmpty(Object[] results, int expectedSize) { + // Validate got back the same number of Result objects, all empty + Assert.assertEquals(expectedSize, results.length); + for (Object result : results) { + validateEmpty(result); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java b/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java new file mode 100644 index 0000000..d479259 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java @@ -0,0 +1,540 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Run tests related to {@link TimestampsFilter} using HBase client APIs. + * Sets up the HBase mini cluster once at start. Each creates a table + * named for the method and does its stuff against that. + */ +@Category(LargeTests.class) +public class TestMultipleTimestamps { + final Log LOG = LogFactory.getLog(getClass()); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + // Nothing to do. + } + + /** + * @throws java.lang.Exception + */ + @After + public void tearDown() throws Exception { + // Nothing to do. + } + + @Test + public void testReseeksWithOneColumnMiltipleTimestamp() throws IOException { + byte [] TABLE = Bytes.toBytes("testReseeksWithOne" + + "ColumnMiltipleTimestamps"); + byte [] FAMILY = Bytes.toBytes("event_log"); + byte [][] FAMILIES = new byte[][] { FAMILY }; + + // create table; set versions to max... + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); + + Integer[] putRows = new Integer[] {1, 3, 5, 7}; + Integer[] putColumns = new Integer[] { 1, 3, 5}; + Long[] putTimestamps = new Long[] {1L, 2L, 3L, 4L, 5L}; + + Integer[] scanRows = new Integer[] {3, 5}; + Integer[] scanColumns = new Integer[] {3}; + Long[] scanTimestamps = new Long[] {3L, 4L}; + int scanMaxVersions = 2; + + put(ht, FAMILY, putRows, putColumns, putTimestamps); + + TEST_UTIL.flush(TABLE); + + ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns, + scanTimestamps, scanMaxVersions); + + KeyValue[] kvs; + + kvs = scanner.next().raw(); + assertEquals(2, kvs.length); + checkOneCell(kvs[0], FAMILY, 3, 3, 4); + checkOneCell(kvs[1], FAMILY, 3, 3, 3); + kvs = scanner.next().raw(); + assertEquals(2, kvs.length); + checkOneCell(kvs[0], FAMILY, 5, 3, 4); + checkOneCell(kvs[1], FAMILY, 5, 3, 3); + + ht.close(); + } + + @Test + public void testReseeksWithMultipleColumnOneTimestamp() throws IOException { + LOG.info("testReseeksWithMultipleColumnOneTimestamp"); + byte [] TABLE = Bytes.toBytes("testReseeksWithMultiple" + + "ColumnOneTimestamps"); + byte [] FAMILY = Bytes.toBytes("event_log"); + byte [][] FAMILIES = new byte[][] { FAMILY }; + + // create table; set versions to max... + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); + + Integer[] putRows = new Integer[] {1, 3, 5, 7}; + Integer[] putColumns = new Integer[] { 1, 3, 5}; + Long[] putTimestamps = new Long[] {1L, 2L, 3L, 4L, 5L}; + + Integer[] scanRows = new Integer[] {3, 5}; + Integer[] scanColumns = new Integer[] {3,4}; + Long[] scanTimestamps = new Long[] {3L}; + int scanMaxVersions = 2; + + put(ht, FAMILY, putRows, putColumns, putTimestamps); + + TEST_UTIL.flush(TABLE); + + ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns, + scanTimestamps, scanMaxVersions); + + KeyValue[] kvs; + + kvs = scanner.next().raw(); + assertEquals(1, kvs.length); + checkOneCell(kvs[0], FAMILY, 3, 3, 3); + kvs = scanner.next().raw(); + assertEquals(1, kvs.length); + checkOneCell(kvs[0], FAMILY, 5, 3, 3); + + ht.close(); + } + + @Test + public void testReseeksWithMultipleColumnMultipleTimestamp() throws + IOException { + LOG.info("testReseeksWithMultipleColumnMultipleTimestamp"); + + byte [] TABLE = Bytes.toBytes("testReseeksWithMultiple" + + "ColumnMiltipleTimestamps"); + byte [] FAMILY = Bytes.toBytes("event_log"); + byte [][] FAMILIES = new byte[][] { FAMILY }; + + // create table; set versions to max... + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); + + Integer[] putRows = new Integer[] {1, 3, 5, 7}; + Integer[] putColumns = new Integer[] { 1, 3, 5}; + Long[] putTimestamps = new Long[] {1L, 2L, 3L, 4L, 5L}; + + Integer[] scanRows = new Integer[] {5, 7}; + Integer[] scanColumns = new Integer[] {3, 4, 5}; + Long[] scanTimestamps = new Long[] {2l, 3L}; + int scanMaxVersions = 2; + + put(ht, FAMILY, putRows, putColumns, putTimestamps); + + TEST_UTIL.flush(TABLE); + + ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns, + scanTimestamps, scanMaxVersions); + + KeyValue[] kvs; + + kvs = scanner.next().raw(); + assertEquals(4, kvs.length); + checkOneCell(kvs[0], FAMILY, 5, 3, 3); + checkOneCell(kvs[1], FAMILY, 5, 3, 2); + checkOneCell(kvs[2], FAMILY, 5, 5, 3); + checkOneCell(kvs[3], FAMILY, 5, 5, 2); + kvs = scanner.next().raw(); + assertEquals(4, kvs.length); + checkOneCell(kvs[0], FAMILY, 7, 3, 3); + checkOneCell(kvs[1], FAMILY, 7, 3, 2); + checkOneCell(kvs[2], FAMILY, 7, 5, 3); + checkOneCell(kvs[3], FAMILY, 7, 5, 2); + + ht.close(); + } + + @Test + public void testReseeksWithMultipleFiles() throws IOException { + LOG.info("testReseeksWithMultipleFiles"); + byte [] TABLE = Bytes.toBytes("testReseeksWithMultipleFiles"); + byte [] FAMILY = Bytes.toBytes("event_log"); + byte [][] FAMILIES = new byte[][] { FAMILY }; + + // create table; set versions to max... + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); + + Integer[] putRows1 = new Integer[] {1, 2, 3}; + Integer[] putColumns1 = new Integer[] { 2, 5, 6}; + Long[] putTimestamps1 = new Long[] {1L, 2L, 5L}; + + Integer[] putRows2 = new Integer[] {6, 7}; + Integer[] putColumns2 = new Integer[] {3, 6}; + Long[] putTimestamps2 = new Long[] {4L, 5L}; + + Integer[] putRows3 = new Integer[] {2, 3, 5}; + Integer[] putColumns3 = new Integer[] {1, 2, 3}; + Long[] putTimestamps3 = new Long[] {4L,8L}; + + + Integer[] scanRows = new Integer[] {3, 5, 7}; + Integer[] scanColumns = new Integer[] {3, 4, 5}; + Long[] scanTimestamps = new Long[] {2l, 4L}; + int scanMaxVersions = 5; + + put(ht, FAMILY, putRows1, putColumns1, putTimestamps1); + TEST_UTIL.flush(TABLE); + put(ht, FAMILY, putRows2, putColumns2, putTimestamps2); + TEST_UTIL.flush(TABLE); + put(ht, FAMILY, putRows3, putColumns3, putTimestamps3); + + ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns, + scanTimestamps, scanMaxVersions); + + KeyValue[] kvs; + + kvs = scanner.next().raw(); + assertEquals(2, kvs.length); + checkOneCell(kvs[0], FAMILY, 3, 3, 4); + checkOneCell(kvs[1], FAMILY, 3, 5, 2); + + kvs = scanner.next().raw(); + assertEquals(1, kvs.length); + checkOneCell(kvs[0], FAMILY, 5, 3, 4); + + kvs = scanner.next().raw(); + assertEquals(1, kvs.length); + checkOneCell(kvs[0], FAMILY, 6, 3, 4); + + kvs = scanner.next().raw(); + assertEquals(1, kvs.length); + checkOneCell(kvs[0], FAMILY, 7, 3, 4); + + ht.close(); + } + + @Test + public void testWithVersionDeletes() throws Exception { + + // first test from memstore (without flushing). + testWithVersionDeletes(false); + + // run same test against HFiles (by forcing a flush). + testWithVersionDeletes(true); + } + + public void testWithVersionDeletes(boolean flushTables) throws IOException { + LOG.info("testWithVersionDeletes_"+ + (flushTables ? "flush" : "noflush")); + + byte [] TABLE = Bytes.toBytes("testWithVersionDeletes_" + + (flushTables ? "flush" : "noflush")); + + byte [] FAMILY = Bytes.toBytes("event_log"); + byte [][] FAMILIES = new byte[][] { FAMILY }; + + // create table; set versions to max... + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); + + // For row:0, col:0: insert versions 1 through 5. + putNVersions(ht, FAMILY, 0, 0, 1, 5); + + if (flushTables) { + TEST_UTIL.flush(TABLE); + } + + // delete version 4. + deleteOneVersion(ht, FAMILY, 0, 0, 4); + + // request a bunch of versions including the deleted version. We should + // only get back entries for the versions that exist. + KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, + Arrays.asList(2L, 3L, 4L, 5L)); + assertEquals(3, kvs.length); + checkOneCell(kvs[0], FAMILY, 0, 0, 5); + checkOneCell(kvs[1], FAMILY, 0, 0, 3); + checkOneCell(kvs[2], FAMILY, 0, 0, 2); + + ht.close(); + } + + @Test + public void testWithMultipleVersionDeletes() throws IOException { + LOG.info("testWithMultipleVersionDeletes"); + + byte [] TABLE = Bytes.toBytes("testWithMultipleVersionDeletes"); + byte [] FAMILY = Bytes.toBytes("event_log"); + byte [][] FAMILIES = new byte[][] { FAMILY }; + + // create table; set versions to max... + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); + + // For row:0, col:0: insert versions 1 through 5. + putNVersions(ht, FAMILY, 0, 0, 1, 5); + + TEST_UTIL.flush(TABLE); + + // delete all versions before 4. + deleteAllVersionsBefore(ht, FAMILY, 0, 0, 4); + + // request a bunch of versions including the deleted version. We should + // only get back entries for the versions that exist. + KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L)); + assertEquals(0, kvs.length); + + ht.close(); + } + + @Test + public void testWithColumnDeletes() throws IOException { + byte [] TABLE = Bytes.toBytes("testWithColumnDeletes"); + byte [] FAMILY = Bytes.toBytes("event_log"); + byte [][] FAMILIES = new byte[][] { FAMILY }; + + // create table; set versions to max... + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); + + // For row:0, col:0: insert versions 1 through 5. + putNVersions(ht, FAMILY, 0, 0, 1, 5); + + TEST_UTIL.flush(TABLE); + + // delete all versions before 4. + deleteColumn(ht, FAMILY, 0, 0); + + // request a bunch of versions including the deleted version. We should + // only get back entries for the versions that exist. + KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L)); + assertEquals(0, kvs.length); + + ht.close(); + } + + @Test + public void testWithFamilyDeletes() throws IOException { + byte [] TABLE = Bytes.toBytes("testWithFamilyDeletes"); + byte [] FAMILY = Bytes.toBytes("event_log"); + byte [][] FAMILIES = new byte[][] { FAMILY }; + + // create table; set versions to max... + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); + + // For row:0, col:0: insert versions 1 through 5. + putNVersions(ht, FAMILY, 0, 0, 1, 5); + + TEST_UTIL.flush(TABLE); + + // delete all versions before 4. + deleteFamily(ht, FAMILY, 0); + + // request a bunch of versions including the deleted version. We should + // only get back entries for the versions that exist. + KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L)); + assertEquals(0, kvs.length); + + ht.close(); + } + + /** + * Assert that the passed in KeyValue has expected contents for the + * specified row, column & timestamp. + */ + private void checkOneCell(KeyValue kv, byte[] cf, + int rowIdx, int colIdx, long ts) { + + String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts; + + assertEquals("Row mismatch which checking: " + ctx, + "row:"+ rowIdx, Bytes.toString(kv.getRow())); + + assertEquals("ColumnFamily mismatch while checking: " + ctx, + Bytes.toString(cf), Bytes.toString(kv.getFamily())); + + assertEquals("Column qualifier mismatch while checking: " + ctx, + "column:" + colIdx, + Bytes.toString(kv.getQualifier())); + + assertEquals("Timestamp mismatch while checking: " + ctx, + ts, kv.getTimestamp()); + + assertEquals("Value mismatch while checking: " + ctx, + "value-version-" + ts, Bytes.toString(kv.getValue())); + } + + /** + * Uses the TimestampFilter on a Get to request a specified list of + * versions for the row/column specified by rowIdx & colIdx. + * + */ + private KeyValue[] getNVersions(HTable ht, byte[] cf, int rowIdx, + int colIdx, List versions) + throws IOException { + byte row[] = Bytes.toBytes("row:" + rowIdx); + byte column[] = Bytes.toBytes("column:" + colIdx); + Get get = new Get(row); + get.addColumn(cf, column); + get.setMaxVersions(); + get.setTimeRange(Collections.min(versions), Collections.max(versions)+1); + Result result = ht.get(get); + + return result.raw(); + } + + private ResultScanner scan(HTable ht, byte[] cf, + Integer[] rowIndexes, Integer[] columnIndexes, + Long[] versions, int maxVersions) + throws IOException { + Arrays.asList(rowIndexes); + byte startRow[] = Bytes.toBytes("row:" + + Collections.min( Arrays.asList(rowIndexes))); + byte endRow[] = Bytes.toBytes("row:" + + Collections.max( Arrays.asList(rowIndexes))+1); + Scan scan = new Scan(startRow, endRow); + for (Integer colIdx: columnIndexes) { + byte column[] = Bytes.toBytes("column:" + colIdx); + scan.addColumn(cf, column); + } + scan.setMaxVersions(maxVersions); + scan.setTimeRange(Collections.min(Arrays.asList(versions)), + Collections.max(Arrays.asList(versions))+1); + ResultScanner scanner = ht.getScanner(scan); + return scanner; + } + + private void put(HTable ht, byte[] cf, Integer[] rowIndexes, + Integer[] columnIndexes, Long[] versions) + throws IOException { + for (int rowIdx: rowIndexes) { + byte row[] = Bytes.toBytes("row:" + rowIdx); + Put put = new Put(row); + put.setWriteToWAL(false); + for(int colIdx: columnIndexes) { + byte column[] = Bytes.toBytes("column:" + colIdx); + for (long version: versions) { + put.add(cf, column, version, Bytes.toBytes("value-version-" + + version)); + } + } + ht.put(put); + } + } + + /** + * Insert in specific row/column versions with timestamps + * versionStart..versionEnd. + */ + private void putNVersions(HTable ht, byte[] cf, int rowIdx, int colIdx, + long versionStart, long versionEnd) + throws IOException { + byte row[] = Bytes.toBytes("row:" + rowIdx); + byte column[] = Bytes.toBytes("column:" + colIdx); + Put put = new Put(row); + put.setWriteToWAL(false); + + for (long idx = versionStart; idx <= versionEnd; idx++) { + put.add(cf, column, idx, Bytes.toBytes("value-version-" + idx)); + } + + ht.put(put); + } + + /** + * For row/column specified by rowIdx/colIdx, delete the cell + * corresponding to the specified version. + */ + private void deleteOneVersion(HTable ht, byte[] cf, int rowIdx, + int colIdx, long version) + throws IOException { + byte row[] = Bytes.toBytes("row:" + rowIdx); + byte column[] = Bytes.toBytes("column:" + colIdx); + Delete del = new Delete(row); + del.deleteColumn(cf, column, version); + ht.delete(del); + } + + /** + * For row/column specified by rowIdx/colIdx, delete all cells + * preceeding the specified version. + */ + private void deleteAllVersionsBefore(HTable ht, byte[] cf, int rowIdx, + int colIdx, long version) + throws IOException { + byte row[] = Bytes.toBytes("row:" + rowIdx); + byte column[] = Bytes.toBytes("column:" + colIdx); + Delete del = new Delete(row); + del.deleteColumns(cf, column, version); + ht.delete(del); + } + + private void deleteColumn(HTable ht, byte[] cf, int rowIdx, int colIdx) throws IOException { + byte row[] = Bytes.toBytes("row:" + rowIdx); + byte column[] = Bytes.toBytes("column:" + colIdx); + Delete del = new Delete(row); + del.deleteColumns(cf, column); + ht.delete(del); + } + + private void deleteFamily(HTable ht, byte[] cf, int rowIdx) throws IOException { + byte row[] = Bytes.toBytes("row:" + rowIdx); + Delete del = new Delete(row); + del.deleteFamily(cf); + ht.delete(del); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java b/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java new file mode 100644 index 0000000..43f856f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java @@ -0,0 +1,376 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; +import org.apache.hadoop.hbase.filter.ColumnPaginationFilter; +import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; +import org.apache.hadoop.hbase.filter.ColumnRangeFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.DependentColumnFilter; +import org.apache.hadoop.hbase.filter.FamilyFilter; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.InclusiveStopFilter; +import org.apache.hadoop.hbase.filter.KeyOnlyFilter; +import org.apache.hadoop.hbase.filter.MultipleColumnPrefixFilter; +import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; +import org.apache.hadoop.hbase.filter.SkipFilter; +import org.apache.hadoop.hbase.filter.TimestampsFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.apache.hadoop.hbase.filter.WhileMatchFilter; +import org.apache.hadoop.hbase.util.Bytes; + +import org.codehaus.jackson.map.ObjectMapper; +import org.junit.experimental.categories.Category; + +/** + * Run tests that use the functionality of the Operation superclass for + * Puts, Gets, Deletes, Scans, and MultiPuts. + */ +@Category(SmallTests.class) +public class TestOperation { + private static byte [] ROW = Bytes.toBytes("testRow"); + private static byte [] FAMILY = Bytes.toBytes("testFamily"); + private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); + private static byte [] VALUE = Bytes.toBytes("testValue"); + + private static ObjectMapper mapper = new ObjectMapper(); + + private static List TS_LIST = Arrays.asList(2L, 3L, 5L); + private static TimestampsFilter TS_FILTER = new TimestampsFilter(TS_LIST); + private static String STR_TS_FILTER = + TS_FILTER.getClass().getSimpleName() + " (3/3): [2, 3, 5]"; + + private static List L_TS_LIST = + Arrays.asList(0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L); + private static TimestampsFilter L_TS_FILTER = + new TimestampsFilter(L_TS_LIST); + private static String STR_L_TS_FILTER = + L_TS_FILTER.getClass().getSimpleName() + " (5/11): [0, 1, 2, 3, 4]"; + + private static String COL_NAME_1 = "col1"; + private static ColumnPrefixFilter COL_PRE_FILTER = + new ColumnPrefixFilter(COL_NAME_1.getBytes()); + private static String STR_COL_PRE_FILTER = + COL_PRE_FILTER.getClass().getSimpleName() + " " + COL_NAME_1; + + private static String COL_NAME_2 = "col2"; + private static ColumnRangeFilter CR_FILTER = new ColumnRangeFilter( + COL_NAME_1.getBytes(), true, COL_NAME_2.getBytes(), false); + private static String STR_CR_FILTER = CR_FILTER.getClass().getSimpleName() + + " [" + COL_NAME_1 + ", " + COL_NAME_2 + ")"; + + private static int COL_COUNT = 9; + private static ColumnCountGetFilter CCG_FILTER = + new ColumnCountGetFilter(COL_COUNT); + private static String STR_CCG_FILTER = + CCG_FILTER.getClass().getSimpleName() + " " + COL_COUNT; + + private static int LIMIT = 3; + private static int OFFSET = 4; + private static ColumnPaginationFilter CP_FILTER = + new ColumnPaginationFilter(LIMIT, OFFSET); + private static String STR_CP_FILTER = CP_FILTER.getClass().getSimpleName() + + " (" + LIMIT + ", " + OFFSET + ")"; + + private static String STOP_ROW_KEY = "stop"; + private static InclusiveStopFilter IS_FILTER = + new InclusiveStopFilter(STOP_ROW_KEY.getBytes()); + private static String STR_IS_FILTER = + IS_FILTER.getClass().getSimpleName() + " " + STOP_ROW_KEY; + + private static String PREFIX = "prefix"; + private static PrefixFilter PREFIX_FILTER = + new PrefixFilter(PREFIX.getBytes()); + private static String STR_PREFIX_FILTER = "PrefixFilter " + PREFIX; + + private static byte[][] PREFIXES = { + "0".getBytes(), "1".getBytes(), "2".getBytes()}; + private static MultipleColumnPrefixFilter MCP_FILTER = + new MultipleColumnPrefixFilter(PREFIXES); + private static String STR_MCP_FILTER = + MCP_FILTER.getClass().getSimpleName() + " (3/3): [0, 1, 2]"; + + private static byte[][] L_PREFIXES = { + "0".getBytes(), "1".getBytes(), "2".getBytes(), "3".getBytes(), + "4".getBytes(), "5".getBytes(), "6".getBytes(), "7".getBytes()}; + private static MultipleColumnPrefixFilter L_MCP_FILTER = + new MultipleColumnPrefixFilter(L_PREFIXES); + private static String STR_L_MCP_FILTER = + L_MCP_FILTER.getClass().getSimpleName() + " (5/8): [0, 1, 2, 3, 4]"; + + private static int PAGE_SIZE = 9; + private static PageFilter PAGE_FILTER = new PageFilter(PAGE_SIZE); + private static String STR_PAGE_FILTER = + PAGE_FILTER.getClass().getSimpleName() + " " + PAGE_SIZE; + + private static SkipFilter SKIP_FILTER = new SkipFilter(L_TS_FILTER); + private static String STR_SKIP_FILTER = + SKIP_FILTER.getClass().getSimpleName() + " " + STR_L_TS_FILTER; + + private static WhileMatchFilter WHILE_FILTER = + new WhileMatchFilter(L_TS_FILTER); + private static String STR_WHILE_FILTER = + WHILE_FILTER.getClass().getSimpleName() + " " + STR_L_TS_FILTER; + + private static KeyOnlyFilter KEY_ONLY_FILTER = new KeyOnlyFilter(); + private static String STR_KEY_ONLY_FILTER = + KEY_ONLY_FILTER.getClass().getSimpleName(); + + private static FirstKeyOnlyFilter FIRST_KEY_ONLY_FILTER = + new FirstKeyOnlyFilter(); + private static String STR_FIRST_KEY_ONLY_FILTER = + FIRST_KEY_ONLY_FILTER.getClass().getSimpleName(); + + private static CompareOp CMP_OP = CompareOp.EQUAL; + private static byte[] CMP_VALUE = "value".getBytes(); + private static BinaryComparator BC = new BinaryComparator(CMP_VALUE); + private static DependentColumnFilter DC_FILTER = + new DependentColumnFilter(FAMILY, QUALIFIER, true, CMP_OP, BC); + private static String STR_DC_FILTER = String.format( + "%s (%s, %s, %s, %s, %s)", DC_FILTER.getClass().getSimpleName(), + Bytes.toStringBinary(FAMILY), Bytes.toStringBinary(QUALIFIER), true, + CMP_OP.name(), Bytes.toStringBinary(BC.getValue())); + + private static FamilyFilter FAMILY_FILTER = new FamilyFilter(CMP_OP, BC); + private static String STR_FAMILY_FILTER = + FAMILY_FILTER.getClass().getSimpleName() + " (EQUAL, value)"; + + private static QualifierFilter QUALIFIER_FILTER = + new QualifierFilter(CMP_OP, BC); + private static String STR_QUALIFIER_FILTER = + QUALIFIER_FILTER.getClass().getSimpleName() + " (EQUAL, value)"; + + private static RowFilter ROW_FILTER = new RowFilter(CMP_OP, BC); + private static String STR_ROW_FILTER = + ROW_FILTER.getClass().getSimpleName() + " (EQUAL, value)"; + + private static ValueFilter VALUE_FILTER = new ValueFilter(CMP_OP, BC); + private static String STR_VALUE_FILTER = + VALUE_FILTER.getClass().getSimpleName() + " (EQUAL, value)"; + + private static SingleColumnValueFilter SCV_FILTER = + new SingleColumnValueFilter(FAMILY, QUALIFIER, CMP_OP, CMP_VALUE); + private static String STR_SCV_FILTER = String.format("%s (%s, %s, %s, %s)", + SCV_FILTER.getClass().getSimpleName(), Bytes.toStringBinary(FAMILY), + Bytes.toStringBinary(QUALIFIER), CMP_OP.name(), + Bytes.toStringBinary(CMP_VALUE)); + + private static SingleColumnValueExcludeFilter SCVE_FILTER = + new SingleColumnValueExcludeFilter(FAMILY, QUALIFIER, CMP_OP, CMP_VALUE); + private static String STR_SCVE_FILTER = String.format("%s (%s, %s, %s, %s)", + SCVE_FILTER.getClass().getSimpleName(), Bytes.toStringBinary(FAMILY), + Bytes.toStringBinary(QUALIFIER), CMP_OP.name(), + Bytes.toStringBinary(CMP_VALUE)); + + private static FilterList AND_FILTER_LIST = new FilterList( + Operator.MUST_PASS_ALL, Arrays.asList((Filter) TS_FILTER, L_TS_FILTER, + CR_FILTER)); + private static String STR_AND_FILTER_LIST = String.format( + "%s AND (3/3): [%s, %s, %s]", AND_FILTER_LIST.getClass().getSimpleName(), + STR_TS_FILTER, STR_L_TS_FILTER, STR_CR_FILTER); + + private static FilterList OR_FILTER_LIST = new FilterList( + Operator.MUST_PASS_ONE, Arrays.asList((Filter) TS_FILTER, L_TS_FILTER, + CR_FILTER)); + private static String STR_OR_FILTER_LIST = String.format( + "%s OR (3/3): [%s, %s, %s]", AND_FILTER_LIST.getClass().getSimpleName(), + STR_TS_FILTER, STR_L_TS_FILTER, STR_CR_FILTER); + + private static FilterList L_FILTER_LIST = new FilterList( + Arrays.asList((Filter) TS_FILTER, L_TS_FILTER, CR_FILTER, COL_PRE_FILTER, + CCG_FILTER, CP_FILTER, PREFIX_FILTER, PAGE_FILTER)); + private static String STR_L_FILTER_LIST = String.format( + "%s AND (5/8): [%s, %s, %s, %s, %s]", + L_FILTER_LIST.getClass().getSimpleName(), STR_TS_FILTER, STR_L_TS_FILTER, + STR_CR_FILTER, STR_COL_PRE_FILTER, STR_CCG_FILTER, STR_CP_FILTER); + + private static Filter[] FILTERS = { + TS_FILTER, // TimestampsFilter + L_TS_FILTER, // TimestampsFilter + COL_PRE_FILTER, // ColumnPrefixFilter + CP_FILTER, // ColumnPaginationFilter + CR_FILTER, // ColumnRangeFilter + CCG_FILTER, // ColumnCountGetFilter + IS_FILTER, // InclusiveStopFilter + PREFIX_FILTER, // PrefixFilter + PAGE_FILTER, // PageFilter + SKIP_FILTER, // SkipFilter + WHILE_FILTER, // WhileMatchFilter + KEY_ONLY_FILTER, // KeyOnlyFilter + FIRST_KEY_ONLY_FILTER, // FirstKeyOnlyFilter + MCP_FILTER, // MultipleColumnPrefixFilter + L_MCP_FILTER, // MultipleColumnPrefixFilter + DC_FILTER, // DependentColumnFilter + FAMILY_FILTER, // FamilyFilter + QUALIFIER_FILTER, // QualifierFilter + ROW_FILTER, // RowFilter + VALUE_FILTER, // ValueFilter + SCV_FILTER, // SingleColumnValueFilter + SCVE_FILTER, // SingleColumnValueExcludeFilter + AND_FILTER_LIST, // FilterList + OR_FILTER_LIST, // FilterList + L_FILTER_LIST, // FilterList + }; + + private static String[] FILTERS_INFO = { + STR_TS_FILTER, // TimestampsFilter + STR_L_TS_FILTER, // TimestampsFilter + STR_COL_PRE_FILTER, // ColumnPrefixFilter + STR_CP_FILTER, // ColumnPaginationFilter + STR_CR_FILTER, // ColumnRangeFilter + STR_CCG_FILTER, // ColumnCountGetFilter + STR_IS_FILTER, // InclusiveStopFilter + STR_PREFIX_FILTER, // PrefixFilter + STR_PAGE_FILTER, // PageFilter + STR_SKIP_FILTER, // SkipFilter + STR_WHILE_FILTER, // WhileMatchFilter + STR_KEY_ONLY_FILTER, // KeyOnlyFilter + STR_FIRST_KEY_ONLY_FILTER, // FirstKeyOnlyFilter + STR_MCP_FILTER, // MultipleColumnPrefixFilter + STR_L_MCP_FILTER, // MultipleColumnPrefixFilter + STR_DC_FILTER, // DependentColumnFilter + STR_FAMILY_FILTER, // FamilyFilter + STR_QUALIFIER_FILTER, // QualifierFilter + STR_ROW_FILTER, // RowFilter + STR_VALUE_FILTER, // ValueFilter + STR_SCV_FILTER, // SingleColumnValueFilter + STR_SCVE_FILTER, // SingleColumnValueExcludeFilter + STR_AND_FILTER_LIST, // FilterList + STR_OR_FILTER_LIST, // FilterList + STR_L_FILTER_LIST, // FilterList + }; + + static { + assertEquals("The sizes of static arrays do not match: " + + "[FILTERS: %d <=> FILTERS_INFO: %d]", + FILTERS.length, FILTERS_INFO.length); + } + + /** + * Test the client Operations' JSON encoding to ensure that produced JSON is + * parseable and that the details are present and not corrupted. + * @throws IOException + */ + @Test + public void testOperationJSON() + throws IOException { + // produce a Scan Operation + Scan scan = new Scan(ROW); + scan.addColumn(FAMILY, QUALIFIER); + // get its JSON representation, and parse it + String json = scan.toJSON(); + Map parsedJSON = mapper.readValue(json, HashMap.class); + // check for the row + assertEquals("startRow incorrect in Scan.toJSON()", + Bytes.toStringBinary(ROW), parsedJSON.get("startRow")); + // check for the family and the qualifier. + List familyInfo = (List) ((Map) parsedJSON.get("families")).get( + Bytes.toStringBinary(FAMILY)); + assertNotNull("Family absent in Scan.toJSON()", familyInfo); + assertEquals("Qualifier absent in Scan.toJSON()", 1, familyInfo.size()); + assertEquals("Qualifier incorrect in Scan.toJSON()", + Bytes.toStringBinary(QUALIFIER), + familyInfo.get(0)); + + // produce a Get Operation + Get get = new Get(ROW); + get.addColumn(FAMILY, QUALIFIER); + // get its JSON representation, and parse it + json = get.toJSON(); + parsedJSON = mapper.readValue(json, HashMap.class); + // check for the row + assertEquals("row incorrect in Get.toJSON()", + Bytes.toStringBinary(ROW), parsedJSON.get("row")); + // check for the family and the qualifier. + familyInfo = (List) ((Map) parsedJSON.get("families")).get( + Bytes.toStringBinary(FAMILY)); + assertNotNull("Family absent in Get.toJSON()", familyInfo); + assertEquals("Qualifier absent in Get.toJSON()", 1, familyInfo.size()); + assertEquals("Qualifier incorrect in Get.toJSON()", + Bytes.toStringBinary(QUALIFIER), + familyInfo.get(0)); + + // produce a Put operation + Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, VALUE); + // get its JSON representation, and parse it + json = put.toJSON(); + parsedJSON = mapper.readValue(json, HashMap.class); + // check for the row + assertEquals("row absent in Put.toJSON()", + Bytes.toStringBinary(ROW), parsedJSON.get("row")); + // check for the family and the qualifier. + familyInfo = (List) ((Map) parsedJSON.get("families")).get( + Bytes.toStringBinary(FAMILY)); + assertNotNull("Family absent in Put.toJSON()", familyInfo); + assertEquals("KeyValue absent in Put.toJSON()", 1, familyInfo.size()); + Map kvMap = (Map) familyInfo.get(0); + assertEquals("Qualifier incorrect in Put.toJSON()", + Bytes.toStringBinary(QUALIFIER), + kvMap.get("qualifier")); + assertEquals("Value length incorrect in Put.toJSON()", + VALUE.length, kvMap.get("vlen")); + + // produce a Delete operation + Delete delete = new Delete(ROW); + delete.deleteColumn(FAMILY, QUALIFIER); + // get its JSON representation, and parse it + json = delete.toJSON(); + parsedJSON = mapper.readValue(json, HashMap.class); + // check for the row + assertEquals("row absent in Delete.toJSON()", + Bytes.toStringBinary(ROW), parsedJSON.get("row")); + // check for the family and the qualifier. + familyInfo = (List) ((Map) parsedJSON.get("families")).get( + Bytes.toStringBinary(FAMILY)); + assertNotNull("Family absent in Delete.toJSON()", familyInfo); + assertEquals("KeyValue absent in Delete.toJSON()", 1, familyInfo.size()); + kvMap = (Map) familyInfo.get(0); + assertEquals("Qualifier incorrect in Delete.toJSON()", + Bytes.toStringBinary(QUALIFIER), kvMap.get("qualifier")); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestPutDotHas.java b/src/test/java/org/apache/hadoop/hbase/client/TestPutDotHas.java new file mode 100644 index 0000000..49cfcdc --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestPutDotHas.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +/** + * Addresses HBASE-6047 + * We test put.has call with all of its polymorphic magic + */ +public class TestPutDotHas { + + public static final byte[] ROW_01 = Bytes.toBytes("row-01"); + public static final byte[] QUALIFIER_01 = Bytes.toBytes("qualifier-01"); + public static final byte[] VALUE_01 = Bytes.toBytes("value-01"); + public static final byte[] FAMILY_01 = Bytes.toBytes("family-01"); + public static final long TS = 1234567L; + public Put put = new Put(ROW_01); + + @Before + public void setUp() { + put.add(FAMILY_01, QUALIFIER_01, TS, VALUE_01); + } + + @Test + public void testHasIgnoreValueIgnoreTS() { + Assert.assertTrue(put.has(FAMILY_01, QUALIFIER_01)); + Assert.assertFalse(put.has(QUALIFIER_01, FAMILY_01)); + } + + @Test + public void testHasIgnoreValue() { + Assert.assertTrue(put.has(FAMILY_01, QUALIFIER_01, TS)); + Assert.assertFalse(put.has(FAMILY_01, QUALIFIER_01, TS + 1)); + } + + @Test + public void testHasIgnoreTS() { + Assert.assertTrue(put.has(FAMILY_01, QUALIFIER_01, VALUE_01)); + Assert.assertFalse(put.has(FAMILY_01, VALUE_01, QUALIFIER_01)); + } + + @Test + public void testHas() { + Assert.assertTrue(put.has(FAMILY_01, QUALIFIER_01, TS, VALUE_01)); + // Bad TS + Assert.assertFalse(put.has(FAMILY_01, QUALIFIER_01, TS + 1, VALUE_01)); + // Bad Value + Assert.assertFalse(put.has(FAMILY_01, QUALIFIER_01, TS, QUALIFIER_01)); + // Bad Family + Assert.assertFalse(put.has(QUALIFIER_01, QUALIFIER_01, TS, VALUE_01)); + // Bad Qual + Assert.assertFalse(put.has(FAMILY_01, FAMILY_01, TS, VALUE_01)); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java new file mode 100644 index 0000000..f3af219 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java @@ -0,0 +1,299 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.MD5Hash; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test clone/restore snapshots from the client + */ +@Category(LargeTests.class) +public class TestRestoreSnapshotFromClient { + final Log LOG = LogFactory.getLog(getClass()); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private final byte[] FAMILY = Bytes.toBytes("cf"); + + private byte[] emptySnapshot; + private byte[] snapshotName0; + private byte[] snapshotName1; + private byte[] snapshotName2; + private int snapshot0Rows; + private int snapshot1Rows; + private byte[] tableName; + private HBaseAdmin admin; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true); + TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10); + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); + TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6); + TEST_UTIL.getConfiguration().setBoolean( + "hbase.master.enabletable.roundrobin", true); + TEST_UTIL.startMiniCluster(3); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Initialize the tests with a table filled with some data + * and two snapshots (snapshotName0, snapshotName1) of different states. + * The tableName, snapshotNames and the number of rows in the snapshot are initialized. + */ + @Before + public void setup() throws Exception { + this.admin = TEST_UTIL.getHBaseAdmin(); + + long tid = System.currentTimeMillis(); + tableName = Bytes.toBytes("testtb-" + tid); + emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid); + snapshotName0 = Bytes.toBytes("snaptb0-" + tid); + snapshotName1 = Bytes.toBytes("snaptb1-" + tid); + snapshotName2 = Bytes.toBytes("snaptb2-" + tid); + + // create Table and disable it + createTable(tableName, FAMILY); + admin.disableTable(tableName); + + // take an empty snapshot + admin.snapshot(emptySnapshot, tableName); + + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + try { + // enable table and insert data + admin.enableTable(tableName); + loadData(table, 500, FAMILY); + snapshot0Rows = TEST_UTIL.countRows(table); + admin.disableTable(tableName); + + // take a snapshot + admin.snapshot(snapshotName0, tableName); + + // enable table and insert more data + admin.enableTable(tableName); + loadData(table, 500, FAMILY); + snapshot1Rows = TEST_UTIL.countRows(table); + admin.disableTable(tableName); + + // take a snapshot of the updated table + admin.snapshot(snapshotName1, tableName); + + // re-enable table + admin.enableTable(tableName); + } finally { + table.close(); + } + } + + @After + public void tearDown() throws Exception { + if (admin.tableExists(tableName)) { + TEST_UTIL.deleteTable(tableName); + } + admin.deleteSnapshot(snapshotName0); + admin.deleteSnapshot(snapshotName1); + + // Ensure the archiver to be empty + MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); + mfs.getFileSystem().delete( + new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true); + } + + @Test + public void testRestoreSnapshot() throws IOException { + verifyRowCount(tableName, snapshot1Rows); + + // Restore from snapshot-0 + admin.disableTable(tableName); + admin.restoreSnapshot(snapshotName0); + admin.enableTable(tableName); + verifyRowCount(tableName, snapshot0Rows); + + // Restore from emptySnapshot + admin.disableTable(tableName); + admin.restoreSnapshot(emptySnapshot); + admin.enableTable(tableName); + verifyRowCount(tableName, 0); + + // Restore from snapshot-1 + admin.disableTable(tableName); + admin.restoreSnapshot(snapshotName1); + admin.enableTable(tableName); + verifyRowCount(tableName, snapshot1Rows); + } + + @Test + public void testRestoreSchemaChange() throws IOException { + byte[] TEST_FAMILY2 = Bytes.toBytes("cf2"); + + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + + // Add one column family and put some data in it + admin.disableTable(tableName); + admin.addColumn(tableName, new HColumnDescriptor(TEST_FAMILY2)); + admin.enableTable(tableName); + assertEquals(2, table.getTableDescriptor().getFamilies().size()); + HTableDescriptor htd = admin.getTableDescriptor(tableName); + assertEquals(2, htd.getFamilies().size()); + loadData(table, 500, TEST_FAMILY2); + long snapshot2Rows = snapshot1Rows + 500; + assertEquals(snapshot2Rows, TEST_UTIL.countRows(table)); + assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2)); + Set fsFamilies = getFamiliesFromFS(tableName); + assertEquals(2, fsFamilies.size()); + table.close(); + + // Take a snapshot + admin.disableTable(tableName); + admin.snapshot(snapshotName2, tableName); + + // Restore the snapshot (without the cf) + admin.restoreSnapshot(snapshotName0); + assertEquals(1, table.getTableDescriptor().getFamilies().size()); + admin.enableTable(tableName); + try { + TEST_UTIL.countRows(table, TEST_FAMILY2); + fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists"); + } catch (NoSuchColumnFamilyException e) { + // expected + } + assertEquals(snapshot0Rows, TEST_UTIL.countRows(table)); + htd = admin.getTableDescriptor(tableName); + assertEquals(1, htd.getFamilies().size()); + fsFamilies = getFamiliesFromFS(tableName); + assertEquals(1, fsFamilies.size()); + table.close(); + + // Restore back the snapshot (with the cf) + admin.disableTable(tableName); + admin.restoreSnapshot(snapshotName2); + admin.enableTable(tableName); + htd = admin.getTableDescriptor(tableName); + assertEquals(2, htd.getFamilies().size()); + assertEquals(2, table.getTableDescriptor().getFamilies().size()); + assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2)); + assertEquals(snapshot2Rows, TEST_UTIL.countRows(table)); + fsFamilies = getFamiliesFromFS(tableName); + assertEquals(2, fsFamilies.size()); + table.close(); + } + + @Test + public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException { + byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis()); + admin.cloneSnapshot(snapshotName0, clonedTableName); + verifyRowCount(clonedTableName, snapshot0Rows); + admin.disableTable(clonedTableName); + admin.snapshot(snapshotName2, clonedTableName); + admin.deleteTable(clonedTableName); + waitCleanerRun(); + + admin.cloneSnapshot(snapshotName2, clonedTableName); + verifyRowCount(clonedTableName, snapshot0Rows); + admin.disableTable(clonedTableName); + admin.deleteTable(clonedTableName); + } + + // ========================================================================== + // Helpers + // ========================================================================== + private void createTable(final byte[] tableName, final byte[]... families) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + for (byte[] family: families) { + HColumnDescriptor hcd = new HColumnDescriptor(family); + htd.addFamily(hcd); + } + byte[][] splitKeys = new byte[16][]; + byte[] hex = Bytes.toBytes("0123456789abcdef"); + for (int i = 0; i < 16; ++i) { + splitKeys[i] = new byte[] { hex[i] }; + } + admin.createTable(htd, splitKeys); + } + + public void loadData(final HTable table, int rows, byte[]... families) throws IOException { + byte[] qualifier = Bytes.toBytes("q"); + table.setAutoFlush(false); + while (rows-- > 0) { + byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows)); + byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value)); + Put put = new Put(key); + put.setWriteToWAL(false); + for (byte[] family: families) { + put.add(family, qualifier, value); + } + table.put(put); + } + table.flushCommits(); + } + + private void waitCleanerRun() throws InterruptedException { + TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting(); + } + + private Set getFamiliesFromFS(final byte[] tableName) throws IOException { + MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); + Set families = new HashSet(); + Path tableDir = HTableDescriptor.getTableDir(mfs.getRootDir(), tableName); + for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) { + for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) { + families.add(familyDir.getName()); + } + } + return families; + } + + private void verifyRowCount(final byte[] tableName, long expectedRows) throws IOException { + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + assertEquals(expectedRows, TEST_UTIL.countRows(table)); + table.close(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestResult.java b/src/test/java/org/apache/hadoop/hbase/client/TestResult.java new file mode 100644 index 0000000..f9e29c2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestResult.java @@ -0,0 +1,129 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +import static org.apache.hadoop.hbase.HBaseTestCase.assertByteEquals; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; + +@Category(SmallTests.class) +public class TestResult extends TestCase { + + static KeyValue[] genKVs(final byte[] row, final byte[] family, + final byte[] value, + final long timestamp, + final int cols) { + KeyValue [] kvs = new KeyValue[cols]; + + for (int i = 0; i < cols ; i++) { + kvs[i] = new KeyValue( + row, family, Bytes.toBytes(i), + timestamp, + Bytes.add(value, Bytes.toBytes(i))); + } + return kvs; + } + + static final byte [] row = Bytes.toBytes("row"); + static final byte [] family = Bytes.toBytes("family"); + static final byte [] value = Bytes.toBytes("value"); + + public void testBasic() throws Exception { + KeyValue [] kvs = genKVs(row, family, value, 1, 100); + + Arrays.sort(kvs, KeyValue.COMPARATOR); + + Result r = new Result(kvs); + + for (int i = 0; i < 100; ++i) { + final byte[] qf = Bytes.toBytes(i); + + List ks = r.getColumn(family, qf); + assertEquals(1, ks.size()); + assertByteEquals(qf, ks.get(0).getQualifier()); + + assertEquals(ks.get(0), r.getColumnLatest(family, qf)); + assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf)); + assertTrue(r.containsColumn(family, qf)); + } + } + public void testMultiVersion() throws Exception { + KeyValue [] kvs1 = genKVs(row, family, value, 1, 100); + KeyValue [] kvs2 = genKVs(row, family, value, 200, 100); + + KeyValue [] kvs = new KeyValue[kvs1.length+kvs2.length]; + System.arraycopy(kvs1, 0, kvs, 0, kvs1.length); + System.arraycopy(kvs2, 0, kvs, kvs1.length, kvs2.length); + + Arrays.sort(kvs, KeyValue.COMPARATOR); + + Result r = new Result(kvs); + for (int i = 0; i < 100; ++i) { + final byte[] qf = Bytes.toBytes(i); + + List ks = r.getColumn(family, qf); + assertEquals(2, ks.size()); + assertByteEquals(qf, ks.get(0).getQualifier()); + assertEquals(200, ks.get(0).getTimestamp()); + + assertEquals(ks.get(0), r.getColumnLatest(family, qf)); + assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf)); + assertTrue(r.containsColumn(family, qf)); + } + } + + /** + * Verify that Result.compareResults(...) behaves correctly. + */ + public void testCompareResults() throws Exception { + byte [] value1 = Bytes.toBytes("value1"); + byte [] qual = Bytes.toBytes("qual"); + + KeyValue kv1 = new KeyValue(row, family, qual, value); + KeyValue kv2 = new KeyValue(row, family, qual, value1); + + Result r1 = new Result(new KeyValue[] {kv1}); + Result r2 = new Result(new KeyValue[] {kv2}); + // no exception thrown + Result.compareResults(r1, r1); + try { + // these are different (HBASE-4800) + Result.compareResults(r1, r2); + fail(); + } catch (Exception x) { + assertTrue(x.getMessage().startsWith("This result was different:")); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestScan.java b/src/test/java/org/apache/hadoop/hbase/client/TestScan.java new file mode 100644 index 0000000..cdb40bb --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestScan.java @@ -0,0 +1,114 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +// TODO: cover more test cases +@Category(SmallTests.class) +public class TestScan { + @Test + public void testAttributesSerialization() throws IOException { + Scan scan = new Scan(); + scan.setAttribute("attribute1", Bytes.toBytes("value1")); + scan.setAttribute("attribute2", Bytes.toBytes("value2")); + scan.setAttribute("attribute3", Bytes.toBytes("value3")); + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DataOutput out = new DataOutputStream(byteArrayOutputStream); + scan.write(out); + + Scan scan2 = new Scan(); + Assert.assertTrue(scan2.getAttributesMap().isEmpty()); + + scan2.readFields(new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))); + + Assert.assertNull(scan2.getAttribute("absent")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), scan2.getAttribute("attribute1"))); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), scan2.getAttribute("attribute2"))); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value3"), scan2.getAttribute("attribute3"))); + Assert.assertEquals(3, scan2.getAttributesMap().size()); + } + + @Test + public void testScanAttributes() { + Scan scan = new Scan(); + Assert.assertTrue(scan.getAttributesMap().isEmpty()); + Assert.assertNull(scan.getAttribute("absent")); + + scan.setAttribute("absent", null); + Assert.assertTrue(scan.getAttributesMap().isEmpty()); + Assert.assertNull(scan.getAttribute("absent")); + + // adding attribute + scan.setAttribute("attribute1", Bytes.toBytes("value1")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), scan.getAttribute("attribute1"))); + Assert.assertEquals(1, scan.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), scan.getAttributesMap().get("attribute1"))); + + // overriding attribute value + scan.setAttribute("attribute1", Bytes.toBytes("value12")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), scan.getAttribute("attribute1"))); + Assert.assertEquals(1, scan.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), scan.getAttributesMap().get("attribute1"))); + + // adding another attribute + scan.setAttribute("attribute2", Bytes.toBytes("value2")); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), scan.getAttribute("attribute2"))); + Assert.assertEquals(2, scan.getAttributesMap().size()); + Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), scan.getAttributesMap().get("attribute2"))); + + // removing attribute + scan.setAttribute("attribute2", null); + Assert.assertNull(scan.getAttribute("attribute2")); + Assert.assertEquals(1, scan.getAttributesMap().size()); + Assert.assertNull(scan.getAttributesMap().get("attribute2")); + + // removing non-existed attribute + scan.setAttribute("attribute2", null); + Assert.assertNull(scan.getAttribute("attribute2")); + Assert.assertEquals(1, scan.getAttributesMap().size()); + Assert.assertNull(scan.getAttributesMap().get("attribute2")); + + // removing another attribute + scan.setAttribute("attribute1", null); + Assert.assertNull(scan.getAttribute("attribute1")); + Assert.assertTrue(scan.getAttributesMap().isEmpty()); + Assert.assertNull(scan.getAttributesMap().get("attribute1")); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java b/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java new file mode 100644 index 0000000..d4de897 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java @@ -0,0 +1,226 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test various scanner timeout issues. + */ +@Category(LargeTests.class) +public class TestScannerTimeout { + + private final static HBaseTestingUtility + TEST_UTIL = new HBaseTestingUtility(); + + final Log LOG = LogFactory.getLog(getClass()); + private final static byte[] SOME_BYTES = Bytes.toBytes("f"); + private final static byte[] TABLE_NAME = Bytes.toBytes("t"); + private final static int NB_ROWS = 10; + // Be careful w/ what you set this timer too... it can get in the way of + // the mini cluster coming up -- the verification in particular. + private final static int SCANNER_TIMEOUT = 15000; + private final static int SCANNER_CACHING = 5; + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration c = TEST_UTIL.getConfiguration(); + c.setInt("hbase.regionserver.lease.period", SCANNER_TIMEOUT); + // We need more than one region server for this test + TEST_UTIL.startMiniCluster(2); + HTable table = TEST_UTIL.createTable(TABLE_NAME, SOME_BYTES); + for (int i = 0; i < NB_ROWS; i++) { + Put put = new Put(Bytes.toBytes(i)); + put.add(SOME_BYTES, SOME_BYTES, SOME_BYTES); + table.put(put); + } + table.close(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + TEST_UTIL.ensureSomeNonStoppedRegionServersAvailable(2); + } + + /** + * Test that we do get a ScannerTimeoutException + * @throws Exception + */ + @Test(timeout=300000) + public void test2481() throws Exception { + LOG.info("START ************ test2481"); + Scan scan = new Scan(); + HTable table = + new HTable(new Configuration(TEST_UTIL.getConfiguration()), TABLE_NAME); + ResultScanner r = table.getScanner(scan); + int count = 0; + try { + Result res = r.next(); + while (res != null) { + count++; + if (count == 5) { + // Sleep just a bit more to be sure + Thread.sleep(SCANNER_TIMEOUT+100); + } + res = r.next(); + } + } catch (ScannerTimeoutException e) { + LOG.info("Got the timeout " + e.getMessage(), e); + return; + } finally { + table.close(); + } + fail("We should be timing out"); + LOG.info("END ************ test2481"); + } + + /** + * Test that scanner can continue even if the region server it was reading + * from failed. Before 2772, it reused the same scanner id. + * @throws Exception + */ + @Test(timeout=300000) + public void test2772() throws Exception { + LOG.info("START************ test2772"); + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME); + Scan scan = new Scan(); + // Set a very high timeout, we want to test what happens when a RS + // fails but the region is recovered before the lease times out. + // Since the RS is already created, this conf is client-side only for + // this new table + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + conf.setInt( + HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, SCANNER_TIMEOUT*100); + HTable higherScanTimeoutTable = new HTable(conf, TABLE_NAME); + ResultScanner r = higherScanTimeoutTable.getScanner(scan); + // This takes way less than SCANNER_TIMEOUT*100 + rs.abort("die!"); + Result[] results = r.next(NB_ROWS); + assertEquals(NB_ROWS, results.length); + r.close(); + higherScanTimeoutTable.close(); + LOG.info("END ************ test2772"); + + } + + /** + * Test that scanner won't miss any rows if the region server it was reading + * from failed. Before 3686, it would skip rows in the scan. + * @throws Exception + */ + @Test(timeout=300000) + public void test3686a() throws Exception { + LOG.info("START ************ TEST3686A---1"); + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME); + LOG.info("START ************ TEST3686A---1111"); + + Scan scan = new Scan(); + scan.setCaching(SCANNER_CACHING); + LOG.info("************ TEST3686A"); + MetaReader.fullScanMetaAndPrint(TEST_UTIL.getHBaseCluster().getMaster().getCatalogTracker()); + HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME); + LOG.info("START ************ TEST3686A---22"); + + ResultScanner r = table.getScanner(scan); + LOG.info("START ************ TEST3686A---33"); + + int count = 1; + r.next(); + LOG.info("START ************ TEST3686A---44"); + + // Kill after one call to next(), which got 5 rows. + rs.abort("die!"); + while(r.next() != null) { + count ++; + } + assertEquals(NB_ROWS, count); + r.close(); + table.close(); + LOG.info("************ END TEST3686A"); + } + + /** + * Make sure that no rows are lost if the scanner timeout is longer on the + * client than the server, and the scan times out on the server but not the + * client. + * @throws Exception + */ + @Test(timeout=300000) + public void test3686b() throws Exception { + LOG.info("START ************ test3686b"); + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME); + Scan scan = new Scan(); + scan.setCaching(SCANNER_CACHING); + // Set a very high timeout, we want to test what happens when a RS + // fails but the region is recovered before the lease times out. + // Since the RS is already created, this conf is client-side only for + // this new table + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + conf.setInt( + HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, SCANNER_TIMEOUT*100); + HTable higherScanTimeoutTable = new HTable(conf, TABLE_NAME); + ResultScanner r = higherScanTimeoutTable.getScanner(scan); + int count = 1; + r.next(); + // Sleep, allowing the scan to timeout on the server but not on the client. + Thread.sleep(SCANNER_TIMEOUT+2000); + while(r.next() != null) { + count ++; + } + assertEquals(NB_ROWS, count); + r.close(); + higherScanTimeoutTable.close(); + LOG.info("END ************ END test3686b"); + + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestShell.java b/src/test/java/org/apache/hadoop/hbase/client/TestShell.java new file mode 100644 index 0000000..624913b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestShell.java @@ -0,0 +1,77 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.LargeTests; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.jruby.embed.ScriptingContainer; +import org.jruby.embed.PathType; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestShell { + final Log LOG = LogFactory.getLog(getClass()); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static ScriptingContainer jruby = new ScriptingContainer(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Start mini cluster + TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true); + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); + TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6); + TEST_UTIL.startMiniCluster(); + + // Configure jruby runtime + List loadPaths = new ArrayList(); + loadPaths.add("src/main/ruby"); + loadPaths.add("src/test/ruby"); + jruby.getProvider().setLoadPaths(loadPaths); + jruby.put("$TEST_CLUSTER", TEST_UTIL); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testRunShellTests() throws IOException { + // Start all ruby tests + jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb"); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java new file mode 100644 index 0000000..af9ad98 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.ipc.HMasterInterface; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.snapshot.HSnapshotDescription; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import com.google.protobuf.RpcController; + +/** + * Test snapshot logic from the client + */ +@Category(SmallTests.class) +public class TestSnapshotFromAdmin { + + private static final Log LOG = LogFactory.getLog(TestSnapshotFromAdmin.class); + + /** + * Test that the logic for doing 'correct' back-off based on exponential increase and the max-time + * passed from the server ensures the correct overall waiting for the snapshot to finish. + * @throws Exception + */ + @Test(timeout = 60000) + public void testBackoffLogic() throws Exception { + final int maxWaitTime = 7500; + final int numRetries = 10; + final int pauseTime = 500; + // calculate the wait time, if we just do straight backoff (ignoring the expected time from + // master) + long ignoreExpectedTime = 0; + for (int i = 0; i < 6; i++) { + ignoreExpectedTime += HConstants.RETRY_BACKOFF[i] * pauseTime; + } + // the correct wait time, capping at the maxTime/tries + fudge room + final long time = pauseTime * 3 + ((maxWaitTime / numRetries) * 3) + 300; + assertTrue("Capped snapshot wait time isn't less that the uncapped backoff time " + + "- further testing won't prove anything.", time < ignoreExpectedTime); + + // setup the mocks + HConnectionManager.HConnectionImplementation mockConnection = Mockito + .mock(HConnectionManager.HConnectionImplementation.class); + Configuration conf = HBaseConfiguration.create(); + // setup the conf to match the expected properties + conf.setInt("hbase.client.retries.number", numRetries); + conf.setLong("hbase.client.pause", pauseTime); + // mock the master admin to our mock + HMasterInterface mockMaster = Mockito.mock(HMasterInterface.class); + Mockito.when(mockConnection.getConfiguration()).thenReturn(conf); + Mockito.when(mockConnection.getMaster()).thenReturn(mockMaster); + // set the max wait time for the snapshot to complete + Mockito + .when( + mockMaster.snapshot( + Mockito.any(HSnapshotDescription.class))).thenReturn((long)maxWaitTime); + + // first five times, we return false, last we get success + Mockito.when( + mockMaster.isSnapshotDone( + Mockito.any(HSnapshotDescription.class))).thenReturn(false, false, + false, false, false, true); + + // setup the admin and run the test + HBaseAdmin admin = new HBaseAdmin(mockConnection); + String snapshot = "snapshot"; + String table = "table"; + // get start time + long start = System.currentTimeMillis(); + admin.snapshot(snapshot, table); + long finish = System.currentTimeMillis(); + long elapsed = (finish - start); + assertTrue("Elapsed time:" + elapsed + " is more than expected max:" + time, elapsed <= time); + admin.close(); + } + + /** + * Make sure that we validate the snapshot name and the table name before we pass anything across + * the wire + * @throws Exception on failure + */ + @Test + public void testValidateSnapshotName() throws Exception { + HConnectionManager.HConnectionImplementation mockConnection = Mockito + .mock(HConnectionManager.HConnectionImplementation.class); + Configuration conf = HBaseConfiguration.create(); + Mockito.when(mockConnection.getConfiguration()).thenReturn(conf); + HBaseAdmin admin = new HBaseAdmin(mockConnection); + SnapshotDescription.Builder builder = SnapshotDescription.newBuilder(); + // check that invalid snapshot names fail + failSnapshotStart(admin, builder.setName(HConstants.SNAPSHOT_DIR_NAME).build()); + failSnapshotStart(admin, builder.setName("-snapshot").build()); + failSnapshotStart(admin, builder.setName("snapshot fails").build()); + failSnapshotStart(admin, builder.setName("snap$hot").build()); + // check the table name also get verified + failSnapshotStart(admin, builder.setName("snapshot").setTable(".table").build()); + failSnapshotStart(admin, builder.setName("snapshot").setTable("-table").build()); + failSnapshotStart(admin, builder.setName("snapshot").setTable("table fails").build()); + failSnapshotStart(admin, builder.setName("snapshot").setTable("tab%le").build()); + + // mock the master connection + HMasterInterface master = Mockito.mock(HMasterInterface.class); + Mockito.when(mockConnection.getMaster()).thenReturn(master); + + Mockito.when( + master.snapshot(Mockito.any(HSnapshotDescription.class))).thenReturn((long)0); + Mockito.when( + master.isSnapshotDone( + Mockito.any(HSnapshotDescription.class))).thenReturn(true); + + // make sure that we can use valid names + admin.snapshot(builder.setName("snapshot").setTable("table").build()); + } + + private void failSnapshotStart(HBaseAdmin admin, SnapshotDescription snapshot) throws IOException { + try { + admin.snapshot(snapshot); + fail("Snapshot should not have succeed with name:" + snapshot.getName()); + } catch (IllegalArgumentException e) { + LOG.debug("Correctly failed to start snapshot:" + e.getMessage()); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java new file mode 100644 index 0000000..e1b5ff8 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java @@ -0,0 +1,231 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.snapshot.HSnapshotDescription; +import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test create/using/deleting snapshots from the client + *

    + * This is an end-to-end test for the snapshot utility + */ +@Category(LargeTests.class) +public class TestSnapshotFromClient { + private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final int NUM_RS = 2; + private static final String STRING_TABLE_NAME = "test"; + private static final byte[] TEST_FAM = Bytes.toBytes("fam"); + private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME); + + /** + * Setup the config for the cluster + * @throws Exception on failure + */ + @BeforeClass + public static void setupCluster() throws Exception { + setupConf(UTIL.getConfiguration()); + UTIL.startMiniCluster(NUM_RS); + } + + private static void setupConf(Configuration conf) { + // disable the ui + conf.setInt("hbase.regionsever.info.port", -1); + // change the flush size to a small amount, regulating number of store files + conf.setInt("hbase.hregion.memstore.flush.size", 25000); + // so make sure we get a compaction when doing a load, but keep around some + // files in the store + conf.setInt("hbase.hstore.compaction.min", 10); + conf.setInt("hbase.hstore.compactionThreshold", 10); + // block writes if we get to 12 store files + conf.setInt("hbase.hstore.blockingStoreFiles", 12); + // drop the number of attempts for the hbase admin + conf.setInt("hbase.client.retries.number", 1); + // Enable snapshot + conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + // prevent aggressive region split + conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, + ConstantSizeRegionSplitPolicy.class.getName()); + } + + @Before + public void setup() throws Exception { + UTIL.createTable(TABLE_NAME, TEST_FAM); + } + + @After + public void tearDown() throws Exception { + UTIL.deleteTable(TABLE_NAME); + // and cleanup the archive directory + try { + UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true); + } catch (IOException e) { + LOG.warn("Failure to delete archive directory", e); + } + } + + @AfterClass + public static void cleanupTest() throws Exception { + try { + UTIL.shutdownMiniCluster(); + } catch (Exception e) { + LOG.warn("failure shutting down cluster", e); + } + } + + /** + * Test snapshotting not allowed .META. and -ROOT- + * @throws Exception + */ + @Test + public void testMetaTablesSnapshot() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + byte[] snapshotName = Bytes.toBytes("metaSnapshot"); + + try { + admin.snapshot(snapshotName, HConstants.META_TABLE_NAME); + fail("taking a snapshot of .META. should not be allowed"); + } catch (IllegalArgumentException e) { + // expected + } + + try { + admin.snapshot(snapshotName, HConstants.ROOT_TABLE_NAME); + fail("taking a snapshot of -ROOT- should not be allowed"); + } catch (IllegalArgumentException e) { + // expected + } + } + + /** + * Test snapshotting a table that is offline + * @throws Exception + */ + @Test + public void testOfflineTableSnapshot() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + // make sure we don't fail on listing snapshots + SnapshotTestingUtils.assertNoSnapshots(admin); + + // put some stuff in the table + HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME); + UTIL.loadTable(table, TEST_FAM); + + // get the name of all the regionservers hosting the snapshotted table + Set snapshotServers = new HashSet(); + List servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + for (RegionServerThread server : servers) { + if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) { + snapshotServers.add(server.getRegionServer().getServerName().toString()); + } + } + + LOG.debug("FS state before disable:"); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), + FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + // XXX if this is flakey, might want to consider using the async version and looping as + // disableTable can succeed and still timeout. + admin.disableTable(TABLE_NAME); + + LOG.debug("FS state before snapshot:"); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), + FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + + // take a snapshot of the disabled table + byte[] snapshot = Bytes.toBytes("offlineTableSnapshot"); + admin.snapshot(snapshot, TABLE_NAME); + LOG.debug("Snapshot completed."); + + // make sure we have the snapshot + List snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, + snapshot, TABLE_NAME); + + // make sure its a valid snapshot + FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); + Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); + LOG.debug("FS state after snapshot:"); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), + FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + + SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, + admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers); + + admin.deleteSnapshot(snapshot); + snapshots = admin.listSnapshots(); + SnapshotTestingUtils.assertNoSnapshots(admin); + } + + @Test + public void testSnapshotFailsOnNonExistantTable() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + // make sure we don't fail on listing snapshots + SnapshotTestingUtils.assertNoSnapshots(admin); + String tableName = "_not_a_table"; + + // make sure the table doesn't exist + boolean fail = false; + do { + try { + admin.getTableDescriptor(Bytes.toBytes(tableName)); + fail = true; + LOG.error("Table:" + tableName + " already exists, checking a new name"); + tableName = tableName+"!"; + } catch (TableNotFoundException e) { + fail = false; + } + } while (fail); + + // snapshot the non-existant table + try { + admin.snapshot("fail", tableName); + fail("Snapshot succeeded even though there is not table."); + } catch (SnapshotCreationException e) { + LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage()); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java new file mode 100644 index 0000000..2c1c85e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotsFromAdmin.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.ipc.HMasterInterface; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.snapshot.HSnapshotDescription; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import com.google.protobuf.RpcController; + +/** + * Test snapshot logic from the client + */ +@Category(SmallTests.class) +public class TestSnapshotsFromAdmin { + + private static final Log LOG = LogFactory.getLog(TestSnapshotsFromAdmin.class); + + /** + * Test that the logic for doing 'correct' back-off based on exponential increase and the max-time + * passed from the server ensures the correct overall waiting for the snapshot to finish. + * @throws Exception + */ + @Test(timeout = 60000) + public void testBackoffLogic() throws Exception { + final int maxWaitTime = 7500; + final int numRetries = 10; + final int pauseTime = 500; + // calculate the wait time, if we just do straight backoff (ignoring the expected time from + // master) + long ignoreExpectedTime = 0; + for (int i = 0; i < 6; i++) { + ignoreExpectedTime += HConstants.RETRY_BACKOFF[i] * pauseTime; + } + // the correct wait time, capping at the maxTime/tries + fudge room + final long time = pauseTime * 3 + ((maxWaitTime / numRetries) * 3) + 300; + assertTrue("Capped snapshot wait time isn't less that the uncapped backoff time " + + "- further testing won't prove anything.", time < ignoreExpectedTime); + + // setup the mocks + HConnectionManager.HConnectionImplementation mockConnection = Mockito + .mock(HConnectionManager.HConnectionImplementation.class); + Configuration conf = HBaseConfiguration.create(); + // setup the conf to match the expected properties + conf.setInt("hbase.client.retries.number", numRetries); + conf.setLong("hbase.client.pause", pauseTime); + // mock the master admin to our mock + HMasterInterface mockMaster = Mockito.mock(HMasterInterface.class); + Mockito.when(mockConnection.getConfiguration()).thenReturn(conf); + Mockito.when(mockConnection.getMaster()).thenReturn(mockMaster); + // set the max wait time for the snapshot to complete + Mockito + .when( + mockMaster.snapshot( + Mockito.any(HSnapshotDescription.class))).thenReturn((long)maxWaitTime); + // first five times, we return false, last we get success + Mockito.when( + mockMaster.isSnapshotDone( + Mockito.any(HSnapshotDescription.class))).thenReturn(false, false, + false, false, false, true); + + // setup the admin and run the test + HBaseAdmin admin = new HBaseAdmin(mockConnection); + String snapshot = "snasphot"; + String table = "table"; + // get start time + long start = System.currentTimeMillis(); + admin.snapshot(snapshot, table); + long finish = System.currentTimeMillis(); + long elapsed = (finish - start); + assertTrue("Elapsed time:" + elapsed + " is more than expected max:" + time, elapsed <= time); + } + + /** + * Make sure that we validate the snapshot name and the table name before we pass anything across + * the wire + * @throws IOException on failure + */ + @Test + public void testValidateSnapshotName() throws IOException { + HConnectionManager.HConnectionImplementation mockConnection = Mockito + .mock(HConnectionManager.HConnectionImplementation.class); + Configuration conf = HBaseConfiguration.create(); + Mockito.when(mockConnection.getConfiguration()).thenReturn(conf); + HBaseAdmin admin = new HBaseAdmin(mockConnection); + SnapshotDescription.Builder builder = SnapshotDescription.newBuilder(); + // check that invalid snapshot names fail + failSnapshotStart(admin, builder.setName(HConstants.SNAPSHOT_DIR_NAME).build()); + failSnapshotStart(admin, builder.setName("-snapshot").build()); + failSnapshotStart(admin, builder.setName("snapshot fails").build()); + failSnapshotStart(admin, builder.setName("snap$hot").build()); + // check the table name also get verified + failSnapshotStart(admin, builder.setName("snapshot").setTable(".table").build()); + failSnapshotStart(admin, builder.setName("snapshot").setTable("-table").build()); + failSnapshotStart(admin, builder.setName("snapshot").setTable("table fails").build()); + failSnapshotStart(admin, builder.setName("snapshot").setTable("tab%le").build()); + } + + private void failSnapshotStart(HBaseAdmin admin, SnapshotDescription snapshot) throws IOException { + try { + admin.snapshot(snapshot); + fail("Snapshot should not have succeed with name:" + snapshot.getName()); + } catch (IllegalArgumentException e) { + LOG.debug("Correctly failed to start snapshot:" + e.getMessage()); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestTimestampsFilter.java b/src/test/java/org/apache/hadoop/hbase/client/TestTimestampsFilter.java new file mode 100644 index 0000000..837ce28 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/TestTimestampsFilter.java @@ -0,0 +1,390 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.TimestampsFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Run tests related to {@link TimestampsFilter} using HBase client APIs. + * Sets up the HBase mini cluster once at start. Each creates a table + * named for the method and does its stuff against that. + */ +@Category(MediumTests.class) +public class TestTimestampsFilter { + final Log LOG = LogFactory.getLog(getClass()); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + // Nothing to do. + } + + /** + * @throws java.lang.Exception + */ + @After + public void tearDown() throws Exception { + // Nothing to do. + } + + /** + * Test from client side for TimestampsFilter. + * + * The TimestampsFilter provides the ability to request cells (KeyValues) + * whose timestamp/version is in the specified list of timestamps/version. + * + * @throws Exception + */ + @Test + public void testTimestampsFilter() throws Exception { + byte [] TABLE = Bytes.toBytes("testTimestampsFilter"); + byte [] FAMILY = Bytes.toBytes("event_log"); + byte [][] FAMILIES = new byte[][] { FAMILY }; + KeyValue kvs[]; + + // create table; set versions to max... + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); + + for (int rowIdx = 0; rowIdx < 5; rowIdx++) { + for (int colIdx = 0; colIdx < 5; colIdx++) { + // insert versions 201..300 + putNVersions(ht, FAMILY, rowIdx, colIdx, 201, 300); + // insert versions 1..100 + putNVersions(ht, FAMILY, rowIdx, colIdx, 1, 100); + } + } + + // do some verification before flush + verifyInsertedValues(ht, FAMILY); + + TEST_UTIL.flush(); + + // do some verification after flush + verifyInsertedValues(ht, FAMILY); + + // Insert some more versions after flush. These should be in memstore. + // After this we should have data in both memstore & HFiles. + for (int rowIdx = 0; rowIdx < 5; rowIdx++) { + for (int colIdx = 0; colIdx < 5; colIdx++) { + putNVersions(ht, FAMILY, rowIdx, colIdx, 301, 400); + putNVersions(ht, FAMILY, rowIdx, colIdx, 101, 200); + } + } + + for (int rowIdx = 0; rowIdx < 5; rowIdx++) { + for (int colIdx = 0; colIdx < 5; colIdx++) { + kvs = getNVersions(ht, FAMILY, rowIdx, colIdx, + Arrays.asList(505L, 5L, 105L, 305L, 205L)); + assertEquals(4, kvs.length); + checkOneCell(kvs[0], FAMILY, rowIdx, colIdx, 305); + checkOneCell(kvs[1], FAMILY, rowIdx, colIdx, 205); + checkOneCell(kvs[2], FAMILY, rowIdx, colIdx, 105); + checkOneCell(kvs[3], FAMILY, rowIdx, colIdx, 5); + } + } + + // Request an empty list of versions using the Timestamps filter; + // Should return none. + kvs = getNVersions(ht, FAMILY, 2, 2, new ArrayList()); + assertEquals(0, kvs.length); + + // + // Test the filter using a Scan operation + // Scan rows 0..4. For each row, get all its columns, but only + // those versions of the columns with the specified timestamps. + Result[] results = scanNVersions(ht, FAMILY, 0, 4, + Arrays.asList(6L, 106L, 306L)); + assertEquals("# of rows returned from scan", 5, results.length); + for (int rowIdx = 0; rowIdx < 5; rowIdx++) { + kvs = results[rowIdx].raw(); + // each row should have 5 columns. + // And we have requested 3 versions for each. + assertEquals("Number of KeyValues in result for row:" + rowIdx, + 3*5, kvs.length); + for (int colIdx = 0; colIdx < 5; colIdx++) { + int offset = colIdx * 3; + checkOneCell(kvs[offset + 0], FAMILY, rowIdx, colIdx, 306); + checkOneCell(kvs[offset + 1], FAMILY, rowIdx, colIdx, 106); + checkOneCell(kvs[offset + 2], FAMILY, rowIdx, colIdx, 6); + } + } + ht.close(); + } + + @Test + public void testMultiColumns() throws Exception { + byte [] TABLE = Bytes.toBytes("testTimestampsFilterMultiColumns"); + byte [] FAMILY = Bytes.toBytes("event_log"); + byte [][] FAMILIES = new byte[][] { FAMILY }; + KeyValue kvs[]; + + // create table; set versions to max... + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); + + Put p = new Put(Bytes.toBytes("row")); + p.add(FAMILY, Bytes.toBytes("column0"), 3, Bytes.toBytes("value0-3")); + p.add(FAMILY, Bytes.toBytes("column1"), 3, Bytes.toBytes("value1-3")); + p.add(FAMILY, Bytes.toBytes("column2"), 1, Bytes.toBytes("value2-1")); + p.add(FAMILY, Bytes.toBytes("column2"), 2, Bytes.toBytes("value2-2")); + p.add(FAMILY, Bytes.toBytes("column2"), 3, Bytes.toBytes("value2-3")); + p.add(FAMILY, Bytes.toBytes("column3"), 2, Bytes.toBytes("value3-2")); + p.add(FAMILY, Bytes.toBytes("column4"), 1, Bytes.toBytes("value4-1")); + p.add(FAMILY, Bytes.toBytes("column4"), 2, Bytes.toBytes("value4-2")); + p.add(FAMILY, Bytes.toBytes("column4"), 3, Bytes.toBytes("value4-3")); + ht.put(p); + + ArrayList timestamps = new ArrayList(); + timestamps.add(new Long(3)); + TimestampsFilter filter = new TimestampsFilter(timestamps); + + Get g = new Get(Bytes.toBytes("row")); + g.setFilter(filter); + g.setMaxVersions(); + g.addColumn(FAMILY, Bytes.toBytes("column2")); + g.addColumn(FAMILY, Bytes.toBytes("column4")); + + Result result = ht.get(g); + for (KeyValue kv : result.list()) { + System.out.println("found row " + Bytes.toString(kv.getRow()) + + ", column " + Bytes.toString(kv.getQualifier()) + ", value " + + Bytes.toString(kv.getValue())); + } + + assertEquals(result.list().size(), 2); + assertEquals(Bytes.toString(result.list().get(0).getValue()), + "value2-3"); + assertEquals(Bytes.toString(result.list().get(1).getValue()), + "value4-3"); + + ht.close(); + } + + /** + * Test TimestampsFilter in the presence of version deletes. + * + * @throws Exception + */ + @Test + public void testWithVersionDeletes() throws Exception { + + // first test from memstore (without flushing). + testWithVersionDeletes(false); + + // run same test against HFiles (by forcing a flush). + testWithVersionDeletes(true); + } + + private void testWithVersionDeletes(boolean flushTables) throws IOException { + byte [] TABLE = Bytes.toBytes("testWithVersionDeletes_" + + (flushTables ? "flush" : "noflush")); + byte [] FAMILY = Bytes.toBytes("event_log"); + byte [][] FAMILIES = new byte[][] { FAMILY }; + + // create table; set versions to max... + HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); + + // For row:0, col:0: insert versions 1 through 5. + putNVersions(ht, FAMILY, 0, 0, 1, 5); + + // delete version 4. + deleteOneVersion(ht, FAMILY, 0, 0, 4); + + if (flushTables) { + TEST_UTIL.flush(); + } + + // request a bunch of versions including the deleted version. We should + // only get back entries for the versions that exist. + KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L, 4L, 5L)); + assertEquals(3, kvs.length); + checkOneCell(kvs[0], FAMILY, 0, 0, 5); + checkOneCell(kvs[1], FAMILY, 0, 0, 3); + checkOneCell(kvs[2], FAMILY, 0, 0, 2); + + ht.close(); + } + + private void verifyInsertedValues(HTable ht, byte[] cf) throws IOException { + for (int rowIdx = 0; rowIdx < 5; rowIdx++) { + for (int colIdx = 0; colIdx < 5; colIdx++) { + // ask for versions that exist. + KeyValue[] kvs = getNVersions(ht, cf, rowIdx, colIdx, + Arrays.asList(5L, 300L, 6L, 80L)); + assertEquals(4, kvs.length); + checkOneCell(kvs[0], cf, rowIdx, colIdx, 300); + checkOneCell(kvs[1], cf, rowIdx, colIdx, 80); + checkOneCell(kvs[2], cf, rowIdx, colIdx, 6); + checkOneCell(kvs[3], cf, rowIdx, colIdx, 5); + + // ask for versions that do not exist. + kvs = getNVersions(ht, cf, rowIdx, colIdx, + Arrays.asList(101L, 102L)); + assertEquals(0, kvs.length); + + // ask for some versions that exist and some that do not. + kvs = getNVersions(ht, cf, rowIdx, colIdx, + Arrays.asList(1L, 300L, 105L, 70L, 115L)); + assertEquals(3, kvs.length); + checkOneCell(kvs[0], cf, rowIdx, colIdx, 300); + checkOneCell(kvs[1], cf, rowIdx, colIdx, 70); + checkOneCell(kvs[2], cf, rowIdx, colIdx, 1); + } + } + } + + /** + * Assert that the passed in KeyValue has expected contents for the + * specified row, column & timestamp. + */ + private void checkOneCell(KeyValue kv, byte[] cf, + int rowIdx, int colIdx, long ts) { + + String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts; + + assertEquals("Row mismatch which checking: " + ctx, + "row:"+ rowIdx, Bytes.toString(kv.getRow())); + + assertEquals("ColumnFamily mismatch while checking: " + ctx, + Bytes.toString(cf), Bytes.toString(kv.getFamily())); + + assertEquals("Column qualifier mismatch while checking: " + ctx, + "column:" + colIdx, + Bytes.toString(kv.getQualifier())); + + assertEquals("Timestamp mismatch while checking: " + ctx, + ts, kv.getTimestamp()); + + assertEquals("Value mismatch while checking: " + ctx, + "value-version-" + ts, Bytes.toString(kv.getValue())); + } + + /** + * Uses the TimestampFilter on a Get to request a specified list of + * versions for the row/column specified by rowIdx & colIdx. + * + */ + private KeyValue[] getNVersions(HTable ht, byte[] cf, int rowIdx, + int colIdx, List versions) + throws IOException { + byte row[] = Bytes.toBytes("row:" + rowIdx); + byte column[] = Bytes.toBytes("column:" + colIdx); + Filter filter = new TimestampsFilter(versions); + Get get = new Get(row); + get.addColumn(cf, column); + get.setFilter(filter); + get.setMaxVersions(); + Result result = ht.get(get); + + return result.raw(); + } + + /** + * Uses the TimestampFilter on a Scan to request a specified list of + * versions for the rows from startRowIdx to endRowIdx (both inclusive). + */ + private Result[] scanNVersions(HTable ht, byte[] cf, int startRowIdx, + int endRowIdx, List versions) + throws IOException { + byte startRow[] = Bytes.toBytes("row:" + startRowIdx); + byte endRow[] = Bytes.toBytes("row:" + endRowIdx + 1); // exclusive + Filter filter = new TimestampsFilter(versions); + Scan scan = new Scan(startRow, endRow); + scan.setFilter(filter); + scan.setMaxVersions(); + ResultScanner scanner = ht.getScanner(scan); + return scanner.next(endRowIdx - startRowIdx + 1); + } + + /** + * Insert in specific row/column versions with timestamps + * versionStart..versionEnd. + */ + private void putNVersions(HTable ht, byte[] cf, int rowIdx, int colIdx, + long versionStart, long versionEnd) + throws IOException { + byte row[] = Bytes.toBytes("row:" + rowIdx); + byte column[] = Bytes.toBytes("column:" + colIdx); + Put put = new Put(row); + put.setWriteToWAL(false); + + for (long idx = versionStart; idx <= versionEnd; idx++) { + put.add(cf, column, idx, Bytes.toBytes("value-version-" + idx)); + } + + ht.put(put); + } + + /** + * For row/column specified by rowIdx/colIdx, delete the cell + * corresponding to the specified version. + */ + private void deleteOneVersion(HTable ht, byte[] cf, int rowIdx, + int colIdx, long version) + throws IOException { + byte row[] = Bytes.toBytes("row:" + rowIdx); + byte column[] = Bytes.toBytes("column:" + colIdx); + Delete del = new Delete(row); + del.deleteColumn(cf, column, version); + ht.delete(del); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + + diff --git a/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java b/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java new file mode 100644 index 0000000..6dabc27 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client.replication; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.fail; +import static org.junit.Assert.assertEquals; + +/** + * Unit testing of ReplicationAdmin + */ +@Category(MediumTests.class) +public class TestReplicationAdmin { + + private static final Log LOG = + LogFactory.getLog(TestReplicationAdmin.class); + private final static HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + private final String ID_ONE = "1"; + private final String KEY_ONE = "127.0.0.1:2181:/hbase"; + private final String ID_SECOND = "2"; + private final String KEY_SECOND = "127.0.0.1:2181:/hbase2"; + + private static ReplicationSourceManager manager; + private static ReplicationAdmin admin; + private static AtomicBoolean replicating = new AtomicBoolean(true); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true); + admin = new ReplicationAdmin(conf); + Path oldLogDir = new Path(TEST_UTIL.getDataTestDir(), + HConstants.HREGION_OLDLOGDIR_NAME); + Path logDir = new Path(TEST_UTIL.getDataTestDir(), + HConstants.HREGION_LOGDIR_NAME); + manager = new ReplicationSourceManager(admin.getReplicationZk(), conf, + // The following stopper never stops so that we can respond + // to zk notification + new Stoppable() { + @Override + public void stop(String why) {} + @Override + public boolean isStopped() {return false;} + }, FileSystem.get(conf), replicating, logDir, oldLogDir); + } + + /** + * Simple testing of adding and removing peers, basically shows that + * all interactions with ZK work + * @throws Exception + */ + @Test + public void testAddRemovePeer() throws Exception { + assertEquals(0, manager.getSources().size()); + // Add a valid peer + admin.addPeer(ID_ONE, KEY_ONE); + // try adding the same (fails) + try { + admin.addPeer(ID_ONE, KEY_ONE); + } catch (IllegalArgumentException iae) { + // OK! + } + assertEquals(1, admin.getPeersCount()); + // Try to remove an inexisting peer + try { + admin.removePeer(ID_SECOND); + fail(); + } catch (IllegalArgumentException iae) { + // OK! + } + assertEquals(1, admin.getPeersCount()); + // Add a second since multi-slave is supported + try { + admin.addPeer(ID_SECOND, KEY_SECOND); + } catch (IllegalStateException iae) { + fail(); + // OK! + } + assertEquals(2, admin.getPeersCount()); + // Remove the first peer we added + admin.removePeer(ID_ONE); + assertEquals(1, admin.getPeersCount()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/AllFailConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/AllFailConstraint.java new file mode 100644 index 0000000..03fec35 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/AllFailConstraint.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.client.Put; + +/** + * Always fail the put. + */ +public class AllFailConstraint extends BaseConstraint { + + @Override + public void check(Put p) throws ConstraintException { + throw new ConstraintException("AllFailConstraint fails for all puts"); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/AllPassConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/AllPassConstraint.java new file mode 100644 index 0000000..a33dfdb --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/AllPassConstraint.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.client.Put; + +/** + * Simple test constraint that always allows the put to pass + */ +public class AllPassConstraint extends BaseConstraint { + + @Override + public void check(Put p) { + // Do nothing - it passes + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/CheckConfigurationConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/CheckConfigurationConstraint.java new file mode 100644 index 0000000..c7b0604 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/CheckConfigurationConstraint.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Put; + +/** + * Test Constraint to check to make sure the configuration is set + */ +public class CheckConfigurationConstraint extends BaseConstraint { + + private static String key = "testKey"; + private static String value = "testValue"; + + public static Configuration getConfiguration() { + Configuration conf = new Configuration(false); + conf.set(key, value); + return conf; + } + + @Override + public void check(Put p) { + // NOOP + } + + @Override + public void setConf(Configuration conf) { + super.setConf(conf); + if (conf != null) { + String val = conf.get(key); + if (val == null || !val.equals(value)) + throw new IllegalArgumentException( + "Configuration was not passed correctly"); + // and then check to make sure we got a fresh config by checking for a + // hadoop-based config value, and if we don't find it, its fine + if (conf.getRaw("fs.file.impl") != null) + throw new IllegalArgumentException( + "Configuration was created using 'new Configuration()', should be " + + "done via 'new Configuration(false) to exclude defaut hadoop " + + "configurations values."); + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/RuntimeFailConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/RuntimeFailConstraint.java new file mode 100644 index 0000000..64f6e6d --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/RuntimeFailConstraint.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.client.Put; + +/** + * Always non-gracefully fail on attempt + */ +public class RuntimeFailConstraint extends BaseConstraint { + + @Override + public void check(Put p) throws ConstraintException { + throw new RuntimeException( + "RuntimeFailConstraint always throws a runtime exception"); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java new file mode 100644 index 0000000..e0f2569 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java @@ -0,0 +1,262 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Do the complex testing of constraints against a minicluster + */ +@Category(MediumTests.class) +public class TestConstraint { + private static final Log LOG = LogFactory + .getLog(TestConstraint.class); + + private static HBaseTestingUtility util; + private static final byte[] tableName = Bytes.toBytes("test"); + private static final byte[] dummy = Bytes.toBytes("dummy"); + private static final byte[] row1 = Bytes.toBytes("r1"); + private static final byte[] test = Bytes.toBytes("test"); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + util = new HBaseTestingUtility(); + util.startMiniCluster(); + } + + /** + * Test that we run a passing constraint + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test + public void testConstraintPasses() throws Exception { + // create the table + // it would be nice if this was also a method on the util + HTableDescriptor desc = new HTableDescriptor(tableName); + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + // add a constraint + Constraints.add(desc, CheckWasRunConstraint.class); + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we don't fail on a valid put + Put put = new Put(row1); + byte[] value = Integer.toString(10).getBytes(); + put.add(dummy, new byte[0], value); + table.put(put); + + assertTrue(CheckWasRunConstraint.wasRun); + } + + /** + * Test that constraints will fail properly + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test(timeout = 60000) + public void testConstraintFails() throws Exception { + + // create the table + // it would be nice if this was also a method on the util + HTableDescriptor desc = new HTableDescriptor(tableName); + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + + // add a constraint that is sure to fail + Constraints.add(desc, AllFailConstraint.class); + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we do fail on violation + Put put = new Put(row1); + put.add(dummy, new byte[0], "fail".getBytes()); + LOG.warn("Doing put in table"); + try { + table.put(put); + fail("This put should not have suceeded - AllFailConstraint was not run!"); + } catch (RetriesExhaustedWithDetailsException e) { + List causes = e.getCauses(); + assertEquals( + "More than one failure cause - should only be the failure constraint exception", + 1, causes.size()); + Throwable t = causes.get(0); + assertEquals(ConstraintException.class, t.getClass()); + } + table.close(); + } + + /** + * Check that if we just disable one constraint, then + * @throws Throwable + */ + @SuppressWarnings("unchecked") + @Test + public void testDisableConstraint() throws Throwable { + // create the table + HTableDescriptor desc = new HTableDescriptor(tableName); + // add a family to the table + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + // add a constraint to make sure it others get run + Constraints.add(desc, CheckWasRunConstraint.class); + + // Add Constraint to check + Constraints.add(desc, AllFailConstraint.class); + + // and then disable the failing constraint + Constraints.disableConstraint(desc, AllFailConstraint.class); + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we don't fail because its disabled + Put put = new Put(row1); + put.add(dummy, new byte[0], "pass".getBytes()); + table.put(put); + + assertTrue(CheckWasRunConstraint.wasRun); + } + + /** + * Test that if we disable all constraints, then nothing gets run + * @throws Throwable + */ + @SuppressWarnings("unchecked") + @Test + public void testDisableConstraints() throws Throwable { + // create the table + HTableDescriptor desc = new HTableDescriptor(tableName); + // add a family to the table + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + // add a constraint to check to see if is run + Constraints.add(desc, CheckWasRunConstraint.class); + + // then disable all the constraints + Constraints.disable(desc); + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we do fail on violation + Put put = new Put(row1); + put.add(dummy, new byte[0], "pass".getBytes()); + LOG.warn("Doing put in table"); + table.put(put); + + assertFalse(CheckWasRunConstraint.wasRun); + } + + /** + * Check to make sure a constraint is unloaded when it fails + * @throws Exception + */ + @Test + public void testIsUnloaded() throws Exception { + // create the table + HTableDescriptor desc = new HTableDescriptor(tableName); + // add a family to the table + for (byte[] family : new byte[][] { dummy, test }) { + desc.addFamily(new HColumnDescriptor(family)); + } + // make sure that constraints are unloaded + Constraints.add(desc, RuntimeFailConstraint.class); + // add a constraint to check to see if is run + Constraints.add(desc, CheckWasRunConstraint.class); + CheckWasRunConstraint.wasRun = false; + + util.getHBaseAdmin().createTable(desc); + HTable table = new HTable(util.getConfiguration(), tableName); + table.setAutoFlush(true); + + // test that we do fail on violation + Put put = new Put(row1); + put.add(dummy, new byte[0], "pass".getBytes()); + + try{ + table.put(put); + fail("RuntimeFailConstraint wasn't triggered - this put shouldn't work!"); + } catch (Exception e) {// NOOP + } + + // try the put again, this time constraints are not used, so it works + table.put(put); + // and we make sure that constraints were not run... + assertFalse(CheckWasRunConstraint.wasRun); + table.close(); + } + + @After + public void cleanup() throws Exception { + // cleanup + CheckWasRunConstraint.wasRun = false; + util.getHBaseAdmin().disableTable(tableName); + util.getHBaseAdmin().deleteTable(tableName); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + util.shutdownMiniCluster(); + } + + /** + * Constraint to check that it was actually run (or not) + */ + public static class CheckWasRunConstraint extends BaseConstraint { + public static boolean wasRun = false; + + @Override + public void check(Put p) { + wasRun = true; + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraints.java b/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraints.java new file mode 100644 index 0000000..777da49 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraints.java @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.constraint.TestConstraint.CheckWasRunConstraint; +import org.apache.hadoop.hbase.constraint.WorksConstraint.NameConstraint; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test reading/writing the constraints into the {@link HTableDescriptor} + */ +@Category(SmallTests.class) +public class TestConstraints { + + @SuppressWarnings("unchecked") + @Test + public void testSimpleReadWrite() throws Throwable { + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.add(desc, WorksConstraint.class); + + List constraints = Constraints.getConstraints(desc, + this.getClass().getClassLoader()); + assertEquals(1, constraints.size()); + + assertEquals(WorksConstraint.class, constraints.get(0).getClass()); + + // Check that we can add more than 1 constraint and that ordering is + // preserved + Constraints.add(desc, AlsoWorks.class, NameConstraint.class); + constraints = Constraints.getConstraints(desc, this.getClass() + .getClassLoader()); + assertEquals(3, constraints.size()); + + assertEquals(WorksConstraint.class, constraints.get(0).getClass()); + assertEquals(AlsoWorks.class, constraints.get(1).getClass()); + assertEquals(NameConstraint.class, constraints.get(2).getClass()); + + } + + @SuppressWarnings("unchecked") + @Test + public void testReadWriteWithConf() throws Throwable { + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.add( + desc, + new Pair, Configuration>( + CheckConfigurationConstraint.class, CheckConfigurationConstraint + .getConfiguration())); + + List c = Constraints.getConstraints(desc, this + .getClass().getClassLoader()); + assertEquals(1, c.size()); + + assertEquals(CheckConfigurationConstraint.class, c.get(0).getClass()); + + // check to make sure that we overwrite configurations + Constraints.add(desc, new Pair, Configuration>( + CheckConfigurationConstraint.class, new Configuration(false))); + + try { + Constraints.getConstraints(desc, this.getClass().getClassLoader()); + fail("No exception thrown - configuration not overwritten"); + } catch (IllegalArgumentException e) { + // expect to have the exception, so don't do anything + } + } + + /** + * Test that Constraints are properly enabled, disabled, and removed + * + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test + public void testEnableDisableRemove() throws Exception { + HTableDescriptor desc = new HTableDescriptor("table"); + // check general enabling/disabling of constraints + // first add a constraint + Constraints.add(desc, AllPassConstraint.class); + // make sure everything is enabled + assertTrue(Constraints.enabled(desc, AllPassConstraint.class)); + assertTrue(desc.hasCoprocessor(ConstraintProcessor.class.getName())); + + // check disabling + Constraints.disable(desc); + assertFalse(desc.hasCoprocessor(ConstraintProcessor.class.getName())); + // make sure the added constraints are still present + assertTrue(Constraints.enabled(desc, AllPassConstraint.class)); + + // check just removing the single constraint + Constraints.remove(desc, AllPassConstraint.class); + assertFalse(Constraints.has(desc, AllPassConstraint.class)); + + // Add back the single constraint + Constraints.add(desc, AllPassConstraint.class); + + // and now check that when we remove constraints, all are gone + Constraints.remove(desc); + assertFalse(desc.hasCoprocessor(ConstraintProcessor.class.getName())); + assertFalse(Constraints.has(desc, AllPassConstraint.class)); + + } + + /** + * Test that when we update a constraint the ordering is not modified. + * + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test + public void testUpdateConstraint() throws Exception { + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.add(desc, CheckConfigurationConstraint.class, + CheckWasRunConstraint.class); + Constraints.setConfiguration(desc, CheckConfigurationConstraint.class, + CheckConfigurationConstraint.getConfiguration()); + + List constraints = Constraints.getConstraints(desc, + this.getClass().getClassLoader()); + + assertEquals(2, constraints.size()); + + // check to make sure the order didn't change + assertEquals(CheckConfigurationConstraint.class, constraints.get(0) + .getClass()); + assertEquals(CheckWasRunConstraint.class, constraints.get(1).getClass()); + } + + /** + * Test that if a constraint hasn't been set that there are no problems with + * attempting to remove it. + * + * @throws Throwable + * on failure. + */ + @Test + public void testRemoveUnsetConstraint() throws Throwable { + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.remove(desc); + Constraints.remove(desc, AlsoWorks.class); + } + + @Test + public void testConfigurationPreserved() throws Throwable { + Configuration conf = new Configuration(); + conf.setBoolean("_ENABLED", false); + conf.setLong("_PRIORITY", 10); + HTableDescriptor desc = new HTableDescriptor("table"); + Constraints.add(desc, AlsoWorks.class, conf); + Constraints.add(desc, WorksConstraint.class); + assertFalse(Constraints.enabled(desc, AlsoWorks.class)); + List constraints = Constraints.getConstraints(desc, + this.getClass().getClassLoader()); + for (Constraint c : constraints) { + Configuration storedConf = c.getConf(); + if (c instanceof AlsoWorks) + assertEquals(10, storedConf.getLong("_PRIORITY", -1)); + // its just a worksconstraint + else + assertEquals(2, storedConf.getLong("_PRIORITY", -1)); + + } + + } + + // ---------- Constraints just used for testing + + /** + * Also just works + */ + public static class AlsoWorks extends BaseConstraint { + @Override + public void check(Put p) { + // NOOP + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/constraint/WorksConstraint.java b/src/test/java/org/apache/hadoop/hbase/constraint/WorksConstraint.java new file mode 100644 index 0000000..fa94230 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/constraint/WorksConstraint.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.constraint; + +import org.apache.hadoop.hbase.client.Put; + +/** + * It just works + */ +public class WorksConstraint extends BaseConstraint { + @Override + public void check(Put p) { + // NOOP + } + + /** + * Constraint to check that the naming of constraints doesn't mess up the + * pattern matching.(that constraint $___Constraint$NameConstraint isn't a + * problem) + */ + public static class NameConstraint extends WorksConstraint { + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationEndpoint.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationEndpoint.java new file mode 100644 index 0000000..38c55aa --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationEndpoint.java @@ -0,0 +1,62 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.util.Bytes; + + +/** + * The aggregation implementation at a region. + */ +public class ColumnAggregationEndpoint extends BaseEndpointCoprocessor +implements ColumnAggregationProtocol { + + @Override + public long sum(byte[] family, byte[] qualifier) + throws IOException { + // aggregate at each region + Scan scan = new Scan(); + scan.addColumn(family, qualifier); + int sumResult = 0; + + InternalScanner scanner = ((RegionCoprocessorEnvironment)getEnvironment()) + .getRegion().getScanner(scan); + try { + List curVals = new ArrayList(); + boolean done = false; + do { + curVals.clear(); + done = scanner.next(curVals); + KeyValue kv = curVals.get(0); + sumResult += Bytes.toInt(kv.getBuffer(), kv.getValueOffset()); + } while (done); + } finally { + scanner.close(); + } + return sumResult; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationProtocol.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationProtocol.java new file mode 100644 index 0000000..658c07b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/ColumnAggregationProtocol.java @@ -0,0 +1,39 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import java.io.IOException; + +/** + * A sample protocol for performing aggregation at regions. + */ +public interface ColumnAggregationProtocol extends CoprocessorProtocol { + /** + * Perform aggregation for a given column at the region. The aggregation + * will include all the rows inside the region. It can be extended to + * allow passing start and end rows for a fine-grained aggregation. + * @param family family + * @param qualifier qualifier + * @return Aggregation of the column. + * @throws exception. + */ + public long sum(byte[] family, byte[] qualifier) throws IOException; +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/GenericEndpoint.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/GenericEndpoint.java new file mode 100644 index 0000000..a35db9a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/GenericEndpoint.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +public class GenericEndpoint extends BaseEndpointCoprocessor implements + GenericProtocol { + + @Override + public T doWork(T genericObject) { + return genericObject; + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/GenericProtocol.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/GenericProtocol.java new file mode 100644 index 0000000..aa9c03e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/GenericProtocol.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; + +public interface GenericProtocol extends CoprocessorProtocol { + + /** + * Simple interface to allow the passing of a generic parameter to see if the + * RPC framework can accommodate generics. + * + * @param + * @param genericObject + * @return + */ + public T doWork(T genericObject); + +} diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java new file mode 100644 index 0000000..ff9c502 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java @@ -0,0 +1,161 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.List; +import java.util.Arrays; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; + +/** + * Class for testing WALObserver coprocessor. + * + * It will monitor WAL writing and restoring, and modify passed-in WALEdit, i.e, + * ignore specified columns when writing, or add a KeyValue. On the other + * side, it checks whether the ignored column is still in WAL when Restoreed + * at region reconstruct. + */ +public class SampleRegionWALObserver extends BaseRegionObserver +implements WALObserver { + + private static final Log LOG = LogFactory.getLog(SampleRegionWALObserver.class); + + private byte[] tableName; + private byte[] row; + private byte[] ignoredFamily; + private byte[] ignoredQualifier; + private byte[] addedFamily; + private byte[] addedQualifier; + private byte[] changedFamily; + private byte[] changedQualifier; + + private boolean preWALWriteCalled = false; + private boolean postWALWriteCalled = false; + private boolean preWALRestoreCalled = false; + private boolean postWALRestoreCalled = false; + + /** + * Set values: with a table name, a column name which will be ignored, and + * a column name which will be added to WAL. + */ + public void setTestValues(byte[] tableName, byte[] row, byte[] igf, byte[] igq, + byte[] chf, byte[] chq, byte[] addf, byte[] addq) { + this.row = row; + this.tableName = tableName; + this.ignoredFamily = igf; + this.ignoredQualifier = igq; + this.addedFamily = addf; + this.addedQualifier = addq; + this.changedFamily = chf; + this.changedQualifier = chq; + } + + + @Override + public void postWALWrite(ObserverContext env, + HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { + postWALWriteCalled = true; + } + + @Override + public boolean preWALWrite(ObserverContext env, + HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { + boolean bypass = false; + // check table name matches or not. + if (!Arrays.equals(HRegionInfo.getTableName(info.getRegionName()), this.tableName)) { + return bypass; + } + preWALWriteCalled = true; + // here we're going to remove one keyvalue from the WALEdit, and add + // another one to it. + List kvs = logEdit.getKeyValues(); + KeyValue deletedKV = null; + for (KeyValue kv : kvs) { + // assume only one kv from the WALEdit matches. + byte[] family = kv.getFamily(); + byte[] qulifier = kv.getQualifier(); + + if (Arrays.equals(family, ignoredFamily) && + Arrays.equals(qulifier, ignoredQualifier)) { + LOG.debug("Found the KeyValue from WALEdit which should be ignored."); + deletedKV = kv; + } + if (Arrays.equals(family, changedFamily) && + Arrays.equals(qulifier, changedQualifier)) { + LOG.debug("Found the KeyValue from WALEdit which should be changed."); + kv.getBuffer()[kv.getValueOffset()] += 1; + } + } + kvs.add(new KeyValue(row, addedFamily, addedQualifier)); + if (deletedKV != null) { + LOG.debug("About to delete a KeyValue from WALEdit."); + kvs.remove(deletedKV); + } + return bypass; + } + + /** + * Triggered before {@link org.apache.hadoop.hbase.regionserver.HRegion} when WAL is + * Restoreed. + */ + @Override + public void preWALRestore(ObserverContext env, + HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { + preWALRestoreCalled = true; + } + + /** + * Triggered after {@link org.apache.hadoop.hbase.regionserver.HRegion} when WAL is + * Restoreed. + */ + @Override + public void postWALRestore(ObserverContext env, + HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { + postWALRestoreCalled = true; + } + + public boolean isPreWALWriteCalled() { + return preWALWriteCalled; + } + + public boolean isPostWALWriteCalled() { + return postWALWriteCalled; + } + + public boolean isPreWALRestoreCalled() { + LOG.debug(SampleRegionWALObserver.class.getName() + + ".isPreWALRestoreCalled is called."); + return preWALRestoreCalled; + } + + public boolean isPostWALRestoreCalled() { + LOG.debug(SampleRegionWALObserver.class.getName() + + ".isPostWALRestoreCalled is called."); + return postWALRestoreCalled; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java new file mode 100644 index 0000000..30431b2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java @@ -0,0 +1,621 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Arrays; +import java.util.NavigableSet; + +import com.google.common.collect.ImmutableList; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.Leases; +import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +import org.apache.hadoop.hbase.regionserver.OperationStatus; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.SplitTransaction.SplitInfo; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +/** + * A sample region observer that tests the RegionObserver interface. + * It works with TestRegionObserverInterface to provide the test case. + */ +public class SimpleRegionObserver extends BaseRegionObserver implements RegionObserverExt { + static final Log LOG = LogFactory.getLog(TestRegionObserverInterface.class); + + boolean beforeDelete = true; + boolean scannerOpened = false; + boolean hadPreOpen; + boolean hadPostOpen; + boolean hadPreClose; + boolean hadPostClose; + boolean hadPreFlush; + boolean hadPreFlushScannerOpen; + boolean hadPostFlush; + boolean hadPreSplit; + boolean hadPostSplit; + boolean hadPreCompactSelect; + boolean hadPostCompactSelect; + boolean hadPreCompactScanner; + boolean hadPreCompact; + boolean hadPostCompact; + boolean hadPreGet = false; + boolean hadPostGet = false; + boolean hadPrePut = false; + boolean hadPostPut = false; + boolean hadPreDeleted = false; + boolean hadPostDeleted = false; + boolean hadPreGetClosestRowBefore = false; + boolean hadPostGetClosestRowBefore = false; + boolean hadPreIncrement = false; + boolean hadPostIncrement = false; + boolean hadPreWALRestored = false; + boolean hadPostWALRestored = false; + boolean hadPreScannerNext = false; + boolean hadPostScannerNext = false; + boolean hadPreScannerClose = false; + boolean hadPostScannerClose = false; + boolean hadPreScannerOpen = false; + boolean hadPreStoreScannerOpen = false; + boolean hadPostScannerOpen = false; + boolean hadPreBulkLoadHFile = false; + boolean hadPostBulkLoadHFile = false; + boolean hadPreBatchMutate = false; + boolean hadPostBatchMutate = false; + + @Override + public void start(CoprocessorEnvironment e) throws IOException { + // this only makes sure that leases and locks are available to coprocessors + // from external packages + RegionCoprocessorEnvironment re = (RegionCoprocessorEnvironment)e; + Leases leases = re.getRegionServerServices().getLeases(); + leases.createLease("x", null); + leases.cancelLease("x"); + } + + @Override + public void preOpen(ObserverContext c) { + hadPreOpen = true; + } + + @Override + public void postOpen(ObserverContext c) { + hadPostOpen = true; + } + + public boolean wasOpened() { + return hadPreOpen && hadPostOpen; + } + + @Override + public void preClose(ObserverContext c, boolean abortRequested) { + hadPreClose = true; + } + + @Override + public void postClose(ObserverContext c, boolean abortRequested) { + hadPostClose = true; + } + + public boolean wasClosed() { + return hadPreClose && hadPostClose; + } + + @Override + public InternalScanner preFlush(ObserverContext c, Store store, InternalScanner scanner) { + hadPreFlush = true; + return scanner; + } + + @Override + public InternalScanner preFlushScannerOpen(final ObserverContext c, + Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException { + hadPreFlushScannerOpen = true; + return null; + } + + @Override + public void postFlush(ObserverContext c, Store store, StoreFile resultFile) { + hadPostFlush = true; + } + + public boolean wasFlushed() { + return hadPreFlush && hadPostFlush; + } + + @Override + public void preSplit(ObserverContext c) { + hadPreSplit = true; + } + + @Override + public void postSplit(ObserverContext c, HRegion l, HRegion r) { + hadPostSplit = true; + } + + public boolean wasSplit() { + return hadPreSplit && hadPostSplit; + } + + @Override + public void preCompactSelection(ObserverContext c, + Store store, List candidates) { + hadPreCompactSelect = true; + } + + @Override + public void postCompactSelection(ObserverContext c, + Store store, ImmutableList selected) { + hadPostCompactSelect = true; + } + + @Override + public InternalScanner preCompact(ObserverContext e, + Store store, InternalScanner scanner) { + hadPreCompact = true; + return scanner; + } + + @Override + public InternalScanner preCompactScannerOpen(final ObserverContext c, + Store store, List scanners, ScanType scanType, long earliestPutTs, + InternalScanner s) throws IOException { + hadPreCompactScanner = true; + return null; + } + + @Override + public void postCompact(ObserverContext e, + Store store, StoreFile resultFile) { + hadPostCompact = true; + } + + public boolean wasCompacted() { + return hadPreCompact && hadPostCompact; + } + + @Override + public RegionScanner preScannerOpen(final ObserverContext c, + final Scan scan, + final RegionScanner s) throws IOException { + hadPreScannerOpen = true; + return null; + } + + @Override + public KeyValueScanner preStoreScannerOpen(final ObserverContext c, + final Store store, final Scan scan, final NavigableSet targetCols, + final KeyValueScanner s) throws IOException { + hadPreStoreScannerOpen = true; + return null; + } + + @Override + public RegionScanner postScannerOpen(final ObserverContext c, + final Scan scan, final RegionScanner s) + throws IOException { + hadPostScannerOpen = true; + return s; + } + + @Override + public boolean preScannerNext(final ObserverContext c, + final InternalScanner s, final List results, + final int limit, final boolean hasMore) throws IOException { + hadPreScannerNext = true; + return hasMore; + } + + @Override + public boolean postScannerNext(final ObserverContext c, + final InternalScanner s, final List results, final int limit, + final boolean hasMore) throws IOException { + hadPostScannerNext = true; + return hasMore; + } + + @Override + public void preScannerClose(final ObserverContext c, + final InternalScanner s) throws IOException { + hadPreScannerClose = true; + } + + @Override + public void postScannerClose(final ObserverContext c, + final InternalScanner s) throws IOException { + hadPostScannerClose = true; + } + + @Override + public void preGet(final ObserverContext c, final Get get, + final List results) throws IOException { + RegionCoprocessorEnvironment e = c.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + assertNotNull(get); + assertNotNull(results); + hadPreGet = true; + } + + @Override + public void postGet(final ObserverContext c, final Get get, + final List results) { + RegionCoprocessorEnvironment e = c.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + assertNotNull(get); + assertNotNull(results); + if (Arrays.equals(e.getRegion().getTableDesc().getName(), + TestRegionObserverInterface.TEST_TABLE)) { + boolean foundA = false; + boolean foundB = false; + boolean foundC = false; + for (KeyValue kv: results) { + if (Bytes.equals(kv.getFamily(), TestRegionObserverInterface.A)) { + foundA = true; + } + if (Bytes.equals(kv.getFamily(), TestRegionObserverInterface.B)) { + foundB = true; + } + if (Bytes.equals(kv.getFamily(), TestRegionObserverInterface.C)) { + foundC = true; + } + } + assertTrue(foundA); + assertTrue(foundB); + assertTrue(foundC); + } + hadPostGet = true; + } + + @Override + public void prePut(final ObserverContext c, + final Put put, final WALEdit edit, + final boolean writeToWAL) throws IOException { + Map> familyMap = put.getFamilyMap(); + RegionCoprocessorEnvironment e = c.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + assertNotNull(familyMap); + if (Arrays.equals(e.getRegion().getTableDesc().getName(), + TestRegionObserverInterface.TEST_TABLE)) { + List kvs = familyMap.get(TestRegionObserverInterface.A); + assertNotNull(kvs); + assertNotNull(kvs.get(0)); + assertTrue(Bytes.equals(kvs.get(0).getQualifier(), + TestRegionObserverInterface.A)); + kvs = familyMap.get(TestRegionObserverInterface.B); + assertNotNull(kvs); + assertNotNull(kvs.get(0)); + assertTrue(Bytes.equals(kvs.get(0).getQualifier(), + TestRegionObserverInterface.B)); + kvs = familyMap.get(TestRegionObserverInterface.C); + assertNotNull(kvs); + assertNotNull(kvs.get(0)); + assertTrue(Bytes.equals(kvs.get(0).getQualifier(), + TestRegionObserverInterface.C)); + } + hadPrePut = true; + } + + @Override + public void postPut(final ObserverContext c, + final Put put, final WALEdit edit, + final boolean writeToWAL) throws IOException { + Map> familyMap = put.getFamilyMap(); + RegionCoprocessorEnvironment e = c.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + assertNotNull(familyMap); + List kvs = familyMap.get(TestRegionObserverInterface.A); + if (Arrays.equals(e.getRegion().getTableDesc().getName(), + TestRegionObserverInterface.TEST_TABLE)) { + assertNotNull(kvs); + assertNotNull(kvs.get(0)); + assertTrue(Bytes.equals(kvs.get(0).getQualifier(), + TestRegionObserverInterface.A)); + kvs = familyMap.get(TestRegionObserverInterface.B); + assertNotNull(kvs); + assertNotNull(kvs.get(0)); + assertTrue(Bytes.equals(kvs.get(0).getQualifier(), + TestRegionObserverInterface.B)); + kvs = familyMap.get(TestRegionObserverInterface.C); + assertNotNull(kvs); + assertNotNull(kvs.get(0)); + assertTrue(Bytes.equals(kvs.get(0).getQualifier(), + TestRegionObserverInterface.C)); + } + hadPostPut = true; + } + + @Override + public void preDelete(final ObserverContext c, + final Delete delete, final WALEdit edit, + final boolean writeToWAL) throws IOException { + Map> familyMap = delete.getFamilyMap(); + RegionCoprocessorEnvironment e = c.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + assertNotNull(familyMap); + if (beforeDelete) { + hadPreDeleted = true; + } + } + + @Override + public void postDelete(final ObserverContext c, + final Delete delete, final WALEdit edit, + final boolean writeToWAL) throws IOException { + Map> familyMap = delete.getFamilyMap(); + RegionCoprocessorEnvironment e = c.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + assertNotNull(familyMap); + beforeDelete = false; + hadPostDeleted = true; + } + + @Override + public void preBatchMutate(ObserverContext c, + MiniBatchOperationInProgress> miniBatchOp) throws IOException { + RegionCoprocessorEnvironment e = c.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + assertNotNull(miniBatchOp); + hadPreBatchMutate = true; + } + + @Override + public void postBatchMutate(final ObserverContext c, + final MiniBatchOperationInProgress> miniBatchOp) throws IOException { + RegionCoprocessorEnvironment e = c.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + assertNotNull(miniBatchOp); + hadPostBatchMutate = true; + } + + @Override + public void preGetClosestRowBefore(final ObserverContext c, + final byte[] row, final byte[] family, final Result result) + throws IOException { + RegionCoprocessorEnvironment e = c.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + assertNotNull(row); + assertNotNull(result); + if (beforeDelete) { + hadPreGetClosestRowBefore = true; + } + } + + @Override + public void postGetClosestRowBefore(final ObserverContext c, + final byte[] row, final byte[] family, final Result result) + throws IOException { + RegionCoprocessorEnvironment e = c.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + assertNotNull(row); + assertNotNull(result); + hadPostGetClosestRowBefore = true; + } + + @Override + public Result preIncrement(final ObserverContext c, + final Increment increment) throws IOException { + hadPreIncrement = true; + return null; + } + + @Override + public Result postIncrement(final ObserverContext c, + final Increment increment, final Result result) throws IOException { + hadPostIncrement = true; + return result; + } + + @Override + public void preBulkLoadHFile(ObserverContext ctx, + List> familyPaths) throws IOException { + RegionCoprocessorEnvironment e = ctx.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + if (Arrays.equals(e.getRegion().getTableDesc().getName(), + TestRegionObserverInterface.TEST_TABLE)) { + assertNotNull(familyPaths); + assertEquals(1,familyPaths.size()); + assertArrayEquals(familyPaths.get(0).getFirst(), TestRegionObserverInterface.A); + String familyPath = familyPaths.get(0).getSecond(); + String familyName = Bytes.toString(TestRegionObserverInterface.A); + assertEquals(familyPath.substring(familyPath.length()-familyName.length()-1),"/"+familyName); + } + hadPreBulkLoadHFile = true; + } + + @Override + public boolean postBulkLoadHFile(ObserverContext ctx, + List> familyPaths, boolean hasLoaded) throws IOException { + RegionCoprocessorEnvironment e = ctx.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + if (Arrays.equals(e.getRegion().getTableDesc().getName(), + TestRegionObserverInterface.TEST_TABLE)) { + assertNotNull(familyPaths); + assertEquals(1,familyPaths.size()); + assertArrayEquals(familyPaths.get(0).getFirst(), TestRegionObserverInterface.A); + String familyPath = familyPaths.get(0).getSecond(); + String familyName = Bytes.toString(TestRegionObserverInterface.A); + assertEquals(familyPath.substring(familyPath.length()-familyName.length()-1),"/"+familyName); + } + hadPostBulkLoadHFile = true; + return hasLoaded; + } + + public boolean hadPreGet() { + return hadPreGet; + } + + public boolean hadPostGet() { + return hadPostGet; + } + + public boolean hadPrePut() { + return hadPrePut; + } + + public boolean hadPostPut() { + return hadPostPut; + } + + public boolean hadPreBatchMutate() { + return hadPreBatchMutate; + } + + public boolean hadPostBatchMutate() { + return hadPostBatchMutate; + } + + public boolean hadDelete() { + return !beforeDelete; + } + + public boolean hadPreIncrement() { + return hadPreIncrement; + } + + public boolean hadPostIncrement() { + return hadPostIncrement; + } + + public boolean hadPreWALRestored() { + return hadPreWALRestored; + } + + public boolean hadPostWALRestored() { + return hadPostWALRestored; + } + public boolean wasScannerNextCalled() { + return hadPreScannerNext && hadPostScannerNext; + } + public boolean wasScannerCloseCalled() { + return hadPreScannerClose && hadPostScannerClose; + } + public boolean wasScannerOpenCalled() { + return hadPreScannerOpen && hadPostScannerOpen; + } + public boolean hadDeleted() { + return hadPreDeleted && hadPostDeleted; + } + + public boolean hadPostBulkLoadHFile() { + return hadPostBulkLoadHFile; + } + + public boolean hadPreBulkLoadHFile() { + return hadPreBulkLoadHFile; + } + + @Override + public void preBatchMutate(ObserverContext ctx, + List> mutationVsBatchOp, WALEdit edit) throws IOException { + RegionCoprocessorEnvironment e = ctx.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + assertNotNull(edit); + hadPreBatchMutate = true; + } + + @Override + public void postBatchMutate(ObserverContext ctx, + List mutations, WALEdit walEdit) throws IOException { + RegionCoprocessorEnvironment e = ctx.getEnvironment(); + assertNotNull(e); + assertNotNull(e.getRegion()); + assertNotNull(walEdit); + hadPostBatchMutate = true; + } + + @Override + public void postCompleteBatchMutate(ObserverContext ctx, + List mutations) throws IOException { + // TODO Auto-generated method stub + + } + + @Override + public boolean postFilterRow(ObserverContext ctx, + InternalScanner s, byte[] currentRow) throws IOException { + // TODO Auto-generated method stub + return false; + } + + @Override + public SplitInfo preSplitBeforePONR(ObserverContext ctx, + byte[] splitKey) throws IOException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void preRollBack(ObserverContext ctx) throws IOException { + // TODO Auto-generated method stub + + } + + @Override + public void postCloseRegionOperation(ObserverContext ctx) + throws IOException { + // TODO Auto-generated method stub + + } + + @Override + public void postStartRegionOperation(ObserverContext ctx) + throws IOException { + // TODO Auto-generated method stub + + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAggregateProtocol.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAggregateProtocol.java new file mode 100644 index 0000000..28af316 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAggregateProtocol.java @@ -0,0 +1,810 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import static org.junit.Assert.assertEquals; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.coprocessor.AggregationClient; +import org.apache.hadoop.hbase.client.coprocessor.LongColumnInterpreter; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * A test class to cover aggregate functions, that can be implemented using + * Coprocessors. + */ +@Category(MediumTests.class) +public class TestAggregateProtocol { + protected static Log myLog = LogFactory.getLog(TestAggregateProtocol.class); + + /** + * Creating the test infrastructure. + */ + private static final byte[] TEST_TABLE = Bytes.toBytes("TestTable"); + private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily"); + private static final byte[] TEST_QUALIFIER = Bytes.toBytes("TestQualifier"); + private static final byte[] TEST_MULTI_CQ = Bytes.toBytes("TestMultiCQ"); + + private static byte[] ROW = Bytes.toBytes("testRow"); + private static final int ROWSIZE = 20; + private static final int rowSeperator1 = 5; + private static final int rowSeperator2 = 12; + private static byte[][] ROWS = makeN(ROW, ROWSIZE); + + private static HBaseTestingUtility util = new HBaseTestingUtility(); + private static MiniHBaseCluster cluster = null; + private static Configuration conf = util.getConfiguration(); + + /** + * A set up method to start the test cluster. AggregateProtocolImpl is + * registered and will be loaded during region startup. + * @throws Exception + */ + @BeforeClass + public static void setupBeforeClass() throws Exception { + + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + "org.apache.hadoop.hbase.coprocessor.AggregateImplementation"); + + util.startMiniCluster(2); + cluster = util.getMiniHBaseCluster(); + HTable table = util.createTable(TEST_TABLE, TEST_FAMILY); + util.createMultiRegions(util.getConfiguration(), table, TEST_FAMILY, + new byte[][] { HConstants.EMPTY_BYTE_ARRAY, ROWS[rowSeperator1], + ROWS[rowSeperator2] }); + /** + * The testtable has one CQ which is always populated and one variable CQ + * for each row rowkey1: CF:CQ CF:CQ1 rowKey2: CF:CQ CF:CQ2 + */ + for (int i = 0; i < ROWSIZE; i++) { + Put put = new Put(ROWS[i]); + put.setWriteToWAL(false); + Long l = new Long(i); + put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(l)); + table.put(put); + Put p2 = new Put(ROWS[i]); + put.setWriteToWAL(false); + p2.add(TEST_FAMILY, Bytes.add(TEST_MULTI_CQ, Bytes.toBytes(l)), Bytes + .toBytes(l * 10)); + table.put(p2); + } + table.close(); + } + + /** + * Shutting down the cluster + * @throws Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + util.shutdownMiniCluster(); + } + + /** + * an infrastructure method to prepare rows for the testtable. + * @param base + * @param n + * @return + */ + private static byte[][] makeN(byte[] base, int n) { + byte[][] ret = new byte[n][]; + for (int i = 0; i < n; i++) { + ret[i] = Bytes.add(base, Bytes.toBytes(i)); + } + return ret; + } + + /** + * ****************** Test cases for Median ********************** + */ + /** + * @throws Throwable + */ + @Test + public void testMedianWithValidRange() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY,TEST_QUALIFIER); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long median = aClient.median(TEST_TABLE, ci, + scan); + assertEquals(8L, median); + } + + /** + * **************************** ROW COUNT Test cases ******************* + */ + + /** + * This will test rowcount with a valid range, i.e., a subset of rows. It will + * be the most common use case. + * @throws Throwable + */ + @Test + public void testRowCountWithValidRange() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[2]); + scan.setStopRow(ROWS[14]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long rowCount = aClient.rowCount(TEST_TABLE, ci, scan); + assertEquals(12, rowCount); + } + + /** + * This will test the row count on the entire table. Startrow and endrow will + * be null. + * @throws Throwable + */ + @Test + public void testRowCountAllTable() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long rowCount = aClient.rowCount(TEST_TABLE, ci, + scan); + assertEquals(ROWSIZE, rowCount); + } + + /** + * This will test the row count with startrow > endrow. The result should be + * -1. + * @throws Throwable + */ + @Test + public void testRowCountWithInvalidRange1() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[2]); + + final ColumnInterpreter ci = new LongColumnInterpreter(); + long rowCount = -1; + try { + rowCount = aClient.rowCount(TEST_TABLE, ci, scan); + } catch (Throwable e) { + myLog.error("Exception thrown in the invalidRange method" + + e.getStackTrace()); + } + assertEquals(-1, rowCount); + } + + /** + * This will test the row count with startrow = endrow and they will be + * non-null. The result should be 0, as it assumes a non-get query. + * @throws Throwable + */ + @Test + public void testRowCountWithInvalidRange2() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[5]); + + final ColumnInterpreter ci = new LongColumnInterpreter(); + long rowCount = -1; + try { + rowCount = aClient.rowCount(TEST_TABLE, ci, scan); + } catch (Throwable e) { + rowCount = 0; + } + assertEquals(0, rowCount); + } + + /** + * This should return a 0 + */ + @Test + public void testRowCountWithNullCF() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long rowCount = -1; + try { + rowCount = aClient.rowCount(TEST_TABLE, ci, scan); + } catch (Throwable e) { + rowCount = 0; + } + assertEquals(0, rowCount); + } + + @Test + public void testRowCountWithNullCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long rowCount = aClient.rowCount(TEST_TABLE, ci, + scan); + assertEquals(20, rowCount); + } + + @Test + public void testRowCountWithPrefixFilter() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Filter f = new PrefixFilter(Bytes.toBytes("foo:bar")); + scan.setFilter(f); + long rowCount = aClient.rowCount(TEST_TABLE, ci, + scan); + assertEquals(0, rowCount); + } + + /** + * ***************Test cases for Maximum ******************* + */ + + /** + * give max for the entire table. + * @throws Throwable + */ + @Test + public void testMaxWithValidRange() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long maximum = aClient.max(TEST_TABLE, ci, scan); + assertEquals(19, maximum); + } + + /** + * @throws Throwable + */ + @Test + public void testMaxWithValidRange2() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long max = aClient.max(TEST_TABLE, ci, scan); + assertEquals(14, max); + } + + @Test + public void testMaxWithValidRangeWithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long maximum = aClient.max(TEST_TABLE, ci, scan); + assertEquals(190, maximum); + } + + @Test + public void testMaxWithValidRange2WithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[7]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long max = aClient.max(TEST_TABLE, ci, scan); + assertEquals(60, max); + } + + @Test + public void testMaxWithValidRangeWithNullCF() { + AggregationClient aClient = new AggregationClient(conf); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Scan scan = new Scan(); + Long max = null; + try { + max = aClient.max(TEST_TABLE, ci, scan); + } catch (Throwable e) { + max = null; + } + assertEquals(null, max);// CP will throw an IOException about the + // null column family, and max will be set to 0 + } + + @Test + public void testMaxWithInvalidRange() { + AggregationClient aClient = new AggregationClient(conf); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Scan scan = new Scan(); + scan.setStartRow(ROWS[4]); + scan.setStopRow(ROWS[2]); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + long max = Long.MIN_VALUE; + try { + max = aClient.max(TEST_TABLE, ci, scan); + } catch (Throwable e) { + max = 0; + } + assertEquals(0, max);// control should go to the catch block + } + + @Test + public void testMaxWithInvalidRange2() throws Throwable { + long max = Long.MIN_VALUE; + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[4]); + scan.setStopRow(ROWS[4]); + try { + AggregationClient aClient = new AggregationClient(conf); + final ColumnInterpreter ci = new LongColumnInterpreter(); + max = aClient.max(TEST_TABLE, ci, scan); + } catch (Exception e) { + max = 0; + } + assertEquals(0, max);// control should go to the catch block + } + + @Test + public void testMaxWithFilter() throws Throwable { + Long max = 0l; + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + Filter f = new PrefixFilter(Bytes.toBytes("foo:bar")); + scan.setFilter(f); + final ColumnInterpreter ci = new LongColumnInterpreter(); + max = aClient.max(TEST_TABLE, ci, scan); + assertEquals(null, max); + } + + /** + * **************************Test cases for Minimum *********************** + */ + + /** + * @throws Throwable + */ + @Test + public void testMinWithValidRange() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(HConstants.EMPTY_START_ROW); + scan.setStopRow(HConstants.EMPTY_END_ROW); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Long min = aClient.min(TEST_TABLE, ci, + scan); + assertEquals(0l, min.longValue()); + } + + /** + * @throws Throwable + */ + @Test + public void testMinWithValidRange2() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long min = aClient.min(TEST_TABLE, ci, scan); + assertEquals(5, min); + } + + @Test + public void testMinWithValidRangeWithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(HConstants.EMPTY_START_ROW); + scan.setStopRow(HConstants.EMPTY_END_ROW); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long min = aClient.min(TEST_TABLE, ci, + scan); + assertEquals(0, min); + } + + @Test + public void testMinWithValidRange2WithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[7]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long min = aClient.min(TEST_TABLE, ci, scan); + assertEquals(6, min); + } + + @Test + public void testMinWithValidRangeWithNullCF() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Long min = null; + try { + min = aClient.min(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, min);// CP will throw an IOException about the + // null column family, and max will be set to 0 + } + + @Test + public void testMinWithInvalidRange() { + AggregationClient aClient = new AggregationClient(conf); + Long min = null; + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[4]); + scan.setStopRow(ROWS[2]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + try { + min = aClient.min(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, min);// control should go to the catch block + } + + @Test + public void testMinWithInvalidRange2() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[6]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Long min = null; + try { + min = aClient.min(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, min);// control should go to the catch block + } + + @Test + public void testMinWithFilter() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + Filter f = new PrefixFilter(Bytes.toBytes("foo:bar")); + scan.setFilter(f); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Long min = null; + min = aClient.min(TEST_TABLE, ci, scan); + assertEquals(null, min); + } + + /** + * *************** Test cases for Sum ********************* + */ + /** + * @throws Throwable + */ + @Test + public void testSumWithValidRange() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY,TEST_QUALIFIER); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long sum = aClient.sum(TEST_TABLE, ci, + scan); + assertEquals(190, sum); + } + + /** + * @throws Throwable + */ + @Test + public void testSumWithValidRange2() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY,TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long sum = aClient.sum(TEST_TABLE, ci, scan); + assertEquals(95, sum); + } + + @Test + public void testSumWithValidRangeWithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long sum = aClient.sum(TEST_TABLE, ci, + scan); + assertEquals(190 + 1900, sum); + } + + @Test + public void testSumWithValidRange2WithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[7]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + long sum = aClient.sum(TEST_TABLE, ci, scan); + assertEquals(6 + 60, sum); + } + + @Test + public void testSumWithValidRangeWithNullCF() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[7]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Long sum = null; + try { + sum = aClient.sum(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, sum);// CP will throw an IOException about the + // null column family, and max will be set to 0 + } + + @Test + public void testSumWithInvalidRange() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[2]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Long sum = null; + try { + sum = aClient.sum(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, sum);// control should go to the catch block + } + + @Test + public void testSumWithFilter() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Filter f = new PrefixFilter(Bytes.toBytes("foo:bar")); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setFilter(f); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Long sum = null; + sum = aClient.sum(TEST_TABLE, ci, scan); + assertEquals(null, sum); + } + + /** + * ****************************** Test Cases for Avg ************** + */ + /** + * @throws Throwable + */ + @Test + public void testAvgWithValidRange() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY,TEST_QUALIFIER); + final ColumnInterpreter ci = new LongColumnInterpreter(); + double avg = aClient.avg(TEST_TABLE, ci, + scan); + assertEquals(9.5, avg, 0); + } + + /** + * @throws Throwable + */ + @Test + public void testAvgWithValidRange2() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY,TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + double avg = aClient.avg(TEST_TABLE, ci, scan); + assertEquals(9.5, avg, 0); + } + + @Test + public void testAvgWithValidRangeWithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + final ColumnInterpreter ci = new LongColumnInterpreter(); + double avg = aClient.avg(TEST_TABLE, ci, + scan); + assertEquals(104.5, avg, 0); + } + + @Test + public void testAvgWithValidRange2WithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[7]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + double avg = aClient.avg(TEST_TABLE, ci, scan); + assertEquals(6 + 60, avg, 0); + } + + @Test + public void testAvgWithValidRangeWithNullCF() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Double avg = null; + try { + avg = aClient.avg(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, avg);// CP will throw an IOException about the + // null column family, and max will be set to 0 + } + + @Test + public void testAvgWithInvalidRange() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY,TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[1]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Double avg = null; + try { + avg = aClient.avg(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, avg);// control should go to the catch block + } + + @Test + public void testAvgWithFilter() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY,TEST_QUALIFIER); + Filter f = new PrefixFilter(Bytes.toBytes("foo:bar")); + scan.setFilter(f); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Double avg = null; + avg = aClient.avg(TEST_TABLE, ci, scan); + assertEquals(Double.NaN, avg, 0); + } + + /** + * ****************** Test cases for STD ********************** + */ + /** + * @throws Throwable + */ + @Test + public void testStdWithValidRange() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY,TEST_QUALIFIER); + final ColumnInterpreter ci = new LongColumnInterpreter(); + double std = aClient.std(TEST_TABLE, ci, + scan); + assertEquals(5.766, std, 0.05d); + } + + /** + * @throws Throwable + */ + @Test + public void testStdWithValidRange2() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY,TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + double std = aClient.std(TEST_TABLE, ci, scan); + assertEquals(2.87, std, 0.05d); + } + + @Test + public void testStdWithValidRangeWithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + final ColumnInterpreter ci = new LongColumnInterpreter(); + double std = aClient.std(TEST_TABLE, ci, + scan); + assertEquals(63.42, std, 0.05d); + } + + @Test + public void testStdWithValidRange2WithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[7]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + double std = aClient.std(TEST_TABLE, ci, scan); + assertEquals(0, std, 0); + } + + @Test + public void testStdWithValidRangeWithNullCF() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[17]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Double std = null; + try { + std = aClient.std(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, std);// CP will throw an IOException about the + // null column family, and max will be set to 0 + } + + @Test + public void testStdWithInvalidRange() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[1]); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Double std = null; + try { + std = aClient.std(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, std);// control should go to the catch block + } + + @Test + public void testStdWithFilter() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Filter f = new PrefixFilter(Bytes.toBytes("foo:bar")); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setFilter(f); + final ColumnInterpreter ci = new LongColumnInterpreter(); + Double std = null; + std = aClient.std(TEST_TABLE, ci, scan); + assertEquals(Double.NaN, std, 0); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestBigDecimalColumnInterpreter.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestBigDecimalColumnInterpreter.java new file mode 100644 index 0000000..62a0d00 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestBigDecimalColumnInterpreter.java @@ -0,0 +1,670 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import static org.junit.Assert.assertEquals; +import java.math.BigDecimal; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.coprocessor.AggregationClient; +import org.apache.hadoop.hbase.client.coprocessor.BigDecimalColumnInterpreter; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * A test class to test BigDecimalColumnInterpreter for AggregationsProtocol + */ +@Category(MediumTests.class) +public class TestBigDecimalColumnInterpreter { + protected static Log myLog = LogFactory.getLog(TestBigDecimalColumnInterpreter.class); + + /** + * Creating the test infrastructure. + */ + private static final byte[] TEST_TABLE = Bytes.toBytes("TestTable"); + private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily"); + private static final byte[] TEST_QUALIFIER = Bytes.toBytes("TestQualifier"); + private static final byte[] TEST_MULTI_CQ = Bytes.toBytes("TestMultiCQ"); + + private static byte[] ROW = Bytes.toBytes("testRow"); + private static final int ROWSIZE = 20; + private static final int rowSeperator1 = 5; + private static final int rowSeperator2 = 12; + private static byte[][] ROWS = makeN(ROW, ROWSIZE); + + private static HBaseTestingUtility util = new HBaseTestingUtility(); + private static Configuration conf = util.getConfiguration(); + + /** + * A set up method to start the test cluster. AggregateProtocolImpl is registered and will be + * loaded during region startup. + * @throws Exception + */ + @BeforeClass + public static void setupBeforeClass() throws Exception { + + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + "org.apache.hadoop.hbase.coprocessor.AggregateImplementation"); + + util.startMiniCluster(2); + HTable table = util.createTable(TEST_TABLE, TEST_FAMILY); + util.createMultiRegions(util.getConfiguration(), table, TEST_FAMILY, new byte[][] { + HConstants.EMPTY_BYTE_ARRAY, ROWS[rowSeperator1], ROWS[rowSeperator2] }); + /** + * The testtable has one CQ which is always populated and one variable CQ for each row rowkey1: + * CF:CQ CF:CQ1 rowKey2: CF:CQ CF:CQ2 + */ + for (int i = 0; i < ROWSIZE; i++) { + Put put = new Put(ROWS[i]); + put.setWriteToWAL(false); + BigDecimal bd = new BigDecimal(i); + put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(bd)); + table.put(put); + Put p2 = new Put(ROWS[i]); + put.setWriteToWAL(false); + p2.add(TEST_FAMILY, Bytes.add(TEST_MULTI_CQ, Bytes.toBytes(bd)), + Bytes.toBytes(bd.multiply(new BigDecimal("0.10")))); + table.put(p2); + } + table.close(); + } + + /** + * Shutting down the cluster + * @throws Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + util.shutdownMiniCluster(); + } + + /** + * an infrastructure method to prepare rows for the testtable. + * @param base + * @param n + * @return + */ + private static byte[][] makeN(byte[] base, int n) { + byte[][] ret = new byte[n][]; + for (int i = 0; i < n; i++) { + ret[i] = Bytes.add(base, Bytes.toBytes(i)); + } + return ret; + } + + /** + * ****************** Test cases for Median ********************** + */ + /** + * @throws Throwable + */ + @Test + public void testMedianWithValidRange() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal median = aClient.median(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("8.00"), median); + } + + /** + * ***************Test cases for Maximum ******************* + */ + + /** + * give max for the entire table. + * @throws Throwable + */ + @Test + public void testMax() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal maximum = aClient.max(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("19.00"), maximum); + } + + /** + * @throws Throwable + */ + @Test + public void testMaxWithValidRange2() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal max = aClient.max(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("14.00"), max); + } + + @Test + public void testMaxWithValidRangeWithoutCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal maximum = aClient.max(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("19.00"), maximum); + } + + @Test + public void testMaxWithValidRange2WithoutCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[7]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal max = aClient.max(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("6.00"), max); + } + + @Test + public void testMaxWithValidRangeWithNullCF() { + AggregationClient aClient = new AggregationClient(conf); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + Scan scan = new Scan(); + BigDecimal max = null; + try { + max = aClient.max(TEST_TABLE, ci, scan); + } catch (Throwable e) { + max = null; + } + assertEquals(null, max);// CP will throw an IOException about the + // null column family, and max will be set to 0 + } + + @Test + public void testMaxWithInvalidRange() { + AggregationClient aClient = new AggregationClient(conf); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + Scan scan = new Scan(); + scan.setStartRow(ROWS[4]); + scan.setStopRow(ROWS[2]); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + BigDecimal max = new BigDecimal(Long.MIN_VALUE); + ; + try { + max = aClient.max(TEST_TABLE, ci, scan); + } catch (Throwable e) { + max = BigDecimal.ZERO; + } + assertEquals(BigDecimal.ZERO, max);// control should go to the catch block + } + + @Test + public void testMaxWithInvalidRange2() throws Throwable { + BigDecimal max = new BigDecimal(Long.MIN_VALUE); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[4]); + scan.setStopRow(ROWS[4]); + try { + AggregationClient aClient = new AggregationClient(conf); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + max = aClient.max(TEST_TABLE, ci, scan); + } catch (Exception e) { + max = BigDecimal.ZERO; + } + assertEquals(BigDecimal.ZERO, max);// control should go to the catch block + } + + @Test + public void testMaxWithFilter() throws Throwable { + BigDecimal max = BigDecimal.ZERO; + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + Filter f = new PrefixFilter(Bytes.toBytes("foo:bar")); + scan.setFilter(f); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + max = aClient.max(TEST_TABLE, ci, scan); + assertEquals(null, max); + } + + /** + * **************************Test cases for Minimum *********************** + */ + + /** + * @throws Throwable + */ + @Test + public void testMinWithValidRange() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(HConstants.EMPTY_START_ROW); + scan.setStopRow(HConstants.EMPTY_END_ROW); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal min = aClient.min(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("0.00"), min); + } + + /** + * @throws Throwable + */ + @Test + public void testMinWithValidRange2() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal min = aClient.min(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("5.00"), min); + } + + @Test + public void testMinWithValidRangeWithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(HConstants.EMPTY_START_ROW); + scan.setStopRow(HConstants.EMPTY_END_ROW); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal min = aClient.min(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("0.00"), min); + } + + @Test + public void testMinWithValidRange2WithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[7]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal min = aClient.min(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("0.60"), min); + } + + @Test + public void testMinWithValidRangeWithNullCF() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal min = null; + try { + min = aClient.min(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, min);// CP will throw an IOException about the + // null column family, and max will be set to 0 + } + + @Test + public void testMinWithInvalidRange() { + AggregationClient aClient = new AggregationClient(conf); + BigDecimal min = null; + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[4]); + scan.setStopRow(ROWS[2]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + try { + min = aClient.min(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, min);// control should go to the catch block + } + + @Test + public void testMinWithInvalidRange2() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[6]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal min = null; + try { + min = aClient.min(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, min);// control should go to the catch block + } + + @Test + public void testMinWithFilter() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + Filter f = new PrefixFilter(Bytes.toBytes("foo:bar")); + scan.setFilter(f); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal min = null; + min = aClient.min(TEST_TABLE, ci, scan); + assertEquals(null, min); + } + + /** + * *************** Test cases for Sum ********************* + */ + /** + * @throws Throwable + */ + @Test + public void testSumWithValidRange() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal sum = aClient.sum(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("190.00"), sum); + } + + /** + * @throws Throwable + */ + @Test + public void testSumWithValidRange2() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal sum = aClient.sum(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("95.00"), sum); + } + + @Test + public void testSumWithValidRangeWithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal sum = aClient.sum(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("209.00"), sum); // 190 + 19 + } + + @Test + public void testSumWithValidRange2WithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[7]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal sum = aClient.sum(TEST_TABLE, ci, scan); + assertEquals(new BigDecimal("6.60"), sum); // 6 + 60 + } + + @Test + public void testSumWithValidRangeWithNullCF() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[7]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal sum = null; + try { + sum = aClient.sum(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, sum);// CP will throw an IOException about the + // null column family, and max will be set to 0 + } + + @Test + public void testSumWithInvalidRange() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[2]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal sum = null; + try { + sum = aClient.sum(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, sum);// control should go to the catch block + } + + @Test + public void testSumWithFilter() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Filter f = new PrefixFilter(Bytes.toBytes("foo:bar")); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setFilter(f); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + BigDecimal sum = null; + sum = aClient.sum(TEST_TABLE, ci, scan); + assertEquals(null, sum); + } + + /** + * ****************************** Test Cases for Avg ************** + */ + /** + * @throws Throwable + */ + @Test + public void testAvgWithValidRange() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + double avg = aClient.avg(TEST_TABLE, ci, scan); + assertEquals(9.5, avg, 0); + } + + /** + * @throws Throwable + */ + @Test + public void testAvgWithValidRange2() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + double avg = aClient.avg(TEST_TABLE, ci, scan); + assertEquals(9.5, avg, 0); + } + + @Test + public void testAvgWithValidRangeWithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + double avg = aClient.avg(TEST_TABLE, ci, scan); + assertEquals(10.45, avg, 0.01); + } + + @Test + public void testAvgWithValidRange2WithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[7]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + double avg = aClient.avg(TEST_TABLE, ci, scan); + assertEquals(6 + 0.60, avg, 0); + } + + @Test + public void testAvgWithValidRangeWithNullCF() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + Double avg = null; + try { + avg = aClient.avg(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, avg);// CP will throw an IOException about the + // null column family, and max will be set to 0 + } + + @Test + public void testAvgWithInvalidRange() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[1]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + Double avg = null; + try { + avg = aClient.avg(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, avg);// control should go to the catch block + } + + @Test + public void testAvgWithFilter() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + Filter f = new PrefixFilter(Bytes.toBytes("foo:bar")); + scan.setFilter(f); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + Double avg = null; + avg = aClient.avg(TEST_TABLE, ci, scan); + assertEquals(Double.NaN, avg, 0); + } + + /** + * ****************** Test cases for STD ********************** + */ + /** + * @throws Throwable + */ + @Test + public void testStdWithValidRange() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + double std = aClient.std(TEST_TABLE, ci, scan); + assertEquals(5.766, std, 0.05d); + } + + /** + * need to change this + * @throws Throwable + */ + @Test + public void testStdWithValidRange2() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addColumn(TEST_FAMILY, TEST_QUALIFIER); + scan.setStartRow(ROWS[5]); + scan.setStopRow(ROWS[15]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + double std = aClient.std(TEST_TABLE, ci, scan); + assertEquals(2.87, std, 0.05d); + } + + /** + * need to change this + * @throws Throwable + */ + @Test + public void testStdWithValidRangeWithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + double std = aClient.std(TEST_TABLE, ci, scan); + assertEquals(6.342, std, 0.05d); + } + + @Test + public void testStdWithValidRange2WithNoCQ() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[7]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + double std = aClient.std(TEST_TABLE, ci, scan); + System.out.println("std is:" + std); + assertEquals(0, std, 0.05d); + } + + @Test + public void testStdWithValidRangeWithNullCF() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[17]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + Double std = null; + try { + std = aClient.std(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, std);// CP will throw an IOException about the + // null column family, and max will be set to 0 + } + + @Test + public void testStdWithInvalidRange() { + AggregationClient aClient = new AggregationClient(conf); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setStartRow(ROWS[6]); + scan.setStopRow(ROWS[1]); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + Double std = null; + try { + std = aClient.std(TEST_TABLE, ci, scan); + } catch (Throwable e) { + } + assertEquals(null, std);// control should go to the catch block + } + + @Test + public void testStdWithFilter() throws Throwable { + AggregationClient aClient = new AggregationClient(conf); + Filter f = new PrefixFilter(Bytes.toBytes("foo:bar")); + Scan scan = new Scan(); + scan.addFamily(TEST_FAMILY); + scan.setFilter(f); + final ColumnInterpreter ci = new BigDecimalColumnInterpreter(); + Double std = null; + std = aClient.std(TEST_TABLE, ci, scan); + assertEquals(Double.NaN, std, 0); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java new file mode 100644 index 0000000..ba1b528 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java @@ -0,0 +1,573 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.regionserver.HRegion; + +import org.apache.hadoop.hbase.util.ClassLoaderTestHelper; +import org.apache.hadoop.hbase.util.CoprocessorClassLoader; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.*; +import java.util.*; +import java.util.jar.*; + +import org.junit.*; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +/** + * Test coprocessors class loading. + */ +@Category(MediumTests.class) +public class TestClassLoading { + private static final Log LOG = LogFactory.getLog(TestClassLoading.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private static MiniDFSCluster cluster; + + static final int BUFFER_SIZE = 4096; + static final String tableName = "TestClassLoading"; + static final String cpName1 = "TestCP1"; + static final String cpName2 = "TestCP2"; + static final String cpName3 = "TestCP3"; + static final String cpName4 = "TestCP4"; + static final String cpName5 = "TestCP5"; + static final String cpName6 = "TestCP6"; + static final String cpNameInvalid = "TestCPInvalid"; + + private static Class regionCoprocessor1 = ColumnAggregationEndpoint.class; + private static Class regionCoprocessor2 = GenericEndpoint.class; + private static Class regionServerCoprocessor = SampleRegionWALObserver.class; + private static Class masterCoprocessor = BaseMasterObserver.class; + + private static final String[] regionServerSystemCoprocessors = + new String[]{ + regionServerCoprocessor.getSimpleName() + }; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + + // regionCoprocessor1 will be loaded on all regionservers, since it is + // loaded for any tables (user or meta). + conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + regionCoprocessor1.getName()); + + // regionCoprocessor2 will be loaded only on regionservers that serve a + // user table region. Therefore, if there are no user tables loaded, + // this coprocessor will not be loaded on any regionserver. + conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, + regionCoprocessor2.getName()); + + conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, + regionServerCoprocessor.getName()); + conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, + masterCoprocessor.getName()); + TEST_UTIL.startMiniCluster(1); + cluster = TEST_UTIL.getDFSCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + static File buildCoprocessorJar(String className) throws Exception { + String code = "import org.apache.hadoop.hbase.coprocessor.*;" + + "public class " + className + " extends BaseRegionObserver {}"; + return ClassLoaderTestHelper.buildJar( + TEST_UTIL.getDataTestDir().toString(), className, code); + } + + @Test + // HBASE-3516: Test CP Class loading from HDFS + public void testClassLoadingFromHDFS() throws Exception { + FileSystem fs = cluster.getFileSystem(); + + File jarFile1 = buildCoprocessorJar(cpName1); + File jarFile2 = buildCoprocessorJar(cpName2); + + // copy the jars into dfs + fs.copyFromLocalFile(new Path(jarFile1.getPath()), + new Path(fs.getUri().toString() + Path.SEPARATOR)); + String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR + + jarFile1.getName(); + Path pathOnHDFS1 = new Path(jarFileOnHDFS1); + assertTrue("Copy jar file to HDFS failed.", + fs.exists(pathOnHDFS1)); + LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1); + + fs.copyFromLocalFile(new Path(jarFile2.getPath()), + new Path(fs.getUri().toString() + Path.SEPARATOR)); + String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR + + jarFile2.getName(); + Path pathOnHDFS2 = new Path(jarFileOnHDFS2); + assertTrue("Copy jar file to HDFS failed.", + fs.exists(pathOnHDFS2)); + LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2); + + // create a table that references the coprocessors + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor("test")); + // without configuration values + htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 + + "|" + Coprocessor.PRIORITY_USER); + // with configuration values + htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 + + "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3"); + // same jar but invalid class name (should fail to load this class) + htd.setValue("COPROCESSOR$3", jarFileOnHDFS2.toString() + "|" + cpNameInvalid + + "|" + Coprocessor.PRIORITY_USER); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(tableName)) { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + CoprocessorClassLoader.clearCache(); + byte[] startKey = {10, 63}; + byte[] endKey = {12, 43}; + admin.createTable(htd, startKey, endKey, 4); + waitForTable(htd.getName()); + + // verify that the coprocessors were loaded + boolean foundTableRegion=false; + boolean found_invalid = true, found1 = true, found2 = true, found2_k1 = true, + found2_k2 = true, found2_k3 = true; + Map> regionsActiveClassLoaders = + new HashMap>(); + MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster(); + for (HRegion region: + hbase.getRegionServer(0).getOnlineRegionsLocalContext()) { + if (region.getRegionNameAsString().startsWith(tableName)) { + foundTableRegion = true; + CoprocessorEnvironment env; + env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1); + found1 = found1 && (env != null); + env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2); + found2 = found2 && (env != null); + if (env != null) { + Configuration conf = env.getConfiguration(); + found2_k1 = found2_k1 && (conf.get("k1") != null); + found2_k2 = found2_k2 && (conf.get("k2") != null); + found2_k3 = found2_k3 && (conf.get("k3") != null); + } else { + found2_k1 = found2_k2 = found2_k3 = false; + } + env = region.getCoprocessorHost().findCoprocessorEnvironment(cpNameInvalid); + found_invalid = found_invalid && (env != null); + + regionsActiveClassLoaders + .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders()); + } + } + + assertTrue("No region was found for table " + tableName, foundTableRegion); + assertTrue("Class " + cpName1 + " was missing on a region", found1); + assertTrue("Class " + cpName2 + " was missing on a region", found2); + //an invalid CP class name is defined for this table, validate that it is not loaded + assertFalse("Class " + cpNameInvalid + " was found on a region", found_invalid); + assertTrue("Configuration key 'k1' was missing on a region", found2_k1); + assertTrue("Configuration key 'k2' was missing on a region", found2_k2); + assertTrue("Configuration key 'k3' was missing on a region", found2_k3); + // check if CP classloaders are cached + assertNotNull(jarFileOnHDFS1 + " was not cached", + CoprocessorClassLoader.getIfCached(pathOnHDFS1)); + assertNotNull(jarFileOnHDFS2 + " was not cached", + CoprocessorClassLoader.getIfCached(pathOnHDFS2)); + //two external jar used, should be one classloader per jar + assertEquals("The number of cached classloaders should be equal to the number" + + " of external jar files", + 2, CoprocessorClassLoader.getAllCached().size()); + //check if region active classloaders are shared across all RS regions + Set externalClassLoaders = new HashSet( + CoprocessorClassLoader.getAllCached()); + for (Map.Entry> regionCP : regionsActiveClassLoaders.entrySet()) { + assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached." + + " ClassLoader Cache:" + externalClassLoaders + + " Region ClassLoaders:" + regionCP.getValue(), + externalClassLoaders.containsAll(regionCP.getValue())); + } + } + + private String getLocalPath(File file) { + return new Path(file.toURI()).toString(); + } + + @Test + // HBASE-3516: Test CP Class loading from local file system + public void testClassLoadingFromLocalFS() throws Exception { + File jarFile = buildCoprocessorJar(cpName3); + + // create a table that references the jar + HTableDescriptor htd = new HTableDescriptor(cpName3); + htd.addFamily(new HColumnDescriptor("test")); + htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" + + Coprocessor.PRIORITY_USER); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + admin.createTable(htd); + waitForTable(htd.getName()); + + // verify that the coprocessor was loaded + boolean found = false; + MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster(); + for (HRegion region: + hbase.getRegionServer(0).getOnlineRegionsLocalContext()) { + if (region.getRegionNameAsString().startsWith(cpName3)) { + found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null); + } + } + assertTrue("Class " + cpName3 + " was missing on a region", found); + } + + @Test + // HBASE-6308: Test CP classloader is the CoprocessorClassLoader + public void testPrivateClassLoader() throws Exception { + File jarFile = buildCoprocessorJar(cpName4); + + // create a table that references the jar + HTableDescriptor htd = new HTableDescriptor(cpName4); + htd.addFamily(new HColumnDescriptor("test")); + htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" + + Coprocessor.PRIORITY_USER); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + admin.createTable(htd); + waitForTable(htd.getName()); + + // verify that the coprocessor was loaded correctly + boolean found = false; + MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster(); + for (HRegion region: + hbase.getRegionServer(0).getOnlineRegionsLocalContext()) { + if (region.getRegionNameAsString().startsWith(cpName4)) { + Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4); + if (cp != null) { + found = true; + assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader", + cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class); + } + } + } + assertTrue("Class " + cpName4 + " was missing on a region", found); + } + + @Test + // HBase-3810: Registering a Coprocessor at HTableDescriptor should be + // less strict + public void testHBase3810() throws Exception { + // allowed value pattern: [path] | class name | [priority] | [key values] + + File jarFile1 = buildCoprocessorJar(cpName1); + File jarFile2 = buildCoprocessorJar(cpName2); + File jarFile5 = buildCoprocessorJar(cpName5); + File jarFile6 = buildCoprocessorJar(cpName6); + + String cpKey1 = "COPROCESSOR$1"; + String cpKey2 = " Coprocessor$2 "; + String cpKey3 = " coprocessor$03 "; + + String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" + + Coprocessor.PRIORITY_USER; + String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | "; + // load from default class loader + String cpValue3 = + " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v "; + + // create a table that references the jar + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor("test")); + + // add 3 coprocessors by setting htd attributes directly. + htd.setValue(cpKey1, cpValue1); + htd.setValue(cpKey2, cpValue2); + htd.setValue(cpKey3, cpValue3); + + // add 2 coprocessor by using new htd.addCoprocessor() api + htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)), + Coprocessor.PRIORITY_USER, null); + Map kvs = new HashMap(); + kvs.put("k1", "v1"); + kvs.put("k2", "v2"); + kvs.put("k3", "v3"); + htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)), + Coprocessor.PRIORITY_USER, kvs); + + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(tableName)) { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + admin.createTable(htd); + waitForTable(htd.getName()); + + // verify that the coprocessor was loaded + boolean found_2 = false, found_1 = false, found_3 = false, + found_5 = false, found_6 = false; + boolean found6_k1 = false, found6_k2 = false, found6_k3 = false, + found6_k4 = false; + + MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster(); + for (HRegion region: + hbase.getRegionServer(0).getOnlineRegionsLocalContext()) { + if (region.getRegionNameAsString().startsWith(tableName)) { + found_1 = found_1 || + (region.getCoprocessorHost().findCoprocessor(cpName1) != null); + found_2 = found_2 || + (region.getCoprocessorHost().findCoprocessor(cpName2) != null); + found_3 = found_3 || + (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver") + != null); + found_5 = found_5 || + (region.getCoprocessorHost().findCoprocessor(cpName5) != null); + + CoprocessorEnvironment env = + region.getCoprocessorHost().findCoprocessorEnvironment(cpName6); + if (env != null) { + found_6 = true; + Configuration conf = env.getConfiguration(); + found6_k1 = conf.get("k1") != null; + found6_k2 = conf.get("k2") != null; + found6_k3 = conf.get("k3") != null; + } + } + } + + assertTrue("Class " + cpName1 + " was missing on a region", found_1); + assertTrue("Class " + cpName2 + " was missing on a region", found_2); + assertTrue("Class SimpleRegionObserver was missing on a region", found_3); + assertTrue("Class " + cpName5 + " was missing on a region", found_5); + assertTrue("Class " + cpName6 + " was missing on a region", found_6); + + assertTrue("Configuration key 'k1' was missing on a region", found6_k1); + assertTrue("Configuration key 'k2' was missing on a region", found6_k2); + assertTrue("Configuration key 'k3' was missing on a region", found6_k3); + assertFalse("Configuration key 'k4' wasn't configured", found6_k4); + } + + @Test + public void testClassLoadingFromLibDirInJar() throws Exception { + loadingClassFromLibDirInJar("/lib/"); + } + + @Test + public void testClassLoadingFromRelativeLibDirInJar() throws Exception { + loadingClassFromLibDirInJar("lib/"); + } + + void loadingClassFromLibDirInJar(String libPrefix) throws Exception { + FileSystem fs = cluster.getFileSystem(); + + File innerJarFile1 = buildCoprocessorJar(cpName1); + File innerJarFile2 = buildCoprocessorJar(cpName2); + File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar"); + + byte buffer[] = new byte[BUFFER_SIZE]; + // TODO: code here and elsewhere in this file is duplicated w/TestClassFinder. + // Some refactoring may be in order... + // Open archive file + FileOutputStream stream = new FileOutputStream(outerJarFile); + JarOutputStream out = new JarOutputStream(stream, new Manifest()); + + for (File jarFile: new File[] { innerJarFile1, innerJarFile2 }) { + // Add archive entry + JarEntry jarAdd = new JarEntry(libPrefix + jarFile.getName()); + jarAdd.setTime(jarFile.lastModified()); + out.putNextEntry(jarAdd); + + // Write file to archive + FileInputStream in = new FileInputStream(jarFile); + while (true) { + int nRead = in.read(buffer, 0, buffer.length); + if (nRead <= 0) + break; + out.write(buffer, 0, nRead); + } + in.close(); + } + out.close(); + stream.close(); + LOG.info("Adding jar file to outer jar file completed"); + + // copy the jars into dfs + fs.copyFromLocalFile(new Path(outerJarFile.getPath()), + new Path(fs.getUri().toString() + Path.SEPARATOR)); + String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR + + outerJarFile.getName(); + assertTrue("Copy jar file to HDFS failed.", + fs.exists(new Path(jarFileOnHDFS))); + LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS); + + // create a table that references the coprocessors + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor("test")); + // without configuration values + htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 + + "|" + Coprocessor.PRIORITY_USER); + // with configuration values + htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 + + "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3"); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(tableName)) { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + admin.createTable(htd); + waitForTable(htd.getName()); + + // verify that the coprocessors were loaded + boolean found1 = false, found2 = false, found2_k1 = false, + found2_k2 = false, found2_k3 = false; + MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster(); + for (HRegion region: + hbase.getRegionServer(0).getOnlineRegionsLocalContext()) { + if (region.getRegionNameAsString().startsWith(tableName)) { + CoprocessorEnvironment env; + env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1); + if (env != null) { + found1 = true; + } + env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2); + if (env != null) { + found2 = true; + Configuration conf = env.getConfiguration(); + found2_k1 = conf.get("k1") != null; + found2_k2 = conf.get("k2") != null; + found2_k3 = conf.get("k3") != null; + } + } + } + assertTrue("Class " + cpName1 + " was missing on a region", found1); + assertTrue("Class " + cpName2 + " was missing on a region", found2); + assertTrue("Configuration key 'k1' was missing on a region", found2_k1); + assertTrue("Configuration key 'k2' was missing on a region", found2_k2); + assertTrue("Configuration key 'k3' was missing on a region", found2_k3); + } + + @Test + public void testRegionServerCoprocessorsReported() throws Exception { + // This was a test for HBASE-4070. + // We are removing coprocessors from region load in HBASE-5258. + // Therefore, this test now only checks system coprocessors. + + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + assertAllRegionServers(regionServerSystemCoprocessors,null); + } + + /** + * return the subset of all regionservers + * (actually returns set of HServerLoads) + * which host some region in a given table. + * used by assertAllRegionServers() below to + * test reporting of loaded coprocessors. + * @param tableName : given table. + * @return subset of all servers. + */ + Map serversForTable(String tableName) { + Map serverLoadHashMap = + new HashMap(); + for(Map.Entry server: + TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager(). + getOnlineServers().entrySet()) { + for(Map.Entry region: + server.getValue().getRegionsLoad().entrySet()) { + if (region.getValue().getNameAsString().equals(tableName)) { + // this server server hosts a region of tableName: add this server.. + serverLoadHashMap.put(server.getKey(),server.getValue()); + // .. and skip the rest of the regions that it hosts. + break; + } + } + } + return serverLoadHashMap; + } + + void assertAllRegionServers(String[] expectedCoprocessors, String tableName) + throws InterruptedException { + Map servers; + String[] actualCoprocessors = null; + boolean success = false; + for(int i = 0; i < 5; i++) { + if (tableName == null) { + //if no tableName specified, use all servers. + servers = + TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager(). + getOnlineServers(); + } else { + servers = serversForTable(tableName); + } + boolean any_failed = false; + for(Map.Entry server: servers.entrySet()) { + actualCoprocessors = server.getValue().getRsCoprocessors(); + if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) { + LOG.debug("failed comparison: actual: " + + Arrays.toString(actualCoprocessors) + + " ; expected: " + Arrays.toString(expectedCoprocessors)); + any_failed = true; + break; + } + } + if (any_failed == false) { + success = true; + break; + } + LOG.debug("retrying after failed comparison: " + i); + Thread.sleep(1000); + } + assertTrue(success); + } + + @Test + public void testMasterCoprocessorsReported() { + // HBASE 4070: Improve region server metrics to report loaded coprocessors + // to master: verify that the master is reporting the correct set of + // loaded coprocessors. + final String loadedMasterCoprocessorsVerify = + "[" + masterCoprocessor.getSimpleName() + "]"; + String loadedMasterCoprocessors = + java.util.Arrays.toString( + TEST_UTIL.getHBaseCluster().getMaster().getCoprocessors()); + assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors); + } + + private void waitForTable(byte[] name) throws InterruptedException, IOException { + // First wait until all regions are online + TEST_UTIL.waitTableEnabled(name, 5000); + // Now wait a bit longer for the coprocessor hosts to load the CPs + Thread.sleep(1000); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java new file mode 100644 index 0000000..1c596d5 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java @@ -0,0 +1,254 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.client.coprocessor.Exec; +import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.Text; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * TestEndpoint: test cases to verify coprocessor Endpoint + */ +@Category(MediumTests.class) +public class TestCoprocessorEndpoint { + + private static final byte[] TEST_TABLE = Bytes.toBytes("TestTable"); + private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily"); + private static final byte[] TEST_QUALIFIER = Bytes.toBytes("TestQualifier"); + private static byte[] ROW = Bytes.toBytes("testRow"); + + private static final String protocolName = "org.apache.hadoop.hbase.CustomProtocol"; + private static final String methodName = "myFunc"; + + private static final int ROWSIZE = 20; + private static final int rowSeperator1 = 5; + private static final int rowSeperator2 = 12; + private static byte[][] ROWS = makeN(ROW, ROWSIZE); + + private static HBaseTestingUtility util = new HBaseTestingUtility(); + private static MiniHBaseCluster cluster = null; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // set configure to indicate which cp should be loaded + Configuration conf = util.getConfiguration(); + conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + "org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint", + "org.apache.hadoop.hbase.coprocessor.GenericEndpoint"); + conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, + "org.apache.hadoop.hbase.coprocessor.GenericEndpoint"); + + util.startMiniCluster(2); + cluster = util.getMiniHBaseCluster(); + + HTable table = util.createTable(TEST_TABLE, TEST_FAMILY); + util.createMultiRegions(util.getConfiguration(), table, TEST_FAMILY, + new byte[][] { HConstants.EMPTY_BYTE_ARRAY, + ROWS[rowSeperator1], ROWS[rowSeperator2] }); + + for (int i = 0; i < ROWSIZE; i++) { + Put put = new Put(ROWS[i]); + put.setWriteToWAL(false); + put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(i)); + table.put(put); + } + + // sleep here is an ugly hack to allow region transitions to finish + long timeout = System.currentTimeMillis() + (15 * 1000); + while ((System.currentTimeMillis() < timeout) && + (table.getRegionsInfo().size() != 2)) { + Thread.sleep(250); + } + table.close(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + util.shutdownMiniCluster(); + } + + @Test + public void testGeneric() throws Throwable { + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + GenericProtocol protocol = table.coprocessorProxy(GenericProtocol.class, + Bytes.toBytes("testRow")); + String workResult1 = protocol.doWork("foo"); + assertEquals("foo", workResult1); + byte[] workResult2 = protocol.doWork(new byte[]{1}); + assertArrayEquals(new byte[]{1}, workResult2); + byte workResult3 = protocol.doWork((byte)1); + assertEquals((byte)1, workResult3); + char workResult4 = protocol.doWork('c'); + assertEquals('c', workResult4); + boolean workResult5 = protocol.doWork(true); + assertEquals(true, workResult5); + short workResult6 = protocol.doWork((short)1); + assertEquals((short)1, workResult6); + int workResult7 = protocol.doWork(5); + assertEquals(5, workResult7); + long workResult8 = protocol.doWork(5l); + assertEquals(5l, workResult8); + double workResult9 = protocol.doWork(6d); + assertEquals(6d, workResult9, 0.01); + float workResult10 = protocol.doWork(6f); + assertEquals(6f, workResult10, 0.01); + Text workResult11 = protocol.doWork(new Text("foo")); + assertEquals(new Text("foo"), workResult11); + table.close(); + } + + @Test + public void testMasterGeneric() throws Throwable { + HBaseAdmin admin = new HBaseAdmin(util.getConfiguration()); + GenericProtocol protocol = admin.coprocessorProxy(GenericProtocol.class); + String workResult1 = protocol.doWork("foo"); + assertEquals("foo", workResult1); + byte[] workResult2 = protocol.doWork(new byte[]{1}); + assertArrayEquals(new byte[]{1}, workResult2); + byte workResult3 = protocol.doWork((byte)1); + assertEquals((byte)1, workResult3); + char workResult4 = protocol.doWork('c'); + assertEquals('c', workResult4); + boolean workResult5 = protocol.doWork(true); + assertEquals(true, workResult5); + short workResult6 = protocol.doWork((short)1); + assertEquals((short)1, workResult6); + int workResult7 = protocol.doWork(5); + assertEquals(5, workResult7); + long workResult8 = protocol.doWork(5l); + assertEquals(5l, workResult8); + double workResult9 = protocol.doWork(6d); + assertEquals(6d, workResult9, 0.01); + float workResult10 = protocol.doWork(6f); + assertEquals(6f, workResult10, 0.01); + Text workResult11 = protocol.doWork(new Text("foo")); + assertEquals(new Text("foo"), workResult11); + } + + @Ignore @Test + public void testAggregation() throws Throwable { + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + Map results; + + // scan: for all regions + results = table + .coprocessorExec(ColumnAggregationProtocol.class, + ROWS[rowSeperator1 - 1], ROWS[rowSeperator2 + 1], + new Batch.Call() { + public Long call(ColumnAggregationProtocol instance) + throws IOException { + return instance.sum(TEST_FAMILY, TEST_QUALIFIER); + } + }); + int sumResult = 0; + int expectedResult = 0; + for (Map.Entry e : results.entrySet()) { + sumResult += e.getValue(); + } + for (int i = 0; i < ROWSIZE; i++) { + expectedResult += i; + } + assertEquals("Invalid result", sumResult, expectedResult); + + results.clear(); + + // scan: for region 2 and region 3 + results = table + .coprocessorExec(ColumnAggregationProtocol.class, + ROWS[rowSeperator1 + 1], ROWS[rowSeperator2 + 1], + new Batch.Call() { + public Long call(ColumnAggregationProtocol instance) + throws IOException { + return instance.sum(TEST_FAMILY, TEST_QUALIFIER); + } + }); + sumResult = 0; + expectedResult = 0; + for (Map.Entry e : results.entrySet()) { + sumResult += e.getValue(); + } + for (int i = rowSeperator1; i < ROWSIZE; i++) { + expectedResult += i; + } + assertEquals("Invalid result", sumResult, expectedResult); + table.close(); + } + + @Test + public void testExecDeserialization() throws IOException { + DataOutputBuffer dob = new DataOutputBuffer(); + dob.writeUTF(methodName); + dob.writeInt(1); + Scan scan = new Scan(); + HbaseObjectWritable.writeObject(dob, scan, Scan.class, new Configuration()); + dob.writeUTF("org.apache.hadoop.hbase.client.Scan"); + Bytes.writeByteArray(dob, new byte[]{'a'}); + // this is the dynamic protocol name + dob.writeUTF(protocolName); + + DataInputBuffer dib = new DataInputBuffer(); + dib.reset(dob.getData(), dob.getLength()); + + Exec after = new Exec(); + after.setConf(HBaseConfiguration.create()); + after.readFields(dib); + // no error thrown + assertEquals(after.getProtocolName(), protocolName); + assertEquals(after.getMethodName(), methodName); + } + + private static byte[][] makeN(byte[] base, int n) { + byte[][] ret = new byte[n][]; + for (int i = 0; i < n; i++) { + ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%02d", i))); + } + return ret; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java new file mode 100644 index 0000000..88becfc --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java @@ -0,0 +1,494 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.SplitTransaction; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.PairOfSameType; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import static org.mockito.Mockito.when; + +@Category(SmallTests.class) +public class TestCoprocessorInterface extends HBaseTestCase { + static final Log LOG = LogFactory.getLog(TestCoprocessorInterface.class); + static final String DIR = "test/build/data/TestCoprocessorInterface/"; + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + private static class CustomScanner implements RegionScanner { + + private RegionScanner delegate; + + public CustomScanner(RegionScanner delegate) { + this.delegate = delegate; + } + + @Override + public boolean next(List results) throws IOException { + return delegate.next(results); + } + + @Override + public boolean next(List results, String metric) + throws IOException { + return delegate.next(results, metric); + } + + @Override + public boolean next(List result, int limit) throws IOException { + return delegate.next(result, limit); + } + + @Override + public boolean next(List result, int limit, String metric) + throws IOException { + return delegate.next(result, limit, metric); + } + + @Override + public boolean nextRaw(List result, int limit, String metric) + throws IOException { + return delegate.nextRaw(result, limit, metric); + } + + @Override + public boolean nextRaw(List result, String metric) + throws IOException { + return delegate.nextRaw(result, metric); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + @Override + public HRegionInfo getRegionInfo() { + return delegate.getRegionInfo(); + } + + @Override + public boolean isFilterDone() { + return delegate.isFilterDone(); + } + + @Override + public boolean reseek(byte[] row) throws IOException { + return false; + } + + @Override + public long getMvccReadPoint() { + return delegate.getMvccReadPoint(); + } + } + + public static class CoprocessorImpl extends BaseRegionObserver { + + private boolean startCalled; + private boolean stopCalled; + private boolean preOpenCalled; + private boolean postOpenCalled; + private boolean preCloseCalled; + private boolean postCloseCalled; + private boolean preCompactCalled; + private boolean postCompactCalled; + private boolean preFlushCalled; + private boolean postFlushCalled; + private boolean preSplitCalled; + private boolean postSplitCalled; + private ConcurrentMap sharedData; + + @Override + public void start(CoprocessorEnvironment e) { + sharedData = ((RegionCoprocessorEnvironment)e).getSharedData(); + // using new String here, so that there will be new object on each invocation + sharedData.putIfAbsent("test1", new Object()); + startCalled = true; + } + + @Override + public void stop(CoprocessorEnvironment e) { + sharedData = null; + stopCalled = true; + } + + @Override + public void preOpen(ObserverContext e) { + preOpenCalled = true; + } + @Override + public void postOpen(ObserverContext e) { + postOpenCalled = true; + } + @Override + public void preClose(ObserverContext e, boolean abortRequested) { + preCloseCalled = true; + } + @Override + public void postClose(ObserverContext e, boolean abortRequested) { + postCloseCalled = true; + } + @Override + public InternalScanner preCompact(ObserverContext e, + Store store, InternalScanner scanner) { + preCompactCalled = true; + return scanner; + } + @Override + public void postCompact(ObserverContext e, + Store store, StoreFile resultFile) { + postCompactCalled = true; + } + @Override + public void preFlush(ObserverContext e) { + preFlushCalled = true; + } + @Override + public void postFlush(ObserverContext e) { + postFlushCalled = true; + } + @Override + public void preSplit(ObserverContext e) { + preSplitCalled = true; + } + @Override + public void postSplit(ObserverContext e, HRegion l, HRegion r) { + postSplitCalled = true; + } + + @Override + public RegionScanner postScannerOpen(final ObserverContext e, + final Scan scan, final RegionScanner s) throws IOException { + return new CustomScanner(s); + } + + boolean wasStarted() { + return startCalled; + } + boolean wasStopped() { + return stopCalled; + } + boolean wasOpened() { + return (preOpenCalled && postOpenCalled); + } + boolean wasClosed() { + return (preCloseCalled && postCloseCalled); + } + boolean wasFlushed() { + return (preFlushCalled && postFlushCalled); + } + boolean wasCompacted() { + return (preCompactCalled && postCompactCalled); + } + boolean wasSplit() { + return (preSplitCalled && postSplitCalled); + } + Map getSharedData() { + return sharedData; + } + } + + public static class CoprocessorII extends BaseRegionObserver { + private ConcurrentMap sharedData; + @Override + public void start(CoprocessorEnvironment e) { + sharedData = ((RegionCoprocessorEnvironment)e).getSharedData(); + sharedData.putIfAbsent("test2", new Object()); + } + @Override + public void stop(CoprocessorEnvironment e) { + sharedData = null; + } + @Override + public void preGet(final ObserverContext e, + final Get get, final List results) throws IOException { + if (1/0 == 1) { + e.complete(); + } + } + + Map getSharedData() { + return sharedData; + } + } + + public void testSharedData() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [][] families = { fam1, fam2, fam3 }; + + Configuration hc = initSplit(); + HRegion region = initHRegion(tableName, getName(), hc, + new Class[]{}, families); + + for (int i = 0; i < 3; i++) { + addContent(region, fam3); + region.flushcache(); + } + + region.compactStores(); + + byte [] splitRow = region.checkSplit(); + + assertNotNull(splitRow); + HRegion [] regions = split(region, splitRow); + for (int i = 0; i < regions.length; i++) { + regions[i] = reopenRegion(regions[i], CoprocessorImpl.class, CoprocessorII.class); + } + Coprocessor c = regions[0].getCoprocessorHost(). + findCoprocessor(CoprocessorImpl.class.getName()); + Coprocessor c2 = regions[0].getCoprocessorHost(). + findCoprocessor(CoprocessorII.class.getName()); + Object o = ((CoprocessorImpl)c).getSharedData().get("test1"); + Object o2 = ((CoprocessorII)c2).getSharedData().get("test2"); + assertNotNull(o); + assertNotNull(o2); + // to coprocessors get different sharedDatas + assertFalse(((CoprocessorImpl)c).getSharedData() == ((CoprocessorII)c2).getSharedData()); + for (int i = 1; i < regions.length; i++) { + c = regions[i].getCoprocessorHost(). + findCoprocessor(CoprocessorImpl.class.getName()); + c2 = regions[i].getCoprocessorHost(). + findCoprocessor(CoprocessorII.class.getName()); + // make sure that all coprocessor of a class have identical sharedDatas + assertTrue(((CoprocessorImpl)c).getSharedData().get("test1") == o); + assertTrue(((CoprocessorII)c2).getSharedData().get("test2") == o2); + } + // now have all Environments fail + for (int i = 0; i < regions.length; i++) { + try { + Get g = new Get(regions[i].getStartKey()); + regions[i].get(g, null); + fail(); + } catch (DoNotRetryIOException xc) { + } + assertNull(regions[i].getCoprocessorHost(). + findCoprocessor(CoprocessorII.class.getName())); + } + c = regions[0].getCoprocessorHost(). + findCoprocessor(CoprocessorImpl.class.getName()); + assertTrue(((CoprocessorImpl)c).getSharedData().get("test1") == o); + c = c2 = null; + // perform a GC + System.gc(); + // reopen the region + region = reopenRegion(regions[0], CoprocessorImpl.class, CoprocessorII.class); + c = region.getCoprocessorHost(). + findCoprocessor(CoprocessorImpl.class.getName()); + // CPimpl is unaffected, still the same reference + assertTrue(((CoprocessorImpl)c).getSharedData().get("test1") == o); + c2 = region.getCoprocessorHost(). + findCoprocessor(CoprocessorII.class.getName()); + // new map and object created, hence the reference is different + // hence the old entry was indeed removed by the GC and new one has been created + assertFalse(((CoprocessorII)c2).getSharedData().get("test2") == o2); + } + + public void testCoprocessorInterface() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [][] families = { fam1, fam2, fam3 }; + + Configuration hc = initSplit(); + HRegion region = initHRegion(tableName, getName(), hc, + new Class[]{CoprocessorImpl.class}, families); + for (int i = 0; i < 3; i++) { + addContent(region, fam3); + region.flushcache(); + } + + region.compactStores(); + + byte [] splitRow = region.checkSplit(); + + assertNotNull(splitRow); + HRegion [] regions = split(region, splitRow); + for (int i = 0; i < regions.length; i++) { + regions[i] = reopenRegion(regions[i], CoprocessorImpl.class); + } + region.close(); + region.getLog().closeAndDelete(); + Coprocessor c = region.getCoprocessorHost(). + findCoprocessor(CoprocessorImpl.class.getName()); + + // HBASE-4197 + Scan s = new Scan(); + RegionScanner scanner = regions[0].getCoprocessorHost().postScannerOpen(s, regions[0].getScanner(s)); + assertTrue(scanner instanceof CustomScanner); + // this would throw an exception before HBASE-4197 + scanner.next(new ArrayList()); + + assertTrue("Coprocessor not started", ((CoprocessorImpl)c).wasStarted()); + assertTrue("Coprocessor not stopped", ((CoprocessorImpl)c).wasStopped()); + assertTrue(((CoprocessorImpl)c).wasOpened()); + assertTrue(((CoprocessorImpl)c).wasClosed()); + assertTrue(((CoprocessorImpl)c).wasFlushed()); + assertTrue(((CoprocessorImpl)c).wasCompacted()); + assertTrue(((CoprocessorImpl)c).wasSplit()); + + for (int i = 0; i < regions.length; i++) { + regions[i].close(); + regions[i].getLog().closeAndDelete(); + c = region.getCoprocessorHost() + .findCoprocessor(CoprocessorImpl.class.getName()); + assertTrue("Coprocessor not started", ((CoprocessorImpl)c).wasStarted()); + assertTrue("Coprocessor not stopped", ((CoprocessorImpl)c).wasStopped()); + assertTrue(((CoprocessorImpl)c).wasOpened()); + assertTrue(((CoprocessorImpl)c).wasClosed()); + assertTrue(((CoprocessorImpl)c).wasCompacted()); + } + } + + HRegion reopenRegion(final HRegion closedRegion, Class ... implClasses) + throws IOException { + //HRegionInfo info = new HRegionInfo(tableName, null, null, false); + HRegion r = new HRegion(closedRegion); + r.initialize(); + + // this following piece is a hack. currently a coprocessorHost + // is secretly loaded at OpenRegionHandler. we don't really + // start a region server here, so just manually create cphost + // and set it to region. + RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf); + r.setCoprocessorHost(host); + + for (Class implClass : implClasses) { + host.load(implClass, Coprocessor.PRIORITY_USER, conf); + } + // we need to manually call pre- and postOpen here since the + // above load() is not the real case for CP loading. A CP is + // expected to be loaded by default from 1) configuration; or 2) + // HTableDescriptor. If it's loaded after HRegion initialized, + // the pre- and postOpen() won't be triggered automatically. + // Here we have to call pre and postOpen explicitly. + host.preOpen(); + host.postOpen(); + return r; + } + + HRegion initHRegion (byte [] tableName, String callingMethod, + Configuration conf, Class [] implClasses, byte [][] families) + throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + for(byte [] family : families) { + htd.addFamily(new HColumnDescriptor(family)); + } + HRegionInfo info = new HRegionInfo(tableName, null, null, false); + Path path = new Path(DIR + callingMethod); + HRegion r = HRegion.createHRegion(info, path, conf, htd); + + // this following piece is a hack. + RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf); + r.setCoprocessorHost(host); + + for (Class implClass : implClasses) { + host.load(implClass, Coprocessor.PRIORITY_USER, conf); + Coprocessor c = host.findCoprocessor(implClass.getName()); + assertNotNull(c); + } + + // Here we have to call pre and postOpen explicitly. + host.preOpen(); + host.postOpen(); + return r; + } + + Configuration initSplit() { + // Always compact if there is more than one store file. + TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 2); + // Make lease timeout longer, lease checks less frequent + TEST_UTIL.getConfiguration().setInt( + "hbase.master.lease.thread.wakefrequency", 5 * 1000); + TEST_UTIL.getConfiguration().setInt( + "hbase.regionserver.lease.period", 10 * 1000); + // Increase the amount of time between client retries + TEST_UTIL.getConfiguration().setLong("hbase.client.pause", 15 * 1000); + // This size should make it so we always split using the addContent + // below. After adding all data, the first region is 1.3M + TEST_UTIL.getConfiguration().setLong(HConstants.HREGION_MAX_FILESIZE, + 1024 * 128); + TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", + true); + + return TEST_UTIL.getConfiguration(); + } + + private HRegion [] split(final HRegion r, final byte [] splitRow) + throws IOException { + + HRegion[] regions = new HRegion[2]; + + SplitTransaction st = new SplitTransaction(r, splitRow); + int i = 0; + + if (!st.prepare()) { + // test fails. + assertTrue(false); + } + try { + Server mockServer = Mockito.mock(Server.class); + when(mockServer.getConfiguration()).thenReturn( + TEST_UTIL.getConfiguration()); + PairOfSameType daughters = st.execute(mockServer, null); + for (HRegion each_daughter: daughters) { + regions[i] = each_daughter; + i++; + } + } catch (IOException ioe) { + LOG.info("Split transaction of " + r.getRegionNameAsString() + + " failed:" + ioe.getMessage()); + assertTrue(false); + } catch (RuntimeException e) { + LOG.info("Failed rollback of failed split of " + + r.getRegionNameAsString() + e.getMessage()); + } + + assertTrue(i == 2); + return regions; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + + + diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java new file mode 100644 index 0000000..0e0b422 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java @@ -0,0 +1,230 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.io.InterruptedIOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterCoprocessorHost; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +/** + * Tests unhandled exceptions thrown by coprocessors running on master. + * Expected result is that the master will abort with an informative + * error message describing the set of its loaded coprocessors for crash diagnosis. + * (HBASE-4014). + */ +@Category(MediumTests.class) +public class TestMasterCoprocessorExceptionWithAbort { + + public static class MasterTracker extends ZooKeeperNodeTracker { + public boolean masterZKNodeWasDeleted = false; + + public MasterTracker(ZooKeeperWatcher zkw, String masterNode, Abortable abortable) { + super(zkw, masterNode, abortable); + } + + @Override + public synchronized void nodeDeleted(String path) { + if (path.equals("/hbase/master")) { + masterZKNodeWasDeleted = true; + } + } + } + + public static class CreateTableThread extends Thread { + HBaseTestingUtility UTIL; + public CreateTableThread(HBaseTestingUtility UTIL) { + this.UTIL = UTIL; + } + + @Override + public void run() { + // create a table : master coprocessor will throw an exception and not + // catch it. + HTableDescriptor htd = new HTableDescriptor(TEST_TABLE); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + try { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + admin.createTable(htd); + fail("BuggyMasterObserver failed to throw an exception."); + } catch (IOException e) { + assertEquals("HBaseAdmin threw an interrupted IOException as expected.", + e.getClass().getName(), "java.io.InterruptedIOException"); + } + } + } + + public static class BuggyMasterObserver extends BaseMasterObserver { + private boolean preCreateTableCalled; + private boolean postCreateTableCalled; + private boolean startCalled; + private boolean postStartMasterCalled; + + @Override + public void postCreateTable(ObserverContext env, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + // cause a NullPointerException and don't catch it: this will cause the + // master to abort(). + Integer i; + i = null; + i = i++; + } + + public boolean wasCreateTableCalled() { + return preCreateTableCalled && postCreateTableCalled; + } + + @Override + public void postStartMaster(ObserverContext ctx) + throws IOException { + postStartMasterCalled = true; + } + + public boolean wasStartMasterCalled() { + return postStartMasterCalled; + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + startCalled = true; + } + + public boolean wasStarted() { + return startCalled; + } + } + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static byte[] TEST_TABLE = Bytes.toBytes("observed_table"); + private static byte[] TEST_FAMILY = Bytes.toBytes("fam1"); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, + BuggyMasterObserver.class.getName()); + conf.set("hbase.coprocessor.abortonerror", "true"); + UTIL.startMiniCluster(); + } + + @AfterClass + public static void teardownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test(timeout=30000) + public void testExceptionFromCoprocessorWhenCreatingTable() + throws IOException { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + + HMaster master = cluster.getMaster(); + MasterCoprocessorHost host = master.getCoprocessorHost(); + BuggyMasterObserver cp = (BuggyMasterObserver)host.findCoprocessor( + BuggyMasterObserver.class.getName()); + assertFalse("No table created yet", cp.wasCreateTableCalled()); + + // set a watch on the zookeeper /hbase/master node. If the master dies, + // the node will be deleted. + ZooKeeperWatcher zkw = new ZooKeeperWatcher(UTIL.getConfiguration(), + "unittest", new Abortable() { + @Override + public void abort(String why, Throwable e) { + throw new RuntimeException("Fatal ZK error: " + why, e); + } + @Override + public boolean isAborted() { + return false; + } + }); + + MasterTracker masterTracker = new MasterTracker(zkw,"/hbase/master", + new Abortable() { + @Override + public void abort(String why, Throwable e) { + throw new RuntimeException("Fatal ZK master tracker error, why=", e); + } + @Override + public boolean isAborted() { + return false; + } + }); + + masterTracker.start(); + zkw.registerListener(masterTracker); + + // Test (part of the) output that should have be printed by master when it aborts: + // (namely the part that shows the set of loaded coprocessors). + // In this test, there is only a single coprocessor (BuggyMasterObserver). + assertTrue(master.getLoadedCoprocessors(). + equals("[" + + TestMasterCoprocessorExceptionWithAbort.BuggyMasterObserver.class.getName() + + "]")); + + CreateTableThread createTableThread = new CreateTableThread(UTIL); + + // Attempting to create a table (using createTableThread above) triggers an NPE in BuggyMasterObserver. + // Master will then abort and the /hbase/master zk node will be deleted. + createTableThread.start(); + + // Wait up to 30 seconds for master's /hbase/master zk node to go away after master aborts. + for (int i = 0; i < 30; i++) { + if (masterTracker.masterZKNodeWasDeleted == true) { + break; + } + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + fail("InterruptedException while waiting for master zk node to " + + "be deleted."); + } + } + + assertTrue("Master aborted on coprocessor exception, as expected.", + masterTracker.masterZKNodeWasDeleted); + + createTableThread.interrupt(); + try { + createTableThread.join(1000); + } catch (InterruptedException e) { + assertTrue("Ignoring InterruptedException while waiting for " + + " createTableThread.join().", true); + } + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java new file mode 100644 index 0000000..d7e0f65 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java @@ -0,0 +1,223 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.io.InterruptedIOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterCoprocessorHost; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +/** + * Tests unhandled exceptions thrown by coprocessors running on master. + * Expected result is that the master will remove the buggy coprocessor from + * its set of coprocessors and throw a org.apache.hadoop.hbase.DoNotRetryIOException + * back to the client. + * (HBASE-4014). + */ +@Category(MediumTests.class) +public class TestMasterCoprocessorExceptionWithRemove { + + public static class MasterTracker extends ZooKeeperNodeTracker { + public boolean masterZKNodeWasDeleted = false; + + public MasterTracker(ZooKeeperWatcher zkw, String masterNode, Abortable abortable) { + super(zkw, masterNode, abortable); + } + + @Override + public synchronized void nodeDeleted(String path) { + if (path.equals("/hbase/master")) { + masterZKNodeWasDeleted = true; + } + } + } + + public static class BuggyMasterObserver extends BaseMasterObserver { + private boolean preCreateTableCalled; + private boolean postCreateTableCalled; + private boolean startCalled; + private boolean postStartMasterCalled; + + @Override + public void postCreateTable(ObserverContext env, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + // Cause a NullPointerException and don't catch it: this should cause the + // master to throw an o.apache.hadoop.hbase.DoNotRetryIOException to the + // client. + Integer i; + i = null; + i = i++; + } + + public boolean wasCreateTableCalled() { + return preCreateTableCalled && postCreateTableCalled; + } + + @Override + public void postStartMaster(ObserverContext ctx) + throws IOException { + postStartMasterCalled = true; + } + + public boolean wasStartMasterCalled() { + return postStartMasterCalled; + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + startCalled = true; + } + + public boolean wasStarted() { + return startCalled; + } + } + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static byte[] TEST_TABLE1 = Bytes.toBytes("observed_table1"); + private static byte[] TEST_FAMILY1 = Bytes.toBytes("fam1"); + + private static byte[] TEST_TABLE2 = Bytes.toBytes("table2"); + private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2"); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, + BuggyMasterObserver.class.getName()); + UTIL.startMiniCluster(); + } + + @AfterClass + public static void teardownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test(timeout=30000) + public void testExceptionFromCoprocessorWhenCreatingTable() + throws IOException { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + + HMaster master = cluster.getMaster(); + MasterCoprocessorHost host = master.getCoprocessorHost(); + BuggyMasterObserver cp = (BuggyMasterObserver)host.findCoprocessor( + BuggyMasterObserver.class.getName()); + assertFalse("No table created yet", cp.wasCreateTableCalled()); + + // Set a watch on the zookeeper /hbase/master node. If the master dies, + // the node will be deleted. + // Master should *NOT* die: + // we are testing that the default setting of hbase.coprocessor.abortonerror + // =false + // is respected. + ZooKeeperWatcher zkw = new ZooKeeperWatcher(UTIL.getConfiguration(), + "unittest", new Abortable() { + @Override + public void abort(String why, Throwable e) { + throw new RuntimeException("Fatal ZK error: " + why, e); + } + @Override + public boolean isAborted() { + return false; + } + }); + + MasterTracker masterTracker = new MasterTracker(zkw,"/hbase/master", + new Abortable() { + @Override + public void abort(String why, Throwable e) { + throw new RuntimeException("Fatal Zookeeper tracker error, why=", e); + } + @Override + public boolean isAborted() { + return false; + } + }); + + masterTracker.start(); + zkw.registerListener(masterTracker); + + // Test (part of the) output that should have be printed by master when it aborts: + // (namely the part that shows the set of loaded coprocessors). + // In this test, there is only a single coprocessor (BuggyMasterObserver). + String coprocessorName = + BuggyMasterObserver.class.getName(); + assertTrue(master.getLoadedCoprocessors().equals("[" + coprocessorName + "]")); + + HTableDescriptor htd1 = new HTableDescriptor(TEST_TABLE1); + htd1.addFamily(new HColumnDescriptor(TEST_FAMILY1)); + + boolean threwDNRE = false; + try { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + admin.createTable(htd1); + } catch (IOException e) { + if (e.getClass().getName().equals("org.apache.hadoop.hbase.DoNotRetryIOException")) { + threwDNRE = true; + } + } finally { + assertTrue(threwDNRE); + } + + // wait for a few seconds to make sure that the Master hasn't aborted. + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + fail("InterruptedException while sleeping."); + } + + assertFalse("Master survived coprocessor NPE, as expected.", + masterTracker.masterZKNodeWasDeleted); + + String loadedCoprocessors = master.getLoadedCoprocessors(); + assertTrue(loadedCoprocessors.equals("[" + coprocessorName + "]")); + + // Verify that BuggyMasterObserver has been removed due to its misbehavior + // by creating another table: should not have a problem this time. + HTableDescriptor htd2 = new HTableDescriptor(TEST_TABLE2); + htd2.addFamily(new HColumnDescriptor(TEST_FAMILY2)); + HBaseAdmin admin = UTIL.getHBaseAdmin(); + try { + admin.createTable(htd2); + } catch (IOException e) { + fail("Failed to create table after buggy coprocessor removal: " + e); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java new file mode 100644 index 0000000..57c367a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java @@ -0,0 +1,1134 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.concurrent.CountDownLatch; + +import junit.framework.Assert; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterCoprocessorHost; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests invocation of the {@link org.apache.hadoop.hbase.coprocessor.MasterObserver} + * interface hooks at all appropriate times during normal HMaster operations. + */ +@Category(MediumTests.class) +public class TestMasterObserver { + private static final Log LOG = LogFactory.getLog(TestMasterObserver.class); + + public static CountDownLatch countDown = new CountDownLatch(1); + + public static class CPMasterObserver implements MasterObserver { + + private boolean bypass = false; + private boolean preCreateTableCalled; + private boolean postCreateTableCalled; + private boolean preDeleteTableCalled; + private boolean postDeleteTableCalled; + private boolean preModifyTableCalled; + private boolean postModifyTableCalled; + private boolean preAddColumnCalled; + private boolean postAddColumnCalled; + private boolean preModifyColumnCalled; + private boolean postModifyColumnCalled; + private boolean preDeleteColumnCalled; + private boolean postDeleteColumnCalled; + private boolean preEnableTableCalled; + private boolean postEnableTableCalled; + private boolean preDisableTableCalled; + private boolean postDisableTableCalled; + private boolean preMoveCalled; + private boolean postMoveCalled; + private boolean preAssignCalled; + private boolean postAssignCalled; + private boolean preUnassignCalled; + private boolean postUnassignCalled; + private boolean preBalanceCalled; + private boolean postBalanceCalled; + private boolean preBalanceSwitchCalled; + private boolean postBalanceSwitchCalled; + private boolean preShutdownCalled; + private boolean preStopMasterCalled; + private boolean postStartMasterCalled; + private boolean startCalled; + private boolean stopCalled; + private boolean preCreateTableHandlerCalled; + private boolean postCreateTableHandlerCalled; + private boolean preDeleteTableHandlerCalled; + private boolean postDeleteTableHandlerCalled; + private boolean preAddColumnHandlerCalled; + private boolean postAddColumnHandlerCalled; + private boolean preModifyColumnHandlerCalled; + private boolean postModifyColumnHandlerCalled; + private boolean preDeleteColumnHandlerCalled; + private boolean postDeleteColumnHandlerCalled; + private boolean preEnableTableHandlerCalled; + private boolean postEnableTableHandlerCalled; + private boolean preDisableTableHandlerCalled; + private boolean postDisableTableHandlerCalled; + private boolean preModifyTableHandlerCalled; + private boolean postModifyTableHandlerCalled; + private boolean preSnapshotCalled; + private boolean postSnapshotCalled; + private boolean preCloneSnapshotCalled; + private boolean postCloneSnapshotCalled; + private boolean preRestoreSnapshotCalled; + private boolean postRestoreSnapshotCalled; + private boolean preDeleteSnapshotCalled; + private boolean postDeleteSnapshotCalled; + + public void enableBypass(boolean bypass) { + this.bypass = bypass; + } + + public void resetStates() { + preCreateTableCalled = false; + postCreateTableCalled = false; + preDeleteTableCalled = false; + postDeleteTableCalled = false; + preModifyTableCalled = false; + postModifyTableCalled = false; + preAddColumnCalled = false; + postAddColumnCalled = false; + preModifyColumnCalled = false; + postModifyColumnCalled = false; + preDeleteColumnCalled = false; + postDeleteColumnCalled = false; + preEnableTableCalled = false; + postEnableTableCalled = false; + preDisableTableCalled = false; + postDisableTableCalled = false; + preMoveCalled= false; + postMoveCalled = false; + preAssignCalled = false; + postAssignCalled = false; + preUnassignCalled = false; + postUnassignCalled = false; + preBalanceCalled = false; + postBalanceCalled = false; + preBalanceSwitchCalled = false; + postBalanceSwitchCalled = false; + preCreateTableHandlerCalled = false; + postCreateTableHandlerCalled = false; + preDeleteTableHandlerCalled = false; + postDeleteTableHandlerCalled = false; + preModifyTableHandlerCalled = false; + postModifyTableHandlerCalled = false; + preAddColumnHandlerCalled = false; + postAddColumnHandlerCalled = false; + preModifyColumnHandlerCalled = false; + postModifyColumnHandlerCalled = false; + preDeleteColumnHandlerCalled = false; + postDeleteColumnHandlerCalled = false; + preEnableTableHandlerCalled = false; + postEnableTableHandlerCalled = false; + preDisableTableHandlerCalled = false; + postDisableTableHandlerCalled = false; + preSnapshotCalled = false; + postSnapshotCalled = false; + preCloneSnapshotCalled = false; + postCloneSnapshotCalled = false; + preRestoreSnapshotCalled = false; + postRestoreSnapshotCalled = false; + preDeleteSnapshotCalled = false; + postDeleteSnapshotCalled = false; + } + + @Override + public void preCreateTable(ObserverContext env, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + if (bypass) { + env.bypass(); + } + preCreateTableCalled = true; + } + + @Override + public void postCreateTable(ObserverContext env, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + postCreateTableCalled = true; + } + + public boolean wasCreateTableCalled() { + return preCreateTableCalled && postCreateTableCalled; + } + + public boolean preCreateTableCalledOnly() { + return preCreateTableCalled && !postCreateTableCalled; + } + + @Override + public void preDeleteTable(ObserverContext env, + byte[] tableName) throws IOException { + if (bypass) { + env.bypass(); + } + preDeleteTableCalled = true; + } + + @Override + public void postDeleteTable(ObserverContext env, + byte[] tableName) throws IOException { + postDeleteTableCalled = true; + } + + public boolean wasDeleteTableCalled() { + return preDeleteTableCalled && postDeleteTableCalled; + } + + public boolean preDeleteTableCalledOnly() { + return preDeleteTableCalled && !postDeleteTableCalled; + } + + @Override + public void preModifyTable(ObserverContext env, + byte[] tableName, HTableDescriptor htd) throws IOException { + if (bypass) { + env.bypass(); + } + preModifyTableCalled = true; + } + + @Override + public void postModifyTable(ObserverContext env, + byte[] tableName, HTableDescriptor htd) throws IOException { + postModifyTableCalled = true; + } + + public boolean wasModifyTableCalled() { + return preModifyTableCalled && postModifyTableCalled; + } + + public boolean preModifyTableCalledOnly() { + return preModifyTableCalled && !postModifyTableCalled; + } + + @Override + public void preAddColumn(ObserverContext env, + byte[] tableName, HColumnDescriptor column) throws IOException { + if (bypass) { + env.bypass(); + } + preAddColumnCalled = true; + } + + @Override + public void postAddColumn(ObserverContext env, + byte[] tableName, HColumnDescriptor column) throws IOException { + postAddColumnCalled = true; + } + + public boolean wasAddColumnCalled() { + return preAddColumnCalled && postAddColumnCalled; + } + + public boolean preAddColumnCalledOnly() { + return preAddColumnCalled && !postAddColumnCalled; + } + + @Override + public void preModifyColumn(ObserverContext env, + byte[] tableName, HColumnDescriptor descriptor) throws IOException { + if (bypass) { + env.bypass(); + } + preModifyColumnCalled = true; + } + + @Override + public void postModifyColumn(ObserverContext env, + byte[] tableName, HColumnDescriptor descriptor) throws IOException { + postModifyColumnCalled = true; + } + + public boolean wasModifyColumnCalled() { + return preModifyColumnCalled && postModifyColumnCalled; + } + + public boolean preModifyColumnCalledOnly() { + return preModifyColumnCalled && !postModifyColumnCalled; + } + + @Override + public void preDeleteColumn(ObserverContext env, + byte[] tableName, byte[] c) throws IOException { + if (bypass) { + env.bypass(); + } + preDeleteColumnCalled = true; + } + + @Override + public void postDeleteColumn(ObserverContext env, + byte[] tableName, byte[] c) throws IOException { + postDeleteColumnCalled = true; + } + + public boolean wasDeleteColumnCalled() { + return preDeleteColumnCalled && postDeleteColumnCalled; + } + + public boolean preDeleteColumnCalledOnly() { + return preDeleteColumnCalled && !postDeleteColumnCalled; + } + + @Override + public void preEnableTable(ObserverContext env, + byte[] tableName) throws IOException { + if (bypass) { + env.bypass(); + } + preEnableTableCalled = true; + } + + @Override + public void postEnableTable(ObserverContext env, + byte[] tableName) throws IOException { + postEnableTableCalled = true; + } + + public boolean wasEnableTableCalled() { + return preEnableTableCalled && postEnableTableCalled; + } + + public boolean preEnableTableCalledOnly() { + return preEnableTableCalled && !postEnableTableCalled; + } + + @Override + public void preDisableTable(ObserverContext env, + byte[] tableName) throws IOException { + if (bypass) { + env.bypass(); + } + preDisableTableCalled = true; + } + + @Override + public void postDisableTable(ObserverContext env, + byte[] tableName) throws IOException { + postDisableTableCalled = true; + } + + public boolean wasDisableTableCalled() { + return preDisableTableCalled && postDisableTableCalled; + } + + public boolean preDisableTableCalledOnly() { + return preDisableTableCalled && !postDisableTableCalled; + } + + @Override + public void preMove(ObserverContext env, + HRegionInfo region, ServerName srcServer, ServerName destServer) + throws IOException { + if (bypass) { + env.bypass(); + } + preMoveCalled = true; + } + + @Override + public void postMove(ObserverContext env, HRegionInfo region, + ServerName srcServer, ServerName destServer) + throws IOException { + postMoveCalled = true; + } + + public boolean wasMoveCalled() { + return preMoveCalled && postMoveCalled; + } + + public boolean preMoveCalledOnly() { + return preMoveCalled && !postMoveCalled; + } + + @Override + public void preAssign(ObserverContext env, + final HRegionInfo regionInfo) throws IOException { + if (bypass) { + env.bypass(); + } + preAssignCalled = true; + } + + @Override + public void postAssign(ObserverContext env, + final HRegionInfo regionInfo) throws IOException { + postAssignCalled = true; + } + + public boolean wasAssignCalled() { + return preAssignCalled && postAssignCalled; + } + + public boolean preAssignCalledOnly() { + return preAssignCalled && !postAssignCalled; + } + + @Override + public void preUnassign(ObserverContext env, + final HRegionInfo regionInfo, final boolean force) throws IOException { + if (bypass) { + env.bypass(); + } + preUnassignCalled = true; + } + + @Override + public void postUnassign(ObserverContext env, + final HRegionInfo regionInfo, final boolean force) throws IOException { + postUnassignCalled = true; + } + + public boolean wasUnassignCalled() { + return preUnassignCalled && postUnassignCalled; + } + + public boolean preUnassignCalledOnly() { + return preUnassignCalled && !postUnassignCalled; + } + + @Override + public void preBalance(ObserverContext env) + throws IOException { + if (bypass) { + env.bypass(); + } + preBalanceCalled = true; + } + + @Override + public void postBalance(ObserverContext env) + throws IOException { + postBalanceCalled = true; + } + + public boolean wasBalanceCalled() { + return preBalanceCalled && postBalanceCalled; + } + + public boolean preBalanceCalledOnly() { + return preBalanceCalled && !postBalanceCalled; + } + + @Override + public boolean preBalanceSwitch(ObserverContext env, boolean b) + throws IOException { + if (bypass) { + env.bypass(); + } + preBalanceSwitchCalled = true; + return b; + } + + @Override + public void postBalanceSwitch(ObserverContext env, + boolean oldValue, boolean newValue) throws IOException { + postBalanceSwitchCalled = true; + } + + public boolean wasBalanceSwitchCalled() { + return preBalanceSwitchCalled && postBalanceSwitchCalled; + } + + public boolean preBalanceSwitchCalledOnly() { + return preBalanceSwitchCalled && !postBalanceSwitchCalled; + } + + @Override + public void preShutdown(ObserverContext env) + throws IOException { + preShutdownCalled = true; + } + + @Override + public void preStopMaster(ObserverContext env) + throws IOException { + preStopMasterCalled = true; + } + + @Override + public void postStartMaster(ObserverContext ctx) + throws IOException { + postStartMasterCalled = true; + } + + public boolean wasStartMasterCalled() { + return postStartMasterCalled; + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + startCalled = true; + } + + @Override + public void stop(CoprocessorEnvironment env) throws IOException { + stopCalled = true; + } + + public boolean wasStarted() { return startCalled; } + + public boolean wasStopped() { return stopCalled; } + + @Override + public void preCreateTableHandler(ObserverContext env, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + if (bypass) { + env.bypass(); + } + preCreateTableHandlerCalled = true; + } + + @Override + public void postCreateTableHandler(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + postCreateTableHandlerCalled = true; + countDown.countDown(); + } + + public boolean wasPreCreateTableHandlerCalled() { + return preCreateTableHandlerCalled; + } + + public boolean wasCreateTableHandlerCalled() { + return preCreateTableHandlerCalled && postCreateTableHandlerCalled; + } + + public boolean wasCreateTableHandlerCalledOnly() { + return preCreateTableHandlerCalled && !postCreateTableHandlerCalled; + } + + @Override + public void preDeleteTableHandler(ObserverContext env, + byte[] tableName) throws IOException { + if (bypass) { + env.bypass(); + } + preDeleteTableHandlerCalled = true; + } + + @Override + public void postDeleteTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + postDeleteTableHandlerCalled = true; + } + + public boolean wasDeleteTableHandlerCalled() { + return preDeleteTableHandlerCalled && postDeleteTableHandlerCalled; + } + + public boolean wasDeleteTableHandlerCalledOnly() { + return preDeleteTableHandlerCalled && !postDeleteTableHandlerCalled; + } + + @Override + public void preModifyTableHandler(ObserverContext env, + byte[] tableName, HTableDescriptor htd) throws IOException { + if (bypass) { + env.bypass(); + } + preModifyTableHandlerCalled = true; + } + + @Override + public void postModifyTableHandler(ObserverContext env, + byte[] tableName, HTableDescriptor htd) throws IOException { + postModifyTableHandlerCalled = true; + } + + public boolean wasModifyTableHandlerCalled() { + return preModifyColumnHandlerCalled && postModifyColumnHandlerCalled; + } + + public boolean wasModifyTableHandlerCalledOnly() { + return preModifyColumnHandlerCalled && !postModifyColumnHandlerCalled; + } + + @Override + public void preAddColumnHandler(ObserverContext env, + byte[] tableName, HColumnDescriptor column) throws IOException { + if (bypass) { + env.bypass(); + } + preAddColumnHandlerCalled = true; + } + + @Override + public void postAddColumnHandler(ObserverContext ctx, + byte[] tableName, HColumnDescriptor column) throws IOException { + postAddColumnHandlerCalled = true; + } + + public boolean wasAddColumnHandlerCalled() { + return preAddColumnHandlerCalled && postAddColumnHandlerCalled; + } + + public boolean preAddColumnHandlerCalledOnly() { + return preAddColumnHandlerCalled && !postAddColumnHandlerCalled; + } + + @Override + public void preModifyColumnHandler(ObserverContext env, + byte[] tableName, HColumnDescriptor descriptor) throws IOException { + if (bypass) { + env.bypass(); + } + preModifyColumnHandlerCalled = true; + } + + @Override + public void postModifyColumnHandler(ObserverContext ctx, + byte[] tableName, HColumnDescriptor descriptor) throws IOException { + postModifyColumnHandlerCalled = true; + } + + public boolean wasModifyColumnHandlerCalled() { + return preModifyColumnHandlerCalled && postModifyColumnHandlerCalled; + } + + public boolean preModifyColumnHandlerCalledOnly() { + return preModifyColumnHandlerCalled && !postModifyColumnHandlerCalled; + } + + @Override + public void preDeleteColumnHandler(ObserverContext env, + byte[] tableName, byte[] c) throws IOException { + if (bypass) { + env.bypass(); + } + preDeleteColumnHandlerCalled = true; + } + + @Override + public void postDeleteColumnHandler(ObserverContext ctx, + byte[] tableName, byte[] c) throws IOException { + postDeleteColumnHandlerCalled = true; + } + + public boolean wasDeleteColumnHandlerCalled() { + return preDeleteColumnHandlerCalled && postDeleteColumnHandlerCalled; + } + + public boolean preDeleteColumnHandlerCalledOnly() { + return preDeleteColumnHandlerCalled && !postDeleteColumnHandlerCalled; + } + + @Override + public void preEnableTableHandler(ObserverContext env, + byte[] tableName) throws IOException { + if (bypass) { + env.bypass(); + } + preEnableTableHandlerCalled = true; + } + + @Override + public void postEnableTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + postEnableTableHandlerCalled = true; + } + + public boolean wasEnableTableHandlerCalled() { + return preEnableTableHandlerCalled && postEnableTableHandlerCalled; + } + + public boolean preEnableTableHandlerCalledOnly() { + return preEnableTableHandlerCalled && !postEnableTableHandlerCalled; + } + + @Override + public void preDisableTableHandler(ObserverContext env, + byte[] tableName) throws IOException { + if (bypass) { + env.bypass(); + } + preDisableTableHandlerCalled = true; + } + + @Override + public void postDisableTableHandler(ObserverContext ctx, + byte[] tableName) throws IOException { + postDisableTableHandlerCalled = true; + } + + public boolean wasDisableTableHandlerCalled() { + return preDisableTableHandlerCalled && postDisableTableHandlerCalled; + } + + public boolean preDisableTableHandlerCalledOnly() { + return preDisableTableHandlerCalled && !postDisableTableHandlerCalled; + } + + @Override + public void preSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + preSnapshotCalled = true; + } + + @Override + public void postSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + postSnapshotCalled = true; + } + + public boolean wasSnapshotCalled() { + return preSnapshotCalled && postSnapshotCalled; + } + + @Override + public void preCloneSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + preCloneSnapshotCalled = true; + } + + @Override + public void postCloneSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + postCloneSnapshotCalled = true; + } + + public boolean wasCloneSnapshotCalled() { + return preCloneSnapshotCalled && postCloneSnapshotCalled; + } + + @Override + public void preRestoreSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + preRestoreSnapshotCalled = true; + } + + @Override + public void postRestoreSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + throws IOException { + postRestoreSnapshotCalled = true; + } + + public boolean wasRestoreSnapshotCalled() { + return preRestoreSnapshotCalled && postRestoreSnapshotCalled; + } + + @Override + public void preDeleteSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot) throws IOException { + preDeleteSnapshotCalled = true; + } + + @Override + public void postDeleteSnapshot(final ObserverContext ctx, + final SnapshotDescription snapshot) throws IOException { + postDeleteSnapshotCalled = true; + } + + public boolean wasDeleteSnapshotCalled() { + return preDeleteSnapshotCalled && postDeleteSnapshotCalled; + } + } + + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static byte[] TEST_SNAPSHOT = Bytes.toBytes("observed_snapshot"); + private static byte[] TEST_TABLE = Bytes.toBytes("observed_table"); + private static byte[] TEST_CLONE = Bytes.toBytes("observed_clone"); + private static byte[] TEST_FAMILY = Bytes.toBytes("fam1"); + private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2"); + private static byte[] TEST_FAMILY3 = Bytes.toBytes("fam3"); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, + CPMasterObserver.class.getName()); + // Enable snapshot + conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + // We need more than one data server on this test + UTIL.startMiniCluster(2); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test + public void testStarted() throws Exception { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + + HMaster master = cluster.getMaster(); + assertTrue("Master should be active", master.isActiveMaster()); + MasterCoprocessorHost host = master.getCoprocessorHost(); + assertNotNull("CoprocessorHost should not be null", host); + CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor( + CPMasterObserver.class.getName()); + assertNotNull("CPMasterObserver coprocessor not found or not installed!", cp); + + // check basic lifecycle + assertTrue("MasterObserver should have been started", cp.wasStarted()); + assertTrue("postStartMaster() hook should have been called", + cp.wasStartMasterCalled()); + } + + @Test + public void testTableOperations() throws Exception { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + + HMaster master = cluster.getMaster(); + MasterCoprocessorHost host = master.getCoprocessorHost(); + CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor( + CPMasterObserver.class.getName()); + cp.enableBypass(true); + cp.resetStates(); + assertFalse("No table created yet", cp.wasCreateTableCalled()); + + // create a table + HTableDescriptor htd = new HTableDescriptor(TEST_TABLE); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + HBaseAdmin admin = UTIL.getHBaseAdmin(); + + admin.createTable(htd); + // preCreateTable can't bypass default action. + assertTrue("Test table should be created", cp.wasCreateTableCalled()); + countDown.await(); + assertTrue("Table pre create handler called.", cp.wasPreCreateTableHandlerCalled()); + assertTrue("Table create handler should be called.", cp.wasCreateTableHandlerCalled()); + + countDown = new CountDownLatch(1); + admin.disableTable(TEST_TABLE); + assertTrue(admin.isTableDisabled(TEST_TABLE)); + // preDisableTable can't bypass default action. + assertTrue("Coprocessor should have been called on table disable", + cp.wasDisableTableCalled()); + + assertTrue("Disable table handler should be called.", cp.wasDisableTableHandlerCalled()); + assertTrue("Disable table handler should be called.", cp.wasDisableTableHandlerCalled()); + + // enable + assertFalse(cp.wasEnableTableCalled()); + admin.enableTable(TEST_TABLE); + assertTrue(admin.isTableEnabled(TEST_TABLE)); + // preEnableTable can't bypass default action. + assertTrue("Coprocessor should have been called on table enable", + cp.wasEnableTableCalled()); + + assertTrue("Enable table handler should be called.", cp.wasEnableTableHandlerCalled()); + + admin.disableTable(TEST_TABLE); + assertTrue(admin.isTableDisabled(TEST_TABLE)); + + // modify table + htd.setMaxFileSize(512 * 1024 * 1024); + modifyTableSync(admin, TEST_TABLE, htd); + // preModifyTable can't bypass default action. + assertTrue("Test table should have been modified", + cp.wasModifyTableCalled()); + + // add a column family + admin.addColumn(TEST_TABLE, new HColumnDescriptor(TEST_FAMILY2)); + assertTrue("New column family shouldn't have been added to test table", + cp.preAddColumnCalledOnly()); + + // modify a column family + HColumnDescriptor hcd1 = new HColumnDescriptor(TEST_FAMILY2); + hcd1.setMaxVersions(25); + admin.modifyColumn(TEST_TABLE, hcd1); + assertTrue("Second column family should be modified", + cp.preModifyColumnCalledOnly()); + + // delete table + admin.deleteTable(TEST_TABLE); + assertFalse("Test table should have been deleted", + admin.tableExists(TEST_TABLE)); + // preDeleteTable can't bypass default action. + assertTrue("Coprocessor should have been called on table delete", + cp.wasDeleteTableCalled()); + assertTrue("Delete table handler should be called.", cp.wasDeleteTableHandlerCalled()); + + // turn off bypass, run the tests again + cp.enableBypass(false); + cp.resetStates(); + + admin.createTable(htd); + assertTrue("Test table should be created", cp.wasCreateTableCalled()); + + countDown.await(); + assertTrue("Table pre create handler called.", cp.wasPreCreateTableHandlerCalled()); + assertTrue("Table create handler should be called.", cp.wasCreateTableHandlerCalled()); + + // disable + assertFalse(cp.wasDisableTableCalled()); + + assertFalse(cp.wasDisableTableHandlerCalled()); + admin.disableTable(TEST_TABLE); + assertTrue(admin.isTableDisabled(TEST_TABLE)); + assertTrue("Coprocessor should have been called on table disable", + cp.wasDisableTableCalled()); + + assertTrue("Disable table handler should be called.", cp.wasDisableTableHandlerCalled()); + // modify table + htd.setMaxFileSize(512 * 1024 * 1024); + modifyTableSync(admin, TEST_TABLE, htd); + assertTrue("Test table should have been modified", + cp.wasModifyTableCalled()); + + // add a column family + admin.addColumn(TEST_TABLE, new HColumnDescriptor(TEST_FAMILY2)); + assertTrue("New column family should have been added to test table", + cp.wasAddColumnCalled()); + + assertTrue("Add column handler should be called.", cp.wasAddColumnHandlerCalled()); + // modify a column family + HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY2); + hcd.setMaxVersions(25); + admin.modifyColumn(TEST_TABLE, hcd); + assertTrue("Second column family should be modified", + cp.wasModifyColumnCalled()); + + assertTrue("Modify table handler should be called.", cp.wasModifyColumnHandlerCalled()); + // enable + assertFalse(cp.wasEnableTableCalled()); + assertFalse(cp.wasEnableTableHandlerCalled()); + admin.enableTable(TEST_TABLE); + assertTrue(admin.isTableEnabled(TEST_TABLE)); + assertTrue("Coprocessor should have been called on table enable", + cp.wasEnableTableCalled()); + + assertTrue("Enable table handler should be called.", cp.wasEnableTableHandlerCalled()); + // disable again + admin.disableTable(TEST_TABLE); + assertTrue(admin.isTableDisabled(TEST_TABLE)); + + // delete column + assertFalse("No column family deleted yet", cp.wasDeleteColumnCalled()); + assertFalse("Delete table column handler should not be called.", + cp.wasDeleteColumnHandlerCalled()); + admin.deleteColumn(TEST_TABLE, TEST_FAMILY2); + HTableDescriptor tableDesc = admin.getTableDescriptor(TEST_TABLE); + assertNull("'"+Bytes.toString(TEST_FAMILY2)+"' should have been removed", + tableDesc.getFamily(TEST_FAMILY2)); + assertTrue("Coprocessor should have been called on column delete", + cp.wasDeleteColumnCalled()); + + assertTrue("Delete table column handler should be called.", cp.wasDeleteColumnHandlerCalled()); + // delete table + assertFalse("No table deleted yet", cp.wasDeleteTableCalled()); + assertFalse("Delete table handler should not be called.", cp.wasDeleteTableHandlerCalled()); + admin.deleteTable(TEST_TABLE); + assertFalse("Test table should have been deleted", + admin.tableExists(TEST_TABLE)); + assertTrue("Coprocessor should have been called on table delete", + cp.wasDeleteTableCalled()); + assertTrue("Delete table handler should be called.", cp.wasDeleteTableHandlerCalled()); + } + + private void modifyTableSync(HBaseAdmin admin, byte[] tableName, HTableDescriptor htd) + throws IOException { + admin.modifyTable(tableName, htd); + //wait until modify table finishes + for (int t = 0; t < 100; t++) { //10 sec timeout + HTableDescriptor td = admin.getTableDescriptor(htd.getName()); + if (td.equals(htd)) { + break; + } + Threads.sleep(100); + } + } + + @Test + public void testRegionTransitionOperations() throws Exception { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + + HMaster master = cluster.getMaster(); + MasterCoprocessorHost host = master.getCoprocessorHost(); + CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor( + CPMasterObserver.class.getName()); + cp.enableBypass(false); + cp.resetStates(); + + HTable table = UTIL.createTable(TEST_TABLE, TEST_FAMILY); + UTIL.createMultiRegions(table, TEST_FAMILY); + UTIL.waitUntilAllRegionsAssigned(TEST_TABLE); + + NavigableMap regions = table.getRegionLocations(); + Map.Entry firstGoodPair = null; + for (Map.Entry e: regions.entrySet()) { + if (e.getValue() != null) { + firstGoodPair = e; + break; + } + } + assertNotNull("Found a non-null entry", firstGoodPair); + LOG.info("Found " + firstGoodPair.toString()); + // Try to force a move + Collection servers = master.getClusterStatus().getServers(); + String destName = null; + String firstRegionHostnamePortStr = firstGoodPair.getValue().toString(); + LOG.info("firstRegionHostnamePortStr=" + firstRegionHostnamePortStr); + boolean found = false; + // Find server that is NOT carrying the first region + for (ServerName info : servers) { + LOG.info("ServerName=" + info); + if (!firstRegionHostnamePortStr.equals(info.getHostAndPort())) { + destName = info.toString(); + found = true; + break; + } + } + assertTrue("Found server", found); + LOG.info("Found " + destName); + master.move(firstGoodPair.getKey().getEncodedNameAsBytes(), + Bytes.toBytes(destName)); + assertTrue("Coprocessor should have been called on region move", + cp.wasMoveCalled()); + + // make sure balancer is on + master.balanceSwitch(true); + assertTrue("Coprocessor should have been called on balance switch", + cp.wasBalanceSwitchCalled()); + + // force region rebalancing + master.balanceSwitch(false); + // move half the open regions from RS 0 to RS 1 + HRegionServer rs = cluster.getRegionServer(0); + byte[] destRS = Bytes.toBytes(cluster.getRegionServer(1).getServerName().toString()); + //Make sure no regions are in transition now + waitForRITtoBeZero(master); + List openRegions = rs.getOnlineRegions(); + int moveCnt = openRegions.size()/2; + for (int i=0; i transRegions = + mgr.getRegionsInTransition().values(); + for (AssignmentManager.RegionState state : transRegions) { + mgr.waitOnRegionToClearRegionsInTransition(state.getRegion()); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java new file mode 100644 index 0000000..7a118da --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java @@ -0,0 +1,200 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; + +@Category(MediumTests.class) +public class TestRegionObserverBypass { + private static HBaseTestingUtility util; + private static final byte[] tableName = Bytes.toBytes("test"); + private static final byte[] dummy = Bytes.toBytes("dummy"); + private static final byte[] row1 = Bytes.toBytes("r1"); + private static final byte[] row2 = Bytes.toBytes("r2"); + private static final byte[] row3 = Bytes.toBytes("r3"); + private static final byte[] test = Bytes.toBytes("test"); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration conf = HBaseConfiguration.create(); + conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, + TestCoprocessor.class.getName()); + util = new HBaseTestingUtility(conf); + util.startMiniCluster(); + util.createTable(tableName, new byte[][] {dummy, test}); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + util.shutdownMiniCluster(); + } + + /** + * do a single put that is bypassed by a RegionObserver + * @throws Exception + */ + @Test + public void testSimple() throws Exception { + HTable t = new HTable(util.getConfiguration(), tableName); + Put p = new Put(row1); + p.add(test,dummy,dummy); + // before HBASE-4331, this would throw an exception + t.put(p); + checkRowAndDelete(t,row1,0); + t.close(); + } + + /** + * Test various multiput operations. + * @throws Exception + */ + @Test + public void testMulti() throws Exception { + HTable t = new HTable(util.getConfiguration(), tableName); + List puts = new ArrayList(); + Put p = new Put(row1); + p.add(dummy,dummy,dummy); + puts.add(p); + p = new Put(row2); + p.add(test,dummy,dummy); + puts.add(p); + p = new Put(row3); + p.add(test,dummy,dummy); + puts.add(p); + // before HBASE-4331, this would throw an exception + t.put(puts); + checkRowAndDelete(t,row1,1); + checkRowAndDelete(t,row2,0); + checkRowAndDelete(t,row3,0); + + puts.clear(); + p = new Put(row1); + p.add(test,dummy,dummy); + puts.add(p); + p = new Put(row2); + p.add(test,dummy,dummy); + puts.add(p); + p = new Put(row3); + p.add(test,dummy,dummy); + puts.add(p); + // before HBASE-4331, this would throw an exception + t.put(puts); + checkRowAndDelete(t,row1,0); + checkRowAndDelete(t,row2,0); + checkRowAndDelete(t,row3,0); + + puts.clear(); + p = new Put(row1); + p.add(test,dummy,dummy); + puts.add(p); + p = new Put(row2); + p.add(test,dummy,dummy); + puts.add(p); + p = new Put(row3); + p.add(dummy,dummy,dummy); + puts.add(p); + // this worked fine even before HBASE-4331 + t.put(puts); + checkRowAndDelete(t,row1,0); + checkRowAndDelete(t,row2,0); + checkRowAndDelete(t,row3,1); + + puts.clear(); + p = new Put(row1); + p.add(dummy,dummy,dummy); + puts.add(p); + p = new Put(row2); + p.add(test,dummy,dummy); + puts.add(p); + p = new Put(row3); + p.add(dummy,dummy,dummy); + puts.add(p); + // this worked fine even before HBASE-4331 + t.put(puts); + checkRowAndDelete(t,row1,1); + checkRowAndDelete(t,row2,0); + checkRowAndDelete(t,row3,1); + + puts.clear(); + p = new Put(row1); + p.add(test,dummy,dummy); + puts.add(p); + p = new Put(row2); + p.add(dummy,dummy,dummy); + puts.add(p); + p = new Put(row3); + p.add(test,dummy,dummy); + puts.add(p); + // before HBASE-4331, this would throw an exception + t.put(puts); + checkRowAndDelete(t,row1,0); + checkRowAndDelete(t,row2,1); + checkRowAndDelete(t,row3,0); + t.close(); + } + + private void checkRowAndDelete(HTable t, byte[] row, int count) throws IOException { + Get g = new Get(row); + Result r = t.get(g); + assertEquals(count, r.size()); + Delete d = new Delete(row); + t.delete(d); + } + + public static class TestCoprocessor extends BaseRegionObserver { + @Override + public void prePut(final ObserverContext e, + final Put put, final WALEdit edit, final boolean writeToWAL) + throws IOException { + Map> familyMap = put.getFamilyMap(); + if (familyMap.containsKey(test)) { + e.bypass(); + } + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java new file mode 100644 index 0000000..bba0365 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java @@ -0,0 +1,528 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.JVMClusterUtil; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +@Category(MediumTests.class) +public class TestRegionObserverInterface { + static final Log LOG = LogFactory.getLog(TestRegionObserverInterface.class); + static final String DIR = "test/build/data/TestRegionObserver/"; + + public static final byte[] TEST_TABLE = Bytes.toBytes("TestTable"); + public final static byte[] A = Bytes.toBytes("a"); + public final static byte[] B = Bytes.toBytes("b"); + public final static byte[] C = Bytes.toBytes("c"); + public final static byte[] ROW = Bytes.toBytes("testrow"); + + private static HBaseTestingUtility util = new HBaseTestingUtility(); + private static MiniHBaseCluster cluster = null; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // set configure to indicate which cp should be loaded + Configuration conf = util.getConfiguration(); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver"); + + util.startMiniCluster(); + cluster = util.getMiniHBaseCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + util.shutdownMiniCluster(); + } + + @Test + public void testRegionObserver() throws IOException { + byte[] tableName = TEST_TABLE; + // recreate table every time in order to reset the status of the + // coproccessor. + HTable table = util.createTable(tableName, new byte[][] {A, B, C}); + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", + "hadDelete"}, + TEST_TABLE, + new Boolean[] {false, false, false, false, false}); + + Put put = new Put(ROW); + put.add(A, A, A); + put.add(B, B, B); + put.add(C, C, C); + table.put(put); + + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", + "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"}, + TEST_TABLE, + new Boolean[] {false, false, true, true, true, true, false} + ); + + Get get = new Get(ROW); + get.addColumn(A, A); + get.addColumn(B, B); + get.addColumn(C, C); + table.get(get); + + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", + "hadDelete"}, + TEST_TABLE, + new Boolean[] {true, true, true, true, false} + ); + + Delete delete = new Delete(ROW); + delete.deleteColumn(A, A); + delete.deleteColumn(B, B); + delete.deleteColumn(C, C); + table.delete(delete); + + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", + "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"}, + TEST_TABLE, + new Boolean[] {true, true, true, true, true, true, true} + ); + util.deleteTable(tableName); + table.close(); + } + + @Test + public void testRowMutation() throws IOException { + byte[] tableName = TEST_TABLE; + HTable table = util.createTable(tableName, new byte[][] {A, B, C}); + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", + "hadDeleted"}, + TEST_TABLE, + new Boolean[] {false, false, false, false, false}); + + Put put = new Put(ROW); + put.add(A, A, A); + put.add(B, B, B); + put.add(C, C, C); + + Delete delete = new Delete(ROW); + delete.deleteColumn(A, A); + delete.deleteColumn(B, B); + delete.deleteColumn(C, C); + + RowMutations arm = new RowMutations(ROW); + arm.add(put); + arm.add(delete); + table.mutateRow(arm); + + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", + "hadDeleted"}, + TEST_TABLE, + new Boolean[] {false, false, true, true, true} + ); + util.deleteTable(tableName); + table.close(); + } + + @Test + public void testIncrementHook() throws IOException { + byte[] tableName = TEST_TABLE; + + HTable table = util.createTable(tableName, new byte[][] {A, B, C}); + Increment inc = new Increment(Bytes.toBytes(0)); + inc.addColumn(A, A, 1); + + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadPreIncrement", "hadPostIncrement"}, + tableName, + new Boolean[] {false, false} + ); + + table.increment(inc); + + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadPreIncrement", "hadPostIncrement"}, + tableName, + new Boolean[] {true, true} + ); + util.deleteTable(tableName); + table.close(); + } + + @Test + // HBase-3583 + public void testHBase3583() throws IOException { + byte[] tableName = Bytes.toBytes("testHBase3583"); + util.createTable(tableName, new byte[][] {A, B, C}); + + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled", + "wasScannerCloseCalled"}, + tableName, + new Boolean[] {false, false, false, false} + ); + + HTable table = new HTable(util.getConfiguration(), tableName); + Put put = new Put(ROW); + put.add(A, A, A); + table.put(put); + + Get get = new Get(ROW); + get.addColumn(A, A); + table.get(get); + + // verify that scannerNext and scannerClose upcalls won't be invoked + // when we perform get(). + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled", + "wasScannerCloseCalled"}, + tableName, + new Boolean[] {true, true, false, false} + ); + + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + try { + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + } + } finally { + scanner.close(); + } + + // now scanner hooks should be invoked. + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"wasScannerNextCalled", "wasScannerCloseCalled"}, + tableName, + new Boolean[] {true, true} + ); + util.deleteTable(tableName); + table.close(); + } + + @Test + // HBase-3758 + public void testHBase3758() throws IOException { + byte[] tableName = Bytes.toBytes("testHBase3758"); + util.createTable(tableName, new byte[][] {A, B, C}); + + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadDeleted", "wasScannerOpenCalled"}, + tableName, + new Boolean[] {false, false} + ); + + HTable table = new HTable(util.getConfiguration(), tableName); + Put put = new Put(ROW); + put.add(A, A, A); + table.put(put); + + Delete delete = new Delete(ROW); + table.delete(delete); + + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadDeleted", "wasScannerOpenCalled"}, + tableName, + new Boolean[] {true, false} + ); + + Scan s = new Scan(); + ResultScanner scanner = table.getScanner(s); + try { + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + } + } finally { + scanner.close(); + } + + // now scanner hooks should be invoked. + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"wasScannerOpenCalled"}, + tableName, + new Boolean[] {true} + ); + util.deleteTable(tableName); + table.close(); + } + + /* Overrides compaction to only output rows with keys that are even numbers */ + public static class EvenOnlyCompactor extends BaseRegionObserver { + long lastCompaction; + long lastFlush; + + @Override + public InternalScanner preCompact(ObserverContext e, + Store store, final InternalScanner scanner) { + return new InternalScanner() { + @Override + public boolean next(List results) throws IOException { + return next(results, -1); + } + + @Override + public boolean next(List results, String metric) + throws IOException { + return next(results, -1, metric); + } + + @Override + public boolean next(List results, int limit) + throws IOException{ + return next(results, limit, null); + } + + @Override + public boolean next(List results, int limit, String metric) + throws IOException { + List internalResults = new ArrayList(); + boolean hasMore; + do { + hasMore = scanner.next(internalResults, limit, metric); + if (!internalResults.isEmpty()) { + long row = Bytes.toLong(internalResults.get(0).getRow()); + if (row % 2 == 0) { + // return this row + break; + } + // clear and continue + internalResults.clear(); + } + } while (hasMore); + + if (!internalResults.isEmpty()) { + results.addAll(internalResults); + } + return hasMore; + } + + @Override + public void close() throws IOException { + scanner.close(); + } + }; + } + + @Override + public void postCompact(ObserverContext e, + Store store, StoreFile resultFile) { + lastCompaction = EnvironmentEdgeManager.currentTimeMillis(); + } + + @Override + public void postFlush(ObserverContext e) { + lastFlush = EnvironmentEdgeManager.currentTimeMillis(); + } + } + /** + * Tests overriding compaction handling via coprocessor hooks + * @throws Exception + */ + @Test + public void testCompactionOverride() throws Exception { + byte[] compactTable = Bytes.toBytes("TestCompactionOverride"); + HBaseAdmin admin = util.getHBaseAdmin(); + if (admin.tableExists(compactTable)) { + admin.disableTable(compactTable); + admin.deleteTable(compactTable); + } + + HTableDescriptor htd = new HTableDescriptor(compactTable); + htd.addFamily(new HColumnDescriptor(A)); + htd.addCoprocessor(EvenOnlyCompactor.class.getName()); + admin.createTable(htd); + + HTable table = new HTable(util.getConfiguration(), compactTable); + for (long i=1; i<=10; i++) { + byte[] iBytes = Bytes.toBytes(i); + Put put = new Put(iBytes); + put.setWriteToWAL(false); + put.add(A, A, iBytes); + table.put(put); + } + + HRegion firstRegion = cluster.getRegions(compactTable).get(0); + Coprocessor cp = firstRegion.getCoprocessorHost().findCoprocessor( + EvenOnlyCompactor.class.getName()); + assertNotNull("EvenOnlyCompactor coprocessor should be loaded", cp); + EvenOnlyCompactor compactor = (EvenOnlyCompactor)cp; + + // force a compaction + long ts = System.currentTimeMillis(); + admin.flush(compactTable); + // wait for flush + for (int i=0; i<10; i++) { + if (compactor.lastFlush >= ts) { + break; + } + Thread.sleep(1000); + } + assertTrue("Flush didn't complete", compactor.lastFlush >= ts); + LOG.debug("Flush complete"); + + ts = compactor.lastFlush; + admin.majorCompact(compactTable); + // wait for compaction + for (int i=0; i<30; i++) { + if (compactor.lastCompaction >= ts) { + break; + } + Thread.sleep(1000); + } + LOG.debug("Last compaction was at "+compactor.lastCompaction); + assertTrue("Compaction didn't complete", compactor.lastCompaction >= ts); + + // only even rows should remain + ResultScanner scanner = table.getScanner(new Scan()); + try { + for (long i=2; i<=10; i+=2) { + Result r = scanner.next(); + assertNotNull(r); + assertFalse(r.isEmpty()); + byte[] iBytes = Bytes.toBytes(i); + assertArrayEquals("Row should be "+i, r.getRow(), iBytes); + assertArrayEquals("Value should be "+i, r.getValue(A, A), iBytes); + } + } finally { + scanner.close(); + } + table.close(); + } + + @Test + public void bulkLoadHFileTest() throws Exception { + String testName = TestRegionObserverInterface.class.getName()+".bulkLoadHFileTest"; + byte[] tableName = TEST_TABLE; + Configuration conf = util.getConfiguration(); + HTable table = util.createTable(tableName, new byte[][] {A, B, C}); + + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"}, + tableName, + new Boolean[] {false, false} + ); + + FileSystem fs = util.getTestFileSystem(); + final Path dir = util.getDataTestDir(testName).makeQualified(fs); + Path familyDir = new Path(dir, Bytes.toString(A)); + + createHFile(util.getConfiguration(), fs, new Path(familyDir,Bytes.toString(A)), A, A); + + //Bulk load + new LoadIncrementalHFiles(conf).doBulkLoad(dir, new HTable(conf, tableName)); + + verifyMethodResult(SimpleRegionObserver.class, + new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"}, + tableName, + new Boolean[] {true, true} + ); + util.deleteTable(tableName); + table.close(); + } + + // check each region whether the coprocessor upcalls are called or not. + private void verifyMethodResult(Class c, String methodName[], byte[] tableName, + Object value[]) throws IOException { + try { + for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) { + for (HRegionInfo r : t.getRegionServer().getOnlineRegions()) { + if (!Arrays.equals(r.getTableName(), tableName)) { + continue; + } + RegionCoprocessorHost cph = t.getRegionServer().getOnlineRegion(r.getRegionName()). + getCoprocessorHost(); + + Coprocessor cp = cph.findCoprocessor(c.getName()); + assertNotNull(cp); + for (int i = 0; i < methodName.length; ++i) { + Method m = c.getMethod(methodName[i]); + Object o = m.invoke(cp); + assertTrue("Result of " + c.getName() + "." + methodName[i] + + " is expected to be " + value[i].toString() + + ", while we get " + o.toString(), o.equals(value[i])); + } + } + } + } catch (Exception e) { + throw new IOException(e.toString()); + } + } + + private static void createHFile( + Configuration conf, + FileSystem fs, Path path, + byte[] family, byte[] qualifier) throws IOException { + HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)) + .withPath(fs, path) + .withComparator(KeyValue.KEY_COMPARATOR) + .create(); + long now = System.currentTimeMillis(); + try { + for (int i =1;i<=9;i++) { + KeyValue kv = new KeyValue(Bytes.toBytes(i+""), family, qualifier, now, Bytes.toBytes(i+"")); + writer.append(kv); + } + } finally { + writer.close(); + } + } + + private static byte [][] makeN(byte [] base, int n) { + byte [][] ret = new byte[n][]; + for(int i=0;i c, + Store store, Scan scan, NavigableSet targetCols, KeyValueScanner s) + throws IOException { + scan.setFilter(new NoDataFilter()); + return new StoreScanner(store, store.getScanInfo(), scan, targetCols); + } + } + + /** + * Don't allow any data in a flush by creating a custom {@link StoreScanner}. + */ + public static class NoDataFromFlush extends BaseRegionObserver { + @Override + public InternalScanner preFlushScannerOpen(ObserverContext c, + Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException { + Scan scan = new Scan(); + scan.setFilter(new NoDataFilter()); + return new StoreScanner(store, store.getScanInfo(), scan, + Collections.singletonList(memstoreScanner), ScanType.MINOR_COMPACT, store.getHRegion() + .getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP); + } + } + + /** + * Don't allow any data to be written out in the compaction by creating a custom + * {@link StoreScanner}. + */ + public static class NoDataFromCompaction extends BaseRegionObserver { + @Override + public InternalScanner preCompactScannerOpen(ObserverContext c, + Store store, List scanners, ScanType scanType, + long earliestPutTs, InternalScanner s) throws IOException { + Scan scan = new Scan(); + scan.setFilter(new NoDataFilter()); + return new StoreScanner(store, store.getScanInfo(), scan, scanners, ScanType.MINOR_COMPACT, + store.getHRegion().getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP); + } + } + + HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf, + byte[]... families) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + for (byte[] family : families) { + htd.addFamily(new HColumnDescriptor(family)); + } + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + Path path = new Path(DIR + callingMethod); + HRegion r = HRegion.createHRegion(info, path, conf, htd); + // this following piece is a hack. currently a coprocessorHost + // is secretly loaded at OpenRegionHandler. we don't really + // start a region server here, so just manually create cphost + // and set it to region. + RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf); + r.setCoprocessorHost(host); + return r; + } + + @Test + public void testRegionObserverScanTimeStacking() throws Exception { + byte[] ROW = Bytes.toBytes("testRow"); + byte[] TABLE = Bytes.toBytes(getClass().getName()); + byte[] A = Bytes.toBytes("A"); + byte[][] FAMILIES = new byte[][] { A }; + + Configuration conf = HBaseConfiguration.create(); + HRegion region = initHRegion(TABLE, getClass().getName(), conf, FAMILIES); + RegionCoprocessorHost h = region.getCoprocessorHost(); + h.load(NoDataFromScan.class, Coprocessor.PRIORITY_HIGHEST, conf); + h.load(EmptyRegionObsever.class, Coprocessor.PRIORITY_USER, conf); + + Put put = new Put(ROW); + put.add(A, A, A); + region.put(put); + + Get get = new Get(ROW); + Result r = region.get(get); + assertNull( + "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: " + + r, r.list()); + } + + @Test + public void testRegionObserverFlushTimeStacking() throws Exception { + byte[] ROW = Bytes.toBytes("testRow"); + byte[] TABLE = Bytes.toBytes(getClass().getName()); + byte[] A = Bytes.toBytes("A"); + byte[][] FAMILIES = new byte[][] { A }; + + Configuration conf = HBaseConfiguration.create(); + HRegion region = initHRegion(TABLE, getClass().getName(), conf, FAMILIES); + RegionCoprocessorHost h = region.getCoprocessorHost(); + h.load(NoDataFromFlush.class, Coprocessor.PRIORITY_HIGHEST, conf); + h.load(EmptyRegionObsever.class, Coprocessor.PRIORITY_USER, conf); + + // put a row and flush it to disk + Put put = new Put(ROW); + put.add(A, A, A); + region.put(put); + region.flushcache(); + Get get = new Get(ROW); + Result r = region.get(get); + assertNull( + "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: " + + r, r.list()); + } + + /** + * Unfortunately, the easiest way to test this is to spin up a mini-cluster since we want to do + * the usual compaction mechanism on the region, rather than going through the backdoor to the + * region + */ + @Test + @Category(MediumTests.class) + public void testRegionObserverCompactionTimeStacking() throws Exception { + // setup a mini cluster so we can do a real compaction on a region + Configuration conf = UTIL.getConfiguration(); + conf.setInt("hbase.hstore.compaction.min", 2); + UTIL.startMiniCluster(); + String tableName = "testRegionObserverCompactionTimeStacking"; + byte[] ROW = Bytes.toBytes("testRow"); + byte[] A = Bytes.toBytes("A"); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(A)); + desc.addCoprocessor(EmptyRegionObsever.class.getName(), null, Coprocessor.PRIORITY_USER, null); + desc.addCoprocessor(NoDataFromCompaction.class.getName(), null, Coprocessor.PRIORITY_HIGHEST, + null); + + HBaseAdmin admin = UTIL.getHBaseAdmin(); + admin.createTable(desc); + + HTable table = new HTable(conf, desc.getName()); + + // put a row and flush it to disk + Put put = new Put(ROW); + put.add(A, A, A); + table.put(put); + table.flushCommits(); + + HRegionServer rs = UTIL.getRSForFirstRegionInTable(desc.getName()); + List regions = rs.getOnlineRegions(desc.getName()); + assertEquals("More than 1 region serving test table with 1 row", 1, regions.size()); + HRegion region = regions.get(0); + admin.flush(region.getRegionName()); + + // put another row and flush that too + put = new Put(Bytes.toBytes("anotherrow")); + put.add(A, A, A); + table.put(put); + table.flushCommits(); + admin.flush(region.getRegionName()); + + // run a compaction, which normally would should get rid of the data + Store s = region.getStores().get(A); + CountDownLatch latch = new CountDownLatch(1); + WaitableCompactionRequest request = new WaitableCompactionRequest(region, s, latch); + rs.compactSplitThread.requestCompaction(region, s, + "compact for testRegionObserverCompactionTimeStacking", Store.PRIORITY_USER, request); + // wait for the compaction to complete + latch.await(); + + // check both rows to ensure that they aren't there + Get get = new Get(ROW); + Result r = table.get(get); + assertNull( + "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: " + + r, r.list()); + + get = new Get(Bytes.toBytes("anotherrow")); + r = table.get(get); + assertNull( + "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor Found: " + + r, r.list()); + + table.close(); + UTIL.shutdownMiniCluster(); + } + + /** + * A simple compaction on which you can wait for the passed in latch until the compaction finishes + * (either successfully or if it failed). + */ + public static class WaitableCompactionRequest extends CompactionRequest { + private CountDownLatch done; + + /** + * Constructor for a custom compaction. Uses the setXXX methods to update the state of the + * compaction before being used. + */ + public WaitableCompactionRequest(HRegion region, Store store, CountDownLatch finished) { + super(region, store, Store.PRIORITY_USER); + this.done = finished; + } + + @Override + public void finishRequest() { + super.finishRequest(); + this.done.countDown(); + } + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java new file mode 100644 index 0000000..002d611 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java @@ -0,0 +1,142 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestRegionObserverStacking extends TestCase { + static final String DIR = "test/build/data/TestRegionObserverStacking/"; + + public static class ObserverA extends BaseRegionObserver { + long id; + @Override + public void postPut(final ObserverContext c, + final Put put, final WALEdit edit, + final boolean writeToWAL) + throws IOException { + id = System.currentTimeMillis(); + try { + Thread.sleep(10); + } catch (InterruptedException ex) { + } + } + } + + public static class ObserverB extends BaseRegionObserver { + long id; + @Override + public void postPut(final ObserverContext c, + final Put put, final WALEdit edit, + final boolean writeToWAL) + throws IOException { + id = System.currentTimeMillis(); + try { + Thread.sleep(10); + } catch (InterruptedException ex) { + } + } + } + + public static class ObserverC extends BaseRegionObserver { + long id; + + @Override + public void postPut(final ObserverContext c, + final Put put, final WALEdit edit, + final boolean writeToWAL) + throws IOException { + id = System.currentTimeMillis(); + try { + Thread.sleep(10); + } catch (InterruptedException ex) { + } + } + } + + HRegion initHRegion (byte [] tableName, String callingMethod, + Configuration conf, byte [] ... families) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + for(byte [] family : families) { + htd.addFamily(new HColumnDescriptor(family)); + } + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + Path path = new Path(DIR + callingMethod); + HRegion r = HRegion.createHRegion(info, path, conf, htd); + // this following piece is a hack. currently a coprocessorHost + // is secretly loaded at OpenRegionHandler. we don't really + // start a region server here, so just manually create cphost + // and set it to region. + RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf); + r.setCoprocessorHost(host); + return r; + } + + public void testRegionObserverStacking() throws Exception { + byte[] ROW = Bytes.toBytes("testRow"); + byte[] TABLE = Bytes.toBytes(getClass().getName()); + byte[] A = Bytes.toBytes("A"); + byte[][] FAMILIES = new byte[][] { A } ; + + Configuration conf = HBaseConfiguration.create(); + HRegion region = initHRegion(TABLE, getClass().getName(), + conf, FAMILIES); + RegionCoprocessorHost h = region.getCoprocessorHost(); + h.load(ObserverA.class, Coprocessor.PRIORITY_HIGHEST, conf); + h.load(ObserverB.class, Coprocessor.PRIORITY_USER, conf); + h.load(ObserverC.class, Coprocessor.PRIORITY_LOWEST, conf); + + Put put = new Put(ROW); + put.add(A, A, A); + int lockid = region.obtainRowLock(ROW); + region.put(put, lockid); + region.releaseRowLock(lockid); + + Coprocessor c = h.findCoprocessor(ObserverA.class.getName()); + long idA = ((ObserverA)c).id; + c = h.findCoprocessor(ObserverB.class.getName()); + long idB = ((ObserverB)c).id; + c = h.findCoprocessor(ObserverC.class.getName()); + long idC = ((ObserverC)c).id; + + assertTrue(idA < idB); + assertTrue(idB < idC); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java new file mode 100644 index 0000000..ae5b9fa --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java @@ -0,0 +1,178 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +/** + * Tests unhandled exceptions thrown by coprocessors running on a regionserver.. + * Expected result is that the regionserver will abort with an informative + * error message describing the set of its loaded coprocessors for crash + * diagnosis. (HBASE-4014). + */ +@Category(MediumTests.class) +public class TestRegionServerCoprocessorExceptionWithAbort { + static final Log LOG = LogFactory.getLog(TestRegionObserverInterface.class); + + private class zkwAbortable implements Abortable { + @Override + public void abort(String why, Throwable e) { + throw new RuntimeException("Fatal ZK rs tracker error, why=", e); + } + @Override + public boolean isAborted() { + return false; + } + }; + + private class RSTracker extends ZooKeeperNodeTracker { + public boolean regionZKNodeWasDeleted = false; + public String rsNode; + private Thread mainThread; + + public RSTracker(ZooKeeperWatcher zkw, String rsNode, Thread mainThread) { + super(zkw, rsNode, new zkwAbortable()); + this.rsNode = rsNode; + this.mainThread = mainThread; + } + + @Override + public synchronized void nodeDeleted(String path) { + if (path.equals(rsNode)) { + regionZKNodeWasDeleted = true; + mainThread.interrupt(); + } + } + } + private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + static final int timeout = 30000; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // set configure to indicate which cp should be loaded + Configuration conf = TEST_UTIL.getConfiguration(); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + BuggyRegionObserver.class.getName()); + conf.set("hbase.coprocessor.abortonerror", "true"); + TEST_UTIL.startMiniCluster(2); + } + + @AfterClass + public static void teardownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testExceptionFromCoprocessorDuringPut() + throws IOException { + // When we try to write to TEST_TABLE, the buggy coprocessor will + // cause a NullPointerException, which will cause the regionserver (which + // hosts the region we attempted to write to) to abort. + byte[] TEST_TABLE = Bytes.toBytes("observed_table"); + byte[] TEST_FAMILY = Bytes.toBytes("aaa"); + + HTable table = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY); + TEST_UTIL.createMultiRegions(table, TEST_FAMILY); + TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE); + + // Note which regionServer will abort (after put is attempted). + final HRegionServer regionServer = + TEST_UTIL.getRSForFirstRegionInTable(TEST_TABLE); + + // add watch so we can know when this regionserver aborted. + ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + "unittest", new zkwAbortable()); + + RSTracker rsTracker = new RSTracker(zkw, + "/hbase/rs/"+regionServer.getServerName(), Thread.currentThread()); + rsTracker.start(); + zkw.registerListener(rsTracker); + + boolean caughtInterruption = false; + try { + final byte[] ROW = Bytes.toBytes("aaa"); + Put put = new Put(ROW); + put.add(TEST_FAMILY, ROW, ROW); + table.put(put); + } catch (IOException e) { + // Depending on exact timing of the threads involved, zkw's interruption + // might be caught here ... + if (e.getCause().getClass().equals(InterruptedException.class)) { + LOG.debug("caught interruption here (during put())."); + caughtInterruption = true; + } else { + fail("put() failed: " + e); + } + } + if (caughtInterruption == false) { + try { + Thread.sleep(timeout); + fail("RegionServer did not abort within 30 seconds."); + } catch (InterruptedException e) { + // .. or it might be caught here. + LOG.debug("caught interruption here (during sleep())."); + caughtInterruption = true; + } + } + assertTrue("Main thread caught interruption.",caughtInterruption); + assertTrue("RegionServer aborted on coprocessor exception, as expected.", + rsTracker.regionZKNodeWasDeleted); + table.close(); + } + + public static class BuggyRegionObserver extends SimpleRegionObserver { + @Override + public void prePut(final ObserverContext c, + final Put put, final WALEdit edit, + final boolean writeToWAL) { + String tableName = + c.getEnvironment().getRegion().getRegionInfo().getTableNameAsString(); + if (tableName.equals("observed_table")) { + Integer i = null; + i = i + 1; + } + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java new file mode 100644 index 0000000..d191665 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java @@ -0,0 +1,144 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +/** + * Tests unhandled exceptions thrown by coprocessors running on regionserver. + * Expected result is that the master will remove the buggy coprocessor from + * its set of coprocessors and throw a org.apache.hadoop.hbase.DoNotRetryIOException + * back to the client. + * (HBASE-4014). + */ +@Category(MediumTests.class) +public class TestRegionServerCoprocessorExceptionWithRemove { + public static class BuggyRegionObserver extends SimpleRegionObserver { + @SuppressWarnings("null") + @Override + public void prePut(final ObserverContext c, + final Put put, final WALEdit edit, + final boolean writeToWAL) { + String tableName = + c.getEnvironment().getRegion().getRegionInfo().getTableNameAsString(); + if (tableName.equals("observed_table")) { + Integer i = null; + i = i + 1; + } + } + } + + private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // set configure to indicate which cp should be loaded + Configuration conf = TEST_UTIL.getConfiguration(); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + BuggyRegionObserver.class.getName()); + TEST_UTIL.startMiniCluster(2); + } + + @AfterClass + public static void teardownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test(timeout=30000) + public void testExceptionFromCoprocessorDuringPut() + throws IOException { + // Set watches on the zookeeper nodes for all of the regionservers in the + // cluster. When we try to write to TEST_TABLE, the buggy coprocessor will + // cause a NullPointerException, which will cause the regionserver (which + // hosts the region we attempted to write to) to abort. In turn, this will + // cause the nodeDeleted() method of the DeadRegionServer tracker to + // execute, which will set the rsZKNodeDeleted flag to true, which will + // pass this test. + + byte[] TEST_TABLE = Bytes.toBytes("observed_table"); + byte[] TEST_FAMILY = Bytes.toBytes("aaa"); + + HTable table = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY); + TEST_UTIL.createMultiRegions(table, TEST_FAMILY); + TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE); + // Note which regionServer that should survive the buggy coprocessor's + // prePut(). + HRegionServer regionServer = + TEST_UTIL.getRSForFirstRegionInTable(TEST_TABLE); + + // same logic as {@link TestMasterCoprocessorExceptionWithRemove}, + // but exception will be RetriesExhaustedWithDetailException rather + // than DoNotRetryIOException. The latter exception is what the RegionServer + // will have actually thrown, but the client will wrap this in a + // RetriesExhaustedWithDetailException. + // We will verify that "DoNotRetryIOException" appears in the text of the + // the exception's detailMessage. + boolean threwDNRE = false; + try { + final byte[] ROW = Bytes.toBytes("aaa"); + Put put = new Put(ROW); + put.add(TEST_FAMILY, ROW, ROW); + table.put(put); + } catch (RetriesExhaustedWithDetailsException e) { + // below, could call instead : + // startsWith("Failed 1 action: DoNotRetryIOException.") + // But that might be too brittle if client-side + // DoNotRetryIOException-handler changes its message. + assertTrue(e.getMessage().contains("DoNotRetryIOException")); + threwDNRE = true; + } finally { + assertTrue(threwDNRE); + } + + // Wait 3 seconds for the regionserver to abort: expected result is that + // it will survive and not abort. + for (int i = 0; i < 3; i++) { + assertFalse(regionServer.isAborted()); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + fail("InterruptedException while waiting for regionserver " + + "zk node to be deleted."); + } + } + table.close(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java new file mode 100644 index 0000000..36dd289 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java @@ -0,0 +1,414 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter; +import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +/** + * Tests invocation of the {@link org.apache.hadoop.hbase.coprocessor.MasterObserver} + * interface hooks at all appropriate times during normal HMaster operations. + */ +@Category(MediumTests.class) +public class TestWALObserver { + private static final Log LOG = LogFactory.getLog(TestWALObserver.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private static byte[] TEST_TABLE = Bytes.toBytes("observedTable"); + private static byte[][] TEST_FAMILY = { Bytes.toBytes("fam1"), + Bytes.toBytes("fam2"), + Bytes.toBytes("fam3"), + }; + private static byte[][] TEST_QUALIFIER = { Bytes.toBytes("q1"), + Bytes.toBytes("q2"), + Bytes.toBytes("q3"), + }; + private static byte[][] TEST_VALUE = { Bytes.toBytes("v1"), + Bytes.toBytes("v2"), + Bytes.toBytes("v3"), + }; + private static byte[] TEST_ROW = Bytes.toBytes("testRow"); + + private Configuration conf; + private FileSystem fs; + private Path dir; + private MiniDFSCluster cluster; + private Path hbaseRootDir; + private Path oldLogDir; + private Path logDir; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, + SampleRegionWALObserver.class.getName()); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + SampleRegionWALObserver.class.getName()); + conf.setBoolean("dfs.support.append", true); + conf.setInt("dfs.client.block.recovery.retries", 2); + + TEST_UTIL.startMiniCluster(1); + Path hbaseRootDir = + TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase")); + LOG.info("hbase.rootdir=" + hbaseRootDir); + conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString()); + } + + @AfterClass + public static void teardownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); + //this.cluster = TEST_UTIL.getDFSCluster(); + this.fs = TEST_UTIL.getDFSCluster().getFileSystem(); + this.hbaseRootDir = new Path(conf.get(HConstants.HBASE_DIR)); + this.dir = new Path(this.hbaseRootDir, TestWALObserver.class.getName()); + this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME); + this.logDir = new Path(this.hbaseRootDir, HConstants.HREGION_LOGDIR_NAME); + + if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) { + TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true); + } + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true); + } + + /** + * Test WAL write behavior with WALObserver. The coprocessor monitors + * a WALEdit written to WAL, and ignore, modify, and add KeyValue's for the + * WALEdit. + */ + @Test + public void testWALObserverWriteToWAL() throws Exception { + + HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE)); + final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE)); + HRegion region2 = HRegion.createHRegion(hri, + hbaseRootDir, this.conf, htd); + + Path basedir = new Path(this.hbaseRootDir, Bytes.toString(TEST_TABLE)); + deleteDir(basedir); + fs.mkdirs(new Path(basedir, hri.getEncodedName())); + + HLog log = new HLog(this.fs, this.dir, this.oldLogDir, this.conf); + SampleRegionWALObserver cp = getCoprocessor(log); + + // TEST_FAMILY[0] shall be removed from WALEdit. + // TEST_FAMILY[1] value shall be changed. + // TEST_FAMILY[2] shall be added to WALEdit, although it's not in the put. + cp.setTestValues(TEST_TABLE, TEST_ROW, TEST_FAMILY[0], TEST_QUALIFIER[0], + TEST_FAMILY[1], TEST_QUALIFIER[1], + TEST_FAMILY[2], TEST_QUALIFIER[2]); + + assertFalse(cp.isPreWALWriteCalled()); + assertFalse(cp.isPostWALWriteCalled()); + + // TEST_FAMILY[2] is not in the put, however it shall be added by the tested + // coprocessor. + // Use a Put to create familyMap. + Put p = creatPutWith2Families(TEST_ROW); + + Map> familyMap = p.getFamilyMap(); + WALEdit edit = new WALEdit(); + addFamilyMapToWALEdit(familyMap, edit); + + boolean foundFamily0 = false; + boolean foundFamily2 = false; + boolean modifiedFamily1 = false; + + List kvs = edit.getKeyValues(); + + for (KeyValue kv : kvs) { + if (Arrays.equals(kv.getFamily(), TEST_FAMILY[0])) { + foundFamily0 = true; + } + if (Arrays.equals(kv.getFamily(), TEST_FAMILY[2])) { + foundFamily2 = true; + } + if (Arrays.equals(kv.getFamily(), TEST_FAMILY[1])) { + if (!Arrays.equals(kv.getValue(), TEST_VALUE[1])) { + modifiedFamily1 = true; + } + } + } + assertTrue(foundFamily0); + assertFalse(foundFamily2); + assertFalse(modifiedFamily1); + + // it's where WAL write cp should occur. + long now = EnvironmentEdgeManager.currentTimeMillis(); + log.append(hri, hri.getTableName(), edit, now, htd); + + // the edit shall have been change now by the coprocessor. + foundFamily0 = false; + foundFamily2 = false; + modifiedFamily1 = false; + for (KeyValue kv : kvs) { + if (Arrays.equals(kv.getFamily(), TEST_FAMILY[0])) { + foundFamily0 = true; + } + if (Arrays.equals(kv.getFamily(), TEST_FAMILY[2])) { + foundFamily2 = true; + } + if (Arrays.equals(kv.getFamily(), TEST_FAMILY[1])) { + if (!Arrays.equals(kv.getValue(), TEST_VALUE[1])) { + modifiedFamily1 = true; + } + } + } + assertFalse(foundFamily0); + assertTrue(foundFamily2); + assertTrue(modifiedFamily1); + + assertTrue(cp.isPreWALWriteCalled()); + assertTrue(cp.isPostWALWriteCalled()); + } + + /** + * Test WAL replay behavior with WALObserver. + */ + @Test + public void testWALCoprocessorReplay() throws Exception { + // WAL replay is handled at HRegion::replayRecoveredEdits(), which is + // ultimately called by HRegion::initialize() + byte[] tableName = Bytes.toBytes("testWALCoprocessorReplay"); + final HTableDescriptor htd = getBasic3FamilyHTableDescriptor(Bytes.toString(tableName)); + //final HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(tableName)); + //final HRegionInfo hri1 = createBasic3FamilyHRegionInfo(Bytes.toString(tableName)); + final HRegionInfo hri = new HRegionInfo(tableName, null, null); + + final Path basedir = new Path(this.hbaseRootDir, Bytes.toString(tableName)); + deleteDir(basedir); + fs.mkdirs(new Path(basedir, hri.getEncodedName())); + + final Configuration newConf = HBaseConfiguration.create(this.conf); + + HRegion region2 = HRegion.createHRegion(hri, + hbaseRootDir, newConf,htd); + + + //HLog wal = new HLog(this.fs, this.dir, this.oldLogDir, this.conf); + HLog wal = createWAL(this.conf); + //Put p = creatPutWith2Families(TEST_ROW); + WALEdit edit = new WALEdit(); + long now = EnvironmentEdgeManager.currentTimeMillis(); + //addFamilyMapToWALEdit(p.getFamilyMap(), edit); + final int countPerFamily = 1000; + //for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) { + for (HColumnDescriptor hcd: htd.getFamilies()) { + //addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily, + //EnvironmentEdgeManager.getDelegate(), wal); + addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily, + EnvironmentEdgeManager.getDelegate(), wal, htd); + } + wal.append(hri, tableName, edit, now, htd); + // sync to fs. + wal.sync(); + + User user = HBaseTestingUtility.getDifferentUser(newConf, + ".replay.wal.secondtime"); + user.runAs(new PrivilegedExceptionAction() { + public Object run() throws Exception { + Path p = runWALSplit(newConf); + LOG.info("WALSplit path == " + p); + FileSystem newFS = FileSystem.get(newConf); + // Make a new wal for new region open. + HLog wal2 = createWAL(newConf); + Path tableDir = + HTableDescriptor.getTableDir(hbaseRootDir, hri.getTableName()); + HRegion region = new HRegion(tableDir, wal2, FileSystem.get(newConf), + newConf, hri, htd, TEST_UTIL.getHBaseCluster().getRegionServer(0)); + + long seqid2 = region.initialize(); + SampleRegionWALObserver cp2 = + (SampleRegionWALObserver)region.getCoprocessorHost().findCoprocessor( + SampleRegionWALObserver.class.getName()); + // TODO: asserting here is problematic. + assertNotNull(cp2); + assertTrue(cp2.isPreWALRestoreCalled()); + assertTrue(cp2.isPostWALRestoreCalled()); + region.close(); + wal2.closeAndDelete(); + return null; + } + }); + } + + /** + * Test to see CP loaded successfully or not. There is a duplication + * at TestHLog, but the purpose of that one is to see whether the loaded + * CP will impact existing HLog tests or not. + */ + @Test + public void testWALObserverLoaded() throws Exception { + HLog log = new HLog(fs, dir, oldLogDir, conf); + assertNotNull(getCoprocessor(log)); + } + + private SampleRegionWALObserver getCoprocessor(HLog wal) throws Exception { + WALCoprocessorHost host = wal.getCoprocessorHost(); + Coprocessor c = host.findCoprocessor(SampleRegionWALObserver.class.getName()); + return (SampleRegionWALObserver)c; + } + + /* + * Creates an HRI around an HTD that has tableName and three + * column families named. + * @param tableName Name of table to use when we create HTableDescriptor. + */ + private HRegionInfo createBasic3FamilyHRegionInfo(final String tableName) { + HTableDescriptor htd = new HTableDescriptor(tableName); + + for (int i = 0; i < TEST_FAMILY.length; i++ ) { + HColumnDescriptor a = new HColumnDescriptor(TEST_FAMILY[i]); + htd.addFamily(a); + } + return new HRegionInfo(htd.getName(), null, null, false); + } + + /* + * @param p Directory to cleanup + */ + private void deleteDir(final Path p) throws IOException { + if (this.fs.exists(p)) { + if (!this.fs.delete(p, true)) { + throw new IOException("Failed remove of " + p); + } + } + } + + private Put creatPutWith2Families(byte[] row) throws IOException { + Put p = new Put(row); + for (int i = 0; i < TEST_FAMILY.length-1; i++ ) { + p.add(TEST_FAMILY[i], TEST_QUALIFIER[i], + TEST_VALUE[i]); + } + return p; + } + + /** + * Copied from HRegion. + * + * @param familyMap map of family->edits + * @param walEdit the destination entry to append into + */ + private void addFamilyMapToWALEdit(Map> familyMap, + WALEdit walEdit) { + for (List edits : familyMap.values()) { + for (KeyValue kv : edits) { + walEdit.add(kv); + } + } + } + private Path runWALSplit(final Configuration c) throws IOException { + FileSystem fs = FileSystem.get(c); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(c, + this.hbaseRootDir, this.logDir, this.oldLogDir, fs); + List splits = logSplitter.splitLog(); + // Split should generate only 1 file since there's only 1 region + assertEquals(1, splits.size()); + // Make sure the file exists + assertTrue(fs.exists(splits.get(0))); + LOG.info("Split file=" + splits.get(0)); + return splits.get(0); + } + private HLog createWAL(final Configuration c) throws IOException { + HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c); + return wal; + } + private void addWALEdits (final byte [] tableName, final HRegionInfo hri, + final byte [] rowName, final byte [] family, + final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd) + throws IOException { + String familyStr = Bytes.toString(family); + for (int j = 0; j < count; j++) { + byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); + byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); + WALEdit edit = new WALEdit(); + edit.add(new KeyValue(rowName, family, qualifierBytes, + ee.currentTimeMillis(), columnBytes)); + wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd); + } + } + private HTableDescriptor getBasic3FamilyHTableDescriptor( + final String tableName) { + HTableDescriptor htd = new HTableDescriptor(tableName); + + for (int i = 0; i < TEST_FAMILY.length; i++ ) { + HColumnDescriptor a = new HColumnDescriptor(TEST_FAMILY[i]); + htd.addFamily(a); + } + return htd; + } + + private HTableDescriptor createBasic3FamilyHTD(final String tableName) { + HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a")); + htd.addFamily(a); + HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b")); + htd.addFamily(b); + HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c")); + htd.addFamily(c); + return htd; + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + + diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestBulkDeleteProtocol.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestBulkDeleteProtocol.java new file mode 100644 index 0000000..8b4d513 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestBulkDeleteProtocol.java @@ -0,0 +1,407 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor.example; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.example.BulkDeleteProtocol.DeleteType; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestBulkDeleteProtocol { + private static final byte[] FAMILY1 = Bytes.toBytes("cf1"); + private static final byte[] FAMILY2 = Bytes.toBytes("cf2"); + private static final byte[] QUALIFIER1 = Bytes.toBytes("c1"); + private static final byte[] QUALIFIER2 = Bytes.toBytes("c2"); + private static final byte[] QUALIFIER3 = Bytes.toBytes("c3"); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, + BulkDeleteEndpoint.class.getName()); + TEST_UTIL.startMiniCluster(2); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testBulkDeleteEndpoint() throws Throwable { + byte[] tableName = Bytes.toBytes("testBulkDeleteEndpoint"); + HTable ht = createTable(tableName); + List puts = new ArrayList(100); + for (int j = 0; j < 100; j++) { + byte[] rowkey = Bytes.toBytes(j); + puts.add(createPut(rowkey, "v1")); + } + ht.put(puts); + // Deleting all the rows. + long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, new Scan(), 500, DeleteType.ROW, + null); + assertEquals(100, noOfRowsDeleted); + + int rows = 0; + for (Result result : ht.getScanner(new Scan())) { + rows++; + } + assertEquals(0, rows); + } + + @Test + public void testBulkDeleteEndpointWhenRowBatchSizeLessThanRowsToDeleteFromARegion() + throws Throwable { + byte[] tableName = Bytes + .toBytes("testBulkDeleteEndpointWhenRowBatchSizeLessThanRowsToDeleteFromARegion"); + HTable ht = createTable(tableName); + List puts = new ArrayList(100); + for (int j = 0; j < 100; j++) { + byte[] rowkey = Bytes.toBytes(j); + puts.add(createPut(rowkey, "v1")); + } + ht.put(puts); + // Deleting all the rows. + long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, new Scan(), 10, DeleteType.ROW, null); + assertEquals(100, noOfRowsDeleted); + + int rows = 0; + for (Result result : ht.getScanner(new Scan())) { + rows++; + } + assertEquals(0, rows); + } + + private long invokeBulkDeleteProtocol(byte[] tableName, final Scan scan, final int rowBatchSize, + final byte deleteType, final Long timeStamp) throws Throwable { + HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); + long noOfDeletedRows = 0L; + Batch.Call callable = + new Batch.Call() { + public BulkDeleteResponse call(BulkDeleteProtocol instance) throws IOException { + return instance.delete(scan, deleteType, timeStamp, rowBatchSize); + } + }; + Map result = ht.coprocessorExec(BulkDeleteProtocol.class, + scan.getStartRow(), scan.getStopRow(), callable); + for (BulkDeleteResponse response : result.values()) { + noOfDeletedRows += response.getRowsDeleted(); + } + return noOfDeletedRows; + } + + @Test + public void testBulkDeleteWithConditionBasedDelete() throws Throwable { + byte[] tableName = Bytes.toBytes("testBulkDeleteWithConditionBasedDelete"); + HTable ht = createTable(tableName); + List puts = new ArrayList(100); + for (int j = 0; j < 100; j++) { + byte[] rowkey = Bytes.toBytes(j); + String value = (j % 10 == 0) ? "v1" : "v2"; + puts.add(createPut(rowkey, value)); + } + ht.put(puts); + Scan scan = new Scan(); + FilterList fl = new FilterList(Operator.MUST_PASS_ALL); + SingleColumnValueFilter scvf = new SingleColumnValueFilter(FAMILY1, QUALIFIER3, + CompareOp.EQUAL, Bytes.toBytes("v1")); + //fl.addFilter(new FirstKeyOnlyFilter()); + fl.addFilter(scvf); + scan.setFilter(fl); + // Deleting all the rows where cf1:c1=v1 + long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.ROW, null); + assertEquals(10, noOfRowsDeleted); + + int rows = 0; + for (Result result : ht.getScanner(new Scan())) { + rows++; + } + assertEquals(90, rows); + } + + @Test + public void testBulkDeleteColumn() throws Throwable { + byte[] tableName = Bytes.toBytes("testBulkDeleteColumn"); + HTable ht = createTable(tableName); + List puts = new ArrayList(100); + for (int j = 0; j < 100; j++) { + byte[] rowkey = Bytes.toBytes(j); + String value = (j % 10 == 0) ? "v1" : "v2"; + puts.add(createPut(rowkey, value)); + } + ht.put(puts); + Scan scan = new Scan (); + scan.addColumn(FAMILY1, QUALIFIER2); + // Delete the column cf1:col2 + long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.COLUMN, null); + assertEquals(100, noOfRowsDeleted); + + int rows = 0; + for (Result result : ht.getScanner(new Scan())) { + assertEquals(2, result.getFamilyMap(FAMILY1).size()); + assertTrue(result.getColumn(FAMILY1, QUALIFIER2).isEmpty()); + assertEquals(1, result.getColumn(FAMILY1, QUALIFIER1).size()); + assertEquals(1, result.getColumn(FAMILY1, QUALIFIER3).size()); + rows++; + } + assertEquals(100, rows); + } + + @Test + public void testBulkDeleteFamily() throws Throwable { + byte[] tableName = Bytes.toBytes("testBulkDeleteFamily"); + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(FAMILY1)); + htd.addFamily(new HColumnDescriptor(FAMILY2)); + TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5); + HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); + List puts = new ArrayList(100); + for (int j = 0; j < 100; j++) { + Put put = new Put(Bytes.toBytes(j)); + put.add(FAMILY1, QUALIFIER1, "v1".getBytes()); + put.add(FAMILY2, QUALIFIER2, "v2".getBytes()); + puts.add(put); + } + ht.put(puts); + Scan scan = new Scan (); + scan.addFamily(FAMILY1); + // Delete the column family cf1 + long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.FAMILY, null); + assertEquals(100, noOfRowsDeleted); + int rows = 0; + for (Result result : ht.getScanner(new Scan())) { + assertTrue(result.getFamilyMap(FAMILY1).isEmpty()); + assertEquals(1, result.getColumn(FAMILY2, QUALIFIER2).size()); + rows++; + } + assertEquals(100, rows); + } + + @Test + public void testBulkDeleteColumnVersion() throws Throwable { + byte[] tableName = Bytes.toBytes("testBulkDeleteColumnVersion"); + HTable ht = createTable(tableName); + List puts = new ArrayList(100); + for (int j = 0; j < 100; j++) { + Put put = new Put(Bytes.toBytes(j)); + byte[] value = "v1".getBytes(); + put.add(FAMILY1, QUALIFIER1, 1234L, value); + put.add(FAMILY1, QUALIFIER2, 1234L, value); + put.add(FAMILY1, QUALIFIER3, 1234L, value); + // Latest version values + value = "v2".getBytes(); + put.add(FAMILY1, QUALIFIER1, value); + put.add(FAMILY1, QUALIFIER2, value); + put.add(FAMILY1, QUALIFIER3, value); + put.add(FAMILY1, null, value); + puts.add(put); + } + ht.put(puts); + Scan scan = new Scan (); + scan.addFamily(FAMILY1); + // Delete the latest version values of all the columns in family cf1. + long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.VERSION, + HConstants.LATEST_TIMESTAMP); + assertEquals(100, noOfRowsDeleted); + int rows = 0; + scan = new Scan (); + scan.setMaxVersions(); + for (Result result : ht.getScanner(scan)) { + assertEquals(3, result.getFamilyMap(FAMILY1).size()); + List column = result.getColumn(FAMILY1, QUALIFIER1); + assertEquals(1, column.size()); + assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue())); + + column = result.getColumn(FAMILY1, QUALIFIER2); + assertEquals(1, column.size()); + assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue())); + + column = result.getColumn(FAMILY1, QUALIFIER3); + assertEquals(1, column.size()); + assertTrue(Bytes.equals("v1".getBytes(), column.get(0).getValue())); + rows++; + } + assertEquals(100, rows); + } + + @Test + public void testBulkDeleteColumnVersionBasedOnTS() throws Throwable { + byte[] tableName = Bytes.toBytes("testBulkDeleteColumnVersionBasedOnTS"); + HTable ht = createTable(tableName); + List puts = new ArrayList(100); + for (int j = 0; j < 100; j++) { + Put put = new Put(Bytes.toBytes(j)); + // TS = 1000L + byte[] value = "v1".getBytes(); + put.add(FAMILY1, QUALIFIER1, 1000L, value); + put.add(FAMILY1, QUALIFIER2, 1000L, value); + put.add(FAMILY1, QUALIFIER3, 1000L, value); + // TS = 1234L + value = "v2".getBytes(); + put.add(FAMILY1, QUALIFIER1, 1234L, value); + put.add(FAMILY1, QUALIFIER2, 1234L, value); + put.add(FAMILY1, QUALIFIER3, 1234L, value); + // Latest version values + value = "v3".getBytes(); + put.add(FAMILY1, QUALIFIER1, value); + put.add(FAMILY1, QUALIFIER2, value); + put.add(FAMILY1, QUALIFIER3, value); + puts.add(put); + } + ht.put(puts); + Scan scan = new Scan (); + scan.addColumn(FAMILY1, QUALIFIER3); + // Delete the column cf1:c3's one version at TS=1234 + long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.VERSION, 1234L); + assertEquals(100, noOfRowsDeleted); + int rows = 0; + scan = new Scan (); + scan.setMaxVersions(); + for (Result result : ht.getScanner(scan)) { + assertEquals(3, result.getFamilyMap(FAMILY1).size()); + assertEquals(3, result.getColumn(FAMILY1, QUALIFIER1).size()); + assertEquals(3, result.getColumn(FAMILY1, QUALIFIER2).size()); + List column = result.getColumn(FAMILY1, QUALIFIER3); + assertEquals(2, column.size()); + assertTrue(Bytes.equals("v3".getBytes(), column.get(0).getValue())); + assertTrue(Bytes.equals("v1".getBytes(), column.get(1).getValue())); + rows++; + } + assertEquals(100, rows); + } + + @Test + public void testBulkDeleteWithNumberOfVersions() throws Throwable { + byte[] tableName = Bytes.toBytes("testBulkDeleteWithNumberOfVersions"); + HTable ht = createTable(tableName); + List puts = new ArrayList(100); + for (int j = 0; j < 100; j++) { + Put put = new Put(Bytes.toBytes(j)); + // TS = 1000L + byte[] value = "v1".getBytes(); + put.add(FAMILY1, QUALIFIER1, 1000L, value); + put.add(FAMILY1, QUALIFIER2, 1000L, value); + put.add(FAMILY1, QUALIFIER3, 1000L, value); + // TS = 1234L + value = "v2".getBytes(); + put.add(FAMILY1, QUALIFIER1, 1234L, value); + put.add(FAMILY1, QUALIFIER2, 1234L, value); + put.add(FAMILY1, QUALIFIER3, 1234L, value); + // TS = 2000L + value = "v3".getBytes(); + put.add(FAMILY1, QUALIFIER1, 2000L, value); + put.add(FAMILY1, QUALIFIER2, 2000L, value); + put.add(FAMILY1, QUALIFIER3, 2000L, value); + // Latest version values + value = "v4".getBytes(); + put.add(FAMILY1, QUALIFIER1, value); + put.add(FAMILY1, QUALIFIER2, value); + put.add(FAMILY1, QUALIFIER3, value); + puts.add(put); + } + ht.put(puts); + + // Delete all the versions of columns cf1:c1 and cf1:c2 falling with the time range + // [1000,2000) + final Scan scan = new Scan(); + scan.addColumn(FAMILY1, QUALIFIER1); + scan.addColumn(FAMILY1, QUALIFIER2); + scan.setTimeRange(1000L, 2000L); + scan.setMaxVersions(); + + long noOfDeletedRows = 0L; + long noOfVersionsDeleted = 0L; + Batch.Call callable = + new Batch.Call() { + public BulkDeleteResponse call(BulkDeleteProtocol instance) throws IOException { + return instance.delete(scan, DeleteType.VERSION, null, 500); + } + }; + Map result = ht.coprocessorExec(BulkDeleteProtocol.class, + scan.getStartRow(), scan.getStopRow(), callable); + for (BulkDeleteResponse response : result.values()) { + noOfDeletedRows += response.getRowsDeleted(); + noOfVersionsDeleted += response.getVersionsDeleted(); + } + assertEquals(100, noOfDeletedRows); + assertEquals(400, noOfVersionsDeleted); + + int rows = 0; + Scan scan1 = new Scan (); + scan1.setMaxVersions(); + for (Result res : ht.getScanner(scan1)) { + assertEquals(3, res.getFamilyMap(FAMILY1).size()); + List column = res.getColumn(FAMILY1, QUALIFIER1); + assertEquals(2, column.size()); + assertTrue(Bytes.equals("v4".getBytes(), column.get(0).getValue())); + assertTrue(Bytes.equals("v3".getBytes(), column.get(1).getValue())); + column = res.getColumn(FAMILY1, QUALIFIER2); + assertEquals(2, column.size()); + assertTrue(Bytes.equals("v4".getBytes(), column.get(0).getValue())); + assertTrue(Bytes.equals("v3".getBytes(), column.get(1).getValue())); + assertEquals(4, res.getColumn(FAMILY1, QUALIFIER3).size()); + rows++; + } + assertEquals(100, rows); + } + + private HTable createTable(byte[] tableName) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor(FAMILY1); + hcd.setMaxVersions(10);// Just setting 10 as I am not testing with more than 10 versions here + htd.addFamily(hcd); + TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5); + HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); + return ht; + } + + private Put createPut(byte[] rowkey, String value) throws IOException { + Put put = new Put(rowkey); + put.add(FAMILY1, QUALIFIER1, value.getBytes()); + put.add(FAMILY1, QUALIFIER2, value.getBytes()); + put.add(FAMILY1, QUALIFIER3, value.getBytes()); + return put; + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java b/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java new file mode 100644 index 0000000..ee05cb7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor.example; + +import static org.junit.Assert.assertEquals; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.ZooKeeper; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestZooKeeperScanPolicyObserver { + private static final Log LOG = LogFactory.getLog(TestZooKeeperScanPolicyObserver.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final byte[] F = Bytes.toBytes("fam"); + private static final byte[] Q = Bytes.toBytes("qual"); + private static final byte[] R = Bytes.toBytes("row"); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Test we can first start the ZK cluster by itself + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + ZooKeeperScanPolicyObserver.class.getName()); + TEST_UTIL.startMiniZKCluster(); + TEST_UTIL.startMiniCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testScanPolicyObserver() throws Exception { + byte[] tableName = Bytes.toBytes("testScanPolicyObserver"); + HTableDescriptor desc = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor(F) + .setMaxVersions(10) + .setTimeToLive(1); + desc.addFamily(hcd); + TEST_UTIL.getHBaseAdmin().createTable(desc); + HTable t = new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName); + long now = EnvironmentEdgeManager.currentTimeMillis(); + + ZooKeeperWatcher zkw = HConnectionManager.getConnection(TEST_UTIL.getConfiguration()) + .getZooKeeperWatcher(); + ZooKeeper zk = zkw.getRecoverableZooKeeper().getZooKeeper(); + ZKUtil.createWithParents(zkw, ZooKeeperScanPolicyObserver.node); + // let's say test last backup was 1h ago + // using plain ZK here, because RecoverableZooKeeper add extra encoding to the data + zk.setData(ZooKeeperScanPolicyObserver.node, Bytes.toBytes(now - 3600*1000), -1); + + LOG.debug("Set time: "+Bytes.toLong(Bytes.toBytes(now - 3600*1000))); + + // sleep for 1s to give the ZK change a chance to reach the watcher in the observer. + // TODO: Better to wait for the data to be propagated + Thread.sleep(1000); + + long ts = now - 2000; + Put p = new Put(R); + p.add(F, Q, ts, Q); + t.put(p); + p = new Put(R); + p.add(F, Q, ts+1, Q); + t.put(p); + + // these two should be expired but for the override + // (their ts was 2s in the past) + Get g = new Get(R); + g.setMaxVersions(10); + Result r = t.get(g); + // still there? + assertEquals(2, r.size()); + + TEST_UTIL.flush(tableName); + TEST_UTIL.compact(tableName, true); + + g = new Get(R); + g.setMaxVersions(10); + r = t.get(g); + // still there? + assertEquals(2, r.size()); + zk.setData(ZooKeeperScanPolicyObserver.node, Bytes.toBytes(now), -1); + LOG.debug("Set time: "+now); + + TEST_UTIL.compact(tableName, true); + + g = new Get(R); + g.setMaxVersions(10); + r = t.get(g); + // should be gone now + assertEquals(0, r.size()); + t.close(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionDispatcher.java b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionDispatcher.java new file mode 100644 index 0000000..e5c47b1 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionDispatcher.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.errorhandling; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test that we propagate errors through an dispatcher exactly once via different failure + * injection mechanisms. + */ +@Category(SmallTests.class) +public class TestForeignExceptionDispatcher { + private static final Log LOG = LogFactory.getLog(TestForeignExceptionDispatcher.class); + + /** + * Exception thrown from the test + */ + final ForeignException EXTEXN = new ForeignException("FORTEST", new IllegalArgumentException("FORTEST")); + final ForeignException EXTEXN2 = new ForeignException("FORTEST2", new IllegalArgumentException("FORTEST2")); + + /** + * Tests that a dispatcher only dispatches only the first exception, and does not propagate + * subsequent exceptions. + */ + @Test + public void testErrorPropagation() { + ForeignExceptionListener listener1 = Mockito.mock(ForeignExceptionListener.class); + ForeignExceptionListener listener2 = Mockito.mock(ForeignExceptionListener.class); + ForeignExceptionDispatcher dispatcher = new ForeignExceptionDispatcher(); + + // add the listeners + dispatcher.addListener(listener1); + dispatcher.addListener(listener2); + + // create an artificial error + dispatcher.receive(EXTEXN); + + // make sure the listeners got the error + Mockito.verify(listener1, Mockito.times(1)).receive(EXTEXN); + Mockito.verify(listener2, Mockito.times(1)).receive(EXTEXN); + + // make sure that we get an exception + try { + dispatcher.rethrowException(); + fail("Monitor should have thrown an exception after getting error."); + } catch (ForeignException ex) { + assertTrue("Got an unexpected exception:" + ex, ex.getCause() == EXTEXN.getCause()); + LOG.debug("Got the testing exception!"); + } + + // push another error, which should be not be passed to listeners + dispatcher.receive(EXTEXN2); + Mockito.verify(listener1, Mockito.never()).receive(EXTEXN2); + Mockito.verify(listener2, Mockito.never()).receive(EXTEXN2); + } + + @Test + public void testSingleDispatcherWithTimer() { + ForeignExceptionListener listener1 = Mockito.mock(ForeignExceptionListener.class); + ForeignExceptionListener listener2 = Mockito.mock(ForeignExceptionListener.class); + + ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(); + + // add the listeners + monitor.addListener(listener1); + monitor.addListener(listener2); + + TimeoutExceptionInjector timer = new TimeoutExceptionInjector(monitor, 1000); + timer.start(); + timer.trigger(); + + assertTrue("Monitor didn't get timeout", monitor.hasException()); + + // verify that that we propagated the error + Mockito.verify(listener1).receive(Mockito.any(ForeignException.class)); + Mockito.verify(listener2).receive(Mockito.any(ForeignException.class)); + } + + /** + * Test that the dispatcher can receive an error via the timer mechanism. + */ + @Test + public void testAttemptTimer() { + ForeignExceptionListener listener1 = Mockito.mock(ForeignExceptionListener.class); + ForeignExceptionListener listener2 = Mockito.mock(ForeignExceptionListener.class); + ForeignExceptionDispatcher orchestrator = new ForeignExceptionDispatcher(); + + // add the listeners + orchestrator.addListener(listener1); + orchestrator.addListener(listener2); + + // now create a timer and check for that error + TimeoutExceptionInjector timer = new TimeoutExceptionInjector(orchestrator, 1000); + timer.start(); + timer.trigger(); + // make sure that we got the timer error + Mockito.verify(listener1, Mockito.times(1)).receive(Mockito.any(ForeignException.class)); + Mockito.verify(listener2, Mockito.times(1)).receive(Mockito.any(ForeignException.class)); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionSerialization.java b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionSerialization.java new file mode 100644 index 0000000..11363fe --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestForeignExceptionSerialization.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.errorhandling; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.protobuf.InvalidProtocolBufferException; + +/** + * Test that we correctly serialize exceptions from a remote source + */ +@Category(SmallTests.class) +public class TestForeignExceptionSerialization { + private static final String srcName = "someNode"; + + /** + * Verify that we get back similar stack trace information before an after serialization. + * @throws InvalidProtocolBufferException + */ + @Test + public void testSimpleException() throws InvalidProtocolBufferException { + String data = "some bytes"; + ForeignException in = new ForeignException("SRC", new IllegalArgumentException(data)); + // check that we get the data back out + ForeignException e = ForeignException.deserialize(ForeignException.serialize(srcName, in)); + assertNotNull(e); + + // now check that we get the right stack trace + StackTraceElement elem = new StackTraceElement(this.getClass().toString(), "method", "file", 1); + in.setStackTrace(new StackTraceElement[] { elem }); + e = ForeignException.deserialize(ForeignException.serialize(srcName, in)); + + assertNotNull(e); + assertEquals("Stack trace got corrupted", elem, e.getCause().getStackTrace()[0]); + assertEquals("Got an unexpectedly long stack trace", 1, e.getCause().getStackTrace().length); + } + + /** + * Compare that a generic exception's stack trace has the same stack trace elements after + * serialization and deserialization + * @throws InvalidProtocolBufferException + */ + @Test + public void testRemoteFromLocal() throws InvalidProtocolBufferException { + String errorMsg = "some message"; + Exception generic = new Exception(errorMsg); + generic.printStackTrace(); + assertTrue(generic.getMessage().contains(errorMsg)); + + ForeignException e = ForeignException.deserialize(ForeignException.serialize(srcName, generic)); + assertArrayEquals("Local stack trace got corrupted", generic.getStackTrace(), e.getCause().getStackTrace()); + + e.printStackTrace(); // should have ForeignException and source node in it. + assertTrue(e.getCause().getCause() == null); + + // verify that original error message is present in Foreign exception message + assertTrue(e.getCause().getMessage().contains(errorMsg)); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java new file mode 100644 index 0000000..641dbe0 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/errorhandling/TestTimeoutExceptionInjector.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.errorhandling; + +import static org.junit.Assert.fail; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test the {@link TimeoutExceptionInjector} to ensure we fulfill contracts + */ +@Category(SmallTests.class) +public class TestTimeoutExceptionInjector { + + private static final Log LOG = LogFactory.getLog(TestTimeoutExceptionInjector.class); + + /** + * Test that a manually triggered timer fires an exception. + */ + @Test(timeout = 60000) + public void testTimerTrigger() { + final long time = 10000000; // pick a value that is very far in the future + ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class); + TimeoutExceptionInjector timer = new TimeoutExceptionInjector(listener, time); + timer.start(); + timer.trigger(); + Mockito.verify(listener, Mockito.times(1)).receive(Mockito.any(ForeignException.class)); + } + + /** + * Test that a manually triggered exception with data fires with the data in receiveError. + */ + @Test + public void testTimerPassesOnErrorInfo() { + final long time = 1000000; + ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class); + TimeoutExceptionInjector timer = new TimeoutExceptionInjector(listener, time); + timer.start(); + timer.trigger(); + Mockito.verify(listener).receive(Mockito.any(ForeignException.class)); + } + + /** + * Demonstrate TimeoutExceptionInjector semantics -- completion means no more exceptions passed to + * error listener. + */ + @Test(timeout = 60000) + public void testStartAfterComplete() throws InterruptedException { + final long time = 10; + ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class); + TimeoutExceptionInjector timer = new TimeoutExceptionInjector(listener, time); + timer.complete(); + try { + timer.start(); + fail("Timer should fail to start after complete."); + } catch (IllegalStateException e) { + LOG.debug("Correctly failed timer: " + e.getMessage()); + } + Thread.sleep(time + 1); + Mockito.verifyZeroInteractions(listener); + } + + /** + * Demonstrate TimeoutExceptionInjector semantics -- triggering fires exception and completes + * the timer. + */ + @Test(timeout = 60000) + public void testStartAfterTrigger() throws InterruptedException { + final long time = 10; + ForeignExceptionListener listener = Mockito.mock(ForeignExceptionListener.class); + TimeoutExceptionInjector timer = new TimeoutExceptionInjector(listener, time); + timer.trigger(); + try { + timer.start(); + fail("Timer should fail to start after complete."); + } catch (IllegalStateException e) { + LOG.debug("Correctly failed timer: " + e.getMessage()); + } + Thread.sleep(time * 2); + Mockito.verify(listener, Mockito.times(1)).receive(Mockito.any(ForeignException.class)); + Mockito.verifyNoMoreInteractions(listener); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/executor/TestExecutorService.java b/src/test/java/org/apache/hadoop/hbase/executor/TestExecutorService.java new file mode 100644 index 0000000..805421f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/executor/TestExecutorService.java @@ -0,0 +1,182 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.executor; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.executor.ExecutorService.Executor; +import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorStatus; +import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.mockito.Mockito.*; + +@Category(SmallTests.class) +public class TestExecutorService { + private static final Log LOG = LogFactory.getLog(TestExecutorService.class); + + @Test + public void testExecutorService() throws Exception { + int maxThreads = 5; + int maxTries = 10; + int sleepInterval = 10; + + Server mockedServer = mock(Server.class); + when(mockedServer.getConfiguration()).thenReturn(HBaseConfiguration.create()); + + // Start an executor service pool with max 5 threads + ExecutorService executorService = new ExecutorService("unit_test"); + executorService.startExecutorService( + ExecutorType.MASTER_SERVER_OPERATIONS, maxThreads); + + Executor executor = + executorService.getExecutor(ExecutorType.MASTER_SERVER_OPERATIONS); + ThreadPoolExecutor pool = executor.threadPoolExecutor; + + // Assert no threads yet + assertEquals(0, pool.getPoolSize()); + + AtomicBoolean lock = new AtomicBoolean(true); + AtomicInteger counter = new AtomicInteger(0); + + // Submit maxThreads executors. + for (int i = 0; i < maxThreads; i++) { + executorService.submit( + new TestEventHandler(mockedServer, EventType.M_SERVER_SHUTDOWN, + lock, counter)); + } + + // The TestEventHandler will increment counter when it starts. + int tries = 0; + while (counter.get() < maxThreads && tries < maxTries) { + LOG.info("Waiting for all event handlers to start..."); + Thread.sleep(sleepInterval); + tries++; + } + + // Assert that pool is at max threads. + assertEquals(maxThreads, counter.get()); + assertEquals(maxThreads, pool.getPoolSize()); + + ExecutorStatus status = executor.getStatus(); + assertTrue(status.queuedEvents.isEmpty()); + assertEquals(5, status.running.size()); + checkStatusDump(status); + + + // Now interrupt the running Executor + synchronized (lock) { + lock.set(false); + lock.notifyAll(); + } + + // Executor increments counter again on way out so.... test that happened. + while (counter.get() < (maxThreads * 2) && tries < maxTries) { + System.out.println("Waiting for all event handlers to finish..."); + Thread.sleep(sleepInterval); + tries++; + } + + assertEquals(maxThreads * 2, counter.get()); + assertEquals(maxThreads, pool.getPoolSize()); + + // Add more than the number of threads items. + // Make sure we don't get RejectedExecutionException. + for (int i = 0; i < (2 * maxThreads); i++) { + executorService.submit( + new TestEventHandler(mockedServer, EventType.M_SERVER_SHUTDOWN, + lock, counter)); + } + // Now interrupt the running Executor + synchronized (lock) { + lock.set(false); + lock.notifyAll(); + } + + // Make sure threads are still around even after their timetolive expires. + Thread.sleep(executor.keepAliveTimeInMillis * 2); + assertEquals(maxThreads, pool.getPoolSize()); + + executorService.shutdown(); + + assertEquals(0, executorService.getAllExecutorStatuses().size()); + + // Test that submit doesn't throw NPEs + executorService.submit( + new TestEventHandler(mockedServer, EventType.M_SERVER_SHUTDOWN, + lock, counter)); + } + + private void checkStatusDump(ExecutorStatus status) throws IOException { + StringWriter sw = new StringWriter(); + status.dumpTo(sw, ""); + String dump = sw.toString(); + LOG.info("Got status dump:\n" + dump); + + assertTrue(dump.contains("Waiting on java.util.concurrent.atomic.AtomicBoolean")); + } + + public static class TestEventHandler extends EventHandler { + private AtomicBoolean lock; + private AtomicInteger counter; + + public TestEventHandler(Server server, EventType eventType, + AtomicBoolean lock, AtomicInteger counter) { + super(server, eventType); + this.lock = lock; + this.counter = counter; + } + + @Override + public void process() throws IOException { + int num = counter.incrementAndGet(); + LOG.info("Running process #" + num + ", threadName=" + + Thread.currentThread().getName()); + synchronized (lock) { + while (lock.get()) { + try { + lock.wait(); + } catch (InterruptedException e) { + // do nothing + } + } + } + counter.incrementAndGet(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java b/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java new file mode 100644 index 0000000..50888af --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java @@ -0,0 +1,97 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * under the License. + */ +package org.apache.hadoop.hbase.filter; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.experimental.categories.Category; + +/** + * Tests for the bit comparator + */ +@Category(SmallTests.class) +public class TestBitComparator extends TestCase { + + private static byte[] zeros = new byte[]{0, 0, 0, 0, 0, 0}; + private static byte[] ones = new byte[]{1, 1, 1, 1, 1, 1}; + private static byte[] data0 = new byte[]{0, 1, 2, 4, 8, 15}; + private static byte[] data1 = new byte[]{15, 0, 0, 0, 0, 0}; + private static byte[] data2 = new byte[]{0, 0, 0, 0, 0, 15}; + private static byte[] data3 = new byte[]{15, 15, 15, 15, 15}; + + // data for testing compareTo method with offset and length parameters + private static byte[] data1_2 = new byte[]{15, 15, 0, 0, 0, 0, 0, 15}; + private static byte[] data2_2 = new byte[]{15, 0, 0, 0, 0, 0, 15, 15}; + + private final int Equal = 0; + private final int NotEqual = 1; + + public void testANDOperation() { + testOperation(zeros, ones, BitComparator.BitwiseOp.AND, NotEqual); + testOperation(data1, ones, BitComparator.BitwiseOp.AND, Equal); + testOperation(data1, data0, BitComparator.BitwiseOp.AND, NotEqual); + testOperation(data2, data1, BitComparator.BitwiseOp.AND, NotEqual); + testOperation(ones, data0, BitComparator.BitwiseOp.AND, Equal); + testOperation(ones, data3, BitComparator.BitwiseOp.AND, NotEqual); + } + + public void testOROperation() { + testOperation(ones, zeros, BitComparator.BitwiseOp.OR, Equal); + testOperation(zeros, zeros, BitComparator.BitwiseOp.OR, NotEqual); + testOperation(data1, zeros, BitComparator.BitwiseOp.OR, Equal); + testOperation(data2, data1, BitComparator.BitwiseOp.OR, Equal); + testOperation(ones, data3, BitComparator.BitwiseOp.OR, NotEqual); + } + + public void testXOROperation() { + testOperation(ones, zeros, BitComparator.BitwiseOp.XOR, Equal); + testOperation(zeros, zeros, BitComparator.BitwiseOp.XOR, NotEqual); + testOperation(ones, ones, BitComparator.BitwiseOp.XOR, NotEqual); + testOperation(data2, data1, BitComparator.BitwiseOp.XOR, Equal); + testOperation(ones, data3, BitComparator.BitwiseOp.XOR, NotEqual); + } + + private void testOperation(byte[] data, byte[] comparatorBytes, BitComparator.BitwiseOp operator, int expected) { + BitComparator comparator = new BitComparator(comparatorBytes, operator); + assertEquals(comparator.compareTo(data), expected); + } + + public void testANDOperationWithOffset() { + testOperationWithOffset(data1_2, ones, BitComparator.BitwiseOp.AND, Equal); + testOperationWithOffset(data1_2, data0, BitComparator.BitwiseOp.AND, NotEqual); + testOperationWithOffset(data2_2, data1, BitComparator.BitwiseOp.AND, NotEqual); + } + + public void testOROperationWithOffset() { + testOperationWithOffset(data1_2, zeros, BitComparator.BitwiseOp.OR, Equal); + testOperationWithOffset(data2_2, data1, BitComparator.BitwiseOp.OR, Equal); + } + + public void testXOROperationWithOffset() { + testOperationWithOffset(data2_2, data1, BitComparator.BitwiseOp.XOR, Equal); + } + + private void testOperationWithOffset(byte[] data, byte[] comparatorBytes, BitComparator.BitwiseOp operator, int expected) { + BitComparator comparator = new BitComparator(comparatorBytes, operator); + assertEquals(comparator.compareTo(data, 1, comparatorBytes.length), expected); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestColumnCountGetFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnCountGetFilter.java new file mode 100644 index 0000000..f725e61 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnCountGetFilter.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueTestUtil; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestColumnCountGetFilter { + + private final static HBaseTestingUtility TEST_UTIL = new + HBaseTestingUtility(); + + @Test + public void testColumnCountGetFilter() throws IOException { + String family = "Family"; + HTableDescriptor htd = new HTableDescriptor("testColumnCountGetFilter"); + htd.addFamily(new HColumnDescriptor(family)); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + try { + String valueString = "ValueString"; + String row = "row-1"; + List columns = generateRandomWords(10000, "column"); + Put p = new Put(Bytes.toBytes(row)); + p.setWriteToWAL(false); + for (String column : columns) { + KeyValue kv = KeyValueTestUtil.create(row, family, column, 0, valueString); + p.add(kv); + } + region.put(p); + + Get get = new Get(row.getBytes()); + Filter filter = new ColumnCountGetFilter(100); + get.setFilter(filter); + Scan scan = new Scan(get); + InternalScanner scanner = region.getScanner(scan); + List results = new ArrayList(); + scanner.next(results); + assertEquals(100, results.size()); + } finally { + region.close(); + region.getLog().closeAndDelete(); + } + + region.close(); + region.getLog().closeAndDelete(); + } + + @Test + public void testColumnCountGetFilterWithFilterList() throws IOException { + String family = "Family"; + HTableDescriptor htd = new HTableDescriptor("testColumnCountGetFilter"); + htd.addFamily(new HColumnDescriptor(family)); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + try { + String valueString = "ValueString"; + String row = "row-1"; + List columns = generateRandomWords(10000, "column"); + Put p = new Put(Bytes.toBytes(row)); + p.setWriteToWAL(false); + for (String column : columns) { + KeyValue kv = KeyValueTestUtil.create(row, family, column, 0, valueString); + p.add(kv); + } + region.put(p); + + Get get = new Get(row.getBytes()); + FilterList filterLst = new FilterList (); + filterLst.addFilter( new ColumnCountGetFilter(100)); + get.setFilter(filterLst); + Scan scan = new Scan(get); + InternalScanner scanner = region.getScanner(scan); + List results = new ArrayList(); + scanner.next(results); + assertEquals(100, results.size()); + } finally { + region.close(); + region.getLog().closeAndDelete(); + } + + region.close(); + region.getLog().closeAndDelete(); + } + + List generateRandomWords(int numberOfWords, String suffix) { + Set wordSet = new HashSet(); + for (int i = 0; i < numberOfWords; i++) { + int lengthOfWords = (int) (Math.random()*2) + 1; + char[] wordChar = new char[lengthOfWords]; + for (int j = 0; j < wordChar.length; j++) { + wordChar[j] = (char) (Math.random() * 26 + 97); + } + String word; + if (suffix == null) { + word = new String(wordChar); + } else { + word = new String(wordChar) + suffix; + } + wordSet.add(word); + } + List wordList = new ArrayList(wordSet); + return wordList; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java new file mode 100644 index 0000000..720d882 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java @@ -0,0 +1,102 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +/** + * Test for the ColumnPaginationFilter, used mainly to test the successful serialization of the filter. + * More test functionality can be found within {@link org.apache.hadoop.hbase.filter.TestFilter#testColumnPaginationFilter()} + */ +@Category(SmallTests.class) +public class TestColumnPaginationFilter extends TestCase +{ + private static final byte[] ROW = Bytes.toBytes("row_1_test"); + private static final byte[] COLUMN_FAMILY = Bytes.toBytes("test"); + private static final byte[] VAL_1 = Bytes.toBytes("a"); + private static final byte [] COLUMN_QUALIFIER = Bytes.toBytes("foo"); + + private Filter columnPaginationFilter; + + @Override + protected void setUp() throws Exception { + super.setUp(); + columnPaginationFilter = getColumnPaginationFilter(); + + } + private Filter getColumnPaginationFilter() { + return new ColumnPaginationFilter(1,0); + } + + private Filter serializationTest(Filter filter) throws Exception { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(stream); + filter.write(out); + out.close(); + byte[] buffer = stream.toByteArray(); + + DataInputStream in = + new DataInputStream(new ByteArrayInputStream(buffer)); + Filter newFilter = new ColumnPaginationFilter(); + newFilter.readFields(in); + + return newFilter; + } + + + /** + * The more specific functionality tests are contained within the TestFilters class. This class is mainly for testing + * serialization + * + * @param filter + * @throws Exception + */ + private void basicFilterTests(ColumnPaginationFilter filter) throws Exception + { + KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_1); + assertTrue("basicFilter1", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE_AND_NEXT_COL); + } + + /** + * Tests serialization + * @throws Exception + */ + public void testSerialization() throws Exception { + Filter newFilter = serializationTest(columnPaginationFilter); + basicFilterTests((ColumnPaginationFilter)newFilter); + } + + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java new file mode 100644 index 0000000..9f89914 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPrefixFilter.java @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestColumnPrefixFilter { + + private final static HBaseTestingUtility TEST_UTIL = new + HBaseTestingUtility(); + + @Test + public void testColumnPrefixFilter() throws IOException { + String family = "Family"; + HTableDescriptor htd = new HTableDescriptor("TestColumnPrefixFilter"); + htd.addFamily(new HColumnDescriptor(family)); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + try { + List rows = generateRandomWords(100, "row"); + List columns = generateRandomWords(10000, "column"); + long maxTimestamp = 2; + + List kvList = new ArrayList(); + + Map> prefixMap = new HashMap>(); + + prefixMap.put("p", new ArrayList()); + prefixMap.put("s", new ArrayList()); + + String valueString = "ValueString"; + + for (String row: rows) { + Put p = new Put(Bytes.toBytes(row)); + p.setWriteToWAL(false); + for (String column: columns) { + for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { + KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, + valueString); + p.add(kv); + kvList.add(kv); + for (String s: prefixMap.keySet()) { + if (column.startsWith(s)) { + prefixMap.get(s).add(kv); + } + } + } + } + region.put(p); + } + + ColumnPrefixFilter filter; + Scan scan = new Scan(); + scan.setMaxVersions(); + for (String s: prefixMap.keySet()) { + filter = new ColumnPrefixFilter(Bytes.toBytes(s)); + + scan.setFilter(filter); + + InternalScanner scanner = region.getScanner(scan); + List results = new ArrayList(); + while(scanner.next(results)); + assertEquals(prefixMap.get(s).size(), results.size()); + } + } finally { + region.close(); + region.getLog().closeAndDelete(); + } + + region.close(); + region.getLog().closeAndDelete(); + } + + @Test + public void testColumnPrefixFilterWithFilterList() throws IOException { + String family = "Family"; + HTableDescriptor htd = new HTableDescriptor("TestColumnPrefixFilter"); + htd.addFamily(new HColumnDescriptor(family)); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + try { + List rows = generateRandomWords(100, "row"); + List columns = generateRandomWords(10000, "column"); + long maxTimestamp = 2; + + List kvList = new ArrayList(); + + Map> prefixMap = new HashMap>(); + + prefixMap.put("p", new ArrayList()); + prefixMap.put("s", new ArrayList()); + + String valueString = "ValueString"; + + for (String row: rows) { + Put p = new Put(Bytes.toBytes(row)); + p.setWriteToWAL(false); + for (String column: columns) { + for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { + KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, + valueString); + p.add(kv); + kvList.add(kv); + for (String s: prefixMap.keySet()) { + if (column.startsWith(s)) { + prefixMap.get(s).add(kv); + } + } + } + } + region.put(p); + } + + ColumnPrefixFilter filter; + Scan scan = new Scan(); + scan.setMaxVersions(); + for (String s: prefixMap.keySet()) { + filter = new ColumnPrefixFilter(Bytes.toBytes(s)); + + //this is how this test differs from the one above + FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); + filterList.addFilter(filter); + scan.setFilter(filterList); + + InternalScanner scanner = region.getScanner(scan); + List results = new ArrayList(); + while(scanner.next(results)); + assertEquals(prefixMap.get(s).size(), results.size()); + } + } finally { + region.close(); + region.getLog().closeAndDelete(); + } + + region.close(); + region.getLog().closeAndDelete(); + } + + List generateRandomWords(int numberOfWords, String suffix) { + Set wordSet = new HashSet(); + for (int i = 0; i < numberOfWords; i++) { + int lengthOfWords = (int) (Math.random()*2) + 1; + char[] wordChar = new char[lengthOfWords]; + for (int j = 0; j < wordChar.length; j++) { + wordChar[j] = (char) (Math.random() * 26 + 97); + } + String word; + if (suffix == null) { + word = new String(wordChar); + } else { + word = new String(wordChar) + suffix; + } + wordSet.add(word); + } + List wordList = new ArrayList(wordSet); + return wordList; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestColumnRangeFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnRangeFilter.java new file mode 100644 index 0000000..9212b07 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestColumnRangeFilter.java @@ -0,0 +1,265 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + + +class StringRange { + private String start = null; + private String end = null; + private boolean startInclusive = true; + private boolean endInclusive = false; + + public StringRange(String start, boolean startInclusive, String end, + boolean endInclusive) { + this.start = start; + this.startInclusive = startInclusive; + this.end = end; + this.endInclusive = endInclusive; + } + + public String getStart() { + return this.start; + } + + public String getEnd() { + return this.end; + } + + public boolean isStartInclusive() { + return this.startInclusive; + } + + public boolean isEndInclusive() { + return this.endInclusive; + } + + @Override + public int hashCode() { + int hashCode = 0; + if (this.start != null) { + hashCode ^= this.start.hashCode(); + } + + if (this.end != null) { + hashCode ^= this.end.hashCode(); + } + return hashCode; + } + + @Override + public String toString() { + String result = (this.startInclusive ? "[" : "(") + + (this.start == null ? null : this.start) + ", " + + (this.end == null ? null : this.end) + + (this.endInclusive ? "]" : ")"); + return result; + } + + public boolean inRange(String value) { + boolean afterStart = true; + if (this.start != null) { + int startCmp = value.compareTo(this.start); + afterStart = this.startInclusive ? startCmp >= 0 : startCmp > 0; + } + + boolean beforeEnd = true; + if (this.end != null) { + int endCmp = value.compareTo(this.end); + beforeEnd = this.endInclusive ? endCmp <= 0 : endCmp < 0; + } + + return afterStart && beforeEnd; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + + +@Category(MediumTests.class) +public class TestColumnRangeFilter { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private final Log LOG = LogFactory.getLog(this.getClass()); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + // Nothing to do. + } + + /** + * @throws java.lang.Exception + */ + @After + public void tearDown() throws Exception { + // Nothing to do. + } + + @Test + public void TestColumnRangeFilterClient() throws Exception { + String family = "Family"; + String table = "TestColumnRangeFilterClient"; + HTable ht = TEST_UTIL.createTable(Bytes.toBytes(table), + Bytes.toBytes(family), Integer.MAX_VALUE); + + List rows = generateRandomWords(10, 8); + long maxTimestamp = 2; + List columns = generateRandomWords(20000, 8); + + List kvList = new ArrayList(); + + Map> rangeMap = new HashMap>(); + + rangeMap.put(new StringRange(null, true, "b", false), + new ArrayList()); + rangeMap.put(new StringRange("p", true, "q", false), + new ArrayList()); + rangeMap.put(new StringRange("r", false, "s", true), + new ArrayList()); + rangeMap.put(new StringRange("z", false, null, false), + new ArrayList()); + String valueString = "ValueString"; + + for (String row : rows) { + Put p = new Put(Bytes.toBytes(row)); + p.setWriteToWAL(false); + for (String column : columns) { + for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { + KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, + valueString); + p.add(kv); + kvList.add(kv); + for (StringRange s : rangeMap.keySet()) { + if (s.inRange(column)) { + rangeMap.get(s).add(kv); + } + } + } + } + ht.put(p); + } + + TEST_UTIL.flush(); + + ColumnRangeFilter filter; + Scan scan = new Scan(); + scan.setMaxVersions(); + for (StringRange s : rangeMap.keySet()) { + filter = new ColumnRangeFilter(s.getStart() == null ? null + : Bytes.toBytes(s.getStart()), s.isStartInclusive(), + s.getEnd() == null ? null : Bytes.toBytes(s.getEnd()), + s.isEndInclusive()); + scan.setFilter(filter); + ResultScanner scanner = ht.getScanner(scan); + List results = new ArrayList(); + LOG.info("scan column range: " + s.toString()); + long timeBeforeScan = System.currentTimeMillis(); + + Result result; + while ((result = scanner.next()) != null) { + for (KeyValue kv : result.list()) { + results.add(kv); + } + } + long scanTime = System.currentTimeMillis() - timeBeforeScan; + scanner.close(); + LOG.info("scan time = " + scanTime + "ms"); + LOG.info("found " + results.size() + " results"); + LOG.info("Expecting " + rangeMap.get(s).size() + " results"); + + /* + for (KeyValue kv : results) { + LOG.info("found row " + Bytes.toString(kv.getRow()) + ", column " + + Bytes.toString(kv.getQualifier())); + } + */ + + assertEquals(rangeMap.get(s).size(), results.size()); + } + ht.close(); + } + + List generateRandomWords(int numberOfWords, int maxLengthOfWords) { + Set wordSet = new HashSet(); + for (int i = 0; i < numberOfWords; i++) { + int lengthOfWords = (int) (Math.random() * maxLengthOfWords) + 1; + char[] wordChar = new char[lengthOfWords]; + for (int j = 0; j < wordChar.length; j++) { + wordChar[j] = (char) (Math.random() * 26 + 97); + } + String word = new String(wordChar); + wordSet.add(word); + } + List wordList = new ArrayList(wordSet); + return wordList; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java new file mode 100644 index 0000000..8e13a5b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestDependentColumnFilter.java @@ -0,0 +1,250 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter.ReturnCode; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestDependentColumnFilter extends TestCase { + private final Log LOG = LogFactory.getLog(this.getClass()); + private static final byte[][] ROWS = { + Bytes.toBytes("test1"),Bytes.toBytes("test2") + }; + private static final byte[][] FAMILIES = { + Bytes.toBytes("familyOne"),Bytes.toBytes("familyTwo") + }; + private static final long STAMP_BASE = System.currentTimeMillis(); + private static final long[] STAMPS = { + STAMP_BASE-100, STAMP_BASE-200, STAMP_BASE-300 + }; + private static final byte[] QUALIFIER = Bytes.toBytes("qualifier"); + private static final byte[][] BAD_VALS = { + Bytes.toBytes("bad1"), Bytes.toBytes("bad2"), Bytes.toBytes("bad3") + }; + private static final byte[] MATCH_VAL = Bytes.toBytes("match"); + private HBaseTestingUtility testUtil; + + List testVals; + private HRegion region; + + @Override + protected void setUp() throws Exception { + super.setUp(); + + testUtil = new HBaseTestingUtility(); + + testVals = makeTestVals(); + + HTableDescriptor htd = new HTableDescriptor(getName()); + htd.addFamily(new HColumnDescriptor(FAMILIES[0])); + htd.addFamily(new HColumnDescriptor(FAMILIES[1])); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + this.region = HRegion.createHRegion(info, testUtil.getDataTestDir(), + testUtil.getConfiguration(), htd); + addData(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + this.region.close(); + region.getLog().closeAndDelete(); + } + + private void addData() throws IOException { + Put put = new Put(ROWS[0]); + // add in an entry for each stamp, with 2 as a "good" value + put.add(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]); + put.add(FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]); + put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL); + // add in entries for stamps 0 and 2. + // without a value check both will be "accepted" + // with one 2 will be accepted(since the corresponding ts entry + // has a matching value + put.add(FAMILIES[1], QUALIFIER, STAMPS[0], BAD_VALS[0]); + put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]); + + this.region.put(put); + + put = new Put(ROWS[1]); + put.add(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]); + // there is no corresponding timestamp for this so it should never pass + put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL); + // if we reverse the qualifiers this one should pass + put.add(FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL); + // should pass + put.add(FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]); + + this.region.put(put); + } + + private List makeTestVals() { + List testVals = new ArrayList(); + testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0])); + testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1])); + testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2])); + testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL)); + testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2])); + + return testVals; + } + + /** + * This shouldn't be confused with TestFilter#verifyScan + * as expectedKeys is not the per row total, but the scan total + * + * @param s + * @param expectedRows + * @param expectedCells + * @throws IOException + */ + private void verifyScan(Scan s, long expectedRows, long expectedCells) + throws IOException { + InternalScanner scanner = this.region.getScanner(s); + List results = new ArrayList(); + int i = 0; + int cells = 0; + for (boolean done = true; done; i++) { + done = scanner.next(results); + Arrays.sort(results.toArray(new KeyValue[results.size()]), + KeyValue.COMPARATOR); + LOG.info("counter=" + i + ", " + results); + if (results.isEmpty()) break; + cells += results.size(); + assertTrue("Scanned too many rows! Only expected " + expectedRows + + " total but already scanned " + (i+1), expectedRows > i); + assertTrue("Expected " + expectedCells + " cells total but " + + "already scanned " + cells, expectedCells >= cells); + results.clear(); + } + assertEquals("Expected " + expectedRows + " rows but scanned " + i + + " rows", expectedRows, i); + assertEquals("Expected " + expectedCells + " cells but scanned " + cells + + " cells", expectedCells, cells); + } + + /** + * Test scans using a DependentColumnFilter + */ + public void testScans() throws Exception { + Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER); + + Scan scan = new Scan(); + scan.setFilter(filter); + scan.setMaxVersions(Integer.MAX_VALUE); + + verifyScan(scan, 2, 8); + + // drop the filtering cells + filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true); + scan = new Scan(); + scan.setFilter(filter); + scan.setMaxVersions(Integer.MAX_VALUE); + + verifyScan(scan, 2, 3); + + // include a comparator operation + filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, false, + CompareOp.EQUAL, new BinaryComparator(MATCH_VAL)); + scan = new Scan(); + scan.setFilter(filter); + scan.setMaxVersions(Integer.MAX_VALUE); + + /* + * expecting to get the following 3 cells + * row 0 + * put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL); + * put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]); + * row 1 + * put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL); + */ + verifyScan(scan, 2, 3); + + // include a comparator operation and drop comparator + filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, + CompareOp.EQUAL, new BinaryComparator(MATCH_VAL)); + scan = new Scan(); + scan.setFilter(filter); + scan.setMaxVersions(Integer.MAX_VALUE); + + /* + * expecting to get the following 1 cell + * row 0 + * put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]); + */ + verifyScan(scan, 1, 1); + + } + + /** + * Test that the filter correctly drops rows without a corresponding timestamp + * + * @throws Exception + */ + public void testFilterDropping() throws Exception { + Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER); + List accepted = new ArrayList(); + for(KeyValue val : testVals) { + if(filter.filterKeyValue(val) == ReturnCode.INCLUDE) { + accepted.add(val); + } + } + assertEquals("check all values accepted from filterKeyValue", 5, accepted.size()); + + filter.filterRow(accepted); + assertEquals("check filterRow(List) dropped cell without corresponding column entry", 4, accepted.size()); + + // start do it again with dependent column dropping on + filter = new DependentColumnFilter(FAMILIES[1], QUALIFIER, true); + accepted.clear(); + for(KeyValue val : testVals) { + if(filter.filterKeyValue(val) == ReturnCode.INCLUDE) { + accepted.add(val); + } + } + assertEquals("check the filtering column cells got dropped", 2, accepted.size()); + + filter.filterRow(accepted); + assertEquals("check cell retention", 2, accepted.size()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java new file mode 100644 index 0000000..d583700 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java @@ -0,0 +1,1669 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import junit.framework.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +import com.google.common.base.Throwables; + +/** + * Test filters at the HRegion doorstep. + */ +@Category(SmallTests.class) +public class TestFilter extends HBaseTestCase { + private final static Log LOG = LogFactory.getLog(TestFilter.class); + private HRegion region; + + // + // Rows, Qualifiers, and Values are in two groups, One and Two. + // + + private static final byte [][] ROWS_ONE = { + Bytes.toBytes("testRowOne-0"), Bytes.toBytes("testRowOne-1"), + Bytes.toBytes("testRowOne-2"), Bytes.toBytes("testRowOne-3") + }; + + private static final byte [][] ROWS_TWO = { + Bytes.toBytes("testRowTwo-0"), Bytes.toBytes("testRowTwo-1"), + Bytes.toBytes("testRowTwo-2"), Bytes.toBytes("testRowTwo-3") + }; + + private static final byte [][] ROWS_THREE = { + Bytes.toBytes("testRowThree-0"), Bytes.toBytes("testRowThree-1"), + Bytes.toBytes("testRowThree-2"), Bytes.toBytes("testRowThree-3") + }; + + private static final byte [][] ROWS_FOUR = { + Bytes.toBytes("testRowFour-0"), Bytes.toBytes("testRowFour-1"), + Bytes.toBytes("testRowFour-2"), Bytes.toBytes("testRowFour-3") + }; + + private static final byte [][] FAMILIES = { + Bytes.toBytes("testFamilyOne"), Bytes.toBytes("testFamilyTwo") + }; + + private static final byte [][] FAMILIES_1 = { + Bytes.toBytes("testFamilyThree"), Bytes.toBytes("testFamilyFour") + }; + + private static final byte [][] QUALIFIERS_ONE = { + Bytes.toBytes("testQualifierOne-0"), Bytes.toBytes("testQualifierOne-1"), + Bytes.toBytes("testQualifierOne-2"), Bytes.toBytes("testQualifierOne-3") + }; + + private static final byte [][] QUALIFIERS_TWO = { + Bytes.toBytes("testQualifierTwo-0"), Bytes.toBytes("testQualifierTwo-1"), + Bytes.toBytes("testQualifierTwo-2"), Bytes.toBytes("testQualifierTwo-3") + }; + + private static final byte [][] QUALIFIERS_THREE = { + Bytes.toBytes("testQualifierThree-0"), Bytes.toBytes("testQualifierThree-1"), + Bytes.toBytes("testQualifierThree-2"), Bytes.toBytes("testQualifierThree-3") + }; + + private static final byte [][] QUALIFIERS_FOUR = { + Bytes.toBytes("testQualifierFour-0"), Bytes.toBytes("testQualifierFour-1"), + Bytes.toBytes("testQualifierFour-2"), Bytes.toBytes("testQualifierFour-3") + }; + + private static final byte [][] VALUES = { + Bytes.toBytes("testValueOne"), Bytes.toBytes("testValueTwo") + }; + + byte [][] NEW_FAMILIES = { + Bytes.toBytes("f1"), Bytes.toBytes("f2") + }; + + private long numRows = ROWS_ONE.length + ROWS_TWO.length; + private long colsPerRow = FAMILIES.length * QUALIFIERS_ONE.length; + + + protected void setUp() throws Exception { + super.setUp(); + HTableDescriptor htd = new HTableDescriptor(getName()); + htd.addFamily(new HColumnDescriptor(FAMILIES[0])); + htd.addFamily(new HColumnDescriptor(FAMILIES[1])); + htd.addFamily(new HColumnDescriptor(FAMILIES_1[0])); + htd.addFamily(new HColumnDescriptor(FAMILIES_1[1])); + htd.addFamily(new HColumnDescriptor(NEW_FAMILIES[0])); + htd.addFamily(new HColumnDescriptor(NEW_FAMILIES[1])); + htd.addFamily(new HColumnDescriptor(FAMILIES_1[1])); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + this.region = HRegion.createHRegion(info, this.testDir, this.conf, htd); + + // Insert first half + for(byte [] ROW : ROWS_ONE) { + Put p = new Put(ROW); + p.setWriteToWAL(false); + for(byte [] QUALIFIER : QUALIFIERS_ONE) { + p.add(FAMILIES[0], QUALIFIER, VALUES[0]); + } + this.region.put(p); + } + for(byte [] ROW : ROWS_TWO) { + Put p = new Put(ROW); + p.setWriteToWAL(false); + for(byte [] QUALIFIER : QUALIFIERS_TWO) { + p.add(FAMILIES[1], QUALIFIER, VALUES[1]); + } + this.region.put(p); + } + + // Flush + this.region.flushcache(); + + // Insert second half (reverse families) + for(byte [] ROW : ROWS_ONE) { + Put p = new Put(ROW); + p.setWriteToWAL(false); + for(byte [] QUALIFIER : QUALIFIERS_ONE) { + p.add(FAMILIES[1], QUALIFIER, VALUES[0]); + } + this.region.put(p); + } + for(byte [] ROW : ROWS_TWO) { + Put p = new Put(ROW); + p.setWriteToWAL(false); + for(byte [] QUALIFIER : QUALIFIERS_TWO) { + p.add(FAMILIES[0], QUALIFIER, VALUES[1]); + } + this.region.put(p); + } + + // Delete the second qualifier from all rows and families + for(byte [] ROW : ROWS_ONE) { + Delete d = new Delete(ROW); + d.deleteColumns(FAMILIES[0], QUALIFIERS_ONE[1]); + d.deleteColumns(FAMILIES[1], QUALIFIERS_ONE[1]); + this.region.delete(d, null, false); + } + for(byte [] ROW : ROWS_TWO) { + Delete d = new Delete(ROW); + d.deleteColumns(FAMILIES[0], QUALIFIERS_TWO[1]); + d.deleteColumns(FAMILIES[1], QUALIFIERS_TWO[1]); + this.region.delete(d, null, false); + } + colsPerRow -= 2; + + // Delete the second rows from both groups, one column at a time + for(byte [] QUALIFIER : QUALIFIERS_ONE) { + Delete d = new Delete(ROWS_ONE[1]); + d.deleteColumns(FAMILIES[0], QUALIFIER); + d.deleteColumns(FAMILIES[1], QUALIFIER); + this.region.delete(d, null, false); + } + for(byte [] QUALIFIER : QUALIFIERS_TWO) { + Delete d = new Delete(ROWS_TWO[1]); + d.deleteColumns(FAMILIES[0], QUALIFIER); + d.deleteColumns(FAMILIES[1], QUALIFIER); + this.region.delete(d, null, false); + } + numRows -= 2; + } + + protected void tearDown() throws Exception { + HLog hlog = region.getLog(); + region.close(); + hlog.closeAndDelete(); + super.tearDown(); + } + + + public void testRegionScannerReseek() throws Exception { + // create new rows and column family to show how reseek works.. + for (byte[] ROW : ROWS_THREE) { + Put p = new Put(ROW); + p.setWriteToWAL(true); + for (byte[] QUALIFIER : QUALIFIERS_THREE) { + p.add(FAMILIES[0], QUALIFIER, VALUES[0]); + + } + this.region.put(p); + } + for (byte[] ROW : ROWS_FOUR) { + Put p = new Put(ROW); + p.setWriteToWAL(false); + for (byte[] QUALIFIER : QUALIFIERS_FOUR) { + p.add(FAMILIES[1], QUALIFIER, VALUES[1]); + } + this.region.put(p); + } + // Flush + this.region.flushcache(); + + // Insert second half (reverse families) + for (byte[] ROW : ROWS_THREE) { + Put p = new Put(ROW); + p.setWriteToWAL(false); + for (byte[] QUALIFIER : QUALIFIERS_THREE) { + p.add(FAMILIES[1], QUALIFIER, VALUES[0]); + } + this.region.put(p); + } + for (byte[] ROW : ROWS_FOUR) { + Put p = new Put(ROW); + p.setWriteToWAL(false); + for (byte[] QUALIFIER : QUALIFIERS_FOUR) { + p.add(FAMILIES[0], QUALIFIER, VALUES[1]); + } + this.region.put(p); + } + + Scan s = new Scan(); + // set a start row + s.setStartRow(ROWS_FOUR[1]); + RegionScanner scanner = region.getScanner(s); + + // reseek to row three. + scanner.reseek(ROWS_THREE[1]); + List results = new ArrayList(); + + // the results should belong to ROWS_THREE[1] + scanner.next(results); + for (KeyValue keyValue : results) { + assertEquals("The rows with ROWS_TWO as row key should be appearing.", + Bytes.toString(keyValue.getRow()), Bytes.toString(ROWS_THREE[1])); + } + // again try to reseek to a value before ROWS_THREE[1] + scanner.reseek(ROWS_ONE[1]); + results = new ArrayList(); + // This time no seek would have been done to ROWS_ONE[1] + scanner.next(results); + for (KeyValue keyValue : results) { + assertFalse("Cannot rewind back to a value less than previous reseek.", + Bytes.toString(keyValue.getRow()).contains("testRowOne")); + } + } + + public void testNoFilter() throws Exception { + // No filter + long expectedRows = this.numRows; + long expectedKeys = this.colsPerRow; + + // Both families + Scan s = new Scan(); + verifyScan(s, expectedRows, expectedKeys); + + // One family + s = new Scan(); + s.addFamily(FAMILIES[0]); + verifyScan(s, expectedRows, expectedKeys/2); + } + + public void testPrefixFilter() throws Exception { + // Grab rows from group one (half of total) + long expectedRows = this.numRows / 2; + long expectedKeys = this.colsPerRow; + Scan s = new Scan(); + s.setFilter(new PrefixFilter(Bytes.toBytes("testRowOne"))); + verifyScan(s, expectedRows, expectedKeys); + } + + public void testPageFilter() throws Exception { + + // KVs in first 6 rows + KeyValue [] expectedKVs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) + }; + + // Grab all 6 rows + long expectedRows = 6; + long expectedKeys = this.colsPerRow; + Scan s = new Scan(); + s.setFilter(new PageFilter(expectedRows)); + verifyScan(s, expectedRows, expectedKeys); + s.setFilter(new PageFilter(expectedRows)); + verifyScanFull(s, expectedKVs); + + // Grab first 4 rows (6 cols per row) + expectedRows = 4; + expectedKeys = this.colsPerRow; + s = new Scan(); + s.setFilter(new PageFilter(expectedRows)); + verifyScan(s, expectedRows, expectedKeys); + s.setFilter(new PageFilter(expectedRows)); + verifyScanFull(s, Arrays.copyOf(expectedKVs, 24)); + + // Grab first 2 rows + expectedRows = 2; + expectedKeys = this.colsPerRow; + s = new Scan(); + s.setFilter(new PageFilter(expectedRows)); + verifyScan(s, expectedRows, expectedKeys); + s.setFilter(new PageFilter(expectedRows)); + verifyScanFull(s, Arrays.copyOf(expectedKVs, 12)); + + // Grab first row + expectedRows = 1; + expectedKeys = this.colsPerRow; + s = new Scan(); + s.setFilter(new PageFilter(expectedRows)); + verifyScan(s, expectedRows, expectedKeys); + s.setFilter(new PageFilter(expectedRows)); + verifyScanFull(s, Arrays.copyOf(expectedKVs, 6)); + + } + + /** + * Tests the the {@link WhileMatchFilter} works in combination with a + * {@link Filter} that uses the + * {@link Filter#filterRow()} method. + * + * See HBASE-2258. + * + * @throws Exception + */ + public void testWhileMatchFilterWithFilterRow() throws Exception { + final int pageSize = 4; + + Scan s = new Scan(); + WhileMatchFilter filter = new WhileMatchFilter(new PageFilter(pageSize)); + s.setFilter(filter); + + InternalScanner scanner = this.region.getScanner(s); + int scannerCounter = 0; + while (true) { + boolean isMoreResults = scanner.next(new ArrayList()); + scannerCounter++; + + if (scannerCounter >= pageSize) { + Assert.assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining()); + } + if (!isMoreResults) { + break; + } + } + Assert.assertEquals("The page filter returned more rows than expected", pageSize, scannerCounter); + } + + /** + * Tests the the {@link WhileMatchFilter} works in combination with a + * {@link Filter} that uses the + * {@link Filter#filterRowKey(byte[], int, int)} method. + * + * See HBASE-2258. + * + * @throws Exception + */ + public void testWhileMatchFilterWithFilterRowKey() throws Exception { + Scan s = new Scan(); + String prefix = "testRowOne"; + WhileMatchFilter filter = new WhileMatchFilter(new PrefixFilter(Bytes.toBytes(prefix))); + s.setFilter(filter); + + InternalScanner scanner = this.region.getScanner(s); + while (true) { + ArrayList values = new ArrayList(); + boolean isMoreResults = scanner.next(values); + if (!isMoreResults || !Bytes.toString(values.get(0).getRow()).startsWith(prefix)) { + Assert.assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining()); + } + if (!isMoreResults) { + break; + } + } + } + + /** + * Tests the the {@link WhileMatchFilter} works in combination with a + * {@link Filter} that uses the + * {@link Filter#filterKeyValue(org.apache.hadoop.hbase.KeyValue)} method. + * + * See HBASE-2258. + * + * @throws Exception + */ + public void testWhileMatchFilterWithFilterKeyValue() throws Exception { + Scan s = new Scan(); + WhileMatchFilter filter = new WhileMatchFilter( + new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], CompareOp.EQUAL, Bytes.toBytes("foo")) + ); + s.setFilter(filter); + + InternalScanner scanner = this.region.getScanner(s); + while (true) { + ArrayList values = new ArrayList(); + boolean isMoreResults = scanner.next(values); + Assert.assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining()); + if (!isMoreResults) { + break; + } + } + } + + public void testInclusiveStopFilter() throws IOException { + + // Grab rows from group one + + // If we just use start/stop row, we get total/2 - 1 rows + long expectedRows = (this.numRows / 2) - 1; + long expectedKeys = this.colsPerRow; + Scan s = new Scan(Bytes.toBytes("testRowOne-0"), + Bytes.toBytes("testRowOne-3")); + verifyScan(s, expectedRows, expectedKeys); + + // Now use start row with inclusive stop filter + expectedRows = this.numRows / 2; + s = new Scan(Bytes.toBytes("testRowOne-0")); + s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowOne-3"))); + verifyScan(s, expectedRows, expectedKeys); + + // Grab rows from group two + + // If we just use start/stop row, we get total/2 - 1 rows + expectedRows = (this.numRows / 2) - 1; + expectedKeys = this.colsPerRow; + s = new Scan(Bytes.toBytes("testRowTwo-0"), + Bytes.toBytes("testRowTwo-3")); + verifyScan(s, expectedRows, expectedKeys); + + // Now use start row with inclusive stop filter + expectedRows = this.numRows / 2; + s = new Scan(Bytes.toBytes("testRowTwo-0")); + s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowTwo-3"))); + verifyScan(s, expectedRows, expectedKeys); + + } + + public void testQualifierFilter() throws IOException { + + // Match two keys (one from each family) in half the rows + long expectedRows = this.numRows / 2; + long expectedKeys = 2; + Filter f = new QualifierFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + Scan s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys less than same qualifier + // Expect only two keys (one from each family) in half the rows + expectedRows = this.numRows / 2; + expectedKeys = 2; + f = new QualifierFilter(CompareOp.LESS, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys less than or equal + // Expect four keys (two from each family) in half the rows + expectedRows = this.numRows / 2; + expectedKeys = 4; + f = new QualifierFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys not equal + // Expect four keys (two from each family) + // Only look in first group of rows + expectedRows = this.numRows / 2; + expectedKeys = 4; + f = new QualifierFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(HConstants.EMPTY_START_ROW, Bytes.toBytes("testRowTwo")); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys greater or equal + // Expect four keys (two from each family) + // Only look in first group of rows + expectedRows = this.numRows / 2; + expectedKeys = 4; + f = new QualifierFilter(CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(HConstants.EMPTY_START_ROW, Bytes.toBytes("testRowTwo")); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys greater + // Expect two keys (one from each family) + // Only look in first group of rows + expectedRows = this.numRows / 2; + expectedKeys = 2; + f = new QualifierFilter(CompareOp.GREATER, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(HConstants.EMPTY_START_ROW, Bytes.toBytes("testRowTwo")); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys not equal to + // Look across rows and fully validate the keys and ordering + // Expect varied numbers of keys, 4 per row in group one, 6 per row in group two + f = new QualifierFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(QUALIFIERS_ONE[2])); + s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + + + // Test across rows and groups with a regex + // Filter out "test*-2" + // Expect 4 keys per row across both groups + f = new QualifierFilter(CompareOp.NOT_EQUAL, + new RegexStringComparator("test.+-2")); + s = new Scan(); + s.setFilter(f); + + kvs = new KeyValue [] { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + + } + + public void testFamilyFilter() throws IOException { + + // Match family, only half of columns returned. + long expectedRows = this.numRows; + long expectedKeys = this.colsPerRow / 2; + Filter f = new FamilyFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testFamilyOne"))); + Scan s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys less than given family, should return nothing + expectedRows = 0; + expectedKeys = 0; + f = new FamilyFilter(CompareOp.LESS, + new BinaryComparator(Bytes.toBytes("testFamily"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys less than or equal, should return half of columns + expectedRows = this.numRows; + expectedKeys = this.colsPerRow / 2; + f = new FamilyFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testFamilyOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys from second family + // look only in second group of rows + expectedRows = this.numRows / 2; + expectedKeys = this.colsPerRow / 2; + f = new FamilyFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testFamilyOne"))); + s = new Scan(HConstants.EMPTY_START_ROW, Bytes.toBytes("testRowTwo")); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match all columns + // look only in second group of rows + expectedRows = this.numRows / 2; + expectedKeys = this.colsPerRow; + f = new FamilyFilter(CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testFamilyOne"))); + s = new Scan(HConstants.EMPTY_START_ROW, Bytes.toBytes("testRowTwo")); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match all columns in second family + // look only in second group of rows + expectedRows = this.numRows / 2; + expectedKeys = this.colsPerRow / 2; + f = new FamilyFilter(CompareOp.GREATER, + new BinaryComparator(Bytes.toBytes("testFamilyOne"))); + s = new Scan(HConstants.EMPTY_START_ROW, Bytes.toBytes("testRowTwo")); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys not equal to given family + // Look across rows and fully validate the keys and ordering + f = new FamilyFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(FAMILIES[1])); + s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + + + // Test across rows and groups with a regex + // Filter out "test*-2" + // Expect 4 keys per row across both groups + f = new FamilyFilter(CompareOp.NOT_EQUAL, + new RegexStringComparator("test.*One")); + s = new Scan(); + s.setFilter(f); + + kvs = new KeyValue [] { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + + } + + + public void testRowFilter() throws IOException { + + // Match a single row, all keys + long expectedRows = 1; + long expectedKeys = this.colsPerRow; + Filter f = new RowFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + Scan s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match a two rows, one from each group, using regex + expectedRows = 2; + expectedKeys = this.colsPerRow; + f = new RowFilter(CompareOp.EQUAL, + new RegexStringComparator("testRow.+-2")); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match rows less than + // Expect all keys in one row + expectedRows = 1; + expectedKeys = this.colsPerRow; + f = new RowFilter(CompareOp.LESS, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match rows less than or equal + // Expect all keys in two rows + expectedRows = 2; + expectedKeys = this.colsPerRow; + f = new RowFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match rows not equal + // Expect all keys in all but one row + expectedRows = this.numRows - 1; + expectedKeys = this.colsPerRow; + f = new RowFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys greater or equal + // Expect all keys in all but one row + expectedRows = this.numRows - 1; + expectedKeys = this.colsPerRow; + f = new RowFilter(CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys greater + // Expect all keys in all but two rows + expectedRows = this.numRows - 2; + expectedKeys = this.colsPerRow; + f = new RowFilter(CompareOp.GREATER, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match rows not equal to testRowTwo-2 + // Look across rows and fully validate the keys and ordering + // Should see all keys in all rows but testRowTwo-2 + f = new RowFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + + + // Test across rows and groups with a regex + // Filter out everything that doesn't match "*-2" + // Expect all keys in two rows + f = new RowFilter(CompareOp.EQUAL, + new RegexStringComparator(".+-2")); + s = new Scan(); + s.setFilter(f); + + kvs = new KeyValue [] { + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) + }; + verifyScanFull(s, kvs); + + } + + public void testValueFilter() throws IOException { + + // Match group one rows + long expectedRows = this.numRows / 2; + long expectedKeys = this.colsPerRow; + Filter f = new ValueFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + Scan s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match group two rows + expectedRows = this.numRows / 2; + expectedKeys = this.colsPerRow; + f = new ValueFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testValueTwo"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match all values using regex + expectedRows = this.numRows; + expectedKeys = this.colsPerRow; + f = new ValueFilter(CompareOp.EQUAL, + new RegexStringComparator("testValue((One)|(Two))")); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values less than + // Expect group one rows + expectedRows = this.numRows / 2; + expectedKeys = this.colsPerRow; + f = new ValueFilter(CompareOp.LESS, + new BinaryComparator(Bytes.toBytes("testValueTwo"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values less than or equal + // Expect all rows + expectedRows = this.numRows; + expectedKeys = this.colsPerRow; + f = new ValueFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueTwo"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values less than or equal + // Expect group one rows + expectedRows = this.numRows / 2; + expectedKeys = this.colsPerRow; + f = new ValueFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values not equal + // Expect half the rows + expectedRows = this.numRows / 2; + expectedKeys = this.colsPerRow; + f = new ValueFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values greater or equal + // Expect all rows + expectedRows = this.numRows; + expectedKeys = this.colsPerRow; + f = new ValueFilter(CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values greater + // Expect half rows + expectedRows = this.numRows / 2; + expectedKeys = this.colsPerRow; + f = new ValueFilter(CompareOp.GREATER, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values not equal to testValueOne + // Look across rows and fully validate the keys and ordering + // Should see all keys in all group two rows + f = new ValueFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + } + + public void testSkipFilter() throws IOException { + + // Test for qualifier regex: "testQualifierOne-2" + // Should only get rows from second group, and all keys + Filter f = new SkipFilter(new QualifierFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2")))); + Scan s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + } + + // TODO: This is important... need many more tests for ordering, etc + // There are limited tests elsewhere but we need HRegion level ones here + public void testFilterList() throws IOException { + + // Test getting a single row, single key using Row, Qualifier, and Value + // regular expression and substring filters + // Use must pass all + List filters = new ArrayList(); + filters.add(new RowFilter(CompareOp.EQUAL, new RegexStringComparator(".+-2"))); + filters.add(new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator(".+-2"))); + filters.add(new ValueFilter(CompareOp.EQUAL, new SubstringComparator("One"))); + Filter f = new FilterList(Operator.MUST_PASS_ALL, filters); + Scan s = new Scan(); + s.addFamily(FAMILIES[0]); + s.setFilter(f); + KeyValue [] kvs = { + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]) + }; + verifyScanFull(s, kvs); + + // Test getting everything with a MUST_PASS_ONE filter including row, qf, val + // regular expression and substring filters + filters.clear(); + filters.add(new RowFilter(CompareOp.EQUAL, new RegexStringComparator(".+Two.+"))); + filters.add(new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator(".+-2"))); + filters.add(new ValueFilter(CompareOp.EQUAL, new SubstringComparator("One"))); + f = new FilterList(Operator.MUST_PASS_ONE, filters); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, this.numRows, this.colsPerRow); + + + } + + public void testFirstKeyOnlyFilter() throws IOException { + Scan s = new Scan(); + s.setFilter(new FirstKeyOnlyFilter()); + // Expected KVs, the first KV from each of the remaining 6 rows + KeyValue [] kvs = { + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]) + }; + verifyScanFull(s, kvs); + } + + public void testFilterListWithSingleColumnValueFilter() throws IOException { + // Test for HBASE-3191 + + // Scan using SingleColumnValueFilter + SingleColumnValueFilter f1 = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], + CompareOp.EQUAL, VALUES[0]); + f1.setFilterIfMissing( true ); + Scan s1 = new Scan(); + s1.addFamily(FAMILIES[0]); + s1.setFilter(f1); + KeyValue [] kvs1 = { + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + }; + verifyScanNoEarlyOut(s1, 3, 3); + verifyScanFull(s1, kvs1); + + // Scan using another SingleColumnValueFilter, expect disjoint result + SingleColumnValueFilter f2 = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_TWO[0], + CompareOp.EQUAL, VALUES[1]); + f2.setFilterIfMissing( true ); + Scan s2 = new Scan(); + s2.addFamily(FAMILIES[0]); + s2.setFilter(f2); + KeyValue [] kvs2 = { + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanNoEarlyOut(s2, 3, 3); + verifyScanFull(s2, kvs2); + + // Scan, ORing the two previous filters, expect unified result + FilterList f = new FilterList(Operator.MUST_PASS_ONE); + f.addFilter(f1); + f.addFilter(f2); + Scan s = new Scan(); + s.addFamily(FAMILIES[0]); + s.setFilter(f); + KeyValue [] kvs = { + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanNoEarlyOut(s, 6, 3); + verifyScanFull(s, kvs); + } + + public void testSingleColumnValueFilter() throws IOException { + + // From HBASE-1821 + // Desired action is to combine two SCVF in a FilterList + // Want to return only rows that match both conditions + + // Need to change one of the group one columns to use group two value + Put p = new Put(ROWS_ONE[2]); + p.add(FAMILIES[0], QUALIFIERS_ONE[2], VALUES[1]); + this.region.put(p); + + // Now let's grab rows that have Q_ONE[0](VALUES[0]) and Q_ONE[2](VALUES[1]) + // Since group two rows don't have these qualifiers, they will pass + // so limiting scan to group one + List filters = new ArrayList(); + filters.add(new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], + CompareOp.EQUAL, VALUES[0])); + filters.add(new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[2], + CompareOp.EQUAL, VALUES[1])); + Filter f = new FilterList(Operator.MUST_PASS_ALL, filters); + Scan s = new Scan(ROWS_ONE[0], ROWS_TWO[0]); + s.addFamily(FAMILIES[0]); + s.setFilter(f); + // Expect only one row, all qualifiers + KeyValue [] kvs = { + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[1]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]) + }; + verifyScanNoEarlyOut(s, 1, 3); + verifyScanFull(s, kvs); + + // In order to get expected behavior without limiting to group one + // need to wrap SCVFs in SkipFilters + filters = new ArrayList(); + filters.add(new SkipFilter(new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], + CompareOp.EQUAL, VALUES[0]))); + filters.add(new SkipFilter(new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[2], + CompareOp.EQUAL, VALUES[1]))); + f = new FilterList(Operator.MUST_PASS_ALL, filters); + s = new Scan(ROWS_ONE[0], ROWS_TWO[0]); + s.addFamily(FAMILIES[0]); + s.setFilter(f); + // Expect same KVs + verifyScanNoEarlyOut(s, 1, 3); + verifyScanFull(s, kvs); + + // More tests from HBASE-1821 for Clint and filterIfMissing flag + + byte [][] ROWS_THREE = { + Bytes.toBytes("rowThree-0"), Bytes.toBytes("rowThree-1"), + Bytes.toBytes("rowThree-2"), Bytes.toBytes("rowThree-3") + }; + + // Give row 0 and 2 QUALIFIERS_ONE[0] (VALUE[0] VALUE[1]) + // Give row 1 and 3 QUALIFIERS_ONE[1] (VALUE[0] VALUE[1]) + + KeyValue [] srcKVs = new KeyValue [] { + new KeyValue(ROWS_THREE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_THREE[1], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[1]), + new KeyValue(ROWS_THREE[2], FAMILIES[0], QUALIFIERS_ONE[1], VALUES[0]), + new KeyValue(ROWS_THREE[3], FAMILIES[0], QUALIFIERS_ONE[1], VALUES[1]) + }; + + for(KeyValue kv : srcKVs) { + Put put = new Put(kv.getRow()).add(kv); + put.setWriteToWAL(false); + this.region.put(put); + } + + // Match VALUES[0] against QUALIFIERS_ONE[0] with filterIfMissing = false + // Expect 3 rows (0, 2, 3) + SingleColumnValueFilter scvf = new SingleColumnValueFilter(FAMILIES[0], + QUALIFIERS_ONE[0], CompareOp.EQUAL, VALUES[0]); + s = new Scan(ROWS_THREE[0], Bytes.toBytes("rowThree-4")); + s.addFamily(FAMILIES[0]); + s.setFilter(scvf); + kvs = new KeyValue [] { srcKVs[0], srcKVs[2], srcKVs[3] }; + verifyScanFull(s, kvs); + + // Match VALUES[0] against QUALIFIERS_ONE[0] with filterIfMissing = true + // Expect 1 row (0) + scvf = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], + CompareOp.EQUAL, VALUES[0]); + scvf.setFilterIfMissing(true); + s = new Scan(ROWS_THREE[0], Bytes.toBytes("rowThree-4")); + s.addFamily(FAMILIES[0]); + s.setFilter(scvf); + kvs = new KeyValue [] { srcKVs[0] }; + verifyScanFull(s, kvs); + + // Match VALUES[1] against QUALIFIERS_ONE[1] with filterIfMissing = true + // Expect 1 row (3) + scvf = new SingleColumnValueFilter(FAMILIES[0], + QUALIFIERS_ONE[1], CompareOp.EQUAL, VALUES[1]); + scvf.setFilterIfMissing(true); + s = new Scan(ROWS_THREE[0], Bytes.toBytes("rowThree-4")); + s.addFamily(FAMILIES[0]); + s.setFilter(scvf); + kvs = new KeyValue [] { srcKVs[3] }; + verifyScanFull(s, kvs); + + // Add QUALIFIERS_ONE[1] to ROWS_THREE[0] with VALUES[0] + KeyValue kvA = new KeyValue(ROWS_THREE[0], FAMILIES[0], QUALIFIERS_ONE[1], VALUES[0]); + this.region.put(new Put(kvA.getRow()).add(kvA)); + + // Match VALUES[1] against QUALIFIERS_ONE[1] with filterIfMissing = true + // Expect 1 row (3) + scvf = new SingleColumnValueFilter(FAMILIES[0], + QUALIFIERS_ONE[1], CompareOp.EQUAL, VALUES[1]); + scvf.setFilterIfMissing(true); + s = new Scan(ROWS_THREE[0], Bytes.toBytes("rowThree-4")); + s.addFamily(FAMILIES[0]); + s.setFilter(scvf); + kvs = new KeyValue [] { srcKVs[3] }; + verifyScanFull(s, kvs); + + } + + private void verifyScan(Scan s, long expectedRows, long expectedKeys) + throws IOException { + InternalScanner scanner = this.region.getScanner(s); + List results = new ArrayList(); + int i = 0; + for (boolean done = true; done; i++) { + done = scanner.next(results); + Arrays.sort(results.toArray(new KeyValue[results.size()]), + KeyValue.COMPARATOR); + LOG.info("counter=" + i + ", " + results); + if (results.isEmpty()) break; + assertTrue("Scanned too many rows! Only expected " + expectedRows + + " total but already scanned " + (i+1), expectedRows > i); + assertEquals("Expected " + expectedKeys + " keys per row but " + + "returned " + results.size(), expectedKeys, results.size()); + results.clear(); + } + assertEquals("Expected " + expectedRows + " rows but scanned " + i + + " rows", expectedRows, i); + } + + private void verifyScanNoEarlyOut(Scan s, long expectedRows, + long expectedKeys) + throws IOException { + InternalScanner scanner = this.region.getScanner(s); + List results = new ArrayList(); + int i = 0; + for (boolean done = true; done; i++) { + done = scanner.next(results); + Arrays.sort(results.toArray(new KeyValue[results.size()]), + KeyValue.COMPARATOR); + LOG.info("counter=" + i + ", " + results); + if(results.isEmpty()) break; + assertTrue("Scanned too many rows! Only expected " + expectedRows + + " total but already scanned " + (i+1), expectedRows > i); + assertEquals("Expected " + expectedKeys + " keys per row but " + + "returned " + results.size(), expectedKeys, results.size()); + results.clear(); + } + assertEquals("Expected " + expectedRows + " rows but scanned " + i + + " rows", expectedRows, i); + } + + private void verifyScanFull(Scan s, KeyValue [] kvs) + throws IOException { + InternalScanner scanner = this.region.getScanner(s); + List results = new ArrayList(); + int row = 0; + int idx = 0; + for (boolean done = true; done; row++) { + done = scanner.next(results); + Arrays.sort(results.toArray(new KeyValue[results.size()]), + KeyValue.COMPARATOR); + if(results.isEmpty()) break; + assertTrue("Scanned too many keys! Only expected " + kvs.length + + " total but already scanned " + (results.size() + idx) + + (results.isEmpty() ? "" : "(" + results.get(0).toString() + ")"), + kvs.length >= idx + results.size()); + for(KeyValue kv : results) { + LOG.info("row=" + row + ", result=" + kv.toString() + + ", match=" + kvs[idx].toString()); + assertTrue("Row mismatch", + Bytes.equals(kv.getRow(), kvs[idx].getRow())); + assertTrue("Family mismatch", + Bytes.equals(kv.getFamily(), kvs[idx].getFamily())); + assertTrue("Qualifier mismatch", + Bytes.equals(kv.getQualifier(), kvs[idx].getQualifier())); + assertTrue("Value mismatch", + Bytes.equals(kv.getValue(), kvs[idx].getValue())); + idx++; + } + results.clear(); + } + LOG.info("Looked at " + row + " rows with " + idx + " keys"); + assertEquals("Expected " + kvs.length + " total keys but scanned " + idx, + kvs.length, idx); + } + + private void verifyScanFullNoValues(Scan s, KeyValue [] kvs, boolean useLen) + throws IOException { + InternalScanner scanner = this.region.getScanner(s); + List results = new ArrayList(); + int row = 0; + int idx = 0; + for (boolean more = true; more; row++) { + more = scanner.next(results); + Arrays.sort(results.toArray(new KeyValue[results.size()]), + KeyValue.COMPARATOR); + if(results.isEmpty()) break; + assertTrue("Scanned too many keys! Only expected " + kvs.length + + " total but already scanned " + (results.size() + idx) + + (results.isEmpty() ? "" : "(" + results.get(0).toString() + ")"), + kvs.length >= idx + results.size()); + for(KeyValue kv : results) { + LOG.info("row=" + row + ", result=" + kv.toString() + + ", match=" + kvs[idx].toString()); + assertTrue("Row mismatch", + Bytes.equals(kv.getRow(), kvs[idx].getRow())); + assertTrue("Family mismatch", + Bytes.equals(kv.getFamily(), kvs[idx].getFamily())); + assertTrue("Qualifier mismatch", + Bytes.equals(kv.getQualifier(), kvs[idx].getQualifier())); + assertFalse("Should not have returned whole value", + Bytes.equals(kv.getValue(), kvs[idx].getValue())); + if (useLen) { + assertEquals("Value in result is not SIZEOF_INT", + kv.getValue().length, Bytes.SIZEOF_INT); + LOG.info("idx = " + idx + ", len=" + kvs[idx].getValueLength() + + ", actual=" + Bytes.toInt(kv.getValue())); + assertEquals("Scan value should be the length of the actual value. ", + kvs[idx].getValueLength(), Bytes.toInt(kv.getValue()) ); + LOG.info("good"); + } else { + assertEquals("Value in result is not empty", + kv.getValue().length, 0); + } + idx++; + } + results.clear(); + } + LOG.info("Looked at " + row + " rows with " + idx + " keys"); + assertEquals("Expected " + kvs.length + " total keys but scanned " + idx, + kvs.length, idx); + } + + + public void testColumnPaginationFilter() throws Exception { + // Test that the filter skips multiple column versions. + Put p = new Put(ROWS_ONE[0]); + p.setWriteToWAL(false); + p.add(FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]); + this.region.put(p); + this.region.flushcache(); + + // Set of KVs (page: 1; pageSize: 1) - the first set of 1 column per row + KeyValue [] expectedKVs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]) + }; + + + // Set of KVs (page: 3; pageSize: 1) - the third set of 1 column per row + KeyValue [] expectedKVs2 = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + }; + + // Set of KVs (page: 2; pageSize 2) - the 2nd set of 2 columns per row + KeyValue [] expectedKVs3 = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + }; + + + // Set of KVs (page: 2; pageSize 2) - the 2nd set of 2 columns per row + KeyValue [] expectedKVs4 = { + + }; + + long expectedRows = this.numRows; + long expectedKeys = 1; + Scan s = new Scan(); + + + // Page 1; 1 Column per page (Limit 1, Offset 0) + s.setFilter(new ColumnPaginationFilter(1,0)); + verifyScan(s, expectedRows, expectedKeys); + this.verifyScanFull(s, expectedKVs); + + // Page 3; 1 Result per page (Limit 1, Offset 2) + s.setFilter(new ColumnPaginationFilter(1,2)); + verifyScan(s, expectedRows, expectedKeys); + this.verifyScanFull(s, expectedKVs2); + + // Page 2; 2 Results per page (Limit 2, Offset 2) + s.setFilter(new ColumnPaginationFilter(2,2)); + expectedKeys = 2; + verifyScan(s, expectedRows, expectedKeys); + this.verifyScanFull(s, expectedKVs3); + + // Page 8; 20 Results per page (no results) (Limit 20, Offset 140) + s.setFilter(new ColumnPaginationFilter(20,140)); + expectedKeys = 0; + expectedRows = 0; + verifyScan(s, expectedRows, 0); + this.verifyScanFull(s, expectedKVs4); + } + + public void testKeyOnlyFilter() throws Exception { + + // KVs in first 6 rows + KeyValue [] expectedKVs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) + }; + + // Grab all 6 rows + long expectedRows = 6; + long expectedKeys = this.colsPerRow; + for (boolean useLen : new boolean[]{false,true}) { + Scan s = new Scan(); + s.setFilter(new KeyOnlyFilter(useLen)); + verifyScan(s, expectedRows, expectedKeys); + verifyScanFullNoValues(s, expectedKVs, useLen); + } + } + + /** + * Filter which makes sleeps for a second between each row of a scan. + * This can be useful for manual testing of bugs like HBASE-5973. For example: + * + * create 't1', 'f1' + * 1.upto(100) { |x| put 't1', 'r' + x.to_s, 'f1:q1', 'hi' } + * import org.apache.hadoop.hbase.filter.TestFilter + * scan 't1', { FILTER => TestFilter::SlowScanFilter.new(), CACHE => 50 } + * + */ + public static class SlowScanFilter extends FilterBase { + private static Thread ipcHandlerThread = null; + + @Override + public void readFields(DataInput arg0) throws IOException { + } + + @Override + public void write(DataOutput arg0) throws IOException { + } + + @Override + public boolean filterRow() { + ipcHandlerThread = Thread.currentThread(); + try { + LOG.info("Handler thread " + ipcHandlerThread + " sleeping in filter..."); + Thread.sleep(1000); + } catch (InterruptedException e) { + Throwables.propagate(e); + } + return super.filterRow(); + } + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java new file mode 100644 index 0000000..b3a1596 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java @@ -0,0 +1,403 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +/** + * Tests filter sets + * + */ +@Category(SmallTests.class) +public class TestFilterList extends TestCase { + static final int MAX_PAGES = 2; + static final char FIRST_CHAR = 'a'; + static final char LAST_CHAR = 'e'; + static byte[] GOOD_BYTES = Bytes.toBytes("abc"); + static byte[] BAD_BYTES = Bytes.toBytes("def"); + + /** + * Test "must pass one" + * @throws Exception + */ + public void testMPONE() throws Exception { + List filters = new ArrayList(); + filters.add(new PageFilter(MAX_PAGES)); + filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy")))); + Filter filterMPONE = + new FilterList(FilterList.Operator.MUST_PASS_ONE, filters); + /* Filter must do all below steps: + *

      + *
    • {@link #reset()}
      • + *
    • {@link #filterAllRemaining()} -> true indicates scan is over, false, keep going on.
      • + *
    • {@link #filterRowKey(byte[],int,int)} -> true to drop this row, + * if false, we will also call
      • + *
    • {@link #filterKeyValue(org.apache.hadoop.hbase.KeyValue)} -> true to drop this key/value
      • + *
    • {@link #filterRow()} -> last chance to drop entire row based on the sequence of + * filterValue() calls. Eg: filter a row if it doesn't contain a specified column. + *
      • + *
    + */ + filterMPONE.reset(); + assertFalse(filterMPONE.filterAllRemaining()); + + /* Will pass both */ + byte [] rowkey = Bytes.toBytes("yyyyyyyyy"); + for (int i = 0; i < MAX_PAGES - 1; i++) { + assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); + assertFalse(filterMPONE.filterRow()); + KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), + Bytes.toBytes(i)); + assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); + } + + /* Only pass PageFilter */ + rowkey = Bytes.toBytes("z"); + assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); + assertFalse(filterMPONE.filterRow()); + KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0), + Bytes.toBytes(0)); + assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); + + /* PageFilter will fail now, but should pass because we match yyy */ + rowkey = Bytes.toBytes("yyy"); + assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); + assertFalse(filterMPONE.filterRow()); + kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0), + Bytes.toBytes(0)); + assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); + + /* We should filter any row */ + rowkey = Bytes.toBytes("z"); + assertTrue(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); + assertTrue(filterMPONE.filterAllRemaining()); + + } + + /** + * Test "must pass all" + * @throws Exception + */ + public void testMPALL() throws Exception { + List filters = new ArrayList(); + filters.add(new PageFilter(MAX_PAGES)); + filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy")))); + Filter filterMPALL = + new FilterList(FilterList.Operator.MUST_PASS_ALL, filters); + /* Filter must do all below steps: + *
      + *
    • {@link #reset()}
      • + *
    • {@link #filterAllRemaining()} -> true indicates scan is over, false, keep going on.
      • + *
    • {@link #filterRowKey(byte[],int,int)} -> true to drop this row, + * if false, we will also call
      • + *
    • {@link #filterKeyValue(org.apache.hadoop.hbase.KeyValue)} -> true to drop this key/value
      • + *
    • {@link #filterRow()} -> last chance to drop entire row based on the sequence of + * filterValue() calls. Eg: filter a row if it doesn't contain a specified column. + *
      • + *
    + */ + filterMPALL.reset(); + assertFalse(filterMPALL.filterAllRemaining()); + byte [] rowkey = Bytes.toBytes("yyyyyyyyy"); + for (int i = 0; i < MAX_PAGES - 1; i++) { + assertFalse(filterMPALL.filterRowKey(rowkey, 0, rowkey.length)); + KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), + Bytes.toBytes(i)); + assertTrue(Filter.ReturnCode.INCLUDE == filterMPALL.filterKeyValue(kv)); + } + filterMPALL.reset(); + rowkey = Bytes.toBytes("z"); + assertTrue(filterMPALL.filterRowKey(rowkey, 0, rowkey.length)); + // Should fail here; row should be filtered out. + KeyValue kv = new KeyValue(rowkey, rowkey, rowkey, rowkey); + assertTrue(Filter.ReturnCode.NEXT_ROW == filterMPALL.filterKeyValue(kv)); + } + + /** + * Test list ordering + * @throws Exception + */ + public void testOrdering() throws Exception { + List filters = new ArrayList(); + filters.add(new PrefixFilter(Bytes.toBytes("yyy"))); + filters.add(new PageFilter(MAX_PAGES)); + Filter filterMPONE = + new FilterList(FilterList.Operator.MUST_PASS_ONE, filters); + /* Filter must do all below steps: + *
      + *
    • {@link #reset()}
      • + *
    • {@link #filterAllRemaining()} -> true indicates scan is over, false, keep going on.
      • + *
    • {@link #filterRowKey(byte[],int,int)} -> true to drop this row, + * if false, we will also call
      • + *
    • {@link #filterKeyValue(org.apache.hadoop.hbase.KeyValue)} -> true to drop this key/value
      • + *
    • {@link #filterRow()} -> last chance to drop entire row based on the sequence of + * filterValue() calls. Eg: filter a row if it doesn't contain a specified column. + *
      • + *
    + */ + filterMPONE.reset(); + assertFalse(filterMPONE.filterAllRemaining()); + + /* We should be able to fill MAX_PAGES without incrementing page counter */ + byte [] rowkey = Bytes.toBytes("yyyyyyyy"); + for (int i = 0; i < MAX_PAGES; i++) { + assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); + KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), + Bytes.toBytes(i)); + assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); + assertFalse(filterMPONE.filterRow()); + } + + /* Now let's fill the page filter */ + rowkey = Bytes.toBytes("xxxxxxx"); + for (int i = 0; i < MAX_PAGES; i++) { + assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); + KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), + Bytes.toBytes(i)); + assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); + assertFalse(filterMPONE.filterRow()); + } + + /* We should still be able to include even though page filter is at max */ + rowkey = Bytes.toBytes("yyy"); + for (int i = 0; i < MAX_PAGES; i++) { + assertFalse(filterMPONE.filterRowKey(rowkey, 0, rowkey.length)); + KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), + Bytes.toBytes(i)); + assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterKeyValue(kv)); + assertFalse(filterMPONE.filterRow()); + } + } + + /** + * Test serialization + * @throws Exception + */ + public void testSerialization() throws Exception { + List filters = new ArrayList(); + filters.add(new PageFilter(MAX_PAGES)); + filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy")))); + Filter filterMPALL = + new FilterList(FilterList.Operator.MUST_PASS_ALL, filters); + + // Decompose filterMPALL to bytes. + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(stream); + filterMPALL.write(out); + out.close(); + byte[] buffer = stream.toByteArray(); + + // Recompose filterMPALL. + DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer)); + FilterList newFilter = new FilterList(); + newFilter.readFields(in); + + // TODO: Run TESTS!!! + } + + /** + * Test filterKeyValue logic. + * @throws Exception + */ + public void testFilterKeyValue() throws Exception { + Filter includeFilter = new FilterBase() { + @Override + public Filter.ReturnCode filterKeyValue(KeyValue v) { + return Filter.ReturnCode.INCLUDE; + } + + @Override + public void readFields(DataInput arg0) throws IOException {} + + @Override + public void write(DataOutput arg0) throws IOException {} + }; + + Filter alternateFilter = new FilterBase() { + boolean returnInclude = true; + + @Override + public Filter.ReturnCode filterKeyValue(KeyValue v) { + Filter.ReturnCode returnCode = returnInclude ? Filter.ReturnCode.INCLUDE : + Filter.ReturnCode.SKIP; + returnInclude = !returnInclude; + return returnCode; + } + + @Override + public void readFields(DataInput arg0) throws IOException {} + + @Override + public void write(DataOutput arg0) throws IOException {} + }; + + Filter alternateIncludeFilter = new FilterBase() { + boolean returnIncludeOnly = false; + + @Override + public Filter.ReturnCode filterKeyValue(KeyValue v) { + Filter.ReturnCode returnCode = returnIncludeOnly ? Filter.ReturnCode.INCLUDE : + Filter.ReturnCode.INCLUDE_AND_NEXT_COL; + returnIncludeOnly = !returnIncludeOnly; + return returnCode; + } + + @Override + public void readFields(DataInput arg0) throws IOException {} + + @Override + public void write(DataOutput arg0) throws IOException {} + }; + + // Check must pass one filter. + FilterList mpOnefilterList = new FilterList(Operator.MUST_PASS_ONE, + Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter })); + // INCLUDE, INCLUDE, INCLUDE_AND_NEXT_COL. + assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpOnefilterList.filterKeyValue(null)); + // INCLUDE, SKIP, INCLUDE. + assertEquals(Filter.ReturnCode.INCLUDE, mpOnefilterList.filterKeyValue(null)); + + // Check must pass all filter. + FilterList mpAllfilterList = new FilterList(Operator.MUST_PASS_ALL, + Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter })); + // INCLUDE, INCLUDE, INCLUDE_AND_NEXT_COL. + assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpAllfilterList.filterKeyValue(null)); + // INCLUDE, SKIP, INCLUDE. + assertEquals(Filter.ReturnCode.SKIP, mpAllfilterList.filterKeyValue(null)); + } + + /** + * Test pass-thru of hints. + */ + public void testHintPassThru() throws Exception { + + final KeyValue minKeyValue = new KeyValue(Bytes.toBytes(0L), null, null); + final KeyValue maxKeyValue = new KeyValue(Bytes.toBytes(Long.MAX_VALUE), + null, null); + + Filter filterNoHint = new FilterBase() { + @Override + public void readFields(DataInput arg0) throws IOException {} + + @Override + public void write(DataOutput arg0) throws IOException {} + }; + + Filter filterMinHint = new FilterBase() { + @Override + public KeyValue getNextKeyHint(KeyValue currentKV) { + return minKeyValue; + } + + @Override + public void readFields(DataInput arg0) throws IOException {} + + @Override + public void write(DataOutput arg0) throws IOException {} + }; + + Filter filterMaxHint = new FilterBase() { + @Override + public KeyValue getNextKeyHint(KeyValue currentKV) { + return new KeyValue(Bytes.toBytes(Long.MAX_VALUE), null, null); + } + + @Override + public void readFields(DataInput arg0) throws IOException {} + + @Override + public void write(DataOutput arg0) throws IOException {} + }; + + // MUST PASS ONE + + // Should take the min if given two hints + FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, + Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } )); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), + minKeyValue)); + + // Should have no hint if any filter has no hint + filterList = new FilterList(Operator.MUST_PASS_ONE, + Arrays.asList( + new Filter [] { filterMinHint, filterMaxHint, filterNoHint } )); + assertNull(filterList.getNextKeyHint(null)); + filterList = new FilterList(Operator.MUST_PASS_ONE, + Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } )); + assertNull(filterList.getNextKeyHint(null)); + + // Should give max hint if its the only one + filterList = new FilterList(Operator.MUST_PASS_ONE, + Arrays.asList(new Filter [] { filterMaxHint, filterMaxHint } )); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), + maxKeyValue)); + + // MUST PASS ALL + + // Should take the max if given two hints + filterList = new FilterList(Operator.MUST_PASS_ALL, + Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } )); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), + maxKeyValue)); + + // Should have max hint even if a filter has no hint + filterList = new FilterList(Operator.MUST_PASS_ALL, + Arrays.asList( + new Filter [] { filterMinHint, filterMaxHint, filterNoHint } )); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), + maxKeyValue)); + filterList = new FilterList(Operator.MUST_PASS_ALL, + Arrays.asList(new Filter [] { filterNoHint, filterMaxHint } )); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), + maxKeyValue)); + filterList = new FilterList(Operator.MUST_PASS_ALL, + Arrays.asList(new Filter [] { filterNoHint, filterMinHint } )); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), + minKeyValue)); + + // Should give min hint if its the only one + filterList = new FilterList(Operator.MUST_PASS_ALL, + Arrays.asList(new Filter [] { filterNoHint, filterMinHint } )); + assertEquals(0, KeyValue.COMPARATOR.compare(filterList.getNextKeyHint(null), + minKeyValue)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java new file mode 100644 index 0000000..4faca82 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java @@ -0,0 +1,204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestFuzzyRowFilter { + @Test + public void testSatisfies() { + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, + FuzzyRowFilter.satisfies(new byte[]{1, (byte) -128, 0, 0, 1}, // row to check + new byte[]{1, 0, 1}, // fuzzy row + new byte[]{0, 1, 0})); // mask + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES, + FuzzyRowFilter.satisfies(new byte[]{1, (byte) -128, 1, 0, 1}, + new byte[]{1, 0, 1}, + new byte[]{0, 1, 0})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, + FuzzyRowFilter.satisfies(new byte[]{1, (byte) -128, 2, 0, 1}, + new byte[]{1, 0, 1}, + new byte[]{0, 1, 0})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT, + FuzzyRowFilter.satisfies(new byte[]{2, 3, 1, 1, 1}, + new byte[]{1, 0, 1}, + new byte[]{0, 1, 0})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES, + FuzzyRowFilter.satisfies(new byte[]{1, 2, 1, 3, 3}, + new byte[]{1, 2, 0, 3}, + new byte[]{0, 0, 1, 0})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, + FuzzyRowFilter.satisfies(new byte[]{1, 1, 1, 3, 0}, // row to check + new byte[]{1, 2, 0, 3}, // fuzzy row + new byte[]{0, 0, 1, 0})); // mask + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, + FuzzyRowFilter.satisfies(new byte[]{1, 1, 1, 3, 0}, + new byte[]{1, (byte) 245, 0, 3}, + new byte[]{0, 0, 1, 0})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT, + FuzzyRowFilter.satisfies(new byte[]{1, (byte) 245, 1, 3, 0}, + new byte[]{1, 1, 0, 3}, + new byte[]{0, 0, 1, 0})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT, + FuzzyRowFilter.satisfies(new byte[]{1, 3, 1, 3, 0}, + new byte[]{1, 2, 0, 3}, + new byte[]{0, 0, 1, 0})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT, + FuzzyRowFilter.satisfies(new byte[]{2, 1, 1, 1, 0}, + new byte[]{1, 2, 0, 3}, + new byte[]{0, 0, 1, 0})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, + FuzzyRowFilter.satisfies(new byte[]{1, 2, 1, 0, 1}, + new byte[]{0, 1, 2}, + new byte[]{1, 0, 0})); + } + + @Test + public void testGetNextForFuzzyRule() { + assertNext( + new byte[]{0, 1, 2}, // fuzzy row + new byte[]{1, 0, 0}, // mask + new byte[]{1, 2, 1, 0, 1}, // current + new byte[]{2, 1, 2, 0, 0}); // expected next + + assertNext( + new byte[]{0, 1, 2}, // fuzzy row + new byte[]{1, 0, 0}, // mask + new byte[]{1, 1, 2, 0, 1}, // current + new byte[]{1, 1, 2, 0, 2}); // expected next + + assertNext( + new byte[]{0, 1, 0, 2, 0}, // fuzzy row + new byte[]{1, 0, 1, 0, 1}, // mask + new byte[]{1, 0, 2, 0, 1}, // current + new byte[]{1, 1, 0, 2, 0}); // expected next + + assertNext( + new byte[]{1, 0, 1}, + new byte[]{0, 1, 0}, + new byte[]{1, (byte) 128, 2, 0, 1}, + new byte[]{1, (byte) 129, 1, 0, 0}); + + assertNext( + new byte[]{0, 1, 0, 1}, + new byte[]{1, 0, 1, 0}, + new byte[]{5, 1, 0, 1}, + new byte[]{5, 1, 1, 1}); + + assertNext( + new byte[]{0, 1, 0, 1}, + new byte[]{1, 0, 1, 0}, + new byte[]{5, 1, 0, 1, 1}, + new byte[]{5, 1, 0, 1, 2}); + + assertNext( + new byte[]{0, 1, 0, 0}, // fuzzy row + new byte[]{1, 0, 1, 1}, // mask + new byte[]{5, 1, (byte) 255, 1}, // current + new byte[]{5, 1, (byte) 255, 2}); // expected next + + assertNext( + new byte[]{0, 1, 0, 1}, // fuzzy row + new byte[]{1, 0, 1, 0}, // mask + new byte[]{5, 1, (byte) 255, 1}, // current + new byte[]{6, 1, 0, 1}); // expected next + + assertNext( + new byte[]{0, 1, 0, 1}, // fuzzy row + new byte[]{1, 0, 1, 0}, // mask + new byte[]{5, 1, (byte) 255, 0}, // current + new byte[]{5, 1, (byte) 255, 1}); // expected next + + assertNext( + new byte[]{5, 1, 1, 0}, + new byte[]{0, 0, 1, 1}, + new byte[]{5, 1, (byte) 255, 1}, + new byte[]{5, 1, (byte) 255, 2}); + + assertNext( + new byte[]{1, 1, 1, 1}, + new byte[]{0, 0, 1, 1}, + new byte[]{1, 1, 2, 2}, + new byte[]{1, 1, 2, 3}); + + assertNext( + new byte[]{1, 1, 1, 1}, + new byte[]{0, 0, 1, 1}, + new byte[]{1, 1, 3, 2}, + new byte[]{1, 1, 3, 3}); + + assertNext( + new byte[]{1, 1, 1, 1}, + new byte[]{1, 1, 1, 1}, + new byte[]{1, 1, 2, 3}, + new byte[]{1, 1, 2, 4}); + + assertNext( + new byte[]{1, 1, 1, 1}, + new byte[]{1, 1, 1, 1}, + new byte[]{1, 1, 3, 2}, + new byte[]{1, 1, 3, 3}); + + assertNext( + new byte[]{1, 1, 0, 0}, + new byte[]{0, 0, 1, 1}, + new byte[]{0, 1, 3, 2}, + new byte[]{1, 1, 0, 0}); + + // No next for this one + Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule( + new byte[]{2, 3, 1, 1, 1}, // row to check + new byte[]{1, 0, 1}, // fuzzy row + new byte[]{0, 1, 0})); // mask + Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule( + new byte[]{1, (byte) 245, 1, 3, 0}, + new byte[]{1, 1, 0, 3}, + new byte[]{0, 0, 1, 0})); + Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule( + new byte[]{1, 3, 1, 3, 0}, + new byte[]{1, 2, 0, 3}, + new byte[]{0, 0, 1, 0})); + Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule( + new byte[]{2, 1, 1, 1, 0}, + new byte[]{1, 2, 0, 3}, + new byte[]{0, 0, 1, 0})); + } + + private void assertNext(byte[] fuzzyRow, byte[] mask, byte[] current, byte[] expected) { + byte[] nextForFuzzyRule = FuzzyRowFilter.getNextForFuzzyRule(current, fuzzyRow, mask); + Assert.assertArrayEquals(expected, nextForFuzzyRule); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestInclusiveStopFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestInclusiveStopFilter.java new file mode 100644 index 0000000..a12bd8e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestInclusiveStopFilter.java @@ -0,0 +1,97 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +/** + * Tests the inclusive stop row filter + */ +@Category(SmallTests.class) +public class TestInclusiveStopFilter extends TestCase { + private final byte [] STOP_ROW = Bytes.toBytes("stop_row"); + private final byte [] GOOD_ROW = Bytes.toBytes("good_row"); + private final byte [] PAST_STOP_ROW = Bytes.toBytes("zzzzzz"); + + Filter mainFilter; + + @Override + protected void setUp() throws Exception { + super.setUp(); + mainFilter = new InclusiveStopFilter(STOP_ROW); + } + + /** + * Tests identification of the stop row + * @throws Exception + */ + public void testStopRowIdentification() throws Exception { + stopRowTests(mainFilter); + } + + /** + * Tests serialization + * @throws Exception + */ + public void testSerialization() throws Exception { + // Decompose mainFilter to bytes. + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(stream); + mainFilter.write(out); + out.close(); + byte[] buffer = stream.toByteArray(); + + // Recompose mainFilter. + DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer)); + Filter newFilter = new InclusiveStopFilter(); + newFilter.readFields(in); + + // Ensure the serialization preserved the filter by running a full test. + stopRowTests(newFilter); + } + + private void stopRowTests(Filter filter) throws Exception { + assertFalse("Filtering on " + Bytes.toString(GOOD_ROW), + filter.filterRowKey(GOOD_ROW, 0, GOOD_ROW.length)); + assertFalse("Filtering on " + Bytes.toString(STOP_ROW), + filter.filterRowKey(STOP_ROW, 0, STOP_ROW.length)); + assertTrue("Filtering on " + Bytes.toString(PAST_STOP_ROW), + filter.filterRowKey(PAST_STOP_ROW, 0, PAST_STOP_ROW.length)); + + assertTrue("FilterAllRemaining", filter.filterAllRemaining()); + assertFalse("FilterNotNull", filter.filterRow()); + + assertFalse("Filter a null", filter.filterRowKey(null, 0, 0)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestMultipleColumnPrefixFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestMultipleColumnPrefixFilter.java new file mode 100644 index 0000000..cd555bc --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestMultipleColumnPrefixFilter.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestMultipleColumnPrefixFilter { + + private final static HBaseTestingUtility TEST_UTIL = new + HBaseTestingUtility(); + + @Test + public void testMultipleColumnPrefixFilter() throws IOException { + String family = "Family"; + HTableDescriptor htd = new HTableDescriptor("TestMultipleColumnPrefixFilter"); + htd.addFamily(new HColumnDescriptor(family)); + // HRegionInfo info = new HRegionInfo(htd, null, null, false); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + + List rows = generateRandomWords(100, "row"); + List columns = generateRandomWords(10000, "column"); + long maxTimestamp = 2; + + List kvList = new ArrayList(); + + Map> prefixMap = new HashMap>(); + + prefixMap.put("p", new ArrayList()); + prefixMap.put("q", new ArrayList()); + prefixMap.put("s", new ArrayList()); + + String valueString = "ValueString"; + + for (String row: rows) { + Put p = new Put(Bytes.toBytes(row)); + p.setWriteToWAL(false); + for (String column: columns) { + for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { + KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, + valueString); + p.add(kv); + kvList.add(kv); + for (String s: prefixMap.keySet()) { + if (column.startsWith(s)) { + prefixMap.get(s).add(kv); + } + } + } + } + region.put(p); + } + + MultipleColumnPrefixFilter filter; + Scan scan = new Scan(); + scan.setMaxVersions(); + byte [][] filter_prefix = new byte [2][]; + filter_prefix[0] = new byte [] {'p'}; + filter_prefix[1] = new byte [] {'q'}; + + filter = new MultipleColumnPrefixFilter(filter_prefix); + scan.setFilter(filter); + List results = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + while(scanner.next(results)); + assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size()); + + region.close(); + region.getLog().closeAndDelete(); + } + + @Test + public void testMultipleColumnPrefixFilterWithManyFamilies() throws IOException { + String family1 = "Family1"; + String family2 = "Family2"; + HTableDescriptor htd = new HTableDescriptor("TestMultipleColumnPrefixFilter"); + htd.addFamily(new HColumnDescriptor(family1)); + htd.addFamily(new HColumnDescriptor(family2)); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + + List rows = generateRandomWords(100, "row"); + List columns = generateRandomWords(10000, "column"); + long maxTimestamp = 3; + + List kvList = new ArrayList(); + + Map> prefixMap = new HashMap>(); + + prefixMap.put("p", new ArrayList()); + prefixMap.put("q", new ArrayList()); + prefixMap.put("s", new ArrayList()); + + String valueString = "ValueString"; + + for (String row: rows) { + Put p = new Put(Bytes.toBytes(row)); + p.setWriteToWAL(false); + for (String column: columns) { + for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { + double rand = Math.random(); + KeyValue kv; + if (rand < 0.5) + kv = KeyValueTestUtil.create(row, family1, column, timestamp, + valueString); + else + kv = KeyValueTestUtil.create(row, family2, column, timestamp, + valueString); + p.add(kv); + kvList.add(kv); + for (String s: prefixMap.keySet()) { + if (column.startsWith(s)) { + prefixMap.get(s).add(kv); + } + } + } + } + region.put(p); + } + + MultipleColumnPrefixFilter filter; + Scan scan = new Scan(); + scan.setMaxVersions(); + byte [][] filter_prefix = new byte [2][]; + filter_prefix[0] = new byte [] {'p'}; + filter_prefix[1] = new byte [] {'q'}; + + filter = new MultipleColumnPrefixFilter(filter_prefix); + scan.setFilter(filter); + List results = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + while(scanner.next(results)); + assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size()); + + region.close(); + region.getLog().closeAndDelete(); + } + + @Test + public void testMultipleColumnPrefixFilterWithColumnPrefixFilter() throws IOException { + String family = "Family"; + HTableDescriptor htd = new HTableDescriptor("TestMultipleColumnPrefixFilter"); + htd.addFamily(new HColumnDescriptor(family)); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(),htd); + + List rows = generateRandomWords(100, "row"); + List columns = generateRandomWords(10000, "column"); + long maxTimestamp = 2; + + String valueString = "ValueString"; + + for (String row: rows) { + Put p = new Put(Bytes.toBytes(row)); + p.setWriteToWAL(false); + for (String column: columns) { + for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { + KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, + valueString); + p.add(kv); + } + } + region.put(p); + } + + MultipleColumnPrefixFilter multiplePrefixFilter; + Scan scan1 = new Scan(); + scan1.setMaxVersions(); + byte [][] filter_prefix = new byte [1][]; + filter_prefix[0] = new byte [] {'p'}; + + multiplePrefixFilter = new MultipleColumnPrefixFilter(filter_prefix); + scan1.setFilter(multiplePrefixFilter); + List results1 = new ArrayList(); + InternalScanner scanner1 = region.getScanner(scan1); + while(scanner1.next(results1)); + + ColumnPrefixFilter singlePrefixFilter; + Scan scan2 = new Scan(); + scan2.setMaxVersions(); + singlePrefixFilter = new ColumnPrefixFilter(Bytes.toBytes("p")); + + scan2.setFilter(singlePrefixFilter); + List results2 = new ArrayList(); + InternalScanner scanner2 = region.getScanner(scan1); + while(scanner2.next(results2)); + + assertEquals(results1.size(), results2.size()); + + region.close(); + region.getLog().closeAndDelete(); + } + + List generateRandomWords(int numberOfWords, String suffix) { + Set wordSet = new HashSet(); + for (int i = 0; i < numberOfWords; i++) { + int lengthOfWords = (int) (Math.random()*2) + 1; + char[] wordChar = new char[lengthOfWords]; + for (int j = 0; j < wordChar.length; j++) { + wordChar[j] = (char) (Math.random() * 26 + 97); + } + String word; + if (suffix == null) { + word = new String(wordChar); + } else { + word = new String(wordChar) + suffix; + } + wordSet.add(word); + } + List wordList = new ArrayList(wordSet); + return wordList; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestPageFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestPageFilter.java new file mode 100644 index 0000000..ee9a961 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestPageFilter.java @@ -0,0 +1,98 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.experimental.categories.Category; + +/** + * Tests for the page filter + */ +@Category(SmallTests.class) +public class TestPageFilter extends TestCase { + static final int ROW_LIMIT = 3; + + /** + * test page size filter + * @throws Exception + */ + public void testPageSize() throws Exception { + Filter f = new PageFilter(ROW_LIMIT); + pageSizeTests(f); + } + + /** + * Test filter serialization + * @throws Exception + */ + public void testSerialization() throws Exception { + Filter f = new PageFilter(ROW_LIMIT); + // Decompose mainFilter to bytes. + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(stream); + f.write(out); + out.close(); + byte[] buffer = stream.toByteArray(); + // Recompose mainFilter. + DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer)); + Filter newFilter = new PageFilter(); + newFilter.readFields(in); + + // Ensure the serialization preserved the filter by running a full test. + pageSizeTests(newFilter); + } + + private void pageSizeTests(Filter f) throws Exception { + testFiltersBeyondPageSize(f, ROW_LIMIT); + } + + private void testFiltersBeyondPageSize(final Filter f, final int pageSize) { + int count = 0; + for (int i = 0; i < (pageSize * 2); i++) { + boolean filterOut = f.filterRow(); + + if(filterOut) { + break; + } else { + count++; + } + + // If at last row, should tell us to skip all remaining + if(count == pageSize) { + assertTrue(f.filterAllRemaining()); + } else { + assertFalse(f.filterAllRemaining()); + } + + } + assertEquals(pageSize, count); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestParseFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestParseFilter.java new file mode 100644 index 0000000..922a6c1 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestParseFilter.java @@ -0,0 +1,672 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * This class tests ParseFilter.java + * It tests the entire work flow from when a string is given by the user + * and how it is parsed to construct the corresponding Filter object + */ +@Category(SmallTests.class) +public class TestParseFilter { + + ParseFilter f; + Filter filter; + + @Before + public void setUp() throws Exception { + f = new ParseFilter(); + } + + @After + public void tearDown() throws Exception { + // Nothing to do. + } + + @Test + public void testKeyOnlyFilter() throws IOException { + String filterString = "KeyOnlyFilter()"; + doTestFilter(filterString, KeyOnlyFilter.class); + + String filterString2 = "KeyOnlyFilter ('') "; + byte [] filterStringAsByteArray2 = Bytes.toBytes(filterString2); + try { + filter = f.parseFilterString(filterStringAsByteArray2); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println(e.getMessage()); + } + } + + @Test + public void testFirstKeyOnlyFilter() throws IOException { + String filterString = " FirstKeyOnlyFilter( ) "; + doTestFilter(filterString, FirstKeyOnlyFilter.class); + + String filterString2 = " FirstKeyOnlyFilter ('') "; + byte [] filterStringAsByteArray2 = Bytes.toBytes(filterString2); + try { + filter = f.parseFilterString(filterStringAsByteArray2); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println(e.getMessage()); + } + } + + @Test + public void testPrefixFilter() throws IOException { + String filterString = " PrefixFilter('row' ) "; + PrefixFilter prefixFilter = doTestFilter(filterString, PrefixFilter.class); + byte [] prefix = prefixFilter.getPrefix(); + assertEquals(new String(prefix), "row"); + + + filterString = " PrefixFilter(row)"; + try { + doTestFilter(filterString, PrefixFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println(e.getMessage()); + } + } + + @Test + public void testColumnPrefixFilter() throws IOException { + String filterString = " ColumnPrefixFilter('qualifier' ) "; + ColumnPrefixFilter columnPrefixFilter = + doTestFilter(filterString, ColumnPrefixFilter.class); + byte [] columnPrefix = columnPrefixFilter.getPrefix(); + assertEquals(new String(columnPrefix), "qualifier"); + } + + @Test + public void testMultipleColumnPrefixFilter() throws IOException { + String filterString = " MultipleColumnPrefixFilter('qualifier1', 'qualifier2' ) "; + MultipleColumnPrefixFilter multipleColumnPrefixFilter = + doTestFilter(filterString, MultipleColumnPrefixFilter.class); + byte [][] prefixes = multipleColumnPrefixFilter.getPrefix(); + assertEquals(new String(prefixes[0]), "qualifier1"); + assertEquals(new String(prefixes[1]), "qualifier2"); + } + + @Test + public void testColumnCountGetFilter() throws IOException { + String filterString = " ColumnCountGetFilter(4)"; + ColumnCountGetFilter columnCountGetFilter = + doTestFilter(filterString, ColumnCountGetFilter.class); + int limit = columnCountGetFilter.getLimit(); + assertEquals(limit, 4); + + filterString = " ColumnCountGetFilter('abc')"; + try { + doTestFilter(filterString, ColumnCountGetFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println(e.getMessage()); + } + + filterString = " ColumnCountGetFilter(2147483648)"; + try { + doTestFilter(filterString, ColumnCountGetFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println(e.getMessage()); + } + } + + @Test + public void testPageFilter() throws IOException { + String filterString = " PageFilter(4)"; + PageFilter pageFilter = + doTestFilter(filterString, PageFilter.class); + long pageSize = pageFilter.getPageSize(); + assertEquals(pageSize, 4); + + filterString = " PageFilter('123')"; + try { + doTestFilter(filterString, PageFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println("PageFilter needs an int as an argument"); + } + } + + @Test + public void testColumnPaginationFilter() throws IOException { + String filterString = "ColumnPaginationFilter(4, 6)"; + ColumnPaginationFilter columnPaginationFilter = + doTestFilter(filterString, ColumnPaginationFilter.class); + int limit = columnPaginationFilter.getLimit(); + assertEquals(limit, 4); + int offset = columnPaginationFilter.getOffset(); + assertEquals(offset, 6); + + filterString = " ColumnPaginationFilter('124')"; + try { + doTestFilter(filterString, ColumnPaginationFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println("ColumnPaginationFilter needs two arguments"); + } + + filterString = " ColumnPaginationFilter('4' , '123a')"; + try { + doTestFilter(filterString, ColumnPaginationFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println("ColumnPaginationFilter needs two ints as arguments"); + } + + filterString = " ColumnPaginationFilter('4' , '-123')"; + try { + doTestFilter(filterString, ColumnPaginationFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println("ColumnPaginationFilter arguments should not be negative"); + } + } + + @Test + public void testInclusiveStopFilter() throws IOException { + String filterString = "InclusiveStopFilter ('row 3')"; + InclusiveStopFilter inclusiveStopFilter = + doTestFilter(filterString, InclusiveStopFilter.class); + byte [] stopRowKey = inclusiveStopFilter.getStopRowKey(); + assertEquals(new String(stopRowKey), "row 3"); + } + + + @Test + public void testTimestampsFilter() throws IOException { + String filterString = "TimestampsFilter(9223372036854775806, 6)"; + TimestampsFilter timestampsFilter = + doTestFilter(filterString, TimestampsFilter.class); + List timestamps = timestampsFilter.getTimestamps(); + assertEquals(timestamps.size(), 2); + assertEquals(timestamps.get(0), new Long(6)); + + filterString = "TimestampsFilter()"; + timestampsFilter = doTestFilter(filterString, TimestampsFilter.class); + timestamps = timestampsFilter.getTimestamps(); + assertEquals(timestamps.size(), 0); + + filterString = "TimestampsFilter(9223372036854775808, 6)"; + try { + doTestFilter(filterString, ColumnPaginationFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println("Long Argument was too large"); + } + + filterString = "TimestampsFilter(-45, 6)"; + try { + doTestFilter(filterString, ColumnPaginationFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println("Timestamp Arguments should not be negative"); + } + } + + @Test + public void testRowFilter() throws IOException { + String filterString = "RowFilter ( =, 'binary:regionse')"; + RowFilter rowFilter = + doTestFilter(filterString, RowFilter.class); + assertEquals(CompareFilter.CompareOp.EQUAL, rowFilter.getOperator()); + assertTrue(rowFilter.getComparator() instanceof BinaryComparator); + BinaryComparator binaryComparator = (BinaryComparator) rowFilter.getComparator(); + assertEquals("regionse", new String(binaryComparator.getValue())); + } + + @Test + public void testFamilyFilter() throws IOException { + String filterString = "FamilyFilter(>=, 'binaryprefix:pre')"; + FamilyFilter familyFilter = + doTestFilter(filterString, FamilyFilter.class); + assertEquals(CompareFilter.CompareOp.GREATER_OR_EQUAL, familyFilter.getOperator()); + assertTrue(familyFilter.getComparator() instanceof BinaryPrefixComparator); + BinaryPrefixComparator binaryPrefixComparator = + (BinaryPrefixComparator) familyFilter.getComparator(); + assertEquals("pre", new String(binaryPrefixComparator.getValue())); + } + + @Test + public void testQualifierFilter() throws IOException { + String filterString = "QualifierFilter(=, 'regexstring:pre*')"; + QualifierFilter qualifierFilter = + doTestFilter(filterString, QualifierFilter.class); + assertEquals(CompareFilter.CompareOp.EQUAL, qualifierFilter.getOperator()); + assertTrue(qualifierFilter.getComparator() instanceof RegexStringComparator); + RegexStringComparator regexStringComparator = + (RegexStringComparator) qualifierFilter.getComparator(); + assertEquals("pre*", new String(regexStringComparator.getValue())); + } + + @Test + public void testValueFilter() throws IOException { + String filterString = "ValueFilter(!=, 'substring:pre')"; + ValueFilter valueFilter = + doTestFilter(filterString, ValueFilter.class); + assertEquals(CompareFilter.CompareOp.NOT_EQUAL, valueFilter.getOperator()); + assertTrue(valueFilter.getComparator() instanceof SubstringComparator); + SubstringComparator substringComparator = + (SubstringComparator) valueFilter.getComparator(); + assertEquals("pre", new String(substringComparator.getValue())); + } + + @Test + public void testColumnRangeFilter() throws IOException { + String filterString = "ColumnRangeFilter('abc', true, 'xyz', false)"; + ColumnRangeFilter columnRangeFilter = + doTestFilter(filterString, ColumnRangeFilter.class); + assertEquals("abc", new String(columnRangeFilter.getMinColumn())); + assertEquals("xyz", new String(columnRangeFilter.getMaxColumn())); + assertTrue(columnRangeFilter.isMinColumnInclusive()); + assertFalse(columnRangeFilter.isMaxColumnInclusive()); + } + + @Test + public void testDependentColumnFilter() throws IOException { + String filterString = "DependentColumnFilter('family', 'qualifier', true, =, 'binary:abc')"; + DependentColumnFilter dependentColumnFilter = + doTestFilter(filterString, DependentColumnFilter.class); + assertEquals("family", new String(dependentColumnFilter.getFamily())); + assertEquals("qualifier", new String(dependentColumnFilter.getQualifier())); + assertTrue(dependentColumnFilter.getDropDependentColumn()); + assertEquals(CompareFilter.CompareOp.EQUAL, dependentColumnFilter.getOperator()); + assertTrue(dependentColumnFilter.getComparator() instanceof BinaryComparator); + BinaryComparator binaryComparator = (BinaryComparator)dependentColumnFilter.getComparator(); + assertEquals("abc", new String(binaryComparator.getValue())); + } + + @Test + public void testSingleColumnValueFilter() throws IOException { + String filterString = "SingleColumnValueFilter " + + "('family', 'qualifier', >=, 'binary:a', true, false)"; + SingleColumnValueFilter singleColumnValueFilter = + doTestFilter(filterString, SingleColumnValueFilter.class); + assertEquals("family", new String(singleColumnValueFilter.getFamily())); + assertEquals("qualifier", new String(singleColumnValueFilter.getQualifier())); + assertEquals(singleColumnValueFilter.getOperator(), CompareFilter.CompareOp.GREATER_OR_EQUAL); + assertTrue(singleColumnValueFilter.getComparator() instanceof BinaryComparator); + BinaryComparator binaryComparator = (BinaryComparator) singleColumnValueFilter.getComparator(); + assertEquals(new String(binaryComparator.getValue()), "a"); + assertTrue(singleColumnValueFilter.getFilterIfMissing()); + assertFalse(singleColumnValueFilter.getLatestVersionOnly()); + + + filterString = "SingleColumnValueFilter ('family', 'qualifier', >, 'binaryprefix:a')"; + singleColumnValueFilter = doTestFilter(filterString, SingleColumnValueFilter.class); + assertEquals("family", new String(singleColumnValueFilter.getFamily())); + assertEquals("qualifier", new String(singleColumnValueFilter.getQualifier())); + assertEquals(singleColumnValueFilter.getOperator(), CompareFilter.CompareOp.GREATER); + assertTrue(singleColumnValueFilter.getComparator() instanceof BinaryPrefixComparator); + BinaryPrefixComparator binaryPrefixComparator = + (BinaryPrefixComparator) singleColumnValueFilter.getComparator(); + assertEquals(new String(binaryPrefixComparator.getValue()), "a"); + assertFalse(singleColumnValueFilter.getFilterIfMissing()); + assertTrue(singleColumnValueFilter.getLatestVersionOnly()); + } + + @Test + public void testSingleColumnValueExcludeFilter() throws IOException { + String filterString = + "SingleColumnValueExcludeFilter ('family', 'qualifier', <, 'binaryprefix:a')"; + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter = + doTestFilter(filterString, SingleColumnValueExcludeFilter.class); + assertEquals(singleColumnValueExcludeFilter.getOperator(), CompareFilter.CompareOp.LESS); + assertEquals("family", new String(singleColumnValueExcludeFilter.getFamily())); + assertEquals("qualifier", new String(singleColumnValueExcludeFilter.getQualifier())); + assertEquals(new String(singleColumnValueExcludeFilter.getComparator().getValue()), "a"); + assertFalse(singleColumnValueExcludeFilter.getFilterIfMissing()); + assertTrue(singleColumnValueExcludeFilter.getLatestVersionOnly()); + + filterString = "SingleColumnValueExcludeFilter " + + "('family', 'qualifier', <=, 'binaryprefix:a', true, false)"; + singleColumnValueExcludeFilter = + doTestFilter(filterString, SingleColumnValueExcludeFilter.class); + assertEquals("family", new String(singleColumnValueExcludeFilter.getFamily())); + assertEquals("qualifier", new String(singleColumnValueExcludeFilter.getQualifier())); + assertEquals(singleColumnValueExcludeFilter.getOperator(), + CompareFilter.CompareOp.LESS_OR_EQUAL); + assertTrue(singleColumnValueExcludeFilter.getComparator() instanceof BinaryPrefixComparator); + BinaryPrefixComparator binaryPrefixComparator = + (BinaryPrefixComparator) singleColumnValueExcludeFilter.getComparator(); + assertEquals(new String(binaryPrefixComparator.getValue()), "a"); + assertTrue(singleColumnValueExcludeFilter.getFilterIfMissing()); + assertFalse(singleColumnValueExcludeFilter.getLatestVersionOnly()); + } + + @Test + public void testSkipFilter() throws IOException { + String filterString = "SKIP ValueFilter( =, 'binary:0')"; + SkipFilter skipFilter = + doTestFilter(filterString, SkipFilter.class); + assertTrue(skipFilter.getFilter() instanceof ValueFilter); + ValueFilter valueFilter = (ValueFilter) skipFilter.getFilter(); + + assertEquals(CompareFilter.CompareOp.EQUAL, valueFilter.getOperator()); + assertTrue(valueFilter.getComparator() instanceof BinaryComparator); + BinaryComparator binaryComparator = (BinaryComparator) valueFilter.getComparator(); + assertEquals("0", new String(binaryComparator.getValue())); + } + + @Test + public void testWhileFilter() throws IOException { + String filterString = " WHILE RowFilter ( !=, 'binary:row1')"; + WhileMatchFilter whileMatchFilter = + doTestFilter(filterString, WhileMatchFilter.class); + assertTrue(whileMatchFilter.getFilter() instanceof RowFilter); + RowFilter rowFilter = (RowFilter) whileMatchFilter.getFilter(); + + assertEquals(CompareFilter.CompareOp.NOT_EQUAL, rowFilter.getOperator()); + assertTrue(rowFilter.getComparator() instanceof BinaryComparator); + BinaryComparator binaryComparator = (BinaryComparator) rowFilter.getComparator(); + assertEquals("row1", new String(binaryComparator.getValue())); + } + + @Test + public void testCompoundFilter1() throws IOException { + String filterString = " (PrefixFilter ('realtime')AND FirstKeyOnlyFilter())"; + FilterList filterList = + doTestFilter(filterString, FilterList.class); + ArrayList filters = (ArrayList) filterList.getFilters(); + + assertTrue(filters.get(0) instanceof PrefixFilter); + assertTrue(filters.get(1) instanceof FirstKeyOnlyFilter); + PrefixFilter PrefixFilter = (PrefixFilter) filters.get(0); + byte [] prefix = PrefixFilter.getPrefix(); + assertEquals(new String(prefix), "realtime"); + FirstKeyOnlyFilter firstKeyOnlyFilter = (FirstKeyOnlyFilter) filters.get(1); + } + + @Test + public void testCompoundFilter2() throws IOException { + String filterString = "(PrefixFilter('realtime') AND QualifierFilter (>=, 'binary:e'))" + + "OR FamilyFilter (=, 'binary:qualifier') "; + FilterList filterList = + doTestFilter(filterString, FilterList.class); + ArrayList filterListFilters = (ArrayList) filterList.getFilters(); + assertTrue(filterListFilters.get(0) instanceof FilterList); + assertTrue(filterListFilters.get(1) instanceof FamilyFilter); + assertEquals(filterList.getOperator(), FilterList.Operator.MUST_PASS_ONE); + + filterList = (FilterList) filterListFilters.get(0); + FamilyFilter familyFilter = (FamilyFilter) filterListFilters.get(1); + + filterListFilters = (ArrayList)filterList.getFilters(); + assertTrue(filterListFilters.get(0) instanceof PrefixFilter); + assertTrue(filterListFilters.get(1) instanceof QualifierFilter); + assertEquals(filterList.getOperator(), FilterList.Operator.MUST_PASS_ALL); + + assertEquals(CompareFilter.CompareOp.EQUAL, familyFilter.getOperator()); + assertTrue(familyFilter.getComparator() instanceof BinaryComparator); + BinaryComparator binaryComparator = (BinaryComparator) familyFilter.getComparator(); + assertEquals("qualifier", new String(binaryComparator.getValue())); + + PrefixFilter prefixFilter = (PrefixFilter) filterListFilters.get(0); + byte [] prefix = prefixFilter.getPrefix(); + assertEquals(new String(prefix), "realtime"); + + QualifierFilter qualifierFilter = (QualifierFilter) filterListFilters.get(1); + assertEquals(CompareFilter.CompareOp.GREATER_OR_EQUAL, qualifierFilter.getOperator()); + assertTrue(qualifierFilter.getComparator() instanceof BinaryComparator); + binaryComparator = (BinaryComparator) qualifierFilter.getComparator(); + assertEquals("e", new String(binaryComparator.getValue())); + } + + @Test + public void testCompoundFilter3() throws IOException { + String filterString = " ColumnPrefixFilter ('realtime')AND " + + "FirstKeyOnlyFilter() OR SKIP FamilyFilter(=, 'substring:hihi')"; + FilterList filterList = + doTestFilter(filterString, FilterList.class); + ArrayList filters = (ArrayList) filterList.getFilters(); + + assertTrue(filters.get(0) instanceof FilterList); + assertTrue(filters.get(1) instanceof SkipFilter); + + filterList = (FilterList) filters.get(0); + SkipFilter skipFilter = (SkipFilter) filters.get(1); + + filters = (ArrayList) filterList.getFilters(); + assertTrue(filters.get(0) instanceof ColumnPrefixFilter); + assertTrue(filters.get(1) instanceof FirstKeyOnlyFilter); + + ColumnPrefixFilter columnPrefixFilter = (ColumnPrefixFilter) filters.get(0); + byte [] columnPrefix = columnPrefixFilter.getPrefix(); + assertEquals(new String(columnPrefix), "realtime"); + + FirstKeyOnlyFilter firstKeyOnlyFilter = (FirstKeyOnlyFilter) filters.get(1); + + assertTrue(skipFilter.getFilter() instanceof FamilyFilter); + FamilyFilter familyFilter = (FamilyFilter) skipFilter.getFilter(); + + assertEquals(CompareFilter.CompareOp.EQUAL, familyFilter.getOperator()); + assertTrue(familyFilter.getComparator() instanceof SubstringComparator); + SubstringComparator substringComparator = + (SubstringComparator) familyFilter.getComparator(); + assertEquals("hihi", new String(substringComparator.getValue())); + } + + @Test + public void testCompoundFilter4() throws IOException { + String filterString = " ColumnPrefixFilter ('realtime') OR " + + "FirstKeyOnlyFilter() OR SKIP FamilyFilter(=, 'substring:hihi')"; + FilterList filterList = + doTestFilter(filterString, FilterList.class); + ArrayList filters = (ArrayList) filterList.getFilters(); + + assertTrue(filters.get(0) instanceof ColumnPrefixFilter); + assertTrue(filters.get(1) instanceof FirstKeyOnlyFilter); + assertTrue(filters.get(2) instanceof SkipFilter); + + ColumnPrefixFilter columnPrefixFilter = (ColumnPrefixFilter) filters.get(0); + FirstKeyOnlyFilter firstKeyOnlyFilter = (FirstKeyOnlyFilter) filters.get(1); + SkipFilter skipFilter = (SkipFilter) filters.get(2); + + byte [] columnPrefix = columnPrefixFilter.getPrefix(); + assertEquals(new String(columnPrefix), "realtime"); + + assertTrue(skipFilter.getFilter() instanceof FamilyFilter); + FamilyFilter familyFilter = (FamilyFilter) skipFilter.getFilter(); + + assertEquals(CompareFilter.CompareOp.EQUAL, familyFilter.getOperator()); + assertTrue(familyFilter.getComparator() instanceof SubstringComparator); + SubstringComparator substringComparator = + (SubstringComparator) familyFilter.getComparator(); + assertEquals("hihi", new String(substringComparator.getValue())); + } + + @Test + public void testIncorrectCompareOperator() throws IOException { + String filterString = "RowFilter ('>>' , 'binary:region')"; + try { + doTestFilter(filterString, RowFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println("Incorrect compare operator >>"); + } + } + + @Test + public void testIncorrectComparatorType () throws IOException { + String filterString = "RowFilter ('>=' , 'binaryoperator:region')"; + try { + doTestFilter(filterString, RowFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println("Incorrect comparator type: binaryoperator"); + } + + filterString = "RowFilter ('>=' 'regexstring:pre*')"; + try { + doTestFilter(filterString, RowFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println("RegexStringComparator can only be used with EQUAL or NOT_EQUAL"); + } + + filterString = "SingleColumnValueFilter" + + " ('family', 'qualifier', '>=', 'substring:a', 'true', 'false')')"; + try { + doTestFilter(filterString, RowFilter.class); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println("SubtringComparator can only be used with EQUAL or NOT_EQUAL"); + } + } + + @Test + public void testPrecedence1() throws IOException { + String filterString = " (PrefixFilter ('realtime')AND FirstKeyOnlyFilter()" + + " OR KeyOnlyFilter())"; + FilterList filterList = + doTestFilter(filterString, FilterList.class); + + ArrayList filters = (ArrayList) filterList.getFilters(); + + assertTrue(filters.get(0) instanceof FilterList); + assertTrue(filters.get(1) instanceof KeyOnlyFilter); + + filterList = (FilterList) filters.get(0); + filters = (ArrayList) filterList.getFilters(); + + assertTrue(filters.get(0) instanceof PrefixFilter); + assertTrue(filters.get(1) instanceof FirstKeyOnlyFilter); + + PrefixFilter prefixFilter = (PrefixFilter)filters.get(0); + byte [] prefix = prefixFilter.getPrefix(); + assertEquals(new String(prefix), "realtime"); + } + + @Test + public void testPrecedence2() throws IOException { + String filterString = " PrefixFilter ('realtime')AND SKIP FirstKeyOnlyFilter()" + + "OR KeyOnlyFilter()"; + FilterList filterList = + doTestFilter(filterString, FilterList.class); + ArrayList filters = (ArrayList) filterList.getFilters(); + + assertTrue(filters.get(0) instanceof FilterList); + assertTrue(filters.get(1) instanceof KeyOnlyFilter); + + filterList = (FilterList) filters.get(0); + filters = (ArrayList) filterList.getFilters(); + + assertTrue(filters.get(0) instanceof PrefixFilter); + assertTrue(filters.get(1) instanceof SkipFilter); + + PrefixFilter prefixFilter = (PrefixFilter)filters.get(0); + byte [] prefix = prefixFilter.getPrefix(); + assertEquals(new String(prefix), "realtime"); + + SkipFilter skipFilter = (SkipFilter)filters.get(1); + assertTrue(skipFilter.getFilter() instanceof FirstKeyOnlyFilter); + } + + @Test + public void testUnescapedQuote1 () throws IOException { + String filterString = "InclusiveStopFilter ('row''3')"; + InclusiveStopFilter inclusiveStopFilter = + doTestFilter(filterString, InclusiveStopFilter.class); + byte [] stopRowKey = inclusiveStopFilter.getStopRowKey(); + assertEquals(new String(stopRowKey), "row'3"); + } + + @Test + public void testUnescapedQuote2 () throws IOException { + String filterString = "InclusiveStopFilter ('row''3''')"; + InclusiveStopFilter inclusiveStopFilter = + doTestFilter(filterString, InclusiveStopFilter.class); + byte [] stopRowKey = inclusiveStopFilter.getStopRowKey(); + assertEquals(new String(stopRowKey), "row'3'"); + } + + @Test + public void testUnescapedQuote3 () throws IOException { + String filterString = " InclusiveStopFilter ('''')"; + InclusiveStopFilter inclusiveStopFilter = + doTestFilter(filterString, InclusiveStopFilter.class); + byte [] stopRowKey = inclusiveStopFilter.getStopRowKey(); + assertEquals(new String(stopRowKey), "'"); + } + + @Test + public void testIncorrectFilterString () throws IOException { + String filterString = "()"; + byte [] filterStringAsByteArray = Bytes.toBytes(filterString); + try { + filter = f.parseFilterString(filterStringAsByteArray); + assertTrue(false); + } catch (IllegalArgumentException e) { + System.out.println(e.getMessage()); + } + } + + @Test + public void testCorrectFilterString () throws IOException { + String filterString = "(FirstKeyOnlyFilter())"; + FirstKeyOnlyFilter firstKeyOnlyFilter = + doTestFilter(filterString, FirstKeyOnlyFilter.class); + } + + @Test + public void testRegisterFilter() { + ParseFilter.registerFilter("MyFilter", "some.class"); + + assertTrue(f.getSupportedFilters().contains("MyFilter")); + } + + private T doTestFilter(String filterString, Class clazz) throws IOException { + byte [] filterStringAsByteArray = Bytes.toBytes(filterString); + filter = f.parseFilterString(filterStringAsByteArray); + assertEquals(clazz, filter.getClass()); + return clazz.cast(filter); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestPrefixFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestPrefixFilter.java new file mode 100644 index 0000000..24a999e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestPrefixFilter.java @@ -0,0 +1,108 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.UnsupportedEncodingException; + +@Category(SmallTests.class) +public class TestPrefixFilter extends TestCase { + Filter mainFilter; + static final char FIRST_CHAR = 'a'; + static final char LAST_CHAR = 'e'; + static final String HOST_PREFIX = "org.apache.site-"; + static byte [] GOOD_BYTES = null; + + static { + try { + GOOD_BYTES = "abc".getBytes(HConstants.UTF8_ENCODING); + } catch (UnsupportedEncodingException e) { + fail(); + } + } + + protected void setUp() throws Exception { + super.setUp(); + this.mainFilter = new PrefixFilter(Bytes.toBytes(HOST_PREFIX)); + } + + public void testPrefixOnRow() throws Exception { + prefixRowTests(mainFilter); + } + + public void testPrefixOnRowInsideWhileMatchRow() throws Exception { + prefixRowTests(new WhileMatchFilter(this.mainFilter), true); + } + + public void testSerialization() throws Exception { + // Decompose mainFilter to bytes. + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(stream); + mainFilter.write(out); + out.close(); + byte[] buffer = stream.toByteArray(); + + // Recompose filter. + DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer)); + Filter newFilter = new PrefixFilter(); + newFilter.readFields(in); + + // Ensure the serialization preserved the filter by running all test. + prefixRowTests(newFilter); + } + + private void prefixRowTests(Filter filter) throws Exception { + prefixRowTests(filter, false); + } + + private void prefixRowTests(Filter filter, boolean lastFilterAllRemaining) + throws Exception { + for (char c = FIRST_CHAR; c <= LAST_CHAR; c++) { + byte [] t = createRow(c); + assertFalse("Failed with character " + c, + filter.filterRowKey(t, 0, t.length)); + assertFalse(filter.filterAllRemaining()); + } + String yahooSite = "com.yahoo.www"; + byte [] yahooSiteBytes = Bytes.toBytes(yahooSite); + assertTrue("Failed with character " + + yahooSite, filter.filterRowKey(yahooSiteBytes, 0, yahooSiteBytes.length)); + assertEquals(filter.filterAllRemaining(), lastFilterAllRemaining); + } + + private byte [] createRow(final char c) { + return Bytes.toBytes(HOST_PREFIX + Character.toString(c)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestRandomRowFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestRandomRowFilter.java new file mode 100644 index 0000000..7731cd9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestRandomRowFilter.java @@ -0,0 +1,99 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; + +import junit.framework.TestCase; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestRandomRowFilter extends TestCase { + protected RandomRowFilter quarterChanceFilter; + + @Override + protected void setUp() throws Exception { + super.setUp(); + quarterChanceFilter = new RandomRowFilter(0.25f); + } + + /** + * Tests basics + * + * @throws Exception + */ + public void testBasics() throws Exception { + int included = 0; + int max = 1000000; + for (int i = 0; i < max; i++) { + if (!quarterChanceFilter.filterRowKey(Bytes.toBytes("row"), 0, Bytes + .toBytes("row").length)) { + included++; + } + } + // Now let's check if the filter included the right number of rows; + // since we're dealing with randomness, we must have a include an epsilon + // tolerance. + int epsilon = max / 100; + assertTrue("Roughly 25% should pass the filter", Math.abs(included - max + / 4) < epsilon); + } + + /** + * Tests serialization + * + * @throws Exception + */ + public void testSerialization() throws Exception { + RandomRowFilter newFilter = serializationTest(quarterChanceFilter); + // use epsilon float comparison + assertTrue("float should be equal", Math.abs(newFilter.getChance() + - quarterChanceFilter.getChance()) < 0.000001f); + } + + private RandomRowFilter serializationTest(RandomRowFilter filter) + throws Exception { + // Decompose filter to bytes. + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(stream); + filter.write(out); + out.close(); + byte[] buffer = stream.toByteArray(); + + // Recompose filter. + DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer)); + RandomRowFilter newFilter = new RandomRowFilter(); + newFilter.readFields(in); + + return newFilter; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java new file mode 100644 index 0000000..31f5db4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java @@ -0,0 +1,91 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import junit.framework.TestCase; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +import java.util.List; +import java.util.ArrayList; + +/** + * Tests for {@link SingleColumnValueExcludeFilter}. Because this filter + * extends {@link SingleColumnValueFilter}, only the added functionality is + * tested. That is, method filterKeyValue(KeyValue). + * + * @author ferdy + * + */ +@Category(SmallTests.class) +public class TestSingleColumnValueExcludeFilter extends TestCase { + private static final byte[] ROW = Bytes.toBytes("test"); + private static final byte[] COLUMN_FAMILY = Bytes.toBytes("test"); + private static final byte[] COLUMN_QUALIFIER = Bytes.toBytes("foo"); + private static final byte[] COLUMN_QUALIFIER_2 = Bytes.toBytes("foo_2"); + private static final byte[] VAL_1 = Bytes.toBytes("a"); + private static final byte[] VAL_2 = Bytes.toBytes("ab"); + + /** + * Test the overridden functionality of filterKeyValue(KeyValue) + * @throws Exception + */ + public void testFilterKeyValue() throws Exception { + Filter filter = new SingleColumnValueExcludeFilter(COLUMN_FAMILY, COLUMN_QUALIFIER, + CompareOp.EQUAL, VAL_1); + + // A 'match' situation + List kvs = new ArrayList(); + KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1); + + kvs.add (new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1)); + kvs.add (new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_1)); + kvs.add (new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1)); + + filter.filterRow(kvs); + + assertEquals("resultSize", kvs.size(), 2); + assertTrue("leftKV1", KeyValue.COMPARATOR.compare(kvs.get(0), kv) == 0); + assertTrue("leftKV2", KeyValue.COMPARATOR.compare(kvs.get(1), kv) == 0); + assertFalse("allRemainingWhenMatch", filter.filterAllRemaining()); + + // A 'mismatch' situation + filter.reset(); + // INCLUDE expected because test column has not yet passed + kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1); + assertTrue("otherColumn", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE); + // Test column will pass (wont match), expect NEXT_ROW + kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_2); + assertTrue("testedMismatch", filter.filterKeyValue(kv) == Filter.ReturnCode.NEXT_ROW); + // After a mismatch (at least with LatestVersionOnly), subsequent columns are EXCLUDE + kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1); + assertTrue("otherColumn", filter.filterKeyValue(kv) == Filter.ReturnCode.NEXT_ROW); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java new file mode 100644 index 0000000..2a0751c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java @@ -0,0 +1,179 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +/** + * Tests the value filter + */ +@Category(SmallTests.class) +public class TestSingleColumnValueFilter extends TestCase { + private static final byte[] ROW = Bytes.toBytes("test"); + private static final byte[] COLUMN_FAMILY = Bytes.toBytes("test"); + private static final byte [] COLUMN_QUALIFIER = Bytes.toBytes("foo"); + private static final byte[] VAL_1 = Bytes.toBytes("a"); + private static final byte[] VAL_2 = Bytes.toBytes("ab"); + private static final byte[] VAL_3 = Bytes.toBytes("abc"); + private static final byte[] VAL_4 = Bytes.toBytes("abcd"); + private static final byte[] FULLSTRING_1 = + Bytes.toBytes("The quick brown fox jumps over the lazy dog."); + private static final byte[] FULLSTRING_2 = + Bytes.toBytes("The slow grey fox trips over the lazy dog."); + private static final String QUICK_SUBSTR = "quick"; + private static final String QUICK_REGEX = ".+quick.+"; + + Filter basicFilter; + Filter substrFilter; + Filter regexFilter; + + @Override + protected void setUp() throws Exception { + super.setUp(); + basicFilter = basicFilterNew(); + substrFilter = substrFilterNew(); + regexFilter = regexFilterNew(); + } + + private Filter basicFilterNew() { + return new SingleColumnValueFilter(COLUMN_FAMILY, COLUMN_QUALIFIER, + CompareOp.GREATER_OR_EQUAL, VAL_2); + } + + private Filter substrFilterNew() { + return new SingleColumnValueFilter(COLUMN_FAMILY, COLUMN_QUALIFIER, + CompareOp.EQUAL, + new SubstringComparator(QUICK_SUBSTR)); + } + + private Filter regexFilterNew() { + return new SingleColumnValueFilter(COLUMN_FAMILY, COLUMN_QUALIFIER, + CompareOp.EQUAL, + new RegexStringComparator(QUICK_REGEX)); + } + + private void basicFilterTests(SingleColumnValueFilter filter) + throws Exception { + KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_2); + assertTrue("basicFilter1", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE); + kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_3); + assertTrue("basicFilter2", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE); + kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_4); + assertTrue("basicFilter3", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE); + assertFalse("basicFilterNotNull", filter.filterRow()); + filter.reset(); + kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_1); + assertTrue("basicFilter4", filter.filterKeyValue(kv) == Filter.ReturnCode.NEXT_ROW); + kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_2); + assertTrue("basicFilter4", filter.filterKeyValue(kv) == Filter.ReturnCode.NEXT_ROW); + assertFalse("basicFilterAllRemaining", filter.filterAllRemaining()); + assertTrue("basicFilterNotNull", filter.filterRow()); + filter.reset(); + filter.setLatestVersionOnly(false); + kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_1); + assertTrue("basicFilter5", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE); + kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_2); + assertTrue("basicFilter5", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE); + assertFalse("basicFilterNotNull", filter.filterRow()); + } + + private void substrFilterTests(Filter filter) + throws Exception { + KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, + FULLSTRING_1); + assertTrue("substrTrue", + filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE); + kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, + FULLSTRING_2); + assertTrue("substrFalse", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE); + assertFalse("substrFilterAllRemaining", filter.filterAllRemaining()); + assertFalse("substrFilterNotNull", filter.filterRow()); + } + + private void regexFilterTests(Filter filter) + throws Exception { + KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, + FULLSTRING_1); + assertTrue("regexTrue", + filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE); + kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, + FULLSTRING_2); + assertTrue("regexFalse", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE); + assertFalse("regexFilterAllRemaining", filter.filterAllRemaining()); + assertFalse("regexFilterNotNull", filter.filterRow()); + } + + private Filter serializationTest(Filter filter) + throws Exception { + // Decompose filter to bytes. + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(stream); + filter.write(out); + out.close(); + byte[] buffer = stream.toByteArray(); + + // Recompose filter. + DataInputStream in = + new DataInputStream(new ByteArrayInputStream(buffer)); + Filter newFilter = new SingleColumnValueFilter(); + newFilter.readFields(in); + + return newFilter; + } + + /** + * Tests identification of the stop row + * @throws Exception + */ + public void testStop() throws Exception { + basicFilterTests((SingleColumnValueFilter)basicFilter); + substrFilterTests(substrFilter); + regexFilterTests(regexFilter); + } + + /** + * Tests serialization + * @throws Exception + */ + public void testSerialization() throws Exception { + Filter newFilter = serializationTest(basicFilter); + basicFilterTests((SingleColumnValueFilter)newFilter); + newFilter = serializationTest(substrFilter); + substrFilterTests(newFilter); + newFilter = serializationTest(regexFilter); + regexFilterTests(newFilter); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java b/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java new file mode 100644 index 0000000..88310ef --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java @@ -0,0 +1,244 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.io; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.io.FileLink; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Test that FileLink switches between alternate locations + * when the current location moves or gets deleted. + */ +@Category(MediumTests.class) +public class TestFileLink { + /** + * Test, on HDFS, that the FileLink is still readable + * even when the current file gets renamed. + */ + @Test + public void testHDFSLinkReadDuringRename() throws Exception { + HBaseTestingUtility testUtil = new HBaseTestingUtility(); + Configuration conf = testUtil.getConfiguration(); + conf.setInt("dfs.blocksize", 1024 * 1024); + conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024); + + testUtil.startMiniDFSCluster(1); + MiniDFSCluster cluster = testUtil.getDFSCluster(); + FileSystem fs = cluster.getFileSystem(); + assertEquals("hdfs", fs.getUri().getScheme()); + + try { + testLinkReadDuringRename(fs, testUtil.getDefaultRootDirPath()); + } finally { + testUtil.shutdownMiniCluster(); + } + } + + /** + * Test, on a local filesystem, that the FileLink is still readable + * even when the current file gets renamed. + */ + @Test + public void testLocalLinkReadDuringRename() throws IOException { + HBaseTestingUtility testUtil = new HBaseTestingUtility(); + FileSystem fs = testUtil.getTestFileSystem(); + assertEquals("file", fs.getUri().getScheme()); + testLinkReadDuringRename(fs, testUtil.getDataTestDir()); + } + + /** + * Test that link is still readable even when the current file gets renamed. + */ + private void testLinkReadDuringRename(FileSystem fs, Path rootDir) throws IOException { + Path originalPath = new Path(rootDir, "test.file"); + Path archivedPath = new Path(rootDir, "archived.file"); + + writeSomeData(fs, originalPath, 256 << 20, (byte)2); + + List files = new ArrayList(); + files.add(originalPath); + files.add(archivedPath); + + FileLink link = new FileLink(files); + FSDataInputStream in = link.open(fs); + try { + byte[] data = new byte[8192]; + long size = 0; + + // Read from origin + int n = in.read(data); + dataVerify(data, n, (byte)2); + size += n; + + // Move origin to archive + assertFalse(fs.exists(archivedPath)); + fs.rename(originalPath, archivedPath); + assertFalse(fs.exists(originalPath)); + assertTrue(fs.exists(archivedPath)); + + // Try to read to the end + while ((n = in.read(data)) > 0) { + dataVerify(data, n, (byte)2); + size += n; + } + + assertEquals(256 << 20, size); + } finally { + in.close(); + if (fs.exists(originalPath)) fs.delete(originalPath); + if (fs.exists(archivedPath)) fs.delete(archivedPath); + } + } + + /** + * Test that link is still readable even when the current file gets deleted. + * + * NOTE: This test is valid only on HDFS. + * When a file is deleted from a local file-system, it is simply 'unlinked'. + * The inode, which contains the file's data, is not deleted until all + * processes have finished with it. + * In HDFS when the request exceed the cached block locations, + * a query to the namenode is performed, using the filename, + * and the deleted file doesn't exists anymore (FileNotFoundException). + */ + @Test + public void testHDFSLinkReadDuringDelete() throws Exception { + HBaseTestingUtility testUtil = new HBaseTestingUtility(); + Configuration conf = testUtil.getConfiguration(); + conf.setInt("dfs.blocksize", 1024 * 1024); + conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024); + + testUtil.startMiniDFSCluster(1); + MiniDFSCluster cluster = testUtil.getDFSCluster(); + FileSystem fs = cluster.getFileSystem(); + assertEquals("hdfs", fs.getUri().getScheme()); + + try { + List files = new ArrayList(); + for (int i = 0; i < 3; i++) { + Path path = new Path(String.format("test-data-%d", i)); + writeSomeData(fs, path, 1 << 20, (byte)i); + files.add(path); + } + + FileLink link = new FileLink(files); + FSDataInputStream in = link.open(fs); + try { + byte[] data = new byte[8192]; + int n; + + // Switch to file 1 + n = in.read(data); + dataVerify(data, n, (byte)0); + fs.delete(files.get(0)); + skipBuffer(in, (byte)0); + + // Switch to file 2 + n = in.read(data); + dataVerify(data, n, (byte)1); + fs.delete(files.get(1)); + skipBuffer(in, (byte)1); + + // Switch to file 3 + n = in.read(data); + dataVerify(data, n, (byte)2); + fs.delete(files.get(2)); + skipBuffer(in, (byte)2); + + // No more files available + try { + n = in.read(data); + assert(n <= 0); + } catch (FileNotFoundException e) { + assertTrue(true); + } + } finally { + in.close(); + } + } finally { + testUtil.shutdownMiniCluster(); + } + } + + /** + * Write up to 'size' bytes with value 'v' into a new file called 'path'. + */ + private void writeSomeData (FileSystem fs, Path path, long size, byte v) throws IOException { + byte[] data = new byte[4096]; + for (int i = 0; i < data.length; i++) { + data[i] = v; + } + + FSDataOutputStream stream = fs.create(path); + try { + long written = 0; + while (written < size) { + stream.write(data, 0, data.length); + written += data.length; + } + } finally { + stream.close(); + } + } + + /** + * Verify that all bytes in 'data' have 'v' as value. + */ + private static void dataVerify(byte[] data, int n, byte v) { + for (int i = 0; i < n; ++i) { + assertEquals(v, data[i]); + } + } + + private static void skipBuffer(FSDataInputStream in, byte v) throws IOException { + byte[] data = new byte[8192]; + try { + int n; + while ((n = in.read(data)) == data.length) { + for (int i = 0; i < data.length; ++i) { + if (data[i] != v) + throw new Exception("File changed"); + } + } + } catch (Exception e) { + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java b/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java new file mode 100644 index 0000000..7d8eaa7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java @@ -0,0 +1,248 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.io; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestHalfStoreFileReader { + + /** + * Test the scanner and reseek of a half hfile scanner. The scanner API + * demands that seekTo and reseekTo() only return < 0 if the key lies + * before the start of the file (with no position on the scanner). Returning + * 0 if perfect match (rare), and return > 1 if we got an imperfect match. + * + * The latter case being the most common, we should generally be returning 1, + * and if we do, there may or may not be a 'next' in the scanner/file. + * + * A bug in the half file scanner was returning -1 at the end of the bottom + * half, and that was causing the infrastructure above to go null causing NPEs + * and other problems. This test reproduces that failure, and also tests + * both the bottom and top of the file while we are at it. + * + * @throws IOException + */ + @Test + public void testHalfScanAndReseek() throws IOException { + HBaseTestingUtility test_util = new HBaseTestingUtility(); + String root_dir = test_util.getDataTestDir("TestHalfStoreFile").toString(); + Path p = new Path(root_dir, "test"); + + Configuration conf = test_util.getConfiguration(); + FileSystem fs = FileSystem.get(conf); + CacheConfig cacheConf = new CacheConfig(conf); + + HFile.Writer w = HFile.getWriterFactory(conf, cacheConf) + .withPath(fs, p) + .withBlockSize(1024) + .withComparator(KeyValue.KEY_COMPARATOR) + .create(); + + // write some things. + List items = genSomeKeys(); + for (KeyValue kv : items) { + w.append(kv); + } + w.close(); + + HFile.Reader r = HFile.createReader(fs, p, cacheConf); + r.loadFileInfo(); + byte [] midkey = r.midkey(); + KeyValue midKV = KeyValue.createKeyValueFromKey(midkey); + midkey = midKV.getRow(); + + //System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey)); + + Reference bottom = new Reference(midkey, Reference.Range.bottom); + doTestOfScanAndReseek(p, fs, bottom, cacheConf); + + Reference top = new Reference(midkey, Reference.Range.top); + doTestOfScanAndReseek(p, fs, top, cacheConf); + + r.close(); + } + + private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, + CacheConfig cacheConf) + throws IOException { + final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p, + cacheConf, bottom, DataBlockEncoding.NONE); + halfreader.loadFileInfo(); + final HFileScanner scanner = halfreader.getScanner(false, false); + + scanner.seekTo(); + KeyValue curr; + do { + curr = scanner.getKeyValue(); + KeyValue reseekKv = + getLastOnCol(curr); + int ret = scanner.reseekTo(reseekKv.getKey()); + assertTrue("reseek to returned: " + ret, ret > 0); + //System.out.println(curr + ": " + ret); + } while (scanner.next()); + + int ret = scanner.reseekTo(getLastOnCol(curr).getKey()); + //System.out.println("Last reseek: " + ret); + assertTrue( ret > 0 ); + + halfreader.close(true); + } + + + // Tests the scanner on an HFile that is backed by HalfStoreFiles + @Test + public void testHalfScanner() throws IOException { + HBaseTestingUtility test_util = new HBaseTestingUtility(); + String root_dir = test_util.getDataTestDir("TestHalfStoreFileScanBefore").toString(); + Path p = new Path(root_dir, "test"); + Configuration conf = test_util.getConfiguration(); + FileSystem fs = FileSystem.get(conf); + CacheConfig cacheConf = new CacheConfig(conf); + + HFile.Writer w = HFile.getWriterFactory(conf, cacheConf) + .withPath(fs, p) + .withBlockSize(1024) + .withComparator(KeyValue.KEY_COMPARATOR) + .create(); + + // write some things. + List items = genSomeKeys(); + for (KeyValue kv : items) { + w.append(kv); + } + w.close(); + + + HFile.Reader r = HFile.createReader(fs, p, cacheConf); + r.loadFileInfo(); + byte[] midkey = r.midkey(); + KeyValue midKV = KeyValue.createKeyValueFromKey(midkey); + midkey = midKV.getRow(); + + Reference bottom = new Reference(midkey, Reference.Range.bottom); + Reference top = new Reference(midkey, Reference.Range.top); + + // Ugly code to get the item before the midkey + KeyValue beforeMidKey = null; + for (KeyValue item : items) { + if (item.equals(midKV)) { + break; + } + beforeMidKey = item; + } + + + // Seek on the splitKey, should be in top, not in bottom + KeyValue foundKeyValue = doTestOfSeekBefore(p, fs, bottom, midKV, cacheConf); + assertEquals(beforeMidKey, foundKeyValue); + + // Seek tot the last thing should be the penultimate on the top, the one before the midkey on the bottom. + foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(items.size() - 1), cacheConf); + assertEquals(items.get(items.size() - 2), foundKeyValue); + + foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(items.size() - 1), cacheConf); + assertEquals(beforeMidKey, foundKeyValue); + + // Try and seek before something that is in the bottom. + foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(0), cacheConf); + assertNull(foundKeyValue); + + // Try and seek before the first thing. + foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(0), cacheConf); + assertNull(foundKeyValue); + + // Try and seek before the second thing in the top and bottom. + foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(1), cacheConf); + assertNull(foundKeyValue); + + foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(1), cacheConf); + assertEquals(items.get(0), foundKeyValue); + + // Try to seek before the splitKey in the top file + foundKeyValue = doTestOfSeekBefore(p, fs, top, midKV, cacheConf); + assertNull(foundKeyValue); + } + + private KeyValue doTestOfSeekBefore(Path p, FileSystem fs, Reference bottom, KeyValue seekBefore, + CacheConfig cacheConfig) + throws IOException { + final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p, + cacheConfig, bottom, DataBlockEncoding.NONE); + halfreader.loadFileInfo(); + final HFileScanner scanner = halfreader.getScanner(false, false); + scanner.seekBefore(seekBefore.getKey()); + return scanner.getKeyValue(); + } + + private KeyValue getLastOnCol(KeyValue curr) { + return KeyValue.createLastOnRow( + curr.getBuffer(), curr.getRowOffset(), curr.getRowLength(), + curr.getBuffer(), curr.getFamilyOffset(), curr.getFamilyLength(), + curr.getBuffer(), curr.getQualifierOffset(), curr.getQualifierLength()); + } + + static final int SIZE = 1000; + + static byte[] _b(String s) { + return Bytes.toBytes(s); + } + + List genSomeKeys() { + List ret = new ArrayList(SIZE); + for (int i = 0; i < SIZE; i++) { + KeyValue kv = + new KeyValue( + _b(String.format("row_%04d", i)), + _b("family"), + _b("qualifier"), + 1000, // timestamp + _b("value")); + ret.add(kv); + } + return ret; + } + + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java b/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java new file mode 100644 index 0000000..892dda1 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/TestHbaseObjectWritable.java @@ -0,0 +1,549 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.Serializable; +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableSet; + +import junit.framework.TestCase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HServerInfo; +import org.apache.hadoop.hbase.HServerLoad; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Action; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.MultiAction; +import org.apache.hadoop.hbase.client.MultiResponse; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Row; +import org.apache.hadoop.hbase.client.RowMutations; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.coprocessor.Exec; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.BitComparator; +import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; +import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; +import org.apache.hadoop.hbase.filter.ColumnRangeFilter; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.DependentColumnFilter; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterBase; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.InclusiveStopFilter; +import org.apache.hadoop.hbase.filter.KeyOnlyFilter; +import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.filter.RandomRowFilter; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.filter.SkipFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.apache.hadoop.hbase.filter.WhileMatchFilter; +import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionOpeningState; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.MapWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparator; +import org.junit.Assert; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; +import com.google.protobuf.Message; + +@Category(SmallTests.class) +public class TestHbaseObjectWritable extends TestCase { + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + } + + @SuppressWarnings("boxing") + public void testReadOldObjectDataInput() throws IOException { + Configuration conf = HBaseConfiguration.create(); + /* + * This is the code used to generate byte[] where + * HbaseObjectWritable used byte for code + * + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(byteStream); + HbaseObjectWritable.writeObject(out, bytes, byte[].class, conf); + byte[] ba = byteStream.toByteArray(); + out.close(); + */ + + /* + * byte array generated by the folowing call + * HbaseObjectWritable.writeObject(out, new Text("Old"), Text.class, conf); + */ + byte[] baForText = {13, 13, 3, 79, 108, 100}; + Text txt = (Text)readByteArray(conf, baForText); + Text oldTxt = new Text("Old"); + assertEquals(txt, oldTxt); + + final byte A = 'A'; + byte [] bytes = new byte[1]; + bytes[0] = A; + /* + * byte array generated by the folowing call + * HbaseObjectWritable.writeObject(out, bytes, byte[].class, conf); + */ + byte[] baForByteArray = { 11, 1, 65 }; + byte[] baOut = (byte[])readByteArray(conf, baForByteArray); + assertTrue(Bytes.equals(baOut, bytes)); + } + + /* + * helper method which reads byte array using HbaseObjectWritable.readObject() + */ + private Object readByteArray(final Configuration conf, final byte[] ba) + throws IOException { + ByteArrayInputStream bais = + new ByteArrayInputStream(ba); + DataInputStream dis = new DataInputStream(bais); + Object product = HbaseObjectWritable.readObject(dis, conf); + dis.close(); + return product; + } + + @SuppressWarnings("boxing") + public void testReadObjectDataInputConfiguration() throws IOException { + Configuration conf = HBaseConfiguration.create(); + // Do primitive type + final int COUNT = 101; + assertTrue(doType(conf, COUNT, int.class).equals(COUNT)); + // Do array + final byte [] testing = "testing".getBytes(); + byte [] result = (byte [])doType(conf, testing, testing.getClass()); + assertTrue(WritableComparator.compareBytes(testing, 0, testing.length, + result, 0, result.length) == 0); + // Do unsupported type. + boolean exception = false; + try { + doType(conf, new Object(), Object.class); + } catch (UnsupportedOperationException uoe) { + exception = true; + } + assertTrue(exception); + // Try odd types + final byte A = 'A'; + byte [] bytes = new byte[1]; + bytes[0] = A; + Object obj = doType(conf, bytes, byte [].class); + assertTrue(((byte [])obj)[0] == A); + // Do 'known' Writable type. + obj = doType(conf, new Text(""), Text.class); + assertTrue(obj instanceof Text); + //List.class + List list = new ArrayList(); + list.add("hello"); + list.add("world"); + list.add("universe"); + obj = doType(conf, list, List.class); + assertTrue(obj instanceof List); + Assert.assertArrayEquals(list.toArray(), ((List)obj).toArray() ); + //List.class with null values + List listWithNulls = new ArrayList(); + listWithNulls.add("hello"); + listWithNulls.add("world"); + listWithNulls.add(null); + obj = doType(conf, listWithNulls, List.class); + assertTrue(obj instanceof List); + Assert.assertArrayEquals(listWithNulls.toArray(), ((List)obj).toArray() ); + //ArrayList.class + ArrayList arr = new ArrayList(); + arr.add("hello"); + arr.add("world"); + arr.add("universe"); + obj = doType(conf, arr, ArrayList.class); + assertTrue(obj instanceof ArrayList); + Assert.assertArrayEquals(list.toArray(), ((ArrayList)obj).toArray() ); + // Check that filters can be serialized + obj = doType(conf, new PrefixFilter(HConstants.EMPTY_BYTE_ARRAY), + PrefixFilter.class); + assertTrue(obj instanceof PrefixFilter); + } + + public void testCustomWritable() throws Exception { + Configuration conf = HBaseConfiguration.create(); + + // test proper serialization of un-encoded custom writables + CustomWritable custom = new CustomWritable("test phrase"); + Object obj = doType(conf, custom, CustomWritable.class); + assertTrue(obj instanceof Writable); + assertTrue(obj instanceof CustomWritable); + assertEquals("test phrase", ((CustomWritable)obj).getValue()); + + // test proper serialization of a custom filter + CustomFilter filt = new CustomFilter("mykey"); + FilterList filtlist = new FilterList(FilterList.Operator.MUST_PASS_ALL); + filtlist.addFilter(filt); + obj = doType(conf, filtlist, FilterList.class); + assertTrue(obj instanceof FilterList); + assertNotNull(((FilterList)obj).getFilters()); + assertEquals(1, ((FilterList)obj).getFilters().size()); + Filter child = ((FilterList)obj).getFilters().get(0); + assertTrue(child instanceof CustomFilter); + assertEquals("mykey", ((CustomFilter)child).getKey()); + } + + public void testCustomSerializable() throws Exception { + Configuration conf = HBaseConfiguration.create(); + + // test proper serialization of un-encoded serialized java objects + CustomSerializable custom = new CustomSerializable("test phrase"); + Object obj = doType(conf, custom, CustomSerializable.class); + assertTrue(obj instanceof Serializable); + assertTrue(obj instanceof CustomSerializable); + assertEquals("test phrase", ((CustomSerializable)obj).getValue()); + } + + private Object doType(final Configuration conf, final Object value, + final Class clazz) + throws IOException { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(byteStream); + HbaseObjectWritable.writeObject(out, value, clazz, conf); + out.close(); + ByteArrayInputStream bais = + new ByteArrayInputStream(byteStream.toByteArray()); + DataInputStream dis = new DataInputStream(bais); + Object product = HbaseObjectWritable.readObject(dis, conf); + dis.close(); + return product; + } + + public static class A extends IntWritable { + public A() {} + public A(int a) {super(a);} + } + + public static class B extends A { + int b; + public B() { } + public B(int a, int b) { + super(a); + this.b = b; + } + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeInt(b); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + this.b = in.readInt(); + } + @Override + public boolean equals(Object o) { + if (o instanceof B) { + return this.get() == ((B) o).get() && this.b == ((B) o).b; + } + return false; + } + } + + /** Tests for serialization of List and Arrays */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void testPolymorphismInSequences() throws Exception { + Configuration conf = HBaseConfiguration.create(); + Object ret; + + //test with lists + List list = Lists.newArrayList(new A(42), new B(10, 100)); + ret = doType(conf, list, list.getClass()); + assertEquals(ret, list); + + //test with Writable[] + Writable[] warr = new Writable[] {new A(42), new B(10, 100)}; + ret = doType(conf, warr, warr.getClass()); + Assert.assertArrayEquals((Writable[])ret, warr); + + //test with arrays + A[] arr = new A[] {new A(42), new B(10, 100)}; + ret = doType(conf, arr, arr.getClass()); + Assert.assertArrayEquals((A[])ret, arr); + + //test with double array + A[][] darr = new A[][] {new A[] { new A(42), new B(10, 100)}, new A[] {new A(12)}}; + ret = doType(conf, darr, darr.getClass()); + Assert.assertArrayEquals((A[][])ret, darr); + + //test with List of arrays + List larr = Lists.newArrayList(arr, new A[] {new A(99)}); + ret = doType(conf, larr, larr.getClass()); + List lret = (List) ret; + assertEquals(larr.size(), lret.size()); + for (int i=0; i, HStoreKey + // KeyValue, LruBlockCache, LruHashMap, Put, HLogKey + + /** + * Test our hard-coded sizing of native java objects + */ + public void testNativeSizes() throws IOException { + @SuppressWarnings("rawtypes") + Class cl = null; + long expected = 0L; + long actual = 0L; + + // ArrayList + cl = ArrayList.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.ARRAYLIST; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // ByteBuffer + cl = ByteBuffer.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.BYTE_BUFFER; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // Integer + cl = Integer.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.INTEGER; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // Map.Entry + // Interface is public, all others are not. Hard to size via ClassSize +// cl = Map.Entry.class; +// expected = ClassSize.estimateBase(cl, false); +// actual = ClassSize.MAP_ENTRY; +// if(expected != actual) { +// ClassSize.estimateBase(cl, true); +// assertEquals(expected, actual); +// } + + // Object + cl = Object.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.OBJECT; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // TreeMap + cl = TreeMap.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.TREEMAP; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // String + cl = String.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.STRING; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // ConcurrentHashMap + cl = ConcurrentHashMap.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.CONCURRENT_HASHMAP; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // ConcurrentSkipListMap + cl = ConcurrentSkipListMap.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.CONCURRENT_SKIPLISTMAP; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // ReentrantReadWriteLock + cl = ReentrantReadWriteLock.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.REENTRANT_LOCK; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // AtomicLong + cl = AtomicLong.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.ATOMIC_LONG; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // AtomicInteger + cl = AtomicInteger.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.ATOMIC_INTEGER; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // AtomicBoolean + cl = AtomicBoolean.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.ATOMIC_BOOLEAN; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // CopyOnWriteArraySet + cl = CopyOnWriteArraySet.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.COPYONWRITE_ARRAYSET; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // CopyOnWriteArrayList + cl = CopyOnWriteArrayList.class; + expected = ClassSize.estimateBase(cl, false); + actual = ClassSize.COPYONWRITE_ARRAYLIST; + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + + } + + /** + * Testing the classes that implements HeapSize and are a part of 0.20. + * Some are not tested here for example BlockIndex which is tested in + * TestHFile since it is a non public class + * @throws IOException + */ + public void testSizes() throws IOException { + @SuppressWarnings("rawtypes") + Class cl = null; + long expected = 0L; + long actual = 0L; + + //KeyValue + cl = KeyValue.class; + expected = ClassSize.estimateBase(cl, false); + KeyValue kv = new KeyValue(); + actual = kv.heapSize(); + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + //Put + cl = Put.class; + expected = ClassSize.estimateBase(cl, false); + //The actual TreeMap is not included in the above calculation + expected += ClassSize.TREEMAP; + Put put = new Put(Bytes.toBytes("")); + actual = put.heapSize(); + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + //LruBlockCache Overhead + cl = LruBlockCache.class; + actual = LruBlockCache.CACHE_FIXED_OVERHEAD; + expected = ClassSize.estimateBase(cl, false); + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // CachedBlock Fixed Overhead + // We really need "deep" sizing but ClassSize does not do this. + // Perhaps we should do all these more in this style.... + cl = CachedBlock.class; + actual = CachedBlock.PER_BLOCK_OVERHEAD; + expected = ClassSize.estimateBase(cl, false); + expected += ClassSize.estimateBase(String.class, false); + expected += ClassSize.estimateBase(ByteBuffer.class, false); + if(expected != actual) { + ClassSize.estimateBase(cl, true); + ClassSize.estimateBase(String.class, true); + ClassSize.estimateBase(ByteBuffer.class, true); + assertEquals(expected, actual); + } + + // MemStore Overhead + cl = MemStore.class; + actual = MemStore.FIXED_OVERHEAD; + expected = ClassSize.estimateBase(cl, false); + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // MemStore Deep Overhead + actual = MemStore.DEEP_OVERHEAD; + expected = ClassSize.estimateBase(cl, false); + expected += ClassSize.estimateBase(ReentrantReadWriteLock.class, false); + expected += ClassSize.estimateBase(AtomicLong.class, false); + expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false); + expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false); + expected += ClassSize.estimateBase(CopyOnWriteArraySet.class, false); + expected += ClassSize.estimateBase(CopyOnWriteArrayList.class, false); + if(expected != actual) { + ClassSize.estimateBase(cl, true); + ClassSize.estimateBase(ReentrantReadWriteLock.class, true); + ClassSize.estimateBase(AtomicLong.class, true); + ClassSize.estimateBase(ConcurrentSkipListMap.class, true); + ClassSize.estimateBase(CopyOnWriteArraySet.class, true); + ClassSize.estimateBase(CopyOnWriteArrayList.class, true); + assertEquals(expected, actual); + } + + // SchemaConfigured + LOG.debug("Heap size for: " + SchemaConfigured.class.getName()); + SchemaConfigured sc = new SchemaConfigured(null, "myTable", "myCF"); + assertEquals(ClassSize.estimateBase(SchemaConfigured.class, true), + sc.heapSize()); + + // Store Overhead + cl = Store.class; + actual = Store.FIXED_OVERHEAD; + expected = ClassSize.estimateBase(cl, false); + if(expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // Region Overhead + cl = HRegion.class; + actual = HRegion.FIXED_OVERHEAD; + expected = ClassSize.estimateBase(cl, false); + if (expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // Block cache key overhead + cl = BlockCacheKey.class; + // Passing zero length file name, because estimateBase does not handle + // deep overhead. + actual = new BlockCacheKey("", 0).heapSize(); + expected = ClassSize.estimateBase(cl, false); + if (expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // Currently NOT testing Deep Overheads of many of these classes. + // Deep overheads cover a vast majority of stuff, but will not be 100% + // accurate because it's unclear when we're referencing stuff that's already + // accounted for. But we have satisfied our two core requirements. + // Sizing is quite accurate now, and our tests will throw errors if + // any of these classes are modified without updating overhead sizes. + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/TestImmutableBytesWritable.java b/src/test/java/org/apache/hadoop/hbase/io/TestImmutableBytesWritable.java new file mode 100644 index 0000000..6217dec --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/TestImmutableBytesWritable.java @@ -0,0 +1,140 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.io; + +import junit.framework.TestCase; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +@Category(SmallTests.class) +public class TestImmutableBytesWritable extends TestCase { + public void testHash() throws Exception { + assertEquals( + new ImmutableBytesWritable(Bytes.toBytes("xxabc"), 2, 3).hashCode(), + new ImmutableBytesWritable(Bytes.toBytes("abc")).hashCode()); + assertEquals( + new ImmutableBytesWritable(Bytes.toBytes("xxabcd"), 2, 3).hashCode(), + new ImmutableBytesWritable(Bytes.toBytes("abc")).hashCode()); + assertNotSame( + new ImmutableBytesWritable(Bytes.toBytes("xxabc"), 2, 3).hashCode(), + new ImmutableBytesWritable(Bytes.toBytes("xxabc"), 2, 2).hashCode()); + } + + public void testSpecificCompare() { + ImmutableBytesWritable ibw1 = new ImmutableBytesWritable(new byte[]{0x0f}); + ImmutableBytesWritable ibw2 = new ImmutableBytesWritable(new byte[]{0x00, 0x00}); + ImmutableBytesWritable.Comparator c = new ImmutableBytesWritable.Comparator(); + assertFalse("ibw1 < ibw2", c.compare( ibw1, ibw2 ) < 0 ); + } + + public void testComparison() throws Exception { + runTests("aa", "b", -1); + runTests("aa", "aa", 0); + runTests("aa", "ab", -1); + runTests("aa", "aaa", -1); + runTests("", "", 0); + runTests("", "a", -1); + } + + private void runTests(String aStr, String bStr, int signum) + throws Exception { + ImmutableBytesWritable a = new ImmutableBytesWritable( + Bytes.toBytes(aStr)); + ImmutableBytesWritable b = new ImmutableBytesWritable( + Bytes.toBytes(bStr)); + + doComparisonsOnObjects(a, b, signum); + doComparisonsOnRaw(a, b, signum); + + // Tests for when the offset is non-zero + a = new ImmutableBytesWritable(Bytes.toBytes("xxx" + aStr), + 3, aStr.length()); + b = new ImmutableBytesWritable(Bytes.toBytes("yy" + bStr), + 2, bStr.length()); + doComparisonsOnObjects(a, b, signum); + doComparisonsOnRaw(a, b, signum); + + // Tests for when offset is nonzero and length doesn't extend to end + a = new ImmutableBytesWritable(Bytes.toBytes("xxx" + aStr + "zzz"), + 3, aStr.length()); + b = new ImmutableBytesWritable(Bytes.toBytes("yy" + bStr + "aaa"), + 2, bStr.length()); + doComparisonsOnObjects(a, b, signum); + doComparisonsOnRaw(a, b, signum); + } + + + private int signum(int i) { + if (i > 0) return 1; + if (i == 0) return 0; + return -1; + } + + private void doComparisonsOnRaw(ImmutableBytesWritable a, + ImmutableBytesWritable b, + int expectedSignum) + throws IOException { + ImmutableBytesWritable.Comparator comparator = + new ImmutableBytesWritable.Comparator(); + + ByteArrayOutputStream baosA = new ByteArrayOutputStream(); + ByteArrayOutputStream baosB = new ByteArrayOutputStream(); + + a.write(new DataOutputStream(baosA)); + b.write(new DataOutputStream(baosB)); + + assertEquals( + "Comparing " + a + " and " + b + " as raw", + signum(comparator.compare(baosA.toByteArray(), 0, baosA.size(), + baosB.toByteArray(), 0, baosB.size())), + expectedSignum); + + assertEquals( + "Comparing " + a + " and " + b + " as raw (inverse)", + -signum(comparator.compare(baosB.toByteArray(), 0, baosB.size(), + baosA.toByteArray(), 0, baosA.size())), + expectedSignum); + } + + private void doComparisonsOnObjects(ImmutableBytesWritable a, + ImmutableBytesWritable b, + int expectedSignum) { + ImmutableBytesWritable.Comparator comparator = + new ImmutableBytesWritable.Comparator(); + assertEquals( + "Comparing " + a + " and " + b + " as objects", + signum(comparator.compare(a, b)), expectedSignum); + assertEquals( + "Comparing " + a + " and " + b + " as objects (inverse)", + -signum(comparator.compare(b, a)), expectedSignum); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/RedundantKVGenerator.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/RedundantKVGenerator.java new file mode 100644 index 0000000..07dcb63 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/RedundantKVGenerator.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.io.encoding; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.ByteBufferUtils; +import org.apache.hadoop.io.WritableUtils; + +/** + * Generate list of key values which are very useful to test data block encoding + * and compression. + */ +public class RedundantKVGenerator { + // row settings + static int DEFAULT_NUMBER_OF_ROW_PREFIXES = 10; + static int DEFAULT_AVERAGE_PREFIX_LENGTH = 6; + static int DEFAULT_PREFIX_LENGTH_VARIANCE = 3; + static int DEFAULT_AVERAGE_SUFFIX_LENGTH = 3; + static int DEFAULT_SUFFIX_LENGTH_VARIANCE = 3; + static int DEFAULT_NUMBER_OF_ROW = 500; + + // qualifier + static float DEFAULT_CHANCE_FOR_SAME_QUALIFIER = 0.5f; + static float DEFAULT_CHANCE_FOR_SIMILIAR_QUALIFIER = 0.4f; + static int DEFAULT_AVERAGE_QUALIFIER_LENGTH = 9; + static int DEFAULT_QUALIFIER_LENGTH_VARIANCE = 3; + + static int DEFAULT_COLUMN_FAMILY_LENGTH = 9; + static int DEFAULT_VALUE_LENGTH = 8; + static float DEFAULT_CHANCE_FOR_ZERO_VALUE = 0.5f; + + static int DEFAULT_BASE_TIMESTAMP_DIVIDE = 1000000; + static int DEFAULT_TIMESTAMP_DIFF_SIZE = 100000000; + + /** + * Default constructor, assumes all parameters from class constants. + */ + public RedundantKVGenerator() { + this(new Random(42L), + DEFAULT_NUMBER_OF_ROW_PREFIXES, + DEFAULT_AVERAGE_PREFIX_LENGTH, + DEFAULT_PREFIX_LENGTH_VARIANCE, + DEFAULT_AVERAGE_SUFFIX_LENGTH, + DEFAULT_SUFFIX_LENGTH_VARIANCE, + DEFAULT_NUMBER_OF_ROW, + + DEFAULT_CHANCE_FOR_SAME_QUALIFIER, + DEFAULT_CHANCE_FOR_SIMILIAR_QUALIFIER, + DEFAULT_AVERAGE_QUALIFIER_LENGTH, + DEFAULT_QUALIFIER_LENGTH_VARIANCE, + + DEFAULT_COLUMN_FAMILY_LENGTH, + DEFAULT_VALUE_LENGTH, + DEFAULT_CHANCE_FOR_ZERO_VALUE, + + DEFAULT_BASE_TIMESTAMP_DIVIDE, + DEFAULT_TIMESTAMP_DIFF_SIZE + ); + } + + + /** + * Various configuration options for generating key values + * @param randomizer pick things by random + */ + public RedundantKVGenerator(Random randomizer, + int numberOfRowPrefixes, + int averagePrefixLength, + int prefixLengthVariance, + int averageSuffixLength, + int suffixLengthVariance, + int numberOfRows, + + float chanceForSameQualifier, + float chanceForSimiliarQualifier, + int averageQualifierLength, + int qualifierLengthVariance, + + int columnFamilyLength, + int valueLength, + float chanceForZeroValue, + + int baseTimestampDivide, + int timestampDiffSize + ) { + this.randomizer = randomizer; + + this.numberOfRowPrefixes = numberOfRowPrefixes; + this.averagePrefixLength = averagePrefixLength; + this.prefixLengthVariance = prefixLengthVariance; + this.averageSuffixLength = averageSuffixLength; + this.suffixLengthVariance = suffixLengthVariance; + this.numberOfRows = numberOfRows; + + this.chanceForSameQualifier = chanceForSameQualifier; + this.chanceForSimiliarQualifier = chanceForSimiliarQualifier; + this.averageQualifierLength = averageQualifierLength; + this.qualifierLengthVariance = qualifierLengthVariance; + + this.columnFamilyLength = columnFamilyLength; + this.valueLength = valueLength; + this.chanceForZeroValue = chanceForZeroValue; + + this.baseTimestampDivide = baseTimestampDivide; + this.timestampDiffSize = timestampDiffSize; + } + + /** Used to generate dataset */ + private Random randomizer; + + // row settings + private int numberOfRowPrefixes; + private int averagePrefixLength = 6; + private int prefixLengthVariance = 3; + private int averageSuffixLength = 3; + private int suffixLengthVariance = 3; + private int numberOfRows = 500; + + // qualifier + private float chanceForSameQualifier = 0.5f; + private float chanceForSimiliarQualifier = 0.4f; + private int averageQualifierLength = 9; + private int qualifierLengthVariance = 3; + + private int columnFamilyLength = 9; + private int valueLength = 8; + private float chanceForZeroValue = 0.5f; + + private int baseTimestampDivide = 1000000; + private int timestampDiffSize = 100000000; + + private List generateRows() { + // generate prefixes + List prefixes = new ArrayList(); + prefixes.add(new byte[0]); + for (int i = 1; i < numberOfRowPrefixes; ++i) { + int prefixLength = averagePrefixLength; + prefixLength += randomizer.nextInt(2 * prefixLengthVariance + 1) - + prefixLengthVariance; + byte[] newPrefix = new byte[prefixLength]; + randomizer.nextBytes(newPrefix); + prefixes.add(newPrefix); + } + + // generate rest of the row + List rows = new ArrayList(); + for (int i = 0; i < numberOfRows; ++i) { + int suffixLength = averageSuffixLength; + suffixLength += randomizer.nextInt(2 * suffixLengthVariance + 1) - + suffixLengthVariance; + int randomPrefix = randomizer.nextInt(prefixes.size()); + byte[] row = new byte[prefixes.get(randomPrefix).length + + suffixLength]; + rows.add(row); + } + + return rows; + } + + /** + * Generate test data useful to test encoders. + * @param howMany How many Key values should be generated. + * @return sorted list of key values + */ + public List generateTestKeyValues(int howMany) { + List result = new ArrayList(); + + List rows = generateRows(); + Map> rowsToQualifier = + new HashMap>(); + + byte[] family = new byte[columnFamilyLength]; + randomizer.nextBytes(family); + + long baseTimestamp = Math.abs(randomizer.nextLong()) / + baseTimestampDivide; + + byte[] value = new byte[valueLength]; + + for (int i = 0; i < howMany; ++i) { + long timestamp = baseTimestamp + randomizer.nextInt( + timestampDiffSize); + Integer rowId = randomizer.nextInt(rows.size()); + byte[] row = rows.get(rowId); + + // generate qualifier, sometimes it is same, sometimes similar, + // occasionally completely different + byte[] qualifier; + float qualifierChance = randomizer.nextFloat(); + if (!rowsToQualifier.containsKey(rowId) || + qualifierChance > chanceForSameQualifier + + chanceForSimiliarQualifier) { + int qualifierLength = averageQualifierLength; + qualifierLength += + randomizer.nextInt(2 * qualifierLengthVariance + 1) - + qualifierLengthVariance; + qualifier = new byte[qualifierLength]; + randomizer.nextBytes(qualifier); + + // add it to map + if (!rowsToQualifier.containsKey(rowId)) { + rowsToQualifier.put(rowId, new ArrayList()); + } + rowsToQualifier.get(rowId).add(qualifier); + } else if (qualifierChance > chanceForSameQualifier) { + // similar qualifier + List previousQualifiers = rowsToQualifier.get(rowId); + byte[] originalQualifier = previousQualifiers.get( + randomizer.nextInt(previousQualifiers.size())); + + qualifier = new byte[originalQualifier.length]; + int commonPrefix = randomizer.nextInt(qualifier.length); + System.arraycopy(originalQualifier, 0, qualifier, 0, commonPrefix); + for (int j = commonPrefix; j < qualifier.length; ++j) { + qualifier[j] = (byte) (randomizer.nextInt() & 0xff); + } + + rowsToQualifier.get(rowId).add(qualifier); + } else { + // same qualifier + List previousQualifiers = rowsToQualifier.get(rowId); + qualifier = previousQualifiers.get( + randomizer.nextInt(previousQualifiers.size())); + } + + if (randomizer.nextFloat() < chanceForZeroValue) { + for (int j = 0; j < value.length; ++j) { + value[j] = (byte) 0; + } + } else { + randomizer.nextBytes(value); + } + + result.add(new KeyValue(row, family, qualifier, timestamp, value)); + } + + Collections.sort(result, KeyValue.COMPARATOR); + + return result; + } + + /** + * Convert list of KeyValues to byte buffer. + * @param keyValues list of KeyValues to be converted. + * @return buffer with content from key values + */ + public static ByteBuffer convertKvToByteBuffer(List keyValues, + boolean includesMemstoreTS) { + int totalSize = 0; + for (KeyValue kv : keyValues) { + totalSize += kv.getLength(); + if (includesMemstoreTS) { + totalSize += WritableUtils.getVIntSize(kv.getMemstoreTS()); + } + } + + ByteBuffer result = ByteBuffer.allocate(totalSize); + for (KeyValue kv : keyValues) { + result.put(kv.getBuffer(), kv.getOffset(), kv.getLength()); + if (includesMemstoreTS) { + ByteBufferUtils.writeVLong(result, kv.getMemstoreTS()); + } + } + + return result; + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java new file mode 100644 index 0000000..2e7de2f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.io.encoding; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestBufferedDataBlockEncoder { + + @Test + public void testEnsureSpaceForKey() { + BufferedDataBlockEncoder.SeekerState state = + new BufferedDataBlockEncoder.SeekerState(); + for (int i = 1; i <= 65536; ++i) { + state.keyLength = i; + state.ensureSpaceForKey(); + state.keyBuffer[state.keyLength - 1] = (byte) ((i - 1) % 0xff); + for (int j = 0; j < i - 1; ++j) { + // Check that earlier bytes were preserved as the buffer grew. + assertEquals((byte) (j % 0xff), state.keyBuffer[j]); + } + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java new file mode 100644 index 0000000..77ce818 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.io.encoding; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests changing data block encoding settings of a column family. + */ +@Category(LargeTests.class) +public class TestChangingEncoding { + + private static final Log LOG = LogFactory.getLog(TestChangingEncoding.class); + + static final String CF = "EncodingTestCF"; + static final byte[] CF_BYTES = Bytes.toBytes(CF); + + private static final int NUM_ROWS_PER_BATCH = 100; + private static final int NUM_COLS_PER_ROW = 20; + + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + private static final Configuration conf = TEST_UTIL.getConfiguration(); + + private static final int TIMEOUT_MS = 240000; + + private HBaseAdmin admin; + private HColumnDescriptor hcd; + + private String tableName; + private static final List ENCODINGS_TO_ITERATE = + createEncodingsToIterate(); + + private static final List createEncodingsToIterate() { + List encodings = new ArrayList( + Arrays.asList(DataBlockEncoding.values())); + encodings.add(DataBlockEncoding.NONE); + return Collections.unmodifiableList(encodings); + } + + /** A zero-based index of the current batch of test data being written */ + private int numBatchesWritten; + + private void prepareTest(String testId) throws IOException { + tableName = "test_table_" + testId; + HTableDescriptor htd = new HTableDescriptor(tableName); + hcd = new HColumnDescriptor(CF); + htd.addFamily(hcd); + admin.createTable(htd); + numBatchesWritten = 0; + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Use a small flush size to create more HFiles. + conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024); + TEST_UTIL.startMiniCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + admin = new HBaseAdmin(conf); + } + + @After + public void tearDown() throws IOException { + admin.close(); + } + + private static byte[] getRowKey(int batchId, int i) { + return Bytes.toBytes("batch" + batchId + "_row" + i); + } + + private static byte[] getQualifier(int j) { + return Bytes.toBytes("col" + j); + } + + private static byte[] getValue(int batchId, int i, int j) { + return Bytes.toBytes("value_for_" + Bytes.toString(getRowKey(batchId, i)) + + "_col" + j); + } + + static void writeTestDataBatch(Configuration conf, String tableName, + int batchId) throws Exception { + LOG.debug("Writing test data batch " + batchId); + HTable table = new HTable(conf, tableName); + for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) { + Put put = new Put(getRowKey(batchId, i)); + for (int j = 0; j < NUM_COLS_PER_ROW; ++j) { + put.add(CF_BYTES, getQualifier(j), + getValue(batchId, i, j)); + table.put(put); + } + } + table.close(); + } + + static void verifyTestDataBatch(Configuration conf, String tableName, + int batchId) throws Exception { + LOG.debug("Verifying test data batch " + batchId); + HTable table = new HTable(conf, tableName); + for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) { + Get get = new Get(getRowKey(batchId, i)); + Result result = table.get(get); + for (int j = 0; j < NUM_COLS_PER_ROW; ++j) { + KeyValue kv = result.getColumnLatest(CF_BYTES, getQualifier(j)); + assertEquals(Bytes.toStringBinary(getValue(batchId, i, j)), + Bytes.toStringBinary(kv.getValue())); + } + } + table.close(); + } + + private void writeSomeNewData() throws Exception { + writeTestDataBatch(conf, tableName, numBatchesWritten); + ++numBatchesWritten; + } + + private void verifyAllData() throws Exception { + for (int i = 0; i < numBatchesWritten; ++i) { + verifyTestDataBatch(conf, tableName, i); + } + } + + private void setEncodingConf(DataBlockEncoding encoding, + boolean encodeOnDisk) throws IOException { + LOG.debug("Setting CF encoding to " + encoding + " (ordinal=" + + encoding.ordinal() + "), encodeOnDisk=" + encodeOnDisk); + admin.disableTable(tableName); + hcd.setDataBlockEncoding(encoding); + hcd.setEncodeOnDisk(encodeOnDisk); + admin.modifyColumn(tableName, hcd); + admin.enableTable(tableName); + } + + @Test(timeout=TIMEOUT_MS) + public void testChangingEncoding() throws Exception { + prepareTest("ChangingEncoding"); + for (boolean encodeOnDisk : new boolean[]{false, true}) { + for (DataBlockEncoding encoding : ENCODINGS_TO_ITERATE) { + setEncodingConf(encoding, encodeOnDisk); + writeSomeNewData(); + verifyAllData(); + } + } + } + + @Test(timeout=TIMEOUT_MS) + public void testChangingEncodingWithCompaction() throws Exception { + prepareTest("ChangingEncodingWithCompaction"); + for (boolean encodeOnDisk : new boolean[]{false, true}) { + for (DataBlockEncoding encoding : ENCODINGS_TO_ITERATE) { + setEncodingConf(encoding, encodeOnDisk); + writeSomeNewData(); + verifyAllData(); + compactAndWait(); + verifyAllData(); + } + } + } + + @Test(timeout=TIMEOUT_MS) + public void testFlippingEncodeOnDisk() throws Exception { + prepareTest("FlippingEncodeOnDisk"); + // The focus of this test case is to flip the "encoding on disk" flag, + // so we only try a couple of encodings. + DataBlockEncoding[] encodings = new DataBlockEncoding[] { + DataBlockEncoding.NONE, DataBlockEncoding.FAST_DIFF }; + for (DataBlockEncoding encoding : encodings) { + boolean[] flagValues; + if (encoding == DataBlockEncoding.NONE) { + // encodeOnDisk does not matter when not using encoding. + flagValues = + new boolean[] { HColumnDescriptor.DEFAULT_ENCODE_ON_DISK }; + } else { + flagValues = new boolean[] { false, true, false, true }; + } + for (boolean encodeOnDisk : flagValues) { + setEncodingConf(encoding, encodeOnDisk); + writeSomeNewData(); + verifyAllData(); + compactAndWait(); + verifyAllData(); + } + } + } + + private void compactAndWait() throws IOException, InterruptedException { + LOG.debug("Compacting table " + tableName); + admin.majorCompact(tableName); + HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); + + // Waiting for the compaction to start, at least .5s. + final long maxWaitime = System.currentTimeMillis() + 500; + boolean cont; + do { + cont = rs.compactSplitThread.getCompactionQueueSize() == 0; + Threads.sleep(1); + } while (cont && System.currentTimeMillis() < maxWaitime); + + while (rs.compactSplitThread.getCompactionQueueSize() > 0) { + Threads.sleep(5); + } + LOG.debug("Compaction queue size reached 0, continuing"); + } + + @Test + public void testCrazyRandomChanges() throws Exception { + prepareTest("RandomChanges"); + Random rand = new Random(2934298742974297L); + for (int i = 0; i < 20; ++i) { + int encodingOrdinal = rand.nextInt(DataBlockEncoding.values().length); + DataBlockEncoding encoding = DataBlockEncoding.values()[encodingOrdinal]; + setEncodingConf(encoding, rand.nextBoolean()); + writeSomeNewData(); + verifyAllData(); + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java new file mode 100644 index 0000000..f7b7397 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.io.encoding; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Random; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Test all of the data block encoding algorithms for correctness. + * Most of the class generate data which will test different branches in code. + */ +@Category(LargeTests.class) +@RunWith(Parameterized.class) +public class TestDataBlockEncoders { + static int NUMBER_OF_KV = 10000; + static int NUM_RANDOM_SEEKS = 10000; + + private RedundantKVGenerator generator = new RedundantKVGenerator(); + private Random randomizer = new Random(42l); + + private final boolean includesMemstoreTS; + + @Parameters + public static Collection parameters() { + return HBaseTestingUtility.BOOLEAN_PARAMETERIZED; + } + + public TestDataBlockEncoders(boolean includesMemstoreTS) { + this.includesMemstoreTS = includesMemstoreTS; + } + + private void testAlgorithm(ByteBuffer dataset, DataBlockEncoder encoder) + throws IOException { + // encode + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dataOut = new DataOutputStream(baos); + encoder.compressKeyValues(dataOut, dataset, includesMemstoreTS); + + // decode + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + DataInputStream dis = new DataInputStream(bais); + ByteBuffer actualDataset; + actualDataset = encoder.uncompressKeyValues(dis, includesMemstoreTS); + + dataset.rewind(); + actualDataset.rewind(); + + assertEquals("Encoding -> decoding gives different results for " + encoder, + Bytes.toStringBinary(dataset), Bytes.toStringBinary(actualDataset)); + } + + /** + * Test data block encoding of empty KeyValue. + * @throws IOException On test failure. + */ + @Test + public void testEmptyKeyValues() throws IOException { + List kvList = new ArrayList(); + byte[] row = new byte[0]; + byte[] family = new byte[0]; + byte[] qualifier = new byte[0]; + byte[] value = new byte[0]; + kvList.add(new KeyValue(row, family, qualifier, 0l, Type.Put, value)); + kvList.add(new KeyValue(row, family, qualifier, 0l, Type.Put, value)); + testEncodersOnDataset(RedundantKVGenerator.convertKvToByteBuffer(kvList, + includesMemstoreTS)); + } + + /** + * Test KeyValues with negative timestamp. + * @throws IOException On test failure. + */ + @Test + public void testNegativeTimestamps() throws IOException { + List kvList = new ArrayList(); + byte[] row = new byte[0]; + byte[] family = new byte[0]; + byte[] qualifier = new byte[0]; + byte[] value = new byte[0]; + kvList.add(new KeyValue(row, family, qualifier, -1l, Type.Put, value)); + kvList.add(new KeyValue(row, family, qualifier, -2l, Type.Put, value)); + testEncodersOnDataset( + RedundantKVGenerator.convertKvToByteBuffer(kvList, + includesMemstoreTS)); + } + + /** + * Test whether compression -> decompression gives the consistent results on + * pseudorandom sample. + * @throws IOException On test failure. + */ + @Test + public void testExecutionOnSample() throws IOException { + testEncodersOnDataset( + RedundantKVGenerator.convertKvToByteBuffer( + generator.generateTestKeyValues(NUMBER_OF_KV), + includesMemstoreTS)); + } + + /** + * Test seeking while file is encoded. + */ + @Test + public void testSeekingOnSample() throws IOException{ + List sampleKv = generator.generateTestKeyValues(NUMBER_OF_KV); + ByteBuffer originalBuffer = + RedundantKVGenerator.convertKvToByteBuffer(sampleKv, + includesMemstoreTS); + List dataBlockEncoders = + DataBlockEncoding.getAllEncoders(); + + // create all seekers + List encodedSeekers = + new ArrayList(); + for (DataBlockEncoder encoder : dataBlockEncoders) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dataOut = new DataOutputStream(baos); + encoder.compressKeyValues(dataOut, originalBuffer, includesMemstoreTS); + ByteBuffer encodedBuffer = ByteBuffer.wrap(baos.toByteArray()); + DataBlockEncoder.EncodedSeeker seeker = + encoder.createSeeker(KeyValue.KEY_COMPARATOR, includesMemstoreTS); + seeker.setCurrentBuffer(encodedBuffer); + encodedSeekers.add(seeker); + } + + // test it! + // try a few random seeks + for (boolean seekBefore : new boolean[] {false, true}) { + for (int i = 0; i < NUM_RANDOM_SEEKS; ++i) { + int keyValueId; + if (!seekBefore) { + keyValueId = randomizer.nextInt(sampleKv.size()); + } else { + keyValueId = randomizer.nextInt(sampleKv.size() - 1) + 1; + } + + KeyValue keyValue = sampleKv.get(keyValueId); + checkSeekingConsistency(encodedSeekers, seekBefore, keyValue); + } + } + + // check edge cases + checkSeekingConsistency(encodedSeekers, false, sampleKv.get(0)); + for (boolean seekBefore : new boolean[] {false, true}) { + checkSeekingConsistency(encodedSeekers, seekBefore, + sampleKv.get(sampleKv.size() - 1)); + KeyValue midKv = sampleKv.get(sampleKv.size() / 2); + KeyValue lastMidKv = midKv.createLastOnRowCol(); + checkSeekingConsistency(encodedSeekers, seekBefore, lastMidKv); + } + } + + /** + * Test iterating on encoded buffers. + */ + @Test + public void testNextOnSample() { + List sampleKv = generator.generateTestKeyValues(NUMBER_OF_KV); + ByteBuffer originalBuffer = + RedundantKVGenerator.convertKvToByteBuffer(sampleKv, + includesMemstoreTS); + List dataBlockEncoders = + DataBlockEncoding.getAllEncoders(); + + for (DataBlockEncoder encoder : dataBlockEncoders) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dataOut = new DataOutputStream(baos); + try { + encoder.compressKeyValues(dataOut, originalBuffer, includesMemstoreTS); + } catch (IOException e) { + throw new RuntimeException(String.format( + "Bug while encoding using '%s'", encoder.toString()), e); + } + + ByteBuffer encodedBuffer = ByteBuffer.wrap(baos.toByteArray()); + DataBlockEncoder.EncodedSeeker seeker = + encoder.createSeeker(KeyValue.KEY_COMPARATOR, includesMemstoreTS); + seeker.setCurrentBuffer(encodedBuffer); + int i = 0; + do { + KeyValue expectedKeyValue = sampleKv.get(i); + ByteBuffer keyValue = seeker.getKeyValueBuffer(); + if (0 != Bytes.compareTo( + keyValue.array(), keyValue.arrayOffset(), keyValue.limit(), + expectedKeyValue.getBuffer(), expectedKeyValue.getOffset(), + expectedKeyValue.getLength())) { + + int commonPrefix = 0; + byte[] left = keyValue.array(); + byte[] right = expectedKeyValue.getBuffer(); + int leftOff = keyValue.arrayOffset(); + int rightOff = expectedKeyValue.getOffset(); + int length = Math.min(keyValue.limit(), expectedKeyValue.getLength()); + while (commonPrefix < length && + left[commonPrefix + leftOff] == right[commonPrefix + rightOff]) { + commonPrefix++; + } + + fail(String.format( + "next() produces wrong results " + + "encoder: %s i: %d commonPrefix: %d" + + "\n expected %s\n actual %s", + encoder.toString(), i, commonPrefix, + Bytes.toStringBinary(expectedKeyValue.getBuffer(), + expectedKeyValue.getOffset(), expectedKeyValue.getLength()), + Bytes.toStringBinary(keyValue))); + } + i++; + } while (seeker.next()); + } + } + + /** + * Test whether the decompression of first key is implemented correctly. + */ + @Test + public void testFirstKeyInBlockOnSample() { + List sampleKv = generator.generateTestKeyValues(NUMBER_OF_KV); + ByteBuffer originalBuffer = + RedundantKVGenerator.convertKvToByteBuffer(sampleKv, + includesMemstoreTS); + List dataBlockEncoders = + DataBlockEncoding.getAllEncoders(); + + for (DataBlockEncoder encoder : dataBlockEncoders) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dataOut = new DataOutputStream(baos); + try { + encoder.compressKeyValues(dataOut, originalBuffer, includesMemstoreTS); + } catch (IOException e) { + throw new RuntimeException(String.format( + "Bug while encoding using '%s'", encoder.toString()), e); + } + + ByteBuffer encodedBuffer = ByteBuffer.wrap(baos.toByteArray()); + ByteBuffer keyBuffer = encoder.getFirstKeyInBlock(encodedBuffer); + KeyValue firstKv = sampleKv.get(0); + if (0 != Bytes.compareTo( + keyBuffer.array(), keyBuffer.arrayOffset(), keyBuffer.limit(), + firstKv.getBuffer(), firstKv.getKeyOffset(), + firstKv.getKeyLength())) { + + int commonPrefix = 0; + int length = Math.min(keyBuffer.limit(), firstKv.getKeyLength()); + while (commonPrefix < length && + keyBuffer.array()[keyBuffer.arrayOffset() + commonPrefix] == + firstKv.getBuffer()[firstKv.getKeyOffset() + commonPrefix]) { + commonPrefix++; + } + fail(String.format("Bug in '%s' commonPrefix %d", + encoder.toString(), commonPrefix)); + } + } + } + + private void checkSeekingConsistency( + List encodedSeekers, boolean seekBefore, + KeyValue keyValue) { + ByteBuffer expectedKeyValue = null; + ByteBuffer expectedKey = null; + ByteBuffer expectedValue = null; + + for (DataBlockEncoder.EncodedSeeker seeker : encodedSeekers) { + seeker.seekToKeyInBlock(keyValue.getBuffer(), + keyValue.getKeyOffset(), keyValue.getKeyLength(), seekBefore); + seeker.rewind(); + + ByteBuffer actualKeyValue = seeker.getKeyValueBuffer(); + ByteBuffer actualKey = seeker.getKeyDeepCopy(); + ByteBuffer actualValue = seeker.getValueShallowCopy(); + + if (expectedKeyValue != null) { + assertEquals(expectedKeyValue, actualKeyValue); + } else { + expectedKeyValue = actualKeyValue; + } + + if (expectedKey != null) { + assertEquals(expectedKey, actualKey); + } else { + expectedKey = actualKey; + } + + if (expectedValue != null) { + assertEquals(expectedValue, actualValue); + } else { + expectedValue = actualValue; + } + } + } + + private void testEncodersOnDataset(ByteBuffer onDataset) + throws IOException{ + List dataBlockEncoders = + DataBlockEncoding.getAllEncoders(); + ByteBuffer dataset = ByteBuffer.allocate(onDataset.capacity()); + onDataset.rewind(); + dataset.put(onDataset); + onDataset.rewind(); + dataset.flip(); + + for (DataBlockEncoder encoder : dataBlockEncoders) { + testAlgorithm(dataset, encoder); + + // ensure that dataset is unchanged + dataset.rewind(); + assertEquals("Input of two methods is changed", onDataset, dataset); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestEncodedSeekers.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestEncodedSeekers.java new file mode 100644 index 0000000..46b5562 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestEncodedSeekers.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.io.encoding; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.LruBlockCache; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.LoadTestKVGenerator; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Tests encoded seekers by loading and reading values. + */ +@Category(SmallTests.class) +@RunWith(Parameterized.class) +public class TestEncodedSeekers { + + private static final String TABLE_NAME = "encodedSeekersTable"; + private static final String CF_NAME = "encodedSeekersCF"; + private static final byte[] CF_BYTES = Bytes.toBytes(CF_NAME); + private static final int MAX_VERSIONS = 5; + + private static final int MIN_VALUE_SIZE = 30; + private static final int MAX_VALUE_SIZE = 60; + private static final int NUM_ROWS = 1000; + private static final int NUM_COLS_PER_ROW = 20; + private static final int NUM_HFILES = 4; + private static final int NUM_ROWS_PER_FLUSH = NUM_ROWS / NUM_HFILES; + + private final HBaseTestingUtility testUtil = new HBaseTestingUtility(); + private final DataBlockEncoding encoding; + private final boolean encodeOnDisk; + + /** Enable when debugging */ + private static final boolean VERBOSE = false; + + @Parameters + public static Collection parameters() { + List paramList = new ArrayList(); + for (DataBlockEncoding encoding : DataBlockEncoding.values()) { + for (boolean encodeOnDisk : new boolean[]{false, true}) { + paramList.add(new Object[] { encoding, encodeOnDisk }); + } + } + return paramList; + } + + public TestEncodedSeekers(DataBlockEncoding encoding, boolean encodeOnDisk) { + this.encoding = encoding; + this.encodeOnDisk = encodeOnDisk; + } + + @Test + public void testEncodedSeeker() throws IOException { + System.err.println("Testing encoded seekers for encoding " + encoding); + LruBlockCache cache = (LruBlockCache) + new CacheConfig(testUtil.getConfiguration()).getBlockCache(); + cache.clearCache(); + + HRegion region = testUtil.createTestRegion( + TABLE_NAME, new HColumnDescriptor(CF_NAME) + .setMaxVersions(MAX_VERSIONS) + .setDataBlockEncoding(encoding) + .setEncodeOnDisk(encodeOnDisk) + ); + + LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator( + MIN_VALUE_SIZE, MAX_VALUE_SIZE); + + // Write + for (int i = 0; i < NUM_ROWS; ++i) { + byte[] key = LoadTestKVGenerator.md5PrefixedKey(i).getBytes(); + for (int j = 0; j < NUM_COLS_PER_ROW; ++j) { + Put put = new Put(key); + byte[] col = Bytes.toBytes(String.valueOf(j)); + byte[] value = dataGenerator.generateRandomSizeValue(key, col); + put.add(CF_BYTES, col, value); + region.put(put); + } + if (i % NUM_ROWS_PER_FLUSH == 0) { + region.flushcache(); + } + } + + for (int doneCompaction = 0; doneCompaction <= 1; ++doneCompaction) { + // Read + for (int i = 0; i < NUM_ROWS; ++i) { + byte[] rowKey = LoadTestKVGenerator.md5PrefixedKey(i).getBytes(); + for (int j = 0; j < NUM_COLS_PER_ROW; ++j) { + if (VERBOSE) { + System.err.println("Reading row " + i + ", column " + j); + } + final String qualStr = String.valueOf(j); + final byte[] qualBytes = Bytes.toBytes(qualStr); + Get get = new Get(rowKey); + get.addColumn(CF_BYTES, qualBytes); + Result result = region.get(get); + assertEquals(1, result.size()); + byte[] value = result.getValue(CF_BYTES, qualBytes); + assertTrue(LoadTestKVGenerator.verify(value, rowKey, qualBytes)); + } + } + + if (doneCompaction == 0) { + // Compact, then read again at the next loop iteration. + region.compactStores(); + } + } + + Map encodingCounts = + cache.getEncodingCountsForTest(); + + // Ensure that compactions don't pollute the cache with unencoded blocks + // in case of in-cache-only encoding. + System.err.println("encodingCounts=" + encodingCounts); + assertEquals(1, encodingCounts.size()); + DataBlockEncoding encodingInCache = + encodingCounts.keySet().iterator().next(); + assertEquals(encoding, encodingInCache); + assertTrue(encodingCounts.get(encodingInCache) > 0); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestLoadAndSwitchEncodeOnDisk.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestLoadAndSwitchEncodeOnDisk.java new file mode 100644 index 0000000..4e63608 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestLoadAndSwitchEncodeOnDisk.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.io.encoding; + +import java.util.Arrays; +import java.util.Collection; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.TestMiniClusterLoadSequential; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized.Parameters; + +/** + * Uses the load tester + */ +@Category(MediumTests.class) +public class TestLoadAndSwitchEncodeOnDisk extends + TestMiniClusterLoadSequential { + + /** We do not alternate the multi-put flag in this test. */ + private static final boolean USE_MULTI_PUT = true; + + /** Un-parameterize the test */ + @Parameters + public static Collection parameters() { + return Arrays.asList(new Object[][]{ new Object[0] }); + } + + public TestLoadAndSwitchEncodeOnDisk() { + super(USE_MULTI_PUT, DataBlockEncoding.PREFIX); + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true); + } + + protected int numKeys() { + return 3000; + } + + @Test(timeout=TIMEOUT_MS) + public void loadTest() throws Exception { + HBaseAdmin admin = new HBaseAdmin(conf); + + compression = Compression.Algorithm.GZ; // used for table setup + super.loadTest(); + + HColumnDescriptor hcd = getColumnDesc(admin); + System.err.println("\nDisabling encode-on-disk. Old column descriptor: " + + hcd + "\n"); + admin.disableTable(TABLE); + hcd.setEncodeOnDisk(false); + admin.modifyColumn(TABLE, hcd); + + System.err.println("\nRe-enabling table\n"); + admin.enableTable(TABLE); + + System.err.println("\nNew column descriptor: " + + getColumnDesc(admin) + "\n"); + + System.err.println("\nCompacting the table\n"); + admin.majorCompact(TABLE); + // Wait until compaction completes + Threads.sleepWithoutInterrupt(5000); + HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); + while (rs.compactSplitThread.getCompactionQueueSize() > 0) { + Threads.sleep(50); + } + + System.err.println("\nDone with the test, shutting down the cluster\n"); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/encoding/TestUpgradeFromHFileV1ToEncoding.java b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestUpgradeFromHFileV1ToEncoding.java new file mode 100644 index 0000000..b525409 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/encoding/TestUpgradeFromHFileV1ToEncoding.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.io.encoding; + +import static org.apache.hadoop.hbase.io.encoding.TestChangingEncoding.CF; +import static org.apache.hadoop.hbase.io.encoding.TestChangingEncoding.CF_BYTES; + +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestUpgradeFromHFileV1ToEncoding { + + private static final Log LOG = + LogFactory.getLog(TestUpgradeFromHFileV1ToEncoding.class); + + private static final String TABLE = "UpgradeTable"; + private static final byte[] TABLE_BYTES = Bytes.toBytes(TABLE); + + private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final Configuration conf = TEST_UTIL.getConfiguration(); + + private static final int NUM_HFILE_V1_BATCHES = 10; + private static final int NUM_HFILE_V2_BATCHES = 20; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Use a small flush size to create more HFiles. + conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024); + conf.setInt(HFile.FORMAT_VERSION_KEY, 1); // Use HFile v1 initially + TEST_UTIL.startMiniCluster(); + LOG.debug("Started an HFile v1 cluster"); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testUpgrade() throws Exception { + int numBatches = 0; + HTableDescriptor htd = new HTableDescriptor(TABLE); + HColumnDescriptor hcd = new HColumnDescriptor(CF); + htd.addFamily(hcd); + HBaseAdmin admin = new HBaseAdmin(conf); + admin.createTable(htd); + admin.close(); + for (int i = 0; i < NUM_HFILE_V1_BATCHES; ++i) { + TestChangingEncoding.writeTestDataBatch(conf, TABLE, numBatches++); + } + TEST_UTIL.shutdownMiniHBaseCluster(); + + conf.setInt(HFile.FORMAT_VERSION_KEY, 2); + TEST_UTIL.startMiniHBaseCluster(1, 1); + LOG.debug("Started an HFile v2 cluster"); + admin = new HBaseAdmin(conf); + htd = admin.getTableDescriptor(TABLE_BYTES); + hcd = htd.getFamily(CF_BYTES); + hcd.setDataBlockEncoding(DataBlockEncoding.PREFIX); + admin.disableTable(TABLE); + admin.modifyColumn(TABLE, hcd); + admin.enableTable(TABLE); + admin.close(); + for (int i = 0; i < NUM_HFILE_V2_BATCHES; ++i) { + TestChangingEncoding.writeTestDataBatch(conf, TABLE, numBatches++); + } + + LOG.debug("Verifying all 'batches', both HFile v1 and encoded HFile v2"); + verifyBatches(numBatches); + + LOG.debug("Doing a manual compaction"); + admin.compact(TABLE); + Thread.sleep(TimeUnit.SECONDS.toMillis(10)); + + LOG.debug("Verify all the data again"); + verifyBatches(numBatches); + } + + private void verifyBatches(int numBatches) throws Exception { + for (int i = 0; i < numBatches; ++i) { + TestChangingEncoding.verifyTestDataBatch(conf, TABLE, i); + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java new file mode 100644 index 0000000..88b8708 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java @@ -0,0 +1,348 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Random; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.MultithreadedTestUtil; +import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; +import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.util.ChecksumType; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; + +public class CacheTestUtils { + + private static final boolean includesMemstoreTS = true; + + /** + * Just checks if heapsize grows when something is cached, and gets smaller + * when the same object is evicted + */ + + public static void testHeapSizeChanges(final BlockCache toBeTested, + final int blockSize) { + HFileBlockPair[] blocks = generateHFileBlocks(blockSize, 1); + long heapSize = ((HeapSize) toBeTested).heapSize(); + toBeTested.cacheBlock(blocks[0].blockName, blocks[0].block); + + /*When we cache something HeapSize should always increase */ + assertTrue(heapSize < ((HeapSize) toBeTested).heapSize()); + + toBeTested.evictBlock(blocks[0].blockName); + + /*Post eviction, heapsize should be the same */ + assertEquals(heapSize, ((HeapSize) toBeTested).heapSize()); + } + public static void testCacheMultiThreaded(final BlockCache toBeTested, + final int blockSize, final int numThreads, final int numQueries, + final double passingScore) throws Exception { + + Configuration conf = new Configuration(); + MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext( + conf); + + final AtomicInteger totalQueries = new AtomicInteger(); + final ConcurrentLinkedQueue blocksToTest = new ConcurrentLinkedQueue(); + final AtomicInteger hits = new AtomicInteger(); + final AtomicInteger miss = new AtomicInteger(); + + HFileBlockPair[] blocks = generateHFileBlocks(numQueries, blockSize); + blocksToTest.addAll(Arrays.asList(blocks)); + + for (int i = 0; i < numThreads; i++) { + TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) { + @Override + public void doAnAction() throws Exception { + if (!blocksToTest.isEmpty()) { + HFileBlockPair ourBlock = blocksToTest.poll(); + // if we run out of blocks to test, then we should stop the tests. + if (ourBlock == null) { + ctx.setStopFlag(true); + return; + } + toBeTested.cacheBlock(ourBlock.blockName, ourBlock.block); + Cacheable retrievedBlock = toBeTested.getBlock(ourBlock.blockName, + false, false); + if (retrievedBlock != null) { + assertEquals(ourBlock.block, retrievedBlock); + toBeTested.evictBlock(ourBlock.blockName); + hits.incrementAndGet(); + assertNull(toBeTested.getBlock(ourBlock.blockName, false, false)); + } else { + miss.incrementAndGet(); + } + totalQueries.incrementAndGet(); + } + } + }; + t.setDaemon(true); + ctx.addThread(t); + } + ctx.startThreads(); + while (!blocksToTest.isEmpty() && ctx.shouldRun()) { + Thread.sleep(10); + } + ctx.stop(); + if (hits.get() / ((double) hits.get() + (double) miss.get()) < passingScore) { + fail("Too many nulls returned. Hits: " + hits.get() + " Misses: " + + miss.get()); + } + } + + public static void testCacheSimple(BlockCache toBeTested, int blockSize, + int numBlocks) throws Exception { + + HFileBlockPair[] blocks = generateHFileBlocks(numBlocks, blockSize); + // Confirm empty + for (HFileBlockPair block : blocks) { + assertNull(toBeTested.getBlock(block.blockName, true, false)); + } + + // Add blocks + for (HFileBlockPair block : blocks) { + toBeTested.cacheBlock(block.blockName, block.block); + } + + // Check if all blocks are properly cached and contain the right + // information, or the blocks are null. + // MapMaker makes no guarantees when it will evict, so neither can we. + + for (HFileBlockPair block : blocks) { + HFileBlock buf = (HFileBlock) toBeTested.getBlock(block.blockName, true, false); + if (buf != null) { + assertEquals(block.block, buf); + } + + } + + // Re-add some duplicate blocks. Hope nothing breaks. + + for (HFileBlockPair block : blocks) { + try { + if (toBeTested.getBlock(block.blockName, true, false) != null) { + toBeTested.cacheBlock(block.blockName, block.block); + fail("Cache should not allow re-caching a block"); + } + } catch (RuntimeException re) { + // expected + } + } + + } + + public static void hammerSingleKey(final BlockCache toBeTested, + int BlockSize, int numThreads, int numQueries) throws Exception { + final BlockCacheKey key = new BlockCacheKey("key", 0); + final byte[] buf = new byte[5 * 1024]; + Arrays.fill(buf, (byte) 5); + + final ByteArrayCacheable bac = new ByteArrayCacheable(buf); + Configuration conf = new Configuration(); + MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext( + conf); + + final AtomicInteger totalQueries = new AtomicInteger(); + toBeTested.cacheBlock(key, bac); + + for (int i = 0; i < numThreads; i++) { + TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) { + @Override + public void doAnAction() throws Exception { + ByteArrayCacheable returned = (ByteArrayCacheable) toBeTested + .getBlock(key, false, false); + assertArrayEquals(buf, returned.buf); + totalQueries.incrementAndGet(); + } + }; + + t.setDaemon(true); + ctx.addThread(t); + } + + ctx.startThreads(); + while (totalQueries.get() < numQueries && ctx.shouldRun()) { + Thread.sleep(10); + } + ctx.stop(); + } + + public static void hammerEviction(final BlockCache toBeTested, int BlockSize, + int numThreads, int numQueries) throws Exception { + + Configuration conf = new Configuration(); + MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext( + conf); + + final AtomicInteger totalQueries = new AtomicInteger(); + + for (int i = 0; i < numThreads; i++) { + final int finalI = i; + + final byte[] buf = new byte[5 * 1024]; + TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) { + @Override + public void doAnAction() throws Exception { + for (int j = 0; j < 100; j++) { + BlockCacheKey key = new BlockCacheKey("key_" + finalI + "_" + j, 0); + Arrays.fill(buf, (byte) (finalI * j)); + final ByteArrayCacheable bac = new ByteArrayCacheable(buf); + + ByteArrayCacheable gotBack = (ByteArrayCacheable) toBeTested + .getBlock(key, true, false); + if (gotBack != null) { + assertArrayEquals(gotBack.buf, bac.buf); + } else { + toBeTested.cacheBlock(key, bac); + } + } + totalQueries.incrementAndGet(); + } + }; + + t.setDaemon(true); + ctx.addThread(t); + } + + ctx.startThreads(); + while (totalQueries.get() < numQueries && ctx.shouldRun()) { + Thread.sleep(10); + } + ctx.stop(); + + assertTrue(toBeTested.getStats().getEvictedCount() > 0); + } + + private static class ByteArrayCacheable implements Cacheable { + + final byte[] buf; + + public ByteArrayCacheable(byte[] buf) { + this.buf = buf; + } + + @Override + public long heapSize() { + return 4 + buf.length; + } + + @Override + public int getSerializedLength() { + return 4 + buf.length; + } + + @Override + public void serialize(ByteBuffer destination) { + destination.putInt(buf.length); + Thread.yield(); + destination.put(buf); + destination.rewind(); + } + + @Override + public CacheableDeserializer getDeserializer() { + return new CacheableDeserializer() { + + @Override + public Cacheable deserialize(ByteBuffer b) throws IOException { + int len = b.getInt(); + Thread.yield(); + byte buf[] = new byte[len]; + b.get(buf); + return new ByteArrayCacheable(buf); + } + }; + } + + @Override + public BlockType getBlockType() { + return BlockType.DATA; + } + + @Override + public SchemaMetrics getSchemaMetrics() { + return SchemaMetrics.getUnknownInstanceForTest(); + } + + } + + private static HFileBlockPair[] generateHFileBlocks(int blockSize, + int numBlocks) { + HFileBlockPair[] returnedBlocks = new HFileBlockPair[numBlocks]; + Random rand = new Random(); + HashSet usedStrings = new HashSet(); + for (int i = 0; i < numBlocks; i++) { + + // The buffer serialized size needs to match the size of BlockSize. So we + // declare our data size to be smaller than it by the serialization space + // required. + + ByteBuffer cachedBuffer = ByteBuffer.allocate(blockSize + - HFileBlock.EXTRA_SERIALIZATION_SPACE); + rand.nextBytes(cachedBuffer.array()); + cachedBuffer.rewind(); + int onDiskSizeWithoutHeader = blockSize + - HFileBlock.EXTRA_SERIALIZATION_SPACE; + int uncompressedSizeWithoutHeader = blockSize + - HFileBlock.EXTRA_SERIALIZATION_SPACE; + long prevBlockOffset = rand.nextLong(); + BlockType.DATA.write(cachedBuffer); + cachedBuffer.putInt(onDiskSizeWithoutHeader); + cachedBuffer.putInt(uncompressedSizeWithoutHeader); + cachedBuffer.putLong(prevBlockOffset); + cachedBuffer.rewind(); + + HFileBlock generated = new HFileBlock(BlockType.DATA, + onDiskSizeWithoutHeader, uncompressedSizeWithoutHeader, + prevBlockOffset, cachedBuffer, HFileBlock.DONT_FILL_HEADER, + blockSize, includesMemstoreTS, HFileBlock.MINOR_VERSION_NO_CHECKSUM, + 0, ChecksumType.NULL.getCode(), + onDiskSizeWithoutHeader + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS); + + String strKey; + /* No conflicting keys */ + for (strKey = new Long(rand.nextLong()).toString(); !usedStrings + .add(strKey); strKey = new Long(rand.nextLong()).toString()) + ; + + returnedBlocks[i] = new HFileBlockPair(); + returnedBlocks[i].blockName = new BlockCacheKey(strKey, 0); + returnedBlocks[i].block = generated; + } + return returnedBlocks; + } + + private static class HFileBlockPair { + BlockCacheKey blockName; + HFileBlock block; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/KVGenerator.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/KVGenerator.java new file mode 100644 index 0000000..b22cb8c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/KVGenerator.java @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.util.Random; + +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.WritableComparator; + +/** + * Generate random pairs. + *

    + * Copied from + * hadoop-3315 tfile. + * Remove after tfile is committed and use the tfile version of this class + * instead.

    + */ +class KVGenerator { + private final Random random; + private final byte[][] dict; + private final boolean sorted; + private final RandomDistribution.DiscreteRNG keyLenRNG, valLenRNG; + private BytesWritable lastKey; + private static final int MIN_KEY_LEN = 4; + private final byte prefix[] = new byte[MIN_KEY_LEN]; + + public KVGenerator(Random random, boolean sorted, + RandomDistribution.DiscreteRNG keyLenRNG, + RandomDistribution.DiscreteRNG valLenRNG, + RandomDistribution.DiscreteRNG wordLenRNG, int dictSize) { + this.random = random; + dict = new byte[dictSize][]; + this.sorted = sorted; + this.keyLenRNG = keyLenRNG; + this.valLenRNG = valLenRNG; + for (int i = 0; i < dictSize; ++i) { + int wordLen = wordLenRNG.nextInt(); + dict[i] = new byte[wordLen]; + random.nextBytes(dict[i]); + } + lastKey = new BytesWritable(); + fillKey(lastKey); + } + + private void fillKey(BytesWritable o) { + int len = keyLenRNG.nextInt(); + if (len < MIN_KEY_LEN) len = MIN_KEY_LEN; + o.setSize(len); + int n = MIN_KEY_LEN; + while (n < len) { + byte[] word = dict[random.nextInt(dict.length)]; + int l = Math.min(word.length, len - n); + System.arraycopy(word, 0, o.get(), n, l); + n += l; + } + if (sorted + && WritableComparator.compareBytes(lastKey.get(), MIN_KEY_LEN, lastKey + .getSize() + - MIN_KEY_LEN, o.get(), MIN_KEY_LEN, o.getSize() - MIN_KEY_LEN) > 0) { + incrementPrefix(); + } + + System.arraycopy(prefix, 0, o.get(), 0, MIN_KEY_LEN); + lastKey.set(o); + } + + private void fillValue(BytesWritable o) { + int len = valLenRNG.nextInt(); + o.setSize(len); + int n = 0; + while (n < len) { + byte[] word = dict[random.nextInt(dict.length)]; + int l = Math.min(word.length, len - n); + System.arraycopy(word, 0, o.get(), n, l); + n += l; + } + } + + private void incrementPrefix() { + for (int i = MIN_KEY_LEN - 1; i >= 0; --i) { + ++prefix[i]; + if (prefix[i] != 0) return; + } + + throw new RuntimeException("Prefix overflown"); + } + + public void next(BytesWritable key, BytesWritable value, boolean dupKey) { + if (dupKey) { + key.set(lastKey); + } + else { + fillKey(key); + } + fillValue(value); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/KeySampler.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/KeySampler.java new file mode 100644 index 0000000..2489029 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/KeySampler.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.util.Random; + +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.hbase.io.hfile.RandomDistribution.DiscreteRNG; + +/* +*

    +* Copied from +* hadoop-3315 tfile. +* Remove after tfile is committed and use the tfile version of this class +* instead.

    +*/ +class KeySampler { + Random random; + int min, max; + DiscreteRNG keyLenRNG; + private static final int MIN_KEY_LEN = 4; + + public KeySampler(Random random, byte [] first, byte [] last, + DiscreteRNG keyLenRNG) { + this.random = random; + min = keyPrefixToInt(first); + max = keyPrefixToInt(last); + this.keyLenRNG = keyLenRNG; + } + + private int keyPrefixToInt(byte [] key) { + byte[] b = key; + int o = 0; + return (b[o] & 0xff) << 24 | (b[o + 1] & 0xff) << 16 + | (b[o + 2] & 0xff) << 8 | (b[o + 3] & 0xff); + } + + public void next(BytesWritable key) { + key.setSize(Math.max(MIN_KEY_LEN, keyLenRNG.nextInt())); + random.nextBytes(key.get()); + int n = random.nextInt(max - min) + min; + byte[] b = key.get(); + b[0] = (byte) (n >> 24); + b[1] = (byte) (n >> 16); + b[2] = (byte) (n >> 8); + b[3] = (byte) n; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/NanoTimer.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/NanoTimer.java new file mode 100644 index 0000000..a133cb4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/NanoTimer.java @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +/** + * A nano-second timer. + *

    + * Copied from + * hadoop-3315 tfile. + * Remove after tfile is committed and use the tfile version of this class + * instead.

    + */ +public class NanoTimer { + private long last = -1; + private boolean started = false; + private long cumulate = 0; + + /** + * Constructor + * + * @param start + * Start the timer upon construction. + */ + public NanoTimer(boolean start) { + if (start) this.start(); + } + + /** + * Start the timer. + * + * Note: No effect if timer is already started. + */ + public void start() { + if (!this.started) { + this.last = System.nanoTime(); + this.started = true; + } + } + + /** + * Stop the timer. + * + * Note: No effect if timer is already stopped. + */ + public void stop() { + if (this.started) { + this.started = false; + this.cumulate += System.nanoTime() - this.last; + } + } + + /** + * Read the timer. + * + * @return the elapsed time in nano-seconds. Note: If the timer is never + * started before, -1 is returned. + */ + public long read() { + if (!readable()) return -1; + + return this.cumulate; + } + + /** + * Reset the timer. + */ + public void reset() { + this.last = -1; + this.started = false; + this.cumulate = 0; + } + + /** + * Checking whether the timer is started + * + * @return true if timer is started. + */ + public boolean isStarted() { + return this.started; + } + + /** + * Format the elapsed time to a human understandable string. + * + * Note: If timer is never started, "ERR" will be returned. + */ + public String toString() { + if (!readable()) { + return "ERR"; + } + + return NanoTimer.nanoTimeToString(this.cumulate); + } + + /** + * A utility method to format a time duration in nano seconds into a human + * understandable stirng. + * + * @param t + * Time duration in nano seconds. + * @return String representation. + */ + public static String nanoTimeToString(long t) { + if (t < 0) return "ERR"; + + if (t == 0) return "0"; + + if (t < 1000) { + return t + "ns"; + } + + double us = (double) t / 1000; + if (us < 1000) { + return String.format("%.2fus", us); + } + + double ms = us / 1000; + if (ms < 1000) { + return String.format("%.2fms", ms); + } + + double ss = ms / 1000; + if (ss < 1000) { + return String.format("%.2fs", ss); + } + + long mm = (long) ss / 60; + ss -= mm * 60; + long hh = mm / 60; + mm -= hh * 60; + long dd = hh / 24; + hh -= dd * 24; + + if (dd > 0) { + return String.format("%dd%dh", dd, hh); + } + + if (hh > 0) { + return String.format("%dh%dm", hh, mm); + } + + if (mm > 0) { + return String.format("%dm%.1fs", mm, ss); + } + + return String.format("%.2fs", ss); + + /** + * StringBuilder sb = new StringBuilder(); String sep = ""; + * + * if (dd > 0) { String unit = (dd > 1) ? "days" : "day"; + * sb.append(String.format("%s%d%s", sep, dd, unit)); sep = " "; } + * + * if (hh > 0) { String unit = (hh > 1) ? "hrs" : "hr"; + * sb.append(String.format("%s%d%s", sep, hh, unit)); sep = " "; } + * + * if (mm > 0) { String unit = (mm > 1) ? "mins" : "min"; + * sb.append(String.format("%s%d%s", sep, mm, unit)); sep = " "; } + * + * if (ss > 0) { String unit = (ss > 1) ? "secs" : "sec"; + * sb.append(String.format("%s%.3f%s", sep, ss, unit)); sep = " "; } + * + * return sb.toString(); + */ + } + + private boolean readable() { + return this.last != -1; + } + + /** + * Simple tester. + * + * @param args + */ + public static void main(String[] args) { + long i = 7; + + for (int x = 0; x < 20; ++x, i *= 7) { + System.out.println(NanoTimer.nanoTimeToString(i)); + } + } +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomDistribution.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomDistribution.java new file mode 100644 index 0000000..7232cad --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomDistribution.java @@ -0,0 +1,271 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Random; + +/** + * A class that generates random numbers that follow some distribution. + *

    + * Copied from + * hadoop-3315 tfile. + * Remove after tfile is committed and use the tfile version of this class + * instead.

    + */ +public class RandomDistribution { + /** + * Interface for discrete (integer) random distributions. + */ + public static interface DiscreteRNG { + /** + * Get the next random number + * + * @return the next random number. + */ + public int nextInt(); + } + + /** + * P(i)=1/(max-min) + */ + public static final class Flat implements DiscreteRNG { + private final Random random; + private final int min; + private final int max; + + /** + * Generate random integers from min (inclusive) to max (exclusive) + * following even distribution. + * + * @param random + * The basic random number generator. + * @param min + * Minimum integer + * @param max + * maximum integer (exclusive). + * + */ + public Flat(Random random, int min, int max) { + if (min >= max) { + throw new IllegalArgumentException("Invalid range"); + } + this.random = random; + this.min = min; + this.max = max; + } + + /** + * @see DiscreteRNG#nextInt() + */ + @Override + public int nextInt() { + return random.nextInt(max - min) + min; + } + } + + /** + * Zipf distribution. The ratio of the probabilities of integer i and j is + * defined as follows: + * + * P(i)/P(j)=((j-min+1)/(i-min+1))^sigma. + */ + public static final class Zipf implements DiscreteRNG { + private static final double DEFAULT_EPSILON = 0.001; + private final Random random; + private final ArrayList k; + private final ArrayList v; + + /** + * Constructor + * + * @param r + * The random number generator. + * @param min + * minimum integer (inclusvie) + * @param max + * maximum integer (exclusive) + * @param sigma + * parameter sigma. (sigma > 1.0) + */ + public Zipf(Random r, int min, int max, double sigma) { + this(r, min, max, sigma, DEFAULT_EPSILON); + } + + /** + * Constructor. + * + * @param r + * The random number generator. + * @param min + * minimum integer (inclusvie) + * @param max + * maximum integer (exclusive) + * @param sigma + * parameter sigma. (sigma > 1.0) + * @param epsilon + * Allowable error percentage (0 < epsilon < 1.0). + */ + public Zipf(Random r, int min, int max, double sigma, double epsilon) { + if ((max <= min) || (sigma <= 1) || (epsilon <= 0) + || (epsilon >= 0.5)) { + throw new IllegalArgumentException("Invalid arguments"); + } + random = r; + k = new ArrayList(); + v = new ArrayList(); + + double sum = 0; + int last = -1; + for (int i = min; i < max; ++i) { + sum += Math.exp(-sigma * Math.log(i - min + 1)); + if ((last == -1) || i * (1 - epsilon) > last) { + k.add(i); + v.add(sum); + last = i; + } + } + + if (last != max - 1) { + k.add(max - 1); + v.add(sum); + } + + v.set(v.size() - 1, 1.0); + + for (int i = v.size() - 2; i >= 0; --i) { + v.set(i, v.get(i) / sum); + } + } + + /** + * @see DiscreteRNG#nextInt() + */ + @Override + public int nextInt() { + double d = random.nextDouble(); + int idx = Collections.binarySearch(v, d); + + if (idx > 0) { + ++idx; + } + else { + idx = -(idx + 1); + } + + if (idx >= v.size()) { + idx = v.size() - 1; + } + + if (idx == 0) { + return k.get(0); + } + + int ceiling = k.get(idx); + int lower = k.get(idx - 1); + + return ceiling - random.nextInt(ceiling - lower); + } + } + + /** + * Binomial distribution. + * + * P(k)=select(n, k)*p^k*(1-p)^(n-k) (k = 0, 1, ..., n) + * + * P(k)=select(max-min-1, k-min)*p^(k-min)*(1-p)^(k-min)*(1-p)^(max-k-1) + */ + public static final class Binomial implements DiscreteRNG { + private final Random random; + private final int min; + private final int n; + private final double[] v; + + private static double select(int n, int k) { + double ret = 1.0; + for (int i = k + 1; i <= n; ++i) { + ret *= (double) i / (i - k); + } + return ret; + } + + private static double power(double p, int k) { + return Math.exp(k * Math.log(p)); + } + + /** + * Generate random integers from min (inclusive) to max (exclusive) + * following Binomial distribution. + * + * @param random + * The basic random number generator. + * @param min + * Minimum integer + * @param max + * maximum integer (exclusive). + * @param p + * parameter. + * + */ + public Binomial(Random random, int min, int max, double p) { + if (min >= max) { + throw new IllegalArgumentException("Invalid range"); + } + this.random = random; + this.min = min; + this.n = max - min - 1; + if (n > 0) { + v = new double[n + 1]; + double sum = 0.0; + for (int i = 0; i <= n; ++i) { + sum += select(n, i) * power(p, i) * power(1 - p, n - i); + v[i] = sum; + } + for (int i = 0; i <= n; ++i) { + v[i] /= sum; + } + } + else { + v = null; + } + } + + /** + * @see DiscreteRNG#nextInt() + */ + @Override + public int nextInt() { + if (v == null) { + return min; + } + double d = random.nextDouble(); + int idx = Arrays.binarySearch(v, d); + if (idx > 0) { + ++idx; + } else { + idx = -(idx + 1); + } + + if (idx >= v.length) { + idx = v.length - 1; + } + return idx + min; + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomSeek.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomSeek.java new file mode 100644 index 0000000..a48a69f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomSeek.java @@ -0,0 +1,129 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.hbase.io.hfile.HFile.Reader; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Random seek test. + */ +public class RandomSeek { + private static List slurp(String fname) throws IOException { + BufferedReader istream = new BufferedReader(new FileReader(fname)); + String str; + List l = new ArrayList(); + while ( (str=istream.readLine()) != null) { + String [] parts = str.split(","); + l.add(parts[0] + ":" + parts[1] + ":" + parts[2]); + } + istream.close(); + return l; + } + + private static String randKey(List keys) { + Random r = new Random(); + //return keys.get(r.nextInt(keys.size())); + return "2" + Integer.toString(7+r.nextInt(2)) + Integer.toString(r.nextInt(100)); + //return new String(r.nextInt(100)); + } + + public static void main(String [] argv) throws IOException { + Configuration conf = new Configuration(); + conf.setInt("io.file.buffer.size", 64*1024); + RawLocalFileSystem rlfs = new RawLocalFileSystem(); + rlfs.setConf(conf); + LocalFileSystem lfs = new LocalFileSystem(rlfs); + + Path path = new Path("/Users/ryan/rfile.big.txt"); + long start = System.currentTimeMillis(); + SimpleBlockCache cache = new SimpleBlockCache(); + CacheConfig cacheConf = new CacheConfig(cache, true, false, false, false, + false, false, false); + + Reader reader = HFile.createReader(lfs, path, cacheConf); + reader.loadFileInfo(); + System.out.println(reader.getTrailer()); + long end = System.currentTimeMillis(); + + System.out.println("Index read time: " + (end - start)); + + List keys = slurp("/Users/ryan/xaa.50k"); + + // Get a scanner that doesn't cache and that uses pread. + HFileScanner scanner = reader.getScanner(false, true); + int count; + long totalBytes = 0; + int notFound = 0; + + start = System.nanoTime(); + for(count = 0; count < 500000; ++count) { + String key = randKey(keys); + byte [] bkey = Bytes.toBytes(key); + int res = scanner.seekTo(bkey); + if (res == 0) { + ByteBuffer k = scanner.getKey(); + ByteBuffer v = scanner.getValue(); + totalBytes += k.limit(); + totalBytes += v.limit(); + } else { + ++ notFound; + } + if (res == -1) { + scanner.seekTo(); + } + // Scan for another 1000 rows. + for (int i = 0; i < 1000; ++i) { + if (!scanner.next()) + break; + ByteBuffer k = scanner.getKey(); + ByteBuffer v = scanner.getValue(); + totalBytes += k.limit(); + totalBytes += v.limit(); + } + + if ( count % 1000 == 0 ) { + end = System.nanoTime(); + + System.out.println("Cache block count: " + cache.size() + " dumped: "+ cache.dumps); + //System.out.println("Cache size: " + cache.heapSize()); + double msTime = ((end - start) / 1000000.0); + System.out.println("Seeked: "+ count + " in " + msTime + " (ms) " + + (1000.0 / msTime ) + " seeks/ms " + + (msTime / 1000.0) + " ms/seek"); + + start = System.nanoTime(); + } + } + System.out.println("Total bytes: " + totalBytes + " not found: " + notFound); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheColumnFamilySummary.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheColumnFamilySummary.java new file mode 100644 index 0000000..7673806 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheColumnFamilySummary.java @@ -0,0 +1,128 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests the BlockCacheColumnFamilySummary class + * + */ +@Category(SmallTests.class) +public class TestBlockCacheColumnFamilySummary { + + + /** + * + */ + @Test + public void testEquals() { + + BlockCacheColumnFamilySummary e1 = new BlockCacheColumnFamilySummary(); + e1.setTable("table1"); + e1.setColumnFamily("cf1"); + + BlockCacheColumnFamilySummary e2 = new BlockCacheColumnFamilySummary(); + e2.setTable("table1"); + e2.setColumnFamily("cf1"); + + assertEquals("bcse", e1, e2); + } + + /** + * + */ + @Test + public void testNotEquals() { + + BlockCacheColumnFamilySummary e1 = new BlockCacheColumnFamilySummary(); + e1.setTable("table1"); + e1.setColumnFamily("cf1"); + + BlockCacheColumnFamilySummary e2 = new BlockCacheColumnFamilySummary(); + e2.setTable("tablexxxxxx"); + e2.setColumnFamily("cf1"); + + assertTrue("bcse", ! e1.equals(e2)); + } + + /** + * + */ + @Test + public void testMapLookup() { + + Map bcs = + new HashMap(); + + BlockCacheColumnFamilySummary e1 = new BlockCacheColumnFamilySummary("table1","cf1"); + + BlockCacheColumnFamilySummary lookup = bcs.get(e1); + + if (lookup == null) { + lookup = BlockCacheColumnFamilySummary.create(e1); + bcs.put(e1,lookup); + lookup.incrementBlocks(); + lookup.incrementHeapSize(100L); + } + + BlockCacheColumnFamilySummary e2 = new BlockCacheColumnFamilySummary("table1","cf1"); + + BlockCacheColumnFamilySummary l2 = bcs.get(e2); + assertEquals("blocks",1,l2.getBlocks()); + assertEquals("heap",100L,l2.getHeapSize()); + } + + /** + * + */ + @Test + public void testMapEntry() { + + Map bcs = + new HashMap(); + + BlockCacheColumnFamilySummary e1 = new BlockCacheColumnFamilySummary("table1","cf1"); + bcs.put(e1, e1); + + BlockCacheColumnFamilySummary e2 = new BlockCacheColumnFamilySummary("table1","cf1"); + bcs.put(e2, e2); + + BlockCacheColumnFamilySummary e3 = new BlockCacheColumnFamilySummary("table1","cf1"); + bcs.put(e3, e3); + + assertEquals("mapSize",1,bcs.size()); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java new file mode 100644 index 0000000..aa20de3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java @@ -0,0 +1,383 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.util.BloomFilterFactory; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ChecksumType; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Tests {@link HFile} cache-on-write functionality for the following block + * types: data blocks, non-root index blocks, and Bloom filter blocks. + */ +@RunWith(Parameterized.class) +@Category(MediumTests.class) +public class TestCacheOnWrite { + + private static final Log LOG = LogFactory.getLog(TestCacheOnWrite.class); + + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + private Configuration conf; + private CacheConfig cacheConf; + private FileSystem fs; + private Random rand = new Random(12983177L); + private Path storeFilePath; + private BlockCache blockCache; + private String testDescription; + + private final CacheOnWriteType cowType; + private final Compression.Algorithm compress; + private final BlockEncoderTestType encoderType; + private final HFileDataBlockEncoder encoder; + + private static final int DATA_BLOCK_SIZE = 2048; + private static final int NUM_KV = 25000; + private static final int INDEX_BLOCK_SIZE = 512; + private static final int BLOOM_BLOCK_SIZE = 4096; + private static final BloomType BLOOM_TYPE = StoreFile.BloomType.ROWCOL; + private static final ChecksumType CKTYPE = ChecksumType.CRC32; + private static final int CKBYTES = 512; + + /** The number of valid key types possible in a store file */ + private static final int NUM_VALID_KEY_TYPES = + KeyValue.Type.values().length - 2; + + private static enum CacheOnWriteType { + DATA_BLOCKS(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, + BlockType.DATA, BlockType.ENCODED_DATA), + BLOOM_BLOCKS(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, + BlockType.BLOOM_CHUNK), + INDEX_BLOCKS(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, + BlockType.LEAF_INDEX, BlockType.INTERMEDIATE_INDEX); + + private final String confKey; + private final BlockType blockType1; + private final BlockType blockType2; + + private CacheOnWriteType(String confKey, BlockType blockType) { + this(confKey, blockType, blockType); + } + + private CacheOnWriteType(String confKey, BlockType blockType1, + BlockType blockType2) { + this.blockType1 = blockType1; + this.blockType2 = blockType2; + this.confKey = confKey; + } + + public boolean shouldBeCached(BlockType blockType) { + return blockType == blockType1 || blockType == blockType2; + } + + public void modifyConf(Configuration conf) { + for (CacheOnWriteType cowType : CacheOnWriteType.values()) { + conf.setBoolean(cowType.confKey, cowType == this); + } + } + + } + + private static final DataBlockEncoding ENCODING_ALGO = + DataBlockEncoding.PREFIX; + + /** Provides fancy names for three combinations of two booleans */ + private static enum BlockEncoderTestType { + NO_BLOCK_ENCODING(false, false), + BLOCK_ENCODING_IN_CACHE_ONLY(false, true), + BLOCK_ENCODING_EVERYWHERE(true, true); + + private final boolean encodeOnDisk; + private final boolean encodeInCache; + + BlockEncoderTestType(boolean encodeOnDisk, boolean encodeInCache) { + this.encodeOnDisk = encodeOnDisk; + this.encodeInCache = encodeInCache; + } + + public HFileDataBlockEncoder getEncoder() { + return new HFileDataBlockEncoderImpl( + encodeOnDisk ? ENCODING_ALGO : DataBlockEncoding.NONE, + encodeInCache ? ENCODING_ALGO : DataBlockEncoding.NONE); + } + } + + public TestCacheOnWrite(CacheOnWriteType cowType, + Compression.Algorithm compress, BlockEncoderTestType encoderType) { + this.cowType = cowType; + this.compress = compress; + this.encoderType = encoderType; + this.encoder = encoderType.getEncoder(); + testDescription = "[cacheOnWrite=" + cowType + ", compress=" + compress + + ", encoderType=" + encoderType + "]"; + System.out.println(testDescription); + } + + @Parameters + public static Collection getParameters() { + List cowTypes = new ArrayList(); + for (CacheOnWriteType cowType : CacheOnWriteType.values()) { + for (Compression.Algorithm compress : + HBaseTestingUtility.COMPRESSION_ALGORITHMS) { + for (BlockEncoderTestType encoderType : + BlockEncoderTestType.values()) { + cowTypes.add(new Object[] { cowType, compress, encoderType }); + } + } + } + return cowTypes; + } + + @Before + public void setUp() throws IOException { + conf = TEST_UTIL.getConfiguration(); + conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION); + conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, INDEX_BLOCK_SIZE); + conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, + BLOOM_BLOCK_SIZE); + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, + cowType.shouldBeCached(BlockType.DATA)); + conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, + cowType.shouldBeCached(BlockType.LEAF_INDEX)); + conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, + cowType.shouldBeCached(BlockType.BLOOM_CHUNK)); + cowType.modifyConf(conf); + fs = HFileSystem.get(conf); + cacheConf = new CacheConfig(conf); + blockCache = cacheConf.getBlockCache(); + } + + @After + public void tearDown() { + cacheConf = new CacheConfig(conf); + blockCache = cacheConf.getBlockCache(); + } + + @Test + public void testStoreFileCacheOnWrite() throws IOException { + writeStoreFile(); + readStoreFile(); + } + + private void readStoreFile() throws IOException { + HFileReaderV2 reader = (HFileReaderV2) HFile.createReaderWithEncoding(fs, + storeFilePath, cacheConf, encoder.getEncodingInCache()); + LOG.info("HFile information: " + reader); + final boolean cacheBlocks = false; + final boolean pread = false; + HFileScanner scanner = reader.getScanner(cacheBlocks, pread); + assertTrue(testDescription, scanner.seekTo()); + + long offset = 0; + HFileBlock prevBlock = null; + EnumMap blockCountByType = + new EnumMap(BlockType.class); + + DataBlockEncoding encodingInCache = + encoderType.getEncoder().getEncodingInCache(); + while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { + long onDiskSize = -1; + if (prevBlock != null) { + onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader(); + } + // Flags: don't cache the block, use pread, this is not a compaction. + // Also, pass null for expected block type to avoid checking it. + HFileBlock block = reader.readBlock(offset, onDiskSize, false, true, + false, null); + BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(), + offset, encodingInCache, block.getBlockType()); + boolean isCached = blockCache.getBlock(blockCacheKey, true, false) != null; + boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType()); + if (shouldBeCached != isCached) { + throw new AssertionError( + "shouldBeCached: " + shouldBeCached+ "\n" + + "isCached: " + isCached + "\n" + + "Test description: " + testDescription + "\n" + + "block: " + block + "\n" + + "encodingInCache: " + encodingInCache + "\n" + + "blockCacheKey: " + blockCacheKey); + } + prevBlock = block; + offset += block.getOnDiskSizeWithHeader(); + BlockType bt = block.getBlockType(); + Integer count = blockCountByType.get(bt); + blockCountByType.put(bt, (count == null ? 0 : count) + 1); + } + + LOG.info("Block count by type: " + blockCountByType); + String countByType = blockCountByType.toString(); + BlockType cachedDataBlockType = + encoderType.encodeInCache ? BlockType.ENCODED_DATA : BlockType.DATA; + assertEquals("{" + cachedDataBlockType + + "=1379, LEAF_INDEX=173, BLOOM_CHUNK=9, INTERMEDIATE_INDEX=24}", + countByType); + + reader.close(); + } + + public static KeyValue.Type generateKeyType(Random rand) { + if (rand.nextBoolean()) { + // Let's make half of KVs puts. + return KeyValue.Type.Put; + } else { + KeyValue.Type keyType = + KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)]; + if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) + { + throw new RuntimeException("Generated an invalid key type: " + keyType + + ". " + "Probably the layout of KeyValue.Type has changed."); + } + return keyType; + } + } + + public void writeStoreFile() throws IOException { + Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), + "test_cache_on_write"); + StoreFile.Writer sfw = new StoreFile.WriterBuilder(conf, cacheConf, fs, + DATA_BLOCK_SIZE) + .withOutputDir(storeFileParentDir) + .withCompression(compress) + .withDataBlockEncoder(encoder) + .withComparator(KeyValue.COMPARATOR) + .withBloomType(BLOOM_TYPE) + .withMaxKeyCount(NUM_KV) + .withChecksumType(CKTYPE) + .withBytesPerChecksum(CKBYTES) + .build(); + + final int rowLen = 32; + for (int i = 0; i < NUM_KV; ++i) { + byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i); + byte[] v = TestHFileWriterV2.randomValue(rand); + int cfLen = rand.nextInt(k.length - rowLen + 1); + KeyValue kv = new KeyValue( + k, 0, rowLen, + k, rowLen, cfLen, + k, rowLen + cfLen, k.length - rowLen - cfLen, + rand.nextLong(), + generateKeyType(rand), + v, 0, v.length); + sfw.append(kv); + } + + sfw.close(); + storeFilePath = sfw.getPath(); + } + + @Test + public void testNotCachingDataBlocksDuringCompaction() throws IOException { + // TODO: need to change this test if we add a cache size threshold for + // compactions, or if we implement some other kind of intelligent logic for + // deciding what blocks to cache-on-write on compaction. + final String table = "CompactionCacheOnWrite"; + final String cf = "myCF"; + final byte[] cfBytes = Bytes.toBytes(cf); + final int maxVersions = 3; + HRegion region = TEST_UTIL.createTestRegion(table, + new HColumnDescriptor(cf) + .setCompressionType(compress) + .setBloomFilterType(BLOOM_TYPE) + .setMaxVersions(maxVersions) + .setDataBlockEncoding(encoder.getEncodingInCache()) + .setEncodeOnDisk(encoder.getEncodingOnDisk() != + DataBlockEncoding.NONE) + ); + int rowIdx = 0; + long ts = EnvironmentEdgeManager.currentTimeMillis(); + for (int iFile = 0; iFile < 5; ++iFile) { + for (int iRow = 0; iRow < 500; ++iRow) { + String rowStr = "" + (rowIdx * rowIdx * rowIdx) + "row" + iFile + "_" + + iRow; + Put p = new Put(Bytes.toBytes(rowStr)); + ++rowIdx; + for (int iCol = 0; iCol < 10; ++iCol) { + String qualStr = "col" + iCol; + String valueStr = "value_" + rowStr + "_" + qualStr; + for (int iTS = 0; iTS < 5; ++iTS) { + p.add(cfBytes, Bytes.toBytes(qualStr), ts++, + Bytes.toBytes(valueStr)); + } + } + region.put(p); + } + region.flushcache(); + } + LruBlockCache blockCache = + (LruBlockCache) new CacheConfig(conf).getBlockCache(); + blockCache.clearCache(); + assertEquals(0, blockCache.getBlockTypeCountsForTest().size()); + Map metricsBefore = SchemaMetrics.getMetricsSnapshot(); + region.compactStores(); + LOG.debug("compactStores() returned"); + SchemaMetrics.validateMetricChanges(metricsBefore); + Map compactionMetrics = SchemaMetrics.diffMetrics( + metricsBefore, SchemaMetrics.getMetricsSnapshot()); + LOG.debug(SchemaMetrics.formatMetrics(compactionMetrics)); + Map blockTypesInCache = + blockCache.getBlockTypeCountsForTest(); + LOG.debug("Block types in cache: " + blockTypesInCache); + assertNull(blockTypesInCache.get(BlockType.DATA)); + region.close(); + blockCache.shutdown(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java new file mode 100644 index 0000000..0dbe3e3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java @@ -0,0 +1,158 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestCachedBlockQueue extends TestCase { + + public void testQueue() throws Exception { + + CachedBlock cb1 = new CachedBlock(1000, "cb1", 1); + CachedBlock cb2 = new CachedBlock(1500, "cb2", 2); + CachedBlock cb3 = new CachedBlock(1000, "cb3", 3); + CachedBlock cb4 = new CachedBlock(1500, "cb4", 4); + CachedBlock cb5 = new CachedBlock(1000, "cb5", 5); + CachedBlock cb6 = new CachedBlock(1750, "cb6", 6); + CachedBlock cb7 = new CachedBlock(1000, "cb7", 7); + CachedBlock cb8 = new CachedBlock(1500, "cb8", 8); + CachedBlock cb9 = new CachedBlock(1000, "cb9", 9); + CachedBlock cb10 = new CachedBlock(1500, "cb10", 10); + + CachedBlockQueue queue = new CachedBlockQueue(10000,1000); + + queue.add(cb1); + queue.add(cb2); + queue.add(cb3); + queue.add(cb4); + queue.add(cb5); + queue.add(cb6); + queue.add(cb7); + queue.add(cb8); + queue.add(cb9); + queue.add(cb10); + + // We expect cb1 through cb8 to be in the queue + long expectedSize = cb1.heapSize() + cb2.heapSize() + cb3.heapSize() + + cb4.heapSize() + cb5.heapSize() + cb6.heapSize() + cb7.heapSize() + + cb8.heapSize(); + + assertEquals(queue.heapSize(), expectedSize); + + for (int i = 1; i <= 8; i++) { + assertEquals(queue.pollLast().getCacheKey().getHfileName(), "cb"+i); + } + } + + public void testQueueSmallBlockEdgeCase() throws Exception { + + CachedBlock cb1 = new CachedBlock(1000, "cb1", 1); + CachedBlock cb2 = new CachedBlock(1500, "cb2", 2); + CachedBlock cb3 = new CachedBlock(1000, "cb3", 3); + CachedBlock cb4 = new CachedBlock(1500, "cb4", 4); + CachedBlock cb5 = new CachedBlock(1000, "cb5", 5); + CachedBlock cb6 = new CachedBlock(1750, "cb6", 6); + CachedBlock cb7 = new CachedBlock(1000, "cb7", 7); + CachedBlock cb8 = new CachedBlock(1500, "cb8", 8); + CachedBlock cb9 = new CachedBlock(1000, "cb9", 9); + CachedBlock cb10 = new CachedBlock(1500, "cb10", 10); + + CachedBlockQueue queue = new CachedBlockQueue(10000,1000); + + queue.add(cb1); + queue.add(cb2); + queue.add(cb3); + queue.add(cb4); + queue.add(cb5); + queue.add(cb6); + queue.add(cb7); + queue.add(cb8); + queue.add(cb9); + queue.add(cb10); + + CachedBlock cb0 = new CachedBlock(10 + CachedBlock.PER_BLOCK_OVERHEAD, "cb0", 0); + queue.add(cb0); + + // This is older so we must include it, but it will not end up kicking + // anything out because (heapSize - cb8.heapSize + cb0.heapSize < maxSize) + // and we must always maintain heapSize >= maxSize once we achieve it. + + // We expect cb0 through cb8 to be in the queue + long expectedSize = cb1.heapSize() + cb2.heapSize() + cb3.heapSize() + + cb4.heapSize() + cb5.heapSize() + cb6.heapSize() + cb7.heapSize() + + cb8.heapSize() + cb0.heapSize(); + + assertEquals(queue.heapSize(), expectedSize); + + for (int i = 0; i <= 8; i++) { + assertEquals(queue.pollLast().getCacheKey().getHfileName(), "cb"+i); + } + } + + private static class CachedBlock extends org.apache.hadoop.hbase.io.hfile.CachedBlock + { + public CachedBlock(final long heapSize, String name, long accessTime) { + super(new BlockCacheKey(name, 0), + new Cacheable() { + @Override + public long heapSize() { + return ((int)(heapSize - CachedBlock.PER_BLOCK_OVERHEAD)); + } + + @Override + public int getSerializedLength() { + return 0; + } + + @Override + public void serialize(ByteBuffer destination) { + } + + @Override + public CacheableDeserializer getDeserializer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public BlockType getBlockType() { + return BlockType.DATA; + } + + @Override + public SchemaMetrics getSchemaMetrics() { + return SchemaMetrics.ALL_SCHEMA_METRICS; + } + }, accessTime, false); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java new file mode 100644 index 0000000..168fc67 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java @@ -0,0 +1,275 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.*; + +import java.io.ByteArrayInputStream; +import java.io.DataOutputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm; +import org.apache.hadoop.hbase.util.ChecksumType; + +import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestChecksum { + // change this value to activate more logs + private static final boolean detailedLogging = true; + private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true }; + + private static final Log LOG = LogFactory.getLog(TestHFileBlock.class); + + static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = { + NONE, GZ }; + + static final int[] BYTES_PER_CHECKSUM = { + 50, 500, 688, 16*1024, (16*1024+980), 64 * 1024}; + + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + private FileSystem fs; + private HFileSystem hfs; + + @Before + public void setUp() throws Exception { + fs = HFileSystem.get(TEST_UTIL.getConfiguration()); + hfs = (HFileSystem)fs; + } + + /** + * Introduce checksum failures and check that we can still read + * the data + */ + @Test + public void testChecksumCorruption() throws IOException { + for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) { + for (boolean pread : new boolean[] { false, true }) { + LOG.info("testChecksumCorruption: Compression algorithm: " + algo + + ", pread=" + pread); + Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_" + + algo); + FSDataOutputStream os = fs.create(path); + HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null, + true, 1, HFile.DEFAULT_CHECKSUM_TYPE, + HFile.DEFAULT_BYTES_PER_CHECKSUM); + long totalSize = 0; + for (int blockId = 0; blockId < 2; ++blockId) { + DataOutputStream dos = hbw.startWriting(BlockType.DATA); + for (int i = 0; i < 1234; ++i) + dos.writeInt(i); + hbw.writeHeaderAndData(os); + totalSize += hbw.getOnDiskSizeWithHeader(); + } + os.close(); + + // Use hbase checksums. + assertEquals(true, hfs.useHBaseChecksum()); + + // Do a read that purposely introduces checksum verification failures. + FSDataInputStream is = fs.open(path); + HFileBlock.FSReader hbr = new FSReaderV2Test(is, algo, + totalSize, HFile.MAX_FORMAT_VERSION, fs, path); + HFileBlock b = hbr.readBlockData(0, -1, -1, pread); + b.sanityCheck(); + assertEquals(4936, b.getUncompressedSizeWithoutHeader()); + assertEquals(algo == GZ ? 2173 : 4936, + b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes()); + // read data back from the hfile, exclude header and checksum + ByteBuffer bb = b.getBufferWithoutHeader(); // read back data + DataInputStream in = new DataInputStream( + new ByteArrayInputStream( + bb.array(), bb.arrayOffset(), bb.limit())); + + // assert that we encountered hbase checksum verification failures + // but still used hdfs checksums and read data successfully. + assertEquals(1, HFile.getChecksumFailuresCount()); + validateData(in); + + // A single instance of hbase checksum failure causes the reader to + // switch off hbase checksum verification for the next 100 read + // requests. Verify that this is correct. + for (int i = 0; i < + HFileBlock.CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD + 1; i++) { + b = hbr.readBlockData(0, -1, -1, pread); + assertEquals(0, HFile.getChecksumFailuresCount()); + } + // The next read should have hbase checksum verification reanabled, + // we verify this by assertng that there was a hbase-checksum failure. + b = hbr.readBlockData(0, -1, -1, pread); + assertEquals(1, HFile.getChecksumFailuresCount()); + + // Since the above encountered a checksum failure, we switch + // back to not checking hbase checksums. + b = hbr.readBlockData(0, -1, -1, pread); + assertEquals(0, HFile.getChecksumFailuresCount()); + is.close(); + + // Now, use a completely new reader. Switch off hbase checksums in + // the configuration. In this case, we should not detect + // any retries within hbase. + HFileSystem newfs = new HFileSystem(TEST_UTIL.getConfiguration(), false); + assertEquals(false, newfs.useHBaseChecksum()); + is = newfs.open(path); + hbr = new FSReaderV2Test(is, algo, + totalSize, HFile.MAX_FORMAT_VERSION, newfs, path); + b = hbr.readBlockData(0, -1, -1, pread); + is.close(); + b.sanityCheck(); + assertEquals(4936, b.getUncompressedSizeWithoutHeader()); + assertEquals(algo == GZ ? 2173 : 4936, + b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes()); + // read data back from the hfile, exclude header and checksum + bb = b.getBufferWithoutHeader(); // read back data + in = new DataInputStream(new ByteArrayInputStream( + bb.array(), bb.arrayOffset(), bb.limit())); + + // assert that we did not encounter hbase checksum verification failures + // but still used hdfs checksums and read data successfully. + assertEquals(0, HFile.getChecksumFailuresCount()); + validateData(in); + } + } + } + + /** + * Test different values of bytesPerChecksum + */ + @Test + public void testChecksumChunks() throws IOException { + Compression.Algorithm algo = NONE; + for (boolean pread : new boolean[] { false, true }) { + for (int bytesPerChecksum : BYTES_PER_CHECKSUM) { + Path path = new Path(TEST_UTIL.getDataTestDir(), "checksumChunk_" + + algo + bytesPerChecksum); + FSDataOutputStream os = fs.create(path); + HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null, + true, 1,HFile.DEFAULT_CHECKSUM_TYPE, bytesPerChecksum); + + // write one block. The block has data + // that is at least 6 times more than the checksum chunk size + long dataSize = 0; + DataOutputStream dos = hbw.startWriting(BlockType.DATA); + for (; dataSize < 6 * bytesPerChecksum;) { + for (int i = 0; i < 1234; ++i) { + dos.writeInt(i); + dataSize += 4; + } + } + hbw.writeHeaderAndData(os); + long totalSize = hbw.getOnDiskSizeWithHeader(); + os.close(); + + long expectedChunks = ChecksumUtil.numChunks( + dataSize + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS, + bytesPerChecksum); + LOG.info("testChecksumChunks: pread=" + pread + + ", bytesPerChecksum=" + bytesPerChecksum + + ", fileSize=" + totalSize + + ", dataSize=" + dataSize + + ", expectedChunks=" + expectedChunks); + + // Verify hbase checksums. + assertEquals(true, hfs.useHBaseChecksum()); + + // Read data back from file. + FSDataInputStream is = fs.open(path); + FSDataInputStream nochecksum = hfs.getNoChecksumFs().open(path); + HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, nochecksum, + algo, totalSize, HFile.MAX_FORMAT_VERSION, hfs, path); + HFileBlock b = hbr.readBlockData(0, -1, -1, pread); + is.close(); + b.sanityCheck(); + assertEquals(dataSize, b.getUncompressedSizeWithoutHeader()); + + // verify that we have the expected number of checksum chunks + assertEquals(totalSize, HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + dataSize + + expectedChunks * HFileBlock.CHECKSUM_SIZE); + + // assert that we did not encounter hbase checksum verification failures + assertEquals(0, HFile.getChecksumFailuresCount()); + } + } + } + + /** + * Test to ensure that these is at least one valid checksum implementation + */ + @Test + public void testChecksumAlgorithm() throws IOException { + ChecksumType type = ChecksumType.CRC32; + assertEquals(ChecksumType.nameToType(type.getName()), type); + assertEquals(ChecksumType.valueOf(type.toString()), type); + } + + private void validateData(DataInputStream in) throws IOException { + // validate data + for (int i = 0; i < 1234; i++) { + int val = in.readInt(); + if (val != i) { + String msg = "testChecksumCorruption: data mismatch at index " + + i + " expected " + i + " found " + val; + LOG.warn(msg); + assertEquals(i, val); + } + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); + + /** + * A class that introduces hbase-checksum failures while + * reading data from hfiles. This should trigger the hdfs level + * checksum validations. + */ + static private class FSReaderV2Test extends HFileBlock.FSReaderV2 { + + FSReaderV2Test(FSDataInputStream istream, Algorithm algo, + long fileSize, int minorVersion, FileSystem fs, + Path path) throws IOException { + super(istream, istream, algo, fileSize, minorVersion, + (HFileSystem)fs, path); + } + + @Override + protected boolean validateBlockChecksum(HFileBlock block, + byte[] data, int hdrSize) throws IOException { + return false; // checksum validation failure + } + } +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java new file mode 100644 index 0000000..7dd63d5 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java @@ -0,0 +1,235 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.hadoop.hbase.*; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import static org.junit.Assert.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +@RunWith(Parameterized.class) +@Category(MediumTests.class) +public class TestFixedFileTrailer { + + private static final Log LOG = LogFactory.getLog(TestFixedFileTrailer.class); + + /** The number of used fields by version. Indexed by version minus one. */ + private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 9, 14 }; + + private HBaseTestingUtility util = new HBaseTestingUtility(); + private FileSystem fs; + private ByteArrayOutputStream baos = new ByteArrayOutputStream(); + private int version; + + static { + assert NUM_FIELDS_BY_VERSION.length == HFile.MAX_FORMAT_VERSION + - HFile.MIN_FORMAT_VERSION + 1; + } + + public TestFixedFileTrailer(int version) { + this.version = version; + } + + @Parameters + public static Collection getParameters() { + List versionsToTest = new ArrayList(); + for (int v = HFile.MIN_FORMAT_VERSION; v <= HFile.MAX_FORMAT_VERSION; ++v) + versionsToTest.add(new Integer[] { v } ); + return versionsToTest; + } + + @Before + public void setUp() throws IOException { + fs = FileSystem.get(util.getConfiguration()); + } + + @Test + public void testTrailer() throws IOException { + FixedFileTrailer t = new FixedFileTrailer(version, + HFileBlock.MINOR_VERSION_NO_CHECKSUM); + t.setDataIndexCount(3); + t.setEntryCount(((long) Integer.MAX_VALUE) + 1); + + if (version == 1) { + t.setFileInfoOffset(876); + } + + if (version == 2) { + t.setLastDataBlockOffset(291); + t.setNumDataIndexLevels(3); + t.setComparatorClass(KeyValue.KEY_COMPARATOR.getClass()); + t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic. + t.setUncompressedDataIndexSize(827398717L); // Something random. + } + + t.setLoadOnOpenOffset(128); + t.setMetaIndexCount(7); + + t.setTotalUncompressedBytes(129731987); + + { + DataOutputStream dos = new DataOutputStream(baos); // Limited scope. + t.serialize(dos); + dos.flush(); + assertEquals(dos.size(), FixedFileTrailer.getTrailerSize(version)); + } + + byte[] bytes = baos.toByteArray(); + baos.reset(); + + assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version)); + + ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + + // Finished writing, trying to read. + { + DataInputStream dis = new DataInputStream(bais); + FixedFileTrailer t2 = new FixedFileTrailer(version, + HFileBlock.MINOR_VERSION_NO_CHECKSUM); + t2.deserialize(dis); + assertEquals(-1, bais.read()); // Ensure we have read everything. + checkLoadedTrailer(version, t, t2); + } + + // Now check what happens if the trailer is corrupted. + Path trailerPath = new Path(util.getDataTestDir(), "trailer_" + + version); + + { + for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1, + HFile.MAX_FORMAT_VERSION + 1}) { + bytes[bytes.length - 1] = invalidVersion; + writeTrailer(trailerPath, null, bytes); + try { + readTrailer(trailerPath); + fail("Exception expected"); + } catch (IllegalArgumentException ex) { + // Make it easy to debug this. + String msg = ex.getMessage(); + String cleanMsg = msg.replaceAll( + "^(java(\\.[a-zA-Z]+)+:\\s+)?|\\s+\\(.*\\)\\s*$", ""); + assertEquals("Actual exception message is \"" + msg + "\".\n" + + "Cleaned-up message", // will be followed by " expected: ..." + "Invalid HFile version: " + invalidVersion, cleanMsg); + LOG.info("Got an expected exception: " + msg); + } + } + + } + + // Now write the trailer into a file and auto-detect the version. + writeTrailer(trailerPath, t, null); + + FixedFileTrailer t4 = readTrailer(trailerPath); + + checkLoadedTrailer(version, t, t4); + + String trailerStr = t.toString(); + assertEquals("Invalid number of fields in the string representation " + + "of the trailer: " + trailerStr, NUM_FIELDS_BY_VERSION[version - 1], + trailerStr.split(", ").length); + assertEquals(trailerStr, t4.toString()); + } + + private FixedFileTrailer readTrailer(Path trailerPath) throws IOException { + FSDataInputStream fsdis = fs.open(trailerPath); + FixedFileTrailer trailerRead = FixedFileTrailer.readFromStream(fsdis, + fs.getFileStatus(trailerPath).getLen()); + fsdis.close(); + return trailerRead; + } + + private void writeTrailer(Path trailerPath, FixedFileTrailer t, + byte[] useBytesInstead) throws IOException { + assert (t == null) != (useBytesInstead == null); // Expect one non-null. + + FSDataOutputStream fsdos = fs.create(trailerPath); + fsdos.write(135); // to make deserializer's job less trivial + if (useBytesInstead != null) { + fsdos.write(useBytesInstead); + } else { + t.serialize(fsdos); + } + fsdos.close(); + } + + private void checkLoadedTrailer(int version, FixedFileTrailer expected, + FixedFileTrailer loaded) throws IOException { + assertEquals(version, loaded.getMajorVersion()); + assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount()); + + assertEquals(Math.min(expected.getEntryCount(), + version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE), + loaded.getEntryCount()); + + if (version == 1) { + assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset()); + } + + if (version == 2) { + assertEquals(expected.getLastDataBlockOffset(), + loaded.getLastDataBlockOffset()); + assertEquals(expected.getNumDataIndexLevels(), + loaded.getNumDataIndexLevels()); + assertEquals(expected.createComparator().getClass().getName(), + loaded.createComparator().getClass().getName()); + assertEquals(expected.getFirstDataBlockOffset(), + loaded.getFirstDataBlockOffset()); + assertTrue( + expected.createComparator() instanceof KeyValue.KeyComparator); + assertEquals(expected.getUncompressedDataIndexSize(), + loaded.getUncompressedDataIndexSize()); + } + + assertEquals(expected.getLoadOnOpenDataOffset(), + loaded.getLoadOnOpenDataOffset()); + assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount()); + + assertEquals(expected.getTotalUncompressedBytes(), + loaded.getTotalUncompressedBytes()); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java new file mode 100644 index 0000000..5f8214e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; +import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/**W + * Make sure we always cache important block types, such as index blocks, as + * long as we have a block cache, even though block caching might be disabled + * for the column family. + */ +@Category(MediumTests.class) +@RunWith(Parameterized.class) +public class TestForceCacheImportantBlocks { + + private final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + private static final String TABLE = "myTable"; + private static final String CF = "myCF"; + private static final byte[] CF_BYTES = Bytes.toBytes(CF); + private static final int MAX_VERSIONS = 3; + private static final int NUM_HFILES = 5; + + private static final int ROWS_PER_HFILE = 100; + private static final int NUM_ROWS = NUM_HFILES * ROWS_PER_HFILE; + private static final int NUM_COLS_PER_ROW = 50; + private static final int NUM_TIMESTAMPS_PER_COL = 50; + + /** Extremely small block size, so that we can get some index blocks */ + private static final int BLOCK_SIZE = 256; + + private static final Algorithm COMPRESSION_ALGORITHM = + Compression.Algorithm.GZ; + private static final BloomType BLOOM_TYPE = BloomType.ROW; + + private final int hfileVersion; + private final boolean cfCacheEnabled; + + @Parameters + public static Collection parameters() { + // HFile versions + return Arrays.asList(new Object[][] { + new Object[] { new Integer(1), false }, + new Object[] { new Integer(1), true }, + new Object[] { new Integer(2), false }, + new Object[] { new Integer(2), true } + }); + } + + public TestForceCacheImportantBlocks(int hfileVersion, + boolean cfCacheEnabled) { + this.hfileVersion = hfileVersion; + this.cfCacheEnabled = cfCacheEnabled; + TEST_UTIL.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, + hfileVersion); + } + + @Test + public void testCacheBlocks() throws IOException { + // Set index block size to be the same as normal block size. + TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, + BLOCK_SIZE); + + SchemaMetrics.setUseTableNameInTest(false); + HColumnDescriptor hcd = + new HColumnDescriptor(Bytes.toBytes(CF)) + .setMaxVersions(MAX_VERSIONS) + .setCompressionType(COMPRESSION_ALGORITHM) + .setBloomFilterType(BLOOM_TYPE); + hcd.setBlocksize(BLOCK_SIZE); + hcd.setBlockCacheEnabled(cfCacheEnabled); + HRegion region = TEST_UTIL.createTestRegion(TABLE, hcd); + writeTestData(region); + Map metricsBefore = SchemaMetrics.getMetricsSnapshot(); + for (int i = 0; i < NUM_ROWS; ++i) { + Get get = new Get(Bytes.toBytes("row" + i)); + region.get(get, null); + } + SchemaMetrics.validateMetricChanges(metricsBefore); + Map metricsAfter = SchemaMetrics.getMetricsSnapshot(); + Map metricsDelta = SchemaMetrics.diffMetrics(metricsBefore, + metricsAfter); + SchemaMetrics metrics = SchemaMetrics.getInstance(TABLE, CF); + List importantBlockCategories = + new ArrayList(); + importantBlockCategories.add(BlockCategory.BLOOM); + if (hfileVersion == 2) { + // We only have index blocks for HFile v2. + importantBlockCategories.add(BlockCategory.INDEX); + } + + for (BlockCategory category : importantBlockCategories) { + String hitsMetricName = getMetricName(metrics, category); + assertTrue("Metric " + hitsMetricName + " was not incremented", + metricsDelta.containsKey(hitsMetricName)); + long hits = metricsDelta.get(hitsMetricName); + assertTrue("Invalid value of " + hitsMetricName + ": " + hits, hits > 0); + } + + if (!cfCacheEnabled) { + // Caching is turned off for the CF, so make sure we are not caching data + // blocks. + String dataHitMetricName = getMetricName(metrics, BlockCategory.DATA); + assertFalse("Nonzero value for metric " + dataHitMetricName, + metricsDelta.containsKey(dataHitMetricName)); + } + } + + private String getMetricName(SchemaMetrics metrics, BlockCategory category) { + String hitsMetricName = + metrics.getBlockMetricName(category, SchemaMetrics.NO_COMPACTION, + BlockMetricType.CACHE_HIT); + return hitsMetricName; + } + + private void writeTestData(HRegion region) throws IOException { + for (int i = 0; i < NUM_ROWS; ++i) { + Put put = new Put(Bytes.toBytes("row" + i)); + for (int j = 0; j < NUM_COLS_PER_ROW; ++j) { + for (long ts = 1; ts < NUM_TIMESTAMPS_PER_COL; ++ts) { + put.add(CF_BYTES, Bytes.toBytes("col" + j), ts, + Bytes.toBytes("value" + i + "_" + j + "_" + ts)); + } + } + region.put(put); + if ((i + 1) % ROWS_PER_HFILE == 0) { + region.flushcache(); + } + } + } + +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java new file mode 100644 index 0000000..44dfb73 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -0,0 +1,379 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.KeyValue.KeyComparator; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.io.hfile.HFile.Reader; +import org.apache.hadoop.hbase.io.hfile.HFile.Writer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Writable; +import org.junit.experimental.categories.Category; + +/** + * test hfile features. + *

    + * Copied from + * hadoop-3315 tfile. + * Remove after tfile is committed and use the tfile version of this class + * instead.

    + */ +@Category(SmallTests.class) +public class TestHFile extends HBaseTestCase { + static final Log LOG = LogFactory.getLog(TestHFile.class); + + private String ROOT_DIR; + private final int minBlockSize = 512; + private static String localFormatter = "%010d"; + private static CacheConfig cacheConf = null; + + @Override + public void setUp() throws Exception { + ROOT_DIR = this.getUnitTestdir("TestHFile").toString(); + super.setUp(); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + + /** + * Test empty HFile. + * Test all features work reasonably when hfile is empty of entries. + * @throws IOException + */ + public void testEmptyHFile() throws IOException { + if (cacheConf == null) cacheConf = new CacheConfig(conf); + Path f = new Path(ROOT_DIR, getName()); + Writer w = + HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).create(); + w.close(); + Reader r = HFile.createReader(fs, f, cacheConf); + r.loadFileInfo(); + assertNull(r.getFirstKey()); + assertNull(r.getLastKey()); + } + + /** + * Create 0-length hfile and show that it fails + */ + public void testCorrupt0LengthHFile() throws IOException { + if (cacheConf == null) cacheConf = new CacheConfig(conf); + Path f = new Path(ROOT_DIR, getName()); + FSDataOutputStream fsos = fs.create(f); + fsos.close(); + + try { + Reader r = HFile.createReader(fs, f, cacheConf); + } catch (CorruptHFileException che) { + // Expected failure + return; + } + fail("Should have thrown exception"); + } + + public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException { + FileStatus fst = fs.getFileStatus(src); + long len = fst.getLen(); + len = len / 2 ; + + // create a truncated hfile + FSDataOutputStream fdos = fs.create(dst); + byte[] buf = new byte[(int)len]; + FSDataInputStream fdis = fs.open(src); + fdis.read(buf); + fdos.write(buf); + fdis.close(); + fdos.close(); + } + + /** + * Create a truncated hfile and verify that exception thrown. + */ + public void testCorruptTruncatedHFile() throws IOException { + if (cacheConf == null) cacheConf = new CacheConfig(conf); + Path f = new Path(ROOT_DIR, getName()); + Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f).create(); + writeSomeRecords(w, 0, 100); + w.close(); + + Path trunc = new Path(f.getParent(), "trucated"); + truncateFile(fs, w.getPath(), trunc); + + try { + Reader r = HFile.createReader(fs, trunc, cacheConf); + } catch (CorruptHFileException che) { + // Expected failure + return; + } + fail("Should have thrown exception"); + } + + // write some records into the tfile + // write them twice + private int writeSomeRecords(Writer writer, int start, int n) + throws IOException { + String value = "value"; + for (int i = start; i < (start + n); i++) { + String key = String.format(localFormatter, Integer.valueOf(i)); + writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key)); + } + return (start + n); + } + + private void readAllRecords(HFileScanner scanner) throws IOException { + readAndCheckbytes(scanner, 0, 100); + } + + // read the records and check + private int readAndCheckbytes(HFileScanner scanner, int start, int n) + throws IOException { + String value = "value"; + int i = start; + for (; i < (start + n); i++) { + ByteBuffer key = scanner.getKey(); + ByteBuffer val = scanner.getValue(); + String keyStr = String.format(localFormatter, Integer.valueOf(i)); + String valStr = value + keyStr; + byte [] keyBytes = Bytes.toBytes(key); + assertTrue("bytes for keys do not match " + keyStr + " " + + Bytes.toString(Bytes.toBytes(key)), + Arrays.equals(Bytes.toBytes(keyStr), keyBytes)); + byte [] valBytes = Bytes.toBytes(val); + assertTrue("bytes for vals do not match " + valStr + " " + + Bytes.toString(valBytes), + Arrays.equals(Bytes.toBytes(valStr), valBytes)); + if (!scanner.next()) { + break; + } + } + assertEquals(i, start + n - 1); + return (start + n); + } + + private byte[] getSomeKey(int rowId) { + return String.format(localFormatter, Integer.valueOf(rowId)).getBytes(); + } + + private void writeRecords(Writer writer) throws IOException { + writeSomeRecords(writer, 0, 100); + writer.close(); + } + + private FSDataOutputStream createFSOutput(Path name) throws IOException { + //if (fs.exists(name)) fs.delete(name, true); + FSDataOutputStream fout = fs.create(name); + return fout; + } + + /** + * test none codecs + */ + void basicWithSomeCodec(String codec) throws IOException { + if (cacheConf == null) cacheConf = new CacheConfig(conf); + Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString()); + FSDataOutputStream fout = createFSOutput(ncTFile); + Writer writer = HFile.getWriterFactory(conf, cacheConf) + .withOutputStream(fout) + .withBlockSize(minBlockSize) + .withCompression(codec) + .create(); + LOG.info(writer); + writeRecords(writer); + fout.close(); + FSDataInputStream fin = fs.open(ncTFile); + Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile), + fs.getFileStatus(ncTFile).getLen(), cacheConf); + System.out.println(cacheConf.toString()); + // Load up the index. + reader.loadFileInfo(); + // Get a scanner that caches and that does not use pread. + HFileScanner scanner = reader.getScanner(true, false); + // Align scanner at start of the file. + scanner.seekTo(); + readAllRecords(scanner); + scanner.seekTo(getSomeKey(50)); + assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)) == 0); + // read the key and see if it matches + ByteBuffer readKey = scanner.getKey(); + assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50), + Bytes.toBytes(readKey))); + + scanner.seekTo(new byte[0]); + ByteBuffer val1 = scanner.getValue(); + scanner.seekTo(new byte[0]); + ByteBuffer val2 = scanner.getValue(); + assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2))); + + reader.close(); + fin.close(); + fs.delete(ncTFile, true); + } + + public void testTFileFeatures() throws IOException { + basicWithSomeCodec("none"); + basicWithSomeCodec("gz"); + } + + private void writeNumMetablocks(Writer writer, int n) { + for (int i = 0; i < n; i++) { + writer.appendMetaBlock("HFileMeta" + i, new Writable() { + private int val; + public Writable setVal(int val) { this.val = val; return this; } + + @Override + public void write(DataOutput out) throws IOException { + out.write(("something to test" + val).getBytes()); + } + + @Override + public void readFields(DataInput in) throws IOException { } + }.setVal(i)); + } + } + + private void someTestingWithMetaBlock(Writer writer) { + writeNumMetablocks(writer, 10); + } + + private void readNumMetablocks(Reader reader, int n) throws IOException { + for (int i = 0; i < n; i++) { + ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false); + ByteBuffer expected = + ByteBuffer.wrap(("something to test" + i).getBytes()); + assertTrue("failed to match metadata", actual.compareTo(expected) == 0); + } + } + + private void someReadingWithMetaBlock(Reader reader) throws IOException { + readNumMetablocks(reader, 10); + } + + private void metablocks(final String compress) throws Exception { + if (cacheConf == null) cacheConf = new CacheConfig(conf); + Path mFile = new Path(ROOT_DIR, "meta.hfile"); + FSDataOutputStream fout = createFSOutput(mFile); + Writer writer = HFile.getWriterFactory(conf, cacheConf) + .withOutputStream(fout) + .withBlockSize(minBlockSize) + .withCompression(compress) + .create(); + someTestingWithMetaBlock(writer); + writer.close(); + fout.close(); + FSDataInputStream fin = fs.open(mFile); + Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile), + this.fs.getFileStatus(mFile).getLen(), cacheConf); + reader.loadFileInfo(); + // No data -- this should return false. + assertFalse(reader.getScanner(false, false).seekTo()); + someReadingWithMetaBlock(reader); + fs.delete(mFile, true); + reader.close(); + fin.close(); + } + + // test meta blocks for tfiles + public void testMetaBlocks() throws Exception { + metablocks("none"); + metablocks("gz"); + } + + public void testNullMetaBlocks() throws Exception { + if (cacheConf == null) cacheConf = new CacheConfig(conf); + for (Compression.Algorithm compressAlgo : + HBaseTestingUtility.COMPRESSION_ALGORITHMS) { + Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile"); + FSDataOutputStream fout = createFSOutput(mFile); + Writer writer = HFile.getWriterFactory(conf, cacheConf) + .withOutputStream(fout) + .withBlockSize(minBlockSize) + .withCompression(compressAlgo) + .create(); + writer.append("foo".getBytes(), "value".getBytes()); + writer.close(); + fout.close(); + Reader reader = HFile.createReader(fs, mFile, cacheConf); + reader.loadFileInfo(); + assertNull(reader.getMetaBlock("non-existant", false)); + } + } + + /** + * Make sure the ordinals for our compression algorithms do not change on us. + */ + public void testCompressionOrdinance() { + assertTrue(Compression.Algorithm.LZO.ordinal() == 0); + assertTrue(Compression.Algorithm.GZ.ordinal() == 1); + assertTrue(Compression.Algorithm.NONE.ordinal() == 2); + assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3); + assertTrue(Compression.Algorithm.LZ4.ordinal() == 4); + } + + public void testComparator() throws IOException { + if (cacheConf == null) cacheConf = new CacheConfig(conf); + Path mFile = new Path(ROOT_DIR, "meta.tfile"); + FSDataOutputStream fout = createFSOutput(mFile); + KeyComparator comparator = new KeyComparator() { + @Override + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, + int l2) { + return -Bytes.compareTo(b1, s1, l1, b2, s2, l2); + } + @Override + public int compare(byte[] o1, byte[] o2) { + return compare(o1, 0, o1.length, o2, 0, o2.length); + } + }; + Writer writer = HFile.getWriterFactory(conf, cacheConf) + .withOutputStream(fout) + .withBlockSize(minBlockSize) + .withComparator(comparator) + .create(); + writer.append("3".getBytes(), "0".getBytes()); + writer.append("2".getBytes(), "0".getBytes()); + writer.append("1".getBytes(), "0".getBytes()); + writer.close(); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java new file mode 100644 index 0000000..7440224 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java @@ -0,0 +1,805 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.*; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.DoubleOutputStream; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ChecksumType; +import org.apache.hadoop.hbase.util.ClassSize; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.io.compress.CompressionOutputStream; +import org.apache.hadoop.io.compress.Compressor; + +import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@Category(MediumTests.class) +@RunWith(Parameterized.class) +public class TestHFileBlock { + // change this value to activate more logs + private static final boolean detailedLogging = false; + private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true }; + + private static final Log LOG = LogFactory.getLog(TestHFileBlock.class); + + static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = { + NONE, GZ }; + + private static final int NUM_TEST_BLOCKS = 1000; + private static final int NUM_READER_THREADS = 26; + + // Used to generate KeyValues + private static int NUM_KEYVALUES = 50; + private static int FIELD_LENGTH = 10; + private static float CHANCE_TO_REPEAT = 0.6f; + + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + private FileSystem fs; + private int uncompressedSizeV1; + + private final boolean includesMemstoreTS; + + public TestHFileBlock(boolean includesMemstoreTS) { + this.includesMemstoreTS = includesMemstoreTS; + } + + @Parameters + public static Collection parameters() { + return HBaseTestingUtility.BOOLEAN_PARAMETERIZED; + } + + @Before + public void setUp() throws IOException { + fs = HFileSystem.get(TEST_UTIL.getConfiguration()); + } + + static void writeTestBlockContents(DataOutputStream dos) throws IOException { + // This compresses really well. + for (int i = 0; i < 1000; ++i) + dos.writeInt(i / 100); + } + + static int writeTestKeyValues(OutputStream dos, int seed, boolean includesMemstoreTS) + throws IOException { + List keyValues = new ArrayList(); + Random randomizer = new Random(42l + seed); // just any fixed number + + // generate keyValues + for (int i = 0; i < NUM_KEYVALUES; ++i) { + byte[] row; + long timestamp; + byte[] family; + byte[] qualifier; + byte[] value; + + // generate it or repeat, it should compress well + if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) { + row = keyValues.get(randomizer.nextInt(keyValues.size())).getRow(); + } else { + row = new byte[FIELD_LENGTH]; + randomizer.nextBytes(row); + } + if (0 == i) { + family = new byte[FIELD_LENGTH]; + randomizer.nextBytes(family); + } else { + family = keyValues.get(0).getFamily(); + } + if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) { + qualifier = keyValues.get( + randomizer.nextInt(keyValues.size())).getQualifier(); + } else { + qualifier = new byte[FIELD_LENGTH]; + randomizer.nextBytes(qualifier); + } + if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) { + value = keyValues.get(randomizer.nextInt(keyValues.size())).getValue(); + } else { + value = new byte[FIELD_LENGTH]; + randomizer.nextBytes(value); + } + if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) { + timestamp = keyValues.get( + randomizer.nextInt(keyValues.size())).getTimestamp(); + } else { + timestamp = randomizer.nextLong(); + } + + keyValues.add(new KeyValue(row, family, qualifier, timestamp, value)); + } + + // sort it and write to stream + int totalSize = 0; + Collections.sort(keyValues, KeyValue.COMPARATOR); + DataOutputStream dataOutputStream = new DataOutputStream(dos); + for (KeyValue kv : keyValues) { + totalSize += kv.getLength(); + dataOutputStream.write(kv.getBuffer(), kv.getOffset(), kv.getLength()); + if (includesMemstoreTS) { + long memstoreTS = randomizer.nextLong(); + WritableUtils.writeVLong(dataOutputStream, memstoreTS); + totalSize += WritableUtils.getVIntSize(memstoreTS); + } + } + + return totalSize; + } + + public byte[] createTestV1Block(Compression.Algorithm algo) + throws IOException { + Compressor compressor = algo.getCompressor(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + OutputStream os = algo.createCompressionStream(baos, compressor, 0); + DataOutputStream dos = new DataOutputStream(os); + BlockType.META.write(dos); // Let's make this a meta block. + writeTestBlockContents(dos); + uncompressedSizeV1 = dos.size(); + dos.flush(); + algo.returnCompressor(compressor); + return baos.toByteArray(); + } + + static HFileBlock.Writer createTestV2Block(Compression.Algorithm algo, + boolean includesMemstoreTS) throws IOException { + final BlockType blockType = BlockType.DATA; + HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null, + includesMemstoreTS, HFileReaderV2.MAX_MINOR_VERSION, + HFile.DEFAULT_CHECKSUM_TYPE, + HFile.DEFAULT_BYTES_PER_CHECKSUM); + DataOutputStream dos = hbw.startWriting(blockType); + writeTestBlockContents(dos); + dos.flush(); + byte[] headerAndData = hbw.getHeaderAndDataForTest(); + assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader()); + hbw.releaseCompressor(); + return hbw; + } + + public String createTestBlockStr(Compression.Algorithm algo, + int correctLength) throws IOException { + HFileBlock.Writer hbw = createTestV2Block(algo, includesMemstoreTS); + byte[] testV2Block = hbw.getHeaderAndDataForTest(); + int osOffset = HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + 9; + if (testV2Block.length == correctLength) { + // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid + // variations across operating systems. + // See http://www.gzip.org/zlib/rfc-gzip.html for gzip format. + // We only make this change when the compressed block length matches. + // Otherwise, there are obviously other inconsistencies. + testV2Block[osOffset] = 3; + } + return Bytes.toStringBinary(testV2Block); + } + + @Test + public void testNoCompression() throws IOException { + assertEquals(4000, createTestV2Block(NONE, includesMemstoreTS). + getBlockForCaching().getUncompressedSizeWithoutHeader()); + } + + @Test + public void testGzipCompression() throws IOException { + final String correctTestBlockStr = + "DATABLK*\\x00\\x00\\x00>\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF" + + "\\xFF\\xFF\\xFF\\xFF" + + "\\x01\\x00\\x00@\\x00\\x00\\x00\\x00[" + // gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html + + "\\x1F\\x8B" // gzip magic signature + + "\\x08" // Compression method: 8 = "deflate" + + "\\x00" // Flags + + "\\x00\\x00\\x00\\x00" // mtime + + "\\x00" // XFL (extra flags) + // OS (0 = FAT filesystems, 3 = Unix). However, this field + // sometimes gets set to 0 on Linux and Mac, so we reset it to 3. + // This appears to be a difference caused by the availability + // (and use) of the native GZ codec. + + "\\x03" + + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa" + + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c" + + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00" + + "\\x00\\x00\\x00\\x00"; // 4 byte checksum (ignored) + final int correctGzipBlockLength = 95; + final String testBlockStr = createTestBlockStr(GZ, correctGzipBlockLength); + // We ignore the block checksum because createTestBlockStr can change the + // gzip header after the block is produced + assertEquals(correctTestBlockStr.substring(0, correctGzipBlockLength - 4), + testBlockStr.substring(0, correctGzipBlockLength - 4)); + } + + @Test + public void testReaderV1() throws IOException { + for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) { + for (boolean pread : new boolean[] { false, true }) { + byte[] block = createTestV1Block(algo); + Path path = new Path(TEST_UTIL.getDataTestDir(), + "blocks_v1_"+ algo); + LOG.info("Creating temporary file at " + path); + FSDataOutputStream os = fs.create(path); + int totalSize = 0; + int numBlocks = 50; + for (int i = 0; i < numBlocks; ++i) { + os.write(block); + totalSize += block.length; + } + os.close(); + + FSDataInputStream is = fs.open(path); + HFileBlock.FSReader hbr = new HFileBlock.FSReaderV1(is, algo, + totalSize); + HFileBlock b; + int numBlocksRead = 0; + long pos = 0; + while (pos < totalSize) { + b = hbr.readBlockData(pos, block.length, uncompressedSizeV1, pread); + b.sanityCheck(); + pos += block.length; + numBlocksRead++; + } + assertEquals(numBlocks, numBlocksRead); + is.close(); + } + } + } + + @Test + public void testReaderV2() throws IOException { + for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) { + for (boolean pread : new boolean[] { false, true }) { + LOG.info("testReaderV2: Compression algorithm: " + algo + + ", pread=" + pread); + Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_" + + algo); + FSDataOutputStream os = fs.create(path); + HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null, + includesMemstoreTS, + HFileReaderV2.MAX_MINOR_VERSION, + HFile.DEFAULT_CHECKSUM_TYPE, + HFile.DEFAULT_BYTES_PER_CHECKSUM); + long totalSize = 0; + for (int blockId = 0; blockId < 2; ++blockId) { + DataOutputStream dos = hbw.startWriting(BlockType.DATA); + for (int i = 0; i < 1234; ++i) + dos.writeInt(i); + hbw.writeHeaderAndData(os); + totalSize += hbw.getOnDiskSizeWithHeader(); + } + os.close(); + + FSDataInputStream is = fs.open(path); + HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, algo, + totalSize); + HFileBlock b = hbr.readBlockData(0, -1, -1, pread); + is.close(); + assertEquals(0, HFile.getChecksumFailuresCount()); + + b.sanityCheck(); + assertEquals(4936, b.getUncompressedSizeWithoutHeader()); + assertEquals(algo == GZ ? 2173 : 4936, + b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes()); + String blockStr = b.toString(); + + if (algo == GZ) { + is = fs.open(path); + hbr = new HFileBlock.FSReaderV2(is, algo, totalSize); + b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + + b.totalChecksumBytes(), -1, pread); + assertEquals(blockStr, b.toString()); + int wrongCompressedSize = 2172; + try { + b = hbr.readBlockData(0, wrongCompressedSize + + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS, -1, pread); + fail("Exception expected"); + } catch (IOException ex) { + String expectedPrefix = "On-disk size without header provided is " + + wrongCompressedSize + ", but block header contains " + + b.getOnDiskSizeWithoutHeader() + "."; + assertTrue("Invalid exception message: '" + ex.getMessage() + + "'.\nMessage is expected to start with: '" + expectedPrefix + + "'", ex.getMessage().startsWith(expectedPrefix)); + } + is.close(); + } + } + } + } + + /** + * Test encoding/decoding data blocks. + * @throws IOException a bug or a problem with temporary files. + */ + @Test + public void testDataBlockEncoding() throws IOException { + final int numBlocks = 5; + for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) { + for (boolean pread : new boolean[] { false, true }) { + for (DataBlockEncoding encoding : DataBlockEncoding.values()) { + Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_" + + algo + "_" + encoding.toString()); + FSDataOutputStream os = fs.create(path); + HFileDataBlockEncoder dataBlockEncoder = + new HFileDataBlockEncoderImpl(encoding); + HFileBlock.Writer hbw = new HFileBlock.Writer(algo, dataBlockEncoder, + includesMemstoreTS, + HFileReaderV2.MAX_MINOR_VERSION, + HFile.DEFAULT_CHECKSUM_TYPE, + HFile.DEFAULT_BYTES_PER_CHECKSUM); + long totalSize = 0; + final List encodedSizes = new ArrayList(); + final List encodedBlocks = new ArrayList(); + for (int blockId = 0; blockId < numBlocks; ++blockId) { + DataOutputStream dos = hbw.startWriting(BlockType.DATA); + writeEncodedBlock(encoding, dos, encodedSizes, encodedBlocks, + blockId, includesMemstoreTS); + + hbw.writeHeaderAndData(os); + totalSize += hbw.getOnDiskSizeWithHeader(); + } + os.close(); + + FSDataInputStream is = fs.open(path); + HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(is, algo, + totalSize); + hbr.setDataBlockEncoder(dataBlockEncoder); + hbr.setIncludesMemstoreTS(includesMemstoreTS); + + HFileBlock b; + int pos = 0; + for (int blockId = 0; blockId < numBlocks; ++blockId) { + b = hbr.readBlockData(pos, -1, -1, pread); + assertEquals(0, HFile.getChecksumFailuresCount()); + b.sanityCheck(); + pos += b.getOnDiskSizeWithHeader(); + + assertEquals((int) encodedSizes.get(blockId), + b.getUncompressedSizeWithoutHeader()); + ByteBuffer actualBuffer = b.getBufferWithoutHeader(); + if (encoding != DataBlockEncoding.NONE) { + // We expect a two-byte big-endian encoding id. + assertEquals(0, actualBuffer.get(0)); + assertEquals(encoding.getId(), actualBuffer.get(1)); + actualBuffer.position(2); + actualBuffer = actualBuffer.slice(); + } + + ByteBuffer expectedBuffer = encodedBlocks.get(blockId); + expectedBuffer.rewind(); + + // test if content matches, produce nice message + assertBuffersEqual(expectedBuffer, actualBuffer, algo, encoding, + pread); + } + is.close(); + } + } + } + } + + static void writeEncodedBlock(DataBlockEncoding encoding, + DataOutputStream dos, final List encodedSizes, + final List encodedBlocks, int blockId, + boolean includesMemstoreTS) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DoubleOutputStream doubleOutputStream = + new DoubleOutputStream(dos, baos); + + final int rawBlockSize = writeTestKeyValues(doubleOutputStream, + blockId, includesMemstoreTS); + + ByteBuffer rawBuf = ByteBuffer.wrap(baos.toByteArray()); + rawBuf.rewind(); + + final int encodedSize; + final ByteBuffer encodedBuf; + if (encoding == DataBlockEncoding.NONE) { + encodedSize = rawBlockSize; + encodedBuf = rawBuf; + } else { + ByteArrayOutputStream encodedOut = new ByteArrayOutputStream(); + encoding.getEncoder().compressKeyValues( + new DataOutputStream(encodedOut), + rawBuf.duplicate(), includesMemstoreTS); + // We need to account for the two-byte encoding algorithm ID that + // comes after the 24-byte block header but before encoded KVs. + encodedSize = encodedOut.size() + DataBlockEncoding.ID_SIZE; + encodedBuf = ByteBuffer.wrap(encodedOut.toByteArray()); + } + encodedSizes.add(encodedSize); + encodedBlocks.add(encodedBuf); + } + + static void assertBuffersEqual(ByteBuffer expectedBuffer, + ByteBuffer actualBuffer, Compression.Algorithm compression, + DataBlockEncoding encoding, boolean pread) { + if (!actualBuffer.equals(expectedBuffer)) { + int prefix = 0; + int minLimit = Math.min(expectedBuffer.limit(), actualBuffer.limit()); + while (prefix < minLimit && + expectedBuffer.get(prefix) == actualBuffer.get(prefix)) { + prefix++; + } + + fail(String.format( + "Content mismath for compression %s, encoding %s, " + + "pread %s, commonPrefix %d, expected %s, got %s", + compression, encoding, pread, prefix, + nextBytesToStr(expectedBuffer, prefix), + nextBytesToStr(actualBuffer, prefix))); + } + } + + /** + * Convert a few next bytes in the given buffer at the given position to + * string. Used for error messages. + */ + private static String nextBytesToStr(ByteBuffer buf, int pos) { + int maxBytes = buf.limit() - pos; + int numBytes = Math.min(16, maxBytes); + return Bytes.toStringBinary(buf.array(), buf.arrayOffset() + pos, + numBytes) + (numBytes < maxBytes ? "..." : ""); + } + + @Test + public void testPreviousOffset() throws IOException { + for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) { + for (boolean pread : BOOLEAN_VALUES) { + for (boolean cacheOnWrite : BOOLEAN_VALUES) { + Random rand = defaultRandom(); + LOG.info("testPreviousOffset:Compression algorithm: " + algo + + ", pread=" + pread + + ", cacheOnWrite=" + cacheOnWrite); + Path path = new Path(TEST_UTIL.getDataTestDir(), "prev_offset"); + List expectedOffsets = new ArrayList(); + List expectedPrevOffsets = new ArrayList(); + List expectedTypes = new ArrayList(); + List expectedContents = cacheOnWrite + ? new ArrayList() : null; + long totalSize = writeBlocks(rand, algo, path, expectedOffsets, + expectedPrevOffsets, expectedTypes, expectedContents); + + FSDataInputStream is = fs.open(path); + HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, algo, + totalSize); + long curOffset = 0; + for (int i = 0; i < NUM_TEST_BLOCKS; ++i) { + if (!pread) { + assertEquals(is.getPos(), curOffset + (i == 0 ? 0 : + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS)); + } + + assertEquals(expectedOffsets.get(i).longValue(), curOffset); + if (detailedLogging) { + LOG.info("Reading block #" + i + " at offset " + curOffset); + } + HFileBlock b = hbr.readBlockData(curOffset, -1, -1, pread); + if (detailedLogging) { + LOG.info("Block #" + i + ": " + b); + } + assertEquals("Invalid block #" + i + "'s type:", + expectedTypes.get(i), b.getBlockType()); + assertEquals("Invalid previous block offset for block " + i + + " of " + "type " + b.getBlockType() + ":", + (long) expectedPrevOffsets.get(i), b.getPrevBlockOffset()); + b.sanityCheck(); + assertEquals(curOffset, b.getOffset()); + + // Now re-load this block knowing the on-disk size. This tests a + // different branch in the loader. + HFileBlock b2 = hbr.readBlockData(curOffset, + b.getOnDiskSizeWithHeader(), -1, pread); + b2.sanityCheck(); + + assertEquals(b.getBlockType(), b2.getBlockType()); + assertEquals(b.getOnDiskSizeWithoutHeader(), + b2.getOnDiskSizeWithoutHeader()); + assertEquals(b.getOnDiskSizeWithHeader(), + b2.getOnDiskSizeWithHeader()); + assertEquals(b.getUncompressedSizeWithoutHeader(), + b2.getUncompressedSizeWithoutHeader()); + assertEquals(b.getPrevBlockOffset(), b2.getPrevBlockOffset()); + assertEquals(curOffset, b2.getOffset()); + assertEquals(b.getBytesPerChecksum(), b2.getBytesPerChecksum()); + assertEquals(b.getOnDiskDataSizeWithHeader(), + b2.getOnDiskDataSizeWithHeader()); + assertEquals(0, HFile.getChecksumFailuresCount()); + + curOffset += b.getOnDiskSizeWithHeader(); + + if (cacheOnWrite) { + // In the cache-on-write mode we store uncompressed bytes so we + // can compare them to what was read by the block reader. + // b's buffer has header + data + checksum while + // expectedContents have header + data only + ByteBuffer bufRead = b.getBufferWithHeader(); + ByteBuffer bufExpected = expectedContents.get(i); + boolean bytesAreCorrect = Bytes.compareTo(bufRead.array(), + bufRead.arrayOffset(), + bufRead.limit() - b.totalChecksumBytes(), + bufExpected.array(), bufExpected.arrayOffset(), + bufExpected.limit()) == 0; + String wrongBytesMsg = ""; + + if (!bytesAreCorrect) { + // Optimization: only construct an error message in case we + // will need it. + wrongBytesMsg = "Expected bytes in block #" + i + " (algo=" + + algo + ", pread=" + pread + + ", cacheOnWrite=" + cacheOnWrite + "):\n"; + wrongBytesMsg += Bytes.toStringBinary(bufExpected.array(), + bufExpected.arrayOffset(), Math.min(32, + bufExpected.limit())) + + ", actual:\n" + + Bytes.toStringBinary(bufRead.array(), + bufRead.arrayOffset(), Math.min(32, bufRead.limit())); + if (detailedLogging) { + LOG.warn("expected header" + + HFileBlock.toStringHeader(bufExpected) + + "\nfound header" + + HFileBlock.toStringHeader(bufRead)); + LOG.warn("bufread offset " + bufRead.arrayOffset() + + " limit " + bufRead.limit() + + " expected offset " + bufExpected.arrayOffset() + + " limit " + bufExpected.limit()); + LOG.warn(wrongBytesMsg); + } + } + assertTrue(wrongBytesMsg, bytesAreCorrect); + } + } + + assertEquals(curOffset, fs.getFileStatus(path).getLen()); + is.close(); + } + } + } + } + + private Random defaultRandom() { + return new Random(189237); + } + + private class BlockReaderThread implements Callable { + private final String clientId; + private final HFileBlock.FSReader hbr; + private final List offsets; + private final List types; + private final long fileSize; + + public BlockReaderThread(String clientId, + HFileBlock.FSReader hbr, List offsets, List types, + long fileSize) { + this.clientId = clientId; + this.offsets = offsets; + this.hbr = hbr; + this.types = types; + this.fileSize = fileSize; + } + + @Override + public Boolean call() throws Exception { + Random rand = new Random(clientId.hashCode()); + long endTime = System.currentTimeMillis() + 10000; + int numBlocksRead = 0; + int numPositionalRead = 0; + int numWithOnDiskSize = 0; + while (System.currentTimeMillis() < endTime) { + int blockId = rand.nextInt(NUM_TEST_BLOCKS); + long offset = offsets.get(blockId); + boolean pread = rand.nextBoolean(); + boolean withOnDiskSize = rand.nextBoolean(); + long expectedSize = + (blockId == NUM_TEST_BLOCKS - 1 ? fileSize + : offsets.get(blockId + 1)) - offset; + + HFileBlock b; + try { + long onDiskSizeArg = withOnDiskSize ? expectedSize : -1; + b = hbr.readBlockData(offset, onDiskSizeArg, -1, pread); + } catch (IOException ex) { + LOG.error("Error in client " + clientId + " trying to read block at " + + offset + ", pread=" + pread + ", withOnDiskSize=" + + withOnDiskSize, ex); + return false; + } + + assertEquals(types.get(blockId), b.getBlockType()); + assertEquals(expectedSize, b.getOnDiskSizeWithHeader()); + assertEquals(offset, b.getOffset()); + + ++numBlocksRead; + if (pread) + ++numPositionalRead; + if (withOnDiskSize) + ++numWithOnDiskSize; + } + LOG.info("Client " + clientId + " successfully read " + numBlocksRead + + " blocks (with pread: " + numPositionalRead + ", with onDiskSize " + + "specified: " + numWithOnDiskSize + ")"); + + return true; + } + + } + + @Test + public void testConcurrentReading() throws Exception { + for (Compression.Algorithm compressAlgo : COMPRESSION_ALGORITHMS) { + Path path = + new Path(TEST_UTIL.getDataTestDir(), "concurrent_reading"); + Random rand = defaultRandom(); + List offsets = new ArrayList(); + List types = new ArrayList(); + writeBlocks(rand, compressAlgo, path, offsets, null, types, null); + FSDataInputStream is = fs.open(path); + long fileSize = fs.getFileStatus(path).getLen(); + HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, compressAlgo, + fileSize); + + Executor exec = Executors.newFixedThreadPool(NUM_READER_THREADS); + ExecutorCompletionService ecs = + new ExecutorCompletionService(exec); + + for (int i = 0; i < NUM_READER_THREADS; ++i) { + ecs.submit(new BlockReaderThread("reader_" + (char) ('A' + i), hbr, + offsets, types, fileSize)); + } + + for (int i = 0; i < NUM_READER_THREADS; ++i) { + Future result = ecs.take(); + assertTrue(result.get()); + if (detailedLogging) { + LOG.info(String.valueOf(i + 1) + + " reader threads finished successfully (algo=" + compressAlgo + + ")"); + } + } + + is.close(); + } + } + + private long writeBlocks(Random rand, Compression.Algorithm compressAlgo, + Path path, List expectedOffsets, List expectedPrevOffsets, + List expectedTypes, List expectedContents + ) throws IOException { + boolean cacheOnWrite = expectedContents != null; + FSDataOutputStream os = fs.create(path); + HFileBlock.Writer hbw = new HFileBlock.Writer(compressAlgo, null, + includesMemstoreTS, + HFileReaderV2.MAX_MINOR_VERSION, + HFile.DEFAULT_CHECKSUM_TYPE, + HFile.DEFAULT_BYTES_PER_CHECKSUM); + Map prevOffsetByType = new HashMap(); + long totalSize = 0; + for (int i = 0; i < NUM_TEST_BLOCKS; ++i) { + long pos = os.getPos(); + int blockTypeOrdinal = rand.nextInt(BlockType.values().length); + if (blockTypeOrdinal == BlockType.ENCODED_DATA.ordinal()) { + blockTypeOrdinal = BlockType.DATA.ordinal(); + } + BlockType bt = BlockType.values()[blockTypeOrdinal]; + DataOutputStream dos = hbw.startWriting(bt); + int size = rand.nextInt(500); + for (int j = 0; j < size; ++j) { + // This might compress well. + dos.writeShort(i + 1); + dos.writeInt(j + 1); + } + + if (expectedOffsets != null) + expectedOffsets.add(os.getPos()); + + if (expectedPrevOffsets != null) { + Long prevOffset = prevOffsetByType.get(bt); + expectedPrevOffsets.add(prevOffset != null ? prevOffset : -1); + prevOffsetByType.put(bt, os.getPos()); + } + + expectedTypes.add(bt); + + hbw.writeHeaderAndData(os); + totalSize += hbw.getOnDiskSizeWithHeader(); + + if (cacheOnWrite) + expectedContents.add(hbw.getUncompressedBufferWithHeader()); + + if (detailedLogging) { + LOG.info("Written block #" + i + " of type " + bt + + ", uncompressed size " + hbw.getUncompressedSizeWithoutHeader() + + " at offset " + pos); + } + } + os.close(); + LOG.info("Created a temporary file at " + path + ", " + + fs.getFileStatus(path).getLen() + " byte, compression=" + + compressAlgo); + return totalSize; + } + + @Test + public void testBlockHeapSize() { + if (ClassSize.is32BitJVM()) { + assertTrue(HFileBlock.BYTE_BUFFER_HEAP_SIZE == 64); + } else { + assertTrue(HFileBlock.BYTE_BUFFER_HEAP_SIZE == 80); + } + + for (int size : new int[] { 100, 256, 12345 }) { + byte[] byteArr = new byte[HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + size]; + ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size); + HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf, + HFileBlock.FILL_HEADER, -1, includesMemstoreTS, + HFileBlock.MINOR_VERSION_NO_CHECKSUM, 0, ChecksumType.NULL.getCode(), + 0); + long byteBufferExpectedSize = + ClassSize.align(ClassSize.estimateBase(buf.getClass(), true) + + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + size); + long hfileBlockExpectedSize = + ClassSize.align(ClassSize.estimateBase(HFileBlock.class, true)); + long expected = hfileBlockExpectedSize + byteBufferExpectedSize; + assertEquals("Block data size: " + size + ", byte buffer expected " + + "size: " + byteBufferExpectedSize + ", HFileBlock class expected " + + "size: " + hfileBlockExpectedSize + ";", expected, + block.heapSize()); + } + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java new file mode 100644 index 0000000..98ea180 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java @@ -0,0 +1,792 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.*; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.compress.CompressionOutputStream; +import org.apache.hadoop.io.compress.Compressor; +import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable; +import org.apache.hadoop.hbase.util.ChecksumType; +import org.apache.hadoop.hbase.util.Pair; +import com.google.common.base.Preconditions; + +import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * This class has unit tests to prove that older versions of + * HFiles (without checksums) are compatible with current readers. + */ +@Category(MediumTests.class) +@RunWith(Parameterized.class) +public class TestHFileBlockCompatibility { + // change this value to activate more logs + private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true }; + + private static final Log LOG = LogFactory.getLog(TestHFileBlockCompatibility.class); + + private static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = { + NONE, GZ }; + + // The mnior version for pre-checksum files + private static int MINOR_VERSION = 0; + + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + private HFileSystem fs; + private int uncompressedSizeV1; + + private final boolean includesMemstoreTS; + + public TestHFileBlockCompatibility(boolean includesMemstoreTS) { + this.includesMemstoreTS = includesMemstoreTS; + } + + @Parameters + public static Collection parameters() { + return HBaseTestingUtility.BOOLEAN_PARAMETERIZED; + } + + @Before + public void setUp() throws IOException { + fs = (HFileSystem)HFileSystem.get(TEST_UTIL.getConfiguration()); + } + + public byte[] createTestV1Block(Compression.Algorithm algo) + throws IOException { + Compressor compressor = algo.getCompressor(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + OutputStream os = algo.createCompressionStream(baos, compressor, 0); + DataOutputStream dos = new DataOutputStream(os); + BlockType.META.write(dos); // Let's make this a meta block. + TestHFileBlock.writeTestBlockContents(dos); + uncompressedSizeV1 = dos.size(); + dos.flush(); + algo.returnCompressor(compressor); + return baos.toByteArray(); + } + + private Writer createTestV2Block(Compression.Algorithm algo) + throws IOException { + final BlockType blockType = BlockType.DATA; + Writer hbw = new Writer(algo, null, + includesMemstoreTS); + DataOutputStream dos = hbw.startWriting(blockType); + TestHFileBlock.writeTestBlockContents(dos); + byte[] headerAndData = hbw.getHeaderAndData(); + assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader()); + hbw.releaseCompressor(); + return hbw; + } + + private String createTestBlockStr(Compression.Algorithm algo, + int correctLength) throws IOException { + Writer hbw = createTestV2Block(algo); + byte[] testV2Block = hbw.getHeaderAndData(); + int osOffset = HFileBlock.HEADER_SIZE_NO_CHECKSUM + 9; + if (testV2Block.length == correctLength) { + // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid + // variations across operating systems. + // See http://www.gzip.org/zlib/rfc-gzip.html for gzip format. + testV2Block[osOffset] = 3; + } + return Bytes.toStringBinary(testV2Block); + } + + @Test + public void testNoCompression() throws IOException { + assertEquals(4000, createTestV2Block(NONE).getBlockForCaching(). + getUncompressedSizeWithoutHeader()); + } + + @Test + public void testGzipCompression() throws IOException { + final String correctTestBlockStr = + "DATABLK*\\x00\\x00\\x00:\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF" + + "\\xFF\\xFF\\xFF\\xFF" + // gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html + + "\\x1F\\x8B" // gzip magic signature + + "\\x08" // Compression method: 8 = "deflate" + + "\\x00" // Flags + + "\\x00\\x00\\x00\\x00" // mtime + + "\\x00" // XFL (extra flags) + // OS (0 = FAT filesystems, 3 = Unix). However, this field + // sometimes gets set to 0 on Linux and Mac, so we reset it to 3. + + "\\x03" + + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa" + + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c" + + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00"; + final int correctGzipBlockLength = 82; + assertEquals(correctTestBlockStr, createTestBlockStr(GZ, + correctGzipBlockLength)); + } + + @Test + public void testReaderV1() throws IOException { + for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) { + for (boolean pread : new boolean[] { false, true }) { + byte[] block = createTestV1Block(algo); + Path path = new Path(TEST_UTIL.getDataTestDir(), + "blocks_v1_"+ algo); + LOG.info("Creating temporary file at " + path); + FSDataOutputStream os = fs.create(path); + int totalSize = 0; + int numBlocks = 50; + for (int i = 0; i < numBlocks; ++i) { + os.write(block); + totalSize += block.length; + } + os.close(); + + FSDataInputStream is = fs.open(path); + HFileBlock.FSReader hbr = new HFileBlock.FSReaderV1(is, algo, + totalSize); + HFileBlock b; + int numBlocksRead = 0; + long pos = 0; + while (pos < totalSize) { + b = hbr.readBlockData(pos, block.length, uncompressedSizeV1, pread); + b.sanityCheck(); + pos += block.length; + numBlocksRead++; + } + assertEquals(numBlocks, numBlocksRead); + is.close(); + } + } + } + + @Test + public void testReaderV2() throws IOException { + for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) { + for (boolean pread : new boolean[] { false, true }) { + LOG.info("testReaderV2: Compression algorithm: " + algo + + ", pread=" + pread); + Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_" + + algo); + FSDataOutputStream os = fs.create(path); + Writer hbw = new Writer(algo, null, + includesMemstoreTS); + long totalSize = 0; + for (int blockId = 0; blockId < 2; ++blockId) { + DataOutputStream dos = hbw.startWriting(BlockType.DATA); + for (int i = 0; i < 1234; ++i) + dos.writeInt(i); + hbw.writeHeaderAndData(os); + totalSize += hbw.getOnDiskSizeWithHeader(); + } + os.close(); + + FSDataInputStream is = fs.open(path); + HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, is, algo, + totalSize, MINOR_VERSION, fs, path); + HFileBlock b = hbr.readBlockData(0, -1, -1, pread); + is.close(); + + b.sanityCheck(); + assertEquals(4936, b.getUncompressedSizeWithoutHeader()); + assertEquals(algo == GZ ? 2173 : 4936, + b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes()); + String blockStr = b.toString(); + + if (algo == GZ) { + is = fs.open(path); + hbr = new HFileBlock.FSReaderV2(is, is, algo, totalSize, MINOR_VERSION, + fs, path); + b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE_NO_CHECKSUM + + b.totalChecksumBytes(), -1, pread); + assertEquals(blockStr, b.toString()); + int wrongCompressedSize = 2172; + try { + b = hbr.readBlockData(0, wrongCompressedSize + + HFileBlock.HEADER_SIZE_NO_CHECKSUM, -1, pread); + fail("Exception expected"); + } catch (IOException ex) { + String expectedPrefix = "On-disk size without header provided is " + + wrongCompressedSize + ", but block header contains " + + b.getOnDiskSizeWithoutHeader() + "."; + assertTrue("Invalid exception message: '" + ex.getMessage() + + "'.\nMessage is expected to start with: '" + expectedPrefix + + "'", ex.getMessage().startsWith(expectedPrefix)); + } + is.close(); + } + } + } + } + + /** + * Test encoding/decoding data blocks. + * @throws IOException a bug or a problem with temporary files. + */ + @Test + public void testDataBlockEncoding() throws IOException { + final int numBlocks = 5; + for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) { + for (boolean pread : new boolean[] { false, true }) { + for (DataBlockEncoding encoding : DataBlockEncoding.values()) { + LOG.info("testDataBlockEncoding algo " + algo + + " pread = " + pread + + " encoding " + encoding); + Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_" + + algo + "_" + encoding.toString()); + FSDataOutputStream os = fs.create(path); + HFileDataBlockEncoder dataBlockEncoder = + new HFileDataBlockEncoderImpl(encoding); + Writer hbw = new Writer(algo, dataBlockEncoder, + includesMemstoreTS); + long totalSize = 0; + final List encodedSizes = new ArrayList(); + final List encodedBlocks = new ArrayList(); + for (int blockId = 0; blockId < numBlocks; ++blockId) { + DataOutputStream dos = hbw.startWriting(BlockType.DATA); + TestHFileBlock.writeEncodedBlock(encoding, dos, encodedSizes, encodedBlocks, + blockId, includesMemstoreTS); + + hbw.writeHeaderAndData(os); + totalSize += hbw.getOnDiskSizeWithHeader(); + } + os.close(); + + FSDataInputStream is = fs.open(path); + HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(is, is, algo, + totalSize, MINOR_VERSION, fs, path); + hbr.setDataBlockEncoder(dataBlockEncoder); + hbr.setIncludesMemstoreTS(includesMemstoreTS); + + HFileBlock b; + int pos = 0; + for (int blockId = 0; blockId < numBlocks; ++blockId) { + b = hbr.readBlockData(pos, -1, -1, pread); + b.sanityCheck(); + pos += b.getOnDiskSizeWithHeader(); + + assertEquals((int) encodedSizes.get(blockId), + b.getUncompressedSizeWithoutHeader()); + ByteBuffer actualBuffer = b.getBufferWithoutHeader(); + if (encoding != DataBlockEncoding.NONE) { + // We expect a two-byte big-endian encoding id. + assertEquals(0, actualBuffer.get(0)); + assertEquals(encoding.getId(), actualBuffer.get(1)); + actualBuffer.position(2); + actualBuffer = actualBuffer.slice(); + } + + ByteBuffer expectedBuffer = encodedBlocks.get(blockId); + expectedBuffer.rewind(); + + // test if content matches, produce nice message + TestHFileBlock.assertBuffersEqual(expectedBuffer, actualBuffer, algo, encoding, + pread); + } + is.close(); + } + } + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); + + + /** + * This is the version of the HFileBlock.Writer that is used to + * create V2 blocks with minor version 0. These blocks do not + * have hbase-level checksums. The code is here to test + * backward compatibility. The reason we do not inherit from + * HFileBlock.Writer is because we never ever want to change the code + * in this class but the code in HFileBlock.Writer will continually + * evolve. + */ + public static final class Writer { + + // These constants are as they were in minorVersion 0. + private static final int HEADER_SIZE = HFileBlock.HEADER_SIZE_NO_CHECKSUM; + private static final boolean DONT_FILL_HEADER = HFileBlock.DONT_FILL_HEADER; + private static final byte[] DUMMY_HEADER = + HFileBlock.DUMMY_HEADER_NO_CHECKSUM; + + private enum State { + INIT, + WRITING, + BLOCK_READY + }; + + /** Writer state. Used to ensure the correct usage protocol. */ + private State state = State.INIT; + + /** Compression algorithm for all blocks this instance writes. */ + private final Compression.Algorithm compressAlgo; + + /** Data block encoder used for data blocks */ + private final HFileDataBlockEncoder dataBlockEncoder; + + /** + * The stream we use to accumulate data in uncompressed format for each + * block. We reset this stream at the end of each block and reuse it. The + * header is written as the first {@link #HEADER_SIZE} bytes into this + * stream. + */ + private ByteArrayOutputStream baosInMemory; + + /** Compressor, which is also reused between consecutive blocks. */ + private Compressor compressor; + + /** Compression output stream */ + private CompressionOutputStream compressionStream; + + /** Underlying stream to write compressed bytes to */ + private ByteArrayOutputStream compressedByteStream; + + /** + * Current block type. Set in {@link #startWriting(BlockType)}. Could be + * changed in {@link #encodeDataBlockForDisk()} from {@link BlockType#DATA} + * to {@link BlockType#ENCODED_DATA}. + */ + private BlockType blockType; + + /** + * A stream that we write uncompressed bytes to, which compresses them and + * writes them to {@link #baosInMemory}. + */ + private DataOutputStream userDataStream; + + /** + * Bytes to be written to the file system, including the header. Compressed + * if compression is turned on. + */ + private byte[] onDiskBytesWithHeader; + + /** + * Valid in the READY state. Contains the header and the uncompressed (but + * potentially encoded, if this is a data block) bytes, so the length is + * {@link #uncompressedSizeWithoutHeader} + {@link HFileBlock#HEADER_SIZE_WITH_CHECKSUMS}. + */ + private byte[] uncompressedBytesWithHeader; + + /** + * Current block's start offset in the {@link HFile}. Set in + * {@link #writeHeaderAndData(FSDataOutputStream)}. + */ + private long startOffset; + + /** + * Offset of previous block by block type. Updated when the next block is + * started. + */ + private long[] prevOffsetByType; + + /** The offset of the previous block of the same type */ + private long prevOffset; + + /** Whether we are including memstore timestamp after every key/value */ + private boolean includesMemstoreTS; + + /** + * @param compressionAlgorithm compression algorithm to use + * @param dataBlockEncoderAlgo data block encoding algorithm to use + */ + public Writer(Compression.Algorithm compressionAlgorithm, + HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS) { + compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm; + this.dataBlockEncoder = dataBlockEncoder != null + ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE; + + baosInMemory = new ByteArrayOutputStream(); + if (compressAlgo != NONE) { + compressor = compressionAlgorithm.getCompressor(); + compressedByteStream = new ByteArrayOutputStream(); + try { + compressionStream = + compressionAlgorithm.createPlainCompressionStream( + compressedByteStream, compressor); + } catch (IOException e) { + throw new RuntimeException("Could not create compression stream " + + "for algorithm " + compressionAlgorithm, e); + } + } + + prevOffsetByType = new long[BlockType.values().length]; + for (int i = 0; i < prevOffsetByType.length; ++i) + prevOffsetByType[i] = -1; + + this.includesMemstoreTS = includesMemstoreTS; + } + + /** + * Starts writing into the block. The previous block's data is discarded. + * + * @return the stream the user can write their data into + * @throws IOException + */ + public DataOutputStream startWriting(BlockType newBlockType) + throws IOException { + if (state == State.BLOCK_READY && startOffset != -1) { + // We had a previous block that was written to a stream at a specific + // offset. Save that offset as the last offset of a block of that type. + prevOffsetByType[blockType.getId()] = startOffset; + } + + startOffset = -1; + blockType = newBlockType; + + baosInMemory.reset(); + baosInMemory.write(DUMMY_HEADER); + + state = State.WRITING; + + // We will compress it later in finishBlock() + userDataStream = new DataOutputStream(baosInMemory); + return userDataStream; + } + + /** + * Returns the stream for the user to write to. The block writer takes care + * of handling compression and buffering for caching on write. Can only be + * called in the "writing" state. + * + * @return the data output stream for the user to write to + */ + DataOutputStream getUserDataStream() { + expectState(State.WRITING); + return userDataStream; + } + + /** + * Transitions the block writer from the "writing" state to the "block + * ready" state. Does nothing if a block is already finished. + */ + private void ensureBlockReady() throws IOException { + Preconditions.checkState(state != State.INIT, + "Unexpected state: " + state); + + if (state == State.BLOCK_READY) + return; + + // This will set state to BLOCK_READY. + finishBlock(); + } + + /** + * An internal method that flushes the compressing stream (if using + * compression), serializes the header, and takes care of the separate + * uncompressed stream for caching on write, if applicable. Sets block + * write state to "block ready". + */ + private void finishBlock() throws IOException { + userDataStream.flush(); + + // This does an array copy, so it is safe to cache this byte array. + uncompressedBytesWithHeader = baosInMemory.toByteArray(); + LOG.warn("Writer.finishBlock user data size with header before compression " + + uncompressedBytesWithHeader.length); + prevOffset = prevOffsetByType[blockType.getId()]; + + // We need to set state before we can package the block up for + // cache-on-write. In a way, the block is ready, but not yet encoded or + // compressed. + state = State.BLOCK_READY; + encodeDataBlockForDisk(); + + doCompression(); + putHeader(uncompressedBytesWithHeader, 0, onDiskBytesWithHeader.length, + uncompressedBytesWithHeader.length); + } + + /** + * Do compression if it is enabled, or re-use the uncompressed buffer if + * it is not. Fills in the compressed block's header if doing compression. + */ + private void doCompression() throws IOException { + // do the compression + if (compressAlgo != NONE) { + compressedByteStream.reset(); + compressedByteStream.write(DUMMY_HEADER); + + compressionStream.resetState(); + + compressionStream.write(uncompressedBytesWithHeader, HEADER_SIZE, + uncompressedBytesWithHeader.length - HEADER_SIZE); + + compressionStream.flush(); + compressionStream.finish(); + + onDiskBytesWithHeader = compressedByteStream.toByteArray(); + putHeader(onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length, + uncompressedBytesWithHeader.length); + } else { + onDiskBytesWithHeader = uncompressedBytesWithHeader; + } + } + + /** + * Encodes this block if it is a data block and encoding is turned on in + * {@link #dataBlockEncoder}. + */ + private void encodeDataBlockForDisk() throws IOException { + if (blockType != BlockType.DATA) { + return; // skip any non-data block + } + + // do data block encoding, if data block encoder is set + ByteBuffer rawKeyValues = ByteBuffer.wrap(uncompressedBytesWithHeader, + HEADER_SIZE, uncompressedBytesWithHeader.length - + HEADER_SIZE).slice(); + Pair encodingResult = + dataBlockEncoder.beforeWriteToDisk(rawKeyValues, + includesMemstoreTS, DUMMY_HEADER); + + BlockType encodedBlockType = encodingResult.getSecond(); + if (encodedBlockType == BlockType.ENCODED_DATA) { + uncompressedBytesWithHeader = encodingResult.getFirst().array(); + blockType = BlockType.ENCODED_DATA; + } else { + // There is no encoding configured. Do some extra sanity-checking. + if (encodedBlockType != BlockType.DATA) { + throw new IOException("Unexpected block type coming out of data " + + "block encoder: " + encodedBlockType); + } + if (userDataStream.size() != + uncompressedBytesWithHeader.length - HEADER_SIZE) { + throw new IOException("Uncompressed size mismatch: " + + userDataStream.size() + " vs. " + + (uncompressedBytesWithHeader.length - HEADER_SIZE)); + } + } + } + + /** + * Put the header into the given byte array at the given offset. + * @param onDiskSize size of the block on disk + * @param uncompressedSize size of the block after decompression (but + * before optional data block decoding) + */ + private void putHeader(byte[] dest, int offset, int onDiskSize, + int uncompressedSize) { + offset = blockType.put(dest, offset); + offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE); + offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE); + Bytes.putLong(dest, offset, prevOffset); + } + + /** + * Similar to {@link #writeHeaderAndData(FSDataOutputStream)}, but records + * the offset of this block so that it can be referenced in the next block + * of the same type. + * + * @param out + * @throws IOException + */ + public void writeHeaderAndData(FSDataOutputStream out) throws IOException { + long offset = out.getPos(); + if (startOffset != -1 && offset != startOffset) { + throw new IOException("A " + blockType + " block written to a " + + "stream twice, first at offset " + startOffset + ", then at " + + offset); + } + startOffset = offset; + + writeHeaderAndData((DataOutputStream) out); + } + + /** + * Writes the header and the compressed data of this block (or uncompressed + * data when not using compression) into the given stream. Can be called in + * the "writing" state or in the "block ready" state. If called in the + * "writing" state, transitions the writer to the "block ready" state. + * + * @param out the output stream to write the + * @throws IOException + */ + private void writeHeaderAndData(DataOutputStream out) throws IOException { + ensureBlockReady(); + out.write(onDiskBytesWithHeader); + } + + /** + * Returns the header or the compressed data (or uncompressed data when not + * using compression) as a byte array. Can be called in the "writing" state + * or in the "block ready" state. If called in the "writing" state, + * transitions the writer to the "block ready" state. + * + * @return header and data as they would be stored on disk in a byte array + * @throws IOException + */ + public byte[] getHeaderAndData() throws IOException { + ensureBlockReady(); + return onDiskBytesWithHeader; + } + + /** + * Releases the compressor this writer uses to compress blocks into the + * compressor pool. Needs to be called before the writer is discarded. + */ + public void releaseCompressor() { + if (compressor != null) { + compressAlgo.returnCompressor(compressor); + compressor = null; + } + } + + /** + * Returns the on-disk size of the data portion of the block. This is the + * compressed size if compression is enabled. Can only be called in the + * "block ready" state. Header is not compressed, and its size is not + * included in the return value. + * + * @return the on-disk size of the block, not including the header. + */ + public int getOnDiskSizeWithoutHeader() { + expectState(State.BLOCK_READY); + return onDiskBytesWithHeader.length - HEADER_SIZE; + } + + /** + * Returns the on-disk size of the block. Can only be called in the + * "block ready" state. + * + * @return the on-disk size of the block ready to be written, including the + * header size + */ + public int getOnDiskSizeWithHeader() { + expectState(State.BLOCK_READY); + return onDiskBytesWithHeader.length; + } + + /** + * The uncompressed size of the block data. Does not include header size. + */ + public int getUncompressedSizeWithoutHeader() { + expectState(State.BLOCK_READY); + return uncompressedBytesWithHeader.length - HEADER_SIZE; + } + + /** + * The uncompressed size of the block data, including header size. + */ + public int getUncompressedSizeWithHeader() { + expectState(State.BLOCK_READY); + return uncompressedBytesWithHeader.length; + } + + /** @return true if a block is being written */ + public boolean isWriting() { + return state == State.WRITING; + } + + /** + * Returns the number of bytes written into the current block so far, or + * zero if not writing the block at the moment. Note that this will return + * zero in the "block ready" state as well. + * + * @return the number of bytes written + */ + public int blockSizeWritten() { + if (state != State.WRITING) + return 0; + return userDataStream.size(); + } + + /** + * Returns the header followed by the uncompressed data, even if using + * compression. This is needed for storing uncompressed blocks in the block + * cache. Can be called in the "writing" state or the "block ready" state. + * + * @return uncompressed block bytes for caching on write + */ + private byte[] getUncompressedDataWithHeader() { + expectState(State.BLOCK_READY); + + return uncompressedBytesWithHeader; + } + + private void expectState(State expectedState) { + if (state != expectedState) { + throw new IllegalStateException("Expected state: " + expectedState + + ", actual state: " + state); + } + } + + /** + * Similar to {@link #getUncompressedBufferWithHeader()} but returns a byte + * buffer. + * + * @return uncompressed block for caching on write in the form of a buffer + */ + public ByteBuffer getUncompressedBufferWithHeader() { + byte[] b = getUncompressedDataWithHeader(); + return ByteBuffer.wrap(b, 0, b.length); + } + + /** + * Takes the given {@link BlockWritable} instance, creates a new block of + * its appropriate type, writes the writable into this block, and flushes + * the block into the output stream. The writer is instructed not to buffer + * uncompressed bytes for cache-on-write. + * + * @param bw the block-writable object to write as a block + * @param out the file system output stream + * @throws IOException + */ + public void writeBlock(BlockWritable bw, FSDataOutputStream out) + throws IOException { + bw.writeToBlock(startWriting(bw.getBlockType())); + writeHeaderAndData(out); + } + + /** + * Creates a new HFileBlock. + */ + public HFileBlock getBlockForCaching() { + return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(), + getUncompressedSizeWithoutHeader(), prevOffset, + getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset, + includesMemstoreTS, MINOR_VERSION, 0, ChecksumType.NULL.getCode(), + getOnDiskSizeWithoutHeader()); + } + } +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java new file mode 100644 index 0000000..6280c21 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java @@ -0,0 +1,621 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader; +import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexChunk; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +@Category(MediumTests.class) +public class TestHFileBlockIndex { + + @Parameters + public static Collection compressionAlgorithms() { + return HBaseTestingUtility.COMPRESSION_ALGORITHMS_PARAMETERIZED; + } + + public TestHFileBlockIndex(Compression.Algorithm compr) { + this.compr = compr; + } + + private static final Log LOG = LogFactory.getLog(TestHFileBlockIndex.class); + + private static final int NUM_DATA_BLOCKS = 1000; + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + private static final int SMALL_BLOCK_SIZE = 4096; + private static final int NUM_KV = 10000; + + private static FileSystem fs; + private Path path; + private Random rand; + private long rootIndexOffset; + private int numRootEntries; + private int numLevels; + private static final List keys = new ArrayList(); + private final Compression.Algorithm compr; + private byte[] firstKeyInFile; + private Configuration conf; + + private static final int[] INDEX_CHUNK_SIZES = { 4096, 512, 384 }; + private static final int[] EXPECTED_NUM_LEVELS = { 2, 3, 4 }; + private static final int[] UNCOMPRESSED_INDEX_SIZES = + { 19187, 21813, 23086 }; + + private static final boolean includesMemstoreTS = true; + + static { + assert INDEX_CHUNK_SIZES.length == EXPECTED_NUM_LEVELS.length; + assert INDEX_CHUNK_SIZES.length == UNCOMPRESSED_INDEX_SIZES.length; + } + + @Before + public void setUp() throws IOException { + keys.clear(); + rand = new Random(2389757); + firstKeyInFile = null; + conf = TEST_UTIL.getConfiguration(); + + // This test requires at least HFile format version 2. + conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION); + + fs = HFileSystem.get(conf); + } + + @Test + public void testBlockIndex() throws IOException { + path = new Path(TEST_UTIL.getDataTestDir(), "block_index_" + compr); + writeWholeIndex(); + readIndex(); + } + + /** + * A wrapper around a block reader which only caches the results of the last + * operation. Not thread-safe. + */ + private static class BlockReaderWrapper implements HFile.CachingBlockReader { + + private HFileBlock.FSReader realReader; + private long prevOffset; + private long prevOnDiskSize; + private boolean prevPread; + private HFileBlock prevBlock; + + public int hitCount = 0; + public int missCount = 0; + + public BlockReaderWrapper(HFileBlock.FSReader realReader) { + this.realReader = realReader; + } + + @Override + public HFileBlock readBlock(long offset, long onDiskSize, + boolean cacheBlock, boolean pread, boolean isCompaction, + BlockType expectedBlockType) + throws IOException { + if (offset == prevOffset && onDiskSize == prevOnDiskSize && + pread == prevPread) { + hitCount += 1; + return prevBlock; + } + + missCount += 1; + prevBlock = realReader.readBlockData(offset, onDiskSize, + -1, pread); + prevOffset = offset; + prevOnDiskSize = onDiskSize; + prevPread = pread; + + return prevBlock; + } + } + + public void readIndex() throws IOException { + long fileSize = fs.getFileStatus(path).getLen(); + LOG.info("Size of " + path + ": " + fileSize); + + FSDataInputStream istream = fs.open(path); + HFileBlock.FSReader blockReader = new HFileBlock.FSReaderV2(istream, + compr, fs.getFileStatus(path).getLen()); + + BlockReaderWrapper brw = new BlockReaderWrapper(blockReader); + HFileBlockIndex.BlockIndexReader indexReader = + new HFileBlockIndex.BlockIndexReader( + Bytes.BYTES_RAWCOMPARATOR, numLevels, brw); + + indexReader.readRootIndex(blockReader.blockRange(rootIndexOffset, + fileSize).nextBlockWithBlockType(BlockType.ROOT_INDEX), numRootEntries); + + long prevOffset = -1; + int i = 0; + int expectedHitCount = 0; + int expectedMissCount = 0; + LOG.info("Total number of keys: " + keys.size()); + for (byte[] key : keys) { + assertTrue(key != null); + assertTrue(indexReader != null); + HFileBlock b = indexReader.seekToDataBlock(key, 0, key.length, null, + true, true, false); + if (Bytes.BYTES_RAWCOMPARATOR.compare(key, firstKeyInFile) < 0) { + assertTrue(b == null); + ++i; + continue; + } + + String keyStr = "key #" + i + ", " + Bytes.toStringBinary(key); + + assertTrue("seekToDataBlock failed for " + keyStr, b != null); + + if (prevOffset == b.getOffset()) { + assertEquals(++expectedHitCount, brw.hitCount); + } else { + LOG.info("First key in a new block: " + keyStr + ", block offset: " + + b.getOffset() + ")"); + assertTrue(b.getOffset() > prevOffset); + assertEquals(++expectedMissCount, brw.missCount); + prevOffset = b.getOffset(); + } + ++i; + } + + istream.close(); + } + + private void writeWholeIndex() throws IOException { + assertEquals(0, keys.size()); + HFileBlock.Writer hbw = new HFileBlock.Writer(compr, null, + includesMemstoreTS, + 1, + HFile.DEFAULT_CHECKSUM_TYPE, + HFile.DEFAULT_BYTES_PER_CHECKSUM); + FSDataOutputStream outputStream = fs.create(path); + HFileBlockIndex.BlockIndexWriter biw = + new HFileBlockIndex.BlockIndexWriter(hbw, null, null); + + for (int i = 0; i < NUM_DATA_BLOCKS; ++i) { + hbw.startWriting(BlockType.DATA).write( + String.valueOf(rand.nextInt(1000)).getBytes()); + long blockOffset = outputStream.getPos(); + hbw.writeHeaderAndData(outputStream); + + byte[] firstKey = null; + for (int j = 0; j < 16; ++j) { + byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i * 16 + j); + keys.add(k); + if (j == 8) + firstKey = k; + } + assertTrue(firstKey != null); + if (firstKeyInFile == null) + firstKeyInFile = firstKey; + biw.addEntry(firstKey, blockOffset, hbw.getOnDiskSizeWithHeader()); + + writeInlineBlocks(hbw, outputStream, biw, false); + } + writeInlineBlocks(hbw, outputStream, biw, true); + rootIndexOffset = biw.writeIndexBlocks(outputStream); + outputStream.close(); + + numLevels = biw.getNumLevels(); + numRootEntries = biw.getNumRootEntries(); + + LOG.info("Index written: numLevels=" + numLevels + ", numRootEntries=" + + numRootEntries + ", rootIndexOffset=" + rootIndexOffset); + } + + private void writeInlineBlocks(HFileBlock.Writer hbw, + FSDataOutputStream outputStream, HFileBlockIndex.BlockIndexWriter biw, + boolean isClosing) throws IOException { + while (biw.shouldWriteBlock(isClosing)) { + long offset = outputStream.getPos(); + biw.writeInlineBlock(hbw.startWriting(biw.getInlineBlockType())); + hbw.writeHeaderAndData(outputStream); + biw.blockWritten(offset, hbw.getOnDiskSizeWithHeader(), + hbw.getUncompressedSizeWithoutHeader()); + LOG.info("Wrote an inline index block at " + offset + ", size " + + hbw.getOnDiskSizeWithHeader()); + } + } + + private static final long getDummyFileOffset(int i) { + return i * 185 + 379; + } + + private static final int getDummyOnDiskSize(int i) { + return i * i * 37 + i * 19 + 13; + } + + @Test + public void testSecondaryIndexBinarySearch() throws IOException { + int numTotalKeys = 99; + assertTrue(numTotalKeys % 2 == 1); // Ensure no one made this even. + + // We only add odd-index keys into the array that we will binary-search. + int numSearchedKeys = (numTotalKeys - 1) / 2; + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + + dos.writeInt(numSearchedKeys); + int curAllEntriesSize = 0; + int numEntriesAdded = 0; + + // Only odd-index elements of this array are used to keep the secondary + // index entries of the corresponding keys. + int secondaryIndexEntries[] = new int[numTotalKeys]; + + for (int i = 0; i < numTotalKeys; ++i) { + byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i * 2); + keys.add(k); + String msgPrefix = "Key #" + i + " (" + Bytes.toStringBinary(k) + "): "; + StringBuilder padding = new StringBuilder(); + while (msgPrefix.length() + padding.length() < 70) + padding.append(' '); + msgPrefix += padding; + if (i % 2 == 1) { + dos.writeInt(curAllEntriesSize); + secondaryIndexEntries[i] = curAllEntriesSize; + LOG.info(msgPrefix + "secondary index entry #" + ((i - 1) / 2) + + ", offset " + curAllEntriesSize); + curAllEntriesSize += k.length + + HFileBlockIndex.SECONDARY_INDEX_ENTRY_OVERHEAD; + ++numEntriesAdded; + } else { + secondaryIndexEntries[i] = -1; + LOG.info(msgPrefix + "not in the searched array"); + } + } + + // Make sure the keys are increasing. + for (int i = 0; i < keys.size() - 1; ++i) + assertTrue(Bytes.BYTES_RAWCOMPARATOR.compare(keys.get(i), + keys.get(i + 1)) < 0); + + dos.writeInt(curAllEntriesSize); + assertEquals(numSearchedKeys, numEntriesAdded); + int secondaryIndexOffset = dos.size(); + assertEquals(Bytes.SIZEOF_INT * (numSearchedKeys + 2), + secondaryIndexOffset); + + for (int i = 1; i <= numTotalKeys - 1; i += 2) { + assertEquals(dos.size(), + secondaryIndexOffset + secondaryIndexEntries[i]); + long dummyFileOffset = getDummyFileOffset(i); + int dummyOnDiskSize = getDummyOnDiskSize(i); + LOG.debug("Storing file offset=" + dummyFileOffset + " and onDiskSize=" + + dummyOnDiskSize + " at offset " + dos.size()); + dos.writeLong(dummyFileOffset); + dos.writeInt(dummyOnDiskSize); + LOG.debug("Stored key " + ((i - 1) / 2) +" at offset " + dos.size()); + dos.write(keys.get(i)); + } + + dos.writeInt(curAllEntriesSize); + + ByteBuffer nonRootIndex = ByteBuffer.wrap(baos.toByteArray()); + for (int i = 0; i < numTotalKeys; ++i) { + byte[] searchKey = keys.get(i); + byte[] arrayHoldingKey = new byte[searchKey.length + + searchKey.length / 2]; + + // To make things a bit more interesting, store the key we are looking + // for at a non-zero offset in a new array. + System.arraycopy(searchKey, 0, arrayHoldingKey, searchKey.length / 2, + searchKey.length); + + int searchResult = BlockIndexReader.binarySearchNonRootIndex( + arrayHoldingKey, searchKey.length / 2, searchKey.length, nonRootIndex, + Bytes.BYTES_RAWCOMPARATOR); + String lookupFailureMsg = "Failed to look up key #" + i + " (" + + Bytes.toStringBinary(searchKey) + ")"; + + int expectedResult; + int referenceItem; + + if (i % 2 == 1) { + // This key is in the array we search as the element (i - 1) / 2. Make + // sure we find it. + expectedResult = (i - 1) / 2; + referenceItem = i; + } else { + // This key is not in the array but between two elements on the array, + // in the beginning, or in the end. The result should be the previous + // key in the searched array, or -1 for i = 0. + expectedResult = i / 2 - 1; + referenceItem = i - 1; + } + + assertEquals(lookupFailureMsg, expectedResult, searchResult); + + // Now test we can get the offset and the on-disk-size using a + // higher-level API function.s + boolean locateBlockResult = + (BlockIndexReader.locateNonRootIndexEntry(nonRootIndex, arrayHoldingKey, + searchKey.length / 2, searchKey.length, Bytes.BYTES_RAWCOMPARATOR) != -1); + + if (i == 0) { + assertFalse(locateBlockResult); + } else { + assertTrue(locateBlockResult); + String errorMsg = "i=" + i + ", position=" + nonRootIndex.position(); + assertEquals(errorMsg, getDummyFileOffset(referenceItem), + nonRootIndex.getLong()); + assertEquals(errorMsg, getDummyOnDiskSize(referenceItem), + nonRootIndex.getInt()); + } + } + + } + + @Test + public void testBlockIndexChunk() throws IOException { + BlockIndexChunk c = new BlockIndexChunk(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int N = 1000; + int[] numSubEntriesAt = new int[N]; + int numSubEntries = 0; + for (int i = 0; i < N; ++i) { + baos.reset(); + DataOutputStream dos = new DataOutputStream(baos); + c.writeNonRoot(dos); + assertEquals(c.getNonRootSize(), dos.size()); + + baos.reset(); + dos = new DataOutputStream(baos); + c.writeRoot(dos); + assertEquals(c.getRootSize(), dos.size()); + + byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i); + numSubEntries += rand.nextInt(5) + 1; + keys.add(k); + c.add(k, getDummyFileOffset(i), getDummyOnDiskSize(i), numSubEntries); + } + + // Test the ability to look up the entry that contains a particular + // deeper-level index block's entry ("sub-entry"), assuming a global + // 0-based ordering of sub-entries. This is needed for mid-key calculation. + for (int i = 0; i < N; ++i) { + for (int j = i == 0 ? 0 : numSubEntriesAt[i - 1]; + j < numSubEntriesAt[i]; + ++j) { + assertEquals(i, c.getEntryBySubEntry(j)); + } + } + } + + /** Checks if the HeapSize calculator is within reason */ + @Test + public void testHeapSizeForBlockIndex() throws IOException { + Class cl = + HFileBlockIndex.BlockIndexReader.class; + long expected = ClassSize.estimateBase(cl, false); + + HFileBlockIndex.BlockIndexReader bi = + new HFileBlockIndex.BlockIndexReader(Bytes.BYTES_RAWCOMPARATOR, 1); + long actual = bi.heapSize(); + + // Since the arrays in BlockIndex(byte [][] blockKeys, long [] blockOffsets, + // int [] blockDataSizes) are all null they are not going to show up in the + // HeapSize calculation, so need to remove those array costs from expected. + expected -= ClassSize.align(3 * ClassSize.ARRAY); + + if (expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + } + + /** + * Testing block index through the HFile writer/reader APIs. Allows to test + * setting index block size through configuration, intermediate-level index + * blocks, and caching index blocks on write. + * + * @throws IOException + */ + @Test + public void testHFileWriterAndReader() throws IOException { + Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), + "hfile_for_block_index"); + CacheConfig cacheConf = new CacheConfig(conf); + BlockCache blockCache = cacheConf.getBlockCache(); + + for (int testI = 0; testI < INDEX_CHUNK_SIZES.length; ++testI) { + int indexBlockSize = INDEX_CHUNK_SIZES[testI]; + int expectedNumLevels = EXPECTED_NUM_LEVELS[testI]; + LOG.info("Index block size: " + indexBlockSize + ", compression: " + + compr); + // Evict all blocks that were cached-on-write by the previous invocation. + blockCache.evictBlocksByHfileName(hfilePath.getName()); + + conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, indexBlockSize); + Set keyStrSet = new HashSet(); + byte[][] keys = new byte[NUM_KV][]; + byte[][] values = new byte[NUM_KV][]; + + // Write the HFile + { + HFile.Writer writer = + HFile.getWriterFactory(conf, cacheConf) + .withPath(fs, hfilePath) + .withBlockSize(SMALL_BLOCK_SIZE) + .withCompression(compr) + .withComparator(KeyValue.KEY_COMPARATOR) + .create(); + Random rand = new Random(19231737); + + for (int i = 0; i < NUM_KV; ++i) { + byte[] row = TestHFileWriterV2.randomOrderedKey(rand, i); + + // Key will be interpreted by KeyValue.KEY_COMPARATOR + byte[] k = KeyValue.createFirstOnRow(row, 0, row.length, row, 0, 0, + row, 0, 0).getKey(); + + byte[] v = TestHFileWriterV2.randomValue(rand); + writer.append(k, v); + keys[i] = k; + values[i] = v; + keyStrSet.add(Bytes.toStringBinary(k)); + + if (i > 0) { + assertTrue(KeyValue.KEY_COMPARATOR.compare(keys[i - 1], + keys[i]) < 0); + } + } + + writer.close(); + } + + // Read the HFile + HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConf); + assertEquals(expectedNumLevels, + reader.getTrailer().getNumDataIndexLevels()); + + assertTrue(Bytes.equals(keys[0], reader.getFirstKey())); + assertTrue(Bytes.equals(keys[NUM_KV - 1], reader.getLastKey())); + LOG.info("Last key: " + Bytes.toStringBinary(keys[NUM_KV - 1])); + + for (boolean pread : new boolean[] { false, true }) { + HFileScanner scanner = reader.getScanner(true, pread); + for (int i = 0; i < NUM_KV; ++i) { + checkSeekTo(keys, scanner, i); + checkKeyValue("i=" + i, keys[i], values[i], scanner.getKey(), + scanner.getValue()); + } + assertTrue(scanner.seekTo()); + for (int i = NUM_KV - 1; i >= 0; --i) { + checkSeekTo(keys, scanner, i); + checkKeyValue("i=" + i, keys[i], values[i], scanner.getKey(), + scanner.getValue()); + } + } + + // Manually compute the mid-key and validate it. + HFileReaderV2 reader2 = (HFileReaderV2) reader; + HFileBlock.FSReader fsReader = reader2.getUncachedBlockReader(); + + HFileBlock.BlockIterator iter = fsReader.blockRange(0, + reader.getTrailer().getLoadOnOpenDataOffset()); + HFileBlock block; + List blockKeys = new ArrayList(); + while ((block = iter.nextBlock()) != null) { + if (block.getBlockType() != BlockType.LEAF_INDEX) + return; + ByteBuffer b = block.getBufferReadOnly(); + int n = b.getInt(); + // One int for the number of items, and n + 1 for the secondary index. + int entriesOffset = Bytes.SIZEOF_INT * (n + 2); + + // Get all the keys from the leaf index block. S + for (int i = 0; i < n; ++i) { + int keyRelOffset = b.getInt(Bytes.SIZEOF_INT * (i + 1)); + int nextKeyRelOffset = b.getInt(Bytes.SIZEOF_INT * (i + 2)); + int keyLen = nextKeyRelOffset - keyRelOffset; + int keyOffset = b.arrayOffset() + entriesOffset + keyRelOffset + + HFileBlockIndex.SECONDARY_INDEX_ENTRY_OVERHEAD; + byte[] blockKey = Arrays.copyOfRange(b.array(), keyOffset, keyOffset + + keyLen); + String blockKeyStr = Bytes.toString(blockKey); + blockKeys.add(blockKey); + + // If the first key of the block is not among the keys written, we + // are not parsing the non-root index block format correctly. + assertTrue("Invalid block key from leaf-level block: " + blockKeyStr, + keyStrSet.contains(blockKeyStr)); + } + } + + // Validate the mid-key. + assertEquals( + Bytes.toStringBinary(blockKeys.get((blockKeys.size() - 1) / 2)), + Bytes.toStringBinary(reader.midkey())); + + assertEquals(UNCOMPRESSED_INDEX_SIZES[testI], + reader.getTrailer().getUncompressedDataIndexSize()); + + reader.close(); + reader2.close(); + } + } + + private void checkSeekTo(byte[][] keys, HFileScanner scanner, int i) + throws IOException { + assertEquals("Failed to seek to key #" + i + " (" + + Bytes.toStringBinary(keys[i]) + ")", 0, scanner.seekTo(keys[i])); + } + + private void assertArrayEqualsBuffer(String msgPrefix, byte[] arr, + ByteBuffer buf) { + assertEquals(msgPrefix + ": expected " + Bytes.toStringBinary(arr) + + ", actual " + Bytes.toStringBinary(buf), 0, Bytes.compareTo(arr, 0, + arr.length, buf.array(), buf.arrayOffset(), buf.limit())); + } + + /** Check a key/value pair after it was read by the reader */ + private void checkKeyValue(String msgPrefix, byte[] expectedKey, + byte[] expectedValue, ByteBuffer keyRead, ByteBuffer valueRead) { + if (!msgPrefix.isEmpty()) + msgPrefix += ". "; + + assertArrayEqualsBuffer(msgPrefix + "Invalid key", expectedKey, keyRead); + assertArrayEqualsBuffer(msgPrefix + "Invalid value", expectedValue, + valueRead); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java new file mode 100644 index 0000000..0c076f2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.encoding.RedundantKVGenerator; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.util.ChecksumType; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +@Category(SmallTests.class) +public class TestHFileDataBlockEncoder { + private Configuration conf; + private final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + private HFileDataBlockEncoderImpl blockEncoder; + private RedundantKVGenerator generator = new RedundantKVGenerator(); + private SchemaConfigured UNKNOWN_TABLE_AND_CF = + SchemaConfigured.createUnknown(); + private boolean includesMemstoreTS; + + /** + * Create test for given data block encoding configuration. + * @param blockEncoder What kind of encoding policy will be used. + */ + public TestHFileDataBlockEncoder(HFileDataBlockEncoderImpl blockEncoder, + boolean includesMemstoreTS) { + this.blockEncoder = blockEncoder; + this.includesMemstoreTS = includesMemstoreTS; + System.err.println("On-disk encoding: " + blockEncoder.getEncodingOnDisk() + + ", in-cache encoding: " + blockEncoder.getEncodingInCache() + + ", includesMemstoreTS: " + includesMemstoreTS); + } + + /** + * Preparation before JUnit test. + */ + @Before + public void setUp() { + conf = TEST_UTIL.getConfiguration(); + SchemaMetrics.configureGlobally(conf); + } + + /** + * Cleanup after JUnit test. + */ + @After + public void tearDown() throws IOException { + TEST_UTIL.cleanupTestDir(); + } + + /** + * Test putting and taking out blocks into cache with different + * encoding options. + */ + @Test + public void testEncodingWithCache() { + HFileBlock block = getSampleHFileBlock(); + LruBlockCache blockCache = + new LruBlockCache(8 * 1024 * 1024, 32 * 1024, TEST_UTIL.getConfiguration()); + HFileBlock cacheBlock = blockEncoder.diskToCacheFormat(block, false); + BlockCacheKey cacheKey = new BlockCacheKey("test", 0); + blockCache.cacheBlock(cacheKey, cacheBlock); + + HeapSize heapSize = blockCache.getBlock(cacheKey, false, false); + assertTrue(heapSize instanceof HFileBlock); + + HFileBlock returnedBlock = (HFileBlock) heapSize;; + + if (blockEncoder.getEncodingInCache() == + DataBlockEncoding.NONE) { + assertEquals(block.getBufferWithHeader(), + returnedBlock.getBufferWithHeader()); + } else { + if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) { + System.out.println(blockEncoder); + } + assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType()); + } + } + + /** + * Test writing to disk. + */ + @Test + public void testEncodingWritePath() { + // usually we have just block without headers, but don't complicate that + HFileBlock block = getSampleHFileBlock(); + Pair result = + blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(), + includesMemstoreTS, HFileBlock.DUMMY_HEADER_WITH_CHECKSUM); + + int size = result.getFirst().limit() - HFileBlock.HEADER_SIZE_WITH_CHECKSUMS; + HFileBlock blockOnDisk = new HFileBlock(result.getSecond(), + size, size, -1, result.getFirst(), HFileBlock.FILL_HEADER, 0, + includesMemstoreTS, block.getMinorVersion(), + block.getBytesPerChecksum(), block.getChecksumType(), + block.getOnDiskDataSizeWithHeader()); + + if (blockEncoder.getEncodingOnDisk() != + DataBlockEncoding.NONE) { + assertEquals(BlockType.ENCODED_DATA, blockOnDisk.getBlockType()); + assertEquals(blockEncoder.getEncodingOnDisk().getId(), + blockOnDisk.getDataBlockEncodingId()); + } else { + assertEquals(BlockType.DATA, blockOnDisk.getBlockType()); + } + } + + /** + * Test converting blocks from disk to cache format. + */ + @Test + public void testEncodingReadPath() { + HFileBlock origBlock = getSampleHFileBlock(); + blockEncoder.diskToCacheFormat(origBlock, false); + } + + private HFileBlock getSampleHFileBlock() { + ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer( + generator.generateTestKeyValues(60), includesMemstoreTS); + int size = keyValues.limit(); + ByteBuffer buf = ByteBuffer.allocate(size + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS); + buf.position(HFileBlock.HEADER_SIZE_WITH_CHECKSUMS); + keyValues.rewind(); + buf.put(keyValues); + HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf, + HFileBlock.FILL_HEADER, 0, includesMemstoreTS, + HFileReaderV2.MAX_MINOR_VERSION, 0, ChecksumType.NULL.getCode(), 0); + UNKNOWN_TABLE_AND_CF.passSchemaMetricsTo(b); + return b; + } + + /** + * @return All possible data block encoding configurations + */ + @Parameters + public static Collection getAllConfigurations() { + List configurations = + new ArrayList(); + + for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) { + for (DataBlockEncoding cacheAlgo : DataBlockEncoding.values()) { + if (diskAlgo != cacheAlgo && diskAlgo != DataBlockEncoding.NONE) { + // We allow (1) the same encoding on disk and in cache, and + // (2) some encoding in cache but no encoding on disk (for testing). + continue; + } + for (boolean includesMemstoreTS : new boolean[] {false, true}) { + configurations.add(new Object[] { + new HFileDataBlockEncoderImpl(diskAlgo, cacheAlgo), + new Boolean(includesMemstoreTS)}); + } + } + } + + return configurations; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java new file mode 100644 index 0000000..419fc0b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.junit.experimental.categories.Category; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; + +/** + * Test a case when an inline index chunk is converted to a root one. This reproduces the bug in + * HBASE-6871. We write a carefully selected number of relatively large keys so that we accumulate + * a leaf index chunk that only goes over the configured index chunk size after adding the last + * key/value. The bug is in that when we close the file, we convert that inline (leaf-level) chunk + * into a root chunk, but then look at the size of that root chunk, find that it is greater than + * the configured chunk size, and split it into a number of intermediate index blocks that should + * really be leaf-level blocks. If more keys were added, we would flush the leaf-level block, add + * another entry to the root-level block, and that would prevent us from upgrading the leaf-level + * chunk to the root chunk, thus not triggering the bug. + */ +@Category(SmallTests.class) +public class TestHFileInlineToRootChunkConversion { + private final HBaseTestingUtility testUtil = new HBaseTestingUtility(); + private final Configuration conf = testUtil.getConfiguration(); + + @Test + public void testWriteHFile() throws Exception { + Path hfPath = new Path(testUtil.getDataTestDir(), + TestHFileInlineToRootChunkConversion.class.getSimpleName() + ".hfile"); + int maxChunkSize = 1024; + FileSystem fs = FileSystem.get(conf); + CacheConfig cacheConf = new CacheConfig(conf); + conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, maxChunkSize); + HFileWriterV2 hfw = + (HFileWriterV2) new HFileWriterV2.WriterFactoryV2(conf, cacheConf) + .withBlockSize(16) + .withPath(fs, hfPath).create(); + List keys = new ArrayList(); + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < 4; ++i) { + sb.append("key" + String.format("%05d", i)); + sb.append("_"); + for (int j = 0; j < 100; ++j) { + sb.append('0' + j); + } + String keyStr = sb.toString(); + sb.setLength(0); + + byte[] k = Bytes.toBytes(keyStr); + System.out.println("Key: " + Bytes.toString(k)); + keys.add(k); + byte[] v = Bytes.toBytes("value" + i); + hfw.append(k, v); + } + hfw.close(); + + HFileReaderV2 reader = (HFileReaderV2) HFile.createReader(fs, hfPath, cacheConf); + HFileScanner scanner = reader.getScanner(true, true); + for (int i = 0; i < keys.size(); ++i) { + scanner.seekTo(keys.get(i)); + } + reader.close(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java new file mode 100644 index 0000000..86348d7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java @@ -0,0 +1,397 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Random; + +import junit.framework.TestCase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.io.compress.GzipCodec; +import org.junit.experimental.categories.Category; + +/** + * Set of long-running tests to measure performance of HFile. + *

    + * Copied from + * hadoop-3315 tfile. + * Remove after tfile is committed and use the tfile version of this class + * instead.

    + */ +@Category(MediumTests.class) +public class TestHFilePerformance extends TestCase { + private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static String ROOT_DIR = + TEST_UTIL.getDataTestDir("TestHFilePerformance").toString(); + private FileSystem fs; + private Configuration conf; + private long startTimeEpoch; + private long finishTimeEpoch; + private DateFormat formatter; + + @Override + public void setUp() throws IOException { + conf = new Configuration(); + fs = FileSystem.get(conf); + formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + } + + public void startTime() { + startTimeEpoch = System.currentTimeMillis(); + System.out.println(formatTime() + " Started timing."); + } + + public void stopTime() { + finishTimeEpoch = System.currentTimeMillis(); + System.out.println(formatTime() + " Stopped timing."); + } + + public long getIntervalMillis() { + return finishTimeEpoch - startTimeEpoch; + } + + public void printlnWithTimestamp(String message) { + System.out.println(formatTime() + " " + message); + } + + /* + * Format millis into minutes and seconds. + */ + public String formatTime(long milis){ + return formatter.format(milis); + } + + public String formatTime(){ + return formatTime(System.currentTimeMillis()); + } + + private FSDataOutputStream createFSOutput(Path name) throws IOException { + if (fs.exists(name)) + fs.delete(name, true); + FSDataOutputStream fout = fs.create(name); + return fout; + } + + //TODO have multiple ways of generating key/value e.g. dictionary words + //TODO to have a sample compressable data, for now, made 1 out of 3 values random + // keys are all random. + + private static class KeyValueGenerator { + Random keyRandomizer; + Random valueRandomizer; + long randomValueRatio = 3; // 1 out of randomValueRatio generated values will be random. + long valueSequence = 0 ; + + + KeyValueGenerator() { + keyRandomizer = new Random(0L); //TODO with seed zero + valueRandomizer = new Random(1L); //TODO with seed one + } + + // Key is always random now. + void getKey(byte[] key) { + keyRandomizer.nextBytes(key); + } + + void getValue(byte[] value) { + if (valueSequence % randomValueRatio == 0) + valueRandomizer.nextBytes(value); + valueSequence++; + } + } + + /** + * + * @param fileType "HFile" or "SequenceFile" + * @param keyLength + * @param valueLength + * @param codecName "none", "lzo", "gz", "snappy" + * @param rows number of rows to be written. + * @param writeMethod used for HFile only. + * @param minBlockSize used for HFile only. + * @throws IOException + */ + //TODO writeMethod: implement multiple ways of writing e.g. A) known length (no chunk) B) using a buffer and streaming (for many chunks). + public void timeWrite(String fileType, int keyLength, int valueLength, + String codecName, long rows, String writeMethod, int minBlockSize) + throws IOException { + System.out.println("File Type: " + fileType); + System.out.println("Writing " + fileType + " with codecName: " + codecName); + long totalBytesWritten = 0; + + + //Using separate randomizer for key/value with seeds matching Sequence File. + byte[] key = new byte[keyLength]; + byte[] value = new byte[valueLength]; + KeyValueGenerator generator = new KeyValueGenerator(); + + startTime(); + + Path path = new Path(ROOT_DIR, fileType + ".Performance"); + System.out.println(ROOT_DIR + path.getName()); + FSDataOutputStream fout = createFSOutput(path); + + if ("HFile".equals(fileType)){ + System.out.println("HFile write method: "); + HFile.Writer writer = HFile.getWriterFactoryNoCache(conf) + .withOutputStream(fout) + .withBlockSize(minBlockSize) + .withCompression(codecName) + .create(); + + // Writing value in one shot. + for (long l=0; l startingMetrics; + + private static final int N = 1000; + + @Before + public void setUp() throws IOException { + startingMetrics = SchemaMetrics.getMetricsSnapshot(); + conf = TEST_UTIL.getConfiguration(); + fs = FileSystem.get(conf); + SchemaMetrics.configureGlobally(conf); + } + + @After + public void tearDown() throws Exception { + SchemaMetrics.validateMetricChanges(startingMetrics); + } + + @Test + public void testReadingExistingVersion1HFile() throws IOException { + URL url = TestHFileReaderV1.class.getResource( + "8e8ab58dcf39412da19833fcd8f687ac"); + Path existingHFilePath = new Path(url.getPath()); + HFile.Reader reader = + HFile.createReader(fs, existingHFilePath, new CacheConfig(conf)); + reader.loadFileInfo(); + FixedFileTrailer trailer = reader.getTrailer(); + + assertEquals(N, reader.getEntries()); + assertEquals(N, trailer.getEntryCount()); + assertEquals(1, trailer.getMajorVersion()); + assertEquals(Compression.Algorithm.GZ, trailer.getCompressionCodec()); + + for (boolean pread : new boolean[] { false, true }) { + int totalDataSize = 0; + int n = 0; + + HFileScanner scanner = reader.getScanner(false, pread); + assertTrue(scanner.seekTo()); + do { + totalDataSize += scanner.getKey().limit() + scanner.getValue().limit() + + Bytes.SIZEOF_INT * 2; + ++n; + } while (scanner.next()); + + // Add magic record sizes, one per data block. + totalDataSize += 8 * trailer.getDataIndexCount(); + + assertEquals(N, n); + assertEquals(trailer.getTotalUncompressedBytes(), totalDataSize); + } + reader.close(); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java new file mode 100644 index 0000000..2160e3e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java @@ -0,0 +1,545 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hbase.io.hfile; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Random; +import java.util.StringTokenizer; + +import junit.framework.TestCase; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.io.hfile.HFile.Reader; +import org.apache.hadoop.hbase.io.hfile.HFile.Writer; +import org.apache.hadoop.io.BytesWritable; +import org.junit.experimental.categories.Category; + +/** + * test the performance for seek. + *

    + * Copied from + * hadoop-3315 tfile. + * Remove after tfile is committed and use the tfile version of this class + * instead.

    + */ +@Category(MediumTests.class) +public class TestHFileSeek extends TestCase { + private static final Log LOG = LogFactory.getLog(TestHFileSeek.class); + private static final boolean USE_PREAD = true; + private MyOptions options; + private Configuration conf; + private Path path; + private FileSystem fs; + private NanoTimer timer; + private Random rng; + private RandomDistribution.DiscreteRNG keyLenGen; + private KVGenerator kvGen; + + @Override + public void setUp() throws IOException { + if (options == null) { + options = new MyOptions(new String[0]); + } + + conf = new Configuration(); + + if (options.useRawFs) { + conf.setClass("fs.file.impl", RawLocalFileSystem.class, FileSystem.class); + } + + conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize); + conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize); + path = new Path(new Path(options.rootDir), options.file); + fs = path.getFileSystem(conf); + timer = new NanoTimer(false); + rng = new Random(options.seed); + keyLenGen = + new RandomDistribution.Zipf(new Random(rng.nextLong()), + options.minKeyLen, options.maxKeyLen, 1.2); + RandomDistribution.DiscreteRNG valLenGen = + new RandomDistribution.Flat(new Random(rng.nextLong()), + options.minValLength, options.maxValLength); + RandomDistribution.DiscreteRNG wordLenGen = + new RandomDistribution.Flat(new Random(rng.nextLong()), + options.minWordLen, options.maxWordLen); + kvGen = + new KVGenerator(rng, true, keyLenGen, valLenGen, wordLenGen, + options.dictSize); + } + + @Override + public void tearDown() { + try { + fs.close(); + } + catch (Exception e) { + // Nothing + } + } + + private static FSDataOutputStream createFSOutput(Path name, FileSystem fs) + throws IOException { + if (fs.exists(name)) { + fs.delete(name, true); + } + FSDataOutputStream fout = fs.create(name); + return fout; + } + + private void createTFile() throws IOException { + long totalBytes = 0; + FSDataOutputStream fout = createFSOutput(path, fs); + try { + Writer writer = HFile.getWriterFactoryNoCache(conf) + .withOutputStream(fout) + .withBlockSize(options.minBlockSize) + .withCompression(options.compress) + .create(); + try { + BytesWritable key = new BytesWritable(); + BytesWritable val = new BytesWritable(); + timer.start(); + for (long i = 0; true; ++i) { + if (i % 1000 == 0) { // test the size for every 1000 rows. + if (fs.getFileStatus(path).getLen() >= options.fileSize) { + break; + } + } + kvGen.next(key, val, false); + byte [] k = new byte [key.getLength()]; + System.arraycopy(key.getBytes(), 0, k, 0, key.getLength()); + byte [] v = new byte [val.getLength()]; + System.arraycopy(val.getBytes(), 0, v, 0, key.getLength()); + writer.append(k, v); + totalBytes += key.getLength(); + totalBytes += val.getLength(); + } + timer.stop(); + } + finally { + writer.close(); + } + } + finally { + fout.close(); + } + double duration = (double)timer.read()/1000; // in us. + long fsize = fs.getFileStatus(path).getLen(); + + System.out.printf( + "time: %s...uncompressed: %.2fMB...raw thrpt: %.2fMB/s\n", + timer.toString(), (double) totalBytes / 1024 / 1024, totalBytes + / duration); + System.out.printf("time: %s...file size: %.2fMB...disk thrpt: %.2fMB/s\n", + timer.toString(), (double) fsize / 1024 / 1024, fsize / duration); + } + + public void seekTFile() throws IOException { + int miss = 0; + long totalBytes = 0; + FSDataInputStream fsdis = fs.open(path); + Reader reader = HFile.createReaderFromStream(path, fsdis, + fs.getFileStatus(path).getLen(), new CacheConfig(conf)); + reader.loadFileInfo(); + KeySampler kSampler = + new KeySampler(rng, reader.getFirstKey(), reader.getLastKey(), + keyLenGen); + HFileScanner scanner = reader.getScanner(false, USE_PREAD); + BytesWritable key = new BytesWritable(); + timer.reset(); + timer.start(); + for (int i = 0; i < options.seekCount; ++i) { + kSampler.next(key); + byte [] k = new byte [key.getLength()]; + System.arraycopy(key.getBytes(), 0, k, 0, key.getLength()); + if (scanner.seekTo(k) >= 0) { + ByteBuffer bbkey = scanner.getKey(); + ByteBuffer bbval = scanner.getValue(); + totalBytes += bbkey.limit(); + totalBytes += bbval.limit(); + } + else { + ++miss; + } + } + timer.stop(); + System.out.printf( + "time: %s...avg seek: %s...%d hit...%d miss...avg I/O size: %.2fKB\n", + timer.toString(), NanoTimer.nanoTimeToString(timer.read() + / options.seekCount), options.seekCount - miss, miss, + (double) totalBytes / 1024 / (options.seekCount - miss)); + + } + + public void testSeeks() throws IOException { + if (options.doCreate()) { + createTFile(); + } + + if (options.doRead()) { + seekTFile(); + } + + if (options.doCreate()) { + fs.delete(path, true); + } + } + + private static class IntegerRange { + private final int from, to; + + public IntegerRange(int from, int to) { + this.from = from; + this.to = to; + } + + public static IntegerRange parse(String s) throws ParseException { + StringTokenizer st = new StringTokenizer(s, " \t,"); + if (st.countTokens() != 2) { + throw new ParseException("Bad integer specification: " + s); + } + int from = Integer.parseInt(st.nextToken()); + int to = Integer.parseInt(st.nextToken()); + return new IntegerRange(from, to); + } + + public int from() { + return from; + } + + public int to() { + return to; + } + } + + private static class MyOptions { + // hard coded constants + int dictSize = 1000; + int minWordLen = 5; + int maxWordLen = 20; + + private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + String rootDir = + TEST_UTIL.getDataTestDir("TestTFileSeek").toString(); + String file = "TestTFileSeek"; + // String compress = "lzo"; DISABLED + String compress = "none"; + int minKeyLen = 10; + int maxKeyLen = 50; + int minValLength = 1024; + int maxValLength = 2 * 1024; + int minBlockSize = 1 * 1024 * 1024; + int fsOutputBufferSize = 1; + int fsInputBufferSize = 0; + // Default writing 10MB. + long fileSize = 10 * 1024 * 1024; + long seekCount = 1000; + long trialCount = 1; + long seed; + boolean useRawFs = false; + + static final int OP_CREATE = 1; + static final int OP_READ = 2; + int op = OP_CREATE | OP_READ; + + boolean proceed = false; + + public MyOptions(String[] args) { + seed = System.nanoTime(); + + try { + Options opts = buildOptions(); + CommandLineParser parser = new GnuParser(); + CommandLine line = parser.parse(opts, args, true); + processOptions(line, opts); + validateOptions(); + } + catch (ParseException e) { + System.out.println(e.getMessage()); + System.out.println("Try \"--help\" option for details."); + setStopProceed(); + } + } + + public boolean proceed() { + return proceed; + } + + private Options buildOptions() { + Option compress = + OptionBuilder.withLongOpt("compress").withArgName("[none|lzo|gz|snappy]") + .hasArg().withDescription("compression scheme").create('c'); + + Option fileSize = + OptionBuilder.withLongOpt("file-size").withArgName("size-in-MB") + .hasArg().withDescription("target size of the file (in MB).") + .create('s'); + + Option fsInputBufferSz = + OptionBuilder.withLongOpt("fs-input-buffer").withArgName("size") + .hasArg().withDescription( + "size of the file system input buffer (in bytes).").create( + 'i'); + + Option fsOutputBufferSize = + OptionBuilder.withLongOpt("fs-output-buffer").withArgName("size") + .hasArg().withDescription( + "size of the file system output buffer (in bytes).").create( + 'o'); + + Option keyLen = + OptionBuilder + .withLongOpt("key-length") + .withArgName("min,max") + .hasArg() + .withDescription( + "the length range of the key (in bytes)") + .create('k'); + + Option valueLen = + OptionBuilder + .withLongOpt("value-length") + .withArgName("min,max") + .hasArg() + .withDescription( + "the length range of the value (in bytes)") + .create('v'); + + Option blockSz = + OptionBuilder.withLongOpt("block").withArgName("size-in-KB").hasArg() + .withDescription("minimum block size (in KB)").create('b'); + + Option operation = + OptionBuilder.withLongOpt("operation").withArgName("r|w|rw").hasArg() + .withDescription( + "action: seek-only, create-only, seek-after-create").create( + 'x'); + + Option rootDir = + OptionBuilder.withLongOpt("root-dir").withArgName("path").hasArg() + .withDescription( + "specify root directory where files will be created.") + .create('r'); + + Option file = + OptionBuilder.withLongOpt("file").withArgName("name").hasArg() + .withDescription("specify the file name to be created or read.") + .create('f'); + + Option seekCount = + OptionBuilder + .withLongOpt("seek") + .withArgName("count") + .hasArg() + .withDescription( + "specify how many seek operations we perform (requires -x r or -x rw.") + .create('n'); + + Option trialCount = + OptionBuilder + .withLongOpt("trials") + .withArgName("n") + .hasArg() + .withDescription( + "specify how many times to run the whole benchmark") + .create('t'); + + Option useRawFs = + OptionBuilder + .withLongOpt("rawfs") + .withDescription("use raw instead of checksummed file system") + .create(); + + Option help = + OptionBuilder.withLongOpt("help").hasArg(false).withDescription( + "show this screen").create("h"); + + return new Options().addOption(compress).addOption(fileSize).addOption( + fsInputBufferSz).addOption(fsOutputBufferSize).addOption(keyLen) + .addOption(blockSz).addOption(rootDir).addOption(valueLen) + .addOption(operation).addOption(seekCount).addOption(file) + .addOption(trialCount).addOption(useRawFs).addOption(help); + + } + + private void processOptions(CommandLine line, Options opts) + throws ParseException { + // --help -h and --version -V must be processed first. + if (line.hasOption('h')) { + HelpFormatter formatter = new HelpFormatter(); + System.out.println("TFile and SeqFile benchmark."); + System.out.println(); + formatter.printHelp(100, + "java ... TestTFileSeqFileComparison [options]", + "\nSupported options:", opts, ""); + return; + } + + if (line.hasOption('c')) { + compress = line.getOptionValue('c'); + } + + if (line.hasOption('d')) { + dictSize = Integer.parseInt(line.getOptionValue('d')); + } + + if (line.hasOption('s')) { + fileSize = Long.parseLong(line.getOptionValue('s')) * 1024 * 1024; + } + + if (line.hasOption('i')) { + fsInputBufferSize = Integer.parseInt(line.getOptionValue('i')); + } + + if (line.hasOption('o')) { + fsOutputBufferSize = Integer.parseInt(line.getOptionValue('o')); + } + + if (line.hasOption('n')) { + seekCount = Integer.parseInt(line.getOptionValue('n')); + } + + if (line.hasOption('t')) { + trialCount = Integer.parseInt(line.getOptionValue('t')); + } + + if (line.hasOption('k')) { + IntegerRange ir = IntegerRange.parse(line.getOptionValue('k')); + minKeyLen = ir.from(); + maxKeyLen = ir.to(); + } + + if (line.hasOption('v')) { + IntegerRange ir = IntegerRange.parse(line.getOptionValue('v')); + minValLength = ir.from(); + maxValLength = ir.to(); + } + + if (line.hasOption('b')) { + minBlockSize = Integer.parseInt(line.getOptionValue('b')) * 1024; + } + + if (line.hasOption('r')) { + rootDir = line.getOptionValue('r'); + } + + if (line.hasOption('f')) { + file = line.getOptionValue('f'); + } + + if (line.hasOption('S')) { + seed = Long.parseLong(line.getOptionValue('S')); + } + + if (line.hasOption('x')) { + String strOp = line.getOptionValue('x'); + if (strOp.equals("r")) { + op = OP_READ; + } + else if (strOp.equals("w")) { + op = OP_CREATE; + } + else if (strOp.equals("rw")) { + op = OP_CREATE | OP_READ; + } + else { + throw new ParseException("Unknown action specifier: " + strOp); + } + } + + useRawFs = line.hasOption("rawfs"); + + proceed = true; + } + + private void validateOptions() throws ParseException { + if (!compress.equals("none") && !compress.equals("lzo") + && !compress.equals("gz") && !compress.equals("snappy")) { + throw new ParseException("Unknown compression scheme: " + compress); + } + + if (minKeyLen >= maxKeyLen) { + throw new ParseException( + "Max key length must be greater than min key length."); + } + + if (minValLength >= maxValLength) { + throw new ParseException( + "Max value length must be greater than min value length."); + } + + if (minWordLen >= maxWordLen) { + throw new ParseException( + "Max word length must be greater than min word length."); + } + return; + } + + private void setStopProceed() { + proceed = false; + } + + public boolean doCreate() { + return (op & OP_CREATE) != 0; + } + + public boolean doRead() { + return (op & OP_READ) != 0; + } + } + + public static void main(String[] argv) throws IOException { + TestHFileSeek testCase = new TestHFileSeek(); + MyOptions options = new MyOptions(argv); + + if (options.proceed == false) { + return; + } + + testCase.options = options; + for (int i = 0; i < options.trialCount; i++) { + LOG.info("Beginning trial " + (i+1)); + testCase.setUp(); + testCase.testSeeks(); + testCase.tearDown(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java new file mode 100644 index 0000000..1ddafa6 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java @@ -0,0 +1,353 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.*; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm; +import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.RawComparator; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableUtils; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * Testing writing a version 2 {@link HFile}. This is a low-level test written + * during the development of {@link HFileWriterV2}. + */ +@Category(SmallTests.class) +@RunWith(Parameterized.class) +public class TestHFileWriterV2 { + + private final boolean useChecksums; + + @Parameterized.Parameters + public static Collection parameters() { + return HBaseTestingUtility.BOOLEAN_PARAMETERIZED; + } + + private static final Log LOG = LogFactory.getLog(TestHFileWriterV2.class); + + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + private Configuration conf; + private FileSystem fs; + + public TestHFileWriterV2(boolean useChecksums) { + this.useChecksums = useChecksums; + } + + @Before + public void setUp() throws IOException { + conf = TEST_UTIL.getConfiguration(); + conf.setBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, useChecksums); + fs = FileSystem.get(conf); + } + + @Test + public void testHFileFormatV2() throws IOException { + Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), + "testHFileFormatV2"); + final Compression.Algorithm compressAlgo = Compression.Algorithm.GZ; + final int entryCount = 10000; + writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, false); + } + + + @Test + public void testMidKeyInHFile() throws IOException{ + Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), + "testMidKeyInHFile"); + Compression.Algorithm compressAlgo = Compression.Algorithm.NONE; + int entryCount = 50000; + writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, true); + } + + private void writeDataAndReadFromHFile(Path hfilePath, + Algorithm compressAlgo, int entryCount, boolean findMidKey) throws IOException { + + HFileWriterV2 writer = (HFileWriterV2) + new HFileWriterV2.WriterFactoryV2(conf, new CacheConfig(conf)) + .withPath(fs, hfilePath) + .withBlockSize(4096) + .withCompression(compressAlgo) + .withComparator(KeyValue.KEY_COMPARATOR) + .create(); + + long totalKeyLength = 0; + long totalValueLength = 0; + + Random rand = new Random(9713312); // Just a fixed seed. + + List keys = new ArrayList(); + List values = new ArrayList(); + + for (int i = 0; i < entryCount; ++i) { + byte[] keyBytes = randomOrderedKey(rand, i); + + // A random-length random value. + byte[] valueBytes = randomValue(rand); + writer.append(keyBytes, valueBytes); + + totalKeyLength += keyBytes.length; + totalValueLength += valueBytes.length; + + keys.add(keyBytes); + values.add(valueBytes); + } + + // Add in an arbitrary order. They will be sorted lexicographically by + // the key. + writer.appendMetaBlock("CAPITAL_OF_USA", new Text("Washington, D.C.")); + writer.appendMetaBlock("CAPITAL_OF_RUSSIA", new Text("Moscow")); + writer.appendMetaBlock("CAPITAL_OF_FRANCE", new Text("Paris")); + + writer.close(); + + + FSDataInputStream fsdis = fs.open(hfilePath); + + // A "manual" version of a new-format HFile reader. This unit test was + // written before the V2 reader was fully implemented. + + long fileSize = fs.getFileStatus(hfilePath).getLen(); + FixedFileTrailer trailer = + FixedFileTrailer.readFromStream(fsdis, fileSize); + + assertEquals(2, trailer.getMajorVersion()); + assertEquals(useChecksums?1:0, trailer.getMinorVersion()); + assertEquals(entryCount, trailer.getEntryCount()); + + HFileBlock.FSReader blockReader = + new HFileBlock.FSReaderV2(fsdis,fsdis, compressAlgo, fileSize, + this.useChecksums?HFileReaderV2.MAX_MINOR_VERSION:HFileReaderV2.MIN_MINOR_VERSION, + null, null); + // Comparator class name is stored in the trailer in version 2. + RawComparator comparator = trailer.createComparator(); + HFileBlockIndex.BlockIndexReader dataBlockIndexReader = + new HFileBlockIndex.BlockIndexReader(comparator, + trailer.getNumDataIndexLevels()); + HFileBlockIndex.BlockIndexReader metaBlockIndexReader = + new HFileBlockIndex.BlockIndexReader( + Bytes.BYTES_RAWCOMPARATOR, 1); + + HFileBlock.BlockIterator blockIter = blockReader.blockRange( + trailer.getLoadOnOpenDataOffset(), + fileSize - trailer.getTrailerSize()); + // Data index. We also read statistics about the block index written after + // the root level. + dataBlockIndexReader.readMultiLevelIndexRoot( + blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), + trailer.getDataIndexCount()); + + if (findMidKey) { + byte[] midkey = dataBlockIndexReader.midkey(); + assertNotNull("Midkey should not be null", midkey); + } + + // Meta index. + metaBlockIndexReader.readRootIndex( + blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX).getByteStream(), + trailer.getMetaIndexCount()); + // File info + FileInfo fileInfo = new FileInfo(); + fileInfo.readFields(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream()); + byte [] keyValueFormatVersion = fileInfo.get( + HFileWriterV2.KEY_VALUE_VERSION); + boolean includeMemstoreTS = keyValueFormatVersion != null && + Bytes.toInt(keyValueFormatVersion) > 0; + + // Counters for the number of key/value pairs and the number of blocks + int entriesRead = 0; + int blocksRead = 0; + long memstoreTS = 0; + + // Scan blocks the way the reader would scan them + fsdis.seek(0); + long curBlockPos = 0; + while (curBlockPos <= trailer.getLastDataBlockOffset()) { + HFileBlock block = blockReader.readBlockData(curBlockPos, -1, -1, false); + assertEquals(BlockType.DATA, block.getBlockType()); + ByteBuffer buf = block.getBufferWithoutHeader(); + while (buf.hasRemaining()) { + int keyLen = buf.getInt(); + int valueLen = buf.getInt(); + + byte[] key = new byte[keyLen]; + buf.get(key); + + byte[] value = new byte[valueLen]; + buf.get(value); + + if (includeMemstoreTS) { + ByteArrayInputStream byte_input = new ByteArrayInputStream(buf.array(), + buf.arrayOffset() + buf.position(), buf.remaining()); + DataInputStream data_input = new DataInputStream(byte_input); + + memstoreTS = WritableUtils.readVLong(data_input); + buf.position(buf.position() + WritableUtils.getVIntSize(memstoreTS)); + } + + // A brute-force check to see that all keys and values are correct. + assertTrue(Bytes.compareTo(key, keys.get(entriesRead)) == 0); + assertTrue(Bytes.compareTo(value, values.get(entriesRead)) == 0); + + ++entriesRead; + } + ++blocksRead; + curBlockPos += block.getOnDiskSizeWithHeader(); + } + LOG.info("Finished reading: entries=" + entriesRead + ", blocksRead=" + + blocksRead); + assertEquals(entryCount, entriesRead); + + // Meta blocks. We can scan until the load-on-open data offset (which is + // the root block index offset in version 2) because we are not testing + // intermediate-level index blocks here. + + int metaCounter = 0; + while (fsdis.getPos() < trailer.getLoadOnOpenDataOffset()) { + LOG.info("Current offset: " + fsdis.getPos() + ", scanning until " + + trailer.getLoadOnOpenDataOffset()); + HFileBlock block = blockReader.readBlockData(curBlockPos, -1, -1, false); + assertEquals(BlockType.META, block.getBlockType()); + Text t = new Text(); + block.readInto(t); + Text expectedText = + (metaCounter == 0 ? new Text("Paris") : metaCounter == 1 ? new Text( + "Moscow") : new Text("Washington, D.C.")); + assertEquals(expectedText, t); + LOG.info("Read meta block data: " + t); + ++metaCounter; + curBlockPos += block.getOnDiskSizeWithHeader(); + } + + fsdis.close(); + } + + + // Static stuff used by various HFile v2 unit tests + + private static final String COLUMN_FAMILY_NAME = "_-myColumnFamily-_"; + private static final int MIN_ROW_OR_QUALIFIER_LENGTH = 64; + private static final int MAX_ROW_OR_QUALIFIER_LENGTH = 128; + + /** + * Generates a random key that is guaranteed to increase as the given index i + * increases. The result consists of a prefix, which is a deterministic + * increasing function of i, and a random suffix. + * + * @param rand + * random number generator to use + * @param i + * @return + */ + public static byte[] randomOrderedKey(Random rand, int i) { + StringBuilder k = new StringBuilder(); + + // The fixed-length lexicographically increasing part of the key. + for (int bitIndex = 31; bitIndex >= 0; --bitIndex) { + if ((i & (1 << bitIndex)) == 0) + k.append("a"); + else + k.append("b"); + } + + // A random-length random suffix of the key. + for (int j = 0; j < rand.nextInt(50); ++j) + k.append(randomReadableChar(rand)); + + byte[] keyBytes = k.toString().getBytes(); + return keyBytes; + } + + public static byte[] randomValue(Random rand) { + StringBuilder v = new StringBuilder(); + for (int j = 0; j < 1 + rand.nextInt(2000); ++j) { + v.append((char) (32 + rand.nextInt(95))); + } + + byte[] valueBytes = v.toString().getBytes(); + return valueBytes; + } + + public static final char randomReadableChar(Random rand) { + int i = rand.nextInt(26 * 2 + 10 + 1); + if (i < 26) + return (char) ('A' + i); + i -= 26; + + if (i < 26) + return (char) ('a' + i); + i -= 26; + + if (i < 10) + return (char) ('0' + i); + i -= 10; + + assert i == 0; + return '_'; + } + + public static byte[] randomRowOrQualifier(Random rand) { + StringBuilder field = new StringBuilder(); + int fieldLen = MIN_ROW_OR_QUALIFIER_LENGTH + + rand.nextInt(MAX_ROW_OR_QUALIFIER_LENGTH + - MIN_ROW_OR_QUALIFIER_LENGTH + 1); + for (int i = 0; i < fieldLen; ++i) + field.append(randomReadableChar(rand)); + return field.toString().getBytes(); + } + + public static KeyValue randomKeyValue(Random rand) { + return new KeyValue(randomRowOrQualifier(rand), + COLUMN_FAMILY_NAME.getBytes(), randomRowOrQualifier(rand), + randomValue(rand)); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java new file mode 100644 index 0000000..65af0d7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java @@ -0,0 +1,699 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Map; +import java.util.Random; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.io.hfile.LruBlockCache.EvictionThread; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.regionserver.metrics.TestSchemaMetrics; +import org.apache.hadoop.hbase.util.ClassSize; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Tests the concurrent LruBlockCache.

    + * + * Tests will ensure it grows and shrinks in size properly, + * evictions run when they're supposed to and do what they should, + * and that cached blocks are accessible when expected to be. + */ +@RunWith(Parameterized.class) +@Category(MediumTests.class) +public class TestLruBlockCache { + + private Map startingMetrics; + private final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + public TestLruBlockCache(boolean useTableName) { + SchemaMetrics.setUseTableNameInTest(useTableName); + } + + @Parameters + public static Collection parameters() { + return TestSchemaMetrics.parameters(); + } + + @Before + public void setUp() throws Exception { + startingMetrics = SchemaMetrics.getMetricsSnapshot(); + } + + @After + public void tearDown() throws Exception { + SchemaMetrics.validateMetricChanges(startingMetrics); + } + + @Test + public void testBackgroundEvictionThread() throws Exception { + long maxSize = 100000; + long blockSize = calculateBlockSizeDefault(maxSize, 9); // room for 9, will evict + + LruBlockCache cache = new LruBlockCache(maxSize, blockSize, TEST_UTIL.getConfiguration()); + + CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block"); + + EvictionThread evictionThread = cache.getEvictionThread(); + assertTrue(evictionThread != null); + + // Make sure eviction thread has entered run method + while (!evictionThread.isEnteringRun()) { + Thread.sleep(1); + } + + // Add all the blocks + for (CachedItem block : blocks) { + cache.cacheBlock(block.cacheKey, block); + } + + // Let the eviction run + int n = 0; + while(cache.getEvictionCount() == 0) { + Thread.sleep(200); + assertTrue(n++ < 20); + } + System.out.println("Background Evictions run: " + cache.getEvictionCount()); + + // A single eviction run should have occurred + assertEquals(cache.getEvictionCount(), 1); + } + + @Test + public void testCacheSimple() throws Exception { + + long maxSize = 1000000; + long blockSize = calculateBlockSizeDefault(maxSize, 101); + + LruBlockCache cache = new LruBlockCache(maxSize, blockSize, TEST_UTIL.getConfiguration()); + + CachedItem [] blocks = generateRandomBlocks(100, blockSize); + + long expectedCacheSize = cache.heapSize(); + + // Confirm empty + for (CachedItem block : blocks) { + assertTrue(cache.getBlock(block.cacheKey, true, false) == null); + } + + // Add blocks + for (CachedItem block : blocks) { + cache.cacheBlock(block.cacheKey, block); + expectedCacheSize += block.cacheBlockHeapSize(); + } + + // Verify correctly calculated cache heap size + assertEquals(expectedCacheSize, cache.heapSize()); + + // Check if all blocks are properly cached and retrieved + for (CachedItem block : blocks) { + HeapSize buf = cache.getBlock(block.cacheKey, true, false); + assertTrue(buf != null); + assertEquals(buf.heapSize(), block.heapSize()); + } + + // Verify correctly calculated cache heap size + assertEquals(expectedCacheSize, cache.heapSize()); + + // Check if all blocks are properly cached and retrieved + for (CachedItem block : blocks) { + HeapSize buf = cache.getBlock(block.cacheKey, true, false); + assertTrue(buf != null); + assertEquals(buf.heapSize(), block.heapSize()); + } + + // Expect no evictions + assertEquals(0, cache.getEvictionCount()); + Thread t = new LruBlockCache.StatisticsThread(cache); + t.start(); + t.join(); + } + + @Test + public void testCacheEvictionSimple() throws Exception { + + long maxSize = 100000; + long blockSize = calculateBlockSizeDefault(maxSize, 10); + + LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false, TEST_UTIL.getConfiguration()); + + CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block"); + + long expectedCacheSize = cache.heapSize(); + + // Add all the blocks + for (CachedItem block : blocks) { + cache.cacheBlock(block.cacheKey, block); + expectedCacheSize += block.cacheBlockHeapSize(); + } + + // A single eviction run should have occurred + assertEquals(1, cache.getEvictionCount()); + + // Our expected size overruns acceptable limit + assertTrue(expectedCacheSize > + (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR)); + + // But the cache did not grow beyond max + assertTrue(cache.heapSize() < maxSize); + + // And is still below the acceptable limit + assertTrue(cache.heapSize() < + (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR)); + + // All blocks except block 0 and 1 should be in the cache + assertTrue(cache.getBlock(blocks[0].cacheKey, true, false) == null); + assertTrue(cache.getBlock(blocks[1].cacheKey, true, false) == null); + for(int i=2;i + (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR)); + + // But the cache did not grow beyond max + assertTrue(cache.heapSize() <= maxSize); + + // And is now below the acceptable limit + assertTrue(cache.heapSize() <= + (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR)); + + // We expect fairness across the two priorities. + // This test makes multi go barely over its limit, in-memory + // empty, and the rest in single. Two single evictions and + // one multi eviction expected. + assertTrue(cache.getBlock(singleBlocks[0].cacheKey, true, false) == null); + assertTrue(cache.getBlock(multiBlocks[0].cacheKey, true, false) == null); + + // And all others to be cached + for(int i=1;i<4;i++) { + assertEquals(cache.getBlock(singleBlocks[i].cacheKey, true, false), + singleBlocks[i]); + assertEquals(cache.getBlock(multiBlocks[i].cacheKey, true, false), + multiBlocks[i]); + } + } + + @Test + public void testCacheEvictionThreePriorities() throws Exception { + + long maxSize = 100000; + long blockSize = calculateBlockSize(maxSize, 10); + + LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false, + (int)Math.ceil(1.2*maxSize/blockSize), + LruBlockCache.DEFAULT_LOAD_FACTOR, + LruBlockCache.DEFAULT_CONCURRENCY_LEVEL, + 0.98f, // min + 0.99f, // acceptable + 0.33f, // single + 0.33f, // multi + 0.34f);// memory + + + CachedItem [] singleBlocks = generateFixedBlocks(5, blockSize, "single"); + CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi"); + CachedItem [] memoryBlocks = generateFixedBlocks(5, blockSize, "memory"); + + long expectedCacheSize = cache.heapSize(); + + // Add 3 blocks from each priority + for(int i=0;i<3;i++) { + + // Just add single blocks + cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]); + expectedCacheSize += singleBlocks[i].cacheBlockHeapSize(); + + // Add and get multi blocks + cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]); + expectedCacheSize += multiBlocks[i].cacheBlockHeapSize(); + cache.getBlock(multiBlocks[i].cacheKey, true, false); + + // Add memory blocks as such + cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true); + expectedCacheSize += memoryBlocks[i].cacheBlockHeapSize(); + + } + + // Do not expect any evictions yet + assertEquals(0, cache.getEvictionCount()); + + // Verify cache size + assertEquals(expectedCacheSize, cache.heapSize()); + + // Insert a single block, oldest single should be evicted + cache.cacheBlock(singleBlocks[3].cacheKey, singleBlocks[3]); + + // Single eviction, one thing evicted + assertEquals(1, cache.getEvictionCount()); + assertEquals(1, cache.getEvictedCount()); + + // Verify oldest single block is the one evicted + assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false)); + + // Change the oldest remaining single block to a multi + cache.getBlock(singleBlocks[1].cacheKey, true, false); + + // Insert another single block + cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]); + + // Two evictions, two evicted. + assertEquals(2, cache.getEvictionCount()); + assertEquals(2, cache.getEvictedCount()); + + // Oldest multi block should be evicted now + assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false)); + + // Insert another memory block + cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true); + + // Three evictions, three evicted. + assertEquals(3, cache.getEvictionCount()); + assertEquals(3, cache.getEvictedCount()); + + // Oldest memory block should be evicted now + assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true, false)); + + // Add a block that is twice as big (should force two evictions) + CachedItem [] bigBlocks = generateFixedBlocks(3, blockSize*3, "big"); + cache.cacheBlock(bigBlocks[0].cacheKey, bigBlocks[0]); + + // Four evictions, six evicted (inserted block 3X size, expect +3 evicted) + assertEquals(4, cache.getEvictionCount()); + assertEquals(6, cache.getEvictedCount()); + + // Expect three remaining singles to be evicted + assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true, false)); + assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true, false)); + assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true, false)); + + // Make the big block a multi block + cache.getBlock(bigBlocks[0].cacheKey, true, false); + + // Cache another single big block + cache.cacheBlock(bigBlocks[1].cacheKey, bigBlocks[1]); + + // Five evictions, nine evicted (3 new) + assertEquals(5, cache.getEvictionCount()); + assertEquals(9, cache.getEvictedCount()); + + // Expect three remaining multis to be evicted + assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false)); + assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false)); + assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true, false)); + + // Cache a big memory block + cache.cacheBlock(bigBlocks[2].cacheKey, bigBlocks[2], true); + + // Six evictions, twelve evicted (3 new) + assertEquals(6, cache.getEvictionCount()); + assertEquals(12, cache.getEvictedCount()); + + // Expect three remaining in-memory to be evicted + assertEquals(null, cache.getBlock(memoryBlocks[1].cacheKey, true, false)); + assertEquals(null, cache.getBlock(memoryBlocks[2].cacheKey, true, false)); + assertEquals(null, cache.getBlock(memoryBlocks[3].cacheKey, true, false)); + + + } + + // test scan resistance + @Test + public void testScanResistance() throws Exception { + + long maxSize = 100000; + long blockSize = calculateBlockSize(maxSize, 10); + + LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false, + (int)Math.ceil(1.2*maxSize/blockSize), + LruBlockCache.DEFAULT_LOAD_FACTOR, + LruBlockCache.DEFAULT_CONCURRENCY_LEVEL, + 0.66f, // min + 0.99f, // acceptable + 0.33f, // single + 0.33f, // multi + 0.34f);// memory + + CachedItem [] singleBlocks = generateFixedBlocks(20, blockSize, "single"); + CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi"); + + // Add 5 multi blocks + for (CachedItem block : multiBlocks) { + cache.cacheBlock(block.cacheKey, block); + cache.getBlock(block.cacheKey, true, false); + } + + // Add 5 single blocks + for(int i=0;i<5;i++) { + cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]); + } + + // An eviction ran + assertEquals(1, cache.getEvictionCount()); + + // To drop down to 2/3 capacity, we'll need to evict 4 blocks + assertEquals(4, cache.getEvictedCount()); + + // Should have been taken off equally from single and multi + assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false)); + assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false)); + assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false)); + assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false)); + + // Let's keep "scanning" by adding single blocks. From here on we only + // expect evictions from the single bucket. + + // Every time we reach 10 total blocks (every 4 inserts) we get 4 single + // blocks evicted. Inserting 13 blocks should yield 3 more evictions and + // 12 more evicted. + + for(int i=5;i<18;i++) { + cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]); + } + + // 4 total evictions, 16 total evicted + assertEquals(4, cache.getEvictionCount()); + assertEquals(16, cache.getEvictedCount()); + + // Should now have 7 total blocks + assertEquals(7, cache.size()); + + } + + // test setMaxSize + @Test + public void testResizeBlockCache() throws Exception { + + long maxSize = 300000; + long blockSize = calculateBlockSize(maxSize, 31); + + LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false, + (int)Math.ceil(1.2*maxSize/blockSize), + LruBlockCache.DEFAULT_LOAD_FACTOR, + LruBlockCache.DEFAULT_CONCURRENCY_LEVEL, + 0.98f, // min + 0.99f, // acceptable + 0.33f, // single + 0.33f, // multi + 0.34f);// memory + + CachedItem [] singleBlocks = generateFixedBlocks(10, blockSize, "single"); + CachedItem [] multiBlocks = generateFixedBlocks(10, blockSize, "multi"); + CachedItem [] memoryBlocks = generateFixedBlocks(10, blockSize, "memory"); + + // Add all blocks from all priorities + for(int i=0;i<10;i++) { + + // Just add single blocks + cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]); + + // Add and get multi blocks + cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]); + cache.getBlock(multiBlocks[i].cacheKey, true, false); + + // Add memory blocks as such + cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true); + } + + // Do not expect any evictions yet + assertEquals(0, cache.getEvictionCount()); + + // Resize to half capacity plus an extra block (otherwise we evict an extra) + cache.setMaxSize((long)(maxSize * 0.5f)); + + // Should have run a single eviction + assertEquals(1, cache.getEvictionCount()); + + // And we expect 1/2 of the blocks to be evicted + assertEquals(15, cache.getEvictedCount()); + + // And the oldest 5 blocks from each category should be gone + for(int i=0;i<5;i++) { + assertEquals(null, cache.getBlock(singleBlocks[i].cacheKey, true, false)); + assertEquals(null, cache.getBlock(multiBlocks[i].cacheKey, true, false)); + assertEquals(null, cache.getBlock(memoryBlocks[i].cacheKey, true, false)); + } + + // And the newest 5 blocks should still be accessible + for(int i=5;i<10;i++) { + assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].cacheKey, true, false)); + assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].cacheKey, true, false)); + assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].cacheKey, true, false)); + } + } + + // test metricsPastNPeriods + @Test + public void testPastNPeriodsMetrics() throws Exception { + double delta = 0.01; + + // 3 total periods + CacheStats stats = new CacheStats(3); + + // No accesses, should be 0 + stats.rollMetricsPeriod(); + assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta); + assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta); + + // period 1, 1 hit caching, 1 hit non-caching, 2 miss non-caching + // should be (2/4)=0.5 and (1/1)=1 + stats.hit(false); + stats.hit(true); + stats.miss(false); + stats.miss(false); + stats.rollMetricsPeriod(); + assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta); + assertEquals(1.0, stats.getHitCachingRatioPastNPeriods(), delta); + + // period 2, 1 miss caching, 3 miss non-caching + // should be (2/8)=0.25 and (1/2)=0.5 + stats.miss(true); + stats.miss(false); + stats.miss(false); + stats.miss(false); + stats.rollMetricsPeriod(); + assertEquals(0.25, stats.getHitRatioPastNPeriods(), delta); + assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta); + + // period 3, 2 hits of each type + // should be (6/12)=0.5 and (3/4)=0.75 + stats.hit(false); + stats.hit(true); + stats.hit(false); + stats.hit(true); + stats.rollMetricsPeriod(); + assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta); + assertEquals(0.75, stats.getHitCachingRatioPastNPeriods(), delta); + + // period 4, evict period 1, two caching misses + // should be (4/10)=0.4 and (2/5)=0.4 + stats.miss(true); + stats.miss(true); + stats.rollMetricsPeriod(); + assertEquals(0.4, stats.getHitRatioPastNPeriods(), delta); + assertEquals(0.4, stats.getHitCachingRatioPastNPeriods(), delta); + + // period 5, evict period 2, 2 caching misses, 2 non-caching hit + // should be (6/10)=0.6 and (2/6)=1/3 + stats.miss(true); + stats.miss(true); + stats.hit(false); + stats.hit(false); + stats.rollMetricsPeriod(); + assertEquals(0.6, stats.getHitRatioPastNPeriods(), delta); + assertEquals((double)1/3, stats.getHitCachingRatioPastNPeriods(), delta); + + // period 6, evict period 3 + // should be (2/6)=1/3 and (0/4)=0 + stats.rollMetricsPeriod(); + assertEquals((double)1/3, stats.getHitRatioPastNPeriods(), delta); + assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta); + + // period 7, evict period 4 + // should be (2/4)=0.5 and (0/2)=0 + stats.rollMetricsPeriod(); + assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta); + assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta); + + // period 8, evict period 5 + // should be 0 and 0 + stats.rollMetricsPeriod(); + assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta); + assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta); + + // period 9, one of each + // should be (2/4)=0.5 and (1/2)=0.5 + stats.miss(true); + stats.miss(false); + stats.hit(true); + stats.hit(false); + stats.rollMetricsPeriod(); + assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta); + assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta); + } + + private CachedItem [] generateFixedBlocks(int numBlocks, int size, String pfx) { + CachedItem [] blocks = new CachedItem[numBlocks]; + for(int i=0;i getDeserializer() { + return null; + } + + @Override + public void serialize(ByteBuffer destination) { + } + + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java new file mode 100644 index 0000000..6c859fd --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java @@ -0,0 +1,103 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.util.ArrayList; +import java.util.List; + +import junit.framework.Assert; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +/** + * Test {@link HFileScanner#reseekTo(byte[])} + */ +@Category(SmallTests.class) +public class TestReseekTo { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @Test + public void testReseekTo() throws Exception { + + Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile"); + FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile); + CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration()); + HFile.Writer writer = HFile.getWriterFactory( + TEST_UTIL.getConfiguration(), cacheConf) + .withOutputStream(fout) + .withBlockSize(4000) + .create(); + int numberOfKeys = 1000; + + String valueString = "Value"; + + List keyList = new ArrayList(); + List valueList = new ArrayList(); + + for (int key = 0; key < numberOfKeys; key++) { + String value = valueString + key; + keyList.add(key); + valueList.add(value); + writer.append(Bytes.toBytes(key), Bytes.toBytes(value)); + } + writer.close(); + fout.close(); + + HFile.Reader reader = HFile.createReader(TEST_UTIL.getTestFileSystem(), + ncTFile, cacheConf); + reader.loadFileInfo(); + HFileScanner scanner = reader.getScanner(false, true); + + scanner.seekTo(); + for (int i = 0; i < keyList.size(); i++) { + Integer key = keyList.get(i); + String value = valueList.get(i); + long start = System.nanoTime(); + scanner.seekTo(Bytes.toBytes(key)); + assertEquals(value, scanner.getValueString()); + } + + scanner.seekTo(); + for (int i = 0; i < keyList.size(); i += 10) { + Integer key = keyList.get(i); + String value = valueList.get(i); + long start = System.nanoTime(); + scanner.reseekTo(Bytes.toBytes(key)); + assertEquals("i is " + i, value, scanner.getValueString()); + } + + reader.close(); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java new file mode 100644 index 0000000..2699832 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingKeyRange.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Test the optimization that does not scan files where all key ranges are excluded. + */ +@RunWith(Parameterized.class) +@Category(SmallTests.class) +public class TestScannerSelectionUsingKeyRange { + private static final Log LOG = LogFactory.getLog(TestScannerSelectionUsingKeyRange.class); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static String TABLE = "myTable"; + private static String FAMILY = "myCF"; + private static byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY); + private static final int NUM_ROWS = 8; + private static final int NUM_COLS_PER_ROW = 5; + private static final int NUM_FILES = 2; + private static final Map TYPE_COUNT = new HashMap(3); + static { + TYPE_COUNT.put(BloomType.ROWCOL, 2); + TYPE_COUNT.put(BloomType.ROW, 2); + TYPE_COUNT.put(BloomType.NONE, 2); + } + + private BloomType bloomType; + private int expectedCount; + + @Parameters + public static Collection parameters() { + List params = new ArrayList(); + for (Object type : TYPE_COUNT.keySet()) { + params.add(new Object[] { type, TYPE_COUNT.get(type) }); + } + return params; + } + + public TestScannerSelectionUsingKeyRange(Object expectedType, Object expectedCount) { + bloomType = (BloomType)expectedType; + expectedCount = expectedCount; + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.cleanupTestDir(); + } + + @Test + public void testScannerSelection() throws IOException { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setInt("hbase.hstore.compactionThreshold", 10000); + HColumnDescriptor hcd = new HColumnDescriptor(FAMILY_BYTES).setBlockCacheEnabled(true) + .setBloomFilterType(bloomType); + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(hcd); + HRegionInfo info = new HRegionInfo(Bytes.toBytes(TABLE)); + HRegion region = HRegion.createHRegion(info, TEST_UTIL.getClusterTestDir(), conf, htd); + + for (int iFile = 0; iFile < NUM_FILES; ++iFile) { + for (int iRow = 0; iRow < NUM_ROWS; ++iRow) { + Put put = new Put(Bytes.toBytes("row" + iRow)); + for (int iCol = 0; iCol < NUM_COLS_PER_ROW; ++iCol) { + put.add(FAMILY_BYTES, Bytes.toBytes("col" + iCol), + Bytes.toBytes("value" + iFile + "_" + iRow + "_" + iCol)); + } + region.put(put); + } + region.flushcache(); + } + + Scan scan = new Scan(Bytes.toBytes("aaa"), Bytes.toBytes("aaz")); + CacheConfig cacheConf = new CacheConfig(conf); + LruBlockCache cache = (LruBlockCache) cacheConf.getBlockCache(); + cache.clearCache(); + Map metricsBefore = SchemaMetrics.getMetricsSnapshot(); + SchemaMetrics.validateMetricChanges(metricsBefore); + InternalScanner scanner = region.getScanner(scan); + List results = new ArrayList(); + while (scanner.next(results)) { + } + scanner.close(); + assertEquals(0, results.size()); + Set accessedFiles = cache.getCachedFileNamesForTest(); + assertEquals(accessedFiles.size(), 0); + //assertEquals(cache.getBlockCount(), 0); + Map diffMetrics = SchemaMetrics.diffMetrics(metricsBefore, + SchemaMetrics.getMetricsSnapshot()); + SchemaMetrics schemaMetrics = SchemaMetrics.getInstance(TABLE, FAMILY); + long dataBlockRead = SchemaMetrics.getLong(diffMetrics, + schemaMetrics.getBlockMetricName(BlockCategory.DATA, false, BlockMetricType.READ_COUNT)); + assertEquals(dataBlockRead, 0); + region.close(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java new file mode 100644 index 0000000..44aa3e4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Test the optimization that does not scan files where all timestamps are + * expired. + */ +@RunWith(Parameterized.class) +@Category(SmallTests.class) +public class TestScannerSelectionUsingTTL { + + private static final Log LOG = + LogFactory.getLog(TestScannerSelectionUsingTTL.class); + + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + private static String TABLE = "myTable"; + private static String FAMILY = "myCF"; + private static byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY); + + private static final int TTL_SECONDS = 2; + private static final int TTL_MS = TTL_SECONDS * 1000; + + private static final int NUM_EXPIRED_FILES = 2; + private static final int NUM_ROWS = 8; + private static final int NUM_COLS_PER_ROW = 5; + + public final int numFreshFiles, totalNumFiles; + + /** Whether we are specifying the exact files to compact */ + private final boolean explicitCompaction; + + @Parameters + public static Collection parameters() { + List params = new ArrayList(); + for (int numFreshFiles = 1; numFreshFiles <= 3; ++numFreshFiles) { + for (boolean explicitCompaction : new boolean[] { false, true }) { + params.add(new Object[] { numFreshFiles, explicitCompaction }); + } + } + return params; + } + + public TestScannerSelectionUsingTTL(int numFreshFiles, + boolean explicitCompaction) { + this.numFreshFiles = numFreshFiles; + this.totalNumFiles = numFreshFiles + NUM_EXPIRED_FILES; + this.explicitCompaction = explicitCompaction; + } + + @Test + public void testScannerSelection() throws IOException { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setBoolean("hbase.store.delete.expired.storefile", false); + HColumnDescriptor hcd = + new HColumnDescriptor(FAMILY_BYTES) + .setMaxVersions(Integer.MAX_VALUE) + .setTimeToLive(TTL_SECONDS); + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(hcd); + HRegionInfo info = new HRegionInfo(Bytes.toBytes(TABLE)); + HRegion region = + HRegion.createHRegion(info, TEST_UTIL.getClusterTestDir(), + conf, htd); + + for (int iFile = 0; iFile < totalNumFiles; ++iFile) { + if (iFile == NUM_EXPIRED_FILES) { + Threads.sleepWithoutInterrupt(TTL_MS); + } + + for (int iRow = 0; iRow < NUM_ROWS; ++iRow) { + Put put = new Put(Bytes.toBytes("row" + iRow)); + for (int iCol = 0; iCol < NUM_COLS_PER_ROW; ++iCol) { + put.add(FAMILY_BYTES, Bytes.toBytes("col" + iCol), + Bytes.toBytes("value" + iFile + "_" + iRow + "_" + iCol)); + } + region.put(put); + } + region.flushcache(); + } + + Scan scan = new Scan(); + scan.setMaxVersions(Integer.MAX_VALUE); + CacheConfig cacheConf = new CacheConfig(conf); + LruBlockCache cache = (LruBlockCache) cacheConf.getBlockCache(); + cache.clearCache(); + InternalScanner scanner = region.getScanner(scan); + List results = new ArrayList(); + final int expectedKVsPerRow = numFreshFiles * NUM_COLS_PER_ROW; + int numReturnedRows = 0; + LOG.info("Scanning the entire table"); + while (scanner.next(results) || results.size() > 0) { + assertEquals(expectedKVsPerRow, results.size()); + ++numReturnedRows; + results.clear(); + } + assertEquals(NUM_ROWS, numReturnedRows); + Set accessedFiles = cache.getCachedFileNamesForTest(); + LOG.debug("Files accessed during scan: " + accessedFiles); + + Map metricsBeforeCompaction = + SchemaMetrics.getMetricsSnapshot(); + + // Exercise both compaction codepaths. + if (explicitCompaction) { + region.getStore(FAMILY_BYTES).compactRecentForTesting(totalNumFiles); + } else { + region.compactStores(); + } + + SchemaMetrics.validateMetricChanges(metricsBeforeCompaction); + Map compactionMetrics = + SchemaMetrics.diffMetrics(metricsBeforeCompaction, + SchemaMetrics.getMetricsSnapshot()); + long compactionDataBlocksRead = SchemaMetrics.getLong( + compactionMetrics, + SchemaMetrics.getInstance(TABLE, FAMILY).getBlockMetricName( + BlockCategory.DATA, true, BlockMetricType.READ_COUNT)); + assertEquals("Invalid number of blocks accessed during compaction. " + + "We only expect non-expired files to be accessed.", + numFreshFiles, compactionDataBlocksRead); + region.close(); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java new file mode 100644 index 0000000..1a2ce15 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java @@ -0,0 +1,243 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.io.IOException; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +/** + * Test {@link HFileScanner#seekTo(byte[])} and its variants. + */ +@Category(SmallTests.class) +public class TestSeekTo extends HBaseTestCase { + + static KeyValue toKV(String row) { + return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes + .toBytes("qualifier"), Bytes.toBytes("value")); + } + + static String toRowStr(KeyValue kv) { + return Bytes.toString(kv.getRow()); + } + + Path makeNewFile() throws IOException { + Path ncTFile = new Path(this.testDir, "basic.hfile"); + FSDataOutputStream fout = this.fs.create(ncTFile); + int blocksize = toKV("a").getLength() * 3; + HFile.Writer writer = HFile.getWriterFactoryNoCache(conf) + .withOutputStream(fout) + .withBlockSize(blocksize) + .create(); + // 4 bytes * 3 * 2 for each key/value + + // 3 for keys, 15 for values = 42 (woot) + writer.append(toKV("c")); + writer.append(toKV("e")); + writer.append(toKV("g")); + // block transition + writer.append(toKV("i")); + writer.append(toKV("k")); + writer.close(); + fout.close(); + return ncTFile; + } + + public void testSeekBefore() throws Exception { + Path p = makeNewFile(); + HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf)); + reader.loadFileInfo(); + HFileScanner scanner = reader.getScanner(false, true); + assertEquals(false, scanner.seekBefore(toKV("a").getKey())); + + assertEquals(false, scanner.seekBefore(toKV("c").getKey())); + + assertEquals(true, scanner.seekBefore(toKV("d").getKey())); + assertEquals("c", toRowStr(scanner.getKeyValue())); + + assertEquals(true, scanner.seekBefore(toKV("e").getKey())); + assertEquals("c", toRowStr(scanner.getKeyValue())); + + assertEquals(true, scanner.seekBefore(toKV("f").getKey())); + assertEquals("e", toRowStr(scanner.getKeyValue())); + + assertEquals(true, scanner.seekBefore(toKV("g").getKey())); + assertEquals("e", toRowStr(scanner.getKeyValue())); + + assertEquals(true, scanner.seekBefore(toKV("h").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + assertEquals(true, scanner.seekBefore(toKV("i").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + assertEquals(true, scanner.seekBefore(toKV("j").getKey())); + assertEquals("i", toRowStr(scanner.getKeyValue())); + assertEquals(true, scanner.seekBefore(toKV("k").getKey())); + assertEquals("i", toRowStr(scanner.getKeyValue())); + assertEquals(true, scanner.seekBefore(toKV("l").getKey())); + assertEquals("k", toRowStr(scanner.getKeyValue())); + + reader.close(); + } + + public void testSeekBeforeWithReSeekTo() throws Exception { + Path p = makeNewFile(); + HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf)); + reader.loadFileInfo(); + HFileScanner scanner = reader.getScanner(false, true); + assertEquals(false, scanner.seekBefore(toKV("a").getKey())); + assertEquals(false, scanner.seekBefore(toKV("b").getKey())); + assertEquals(false, scanner.seekBefore(toKV("c").getKey())); + + // seekBefore d, so the scanner points to c + assertEquals(true, scanner.seekBefore(toKV("d").getKey())); + assertEquals("c", toRowStr(scanner.getKeyValue())); + // reseekTo e and g + assertEquals(0, scanner.reseekTo(toKV("c").getKey())); + assertEquals("c", toRowStr(scanner.getKeyValue())); + assertEquals(0, scanner.reseekTo(toKV("g").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + + // seekBefore e, so the scanner points to c + assertEquals(true, scanner.seekBefore(toKV("e").getKey())); + assertEquals("c", toRowStr(scanner.getKeyValue())); + // reseekTo e and g + assertEquals(0, scanner.reseekTo(toKV("e").getKey())); + assertEquals("e", toRowStr(scanner.getKeyValue())); + assertEquals(0, scanner.reseekTo(toKV("g").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + + // seekBefore f, so the scanner points to e + assertEquals(true, scanner.seekBefore(toKV("f").getKey())); + assertEquals("e", toRowStr(scanner.getKeyValue())); + // reseekTo e and g + assertEquals(0, scanner.reseekTo(toKV("e").getKey())); + assertEquals("e", toRowStr(scanner.getKeyValue())); + assertEquals(0, scanner.reseekTo(toKV("g").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + + // seekBefore g, so the scanner points to e + assertEquals(true, scanner.seekBefore(toKV("g").getKey())); + assertEquals("e", toRowStr(scanner.getKeyValue())); + // reseekTo e and g again + assertEquals(0, scanner.reseekTo(toKV("e").getKey())); + assertEquals("e", toRowStr(scanner.getKeyValue())); + assertEquals(0, scanner.reseekTo(toKV("g").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + + // seekBefore h, so the scanner points to g + assertEquals(true, scanner.seekBefore(toKV("h").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + // reseekTo g + assertEquals(0, scanner.reseekTo(toKV("g").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + + // seekBefore i, so the scanner points to g + assertEquals(true, scanner.seekBefore(toKV("i").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + // reseekTo g + assertEquals(0, scanner.reseekTo(toKV("g").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + + // seekBefore j, so the scanner points to i + assertEquals(true, scanner.seekBefore(toKV("j").getKey())); + assertEquals("i", toRowStr(scanner.getKeyValue())); + // reseekTo i + assertEquals(0, scanner.reseekTo(toKV("i").getKey())); + assertEquals("i", toRowStr(scanner.getKeyValue())); + + // seekBefore k, so the scanner points to i + assertEquals(true, scanner.seekBefore(toKV("k").getKey())); + assertEquals("i", toRowStr(scanner.getKeyValue())); + // reseekTo i and k + assertEquals(0, scanner.reseekTo(toKV("i").getKey())); + assertEquals("i", toRowStr(scanner.getKeyValue())); + assertEquals(0, scanner.reseekTo(toKV("k").getKey())); + assertEquals("k", toRowStr(scanner.getKeyValue())); + + // seekBefore l, so the scanner points to k + assertEquals(true, scanner.seekBefore(toKV("l").getKey())); + assertEquals("k", toRowStr(scanner.getKeyValue())); + // reseekTo k + assertEquals(0, scanner.reseekTo(toKV("k").getKey())); + assertEquals("k", toRowStr(scanner.getKeyValue())); + } + + public void testSeekTo() throws Exception { + Path p = makeNewFile(); + HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf)); + reader.loadFileInfo(); + assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount()); + HFileScanner scanner = reader.getScanner(false, true); + // lies before the start of the file. + assertEquals(-1, scanner.seekTo(toKV("a").getKey())); + + assertEquals(1, scanner.seekTo(toKV("d").getKey())); + assertEquals("c", toRowStr(scanner.getKeyValue())); + + // Across a block boundary now. + assertEquals(1, scanner.seekTo(toKV("h").getKey())); + assertEquals("g", toRowStr(scanner.getKeyValue())); + + assertEquals(1, scanner.seekTo(toKV("l").getKey())); + assertEquals("k", toRowStr(scanner.getKeyValue())); + + reader.close(); + } + + public void testBlockContainingKey() throws Exception { + Path p = makeNewFile(); + HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf)); + reader.loadFileInfo(); + HFileBlockIndex.BlockIndexReader blockIndexReader = + reader.getDataBlockIndexReader(); + System.out.println(blockIndexReader.toString()); + int klen = toKV("a").getKey().length; + // falls before the start of the file. + assertEquals(-1, blockIndexReader.rootBlockContainingKey( + toKV("a").getKey(), 0, klen)); + assertEquals(0, blockIndexReader.rootBlockContainingKey( + toKV("c").getKey(), 0, klen)); + assertEquals(0, blockIndexReader.rootBlockContainingKey( + toKV("d").getKey(), 0, klen)); + assertEquals(0, blockIndexReader.rootBlockContainingKey( + toKV("e").getKey(), 0, klen)); + assertEquals(0, blockIndexReader.rootBlockContainingKey( + toKV("g").getKey(), 0, klen)); + assertEquals(0, blockIndexReader.rootBlockContainingKey( + toKV("h").getKey(), 0, klen)); + assertEquals(1, blockIndexReader.rootBlockContainingKey( + toKV("i").getKey(), 0, klen)); + assertEquals(1, blockIndexReader.rootBlockContainingKey( + toKV("j").getKey(), 0, klen)); + assertEquals(1, blockIndexReader.rootBlockContainingKey( + toKV("k").getKey(), 0, klen)); + assertEquals(1, blockIndexReader.rootBlockContainingKey( + toKV("l").getKey(), 0, klen)); + + reader.close(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java new file mode 100644 index 0000000..4f21fbd --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java @@ -0,0 +1,87 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile.slab; + +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.io.hfile.CacheTestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests SingleSlabCache. + *

    + * + * Tests will ensure that evictions operate when they're supposed to and do what + * they should, and that cached blocks are accessible when expected to be. + */ +// Starts 100 threads, high variability of execution time => Medium +@Category(MediumTests.class) +public class TestSingleSizeCache { + SingleSizeCache cache; + final int CACHE_SIZE = 1000000; + final int NUM_BLOCKS = 100; + final int BLOCK_SIZE = CACHE_SIZE / NUM_BLOCKS; + final int NUM_THREADS = 100; + final int NUM_QUERIES = 10000; + + @Before + public void setup() { + cache = new SingleSizeCache(BLOCK_SIZE, NUM_BLOCKS, null); + } + + @After + public void tearDown() { + cache.shutdown(); + } + + @Test + public void testCacheSimple() throws Exception { + CacheTestUtils.testCacheSimple(cache, BLOCK_SIZE, NUM_QUERIES); + } + + @Test + public void testCacheMultiThreaded() throws Exception { + CacheTestUtils.testCacheMultiThreaded(cache, BLOCK_SIZE, + NUM_THREADS, NUM_QUERIES, 0.80); + } + + @Test + public void testCacheMultiThreadedSingleKey() throws Exception { + CacheTestUtils.hammerSingleKey(cache, BLOCK_SIZE, NUM_THREADS, NUM_QUERIES); + } + + @Test + public void testCacheMultiThreadedEviction() throws Exception { + CacheTestUtils.hammerEviction(cache, BLOCK_SIZE, NUM_THREADS, NUM_QUERIES); + } + + @Test + public void testHeapSizeChanges(){ + CacheTestUtils.testHeapSizeChanges(cache, BLOCK_SIZE); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlab.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlab.java new file mode 100644 index 0000000..124d131 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlab.java @@ -0,0 +1,81 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.io.hfile.slab; + +import static org.junit.Assert.*; +import java.nio.ByteBuffer; + +import org.apache.hadoop.hbase.SmallTests; +import org.junit.*; +import org.junit.experimental.categories.Category; + +/**Test cases for Slab.java*/ +@Category(SmallTests.class) +public class TestSlab { + static final int BLOCKSIZE = 1000; + static final int NUMBLOCKS = 100; + Slab testSlab; + ByteBuffer[] buffers = new ByteBuffer[NUMBLOCKS]; + + @Before + public void setUp() { + testSlab = new Slab(BLOCKSIZE, NUMBLOCKS); + } + + @After + public void tearDown() { + testSlab.shutdown(); + } + + @Test + public void testBasicFunctionality() throws InterruptedException { + for (int i = 0; i < NUMBLOCKS; i++) { + buffers[i] = testSlab.alloc(BLOCKSIZE); + assertEquals(BLOCKSIZE, buffers[i].limit()); + } + + // write an unique integer to each allocated buffer. + for (int i = 0; i < NUMBLOCKS; i++) { + buffers[i].putInt(i); + } + + // make sure the bytebuffers remain unique (the slab allocator hasn't + // allocated the same chunk of memory twice) + for (int i = 0; i < NUMBLOCKS; i++) { + buffers[i].putInt(i); + } + + for (int i = 0; i < NUMBLOCKS; i++) { + testSlab.free(buffers[i]); // free all the buffers. + } + + for (int i = 0; i < NUMBLOCKS; i++) { + buffers[i] = testSlab.alloc(BLOCKSIZE); + assertEquals(BLOCKSIZE, buffers[i].limit()); + } + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java new file mode 100644 index 0000000..a02f109 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java @@ -0,0 +1,115 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile.slab; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.io.hfile.CacheTestUtils; +import org.apache.hadoop.hbase.io.hfile.slab.SlabCache.SlabStats; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.Ignore; +import org.junit.experimental.categories.Category; + +/** + * Basic test of SlabCache. Puts and gets. + *

    + * + * Tests will ensure that blocks that are uncached are identical to the ones + * being cached, and that the cache never exceeds its capacity. Note that its + * fine if the cache evicts before it reaches max capacity - Guava Mapmaker may + * choose to evict at any time. + * + */ +// Starts 50 threads, high variability of execution time => Medium +@Category(MediumTests.class) +public class TestSlabCache { + static final int CACHE_SIZE = 1000000; + static final int NUM_BLOCKS = 101; + static final int BLOCK_SIZE = CACHE_SIZE / NUM_BLOCKS; + static final int NUM_THREADS = 50; + static final int NUM_QUERIES = 10000; + SlabCache cache; + + @Before + public void setup() { + cache = new SlabCache(CACHE_SIZE + BLOCK_SIZE * 2, BLOCK_SIZE); + cache.addSlabByConf(new Configuration()); + } + + @After + public void tearDown() { + cache.shutdown(); + } + + @Test + public void testElementPlacement() { + assertEquals(cache.getHigherBlock(BLOCK_SIZE).getKey().intValue(), + (BLOCK_SIZE * 11 / 10)); + assertEquals(cache.getHigherBlock((BLOCK_SIZE * 2)).getKey() + .intValue(), (BLOCK_SIZE * 21 / 10)); + } + + @Test + public void testCacheSimple() throws Exception { + CacheTestUtils.testCacheSimple(cache, BLOCK_SIZE, NUM_QUERIES); + } + + @Test + public void testCacheMultiThreaded() throws Exception { + CacheTestUtils.testCacheMultiThreaded(cache, BLOCK_SIZE, NUM_THREADS, + NUM_QUERIES, 0.80); + } + + @Test + public void testCacheMultiThreadedSingleKey() throws Exception { + CacheTestUtils.hammerSingleKey(cache, BLOCK_SIZE, NUM_THREADS, NUM_QUERIES); + } + + @Test + public void testCacheMultiThreadedEviction() throws Exception { + CacheTestUtils.hammerEviction(cache, BLOCK_SIZE, 10, NUM_QUERIES); + } + + @Test + /*Just checks if ranges overlap*/ + public void testStatsArithmetic(){ + SlabStats test = cache.requestStats; + for(int i = 0; i < test.NUMDIVISIONS; i++){ + assertTrue("Upper for index " + i + " is " + test.getUpperBound(i) + + " lower " + test.getLowerBound(i + 1), + test.getUpperBound(i) <= test.getLowerBound(i + 1)); + } + } + + @Test + public void testHeapSizeChanges(){ + CacheTestUtils.testHeapSizeChanges(cache, BLOCK_SIZE); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java new file mode 100644 index 0000000..ee390cd --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java @@ -0,0 +1,344 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.ipc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; + +import junit.framework.Assert; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.ipc.VersionedProtocol; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; +import org.apache.log4j.Level; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test that delayed RPCs work. Fire up three calls, the first of which should + * be delayed. Check that the last two, which are undelayed, return before the + * first one. + */ +@Category(MediumTests.class) // Fails sometimes with small tests +public class TestDelayedRpc { + public static RpcServer rpcServer; + + public static final int UNDELAYED = 0; + public static final int DELAYED = 1; + + @Test + public void testDelayedRpcImmediateReturnValue() throws Exception { + testDelayedRpc(false); + } + + @Test + public void testDelayedRpcDelayedReturnValue() throws Exception { + testDelayedRpc(true); + } + + private void testDelayedRpc(boolean delayReturnValue) throws Exception { + Configuration conf = HBaseConfiguration.create(); + InetSocketAddress isa = new InetSocketAddress("localhost", 0); + + rpcServer = HBaseRPC.getServer(new TestRpcImpl(delayReturnValue), + new Class[]{ TestRpcImpl.class }, + isa.getHostName(), isa.getPort(), 1, 0, true, conf, 0); + RpcEngine rpcEngine = null; + try { + rpcServer.start(); + rpcEngine = HBaseRPC.getProtocolEngine(conf); + + TestRpc client = rpcEngine.getProxy(TestRpc.class, 0, + rpcServer.getListenerAddress(), conf, 1000); + + List results = new ArrayList(); + + TestThread th1 = new TestThread(client, true, results); + TestThread th2 = new TestThread(client, false, results); + TestThread th3 = new TestThread(client, false, results); + th1.start(); + Thread.sleep(100); + th2.start(); + Thread.sleep(200); + th3.start(); + + th1.join(); + th2.join(); + th3.join(); + + assertEquals(UNDELAYED, results.get(0).intValue()); + assertEquals(UNDELAYED, results.get(1).intValue()); + assertEquals(results.get(2).intValue(), delayReturnValue ? DELAYED : + 0xDEADBEEF); + } finally { + if (rpcEngine != null) { + rpcEngine.close(); + } + } + } + + private static class ListAppender extends AppenderSkeleton { + private List messages = new ArrayList(); + + @Override + protected void append(LoggingEvent event) { + messages.add(event.getMessage().toString()); + } + + @Override + public void close() { + } + + @Override + public boolean requiresLayout() { + return false; + } + + public List getMessages() { + return messages; + } + } + + @Test + public void testTooManyDelayedRpcs() throws Exception { + Configuration conf = HBaseConfiguration.create(); + final int MAX_DELAYED_RPC = 10; + conf.setInt("hbase.ipc.warn.delayedrpc.number", MAX_DELAYED_RPC); + + ListAppender listAppender = new ListAppender(); + Logger log = Logger.getLogger("org.apache.hadoop.ipc.HBaseServer"); + log.addAppender(listAppender); + log.setLevel(Level.WARN); + + InetSocketAddress isa = new InetSocketAddress("localhost", 0); + rpcServer = HBaseRPC.getServer(new TestRpcImpl(true), + new Class[]{ TestRpcImpl.class }, + isa.getHostName(), isa.getPort(), 1, 0, true, conf, 0); + RpcEngine rpcEngine = null; + try { + rpcServer.start(); + rpcEngine = HBaseRPC.getProtocolEngine(conf); + + TestRpc client = rpcEngine.getProxy(TestRpc.class, 0, + rpcServer.getListenerAddress(), conf, 1000); + + Thread threads[] = new Thread[MAX_DELAYED_RPC + 1]; + + for (int i = 0; i < MAX_DELAYED_RPC; i++) { + threads[i] = new TestThread(client, true, null); + threads[i].start(); + } + + /* No warnings till here. */ + assertTrue(listAppender.getMessages().isEmpty()); + + /* This should give a warning. */ + threads[MAX_DELAYED_RPC] = new TestThread(client, true, null); + threads[MAX_DELAYED_RPC].start(); + + for (int i = 0; i < MAX_DELAYED_RPC; i++) { + threads[i].join(); + } + + assertFalse(listAppender.getMessages().isEmpty()); + assertTrue(listAppender.getMessages().get(0).startsWith( + "Too many delayed calls")); + + log.removeAppender(listAppender); + } finally { + if (rpcEngine != null) { + rpcEngine.close(); + } + } + } + + public interface TestRpc extends VersionedProtocol { + public static final long VERSION = 1L; + int test(boolean delay); + } + + private static class TestRpcImpl implements TestRpc { + /** + * Should the return value of delayed call be set at the end of the delay + * or at call return. + */ + private boolean delayReturnValue; + + /** + * @param delayReturnValue Should the response to the delayed call be set + * at the start or the end of the delay. + */ + public TestRpcImpl(boolean delayReturnValue) { + this.delayReturnValue = delayReturnValue; + } + + @Override + public int test(final boolean delay) { + if (!delay) { + return UNDELAYED; + } + final Delayable call = HBaseServer.getCurrentCall(); + call.startDelay(delayReturnValue); + new Thread() { + public void run() { + try { + Thread.sleep(500); + call.endDelay(delayReturnValue ? DELAYED : null); + } catch (Exception e) { + e.printStackTrace(); + } + } + }.start(); + // This value should go back to client only if the response is set + // immediately at delay time. + return 0xDEADBEEF; + } + + @Override + public long getProtocolVersion(String arg0, long arg1) throws IOException { + return 0; + } + + @Override + public ProtocolSignature getProtocolSignature(String protocol, + long clientVersion, int clientMethodsHash) throws IOException { + Method [] methods = this.getClass().getMethods(); + int [] hashes = new int [methods.length]; + for (int i = 0; i < methods.length; i++) { + hashes[i] = methods[i].hashCode(); + } + return new ProtocolSignature(clientVersion, hashes); + } + } + + private static class TestThread extends Thread { + private TestRpc server; + private boolean delay; + private List results; + + public TestThread(TestRpc server, boolean delay, List results) { + this.server = server; + this.delay = delay; + this.results = results; + } + + @Override + public void run() { + try { + Integer result = new Integer(server.test(delay)); + if (results != null) { + synchronized (results) { + results.add(result); + } + } + } catch (Exception e) { + fail("Unexpected exception: "+e.getMessage()); + } + } + } + + @Test + public void testEndDelayThrowing() throws IOException { + Configuration conf = HBaseConfiguration.create(); + InetSocketAddress isa = new InetSocketAddress("localhost", 0); + + rpcServer = HBaseRPC.getServer(new FaultyTestRpc(), + new Class[]{ TestRpcImpl.class }, + isa.getHostName(), isa.getPort(), 1, 0, true, conf, 0); + RpcEngine rpcEngine = null; + try { + rpcServer.start(); + rpcEngine = HBaseRPC.getProtocolEngine(conf); + + TestRpc client = rpcEngine.getProxy(TestRpc.class, 0, + rpcServer.getListenerAddress(), conf, 1000); + + int result = 0xDEADBEEF; + + try { + result = client.test(false); + } catch (Exception e) { + fail("No exception should have been thrown."); + } + assertEquals(result, UNDELAYED); + + boolean caughtException = false; + try { + result = client.test(true); + } catch(Exception e) { + // Exception thrown by server is enclosed in a RemoteException. + if (e.getCause().getMessage().startsWith( + "java.lang.Exception: Something went wrong")) + caughtException = true; + } + assertTrue(caughtException); + } finally { + if (rpcEngine != null) { + rpcEngine.close(); + } + } + } + + /** + * Delayed calls to this class throw an exception. + */ + private static class FaultyTestRpc implements TestRpc { + @Override + public int test(boolean delay) { + if (!delay) + return UNDELAYED; + Delayable call = HBaseServer.getCurrentCall(); + call.startDelay(true); + try { + call.endDelayThrowing(new Exception("Something went wrong")); + } catch (IOException e) { + e.printStackTrace(); + } + // Client will receive the Exception, not this value. + return DELAYED; + } + + @Override + public long getProtocolVersion(String arg0, long arg1) throws IOException { + return 0; + } + + @Override + public ProtocolSignature getProtocolSignature(String protocol, + long clientVersion, int clientMethodsHash) throws IOException { + return new ProtocolSignature(clientVersion, new int [] {}); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java new file mode 100644 index 0000000..d0eb78b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestPBOnWritableRpc.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.ipc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.DescriptorProtos.EnumDescriptorProto; + +/** Unit tests to test PB-based types on WritableRpcEngine. */ +@Category(MediumTests.class) +public class TestPBOnWritableRpc { + + private static Configuration conf = new Configuration(); + + public interface TestProtocol extends VersionedProtocol { + public static final long VERSION = 1L; + + String echo(String value) throws IOException; + Writable echo(Writable value) throws IOException; + + DescriptorProtos.EnumDescriptorProto exchangeProto( + DescriptorProtos.EnumDescriptorProto arg); + } + + public static class TestImpl implements TestProtocol { + public long getProtocolVersion(String protocol, long clientVersion) { + return TestProtocol.VERSION; + } + + public ProtocolSignature getProtocolSignature(String protocol, long clientVersion, + int hashcode) { + return new ProtocolSignature(TestProtocol.VERSION, null); + } + + @Override + public String echo(String value) throws IOException { return value; } + + @Override + public Writable echo(Writable writable) { + return writable; + } + + @Override + public EnumDescriptorProto exchangeProto(EnumDescriptorProto arg) { + return arg; + } + } + + @Test(timeout=60000) + public void testCalls() throws Exception { + testCallsInternal(conf); + } + + private void testCallsInternal(Configuration conf) throws Exception { + RpcServer rpcServer = HBaseRPC.getServer(new TestImpl(), + new Class[] {TestProtocol.class}, + "localhost", // BindAddress is IP we got for this server. + 0, // port number + 2, // number of handlers + 0, // we dont use high priority handlers in master + conf.getBoolean("hbase.rpc.verbose", false), conf, + 0); + RpcEngine rpcEngine = null; + try { + rpcServer.start(); + rpcEngine = HBaseRPC.getProtocolEngine(conf); + + InetSocketAddress isa = rpcServer.getListenerAddress(); + TestProtocol proxy = HBaseRPC.waitForProxy(rpcEngine, + TestProtocol.class, TestProtocol.VERSION, + isa, conf, -1, 8000, 8000); + + String stringResult = proxy.echo("foo"); + assertEquals(stringResult, "foo"); + + stringResult = proxy.echo((String)null); + assertEquals(stringResult, null); + + Text utf8Result = (Text)proxy.echo(new Text("hello world")); + assertEquals(utf8Result, new Text("hello world")); + + utf8Result = (Text)proxy.echo((Text)null); + assertEquals(utf8Result, null); + + // Test protobufs + EnumDescriptorProto sendProto = + EnumDescriptorProto.newBuilder().setName("test").build(); + EnumDescriptorProto retProto = proxy.exchangeProto(sendProto); + assertEquals(sendProto, retProto); + assertNotSame(sendProto, retProto); + } finally { + rpcServer.stop(); + if (rpcEngine != null) { + rpcEngine.close(); + } + } + } + + public static void main(String[] args) throws Exception { + new TestPBOnWritableRpc().testCallsInternal(conf); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java b/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java new file mode 100644 index 0000000..a8a0c6c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/ipc/TestProtocolExtension.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.ipc; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** Unit test for Protocol extending common interface. */ +@Category(SmallTests.class) +public class TestProtocolExtension { + private static final String ADDRESS = "0.0.0.0"; + + public static final Log LOG = + LogFactory.getLog(TestProtocolExtension.class); + + private static Configuration conf = new Configuration(); + + public interface ProtocolExtention { + void logClassName(); + } + + public interface TestProtocol extends VersionedProtocol, ProtocolExtention { + public static final long VERSION = 7L; + + void ping() throws IOException; + + // @Override // Uncomment to make the test pass + // public void logClassName(); +} + + public static class TestImpl implements TestProtocol { + public long getProtocolVersion(String protocol, long clientVersion) { + return TestProtocol.VERSION; + } + + @Override + public void ping() {} + + @Override + public void logClassName() { + LOG.info(this.getClass().getName()); + } + + @Override + public ProtocolSignature getProtocolSignature(String protocol, + long clientVersion, int clientMethodsHash) throws IOException { + return new ProtocolSignature(VERSION, null); + } + } + + @Test + public void testCalls() throws Exception { + RpcServer server = HBaseRPC.getServer(TestProtocol.class, + new TestImpl(), + new Class[]{ProtocolExtention.class}, + ADDRESS, + 6016, + 10, 10, false, + conf, 10); + RpcEngine rpcEngine = null; + try { + server.start(); + rpcEngine = HBaseRPC.getProtocolEngine(conf); + + InetSocketAddress addr = server.getListenerAddress(); + TestProtocol proxy = rpcEngine.getProxy( + TestProtocol.class, TestProtocol.VERSION, addr, conf, 10000); + + proxy.ping(); + + proxy.logClassName(); + } finally { + server.stop(); + if (rpcEngine != null) { + rpcEngine.close(); + } + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java new file mode 100644 index 0000000..38be7e4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java @@ -0,0 +1,393 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapred; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** + * This tests the TableInputFormat and its recovery semantics + * + */ +@Category(LargeTests.class) +public class TestTableInputFormat { + + private static final Log LOG = LogFactory.getLog(TestTableInputFormat.class); + + private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + static final byte[] FAMILY = Bytes.toBytes("family"); + + private static final byte[][] columns = new byte[][] { FAMILY }; + + @BeforeClass + public static void beforeClass() throws Exception { + UTIL.startMiniCluster(); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Before + public void before() throws IOException { + LOG.info("before"); + UTIL.ensureSomeRegionServersAvailable(1); + LOG.info("before done"); + } + + /** + * Setup a table with two rows and values. + * + * @param tableName + * @return + * @throws IOException + */ + public static HTable createTable(byte[] tableName) throws IOException { + HTable table = UTIL.createTable(tableName, FAMILY); + Put p = new Put("aaa".getBytes()); + p.add(FAMILY, null, "value aaa".getBytes()); + table.put(p); + p = new Put("bbb".getBytes()); + p.add(FAMILY, null, "value bbb".getBytes()); + table.put(p); + return table; + } + + /** + * Verify that the result and key have expected values. + * + * @param r + * @param key + * @param expectedKey + * @param expectedValue + * @return + */ + static boolean checkResult(Result r, ImmutableBytesWritable key, + byte[] expectedKey, byte[] expectedValue) { + assertEquals(0, key.compareTo(expectedKey)); + Map vals = r.getFamilyMap(FAMILY); + byte[] value = vals.values().iterator().next(); + assertTrue(Arrays.equals(value, expectedValue)); + return true; // if succeed + } + + /** + * Create table data and run tests on specified htable using the + * o.a.h.hbase.mapred API. + * + * @param table + * @throws IOException + */ + static void runTestMapred(HTable table) throws IOException { + org.apache.hadoop.hbase.mapred.TableRecordReader trr = + new org.apache.hadoop.hbase.mapred.TableRecordReader(); + trr.setStartRow("aaa".getBytes()); + trr.setEndRow("zzz".getBytes()); + trr.setHTable(table); + trr.setInputColumns(columns); + + trr.init(); + Result r = new Result(); + ImmutableBytesWritable key = new ImmutableBytesWritable(); + + boolean more = trr.next(key, r); + assertTrue(more); + checkResult(r, key, "aaa".getBytes(), "value aaa".getBytes()); + + more = trr.next(key, r); + assertTrue(more); + checkResult(r, key, "bbb".getBytes(), "value bbb".getBytes()); + + // no more data + more = trr.next(key, r); + assertFalse(more); + } + + /** + * Create table data and run tests on specified htable using the + * o.a.h.hbase.mapreduce API. + * + * @param table + * @throws IOException + * @throws InterruptedException + */ + static void runTestMapreduce(HTable table) throws IOException, + InterruptedException { + org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl trr = + new org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl(); + Scan s = new Scan(); + s.setStartRow("aaa".getBytes()); + s.setStopRow("zzz".getBytes()); + s.addFamily(FAMILY); + trr.setScan(s); + trr.setHTable(table); + + trr.initialize(null, null); + Result r = new Result(); + ImmutableBytesWritable key = new ImmutableBytesWritable(); + + boolean more = trr.nextKeyValue(); + assertTrue(more); + key = trr.getCurrentKey(); + r = trr.getCurrentValue(); + checkResult(r, key, "aaa".getBytes(), "value aaa".getBytes()); + + more = trr.nextKeyValue(); + assertTrue(more); + key = trr.getCurrentKey(); + r = trr.getCurrentValue(); + checkResult(r, key, "bbb".getBytes(), "value bbb".getBytes()); + + // no more data + more = trr.nextKeyValue(); + assertFalse(more); + } + + /** + * Create a table that IOE's on first scanner next call + * + * @throws IOException + */ + static HTable createIOEScannerTable(byte[] name, final int failCnt) + throws IOException { + // build up a mock scanner stuff to fail the first time + Answer a = new Answer() { + int cnt = 0; + + @Override + public ResultScanner answer(InvocationOnMock invocation) throws Throwable { + // first invocation return the busted mock scanner + if (cnt++ < failCnt) { + // create mock ResultScanner that always fails. + Scan scan = mock(Scan.class); + doReturn("bogus".getBytes()).when(scan).getStartRow(); // avoid npe + ResultScanner scanner = mock(ResultScanner.class); + // simulate TimeoutException / IOException + doThrow(new IOException("Injected exception")).when(scanner).next(); + return scanner; + } + + // otherwise return the real scanner. + return (ResultScanner) invocation.callRealMethod(); + } + }; + + HTable htable = spy(createTable(name)); + doAnswer(a).when(htable).getScanner((Scan) anyObject()); + return htable; + } + + /** + * Create a table that throws a DoNoRetryIOException on first scanner next + * call + * + * @throws IOException + */ + static HTable createDNRIOEScannerTable(byte[] name, final int failCnt) + throws IOException { + // build up a mock scanner stuff to fail the first time + Answer a = new Answer() { + int cnt = 0; + + @Override + public ResultScanner answer(InvocationOnMock invocation) throws Throwable { + // first invocation return the busted mock scanner + if (cnt++ < failCnt) { + // create mock ResultScanner that always fails. + Scan scan = mock(Scan.class); + doReturn("bogus".getBytes()).when(scan).getStartRow(); // avoid npe + ResultScanner scanner = mock(ResultScanner.class); + + invocation.callRealMethod(); // simulate UnknownScannerException + doThrow( + new UnknownScannerException("Injected simulated TimeoutException")) + .when(scanner).next(); + return scanner; + } + + // otherwise return the real scanner. + return (ResultScanner) invocation.callRealMethod(); + } + }; + + HTable htable = spy(createTable(name)); + doAnswer(a).when(htable).getScanner((Scan) anyObject()); + return htable; + } + + /** + * Run test assuming no errors using mapred api. + * + * @throws IOException + */ + @Test + public void testTableRecordReader() throws IOException { + HTable table = createTable("table1".getBytes()); + runTestMapred(table); + } + + /** + * Run test assuming Scanner IOException failure using mapred api, + * + * @throws IOException + */ + @Test + public void testTableRecordReaderScannerFail() throws IOException { + HTable htable = createIOEScannerTable("table2".getBytes(), 1); + runTestMapred(htable); + } + + /** + * Run test assuming Scanner IOException failure using mapred api, + * + * @throws IOException + */ + @Test(expected = IOException.class) + public void testTableRecordReaderScannerFailTwice() throws IOException { + HTable htable = createIOEScannerTable("table3".getBytes(), 2); + runTestMapred(htable); + } + + /** + * Run test assuming UnknownScannerException (which is a type of + * DoNotRetryIOException) using mapred api. + * + * @throws DoNotRetryIOException + */ + @Test + public void testTableRecordReaderScannerTimeout() throws IOException { + HTable htable = createDNRIOEScannerTable("table4".getBytes(), 1); + runTestMapred(htable); + } + + /** + * Run test assuming UnknownScannerException (which is a type of + * DoNotRetryIOException) using mapred api. + * + * @throws DoNotRetryIOException + */ + @Test(expected = DoNotRetryIOException.class) + public void testTableRecordReaderScannerTimeoutTwice() throws IOException { + HTable htable = createDNRIOEScannerTable("table5".getBytes(), 2); + runTestMapred(htable); + } + + /** + * Run test assuming no errors using newer mapreduce api + * + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testTableRecordReaderMapreduce() throws IOException, + InterruptedException { + HTable table = createTable("table1-mr".getBytes()); + runTestMapreduce(table); + } + + /** + * Run test assuming Scanner IOException failure using newer mapreduce api + * + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testTableRecordReaderScannerFailMapreduce() throws IOException, + InterruptedException { + HTable htable = createIOEScannerTable("table2-mr".getBytes(), 1); + runTestMapreduce(htable); + } + + /** + * Run test assuming Scanner IOException failure using newer mapreduce api + * + * @throws IOException + * @throws InterruptedException + */ + @Test(expected = IOException.class) + public void testTableRecordReaderScannerFailMapreduceTwice() throws IOException, + InterruptedException { + HTable htable = createIOEScannerTable("table3-mr".getBytes(), 2); + runTestMapreduce(htable); + } + + /** + * Run test assuming UnknownScannerException (which is a type of + * DoNotRetryIOException) using newer mapreduce api + * + * @throws InterruptedException + * @throws DoNotRetryIOException + */ + @Test + public void testTableRecordReaderScannerTimeoutMapreduce() + throws IOException, InterruptedException { + HTable htable = createDNRIOEScannerTable("table4-mr".getBytes(), 1); + runTestMapreduce(htable); + } + + /** + * Run test assuming UnknownScannerException (which is a type of + * DoNotRetryIOException) using newer mapreduce api + * + * @throws InterruptedException + * @throws DoNotRetryIOException + */ + @Test(expected = DoNotRetryIOException.class) + public void testTableRecordReaderScannerTimeoutMapreduceTwice() + throws IOException, InterruptedException { + HTable htable = createDNRIOEScannerTable("table5-mr".getBytes(), 2); + runTestMapreduce(htable); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java b/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java new file mode 100644 index 0000000..7fe273b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java @@ -0,0 +1,268 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapred; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.TableInputFormat; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MapReduceBase; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.RunningJob; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.fail; +import static org.junit.Assert.assertTrue; + +/** + * Test Map/Reduce job over HBase tables. The map/reduce process we're testing + * on our tables is simple - take every row in the table, reverse the value of + * a particular cell, and write it back to the table. + */ +@Category(LargeTests.class) +public class TestTableMapReduce { + private static final Log LOG = + LogFactory.getLog(TestTableMapReduce.class.getName()); + private static final HBaseTestingUtility UTIL = + new HBaseTestingUtility(); + static final byte[] MULTI_REGION_TABLE_NAME = Bytes.toBytes("mrtest"); + static final byte[] INPUT_FAMILY = Bytes.toBytes("contents"); + static final byte[] OUTPUT_FAMILY = Bytes.toBytes("text"); + + private static final byte [][] columns = new byte [][] { + INPUT_FAMILY, + OUTPUT_FAMILY + }; + + @BeforeClass + public static void beforeClass() throws Exception { + UTIL.startMiniCluster(); + HTable table = UTIL.createTable(MULTI_REGION_TABLE_NAME, new byte[][] {INPUT_FAMILY, OUTPUT_FAMILY}); + UTIL.createMultiRegions(table, INPUT_FAMILY); + UTIL.loadTable(table, INPUT_FAMILY); + UTIL.startMiniMapReduceCluster(); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniMapReduceCluster(); + UTIL.shutdownMiniCluster(); + } + + /** + * Pass the given key and processed record reduce + */ + public static class ProcessContentsMapper + extends MapReduceBase + implements TableMap { + /** + * Pass the key, and reversed value to reduce + * @param key + * @param value + * @param output + * @param reporter + * @throws IOException + */ + public void map(ImmutableBytesWritable key, Result value, + OutputCollector output, + Reporter reporter) + throws IOException { + if (value.size() != 1) { + throw new IOException("There should only be one input column"); + } + Map>> + cf = value.getMap(); + if(!cf.containsKey(INPUT_FAMILY)) { + throw new IOException("Wrong input columns. Missing: '" + + Bytes.toString(INPUT_FAMILY) + "'."); + } + + // Get the original value and reverse it + + String originalValue = new String(value.getValue(INPUT_FAMILY, null), + HConstants.UTF8_ENCODING); + StringBuilder newValue = new StringBuilder(originalValue); + newValue.reverse(); + + // Now set the value to be collected + + Put outval = new Put(key.get()); + outval.add(OUTPUT_FAMILY, null, Bytes.toBytes(newValue.toString())); + output.collect(key, outval); + } + } + + /** + * Test a map/reduce against a multi-region table + * @throws IOException + */ + @Test + public void testMultiRegionTable() throws IOException { + runTestOnTable(new HTable(UTIL.getConfiguration(), MULTI_REGION_TABLE_NAME)); + } + + private void runTestOnTable(HTable table) throws IOException { + JobConf jobConf = null; + try { + LOG.info("Before map/reduce startup"); + jobConf = new JobConf(UTIL.getConfiguration(), TestTableMapReduce.class); + jobConf.setJobName("process column contents"); + jobConf.setNumReduceTasks(1); + TableMapReduceUtil.initTableMapJob(Bytes.toString(table.getTableName()), + Bytes.toString(INPUT_FAMILY), ProcessContentsMapper.class, + ImmutableBytesWritable.class, Put.class, jobConf); + TableMapReduceUtil.initTableReduceJob(Bytes.toString(table.getTableName()), + IdentityTableReduce.class, jobConf); + + LOG.info("Started " + Bytes.toString(table.getTableName())); + RunningJob job = JobClient.runJob(jobConf); + assertTrue(job.isSuccessful()); + LOG.info("After map/reduce completion"); + + // verify map-reduce results + verify(Bytes.toString(table.getTableName())); + } finally { + if (jobConf != null) { + FileUtil.fullyDelete(new File(jobConf.get("hadoop.tmp.dir"))); + } + } + } + + private void verify(String tableName) throws IOException { + HTable table = new HTable(UTIL.getConfiguration(), tableName); + boolean verified = false; + long pause = UTIL.getConfiguration().getLong("hbase.client.pause", 5 * 1000); + int numRetries = UTIL.getConfiguration().getInt("hbase.client.retries.number", 5); + for (int i = 0; i < numRetries; i++) { + try { + LOG.info("Verification attempt #" + i); + verifyAttempt(table); + verified = true; + break; + } catch (NullPointerException e) { + // If here, a cell was empty. Presume its because updates came in + // after the scanner had been opened. Wait a while and retry. + LOG.debug("Verification attempt failed: " + e.getMessage()); + } + try { + Thread.sleep(pause); + } catch (InterruptedException e) { + // continue + } + } + assertTrue(verified); + } + + /** + * Looks at every value of the mapreduce output and verifies that indeed + * the values have been reversed. + * @param table Table to scan. + * @throws IOException + * @throws NullPointerException if we failed to find a cell value + */ + private void verifyAttempt(final HTable table) throws IOException, NullPointerException { + Scan scan = new Scan(); + TableInputFormat.addColumns(scan, columns); + ResultScanner scanner = table.getScanner(scan); + try { + Iterator itr = scanner.iterator(); + assertTrue(itr.hasNext()); + while(itr.hasNext()) { + Result r = itr.next(); + if (LOG.isDebugEnabled()) { + if (r.size() > 2 ) { + throw new IOException("Too many results, expected 2 got " + + r.size()); + } + } + byte[] firstValue = null; + byte[] secondValue = null; + int count = 0; + for(KeyValue kv : r.list()) { + if (count == 0) { + firstValue = kv.getValue(); + } + if (count == 1) { + secondValue = kv.getValue(); + } + count++; + if (count == 2) { + break; + } + } + + + String first = ""; + if (firstValue == null) { + throw new NullPointerException(Bytes.toString(r.getRow()) + + ": first value is null"); + } + first = new String(firstValue, HConstants.UTF8_ENCODING); + + String second = ""; + if (secondValue == null) { + throw new NullPointerException(Bytes.toString(r.getRow()) + + ": second value is null"); + } + byte[] secondReversed = new byte[secondValue.length]; + for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) { + secondReversed[i] = secondValue[j]; + } + second = new String(secondReversed, HConstants.UTF8_ENCODING); + + if (first.compareTo(second) != 0) { + if (LOG.isDebugEnabled()) { + LOG.debug("second key is not the reverse of first. row=" + + r.getRow() + ", first value=" + first + ", second value=" + + second); + } + fail(); + } + } + } finally { + scanner.close(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java new file mode 100644 index 0000000..76de6a3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.JobID; + +/** + * This class provides shims for HBase to interact with the Hadoop 1.0.x and the + * Hadoop 0.23.x series. + * + * NOTE: No testing done against 0.22.x, or 0.21.x. + */ +abstract public class MapreduceTestingShim { + private static MapreduceTestingShim instance; + + static { + try { + // This class exists in hadoop 0.22+ but not in Hadoop 20.x/1.x + Class c = Class + .forName("org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl"); + instance = new MapreduceV2Shim(); + } catch (Exception e) { + instance = new MapreduceV1Shim(); + } + } + + abstract public JobContext newJobContext(Configuration jobConf) + throws IOException; + + public static JobContext createJobContext(Configuration jobConf) + throws IOException { + return instance.newJobContext(jobConf); + } + + private static class MapreduceV1Shim extends MapreduceTestingShim { + public JobContext newJobContext(Configuration jobConf) throws IOException { + // Implementing: + // return new JobContext(jobConf, new JobID()); + JobID jobId = new JobID(); + Constructor c; + try { + c = JobContext.class.getConstructor(Configuration.class, JobID.class); + return c.newInstance(jobConf, jobId); + } catch (Exception e) { + throw new IllegalStateException( + "Failed to instantiate new JobContext(jobConf, new JobID())", e); + } + } + }; + + private static class MapreduceV2Shim extends MapreduceTestingShim { + public JobContext newJobContext(Configuration jobConf) { + // Implementing: + // return Job.getInstance(jobConf); + try { + Method m = Job.class.getMethod("getInstance", Configuration.class); + return (JobContext) m.invoke(null, jobConf); // static method, then arg + } catch (Exception e) { + e.printStackTrace(); + throw new IllegalStateException( + "Failed to return from Job.getInstance(jobConf)"); + } + } + }; + +} diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/NMapInputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/NMapInputFormat.java new file mode 100644 index 0000000..f4b3f65 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/NMapInputFormat.java @@ -0,0 +1,136 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +/** + * Input format that creates a configurable number of map tasks + * each provided with a single row of NullWritables. This can be + * useful when trying to write mappers which don't have any real + * input (eg when the mapper is simply producing random data as output) + */ +public class NMapInputFormat extends InputFormat { + private static final String NMAPS_KEY = "nmapinputformat.num.maps"; + + @Override + public RecordReader createRecordReader( + InputSplit split, + TaskAttemptContext tac) throws IOException, InterruptedException { + return new SingleRecordReader( + NullWritable.get(), NullWritable.get()); + } + + @Override + public List getSplits(JobContext context) throws IOException, + InterruptedException { + int count = getNumMapTasks(context.getConfiguration()); + List splits = new ArrayList(count); + for (int i = 0; i < count; i++) { + splits.add(new NullInputSplit()); + } + return splits; + } + + public static void setNumMapTasks(Configuration conf, int numTasks) { + conf.setInt(NMAPS_KEY, numTasks); + } + + public static int getNumMapTasks(Configuration conf) { + return conf.getInt(NMAPS_KEY, 1); + } + + private static class NullInputSplit extends InputSplit implements Writable { + @Override + public long getLength() throws IOException, InterruptedException { + return 0; + } + + @Override + public String[] getLocations() throws IOException, InterruptedException { + return new String[] {}; + } + + @Override + public void readFields(DataInput in) throws IOException { + } + + @Override + public void write(DataOutput out) throws IOException { + } + } + + private static class SingleRecordReader + extends RecordReader { + + private final K key; + private final V value; + boolean providedKey = false; + + SingleRecordReader(K key, V value) { + this.key = key; + this.value = value; + } + + @Override + public void close() { + } + + @Override + public K getCurrentKey() { + return key; + } + + @Override + public V getCurrentValue(){ + return value; + } + + @Override + public float getProgress() { + return 0; + } + + @Override + public void initialize(InputSplit split, TaskAttemptContext tac) { + } + + @Override + public boolean nextKeyValue() { + if (providedKey) return false; + providedKey = true; + return true; + } + + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java new file mode 100644 index 0000000..5bf8bbd --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java @@ -0,0 +1,848 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.Callable; +import java.util.Random; + +import junit.framework.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFile.Reader; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import com.google.common.collect.Lists; + +/** + * Simple test for {@link KeyValueSortReducer} and {@link HFileOutputFormat}. + * Sets up and runs a mapreduce job that writes hfile output. + * Creates a few inner classes to implement splits and an inputformat that + * emits keys and values like those of {@link PerformanceEvaluation}. + */ +@Category(LargeTests.class) +public class TestHFileOutputFormat { + private final static int ROWSPERSPLIT = 1024; + + private static final byte[][] FAMILIES + = { Bytes.add(PerformanceEvaluation.FAMILY_NAME, Bytes.toBytes("-A")) + , Bytes.add(PerformanceEvaluation.FAMILY_NAME, Bytes.toBytes("-B"))}; + private static final byte[] TABLE_NAME = Bytes.toBytes("TestTable"); + + private HBaseTestingUtility util = new HBaseTestingUtility(); + + private static Log LOG = LogFactory.getLog(TestHFileOutputFormat.class); + + /** + * Simple mapper that makes KeyValue output. + */ + static class RandomKVGeneratingMapper + extends Mapper { + + private int keyLength; + private static final int KEYLEN_DEFAULT=10; + private static final String KEYLEN_CONF="randomkv.key.length"; + + private int valLength; + private static final int VALLEN_DEFAULT=10; + private static final String VALLEN_CONF="randomkv.val.length"; + + @Override + protected void setup(Context context) throws IOException, + InterruptedException { + super.setup(context); + + Configuration conf = context.getConfiguration(); + keyLength = conf.getInt(KEYLEN_CONF, KEYLEN_DEFAULT); + valLength = conf.getInt(VALLEN_CONF, VALLEN_DEFAULT); + } + + protected void map( + NullWritable n1, NullWritable n2, + Mapper.Context context) + throws java.io.IOException ,InterruptedException + { + + byte keyBytes[] = new byte[keyLength]; + byte valBytes[] = new byte[valLength]; + + int taskId = context.getTaskAttemptID().getTaskID().getId(); + assert taskId < Byte.MAX_VALUE : "Unit tests dont support > 127 tasks!"; + + Random random = new Random(); + for (int i = 0; i < ROWSPERSPLIT; i++) { + + random.nextBytes(keyBytes); + // Ensure that unique tasks generate unique keys + keyBytes[keyLength - 1] = (byte)(taskId & 0xFF); + random.nextBytes(valBytes); + ImmutableBytesWritable key = new ImmutableBytesWritable(keyBytes); + + for (byte[] family : TestHFileOutputFormat.FAMILIES) { + KeyValue kv = new KeyValue(keyBytes, family, + PerformanceEvaluation.QUALIFIER_NAME, valBytes); + context.write(key, kv); + } + } + } + } + + @Before + public void cleanupDir() throws IOException { + util.cleanupTestDir(); + } + + + private void setupRandomGeneratorMapper(Job job) { + job.setInputFormatClass(NMapInputFormat.class); + job.setMapperClass(RandomKVGeneratingMapper.class); + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + job.setMapOutputValueClass(KeyValue.class); + } + + /** + * Test that {@link HFileOutputFormat} RecordWriter amends timestamps if + * passed a keyvalue whose timestamp is {@link HConstants#LATEST_TIMESTAMP}. + * @see HBASE-2615 + */ + @Test + public void test_LATEST_TIMESTAMP_isReplaced() + throws Exception { + Configuration conf = new Configuration(this.util.getConfiguration()); + RecordWriter writer = null; + TaskAttemptContext context = null; + Path dir = + util.getDataTestDir("test_LATEST_TIMESTAMP_isReplaced"); + try { + Job job = new Job(conf); + FileOutputFormat.setOutputPath(job, dir); + context = getTestTaskAttemptContext(job); + HFileOutputFormat hof = new HFileOutputFormat(); + writer = hof.getRecordWriter(context); + final byte [] b = Bytes.toBytes("b"); + + // Test 1. Pass a KV that has a ts of LATEST_TIMESTAMP. It should be + // changed by call to write. Check all in kv is same but ts. + KeyValue kv = new KeyValue(b, b, b); + KeyValue original = kv.clone(); + writer.write(new ImmutableBytesWritable(), kv); + assertFalse(original.equals(kv)); + assertTrue(Bytes.equals(original.getRow(), kv.getRow())); + assertTrue(original.matchingColumn(kv.getFamily(), kv.getQualifier())); + assertNotSame(original.getTimestamp(), kv.getTimestamp()); + assertNotSame(HConstants.LATEST_TIMESTAMP, kv.getTimestamp()); + + // Test 2. Now test passing a kv that has explicit ts. It should not be + // changed by call to record write. + kv = new KeyValue(b, b, b, kv.getTimestamp() - 1, b); + original = kv.clone(); + writer.write(new ImmutableBytesWritable(), kv); + assertTrue(original.equals(kv)); + } finally { + if (writer != null && context != null) writer.close(context); + dir.getFileSystem(conf).delete(dir, true); + } + } + + /** + * @return True if the available mapreduce is post-0.20. + */ + private static boolean isPost020MapReduce() { + // Here is a coarse test for post 0.20 hadoop; TAC became an interface. + return TaskAttemptContext.class.isInterface(); + } + + private TaskAttemptContext getTestTaskAttemptContext(final Job job) + throws IOException, Exception { + TaskAttemptContext context; + if (isPost020MapReduce()) { + TaskAttemptID id = + TaskAttemptID.forName("attempt_200707121733_0001_m_000000_0"); + Class clazz = + Class.forName("org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl"); + Constructor c = clazz. + getConstructor(Configuration.class, TaskAttemptID.class); + context = (TaskAttemptContext)c.newInstance(job.getConfiguration(), id); + } else { + context = org.apache.hadoop.hbase.mapreduce.hadoopbackport.InputSampler. + getTaskAttemptContext(job); + } + return context; + } + + /* + * Test that {@link HFileOutputFormat} creates an HFile with TIMERANGE + * metadata used by time-restricted scans. + */ + @Test + public void test_TIMERANGE() throws Exception { + Configuration conf = new Configuration(this.util.getConfiguration()); + RecordWriter writer = null; + TaskAttemptContext context = null; + Path dir = + util.getDataTestDir("test_TIMERANGE_present"); + LOG.info("Timerange dir writing to dir: "+ dir); + try { + // build a record writer using HFileOutputFormat + Job job = new Job(conf); + FileOutputFormat.setOutputPath(job, dir); + context = getTestTaskAttemptContext(job); + HFileOutputFormat hof = new HFileOutputFormat(); + writer = hof.getRecordWriter(context); + + // Pass two key values with explicit times stamps + final byte [] b = Bytes.toBytes("b"); + + // value 1 with timestamp 2000 + KeyValue kv = new KeyValue(b, b, b, 2000, b); + KeyValue original = kv.clone(); + writer.write(new ImmutableBytesWritable(), kv); + assertEquals(original,kv); + + // value 2 with timestamp 1000 + kv = new KeyValue(b, b, b, 1000, b); + original = kv.clone(); + writer.write(new ImmutableBytesWritable(), kv); + assertEquals(original, kv); + + // verify that the file has the proper FileInfo. + writer.close(context); + + // the generated file lives 1 directory down from the attempt directory + // and is the only file, e.g. + // _attempt__0000_r_000000_0/b/1979617994050536795 + FileSystem fs = FileSystem.get(conf); + Path attemptDirectory = hof.getDefaultWorkFile(context, "").getParent(); + FileStatus[] sub1 = fs.listStatus(attemptDirectory); + FileStatus[] file = fs.listStatus(sub1[0].getPath()); + + // open as HFile Reader and pull out TIMERANGE FileInfo. + HFile.Reader rd = HFile.createReader(fs, file[0].getPath(), + new CacheConfig(conf)); + Map finfo = rd.loadFileInfo(); + byte[] range = finfo.get("TIMERANGE".getBytes()); + assertNotNull(range); + + // unmarshall and check values. + TimeRangeTracker timeRangeTracker = new TimeRangeTracker(); + Writables.copyWritable(range, timeRangeTracker); + LOG.info(timeRangeTracker.getMinimumTimestamp() + + "...." + timeRangeTracker.getMaximumTimestamp()); + assertEquals(1000, timeRangeTracker.getMinimumTimestamp()); + assertEquals(2000, timeRangeTracker.getMaximumTimestamp()); + rd.close(); + } finally { + if (writer != null && context != null) writer.close(context); + dir.getFileSystem(conf).delete(dir, true); + } + } + + /** + * Run small MR job. + */ + @Test + public void testWritingPEData() throws Exception { + Configuration conf = util.getConfiguration(); + Path testDir = util.getDataTestDir("testWritingPEData"); + FileSystem fs = testDir.getFileSystem(conf); + + // Set down this value or we OOME in eclipse. + conf.setInt("io.sort.mb", 20); + // Write a few files. + conf.setLong(HConstants.HREGION_MAX_FILESIZE, 64 * 1024); + + Job job = new Job(conf, "testWritingPEData"); + setupRandomGeneratorMapper(job); + // This partitioner doesn't work well for number keys but using it anyways + // just to demonstrate how to configure it. + byte[] startKey = new byte[RandomKVGeneratingMapper.KEYLEN_DEFAULT]; + byte[] endKey = new byte[RandomKVGeneratingMapper.KEYLEN_DEFAULT]; + + Arrays.fill(startKey, (byte)0); + Arrays.fill(endKey, (byte)0xff); + + job.setPartitionerClass(SimpleTotalOrderPartitioner.class); + // Set start and end rows for partitioner. + SimpleTotalOrderPartitioner.setStartKey(job.getConfiguration(), startKey); + SimpleTotalOrderPartitioner.setEndKey(job.getConfiguration(), endKey); + job.setReducerClass(KeyValueSortReducer.class); + job.setOutputFormatClass(HFileOutputFormat.class); + job.setNumReduceTasks(4); + + FileOutputFormat.setOutputPath(job, testDir); + assertTrue(job.waitForCompletion(false)); + FileStatus [] files = fs.listStatus(testDir); + assertTrue(files.length > 0); + } + + @Test + public void testJobConfiguration() throws Exception { + Job job = new Job(); + HTable table = Mockito.mock(HTable.class); + setupMockStartKeys(table); + HFileOutputFormat.configureIncrementalLoad(job, table); + assertEquals(job.getNumReduceTasks(), 4); + } + + private byte [][] generateRandomStartKeys(int numKeys) { + Random random = new Random(); + byte[][] ret = new byte[numKeys][]; + // first region start key is always empty + ret[0] = HConstants.EMPTY_BYTE_ARRAY; + for (int i = 1; i < numKeys; i++) { + ret[i] = PerformanceEvaluation.generateValue(random); + } + return ret; + } + + @Test + public void testMRIncrementalLoad() throws Exception { + doIncrementalLoadTest(false); + } + + @Test + public void testMRIncrementalLoadWithSplit() throws Exception { + doIncrementalLoadTest(true); + } + + private void doIncrementalLoadTest( + boolean shouldChangeRegions) throws Exception { + Configuration conf = util.getConfiguration(); + Path testDir = util.getDataTestDir("testLocalMRIncrementalLoad"); + byte[][] startKeys = generateRandomStartKeys(5); + + try { + util.startMiniCluster(); + HBaseAdmin admin = new HBaseAdmin(conf); + HTable table = util.createTable(TABLE_NAME, FAMILIES); + assertEquals("Should start with empty table", + 0, util.countRows(table)); + int numRegions = util.createMultiRegions( + util.getConfiguration(), table, FAMILIES[0], startKeys); + assertEquals("Should make 5 regions", numRegions, 5); + + // Generate the bulk load files + util.startMiniMapReduceCluster(); + runIncrementalPELoad(conf, table, testDir); + // This doesn't write into the table, just makes files + assertEquals("HFOF should not touch actual table", + 0, util.countRows(table)); + + + // Make sure that a directory was created for every CF + int dir = 0; + for (FileStatus f : testDir.getFileSystem(conf).listStatus(testDir)) { + for (byte[] family : FAMILIES) { + if (Bytes.toString(family).equals(f.getPath().getName())) { + ++dir; + } + } + } + assertEquals("Column family not found in FS.", FAMILIES.length, dir); + + // handle the split case + if (shouldChangeRegions) { + LOG.info("Changing regions in table"); + admin.disableTable(table.getTableName()); + while(util.getMiniHBaseCluster().getMaster().getAssignmentManager(). + isRegionsInTransition()) { + Threads.sleep(200); + LOG.info("Waiting on table to finish disabling"); + } + byte[][] newStartKeys = generateRandomStartKeys(15); + util.createMultiRegions( + util.getConfiguration(), table, FAMILIES[0], newStartKeys); + admin.enableTable(table.getTableName()); + while (table.getRegionsInfo().size() != 15 || + !admin.isTableAvailable(table.getTableName())) { + Thread.sleep(200); + LOG.info("Waiting for new region assignment to happen"); + } + } + + // Perform the actual load + new LoadIncrementalHFiles(conf).doBulkLoad(testDir, table); + + // Ensure data shows up + int expectedRows = NMapInputFormat.getNumMapTasks(conf) * ROWSPERSPLIT; + assertEquals("LoadIncrementalHFiles should put expected data in table", + expectedRows, util.countRows(table)); + Scan scan = new Scan(); + ResultScanner results = table.getScanner(scan); + int count = 0; + for (Result res : results) { + count++; + assertEquals(FAMILIES.length, res.raw().length); + KeyValue first = res.raw()[0]; + for (KeyValue kv : res.raw()) { + assertTrue(KeyValue.COMPARATOR.matchingRows(first, kv)); + assertTrue(Bytes.equals(first.getValue(), kv.getValue())); + } + } + results.close(); + String tableDigestBefore = util.checksumRows(table); + + // Cause regions to reopen + admin.disableTable(TABLE_NAME); + while (!admin.isTableDisabled(TABLE_NAME)) { + Thread.sleep(200); + LOG.info("Waiting for table to disable"); + } + admin.enableTable(TABLE_NAME); + util.waitTableAvailable(TABLE_NAME, 30000); + assertEquals("Data should remain after reopening of regions", + tableDigestBefore, util.checksumRows(table)); + } finally { + util.shutdownMiniMapReduceCluster(); + util.shutdownMiniCluster(); + } + } + + private void runIncrementalPELoad( + Configuration conf, HTable table, Path outDir) + throws Exception { + Job job = new Job(conf, "testLocalMRIncrementalLoad"); + setupRandomGeneratorMapper(job); + HFileOutputFormat.configureIncrementalLoad(job, table); + FileOutputFormat.setOutputPath(job, outDir); + + Assert.assertFalse( util.getTestFileSystem().exists(outDir)) ; + + assertEquals(table.getRegionsInfo().size(), + job.getNumReduceTasks()); + + assertTrue(job.waitForCompletion(true)); + } + + /** + * Test for + * {@link HFileOutputFormat#createFamilyCompressionMap(Configuration)}. Tests + * that the compression map is correctly deserialized from configuration + * + * @throws IOException + */ + @Test + public void testCreateFamilyCompressionMap() throws IOException { + for (int numCfs = 0; numCfs <= 3; numCfs++) { + Configuration conf = new Configuration(this.util.getConfiguration()); + Map familyToCompression = getMockColumnFamilies(numCfs); + HTable table = Mockito.mock(HTable.class); + setupMockColumnFamilies(table, familyToCompression); + HFileOutputFormat.configureCompression(table, conf); + + // read back family specific compression setting from the configuration + Map retrievedFamilyToCompressionMap = HFileOutputFormat.createFamilyCompressionMap(conf); + + // test that we have a value for all column families that matches with the + // used mock values + for (Entry entry : familyToCompression.entrySet()) { + assertEquals("Compression configuration incorrect for column family:" + entry.getKey(), entry.getValue() + .getName(), retrievedFamilyToCompressionMap.get(entry.getKey().getBytes())); + } + } + } + + private void setupMockColumnFamilies(HTable table, + Map familyToCompression) throws IOException + { + HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAME); + for (Entry entry : familyToCompression.entrySet()) { + mockTableDescriptor.addFamily(new HColumnDescriptor(entry.getKey()) + .setMaxVersions(1) + .setCompressionType(entry.getValue()) + .setBlockCacheEnabled(false) + .setTimeToLive(0)); + } + Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor(); + } + + private void setupMockStartKeys(HTable table) throws IOException { + byte[][] mockKeys = new byte[][] { + HConstants.EMPTY_BYTE_ARRAY, + Bytes.toBytes("aaa"), + Bytes.toBytes("ggg"), + Bytes.toBytes("zzz") + }; + Mockito.doReturn(mockKeys).when(table).getStartKeys(); + } + + /** + * @return a map from column family names to compression algorithms for + * testing column family compression. Column family names have special characters + */ + private Map getMockColumnFamilies(int numCfs) { + Map familyToCompression = new HashMap(); + // use column family names having special characters + if (numCfs-- > 0) { + familyToCompression.put("Family1!@#!@#&", Compression.Algorithm.LZO); + } + if (numCfs-- > 0) { + familyToCompression.put("Family2=asdads&!AASD", Compression.Algorithm.SNAPPY); + } + if (numCfs-- > 0) { + familyToCompression.put("Family2=asdads&!AASD", Compression.Algorithm.GZ); + } + if (numCfs-- > 0) { + familyToCompression.put("Family3", Compression.Algorithm.NONE); + } + return familyToCompression; + } + + /** + * Test that {@link HFileOutputFormat} RecordWriter uses compression and + * bloom filter settings from the column family descriptor + */ + @Test + public void testColumnFamilySettings() throws Exception { + Configuration conf = new Configuration(this.util.getConfiguration()); + RecordWriter writer = null; + TaskAttemptContext context = null; + Path dir = util.getDataTestDir("testColumnFamilySettings"); + + // Setup table descriptor + HTable table = Mockito.mock(HTable.class); + HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); + Mockito.doReturn(htd).when(table).getTableDescriptor(); + for (HColumnDescriptor hcd: this.util.generateColumnDescriptors()) { + htd.addFamily(hcd); + } + + // set up the table to return some mock keys + setupMockStartKeys(table); + + try { + // partial map red setup to get an operational writer for testing + // We turn off the sequence file compression, because DefaultCodec + // pollutes the GZip codec pool with an incompatible compressor. + conf.set("io.seqfile.compression.type", "NONE"); + Job job = new Job(conf, "testLocalMRIncrementalLoad"); + setupRandomGeneratorMapper(job); + HFileOutputFormat.configureIncrementalLoad(job, table); + FileOutputFormat.setOutputPath(job, dir); + context = getTestTaskAttemptContext(job); + HFileOutputFormat hof = new HFileOutputFormat(); + writer = hof.getRecordWriter(context); + + // write out random rows + writeRandomKeyValues(writer, context, htd.getFamiliesKeys(), ROWSPERSPLIT); + writer.close(context); + + // Make sure that a directory was created for every CF + FileSystem fs = dir.getFileSystem(conf); + + // commit so that the filesystem has one directory per column family + hof.getOutputCommitter(context).commitTask(context); + hof.getOutputCommitter(context).commitJob(context); + FileStatus[] families = FSUtils.listStatus(fs, dir, new FSUtils.FamilyDirFilter(fs)); + assertEquals(htd.getFamilies().size(), families.length); + for (FileStatus f : families) { + String familyStr = f.getPath().getName(); + HColumnDescriptor hcd = htd.getFamily(Bytes.toBytes(familyStr)); + // verify that the compression on this file matches the configured + // compression + Path dataFilePath = fs.listStatus(f.getPath())[0].getPath(); + Reader reader = HFile.createReader(fs, dataFilePath, new CacheConfig(conf)); + Map fileInfo = reader.loadFileInfo(); + + byte[] bloomFilter = fileInfo.get(StoreFile.BLOOM_FILTER_TYPE_KEY); + if (bloomFilter == null) bloomFilter = Bytes.toBytes("NONE"); + assertEquals("Incorrect bloom filter used for column family " + familyStr + + "(reader: " + reader + ")", + hcd.getBloomFilterType(), StoreFile.BloomType.valueOf(Bytes.toString(bloomFilter))); + assertEquals("Incorrect compression used for column family " + familyStr + + "(reader: " + reader + ")", hcd.getCompression(), reader.getCompressionAlgorithm()); + } + } finally { + dir.getFileSystem(conf).delete(dir, true); + } + } + + /** + * Write random values to the writer assuming a table created using + * {@link #FAMILIES} as column family descriptors + */ + private void writeRandomKeyValues(RecordWriter writer, + TaskAttemptContext context, Set families, int numRows) + throws IOException, InterruptedException { + byte keyBytes[] = new byte[Bytes.SIZEOF_INT]; + int valLength = 10; + byte valBytes[] = new byte[valLength]; + + int taskId = context.getTaskAttemptID().getTaskID().getId(); + assert taskId < Byte.MAX_VALUE : "Unit tests dont support > 127 tasks!"; + + Random random = new Random(); + for (int i = 0; i < numRows; i++) { + + Bytes.putInt(keyBytes, 0, i); + random.nextBytes(valBytes); + ImmutableBytesWritable key = new ImmutableBytesWritable(keyBytes); + + for (byte[] family : families) { + KeyValue kv = new KeyValue(keyBytes, family, + PerformanceEvaluation.QUALIFIER_NAME, valBytes); + writer.write(key, kv); + } + } + } + + /** + * This test is to test the scenario happened in HBASE-6901. + * All files are bulk loaded and excluded from minor compaction. + * Without the fix of HBASE-6901, an ArrayIndexOutOfBoundsException + * will be thrown. + */ + @Test + public void testExcludeAllFromMinorCompaction() throws Exception { + Configuration conf = util.getConfiguration(); + conf.setInt("hbase.hstore.compaction.min", 2); + generateRandomStartKeys(5); + + try { + util.startMiniCluster(); + final FileSystem fs = util.getDFSCluster().getFileSystem(); + HBaseAdmin admin = new HBaseAdmin(conf); + HTable table = util.createTable(TABLE_NAME, FAMILIES); + assertEquals("Should start with empty table", 0, util.countRows(table)); + + // deep inspection: get the StoreFile dir + final Path storePath = Store.getStoreHomedir( + HTableDescriptor.getTableDir(FSUtils.getRootDir(conf), TABLE_NAME), + admin.getTableRegions(TABLE_NAME).get(0).getEncodedName(), + FAMILIES[0]); + assertEquals(0, fs.listStatus(storePath).length); + + // Generate two bulk load files + conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", + true); + util.startMiniMapReduceCluster(); + + for (int i = 0; i < 2; i++) { + Path testDir = util.getDataTestDir("testExcludeAllFromMinorCompaction_" + i); + runIncrementalPELoad(conf, table, testDir); + // Perform the actual load + new LoadIncrementalHFiles(conf).doBulkLoad(testDir, table); + } + + // Ensure data shows up + int expectedRows = 2 * NMapInputFormat.getNumMapTasks(conf) * ROWSPERSPLIT; + assertEquals("LoadIncrementalHFiles should put expected data in table", + expectedRows, util.countRows(table)); + + // should have a second StoreFile now + assertEquals(2, fs.listStatus(storePath).length); + + // minor compactions shouldn't get rid of the file + admin.compact(TABLE_NAME); + try { + quickPoll(new Callable() { + public Boolean call() throws Exception { + return fs.listStatus(storePath).length == 1; + } + }, 5000); + throw new IOException("SF# = " + fs.listStatus(storePath).length); + } catch (AssertionError ae) { + // this is expected behavior + } + + // a major compaction should work though + admin.majorCompact(TABLE_NAME); + quickPoll(new Callable() { + public Boolean call() throws Exception { + return fs.listStatus(storePath).length == 1; + } + }, 5000); + + } finally { + util.shutdownMiniMapReduceCluster(); + util.shutdownMiniCluster(); + } + } + + @Test + public void testExcludeMinorCompaction() throws Exception { + Configuration conf = util.getConfiguration(); + conf.setInt("hbase.hstore.compaction.min", 2); + Path testDir = util.getDataTestDir("testExcludeMinorCompaction"); + generateRandomStartKeys(5); + + try { + util.startMiniCluster(); + final FileSystem fs = util.getDFSCluster().getFileSystem(); + HBaseAdmin admin = new HBaseAdmin(conf); + HTable table = util.createTable(TABLE_NAME, FAMILIES); + assertEquals("Should start with empty table", 0, util.countRows(table)); + + // deep inspection: get the StoreFile dir + final Path storePath = Store.getStoreHomedir( + HTableDescriptor.getTableDir(FSUtils.getRootDir(conf), TABLE_NAME), + admin.getTableRegions(TABLE_NAME).get(0).getEncodedName(), + FAMILIES[0]); + assertEquals(0, fs.listStatus(storePath).length); + + // put some data in it and flush to create a storefile + Put p = new Put(Bytes.toBytes("test")); + p.add(FAMILIES[0], Bytes.toBytes("1"), Bytes.toBytes("1")); + table.put(p); + admin.flush(TABLE_NAME); + assertEquals(1, util.countRows(table)); + quickPoll(new Callable() { + public Boolean call() throws Exception { + return fs.listStatus(storePath).length == 1; + } + }, 5000); + + // Generate a bulk load file with more rows + conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", + true); + util.startMiniMapReduceCluster(); + runIncrementalPELoad(conf, table, testDir); + + // Perform the actual load + new LoadIncrementalHFiles(conf).doBulkLoad(testDir, table); + + // Ensure data shows up + int expectedRows = NMapInputFormat.getNumMapTasks(conf) * ROWSPERSPLIT; + assertEquals("LoadIncrementalHFiles should put expected data in table", + expectedRows + 1, util.countRows(table)); + + // should have a second StoreFile now + assertEquals(2, fs.listStatus(storePath).length); + + // minor compactions shouldn't get rid of the file + admin.compact(TABLE_NAME); + try { + quickPoll(new Callable() { + public Boolean call() throws Exception { + return fs.listStatus(storePath).length == 1; + } + }, 5000); + throw new IOException("SF# = " + fs.listStatus(storePath).length); + } catch (AssertionError ae) { + // this is expected behavior + } + + // a major compaction should work though + admin.majorCompact(TABLE_NAME); + quickPoll(new Callable() { + public Boolean call() throws Exception { + return fs.listStatus(storePath).length == 1; + } + }, 5000); + + } finally { + util.shutdownMiniMapReduceCluster(); + util.shutdownMiniCluster(); + } + } + + private void quickPoll(Callable c, int waitMs) throws Exception { + int sleepMs = 10; + int retries = (int) Math.ceil(((double) waitMs) / sleepMs); + while (retries-- > 0) { + if (c.call().booleanValue()) { + return; + } + Thread.sleep(sleepMs); + } + fail(); + } + + public static void main(String args[]) throws Exception { + new TestHFileOutputFormat().manualTest(args); + } + + public void manualTest(String args[]) throws Exception { + Configuration conf = HBaseConfiguration.create(); + util = new HBaseTestingUtility(conf); + if ("newtable".equals(args[0])) { + byte[] tname = args[1].getBytes(); + HTable table = util.createTable(tname, FAMILIES); + HBaseAdmin admin = new HBaseAdmin(conf); + admin.disableTable(tname); + byte[][] startKeys = generateRandomStartKeys(5); + util.createMultiRegions(conf, table, FAMILIES[0], startKeys); + admin.enableTable(tname); + } else if ("incremental".equals(args[0])) { + byte[] tname = args[1].getBytes(); + HTable table = new HTable(conf, tname); + Path outDir = new Path("incremental-out"); + runIncrementalPELoad(conf, table, outDir); + } else { + throw new RuntimeException( + "usage: TestHFileOutputFormat newtable | incremental"); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java new file mode 100644 index 0000000..f91187b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java @@ -0,0 +1,238 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.mapreduce.HLogInputFormat.HLogRecordReader; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.MapReduceTestUtil; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * JUnit tests for the HLogRecordReader + */ +@Category(MediumTests.class) +public class TestHLogRecordReader { + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static Configuration conf; + private static FileSystem fs; + private static Path hbaseDir; + private static final byte [] tableName = Bytes.toBytes(getName()); + private static final byte [] rowName = tableName; + private static final HRegionInfo info = new HRegionInfo(tableName, + Bytes.toBytes(""), Bytes.toBytes(""), false); + private static final byte [] family = Bytes.toBytes("column"); + private static final byte [] value = Bytes.toBytes("value"); + private static HTableDescriptor htd; + private static Path logDir; + private static Path oldLogDir; + + private static String getName() { + return "TestHLogRecordReader"; + } + + @Before + public void setUp() throws Exception { + FileStatus[] entries = fs.listStatus(hbaseDir); + for (FileStatus dir : entries) { + fs.delete(dir.getPath(), true); + } + + } + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Make block sizes small. + conf = TEST_UTIL.getConfiguration(); + conf.setInt("dfs.blocksize", 1024 * 1024); + conf.setInt("dfs.replication", 1); + TEST_UTIL.startMiniDFSCluster(1); + + conf = TEST_UTIL.getConfiguration(); + fs = TEST_UTIL.getDFSCluster().getFileSystem(); + + hbaseDir = TEST_UTIL.createRootDir(); + logDir = new Path(hbaseDir, HConstants.HREGION_LOGDIR_NAME); + oldLogDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME); + htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(family)); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Test partial reads from the log based on passed time range + * @throws Exception + */ + @Test + public void testPartialRead() throws Exception { + HLog log = new HLog(fs, logDir, oldLogDir, conf); + long ts = System.currentTimeMillis(); + WALEdit edit = new WALEdit(); + edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"), + ts, value)); + log.append(info, tableName, edit, + ts, htd); + edit = new WALEdit(); + edit.add(new KeyValue(rowName, family, Bytes.toBytes("2"), + ts+1, value)); + log.append(info, tableName, edit, + ts+1, htd); + log.rollWriter(); + + Thread.sleep(1); + long ts1 = System.currentTimeMillis(); + + edit = new WALEdit(); + edit.add(new KeyValue(rowName, family, Bytes.toBytes("3"), + ts1+1, value)); + log.append(info, tableName, edit, + ts1+1, htd); + edit = new WALEdit(); + edit.add(new KeyValue(rowName, family, Bytes.toBytes("4"), + ts1+2, value)); + log.append(info, tableName, edit, + ts1+2, htd); + log.close(); + + HLogInputFormat input = new HLogInputFormat(); + Configuration jobConf = new Configuration(conf); + jobConf.set("mapred.input.dir", logDir.toString()); + jobConf.setLong(HLogInputFormat.END_TIME_KEY, ts); + + // only 1st file is considered, and only its 1st entry is used + List splits = input.getSplits(MapreduceTestingShim.createJobContext(jobConf)); + assertEquals(1, splits.size()); + testSplit(splits.get(0), Bytes.toBytes("1")); + + jobConf.setLong(HLogInputFormat.START_TIME_KEY, ts+1); + jobConf.setLong(HLogInputFormat.END_TIME_KEY, ts1+1); + splits = input.getSplits(MapreduceTestingShim.createJobContext(jobConf)); + // both files need to be considered + assertEquals(2, splits.size()); + // only the 2nd entry from the 1st file is used + testSplit(splits.get(0), Bytes.toBytes("2")); + // only the 1nd entry from the 2nd file is used + testSplit(splits.get(1), Bytes.toBytes("3")); + } + + /** + * Test basic functionality + * @throws Exception + */ + @Test + public void testHLogRecordReader() throws Exception { + HLog log = new HLog(fs, logDir, oldLogDir, conf); + byte [] value = Bytes.toBytes("value"); + WALEdit edit = new WALEdit(); + edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"), + System.currentTimeMillis(), value)); + log.append(info, tableName, edit, + System.currentTimeMillis(), htd); + + Thread.sleep(1); // make sure 2nd log gets a later timestamp + long secondTs = System.currentTimeMillis(); + log.rollWriter(); + + edit = new WALEdit(); + edit.add(new KeyValue(rowName, family, Bytes.toBytes("2"), + System.currentTimeMillis(), value)); + log.append(info, tableName, edit, + System.currentTimeMillis(), htd); + log.close(); + long thirdTs = System.currentTimeMillis(); + + // should have 2 log files now + HLogInputFormat input = new HLogInputFormat(); + Configuration jobConf = new Configuration(conf); + jobConf.set("mapred.input.dir", logDir.toString()); + + // make sure both logs are found + List splits = input.getSplits(MapreduceTestingShim.createJobContext(jobConf)); + assertEquals(2, splits.size()); + + // should return exactly one KV + testSplit(splits.get(0), Bytes.toBytes("1")); + // same for the 2nd split + testSplit(splits.get(1), Bytes.toBytes("2")); + + // now test basic time ranges: + + // set an endtime, the 2nd log file can be ignored completely. + jobConf.setLong(HLogInputFormat.END_TIME_KEY, secondTs-1); + splits = input.getSplits(MapreduceTestingShim.createJobContext(jobConf)); + assertEquals(1, splits.size()); + testSplit(splits.get(0), Bytes.toBytes("1")); + + // now set a start time + jobConf.setLong(HLogInputFormat.END_TIME_KEY, Long.MAX_VALUE); + jobConf.setLong(HLogInputFormat.START_TIME_KEY, thirdTs); + splits = input.getSplits(MapreduceTestingShim.createJobContext(jobConf)); + // both logs need to be considered + assertEquals(2, splits.size()); + // but both readers skip all edits + testSplit(splits.get(0)); + testSplit(splits.get(1)); + } + + /** + * Create a new reader from the split, and match the edits against the passed columns. + */ + private void testSplit(InputSplit split, byte[]... columns) throws Exception { + HLogRecordReader reader = new HLogRecordReader(); + reader.initialize(split, MapReduceTestUtil.createDummyMapTaskAttemptContext(conf)); + + for (byte[] column : columns) { + assertTrue(reader.nextKeyValue()); + assertTrue(Bytes + .equals(column, reader.getCurrentValue().getKeyValues().get(0).getQualifier())); + } + assertFalse(reader.nextKeyValue()); + reader.close(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java new file mode 100644 index 0000000..c0cd2f1 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java @@ -0,0 +1,346 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.util.GenericOptionsParser; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestImportExport { + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final byte[] ROW1 = Bytes.toBytes("row1"); + private static final byte[] ROW2 = Bytes.toBytes("row2"); + private static final String FAMILYA_STRING = "a"; + private static final String FAMILYB_STRING = "b"; + private static final byte[] FAMILYA = Bytes.toBytes(FAMILYA_STRING); + private static final byte[] FAMILYB = Bytes.toBytes(FAMILYB_STRING); + private static final byte[] QUAL = Bytes.toBytes("q"); + private static final String OUTPUT_DIR = "outputdir"; + + private static MiniHBaseCluster cluster; + private static long now = System.currentTimeMillis(); + + @BeforeClass + public static void beforeClass() throws Exception { + cluster = UTIL.startMiniCluster(); + UTIL.startMiniMapReduceCluster(); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniMapReduceCluster(); + UTIL.shutdownMiniCluster(); + } + + @Before + @After + public void cleanup() throws Exception { + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + fs.delete(new Path(OUTPUT_DIR), true); + } + + /** + * Test simple replication case with column mapping + * @throws Exception + */ + @Test + public void testSimpleCase() throws Exception { + String EXPORT_TABLE = "exportSimpleCase"; + HTable t = UTIL.createTable(Bytes.toBytes(EXPORT_TABLE), FAMILYA); + Put p = new Put(ROW1); + p.add(FAMILYA, QUAL, now, QUAL); + p.add(FAMILYA, QUAL, now+1, QUAL); + p.add(FAMILYA, QUAL, now+2, QUAL); + t.put(p); + p = new Put(ROW2); + p.add(FAMILYA, QUAL, now, QUAL); + p.add(FAMILYA, QUAL, now+1, QUAL); + p.add(FAMILYA, QUAL, now+2, QUAL); + t.put(p); + + String[] args = new String[] { + EXPORT_TABLE, + OUTPUT_DIR, + "1000" + }; + + GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args); + Configuration conf = opts.getConfiguration(); + args = opts.getRemainingArgs(); + + Job job = Export.createSubmittableJob(conf, args); + job.getConfiguration().set("mapreduce.framework.name", "yarn"); + job.waitForCompletion(false); + assertTrue(job.isSuccessful()); + + + String IMPORT_TABLE = "importTableSimpleCase"; + t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB); + args = new String[] { + "-D" + Import.CF_RENAME_PROP + "="+FAMILYA_STRING+":"+FAMILYB_STRING, + IMPORT_TABLE, + OUTPUT_DIR + }; + + opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args); + conf = opts.getConfiguration(); + args = opts.getRemainingArgs(); + + job = Import.createSubmittableJob(conf, args); + job.getConfiguration().set("mapreduce.framework.name", "yarn"); + job.waitForCompletion(false); + assertTrue(job.isSuccessful()); + + Get g = new Get(ROW1); + g.setMaxVersions(); + Result r = t.get(g); + assertEquals(3, r.size()); + g = new Get(ROW2); + g.setMaxVersions(); + r = t.get(g); + assertEquals(3, r.size()); + } + + /** + * Test export .META. table + * + * @throws Exception + */ + @Test + public void testMetaExport() throws Exception { + String EXPORT_TABLE = ".META."; + String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1", "0", "0" }; + GenericOptionsParser opts = new GenericOptionsParser(new Configuration( + cluster.getConfiguration()), args); + Configuration conf = opts.getConfiguration(); + args = opts.getRemainingArgs(); + + Job job = Export.createSubmittableJob(conf, args); + job.getConfiguration().set("mapreduce.framework.name", "yarn"); + job.waitForCompletion(false); + assertTrue(job.isSuccessful()); + } + + @Test + public void testWithDeletes() throws Exception { + String EXPORT_TABLE = "exportWithDeletes"; + HTableDescriptor desc = new HTableDescriptor(EXPORT_TABLE); + desc.addFamily(new HColumnDescriptor(FAMILYA) + .setMaxVersions(5) + .setKeepDeletedCells(true) + ); + UTIL.getHBaseAdmin().createTable(desc); + HTable t = new HTable(UTIL.getConfiguration(), EXPORT_TABLE); + + Put p = new Put(ROW1); + p.add(FAMILYA, QUAL, now, QUAL); + p.add(FAMILYA, QUAL, now+1, QUAL); + p.add(FAMILYA, QUAL, now+2, QUAL); + p.add(FAMILYA, QUAL, now+3, QUAL); + p.add(FAMILYA, QUAL, now+4, QUAL); + t.put(p); + + Delete d = new Delete(ROW1, now+3, null); + t.delete(d); + d = new Delete(ROW1); + d.deleteColumns(FAMILYA, QUAL, now+2); + t.delete(d); + + String[] args = new String[] { + "-D" + Export.RAW_SCAN + "=true", + EXPORT_TABLE, + OUTPUT_DIR, + "1000" + }; + + GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args); + Configuration conf = opts.getConfiguration(); + args = opts.getRemainingArgs(); + + Job job = Export.createSubmittableJob(conf, args); + job.getConfiguration().set("mapreduce.framework.name", "yarn"); + job.waitForCompletion(false); + assertTrue(job.isSuccessful()); + + + String IMPORT_TABLE = "importWithDeletes"; + desc = new HTableDescriptor(IMPORT_TABLE); + desc.addFamily(new HColumnDescriptor(FAMILYA) + .setMaxVersions(5) + .setKeepDeletedCells(true) + ); + UTIL.getHBaseAdmin().createTable(desc); + t.close(); + t = new HTable(UTIL.getConfiguration(), IMPORT_TABLE); + args = new String[] { + IMPORT_TABLE, + OUTPUT_DIR + }; + + opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args); + conf = opts.getConfiguration(); + args = opts.getRemainingArgs(); + + job = Import.createSubmittableJob(conf, args); + job.getConfiguration().set("mapreduce.framework.name", "yarn"); + job.waitForCompletion(false); + assertTrue(job.isSuccessful()); + + Scan s = new Scan(); + s.setMaxVersions(); + s.setRaw(true); + ResultScanner scanner = t.getScanner(s); + Result r = scanner.next(); + KeyValue[] res = r.raw(); + assertTrue(res[0].isDeleteFamily()); + assertEquals(now+4, res[1].getTimestamp()); + assertEquals(now+3, res[2].getTimestamp()); + assertTrue(res[3].isDelete()); + assertEquals(now+2, res[4].getTimestamp()); + assertEquals(now+1, res[5].getTimestamp()); + assertEquals(now, res[6].getTimestamp()); + t.close(); + } + + @Test + public void testWithFilter() throws Exception { + String EXPORT_TABLE = "exportSimpleCase_ImportWithFilter"; + HTableDescriptor desc = new HTableDescriptor(EXPORT_TABLE); + desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5)); + UTIL.getHBaseAdmin().createTable(desc); + HTable exportTable = new HTable(UTIL.getConfiguration(), EXPORT_TABLE); + + Put p = new Put(ROW1); + p.add(FAMILYA, QUAL, now, QUAL); + p.add(FAMILYA, QUAL, now + 1, QUAL); + p.add(FAMILYA, QUAL, now + 2, QUAL); + p.add(FAMILYA, QUAL, now + 3, QUAL); + p.add(FAMILYA, QUAL, now + 4, QUAL); + exportTable.put(p); + + String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, "1000" }; + + GenericOptionsParser opts = new GenericOptionsParser(new Configuration( + cluster.getConfiguration()), args); + Configuration conf = opts.getConfiguration(); + args = opts.getRemainingArgs(); + + Job job = Export.createSubmittableJob(conf, args); + job.getConfiguration().set("mapreduce.framework.name", "yarn"); + job.waitForCompletion(false); + assertTrue(job.isSuccessful()); + + String IMPORT_TABLE = "importWithFilter"; + desc = new HTableDescriptor(IMPORT_TABLE); + desc.addFamily(new HColumnDescriptor(FAMILYA).setMaxVersions(5)); + UTIL.getHBaseAdmin().createTable(desc); + + HTable importTable = new HTable(UTIL.getConfiguration(), IMPORT_TABLE); + args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + PrefixFilter.class.getName(), + "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, OUTPUT_DIR, + "1000" }; + + opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args); + conf = opts.getConfiguration(); + args = opts.getRemainingArgs(); + + job = Import.createSubmittableJob(conf, args); + job.getConfiguration().set("mapreduce.framework.name", "yarn"); + job.waitForCompletion(false); + assertTrue(job.isSuccessful()); + + // get the count of the source table for that time range + PrefixFilter filter = new PrefixFilter(ROW1); + int count = getCount(exportTable, filter); + + Assert.assertEquals("Unexpected row count between export and import tables", count, + getCount(importTable, null)); + + // and then test that a broken command doesn't bork everything - easier here because we don't + // need to re-run the export job + + args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + Filter.class.getName(), + "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", EXPORT_TABLE, + OUTPUT_DIR, "1000" }; + + opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args); + conf = opts.getConfiguration(); + args = opts.getRemainingArgs(); + + job = Import.createSubmittableJob(conf, args); + job.getConfiguration().set("mapreduce.framework.name", "yarn"); + job.waitForCompletion(false); + assertFalse("Job succeeedd, but it had a non-instantiable filter!", job.isSuccessful()); + + // cleanup + exportTable.close(); + importTable.close(); + } + + /** + * Count the number of keyvalues in the specified table for the given timerange + * @param start + * @param end + * @param table + * @return + * @throws IOException + */ + private int getCount(HTable table, Filter filter) throws IOException { + Scan scan = new Scan(); + scan.setFilter(filter); + ResultScanner results = table.getScanner(scan); + int count = 0; + for (Result res : results) { + count += res.size(); + } + results.close(); + return count; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java new file mode 100644 index 0000000..77d044b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java @@ -0,0 +1,362 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.ArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.GenericOptionsParser; + +import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser; +import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser.BadTsvLineException; +import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser.ParsedLine; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Result; + +import org.junit.Test; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +@Category(MediumTests.class) +public class TestImportTsv { + private static final Log LOG = LogFactory.getLog(TestImportTsv.class); + + @Test + public void testTsvParserSpecParsing() { + TsvParser parser; + + parser = new TsvParser("HBASE_ROW_KEY", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertFalse(parser.hasTimestamp()); + + parser = new TsvParser("HBASE_ROW_KEY,col1:scol1", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1)); + assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertFalse(parser.hasTimestamp()); + + parser = new TsvParser("HBASE_ROW_KEY,col1:scol1,col1:scol2", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1)); + assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(2)); + assertBytesEquals(Bytes.toBytes("scol2"), parser.getQualifier(2)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertFalse(parser.hasTimestamp()); + + parser = new TsvParser("HBASE_ROW_KEY,col1:scol1,HBASE_TS_KEY,col1:scol2", + "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1)); + assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1)); + assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(3)); + assertBytesEquals(Bytes.toBytes("scol2"), parser.getQualifier(3)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertTrue(parser.hasTimestamp()); + assertEquals(2, parser.getTimestampKeyColumnIndex()); + } + + @Test + public void testTsvParser() throws BadTsvLineException { + TsvParser parser = new TsvParser("col_a,col_b:qual,HBASE_ROW_KEY,col_d", "\t"); + assertBytesEquals(Bytes.toBytes("col_a"), parser.getFamily(0)); + assertBytesEquals(HConstants.EMPTY_BYTE_ARRAY, parser.getQualifier(0)); + assertBytesEquals(Bytes.toBytes("col_b"), parser.getFamily(1)); + assertBytesEquals(Bytes.toBytes("qual"), parser.getQualifier(1)); + assertNull(parser.getFamily(2)); + assertNull(parser.getQualifier(2)); + assertEquals(2, parser.getRowKeyColumnIndex()); + + assertEquals(TsvParser.DEFAULT_TIMESTAMP_COLUMN_INDEX, parser + .getTimestampKeyColumnIndex()); + + byte[] line = Bytes.toBytes("val_a\tval_b\tval_c\tval_d"); + ParsedLine parsed = parser.parse(line, line.length); + checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line))); + } + + + @Test + public void testTsvParserWithTimestamp() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,HBASE_TS_KEY,col_a,", "\t"); + assertNull(parser.getFamily(0)); + assertNull(parser.getQualifier(0)); + assertNull(parser.getFamily(1)); + assertNull(parser.getQualifier(1)); + assertBytesEquals(Bytes.toBytes("col_a"), parser.getFamily(2)); + assertBytesEquals(HConstants.EMPTY_BYTE_ARRAY, parser.getQualifier(2)); + assertEquals(0, parser.getRowKeyColumnIndex()); + assertEquals(1, parser.getTimestampKeyColumnIndex()); + + byte[] line = Bytes.toBytes("rowkey\t1234\tval_a"); + ParsedLine parsed = parser.parse(line, line.length); + assertEquals(1234l, parsed.getTimestamp(-1)); + checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line))); + } + + private void checkParsing(ParsedLine parsed, Iterable expected) { + ArrayList parsedCols = new ArrayList(); + for (int i = 0; i < parsed.getColumnCount(); i++) { + parsedCols.add(Bytes.toString( + parsed.getLineBytes(), + parsed.getColumnOffset(i), + parsed.getColumnLength(i))); + } + if (!Iterables.elementsEqual(parsedCols, expected)) { + fail("Expected: " + Joiner.on(",").join(expected) + "\n" + + "Got:" + Joiner.on(",").join(parsedCols)); + } + } + + private void assertBytesEquals(byte[] a, byte[] b) { + assertEquals(Bytes.toStringBinary(a), Bytes.toStringBinary(b)); + } + + /** + * Test cases that throw BadTsvLineException + */ + @Test(expected=BadTsvLineException.class) + public void testTsvParserBadTsvLineExcessiveColumns() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t"); + byte[] line = Bytes.toBytes("val_a\tval_b\tval_c"); + parser.parse(line, line.length); + } + + @Test(expected=BadTsvLineException.class) + public void testTsvParserBadTsvLineZeroColumn() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t"); + byte[] line = Bytes.toBytes(""); + parser.parse(line, line.length); + } + + @Test(expected=BadTsvLineException.class) + public void testTsvParserBadTsvLineOnlyKey() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t"); + byte[] line = Bytes.toBytes("key_only"); + parser.parse(line, line.length); + } + + @Test(expected=BadTsvLineException.class) + public void testTsvParserBadTsvLineNoRowKey() throws BadTsvLineException { + TsvParser parser = new TsvParser("col_a,HBASE_ROW_KEY", "\t"); + byte[] line = Bytes.toBytes("only_cola_data_and_no_row_key"); + parser.parse(line, line.length); + } + + @Test(expected = BadTsvLineException.class) + public void testTsvParserInvalidTimestamp() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,HBASE_TS_KEY,col_a,", "\t"); + assertEquals(1, parser.getTimestampKeyColumnIndex()); + byte[] line = Bytes.toBytes("rowkey\ttimestamp\tval_a"); + ParsedLine parsed = parser.parse(line, line.length); + assertEquals(-1, parsed.getTimestamp(-1)); + checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line))); + } + + @Test(expected = BadTsvLineException.class) + public void testTsvParserNoTimestampValue() throws BadTsvLineException { + TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a,HBASE_TS_KEY", "\t"); + assertEquals(2, parser.getTimestampKeyColumnIndex()); + byte[] line = Bytes.toBytes("rowkey\tval_a"); + parser.parse(line, line.length); + } + + + @Test + public void testMROnTable() + throws Exception { + String TABLE_NAME = "TestTable"; + String FAMILY = "FAM"; + String INPUT_FILE = "InputFile.esv"; + + // Prepare the arguments required for the test. + String[] args = new String[] { + "-D" + ImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,FAM:A,FAM:B", + "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=\u001b", + TABLE_NAME, + INPUT_FILE + }; + + doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, null, args, 1); + } + + @Test + public void testMROnTableWithTimestamp() throws Exception { + String TABLE_NAME = "TestTable"; + String FAMILY = "FAM"; + String INPUT_FILE = "InputFile1.csv"; + + // Prepare the arguments required for the test. + String[] args = new String[] { + "-D" + ImportTsv.COLUMNS_CONF_KEY + + "=HBASE_ROW_KEY,HBASE_TS_KEY,FAM:A,FAM:B", + "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=,", TABLE_NAME, INPUT_FILE }; + + String data = "KEY,1234,VALUE1,VALUE2\n"; + doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, data, args, 1); + } + + + @Test + public void testMROnTableWithCustomMapper() + throws Exception { + String TABLE_NAME = "TestTable"; + String FAMILY = "FAM"; + String INPUT_FILE = "InputFile2.esv"; + + // Prepare the arguments required for the test. + String[] args = new String[] { + "-D" + ImportTsv.MAPPER_CONF_KEY + "=org.apache.hadoop.hbase.mapreduce.TsvImporterCustomTestMapper", + TABLE_NAME, + INPUT_FILE + }; + + doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, null, args, 3); + } + + private void doMROnTableTest(String inputFile, String family, String tableName, + String data, String[] args, int valueMultiplier) throws Exception { + + // Cluster + HBaseTestingUtility htu1 = new HBaseTestingUtility(); + + htu1.startMiniCluster(); + htu1.startMiniMapReduceCluster(); + + GenericOptionsParser opts = new GenericOptionsParser(htu1.getConfiguration(), args); + Configuration conf = opts.getConfiguration(); + args = opts.getRemainingArgs(); + + try { + FileSystem fs = FileSystem.get(conf); + FSDataOutputStream op = fs.create(new Path(inputFile), true); + if (data == null) { + data = "KEY\u001bVALUE1\u001bVALUE2\n"; + } + op.write(Bytes.toBytes(data)); + op.close(); + + final byte[] FAM = Bytes.toBytes(family); + final byte[] TAB = Bytes.toBytes(tableName); + if (conf.get(ImportTsv.BULK_OUTPUT_CONF_KEY) == null) { + HTableDescriptor desc = new HTableDescriptor(TAB); + desc.addFamily(new HColumnDescriptor(FAM)); + HBaseAdmin admin = new HBaseAdmin(conf); + admin.createTable(desc); + admin.close(); + } else { // set the hbaseAdmin as we are not going through main() + LOG.info("set the hbaseAdmin"); + ImportTsv.createHbaseAdmin(conf); + } + Job job = ImportTsv.createSubmittableJob(conf, args); + job.waitForCompletion(false); + assertTrue(job.isSuccessful()); + + HTable table = new HTable(new Configuration(conf), TAB); + boolean verified = false; + long pause = conf.getLong("hbase.client.pause", 5 * 1000); + int numRetries = conf.getInt("hbase.client.retries.number", 5); + for (int i = 0; i < numRetries; i++) { + try { + Scan scan = new Scan(); + // Scan entire family. + scan.addFamily(FAM); + ResultScanner resScanner = table.getScanner(scan); + for (Result res : resScanner) { + assertTrue(res.size() == 2); + List kvs = res.list(); + assertEquals(toU8Str(kvs.get(0).getRow()), + toU8Str(Bytes.toBytes("KEY"))); + assertEquals(toU8Str(kvs.get(1).getRow()), + toU8Str(Bytes.toBytes("KEY"))); + assertEquals(toU8Str(kvs.get(0).getValue()), + toU8Str(Bytes.toBytes("VALUE" + valueMultiplier))); + assertEquals(toU8Str(kvs.get(1).getValue()), + toU8Str(Bytes.toBytes("VALUE" + 2*valueMultiplier))); + // Only one result set is expected, so let it loop. + } + verified = true; + break; + } catch (NullPointerException e) { + // If here, a cell was empty. Presume its because updates came in + // after the scanner had been opened. Wait a while and retry. + } + try { + Thread.sleep(pause); + } catch (InterruptedException e) { + // continue + } + } + table.close(); + assertTrue(verified); + } finally { + htu1.shutdownMiniMapReduceCluster(); + htu1.shutdownMiniCluster(); + } + } + + @Test + public void testBulkOutputWithoutAnExistingTable() throws Exception { + String TABLE_NAME = "TestTable"; + String FAMILY = "FAM"; + String INPUT_FILE = "InputFile2.esv"; + + // Prepare the arguments required for the test. + String[] args = new String[] { + "-D" + ImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,FAM:A,FAM:B", + "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=\u001b", + "-D" + ImportTsv.BULK_OUTPUT_CONF_KEY + "=output", TABLE_NAME, + INPUT_FILE }; + doMROnTableTest(INPUT_FILE, FAMILY, TABLE_NAME, null, args, 3); + } + + public static String toU8Str(byte[] bytes) throws UnsupportedEncodingException { + return new String(bytes); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java new file mode 100644 index 0000000..05abd3a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java @@ -0,0 +1,342 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.TreeMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; +import org.junit.experimental.categories.Category; + +/** + * Test cases for the "load" half of the HFileOutputFormat bulk load + * functionality. These tests run faster than the full MR cluster + * tests in TestHFileOutputFormat + */ +@Category(LargeTests.class) +public class TestLoadIncrementalHFiles { + private static final byte[] QUALIFIER = Bytes.toBytes("myqual"); + private static final byte[] FAMILY = Bytes.toBytes("myfam"); + + private static final byte[][] SPLIT_KEYS = new byte[][] { + Bytes.toBytes("ddd"), + Bytes.toBytes("ppp") + }; + + public static int BLOCKSIZE = 64*1024; + public static String COMPRESSION = + Compression.Algorithm.NONE.getName(); + + static HBaseTestingUtility util = new HBaseTestingUtility(); + //used by secure subclass + static boolean useSecure = false; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + util.startMiniCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + util.shutdownMiniCluster(); + } + + /** + * Test case that creates some regions and loads + * HFiles that fit snugly inside those regions + */ + @Test + public void testSimpleLoad() throws Exception { + runTest("testSimpleLoad", BloomType.NONE, + new byte[][][] { + new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("cccc") }, + new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") }, + }); + } + + /** + * Test case that creates some regions and loads + * HFiles that cross the boundaries of those regions + */ + @Test + public void testRegionCrossingLoad() throws Exception { + runTest("testRegionCrossingLoad", BloomType.NONE, + new byte[][][] { + new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") }, + new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") }, + }); + } + + /** + * Test loading into a column family that has a ROW bloom filter. + */ + @Test + public void testRegionCrossingRowBloom() throws Exception { + runTest("testRegionCrossingLoadRowBloom", BloomType.ROW, + new byte[][][] { + new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") }, + new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") }, + }); + } + + /** + * Test loading into a column family that has a ROWCOL bloom filter. + */ + @Test + public void testRegionCrossingRowColBloom() throws Exception { + runTest("testRegionCrossingLoadRowColBloom", BloomType.ROWCOL, + new byte[][][] { + new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") }, + new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") }, + }); + } + + private void runTest(String testName, BloomType bloomType, + byte[][][] hfileRanges) throws Exception { + Path dir = util.getDataTestDir(testName); + FileSystem fs = util.getTestFileSystem(); + dir = dir.makeQualified(fs); + Path familyDir = new Path(dir, Bytes.toString(FAMILY)); + + int hfileIdx = 0; + for (byte[][] range : hfileRanges) { + byte[] from = range[0]; + byte[] to = range[1]; + createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_" + + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000); + } + int expectedRows = hfileIdx * 1000; + + final byte[] TABLE = Bytes.toBytes("mytable_"+testName); + + HBaseAdmin admin = new HBaseAdmin(util.getConfiguration()); + HTableDescriptor htd = new HTableDescriptor(TABLE); + HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY); + familyDesc.setBloomFilterType(bloomType); + htd.addFamily(familyDesc); + admin.createTable(htd, SPLIT_KEYS); + + HTable table = new HTable(util.getConfiguration(), TABLE); + util.waitTableAvailable(TABLE, 30000); + LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration(), useSecure); + loader.doBulkLoad(dir, table); + + assertEquals(expectedRows, util.countRows(table)); + } + + /** + * Test loading into a column family that does not exist. + */ + @Test + public void testNonexistentColumnFamilyLoad() throws Exception { + String testName = "testNonexistentColumnFamilyLoad"; + byte[][][] hfileRanges = new byte[][][] { + new byte[][]{ Bytes.toBytes("aaa"), Bytes.toBytes("ccc") }, + new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") }, + }; + + Path dir = util.getDataTestDir(testName); + FileSystem fs = util.getTestFileSystem(); + dir = dir.makeQualified(fs); + Path familyDir = new Path(dir, Bytes.toString(FAMILY)); + + int hfileIdx = 0; + for (byte[][] range : hfileRanges) { + byte[] from = range[0]; + byte[] to = range[1]; + createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_" + + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000); + } + + final byte[] TABLE = Bytes.toBytes("mytable_"+testName); + + HBaseAdmin admin = new HBaseAdmin(util.getConfiguration()); + HTableDescriptor htd = new HTableDescriptor(TABLE); + admin.createTable(htd, SPLIT_KEYS); + + HTable table = new HTable(util.getConfiguration(), TABLE); + util.waitTableAvailable(TABLE, 30000); + LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration(), false); + try { + loader.doBulkLoad(dir, table); + assertTrue("Loading into table with non-existent family should have failed", false); + } catch (Exception e) { + assertTrue("IOException expected", e instanceof IOException); + } + table.close(); + admin.close(); + } + + @Test + public void testSplitStoreFile() throws IOException { + Path dir = util.getDataTestDir("testSplitHFile"); + FileSystem fs = util.getTestFileSystem(); + Path testIn = new Path(dir, "testhfile"); + HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY); + createHFile(util.getConfiguration(), fs, testIn, FAMILY, QUALIFIER, + Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000); + + Path bottomOut = new Path(dir, "bottom.out"); + Path topOut = new Path(dir, "top.out"); + + LoadIncrementalHFiles.splitStoreFile( + util.getConfiguration(), testIn, + familyDesc, Bytes.toBytes("ggg"), + bottomOut, + topOut); + + int rowCount = verifyHFile(bottomOut); + rowCount += verifyHFile(topOut); + assertEquals(1000, rowCount); + } + + private int verifyHFile(Path p) throws IOException { + Configuration conf = util.getConfiguration(); + HFile.Reader reader = HFile.createReader( + p.getFileSystem(conf), p, new CacheConfig(conf)); + reader.loadFileInfo(); + HFileScanner scanner = reader.getScanner(false, false); + scanner.seekTo(); + int count = 0; + do { + count++; + } while (scanner.next()); + assertTrue(count > 0); + reader.close(); + return count; + } + + + /** + * Create an HFile with the given number of rows between a given + * start key and end key. + * TODO put me in an HFileTestUtil or something? + */ + static void createHFile( + Configuration conf, + FileSystem fs, Path path, + byte[] family, byte[] qualifier, + byte[] startKey, byte[] endKey, int numRows) throws IOException + { + HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)) + .withPath(fs, path) + .withBlockSize(BLOCKSIZE) + .withCompression(COMPRESSION) + .withComparator(KeyValue.KEY_COMPARATOR) + .create(); + long now = System.currentTimeMillis(); + try { + // subtract 2 since iterateOnSplits doesn't include boundary keys + for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows-2)) { + KeyValue kv = new KeyValue(key, family, qualifier, now, key); + writer.append(kv); + } + } finally { + writer.close(); + } + } + + private void addStartEndKeysForTest(TreeMap map, byte[] first, byte[] last) { + Integer value = map.containsKey(first)?(Integer)map.get(first):0; + map.put(first, value+1); + + value = map.containsKey(last)?(Integer)map.get(last):0; + map.put(last, value-1); + } + + @Test + public void testInferBoundaries() { + TreeMap map = new TreeMap(Bytes.BYTES_COMPARATOR); + + /* Toy example + * c---------i o------p s---------t v------x + * a------e g-----k m-------------q r----s u----w + * + * Should be inferred as: + * a-----------------k m-------------q r--------------t u---------x + * + * The output should be (m,r,u) + */ + + String first; + String last; + + first = "a"; last = "e"; + addStartEndKeysForTest(map, first.getBytes(), last.getBytes()); + + first = "r"; last = "s"; + addStartEndKeysForTest(map, first.getBytes(), last.getBytes()); + + first = "o"; last = "p"; + addStartEndKeysForTest(map, first.getBytes(), last.getBytes()); + + first = "g"; last = "k"; + addStartEndKeysForTest(map, first.getBytes(), last.getBytes()); + + first = "v"; last = "x"; + addStartEndKeysForTest(map, first.getBytes(), last.getBytes()); + + first = "c"; last = "i"; + addStartEndKeysForTest(map, first.getBytes(), last.getBytes()); + + first = "m"; last = "q"; + addStartEndKeysForTest(map, first.getBytes(), last.getBytes()); + + first = "s"; last = "t"; + addStartEndKeysForTest(map, first.getBytes(), last.getBytes()); + + first = "u"; last = "w"; + addStartEndKeysForTest(map, first.getBytes(), last.getBytes()); + + byte[][] keysArray = LoadIncrementalHFiles.inferBoundaries(map); + byte[][] compare = new byte[3][]; + compare[0] = "m".getBytes(); + compare[1] = "r".getBytes(); + compare[2] = "u".getBytes(); + + assertEquals(keysArray.length, 3); + + for (int row = 0; row nm : r.getNoVersionMap().values()) { + for (byte[] val : nm.values()) { + assertTrue(Bytes.equals(val, value(value))); + } + } + } + assertEquals(count, i); + } catch (IOException e) { + fail("Failed due to exception"); + } + } + + /** + * Test that shows that exception thrown from the RS side will result in an + * exception on the LIHFile client. + */ + @Test(expected=IOException.class) + public void testBulkLoadPhaseFailure() throws Exception { + String table = "bulkLoadPhaseFailure"; + setupTable(table, 10); + + final AtomicInteger attmptedCalls = new AtomicInteger(); + final AtomicInteger failedCalls = new AtomicInteger(); + LoadIncrementalHFiles lih = new LoadIncrementalHFiles( + util.getConfiguration(), useSecure) { + + protected List tryAtomicRegionLoad(final HConnection conn, + byte[] tableName, final byte[] first, Collection lqis) + throws IOException { + int i = attmptedCalls.incrementAndGet(); + if (i == 1) { + HConnection errConn = null; + try { + errConn = getMockedConnection(util.getConfiguration()); + } catch (Exception e) { + LOG.fatal("mocking cruft, should never happen", e); + throw new RuntimeException("mocking cruft, should never happen"); + } + failedCalls.incrementAndGet(); + return super.tryAtomicRegionLoad(errConn, tableName, first, lqis); + } + + return super.tryAtomicRegionLoad(conn, tableName, first, lqis); + } + }; + + // create HFiles for different column families + Path dir = buildBulkFiles(table, 1); + HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table)); + lih.doBulkLoad(dir, t); + + fail("doBulkLoad should have thrown an exception"); + } + + private HConnection getMockedConnection(final Configuration conf) + throws IOException { + HConnection c = Mockito.mock(HConnection.class); + Mockito.when(c.getConfiguration()).thenReturn(conf); + Mockito.doNothing().when(c).close(); + // Make it so we return a particular location when asked. + final HRegionLocation loc = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, + "example.org", 1234); + Mockito.when(c.getRegionLocation((byte[]) Mockito.any(), + (byte[]) Mockito.any(), Mockito.anyBoolean())). + thenReturn(loc); + Mockito.when(c.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any())). + thenReturn(loc); + HRegionInterface hri = Mockito.mock(HRegionInterface.class); + Mockito.when(hri.bulkLoadHFiles(Mockito.anyList(), (byte [])Mockito.any())). + thenThrow(new IOException("injecting bulk load error")); + Mockito.when(c.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())). + thenReturn(hri); + return c; + } + + /** + * This test exercises the path where there is a split after initial + * validation but before the atomic bulk load call. We cannot use presplitting + * to test this path, so we actually inject a split just before the atomic + * region load. + */ + @Test + public void testSplitWhileBulkLoadPhase() throws Exception { + final String table = "splitWhileBulkloadPhase"; + setupTable(table, 10); + populateTable(table,1); + assertExpectedTable(table, ROWCOUNT, 1); + + // Now let's cause trouble. This will occur after checks and cause bulk + // files to fail when attempt to atomically import. This is recoverable. + final AtomicInteger attemptedCalls = new AtomicInteger(); + LoadIncrementalHFiles lih2 = new LoadIncrementalHFiles( + util.getConfiguration(), useSecure) { + + protected void bulkLoadPhase(final HTable htable, final HConnection conn, + ExecutorService pool, Deque queue, + final Multimap regionGroups) throws IOException { + int i = attemptedCalls.incrementAndGet(); + if (i == 1) { + // On first attempt force a split. + forceSplit(table); + } + + super.bulkLoadPhase(htable, conn, pool, queue, regionGroups); + } + }; + + // create HFiles for different column families + HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table)); + Path bulk = buildBulkFiles(table, 2); + lih2.doBulkLoad(bulk, t); + + // check that data was loaded + // The three expected attempts are 1) failure because need to split, 2) + // load of split top 3) load of split bottom + assertEquals(attemptedCalls.get(), 3); + assertExpectedTable(table, ROWCOUNT, 2); + } + + /** + * This test splits a table and attempts to bulk load. The bulk import files + * should be split before atomically importing. + */ + @Test + public void testGroupOrSplitPresplit() throws Exception { + final String table = "groupOrSplitPresplit"; + setupTable(table, 10); + populateTable(table, 1); + assertExpectedTable(table, ROWCOUNT, 1); + forceSplit(table); + + final AtomicInteger countedLqis= new AtomicInteger(); + LoadIncrementalHFiles lih = new LoadIncrementalHFiles( + util.getConfiguration(), useSecure) { + protected List groupOrSplit( + Multimap regionGroups, + final LoadQueueItem item, final HTable htable, + final Pair startEndKeys) throws IOException { + List lqis = super.groupOrSplit(regionGroups, item, htable, startEndKeys); + if (lqis != null) { + countedLqis.addAndGet(lqis.size()); + } + return lqis; + } + }; + + // create HFiles for different column families + Path bulk = buildBulkFiles(table, 2); + HTable ht = new HTable(util.getConfiguration(), Bytes.toBytes(table)); + lih.doBulkLoad(bulk, ht); + + assertExpectedTable(table, ROWCOUNT, 2); + assertEquals(20, countedLqis.get()); + } + + /** + * This simulates an remote exception which should cause LIHF to exit with an + * exception. + */ + @Test(expected = IOException.class) + public void testGroupOrSplitFailure() throws Exception { + String table = "groupOrSplitFailure"; + setupTable(table, 10); + + LoadIncrementalHFiles lih = new LoadIncrementalHFiles( + util.getConfiguration(), useSecure) { + int i = 0; + + protected List groupOrSplit( + Multimap regionGroups, + final LoadQueueItem item, final HTable table, + final Pair startEndKeys) throws IOException { + i++; + + if (i == 5) { + throw new IOException("failure"); + } + return super.groupOrSplit(regionGroups, item, table, startEndKeys); + } + }; + + // create HFiles for different column families + Path dir = buildBulkFiles(table,1); + HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table)); + lih.doBulkLoad(dir, t); + + fail("doBulkLoad should have thrown an exception"); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java new file mode 100644 index 0000000..89802f2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java @@ -0,0 +1,254 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests various scan start and stop row scenarios. This is set in a scan and + * tested in a MapReduce job to see if that is handed over and done properly + * too. + */ +@Category(LargeTests.class) +public class TestMultiTableInputFormat { + + static final Log LOG = LogFactory.getLog(TestMultiTableInputFormat.class); + static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + static final String TABLE_NAME = "scantest"; + static final byte[] INPUT_FAMILY = Bytes.toBytes("contents"); + static final String KEY_STARTROW = "startRow"; + static final String KEY_LASTROW = "stpRow"; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // switch TIF to log at DEBUG level + TEST_UTIL.enableDebug(MultiTableInputFormat.class); + TEST_UTIL.enableDebug(MultiTableInputFormatBase.class); + // start mini hbase cluster + TEST_UTIL.startMiniCluster(3); + // create and fill table + for (int i = 0; i < 3; i++) { + HTable table = + TEST_UTIL.createTable(Bytes.toBytes(TABLE_NAME + String.valueOf(i)), + INPUT_FAMILY); + TEST_UTIL.createMultiRegions(table, INPUT_FAMILY); + TEST_UTIL.loadTable(table, INPUT_FAMILY); + } + // start MR cluster + TEST_UTIL.startMiniMapReduceCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniMapReduceCluster(); + TEST_UTIL.shutdownMiniCluster(); + } + + @After + public void tearDown() throws Exception { + Configuration c = TEST_UTIL.getConfiguration(); + FileUtil.fullyDelete(new File(c.get("hadoop.tmp.dir"))); + } + + /** + * Pass the key and value to reducer. + */ + public static class ScanMapper extends + TableMapper { + /** + * Pass the key and value to reduce. + * + * @param key The key, here "aaa", "aab" etc. + * @param value The value is the same as the key. + * @param context The task context. + * @throws IOException When reading the rows fails. + */ + @Override + public void map(ImmutableBytesWritable key, Result value, Context context) + throws IOException, InterruptedException { + if (value.size() != 1) { + throw new IOException("There should only be one input column"); + } + Map>> cf = + value.getMap(); + if (!cf.containsKey(INPUT_FAMILY)) { + throw new IOException("Wrong input columns. Missing: '" + + Bytes.toString(INPUT_FAMILY) + "'."); + } + String val = Bytes.toStringBinary(value.getValue(INPUT_FAMILY, null)); + LOG.debug("map: key -> " + Bytes.toStringBinary(key.get()) + + ", value -> " + val); + context.write(key, key); + } + } + + /** + * Checks the last and first keys seen against the scanner boundaries. + */ + public static class ScanReducer + extends + Reducer { + private String first = null; + private String last = null; + + protected void reduce(ImmutableBytesWritable key, + Iterable values, Context context) + throws IOException, InterruptedException { + int count = 0; + for (ImmutableBytesWritable value : values) { + String val = Bytes.toStringBinary(value.get()); + LOG.debug("reduce: key[" + count + "] -> " + + Bytes.toStringBinary(key.get()) + ", value -> " + val); + if (first == null) first = val; + last = val; + count++; + } + assertEquals(3, count); + } + + protected void cleanup(Context context) throws IOException, + InterruptedException { + Configuration c = context.getConfiguration(); + String startRow = c.get(KEY_STARTROW); + String lastRow = c.get(KEY_LASTROW); + LOG.info("cleanup: first -> \"" + first + "\", start row -> \"" + + startRow + "\""); + LOG.info("cleanup: last -> \"" + last + "\", last row -> \"" + lastRow + + "\""); + if (startRow != null && startRow.length() > 0) { + assertEquals(startRow, first); + } + if (lastRow != null && lastRow.length() > 0) { + assertEquals(lastRow, last); + } + } + } + + @Test + public void testScanEmptyToEmpty() throws IOException, InterruptedException, + ClassNotFoundException { + testScan(null, null, null); + } + + @Test + public void testScanEmptyToAPP() throws IOException, InterruptedException, + ClassNotFoundException { + testScan(null, "app", "apo"); + } + + @Test + public void testScanOBBToOPP() throws IOException, InterruptedException, + ClassNotFoundException { + testScan("obb", "opp", "opo"); + } + + @Test + public void testScanOPPToEmpty() throws IOException, InterruptedException, + ClassNotFoundException { + testScan("opp", null, "zzz"); + } + + @Test + public void testScanYZYToEmpty() throws IOException, InterruptedException, + ClassNotFoundException { + testScan("yzy", null, "zzz"); + } + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + private void testScan(String start, String stop, String last) + throws IOException, InterruptedException, ClassNotFoundException { + String jobName = + "Scan" + (start != null ? start.toUpperCase() : "Empty") + "To" + + (stop != null ? stop.toUpperCase() : "Empty"); + LOG.info("Before map/reduce startup - job " + jobName); + Configuration c = new Configuration(TEST_UTIL.getConfiguration()); + + c.set(KEY_STARTROW, start != null ? start : ""); + c.set(KEY_LASTROW, last != null ? last : ""); + + List scans = new ArrayList(); + + for(int i=0; i<3; i++){ + Scan scan = new Scan(); + + scan.addFamily(INPUT_FAMILY); + scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(TABLE_NAME + i)); + + if (start != null) { + scan.setStartRow(Bytes.toBytes(start)); + } + if (stop != null) { + scan.setStopRow(Bytes.toBytes(stop)); + } + + scans.add(scan); + + LOG.info("scan before: " + scan); + } + + Job job = new Job(c, jobName); + + TableMapReduceUtil.initTableMapperJob(scans, ScanMapper.class, + ImmutableBytesWritable.class, ImmutableBytesWritable.class, job); + job.setReducerClass(ScanReducer.class); + job.setNumReduceTasks(1); // one to get final "first" and "last" key + FileOutputFormat.setOutputPath(job, new Path(job.getJobName())); + LOG.info("Started " + job.getJobName()); + job.waitForCompletion(true); + assertTrue(job.isSuccessful()); + LOG.info("After map/reduce completion - job " + jobName); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java new file mode 100644 index 0000000..f77b7eb --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java @@ -0,0 +1,261 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.fail; +import static org.junit.Assert.assertTrue; + +/** + * Test Map/Reduce job over HBase tables. The map/reduce process we're testing + * on our tables is simple - take every row in the table, reverse the value of + * a particular cell, and write it back to the table. + */ +@Category(LargeTests.class) +public class TestMultithreadedTableMapper { + private static final Log LOG = LogFactory.getLog(TestMultithreadedTableMapper.class); + private static final HBaseTestingUtility UTIL = + new HBaseTestingUtility(); + static final byte[] MULTI_REGION_TABLE_NAME = Bytes.toBytes("mrtest"); + static final byte[] INPUT_FAMILY = Bytes.toBytes("contents"); + static final byte[] OUTPUT_FAMILY = Bytes.toBytes("text"); + static final int NUMBER_OF_THREADS = 10; + + @BeforeClass + public static void beforeClass() throws Exception { + UTIL.startMiniCluster(); + HTable table = UTIL.createTable(MULTI_REGION_TABLE_NAME, new byte[][] {INPUT_FAMILY, OUTPUT_FAMILY}); + UTIL.createMultiRegions(table, INPUT_FAMILY); + UTIL.loadTable(table, INPUT_FAMILY); + UTIL.startMiniMapReduceCluster(); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniMapReduceCluster(); + UTIL.shutdownMiniCluster(); + } + + /** + * Pass the given key and processed record reduce + */ + public static class ProcessContentsMapper + extends TableMapper { + + /** + * Pass the key, and reversed value to reduce + * + * @param key + * @param value + * @param context + * @throws IOException + */ + public void map(ImmutableBytesWritable key, Result value, + Context context) + throws IOException, InterruptedException { + if (value.size() != 1) { + throw new IOException("There should only be one input column"); + } + Map>> + cf = value.getMap(); + if(!cf.containsKey(INPUT_FAMILY)) { + throw new IOException("Wrong input columns. Missing: '" + + Bytes.toString(INPUT_FAMILY) + "'."); + } + // Get the original value and reverse it + String originalValue = new String(value.getValue(INPUT_FAMILY, null), + HConstants.UTF8_ENCODING); + StringBuilder newValue = new StringBuilder(originalValue); + newValue.reverse(); + // Now set the value to be collected + Put outval = new Put(key.get()); + outval.add(OUTPUT_FAMILY, null, Bytes.toBytes(newValue.toString())); + context.write(key, outval); + } + } + + /** + * Test multithreadedTableMappper map/reduce against a multi-region table + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testMultithreadedTableMapper() + throws IOException, InterruptedException, ClassNotFoundException { + runTestOnTable(new HTable(new Configuration(UTIL.getConfiguration()), + MULTI_REGION_TABLE_NAME)); + } + + private void runTestOnTable(HTable table) + throws IOException, InterruptedException, ClassNotFoundException { + Job job = null; + try { + LOG.info("Before map/reduce startup"); + job = new Job(table.getConfiguration(), "process column contents"); + job.setNumReduceTasks(1); + Scan scan = new Scan(); + scan.addFamily(INPUT_FAMILY); + TableMapReduceUtil.initTableMapperJob( + Bytes.toString(table.getTableName()), scan, + MultithreadedTableMapper.class, ImmutableBytesWritable.class, + Put.class, job); + MultithreadedTableMapper.setMapperClass(job, ProcessContentsMapper.class); + MultithreadedTableMapper.setNumberOfThreads(job, NUMBER_OF_THREADS); + TableMapReduceUtil.initTableReducerJob( + Bytes.toString(table.getTableName()), + IdentityTableReducer.class, job); + FileOutputFormat.setOutputPath(job, new Path("test")); + LOG.info("Started " + Bytes.toString(table.getTableName())); + assertTrue(job.waitForCompletion(true)); + LOG.info("After map/reduce completion"); + // verify map-reduce results + verify(Bytes.toString(table.getTableName())); + } finally { + table.close(); + if (job != null) { + FileUtil.fullyDelete( + new File(job.getConfiguration().get("hadoop.tmp.dir"))); + } + } + } + + private void verify(String tableName) throws IOException { + HTable table = new HTable(new Configuration(UTIL.getConfiguration()), tableName); + boolean verified = false; + long pause = UTIL.getConfiguration().getLong("hbase.client.pause", 5 * 1000); + int numRetries = UTIL.getConfiguration().getInt("hbase.client.retries.number", 5); + for (int i = 0; i < numRetries; i++) { + try { + LOG.info("Verification attempt #" + i); + verifyAttempt(table); + verified = true; + break; + } catch (NullPointerException e) { + // If here, a cell was empty. Presume its because updates came in + // after the scanner had been opened. Wait a while and retry. + LOG.debug("Verification attempt failed: " + e.getMessage()); + } + try { + Thread.sleep(pause); + } catch (InterruptedException e) { + // continue + } + } + assertTrue(verified); + table.close(); + } + + /** + * Looks at every value of the mapreduce output and verifies that indeed + * the values have been reversed. + * + * @param table Table to scan. + * @throws IOException + * @throws NullPointerException if we failed to find a cell value + */ + private void verifyAttempt(final HTable table) + throws IOException, NullPointerException { + Scan scan = new Scan(); + scan.addFamily(INPUT_FAMILY); + scan.addFamily(OUTPUT_FAMILY); + ResultScanner scanner = table.getScanner(scan); + try { + Iterator itr = scanner.iterator(); + assertTrue(itr.hasNext()); + while(itr.hasNext()) { + Result r = itr.next(); + if (LOG.isDebugEnabled()) { + if (r.size() > 2 ) { + throw new IOException("Too many results, expected 2 got " + + r.size()); + } + } + byte[] firstValue = null; + byte[] secondValue = null; + int count = 0; + for(KeyValue kv : r.list()) { + if (count == 0) { + firstValue = kv.getValue(); + }else if (count == 1) { + secondValue = kv.getValue(); + }else if (count == 2) { + break; + } + count++; + } + String first = ""; + if (firstValue == null) { + throw new NullPointerException(Bytes.toString(r.getRow()) + + ": first value is null"); + } + first = new String(firstValue, HConstants.UTF8_ENCODING); + String second = ""; + if (secondValue == null) { + throw new NullPointerException(Bytes.toString(r.getRow()) + + ": second value is null"); + } + byte[] secondReversed = new byte[secondValue.length]; + for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) { + secondReversed[i] = secondValue[j]; + } + second = new String(secondReversed, HConstants.UTF8_ENCODING); + if (first.compareTo(second) != 0) { + if (LOG.isDebugEnabled()) { + LOG.debug("second key is not the reverse of first. row=" + + Bytes.toStringBinary(r.getRow()) + ", first value=" + first + + ", second value=" + second); + } + fail(); + } + } + } finally { + scanner.close(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSimpleTotalOrderPartitioner.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSimpleTotalOrderPartitioner.java new file mode 100644 index 0000000..7ec759a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSimpleTotalOrderPartitioner.java @@ -0,0 +1,74 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +/** + * Test of simple partitioner. + */ +@Category(SmallTests.class) +public class TestSimpleTotalOrderPartitioner extends HBaseTestCase { + public void testSplit() throws Exception { + String start = "a"; + String end = "{"; + SimpleTotalOrderPartitioner p = + new SimpleTotalOrderPartitioner(); + this.conf.set(SimpleTotalOrderPartitioner.START, start); + this.conf.set(SimpleTotalOrderPartitioner.END, end); + p.setConf(this.conf); + ImmutableBytesWritable c = new ImmutableBytesWritable(Bytes.toBytes("c")); + // If one reduce, partition should be 0. + int partition = p.getPartition(c, HConstants.EMPTY_BYTE_ARRAY, 1); + assertEquals(0, partition); + // If two reduces, partition should be 0. + partition = p.getPartition(c, HConstants.EMPTY_BYTE_ARRAY, 2); + assertEquals(0, partition); + // Divide in 3. + partition = p.getPartition(c, HConstants.EMPTY_BYTE_ARRAY, 3); + assertEquals(0, partition); + ImmutableBytesWritable q = new ImmutableBytesWritable(Bytes.toBytes("q")); + partition = p.getPartition(q, HConstants.EMPTY_BYTE_ARRAY, 2); + assertEquals(1, partition); + partition = p.getPartition(q, HConstants.EMPTY_BYTE_ARRAY, 3); + assertEquals(2, partition); + // What about end and start keys. + ImmutableBytesWritable startBytes = + new ImmutableBytesWritable(Bytes.toBytes(start)); + partition = p.getPartition(startBytes, HConstants.EMPTY_BYTE_ARRAY, 2); + assertEquals(0, partition); + partition = p.getPartition(startBytes, HConstants.EMPTY_BYTE_ARRAY, 3); + assertEquals(0, partition); + ImmutableBytesWritable endBytes = + new ImmutableBytesWritable(Bytes.toBytes("z")); + partition = p.getPartition(endBytes, HConstants.EMPTY_BYTE_ARRAY, 2); + assertEquals(1, partition); + partition = p.getPartition(endBytes, HConstants.EMPTY_BYTE_ARRAY, 3); + assertEquals(2, partition); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan1.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan1.java new file mode 100644 index 0000000..77ea47a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan1.java @@ -0,0 +1,99 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.IOException; + +import org.apache.hadoop.hbase.LargeTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * TestTableInputFormatScan part 1. + * @see TestTableInputFormatScanBase + */ +@Category(LargeTests.class) +public class TestTableInputFormatScan1 extends TestTableInputFormatScanBase { + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testScanEmptyToEmpty() + throws IOException, InterruptedException, ClassNotFoundException { + testScan(null, null, null); + } + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testScanEmptyToAPP() + throws IOException, InterruptedException, ClassNotFoundException { + testScan(null, "app", "apo"); + } + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testScanEmptyToBBA() + throws IOException, InterruptedException, ClassNotFoundException { + testScan(null, "bba", "baz"); + } + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testScanEmptyToBBB() + throws IOException, InterruptedException, ClassNotFoundException { + testScan(null, "bbb", "bba"); + } + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testScanEmptyToOPP() + throws IOException, InterruptedException, ClassNotFoundException { + testScan(null, "opp", "opo"); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan2.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan2.java new file mode 100644 index 0000000..f35bbd1 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan2.java @@ -0,0 +1,117 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.IOException; + +import org.apache.hadoop.hbase.LargeTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * TestTableInputFormatScan part 2. + * @see TestTableInputFormatScanBase + */ +@Category(LargeTests.class) +public class TestTableInputFormatScan2 extends TestTableInputFormatScanBase { + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testScanOBBToOPP() + throws IOException, InterruptedException, ClassNotFoundException { + testScan("obb", "opp", "opo"); + } + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testScanOBBToQPP() + throws IOException, InterruptedException, ClassNotFoundException { + testScan("obb", "qpp", "qpo"); + } + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testScanOPPToEmpty() + throws IOException, InterruptedException, ClassNotFoundException { + testScan("opp", null, "zzz"); + } + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testScanYYXToEmpty() + throws IOException, InterruptedException, ClassNotFoundException { + testScan("yyx", null, "zzz"); + } + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testScanYYYToEmpty() + throws IOException, InterruptedException, ClassNotFoundException { + testScan("yyy", null, "zzz"); + } + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testScanYZYToEmpty() + throws IOException, InterruptedException, ClassNotFoundException { + testScan("yzy", null, "zzz"); + } + + @Test + public void testScanFromConfiguration() + throws IOException, InterruptedException, ClassNotFoundException { + testScanFromConfiguration("bba", "bbd", "bbc"); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java new file mode 100644 index 0000000..7659604 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java @@ -0,0 +1,239 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Map; +import java.util.NavigableMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** + *

    + * Tests various scan start and stop row scenarios. This is set in a scan and + * tested in a MapReduce job to see if that is handed over and done properly + * too. + *

    + *

    + * This test is broken into two parts in order to side-step the test timeout + * period of 900, as documented in HBASE-8326. + *

    + */ +public abstract class TestTableInputFormatScanBase { + + static final Log LOG = LogFactory.getLog(TestTableInputFormatScanBase.class); + static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + static final byte[] TABLE_NAME = Bytes.toBytes("scantest"); + static final byte[] INPUT_FAMILY = Bytes.toBytes("contents"); + static final String KEY_STARTROW = "startRow"; + static final String KEY_LASTROW = "stpRow"; + + private static HTable table = null; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // switch TIF to log at DEBUG level + TEST_UTIL.enableDebug(TableInputFormat.class); + TEST_UTIL.enableDebug(TableInputFormatBase.class); + // start mini hbase cluster + TEST_UTIL.startMiniCluster(3); + // create and fill table + table = TEST_UTIL.createTable(TABLE_NAME, INPUT_FAMILY); + TEST_UTIL.createMultiRegions(table, INPUT_FAMILY); + TEST_UTIL.loadTable(table, INPUT_FAMILY); + // start MR cluster + TEST_UTIL.startMiniMapReduceCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniMapReduceCluster(); + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Pass the key and value to reduce. + */ + public static class ScanMapper + extends TableMapper { + + /** + * Pass the key and value to reduce. + * + * @param key The key, here "aaa", "aab" etc. + * @param value The value is the same as the key. + * @param context The task context. + * @throws IOException When reading the rows fails. + */ + @Override + public void map(ImmutableBytesWritable key, Result value, + Context context) + throws IOException, InterruptedException { + if (value.size() != 1) { + throw new IOException("There should only be one input column"); + } + Map>> + cf = value.getMap(); + if(!cf.containsKey(INPUT_FAMILY)) { + throw new IOException("Wrong input columns. Missing: '" + + Bytes.toString(INPUT_FAMILY) + "'."); + } + String val = Bytes.toStringBinary(value.getValue(INPUT_FAMILY, null)); + LOG.info("map: key -> " + Bytes.toStringBinary(key.get()) + + ", value -> " + val); + context.write(key, key); + } + + } + + /** + * Checks the last and first key seen against the scanner boundaries. + */ + public static class ScanReducer + extends Reducer { + + private String first = null; + private String last = null; + + protected void reduce(ImmutableBytesWritable key, + Iterable values, Context context) + throws IOException ,InterruptedException { + int count = 0; + for (ImmutableBytesWritable value : values) { + String val = Bytes.toStringBinary(value.get()); + LOG.info("reduce: key[" + count + "] -> " + + Bytes.toStringBinary(key.get()) + ", value -> " + val); + if (first == null) first = val; + last = val; + count++; + } + } + + protected void cleanup(Context context) + throws IOException, InterruptedException { + Configuration c = context.getConfiguration(); + String startRow = c.get(KEY_STARTROW); + String lastRow = c.get(KEY_LASTROW); + LOG.info("cleanup: first -> \"" + first + "\", start row -> \"" + startRow + "\""); + LOG.info("cleanup: last -> \"" + last + "\", last row -> \"" + lastRow + "\""); + if (startRow != null && startRow.length() > 0) { + assertEquals(startRow, first); + } + if (lastRow != null && lastRow.length() > 0) { + assertEquals(lastRow, last); + } + } + + } + + /** + * Tests an MR Scan initialized from properties set in the Configuration. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + protected void testScanFromConfiguration(String start, String stop, String last) + throws IOException, InterruptedException, ClassNotFoundException { + String jobName = "ScanFromConfig" + (start != null ? start.toUpperCase() : "Empty") + + "To" + (stop != null ? stop.toUpperCase() : "Empty"); + Configuration c = new Configuration(TEST_UTIL.getConfiguration()); + c.set(TableInputFormat.INPUT_TABLE, Bytes.toString(TABLE_NAME)); + c.set(TableInputFormat.SCAN_COLUMN_FAMILY, Bytes.toString(INPUT_FAMILY)); + c.set(KEY_STARTROW, start != null ? start : ""); + c.set(KEY_LASTROW, last != null ? last : ""); + + if (start != null) { + c.set(TableInputFormat.SCAN_ROW_START, start); + } + + if (stop != null) { + c.set(TableInputFormat.SCAN_ROW_STOP, stop); + } + + Job job = new Job(c, jobName); + job.setMapperClass(ScanMapper.class); + job.setReducerClass(ScanReducer.class); + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + job.setMapOutputValueClass(ImmutableBytesWritable.class); + job.setInputFormatClass(TableInputFormat.class); + job.setNumReduceTasks(1); + FileOutputFormat.setOutputPath(job, new Path(job.getJobName())); + TableMapReduceUtil.addDependencyJars(job); + assertTrue(job.waitForCompletion(true)); + } + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + protected void testScan(String start, String stop, String last) + throws IOException, InterruptedException, ClassNotFoundException { + String jobName = "Scan" + (start != null ? start.toUpperCase() : "Empty") + + "To" + (stop != null ? stop.toUpperCase() : "Empty"); + LOG.info("Before map/reduce startup - job " + jobName); + Configuration c = new Configuration(TEST_UTIL.getConfiguration()); + Scan scan = new Scan(); + scan.addFamily(INPUT_FAMILY); + if (start != null) { + scan.setStartRow(Bytes.toBytes(start)); + } + c.set(KEY_STARTROW, start != null ? start : ""); + if (stop != null) { + scan.setStopRow(Bytes.toBytes(stop)); + } + c.set(KEY_LASTROW, last != null ? last : ""); + LOG.info("scan before: " + scan); + Job job = new Job(c, jobName); + TableMapReduceUtil.initTableMapperJob( + Bytes.toString(TABLE_NAME), scan, ScanMapper.class, + ImmutableBytesWritable.class, ImmutableBytesWritable.class, job); + job.setReducerClass(ScanReducer.class); + job.setNumReduceTasks(1); // one to get final "first" and "last" key + FileOutputFormat.setOutputPath(job, new Path(job.getJobName())); + LOG.info("Started " + job.getJobName()); + assertTrue(job.waitForCompletion(true)); + LOG.info("After map/reduce completion - job " + jobName); + } + +} + diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java new file mode 100644 index 0000000..b351444 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java @@ -0,0 +1,286 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.fail; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +/** + * Test Map/Reduce job over HBase tables. The map/reduce process we're testing + * on our tables is simple - take every row in the table, reverse the value of + * a particular cell, and write it back to the table. + */ +@Category(LargeTests.class) +public class TestTableMapReduce { + private static final Log LOG = LogFactory.getLog(TestTableMapReduce.class); + private static final HBaseTestingUtility UTIL = + new HBaseTestingUtility(); + static final byte[] MULTI_REGION_TABLE_NAME = Bytes.toBytes("mrtest"); + static final byte[] INPUT_FAMILY = Bytes.toBytes("contents"); + static final byte[] OUTPUT_FAMILY = Bytes.toBytes("text"); + + @BeforeClass + public static void beforeClass() throws Exception { + UTIL.startMiniCluster(); + HTable table = UTIL.createTable(MULTI_REGION_TABLE_NAME, new byte[][] {INPUT_FAMILY, OUTPUT_FAMILY}); + UTIL.createMultiRegions(table, INPUT_FAMILY); + UTIL.loadTable(table, INPUT_FAMILY); + UTIL.startMiniMapReduceCluster(); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniMapReduceCluster(); + UTIL.shutdownMiniCluster(); + } + + /** + * Pass the given key and processed record reduce + */ + public static class ProcessContentsMapper + extends TableMapper { + + /** + * Pass the key, and reversed value to reduce + * + * @param key + * @param value + * @param context + * @throws IOException + */ + public void map(ImmutableBytesWritable key, Result value, + Context context) + throws IOException, InterruptedException { + if (value.size() != 1) { + throw new IOException("There should only be one input column"); + } + Map>> + cf = value.getMap(); + if(!cf.containsKey(INPUT_FAMILY)) { + throw new IOException("Wrong input columns. Missing: '" + + Bytes.toString(INPUT_FAMILY) + "'."); + } + + // Get the original value and reverse it + String originalValue = new String(value.getValue(INPUT_FAMILY, null), + HConstants.UTF8_ENCODING); + StringBuilder newValue = new StringBuilder(originalValue); + newValue.reverse(); + // Now set the value to be collected + Put outval = new Put(key.get()); + outval.add(OUTPUT_FAMILY, null, Bytes.toBytes(newValue.toString())); + context.write(key, outval); + } + } + + /** + * Test a map/reduce against a multi-region table + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + @Test + public void testMultiRegionTable() + throws IOException, InterruptedException, ClassNotFoundException { + runTestOnTable(new HTable(new Configuration(UTIL.getConfiguration()), + MULTI_REGION_TABLE_NAME)); + } + + private void runTestOnTable(HTable table) + throws IOException, InterruptedException, ClassNotFoundException { + Job job = null; + try { + LOG.info("Before map/reduce startup"); + job = new Job(table.getConfiguration(), "process column contents"); + job.setNumReduceTasks(1); + Scan scan = new Scan(); + scan.addFamily(INPUT_FAMILY); + TableMapReduceUtil.initTableMapperJob( + Bytes.toString(table.getTableName()), scan, + ProcessContentsMapper.class, ImmutableBytesWritable.class, + Put.class, job); + TableMapReduceUtil.initTableReducerJob( + Bytes.toString(table.getTableName()), + IdentityTableReducer.class, job); + FileOutputFormat.setOutputPath(job, new Path("test")); + LOG.info("Started " + Bytes.toString(table.getTableName())); + assertTrue(job.waitForCompletion(true)); + LOG.info("After map/reduce completion"); + + // verify map-reduce results + verify(Bytes.toString(table.getTableName())); + } finally { + table.close(); + if (job != null) { + FileUtil.fullyDelete( + new File(job.getConfiguration().get("hadoop.tmp.dir"))); + } + } + } + + private void verify(String tableName) throws IOException { + HTable table = new HTable(new Configuration(UTIL.getConfiguration()), tableName); + boolean verified = false; + long pause = UTIL.getConfiguration().getLong("hbase.client.pause", 5 * 1000); + int numRetries = UTIL.getConfiguration().getInt("hbase.client.retries.number", 5); + for (int i = 0; i < numRetries; i++) { + try { + LOG.info("Verification attempt #" + i); + verifyAttempt(table); + verified = true; + break; + } catch (NullPointerException e) { + // If here, a cell was empty. Presume its because updates came in + // after the scanner had been opened. Wait a while and retry. + LOG.debug("Verification attempt failed: " + e.getMessage()); + } + try { + Thread.sleep(pause); + } catch (InterruptedException e) { + // continue + } + } + assertTrue(verified); + table.close(); + } + + /** + * Looks at every value of the mapreduce output and verifies that indeed + * the values have been reversed. + * + * @param table Table to scan. + * @throws IOException + * @throws NullPointerException if we failed to find a cell value + */ + private void verifyAttempt(final HTable table) throws IOException, NullPointerException { + Scan scan = new Scan(); + scan.addFamily(INPUT_FAMILY); + scan.addFamily(OUTPUT_FAMILY); + ResultScanner scanner = table.getScanner(scan); + try { + Iterator itr = scanner.iterator(); + assertTrue(itr.hasNext()); + while(itr.hasNext()) { + Result r = itr.next(); + if (LOG.isDebugEnabled()) { + if (r.size() > 2 ) { + throw new IOException("Too many results, expected 2 got " + + r.size()); + } + } + byte[] firstValue = null; + byte[] secondValue = null; + int count = 0; + for(KeyValue kv : r.list()) { + if (count == 0) { + firstValue = kv.getValue(); + } + if (count == 1) { + secondValue = kv.getValue(); + } + count++; + if (count == 2) { + break; + } + } + + String first = ""; + if (firstValue == null) { + throw new NullPointerException(Bytes.toString(r.getRow()) + + ": first value is null"); + } + first = new String(firstValue, HConstants.UTF8_ENCODING); + + String second = ""; + if (secondValue == null) { + throw new NullPointerException(Bytes.toString(r.getRow()) + + ": second value is null"); + } + byte[] secondReversed = new byte[secondValue.length]; + for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) { + secondReversed[i] = secondValue[j]; + } + second = new String(secondReversed, HConstants.UTF8_ENCODING); + + if (first.compareTo(second) != 0) { + if (LOG.isDebugEnabled()) { + LOG.debug("second key is not the reverse of first. row=" + + Bytes.toStringBinary(r.getRow()) + ", first value=" + first + + ", second value=" + second); + } + fail(); + } + } + } finally { + scanner.close(); + } + } + + /** + * Test that we add tmpjars correctly including the ZK jar. + */ + public void testAddDependencyJars() throws Exception { + Job job = new Job(); + TableMapReduceUtil.addDependencyJars(job); + String tmpjars = job.getConfiguration().get("tmpjars"); + + System.err.println("tmpjars: " + tmpjars); + assertTrue(tmpjars.contains("zookeeper")); + assertFalse(tmpjars.contains("guava")); + + System.err.println("appending guava jar"); + TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + com.google.common.base.Function.class); + tmpjars = job.getConfiguration().get("tmpjars"); + assertTrue(tmpjars.contains("guava")); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSplit.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSplit.java new file mode 100644 index 0000000..ded4dd6 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSplit.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.HashSet; + +import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@Category(SmallTests.class) +public class TestTableSplit { + @Test + public void testHashCode() { + TableSplit split1 = new TableSplit("table".getBytes(), "row-start".getBytes(), "row-end".getBytes(), "location"); + TableSplit split2 = new TableSplit("table".getBytes(), "row-start".getBytes(), "row-end".getBytes(), "location"); + assertEquals (split1, split2); + assertTrue (split1.hashCode() == split2.hashCode()); + HashSet set = new HashSet(2); + set.add(split1); + set.add(split2); + assertTrue(set.size() == 1); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java new file mode 100644 index 0000000..74ea4ec --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java @@ -0,0 +1,214 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.MapWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestTimeRangeMapRed { + private final static Log log = LogFactory.getLog(TestTimeRangeMapRed.class); + private static final HBaseTestingUtility UTIL = + new HBaseTestingUtility(); + private HBaseAdmin admin; + + private static final byte [] KEY = Bytes.toBytes("row1"); + private static final NavigableMap TIMESTAMP = + new TreeMap(); + static { + TIMESTAMP.put((long)1245620000, false); + TIMESTAMP.put((long)1245620005, true); // include + TIMESTAMP.put((long)1245620010, true); // include + TIMESTAMP.put((long)1245620055, true); // include + TIMESTAMP.put((long)1245620100, true); // include + TIMESTAMP.put((long)1245620150, false); + TIMESTAMP.put((long)1245620250, false); + } + static final long MINSTAMP = 1245620005; + static final long MAXSTAMP = 1245620100 + 1; // maxStamp itself is excluded. so increment it. + + static final byte[] TABLE_NAME = Bytes.toBytes("table123"); + static final byte[] FAMILY_NAME = Bytes.toBytes("text"); + static final byte[] COLUMN_NAME = Bytes.toBytes("input"); + + @BeforeClass + public static void beforeClass() throws Exception { + UTIL.startMiniCluster(); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Before + public void before() throws MasterNotRunningException, ZooKeeperConnectionException { + this.admin = new HBaseAdmin(UTIL.getConfiguration()); + } + + @After + public void after() throws IOException { + this.admin.close(); + } + + private static class ProcessTimeRangeMapper + extends TableMapper + implements Configurable { + + private Configuration conf = null; + private HTable table = null; + + @Override + public void map(ImmutableBytesWritable key, Result result, + Context context) + throws IOException { + List tsList = new ArrayList(); + for (KeyValue kv : result.list()) { + tsList.add(kv.getTimestamp()); + } + + for (Long ts : tsList) { + Put put = new Put(key.get()); + put.setWriteToWAL(false); + put.add(FAMILY_NAME, COLUMN_NAME, ts, Bytes.toBytes(true)); + table.put(put); + } + table.flushCommits(); + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void setConf(Configuration configuration) { + this.conf = configuration; + try { + table = new HTable(HBaseConfiguration.create(conf), TABLE_NAME); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + @Test + public void testTimeRangeMapRed() + throws IOException, InterruptedException, ClassNotFoundException { + final HTableDescriptor desc = new HTableDescriptor(TABLE_NAME); + final HColumnDescriptor col = new HColumnDescriptor(FAMILY_NAME); + col.setMaxVersions(Integer.MAX_VALUE); + desc.addFamily(col); + admin.createTable(desc); + HTable table = new HTable(UTIL.getConfiguration(), desc.getName()); + prepareTest(table); + runTestOnTable(); + verify(table); + } + + private void prepareTest(final HTable table) throws IOException { + for (Map.Entry entry : TIMESTAMP.entrySet()) { + Put put = new Put(KEY); + put.setWriteToWAL(false); + put.add(FAMILY_NAME, COLUMN_NAME, entry.getKey(), Bytes.toBytes(false)); + table.put(put); + } + table.flushCommits(); + } + + private void runTestOnTable() + throws IOException, InterruptedException, ClassNotFoundException { + UTIL.startMiniMapReduceCluster(1); + Job job = null; + try { + job = new Job(UTIL.getConfiguration(), "test123"); + job.setOutputFormatClass(NullOutputFormat.class); + job.setNumReduceTasks(0); + Scan scan = new Scan(); + scan.addColumn(FAMILY_NAME, COLUMN_NAME); + scan.setTimeRange(MINSTAMP, MAXSTAMP); + scan.setMaxVersions(); + TableMapReduceUtil.initTableMapperJob(Bytes.toString(TABLE_NAME), + scan, ProcessTimeRangeMapper.class, Text.class, Text.class, job); + job.waitForCompletion(true); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } finally { + UTIL.shutdownMiniMapReduceCluster(); + if (job != null) { + FileUtil.fullyDelete( + new File(job.getConfiguration().get("hadoop.tmp.dir"))); + } + } + } + + private void verify(final HTable table) throws IOException { + Scan scan = new Scan(); + scan.addColumn(FAMILY_NAME, COLUMN_NAME); + scan.setMaxVersions(1); + ResultScanner scanner = table.getScanner(scan); + for (Result r: scanner) { + for (KeyValue kv : r.list()) { + log.debug(Bytes.toString(r.getRow()) + "\t" + Bytes.toString(kv.getFamily()) + + "\t" + Bytes.toString(kv.getQualifier()) + + "\t" + kv.getTimestamp() + "\t" + Bytes.toBoolean(kv.getValue())); + org.junit.Assert.assertEquals(TIMESTAMP.get(kv.getTimestamp()), + (Boolean)Bytes.toBoolean(kv.getValue())); + } + } + scanner.close(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java new file mode 100644 index 0000000..93653af --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNull; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Basic test for the WALPlayer M/R tool + */ +@Category(LargeTests.class) +public class TestWALPlayer { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static MiniHBaseCluster cluster; + + @BeforeClass + public static void beforeClass() throws Exception { + cluster = TEST_UTIL.startMiniCluster(); + TEST_UTIL.startMiniMapReduceCluster(); + } + + @AfterClass + public static void afterClass() throws Exception { + TEST_UTIL.shutdownMiniMapReduceCluster(); + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Simple end-to-end test + * @throws Exception + */ + @Test + public void testWALPlayer() throws Exception { + final byte[] TABLENAME1 = Bytes.toBytes("testWALPlayer1"); + final byte[] TABLENAME2 = Bytes.toBytes("testWALPlayer2"); + final byte[] FAMILY = Bytes.toBytes("family"); + final byte[] COLUMN1 = Bytes.toBytes("c1"); + final byte[] COLUMN2 = Bytes.toBytes("c2"); + final byte[] ROW = Bytes.toBytes("row"); + HTable t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY); + HTable t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY); + + // put a row into the first table + Put p = new Put(ROW); + p.add(FAMILY, COLUMN1, COLUMN1); + p.add(FAMILY, COLUMN2, COLUMN2); + t1.put(p); + // delete one column + Delete d = new Delete(ROW); + d.deleteColumns(FAMILY, COLUMN1); + t1.delete(d); + + // replay the WAL, map table 1 to table 2 + HLog log = cluster.getRegionServer(0).getWAL(); + log.rollWriter(); + String walInputDir = new Path(cluster.getMaster().getMasterFileSystem() + .getRootDir(), HConstants.HREGION_LOGDIR_NAME).toString(); + + WALPlayer player = new WALPlayer(TEST_UTIL.getConfiguration()); + assertEquals(0, player.run(new String[] { walInputDir, Bytes.toString(TABLENAME1), + Bytes.toString(TABLENAME2) })); + + // verify the WAL was player into table 2 + Get g = new Get(ROW); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(Bytes.equals(COLUMN2, r.raw()[0].getQualifier())); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapper.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapper.java new file mode 100644 index 0000000..a9e0ac2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapper.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.KeyValue; + +import java.io.IOException; + +/** + * Dummy mapper used for unit tests to verify that the mapper can be injected. + * This approach would be used if a custom transformation needed to be done after + * reading the input data before writing it to HFiles. + */ +public class TsvImporterCustomTestMapper extends TsvImporterMapper { + + @Override + protected void setup(Context context) { + doSetup(context); + } + + /** + * Convert a line of TSV text into an HBase table row after transforming the + * values by multiplying them by 3. + */ + @Override + public void map(LongWritable offset, Text value, Context context) + throws IOException { + byte[] family = Bytes.toBytes("FAM"); + final byte[][] qualifiers = { Bytes.toBytes("A"), Bytes.toBytes("B") }; + + // do some basic line parsing + byte[] lineBytes = value.getBytes(); + String[] valueTokens = new String(lineBytes, "UTF-8").split("\u001b"); + + // create the rowKey and Put + ImmutableBytesWritable rowKey = + new ImmutableBytesWritable(Bytes.toBytes(valueTokens[0])); + Put put = new Put(rowKey.copyBytes()); + put.setWriteToWAL(false); + + //The value should look like this: VALUE1 or VALUE2. Let's multiply + //the integer by 3 + for(int i = 1; i < valueTokens.length; i++) { + String prefix = valueTokens[i].substring(0, "VALUE".length()); + String suffix = valueTokens[i].substring("VALUE".length()); + String newValue = prefix + Integer.parseInt(suffix) * 3; + + KeyValue kv = new KeyValue(rowKey.copyBytes(), family, + qualifiers[i-1], Bytes.toBytes(newValue)); + put.add(kv); + } + + try { + context.write(rowKey, put); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java b/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java new file mode 100644 index 0000000..fb56993 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.mapreduce.hadoopbackport; + +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.Writer; +import java.text.MessageFormat; +import java.util.Properties; +import java.util.jar.JarInputStream; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; + +/** + * This file was forked from hadoop/common/branches/branch-2@1350012. + */ +@Category(SmallTests.class) +public class TestJarFinder { + + @Test + public void testJar() throws Exception { + + //picking a class that is for sure in a JAR in the classpath + String jar = JarFinder.getJar(LogFactory.class); + Assert.assertTrue(new File(jar).exists()); + } + + private static void delete(File file) throws IOException { + if (file.getAbsolutePath().length() < 5) { + throw new IllegalArgumentException( + MessageFormat.format("Path [{0}] is too short, not deleting", + file.getAbsolutePath())); + } + if (file.exists()) { + if (file.isDirectory()) { + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + delete(child); + } + } + } + if (!file.delete()) { + throw new RuntimeException( + MessageFormat.format("Could not delete path [{0}]", + file.getAbsolutePath())); + } + } + } + + @Test + public void testExpandedClasspath() throws Exception { + //picking a class that is for sure in a directory in the classpath + //in this case the JAR is created on the fly + String jar = JarFinder.getJar(TestJarFinder.class); + Assert.assertTrue(new File(jar).exists()); + } + + @Test + public void testExistingManifest() throws Exception { + File dir = new File(System.getProperty("test.build.dir", "target/test-dir"), + TestJarFinder.class.getName() + "-testExistingManifest"); + delete(dir); + dir.mkdirs(); + + File metaInfDir = new File(dir, "META-INF"); + metaInfDir.mkdirs(); + File manifestFile = new File(metaInfDir, "MANIFEST.MF"); + Manifest manifest = new Manifest(); + OutputStream os = new FileOutputStream(manifestFile); + manifest.write(os); + os.close(); + + File propsFile = new File(dir, "props.properties"); + Writer writer = new FileWriter(propsFile); + new Properties().store(writer, ""); + writer.close(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JarOutputStream zos = new JarOutputStream(baos); + JarFinder.jarDir(dir, "", zos); + JarInputStream jis = + new JarInputStream(new ByteArrayInputStream(baos.toByteArray())); + Assert.assertNotNull(jis.getManifest()); + jis.close(); + } + + @Test + public void testNoManifest() throws Exception { + File dir = new File(System.getProperty("test.build.dir", "target/test-dir"), + TestJarFinder.class.getName() + "-testNoManifest"); + delete(dir); + dir.mkdirs(); + File propsFile = new File(dir, "props.properties"); + Writer writer = new FileWriter(propsFile); + new Properties().store(writer, ""); + writer.close(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JarOutputStream zos = new JarOutputStream(baos); + JarFinder.jarDir(dir, "", zos); + JarInputStream jis = + new JarInputStream(new ByteArrayInputStream(baos.toByteArray())); + Assert.assertNotNull(jis.getManifest()); + jis.close(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/Mocking.java b/src/test/java/org/apache/hadoop/hbase/master/Mocking.java new file mode 100644 index 0000000..6dd379f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/Mocking.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; + +/** + * Package scoped mocking utility. + */ +public class Mocking { + + static void waitForRegionPendingOpenInRIT(AssignmentManager am, String encodedName) + throws InterruptedException { + // We used to do a check like this: + //!Mocking.verifyRegionState(this.watcher, REGIONINFO, EventType.M_ZK_REGION_OFFLINE)) { + // There is a race condition with this: because we may do the transition to + // RS_ZK_REGION_OPENING before the RIT is internally updated. We need to wait for the + // RIT to be as we need it to be instead. This cannot happen in a real cluster as we + // update the RIT before sending the openRegion request. + + boolean wait = true; + while (wait) { + RegionState state = am.getRegionsInTransition().get(encodedName); + if (state != null && state.isPendingOpen()){ + wait = false; + } else { + Thread.sleep(1); + } + } + } + + static void waitForRegionOfflineInRIT(AssignmentManager am, String encodedName) + throws InterruptedException { + boolean wait = true; + while (wait) { + RegionState state = am.getRegionsInTransition().get(encodedName); + if (state != null && state.isOffline()) { + wait = false; + } else { + Thread.sleep(1); + } + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java new file mode 100644 index 0000000..05f6b1a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java @@ -0,0 +1,315 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.concurrent.Semaphore; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.monitoring.MonitoredTask; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test the {@link ActiveMasterManager}. + */ +@Category(MediumTests.class) +public class TestActiveMasterManager { + private final static Log LOG = LogFactory.getLog(TestActiveMasterManager.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + @Test public void testRestartMaster() throws IOException, KeeperException { + ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + "testActiveMasterManagerFromZK", null, true); + try { + ZKUtil.deleteNode(zk, zk.masterAddressZNode); + ZKUtil.deleteNode(zk, zk.clusterStateZNode); + } catch(KeeperException.NoNodeException nne) {} + + // Create the master node with a dummy address + ServerName master = new ServerName("localhost", 1, System.currentTimeMillis()); + // Should not have a master yet + DummyMaster dummyMaster = new DummyMaster(zk,master); + ClusterStatusTracker clusterStatusTracker = + dummyMaster.getClusterStatusTracker(); + ActiveMasterManager activeMasterManager = + dummyMaster.getActiveMasterManager(); + assertFalse(activeMasterManager.clusterHasActiveMaster.get()); + + // First test becoming the active master uninterrupted + MonitoredTask status = Mockito.mock(MonitoredTask.class); + clusterStatusTracker.setClusterUp(); + + activeMasterManager.blockUntilBecomingActiveMaster(status,clusterStatusTracker); + assertTrue(activeMasterManager.clusterHasActiveMaster.get()); + assertMaster(zk, master); + + // Now pretend master restart + DummyMaster secondDummyMaster = new DummyMaster(zk,master); + ActiveMasterManager secondActiveMasterManager = + secondDummyMaster.getActiveMasterManager(); + assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get()); + activeMasterManager.blockUntilBecomingActiveMaster(status,clusterStatusTracker); + assertTrue(activeMasterManager.clusterHasActiveMaster.get()); + assertMaster(zk, master); + } + + /** + * Unit tests that uses ZooKeeper but does not use the master-side methods + * but rather acts directly on ZK. + * @throws Exception + */ + @Test + public void testActiveMasterManagerFromZK() throws Exception { + ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + "testActiveMasterManagerFromZK", null, true); + try { + ZKUtil.deleteNode(zk, zk.masterAddressZNode); + ZKUtil.deleteNode(zk, zk.clusterStateZNode); + } catch(KeeperException.NoNodeException nne) {} + + // Create the master node with a dummy address + ServerName firstMasterAddress = + new ServerName("localhost", 1, System.currentTimeMillis()); + ServerName secondMasterAddress = + new ServerName("localhost", 2, System.currentTimeMillis()); + + // Should not have a master yet + DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress); + ActiveMasterManager activeMasterManager = + ms1.getActiveMasterManager(); + assertFalse(activeMasterManager.clusterHasActiveMaster.get()); + + // First test becoming the active master uninterrupted + ClusterStatusTracker clusterStatusTracker = + ms1.getClusterStatusTracker(); + clusterStatusTracker.setClusterUp(); + activeMasterManager.blockUntilBecomingActiveMaster( + Mockito.mock(MonitoredTask.class),clusterStatusTracker); + assertTrue(activeMasterManager.clusterHasActiveMaster.get()); + assertMaster(zk, firstMasterAddress); + + // New manager will now try to become the active master in another thread + WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress); + t.start(); + // Wait for this guy to figure out there is another active master + // Wait for 1 second at most + int sleeps = 0; + while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) { + Thread.sleep(10); + sleeps++; + } + + // Both should see that there is an active master + assertTrue(activeMasterManager.clusterHasActiveMaster.get()); + assertTrue(t.manager.clusterHasActiveMaster.get()); + // But secondary one should not be the active master + assertFalse(t.isActiveMaster); + + // Close the first server and delete it's master node + ms1.stop("stopping first server"); + + // Use a listener to capture when the node is actually deleted + NodeDeletionListener listener = new NodeDeletionListener(zk, zk.masterAddressZNode); + zk.registerListener(listener); + + LOG.info("Deleting master node"); + ZKUtil.deleteNode(zk, zk.masterAddressZNode); + + // Wait for the node to be deleted + LOG.info("Waiting for active master manager to be notified"); + listener.waitForDeletion(); + LOG.info("Master node deleted"); + + // Now we expect the secondary manager to have and be the active master + // Wait for 1 second at most + sleeps = 0; + while(!t.isActiveMaster && sleeps < 100) { + Thread.sleep(10); + sleeps++; + } + LOG.debug("Slept " + sleeps + " times"); + + assertTrue(t.manager.clusterHasActiveMaster.get()); + assertTrue(t.isActiveMaster); + + LOG.info("Deleting master node"); + ZKUtil.deleteNode(zk, zk.masterAddressZNode); + } + + /** + * Assert there is an active master and that it has the specified address. + * @param zk + * @param thisMasterAddress + * @throws KeeperException + */ + private void assertMaster(ZooKeeperWatcher zk, + ServerName expectedAddress) + throws KeeperException { + ServerName readAddress = + ServerName.parseVersionedServerName(ZKUtil.getData(zk, zk.masterAddressZNode)); + assertNotNull(readAddress); + assertTrue(expectedAddress.equals(readAddress)); + } + + public static class WaitToBeMasterThread extends Thread { + + ActiveMasterManager manager; + DummyMaster dummyMaster; + boolean isActiveMaster; + + public WaitToBeMasterThread(ZooKeeperWatcher zk, ServerName address) { + this.dummyMaster = new DummyMaster(zk,address); + this.manager = this.dummyMaster.getActiveMasterManager(); + isActiveMaster = false; + } + + @Override + public void run() { + manager.blockUntilBecomingActiveMaster( + Mockito.mock(MonitoredTask.class), + this.dummyMaster.getClusterStatusTracker()); + LOG.info("Second master has become the active master!"); + isActiveMaster = true; + } + } + + public static class NodeDeletionListener extends ZooKeeperListener { + private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class); + + private Semaphore lock; + private String node; + + public NodeDeletionListener(ZooKeeperWatcher watcher, String node) { + super(watcher); + lock = new Semaphore(0); + this.node = node; + } + + @Override + public void nodeDeleted(String path) { + if(path.equals(node)) { + LOG.debug("nodeDeleted(" + path + ")"); + lock.release(); + } + } + + public void waitForDeletion() throws InterruptedException { + lock.acquire(); + } + } + + /** + * Dummy Master Implementation. + */ + public static class DummyMaster implements Server { + private volatile boolean stopped; + private ClusterStatusTracker clusterStatusTracker; + private ActiveMasterManager activeMasterManager; + + public DummyMaster(ZooKeeperWatcher zk, ServerName master) { + this.clusterStatusTracker = + new ClusterStatusTracker(zk, this); + clusterStatusTracker.start(); + + this.activeMasterManager = + new ActiveMasterManager(zk, master, this); + zk.registerListener(activeMasterManager); + } + + @Override + public void abort(final String msg, final Throwable t) {} + + @Override + public boolean isAborted() { + return false; + } + + @Override + public Configuration getConfiguration() { + return null; + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + return null; + } + + @Override + public ServerName getServerName() { + return null; + } + + @Override + public boolean isStopped() { + return this.stopped; + } + + @Override + public void stop(String why) { + this.stopped = true; + } + + @Override + public CatalogTracker getCatalogTracker() { + return null; + } + + public ClusterStatusTracker getClusterStatusTracker() { + return clusterStatusTracker; + } + + public ActiveMasterManager getActiveMasterManager() { + return activeMasterManager; + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java new file mode 100644 index 0000000..7d6ea31 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java @@ -0,0 +1,1204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HServerLoad; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionTestingUtility; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.executor.ExecutorService; +import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType; +import org.apache.hadoop.hbase.executor.RegionTransitionData; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; +import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State; +import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler; +import org.apache.hadoop.hbase.regionserver.RegionOpeningState; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.NodeExistsException; +import org.apache.zookeeper.Watcher; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.mockito.internal.util.reflection.Whitebox; + +import com.google.protobuf.ServiceException; + + +/** + * Test {@link AssignmentManager} + */ +@Category(MediumTests.class) +public class TestAssignmentManager { + private static final HBaseTestingUtility HTU = new HBaseTestingUtility(); + private static final ServerName SERVERNAME_A = + new ServerName("example.org", 1234, 5678); + private static final ServerName SERVERNAME_B = + new ServerName("example.org", 0, 5678); + private static final HRegionInfo REGIONINFO = + new HRegionInfo(Bytes.toBytes("t"), + HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); + private static final HRegionInfo REGIONINFO_2 = new HRegionInfo(Bytes.toBytes("t"), + Bytes.toBytes("a"),Bytes.toBytes( "b")); + private static int assignmentCount; + private static boolean enabling = false; + + // Mocked objects or; get redone for each test. + private Server server; + private ServerManager serverManager; + private ZooKeeperWatcher watcher; + private LoadBalancer balancer; + + @BeforeClass + public static void beforeClass() throws Exception { + HTU.startMiniZKCluster(); + } + + @AfterClass + public static void afterClass() throws IOException { + HTU.shutdownMiniZKCluster(); + } + + @Before + public void before() throws ZooKeeperConnectionException, IOException { + // TODO: Make generic versions of what we do below and put up in a mocking + // utility class or move up into HBaseTestingUtility. + + // Mock a Server. Have it return a legit Configuration and ZooKeeperWatcher. + // If abort is called, be sure to fail the test (don't just swallow it + // silently as is mockito default). + this.server = Mockito.mock(Server.class); + Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration()); + this.watcher = + new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true); + Mockito.when(server.getZooKeeper()).thenReturn(this.watcher); + Mockito.doThrow(new RuntimeException("Aborted")). + when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject()); + + // Mock a ServerManager. Say server SERVERNAME_{A,B} are online. Also + // make it so if close or open, we return 'success'. + this.serverManager = Mockito.mock(ServerManager.class); + Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true); + Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true); + final Map onlineServers = new HashMap(); + onlineServers.put(SERVERNAME_B, new HServerLoad()); + onlineServers.put(SERVERNAME_A, new HServerLoad()); + Mockito.when(this.serverManager.getOnlineServersList()).thenReturn( + new ArrayList(onlineServers.keySet())); + Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers); + Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)). + thenReturn(true); + Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)). + thenReturn(true); + // Ditto on open. + Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1)). + thenReturn(RegionOpeningState.OPENED); + Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1)). + thenReturn(RegionOpeningState.OPENED); + } + + @After + public void after() throws KeeperException { + if (this.watcher != null) { + // Clean up all znodes + ZKAssign.deleteAllNodes(this.watcher); + this.watcher.close(); + } + } + + /** + * Test a balance going on at same time as a master failover + * + * @throws IOException + * @throws KeeperException + * @throws InterruptedException + */ + @Test(timeout = 60000) + public void testBalanceOnMasterFailoverScenarioWithOpenedNode() + throws IOException, KeeperException, InterruptedException { + AssignmentManagerWithExtrasForTesting am = + setUpMockedAssignmentManager(this.server, this.serverManager); + try { + createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO); + startFakeFailedOverMasterAssignmentManager(am, this.watcher); + while (!am.processRITInvoked) Thread.sleep(1); + // Now fake the region closing successfully over on the regionserver; the + // regionserver will have set the region in CLOSED state. This will + // trigger callback into AM. The below zk close call is from the RS close + // region handler duplicated here because its down deep in a private + // method hard to expose. + int versionid = + ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1); + assertNotSame(versionid, -1); + Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName()); + + // Get the OFFLINE version id. May have to wait some for it to happen. + // OPENING below + while (true) { + int vid = ZKAssign.getVersion(this.watcher, REGIONINFO); + if (vid != versionid) { + versionid = vid; + break; + } + } + assertNotSame(-1, versionid); + // This uglyness below is what the openregionhandler on RS side does. + versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO, + SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE, + EventType.RS_ZK_REGION_OPENING, versionid); + assertNotSame(-1, versionid); + // Move znode from OPENING to OPENED as RS does on successful open. + versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, + SERVERNAME_B, versionid); + assertNotSame(-1, versionid); + am.gate.set(false); + // Block here until our znode is cleared or until this test times out. + ZKAssign.blockUntilNoRIT(watcher); + } finally { + am.getExecutorService().shutdown(); + am.shutdown(); + } + } + + @Test(timeout = 60000) + public void testBalanceOnMasterFailoverScenarioWithClosedNode() + throws IOException, KeeperException, InterruptedException { + AssignmentManagerWithExtrasForTesting am = + setUpMockedAssignmentManager(this.server, this.serverManager); + try { + createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO); + startFakeFailedOverMasterAssignmentManager(am, this.watcher); + while (!am.processRITInvoked) Thread.sleep(1); + // Now fake the region closing successfully over on the regionserver; the + // regionserver will have set the region in CLOSED state. This will + // trigger callback into AM. The below zk close call is from the RS close + // region handler duplicated here because its down deep in a private + // method hard to expose. + int versionid = + ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1); + assertNotSame(versionid, -1); + am.gate.set(false); + Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName()); + + // Get current versionid else will fail on transition from OFFLINE to + // OPENING below + while (true) { + int vid = ZKAssign.getVersion(this.watcher, REGIONINFO); + if (vid != versionid) { + versionid = vid; + break; + } + } + assertNotSame(-1, versionid); + // This uglyness below is what the openregionhandler on RS side does. + versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO, + SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE, + EventType.RS_ZK_REGION_OPENING, versionid); + assertNotSame(-1, versionid); + // Move znode from OPENING to OPENED as RS does on successful open. + versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, + SERVERNAME_B, versionid); + assertNotSame(-1, versionid); + + // Block here until our znode is cleared or until this test timesout. + ZKAssign.blockUntilNoRIT(watcher); + } finally { + am.getExecutorService().shutdown(); + am.shutdown(); + } + } + + @Test(timeout = 60000) + public void testBalanceOnMasterFailoverScenarioWithOfflineNode() + throws IOException, KeeperException, InterruptedException { + AssignmentManagerWithExtrasForTesting am = + setUpMockedAssignmentManager(this.server, this.serverManager); + try { + createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO); + startFakeFailedOverMasterAssignmentManager(am, this.watcher); + while (!am.processRITInvoked) Thread.sleep(1); + // Now fake the region closing successfully over on the regionserver; the + // regionserver will have set the region in CLOSED state. This will + // trigger callback into AM. The below zk close call is from the RS close + // region handler duplicated here because its down deep in a private + // method hard to expose. + int versionid = + ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1); + assertNotSame(versionid, -1); + Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName()); + + am.gate.set(false); + // Get current versionid else will fail on transition from OFFLINE to + // OPENING below + while (true) { + int vid = ZKAssign.getVersion(this.watcher, REGIONINFO); + if (vid != versionid) { + versionid = vid; + break; + } + } + assertNotSame(-1, versionid); + // This uglyness below is what the openregionhandler on RS side does. + versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO, + SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE, + EventType.RS_ZK_REGION_OPENING, versionid); + assertNotSame(-1, versionid); + // Move znode from OPENING to OPENED as RS does on successful open. + versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, + SERVERNAME_B, versionid); + assertNotSame(-1, versionid); + // Block here until our znode is cleared or until this test timesout. + ZKAssign.blockUntilNoRIT(watcher); + } finally { + am.getExecutorService().shutdown(); + am.shutdown(); + } + } + + private void createRegionPlanAndBalance(final AssignmentManager am, + final ServerName from, final ServerName to, final HRegionInfo hri) { + // Call the balance function but fake the region being online first at + // servername from. + am.regionOnline(hri, from); + // Balance region from 'from' to 'to'. It calls unassign setting CLOSING state + // up in zk. Create a plan and balance + am.balance(new RegionPlan(hri, from, to)); + } + + + /** + * Tests AssignmentManager balance function. Runs a balance moving a region + * from one server to another mocking regionserver responding over zk. + * @throws IOException + * @throws KeeperException + * @throws InterruptedException + */ + @Test(timeout = 60000) + public void testBalance() + throws IOException, KeeperException, InterruptedException { + // Create and startup an executor. This is used by AssignmentManager + // handling zk callbacks. + ExecutorService executor = startupMasterExecutor("testBalanceExecutor"); + + // We need a mocked catalog tracker. + CatalogTracker ct = Mockito.mock(CatalogTracker.class); + LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server + .getConfiguration()); + // Create an AM. + AssignmentManager am = new AssignmentManager(this.server, + this.serverManager, ct, balancer, executor); + try { + // Make sure our new AM gets callbacks; once registered, can't unregister. + // Thats ok because we make a new zk watcher for each test. + this.watcher.registerListenerFirst(am); + // Call the balance function but fake the region being online first at + // SERVERNAME_A. Create a balance plan. + am.regionOnline(REGIONINFO, SERVERNAME_A); + // Balance region from A to B. + RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B); + am.balance(plan); + + // Now fake the region closing successfully over on the regionserver; the + // regionserver will have set the region in CLOSED state. This will + // trigger callback into AM. The below zk close call is from the RS close + // region handler duplicated here because its down deep in a private + // method hard to expose. + int versionid = + ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1); + assertNotSame(versionid, -1); + // AM is going to notice above CLOSED and queue up a new assign. The + // assign will go to open the region in the new location set by the + // balancer. The zk node will be OFFLINE waiting for regionserver to + // transition it through OPENING, OPENED. Wait till we see the RIT + // before we proceed. + Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName()); + // Get current versionid else will fail on transition from OFFLINE to OPENING below + while (true) { + int vid = ZKAssign.getVersion(this.watcher, REGIONINFO); + if (vid != versionid) { + versionid = vid; + break; + } + } + assertNotSame(-1, versionid); + // This uglyness below is what the openregionhandler on RS side does. + versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO, + SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE, + EventType.RS_ZK_REGION_OPENING, versionid); + assertNotSame(-1, versionid); + // Move znode from OPENING to OPENED as RS does on successful open. + versionid = + ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid); + assertNotSame(-1, versionid); + // Wait on the handler removing the OPENED znode. + while(am.isRegionInTransition(REGIONINFO) != null) Threads.sleep(1); + } finally { + executor.shutdown(); + am.shutdown(); + // Clean up all znodes + ZKAssign.deleteAllNodes(this.watcher); + } + } + + /** + * Run a simple server shutdown handler. + * @throws KeeperException + * @throws IOException + */ + @Test + public void testShutdownHandler() throws KeeperException, IOException { + // Create and startup an executor. This is used by AssignmentManager + // handling zk callbacks. + ExecutorService executor = startupMasterExecutor("testShutdownHandler"); + + // We need a mocked catalog tracker. + CatalogTracker ct = Mockito.mock(CatalogTracker.class); + LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server + .getConfiguration()); + // Create an AM. + AssignmentManager am = + new AssignmentManager(this.server, this.serverManager, ct, balancer, executor); + try { + processServerShutdownHandler(ct, am, false, null); + } finally { + executor.shutdown(); + am.shutdown(); + // Clean up all znodes + ZKAssign.deleteAllNodes(this.watcher); + } + } + + /** + * To test closed region handler to remove rit and delete corresponding znode if region in pending + * close or closing while processing shutdown of a region server.(HBASE-5927). + * @throws KeeperException + * @throws IOException + */ + @Test + public void testSSHWhenDisableTableInProgress() + throws KeeperException, IOException { + testCaseWithPartiallyDisabledState(TableState.DISABLING, false); + testCaseWithPartiallyDisabledState(TableState.DISABLED, false); + } + + @Test + public void testSSHWhenDisablingTableRegionsInOpeningState() + throws KeeperException, IOException { + testCaseWithPartiallyDisabledState(TableState.DISABLING, true); + testCaseWithPartiallyDisabledState(TableState.DISABLED, true); + } + + + /** + * To test if the split region is removed from RIT if the region was in SPLITTING state + * but the RS has actually completed the splitting in META but went down. See HBASE-6070 + * and also HBASE-5806 + * @throws KeeperException + * @throws IOException + */ + @Test + public void testSSHWhenSplitRegionInProgress() + throws KeeperException, IOException, Exception { + // true indicates the region is split but still in RIT + testCaseWithSplitRegionPartial(true); + // false indicate the region is not split + testCaseWithSplitRegionPartial(false); + + } + + private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException, IOException, + NodeExistsException, InterruptedException { + // Create and startup an executor. This is used by AssignmentManager + // handling zk callbacks. + ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress"); + + // We need a mocked catalog tracker. + CatalogTracker ct = Mockito.mock(CatalogTracker.class); + // Create an AM. + AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager); + // adding region to regions and servers maps. + am.regionOnline(REGIONINFO, SERVERNAME_A); + // adding region in pending close. + am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO, + State.SPLITTING, System.currentTimeMillis(), SERVERNAME_A)); + am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString()); + + RegionTransitionData data = new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING, + REGIONINFO.getRegionName(), SERVERNAME_A); + String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName()); + // create znode in M_ZK_REGION_CLOSING state. + ZKUtil.createAndWatch(this.watcher, node, data.getBytes()); + + try { + processServerShutdownHandler(ct, am, regionSplitDone, null); + // check znode deleted or not. + // In both cases the znode should be deleted. + + if(regionSplitDone){ + assertTrue("Region state of region in SPLITTING should be removed from rit.", + am.regionsInTransition.isEmpty()); + } + else{ + while (!am.assignInvoked) { + Thread.sleep(1); + } + assertTrue("Assign should be invoked.", am.assignInvoked); + } + } finally { + REGIONINFO.setOffline(false); + REGIONINFO.setSplit(false); + executor.shutdown(); + am.shutdown(); + // Clean up all znodes + ZKAssign.deleteAllNodes(this.watcher); + } + } + + private void testCaseWithPartiallyDisabledState(TableState state, boolean opening) + throws KeeperException, IOException, NodeExistsException { + // Create and startup an executor. This is used by AssignmentManager + // handling zk callbacks. + ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress"); + + // We need a mocked catalog tracker. + CatalogTracker ct = Mockito.mock(CatalogTracker.class); + LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration()); + // Create an AM. + AssignmentManager am = new AssignmentManager(this.server, this.serverManager, ct, balancer, + executor); + if (opening) { + am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO, + State.OPENING, System.currentTimeMillis(), SERVERNAME_A)); + } else { + // adding region to regions and servers maps. + am.regionOnline(REGIONINFO, SERVERNAME_A); + // adding region in pending close. + am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO, + State.PENDING_CLOSE, System.currentTimeMillis(), SERVERNAME_A)); + } + if (state == TableState.DISABLING) { + am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString()); + } else { + am.getZKTable().setDisabledTable(REGIONINFO.getTableNameAsString()); + } + RegionTransitionData data = null; + if (opening) { + data = + new RegionTransitionData(EventType.RS_ZK_REGION_OPENING, REGIONINFO.getRegionName(), + SERVERNAME_A); + + } else { + data = + new RegionTransitionData(EventType.M_ZK_REGION_CLOSING, REGIONINFO.getRegionName(), + SERVERNAME_A); + } + String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName()); + // create znode in M_ZK_REGION_CLOSING state. + ZKUtil.createAndWatch(this.watcher, node, data.getBytes()); + + try { + processServerShutdownHandler(ct, am, false, null); + // check znode deleted or not. + // In both cases the znode should be deleted. + assertTrue("The znode should be deleted.",ZKUtil.checkExists(this.watcher, node) == -1); + assertTrue("Region state of region in pending close should be removed from rit.", + am.regionsInTransition.isEmpty()); + } finally { + executor.shutdown(); + am.shutdown(); + // Clean up all znodes + ZKAssign.deleteAllNodes(this.watcher); + } + } + + private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am, + boolean splitRegion, ServerName sn) + throws IOException { + // Make sure our new AM gets callbacks; once registered, can't unregister. + // Thats ok because we make a new zk watcher for each test. + this.watcher.registerListenerFirst(am); + // Need to set up a fake scan of meta for the servershutdown handler + // Make an RS Interface implementation. Make it so a scanner can go against it. + HRegionInterface implementation = Mockito.mock(HRegionInterface.class); + // Get a meta row result that has region up on SERVERNAME_A + + Result r = null; + if (sn == null) { + if (splitRegion) { + r = getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A); + } else { + r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A); + } + } else { + if (sn.equals(SERVERNAME_A)) { + r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A); + } else if (sn.equals(SERVERNAME_B)) { + r = new Result(new KeyValue[0]); + } + } + + Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())). + thenReturn(System.currentTimeMillis()); + // Return a good result first and then return null to indicate end of scan + Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())). + thenReturn(new Result [] {r}, (Result [])null); + + // Get a connection w/ mocked up common methods. + HConnection connection = + HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(), + implementation, SERVERNAME_B, REGIONINFO); + + // Make it so we can get a catalogtracker from servermanager.. .needed + // down in guts of server shutdown handler. + Mockito.when(ct.getConnection()).thenReturn(connection); + Mockito.when(this.server.getCatalogTracker()).thenReturn(ct); + + // Now make a server shutdown handler instance and invoke process. + // Have it that SERVERNAME_A died. + DeadServer deadServers = new DeadServer(); + deadServers.add(SERVERNAME_A); + // I need a services instance that will return the AM + MasterServices services = Mockito.mock(MasterServices.class); + Mockito.when(services.getAssignmentManager()).thenReturn(am); + Mockito.when(services.getZooKeeper()).thenReturn(this.watcher); + ServerShutdownHandler handler = null; + if (sn != null) { + handler = new ServerShutdownHandler(this.server, services, deadServers, sn, false); + } else { + handler = new ServerShutdownHandler(this.server, services, deadServers, SERVERNAME_A, false); + } + handler.process(); + // The region in r will have been assigned. It'll be up in zk as unassigned. + } + + /** + * @param sn ServerName to use making startcode and server in meta + * @param hri Region to serialize into HRegionInfo + * @return A mocked up Result that fakes a Get on a row in the + * .META. table. + * @throws IOException + */ + private Result getMetaTableRowResult(final HRegionInfo hri, + final ServerName sn) + throws IOException { + // TODO: Move to a utilities class. More than one test case can make use + // of this facility. + List kvs = new ArrayList(); + kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, + HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri))); + kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, + HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, + Bytes.toBytes(sn.getHostAndPort()))); + kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, + HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, + Bytes.toBytes(sn.getStartcode()))); + return new Result(kvs); + } + + /** + * @param sn ServerName to use making startcode and server in meta + * @param hri Region to serialize into HRegionInfo + * @return A mocked up Result that fakes a Get on a row in the + * .META. table. + * @throws IOException + */ + private Result getMetaTableRowResultAsSplitRegion(final HRegionInfo hri, final ServerName sn) + throws IOException { + hri.setOffline(true); + hri.setSplit(true); + return getMetaTableRowResult(hri, sn); + } + + /** + * Create and startup executor pools. Start same set as master does (just + * run a few less). + * @param name Name to give our executor + * @return Created executor (be sure to call shutdown when done). + */ + private ExecutorService startupMasterExecutor(final String name) { + // TODO: Move up into HBaseTestingUtility? Generally useful. + ExecutorService executor = new ExecutorService(name); + executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3); + executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3); + executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3); + executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3); + return executor; + } + + @Test + public void testUnassignWithSplitAtSameTime() throws KeeperException, IOException { + // Region to use in test. + final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO; + // First amend the servermanager mock so that when we do send close of the + // first meta region on SERVERNAME_A, it will return true rather than + // default null. + Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true); + // Need a mocked catalog tracker. + CatalogTracker ct = Mockito.mock(CatalogTracker.class); + LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server + .getConfiguration()); + // Create an AM. + AssignmentManager am = + new AssignmentManager(this.server, this.serverManager, ct, balancer, null); + try { + // First make sure my mock up basically works. Unassign a region. + unassign(am, SERVERNAME_A, hri); + // This delete will fail if the previous unassign did wrong thing. + ZKAssign.deleteClosingNode(this.watcher, hri); + // Now put a SPLITTING region in the way. I don't have to assert it + // go put in place. This method puts it in place then asserts it still + // owns it by moving state from SPLITTING to SPLITTING. + int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A); + // Now, retry the unassign with the SPLTTING in place. It should just + // complete without fail; a sort of 'silent' recognition that the + // region to unassign has been split and no longer exists: TOOD: what if + // the split fails and the parent region comes back to life? + unassign(am, SERVERNAME_A, hri); + // This transition should fail if the znode has been messed with. + ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A, + EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version); + assertTrue(am.isRegionInTransition(hri) == null); + } finally { + am.shutdown(); + } + } + + /** + * Tests the processDeadServersAndRegionsInTransition should not fail with NPE + * when it failed to get the children. Let's abort the system in this + * situation + * @throws ServiceException + */ + @Test(timeout = 60000) + public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE() + throws IOException, KeeperException, InterruptedException, ServiceException { + final RecoverableZooKeeper recoverableZk = Mockito + .mock(RecoverableZooKeeper.class); + AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager( + this.server, this.serverManager); + Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest", + null) { + public RecoverableZooKeeper getRecoverableZooKeeper() { + return recoverableZk; + } + }; + ((ZooKeeperWatcher) zkw).registerListener(am); + Mockito.doThrow(new InterruptedException()).when(recoverableZk) + .getChildren("/hbase/unassigned", zkw); + am.setWatcher((ZooKeeperWatcher) zkw); + try { + am.processDeadServersAndRegionsInTransition(); + fail("Expected to abort"); + } catch (NullPointerException e) { + fail("Should not throw NPE"); + } catch (RuntimeException e) { + assertEquals("Aborted", e.getLocalizedMessage()); + } + } + + /** + * Creates a new ephemeral node in the SPLITTING state for the specified region. + * Create it ephemeral in case regionserver dies mid-split. + * + *

    Does not transition nodes from other states. If a node already exists + * for this region, a {@link NodeExistsException} will be thrown. + * + * @param zkw zk reference + * @param region region to be created as offline + * @param serverName server event originates from + * @return Version of znode created. + * @throws KeeperException + * @throws IOException + */ + // Copied from SplitTransaction rather than open the method over there in + // the regionserver package. + private static int createNodeSplitting(final ZooKeeperWatcher zkw, + final HRegionInfo region, final ServerName serverName) + throws KeeperException, IOException { + RegionTransitionData data = + new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING, + region.getRegionName(), serverName); + + String node = ZKAssign.getNodeName(zkw, region.getEncodedName()); + if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, data.getBytes())) { + throw new IOException("Failed create of ephemeral " + node); + } + // Transition node from SPLITTING to SPLITTING and pick up version so we + // can be sure this znode is ours; version is needed deleting. + return transitionNodeSplitting(zkw, region, serverName, -1); + } + + // Copied from SplitTransaction rather than open the method over there in + // the regionserver package. + private static int transitionNodeSplitting(final ZooKeeperWatcher zkw, + final HRegionInfo parent, + final ServerName serverName, final int version) + throws KeeperException, IOException { + return ZKAssign.transitionNode(zkw, parent, serverName, + EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version); + } + + private void unassign(final AssignmentManager am, final ServerName sn, + final HRegionInfo hri) { + // Before I can unassign a region, I need to set it online. + am.regionOnline(hri, sn); + // Unassign region. + am.unassign(hri); + } + + /** + * Create an {@link AssignmentManagerWithExtrasForTesting} that has mocked + * {@link CatalogTracker} etc. + * @param server + * @param manager + * @return An AssignmentManagerWithExtras with mock connections, etc. + * @throws IOException + * @throws KeeperException + */ + private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server, + final ServerManager manager) + throws IOException, KeeperException { + // We need a mocked catalog tracker. Its used by our AM instance. + CatalogTracker ct = Mockito.mock(CatalogTracker.class); + // Make an RS Interface implementation. Make it so a scanner can go against + // it and a get to return the single region, REGIONINFO, this test is + // messing with. Needed when "new master" joins cluster. AM will try and + // rebuild its list of user regions and it will also get the HRI that goes + // with an encoded name by doing a Get on .META. + HRegionInterface ri = Mockito.mock(HRegionInterface.class); + // Get a meta row result that has region up on SERVERNAME_A for REGIONINFO + Result[] result = null; + if (enabling) { + result = new Result[2]; + result[0] = getMetaTableRowResult(REGIONINFO, SERVERNAME_A); + result[1] = getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A); + } + Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A); + Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())). + thenReturn(System.currentTimeMillis()); + if (enabling) { + Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(result, result, result, + (Result[]) null); + // If a get, return the above result too for REGIONINFO_2 + Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn( + getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A)); + } else { + // Return good result 'r' first and then return null to indicate end of scan + Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r }); + // If a get, return the above result too for REGIONINFO + Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(r); + } + // Get a connection w/ mocked up common methods. + HConnection connection = HConnectionTestingUtility. + getMockedConnectionAndDecorate(HTU.getConfiguration(), ri, SERVERNAME_B, + REGIONINFO); + // Make it so we can get the connection from our mocked catalogtracker + Mockito.when(ct.getConnection()).thenReturn(connection); + // Create and startup an executor. Used by AM handling zk callbacks. + ExecutorService executor = startupMasterExecutor("mockedAMExecutor"); + this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration()); + AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting( + server, manager, ct, balancer, executor); + return am; + } + + /** + * TestCase verifies that the regionPlan is updated whenever a region fails to open + * and the master tries to process RS_ZK_FAILED_OPEN state.(HBASE-5546). + */ + @Test + public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException, + ServiceException, InterruptedException { + this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + MockedLoadBalancer.class, LoadBalancer.class); + AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, + this.serverManager); + try { + // Boolean variable used for waiting until randomAssignment is called and new + // plan is generated. + AtomicBoolean gate = new AtomicBoolean(false); + if (balancer instanceof MockedLoadBalancer) { + ((MockedLoadBalancer) balancer).setGateVariable(gate); + } + ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A); + int v = ZKAssign.getVersion(this.watcher, REGIONINFO); + ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE, + EventType.RS_ZK_REGION_FAILED_OPEN, v); + String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName()); + RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(), + SERVERNAME_A); + am.regionsInTransition.put(REGIONINFO.getEncodedName(), state); + // a dummy plan inserted into the regionPlans. This plan is cleared and new one is formed + am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A)); + RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName()); + List serverList = new ArrayList(2); + serverList.add(SERVERNAME_B); + Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(serverList); + am.nodeDataChanged(path); + // here we are waiting until the random assignment in the load balancer is called. + while (!gate.get()) { + Thread.sleep(10); + } + // new region plan may take some time to get updated after random assignment is called and + // gate is set to true. + RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName()); + while (newRegionPlan == null) { + Thread.sleep(10); + newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName()); + } + // the new region plan created may contain the same RS as destination but it should + // be new plan. + assertNotSame("Same region plan should not come", regionPlan, newRegionPlan); + assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals( + newRegionPlan.getDestination()))); + Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName()); + } finally { + this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + DefaultLoadBalancer.class, LoadBalancer.class); + am.shutdown(); + } + } + + /** + * Test verifies whether assignment is skipped for regions of tables in DISABLING state during + * clean cluster startup. See HBASE-6281. + * + * @throws KeeperException + * @throws IOException + * @throws Exception + */ + @Test + public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup() + throws KeeperException, IOException, Exception { + this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + MockedLoadBalancer.class, LoadBalancer.class); + Mockito.when(this.serverManager.getOnlineServers()).thenReturn( + new HashMap(0)); + List destServers = new ArrayList(1); + destServers.add(SERVERNAME_A); + Mockito.when(this.serverManager.getDrainingServersList()).thenReturn(destServers); + // To avoid cast exception in DisableTableHandler process. + //Server server = new HMaster(HTU.getConfiguration()); + HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0); + Server server = new HMaster(HTU.getConfiguration()); + AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server, + this.serverManager); + AtomicBoolean gate = new AtomicBoolean(false); + if (balancer instanceof MockedLoadBalancer) { + ((MockedLoadBalancer) balancer).setGateVariable(gate); + } + try{ + // set table in disabling state. + am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString()); + am.joinCluster(); + // should not call retainAssignment if we get empty regions in assignAllUserRegions. + assertFalse( + "Assign should not be invoked for disabling table regions during clean cluster startup.", + gate.get()); + // need to change table state from disabling to disabled. + assertTrue("Table should be disabled.", + am.getZKTable().isDisabledTable(REGIONINFO.getTableNameAsString())); + } finally { + this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + DefaultLoadBalancer.class, LoadBalancer.class); + am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString()); + am.shutdown(); + } + } + + /** + * Test verifies whether all the enabling table regions assigned only once during master startup. + * + * @throws KeeperException + * @throws IOException + * @throws Exception + */ + @Test + public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception { + enabling = true; + this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + DefaultLoadBalancer.class, LoadBalancer.class); + Map serverAndLoad = new HashMap(); + serverAndLoad.put(SERVERNAME_A, null); + Mockito.when(this.serverManager.getOnlineServers()).thenReturn(serverAndLoad); + Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(false); + Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true); + HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0); + Server server = new HMaster(HTU.getConfiguration()); + Whitebox.setInternalState(server, "serverManager", this.serverManager); + assignmentCount = 0; + AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server, + this.serverManager); + am.regionOnline(new HRegionInfo("t1".getBytes(), HConstants.EMPTY_START_ROW, + HConstants.EMPTY_END_ROW), SERVERNAME_A); + am.gate.set(false); + try { + // set table in enabling state. + am.getZKTable().setEnablingTable(REGIONINFO.getTableNameAsString()); + ZKAssign.createNodeOffline(this.watcher, REGIONINFO_2, SERVERNAME_B); + + am.joinCluster(); + while (!am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString())) { + Thread.sleep(10); + } + assertEquals("Number of assignments should be equal.", 2, assignmentCount); + assertTrue("Table should be enabled.", + am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString())); + } finally { + enabling = false; + am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString()); + am.shutdown(); + ZKAssign.deleteAllNodes(this.watcher); + assignmentCount = 0; + } + } + + + + /** + * When region in transition if region server opening the region gone down then region assignment + * taking long time(Waiting for timeout monitor to trigger assign). HBASE-5396(HBASE-6060) fixes this + * scenario. This test case verifies whether SSH calling assign for the region in transition or not. + * + * @throws KeeperException + * @throws IOException + * @throws ServiceException + */ + @Test + public void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown() throws KeeperException, IOException, + ServiceException { + testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(true); + testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(false); + } + + private void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(boolean regionInOffline) + throws IOException, KeeperException, ServiceException { + // We need a mocked catalog tracker. + CatalogTracker ct = Mockito.mock(CatalogTracker.class); + // Create an AM. + AssignmentManagerWithExtrasForTesting am = + setUpMockedAssignmentManager(this.server, this.serverManager); + // adding region in pending open. + if (regionInOffline) { + ServerName MASTER_SERVERNAME = new ServerName("example.org", 1111, 1111); + am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO, + State.OFFLINE, System.currentTimeMillis(), MASTER_SERVERNAME)); + } else { + am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO, + State.OPENING, System.currentTimeMillis(), SERVERNAME_B)); + } + // adding region plan + am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B)); + am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString()); + + try { + processServerShutdownHandler(ct, am, false, SERVERNAME_A); + processServerShutdownHandler(ct, am, false, SERVERNAME_B); + if(regionInOffline){ + assertFalse("Assign should not be invoked.", am.assignInvoked); + } else { + assertTrue("Assign should be invoked.", am.assignInvoked); + } + } finally { + am.regionsInTransition.remove(REGIONINFO.getEncodedName()); + am.regionPlans.remove(REGIONINFO.getEncodedName()); + } + } + + /** + * Mocked load balancer class used in the testcase to make sure that the testcase waits until + * random assignment is called and the gate variable is set to true. + */ + public static class MockedLoadBalancer extends DefaultLoadBalancer { + private AtomicBoolean gate; + + public void setGateVariable(AtomicBoolean gate) { + this.gate = gate; + } + + @Override + public ServerName randomAssignment(HRegionInfo region, List servers) { + ServerName randomServerName = super.randomAssignment(region, servers); + this.gate.set(true); + return randomServerName; + } + + @Override + public Map> retainAssignment( + Map regions, List servers) { + this.gate.set(true); + return super.retainAssignment(regions, servers); + } + + } + + /** + * An {@link AssignmentManager} with some extra facility used testing + */ + class AssignmentManagerWithExtrasForTesting extends AssignmentManager { + // Keep a reference so can give it out below in {@link #getExecutorService} + private final ExecutorService es; + // Ditto for ct + private final CatalogTracker ct; + boolean processRITInvoked = false; + boolean assignInvoked = false; + AtomicBoolean gate = new AtomicBoolean(true); + + public AssignmentManagerWithExtrasForTesting(final Server master, + final ServerManager serverManager, final CatalogTracker catalogTracker, + final LoadBalancer balancer, final ExecutorService service) + throws KeeperException, IOException { + super(master, serverManager, catalogTracker, balancer, service); + this.es = service; + this.ct = catalogTracker; + } + + @Override + boolean processRegionInTransition(String encodedRegionName, + HRegionInfo regionInfo, + Map>> deadServers) + throws KeeperException, IOException { + this.processRITInvoked = true; + return super.processRegionInTransition(encodedRegionName, regionInfo, + deadServers); + } + @Override + void processRegionsInTransition(final RegionTransitionData data, + final HRegionInfo regionInfo, + final Map>> deadServers, + final int expectedVersion) throws KeeperException { + while (this.gate.get()) Threads.sleep(1); + super.processRegionsInTransition(data, regionInfo, deadServers, expectedVersion); + } + + @Override + public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan, + boolean hijack) { + if (enabling) { + assignmentCount++; + this.regionOnline(region, SERVERNAME_A); + } else { + assignInvoked = true; + super.assign(region, setOfflineInZK, forceNewPlan, hijack); + } + } + + @Override + public ServerName getRegionServerOfRegion(HRegionInfo hri) { + return SERVERNAME_A; + } + + /** reset the watcher */ + void setWatcher(ZooKeeperWatcher watcher) { + this.watcher = watcher; + } + + /** + * @return ExecutorService used by this instance. + */ + ExecutorService getExecutorService() { + return this.es; + } + + /** + * @return CatalogTracker used by this AM (Its a mock). + */ + CatalogTracker getCatalogTracker() { + return this.ct; + } + } + + /** + * Call joinCluster on the passed AssignmentManager. Do it in a thread + * so it runs independent of what all else is going on. Try to simulate + * an AM running insided a failed over master by clearing all in-memory + * AM state first. + */ + private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am, + final ZooKeeperWatcher watcher) { + // Make sure our new AM gets callbacks; once registered, we can't unregister. + // Thats ok because we make a new zk watcher for each test. + watcher.registerListenerFirst(am); + Thread t = new Thread("RunAmJoinCluster") { + public void run() { + // Call the joinCluster function as though we were doing a master + // failover at this point. It will stall just before we go to add + // the RIT region to our RIT Map in AM at processRegionsInTransition. + // First clear any inmemory state from AM so it acts like a new master + // coming on line. + am.regionsInTransition.clear(); + am.regionPlans.clear(); + try { + am.joinCluster(); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (KeeperException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }; + }; + t.start(); + while (!t.isAlive()) Threads.sleep(1); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java new file mode 100644 index 0000000..13a3c27 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java @@ -0,0 +1,911 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.TableDescriptors; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HConnectionTestingUtility; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.executor.ExecutorService; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +@Category(SmallTests.class) +public class TestCatalogJanitor { + /** + * Pseudo server for below tests. + * Be sure to call stop on the way out else could leave some mess around. + */ + class MockServer implements Server { + private final HConnection connection; + private final Configuration c; + private final CatalogTracker ct; + + MockServer(final HBaseTestingUtility htu) + throws NotAllMetaRegionsOnlineException, IOException, InterruptedException { + this.c = htu.getConfiguration(); + // Mock an HConnection and a HRegionInterface implementation. Have the + // HConnection return the HRI. Have the HRI return a few mocked up responses + // to make our test work. + this.connection = + HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c, + Mockito.mock(HRegionInterface.class), + new ServerName("example.org,12345,6789"), + HRegionInfo.FIRST_META_REGIONINFO); + // Set hbase.rootdir into test dir. + FileSystem fs = FileSystem.get(this.c); + Path rootdir = fs.makeQualified(new Path(this.c.get(HConstants.HBASE_DIR))); + this.c.set(HConstants.HBASE_DIR, rootdir.toString()); + this.ct = Mockito.mock(CatalogTracker.class); + HRegionInterface hri = Mockito.mock(HRegionInterface.class); + Mockito.when(this.ct.getConnection()).thenReturn(this.connection); + Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri); + } + + @Override + public CatalogTracker getCatalogTracker() { + return this.ct; + } + + @Override + public Configuration getConfiguration() { + return this.c; + } + + @Override + public ServerName getServerName() { + return new ServerName("mockserver.example.org", 1234, -1L); + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + return null; + } + + @Override + public void abort(String why, Throwable e) { + //no-op + } + + @Override + public boolean isAborted() { + return false; + } + + @Override + public boolean isStopped() { + return false; + } + + @Override + public void stop(String why) { + if (this.ct != null) { + this.ct.stop(); + } + if (this.connection != null) { + HConnectionManager.deleteConnection(this.connection.getConfiguration()); + } + } + } + + /** + * Mock MasterServices for tests below. + */ + class MockMasterServices implements MasterServices { + private final MasterFileSystem mfs; + private final AssignmentManager asm; + + MockMasterServices(final Server server) throws IOException { + this.mfs = new MasterFileSystem(server, this, null, false); + this.asm = Mockito.mock(AssignmentManager.class); + } + + @Override + public void checkTableModifiable(byte[] tableName) throws IOException { + //no-op + } + + @Override + public void createTable(HTableDescriptor desc, byte[][] splitKeys) + throws IOException { + // no-op + } + + @Override + public AssignmentManager getAssignmentManager() { + return this.asm; + } + + @Override + public ExecutorService getExecutorService() { + return null; + } + + @Override + public MasterFileSystem getMasterFileSystem() { + return this.mfs; + } + + @Override + public ServerManager getServerManager() { + return null; + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + return null; + } + + @Override + public CatalogTracker getCatalogTracker() { + return null; + } + + @Override + public Configuration getConfiguration() { + return mfs.conf; + } + + @Override + public ServerName getServerName() { + return null; + } + + @Override + public void abort(String why, Throwable e) { + //no-op + } + + @Override + public boolean isAborted() { + return false; + } + + private boolean stopped = false; + + @Override + public void stop(String why) { + stopped = true; + } + + @Override + public boolean isStopped() { + return stopped; + } + + @Override + public TableDescriptors getTableDescriptors() { + return new TableDescriptors() { + @Override + public HTableDescriptor remove(String tablename) throws IOException { + return null; + } + + @Override + public Map getAll() throws IOException { + return null; + } + + @Override + public HTableDescriptor get(byte[] tablename) + throws IOException { + return get(Bytes.toString(tablename)); + } + + @Override + public HTableDescriptor get(String tablename) + throws IOException { + return createHTableDescriptor(); + } + + @Override + public void add(HTableDescriptor htd) throws IOException { + } + }; + } + + @Override + public boolean isServerShutdownHandlerEnabled() { + return true; + } + + @Override + public MasterCoprocessorHost getCoprocessorHost() { + return null; + } + + @Override + public boolean registerProtocol(Class protocol, T handler) { + return false; + } + + @Override + public void deleteTable(byte[] tableName) throws IOException { + } + + @Override + public void modifyTable(byte[] tableName, HTableDescriptor descriptor) throws IOException { + } + + @Override + public void enableTable(byte[] tableName) throws IOException { + } + + @Override + public void disableTable(byte[] tableName) throws IOException { + } + + @Override + public void addColumn(byte[] tableName, HColumnDescriptor column) throws IOException { + } + + @Override + public void modifyColumn(byte[] tableName, HColumnDescriptor descriptor) throws IOException { + } + + @Override + public void deleteColumn(byte[] tableName, byte[] columnName) throws IOException { + } + + @Override + public boolean shouldSplitMetaSeparately() { + return false; + } + } + + @Test + public void testGetHRegionInfo() throws IOException { + assertNull(CatalogJanitor.getHRegionInfo(new Result())); + List kvs = new ArrayList(); + Result r = new Result(kvs); + assertNull(CatalogJanitor.getHRegionInfo(r)); + byte [] f = HConstants.CATALOG_FAMILY; + // Make a key value that doesn't have the expected qualifier. + kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f, + HConstants.SERVER_QUALIFIER, f)); + r = new Result(kvs); + assertNull(CatalogJanitor.getHRegionInfo(r)); + // Make a key that does not have a regioninfo value. + kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f, + HConstants.REGIONINFO_QUALIFIER, f)); + HRegionInfo hri = CatalogJanitor.getHRegionInfo(new Result(kvs)); + assertTrue(hri == null); + // OK, give it what it expects + kvs.clear(); + kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, f, + HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO))); + hri = CatalogJanitor.getHRegionInfo(new Result(kvs)); + assertNotNull(hri); + assertTrue(hri.equals(HRegionInfo.FIRST_META_REGIONINFO)); + } + + @Test + public void testCleanParent() throws IOException, InterruptedException { + HBaseTestingUtility htu = new HBaseTestingUtility(); + setRootDirAndCleanIt(htu, "testCleanParent"); + Server server = new MockServer(htu); + try { + MasterServices services = new MockMasterServices(server); + CatalogJanitor janitor = new CatalogJanitor(server, services); + // Create regions. + HTableDescriptor htd = new HTableDescriptor("table"); + htd.addFamily(new HColumnDescriptor("f")); + HRegionInfo parent = + new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), + Bytes.toBytes("eee")); + HRegionInfo splita = + new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), + Bytes.toBytes("ccc")); + HRegionInfo splitb = + new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), + Bytes.toBytes("eee")); + // Test that when both daughter regions are in place, that we do not + // remove the parent. + List kvs = new ArrayList(); + kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY, + HConstants.SPLITA_QUALIFIER, Writables.getBytes(splita))); + kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY, + HConstants.SPLITB_QUALIFIER, Writables.getBytes(splitb))); + Result r = new Result(kvs); + // Add a reference under splitA directory so we don't clear out the parent. + Path rootdir = services.getMasterFileSystem().getRootDir(); + Path tabledir = + HTableDescriptor.getTableDir(rootdir, htd.getName()); + Path storedir = Store.getStoreHomedir(tabledir, splita.getEncodedName(), + htd.getColumnFamilies()[0].getName()); + Reference ref = new Reference(Bytes.toBytes("ccc"), Reference.Range.top); + long now = System.currentTimeMillis(); + // Reference name has this format: StoreFile#REF_NAME_PARSER + Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); + FileSystem fs = services.getMasterFileSystem().getFileSystem(); + Path path = ref.write(fs, p); + assertTrue(fs.exists(path)); + assertFalse(janitor.cleanParent(parent, r)); + // Remove the reference file and try again. + assertTrue(fs.delete(p, true)); + assertTrue(janitor.cleanParent(parent, r)); + } finally { + server.stop("shutdown"); + } + } + + /** + * Make sure parent gets cleaned up even if daughter is cleaned up before it. + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testParentCleanedEvenIfDaughterGoneFirst() + throws IOException, InterruptedException { + parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst( + "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee")); + } + + /** + * Make sure last parent with empty end key gets cleaned up even if daughter is cleaned up before it. + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testLastParentCleanedEvenIfDaughterGoneFirst() + throws IOException, InterruptedException { + parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst( + "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]); + } + + /** + * Make sure parent with specified end key gets cleaned up even if daughter is cleaned up before it. + * + * @param rootDir the test case name, used as the HBase testing utility root + * @param lastEndKey the end key of the split parent + * @throws IOException + * @throws InterruptedException + */ + private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst( + final String rootDir, final byte[] lastEndKey) + throws IOException, InterruptedException { + HBaseTestingUtility htu = new HBaseTestingUtility(); + setRootDirAndCleanIt(htu, rootDir); + Server server = new MockServer(htu); + MasterServices services = new MockMasterServices(server); + CatalogJanitor janitor = new CatalogJanitor(server, services); + final HTableDescriptor htd = createHTableDescriptor(); + + // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc. + + // Parent + HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), + lastEndKey); + // Sleep a second else the encoded name on these regions comes out + // same for all with same start key and made in same second. + Thread.sleep(1001); + + // Daughter a + HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), + Bytes.toBytes("ccc")); + Thread.sleep(1001); + // Make daughters of daughter a; splitaa and splitab. + HRegionInfo splitaa = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), + Bytes.toBytes("bbb")); + HRegionInfo splitab = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"), + Bytes.toBytes("ccc")); + + // Daughter b + HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), + lastEndKey); + Thread.sleep(1001); + // Make Daughters of daughterb; splitba and splitbb. + HRegionInfo splitba = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), + Bytes.toBytes("ddd")); + HRegionInfo splitbb = new HRegionInfo(htd.getName(), Bytes.toBytes("ddd"), + lastEndKey); + + // First test that our Comparator works right up in CatalogJanitor. + // Just fo kicks. + SortedMap regions = + new TreeMap(new CatalogJanitor.SplitParentFirstComparator()); + // Now make sure that this regions map sorts as we expect it to. + regions.put(parent, createResult(parent, splita, splitb)); + regions.put(splitb, createResult(splitb, splitba, splitbb)); + regions.put(splita, createResult(splita, splitaa, splitab)); + // Assert its properly sorted. + int index = 0; + for (Map.Entry e: regions.entrySet()) { + if (index == 0) { + assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName())); + } else if (index == 1) { + assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName())); + } else if (index == 2) { + assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName())); + } + index++; + } + + // Now play around with the cleanParent function. Create a ref from splita + // up to the parent. + Path splitaRef = + createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false); + // Make sure actual super parent sticks around because splita has a ref. + assertFalse(janitor.cleanParent(parent, regions.get(parent))); + + //splitba, and split bb, do not have dirs in fs. That means that if + // we test splitb, it should get cleaned up. + assertTrue(janitor.cleanParent(splitb, regions.get(splitb))); + + // Now remove ref from splita to parent... so parent can be let go and so + // the daughter splita can be split (can't split if still references). + // BUT make the timing such that the daughter gets cleaned up before we + // can get a chance to let go of the parent. + FileSystem fs = FileSystem.get(htu.getConfiguration()); + assertTrue(fs.delete(splitaRef, true)); + // Create the refs from daughters of splita. + Path splitaaRef = + createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false); + Path splitabRef = + createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true); + + // Test splita. It should stick around because references from splitab, etc. + assertFalse(janitor.cleanParent(splita, regions.get(splita))); + + // Now clean up parent daughter first. Remove references from its daughters. + assertTrue(fs.delete(splitaaRef, true)); + assertTrue(fs.delete(splitabRef, true)); + assertTrue(janitor.cleanParent(splita, regions.get(splita))); + + // Super parent should get cleaned up now both splita and splitb are gone. + assertTrue(janitor.cleanParent(parent, regions.get(parent))); + + services.stop("test finished"); + janitor.join(); + } + + /** + * CatalogJanitor.scan() should not clean parent regions if their own + * parents are still referencing them. This ensures that grandfather regions + * do not point to deleted parent regions. + */ + @Test + public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception { + HBaseTestingUtility htu = new HBaseTestingUtility(); + setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents"); + Server server = new MockServer(htu); + MasterServices services = new MockMasterServices(server); + + final HTableDescriptor htd = createHTableDescriptor(); + + // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc. + + // Parent + HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), + new byte[0], true); + // Sleep a second else the encoded name on these regions comes out + // same for all with same start key and made in same second. + Thread.sleep(1001); + + // Daughter a + HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), + Bytes.toBytes("ccc"), true); + Thread.sleep(1001); + // Make daughters of daughter a; splitaa and splitab. + HRegionInfo splitaa = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), + Bytes.toBytes("bbb"), false); + HRegionInfo splitab = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"), + Bytes.toBytes("ccc"), false); + + // Daughter b + HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), + new byte[0]); + Thread.sleep(1001); + + final Map splitParents = + new TreeMap(new SplitParentFirstComparator()); + splitParents.put(parent, makeResultFromHRegionInfo(parent, splita, splitb)); + splita.setOffline(true);//simulate that splita goes offline when it is split + splitParents.put(splita, makeResultFromHRegionInfo(splita, splitaa, splitab)); + + CatalogJanitor janitor = spy(new CatalogJanitor(server, services)); + doReturn(new Pair>( + 10, splitParents)).when(janitor).getSplitParents(); + + //create ref from splita to parent + Path splitaRef = + createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false); + + //parent and A should not be removed + assertEquals(0, janitor.scan()); + + //now delete the ref + FileSystem fs = FileSystem.get(htu.getConfiguration()); + assertTrue(fs.delete(splitaRef, true)); + + //now, both parent, and splita can be deleted + assertEquals(2, janitor.scan()); + + services.stop("test finished"); + janitor.join(); + } + + @Test + public void testSplitParentFirstComparator() { + SplitParentFirstComparator comp = new SplitParentFirstComparator(); + final HTableDescriptor htd = createHTableDescriptor(); + + /* Region splits: + * + * rootRegion --- firstRegion --- firstRegiona + * | |- firstRegionb + * | + * |- lastRegion --- lastRegiona --- lastRegionaa + * | |- lastRegionab + * |- lastRegionb + * + * rootRegion : [] - [] + * firstRegion : [] - bbb + * lastRegion : bbb - [] + * firstRegiona : [] - aaa + * firstRegionb : aaa - bbb + * lastRegiona : bbb - ddd + * lastRegionb : ddd - [] + */ + + // root region + HRegionInfo rootRegion = new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW, + HConstants.EMPTY_END_ROW, true); + HRegionInfo firstRegion = new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW, + Bytes.toBytes("bbb"), true); + HRegionInfo lastRegion = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"), + HConstants.EMPTY_END_ROW, true); + + assertTrue(comp.compare(rootRegion, rootRegion) == 0); + assertTrue(comp.compare(firstRegion, firstRegion) == 0); + assertTrue(comp.compare(lastRegion, lastRegion) == 0); + assertTrue(comp.compare(rootRegion, firstRegion) < 0); + assertTrue(comp.compare(rootRegion, lastRegion) < 0); + assertTrue(comp.compare(firstRegion, lastRegion) < 0); + + //first region split into a, b + HRegionInfo firstRegiona = new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW, + Bytes.toBytes("aaa"), true); + HRegionInfo firstRegionb = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), + Bytes.toBytes("bbb"), true); + //last region split into a, b + HRegionInfo lastRegiona = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"), + Bytes.toBytes("ddd"), true); + HRegionInfo lastRegionb = new HRegionInfo(htd.getName(), Bytes.toBytes("ddd"), + HConstants.EMPTY_END_ROW, true); + + assertTrue(comp.compare(firstRegiona, firstRegiona) == 0); + assertTrue(comp.compare(firstRegionb, firstRegionb) == 0); + assertTrue(comp.compare(rootRegion, firstRegiona) < 0); + assertTrue(comp.compare(rootRegion, firstRegionb) < 0); + assertTrue(comp.compare(firstRegion, firstRegiona) < 0); + assertTrue(comp.compare(firstRegion, firstRegionb) < 0); + assertTrue(comp.compare(firstRegiona, firstRegionb) < 0); + + assertTrue(comp.compare(lastRegiona, lastRegiona) == 0); + assertTrue(comp.compare(lastRegionb, lastRegionb) == 0); + assertTrue(comp.compare(rootRegion, lastRegiona) < 0); + assertTrue(comp.compare(rootRegion, lastRegionb) < 0); + assertTrue(comp.compare(lastRegion, lastRegiona) < 0); + assertTrue(comp.compare(lastRegion, lastRegionb) < 0); + assertTrue(comp.compare(lastRegiona, lastRegionb) < 0); + + assertTrue(comp.compare(firstRegiona, lastRegiona) < 0); + assertTrue(comp.compare(firstRegiona, lastRegionb) < 0); + assertTrue(comp.compare(firstRegionb, lastRegiona) < 0); + assertTrue(comp.compare(firstRegionb, lastRegionb) < 0); + + HRegionInfo lastRegionaa = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"), + Bytes.toBytes("ccc"), false); + HRegionInfo lastRegionab = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), + Bytes.toBytes("ddd"), false); + + assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0); + assertTrue(comp.compare(lastRegiona, lastRegionab) < 0); + assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0); + + } + + @Test + public void testArchiveOldRegion() throws Exception { + String table = "table"; + HBaseTestingUtility htu = new HBaseTestingUtility(); + setRootDirAndCleanIt(htu, "testCleanParent"); + Server server = new MockServer(htu); + MasterServices services = new MockMasterServices(server); + + // create the janitor + CatalogJanitor janitor = new CatalogJanitor(server, services); + + // Create regions. + HTableDescriptor htd = new HTableDescriptor(table); + htd.addFamily(new HColumnDescriptor("f")); + HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee")); + HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc")); + HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee")); + // Test that when both daughter regions are in place, that we do not + // remove the parent. + List kvs = new ArrayList(); + kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY, + HConstants.SPLITA_QUALIFIER, Writables.getBytes(splita))); + kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY, + HConstants.SPLITB_QUALIFIER, Writables.getBytes(splitb))); + Result r = new Result(kvs); + + FileSystem fs = FileSystem.get(htu.getConfiguration()); + Path rootdir = services.getMasterFileSystem().getRootDir(); + // have to set the root directory since we use it in HFileDisposer to figure out to get to the + // archive directory. Otherwise, it just seems to pick the first root directory it can find (so + // the single test passes, but when the full suite is run, things get borked). + FSUtils.setRootDir(fs.getConf(), rootdir); + Path tabledir = HTableDescriptor.getTableDir(rootdir, htd.getName()); + Path storedir = Store.getStoreHomedir(tabledir, parent.getEncodedName(), + htd.getColumnFamilies()[0].getName()); + + // delete the file and ensure that the files have been archived + Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent, + tabledir, htd.getColumnFamilies()[0].getName()); + + // enable archiving, make sure that files get archived + addMockStoreFiles(2, services, storedir); + // get the current store files for comparison + FileStatus[] storeFiles = fs.listStatus(storedir); + for (FileStatus file : storeFiles) { + System.out.println("Have store file:" + file.getPath()); + } + + // do the cleaning of the parent + assertTrue(janitor.cleanParent(parent, r)); + + // and now check to make sure that the files have actually been archived + FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive); + assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs); + + // cleanup + services.stop("Test finished"); + server.stop("shutdown"); + janitor.join(); + } + + /** + * Test that if a store file with the same name is present as those already backed up cause the + * already archived files to be timestamped backup + */ + @Test + public void testDuplicateHFileResolution() throws Exception { + String table = "table"; + HBaseTestingUtility htu = new HBaseTestingUtility(); + setRootDirAndCleanIt(htu, "testCleanParent"); + Server server = new MockServer(htu); + MasterServices services = new MockMasterServices(server); + + // create the janitor + CatalogJanitor janitor = new CatalogJanitor(server, services); + + // Create regions. + HTableDescriptor htd = new HTableDescriptor(table); + htd.addFamily(new HColumnDescriptor("f")); + HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee")); + HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc")); + HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee")); + // Test that when both daughter regions are in place, that we do not + // remove the parent. + List kvs = new ArrayList(); + kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY, + HConstants.SPLITA_QUALIFIER, Writables.getBytes(splita))); + kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY, + HConstants.SPLITB_QUALIFIER, Writables.getBytes(splitb))); + Result r = new Result(kvs); + + FileSystem fs = FileSystem.get(htu.getConfiguration()); + + Path rootdir = services.getMasterFileSystem().getRootDir(); + // have to set the root directory since we use it in HFileDisposer to figure out to get to the + // archive directory. Otherwise, it just seems to pick the first root directory it can find (so + // the single test passes, but when the full suite is run, things get borked). + FSUtils.setRootDir(fs.getConf(), rootdir); + Path tabledir = HTableDescriptor.getTableDir(rootdir, parent.getTableName()); + Path storedir = Store.getStoreHomedir(tabledir, parent.getEncodedName(), + htd.getColumnFamilies()[0].getName()); + System.out.println("Old root:" + rootdir); + System.out.println("Old table:" + tabledir); + System.out.println("Old store:" + storedir); + + Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent, + tabledir, htd.getColumnFamilies()[0].getName()); + System.out.println("Old archive:" + storeArchive); + + // enable archiving, make sure that files get archived + addMockStoreFiles(2, services, storedir); + // get the current store files for comparison + FileStatus[] storeFiles = fs.listStatus(storedir); + + // do the cleaning of the parent + assertTrue(janitor.cleanParent(parent, r)); + + // and now check to make sure that the files have actually been archived + FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive); + assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs); + + // now add store files with the same names as before to check backup + // enable archiving, make sure that files get archived + addMockStoreFiles(2, services, storedir); + + // do the cleaning of the parent + assertTrue(janitor.cleanParent(parent, r)); + + // and now check to make sure that the files have actually been archived + archivedStoreFiles = fs.listStatus(storeArchive); + assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true); + + // cleanup + services.stop("Test finished"); + server.stop("shutdown"); + janitor.join(); + } + + private void addMockStoreFiles(int count, MasterServices services, Path storedir) + throws IOException { + // get the existing store files + FileSystem fs = services.getMasterFileSystem().getFileSystem(); + fs.mkdirs(storedir); + // create the store files in the parent + for (int i = 0; i < count; i++) { + Path storeFile = new Path(storedir, "_store" + i); + FSDataOutputStream dos = fs.create(storeFile, true); + dos.writeBytes("Some data: " + i); + dos.close(); + } + // make sure the mock store files are there + FileStatus[] storeFiles = fs.listStatus(storedir); + assertEquals(count, storeFiles.length); + } + + private Result makeResultFromHRegionInfo(HRegionInfo region, HRegionInfo splita, + HRegionInfo splitb) throws IOException { + List kvs = new ArrayList(); + kvs.add(new KeyValue( + region.getRegionName(), + HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(region))); + + if (splita != null) { + kvs.add(new KeyValue( + region.getRegionName(), + HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER, + Writables.getBytes(splita))); + } + + if (splitb != null) { + kvs.add(new KeyValue( + region.getRegionName(), + HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER, + Writables.getBytes(splitb))); + } + + return new Result(kvs); + } + + private String setRootDirAndCleanIt(final HBaseTestingUtility htu, + final String subdir) + throws IOException { + Path testdir = htu.getDataTestDir(subdir); + FileSystem fs = FileSystem.get(htu.getConfiguration()); + if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true)); + htu.getConfiguration().set(HConstants.HBASE_DIR, testdir.toString()); + return htu.getConfiguration().get(HConstants.HBASE_DIR); + } + + /** + * @param services Master services instance. + * @param htd + * @param parent + * @param daughter + * @param midkey + * @param top True if we are to write a 'top' reference. + * @return Path to reference we created. + * @throws IOException + */ + private Path createReferences(final MasterServices services, + final HTableDescriptor htd, final HRegionInfo parent, + final HRegionInfo daughter, final byte [] midkey, final boolean top) + throws IOException { + Path rootdir = services.getMasterFileSystem().getRootDir(); + Path tabledir = HTableDescriptor.getTableDir(rootdir, parent.getTableName()); + Path storedir = Store.getStoreHomedir(tabledir, daughter.getEncodedName(), + htd.getColumnFamilies()[0].getName()); + Reference ref = new Reference(midkey, + top? Reference.Range.top: Reference.Range.bottom); + long now = System.currentTimeMillis(); + // Reference name has this format: StoreFile#REF_NAME_PARSER + Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); + FileSystem fs = services.getMasterFileSystem().getFileSystem(); + ref.write(fs, p); + return p; + } + + private Result createResult(final HRegionInfo parent, final HRegionInfo a, + final HRegionInfo b) + throws IOException { + List kvs = new ArrayList(); + kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY, + HConstants.SPLITA_QUALIFIER, Writables.getBytes(a))); + kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY, + HConstants.SPLITB_QUALIFIER, Writables.getBytes(b))); + return new Result(kvs); + } + + private HTableDescriptor createHTableDescriptor() { + HTableDescriptor htd = new HTableDescriptor("t"); + htd.addFamily(new HColumnDescriptor("f")); + return htd; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java b/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java new file mode 100644 index 0000000..7899bf0 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java @@ -0,0 +1,112 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.fail; + +import java.net.InetAddress; + +import junit.framework.Assert; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestClockSkewDetection { + private static final Log LOG = + LogFactory.getLog(TestClockSkewDetection.class); + + @Test + public void testClockSkewDetection() throws Exception { + final Configuration conf = HBaseConfiguration.create(); + ServerManager sm = new ServerManager(new Server() { + @Override + public CatalogTracker getCatalogTracker() { + return null; + } + + @Override + public Configuration getConfiguration() { + return conf; + } + + @Override + public ServerName getServerName() { + return null; + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + return null; + } + + @Override + public void abort(String why, Throwable e) {} + + @Override + public boolean isAborted() { + return false; + } + + @Override + public boolean isStopped() { + return false; + } + + @Override + public void stop(String why) { + }}, null, false); + + LOG.debug("regionServerStartup 1"); + InetAddress ia1 = InetAddress.getLocalHost(); + sm.regionServerStartup(ia1, 1234, -1, System.currentTimeMillis()); + + final Configuration c = HBaseConfiguration.create(); + long maxSkew = c.getLong("hbase.master.maxclockskew", 30000); + long warningSkew = c.getLong("hbase.master.warningclockskew", 1000); + + try { + LOG.debug("regionServerStartup 2"); + InetAddress ia2 = InetAddress.getLocalHost(); + sm.regionServerStartup(ia2, 1235, -1, System.currentTimeMillis() - maxSkew * 2); + fail("HMaster should have thrown an ClockOutOfSyncException but didn't."); + } catch(ClockOutOfSyncException e) { + //we want an exception + LOG.info("Recieved expected exception: "+e); + } + + // make sure values above warning threshold but below max threshold don't kill + LOG.debug("regionServerStartup 3"); + InetAddress ia3 = InetAddress.getLocalHost(); + sm.regionServerStartup(ia3, 1236, -1, System.currentTimeMillis() - warningSkew * 2); + + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java b/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java new file mode 100644 index 0000000..d24c879 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.ServerName; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestDeadServer { + @Test public void testIsDead() { + DeadServer ds = new DeadServer(); + final ServerName hostname123 = new ServerName("127.0.0.1", 123, 3L); + ds.add(hostname123); + assertTrue(ds.areDeadServersInProgress()); + ds.finish(hostname123); + assertFalse(ds.areDeadServersInProgress()); + final ServerName hostname1234 = new ServerName("127.0.0.2", 1234, 4L); + ds.add(hostname1234); + assertTrue(ds.areDeadServersInProgress()); + ds.finish(hostname1234); + assertFalse(ds.areDeadServersInProgress()); + final ServerName hostname12345 = new ServerName("127.0.0.2", 12345, 4L); + ds.add(hostname12345); + assertTrue(ds.areDeadServersInProgress()); + ds.finish(hostname12345); + assertFalse(ds.areDeadServersInProgress()); + + // Already dead = 127.0.0.1,9090,112321 + // Coming back alive = 127.0.0.1,9090,223341 + + final ServerName deadServer = new ServerName("127.0.0.1", 9090, 112321L); + assertFalse(ds.cleanPreviousInstance(deadServer)); + ds.add(deadServer); + assertTrue(ds.isDeadServer(deadServer)); + final ServerName deadServerHostComingAlive = + new ServerName("127.0.0.1", 9090, 112321L); + assertTrue(ds.cleanPreviousInstance(deadServerHostComingAlive)); + assertFalse(ds.isDeadServer(deadServer)); + assertFalse(ds.cleanPreviousInstance(deadServerHostComingAlive)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDefaultLoadBalancer.java b/src/test/java/org/apache/hadoop/hbase/master/TestDefaultLoadBalancer.java new file mode 100644 index 0000000..40721cc --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestDefaultLoadBalancer.java @@ -0,0 +1,513 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Random; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + + +/** + * Test the load balancer that is created by default. + */ +@Category(MediumTests.class) +public class TestDefaultLoadBalancer { + private static final Log LOG = LogFactory.getLog(TestDefaultLoadBalancer.class); + + private static LoadBalancer loadBalancer; + + private static Random rand; + + @BeforeClass + public static void beforeAllTests() throws Exception { + Configuration conf = HBaseConfiguration.create(); + conf.set("hbase.regions.slop", "0"); + loadBalancer = new DefaultLoadBalancer(); + loadBalancer.setConf(conf); + rand = new Random(); + } + + // int[testnum][servernumber] -> numregions + int [][] clusterStateMocks = new int [][] { + // 1 node + new int [] { 0 }, + new int [] { 1 }, + new int [] { 10 }, + // 2 node + new int [] { 0, 0 }, + new int [] { 2, 0 }, + new int [] { 2, 1 }, + new int [] { 2, 2 }, + new int [] { 2, 3 }, + new int [] { 2, 4 }, + new int [] { 1, 1 }, + new int [] { 0, 1 }, + new int [] { 10, 1 }, + new int [] { 14, 1432 }, + new int [] { 47, 53 }, + // 3 node + new int [] { 0, 1, 2 }, + new int [] { 1, 2, 3 }, + new int [] { 0, 2, 2 }, + new int [] { 0, 3, 0 }, + new int [] { 0, 4, 0 }, + new int [] { 20, 20, 0 }, + // 4 node + new int [] { 0, 1, 2, 3 }, + new int [] { 4, 0, 0, 0 }, + new int [] { 5, 0, 0, 0 }, + new int [] { 6, 6, 0, 0 }, + new int [] { 6, 2, 0, 0 }, + new int [] { 6, 1, 0, 0 }, + new int [] { 6, 0, 0, 0 }, + new int [] { 4, 4, 4, 7 }, + new int [] { 4, 4, 4, 8 }, + new int [] { 0, 0, 0, 7 }, + // 5 node + new int [] { 1, 1, 1, 1, 4 }, + // more nodes + new int [] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 10 }, + new int [] { 6, 6, 5, 6, 6, 6, 6, 6, 6, 1 }, + new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 54 }, + new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 55 }, + new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 }, + new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 16 }, + new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 8 }, + new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 9 }, + new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 10 }, + new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 123 }, + new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 155 }, + new int [] { 0, 0, 144, 1, 1, 1, 1, 1123, 133, 138, 12, 1444 }, + new int [] { 0, 0, 144, 1, 0, 4, 1, 1123, 133, 138, 12, 1444 }, + new int [] { 1538, 1392, 1561, 1557, 1535, 1553, 1385, 1542, 1619 } + }; + + int [][] regionsAndServersMocks = new int [][] { + // { num regions, num servers } + new int [] { 0, 0 }, + new int [] { 0, 1 }, + new int [] { 1, 1 }, + new int [] { 2, 1 }, + new int [] { 10, 1 }, + new int [] { 1, 2 }, + new int [] { 2, 2 }, + new int [] { 3, 2 }, + new int [] { 1, 3 }, + new int [] { 2, 3 }, + new int [] { 3, 3 }, + new int [] { 25, 3 }, + new int [] { 2, 10 }, + new int [] { 2, 100 }, + new int [] { 12, 10 }, + new int [] { 12, 100 }, + }; + + /** + * Test the load balancing algorithm. + * + * Invariant is that all servers should be hosting either + * floor(average) or ceiling(average) + * + * @throws Exception + */ + @Test + public void testBalanceCluster() throws Exception { + + for(int [] mockCluster : clusterStateMocks) { + Map> servers = mockClusterServers(mockCluster); + List list = convertToList(servers); + LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list)); + List plans = loadBalancer.balanceCluster(servers); + List balancedCluster = reconcile(list, plans); + LOG.info("Mock Balance : " + printMock(balancedCluster)); + assertClusterAsBalanced(balancedCluster); + for(Map.Entry> entry : servers.entrySet()) { + returnRegions(entry.getValue()); + returnServer(entry.getKey()); + } + } + + } + + /** + * Invariant is that all servers have between floor(avg) and ceiling(avg) + * number of regions. + */ + public void assertClusterAsBalanced(List servers) { + int numServers = servers.size(); + int numRegions = 0; + int maxRegions = 0; + int minRegions = Integer.MAX_VALUE; + for(ServerAndLoad server : servers) { + int nr = server.getLoad(); + if(nr > maxRegions) { + maxRegions = nr; + } + if(nr < minRegions) { + minRegions = nr; + } + numRegions += nr; + } + if(maxRegions - minRegions < 2) { + // less than 2 between max and min, can't balance + return; + } + int min = numRegions / numServers; + int max = numRegions % numServers == 0 ? min : min + 1; + + for(ServerAndLoad server : servers) { + assertTrue(server.getLoad() <= max); + assertTrue(server.getLoad() >= min); + } + } + + /** + * Tests immediate assignment. + * + * Invariant is that all regions have an assignment. + * + * @throws Exception + */ + @Test + public void testImmediateAssignment() throws Exception { + for(int [] mock : regionsAndServersMocks) { + LOG.debug("testImmediateAssignment with " + mock[0] + " regions and " + mock[1] + " servers"); + List regions = randomRegions(mock[0]); + List servers = randomServers(mock[1], 0); + List list = getListOfServerNames(servers); + Map assignments = + loadBalancer.immediateAssignment(regions, list); + assertImmediateAssignment(regions, list, assignments); + returnRegions(regions); + returnServers(list); + } + } + + /** + * All regions have an assignment. + * @param regions + * @param servers + * @param assignments + */ + private void assertImmediateAssignment(List regions, + List servers, Map assignments) { + for(HRegionInfo region : regions) { + assertTrue(assignments.containsKey(region)); + } + } + + /** + * Tests the bulk assignment used during cluster startup. + * + * Round-robin. Should yield a balanced cluster so same invariant as the load + * balancer holds, all servers holding either floor(avg) or ceiling(avg). + * + * @throws Exception + */ + @Test + public void testBulkAssignment() throws Exception { + for(int [] mock : regionsAndServersMocks) { + LOG.debug("testBulkAssignment with " + mock[0] + " regions and " + mock[1] + " servers"); + List regions = randomRegions(mock[0]); + List servers = randomServers(mock[1], 0); + List list = getListOfServerNames(servers); + Map> assignments = + loadBalancer.roundRobinAssignment(regions, list); + float average = (float)regions.size()/servers.size(); + int min = (int)Math.floor(average); + int max = (int)Math.ceil(average); + if(assignments != null && !assignments.isEmpty()) { + for(List regionList : assignments.values()) { + assertTrue(regionList.size() == min || regionList.size() == max); + } + } + returnRegions(regions); + returnServers(list); + } + } + + /** + * Test the cluster startup bulk assignment which attempts to retain + * assignment info. + * @throws Exception + */ + @Test + public void testRetainAssignment() throws Exception { + // Test simple case where all same servers are there + List servers = randomServers(10, 10); + List regions = randomRegions(100); + Map existing = + new TreeMap(); + for (int i = 0; i < regions.size(); i++) { + ServerName sn = servers.get(i % servers.size()).getServerName(); + // The old server would have had same host and port, but different + // start code! + ServerName snWithOldStartCode = + new ServerName(sn.getHostname(), sn.getPort(), sn.getStartcode() - 10); + existing.put(regions.get(i), snWithOldStartCode); + } + List listOfServerNames = getListOfServerNames(servers); + Map> assignment = + loadBalancer.retainAssignment(existing, listOfServerNames); + assertRetainedAssignment(existing, listOfServerNames, assignment); + + // Include two new servers that were not there before + List servers2 = + new ArrayList(servers); + servers2.add(randomServer(10)); + servers2.add(randomServer(10)); + listOfServerNames = getListOfServerNames(servers2); + assignment = loadBalancer.retainAssignment(existing, listOfServerNames); + assertRetainedAssignment(existing, listOfServerNames, assignment); + + // Remove two of the servers that were previously there + List servers3 = + new ArrayList(servers); + servers3.remove(0); + servers3.remove(0); + listOfServerNames = getListOfServerNames(servers3); + assignment = loadBalancer.retainAssignment(existing, listOfServerNames); + assertRetainedAssignment(existing, listOfServerNames, assignment); + } + + private List getListOfServerNames(final List sals) { + List list = new ArrayList(); + for (ServerAndLoad e: sals) { + list.add(e.getServerName()); + } + return list; + } + + /** + * Asserts a valid retained assignment plan. + *

    + * Must meet the following conditions: + *

      + *
    • Every input region has an assignment, and to an online server + *
    • If a region had an existing assignment to a server with the same + * address a a currently online server, it will be assigned to it + *
    + * @param existing + * @param servers + * @param assignment + */ + private void assertRetainedAssignment( + Map existing, List servers, + Map> assignment) { + // Verify condition 1, every region assigned, and to online server + Set onlineServerSet = new TreeSet(servers); + Set assignedRegions = new TreeSet(); + for (Map.Entry> a : assignment.entrySet()) { + assertTrue("Region assigned to server that was not listed as online", + onlineServerSet.contains(a.getKey())); + for (HRegionInfo r : a.getValue()) assignedRegions.add(r); + } + assertEquals(existing.size(), assignedRegions.size()); + + // Verify condition 2, if server had existing assignment, must have same + Set onlineHostNames = new TreeSet(); + for (ServerName s : servers) { + onlineHostNames.add(s.getHostname()); + } + + for (Map.Entry> a : assignment.entrySet()) { + ServerName assignedTo = a.getKey(); + for (HRegionInfo r : a.getValue()) { + ServerName address = existing.get(r); + if (address != null && onlineHostNames.contains(address.getHostname())) { + // this region was prevously assigned somewhere, and that + // host is still around, then it should be re-assigned on the + // same host + assertEquals(address.getHostname(), assignedTo.getHostname()); + } + } + } + } + + private String printStats(List servers) { + int numServers = servers.size(); + int totalRegions = 0; + for(ServerAndLoad server : servers) { + totalRegions += server.getLoad(); + } + float average = (float)totalRegions / numServers; + int max = (int)Math.ceil(average); + int min = (int)Math.floor(average); + return "[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max + " min=" + min + "]"; + } + + private List convertToList(final Map> servers) { + List list = + new ArrayList(servers.size()); + for (Map.Entry> e: servers.entrySet()) { + list.add(new ServerAndLoad(e.getKey(), e.getValue().size())); + } + return list; + } + + private String printMock(List balancedCluster) { + SortedSet sorted = + new TreeSet(balancedCluster); + ServerAndLoad [] arr = + sorted.toArray(new ServerAndLoad[sorted.size()]); + StringBuilder sb = new StringBuilder(sorted.size() * 4 + 4); + sb.append("{ "); + for(int i = 0; i < arr.length; i++) { + if (i != 0) { + sb.append(" , "); + } + sb.append(arr[i].getLoad()); + } + sb.append(" }"); + return sb.toString(); + } + + /** + * This assumes the RegionPlan HSI instances are the same ones in the map, so + * actually no need to even pass in the map, but I think it's clearer. + * @param list + * @param plans + * @return + */ + private List reconcile(List list, + List plans) { + List result = + new ArrayList(list.size()); + if (plans == null) return result; + Map map = + new HashMap(list.size()); + for (RegionPlan plan : plans) { + ServerName source = plan.getSource(); + updateLoad(map, source, -1); + ServerName destination = plan.getDestination(); + updateLoad(map, destination, +1); + } + result.clear(); + result.addAll(map.values()); + return result; + } + + private void updateLoad(Map map, + final ServerName sn, final int diff) { + ServerAndLoad sal = map.get(sn); + if (sal == null) return; + sal = new ServerAndLoad(sn, sal.getLoad() + diff); + map.put(sn, sal); + } + + private Map> mockClusterServers( + int [] mockCluster) { + int numServers = mockCluster.length; + Map> servers = + new TreeMap>(); + for(int i = 0; i < numServers; i++) { + int numRegions = mockCluster[i]; + ServerAndLoad sal = randomServer(0); + List regions = randomRegions(numRegions); + servers.put(sal.getServerName(), regions); + } + return servers; + } + + private Queue regionQueue = new LinkedList(); + static int regionId = 0; + + private List randomRegions(int numRegions) { + List regions = new ArrayList(numRegions); + byte [] start = new byte[16]; + byte [] end = new byte[16]; + rand.nextBytes(start); + rand.nextBytes(end); + for(int i=0;i regions) { + regionQueue.addAll(regions); + } + + private Queue serverQueue = new LinkedList(); + + private ServerAndLoad randomServer(final int numRegionsPerServer) { + if (!this.serverQueue.isEmpty()) { + ServerName sn = this.serverQueue.poll(); + return new ServerAndLoad(sn, numRegionsPerServer); + } + String host = "server" + rand.nextInt(100000); + int port = rand.nextInt(60000); + long startCode = rand.nextLong(); + ServerName sn = new ServerName(host, port, startCode); + return new ServerAndLoad(sn, numRegionsPerServer); + } + + private List randomServers(int numServers, int numRegionsPerServer) { + List servers = + new ArrayList(numServers); + for (int i = 0; i < numServers; i++) { + servers.add(randomServer(numRegionsPerServer)); + } + return servers; + } + + private void returnServer(ServerName server) { + serverQueue.add(server); + } + + private void returnServers(List servers) { + this.serverQueue.addAll(servers); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java new file mode 100644 index 0000000..83bd4c1 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -0,0 +1,624 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_mgr_wait_for_zk_delete; +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_final_transistion_failed; +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_preempt_task; +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_acquired; +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_done; +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_err; +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_resigned; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKSplitLog; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestDistributedLogSplitting { + private static final Log LOG = LogFactory.getLog(TestSplitLogManager.class); + static { + Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG); + } + + // Start a cluster with 2 masters and 3 regionservers + final int NUM_MASTERS = 2; + final int NUM_RS = 6; + + MiniHBaseCluster cluster; + HMaster master; + Configuration conf; + HBaseTestingUtility TEST_UTIL; + + + private void startCluster(int num_rs) throws Exception{ + conf = HBaseConfiguration.create(); + startCluster(NUM_MASTERS, num_rs, conf); + } + + private void startCluster(int num_master, int num_rs, Configuration inConf) throws Exception { + ZKSplitLog.Counters.resetCounters(); + LOG.info("Starting cluster"); + this.conf = inConf; + conf.getLong("hbase.splitlog.max.resubmit", 0); + // Make the failure test faster + conf.setInt("zookeeper.recovery.retry", 0); + TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(num_master, num_rs); + cluster = TEST_UTIL.getHBaseCluster(); + LOG.info("Waiting for active/ready master"); + cluster.waitForActiveAndReadyMaster(); + master = cluster.getMaster(); + while (cluster.getLiveRegionServerThreads().size() < num_rs) { + Threads.sleep(1); + } + } + + @After + public void after() throws Exception { + for (MasterThread mt : TEST_UTIL.getHBaseCluster().getLiveMasterThreads()) { + mt.getMaster().abort("closing...", new Exception("Trace info")); + } + + TEST_UTIL.shutdownMiniCluster(); + } + + @Test (timeout=300000) + public void testThreeRSAbort() throws Exception { + LOG.info("testThreeRSAbort"); + final int NUM_REGIONS_TO_CREATE = 40; + final int NUM_ROWS_PER_REGION = 100; + + startCluster(NUM_RS); // NUM_RS=6. + + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, + "distributed log splitting test", null); + + HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); + populateDataInTable(NUM_ROWS_PER_REGION, "family"); + + + List rsts = cluster.getLiveRegionServerThreads(); + assertEquals(NUM_RS, rsts.size()); + rsts.get(0).getRegionServer().abort("testing"); + rsts.get(1).getRegionServer().abort("testing"); + rsts.get(2).getRegionServer().abort("testing"); + + long start = EnvironmentEdgeManager.currentTimeMillis(); + while (cluster.getLiveRegionServerThreads().size() > (NUM_RS - 3)) { + if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) { + assertTrue(false); + } + Thread.sleep(200); + } + + start = EnvironmentEdgeManager.currentTimeMillis(); + while (getAllOnlineRegions(cluster).size() < (NUM_REGIONS_TO_CREATE + 2)) { + if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) { + assertTrue(false); + } + Thread.sleep(200); + } + + assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION, + TEST_UTIL.countRows(ht)); + ht.close(); + } + + @Test (timeout=300000) + public void testRecoveredEdits() throws Exception { + LOG.info("testRecoveredEdits"); + startCluster(NUM_RS); + final int NUM_LOG_LINES = 1000; + final SplitLogManager slm = master.getMasterFileSystem().splitLogManager; + // turn off load balancing to prevent regions from moving around otherwise + // they will consume recovered.edits + master.balanceSwitch(false); + FileSystem fs = master.getMasterFileSystem().getFileSystem(); + + List rsts = cluster.getLiveRegionServerThreads(); + + Path rootdir = FSUtils.getRootDir(conf); + + installTable(new ZooKeeperWatcher(conf, "table-creation", null), + "table", "family", 40); + byte[] table = Bytes.toBytes("table"); + List regions = null; + HRegionServer hrs = null; + for (int i = 0; i < NUM_RS; i++) { + hrs = rsts.get(i).getRegionServer(); + regions = hrs.getOnlineRegions(); + if (regions.size() != 0) break; + } + final Path logDir = new Path(rootdir, HLog.getHLogDirectoryName(hrs + .getServerName().toString())); + + LOG.info("#regions = " + regions.size()); + Iterator it = regions.iterator(); + while (it.hasNext()) { + HRegionInfo region = it.next(); + if (region.isMetaTable()) { + it.remove(); + } + } + makeHLog(hrs.getWAL(), regions, "table", + NUM_LOG_LINES, 100); + + slm.splitLogDistributed(logDir); + + int count = 0; + for (HRegionInfo hri : regions) { + + Path tdir = HTableDescriptor.getTableDir(rootdir, table); + Path editsdir = + HLog.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, + hri.getEncodedName())); + LOG.debug("checking edits dir " + editsdir); + FileStatus[] files = fs.listStatus(editsdir); + assertEquals(1, files.length); + int c = countHLog(files[0].getPath(), fs, conf); + count += c; + LOG.info(c + " edits in " + files[0].getPath()); + } + assertEquals(NUM_LOG_LINES, count); + } + + @Test(timeout = 300000) + public void testMasterStartsUpWithLogSplittingWork() throws Exception { + LOG.info("testMasterStartsUpWithLogSplittingWork"); + Configuration curConf = HBaseConfiguration.create(); + curConf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_RS - 1); + startCluster(2, NUM_RS, curConf); + + final int NUM_REGIONS_TO_CREATE = 40; + final int NUM_LOG_LINES = 1000; + // turn off load balancing to prevent regions from moving around otherwise + // they will consume recovered.edits + master.balanceSwitch(false); + + List rsts = cluster.getLiveRegionServerThreads(); + final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); + HTable ht = installTable(zkw, "table", "f", NUM_REGIONS_TO_CREATE); + + List regions = null; + HRegionServer hrs = null; + for (int i = 0; i < NUM_RS; i++) { + boolean isCarryingMeta = false; + hrs = rsts.get(i).getRegionServer(); + regions = hrs.getOnlineRegions(); + for (HRegionInfo region : regions) { + if (region.isRootRegion() || region.isMetaRegion()) { + isCarryingMeta = true; + break; + } + } + if (isCarryingMeta) { + continue; + } + break; + } + + LOG.info("#regions = " + regions.size()); + Iterator it = regions.iterator(); + while (it.hasNext()) { + HRegionInfo region = it.next(); + if (region.isMetaTable()) { + it.remove(); + } + } + makeHLog(hrs.getWAL(), regions, "table", NUM_LOG_LINES, 100); + + // abort master + abortMaster(cluster); + + // abort RS + int numRS = cluster.getLiveRegionServerThreads().size(); + LOG.info("Aborting region server: " + hrs.getServerName()); + hrs.abort("testing"); + + // wait for the RS dies + long start = EnvironmentEdgeManager.currentTimeMillis(); + while (cluster.getLiveRegionServerThreads().size() > (numRS - 1)) { + if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) { + assertTrue(false); + } + Thread.sleep(200); + } + + Thread.sleep(2000); + LOG.info("Current Open Regions:" + getAllOnlineRegions(cluster).size()); + + startMasterTillNoDeadServers(cluster); + + start = EnvironmentEdgeManager.currentTimeMillis(); + while (getAllOnlineRegions(cluster).size() < (NUM_REGIONS_TO_CREATE + 2)) { + if (EnvironmentEdgeManager.currentTimeMillis() - start > 60000) { + assertTrue("Timedout", false); + } + Thread.sleep(200); + } + + LOG.info("Current Open Regions After Master Node Starts Up:" + + getAllOnlineRegions(cluster).size()); + + assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); + + ht.close(); + } + + /** + * The original intention of this test was to force an abort of a region + * server and to make sure that the failure path in the region servers is + * properly evaluated. But it is difficult to ensure that the region server + * doesn't finish the log splitting before it aborts. Also now, there is + * this code path where the master will preempt the region server when master + * detects that the region server has aborted. + * @throws Exception + */ + @Test (timeout=300000) + public void testWorkerAbort() throws Exception { + LOG.info("testWorkerAbort"); + startCluster(1); + final int NUM_LOG_LINES = 10000; + final SplitLogManager slm = master.getMasterFileSystem().splitLogManager; + FileSystem fs = master.getMasterFileSystem().getFileSystem(); + + final List rsts = cluster.getLiveRegionServerThreads(); + HRegionServer hrs = rsts.get(0).getRegionServer(); + Path rootdir = FSUtils.getRootDir(conf); + final Path logDir = new Path(rootdir, + HLog.getHLogDirectoryName(hrs.getServerName().toString())); + + installTable(new ZooKeeperWatcher(conf, "table-creation", null), + "table", "family", 40); + makeHLog(hrs.getWAL(), hrs.getOnlineRegions(), "table", + NUM_LOG_LINES, 100); + + new Thread() { + public void run() { + waitForCounter(tot_wkr_task_acquired, 0, 1, 1000); + for (RegionServerThread rst : rsts) { + rst.getRegionServer().abort("testing"); + } + } + }.start(); + // slm.splitLogDistributed(logDir); + FileStatus[] logfiles = fs.listStatus(logDir); + TaskBatch batch = new TaskBatch(); + slm.enqueueSplitTask(logfiles[0].getPath().toString(), batch); + //waitForCounter but for one of the 2 counters + long curt = System.currentTimeMillis(); + long waitTime = 80000; + long endt = curt + waitTime; + while (curt < endt) { + if ((tot_wkr_task_resigned.get() + tot_wkr_task_err.get() + + tot_wkr_final_transistion_failed.get() + tot_wkr_task_done.get() + + tot_wkr_preempt_task.get()) == 0) { + Thread.yield(); + curt = System.currentTimeMillis(); + } else { + assertEquals(1, (tot_wkr_task_resigned.get() + tot_wkr_task_err.get() + + tot_wkr_final_transistion_failed.get() + tot_wkr_task_done.get() + + tot_wkr_preempt_task.get())); + return; + } + } + fail("none of the following counters went up in " + waitTime + + " milliseconds - " + + "tot_wkr_task_resigned, tot_wkr_task_err, " + + "tot_wkr_final_transistion_failed, tot_wkr_task_done, " + + "tot_wkr_preempt_task"); + } + + @Test(timeout=30000) + public void testDelayedDeleteOnFailure() throws Exception { + LOG.info("testDelayedDeleteOnFailure"); + startCluster(1); + final SplitLogManager slm = master.getMasterFileSystem().splitLogManager; + final FileSystem fs = master.getMasterFileSystem().getFileSystem(); + final Path logDir = new Path(FSUtils.getRootDir(conf), "x"); + fs.mkdirs(logDir); + ExecutorService executor = null; + try { + final Path corruptedLogFile = new Path(logDir, "x"); + FSDataOutputStream out; + out = fs.create(corruptedLogFile); + out.write(0); + out.write(Bytes.toBytes("corrupted bytes")); + out.close(); + slm.ignoreZKDeleteForTesting = true; + executor = Executors.newSingleThreadExecutor(); + Runnable runnable = new Runnable() { + @Override + public void run() { + try { + // since the logDir is a fake, corrupted one, so the split log worker + // will finish it quickly with error, and this call will fail and throw + // an IOException. + slm.splitLogDistributed(logDir); + } catch (IOException ioe) { + try { + assertTrue(fs.exists(corruptedLogFile)); + // this call will block waiting for the task to be removed from the + // tasks map which is not going to happen since ignoreZKDeleteForTesting + // is set to true, until it is interrupted. + slm.splitLogDistributed(logDir); + } catch (IOException e) { + assertTrue(Thread.currentThread().isInterrupted()); + return; + } + fail("did not get the expected IOException from the 2nd call"); + } + fail("did not get the expected IOException from the 1st call"); + } + }; + Future result = executor.submit(runnable); + try { + result.get(2000, TimeUnit.MILLISECONDS); + } catch (TimeoutException te) { + // it is ok, expected. + } + waitForCounter(tot_mgr_wait_for_zk_delete, 0, 1, 10000); + executor.shutdownNow(); + executor = null; + + // make sure the runnable is finished with no exception thrown. + result.get(); + } finally { + if (executor != null) { + // interrupt the thread in case the test fails in the middle. + // it has no effect if the thread is already terminated. + executor.shutdownNow(); + } + fs.delete(logDir, true); + } + } + + HTable installTable(ZooKeeperWatcher zkw, String tname, String fname, + int nrs ) throws Exception { + // Create a table with regions + byte [] table = Bytes.toBytes(tname); + byte [] family = Bytes.toBytes(fname); + LOG.info("Creating table with " + nrs + " regions"); + HTable ht = TEST_UTIL.createTable(table, family); + int numRegions = TEST_UTIL.createMultiRegions(conf, ht, family, nrs); + assertEquals(nrs, numRegions); + LOG.info("Waiting for no more RIT\n"); + blockUntilNoRIT(zkw, master); + // disable-enable cycle to get rid of table's dead regions left behind + // by createMultiRegions + LOG.debug("Disabling table\n"); + TEST_UTIL.getHBaseAdmin().disableTable(table); + LOG.debug("Waiting for no more RIT\n"); + blockUntilNoRIT(zkw, master); + NavigableSet regions = getAllOnlineRegions(cluster); + LOG.debug("Verifying only catalog regions are assigned\n"); + if (regions.size() != 2) { + for (String oregion : regions) + LOG.debug("Region still online: " + oregion); + } + assertEquals(2, regions.size()); + LOG.debug("Enabling table\n"); + TEST_UTIL.getHBaseAdmin().enableTable(table); + LOG.debug("Waiting for no more RIT\n"); + blockUntilNoRIT(zkw, master); + LOG.debug("Verifying there are " + numRegions + " assigned on cluster\n"); + regions = getAllOnlineRegions(cluster); + assertEquals(numRegions + 2, regions.size()); + return ht; + } + + void populateDataInTable(int nrows, String fname) throws Exception { + byte [] family = Bytes.toBytes(fname); + + List rsts = cluster.getLiveRegionServerThreads(); + assertEquals(NUM_RS, rsts.size()); + + for (RegionServerThread rst : rsts) { + HRegionServer hrs = rst.getRegionServer(); + List hris = hrs.getOnlineRegions(); + for (HRegionInfo hri : hris) { + if (hri.isMetaTable()) { + continue; + } + LOG.debug("adding data to rs = " + rst.getName() + + " region = "+ hri.getRegionNameAsString()); + HRegion region = hrs.getOnlineRegion(hri.getRegionName()); + assertTrue(region != null); + putData(region, hri.getStartKey(), nrows, Bytes.toBytes("q"), family); + } + } + } + + public void makeHLog(HLog log, + List hris, String tname, + int num_edits, int edit_size) throws IOException { + + // remove root and meta region + hris.remove(HRegionInfo.ROOT_REGIONINFO); + hris.remove(HRegionInfo.FIRST_META_REGIONINFO); + byte[] table = Bytes.toBytes(tname); + HTableDescriptor htd = new HTableDescriptor(tname); + byte[] value = new byte[edit_size]; + for (int i = 0; i < edit_size; i++) { + value[i] = (byte) ('a' + (i % 26)); + } + int n = hris.size(); + int[] counts = new int[n]; + if (n > 0) { + for (int i = 0; i < num_edits; i += 1) { + WALEdit e = new WALEdit(); + HRegionInfo curRegionInfo = hris.get(i % n); + byte[] startRow = curRegionInfo.getStartKey(); + if (startRow == null || startRow.length == 0) { + startRow = new byte[] { 0, 0, 0, 0, 1 }; + } + byte[] row = Bytes.incrementBytes(startRow, counts[i % n]); + row = Arrays.copyOfRange(row, 3, 8); // use last 5 bytes because + // HBaseTestingUtility.createMultiRegions use 5 bytes + // key + byte[] family = Bytes.toBytes("f"); + byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i)); + e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value)); + log.append(curRegionInfo, table, e, System.currentTimeMillis(), htd); + counts[i % n] += 1; + } + } + log.sync(); + log.close(); + for (int i = 0; i < n; i++) { + LOG.info("region " + hris.get(i).getRegionNameAsString() + " has " + counts[i] + " edits"); + } + return; + } + + private int countHLog(Path log, FileSystem fs, Configuration conf) + throws IOException { + int count = 0; + HLog.Reader in = HLog.getReader(fs, log, conf); + while (in.next() != null) { + count++; + } + return count; + } + + private void blockUntilNoRIT(ZooKeeperWatcher zkw, HMaster master) + throws KeeperException, InterruptedException { + ZKAssign.blockUntilNoRIT(zkw); + master.assignmentManager.waitUntilNoRegionsInTransition(60000); + } + + private void putData(HRegion region, byte[] startRow, int numRows, byte [] qf, + byte [] ...families) + throws IOException { + for(int i = 0; i < numRows; i++) { + Put put = new Put(Bytes.add(startRow, Bytes.toBytes(i))); + for(byte [] family : families) { + put.add(family, qf, null); + } + region.put(put); + } + } + + private NavigableSet getAllOnlineRegions(MiniHBaseCluster cluster) + throws IOException { + NavigableSet online = new TreeSet(); + for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) { + for (HRegionInfo region : rst.getRegionServer().getOnlineRegions()) { + online.add(region.getRegionNameAsString()); + } + } + return online; + } + + private void waitForCounter(AtomicLong ctr, long oldval, long newval, + long timems) { + long curt = System.currentTimeMillis(); + long endt = curt + timems; + while (curt < endt) { + if (ctr.get() == oldval) { + Thread.yield(); + curt = System.currentTimeMillis(); + } else { + assertEquals(newval, ctr.get()); + return; + } + } + assertTrue(false); + } + + private void abortMaster(MiniHBaseCluster cluster) throws InterruptedException { + for (MasterThread mt : cluster.getLiveMasterThreads()) { + if (mt.getMaster().isActiveMaster()) { + mt.getMaster().abort("Aborting for tests", new Exception("Trace info")); + mt.join(); + break; + } + } + LOG.debug("Master is aborted"); + } + + private void startMasterTillNoDeadServers(MiniHBaseCluster cluster) + throws IOException, InterruptedException { + cluster.startMaster(); + HMaster master = cluster.getMaster(); + while (!master.isInitialized()) { + Thread.sleep(100); + } + ServerManager serverManager = master.getServerManager(); + while (serverManager.areDeadServersInProgress()) { + Thread.sleep(100); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java b/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java new file mode 100644 index 0000000..8c4de71 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java @@ -0,0 +1,75 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.ipc.HBaseRPC; +import org.apache.hadoop.hbase.ipc.HMasterInterface; +import org.apache.hadoop.hbase.ipc.RpcEngine; +import org.apache.hadoop.ipc.RemoteException; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestHMasterRPCException { + + @Test + public void testRPCException() throws Exception { + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + TEST_UTIL.startMiniZKCluster(); + Configuration conf = TEST_UTIL.getConfiguration(); + conf.set(HConstants.MASTER_PORT, "0"); + + HMaster hm = new HMaster(conf); + + ServerName sm = hm.getServerName(); + InetSocketAddress isa = new InetSocketAddress(sm.getHostname(), sm.getPort()); + RpcEngine rpcEngine = null; + try { + rpcEngine = HBaseRPC.getProtocolEngine(conf); + HMasterInterface inf = rpcEngine.getProxy( + HMasterInterface.class, HMasterInterface.VERSION, isa, conf, 100 * 10); + inf.isMasterRunning(); + fail(); + } catch (RemoteException ex) { + assertTrue(ex.getMessage().startsWith( + "org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet")); + } catch (Throwable t) { + fail("Unexpected throwable: " + t); + } finally { + if (rpcEngine != null) { + rpcEngine.close(); + } + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMXBean.java b/src/test/java/org/apache/hadoop/hbase/master/TestMXBean.java new file mode 100644 index 0000000..ec63d81 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestMXBean.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import junit.framework.Assert; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HServerLoad; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestMXBean { + + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + @BeforeClass + public static void setup() throws Exception { + TEST_UTIL.startMiniCluster(1, 4); + } + + @AfterClass + public static void teardown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + private void verifyRegionServers(Map regions) { + Set expected = new HashSet(); + for (int i = 0; i < 4; ++i) { + HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i); + expected.add(rs.getServerName().getServerName()); + } + + int found = 0; + for (java.util.Map.Entry entry : regions.entrySet()) { + if (expected.contains(entry.getKey())) { + ++found; + } + } + Assert.assertEquals(4, found); + } + + @Test + public void testInfo() { + HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); + MXBeanImpl info = MXBeanImpl.init(master); + Assert.assertEquals(master.getAverageLoad(), info.getAverageLoad()); + Assert.assertEquals(master.getClusterId(), info.getClusterId()); + Assert.assertEquals(master.getMasterActiveTime(), + info.getMasterActiveTime()); + Assert.assertEquals(master.getMasterStartTime(), + info.getMasterStartTime()); + Assert.assertEquals(master.getCoprocessors().length, + info.getCoprocessors().length); + Assert.assertEquals(master.getServerManager().getOnlineServersList().size(), + info.getRegionServers().size()); + Assert.assertEquals(master.getAssignmentManager().isRegionsInTransition(), + info.getRegionsInTransition().length > 0); + Assert.assertTrue(info.getRegionServers().size() == 4); + + String zkServers = info.getZookeeperQuorum(); + Assert.assertEquals(zkServers.split(",").length, + TEST_UTIL.getZkCluster().getZooKeeperServerNum()); + + verifyRegionServers(info.getRegionServers()); + + TEST_UTIL.getMiniHBaseCluster().stopRegionServer(3, false); + TEST_UTIL.getMiniHBaseCluster().waitOnRegionServer(3); + Assert.assertTrue(info.getRegionServers().size() == 3); + Assert.assertTrue(info.getDeadRegionServers().length == 1); + + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java b/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java new file mode 100644 index 0000000..65b4d59 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java @@ -0,0 +1,164 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.executor.EventHandler; +import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.base.Joiner; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +@Category(MediumTests.class) +public class TestMaster { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final Log LOG = LogFactory.getLog(TestMaster.class); + private static final byte[] TABLENAME = Bytes.toBytes("TestMaster"); + private static final byte[] FAMILYNAME = Bytes.toBytes("fam"); + + @BeforeClass + public static void beforeAllTests() throws Exception { + // Start a cluster of two regionservers. + TEST_UTIL.startMiniCluster(2); + } + + @AfterClass + public static void afterAllTests() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testMasterOpsWhileSplitting() throws Exception { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + HMaster m = cluster.getMaster(); + + HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILYNAME); + assertTrue(m.assignmentManager.getZKTable().isEnabledTable + (Bytes.toString(TABLENAME))); + TEST_UTIL.loadTable(ht, FAMILYNAME); + ht.close(); + + List> tableRegions = + MetaReader.getTableRegionsAndLocations(m.getCatalogTracker(), + Bytes.toString(TABLENAME)); + LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions)); + assertEquals(1, tableRegions.size()); + assertArrayEquals(HConstants.EMPTY_START_ROW, + tableRegions.get(0).getFirst().getStartKey()); + assertArrayEquals(HConstants.EMPTY_END_ROW, + tableRegions.get(0).getFirst().getEndKey()); + + // Now trigger a split and stop when the split is in progress + CountDownLatch split = new CountDownLatch(1); + CountDownLatch proceed = new CountDownLatch(1); + RegionSplitListener list = new RegionSplitListener(split, proceed); + cluster.getMaster().executorService. + registerListener(EventType.RS_ZK_REGION_SPLIT, list); + + LOG.info("Splitting table"); + TEST_UTIL.getHBaseAdmin().split(TABLENAME); + LOG.info("Waiting for split result to be about to open"); + split.await(60, TimeUnit.SECONDS); + try { + LOG.info("Making sure we can call getTableRegions while opening"); + tableRegions = MetaReader.getTableRegionsAndLocations(m.getCatalogTracker(), + TABLENAME, false); + + LOG.info("Regions: " + Joiner.on(',').join(tableRegions)); + // We have three regions because one is split-in-progress + assertEquals(3, tableRegions.size()); + LOG.info("Making sure we can call getTableRegionClosest while opening"); + Pair pair = + m.getTableRegionForRow(TABLENAME, Bytes.toBytes("cde")); + LOG.info("Result is: " + pair); + Pair tableRegionFromName = + MetaReader.getRegion(m.getCatalogTracker(), + pair.getFirst().getRegionName()); + assertEquals(tableRegionFromName.getFirst(), pair.getFirst()); + } finally { + proceed.countDown(); + } + } + + @Test + public void testMoveRegionWhenNotInitialized() { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + HMaster m = cluster.getMaster(); + try { + m.initialized = false; // fake it, set back later + HRegionInfo meta = HRegionInfo.FIRST_META_REGIONINFO; + m.move(meta.getEncodedNameAsBytes(), null); + fail("Region should not be moved since master is not initialized"); + } catch (IOException ioe) { + assertTrue(ioe.getCause() instanceof PleaseHoldException); + } finally { + m.initialized = true; + } + } + + static class RegionSplitListener implements EventHandlerListener { + CountDownLatch split, proceed; + + public RegionSplitListener(CountDownLatch split, CountDownLatch proceed) { + this.split = split; + this.proceed = proceed; + } + + @Override + public void afterProcess(EventHandler event) { + if (event.getEventType() != EventType.RS_ZK_REGION_SPLIT) { + return; + } + try { + split.countDown(); + proceed.await(60, TimeUnit.SECONDS); + } catch (InterruptedException ie) { + throw new RuntimeException(ie); + } + return; + } + + @Override + public void beforeProcess(EventHandler event) { + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java new file mode 100644 index 0000000..6d1492d --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -0,0 +1,1207 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.executor.RegionTransitionData; +import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKTable; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestMasterFailover { + private static final Log LOG = LogFactory.getLog(TestMasterFailover.class); + + @Test (timeout=180000) + public void testShouldCheckMasterFailOverWhenMETAIsInOpenedState() + throws Exception { + LOG.info("Starting testShouldCheckMasterFailOverWhenMETAIsInOpenedState"); + final int NUM_MASTERS = 1; + final int NUM_RS = 2; + + Configuration conf = HBaseConfiguration.create(); + conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000); + conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 8000); + // Start the cluster + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); + + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + + // Find regionserver carrying meta. + List regionServerThreads = + cluster.getRegionServerThreads(); + int count = -1; + HRegion metaRegion = null; + for (RegionServerThread regionServerThread : regionServerThreads) { + HRegionServer regionServer = regionServerThread.getRegionServer(); + metaRegion = regionServer.getOnlineRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); + count++; + regionServer.abort(""); + if (null != metaRegion) break; + } + HRegionServer regionServer = cluster.getRegionServer(count); + + TEST_UTIL.shutdownMiniHBaseCluster(); + + // Create a ZKW to use in the test + ZooKeeperWatcher zkw = + HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL, + metaRegion, regionServer.getServerName()); + + LOG.info("Staring cluster for second time"); + TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS); + + // Failover should be completed, now wait for no RIT + log("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + + zkw.close(); + // Stop the cluster + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Simple test of master failover. + *

    + * Starts with three masters. Kills a backup master. Then kills the active + * master. Ensures the final master becomes active and we can still contact + * the cluster. + * @throws Exception + */ + @Test (timeout=240000) + public void testSimpleMasterFailover() throws Exception { + + final int NUM_MASTERS = 3; + final int NUM_RS = 3; + + // Create config to use for this cluster + Configuration conf = HBaseConfiguration.create(); + + // Start the cluster + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + + // get all the master threads + List masterThreads = cluster.getMasterThreads(); + + // wait for each to come online + for (MasterThread mt : masterThreads) { + assertTrue(mt.isAlive()); + } + + // verify only one is the active master and we have right number + int numActive = 0; + int activeIndex = -1; + ServerName activeName = null; + HMaster active = null; + for (int i = 0; i < masterThreads.size(); i++) { + if (masterThreads.get(i).getMaster().isActiveMaster()) { + numActive++; + activeIndex = i; + active = masterThreads.get(activeIndex).getMaster(); + activeName = active.getServerName(); + } + } + assertEquals(1, numActive); + assertEquals(NUM_MASTERS, masterThreads.size()); + LOG.info("Active master " + activeName); + + // Check that ClusterStatus reports the correct active and backup masters + assertNotNull(active); + ClusterStatus status = active.getClusterStatus(); + assertTrue(status.getMaster().equals(activeName)); + assertEquals(2, status.getBackupMastersSize()); + assertEquals(2, status.getBackupMasters().size()); + + // attempt to stop one of the inactive masters + int backupIndex = (activeIndex == 0 ? 1 : activeIndex - 1); + HMaster master = cluster.getMaster(backupIndex); + LOG.debug("\n\nStopping a backup master: " + master.getServerName() + "\n"); + cluster.stopMaster(backupIndex, false); + cluster.waitOnMaster(backupIndex); + + // Verify still one active master and it's the same + for (int i = 0; i < masterThreads.size(); i++) { + if (masterThreads.get(i).getMaster().isActiveMaster()) { + assertTrue(activeName.equals(masterThreads.get(i).getMaster().getServerName())); + activeIndex = i; + active = masterThreads.get(activeIndex).getMaster(); + } + } + assertEquals(1, numActive); + assertEquals(2, masterThreads.size()); + int rsCount = masterThreads.get(activeIndex).getMaster().getClusterStatus().getServersSize(); + LOG.info("Active master " + active.getServerName() + " managing " + rsCount + " regions servers"); + assertEquals(3, rsCount); + + // Check that ClusterStatus reports the correct active and backup masters + assertNotNull(active); + status = active.getClusterStatus(); + assertTrue(status.getMaster().equals(activeName)); + assertEquals(1, status.getBackupMastersSize()); + assertEquals(1, status.getBackupMasters().size()); + + // kill the active master + LOG.debug("\n\nStopping the active master " + active.getServerName() + "\n"); + cluster.stopMaster(activeIndex, false); + cluster.waitOnMaster(activeIndex); + + // wait for an active master to show up and be ready + assertTrue(cluster.waitForActiveAndReadyMaster()); + + LOG.debug("\n\nVerifying backup master is now active\n"); + // should only have one master now + assertEquals(1, masterThreads.size()); + + // and he should be active + active = masterThreads.get(0).getMaster(); + assertNotNull(active); + status = active.getClusterStatus(); + ServerName mastername = status.getMaster(); + assertTrue(mastername.equals(active.getServerName())); + assertTrue(active.isActiveMaster()); + assertEquals(0, status.getBackupMastersSize()); + assertEquals(0, status.getBackupMasters().size()); + int rss = status.getServersSize(); + LOG.info("Active master " + mastername.getServerName() + " managing " + + rss + " region servers"); + assertEquals(3, rss); + + // Stop the cluster + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Complex test of master failover that tests as many permutations of the + * different possible states that regions in transition could be in within ZK. + *

    + * This tests the proper handling of these states by the failed-over master + * and includes a thorough testing of the timeout code as well. + *

    + * Starts with a single master and three regionservers. + *

    + * Creates two tables, enabledTable and disabledTable, each containing 5 + * regions. The disabledTable is then disabled. + *

    + * After reaching steady-state, the master is killed. We then mock several + * states in ZK. + *

    + * After mocking them, we will startup a new master which should become the + * active master and also detect that it is a failover. The primary test + * passing condition will be that all regions of the enabled table are + * assigned and all the regions of the disabled table are not assigned. + *

    + * The different scenarios to be tested are below: + *

    + * ZK State: OFFLINE + *

    A node can get into OFFLINE state if

    + *
      + *
    • An RS fails to open a region, so it reverts the state back to OFFLINE + *
    • The Master is assigning the region to a RS before it sends RPC + *
    + *

    We will mock the scenarios

    + *
      + *
    • Master has assigned an enabled region but RS failed so a region is + * not assigned anywhere and is sitting in ZK as OFFLINE
      • + *
    • This seems to cover both cases?
      • + *
    + *

    + * ZK State: CLOSING + *

    A node can get into CLOSING state if

    + *
      + *
    • An RS has begun to close a region + *
    + *

    We will mock the scenarios

    + *
      + *
    • Region of enabled table was being closed but did not complete + *
    • Region of disabled table was being closed but did not complete + *
    + *

    + * ZK State: CLOSED + *

    A node can get into CLOSED state if

    + *
      + *
    • An RS has completed closing a region but not acknowledged by master yet + *
    + *

    We will mock the scenarios

    + *
      + *
    • Region of a table that should be enabled was closed on an RS + *
    • Region of a table that should be disabled was closed on an RS + *
    + *

    + * ZK State: OPENING + *

    A node can get into OPENING state if

    + *
      + *
    • An RS has begun to open a region + *
    + *

    We will mock the scenarios

    + *
      + *
    • RS was opening a region of enabled table but never finishes + *
    + *

    + * ZK State: OPENED + *

    A node can get into OPENED state if

    + *
      + *
    • An RS has finished opening a region but not acknowledged by master yet + *
    + *

    We will mock the scenarios

    + *
      + *
    • Region of a table that should be enabled was opened on an RS + *
    • Region of a table that should be disabled was opened on an RS + *
    + * @throws Exception + */ + @Test (timeout=180000) + public void testMasterFailoverWithMockedRIT() throws Exception { + + final int NUM_MASTERS = 1; + final int NUM_RS = 3; + + // Create config to use for this cluster + Configuration conf = HBaseConfiguration.create(); + // Need to drop the timeout much lower + conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000); + conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 4000); + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 3); + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 3); + + // Start the cluster + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + log("Cluster started"); + + // Create a ZKW to use in the test + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL); + + // get all the master threads + List masterThreads = cluster.getMasterThreads(); + assertEquals(1, masterThreads.size()); + + // only one master thread, let's wait for it to be initialized + assertTrue(cluster.waitForActiveAndReadyMaster()); + HMaster master = masterThreads.get(0).getMaster(); + assertTrue(master.isActiveMaster()); + assertTrue(master.isInitialized()); + + // disable load balancing on this master + master.balanceSwitch(false); + + // create two tables in META, each with 10 regions + byte [] FAMILY = Bytes.toBytes("family"); + byte [][] SPLIT_KEYS = new byte [][] { + new byte[0], Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), + Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"), + Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"), + Bytes.toBytes("iii"), Bytes.toBytes("jjj") + }; + + byte [] enabledTable = Bytes.toBytes("enabledTable"); + HTableDescriptor htdEnabled = new HTableDescriptor(enabledTable); + htdEnabled.addFamily(new HColumnDescriptor(FAMILY)); + + FileSystem filesystem = FileSystem.get(conf); + Path rootdir = filesystem.makeQualified( + new Path(conf.get(HConstants.HBASE_DIR))); + // Write the .tableinfo + FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdEnabled); + + HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getName(), null, null); + createRegion(hriEnabled, rootdir, conf, htdEnabled); + + List enabledRegions = TEST_UTIL.createMultiRegionsInMeta( + TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS); + + byte [] disabledTable = Bytes.toBytes("disabledTable"); + HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable); + htdDisabled.addFamily(new HColumnDescriptor(FAMILY)); + // Write the .tableinfo + FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled); + HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null); + createRegion(hriDisabled, rootdir, conf, htdDisabled); + List disabledRegions = TEST_UTIL.createMultiRegionsInMeta( + TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS); + + log("Regions in META have been created"); + + // at this point we only expect 2 regions to be assigned out (catalogs) + assertEquals(2, cluster.countServedRegions()); + + // Let's just assign everything to first RS + HRegionServer hrs = cluster.getRegionServer(0); + ServerName serverName = hrs.getServerName(); + HRegionInfo closingRegion = enabledRegions.remove(0); + // we'll need some regions to already be assigned out properly on live RS + List enabledAndAssignedRegions = new ArrayList(); + enabledAndAssignedRegions.add(enabledRegions.remove(0)); + enabledAndAssignedRegions.add(enabledRegions.remove(0)); + enabledAndAssignedRegions.add(closingRegion); + + List disabledAndAssignedRegions = new ArrayList(); + disabledAndAssignedRegions.add(disabledRegions.remove(0)); + disabledAndAssignedRegions.add(disabledRegions.remove(0)); + + // now actually assign them + for (HRegionInfo hri : enabledAndAssignedRegions) { + master.assignmentManager.regionPlans.put(hri.getEncodedName(), + new RegionPlan(hri, null, serverName)); + master.assignRegion(hri); + } + for (HRegionInfo hri : disabledAndAssignedRegions) { + master.assignmentManager.regionPlans.put(hri.getEncodedName(), + new RegionPlan(hri, null, serverName)); + master.assignRegion(hri); + } + + // wait for no more RIT + log("Waiting for assignment to finish"); + ZKAssign.blockUntilNoRIT(zkw); + log("Assignment completed"); + + // Stop the master + log("Aborting master"); + cluster.abortMaster(0); + cluster.waitOnMaster(0); + log("Master has aborted"); + + /* + * Now, let's start mocking up some weird states as described in the method + * javadoc. + */ + + List regionsThatShouldBeOnline = new ArrayList(); + List regionsThatShouldBeOffline = new ArrayList(); + + log("Beginning to mock scenarios"); + + // Disable the disabledTable in ZK + ZKTable zktable = new ZKTable(zkw); + zktable.setDisabledTable(Bytes.toString(disabledTable)); + + /* + * ZK = OFFLINE + */ + + // Region that should be assigned but is not and is in ZK as OFFLINE + HRegionInfo region = enabledRegions.remove(0); + regionsThatShouldBeOnline.add(region); + ZKAssign.createNodeOffline(zkw, region, serverName); + + /* + * ZK = CLOSING + */ + regionsThatShouldBeOnline.add(closingRegion); + ZKAssign.createNodeClosing(zkw, closingRegion, serverName); + + /* + * ZK = CLOSED + */ + + // Region of enabled table closed but not ack + region = enabledRegions.remove(0); + regionsThatShouldBeOnline.add(region); + int version = ZKAssign.createNodeClosing(zkw, region, serverName); + ZKAssign.transitionNodeClosed(zkw, region, serverName, version); + + // Region of disabled table closed but not ack + region = disabledRegions.remove(0); + regionsThatShouldBeOffline.add(region); + version = ZKAssign.createNodeClosing(zkw, region, serverName); + ZKAssign.transitionNodeClosed(zkw, region, serverName, version); + + /* + * ZK = OPENING + */ + + // RS was opening a region of enabled table but never finishes + region = enabledRegions.remove(0); + regionsThatShouldBeOnline.add(region); + ZKAssign.createNodeOffline(zkw, region, serverName); + ZKAssign.transitionNodeOpening(zkw, region, serverName); + + /* + * ZK = OPENED + */ + + // Region of enabled table was opened on RS + region = enabledRegions.remove(0); + regionsThatShouldBeOnline.add(region); + ZKAssign.createNodeOffline(zkw, region, serverName); + hrs.openRegion(region); + while (true) { + RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName()); + if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) { + break; + } + Thread.sleep(100); + } + + // Region of disable table was opened on RS + region = disabledRegions.remove(0); + regionsThatShouldBeOffline.add(region); + ZKAssign.createNodeOffline(zkw, region, serverName); + hrs.openRegion(region); + while (true) { + RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName()); + if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) { + break; + } + Thread.sleep(100); + } + + /* + * ZK = NONE + */ + + /* + * DONE MOCKING + */ + + log("Done mocking data up in ZK"); + + // Start up a new master + log("Starting up a new master"); + master = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + // Failover should be completed, now wait for no RIT + log("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + log("No more RIT in ZK, now doing final test verification"); + + // Grab all the regions that are online across RSs + Set onlineRegions = new TreeSet(); + for (JVMClusterUtil.RegionServerThread rst : + cluster.getRegionServerThreads()) { + onlineRegions.addAll(rst.getRegionServer().getOnlineRegions()); + } + + // Now, everything that should be online should be online + for (HRegionInfo hri : regionsThatShouldBeOnline) { + assertTrue(onlineRegions.contains(hri)); + } + + // Everything that should be offline should not be online + for (HRegionInfo hri : regionsThatShouldBeOffline) { + assertFalse(onlineRegions.contains(hri)); + } + + log("Done with verification, all passed, shutting down cluster"); + + // Done, shutdown the cluster + TEST_UTIL.shutdownMiniCluster(); + } + + + /** + * Complex test of master failover that tests as many permutations of the + * different possible states that regions in transition could be in within ZK + * pointing to an RS that has died while no master is around to process it. + *

    + * This tests the proper handling of these states by the failed-over master + * and includes a thorough testing of the timeout code as well. + *

    + * Starts with a single master and two regionservers. + *

    + * Creates two tables, enabledTable and disabledTable, each containing 5 + * regions. The disabledTable is then disabled. + *

    + * After reaching steady-state, the master is killed. We then mock several + * states in ZK. And one of the RS will be killed. + *

    + * After mocking them and killing an RS, we will startup a new master which + * should become the active master and also detect that it is a failover. The + * primary test passing condition will be that all regions of the enabled + * table are assigned and all the regions of the disabled table are not + * assigned. + *

    + * The different scenarios to be tested are below: + *

    + * ZK State: CLOSING + *

    A node can get into CLOSING state if

    + *
      + *
    • An RS has begun to close a region + *
    + *

    We will mock the scenarios

    + *
      + *
    • Region was being closed but the RS died before finishing the close + *
    + * ZK State: OPENED + *

    A node can get into OPENED state if

    + *
      + *
    • An RS has finished opening a region but not acknowledged by master yet + *
    + *

    We will mock the scenarios

    + *
      + *
    • Region of a table that should be enabled was opened by a now-dead RS + *
    • Region of a table that should be disabled was opened by a now-dead RS + *
    + *

    + * ZK State: NONE + *

    A region could not have a transition node if

    + *
      + *
    • The server hosting the region died and no master processed it + *
    + *

    We will mock the scenarios

    + *
      + *
    • Region of enabled table was on a dead RS that was not yet processed + *
    • Region of disabled table was on a dead RS that was not yet processed + *
    + * @throws Exception + */ + @Test(timeout = 180000) + public void testMasterFailoverWithMockedRITOnDeadRS() throws Exception { + + final int NUM_MASTERS = 1; + final int NUM_RS = 2; + + // Create config to use for this cluster + Configuration conf = HBaseConfiguration.create(); + // Need to drop the timeout much lower + conf.setInt("hbase.master.assignment.timeoutmonitor.period", 4000); + conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 8000); + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2); + + // Create and start the cluster + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + log("Cluster started"); + + // Create a ZKW to use in the test + ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + "unittest", new Abortable() { + + @Override + public void abort(String why, Throwable e) { + LOG.error("Fatal ZK Error: " + why, e); + org.junit.Assert.assertFalse("Fatal ZK error", true); + } + + @Override + public boolean isAborted() { + return false; + } + + }); + + // get all the master threads + List masterThreads = cluster.getMasterThreads(); + assertEquals(1, masterThreads.size()); + + // only one master thread, let's wait for it to be initialized + assertTrue(cluster.waitForActiveAndReadyMaster()); + HMaster master = masterThreads.get(0).getMaster(); + assertTrue(master.isActiveMaster()); + assertTrue(master.isInitialized()); + + // disable load balancing on this master + master.balanceSwitch(false); + + // create two tables in META, each with 30 regions + byte [] FAMILY = Bytes.toBytes("family"); + byte[][] SPLIT_KEYS = + TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 30); + + byte [] enabledTable = Bytes.toBytes("enabledTable"); + HTableDescriptor htdEnabled = new HTableDescriptor(enabledTable); + htdEnabled.addFamily(new HColumnDescriptor(FAMILY)); + FileSystem filesystem = FileSystem.get(conf); + Path rootdir = filesystem.makeQualified( + new Path(conf.get(HConstants.HBASE_DIR))); + // Write the .tableinfo + FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdEnabled); + HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getName(), + null, null); + createRegion(hriEnabled, rootdir, conf, htdEnabled); + + List enabledRegions = TEST_UTIL.createMultiRegionsInMeta( + TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS); + + byte [] disabledTable = Bytes.toBytes("disabledTable"); + HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable); + htdDisabled.addFamily(new HColumnDescriptor(FAMILY)); + // Write the .tableinfo + FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled); + HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null); + createRegion(hriDisabled, rootdir, conf, htdDisabled); + + List disabledRegions = TEST_UTIL.createMultiRegionsInMeta( + TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS); + + log("Regions in META have been created"); + + // at this point we only expect 2 regions to be assigned out (catalogs) + assertEquals(2, cluster.countServedRegions()); + + // The first RS will stay online + List regionservers = + cluster.getRegionServerThreads(); + HRegionServer hrs = regionservers.get(0).getRegionServer(); + + // The second RS is going to be hard-killed + RegionServerThread hrsDeadThread = regionservers.get(1); + HRegionServer hrsDead = hrsDeadThread.getRegionServer(); + ServerName deadServerName = hrsDead.getServerName(); + + // we'll need some regions to already be assigned out properly on live RS + List enabledAndAssignedRegions = new ArrayList(); + enabledAndAssignedRegions.addAll(enabledRegions.subList(0, 6)); + enabledRegions.removeAll(enabledAndAssignedRegions); + List disabledAndAssignedRegions = new ArrayList(); + disabledAndAssignedRegions.addAll(disabledRegions.subList(0, 6)); + disabledRegions.removeAll(disabledAndAssignedRegions); + + // now actually assign them + for (HRegionInfo hri : enabledAndAssignedRegions) { + master.assignmentManager.regionPlans.put(hri.getEncodedName(), + new RegionPlan(hri, null, hrs.getServerName())); + master.assignRegion(hri); + } + for (HRegionInfo hri : disabledAndAssignedRegions) { + master.assignmentManager.regionPlans.put(hri.getEncodedName(), + new RegionPlan(hri, null, hrs.getServerName())); + master.assignRegion(hri); + } + + log("Waiting for assignment to finish"); + ZKAssign.blockUntilNoRIT(zkw); + master.assignmentManager.waitUntilNoRegionsInTransition(60000); + log("Assignment completed"); + + assertTrue(" Table must be enabled.", master.getAssignmentManager() + .getZKTable().isEnabledTable("enabledTable")); + // we also need regions assigned out on the dead server + List enabledAndOnDeadRegions = new ArrayList(); + enabledAndOnDeadRegions.addAll(enabledRegions.subList(0, 6)); + enabledRegions.removeAll(enabledAndOnDeadRegions); + List disabledAndOnDeadRegions = new ArrayList(); + disabledAndOnDeadRegions.addAll(disabledRegions.subList(0, 6)); + disabledRegions.removeAll(disabledAndOnDeadRegions); + + // set region plan to server to be killed and trigger assign + for (HRegionInfo hri : enabledAndOnDeadRegions) { + master.assignmentManager.regionPlans.put(hri.getEncodedName(), + new RegionPlan(hri, null, deadServerName)); + master.assignRegion(hri); + } + for (HRegionInfo hri : disabledAndOnDeadRegions) { + master.assignmentManager.regionPlans.put(hri.getEncodedName(), + new RegionPlan(hri, null, deadServerName)); + master.assignRegion(hri); + } + + // wait for no more RIT + log("Waiting for assignment to finish"); + ZKAssign.blockUntilNoRIT(zkw); + master.assignmentManager.waitUntilNoRegionsInTransition(60000); + log("Assignment completed"); + + // Due to master.assignRegion(hri) could fail to assign a region to a specified RS + // therefore, we need make sure that regions are in the expected RS + verifyRegionLocation(hrs, enabledAndAssignedRegions); + verifyRegionLocation(hrs, disabledAndAssignedRegions); + verifyRegionLocation(hrsDead, enabledAndOnDeadRegions); + verifyRegionLocation(hrsDead, disabledAndOnDeadRegions); + + assertTrue(" Didn't get enough regions of enabledTalbe on live rs.", + enabledAndAssignedRegions.size() >= 2); + assertTrue(" Didn't get enough regions of disalbedTable on live rs.", + disabledAndAssignedRegions.size() >= 2); + assertTrue(" Didn't get enough regions of enabledTalbe on dead rs.", + enabledAndOnDeadRegions.size() >= 2); + assertTrue(" Didn't get enough regions of disalbedTable on dead rs.", + disabledAndOnDeadRegions.size() >= 2); + + // Stop the master + log("Aborting master"); + cluster.abortMaster(0); + cluster.waitOnMaster(0); + log("Master has aborted"); + + /* + * Now, let's start mocking up some weird states as described in the method + * javadoc. + */ + + List regionsThatShouldBeOnline = new ArrayList(); + List regionsThatShouldBeOffline = new ArrayList(); + + log("Beginning to mock scenarios"); + + // Disable the disabledTable in ZK + ZKTable zktable = new ZKTable(zkw); + zktable.setDisabledTable(Bytes.toString(disabledTable)); + + assertTrue(" The enabled table should be identified on master fail over.", + zktable.isEnabledTable("enabledTable")); + + /* + * ZK = CLOSING + */ + + // Region of enabled table being closed on dead RS but not finished + HRegionInfo region = enabledAndOnDeadRegions.remove(0); + regionsThatShouldBeOnline.add(region); + ZKAssign.createNodeClosing(zkw, region, deadServerName); + LOG.debug("\n\nRegion of enabled table was CLOSING on dead RS\n" + + region + "\n\n"); + + // Region of disabled table being closed on dead RS but not finished + region = disabledAndOnDeadRegions.remove(0); + regionsThatShouldBeOffline.add(region); + ZKAssign.createNodeClosing(zkw, region, deadServerName); + LOG.debug("\n\nRegion of disabled table was CLOSING on dead RS\n" + + region + "\n\n"); + + /* + * ZK = CLOSED + */ + + // Region of enabled on dead server gets closed but not ack'd by master + region = enabledAndOnDeadRegions.remove(0); + regionsThatShouldBeOnline.add(region); + int version = ZKAssign.createNodeClosing(zkw, region, deadServerName); + ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version); + LOG.debug("\n\nRegion of enabled table was CLOSED on dead RS\n" + + region + "\n\n"); + + // Region of disabled on dead server gets closed but not ack'd by master + region = disabledAndOnDeadRegions.remove(0); + regionsThatShouldBeOffline.add(region); + version = ZKAssign.createNodeClosing(zkw, region, deadServerName); + ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version); + LOG.debug("\n\nRegion of disabled table was CLOSED on dead RS\n" + + region + "\n\n"); + + /* + * ZK = OPENING + */ + + // RS was opening a region of enabled table then died + region = enabledRegions.remove(0); + regionsThatShouldBeOnline.add(region); + ZKAssign.createNodeOffline(zkw, region, deadServerName); + ZKAssign.transitionNodeOpening(zkw, region, deadServerName); + LOG.debug("\n\nRegion of enabled table was OPENING on dead RS\n" + + region + "\n\n"); + + // RS was opening a region of disabled table then died + region = disabledRegions.remove(0); + regionsThatShouldBeOffline.add(region); + ZKAssign.createNodeOffline(zkw, region, deadServerName); + ZKAssign.transitionNodeOpening(zkw, region, deadServerName); + LOG.debug("\n\nRegion of disabled table was OPENING on dead RS\n" + + region + "\n\n"); + + /* + * ZK = OPENED + */ + + // Region of enabled table was opened on dead RS + region = enabledRegions.remove(0); + regionsThatShouldBeOnline.add(region); + ZKAssign.createNodeOffline(zkw, region, deadServerName); + hrsDead.openRegion(region); + while (true) { + RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName()); + if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) { + break; + } + Thread.sleep(100); + } + LOG.debug("\n\nRegion of enabled table was OPENED on dead RS\n" + + region + "\n\n"); + + // Region of disabled table was opened on dead RS + region = disabledRegions.remove(0); + regionsThatShouldBeOffline.add(region); + ZKAssign.createNodeOffline(zkw, region, deadServerName); + hrsDead.openRegion(region); + while (true) { + RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName()); + if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) { + break; + } + Thread.sleep(100); + } + LOG.debug("\n\nRegion of disabled table was OPENED on dead RS\n" + + region + "\n\n"); + + /* + * ZK = NONE + */ + + // Region of enabled table was open at steady-state on dead RS + region = enabledRegions.remove(0); + regionsThatShouldBeOnline.add(region); + ZKAssign.createNodeOffline(zkw, region, deadServerName); + hrsDead.openRegion(region); + while (true) { + RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName()); + if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) { + ZKAssign.deleteOpenedNode(zkw, region.getEncodedName()); + break; + } + Thread.sleep(100); + } + LOG.debug("\n\nRegion of enabled table was open at steady-state on dead RS" + + "\n" + region + "\n\n"); + + // Region of disabled table was open at steady-state on dead RS + region = disabledRegions.remove(0); + regionsThatShouldBeOffline.add(region); + ZKAssign.createNodeOffline(zkw, region, deadServerName); + hrsDead.openRegion(region); + while (true) { + RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName()); + if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) { + ZKAssign.deleteOpenedNode(zkw, region.getEncodedName()); + break; + } + Thread.sleep(100); + } + LOG.debug("\n\nRegion of disabled table was open at steady-state on dead RS" + + "\n" + region + "\n\n"); + + /* + * DONE MOCKING + */ + + log("Done mocking data up in ZK"); + + // Kill the RS that had a hard death + log("Killing RS " + deadServerName); + hrsDead.abort("Killing for unit test"); + log("RS " + deadServerName + " killed"); + + // Start up a new master. Wait until regionserver is completely down + // before starting new master because of hbase-4511. + while (hrsDeadThread.isAlive()) { + Threads.sleep(10); + } + log("Starting up a new master"); + master = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + assertTrue(cluster.waitForActiveAndReadyMaster()); + log("Master is ready"); + + // Let's add some weird states to master in-memory state + + // After HBASE-3181, we need to have some ZK state if we're PENDING_OPEN + // b/c it is impossible for us to get into this state w/o a zk node + // this is not true of PENDING_CLOSE + + // PENDING_OPEN and enabled + region = enabledRegions.remove(0); + regionsThatShouldBeOnline.add(region); + master.assignmentManager.regionsInTransition.put(region.getEncodedName(), + new RegionState(region, RegionState.State.PENDING_OPEN, 0, null)); + ZKAssign.createNodeOffline(zkw, region, master.getServerName()); + // PENDING_OPEN and disabled + region = disabledRegions.remove(0); + regionsThatShouldBeOffline.add(region); + master.assignmentManager.regionsInTransition.put(region.getEncodedName(), + new RegionState(region, RegionState.State.PENDING_OPEN, 0, null)); + ZKAssign.createNodeOffline(zkw, region, master.getServerName()); + // This test is bad. It puts up a PENDING_CLOSE but doesn't say what + // server we were PENDING_CLOSE against -- i.e. an entry in + // AssignmentManager#regions. W/o a server, we NPE trying to resend close. + // In past, there was wonky logic that had us reassign region if no server + // at tail of the unassign. This was removed. Commenting out for now. + // TODO: Remove completely. + /* + // PENDING_CLOSE and enabled + region = enabledRegions.remove(0); + LOG.info("Setting PENDING_CLOSE enabled " + region.getEncodedName()); + regionsThatShouldBeOnline.add(region); + master.assignmentManager.regionsInTransition.put(region.getEncodedName(), + new RegionState(region, RegionState.State.PENDING_CLOSE, 0)); + // PENDING_CLOSE and disabled + region = disabledRegions.remove(0); + LOG.info("Setting PENDING_CLOSE disabled " + region.getEncodedName()); + regionsThatShouldBeOffline.add(region); + master.assignmentManager.regionsInTransition.put(region.getEncodedName(), + new RegionState(region, RegionState.State.PENDING_CLOSE, 0)); + */ + + // Failover should be completed, now wait for no RIT + log("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + log("No more RIT in ZK"); + long now = System.currentTimeMillis(); + final long maxTime = 120000; + boolean done = master.assignmentManager.waitUntilNoRegionsInTransition(maxTime); + if (!done) { + LOG.info("rit=" + master.assignmentManager.getRegionsInTransition()); + } + long elapsed = System.currentTimeMillis() - now; + assertTrue("Elapsed=" + elapsed + ", maxTime=" + maxTime + ", done=" + done, + elapsed < maxTime); + log("No more RIT in RIT map, doing final test verification"); + + // Grab all the regions that are online across RSs + Set onlineRegions = new TreeSet(); + for (JVMClusterUtil.RegionServerThread rst : + cluster.getRegionServerThreads()) { + try { + onlineRegions.addAll(rst.getRegionServer().getOnlineRegions()); + } catch (org.apache.hadoop.hbase.regionserver.RegionServerStoppedException e) { + LOG.info("Got RegionServerStoppedException", e); + } + } + + // Now, everything that should be online should be online + for (HRegionInfo hri : regionsThatShouldBeOnline) { + assertTrue("region=" + hri.getRegionNameAsString(), onlineRegions.contains(hri)); + } + + // Everything that should be offline should not be online + for (HRegionInfo hri : regionsThatShouldBeOffline) { + assertFalse(onlineRegions.contains(hri)); + } + + log("Done with verification, all passed, shutting down cluster"); + + // Done, shutdown the cluster + TEST_UTIL.shutdownMiniCluster(); + } + + @Test(timeout = 180000) + public void testRSKilledWithMockedOpeningRITGoingToDeadRS() throws Exception { + final int NUM_MASTERS = 1; + final int NUM_RS = 2; + + // Create config to use for this cluster + Configuration conf = HBaseConfiguration.create(); + // Need to drop the timeout much lower + conf.setInt("hbase.master.assignment.timeoutmonitor.period", 10000); + conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 30000); + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2); + + // Create and start the cluster + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + log("Cluster started"); + + // Create a ZKW to use in the test + ZooKeeperWatcher zkw = + new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), "unittest", new Abortable() { + + @Override + public void abort(String why, Throwable e) { + LOG.error("Fatal ZK Error: " + why, e); + org.junit.Assert.assertFalse("Fatal ZK error", true); + } + + @Override + public boolean isAborted() { + return false; + } + + }); + + // get all the master threads + List masterThreads = cluster.getMasterThreads(); + assertEquals(1, masterThreads.size()); + + // only one master thread, let's wait for it to be initialized + assertTrue(cluster.waitForActiveAndReadyMaster()); + HMaster master = masterThreads.get(0).getMaster(); + assertTrue(master.isActiveMaster()); + assertTrue(master.isInitialized()); + + // disable load balancing on this master + master.balanceSwitch(false); + + // create two tables in META, each with 30 regions + byte[] FAMILY = Bytes.toBytes("family"); + byte[][] SPLIT_KEYS = + TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 15); + + FileSystem filesystem = FileSystem.get(conf); + Path rootdir = filesystem.makeQualified(new Path(conf.get(HConstants.HBASE_DIR))); + + byte[] disabledTable = Bytes.toBytes("disabledTable"); + HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable); + htdDisabled.addFamily(new HColumnDescriptor(FAMILY)); + // Write the .tableinfo + FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled); + HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null); + createRegion(hriDisabled, rootdir, conf, htdDisabled); + + List tableRegions = + TEST_UTIL.createMultiRegionsInMeta(TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS); + + log("Regions in META have been created"); + + // at this point we only expect 2 regions to be assigned out (catalogs) + assertEquals(2, cluster.countServedRegions()); + + // The first RS will stay online + List regionservers = cluster.getRegionServerThreads(); + HRegionServer hrs = regionservers.get(0).getRegionServer(); + + // The second RS is going to be hard-killed + RegionServerThread hrsDeadThread = regionservers.get(1); + HRegionServer hrsDead = hrsDeadThread.getRegionServer(); + ServerName deadServerName = hrsDead.getServerName(); + + // we'll need some regions to already be assigned out properly on live RS + List assignedRegionsOnLiveRS = new ArrayList(); + assignedRegionsOnLiveRS.addAll(tableRegions.subList(0, 3)); + tableRegions.removeAll(assignedRegionsOnLiveRS); + + // now actually assign them + for (HRegionInfo hri : assignedRegionsOnLiveRS) { + master.assignmentManager.regionPlans.put(hri.getEncodedName(), + new RegionPlan(hri, null, hrs.getServerName())); + master.assignRegion(hri); + } + + log("Waiting for assignment to finish"); + ZKAssign.blockUntilNoRIT(zkw); + master.assignmentManager.waitUntilNoRegionsInTransition(60000); + log("Assignment completed"); + + // Due to master.assignRegion(hri) could fail to assign a region to a specified RS + // therefore, we need make sure that regions are in the expected RS + verifyRegionLocation(hrs, assignedRegionsOnLiveRS); + + assertTrue(" Table must be enabled.", master.getAssignmentManager().getZKTable() + .isEnabledTable("disabledTable")); + + assertTrue(" Didn't get enough regions of enabledTalbe on live rs.", + assignedRegionsOnLiveRS.size() >= 1); + + // Disable the disabledTable in ZK + ZKTable zktable = master.assignmentManager.getZKTable(); + zktable.setDisablingTable("disabledTable"); + + // RS was opening a region of disabled table then died + HRegionInfo region = assignedRegionsOnLiveRS.remove(0); + master.assignmentManager.regionOffline(region); + master.assignmentManager.regionsInTransition.put(region.getEncodedName(), new RegionState( + region, RegionState.State.OPENING, System.currentTimeMillis(), deadServerName)); + ZKAssign.createNodeOffline(zkw, region, deadServerName); + ZKAssign.transitionNodeOpening(zkw, region, deadServerName); + + // Kill the RS that had a hard death + log("Killing RS " + deadServerName); + hrsDead.abort("Killing for unit test"); + while (hrsDeadThread.isAlive()) { + Threads.sleep(10); + } + log("RS " + deadServerName + " killed"); + + log("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + log("No more RIT in ZK"); + assertTrue(master.assignmentManager.waitUntilNoRegionsInTransition(120000)); + } + + /** + * Verify regions are on the expected region server + */ + private void verifyRegionLocation(HRegionServer hrs, List regions) + throws IOException { + List tmpOnlineRegions = hrs.getOnlineRegions(); + Iterator itr = regions.iterator(); + while (itr.hasNext()) { + HRegionInfo tmp = itr.next(); + if (!tmpOnlineRegions.contains(tmp)) { + itr.remove(); + } + } + } + + HRegion createRegion(final HRegionInfo hri, final Path rootdir, final Configuration c, + final HTableDescriptor htd) + throws IOException { + HRegion r = HRegion.createHRegion(hri, rootdir, c, htd); + // The above call to create a region will create an hlog file. Each + // log file create will also create a running thread to do syncing. We need + // to close out this log else we will have a running thread trying to sync + // the file system continuously which is ugly when dfs is taken away at the + // end of the test. + HRegion.closeHRegion(r); + return r; + } + + // TODO: Next test to add is with testing permutations of the RIT or the RS + // killed are hosting ROOT and META regions. + + private void log(String string) { + LOG.info("\n\n" + string + " \n\n"); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java new file mode 100644 index 0000000..0781d11 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test the master filesystem in a local cluster + */ +@Category(MediumTests.class) +public class TestMasterFileSystem { + + private static final Log LOG = LogFactory.getLog(TestMasterFileSystem.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupTest() throws Exception { + UTIL.startMiniCluster(); + } + + @AfterClass + public static void teardownTest() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test + public void testFsUriSetProperly() throws Exception { + HMaster master = UTIL.getMiniHBaseCluster().getMaster(); + MasterFileSystem fs = master.getMasterFileSystem(); + Path masterRoot = FSUtils.getRootDir(fs.conf); + Path rootDir = FSUtils.getRootDir(fs.getFileSystem().getConf()); + // make sure the fs and the found root dir have the same scheme + LOG.debug("from fs uri:" + FileSystem.getDefaultUri(fs.getFileSystem().getConf())); + LOG.debug("from configuration uri:" + FileSystem.getDefaultUri(fs.conf)); + // make sure the set uri matches by forcing it. + assertEquals(masterRoot, rootDir); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java new file mode 100644 index 0000000..ec08b17 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java @@ -0,0 +1,146 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestMasterRestartAfterDisablingTable { + + private static final Log LOG = LogFactory.getLog(TestMasterRestartAfterDisablingTable.class); + + @Test + public void testForCheckingIfEnableAndDisableWorksFineAfterSwitch() + throws Exception { + final int NUM_MASTERS = 2; + final int NUM_RS = 1; + final int NUM_REGIONS_TO_CREATE = 4; + + // Start the cluster + log("Starting cluster"); + Configuration conf = HBaseConfiguration.create(); + conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000); + conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 5000); + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + log("Waiting for active/ready master"); + cluster.waitForActiveAndReadyMaster(); + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "testmasterRestart", null); + HMaster master = cluster.getMaster(); + + // Create a table with regions + byte[] table = Bytes.toBytes("tableRestart"); + byte[] family = Bytes.toBytes("family"); + log("Creating table with " + NUM_REGIONS_TO_CREATE + " regions"); + HTable ht = TEST_UTIL.createTable(table, family); + int numRegions = TEST_UTIL.createMultiRegions(conf, ht, family, + NUM_REGIONS_TO_CREATE); + numRegions += 2; // catalogs + log("Waiting for no more RIT\n"); + blockUntilNoRIT(zkw, master); + log("Disabling table\n"); + TEST_UTIL.getHBaseAdmin().disableTable(table); + + NavigableSet regions = getAllOnlineRegions(cluster); + assertEquals( + "The number of regions for the table tableRestart should be 0 and only" + + "the catalog tables should be present.", 2, regions.size()); + + List masterThreads = cluster.getMasterThreads(); + MasterThread activeMaster = null; + if (masterThreads.get(0).getMaster().isActiveMaster()) { + activeMaster = masterThreads.get(0); + } else { + activeMaster = masterThreads.get(1); + } + activeMaster.getMaster().stop( + "stopping the active master so that the backup can become active"); + cluster.hbaseCluster.waitOnMaster(activeMaster); + cluster.waitForActiveAndReadyMaster(); + + assertTrue("The table should not be in enabled state", cluster.getMaster() + .getAssignmentManager().getZKTable().isDisablingOrDisabledTable( + "tableRestart")); + log("Enabling table\n"); + // Need a new Admin, the previous one is on the old master + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + admin.enableTable(table); + admin.close(); + log("Waiting for no more RIT\n"); + blockUntilNoRIT(zkw, master); + log("Verifying there are " + numRegions + " assigned on cluster\n"); + regions = getAllOnlineRegions(cluster); + assertEquals( + "The assigned regions were not onlined after master switch except for the catalog tables.", + 6, regions.size()); + assertTrue("The table should be in enabled state", cluster.getMaster() + .getAssignmentManager().getZKTable().isEnabledTable("tableRestart")); + ht.close(); + TEST_UTIL.shutdownMiniCluster(); + } + + private void log(String msg) { + LOG.debug("\n\nTRR: " + msg + "\n"); + } + + private void blockUntilNoRIT(ZooKeeperWatcher zkw, HMaster master) + throws KeeperException, InterruptedException { + ZKAssign.blockUntilNoRIT(zkw); + master.assignmentManager.waitUntilNoRegionsInTransition(60000); + } + + private NavigableSet getAllOnlineRegions(MiniHBaseCluster cluster) + throws IOException { + NavigableSet online = new TreeSet(); + for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) { + for (HRegionInfo region : rst.getRegionServer().getOnlineRegions()) { + online.add(region.getRegionNameAsString()); + } + } + return online; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java new file mode 100644 index 0000000..01e6bd8 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java @@ -0,0 +1,140 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.hadoop.hbase.*; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestMasterShutdown { + private static final Log LOG = LogFactory.getLog(TestMasterShutdown.class); + + /** + * Simple test of shutdown. + *

    + * Starts with three masters. Tells the active master to shutdown the cluster. + * Verifies that all masters are properly shutdown. + * @throws Exception + */ + @Test (timeout=240000) + public void testMasterShutdown() throws Exception { + + final int NUM_MASTERS = 3; + final int NUM_RS = 3; + + // Create config to use for this cluster + Configuration conf = HBaseConfiguration.create(); + + // Start the cluster + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + + // get all the master threads + List masterThreads = cluster.getMasterThreads(); + + // wait for each to come online + for (MasterThread mt : masterThreads) { + assertTrue(mt.isAlive()); + } + + // find the active master + HMaster active = null; + for (int i = 0; i < masterThreads.size(); i++) { + if (masterThreads.get(i).getMaster().isActiveMaster()) { + active = masterThreads.get(i).getMaster(); + break; + } + } + assertNotNull(active); + // make sure the other two are backup masters + ClusterStatus status = active.getClusterStatus(); + assertEquals(2, status.getBackupMastersSize()); + assertEquals(2, status.getBackupMasters().size()); + + // tell the active master to shutdown the cluster + active.shutdown(); + + for (int i = NUM_MASTERS - 1; i >= 0 ;--i) { + cluster.waitOnMaster(i); + } + // make sure all the masters properly shutdown + assertEquals(0,masterThreads.size()); + + TEST_UTIL.shutdownMiniCluster(); + } + + @Test//(timeout = 180000) + public void testMasterShutdownBeforeStartingAnyRegionServer() throws Exception { + + final int NUM_MASTERS = 1; + final int NUM_RS = 0; + + // Create config to use for this cluster + Configuration conf = HBaseConfiguration.create(); + + // Start the cluster + final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniDFSCluster(1); + TEST_UTIL.startMiniZKCluster(); + TEST_UTIL.createRootDir(); + final LocalHBaseCluster cluster = + new LocalHBaseCluster(conf, NUM_MASTERS, NUM_RS, HMaster.class, + MiniHBaseCluster.MiniHBaseClusterRegionServer.class); + final MasterThread master = cluster.getMasters().get(0); + master.start(); + Thread shutdownThread = new Thread() { + public void run() { + try { + TEST_UTIL.getHBaseAdmin().shutdown(); + cluster.waitOnMaster(0); + } catch (Exception e) { + } + }; + }; + shutdownThread.start(); + master.join(); + shutdownThread.join(); + + List masterThreads = cluster.getMasters(); + // make sure all the masters properly shutdown + assertEquals(0, masterThreads.size()); + + TEST_UTIL.shutdownMiniZKCluster(); + TEST_UTIL.cleanupTestDir(); + TEST_UTIL.shutdownMiniDFSCluster(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java new file mode 100644 index 0000000..be5dea5 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java @@ -0,0 +1,198 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.HashSet; +import java.util.List; +import java.util.NavigableMap; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hbase.tmpl.master.AssignmentManagerStatusTmpl; +import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +/** + * Tests for the master status page and its template. + */ +@Category(MediumTests.class) +public class TestMasterStatusServlet { + + private HMaster master; + private Configuration conf; + private HBaseAdmin admin; + + static final ServerName FAKE_HOST = + new ServerName("fakehost", 12345, 1234567890); + static final HTableDescriptor FAKE_TABLE = + new HTableDescriptor("mytable"); + static final HRegionInfo FAKE_HRI = + new HRegionInfo(FAKE_TABLE.getName(), Bytes.toBytes("a"), Bytes.toBytes("b")); + + @Before + public void setupBasicMocks() { + conf = HBaseConfiguration.create(); + + master = Mockito.mock(HMaster.class); + Mockito.doReturn(FAKE_HOST).when(master).getServerName(); + Mockito.doReturn(conf).when(master).getConfiguration(); + + // Fake serverManager + ServerManager serverManager = Mockito.mock(ServerManager.class); + Mockito.doReturn(1.0).when(serverManager).getAverageLoad(); + Mockito.doReturn(serverManager).when(master).getServerManager(); + + // Fake AssignmentManager and RIT + AssignmentManager am = Mockito.mock(AssignmentManager.class); + NavigableMap regionsInTransition = + Maps.newTreeMap(); + regionsInTransition.put("r1", + new RegionState(FAKE_HRI, RegionState.State.CLOSING, 12345L, FAKE_HOST)); + Mockito.doReturn(regionsInTransition).when(am).getRegionsInTransition(); + Mockito.doReturn(am).when(master).getAssignmentManager(); + + // Fake ZKW + ZooKeeperWatcher zkw = Mockito.mock(ZooKeeperWatcher.class); + Mockito.doReturn("fakequorum").when(zkw).getQuorum(); + Mockito.doReturn(zkw).when(master).getZooKeeperWatcher(); + + // Mock admin + admin = Mockito.mock(HBaseAdmin.class); + } + + + private void setupMockTables() throws IOException { + HTableDescriptor tables[] = new HTableDescriptor[] { + new HTableDescriptor("foo"), + new HTableDescriptor("bar") + }; + Mockito.doReturn(tables).when(admin).listTables(); + } + + @Test + public void testStatusTemplateNoTables() throws IOException { + new MasterStatusTmpl().render(new StringWriter(), + master, admin); + } + + @Test + public void testStatusTemplateRootAvailable() throws IOException { + new MasterStatusTmpl() + .setRootLocation(new ServerName("rootserver:123,12345")) + .render(new StringWriter(), + master, admin); + } + + @Test + public void testStatusTemplateRootAndMetaAvailable() throws IOException { + setupMockTables(); + + new MasterStatusTmpl() + .setRootLocation(new ServerName("rootserver:123,12345")) + .setMetaLocation(new ServerName("metaserver:123,12345")) + .render(new StringWriter(), + master, admin); + } + + @Test + public void testStatusTemplateWithServers() throws IOException { + setupMockTables(); + + List servers = Lists.newArrayList( + new ServerName("rootserver:123,12345"), + new ServerName("metaserver:123,12345")); + Set deadServers = new HashSet( + Lists.newArrayList( + new ServerName("badserver:123,12345"), + new ServerName("uglyserver:123,12345")) + ); + + new MasterStatusTmpl() + .setRootLocation(new ServerName("rootserver:123,12345")) + .setMetaLocation(new ServerName("metaserver:123,12345")) + .setServers(servers) + .setDeadServers(deadServers) + .render(new StringWriter(), + master, admin); + } + + @Test + public void testAssignmentManagerTruncatedList() throws IOException { + AssignmentManager am = Mockito.mock(AssignmentManager.class); + + // Add 100 regions as in-transition + NavigableMap regionsInTransition = + Maps.newTreeMap(); + for (byte i = 0; i < 100; i++) { + HRegionInfo hri = new HRegionInfo(FAKE_TABLE.getName(), + new byte[]{i}, new byte[]{(byte) (i+1)}); + regionsInTransition.put(hri.getEncodedName(), + new RegionState(hri, RegionState.State.CLOSING, 12345L, FAKE_HOST)); + } + // Add META in transition as well + regionsInTransition.put( + HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(), + new RegionState(HRegionInfo.FIRST_META_REGIONINFO, + RegionState.State.CLOSING, 12345L, FAKE_HOST)); + Mockito.doReturn(regionsInTransition).when(am).getRegionsInTransition(); + + // Render to a string + StringWriter sw = new StringWriter(); + new AssignmentManagerStatusTmpl() + .setLimit(50) + .render(sw, am); + String result = sw.toString(); + + // Should always include META + assertTrue(result.contains(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())); + + // Make sure we only see 50 of them + Matcher matcher = Pattern.compile("CLOSING").matcher(result); + int count = 0; + while (matcher.find()) { + count++; + } + assertEquals(50, count); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java new file mode 100644 index 0000000..762a2b8 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java @@ -0,0 +1,545 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test transitions of state across the master. Sets up the cluster once and + * then runs a couple of tests. + */ +@Category(LargeTests.class) +public class TestMasterTransitions { + private static final Log LOG = LogFactory.getLog(TestMasterTransitions.class); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String TABLENAME = "master_transitions"; + private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"), + Bytes.toBytes("b"), Bytes.toBytes("c")}; + + /** + * Start up a mini cluster and put a small table of many empty regions into it. + * @throws Exception + */ + @BeforeClass public static void beforeAllTests() throws Exception { + TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); + TEST_UTIL.startMiniCluster(2); + // Create a table of three families. This will assign a region. + byte[] tableName = Bytes.toBytes(TABLENAME); + TEST_UTIL.createTable(tableName, FAMILIES); + HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); + int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily()); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + addToEachStartKey(countOfRegions); + t.close(); + } + + @AfterClass public static void afterAllTests() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before public void setup() throws IOException { + TEST_UTIL.ensureSomeRegionServersAvailable(2); + } + + /** + * Listener for regionserver events testing hbase-2428 (Infinite loop of + * region closes if META region is offline). In particular, listen + * for the close of the 'metaServer' and when it comes in, requeue it with a + * delay as though there were an issue processing the shutdown. As part of + * the requeuing, send over a close of a region on 'otherServer' so it comes + * into a master that has its meta region marked as offline. + */ + /* + static class HBase2428Listener implements RegionServerOperationListener { + // Map of what we've delayed so we don't do do repeated delays. + private final Set postponed = + new CopyOnWriteArraySet(); + private boolean done = false;; + private boolean metaShutdownReceived = false; + private final HServerAddress metaAddress; + private final MiniHBaseCluster cluster; + private final int otherServerIndex; + private final HRegionInfo hri; + private int closeCount = 0; + static final int SERVER_DURATION = 3 * 1000; + static final int CLOSE_DURATION = 1 * 1000; + + HBase2428Listener(final MiniHBaseCluster c, final HServerAddress metaAddress, + final HRegionInfo closingHRI, final int otherServerIndex) { + this.cluster = c; + this.metaAddress = metaAddress; + this.hri = closingHRI; + this.otherServerIndex = otherServerIndex; + } + + @Override + public boolean process(final RegionServerOperation op) throws IOException { + // If a regionserver shutdown and its of the meta server, then we want to + // delay the processing of the shutdown and send off a close of a region on + // the 'otherServer. + boolean result = true; + if (op instanceof ProcessServerShutdown) { + ProcessServerShutdown pss = (ProcessServerShutdown)op; + if (pss.getDeadServerAddress().equals(this.metaAddress)) { + // Don't postpone more than once. + if (!this.postponed.contains(pss)) { + // Close some region. + this.cluster.addMessageToSendRegionServer(this.otherServerIndex, + new HMsg(HMsg.Type.MSG_REGION_CLOSE, hri, + Bytes.toBytes("Forcing close in test"))); + this.postponed.add(pss); + // Put off the processing of the regionserver shutdown processing. + pss.setDelay(SERVER_DURATION); + this.metaShutdownReceived = true; + // Return false. This will add this op to the delayed queue. + result = false; + } + } + } else { + // Have the close run frequently. + if (isWantedCloseOperation(op) != null) { + op.setDelay(CLOSE_DURATION); + // Count how many times it comes through here. + this.closeCount++; + } + } + return result; + } + + public void processed(final RegionServerOperation op) { + if (isWantedCloseOperation(op) != null) return; + this.done = true; + } +*/ + /* + * @param op + * @return Null if not the wanted ProcessRegionClose, else op + * cast as a ProcessRegionClose. + */ + /* + private ProcessRegionClose isWantedCloseOperation(final RegionServerOperation op) { + // Count every time we get a close operation. + if (op instanceof ProcessRegionClose) { + ProcessRegionClose c = (ProcessRegionClose)op; + if (c.regionInfo.equals(hri)) { + return c; + } + } + return null; + } + + boolean isDone() { + return this.done; + } + + boolean isMetaShutdownReceived() { + return metaShutdownReceived; + } + + int getCloseCount() { + return this.closeCount; + } + + @Override + public boolean process(HServerInfo serverInfo, HMsg incomingMsg) { + return true; + } + } +*/ + /** + * In 2428, the meta region has just been set offline and then a close comes + * in. + * @see HBASE-2428 + */ + @Ignore @Test (timeout=300000) public void testRegionCloseWhenNoMetaHBase2428() + throws Exception { + /* + LOG.info("Running testRegionCloseWhenNoMetaHBase2428"); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + final HMaster master = cluster.getMaster(); + int metaIndex = cluster.getServerWithMeta(); + // Figure the index of the server that is not server the .META. + int otherServerIndex = -1; + for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) { + if (i == metaIndex) continue; + otherServerIndex = i; + break; + } + final HRegionServer otherServer = cluster.getRegionServer(otherServerIndex); + final HRegionServer metaHRS = cluster.getRegionServer(metaIndex); + + // Get a region out on the otherServer. + final HRegionInfo hri = + otherServer.getOnlineRegions().iterator().next().getRegionInfo(); + + // Add our RegionServerOperationsListener + HBase2428Listener listener = new HBase2428Listener(cluster, + metaHRS.getHServerInfo().getServerAddress(), hri, otherServerIndex); + master.getRegionServerOperationQueue(). + registerRegionServerOperationListener(listener); + try { + // Now close the server carrying meta. + cluster.abortRegionServer(metaIndex); + + // First wait on receipt of meta server shutdown message. + while(!listener.metaShutdownReceived) Threads.sleep(100); + while(!listener.isDone()) Threads.sleep(10); + // We should not have retried the close more times than it took for the + // server shutdown message to exit the delay queue and get processed + // (Multiple by two to add in some slop in case of GC or something). + assertTrue(listener.getCloseCount() > 1); + assertTrue(listener.getCloseCount() < + ((HBase2428Listener.SERVER_DURATION/HBase2428Listener.CLOSE_DURATION) * 2)); + + // Assert the closed region came back online + assertRegionIsBackOnline(hri); + } finally { + master.getRegionServerOperationQueue(). + unregisterRegionServerOperationListener(listener); + } + */ + } + + /** + * Test adding in a new server before old one on same host+port is dead. + * Make the test more onerous by having the server under test carry the meta. + * If confusion between old and new, purportedly meta never comes back. Test + * that meta gets redeployed. + */ + @Ignore @Test (timeout=300000) public void testAddingServerBeforeOldIsDead2413() + throws IOException { + /* + LOG.info("Running testAddingServerBeforeOldIsDead2413"); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + int count = count(); + int metaIndex = cluster.getServerWithMeta(); + MiniHBaseClusterRegionServer metaHRS = + (MiniHBaseClusterRegionServer)cluster.getRegionServer(metaIndex); + int port = metaHRS.getServerInfo().getServerAddress().getPort(); + Configuration c = TEST_UTIL.getConfiguration(); + String oldPort = c.get(HConstants.REGIONSERVER_PORT, "0"); + try { + LOG.info("KILLED=" + metaHRS); + metaHRS.kill(); + c.set(HConstants.REGIONSERVER_PORT, Integer.toString(port)); + // Try and start new regionserver. It might clash with the old + // regionserver port so keep trying to get past the BindException. + HRegionServer hrs = null; + while (true) { + try { + hrs = cluster.startRegionServer().getRegionServer(); + break; + } catch (IOException e) { + if (e.getCause() != null && e.getCause() instanceof InvocationTargetException) { + InvocationTargetException ee = (InvocationTargetException)e.getCause(); + if (ee.getCause() != null && ee.getCause() instanceof BindException) { + LOG.info("BindException; retrying: " + e.toString()); + } + } + } + } + LOG.info("STARTED=" + hrs); + // Wait until he's been given at least 3 regions before we go on to try + // and count rows in table. + while (hrs.getOnlineRegions().size() < 3) Threads.sleep(100); + LOG.info(hrs.toString() + " has " + hrs.getOnlineRegions().size() + + " regions"); + assertEquals(count, count()); + } finally { + c.set(HConstants.REGIONSERVER_PORT, oldPort); + } + */ + } + + /** + * HBase2482 is about outstanding region openings. If any are outstanding + * when a regionserver goes down, then they'll never deploy. They'll be + * stuck in the regions-in-transition list for ever. This listener looks + * for a region opening HMsg and if its from the server passed on construction, + * then we kill it. It also looks out for a close message on the victim + * server because that signifies start of the fireworks. + */ + /* + static class HBase2482Listener implements RegionServerOperationListener { + private final HRegionServer victim; + private boolean abortSent = false; + // We closed regions on new server. + private volatile boolean closed = false; + // Copy of regions on new server + private final Collection copyOfOnlineRegions; + // This is the region that was in transition on the server we aborted. Test + // passes if this region comes back online successfully. + private HRegionInfo regionToFind; + + HBase2482Listener(final HRegionServer victim) { + this.victim = victim; + // Copy regions currently open on this server so I can notice when + // there is a close. + this.copyOfOnlineRegions = + this.victim.getCopyOfOnlineRegionsSortedBySize().values(); + } + + @Override + public boolean process(HServerInfo serverInfo, HMsg incomingMsg) { + if (!victim.getServerInfo().equals(serverInfo) || + this.abortSent || !this.closed) { + return true; + } + if (!incomingMsg.isType(HMsg.Type.MSG_REPORT_PROCESS_OPEN)) return true; + // Save the region that is in transition so can test later it came back. + this.regionToFind = incomingMsg.getRegionInfo(); + String msg = "ABORTING " + this.victim + " because got a " + + HMsg.Type.MSG_REPORT_PROCESS_OPEN + " on this server for " + + incomingMsg.getRegionInfo().getRegionNameAsString(); + this.victim.abort(msg); + this.abortSent = true; + return true; + } + + @Override + public boolean process(RegionServerOperation op) throws IOException { + return true; + } + + @Override + public void processed(RegionServerOperation op) { + if (this.closed || !(op instanceof ProcessRegionClose)) return; + ProcessRegionClose close = (ProcessRegionClose)op; + for (HRegion r: this.copyOfOnlineRegions) { + if (r.getRegionInfo().equals(close.regionInfo)) { + // We've closed one of the regions that was on the victim server. + // Now can start testing for when all regions are back online again + LOG.info("Found close of " + + r.getRegionInfo().getRegionNameAsString() + + "; setting close happened flag"); + this.closed = true; + break; + } + } + } + } +*/ + /** + * In 2482, a RS with an opening region on it dies. The said region is then + * stuck in the master's regions-in-transition and never leaves it. This + * test works by bringing up a new regionserver, waiting for the load + * balancer to give it some regions. Then, we close all on the new server. + * After sending all the close messages, we send the new regionserver the + * special blocking message so it can not process any more messages. + * Meantime reopening of the just-closed regions is backed up on the new + * server. Soon as master gets an opening region from the new regionserver, + * we kill it. We then wait on all regions to come back on line. If bug + * is fixed, this should happen soon as the processing of the killed server is + * done. + * @see HBASE-2482 + */ + @Ignore @Test (timeout=300000) public void testKillRSWithOpeningRegion2482() + throws Exception { + /* + LOG.info("Running testKillRSWithOpeningRegion2482"); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + if (cluster.getLiveRegionServerThreads().size() < 2) { + // Need at least two servers. + cluster.startRegionServer(); + } + // Count how many regions are online. They need to be all back online for + // this test to succeed. + int countOfMetaRegions = countOfMetaRegions(); + // Add a listener on the server. + HMaster m = cluster.getMaster(); + // Start new regionserver. + MiniHBaseClusterRegionServer hrs = + (MiniHBaseClusterRegionServer)cluster.startRegionServer().getRegionServer(); + LOG.info("Started new regionserver: " + hrs.toString()); + // Wait until has some regions before proceeding. Balancer will give it some. + int minimumRegions = + countOfMetaRegions/(cluster.getRegionServerThreads().size() * 2); + while (hrs.getOnlineRegions().size() < minimumRegions) Threads.sleep(100); + // Set the listener only after some regions have been opened on new server. + HBase2482Listener listener = new HBase2482Listener(hrs); + m.getRegionServerOperationQueue(). + registerRegionServerOperationListener(listener); + try { + // Go close all non-catalog regions on this new server + closeAllNonCatalogRegions(cluster, hrs); + // After all closes, add blocking message before the region opens start to + // come in. + cluster.addMessageToSendRegionServer(hrs, + new HMsg(HMsg.Type.TESTING_BLOCK_REGIONSERVER)); + // Wait till one of the above close messages has an effect before we start + // wait on all regions back online. + while (!listener.closed) Threads.sleep(100); + LOG.info("Past close"); + // Make sure the abort server message was sent. + while(!listener.abortSent) Threads.sleep(100); + LOG.info("Past abort send; waiting on all regions to redeploy"); + // Now wait for regions to come back online. + assertRegionIsBackOnline(listener.regionToFind); + } finally { + m.getRegionServerOperationQueue(). + unregisterRegionServerOperationListener(listener); + } + */ + } + + /* + * @return Count of all non-catalog regions on the designated server + */ +/* + private int closeAllNonCatalogRegions(final MiniHBaseCluster cluster, + final MiniHBaseCluster.MiniHBaseClusterRegionServer hrs) + throws IOException { + int countOfRegions = 0; + for (HRegion r: hrs.getOnlineRegions()) { + if (r.getRegionInfo().isMetaRegion()) continue; + cluster.addMessageToSendRegionServer(hrs, + new HMsg(HMsg.Type.MSG_REGION_CLOSE, r.getRegionInfo())); + LOG.info("Sent close of " + r.getRegionInfo().getRegionNameAsString() + + " on " + hrs.toString()); + countOfRegions++; + } + return countOfRegions; + } + + private void assertRegionIsBackOnline(final HRegionInfo hri) + throws IOException { + // Region should have an entry in its startkey because of addRowToEachRegion. + byte [] row = getStartKey(hri); + HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); + Get g = new Get(row); + assertTrue((t.get(g)).size() > 0); + } + + /* + * @return Count of regions in meta table. + * @throws IOException + */ + /* + private static int countOfMetaRegions() + throws IOException { + HTable meta = new HTable(TEST_UTIL.getConfiguration(), + HConstants.META_TABLE_NAME); + int rows = 0; + Scan scan = new Scan(); + scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); + ResultScanner s = meta.getScanner(scan); + for (Result r = null; (r = s.next()) != null;) { + byte [] b = + r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); + if (b == null || b.length <= 0) break; + rows++; + } + s.close(); + return rows; + } +*/ + /* + * Add to each of the regions in .META. a value. Key is the startrow of the + * region (except its 'aaa' for first region). Actual value is the row name. + * @param expected + * @return + * @throws IOException + */ + private static int addToEachStartKey(final int expected) throws IOException { + HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); + HTable meta = new HTable(TEST_UTIL.getConfiguration(), + HConstants.META_TABLE_NAME); + int rows = 0; + Scan scan = new Scan(); + scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + ResultScanner s = meta.getScanner(scan); + for (Result r = null; (r = s.next()) != null;) { + byte [] b = + r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + if (b == null || b.length <= 0) break; + HRegionInfo hri = Writables.getHRegionInfo(b); + // If start key, add 'aaa'. + byte [] row = getStartKey(hri); + Put p = new Put(row); + p.setWriteToWAL(false); + p.add(getTestFamily(), getTestQualifier(), row); + t.put(p); + rows++; + } + s.close(); + Assert.assertEquals(expected, rows); + t.close(); + meta.close(); + return rows; + } + + /* + * @return Count of rows in TABLENAME + * @throws IOException + */ + private static int count() throws IOException { + HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); + int rows = 0; + Scan scan = new Scan(); + ResultScanner s = t.getScanner(scan); + for (Result r = null; (r = s.next()) != null;) { + rows++; + } + s.close(); + LOG.info("Counted=" + rows); + t.close(); + return rows; + } + + /* + * @param hri + * @return Start key for hri (If start key is '', then return 'aaa'. + */ + private static byte [] getStartKey(final HRegionInfo hri) { + return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())? + Bytes.toBytes("aaa"): hri.getStartKey(); + } + + private static byte [] getTestFamily() { + return FAMILIES[0]; + } + + private static byte [] getTestQualifier() { + return getTestFamily(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java new file mode 100644 index 0000000..d643e90 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterZKSessionRecovery.java @@ -0,0 +1,174 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test cases for master to recover from ZK session expiry. + */ +@Category(MediumTests.class) +public class TestMasterZKSessionRecovery { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + /** + * The default timeout is 5 minutes. + * Shorten it so that the test won't wait for too long. + */ + static { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setLong("hbase.master.zksession.recover.timeout", 50000); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + MockLoadBalancer.class, LoadBalancer.class); + } + + @Before + public void setUp() throws Exception { + // Start a cluster of one regionserver. + TEST_UTIL.startMiniCluster(1); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Tests that the master does not call retainAssignment after recovery from + * expired zookeeper session. Without the HBASE-6046 fix master always tries + * to assign all the user regions by calling retainAssignment. + */ + @Test + public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Exception { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + cluster.startRegionServer(); + HMaster m = cluster.getMaster(); + ZooKeeperWatcher zkw = m.getZooKeeperWatcher(); + int expectedNumOfListeners = zkw.getNumberOfListeners(); + // now the cluster is up. So assign some regions. + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), + Bytes.toBytes("c"), Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"), + Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j") }; + + String tableName = "testRegionAssignmentAfterMasterRecoveryDueToZKExpiry"; + admin.createTable(new HTableDescriptor(tableName), SPLIT_KEYS); + ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL); + ZKAssign.blockUntilNoRIT(zooKeeperWatcher); + m.getZooKeeperWatcher().close(); + MockLoadBalancer.retainAssignCalled = false; + m.abort("Test recovery from zk session expired", new KeeperException.SessionExpiredException()); + assertFalse(m.isStopped()); + // The recovered master should not call retainAssignment, as it is not a + // clean startup. + assertFalse("Retain assignment should not be called", MockLoadBalancer.retainAssignCalled); + // number of listeners should be same as the value before master aborted + assertEquals(expectedNumOfListeners, zkw.getNumberOfListeners()); + } + + static class MockLoadBalancer extends DefaultLoadBalancer { + static boolean retainAssignCalled = false; + + @Override + public Map> retainAssignment( + Map regions, List servers) { + retainAssignCalled = true; + return super.retainAssignment(regions, servers); + } + } + + /** + * Tests whether the logs are split when master recovers from a expired + * zookeeper session and an RS goes down. + */ + @Test(timeout = 60000) + public void testLogSplittingAfterMasterRecoveryDueToZKExpiry() throws IOException, + KeeperException, InterruptedException { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + cluster.startRegionServer(); + HMaster m = cluster.getMaster(); + // now the cluster is up. So assign some regions. + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("1"), Bytes.toBytes("2"), + Bytes.toBytes("3"), Bytes.toBytes("4"), Bytes.toBytes("5") }; + + String tableName = "testLogSplittingAfterMasterRecoveryDueToZKExpiry"; + HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("col"); + htd.addFamily(hcd); + admin.createTable(htd, SPLIT_KEYS); + ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL); + ZKAssign.blockUntilNoRIT(zooKeeperWatcher); + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + + Put p = null; + int numberOfPuts = 0; + for (numberOfPuts = 0; numberOfPuts < 6; numberOfPuts++) { + p = new Put(Bytes.toBytes(numberOfPuts)); + p.add(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("value" + numberOfPuts)); + table.put(p); + } + m.getZooKeeperWatcher().close(); + m.abort("Test recovery from zk session expired", new KeeperException.SessionExpiredException()); + assertFalse(m.isStopped()); + cluster.getRegionServer(0).abort("Aborting"); + // Without patch for HBASE-6046 this test case will always timeout + // with patch the test case should pass. + Scan scan = new Scan(); + int numberOfRows = 0; + ResultScanner scanner = table.getScanner(scan); + Result[] result = scanner.next(1); + while (result != null && result.length > 0) { + numberOfRows++; + result = scanner.next(1); + } + assertEquals("Number of rows should be equal to number of puts.", numberOfPuts, numberOfRows); + } +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestOpenedRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/master/TestOpenedRegionHandler.java new file mode 100644 index 0000000..a38b49d --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestOpenedRegionHandler.java @@ -0,0 +1,215 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; +import org.apache.hadoop.hbase.master.handler.OpenedRegionHandler; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.MockServer; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKTable; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.data.Stat; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +@Category(MediumTests.class) +public class TestOpenedRegionHandler { + + private static final Log LOG = LogFactory + .getLog(TestOpenedRegionHandler.class); + + private HBaseTestingUtility TEST_UTIL; + private final int NUM_MASTERS = 1; + private final int NUM_RS = 1; + private Configuration conf; + private Configuration resetConf; + private ZooKeeperWatcher zkw; + + @Before + public void setUp() throws Exception { + conf = HBaseConfiguration.create(); + TEST_UTIL = new HBaseTestingUtility(conf); + } + + @After + public void tearDown() throws Exception { + // Stop the cluster + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testOpenedRegionHandlerOnMasterRestart() throws Exception { + // Start the cluster + log("Starting cluster"); + conf = HBaseConfiguration.create(); + resetConf = conf; + conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000); + conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 5000); + TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + String tableName = "testOpenedRegionHandlerOnMasterRestart"; + MiniHBaseCluster cluster = createRegions(tableName); + abortMaster(cluster); + + HRegionServer regionServer = cluster.getRegionServer(0); + HRegion region = getRegionBeingServed(cluster, regionServer); + + // forcefully move a region to OPENED state in zk + // Create a ZKW to use in the test + zkw = HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL, + region, regionServer.getServerName()); + + // Start up a new master + log("Starting up a new master"); + cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + // Failover should be completed, now wait for no RIT + log("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + } + @Test + public void testShouldNotCompeleteOpenedRegionSuccessfullyIfVersionMismatches() + throws Exception { + HRegion region = null; + try { + int testIndex = 0; + TEST_UTIL.startMiniZKCluster(); + final Server server = new MockServer(TEST_UTIL); + HTableDescriptor htd = new HTableDescriptor( + "testShouldNotCompeleteOpenedRegionSuccessfullyIfVersionMismatches"); + HRegionInfo hri = new HRegionInfo(htd.getName(), + Bytes.toBytes(testIndex), Bytes.toBytes(testIndex + 1)); + region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + assertNotNull(region); + AssignmentManager am = Mockito.mock(AssignmentManager.class); + when(am.isRegionInTransition(hri)).thenReturn( + new RegionState(region.getRegionInfo(), RegionState.State.OPEN, + System.currentTimeMillis(), server.getServerName())); + // create a node with OPENED state + zkw = HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL, + region, server.getServerName()); + when(am.getZKTable()).thenReturn(new ZKTable(zkw)); + Stat stat = new Stat(); + String nodeName = ZKAssign.getNodeName(zkw, region.getRegionInfo() + .getEncodedName()); + ZKUtil.getDataAndWatch(zkw, nodeName, stat); + + // use the version for the OpenedRegionHandler + OpenedRegionHandler handler = new OpenedRegionHandler(server, am, region + .getRegionInfo(), server.getServerName(), stat.getVersion()); + // Once again overwrite the same znode so that the version changes. + ZKAssign.transitionNode(zkw, region.getRegionInfo(), server + .getServerName(), EventType.RS_ZK_REGION_OPENED, + EventType.RS_ZK_REGION_OPENED, stat.getVersion()); + + // Should not invoke assignmentmanager.regionOnline. If it is + // invoked as per current mocking it will throw null pointer exception. + boolean expectedException = false; + try { + handler.process(); + } catch (Exception e) { + expectedException = true; + } + assertFalse("The process method should not throw any exception.", + expectedException); + List znodes = ZKUtil.listChildrenAndWatchForNewChildren(zkw, + zkw.assignmentZNode); + String regionName = znodes.get(0); + assertEquals("The region should not be opened successfully.", regionName, + region.getRegionInfo().getEncodedName()); + } finally { + region.close(); + region.getLog().closeAndDelete(); + TEST_UTIL.shutdownMiniZKCluster(); + } + } + private MiniHBaseCluster createRegions(String tableName) + throws InterruptedException, ZooKeeperConnectionException, IOException, + KeeperException { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + log("Waiting for active/ready master"); + cluster.waitForActiveAndReadyMaster(); + zkw = new ZooKeeperWatcher(conf, "testOpenedRegionHandler", null); + + // Create a table with regions + byte[] table = Bytes.toBytes(tableName); + byte[] family = Bytes.toBytes("family"); + TEST_UTIL.createTable(table, family); + + //wait till the regions are online + log("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + + return cluster; + } + private void abortMaster(MiniHBaseCluster cluster) { + // Stop the master + log("Aborting master"); + cluster.abortMaster(0); + cluster.waitOnMaster(0); + log("Master has aborted"); + } + private HRegion getRegionBeingServed(MiniHBaseCluster cluster, + HRegionServer regionServer) { + Collection onlineRegionsLocalContext = regionServer + .getOnlineRegionsLocalContext(); + Iterator iterator = onlineRegionsLocalContext.iterator(); + HRegion region = null; + while (iterator.hasNext()) { + region = iterator.next(); + if (!region.getRegionInfo().isMetaTable()) { + break; + } + } + return region; + } + private void log(String msg) { + LOG.debug("\n\nTRR: " + msg + "\n"); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java b/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java new file mode 100644 index 0000000..3b8b6c6 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java @@ -0,0 +1,140 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.MetaScanner; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.After; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestRestartCluster { + private static final Log LOG = LogFactory.getLog(TestRestartCluster.class); + private HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static final byte[] TABLENAME = Bytes.toBytes("master_transitions"); + private static final byte [][] FAMILIES = {Bytes.toBytes("a")}; + private static final byte [][] TABLES = { + Bytes.toBytes("restartTableOne"), + Bytes.toBytes("restartTableTwo"), + Bytes.toBytes("restartTableThree") + }; + private static final byte [] FAMILY = Bytes.toBytes("family"); + + @After public void tearDown() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test (timeout=300000) public void testRestartClusterAfterKill() + throws Exception { + UTIL.startMiniZKCluster(); + ZooKeeperWatcher zooKeeper = + new ZooKeeperWatcher(UTIL.getConfiguration(), "cluster1", null, true); + + // create the unassigned region, throw up a region opened state for META + String unassignedZNode = zooKeeper.assignmentZNode; + ZKUtil.createAndFailSilent(zooKeeper, unassignedZNode); + + ServerName sn = new ServerName(HMaster.MASTER, -1, System.currentTimeMillis()); + + ZKAssign.createNodeOffline(zooKeeper, HRegionInfo.ROOT_REGIONINFO, sn); + + ZKAssign.createNodeOffline(zooKeeper, HRegionInfo.FIRST_META_REGIONINFO, sn); + + LOG.debug("Created UNASSIGNED zNode for ROOT and META regions in state " + + EventType.M_ZK_REGION_OFFLINE); + + // start the HB cluster + LOG.info("Starting HBase cluster..."); + UTIL.startMiniCluster(2); + + UTIL.createTable(TABLENAME, FAMILIES); + LOG.info("Created a table, waiting for table to be available..."); + UTIL.waitTableAvailable(TABLENAME, 60*1000); + + LOG.info("Master deleted unassigned region and started up successfully."); + } + + @Test (timeout=300000) + public void testClusterRestart() throws Exception { + UTIL.startMiniCluster(3); + while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) { + Threads.sleep(1); + } + LOG.info("\n\nCreating tables"); + for(byte [] TABLE : TABLES) { + UTIL.createTable(TABLE, FAMILY); + } + for(byte [] TABLE : TABLES) { + UTIL.waitTableAvailable(TABLE, 30000); + } + + List allRegions = + MetaScanner.listAllRegions(UTIL.getConfiguration()); + assertEquals(3, allRegions.size()); + + LOG.info("\n\nShutting down cluster"); + UTIL.shutdownMiniHBaseCluster(); + + LOG.info("\n\nSleeping a bit"); + Thread.sleep(2000); + + LOG.info("\n\nStarting cluster the second time"); + UTIL.restartHBaseCluster(3); + + // Need to use a new 'Configuration' so we make a new HConnection. + // Otherwise we're reusing an HConnection that has gone stale because + // the shutdown of the cluster also called shut of the connection. + allRegions = MetaScanner. + listAllRegions(new Configuration(UTIL.getConfiguration())); + assertEquals(3, allRegions.size()); + + LOG.info("\n\nWaiting for tables to be available"); + for(byte [] TABLE: TABLES) { + try { + UTIL.createTable(TABLE, FAMILY); + assertTrue("Able to create table that should already exist", false); + } catch(TableExistsException tee) { + LOG.info("Table already exists as expected"); + } + UTIL.waitTableAvailable(TABLE, 30000); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java b/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java new file mode 100644 index 0000000..9462d11 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java @@ -0,0 +1,415 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.List; +import java.util.NavigableSet; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests the restarting of everything as done during rolling restarts. + */ +@Category(LargeTests.class) +public class TestRollingRestart { + private static final Log LOG = LogFactory.getLog(TestRollingRestart.class); + + @Test (timeout=300000) + public void testBasicRollingRestart() throws Exception { + + // Start a cluster with 2 masters and 4 regionservers + final int NUM_MASTERS = 2; + final int NUM_RS = 3; + final int NUM_REGIONS_TO_CREATE = 20; + + int expectedNumRS = 3; + + // Start the cluster + log("Starting cluster"); + Configuration conf = HBaseConfiguration.create(); + conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000); + conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 5000); + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + log("Waiting for active/ready master"); + cluster.waitForActiveAndReadyMaster(); + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "testRollingRestart", + null); + HMaster master = cluster.getMaster(); + + // Create a table with regions + byte [] table = Bytes.toBytes("tableRestart"); + byte [] family = Bytes.toBytes("family"); + log("Creating table with " + NUM_REGIONS_TO_CREATE + " regions"); + HTable ht = TEST_UTIL.createTable(table, family); + int numRegions = TEST_UTIL.createMultiRegions(conf, ht, family, + NUM_REGIONS_TO_CREATE); + numRegions += 2; // catalogs + log("Waiting for no more RIT\n"); + blockUntilNoRIT(zkw, master); + log("Disabling table\n"); + TEST_UTIL.getHBaseAdmin().disableTable(table); + log("Waiting for no more RIT\n"); + blockUntilNoRIT(zkw, master); + NavigableSet regions = getAllOnlineRegions(cluster); + log("Verifying only catalog regions are assigned\n"); + if (regions.size() != 2) { + for (String oregion : regions) log("Region still online: " + oregion); + } + assertEquals(2, regions.size()); + log("Enabling table\n"); + TEST_UTIL.getHBaseAdmin().enableTable(table); + log("Waiting for no more RIT\n"); + blockUntilNoRIT(zkw, master); + log("Verifying there are " + numRegions + " assigned on cluster\n"); + regions = getAllOnlineRegions(cluster); + assertRegionsAssigned(cluster, regions); + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + + // Add a new regionserver + log("Adding a fourth RS"); + RegionServerThread restarted = cluster.startRegionServer(); + expectedNumRS++; + restarted.waitForServerOnline(); + log("Additional RS is online"); + log("Waiting for no more RIT"); + blockUntilNoRIT(zkw, master); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + + // Master Restarts + List masterThreads = cluster.getMasterThreads(); + MasterThread activeMaster = null; + MasterThread backupMaster = null; + assertEquals(2, masterThreads.size()); + if (masterThreads.get(0).getMaster().isActiveMaster()) { + activeMaster = masterThreads.get(0); + backupMaster = masterThreads.get(1); + } else { + activeMaster = masterThreads.get(1); + backupMaster = masterThreads.get(0); + } + + // Bring down the backup master + log("Stopping backup master\n\n"); + backupMaster.getMaster().stop("Stop of backup during rolling restart"); + cluster.hbaseCluster.waitOnMaster(backupMaster); + + // Bring down the primary master + log("Stopping primary master\n\n"); + activeMaster.getMaster().stop("Stop of active during rolling restart"); + cluster.hbaseCluster.waitOnMaster(activeMaster); + + // Start primary master + log("Restarting primary master\n\n"); + activeMaster = cluster.startMaster(); + cluster.waitForActiveAndReadyMaster(); + master = activeMaster.getMaster(); + + // Start backup master + log("Restarting backup master\n\n"); + backupMaster = cluster.startMaster(); + + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + + // RegionServer Restarts + + // Bring them down, one at a time, waiting between each to complete + List regionServers = + cluster.getLiveRegionServerThreads(); + int num = 1; + int total = regionServers.size(); + for (RegionServerThread rst : regionServers) { + ServerName serverName = rst.getRegionServer().getServerName(); + log("Stopping region server " + num + " of " + total + " [ " + + serverName + "]"); + rst.getRegionServer().stop("Stopping RS during rolling restart"); + cluster.hbaseCluster.waitOnRegionServer(rst); + log("Waiting for RS shutdown to be handled by master"); + waitForRSShutdownToStartAndFinish(activeMaster, serverName); + log("RS shutdown done, waiting for no more RIT"); + blockUntilNoRIT(zkw, master); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + expectedNumRS--; + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + log("Restarting region server " + num + " of " + total); + restarted = cluster.startRegionServer(); + restarted.waitForServerOnline(); + expectedNumRS++; + log("Region server " + num + " is back online"); + log("Waiting for no more RIT"); + blockUntilNoRIT(zkw, master); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + num++; + } + Thread.sleep(2000); + assertRegionsAssigned(cluster, regions); + + // Bring the RS hosting ROOT down and the RS hosting META down at once + RegionServerThread rootServer = getServerHostingRoot(cluster); + RegionServerThread metaServer = getServerHostingMeta(cluster); + if (rootServer == metaServer) { + log("ROOT and META on the same server so killing another random server"); + int i=0; + while (rootServer == metaServer) { + metaServer = cluster.getRegionServerThreads().get(i); + i++; + } + } + log("Stopping server hosting ROOT"); + rootServer.getRegionServer().stop("Stopping ROOT server"); + log("Stopping server hosting META #1"); + metaServer.getRegionServer().stop("Stopping META server"); + cluster.hbaseCluster.waitOnRegionServer(rootServer); + log("Root server down"); + cluster.hbaseCluster.waitOnRegionServer(metaServer); + log("Meta server down #1"); + expectedNumRS -= 2; + log("Waiting for meta server #1 RS shutdown to be handled by master"); + waitForRSShutdownToStartAndFinish(activeMaster, + metaServer.getRegionServer().getServerName()); + log("Waiting for no more RIT"); + long start = System.currentTimeMillis(); + do { + blockUntilNoRIT(zkw, master); + } while (getNumberOfOnlineRegions(cluster) < numRegions + && System.currentTimeMillis()-start < 60000); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + + // Kill off the server hosting META again + metaServer = getServerHostingMeta(cluster); + log("Stopping server hosting META #2"); + metaServer.getRegionServer().stop("Stopping META server"); + cluster.hbaseCluster.waitOnRegionServer(metaServer); + log("Meta server down"); + expectedNumRS--; + log("Waiting for RS shutdown to be handled by master"); + waitForRSShutdownToStartAndFinish(activeMaster, + metaServer.getRegionServer().getServerName()); + log("RS shutdown done, waiting for no more RIT"); + blockUntilNoRIT(zkw, master); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + + // Start 3 RS again + cluster.startRegionServer().waitForServerOnline(); + cluster.startRegionServer().waitForServerOnline(); + cluster.startRegionServer().waitForServerOnline(); + Thread.sleep(1000); + log("Waiting for no more RIT"); + blockUntilNoRIT(zkw, master); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + // Shutdown server hosting META + metaServer = getServerHostingMeta(cluster); + log("Stopping server hosting META (1 of 3)"); + metaServer.getRegionServer().stop("Stopping META server"); + cluster.hbaseCluster.waitOnRegionServer(metaServer); + log("Meta server down (1 of 3)"); + log("Waiting for RS shutdown to be handled by master"); + waitForRSShutdownToStartAndFinish(activeMaster, + metaServer.getRegionServer().getServerName()); + log("RS shutdown done, waiting for no more RIT"); + blockUntilNoRIT(zkw, master); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + + // Shutdown server hosting META again + metaServer = getServerHostingMeta(cluster); + log("Stopping server hosting META (2 of 3)"); + metaServer.getRegionServer().stop("Stopping META server"); + cluster.hbaseCluster.waitOnRegionServer(metaServer); + log("Meta server down (2 of 3)"); + log("Waiting for RS shutdown to be handled by master"); + waitForRSShutdownToStartAndFinish(activeMaster, + metaServer.getRegionServer().getServerName()); + log("RS shutdown done, waiting for no more RIT"); + blockUntilNoRIT(zkw, master); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + + // Shutdown server hosting META again + metaServer = getServerHostingMeta(cluster); + log("Stopping server hosting META (3 of 3)"); + metaServer.getRegionServer().stop("Stopping META server"); + cluster.hbaseCluster.waitOnRegionServer(metaServer); + log("Meta server down (3 of 3)"); + log("Waiting for RS shutdown to be handled by master"); + waitForRSShutdownToStartAndFinish(activeMaster, + metaServer.getRegionServer().getServerName()); + log("RS shutdown done, waiting for no more RIT"); + blockUntilNoRIT(zkw, master); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + + if (cluster.getRegionServerThreads().size() != 1) { + log("Online regionservers:"); + for (RegionServerThread rst : cluster.getRegionServerThreads()) { + log("RS: " + rst.getRegionServer().getServerName()); + } + } + assertEquals(1, cluster.getRegionServerThreads().size()); + + + // TODO: Bring random 3 of 4 RS down at the same time + + ht.close(); + // Stop the cluster + TEST_UTIL.shutdownMiniCluster(); + } + + private void blockUntilNoRIT(ZooKeeperWatcher zkw, HMaster master) + throws KeeperException, InterruptedException { + ZKAssign.blockUntilNoRIT(zkw); + master.assignmentManager.waitUntilNoRegionsInTransition(60000); + } + + private void waitForRSShutdownToStartAndFinish(MasterThread activeMaster, + ServerName serverName) throws InterruptedException { + ServerManager sm = activeMaster.getMaster().getServerManager(); + // First wait for it to be in dead list + while (!sm.getDeadServers().contains(serverName)) { + log("Waiting for [" + serverName + "] to be listed as dead in master"); + Thread.sleep(1); + } + log("Server [" + serverName + "] marked as dead, waiting for it to " + + "finish dead processing"); + while (sm.areDeadServersInProgress()) { + log("Server [" + serverName + "] still being processed, waiting"); + Thread.sleep(100); + } + log("Server [" + serverName + "] done with server shutdown processing"); + } + + private void log(String msg) { + LOG.debug("\n\nTRR: " + msg + "\n"); + } + + private RegionServerThread getServerHostingMeta(MiniHBaseCluster cluster) + throws IOException { + return getServerHosting(cluster, HRegionInfo.FIRST_META_REGIONINFO); + } + + private RegionServerThread getServerHostingRoot(MiniHBaseCluster cluster) + throws IOException { + return getServerHosting(cluster, HRegionInfo.ROOT_REGIONINFO); + } + + private RegionServerThread getServerHosting(MiniHBaseCluster cluster, + HRegionInfo region) throws IOException { + for (RegionServerThread rst : cluster.getRegionServerThreads()) { + if (rst.getRegionServer().getOnlineRegions().contains(region)) { + return rst; + } + } + return null; + } + + private int getNumberOfOnlineRegions(MiniHBaseCluster cluster) { + int numFound = 0; + for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) { + numFound += rst.getRegionServer().getNumberOfOnlineRegions(); + } + return numFound; + } + + private void assertRegionsAssigned(MiniHBaseCluster cluster, + Set expectedRegions) throws IOException { + int numFound = getNumberOfOnlineRegions(cluster); + if (expectedRegions.size() > numFound) { + log("Expected to find " + expectedRegions.size() + " but only found" + + " " + numFound); + NavigableSet foundRegions = getAllOnlineRegions(cluster); + for (String region : expectedRegions) { + if (!foundRegions.contains(region)) { + log("Missing region: " + region); + } + } + assertEquals(expectedRegions.size(), numFound); + } else if (expectedRegions.size() < numFound) { + int doubled = numFound - expectedRegions.size(); + log("Expected to find " + expectedRegions.size() + " but found" + + " " + numFound + " (" + doubled + " double assignments?)"); + NavigableSet doubleRegions = getDoubleAssignedRegions(cluster); + for (String region : doubleRegions) { + log("Region is double assigned: " + region); + } + assertEquals(expectedRegions.size(), numFound); + } else { + log("Success! Found expected number of " + numFound + " regions"); + } + } + + private NavigableSet getAllOnlineRegions(MiniHBaseCluster cluster) + throws IOException { + NavigableSet online = new TreeSet(); + for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) { + for (HRegionInfo region : rst.getRegionServer().getOnlineRegions()) { + online.add(region.getRegionNameAsString()); + } + } + return online; + } + + private NavigableSet getDoubleAssignedRegions( + MiniHBaseCluster cluster) throws IOException { + NavigableSet online = new TreeSet(); + NavigableSet doubled = new TreeSet(); + for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) { + for (HRegionInfo region : rst.getRegionServer().getOnlineRegions()) { + if(!online.add(region.getRegionNameAsString())) { + doubled.add(region.getRegionNameAsString()); + } + } + } + return doubled; + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java new file mode 100644 index 0000000..cb1d3e8 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java @@ -0,0 +1,514 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.*; + +import java.io.IOException; +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; + +import junit.framework.Assert; + +import static org.junit.Assert.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.master.SplitLogManager.Task; +import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch; +import org.apache.hadoop.hbase.regionserver.TestMasterAddressManager.NodeCreationListener; +import org.apache.hadoop.hbase.zookeeper.ZKSplitLog; +import org.apache.hadoop.hbase.zookeeper.ZKSplitLog.TaskState; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs.Ids; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +@Category(MediumTests.class) +public class TestSplitLogManager { + private static final Log LOG = LogFactory.getLog(TestSplitLogManager.class); + static { + Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG); + } + + private ZooKeeperWatcher zkw; + private static boolean stopped = false; + private SplitLogManager slm; + private Configuration conf; + private int to; + private final ServerManager sm = Mockito.mock(ServerManager.class); + private final MasterServices master = Mockito.mock(MasterServices.class); + + private final static HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + static Stoppable stopper = new Stoppable() { + @Override + public void stop(String why) { + stopped = true; + } + + @Override + public boolean isStopped() { + return stopped; + } + + }; + + @Before + public void setup() throws Exception { + TEST_UTIL.startMiniZKCluster(); + conf = TEST_UTIL.getConfiguration(); + zkw = new ZooKeeperWatcher(conf, "split-log-manager-tests", null); + ZKUtil.deleteChildrenRecursively(zkw, zkw.baseZNode); + ZKUtil.createAndFailSilent(zkw, zkw.baseZNode); + assertTrue(ZKUtil.checkExists(zkw, zkw.baseZNode) != -1); + LOG.debug(zkw.baseZNode + " created"); + ZKUtil.createAndFailSilent(zkw, zkw.splitLogZNode); + assertTrue(ZKUtil.checkExists(zkw, zkw.splitLogZNode) != -1); + LOG.debug(zkw.splitLogZNode + " created"); + + stopped = false; + resetCounters(); + to = 6000; + conf.setInt("hbase.splitlog.manager.timeout", to); + conf.setInt("hbase.splitlog.manager.unassigned.timeout", 2 * to); + conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 100); + to = to + 4 * 100; + + // By default, we let the test manage the error as before, so the server + // does not appear as dead from the master point of view, only from the split log pov. + Mockito.when(sm.isServerOnline(Mockito.any(ServerName.class))).thenReturn(true); + Mockito.when(master.getServerManager()).thenReturn(sm); + } + + @After + public void teardown() throws IOException, KeeperException { + stopper.stop(""); + slm.stop(); + TEST_UTIL.shutdownMiniZKCluster(); + } + + private interface Expr { + public long eval(); + } + + private void waitForCounter(final AtomicLong ctr, long oldval, long newval, + long timems) { + Expr e = new Expr() { + public long eval() { + return ctr.get(); + } + }; + waitForCounter(e, oldval, newval, timems); + return; + } + + private void waitForCounter(Expr e, long oldval, long newval, + long timems) { + long curt = System.currentTimeMillis(); + long endt = curt + timems; + while (curt < endt) { + if (e.eval() == oldval) { + try { + Thread.sleep(10); + } catch (InterruptedException eintr) { + } + curt = System.currentTimeMillis(); + } else { + assertEquals(newval, e.eval()); + return; + } + } + assertTrue(false); + } + + private String submitTaskAndWait(TaskBatch batch, String name) + throws KeeperException, InterruptedException { + String tasknode = ZKSplitLog.getEncodedNodeName(zkw, name); + NodeCreationListener listener = new NodeCreationListener(zkw, tasknode); + zkw.registerListener(listener); + ZKUtil.watchAndCheckExists(zkw, tasknode); + + slm.enqueueSplitTask(name, batch); + assertEquals(1, batch.installed); + assertTrue(slm.findOrCreateOrphanTask(tasknode).batch == batch); + assertEquals(1L, tot_mgr_node_create_queued.get()); + + LOG.debug("waiting for task node creation"); + listener.waitForCreation(); + LOG.debug("task created"); + return tasknode; + } + + /** + * Test whether the splitlog correctly creates a task in zookeeper + * @throws Exception + */ + @Test + public void testTaskCreation() throws Exception { + LOG.info("TestTaskCreation - test the creation of a task in zk"); + + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + TaskBatch batch = new TaskBatch(); + + String tasknode = submitTaskAndWait(batch, "foo/1"); + + byte[] data = ZKUtil.getData(zkw, tasknode); + LOG.info("Task node created " + new String(data)); + assertTrue(TaskState.TASK_UNASSIGNED.equals(data, "dummy-master")); + } + + @Test + public void testOrphanTaskAcquisition() throws Exception { + LOG.info("TestOrphanTaskAcquisition"); + + String tasknode = ZKSplitLog.getEncodedNodeName(zkw, "orphan/test/slash"); + zkw.getRecoverableZooKeeper().create(tasknode, + TaskState.TASK_OWNED.get("dummy-worker"), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2); + Task task = slm.findOrCreateOrphanTask(tasknode); + assertTrue(task.isOrphan()); + waitForCounter(tot_mgr_heartbeat, 0, 1, to/2); + assertFalse(task.isUnassigned()); + long curt = System.currentTimeMillis(); + assertTrue((task.last_update <= curt) && + (task.last_update > (curt - 1000))); + LOG.info("waiting for manager to resubmit the orphan task"); + waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2); + assertTrue(task.isUnassigned()); + waitForCounter(tot_mgr_rescan, 0, 1, to + to/2); + } + + @Test + public void testUnassignedOrphan() throws Exception { + LOG.info("TestUnassignedOrphan - an unassigned task is resubmitted at" + + " startup"); + String tasknode = ZKSplitLog.getEncodedNodeName(zkw, "orphan/test/slash"); + //create an unassigned orphan task + zkw.getRecoverableZooKeeper().create(tasknode, + TaskState.TASK_UNASSIGNED.get("dummy-worker"), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + int version = ZKUtil.checkExists(zkw, tasknode); + + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2); + Task task = slm.findOrCreateOrphanTask(tasknode); + assertTrue(task.isOrphan()); + assertTrue(task.isUnassigned()); + // wait for RESCAN node to be created + waitForCounter(tot_mgr_rescan, 0, 1, to/2); + Task task2 = slm.findOrCreateOrphanTask(tasknode); + assertTrue(task == task2); + LOG.debug("task = " + task); + assertEquals(1L, tot_mgr_resubmit.get()); + assertEquals(1, task.incarnation); + assertEquals(0, task.unforcedResubmits); + assertTrue(task.isOrphan()); + assertTrue(task.isUnassigned()); + assertTrue(ZKUtil.checkExists(zkw, tasknode) > version); + } + + @Test + public void testMultipleResubmits() throws Exception { + LOG.info("TestMultipleResbmits - no indefinite resubmissions"); + + conf.setInt("hbase.splitlog.max.resubmit", 2); + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + TaskBatch batch = new TaskBatch(); + + String tasknode = submitTaskAndWait(batch, "foo/1"); + int version = ZKUtil.checkExists(zkw, tasknode); + + ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker1")); + waitForCounter(tot_mgr_heartbeat, 0, 1, to/2); + waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2); + int version1 = ZKUtil.checkExists(zkw, tasknode); + assertTrue(version1 > version); + ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker2")); + waitForCounter(tot_mgr_heartbeat, 1, 2, to/2); + waitForCounter(tot_mgr_resubmit, 1, 2, to + to/2); + int version2 = ZKUtil.checkExists(zkw, tasknode); + assertTrue(version2 > version1); + ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker3")); + waitForCounter(tot_mgr_heartbeat, 1, 2, to/2); + waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + to/2); + Thread.sleep(to + to/2); + assertEquals(2L, tot_mgr_resubmit.get()); + } + + @Test + public void testRescanCleanup() throws Exception { + LOG.info("TestRescanCleanup - ensure RESCAN nodes are cleaned up"); + + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + TaskBatch batch = new TaskBatch(); + + String tasknode = submitTaskAndWait(batch, "foo/1"); + int version = ZKUtil.checkExists(zkw, tasknode); + + ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker1")); + waitForCounter(tot_mgr_heartbeat, 0, 1, to/2); + waitForCounter(new Expr() { + @Override + public long eval() { + return (tot_mgr_resubmit.get() + tot_mgr_resubmit_failed.get()); + } + }, 0, 1, 5*60000); // wait long enough + Assert + .assertEquals("Could not run test. Lost ZK connection?", 0, tot_mgr_resubmit_failed.get()); + int version1 = ZKUtil.checkExists(zkw, tasknode); + assertTrue(version1 > version); + byte[] taskstate = ZKUtil.getData(zkw, tasknode); + assertTrue(Arrays.equals(TaskState.TASK_UNASSIGNED.get("dummy-master"), taskstate)); + + waitForCounter(tot_mgr_rescan_deleted, 0, 1, to / 2); + } + + @Test + public void testTaskDone() throws Exception { + LOG.info("TestTaskDone - cleanup task node once in DONE state"); + + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + TaskBatch batch = new TaskBatch(); + String tasknode = submitTaskAndWait(batch, "foo/1"); + ZKUtil.setData(zkw, tasknode, TaskState.TASK_DONE.get("worker")); + synchronized (batch) { + while (batch.installed != batch.done) { + batch.wait(); + } + } + waitForCounter(tot_mgr_task_deleted, 0, 1, to/2); + assertTrue(ZKUtil.checkExists(zkw, tasknode) == -1); + } + + @Test + public void testTaskErr() throws Exception { + LOG.info("TestTaskErr - cleanup task node once in ERR state"); + + conf.setInt("hbase.splitlog.max.resubmit", 0); + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + TaskBatch batch = new TaskBatch(); + + String tasknode = submitTaskAndWait(batch, "foo/1"); + ZKUtil.setData(zkw, tasknode, TaskState.TASK_ERR.get("worker")); + synchronized (batch) { + while (batch.installed != batch.error) { + batch.wait(); + } + } + waitForCounter(tot_mgr_task_deleted, 0, 1, to/2); + assertTrue(ZKUtil.checkExists(zkw, tasknode) == -1); + conf.setInt("hbase.splitlog.max.resubmit", ZKSplitLog.DEFAULT_MAX_RESUBMIT); + } + + @Test + public void testTaskResigned() throws Exception { + LOG.info("TestTaskResigned - resubmit task node once in RESIGNED state"); + + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + TaskBatch batch = new TaskBatch(); + String tasknode = submitTaskAndWait(batch, "foo/1"); + ZKUtil.setData(zkw, tasknode, TaskState.TASK_RESIGNED.get("worker")); + int version = ZKUtil.checkExists(zkw, tasknode); + + waitForCounter(tot_mgr_resubmit, 0, 1, to/2); + int version1 = ZKUtil.checkExists(zkw, tasknode); + assertTrue(version1 > version); + + byte[] taskstate = ZKUtil.getData(zkw, tasknode); + assertTrue(Arrays.equals(taskstate, + TaskState.TASK_UNASSIGNED.get("dummy-master"))); + } + + @Test + public void testUnassignedTimeout() throws Exception { + LOG.info("TestUnassignedTimeout - iff all tasks are unassigned then" + + " resubmit"); + + // create an orphan task in OWNED state + String tasknode1 = ZKSplitLog.getEncodedNodeName(zkw, "orphan/1"); + zkw.getRecoverableZooKeeper().create(tasknode1, + TaskState.TASK_OWNED.get("dummy-worker"), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2); + + + // submit another task which will stay in unassigned mode + TaskBatch batch = new TaskBatch(); + submitTaskAndWait(batch, "foo/1"); + + // keep updating the orphan owned node every to/2 seconds + for (int i = 0; i < (3 * to)/100; i++) { + Thread.sleep(100); + ZKUtil.setData(zkw, tasknode1, + TaskState.TASK_OWNED.get("dummy-worker")); + } + + // since we have stopped heartbeating the owned node therefore it should + // get resubmitted + LOG.info("waiting for manager to resubmit the orphan task"); + waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2); + + // now all the nodes are unassigned. manager should post another rescan + waitForCounter(tot_mgr_resubmit_unassigned, 0, 1, 2 * to + to/2); + } + + @Test + public void testDeadWorker() throws Exception { + LOG.info("testDeadWorker"); + + conf.setLong("hbase.splitlog.max.resubmit", 0); + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + TaskBatch batch = new TaskBatch(); + + String tasknode = submitTaskAndWait(batch, "foo/1"); + int version = ZKUtil.checkExists(zkw, tasknode); + + ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get("worker1")); + waitForCounter(tot_mgr_heartbeat, 0, 1, to/2); + slm.handleDeadWorker("worker1"); + waitForCounter(tot_mgr_resubmit, 0, 1, to/2); + waitForCounter(tot_mgr_resubmit_dead_server_task, 0, 1, to + to/2); + + int version1 = ZKUtil.checkExists(zkw, tasknode); + assertTrue(version1 > version); + byte[] taskstate = ZKUtil.getData(zkw, tasknode); + assertTrue(Arrays.equals(TaskState.TASK_UNASSIGNED.get("dummy-master"), + taskstate)); + return; + } + + @Test + public void testEmptyLogDir() throws Exception { + LOG.info("testEmptyLogDir"); + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + Path emptyLogDirPath = new Path(fs.getWorkingDirectory(), + UUID.randomUUID().toString()); + fs.mkdirs(emptyLogDirPath); + slm.splitLogDistributed(emptyLogDirPath); + assertFalse(fs.exists(emptyLogDirPath)); + } + + @Test(timeout=45000) + public void testVanishingTaskZNode() throws Exception { + LOG.info("testVanishingTaskZNode"); + conf.setInt("hbase.splitlog.manager.unassigned.timeout", 0); + conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 1000); + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + final Path logDir = new Path(fs.getWorkingDirectory(), + UUID.randomUUID().toString()); + fs.mkdirs(logDir); + Thread thread = null; + try { + Path logFile = new Path(logDir, UUID.randomUUID().toString()); + fs.createNewFile(logFile); + thread = new Thread() { + public void run() { + try { + // this call will block because there are no SplitLogWorkers, + // until the task znode is deleted below. Then the call will + // complete successfully, assuming the log is split. + slm.splitLogDistributed(logDir); + } catch (Exception e) { + LOG.warn("splitLogDistributed failed", e); + } + } + }; + thread.start(); + waitForCounter(tot_mgr_node_create_result, 0, 1, 10000); + String znode = ZKSplitLog.getEncodedNodeName(zkw, logFile.toString()); + // remove the task znode, to finish the distributed log splitting + ZKUtil.deleteNode(zkw, znode); + waitForCounter(tot_mgr_get_data_nonode, 0, 1, 30000); + waitForCounter(tot_mgr_log_split_batch_success, 0, 1, to/2); + assertTrue(fs.exists(logFile)); + } finally { + if (thread != null) { + // interrupt the thread in case the test fails in the middle. + // it has no effect if the thread is already terminated. + thread.interrupt(); + } + fs.delete(logDir, true); + } + } + + @Test + public void testWorkerCrash() throws Exception { + conf.setInt("hbase.splitlog.max.resubmit", ZKSplitLog.DEFAULT_MAX_RESUBMIT); + slm = new SplitLogManager(zkw, conf, stopper, master, "dummy-master", null); + slm.finishInitialization(); + TaskBatch batch = new TaskBatch(); + + String tasknode = submitTaskAndWait(batch, "foo/1"); + final ServerName worker1 = new ServerName("worker1,1,1"); + + ZKUtil.setData(zkw, tasknode, TaskState.TASK_OWNED.get(worker1.getServerName())); + if (tot_mgr_heartbeat.get() == 0) waitForCounter(tot_mgr_heartbeat, 0, 1, to / 2); + + // Not yet resubmitted. + Assert.assertEquals(0, tot_mgr_resubmit.get()); + + // This server becomes dead + Mockito.when(sm.isServerOnline(worker1)).thenReturn(false); + + Thread.sleep(1300); // The timeout checker is done every 1000 ms (hardcoded). + + // It has been resubmitted + Assert.assertEquals(1, tot_mgr_resubmit.get()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java new file mode 100644 index 0000000..516c028 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedOpenCloseRegion.java @@ -0,0 +1,452 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + + +import java.io.IOException; +import java.util.Collection; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.executor.EventHandler; +import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.master.handler.TotesHRegionInfo; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.util.Writables; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.mockito.internal.util.reflection.Whitebox; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.junit.Assert.assertFalse; + +/** + * Test open and close of regions using zk. + */ +@Category(MediumTests.class) +public class TestZKBasedOpenCloseRegion { + private static final Log LOG = LogFactory.getLog(TestZKBasedOpenCloseRegion.class); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String TABLENAME = "TestZKBasedOpenCloseRegion"; + private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"), + Bytes.toBytes("b"), Bytes.toBytes("c")}; + private static int countOfRegions; + + @BeforeClass public static void beforeAllTests() throws Exception { + Configuration c = TEST_UTIL.getConfiguration(); + c.setClass(HConstants.REGION_SERVER_IMPL, TestZKBasedOpenCloseRegionRegionServer.class, + HRegionServer.class); + c.setBoolean("dfs.support.append", true); + c.setInt("hbase.regionserver.info.port", 0); + TEST_UTIL.startMiniCluster(2); + TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES); + HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); + countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily()); + waitUntilAllRegionsAssigned(); + addToEachStartKey(countOfRegions); + t.close(); + } + + @AfterClass public static void afterAllTests() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before public void setup() throws IOException { + if (TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size() < 2) { + // Need at least two servers. + LOG.info("Started new server=" + + TEST_UTIL.getHBaseCluster().startRegionServer()); + + } + waitUntilAllRegionsAssigned(); + } + + /** + * Special HRegionServer used in these tests that allows access to + * {@link #addRegionsInTransition(HRegionInfo, String)}. + */ + public static class TestZKBasedOpenCloseRegionRegionServer extends HRegionServer { + public TestZKBasedOpenCloseRegionRegionServer(Configuration conf) + throws IOException, InterruptedException { + super(conf); + } + @Override + public boolean addRegionsInTransition(HRegionInfo region, + String currentAction) throws RegionAlreadyInTransitionException { + return super.addRegionsInTransition(region, currentAction); + } + } + + /** + * Test we reopen a region once closed. + * @throws Exception + */ + @Test (timeout=300000) public void testReOpenRegion() + throws Exception { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + LOG.info("Number of region servers = " + + cluster.getLiveRegionServerThreads().size()); + + int rsIdx = 0; + HRegionServer regionServer = + TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx); + HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions()); + LOG.debug("Asking RS to close region " + hri.getRegionNameAsString()); + + AtomicBoolean closeEventProcessed = new AtomicBoolean(false); + AtomicBoolean reopenEventProcessed = new AtomicBoolean(false); + + EventHandlerListener closeListener = + new ReopenEventListener(hri.getRegionNameAsString(), + closeEventProcessed, EventType.RS_ZK_REGION_CLOSED); + cluster.getMaster().executorService. + registerListener(EventType.RS_ZK_REGION_CLOSED, closeListener); + + EventHandlerListener openListener = + new ReopenEventListener(hri.getRegionNameAsString(), + reopenEventProcessed, EventType.RS_ZK_REGION_OPENED); + cluster.getMaster().executorService. + registerListener(EventType.RS_ZK_REGION_OPENED, openListener); + + LOG.info("Unassign " + hri.getRegionNameAsString()); + cluster.getMaster().assignmentManager.unassign(hri); + + while (!closeEventProcessed.get()) { + Threads.sleep(100); + } + + while (!reopenEventProcessed.get()) { + Threads.sleep(100); + } + + LOG.info("Done with testReOpenRegion"); + } + + private HRegionInfo getNonMetaRegion(final Collection regions) { + HRegionInfo hri = null; + for (HRegionInfo i: regions) { + LOG.info(i.getRegionNameAsString()); + if (!i.isMetaRegion()) { + hri = i; + break; + } + } + return hri; + } + + public static class ReopenEventListener implements EventHandlerListener { + private static final Log LOG = LogFactory.getLog(ReopenEventListener.class); + String regionName; + AtomicBoolean eventProcessed; + EventType eventType; + + public ReopenEventListener(String regionName, + AtomicBoolean eventProcessed, EventType eventType) { + this.regionName = regionName; + this.eventProcessed = eventProcessed; + this.eventType = eventType; + } + + @Override + public void beforeProcess(EventHandler event) { + if(event.getEventType() == eventType) { + LOG.info("Received " + eventType + " and beginning to process it"); + } + } + + @Override + public void afterProcess(EventHandler event) { + LOG.info("afterProcess(" + event + ")"); + if(event.getEventType() == eventType) { + LOG.info("Finished processing " + eventType); + String regionName = ""; + if(eventType == EventType.RS_ZK_REGION_OPENED) { + TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event; + regionName = hriCarrier.getHRegionInfo().getRegionNameAsString(); + } else if(eventType == EventType.RS_ZK_REGION_CLOSED) { + TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event; + regionName = hriCarrier.getHRegionInfo().getRegionNameAsString(); + } + if(this.regionName.equals(regionName)) { + eventProcessed.set(true); + } + synchronized(eventProcessed) { + eventProcessed.notifyAll(); + } + } + } + } + + public static class CloseRegionEventListener implements EventHandlerListener { + private static final Log LOG = LogFactory.getLog(CloseRegionEventListener.class); + String regionToClose; + AtomicBoolean closeEventProcessed; + + public CloseRegionEventListener(String regionToClose, + AtomicBoolean closeEventProcessed) { + this.regionToClose = regionToClose; + this.closeEventProcessed = closeEventProcessed; + } + + @Override + public void afterProcess(EventHandler event) { + LOG.info("afterProcess(" + event + ")"); + if(event.getEventType() == EventType.RS_ZK_REGION_CLOSED) { + LOG.info("Finished processing CLOSE REGION"); + TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event; + if (regionToClose.equals(hriCarrier.getHRegionInfo().getRegionNameAsString())) { + LOG.info("Setting closeEventProcessed flag"); + closeEventProcessed.set(true); + } else { + LOG.info("Region to close didn't match"); + } + } + } + + @Override + public void beforeProcess(EventHandler event) { + if(event.getEventType() == EventType.M_RS_CLOSE_REGION) { + LOG.info("Received CLOSE RPC and beginning to process it"); + } + } + } + + /** + * This test shows how a region won't be able to be assigned to a RS + * if it's already "processing" it. + * @throws Exception + */ + @Test + public void testRSAlreadyProcessingRegion() throws Exception { + LOG.info("starting testRSAlreadyProcessingRegion"); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + + HRegionServer hr0 = + cluster.getLiveRegionServerThreads().get(0).getRegionServer(); + HRegionServer hr1 = + cluster.getLiveRegionServerThreads().get(1).getRegionServer(); + HRegionInfo hri = getNonMetaRegion(hr0.getOnlineRegions()); + + // Fake that hr1 is processing the region. At top of this test we made a + // regionserver that gave access addRegionsInTransition. Need to cast as + // TestZKBasedOpenCloseRegionRegionServer. + ((TestZKBasedOpenCloseRegionRegionServer) hr1).addRegionsInTransition(hri, "OPEN"); + + AtomicBoolean reopenEventProcessed = new AtomicBoolean(false); + EventHandlerListener openListener = + new ReopenEventListener(hri.getRegionNameAsString(), + reopenEventProcessed, EventType.RS_ZK_REGION_OPENED); + cluster.getMaster().executorService. + registerListener(EventType.RS_ZK_REGION_OPENED, openListener); + + // now ask the master to move the region to hr1, will fail + TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(), + Bytes.toBytes(hr1.getServerName().toString())); + + // make sure the region came back + assertEquals(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()), null); + + // remove the block and reset the boolean + hr1.removeFromRegionsInTransition(hri); + reopenEventProcessed.set(false); + + // now try moving a region when there is no region in transition. + hri = getNonMetaRegion(hr1.getOnlineRegions()); + + openListener = + new ReopenEventListener(hri.getRegionNameAsString(), + reopenEventProcessed, EventType.RS_ZK_REGION_OPENED); + + cluster.getMaster().executorService. + registerListener(EventType.RS_ZK_REGION_OPENED, openListener); + + TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(), + Bytes.toBytes(hr0.getServerName().toString())); + + while (!reopenEventProcessed.get()) { + Threads.sleep(100); + } + + // make sure the region has moved from the original RS + assertTrue(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()) == null); + + } + + @Test (timeout=300000) public void testCloseRegion() + throws Exception { + LOG.info("Running testCloseRegion"); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + LOG.info("Number of region servers = " + cluster.getLiveRegionServerThreads().size()); + + int rsIdx = 0; + HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx); + HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions()); + LOG.debug("Asking RS to close region " + hri.getRegionNameAsString()); + + AtomicBoolean closeEventProcessed = new AtomicBoolean(false); + EventHandlerListener listener = + new CloseRegionEventListener(hri.getRegionNameAsString(), + closeEventProcessed); + cluster.getMaster().executorService.registerListener(EventType.RS_ZK_REGION_CLOSED, listener); + + cluster.getMaster().assignmentManager.unassign(hri); + + while (!closeEventProcessed.get()) { + Threads.sleep(100); + } + LOG.info("Done with testCloseRegion"); + } + + /** + * If region open fails with IOException in openRegion() while doing tableDescriptors.get() + * the region should not add into regionsInTransitionInRS map + * @throws Exception + */ + @Test + public void testRegionOpenFailsDueToIOException() throws Exception { + HRegionInfo REGIONINFO = new HRegionInfo(Bytes.toBytes("t"), + HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); + HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0); + TableDescriptors htd = Mockito.mock(TableDescriptors.class); + Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors"); + Whitebox.setInternalState(regionServer, "tableDescriptors", htd); + Mockito.doThrow(new IOException()).when(htd).get((byte[]) Mockito.any()); + try { + regionServer.openRegion(REGIONINFO); + fail("It should throw IOException "); + } catch (IOException e) { + } + Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState); + assertFalse("Region should not be in RIT", + regionServer.containsKeyInRegionsInTransition(REGIONINFO)); + } + + private static void waitUntilAllRegionsAssigned() + throws IOException { + HTable meta = new HTable(TEST_UTIL.getConfiguration(), + HConstants.META_TABLE_NAME); + while (true) { + int rows = 0; + Scan scan = new Scan(); + scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); + ResultScanner s = meta.getScanner(scan); + for (Result r = null; (r = s.next()) != null;) { + byte [] b = + r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); + if (b == null || b.length <= 0) { + break; + } + rows++; + } + s.close(); + // If I get to here and all rows have a Server, then all have been assigned. + if (rows >= countOfRegions) { + break; + } + LOG.info("Found=" + rows); + Threads.sleep(1000); + } + meta.close(); + } + + /* + * Add to each of the regions in .META. a value. Key is the startrow of the + * region (except its 'aaa' for first region). Actual value is the row name. + * @param expected + * @return + * @throws IOException + */ + private static int addToEachStartKey(final int expected) throws IOException { + HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); + HTable meta = new HTable(TEST_UTIL.getConfiguration(), + HConstants.META_TABLE_NAME); + int rows = 0; + Scan scan = new Scan(); + scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + ResultScanner s = meta.getScanner(scan); + for (Result r = null; (r = s.next()) != null;) { + byte [] b = + r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + if (b == null || b.length <= 0) { + break; + } + HRegionInfo hri = Writables.getHRegionInfo(b); + // If start key, add 'aaa'. + byte [] row = getStartKey(hri); + Put p = new Put(row); + p.setWriteToWAL(false); + p.add(getTestFamily(), getTestQualifier(), row); + t.put(p); + rows++; + } + s.close(); + Assert.assertEquals(expected, rows); + t.close(); + meta.close(); + return rows; + } + + private static byte [] getStartKey(final HRegionInfo hri) { + return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())? + Bytes.toBytes("aaa"): hri.getStartKey(); + } + + private static byte [] getTestFamily() { + return FAMILIES[0]; + } + + private static byte [] getTestQualifier() { + return getTestFamily(); + } + + public static void main(String args[]) throws Exception { + TestZKBasedOpenCloseRegion.beforeAllTests(); + + TestZKBasedOpenCloseRegion test = new TestZKBasedOpenCloseRegion(); + test.setup(); + test.testCloseRegion(); + + TestZKBasedOpenCloseRegion.afterAllTests(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java new file mode 100644 index 0000000..3ce4de3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java @@ -0,0 +1,331 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.cleaner; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.After; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +@Category(SmallTests.class) +public class TestCleanerChore { + + private static final Log LOG = LogFactory.getLog(TestCleanerChore.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @After + public void cleanup() throws Exception { + // delete and recreate the test directory, ensuring a clean test dir between tests + UTIL.cleanupTestDir(); + } + + @Test + public void testSavesFilesOnRequest() throws Exception { + Stoppable stop = new StoppableImplementation(); + Configuration conf = UTIL.getConfiguration(); + Path testDir = UTIL.getDataTestDir(); + FileSystem fs = UTIL.getTestFileSystem(); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, NeverDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + + // create the directory layout in the directory to clean + Path parent = new Path(testDir, "parent"); + Path file = new Path(parent, "someFile"); + fs.mkdirs(parent); + // touch a new file + fs.create(file).close(); + assertTrue("Test file didn't get created.", fs.exists(file)); + + // run the chore + chore.chore(); + + // verify all the files got deleted + assertTrue("File didn't get deleted", fs.exists(file)); + assertTrue("Empty directory didn't get deleted", fs.exists(parent)); + } + + @Test + public void testDeletesEmptyDirectories() throws Exception { + Stoppable stop = new StoppableImplementation(); + Configuration conf = UTIL.getConfiguration(); + Path testDir = UTIL.getDataTestDir(); + FileSystem fs = UTIL.getTestFileSystem(); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, AlwaysDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + + // create the directory layout in the directory to clean + Path parent = new Path(testDir, "parent"); + Path child = new Path(parent, "child"); + Path emptyChild = new Path(parent, "emptyChild"); + Path file = new Path(child, "someFile"); + fs.mkdirs(child); + fs.mkdirs(emptyChild); + // touch a new file + fs.create(file).close(); + // also create a file in the top level directory + Path topFile = new Path(testDir, "topFile"); + fs.create(topFile).close(); + assertTrue("Test file didn't get created.", fs.exists(file)); + assertTrue("Test file didn't get created.", fs.exists(topFile)); + + // run the chore + chore.chore(); + + // verify all the files got deleted + assertFalse("File didn't get deleted", fs.exists(topFile)); + assertFalse("File didn't get deleted", fs.exists(file)); + assertFalse("Empty directory didn't get deleted", fs.exists(child)); + assertFalse("Empty directory didn't get deleted", fs.exists(parent)); + } + + /** + * Test to make sure that we don't attempt to ask the delegate whether or not we should preserve a + * directory. + * @throws Exception on failure + */ + @Test + public void testDoesNotCheckDirectories() throws Exception { + Stoppable stop = new StoppableImplementation(); + Configuration conf = UTIL.getConfiguration(); + Path testDir = UTIL.getDataTestDir(); + FileSystem fs = UTIL.getTestFileSystem(); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, AlwaysDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + // spy on the delegate to ensure that we don't check for directories + AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0); + AlwaysDelete spy = Mockito.spy(delegate); + chore.cleanersChain.set(0, spy); + + // create the directory layout in the directory to clean + Path parent = new Path(testDir, "parent"); + Path file = new Path(parent, "someFile"); + fs.mkdirs(parent); + // touch a new file + fs.create(file).close(); + assertTrue("Test file didn't get created.", fs.exists(file)); + + chore.chore(); + // make sure we never checked the directory + Mockito.verify(spy, Mockito.never()).isFileDeletable(parent); + Mockito.reset(spy); + } + + @Test + public void testStoppedCleanerDoesNotDeleteFiles() throws Exception { + Stoppable stop = new StoppableImplementation(); + Configuration conf = UTIL.getConfiguration(); + Path testDir = UTIL.getDataTestDir(); + FileSystem fs = UTIL.getTestFileSystem(); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, AlwaysDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + + // also create a file in the top level directory + Path topFile = new Path(testDir, "topFile"); + fs.create(topFile).close(); + assertTrue("Test file didn't get created.", fs.exists(topFile)); + + // stop the chore + stop.stop("testing stop"); + + // run the chore + chore.chore(); + + // test that the file still exists + assertTrue("File got deleted while chore was stopped", fs.exists(topFile)); + } + + /** + * While cleaning a directory, all the files in the directory may be deleted, but there may be + * another file added, in which case the directory shouldn't be deleted. + * @throws IOException on failure + */ + @Test + public void testCleanerDoesNotDeleteDirectoryWithLateAddedFiles() throws IOException { + Stoppable stop = new StoppableImplementation(); + Configuration conf = UTIL.getConfiguration(); + final Path testDir = UTIL.getDataTestDir(); + final FileSystem fs = UTIL.getTestFileSystem(); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, AlwaysDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + // spy on the delegate to ensure that we don't check for directories + AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0); + AlwaysDelete spy = Mockito.spy(delegate); + chore.cleanersChain.set(0, spy); + + // create the directory layout in the directory to clean + final Path parent = new Path(testDir, "parent"); + Path file = new Path(parent, "someFile"); + fs.mkdirs(parent); + // touch a new file + fs.create(file).close(); + assertTrue("Test file didn't get created.", fs.exists(file)); + final Path addedFile = new Path(parent, "addedFile"); + + // when we attempt to delete the original file, add another file in the same directory + Mockito.doAnswer(new Answer() { + @Override + public Boolean answer(InvocationOnMock invocation) throws Throwable { + fs.create(addedFile).close(); + FSUtils.logFileSystemState(fs, testDir, LOG); + return (Boolean) invocation.callRealMethod(); + } + }).when(spy).isFileDeletable(Mockito.any(Path.class)); + + // run the chore + chore.chore(); + + // make sure all the directories + added file exist, but the original file is deleted + assertTrue("Added file unexpectedly deleted", fs.exists(addedFile)); + assertTrue("Parent directory deleted unexpectedly", fs.exists(parent)); + assertFalse("Original file unexpectedly retained", fs.exists(file)); + Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(Path.class)); + Mockito.reset(spy); + } + + /** + * The cleaner runs in a loop, where it first checks to see all the files under a directory can be + * deleted. If they all can, then we try to delete the directory. However, a file may be added + * that directory to after the original check. This ensures that we don't accidentally delete that + * directory on and don't get spurious IOExceptions. + *

    + * This was from HBASE-7465. + * @throws Exception on failure + */ + @Test + public void testNoExceptionFromDirectoryWithRacyChildren() throws Exception { + Stoppable stop = new StoppableImplementation(); + // need to use a localutil to not break the rest of the test that runs on the local FS, which + // gets hosed when we start to use a minicluster. + HBaseTestingUtility localUtil = new HBaseTestingUtility(); + Configuration conf = localUtil.getConfiguration(); + final Path testDir = UTIL.getDataTestDir(); + final FileSystem fs = UTIL.getTestFileSystem(); + LOG.debug("Writing test data to: " + testDir); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, AlwaysDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + // spy on the delegate to ensure that we don't check for directories + AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0); + AlwaysDelete spy = Mockito.spy(delegate); + chore.cleanersChain.set(0, spy); + + // create the directory layout in the directory to clean + final Path parent = new Path(testDir, "parent"); + Path file = new Path(parent, "someFile"); + fs.mkdirs(parent); + // touch a new file + fs.create(file).close(); + assertTrue("Test file didn't get created.", fs.exists(file)); + final Path racyFile = new Path(parent, "addedFile"); + + // when we attempt to delete the original file, add another file in the same directory + Mockito.doAnswer(new Answer() { + @Override + public Boolean answer(InvocationOnMock invocation) throws Throwable { + fs.create(racyFile).close(); + FSUtils.logFileSystemState(fs, testDir, LOG); + return (Boolean) invocation.callRealMethod(); + } + }).when(spy).isFileDeletable(Mockito.any(Path.class)); + + // attempt to delete the directory, which + if (chore.checkAndDeleteDirectory(parent)) { + throw new Exception( + "Reported success deleting directory, should have failed when adding file mid-iteration"); + } + + // make sure all the directories + added file exist, but the original file is deleted + assertTrue("Added file unexpectedly deleted", fs.exists(racyFile)); + assertTrue("Parent directory deleted unexpectedly", fs.exists(parent)); + assertFalse("Original file unexpectedly retained", fs.exists(file)); + Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(Path.class)); + } + + private static class AllValidPaths extends CleanerChore { + + public AllValidPaths(String name, Stoppable s, Configuration conf, FileSystem fs, + Path oldFileDir, String confkey) { + super(name, Integer.MAX_VALUE, s, conf, fs, oldFileDir, confkey); + } + + // all paths are valid + @Override + protected boolean validate(Path file) { + return true; + } + }; + + public static class AlwaysDelete extends BaseHFileCleanerDelegate { + @Override + public boolean isFileDeletable(Path file) { + return true; + } + } + + public static class NeverDelete extends BaseHFileCleanerDelegate { + @Override + public boolean isFileDeletable(Path file) { + return false; + } + } + + /** + * Simple helper class that just keeps track of whether or not its stopped. + */ + private static class StoppableImplementation implements Stoppable { + private volatile boolean stop; + + @Override + public void stop(String why) { + this.stop = true; + } + + @Override + public boolean isStopped() { + return this.stop; + } + + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java new file mode 100644 index 0000000..2392099 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java @@ -0,0 +1,241 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.cleaner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestHFileCleaner { + private static final Log LOG = LogFactory.getLog(TestHFileCleaner.class); + + private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupCluster() throws Exception { + // have to use a minidfs cluster because the localfs doesn't modify file times correctly + UTIL.startMiniDFSCluster(1); + } + + @AfterClass + public static void shutdownCluster() throws Exception { + UTIL.shutdownMiniDFSCluster(); + } + + @Test + public void testTTLCleaner() throws IOException, InterruptedException { + FileSystem fs = UTIL.getDFSCluster().getFileSystem(); + Path root = UTIL.getDataTestDir(); + Path file = new Path(root, "file"); + fs.createNewFile(file); + long createTime = System.currentTimeMillis(); + assertTrue("Test file not created!", fs.exists(file)); + TimeToLiveHFileCleaner cleaner = new TimeToLiveHFileCleaner(); + // update the time info for the file, so the cleaner removes it + fs.setTimes(file, createTime - 100, -1); + Configuration conf = UTIL.getConfiguration(); + conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 100); + cleaner.setConf(conf); + assertTrue("File not set deletable - check mod time:" + getFileStats(file, fs) + + " with create time:" + createTime, cleaner.isFileDeletable(file)); + } + + /** + * @param file to check + * @return loggable information about the file + */ + private String getFileStats(Path file, FileSystem fs) throws IOException { + FileStatus status = fs.getFileStatus(file); + return "File" + file + ", mtime:" + status.getModificationTime() + ", atime:" + + status.getAccessTime(); + } + + @Test(timeout = 60 *1000) + public void testHFileCleaning() throws Exception { + final EnvironmentEdge originalEdge = EnvironmentEdgeManager.getDelegate(); + String prefix = "someHFileThatWouldBeAUUID"; + Configuration conf = UTIL.getConfiguration(); + // set TTL + long ttl = 2000; + conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, + "org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner"); + conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl); + Server server = new DummyServer(); + Path archivedHfileDir = new Path(UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY); + FileSystem fs = FileSystem.get(conf); + HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir); + + // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files + final long createTime = System.currentTimeMillis(); + fs.delete(archivedHfileDir, true); + fs.mkdirs(archivedHfileDir); + // Case 1: 1 invalid file, which should be deleted directly + fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd")); + // Case 2: 1 "recent" file, not even deletable for the first log cleaner + // (TimeToLiveLogCleaner), so we are not going down the chain + LOG.debug("Now is: " + createTime); + for (int i = 1; i < 32; i++) { + // Case 3: old files which would be deletable for the first log cleaner + // (TimeToLiveHFileCleaner), + Path fileName = new Path(archivedHfileDir, (prefix + "." + (createTime + i))); + fs.createNewFile(fileName); + // set the creation time past ttl to ensure that it gets removed + fs.setTimes(fileName, createTime - ttl - 1, -1); + LOG.debug("Creating " + getFileStats(fileName, fs)); + } + + // Case 2: 1 newer file, not even deletable for the first log cleaner + // (TimeToLiveLogCleaner), so we are not going down the chain + Path saved = new Path(archivedHfileDir, "thisFileShouldBeSaved.00000000000"); + fs.createNewFile(saved); + // set creation time in the future, so definitely within TTL + fs.setTimes(saved, createTime + (ttl * 2), -1); + LOG.debug("Creating " + getFileStats(saved, fs)); + + assertEquals(33, fs.listStatus(archivedHfileDir).length); + + // set a custom edge manager to handle time checking + EnvironmentEdge setTime = new EnvironmentEdge() { + @Override + public long currentTimeMillis() { + return createTime; + } + }; + EnvironmentEdgeManager.injectEdge(setTime); + + // run the chore + cleaner.chore(); + + for (FileStatus file : fs.listStatus(archivedHfileDir)) { + LOG.debug("Kept hfile: " + file.getPath()); + } + + // ensure we only end up with the saved file + assertEquals("Didn't dev expected number of files in the archive!", 1, + fs.listStatus(archivedHfileDir).length); + + cleaner.interrupt(); + // reset the edge back to the original edge + EnvironmentEdgeManager.injectEdge(originalEdge); + } + + @Test + public void testRemovesEmptyDirectories() throws Exception { + Configuration conf = UTIL.getConfiguration(); + // no cleaner policies = delete all files + conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); + Server server = new DummyServer(); + Path archivedHfileDir = new Path(UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY); + + // setup the cleaner + FileSystem fs = UTIL.getDFSCluster().getFileSystem(); + HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir); + + // make all the directories for archiving files + Path table = new Path(archivedHfileDir, "table"); + Path region = new Path(table, "regionsomthing"); + Path family = new Path(region, "fam"); + Path file = new Path(family, "file12345"); + fs.mkdirs(family); + if (!fs.exists(family)) throw new RuntimeException("Couldn't create test family:" + family); + fs.create(file).close(); + if (!fs.exists(file)) throw new RuntimeException("Test file didn't get created:" + file); + + // run the chore to cleanup the files (and the directories above it) + cleaner.chore(); + + // make sure all the parent directories get removed + assertFalse("family directory not removed for empty directory", fs.exists(family)); + assertFalse("region directory not removed for empty directory", fs.exists(region)); + assertFalse("table directory not removed for empty directory", fs.exists(table)); + assertTrue("archive directory", fs.exists(archivedHfileDir)); + } + + static class DummyServer implements Server { + + @Override + public Configuration getConfiguration() { + return UTIL.getConfiguration(); + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + try { + return new ZooKeeperWatcher(getConfiguration(), "dummy server", this); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public CatalogTracker getCatalogTracker() { + return null; + } + + @Override + public ServerName getServerName() { + return new ServerName("regionserver,60020,000000"); + } + + @Override + public void abort(String why, Throwable e) { + } + + @Override + public boolean isAborted() { + return false; + } + + @Override + public void stop(String why) {} + + @Override + public boolean isStopped() { + return false; + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java new file mode 100644 index 0000000..c55084d --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.cleaner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.backup.HFileArchiver; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test the HFileLink Cleaner. + * HFiles with links cannot be deleted until a link is present. + */ +@Category(SmallTests.class) +public class TestHFileLinkCleaner { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @Test + public void testHFileLinkCleaning() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDataTestDir().toString()); + conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName()); + Path rootDir = FSUtils.getRootDir(conf); + FileSystem fs = FileSystem.get(conf); + + final String tableName = "test-table"; + final String tableLinkName = "test-link"; + final String hfileName = "1234567890"; + final String familyName = "cf"; + + HRegionInfo hri = new HRegionInfo(Bytes.toBytes(tableName)); + HRegionInfo hriLink = new HRegionInfo(Bytes.toBytes(tableLinkName)); + + Path archiveDir = HFileArchiveUtil.getArchivePath(conf); + Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, + tableName, hri.getEncodedName(), familyName); + Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, + tableLinkName, hriLink.getEncodedName(), familyName); + + // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf); + Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName); + fs.mkdirs(familyPath); + Path hfilePath = new Path(familyPath, hfileName); + fs.createNewFile(hfilePath); + + // Create link to hfile + Path familyLinkPath = getFamilyDirPath(rootDir, tableLinkName, + hriLink.getEncodedName(), familyName); + fs.mkdirs(familyLinkPath); + HFileLink.create(conf, fs, familyLinkPath, hri, hfileName); + Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName); + assertTrue(fs.exists(linkBackRefDir)); + FileStatus[] backRefs = fs.listStatus(linkBackRefDir); + assertEquals(1, backRefs.length); + Path linkBackRef = backRefs[0].getPath(); + + // Initialize cleaner + final long ttl = 1000; + conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl); + Server server = new DummyServer(); + HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir); + + // Link backref cannot be removed + cleaner.chore(); + assertTrue(fs.exists(linkBackRef)); + assertTrue(fs.exists(hfilePath)); + + // Link backref can be removed + fs.rename(new Path(rootDir, tableLinkName), new Path(archiveDir, tableLinkName)); + cleaner.chore(); + assertFalse("Link should be deleted", fs.exists(linkBackRef)); + + // HFile can be removed + Thread.sleep(ttl * 2); + cleaner.chore(); + assertFalse("HFile should be deleted", fs.exists(hfilePath)); + + // Remove everything + for (int i = 0; i < 4; ++i) { + Thread.sleep(ttl * 2); + cleaner.chore(); + } + assertFalse("HFile should be deleted", fs.exists(new Path(archiveDir, tableName))); + assertFalse("Link should be deleted", fs.exists(new Path(archiveDir, tableLinkName))); + + cleaner.interrupt(); + } + + private static Path getFamilyDirPath (final Path rootDir, final String table, + final String region, final String family) { + return new Path(new Path(new Path(rootDir, table), region), family); + } + + static class DummyServer implements Server { + + @Override + public Configuration getConfiguration() { + return TEST_UTIL.getConfiguration(); + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + try { + return new ZooKeeperWatcher(getConfiguration(), "dummy server", this); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public CatalogTracker getCatalogTracker() { + return null; + } + + @Override + public ServerName getServerName() { + return new ServerName("regionserver,60020,000000"); + } + + @Override + public void abort(String why, Throwable e) {} + + @Override + public boolean isAborted() { + return false; + } + + @Override + public void stop(String why) {} + + @Override + public boolean isStopped() { + return false; + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java new file mode 100644 index 0000000..e38309c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java @@ -0,0 +1,182 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.cleaner; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.net.URLEncoder; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.master.cleaner.LogCleaner; +import org.apache.hadoop.hbase.replication.ReplicationZookeeper; +import org.apache.hadoop.hbase.replication.regionserver.Replication; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestLogsCleaner { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + @Test + public void testLogCleaning() throws Exception{ + Configuration conf = TEST_UTIL.getConfiguration(); + // set TTL + long ttl = 2000; + conf.setLong("hbase.master.logcleaner.ttl", ttl); + conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true); + Replication.decorateMasterConfiguration(conf); + Server server = new DummyServer(); + ReplicationZookeeper zkHelper = + new ReplicationZookeeper(server, new AtomicBoolean(true)); + + Path oldLogDir = new Path(TEST_UTIL.getDataTestDir(), + HConstants.HREGION_OLDLOGDIR_NAME); + String fakeMachineName = + URLEncoder.encode(server.getServerName().toString(), "UTF8"); + + FileSystem fs = FileSystem.get(conf); + LogCleaner cleaner = new LogCleaner(1000, server, conf, fs, oldLogDir); + + // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files + long now = System.currentTimeMillis(); + fs.delete(oldLogDir, true); + fs.mkdirs(oldLogDir); + // Case 1: 2 invalid files, which would be deleted directly + fs.createNewFile(new Path(oldLogDir, "a")); + fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + "a")); + // Case 2: 1 "recent" file, not even deletable for the first log cleaner + // (TimeToLiveLogCleaner), so we are not going down the chain + System.out.println("Now is: " + now); + for (int i = 1; i < 31; i++) { + // Case 3: old files which would be deletable for the first log cleaner + // (TimeToLiveLogCleaner), and also for the second (ReplicationLogCleaner) + Path fileName = new Path(oldLogDir, fakeMachineName + "." + (now - i) ); + fs.createNewFile(fileName); + // Case 4: put 3 old log files in ZK indicating that they are scheduled + // for replication so these files would pass the first log cleaner + // (TimeToLiveLogCleaner) but would be rejected by the second + // (ReplicationLogCleaner) + if (i % (30/3) == 1) { + zkHelper.addLogToList(fileName.getName(), fakeMachineName); + System.out.println("Replication log file: " + fileName); + } + } + + // sleep for sometime to get newer modifcation time + Thread.sleep(ttl); + fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + now)); + + // Case 2: 1 newer file, not even deletable for the first log cleaner + // (TimeToLiveLogCleaner), so we are not going down the chain + fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + (now + 10000) )); + + for (FileStatus stat : fs.listStatus(oldLogDir)) { + System.out.println(stat.getPath().toString()); + } + + assertEquals(34, fs.listStatus(oldLogDir).length); + + cleaner.chore(); + + // We end up with the current log file, a newer one and the 3 old log + // files which are scheduled for replication + assertEquals(5, fs.listStatus(oldLogDir).length); + + for (FileStatus file : fs.listStatus(oldLogDir)) { + System.out.println("Kept log files: " + file.getPath().getName()); + } + } + + static class DummyServer implements Server { + + @Override + public Configuration getConfiguration() { + return TEST_UTIL.getConfiguration(); + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + try { + return new ZooKeeperWatcher(getConfiguration(), "dummy server", this); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public CatalogTracker getCatalogTracker() { + return null; + } + + @Override + public ServerName getServerName() { + return new ServerName("regionserver,60020,000000"); + } + + @Override + public void abort(String why, Throwable e) {} + + @Override + public boolean isAborted() { + return false; + } + + @Override + public void stop(String why) {} + + @Override + public boolean isStopped() { + return false; + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java new file mode 100644 index 0000000..92a8bb8 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java @@ -0,0 +1,386 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.cleaner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.snapshot.DisabledTableSnapshotHandler; +import org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.snapshot.HSnapshotDescription; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; +import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import com.google.common.collect.Lists; + +/** + * Test the master-related aspects of a snapshot + */ +@Category(MediumTests.class) +public class TestSnapshotFromMaster { + + private static final Log LOG = LogFactory.getLog(TestSnapshotFromMaster.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final int NUM_RS = 2; + private static Path rootDir; + private static Path snapshots; + private static FileSystem fs; + private static HMaster master; + + // for hfile archiving test. + private static Path archiveDir; + private static final String STRING_TABLE_NAME = "test"; + private static final byte[] TEST_FAM = Bytes.toBytes("fam"); + private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME); + // refresh the cache every 1/2 second + private static final long cacheRefreshPeriod = 500; + + /** + * Setup the config for the cluster + */ + @BeforeClass + public static void setupCluster() throws Exception { + setupConf(UTIL.getConfiguration()); + UTIL.startMiniCluster(NUM_RS); + fs = UTIL.getDFSCluster().getFileSystem(); + master = UTIL.getMiniHBaseCluster().getMaster(); + rootDir = master.getMasterFileSystem().getRootDir(); + snapshots = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); + archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); + } + + private static void setupConf(Configuration conf) { + // disable the ui + conf.setInt("hbase.regionsever.info.port", -1); + // change the flush size to a small amount, regulating number of store files + conf.setInt("hbase.hregion.memstore.flush.size", 25000); + // so make sure we get a compaction when doing a load, but keep around some + // files in the store + conf.setInt("hbase.hstore.compaction.min", 3); + conf.setInt("hbase.hstore.compactionThreshold", 5); + // block writes if we get to 12 store files + conf.setInt("hbase.hstore.blockingStoreFiles", 12); + // drop the number of attempts for the hbase admin + conf.setInt("hbase.client.retries.number", 1); + // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner) + conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); + conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, ""); + // Enable snapshot + conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod); + + // prevent aggressive region split + conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, + ConstantSizeRegionSplitPolicy.class.getName()); + } + + @Before + public void setup() throws Exception { + UTIL.createTable(TABLE_NAME, TEST_FAM); + master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(STRING_TABLE_NAME, null); + } + + @After + public void tearDown() throws Exception { + UTIL.deleteTable(TABLE_NAME); + + // delete the archive directory, if its exists + if (fs.exists(archiveDir)) { + if (!fs.delete(archiveDir, true)) { + throw new IOException("Couldn't delete archive directory (" + archiveDir + + " for an unknown reason"); + } + } + + // delete the snapshot directory, if its exists + if (fs.exists(snapshots)) { + if (!fs.delete(snapshots, true)) { + throw new IOException("Couldn't delete snapshots directory (" + snapshots + + " for an unknown reason"); + } + } + } + + @AfterClass + public static void cleanupTest() throws Exception { + try { + UTIL.shutdownMiniCluster(); + } catch (Exception e) { + // NOOP; + } + } + + /** + * Test that the contract from the master for checking on a snapshot are valid. + *

    + *

      + *
    1. If a snapshot fails with an error, we expect to get the source error.
      2. + *
    2. If there is no snapshot name supplied, we should get an error.
      3. + *
    3. If asking about a snapshot has hasn't occurred, you should get an error.
      4. + *
    + */ + @Test(timeout = 60000) + public void testIsDoneContract() throws Exception { + + String snapshotName = "asyncExpectedFailureTest"; + + // check that we get an exception when looking up snapshot where one hasn't happened + SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(), + UnknownSnapshotException.class); + + // and that we get the same issue, even if we specify a name + SnapshotDescription desc = SnapshotDescription.newBuilder() + .setName(snapshotName).setTable(STRING_TABLE_NAME).build(); + SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(desc), + UnknownSnapshotException.class); + + // set a mock handler to simulate a snapshot + DisabledTableSnapshotHandler mockHandler = Mockito.mock(DisabledTableSnapshotHandler.class); + Mockito.when(mockHandler.getException()).thenReturn(null); + Mockito.when(mockHandler.getSnapshot()).thenReturn(desc); + Mockito.when(mockHandler.isFinished()).thenReturn(new Boolean(true)); + Mockito.when(mockHandler.getCompletionTimestamp()) + .thenReturn(EnvironmentEdgeManager.currentTimeMillis()); + + master.getSnapshotManagerForTesting() + .setSnapshotHandlerForTesting(STRING_TABLE_NAME, mockHandler); + + // if we do a lookup without a snapshot name, we should fail - you should always know your name + SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(), + UnknownSnapshotException.class); + + // then do the lookup for the snapshot that it is done + boolean isDone = master.isSnapshotDone(new HSnapshotDescription(desc)); + assertTrue("Snapshot didn't complete when it should have.", isDone); + + // now try the case where we are looking for a snapshot we didn't take + desc = SnapshotDescription.newBuilder().setName("Not A Snapshot").build(); + SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(desc), + UnknownSnapshotException.class); + + // then create a snapshot to the fs and make sure that we can find it when checking done + snapshotName = "completed"; + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + desc = desc.toBuilder().setName(snapshotName).build(); + SnapshotDescriptionUtils.writeSnapshotInfo(desc, snapshotDir, fs); + + isDone = master.isSnapshotDone(new HSnapshotDescription(desc)); + assertTrue("Completed, on-disk snapshot not found", isDone); + } + + @Test + public void testGetCompletedSnapshots() throws Exception { + // first check when there are no snapshots + List snapshots = master.getCompletedSnapshots(); + assertEquals("Found unexpected number of snapshots", 0, snapshots.size()); + + // write one snapshot to the fs + String snapshotName = "completed"; + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); + SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); + + // check that we get one snapshot + snapshots = master.getCompletedSnapshots(); + assertEquals("Found unexpected number of snapshots", 1, snapshots.size()); + List expected = Lists.newArrayList(new HSnapshotDescription(snapshot)); + assertEquals("Returned snapshots don't match created snapshots", expected, snapshots); + + // write a second snapshot + snapshotName = "completed_two"; + snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); + SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); + expected.add(new HSnapshotDescription(snapshot)); + + // check that we get one snapshot + snapshots = master.getCompletedSnapshots(); + assertEquals("Found unexpected number of snapshots", 2, snapshots.size()); + assertEquals("Returned snapshots don't match created snapshots", expected, snapshots); + } + + @Test + public void testDeleteSnapshot() throws Exception { + + String snapshotName = "completed"; + SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); + + try { + master.deleteSnapshot(new HSnapshotDescription(snapshot)); + fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist"); + } catch (IOException e) { + LOG.debug("Correctly failed delete of non-existant snapshot:" + e.getMessage()); + } + + // write one snapshot to the fs + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); + + // then delete the existing snapshot,which shouldn't cause an exception to be thrown + master.deleteSnapshot(new HSnapshotDescription(snapshot)); + } + + /** + * Test that the snapshot hfile archive cleaner works correctly. HFiles that are in snapshots + * should be retained, while those that are not in a snapshot should be deleted. + * @throws Exception on failure + */ + @Test + public void testSnapshotHFileArchiving() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + // make sure we don't fail on listing snapshots + SnapshotTestingUtils.assertNoSnapshots(admin); + // load the table + UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM); + + // disable the table so we can take a snapshot + admin.disableTable(TABLE_NAME); + + // take a snapshot of the table + String snapshotName = "snapshot"; + byte[] snapshotNameBytes = Bytes.toBytes(snapshotName); + admin.snapshot(snapshotNameBytes, TABLE_NAME); + + Configuration conf = master.getConfiguration(); + LOG.info("After snapshot File-System state"); + FSUtils.logFileSystemState(fs, rootDir, LOG); + + // ensure we only have one snapshot + SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshotNameBytes, TABLE_NAME); + + // renable the table so we can compact the regions + admin.enableTable(TABLE_NAME); + + // compact the files so we get some archived files for the table we just snapshotted + List regions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); + for (HRegion region : regions) { + region.waitForFlushesAndCompactions(); // enable can trigger a compaction, wait for it. + region.compactStores(); + } + LOG.info("After compaction File-System state"); + FSUtils.logFileSystemState(fs, rootDir, LOG); + + // make sure the cleaner has run + LOG.debug("Running hfile cleaners"); + ensureHFileCleanersRun(); + LOG.info("After cleaners File-System state: " + rootDir); + FSUtils.logFileSystemState(fs, rootDir, LOG); + + // get the snapshot files for the table + Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + FileStatus[] snapshotHFiles = SnapshotTestingUtils.listHFiles(fs, snapshotTable); + // check that the files in the archive contain the ones that we need for the snapshot + LOG.debug("Have snapshot hfiles:"); + for (FileStatus file : snapshotHFiles) { + LOG.debug(file.getPath()); + } + // get the archived files for the table + Collection files = getArchivedHFiles(archiveDir, rootDir, fs, STRING_TABLE_NAME); + + // and make sure that there is a proper subset + for (FileStatus file : snapshotHFiles) { + assertTrue("Archived hfiles " + files + " is missing snapshot file:" + file.getPath(), + files.contains(file.getPath().getName())); + } + + // delete the existing snapshot + admin.deleteSnapshot(snapshotNameBytes); + SnapshotTestingUtils.assertNoSnapshots(admin); + + // make sure that we don't keep around the hfiles that aren't in a snapshot + // make sure we wait long enough to refresh the snapshot hfile + List delegates = UTIL.getMiniHBaseCluster().getMaster() + .getHFileCleaner().cleanersChain; + for (BaseHFileCleanerDelegate delegate: delegates) { + if (delegate instanceof SnapshotHFileCleaner) { + ((SnapshotHFileCleaner)delegate).getFileCacheForTesting().triggerCacheRefreshForTesting(); + } + } + // run the cleaner again + LOG.debug("Running hfile cleaners"); + ensureHFileCleanersRun(); + LOG.info("After delete snapshot cleaners run File-System state"); + FSUtils.logFileSystemState(fs, rootDir, LOG); + + files = getArchivedHFiles(archiveDir, rootDir, fs, STRING_TABLE_NAME); + assertEquals("Still have some hfiles in the archive, when their snapshot has been deleted.", 0, + files.size()); + } + + /** + * @return all the HFiles for a given table that have been archived + * @throws IOException on expected failure + */ + private final Collection getArchivedHFiles(Path archiveDir, Path rootDir, + FileSystem fs, String tableName) throws IOException { + Path tableArchive = new Path(archiveDir, tableName); + FileStatus[] archivedHFiles = SnapshotTestingUtils.listHFiles(fs, tableArchive); + List files = new ArrayList(archivedHFiles.length); + LOG.debug("Have archived hfiles: " + tableArchive); + for (FileStatus file : archivedHFiles) { + LOG.debug(file.getPath()); + files.add(file.getPath().getName()); + } + // sort the archived files + + Collections.sort(files); + return files; + } + + /** + * Make sure the {@link HFileCleaner HFileCleaners} run at least once + */ + private static void ensureHFileCleanersRun() { + UTIL.getHBaseCluster().getMaster().getHFileCleaner().chore(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java b/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java new file mode 100644 index 0000000..1f85eb2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java @@ -0,0 +1,147 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.handler; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.master.TestMaster; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKTable; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestCreateTableHandler { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final Log LOG = LogFactory.getLog(TestCreateTableHandler.class); + private static final byte[] TABLENAME = Bytes.toBytes("TestCreateTableHandler"); + private static final byte[] FAMILYNAME = Bytes.toBytes("fam"); + private static boolean throwException = false; + + @Before + public void setUp() throws Exception { + TEST_UTIL.startMiniCluster(1); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + throwException = true; + } + + @Test + public void testCreateTableHandlerIfCalledTwoTimesAndFirstOneIsUnderProgress() throws Exception { + final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + final HMaster m = cluster.getMaster(); + final HTableDescriptor desc = new HTableDescriptor(TABLENAME); + desc.addFamily(new HColumnDescriptor(FAMILYNAME)); + final HRegionInfo[] hRegionInfos = new HRegionInfo[] { new HRegionInfo(desc.getName(), null, + null) }; + CustomCreateTableHandler handler = new CustomCreateTableHandler(m, m.getMasterFileSystem(), + m.getServerManager(), desc, cluster.getConfiguration(), hRegionInfos, + m.getCatalogTracker(), m.getAssignmentManager()); + throwException = true; + handler.process(); + throwException = false; + CustomCreateTableHandler handler1 = new CustomCreateTableHandler(m, m.getMasterFileSystem(), + m.getServerManager(), desc, cluster.getConfiguration(), hRegionInfos, + m.getCatalogTracker(), m.getAssignmentManager()); + handler1.process(); + for (int i = 0; i < 100; i++) { + if (!TEST_UTIL.getHBaseAdmin().isTableAvailable(TABLENAME)) { + Thread.sleep(200); + } + } + assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(TABLENAME)); + + } + + @Test (timeout=60000) + public void testMasterRestartAfterEnablingNodeIsCreated() throws Exception { + byte[] tableName = Bytes.toBytes("testMasterRestartAfterEnablingNodeIsCreated"); + final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + final HMaster m = cluster.getMaster(); + final HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILYNAME)); + final HRegionInfo[] hRegionInfos = new HRegionInfo[] { new HRegionInfo(desc.getName(), null, + null) }; + CustomCreateTableHandler handler = new CustomCreateTableHandler(m, m.getMasterFileSystem(), + m.getServerManager(), desc, cluster.getConfiguration(), hRegionInfos, + m.getCatalogTracker(), m.getAssignmentManager()); + throwException = true; + handler.process(); + abortAndStartNewMaster(cluster); + assertTrue(cluster.getLiveMasterThreads().size() == 1); + + } + + private void abortAndStartNewMaster(final MiniHBaseCluster cluster) throws IOException { + cluster.abortMaster(0); + cluster.waitOnMaster(0); + LOG.info("Starting new master"); + cluster.startMaster(); + LOG.info("Waiting for master to become active."); + cluster.waitForActiveAndReadyMaster(); + } + + private static class CustomCreateTableHandler extends CreateTableHandler { + public CustomCreateTableHandler(Server server, MasterFileSystem fileSystemManager, + ServerManager sm, HTableDescriptor hTableDescriptor, Configuration conf, + HRegionInfo[] newRegions, CatalogTracker ct, AssignmentManager am) + throws NotAllMetaRegionsOnlineException, TableExistsException, IOException { + super(server, fileSystemManager, sm, hTableDescriptor, conf, newRegions, ct, am); + } + + @Override + protected List handleCreateHdfsRegions(Path tableRootDir, String tableName) + throws IOException { + if (throwException) { + throw new IOException("Test throws exceptions."); + } + return super.handleCreateHdfsRegions(tableRootDir, tableName); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDeleteFamilyHandler.java b/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDeleteFamilyHandler.java new file mode 100644 index 0000000..ad56bd4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDeleteFamilyHandler.java @@ -0,0 +1,159 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.handler; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestTableDeleteFamilyHandler { + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String TABLENAME = "column_family_handlers"; + private static final byte[][] FAMILIES = new byte[][] { Bytes.toBytes("cf1"), + Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; + + /** + * Start up a mini cluster and put a small table of empty regions into it. + * + * @throws Exception + */ + @BeforeClass + public static void beforeAllTests() throws Exception { + + TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); + TEST_UTIL.startMiniCluster(2); + + // Create a table of three families. This will assign a region. + TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES); + HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); + + // Create multiple regions in all the three column families + TEST_UTIL.createMultiRegions(t, FAMILIES[0]); + + // Load the table with data for all families + TEST_UTIL.loadTable(t, FAMILIES); + + TEST_UTIL.flush(); + + t.close(); + } + + @AfterClass + public static void afterAllTests() throws Exception { + TEST_UTIL.deleteTable(Bytes.toBytes(TABLENAME)); + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setup() throws IOException, InterruptedException { + TEST_UTIL.ensureSomeRegionServersAvailable(2); + } + + @Test + public void deleteColumnFamilyWithMultipleRegions() throws Exception { + + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + HTableDescriptor beforehtd = admin.getTableDescriptor(Bytes + .toBytes(TABLENAME)); + + FileSystem fs = TEST_UTIL.getDFSCluster().getFileSystem(); + + // 1 - Check if table exists in descriptor + assertTrue(admin.isTableAvailable(TABLENAME)); + + // 2 - Check if all three families exist in descriptor + assertEquals(3, beforehtd.getColumnFamilies().length); + HColumnDescriptor[] families = beforehtd.getColumnFamilies(); + for (int i = 0; i < families.length; i++) { + + assertTrue(families[i].getNameAsString().equals("cf" + (i + 1))); + } + + // 3 - Check if table exists in FS + Path tableDir = new Path(TEST_UTIL.getDefaultRootDirPath().toString() + "/" + + TABLENAME); + assertTrue(fs.exists(tableDir)); + + // 4 - Check if all the 3 column families exist in FS + FileStatus[] fileStatus = fs.listStatus(tableDir); + for (int i = 0; i < fileStatus.length; i++) { + if (fileStatus[i].isDir() == true) { + FileStatus[] cf = fs.listStatus(fileStatus[i].getPath()); + int k = 1; + for (int j = 0; j < cf.length; j++) { + if (cf[j].isDir() == true + && cf[j].getPath().getName().startsWith(".") == false) { + assertTrue(cf[j].getPath().getName().equals("cf" + k)); + k++; + } + } + } + } + + // TEST - Disable and delete the column family + admin.disableTable(TABLENAME); + admin.deleteColumn(TABLENAME, "cf2"); + + // 5 - Check if only 2 column families exist in the descriptor + HTableDescriptor afterhtd = admin.getTableDescriptor(Bytes + .toBytes(TABLENAME)); + assertEquals(2, afterhtd.getColumnFamilies().length); + HColumnDescriptor[] newFamilies = afterhtd.getColumnFamilies(); + assertTrue(newFamilies[0].getNameAsString().equals("cf1")); + assertTrue(newFamilies[1].getNameAsString().equals("cf3")); + + // 6 - Check if the second column family is gone from the FS + fileStatus = fs.listStatus(tableDir); + for (int i = 0; i < fileStatus.length; i++) { + if (fileStatus[i].isDir() == true) { + FileStatus[] cf = fs.listStatus(fileStatus[i].getPath()); + for (int j = 0; j < cf.length; j++) { + if (cf[j].isDir() == true) { + assertFalse(cf[j].getPath().getName().equals("cf2")); + } + } + } + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDescriptorModification.java b/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDescriptorModification.java new file mode 100644 index 0000000..8b6e8b9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDescriptorModification.java @@ -0,0 +1,160 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.handler; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Set; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Verify that the HTableDescriptor is updated after + * addColumn(), deleteColumn() and modifyTable() operations. + */ +@Category(LargeTests.class) +public class TestTableDescriptorModification { + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final byte[] TABLE_NAME = Bytes.toBytes("table"); + private static final byte[] FAMILY_0 = Bytes.toBytes("cf0"); + private static final byte[] FAMILY_1 = Bytes.toBytes("cf1"); + + /** + * Start up a mini cluster and put a small table of empty regions into it. + * + * @throws Exception + */ + @BeforeClass + public static void beforeAllTests() throws Exception { + TEST_UTIL.startMiniCluster(1); + } + + @AfterClass + public static void afterAllTests() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testModifyTable() throws IOException { + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + // Create a table with one family + HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME); + baseHtd.addFamily(new HColumnDescriptor(FAMILY_0)); + admin.createTable(baseHtd); + admin.disableTable(TABLE_NAME); + try { + // Verify the table descriptor + verifyTableDescriptor(TABLE_NAME, FAMILY_0); + + // Modify the table adding another family and verify the descriptor + HTableDescriptor modifiedHtd = new HTableDescriptor(TABLE_NAME); + modifiedHtd.addFamily(new HColumnDescriptor(FAMILY_0)); + modifiedHtd.addFamily(new HColumnDescriptor(FAMILY_1)); + admin.modifyTable(TABLE_NAME, modifiedHtd); + verifyTableDescriptor(TABLE_NAME, FAMILY_0, FAMILY_1); + } finally { + admin.deleteTable(TABLE_NAME); + } + } + + @Test + public void testAddColumn() throws IOException { + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + // Create a table with two families + HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME); + baseHtd.addFamily(new HColumnDescriptor(FAMILY_0)); + admin.createTable(baseHtd); + admin.disableTable(TABLE_NAME); + try { + // Verify the table descriptor + verifyTableDescriptor(TABLE_NAME, FAMILY_0); + + // Modify the table removing one family and verify the descriptor + admin.addColumn(TABLE_NAME, new HColumnDescriptor(FAMILY_1)); + verifyTableDescriptor(TABLE_NAME, FAMILY_0, FAMILY_1); + } finally { + admin.deleteTable(TABLE_NAME); + } + } + + @Test + public void testDeleteColumn() throws IOException { + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + // Create a table with two families + HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME); + baseHtd.addFamily(new HColumnDescriptor(FAMILY_0)); + baseHtd.addFamily(new HColumnDescriptor(FAMILY_1)); + admin.createTable(baseHtd); + admin.disableTable(TABLE_NAME); + try { + // Verify the table descriptor + verifyTableDescriptor(TABLE_NAME, FAMILY_0, FAMILY_1); + + // Modify the table removing one family and verify the descriptor + admin.deleteColumn(TABLE_NAME, FAMILY_1); + verifyTableDescriptor(TABLE_NAME, FAMILY_0); + } finally { + admin.deleteTable(TABLE_NAME); + } + } + + private void verifyTableDescriptor(final byte[] tableName, final byte[]... families) + throws IOException { + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + + // Verify descriptor from master + HTableDescriptor htd = admin.getTableDescriptor(tableName); + verifyTableDescriptor(htd, tableName, families); + + // Verify descriptor from HDFS + MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); + Path tableDir = HTableDescriptor.getTableDir(mfs.getRootDir(), tableName); + htd = FSTableDescriptors.getTableDescriptor(mfs.getFileSystem(), tableDir); + verifyTableDescriptor(htd, tableName, families); + } + + private void verifyTableDescriptor(final HTableDescriptor htd, + final byte[] tableName, final byte[]... families) { + Set htdFamilies = htd.getFamiliesKeys(); + assertTrue(Bytes.equals(tableName, htd.getName())); + assertEquals(families.length, htdFamilies.size()); + for (byte[] familyName: families) { + assertTrue("Expected family " + Bytes.toString(familyName), htdFamilies.contains(familyName)); + } + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java new file mode 100644 index 0000000..a6e627c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java @@ -0,0 +1,230 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.snapshot; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; +import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test that we correctly reload the cache, filter directories, etc. + */ +@Category(MediumTests.class) +public class TestSnapshotFileCache { + + private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static FileSystem fs; + private static Path rootDir; + + @BeforeClass + public static void startCluster() throws Exception { + UTIL.startMiniDFSCluster(1); + fs = UTIL.getDFSCluster().getFileSystem(); + rootDir = UTIL.getDefaultRootDirPath(); + } + + @AfterClass + public static void stopCluster() throws Exception { + UTIL.shutdownMiniDFSCluster(); + } + + @After + public void cleanupFiles() throws Exception { + // cleanup the snapshot directory + Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); + fs.delete(snapshotDir, true); + } + + @Test(timeout = 10000000) + public void testLoadAndDelete() throws Exception { + // don't refresh the cache unless we tell it to + long period = Long.MAX_VALUE; + Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + "test-snapshot-file-cache-refresh", new SnapshotFiles()); + + Path snapshot = new Path(snapshotDir, "snapshot"); + Path region = new Path(snapshot, "7e91021"); + Path family = new Path(region, "fam"); + Path file1 = new Path(family, "file1"); + Path file2 = new Path(family, "file2"); + + // create two hfiles under the snapshot + fs.create(file1); + fs.create(file2); + + FSUtils.logFileSystemState(fs, rootDir, LOG); + + // then make sure the cache finds them + assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName())); + assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName())); + String not = "file-shouldn't-be-found"; + assertFalse("Cache found '" + not + "', but it shouldn't have.", cache.contains(not)); + + // make sure we get a little bit of separation in the modification times + // its okay if we sleep a little longer (b/c of GC pause), as long as we sleep a little + Thread.sleep(10); + + LOG.debug("Deleting snapshot."); + // then delete the snapshot and make sure that we can still find the files + if (!fs.delete(snapshot, true)) { + throw new IOException("Couldn't delete " + snapshot + " for an unknown reason."); + } + FSUtils.logFileSystemState(fs, rootDir, LOG); + + + LOG.debug("Checking to see if file is deleted."); + assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName())); + assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName())); + + // then trigger a refresh + cache.triggerCacheRefreshForTesting(); + // and not it shouldn't find those files + assertFalse("Cache found '" + file1 + "', but it shouldn't have.", + cache.contains(file1.getName())); + assertFalse("Cache found '" + file2 + "', but it shouldn't have.", + cache.contains(file2.getName())); + + fs.delete(snapshotDir, true); + } + + @Test + public void testLoadsTmpDir() throws Exception { + // don't refresh the cache unless we tell it to + long period = Long.MAX_VALUE; + Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + "test-snapshot-file-cache-refresh", new SnapshotFiles()); + + // create a file in a 'completed' snapshot + Path snapshot = new Path(snapshotDir, "snapshot"); + Path region = new Path(snapshot, "7e91021"); + Path family = new Path(region, "fam"); + Path file1 = new Path(family, "file1"); + fs.create(file1); + + // create an 'in progress' snapshot + SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build(); + snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); + region = new Path(snapshot, "7e91021"); + family = new Path(region, "fam"); + Path file2 = new Path(family, "file2"); + fs.create(file2); + + FSUtils.logFileSystemState(fs, rootDir, LOG); + + // then make sure the cache finds both files + assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName())); + assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName())); + } + + @Test + public void testJustFindLogsDirectory() throws Exception { + // don't refresh the cache unless we tell it to + long period = Long.MAX_VALUE; + Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + "test-snapshot-file-cache-refresh", new SnapshotFileCache.SnapshotFileInspector() { + public Collection filesUnderSnapshot(final Path snapshotDir) + throws IOException { + return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir); + } + }); + + // create a file in a 'completed' snapshot + Path snapshot = new Path(snapshotDir, "snapshot"); + Path region = new Path(snapshot, "7e91021"); + Path family = new Path(region, "fam"); + Path file1 = new Path(family, "file1"); + fs.create(file1); + + // and another file in the logs directory + Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server"); + Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552"); + fs.create(log); + + FSUtils.logFileSystemState(fs, rootDir, LOG); + + // then make sure the cache only finds the log files + assertFalse("Cache found '" + file1 + "', but it shouldn't have.", + cache.contains(file1.getName())); + assertTrue("Cache didn't find:" + log, cache.contains(log.getName())); + } + + @Test + public void testReloadModifiedDirectory() throws IOException { + // don't refresh the cache unless we tell it to + long period = Long.MAX_VALUE; + Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + "test-snapshot-file-cache-refresh", new SnapshotFiles()); + + Path snapshot = new Path(snapshotDir, "snapshot"); + Path region = new Path(snapshot, "7e91021"); + Path family = new Path(region, "fam"); + Path file1 = new Path(family, "file1"); + Path file2 = new Path(family, "file2"); + + // create two hfiles under the snapshot + fs.create(file1); + fs.create(file2); + + FSUtils.logFileSystemState(fs, rootDir, LOG); + + assertTrue("Cache didn't find " + file1, cache.contains(file1.getName())); + + // now delete the snapshot and add a file with a different name + fs.delete(snapshot, true); + Path file3 = new Path(family, "new_file"); + fs.create(file3); + + FSUtils.logFileSystemState(fs, rootDir, LOG); + assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName())); + } + + class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { + public Collection filesUnderSnapshot(final Path snapshotDir) throws IOException { + Collection files = new HashSet(); + files.addAll(SnapshotReferenceUtil.getHLogNames(fs, snapshotDir)); + files.addAll(SnapshotReferenceUtil.getHFileNames(fs, snapshotDir)); + return files; + } + }; +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java new file mode 100644 index 0000000..03847c4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.snapshot; + +import static org.junit.Assert.assertFalse; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test that the snapshot hfile cleaner finds hfiles referenced in a snapshot + */ +@Category(SmallTests.class) +public class TestSnapshotHFileCleaner { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @AfterClass + public static void cleanup() throws IOException { + Configuration conf = TEST_UTIL.getConfiguration(); + Path rootDir = FSUtils.getRootDir(conf); + FileSystem fs = FileSystem.get(conf); + // cleanup + fs.delete(rootDir, true); + } + + @Test + public void testFindsSnapshotFilesWhenCleaning() throws IOException { + Configuration conf = TEST_UTIL.getConfiguration(); + FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); + Path rootDir = FSUtils.getRootDir(conf); + Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY); + + FileSystem fs = FileSystem.get(conf); + SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner(); + cleaner.setConf(conf); + + // write an hfile to the snapshot directory + String snapshotName = "snapshot"; + byte[] snapshot = Bytes.toBytes(snapshotName); + String table = "table"; + byte[] tableName = Bytes.toBytes(table); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + HRegionInfo mockRegion = new HRegionInfo(tableName); + Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName()); + Path familyDir = new Path(regionSnapshotDir, "family"); + // create a reference to a supposedly valid hfile + String hfile = "fd1e73e8a96c486090c5cec07b4894c4"; + Path refFile = new Path(familyDir, hfile); + + // make sure the reference file exists + fs.create(refFile); + + // create the hfile in the archive + fs.mkdirs(archivedHfileDir); + fs.createNewFile(new Path(archivedHfileDir, hfile)); + + // make sure that the file isn't deletable + assertFalse(cleaner.isFileDeletable(new Path(hfile))); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java new file mode 100644 index 0000000..f0f2ebd --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotLogCleaner.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.snapshot; + +import static org.junit.Assert.assertFalse; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test that the snapshot log cleaner finds logs referenced in a snapshot + */ +@Category(SmallTests.class) +public class TestSnapshotLogCleaner { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @AfterClass + public static void cleanup() throws IOException { + Configuration conf = TEST_UTIL.getConfiguration(); + Path rootDir = FSUtils.getRootDir(conf); + FileSystem fs = FileSystem.get(conf); + // cleanup + fs.delete(rootDir, true); + } + + @Test + public void testFindsSnapshotFilesWhenCleaning() throws IOException { + Configuration conf = TEST_UTIL.getConfiguration(); + FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); + Path rootDir = FSUtils.getRootDir(conf); + FileSystem fs = FileSystem.get(conf); + SnapshotLogCleaner cleaner = new SnapshotLogCleaner(); + cleaner.setConf(conf); + + // write an hfile to the snapshot directory + String snapshotName = "snapshot"; + byte[] snapshot = Bytes.toBytes(snapshotName); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + Path snapshotLogDir = new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME); + String timestamp = "1339643343027"; + String hostFromMaster = "localhost%2C59648%2C1339643336601"; + + Path hostSnapshotLogDir = new Path(snapshotLogDir, hostFromMaster); + String snapshotlogfile = hostFromMaster + "." + timestamp + ".hbase"; + + // add the reference to log in the snapshot + fs.create(new Path(hostSnapshotLogDir, snapshotlogfile)); + + // now check to see if that log file would get deleted. + Path oldlogDir = new Path(rootDir, ".oldlogs"); + Path logFile = new Path(oldlogDir, snapshotlogfile); + fs.create(logFile); + + // make sure that the file isn't deletable + assertFalse(cleaner.isFileDeletable(logFile)); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java new file mode 100644 index 0000000..47c57bf --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.snapshot; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.executor.ExecutorService; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; +import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner; +import org.apache.hadoop.hbase.master.metrics.MasterMetrics; +import org.apache.hadoop.hbase.procedure.ProcedureCoordinator; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.zookeeper.KeeperException; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test basic snapshot manager functionality + */ +@Category(SmallTests.class) +public class TestSnapshotManager { + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + MasterServices services = Mockito.mock(MasterServices.class); + MasterMetrics metrics = Mockito.mock(MasterMetrics.class); + ProcedureCoordinator coordinator = Mockito.mock(ProcedureCoordinator.class); + ExecutorService pool = Mockito.mock(ExecutorService.class); + MasterFileSystem mfs = Mockito.mock(MasterFileSystem.class); + FileSystem fs; + { + try { + fs = UTIL.getTestFileSystem(); + } catch (IOException e) { + throw new RuntimeException("Couldn't get test filesystem", e); + } + } + + private SnapshotManager getNewManager() throws IOException, KeeperException { + return getNewManager(UTIL.getConfiguration()); + } + + private SnapshotManager getNewManager(final Configuration conf) + throws IOException, KeeperException { + Mockito.reset(services); + Mockito.when(services.getConfiguration()).thenReturn(conf); + Mockito.when(services.getMasterFileSystem()).thenReturn(mfs); + Mockito.when(mfs.getFileSystem()).thenReturn(fs); + Mockito.when(mfs.getRootDir()).thenReturn(UTIL.getDataTestDir()); + return new SnapshotManager(services, metrics, coordinator, pool); + } + + @Test + public void testInProcess() throws KeeperException, IOException { + String tableName = "testTable"; + SnapshotManager manager = getNewManager(); + TakeSnapshotHandler handler = Mockito.mock(TakeSnapshotHandler.class); + assertFalse("Manager is in process when there is no current handler", + manager.isTakingSnapshot(tableName)); + manager.setSnapshotHandlerForTesting(tableName, handler); + Mockito.when(handler.isFinished()).thenReturn(false); + assertTrue("Manager isn't in process when handler is running", + manager.isTakingSnapshot(tableName)); + Mockito.when(handler.isFinished()).thenReturn(true); + assertFalse("Manager is process when handler isn't running", + manager.isTakingSnapshot(tableName)); + } + + /** + * Verify the snapshot support based on the configuration. + */ + @Test + public void testSnapshotSupportConfiguration() throws Exception { + // No configuration (no cleaners, not enabled): snapshot feature disabled + Configuration conf = new Configuration(); + SnapshotManager manager = getNewManager(conf); + assertFalse("Snapshot should be disabled with no configuration", isSnapshotSupported(manager)); + + // force snapshot feature to be enabled + conf = new Configuration(); + conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + manager = getNewManager(conf); + assertTrue("Snapshot should be enabled", isSnapshotSupported(manager)); + + // force snapshot feature to be disabled + conf = new Configuration(); + conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false); + manager = getNewManager(conf); + assertFalse("Snapshot should be disabled", isSnapshotSupported(manager)); + + // force snapshot feature to be disabled, even if cleaners are present + conf = new Configuration(); + conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, + SnapshotHFileCleaner.class.getName(), HFileLinkCleaner.class.getName()); + conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, SnapshotLogCleaner.class.getName()); + conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false); + manager = getNewManager(conf); + assertFalse("Snapshot should be disabled", isSnapshotSupported(manager)); + + // cleaners are present, but missing snapshot enabled property + conf = new Configuration(); + conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, + SnapshotHFileCleaner.class.getName(), HFileLinkCleaner.class.getName()); + conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, SnapshotLogCleaner.class.getName()); + manager = getNewManager(conf); + assertTrue("Snapshot should be enabled, because cleaners are present", + isSnapshotSupported(manager)); + + // Create a "test snapshot" + Path rootDir = UTIL.getDataTestDir(); + Path testSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( + "testSnapshotSupportConfiguration", rootDir); + fs.mkdirs(testSnapshotDir); + try { + // force snapshot feature to be disabled, but snapshots are present + conf = new Configuration(); + conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, false); + manager = getNewManager(conf); + fail("Master should not start when snapshot is disabled, but snapshots are present"); + } catch (UnsupportedOperationException e) { + // expected + } finally { + fs.delete(testSnapshotDir, true); + } + } + + private boolean isSnapshotSupported(final SnapshotManager manager) { + try { + manager.checkSnapshotSupport(); + return true; + } catch (UnsupportedOperationException e) { + return false; + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestExactCounterMetric.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestExactCounterMetric.java new file mode 100644 index 0000000..a0ba248 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestExactCounterMetric.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.metrics; + +import java.util.List; + +import junit.framework.Assert; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestExactCounterMetric { + + @Test + public void testBasic() { + final ExactCounterMetric counter = new ExactCounterMetric("testCounter", null); + for (int i = 1; i <= 10; i++) { + for (int j = 0; j < i; j++) { + counter.update(i + ""); + } + } + + List> topFive = counter.getTop(5); + Long i = 10L; + for (Pair entry : topFive) { + Assert.assertEquals(i + "", entry.getFirst()); + Assert.assertEquals(i, entry.getSecond()); + i--; + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java new file mode 100644 index 0000000..9d09486 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestExponentiallyDecayingSample.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.metrics; + +import junit.framework.Assert; + +import com.yammer.metrics.stats.ExponentiallyDecayingSample; +import com.yammer.metrics.stats.Snapshot; + +import org.apache.hadoop.hbase.SmallTests; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestExponentiallyDecayingSample { + + @Test + public void testBasic() { + final ExponentiallyDecayingSample sample = + new ExponentiallyDecayingSample(100, 0.99); + + for (int i = 0; i < 1000; i++) { + sample.update(i); + } + Assert.assertEquals(100, sample.size()); + + final Snapshot snapshot = sample.getSnapshot(); + Assert.assertEquals(100, snapshot.size()); + + for (double i : snapshot.getValues()) { + Assert.assertTrue(i >= 0.0 && i < 1000.0); + } + } + + @Test + public void testTooBig() throws Exception { + final ExponentiallyDecayingSample sample = + new ExponentiallyDecayingSample(100, 0.99); + for (int i = 0; i < 10; i++) { + sample.update(i); + } + Assert.assertEquals(10, sample.size()); + + final Snapshot snapshot = sample.getSnapshot(); + Assert.assertEquals(10, sample.size()); + + for (double i : snapshot.getValues()) { + Assert.assertTrue(i >= 0.0 && i < 1000.0); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java new file mode 100644 index 0000000..da0e87e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsHistogram.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.metrics; + +import java.util.Arrays; +import java.util.Random; + +import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram; +import org.apache.hadoop.hbase.SmallTests; +import com.yammer.metrics.stats.Snapshot; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestMetricsHistogram { + + @Test + public void testBasicUniform() { + MetricsHistogram h = new MetricsHistogram("testHistogram", null); + + for (int i = 0; i < 100; i++) { + h.update(i); + } + + Assert.assertEquals(100, h.getCount()); + Assert.assertEquals(0, h.getMin()); + Assert.assertEquals(99, h.getMax()); + } + + private static int safeIndex(int i, int len) { + if (i < len && i>= 0) { + return i; + } else if (i >= len) { + return len - 1; + } else { + return 0; + } + } + + @Test + public void testRandom() { + final Random r = new Random(); + final MetricsHistogram h = new MetricsHistogram("testHistogram", null); + + final long[] data = new long[1000]; + + for (int i = 0; i < data.length; i++) { + data[i] = (long) (r.nextGaussian() * 10000.0); + h.update(data[i]); + } + + final Snapshot s = h.getSnapshot(); + Arrays.sort(data); + + // as long as the histogram chooses an item with index N+/-slop, accept it + final int slop = 20; + + // make sure the median, 75th percentile and 95th percentile are good + final int medianIndex = data.length / 2; + final long minAcceptableMedian = data[safeIndex(medianIndex - slop, + data.length)]; + final long maxAcceptableMedian = data[safeIndex(medianIndex + slop, + data.length)]; + Assert.assertTrue(s.getMedian() >= minAcceptableMedian + && s.getMedian() <= maxAcceptableMedian); + + final int seventyFifthIndex = (int) (data.length * 0.75); + final long minAcceptableseventyFifth = data[safeIndex(seventyFifthIndex + - slop, data.length)]; + final long maxAcceptableseventyFifth = data[safeIndex(seventyFifthIndex + + slop, data.length)]; + Assert.assertTrue(s.get75thPercentile() >= minAcceptableseventyFifth + && s.get75thPercentile() <= maxAcceptableseventyFifth); + + final int ninetyFifthIndex = (int) (data.length * 0.95); + final long minAcceptableninetyFifth = data[safeIndex(ninetyFifthIndex + - slop, data.length)]; + final long maxAcceptableninetyFifth = data[safeIndex(ninetyFifthIndex + + slop, data.length)]; + Assert.assertTrue(s.get95thPercentile() >= minAcceptableninetyFifth + && s.get95thPercentile() <= maxAcceptableninetyFifth); + + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsMBeanBase.java b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsMBeanBase.java new file mode 100644 index 0000000..96e7772 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsMBeanBase.java @@ -0,0 +1,184 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.management.AttributeNotFoundException; +import javax.management.MBeanAttributeInfo; +import javax.management.MBeanException; +import javax.management.MBeanInfo; +import javax.management.ReflectionException; +import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram; +import com.yammer.metrics.stats.Snapshot; + +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.metrics.MetricsContext; +import org.apache.hadoop.metrics.MetricsRecord; +import org.apache.hadoop.metrics.MetricsUtil; +import org.apache.hadoop.metrics.util.MetricsIntValue; +import org.apache.hadoop.metrics.util.MetricsRegistry; +import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestMetricsMBeanBase extends TestCase { + + private class TestStatistics extends MetricsMBeanBase { + public TestStatistics(MetricsRegistry registry) { + super(registry, "TestStatistics"); + } + } + + private MetricsRegistry registry; + private MetricsRecord metricsRecord; + private TestStatistics stats; + private MetricsRate metricsRate; + private MetricsIntValue intValue; + private MetricsTimeVaryingRate varyRate; + + public void setUp() { + this.registry = new MetricsRegistry(); + this.metricsRate = new MetricsRate("metricsRate", registry, "test"); + this.intValue = new MetricsIntValue("intValue", registry, "test"); + this.varyRate = new MetricsTimeVaryingRate("varyRate", registry, "test"); + this.stats = new TestStatistics(registry); + MetricsContext context = MetricsUtil.getContext("hbase"); + this.metricsRecord = MetricsUtil.createRecord(context, "test"); + this.metricsRecord.setTag("TestStatistics", "test"); + //context.registerUpdater(this); + + } + + public void tearDown() { + + } + + public void testGetAttribute() throws Exception { + this.metricsRate.inc(2); + Thread.sleep(1000); + this.metricsRate.pushMetric(this.metricsRecord); + this.intValue.set(5); + this.intValue.pushMetric(this.metricsRecord); + this.varyRate.inc(10); + this.varyRate.inc(50); + this.varyRate.pushMetric(this.metricsRecord); + + + assertEquals( 2.0, (Float)this.stats.getAttribute("metricsRate"), 0.005 ); + assertEquals( 5, this.stats.getAttribute("intValue") ); + assertEquals( 10L, this.stats.getAttribute("varyRateMinTime") ); + assertEquals( 50L, this.stats.getAttribute("varyRateMaxTime") ); + assertEquals( 30L, this.stats.getAttribute("varyRateAvgTime") ); + assertEquals( 2, this.stats.getAttribute("varyRateNumOps") ); + } + + public void testGetMBeanInfo() { + MBeanInfo info = this.stats.getMBeanInfo(); + MBeanAttributeInfo[] attributes = info.getAttributes(); + assertEquals( 6, attributes.length ); + + Map attributeByName = + new HashMap(attributes.length); + for (MBeanAttributeInfo attr : attributes) + attributeByName.put(attr.getName(), attr); + + assertAttribute( attributeByName.get("metricsRate"), + "metricsRate", "java.lang.Float", "test"); + assertAttribute( attributeByName.get("intValue"), + "intValue", "java.lang.Integer", "test"); + assertAttribute( attributeByName.get("varyRateMinTime"), + "varyRateMinTime", "java.lang.Long", "test"); + assertAttribute( attributeByName.get("varyRateMaxTime"), + "varyRateMaxTime", "java.lang.Long", "test"); + assertAttribute( attributeByName.get("varyRateAvgTime"), + "varyRateAvgTime", "java.lang.Long", "test"); + assertAttribute( attributeByName.get("varyRateNumOps"), + "varyRateNumOps", "java.lang.Integer", "test"); + } + + public void testMetricsMBeanBaseHistogram() + throws ReflectionException, AttributeNotFoundException, MBeanException { + MetricsRegistry mr = new MetricsRegistry(); + MetricsHistogram histo = mock(MetricsHistogram.class); + Snapshot snap = mock(Snapshot.class); + + //Set up the mocks + String histoName = "MockHisto"; + when(histo.getName()).thenReturn(histoName); + when(histo.getCount()).thenReturn(20l); + when(histo.getMin()).thenReturn(1l); + when(histo.getMax()).thenReturn(999l); + when(histo.getMean()).thenReturn(500.2); + when(histo.getStdDev()).thenReturn(1.2); + when(histo.getSnapshot()).thenReturn(snap); + + when(snap.getMedian()).thenReturn(490.0); + when(snap.get75thPercentile()).thenReturn(550.0); + when(snap.get95thPercentile()).thenReturn(900.0); + when(snap.get99thPercentile()).thenReturn(990.0); + + mr.add("myTestHisto", histo); + + MetricsMBeanBase mBeanBase = new MetricsMBeanBase(mr, "test"); + + assertEquals(new Long(20), mBeanBase + .getAttribute(histoName + MetricsHistogram.NUM_OPS_METRIC_NAME)); + assertEquals(new Long(1), mBeanBase + .getAttribute(histoName + MetricsHistogram.MIN_METRIC_NAME)); + assertEquals(new Long(999), mBeanBase + .getAttribute(histoName + MetricsHistogram.MAX_METRIC_NAME)); + assertEquals(new Float(500.2), mBeanBase + .getAttribute(histoName + MetricsHistogram.MEAN_METRIC_NAME)); + assertEquals(new Float(1.2), mBeanBase + .getAttribute(histoName + MetricsHistogram.STD_DEV_METRIC_NAME)); + + assertEquals(new Float(490.0), mBeanBase + .getAttribute(histoName + MetricsHistogram.MEDIAN_METRIC_NAME)); + assertEquals(new Float(550.0), mBeanBase + .getAttribute(histoName + MetricsHistogram.SEVENTY_FIFTH_PERCENTILE_METRIC_NAME)); + assertEquals(new Float(900.0), mBeanBase + .getAttribute(histoName + MetricsHistogram.NINETY_FIFTH_PERCENTILE_METRIC_NAME)); + assertEquals(new Float(990.0), mBeanBase + .getAttribute(histoName + MetricsHistogram.NINETY_NINETH_PERCENTILE_METRIC_NAME)); + } + + protected void assertAttribute(MBeanAttributeInfo attr, String name, + String type, String description) { + + assertEquals(attr.getName(), name); + assertEquals(attr.getType(), type); + assertEquals(attr.getDescription(), description); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java b/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java new file mode 100644 index 0000000..0700508 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.metrics.file; + +import java.io.File; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Collection; +import java.util.Date; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hbase.SmallTests; + +import org.junit.*; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +/** + * Test for TimeStampingMetricsContext functionality. + * (FQN class names are used to suppress javac warnings in imports.) + */ +@Category(SmallTests.class) +@SuppressWarnings("deprecation") +public class TestTimeStampingMetricsContext { + + private static final int updatePeriodSeconds = 2; + + private TimeStampingFileContext mc; + + @Test + public void testFileUpdate() throws Exception { + final Date start = new Date(); + final File metricOutFile = FileUtil.createLocalTempFile( + new File(FileUtils.getTempDirectory(),getClass().getName() + "-out-"), "", true); + assertTrue(metricOutFile.exists()); + assertEquals(0L, metricOutFile.length()); + + mc = new TimeStampingFileContext(); + org.apache.hadoop.metrics.ContextFactory cf + = org.apache.hadoop.metrics.ContextFactory.getFactory(); + cf.setAttribute("test1.fileName", metricOutFile.getAbsolutePath()); + cf.setAttribute("test1.period", Integer.toString(updatePeriodSeconds)); + mc.init("test1", cf); + + assertEquals("test1", mc.getContextName()); + + org.apache.hadoop.metrics.MetricsRecord r = mc.createRecord("testRecord"); + r.setTag("testTag1", "testTagValue1"); + r.setTag("testTag2", "testTagValue2"); + r.setMetric("testMetric1", 1); + r.setMetric("testMetric2", 33); + r.update(); + + mc.startMonitoring(); + assertTrue(mc.isMonitoring()); + + // wait 3/2 of the update period: + Thread.sleep((1000 * updatePeriodSeconds * 3)/2); + + mc.stopMonitoring(); + assertFalse(mc.isMonitoring()); + + mc.close(); + + Map> m = mc.getAllRecords(); + assertEquals(1, m.size()); + Collection outputRecords = m.get("testRecord"); + assertNotNull(outputRecords); + assertEquals(1, outputRecords.size()); + org.apache.hadoop.metrics.spi.OutputRecord outputRecord = outputRecords.iterator().next(); + assertNotNull(outputRecord); + + String outStr = FileUtils.readFileToString(metricOutFile); + assertTrue(outStr.length() > 0); + int pos = outStr.indexOf(" "); + String time = outStr.substring(0, pos); + + DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + Date date = df.parse(time); + assertTrue(date.after(start)); + assertTrue(date.before(new Date())); + + String reminder = outStr.substring(pos); + assertEquals(" test1.testRecord: testTag1=testTagValue1, testTag2=testTagValue2, testMetric1=1," + +" testMetric2=33\n", reminder); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/migration/TestMigrationFrom090To092.java b/src/test/java/org/apache/hadoop/hbase/migration/TestMigrationFrom090To092.java new file mode 100644 index 0000000..c3651ac --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/migration/TestMigrationFrom090To092.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.migration; + + +import java.io.IOException; + +import junit.framework.Assert; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD; +import org.apache.hadoop.hbase.util.Writables; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Migration tests that do not need spin up of a cluster. + * @deprecated Remove after we release 0.92 + */ +@Category(SmallTests.class) +public class TestMigrationFrom090To092 { + @Test + public void testMigrateHRegionInfoFromVersion0toVersion1() + throws IOException { + HTableDescriptor htd = + getHTableDescriptor("testMigrateHRegionInfoFromVersion0toVersion1"); + HRegionInfo090x ninety = + new HRegionInfo090x(htd, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); + byte [] bytes = Writables.getBytes(ninety); + // Now deserialize into an HRegionInfo + HRegionInfo hri = Writables.getHRegionInfo(bytes); + Assert.assertEquals(hri.getTableNameAsString(), + ninety.getTableDesc().getNameAsString()); + Assert.assertEquals(HRegionInfo.VERSION, hri.getVersion()); + } + + private HTableDescriptor getHTableDescriptor(final String name) { + HTableDescriptor htd = new HTableDescriptor(name); + htd.addFamily(new HColumnDescriptor("family")); + return htd; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/monitoring/TestMemoryBoundedLogMessageBuffer.java b/src/test/java/org/apache/hadoop/hbase/monitoring/TestMemoryBoundedLogMessageBuffer.java new file mode 100644 index 0000000..ac76887 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/monitoring/TestMemoryBoundedLogMessageBuffer.java @@ -0,0 +1,79 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.monitoring; + +import static org.junit.Assert.*; + +import java.io.PrintWriter; +import java.io.StringWriter; + +import org.apache.hadoop.hbase.MediumTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test case for the MemoryBoundedLogMessageBuffer utility. + * Ensures that it uses no more memory than it's supposed to, + * and that it properly deals with multibyte encodings. + */ +@Category(MediumTests.class) +public class TestMemoryBoundedLogMessageBuffer { + + private static final long TEN_KB = 10 * 1024; + private static final String JP_TEXT = "こんにちは"; + + @Test + public void testBuffer() { + MemoryBoundedLogMessageBuffer buf = + new MemoryBoundedLogMessageBuffer(TEN_KB); + + for (int i = 0; i < 1000; i++) { + buf.add("hello " + i); + } + assertTrue("Usage too big: " + buf.estimateHeapUsage(), + buf.estimateHeapUsage() < TEN_KB); + assertTrue("Too many retained: " + buf.getMessages().size(), + buf.getMessages().size() < 100); + StringWriter sw = new StringWriter(); + buf.dumpTo(new PrintWriter(sw)); + String dump = sw.toString(); + assertFalse("The early log messages should be evicted", + dump.contains("hello 1\n")); + assertTrue("The late log messages should be retained", + dump.contains("hello 999\n")); + } + + @Test + public void testNonAsciiEncoding() { + MemoryBoundedLogMessageBuffer buf = + new MemoryBoundedLogMessageBuffer(TEN_KB); + + buf.add(JP_TEXT); + StringWriter sw = new StringWriter(); + buf.dumpTo(new PrintWriter(sw)); + String dump = sw.toString(); + assertTrue(dump.contains(JP_TEXT)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/monitoring/TestTaskMonitor.java b/src/test/java/org/apache/hadoop/hbase/monitoring/TestTaskMonitor.java new file mode 100644 index 0000000..98de024 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/monitoring/TestTaskMonitor.java @@ -0,0 +1,109 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.monitoring; + +import static org.junit.Assert.*; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.hbase.MediumTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestTaskMonitor { + + @Test + public void testTaskMonitorBasics() { + TaskMonitor tm = new TaskMonitor(); + assertTrue("Task monitor should start empty", + tm.getTasks().isEmpty()); + + // Make a task and fetch it back out + MonitoredTask task = tm.createStatus("Test task"); + MonitoredTask taskFromTm = tm.getTasks().get(0); + + // Make sure the state is reasonable. + assertEquals(task.getDescription(), taskFromTm.getDescription()); + assertEquals(-1, taskFromTm.getCompletionTimestamp()); + assertEquals(MonitoredTask.State.RUNNING, taskFromTm.getState()); + + // Mark it as finished + task.markComplete("Finished!"); + assertEquals(MonitoredTask.State.COMPLETE, task.getState()); + + // It should still show up in the TaskMonitor list + assertEquals(1, tm.getTasks().size()); + + // If we mark its completion time back a few minutes, it should get gced + task.expireNow(); + assertEquals(0, tm.getTasks().size()); + } + + @Test + public void testTasksGetAbortedOnLeak() throws InterruptedException { + final TaskMonitor tm = new TaskMonitor(); + assertTrue("Task monitor should start empty", + tm.getTasks().isEmpty()); + + final AtomicBoolean threadSuccess = new AtomicBoolean(false); + // Make a task in some other thread and leak it + Thread t = new Thread() { + @Override + public void run() { + MonitoredTask task = tm.createStatus("Test task"); + assertEquals(MonitoredTask.State.RUNNING, task.getState()); + threadSuccess.set(true); + } + }; + t.start(); + t.join(); + // Make sure the thread saw the correct state + assertTrue(threadSuccess.get()); + + // Make sure the leaked reference gets cleared + System.gc(); + System.gc(); + System.gc(); + + // Now it should be aborted + MonitoredTask taskFromTm = tm.getTasks().get(0); + assertEquals(MonitoredTask.State.ABORTED, taskFromTm.getState()); + } + + @Test + public void testTaskLimit() throws Exception { + TaskMonitor tm = new TaskMonitor(); + for (int i = 0; i < TaskMonitor.MAX_TASKS + 10; i++) { + tm.createStatus("task " + i); + } + // Make sure it was limited correctly + assertEquals(TaskMonitor.MAX_TASKS, tm.getTasks().size()); + // Make sure we culled the earlier tasks, not later + // (i.e. tasks 0 through 9 should have been deleted) + assertEquals("task 10", tm.getTasks().get(0).getDescription()); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java new file mode 100644 index 0000000..4026f39 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java @@ -0,0 +1,234 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.procedure; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.errorhandling.ForeignException; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Demonstrate how Procedure handles single members, multiple members, and errors semantics + */ +@Category(SmallTests.class) +public class TestProcedure { + + ProcedureCoordinator coord; + + @Before + public void setup() { + coord = mock(ProcedureCoordinator.class); + final ProcedureCoordinatorRpcs comms = mock(ProcedureCoordinatorRpcs.class); + when(coord.getRpcs()).thenReturn(comms); // make it not null + } + + class LatchedProcedure extends Procedure { + CountDownLatch startedAcquireBarrier = new CountDownLatch(1); + CountDownLatch startedDuringBarrier = new CountDownLatch(1); + CountDownLatch completedProcedure = new CountDownLatch(1); + + public LatchedProcedure(ProcedureCoordinator coord, ForeignExceptionDispatcher monitor, + long wakeFreq, long timeout, String opName, byte[] data, + List expectedMembers) { + super(coord, monitor, wakeFreq, timeout, opName, data, expectedMembers); + } + + @Override + public void sendGlobalBarrierStart() { + startedAcquireBarrier.countDown(); + } + + @Override + public void sendGlobalBarrierReached() { + startedDuringBarrier.countDown(); + } + + @Override + public void sendGlobalBarrierComplete() { + completedProcedure.countDown(); + } + }; + + /** + * With a single member, verify ordered execution. The Coordinator side is run in a separate + * thread so we can only trigger from members and wait for particular state latches. + */ + @Test(timeout = 60000) + public void testSingleMember() throws Exception { + // The member + List members = new ArrayList(); + members.add("member"); + LatchedProcedure proc = new LatchedProcedure(coord, new ForeignExceptionDispatcher(), 100, + Integer.MAX_VALUE, "op", null, members); + final LatchedProcedure procspy = spy(proc); + // coordinator: start the barrier procedure + new Thread() { + public void run() { + procspy.call(); + } + }.start(); + + // coordinator: wait for the barrier to be acquired, then send start barrier + proc.startedAcquireBarrier.await(); + + // we only know that {@link Procedure#sendStartBarrier()} was called, and others are blocked. + verify(procspy).sendGlobalBarrierStart(); + verify(procspy, never()).sendGlobalBarrierReached(); + verify(procspy, never()).sendGlobalBarrierComplete(); + verify(procspy, never()).barrierAcquiredByMember(anyString()); + + // member: trigger global barrier acquisition + proc.barrierAcquiredByMember(members.get(0)); + + // coordinator: wait for global barrier to be acquired. + proc.acquiredBarrierLatch.await(); + verify(procspy).sendGlobalBarrierStart(); // old news + + // since two threads, we cannot guarantee that {@link Procedure#sendSatsifiedBarrier()} was + // or was not called here. + + // member: trigger global barrier release + proc.barrierReleasedByMember(members.get(0)); + + // coordinator: wait for procedure to be completed + proc.completedProcedure.await(); + verify(procspy).sendGlobalBarrierReached(); + verify(procspy).sendGlobalBarrierComplete(); + verify(procspy, never()).receive(any(ForeignException.class)); + } + + @Test(timeout=60000) + public void testMultipleMember() throws Exception { + // 2 members + List members = new ArrayList(); + members.add("member1"); + members.add("member2"); + + LatchedProcedure proc = new LatchedProcedure(coord, new ForeignExceptionDispatcher(), 100, + Integer.MAX_VALUE, "op", null, members); + final LatchedProcedure procspy = spy(proc); + // start the barrier procedure + new Thread() { + public void run() { + procspy.call(); + } + }.start(); + + // coordinator: wait for the barrier to be acquired, then send start barrier + procspy.startedAcquireBarrier.await(); + + // we only know that {@link Procedure#sendStartBarrier()} was called, and others are blocked. + verify(procspy).sendGlobalBarrierStart(); + verify(procspy, never()).sendGlobalBarrierReached(); + verify(procspy, never()).sendGlobalBarrierComplete(); + verify(procspy, never()).barrierAcquiredByMember(anyString()); // no externals + + // member0: [1/2] trigger global barrier acquisition. + procspy.barrierAcquiredByMember(members.get(0)); + + // coordinator not satisified. + verify(procspy).sendGlobalBarrierStart(); + verify(procspy, never()).sendGlobalBarrierReached(); + verify(procspy, never()).sendGlobalBarrierComplete(); + + // member 1: [2/2] trigger global barrier acquisition. + procspy.barrierAcquiredByMember(members.get(1)); + + // coordinator: wait for global barrier to be acquired. + procspy.startedDuringBarrier.await(); + verify(procspy).sendGlobalBarrierStart(); // old news + + // member 1, 2: trigger global barrier release + procspy.barrierReleasedByMember(members.get(0)); + procspy.barrierReleasedByMember(members.get(1)); + + // coordinator wait for procedure to be completed + procspy.completedProcedure.await(); + verify(procspy).sendGlobalBarrierReached(); + verify(procspy).sendGlobalBarrierComplete(); + verify(procspy, never()).receive(any(ForeignException.class)); + } + + @Test(timeout = 60000) + public void testErrorPropagation() throws Exception { + List members = new ArrayList(); + members.add("member"); + Procedure proc = new Procedure(coord, new ForeignExceptionDispatcher(), 100, + Integer.MAX_VALUE, "op", null, members); + final Procedure procspy = spy(proc); + + ForeignException cause = new ForeignException("SRC", "External Exception"); + proc.receive(cause); + + // start the barrier procedure + Thread t = new Thread() { + public void run() { + procspy.call(); + } + }; + t.start(); + t.join(); + + verify(procspy, never()).sendGlobalBarrierStart(); + verify(procspy, never()).sendGlobalBarrierReached(); + verify(procspy).sendGlobalBarrierComplete(); + } + + @Test(timeout = 60000) + public void testBarrieredErrorPropagation() throws Exception { + List members = new ArrayList(); + members.add("member"); + LatchedProcedure proc = new LatchedProcedure(coord, new ForeignExceptionDispatcher(), 100, + Integer.MAX_VALUE, "op", null, members); + final LatchedProcedure procspy = spy(proc); + + // start the barrier procedure + Thread t = new Thread() { + public void run() { + procspy.call(); + } + }; + t.start(); + + // now test that we can put an error in before the commit phase runs + procspy.startedAcquireBarrier.await(); + ForeignException cause = new ForeignException("SRC", "External Exception"); + procspy.receive(cause); + procspy.barrierAcquiredByMember(members.get(0)); + t.join(); + + // verify state of all the object + verify(procspy).sendGlobalBarrierStart(); + verify(procspy).sendGlobalBarrierComplete(); + verify(procspy, never()).sendGlobalBarrierReached(); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java new file mode 100644 index 0000000..f48aa23 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java @@ -0,0 +1,349 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.procedure; + +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyListOf; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.errorhandling.ForeignException; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.junit.After; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.InOrder; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import com.google.common.collect.Lists; + +/** + * Test Procedure coordinator operation. + *

    + * This only works correctly when we do class level parallelization of tests. If we do method + * level serialization this class will likely throw all kinds of errors. + */ +@Category(SmallTests.class) +public class TestProcedureCoordinator { + // general test constants + private static final long WAKE_FREQUENCY = 1000; + private static final long TIMEOUT = 100000; + private static final long POOL_KEEP_ALIVE = 1; + private static final String nodeName = "node"; + private static final String procName = "some op"; + private static final byte[] procData = new byte[0]; + private static final List expected = Lists.newArrayList("remote1", "remote2"); + + // setup the mocks + private final ProcedureCoordinatorRpcs controller = mock(ProcedureCoordinatorRpcs.class); + private final Procedure task = mock(Procedure.class); + private final ForeignExceptionDispatcher monitor = mock(ForeignExceptionDispatcher.class); + + // handle to the coordinator for each test + private ProcedureCoordinator coordinator; + + @After + public void resetTest() throws IOException { + // reset all the mocks used for the tests + reset(controller, task, monitor); + // close the open coordinator, if it was used + if (coordinator != null) coordinator.close(); + } + + private ProcedureCoordinator buildNewCoordinator() { + ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(nodeName, POOL_KEEP_ALIVE, 1, WAKE_FREQUENCY); + return spy(new ProcedureCoordinator(controller, pool)); + } + + /** + * Currently we can only handle one procedure at a time. This makes sure we handle that and + * reject submitting more. + */ + @Test + public void testThreadPoolSize() throws Exception { + ProcedureCoordinator coordinator = buildNewCoordinator(); + Procedure proc = new Procedure(coordinator, monitor, + WAKE_FREQUENCY, TIMEOUT, procName, procData, expected); + Procedure procSpy = spy(proc); + + Procedure proc2 = new Procedure(coordinator, monitor, + WAKE_FREQUENCY, TIMEOUT, procName +"2", procData, expected); + Procedure procSpy2 = spy(proc2); + when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class))) + .thenReturn(procSpy, procSpy2); + + coordinator.startProcedure(procSpy.getErrorMonitor(), procName, procData, expected); + // null here means second procedure failed to start. + assertNull("Coordinator successfully ran two tasks at once with a single thread pool.", + coordinator.startProcedure(proc2.getErrorMonitor(), "another op", procData, expected)); + } + + /** + * Check handling a connection failure correctly if we get it during the acquiring phase + */ + @Test(timeout = 60000) + public void testUnreachableControllerDuringPrepare() throws Exception { + coordinator = buildNewCoordinator(); + // setup the proc + List expected = Arrays.asList("cohort"); + Procedure proc = new Procedure(coordinator, WAKE_FREQUENCY, + TIMEOUT, procName, procData, expected); + final Procedure procSpy = spy(proc); + + when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class))) + .thenReturn(procSpy); + + // use the passed controller responses + IOException cause = new IOException("Failed to reach comms during acquire"); + doThrow(cause).when(controller) + .sendGlobalBarrierAcquire(eq(procSpy), eq(procData), anyListOf(String.class)); + + // run the operation + proc = coordinator.startProcedure(proc.getErrorMonitor(), procName, procData, expected); + // and wait for it to finish + proc.waitForCompleted(); + verify(procSpy, atLeastOnce()).receive(any(ForeignException.class)); + verify(coordinator, times(1)).rpcConnectionFailure(anyString(), eq(cause)); + verify(controller, times(1)).sendGlobalBarrierAcquire(procSpy, procData, expected); + verify(controller, never()).sendGlobalBarrierReached(any(Procedure.class), + anyListOf(String.class)); + } + + /** + * Check handling a connection failure correctly if we get it during the barrier phase + */ + @Test(timeout = 60000) + public void testUnreachableControllerDuringCommit() throws Exception { + coordinator = buildNewCoordinator(); + + // setup the task and spy on it + List expected = Arrays.asList("cohort"); + final Procedure spy = spy(new Procedure(coordinator, + WAKE_FREQUENCY, TIMEOUT, procName, procData, expected)); + + when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class))) + .thenReturn(spy); + + // use the passed controller responses + IOException cause = new IOException("Failed to reach controller during prepare"); + doAnswer(new AcquireBarrierAnswer(procName, new String[] { "cohort" })) + .when(controller).sendGlobalBarrierAcquire(eq(spy), eq(procData), anyListOf(String.class)); + doThrow(cause).when(controller).sendGlobalBarrierReached(eq(spy), anyListOf(String.class)); + + // run the operation + Procedure task = coordinator.startProcedure(spy.getErrorMonitor(), procName, procData, expected); + // and wait for it to finish + task.waitForCompleted(); + verify(spy, atLeastOnce()).receive(any(ForeignException.class)); + verify(coordinator, times(1)).rpcConnectionFailure(anyString(), eq(cause)); + verify(controller, times(1)).sendGlobalBarrierAcquire(eq(spy), + eq(procData), anyListOf(String.class)); + verify(controller, times(1)).sendGlobalBarrierReached(any(Procedure.class), + anyListOf(String.class)); + } + + @Test(timeout = 60000) + public void testNoCohort() throws Exception { + runSimpleProcedure(); + } + + @Test(timeout = 60000) + public void testSingleCohortOrchestration() throws Exception { + runSimpleProcedure("one"); + } + + @Test(timeout = 60000) + public void testMultipleCohortOrchestration() throws Exception { + runSimpleProcedure("one", "two", "three", "four"); + } + + public void runSimpleProcedure(String... members) throws Exception { + coordinator = buildNewCoordinator(); + Procedure task = new Procedure(coordinator, monitor, WAKE_FREQUENCY, + TIMEOUT, procName, procData, Arrays.asList(members)); + final Procedure spy = spy(task); + runCoordinatedProcedure(spy, members); + } + + /** + * Test that if nodes join the barrier early we still correctly handle the progress + */ + @Test(timeout = 60000) + public void testEarlyJoiningBarrier() throws Exception { + final String[] cohort = new String[] { "one", "two", "three", "four" }; + coordinator = buildNewCoordinator(); + final ProcedureCoordinator ref = coordinator; + Procedure task = new Procedure(coordinator, monitor, WAKE_FREQUENCY, + TIMEOUT, procName, procData, Arrays.asList(cohort)); + final Procedure spy = spy(task); + + AcquireBarrierAnswer prepare = new AcquireBarrierAnswer(procName, cohort) { + public void doWork() { + // then do some fun where we commit before all nodes have prepared + // "one" commits before anyone else is done + ref.memberAcquiredBarrier(this.opName, this.cohort[0]); + ref.memberFinishedBarrier(this.opName, this.cohort[0]); + // but "two" takes a while + ref.memberAcquiredBarrier(this.opName, this.cohort[1]); + // "three"jumps ahead + ref.memberAcquiredBarrier(this.opName, this.cohort[2]); + ref.memberFinishedBarrier(this.opName, this.cohort[2]); + // and "four" takes a while + ref.memberAcquiredBarrier(this.opName, this.cohort[3]); + } + }; + + BarrierAnswer commit = new BarrierAnswer(procName, cohort) { + @Override + public void doWork() { + ref.memberFinishedBarrier(opName, this.cohort[1]); + ref.memberFinishedBarrier(opName, this.cohort[3]); + } + }; + runCoordinatedOperation(spy, prepare, commit, cohort); + } + + /** + * Just run a procedure with the standard name and data, with not special task for the mock + * coordinator (it works just like a regular coordinator). For custom behavior see + * {@link #runCoordinatedOperation(Procedure, AcquireBarrierAnswer, BarrierAnswer, String[])} + * . + * @param spy Spy on a real {@link Procedure} + * @param cohort expected cohort members + * @throws Exception on failure + */ + public void runCoordinatedProcedure(Procedure spy, String... cohort) throws Exception { + runCoordinatedOperation(spy, new AcquireBarrierAnswer(procName, cohort), + new BarrierAnswer(procName, cohort), cohort); + } + + public void runCoordinatedOperation(Procedure spy, AcquireBarrierAnswer prepare, + String... cohort) throws Exception { + runCoordinatedOperation(spy, prepare, new BarrierAnswer(procName, cohort), cohort); + } + + public void runCoordinatedOperation(Procedure spy, BarrierAnswer commit, + String... cohort) throws Exception { + runCoordinatedOperation(spy, new AcquireBarrierAnswer(procName, cohort), commit, cohort); + } + + public void runCoordinatedOperation(Procedure spy, AcquireBarrierAnswer prepareOperation, + BarrierAnswer commitOperation, String... cohort) throws Exception { + List expected = Arrays.asList(cohort); + when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class))) + .thenReturn(spy); + + // use the passed controller responses + doAnswer(prepareOperation).when(controller).sendGlobalBarrierAcquire(spy, procData, expected); + doAnswer(commitOperation).when(controller) + .sendGlobalBarrierReached(eq(spy), anyListOf(String.class)); + + // run the operation + Procedure task = coordinator.startProcedure(spy.getErrorMonitor(), procName, procData, expected); + // and wait for it to finish + task.waitForCompleted(); + + // make sure we mocked correctly + prepareOperation.ensureRan(); + // we never got an exception + InOrder inorder = inOrder(spy, controller); + inorder.verify(spy).sendGlobalBarrierStart(); + inorder.verify(controller).sendGlobalBarrierAcquire(task, procData, expected); + inorder.verify(spy).sendGlobalBarrierReached(); + inorder.verify(controller).sendGlobalBarrierReached(eq(task), anyListOf(String.class)); + } + + private abstract class OperationAnswer implements Answer { + private boolean ran = false; + + public void ensureRan() { + assertTrue("Prepare mocking didn't actually run!", ran); + } + + @Override + public final Void answer(InvocationOnMock invocation) throws Throwable { + this.ran = true; + doWork(); + return null; + } + + protected abstract void doWork() throws Throwable; + } + + /** + * Just tell the current coordinator that each of the nodes has prepared + */ + private class AcquireBarrierAnswer extends OperationAnswer { + protected final String[] cohort; + protected final String opName; + + public AcquireBarrierAnswer(String opName, String... cohort) { + this.cohort = cohort; + this.opName = opName; + } + + @Override + public void doWork() { + if (cohort == null) return; + for (String member : cohort) { + TestProcedureCoordinator.this.coordinator.memberAcquiredBarrier(opName, member); + } + } + } + + /** + * Just tell the current coordinator that each of the nodes has committed + */ + private class BarrierAnswer extends OperationAnswer { + protected final String[] cohort; + protected final String opName; + + public BarrierAnswer(String opName, String... cohort) { + this.cohort = cohort; + this.opName = opName; + } + + @Override + public void doWork() { + if (cohort == null) return; + for (String member : cohort) { + TestProcedureCoordinator.this.coordinator.memberFinishedBarrier(opName, member); + } + } + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java new file mode 100644 index 0000000..5af2e40 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureMember.java @@ -0,0 +1,444 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.procedure; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.errorhandling.ForeignException; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.errorhandling.TimeoutException; +import org.apache.hadoop.hbase.procedure.Subprocedure.SubprocedureImpl; +import org.junit.After; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.InOrder; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** + * Test the procedure member, and it's error handling mechanisms. + */ +@Category(SmallTests.class) +public class TestProcedureMember { + private static final long WAKE_FREQUENCY = 100; + private static final long TIMEOUT = 100000; + private static final long POOL_KEEP_ALIVE = 1; + + private final String op = "some op"; + private final byte[] data = new byte[0]; + private final ForeignExceptionDispatcher mockListener = Mockito + .spy(new ForeignExceptionDispatcher()); + private final SubprocedureFactory mockBuilder = mock(SubprocedureFactory.class); + private final ProcedureMemberRpcs mockMemberComms = Mockito + .mock(ProcedureMemberRpcs.class); + private ProcedureMember member; + private ForeignExceptionDispatcher dispatcher; + Subprocedure spySub; + + /** + * Reset all the mock objects + */ + @After + public void resetTest() { + reset(mockListener, mockBuilder, mockMemberComms); + if (member != null) + try { + member.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * Build a member using the class level mocks + * @return member to use for tests + */ + private ProcedureMember buildCohortMember() { + String name = "node"; + ThreadPoolExecutor pool = ProcedureMember.defaultPool(WAKE_FREQUENCY, POOL_KEEP_ALIVE, 1, name); + return new ProcedureMember(mockMemberComms, pool, mockBuilder); + } + + /** + * Setup a procedure member that returns the spied-upon {@link Subprocedure}. + */ + private void buildCohortMemberPair() throws IOException { + dispatcher = new ForeignExceptionDispatcher(); + String name = "node"; + ThreadPoolExecutor pool = ProcedureMember.defaultPool(WAKE_FREQUENCY, POOL_KEEP_ALIVE, 1, name); + member = new ProcedureMember(mockMemberComms, pool, mockBuilder); + when(mockMemberComms.getMemberName()).thenReturn("membername"); // needed for generating exception + Subprocedure subproc = new EmptySubprocedure(member, dispatcher); + spySub = spy(subproc); + when(mockBuilder.buildSubprocedure(op, data)).thenReturn(spySub); + addCommitAnswer(); + } + + + /** + * Add a 'in barrier phase' response to the mock controller when it gets a acquired notification + */ + private void addCommitAnswer() throws IOException { + doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + member.receivedReachedGlobalBarrier(op); + return null; + } + }).when(mockMemberComms).sendMemberAcquired(any(Subprocedure.class)); + } + + /** + * Test the normal sub procedure execution case. + */ + @Test(timeout = 60000) + public void testSimpleRun() throws Exception { + member = buildCohortMember(); + EmptySubprocedure subproc = new EmptySubprocedure(member, mockListener); + EmptySubprocedure spy = spy(subproc); + when(mockBuilder.buildSubprocedure(op, data)).thenReturn(spy); + + // when we get a prepare, then start the commit phase + addCommitAnswer(); + + // run the operation + // build a new operation + Subprocedure subproc1 = member.createSubprocedure(op, data); + member.submitSubprocedure(subproc1); + // and wait for it to finish + subproc.waitForLocallyCompleted(); + + // make sure everything ran in order + InOrder order = inOrder(mockMemberComms, spy); + order.verify(spy).acquireBarrier(); + order.verify(mockMemberComms).sendMemberAcquired(eq(spy)); + order.verify(spy).insideBarrier(); + order.verify(mockMemberComms).sendMemberCompleted(eq(spy)); + order.verify(mockMemberComms, never()).sendMemberAborted(eq(spy), + any(ForeignException.class)); + } + + /** + * Make sure we call cleanup etc, when we have an exception during + * {@link Subprocedure#acquireBarrier()}. + */ + @Test(timeout = 60000) + public void testMemberPrepareException() throws Exception { + buildCohortMemberPair(); + + // mock an exception on Subprocedure's prepare + doAnswer( + new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + throw new IOException("Forced IOException in member acquireBarrier"); + } + }).when(spySub).acquireBarrier(); + + // run the operation + // build a new operation + Subprocedure subproc = member.createSubprocedure(op, data); + member.submitSubprocedure(subproc); + // if the operation doesn't die properly, then this will timeout + member.closeAndWait(TIMEOUT); + + // make sure everything ran in order + InOrder order = inOrder(mockMemberComms, spySub); + order.verify(spySub).acquireBarrier(); + // Later phases not run + order.verify(mockMemberComms, never()).sendMemberAcquired(eq(spySub)); + order.verify(spySub, never()).insideBarrier(); + order.verify(mockMemberComms, never()).sendMemberCompleted(eq(spySub)); + // error recovery path exercised + order.verify(spySub).cancel(anyString(), any(Exception.class)); + order.verify(spySub).cleanup(any(Exception.class)); + } + + /** + * Make sure we call cleanup etc, when we have an exception during prepare. + */ + @Test(timeout = 60000) + public void testSendMemberAcquiredCommsFailure() throws Exception { + buildCohortMemberPair(); + + // mock an exception on Subprocedure's prepare + doAnswer( + new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + throw new IOException("Forced IOException in memeber prepare"); + } + }).when(mockMemberComms).sendMemberAcquired(any(Subprocedure.class)); + + // run the operation + // build a new operation + Subprocedure subproc = member.createSubprocedure(op, data); + member.submitSubprocedure(subproc); + // if the operation doesn't die properly, then this will timeout + member.closeAndWait(TIMEOUT); + + // make sure everything ran in order + InOrder order = inOrder(mockMemberComms, spySub); + order.verify(spySub).acquireBarrier(); + order.verify(mockMemberComms).sendMemberAcquired(eq(spySub)); + + // Later phases not run + order.verify(spySub, never()).insideBarrier(); + order.verify(mockMemberComms, never()).sendMemberCompleted(eq(spySub)); + // error recovery path exercised + order.verify(spySub).cancel(anyString(), any(Exception.class)); + order.verify(spySub).cleanup(any(Exception.class)); + } + + /** + * Fail correctly if coordinator aborts the procedure. The subprocedure will not interrupt a + * running {@link Subprocedure#prepare} -- prepare needs to finish first, and the the abort + * is checked. Thus, the {@link Subprocedure#prepare} should succeed but later get rolled back + * via {@link Subprocedure#cleanup}. + */ + @Test(timeout = 60000) + public void testCoordinatorAbort() throws Exception { + buildCohortMemberPair(); + + // mock that another node timed out or failed to prepare + final TimeoutException oate = new TimeoutException("bogus timeout", 1,2,0); + doAnswer( + new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + // inject a remote error (this would have come from an external thread) + spySub.cancel("bogus message", oate); + // sleep the wake frequency since that is what we promised + Thread.sleep(WAKE_FREQUENCY); + return null; + } + }).when(spySub).waitForReachedGlobalBarrier(); + + // run the operation + // build a new operation + Subprocedure subproc = member.createSubprocedure(op, data); + member.submitSubprocedure(subproc); + // if the operation doesn't die properly, then this will timeout + member.closeAndWait(TIMEOUT); + + // make sure everything ran in order + InOrder order = inOrder(mockMemberComms, spySub); + order.verify(spySub).acquireBarrier(); + order.verify(mockMemberComms).sendMemberAcquired(eq(spySub)); + // Later phases not run + order.verify(spySub, never()).insideBarrier(); + order.verify(mockMemberComms, never()).sendMemberCompleted(eq(spySub)); + // error recovery path exercised + order.verify(spySub).cancel(anyString(), any(Exception.class)); + order.verify(spySub).cleanup(any(Exception.class)); + } + + /** + * Handle failures if a member's commit phase fails. + * + * NOTE: This is the core difference that makes this different from traditional 2PC. In true + * 2PC the transaction is committed just before the coordinator sends commit messages to the + * member. Members are then responsible for reading its TX log. This implementation actually + * rolls back, and thus breaks the normal TX guarantees. + */ + @Test(timeout = 60000) + public void testMemberCommitException() throws Exception { + buildCohortMemberPair(); + + // mock an exception on Subprocedure's prepare + doAnswer( + new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + throw new IOException("Forced IOException in memeber prepare"); + } + }).when(spySub).insideBarrier(); + + // run the operation + // build a new operation + Subprocedure subproc = member.createSubprocedure(op, data); + member.submitSubprocedure(subproc); + // if the operation doesn't die properly, then this will timeout + member.closeAndWait(TIMEOUT); + + // make sure everything ran in order + InOrder order = inOrder(mockMemberComms, spySub); + order.verify(spySub).acquireBarrier(); + order.verify(mockMemberComms).sendMemberAcquired(eq(spySub)); + order.verify(spySub).insideBarrier(); + + // Later phases not run + order.verify(mockMemberComms, never()).sendMemberCompleted(eq(spySub)); + // error recovery path exercised + order.verify(spySub).cancel(anyString(), any(Exception.class)); + order.verify(spySub).cleanup(any(Exception.class)); + } + + /** + * Handle Failures if a member's commit phase succeeds but notification to coordinator fails + * + * NOTE: This is the core difference that makes this different from traditional 2PC. In true + * 2PC the transaction is committed just before the coordinator sends commit messages to the + * member. Members are then responsible for reading its TX log. This implementation actually + * rolls back, and thus breaks the normal TX guarantees. + */ + @Test(timeout = 60000) + public void testMemberCommitCommsFailure() throws Exception { + buildCohortMemberPair(); + final TimeoutException oate = new TimeoutException("bogus timeout",1,2,0); + doAnswer( + new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + // inject a remote error (this would have come from an external thread) + spySub.cancel("commit comms fail", oate); + // sleep the wake frequency since that is what we promised + Thread.sleep(WAKE_FREQUENCY); + return null; + } + }).when(mockMemberComms).sendMemberCompleted(any(Subprocedure.class)); + + // run the operation + // build a new operation + Subprocedure subproc = member.createSubprocedure(op, data); + member.submitSubprocedure(subproc); + // if the operation doesn't die properly, then this will timeout + member.closeAndWait(TIMEOUT); + + // make sure everything ran in order + InOrder order = inOrder(mockMemberComms, spySub); + order.verify(spySub).acquireBarrier(); + order.verify(mockMemberComms).sendMemberAcquired(eq(spySub)); + order.verify(spySub).insideBarrier(); + order.verify(mockMemberComms).sendMemberCompleted(eq(spySub)); + // error recovery path exercised + order.verify(spySub).cancel(anyString(), any(Exception.class)); + order.verify(spySub).cleanup(any(Exception.class)); + } + + /** + * Fail correctly on getting an external error while waiting for the prepared latch + * @throws Exception on failure + */ + @Test(timeout = 1000) + public void testPropagateConnectionErrorBackToManager() throws Exception { + // setup the operation + member = buildCohortMember(); + ProcedureMember memberSpy = spy(member); + + // setup the commit and the spy + final ForeignExceptionDispatcher dispatcher = new ForeignExceptionDispatcher(); + ForeignExceptionDispatcher dispSpy = spy(dispatcher); + Subprocedure commit = new EmptySubprocedure(member, dispatcher); + Subprocedure spy = spy(commit); + when(mockBuilder.buildSubprocedure(op, data)).thenReturn(spy); + + // fail during the prepare phase + doThrow(new ForeignException("SRC", "prepare exception")).when(spy).acquireBarrier(); + // and throw a connection error when we try to tell the controller about it + doThrow(new IOException("Controller is down!")).when(mockMemberComms) + .sendMemberAborted(eq(spy), any(ForeignException.class)); + + + // run the operation + // build a new operation + Subprocedure subproc = memberSpy.createSubprocedure(op, data); + memberSpy.submitSubprocedure(subproc); + // if the operation doesn't die properly, then this will timeout + memberSpy.closeAndWait(TIMEOUT); + + // make sure everything ran in order + InOrder order = inOrder(mockMemberComms, spy, dispSpy); + // make sure we acquire. + order.verify(spy).acquireBarrier(); + order.verify(mockMemberComms, never()).sendMemberAcquired(spy); + + // TODO Need to do another refactor to get this to propagate to the coordinator. + // make sure we pass a remote exception back the controller +// order.verify(mockMemberComms).sendMemberAborted(eq(spy), +// any(ExternalException.class)); +// order.verify(dispSpy).receiveError(anyString(), +// any(ExternalException.class), any()); + } + + /** + * Test that the cohort member correctly doesn't attempt to start a task when the builder cannot + * correctly build a new task for the requested operation + * @throws Exception on failure + */ + @Test + public void testNoTaskToBeRunFromRequest() throws Exception { + ThreadPoolExecutor pool = mock(ThreadPoolExecutor.class); + when(mockBuilder.buildSubprocedure(op, data)).thenReturn(null) + .thenThrow(new IllegalStateException("Wrong state!"), new IllegalArgumentException("can't understand the args")); + member = new ProcedureMember(mockMemberComms, pool, mockBuilder); + // builder returns null + // build a new operation + Subprocedure subproc = member.createSubprocedure(op, data); + member.submitSubprocedure(subproc); + // throws an illegal state exception + try { + // build a new operation + Subprocedure subproc2 = member.createSubprocedure(op, data); + member.submitSubprocedure(subproc2); + } catch (IllegalStateException ise) { + } + // throws an illegal argument exception + try { + // build a new operation + Subprocedure subproc3 = member.createSubprocedure(op, data); + member.submitSubprocedure(subproc3); + } catch (IllegalArgumentException iae) { + } + + // no request should reach the pool + verifyZeroInteractions(pool); + // get two abort requests + // TODO Need to do another refactor to get this to propagate to the coordinator. + // verify(mockMemberComms, times(2)).sendMemberAborted(any(Subprocedure.class), any(ExternalException.class)); + } + + /** + * Helper {@link Procedure} who's phase for each step is just empty + */ + public class EmptySubprocedure extends SubprocedureImpl { + public EmptySubprocedure(ProcedureMember member, ForeignExceptionDispatcher dispatcher) { + super( member, op, dispatcher, + // TODO 1000000 is an arbitrary number that I picked. + WAKE_FREQUENCY, TIMEOUT); + } + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java new file mode 100644 index 0000000..a0fc549 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java @@ -0,0 +1,405 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.procedure; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyListOf; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.errorhandling.ForeignException; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.errorhandling.TimeoutException; +import org.apache.hadoop.hbase.procedure.Subprocedure.SubprocedureImpl; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.mockito.internal.matchers.ArrayEquals; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.mockito.verification.VerificationMode; + +import com.google.common.collect.Lists; + +/** + * Cluster-wide testing of a distributed three-phase commit using a 'real' zookeeper cluster + */ +@Category(MediumTests.class) +public class TestZKProcedure { + + private static final Log LOG = LogFactory.getLog(TestZKProcedure.class); + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final String COORDINATOR_NODE_NAME = "coordinator"; + private static final long KEEP_ALIVE = 100; // seconds + private static final int POOL_SIZE = 1; + private static final long TIMEOUT = 10000; // when debugging make this larger for debugging + private static final long WAKE_FREQUENCY = 500; + private static final String opName = "op"; + private static final byte[] data = new byte[] { 1, 2 }; // TODO what is this used for? + private static final VerificationMode once = Mockito.times(1); + + @BeforeClass + public static void setupTest() throws Exception { + UTIL.startMiniZKCluster(); + } + + @AfterClass + public static void cleanupTest() throws Exception { + UTIL.shutdownMiniZKCluster(); + } + + private static ZooKeeperWatcher newZooKeeperWatcher() throws IOException { + return new ZooKeeperWatcher(UTIL.getConfiguration(), "testing utility", new Abortable() { + @Override + public void abort(String why, Throwable e) { + throw new RuntimeException( + "Unexpected abort in distributed three phase commit test:" + why, e); + } + + @Override + public boolean isAborted() { + return false; + } + }); + } + + @Test + public void testEmptyMemberSet() throws Exception { + runCommit(); + } + + @Test + public void testSingleMember() throws Exception { + runCommit("one"); + } + + @Test + public void testMultipleMembers() throws Exception { + runCommit("one", "two", "three", "four" ); + } + + private void runCommit(String... members) throws Exception { + // make sure we just have an empty list + if (members == null) { + members = new String[0]; + } + List expected = Arrays.asList(members); + + // setup the constants + ZooKeeperWatcher coordZkw = newZooKeeperWatcher(); + String opDescription = "coordination test - " + members.length + " cohort members"; + + // start running the controller + ZKProcedureCoordinatorRpcs coordinatorComms = new ZKProcedureCoordinatorRpcs( + coordZkw, opDescription, COORDINATOR_NODE_NAME); + ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(COORDINATOR_NODE_NAME, KEEP_ALIVE, POOL_SIZE, WAKE_FREQUENCY); + ProcedureCoordinator coordinator = new ProcedureCoordinator(coordinatorComms, pool) { + @Override + public Procedure createProcedure(ForeignExceptionDispatcher fed, String procName, byte[] procArgs, + List expectedMembers) { + return Mockito.spy(super.createProcedure(fed, procName, procArgs, expectedMembers)); + } + }; + + // build and start members + // NOTE: There is a single subprocedure builder for all members here. + SubprocedureFactory subprocFactory = Mockito.mock(SubprocedureFactory.class); + List> procMembers = new ArrayList>( + members.length); + // start each member + for (String member : members) { + ZooKeeperWatcher watcher = newZooKeeperWatcher(); + ZKProcedureMemberRpcs comms = new ZKProcedureMemberRpcs(watcher, opDescription, member); + ThreadPoolExecutor pool2 = ProcedureMember.defaultPool(WAKE_FREQUENCY, KEEP_ALIVE, 1, member); + ProcedureMember procMember = new ProcedureMember(comms, pool2, subprocFactory); + procMembers.add(new Pair(procMember, comms)); + comms.start(procMember); + } + + // setup mock member subprocedures + final List subprocs = new ArrayList(); + for (int i = 0; i < procMembers.size(); i++) { + ForeignExceptionDispatcher cohortMonitor = new ForeignExceptionDispatcher(); + Subprocedure commit = Mockito + .spy(new SubprocedureImpl(procMembers.get(i).getFirst(), opName, cohortMonitor, + WAKE_FREQUENCY, TIMEOUT)); + subprocs.add(commit); + } + + // link subprocedure to buildNewOperation invocation. + final AtomicInteger i = new AtomicInteger(0); // NOTE: would be racy if not an AtomicInteger + Mockito.when(subprocFactory.buildSubprocedure(Mockito.eq(opName), + (byte[]) Mockito.argThat(new ArrayEquals(data)))).thenAnswer( + new Answer() { + @Override + public Subprocedure answer(InvocationOnMock invocation) throws Throwable { + int index = i.getAndIncrement(); + LOG.debug("Task size:" + subprocs.size() + ", getting:" + index); + Subprocedure commit = subprocs.get(index); + return commit; + } + }); + + // setup spying on the coordinator +// Procedure proc = Mockito.spy(procBuilder.createProcedure(coordinator, opName, data, expected)); +// Mockito.when(procBuilder.build(coordinator, opName, data, expected)).thenReturn(proc); + + // start running the operation + Procedure task = coordinator.startProcedure(new ForeignExceptionDispatcher(), opName, data, expected); +// assertEquals("Didn't mock coordinator task", proc, task); + + // verify all things ran as expected +// waitAndVerifyProc(proc, once, once, never(), once, false); + waitAndVerifyProc(task, once, once, never(), once, false); + verifyCohortSuccessful(expected, subprocFactory, subprocs, once, once, never(), once, false); + + // close all the things + closeAll(coordinator, coordinatorComms, procMembers); + } + + /** + * Test a distributed commit with multiple cohort members, where one of the cohort members has a + * timeout exception during the prepare stage. + */ + @Test + public void testMultiCohortWithMemberTimeoutDuringPrepare() throws Exception { + String opDescription = "error injection coordination"; + String[] cohortMembers = new String[] { "one", "two", "three" }; + List expected = Lists.newArrayList(cohortMembers); + // error constants + final int memberErrorIndex = 2; + final CountDownLatch coordinatorReceivedErrorLatch = new CountDownLatch(1); + + // start running the coordinator and its controller + ZooKeeperWatcher coordinatorWatcher = newZooKeeperWatcher(); + ZKProcedureCoordinatorRpcs coordinatorController = new ZKProcedureCoordinatorRpcs( + coordinatorWatcher, opDescription, COORDINATOR_NODE_NAME); + ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(COORDINATOR_NODE_NAME, KEEP_ALIVE, POOL_SIZE, WAKE_FREQUENCY); + ProcedureCoordinator coordinator = spy(new ProcedureCoordinator(coordinatorController, pool)); + + // start a member for each node + SubprocedureFactory subprocFactory = Mockito.mock(SubprocedureFactory.class); + List> members = new ArrayList>( + expected.size()); + for (String member : expected) { + ZooKeeperWatcher watcher = newZooKeeperWatcher(); + ZKProcedureMemberRpcs controller = new ZKProcedureMemberRpcs(watcher, opDescription, member); + ThreadPoolExecutor pool2 = ProcedureMember.defaultPool(WAKE_FREQUENCY, KEEP_ALIVE, 1, member); + ProcedureMember mem = new ProcedureMember(controller, pool2, subprocFactory); + members.add(new Pair(mem, controller)); + controller.start(mem); + } + + // setup mock subprocedures + final List cohortTasks = new ArrayList(); + final int[] elem = new int[1]; + for (int i = 0; i < members.size(); i++) { + ForeignExceptionDispatcher cohortMonitor = new ForeignExceptionDispatcher(); + ProcedureMember comms = members.get(i).getFirst(); + Subprocedure commit = Mockito + .spy(new SubprocedureImpl(comms, opName, cohortMonitor, WAKE_FREQUENCY, TIMEOUT)); + // This nasty bit has one of the impls throw a TimeoutException + Mockito.doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + int index = elem[0]; + if (index == memberErrorIndex) { + LOG.debug("Sending error to coordinator"); + ForeignException remoteCause = new ForeignException("TIMER", + new TimeoutException("subprocTimeout" , 1, 2, 0)); + Subprocedure r = ((Subprocedure) invocation.getMock()); + LOG.error("Remote commit failure, not propagating error:" + remoteCause); + r.monitor.receive(remoteCause); + // don't complete the error phase until the coordinator has gotten the error + // notification (which ensures that we never progress past prepare) + try { + Procedure.waitForLatch(coordinatorReceivedErrorLatch, new ForeignExceptionDispatcher(), + WAKE_FREQUENCY, "coordinator received error"); + } catch (InterruptedException e) { + LOG.debug("Wait for latch interrupted, done:" + (coordinatorReceivedErrorLatch.getCount() == 0)); + // reset the interrupt status on the thread + Thread.currentThread().interrupt(); + } + } + elem[0] = ++index; + return null; + } + }).when(commit).acquireBarrier(); + cohortTasks.add(commit); + } + + // pass out a task per member + final int[] i = new int[] { 0 }; + Mockito.when( + subprocFactory.buildSubprocedure(Mockito.eq(opName), + (byte[]) Mockito.argThat(new ArrayEquals(data)))).thenAnswer( + new Answer() { + @Override + public Subprocedure answer(InvocationOnMock invocation) throws Throwable { + int index = i[0]; + Subprocedure commit = cohortTasks.get(index); + index++; + i[0] = index; + return commit; + } + }); + + // setup spying on the coordinator + ForeignExceptionDispatcher coordinatorTaskErrorMonitor = Mockito + .spy(new ForeignExceptionDispatcher()); + Procedure coordinatorTask = Mockito.spy(new Procedure(coordinator, + coordinatorTaskErrorMonitor, WAKE_FREQUENCY, TIMEOUT, + opName, data, expected)); + when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(opName), eq(data), anyListOf(String.class))) + .thenReturn(coordinatorTask); + // count down the error latch when we get the remote error + Mockito.doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + // pass on the error to the master + invocation.callRealMethod(); + // then count down the got error latch + coordinatorReceivedErrorLatch.countDown(); + return null; + } + }).when(coordinatorTask).receive(Mockito.any(ForeignException.class)); + + // ---------------------------- + // start running the operation + // ---------------------------- + + Procedure task = coordinator.startProcedure(coordinatorTaskErrorMonitor, opName, data, expected); + assertEquals("Didn't mock coordinator task", coordinatorTask, task); + + // wait for the task to complete + try { + task.waitForCompleted(); + } catch (ForeignException fe) { + // this may get caught or may not + } + + // ------------- + // verification + // ------------- + waitAndVerifyProc(coordinatorTask, once, never(), once, once, true); + verifyCohortSuccessful(expected, subprocFactory, cohortTasks, once, never(), once, + once, true); + + // close all the open things + closeAll(coordinator, coordinatorController, members); + } + + /** + * Wait for the coordinator task to complete, and verify all the mocks + * @param task to wait on + * @throws Exception on unexpected failure + */ + private void waitAndVerifyProc(Procedure proc, VerificationMode prepare, + VerificationMode commit, VerificationMode cleanup, VerificationMode finish, boolean opHasError) + throws Exception { + boolean caughtError = false; + try { + proc.waitForCompleted(); + } catch (ForeignException fe) { + caughtError = true; + } + // make sure that the task called all the expected phases + Mockito.verify(proc, prepare).sendGlobalBarrierStart(); + Mockito.verify(proc, commit).sendGlobalBarrierReached(); + Mockito.verify(proc, finish).sendGlobalBarrierComplete(); + assertEquals("Operation error state was unexpected", opHasError, proc.getErrorMonitor() + .hasException()); + assertEquals("Operation error state was unexpected", opHasError, caughtError); + + } + + /** + * Wait for the coordinator task to complete, and verify all the mocks + * @param task to wait on + * @throws Exception on unexpected failure + */ + private void waitAndVerifySubproc(Subprocedure op, VerificationMode prepare, + VerificationMode commit, VerificationMode cleanup, VerificationMode finish, boolean opHasError) + throws Exception { + boolean caughtError = false; + try { + op.waitForLocallyCompleted(); + } catch (ForeignException fe) { + caughtError = true; + } + // make sure that the task called all the expected phases + Mockito.verify(op, prepare).acquireBarrier(); + Mockito.verify(op, commit).insideBarrier(); + // We cannot guarantee that cleanup has run so we don't check it. + + assertEquals("Operation error state was unexpected", opHasError, op.getErrorCheckable() + .hasException()); + assertEquals("Operation error state was unexpected", opHasError, caughtError); + + } + + private void verifyCohortSuccessful(List cohortNames, + SubprocedureFactory subprocFactory, Iterable cohortTasks, + VerificationMode prepare, VerificationMode commit, VerificationMode cleanup, + VerificationMode finish, boolean opHasError) throws Exception { + + // make sure we build the correct number of cohort members + Mockito.verify(subprocFactory, Mockito.times(cohortNames.size())).buildSubprocedure( + Mockito.eq(opName), (byte[]) Mockito.argThat(new ArrayEquals(data))); + // verify that we ran each of the operations cleanly + int j = 0; + for (Subprocedure op : cohortTasks) { + LOG.debug("Checking mock:" + (j++)); + waitAndVerifySubproc(op, prepare, commit, cleanup, finish, opHasError); + } + } + + private void closeAll( + ProcedureCoordinator coordinator, + ZKProcedureCoordinatorRpcs coordinatorController, + List> cohort) + throws IOException { + // make sure we close all the resources + for (Pair member : cohort) { + member.getFirst().close(); + member.getSecond().close(); + } + coordinator.close(); + coordinatorController.close(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java new file mode 100644 index 0000000..745d75a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedureControllers.java @@ -0,0 +1,429 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.procedure; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.errorhandling.ForeignException; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.mockito.verification.VerificationMode; + +import com.google.common.collect.Lists; + +/** + * Test zookeeper-based, procedure controllers + */ +@Category(MediumTests.class) +public class TestZKProcedureControllers { + + static final Log LOG = LogFactory.getLog(TestZKProcedureControllers.class); + private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final String COHORT_NODE_NAME = "expected"; + private static final String CONTROLLER_NODE_NAME = "controller"; + private static final VerificationMode once = Mockito.times(1); + + @BeforeClass + public static void setupTest() throws Exception { + UTIL.startMiniZKCluster(); + } + + @AfterClass + public static void cleanupTest() throws Exception { + UTIL.shutdownMiniZKCluster(); + } + + /** + * Smaller test to just test the actuation on the cohort member + * @throws Exception on failure + */ + @Test(timeout = 60000) + public void testSimpleZKCohortMemberController() throws Exception { + ZooKeeperWatcher watcher = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + final String operationName = "instanceTest"; + + final Subprocedure sub = Mockito.mock(Subprocedure.class); + Mockito.when(sub.getName()).thenReturn(operationName); + + final byte[] data = new byte[] { 1, 2, 3 }; + final CountDownLatch prepared = new CountDownLatch(1); + final CountDownLatch committed = new CountDownLatch(1); + + final ForeignExceptionDispatcher monitor = spy(new ForeignExceptionDispatcher()); + final ZKProcedureMemberRpcs controller = new ZKProcedureMemberRpcs( + watcher, "testSimple", COHORT_NODE_NAME); + + // mock out cohort member callbacks + final ProcedureMember member = Mockito + .mock(ProcedureMember.class); + Mockito.doReturn(sub).when(member).createSubprocedure(operationName, data); + Mockito.doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + controller.sendMemberAcquired(sub); + prepared.countDown(); + return null; + } + }).when(member).submitSubprocedure(sub); + Mockito.doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + controller.sendMemberCompleted(sub); + committed.countDown(); + return null; + } + }).when(member).receivedReachedGlobalBarrier(operationName); + + // start running the listener + controller.start(member); + + // set a prepare node from a 'coordinator' + String prepare = ZKProcedureUtil.getAcquireBarrierNode(controller.getZkController(), operationName); + ZKUtil.createSetData(watcher, prepare, ProtobufUtil.prependPBMagic(data)); + // wait for the operation to be prepared + prepared.await(); + + // create the commit node so we update the operation to enter the commit phase + String commit = ZKProcedureUtil.getReachedBarrierNode(controller.getZkController(), operationName); + LOG.debug("Found prepared, posting commit node:" + commit); + ZKUtil.createAndFailSilent(watcher, commit); + LOG.debug("Commit node:" + commit + ", exists:" + ZKUtil.checkExists(watcher, commit)); + committed.await(); + + verify(monitor, never()).receive(Mockito.any(ForeignException.class)); + // XXX: broken due to composition. +// verify(member, never()).getManager().controllerConnectionFailure(Mockito.anyString(), +// Mockito.any(IOException.class)); + // cleanup after the test + ZKUtil.deleteNodeRecursively(watcher, controller.getZkController().getBaseZnode()); + assertEquals("Didn't delete prepare node", -1, ZKUtil.checkExists(watcher, prepare)); + assertEquals("Didn't delete commit node", -1, ZKUtil.checkExists(watcher, commit)); + } + + @Test(timeout = 60000) + public void testZKCoordinatorControllerWithNoCohort() throws Exception { + final String operationName = "no cohort controller test"; + final byte[] data = new byte[] { 1, 2, 3 }; + + runMockCommitWithOrchestratedControllers(startCoordinatorFirst, operationName, data); + runMockCommitWithOrchestratedControllers(startCohortFirst, operationName, data); + } + + @Test(timeout = 60000) + public void testZKCoordinatorControllerWithSingleMemberCohort() throws Exception { + final String operationName = "single member controller test"; + final byte[] data = new byte[] { 1, 2, 3 }; + + runMockCommitWithOrchestratedControllers(startCoordinatorFirst, operationName, data, "cohort"); + runMockCommitWithOrchestratedControllers(startCohortFirst, operationName, data, "cohort"); + } + + @Test(timeout = 60000) + public void testZKCoordinatorControllerMultipleCohort() throws Exception { + final String operationName = "multi member controller test"; + final byte[] data = new byte[] { 1, 2, 3 }; + + runMockCommitWithOrchestratedControllers(startCoordinatorFirst, operationName, data, "cohort", + "cohort2", "cohort3"); + runMockCommitWithOrchestratedControllers(startCohortFirst, operationName, data, "cohort", + "cohort2", "cohort3"); + } + + private void runMockCommitWithOrchestratedControllers(StartControllers controllers, + String operationName, byte[] data, String... cohort) throws Exception { + ZooKeeperWatcher watcher = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + List expected = Lists.newArrayList(cohort); + + final Subprocedure sub = Mockito.mock(Subprocedure.class); + Mockito.when(sub.getName()).thenReturn(operationName); + + CountDownLatch prepared = new CountDownLatch(expected.size()); + CountDownLatch committed = new CountDownLatch(expected.size()); + // mock out coordinator so we can keep track of zk progress + ProcedureCoordinator coordinator = setupMockCoordinator(operationName, + prepared, committed); + + ProcedureMember member = Mockito.mock(ProcedureMember.class); + + Pair> pair = controllers + .start(watcher, operationName, coordinator, CONTROLLER_NODE_NAME, member, expected); + ZKProcedureCoordinatorRpcs controller = pair.getFirst(); + List cohortControllers = pair.getSecond(); + // start the operation + Procedure p = Mockito.mock(Procedure.class); + Mockito.when(p.getName()).thenReturn(operationName); + + controller.sendGlobalBarrierAcquire(p, data, expected); + + // post the prepare node for each expected node + for (ZKProcedureMemberRpcs cc : cohortControllers) { + cc.sendMemberAcquired(sub); + } + + // wait for all the notifications to reach the coordinator + prepared.await(); + // make sure we got the all the nodes and no more + Mockito.verify(coordinator, times(expected.size())).memberAcquiredBarrier(Mockito.eq(operationName), + Mockito.anyString()); + + // kick off the commit phase + controller.sendGlobalBarrierReached(p, expected); + + // post the committed node for each expected node + for (ZKProcedureMemberRpcs cc : cohortControllers) { + cc.sendMemberCompleted(sub); + } + + // wait for all commit notifications to reach the coordinator + committed.await(); + // make sure we got the all the nodes and no more + Mockito.verify(coordinator, times(expected.size())).memberFinishedBarrier(Mockito.eq(operationName), + Mockito.anyString()); + + controller.resetMembers(p); + + // verify all behavior + verifyZooKeeperClean(operationName, watcher, controller.getZkProcedureUtil()); + verifyCohort(member, cohortControllers.size(), operationName, data); + verifyCoordinator(operationName, coordinator, expected); + } + + // TODO Broken by composition. +// @Test +// public void testCoordinatorControllerHandlesEarlyPrepareNodes() throws Exception { +// runEarlyPrepareNodes(startCoordinatorFirst, "testEarlyPreparenodes", new byte[] { 1, 2, 3 }, +// "cohort1", "cohort2"); +// runEarlyPrepareNodes(startCohortFirst, "testEarlyPreparenodes", new byte[] { 1, 2, 3 }, +// "cohort1", "cohort2"); +// } + + public void runEarlyPrepareNodes(StartControllers controllers, String operationName, byte[] data, + String... cohort) throws Exception { + ZooKeeperWatcher watcher = HBaseTestingUtility.getZooKeeperWatcher(UTIL); + List expected = Lists.newArrayList(cohort); + + final Subprocedure sub = Mockito.mock(Subprocedure.class); + Mockito.when(sub.getName()).thenReturn(operationName); + + final CountDownLatch prepared = new CountDownLatch(expected.size()); + final CountDownLatch committed = new CountDownLatch(expected.size()); + // mock out coordinator so we can keep track of zk progress + ProcedureCoordinator coordinator = setupMockCoordinator(operationName, + prepared, committed); + + ProcedureMember member = Mockito.mock(ProcedureMember.class); + Procedure p = Mockito.mock(Procedure.class); + Mockito.when(p.getName()).thenReturn(operationName); + + Pair> pair = controllers + .start(watcher, operationName, coordinator, CONTROLLER_NODE_NAME, member, expected); + ZKProcedureCoordinatorRpcs controller = pair.getFirst(); + List cohortControllers = pair.getSecond(); + + // post 1/2 the prepare nodes early + for (int i = 0; i < cohortControllers.size() / 2; i++) { + cohortControllers.get(i).sendMemberAcquired(sub); + } + + // start the operation + controller.sendGlobalBarrierAcquire(p, data, expected); + + // post the prepare node for each expected node + for (ZKProcedureMemberRpcs cc : cohortControllers) { + cc.sendMemberAcquired(sub); + } + + // wait for all the notifications to reach the coordinator + prepared.await(); + // make sure we got the all the nodes and no more + Mockito.verify(coordinator, times(expected.size())).memberAcquiredBarrier(Mockito.eq(operationName), + Mockito.anyString()); + + // kick off the commit phase + controller.sendGlobalBarrierReached(p, expected); + + // post the committed node for each expected node + for (ZKProcedureMemberRpcs cc : cohortControllers) { + cc.sendMemberCompleted(sub); + } + + // wait for all commit notifications to reach the coordiantor + committed.await(); + // make sure we got the all the nodes and no more + Mockito.verify(coordinator, times(expected.size())).memberFinishedBarrier(Mockito.eq(operationName), + Mockito.anyString()); + + controller.resetMembers(p); + + // verify all behavior + verifyZooKeeperClean(operationName, watcher, controller.getZkProcedureUtil()); + verifyCohort(member, cohortControllers.size(), operationName, data); + verifyCoordinator(operationName, coordinator, expected); + } + + /** + * @return a mock {@link ProcedureCoordinator} that just counts down the + * prepared and committed latch for called to the respective method + */ + private ProcedureCoordinator setupMockCoordinator(String operationName, + final CountDownLatch prepared, final CountDownLatch committed) { + ProcedureCoordinator coordinator = Mockito + .mock(ProcedureCoordinator.class); + Mockito.mock(ProcedureCoordinator.class); + Mockito.doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + prepared.countDown(); + return null; + } + }).when(coordinator).memberAcquiredBarrier(Mockito.eq(operationName), Mockito.anyString()); + Mockito.doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + committed.countDown(); + return null; + } + }).when(coordinator).memberFinishedBarrier(Mockito.eq(operationName), Mockito.anyString()); + return coordinator; + } + + /** + * Verify that the prepare, commit and abort nodes for the operation are removed from zookeeper + */ + private void verifyZooKeeperClean(String operationName, ZooKeeperWatcher watcher, + ZKProcedureUtil controller) throws Exception { + String prepare = ZKProcedureUtil.getAcquireBarrierNode(controller, operationName); + String commit = ZKProcedureUtil.getReachedBarrierNode(controller, operationName); + String abort = ZKProcedureUtil.getAbortNode(controller, operationName); + assertEquals("Didn't delete prepare node", -1, ZKUtil.checkExists(watcher, prepare)); + assertEquals("Didn't delete commit node", -1, ZKUtil.checkExists(watcher, commit)); + assertEquals("Didn't delete abort node", -1, ZKUtil.checkExists(watcher, abort)); + } + + /** + * Verify the cohort controller got called once per expected node to start the operation + */ + private void verifyCohort(ProcedureMember member, int cohortSize, + String operationName, byte[] data) { +// verify(member, Mockito.times(cohortSize)).submitSubprocedure(Mockito.eq(operationName), +// (byte[]) Mockito.argThat(new ArrayEquals(data))); + verify(member, Mockito.times(cohortSize)).submitSubprocedure(Mockito.any(Subprocedure.class)); + + } + + /** + * Verify that the coordinator only got called once for each expected node + */ + private void verifyCoordinator(String operationName, + ProcedureCoordinator coordinator, List expected) { + // verify that we got all the expected nodes + for (String node : expected) { + verify(coordinator, once).memberAcquiredBarrier(operationName, node); + verify(coordinator, once).memberFinishedBarrier(operationName, node); + } + } + + /** + * Specify how the controllers that should be started (not spy/mockable) for the test. + */ + private abstract class StartControllers { + public abstract Pair> start( + ZooKeeperWatcher watcher, String operationName, + ProcedureCoordinator coordinator, String controllerName, + ProcedureMember member, List cohortNames) throws Exception; + } + + private final StartControllers startCoordinatorFirst = new StartControllers() { + + @Override + public Pair> start( + ZooKeeperWatcher watcher, String operationName, + ProcedureCoordinator coordinator, String controllerName, + ProcedureMember member, List expected) throws Exception { + // start the controller + ZKProcedureCoordinatorRpcs controller = new ZKProcedureCoordinatorRpcs( + watcher, operationName, CONTROLLER_NODE_NAME); + controller.start(coordinator); + + // make a cohort controller for each expected node + + List cohortControllers = new ArrayList(); + for (String nodeName : expected) { + ZKProcedureMemberRpcs cc = new ZKProcedureMemberRpcs( + watcher, operationName, nodeName); + cc.start(member); + cohortControllers.add(cc); + } + return new Pair>( + controller, cohortControllers); + } + }; + + /** + * Check for the possible race condition where a cohort member starts after the controller and + * therefore could miss a new operation + */ + private final StartControllers startCohortFirst = new StartControllers() { + + @Override + public Pair> start( + ZooKeeperWatcher watcher, String operationName, + ProcedureCoordinator coordinator, String controllerName, + ProcedureMember member, List expected) throws Exception { + + // make a cohort controller for each expected node + List cohortControllers = new ArrayList(); + for (String nodeName : expected) { + ZKProcedureMemberRpcs cc = new ZKProcedureMemberRpcs( + watcher, operationName, nodeName); + cc.start(member); + cohortControllers.add(cc); + } + + // start the controller + ZKProcedureCoordinatorRpcs controller = new ZKProcedureCoordinatorRpcs( + watcher, operationName, CONTROLLER_NODE_NAME); + controller.start(coordinator); + + return new Pair>( + controller, cohortControllers); + } + }; +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/CheckedArchivingHFileCleaner.java b/src/test/java/org/apache/hadoop/hbase/regionserver/CheckedArchivingHFileCleaner.java new file mode 100644 index 0000000..efc9cb6 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/CheckedArchivingHFileCleaner.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate; + +/** + * HFile archive cleaner that just tells you if it has been run already or not (and allows resets) - + * always attempts to delete the passed file. + *

    + * Just a helper class for testing to make sure the cleaner has been run. + */ +public class CheckedArchivingHFileCleaner extends BaseHFileCleanerDelegate { + + private static boolean checked; + + @Override + public boolean isFileDeletable(Path file) { + checked = true; + return true; + } + + public static boolean getChecked() { + return checked; + } + + public static void resetCheck() { + checked = false; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java new file mode 100644 index 0000000..f1cd8a0 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java @@ -0,0 +1,308 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Random; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.util.BloomFilterFactory; +import org.apache.hadoop.io.BytesWritable; + +/** + * Creates an HFile with random key/value pairs. + */ +public class CreateRandomStoreFile { + + /** + * As much as this number of bytes can be added or subtracted from key/value + * lengths. + */ + private static final int LEN_VARIATION = 5; + + private static final Log LOG = + LogFactory.getLog(CreateRandomStoreFile.class); + private static final String OUTPUT_DIR_OPTION = "o"; + private static final String NUM_KV_OPTION = "n"; + private static final String HFILE_VERSION_OPTION = "h"; + private static final String KEY_SIZE_OPTION = "k"; + private static final String VALUE_SIZE_OPTION = "v"; + private static final String COMPRESSION_OPTION = "c"; + private static final String BLOOM_FILTER_OPTION = "bf"; + private static final String BLOCK_SIZE_OPTION = "bs"; + private static final String BLOOM_BLOCK_SIZE_OPTION = "bfbs"; + private static final String INDEX_BLOCK_SIZE_OPTION = "ibs"; + + /** The exit code this command-line tool returns on failure */ + private static final int EXIT_FAILURE = 1; + + /** The number of valid key types in a store file */ + private static final int NUM_VALID_KEY_TYPES = + KeyValue.Type.values().length - 2; + + private Options options = new Options(); + + private int keyPrefixLen, keyLen, rowLen, cfLen, valueLen; + private Random rand; + + /** + * Runs the tools. + * + * @param args command-line arguments + * @return true in case of success + * @throws IOException + */ + public boolean run(String[] args) throws IOException { + options.addOption(OUTPUT_DIR_OPTION, "output_dir", true, + "Output directory"); + options.addOption(NUM_KV_OPTION, "num_kv", true, + "Number of key/value pairs"); + options.addOption(KEY_SIZE_OPTION, "key_size", true, "Average key size"); + options.addOption(VALUE_SIZE_OPTION, "value_size", true, + "Average value size"); + options.addOption(HFILE_VERSION_OPTION, "hfile_version", true, + "HFile version to create"); + options.addOption(COMPRESSION_OPTION, "compression", true, + " Compression type, one of " + + Arrays.toString(Compression.Algorithm.values())); + options.addOption(BLOOM_FILTER_OPTION, "bloom_filter", true, + "Bloom filter type, one of " + + Arrays.toString(StoreFile.BloomType.values())); + options.addOption(BLOCK_SIZE_OPTION, "block_size", true, + "HFile block size"); + options.addOption(BLOOM_BLOCK_SIZE_OPTION, "bloom_block_size", true, + "Compound Bloom filters block size"); + options.addOption(INDEX_BLOCK_SIZE_OPTION, "index_block_size", true, + "Index block size"); + + if (args.length == 0) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp(CreateRandomStoreFile.class.getSimpleName(), options, + true); + return false; + } + + CommandLineParser parser = new PosixParser(); + CommandLine cmdLine; + try { + cmdLine = parser.parse(options, args); + } catch (ParseException ex) { + LOG.error(ex); + return false; + } + + if (!cmdLine.hasOption(OUTPUT_DIR_OPTION)) { + LOG.error("Output directory is not specified"); + return false; + } + + if (!cmdLine.hasOption(NUM_KV_OPTION)) { + LOG.error("The number of keys/values not specified"); + return false; + } + + if (!cmdLine.hasOption(KEY_SIZE_OPTION)) { + LOG.error("Key size is not specified"); + return false; + } + + if (!cmdLine.hasOption(VALUE_SIZE_OPTION)) { + LOG.error("Value size not specified"); + return false; + } + + Configuration conf = HBaseConfiguration.create(); + + Path outputDir = new Path(cmdLine.getOptionValue(OUTPUT_DIR_OPTION)); + + long numKV = Long.parseLong(cmdLine.getOptionValue(NUM_KV_OPTION)); + configureKeyValue(numKV, + Integer.parseInt(cmdLine.getOptionValue(KEY_SIZE_OPTION)), + Integer.parseInt(cmdLine.getOptionValue(VALUE_SIZE_OPTION))); + + FileSystem fs = FileSystem.get(conf); + + Compression.Algorithm compr = Compression.Algorithm.NONE; + if (cmdLine.hasOption(COMPRESSION_OPTION)) { + compr = Compression.Algorithm.valueOf( + cmdLine.getOptionValue(COMPRESSION_OPTION)); + } + + StoreFile.BloomType bloomType = StoreFile.BloomType.NONE; + if (cmdLine.hasOption(BLOOM_FILTER_OPTION)) { + bloomType = StoreFile.BloomType.valueOf(cmdLine.getOptionValue( + BLOOM_FILTER_OPTION)); + } + + int blockSize = HFile.DEFAULT_BLOCKSIZE; + if (cmdLine.hasOption(BLOCK_SIZE_OPTION)) + blockSize = Integer.valueOf(cmdLine.getOptionValue(BLOCK_SIZE_OPTION)); + + if (cmdLine.hasOption(BLOOM_BLOCK_SIZE_OPTION)) { + conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, + Integer.valueOf(cmdLine.getOptionValue(BLOOM_BLOCK_SIZE_OPTION))); + } + + if (cmdLine.hasOption(INDEX_BLOCK_SIZE_OPTION)) { + conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, + Integer.valueOf(cmdLine.getOptionValue(INDEX_BLOCK_SIZE_OPTION))); + } + + StoreFile.Writer sfw = new StoreFile.WriterBuilder(conf, + new CacheConfig(conf), fs, blockSize) + .withOutputDir(outputDir) + .withCompression(compr) + .withBloomType(bloomType) + .withMaxKeyCount(numKV) + .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE) + .withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM) + .build(); + + rand = new Random(); + LOG.info("Writing " + numKV + " key/value pairs"); + for (long i = 0; i < numKV; ++i) { + sfw.append(generateKeyValue(i)); + } + + int numMetaBlocks = rand.nextInt(10) + 1; + LOG.info("Writing " + numMetaBlocks + " meta blocks"); + for (int metaI = 0; metaI < numMetaBlocks; ++metaI) { + sfw.getHFileWriter().appendMetaBlock(generateString(), + new BytesWritable(generateValue())); + } + sfw.close(); + + Path storeFilePath = sfw.getPath(); + long fileSize = fs.getFileStatus(storeFilePath).getLen(); + LOG.info("Created " + storeFilePath + ", " + fileSize + " bytes"); + + return true; + } + + private void configureKeyValue(long numKV, int keyLen, int valueLen) { + numKV = Math.abs(numKV); + keyLen = Math.abs(keyLen); + keyPrefixLen = 0; + while (numKV != 0) { + numKV >>>= 8; + ++keyPrefixLen; + } + + this.keyLen = Math.max(keyPrefixLen, keyLen); + this.valueLen = valueLen; + + // Arbitrarily split the key into row, column family, and qualifier. + rowLen = keyPrefixLen / 3; + cfLen = keyPrefixLen / 4; + } + + private int nextInRange(int range) { + return rand.nextInt(2 * range + 1) - range; + } + + public KeyValue generateKeyValue(long i) { + byte[] k = generateKey(i); + byte[] v = generateValue(); + + return new KeyValue( + k, 0, rowLen, + k, rowLen, cfLen, + k, rowLen + cfLen, k.length - rowLen - cfLen, + rand.nextLong(), + generateKeyType(rand), + v, 0, v.length); + } + + public static KeyValue.Type generateKeyType(Random rand) { + if (rand.nextBoolean()) { + // Let's make half of KVs puts. + return KeyValue.Type.Put; + } else { + KeyValue.Type keyType = + KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)]; + if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) + { + throw new RuntimeException("Generated an invalid key type: " + keyType + + ". " + "Probably the layout of KeyValue.Type has changed."); + } + return keyType; + } + } + + private String generateString() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < rand.nextInt(10); ++i) { + sb.append((char) ('A' + rand.nextInt(26))); + } + return sb.toString(); + } + + private byte[] generateKey(long i) { + byte[] k = new byte[Math.max(keyPrefixLen, keyLen + + nextInRange(LEN_VARIATION))]; + for (int pos = keyPrefixLen - 1; pos >= 0; --pos) { + k[pos] = (byte) (i & 0xFF); + i >>>= 8; + } + for (int pos = keyPrefixLen; pos < k.length; ++pos) { + k[pos] = (byte) rand.nextInt(256); + } + return k; + } + + private byte[] generateValue() { + byte[] v = new byte[Math.max(1, valueLen + nextInRange(LEN_VARIATION))]; + for (int i = 0; i < v.length; ++i) { + v[i] = (byte) rand.nextInt(256); + } + return v; + } + + public static void main(String[] args) { + CreateRandomStoreFile app = new CreateRandomStoreFile(); + try { + if (!app.run(args)) + System.exit(EXIT_FAILURE); + } catch (IOException ex) { + LOG.error(ex); + System.exit(EXIT_FAILURE); + } + + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java b/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java new file mode 100644 index 0000000..effa74c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java @@ -0,0 +1,601 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.io.encoding.EncodedDataBlock; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder; +import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.compress.Compressor; +import org.apache.hadoop.io.compress.Decompressor; + +/** + * Tests various algorithms for key compression on an existing HFile. Useful + * for testing, debugging and benchmarking. + */ +public class DataBlockEncodingTool { + private static final Log LOG = LogFactory.getLog( + DataBlockEncodingTool.class); + + private static final boolean includesMemstoreTS = true; + + /** + * How many times should benchmark run. + * More times means better data in terms of statistics. + * It has to be larger than BENCHMARK_N_OMIT. + */ + public static int BENCHMARK_N_TIMES = 12; + + /** + * How many first runs should omit benchmark. + * Usually it is one in order to exclude setup cost. + * Has to be 0 or larger. + */ + public static int BENCHMARK_N_OMIT = 2; + + /** Compression algorithm to use if not specified on the command line */ + private static final Algorithm DEFAULT_COMPRESSION = + Compression.Algorithm.GZ; + + private List codecs = new ArrayList(); + private int totalPrefixLength = 0; + private int totalKeyLength = 0; + private int totalValueLength = 0; + private int totalKeyRedundancyLength = 0; + + final private String compressionAlgorithmName; + final private Algorithm compressionAlgorithm; + final private Compressor compressor; + final private Decompressor decompressor; + + /** + * @param compressionAlgorithmName What kind of algorithm should be used + * as baseline for comparison (e.g. lzo, gz). + */ + public DataBlockEncodingTool(String compressionAlgorithmName) { + this.compressionAlgorithmName = compressionAlgorithmName; + this.compressionAlgorithm = Compression.getCompressionAlgorithmByName( + compressionAlgorithmName); + this.compressor = this.compressionAlgorithm.getCompressor(); + this.decompressor = this.compressionAlgorithm.getDecompressor(); + } + + /** + * Check statistics for given HFile for different data block encoders. + * @param scanner Of file which will be compressed. + * @param kvLimit Maximal count of KeyValue which will be processed. + * @throws IOException thrown if scanner is invalid + */ + public void checkStatistics(final KeyValueScanner scanner, final int kvLimit) + throws IOException { + scanner.seek(KeyValue.LOWESTKEY); + + KeyValue currentKv; + + byte[] previousKey = null; + byte[] currentKey; + + List dataBlockEncoders = + DataBlockEncoding.getAllEncoders(); + + for (DataBlockEncoder d : dataBlockEncoders) { + codecs.add(new EncodedDataBlock(d, includesMemstoreTS)); + } + + int j = 0; + while ((currentKv = scanner.next()) != null && j < kvLimit) { + // Iterates through key/value pairs + j++; + currentKey = currentKv.getKey(); + if (previousKey != null) { + for (int i = 0; i < previousKey.length && i < currentKey.length && + previousKey[i] == currentKey[i]; ++i) { + totalKeyRedundancyLength++; + } + } + + for (EncodedDataBlock codec : codecs) { + codec.addKv(currentKv); + } + + previousKey = currentKey; + + totalPrefixLength += currentKv.getLength() - currentKv.getKeyLength() - + currentKv.getValueLength(); + totalKeyLength += currentKv.getKeyLength(); + totalValueLength += currentKv.getValueLength(); + } + } + + /** + * Verify if all data block encoders are working properly. + * + * @param scanner Of file which was compressed. + * @param kvLimit Maximal count of KeyValue which will be processed. + * @return true if all data block encoders compressed/decompressed correctly. + * @throws IOException thrown if scanner is invalid + */ + public boolean verifyCodecs(final KeyValueScanner scanner, final int kvLimit) + throws IOException { + KeyValue currentKv; + + scanner.seek(KeyValue.LOWESTKEY); + List> codecIterators = + new ArrayList>(); + for(EncodedDataBlock codec : codecs) { + codecIterators.add(codec.getIterator()); + } + + int j = 0; + while ((currentKv = scanner.next()) != null && j < kvLimit) { + // Iterates through key/value pairs + ++j; + for (Iterator it : codecIterators) { + KeyValue codecKv = it.next(); + if (codecKv == null || 0 != Bytes.compareTo( + codecKv.getBuffer(), codecKv.getOffset(), codecKv.getLength(), + currentKv.getBuffer(), currentKv.getOffset(), + currentKv.getLength())) { + if (codecKv == null) { + LOG.error("There is a bug in codec " + it + + " it returned null KeyValue,"); + } else { + int prefix = 0; + int limitLength = 2 * Bytes.SIZEOF_INT + + Math.min(codecKv.getLength(), currentKv.getLength()); + while (prefix < limitLength && + codecKv.getBuffer()[prefix + codecKv.getOffset()] == + currentKv.getBuffer()[prefix + currentKv.getOffset()]) { + prefix++; + } + + LOG.error("There is bug in codec " + it.toString() + + "\n on element " + j + + "\n codecKv.getKeyLength() " + codecKv.getKeyLength() + + "\n codecKv.getValueLength() " + codecKv.getValueLength() + + "\n codecKv.getLength() " + codecKv.getLength() + + "\n currentKv.getKeyLength() " + currentKv.getKeyLength() + + "\n currentKv.getValueLength() " + currentKv.getValueLength() + + "\n codecKv.getLength() " + currentKv.getLength() + + "\n currentKV rowLength " + currentKv.getRowLength() + + " familyName " + currentKv.getFamilyLength() + + " qualifier " + currentKv.getQualifierLength() + + "\n prefix " + prefix + + "\n codecKv '" + Bytes.toStringBinary(codecKv.getBuffer(), + codecKv.getOffset(), prefix) + "' diff '" + + Bytes.toStringBinary(codecKv.getBuffer(), + codecKv.getOffset() + prefix, codecKv.getLength() - + prefix) + "'" + + "\n currentKv '" + Bytes.toStringBinary( + currentKv.getBuffer(), + currentKv.getOffset(), prefix) + "' diff '" + + Bytes.toStringBinary(currentKv.getBuffer(), + currentKv.getOffset() + prefix, currentKv.getLength() - + prefix) + "'" + ); + } + return false; + } + } + } + + LOG.info("Verification was successful!"); + + return true; + } + + /** + * Benchmark codec's speed. + */ + public void benchmarkCodecs() { + int prevTotalSize = -1; + for (EncodedDataBlock codec : codecs) { + prevTotalSize = benchmarkEncoder(prevTotalSize, codec); + } + + byte[] buffer = codecs.get(0).getRawKeyValues(); + + benchmarkDefaultCompression(prevTotalSize, buffer); + } + + /** + * Benchmark compression/decompression throughput. + * @param previousTotalSize Total size used for verification. Use -1 if + * unknown. + * @param codec Tested encoder. + * @return Size of uncompressed data. + */ + private int benchmarkEncoder(int previousTotalSize, EncodedDataBlock codec) { + int prevTotalSize = previousTotalSize; + int totalSize = 0; + + // decompression time + List durations = new ArrayList(); + for (int itTime = 0; itTime < BENCHMARK_N_TIMES; ++itTime) { + totalSize = 0; + + Iterator it; + + it = codec.getIterator(); + + // count only the algorithm time, without memory allocations + // (expect first time) + final long startTime = System.nanoTime(); + while (it.hasNext()) { + totalSize += it.next().getLength(); + } + final long finishTime = System.nanoTime(); + if (itTime >= BENCHMARK_N_OMIT) { + durations.add(finishTime - startTime); + } + + if (prevTotalSize != -1 && prevTotalSize != totalSize) { + throw new IllegalStateException(String.format( + "Algorithm '%s' decoded data to different size", codec.toString())); + } + prevTotalSize = totalSize; + } + + // compression time + List compressDurations = new ArrayList(); + for (int itTime = 0; itTime < BENCHMARK_N_TIMES; ++itTime) { + final long startTime = System.nanoTime(); + codec.doCompressData(); + final long finishTime = System.nanoTime(); + if (itTime >= BENCHMARK_N_OMIT) { + compressDurations.add(finishTime - startTime); + } + } + + System.out.println(codec.toString() + ":"); + printBenchmarkResult(totalSize, compressDurations, false); + printBenchmarkResult(totalSize, durations, true); + + return prevTotalSize; + } + + private void benchmarkDefaultCompression(int totalSize, byte[] rawBuffer) { + benchmarkAlgorithm(compressionAlgorithm, compressor, decompressor, + compressionAlgorithmName.toUpperCase(), rawBuffer, 0, totalSize); + } + + /** + * Check decompress performance of a given algorithm and print it. + * @param algorithm Compression algorithm. + * @param compressorCodec Compressor to be tested. + * @param decompressorCodec Decompressor of the same algorithm. + * @param name Name of algorithm. + * @param buffer Buffer to be compressed. + * @param offset Position of the beginning of the data. + * @param length Length of data in buffer. + */ + public static void benchmarkAlgorithm( + Compression.Algorithm algorithm, + Compressor compressorCodec, + Decompressor decompressorCodec, + String name, + byte[] buffer, int offset, int length) { + System.out.println(name + ":"); + + // compress it + List compressDurations = new ArrayList(); + ByteArrayOutputStream compressedStream = new ByteArrayOutputStream(); + OutputStream compressingStream; + try { + for (int itTime = 0; itTime < BENCHMARK_N_TIMES; ++itTime) { + final long startTime = System.nanoTime(); + compressingStream = algorithm.createCompressionStream( + compressedStream, compressorCodec, 0); + compressingStream.write(buffer, offset, length); + compressingStream.flush(); + compressedStream.toByteArray(); + + final long finishTime = System.nanoTime(); + + // add time record + if (itTime >= BENCHMARK_N_OMIT) { + compressDurations.add(finishTime - startTime); + } + + if (itTime + 1 < BENCHMARK_N_TIMES) { // not the last one + compressedStream.reset(); + } + } + } catch (IOException e) { + throw new RuntimeException(String.format( + "Benchmark, or encoding algorithm '%s' cause some stream problems", + name), e); + } + printBenchmarkResult(length, compressDurations, false); + + + byte[] compBuffer = compressedStream.toByteArray(); + + // uncompress it several times and measure performance + List durations = new ArrayList(); + for (int itTime = 0; itTime < BENCHMARK_N_TIMES; ++itTime) { + final long startTime = System.nanoTime(); + byte[] newBuf = new byte[length + 1]; + + try { + + ByteArrayInputStream downStream = new ByteArrayInputStream(compBuffer, + 0, compBuffer.length); + InputStream decompressedStream = algorithm.createDecompressionStream( + downStream, decompressorCodec, 0); + + int destOffset = 0; + int nextChunk; + while ((nextChunk = decompressedStream.available()) > 0) { + destOffset += decompressedStream.read(newBuf, destOffset, nextChunk); + } + decompressedStream.close(); + + // iterate over KeyValue + KeyValue kv; + for (int pos = 0; pos < length; pos += kv.getLength()) { + kv = new KeyValue(newBuf, pos); + } + + } catch (IOException e) { + throw new RuntimeException(String.format( + "Decoding path in '%s' algorithm cause exception ", name), e); + } + + final long finishTime = System.nanoTime(); + + // check correctness + if (0 != Bytes.compareTo(buffer, 0, length, newBuf, 0, length)) { + int prefix = 0; + for(; prefix < buffer.length && prefix < newBuf.length; ++prefix) { + if (buffer[prefix] != newBuf[prefix]) { + break; + } + } + throw new RuntimeException(String.format( + "Algorithm '%s' is corrupting the data", name)); + } + + // add time record + if (itTime >= BENCHMARK_N_OMIT) { + durations.add(finishTime - startTime); + } + } + printBenchmarkResult(length, durations, true); + } + + private static void printBenchmarkResult(int totalSize, + List durationsInNanoSed, boolean isDecompression) { + long meanTime = 0; + for (long time : durationsInNanoSed) { + meanTime += time; + } + meanTime /= durationsInNanoSed.size(); + + long standardDev = 0; + for (long time : durationsInNanoSed) { + standardDev += (time - meanTime) * (time - meanTime); + } + standardDev = (long) Math.sqrt(standardDev / durationsInNanoSed.size()); + + final double million = 1000.0 * 1000.0 * 1000.0; + double mbPerSec = (totalSize * million) / (1024.0 * 1024.0 * meanTime); + double mbPerSecDev = (totalSize * million) / + (1024.0 * 1024.0 * (meanTime - standardDev)); + + System.out.println(String.format( + " %s performance:%s %6.2f MB/s (+/- %.2f MB/s)", + isDecompression ? "Decompression" : "Compression", + isDecompression ? "" : " ", + mbPerSec, mbPerSecDev - mbPerSec)); + } + + /** + * Display statistics of different compression algorithms. + */ + public void displayStatistics() { + int totalLength = totalPrefixLength + totalKeyLength + totalValueLength; + if (compressor != null) { // might be null e.g. for pure-Java GZIP + compressor.reset(); + } + + for(EncodedDataBlock codec : codecs) { + System.out.println(codec.toString()); + int saved = totalKeyLength + totalPrefixLength + totalValueLength + - codec.getSize(); + System.out.println( + String.format(" Saved bytes: %8d", saved)); + double keyRatio = (saved * 100.0) / (totalPrefixLength + totalKeyLength); + double allRatio = (saved * 100.0) / totalLength; + System.out.println( + String.format(" Key compression ratio: %.2f %%", keyRatio)); + System.out.println( + String.format(" All compression ratio: %.2f %%", allRatio)); + + String compressedSizeCaption = + String.format(" %s compressed size: ", + compressionAlgorithmName.toUpperCase()); + String compressOnlyRatioCaption = + String.format(" %s compression ratio: ", + compressionAlgorithmName.toUpperCase()); + + if (compressor != null) { + int compressedSize = codec.checkCompressedSize(compressor); + System.out.println(compressedSizeCaption + + String.format("%8d", compressedSize)); + double compressOnlyRatio = + 100.0 * (1.0 - compressedSize / (0.0 + totalLength)); + System.out.println(compressOnlyRatioCaption + + String.format("%.2f %%", compressOnlyRatio)); + } else { + System.out.println(compressedSizeCaption + "N/A"); + System.out.println(compressOnlyRatioCaption + "N/A"); + } + } + + System.out.println( + String.format("Total KV prefix length: %8d", totalPrefixLength)); + System.out.println( + String.format("Total key length: %8d", totalKeyLength)); + System.out.println( + String.format("Total key redundancy: %8d", + totalKeyRedundancyLength)); + System.out.println( + String.format("Total value length: %8d", totalValueLength)); + } + + /** + * Test a data block encoder on the given HFile. Output results to console. + * @param kvLimit The limit of KeyValue which will be analyzed. + * @param hfilePath an HFile path on the file system. + * @param compressionName Compression algorithm used for comparison. + * @param doBenchmark Run performance benchmarks. + * @param doVerify Verify correctness. + * @throws IOException When pathName is incorrect. + */ + public static void testCodecs(Configuration conf, int kvLimit, + String hfilePath, String compressionName, boolean doBenchmark, + boolean doVerify) throws IOException { + // create environment + Path path = new Path(hfilePath); + CacheConfig cacheConf = new CacheConfig(conf); + FileSystem fs = FileSystem.get(conf); + StoreFile hsf = new StoreFile(fs, path, conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + + StoreFile.Reader reader = hsf.createReader(); + reader.loadFileInfo(); + KeyValueScanner scanner = reader.getStoreFileScanner(true, true); + + // run the utilities + DataBlockEncodingTool comp = new DataBlockEncodingTool(compressionName); + comp.checkStatistics(scanner, kvLimit); + if (doVerify) { + comp.verifyCodecs(scanner, kvLimit); + } + if (doBenchmark) { + comp.benchmarkCodecs(); + } + comp.displayStatistics(); + + // cleanup + scanner.close(); + reader.close(cacheConf.shouldEvictOnClose()); + } + + private static void printUsage(Options options) { + System.err.println("Usage:"); + System.err.println(String.format("./hbase %s ", + DataBlockEncodingTool.class.getName())); + System.err.println("Options:"); + for (Object it : options.getOptions()) { + Option opt = (Option) it; + if (opt.hasArg()) { + System.err.println(String.format("-%s %s: %s", opt.getOpt(), + opt.getArgName(), opt.getDescription())); + } else { + System.err.println(String.format("-%s: %s", opt.getOpt(), + opt.getDescription())); + } + } + } + + /** + * A command line interface to benchmarks. + * @param args Should have length at least 1 and holds the file path to HFile. + * @throws IOException If you specified the wrong file. + */ + public static void main(final String[] args) throws IOException { + // set up user arguments + Options options = new Options(); + options.addOption("f", true, "HFile to analyse (REQUIRED)"); + options.getOption("f").setArgName("FILENAME"); + options.addOption("n", true, + "Limit number of KeyValue which will be analysed"); + options.getOption("n").setArgName("NUMBER"); + options.addOption("b", false, "Measure read throughput"); + options.addOption("c", false, "Omit corectness tests."); + options.addOption("a", true, + "What kind of compression algorithm use for comparison."); + + // parse arguments + CommandLineParser parser = new PosixParser(); + CommandLine cmd = null; + try { + cmd = parser.parse(options, args); + } catch (ParseException e) { + System.err.println("Could not parse arguments!"); + System.exit(-1); + return; // avoid warning + } + + int kvLimit = Integer.MAX_VALUE; + if (cmd.hasOption("n")) { + kvLimit = Integer.parseInt(cmd.getOptionValue("n")); + } + + // basic argument sanity checks + if (!cmd.hasOption("f")) { + System.err.println("ERROR: Filename is required!"); + printUsage(options); + System.exit(-1); + } + + String pathName = cmd.getOptionValue("f"); + String compressionName = DEFAULT_COMPRESSION.getName(); + if (cmd.hasOption("a")) { + compressionName = cmd.getOptionValue("a").toLowerCase(); + } + boolean doBenchmark = cmd.hasOption("b"); + boolean doVerify = !cmd.hasOption("c"); + + final Configuration conf = HBaseConfiguration.create(); + try { + testCodecs(conf, kvLimit, pathName, compressionName, doBenchmark, + doVerify); + } finally { + (new CacheConfig(conf)).getBlockCache().shutdown(); + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java b/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java new file mode 100644 index 0000000..7f67a72 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; +import org.apache.hadoop.hbase.io.hfile.LruBlockCache; +import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; + +/** + * Test seek performance for encoded data blocks. Read an HFile and do several + * random seeks. + */ +public class EncodedSeekPerformanceTest { + private static final double NANOSEC_IN_SEC = 1000.0 * 1000.0 * 1000.0; + private static final double BYTES_IN_MEGABYTES = 1024.0 * 1024.0; + /** Default number of seeks which will be used in benchmark. */ + public static int DEFAULT_NUMBER_OF_SEEKS = 10000; + + private final HBaseTestingUtility testingUtility = new HBaseTestingUtility(); + private Configuration configuration = testingUtility.getConfiguration(); + private CacheConfig cacheConf = new CacheConfig(configuration); + private Random randomizer; + private int numberOfSeeks; + + /** Use this benchmark with default options */ + public EncodedSeekPerformanceTest() { + configuration.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.5f); + randomizer = new Random(42l); + numberOfSeeks = DEFAULT_NUMBER_OF_SEEKS; + } + + private List prepareListOfTestSeeks(Path path) throws IOException { + List allKeyValues = new ArrayList(); + + // read all of the key values + StoreFile storeFile = new StoreFile(testingUtility.getTestFileSystem(), + path, configuration, cacheConf, BloomType.NONE, + NoOpDataBlockEncoder.INSTANCE); + + StoreFile.Reader reader = storeFile.createReader(); + StoreFileScanner scanner = reader.getStoreFileScanner(true, false); + KeyValue current; + + scanner.seek(KeyValue.LOWESTKEY); + while (null != (current = scanner.next())) { + allKeyValues.add(current); + } + + storeFile.closeReader(cacheConf.shouldEvictOnClose()); + + // pick seeks by random + List seeks = new ArrayList(); + for (int i = 0; i < numberOfSeeks; ++i) { + KeyValue keyValue = allKeyValues.get( + randomizer.nextInt(allKeyValues.size())); + seeks.add(keyValue); + } + + clearBlockCache(); + + return seeks; + } + + private void runTest(Path path, HFileDataBlockEncoder blockEncoder, + List seeks) throws IOException { + // read all of the key values + StoreFile storeFile = new StoreFile(testingUtility.getTestFileSystem(), + path, configuration, cacheConf, BloomType.NONE, blockEncoder); + + long totalSize = 0; + + StoreFile.Reader reader = storeFile.createReader(); + StoreFileScanner scanner = reader.getStoreFileScanner(true, false); + + long startReadingTime = System.nanoTime(); + KeyValue current; + scanner.seek(KeyValue.LOWESTKEY); + while (null != (current = scanner.next())) { // just iterate it! + if (current.getLength() < 0) { + throw new IOException("Negative KV size: " + current); + } + totalSize += current.getLength(); + } + long finishReadingTime = System.nanoTime(); + + // do seeks + long startSeeksTime = System.nanoTime(); + for (KeyValue keyValue : seeks) { + scanner.seek(keyValue); + KeyValue toVerify = scanner.next(); + if (!keyValue.equals(toVerify)) { + System.out.println(String.format("KeyValue doesn't match:\n" + + "Orig key: %s\n" + + "Ret key: %s", keyValue.getKeyString(), toVerify.getKeyString())); + break; + } + } + long finishSeeksTime = System.nanoTime(); + if (finishSeeksTime < startSeeksTime) { + throw new AssertionError("Finish time " + finishSeeksTime + + " is earlier than start time " + startSeeksTime); + } + + // write some stats + double readInMbPerSec = (totalSize * NANOSEC_IN_SEC) / + (BYTES_IN_MEGABYTES * (finishReadingTime - startReadingTime)); + double seeksPerSec = (seeks.size() * NANOSEC_IN_SEC) / + (finishSeeksTime - startSeeksTime); + + storeFile.closeReader(cacheConf.shouldEvictOnClose()); + clearBlockCache(); + + System.out.println(blockEncoder); + System.out.printf(" Read speed: %8.2f (MB/s)\n", readInMbPerSec); + System.out.printf(" Seeks per second: %8.2f (#/s)\n", seeksPerSec); + System.out.printf(" Total KV size: %d\n", totalSize); + } + + /** + * @param path Path to the HFile which will be used. + * @param encoders List of encoders which will be used for tests. + * @throws IOException if there is a bug while reading from disk + */ + public void runTests(Path path, List encoders) + throws IOException { + List seeks = prepareListOfTestSeeks(path); + + for (HFileDataBlockEncoder blockEncoder : encoders) { + runTest(path, blockEncoder, seeks); + } + } + + /** + * Command line interface: + * @param args Takes one argument - file size. + * @throws IOException if there is a bug while reading from disk + */ + public static void main(final String[] args) throws IOException { + if (args.length < 1) { + printUsage(); + System.exit(-1); + } + + Path path = new Path(args[0]); + List encoders = + new ArrayList(); + + encoders.add(new HFileDataBlockEncoderImpl(DataBlockEncoding.NONE)); + for (DataBlockEncoding encodingAlgo : DataBlockEncoding.values()) { + encoders.add(new HFileDataBlockEncoderImpl(DataBlockEncoding.NONE, + encodingAlgo)); + } + + EncodedSeekPerformanceTest utility = new EncodedSeekPerformanceTest(); + utility.runTests(path, encoders); + + System.exit(0); + } + + private static void printUsage() { + System.out.println("Usage: one argument, name of the HFile"); + } + + private void clearBlockCache() { + ((LruBlockCache) cacheConf.getBlockCache()).clearCache(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java b/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java new file mode 100644 index 0000000..e6ff173 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java @@ -0,0 +1,850 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.SortedSet; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; +import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter; +import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.LoadTestTool; +import org.apache.hadoop.hbase.util.MD5Hash; +import org.apache.hadoop.util.StringUtils; + +/** + * Tests HFile read/write workloads, such as merging HFiles and random reads. + */ +public class HFileReadWriteTest { + + private static final String TABLE_NAME = "MyTable"; + + private static enum Workload { + MERGE("merge", "Merge the specified HFiles", 1, Integer.MAX_VALUE), + RANDOM_READS("read", "Perform a random read benchmark on the given HFile", + 1, 1); + + private String option; + private String description; + + public final int minNumInputFiles; + public final int maxNumInputFiles; + + Workload(String option, String description, int minNumInputFiles, + int maxNumInputFiles) { + this.option = option; + this.description = description; + this.minNumInputFiles = minNumInputFiles; + this.maxNumInputFiles = maxNumInputFiles; + } + + static OptionGroup getOptionGroup() { + OptionGroup optionGroup = new OptionGroup(); + for (Workload w : values()) + optionGroup.addOption(new Option(w.option, w.description)); + return optionGroup; + } + + private static String getOptionListStr() { + StringBuilder sb = new StringBuilder(); + for (Workload w : values()) { + if (sb.length() > 0) + sb.append(", "); + sb.append("-" + w.option); + } + return sb.toString(); + } + + static Workload fromCmdLine(CommandLine cmdLine) { + for (Workload w : values()) { + if (cmdLine.hasOption(w.option)) + return w; + } + LOG.error("No workload specified. Specify one of the options: " + + getOptionListStr()); + return null; + } + + public String onlyUsedFor() { + return ". Only used for the " + this + " workload."; + } + } + + private static final String OUTPUT_DIR_OPTION = "output_dir"; + private static final String COMPRESSION_OPTION = "compression"; + private static final String BLOOM_FILTER_OPTION = "bloom"; + private static final String BLOCK_SIZE_OPTION = "block_size"; + private static final String DURATION_OPTION = "duration"; + private static final String NUM_THREADS_OPTION = "num_threads"; + + private static final Log LOG = LogFactory.getLog(HFileReadWriteTest.class); + + private Workload workload; + private FileSystem fs; + private Configuration conf; + private CacheConfig cacheConf; + private List inputFileNames; + private Path outputDir; + private int numReadThreads; + private int durationSec; + private DataBlockEncoding dataBlockEncoding; + private boolean encodeInCacheOnly; + private HFileDataBlockEncoder dataBlockEncoder = + NoOpDataBlockEncoder.INSTANCE; + + private StoreFile.BloomType bloomType = StoreFile.BloomType.NONE; + private int blockSize; + private Compression.Algorithm compression = Compression.Algorithm.NONE; + + private byte[] firstRow, lastRow; + + private AtomicLong numSeeks = new AtomicLong(); + private AtomicLong numKV = new AtomicLong(); + private AtomicLong totalBytes = new AtomicLong(); + + private byte[] family; + + private long endTime = Long.MAX_VALUE; + + private SortedSet keysRead = new ConcurrentSkipListSet(); + private List inputStoreFiles; + + public HFileReadWriteTest() { + conf = HBaseConfiguration.create(); + cacheConf = new CacheConfig(conf); + } + + @SuppressWarnings("unchecked") + public boolean parseOptions(String args[]) { + + Options options = new Options(); + options.addOption(OUTPUT_DIR_OPTION, true, "Output directory" + + Workload.MERGE.onlyUsedFor()); + options.addOption(COMPRESSION_OPTION, true, " Compression type, one of " + + Arrays.toString(Compression.Algorithm.values()) + + Workload.MERGE.onlyUsedFor()); + options.addOption(BLOOM_FILTER_OPTION, true, "Bloom filter type, one of " + + Arrays.toString(StoreFile.BloomType.values()) + + Workload.MERGE.onlyUsedFor()); + options.addOption(BLOCK_SIZE_OPTION, true, "HFile block size" + + Workload.MERGE.onlyUsedFor()); + options.addOption(DURATION_OPTION, true, "The amount of time to run the " + + "random read workload for" + Workload.RANDOM_READS.onlyUsedFor()); + options.addOption(NUM_THREADS_OPTION, true, "The number of random " + + "reader threads" + Workload.RANDOM_READS.onlyUsedFor()); + options.addOption(NUM_THREADS_OPTION, true, "The number of random " + + "reader threads" + Workload.RANDOM_READS.onlyUsedFor()); + options.addOption(LoadTestTool.OPT_DATA_BLOCK_ENCODING, true, + LoadTestTool.OPT_DATA_BLOCK_ENCODING_USAGE); + options.addOption(LoadTestTool.OPT_ENCODE_IN_CACHE_ONLY, false, + LoadTestTool.OPT_ENCODE_IN_CACHE_ONLY_USAGE); + options.addOptionGroup(Workload.getOptionGroup()); + + if (args.length == 0) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp(HFileReadWriteTest.class.getSimpleName(), + options, true); + return false; + } + + CommandLineParser parser = new PosixParser(); + CommandLine cmdLine; + try { + cmdLine = parser.parse(options, args); + } catch (ParseException ex) { + LOG.error(ex); + return false; + } + + workload = Workload.fromCmdLine(cmdLine); + if (workload == null) + return false; + + inputFileNames = (List) cmdLine.getArgList(); + + if (inputFileNames.size() == 0) { + LOG.error("No input file names specified"); + return false; + } + + if (inputFileNames.size() < workload.minNumInputFiles) { + LOG.error("Too few input files: at least " + workload.minNumInputFiles + + " required"); + return false; + } + + if (inputFileNames.size() > workload.maxNumInputFiles) { + LOG.error("Too many input files: at most " + workload.minNumInputFiles + + " allowed"); + return false; + } + + if (cmdLine.hasOption(COMPRESSION_OPTION)) { + compression = Compression.Algorithm.valueOf( + cmdLine.getOptionValue(COMPRESSION_OPTION)); + } + + if (cmdLine.hasOption(BLOOM_FILTER_OPTION)) { + bloomType = StoreFile.BloomType.valueOf(cmdLine.getOptionValue( + BLOOM_FILTER_OPTION)); + } + + encodeInCacheOnly = + cmdLine.hasOption(LoadTestTool.OPT_ENCODE_IN_CACHE_ONLY); + + if (cmdLine.hasOption(LoadTestTool.OPT_DATA_BLOCK_ENCODING)) { + dataBlockEncoding = DataBlockEncoding.valueOf( + cmdLine.getOptionValue(LoadTestTool.OPT_DATA_BLOCK_ENCODING)); + // Optionally encode on disk, always encode in cache. + dataBlockEncoder = new HFileDataBlockEncoderImpl( + encodeInCacheOnly ? DataBlockEncoding.NONE : dataBlockEncoding, + dataBlockEncoding); + } else { + if (encodeInCacheOnly) { + LOG.error("The -" + LoadTestTool.OPT_ENCODE_IN_CACHE_ONLY + + " option does not make sense without -" + + LoadTestTool.OPT_DATA_BLOCK_ENCODING); + return false; + } + } + + blockSize = conf.getInt("hfile.min.blocksize.size", 65536); + if (cmdLine.hasOption(BLOCK_SIZE_OPTION)) + blockSize = Integer.valueOf(cmdLine.getOptionValue(BLOCK_SIZE_OPTION)); + + if (workload == Workload.MERGE) { + String outputDirStr = cmdLine.getOptionValue(OUTPUT_DIR_OPTION); + if (outputDirStr == null) { + LOG.error("Output directory is not specified"); + return false; + } + outputDir = new Path(outputDirStr); + // Will be checked for existence in validateConfiguration. + } + + if (workload == Workload.RANDOM_READS) { + if (!requireOptions(cmdLine, new String[] { DURATION_OPTION, + NUM_THREADS_OPTION })) { + return false; + } + + durationSec = Integer.parseInt(cmdLine.getOptionValue(DURATION_OPTION)); + numReadThreads = Integer.parseInt( + cmdLine.getOptionValue(NUM_THREADS_OPTION)); + } + + Collections.sort(inputFileNames); + + return true; + } + + /** @return true if all the given options are specified */ + private boolean requireOptions(CommandLine cmdLine, + String[] requiredOptions) { + for (String option : requiredOptions) + if (!cmdLine.hasOption(option)) { + LOG.error("Required option -" + option + " not specified"); + return false; + } + return true; + } + + public boolean validateConfiguration() throws IOException { + fs = FileSystem.get(conf); + + for (String inputFileName : inputFileNames) { + Path path = new Path(inputFileName); + if (!fs.exists(path)) { + LOG.error("File " + inputFileName + " does not exist"); + return false; + } + + if (fs.getFileStatus(path).isDir()) { + LOG.error(inputFileName + " is a directory"); + return false; + } + } + + if (outputDir != null && + (!fs.exists(outputDir) || !fs.getFileStatus(outputDir).isDir())) { + LOG.error(outputDir.toString() + " does not exist or is not a " + + "directory"); + return false; + } + + return true; + } + + public void runMergeWorkload() throws IOException { + long maxKeyCount = prepareForMerge(); + + List scanners = + StoreFileScanner.getScannersForStoreFiles(inputStoreFiles, false, + false); + + HColumnDescriptor columnDescriptor = new HColumnDescriptor( + HFileReadWriteTest.class.getSimpleName()); + columnDescriptor.setBlocksize(blockSize); + columnDescriptor.setBloomFilterType(bloomType); + columnDescriptor.setCompressionType(compression); + columnDescriptor.setDataBlockEncoding(dataBlockEncoding); + HRegionInfo regionInfo = new HRegionInfo(); + HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); + HRegion region = new HRegion(outputDir, null, fs, conf, regionInfo, htd, + null); + Store store = new Store(outputDir, region, columnDescriptor, fs, conf); + + StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, + new CacheConfig(conf), fs, blockSize) + .withOutputDir(outputDir) + .withCompression(compression) + .withDataBlockEncoder(dataBlockEncoder) + .withBloomType(bloomType) + .withMaxKeyCount(maxKeyCount) + .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE) + .withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM) + .build(); + + StatisticsPrinter statsPrinter = new StatisticsPrinter(); + statsPrinter.startThread(); + + try { + performMerge(scanners, store, writer); + writer.close(); + } finally { + statsPrinter.requestStop(); + } + + Path resultPath = writer.getPath(); + + resultPath = tryUsingSimpleOutputPath(resultPath); + + long fileSize = fs.getFileStatus(resultPath).getLen(); + LOG.info("Created " + resultPath + ", size " + fileSize); + + System.out.println(); + System.out.println("HFile information for " + resultPath); + System.out.println(); + + HFilePrettyPrinter hfpp = new HFilePrettyPrinter(); + hfpp.run(new String[] { "-m", "-f", resultPath.toString() }); + } + + private Path tryUsingSimpleOutputPath(Path resultPath) throws IOException { + if (inputFileNames.size() == 1) { + // In case of only one input set output to be consistent with the + // input name. + + Path inputPath = new Path(inputFileNames.get(0)); + Path betterOutputPath = new Path(outputDir, + inputPath.getName()); + if (!fs.exists(betterOutputPath)) { + fs.rename(resultPath, betterOutputPath); + resultPath = betterOutputPath; + } + } + return resultPath; + } + + private void performMerge(List scanners, Store store, + StoreFile.Writer writer) throws IOException { + InternalScanner scanner = null; + try { + Scan scan = new Scan(); + + // Include deletes + scanner = new StoreScanner(store, store.scanInfo, scan, scanners, + ScanType.MAJOR_COMPACT, Long.MIN_VALUE, Long.MIN_VALUE); + + ArrayList kvs = new ArrayList(); + + while (scanner.next(kvs) || kvs.size() != 0) { + numKV.addAndGet(kvs.size()); + for (KeyValue kv : kvs) { + totalBytes.addAndGet(kv.getLength()); + writer.append(kv); + } + kvs.clear(); + } + } finally { + if (scanner != null) + scanner.close(); + } + } + + /** + * @return the total key count in the files being merged + * @throws IOException + */ + private long prepareForMerge() throws IOException { + LOG.info("Merging " + inputFileNames); + LOG.info("Using block size: " + blockSize); + inputStoreFiles = new ArrayList(); + + long maxKeyCount = 0; + for (String fileName : inputFileNames) { + Path filePath = new Path(fileName); + + // Open without caching. + StoreFile sf = openStoreFile(filePath, false); + sf.createReader(); + inputStoreFiles.add(sf); + + StoreFile.Reader r = sf.getReader(); + if (r != null) { + long keyCount = r.getFilterEntries(); + maxKeyCount += keyCount; + LOG.info("Compacting: " + sf + "; keyCount = " + keyCount + + "; Bloom Type = " + r.getBloomFilterType().toString() + + "; Size = " + StringUtils.humanReadableInt(r.length())); + } + } + return maxKeyCount; + } + + public HFile.Reader[] getHFileReaders() { + HFile.Reader readers[] = new HFile.Reader[inputStoreFiles.size()]; + for (int i = 0; i < inputStoreFiles.size(); ++i) + readers[i] = inputStoreFiles.get(i).getReader().getHFileReader(); + return readers; + } + + private StoreFile openStoreFile(Path filePath, boolean blockCache) + throws IOException { + // We are passing the ROWCOL Bloom filter type, but StoreFile will still + // use the Bloom filter type specified in the HFile. + return new StoreFile(fs, filePath, conf, cacheConf, + StoreFile.BloomType.ROWCOL, dataBlockEncoder); + } + + public static int charToHex(int c) { + if ('0' <= c && c <= '9') + return c - '0'; + if ('a' <= c && c <= 'f') + return 10 + c - 'a'; + return -1; + } + + public static int hexToChar(int h) { + h &= 0xff; + if (0 <= h && h <= 9) + return '0' + h; + if (10 <= h && h <= 15) + return 'a' + h - 10; + return -1; + } + + public static byte[] createRandomRow(Random rand, byte[] first, byte[] last) + { + int resultLen = Math.max(first.length, last.length); + int minLen = Math.min(first.length, last.length); + byte[] result = new byte[resultLen]; + boolean greaterThanFirst = false; + boolean lessThanLast = false; + + for (int i = 0; i < resultLen; ++i) { + // Generate random hex characters if both first and last row are hex + // at this position. + boolean isHex = i < minLen && charToHex(first[i]) != -1 + && charToHex(last[i]) != -1; + + // If our key is already greater than the first key, we can use + // arbitrarily low values. + int low = greaterThanFirst || i >= first.length ? 0 : first[i] & 0xff; + + // If our key is already less than the last key, we can use arbitrarily + // high values. + int high = lessThanLast || i >= last.length ? 0xff : last[i] & 0xff; + + // Randomly select the next byte between the lowest and the highest + // value allowed for this position. Restrict to hex characters if + // necessary. We are generally biased towards border cases, which is OK + // for test. + + int r; + if (isHex) { + // Use hex chars. + if (low < '0') + low = '0'; + + if (high > 'f') + high = 'f'; + + int lowHex = charToHex(low); + int highHex = charToHex(high); + r = hexToChar(lowHex + rand.nextInt(highHex - lowHex + 1)); + } else { + r = low + rand.nextInt(high - low + 1); + } + + if (r > low) + greaterThanFirst = true; + + if (r < high) + lessThanLast = true; + + result[i] = (byte) r; + } + + if (Bytes.compareTo(result, first) < 0) { + throw new IllegalStateException("Generated key " + + Bytes.toStringBinary(result) + " is less than the first key " + + Bytes.toStringBinary(first)); + } + + if (Bytes.compareTo(result, last) > 0) { + throw new IllegalStateException("Generated key " + + Bytes.toStringBinary(result) + " is greater than te last key " + + Bytes.toStringBinary(last)); + } + + return result; + } + + private static byte[] createRandomQualifier(Random rand) { + byte[] q = new byte[10 + rand.nextInt(30)]; + rand.nextBytes(q); + return q; + } + + private class RandomReader implements Callable { + + private int readerId; + private StoreFile.Reader reader; + private boolean pread; + + public RandomReader(int readerId, StoreFile.Reader reader, + boolean pread) + { + this.readerId = readerId; + this.reader = reader; + this.pread = pread; + } + + @Override + public Boolean call() throws Exception { + Thread.currentThread().setName("reader " + readerId); + Random rand = new Random(); + StoreFileScanner scanner = reader.getStoreFileScanner(true, pread); + + while (System.currentTimeMillis() < endTime) { + byte[] row = createRandomRow(rand, firstRow, lastRow); + KeyValue kvToSeek = new KeyValue(row, family, + createRandomQualifier(rand)); + if (rand.nextDouble() < 0.0001) { + LOG.info("kvToSeek=" + kvToSeek); + } + boolean seekResult; + try { + seekResult = scanner.seek(kvToSeek); + } catch (IOException ex) { + throw new IOException("Seek failed for key " + kvToSeek + ", pread=" + + pread, ex); + } + numSeeks.incrementAndGet(); + if (!seekResult) { + error("Seek returned false for row " + Bytes.toStringBinary(row)); + return false; + } + for (int i = 0; i < rand.nextInt(10) + 1; ++i) { + KeyValue kv = scanner.next(); + numKV.incrementAndGet(); + if (i == 0 && kv == null) { + error("scanner.next() returned null at the first iteration for " + + "row " + Bytes.toStringBinary(row)); + return false; + } + if (kv == null) + break; + + String keyHashStr = MD5Hash.getMD5AsHex(kv.getKey()); + keysRead.add(keyHashStr); + totalBytes.addAndGet(kv.getLength()); + } + } + + return true; + } + + private void error(String msg) { + LOG.error("error in reader " + readerId + " (pread=" + pread + "): " + + msg); + } + + } + + private class StatisticsPrinter implements Callable { + + private volatile boolean stopRequested; + private volatile Thread thread; + private long totalSeekAndReads, totalPositionalReads; + + /** + * Run the statistics collector in a separate thread without an executor. + */ + public void startThread() { + new Thread() { + @Override + public void run() { + try { + call(); + } catch (Exception e) { + LOG.error(e); + } + } + }.start(); + } + + @Override + public Boolean call() throws Exception { + LOG.info("Starting statistics printer"); + thread = Thread.currentThread(); + thread.setName(StatisticsPrinter.class.getSimpleName()); + long startTime = System.currentTimeMillis(); + long curTime; + while ((curTime = System.currentTimeMillis()) < endTime && + !stopRequested) { + long elapsedTime = curTime - startTime; + printStats(elapsedTime); + try { + Thread.sleep(1000 - elapsedTime % 1000); + } catch (InterruptedException iex) { + Thread.currentThread().interrupt(); + if (stopRequested) + break; + } + } + printStats(curTime - startTime); + LOG.info("Stopping statistics printer"); + return true; + } + + private void printStats(long elapsedTime) { + long numSeeksL = numSeeks.get(); + double timeSec = elapsedTime / 1000.0; + double seekPerSec = numSeeksL / timeSec; + long kvCount = numKV.get(); + double kvPerSec = kvCount / timeSec; + long bytes = totalBytes.get(); + double bytesPerSec = bytes / timeSec; + + // readOps and preadOps counters get reset on access, so we have to + // accumulate them here. HRegion metrics publishing thread should not + // be running in this tool, so no one else should be resetting these + // metrics. + totalSeekAndReads += HFile.getReadOps(); + totalPositionalReads += HFile.getPreadOps(); + long totalBlocksRead = totalSeekAndReads + totalPositionalReads; + + double blkReadPerSec = totalBlocksRead / timeSec; + + double seekReadPerSec = totalSeekAndReads / timeSec; + double preadPerSec = totalPositionalReads / timeSec; + + boolean isRead = workload == Workload.RANDOM_READS; + + StringBuilder sb = new StringBuilder(); + sb.append("Time: " + (long) timeSec + " sec"); + if (isRead) + sb.append(", seek/sec: " + (long) seekPerSec); + sb.append(", kv/sec: " + (long) kvPerSec); + sb.append(", bytes/sec: " + (long) bytesPerSec); + sb.append(", blk/sec: " + (long) blkReadPerSec); + sb.append(", total KV: " + numKV); + sb.append(", total bytes: " + totalBytes); + sb.append(", total blk: " + totalBlocksRead); + + sb.append(", seekRead/sec: " + (long) seekReadPerSec); + sb.append(", pread/sec: " + (long) preadPerSec); + + if (isRead) + sb.append(", unique keys: " + (long) keysRead.size()); + + LOG.info(sb.toString()); + } + + public void requestStop() { + stopRequested = true; + if (thread != null) + thread.interrupt(); + } + + } + + public boolean runRandomReadWorkload() throws IOException { + if (inputFileNames.size() != 1) { + throw new IOException("Need exactly one input file for random reads: " + + inputFileNames); + } + + Path inputPath = new Path(inputFileNames.get(0)); + + // Make sure we are using caching. + StoreFile storeFile = openStoreFile(inputPath, true); + + StoreFile.Reader reader = storeFile.createReader(); + + LOG.info("First key: " + Bytes.toStringBinary(reader.getFirstKey())); + LOG.info("Last key: " + Bytes.toStringBinary(reader.getLastKey())); + + KeyValue firstKV = KeyValue.createKeyValueFromKey(reader.getFirstKey()); + firstRow = firstKV.getRow(); + + KeyValue lastKV = KeyValue.createKeyValueFromKey(reader.getLastKey()); + lastRow = lastKV.getRow(); + + byte[] family = firstKV.getFamily(); + if (!Bytes.equals(family, lastKV.getFamily())) { + LOG.error("First and last key have different families: " + + Bytes.toStringBinary(family) + " and " + + Bytes.toStringBinary(lastKV.getFamily())); + return false; + } + + if (Bytes.equals(firstRow, lastRow)) { + LOG.error("First and last row are the same, cannot run read workload: " + + "firstRow=" + Bytes.toStringBinary(firstRow) + ", " + + "lastRow=" + Bytes.toStringBinary(lastRow)); + return false; + } + + ExecutorService exec = Executors.newFixedThreadPool(numReadThreads + 1); + int numCompleted = 0; + int numFailed = 0; + try { + ExecutorCompletionService ecs = + new ExecutorCompletionService(exec); + endTime = System.currentTimeMillis() + 1000 * durationSec; + boolean pread = true; + for (int i = 0; i < numReadThreads; ++i) + ecs.submit(new RandomReader(i, reader, pread)); + ecs.submit(new StatisticsPrinter()); + Future result; + while (true) { + try { + result = ecs.poll(endTime + 1000 - System.currentTimeMillis(), + TimeUnit.MILLISECONDS); + if (result == null) + break; + try { + if (result.get()) { + ++numCompleted; + } else { + ++numFailed; + } + } catch (ExecutionException e) { + LOG.error("Worker thread failure", e.getCause()); + ++numFailed; + } + } catch (InterruptedException ex) { + LOG.error("Interrupted after " + numCompleted + + " workers completed"); + Thread.currentThread().interrupt(); + continue; + } + + } + } finally { + storeFile.closeReader(true); + exec.shutdown(); + + BlockCache c = cacheConf.getBlockCache(); + if (c != null) { + c.shutdown(); + } + } + LOG.info("Worker threads completed: " + numCompleted); + LOG.info("Worker threads failed: " + numFailed); + return true; + } + + public boolean run() throws IOException { + LOG.info("Workload: " + workload); + switch (workload) { + case MERGE: + runMergeWorkload(); + break; + case RANDOM_READS: + return runRandomReadWorkload(); + default: + LOG.error("Unknown workload: " + workload); + return false; + } + + return true; + } + + private static void failure() { + System.exit(1); + } + + public static void main(String[] args) { + HFileReadWriteTest app = new HFileReadWriteTest(); + if (!app.parseOptions(args)) + failure(); + + try { + if (!app.validateConfiguration() || + !app.run()) + failure(); + } catch (IOException ex) { + LOG.error(ex); + failure(); + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/KeyValueScanFixture.java b/src/test/java/org/apache/hadoop/hbase/regionserver/KeyValueScanFixture.java new file mode 100644 index 0000000..08fb91e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/KeyValueScanFixture.java @@ -0,0 +1,49 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.util.CollectionBackedScanner; +import org.apache.hadoop.hbase.KeyValue; + +import java.util.ArrayList; +import java.util.List; + +/** + * A fixture that implements and presents a KeyValueScanner. + * It takes a list of key/values which is then sorted according + * to the provided comparator, and then the whole thing pretends + * to be a store file scanner. + */ +public class KeyValueScanFixture extends CollectionBackedScanner { + public KeyValueScanFixture(KeyValue.KVComparator comparator, + KeyValue... incData) { + super(comparator, incData); + } + + public static List scanFixture(KeyValue[] ... kvArrays) { + ArrayList scanners = new ArrayList(); + for (KeyValue [] kvs : kvArrays) { + scanners.add(new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)); + } + return scanners; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java b/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java new file mode 100644 index 0000000..c933ade --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.NavigableSet; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.TestFromClientSideWithCoprocessor; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; + +/** + * RegionObserver that just reimplements the default behavior, + * in order to validate that all the necessary APIs for this are public + * This observer is also used in {@link TestFromClientSideWithCoprocessor} and + * {@link TestCompactionWithCoprocessor} to make sure that a wide range + * of functionality still behaves as expected. + */ +public class NoOpScanPolicyObserver extends BaseRegionObserver { + /** + * Reimplement the default behavior + */ + @Override + public InternalScanner preFlushScannerOpen(final ObserverContext c, + Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException { + Store.ScanInfo oldSI = store.getScanInfo(); + Store.ScanInfo scanInfo = new Store.ScanInfo(store.getFamily(), oldSI.getTtl(), + oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + Scan scan = new Scan(); + scan.setMaxVersions(oldSI.getMaxVersions()); + return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner), + ScanType.MINOR_COMPACT, store.getHRegion().getSmallestReadPoint(), + HConstants.OLDEST_TIMESTAMP); + } + + /** + * Reimplement the default behavior + */ + @Override + public InternalScanner preCompactScannerOpen(final ObserverContext c, + Store store, List scanners, ScanType scanType, long earliestPutTs, + InternalScanner s) throws IOException { + // this demonstrates how to override the scanners default behavior + Store.ScanInfo oldSI = store.getScanInfo(); + Store.ScanInfo scanInfo = new Store.ScanInfo(store.getFamily(), oldSI.getTtl(), + oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + Scan scan = new Scan(); + scan.setMaxVersions(oldSI.getMaxVersions()); + return new StoreScanner(store, scanInfo, scan, scanners, scanType, store.getHRegion() + .getSmallestReadPoint(), earliestPutTs); + } + + @Override + public KeyValueScanner preStoreScannerOpen(final ObserverContext c, + Store store, final Scan scan, final NavigableSet targetCols, KeyValueScanner s) + throws IOException { + return new StoreScanner(store, store.getScanInfo(), scan, targetCols); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java b/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java new file mode 100644 index 0000000..cac2989 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java @@ -0,0 +1,55 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.Put; + +/** + * A region server that will OOME. + * Everytime {@link #put(regionName, Put)} is called, we add + * keep around a reference to the batch. Use this class to test OOME extremes. + * Needs to be started manually as in + * ${HBASE_HOME}/bin/hbase ./bin/hbase org.apache.hadoop.hbase.OOMERegionServer start. + */ +public class OOMERegionServer extends HRegionServer { + private List retainer = new ArrayList(); + + public OOMERegionServer(HBaseConfiguration conf) throws IOException, InterruptedException { + super(conf); + } + + public void put(byte [] regionName, Put put) + throws IOException { + super.put(regionName, put); + for (int i = 0; i < 30; i++) { + // Add the batch update 30 times to bring on the OOME faster. + this.retainer.add(put); + } + } + + public static void main(String[] args) throws Exception { + new HRegionServerCommandLine(OOMERegionServer.class).doMain(args); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java new file mode 100644 index 0000000..6cbb2bc --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java @@ -0,0 +1,447 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.RowMutations; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; +import org.junit.experimental.categories.Category; + + +/** + * Testing of HRegion.incrementColumnValue, HRegion.increment, + * and HRegion.append + */ +@Category(MediumTests.class) // Starts 100 threads +public class TestAtomicOperation extends HBaseTestCase { + static final Log LOG = LogFactory.getLog(TestAtomicOperation.class); + + HRegion region = null; + private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final String DIR = TEST_UTIL.getDataTestDir("TestAtomicOperation").toString(); + + + // Test names + static final byte[] tableName = Bytes.toBytes("testtable");; + static final byte[] qual1 = Bytes.toBytes("qual1"); + static final byte[] qual2 = Bytes.toBytes("qual2"); + static final byte[] qual3 = Bytes.toBytes("qual3"); + static final byte[] value1 = Bytes.toBytes("value1"); + static final byte[] value2 = Bytes.toBytes("value2"); + static final byte [] row = Bytes.toBytes("rowA"); + static final byte [] row2 = Bytes.toBytes("rowB"); + + /** + * @see org.apache.hadoop.hbase.HBaseTestCase#setUp() + */ + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + EnvironmentEdgeManagerTestHelper.reset(); + } + + ////////////////////////////////////////////////////////////////////////////// + // New tests that doesn't spin up a mini cluster but rather just test the + // individual code pieces in the HRegion. + ////////////////////////////////////////////////////////////////////////////// + + /** + * Test basic append operation. + * More tests in + * @see org.apache.hadoop.hbase.client.TestFromClientSide#testAppend() + */ + public void testAppend() throws IOException { + initHRegion(tableName, getName(), fam1); + String v1 = "Ultimate Answer to the Ultimate Question of Life,"+ + " The Universe, and Everything"; + String v2 = " is... 42."; + Append a = new Append(row); + a.setReturnResults(false); + a.add(fam1, qual1, Bytes.toBytes(v1)); + a.add(fam1, qual2, Bytes.toBytes(v2)); + assertNull(region.append(a, null, true)); + a = new Append(row); + a.add(fam1, qual1, Bytes.toBytes(v2)); + a.add(fam1, qual2, Bytes.toBytes(v1)); + Result result = region.append(a, null, true); + assertEquals(0, Bytes.compareTo(Bytes.toBytes(v1+v2), result.getValue(fam1, qual1))); + assertEquals(0, Bytes.compareTo(Bytes.toBytes(v2+v1), result.getValue(fam1, qual2))); + } + + /** + * Test one increment command. + */ + public void testIncrementColumnValue() throws IOException { + LOG.info("Starting test testIncrementColumnValue"); + initHRegion(tableName, getName(), fam1); + + long value = 1L; + long amount = 3L; + + Put put = new Put(row); + put.add(fam1, qual1, Bytes.toBytes(value)); + region.put(put); + + long result = region.incrementColumnValue(row, fam1, qual1, amount, true); + + assertEquals(value+amount, result); + + Store store = region.getStore(fam1); + // ICV removes any extra values floating around in there. + assertEquals(1, store.memstore.kvset.size()); + assertTrue(store.memstore.snapshot.isEmpty()); + + assertICV(row, fam1, qual1, value+amount); + } + + /** + * Test multi-threaded increments. + */ + public void testIncrementMultiThreads() throws IOException { + + LOG.info("Starting test testIncrementMultiThreads"); + initHRegion(tableName, getName(), fam1); + + // create 100 threads, each will increment by its own quantity + int numThreads = 100; + int incrementsPerThread = 1000; + Incrementer[] all = new Incrementer[numThreads]; + int expectedTotal = 0; + + // create all threads + for (int i = 0; i < numThreads; i++) { + all[i] = new Incrementer(region, i, i, incrementsPerThread); + expectedTotal += (i * incrementsPerThread); + } + + // run all threads + for (int i = 0; i < numThreads; i++) { + all[i].start(); + } + + // wait for all threads to finish + for (int i = 0; i < numThreads; i++) { + try { + all[i].join(); + } catch (InterruptedException e) { + } + } + assertICV(row, fam1, qual1, expectedTotal); + LOG.info("testIncrementMultiThreads successfully verified that total is " + + expectedTotal); + } + + + private void assertICV(byte [] row, + byte [] familiy, + byte[] qualifier, + long amount) throws IOException { + // run a get and see? + Get get = new Get(row); + get.addColumn(familiy, qualifier); + Result result = region.get(get, null); + assertEquals(1, result.size()); + + KeyValue kv = result.raw()[0]; + long r = Bytes.toLong(kv.getValue()); + assertEquals(amount, r); + } + + private void initHRegion (byte [] tableName, String callingMethod, + byte[] ... families) + throws IOException { + initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families); + } + + private void initHRegion (byte [] tableName, String callingMethod, + Configuration conf, byte [] ... families) + throws IOException{ + HTableDescriptor htd = new HTableDescriptor(tableName); + for(byte [] family : families) { + htd.addFamily(new HColumnDescriptor(family)); + } + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + Path path = new Path(DIR + callingMethod); + if (fs.exists(path)) { + if (!fs.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + region = HRegion.createHRegion(info, path, conf, htd); + } + + /** + * A thread that makes a few increment calls + */ + public static class Incrementer extends Thread { + + private final HRegion region; + private final int threadNumber; + private final int numIncrements; + private final int amount; + + private int count; + + public Incrementer(HRegion region, + int threadNumber, int amount, int numIncrements) { + this.region = region; + this.threadNumber = threadNumber; + this.numIncrements = numIncrements; + this.count = 0; + this.amount = amount; + setDaemon(true); + } + + @Override + public void run() { + for (int i=0; i rowsToLock = Arrays.asList(row, row2); + // create all threads + for (int i = 0; i < numThreads; i++) { + all[i] = new AtomicOperation(region, opsPerThread, timeStamps, failures) { + @Override + public void run() { + boolean op = true; + for (int i=0; i mrm = new ArrayList(); + if (op) { + Put p = new Put(row2, ts); + p.add(fam1, qual1, value1); + mrm.add(p); + Delete d = new Delete(row); + d.deleteColumns(fam1, qual1, ts); + mrm.add(d); + } else { + Delete d = new Delete(row2); + d.deleteColumns(fam1, qual1, ts); + mrm.add(d); + Put p = new Put(row, ts); + p.add(fam1, qual1, value2); + mrm.add(p); + } + region.mutateRowsWithLocks(mrm, rowsToLock); + op ^= true; + // check: should always see exactly one column + Scan s = new Scan(row); + RegionScanner rs = region.getScanner(s); + List r = new ArrayList(); + while(rs.next(r)); + rs.close(); + if (r.size() != 1) { + LOG.debug(r); + failures.incrementAndGet(); + fail(); + } + } catch (IOException e) { + e.printStackTrace(); + failures.incrementAndGet(); + fail(); + } + } + } + }; + } + + // run all threads + for (int i = 0; i < numThreads; i++) { + all[i].start(); + } + + // wait for all threads to finish + for (int i = 0; i < numThreads; i++) { + try { + all[i].join(); + } catch (InterruptedException e) { + } + } + assertEquals(0, failures.get()); + } + + public static class AtomicOperation extends Thread { + protected final HRegion region; + protected final int numOps; + protected final AtomicLong timeStamps; + protected final AtomicInteger failures; + protected final Random r = new Random(); + + public AtomicOperation(HRegion region, int numOps, AtomicLong timeStamps, + AtomicInteger failures) { + this.region = region; + this.numOps = numOps; + this.timeStamps = timeStamps; + this.failures = failures; + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java new file mode 100644 index 0000000..3813ea4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java @@ -0,0 +1,459 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestBlocksRead extends HBaseTestCase { + static final Log LOG = LogFactory.getLog(TestBlocksRead.class); + static final BloomType[] BLOOM_TYPE = new BloomType[] { BloomType.ROWCOL, + BloomType.ROW, BloomType.NONE }; + + private static BlockCache blockCache; + + private HBaseConfiguration getConf() { + HBaseConfiguration conf = new HBaseConfiguration(); + + // disable compactions in this test. + conf.setInt("hbase.hstore.compactionThreshold", 10000); + return conf; + } + + HRegion region = null; + private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final String DIR = TEST_UTIL.getDataTestDir("TestBlocksRead").toString(); + + /** + * @see org.apache.hadoop.hbase.HBaseTestCase#setUp() + */ + @SuppressWarnings("deprecation") + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + @SuppressWarnings("deprecation") + @Override + protected void tearDown() throws Exception { + super.tearDown(); + EnvironmentEdgeManagerTestHelper.reset(); + } + + /** + * Callers must afterward call {@link HRegion#closeHRegion(HRegion)} + * @param tableName + * @param callingMethod + * @param conf + * @param families + * @throws IOException + * @return created and initialized region. + */ + private HRegion initHRegion(byte[] tableName, String callingMethod, + HBaseConfiguration conf, String family) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor familyDesc; + for (int i = 0; i < BLOOM_TYPE.length; i++) { + BloomType bloomType = BLOOM_TYPE[i]; + familyDesc = new HColumnDescriptor(family + "_" + bloomType) + .setBlocksize(1) + .setBloomFilterType(BLOOM_TYPE[i]); + htd.addFamily(familyDesc); + } + + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + Path path = new Path(DIR + callingMethod); + HRegion r = HRegion.createHRegion(info, path, conf, htd); + blockCache = new CacheConfig(conf).getBlockCache(); + return r; + } + + private void putData(String family, String row, String col, long version) + throws IOException { + for (int i = 0; i < BLOOM_TYPE.length; i++) { + putData(Bytes.toBytes(family + "_" + BLOOM_TYPE[i]), row, col, version, + version); + } + } + + // generates a value to put for a row/col/version. + private static byte[] genValue(String row, String col, long version) { + return Bytes.toBytes("Value:" + row + "#" + col + "#" + version); + } + + private void putData(byte[] cf, String row, String col, long versionStart, + long versionEnd) throws IOException { + byte columnBytes[] = Bytes.toBytes(col); + Put put = new Put(Bytes.toBytes(row)); + put.setWriteToWAL(false); + + for (long version = versionStart; version <= versionEnd; version++) { + put.add(cf, columnBytes, version, genValue(row, col, version)); + } + region.put(put); + } + + private KeyValue[] getData(String family, String row, List columns, + int expBlocks) throws IOException { + return getData(family, row, columns, expBlocks, expBlocks, expBlocks); + } + + private KeyValue[] getData(String family, String row, List columns, + int expBlocksRowCol, int expBlocksRow, int expBlocksNone) + throws IOException { + int[] expBlocks = new int[] { expBlocksRowCol, expBlocksRow, expBlocksNone }; + KeyValue[] kvs = null; + + for (int i = 0; i < BLOOM_TYPE.length; i++) { + BloomType bloomType = BLOOM_TYPE[i]; + byte[] cf = Bytes.toBytes(family + "_" + bloomType); + long blocksStart = getBlkAccessCount(cf); + Get get = new Get(Bytes.toBytes(row)); + + for (String column : columns) { + get.addColumn(cf, Bytes.toBytes(column)); + } + + kvs = region.get(get, null).raw(); + long blocksEnd = getBlkAccessCount(cf); + if (expBlocks[i] != -1) { + assertEquals("Blocks Read Check for Bloom: " + bloomType, expBlocks[i], + blocksEnd - blocksStart); + } + System.out.println("Blocks Read for Bloom: " + bloomType + " = " + + (blocksEnd - blocksStart) + "Expected = " + expBlocks[i]); + } + return kvs; + } + + private KeyValue[] getData(String family, String row, String column, + int expBlocks) throws IOException { + return getData(family, row, Arrays.asList(column), expBlocks, expBlocks, + expBlocks); + } + + private KeyValue[] getData(String family, String row, String column, + int expBlocksRowCol, int expBlocksRow, int expBlocksNone) + throws IOException { + return getData(family, row, Arrays.asList(column), expBlocksRowCol, + expBlocksRow, expBlocksNone); + } + + private void deleteFamily(String family, String row, long version) + throws IOException { + Delete del = new Delete(Bytes.toBytes(row)); + del.deleteFamily(Bytes.toBytes(family + "_ROWCOL"), version); + del.deleteFamily(Bytes.toBytes(family + "_ROW"), version); + del.deleteFamily(Bytes.toBytes(family + "_NONE"), version); + region.delete(del, null, true); + } + + private static void verifyData(KeyValue kv, String expectedRow, + String expectedCol, long expectedVersion) { + assertEquals("RowCheck", expectedRow, Bytes.toString(kv.getRow())); + assertEquals("ColumnCheck", expectedCol, Bytes.toString(kv.getQualifier())); + assertEquals("TSCheck", expectedVersion, kv.getTimestamp()); + assertEquals("ValueCheck", + Bytes.toString(genValue(expectedRow, expectedCol, expectedVersion)), + Bytes.toString(kv.getValue())); + } + + private static long getBlkAccessCount(byte[] cf) { + return HFile.dataBlockReadCnt.get(); + } + + private static long getBlkCount() { + return blockCache.getBlockCount(); + } + + /** + * Test # of blocks read for some simple seek cases. + * + * @throws Exception + */ + @Test + public void testBlocksRead() throws Exception { + byte[] TABLE = Bytes.toBytes("testBlocksRead"); + String FAMILY = "cf1"; + KeyValue kvs[]; + HBaseConfiguration conf = getConf(); + this.region = initHRegion(TABLE, getName(), conf, FAMILY); + + try { + putData(FAMILY, "row", "col1", 1); + putData(FAMILY, "row", "col2", 2); + putData(FAMILY, "row", "col3", 3); + putData(FAMILY, "row", "col4", 4); + putData(FAMILY, "row", "col5", 5); + putData(FAMILY, "row", "col6", 6); + putData(FAMILY, "row", "col7", 7); + region.flushcache(); + + // Expected block reads: 1 + // The top block has the KV we are + // interested. So only 1 seek is needed. + kvs = getData(FAMILY, "row", "col1", 1); + assertEquals(1, kvs.length); + verifyData(kvs[0], "row", "col1", 1); + + // Expected block reads: 2 + // The top block and next block has the KVs we are + // interested. So only 2 seek is needed. + kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 2); + assertEquals(2, kvs.length); + verifyData(kvs[0], "row", "col1", 1); + verifyData(kvs[1], "row", "col2", 2); + + // Expected block reads: 3 + // The first 2 seeks is to find out col2. [HBASE-4443] + // One additional seek for col3 + // So 3 seeks are needed. + kvs = getData(FAMILY, "row", Arrays.asList("col2", "col3"), 3); + assertEquals(2, kvs.length); + verifyData(kvs[0], "row", "col2", 2); + verifyData(kvs[1], "row", "col3", 3); + + // Expected block reads: 2. [HBASE-4443] + kvs = getData(FAMILY, "row", Arrays.asList("col5"), 2); + assertEquals(1, kvs.length); + verifyData(kvs[0], "row", "col5", 5); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + /** + * Test # of blocks read (targetted at some of the cases Lazy Seek optimizes). + * + * @throws Exception + */ + @Test + public void testLazySeekBlocksRead() throws Exception { + byte[] TABLE = Bytes.toBytes("testLazySeekBlocksRead"); + String FAMILY = "cf1"; + KeyValue kvs[]; + HBaseConfiguration conf = getConf(); + this.region = initHRegion(TABLE, getName(), conf, FAMILY); + + try { + // File 1 + putData(FAMILY, "row", "col1", 1); + putData(FAMILY, "row", "col2", 2); + region.flushcache(); + + // File 2 + putData(FAMILY, "row", "col1", 3); + putData(FAMILY, "row", "col2", 4); + region.flushcache(); + + // Expected blocks read: 1. + // File 2's top block is also the KV we are + // interested. So only 1 seek is needed. + kvs = getData(FAMILY, "row", Arrays.asList("col1"), 1); + assertEquals(1, kvs.length); + verifyData(kvs[0], "row", "col1", 3); + + // Expected blocks read: 2 + // File 2's top block has the "col1" KV we are + // interested. We also need "col2" which is in a block + // of its own. So, we need that block as well. + kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 2); + assertEquals(2, kvs.length); + verifyData(kvs[0], "row", "col1", 3); + verifyData(kvs[1], "row", "col2", 4); + + // File 3: Add another column + putData(FAMILY, "row", "col3", 5); + region.flushcache(); + + // Expected blocks read: 1 + // File 3's top block has the "col3" KV we are + // interested. So only 1 seek is needed. + kvs = getData(FAMILY, "row", "col3", 1); + assertEquals(1, kvs.length); + verifyData(kvs[0], "row", "col3", 5); + + // Get a column from older file. + // For ROWCOL Bloom filter: Expected blocks read: 1. + // For ROW Bloom filter: Expected blocks read: 2. + // For NONE Bloom filter: Expected blocks read: 2. + kvs = getData(FAMILY, "row", Arrays.asList("col1"), 1, 2, 2); + assertEquals(1, kvs.length); + verifyData(kvs[0], "row", "col1", 3); + + // File 4: Delete the entire row. + deleteFamily(FAMILY, "row", 6); + region.flushcache(); + + // For ROWCOL Bloom filter: Expected blocks read: 2. + // For ROW Bloom filter: Expected blocks read: 3. + // For NONE Bloom filter: Expected blocks read: 3. + kvs = getData(FAMILY, "row", "col1", 2, 3, 3); + assertEquals(0, kvs.length); + kvs = getData(FAMILY, "row", "col2", 3, 4, 4); + assertEquals(0, kvs.length); + kvs = getData(FAMILY, "row", "col3", 2); + assertEquals(0, kvs.length); + kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 4); + assertEquals(0, kvs.length); + + // File 5: Delete + deleteFamily(FAMILY, "row", 10); + region.flushcache(); + + // File 6: some more puts, but with timestamps older than the + // previous delete. + putData(FAMILY, "row", "col1", 7); + putData(FAMILY, "row", "col2", 8); + putData(FAMILY, "row", "col3", 9); + region.flushcache(); + + // Baseline expected blocks read: 8. [HBASE-4532] + kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 5); + assertEquals(0, kvs.length); + + // File 7: Put back new data + putData(FAMILY, "row", "col1", 11); + putData(FAMILY, "row", "col2", 12); + putData(FAMILY, "row", "col3", 13); + region.flushcache(); + + + // Expected blocks read: 5. [HBASE-4585] + kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 5); + assertEquals(3, kvs.length); + verifyData(kvs[0], "row", "col1", 11); + verifyData(kvs[1], "row", "col2", 12); + verifyData(kvs[2], "row", "col3", 13); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + /** + * Test # of blocks read to ensure disabling cache-fill on Scan works. + * @throws Exception + */ + @Test + public void testBlocksStoredWhenCachingDisabled() throws Exception { + byte [] TABLE = Bytes.toBytes("testBlocksReadWhenCachingDisabled"); + String FAMILY = "cf1"; + + HBaseConfiguration conf = getConf(); + this.region = initHRegion(TABLE, getName(), conf, FAMILY); + + try { + putData(FAMILY, "row", "col1", 1); + putData(FAMILY, "row", "col2", 2); + region.flushcache(); + + // Execute a scan with caching turned off + // Expected blocks stored: 0 + long blocksStart = getBlkCount(); + Scan scan = new Scan(); + scan.setCacheBlocks(false); + RegionScanner rs = region.getScanner(scan); + List result = new ArrayList(2); + rs.next(result); + assertEquals(2 * BLOOM_TYPE.length, result.size()); + rs.close(); + long blocksEnd = getBlkCount(); + + assertEquals(blocksStart, blocksEnd); + + // Execute with caching turned on + // Expected blocks stored: 2 + blocksStart = blocksEnd; + scan.setCacheBlocks(true); + rs = region.getScanner(scan); + result = new ArrayList(2); + rs.next(result); + assertEquals(2 * BLOOM_TYPE.length, result.size()); + rs.close(); + blocksEnd = getBlkCount(); + + assertEquals(2 * BLOOM_TYPE.length, blocksEnd - blocksStart); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + @Test + public void testLazySeekBlocksReadWithDelete() throws Exception { + byte[] TABLE = Bytes.toBytes("testLazySeekBlocksReadWithDelete"); + String FAMILY = "cf1"; + KeyValue kvs[]; + HBaseConfiguration conf = getConf(); + this.region = initHRegion(TABLE, getName(), conf, FAMILY); + try { + deleteFamily(FAMILY, "row", 200); + for (int i = 0; i < 100; i++) { + putData(FAMILY, "row", "col" + i, i); + } + putData(FAMILY, "row", "col99", 201); + region.flushcache(); + + kvs = getData(FAMILY, "row", Arrays.asList("col0"), 2); + assertEquals(0, kvs.length); + + kvs = getData(FAMILY, "row", Arrays.asList("col99"), 2); + assertEquals(1, kvs.length); + verifyData(kvs[0], "row", "col99", 201); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java new file mode 100644 index 0000000..19c690f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@SuppressWarnings("deprecation") +@Category(SmallTests.class) +public class TestBlocksScanned extends HBaseTestCase { + private static byte [] TABLE = Bytes.toBytes("TestBlocksScanned"); + private static byte [] FAMILY = Bytes.toBytes("family"); + private static byte [] COL = Bytes.toBytes("col"); + private static byte [] START_KEY = Bytes.toBytes("aaa"); + private static byte [] END_KEY = Bytes.toBytes("zzz"); + private static int BLOCK_SIZE = 70; + + private static HBaseTestingUtility TEST_UTIL = null; + private static HTableDescriptor TESTTABLEDESC = null; + + @Override + public void setUp() throws Exception { + super.setUp(); + SchemaMetrics.setUseTableNameInTest(true); + TEST_UTIL = new HBaseTestingUtility(); + TESTTABLEDESC = new HTableDescriptor(TABLE); + + TESTTABLEDESC.addFamily( + new HColumnDescriptor(FAMILY) + .setMaxVersions(10) + .setBlockCacheEnabled(true) + .setBlocksize(BLOCK_SIZE) + .setCompressionType(Compression.Algorithm.NONE) + ); + } + + @Test + public void testBlocksScanned() throws Exception { + HRegion r = createNewHRegion(TESTTABLEDESC, START_KEY, END_KEY, + TEST_UTIL.getConfiguration()); + addContent(r, FAMILY, COL); + r.flushcache(); + + // Get the per-cf metrics + SchemaMetrics schemaMetrics = + SchemaMetrics.getInstance(Bytes.toString(TABLE), Bytes.toString(FAMILY)); + Map schemaMetricSnapshot = SchemaMetrics.getMetricsSnapshot(); + + // Do simple test of getting one row only first. + Scan scan = new Scan(Bytes.toBytes("aaa"), Bytes.toBytes("aaz")); + scan.addColumn(FAMILY, COL); + scan.setMaxVersions(1); + + InternalScanner s = r.getScanner(scan); + List results = new ArrayList(); + while (s.next(results)); + s.close(); + + int expectResultSize = 'z' - 'a'; + Assert.assertEquals(expectResultSize, results.size()); + + int kvPerBlock = (int) Math.ceil(BLOCK_SIZE / (double) results.get(0).getLength()); + Assert.assertEquals(2, kvPerBlock); + + long expectDataBlockRead = (long) Math.ceil(expectResultSize / (double) kvPerBlock); + long expectIndexBlockRead = expectDataBlockRead; + + verifyDataAndIndexBlockRead(schemaMetricSnapshot, schemaMetrics, + expectDataBlockRead, expectIndexBlockRead); + } + + private void verifyDataAndIndexBlockRead(Map previousMetricSnapshot, + SchemaMetrics schemaMetrics, long expectDataBlockRead, long expectedIndexBlockRead){ + Map currentMetricsSnapshot = SchemaMetrics.getMetricsSnapshot(); + Map diffs = + SchemaMetrics.diffMetrics(previousMetricSnapshot, currentMetricsSnapshot); + + long dataBlockRead = SchemaMetrics.getLong(diffs, + schemaMetrics.getBlockMetricName(BlockCategory.DATA, false, BlockMetricType.READ_COUNT)); + long indexBlockRead = SchemaMetrics.getLong(diffs, + schemaMetrics.getBlockMetricName(BlockCategory.INDEX, false, BlockMetricType.READ_COUNT)); + + Assert.assertEquals(expectDataBlockRead, dataBlockRead); + Assert.assertEquals(expectedIndexBlockRead, indexBlockRead); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java new file mode 100644 index 0000000..863b97c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java @@ -0,0 +1,271 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.io.hfile.BlockType; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileBlock; +import org.apache.hadoop.hbase.io.hfile.HFileReaderV2; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.io.hfile.TestHFileWriterV2; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Bytes; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Tests {@link HFile} cache-on-write functionality for data blocks, non-root + * index blocks, and Bloom filter blocks, as specified by the column family. + */ +@RunWith(Parameterized.class) +@Category(MediumTests.class) +public class TestCacheOnWriteInSchema { + + private static final Log LOG = LogFactory.getLog(TestCacheOnWriteInSchema.class); + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String DIR = TEST_UTIL.getDataTestDir("TestCacheOnWriteInSchema").toString(); + private static final byte [] table = Bytes.toBytes("table"); + private static byte [] family = Bytes.toBytes("family"); + private static final int NUM_KV = 25000; + private static final Random rand = new Random(12983177L); + /** The number of valid key types possible in a store file */ + private static final int NUM_VALID_KEY_TYPES = + KeyValue.Type.values().length - 2; + + private static enum CacheOnWriteType { + DATA_BLOCKS(BlockType.DATA, BlockType.ENCODED_DATA), + BLOOM_BLOCKS(BlockType.BLOOM_CHUNK), + INDEX_BLOCKS(BlockType.LEAF_INDEX, BlockType.INTERMEDIATE_INDEX); + + private final BlockType blockType1; + private final BlockType blockType2; + + private CacheOnWriteType(BlockType blockType) { + this(blockType, blockType); + } + + private CacheOnWriteType(BlockType blockType1, BlockType blockType2) { + this.blockType1 = blockType1; + this.blockType2 = blockType2; + } + + public boolean shouldBeCached(BlockType blockType) { + return blockType == blockType1 || blockType == blockType2; + } + + public void modifyFamilySchema(HColumnDescriptor family) { + switch (this) { + case DATA_BLOCKS: + family.setCacheDataOnWrite(true); + break; + case BLOOM_BLOCKS: + family.setCacheBloomsOnWrite(true); + break; + case INDEX_BLOCKS: + family.setCacheIndexesOnWrite(true); + break; + } + } + } + + private final CacheOnWriteType cowType; + private Configuration conf; + private final String testDescription; + private Store store; + private FileSystem fs; + + public TestCacheOnWriteInSchema(CacheOnWriteType cowType) { + this.cowType = cowType; + testDescription = "[cacheOnWrite=" + cowType + "]"; + System.out.println(testDescription); + } + + @Parameters + public static Collection getParameters() { + List cowTypes = new ArrayList(); + for (CacheOnWriteType cowType : CacheOnWriteType.values()) { + cowTypes.add(new Object[] { cowType }); + } + return cowTypes; + } + + @Before + public void setUp() throws IOException { + conf = TEST_UTIL.getConfiguration(); + conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION); + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false); + conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false); + conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, false); + + fs = HFileSystem.get(conf); + + // Create the schema + HColumnDescriptor hcd = new HColumnDescriptor(family); + hcd.setBloomFilterType(BloomType.ROWCOL); + cowType.modifyFamilySchema(hcd); + HTableDescriptor htd = new HTableDescriptor(table); + htd.addFamily(hcd); + + // Create a store based on the schema + Path basedir = new Path(DIR); + Path logdir = new Path(DIR+"/logs"); + Path oldLogDir = new Path(basedir, HConstants.HREGION_OLDLOGDIR_NAME); + fs.delete(logdir, true); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HLog hlog = new HLog(fs, logdir, oldLogDir, conf); + HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); + store = new Store(basedir, region, hcd, fs, conf); + } + + @After + public void tearDown() { + try { + fs.delete(new Path(DIR), true); + } catch (IOException e) { + LOG.error("Could not delete " + DIR, e); + } + } + + @Test + public void testCacheOnWriteInSchema() throws IOException { + // Write some random data into the store + StoreFile.Writer writer = store.createWriterInTmp(Integer.MAX_VALUE, + Compression.Algorithm.NONE, false, true); + writeStoreFile(writer); + writer.close(); + // Verify the block types of interest were cached on write + readStoreFile(writer.getPath()); + } + + private void readStoreFile(Path path) throws IOException { + CacheConfig cacheConf = store.getCacheConfig(); + BlockCache cache = cacheConf.getBlockCache(); + StoreFile sf = new StoreFile(fs, path, conf, cacheConf, + BloomType.ROWCOL, null); + store.passSchemaMetricsTo(sf); + HFileReaderV2 reader = (HFileReaderV2) sf.createReader().getHFileReader(); + try { + // Open a scanner with (on read) caching disabled + HFileScanner scanner = reader.getScanner(false, false); + assertTrue(testDescription, scanner.seekTo()); + // Cribbed from io.hfile.TestCacheOnWrite + long offset = 0; + HFileBlock prevBlock = null; + while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { + long onDiskSize = -1; + if (prevBlock != null) { + onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader(); + } + // Flags: don't cache the block, use pread, this is not a compaction. + // Also, pass null for expected block type to avoid checking it. + HFileBlock block = reader.readBlock(offset, onDiskSize, false, true, + false, null); + BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(), + offset); + boolean isCached = cache.getBlock(blockCacheKey, true, false) != null; + boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType()); + if (shouldBeCached != isCached) { + throw new AssertionError( + "shouldBeCached: " + shouldBeCached+ "\n" + + "isCached: " + isCached + "\n" + + "Test description: " + testDescription + "\n" + + "block: " + block + "\n" + + "blockCacheKey: " + blockCacheKey); + } + prevBlock = block; + offset += block.getOnDiskSizeWithHeader(); + } + } finally { + reader.close(); + } + } + + private static KeyValue.Type generateKeyType(Random rand) { + if (rand.nextBoolean()) { + // Let's make half of KVs puts. + return KeyValue.Type.Put; + } else { + KeyValue.Type keyType = + KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)]; + if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) + { + throw new RuntimeException("Generated an invalid key type: " + keyType + + ". " + "Probably the layout of KeyValue.Type has changed."); + } + return keyType; + } + } + + private void writeStoreFile(StoreFile.Writer writer) throws IOException { + final int rowLen = 32; + for (int i = 0; i < NUM_KV; ++i) { + byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i); + byte[] v = TestHFileWriterV2.randomValue(rand); + int cfLen = rand.nextInt(k.length - rowLen + 1); + KeyValue kv = new KeyValue( + k, 0, rowLen, + k, rowLen, cfLen, + k, rowLen + cfLen, k.length - rowLen - cfLen, + rand.nextLong(), + generateKeyType(rand), + v, 0, v.length); + writer.append(kv); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java new file mode 100644 index 0000000..e865a8a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java @@ -0,0 +1,298 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestColumnSeeking { + + private final static HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + static final Log LOG = LogFactory.getLog(TestColumnSeeking.class); + + @SuppressWarnings("unchecked") + @Test + public void testDuplicateVersions() throws IOException { + String family = "Family"; + byte[] familyBytes = Bytes.toBytes("Family"); + String table = "TestDuplicateVersions"; + + HColumnDescriptor hcd = + new HColumnDescriptor(familyBytes).setMaxVersions(1000); + HTableDescriptor htd = new HTableDescriptor(table); + htd.addFamily(hcd); + HRegionInfo info = new HRegionInfo(Bytes.toBytes(table), null, null, false); + HRegion region = + HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL + .getConfiguration(), htd); + try { + List rows = generateRandomWords(10, "row"); + List allColumns = generateRandomWords(10, "column"); + List values = generateRandomWords(100, "value"); + + long maxTimestamp = 2; + double selectPercent = 0.5; + int numberOfTests = 5; + double flushPercentage = 0.2; + double minorPercentage = 0.2; + double majorPercentage = 0.2; + double putPercentage = 0.2; + + HashMap allKVMap = new HashMap(); + + HashMap[] kvMaps = new HashMap[numberOfTests]; + ArrayList[] columnLists = new ArrayList[numberOfTests]; + + for (int i = 0; i < numberOfTests; i++) { + kvMaps[i] = new HashMap(); + columnLists[i] = new ArrayList(); + for (String column : allColumns) { + if (Math.random() < selectPercent) { + columnLists[i].add(column); + } + } + } + + for (String value : values) { + for (String row : rows) { + Put p = new Put(Bytes.toBytes(row)); + p.setWriteToWAL(false); + for (String column : allColumns) { + for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { + KeyValue kv = + KeyValueTestUtil.create(row, family, column, timestamp, value); + if (Math.random() < putPercentage) { + p.add(kv); + allKVMap.put(kv.getKeyString(), kv); + for (int i = 0; i < numberOfTests; i++) { + if (columnLists[i].contains(column)) { + kvMaps[i].put(kv.getKeyString(), kv); + } + } + } + } + } + region.put(p); + if (Math.random() < flushPercentage) { + LOG.info("Flushing... "); + region.flushcache(); + } + + if (Math.random() < minorPercentage) { + LOG.info("Minor compacting... "); + region.compactStores(false); + } + + if (Math.random() < majorPercentage) { + LOG.info("Major compacting... "); + region.compactStores(true); + } + } + } + + for (int i = 0; i < numberOfTests + 1; i++) { + Collection kvSet; + Scan scan = new Scan(); + scan.setMaxVersions(); + if (i < numberOfTests) { + if (columnLists[i].size() == 0) continue; // HBASE-7700 + kvSet = kvMaps[i].values(); + for (String column : columnLists[i]) { + scan.addColumn(familyBytes, Bytes.toBytes(column)); + } + LOG.info("ExplicitColumns scanner"); + LOG.info("Columns: " + columnLists[i].size() + " Keys: " + + kvSet.size()); + } else { + kvSet = allKVMap.values(); + LOG.info("Wildcard scanner"); + LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size()); + + } + InternalScanner scanner = region.getScanner(scan); + List results = new ArrayList(); + while (scanner.next(results)) + ; + assertEquals(kvSet.size(), results.size()); + assertTrue(results.containsAll(kvSet)); + } + } finally { + HRegion.closeHRegion(region); + } + + region.close(); + region.getLog().closeAndDelete(); + } + + @SuppressWarnings("unchecked") + @Test + public void testReseeking() throws IOException { + String family = "Family"; + byte[] familyBytes = Bytes.toBytes("Family"); + String table = "TestSingleVersions"; + + HTableDescriptor htd = new HTableDescriptor(table); + htd.addFamily(new HColumnDescriptor(family)); + + HRegionInfo info = new HRegionInfo(Bytes.toBytes(table), null, null, false); + HRegion region = + HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL + .getConfiguration(), htd); + + List rows = generateRandomWords(10, "row"); + List allColumns = generateRandomWords(100, "column"); + + long maxTimestamp = 2; + double selectPercent = 0.5; + int numberOfTests = 5; + double flushPercentage = 0.2; + double minorPercentage = 0.2; + double majorPercentage = 0.2; + double putPercentage = 0.2; + + HashMap allKVMap = new HashMap(); + + HashMap[] kvMaps = new HashMap[numberOfTests]; + ArrayList[] columnLists = new ArrayList[numberOfTests]; + String valueString = "Value"; + + for (int i = 0; i < numberOfTests; i++) { + kvMaps[i] = new HashMap(); + columnLists[i] = new ArrayList(); + for (String column : allColumns) { + if (Math.random() < selectPercent) { + columnLists[i].add(column); + } + } + } + + for (String row : rows) { + Put p = new Put(Bytes.toBytes(row)); + p.setWriteToWAL(false); + for (String column : allColumns) { + for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) { + KeyValue kv = + KeyValueTestUtil.create(row, family, column, timestamp, + valueString); + if (Math.random() < putPercentage) { + p.add(kv); + allKVMap.put(kv.getKeyString(), kv); + for (int i = 0; i < numberOfTests; i++) { + if (columnLists[i].contains(column)) { + kvMaps[i].put(kv.getKeyString(), kv); + } + } + } + + } + } + region.put(p); + if (Math.random() < flushPercentage) { + LOG.info("Flushing... "); + region.flushcache(); + } + + if (Math.random() < minorPercentage) { + LOG.info("Minor compacting... "); + region.compactStores(false); + } + + if (Math.random() < majorPercentage) { + LOG.info("Major compacting... "); + region.compactStores(true); + } + } + + for (int i = 0; i < numberOfTests + 1; i++) { + Collection kvSet; + Scan scan = new Scan(); + scan.setMaxVersions(); + if (i < numberOfTests) { + if (columnLists[i].size() == 0) continue; // HBASE-7700 + kvSet = kvMaps[i].values(); + for (String column : columnLists[i]) { + scan.addColumn(familyBytes, Bytes.toBytes(column)); + } + LOG.info("ExplicitColumns scanner"); + LOG.info("Columns: " + columnLists[i].size() + " Keys: " + + kvSet.size()); + } else { + kvSet = allKVMap.values(); + LOG.info("Wildcard scanner"); + LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size()); + + } + InternalScanner scanner = region.getScanner(scan); + List results = new ArrayList(); + while (scanner.next(results)) + ; + assertEquals(kvSet.size(), results.size()); + assertTrue(results.containsAll(kvSet)); + } + + region.close(); + region.getLog().closeAndDelete(); + } + + List generateRandomWords(int numberOfWords, String suffix) { + Set wordSet = new HashSet(); + for (int i = 0; i < numberOfWords; i++) { + int lengthOfWords = (int) (Math.random() * 5) + 1; + char[] wordChar = new char[lengthOfWords]; + for (int j = 0; j < wordChar.length; j++) { + wordChar[j] = (char) (Math.random() * 26 + 97); + } + String word; + if (suffix == null) { + word = new String(wordChar); + } else { + word = new String(wordChar) + suffix; + } + wordSet.add(word); + } + List wordList = new ArrayList(wordSet); + return wordList; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java new file mode 100644 index 0000000..451ada6 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java @@ -0,0 +1,289 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder; +import org.apache.hadoop.hbase.regionserver.compactions.CompactSelection; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Bytes; + +import com.google.common.collect.Lists; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestCompactSelection extends TestCase { + private final static Log LOG = LogFactory.getLog(TestCompactSelection.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private Configuration conf; + private Store store; + private static final String DIR= + TEST_UTIL.getDataTestDir("TestCompactSelection").toString(); + private static Path TEST_FILE; + + private static final int minFiles = 3; + private static final int maxFiles = 5; + + private static final long minSize = 10; + private static final long maxSize = 1000; + + + @Override + public void setUp() throws Exception { + // setup config values necessary for store + this.conf = TEST_UTIL.getConfiguration(); + this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0); + this.conf.setInt("hbase.hstore.compaction.min", minFiles); + this.conf.setInt("hbase.hstore.compaction.max", maxFiles); + this.conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, minSize); + this.conf.setLong("hbase.hstore.compaction.max.size", maxSize); + this.conf.setFloat("hbase.hstore.compaction.ratio", 1.0F); + + //Setting up a Store + Path basedir = new Path(DIR); + Path logdir = new Path(DIR+"/logs"); + Path oldLogDir = new Path(basedir, HConstants.HREGION_OLDLOGDIR_NAME); + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family")); + FileSystem fs = FileSystem.get(conf); + + fs.delete(logdir, true); + + HTableDescriptor htd = new HTableDescriptor(Bytes.toBytes("table")); + htd.addFamily(hcd); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + + HLog hlog = new HLog(fs, logdir, oldLogDir, conf); + HRegion region = HRegion.createHRegion(info, basedir, conf, htd); + HRegion.closeHRegion(region); + Path tableDir = new Path(basedir, Bytes.toString(htd.getName())); + region = new HRegion(tableDir, hlog, fs, conf, info, htd, null); + + store = new Store(basedir, region, hcd, fs, conf); + TEST_FILE = StoreFile.getRandomFilename(fs, store.getHomedir()); + fs.create(TEST_FILE); + } + + // used so our tests don't deal with actual StoreFiles + static class MockStoreFile extends StoreFile { + long length = 0; + boolean isRef = false; + + MockStoreFile(long length, boolean isRef) throws IOException { + super(TEST_UTIL.getTestFileSystem(), TEST_FILE, + TEST_UTIL.getConfiguration(), + new CacheConfig(TEST_UTIL.getConfiguration()), BloomType.NONE, + NoOpDataBlockEncoder.INSTANCE); + this.length = length; + this.isRef = isRef; + } + + void setLength(long newLen) { + this.length = newLen; + } + + @Override + boolean isMajorCompaction() { + return false; + } + + @Override + boolean isReference() { + return this.isRef; + } + + @Override + public StoreFile.Reader getReader() { + final long len = this.length; + return new StoreFile.Reader() { + @Override + public long length() { + return len; + } + }; + } + } + + List sfCreate(long ... sizes) throws IOException { + return sfCreate(false, sizes); + } + + List sfCreate(boolean isReference, long ... sizes) + throws IOException { + List ret = Lists.newArrayList(); + for (long i : sizes) { + ret.add(new MockStoreFile(i, isReference)); + } + return ret; + } + + long[] getSizes(List sfList) { + long[] aNums = new long[sfList.size()]; + for (int i=0; i candidates, long ... expected) + throws IOException { + compactEquals(candidates, false, expected); + } + + void compactEquals(List candidates, boolean forcemajor, + long ... expected) + throws IOException { + store.forceMajor = forcemajor; + List actual = store.compactSelection(candidates).getFilesToCompact(); + store.forceMajor = false; + assertEquals(Arrays.toString(expected), Arrays.toString(getSizes(actual))); + } + + public void testCompactionRatio() throws IOException { + /* + * NOTE: these tests are specific to describe the implementation of the + * current compaction algorithm. Developed to ensure that refactoring + * doesn't implicitly alter this. + */ + long tooBig = maxSize + 1; + + // default case. preserve user ratio on size + compactEquals(sfCreate(100,50,23,12,12), 23, 12, 12); + // less than compact threshold = don't compact + compactEquals(sfCreate(100,50,25,12,12) /* empty */); + // greater than compact size = skip those + compactEquals(sfCreate(tooBig, tooBig, 700, 700, 700), 700, 700, 700); + // big size + threshold + compactEquals(sfCreate(tooBig, tooBig, 700,700) /* empty */); + // small files = don't care about ratio + compactEquals(sfCreate(8,3,1), 8,3,1); + /* TODO: add sorting + unit test back in when HBASE-2856 is fixed + // sort first so you don't include huge file the tail end + // happens with HFileOutputFormat bulk migration + compactEquals(sfCreate(100,50,23,12,12, 500), 23, 12, 12); + */ + // don't exceed max file compact threshold + assertEquals(maxFiles, + store.compactSelection(sfCreate(7,6,5,4,3,2,1)).getFilesToCompact().size()); + // note: file selection starts with largest to smallest. + compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3); + + /* MAJOR COMPACTION */ + // if a major compaction has been forced, then compact everything + compactEquals(sfCreate(50,25,12,12), true, 50, 25, 12, 12); + // also choose files < threshold on major compaction + compactEquals(sfCreate(12,12), true, 12, 12); + // even if one of those files is too big + compactEquals(sfCreate(tooBig, 12,12), true, tooBig, 12, 12); + // don't exceed max file compact threshold, even with major compaction + store.forceMajor = true; + compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3); + store.forceMajor = false; + + // if we exceed maxCompactSize, downgrade to minor + // if not, it creates a 'snowball effect' when files >> maxCompactSize: + // the last file in compaction is the aggregate of all previous compactions + compactEquals(sfCreate(100,50,23,12,12), true, 23, 12, 12); + conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1); + conf.setFloat("hbase.hregion.majorcompaction.jitter", 0); + try { + // trigger an aged major compaction + compactEquals(sfCreate(50,25,12,12), 50, 25, 12, 12); + // major sure exceeding maxCompactSize also downgrades aged minors + compactEquals(sfCreate(100,50,23,12,12), 23, 12, 12); + } finally { + conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24); + conf.setFloat("hbase.hregion.majorcompaction.jitter", 0.20F); + } + + /* REFERENCES == file is from a region that was split */ + // treat storefiles that have references like a major compaction + compactEquals(sfCreate(true, 100,50,25,12,12), 100, 50, 25, 12, 12); + // reference files shouldn't obey max threshold + compactEquals(sfCreate(true, tooBig, 12,12), tooBig, 12, 12); + // reference files should obey max file compact to avoid OOM + assertEquals(maxFiles, + store.compactSelection(sfCreate(true, 7,6,5,4,3,2,1)).getFilesToCompact().size()); + // reference compaction + compactEquals(sfCreate(true, 7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1); + + // empty case + compactEquals(new ArrayList() /* empty */); + // empty case (because all files are too big) + compactEquals(sfCreate(tooBig, tooBig) /* empty */); + } + + public void testOffPeakCompactionRatio() throws IOException { + /* + * NOTE: these tests are specific to describe the implementation of the + * current compaction algorithm. Developed to ensure that refactoring + * doesn't implicitly alter this. + */ + long tooBig = maxSize + 1; + + Calendar calendar = new GregorianCalendar(); + int hourOfDay = calendar.get(Calendar.HOUR_OF_DAY); + LOG.debug("Hour of day = " + hourOfDay); + int hourPlusOne = ((hourOfDay+1+24)%24); + int hourMinusOne = ((hourOfDay-1+24)%24); + int hourMinusTwo = ((hourOfDay-2+24)%24); + + // check compact selection without peak hour setting + LOG.debug("Testing compact selection without off-peak settings..."); + compactEquals(sfCreate(999,50,12,12,1), 12, 12, 1); + + // set an off-peak compaction threshold + this.conf.setFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F); + + // set peak hour to current time and check compact selection + this.conf.setLong("hbase.offpeak.start.hour", hourMinusOne); + this.conf.setLong("hbase.offpeak.end.hour", hourPlusOne); + LOG.debug("Testing compact selection with off-peak settings (" + + hourMinusOne + ", " + hourPlusOne + ")"); + compactEquals(sfCreate(999,50,12,12, 1), 50, 12, 12, 1); + + // set peak hour outside current selection and check compact selection + this.conf.setLong("hbase.offpeak.start.hour", hourMinusTwo); + this.conf.setLong("hbase.offpeak.end.hour", hourMinusOne); + LOG.debug("Testing compact selection with off-peak settings (" + + hourMinusTwo + ", " + hourMinusOne + ")"); + compactEquals(sfCreate(999,50,12,12, 1), 12, 12, 1); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java new file mode 100644 index 0000000..4971a86 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java @@ -0,0 +1,753 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.CountDownLatch; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + + +/** + * Test compactions + */ +@Category(SmallTests.class) +public class TestCompaction extends HBaseTestCase { + static final Log LOG = LogFactory.getLog(TestCompaction.class.getName()); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private HRegion r = null; + private HTableDescriptor htd = null; + private Path compactionDir = null; + private Path regionCompactionDir = null; + private static final byte [] COLUMN_FAMILY = fam1; + private final byte [] STARTROW = Bytes.toBytes(START_KEY); + private static final byte [] COLUMN_FAMILY_TEXT = COLUMN_FAMILY; + private int compactionThreshold; + private byte[] firstRowBytes, secondRowBytes, thirdRowBytes; + final private byte[] col1, col2; + private static final long MAX_FILES_TO_COMPACT = 10; + + /** constructor */ + public TestCompaction() throws Exception { + super(); + + // Set cache flush size to 1MB + conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024); + conf.setInt("hbase.hregion.memstore.block.multiplier", 100); + compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3); + + firstRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING); + secondRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING); + // Increment the least significant character so we get to next row. + secondRowBytes[START_KEY_BYTES.length - 1]++; + thirdRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING); + thirdRowBytes[START_KEY_BYTES.length - 1]++; + thirdRowBytes[START_KEY_BYTES.length - 1]++; + col1 = "column1".getBytes(HConstants.UTF8_ENCODING); + col2 = "column2".getBytes(HConstants.UTF8_ENCODING); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + this.htd = createTableDescriptor(getName()); + this.r = createNewHRegion(htd, null, null); + } + + @Override + public void tearDown() throws Exception { + HLog hlog = r.getLog(); + this.r.close(); + hlog.closeAndDelete(); + super.tearDown(); + } + + /** + * Test that on a major compaction, if all cells are expired or deleted, then + * we'll end up with no product. Make sure scanner over region returns + * right answer in this case - and that it just basically works. + * @throws IOException + */ + public void testMajorCompactingToNoOutput() throws IOException { + createStoreFile(r); + for (int i = 0; i < compactionThreshold; i++) { + createStoreFile(r); + } + // Now delete everything. + InternalScanner s = r.getScanner(new Scan()); + do { + List results = new ArrayList(); + boolean result = s.next(results); + r.delete(new Delete(results.get(0).getRow()), null, false); + if (!result) break; + } while(true); + s.close(); + // Flush + r.flushcache(); + // Major compact. + r.compactStores(true); + s = r.getScanner(new Scan()); + int counter = 0; + do { + List results = new ArrayList(); + boolean result = s.next(results); + if (!result) break; + counter++; + } while(true); + assertEquals(0, counter); + } + + /** + * Run compaction and flushing memstore + * Assert deletes get cleaned up. + * @throws Exception + */ + public void testMajorCompaction() throws Exception { + majorCompaction(); + } + + public void testDataBlockEncodingInCacheOnly() throws Exception { + majorCompactionWithDataBlockEncoding(true); + } + + public void testDataBlockEncodingEverywhere() throws Exception { + majorCompactionWithDataBlockEncoding(false); + } + + public void majorCompactionWithDataBlockEncoding(boolean inCacheOnly) + throws Exception { + Map replaceBlockCache = + new HashMap(); + for (Entry pair : r.getStores().entrySet()) { + Store store = pair.getValue(); + HFileDataBlockEncoder blockEncoder = store.getDataBlockEncoder(); + replaceBlockCache.put(pair.getValue(), blockEncoder); + final DataBlockEncoding inCache = DataBlockEncoding.PREFIX; + final DataBlockEncoding onDisk = inCacheOnly ? DataBlockEncoding.NONE : + inCache; + store.setDataBlockEncoderInTest(new HFileDataBlockEncoderImpl( + onDisk, inCache)); + } + + majorCompaction(); + + // restore settings + for (Entry entry : + replaceBlockCache.entrySet()) { + entry.getKey().setDataBlockEncoderInTest(entry.getValue()); + } + } + + private void majorCompaction() throws Exception { + createStoreFile(r); + for (int i = 0; i < compactionThreshold; i++) { + createStoreFile(r); + } + // Add more content. + addContent(new HRegionIncommon(r), Bytes.toString(COLUMN_FAMILY)); + + // Now there are about 5 versions of each column. + // Default is that there only 3 (MAXVERSIONS) versions allowed per column. + // + // Assert == 3 when we ask for versions. + Result result = r.get(new Get(STARTROW).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null); + assertEquals(compactionThreshold, result.size()); + + // see if CompactionProgress is in place but null + for (Store store: this.r.stores.values()) { + assertNull(store.getCompactionProgress()); + } + + r.flushcache(); + r.compactStores(true); + + // see if CompactionProgress has done its thing on at least one store + int storeCount = 0; + for (Store store: this.r.stores.values()) { + CompactionProgress progress = store.getCompactionProgress(); + if( progress != null ) { + ++storeCount; + assertTrue(progress.currentCompactedKVs > 0); + assertTrue(progress.totalCompactingKVs > 0); + } + assertTrue(storeCount > 0); + } + + // look at the second row + // Increment the least significant character so we get to next row. + byte [] secondRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING); + secondRowBytes[START_KEY_BYTES.length - 1]++; + + // Always 3 versions if that is what max versions is. + result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT). + setMaxVersions(100), null); + LOG.debug("Row " + Bytes.toStringBinary(secondRowBytes) + " after " + + "initial compaction: " + result); + assertEquals("Invalid number of versions of row " + + Bytes.toStringBinary(secondRowBytes) + ".", compactionThreshold, + result.size()); + + // Now add deletes to memstore and then flush it. + // That will put us over + // the compaction threshold of 3 store files. Compacting these store files + // should result in a compacted store file that has no references to the + // deleted row. + LOG.debug("Adding deletes to memstore and flushing"); + Delete delete = new Delete(secondRowBytes, System.currentTimeMillis(), null); + byte [][] famAndQf = {COLUMN_FAMILY, null}; + delete.deleteFamily(famAndQf[0]); + r.delete(delete, null, true); + + // Assert deleted. + result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null ); + assertTrue("Second row should have been deleted", result.isEmpty()); + + r.flushcache(); + + result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null ); + assertTrue("Second row should have been deleted", result.isEmpty()); + + // Add a bit of data and flush. Start adding at 'bbb'. + createSmallerStoreFile(this.r); + r.flushcache(); + // Assert that the second row is still deleted. + result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null ); + assertTrue("Second row should still be deleted", result.isEmpty()); + + // Force major compaction. + r.compactStores(true); + assertEquals(r.getStore(COLUMN_FAMILY_TEXT).getStorefiles().size(), 1); + + result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null ); + assertTrue("Second row should still be deleted", result.isEmpty()); + + // Make sure the store files do have some 'aaa' keys in them -- exactly 3. + // Also, that compacted store files do not have any secondRowBytes because + // they were deleted. + verifyCounts(3,0); + + // Multiple versions allowed for an entry, so the delete isn't enough + // Lower TTL and expire to ensure that all our entries have been wiped + final int ttl = 1000; + for (Store store: this.r.stores.values()) { + Store.ScanInfo old = store.scanInfo; + Store.ScanInfo si = new Store.ScanInfo(old.getFamily(), + old.getMinVersions(), old.getMaxVersions(), ttl, + old.getKeepDeletedCells(), 0, old.getComparator()); + store.scanInfo = si; + } + Thread.sleep(1000); + + r.compactStores(true); + int count = count(); + assertEquals("Should not see anything after TTL has expired", 0, count); + } + + public void testTimeBasedMajorCompaction() throws Exception { + // create 2 storefiles and force a major compaction to reset the time + int delay = 10 * 1000; // 10 sec + float jitterPct = 0.20f; // 20% + conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, delay); + conf.setFloat("hbase.hregion.majorcompaction.jitter", jitterPct); + + Store s = r.getStore(COLUMN_FAMILY); + try { + createStoreFile(r); + createStoreFile(r); + r.compactStores(true); + + // add one more file & verify that a regular compaction won't work + createStoreFile(r); + r.compactStores(false); + assertEquals(2, s.getStorefilesCount()); + + // ensure that major compaction time is deterministic + long mcTime = s.getNextMajorCompactTime(); + for (int i = 0; i < 10; ++i) { + assertEquals(mcTime, s.getNextMajorCompactTime()); + } + + // ensure that the major compaction time is within the variance + long jitter = Math.round(delay * jitterPct); + assertTrue(delay - jitter <= mcTime && mcTime <= delay + jitter); + + // wait until the time-based compaction interval + Thread.sleep(mcTime); + + // trigger a compaction request and ensure that it's upgraded to major + r.compactStores(false); + assertEquals(1, s.getStorefilesCount()); + } finally { + // reset the timed compaction settings + conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24); + conf.setFloat("hbase.hregion.majorcompaction.jitter", 0.20F); + // run a major to reset the cache + createStoreFile(r); + r.compactStores(true); + assertEquals(1, s.getStorefilesCount()); + } + } + + public void testMinorCompactionWithDeleteRow() throws Exception { + Delete deleteRow = new Delete(secondRowBytes); + testMinorCompactionWithDelete(deleteRow); + } + public void testMinorCompactionWithDeleteColumn1() throws Exception { + Delete dc = new Delete(secondRowBytes); + /* delete all timestamps in the column */ + dc.deleteColumns(fam2, col2); + testMinorCompactionWithDelete(dc); + } + public void testMinorCompactionWithDeleteColumn2() throws Exception { + Delete dc = new Delete(secondRowBytes); + dc.deleteColumn(fam2, col2); + /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3. + * we only delete the latest version. One might expect to see only + * versions 1 and 2. HBase differs, and gives us 0, 1 and 2. + * This is okay as well. Since there was no compaction done before the + * delete, version 0 seems to stay on. + */ + //testMinorCompactionWithDelete(dc, 2); + testMinorCompactionWithDelete(dc, 3); + } + public void testMinorCompactionWithDeleteColumnFamily() throws Exception { + Delete deleteCF = new Delete(secondRowBytes); + deleteCF.deleteFamily(fam2); + testMinorCompactionWithDelete(deleteCF); + } + public void testMinorCompactionWithDeleteVersion1() throws Exception { + Delete deleteVersion = new Delete(secondRowBytes); + deleteVersion.deleteColumns(fam2, col2, 2); + /* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3. + * We delete versions 0 ... 2. So, we still have one remaining. + */ + testMinorCompactionWithDelete(deleteVersion, 1); + } + public void testMinorCompactionWithDeleteVersion2() throws Exception { + Delete deleteVersion = new Delete(secondRowBytes); + deleteVersion.deleteColumn(fam2, col2, 1); + /* + * the table has 4 versions: 0, 1, 2, and 3. + * We delete 1. + * Should have 3 remaining. + */ + testMinorCompactionWithDelete(deleteVersion, 3); + } + + /* + * A helper function to test the minor compaction algorithm. We check that + * the delete markers are left behind. Takes delete as an argument, which + * can be any delete (row, column, columnfamliy etc), that essentially + * deletes row2 and column2. row1 and column1 should be undeleted + */ + private void testMinorCompactionWithDelete(Delete delete) throws Exception { + testMinorCompactionWithDelete(delete, 0); + } + private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete) throws Exception { + HRegionIncommon loader = new HRegionIncommon(r); + for (int i = 0; i < compactionThreshold + 1; i++) { + addContent(loader, Bytes.toString(fam1), Bytes.toString(col1), firstRowBytes, thirdRowBytes, i); + addContent(loader, Bytes.toString(fam1), Bytes.toString(col2), firstRowBytes, thirdRowBytes, i); + addContent(loader, Bytes.toString(fam2), Bytes.toString(col1), firstRowBytes, thirdRowBytes, i); + addContent(loader, Bytes.toString(fam2), Bytes.toString(col2), firstRowBytes, thirdRowBytes, i); + r.flushcache(); + } + + Result result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100), null); + assertEquals(compactionThreshold, result.size()); + result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100), null); + assertEquals(compactionThreshold, result.size()); + + // Now add deletes to memstore and then flush it. That will put us over + // the compaction threshold of 3 store files. Compacting these store files + // should result in a compacted store file that has no references to the + // deleted row. + r.delete(delete, null, true); + + // Make sure that we have only deleted family2 from secondRowBytes + result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100), null); + assertEquals(expectedResultsAfterDelete, result.size()); + // but we still have firstrow + result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100), null); + assertEquals(compactionThreshold, result.size()); + + r.flushcache(); + // should not change anything. + // Let us check again + + // Make sure that we have only deleted family2 from secondRowBytes + result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100), null); + assertEquals(expectedResultsAfterDelete, result.size()); + // but we still have firstrow + result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100), null); + assertEquals(compactionThreshold, result.size()); + + // do a compaction + Store store2 = this.r.stores.get(fam2); + int numFiles1 = store2.getStorefiles().size(); + assertTrue("Was expecting to see 4 store files", numFiles1 > compactionThreshold); // > 3 + store2.compactRecentForTesting(compactionThreshold); // = 3 + int numFiles2 = store2.getStorefiles().size(); + // Check that we did compact + assertTrue("Number of store files should go down", numFiles1 > numFiles2); + // Check that it was a minor compaction. + assertTrue("Was not supposed to be a major compaction", numFiles2 > 1); + + // Make sure that we have only deleted family2 from secondRowBytes + result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100), null); + assertEquals(expectedResultsAfterDelete, result.size()); + // but we still have firstrow + result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100), null); + assertEquals(compactionThreshold, result.size()); + } + + private void verifyCounts(int countRow1, int countRow2) throws Exception { + int count1 = 0; + int count2 = 0; + for (StoreFile f: this.r.stores.get(COLUMN_FAMILY_TEXT).getStorefiles()) { + HFileScanner scanner = f.getReader().getScanner(false, false); + scanner.seekTo(); + do { + byte [] row = scanner.getKeyValue().getRow(); + if (Bytes.equals(row, STARTROW)) { + count1++; + } else if(Bytes.equals(row, secondRowBytes)) { + count2++; + } + } while(scanner.next()); + } + assertEquals(countRow1,count1); + assertEquals(countRow2,count2); + } + + /** + * Verify that you can stop a long-running compaction + * (used during RS shutdown) + * @throws Exception + */ + public void testInterruptCompaction() throws Exception { + assertEquals(0, count()); + + // lower the polling interval for this test + int origWI = Store.closeCheckInterval; + Store.closeCheckInterval = 10*1000; // 10 KB + + try { + // Create a couple store files w/ 15KB (over 10KB interval) + int jmax = (int) Math.ceil(15.0/compactionThreshold); + byte [] pad = new byte[1000]; // 1 KB chunk + for (int i = 0; i < compactionThreshold; i++) { + HRegionIncommon loader = new HRegionIncommon(r); + Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(i))); + p.setWriteToWAL(false); + for (int j = 0; j < jmax; j++) { + p.add(COLUMN_FAMILY, Bytes.toBytes(j), pad); + } + addContent(loader, Bytes.toString(COLUMN_FAMILY)); + loader.put(p); + loader.flushcache(); + } + + HRegion spyR = spy(r); + doAnswer(new Answer() { + public Object answer(InvocationOnMock invocation) throws Throwable { + r.writestate.writesEnabled = false; + return invocation.callRealMethod(); + } + }).when(spyR).doRegionCompactionPrep(); + + // force a minor compaction, but not before requesting a stop + spyR.compactStores(); + + // ensure that the compaction stopped, all old files are intact, + Store s = r.stores.get(COLUMN_FAMILY); + assertEquals(compactionThreshold, s.getStorefilesCount()); + assertTrue(s.getStorefilesSize() > 15*1000); + // and no new store files persisted past compactStores() + FileStatus[] ls = FileSystem.get(conf).listStatus(r.getTmpDir()); + assertEquals(0, ls.length); + + } finally { + // don't mess up future tests + r.writestate.writesEnabled = true; + Store.closeCheckInterval = origWI; + + // Delete all Store information once done using + for (int i = 0; i < compactionThreshold; i++) { + Delete delete = new Delete(Bytes.add(STARTROW, Bytes.toBytes(i))); + byte [][] famAndQf = {COLUMN_FAMILY, null}; + delete.deleteFamily(famAndQf[0]); + r.delete(delete, null, true); + } + r.flushcache(); + + // Multiple versions allowed for an entry, so the delete isn't enough + // Lower TTL and expire to ensure that all our entries have been wiped + final int ttl = 1000; + for (Store store: this.r.stores.values()) { + Store.ScanInfo old = store.scanInfo; + Store.ScanInfo si = new Store.ScanInfo(old.getFamily(), + old.getMinVersions(), old.getMaxVersions(), ttl, + old.getKeepDeletedCells(), 0, old.getComparator()); + store.scanInfo = si; + } + Thread.sleep(ttl); + + r.compactStores(true); + assertEquals(0, count()); + } + } + + private int count() throws IOException { + int count = 0; + for (StoreFile f: this.r.stores. + get(COLUMN_FAMILY_TEXT).getStorefiles()) { + HFileScanner scanner = f.getReader().getScanner(false, false); + if (!scanner.seekTo()) { + continue; + } + do { + count++; + } while(scanner.next()); + } + return count; + } + + private void createStoreFile(final HRegion region) throws IOException { + createStoreFile(region, Bytes.toString(COLUMN_FAMILY)); + } + + private void createStoreFile(final HRegion region, String family) throws IOException { + HRegionIncommon loader = new HRegionIncommon(region); + addContent(loader, family); + loader.flushcache(); + } + + private void createSmallerStoreFile(final HRegion region) throws IOException { + HRegionIncommon loader = new HRegionIncommon(region); + addContent(loader, Bytes.toString(COLUMN_FAMILY), ("" + + "bbb").getBytes(), null); + loader.flushcache(); + } + + public void testCompactionWithCorruptResult() throws Exception { + int nfiles = 10; + for (int i = 0; i < nfiles; i++) { + createStoreFile(r); + } + Store store = r.getStore(COLUMN_FAMILY); + + List storeFiles = store.getStorefiles(); + long maxId = StoreFile.getMaxSequenceIdInList(storeFiles); + Compactor tool = new Compactor(this.conf); + + StoreFile.Writer compactedFile = tool.compactForTesting(store, this.conf, storeFiles, false, + maxId); + + // Now lets corrupt the compacted file. + FileSystem fs = FileSystem.get(conf); + Path origPath = compactedFile.getPath(); + Path homedir = store.getHomedir(); + Path dstPath = new Path(homedir, origPath.getName()); + FSDataOutputStream stream = fs.create(origPath, null, true, 512, (short) 3, + (long) 1024, + null); + stream.writeChars("CORRUPT FILE!!!!"); + stream.close(); + + try { + store.completeCompaction(storeFiles, compactedFile); + } catch (Exception e) { + // The complete compaction should fail and the corrupt file should remain + // in the 'tmp' directory; + assert (fs.exists(origPath)); + assert (!fs.exists(dstPath)); + System.out.println("testCompactionWithCorruptResult Passed"); + return; + } + fail("testCompactionWithCorruptResult failed since no exception was" + + "thrown while completing a corrupt file"); + } + + /** + * Test for HBASE-5920 - Test user requested major compactions always occurring + */ + public void testNonUserMajorCompactionRequest() throws Exception { + Store store = r.getStore(COLUMN_FAMILY); + createStoreFile(r); + for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) { + createStoreFile(r); + } + store.triggerMajorCompaction(); + + CompactionRequest request = store.requestCompaction(Store.NO_PRIORITY, null); + assertNotNull("Expected to receive a compaction request", request); + assertEquals( + "System-requested major compaction should not occur if there are too many store files", + false, + request.isMajor()); + } + + /** + * Test for HBASE-5920 + */ + public void testUserMajorCompactionRequest() throws IOException{ + Store store = r.getStore(COLUMN_FAMILY); + createStoreFile(r); + for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) { + createStoreFile(r); + } + store.triggerMajorCompaction(); + CompactionRequest request = store.requestCompaction(Store.PRIORITY_USER, null); + assertNotNull("Expected to receive a compaction request", request); + assertEquals( + "User-requested major compaction should always occur, even if there are too many store files", + true, + request.isMajor()); + } + + /** + * Create a custom compaction request and be sure that we can track it through the queue, knowing + * when the compaction is completed. + */ + public void testTrackingCompactionRequest() throws Exception { + // setup a compact/split thread on a mock server + HRegionServer mockServer = Mockito.mock(HRegionServer.class); + Mockito.when(mockServer.getConfiguration()).thenReturn(r.getConf()); + CompactSplitThread thread = new CompactSplitThread(mockServer); + Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread); + // simple stop for the metrics - we ignore any updates in the test + RegionServerMetrics mockMetrics = Mockito.mock(RegionServerMetrics.class); + Mockito.when(mockServer.getMetrics()).thenReturn(mockMetrics); + + // setup a region/store with some files + Store store = r.getStore(COLUMN_FAMILY); + createStoreFile(r); + for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) { + createStoreFile(r); + } + + CountDownLatch latch = new CountDownLatch(1); + TrackableCompactionRequest request = new TrackableCompactionRequest(r, store, latch); + thread.requestCompaction(r, store, "test custom comapction", Store.PRIORITY_USER, request); + // wait for the latch to complete. + latch.await(); + + thread.interruptIfNecessary(); + } + + public void testMultipleCustomCompactionRequests() throws Exception { + // setup a compact/split thread on a mock server + HRegionServer mockServer = Mockito.mock(HRegionServer.class); + Mockito.when(mockServer.getConfiguration()).thenReturn(r.getConf()); + CompactSplitThread thread = new CompactSplitThread(mockServer); + Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread); + // simple stop for the metrics - we ignore any updates in the test + RegionServerMetrics mockMetrics = Mockito.mock(RegionServerMetrics.class); + Mockito.when(mockServer.getMetrics()).thenReturn(mockMetrics); + + // setup a region/store with some files + int numStores = r.getStores().size(); + List requests = new ArrayList(numStores); + CountDownLatch latch = new CountDownLatch(numStores); + // create some store files and setup requests for each store on which we want to do a + // compaction + for (Store store : r.getStores().values()) { + createStoreFile(r, store.getColumnFamilyName()); + createStoreFile(r, store.getColumnFamilyName()); + createStoreFile(r, store.getColumnFamilyName()); + requests.add(new TrackableCompactionRequest(r, store, latch)); + } + + thread.requestCompaction(r, "test mulitple custom comapctions", Store.PRIORITY_USER, + Collections.unmodifiableList(requests)); + + // wait for the latch to complete. + latch.await(); + + thread.interruptIfNecessary(); + } + + /** + * Simple {@link CompactionRequest} on which you can wait until the requested compaction finishes. + */ + public static class TrackableCompactionRequest extends CompactionRequest { + private CountDownLatch done; + + /** + * Constructor for a custom compaction. Uses the setXXX methods to update the state of the + * compaction before being used. + */ + public TrackableCompactionRequest(HRegion region, Store store, CountDownLatch finished) { + super(region, store, Store.PRIORITY_USER); + this.done = finished; + } + + @Override + public void run() { + super.run(); + this.done.countDown(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java new file mode 100644 index 0000000..79594d7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java @@ -0,0 +1,241 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.CompactionState; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** Unit tests to test retrieving table/region compaction state*/ +@Category(LargeTests.class) +public class TestCompactionState { + final static Log LOG = LogFactory.getLog(TestCompactionState.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static Random random = new Random(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test(timeout=60000) + public void testMajorCompaction() throws IOException, InterruptedException { + compaction("testMajorCompaction", 8, CompactionState.MAJOR, false); + } + + @Test(timeout=60000) + public void testMinorCompaction() throws IOException, InterruptedException { + compaction("testMinorCompaction", 15, CompactionState.MINOR, false); + } + + @Test(timeout=60000) + public void testMajorCompactionOnFamily() throws IOException, InterruptedException { + compaction("testMajorCompactionOnFamily", 8, CompactionState.MAJOR, true); + } + + @Test(timeout=60000) + public void testMinorCompactionOnFamily() throws IOException, InterruptedException { + compaction("testMinorCompactionOnFamily", 15, CompactionState.MINOR, true); + } + + @Test + public void testInvalidColumnFamily() throws IOException, InterruptedException { + byte [] table = Bytes.toBytes("testInvalidColumnFamily"); + byte [] family = Bytes.toBytes("family"); + byte [] fakecf = Bytes.toBytes("fakecf"); + boolean caughtMinorCompact = false; + boolean caughtMajorCompact = false; + HTable ht = null; + try { + ht = TEST_UTIL.createTable(table, family); + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + try { + admin.compact(table, fakecf); + } catch (IOException ioe) { + caughtMinorCompact = true; + } + try { + admin.majorCompact(table, fakecf); + } catch (IOException ioe) { + caughtMajorCompact = true; + } + } finally { + if (ht != null) { + TEST_UTIL.deleteTable(table); + } + assertTrue(caughtMinorCompact); + assertTrue(caughtMajorCompact); + } + } + + /** + * Load data to a table, flush it to disk, trigger compaction, + * confirm the compaction state is right and wait till it is done. + * + * @param tableName + * @param flushes + * @param expectedState + * @param singleFamily otherwise, run compaction on all cfs + * @throws IOException + * @throws InterruptedException + */ + private void compaction(final String tableName, final int flushes, + final CompactionState expectedState, boolean singleFamily) + throws IOException, InterruptedException { + // Create a table with regions + byte [] table = Bytes.toBytes(tableName); + byte [] family = Bytes.toBytes("family"); + byte [][] families = + {family, Bytes.add(family, Bytes.toBytes("2")), Bytes.add(family, Bytes.toBytes("3"))}; + HTable ht = null; + try { + ht = TEST_UTIL.createTable(table, families); + loadData(ht, families, 3000, flushes); + HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); + List regions = rs.getOnlineRegions(table); + int countBefore = countStoreFilesInFamilies(regions, families); + int countBeforeSingleFamily = countStoreFilesInFamily(regions, family); + assertTrue(countBefore > 0); // there should be some data files + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + if (expectedState == CompactionState.MINOR) { + if (singleFamily) { + admin.compact(table, family); + } else { + admin.compact(table); + } + } else { + if (singleFamily) { + admin.majorCompact(table, family); + } else { + admin.majorCompact(table); + } + } + long curt = System.currentTimeMillis(); + long waitTime = 5000; + long endt = curt + waitTime; + CompactionState state = admin.getCompactionState(table); + while (state == CompactionState.NONE && curt < endt) { + Thread.sleep(10); + state = admin.getCompactionState(table); + curt = System.currentTimeMillis(); + } + // Now, should have the right compaction state, + // otherwise, the compaction should have already been done + if (expectedState != state) { + for (HRegion region: regions) { + state = CompactionRequest.getCompactionState(region.getRegionId()); + assertEquals(CompactionState.NONE, state); + } + } else { + curt = System.currentTimeMillis(); + waitTime = 20000; + endt = curt + waitTime; + state = admin.getCompactionState(table); + while (state != CompactionState.NONE && curt < endt) { + Thread.sleep(10); + state = admin.getCompactionState(table); + curt = System.currentTimeMillis(); + } + // Now, compaction should be done. + assertEquals(CompactionState.NONE, state); + } + int countAfter = countStoreFilesInFamilies(regions, families); + int countAfterSingleFamily = countStoreFilesInFamily(regions, family); + assertTrue(countAfter < countBefore); + if (!singleFamily) { + if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter); + else assertTrue(families.length < countAfter); + } else { + int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily; + // assert only change was to single column family + assertTrue(singleFamDiff == (countBefore - countAfter)); + if (expectedState == CompactionState.MAJOR) { + assertTrue(1 == countAfterSingleFamily); + } else { + assertTrue(1 < countAfterSingleFamily); + } + } + } finally { + if (ht != null) { + TEST_UTIL.deleteTable(table); + } + } + } + + private static int countStoreFilesInFamily( + List regions, final byte[] family) { + return countStoreFilesInFamilies(regions, new byte[][]{family}); + } + + private static int countStoreFilesInFamilies(List regions, final byte[][] families) { + int count = 0; + for (HRegion region: regions) { + count += region.getStoreFileList(families).size(); + } + return count; + } + + private static void loadData(final HTable ht, final byte[][] families, + final int rows, final int flushes) throws IOException { + List puts = new ArrayList(rows); + byte[] qualifier = Bytes.toBytes("val"); + for (int i = 0; i < flushes; i++) { + for (int k = 0; k < rows; k++) { + byte[] row = Bytes.toBytes(random.nextLong()); + Put p = new Put(row); + for (int j = 0; j < families.length; ++j) { + p.add(families[ j ], qualifier, row); + } + puts.add(p); + } + ht.put(puts); + ht.flushCommits(); + TEST_UTIL.flush(); + puts.clear(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionWithCoprocessor.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionWithCoprocessor.java new file mode 100644 index 0000000..ba30a9f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionWithCoprocessor.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.junit.experimental.categories.Category; + +/** + * Make sure all compaction tests still pass with the preFlush and preCompact + * overridden to implement the default behavior + */ +@Category(MediumTests.class) +public class TestCompactionWithCoprocessor extends TestCompaction { + /** constructor */ + public TestCompactionWithCoprocessor() throws Exception { + super(); + conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, + NoOpScanPolicyObserver.class.getName()); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java new file mode 100644 index 0000000..aeaf625 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java @@ -0,0 +1,371 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.TestHFileWriterV2; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.util.BloomFilterFactory; +import org.apache.hadoop.hbase.util.ByteBloomFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CompoundBloomFilter; +import org.apache.hadoop.hbase.util.CompoundBloomFilterBase; +import org.apache.hadoop.hbase.util.CompoundBloomFilterWriter; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests writing Bloom filter blocks in the same part of the file as data + * blocks. + */ +@Category(SmallTests.class) +public class TestCompoundBloomFilter { + + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + private static final Log LOG = LogFactory.getLog( + TestCompoundBloomFilter.class); + + private static final int NUM_TESTS = 9; + private static final BloomType BLOOM_TYPES[] = { BloomType.ROW, + BloomType.ROW, BloomType.ROWCOL, BloomType.ROWCOL, BloomType.ROW, + BloomType.ROWCOL, BloomType.ROWCOL, BloomType.ROWCOL, BloomType.ROW }; + + private static final int NUM_KV[]; + static { + final int N = 10000; // Only used in initialization. + NUM_KV = new int[] { 21870, N, N, N, N, 1000, N, 7500, 7500}; + assert NUM_KV.length == NUM_TESTS; + } + + private static final int BLOCK_SIZES[]; + static { + final int blkSize = 65536; + BLOCK_SIZES = new int[] { 512, 1000, blkSize, blkSize, blkSize, 128, 300, + blkSize, blkSize }; + assert BLOCK_SIZES.length == NUM_TESTS; + } + + /** + * Be careful not to specify too high a Bloom filter block size, otherwise + * there will only be one oversized chunk and the observed false positive + * rate will be too low. + */ + private static final int BLOOM_BLOCK_SIZES[] = { 1000, 4096, 4096, 4096, + 8192, 128, 1024, 600, 600 }; + static { assert BLOOM_BLOCK_SIZES.length == NUM_TESTS; } + + private static final double TARGET_ERROR_RATES[] = { 0.025, 0.01, 0.015, + 0.01, 0.03, 0.01, 0.01, 0.07, 0.07 }; + static { assert TARGET_ERROR_RATES.length == NUM_TESTS; } + + /** A false positive rate that is obviously too high. */ + private static final double TOO_HIGH_ERROR_RATE; + static { + double m = 0; + for (double errorRate : TARGET_ERROR_RATES) + m = Math.max(m, errorRate); + TOO_HIGH_ERROR_RATE = m + 0.03; + } + + private static Configuration conf; + private static CacheConfig cacheConf; + private FileSystem fs; + private BlockCache blockCache; + + /** A message of the form "in test#:" to include in logging. */ + private String testIdMsg; + + private static final int GENERATION_SEED = 2319; + private static final int EVALUATION_SEED = 135; + + @Before + public void setUp() throws IOException { + conf = TEST_UTIL.getConfiguration(); + + // This test requires the most recent HFile format (i.e. v2). + conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION); + + fs = FileSystem.get(conf); + + cacheConf = new CacheConfig(conf); + blockCache = cacheConf.getBlockCache(); + assertNotNull(blockCache); + } + + private List createSortedKeyValues(Random rand, int n) { + List kvList = new ArrayList(n); + for (int i = 0; i < n; ++i) + kvList.add(TestHFileWriterV2.randomKeyValue(rand)); + Collections.sort(kvList, KeyValue.COMPARATOR); + return kvList; + } + + @Test + public void testCompoundBloomFilter() throws IOException { + conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true); + for (int t = 0; t < NUM_TESTS; ++t) { + conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, + (float) TARGET_ERROR_RATES[t]); + + testIdMsg = "in test #" + t + ":"; + Random generationRand = new Random(GENERATION_SEED); + List kvs = createSortedKeyValues(generationRand, NUM_KV[t]); + BloomType bt = BLOOM_TYPES[t]; + Path sfPath = writeStoreFile(t, bt, kvs); + readStoreFile(t, bt, kvs, sfPath); + } + } + + /** + * Validates the false positive ratio by computing its z-value and comparing + * it to the provided threshold. + * + * @param falsePosRate experimental positive rate + * @param nTrials the number of Bloom filter checks + * @param zValueBoundary z-value boundary, positive for an upper bound and + * negative for a lower bound + * @param cbf the compound Bloom filter we are using + * @param additionalMsg additional message to include in log output and + * assertion failures + */ + private void validateFalsePosRate(double falsePosRate, int nTrials, + double zValueBoundary, CompoundBloomFilter cbf, String additionalMsg) { + double p = BloomFilterFactory.getErrorRate(conf); + double zValue = (falsePosRate - p) / Math.sqrt(p * (1 - p) / nTrials); + + String assortedStatsStr = " (targetErrorRate=" + p + ", falsePosRate=" + + falsePosRate + ", nTrials=" + nTrials + ")"; + LOG.info("z-value is " + zValue + assortedStatsStr); + + boolean isUpperBound = zValueBoundary > 0; + + if (isUpperBound && zValue > zValueBoundary || + !isUpperBound && zValue < zValueBoundary) { + String errorMsg = "False positive rate z-value " + zValue + " is " + + (isUpperBound ? "higher" : "lower") + " than " + zValueBoundary + + assortedStatsStr + ". Per-chunk stats:\n" + + cbf.formatTestingStats(); + fail(errorMsg + additionalMsg); + } + } + + private void readStoreFile(int t, BloomType bt, List kvs, + Path sfPath) throws IOException { + StoreFile sf = new StoreFile(fs, sfPath, conf, cacheConf, bt, + NoOpDataBlockEncoder.INSTANCE); + StoreFile.Reader r = sf.createReader(); + final boolean pread = true; // does not really matter + StoreFileScanner scanner = r.getStoreFileScanner(true, pread); + + { + // Test for false negatives (not allowed). + int numChecked = 0; + for (KeyValue kv : kvs) { + byte[] row = kv.getRow(); + boolean present = isInBloom(scanner, row, kv.getQualifier()); + assertTrue(testIdMsg + " Bloom filter false negative on row " + + Bytes.toStringBinary(row) + " after " + numChecked + + " successful checks", present); + ++numChecked; + } + } + + // Test for false positives (some percentage allowed). We test in two modes: + // "fake lookup" which ignores the key distribution, and production mode. + for (boolean fakeLookupEnabled : new boolean[] { true, false }) { + ByteBloomFilter.setFakeLookupMode(fakeLookupEnabled); + try { + String fakeLookupModeStr = ", fake lookup is " + (fakeLookupEnabled ? + "enabled" : "disabled"); + CompoundBloomFilter cbf = (CompoundBloomFilter) r.getGeneralBloomFilter(); + cbf.enableTestingStats(); + int numFalsePos = 0; + Random rand = new Random(EVALUATION_SEED); + int nTrials = NUM_KV[t] * 10; + for (int i = 0; i < nTrials; ++i) { + byte[] query = TestHFileWriterV2.randomRowOrQualifier(rand); + if (isInBloom(scanner, query, bt, rand)) { + numFalsePos += 1; + } + } + double falsePosRate = numFalsePos * 1.0 / nTrials; + LOG.debug(String.format(testIdMsg + + " False positives: %d out of %d (%f)", + numFalsePos, nTrials, falsePosRate) + fakeLookupModeStr); + + // Check for obvious Bloom filter crashes. + assertTrue("False positive is too high: " + falsePosRate + " (greater " + + "than " + TOO_HIGH_ERROR_RATE + ")" + fakeLookupModeStr, + falsePosRate < TOO_HIGH_ERROR_RATE); + + // Now a more precise check to see if the false positive rate is not + // too high. The reason we use a relaxed restriction for the real-world + // case as opposed to the "fake lookup" case is that our hash functions + // are not completely independent. + + double maxZValue = fakeLookupEnabled ? 1.96 : 2.5; + validateFalsePosRate(falsePosRate, nTrials, maxZValue, cbf, + fakeLookupModeStr); + + // For checking the lower bound we need to eliminate the last chunk, + // because it is frequently smaller and the false positive rate in it + // is too low. This does not help if there is only one under-sized + // chunk, though. + int nChunks = cbf.getNumChunks(); + if (nChunks > 1) { + numFalsePos -= cbf.getNumPositivesForTesting(nChunks - 1); + nTrials -= cbf.getNumQueriesForTesting(nChunks - 1); + falsePosRate = numFalsePos * 1.0 / nTrials; + LOG.info(testIdMsg + " False positive rate without last chunk is " + + falsePosRate + fakeLookupModeStr); + } + + validateFalsePosRate(falsePosRate, nTrials, -2.58, cbf, + fakeLookupModeStr); + } finally { + ByteBloomFilter.setFakeLookupMode(false); + } + } + + r.close(true); // end of test so evictOnClose + } + + private boolean isInBloom(StoreFileScanner scanner, byte[] row, BloomType bt, + Random rand) { + return isInBloom(scanner, row, + TestHFileWriterV2.randomRowOrQualifier(rand)); + } + + private boolean isInBloom(StoreFileScanner scanner, byte[] row, + byte[] qualifier) { + Scan scan = new Scan(row, row); + TreeSet columns = new TreeSet(Bytes.BYTES_COMPARATOR); + columns.add(qualifier); + return scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE); + } + + private Path writeStoreFile(int t, BloomType bt, List kvs) + throws IOException { + conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, + BLOOM_BLOCK_SIZES[t]); + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true); + cacheConf = new CacheConfig(conf); + + StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConf, fs, + BLOCK_SIZES[t]) + .withOutputDir(TEST_UTIL.getDataTestDir()) + .withBloomType(bt) + .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE) + .withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM) + .build(); + + assertTrue(w.hasGeneralBloom()); + assertTrue(w.getGeneralBloomWriter() instanceof CompoundBloomFilterWriter); + CompoundBloomFilterWriter cbbf = + (CompoundBloomFilterWriter) w.getGeneralBloomWriter(); + + int keyCount = 0; + KeyValue prev = null; + LOG.debug("Total keys/values to insert: " + kvs.size()); + for (KeyValue kv : kvs) { + w.append(kv); + + // Validate the key count in the Bloom filter. + boolean newKey = true; + if (prev != null) { + newKey = !(bt == BloomType.ROW ? KeyValue.COMPARATOR.matchingRows(kv, + prev) : KeyValue.COMPARATOR.matchingRowColumn(kv, prev)); + } + if (newKey) + ++keyCount; + assertEquals(keyCount, cbbf.getKeyCount()); + + prev = kv; + } + w.close(); + + return w.getPath(); + } + + @Test + public void testCompoundBloomSizing() { + int bloomBlockByteSize = 4096; + int bloomBlockBitSize = bloomBlockByteSize * 8; + double targetErrorRate = 0.01; + long maxKeysPerChunk = ByteBloomFilter.idealMaxKeys(bloomBlockBitSize, + targetErrorRate); + + long bloomSize1 = bloomBlockByteSize * 8; + long bloomSize2 = ByteBloomFilter.computeBitSize(maxKeysPerChunk, + targetErrorRate); + + double bloomSizeRatio = (bloomSize2 * 1.0 / bloomSize1); + assertTrue(Math.abs(bloomSizeRatio - 0.9999) < 0.0001); + } + + @Test + public void testCreateKey() { + CompoundBloomFilterBase cbfb = new CompoundBloomFilterBase(); + byte[] row = "myRow".getBytes(); + byte[] qualifier = "myQualifier".getBytes(); + byte[] rowKey = cbfb.createBloomKey(row, 0, row.length, + row, 0, 0); + byte[] rowColKey = cbfb.createBloomKey(row, 0, row.length, + qualifier, 0, qualifier.length); + KeyValue rowKV = KeyValue.createKeyValueFromKey(rowKey); + KeyValue rowColKV = KeyValue.createKeyValueFromKey(rowColKey); + assertEquals(rowKV.getTimestamp(), rowColKV.getTimestamp()); + assertEquals(Bytes.toStringBinary(rowKV.getRow()), + Bytes.toStringBinary(rowColKV.getRow())); + assertEquals(0, rowKV.getQualifier().length); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java new file mode 100644 index 0000000..12bc40b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java @@ -0,0 +1,116 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.regionserver.CompoundConfiguration; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.SmallTests; + +import org.junit.experimental.categories.Category; +import org.junit.Test; + +import junit.framework.TestCase; + +@Category(SmallTests.class) +public class TestCompoundConfiguration extends TestCase { + private Configuration baseConf; + + @Override + protected void setUp() throws Exception { + baseConf = new Configuration(); + baseConf.set("A", "1"); + baseConf.setInt("B", 2); + baseConf.set("C", "3"); + } + + @Test + public void testBasicFunctionality() throws ClassNotFoundException { + CompoundConfiguration compoundConf = new CompoundConfiguration() + .add(baseConf); + assertEquals("1", compoundConf.get("A")); + assertEquals(2, compoundConf.getInt("B", 0)); + assertEquals(3, compoundConf.getInt("C", 0)); + assertEquals(0, compoundConf.getInt("D", 0)); + + assertEquals(CompoundConfiguration.class, compoundConf + .getClassByName(CompoundConfiguration.class.getName())); + try { + compoundConf.getClassByName("bad_class_name"); + fail("Trying to load bad_class_name should throw an exception"); + } catch (ClassNotFoundException e) { + // win! + } + } + + @Test + public void testWithConfig() { + Configuration conf = new Configuration(); + conf.set("B", "2b"); + conf.set("C", "33"); + conf.set("D", "4"); + + CompoundConfiguration compoundConf = new CompoundConfiguration() + .add(baseConf) + .add(conf); + assertEquals("1", compoundConf.get("A")); + assertEquals("2b", compoundConf.get("B")); + assertEquals(33, compoundConf.getInt("C", 0)); + assertEquals("4", compoundConf.get("D")); + assertEquals(4, compoundConf.getInt("D", 0)); + assertNull(compoundConf.get("E")); + assertEquals(6, compoundConf.getInt("F", 6)); + } + + private ImmutableBytesWritable strToIbw(String s) { + return new ImmutableBytesWritable(Bytes.toBytes(s)); + } + + @Test + public void testWithIbwMap() { + Map map = + new HashMap(); + map.put(strToIbw("B"), strToIbw("2b")); + map.put(strToIbw("C"), strToIbw("33")); + map.put(strToIbw("D"), strToIbw("4")); + // unlike config, note that IBW Maps can accept null values + map.put(strToIbw("G"), null); + + CompoundConfiguration compoundConf = new CompoundConfiguration() + .add(baseConf) + .add(map); + assertEquals("1", compoundConf.get("A")); + assertEquals("2b", compoundConf.get("B")); + assertEquals(33, compoundConf.getInt("C", 0)); + assertEquals("4", compoundConf.get("D")); + assertEquals(4, compoundConf.getInt("D", 0)); + assertNull(compoundConf.get("E")); + assertEquals(6, compoundConf.getInt("F", 6)); + assertNull(compoundConf.get("G")); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java new file mode 100644 index 0000000..4e62a1f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java @@ -0,0 +1,492 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Random; +import java.util.Set; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Chore; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.NotServingRegionException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.catalog.MetaEditor; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.MetaScanner; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.PairOfSameType; +import org.apache.hadoop.hbase.util.StoppableImplementation; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Iterators; +import com.google.common.collect.Sets; + +@Category(LargeTests.class) +public class TestEndToEndSplitTransaction { + private static final Log LOG = LogFactory.getLog(TestEndToEndSplitTransaction.class); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final Configuration conf = TEST_UTIL.getConfiguration(); + + @BeforeClass + public static void beforeAllTests() throws Exception { + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5); + TEST_UTIL.startMiniCluster(); + } + + @AfterClass + public static void afterAllTests() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testMasterOpsWhileSplitting() throws Exception { + byte[] tableName = Bytes.toBytes("TestSplit"); + byte[] familyName = Bytes.toBytes("fam"); + HTable ht = TEST_UTIL.createTable(tableName, familyName); + TEST_UTIL.loadTable(ht, familyName); + ht.close(); + HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(0); + byte []firstRow = Bytes.toBytes("aaa"); + byte []splitRow = Bytes.toBytes("lll"); + byte []lastRow = Bytes.toBytes("zzz"); + HConnection con = HConnectionManager + .getConnection(TEST_UTIL.getConfiguration()); + // this will also cache the region + byte[] regionName = con.locateRegion(tableName, splitRow).getRegionInfo() + .getRegionName(); + HRegion region = server.getRegion(regionName); + SplitTransaction split = new SplitTransaction(region, splitRow); + split.prepare(); + + // 1. phase I + PairOfSameType regions = split.createDaughters(server, server); + assertFalse(test(con, tableName, firstRow, server)); + assertFalse(test(con, tableName, lastRow, server)); + + // passing null as services prevents final step + // 2, most of phase II + split.openDaughters(server, null, regions.getFirst(), regions.getSecond()); + assertFalse(test(con, tableName, firstRow, server)); + assertFalse(test(con, tableName, lastRow, server)); + + // 3. finish phase II + // note that this replicates some code from SplitTransaction + // 2nd daughter first + server.postOpenDeployTasks(regions.getSecond(), server.getCatalogTracker(), true); + // Add to online regions + server.addToOnlineRegions(regions.getSecond()); + // THIS is the crucial point: + // the 2nd daughter was added, so querying before the split key should fail. + assertFalse(test(con, tableName, firstRow, server)); + // past splitkey is ok. + assertTrue(test(con, tableName, lastRow, server)); + + // first daughter second + server.postOpenDeployTasks(regions.getFirst(), server.getCatalogTracker(), true); + // Add to online regions + server.addToOnlineRegions(regions.getFirst()); + assertTrue(test(con, tableName, firstRow, server)); + assertTrue(test(con, tableName, lastRow, server)); + + // 4. phase III + split.transitionZKNode(server, server, regions.getFirst(), + regions.getSecond()); + assertTrue(test(con, tableName, firstRow, server)); + assertTrue(test(con, tableName, lastRow, server)); + } + + /** + * attempt to locate the region and perform a get and scan + * @return True if successful, False otherwise. + */ + private boolean test(HConnection con, byte[] tableName, byte[] row, + HRegionServer server) { + // not using HTable to avoid timeouts and retries + try { + byte[] regionName = con.relocateRegion(tableName, row).getRegionInfo() + .getRegionName(); + // get and scan should now succeed without exception + server.get(regionName, new Get(row)); + server.openScanner(regionName, new Scan(row)); + } catch (IOException x) { + return false; + } + return true; + } + + /** + * Tests that the client sees meta table changes as atomic during splits + */ + @Test + public void testFromClientSideWhileSplitting() throws Throwable { + LOG.info("Starting testFromClientSideWhileSplitting"); + final byte[] TABLENAME = Bytes.toBytes("testFromClientSideWhileSplitting"); + final byte[] FAMILY = Bytes.toBytes("family"); + + //SplitTransaction will update the meta table by offlining the parent region, and adding info + //for daughters. + HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY); + + Stoppable stopper = new StoppableImplementation(); + RegionSplitter regionSplitter = new RegionSplitter(table); + RegionChecker regionChecker = new RegionChecker(conf, stopper, TABLENAME); + + regionChecker.start(); + regionSplitter.start(); + + //wait until the splitter is finished + regionSplitter.join(); + stopper.stop(null); + + if (regionChecker.ex != null) { + throw regionChecker.ex; + } + + if (regionSplitter.ex != null) { + throw regionSplitter.ex; + } + + //one final check + regionChecker.verify(); + } + + static class RegionSplitter extends Thread { + Throwable ex; + HTable table; + byte[] tableName, family; + HBaseAdmin admin; + HRegionServer rs; + + RegionSplitter(HTable table) throws IOException { + this.table = table; + this.tableName = table.getTableName(); + this.family = table.getTableDescriptor().getFamiliesKeys().iterator().next(); + admin = TEST_UTIL.getHBaseAdmin(); + rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); + } + + public void run() { + try { + Random random = new Random(); + for (int i=0; i< 5; i++) { + NavigableMap regions = MetaScanner.allTableRegions(conf, tableName, false); + if (regions.size() == 0) { + continue; + } + int regionIndex = random.nextInt(regions.size()); + + //pick a random region and split it into two + HRegionInfo region = Iterators.get(regions.keySet().iterator(), regionIndex); + + //pick the mid split point + int start = 0, end = Integer.MAX_VALUE; + if (region.getStartKey().length > 0) { + start = Bytes.toInt(region.getStartKey()); + } + if (region.getEndKey().length > 0) { + end = Bytes.toInt(region.getEndKey()); + } + int mid = start + ((end - start) / 2); + byte[] splitPoint = Bytes.toBytes(mid); + + //put some rows to the regions + addData(start); + addData(mid); + + flushAndBlockUntilDone(admin, rs, region.getRegionName()); + compactAndBlockUntilDone(admin, rs, region.getRegionName()); + + log("Initiating region split for:" + region.getRegionNameAsString()); + try { + admin.split(region.getRegionName(), splitPoint); + //wait until the split is complete + blockUntilRegionSplit(conf, 50000, region.getRegionName(), true); + + } catch (NotServingRegionException ex) { + //ignore + } + } + } catch (Throwable ex) { + this.ex = ex; + } + } + + void addData(int start) throws IOException { + for (int i=start; i< start + 100; i++) { + Put put = new Put(Bytes.toBytes(i)); + + put.add(family, family, Bytes.toBytes(i)); + table.put(put); + } + table.flushCommits(); + } + } + + /** + * Checks regions using MetaScanner, MetaReader and HTable methods + */ + static class RegionChecker extends Chore { + Configuration conf; + byte[] tableName; + Throwable ex; + + RegionChecker(Configuration conf, Stoppable stopper, byte[] tableName) { + super("RegionChecker", 10, stopper); + this.conf = conf; + this.tableName = tableName; + this.setDaemon(true); + } + + /** verify region boundaries obtained from MetaScanner */ + void verifyRegionsUsingMetaScanner() throws Exception { + + //MetaScanner.allTableRegions() + NavigableMap regions = MetaScanner.allTableRegions(conf, tableName, + false); + verifyTableRegions(regions.keySet()); + + //MetaScanner.listAllRegions() + List regionList = MetaScanner.listAllRegions(conf, false); + verifyTableRegions(Sets.newTreeSet(regionList)); + } + + /** verify region boundaries obtained from HTable.getStartEndKeys() */ + void verifyRegionsUsingHTable() throws IOException { + HTable table = null; + try { + //HTable.getStartEndKeys() + table = new HTable(conf, tableName); + Pair keys = table.getStartEndKeys(); + verifyStartEndKeys(keys); + + //HTable.getRegionsInfo() + Map regions = table.getRegionsInfo(); + verifyTableRegions(regions.keySet()); + } finally { + IOUtils.closeQuietly(table); + } + } + + void verify() throws Exception { + verifyRegionsUsingMetaScanner(); + verifyRegionsUsingHTable(); + } + + void verifyTableRegions(Set regions) { + log("Verifying " + regions.size() + " regions"); + + byte[][] startKeys = new byte[regions.size()][]; + byte[][] endKeys = new byte[regions.size()][]; + + int i=0; + for (HRegionInfo region : regions) { + startKeys[i] = region.getStartKey(); + endKeys[i] = region.getEndKey(); + i++; + } + + Pair keys = new Pair(startKeys, endKeys); + verifyStartEndKeys(keys); + } + + void verifyStartEndKeys(Pair keys) { + byte[][] startKeys = keys.getFirst(); + byte[][] endKeys = keys.getSecond(); + assertEquals(startKeys.length, endKeys.length); + assertTrue("Found 0 regions for the table", startKeys.length > 0); + + assertArrayEquals("Start key for the first region is not byte[0]", + HConstants.EMPTY_START_ROW, startKeys[0]); + byte[] prevEndKey = HConstants.EMPTY_START_ROW; + + // ensure that we do not have any gaps + for (int i=0; i 0) { + Threads.sleep(50); + } + } + + public static void compactAndBlockUntilDone(HBaseAdmin admin, HRegionServer rs, byte[] regionName) + throws IOException, InterruptedException { + log("Compacting region: " + Bytes.toStringBinary(regionName)); + admin.majorCompact(regionName); + log("blocking until compaction is complete: " + Bytes.toStringBinary(regionName)); + Threads.sleepWithoutInterrupt(500); + while (rs.compactSplitThread.getCompactionQueueSize() > 0) { + Threads.sleep(50); + } + } + + /** Blocks until the region split is complete in META and region server opens the daughters */ + public static void blockUntilRegionSplit(Configuration conf, long timeout, + final byte[] regionName, boolean waitForDaughters) + throws IOException, InterruptedException { + long start = System.currentTimeMillis(); + log("blocking until region is split:" + Bytes.toStringBinary(regionName)); + HRegionInfo daughterA = null, daughterB = null; + HTable metaTable = new HTable(conf, HConstants.META_TABLE_NAME); + + try { + while (System.currentTimeMillis() - start < timeout) { + Result result = getRegionRow(metaTable, regionName); + if (result == null) { + break; + } + + HRegionInfo region = MetaReader.parseCatalogResult(result).getFirst(); + if(region.isSplitParent()) { + log("found parent region: " + region.toString()); + PairOfSameType pair = MetaReader.getDaughterRegions(result); + daughterA = pair.getFirst(); + daughterB = pair.getSecond(); + break; + } + Threads.sleep(100); + } + + //if we are here, this means the region split is complete or timed out + if (waitForDaughters) { + long rem = timeout - (System.currentTimeMillis() - start); + blockUntilRegionIsInMeta(metaTable, rem, daughterA); + + rem = timeout - (System.currentTimeMillis() - start); + blockUntilRegionIsInMeta(metaTable, rem, daughterB); + + rem = timeout - (System.currentTimeMillis() - start); + blockUntilRegionIsOpened(conf, rem, daughterA); + + rem = timeout - (System.currentTimeMillis() - start); + blockUntilRegionIsOpened(conf, rem, daughterB); + } + } finally { + IOUtils.closeQuietly(metaTable); + } + } + + public static Result getRegionRow(HTable metaTable, byte[] regionName) throws IOException { + Get get = new Get(regionName); + return metaTable.get(get); + } + + public static void blockUntilRegionIsInMeta(HTable metaTable, long timeout, HRegionInfo hri) + throws IOException, InterruptedException { + log("blocking until region is in META: " + hri.getRegionNameAsString()); + long start = System.currentTimeMillis(); + while (System.currentTimeMillis() - start < timeout) { + Result result = getRegionRow(metaTable, hri.getRegionName()); + if (result != null) { + HRegionInfo info = MetaReader.parseCatalogResult(result).getFirst(); + if (info != null && !info.isOffline()) { + log("found region in META: " + hri.getRegionNameAsString()); + break; + } + } + Threads.sleep(10); + } + } + + public static void blockUntilRegionIsOpened(Configuration conf, long timeout, HRegionInfo hri) + throws IOException, InterruptedException { + log("blocking until region is opened for reading:" + hri.getRegionNameAsString()); + long start = System.currentTimeMillis(); + HTable table = new HTable(conf, hri.getTableName()); + + try { + Get get = new Get(hri.getStartKey()); + while (System.currentTimeMillis() - start < timeout) { + try { + table.get(get); + break; + } catch(IOException ex) { + //wait some more + } + Threads.sleep(10); + } + } finally { + IOUtils.closeQuietly(table); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java new file mode 100644 index 0000000..246b145 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java @@ -0,0 +1,204 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; +import java.util.Arrays; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + + +@Category(SmallTests.class) +public class TestExplicitColumnTracker extends HBaseTestCase { + private boolean PRINT = false; + + private final byte[] col1 = Bytes.toBytes("col1"); + private final byte[] col2 = Bytes.toBytes("col2"); + private final byte[] col3 = Bytes.toBytes("col3"); + private final byte[] col4 = Bytes.toBytes("col4"); + private final byte[] col5 = Bytes.toBytes("col5"); + + private void runTest(int maxVersions, + TreeSet trackColumns, + List scannerColumns, + List expected) throws IOException { + ColumnTracker exp = new ExplicitColumnTracker( + trackColumns, 0, maxVersions, Long.MIN_VALUE); + + + //Initialize result + List result = new ArrayList(); + + long timestamp = 0; + //"Match" + for(byte [] col : scannerColumns){ + result.add(exp.checkColumn(col, 0, col.length, ++timestamp, + KeyValue.Type.Put.getCode(), false)); + } + + assertEquals(expected.size(), result.size()); + for(int i=0; i< expected.size(); i++){ + assertEquals(expected.get(i), result.get(i)); + if(PRINT){ + System.out.println("Expected " +expected.get(i) + ", actual " + + result.get(i)); + } + } + } + + public void testGet_SingleVersion() throws IOException{ + if(PRINT){ + System.out.println("SingleVersion"); + } + + //Create tracker + TreeSet columns = new TreeSet(Bytes.BYTES_COMPARATOR); + //Looking for every other + columns.add(col2); + columns.add(col4); + List expected = new ArrayList(); + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); // col1 + expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); // col2 + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); // col3 + expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW); // col4 + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW); // col5 + int maxVersions = 1; + + //Create "Scanner" + List scanner = new ArrayList(); + scanner.add(col1); + scanner.add(col2); + scanner.add(col3); + scanner.add(col4); + scanner.add(col5); + + runTest(maxVersions, columns, scanner, expected); + } + + public void testGet_MultiVersion() throws IOException{ + if(PRINT){ + System.out.println("\nMultiVersion"); + } + + //Create tracker + TreeSet columns = new TreeSet(Bytes.BYTES_COMPARATOR); + //Looking for every other + columns.add(col2); + columns.add(col4); + + List expected = new ArrayList(); + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); + + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); // col2; 1st version + expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); // col2; 2nd version + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); + + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); + + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); // col4; 1st version + expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW); // col4; 2nd version + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW); + + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW); + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW); + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW); + int maxVersions = 2; + + //Create "Scanner" + List scanner = new ArrayList(); + scanner.add(col1); + scanner.add(col1); + scanner.add(col1); + scanner.add(col2); + scanner.add(col2); + scanner.add(col2); + scanner.add(col3); + scanner.add(col3); + scanner.add(col3); + scanner.add(col4); + scanner.add(col4); + scanner.add(col4); + scanner.add(col5); + scanner.add(col5); + scanner.add(col5); + + //Initialize result + runTest(maxVersions, columns, scanner, expected); + } + + + /** + * hbase-2259 + */ + public void testStackOverflow() throws IOException{ + int maxVersions = 1; + TreeSet columns = new TreeSet(Bytes.BYTES_COMPARATOR); + for (int i = 0; i < 100000; i++) { + columns.add(Bytes.toBytes("col"+i)); + } + + ColumnTracker explicit = new ExplicitColumnTracker(columns, 0, maxVersions, + Long.MIN_VALUE); + for (int i = 0; i < 100000; i+=2) { + byte [] col = Bytes.toBytes("col"+i); + explicit.checkColumn(col, 0, col.length, 1, KeyValue.Type.Put.getCode(), + false); + } + explicit.update(); + + for (int i = 1; i < 100000; i+=2) { + byte [] col = Bytes.toBytes("col"+i); + explicit.checkColumn(col, 0, col.length, 1, KeyValue.Type.Put.getCode(), + false); + } + } + + /** + * Regression test for HBASE-2545 + */ + public void testInfiniteLoop() throws IOException { + TreeSet columns = new TreeSet(Bytes.BYTES_COMPARATOR); + columns.addAll(Arrays.asList(new byte[][] { + col2, col3, col5 })); + List scanner = Arrays.asList( + new byte[][] { col1, col4 }); + List expected = Arrays.asList( + new ScanQueryMatcher.MatchCode[] { + ScanQueryMatcher.MatchCode.SEEK_NEXT_COL, + ScanQueryMatcher.MatchCode.SEEK_NEXT_COL }); + runTest(1, columns, scanner, expected); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java new file mode 100644 index 0000000..89dfbf7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java @@ -0,0 +1,268 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.ref.SoftReference; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FilterFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PositionedReadable; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + + +/** + * Test cases that ensure that file system level errors are bubbled up + * appropriately to clients, rather than swallowed. + */ +@Category(MediumTests.class) +public class TestFSErrorsExposed { + private static final Log LOG = LogFactory.getLog(TestFSErrorsExposed.class); + + HBaseTestingUtility util = new HBaseTestingUtility(); + + /** + * Injects errors into the pread calls of an on-disk file, and makes + * sure those bubble up to the HFile scanner + */ + @Test + public void testHFileScannerThrowsErrors() throws IOException { + Path hfilePath = new Path(new Path( + util.getDataTestDir("internalScannerExposesErrors"), + "regionname"), "familyname"); + HFileSystem hfs = (HFileSystem)util.getTestFileSystem(); + FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs()); + FileSystem fs = new HFileSystem(faultyfs); + CacheConfig cacheConf = new CacheConfig(util.getConfiguration()); + StoreFile.Writer writer = new StoreFile.WriterBuilder( + util.getConfiguration(), cacheConf, hfs, 2*1024) + .withOutputDir(hfilePath) + .build(); + TestStoreFile.writeStoreFile( + writer, Bytes.toBytes("cf"), Bytes.toBytes("qual")); + + StoreFile sf = new StoreFile(fs, writer.getPath(), + util.getConfiguration(), cacheConf, StoreFile.BloomType.NONE, + NoOpDataBlockEncoder.INSTANCE); + + StoreFile.Reader reader = sf.createReader(); + HFileScanner scanner = reader.getScanner(false, true); + + FaultyInputStream inStream = faultyfs.inStreams.get(0).get(); + assertNotNull(inStream); + + scanner.seekTo(); + // Do at least one successful read + assertTrue(scanner.next()); + + faultyfs.startFaults(); + + try { + int scanned=0; + while (scanner.next()) { + scanned++; + } + fail("Scanner didn't throw after faults injected"); + } catch (IOException ioe) { + LOG.info("Got expected exception", ioe); + assertTrue(ioe.getMessage().contains("Fault")); + } + reader.close(true); // end of test so evictOnClose + } + + /** + * Injects errors into the pread calls of an on-disk file, and makes + * sure those bubble up to the StoreFileScanner + */ + @Test + public void testStoreFileScannerThrowsErrors() throws IOException { + Path hfilePath = new Path(new Path( + util.getDataTestDir("internalScannerExposesErrors"), + "regionname"), "familyname"); + HFileSystem hfs = (HFileSystem)util.getTestFileSystem(); + FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs()); + HFileSystem fs = new HFileSystem(faultyfs); + CacheConfig cacheConf = new CacheConfig(util.getConfiguration()); + StoreFile.Writer writer = new StoreFile.WriterBuilder( + util.getConfiguration(), cacheConf, hfs, 2 * 1024) + .withOutputDir(hfilePath) + .build(); + TestStoreFile.writeStoreFile( + writer, Bytes.toBytes("cf"), Bytes.toBytes("qual")); + + StoreFile sf = new StoreFile(fs, writer.getPath(), util.getConfiguration(), + cacheConf, BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + + List scanners = StoreFileScanner.getScannersForStoreFiles( + Collections.singletonList(sf), false, true, false); + KeyValueScanner scanner = scanners.get(0); + + FaultyInputStream inStream = faultyfs.inStreams.get(0).get(); + assertNotNull(inStream); + + scanner.seek(KeyValue.LOWESTKEY); + // Do at least one successful read + assertNotNull(scanner.next()); + faultyfs.startFaults(); + + try { + int scanned=0; + while (scanner.next() != null) { + scanned++; + } + fail("Scanner didn't throw after faults injected"); + } catch (IOException ioe) { + LOG.info("Got expected exception", ioe); + assertTrue(ioe.getMessage().contains("Could not iterate")); + } + scanner.close(); + } + + /** + * Cluster test which starts a region server with a region, then + * removes the data from HDFS underneath it, and ensures that + * errors are bubbled to the client. + */ + @Test + public void testFullSystemBubblesFSErrors() throws Exception { + try { + // We set it not to run or it will trigger server shutdown while sync'ing + // because all the datanodes are bad + util.getConfiguration().setInt( + "hbase.regionserver.optionallogflushinterval", Integer.MAX_VALUE); + util.startMiniCluster(1); + byte[] tableName = Bytes.toBytes("table"); + byte[] fam = Bytes.toBytes("fam"); + + HBaseAdmin admin = new HBaseAdmin(util.getConfiguration()); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(fam) + .setMaxVersions(1) + .setBlockCacheEnabled(false) + ); + admin.createTable(desc); + // Make it fail faster. + util.getConfiguration().setInt("hbase.client.retries.number", 1); + // Make a new Configuration so it makes a new connection that has the + // above configuration on it; else we use the old one w/ 10 as default. + HTable table = new HTable(new Configuration(util.getConfiguration()), tableName); + + // Load some data + util.loadTable(table, fam); + table.flushCommits(); + util.flush(); + util.countRows(table); + + // Kill the DFS cluster + util.getDFSCluster().shutdownDataNodes(); + + try { + util.countRows(table); + fail("Did not fail to count after removing data"); + } catch (Exception e) { + LOG.info("Got expected error", e); + assertTrue(e.getMessage().contains("Could not seek")); + } + + } finally { + util.shutdownMiniCluster(); + } + } + + static class FaultyFileSystem extends FilterFileSystem { + List> inStreams = + new ArrayList>(); + + public FaultyFileSystem(FileSystem testFileSystem) { + super(testFileSystem); + } + + @Override + public FSDataInputStream open(Path p, int bufferSize) throws IOException { + FSDataInputStream orig = fs.open(p, bufferSize); + FaultyInputStream faulty = new FaultyInputStream(orig); + inStreams.add(new SoftReference(faulty)); + return faulty; + } + + /** + * Starts to simulate faults on all streams opened so far + */ + public void startFaults() { + for (SoftReference is: inStreams) { + is.get().startFaults(); + } + } + } + + static class FaultyInputStream extends FSDataInputStream { + boolean faultsStarted = false; + + public FaultyInputStream(InputStream in) throws IOException { + super(in); + } + + public void startFaults() { + faultsStarted = true; + } + + public int read(long position, byte[] buffer, int offset, int length) + throws IOException { + injectFault(); + return ((PositionedReadable)in).read(position, buffer, offset, length); + } + + private void injectFault() throws IOException { + if (faultsStarted) { + throw new IOException("Fault injected"); + } + } + } + + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java new file mode 100644 index 0000000..5f97167 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java @@ -0,0 +1,348 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.experimental.categories.Category; + +/** + * {@link TestGet} is a medley of tests of get all done up as a single test. + * This class + */ +@Category(SmallTests.class) +public class TestGetClosestAtOrBefore extends HBaseTestCase { + private static final Log LOG = LogFactory.getLog(TestGetClosestAtOrBefore.class); + + private static final byte[] T00 = Bytes.toBytes("000"); + private static final byte[] T10 = Bytes.toBytes("010"); + private static final byte[] T11 = Bytes.toBytes("011"); + private static final byte[] T12 = Bytes.toBytes("012"); + private static final byte[] T20 = Bytes.toBytes("020"); + private static final byte[] T30 = Bytes.toBytes("030"); + private static final byte[] T31 = Bytes.toBytes("031"); + private static final byte[] T35 = Bytes.toBytes("035"); + private static final byte[] T40 = Bytes.toBytes("040"); + + + + public void testUsingMetaAndBinary() throws IOException { + FileSystem filesystem = FileSystem.get(conf); + Path rootdir = testDir; + // Up flush size else we bind up when we use default catalog flush of 16k. + HTableDescriptor.META_TABLEDESC.setMemStoreFlushSize(64 * 1024 * 1024); + + HRegion mr = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, + rootdir, this.conf, HTableDescriptor.META_TABLEDESC); + try { + // Write rows for three tables 'A', 'B', and 'C'. + for (char c = 'A'; c < 'D'; c++) { + HTableDescriptor htd = new HTableDescriptor("" + c); + final int last = 128; + final int interval = 2; + for (int i = 0; i <= last; i += interval) { + HRegionInfo hri = new HRegionInfo(htd.getName(), + i == 0? HConstants.EMPTY_BYTE_ARRAY: Bytes.toBytes((byte)i), + i == last? HConstants.EMPTY_BYTE_ARRAY: Bytes.toBytes((byte)i + interval)); + Put put = new Put(hri.getRegionName()); + put.setWriteToWAL(false); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + mr.put(put, false); + } + } + InternalScanner s = mr.getScanner(new Scan()); + try { + List keys = new ArrayList(); + while(s.next(keys)) { + LOG.info(keys); + keys.clear(); + } + } finally { + s.close(); + } + findRow(mr, 'C', 44, 44); + findRow(mr, 'C', 45, 44); + findRow(mr, 'C', 46, 46); + findRow(mr, 'C', 43, 42); + mr.flushcache(); + findRow(mr, 'C', 44, 44); + findRow(mr, 'C', 45, 44); + findRow(mr, 'C', 46, 46); + findRow(mr, 'C', 43, 42); + // Now delete 'C' and make sure I don't get entries from 'B'. + byte [] firstRowInC = HRegionInfo.createRegionName(Bytes.toBytes("" + 'C'), + HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false); + Scan scan = new Scan(firstRowInC); + s = mr.getScanner(scan); + try { + List keys = new ArrayList(); + while (s.next(keys)) { + mr.delete(new Delete(keys.get(0).getRow()), null, false); + keys.clear(); + } + } finally { + s.close(); + } + // Assert we get null back (pass -1). + findRow(mr, 'C', 44, -1); + findRow(mr, 'C', 45, -1); + findRow(mr, 'C', 46, -1); + findRow(mr, 'C', 43, -1); + mr.flushcache(); + findRow(mr, 'C', 44, -1); + findRow(mr, 'C', 45, -1); + findRow(mr, 'C', 46, -1); + findRow(mr, 'C', 43, -1); + } finally { + if (mr != null) { + try { + mr.close(); + } catch (Exception e) { + e.printStackTrace(); + } + mr.getLog().closeAndDelete(); + } + } + } + + /* + * @param mr + * @param table + * @param rowToFind + * @param answer Pass -1 if we're not to find anything. + * @return Row found. + * @throws IOException + */ + private byte [] findRow(final HRegion mr, final char table, + final int rowToFind, final int answer) + throws IOException { + byte [] tableb = Bytes.toBytes("" + table); + // Find the row. + byte [] tofindBytes = Bytes.toBytes((short)rowToFind); + byte [] metaKey = HRegionInfo.createRegionName(tableb, tofindBytes, + HConstants.NINES, false); + LOG.info("find=" + new String(metaKey)); + Result r = mr.getClosestRowBefore(metaKey); + if (answer == -1) { + assertNull(r); + return null; + } + assertTrue(Bytes.compareTo(Bytes.toBytes((short)answer), + extractRowFromMetaRow(r.getRow())) == 0); + return r.getRow(); + } + + private byte [] extractRowFromMetaRow(final byte [] b) { + int firstDelimiter = KeyValue.getDelimiter(b, 0, b.length, + HRegionInfo.DELIMITER); + int lastDelimiter = KeyValue.getDelimiterInReverse(b, 0, b.length, + HRegionInfo.DELIMITER); + int length = lastDelimiter - firstDelimiter - 1; + byte [] row = new byte[length]; + System.arraycopy(b, firstDelimiter + 1, row, 0, length); + return row; + } + + /** + * Test file of multiple deletes and with deletes as final key. + * @see HBASE-751 + */ + public void testGetClosestRowBefore3() throws IOException{ + HRegion region = null; + byte [] c0 = COLUMNS[0]; + byte [] c1 = COLUMNS[1]; + try { + HTableDescriptor htd = createTableDescriptor(getName()); + region = createNewHRegion(htd, null, null); + + Put p = new Put(T00); + p.add(c0, c0, T00); + region.put(p); + + p = new Put(T10); + p.add(c0, c0, T10); + region.put(p); + + p = new Put(T20); + p.add(c0, c0, T20); + region.put(p); + + Result r = region.getClosestRowBefore(T20, c0); + assertTrue(Bytes.equals(T20, r.getRow())); + + Delete d = new Delete(T20); + d.deleteColumn(c0, c0); + region.delete(d, null, false); + + r = region.getClosestRowBefore(T20, c0); + assertTrue(Bytes.equals(T10, r.getRow())); + + p = new Put(T30); + p.add(c0, c0, T30); + region.put(p); + + r = region.getClosestRowBefore(T30, c0); + assertTrue(Bytes.equals(T30, r.getRow())); + + d = new Delete(T30); + d.deleteColumn(c0, c0); + region.delete(d, null, false); + + r = region.getClosestRowBefore(T30, c0); + assertTrue(Bytes.equals(T10, r.getRow())); + r = region.getClosestRowBefore(T31, c0); + assertTrue(Bytes.equals(T10, r.getRow())); + + region.flushcache(); + + // try finding "010" after flush + r = region.getClosestRowBefore(T30, c0); + assertTrue(Bytes.equals(T10, r.getRow())); + r = region.getClosestRowBefore(T31, c0); + assertTrue(Bytes.equals(T10, r.getRow())); + + // Put into a different column family. Should make it so I still get t10 + p = new Put(T20); + p.add(c1, c1, T20); + region.put(p); + + r = region.getClosestRowBefore(T30, c0); + assertTrue(Bytes.equals(T10, r.getRow())); + r = region.getClosestRowBefore(T31, c0); + assertTrue(Bytes.equals(T10, r.getRow())); + + region.flushcache(); + + r = region.getClosestRowBefore(T30, c0); + assertTrue(Bytes.equals(T10, r.getRow())); + r = region.getClosestRowBefore(T31, c0); + assertTrue(Bytes.equals(T10, r.getRow())); + + // Now try combo of memcache and mapfiles. Delete the t20 COLUMS[1] + // in memory; make sure we get back t10 again. + d = new Delete(T20); + d.deleteColumn(c1, c1); + region.delete(d, null, false); + r = region.getClosestRowBefore(T30, c0); + assertTrue(Bytes.equals(T10, r.getRow())); + + // Ask for a value off the end of the file. Should return t10. + r = region.getClosestRowBefore(T31, c0); + assertTrue(Bytes.equals(T10, r.getRow())); + region.flushcache(); + r = region.getClosestRowBefore(T31, c0); + assertTrue(Bytes.equals(T10, r.getRow())); + + // Ok. Let the candidate come out of hfile but have delete of + // the candidate be in memory. + p = new Put(T11); + p.add(c0, c0, T11); + region.put(p); + d = new Delete(T10); + d.deleteColumn(c1, c1); + r = region.getClosestRowBefore(T12, c0); + assertTrue(Bytes.equals(T11, r.getRow())); + } finally { + if (region != null) { + try { + region.close(); + } catch (Exception e) { + e.printStackTrace(); + } + region.getLog().closeAndDelete(); + } + } + } + + /** For HBASE-694 */ + public void testGetClosestRowBefore2() throws IOException{ + HRegion region = null; + byte [] c0 = COLUMNS[0]; + try { + HTableDescriptor htd = createTableDescriptor(getName()); + region = createNewHRegion(htd, null, null); + + Put p = new Put(T10); + p.add(c0, c0, T10); + region.put(p); + + p = new Put(T30); + p.add(c0, c0, T30); + region.put(p); + + p = new Put(T40); + p.add(c0, c0, T40); + region.put(p); + + // try finding "035" + Result r = region.getClosestRowBefore(T35, c0); + assertTrue(Bytes.equals(T30, r.getRow())); + + region.flushcache(); + + // try finding "035" + r = region.getClosestRowBefore(T35, c0); + assertTrue(Bytes.equals(T30, r.getRow())); + + p = new Put(T20); + p.add(c0, c0, T20); + region.put(p); + + // try finding "035" + r = region.getClosestRowBefore(T35, c0); + assertTrue(Bytes.equals(T30, r.getRow())); + + region.flushcache(); + + // try finding "035" + r = region.getClosestRowBefore(T35, c0); + assertTrue(Bytes.equals(T30, r.getRow())); + } finally { + if (region != null) { + try { + region.close(); + } catch (Exception e) { + e.printStackTrace(); + } + region.getLog().closeAndDelete(); + } + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java new file mode 100644 index 0000000..f5ce0b0 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHBase7051.java @@ -0,0 +1,204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.MultithreadedTestUtil; +import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext; +import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + +/** + * Test of HBASE-7051; that checkAndPuts and puts behave atomically with respect to each other. + * Rather than perform a bunch of trials to verify atomicity, this test recreates a race condition + * that causes the test to fail if checkAndPut doesn't wait for outstanding put transactions + * to complete. It does this by invasively overriding HRegion function to affect the timing of + * the operations. + */ +@Category(SmallTests.class) +public class TestHBase7051 { + + private static CountDownLatch latch = new CountDownLatch(1); + private enum TestStep { + INIT, // initial put of 10 to set value of the cell + PUT_STARTED, // began doing a put of 50 to cell + PUT_COMPLETED, // put complete (released RowLock, but may not have advanced MVCC). + CHECKANDPUT_STARTED, // began checkAndPut: if 10 -> 11 + CHECKANDPUT_COMPLETED // completed checkAndPut + // NOTE: at the end of these steps, the value of the cell should be 50, not 11! + } + private static volatile TestStep testStep = TestStep.INIT; + private final String family = "f1"; + + @Test + public void testPutAndCheckAndPutInParallel() throws Exception { + + final String tableName = "testPutAndCheckAndPut"; + Configuration conf = HBaseConfiguration.create(); + conf.setClass(HConstants.REGION_IMPL, MockHRegion.class, HeapSize.class); + final MockHRegion region = (MockHRegion) TestHRegion.initHRegion(Bytes.toBytes(tableName), + tableName, conf, Bytes.toBytes(family)); + + List> putsAndLocks = Lists.newArrayList(); + Put[] puts = new Put[1]; + Put put = new Put(Bytes.toBytes("r1")); + put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("10")); + puts[0] = put; + Pair pair = new Pair(puts[0], null); + + putsAndLocks.add(pair); + + region.batchMutate(putsAndLocks.toArray(new Pair[0])); + MultithreadedTestUtil.TestContext ctx = + new MultithreadedTestUtil.TestContext(conf); + ctx.addThread(new PutThread(ctx, region)); + ctx.addThread(new CheckAndPutThread(ctx, region)); + ctx.startThreads(); + while (testStep != TestStep.CHECKANDPUT_COMPLETED) { + Thread.sleep(100); + } + ctx.stop(); + Scan s = new Scan(); + RegionScanner scanner = region.getScanner(s); + List results = new ArrayList(); + scanner.next(results, 2); + for (KeyValue keyValue : results) { + assertEquals("50",Bytes.toString(keyValue.getValue())); + } + + } + + private class PutThread extends TestThread { + private MockHRegion region; + PutThread(TestContext ctx, MockHRegion region) { + super(ctx); + this.region = region; + } + + public void doWork() throws Exception { + List> putsAndLocks = Lists.newArrayList(); + Put[] puts = new Put[1]; + Put put = new Put(Bytes.toBytes("r1")); + put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("50")); + puts[0] = put; + Pair pair = new Pair(puts[0], null); + putsAndLocks.add(pair); + testStep = TestStep.PUT_STARTED; + region.batchMutate(putsAndLocks.toArray(new Pair[0])); + } + } + + private class CheckAndPutThread extends TestThread { + private MockHRegion region; + CheckAndPutThread(TestContext ctx, MockHRegion region) { + super(ctx); + this.region = region; + } + + public void doWork() throws Exception { + Put[] puts = new Put[1]; + Put put = new Put(Bytes.toBytes("r1")); + put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("11")); + puts[0] = put; + while (testStep != TestStep.PUT_COMPLETED) { + Thread.sleep(100); + } + testStep = TestStep.CHECKANDPUT_STARTED; + region.checkAndMutate(Bytes.toBytes("r1"), Bytes.toBytes(family), Bytes.toBytes("q1"), + CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("10")), put, null, true); + testStep = TestStep.CHECKANDPUT_COMPLETED; + } + } + + public static class MockHRegion extends HRegion { + + public MockHRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf, + final HRegionInfo regionInfo, final HTableDescriptor htd, RegionServerServices rsServices) { + super(tableDir, log, fs, conf, regionInfo, htd, rsServices); + } + + @Override + public void releaseRowLock(Integer lockId) { + if (testStep == TestStep.INIT) { + super.releaseRowLock(lockId); + return; + } + + if (testStep == TestStep.PUT_STARTED) { + try { + testStep = TestStep.PUT_COMPLETED; + super.releaseRowLock(lockId); + // put has been written to the memstore and the row lock has been released, but the + // MVCC has not been advanced. Prior to fixing HBASE-7051, the following order of + // operations would cause the non-atomicity to show up: + // 1) Put releases row lock (where we are now) + // 2) CheckAndPut grabs row lock and reads the value prior to the put (10) + // because the MVCC has not advanced + // 3) Put advances MVCC + // So, in order to recreate this order, we wait for the checkAndPut to grab the rowLock + // (see below), and then wait some more to give the checkAndPut time to read the old + // value. + latch.await(); + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + else if (testStep == TestStep.CHECKANDPUT_STARTED) { + super.releaseRowLock(lockId); + } + } + + @Override + public Integer getLock(Integer lockid, byte[] row, boolean waitForLock) throws IOException { + if (testStep == TestStep.CHECKANDPUT_STARTED) { + latch.countDown(); + } + return super.getLock(lockid, row, waitForLock); + } + + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java new file mode 100644 index 0000000..28419d7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -0,0 +1,4333 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HConstants.OperationStatusCode; +import org.apache.hadoop.hbase.HDFSBlocksDistribution; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.MultithreadedTestUtil; +import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread; +import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterBase; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.NullComparator; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; +import org.apache.hadoop.hbase.monitoring.MonitoredTask; +import org.apache.hadoop.hbase.monitoring.TaskMonitor; +import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; +import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge; +import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.PairOfSameType; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import com.google.common.collect.Lists; + + +/** + * Basic stand-alone testing of HRegion. + * + * A lot of the meta information for an HRegion now lives inside other + * HRegions or in the HBaseMaster, so only basic testing is possible. + */ +@Category(MediumTests.class) +@SuppressWarnings("deprecation") +public class TestHRegion extends HBaseTestCase { + // Do not spin up clusters in here. If you need to spin up a cluster, do it + // over in TestHRegionOnCluster. + static final Log LOG = LogFactory.getLog(TestHRegion.class); + + private static final String COLUMN_FAMILY = "MyCF"; + + HRegion region = null; + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString(); + + private final int MAX_VERSIONS = 2; + + // Test names + protected final byte[] tableName = Bytes.toBytes("testtable");; + protected final byte[] qual1 = Bytes.toBytes("qual1"); + protected final byte[] qual2 = Bytes.toBytes("qual2"); + protected final byte[] qual3 = Bytes.toBytes("qual3"); + protected final byte[] value1 = Bytes.toBytes("value1"); + protected final byte[] value2 = Bytes.toBytes("value2"); + protected final byte [] row = Bytes.toBytes("rowA"); + protected final byte [] row2 = Bytes.toBytes("rowB"); + + + private Map startingMetrics; + + /** + * @see org.apache.hadoop.hbase.HBaseTestCase#setUp() + */ + @Override + protected void setUp() throws Exception { + startingMetrics = SchemaMetrics.getMetricsSnapshot(); + super.setUp(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + EnvironmentEdgeManagerTestHelper.reset(); + SchemaMetrics.validateMetricChanges(startingMetrics); + } + + ////////////////////////////////////////////////////////////////////////////// + // New tests that doesn't spin up a mini cluster but rather just test the + // individual code pieces in the HRegion. Putting files locally in + // /tmp/testtable + ////////////////////////////////////////////////////////////////////////////// + + public void testCompactionAffectedByScanners() throws Exception { + String method = "testCompactionAffectedByScanners"; + byte[] tableName = Bytes.toBytes(method); + byte[] family = Bytes.toBytes("family"); + this.region = initHRegion(tableName, method, conf, family); + + Put put = new Put(Bytes.toBytes("r1")); + put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); + region.put(put); + region.flushcache(); + + + Scan scan = new Scan(); + scan.setMaxVersions(3); + // open the first scanner + RegionScanner scanner1 = region.getScanner(scan); + + Delete delete = new Delete(Bytes.toBytes("r1")); + region.delete(delete, null, false); + region.flushcache(); + + // open the second scanner + RegionScanner scanner2 = region.getScanner(scan); + + List results = new ArrayList(); + + System.out.println("Smallest read point:" + region.getSmallestReadPoint()); + + // make a major compaction + region.compactStores(true); + + // open the third scanner + RegionScanner scanner3 = region.getScanner(scan); + + // get data from scanner 1, 2, 3 after major compaction + scanner1.next(results); + System.out.println(results); + assertEquals(1, results.size()); + + results.clear(); + scanner2.next(results); + System.out.println(results); + assertEquals(0, results.size()); + + results.clear(); + scanner3.next(results); + System.out.println(results); + assertEquals(0, results.size()); + } + + @Test + public void testToShowNPEOnRegionScannerReseek() throws Exception{ + String method = "testToShowNPEOnRegionScannerReseek"; + byte[] tableName = Bytes.toBytes(method); + byte[] family = Bytes.toBytes("family"); + this.region = initHRegion(tableName, method, conf, family); + + Put put = new Put(Bytes.toBytes("r1")); + put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); + region.put(put); + put = new Put(Bytes.toBytes("r2")); + put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); + region.put(put); + region.flushcache(); + + + Scan scan = new Scan(); + scan.setMaxVersions(3); + // open the first scanner + RegionScanner scanner1 = region.getScanner(scan); + + System.out.println("Smallest read point:" + region.getSmallestReadPoint()); + + region.compactStores(true); + + scanner1.reseek(Bytes.toBytes("r2")); + List results = new ArrayList(); + scanner1.next(results); + KeyValue keyValue = results.get(0); + Assert.assertTrue(Bytes.compareTo(keyValue.getRow(), Bytes.toBytes("r2")) == 0); + scanner1.close(); + } + + public void testSkipRecoveredEditsReplay() throws Exception { + String method = "testSkipRecoveredEditsReplay"; + byte[] tableName = Bytes.toBytes(method); + byte[] family = Bytes.toBytes("family"); + this.region = initHRegion(tableName, method, conf, family); + try { + Path regiondir = region.getRegionDir(); + FileSystem fs = region.getFilesystem(); + byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); + + Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir); + + long maxSeqId = 1050; + long minSeqId = 1000; + + for (long i = minSeqId; i <= maxSeqId; i += 10) { + Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); + fs.create(recoveredEdits); + HLog.Writer writer = HLog.createWriter(fs, recoveredEdits, conf); + + long time = System.nanoTime(); + WALEdit edit = new WALEdit(); + edit.add(new KeyValue(row, family, Bytes.toBytes(i), + time, KeyValue.Type.Put, Bytes.toBytes(i))); + writer.append(new HLog.Entry(new HLogKey(regionName, tableName, + i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); + + writer.close(); + } + MonitoredTask status = TaskMonitor.get().createStatus(method); + Map maxSeqIdInStores = new TreeMap( + Bytes.BYTES_COMPARATOR); + for (Store store : region.getStores().values()) { + maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), + minSeqId - 1); + } + long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status); + assertEquals(maxSeqId, seqId); + Get get = new Get(row); + Result result = region.get(get, null); + for (long i = minSeqId; i <= maxSeqId; i += 10) { + List kvs = result.getColumn(family, Bytes.toBytes(i)); + assertEquals(1, kvs.size()); + assertEquals(Bytes.toBytes(i), kvs.get(0).getValue()); + } + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception { + String method = "testSkipRecoveredEditsReplaySomeIgnored"; + byte[] tableName = Bytes.toBytes(method); + byte[] family = Bytes.toBytes("family"); + this.region = initHRegion(tableName, method, conf, family); + try { + Path regiondir = region.getRegionDir(); + FileSystem fs = region.getFilesystem(); + byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); + + Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir); + + long maxSeqId = 1050; + long minSeqId = 1000; + + for (long i = minSeqId; i <= maxSeqId; i += 10) { + Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); + fs.create(recoveredEdits); + HLog.Writer writer = HLog.createWriter(fs, recoveredEdits, conf); + + long time = System.nanoTime(); + WALEdit edit = new WALEdit(); + edit.add(new KeyValue(row, family, Bytes.toBytes(i), + time, KeyValue.Type.Put, Bytes.toBytes(i))); + writer.append(new HLog.Entry(new HLogKey(regionName, tableName, + i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); + + writer.close(); + } + long recoverSeqId = 1030; + MonitoredTask status = TaskMonitor.get().createStatus(method); + Map maxSeqIdInStores = new TreeMap( + Bytes.BYTES_COMPARATOR); + for (Store store : region.getStores().values()) { + maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), + recoverSeqId - 1); + } + long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status); + assertEquals(maxSeqId, seqId); + Get get = new Get(row); + Result result = region.get(get, null); + for (long i = minSeqId; i <= maxSeqId; i += 10) { + List kvs = result.getColumn(family, Bytes.toBytes(i)); + if (i < recoverSeqId) { + assertEquals(0, kvs.size()); + } else { + assertEquals(1, kvs.size()); + assertEquals(Bytes.toBytes(i), kvs.get(0).getValue()); + } + } + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testSkipRecoveredEditsReplayAllIgnored() throws Exception { + String method = "testSkipRecoveredEditsReplayAllIgnored"; + byte[] tableName = Bytes.toBytes(method); + byte[] family = Bytes.toBytes("family"); + this.region = initHRegion(tableName, method, conf, family); + try { + Path regiondir = region.getRegionDir(); + FileSystem fs = region.getFilesystem(); + + Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir); + for (int i = 1000; i < 1050; i += 10) { + Path recoveredEdits = new Path( + recoveredEditsDir, String.format("%019d", i)); + FSDataOutputStream dos= fs.create(recoveredEdits); + dos.writeInt(i); + dos.close(); + } + long minSeqId = 2000; + Path recoveredEdits = new Path( + recoveredEditsDir, String.format("%019d", minSeqId-1)); + FSDataOutputStream dos= fs.create(recoveredEdits); + dos.close(); + + Map maxSeqIdInStores = new TreeMap( + Bytes.BYTES_COMPARATOR); + for (Store store : region.getStores().values()) { + maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId); + } + long seqId = region.replayRecoveredEditsIfAny(regiondir, + maxSeqIdInStores, null, null); + assertEquals(minSeqId, seqId); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testGetWhileRegionClose() throws IOException { + Configuration hc = initSplit(); + int numRows = 100; + byte [][] families = {fam1, fam2, fam3}; + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, hc, families); + try { + // Put data in region + final int startRow = 100; + putData(startRow, numRows, qual1, families); + putData(startRow, numRows, qual2, families); + putData(startRow, numRows, qual3, families); + // this.region.flushcache(); + final AtomicBoolean done = new AtomicBoolean(false); + final AtomicInteger gets = new AtomicInteger(0); + GetTillDoneOrException [] threads = new GetTillDoneOrException[10]; + try { + // Set ten threads running concurrently getting from the region. + for (int i = 0; i < threads.length / 2; i++) { + threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), + done, gets); + threads[i].setDaemon(true); + threads[i].start(); + } + // Artificially make the condition by setting closing flag explicitly. + // I can't make the issue happen with a call to region.close(). + this.region.closing.set(true); + for (int i = threads.length / 2; i < threads.length; i++) { + threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), + done, gets); + threads[i].setDaemon(true); + threads[i].start(); + } + } finally { + if (this.region != null) { + this.region.close(); + this.region.getLog().closeAndDelete(); + } + } + done.set(true); + for (GetTillDoneOrException t: threads) { + try { + t.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + if (t.e != null) { + LOG.info("Exception=" + t.e); + assertFalse("Found a NPE in " + t.getName(), + t.e instanceof NullPointerException); + } + } + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + /* + * Thread that does get on single row until 'done' flag is flipped. If an + * exception causes us to fail, it records it. + */ + class GetTillDoneOrException extends Thread { + private final Get g; + private final AtomicBoolean done; + private final AtomicInteger count; + private Exception e; + + GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, + final AtomicInteger c) { + super("getter." + i); + this.g = new Get(r); + this.done = d; + this.count = c; + } + + @Override + public void run() { + while (!this.done.get()) { + try { + assertTrue(region.get(g, null).size() > 0); + this.count.incrementAndGet(); + } catch (Exception e) { + this.e = e; + break; + } + } + } + } + + /* + * An involved filter test. Has multiple column families and deletes in mix. + */ + public void testWeirdCacheBehaviour() throws Exception { + byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour"); + byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), + Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"), + Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") }; + this.region = initHRegion(TABLE, getName(), conf, FAMILIES); + try { + String value = "this is the value"; + String value2 = "this is some other value"; + String keyPrefix1 = "prefix1"; // UUID.randomUUID().toString(); + String keyPrefix2 = "prefix2"; // UUID.randomUUID().toString(); + String keyPrefix3 = "prefix3"; // UUID.randomUUID().toString(); + putRows(this.region, 3, value, keyPrefix1); + putRows(this.region, 3, value, keyPrefix2); + putRows(this.region, 3, value, keyPrefix3); + // this.region.flushCommits(); + putRows(this.region, 3, value2, keyPrefix1); + putRows(this.region, 3, value2, keyPrefix2); + putRows(this.region, 3, value2, keyPrefix3); + System.out.println("Checking values for key: " + keyPrefix1); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix1, value2, this.region)); + System.out.println("Checking values for key: " + keyPrefix2); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix2, value2, this.region)); + System.out.println("Checking values for key: " + keyPrefix3); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix3, value2, this.region)); + deleteColumns(this.region, value2, keyPrefix1); + deleteColumns(this.region, value2, keyPrefix2); + deleteColumns(this.region, value2, keyPrefix3); + System.out.println("Starting important checks....."); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, + 0, getNumberOfRows(keyPrefix1, value2, this.region)); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, + 0, getNumberOfRows(keyPrefix2, value2, this.region)); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, + 0, getNumberOfRows(keyPrefix3, value2, this.region)); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testAppendWithReadOnlyTable() throws Exception { + byte[] TABLE = Bytes.toBytes("readOnlyTable"); + this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily")); + boolean exceptionCaught = false; + Append append = new Append(Bytes.toBytes("somerow")); + append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), + Bytes.toBytes("somevalue")); + try { + region.append(append, false); + } catch (IOException e) { + exceptionCaught = true; + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + assertTrue(exceptionCaught == true); + } + + public void testIncrWithReadOnlyTable() throws Exception { + byte[] TABLE = Bytes.toBytes("readOnlyTable"); + this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily")); + boolean exceptionCaught = false; + Increment inc = new Increment(Bytes.toBytes("somerow")); + inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L); + try { + region.increment(inc, false); + } catch (IOException e) { + exceptionCaught = true; + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + assertTrue(exceptionCaught == true); + } + + private void deleteColumns(HRegion r, String value, String keyPrefix) + throws IOException { + InternalScanner scanner = buildScanner(keyPrefix, value, r); + int count = 0; + boolean more = false; + List results = new ArrayList(); + do { + more = scanner.next(results); + if (results != null && !results.isEmpty()) + count++; + else + break; + Delete delete = new Delete(results.get(0).getRow()); + delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2")); + r.delete(delete, null, false); + results.clear(); + } while (more); + assertEquals("Did not perform correct number of deletes", 3, count); + } + + private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception { + InternalScanner resultScanner = buildScanner(keyPrefix, value, r); + int numberOfResults = 0; + List results = new ArrayList(); + boolean more = false; + do { + more = resultScanner.next(results); + if (results != null && !results.isEmpty()) numberOfResults++; + else break; + for (KeyValue kv: results) { + System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(kv.getValue())); + } + results.clear(); + } while(more); + return numberOfResults; + } + + private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) + throws IOException { + // Defaults FilterList.Operator.MUST_PASS_ALL. + FilterList allFilters = new FilterList(); + allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); + // Only return rows where this column value exists in the row. + SingleColumnValueFilter filter = + new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), + Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)); + filter.setFilterIfMissing(true); + allFilters.addFilter(filter); + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("trans-blob")); + scan.addFamily(Bytes.toBytes("trans-type")); + scan.addFamily(Bytes.toBytes("trans-date")); + scan.addFamily(Bytes.toBytes("trans-tags")); + scan.addFamily(Bytes.toBytes("trans-group")); + scan.setFilter(allFilters); + return r.getScanner(scan); + } + + private void putRows(HRegion r, int numRows, String value, String key) + throws IOException { + for (int i = 0; i < numRows; i++) { + String row = key + "_" + i/* UUID.randomUUID().toString() */; + System.out.println(String.format("Saving row: %s, with value %s", row, + value)); + Put put = new Put(Bytes.toBytes(row)); + put.setWriteToWAL(false); + put.add(Bytes.toBytes("trans-blob"), null, + Bytes.toBytes("value for blob")); + put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement")); + put.add(Bytes.toBytes("trans-date"), null, + Bytes.toBytes("20090921010101999")); + put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), + Bytes.toBytes(value)); + put.add(Bytes.toBytes("trans-group"), null, + Bytes.toBytes("adhocTransactionGroupId")); + r.put(put); + } + } + + public void testFamilyWithAndWithoutColon() throws Exception { + byte [] b = Bytes.toBytes(getName()); + byte [] cf = Bytes.toBytes(COLUMN_FAMILY); + this.region = initHRegion(b, getName(), conf, cf); + try { + Put p = new Put(b); + byte [] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":"); + p.add(cfwithcolon, cfwithcolon, cfwithcolon); + boolean exception = false; + try { + this.region.put(p); + } catch (NoSuchColumnFamilyException e) { + exception = true; + } + assertTrue(exception); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + @SuppressWarnings("unchecked") + public void testBatchPut() throws Exception { + byte[] b = Bytes.toBytes(getName()); + byte[] cf = Bytes.toBytes(COLUMN_FAMILY); + byte[] qual = Bytes.toBytes("qual"); + byte[] val = Bytes.toBytes("val"); + this.region = initHRegion(b, getName(), conf, cf); + try { + HLog.getSyncTime(); // clear counter from prior tests + assertEquals(0, HLog.getSyncTime().count); + + LOG.info("First a batch put with all valid puts"); + final Put[] puts = new Put[10]; + for (int i = 0; i < 10; i++) { + puts[i] = new Put(Bytes.toBytes("row_" + i)); + puts[i].add(cf, qual, val); + } + + OperationStatus[] codes = this.region.put(puts); + assertEquals(10, codes.length); + for (int i = 0; i < 10; i++) { + assertEquals(OperationStatusCode.SUCCESS, codes[i] + .getOperationStatusCode()); + } + assertEquals(1, HLog.getSyncTime().count); + + LOG.info("Next a batch put with one invalid family"); + puts[5].add(Bytes.toBytes("BAD_CF"), qual, val); + codes = this.region.put(puts); + assertEquals(10, codes.length); + for (int i = 0; i < 10; i++) { + assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : + OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); + } + assertEquals(1, HLog.getSyncTime().count); + + LOG.info("Next a batch put that has to break into two batches to avoid a lock"); + Integer lockedRow = region.obtainRowLock(Bytes.toBytes("row_2")); + + MultithreadedTestUtil.TestContext ctx = + new MultithreadedTestUtil.TestContext(conf); + final AtomicReference retFromThread = + new AtomicReference(); + TestThread putter = new TestThread(ctx) { + @Override + public void doWork() throws IOException { + retFromThread.set(region.put(puts)); + } + }; + LOG.info("...starting put thread while holding lock"); + ctx.addThread(putter); + ctx.startThreads(); + + LOG.info("...waiting for put thread to sync first time"); + long startWait = System.currentTimeMillis(); + while (HLog.getSyncTime().count == 0) { + Thread.sleep(100); + if (System.currentTimeMillis() - startWait > 10000) { + fail("Timed out waiting for thread to sync first minibatch"); + } + } + LOG.info("...releasing row lock, which should let put thread continue"); + region.releaseRowLock(lockedRow); + LOG.info("...joining on thread"); + ctx.stop(); + LOG.info("...checking that next batch was synced"); + assertEquals(1, HLog.getSyncTime().count); + codes = retFromThread.get(); + for (int i = 0; i < 10; i++) { + assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : + OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); + } + + LOG.info("Nexta, a batch put which uses an already-held lock"); + lockedRow = region.obtainRowLock(Bytes.toBytes("row_2")); + LOG.info("...obtained row lock"); + List> putsAndLocks = Lists.newArrayList(); + for (int i = 0; i < 10; i++) { + Pair pair = new Pair(puts[i], null); + if (i == 2) pair.setSecond(lockedRow); + putsAndLocks.add(pair); + } + + codes = region.put(putsAndLocks.toArray(new Pair[0])); + LOG.info("...performed put"); + for (int i = 0; i < 10; i++) { + assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : + OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); + } + // Make sure we didn't do an extra batch + assertEquals(1, HLog.getSyncTime().count); + + // Make sure we still hold lock + assertTrue(region.isRowLocked(lockedRow)); + LOG.info("...releasing lock"); + region.releaseRowLock(lockedRow); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testBatchPutWithTsSlop() throws Exception { + byte[] b = Bytes.toBytes(getName()); + byte[] cf = Bytes.toBytes(COLUMN_FAMILY); + byte[] qual = Bytes.toBytes("qual"); + byte[] val = Bytes.toBytes("val"); + Configuration conf = HBaseConfiguration.create(this.conf); + + // add data with a timestamp that is too recent for range. Ensure assert + conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); + this.region = initHRegion(b, getName(), conf, cf); + + try{ + HLog.getSyncTime(); // clear counter from prior tests + assertEquals(0, HLog.getSyncTime().count); + + final Put[] puts = new Put[10]; + for (int i = 0; i < 10; i++) { + puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100); + puts[i].add(cf, qual, val); + } + + OperationStatus[] codes = this.region.put(puts); + assertEquals(10, codes.length); + for (int i = 0; i < 10; i++) { + assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i] + .getOperationStatusCode()); + } + assertEquals(0, HLog.getSyncTime().count); + + + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + + } + + ////////////////////////////////////////////////////////////////////////////// + // checkAndMutate tests + ////////////////////////////////////////////////////////////////////////////// + public void testCheckAndMutate_WithEmptyRowValue() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [] row1 = Bytes.toBytes("row1"); + byte [] fam1 = Bytes.toBytes("fam1"); + byte [] qf1 = Bytes.toBytes("qualifier"); + byte [] emptyVal = new byte[] {}; + byte [] val1 = Bytes.toBytes("value1"); + byte [] val2 = Bytes.toBytes("value2"); + Integer lockId = null; + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, fam1); + try { + //Putting empty data in key + Put put = new Put(row1); + put.add(fam1, qf1, emptyVal); + + //checkAndPut with empty value + boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new BinaryComparator(emptyVal), put, lockId, true); + assertTrue(res); + + //Putting data in key + put = new Put(row1); + put.add(fam1, qf1, val1); + + //checkAndPut with correct value + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new BinaryComparator(emptyVal), put, lockId, true); + assertTrue(res); + + // not empty anymore + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new BinaryComparator(emptyVal), put, lockId, true); + assertFalse(res); + + Delete delete = new Delete(row1); + delete.deleteColumn(fam1, qf1); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new BinaryComparator(emptyVal), delete, lockId, true); + assertFalse(res); + + put = new Put(row1); + put.add(fam1, qf1, val2); + //checkAndPut with correct value + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new BinaryComparator(val1), put, lockId, true); + assertTrue(res); + + //checkAndDelete with correct value + delete = new Delete(row1); + delete.deleteColumn(fam1, qf1); + delete.deleteColumn(fam1, qf1); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new BinaryComparator(val2), delete, lockId, true); + assertTrue(res); + + delete = new Delete(row1); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new BinaryComparator(emptyVal), delete, lockId, true); + assertTrue(res); + + //checkAndPut looking for a null value + put = new Put(row1); + put.add(fam1, qf1, val1); + + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new NullComparator(), put, lockId, true); + assertTrue(res); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testCheckAndMutate_WithWrongValue() throws IOException{ + byte [] tableName = Bytes.toBytes("testtable"); + byte [] row1 = Bytes.toBytes("row1"); + byte [] fam1 = Bytes.toBytes("fam1"); + byte [] qf1 = Bytes.toBytes("qualifier"); + byte [] val1 = Bytes.toBytes("value1"); + byte [] val2 = Bytes.toBytes("value2"); + Integer lockId = null; + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, fam1); + try { + //Putting data in key + Put put = new Put(row1); + put.add(fam1, qf1, val1); + region.put(put); + + //checkAndPut with wrong value + boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new BinaryComparator(val2), put, lockId, true); + assertEquals(false, res); + + //checkAndDelete with wrong value + Delete delete = new Delete(row1); + delete.deleteFamily(fam1); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new BinaryComparator(val2), delete, lockId, true); + assertEquals(false, res); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testCheckAndMutate_WithCorrectValue() throws IOException{ + byte [] tableName = Bytes.toBytes("testtable"); + byte [] row1 = Bytes.toBytes("row1"); + byte [] fam1 = Bytes.toBytes("fam1"); + byte [] qf1 = Bytes.toBytes("qualifier"); + byte [] val1 = Bytes.toBytes("value1"); + Integer lockId = null; + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, fam1); + try { + //Putting data in key + Put put = new Put(row1); + put.add(fam1, qf1, val1); + region.put(put); + + //checkAndPut with correct value + boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new BinaryComparator(val1), put, lockId, true); + assertEquals(true, res); + + //checkAndDelete with correct value + Delete delete = new Delete(row1); + delete.deleteColumn(fam1, qf1); + res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new BinaryComparator(val1), put, lockId, true); + assertEquals(true, res); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testCheckAndPut_ThatPutWasWritten() throws IOException{ + byte [] tableName = Bytes.toBytes("testtable"); + byte [] row1 = Bytes.toBytes("row1"); + byte [] fam1 = Bytes.toBytes("fam1"); + byte [] fam2 = Bytes.toBytes("fam2"); + byte [] qf1 = Bytes.toBytes("qualifier"); + byte [] val1 = Bytes.toBytes("value1"); + byte [] val2 = Bytes.toBytes("value2"); + Integer lockId = null; + + byte [][] families = {fam1, fam2}; + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, families); + try { + //Putting data in the key to check + Put put = new Put(row1); + put.add(fam1, qf1, val1); + region.put(put); + + //Creating put to add + long ts = System.currentTimeMillis(); + KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); + put = new Put(row1); + put.add(kv); + + //checkAndPut with wrong value + Store store = region.getStore(fam1); + store.memstore.kvset.size(); + + boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, + new BinaryComparator(val1), put, lockId, true); + assertEquals(true, res); + store.memstore.kvset.size(); + + Get get = new Get(row1); + get.addColumn(fam2, qf1); + KeyValue [] actual = region.get(get, null).raw(); + + KeyValue [] expected = {kv}; + + assertEquals(expected.length, actual.length); + for(int i=0; i kvs = new ArrayList(); + kvs.add(new KeyValue(row1, fam4, null, null)); + + + //testing existing family + byte [] family = fam2; + try { + Map> deleteMap = new HashMap>(); + deleteMap.put(family, kvs); + region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true); + } catch (Exception e) { + assertTrue("Family " +new String(family)+ " does not exist", false); + } + + //testing non existing family + boolean ok = false; + family = fam4; + try { + Map> deleteMap = new HashMap>(); + deleteMap.put(family, kvs); + region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true); + } catch (Exception e) { + ok = true; + } + assertEquals("Family " +new String(family)+ " does exist", true, ok); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testDelete_mixed() throws IOException, InterruptedException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [] fam = Bytes.toBytes("info"); + byte [][] families = {fam}; + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, families); + try { + EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); + + byte [] row = Bytes.toBytes("table_name"); + // column names + byte [] serverinfo = Bytes.toBytes("serverinfo"); + byte [] splitA = Bytes.toBytes("splitA"); + byte [] splitB = Bytes.toBytes("splitB"); + + // add some data: + Put put = new Put(row); + put.add(fam, splitA, Bytes.toBytes("reference_A")); + region.put(put); + + put = new Put(row); + put.add(fam, splitB, Bytes.toBytes("reference_B")); + region.put(put); + + put = new Put(row); + put.add(fam, serverinfo, Bytes.toBytes("ip_address")); + region.put(put); + + // ok now delete a split: + Delete delete = new Delete(row); + delete.deleteColumns(fam, splitA); + region.delete(delete, null, true); + + // assert some things: + Get get = new Get(row).addColumn(fam, serverinfo); + Result result = region.get(get, null); + assertEquals(1, result.size()); + + get = new Get(row).addColumn(fam, splitA); + result = region.get(get, null); + assertEquals(0, result.size()); + + get = new Get(row).addColumn(fam, splitB); + result = region.get(get, null); + assertEquals(1, result.size()); + + // Assert that after a delete, I can put. + put = new Put(row); + put.add(fam, splitA, Bytes.toBytes("reference_A")); + region.put(put); + get = new Get(row); + result = region.get(get, null); + assertEquals(3, result.size()); + + // Now delete all... then test I can add stuff back + delete = new Delete(row); + region.delete(delete, null, false); + assertEquals(0, region.get(get, null).size()); + + region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A"))); + result = region.get(get, null); + assertEquals(1, result.size()); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testDeleteRowWithFutureTs() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [] fam = Bytes.toBytes("info"); + byte [][] families = {fam}; + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, families); + try { + byte [] row = Bytes.toBytes("table_name"); + // column names + byte [] serverinfo = Bytes.toBytes("serverinfo"); + + // add data in the far future + Put put = new Put(row); + put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP-5,Bytes.toBytes("value")); + region.put(put); + + // now delete something in the present + Delete delete = new Delete(row); + region.delete(delete, null, true); + + // make sure we still see our data + Get get = new Get(row).addColumn(fam, serverinfo); + Result result = region.get(get, null); + assertEquals(1, result.size()); + + // delete the future row + delete = new Delete(row,HConstants.LATEST_TIMESTAMP-3,null); + region.delete(delete, null, true); + + // make sure it is gone + get = new Get(row).addColumn(fam, serverinfo); + result = region.get(get, null); + assertEquals(0, result.size()); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + /** + * Tests that the special LATEST_TIMESTAMP option for puts gets + * replaced by the actual timestamp + */ + public void testPutWithLatestTS() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [] fam = Bytes.toBytes("info"); + byte [][] families = {fam}; + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, families); + try { + byte [] row = Bytes.toBytes("row1"); + // column names + byte [] qual = Bytes.toBytes("qual"); + + // add data with LATEST_TIMESTAMP, put without WAL + Put put = new Put(row); + put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); + region.put(put, false); + + // Make sure it shows up with an actual timestamp + Get get = new Get(row).addColumn(fam, qual); + Result result = region.get(get, null); + assertEquals(1, result.size()); + KeyValue kv = result.raw()[0]; + LOG.info("Got: " + kv); + assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", + kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); + + // Check same with WAL enabled (historically these took different + // code paths, so check both) + row = Bytes.toBytes("row2"); + put = new Put(row); + put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); + region.put(put, true); + + // Make sure it shows up with an actual timestamp + get = new Get(row).addColumn(fam, qual); + result = region.get(get, null); + assertEquals(1, result.size()); + kv = result.raw()[0]; + LOG.info("Got: " + kv); + assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", + kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + + } + + + /** + * Tests that there is server-side filtering for invalid timestamp upper + * bound. Note that the timestamp lower bound is automatically handled for us + * by the TTL field. + */ + public void testPutWithTsSlop() throws IOException { + byte[] tableName = Bytes.toBytes("testtable"); + byte[] fam = Bytes.toBytes("info"); + byte[][] families = { fam }; + String method = this.getName(); + Configuration conf = HBaseConfiguration.create(this.conf); + + // add data with a timestamp that is too recent for range. Ensure assert + conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); + this.region = initHRegion(tableName, method, conf, families); + boolean caughtExcep = false; + try { + try { + // no TS specified == use latest. should not error + region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes + .toBytes("value")), false); + // TS out of range. should error + region.put(new Put(row).add(fam, Bytes.toBytes("qual"), + System.currentTimeMillis() + 2000, + Bytes.toBytes("value")), false); + fail("Expected IOE for TS out of configured timerange"); + } catch (DoNotRetryIOException ioe) { + LOG.debug("Received expected exception", ioe); + caughtExcep = true; + } + assertTrue("Should catch FailedSanityCheckException", caughtExcep); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testScanner_DeleteOneFamilyNotAnother() throws IOException { + byte [] tableName = Bytes.toBytes("test_table"); + byte [] fam1 = Bytes.toBytes("columnA"); + byte [] fam2 = Bytes.toBytes("columnB"); + this.region = initHRegion(tableName, getName(), conf, fam1, fam2); + try { + byte [] rowA = Bytes.toBytes("rowA"); + byte [] rowB = Bytes.toBytes("rowB"); + + byte [] value = Bytes.toBytes("value"); + + Delete delete = new Delete(rowA); + delete.deleteFamily(fam1); + + region.delete(delete, null, true); + + // now create data. + Put put = new Put(rowA); + put.add(fam2, null, value); + region.put(put); + + put = new Put(rowB); + put.add(fam1, null, value); + put.add(fam2, null, value); + region.put(put); + + Scan scan = new Scan(); + scan.addFamily(fam1).addFamily(fam2); + InternalScanner s = region.getScanner(scan); + List results = new ArrayList(); + s.next(results); + assertTrue(Bytes.equals(rowA, results.get(0).getRow())); + + results.clear(); + s.next(results); + assertTrue(Bytes.equals(rowB, results.get(0).getRow())); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testDeleteColumns_PostInsert() throws IOException, + InterruptedException { + Delete delete = new Delete(row); + delete.deleteColumns(fam1, qual1); + doTestDelete_AndPostInsert(delete); + } + + public void testDeleteFamily_PostInsert() throws IOException, InterruptedException { + Delete delete = new Delete(row); + delete.deleteFamily(fam1); + doTestDelete_AndPostInsert(delete); + } + + public void doTestDelete_AndPostInsert(Delete delete) + throws IOException, InterruptedException { + this.region = initHRegion(tableName, getName(), conf, fam1); + try { + EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); + Put put = new Put(row); + put.add(fam1, qual1, value1); + region.put(put); + + // now delete the value: + region.delete(delete, null, true); + + + // ok put data: + put = new Put(row); + put.add(fam1, qual1, value2); + region.put(put); + + // ok get: + Get get = new Get(row); + get.addColumn(fam1, qual1); + + Result r = region.get(get, null); + assertEquals(1, r.size()); + assertByteEquals(value2, r.getValue(fam1, qual1)); + + // next: + Scan scan = new Scan(row); + scan.addColumn(fam1, qual1); + InternalScanner s = region.getScanner(scan); + + List results = new ArrayList(); + assertEquals(false, s.next(results)); + assertEquals(1, results.size()); + KeyValue kv = results.get(0); + + assertByteEquals(value2, kv.getValue()); + assertByteEquals(fam1, kv.getFamily()); + assertByteEquals(qual1, kv.getQualifier()); + assertByteEquals(row, kv.getRow()); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testDelete_CheckTimestampUpdated() + throws IOException { + byte [] row1 = Bytes.toBytes("row1"); + byte [] col1 = Bytes.toBytes("col1"); + byte [] col2 = Bytes.toBytes("col2"); + byte [] col3 = Bytes.toBytes("col3"); + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, fam1); + try { + //Building checkerList + List kvs = new ArrayList(); + kvs.add(new KeyValue(row1, fam1, col1, null)); + kvs.add(new KeyValue(row1, fam1, col2, null)); + kvs.add(new KeyValue(row1, fam1, col3, null)); + + Map> deleteMap = new HashMap>(); + deleteMap.put(fam1, kvs); + region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true); + + // extract the key values out the memstore: + // This is kinda hacky, but better than nothing... + long now = System.currentTimeMillis(); + KeyValue firstKv = region.getStore(fam1).memstore.kvset.first(); + assertTrue(firstKv.getTimestamp() <= now); + now = firstKv.getTimestamp(); + for (KeyValue kv: region.getStore(fam1).memstore.kvset) { + assertTrue(kv.getTimestamp() <= now); + now = kv.getTimestamp(); + } + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + ////////////////////////////////////////////////////////////////////////////// + // Get tests + ////////////////////////////////////////////////////////////////////////////// + public void testGet_FamilyChecker() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [] row1 = Bytes.toBytes("row1"); + byte [] fam1 = Bytes.toBytes("fam1"); + byte [] fam2 = Bytes.toBytes("False"); + byte [] col1 = Bytes.toBytes("col1"); + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, fam1); + try { + Get get = new Get(row1); + get.addColumn(fam2, col1); + + //Test + try { + region.get(get, null); + } catch (DoNotRetryIOException e) { + assertFalse(false); + return; + } + assertFalse(true); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testGet_Basic() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [] row1 = Bytes.toBytes("row1"); + byte [] fam1 = Bytes.toBytes("fam1"); + byte [] col1 = Bytes.toBytes("col1"); + byte [] col2 = Bytes.toBytes("col2"); + byte [] col3 = Bytes.toBytes("col3"); + byte [] col4 = Bytes.toBytes("col4"); + byte [] col5 = Bytes.toBytes("col5"); + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, fam1); + try { + //Add to memstore + Put put = new Put(row1); + put.add(fam1, col1, null); + put.add(fam1, col2, null); + put.add(fam1, col3, null); + put.add(fam1, col4, null); + put.add(fam1, col5, null); + region.put(put); + + Get get = new Get(row1); + get.addColumn(fam1, col2); + get.addColumn(fam1, col4); + //Expected result + KeyValue kv1 = new KeyValue(row1, fam1, col2); + KeyValue kv2 = new KeyValue(row1, fam1, col4); + KeyValue [] expected = {kv1, kv2}; + + //Test + Result res = region.get(get, null); + assertEquals(expected.length, res.size()); + for(int i=0; i result = new ArrayList(); + s.next(result); + + assertEquals(expected.length, result.size()); + for(int i=0; ithreads = new ArrayList(threadCount); + for (int i = 0; i < threadCount; i++) { + threads.add(new Thread(Integer.toString(i)) { + @Override + public void run() { + Integer [] lockids = new Integer[lockCount]; + // Get locks. + for (int i = 0; i < lockCount; i++) { + try { + byte [] rowid = Bytes.toBytes(Integer.toString(i)); + lockids[i] = region.obtainRowLock(rowid); + assertEquals(rowid, region.getRowFromLock(lockids[i])); + LOG.debug(getName() + " locked " + Bytes.toString(rowid)); + } catch (IOException e) { + e.printStackTrace(); + } + } + LOG.debug(getName() + " set " + + Integer.toString(lockCount) + " locks"); + + // Abort outstanding locks. + for (int i = lockCount - 1; i >= 0; i--) { + region.releaseRowLock(lockids[i]); + LOG.debug(getName() + " unlocked " + i); + } + LOG.debug(getName() + " released " + + Integer.toString(lockCount) + " locks"); + } + }); + } + + // Startup all our threads. + for (Thread t : threads) { + t.start(); + } + + // Now wait around till all are done. + for (Thread t: threads) { + while (t.isAlive()) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + // Go around again. + } + } + } + LOG.info("locks completed."); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + ////////////////////////////////////////////////////////////////////////////// + // Merge test + ////////////////////////////////////////////////////////////////////////////// + public void testMerge() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [][] families = {fam1, fam2, fam3}; + Configuration hc = initSplit(); + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, hc, families); + try { + LOG.info("" + addContent(region, fam3)); + region.flushcache(); + region.compactStores(); + byte [] splitRow = region.checkSplit(); + assertNotNull(splitRow); + LOG.info("SplitRow: " + Bytes.toString(splitRow)); + HRegion [] subregions = splitRegion(region, splitRow); + try { + // Need to open the regions. + for (int i = 0; i < subregions.length; i++) { + openClosedRegion(subregions[i]); + subregions[i].compactStores(); + } + Path oldRegionPath = region.getRegionDir(); + Path oldRegion1 = subregions[0].getRegionDir(); + Path oldRegion2 = subregions[1].getRegionDir(); + long startTime = System.currentTimeMillis(); + region = HRegion.mergeAdjacent(subregions[0], subregions[1]); + LOG.info("Merge regions elapsed time: " + + ((System.currentTimeMillis() - startTime) / 1000.0)); + fs.delete(oldRegion1, true); + fs.delete(oldRegion2, true); + fs.delete(oldRegionPath, true); + LOG.info("splitAndMerge completed."); + } finally { + for (int i = 0; i < subregions.length; i++) { + try { + subregions[i].close(); + subregions[i].getLog().closeAndDelete(); + } catch (IOException e) { + // Ignore. + } + } + } + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + /** + * @param parent Region to split. + * @param midkey Key to split around. + * @return The Regions we created. + * @throws IOException + */ + HRegion [] splitRegion(final HRegion parent, final byte [] midkey) + throws IOException { + PairOfSameType result = null; + SplitTransaction st = new SplitTransaction(parent, midkey); + // If prepare does not return true, for some reason -- logged inside in + // the prepare call -- we are not ready to split just now. Just return. + if (!st.prepare()) return null; + try { + result = st.execute(null, null); + } catch (IOException ioe) { + try { + LOG.info("Running rollback of failed split of " + + parent.getRegionNameAsString() + "; " + ioe.getMessage()); + st.rollback(null, null); + LOG.info("Successful rollback of failed split of " + + parent.getRegionNameAsString()); + return null; + } catch (RuntimeException e) { + // If failed rollback, kill this server to avoid having a hole in table. + LOG.info("Failed rollback of failed split of " + + parent.getRegionNameAsString() + " -- aborting server", e); + } + } + return new HRegion [] {result.getFirst(), result.getSecond()}; + } + + ////////////////////////////////////////////////////////////////////////////// + // Scanner tests + ////////////////////////////////////////////////////////////////////////////// + public void testGetScanner_WithOkFamilies() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [] fam1 = Bytes.toBytes("fam1"); + byte [] fam2 = Bytes.toBytes("fam2"); + + byte [][] families = {fam1, fam2}; + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, families); + try { + Scan scan = new Scan(); + scan.addFamily(fam1); + scan.addFamily(fam2); + try { + region.getScanner(scan); + } catch (Exception e) { + assertTrue("Families could not be found in Region", false); + } + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testGetScanner_WithNotOkFamilies() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [] fam1 = Bytes.toBytes("fam1"); + byte [] fam2 = Bytes.toBytes("fam2"); + + byte [][] families = {fam1}; + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, families); + try { + Scan scan = new Scan(); + scan.addFamily(fam2); + boolean ok = false; + try { + region.getScanner(scan); + } catch (Exception e) { + ok = true; + } + assertTrue("Families could not be found in Region", ok); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testGetScanner_WithNoFamilies() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [] row1 = Bytes.toBytes("row1"); + byte [] fam1 = Bytes.toBytes("fam1"); + byte [] fam2 = Bytes.toBytes("fam2"); + byte [] fam3 = Bytes.toBytes("fam3"); + byte [] fam4 = Bytes.toBytes("fam4"); + + byte [][] families = {fam1, fam2, fam3, fam4}; + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, families); + try { + + //Putting data in Region + Put put = new Put(row1); + put.add(fam1, null, null); + put.add(fam2, null, null); + put.add(fam3, null, null); + put.add(fam4, null, null); + region.put(put); + + Scan scan = null; + HRegion.RegionScannerImpl is = null; + + //Testing to see how many scanners that is produced by getScanner, starting + //with known number, 2 - current = 1 + scan = new Scan(); + scan.addFamily(fam2); + scan.addFamily(fam4); + is = (RegionScannerImpl) region.getScanner(scan); + MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC()); + assertEquals(1, ((RegionScannerImpl)is).storeHeap.getHeap().size()); + + scan = new Scan(); + is = (RegionScannerImpl) region.getScanner(scan); + MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC()); + assertEquals(families.length -1, + ((RegionScannerImpl)is).storeHeap.getHeap().size()); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + /** + * This method tests https://issues.apache.org/jira/browse/HBASE-2516. + * @throws IOException + */ + public void testGetScanner_WithRegionClosed() throws IOException { + byte[] tableName = Bytes.toBytes("testtable"); + byte[] fam1 = Bytes.toBytes("fam1"); + byte[] fam2 = Bytes.toBytes("fam2"); + + byte[][] families = {fam1, fam2}; + + //Setting up region + String method = this.getName(); + try { + this.region = initHRegion(tableName, method, conf, families); + } catch (IOException e) { + e.printStackTrace(); + fail("Got IOException during initHRegion, " + e.getMessage()); + } + try { + region.closed.set(true); + try { + region.getScanner(null); + fail("Expected to get an exception during getScanner on a region that is closed"); + } catch (org.apache.hadoop.hbase.NotServingRegionException e) { + //this is the correct exception that is expected + } catch (IOException e) { + fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: " + + e.getMessage()); + } + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testRegionScanner_Next() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [] row1 = Bytes.toBytes("row1"); + byte [] row2 = Bytes.toBytes("row2"); + byte [] fam1 = Bytes.toBytes("fam1"); + byte [] fam2 = Bytes.toBytes("fam2"); + byte [] fam3 = Bytes.toBytes("fam3"); + byte [] fam4 = Bytes.toBytes("fam4"); + + byte [][] families = {fam1, fam2, fam3, fam4}; + long ts = System.currentTimeMillis(); + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, families); + try { + //Putting data in Region + Put put = null; + put = new Put(row1); + put.add(fam1, null, ts, null); + put.add(fam2, null, ts, null); + put.add(fam3, null, ts, null); + put.add(fam4, null, ts, null); + region.put(put); + + put = new Put(row2); + put.add(fam1, null, ts, null); + put.add(fam2, null, ts, null); + put.add(fam3, null, ts, null); + put.add(fam4, null, ts, null); + region.put(put); + + Scan scan = new Scan(); + scan.addFamily(fam2); + scan.addFamily(fam4); + InternalScanner is = region.getScanner(scan); + + List res = null; + + //Result 1 + List expected1 = new ArrayList(); + expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null)); + expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null)); + + res = new ArrayList(); + is.next(res); + for(int i=0; i expected2 = new ArrayList(); + expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null)); + expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null)); + + res = new ArrayList(); + is.next(res); + for(int i=0; i expected = new ArrayList(); + expected.add(kv13); + expected.add(kv12); + + Scan scan = new Scan(row1); + scan.addColumn(fam1, qf1); + scan.setMaxVersions(MAX_VERSIONS); + List actual = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + + boolean hasNext = scanner.next(actual); + assertEquals(false, hasNext); + + //Verify result + for(int i=0; i expected = new ArrayList(); + expected.add(kv13); + expected.add(kv12); + expected.add(kv23); + expected.add(kv22); + + Scan scan = new Scan(row1); + scan.addColumn(fam1, qf1); + scan.addColumn(fam1, qf2); + scan.setMaxVersions(MAX_VERSIONS); + List actual = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + + boolean hasNext = scanner.next(actual); + assertEquals(false, hasNext); + + //Verify result + for(int i=0; i expected = new ArrayList(); + expected.add(kv14); + expected.add(kv13); + expected.add(kv12); + expected.add(kv24); + expected.add(kv23); + expected.add(kv22); + + Scan scan = new Scan(row1); + scan.addColumn(fam1, qf1); + scan.addColumn(fam1, qf2); + int versions = 3; + scan.setMaxVersions(versions); + List actual = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + + boolean hasNext = scanner.next(actual); + assertEquals(false, hasNext); + + //Verify result + for(int i=0; i expected = new ArrayList(); + expected.add(kv13); + expected.add(kv12); + expected.add(kv23); + expected.add(kv22); + + Scan scan = new Scan(row1); + scan.addFamily(fam1); + scan.setMaxVersions(MAX_VERSIONS); + List actual = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + + boolean hasNext = scanner.next(actual); + assertEquals(false, hasNext); + + //Verify result + for(int i=0; i expected = new ArrayList(); + expected.add(kv13); + expected.add(kv12); + expected.add(kv23); + expected.add(kv22); + + Scan scan = new Scan(row1); + scan.addFamily(fam1); + scan.setMaxVersions(MAX_VERSIONS); + List actual = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + + boolean hasNext = scanner.next(actual); + assertEquals(false, hasNext); + + //Verify result + for(int i=0; i results = new ArrayList(); + assertEquals(false, s.next(results)); + assertEquals(0, results.size()); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testIncrementColumnValue_UpdatingInPlace() throws IOException { + this.region = initHRegion(tableName, getName(), conf, fam1); + try { + long value = 1L; + long amount = 3L; + + Put put = new Put(row); + put.add(fam1, qual1, Bytes.toBytes(value)); + region.put(put); + + long result = region.incrementColumnValue(row, fam1, qual1, amount, true); + + assertEquals(value+amount, result); + + Store store = region.getStore(fam1); + // ICV removes any extra values floating around in there. + assertEquals(1, store.memstore.kvset.size()); + assertTrue(store.memstore.snapshot.isEmpty()); + + assertICV(row, fam1, qual1, value+amount); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testIncrementColumnValue_BumpSnapshot() throws IOException { + ManualEnvironmentEdge mee = new ManualEnvironmentEdge(); + EnvironmentEdgeManagerTestHelper.injectEdge(mee); + this.region = initHRegion(tableName, getName(), conf, fam1); + try { + long value = 42L; + long incr = 44L; + + // first put something in kvset, then snapshot it. + Put put = new Put(row); + put.add(fam1, qual1, Bytes.toBytes(value)); + region.put(put); + + // get the store in question: + Store s = region.getStore(fam1); + s.snapshot(); //bam + + // now increment: + long newVal = region.incrementColumnValue(row, fam1, qual1, + incr, false); + + assertEquals(value+incr, newVal); + + // get both versions: + Get get = new Get(row); + get.setMaxVersions(); + get.addColumn(fam1,qual1); + + Result r = region.get(get, null); + assertEquals(2, r.size()); + KeyValue first = r.raw()[0]; + KeyValue second = r.raw()[1]; + + assertTrue("ICV failed to upgrade timestamp", + first.getTimestamp() != second.getTimestamp()); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testIncrementColumnValue_ConcurrentFlush() throws IOException { + this.region = initHRegion(tableName, getName(), conf, fam1); + try { + long value = 1L; + long amount = 3L; + + Put put = new Put(row); + put.add(fam1, qual1, Bytes.toBytes(value)); + region.put(put); + + // now increment during a flush + Thread t = new Thread() { + public void run() { + try { + region.flushcache(); + } catch (IOException e) { + LOG.info("test ICV, got IOE during flushcache()"); + } + } + }; + t.start(); + long r = region.incrementColumnValue(row, fam1, qual1, amount, true); + assertEquals(value+amount, r); + + // this also asserts there is only 1 KeyValue in the set. + assertICV(row, fam1, qual1, value+amount); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testIncrementColumnValue_heapSize() throws IOException { + EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); + + this.region = initHRegion(tableName, getName(), conf, fam1); + try { + long byAmount = 1L; + long size; + + for( int i = 0; i < 1000 ; i++) { + region.incrementColumnValue(row, fam1, qual1, byAmount, true); + + size = region.memstoreSize.get(); + assertTrue("memstore size: " + size, size >= 0); + } + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testIncrementColumnValue_UpdatingInPlace_Negative() + throws IOException { + this.region = initHRegion(tableName, getName(), conf, fam1); + try { + long value = 3L; + long amount = -1L; + + Put put = new Put(row); + put.add(fam1, qual1, Bytes.toBytes(value)); + region.put(put); + + long result = region.incrementColumnValue(row, fam1, qual1, amount, true); + assertEquals(value+amount, result); + + assertICV(row, fam1, qual1, value+amount); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testIncrementColumnValue_AddingNew() + throws IOException { + this.region = initHRegion(tableName, getName(), conf, fam1); + try { + long value = 1L; + long amount = 3L; + + Put put = new Put(row); + put.add(fam1, qual1, Bytes.toBytes(value)); + put.add(fam1, qual2, Bytes.toBytes(value)); + region.put(put); + + long result = region.incrementColumnValue(row, fam1, qual3, amount, true); + assertEquals(amount, result); + + Get get = new Get(row); + get.addColumn(fam1, qual3); + Result rr = region.get(get, null); + assertEquals(1, rr.size()); + + // ensure none of the other cols were incremented. + assertICV(row, fam1, qual1, value); + assertICV(row, fam1, qual2, value); + assertICV(row, fam1, qual3, amount); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testIncrementColumnValue_UpdatingFromSF() throws IOException { + this.region = initHRegion(tableName, getName(), conf, fam1); + try { + long value = 1L; + long amount = 3L; + + Put put = new Put(row); + put.add(fam1, qual1, Bytes.toBytes(value)); + put.add(fam1, qual2, Bytes.toBytes(value)); + region.put(put); + + // flush to disk. + region.flushcache(); + + Store store = region.getStore(fam1); + assertEquals(0, store.memstore.kvset.size()); + + long r = region.incrementColumnValue(row, fam1, qual1, amount, true); + assertEquals(value+amount, r); + + assertICV(row, fam1, qual1, value+amount); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testIncrementColumnValue_AddingNewAfterSFCheck() + throws IOException { + this.region = initHRegion(tableName, getName(), conf, fam1); + try { + long value = 1L; + long amount = 3L; + + Put put = new Put(row); + put.add(fam1, qual1, Bytes.toBytes(value)); + put.add(fam1, qual2, Bytes.toBytes(value)); + region.put(put); + region.flushcache(); + + Store store = region.getStore(fam1); + assertEquals(0, store.memstore.kvset.size()); + + long r = region.incrementColumnValue(row, fam1, qual3, amount, true); + assertEquals(amount, r); + + assertICV(row, fam1, qual3, amount); + + region.flushcache(); + + // ensure that this gets to disk. + assertICV(row, fam1, qual3, amount); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + /** + * Added for HBASE-3235. + * + * When the initial put and an ICV update were arriving with the same timestamp, + * the initial Put KV was being skipped during {@link MemStore#upsert(KeyValue)} + * causing the iteration for matching KVs, causing the update-in-place to not + * happen and the ICV put to effectively disappear. + * @throws IOException + */ + public void testIncrementColumnValue_UpdatingInPlace_TimestampClobber() throws IOException { + this.region = initHRegion(tableName, getName(), conf, fam1); + try { + long value = 1L; + long amount = 3L; + long now = EnvironmentEdgeManager.currentTimeMillis(); + ManualEnvironmentEdge mock = new ManualEnvironmentEdge(); + mock.setValue(now); + EnvironmentEdgeManagerTestHelper.injectEdge(mock); + + // verify we catch an ICV on a put with the same timestamp + Put put = new Put(row); + put.add(fam1, qual1, now, Bytes.toBytes(value)); + region.put(put); + + long result = region.incrementColumnValue(row, fam1, qual1, amount, true); + + assertEquals(value+amount, result); + + Store store = region.getStore(fam1); + // ICV should update the existing Put with the same timestamp + assertEquals(1, store.memstore.kvset.size()); + assertTrue(store.memstore.snapshot.isEmpty()); + + assertICV(row, fam1, qual1, value+amount); + + // verify we catch an ICV even when the put ts > now + put = new Put(row); + put.add(fam1, qual2, now+1, Bytes.toBytes(value)); + region.put(put); + + result = region.incrementColumnValue(row, fam1, qual2, amount, true); + + assertEquals(value+amount, result); + + store = region.getStore(fam1); + // ICV should update the existing Put with the same timestamp + assertEquals(2, store.memstore.kvset.size()); + assertTrue(store.memstore.snapshot.isEmpty()); + + assertICV(row, fam1, qual2, value+amount); + EnvironmentEdgeManagerTestHelper.reset(); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testIncrementColumnValue_WrongInitialSize() throws IOException { + this.region = initHRegion(tableName, getName(), conf, fam1); + try { + byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L)); + int row1Field1 = 0; + int row1Field2 = 1; + Put put1 = new Put(row1); + put1.add(fam1, qual1, Bytes.toBytes(row1Field1)); + put1.add(fam1, qual2, Bytes.toBytes(row1Field2)); + region.put(put1); + + long result; + try { + result = region.incrementColumnValue(row1, fam1, qual1, 1, true); + fail("Expected to fail here"); + } catch (Exception exception) { + // Expected. + } + + + assertICV(row1, fam1, qual1, row1Field1); + assertICV(row1, fam1, qual2, row1Field2); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testIncrement_WrongInitialSize() throws IOException { + this.region = initHRegion(tableName, getName(), conf, fam1); + try { + byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L)); + long row1Field1 = 0; + int row1Field2 = 1; + Put put1 = new Put(row1); + put1.add(fam1, qual1, Bytes.toBytes(row1Field1)); + put1.add(fam1, qual2, Bytes.toBytes(row1Field2)); + region.put(put1); + Increment increment = new Increment(row1); + increment.addColumn(fam1, qual1, 1); + + //here we should be successful as normal + region.increment(increment, null, true); + assertICV(row1, fam1, qual1, row1Field1 + 1); + + //failed to increment + increment = new Increment(row1); + increment.addColumn(fam1, qual2, 1); + try { + region.increment(increment, null, true); + fail("Expected to fail here"); + } catch (Exception exception) { + // Expected. + } + assertICV(row1, fam1, qual2, row1Field2); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + private void assertICV(byte [] row, + byte [] familiy, + byte[] qualifier, + long amount) throws IOException { + // run a get and see? + Get get = new Get(row); + get.addColumn(familiy, qualifier); + Result result = region.get(get, null); + assertEquals(1, result.size()); + + KeyValue kv = result.raw()[0]; + long r = Bytes.toLong(kv.getValue()); + assertEquals(amount, r); + } + + private void assertICV(byte [] row, + byte [] familiy, + byte[] qualifier, + int amount) throws IOException { + // run a get and see? + Get get = new Get(row); + get.addColumn(familiy, qualifier); + Result result = region.get(get, null); + assertEquals(1, result.size()); + + KeyValue kv = result.raw()[0]; + int r = Bytes.toInt(kv.getValue()); + assertEquals(amount, r); + } + + public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() + throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [] row1 = Bytes.toBytes("row1"); + byte [] fam1 = Bytes.toBytes("fam1"); + byte [] qf1 = Bytes.toBytes("qualifier1"); + byte [] qf2 = Bytes.toBytes("quateslifier2"); + + long ts1 = 1; + long ts2 = ts1 + 1; + long ts3 = ts1 + 2; + long ts4 = ts1 + 3; + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, conf, fam1); + try { + //Putting data in Region + KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); + KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); + KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); + KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); + + KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); + KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); + KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); + KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); + + Put put = null; + put = new Put(row1); + put.add(kv14); + put.add(kv24); + region.put(put); + region.flushcache(); + + put = new Put(row1); + put.add(kv23); + put.add(kv13); + region.put(put); + region.flushcache(); + + put = new Put(row1); + put.add(kv22); + put.add(kv12); + region.put(put); + region.flushcache(); + + put = new Put(row1); + put.add(kv21); + put.add(kv11); + region.put(put); + + //Expected + List expected = new ArrayList(); + expected.add(kv14); + expected.add(kv13); + expected.add(kv12); + expected.add(kv24); + expected.add(kv23); + expected.add(kv22); + + Scan scan = new Scan(row1); + int versions = 3; + scan.setMaxVersions(versions); + List actual = new ArrayList(); + InternalScanner scanner = region.getScanner(scan); + + boolean hasNext = scanner.next(actual); + assertEquals(false, hasNext); + + //Verify result + for(int i=0; i results = new ArrayList(); + assertTrue(s.next(results)); + assertEquals(results.size(), 1); + results.clear(); + + assertTrue(s.next(results)); + assertEquals(results.size(), 3); + assertTrue("orderCheck", results.get(0).matchingFamily(cf_alpha)); + assertTrue("orderCheck", results.get(1).matchingFamily(cf_essential)); + assertTrue("orderCheck", results.get(2).matchingFamily(cf_joined)); + results.clear(); + + assertFalse(s.next(results)); + assertEquals(results.size(), 0); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + /** + * HBASE-5416 + * + * Test case when scan limits amount of KVs returned on each next() call. + */ + public void testScanner_JoinedScannersWithLimits() throws IOException { + final byte [] tableName = Bytes.toBytes("testTable"); + final byte [] cf_first = Bytes.toBytes("first"); + final byte [] cf_second = Bytes.toBytes("second"); + + this.region = initHRegion(tableName, getName(), conf, cf_first, cf_second); + try { + final byte [] col_a = Bytes.toBytes("a"); + final byte [] col_b = Bytes.toBytes("b"); + + Put put; + + for (int i = 0; i < 10; i++) { + put = new Put(Bytes.toBytes("r" + Integer.toString(i))); + put.add(cf_first, col_a, Bytes.toBytes(i)); + if (i < 5) { + put.add(cf_first, col_b, Bytes.toBytes(i)); + put.add(cf_second, col_a, Bytes.toBytes(i)); + put.add(cf_second, col_b, Bytes.toBytes(i)); + } + region.put(put); + } + + Scan scan = new Scan(); + scan.setLoadColumnFamiliesOnDemand(true); + Filter bogusFilter = new FilterBase() { + @Override + public boolean isFamilyEssential(byte[] name) { + return Bytes.equals(name, cf_first); + } + @Override + public void readFields(DataInput arg0) throws IOException { + } + + @Override + public void write(DataOutput arg0) throws IOException { + } + }; + + scan.setFilter(bogusFilter); + InternalScanner s = region.getScanner(scan); + + // Our data looks like this: + // r0: first:a, first:b, second:a, second:b + // r1: first:a, first:b, second:a, second:b + // r2: first:a, first:b, second:a, second:b + // r3: first:a, first:b, second:a, second:b + // r4: first:a, first:b, second:a, second:b + // r5: first:a + // r6: first:a + // r7: first:a + // r8: first:a + // r9: first:a + + // But due to next's limit set to 3, we should get this: + // r0: first:a, first:b, second:a + // r0: second:b + // r1: first:a, first:b, second:a + // r1: second:b + // r2: first:a, first:b, second:a + // r2: second:b + // r3: first:a, first:b, second:a + // r3: second:b + // r4: first:a, first:b, second:a + // r4: second:b + // r5: first:a + // r6: first:a + // r7: first:a + // r8: first:a + // r9: first:a + + List results = new ArrayList(); + int index = 0; + while (true) { + boolean more = s.next(results, 3); + if ((index >> 1) < 5) { + if (index % 2 == 0) + assertEquals(results.size(), 3); + else + assertEquals(results.size(), 1); + } + else + assertEquals(results.size(), 1); + results.clear(); + index++; + if (!more) break; + } + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + ////////////////////////////////////////////////////////////////////////////// + // Split test + ////////////////////////////////////////////////////////////////////////////// + /** + * Splits twice and verifies getting from each of the split regions. + * @throws Exception + */ + public void testBasicSplit() throws Exception { + byte [] tableName = Bytes.toBytes("testtable"); + byte [][] families = {fam1, fam2, fam3}; + + Configuration hc = initSplit(); + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, hc, families); + + try { + LOG.info("" + addContent(region, fam3)); + region.flushcache(); + region.compactStores(); + byte [] splitRow = region.checkSplit(); + assertNotNull(splitRow); + LOG.info("SplitRow: " + Bytes.toString(splitRow)); + HRegion [] regions = splitRegion(region, splitRow); + try { + // Need to open the regions. + // TODO: Add an 'open' to HRegion... don't do open by constructing + // instance. + for (int i = 0; i < regions.length; i++) { + regions[i] = openClosedRegion(regions[i]); + } + // Assert can get rows out of new regions. Should be able to get first + // row from first region and the midkey from second region. + assertGet(regions[0], fam3, Bytes.toBytes(START_KEY)); + assertGet(regions[1], fam3, splitRow); + // Test I can get scanner and that it starts at right place. + assertScan(regions[0], fam3, + Bytes.toBytes(START_KEY)); + assertScan(regions[1], fam3, splitRow); + // Now prove can't split regions that have references. + for (int i = 0; i < regions.length; i++) { + // Add so much data to this region, we create a store file that is > + // than one of our unsplitable references. it will. + for (int j = 0; j < 2; j++) { + addContent(regions[i], fam3); + } + addContent(regions[i], fam2); + addContent(regions[i], fam1); + regions[i].flushcache(); + } + + byte [][] midkeys = new byte [regions.length][]; + // To make regions splitable force compaction. + for (int i = 0; i < regions.length; i++) { + regions[i].compactStores(); + midkeys[i] = regions[i].checkSplit(); + } + + TreeMap sortedMap = new TreeMap(); + // Split these two daughter regions so then I'll have 4 regions. Will + // split because added data above. + for (int i = 0; i < regions.length; i++) { + HRegion[] rs = null; + if (midkeys[i] != null) { + rs = splitRegion(regions[i], midkeys[i]); + for (int j = 0; j < rs.length; j++) { + sortedMap.put(Bytes.toString(rs[j].getRegionName()), + openClosedRegion(rs[j])); + } + } + } + LOG.info("Made 4 regions"); + // The splits should have been even. Test I can get some arbitrary row + // out of each. + int interval = (LAST_CHAR - FIRST_CHAR) / 3; + byte[] b = Bytes.toBytes(START_KEY); + for (HRegion r : sortedMap.values()) { + assertGet(r, fam3, b); + b[0] += interval; + } + } finally { + for (int i = 0; i < regions.length; i++) { + try { + regions[i].close(); + } catch (IOException e) { + // Ignore. + } + } + } + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testSplitRegion() throws IOException { + byte [] tableName = Bytes.toBytes("testtable"); + byte [] qualifier = Bytes.toBytes("qualifier"); + Configuration hc = initSplit(); + int numRows = 10; + byte [][] families = {fam1, fam3}; + + //Setting up region + String method = this.getName(); + this.region = initHRegion(tableName, method, hc, families); + + //Put data in region + int startRow = 100; + putData(startRow, numRows, qualifier, families); + int splitRow = startRow + numRows; + putData(splitRow, numRows, qualifier, families); + region.flushcache(); + + HRegion [] regions = null; + try { + regions = splitRegion(region, Bytes.toBytes("" + splitRow)); + //Opening the regions returned. + for (int i = 0; i < regions.length; i++) { + regions[i] = openClosedRegion(regions[i]); + } + //Verifying that the region has been split + assertEquals(2, regions.length); + + //Verifying that all data is still there and that data is in the right + //place + verifyData(regions[0], startRow, numRows, qualifier, families); + verifyData(regions[1], splitRow, numRows, qualifier, families); + + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + + /** + * Flushes the cache in a thread while scanning. The tests verify that the + * scan is coherent - e.g. the returned results are always of the same or + * later update as the previous results. + * @throws IOException scan / compact + * @throws InterruptedException thread join + */ + public void testFlushCacheWhileScanning() throws IOException, InterruptedException { + byte[] tableName = Bytes.toBytes("testFlushCacheWhileScanning"); + byte[] family = Bytes.toBytes("family"); + int numRows = 1000; + int flushAndScanInterval = 10; + int compactInterval = 10 * flushAndScanInterval; + + String method = "testFlushCacheWhileScanning"; + this.region = initHRegion(tableName,method, conf, family); + try { + FlushThread flushThread = new FlushThread(); + flushThread.start(); + + Scan scan = new Scan(); + scan.addFamily(family); + scan.setFilter(new SingleColumnValueFilter(family, qual1, + CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(5L)))); + + int expectedCount = 0; + List res = new ArrayList(); + + boolean toggle=true; + for (long i = 0; i < numRows; i++) { + Put put = new Put(Bytes.toBytes(i)); + put.setWriteToWAL(false); + put.add(family, qual1, Bytes.toBytes(i % 10)); + region.put(put); + + if (i != 0 && i % compactInterval == 0) { + //System.out.println("iteration = " + i); + region.compactStores(true); + } + + if (i % 10 == 5L) { + expectedCount++; + } + + if (i != 0 && i % flushAndScanInterval == 0) { + res.clear(); + InternalScanner scanner = region.getScanner(scan); + if (toggle) { + flushThread.flush(); + } + while (scanner.next(res)) ; + if (!toggle) { + flushThread.flush(); + } + assertEquals("i=" + i, expectedCount, res.size()); + toggle = !toggle; + } + } + + flushThread.done(); + flushThread.join(); + flushThread.checkNoError(); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + protected class FlushThread extends Thread { + private volatile boolean done; + private Throwable error = null; + + public void done() { + done = true; + synchronized (this) { + interrupt(); + } + } + + public void checkNoError() { + if (error != null) { + assertNull(error); + } + } + + @Override + public void run() { + done = false; + while (!done) { + synchronized (this) { + try { + wait(); + } catch (InterruptedException ignored) { + if (done) { + break; + } + } + } + try { + region.flushcache(); + } catch (IOException e) { + if (!done) { + LOG.error("Error while flusing cache", e); + error = e; + } + break; + } + } + + } + + public void flush() { + synchronized (this) { + notify(); + } + + } + } + + /** + * Writes very wide records and scans for the latest every time.. + * Flushes and compacts the region every now and then to keep things + * realistic. + * + * @throws IOException by flush / scan / compaction + * @throws InterruptedException when joining threads + */ + public void testWritesWhileScanning() + throws IOException, InterruptedException { + byte[] tableName = Bytes.toBytes("testWritesWhileScanning"); + int testCount = 100; + int numRows = 1; + int numFamilies = 10; + int numQualifiers = 100; + int flushInterval = 7; + int compactInterval = 5 * flushInterval; + byte[][] families = new byte[numFamilies][]; + for (int i = 0; i < numFamilies; i++) { + families[i] = Bytes.toBytes("family" + i); + } + byte[][] qualifiers = new byte[numQualifiers][]; + for (int i = 0; i < numQualifiers; i++) { + qualifiers[i] = Bytes.toBytes("qual" + i); + } + + String method = "testWritesWhileScanning"; + this.region = initHRegion(tableName, method, conf, families); + try { + PutThread putThread = new PutThread(numRows, families, qualifiers); + putThread.start(); + putThread.waitForFirstPut(); + + FlushThread flushThread = new FlushThread(); + flushThread.start(); + + Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1")); + // scan.setFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, + // new BinaryComparator(Bytes.toBytes("row0")))); + + int expectedCount = numFamilies * numQualifiers; + List res = new ArrayList(); + + long prevTimestamp = 0L; + for (int i = 0; i < testCount; i++) { + + if (i != 0 && i % compactInterval == 0) { + region.compactStores(true); + } + + if (i != 0 && i % flushInterval == 0) { + //System.out.println("flush scan iteration = " + i); + flushThread.flush(); + } + + boolean previousEmpty = res.isEmpty(); + res.clear(); + InternalScanner scanner = region.getScanner(scan); + while (scanner.next(res)) ; + if (!res.isEmpty() || !previousEmpty || i > compactInterval) { + assertEquals("i=" + i, expectedCount, res.size()); + long timestamp = res.get(0).getTimestamp(); + assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp, + timestamp >= prevTimestamp); + prevTimestamp = timestamp; + } + } + + putThread.done(); + + region.flushcache(); + + putThread.join(); + putThread.checkNoError(); + + flushThread.done(); + flushThread.join(); + flushThread.checkNoError(); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + protected class PutThread extends Thread { + private volatile boolean done; + private volatile int numPutsFinished = 0; + + private Throwable error = null; + private int numRows; + private byte[][] families; + private byte[][] qualifiers; + + private PutThread(int numRows, byte[][] families, + byte[][] qualifiers) { + this.numRows = numRows; + this.families = families; + this.qualifiers = qualifiers; + } + + /** + * Block until this thread has put at least one row. + */ + public void waitForFirstPut() throws InterruptedException { + // wait until put thread actually puts some data + while (numPutsFinished == 0) { + checkNoError(); + Thread.sleep(50); + } + } + + public void done() { + done = true; + synchronized (this) { + interrupt(); + } + } + + public void checkNoError() { + if (error != null) { + assertNull(error); + } + } + + @Override + public void run() { + done = false; + while (!done) { + try { + for (int r = 0; r < numRows; r++) { + byte[] row = Bytes.toBytes("row" + r); + Put put = new Put(row); + put.setWriteToWAL(false); + byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished)); + for (byte[] family : families) { + for (byte[] qualifier : qualifiers) { + put.add(family, qualifier, (long) numPutsFinished, value); + } + } +// System.out.println("Putting of kvsetsize=" + put.size()); + region.put(put); + numPutsFinished++; + if (numPutsFinished > 0 && numPutsFinished % 47 == 0) { + System.out.println("put iteration = " + numPutsFinished); + Delete delete = new Delete(row, (long)numPutsFinished-30, null); + region.delete(delete, null, true); + } + numPutsFinished++; + } + } catch (InterruptedIOException e) { + // This is fine. It means we are done, or didn't get the lock on time + } catch (IOException e) { + LOG.error("error while putting records", e); + error = e; + break; + } + } + + } + + } + + + /** + * Writes very wide records and gets the latest row every time.. + * Flushes and compacts the region aggressivly to catch issues. + * + * @throws IOException by flush / scan / compaction + * @throws InterruptedException when joining threads + */ + public void testWritesWhileGetting() + throws Exception { + byte[] tableName = Bytes.toBytes("testWritesWhileGetting"); + int testCount = 100; + int numRows = 1; + int numFamilies = 10; + int numQualifiers = 100; + int compactInterval = 100; + byte[][] families = new byte[numFamilies][]; + for (int i = 0; i < numFamilies; i++) { + families[i] = Bytes.toBytes("family" + i); + } + byte[][] qualifiers = new byte[numQualifiers][]; + for (int i = 0; i < numQualifiers; i++) { + qualifiers[i] = Bytes.toBytes("qual" + i); + } + + Configuration conf = HBaseConfiguration.create(this.conf); + + String method = "testWritesWhileGetting"; + // This test flushes constantly and can cause many files to be created, possibly + // extending over the ulimit. Make sure compactions are aggressive in reducing + // the number of HFiles created. + conf.setInt("hbase.hstore.compaction.min", 1); + conf.setInt("hbase.hstore.compaction.max", 1000); + this.region = initHRegion(tableName, method, conf, families); + PutThread putThread = null; + MultithreadedTestUtil.TestContext ctx = + new MultithreadedTestUtil.TestContext(conf); + try { + putThread = new PutThread(numRows, families, qualifiers); + putThread.start(); + putThread.waitForFirstPut(); + + // Add a thread that flushes as fast as possible + ctx.addThread(new RepeatingTestThread(ctx) { + private int flushesSinceCompact = 0; + private final int maxFlushesSinceCompact = 20; + public void doAnAction() throws Exception { + if (region.flushcache()) { + ++flushesSinceCompact; + } + // Compact regularly to avoid creating too many files and exceeding the ulimit. + if (flushesSinceCompact == maxFlushesSinceCompact) { + region.compactStores(false); + flushesSinceCompact = 0; + } + } + }); + ctx.startThreads(); + + Get get = new Get(Bytes.toBytes("row0")); + Result result = null; + + int expectedCount = numFamilies * numQualifiers; + + long prevTimestamp = 0L; + for (int i = 0; i < testCount; i++) { + + boolean previousEmpty = result == null || result.isEmpty(); + result = region.get(get, null); + if (!result.isEmpty() || !previousEmpty || i > compactInterval) { + assertEquals("i=" + i, expectedCount, result.size()); + // TODO this was removed, now what dangit?! + // search looking for the qualifier in question? + long timestamp = 0; + for (KeyValue kv : result.raw()) { + if (Bytes.equals(kv.getFamily(), families[0]) + && Bytes.equals(kv.getQualifier(), qualifiers[0])) { + timestamp = kv.getTimestamp(); + } + } + assertTrue(timestamp >= prevTimestamp); + prevTimestamp = timestamp; + KeyValue previousKV = null; + + for (KeyValue kv : result.raw()) { + byte[] thisValue = kv.getValue(); + if (previousKV != null) { + if (Bytes.compareTo(previousKV.getValue(), thisValue) != 0) { + LOG.warn("These two KV should have the same value." + + " Previous KV:" + + previousKV + "(memStoreTS:" + previousKV.getMemstoreTS() + ")" + + ", New KV: " + + kv + "(memStoreTS:" + kv.getMemstoreTS() + ")" + ); + assertEquals(0, Bytes.compareTo(previousKV.getValue(), thisValue)); + } + } + previousKV = kv; + } + } + } + } finally { + if (putThread != null) putThread.done(); + + region.flushcache(); + + if (putThread != null) { + putThread.join(); + putThread.checkNoError(); + } + + ctx.stop(); + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testHolesInMeta() throws Exception { + String method = "testHolesInMeta"; + byte[] tableName = Bytes.toBytes(method); + byte[] family = Bytes.toBytes("family"); + this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, + conf, false, family); + try { + byte[] rowNotServed = Bytes.toBytes("a"); + Get g = new Get(rowNotServed); + try { + region.get(g, null); + fail(); + } catch (WrongRegionException x) { + // OK + } + byte[] row = Bytes.toBytes("y"); + g = new Get(row); + region.get(g, null); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + /** + * Testcase to check state of region initialization task set to ABORTED or not if any exceptions + * during initialization + * + * @throws Exception + */ + @Test + public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception { + HRegionInfo info = null; + try { + FileSystem fs = Mockito.mock(FileSystem.class); + Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException()); + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor("cf")); + info = new HRegionInfo(htd.getName(), HConstants.EMPTY_BYTE_ARRAY, + HConstants.EMPTY_BYTE_ARRAY, false); + Path path = new Path(DIR + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization"); + // no where we are instantiating HStore in this test case so useTableNameGlobally is null. To + // avoid NullPointerException we are setting useTableNameGlobally to false. + SchemaMetrics.setUseTableNameInTest(false); + region = HRegion.newHRegion(path, null, fs, conf, info, htd, null); + // region initialization throws IOException and set task state to ABORTED. + region.initialize(); + fail("Region initialization should fail due to IOException"); + } catch (IOException io) { + List tasks = TaskMonitor.get().getTasks(); + for (MonitoredTask monitoredTask : tasks) { + if (!(monitoredTask instanceof MonitoredRPCHandler) + && monitoredTask.getDescription().contains(region.toString())) { + assertTrue("Region state should be ABORTED.", + monitoredTask.getState().equals(MonitoredTask.State.ABORTED)); + break; + } + } + } finally { + HRegion.closeHRegion(region); + } + } + + public void testIndexesScanWithOneDeletedRow() throws IOException { + byte[] tableName = Bytes.toBytes("testIndexesScanWithOneDeletedRow"); + byte[] family = Bytes.toBytes("family"); + + //Setting up region + String method = "testIndexesScanWithOneDeletedRow"; + this.region = initHRegion(tableName, method, conf, family); + try { + Put put = new Put(Bytes.toBytes(1L)); + put.add(family, qual1, 1L, Bytes.toBytes(1L)); + region.put(put); + + region.flushcache(); + + Delete delete = new Delete(Bytes.toBytes(1L), 1L, null); + //delete.deleteColumn(family, qual1); + region.delete(delete, null, true); + + put = new Put(Bytes.toBytes(2L)); + put.add(family, qual1, 2L, Bytes.toBytes(2L)); + region.put(put); + + Scan idxScan = new Scan(); + idxScan.addFamily(family); + idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, + Arrays.asList(new SingleColumnValueFilter(family, qual1, + CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.toBytes(0L))), + new SingleColumnValueFilter(family, qual1, CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes(3L))) + ))); + InternalScanner scanner = region.getScanner(idxScan); + List res = new ArrayList(); + + //long start = System.nanoTime(); + while (scanner.next(res)) ; + //long end = System.nanoTime(); + //System.out.println("memStoreEmpty=" + memStoreEmpty + ", time=" + (end - start)/1000000D); + assertEquals(1L, res.size()); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + ////////////////////////////////////////////////////////////////////////////// + // Bloom filter test + ////////////////////////////////////////////////////////////////////////////// + public void testBloomFilterSize() throws IOException { + byte [] tableName = Bytes.toBytes("testBloomFilterSize"); + byte [] row1 = Bytes.toBytes("row1"); + byte [] fam1 = Bytes.toBytes("fam1"); + byte [] qf1 = Bytes.toBytes("col"); + byte [] val1 = Bytes.toBytes("value1"); + // Create Table + HColumnDescriptor hcd = new HColumnDescriptor(fam1) + .setMaxVersions(Integer.MAX_VALUE) + .setBloomFilterType(BloomType.ROWCOL); + + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(hcd); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + Path path = new Path(DIR + "testBloomFilterSize"); + this.region = HRegion.createHRegion(info, path, conf, htd); + try { + int num_unique_rows = 10; + int duplicate_multiplier =2; + int num_storefiles = 4; + + int version = 0; + for (int f =0 ; f < num_storefiles; f++) { + for (int i = 0; i < duplicate_multiplier; i ++) { + for (int j = 0; j < num_unique_rows; j++) { + Put put = new Put(Bytes.toBytes("row" + j)); + put.setWriteToWAL(false); + put.add(fam1, qf1, version++, val1); + region.put(put); + } + } + region.flushcache(); + } + //before compaction + Store store = region.getStore(fam1); + List storeFiles = store.getStorefiles(); + for (StoreFile storefile : storeFiles) { + StoreFile.Reader reader = storefile.getReader(); + reader.loadFileInfo(); + reader.loadBloomfilter(); + assertEquals(num_unique_rows*duplicate_multiplier, reader.getEntries()); + assertEquals(num_unique_rows, reader.getFilterEntries()); + } + + region.compactStores(true); + + //after compaction + storeFiles = store.getStorefiles(); + for (StoreFile storefile : storeFiles) { + StoreFile.Reader reader = storefile.getReader(); + reader.loadFileInfo(); + reader.loadBloomfilter(); + assertEquals(num_unique_rows*duplicate_multiplier*num_storefiles, + reader.getEntries()); + assertEquals(num_unique_rows, reader.getFilterEntries()); + } + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + public void testAllColumnsWithBloomFilter() throws IOException { + byte [] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter"); + byte [] FAMILY = Bytes.toBytes("family"); + + //Create table + HColumnDescriptor hcd = new HColumnDescriptor(FAMILY) + .setMaxVersions(Integer.MAX_VALUE) + .setBloomFilterType(BloomType.ROWCOL); + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(hcd); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + Path path = new Path(DIR + "testAllColumnsWithBloomFilter"); + this.region = HRegion.createHRegion(info, path, conf, htd); + try { + // For row:0, col:0: insert versions 1 through 5. + byte row[] = Bytes.toBytes("row:" + 0); + byte column[] = Bytes.toBytes("column:" + 0); + Put put = new Put(row); + put.setWriteToWAL(false); + for (long idx = 1; idx <= 4; idx++) { + put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx)); + } + region.put(put); + + //Flush + region.flushcache(); + + //Get rows + Get get = new Get(row); + get.setMaxVersions(); + KeyValue[] kvs = region.get(get, null).raw(); + + //Check if rows are correct + assertEquals(4, kvs.length); + checkOneCell(kvs[0], FAMILY, 0, 0, 4); + checkOneCell(kvs[1], FAMILY, 0, 0, 3); + checkOneCell(kvs[2], FAMILY, 0, 0, 2); + checkOneCell(kvs[3], FAMILY, 0, 0, 1); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + /** + * Testcase to cover bug-fix for HBASE-2823 + * Ensures correct delete when issuing delete row + * on columns with bloom filter set to row+col (BloomType.ROWCOL) + */ + public void testDeleteRowWithBloomFilter() throws IOException { + byte [] tableName = Bytes.toBytes("testDeleteRowWithBloomFilter"); + byte [] familyName = Bytes.toBytes("familyName"); + + // Create Table + HColumnDescriptor hcd = new HColumnDescriptor(familyName) + .setMaxVersions(Integer.MAX_VALUE) + .setBloomFilterType(BloomType.ROWCOL); + + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(hcd); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + Path path = new Path(DIR + "TestDeleteRowWithBloomFilter"); + this.region = HRegion.createHRegion(info, path, conf, htd); + try { + // Insert some data + byte row[] = Bytes.toBytes("row1"); + byte col[] = Bytes.toBytes("col1"); + + Put put = new Put(row); + put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue")); + region.put(put); + region.flushcache(); + + Delete del = new Delete(row); + region.delete(del, null, true); + region.flushcache(); + + // Get remaining rows (should have none) + Get get = new Get(row); + get.addColumn(familyName, col); + + KeyValue[] keyValues = region.get(get, null).raw(); + assertTrue(keyValues.length == 0); + } finally { + HRegion.closeHRegion(this.region); + this.region = null; + } + } + + @Test public void testgetHDFSBlocksDistribution() throws Exception { + HBaseTestingUtility htu = new HBaseTestingUtility(); + final int DEFAULT_BLOCK_SIZE = 1024; + htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE); + htu.getConfiguration().setInt("dfs.replication", 2); + + + // set up a cluster with 3 nodes + MiniHBaseCluster cluster = null; + String dataNodeHosts[] = new String[] { "host1", "host2", "host3" }; + int regionServersCount = 3; + + try { + cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts); + byte [][] families = {fam1, fam2}; + HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families); + + //Setting up region + byte row[] = Bytes.toBytes("row1"); + byte col[] = Bytes.toBytes("col1"); + + Put put = new Put(row); + put.add(fam1, col, 1, Bytes.toBytes("test1")); + put.add(fam2, col, 1, Bytes.toBytes("test2")); + ht.put(put); + + HRegion firstRegion = htu.getHBaseCluster(). + getRegions(Bytes.toBytes(this.getName())).get(0); + firstRegion.flushcache(); + HDFSBlocksDistribution blocksDistribution1 = + firstRegion.getHDFSBlocksDistribution(); + + // given the default replication factor is 2 and we have 2 HFiles, + // we will have total of 4 replica of blocks on 3 datanodes; thus there + // must be at least one host that have replica for 2 HFiles. That host's + // weight will be equal to the unique block weight. + long uniqueBlocksWeight1 = + blocksDistribution1.getUniqueBlocksTotalWeight(); + + String topHost = blocksDistribution1.getTopHosts().get(0); + long topHostWeight = blocksDistribution1.getWeight(topHost); + assertTrue(uniqueBlocksWeight1 == topHostWeight); + + // use the static method to compute the value, it should be the same. + // static method is used by load balancer or other components + HDFSBlocksDistribution blocksDistribution2 = + HRegion.computeHDFSBlocksDistribution(htu.getConfiguration(), + firstRegion.getTableDesc(), + firstRegion.getRegionInfo().getEncodedName()); + long uniqueBlocksWeight2 = + blocksDistribution2.getUniqueBlocksTotalWeight(); + + assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2); + + ht.close(); + } finally { + if (cluster != null) { + htu.shutdownMiniCluster(); + } + } + } + + /** + * Test case to check put function with memstore flushing for same row, same ts + * @throws Exception + */ + public void testPutWithMemStoreFlush() throws Exception { + Configuration conf = HBaseConfiguration.create(); + String method = "testPutWithMemStoreFlush"; + byte[] tableName = Bytes.toBytes(method); + byte[] family = Bytes.toBytes("family");; + byte[] qualifier = Bytes.toBytes("qualifier"); + byte[] row = Bytes.toBytes("putRow"); + byte[] value = null; + this.region = initHRegion(tableName, method, conf, family); + Put put = null; + Get get = null; + List kvs = null; + Result res = null; + + put = new Put(row); + value = Bytes.toBytes("value0"); + put.add(family, qualifier, 1234567l, value); + region.put(put); + get = new Get(row); + get.addColumn(family, qualifier); + get.setMaxVersions(); + res = this.region.get(get, null); + kvs = res.getColumn(family, qualifier); + assertEquals(1, kvs.size()); + assertEquals(Bytes.toBytes("value0"), kvs.get(0).getValue()); + + region.flushcache(); + get = new Get(row); + get.addColumn(family, qualifier); + get.setMaxVersions(); + res = this.region.get(get, null); + kvs = res.getColumn(family, qualifier); + assertEquals(1, kvs.size()); + assertEquals(Bytes.toBytes("value0"), kvs.get(0).getValue()); + + put = new Put(row); + value = Bytes.toBytes("value1"); + put.add(family, qualifier, 1234567l, value); + region.put(put); + get = new Get(row); + get.addColumn(family, qualifier); + get.setMaxVersions(); + res = this.region.get(get, null); + kvs = res.getColumn(family, qualifier); + assertEquals(1, kvs.size()); + assertEquals(Bytes.toBytes("value1"), kvs.get(0).getValue()); + + region.flushcache(); + get = new Get(row); + get.addColumn(family, qualifier); + get.setMaxVersions(); + res = this.region.get(get, null); + kvs = res.getColumn(family, qualifier); + assertEquals(1, kvs.size()); + assertEquals(Bytes.toBytes("value1"), kvs.get(0).getValue()); + } + + /** + * TestCase for increment + * + */ + private static class Incrementer implements Runnable { + private HRegion region; + private final static byte[] incRow = Bytes.toBytes("incRow"); + private final static byte[] family = Bytes.toBytes("family"); + private final static byte[] qualifier = Bytes.toBytes("qualifier"); + private final static long ONE = 1l; + private int incCounter; + + public Incrementer(HRegion region, int incCounter) { + this.region = region; + this.incCounter = incCounter; + } + + @Override + public void run() { + int count = 0; + while (count < incCounter) { + Increment inc = new Increment(incRow); + inc.addColumn(family, qualifier, ONE); + count++; + try { + region.increment(inc, null, true); + } catch (IOException e) { + e.printStackTrace(); + break; + } + } + } + } + + /** + * TestCase for append + * + */ + private static class Appender implements Runnable { + private HRegion region; + private final static byte[] appendRow = Bytes.toBytes("appendRow"); + private final static byte[] family = Bytes.toBytes("family"); + private final static byte[] qualifier = Bytes.toBytes("qualifier"); + private final static byte[] CHAR = Bytes.toBytes("a"); + private int appendCounter; + + public Appender(HRegion region, int appendCounter) { + this.region = region; + this.appendCounter = appendCounter; + } + + @Override + public void run() { + int count = 0; + while (count < appendCounter) { + Append app = new Append(appendRow); + app.add(family, qualifier, CHAR); + count++; + try { + region.append(app, null, true); + } catch (IOException e) { + e.printStackTrace(); + break; + } + } + } + } + + /** + * Test case to check append function with memstore flushing + * + * @throws Exception + */ + @Test + public void testParallelAppendWithMemStoreFlush() throws Exception { + Configuration conf = HBaseConfiguration.create(); + String method = "testParallelAppendWithMemStoreFlush"; + byte[] tableName = Bytes.toBytes(method); + byte[] family = Appender.family; + this.region = initHRegion(tableName, method, conf, family); + final HRegion region = this.region; + final AtomicBoolean appendDone = new AtomicBoolean(false); + Runnable flusher = new Runnable() { + @Override + public void run() { + while (!appendDone.get()) { + try { + region.flushcache(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + }; + + // after all append finished, the value will append to threadNum * appendCounter Appender.CHAR + int threadNum = 20; + int appendCounter = 100; + byte[] expected = new byte[threadNum * appendCounter]; + for (int i = 0; i < threadNum * appendCounter; i++) { + System.arraycopy(Appender.CHAR, 0, expected, i, 1); + } + Thread[] appenders = new Thread[threadNum]; + Thread flushThread = new Thread(flusher); + for (int i = 0; i < threadNum; i++) { + appenders[i] = new Thread(new Appender(this.region, appendCounter)); + appenders[i].start(); + } + flushThread.start(); + for (int i = 0; i < threadNum; i++) { + appenders[i].join(); + } + + appendDone.set(true); + flushThread.join(); + + Get get = new Get(Appender.appendRow); + get.addColumn(Appender.family, Appender.qualifier); + get.setMaxVersions(1); + Result res = this.region.get(get, null); + List kvs = res.getColumn(Appender.family, Appender.qualifier); + + // we just got the latest version + assertEquals(kvs.size(), 1); + KeyValue kv = kvs.get(0); + byte[] appendResult = new byte[kv.getValueLength()]; + System.arraycopy(kv.getBuffer(), kv.getValueOffset(), appendResult, 0, kv.getValueLength()); + assertEquals(expected, appendResult); + this.region = null; + } + + /** + * Test case to check increment function with memstore flushing + * @throws Exception + */ + @Test + public void testParallelIncrementWithMemStoreFlush() throws Exception { + String method = "testParallelIncrementWithMemStoreFlush"; + byte[] tableName = Bytes.toBytes(method); + byte[] family = Incrementer.family; + this.region = initHRegion(tableName, method, conf, family); + final HRegion region = this.region; + final AtomicBoolean incrementDone = new AtomicBoolean(false); + Runnable reader = new Runnable() { + @Override + public void run() { + while (!incrementDone.get()) { + try { + region.flushcache(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + }; + + //after all increment finished, the row will increment to 20*100 = 2000 + int threadNum = 20; + int incCounter = 100; + long expected = threadNum * incCounter; + Thread[] incrementers = new Thread[threadNum]; + Thread flushThread = new Thread(reader); + for (int i = 0; i < threadNum; i++) { + incrementers[i] = new Thread(new Incrementer(this.region, incCounter)); + incrementers[i].start(); + } + flushThread.start(); + for (int i = 0; i < threadNum; i++) { + incrementers[i].join(); + } + + incrementDone.set(true); + flushThread.join(); + + Get get = new Get(Incrementer.incRow); + get.addColumn(Incrementer.family, Incrementer.qualifier); + get.setMaxVersions(1); + Result res = this.region.get(get, null); + List kvs = res.getColumn(Incrementer.family, + Incrementer.qualifier); + + //we just got the latest version + assertEquals(kvs.size(), 1); + KeyValue kv = kvs.get(0); + assertEquals(expected, Bytes.toLong(kv.getBuffer(), kv.getValueOffset())); + this.region = null; + } + + private void putData(int startRow, int numRows, byte [] qf, + byte [] ...families) + throws IOException { + for(int i=startRow; ifirstValue. + * @param r + * @param fs + * @param firstValue + * @throws IOException + */ + private void assertScan(final HRegion r, final byte [] fs, + final byte [] firstValue) + throws IOException { + byte [][] families = {fs}; + Scan scan = new Scan(); + for (int i = 0; i < families.length; i++) scan.addFamily(families[i]); + InternalScanner s = r.getScanner(scan); + try { + List curVals = new ArrayList(); + boolean first = true; + OUTER_LOOP: while(s.next(curVals)) { + for (KeyValue kv: curVals) { + byte [] val = kv.getValue(); + byte [] curval = val; + if (first) { + first = false; + assertTrue(Bytes.compareTo(curval, firstValue) == 0); + } else { + // Not asserting anything. Might as well break. + break OUTER_LOOP; + } + } + } + } finally { + s.close(); + } + } + + private Configuration initSplit() { + Configuration conf = HBaseConfiguration.create(this.conf); + + // Always compact if there is more than one store file. + conf.setInt("hbase.hstore.compactionThreshold", 2); + + // Make lease timeout longer, lease checks less frequent + conf.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000); + + conf.setInt(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, 10 * 1000); + + // Increase the amount of time between client retries + conf.setLong("hbase.client.pause", 15 * 1000); + + // This size should make it so we always split using the addContent + // below. After adding all data, the first region is 1.3M + conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128); + return conf; + } + + /** + * @param tableName + * @param callingMethod + * @param conf + * @param families + * @throws IOException + * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done. + */ + public static HRegion initHRegion (byte [] tableName, String callingMethod, + Configuration conf, byte [] ... families) + throws IOException{ + return initHRegion(tableName, null, null, callingMethod, conf, false, families); + } + + /** + * @param tableName + * @param callingMethod + * @param conf + * @param isReadOnly + * @param families + * @throws IOException + * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done. + */ + public static HRegion initHRegion (byte [] tableName, String callingMethod, + Configuration conf, boolean isReadOnly, byte [] ... families) + throws IOException{ + return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families); + } + + /** + * @param tableName + * @param startKey + * @param stopKey + * @param callingMethod + * @param conf + * @param isReadOnly + * @param families + * @throws IOException + * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done. + */ + private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey, + String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families) + throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.setReadOnly(isReadOnly); + for(byte [] family : families) { + htd.addFamily(new HColumnDescriptor(family)); + } + HRegionInfo info = new HRegionInfo(htd.getName(), startKey, stopKey, false); + Path path = new Path(DIR + callingMethod); + FileSystem fs = FileSystem.get(conf); + if (fs.exists(path)) { + if (!fs.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + return HRegion.createHRegion(info, path, conf, htd); + } + + /** + * Assert that the passed in KeyValue has expected contents for the + * specified row, column & timestamp. + */ + private void checkOneCell(KeyValue kv, byte[] cf, + int rowIdx, int colIdx, long ts) { + String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts; + assertEquals("Row mismatch which checking: " + ctx, + "row:"+ rowIdx, Bytes.toString(kv.getRow())); + assertEquals("ColumnFamily mismatch while checking: " + ctx, + Bytes.toString(cf), Bytes.toString(kv.getFamily())); + assertEquals("Column qualifier mismatch while checking: " + ctx, + "column:" + colIdx, Bytes.toString(kv.getQualifier())); + assertEquals("Timestamp mismatch while checking: " + ctx, + ts, kv.getTimestamp()); + assertEquals("Value mismatch while checking: " + ctx, + "value-version-" + ts, Bytes.toString(kv.getValue())); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java new file mode 100644 index 0000000..10a9370 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java @@ -0,0 +1,90 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.RegionTooBusyException; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * TestHRegion with hbase.busy.wait.duration set to 1000 (1 second). + * We can't use parameterized test since TestHRegion is old fashion. + */ +@Category(MediumTests.class) +@SuppressWarnings("deprecation") +public class TestHRegionBusyWait extends TestHRegion { + public TestHRegionBusyWait() { + conf.set("hbase.busy.wait.duration", "1000"); + } + + /** + * Test RegionTooBusyException thrown when region is busy + */ + @Test (timeout=2000) + public void testRegionTooBusy() throws IOException { + String method = "testRegionTooBusy"; + byte[] tableName = Bytes.toBytes(method); + byte[] family = Bytes.toBytes("family"); + region = initHRegion(tableName, method, conf, family); + final AtomicBoolean stopped = new AtomicBoolean(true); + Thread t = new Thread(new Runnable() { + @Override + public void run() { + try { + region.lock.writeLock().lock(); + stopped.set(false); + while (!stopped.get()) { + Thread.sleep(100); + } + } catch (InterruptedException ie) { + } finally { + region.lock.writeLock().unlock(); + } + } + }); + t.start(); + Get get = new Get(row); + try { + while (stopped.get()) { + Thread.sleep(100); + } + region.get(get, null); + fail("Should throw RegionTooBusyException"); + } catch (InterruptedException ie) { + fail("test interrupted"); + } catch (RegionTooBusyException e) { + // Good, expected + } finally { + stopped.set(true); + try { + t.join(); + } catch (Throwable e) { + } + + HRegion.closeHRegion(region); + region = null; + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java new file mode 100644 index 0000000..6dfba41 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java @@ -0,0 +1,149 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.MD5Hash; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestHRegionInfo { + @Test + public void testCreateHRegionInfoName() throws Exception { + String tableName = "tablename"; + final byte [] tn = Bytes.toBytes(tableName); + String startKey = "startkey"; + final byte [] sk = Bytes.toBytes(startKey); + String id = "id"; + + // old format region name + byte [] name = HRegionInfo.createRegionName(tn, sk, id, false); + String nameStr = Bytes.toString(name); + assertEquals(tableName + "," + startKey + "," + id, nameStr); + + + // new format region name. + String md5HashInHex = MD5Hash.getMD5AsHex(name); + assertEquals(HRegionInfo.MD5_HEX_LENGTH, md5HashInHex.length()); + name = HRegionInfo.createRegionName(tn, sk, id, true); + nameStr = Bytes.toString(name); + assertEquals(tableName + "," + startKey + "," + + id + "." + md5HashInHex + ".", + nameStr); + } + + @Test + public void testGetSetOfHTD() throws IOException { + HBaseTestingUtility HTU = new HBaseTestingUtility(); + final String tablename = "testGetSetOfHTD"; + + // Delete the temporary table directory that might still be there from the + // previous test run. + FSTableDescriptors.deleteTableDescriptorIfExists(tablename, + HTU.getConfiguration()); + + HTableDescriptor htd = new HTableDescriptor(tablename); + FSTableDescriptors.createTableDescriptor(htd, HTU.getConfiguration()); + HRegionInfo hri = new HRegionInfo(Bytes.toBytes("testGetSetOfHTD"), + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); + HTableDescriptor htd2 = hri.getTableDesc(); + assertTrue(htd.equals(htd2)); + final String key = "SOME_KEY"; + assertNull(htd.getValue(key)); + final String value = "VALUE"; + htd.setValue(key, value); + hri.setTableDesc(htd); + HTableDescriptor htd3 = hri.getTableDesc(); + assertTrue(htd.equals(htd3)); + } + + @Test + public void testContainsRange() { + HTableDescriptor tableDesc = new HTableDescriptor("testtable"); + HRegionInfo hri = new HRegionInfo( + tableDesc.getName(), Bytes.toBytes("a"), Bytes.toBytes("g")); + // Single row range at start of region + assertTrue(hri.containsRange(Bytes.toBytes("a"), Bytes.toBytes("a"))); + // Fully contained range + assertTrue(hri.containsRange(Bytes.toBytes("b"), Bytes.toBytes("c"))); + // Range overlapping start of region + assertTrue(hri.containsRange(Bytes.toBytes("a"), Bytes.toBytes("c"))); + // Fully contained single-row range + assertTrue(hri.containsRange(Bytes.toBytes("c"), Bytes.toBytes("c"))); + // Range that overlaps end key and hence doesn't fit + assertFalse(hri.containsRange(Bytes.toBytes("a"), Bytes.toBytes("g"))); + // Single row range on end key + assertFalse(hri.containsRange(Bytes.toBytes("g"), Bytes.toBytes("g"))); + // Single row range entirely outside + assertFalse(hri.containsRange(Bytes.toBytes("z"), Bytes.toBytes("z"))); + + // Degenerate range + try { + hri.containsRange(Bytes.toBytes("z"), Bytes.toBytes("a")); + fail("Invalid range did not throw IAE"); + } catch (IllegalArgumentException iae) { + } + } + + @Test + public void testLastRegionCompare() { + HTableDescriptor tableDesc = new HTableDescriptor("testtable"); + HRegionInfo hrip = new HRegionInfo( + tableDesc.getName(), Bytes.toBytes("a"), new byte[0]); + HRegionInfo hric = new HRegionInfo( + tableDesc.getName(), Bytes.toBytes("a"), Bytes.toBytes("b")); + assertTrue(hrip.compareTo(hric) > 0); + } + + @Test + public void testMetaTables() { + assertTrue(HRegionInfo.ROOT_REGIONINFO.isMetaTable()); + assertTrue(HRegionInfo.FIRST_META_REGIONINFO.isMetaTable()); + } + + @Test + public void testComparator() { + byte[] tablename = Bytes.toBytes("comparatorTablename"); + byte[] empty = new byte[0]; + HRegionInfo older = new HRegionInfo(tablename, empty, empty, false, 0L); + HRegionInfo newer = new HRegionInfo(tablename, empty, empty, false, 1L); + assertTrue(older.compareTo(newer) < 0); + assertTrue(newer.compareTo(older) > 0); + assertTrue(older.compareTo(older) == 0); + assertTrue(newer.compareTo(newer) == 0); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java new file mode 100644 index 0000000..e0cf415 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionOnCluster.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.MediumTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests that need to spin up a cluster testing an {@link HRegion}. Use + * {@link TestHRegion} if you don't need a cluster, if you can test w/ a + * standalone {@link HRegion}. + */ +@Category(MediumTests.class) +public class TestHRegionOnCluster { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @Test (timeout=180000) + public void testDataCorrectnessReplayingRecoveredEdits() throws Exception { + final int NUM_MASTERS = 1; + final int NUM_RS = 3; + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + + try { + final byte[] TABLENAME = Bytes + .toBytes("testDataCorrectnessReplayingRecoveredEdits"); + final byte[] FAMILY = Bytes.toBytes("family"); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + + // Create table + HTableDescriptor desc = new HTableDescriptor(TABLENAME); + desc.addFamily(new HColumnDescriptor(FAMILY)); + HBaseAdmin hbaseAdmin = TEST_UTIL.getHBaseAdmin(); + hbaseAdmin.createTable(desc); + + assertTrue(hbaseAdmin.isTableAvailable(TABLENAME)); + + // Put data: r1->v1 + HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); + putDataAndVerify(table, "r1", FAMILY, "v1", 1); + + // Move region to target server + HRegionInfo regionInfo = table.getRegionLocation("r1").getRegionInfo(); + int originServerNum = cluster.getServerWith(regionInfo.getRegionName()); + HRegionServer originServer = cluster.getRegionServer(originServerNum); + int targetServerNum = (originServerNum + 1) % NUM_RS; + HRegionServer targetServer = cluster.getRegionServer(targetServerNum); + assertFalse(originServer.equals(targetServer)); + + do { + Thread.sleep(10); + } while (!originServer.getServerName().equals( + cluster.getMaster().getAssignmentManager().getRegionServerOfRegion(regionInfo))); + + hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(), + Bytes.toBytes(targetServer.getServerName().getServerName())); + + do { + Thread.sleep(10); + } while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum || + !targetServer.getServerName().equals( + cluster.getMaster().getAssignmentManager().getRegionServerOfRegion(regionInfo))); + + // Put data: r2->v2 + putDataAndVerify(table, "r2", FAMILY, "v2", 2); + + // Move region to origin server + hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(), + Bytes.toBytes(originServer.getServerName().getServerName())); + do { + Thread.sleep(1); + } while (cluster.getServerWith(regionInfo.getRegionName()) == targetServerNum); + + // Put data: r3->v3 + putDataAndVerify(table, "r3", FAMILY, "v3", 3); + + // Kill target server + targetServer.kill(); + cluster.getRegionServerThreads().get(targetServerNum).join(); + // Wait until finish processing of shutdown + while (master.getServerManager().areDeadServersInProgress()) { + Thread.sleep(5); + } + // Kill origin server + originServer.kill(); + cluster.getRegionServerThreads().get(originServerNum).join(); + + // Put data: r4->v4 + putDataAndVerify(table, "r4", FAMILY, "v4", 4); + + } finally { + TEST_UTIL.shutdownMiniCluster(); + } + } + + private void putDataAndVerify(HTable table, String row, byte[] family, + String value, int verifyNum) throws IOException { + System.out.println("=========Putting data :" + row); + Put put = new Put(Bytes.toBytes(row)); + put.add(family, Bytes.toBytes("q1"), Bytes.toBytes(value)); + table.put(put); + ResultScanner resultScanner = table.getScanner(new Scan()); + List results = new ArrayList(); + while (true) { + Result r = resultScanner.next(); + if (r == null) + break; + results.add(r); + } + resultScanner.close(); + if (results.size() != verifyNum) { + System.out.println(results); + } + assertEquals(verifyNum, results.size()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java new file mode 100644 index 0000000..a1bf73b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java @@ -0,0 +1,320 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread; +import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.ServerCallable; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Test; + +import com.google.common.collect.Lists; +import org.junit.experimental.categories.Category; + +/** + * Tests bulk loading of HFiles and shows the atomicity or lack of atomicity of + * the region server's bullkLoad functionality. + */ +@Category(LargeTests.class) +public class TestHRegionServerBulkLoad { + final static Log LOG = LogFactory.getLog(TestHRegionServerBulkLoad.class); + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private final static Configuration conf = UTIL.getConfiguration(); + private final static byte[] QUAL = Bytes.toBytes("qual"); + private final static int NUM_CFS = 10; + public static int BLOCKSIZE = 64 * 1024; + public static String COMPRESSION = Compression.Algorithm.NONE.getName(); + + private final static byte[][] families = new byte[NUM_CFS][]; + static { + for (int i = 0; i < NUM_CFS; i++) { + families[i] = Bytes.toBytes(family(i)); + } + } + + static byte[] rowkey(int i) { + return Bytes.toBytes(String.format("row_%08d", i)); + } + + static String family(int i) { + return String.format("family_%04d", i); + } + + /** + * Create an HFile with the given number of rows with a specified value. + */ + public static void createHFile(FileSystem fs, Path path, byte[] family, + byte[] qualifier, byte[] value, int numRows) throws IOException { + HFile.Writer writer = HFile + .getWriterFactory(conf, new CacheConfig(conf)) + .withPath(fs, path) + .withBlockSize(BLOCKSIZE) + .withCompression(COMPRESSION) + .withComparator(KeyValue.KEY_COMPARATOR) + .create(); + long now = System.currentTimeMillis(); + try { + // subtract 2 since iterateOnSplits doesn't include boundary keys + for (int i = 0; i < numRows; i++) { + KeyValue kv = new KeyValue(rowkey(i), family, qualifier, now, value); + writer.append(kv); + } + } finally { + writer.close(); + } + } + + /** + * Thread that does full scans of the table looking for any partially + * completed rows. + * + * Each iteration of this loads 10 hdfs files, which occupies 5 file open file + * handles. So every 10 iterations (500 file handles) it does a region + * compaction to reduce the number of open file handles. + */ + public static class AtomicHFileLoader extends RepeatingTestThread { + final AtomicLong numBulkLoads = new AtomicLong(); + final AtomicLong numCompactions = new AtomicLong(); + private String tableName; + + public AtomicHFileLoader(String tableName, TestContext ctx, + byte targetFamilies[][]) throws IOException { + super(ctx); + this.tableName = tableName; + } + + public void doAnAction() throws Exception { + long iteration = numBulkLoads.getAndIncrement(); + Path dir = UTIL.getDataTestDir(String.format("bulkLoad_%08d", + iteration)); + + // create HFiles for different column families + FileSystem fs = UTIL.getTestFileSystem(); + byte[] val = Bytes.toBytes(String.format("%010d", iteration)); + final List> famPaths = new ArrayList>( + NUM_CFS); + for (int i = 0; i < NUM_CFS; i++) { + Path hfile = new Path(dir, family(i)); + byte[] fam = Bytes.toBytes(family(i)); + createHFile(fs, hfile, fam, QUAL, val, 1000); + famPaths.add(new Pair(fam, hfile.toString())); + } + + // bulk load HFiles + HConnection conn = UTIL.getHBaseAdmin().getConnection(); + byte[] tbl = Bytes.toBytes(tableName); + new ServerCallable(conn, tbl, Bytes + .toBytes("aaa")) { + @Override + public Void call() throws Exception { + LOG.debug("Going to connect to server " + location + " for row " + + Bytes.toStringBinary(row)); + byte[] regionName = location.getRegionInfo().getRegionName(); + server.bulkLoadHFiles(famPaths, regionName); + return null; + } + }.withRetries(); + + // Periodically do compaction to reduce the number of open file handles. + if (numBulkLoads.get() % 10 == 0) { + // 10 * 50 = 500 open file handles! + new ServerCallable(conn, tbl, + Bytes.toBytes("aaa")) { + @Override + public Void call() throws Exception { + LOG.debug("compacting " + location + " for row " + + Bytes.toStringBinary(row)); + server.compactRegion(location.getRegionInfo(), true); + numCompactions.incrementAndGet(); + return null; + } + }.withRetries(); + } + } + } + + /** + * Thread that does full scans of the table looking for any partially + * completed rows. + */ + public static class AtomicScanReader extends RepeatingTestThread { + byte targetFamilies[][]; + HTable table; + AtomicLong numScans = new AtomicLong(); + AtomicLong numRowsScanned = new AtomicLong(); + String TABLE_NAME; + + public AtomicScanReader(String TABLE_NAME, TestContext ctx, + byte targetFamilies[][]) throws IOException { + super(ctx); + this.TABLE_NAME = TABLE_NAME; + this.targetFamilies = targetFamilies; + table = new HTable(conf, TABLE_NAME); + } + + public void doAnAction() throws Exception { + Scan s = new Scan(); + for (byte[] family : targetFamilies) { + s.addFamily(family); + } + ResultScanner scanner = table.getScanner(s); + + for (Result res : scanner) { + byte[] lastRow = null, lastFam = null, lastQual = null; + byte[] gotValue = null; + for (byte[] family : targetFamilies) { + byte qualifier[] = QUAL; + byte thisValue[] = res.getValue(family, qualifier); + if (gotValue != null && thisValue != null + && !Bytes.equals(gotValue, thisValue)) { + + StringBuilder msg = new StringBuilder(); + msg.append("Failed on scan ").append(numScans) + .append(" after scanning ").append(numRowsScanned) + .append(" rows!\n"); + msg.append("Current was " + Bytes.toString(res.getRow()) + "/" + + Bytes.toString(family) + ":" + Bytes.toString(qualifier) + + " = " + Bytes.toString(thisValue) + "\n"); + msg.append("Previous was " + Bytes.toString(lastRow) + "/" + + Bytes.toString(lastFam) + ":" + Bytes.toString(lastQual) + + " = " + Bytes.toString(gotValue)); + throw new RuntimeException(msg.toString()); + } + + lastFam = family; + lastQual = qualifier; + lastRow = res.getRow(); + gotValue = thisValue; + } + numRowsScanned.getAndIncrement(); + } + numScans.getAndIncrement(); + } + } + + /** + * Creates a table with given table name and specified number of column + * families if the table does not already exist. + */ + private void setupTable(String table, int cfs) throws IOException { + try { + LOG.info("Creating table " + table); + HTableDescriptor htd = new HTableDescriptor(table); + for (int i = 0; i < 10; i++) { + htd.addFamily(new HColumnDescriptor(family(i))); + } + + UTIL.getHBaseAdmin().createTable(htd); + } catch (TableExistsException tee) { + LOG.info("Table " + table + " already exists"); + } + } + + /** + * Atomic bulk load. + */ + @Test + public void testAtomicBulkLoad() throws Exception { + String TABLE_NAME = "atomicBulkLoad"; + + int millisToRun = 30000; + int numScanners = 50; + + UTIL.startMiniCluster(1); + try { + runAtomicBulkloadTest(TABLE_NAME, millisToRun, numScanners); + } finally { + UTIL.shutdownMiniCluster(); + } + } + + void runAtomicBulkloadTest(String tableName, int millisToRun, int numScanners) + throws Exception { + setupTable(tableName, 10); + + TestContext ctx = new TestContext(UTIL.getConfiguration()); + + AtomicHFileLoader loader = new AtomicHFileLoader(tableName, ctx, null); + ctx.addThread(loader); + + List scanners = Lists.newArrayList(); + for (int i = 0; i < numScanners; i++) { + AtomicScanReader scanner = new AtomicScanReader(tableName, ctx, families); + scanners.add(scanner); + ctx.addThread(scanner); + } + + ctx.startThreads(); + ctx.waitFor(millisToRun); + ctx.stop(); + + LOG.info("Loaders:"); + LOG.info(" loaded " + loader.numBulkLoads.get()); + LOG.info(" compations " + loader.numCompactions.get()); + + LOG.info("Scanners:"); + for (AtomicScanReader scanner : scanners) { + LOG.info(" scanned " + scanner.numScans.get()); + LOG.info(" verified " + scanner.numRowsScanned.get() + " rows"); + } + } + + /** + * Run test on an HBase instance for 5 minutes. This assumes that the table + * under test only has a single region. + */ + public static void main(String args[]) throws Exception { + try { + Configuration c = HBaseConfiguration.create(); + TestHRegionServerBulkLoad test = new TestHRegionServerBulkLoad(); + test.setConf(c); + test.runAtomicBulkloadTest("atomicTableTest", 5 * 60 * 1000, 50); + } finally { + System.exit(0); // something hangs (believe it is lru threadpool) + } + } + + private void setConf(Configuration c) { + UTIL = new HBaseTestingUtility(c); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java new file mode 100644 index 0000000..ae5da03 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java @@ -0,0 +1,761 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestKeepDeletes extends HBaseTestCase { + private final byte[] T0 = Bytes.toBytes("0"); + private final byte[] T1 = Bytes.toBytes("1"); + private final byte[] T2 = Bytes.toBytes("2"); + private final byte[] T3 = Bytes.toBytes("3"); + private final byte[] T4 = Bytes.toBytes("4"); + private final byte[] T5 = Bytes.toBytes("5"); + private final byte[] T6 = Bytes.toBytes("6"); + + private final byte[] c0 = COLUMNS[0]; + private final byte[] c1 = COLUMNS[1]; + + /** + * Make sure that deleted rows are retained. + * Family delete markers are deleted. + * Column Delete markers are versioned + * Time range scan of deleted rows are possible + */ + public void testBasicScenario() throws Exception { + // keep 3 versions, rows do not expire + HTableDescriptor htd = createTableDescriptor(getName(), 0, 3, + HConstants.FOREVER, true); + HRegion region = createNewHRegion(htd, null, null); + + long ts = System.currentTimeMillis(); + Put p = new Put(T1, ts); + p.add(c0, c0, T1); + region.put(p); + p = new Put(T1, ts+1); + p.add(c0, c0, T2); + region.put(p); + p = new Put(T1, ts+2); + p.add(c0, c0, T3); + region.put(p); + p = new Put(T1, ts+4); + p.add(c0, c0, T4); + region.put(p); + + // now place a delete marker at ts+2 + Delete d = new Delete(T1, ts+2, null); + region.delete(d, null, true); + + // a raw scan can see the delete markers + // (one for each column family) + assertEquals(3, countDeleteMarkers(region)); + + // get something *before* the delete marker + Get g = new Get(T1); + g.setMaxVersions(); + g.setTimeRange(0L, ts+2); + Result r = region.get(g, null); + checkResult(r, c0, c0, T2,T1); + + // flush + region.flushcache(); + + // yep, T2 still there, T1 gone + r = region.get(g, null); + checkResult(r, c0, c0, T2); + + // major compact + region.compactStores(true); + region.compactStores(true); + + // one delete marker left (the others did not + // have older puts) + assertEquals(1, countDeleteMarkers(region)); + + // still there (even after multiple compactions) + r = region.get(g, null); + checkResult(r, c0, c0, T2); + + // a timerange that includes the delete marker won't see past rows + g.setTimeRange(0L, ts+4); + r = region.get(g, null); + assertTrue(r.isEmpty()); + + // two more puts, this will expire the older puts. + p = new Put(T1, ts+5); + p.add(c0, c0, T5); + region.put(p); + p = new Put(T1, ts+6); + p.add(c0, c0, T6); + region.put(p); + + // also add an old put again + // (which is past the max versions) + p = new Put(T1, ts); + p.add(c0, c0, T1); + region.put(p); + r = region.get(g, null); + assertTrue(r.isEmpty()); + + region.flushcache(); + region.compactStores(true); + region.compactStores(true); + + // verify that the delete marker itself was collected + region.put(p); + r = region.get(g, null); + checkResult(r, c0, c0, T1); + assertEquals(0, countDeleteMarkers(region)); + + region.close(); + region.getLog().closeAndDelete(); + } + + /** + * Even when the store does not keep deletes a "raw" scan will + * return everything it can find (unless discarding cells is guaranteed + * to have no effect). + * Assuming this the desired behavior. Could also disallow "raw" scanning + * if the store does not have KEEP_DELETED_CELLS enabled. + * (can be changed easily) + */ + public void testRawScanWithoutKeepingDeletes() throws Exception { + // KEEP_DELETED_CELLS is NOT enabled + HTableDescriptor htd = createTableDescriptor(getName(), 0, 3, + HConstants.FOREVER, false); + HRegion region = createNewHRegion(htd, null, null); + + long ts = System.currentTimeMillis(); + Put p = new Put(T1, ts); + p.add(c0, c0, T1); + region.put(p); + + Delete d = new Delete(T1, ts, null); + d.deleteColumn(c0, c0, ts); + region.delete(d, null, true); + + // scan still returns delete markers and deletes rows + Scan s = new Scan(); + s.setRaw(true); + s.setMaxVersions(); + InternalScanner scan = region.getScanner(s); + List kvs = new ArrayList(); + scan.next(kvs); + assertEquals(2, kvs.size()); + + region.flushcache(); + region.compactStores(true); + + // after compaction they are gone + // (note that this a test with a Store without + // KEEP_DELETED_CELLS) + s = new Scan(); + s.setRaw(true); + s.setMaxVersions(); + scan = region.getScanner(s); + kvs = new ArrayList(); + scan.next(kvs); + assertTrue(kvs.isEmpty()); + + region.close(); + region.getLog().closeAndDelete(); + } + + /** + * basic verification of existing behavior + */ + public void testWithoutKeepingDeletes() throws Exception { + // KEEP_DELETED_CELLS is NOT enabled + HTableDescriptor htd = createTableDescriptor(getName(), 0, 3, + HConstants.FOREVER, false); + HRegion region = createNewHRegion(htd, null, null); + + long ts = System.currentTimeMillis(); + Put p = new Put(T1, ts); + p.add(c0, c0, T1); + region.put(p); + Delete d = new Delete(T1, ts+2, null); + d.deleteColumn(c0, c0, ts); + region.delete(d, null, true); + + // "past" get does not see rows behind delete marker + Get g = new Get(T1); + g.setMaxVersions(); + g.setTimeRange(0L, ts+1); + Result r = region.get(g, null); + assertTrue(r.isEmpty()); + + // "past" scan does not see rows behind delete marker + Scan s = new Scan(); + s.setMaxVersions(); + s.setTimeRange(0L, ts+1); + InternalScanner scanner = region.getScanner(s); + List kvs = new ArrayList(); + while(scanner.next(kvs)); + assertTrue(kvs.isEmpty()); + + // flushing and minor compaction keep delete markers + region.flushcache(); + region.compactStores(); + assertEquals(1, countDeleteMarkers(region)); + region.compactStores(true); + // major compaction deleted it + assertEquals(0, countDeleteMarkers(region)); + + region.close(); + region.getLog().closeAndDelete(); + } + + /** + * The ExplicitColumnTracker does not support "raw" scanning. + */ + public void testRawScanWithColumns() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName(), 0, 3, + HConstants.FOREVER, true); + HRegion region = createNewHRegion(htd, null, null); + + Scan s = new Scan(); + s.setRaw(true); + s.setMaxVersions(); + s.addColumn(c0, c0); + + try { + InternalScanner scan = region.getScanner(s); + fail("raw scanner with columns should have failed"); + } catch (DoNotRetryIOException dnre) { + // ok! + } + + region.close(); + region.getLog().closeAndDelete(); + } + + /** + * Verify that "raw" scanning mode return delete markers and deletes rows. + */ + public void testRawScan() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName(), 0, 3, + HConstants.FOREVER, true); + HRegion region = createNewHRegion(htd, null, null); + + long ts = System.currentTimeMillis(); + Put p = new Put(T1, ts); + p.add(c0, c0, T1); + region.put(p); + p = new Put(T1, ts+2); + p.add(c0, c0, T2); + region.put(p); + p = new Put(T1, ts+4); + p.add(c0, c0, T3); + region.put(p); + + Delete d = new Delete(T1, ts+1, null); + region.delete(d, null, true); + + d = new Delete(T1, ts+2, null); + d.deleteColumn(c0, c0, ts+2); + region.delete(d, null, true); + + d = new Delete(T1, ts+3, null); + d.deleteColumns(c0, c0, ts+3); + region.delete(d, null, true); + + Scan s = new Scan(); + s.setRaw(true); + s.setMaxVersions(); + InternalScanner scan = region.getScanner(s); + List kvs = new ArrayList(); + scan.next(kvs); + assertTrue(kvs.get(0).isDeleteFamily()); + assertEquals(kvs.get(1).getValue(), T3); + assertTrue(kvs.get(2).isDelete()); + assertTrue(kvs.get(3).isDeleteType()); + assertEquals(kvs.get(4).getValue(), T2); + assertEquals(kvs.get(5).getValue(), T1); + + region.close(); + region.getLog().closeAndDelete(); + } + + /** + * Verify that delete markers are removed from an otherwise empty store. + */ + public void testDeleteMarkerExpirationEmptyStore() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName(), 0, 1, + HConstants.FOREVER, true); + HRegion region = createNewHRegion(htd, null, null); + + long ts = System.currentTimeMillis(); + + Delete d = new Delete(T1, ts, null); + d.deleteColumns(c0, c0, ts); + region.delete(d, null, true); + + d = new Delete(T1, ts, null); + d.deleteFamily(c0); + region.delete(d, null, true); + + d = new Delete(T1, ts, null); + d.deleteColumn(c0, c0, ts+1); + region.delete(d, null, true); + + d = new Delete(T1, ts, null); + d.deleteColumn(c0, c0, ts+2); + region.delete(d, null, true); + + // 1 family marker, 1 column marker, 2 version markers + assertEquals(4, countDeleteMarkers(region)); + + // neither flush nor minor compaction removes any marker + region.flushcache(); + assertEquals(4, countDeleteMarkers(region)); + region.compactStores(false); + assertEquals(4, countDeleteMarkers(region)); + + // major compaction removes all, since there are no puts they affect + region.compactStores(true); + assertEquals(0, countDeleteMarkers(region)); + + region.close(); + region.getLog().closeAndDelete(); + } + + /** + * Test delete marker removal from store files. + */ + public void testDeleteMarkerExpiration() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName(), 0, 1, + HConstants.FOREVER, true); + HRegion region = createNewHRegion(htd, null, null); + + long ts = System.currentTimeMillis(); + + Put p = new Put(T1, ts); + p.add(c0, c0, T1); + region.put(p); + + // a put into another store (CF) should have no effect + p = new Put(T1, ts-10); + p.add(c1, c0, T1); + region.put(p); + + // all the following deletes affect the put + Delete d = new Delete(T1, ts, null); + d.deleteColumns(c0, c0, ts); + region.delete(d, null, true); + + d = new Delete(T1, ts, null); + d.deleteFamily(c0, ts); + region.delete(d, null, true); + + d = new Delete(T1, ts, null); + d.deleteColumn(c0, c0, ts+1); + region.delete(d, null, true); + + d = new Delete(T1, ts, null); + d.deleteColumn(c0, c0, ts+2); + region.delete(d, null, true); + + // 1 family marker, 1 column marker, 2 version markers + assertEquals(4, countDeleteMarkers(region)); + + region.flushcache(); + assertEquals(4, countDeleteMarkers(region)); + region.compactStores(false); + assertEquals(4, countDeleteMarkers(region)); + + // another put will push out the earlier put... + p = new Put(T1, ts+3); + p.add(c0, c0, T1); + region.put(p); + + region.flushcache(); + // no markers are collected, since there is an affected put + region.compactStores(true); + assertEquals(4, countDeleteMarkers(region)); + + // the last collections collected the earlier put + // so after this collection all markers + region.compactStores(true); + assertEquals(0, countDeleteMarkers(region)); + + region.close(); + region.getLog().closeAndDelete(); + } + + /** + * Verify correct range demarcation + */ + public void testRanges() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName(), 0, 3, + HConstants.FOREVER, true); + HRegion region = createNewHRegion(htd, null, null); + + long ts = System.currentTimeMillis(); + Put p = new Put(T1, ts); + p.add(c0, c0, T1); + p.add(c0, c1, T1); + p.add(c1, c0, T1); + p.add(c1, c1, T1); + region.put(p); + + p = new Put(T2, ts); + p.add(c0, c0, T1); + p.add(c0, c1, T1); + p.add(c1, c0, T1); + p.add(c1, c1, T1); + region.put(p); + + p = new Put(T1, ts+1); + p.add(c0, c0, T2); + p.add(c0, c1, T2); + p.add(c1, c0, T2); + p.add(c1, c1, T2); + region.put(p); + + p = new Put(T2, ts+1); + p.add(c0, c0, T2); + p.add(c0, c1, T2); + p.add(c1, c0, T2); + p.add(c1, c1, T2); + region.put(p); + + Delete d = new Delete(T1, ts+2, null); + d.deleteColumns(c0, c0, ts+2); + region.delete(d, null, true); + + d = new Delete(T1, ts+2, null); + d.deleteFamily(c1, ts+2); + region.delete(d, null, true); + + d = new Delete(T2, ts+2, null); + d.deleteFamily(c0, ts+2); + region.delete(d, null, true); + + // add an older delete, to make sure it is filtered + d = new Delete(T1, ts-10, null); + d.deleteFamily(c1, ts-10); + region.delete(d, null, true); + + // ts + 2 does NOT include the delete at ts+2 + checkGet(region, T1, c0, c0, ts+2, T2, T1); + checkGet(region, T1, c0, c1, ts+2, T2, T1); + checkGet(region, T1, c1, c0, ts+2, T2, T1); + checkGet(region, T1, c1, c1, ts+2, T2, T1); + + checkGet(region, T2, c0, c0, ts+2, T2, T1); + checkGet(region, T2, c0, c1, ts+2, T2, T1); + checkGet(region, T2, c1, c0, ts+2, T2, T1); + checkGet(region, T2, c1, c1, ts+2, T2, T1); + + // ts + 3 does + checkGet(region, T1, c0, c0, ts+3); + checkGet(region, T1, c0, c1, ts+3, T2, T1); + checkGet(region, T1, c1, c0, ts+3); + checkGet(region, T1, c1, c1, ts+3); + + checkGet(region, T2, c0, c0, ts+3); + checkGet(region, T2, c0, c1, ts+3); + checkGet(region, T2, c1, c0, ts+3, T2, T1); + checkGet(region, T2, c1, c1, ts+3, T2, T1); + + region.close(); + region.getLog().closeAndDelete(); + } + + /** + * Verify that column/version delete makers are sorted + * with their respective puts and removed correctly by + * versioning (i.e. not relying on the store earliestPutTS). + */ + public void testDeleteMarkerVersioning() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName(), 0, 1, + HConstants.FOREVER, true); + HRegion region = createNewHRegion(htd, null, null); + + long ts = System.currentTimeMillis(); + Put p = new Put(T1, ts); + p.add(c0, c0, T1); + region.put(p); + + // this prevents marker collection based on earliestPut + // (cannot keep earliest put per column in the store file) + p = new Put(T1, ts-10); + p.add(c0, c1, T1); + region.put(p); + + Delete d = new Delete(T1, ts, null); + // test corner case (Put and Delete have same TS) + d.deleteColumns(c0, c0, ts); + region.delete(d, null, true); + + d = new Delete(T1, ts+1, null); + d.deleteColumn(c0, c0, ts+1); + region.delete(d, null, true); + + d = new Delete(T1, ts+3, null); + d.deleteColumn(c0, c0, ts+3); + region.delete(d, null, true); + + region.flushcache(); + region.compactStores(true); + region.compactStores(true); + assertEquals(3, countDeleteMarkers(region)); + + // add two more puts, since max version is 1 + // the 2nd put (and all delete markers following) + // will be removed. + p = new Put(T1, ts+2); + p.add(c0, c0, T2); + region.put(p); + + // delete, put, delete, delete, put + assertEquals(3, countDeleteMarkers(region)); + + p = new Put(T1, ts+3); + p.add(c0, c0, T3); + region.put(p); + + // This is potentially questionable behavior. + // This could be changed by not letting the ScanQueryMatcher + // return SEEK_NEXT_COL if a put is past VERSIONS, but instead + // return SKIP if the store has KEEP_DELETED_CELLS set. + // + // As it stands, the 1 here is correct here. + // There are two puts, VERSIONS is one, so after the 1st put the scanner + // knows that there can be no more KVs (put or delete) that have any effect. + // + // delete, put, put | delete, delete + assertEquals(1, countDeleteMarkers(region)); + + // flush cache only sees what is in the memstore + region.flushcache(); + + // Here we have the three markers again, because the flush above + // removed the 2nd put before the file is written. + // So there's only one put, and hence the deletes already in the store + // files cannot be removed safely. + // delete, put, delete, delete + assertEquals(3, countDeleteMarkers(region)); + + region.compactStores(true); + assertEquals(3, countDeleteMarkers(region)); + + // add one more put + p = new Put(T1, ts+4); + p.add(c0, c0, T4); + region.put(p); + + region.flushcache(); + // one trailing delete marker remains (but only one) + // because delete markers do not increase the version count + assertEquals(1, countDeleteMarkers(region)); + region.compactStores(true); + region.compactStores(true); + assertEquals(1, countDeleteMarkers(region)); + + region.close(); + region.getLog().closeAndDelete(); + } + + /** + * Verify scenarios with multiple CFs and columns + */ + public void testWithMixedCFs() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName(), 0, 1, + HConstants.FOREVER, true); + HRegion region = createNewHRegion(htd, null, null); + + long ts = System.currentTimeMillis(); + + Put p = new Put(T1, ts); + p.add(c0, c0, T1); + p.add(c0, c1, T1); + p.add(c1, c0, T1); + p.add(c1, c1, T1); + region.put(p); + + p = new Put(T2, ts+1); + p.add(c0, c0, T2); + p.add(c0, c1, T2); + p.add(c1, c0, T2); + p.add(c1, c1, T2); + region.put(p); + + // family markers are each family + Delete d = new Delete(T1, ts+1, null); + region.delete(d, null, true); + + d = new Delete(T2, ts+2, null); + region.delete(d, null, true); + + Scan s = new Scan(T1); + s.setTimeRange(0, ts+1); + InternalScanner scanner = region.getScanner(s); + List kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(4, kvs.size()); + scanner.close(); + + s = new Scan(T2); + s.setTimeRange(0, ts+2); + scanner = region.getScanner(s); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(4, kvs.size()); + scanner.close(); + + region.close(); + region.getLog().closeAndDelete(); + } + + /** + * Test keeping deleted rows together with min versions set + * @throws Exception + */ + public void testWithMinVersions() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, true); + HRegion region = createNewHRegion(htd, null, null); + + long ts = System.currentTimeMillis() - 2000; // 2s in the past + + Put p = new Put(T1, ts); + p.add(c0, c0, T3); + region.put(p); + p = new Put(T1, ts-1); + p.add(c0, c0, T2); + region.put(p); + p = new Put(T1, ts-3); + p.add(c0, c0, T1); + region.put(p); + p = new Put(T1, ts-4); + p.add(c0, c0, T0); + region.put(p); + + // all puts now are just retained because of min versions = 3 + + // place a family delete marker + Delete d = new Delete(T1, ts-1, null); + region.delete(d, null, true); + // and a column delete marker + d = new Delete(T1, ts-2, null); + d.deleteColumns(c0, c0, ts-1); + region.delete(d, null, true); + + Get g = new Get(T1); + g.setMaxVersions(); + g.setTimeRange(0L, ts-2); + Result r = region.get(g, null); + checkResult(r, c0, c0, T1,T0); + + // 3 families, one column delete marker + assertEquals(4, countDeleteMarkers(region)); + + region.flushcache(); + // no delete marker removes by the flush + assertEquals(4, countDeleteMarkers(region)); + + r = region.get(g, null); + checkResult(r, c0, c0, T1); + p = new Put(T1, ts+1); + p.add(c0, c0, T4); + region.put(p); + region.flushcache(); + + assertEquals(4, countDeleteMarkers(region)); + + r = region.get(g, null); + checkResult(r, c0, c0, T1); + + // this will push out the last put before + // family delete marker + p = new Put(T1, ts+2); + p.add(c0, c0, T5); + region.put(p); + + region.flushcache(); + region.compactStores(true); + // the two family markers without puts are gone + assertEquals(2, countDeleteMarkers(region)); + + // the last compactStores updated the earliestPutTs, + // so after the next compaction the last family delete marker is also gone + region.compactStores(true); + assertEquals(0, countDeleteMarkers(region)); + + region.close(); + region.getLog().closeAndDelete(); + } + + private void checkGet(HRegion region, byte[] row, byte[] fam, byte[] col, + long time, byte[]... vals) throws IOException { + Get g = new Get(row); + g.addColumn(fam, col); + g.setMaxVersions(); + g.setTimeRange(0L, time); + Result r = region.get(g, null); + checkResult(r, fam, col, vals); + + } + + private int countDeleteMarkers(HRegion region) throws IOException { + Scan s = new Scan(); + s.setRaw(true); + s.setMaxVersions(); + InternalScanner scan = region.getScanner(s); + List kvs = new ArrayList(); + int res = 0; + boolean hasMore; + do { + hasMore = scan.next(kvs); + for (KeyValue kv : kvs) { + if(kv.isDelete()) res++; + } + kvs.clear(); + } while (hasMore); + scan.close(); + return res; + } + + private void checkResult(Result r, byte[] fam, byte[] col, byte[] ... vals) { + assertEquals(r.size(), vals.length); + List kvs = r.getColumn(fam, col); + assertEquals(kvs.size(), vals.length); + for (int i=0;i scanners = new ArrayList(); + + private byte[] row1; + private byte[] fam1; + private byte[] col1; + private byte[] data; + + private byte[] row2; + private byte[] fam2; + private byte[] col2; + + private byte[] col3; + private byte[] col4; + private byte[] col5; + + public void setUp() throws Exception { + super.setUp(); + data = Bytes.toBytes("data"); + row1 = Bytes.toBytes("row1"); + fam1 = Bytes.toBytes("fam1"); + col1 = Bytes.toBytes("col1"); + row2 = Bytes.toBytes("row2"); + fam2 = Bytes.toBytes("fam2"); + col2 = Bytes.toBytes("col2"); + col3 = Bytes.toBytes("col3"); + col4 = Bytes.toBytes("col4"); + col5 = Bytes.toBytes("col5"); + } + + public void testSorted() throws IOException{ + //Cases that need to be checked are: + //1. The "smallest" KeyValue is in the same scanners as current + //2. Current scanner gets empty + + List l1 = new ArrayList(); + l1.add(new KeyValue(row1, fam1, col5, data)); + l1.add(new KeyValue(row2, fam1, col1, data)); + l1.add(new KeyValue(row2, fam1, col2, data)); + scanners.add(new Scanner(l1)); + + List l2 = new ArrayList(); + l2.add(new KeyValue(row1, fam1, col1, data)); + l2.add(new KeyValue(row1, fam1, col2, data)); + scanners.add(new Scanner(l2)); + + List l3 = new ArrayList(); + l3.add(new KeyValue(row1, fam1, col3, data)); + l3.add(new KeyValue(row1, fam1, col4, data)); + l3.add(new KeyValue(row1, fam2, col1, data)); + l3.add(new KeyValue(row1, fam2, col2, data)); + l3.add(new KeyValue(row2, fam1, col3, data)); + scanners.add(new Scanner(l3)); + + List expected = new ArrayList(); + expected.add(new KeyValue(row1, fam1, col1, data)); + expected.add(new KeyValue(row1, fam1, col2, data)); + expected.add(new KeyValue(row1, fam1, col3, data)); + expected.add(new KeyValue(row1, fam1, col4, data)); + expected.add(new KeyValue(row1, fam1, col5, data)); + expected.add(new KeyValue(row1, fam2, col1, data)); + expected.add(new KeyValue(row1, fam2, col2, data)); + expected.add(new KeyValue(row2, fam1, col1, data)); + expected.add(new KeyValue(row2, fam1, col2, data)); + expected.add(new KeyValue(row2, fam1, col3, data)); + + //Creating KeyValueHeap + KeyValueHeap kvh = + new KeyValueHeap(scanners, KeyValue.COMPARATOR); + + List actual = new ArrayList(); + while(kvh.peek() != null){ + actual.add(kvh.next()); + } + + assertEquals(expected.size(), actual.size()); + for(int i=0; i l1 = new ArrayList(); + l1.add(new KeyValue(row1, fam1, col5, data)); + l1.add(new KeyValue(row2, fam1, col1, data)); + l1.add(new KeyValue(row2, fam1, col2, data)); + scanners.add(new Scanner(l1)); + + List l2 = new ArrayList(); + l2.add(new KeyValue(row1, fam1, col1, data)); + l2.add(new KeyValue(row1, fam1, col2, data)); + scanners.add(new Scanner(l2)); + + List l3 = new ArrayList(); + l3.add(new KeyValue(row1, fam1, col3, data)); + l3.add(new KeyValue(row1, fam1, col4, data)); + l3.add(new KeyValue(row1, fam2, col1, data)); + l3.add(new KeyValue(row1, fam2, col2, data)); + l3.add(new KeyValue(row2, fam1, col3, data)); + scanners.add(new Scanner(l3)); + + List expected = new ArrayList(); + expected.add(new KeyValue(row2, fam1, col1, data)); + + //Creating KeyValueHeap + KeyValueHeap kvh = + new KeyValueHeap(scanners, KeyValue.COMPARATOR); + + KeyValue seekKv = new KeyValue(row2, fam1, null, null); + kvh.seek(seekKv); + + List actual = new ArrayList(); + actual.add(kvh.peek()); + + assertEquals(expected.size(), actual.size()); + for(int i=0; i l1 = new ArrayList(); + l1.add(new KeyValue(row1, fam1, col5, data)); + l1.add(new KeyValue(row2, fam1, col1, data)); + l1.add(new KeyValue(row2, fam1, col2, data)); + scanners.add(new Scanner(l1)); + + List l2 = new ArrayList(); + l2.add(new KeyValue(row1, fam1, col1, data)); + l2.add(new KeyValue(row1, fam1, col2, data)); + scanners.add(new Scanner(l2)); + + List l3 = new ArrayList(); + l3.add(new KeyValue(row1, fam1, col3, data)); + l3.add(new KeyValue(row1, fam1, col4, data)); + l3.add(new KeyValue(row1, fam2, col1, data)); + l3.add(new KeyValue(row1, fam2, col2, data)); + l3.add(new KeyValue(row2, fam1, col3, data)); + scanners.add(new Scanner(l3)); + + List l4 = new ArrayList(); + scanners.add(new Scanner(l4)); + + //Creating KeyValueHeap + KeyValueHeap kvh = new KeyValueHeap(scanners, KeyValue.COMPARATOR); + + while(kvh.next() != null); + + for(KeyValueScanner scanner : scanners) { + assertTrue(((Scanner)scanner).isClosed()); + } + } + + private static class Scanner extends CollectionBackedScanner { + private Iterator iter; + private KeyValue current; + private boolean closed = false; + + public Scanner(List list) { + super(list); + } + + public void close(){ + closed = true; + } + + public boolean isClosed() { + return closed; + } + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java new file mode 100644 index 0000000..1cc893a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java @@ -0,0 +1,74 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestKeyValueScanFixture extends TestCase { + + + public void testKeyValueScanFixture() throws IOException { + KeyValue kvs[] = new KeyValue[]{ + KeyValueTestUtil.create("RowA", "family", "qf1", + 1, KeyValue.Type.Put, "value-1"), + KeyValueTestUtil.create("RowA", "family", "qf2", + 1, KeyValue.Type.Put, "value-2"), + KeyValueTestUtil.create("RowB", "family", "qf1", + 10, KeyValue.Type.Put, "value-10") + }; + KeyValueScanner scan = new KeyValueScanFixture( + KeyValue.COMPARATOR, kvs); + + KeyValue kv = KeyValue.createFirstOnRow(Bytes.toBytes("RowA")); + // should seek to this: + assertTrue(scan.seek(kv)); + KeyValue res = scan.peek(); + assertEquals(kvs[0], res); + + kv = KeyValue.createFirstOnRow(Bytes.toBytes("RowB")); + assertTrue(scan.seek(kv)); + res = scan.peek(); + assertEquals(kvs[2], res); + + // ensure we pull things out properly: + kv = KeyValue.createFirstOnRow(Bytes.toBytes("RowA")); + assertTrue(scan.seek(kv)); + assertEquals(kvs[0], scan.peek()); + assertEquals(kvs[0], scan.next()); + assertEquals(kvs[1], scan.peek()); + assertEquals(kvs[1], scan.next()); + assertEquals(kvs[2], scan.peek()); + assertEquals(kvs[2], scan.next()); + assertEquals(null, scan.peek()); + assertEquals(null, scan.next()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueSkipListSet.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueSkipListSet.java new file mode 100644 index 0000000..d9158be --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueSkipListSet.java @@ -0,0 +1,155 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.util.Iterator; +import java.util.SortedSet; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestKeyValueSkipListSet extends TestCase { + private final KeyValueSkipListSet kvsls = + new KeyValueSkipListSet(KeyValue.COMPARATOR); + + protected void setUp() throws Exception { + super.setUp(); + this.kvsls.clear(); + } + + public void testAdd() throws Exception { + byte [] bytes = Bytes.toBytes(getName()); + KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); + this.kvsls.add(kv); + assertTrue(this.kvsls.contains(kv)); + assertEquals(1, this.kvsls.size()); + KeyValue first = this.kvsls.first(); + assertTrue(kv.equals(first)); + assertTrue(Bytes.equals(kv.getValue(), first.getValue())); + // Now try overwritting + byte [] overwriteValue = Bytes.toBytes("overwrite"); + KeyValue overwrite = new KeyValue(bytes, bytes, bytes, overwriteValue); + this.kvsls.add(overwrite); + assertEquals(1, this.kvsls.size()); + first = this.kvsls.first(); + assertTrue(Bytes.equals(overwrite.getValue(), first.getValue())); + assertFalse(Bytes.equals(overwrite.getValue(), kv.getValue())); + } + + public void testIterator() throws Exception { + byte [] bytes = Bytes.toBytes(getName()); + byte [] value1 = Bytes.toBytes("1"); + byte [] value2 = Bytes.toBytes("2"); + final int total = 3; + for (int i = 0; i < total; i++) { + this.kvsls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value1)); + } + // Assert that we added 'total' values and that they are in order + int count = 0; + for (KeyValue kv: this.kvsls) { + assertEquals("" + count, Bytes.toString(kv.getQualifier())); + assertTrue(Bytes.equals(kv.getValue(), value1)); + count++; + } + assertEquals(total, count); + // Now overwrite with a new value. + for (int i = 0; i < total; i++) { + this.kvsls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value2)); + } + // Assert that we added 'total' values and that they are in order and that + // we are getting back value2 + count = 0; + for (KeyValue kv: this.kvsls) { + assertEquals("" + count, Bytes.toString(kv.getQualifier())); + assertTrue(Bytes.equals(kv.getValue(), value2)); + count++; + } + assertEquals(total, count); + } + + public void testDescendingIterator() throws Exception { + byte [] bytes = Bytes.toBytes(getName()); + byte [] value1 = Bytes.toBytes("1"); + byte [] value2 = Bytes.toBytes("2"); + final int total = 3; + for (int i = 0; i < total; i++) { + this.kvsls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value1)); + } + // Assert that we added 'total' values and that they are in order + int count = 0; + for (Iterator i = this.kvsls.descendingIterator(); i.hasNext();) { + KeyValue kv = i.next(); + assertEquals("" + (total - (count + 1)), Bytes.toString(kv.getQualifier())); + assertTrue(Bytes.equals(kv.getValue(), value1)); + count++; + } + assertEquals(total, count); + // Now overwrite with a new value. + for (int i = 0; i < total; i++) { + this.kvsls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value2)); + } + // Assert that we added 'total' values and that they are in order and that + // we are getting back value2 + count = 0; + for (Iterator i = this.kvsls.descendingIterator(); i.hasNext();) { + KeyValue kv = i.next(); + assertEquals("" + (total - (count + 1)), Bytes.toString(kv.getQualifier())); + assertTrue(Bytes.equals(kv.getValue(), value2)); + count++; + } + assertEquals(total, count); + } + + public void testHeadTail() throws Exception { + byte [] bytes = Bytes.toBytes(getName()); + byte [] value1 = Bytes.toBytes("1"); + byte [] value2 = Bytes.toBytes("2"); + final int total = 3; + KeyValue splitter = null; + for (int i = 0; i < total; i++) { + KeyValue kv = new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value1); + if (i == 1) splitter = kv; + this.kvsls.add(kv); + } + SortedSet tail = this.kvsls.tailSet(splitter); + assertEquals(2, tail.size()); + SortedSet head = this.kvsls.headSet(splitter); + assertEquals(1, head.size()); + // Now ensure that we get back right answer even when we do tail or head. + // Now overwrite with a new value. + for (int i = 0; i < total; i++) { + this.kvsls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value2)); + } + tail = this.kvsls.tailSet(splitter); + assertTrue(Bytes.equals(tail.first().getValue(), value2)); + head = this.kvsls.headSet(splitter); + assertTrue(Bytes.equals(head.first().getValue(), value2)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMXBean.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMXBean.java new file mode 100644 index 0000000..cfa7905 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMXBean.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import junit.framework.Assert; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.MediumTests; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestMXBean { + + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + @BeforeClass + public static void setup() throws Exception { + TEST_UTIL.startMiniCluster(1, 1); + } + + @AfterClass + public static void teardown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testInfo() { + HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); + HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); + MXBeanImpl info = MXBeanImpl.init(rs); + + Assert.assertEquals(rs.getServerName().getServerName(), + info.getServerName()); + Assert.assertEquals(rs.getCoprocessors().length, + info.getCoprocessors().length); + rs.getConfiguration().setInt("hbase.master.info.port", + master.getServerName().getPort()); + Assert.assertEquals(rs.getZooKeeperWatcher().getQuorum(), + info.getZookeeperQuorum()); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java new file mode 100644 index 0000000..e91d83c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java @@ -0,0 +1,121 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.Semaphore; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestMasterAddressManager { + private static final Log LOG = LogFactory.getLog(TestMasterAddressManager.class); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + /** + * Unit tests that uses ZooKeeper but does not use the master-side methods + * but rather acts directly on ZK. + * @throws Exception + */ + @Test + public void testMasterAddressManagerFromZK() throws Exception { + + ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + "testMasterAddressManagerFromZK", null); + ZKUtil.createAndFailSilent(zk, zk.baseZNode); + + // Should not have a master yet + MasterAddressTracker addressManager = new MasterAddressTracker(zk, null); + addressManager.start(); + assertFalse(addressManager.hasMaster()); + zk.registerListener(addressManager); + + // Use a listener to capture when the node is actually created + NodeCreationListener listener = new NodeCreationListener(zk, zk.masterAddressZNode); + zk.registerListener(listener); + + // Create the master node with a dummy address + String host = "localhost"; + int port = 1234; + ServerName sn = new ServerName(host, port, System.currentTimeMillis()); + LOG.info("Creating master node"); + ZKUtil.createEphemeralNodeAndWatch(zk, zk.masterAddressZNode, sn.getVersionedBytes()); + + // Wait for the node to be created + LOG.info("Waiting for master address manager to be notified"); + listener.waitForCreation(); + LOG.info("Master node created"); + assertTrue(addressManager.hasMaster()); + ServerName pulledAddress = addressManager.getMasterAddress(); + assertTrue(pulledAddress.equals(sn)); + + } + + public static class NodeCreationListener extends ZooKeeperListener { + private static final Log LOG = LogFactory.getLog(NodeCreationListener.class); + + private Semaphore lock; + private String node; + + public NodeCreationListener(ZooKeeperWatcher watcher, String node) { + super(watcher); + lock = new Semaphore(0); + this.node = node; + } + + @Override + public void nodeCreated(String path) { + if(path.equals(node)) { + LOG.debug("nodeCreated(" + path + ")"); + lock.release(); + } + } + + public void waitForCreation() throws InterruptedException { + lock.acquire(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java new file mode 100644 index 0000000..ba89b7c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java @@ -0,0 +1,1093 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.rmi.UnexpectedException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import junit.framework.TestCase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.Store.ScanInfo; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; + +import com.google.common.base.Joiner; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import org.junit.experimental.categories.Category; + +/** memstore test case */ +@Category(SmallTests.class) +public class TestMemStore extends TestCase { + private final Log LOG = LogFactory.getLog(this.getClass()); + private MemStore memstore; + private static final int ROW_COUNT = 10; + private static final int QUALIFIER_COUNT = ROW_COUNT; + private static final byte [] FAMILY = Bytes.toBytes("column"); + private static final byte [] CONTENTS = Bytes.toBytes("contents"); + private static final byte [] BASIC = Bytes.toBytes("basic"); + private static final String CONTENTSTR = "contentstr"; + private MultiVersionConsistencyControl mvcc; + + @Override + public void setUp() throws Exception { + super.setUp(); + this.mvcc = new MultiVersionConsistencyControl(); + this.memstore = new MemStore(); + SchemaMetrics.setUseTableNameInTest(false); + } + + public void testPutSameKey() { + byte [] bytes = Bytes.toBytes(getName()); + KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); + this.memstore.add(kv); + byte [] other = Bytes.toBytes("somethingelse"); + KeyValue samekey = new KeyValue(bytes, bytes, bytes, other); + this.memstore.add(samekey); + KeyValue found = this.memstore.kvset.first(); + assertEquals(1, this.memstore.kvset.size()); + assertTrue(Bytes.toString(found.getValue()), Bytes.equals(samekey.getValue(), + found.getValue())); + } + + /** + * Test memstore snapshot happening while scanning. + * @throws IOException + */ + public void testScanAcrossSnapshot() throws IOException { + int rowCount = addRows(this.memstore); + List memstorescanners = this.memstore.getScanners(); + Scan scan = new Scan(); + List result = new ArrayList(); + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + ScanInfo scanInfo = new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, false, + 0, this.memstore.comparator); + ScanType scanType = ScanType.USER_SCAN; + StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners); + int count = 0; + try { + while (s.next(result)) { + LOG.info(result); + count++; + // Row count is same as column count. + assertEquals(rowCount, result.size()); + result.clear(); + } + } finally { + s.close(); + } + assertEquals(rowCount, count); + for (KeyValueScanner scanner : memstorescanners) { + scanner.close(); + } + + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + memstorescanners = this.memstore.getScanners(); + // Now assert can count same number even if a snapshot mid-scan. + s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners); + count = 0; + try { + while (s.next(result)) { + LOG.info(result); + // Assert the stuff is coming out in right order. + assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0); + count++; + // Row count is same as column count. + assertEquals(rowCount, result.size()); + if (count == 2) { + this.memstore.snapshot(); + LOG.info("Snapshotted"); + } + result.clear(); + } + } finally { + s.close(); + } + assertEquals(rowCount, count); + for (KeyValueScanner scanner : memstorescanners) { + scanner.close(); + } + memstorescanners = this.memstore.getScanners(); + // Assert that new values are seen in kvset as we scan. + long ts = System.currentTimeMillis(); + s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners); + count = 0; + int snapshotIndex = 5; + try { + while (s.next(result)) { + LOG.info(result); + // Assert the stuff is coming out in right order. + assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0); + // Row count is same as column count. + assertEquals("count=" + count + ", result=" + result, rowCount, result.size()); + count++; + if (count == snapshotIndex) { + this.memstore.snapshot(); + this.memstore.clearSnapshot(this.memstore.getSnapshot()); + // Added more rows into kvset. But the scanner wont see these rows. + addRows(this.memstore, ts); + LOG.info("Snapshotted, cleared it and then added values (which wont be seen)"); + } + result.clear(); + } + } finally { + s.close(); + } + assertEquals(rowCount, count); + } + + /** + * A simple test which verifies the 3 possible states when scanning across snapshot. + * @throws IOException + */ + public void testScanAcrossSnapshot2() throws IOException { + // we are going to the scanning across snapshot with two kvs + // kv1 should always be returned before kv2 + final byte[] one = Bytes.toBytes(1); + final byte[] two = Bytes.toBytes(2); + final byte[] f = Bytes.toBytes("f"); + final byte[] q = Bytes.toBytes("q"); + final byte[] v = Bytes.toBytes(3); + + final KeyValue kv1 = new KeyValue(one, f, q, v); + final KeyValue kv2 = new KeyValue(two, f, q, v); + + // use case 1: both kvs in kvset + this.memstore.add(kv1.clone()); + this.memstore.add(kv2.clone()); + verifyScanAcrossSnapshot2(kv1, kv2); + + // use case 2: both kvs in snapshot + this.memstore.snapshot(); + verifyScanAcrossSnapshot2(kv1, kv2); + + // use case 3: first in snapshot second in kvset + this.memstore = new MemStore(); + this.memstore.add(kv1.clone()); + this.memstore.snapshot(); + this.memstore.add(kv2.clone()); + verifyScanAcrossSnapshot2(kv1, kv2); + } + + private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2) + throws IOException { + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + List memstorescanners = this.memstore.getScanners(); + assertEquals(1, memstorescanners.size()); + final KeyValueScanner scanner = memstorescanners.get(0); + scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW)); + assertEquals(kv1, scanner.next()); + assertEquals(kv2, scanner.next()); + assertNull(scanner.next()); + } + + private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected) + throws IOException { + scanner.seek(KeyValue.createFirstOnRow(new byte[]{})); + List returned = Lists.newArrayList(); + + while (true) { + KeyValue next = scanner.next(); + if (next == null) break; + returned.add(next); + } + + assertTrue( + "Got:\n" + Joiner.on("\n").join(returned) + + "\nExpected:\n" + Joiner.on("\n").join(expected), + Iterables.elementsEqual(Arrays.asList(expected), returned)); + assertNull(scanner.peek()); + } + + public void testMemstoreConcurrentControl() throws IOException { + final byte[] row = Bytes.toBytes(1); + final byte[] f = Bytes.toBytes("family"); + final byte[] q1 = Bytes.toBytes("q1"); + final byte[] q2 = Bytes.toBytes("q2"); + final byte[] v = Bytes.toBytes("value"); + + MultiVersionConsistencyControl.WriteEntry w = + mvcc.beginMemstoreInsert(); + + KeyValue kv1 = new KeyValue(row, f, q1, v); + kv1.setMemstoreTS(w.getWriteNumber()); + memstore.add(kv1); + + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + KeyValueScanner s = this.memstore.getScanners().get(0); + assertScannerResults(s, new KeyValue[]{}); + + mvcc.completeMemstoreInsert(w); + + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + s = this.memstore.getScanners().get(0); + assertScannerResults(s, new KeyValue[]{kv1}); + + w = mvcc.beginMemstoreInsert(); + KeyValue kv2 = new KeyValue(row, f, q2, v); + kv2.setMemstoreTS(w.getWriteNumber()); + memstore.add(kv2); + + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + s = this.memstore.getScanners().get(0); + assertScannerResults(s, new KeyValue[]{kv1}); + + mvcc.completeMemstoreInsert(w); + + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + s = this.memstore.getScanners().get(0); + assertScannerResults(s, new KeyValue[]{kv1, kv2}); + } + + /** + * Regression test for HBASE-2616, HBASE-2670. + * When we insert a higher-memstoreTS version of a cell but with + * the same timestamp, we still need to provide consistent reads + * for the same scanner. + */ + public void testMemstoreEditsVisibilityWithSameKey() throws IOException { + final byte[] row = Bytes.toBytes(1); + final byte[] f = Bytes.toBytes("family"); + final byte[] q1 = Bytes.toBytes("q1"); + final byte[] q2 = Bytes.toBytes("q2"); + final byte[] v1 = Bytes.toBytes("value1"); + final byte[] v2 = Bytes.toBytes("value2"); + + // INSERT 1: Write both columns val1 + MultiVersionConsistencyControl.WriteEntry w = + mvcc.beginMemstoreInsert(); + + KeyValue kv11 = new KeyValue(row, f, q1, v1); + kv11.setMemstoreTS(w.getWriteNumber()); + memstore.add(kv11); + + KeyValue kv12 = new KeyValue(row, f, q2, v1); + kv12.setMemstoreTS(w.getWriteNumber()); + memstore.add(kv12); + mvcc.completeMemstoreInsert(w); + + // BEFORE STARTING INSERT 2, SEE FIRST KVS + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + KeyValueScanner s = this.memstore.getScanners().get(0); + assertScannerResults(s, new KeyValue[]{kv11, kv12}); + + // START INSERT 2: Write both columns val2 + w = mvcc.beginMemstoreInsert(); + KeyValue kv21 = new KeyValue(row, f, q1, v2); + kv21.setMemstoreTS(w.getWriteNumber()); + memstore.add(kv21); + + KeyValue kv22 = new KeyValue(row, f, q2, v2); + kv22.setMemstoreTS(w.getWriteNumber()); + memstore.add(kv22); + + // BEFORE COMPLETING INSERT 2, SEE FIRST KVS + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + s = this.memstore.getScanners().get(0); + assertScannerResults(s, new KeyValue[]{kv11, kv12}); + + // COMPLETE INSERT 2 + mvcc.completeMemstoreInsert(w); + + // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS. + // See HBASE-1485 for discussion about what we should do with + // the duplicate-TS inserts + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + s = this.memstore.getScanners().get(0); + assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12}); + } + + /** + * When we insert a higher-memstoreTS deletion of a cell but with + * the same timestamp, we still need to provide consistent reads + * for the same scanner. + */ + public void testMemstoreDeletesVisibilityWithSameKey() throws IOException { + final byte[] row = Bytes.toBytes(1); + final byte[] f = Bytes.toBytes("family"); + final byte[] q1 = Bytes.toBytes("q1"); + final byte[] q2 = Bytes.toBytes("q2"); + final byte[] v1 = Bytes.toBytes("value1"); + // INSERT 1: Write both columns val1 + MultiVersionConsistencyControl.WriteEntry w = + mvcc.beginMemstoreInsert(); + + KeyValue kv11 = new KeyValue(row, f, q1, v1); + kv11.setMemstoreTS(w.getWriteNumber()); + memstore.add(kv11); + + KeyValue kv12 = new KeyValue(row, f, q2, v1); + kv12.setMemstoreTS(w.getWriteNumber()); + memstore.add(kv12); + mvcc.completeMemstoreInsert(w); + + // BEFORE STARTING INSERT 2, SEE FIRST KVS + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + KeyValueScanner s = this.memstore.getScanners().get(0); + assertScannerResults(s, new KeyValue[]{kv11, kv12}); + + // START DELETE: Insert delete for one of the columns + w = mvcc.beginMemstoreInsert(); + KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(), + KeyValue.Type.DeleteColumn); + kvDel.setMemstoreTS(w.getWriteNumber()); + memstore.add(kvDel); + + // BEFORE COMPLETING DELETE, SEE FIRST KVS + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + s = this.memstore.getScanners().get(0); + assertScannerResults(s, new KeyValue[]{kv11, kv12}); + + // COMPLETE DELETE + mvcc.completeMemstoreInsert(w); + + // NOW WE SHOULD SEE DELETE + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + s = this.memstore.getScanners().get(0); + assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12}); + } + + + private static class ReadOwnWritesTester extends Thread { + static final int NUM_TRIES = 1000; + + final byte[] row; + + final byte[] f = Bytes.toBytes("family"); + final byte[] q1 = Bytes.toBytes("q1"); + + final MultiVersionConsistencyControl mvcc; + final MemStore memstore; + + AtomicReference caughtException; + + + public ReadOwnWritesTester(int id, + MemStore memstore, + MultiVersionConsistencyControl mvcc, + AtomicReference caughtException) + { + this.mvcc = mvcc; + this.memstore = memstore; + this.caughtException = caughtException; + row = Bytes.toBytes(id); + } + + public void run() { + try { + internalRun(); + } catch (Throwable t) { + caughtException.compareAndSet(null, t); + } + } + + private void internalRun() throws IOException { + for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) { + MultiVersionConsistencyControl.WriteEntry w = + mvcc.beginMemstoreInsert(); + + // Insert the sequence value (i) + byte[] v = Bytes.toBytes(i); + + KeyValue kv = new KeyValue(row, f, q1, i, v); + kv.setMemstoreTS(w.getWriteNumber()); + memstore.add(kv); + mvcc.completeMemstoreInsert(w); + + // Assert that we can read back + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + + KeyValueScanner s = this.memstore.getScanners().get(0); + s.seek(kv); + + KeyValue ret = s.next(); + assertNotNull("Didnt find own write at all", ret); + assertEquals("Didnt read own writes", + kv.getTimestamp(), ret.getTimestamp()); + } + } + } + + public void testReadOwnWritesUnderConcurrency() throws Throwable { + + int NUM_THREADS = 8; + + ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS]; + AtomicReference caught = new AtomicReference(); + + for (int i = 0; i < NUM_THREADS; i++) { + threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught); + threads[i].start(); + } + + for (int i = 0; i < NUM_THREADS; i++) { + threads[i].join(); + } + + if (caught.get() != null) { + throw caught.get(); + } + } + + /** + * Test memstore snapshots + * @throws IOException + */ + public void testSnapshotting() throws IOException { + final int snapshotCount = 5; + // Add some rows, run a snapshot. Do it a few times. + for (int i = 0; i < snapshotCount; i++) { + addRows(this.memstore); + runSnapshot(this.memstore); + KeyValueSkipListSet ss = this.memstore.getSnapshot(); + assertEquals("History not being cleared", 0, ss.size()); + } + } + + public void testMultipleVersionsSimple() throws Exception { + MemStore m = new MemStore(new Configuration(), KeyValue.COMPARATOR); + byte [] row = Bytes.toBytes("testRow"); + byte [] family = Bytes.toBytes("testFamily"); + byte [] qf = Bytes.toBytes("testQualifier"); + long [] stamps = {1,2,3}; + byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"), + Bytes.toBytes("value2")}; + KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]); + KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]); + KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]); + + m.add(key0); + m.add(key1); + m.add(key2); + + assertTrue("Expected memstore to hold 3 values, actually has " + + m.kvset.size(), m.kvset.size() == 3); + } + + public void testBinary() throws IOException { + MemStore mc = new MemStore(new Configuration(), KeyValue.ROOT_COMPARATOR); + final int start = 43; + final int end = 46; + for (int k = start; k <= end; k++) { + byte [] kk = Bytes.toBytes(k); + byte [] row = + Bytes.toBytes(".META.,table," + Bytes.toString(kk) + ",1," + k); + KeyValue key = new KeyValue(row, CONTENTS, BASIC, + System.currentTimeMillis(), + (CONTENTSTR + k).getBytes(HConstants.UTF8_ENCODING)); + mc.add(key); + System.out.println(key); +// key = new KeyValue(row, Bytes.toBytes(ANCHORNUM + k), +// System.currentTimeMillis(), +// (ANCHORSTR + k).getBytes(HConstants.UTF8_ENCODING)); +// mc.add(key); +// System.out.println(key); + } + int index = start; + for (KeyValue kv: mc.kvset) { + System.out.println(kv); + byte [] b = kv.getRow(); + // Hardcoded offsets into String + String str = Bytes.toString(b, 13, 4); + byte [] bb = Bytes.toBytes(index); + String bbStr = Bytes.toString(bb); + assertEquals(str, bbStr); + index++; + } + } + + ////////////////////////////////////////////////////////////////////////////// + // Get tests + ////////////////////////////////////////////////////////////////////////////// + + /** Test getNextRow from memstore + * @throws InterruptedException + */ + public void testGetNextRow() throws Exception { + MultiVersionConsistencyControl.resetThreadReadPoint(); + addRows(this.memstore); + // Add more versions to make it a little more interesting. + Thread.sleep(1); + addRows(this.memstore); + KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY); + assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty, + new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0); + for (int i = 0; i < ROW_COUNT; i++) { + KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i), + System.currentTimeMillis())); + if (i + 1 == ROW_COUNT) { + assertEquals(nr, null); + } else { + assertTrue(KeyValue.COMPARATOR.compareRows(nr, + new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0); + } + } + //starting from each row, validate results should contain the starting row + for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) { + ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, false, + 0, this.memstore.comparator); + ScanType scanType = ScanType.USER_SCAN; + InternalScanner scanner = new StoreScanner(new Scan( + Bytes.toBytes(startRowId)), scanInfo, scanType, null, + memstore.getScanners()); + List results = new ArrayList(); + for (int i = 0; scanner.next(results); i++) { + int rowId = startRowId + i; + assertTrue("Row name", + KeyValue.COMPARATOR.compareRows(results.get(0), + Bytes.toBytes(rowId)) == 0); + assertEquals("Count of columns", QUALIFIER_COUNT, results.size()); + List row = new ArrayList(); + for (KeyValue kv : results) { + row.add(kv); + } + isExpectedRowWithoutTimestamps(rowId, row); + // Clear out set. Otherwise row results accumulate. + results.clear(); + } + } + } + + public void testGet_memstoreAndSnapShot() throws IOException { + byte [] row = Bytes.toBytes("testrow"); + byte [] fam = Bytes.toBytes("testfamily"); + byte [] qf1 = Bytes.toBytes("testqualifier1"); + byte [] qf2 = Bytes.toBytes("testqualifier2"); + byte [] qf3 = Bytes.toBytes("testqualifier3"); + byte [] qf4 = Bytes.toBytes("testqualifier4"); + byte [] qf5 = Bytes.toBytes("testqualifier5"); + byte [] val = Bytes.toBytes("testval"); + + //Setting up memstore + memstore.add(new KeyValue(row, fam ,qf1, val)); + memstore.add(new KeyValue(row, fam ,qf2, val)); + memstore.add(new KeyValue(row, fam ,qf3, val)); + //Creating a snapshot + memstore.snapshot(); + assertEquals(3, memstore.snapshot.size()); + //Adding value to "new" memstore + assertEquals(0, memstore.kvset.size()); + memstore.add(new KeyValue(row, fam ,qf4, val)); + memstore.add(new KeyValue(row, fam ,qf5, val)); + assertEquals(2, memstore.kvset.size()); + } + + ////////////////////////////////////////////////////////////////////////////// + // Delete tests + ////////////////////////////////////////////////////////////////////////////// + public void testGetWithDelete() throws IOException { + byte [] row = Bytes.toBytes("testrow"); + byte [] fam = Bytes.toBytes("testfamily"); + byte [] qf1 = Bytes.toBytes("testqualifier"); + byte [] val = Bytes.toBytes("testval"); + + long ts1 = System.nanoTime(); + KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val); + long ts2 = ts1 + 1; + KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val); + long ts3 = ts2 +1; + KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val); + memstore.add(put1); + memstore.add(put2); + memstore.add(put3); + + assertEquals(3, memstore.kvset.size()); + + KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val); + memstore.delete(del2); + + List expected = new ArrayList(); + expected.add(put3); + expected.add(del2); + expected.add(put2); + expected.add(put1); + + assertEquals(4, memstore.kvset.size()); + int i = 0; + for(KeyValue kv : memstore.kvset) { + assertEquals(expected.get(i++), kv); + } + } + + public void testGetWithDeleteColumn() throws IOException { + byte [] row = Bytes.toBytes("testrow"); + byte [] fam = Bytes.toBytes("testfamily"); + byte [] qf1 = Bytes.toBytes("testqualifier"); + byte [] val = Bytes.toBytes("testval"); + + long ts1 = System.nanoTime(); + KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val); + long ts2 = ts1 + 1; + KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val); + long ts3 = ts2 +1; + KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val); + memstore.add(put1); + memstore.add(put2); + memstore.add(put3); + + assertEquals(3, memstore.kvset.size()); + + KeyValue del2 = + new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val); + memstore.delete(del2); + + List expected = new ArrayList(); + expected.add(put3); + expected.add(del2); + expected.add(put2); + expected.add(put1); + + + assertEquals(4, memstore.kvset.size()); + int i = 0; + for (KeyValue kv: memstore.kvset) { + assertEquals(expected.get(i++), kv); + } + } + + + public void testGetWithDeleteFamily() throws IOException { + byte [] row = Bytes.toBytes("testrow"); + byte [] fam = Bytes.toBytes("testfamily"); + byte [] qf1 = Bytes.toBytes("testqualifier1"); + byte [] qf2 = Bytes.toBytes("testqualifier2"); + byte [] qf3 = Bytes.toBytes("testqualifier3"); + byte [] val = Bytes.toBytes("testval"); + long ts = System.nanoTime(); + + KeyValue put1 = new KeyValue(row, fam, qf1, ts, val); + KeyValue put2 = new KeyValue(row, fam, qf2, ts, val); + KeyValue put3 = new KeyValue(row, fam, qf3, ts, val); + KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val); + + memstore.add(put1); + memstore.add(put2); + memstore.add(put3); + memstore.add(put4); + + KeyValue del = + new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val); + memstore.delete(del); + + List expected = new ArrayList(); + expected.add(del); + expected.add(put1); + expected.add(put2); + expected.add(put4); + expected.add(put3); + + + + assertEquals(5, memstore.kvset.size()); + int i = 0; + for (KeyValue kv: memstore.kvset) { + assertEquals(expected.get(i++), kv); + } + } + + public void testKeepDeleteInmemstore() { + byte [] row = Bytes.toBytes("testrow"); + byte [] fam = Bytes.toBytes("testfamily"); + byte [] qf = Bytes.toBytes("testqualifier"); + byte [] val = Bytes.toBytes("testval"); + long ts = System.nanoTime(); + memstore.add(new KeyValue(row, fam, qf, ts, val)); + KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val); + memstore.delete(delete); + assertEquals(2, memstore.kvset.size()); + assertEquals(delete, memstore.kvset.first()); + } + + public void testRetainsDeleteVersion() throws IOException { + // add a put to memstore + memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care")); + + // now process a specific delete: + KeyValue delete = KeyValueTestUtil.create( + "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care"); + memstore.delete(delete); + + assertEquals(2, memstore.kvset.size()); + assertEquals(delete, memstore.kvset.first()); + } + public void testRetainsDeleteColumn() throws IOException { + // add a put to memstore + memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care")); + + // now process a specific delete: + KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100, + KeyValue.Type.DeleteColumn, "dont-care"); + memstore.delete(delete); + + assertEquals(2, memstore.kvset.size()); + assertEquals(delete, memstore.kvset.first()); + } + public void testRetainsDeleteFamily() throws IOException { + // add a put to memstore + memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care")); + + // now process a specific delete: + KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100, + KeyValue.Type.DeleteFamily, "dont-care"); + memstore.delete(delete); + + assertEquals(2, memstore.kvset.size()); + assertEquals(delete, memstore.kvset.first()); + } + + //////////////////////////////////// + //Test for timestamps + //////////////////////////////////// + + /** + * Test to ensure correctness when using Memstore with multiple timestamps + */ + public void testMultipleTimestamps() throws IOException { + long[] timestamps = new long[] {20,10,5,1}; + Scan scan = new Scan(); + + for (long timestamp: timestamps) + addRows(memstore,timestamp); + + scan.setTimeRange(0, 2); + assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE)); + + scan.setTimeRange(20, 82); + assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE)); + + scan.setTimeRange(10, 20); + assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE)); + + scan.setTimeRange(8, 12); + assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE)); + + /*This test is not required for correctness but it should pass when + * timestamp range optimization is on*/ + //scan.setTimeRange(28, 42); + //assertTrue(!memstore.shouldSeek(scan)); + } + + //////////////////////////////////// + //Test for upsert with MSLAB + //////////////////////////////////// + + /** + * Test a pathological pattern that shows why we can't currently + * use the MSLAB for upsert workloads. This test inserts data + * in the following pattern: + * + * - row0001 through row1000 (fills up one 2M Chunk) + * - row0002 through row1001 (fills up another 2M chunk, leaves one reference + * to the first chunk + * - row0003 through row1002 (another chunk, another dangling reference) + * + * This causes OOME pretty quickly if we use MSLAB for upsert + * since each 2M chunk is held onto by a single reference. + */ + public void testUpsertMSLAB() throws Exception { + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(MemStore.USEMSLAB_KEY, true); + memstore = new MemStore(conf, KeyValue.COMPARATOR); + + int ROW_SIZE = 2048; + byte[] qualifier = new byte[ROW_SIZE - 4]; + + MemoryMXBean bean = ManagementFactory.getMemoryMXBean(); + for (int i = 0; i < 3; i++) { System.gc(); } + long usageBefore = bean.getHeapMemoryUsage().getUsed(); + + long size = 0; + long ts=0; + + for (int newValue = 0; newValue < 1000; newValue++) { + for (int row = newValue; row < newValue + 1000; row++) { + byte[] rowBytes = Bytes.toBytes(row); + size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts); + } + } + System.out.println("Wrote " + ts + " vals"); + for (int i = 0; i < 3; i++) { System.gc(); } + long usageAfter = bean.getHeapMemoryUsage().getUsed(); + System.out.println("Memory used: " + (usageAfter - usageBefore) + + " (heapsize: " + memstore.heapSize() + + " size: " + size + ")"); + } + + ////////////////////////////////////////////////////////////////////////////// + // Helpers + ////////////////////////////////////////////////////////////////////////////// + private static byte [] makeQualifier(final int i1, final int i2){ + return Bytes.toBytes(Integer.toString(i1) + ";" + + Integer.toString(i2)); + } + + /** + * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased + * as older keyvalues are deleted from the memstore. + * @throws Exception + */ + public void testUpsertMemstoreSize() throws Exception { + Configuration conf = HBaseConfiguration.create(); + memstore = new MemStore(conf, KeyValue.COMPARATOR); + long oldSize = memstore.size.get(); + + KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); + this.memstore.upsert(Collections.singletonList(kv1)); + long newSize = this.memstore.size.get(); + assert(newSize > oldSize); + + KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v"); + this.memstore.upsert(Collections.singletonList(kv2)); + assertEquals(newSize, this.memstore.size.get()); + } + + //////////////////////////////////// + // Test for periodic memstore flushes + // based on time of oldest edit + //////////////////////////////////// + + /** + * Tests that the timeOfOldestEdit is updated correctly for the + * various edit operations in memstore. + * @throws Exception + */ + public void testUpdateToTimeOfOldestEdit() throws Exception { + try { + EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest(); + EnvironmentEdgeManager.injectEdge(edge); + MemStore memstore = new MemStore(); + long t = memstore.timeOfOldestEdit(); + assertEquals(t, Long.MAX_VALUE); + + // test the case that the timeOfOldestEdit is updated after a KV add + memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v")); + t = memstore.timeOfOldestEdit(); + assertTrue(t == 1234); + // snapshot() will reset timeOfOldestEdit. The method will also assert the + // value is reset to Long.MAX_VALUE + t = runSnapshot(memstore); + + // test the case that the timeOfOldestEdit is updated after a KV delete + memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v")); + t = memstore.timeOfOldestEdit(); + assertTrue(t == 1234); + t = runSnapshot(memstore); + + // test the case that the timeOfOldestEdit is updated after a KV upsert + List l = new ArrayList(); + KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); + l.add(kv1); + memstore.upsert(l); + t = memstore.timeOfOldestEdit(); + assertTrue(t == 1234); + } finally { + EnvironmentEdgeManager.reset(); + } + } + + /** + * Tests the HRegion.shouldFlush method - adds an edit in the memstore + * and checks that shouldFlush returns true, and another where it disables + * the periodic flush functionality and tests whether shouldFlush returns + * false. + * @throws Exception + */ + public void testShouldFlush() throws Exception { + Configuration conf = new Configuration(); + conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000); + checkShouldFlush(conf, true); + // test disable flush + conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0); + checkShouldFlush(conf, false); + } + + private void checkShouldFlush(Configuration conf, boolean expected) throws Exception { + try { + EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest(); + EnvironmentEdgeManager.injectEdge(edge); + HBaseTestingUtility hbaseUtility = new HBaseTestingUtility(conf); + HRegion region = hbaseUtility.createTestRegion("foobar", new HColumnDescriptor("foo")); + + Map stores = region.getStores(); + assertTrue(stores.size() == 1); + + Store s = stores.entrySet().iterator().next().getValue(); + edge.setCurrentTimeMillis(1234); + s.add(KeyValueTestUtil.create("r", "f", "q", 100, "v")); + edge.setCurrentTimeMillis(1234 + 100); + assertTrue(region.shouldFlush() == false); + edge.setCurrentTimeMillis(1234 + 10000); + assertTrue(region.shouldFlush() == expected); + } finally { + EnvironmentEdgeManager.reset(); + } + } + + private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge { + long t = 1234; + @Override + public long currentTimeMillis() { + return t; + } + public void setCurrentTimeMillis(long t) { + this.t = t; + } + } + + /** * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT} + * @param hmc Instance to add rows to. + * @return How many rows we added. + * @throws IOException + */ + private int addRows(final MemStore hmc) { + return addRows(hmc, HConstants.LATEST_TIMESTAMP); + } + + /** + * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT} + * @param hmc Instance to add rows to. + * @return How many rows we added. + * @throws IOException + */ + private int addRows(final MemStore hmc, final long ts) { + for (int i = 0; i < ROW_COUNT; i++) { + long timestamp = ts == HConstants.LATEST_TIMESTAMP? + System.currentTimeMillis(): ts; + for (int ii = 0; ii < QUALIFIER_COUNT; ii++) { + byte [] row = Bytes.toBytes(i); + byte [] qf = makeQualifier(i, ii); + hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf)); + } + } + return ROW_COUNT; + } + + private long runSnapshot(final MemStore hmc) throws UnexpectedException { + // Save off old state. + int oldHistorySize = hmc.getSnapshot().size(); + hmc.snapshot(); + KeyValueSkipListSet ss = hmc.getSnapshot(); + // Make some assertions about what just happened. + assertTrue("History size has not increased", oldHistorySize < ss.size()); + long t = memstore.timeOfOldestEdit(); + assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE); + hmc.clearSnapshot(ss); + return t; + } + + private void isExpectedRowWithoutTimestamps(final int rowIndex, + List kvs) { + int i = 0; + for (KeyValue kv: kvs) { + String expectedColname = Bytes.toString(makeQualifier(rowIndex, i++)); + String colnameStr = Bytes.toString(kv.getQualifier()); + assertEquals("Column name", colnameStr, expectedColname); + // Value is column name as bytes. Usually result is + // 100 bytes in size at least. This is the default size + // for BytesWriteable. For comparison, convert bytes to + // String and trim to remove trailing null bytes. + String colvalueStr = Bytes.toString(kv.getBuffer(), kv.getValueOffset(), + kv.getValueLength()); + assertEquals("Content", colnameStr, colvalueStr); + } + } + + private KeyValue getDeleteKV(byte [] row) { + return new KeyValue(row, Bytes.toBytes("test_col"), null, + HConstants.LATEST_TIMESTAMP, KeyValue.Type.Delete, null); + } + + private KeyValue getKV(byte [] row, byte [] value) { + return new KeyValue(row, Bytes.toBytes("test_col"), null, + HConstants.LATEST_TIMESTAMP, value); + } + private static void addRows(int count, final MemStore mem) { + long nanos = System.nanoTime(); + + for (int i = 0 ; i < count ; i++) { + if (i % 1000 == 0) { + + System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000); + nanos = System.nanoTime(); + } + long timestamp = System.currentTimeMillis(); + + for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) { + byte [] row = Bytes.toBytes(i); + byte [] qf = makeQualifier(i, ii); + mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf)); + } + } + } + + + static void doScan(MemStore ms, int iteration) throws IOException { + long nanos = System.nanoTime(); + KeyValueScanner s = ms.getScanners().get(0); + s.seek(KeyValue.createFirstOnRow(new byte[]{})); + + System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000); + int cnt=0; + while(s.next() != null) ++cnt; + + System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos)/1000 + " for: " + cnt); + + } + + public static void main(String [] args) throws IOException { + MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl(); + MemStore ms = new MemStore(); + + long n1 = System.nanoTime(); + addRows(25000, ms); + System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000); + + + System.out.println("foo"); + + MultiVersionConsistencyControl.resetThreadReadPoint(mvcc); + + for (int i = 0 ; i < 50 ; i++) + doScan(ms, i); + + } + + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java new file mode 100644 index 0000000..d7b01ca --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java @@ -0,0 +1,181 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.*; + +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.MultithreadedTestUtil; +import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.regionserver.MemStoreLAB.Allocation; +import org.junit.Test; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.primitives.Ints; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestMemStoreLAB { + + /** + * Test a bunch of random allocations + */ + @Test + public void testLABRandomAllocation() { + Random rand = new Random(); + MemStoreLAB mslab = new MemStoreLAB(); + int expectedOff = 0; + byte[] lastBuffer = null; + // 100K iterations by 0-1K alloc -> 50MB expected + // should be reasonable for unit test and also cover wraparound + // behavior + for (int i = 0; i < 100000; i++) { + int size = rand.nextInt(1000); + Allocation alloc = mslab.allocateBytes(size); + + if (alloc.getData() != lastBuffer) { + expectedOff = 0; + lastBuffer = alloc.getData(); + } + assertEquals(expectedOff, alloc.getOffset()); + assertTrue("Allocation " + alloc + " overruns buffer", + alloc.getOffset() + size <= alloc.getData().length); + expectedOff += size; + } + } + + @Test + public void testLABLargeAllocation() { + MemStoreLAB mslab = new MemStoreLAB(); + Allocation alloc = mslab.allocateBytes(2*1024*1024); + assertNull("2MB allocation shouldn't be satisfied by LAB.", + alloc); + } + + /** + * Test allocation from lots of threads, making sure the results don't + * overlap in any way + */ + @Test + public void testLABThreading() throws Exception { + Configuration conf = new Configuration(); + MultithreadedTestUtil.TestContext ctx = + new MultithreadedTestUtil.TestContext(conf); + + final AtomicInteger totalAllocated = new AtomicInteger(); + + final MemStoreLAB mslab = new MemStoreLAB(); + List> allocations = Lists.newArrayList(); + + for (int i = 0; i < 10; i++) { + final List allocsByThisThread = Lists.newLinkedList(); + allocations.add(allocsByThisThread); + + TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) { + private Random r = new Random(); + @Override + public void doAnAction() throws Exception { + int size = r.nextInt(1000); + Allocation alloc = mslab.allocateBytes(size); + totalAllocated.addAndGet(size); + allocsByThisThread.add(new AllocRecord(alloc, size)); + } + }; + ctx.addThread(t); + } + + ctx.startThreads(); + while (totalAllocated.get() < 50*1024*1024 && ctx.shouldRun()) { + Thread.sleep(10); + } + ctx.stop(); + + // Partition the allocations by the actual byte[] they point into, + // make sure offsets are unique for each chunk + Map> mapsByChunk = + Maps.newHashMap(); + + int sizeCounted = 0; + for (AllocRecord rec : Iterables.concat(allocations)) { + sizeCounted += rec.size; + if (rec.size == 0) continue; + + Map mapForThisByteArray = + mapsByChunk.get(rec.alloc.getData()); + if (mapForThisByteArray == null) { + mapForThisByteArray = Maps.newTreeMap(); + mapsByChunk.put(rec.alloc.getData(), mapForThisByteArray); + } + AllocRecord oldVal = mapForThisByteArray.put(rec.alloc.getOffset(), rec); + assertNull("Already had an entry " + oldVal + " for allocation " + rec, + oldVal); + } + assertEquals("Sanity check test", sizeCounted, totalAllocated.get()); + + // Now check each byte array to make sure allocations don't overlap + for (Map allocsInChunk : mapsByChunk.values()) { + int expectedOff = 0; + for (AllocRecord alloc : allocsInChunk.values()) { + assertEquals(expectedOff, alloc.alloc.getOffset()); + assertTrue("Allocation " + alloc + " overruns buffer", + alloc.alloc.getOffset() + alloc.size <= alloc.alloc.getData().length); + expectedOff += alloc.size; + } + } + + } + + private static class AllocRecord implements Comparable{ + private final Allocation alloc; + private final int size; + public AllocRecord(Allocation alloc, int size) { + super(); + this.alloc = alloc; + this.size = size; + } + + @Override + public int compareTo(AllocRecord e) { + if (alloc.getData() != e.alloc.getData()) { + throw new RuntimeException("Can only compare within a particular array"); + } + return Ints.compare(alloc.getOffset(), e.alloc.getOffset()); + } + + @Override + public String toString() { + return "AllocRecord(alloc=" + alloc + ", size=" + size + ")"; + } + + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java new file mode 100644 index 0000000..631d770 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java @@ -0,0 +1,462 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.filter.TimestampsFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.experimental.categories.Category; + +/** + * Test Minimum Versions feature (HBASE-4071). + */ +@Category(SmallTests.class) +public class TestMinVersions extends HBaseTestCase { + private final byte[] T0 = Bytes.toBytes("0"); + private final byte[] T1 = Bytes.toBytes("1"); + private final byte[] T2 = Bytes.toBytes("2"); + private final byte[] T3 = Bytes.toBytes("3"); + private final byte[] T4 = Bytes.toBytes("4"); + private final byte[] T5 = Bytes.toBytes("5"); + + private final byte[] c0 = COLUMNS[0]; + + /** + * Verify behavior of getClosestBefore(...) + */ + public void testGetClosestBefore() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName(), 1, 1000, 1, false); + HRegion region = createNewHRegion(htd, null, null); + try { + + // 2s in the past + long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000; + + Put p = new Put(T1, ts); + p.add(c0, c0, T1); + region.put(p); + + p = new Put(T1, ts+1); + p.add(c0, c0, T4); + region.put(p); + + p = new Put(T3, ts); + p.add(c0, c0, T3); + region.put(p); + + // now make sure that getClosestBefore(...) get can + // rows that would be expired without minVersion. + // also make sure it gets the latest version + Result r = region.getClosestRowBefore(T1, c0); + checkResult(r, c0, T4); + + r = region.getClosestRowBefore(T2, c0); + checkResult(r, c0, T4); + + // now flush/compact + region.flushcache(); + region.compactStores(true); + + r = region.getClosestRowBefore(T1, c0); + checkResult(r, c0, T4); + + r = region.getClosestRowBefore(T2, c0); + checkResult(r, c0, T4); + } finally { + region.close(); + region.getLog().closeAndDelete(); + } + } + + /** + * Test mixed memstore and storefile scanning + * with minimum versions. + */ + public void testStoreMemStore() throws Exception { + // keep 3 versions minimum + HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, false); + HRegion region = createNewHRegion(htd, null, null); + // 2s in the past + long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000; + + try { + Put p = new Put(T1, ts-1); + p.add(c0, c0, T2); + region.put(p); + + p = new Put(T1, ts-3); + p.add(c0, c0, T0); + region.put(p); + + // now flush/compact + region.flushcache(); + region.compactStores(true); + + p = new Put(T1, ts); + p.add(c0, c0, T3); + region.put(p); + + p = new Put(T1, ts-2); + p.add(c0, c0, T1); + region.put(p); + + p = new Put(T1, ts-3); + p.add(c0, c0, T0); + region.put(p); + + // newest version in the memstore + // the 2nd oldest in the store file + // and the 3rd, 4th oldest also in the memstore + + Get g = new Get(T1); + g.setMaxVersions(); + Result r = region.get(g, null); // this'll use ScanWildcardColumnTracker + checkResult(r, c0, T3,T2,T1); + + g = new Get(T1); + g.setMaxVersions(); + g.addColumn(c0, c0); + r = region.get(g, null); // this'll use ExplicitColumnTracker + checkResult(r, c0, T3,T2,T1); + } finally { + region.close(); + region.getLog().closeAndDelete(); + } + } + + /** + * Make sure the Deletes behave as expected with minimum versions + */ + public void testDelete() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, false); + HRegion region = createNewHRegion(htd, null, null); + + // 2s in the past + long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000; + + try { + Put p = new Put(T1, ts-2); + p.add(c0, c0, T1); + region.put(p); + + p = new Put(T1, ts-1); + p.add(c0, c0, T2); + region.put(p); + + p = new Put(T1, ts); + p.add(c0, c0, T3); + region.put(p); + + Delete d = new Delete(T1, ts-1, null); + region.delete(d, null, true); + + Get g = new Get(T1); + g.setMaxVersions(); + Result r = region.get(g, null); // this'll use ScanWildcardColumnTracker + checkResult(r, c0, T3); + + g = new Get(T1); + g.setMaxVersions(); + g.addColumn(c0, c0); + r = region.get(g, null); // this'll use ExplicitColumnTracker + checkResult(r, c0, T3); + + // now flush/compact + region.flushcache(); + region.compactStores(true); + + // try again + g = new Get(T1); + g.setMaxVersions(); + r = region.get(g, null); // this'll use ScanWildcardColumnTracker + checkResult(r, c0, T3); + + g = new Get(T1); + g.setMaxVersions(); + g.addColumn(c0, c0); + r = region.get(g, null); // this'll use ExplicitColumnTracker + checkResult(r, c0, T3); + } finally { + region.close(); + region.getLog().closeAndDelete(); + } + } + + /** + * Make sure the memstor behaves correctly with minimum versions + */ + public void testMemStore() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1, false); + HRegion region = createNewHRegion(htd, null, null); + + // 2s in the past + long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000; + + try { + // 2nd version + Put p = new Put(T1, ts-2); + p.add(c0, c0, T2); + region.put(p); + + // 3rd version + p = new Put(T1, ts-1); + p.add(c0, c0, T3); + region.put(p); + + // 4th version + p = new Put(T1, ts); + p.add(c0, c0, T4); + region.put(p); + + // now flush/compact + region.flushcache(); + region.compactStores(true); + + // now put the first version (backdated) + p = new Put(T1, ts-3); + p.add(c0, c0, T1); + region.put(p); + + // now the latest change is in the memstore, + // but it is not the latest version + + Result r = region.get(new Get(T1), null); + checkResult(r, c0, T4); + + Get g = new Get(T1); + g.setMaxVersions(); + r = region.get(g, null); // this'll use ScanWildcardColumnTracker + checkResult(r, c0, T4,T3); + + g = new Get(T1); + g.setMaxVersions(); + g.addColumn(c0, c0); + r = region.get(g, null); // this'll use ExplicitColumnTracker + checkResult(r, c0, T4,T3); + + p = new Put(T1, ts+1); + p.add(c0, c0, T5); + region.put(p); + + // now the latest version is in the memstore + + g = new Get(T1); + g.setMaxVersions(); + r = region.get(g, null); // this'll use ScanWildcardColumnTracker + checkResult(r, c0, T5,T4); + + g = new Get(T1); + g.setMaxVersions(); + g.addColumn(c0, c0); + r = region.get(g, null); // this'll use ExplicitColumnTracker + checkResult(r, c0, T5,T4); + } finally { + region.close(); + region.getLog().closeAndDelete(); + } + } + + /** + * Verify basic minimum versions functionality + */ + public void testBaseCase() throws Exception { + // 1 version minimum, 1000 versions maximum, ttl = 1s + HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1, false); + HRegion region = createNewHRegion(htd, null, null); + try { + + // 2s in the past + long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000; + + // 1st version + Put p = new Put(T1, ts-3); + p.add(c0, c0, T1); + region.put(p); + + // 2nd version + p = new Put(T1, ts-2); + p.add(c0, c0, T2); + region.put(p); + + // 3rd version + p = new Put(T1, ts-1); + p.add(c0, c0, T3); + region.put(p); + + // 4th version + p = new Put(T1, ts); + p.add(c0, c0, T4); + region.put(p); + + Result r = region.get(new Get(T1), null); + checkResult(r, c0, T4); + + Get g = new Get(T1); + g.setTimeRange(0L, ts+1); + r = region.get(g, null); + checkResult(r, c0, T4); + + // oldest version still exists + g.setTimeRange(0L, ts-2); + r = region.get(g, null); + checkResult(r, c0, T1); + + // gets see only available versions + // even before compactions + g = new Get(T1); + g.setMaxVersions(); + r = region.get(g, null); // this'll use ScanWildcardColumnTracker + checkResult(r, c0, T4,T3); + + g = new Get(T1); + g.setMaxVersions(); + g.addColumn(c0, c0); + r = region.get(g, null); // this'll use ExplicitColumnTracker + checkResult(r, c0, T4,T3); + + // now flush + region.flushcache(); + + // with HBASE-4241 a flush will eliminate the expired rows + g = new Get(T1); + g.setTimeRange(0L, ts-2); + r = region.get(g, null); + assertTrue(r.isEmpty()); + + // major compaction + region.compactStores(true); + + // after compaction the 4th version is still available + g = new Get(T1); + g.setTimeRange(0L, ts+1); + r = region.get(g, null); + checkResult(r, c0, T4); + + // so is the 3rd + g.setTimeRange(0L, ts); + r = region.get(g, null); + checkResult(r, c0, T3); + + // but the 2nd and earlier versions are gone + g.setTimeRange(0L, ts-1); + r = region.get(g, null); + assertTrue(r.isEmpty()); + } finally { + region.close(); + region.getLog().closeAndDelete(); + } + } + + /** + * Verify that basic filters still behave correctly with + * minimum versions enabled. + */ + public void testFilters() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1, false); + HRegion region = createNewHRegion(htd, null, null); + final byte [] c1 = COLUMNS[1]; + + // 2s in the past + long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000; + try { + + Put p = new Put(T1, ts-3); + p.add(c0, c0, T0); + p.add(c1, c1, T0); + region.put(p); + + p = new Put(T1, ts-2); + p.add(c0, c0, T1); + p.add(c1, c1, T1); + region.put(p); + + p = new Put(T1, ts-1); + p.add(c0, c0, T2); + p.add(c1, c1, T2); + region.put(p); + + p = new Put(T1, ts); + p.add(c0, c0, T3); + p.add(c1, c1, T3); + region.put(p); + + List tss = new ArrayList(); + tss.add(ts-1); + tss.add(ts-2); + + Get g = new Get(T1); + g.addColumn(c1,c1); + g.setFilter(new TimestampsFilter(tss)); + g.setMaxVersions(); + Result r = region.get(g, null); + checkResult(r, c1, T2,T1); + + g = new Get(T1); + g.addColumn(c0,c0); + g.setFilter(new TimestampsFilter(tss)); + g.setMaxVersions(); + r = region.get(g, null); + checkResult(r, c0, T2,T1); + + // now flush/compact + region.flushcache(); + region.compactStores(true); + + g = new Get(T1); + g.addColumn(c1,c1); + g.setFilter(new TimestampsFilter(tss)); + g.setMaxVersions(); + r = region.get(g, null); + checkResult(r, c1, T2); + + g = new Get(T1); + g.addColumn(c0,c0); + g.setFilter(new TimestampsFilter(tss)); + g.setMaxVersions(); + r = region.get(g, null); + checkResult(r, c0, T2); + } finally { + region.close(); + region.getLog().closeAndDelete(); + } + } + + private void checkResult(Result r, byte[] col, byte[] ... vals) { + assertEquals(r.size(), vals.length); + List kvs = r.getColumn(col, col); + assertEquals(kvs.size(), vals.length); + for (int i=0;i[] operations = new Pair[10]; + OperationStatus[] retCodeDetails = new OperationStatus[10]; + WALEdit[] walEditsFromCoprocessors = new WALEdit[10]; + for (int i = 0; i < 10; i++) { + operations[i] = new Pair(new Put(Bytes.toBytes(i)), null); + } + MiniBatchOperationInProgress> miniBatch = + new MiniBatchOperationInProgress>(operations, retCodeDetails, + walEditsFromCoprocessors, 0, 5); + + assertEquals(5, miniBatch.size()); + assertTrue(Bytes.equals(Bytes.toBytes(0), miniBatch.getOperation(0).getFirst().getRow())); + assertTrue(Bytes.equals(Bytes.toBytes(2), miniBatch.getOperation(2).getFirst().getRow())); + assertTrue(Bytes.equals(Bytes.toBytes(4), miniBatch.getOperation(4).getFirst().getRow())); + try { + miniBatch.getOperation(5); + fail("Should throw Exception while accessing out of range"); + } catch (ArrayIndexOutOfBoundsException e) { + } + miniBatch.setOperationStatus(1, OperationStatus.FAILURE); + assertEquals(OperationStatus.FAILURE, retCodeDetails[1]); + try { + miniBatch.setOperationStatus(6, OperationStatus.FAILURE); + fail("Should throw Exception while accessing out of range"); + } catch (ArrayIndexOutOfBoundsException e) { + } + try { + miniBatch.setWalEdit(5, new WALEdit()); + fail("Should throw Exception while accessing out of range"); + } catch (ArrayIndexOutOfBoundsException e) { + } + + miniBatch = new MiniBatchOperationInProgress>(operations, + retCodeDetails, walEditsFromCoprocessors, 7, 10); + try { + miniBatch.setWalEdit(-1, new WALEdit()); + fail("Should throw Exception while accessing out of range"); + } catch (ArrayIndexOutOfBoundsException e) { + } + try { + miniBatch.getOperation(-1); + fail("Should throw Exception while accessing out of range"); + } catch (ArrayIndexOutOfBoundsException e) { + } + try { + miniBatch.getOperation(3); + fail("Should throw Exception while accessing out of range"); + } catch (ArrayIndexOutOfBoundsException e) { + } + try { + miniBatch.getOperationStatus(9); + fail("Should throw Exception while accessing out of range"); + } catch (ArrayIndexOutOfBoundsException e) { + } + try { + miniBatch.setOperationStatus(3, OperationStatus.FAILURE); + fail("Should throw Exception while accessing out of range"); + } catch (ArrayIndexOutOfBoundsException e) { + } + assertTrue(Bytes.equals(Bytes.toBytes(7), miniBatch.getOperation(0).getFirst().getRow())); + assertTrue(Bytes.equals(Bytes.toBytes(9), miniBatch.getOperation(2).getFirst().getRow())); + miniBatch.setOperationStatus(1, OperationStatus.SUCCESS); + assertEquals(OperationStatus.SUCCESS, retCodeDetails[8]); + WALEdit wal = new WALEdit(); + miniBatch.setWalEdit(0, wal); + assertEquals(wal, walEditsFromCoprocessors[7]); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java new file mode 100644 index 0000000..b716c53 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java @@ -0,0 +1,352 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueTestUtil; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Tests optimized scanning of multiple columns. + */ +@RunWith(Parameterized.class) +@Category(MediumTests.class) +public class TestMultiColumnScanner { + + private static final Log LOG = LogFactory.getLog(TestMultiColumnScanner.class); + + private static final String TABLE_NAME = + TestMultiColumnScanner.class.getSimpleName(); + + static final int MAX_VERSIONS = 50; + + private static final String FAMILY = "CF"; + private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY); + + /** + * The size of the column qualifier set used. Increasing this parameter + * exponentially increases test time. + */ + private static final int NUM_COLUMNS = 8; + + private static final int MAX_COLUMN_BIT_MASK = 1 << NUM_COLUMNS - 1; + private static final int NUM_FLUSHES = 10; + private static final int NUM_ROWS = 20; + + /** A large value of type long for use as a timestamp */ + private static final long BIG_LONG = 9111222333444555666L; + + /** + * Timestamps to test with. Cannot use {@link Long#MAX_VALUE} here, because + * it will be replaced by an timestamp auto-generated based on the time. + */ + private static final long[] TIMESTAMPS = new long[] { 1, 3, 5, + Integer.MAX_VALUE, BIG_LONG, Long.MAX_VALUE - 1 }; + + /** The probability that a column is skipped in a store file. */ + private static final double COLUMN_SKIP_IN_STORE_FILE_PROB = 0.7; + + /** The probability of skipping a column in a single row */ + private static final double COLUMN_SKIP_IN_ROW_PROB = 0.1; + + /** The probability of skipping a column everywhere */ + private static final double COLUMN_SKIP_EVERYWHERE_PROB = 0.1; + + /** The probability to delete a row/column pair */ + private static final double DELETE_PROBABILITY = 0.02; + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private final Compression.Algorithm comprAlgo; + private final StoreFile.BloomType bloomType; + private final DataBlockEncoding dataBlockEncoding; + + // Some static sanity-checking. + static { + assertTrue(BIG_LONG > 0.9 * Long.MAX_VALUE); // Guard against typos. + + // Ensure TIMESTAMPS are sorted. + for (int i = 0; i < TIMESTAMPS.length - 1; ++i) + assertTrue(TIMESTAMPS[i] < TIMESTAMPS[i + 1]); + } + + @Before + public void setUp() { + SchemaMetrics.configureGlobally(TEST_UTIL.getConfiguration()); + } + + + @Parameters + public static final Collection parameters() { + List parameters = new ArrayList(); + for (Object[] bloomAndCompressionParams : + HBaseTestingUtility.BLOOM_AND_COMPRESSION_COMBINATIONS) { + for (boolean useDataBlockEncoding : new boolean[]{false, true}) { + parameters.add(ArrayUtils.add(bloomAndCompressionParams, + useDataBlockEncoding)); + } + } + return parameters; + } + + public TestMultiColumnScanner(Compression.Algorithm comprAlgo, + StoreFile.BloomType bloomType, boolean useDataBlockEncoding) { + this.comprAlgo = comprAlgo; + this.bloomType = bloomType; + this.dataBlockEncoding = useDataBlockEncoding ? DataBlockEncoding.PREFIX : + DataBlockEncoding.NONE; + } + + @Test + public void testMultiColumnScanner() throws IOException { + HRegion region = TEST_UTIL.createTestRegion(TABLE_NAME, + new HColumnDescriptor(FAMILY) + .setCompressionType(comprAlgo) + .setBloomFilterType(bloomType) + .setMaxVersions(MAX_VERSIONS) + .setDataBlockEncoding(dataBlockEncoding) + ); + List rows = sequentialStrings("row", NUM_ROWS); + List qualifiers = sequentialStrings("qual", NUM_COLUMNS); + List kvs = new ArrayList(); + Set keySet = new HashSet(); + + // A map from _ to the most recent delete timestamp for + // that column. + Map lastDelTimeMap = new HashMap(); + + Random rand = new Random(29372937L); + Set rowQualSkip = new HashSet(); + + // Skip some columns in some rows. We need to test scanning over a set + // of columns when some of the columns are not there. + for (String row : rows) + for (String qual : qualifiers) + if (rand.nextDouble() < COLUMN_SKIP_IN_ROW_PROB) { + LOG.info("Skipping " + qual + " in row " + row); + rowQualSkip.add(rowQualKey(row, qual)); + } + + // Also skip some columns in all rows. + for (String qual : qualifiers) + if (rand.nextDouble() < COLUMN_SKIP_EVERYWHERE_PROB) { + LOG.info("Skipping " + qual + " in all rows"); + for (String row : rows) + rowQualSkip.add(rowQualKey(row, qual)); + } + + for (int iFlush = 0; iFlush < NUM_FLUSHES; ++iFlush) { + for (String qual : qualifiers) { + // This is where we decide to include or not include this column into + // this store file, regardless of row and timestamp. + if (rand.nextDouble() < COLUMN_SKIP_IN_STORE_FILE_PROB) + continue; + + byte[] qualBytes = Bytes.toBytes(qual); + for (String row : rows) { + Put p = new Put(Bytes.toBytes(row)); + for (long ts : TIMESTAMPS) { + String value = createValue(row, qual, ts); + KeyValue kv = KeyValueTestUtil.create(row, FAMILY, qual, ts, + value); + assertEquals(kv.getTimestamp(), ts); + p.add(kv); + String keyAsString = kv.toString(); + if (!keySet.contains(keyAsString)) { + keySet.add(keyAsString); + kvs.add(kv); + } + } + region.put(p); + + Delete d = new Delete(Bytes.toBytes(row)); + boolean deletedSomething = false; + for (long ts : TIMESTAMPS) + if (rand.nextDouble() < DELETE_PROBABILITY) { + d.deleteColumns(FAMILY_BYTES, qualBytes, ts); + String rowAndQual = row + "_" + qual; + Long whenDeleted = lastDelTimeMap.get(rowAndQual); + lastDelTimeMap.put(rowAndQual, whenDeleted == null ? ts + : Math.max(ts, whenDeleted)); + deletedSomething = true; + } + if (deletedSomething) + region.delete(d, null, true); + } + } + region.flushcache(); + } + + Collections.sort(kvs, KeyValue.COMPARATOR); + for (int maxVersions = 1; maxVersions <= TIMESTAMPS.length; ++maxVersions) { + for (int columnBitMask = 1; columnBitMask <= MAX_COLUMN_BIT_MASK; ++columnBitMask) { + Scan scan = new Scan(); + scan.setMaxVersions(maxVersions); + Set qualSet = new TreeSet(); + { + int columnMaskTmp = columnBitMask; + for (String qual : qualifiers) { + if ((columnMaskTmp & 1) != 0) { + scan.addColumn(FAMILY_BYTES, Bytes.toBytes(qual)); + qualSet.add(qual); + } + columnMaskTmp >>= 1; + } + assertEquals(0, columnMaskTmp); + } + + InternalScanner scanner = region.getScanner(scan); + List results = new ArrayList(); + + int kvPos = 0; + int numResults = 0; + String queryInfo = "columns queried: " + qualSet + " (columnBitMask=" + + columnBitMask + "), maxVersions=" + maxVersions; + + while (scanner.next(results) || results.size() > 0) { + for (KeyValue kv : results) { + while (kvPos < kvs.size() + && !matchesQuery(kvs.get(kvPos), qualSet, maxVersions, + lastDelTimeMap)) { + ++kvPos; + } + String rowQual = getRowQualStr(kv); + String deleteInfo = ""; + Long lastDelTS = lastDelTimeMap.get(rowQual); + if (lastDelTS != null) { + deleteInfo = "; last timestamp when row/column " + rowQual + + " was deleted: " + lastDelTS; + } + assertTrue("Scanner returned additional key/value: " + kv + ", " + + queryInfo + deleteInfo + ";", kvPos < kvs.size()); + assertEquals("Scanner returned wrong key/value; " + queryInfo + + deleteInfo + ";", kvs.get(kvPos), kv); + ++kvPos; + ++numResults; + } + results.clear(); + } + for (; kvPos < kvs.size(); ++kvPos) { + KeyValue remainingKV = kvs.get(kvPos); + assertFalse("Matching column not returned by scanner: " + + remainingKV + ", " + queryInfo + ", results returned: " + + numResults, matchesQuery(remainingKV, qualSet, maxVersions, + lastDelTimeMap)); + } + } + } + assertTrue("This test is supposed to delete at least some row/column " + + "pairs", lastDelTimeMap.size() > 0); + LOG.info("Number of row/col pairs deleted at least once: " + + lastDelTimeMap.size()); + region.close(); + region.getLog().closeAndDelete(); + } + + private static String getRowQualStr(KeyValue kv) { + String rowStr = Bytes.toString(kv.getBuffer(), kv.getRowOffset(), + kv.getRowLength()); + String qualStr = Bytes.toString(kv.getBuffer(), kv.getQualifierOffset(), + kv.getQualifierLength()); + return rowStr + "_" + qualStr; + } + + private static boolean matchesQuery(KeyValue kv, Set qualSet, + int maxVersions, Map lastDelTimeMap) { + Long lastDelTS = lastDelTimeMap.get(getRowQualStr(kv)); + long ts = kv.getTimestamp(); + return qualSet.contains(qualStr(kv)) + && ts >= TIMESTAMPS[TIMESTAMPS.length - maxVersions] + && (lastDelTS == null || ts > lastDelTS); + } + + private static String qualStr(KeyValue kv) { + return Bytes.toString(kv.getBuffer(), kv.getQualifierOffset(), + kv.getQualifierLength()); + } + + private static String rowQualKey(String row, String qual) { + return row + "_" + qual; + } + + static String createValue(String row, String qual, long ts) { + return "value_for_" + row + "_" + qual + "_" + ts; + } + + private static List sequentialStrings(String prefix, int n) { + List lst = new ArrayList(); + for (int i = 0; i < n; ++i) { + StringBuilder sb = new StringBuilder(); + sb.append(prefix + i); + + // Make column length depend on i. + int iBitShifted = i; + while (iBitShifted != 0) { + sb.append((iBitShifted & 1) == 0 ? 'a' : 'b'); + iBitShifted >>= 1; + } + + lst.add(sb.toString()); + } + + return lst; + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConsistencyControl.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConsistencyControl.java new file mode 100644 index 0000000..7908b30 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConsistencyControl.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.experimental.categories.Category; + +import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * This is a hammer test that verifies MultiVersionConsistencyControl in a + * multiple writer single reader scenario. + */ +@Category(SmallTests.class) +public class TestMultiVersionConsistencyControl extends TestCase { + static class Writer implements Runnable { + final AtomicBoolean finished; + final MultiVersionConsistencyControl mvcc; + final AtomicBoolean status; + + Writer(AtomicBoolean finished, MultiVersionConsistencyControl mvcc, AtomicBoolean status) { + this.finished = finished; + this.mvcc = mvcc; + this.status = status; + } + + private Random rnd = new Random(); + public boolean failed = false; + + public void run() { + while (!finished.get()) { + MultiVersionConsistencyControl.WriteEntry e = mvcc.beginMemstoreInsert(); + // System.out.println("Begin write: " + e.getWriteNumber()); + // 10 usec - 500usec (including 0) + int sleepTime = rnd.nextInt(500); + // 500 * 1000 = 500,000ns = 500 usec + // 1 * 100 = 100ns = 1usec + try { + if (sleepTime > 0) Thread.sleep(0, sleepTime * 1000); + } catch (InterruptedException e1) { + } + try { + mvcc.completeMemstoreInsert(e); + } catch (RuntimeException ex) { + // got failure + System.out.println(ex.toString()); + ex.printStackTrace(); + status.set(false); + return; + // Report failure if possible. + } + } + } + } + + public void testParallelism() throws Exception { + final MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl(); + + final AtomicBoolean finished = new AtomicBoolean(false); + + // fail flag for the reader thread + final AtomicBoolean readerFailed = new AtomicBoolean(false); + final AtomicLong failedAt = new AtomicLong(); + Runnable reader = new Runnable() { + public void run() { + long prev = mvcc.memstoreReadPoint(); + while (!finished.get()) { + long newPrev = mvcc.memstoreReadPoint(); + if (newPrev < prev) { + // serious problem. + System.out.println("Reader got out of order, prev: " + prev + " next was: " + newPrev); + readerFailed.set(true); + // might as well give up + failedAt.set(newPrev); + return; + } + } + } + }; + + // writer thread parallelism. + int n = 20; + Thread[] writers = new Thread[n]; + AtomicBoolean[] statuses = new AtomicBoolean[n]; + Thread readThread = new Thread(reader); + + for (int i = 0; i < n; ++i) { + statuses[i] = new AtomicBoolean(true); + writers[i] = new Thread(new Writer(finished, mvcc, statuses[i])); + writers[i].start(); + } + readThread.start(); + + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException ex) { + } + + finished.set(true); + + readThread.join(); + for (int i = 0; i < n; ++i) { + writers[i].join(); + } + + // check failure. + assertFalse(readerFailed.get()); + for (int i = 0; i < n; ++i) { + assertTrue(statuses[i].get()); + } + + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestParallelPut.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestParallelPut.java new file mode 100644 index 0000000..3b0eded --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestParallelPut.java @@ -0,0 +1,252 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants.OperationStatusCode; +import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.NullComparator; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; +import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge; +import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.PairOfSameType; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.Test; + +import com.google.common.collect.Lists; +import org.junit.experimental.categories.Category; + + +/** + * Testing of multiPut in parallel. + * + */ +@Category(MediumTests.class) +public class TestParallelPut extends HBaseTestCase { + static final Log LOG = LogFactory.getLog(TestParallelPut.class); + + private static HRegion region = null; + private static HBaseTestingUtility hbtu = new HBaseTestingUtility(); + private static final String DIR = hbtu.getDataTestDir() + "/TestParallelPut/"; + + // Test names + static final byte[] tableName = Bytes.toBytes("testtable");; + static final byte[] qual1 = Bytes.toBytes("qual1"); + static final byte[] qual2 = Bytes.toBytes("qual2"); + static final byte[] qual3 = Bytes.toBytes("qual3"); + static final byte[] value1 = Bytes.toBytes("value1"); + static final byte[] value2 = Bytes.toBytes("value2"); + static final byte [] row = Bytes.toBytes("rowA"); + static final byte [] row2 = Bytes.toBytes("rowB"); + + /** + * @see org.apache.hadoop.hbase.HBaseTestCase#setUp() + */ + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + EnvironmentEdgeManagerTestHelper.reset(); + } + + ////////////////////////////////////////////////////////////////////////////// + // New tests that don't spin up a mini cluster but rather just test the + // individual code pieces in the HRegion. + ////////////////////////////////////////////////////////////////////////////// + + /** + * Test one put command. + */ + public void testPut() throws IOException { + LOG.info("Starting testPut"); + initHRegion(tableName, getName(), fam1); + + long value = 1L; + + Put put = new Put(row); + put.add(fam1, qual1, Bytes.toBytes(value)); + region.put(put); + + assertGet(row, fam1, qual1, Bytes.toBytes(value)); + } + + /** + * Test multi-threaded Puts. + */ + public void testParallelPuts() throws IOException { + + LOG.info("Starting testParallelPuts"); + initHRegion(tableName, getName(), fam1); + int numOps = 1000; // these many operations per thread + + // create 100 threads, each will do its own puts + int numThreads = 100; + Putter[] all = new Putter[numThreads]; + + // create all threads + for (int i = 0; i < numThreads; i++) { + all[i] = new Putter(region, i, numOps); + } + + // run all threads + for (int i = 0; i < numThreads; i++) { + all[i].start(); + } + + // wait for all threads to finish + for (int i = 0; i < numThreads; i++) { + try { + all[i].join(); + } catch (InterruptedException e) { + LOG.warn("testParallelPuts encountered InterruptedException." + + " Ignoring....", e); + } + } + LOG.info("testParallelPuts successfully verified " + + (numOps * numThreads) + " put operations."); + } + + + static private void assertGet(byte [] row, + byte [] familiy, + byte[] qualifier, + byte[] value) throws IOException { + // run a get and see if the value matches + Get get = new Get(row); + get.addColumn(familiy, qualifier); + Result result = region.get(get, null); + assertEquals(1, result.size()); + + KeyValue kv = result.raw()[0]; + byte[] r = kv.getValue(); + assertTrue(Bytes.compareTo(r, value) == 0); + } + + private void initHRegion(byte [] tableName, String callingMethod, + byte[] ... families) + throws IOException { + initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families); + } + + private void initHRegion(byte [] tableName, String callingMethod, + Configuration conf, byte [] ... families) + throws IOException{ + HTableDescriptor htd = new HTableDescriptor(tableName); + for(byte [] family : families) { + htd.addFamily(new HColumnDescriptor(family)); + } + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + Path path = new Path(DIR + callingMethod); + if (fs.exists(path)) { + if (!fs.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + region = HRegion.createHRegion(info, path, conf, htd); + } + + /** + * A thread that makes a few put calls + */ + public static class Putter extends Thread { + + private final HRegion region; + private final int threadNumber; + private final int numOps; + private final Random rand = new Random(); + byte [] rowkey = null; + + public Putter(HRegion region, int threadNumber, int numOps) { + this.region = region; + this.threadNumber = threadNumber; + this.numOps = numOps; + this.rowkey = Bytes.toBytes((long)threadNumber); // unique rowid per thread + setDaemon(true); + } + + @Override + public void run() { + byte[] value = new byte[100]; + Put[] in = new Put[1]; + + // iterate for the specified number of operations + for (int i=0; i instead + //of just byte [] + + //Expected result + List expected = new ArrayList(); + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW); + expected.add(ScanQueryMatcher.MatchCode.DONE); + + // 2,4,5 + + ScanQueryMatcher qm = new ScanQueryMatcher(scan, new Store.ScanInfo(fam2, + 0, 1, ttl, false, 0, rowComparator), get.getFamilyMap().get(fam2), + EnvironmentEdgeManager.currentTimeMillis() - ttl); + + List memstore = new ArrayList(); + memstore.add(new KeyValue(row1, fam2, col1, 1, data)); + memstore.add(new KeyValue(row1, fam2, col2, 1, data)); + memstore.add(new KeyValue(row1, fam2, col3, 1, data)); + memstore.add(new KeyValue(row1, fam2, col4, 1, data)); + memstore.add(new KeyValue(row1, fam2, col5, 1, data)); + + memstore.add(new KeyValue(row2, fam1, col1, data)); + + List actual = new ArrayList(); + KeyValue k = memstore.get(0); + qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength()); + + for (KeyValue kv : memstore){ + actual.add(qm.match(kv)); + } + + assertEquals(expected.size(), actual.size()); + for(int i=0; i< expected.size(); i++){ + assertEquals(expected.get(i), actual.get(i)); + if(PRINT){ + System.out.println("expected "+expected.get(i)+ + ", actual " +actual.get(i)); + } + } + } + + + public void testMatch_Wildcard() + throws IOException { + //Moving up from the Tracker by using Gets and List instead + //of just byte [] + + //Expected result + List expected = new ArrayList(); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); + expected.add(ScanQueryMatcher.MatchCode.DONE); + + ScanQueryMatcher qm = new ScanQueryMatcher(scan, new Store.ScanInfo(fam2, + 0, 1, ttl, false, 0, rowComparator), null, + EnvironmentEdgeManager.currentTimeMillis() - ttl); + + List memstore = new ArrayList(); + memstore.add(new KeyValue(row1, fam2, col1, 1, data)); + memstore.add(new KeyValue(row1, fam2, col2, 1, data)); + memstore.add(new KeyValue(row1, fam2, col3, 1, data)); + memstore.add(new KeyValue(row1, fam2, col4, 1, data)); + memstore.add(new KeyValue(row1, fam2, col5, 1, data)); + memstore.add(new KeyValue(row2, fam1, col1, 1, data)); + + List actual = new ArrayList(); + + KeyValue k = memstore.get(0); + qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength()); + + for(KeyValue kv : memstore) { + actual.add(qm.match(kv)); + } + + assertEquals(expected.size(), actual.size()); + for(int i=0; i< expected.size(); i++){ + assertEquals(expected.get(i), actual.get(i)); + if(PRINT){ + System.out.println("expected "+expected.get(i)+ + ", actual " +actual.get(i)); + } + } + } + + + /** + * Verify that {@link ScanQueryMatcher} only skips expired KeyValue + * instances and does not exit early from the row (skipping + * later non-expired KeyValues). This version mimics a Get with + * explicitly specified column qualifiers. + * + * @throws IOException + */ + public void testMatch_ExpiredExplicit() + throws IOException { + + long testTTL = 1000; + MatchCode [] expected = new MatchCode[] { + ScanQueryMatcher.MatchCode.SEEK_NEXT_COL, + ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL, + ScanQueryMatcher.MatchCode.SEEK_NEXT_COL, + ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL, + ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW, + ScanQueryMatcher.MatchCode.DONE + }; + + long now = EnvironmentEdgeManager.currentTimeMillis(); + ScanQueryMatcher qm = new ScanQueryMatcher(scan, new Store.ScanInfo(fam2, + 0, 1, testTTL, false, 0, rowComparator), get.getFamilyMap().get(fam2), + now - testTTL); + + KeyValue [] kvs = new KeyValue[] { + new KeyValue(row1, fam2, col1, now-100, data), + new KeyValue(row1, fam2, col2, now-50, data), + new KeyValue(row1, fam2, col3, now-5000, data), + new KeyValue(row1, fam2, col4, now-500, data), + new KeyValue(row1, fam2, col5, now-10000, data), + new KeyValue(row2, fam1, col1, now-10, data) + }; + + KeyValue k = kvs[0]; + qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength()); + + List actual = new ArrayList(kvs.length); + for (KeyValue kv : kvs) { + actual.add( qm.match(kv) ); + } + + assertEquals(expected.length, actual.size()); + for (int i=0; i actual = + new ArrayList(kvs.length); + for (KeyValue kv : kvs) { + actual.add( qm.match(kv) ); + } + + assertEquals(expected.length, actual.size()); + for (int i=0; i puts = new ArrayList(); + Put put1 = new Put(Bytes.toBytes("a")); + put1.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value")); + Put put2 = new Put(Bytes.toBytes("h")); + put2.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value")); + Put put3 = new Put(Bytes.toBytes("o")); + put3.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value")); + puts.add(put1); + puts.add(put2); + puts.add(put3); + table.put(puts); + ResultScanner resultScanner = table.getScanner(new Scan()); + int count = 0; + while (resultScanner.next() != null) { + count++; + } + resultScanner.close(); + table.close(); + assertEquals(3, count); + + /* Starting test */ + cluster.getConfiguration().setBoolean("TestingMaster.sleep", true); + cluster.getConfiguration().setInt("TestingMaster.sleep.duration", 10000); + + /* NO.1 .META. region correctness */ + // First abort master + abortMaster(cluster); + TestingMaster master = startMasterAndWaitTillMetaRegionAssignment(cluster); + + // Second kill meta server + int metaServerNum = cluster.getServerWithMeta(); + int rootServerNum = cluster.getServerWith(HRegionInfo.ROOT_REGIONINFO + .getRegionName()); + HRegionServer metaRS = cluster.getRegionServer(metaServerNum); + LOG.debug("Killing metaRS and carryingRoot = " + + (metaServerNum == rootServerNum)); + metaRS.kill(); + metaRS.join(); + + /* + * Sleep double time of TestingMaster.sleep.duration, so we can ensure that + * master has already assigned ROOTandMETA or is blocking on assigning + * ROOTandMETA + */ + Thread.sleep(10000 * 2); + + waitUntilMasterIsInitialized(master); + + // Third check whether data is correct in meta region + assertTrue(hbaseAdmin.isTableAvailable(TABLENAME)); + + /* + * NO.2 -ROOT- region correctness . If the .META. server killed in the NO.1 + * is also carrying -ROOT- region, it is not needed + */ + if (rootServerNum != metaServerNum) { + // First abort master + abortMaster(cluster); + master = startMasterAndWaitTillMetaRegionAssignment(cluster); + + // Second kill meta server + HRegionServer rootRS = cluster.getRegionServer(rootServerNum); + LOG.debug("Killing rootRS"); + rootRS.kill(); + rootRS.join(); + + /* + * Sleep double time of TestingMaster.sleep.duration, so we can ensure + * that master has already assigned ROOTandMETA or is blocking on + * assigning ROOTandMETA + */ + Thread.sleep(10000 * 2); + waitUntilMasterIsInitialized(master); + + // Third check whether data is correct in meta region + assertTrue(hbaseAdmin.isTableAvailable(TABLENAME)); + } + + + /* NO.3 data region correctness */ + ServerManager serverManager = cluster.getMaster().getServerManager(); + while (serverManager.areDeadServersInProgress()) { + Thread.sleep(100); + } + // Create a ZKW to use in the test + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTUTIL); + ZKAssign.blockUntilNoRIT(zkw); + + table = new HTable(TESTUTIL.getConfiguration(), TABLENAME); + resultScanner = table.getScanner(new Scan()); + count = 0; + while (resultScanner.next() != null) { + count++; + } + resultScanner.close(); + table.close(); + assertEquals(3, count); + } + + @Test (timeout=180000) + public void testMasterFailoverWhenDisablingTableRegionsInRITOnDeadRS() throws Exception { + MiniHBaseCluster cluster = TESTUTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + // disable load balancing on this master + master.balanceSwitch(false); + + final String table = "testMasterFailoverWhenDisablingTableRegionsInRITOnDeadRS"; + byte [] FAMILY = Bytes.toBytes("family"); + byte[][] SPLIT_KEYS = + new byte[][] {Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"), + Bytes.toBytes("d") }; + HTableDescriptor htd = new HTableDescriptor(table); + HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); + htd.addFamily(hcd); + TESTUTIL.getHBaseAdmin().createTable(htd, SPLIT_KEYS); + AssignmentManager am = cluster.getMaster().getAssignmentManager(); + List regionsOfTable = null; + while ((regionsOfTable = am.getRegionsOfTable(table.getBytes())).size() + != (SPLIT_KEYS.length + 1)) { + Thread.sleep(10); + } + HRegionInfo closingRegion = regionsOfTable.get(0); + ServerName serverName = am.getRegionServerOfRegion(closingRegion); + HRegionServer deadRS = null; + for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) { + deadRS = cluster.getRegionServer(i); + if (deadRS.getServerName().equals(serverName)) { + break; + } + } + + // Disable the table in ZK + ZKTable zkTable = am.getZKTable(); + zkTable.setDisablingTable(table); + ZKAssign.createNodeClosing(master.getZooKeeper(), closingRegion, serverName); + + // Stop the master + abortMaster(cluster); + master = startMasterAndWaitTillMetaRegionAssignment(cluster); + deadRS.kill(); + deadRS.join(); + waitUntilMasterIsInitialized(master); + am = cluster.getMaster().getAssignmentManager(); + zkTable = am.getZKTable(); + // wait for no more RIT + ZKAssign.blockUntilNoRIT(master.getZooKeeper()); + while (!master.getAssignmentManager().getZKTable().isDisabledTable(table)) { + Thread.sleep(10); + } + assertTrue("Table should be disabled state.", zkTable.isDisabledTable(table)); + HBaseAdmin admin = new HBaseAdmin(master.getConfiguration()); + admin.deleteTable(table); + } + + private void abortMaster(MiniHBaseCluster cluster) + throws InterruptedException { + for (MasterThread mt : cluster.getLiveMasterThreads()) { + if (mt.getMaster().isActiveMaster()) { + mt.getMaster().abort("Aborting for tests", new Exception("Trace info")); + mt.join(); + break; + } + } + LOG.debug("Master is aborted"); + } + + private TestingMaster startMasterAndWaitTillMetaRegionAssignment(MiniHBaseCluster cluster) + throws IOException, InterruptedException { + TestingMaster master = (TestingMaster) cluster.startMaster().getMaster(); + while (!master.isInitializationStartsMetaRegoinAssignment()) { + Thread.sleep(100); + } + return master; + } + + private void waitUntilMasterIsInitialized(HMaster master) + throws InterruptedException { + while (!master.isInitialized()) { + Thread.sleep(100); + } + while (master.getServerManager().areDeadServersInProgress()) { + Thread.sleep(100); + } + LOG.debug("master isInitialized"); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); + +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java new file mode 100644 index 0000000..8478260 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java @@ -0,0 +1,109 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.List; + +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HServerInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MasterAddressTracker; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics; +import org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import com.google.common.collect.Lists; + +/** + * Tests for the region server status page and its template. + */ +@Category(SmallTests.class) +public class TestRSStatusServlet { + private HRegionServer rs; + + static final int FAKE_IPC_PORT = 1585; + static final int FAKE_WEB_PORT = 1586; + + @SuppressWarnings("deprecation") + private final HServerAddress fakeAddress = + new HServerAddress("localhost", FAKE_IPC_PORT); + @SuppressWarnings("deprecation") + private final HServerInfo fakeInfo = + new HServerInfo(fakeAddress, FAKE_WEB_PORT); + private final RegionServerMetrics metrics = + new RegionServerMetrics(); + private final ServerName fakeMasterAddress = + new ServerName("localhost", 60010, 1212121212); + + @SuppressWarnings("deprecation") + @Before + public void setupBasicMocks() throws IOException { + rs = Mockito.mock(HRegionServer.class); + Mockito.doReturn(HBaseConfiguration.create()) + .when(rs).getConfiguration(); + Mockito.doReturn(fakeInfo).when(rs).getHServerInfo(); + Mockito.doReturn(metrics).when(rs).getMetrics(); + + // Fake ZKW + ZooKeeperWatcher zkw = Mockito.mock(ZooKeeperWatcher.class); + Mockito.doReturn("fakequorum").when(zkw).getQuorum(); + Mockito.doReturn(zkw).when(rs).getZooKeeper(); + + // Fake MasterAddressTracker + MasterAddressTracker mat = Mockito.mock(MasterAddressTracker.class); + Mockito.doReturn(fakeMasterAddress).when(mat).getMasterAddress(); + Mockito.doReturn(mat).when(rs).getMasterAddressManager(); + } + + @Test + public void testBasic() throws IOException { + new RSStatusTmpl().render(new StringWriter(), rs); + } + + @Test + public void testWithRegions() throws IOException { + HTableDescriptor htd = new HTableDescriptor("mytable"); + List regions = Lists.newArrayList( + new HRegionInfo(htd.getName(), Bytes.toBytes("a"), Bytes.toBytes("d")), + new HRegionInfo(htd.getName(), Bytes.toBytes("d"), Bytes.toBytes("z")) + ); + Mockito.doReturn(regions).when(rs).getOnlineRegions(); + + new RSStatusTmpl().render(new StringWriter(), rs); + } + + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java new file mode 100644 index 0000000..d6d61bf --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -0,0 +1,356 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics. + StoreMetricType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + + +/** + * Test metrics incremented on region server operations. + */ +@Category(MediumTests.class) +public class TestRegionServerMetrics { + + private static final Log LOG = + LogFactory.getLog(TestRegionServerMetrics.class.getName()); + + private final static String TABLE_NAME = + TestRegionServerMetrics.class.getSimpleName() + "Table"; + private String[] FAMILIES = new String[] { "cf1", "cf2", "anotherCF" }; + private static final int MAX_VERSIONS = 1; + private static final int NUM_COLS_PER_ROW = 15; + private static final int NUM_FLUSHES = 3; + private static final int NUM_REGIONS = 4; + + private static final SchemaMetrics ALL_METRICS = + SchemaMetrics.ALL_SCHEMA_METRICS; + + private final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + private Map startingMetrics; + + private final int META_AND_ROOT = 2; + + @Before + public void setUp() throws Exception { + SchemaMetrics.setUseTableNameInTest(true); + startingMetrics = SchemaMetrics.getMetricsSnapshot(); + TEST_UTIL.startMiniCluster(); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + SchemaMetrics.validateMetricChanges(startingMetrics); + } + + private void assertTimeVaryingMetricCount(int expectedCount, String table, String cf, + String regionName, String metricPrefix) { + + Integer expectedCountInteger = new Integer(expectedCount); + + if (cf != null) { + String cfKey = + SchemaMetrics.TABLE_PREFIX + table + "." + SchemaMetrics.CF_PREFIX + cf + "." + + metricPrefix; + Pair cfPair = RegionMetricsStorage.getTimeVaryingMetric(cfKey); + assertEquals(expectedCountInteger, cfPair.getSecond()); + } + + if (regionName != null) { + String rKey = + SchemaMetrics.TABLE_PREFIX + table + "." + SchemaMetrics.REGION_PREFIX + regionName + "." + + metricPrefix; + + Pair regionPair = RegionMetricsStorage.getTimeVaryingMetric(rKey); + assertEquals(expectedCountInteger, regionPair.getSecond()); + } + } + + private void assertStoreMetricEquals(long expected, + SchemaMetrics schemaMetrics, StoreMetricType storeMetricType) { + final String storeMetricName = + schemaMetrics.getStoreMetricName(storeMetricType); + Long startValue = startingMetrics.get(storeMetricName); + assertEquals("Invalid value for store metric " + storeMetricName + + " (type " + storeMetricType + ")", expected, + RegionMetricsStorage.getNumericMetric(storeMetricName) + - (startValue != null ? startValue : 0)); + } + + @Test + public void testOperationMetrics() throws IOException { + String cf = "OPCF"; + String otherCf = "otherCF"; + String rk = "testRK"; + String icvCol = "icvCol"; + String appendCol = "appendCol"; + String regionName = null; + HTable hTable = + TEST_UTIL.createTable(TABLE_NAME.getBytes(), + new byte[][] { cf.getBytes(), otherCf.getBytes() }); + Set regionInfos = hTable.getRegionLocations().keySet(); + + regionName = regionInfos.toArray(new HRegionInfo[regionInfos.size()])[0].getEncodedName(); + + //Do a multi put that has one cf. Since they are in different rk's + //The lock will still be obtained and everything will be applied in one multiput. + Put pOne = new Put(rk.getBytes()); + pOne.add(cf.getBytes(), icvCol.getBytes(), Bytes.toBytes(0L)); + Put pTwo = new Put("ignored1RK".getBytes()); + pTwo.add(cf.getBytes(), "ignored".getBytes(), Bytes.toBytes(0L)); + + hTable.put(Arrays.asList(new Put[] {pOne, pTwo})); + + // Do a multiput where the cf doesn't stay consistent. + Put pThree = new Put("ignored2RK".getBytes()); + pThree.add(cf.getBytes(), "ignored".getBytes(), Bytes.toBytes("TEST1")); + Put pFour = new Put("ignored3RK".getBytes()); + pFour.add(otherCf.getBytes(), "ignored".getBytes(), Bytes.toBytes(0L)); + + hTable.put(Arrays.asList(new Put[] { pThree, pFour })); + + hTable.incrementColumnValue(rk.getBytes(), cf.getBytes(), icvCol.getBytes(), 1L); + + Increment i = new Increment(rk.getBytes()); + i.addColumn(cf.getBytes(), icvCol.getBytes(), 1L); + hTable.increment(i); + + Get g = new Get(rk.getBytes()); + g.addColumn(cf.getBytes(), appendCol.getBytes()); + hTable.get(g); + + Append a = new Append(rk.getBytes()); + a.add(cf.getBytes(), appendCol.getBytes(), Bytes.toBytes("-APPEND")); + hTable.append(a); + + Delete dOne = new Delete(rk.getBytes()); + dOne.deleteFamily(cf.getBytes()); + hTable.delete(dOne); + + Delete dTwo = new Delete(rk.getBytes()); + hTable.delete(dTwo); + + // There should be one multi put where the cf is consistent + assertTimeVaryingMetricCount(1, TABLE_NAME, cf, null, "multiput_"); + + // There were two multiputs to the cf. + assertTimeVaryingMetricCount(2, TABLE_NAME, null, regionName, "multiput_"); + + // There was one multiput where the cf was not consistent. + assertTimeVaryingMetricCount(1, TABLE_NAME, "__unknown", null, "multiput_"); + + // One increment and one append + assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "incrementColumnValue_"); + assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "increment_"); + assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "append_"); + + // One delete where the cf is known + assertTimeVaryingMetricCount(1, TABLE_NAME, cf, null, "multidelete_"); + + // two deletes in the region. + assertTimeVaryingMetricCount(2, TABLE_NAME, null, regionName, "multidelete_"); + + // Three gets. one for gets. One for append. One for increment. + assertTimeVaryingMetricCount(4, TABLE_NAME, cf, regionName, "get_"); + + hTable.close(); + } + + @Test + public void testRemoveRegionMetrics() throws IOException, InterruptedException { + String cf = "REMOVECF"; + HTable hTable = TEST_UTIL.createTable(TABLE_NAME.getBytes(), cf.getBytes()); + HRegionInfo[] regionInfos = + hTable.getRegionLocations().keySet() + .toArray(new HRegionInfo[hTable.getRegionLocations().keySet().size()]); + + String regionName = regionInfos[0].getEncodedName(); + + // Do some operations so there are metrics. + Put pOne = new Put("TEST".getBytes()); + pOne.add(cf.getBytes(), "test".getBytes(), "test".getBytes()); + hTable.put(pOne); + + Get g = new Get("TEST".getBytes()); + g.addFamily(cf.getBytes()); + hTable.get(g); + assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "get_"); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + admin.disableTable(TABLE_NAME.getBytes()); + admin.deleteTable(TABLE_NAME.getBytes()); + + assertTimeVaryingMetricCount(0, TABLE_NAME, cf, regionName, "get_"); + + hTable.close(); + } + + @Test + public void testMultipleRegions() throws IOException, InterruptedException { + + TEST_UTIL.createRandomTable( + TABLE_NAME, + Arrays.asList(FAMILIES), + MAX_VERSIONS, NUM_COLS_PER_ROW, NUM_FLUSHES, NUM_REGIONS, 1000); + + final HRegionServer rs = + TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); + + assertEquals(NUM_REGIONS + META_AND_ROOT, rs.getOnlineRegions().size()); + + rs.doMetrics(); + for (HRegion r : TEST_UTIL.getMiniHBaseCluster().getRegions( + Bytes.toBytes(TABLE_NAME))) { + for (Map.Entry storeEntry : r.getStores().entrySet()) { + LOG.info("For region " + r.getRegionNameAsString() + ", CF " + + Bytes.toStringBinary(storeEntry.getKey()) + " found store files " + + ": " + storeEntry.getValue().getStorefiles()); + } + } + + assertStoreMetricEquals(NUM_FLUSHES * NUM_REGIONS * FAMILIES.length + + META_AND_ROOT, ALL_METRICS, StoreMetricType.STORE_FILE_COUNT); + + for (String cf : FAMILIES) { + SchemaMetrics schemaMetrics = SchemaMetrics.getInstance(TABLE_NAME, cf); + assertStoreMetricEquals(NUM_FLUSHES * NUM_REGIONS, schemaMetrics, + StoreMetricType.STORE_FILE_COUNT); + } + + // ensure that the max value is also maintained + final String storeMetricName = ALL_METRICS + .getStoreMetricNameMax(StoreMetricType.STORE_FILE_COUNT); + assertEquals("Invalid value for store metric " + storeMetricName, + NUM_FLUSHES, RegionMetricsStorage.getNumericMetric(storeMetricName)); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); + + private void assertSizeMetric(String table, String[] cfs, int[] metrics) { + // we have getsize & nextsize for each column family + assertEquals(cfs.length * 2, metrics.length); + + for (int i =0; i < cfs.length; ++i) { + String prefix = SchemaMetrics.generateSchemaMetricsPrefix(table, cfs[i]); + String getMetric = prefix + SchemaMetrics.METRIC_GETSIZE; + String nextMetric = prefix + SchemaMetrics.METRIC_NEXTSIZE; + + // verify getsize and nextsize matches + int getSize = RegionMetricsStorage.getNumericMetrics().containsKey(getMetric) ? + RegionMetricsStorage.getNumericMetrics().get(getMetric).intValue() : 0; + int nextSize = RegionMetricsStorage.getNumericMetrics().containsKey(nextMetric) ? + RegionMetricsStorage.getNumericMetrics().get(nextMetric).intValue() : 0; + + assertEquals(metrics[i], getSize); + assertEquals(metrics[cfs.length + i], nextSize); + } + } + + @Test + public void testGetNextSize() throws IOException, InterruptedException { + String rowName = "row1"; + byte[] ROW = Bytes.toBytes(rowName); + String tableName = "SizeMetricTest"; + byte[] TABLE = Bytes.toBytes(tableName); + String cf1Name = "cf1"; + String cf2Name = "cf2"; + String[] cfs = new String[] {cf1Name, cf2Name}; + byte[] CF1 = Bytes.toBytes(cf1Name); + byte[] CF2 = Bytes.toBytes(cf2Name); + + long ts = 1234; + HTable hTable = TEST_UTIL.createTable(TABLE, new byte[][]{CF1, CF2}); + HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + + Put p = new Put(ROW); + p.add(CF1, CF1, ts, CF1); + p.add(CF2, CF2, ts, CF2); + hTable.put(p); + + KeyValue kv1 = new KeyValue(ROW, CF1, CF1, ts, CF1); + KeyValue kv2 = new KeyValue(ROW, CF2, CF2, ts, CF2); + int kvLength = kv1.getLength(); + assertEquals(kvLength, kv2.getLength()); + + // only cf1.getsize is set on Get + hTable.get(new Get(ROW).addFamily(CF1)); + assertSizeMetric(tableName, cfs, new int[] {kvLength, 0, 0, 0}); + + // only cf2.getsize is set on Get + hTable.get(new Get(ROW).addFamily(CF2)); + assertSizeMetric(tableName, cfs, new int[] {kvLength, kvLength, 0, 0}); + + // only cf2.nextsize is set + for (Result res : hTable.getScanner(CF2)) { + } + assertSizeMetric(tableName, cfs, + new int[] {kvLength, kvLength, 0, kvLength}); + + // only cf2.nextsize is set + for (Result res : hTable.getScanner(CF1)) { + } + assertSizeMetric(tableName, cfs, + new int[] {kvLength, kvLength, kvLength, kvLength}); + + // getsize/nextsize should not be set on flush or compaction + for (HRegion hr : TEST_UTIL.getMiniHBaseCluster().getRegions(TABLE)) { + hr.flushcache(); + hr.compactStores(); + } + assertSizeMetric(tableName, cfs, + new int[] {kvLength, kvLength, kvLength, kvLength}); + + hTable.close(); + } +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java new file mode 100644 index 0000000..69e6704 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java @@ -0,0 +1,292 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +@Category(SmallTests.class) +public class TestRegionSplitPolicy { + + private Configuration conf; + private HTableDescriptor htd; + private HRegion mockRegion; + private TreeMap stores; + private static final byte [] TABLENAME = new byte [] {'t'}; + + @Before + public void setupMocks() { + conf = HBaseConfiguration.create(); + HRegionInfo hri = new HRegionInfo(TABLENAME); + htd = new HTableDescriptor(TABLENAME); + mockRegion = Mockito.mock(HRegion.class); + Mockito.doReturn(htd).when(mockRegion).getTableDesc(); + Mockito.doReturn(hri).when(mockRegion).getRegionInfo(); + + stores = new TreeMap(Bytes.BYTES_COMPARATOR); + Mockito.doReturn(stores).when(mockRegion).getStores(); + } + + @Test + public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException { + // Configure IncreasingToUpperBoundRegionSplitPolicy as our split policy + conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, + IncreasingToUpperBoundRegionSplitPolicy.class.getName()); + // Now make it so the mock region has a RegionServerService that will + // return 'online regions'. + RegionServerServices rss = Mockito.mock(RegionServerServices.class); + final List regions = new ArrayList(); + Mockito.when(rss.getOnlineRegions(TABLENAME)).thenReturn(regions); + Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss); + // Set max size for this 'table'. + long maxSplitSize = 1024L; + htd.setMaxFileSize(maxSplitSize); + // Set flush size to 1/4. IncreasingToUpperBoundRegionSplitPolicy + // grows by the square of the number of regions times flushsize each time. + long flushSize = maxSplitSize/4; + conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize); + htd.setMemStoreFlushSize(flushSize); + // If RegionServerService with no regions in it -- 'online regions' == 0 -- + // then IncreasingToUpperBoundRegionSplitPolicy should act like a + // ConstantSizePolicy + IncreasingToUpperBoundRegionSplitPolicy policy = + (IncreasingToUpperBoundRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); + doConstantSizePolicyTests(policy); + + // Add a store in excess of split size. Because there are "no regions" + // on this server -- rss.getOnlineRegions is 0 -- then we should split + // like a constantsizeregionsplitpolicy would + Store mockStore = Mockito.mock(Store.class); + Mockito.doReturn(2000L).when(mockStore).getSize(); + Mockito.doReturn(true).when(mockStore).canSplit(); + stores.put(new byte[]{1}, mockStore); + // It should split + assertTrue(policy.shouldSplit()); + + // Now test that we increase our split size as online regions for a table + // grows. With one region, split size should be flushsize. + regions.add(mockRegion); + Mockito.doReturn(flushSize/2).when(mockStore).getSize(); + // Should not split since store is 1/2 flush size. + assertFalse(policy.shouldSplit()); + // Set size of store to be > flush size and we should split + Mockito.doReturn(flushSize + 1).when(mockStore).getSize(); + assertTrue(policy.shouldSplit()); + // Add another region to the 'online regions' on this server and we should + // now be no longer be splittable since split size has gone up. + regions.add(mockRegion); + assertFalse(policy.shouldSplit()); + // Quadruple (2 squared) the store size and make sure its just over; verify it'll split + Mockito.doReturn((flushSize * 2 * 2) + 1).when(mockStore).getSize(); + assertTrue(policy.shouldSplit()); + + // Finally assert that even if loads of regions, we'll split at max size + assertEquals(maxSplitSize, policy.getSizeToCheck(1000)); + // Assert same is true if count of regions is zero. + assertEquals(maxSplitSize, policy.getSizeToCheck(0)); + } + + @Test + public void testCreateDefault() throws IOException { + conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L); + + // Using a default HTD, should pick up the file size from + // configuration. + ConstantSizeRegionSplitPolicy policy = + (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create( + mockRegion, conf); + assertEquals(1234L, policy.getDesiredMaxFileSize()); + + // If specified in HTD, should use that + htd.setMaxFileSize(9999L); + policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create( + mockRegion, conf); + assertEquals(9999L, policy.getDesiredMaxFileSize()); + } + + /** + * Test setting up a customized split policy + */ + @Test + public void testCustomPolicy() throws IOException { + HTableDescriptor myHtd = new HTableDescriptor(); + myHtd.setValue(HTableDescriptor.SPLIT_POLICY, + KeyPrefixRegionSplitPolicy.class.getName()); + myHtd.setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, String.valueOf(2)); + + HRegion myMockRegion = Mockito.mock(HRegion.class); + Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc(); + Mockito.doReturn(stores).when(myMockRegion).getStores(); + + Store mockStore = Mockito.mock(Store.class); + Mockito.doReturn(2000L).when(mockStore).getSize(); + Mockito.doReturn(true).when(mockStore).canSplit(); + Mockito.doReturn(Bytes.toBytes("abcd")).when(mockStore).getSplitPoint(); + stores.put(new byte[] { 1 }, mockStore); + + KeyPrefixRegionSplitPolicy policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy + .create(myMockRegion, conf); + + assertEquals("ab", Bytes.toString(policy.getSplitPoint())); + + Mockito.doReturn(true).when(myMockRegion).shouldForceSplit(); + Mockito.doReturn(Bytes.toBytes("efgh")).when(myMockRegion) + .getExplicitSplitPoint(); + + policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy + .create(myMockRegion, conf); + + assertEquals("ef", Bytes.toString(policy.getSplitPoint())); + } + + @Test + public void testConstantSizePolicy() throws IOException { + htd.setMaxFileSize(1024L); + ConstantSizeRegionSplitPolicy policy = + (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); + doConstantSizePolicyTests(policy); + } + + /** + * Run through tests for a ConstantSizeRegionSplitPolicy + * @param policy + */ + private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) { + // For no stores, should not split + assertFalse(policy.shouldSplit()); + + // Add a store above the requisite size. Should split. + Store mockStore = Mockito.mock(Store.class); + Mockito.doReturn(2000L).when(mockStore).getSize(); + Mockito.doReturn(true).when(mockStore).canSplit(); + stores.put(new byte[]{1}, mockStore); + + assertTrue(policy.shouldSplit()); + + // Act as if there's a reference file or some other reason it can't split. + // This should prevent splitting even though it's big enough. + Mockito.doReturn(false).when(mockStore).canSplit(); + assertFalse(policy.shouldSplit()); + + // Reset splittability after above + Mockito.doReturn(true).when(mockStore).canSplit(); + + // Set to a small size but turn on forceSplit. Should result in a split. + Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); + Mockito.doReturn(100L).when(mockStore).getSize(); + assertTrue(policy.shouldSplit()); + + // Turn off forceSplit, should not split + Mockito.doReturn(false).when(mockRegion).shouldForceSplit(); + assertFalse(policy.shouldSplit()); + + // Clear families we added above + stores.clear(); + } + + @Test + public void testGetSplitPoint() throws IOException { + ConstantSizeRegionSplitPolicy policy = + (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); + + // For no stores, should not split + assertFalse(policy.shouldSplit()); + assertNull(policy.getSplitPoint()); + + // Add a store above the requisite size. Should split. + Store mockStore = Mockito.mock(Store.class); + Mockito.doReturn(2000L).when(mockStore).getSize(); + Mockito.doReturn(true).when(mockStore).canSplit(); + Mockito.doReturn(Bytes.toBytes("store 1 split")) + .when(mockStore).getSplitPoint(); + stores.put(new byte[]{1}, mockStore); + + assertEquals("store 1 split", + Bytes.toString(policy.getSplitPoint())); + + // Add a bigger store. The split point should come from that one + Store mockStore2 = Mockito.mock(Store.class); + Mockito.doReturn(4000L).when(mockStore2).getSize(); + Mockito.doReturn(true).when(mockStore2).canSplit(); + Mockito.doReturn(Bytes.toBytes("store 2 split")) + .when(mockStore2).getSplitPoint(); + stores.put(new byte[]{2}, mockStore2); + + assertEquals("store 2 split", + Bytes.toString(policy.getSplitPoint())); + } + + @Test + public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException { + HTableDescriptor myHtd = new HTableDescriptor(); + myHtd.setValue(HTableDescriptor.SPLIT_POLICY, + DelimitedKeyPrefixRegionSplitPolicy.class.getName()); + myHtd.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ","); + + HRegion myMockRegion = Mockito.mock(HRegion.class); + Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc(); + Mockito.doReturn(stores).when(myMockRegion).getStores(); + + Store mockStore = Mockito.mock(Store.class); + Mockito.doReturn(2000L).when(mockStore).getSize(); + Mockito.doReturn(true).when(mockStore).canSplit(); + Mockito.doReturn(Bytes.toBytes("ab,cd")).when(mockStore).getSplitPoint(); + stores.put(new byte[] { 1 }, mockStore); + + DelimitedKeyPrefixRegionSplitPolicy policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy + .create(myMockRegion, conf); + + assertEquals("ab", Bytes.toString(policy.getSplitPoint())); + + Mockito.doReturn(true).when(myMockRegion).shouldForceSplit(); + Mockito.doReturn(Bytes.toBytes("efg,h")).when(myMockRegion) + .getExplicitSplitPoint(); + + policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy + .create(myMockRegion, conf); + + assertEquals("efg", Bytes.toString(policy.getSplitPoint())); + + Mockito.doReturn(Bytes.toBytes("ijk")).when(myMockRegion) + .getExplicitSplitPoint(); + assertEquals("ijk", Bytes.toString(policy.getSplitPoint())); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java new file mode 100644 index 0000000..96afbd2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestResettingCounters.java @@ -0,0 +1,108 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestResettingCounters { + + @Test + public void testResettingCounters() throws Exception { + + HBaseTestingUtility htu = new HBaseTestingUtility(); + Configuration conf = htu.getConfiguration(); + FileSystem fs = FileSystem.get(conf); + byte [] table = Bytes.toBytes("table"); + byte [][] families = new byte [][] { + Bytes.toBytes("family1"), + Bytes.toBytes("family2"), + Bytes.toBytes("family3") + }; + int numQualifiers = 10; + byte [][] qualifiers = new byte [numQualifiers][]; + for (int i=0; i 0) { + METRICS.put(name, Boolean.TRUE); + LOG.debug("Set metric "+name+" to "+val); + } + } + } + } + private static Map METRICS = new HashMap(); + + private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static Log LOG = LogFactory.getLog(TestRpcMetrics.class); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // set custom metrics context + ContextFactory factory = ContextFactory.getFactory(); + factory.setAttribute("rpc.class", MockMetricsContext.class.getName()); + // make sure metrics context is setup, otherwise updating won't start + MetricsContext ctx = MetricsUtil.getContext("rpc"); + assertTrue("Wrong MetricContext implementation class", + (ctx instanceof MockMetricsContext)); + + TEST_UTIL.startMiniZKCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + @Test + public void testCustomMetrics() throws Exception { + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.port", 0); + TestRegionServer rs = new TestRegionServer(TEST_UTIL.getConfiguration()); + rs.incTest(5); + + // wait for metrics context update + Thread.sleep(1000); + + String metricName = HBaseRpcMetrics.getMetricName(TestMetrics.class, "test"); + assertTrue("Metric should have set incremented for "+metricName, + wasSet(metricName + "_num_ops")); + } + + public boolean wasSet(String name) { + return METRICS.get(name) != null ? METRICS.get(name) : false; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java new file mode 100644 index 0000000..765f12b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java @@ -0,0 +1,131 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.regionserver.DeleteTracker.DeleteResult; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + + +@Category(SmallTests.class) +public class TestScanDeleteTracker extends HBaseTestCase { + + private ScanDeleteTracker sdt; + private long timestamp = 10L; + private byte deleteType = 0; + + public void setUp() throws Exception { + super.setUp(); + sdt = new ScanDeleteTracker(); + } + + public void testDeletedBy_Delete() { + byte [] qualifier = Bytes.toBytes("qualifier"); + deleteType = KeyValue.Type.Delete.getCode(); + + sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType); + DeleteResult ret = sdt.isDeleted(qualifier, 0, qualifier.length, timestamp); + assertEquals(DeleteResult.VERSION_DELETED, ret); + } + + public void testDeletedBy_DeleteColumn() { + byte [] qualifier = Bytes.toBytes("qualifier"); + deleteType = KeyValue.Type.DeleteColumn.getCode(); + + sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType); + timestamp -= 5; + DeleteResult ret = sdt.isDeleted(qualifier, 0, qualifier.length, timestamp); + assertEquals(DeleteResult.COLUMN_DELETED, ret); + } + + public void testDeletedBy_DeleteFamily() { + byte [] qualifier = Bytes.toBytes("qualifier"); + deleteType = KeyValue.Type.DeleteFamily.getCode(); + + sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType); + + timestamp -= 5; + DeleteResult ret = sdt.isDeleted(qualifier, 0, qualifier.length, timestamp); + assertEquals(DeleteResult.FAMILY_DELETED, ret); + } + + public void testDelete_DeleteColumn() { + byte [] qualifier = Bytes.toBytes("qualifier"); + deleteType = KeyValue.Type.Delete.getCode(); + + sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType); + + timestamp -= 5; + deleteType = KeyValue.Type.DeleteColumn.getCode(); + sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType); + + timestamp -= 5; + DeleteResult ret = sdt.isDeleted(qualifier, 0, qualifier.length, timestamp); + assertEquals(DeleteResult.COLUMN_DELETED, ret); + } + + + public void testDeleteColumn_Delete() { + byte [] qualifier = Bytes.toBytes("qualifier"); + deleteType = KeyValue.Type.DeleteColumn.getCode(); + + sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType); + + qualifier = Bytes.toBytes("qualifier1"); + deleteType = KeyValue.Type.Delete.getCode(); + sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType); + + DeleteResult ret = sdt.isDeleted(qualifier, 0, qualifier.length, timestamp); + assertEquals( DeleteResult.VERSION_DELETED, ret); + } + + //Testing new way where we save the Delete in case of a Delete for specific + //ts, could have just added the last line to the first test, but rather keep + //them separated + public void testDelete_KeepDelete(){ + byte [] qualifier = Bytes.toBytes("qualifier"); + deleteType = KeyValue.Type.Delete.getCode(); + + sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType); + sdt.isDeleted(qualifier, 0, qualifier.length, timestamp); + assertEquals(false ,sdt.isEmpty()); + } + + public void testDelete_KeepVersionZero(){ + byte [] qualifier = Bytes.toBytes("qualifier"); + deleteType = KeyValue.Type.Delete.getCode(); + + long deleteTimestamp = 10; + long valueTimestamp = 0; + + sdt.reset(); + sdt.add(qualifier, 0, qualifier.length, deleteTimestamp, deleteType); + DeleteResult ret = sdt.isDeleted(qualifier, 0, qualifier.length, valueTimestamp); + assertEquals(DeleteResult.NOT_DELETED, ret); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java new file mode 100644 index 0000000..086a408 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java @@ -0,0 +1,130 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestScanWildcardColumnTracker extends HBaseTestCase { + + final static int VERSIONS = 2; + + public void testCheckColumn_Ok() throws IOException { + ScanWildcardColumnTracker tracker = + new ScanWildcardColumnTracker(0, VERSIONS, Long.MIN_VALUE); + + //Create list of qualifiers + List qualifiers = new ArrayList(); + qualifiers.add(Bytes.toBytes("qualifer1")); + qualifiers.add(Bytes.toBytes("qualifer2")); + qualifiers.add(Bytes.toBytes("qualifer3")); + qualifiers.add(Bytes.toBytes("qualifer4")); + + //Setting up expected result + List expected = new ArrayList(); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); + + List actual = new ArrayList(); + + for(byte [] qualifier : qualifiers) { + ScanQueryMatcher.MatchCode mc = tracker.checkColumn(qualifier, 0, + qualifier.length, 1, KeyValue.Type.Put.getCode(), false); + actual.add(mc); + } + + //Compare actual with expected + for(int i=0; i qualifiers = new ArrayList(); + qualifiers.add(Bytes.toBytes("qualifer1")); + qualifiers.add(Bytes.toBytes("qualifer1")); + qualifiers.add(Bytes.toBytes("qualifer1")); + qualifiers.add(Bytes.toBytes("qualifer2")); + + //Setting up expected result + List expected = new ArrayList(); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); + expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); + expected.add(ScanQueryMatcher.MatchCode.INCLUDE); + + List actual = new ArrayList(); + + long timestamp = 0; + for(byte [] qualifier : qualifiers) { + MatchCode mc = tracker.checkColumn(qualifier, 0, qualifier.length, + ++timestamp, KeyValue.Type.Put.getCode(), false); + actual.add(mc); + } + + //Compare actual with expected + for(int i=0; i qualifiers = new ArrayList(); + qualifiers.add(Bytes.toBytes("qualifer2")); + qualifiers.add(Bytes.toBytes("qualifer1")); + + boolean ok = false; + + try { + for(byte [] qualifier : qualifiers) { + tracker.checkColumn(qualifier, 0, qualifier.length, 1, + KeyValue.Type.Put.getCode(), false); + } + } catch (Exception e) { + ok = true; + } + + assertEquals(true, ok); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java new file mode 100644 index 0000000..bdb1231 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java @@ -0,0 +1,221 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueTestUtil; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter; +import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import static org.junit.Assert.*; + +/** + * Test a multi-column scanner when there is a Bloom filter false-positive. + * This is needed for the multi-column Bloom filter optimization. + */ +@RunWith(Parameterized.class) +@Category(SmallTests.class) +public class TestScanWithBloomError { + + private static final Log LOG = + LogFactory.getLog(TestScanWithBloomError.class); + + private static final String TABLE_NAME = "ScanWithBloomError"; + private static final String FAMILY = "myCF"; + private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY); + private static final String ROW = "theRow"; + private static final String QUALIFIER_PREFIX = "qual"; + private static final byte[] ROW_BYTES = Bytes.toBytes(ROW); + private static NavigableSet allColIds = new TreeSet(); + private HRegion region; + private StoreFile.BloomType bloomType; + private FileSystem fs; + private Configuration conf; + + private final static HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + @Parameters + public static final Collection parameters() { + List configurations = new ArrayList(); + for (StoreFile.BloomType bloomType : StoreFile.BloomType.values()) { + configurations.add(new Object[] { bloomType }); + } + return configurations; + } + + public TestScanWithBloomError(StoreFile.BloomType bloomType) { + this.bloomType = bloomType; + } + + @Before + public void setUp() throws IOException{ + conf = TEST_UTIL.getConfiguration(); + fs = FileSystem.get(conf); + } + + @Test + public void testThreeStoreFiles() throws IOException { + region = TEST_UTIL.createTestRegion(TABLE_NAME, + new HColumnDescriptor(FAMILY) + .setCompressionType(Compression.Algorithm.GZ) + .setBloomFilterType(bloomType) + .setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS)); + createStoreFile(new int[] {1, 2, 6}); + createStoreFile(new int[] {1, 2, 3, 7}); + createStoreFile(new int[] {1, 9}); + scanColSet(new int[]{1, 4, 6, 7}, new int[]{1, 6, 7}); + + region.close(); + region.getLog().closeAndDelete(); + } + + private void scanColSet(int[] colSet, int[] expectedResultCols) + throws IOException { + LOG.info("Scanning column set: " + Arrays.toString(colSet)); + Scan scan = new Scan(ROW_BYTES, ROW_BYTES); + addColumnSetToScan(scan, colSet); + RegionScannerImpl scanner = (RegionScannerImpl) region.getScanner(scan); + KeyValueHeap storeHeap = scanner.getStoreHeapForTesting(); + assertEquals(0, storeHeap.getHeap().size()); + StoreScanner storeScanner = + (StoreScanner) storeHeap.getCurrentForTesting(); + @SuppressWarnings({ "unchecked", "rawtypes" }) + List scanners = (List) + (List) storeScanner.getAllScannersForTesting(); + + // Sort scanners by their HFile's modification time. + Collections.sort(scanners, new Comparator() { + @Override + public int compare(StoreFileScanner s1, StoreFileScanner s2) { + Path p1 = s1.getReaderForTesting().getHFileReader().getPath(); + Path p2 = s2.getReaderForTesting().getHFileReader().getPath(); + long t1, t2; + try { + t1 = fs.getFileStatus(p1).getModificationTime(); + t2 = fs.getFileStatus(p2).getModificationTime(); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + return t1 < t2 ? -1 : t1 == t2 ? 1 : 0; + } + }); + + StoreFile.Reader lastStoreFileReader = null; + for (StoreFileScanner sfScanner : scanners) + lastStoreFileReader = sfScanner.getReaderForTesting(); + + new HFilePrettyPrinter().run(new String[]{ "-m", "-p", "-f", + lastStoreFileReader.getHFileReader().getPath().toString()}); + + // Disable Bloom filter for the last store file. The disabled Bloom filter + // will always return "true". + LOG.info("Disabling Bloom filter for: " + + lastStoreFileReader.getHFileReader().getName()); + lastStoreFileReader.disableBloomFilterForTesting(); + + List allResults = new ArrayList(); + + { // Limit the scope of results. + List results = new ArrayList(); + while (scanner.next(results) || results.size() > 0) { + allResults.addAll(results); + results.clear(); + } + } + + List actualIds = new ArrayList(); + for (KeyValue kv : allResults) { + String qual = Bytes.toString(kv.getQualifier()); + assertTrue(qual.startsWith(QUALIFIER_PREFIX)); + actualIds.add(Integer.valueOf(qual.substring( + QUALIFIER_PREFIX.length()))); + } + List expectedIds = new ArrayList(); + for (int expectedId : expectedResultCols) + expectedIds.add(expectedId); + + LOG.info("Column ids returned: " + actualIds + ", expected: " + + expectedIds); + assertEquals(expectedIds.toString(), actualIds.toString()); + } + + private void addColumnSetToScan(Scan scan, int[] colIds) { + for (int colId : colIds) { + scan.addColumn(FAMILY_BYTES, + Bytes.toBytes(qualFromId(colId))); + } + } + + private String qualFromId(int colId) { + return QUALIFIER_PREFIX + colId; + } + + private void createStoreFile(int[] colIds) + throws IOException { + Put p = new Put(ROW_BYTES); + for (int colId : colIds) { + long ts = Long.MAX_VALUE; + String qual = qualFromId(colId); + allColIds.add(colId); + KeyValue kv = KeyValueTestUtil.create(ROW, FAMILY, + qual, ts, TestMultiColumnScanner.createValue(ROW, qual, ts)); + p.add(kv); + } + region.put(p); + region.flushcache(); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java new file mode 100644 index 0000000..32e8d18 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java @@ -0,0 +1,610 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.UnknownScannerException; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.InclusiveStopFilter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.WhileMatchFilter; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.junit.experimental.categories.Category; + +/** + * Test of a long-lived scanner validating as we go. + */ +@Category(SmallTests.class) +public class TestScanner extends HBaseTestCase { + private final Log LOG = LogFactory.getLog(this.getClass()); + + private static final byte [] FIRST_ROW = HConstants.EMPTY_START_ROW; + private static final byte [][] COLS = { HConstants.CATALOG_FAMILY }; + private static final byte [][] EXPLICIT_COLS = { + HConstants.REGIONINFO_QUALIFIER, HConstants.SERVER_QUALIFIER, + // TODO ryan + //HConstants.STARTCODE_QUALIFIER + }; + + static final HTableDescriptor TESTTABLEDESC = + new HTableDescriptor("testscanner"); + static { + TESTTABLEDESC.addFamily( + new HColumnDescriptor(HConstants.CATALOG_FAMILY) + // Ten is an arbitrary number. Keep versions to help debugging. + .setMaxVersions(10) + .setBlockCacheEnabled(false) + .setBlocksize(8 * 1024) + ); + } + /** HRegionInfo for root region */ + public static final HRegionInfo REGION_INFO = + new HRegionInfo(TESTTABLEDESC.getName(), HConstants.EMPTY_BYTE_ARRAY, + HConstants.EMPTY_BYTE_ARRAY); + + private static final byte [] ROW_KEY = REGION_INFO.getRegionName(); + + private static final long START_CODE = Long.MAX_VALUE; + + private HRegion r; + private HRegionIncommon region; + + private byte[] firstRowBytes, secondRowBytes, thirdRowBytes; + final private byte[] col1, col2; + + public TestScanner() throws Exception { + super(); + + firstRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING); + secondRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING); + // Increment the least significant character so we get to next row. + secondRowBytes[START_KEY_BYTES.length - 1]++; + thirdRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING); + thirdRowBytes[START_KEY_BYTES.length - 1]++; + thirdRowBytes[START_KEY_BYTES.length - 1]++; + col1 = "column1".getBytes(HConstants.UTF8_ENCODING); + col2 = "column2".getBytes(HConstants.UTF8_ENCODING); + } + + /** + * Test basic stop row filter works. + * @throws Exception + */ + public void testStopRow() throws Exception { + byte [] startrow = Bytes.toBytes("bbb"); + byte [] stoprow = Bytes.toBytes("ccc"); + try { + this.r = createNewHRegion(TESTTABLEDESC, null, null); + addContent(this.r, HConstants.CATALOG_FAMILY); + List results = new ArrayList(); + // Do simple test of getting one row only first. + Scan scan = new Scan(Bytes.toBytes("abc"), Bytes.toBytes("abd")); + scan.addFamily(HConstants.CATALOG_FAMILY); + + InternalScanner s = r.getScanner(scan); + int count = 0; + while (s.next(results)) { + count++; + } + s.close(); + assertEquals(0, count); + // Now do something a bit more imvolved. + scan = new Scan(startrow, stoprow); + scan.addFamily(HConstants.CATALOG_FAMILY); + + s = r.getScanner(scan); + count = 0; + KeyValue kv = null; + results = new ArrayList(); + for (boolean first = true; s.next(results);) { + kv = results.get(0); + if (first) { + assertTrue(Bytes.BYTES_COMPARATOR.compare(startrow, kv.getRow()) == 0); + first = false; + } + count++; + } + assertTrue(Bytes.BYTES_COMPARATOR.compare(stoprow, kv.getRow()) > 0); + // We got something back. + assertTrue(count > 10); + s.close(); + } finally { + this.r.close(); + this.r.getLog().closeAndDelete(); + } + } + + void rowPrefixFilter(Scan scan) throws IOException { + List results = new ArrayList(); + scan.addFamily(HConstants.CATALOG_FAMILY); + InternalScanner s = r.getScanner(scan); + boolean hasMore = true; + while (hasMore) { + hasMore = s.next(results); + for (KeyValue kv : results) { + assertEquals((byte)'a', kv.getRow()[0]); + assertEquals((byte)'b', kv.getRow()[1]); + } + results.clear(); + } + s.close(); + } + + void rowInclusiveStopFilter(Scan scan, byte[] stopRow) throws IOException { + List results = new ArrayList(); + scan.addFamily(HConstants.CATALOG_FAMILY); + InternalScanner s = r.getScanner(scan); + boolean hasMore = true; + while (hasMore) { + hasMore = s.next(results); + for (KeyValue kv : results) { + assertTrue(Bytes.compareTo(kv.getRow(), stopRow) <= 0); + } + results.clear(); + } + s.close(); + } + + public void testFilters() throws IOException { + try { + this.r = createNewHRegion(TESTTABLEDESC, null, null); + addContent(this.r, HConstants.CATALOG_FAMILY); + byte [] prefix = Bytes.toBytes("ab"); + Filter newFilter = new PrefixFilter(prefix); + Scan scan = new Scan(); + scan.setFilter(newFilter); + rowPrefixFilter(scan); + + byte[] stopRow = Bytes.toBytes("bbc"); + newFilter = new WhileMatchFilter(new InclusiveStopFilter(stopRow)); + scan = new Scan(); + scan.setFilter(newFilter); + rowInclusiveStopFilter(scan, stopRow); + + } finally { + this.r.close(); + this.r.getLog().closeAndDelete(); + } + } + + /** + * Test that closing a scanner while a client is using it doesn't throw + * NPEs but instead a UnknownScannerException. HBASE-2503 + * @throws Exception + */ + public void testRaceBetweenClientAndTimeout() throws Exception { + try { + this.r = createNewHRegion(TESTTABLEDESC, null, null); + addContent(this.r, HConstants.CATALOG_FAMILY); + Scan scan = new Scan(); + InternalScanner s = r.getScanner(scan); + List results = new ArrayList(); + try { + s.next(results); + s.close(); + s.next(results); + fail("We don't want anything more, we should be failing"); + } catch (UnknownScannerException ex) { + // ok! + return; + } + } finally { + this.r.close(); + this.r.getLog().closeAndDelete(); + } + } + + /** The test! + * @throws IOException + */ + public void testScanner() throws IOException { + try { + r = createNewHRegion(TESTTABLEDESC, null, null); + region = new HRegionIncommon(r); + + // Write information to the meta table + + Put put = new Put(ROW_KEY, System.currentTimeMillis(), null); + + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + DataOutputStream s = new DataOutputStream(byteStream); + REGION_INFO.write(s); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + byteStream.toByteArray()); + region.put(put); + + // What we just committed is in the memstore. Verify that we can get + // it back both with scanning and get + + scan(false, null); + getRegionInfo(); + + // Close and re-open + + r.close(); + r = openClosedRegion(r); + region = new HRegionIncommon(r); + + // Verify we can get the data back now that it is on disk. + + scan(false, null); + getRegionInfo(); + + // Store some new information + + String address = "www.example.com:1234"; + + put = new Put(ROW_KEY, System.currentTimeMillis(), null); + put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, + Bytes.toBytes(address)); + +// put.add(HConstants.COL_STARTCODE, Bytes.toBytes(START_CODE)); + + region.put(put); + + // Validate that we can still get the HRegionInfo, even though it is in + // an older row on disk and there is a newer row in the memstore + + scan(true, address.toString()); + getRegionInfo(); + + // flush cache + + region.flushcache(); + + // Validate again + + scan(true, address.toString()); + getRegionInfo(); + + // Close and reopen + + r.close(); + r = openClosedRegion(r); + region = new HRegionIncommon(r); + + // Validate again + + scan(true, address.toString()); + getRegionInfo(); + + // Now update the information again + + address = "bar.foo.com:4321"; + + put = new Put(ROW_KEY, System.currentTimeMillis(), null); + + put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, + Bytes.toBytes(address)); + region.put(put); + + // Validate again + + scan(true, address.toString()); + getRegionInfo(); + + // flush cache + + region.flushcache(); + + // Validate again + + scan(true, address.toString()); + getRegionInfo(); + + // Close and reopen + + r.close(); + r = openClosedRegion(r); + region = new HRegionIncommon(r); + + // Validate again + + scan(true, address.toString()); + getRegionInfo(); + + } finally { + // clean up + r.close(); + r.getLog().closeAndDelete(); + } + } + + /** Compare the HRegionInfo we read from HBase to what we stored */ + private void validateRegionInfo(byte [] regionBytes) throws IOException { + HRegionInfo info = + (HRegionInfo) Writables.getWritable(regionBytes, new HRegionInfo()); + + assertEquals(REGION_INFO.getRegionId(), info.getRegionId()); + assertEquals(0, info.getStartKey().length); + assertEquals(0, info.getEndKey().length); + assertEquals(0, Bytes.compareTo(info.getRegionName(), REGION_INFO.getRegionName())); + //assertEquals(0, info.getTableDesc().compareTo(REGION_INFO.getTableDesc())); + } + + /** Use a scanner to get the region info and then validate the results */ + private void scan(boolean validateStartcode, String serverName) + throws IOException { + InternalScanner scanner = null; + Scan scan = null; + List results = new ArrayList(); + byte [][][] scanColumns = { + COLS, + EXPLICIT_COLS + }; + + for(int i = 0; i < scanColumns.length; i++) { + try { + scan = new Scan(FIRST_ROW); + for (int ii = 0; ii < EXPLICIT_COLS.length; ii++) { + scan.addColumn(COLS[0], EXPLICIT_COLS[ii]); + } + scanner = r.getScanner(scan); + while (scanner.next(results)) { + assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER)); + byte [] val = getColumn(results, HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER).getValue(); + validateRegionInfo(val); + if(validateStartcode) { +// assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY, +// HConstants.STARTCODE_QUALIFIER)); +// val = getColumn(results, HConstants.CATALOG_FAMILY, +// HConstants.STARTCODE_QUALIFIER).getValue(); + assertNotNull(val); + assertFalse(val.length == 0); + long startCode = Bytes.toLong(val); + assertEquals(START_CODE, startCode); + } + + if(serverName != null) { + assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER)); + val = getColumn(results, HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER).getValue(); + assertNotNull(val); + assertFalse(val.length == 0); + String server = Bytes.toString(val); + assertEquals(0, server.compareTo(serverName)); + } + } + } finally { + InternalScanner s = scanner; + scanner = null; + if(s != null) { + s.close(); + } + } + } + } + + private boolean hasColumn(final List kvs, final byte [] family, + final byte [] qualifier) { + for (KeyValue kv: kvs) { + if (kv.matchingFamily(family) && kv.matchingQualifier(qualifier)) { + return true; + } + } + return false; + } + + private KeyValue getColumn(final List kvs, final byte [] family, + final byte [] qualifier) { + for (KeyValue kv: kvs) { + if (kv.matchingFamily(family) && kv.matchingQualifier(qualifier)) { + return kv; + } + } + return null; + } + + + /** Use get to retrieve the HRegionInfo and validate it */ + private void getRegionInfo() throws IOException { + Get get = new Get(ROW_KEY); + get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + Result result = region.get(get, null); + byte [] bytes = result.value(); + validateRegionInfo(bytes); + } + + /** + * Tests to do a sync flush during the middle of a scan. This is testing the StoreScanner + * update readers code essentially. This is not highly concurrent, since its all 1 thread. + * HBase-910. + * @throws Exception + */ + public void testScanAndSyncFlush() throws Exception { + this.r = createNewHRegion(TESTTABLEDESC, null, null); + HRegionIncommon hri = new HRegionIncommon(r); + try { + LOG.info("Added: " + addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY), + Bytes.toString(HConstants.REGIONINFO_QUALIFIER))); + int count = count(hri, -1, false); + assertEquals(count, count(hri, 100, false)); // do a sync flush. + } catch (Exception e) { + LOG.error("Failed", e); + throw e; + } finally { + this.r.close(); + this.r.getLog().closeAndDelete(); + } + } + + /** + * Tests to do a concurrent flush (using a 2nd thread) while scanning. This tests both + * the StoreScanner update readers and the transition from memstore -> snapshot -> store file. + * + * @throws Exception + */ + public void testScanAndRealConcurrentFlush() throws Exception { + this.r = createNewHRegion(TESTTABLEDESC, null, null); + HRegionIncommon hri = new HRegionIncommon(r); + try { + LOG.info("Added: " + addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY), + Bytes.toString(HConstants.REGIONINFO_QUALIFIER))); + int count = count(hri, -1, false); + assertEquals(count, count(hri, 100, true)); // do a true concurrent background thread flush + } catch (Exception e) { + LOG.error("Failed", e); + throw e; + } finally { + this.r.close(); + this.r.getLog().closeAndDelete(); + } + } + + /** + * Make sure scanner returns correct result when we run a major compaction + * with deletes. + * + * @throws Exception + */ + @SuppressWarnings("deprecation") + public void testScanAndConcurrentMajorCompact() throws Exception { + HTableDescriptor htd = createTableDescriptor(getName()); + this.r = createNewHRegion(htd, null, null); + HRegionIncommon hri = new HRegionIncommon(r); + + try { + addContent(hri, Bytes.toString(fam1), Bytes.toString(col1), + firstRowBytes, secondRowBytes); + addContent(hri, Bytes.toString(fam2), Bytes.toString(col1), + firstRowBytes, secondRowBytes); + + Delete dc = new Delete(firstRowBytes); + /* delete column1 of firstRow */ + dc.deleteColumns(fam1, col1); + r.delete(dc, null, true); + r.flushcache(); + + addContent(hri, Bytes.toString(fam1), Bytes.toString(col1), + secondRowBytes, thirdRowBytes); + addContent(hri, Bytes.toString(fam2), Bytes.toString(col1), + secondRowBytes, thirdRowBytes); + r.flushcache(); + + InternalScanner s = r.getScanner(new Scan()); + // run a major compact, column1 of firstRow will be cleaned. + r.compactStores(true); + + List results = new ArrayList(); + s.next(results); + + // make sure returns column2 of firstRow + assertTrue("result is not correct, keyValues : " + results, + results.size() == 1); + assertTrue(Bytes.BYTES_COMPARATOR.compare(firstRowBytes, results.get(0) + .getRow()) == 0); + assertTrue(Bytes.BYTES_COMPARATOR.compare(fam2, results.get(0) + .getFamily()) == 0); + + results = new ArrayList(); + s.next(results); + + // get secondRow + assertTrue(results.size() == 2); + assertTrue(Bytes.BYTES_COMPARATOR.compare(secondRowBytes, results.get(0) + .getRow()) == 0); + assertTrue(Bytes.BYTES_COMPARATOR.compare(fam1, results.get(0) + .getFamily()) == 0); + assertTrue(Bytes.BYTES_COMPARATOR.compare(fam2, results.get(1) + .getFamily()) == 0); + } finally { + this.r.close(); + this.r.getLog().closeAndDelete(); + } + } + + + /* + * @param hri Region + * @param flushIndex At what row we start the flush. + * @param concurrent if the flush should be concurrent or sync. + * @return Count of rows found. + * @throws IOException + */ + private int count(final HRegionIncommon hri, final int flushIndex, + boolean concurrent) + throws IOException { + LOG.info("Taking out counting scan"); + ScannerIncommon s = hri.getScanner(HConstants.CATALOG_FAMILY, EXPLICIT_COLS, + HConstants.EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP); + List values = new ArrayList(); + int count = 0; + boolean justFlushed = false; + while (s.next(values)) { + if (justFlushed) { + LOG.info("after next() just after next flush"); + justFlushed=false; + } + count++; + if (flushIndex == count) { + LOG.info("Starting flush at flush index " + flushIndex); + Thread t = new Thread() { + public void run() { + try { + hri.flushcache(); + LOG.info("Finishing flush"); + } catch (IOException e) { + LOG.info("Failed flush cache"); + } + } + }; + if (concurrent) { + t.start(); // concurrently flush. + } else { + t.run(); // sync flush + } + LOG.info("Continuing on after kicking off background flush"); + justFlushed = true; + } + } + s.close(); + LOG.info("Found " + count + " items"); + return count; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java new file mode 100644 index 0000000..2a092e7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java @@ -0,0 +1,458 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.apache.hadoop.hbase.HBaseTestingUtility.assertKVListsEqual; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Test various seek optimizations for correctness and check if they are + * actually saving I/O operations. + */ +@RunWith(Parameterized.class) +@Category(MediumTests.class) +public class TestSeekOptimizations { + + private static final Log LOG = + LogFactory.getLog(TestSeekOptimizations.class); + + // Constants + private static final String FAMILY = "myCF"; + private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY); + + private static final int PUTS_PER_ROW_COL = 50; + private static final int DELETES_PER_ROW_COL = 10; + + private static final int NUM_ROWS = 3; + private static final int NUM_COLS = 3; + + private static final boolean VERBOSE = false; + + /** + * Disable this when this test fails hopelessly and you need to debug a + * simpler case. + */ + private static final boolean USE_MANY_STORE_FILES = true; + + private static final int[][] COLUMN_SETS = new int[][] { + {}, // All columns + {0}, + {1}, + {0, 2}, + {1, 2}, + {0, 1, 2}, + }; + + // Both start row and end row are inclusive here for the purposes of this + // test. + private static final int[][] ROW_RANGES = new int[][] { + {-1, -1}, + {0, 1}, + {1, 1}, + {1, 2}, + {0, 2} + }; + + private static final int[] MAX_VERSIONS_VALUES = new int[] { 1, 2 }; + + // Instance variables + private HRegion region; + private Put put; + private Delete del; + private Random rand; + private Set putTimestamps = new HashSet(); + private Set delTimestamps = new HashSet(); + private List expectedKVs = new ArrayList(); + + private Compression.Algorithm comprAlgo; + private StoreFile.BloomType bloomType; + + private long totalSeekDiligent, totalSeekLazy; + + private final static HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + @Parameters + public static final Collection parameters() { + return HBaseTestingUtility.BLOOM_AND_COMPRESSION_COMBINATIONS; + } + + public TestSeekOptimizations(Compression.Algorithm comprAlgo, + StoreFile.BloomType bloomType) { + this.comprAlgo = comprAlgo; + this.bloomType = bloomType; + } + + @Before + public void setUp() { + rand = new Random(91238123L); + expectedKVs.clear(); + } + + @Test + public void testMultipleTimestampRanges() throws IOException { + region = TEST_UTIL.createTestRegion(TestSeekOptimizations.class.getName(), + new HColumnDescriptor(FAMILY) + .setCompressionType(comprAlgo) + .setBloomFilterType(bloomType) + ); + + // Delete the given timestamp and everything before. + final long latestDelTS = USE_MANY_STORE_FILES ? 1397 : -1; + + createTimestampRange(1, 50, -1); + createTimestampRange(51, 100, -1); + if (USE_MANY_STORE_FILES) { + createTimestampRange(100, 500, 127); + createTimestampRange(900, 1300, -1); + createTimestampRange(1301, 2500, latestDelTS); + createTimestampRange(2502, 2598, -1); + createTimestampRange(2599, 2999, -1); + } + + prepareExpectedKVs(latestDelTS); + + for (int[] columnArr : COLUMN_SETS) { + for (int[] rowRange : ROW_RANGES) { + for (int maxVersions : MAX_VERSIONS_VALUES) { + for (boolean lazySeekEnabled : new boolean[] { false, true }) { + testScan(columnArr, lazySeekEnabled, rowRange[0], rowRange[1], + maxVersions); + } + } + } + } + + final double seekSavings = 1 - totalSeekLazy * 1.0 / totalSeekDiligent; + System.err.println("For bloom=" + bloomType + ", compr=" + comprAlgo + + " total seeks without optimization: " + totalSeekDiligent + + ", with optimization: " + totalSeekLazy + " (" + + String.format("%.2f%%", totalSeekLazy * 100.0 / totalSeekDiligent) + + "), savings: " + String.format("%.2f%%", + 100.0 * seekSavings) + "\n"); + + // Test that lazy seeks are buying us something. Without the actual + // implementation of the lazy seek optimization this will be 0. + final double expectedSeekSavings = 0.0; + assertTrue("Lazy seek is only saving " + + String.format("%.2f%%", seekSavings * 100) + " seeks but should " + + "save at least " + String.format("%.2f%%", expectedSeekSavings * 100), + seekSavings >= expectedSeekSavings); + } + + private void testScan(final int[] columnArr, final boolean lazySeekEnabled, + final int startRow, final int endRow, int maxVersions) + throws IOException { + StoreScanner.enableLazySeekGlobally(lazySeekEnabled); + final Scan scan = new Scan(); + final Set qualSet = new HashSet(); + for (int iColumn : columnArr) { + String qualStr = getQualStr(iColumn); + scan.addColumn(FAMILY_BYTES, Bytes.toBytes(qualStr)); + qualSet.add(qualStr); + } + scan.setMaxVersions(maxVersions); + scan.setStartRow(rowBytes(startRow)); + + // Adjust for the fact that for multi-row queries the end row is exclusive. + { + final byte[] scannerStopRow = + rowBytes(endRow + (startRow != endRow ? 1 : 0)); + scan.setStopRow(scannerStopRow); + } + + final long initialSeekCount = StoreFileScanner.getSeekCount(); + final InternalScanner scanner = region.getScanner(scan); + final List results = new ArrayList(); + final List actualKVs = new ArrayList(); + + // Such a clumsy do-while loop appears to be the official way to use an + // internalScanner. scanner.next() return value refers to the _next_ + // result, not to the one already returned in results. + boolean hasNext; + do { + hasNext = scanner.next(results); + actualKVs.addAll(results); + results.clear(); + } while (hasNext); + + List filteredKVs = filterExpectedResults(qualSet, + rowBytes(startRow), rowBytes(endRow), maxVersions); + final String rowRestrictionStr = + (startRow == -1 && endRow == -1) ? "all rows" : ( + startRow == endRow ? ("row=" + startRow) : ("startRow=" + + startRow + ", " + "endRow=" + endRow)); + final String columnRestrictionStr = + columnArr.length == 0 ? "all columns" + : ("columns=" + Arrays.toString(columnArr)); + final String testDesc = + "Bloom=" + bloomType + ", compr=" + comprAlgo + ", " + + (scan.isGetScan() ? "Get" : "Scan") + ": " + + columnRestrictionStr + ", " + rowRestrictionStr + + ", maxVersions=" + maxVersions + ", lazySeek=" + lazySeekEnabled; + long seekCount = StoreFileScanner.getSeekCount() - initialSeekCount; + if (VERBOSE) { + System.err.println("Seek count: " + seekCount + ", KVs returned: " + + actualKVs.size() + ". " + testDesc + + (lazySeekEnabled ? "\n" : "")); + } + if (lazySeekEnabled) { + totalSeekLazy += seekCount; + } else { + totalSeekDiligent += seekCount; + } + assertKVListsEqual(testDesc, filteredKVs, actualKVs); + } + + private List filterExpectedResults(Set qualSet, + byte[] startRow, byte[] endRow, int maxVersions) { + final List filteredKVs = new ArrayList(); + final Map verCount = new HashMap(); + for (KeyValue kv : expectedKVs) { + if (startRow.length > 0 && + Bytes.compareTo(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(), + startRow, 0, startRow.length) < 0) { + continue; + } + + // In this unit test the end row is always inclusive. + if (endRow.length > 0 && + Bytes.compareTo(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(), + endRow, 0, endRow.length) > 0) { + continue; + } + + if (!qualSet.isEmpty() && (Bytes.compareTo( + kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(), + FAMILY_BYTES, 0, FAMILY_BYTES.length + ) != 0 || + !qualSet.contains(Bytes.toString(kv.getQualifier())))) { + continue; + } + + final String rowColStr = + Bytes.toStringBinary(kv.getRow()) + "/" + + Bytes.toStringBinary(kv.getFamily()) + ":" + + Bytes.toStringBinary(kv.getQualifier()); + final Integer curNumVer = verCount.get(rowColStr); + final int newNumVer = curNumVer != null ? (curNumVer + 1) : 1; + if (newNumVer <= maxVersions) { + filteredKVs.add(kv); + verCount.put(rowColStr, newNumVer); + } + } + + return filteredKVs; + } + + private void prepareExpectedKVs(long latestDelTS) { + final List filteredKVs = new ArrayList(); + for (KeyValue kv : expectedKVs) { + if (kv.getTimestamp() > latestDelTS || latestDelTS == -1) { + filteredKVs.add(kv); + } + } + expectedKVs = filteredKVs; + Collections.sort(expectedKVs, KeyValue.COMPARATOR); + } + + public void put(String qual, long ts) { + if (!putTimestamps.contains(ts)) { + put.add(FAMILY_BYTES, Bytes.toBytes(qual), ts, createValue(ts)); + putTimestamps.add(ts); + } + if (VERBOSE) { + LOG.info("put: row " + Bytes.toStringBinary(put.getRow()) + + ", cf " + FAMILY + ", qualifier " + qual + ", ts " + ts); + } + } + + private byte[] createValue(long ts) { + return Bytes.toBytes("value" + ts); + } + + public void delAtTimestamp(String qual, long ts) { + del.deleteColumn(FAMILY_BYTES, Bytes.toBytes(qual), ts); + logDelete(qual, ts, "at"); + } + + private void logDelete(String qual, long ts, String delType) { + if (VERBOSE) { + LOG.info("del " + delType + ": row " + + Bytes.toStringBinary(put.getRow()) + ", cf " + FAMILY + + ", qualifier " + qual + ", ts " + ts); + } + } + + private void delUpToTimestamp(String qual, long upToTS) { + del.deleteColumns(FAMILY_BYTES, Bytes.toBytes(qual), upToTS); + logDelete(qual, upToTS, "up to and including"); + } + + private long randLong(long n) { + long l = rand.nextLong(); + if (l == Long.MIN_VALUE) + l = Long.MAX_VALUE; + return Math.abs(l) % n; + } + + private long randBetween(long a, long b) { + long x = a + randLong(b - a + 1); + assertTrue(a <= x && x <= b); + return x; + } + + private final String rowStr(int i) { + return ("row" + i).intern(); + } + + private final byte[] rowBytes(int i) { + if (i == -1) { + return HConstants.EMPTY_BYTE_ARRAY; + } + return Bytes.toBytes(rowStr(i)); + } + + private final String getQualStr(int i) { + return ("qual" + i).intern(); + } + + public void createTimestampRange(long minTS, long maxTS, + long deleteUpToTS) throws IOException { + assertTrue(minTS < maxTS); + assertTrue(deleteUpToTS == -1 + || (minTS <= deleteUpToTS && deleteUpToTS <= maxTS)); + + for (int iRow = 0; iRow < NUM_ROWS; ++iRow) { + final String row = rowStr(iRow); + final byte[] rowBytes = Bytes.toBytes(row); + for (int iCol = 0; iCol < NUM_COLS; ++iCol) { + final String qual = getQualStr(iCol); + final byte[] qualBytes = Bytes.toBytes(qual); + put = new Put(rowBytes); + + putTimestamps.clear(); + put(qual, minTS); + put(qual, maxTS); + for (int i = 0; i < PUTS_PER_ROW_COL; ++i) { + put(qual, randBetween(minTS, maxTS)); + } + + long[] putTimestampList = new long[putTimestamps.size()]; + { + int i = 0; + for (long ts : putTimestamps) { + putTimestampList[i++] = ts; + } + } + + // Delete a predetermined number of particular timestamps + delTimestamps.clear(); + assertTrue(putTimestampList.length >= DELETES_PER_ROW_COL); + int numToDel = DELETES_PER_ROW_COL; + int tsRemaining = putTimestampList.length; + del = new Delete(rowBytes); + for (long ts : putTimestampList) { + if (rand.nextInt(tsRemaining) < numToDel) { + delAtTimestamp(qual, ts); + putTimestamps.remove(ts); + --numToDel; + } + + if (--tsRemaining == 0) { + break; + } + } + + // Another type of delete: everything up to the given timestamp. + if (deleteUpToTS != -1) { + delUpToTimestamp(qual, deleteUpToTS); + } + + region.put(put); + if (!del.isEmpty()) { + region.delete(del, null, true); + } + + // Add remaining timestamps (those we have not deleted) to expected + // results + for (long ts : putTimestamps) { + expectedKVs.add(new KeyValue(rowBytes, FAMILY_BYTES, qualBytes, ts, + KeyValue.Type.Put)); + } + } + } + + region.flushcache(); + } + + @After + public void tearDown() throws IOException { + if (region != null) { + region.close(); + region.getLog().closeAndDelete(); + } + + // We have to re-set the lazy seek flag back to the default so that other + // unit tests are not affected. + StoreScanner.enableLazySeekGlobally( + StoreScanner.LAZY_SEEK_ENABLED_BY_DEFAULT); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java new file mode 100644 index 0000000..e99d251 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java @@ -0,0 +1,374 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Row; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; +import org.apache.hadoop.hbase.ipc.HMasterInterface; +import org.apache.hadoop.hbase.ipc.HMasterRegionInterface; +import org.apache.hadoop.hbase.ipc.ProtocolSignature; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.apache.hadoop.hbase.ipc.VersionedProtocol; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.collect.Lists; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestServerCustomProtocol { + /* Test protocol */ + public static interface PingProtocol extends CoprocessorProtocol { + public String ping(); + public int getPingCount(); + public int incrementCount(int diff); + public String hello(String name); + public void noop(); + } + + /* Test protocol implementation */ + public static class PingHandler implements Coprocessor, PingProtocol, VersionedProtocol { + static long VERSION = 1; + private int counter = 0; + @Override + public String ping() { + counter++; + return "pong"; + } + + @Override + public int getPingCount() { + return counter; + } + + @Override + public int incrementCount(int diff) { + counter += diff; + return counter; + } + + @Override + public String hello(String name) { + if (name == null) { + return "Who are you?"; + } else if ("nobody".equals(name)) { + return null; + } + return "Hello, "+name; + } + + @Override + public void noop() { + // do nothing, just test void return type + } + + @Override + public ProtocolSignature getProtocolSignature( + String protocol, long version, int clientMethodsHashCode) + throws IOException { + return new ProtocolSignature(VERSION, null); + } + + @Override + public long getProtocolVersion(String s, long l) throws IOException { + return VERSION; + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + } + + @Override + public void stop(CoprocessorEnvironment env) throws IOException { + } + } + + private static final byte[] TEST_TABLE = Bytes.toBytes("test"); + private static final byte[] TEST_FAMILY = Bytes.toBytes("f1"); + + private static final byte[] ROW_A = Bytes.toBytes("aaa"); + private static final byte[] ROW_B = Bytes.toBytes("bbb"); + private static final byte[] ROW_C = Bytes.toBytes("ccc"); + + private static final byte[] ROW_AB = Bytes.toBytes("abb"); + private static final byte[] ROW_BC = Bytes.toBytes("bcc"); + + private static HBaseTestingUtility util = new HBaseTestingUtility(); + private static MiniHBaseCluster cluster = null; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + util.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + PingHandler.class.getName()); + util.startMiniCluster(1); + cluster = util.getMiniHBaseCluster(); + + HTable table = util.createTable(TEST_TABLE, TEST_FAMILY); + util.createMultiRegions(util.getConfiguration(), table, TEST_FAMILY, + new byte[][]{ HConstants.EMPTY_BYTE_ARRAY, + ROW_B, ROW_C}); + + Put puta = new Put( ROW_A ); + puta.add(TEST_FAMILY, Bytes.toBytes("col1"), Bytes.toBytes(1)); + table.put(puta); + + Put putb = new Put( ROW_B ); + putb.add(TEST_FAMILY, Bytes.toBytes("col1"), Bytes.toBytes(1)); + table.put(putb); + + Put putc = new Put( ROW_C ); + putc.add(TEST_FAMILY, Bytes.toBytes("col1"), Bytes.toBytes(1)); + table.put(putc); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + util.shutdownMiniCluster(); + } + + @Test + public void testSingleProxy() throws Exception { + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + + PingProtocol pinger = table.coprocessorProxy(PingProtocol.class, ROW_A); + String result = pinger.ping(); + assertEquals("Invalid custom protocol response", "pong", result); + result = pinger.hello("George"); + assertEquals("Invalid custom protocol response", "Hello, George", result); + result = pinger.hello(null); + assertEquals("Should handle NULL parameter", "Who are you?", result); + result = pinger.hello("nobody"); + assertNull(result); + int cnt = pinger.getPingCount(); + assertTrue("Count should be incremented", cnt > 0); + int newcnt = pinger.incrementCount(5); + assertEquals("Counter should have incremented by 5", cnt+5, newcnt); + } + + @Test + public void testSingleMethod() throws Throwable { + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + + List rows = Lists.newArrayList( + new Get(ROW_A), new Get(ROW_B), new Get(ROW_C)); + + Batch.Call call = Batch.forMethod(PingProtocol.class, + "ping"); + Map results = + table.coprocessorExec(PingProtocol.class, ROW_A, ROW_C, call); + + + verifyRegionResults(table, results, ROW_A); + verifyRegionResults(table, results, ROW_B); + verifyRegionResults(table, results, ROW_C); + + Batch.Call helloCall = + Batch.forMethod(PingProtocol.class, "hello", "NAME"); + results = + table.coprocessorExec(PingProtocol.class, ROW_A, ROW_C, helloCall); + + + verifyRegionResults(table, results, "Hello, NAME", ROW_A); + verifyRegionResults(table, results, "Hello, NAME", ROW_B); + verifyRegionResults(table, results, "Hello, NAME", ROW_C); + } + + @Test + public void testRowRange() throws Throwable { + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + + // test empty range + Map results = table.coprocessorExec(PingProtocol.class, + null, null, new Batch.Call() { + public String call(PingProtocol instance) { + return instance.ping(); + } + }); + // should contain all three rows/regions + verifyRegionResults(table, results, ROW_A); + verifyRegionResults(table, results, ROW_B); + verifyRegionResults(table, results, ROW_C); + + // test start row + empty end + results = table.coprocessorExec(PingProtocol.class, ROW_BC, null, + new Batch.Call() { + public String call(PingProtocol instance) { + return instance.ping(); + } + }); + // should contain last 2 regions + HRegionLocation loc = table.getRegionLocation(ROW_A); + assertNull("Should be missing region for row aaa (prior to start row)", + results.get(loc.getRegionInfo().getRegionName())); + verifyRegionResults(table, results, ROW_B); + verifyRegionResults(table, results, ROW_C); + + // test empty start + end + results = table.coprocessorExec(PingProtocol.class, null, ROW_BC, + new Batch.Call() { + public String call(PingProtocol instance) { + return instance.ping(); + } + }); + // should contain the first 2 regions + verifyRegionResults(table, results, ROW_A); + verifyRegionResults(table, results, ROW_B); + loc = table.getRegionLocation(ROW_C); + assertNull("Should be missing region for row ccc (past stop row)", + results.get(loc.getRegionInfo().getRegionName())); + + // test explicit start + end + results = table.coprocessorExec(PingProtocol.class, ROW_AB, ROW_BC, + new Batch.Call() { + public String call(PingProtocol instance) { + return instance.ping(); + } + }); + // should contain first 2 regions + verifyRegionResults(table, results, ROW_A); + verifyRegionResults(table, results, ROW_B); + loc = table.getRegionLocation(ROW_C); + assertNull("Should be missing region for row ccc (past stop row)", + results.get(loc.getRegionInfo().getRegionName())); + + // test single region + results = table.coprocessorExec(PingProtocol.class, ROW_B, ROW_BC, + new Batch.Call() { + public String call(PingProtocol instance) { + return instance.ping(); + } + }); + // should only contain region bbb + verifyRegionResults(table, results, ROW_B); + loc = table.getRegionLocation(ROW_A); + assertNull("Should be missing region for row aaa (prior to start)", + results.get(loc.getRegionInfo().getRegionName())); + loc = table.getRegionLocation(ROW_C); + assertNull("Should be missing region for row ccc (past stop row)", + results.get(loc.getRegionInfo().getRegionName())); + } + + @Test + public void testCompountCall() throws Throwable { + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + + Map results = table.coprocessorExec(PingProtocol.class, + ROW_A, ROW_C, + new Batch.Call() { + public String call(PingProtocol instance) { + return instance.hello(instance.ping()); + } + }); + + verifyRegionResults(table, results, "Hello, pong", ROW_A); + verifyRegionResults(table, results, "Hello, pong", ROW_B); + verifyRegionResults(table, results, "Hello, pong", ROW_C); + } + + @Test + public void testNullCall() throws Throwable { + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + + Map results = table.coprocessorExec(PingProtocol.class, + ROW_A, ROW_C, + new Batch.Call() { + public String call(PingProtocol instance) { + return instance.hello(null); + } + }); + + verifyRegionResults(table, results, "Who are you?", ROW_A); + verifyRegionResults(table, results, "Who are you?", ROW_B); + verifyRegionResults(table, results, "Who are you?", ROW_C); + } + + @Test + public void testNullReturn() throws Throwable { + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + + Map results = table.coprocessorExec(PingProtocol.class, + ROW_A, ROW_C, + new Batch.Call(){ + public String call(PingProtocol instance) { + return instance.hello("nobody"); + } + }); + + verifyRegionResults(table, results, null, ROW_A); + verifyRegionResults(table, results, null, ROW_B); + verifyRegionResults(table, results, null, ROW_C); + } + + @Test + public void testVoidReturnType() throws Throwable { + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + + Map results = table.coprocessorExec(PingProtocol.class, + ROW_A, ROW_C, + new Batch.Call(){ + public Object call(PingProtocol instance) { + instance.noop(); + return null; + } + }); + + assertEquals("Should have results from three regions", 3, results.size()); + // all results should be null + for (Object v : results.values()) { + assertNull(v); + } + } + + private void verifyRegionResults(HTable table, + Map results, byte[] row) throws Exception { + verifyRegionResults(table, results, "pong", row); + } + + private void verifyRegionResults(HTable table, + Map results, String expected, byte[] row) + throws Exception { + HRegionLocation loc = table.getRegionLocation(row); + byte[] region = loc.getRegionInfo().getRegionName(); + assertTrue("Results should contain region " + + Bytes.toStringBinary(region)+" for row '"+Bytes.toStringBinary(row)+"'", + results.containsKey(region)); + assertEquals("Invalid result for row '"+Bytes.toStringBinary(row)+"'", + expected, results.get(region)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java new file mode 100644 index 0000000..6fa56db --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java @@ -0,0 +1,315 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.resetCounters; +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_failed_to_grab_task_lost_race; +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_failed_to_grab_task_owned; +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_preempt_task; +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_acquired; +import static org.apache.hadoop.hbase.zookeeper.ZKSplitLog.Counters.tot_wkr_task_acquired_rescan; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CancelableProgressable; +import org.apache.hadoop.hbase.zookeeper.ZKSplitLog; +import org.apache.hadoop.hbase.zookeeper.ZKSplitLog.TaskState; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.ZooDefs.Ids; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestSplitLogWorker { + private static final Log LOG = LogFactory.getLog(TestSplitLogWorker.class); + static { + Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG); + } + private final static HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + private ZooKeeperWatcher zkw; + private SplitLogWorker slw; + + private void waitForCounter(AtomicLong ctr, long oldval, long newval, + long timems) { + assertTrue("ctr=" + ctr.get() + ", oldval=" + oldval + ", newval=" + newval, + waitForCounterBoolean(ctr, oldval, newval, timems)); + } + + private boolean waitForCounterBoolean(AtomicLong ctr, long oldval, long newval, + long timems) { + long curt = System.currentTimeMillis(); + long endt = curt + timems; + while (curt < endt) { + if (ctr.get() == oldval) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + } + curt = System.currentTimeMillis(); + } else { + assertEquals(newval, ctr.get()); + return true; + } + } + return false; + } + + @Before + public void setup() throws Exception { + TEST_UTIL.startMiniZKCluster(); + zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + "split-log-worker-tests", null); + ZKUtil.deleteChildrenRecursively(zkw, zkw.baseZNode); + ZKUtil.createAndFailSilent(zkw, zkw.baseZNode); + assertTrue(ZKUtil.checkExists(zkw, zkw.baseZNode) != -1); + LOG.debug(zkw.baseZNode + " created"); + ZKUtil.createAndFailSilent(zkw, zkw.splitLogZNode); + assertTrue(ZKUtil.checkExists(zkw, zkw.splitLogZNode) != -1); + LOG.debug(zkw.splitLogZNode + " created"); + resetCounters(); + + } + + @After + public void teardown() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + SplitLogWorker.TaskExecutor neverEndingTask = + new SplitLogWorker.TaskExecutor() { + + @Override + public Status exec(String name, CancelableProgressable p) { + while (true) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + return Status.PREEMPTED; + } + if (!p.progress()) { + return Status.PREEMPTED; + } + } + } + + }; + + @Test + public void testAcquireTaskAtStartup() throws Exception { + LOG.info("testAcquireTaskAtStartup"); + ZKSplitLog.Counters.resetCounters(); + + zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, "tatas"), + TaskState.TASK_UNASSIGNED.get("mgr"), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + + SplitLogWorker slw = new SplitLogWorker(zkw, TEST_UTIL.getConfiguration(), + "rs", neverEndingTask); + slw.start(); + try { + waitForCounter(tot_wkr_task_acquired, 0, 1, 1500); + assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw, + ZKSplitLog.getEncodedNodeName(zkw, "tatas")), "rs")); + } finally { + stopSplitLogWorker(slw); + } + } + + private void stopSplitLogWorker(final SplitLogWorker slw) + throws InterruptedException { + if (slw != null) { + slw.stop(); + slw.worker.join(3000); + if (slw.worker.isAlive()) { + assertTrue(("Could not stop the worker thread slw=" + slw) == null); + } + } + } + + @Test + public void testRaceForTask() throws Exception { + LOG.info("testRaceForTask"); + ZKSplitLog.Counters.resetCounters(); + + zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, "trft"), + TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + + SplitLogWorker slw1 = new SplitLogWorker(zkw, TEST_UTIL.getConfiguration(), + "svr1", neverEndingTask); + SplitLogWorker slw2 = new SplitLogWorker(zkw, TEST_UTIL.getConfiguration(), + "svr2", neverEndingTask); + slw1.start(); + slw2.start(); + try { + waitForCounter(tot_wkr_task_acquired, 0, 1, 1500); + // Assert that either the tot_wkr_failed_to_grab_task_owned count was set of if + // not it, that we fell through to the next counter in line and it was set. + assertTrue(waitForCounterBoolean(tot_wkr_failed_to_grab_task_owned, 0, 1, 1500) || + tot_wkr_failed_to_grab_task_lost_race.get() == 1); + assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw, + ZKSplitLog.getEncodedNodeName(zkw, "trft")), "svr1") || + TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw, + ZKSplitLog.getEncodedNodeName(zkw, "trft")), "svr2")); + } finally { + stopSplitLogWorker(slw1); + stopSplitLogWorker(slw2); + } + } + + @Test + public void testPreemptTask() throws Exception { + LOG.info("testPreemptTask"); + ZKSplitLog.Counters.resetCounters(); + + SplitLogWorker slw = new SplitLogWorker(zkw, TEST_UTIL.getConfiguration(), + "tpt_svr", neverEndingTask); + slw.start(); + try { + Thread.yield(); // let the worker start + Thread.sleep(100); + + // this time create a task node after starting the splitLogWorker + zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, "tpt_task"), + TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + + waitForCounter(tot_wkr_task_acquired, 0, 1, 1500); + assertEquals(1, slw.taskReadySeq); + assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw, + ZKSplitLog.getEncodedNodeName(zkw, "tpt_task")), "tpt_svr")); + + ZKUtil.setData(zkw, ZKSplitLog.getEncodedNodeName(zkw, "tpt_task"), + TaskState.TASK_UNASSIGNED.get("manager")); + waitForCounter(tot_wkr_preempt_task, 0, 1, 1500); + } finally { + stopSplitLogWorker(slw); + } + } + + @Test + public void testMultipleTasks() throws Exception { + LOG.info("testMultipleTasks"); + ZKSplitLog.Counters.resetCounters(); + SplitLogWorker slw = new SplitLogWorker(zkw, TEST_UTIL.getConfiguration(), + "tmt_svr", neverEndingTask); + slw.start(); + try { + Thread.yield(); // let the worker start + Thread.sleep(100); + + zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, "tmt_task"), + TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + + waitForCounter(tot_wkr_task_acquired, 0, 1, 1500); + // now the worker is busy doing the above task + + // create another task + zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, "tmt_task_2"), + TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + + // preempt the first task, have it owned by another worker + ZKUtil.setData(zkw, ZKSplitLog.getEncodedNodeName(zkw, "tmt_task"), + TaskState.TASK_OWNED.get("another-worker")); + waitForCounter(tot_wkr_preempt_task, 0, 1, 1500); + + waitForCounter(tot_wkr_task_acquired, 1, 2, 1500); + assertEquals(2, slw.taskReadySeq); + assertTrue(TaskState.TASK_OWNED.equals(ZKUtil.getData(zkw, + ZKSplitLog.getEncodedNodeName(zkw, "tmt_task_2")), "tmt_svr")); + } finally { + stopSplitLogWorker(slw); + } + } + + @Test + public void testRescan() throws Exception { + LOG.info("testRescan"); + ZKSplitLog.Counters.resetCounters(); + slw = new SplitLogWorker(zkw, TEST_UTIL.getConfiguration(), + "svr", neverEndingTask); + slw.start(); + Thread.yield(); // let the worker start + Thread.sleep(200); + + String task = ZKSplitLog.getEncodedNodeName(zkw, "task"); + zkw.getRecoverableZooKeeper().create(task, + TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + + waitForCounter(tot_wkr_task_acquired, 0, 1, 1500); + // now the worker is busy doing the above task + + // preempt the task, have it owned by another worker + ZKUtil.setData(zkw, task, TaskState.TASK_UNASSIGNED.get("manager")); + waitForCounter(tot_wkr_preempt_task, 0, 1, 1500); + + // create a RESCAN node + String rescan = ZKSplitLog.getEncodedNodeName(zkw, "RESCAN"); + rescan = zkw.getRecoverableZooKeeper().create(rescan, + TaskState.TASK_UNASSIGNED.get("manager"), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT_SEQUENTIAL); + + waitForCounter(tot_wkr_task_acquired, 1, 2, 1500); + // RESCAN node might not have been processed if the worker became busy + // with the above task. preempt the task again so that now the RESCAN + // node is processed + ZKUtil.setData(zkw, task, TaskState.TASK_UNASSIGNED.get("manager")); + waitForCounter(tot_wkr_preempt_task, 1, 2, 1500); + waitForCounter(tot_wkr_task_acquired_rescan, 0, 1, 1500); + + List nodes = ZKUtil.listChildrenNoWatch(zkw, zkw.splitLogZNode); + LOG.debug(nodes); + int num = 0; + for (String node : nodes) { + num++; + if (node.startsWith("RESCAN")) { + String name = ZKSplitLog.getEncodedNodeName(zkw, node); + String fn = ZKSplitLog.getFileName(name); + byte [] data = ZKUtil.getData(zkw, ZKUtil.joinZNode(zkw.splitLogZNode, fn)); + String datastr = Bytes.toString(data); + assertTrue("data=" + datastr, TaskState.TASK_DONE.equals(data, "svr")); + } + } + assertEquals(2, num); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java new file mode 100644 index 0000000..4d6eb90 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java @@ -0,0 +1,363 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.LruBlockCache; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.PairOfSameType; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import com.google.common.collect.ImmutableList; + +/** + * Test the {@link SplitTransaction} class against an HRegion (as opposed to + * running cluster). + */ +@Category(SmallTests.class) +public class TestSplitTransaction { + private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final Path testdir = + TEST_UTIL.getDataTestDir(this.getClass().getName()); + private HRegion parent; + private HLog wal; + private FileSystem fs; + private static final byte [] STARTROW = new byte [] {'a', 'a', 'a'}; + // '{' is next ascii after 'z'. + private static final byte [] ENDROW = new byte [] {'{', '{', '{'}; + private static final byte [] GOOD_SPLIT_ROW = new byte [] {'d', 'd', 'd'}; + private static final byte [] CF = HConstants.CATALOG_FAMILY; + + @Before public void setup() throws IOException { + this.fs = FileSystem.get(TEST_UTIL.getConfiguration()); + this.fs.delete(this.testdir, true); + this.wal = new HLog(fs, new Path(this.testdir, "logs"), + new Path(this.testdir, "archive"), + TEST_UTIL.getConfiguration()); + this.parent = createRegion(this.testdir, this.wal); + TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true); + } + + @After public void teardown() throws IOException { + if (this.parent != null && !this.parent.isClosed()) this.parent.close(); + if (this.fs.exists(this.parent.getRegionDir()) && + !this.fs.delete(this.parent.getRegionDir(), true)) { + throw new IOException("Failed delete of " + this.parent.getRegionDir()); + } + if (this.wal != null) this.wal.closeAndDelete(); + this.fs.delete(this.testdir, true); + } + + @Test public void testFailAfterPONR() throws IOException, KeeperException { + final int rowcount = TEST_UTIL.loadRegion(this.parent, CF); + assertTrue(rowcount > 0); + int parentRowCount = countRows(this.parent); + assertEquals(rowcount, parentRowCount); + + // Start transaction. + SplitTransaction st = prepareGOOD_SPLIT_ROW(); + SplitTransaction spiedUponSt = spy(st); + Mockito + .doThrow(new MockedFailedDaughterOpen()) + .when(spiedUponSt) + .openDaughterRegion((Server) Mockito.anyObject(), + (HRegion) Mockito.anyObject()); + + // Run the execute. Look at what it returns. + boolean expectedException = false; + Server mockServer = Mockito.mock(Server.class); + when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration()); + try { + spiedUponSt.execute(mockServer, null); + } catch (IOException e) { + if (e.getCause() != null && + e.getCause() instanceof MockedFailedDaughterOpen) { + expectedException = true; + } + } + assertTrue(expectedException); + // Run rollback returns that we should restart. + assertFalse(spiedUponSt.rollback(null, null)); + // Make sure that region a and region b are still in the filesystem, that + // they have not been removed; this is supposed to be the case if we go + // past point of no return. + Path tableDir = this.parent.getRegionDir().getParent(); + Path daughterADir = + new Path(tableDir, spiedUponSt.getFirstDaughter().getEncodedName()); + Path daughterBDir = + new Path(tableDir, spiedUponSt.getSecondDaughter().getEncodedName()); + assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterADir)); + assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterBDir)); + } + + /** + * Test straight prepare works. Tries to split on {@link #GOOD_SPLIT_ROW} + * @throws IOException + */ + @Test public void testPrepare() throws IOException { + prepareGOOD_SPLIT_ROW(); + } + + private SplitTransaction prepareGOOD_SPLIT_ROW() { + SplitTransaction st = new SplitTransaction(this.parent, GOOD_SPLIT_ROW); + assertTrue(st.prepare()); + return st; + } + + /** + * Pass a reference store + */ + @Test public void testPrepareWithRegionsWithReference() throws IOException { + // create a mock that will act as a reference StoreFile + StoreFile storeFileMock = Mockito.mock(StoreFile.class); + when(storeFileMock.isReference()).thenReturn(true); + + // add the mock to the parent stores + Store storeMock = Mockito.mock(Store.class); + List storeFileList = new ArrayList(1); + storeFileList.add(storeFileMock); + when(storeMock.getStorefiles()).thenReturn(storeFileList); + when(storeMock.close()).thenReturn(ImmutableList.copyOf(storeFileList)); + this.parent.stores.put(Bytes.toBytes(""), storeMock); + + SplitTransaction st = new SplitTransaction(this.parent, GOOD_SPLIT_ROW); + + assertFalse("a region should not be splittable if it has instances of store file references", + st.prepare()); + } + + /** + * Pass an unreasonable split row. + */ + @Test public void testPrepareWithBadSplitRow() throws IOException { + // Pass start row as split key. + SplitTransaction st = new SplitTransaction(this.parent, STARTROW); + assertFalse(st.prepare()); + st = new SplitTransaction(this.parent, HConstants.EMPTY_BYTE_ARRAY); + assertFalse(st.prepare()); + st = new SplitTransaction(this.parent, new byte [] {'A', 'A', 'A'}); + assertFalse(st.prepare()); + st = new SplitTransaction(this.parent, ENDROW); + assertFalse(st.prepare()); + } + + @Test public void testPrepareWithClosedRegion() throws IOException { + this.parent.close(); + SplitTransaction st = new SplitTransaction(this.parent, GOOD_SPLIT_ROW); + assertFalse(st.prepare()); + } + + @Test public void testWholesomeSplitWithHFileV1() throws IOException { + int defaultVersion = TEST_UTIL.getConfiguration().getInt( + HFile.FORMAT_VERSION_KEY, 2); + TEST_UTIL.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, 1); + try { + for (Store store : this.parent.stores.values()) { + store.getFamily().setBloomFilterType(StoreFile.BloomType.ROW); + } + testWholesomeSplit(); + } finally { + TEST_UTIL.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, + defaultVersion); + } + } + + @Test public void testWholesomeSplit() throws IOException { + final int rowcount = TEST_UTIL.loadRegion(this.parent, CF, true); + assertTrue(rowcount > 0); + int parentRowCount = countRows(this.parent); + assertEquals(rowcount, parentRowCount); + + // Pretend region's blocks are not in the cache, used for + // testWholesomeSplitWithHFileV1 + CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration()); + ((LruBlockCache) cacheConf.getBlockCache()).clearCache(); + + // Start transaction. + SplitTransaction st = prepareGOOD_SPLIT_ROW(); + + // Run the execute. Look at what it returns. + Server mockServer = Mockito.mock(Server.class); + when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration()); + PairOfSameType daughters = st.execute(mockServer, null); + // Do some assertions about execution. + assertTrue(this.fs.exists(st.getSplitDir())); + // Assert the parent region is closed. + assertTrue(this.parent.isClosed()); + + // Assert splitdir is empty -- because its content will have been moved out + // to be under the daughter region dirs. + assertEquals(0, this.fs.listStatus(st.getSplitDir()).length); + // Check daughters have correct key span. + assertTrue(Bytes.equals(this.parent.getStartKey(), + daughters.getFirst().getStartKey())); + assertTrue(Bytes.equals(GOOD_SPLIT_ROW, + daughters.getFirst().getEndKey())); + assertTrue(Bytes.equals(daughters.getSecond().getStartKey(), + GOOD_SPLIT_ROW)); + assertTrue(Bytes.equals(this.parent.getEndKey(), + daughters.getSecond().getEndKey())); + // Count rows. + int daughtersRowCount = 0; + for (HRegion r: daughters) { + // Open so can count its content. + HRegion openRegion = HRegion.openHRegion(this.testdir, r.getRegionInfo(), + r.getTableDesc(), r.getLog(), TEST_UTIL.getConfiguration()); + try { + int count = countRows(openRegion); + assertTrue(count > 0 && count != rowcount); + daughtersRowCount += count; + } finally { + openRegion.close(); + openRegion.getLog().closeAndDelete(); + } + } + assertEquals(rowcount, daughtersRowCount); + // Assert the write lock is no longer held on parent + assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread()); + } + + @Test public void testRollback() throws IOException { + final int rowcount = TEST_UTIL.loadRegion(this.parent, CF); + assertTrue(rowcount > 0); + int parentRowCount = countRows(this.parent); + assertEquals(rowcount, parentRowCount); + + // Start transaction. + SplitTransaction st = prepareGOOD_SPLIT_ROW(); + SplitTransaction spiedUponSt = spy(st); + when(spiedUponSt.createDaughterRegion(spiedUponSt.getSecondDaughter(), null)). + thenThrow(new MockedFailedDaughterCreation()); + // Run the execute. Look at what it returns. + boolean expectedException = false; + Server mockServer = Mockito.mock(Server.class); + when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration()); + try { + spiedUponSt.execute(mockServer, null); + } catch (MockedFailedDaughterCreation e) { + expectedException = true; + } + assertTrue(expectedException); + // Run rollback + assertTrue(spiedUponSt.rollback(null, null)); + + // Assert I can scan parent. + int parentRowCount2 = countRows(this.parent); + assertEquals(parentRowCount, parentRowCount2); + + // Assert rollback cleaned up stuff in fs + assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir, st.getFirstDaughter()))); + assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir, st.getSecondDaughter()))); + assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread()); + + // Now retry the split but do not throw an exception this time. + assertTrue(st.prepare()); + PairOfSameType daughters = st.execute(mockServer, null); + // Count rows. + int daughtersRowCount = 0; + for (HRegion r: daughters) { + // Open so can count its content. + HRegion openRegion = HRegion.openHRegion(this.testdir, r.getRegionInfo(), + r.getTableDesc(), r.getLog(), TEST_UTIL.getConfiguration()); + try { + int count = countRows(openRegion); + assertTrue(count > 0 && count != rowcount); + daughtersRowCount += count; + } finally { + openRegion.close(); + openRegion.getLog().closeAndDelete(); + } + } + assertEquals(rowcount, daughtersRowCount); + // Assert the write lock is no longer held on parent + assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread()); + } + + /** + * Exception used in this class only. + */ + @SuppressWarnings("serial") + private class MockedFailedDaughterCreation extends IOException {} + private class MockedFailedDaughterOpen extends IOException {} + + private int countRows(final HRegion r) throws IOException { + int rowcount = 0; + InternalScanner scanner = r.getScanner(new Scan()); + try { + List kvs = new ArrayList(); + boolean hasNext = true; + while (hasNext) { + hasNext = scanner.next(kvs); + if (!kvs.isEmpty()) rowcount++; + } + } finally { + scanner.close(); + } + return rowcount; + } + + HRegion createRegion(final Path testdir, final HLog wal) + throws IOException { + // Make a region with start and end keys. Use 'aaa', to 'AAA'. The load + // region utility will add rows between 'aaa' and 'zzz'. + HTableDescriptor htd = new HTableDescriptor("table"); + HColumnDescriptor hcd = new HColumnDescriptor(CF); + htd.addFamily(hcd); + HRegionInfo hri = new HRegionInfo(htd.getName(), STARTROW, ENDROW); + HRegion r = HRegion.createHRegion(hri, testdir, TEST_UTIL.getConfiguration(), htd); + r.close(); + r.getLog().closeAndDelete(); + return HRegion.openHRegion(testdir, hri, htd, wal, + TEST_UTIL.getConfiguration()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java new file mode 100644 index 0000000..3c1c944 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -0,0 +1,1031 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.List; +import java.util.NavigableMap; +import java.util.concurrent.CountDownLatch; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.executor.RegionTransitionData; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.handler.SplitRegionHandler; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.NodeExistsException; +import org.apache.zookeeper.data.Stat; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Like {@link TestSplitTransaction} in that we're testing {@link SplitTransaction} + * only the below tests are against a running cluster where {@link TestSplitTransaction} + * is tests against a bare {@link HRegion}. + */ +@Category(LargeTests.class) +public class TestSplitTransactionOnCluster { + private static final Log LOG = + LogFactory.getLog(TestSplitTransactionOnCluster.class); + private HBaseAdmin admin = null; + private MiniHBaseCluster cluster = null; + private static final int NB_SERVERS = 2; + private static CountDownLatch latch = new CountDownLatch(1); + private static volatile boolean secondSplit = false; + private static volatile boolean callRollBack = false; + private static volatile boolean firstSplitCompleted = false; + + private static final HBaseTestingUtility TESTING_UTIL = + new HBaseTestingUtility(); + + @BeforeClass public static void before() throws Exception { + TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000); + // Needed because some tests have splits happening on RS that are killed + // We don't want to wait 3min for the master to figure it out + TESTING_UTIL.getConfiguration().setInt( + "hbase.master.assignment.timeoutmonitor.timeout", 4000); + TESTING_UTIL.startMiniCluster(NB_SERVERS); + } + + @AfterClass public static void after() throws Exception { + TESTING_UTIL.shutdownMiniCluster(); + } + + @Before public void setup() throws IOException { + TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS); + this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration()); + this.cluster = TESTING_UTIL.getMiniHBaseCluster(); + } + + private HRegionInfo getAndCheckSingleTableRegion(final List regions) { + assertEquals(1, regions.size()); + return regions.get(0).getRegionInfo(); + } + + @Test(timeout = 2000000) + public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception { + final byte[] tableName = Bytes + .toBytes("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack"); + HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration()); + try { + // Create table then get the single region for our new table. + HTable t = createTableAndWait(tableName, Bytes.toBytes("cf")); + final List regions = cluster.getRegions(tableName); + HRegionInfo hri = getAndCheckSingleTableRegion(regions); + int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName()); + final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex); + insertData(tableName, admin, t); + // Turn off balancer so it doesn't cut in and mess up our placements. + this.admin.setBalancerRunning(false, false); + // Turn off the meta scanner so it don't remove parent on us. + cluster.getMaster().setCatalogJanitorEnabled(false); + + new Thread() { + public void run() { + SplitTransaction st = null; + st = new MockedSplitTransaction(regions.get(0), Bytes.toBytes("row2")); + try { + st.prepare(); + st.execute(regionServer, regionServer); + } catch (IOException e) { + + } + } + }.start(); + for (int i = 0; !callRollBack && i < 100; i++) { + Thread.sleep(100); + } + assertTrue("Waited too long for rollback", callRollBack); + SplitTransaction st = null; + st = new MockedSplitTransaction(regions.get(0), Bytes.toBytes("row2")); + try { + secondSplit = true; + st.prepare(); + st.execute(regionServer, regionServer); + } catch (IOException e) { + LOG.debug("Rollback started :"+ e.getMessage()); + st.rollback(regionServer, regionServer); + } + for (int i=0; !firstSplitCompleted && i<100; i++) { + Thread.sleep(100); + } + assertTrue("fist split did not complete", firstSplitCompleted); + NavigableMap rit = cluster.getMaster().getAssignmentManager() + .getRegionsInTransition(); + for (int i=0; rit.containsKey(hri.getTableNameAsString()) && i<100; i++) { + Thread.sleep(100); + } + assertFalse("region still in transition", rit.containsKey(hri.getTableNameAsString())); + List onlineRegions = regionServer.getOnlineRegions(tableName); + // Region server side split is successful. + assertEquals("The parent region should be splitted", 2, onlineRegions.size()); + //Should be present in RIT + List regionsOfTable = cluster.getMaster().getAssignmentManager().getRegionsOfTable(tableName); + // Master side should also reflect the same + assertEquals("No of regions in master", 2, regionsOfTable.size()); + } finally { + admin.setBalancerRunning(true, false); + secondSplit = false; + firstSplitCompleted = false; + callRollBack = false; + cluster.getMaster().setCatalogJanitorEnabled(true); + } + if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) { + admin.disableTable(tableName); + admin.deleteTable(tableName); + admin.close(); + } + } + + /** + * A test that intentionally has master fail the processing of the split message. + * Tests that the regionserver split ephemeral node gets cleaned up if it + * crashes and that after we process server shutdown, the daughters are up on + * line. + * @throws IOException + * @throws InterruptedException + * @throws NodeExistsException + * @throws KeeperException + */ + @Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling() + throws IOException, InterruptedException, NodeExistsException, KeeperException { + final byte [] tableName = + Bytes.toBytes("ephemeral"); + + // Create table then get the single region for our new table. + HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); + List regions = cluster.getRegions(tableName); + HRegionInfo hri = getAndCheckSingleTableRegion(regions); + + int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); + + // Turn off balancer so it doesn't cut in and mess up our placements. + this.admin.setBalancerRunning(false, true); + // Turn off the meta scanner so it don't remove parent on us. + cluster.getMaster().setCatalogJanitorEnabled(false); + try { + // Add a bit of load up into the table so splittable. + TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY); + // Get region pre-split. + HRegionServer server = cluster.getRegionServer(tableRegionIndex); + printOutRegions(server, "Initial regions: "); + int regionCount = server.getOnlineRegions().size(); + // Now, before we split, set special flag in master, a flag that has + // it FAIL the processing of split. + SplitRegionHandler.TEST_SKIP = true; + // Now try splitting and it should work. + split(hri, server, regionCount); + // Get daughters + List daughters = checkAndGetDaughters(tableName); + // Assert the ephemeral node is up in zk. + String path = ZKAssign.getNodeName(t.getConnection().getZooKeeperWatcher(), + hri.getEncodedName()); + Stat stats = + t.getConnection().getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false); + LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats); + RegionTransitionData rtd = + ZKAssign.getData(t.getConnection().getZooKeeperWatcher(), + hri.getEncodedName()); + // State could be SPLIT or SPLITTING. + assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT) || + rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING)); + // Now crash the server + cluster.abortRegionServer(tableRegionIndex); + waitUntilRegionServerDead(); + awaitDaughters(tableName, daughters.size()); + + // Assert daughters are online. + regions = cluster.getRegions(tableName); + for (HRegion r: regions) { + assertTrue(daughters.contains(r)); + } + // Finally assert that the ephemeral SPLIT znode was cleaned up. + for (int i=0; i<100; i++) { + // wait a bit (10s max) for the node to disappear + stats = t.getConnection().getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false); + if (stats == null) break; + Thread.sleep(100); + } + LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats); + assertTrue(stats == null); + } finally { + // Set this flag back. + SplitRegionHandler.TEST_SKIP = false; + admin.setBalancerRunning(true, false); + cluster.getMaster().setCatalogJanitorEnabled(true); + t.close(); + } + } + + @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback() + throws IOException, InterruptedException, NodeExistsException, KeeperException { + final byte [] tableName = + Bytes.toBytes("testExistingZnodeBlocksSplitAndWeRollback"); + + // Create table then get the single region for our new table. + HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); + List regions = cluster.getRegions(tableName); + HRegionInfo hri = getAndCheckSingleTableRegion(regions); + + int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); + + // Turn off balancer so it doesn't cut in and mess up our placements. + this.admin.setBalancerRunning(false, true); + // Turn off the meta scanner so it don't remove parent on us. + cluster.getMaster().setCatalogJanitorEnabled(false); + try { + // Add a bit of load up into the table so splittable. + TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY); + // Get region pre-split. + HRegionServer server = cluster.getRegionServer(tableRegionIndex); + printOutRegions(server, "Initial regions: "); + int regionCount = server.getOnlineRegions().size(); + // Insert into zk a blocking znode, a znode of same name as region + // so it gets in way of our splitting. + ZKAssign.createNodeClosing(t.getConnection().getZooKeeperWatcher(), + hri, new ServerName("any.old.server", 1234, -1)); + // Now try splitting.... should fail. And each should successfully + // rollback. + this.admin.split(hri.getRegionNameAsString()); + this.admin.split(hri.getRegionNameAsString()); + this.admin.split(hri.getRegionNameAsString()); + // Wait around a while and assert count of regions remains constant. + for (int i = 0; i < 10; i++) { + Thread.sleep(100); + assertEquals(regionCount, server.getOnlineRegions().size()); + } + // Now clear the zknode + ZKAssign.deleteClosingNode(t.getConnection().getZooKeeperWatcher(), hri); + // Now try splitting and it should work. + split(hri, server, regionCount); + // Get daughters + checkAndGetDaughters(tableName); + // OK, so split happened after we cleared the blocking node. + } finally { + admin.setBalancerRunning(true, false); + cluster.getMaster().setCatalogJanitorEnabled(true); + t.close(); + } + } + + /** + * Messy test that simulates case where SplitTransactions fails to add one + * of the daughters up into the .META. table before crash. We're testing + * fact that the shutdown handler will fixup the missing daughter region + * adding it back into .META. + * @throws IOException + * @throws InterruptedException + */ + @Test (timeout = 300000) public void testShutdownSimpleFixup() + throws IOException, InterruptedException { + final byte [] tableName = Bytes.toBytes("testShutdownSimpleFixup"); + + // Create table then get the single region for our new table. + HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); + List regions = cluster.getRegions(tableName); + HRegionInfo hri = getAndCheckSingleTableRegion(regions); + + int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); + + // Turn off balancer so it doesn't cut in and mess up our placements. + this.admin.setBalancerRunning(false, true); + // Turn off the meta scanner so it don't remove parent on us. + cluster.getMaster().setCatalogJanitorEnabled(false); + try { + // Add a bit of load up into the table so splittable. + TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY); + // Get region pre-split. + HRegionServer server = cluster.getRegionServer(tableRegionIndex); + printOutRegions(server, "Initial regions: "); + int regionCount = server.getOnlineRegions().size(); + // Now split. + split(hri, server, regionCount); + // Get daughters + List daughters = checkAndGetDaughters(tableName); + // Remove one of the daughters from .META. to simulate failed insert of + // daughter region up into .META. + removeDaughterFromMeta(daughters.get(0).getRegionName()); + // Now crash the server + cluster.abortRegionServer(tableRegionIndex); + waitUntilRegionServerDead(); + awaitDaughters(tableName, daughters.size()); + // Assert daughters are online. + regions = cluster.getRegions(tableName); + for (HRegion r: regions) { + assertTrue(daughters.contains(r)); + } + } finally { + admin.setBalancerRunning(true, false); + cluster.getMaster().setCatalogJanitorEnabled(true); + t.close(); + } + } + + /** + * Test that if daughter split on us, we won't do the shutdown handler fixup + * just because we can't find the immediate daughter of an offlined parent. + * @throws IOException + * @throws InterruptedException + */ + @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit() + throws IOException, InterruptedException { + final byte [] tableName = + Bytes.toBytes("testShutdownFixupWhenDaughterHasSplit"); + + // Create table then get the single region for our new table. + HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); + List regions = cluster.getRegions(tableName); + HRegionInfo hri = getAndCheckSingleTableRegion(regions); + + int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); + + // Turn off balancer so it doesn't cut in and mess up our placements. + this.admin.setBalancerRunning(false, true); + // Turn off the meta scanner so it don't remove parent on us. + cluster.getMaster().setCatalogJanitorEnabled(false); + try { + // Add a bit of load up into the table so splittable. + TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY); + // Get region pre-split. + HRegionServer server = cluster.getRegionServer(tableRegionIndex); + printOutRegions(server, "Initial regions: "); + int regionCount = server.getOnlineRegions().size(); + // Now split. + split(hri, server, regionCount); + // Get daughters + List daughters = checkAndGetDaughters(tableName); + // Now split one of the daughters. + regionCount = server.getOnlineRegions().size(); + HRegionInfo daughter = daughters.get(0).getRegionInfo(); + // Compact first to ensure we have cleaned up references -- else the split + // will fail. + this.admin.compact(daughter.getRegionName()); + daughters = cluster.getRegions(tableName); + HRegion daughterRegion = null; + for (HRegion r: daughters) { + if (r.getRegionInfo().equals(daughter)) daughterRegion = r; + } + assertTrue(daughterRegion != null); + for (int i=0; i<100; i++) { + if (!daughterRegion.hasReferences()) break; + Threads.sleep(100); + } + assertFalse("Waiting for refereces to be compacted", daughterRegion.hasReferences()); + split(daughter, server, regionCount); + // Get list of daughters + daughters = cluster.getRegions(tableName); + // Now crash the server + cluster.abortRegionServer(tableRegionIndex); + waitUntilRegionServerDead(); + awaitDaughters(tableName, daughters.size()); + // Assert daughters are online and ONLY the original daughters -- that + // fixup didn't insert one during server shutdown recover. + regions = cluster.getRegions(tableName); + assertEquals(daughters.size(), regions.size()); + for (HRegion r: regions) { + assertTrue(daughters.contains(r)); + } + } finally { + admin.setBalancerRunning(true, false); + cluster.getMaster().setCatalogJanitorEnabled(true); + t.close(); + } + } + + /** + * Verifies HBASE-5806. When splitting is partially done and the master goes down + * when the SPLIT node is in either SPLIT or SPLITTING state. + * + * @throws IOException + * @throws InterruptedException + * @throws NodeExistsException + * @throws KeeperException + */ + @Test(timeout = 300000) + public void testMasterRestartWhenSplittingIsPartial() + throws IOException, InterruptedException, NodeExistsException, + KeeperException { + final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial"); + + // Create table then get the single region for our new table. + HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); + List regions = cluster.getRegions(tableName); + HRegionInfo hri = getAndCheckSingleTableRegion(regions); + + int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); + + // Turn off the meta scanner so it don't remove parent on us. + cluster.getMaster().setCatalogJanitorEnabled(false); + // Turn off balancer so it doesn't cut in and mess up our placements. + this.admin.setBalancerRunning(false, true); + + try { + // Add a bit of load up into the table so splittable. + TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY); + // Get region pre-split. + HRegionServer server = cluster.getRegionServer(tableRegionIndex); + printOutRegions(server, "Initial regions: "); + int regionCount = server.getOnlineRegions().size(); + // Now, before we split, set special flag in master, a flag that has + // it FAIL the processing of split. + SplitRegionHandler.TEST_SKIP = true; + // Now try splitting and it should work. + split(hri, server, regionCount); + // Get daughters + checkAndGetDaughters(tableName); + // Assert the ephemeral node is up in zk. + String path = ZKAssign.getNodeName(t.getConnection() + .getZooKeeperWatcher(), hri.getEncodedName()); + Stat stats = t.getConnection().getZooKeeperWatcher() + .getRecoverableZooKeeper().exists(path, false); + LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + + stats); + RegionTransitionData rtd = ZKAssign.getData(t.getConnection() + .getZooKeeperWatcher(), hri.getEncodedName()); + // State could be SPLIT or SPLITTING. + assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT) + || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING)); + + + // abort and wait for new master. + MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster(); + + this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration()); + + // update the hri to be offlined and splitted. + hri.setOffline(true); + hri.setSplit(true); + ServerName regionServerOfRegion = master.getAssignmentManager() + .getRegionServerOfRegion(hri); + assertTrue(regionServerOfRegion != null); + + } finally { + // Set this flag back. + SplitRegionHandler.TEST_SKIP = false; + admin.setBalancerRunning(true, false); + cluster.getMaster().setCatalogJanitorEnabled(true); + t.close(); + } + } + + + /** + * Verifies HBASE-5806. Here the case is that splitting is completed but before the + * CJ could remove the parent region the master is killed and restarted. + * @throws IOException + * @throws InterruptedException + * @throws NodeExistsException + * @throws KeeperException + */ + @Test (timeout = 300000) + public void testMasterRestartAtRegionSplitPendingCatalogJanitor() + throws IOException, InterruptedException, NodeExistsException, + KeeperException { + final byte[] tableName = Bytes.toBytes("testMasterRestartAtRegionSplitPendingCatalogJanitor"); + + // Create table then get the single region for our new table. + this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration()); + HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); + List regions = cluster.getRegions(tableName); + HRegionInfo hri = getAndCheckSingleTableRegion(regions); + + int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); + + // Turn off balancer so it doesn't cut in and mess up our placements. + this.admin.setBalancerRunning(false, true); + // Turn off the meta scanner so it don't remove parent on us. + cluster.getMaster().setCatalogJanitorEnabled(false); + try { + // Add a bit of load up into the table so splittable. + TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY); + // Get region pre-split. + HRegionServer server = cluster.getRegionServer(tableRegionIndex); + printOutRegions(server, "Initial regions: "); + int regionCount = server.getOnlineRegions().size(); + + split(hri, server, regionCount); + // Get daughters + checkAndGetDaughters(tableName); + // Assert the ephemeral node is up in zk. + String path = ZKAssign.getNodeName(t.getConnection() + .getZooKeeperWatcher(), hri.getEncodedName()); + Stat stats = t.getConnection().getZooKeeperWatcher() + .getRecoverableZooKeeper().exists(path, false); + LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + + stats); + String node = ZKAssign.getNodeName(t.getConnection() + .getZooKeeperWatcher(), hri.getEncodedName()); + Stat stat = new Stat(); + byte[] data = ZKUtil.getDataNoWatch(t.getConnection() + .getZooKeeperWatcher(), node, stat); + // ZKUtil.create + for (int i=0; data != null && i<60; i++) { + Thread.sleep(1000); + data = ZKUtil.getDataNoWatch(t.getConnection().getZooKeeperWatcher(), + node, stat); + + } + assertNull("Waited too long for ZK node to be removed: "+node, data); + + MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster(); + + this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration()); + + hri.setOffline(true); + hri.setSplit(true); + ServerName regionServerOfRegion = master.getAssignmentManager() + .getRegionServerOfRegion(hri); + assertTrue(regionServerOfRegion == null); + } finally { + // Set this flag back. + SplitRegionHandler.TEST_SKIP = false; + this.admin.setBalancerRunning(true, false); + cluster.getMaster().setCatalogJanitorEnabled(true); + t.close(); + } + } + + /** + * While transitioning node from RS_ZK_REGION_SPLITTING to + * RS_ZK_REGION_SPLITTING during region split,if zookeper went down split always + * fails for the region. HBASE-6088 fixes this scenario. + * This test case is to test the znode is deleted(if created) or not in roll back. + * + * @throws IOException + * @throws InterruptedException + * @throws KeeperException + */ + @Test + public void testSplitBeforeSettingSplittingInZK() throws Exception, + InterruptedException, KeeperException { + testSplitBeforeSettingSplittingInZK(true); + testSplitBeforeSettingSplittingInZK(false); + } + + private void testSplitBeforeSettingSplittingInZK(boolean nodeCreated) throws Exception { + final byte[] tableName = Bytes.toBytes("testSplitBeforeSettingSplittingInZK"); + + HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration()); + // Create table then get the single region for our new table. + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor("cf")); + admin.createTable(htd); + + List regions = null; + for (int i=0; i<100; i++) { + regions = cluster.getRegions(tableName); + if (regions.size() > 0) break; + Thread.sleep(100); + } + int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName()); + HRegionServer regionServer = cluster.getRegionServer(regionServerIndex); + SplitTransaction st = null; + if (nodeCreated) { + st = new MockedSplitTransaction(regions.get(0), null) { + @Override + int transitionNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo parent, + ServerName serverName, int version) throws KeeperException, IOException { + throw new TransitionToSplittingFailedException(); + } + }; + } else { + st = new MockedSplitTransaction(regions.get(0), null) { + @Override + void createNodeSplitting(ZooKeeperWatcher zkw, HRegionInfo region, ServerName serverName) + throws KeeperException, IOException { + throw new SplittingNodeCreationFailedException (); + } + }; + } + String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0) + .getRegionInfo().getEncodedName()); + // make sure the client is uptodate + regionServer.getZooKeeper().sync(node); + for (int i = 0; i < 100; i++) { + // We expect the znode to be deleted by this time. Here the znode could be in OPENED state and the + // master has not yet deleted the znode. + if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) { + Thread.sleep(100); + } + } + + try { + st.execute(regionServer, regionServer); + } catch (IOException e) { + // check for the specific instance in case the Split failed due to the existence of the znode in OPENED state. + // This will at least make the test to fail; + if (nodeCreated) { + assertTrue("Should be instance of TransitionToSplittingFailedException", + e instanceof TransitionToSplittingFailedException); + } else { + assertTrue("Should be instance of CreateSplittingNodeFailedException", + e instanceof SplittingNodeCreationFailedException ); + } + node = ZKAssign.getNodeName(regionServer.getZooKeeper(), regions.get(0) + .getRegionInfo().getEncodedName()); + // make sure the client is uptodate + regionServer.getZooKeeper().sync(node); + if (nodeCreated) { + assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1); + } else { + assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1); + } + assertTrue(st.rollback(regionServer, regionServer)); + assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1); + } + if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + } + + @Test + public void testShouldClearRITWhenNodeFoundInSplittingState() throws Exception { + final byte[] tableName = Bytes.toBytes("testShouldClearRITWhenNodeFoundInSplittingState"); + HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration()); + // Create table then get the single region for our new table. + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor("cf")); + admin.createTable(htd); + for (int i = 0; cluster.getRegions(tableName).size() == 0 && i < 100; i++) { + Thread.sleep(100); + } + assertTrue("Table not online", cluster.getRegions(tableName).size() != 0); + + HRegion region = cluster.getRegions(tableName).get(0); + int regionServerIndex = cluster.getServerWith(region.getRegionName()); + HRegionServer regionServer = cluster.getRegionServer(regionServerIndex); + SplitTransaction st = null; + + st = new MockedSplitTransaction(region, null) { + @Override + void createSplitDir(FileSystem fs, Path splitdir) throws IOException { + throw new IOException(""); + } + }; + + try { + st.execute(regionServer, regionServer); + } catch (IOException e) { + String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), region + .getRegionInfo().getEncodedName()); + + assertFalse(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1); + AssignmentManager am = cluster.getMaster().getAssignmentManager(); + for (int i = 0; !am.getRegionsInTransition().containsKey( + region.getRegionInfo().getEncodedName()) + && i < 100; i++) { + Thread.sleep(200); + } + assertTrue("region is not in transition "+region, + am.getRegionsInTransition().containsKey(region.getRegionInfo().getEncodedName())); + RegionState regionState = am.getRegionsInTransition().get(region.getRegionInfo() + .getEncodedName()); + assertTrue(regionState.getState() == RegionState.State.SPLITTING); + assertTrue(st.rollback(regionServer, regionServer)); + assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1); + for (int i=0; am.getRegionsInTransition().containsKey(region.getRegionInfo().getEncodedName()) && i<100; i++) { + // Just in case the nodeDeleted event did not get executed. + Thread.sleep(200); + } + assertFalse("region is still in transition", + am.getRegionsInTransition().containsKey(region.getRegionInfo().getEncodedName())); + } + if (admin.isTableAvailable(tableName) && admin.isTableEnabled(tableName)) { + admin.disableTable(tableName); + admin.deleteTable(tableName); + admin.close(); + } + } + + @Test(timeout = 60000) + public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception { + final byte[] tableName = + Bytes.toBytes("testTableExistsIfTheSpecifiedTableRegionIsSplitParent"); + HRegionServer regionServer = null; + List regions = null; + HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration()); + try { + // Create table then get the single region for our new table. + HTable t = createTableAndWait(tableName, Bytes.toBytes("cf")); + regions = cluster.getRegions(tableName); + int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName()); + regionServer = cluster.getRegionServer(regionServerIndex); + insertData(tableName, admin, t); + // Turn off balancer so it doesn't cut in and mess up our placements. + cluster.getMaster().setCatalogJanitorEnabled(false); + boolean tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(), + Bytes.toString(tableName)); + assertEquals("The specified table should present.", true, tableExists); + SplitTransaction st = new SplitTransaction(regions.get(0), Bytes.toBytes("row2")); + try { + st.prepare(); + st.createDaughters(regionServer, regionServer); + } catch (IOException e) { + + } + tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(), + Bytes.toString(tableName)); + assertEquals("The specified table should present.", true, tableExists); + } finally { + cluster.getMaster().setCatalogJanitorEnabled(true); + admin.close(); + } + } + + private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException, + InterruptedException { + Put p = new Put(Bytes.toBytes("row1")); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1")); + t.put(p); + p = new Put(Bytes.toBytes("row2")); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2")); + t.put(p); + p = new Put(Bytes.toBytes("row3")); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3")); + t.put(p); + p = new Put(Bytes.toBytes("row4")); + p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4")); + t.put(p); + admin.flush(tableName); + } + + public static class MockedSplitTransaction extends SplitTransaction { + + private HRegion currentRegion; + public MockedSplitTransaction(HRegion r, byte[] splitrow) { + super(r, splitrow); + this.currentRegion = r; + } + + @Override + void transitionZKNode(Server server, RegionServerServices services, HRegion a, HRegion b) + throws IOException { + if (this.currentRegion.getRegionInfo().getTableNameAsString() + .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) { + try { + if (!secondSplit){ + callRollBack = true; + latch.await(); + } + } catch (InterruptedException e) { + } + + } + super.transitionZKNode(server, services, a, b); + if (this.currentRegion.getRegionInfo().getTableNameAsString() + .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) { + firstSplitCompleted = true; + } + } + @Override + public boolean rollback(Server server, RegionServerServices services) throws IOException { + if (this.currentRegion.getRegionInfo().getTableNameAsString() + .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) { + if(secondSplit){ + super.rollback(server, services); + latch.countDown(); + return true; + } + } + return super.rollback(server, services); + } + + } + + private List checkAndGetDaughters(byte[] tableName) + throws InterruptedException { + List daughters = null; + // try up to 10s + for (int i=0; i<100; i++) { + daughters = cluster.getRegions(tableName); + if (daughters.size() >= 2) break; + Thread.sleep(100); + } + assertTrue(daughters.size() >= 2); + return daughters; + } + + private MockMasterWithoutCatalogJanitor abortAndWaitForMaster() + throws IOException, InterruptedException { + cluster.abortMaster(0); + cluster.waitOnMaster(0); + cluster.getConfiguration().setClass(HConstants.MASTER_IMPL, + MockMasterWithoutCatalogJanitor.class, HMaster.class); + MockMasterWithoutCatalogJanitor master = null; + master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster(); + cluster.waitForActiveAndReadyMaster(); + return master; + } + + private void split(final HRegionInfo hri, final HRegionServer server, + final int regionCount) + throws IOException, InterruptedException { + this.admin.split(hri.getRegionNameAsString()); + for (int i=0; server.getOnlineRegions().size() <= regionCount && i<100; i++) { + LOG.debug("Waiting on region to split"); + Thread.sleep(100); + } + assertFalse("Waited too long for split", server.getOnlineRegions().size() <= regionCount); + } + + private void removeDaughterFromMeta(final byte [] regionName) throws IOException { + HTable metaTable = + new HTable(TESTING_UTIL.getConfiguration(), HConstants.META_TABLE_NAME); + Delete d = new Delete(regionName); + LOG.info("Deleted " + Bytes.toString(regionName)); + metaTable.delete(d); + } + + /** + * Ensure single table region is not on same server as the single .META. table + * region. + * @param admin + * @param hri + * @return Index of the server hosting the single table region + * @throws UnknownRegionException + * @throws MasterNotRunningException + * @throws ZooKeeperConnectionException + * @throws InterruptedException + */ + private int ensureTableRegionNotOnSameServerAsMeta(final HBaseAdmin admin, + final HRegionInfo hri) + throws UnknownRegionException, MasterNotRunningException, + ZooKeeperConnectionException, InterruptedException { + MiniHBaseCluster cluster = TESTING_UTIL.getMiniHBaseCluster(); + // Now make sure that the table region is not on same server as that hosting + // .META. We don't want .META. replay polluting our test when we later crash + // the table region serving server. + int metaServerIndex = cluster.getServerWithMeta(); + assertTrue(metaServerIndex != -1); + HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex); + int tableRegionIndex = cluster.getServerWith(hri.getRegionName()); + assertTrue(tableRegionIndex != -1); + HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex); + if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) { + HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer); + assertNotNull(hrs); + assertNotNull(hri); + LOG. + info("Moving " + hri.getRegionNameAsString() + " to " + + hrs.getServerName() + "; metaServerIndex=" + metaServerIndex); + for (int i = 0; cluster.getMaster().getAssignmentManager() + .getRegionServerOfRegion(hri) == null + && i < 100; i++) { + Thread.sleep(10); + } + admin.move(hri.getEncodedNameAsBytes(), + Bytes.toBytes(hrs.getServerName().toString())); + } + // Wait till table region is up on the server that is NOT carrying .META.. + for (int i=0; i<100; i++) { + tableRegionIndex = cluster.getServerWith(hri.getRegionName()); + if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break; + LOG.debug("Waiting on region move off the .META. server; current index " + + tableRegionIndex + " and metaServerIndex=" + metaServerIndex); + Thread.sleep(100); + } + assertTrue("Region not moved off .META. server", tableRegionIndex != -1 + && tableRegionIndex != metaServerIndex); + // Verify for sure table region is not on same server as .META. + tableRegionIndex = cluster.getServerWith(hri.getRegionName()); + assertTrue(tableRegionIndex != -1); + assertNotSame(metaServerIndex, tableRegionIndex); + return tableRegionIndex; + } + + /** + * Find regionserver other than the one passed. + * Can't rely on indexes into list of regionservers since crashed servers + * occupy an index. + * @param cluster + * @param notThisOne + * @return A regionserver that is not notThisOne or null if none + * found + */ + private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster, + final HRegionServer notThisOne) { + for (RegionServerThread rst: cluster.getRegionServerThreads()) { + HRegionServer hrs = rst.getRegionServer(); + if (hrs.getServerName().equals(notThisOne.getServerName())) continue; + if (hrs.isStopping() || hrs.isStopped()) continue; + return hrs; + } + return null; + } + + private void printOutRegions(final HRegionServer hrs, final String prefix) + throws IOException { + List regions = hrs.getOnlineRegions(); + for (HRegionInfo region: regions) { + LOG.info(prefix + region.getRegionNameAsString()); + } + } + + private void waitUntilRegionServerDead() throws InterruptedException { + // Wait until the master processes the RS shutdown + for (int i=0; cluster.getMaster().getClusterStatus(). + getServers().size() == NB_SERVERS && i<100; i++) { + LOG.info("Waiting on server to go down"); + Thread.sleep(100); + } + assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus(). + getServers().size() == NB_SERVERS); + } + + private void awaitDaughters(byte[] tableName, int numDaughters) throws InterruptedException { + // Wait till regions are back on line again. + for (int i=0; cluster.getRegions(tableName).size() < numDaughters && i<60; i++) { + LOG.info("Waiting for repair to happen"); + Thread.sleep(1000); + } + if (cluster.getRegions(tableName).size() < numDaughters) { + fail("Waiting too long for daughter regions"); + } + } + + private HTable createTableAndWait(byte[] tableName, byte[] cf) throws IOException, + InterruptedException { + HTable t = TESTING_UTIL.createTable(tableName, cf); + for (int i = 0; cluster.getRegions(tableName).size() == 0 && i < 100; i++) { + Thread.sleep(100); + } + assertTrue("Table not online: "+Bytes.toString(tableName), cluster.getRegions(tableName).size() != 0); + return t; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); + + public static class MockMasterWithoutCatalogJanitor extends HMaster { + + public MockMasterWithoutCatalogJanitor(Configuration conf) throws IOException, KeeperException, + InterruptedException { + super(conf); + } + + protected void startCatalogJanitorChore() { + LOG.debug("Customised master executed."); + } + } + + private static class TransitionToSplittingFailedException extends IOException { + public TransitionToSplittingFailedException() { + super(); + } + } + + private static class SplittingNodeCreationFailedException extends IOException { + public SplittingNodeCreationFailedException () { + super(); + } + } +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java new file mode 100644 index 0000000..94c1f1a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java @@ -0,0 +1,791 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.lang.ref.SoftReference; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.NavigableSet; +import java.util.concurrent.ConcurrentSkipListSet; + +import junit.framework.TestCase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.FilterFileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.monitoring.MonitoredTask; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; +import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; +import org.apache.hadoop.util.Progressable; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import com.google.common.base.Joiner; + +/** + * Test class for the Store + */ +@Category(MediumTests.class) +public class TestStore extends TestCase { + public static final Log LOG = LogFactory.getLog(TestStore.class); + + Store store; + byte [] table = Bytes.toBytes("table"); + byte [] family = Bytes.toBytes("family"); + + byte [] row = Bytes.toBytes("row"); + byte [] row2 = Bytes.toBytes("row2"); + byte [] qf1 = Bytes.toBytes("qf1"); + byte [] qf2 = Bytes.toBytes("qf2"); + byte [] qf3 = Bytes.toBytes("qf3"); + byte [] qf4 = Bytes.toBytes("qf4"); + byte [] qf5 = Bytes.toBytes("qf5"); + byte [] qf6 = Bytes.toBytes("qf6"); + + NavigableSet qualifiers = + new ConcurrentSkipListSet(Bytes.BYTES_COMPARATOR); + + List expected = new ArrayList(); + List result = new ArrayList(); + + long id = System.currentTimeMillis(); + Get get = new Get(row); + + private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString(); + + + /** + * Setup + * @throws IOException + */ + @Override + public void setUp() throws IOException { + qualifiers.add(qf1); + qualifiers.add(qf3); + qualifiers.add(qf5); + + Iterator iter = qualifiers.iterator(); + while(iter.hasNext()){ + byte [] next = iter.next(); + expected.add(new KeyValue(row, family, next, 1, (byte[])null)); + get.addColumn(family, next); + } + } + + private void init(String methodName) throws IOException { + init(methodName, HBaseConfiguration.create()); + } + + private void init(String methodName, Configuration conf) + throws IOException { + HColumnDescriptor hcd = new HColumnDescriptor(family); + // some of the tests write 4 versions and then flush + // (with HBASE-4241, lower versions are collected on flush) + hcd.setMaxVersions(4); + init(methodName, conf, hcd); + } + + private void init(String methodName, Configuration conf, + HColumnDescriptor hcd) throws IOException { + //Setting up a Store + Path basedir = new Path(DIR+methodName); + Path logdir = new Path(DIR+methodName+"/logs"); + Path oldLogDir = new Path(basedir, HConstants.HREGION_OLDLOGDIR_NAME); + FileSystem fs = FileSystem.get(conf); + + fs.delete(logdir, true); + + HTableDescriptor htd = new HTableDescriptor(table); + htd.addFamily(hcd); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HLog hlog = new HLog(fs, logdir, oldLogDir, conf); + HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); + + store = new Store(basedir, region, hcd, fs, conf); + } + + public void testDeleteExpiredStoreFiles() throws Exception { + int storeFileNum = 4; + int ttl = 4; + + Configuration conf = HBaseConfiguration.create(); + // Enable the expired store file deletion + conf.setBoolean("hbase.store.delete.expired.storefile", true); + HColumnDescriptor hcd = new HColumnDescriptor(family); + hcd.setTimeToLive(ttl); + init(getName(), conf, hcd); + + long sleepTime = this.store.scanInfo.getTtl() / storeFileNum; + long timeStamp; + // There are 4 store files and the max time stamp difference among these + // store files will be (this.store.ttl / storeFileNum) + for (int i = 1; i <= storeFileNum; i++) { + LOG.info("Adding some data for the store file #" + i); + timeStamp = EnvironmentEdgeManager.currentTimeMillis(); + this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null)); + this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null)); + this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null)); + flush(i); + Thread.sleep(sleepTime); + } + + // Verify the total number of store files + assertEquals(storeFileNum, this.store.getStorefiles().size()); + + // Each compaction request will find one expired store file and delete it + // by the compaction. + for (int i = 1; i <= storeFileNum; i++) { + // verify the expired store file. + CompactionRequest cr = this.store.requestCompaction(); + // the first is expired normally. + // If not the first compaction, there is another empty store file, + assertEquals(Math.min(i, 2), cr.getFiles().size()); + for (int j = 0; i < cr.getFiles().size(); j++) { + assertTrue(cr.getFiles().get(j).getReader().getMaxTimestamp() < (System + .currentTimeMillis() - this.store.scanInfo.getTtl())); + } + // Verify that the expired store file is compacted to an empty store file. + this.store.compact(cr); + // It is an empty store file. + assertEquals(0, this.store.getStorefiles().get(0).getReader() + .getEntries()); + + // Let the next store file expired. + Thread.sleep(sleepTime); + } + } + + public void testLowestModificationTime() throws Exception { + Configuration conf = HBaseConfiguration.create(); + FileSystem fs = FileSystem.get(conf); + // Initialize region + init(getName(), conf); + + int storeFileNum = 4; + for (int i = 1; i <= storeFileNum; i++) { + LOG.info("Adding some data for the store file #"+i); + this.store.add(new KeyValue(row, family, qf1, i, (byte[])null)); + this.store.add(new KeyValue(row, family, qf2, i, (byte[])null)); + this.store.add(new KeyValue(row, family, qf3, i, (byte[])null)); + flush(i); + } + // after flush; check the lowest time stamp + long lowestTimeStampFromStore = + Store.getLowestTimestamp(store.getStorefiles()); + long lowestTimeStampFromFS = + getLowestTimeStampFromFS(fs,store.getStorefiles()); + assertEquals(lowestTimeStampFromStore,lowestTimeStampFromFS); + + // after compact; check the lowest time stamp + store.compact(store.requestCompaction()); + lowestTimeStampFromStore = Store.getLowestTimestamp(store.getStorefiles()); + lowestTimeStampFromFS = getLowestTimeStampFromFS(fs,store.getStorefiles()); + assertEquals(lowestTimeStampFromStore,lowestTimeStampFromFS); + } + + private static long getLowestTimeStampFromFS(FileSystem fs, + final List candidates) throws IOException { + long minTs = Long.MAX_VALUE; + if (candidates.isEmpty()) { + return minTs; + } + Path[] p = new Path[candidates.size()]; + for (int i = 0; i < candidates.size(); ++i) { + p[i] = candidates.get(i).getPath(); + } + + FileStatus[] stats = fs.listStatus(p); + if (stats == null || stats.length == 0) { + return minTs; + } + for (FileStatus s : stats) { + minTs = Math.min(minTs, s.getModificationTime()); + } + return minTs; + } + + ////////////////////////////////////////////////////////////////////////////// + // Get tests + ////////////////////////////////////////////////////////////////////////////// + + /** + * Test for hbase-1686. + * @throws IOException + */ + public void testEmptyStoreFile() throws IOException { + init(this.getName()); + // Write a store file. + this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null)); + this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null)); + flush(1); + // Now put in place an empty store file. Its a little tricky. Have to + // do manually with hacked in sequence id. + StoreFile f = this.store.getStorefiles().get(0); + Path storedir = f.getPath().getParent(); + long seqid = f.getMaxSequenceId(); + Configuration c = HBaseConfiguration.create(); + FileSystem fs = FileSystem.get(c); + StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c), + fs, StoreFile.DEFAULT_BLOCKSIZE_SMALL) + .withOutputDir(storedir) + .build(); + w.appendMetadata(seqid + 1, false); + w.close(); + this.store.close(); + // Reopen it... should pick up two files + this.store = new Store(storedir.getParent().getParent(), + this.store.getHRegion(), + this.store.getFamily(), fs, c); + System.out.println(this.store.getHRegionInfo().getEncodedName()); + assertEquals(2, this.store.getStorefilesCount()); + + result = HBaseTestingUtility.getFromStoreFile(store, + get.getRow(), + qualifiers); + assertEquals(1, result.size()); + } + + /** + * Getting data from memstore only + * @throws IOException + */ + public void testGet_FromMemStoreOnly() throws IOException { + init(this.getName()); + + //Put data in memstore + this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null)); + this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null)); + this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null)); + this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null)); + this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null)); + this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null)); + + //Get + result = HBaseTestingUtility.getFromStoreFile(store, + get.getRow(), qualifiers); + + //Compare + assertCheck(); + } + + /** + * Getting data from files only + * @throws IOException + */ + public void testGet_FromFilesOnly() throws IOException { + init(this.getName()); + + //Put data in memstore + this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null)); + this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null)); + //flush + flush(1); + + //Add more data + this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null)); + this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null)); + //flush + flush(2); + + //Add more data + this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null)); + this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null)); + //flush + flush(3); + + //Get + result = HBaseTestingUtility.getFromStoreFile(store, + get.getRow(), + qualifiers); + //this.store.get(get, qualifiers, result); + + //Need to sort the result since multiple files + Collections.sort(result, KeyValue.COMPARATOR); + + //Compare + assertCheck(); + } + + /** + * Getting data from memstore and files + * @throws IOException + */ + public void testGet_FromMemStoreAndFiles() throws IOException { + init(this.getName()); + + //Put data in memstore + this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null)); + this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null)); + //flush + flush(1); + + //Add more data + this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null)); + this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null)); + //flush + flush(2); + + //Add more data + this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null)); + this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null)); + + //Get + result = HBaseTestingUtility.getFromStoreFile(store, + get.getRow(), qualifiers); + + //Need to sort the result since multiple files + Collections.sort(result, KeyValue.COMPARATOR); + + //Compare + assertCheck(); + } + + private void flush(int storeFilessize) throws IOException{ + this.store.snapshot(); + flushStore(store, id++); + assertEquals(storeFilessize, this.store.getStorefiles().size()); + assertEquals(0, this.store.memstore.kvset.size()); + } + + private void assertCheck() { + assertEquals(expected.size(), result.size()); + for(int i=0; i 0); + + // then flush. + flushStore(store, id++); + assertEquals(1, this.store.getStorefiles().size()); + // from the one we inserted up there, and a new one + assertEquals(2, this.store.memstore.kvset.size()); + + // how many key/values for this row are there? + Get get = new Get(row); + get.addColumn(family, qf1); + get.setMaxVersions(); // all versions. + List results = new ArrayList(); + + results = HBaseTestingUtility.getFromStoreFile(store, get); + assertEquals(2, results.size()); + + long ts1 = results.get(0).getTimestamp(); + long ts2 = results.get(1).getTimestamp(); + + assertTrue(ts1 > ts2); + + assertEquals(newValue, Bytes.toLong(results.get(0).getValue())); + assertEquals(oldValue, Bytes.toLong(results.get(1).getValue())); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + EnvironmentEdgeManagerTestHelper.reset(); + } + + public void testICV_negMemstoreSize() throws IOException { + init(this.getName()); + + long time = 100; + ManualEnvironmentEdge ee = new ManualEnvironmentEdge(); + ee.setValue(time); + EnvironmentEdgeManagerTestHelper.injectEdge(ee); + long newValue = 3L; + long size = 0; + + + size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1, + System.currentTimeMillis(), + Bytes.toBytes(newValue))); + size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1, + System.currentTimeMillis(), + Bytes.toBytes(newValue))); + size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1, + System.currentTimeMillis(), + Bytes.toBytes(newValue))); + size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1, + System.currentTimeMillis(), + Bytes.toBytes(newValue))); + size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1, + System.currentTimeMillis(), + Bytes.toBytes(newValue))); + + + for ( int i = 0 ; i < 10000 ; ++i) { + newValue++; + + long ret = this.store.updateColumnValue(row, family, qf1, newValue); + long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue); + + if (ret != 0) System.out.println("ret: " + ret); + if (ret2 != 0) System.out.println("ret2: " + ret2); + + assertTrue("ret: " + ret, ret >= 0); + size += ret; + assertTrue("ret2: " + ret2, ret2 >= 0); + size += ret2; + + + if (i % 1000 == 0) + ee.setValue(++time); + } + + long computedSize=0; + for (KeyValue kv : this.store.memstore.kvset) { + long kvsize = this.store.memstore.heapSizeChange(kv, true); + //System.out.println(kv + " size= " + kvsize + " kvsize= " + kv.heapSize()); + computedSize += kvsize; + } + assertEquals(computedSize, size); + } + + public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception { + ManualEnvironmentEdge mee = new ManualEnvironmentEdge(); + EnvironmentEdgeManagerTestHelper.injectEdge(mee); + init(this.getName()); + + long oldValue = 1L; + long newValue = 3L; + this.store.add(new KeyValue(row, family, qf1, + EnvironmentEdgeManager.currentTimeMillis(), + Bytes.toBytes(oldValue))); + + // snapshot the store. + this.store.snapshot(); + + // update during the snapshot, the exact same TS as the Put (lololol) + long ret = this.store.updateColumnValue(row, family, qf1, newValue); + + // memstore should have grown by some amount. + assertTrue(ret > 0); + + // then flush. + flushStore(store, id++); + assertEquals(1, this.store.getStorefiles().size()); + assertEquals(1, this.store.memstore.kvset.size()); + + // now increment again: + newValue += 1; + this.store.updateColumnValue(row, family, qf1, newValue); + + // at this point we have a TS=1 in snapshot, and a TS=2 in kvset, so increment again: + newValue += 1; + this.store.updateColumnValue(row, family, qf1, newValue); + + // the second TS should be TS=2 or higher., even though 'time=1' right now. + + + // how many key/values for this row are there? + Get get = new Get(row); + get.addColumn(family, qf1); + get.setMaxVersions(); // all versions. + List results = new ArrayList(); + + results = HBaseTestingUtility.getFromStoreFile(store, get); + assertEquals(2, results.size()); + + long ts1 = results.get(0).getTimestamp(); + long ts2 = results.get(1).getTimestamp(); + + assertTrue(ts1 > ts2); + assertEquals(newValue, Bytes.toLong(results.get(0).getValue())); + assertEquals(oldValue, Bytes.toLong(results.get(1).getValue())); + + mee.setValue(2); // time goes up slightly + newValue += 1; + this.store.updateColumnValue(row, family, qf1, newValue); + + results = HBaseTestingUtility.getFromStoreFile(store, get); + assertEquals(2, results.size()); + + ts1 = results.get(0).getTimestamp(); + ts2 = results.get(1).getTimestamp(); + + assertTrue(ts1 > ts2); + assertEquals(newValue, Bytes.toLong(results.get(0).getValue())); + assertEquals(oldValue, Bytes.toLong(results.get(1).getValue())); + } + + public void testHandleErrorsInFlush() throws Exception { + LOG.info("Setting up a faulty file system that cannot write"); + + final Configuration conf = HBaseConfiguration.create(); + User user = User.createUserForTesting(conf, + "testhandleerrorsinflush", new String[]{"foo"}); + // Inject our faulty LocalFileSystem + conf.setClass("fs.file.impl", FaultyFileSystem.class, + FileSystem.class); + user.runAs(new PrivilegedExceptionAction() { + public Object run() throws Exception { + // Make sure it worked (above is sensitive to caching details in hadoop core) + FileSystem fs = FileSystem.get(conf); + assertEquals(FaultyFileSystem.class, fs.getClass()); + + // Initialize region + init(getName(), conf); + + LOG.info("Adding some data"); + store.add(new KeyValue(row, family, qf1, 1, (byte[])null)); + store.add(new KeyValue(row, family, qf2, 1, (byte[])null)); + store.add(new KeyValue(row, family, qf3, 1, (byte[])null)); + + LOG.info("Before flush, we should have no files"); + FileStatus[] files = fs.listStatus(store.getHomedir()); + Path[] paths = FileUtil.stat2Paths(files); + System.err.println("Got paths: " + Joiner.on(",").join(paths)); + assertEquals(0, paths.length); + + //flush + try { + LOG.info("Flushing"); + flush(1); + fail("Didn't bubble up IOE!"); + } catch (IOException ioe) { + assertTrue(ioe.getMessage().contains("Fault injected")); + } + + LOG.info("After failed flush, we should still have no files!"); + files = fs.listStatus(store.getHomedir()); + paths = FileUtil.stat2Paths(files); + System.err.println("Got paths: " + Joiner.on(",").join(paths)); + assertEquals(0, paths.length); + return null; + } + }); + } + + + static class FaultyFileSystem extends FilterFileSystem { + List> outStreams = + new ArrayList>(); + private long faultPos = 200; + + public FaultyFileSystem() { + super(new LocalFileSystem()); + System.err.println("Creating faulty!"); + } + + @Override + public FSDataOutputStream create(Path p) throws IOException { + return new FaultyOutputStream(super.create(p), faultPos); + } + + @Override + public FSDataOutputStream create(Path f, FsPermission permission, + boolean overwrite, int bufferSize, short replication, long blockSize, + Progressable progress) throws IOException { + return new FaultyOutputStream(super.create(f, permission, + overwrite, bufferSize, replication, blockSize, progress), faultPos); + } + + public FSDataOutputStream createNonRecursive(Path f, boolean overwrite, + int bufferSize, short replication, long blockSize, Progressable progress) + throws IOException { + // Fake it. Call create instead. The default implementation throws an IOE + // that this is not supported. + return create(f, overwrite, bufferSize, replication, blockSize, progress); + } + } + + static class FaultyOutputStream extends FSDataOutputStream { + volatile long faultPos = Long.MAX_VALUE; + + public FaultyOutputStream(FSDataOutputStream out, + long faultPos) throws IOException { + super(out, null); + this.faultPos = faultPos; + } + + @Override + public void write(byte[] buf, int offset, int length) throws IOException { + System.err.println("faulty stream write at pos " + getPos()); + injectFault(); + super.write(buf, offset, length); + } + + private void injectFault() throws IOException { + if (getPos() >= faultPos) { + throw new IOException("Fault injected"); + } + } + } + + + + private static void flushStore(Store store, long id) throws IOException { + StoreFlusher storeFlusher = store.getStoreFlusher(id); + storeFlusher.prepare(); + storeFlusher.flushCache(Mockito.mock(MonitoredTask.class)); + storeFlusher.commit(Mockito.mock(MonitoredTask.class)); + } + + + + /** + * Generate a list of KeyValues for testing based on given parameters + * @param timestamps + * @param numRows + * @param qualifier + * @param family + * @return + */ + List getKeyValueSet(long[] timestamps, int numRows, + byte[] qualifier, byte[] family) { + List kvList = new ArrayList(); + for (int i=1;i<=numRows;i++) { + byte[] b = Bytes.toBytes(i); + for (long timestamp: timestamps) { + kvList.add(new KeyValue(b, family, qualifier, timestamp, b)); + } + } + return kvList; + } + + /** + * Test to ensure correctness when using Stores with multiple timestamps + * @throws IOException + */ + public void testMultipleTimestamps() throws IOException { + int numRows = 1; + long[] timestamps1 = new long[] {1,5,10,20}; + long[] timestamps2 = new long[] {30,80}; + + init(this.getName()); + + List kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family); + for (KeyValue kv : kvList1) { + this.store.add(kv); + } + + this.store.snapshot(); + flushStore(store, id++); + + List kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family); + for(KeyValue kv : kvList2) { + this.store.add(kv); + } + + List result; + Get get = new Get(Bytes.toBytes(1)); + get.addColumn(family,qf1); + + get.setTimeRange(0,15); + result = HBaseTestingUtility.getFromStoreFile(store, get); + assertTrue(result.size()>0); + + get.setTimeRange(40,90); + result = HBaseTestingUtility.getFromStoreFile(store, get); + assertTrue(result.size()>0); + + get.setTimeRange(10,45); + result = HBaseTestingUtility.getFromStoreFile(store, get); + assertTrue(result.size()>0); + + get.setTimeRange(80,145); + result = HBaseTestingUtility.getFromStoreFile(store, get); + assertTrue(result.size()>0); + + get.setTimeRange(1,2); + result = HBaseTestingUtility.getFromStoreFile(store, get); + assertTrue(result.size()>0); + + get.setTimeRange(90,200); + result = HBaseTestingUtility.getFromStoreFile(store, get); + assertTrue(result.size()==0); + } + + /** + * Test for HBASE-3492 - Test split on empty colfam (no store files). + * + * @throws IOException When the IO operations fail. + */ + public void testSplitWithEmptyColFam() throws IOException { + init(this.getName()); + assertNull(store.getSplitPoint()); + store.getHRegion().forceSplit(null); + assertNull(store.getSplitPoint()); + store.getHRegion().clearSplit_TESTS_ONLY(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java new file mode 100644 index 0000000..a8f2f84 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java @@ -0,0 +1,1067 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.io.HalfStoreFileReader; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.io.Reference.Range; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.CacheStats; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; +import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.util.BloomFilterFactory; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ChecksumType; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import com.google.common.base.Joiner; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; + +/** + * Test HStoreFile + */ +@Category(SmallTests.class) +public class TestStoreFile extends HBaseTestCase { + static final Log LOG = LogFactory.getLog(TestStoreFile.class); + private CacheConfig cacheConf = new CacheConfig(conf); + private String ROOT_DIR; + private Map startingMetrics; + + private static final ChecksumType CKTYPE = ChecksumType.CRC32; + private static final int CKBYTES = 512; + + @Override + public void setUp() throws Exception { + super.setUp(); + startingMetrics = SchemaMetrics.getMetricsSnapshot(); + ROOT_DIR = new Path(this.testDir, "TestStoreFile").toString(); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + SchemaMetrics.validateMetricChanges(startingMetrics); + } + + /** + * Write a file and then assert that we can read from top and bottom halves + * using two HalfMapFiles. + * @throws Exception + */ + public void testBasicHalfMapFile() throws Exception { + // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname. + Path outputDir = new Path(new Path(this.testDir, "7e0102"), + "familyname"); + StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, + this.fs, 2 * 1024) + .withOutputDir(outputDir) + .build(); + writeStoreFile(writer); + checkHalfHFile(new StoreFile(this.fs, writer.getPath(), conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE)); + } + + private void writeStoreFile(final StoreFile.Writer writer) throws IOException { + writeStoreFile(writer, Bytes.toBytes(getName()), Bytes.toBytes(getName())); + } + + // pick an split point (roughly halfway) + byte[] SPLITKEY = new byte[] { (LAST_CHAR + FIRST_CHAR)/2, FIRST_CHAR}; + + /* + * Writes HStoreKey and ImmutableBytes data to passed writer and + * then closes it. + * @param writer + * @throws IOException + */ + public static void writeStoreFile(final StoreFile.Writer writer, byte[] fam, byte[] qualifier) + throws IOException { + long now = System.currentTimeMillis(); + try { + for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) { + for (char e = FIRST_CHAR; e <= LAST_CHAR; e++) { + byte[] b = new byte[] { (byte) d, (byte) e }; + writer.append(new KeyValue(b, fam, qualifier, now, b)); + } + } + } finally { + writer.close(); + } + } + + /** + * Test that our mechanism of writing store files in one region to reference + * store files in other regions works. + * @throws IOException + */ + public void testReference() + throws IOException { + // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname. + Path storedir = new Path(new Path(this.testDir, "7e0102"), "familyname"); + // Make a store file and write data to it. + StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, + this.fs, 8 * 1024) + .withOutputDir(storedir) + .build(); + writeStoreFile(writer); + StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + StoreFile.Reader reader = hsf.createReader(); + // Split on a row, not in middle of row. Midkey returned by reader + // may be in middle of row. Create new one with empty column and + // timestamp. + KeyValue kv = KeyValue.createKeyValueFromKey(reader.midkey()); + byte [] midRow = kv.getRow(); + kv = KeyValue.createKeyValueFromKey(reader.getLastKey()); + byte [] finalRow = kv.getRow(); + // Make a reference + Path refPath = StoreFile.split(fs, storedir, hsf, midRow, Range.top); + StoreFile refHsf = new StoreFile(this.fs, refPath, conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + // Now confirm that I can read from the reference and that it only gets + // keys from top half of the file. + HFileScanner s = refHsf.createReader().getScanner(false, false); + for(boolean first = true; (!s.isSeeked() && s.seekTo()) || s.next();) { + ByteBuffer bb = s.getKey(); + kv = KeyValue.createKeyValueFromKey(bb); + if (first) { + assertTrue(Bytes.equals(kv.getRow(), midRow)); + first = false; + } + } + assertTrue(Bytes.equals(kv.getRow(), finalRow)); + } + + public void testHFileLink() throws IOException { + final String columnFamily = "f"; + + Configuration testConf = new Configuration(this.conf); + FSUtils.setRootDir(testConf, this.testDir); + + HRegionInfo hri = new HRegionInfo(Bytes.toBytes("table-link")); + Path storedir = new Path(new Path(this.testDir, + new Path(hri.getTableNameAsString(), hri.getEncodedName())), columnFamily); + + // Make a store file and write data to it. + StoreFile.Writer writer = new StoreFile.WriterBuilder(testConf, cacheConf, + this.fs, 8 * 1024) + .withOutputDir(storedir) + .build(); + Path storeFilePath = writer.getPath(); + writeStoreFile(writer); + writer.close(); + + Path dstPath = new Path(this.testDir, new Path("test-region", columnFamily)); + HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName()); + Path linkFilePath = new Path(dstPath, + HFileLink.createHFileLinkName(hri, storeFilePath.getName())); + + // Try to open store file from link + StoreFile hsf = new StoreFile(this.fs, linkFilePath, testConf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + assertTrue(hsf.isLink()); + + // Now confirm that I can read from the link + int count = 1; + HFileScanner s = hsf.createReader().getScanner(false, false); + s.seekTo(); + while (s.next()) { + count++; + } + assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count); + } + + /** + * Validate that we can handle valid tables with '.', '_', and '-' chars. + */ + public void testStoreFileNames() { + String[] legalHFileLink = { "MyTable_02=abc012-def345", "MyTable_02.300=abc012-def345", + "MyTable_02-400=abc012-def345", "MyTable_02-400.200=abc012-def345", + "MyTable_02=abc012-def345_SeqId_1_", "MyTable_02=abc012-def345_SeqId_20_" }; + for (String name: legalHFileLink) { + assertTrue("should be a valid link: " + name, HFileLink.isHFileLink(name)); + assertTrue("should be a valid StoreFile" + name, StoreFile.validateStoreFileName(name)); + assertFalse("should not be a valid reference: " + name, StoreFile.isReference(name)); + + String refName = name + ".6789"; + assertTrue("should be a valid link reference: " + refName, StoreFile.isReference(refName)); + assertTrue("should be a valid StoreFile" + refName, StoreFile.validateStoreFileName(refName)); + } + + String[] illegalHFileLink = { ".MyTable_02=abc012-def345", "-MyTable_02.300=abc012-def345", + "MyTable_02-400=abc0_12-def345", "MyTable_02-400.200=abc012-def345...." }; + for (String name: illegalHFileLink) { + assertFalse("should not be a valid link: " + name, HFileLink.isHFileLink(name)); + } + } + + /** + * This test creates an hfile and then the dir structures and files to verify that references + * to hfilelinks (created by snapshot clones) can be properly interpreted. + */ + public void testReferenceToHFileLink() throws IOException { + final String columnFamily = "f"; + + Path rootDir = FSUtils.getRootDir(conf); + + String tablename = "_original-evil-name"; // adding legal table name chars to verify regex handles it. + HRegionInfo hri = new HRegionInfo(Bytes.toBytes(tablename)); + // store dir = /// + Path storedir = new Path(new Path(rootDir, + new Path(hri.getTableNameAsString(), hri.getEncodedName())), columnFamily); + + // Make a store file and write data to it. //// + StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, + this.fs, 8 * 1024) + .withOutputDir(storedir) + .build(); + Path storeFilePath = writer.getPath(); + writeStoreFile(writer); + writer.close(); + + // create link to store file. /clone/region//--

    + String target = "clone"; + Path dstPath = new Path(rootDir, new Path(new Path(target, "7e0102"), columnFamily)); + HFileLink.create(conf, this.fs, dstPath, hri, storeFilePath.getName()); + Path linkFilePath = new Path(dstPath, + HFileLink.createHFileLinkName(hri, storeFilePath.getName())); + + // create splits of the link. + // /clone/splitA//, + // /clone/splitB// + Path splitDirA = new Path(new Path(rootDir, + new Path(target, "571A")), columnFamily); + Path splitDirB = new Path(new Path(rootDir, + new Path(target, "571B")), columnFamily); + StoreFile f = new StoreFile(fs, linkFilePath, conf, cacheConf, BloomType.NONE, + NoOpDataBlockEncoder.INSTANCE); + byte[] splitRow = SPLITKEY; + Path pathA = StoreFile.split(fs, splitDirA, f, splitRow, Range.top); // top + Path pathB = StoreFile.split(fs, splitDirB, f, splitRow, Range.bottom); // bottom + + // OK test the thing + FSUtils.logFileSystemState(fs, rootDir, LOG); + + // There is a case where a file with the hfilelink pattern is actually a daughter + // reference to a hfile link. This code in StoreFile that handles this case. + + // Try to open store file from link + StoreFile hsfA = new StoreFile(this.fs, pathA, conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + + // Now confirm that I can read from the ref to link + int count = 1; + HFileScanner s = hsfA.createReader().getScanner(false, false); + s.seekTo(); + while (s.next()) { + count++; + } + assertTrue(count > 0); // read some rows here + + // Try to open store file from link + StoreFile hsfB = new StoreFile(this.fs, pathB, conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + + // Now confirm that I can read from the ref to link + HFileScanner sB = hsfB.createReader().getScanner(false, false); + sB.seekTo(); + + //count++ as seekTo() will advance the scanner + count++; + while (sB.next()) { + count++; + } + + // read the rest of the rows + assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count); + } + + private void checkHalfHFile(final StoreFile f) + throws IOException { + byte [] midkey = f.createReader().midkey(); + KeyValue midKV = KeyValue.createKeyValueFromKey(midkey); + byte [] midRow = midKV.getRow(); + // Create top split. + Path topDir = Store.getStoreHomedir(this.testDir, "1", + Bytes.toBytes(f.getPath().getParent().getName())); + if (this.fs.exists(topDir)) { + this.fs.delete(topDir, true); + } + Path topPath = StoreFile.split(this.fs, topDir, f, midRow, Range.top); + // Create bottom split. + Path bottomDir = Store.getStoreHomedir(this.testDir, "2", + Bytes.toBytes(f.getPath().getParent().getName())); + if (this.fs.exists(bottomDir)) { + this.fs.delete(bottomDir, true); + } + Path bottomPath = StoreFile.split(this.fs, bottomDir, + f, midRow, Range.bottom); + // Make readers on top and bottom. + StoreFile.Reader top = + new StoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE, + NoOpDataBlockEncoder.INSTANCE).createReader(); + StoreFile.Reader bottom = new StoreFile(this.fs, bottomPath, + conf, cacheConf, BloomType.NONE, + NoOpDataBlockEncoder.INSTANCE).createReader(); + ByteBuffer previous = null; + LOG.info("Midkey: " + midKV.toString()); + ByteBuffer bbMidkeyBytes = ByteBuffer.wrap(midkey); + try { + // Now make two HalfMapFiles and assert they can read the full backing + // file, one from the top and the other from the bottom. + // Test bottom half first. + // Now test reading from the top. + boolean first = true; + ByteBuffer key = null; + HFileScanner topScanner = top.getScanner(false, false); + while ((!topScanner.isSeeked() && topScanner.seekTo()) || + (topScanner.isSeeked() && topScanner.next())) { + key = topScanner.getKey(); + + if (topScanner.getReader().getComparator().compare(key.array(), + key.arrayOffset(), key.limit(), midkey, 0, midkey.length) < 0) { + fail("key=" + Bytes.toStringBinary(key) + " < midkey=" + + Bytes.toStringBinary(midkey)); + } + if (first) { + first = false; + LOG.info("First in top: " + Bytes.toString(Bytes.toBytes(key))); + } + } + LOG.info("Last in top: " + Bytes.toString(Bytes.toBytes(key))); + + first = true; + HFileScanner bottomScanner = bottom.getScanner(false, false); + while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || + bottomScanner.next()) { + previous = bottomScanner.getKey(); + key = bottomScanner.getKey(); + if (first) { + first = false; + LOG.info("First in bottom: " + + Bytes.toString(Bytes.toBytes(previous))); + } + assertTrue(key.compareTo(bbMidkeyBytes) < 0); + } + if (previous != null) { + LOG.info("Last in bottom: " + Bytes.toString(Bytes.toBytes(previous))); + } + // Remove references. + this.fs.delete(topPath, false); + this.fs.delete(bottomPath, false); + + // Next test using a midkey that does not exist in the file. + // First, do a key that is < than first key. Ensure splits behave + // properly. + byte [] badmidkey = Bytes.toBytes(" ."); + topPath = StoreFile.split(this.fs, topDir, f, badmidkey, Range.top); + bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey, + Range.bottom); + + assertNull(bottomPath); + + top = new StoreFile(this.fs, topPath, conf, cacheConf, + StoreFile.BloomType.NONE, + NoOpDataBlockEncoder.INSTANCE).createReader(); + // Now read from the top. + first = true; + topScanner = top.getScanner(false, false); + while ((!topScanner.isSeeked() && topScanner.seekTo()) || + topScanner.next()) { + key = topScanner.getKey(); + assertTrue(topScanner.getReader().getComparator().compare(key.array(), + key.arrayOffset(), key.limit(), badmidkey, 0, badmidkey.length) >= 0); + if (first) { + first = false; + KeyValue keyKV = KeyValue.createKeyValueFromKey(key); + LOG.info("First top when key < bottom: " + keyKV); + String tmp = Bytes.toString(keyKV.getRow()); + for (int i = 0; i < tmp.length(); i++) { + assertTrue(tmp.charAt(i) == 'a'); + } + } + } + KeyValue keyKV = KeyValue.createKeyValueFromKey(key); + LOG.info("Last top when key < bottom: " + keyKV); + String tmp = Bytes.toString(keyKV.getRow()); + for (int i = 0; i < tmp.length(); i++) { + assertTrue(tmp.charAt(i) == 'z'); + } + // Remove references. + this.fs.delete(topPath, false); + + // Test when badkey is > than last key in file ('||' > 'zz'). + badmidkey = Bytes.toBytes("|||"); + topPath = StoreFile.split(this.fs, topDir, f, badmidkey, Range.top); + bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey, + Range.bottom); + + assertNull(topPath); + + bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf, + StoreFile.BloomType.NONE, + NoOpDataBlockEncoder.INSTANCE).createReader(); + first = true; + bottomScanner = bottom.getScanner(false, false); + while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || + bottomScanner.next()) { + key = bottomScanner.getKey(); + if (first) { + first = false; + keyKV = KeyValue.createKeyValueFromKey(key); + LOG.info("First bottom when key > top: " + keyKV); + tmp = Bytes.toString(keyKV.getRow()); + for (int i = 0; i < tmp.length(); i++) { + assertTrue(tmp.charAt(i) == 'a'); + } + } + } + keyKV = KeyValue.createKeyValueFromKey(key); + LOG.info("Last bottom when key > top: " + keyKV); + for (int i = 0; i < tmp.length(); i++) { + assertTrue(Bytes.toString(keyKV.getRow()).charAt(i) == 'z'); + } + } finally { + if (top != null) { + top.close(true); // evict since we are about to delete the file + } + if (bottom != null) { + bottom.close(true); // evict since we are about to delete the file + } + fs.delete(f.getPath(), true); + } + } + + private static final String localFormatter = "%010d"; + + private void bloomWriteRead(StoreFile.Writer writer, FileSystem fs) + throws Exception { + float err = conf.getFloat( + BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, 0); + Path f = writer.getPath(); + long now = System.currentTimeMillis(); + for (int i = 0; i < 2000; i += 2) { + String row = String.format(localFormatter, i); + KeyValue kv = new KeyValue(row.getBytes(), "family".getBytes(), + "col".getBytes(), now, "value".getBytes()); + writer.append(kv); + } + writer.close(); + + StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf, + DataBlockEncoding.NONE); + reader.loadFileInfo(); + reader.loadBloomfilter(); + StoreFileScanner scanner = reader.getStoreFileScanner(false, false); + + // check false positives rate + int falsePos = 0; + int falseNeg = 0; + for (int i = 0; i < 2000; i++) { + String row = String.format(localFormatter, i); + TreeSet columns = new TreeSet(Bytes.BYTES_COMPARATOR); + columns.add("family:col".getBytes()); + + Scan scan = new Scan(row.getBytes(),row.getBytes()); + scan.addColumn("family".getBytes(), "family:col".getBytes()); + boolean exists = scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE); + if (i % 2 == 0) { + if (!exists) falseNeg++; + } else { + if (exists) falsePos++; + } + } + reader.close(true); // evict because we are about to delete the file + fs.delete(f, true); + assertEquals("False negatives: " + falseNeg, 0, falseNeg); + int maxFalsePos = (int) (2 * 2000 * err); + assertTrue("Too many false positives: " + falsePos + " (err=" + err + + ", expected no more than " + maxFalsePos + ")", + falsePos <= maxFalsePos); + } + + public void testBloomFilter() throws Exception { + FileSystem fs = FileSystem.getLocal(conf); + conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, + (float) 0.01); + conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true); + + // write the file + Path f = new Path(ROOT_DIR, getName()); + StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs, + StoreFile.DEFAULT_BLOCKSIZE_SMALL) + .withFilePath(f) + .withBloomType(StoreFile.BloomType.ROW) + .withMaxKeyCount(2000) + .withChecksumType(CKTYPE) + .withBytesPerChecksum(CKBYTES) + .build(); + bloomWriteRead(writer, fs); + } + + public void testDeleteFamilyBloomFilter() throws Exception { + FileSystem fs = FileSystem.getLocal(conf); + conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, + (float) 0.01); + conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true); + float err = conf.getFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, + 0); + + // write the file + Path f = new Path(ROOT_DIR, getName()); + + StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, + fs, StoreFile.DEFAULT_BLOCKSIZE_SMALL) + .withFilePath(f) + .withMaxKeyCount(2000) + .withChecksumType(CKTYPE) + .withBytesPerChecksum(CKBYTES) + .build(); + + // add delete family + long now = System.currentTimeMillis(); + for (int i = 0; i < 2000; i += 2) { + String row = String.format(localFormatter, i); + KeyValue kv = new KeyValue(row.getBytes(), "family".getBytes(), + "col".getBytes(), now, KeyValue.Type.DeleteFamily, "value".getBytes()); + writer.append(kv); + } + writer.close(); + + StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf, + DataBlockEncoding.NONE); + reader.loadFileInfo(); + reader.loadBloomfilter(); + + // check false positives rate + int falsePos = 0; + int falseNeg = 0; + for (int i = 0; i < 2000; i++) { + String row = String.format(localFormatter, i); + byte[] rowKey = Bytes.toBytes(row); + boolean exists = reader.passesDeleteFamilyBloomFilter(rowKey, 0, + rowKey.length); + if (i % 2 == 0) { + if (!exists) + falseNeg++; + } else { + if (exists) + falsePos++; + } + } + assertEquals(1000, reader.getDeleteFamilyCnt()); + reader.close(true); // evict because we are about to delete the file + fs.delete(f, true); + assertEquals("False negatives: " + falseNeg, 0, falseNeg); + int maxFalsePos = (int) (2 * 2000 * err); + assertTrue("Too many false positives: " + falsePos + " (err=" + err + + ", expected no more than " + maxFalsePos, falsePos <= maxFalsePos); + } + + public void testBloomTypes() throws Exception { + float err = (float) 0.01; + FileSystem fs = FileSystem.getLocal(conf); + conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, err); + conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true); + + int rowCount = 50; + int colCount = 10; + int versions = 2; + + // run once using columns and once using rows + StoreFile.BloomType[] bt = + {StoreFile.BloomType.ROWCOL, StoreFile.BloomType.ROW}; + int[] expKeys = {rowCount*colCount, rowCount}; + // below line deserves commentary. it is expected bloom false positives + // column = rowCount*2*colCount inserts + // row-level = only rowCount*2 inserts, but failures will be magnified by + // 2nd for loop for every column (2*colCount) + float[] expErr = {2*rowCount*colCount*err, 2*rowCount*2*colCount*err}; + + for (int x : new int[]{0,1}) { + // write the file + Path f = new Path(ROOT_DIR, getName() + x); + StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, + fs, StoreFile.DEFAULT_BLOCKSIZE_SMALL) + .withFilePath(f) + .withBloomType(bt[x]) + .withMaxKeyCount(expKeys[x]) + .withChecksumType(CKTYPE) + .withBytesPerChecksum(CKBYTES) + .build(); + + long now = System.currentTimeMillis(); + for (int i = 0; i < rowCount*2; i += 2) { // rows + for (int j = 0; j < colCount*2; j += 2) { // column qualifiers + String row = String.format(localFormatter, i); + String col = String.format(localFormatter, j); + for (int k= 0; k < versions; ++k) { // versions + KeyValue kv = new KeyValue(row.getBytes(), + "family".getBytes(), ("col" + col).getBytes(), + now-k, Bytes.toBytes((long)-1)); + writer.append(kv); + } + } + } + writer.close(); + + StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf, + DataBlockEncoding.NONE); + reader.loadFileInfo(); + reader.loadBloomfilter(); + StoreFileScanner scanner = reader.getStoreFileScanner(false, false); + assertEquals(expKeys[x], reader.generalBloomFilter.getKeyCount()); + + // check false positives rate + int falsePos = 0; + int falseNeg = 0; + for (int i = 0; i < rowCount*2; ++i) { // rows + for (int j = 0; j < colCount*2; ++j) { // column qualifiers + String row = String.format(localFormatter, i); + String col = String.format(localFormatter, j); + TreeSet columns = new TreeSet(Bytes.BYTES_COMPARATOR); + columns.add(("col" + col).getBytes()); + + Scan scan = new Scan(row.getBytes(),row.getBytes()); + scan.addColumn("family".getBytes(), ("col"+col).getBytes()); + boolean exists = + scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE); + boolean shouldRowExist = i % 2 == 0; + boolean shouldColExist = j % 2 == 0; + shouldColExist = shouldColExist || bt[x] == StoreFile.BloomType.ROW; + if (shouldRowExist && shouldColExist) { + if (!exists) falseNeg++; + } else { + if (exists) falsePos++; + } + } + } + reader.close(true); // evict because we are about to delete the file + fs.delete(f, true); + System.out.println(bt[x].toString()); + System.out.println(" False negatives: " + falseNeg); + System.out.println(" False positives: " + falsePos); + assertEquals(0, falseNeg); + assertTrue(falsePos < 2*expErr[x]); + } + } + + public void testBloomEdgeCases() throws Exception { + float err = (float)0.005; + FileSystem fs = FileSystem.getLocal(conf); + Path f = new Path(ROOT_DIR, getName()); + conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, err); + conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true); + conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_MAX_KEYS, 1000); + + // This test only runs for HFile format version 1. + conf.setInt(HFile.FORMAT_VERSION_KEY, 1); + + // this should not create a bloom because the max keys is too small + StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs, + StoreFile.DEFAULT_BLOCKSIZE_SMALL) + .withFilePath(f) + .withBloomType(StoreFile.BloomType.ROW) + .withMaxKeyCount(2000) + .withChecksumType(CKTYPE) + .withBytesPerChecksum(CKBYTES) + .build(); + assertFalse(writer.hasGeneralBloom()); + writer.close(); + fs.delete(f, true); + + conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_MAX_KEYS, + Integer.MAX_VALUE); + + // TODO: commented out because we run out of java heap space on trunk + // the below config caused IllegalArgumentException in our production cluster + // however, the resulting byteSize is < MAX_INT, so this should work properly + writer = new StoreFile.WriterBuilder(conf, cacheConf, fs, + StoreFile.DEFAULT_BLOCKSIZE_SMALL) + .withFilePath(f) + .withBloomType(StoreFile.BloomType.ROW) + .withMaxKeyCount(27244696) + .build(); + assertTrue(writer.hasGeneralBloom()); + bloomWriteRead(writer, fs); + + // this, however, is too large and should not create a bloom + // because Java can't create a contiguous array > MAX_INT + writer = new StoreFile.WriterBuilder(conf, cacheConf, fs, + StoreFile.DEFAULT_BLOCKSIZE_SMALL) + .withFilePath(f) + .withBloomType(StoreFile.BloomType.ROW) + .withMaxKeyCount(Integer.MAX_VALUE) + .withChecksumType(CKTYPE) + .withBytesPerChecksum(CKBYTES) + .build(); + assertFalse(writer.hasGeneralBloom()); + writer.close(); + fs.delete(f, true); + } + + public void testFlushTimeComparator() { + assertOrdering(StoreFile.Comparators.FLUSH_TIME, + mockStoreFile(true, 1000, -1, "/foo/123"), + mockStoreFile(true, 1000, -1, "/foo/126"), + mockStoreFile(true, 2000, -1, "/foo/126"), + mockStoreFile(false, -1, 1, "/foo/1"), + mockStoreFile(false, -1, 3, "/foo/2"), + mockStoreFile(false, -1, 5, "/foo/2"), + mockStoreFile(false, -1, 5, "/foo/3")); + } + + /** + * Assert that the given comparator orders the given storefiles in the + * same way that they're passed. + */ + private void assertOrdering(Comparator comparator, StoreFile ... sfs) { + ArrayList sorted = Lists.newArrayList(sfs); + Collections.shuffle(sorted); + Collections.sort(sorted, comparator); + LOG.debug("sfs: " + Joiner.on(",").join(sfs)); + LOG.debug("sorted: " + Joiner.on(",").join(sorted)); + assertTrue(Iterables.elementsEqual(Arrays.asList(sfs), sorted)); + } + + /** + * Create a mock StoreFile with the given attributes. + */ + private StoreFile mockStoreFile(boolean bulkLoad, long bulkTimestamp, + long seqId, String path) { + StoreFile mock = Mockito.mock(StoreFile.class); + Mockito.doReturn(bulkLoad).when(mock).isBulkLoadResult(); + Mockito.doReturn(bulkTimestamp).when(mock).getBulkLoadTimestamp(); + if (bulkLoad) { + // Bulk load files will throw if you ask for their sequence ID + Mockito.doThrow(new IllegalAccessError("bulk load")) + .when(mock).getMaxSequenceId(); + } else { + Mockito.doReturn(seqId).when(mock).getMaxSequenceId(); + } + Mockito.doReturn(new Path(path)).when(mock).getPath(); + String name = "mock storefile, bulkLoad=" + bulkLoad + + " bulkTimestamp=" + bulkTimestamp + + " seqId=" + seqId + + " path=" + path; + Mockito.doReturn(name).when(mock).toString(); + return mock; + } + + /** + * Generate a list of KeyValues for testing based on given parameters + * @param timestamps + * @param numRows + * @param qualifier + * @param family + * @return + */ + List getKeyValueSet(long[] timestamps, int numRows, + byte[] qualifier, byte[] family) { + List kvList = new ArrayList(); + for (int i=1;i<=numRows;i++) { + byte[] b = Bytes.toBytes(i) ; + LOG.info(Bytes.toString(b)); + LOG.info(Bytes.toString(b)); + for (long timestamp: timestamps) + { + kvList.add(new KeyValue(b, family, qualifier, timestamp, b)); + } + } + return kvList; + } + + /** + * Test to ensure correctness when using StoreFile with multiple timestamps + * @throws IOException + */ + public void testMultipleTimestamps() throws IOException { + byte[] family = Bytes.toBytes("familyname"); + byte[] qualifier = Bytes.toBytes("qualifier"); + int numRows = 10; + long[] timestamps = new long[] {20,10,5,1}; + Scan scan = new Scan(); + + // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname. + Path storedir = new Path(new Path(this.testDir, "7e0102"), "familyname"); + Path dir = new Path(storedir, "1234567890"); + StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, + this.fs, 8 * 1024) + .withOutputDir(dir) + .build(); + + List kvList = getKeyValueSet(timestamps,numRows, + family, qualifier); + + for (KeyValue kv : kvList) { + writer.append(kv); + } + writer.appendMetadata(0, false); + writer.close(); + + StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + StoreFile.Reader reader = hsf.createReader(); + StoreFileScanner scanner = reader.getStoreFileScanner(false, false); + TreeSet columns = new TreeSet(Bytes.BYTES_COMPARATOR); + columns.add(qualifier); + + scan.setTimeRange(20, 100); + assertTrue(scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE)); + + scan.setTimeRange(1, 2); + assertTrue(scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE)); + + scan.setTimeRange(8, 10); + assertTrue(scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE)); + + scan.setTimeRange(7, 50); + assertTrue(scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE)); + + // This test relies on the timestamp range optimization + scan.setTimeRange(27, 50); + assertTrue(!scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE)); + } + + public void testCacheOnWriteEvictOnClose() throws Exception { + Configuration conf = this.conf; + + // Find a home for our files (regiondir ("7e0102") and familyname). + Path baseDir = new Path(new Path(this.testDir, "7e0102"),"twoCOWEOC"); + + // Grab the block cache and get the initial hit/miss counts + BlockCache bc = new CacheConfig(conf).getBlockCache(); + assertNotNull(bc); + CacheStats cs = bc.getStats(); + long startHit = cs.getHitCount(); + long startMiss = cs.getMissCount(); + long startEvicted = cs.getEvictedCount(); + + // Let's write a StoreFile with three blocks, with cache on write off + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false); + CacheConfig cacheConf = new CacheConfig(conf); + Path pathCowOff = new Path(baseDir, "123456789"); + StoreFile.Writer writer = writeStoreFile(conf, cacheConf, pathCowOff, 3); + StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + LOG.debug(hsf.getPath().toString()); + + // Read this file, we should see 3 misses + StoreFile.Reader reader = hsf.createReader(); + reader.loadFileInfo(); + StoreFileScanner scanner = reader.getStoreFileScanner(true, true); + scanner.seek(KeyValue.LOWESTKEY); + while (scanner.next() != null); + assertEquals(startHit, cs.getHitCount()); + assertEquals(startMiss + 3, cs.getMissCount()); + assertEquals(startEvicted, cs.getEvictedCount()); + startMiss += 3; + scanner.close(); + reader.close(cacheConf.shouldEvictOnClose()); + + // Now write a StoreFile with three blocks, with cache on write on + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true); + cacheConf = new CacheConfig(conf); + Path pathCowOn = new Path(baseDir, "123456788"); + writer = writeStoreFile(conf, cacheConf, pathCowOn, 3); + hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + + // Read this file, we should see 3 hits + reader = hsf.createReader(); + scanner = reader.getStoreFileScanner(true, true); + scanner.seek(KeyValue.LOWESTKEY); + while (scanner.next() != null); + assertEquals(startHit + 3, cs.getHitCount()); + assertEquals(startMiss, cs.getMissCount()); + assertEquals(startEvicted, cs.getEvictedCount()); + startHit += 3; + scanner.close(); + reader.close(cacheConf.shouldEvictOnClose()); + + // Let's read back the two files to ensure the blocks exactly match + hsf = new StoreFile(this.fs, pathCowOff, conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + StoreFile.Reader readerOne = hsf.createReader(); + readerOne.loadFileInfo(); + StoreFileScanner scannerOne = readerOne.getStoreFileScanner(true, true); + scannerOne.seek(KeyValue.LOWESTKEY); + hsf = new StoreFile(this.fs, pathCowOn, conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + StoreFile.Reader readerTwo = hsf.createReader(); + readerTwo.loadFileInfo(); + StoreFileScanner scannerTwo = readerTwo.getStoreFileScanner(true, true); + scannerTwo.seek(KeyValue.LOWESTKEY); + KeyValue kv1 = null; + KeyValue kv2 = null; + while ((kv1 = scannerOne.next()) != null) { + kv2 = scannerTwo.next(); + assertTrue(kv1.equals(kv2)); + assertTrue(Bytes.compareTo( + kv1.getBuffer(), kv1.getKeyOffset(), kv1.getKeyLength(), + kv2.getBuffer(), kv2.getKeyOffset(), kv2.getKeyLength()) == 0); + assertTrue(Bytes.compareTo( + kv1.getBuffer(), kv1.getValueOffset(), kv1.getValueLength(), + kv2.getBuffer(), kv2.getValueOffset(), kv2.getValueLength()) == 0); + } + assertNull(scannerTwo.next()); + assertEquals(startHit + 6, cs.getHitCount()); + assertEquals(startMiss, cs.getMissCount()); + assertEquals(startEvicted, cs.getEvictedCount()); + startHit += 6; + scannerOne.close(); + readerOne.close(cacheConf.shouldEvictOnClose()); + scannerTwo.close(); + readerTwo.close(cacheConf.shouldEvictOnClose()); + + // Let's close the first file with evict on close turned on + conf.setBoolean("hbase.rs.evictblocksonclose", true); + cacheConf = new CacheConfig(conf); + hsf = new StoreFile(this.fs, pathCowOff, conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + reader = hsf.createReader(); + reader.close(cacheConf.shouldEvictOnClose()); + + // We should have 3 new evictions + assertEquals(startHit, cs.getHitCount()); + assertEquals(startMiss, cs.getMissCount()); + assertEquals(startEvicted + 3, cs.getEvictedCount()); + startEvicted += 3; + + // Let's close the second file with evict on close turned off + conf.setBoolean("hbase.rs.evictblocksonclose", false); + cacheConf = new CacheConfig(conf); + hsf = new StoreFile(this.fs, pathCowOn, conf, cacheConf, + StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE); + reader = hsf.createReader(); + reader.close(cacheConf.shouldEvictOnClose()); + + // We expect no changes + assertEquals(startHit, cs.getHitCount()); + assertEquals(startMiss, cs.getMissCount()); + assertEquals(startEvicted, cs.getEvictedCount()); + } + + private StoreFile.Writer writeStoreFile(Configuration conf, + CacheConfig cacheConf, Path path, int numBlocks) + throws IOException { + // Let's put ~5 small KVs in each block, so let's make 5*numBlocks KVs + int numKVs = 5 * numBlocks; + List kvs = new ArrayList(numKVs); + byte [] b = Bytes.toBytes("x"); + int totalSize = 0; + for (int i=numKVs;i>0;i--) { + KeyValue kv = new KeyValue(b, b, b, i, b); + kvs.add(kv); + // kv has memstoreTS 0, which takes 1 byte to store. + totalSize += kv.getLength() + 1; + } + int blockSize = totalSize / numBlocks; + StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs, + blockSize) + .withFilePath(path) + .withMaxKeyCount(2000) + .withChecksumType(CKTYPE) + .withBytesPerChecksum(CKBYTES) + .build(); + // We'll write N-1 KVs to ensure we don't write an extra block + kvs.remove(kvs.size()-1); + for (KeyValue kv : kvs) { + writer.append(kv); + } + writer.appendMetadata(0, false); + writer.close(); + return writer; + } + + /** + * Check if data block encoding information is saved correctly in HFile's + * file info. + */ + public void testDataBlockEncodingMetaData() throws IOException { + // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname. + Path dir = new Path(new Path(this.testDir, "7e0102"), "familyname"); + Path path = new Path(dir, "1234567890"); + + DataBlockEncoding dataBlockEncoderAlgo = + DataBlockEncoding.FAST_DIFF; + HFileDataBlockEncoder dataBlockEncoder = + new HFileDataBlockEncoderImpl( + dataBlockEncoderAlgo, + dataBlockEncoderAlgo); + cacheConf = new CacheConfig(conf); + StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs, + HFile.DEFAULT_BLOCKSIZE) + .withFilePath(path) + .withDataBlockEncoder(dataBlockEncoder) + .withMaxKeyCount(2000) + .withChecksumType(CKTYPE) + .withBytesPerChecksum(CKBYTES) + .build(); + writer.close(); + + StoreFile storeFile = new StoreFile(fs, writer.getPath(), conf, + cacheConf, BloomType.NONE, dataBlockEncoder); + StoreFile.Reader reader = storeFile.createReader(); + + Map fileInfo = reader.loadFileInfo(); + byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING); + + assertEquals(dataBlockEncoderAlgo.getNameInBytes(), value); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java new file mode 100644 index 0000000..0ffb811 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileBlockCacheSummary.java @@ -0,0 +1,158 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.HTableUtil; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Row; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests the block cache summary functionality in StoreFile, + * which contains the BlockCache + * + */ +@Category(MediumTests.class) +public class TestStoreFileBlockCacheSummary { + final Log LOG = LogFactory.getLog(getClass()); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String TEST_TABLE = "testTable"; + private static final String TEST_TABLE2 = "testTable2"; + private static final String TEST_CF = "testFamily"; + private static byte [] FAMILY = Bytes.toBytes(TEST_CF); + private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); + private static byte [] VALUE = Bytes.toBytes("testValue"); + + private final int TOTAL_ROWS = 4; + + /** + * @throws java.lang.Exception exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + /** + * @throws java.lang.Exception exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + + private Put createPut(byte[] family, String row) { + Put put = new Put( Bytes.toBytes(row)); + put.add(family, QUALIFIER, VALUE); + return put; + } + + /** + * This test inserts data into multiple tables and then reads both tables to ensure + * they are in the block cache. + * + * @throws Exception exception + */ + @Test + public void testBlockCacheSummary() throws Exception { + HTable ht = TEST_UTIL.createTable(Bytes.toBytes(TEST_TABLE), FAMILY); + addRows(ht, FAMILY); + + HTable ht2 = TEST_UTIL.createTable(Bytes.toBytes(TEST_TABLE2), FAMILY); + addRows(ht2, FAMILY); + + TEST_UTIL.flush(); + + scan(ht, FAMILY); + scan(ht2, FAMILY); + + BlockCache bc = + new CacheConfig(TEST_UTIL.getConfiguration()).getBlockCache(); + List bcs = + bc.getBlockCacheColumnFamilySummaries(TEST_UTIL.getConfiguration()); + LOG.info("blockCacheSummary: " + bcs); + + assertEquals("blockCache summary has entries", 3, bcs.size()); + + BlockCacheColumnFamilySummary e = bcs.get(0); + assertEquals("table", "-ROOT-", e.getTable()); + assertEquals("cf", "info", e.getColumnFamily()); + + e = bcs.get(1); + assertEquals("table", TEST_TABLE, e.getTable()); + assertEquals("cf", TEST_CF, e.getColumnFamily()); + + e = bcs.get(2); + assertEquals("table", TEST_TABLE2, e.getTable()); + assertEquals("cf", TEST_CF, e.getColumnFamily()); + + ht.close(); + ht2.close(); + } + + private void addRows(HTable ht, byte[] family) throws IOException { + + List rows = new ArrayList(); + for (int i = 0; i < TOTAL_ROWS;i++) { + rows.add(createPut(family, "row" + i)); + } + + HTableUtil.bucketRsBatch( ht, rows); + } + + private void scan(HTable ht, byte[] family) throws IOException { + Scan scan = new Scan(); + scan.addColumn(family, QUALIFIER); + + int count = 0; + for(@SuppressWarnings("unused") Result result : ht.getScanner(scan)) { + count++; + } + if (TOTAL_ROWS != count) { + throw new IOException("Incorrect number of rows!"); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java new file mode 100644 index 0000000..01f0731 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java @@ -0,0 +1,583 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture. + scanFixture; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; + +import junit.framework.TestCase; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueTestUtil; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.Store.ScanInfo; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; +import org.junit.experimental.categories.Category; + +// Can't be small as it plays with EnvironmentEdgeManager +@Category(MediumTests.class) +public class TestStoreScanner extends TestCase { + private static final String CF_STR = "cf"; + final byte [] CF = Bytes.toBytes(CF_STR); + private ScanInfo scanInfo = new ScanInfo(CF, 0, Integer.MAX_VALUE, + Long.MAX_VALUE, false, 0, KeyValue.COMPARATOR); + private ScanType scanType = ScanType.USER_SCAN; + + public void setUp() throws Exception { + super.setUp(); + SchemaMetrics.setUseTableNameInTest(false); + } + + /* + * Test utility for building a NavigableSet for scanners. + * @param strCols + * @return + */ + NavigableSet getCols(String ...strCols) { + NavigableSet cols = new TreeSet(Bytes.BYTES_COMPARATOR); + for (String col : strCols) { + byte[] bytes = Bytes.toBytes(col); + cols.add(bytes); + } + return cols; + } + + public void testScanTimeRange() throws IOException { + String r1 = "R1"; + // returns only 1 of these 2 even though same timestamp + KeyValue [] kvs = new KeyValue[] { + KeyValueTestUtil.create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"), + }; + List scanners = Arrays.asList( + new KeyValueScanner[] { + new KeyValueScanFixture(KeyValue.COMPARATOR, kvs) + }); + Scan scanSpec = new Scan(Bytes.toBytes(r1)); + scanSpec.setTimeRange(0, 6); + scanSpec.setMaxVersions(); + StoreScanner scan = new StoreScanner(scanSpec, scanInfo, scanType, + getCols("a"), scanners); + List results = new ArrayList(); + assertEquals(true, scan.next(results)); + assertEquals(5, results.size()); + assertEquals(kvs[kvs.length - 1], results.get(0)); + // Scan limited TimeRange + scanSpec = new Scan(Bytes.toBytes(r1)); + scanSpec.setTimeRange(1, 3); + scanSpec.setMaxVersions(); + scan = new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), + scanners); + results = new ArrayList(); + assertEquals(true, scan.next(results)); + assertEquals(2, results.size()); + // Another range. + scanSpec = new Scan(Bytes.toBytes(r1)); + scanSpec.setTimeRange(5, 10); + scanSpec.setMaxVersions(); + scan = new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), + scanners); + results = new ArrayList(); + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + // See how TimeRange and Versions interact. + // Another range. + scanSpec = new Scan(Bytes.toBytes(r1)); + scanSpec.setTimeRange(0, 10); + scanSpec.setMaxVersions(3); + scan = new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), + scanners); + results = new ArrayList(); + assertEquals(true, scan.next(results)); + assertEquals(3, results.size()); + } + + public void testScanSameTimestamp() throws IOException { + // returns only 1 of these 2 even though same timestamp + KeyValue [] kvs = new KeyValue[] { + KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), + }; + List scanners = Arrays.asList( + new KeyValueScanner[] { + new KeyValueScanFixture(KeyValue.COMPARATOR, kvs) + }); + + Scan scanSpec = new Scan(Bytes.toBytes("R1")); + // this only uses maxVersions (default=1) and TimeRange (default=all) + StoreScanner scan = new StoreScanner(scanSpec, scanInfo, scanType, + getCols("a"), scanners); + + List results = new ArrayList(); + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs[0], results.get(0)); + } + + /* + * Test test shows exactly how the matcher's return codes confuses the StoreScanner + * and prevent it from doing the right thing. Seeking once, then nexting twice + * should return R1, then R2, but in this case it doesnt. + * TODO this comment makes no sense above. Appears to do the right thing. + * @throws IOException + */ + public void testWontNextToNext() throws IOException { + // build the scan file: + KeyValue [] kvs = new KeyValue[] { + KeyValueTestUtil.create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care") + }; + List scanners = scanFixture(kvs); + + Scan scanSpec = new Scan(Bytes.toBytes("R1")); + // this only uses maxVersions (default=1) and TimeRange (default=all) + StoreScanner scan = new StoreScanner(scanSpec, scanInfo, scanType, + getCols("a"), scanners); + + List results = new ArrayList(); + scan.next(results); + assertEquals(1, results.size()); + assertEquals(kvs[0], results.get(0)); + // should be ok... + // now scan _next_ again. + results.clear(); + scan.next(results); + assertEquals(1, results.size()); + assertEquals(kvs[2], results.get(0)); + + results.clear(); + scan.next(results); + assertEquals(0, results.size()); + + } + + + public void testDeleteVersionSameTimestamp() throws IOException { + KeyValue [] kvs = new KeyValue [] { + KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), + }; + List scanners = scanFixture(kvs); + Scan scanSpec = new Scan(Bytes.toBytes("R1")); + StoreScanner scan = new StoreScanner(scanSpec, scanInfo, scanType, + getCols("a"), scanners); + + List results = new ArrayList(); + assertFalse(scan.next(results)); + assertEquals(0, results.size()); + } + + /* + * Test the case where there is a delete row 'in front of' the next row, the scanner + * will move to the next row. + */ + public void testDeletedRowThenGoodRow() throws IOException { + KeyValue [] kvs = new KeyValue [] { + KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), + KeyValueTestUtil.create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care") + }; + List scanners = scanFixture(kvs); + Scan scanSpec = new Scan(Bytes.toBytes("R1")); + StoreScanner scan = new StoreScanner(scanSpec, scanInfo, scanType, + getCols("a"), scanners); + + List results = new ArrayList(); + assertEquals(true, scan.next(results)); + assertEquals(0, results.size()); + + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs[2], results.get(0)); + + assertEquals(false, scan.next(results)); + } + + public void testDeleteVersionMaskingMultiplePuts() throws IOException { + long now = System.currentTimeMillis(); + KeyValue [] kvs1 = new KeyValue[] { + KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") + }; + KeyValue [] kvs2 = new KeyValue[] { + KeyValueTestUtil.create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care") + }; + List scanners = scanFixture(kvs1, kvs2); + + StoreScanner scan = new StoreScanner(new Scan(Bytes.toBytes("R1")), + scanInfo, scanType, getCols("a"), scanners); + List results = new ArrayList(); + // the two put at ts=now will be masked by the 1 delete, and + // since the scan default returns 1 version we'll return the newest + // key, which is kvs[2], now-100. + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs2[1], results.get(0)); + } + public void testDeleteVersionsMixedAndMultipleVersionReturn() throws IOException { + long now = System.currentTimeMillis(); + KeyValue [] kvs1 = new KeyValue[] { + KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") + }; + KeyValue [] kvs2 = new KeyValue[] { + KeyValueTestUtil.create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", now+500, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care") + }; + List scanners = scanFixture(kvs1, kvs2); + + Scan scanSpec = new Scan(Bytes.toBytes("R1")).setMaxVersions(2); + StoreScanner scan = new StoreScanner(scanSpec, scanInfo, scanType, + getCols("a"), scanners); + List results = new ArrayList(); + assertEquals(true, scan.next(results)); + assertEquals(2, results.size()); + assertEquals(kvs2[1], results.get(0)); + assertEquals(kvs2[0], results.get(1)); + } + + public void testWildCardOneVersionScan() throws IOException { + KeyValue [] kvs = new KeyValue [] { + KeyValueTestUtil.create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), + }; + List scanners = scanFixture(kvs); + StoreScanner scan = new StoreScanner(new Scan(Bytes.toBytes("R1")), + scanInfo, scanType, null, scanners); + List results = new ArrayList(); + assertEquals(true, scan.next(results)); + assertEquals(2, results.size()); + assertEquals(kvs[0], results.get(0)); + assertEquals(kvs[1], results.get(1)); + } + + public void testWildCardScannerUnderDeletes() throws IOException { + KeyValue [] kvs = new KeyValue [] { + KeyValueTestUtil.create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), // inc + // orphaned delete column. + KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), + // column b + KeyValueTestUtil.create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"), // inc + KeyValueTestUtil.create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), // inc + // column c + KeyValueTestUtil.create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"), // no + KeyValueTestUtil.create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"), // inc + // column d + KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), // inc + KeyValueTestUtil.create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"), // no + KeyValueTestUtil.create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"), // no + + }; + List scanners = scanFixture(kvs); + StoreScanner scan = new StoreScanner(new Scan().setMaxVersions(2), + scanInfo, scanType, null, scanners); + List results = new ArrayList(); + assertEquals(true, scan.next(results)); + assertEquals(5, results.size()); + assertEquals(kvs[0], results.get(0)); + assertEquals(kvs[2], results.get(1)); + assertEquals(kvs[3], results.get(2)); + assertEquals(kvs[6], results.get(3)); + assertEquals(kvs[7], results.get(4)); + } + + public void testDeleteFamily() throws IOException { + KeyValue [] kvs = new KeyValue[] { + KeyValueTestUtil.create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), + }; + List scanners = scanFixture(kvs); + StoreScanner scan = new StoreScanner( + new Scan().setMaxVersions(Integer.MAX_VALUE), scanInfo, scanType, null, + scanners); + List results = new ArrayList(); + assertEquals(true, scan.next(results)); + assertEquals(0, results.size()); + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs[kvs.length-1], results.get(0)); + + assertEquals(false, scan.next(results)); + } + + public void testDeleteColumn() throws IOException { + KeyValue [] kvs = new KeyValue[] { + KeyValueTestUtil.create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care") + }; + List scanners = scanFixture(kvs); + StoreScanner scan = new StoreScanner(new Scan(), scanInfo, scanType, null, + scanners); + List results = new ArrayList(); + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs[3], results.get(0)); + } + + private static final KeyValue [] kvs = new KeyValue[] { + KeyValueTestUtil.create("R1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), + }; + + public void testSkipColumn() throws IOException { + List scanners = scanFixture(kvs); + StoreScanner scan = new StoreScanner(new Scan(), scanInfo, scanType, + getCols("a", "d"), scanners); + + List results = new ArrayList(); + assertEquals(true, scan.next(results)); + assertEquals(2, results.size()); + assertEquals(kvs[0], results.get(0)); + assertEquals(kvs[3], results.get(1)); + results.clear(); + + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs[kvs.length-1], results.get(0)); + + results.clear(); + assertEquals(false, scan.next(results)); + } + + /* + * Test expiration of KeyValues in combination with a configured TTL for + * a column family (as should be triggered in a major compaction). + */ + public void testWildCardTtlScan() throws IOException { + long now = System.currentTimeMillis(); + KeyValue [] kvs = new KeyValue[] { + KeyValueTestUtil.create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care") + }; + List scanners = scanFixture(kvs); + Scan scan = new Scan(); + scan.setMaxVersions(1); + ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, false, 0, + KeyValue.COMPARATOR); + ScanType scanType = ScanType.USER_SCAN; + StoreScanner scanner = + new StoreScanner(scan, scanInfo, scanType, + null, scanners); + + List results = new ArrayList(); + assertEquals(true, scanner.next(results)); + assertEquals(2, results.size()); + assertEquals(kvs[1], results.get(0)); + assertEquals(kvs[2], results.get(1)); + results.clear(); + + assertEquals(true, scanner.next(results)); + assertEquals(3, results.size()); + assertEquals(kvs[4], results.get(0)); + assertEquals(kvs[5], results.get(1)); + assertEquals(kvs[6], results.get(2)); + results.clear(); + + assertEquals(false, scanner.next(results)); + } + + public void testScannerReseekDoesntNPE() throws Exception { + List scanners = scanFixture(kvs); + StoreScanner scan = new StoreScanner(new Scan(), scanInfo, scanType, + getCols("a", "d"), scanners); + + // Previously a updateReaders twice in a row would cause an NPE. In test this would also + // normally cause an NPE because scan.store is null. So as long as we get through these + // two calls we are good and the bug was quashed. + + scan.updateReaders(); + + scan.updateReaders(); + + scan.peek(); + } + + + /** + * TODO this fails, since we don't handle deletions, etc, in peek + */ + public void SKIP_testPeek() throws Exception { + KeyValue [] kvs = new KeyValue [] { + KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), + KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), + }; + List scanners = scanFixture(kvs); + Scan scanSpec = new Scan(Bytes.toBytes("R1")); + StoreScanner scan = new StoreScanner(scanSpec, scanInfo, scanType, + getCols("a"), scanners); + assertNull(scan.peek()); + } + + /** + * Ensure that expired delete family markers don't override valid puts + */ + public void testExpiredDeleteFamily() throws Exception { + long now = System.currentTimeMillis(); + KeyValue [] kvs = new KeyValue[] { + new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000, + KeyValue.Type.DeleteFamily), + KeyValueTestUtil.create("R1", "cf", "a", now-10, KeyValue.Type.Put, + "dont-care"), + }; + List scanners = scanFixture(kvs); + Scan scan = new Scan(); + scan.setMaxVersions(1); + // scanner with ttl equal to 500 + ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, false, 0, + KeyValue.COMPARATOR); + ScanType scanType = ScanType.USER_SCAN; + StoreScanner scanner = + new StoreScanner(scan, scanInfo, scanType, null, scanners); + + List results = new ArrayList(); + assertEquals(true, scanner.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs[1], results.get(0)); + results.clear(); + + assertEquals(false, scanner.next(results)); + } + + public void testDeleteMarkerLongevity() throws Exception { + try { + final long now = System.currentTimeMillis(); + EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() { + public long currentTimeMillis() { + return now; + } + }); + KeyValue[] kvs = new KeyValue[]{ + /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, + now - 100, KeyValue.Type.DeleteFamily), // live + /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, + now - 1000, KeyValue.Type.DeleteFamily), // expired + /*2*/ KeyValueTestUtil.create("R1", "cf", "a", now - 50, + KeyValue.Type.Put, "v3"), // live + /*3*/ KeyValueTestUtil.create("R1", "cf", "a", now - 55, + KeyValue.Type.Delete, "dontcare"), // live + /*4*/ KeyValueTestUtil.create("R1", "cf", "a", now - 55, + KeyValue.Type.Put, "deleted-version v2"), // deleted + /*5*/ KeyValueTestUtil.create("R1", "cf", "a", now - 60, + KeyValue.Type.Put, "v1"), // live + /*6*/ KeyValueTestUtil.create("R1", "cf", "a", now - 65, + KeyValue.Type.Put, "v0"), // max-version reached + /*7*/ KeyValueTestUtil.create("R1", "cf", "a", + now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version + /*8*/ KeyValueTestUtil.create("R1", "cf", "b", now - 600, + KeyValue.Type.DeleteColumn, "dont-care"), //expired + /*9*/ KeyValueTestUtil.create("R1", "cf", "b", now - 70, + KeyValue.Type.Put, "v2"), //live + /*10*/ KeyValueTestUtil.create("R1", "cf", "b", now - 750, + KeyValue.Type.Put, "v1"), //expired + /*11*/ KeyValueTestUtil.create("R1", "cf", "c", now - 500, + KeyValue.Type.Delete, "dontcare"), //expired + /*12*/ KeyValueTestUtil.create("R1", "cf", "c", now - 600, + KeyValue.Type.Put, "v1"), //expired + /*13*/ KeyValueTestUtil.create("R1", "cf", "c", now - 1000, + KeyValue.Type.Delete, "dontcare"), //expired + /*14*/ KeyValueTestUtil.create("R1", "cf", "d", now - 60, + KeyValue.Type.Put, "expired put"), //live + /*15*/ KeyValueTestUtil.create("R1", "cf", "d", now - 100, + KeyValue.Type.Delete, "not-expired delete"), //live + }; + List scanners = scanFixture(kvs); + Scan scan = new Scan(); + scan.setMaxVersions(2); + Store.ScanInfo scanInfo = new Store.ScanInfo(Bytes.toBytes("cf"), + 0 /* minVersions */, + 2 /* maxVersions */, 500 /* ttl */, + false /* keepDeletedCells */, + 200, /* timeToPurgeDeletes */ + KeyValue.COMPARATOR); + StoreScanner scanner = + new StoreScanner(scan, scanInfo, + ScanType.MAJOR_COMPACT, null, scanners, + HConstants.OLDEST_TIMESTAMP); + List results = new ArrayList(); + results = new ArrayList(); + assertEquals(true, scanner.next(results)); + assertEquals(kvs[0], results.get(0)); + assertEquals(kvs[2], results.get(1)); + assertEquals(kvs[3], results.get(2)); + assertEquals(kvs[5], results.get(3)); + assertEquals(kvs[9], results.get(4)); + assertEquals(kvs[14], results.get(5)); + assertEquals(kvs[15], results.get(6)); + assertEquals(7, results.size()); + }finally{ + EnvironmentEdgeManagerTestHelper.reset(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java new file mode 100644 index 0000000..09d3151 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java @@ -0,0 +1,143 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestWideScanner extends HBaseTestCase { + private final Log LOG = LogFactory.getLog(this.getClass()); + + static final byte[] A = Bytes.toBytes("A"); + static final byte[] B = Bytes.toBytes("B"); + static final byte[] C = Bytes.toBytes("C"); + static byte[][] COLUMNS = { A, B, C }; + static final Random rng = new Random(); + static final HTableDescriptor TESTTABLEDESC = + new HTableDescriptor("testwidescan"); + static { + for (byte[] cfName : new byte[][] { A, B, C }) { + TESTTABLEDESC.addFamily(new HColumnDescriptor(cfName) + // Keep versions to help debugging. + .setMaxVersions(100) + .setBlocksize(8 * 1024) + ); + } + } + + /** HRegionInfo for root region */ + HRegion r; + + private int addWideContent(HRegion region) throws IOException { + int count = 0; + for (char c = 'a'; c <= 'c'; c++) { + byte[] row = Bytes.toBytes("ab" + c); + int i, j; + long ts = System.currentTimeMillis(); + for (i = 0; i < 100; i++) { + byte[] b = Bytes.toBytes(String.format("%10d", i)); + for (j = 0; j < 100; j++) { + Put put = new Put(row); + put.setWriteToWAL(false); + put.add(COLUMNS[rng.nextInt(COLUMNS.length)], b, ++ts, b); + region.put(put); + count++; + } + } + } + return count; + } + + public void testWideScanBatching() throws IOException { + final int batch = 256; + try { + this.r = createNewHRegion(TESTTABLEDESC, null, null); + int inserted = addWideContent(this.r); + List results = new ArrayList(); + Scan scan = new Scan(); + scan.addFamily(A); + scan.addFamily(B); + scan.addFamily(C); + scan.setMaxVersions(100); + scan.setBatch(batch); + InternalScanner s = r.getScanner(scan); + int total = 0; + int i = 0; + boolean more; + do { + more = s.next(results); + i++; + LOG.info("iteration #" + i + ", results.size=" + results.size()); + + // assert that the result set is no larger + assertTrue(results.size() <= batch); + + total += results.size(); + + if (results.size() > 0) { + // assert that all results are from the same row + byte[] row = results.get(0).getRow(); + for (KeyValue kv: results) { + assertTrue(Bytes.equals(row, kv.getRow())); + } + } + + results.clear(); + + // trigger ChangedReadersObservers + Iterator scanners = + ((HRegion.RegionScannerImpl)s).storeHeap.getHeap().iterator(); + while (scanners.hasNext()) { + StoreScanner ss = (StoreScanner)scanners.next(); + ss.updateReaders(); + } + } while (more); + + // assert that the scanner returned all values + LOG.info("inserted " + inserted + ", scanned " + total); + assertEquals(total, inserted); + + s.close(); + } finally { + this.r.close(); + this.r.getLog().closeAndDelete(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java new file mode 100644 index 0000000..10ccbac --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java @@ -0,0 +1,224 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.handler; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.executor.RegionTransitionData; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.MockRegionServerServices; +import org.apache.hadoop.hbase.util.MockServer; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.NodeExistsException; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test of the {@link CloseRegionHandler}. + */ +@Category(MediumTests.class) +public class TestCloseRegionHandler { + static final Log LOG = LogFactory.getLog(TestCloseRegionHandler.class); + private final static HBaseTestingUtility HTU = new HBaseTestingUtility(); + private static final HTableDescriptor TEST_HTD = + new HTableDescriptor("TestCloseRegionHandler"); + private HRegionInfo TEST_HRI; + private int testIndex = 0; + + @BeforeClass public static void before() throws Exception { + HTU.startMiniZKCluster(); + } + + @AfterClass public static void after() throws IOException { + HTU.shutdownMiniZKCluster(); + } + + /** + * Before each test, use a different HRI, so the different tests + * don't interfere with each other. This allows us to use just + * a single ZK cluster for the whole suite. + */ + @Before + public void setupHRI() { + TEST_HRI = new HRegionInfo(TEST_HTD.getName(), + Bytes.toBytes(testIndex), + Bytes.toBytes(testIndex + 1)); + testIndex++; + } + + /** + * Test that if we fail a flush, abort gets set on close. + * @see HBASE-4270 + * @throws IOException + * @throws NodeExistsException + * @throws KeeperException + */ + @Test public void testFailedFlushAborts() + throws IOException, NodeExistsException, KeeperException { + final Server server = new MockServer(HTU, false); + final RegionServerServices rss = new MockRegionServerServices(); + HTableDescriptor htd = TEST_HTD; + final HRegionInfo hri = + new HRegionInfo(htd.getName(), HConstants.EMPTY_END_ROW, + HConstants.EMPTY_END_ROW); + HRegion region = + HRegion.createHRegion(hri, HTU.getDataTestDir(), + HTU.getConfiguration(), htd); + try { + assertNotNull(region); + // Spy on the region so can throw exception when close is called. + HRegion spy = Mockito.spy(region); + final boolean abort = false; + Mockito.when(spy.close(abort)). + thenThrow(new RuntimeException("Mocked failed close!")); + // The CloseRegionHandler will try to get an HRegion that corresponds + // to the passed hri -- so insert the region into the online region Set. + rss.addToOnlineRegions(spy); + // Assert the Server is NOT stopped before we call close region. + assertFalse(server.isStopped()); + CloseRegionHandler handler = + new CloseRegionHandler(server, rss, hri, false, false, -1); + boolean throwable = false; + try { + handler.process(); + } catch (Throwable t) { + throwable = true; + } finally { + assertTrue(throwable); + // Abort calls stop so stopped flag should be set. + assertTrue(server.isStopped()); + } + } finally { + HRegion.closeHRegion(region); + } + } + + /** + * Test if close region can handle ZK closing node version mismatch + * @throws IOException + * @throws NodeExistsException + * @throws KeeperException + */ + @Test public void testZKClosingNodeVersionMismatch() + throws IOException, NodeExistsException, KeeperException { + final Server server = new MockServer(HTU); + final MockRegionServerServices rss = new MockRegionServerServices(); + rss.setFileSystem(HTU.getTestFileSystem()); + + HTableDescriptor htd = TEST_HTD; + final HRegionInfo hri = TEST_HRI; + + // open a region first so that it can be closed later + OpenRegion(server, rss, htd, hri); + + // close the region + // Create it CLOSING, which is what Master set before sending CLOSE RPC + int versionOfClosingNode = ZKAssign.createNodeClosing(server.getZooKeeper(), + hri, server.getServerName()); + + // The CloseRegionHandler will validate the expected version + // Given it is set to invalid versionOfClosingNode+1, + // CloseRegionHandler should be M_ZK_REGION_CLOSING + CloseRegionHandler handler = + new CloseRegionHandler(server, rss, hri, false, true, + versionOfClosingNode+1); + handler.process(); + + // Handler should remain in M_ZK_REGION_CLOSING + RegionTransitionData data = + ZKAssign.getData(server.getZooKeeper(), hri.getEncodedName()); + assertTrue(EventType.M_ZK_REGION_CLOSING == data.getEventType()); + } + + /** + * Test if the region can be closed properly + * @throws IOException + * @throws NodeExistsException + * @throws KeeperException + */ + @Test public void testCloseRegion() + throws IOException, NodeExistsException, KeeperException { + final Server server = new MockServer(HTU); + final MockRegionServerServices rss = new MockRegionServerServices(); + rss.setFileSystem(HTU.getTestFileSystem()); + + HTableDescriptor htd = TEST_HTD; + HRegionInfo hri = TEST_HRI; + + // open a region first so that it can be closed later + OpenRegion(server, rss, htd, hri); + + // close the region + // Create it CLOSING, which is what Master set before sending CLOSE RPC + int versionOfClosingNode = ZKAssign.createNodeClosing(server.getZooKeeper(), + hri, server.getServerName()); + + // The CloseRegionHandler will validate the expected version + // Given it is set to correct versionOfClosingNode, + // CloseRegionHandlerit should be RS_ZK_REGION_CLOSED + CloseRegionHandler handler = + new CloseRegionHandler(server, rss, hri, false, true, + versionOfClosingNode); + handler.process(); + // Handler should have transitioned it to RS_ZK_REGION_CLOSED + RegionTransitionData data = + ZKAssign.getData(server.getZooKeeper(), hri.getEncodedName()); + assertTrue(EventType.RS_ZK_REGION_CLOSED == data.getEventType()); + } + private void OpenRegion(Server server, RegionServerServices rss, + HTableDescriptor htd, HRegionInfo hri) + throws IOException, NodeExistsException, KeeperException { + // Create it OFFLINE node, which is what Master set before sending OPEN RPC + + + ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName()); + int version = ZKAssign.transitionNodeOpening(server.getZooKeeper(), hri, server.getServerName()); + OpenRegionHandler openHandler = new OpenRegionHandler(server, rss, hri, htd, version); + openHandler.process(); + RegionTransitionData data = ZKAssign.getData(server.getZooKeeper(), hri.getEncodedName()); + + // delete the node, which is what Master do after the region is opened + ZKAssign.deleteNode(server.getZooKeeper(), hri.getEncodedName(), + EventType.RS_ZK_REGION_OPENED); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java new file mode 100644 index 0000000..a2b3fef --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java @@ -0,0 +1,247 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.handler; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.executor.RegionTransitionData; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.MockRegionServerServices; +import org.apache.hadoop.hbase.util.MockServer; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.NodeExistsException; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test of the {@link OpenRegionHandler}. + */ +@Category(MediumTests.class) +public class TestOpenRegionHandler { + static final Log LOG = LogFactory.getLog(TestOpenRegionHandler.class); + private final static HBaseTestingUtility HTU = new HBaseTestingUtility(); + private static HTableDescriptor TEST_HTD; + private HRegionInfo TEST_HRI; + + private int testIndex = 0; + + @BeforeClass public static void before() throws Exception { + Configuration c = HTU.getConfiguration(); + c.setClass(HConstants.REGION_SERVER_IMPL, TestOpenRegionHandlerRegionServer.class, + HRegionServer.class); + HTU.startMiniCluster(); + TEST_HTD = new HTableDescriptor("TestOpenRegionHandler.java"); + } + + @AfterClass public static void after() throws IOException { + TEST_HTD = null; + try { + HTU.shutdownMiniCluster(); + } catch (Exception e) { + throw new IOException(e); + } + } + + /** + * Before each test, use a different HRI, so the different tests + * don't interfere with each other. This allows us to use just + * a single ZK cluster for the whole suite. + */ + @Before + public void setupHRI() { + TEST_HRI = new HRegionInfo(TEST_HTD.getName(), + Bytes.toBytes(testIndex), + Bytes.toBytes(testIndex + 1)); + testIndex++; + } + + /** + * Test the openregionhandler can deal with its znode being yanked out from + * under it. + * @see HBASE-3627 + * @throws IOException + * @throws NodeExistsException + * @throws KeeperException + */ + @Test public void testYankingRegionFromUnderIt() + throws IOException, NodeExistsException, KeeperException { + final Server server = new MockServer(HTU); + final RegionServerServices rss = new MockRegionServerServices(); + + HTableDescriptor htd = TEST_HTD; + final HRegionInfo hri = TEST_HRI; + HRegion region = + HRegion.createHRegion(hri, HTU.getDataTestDir(), HTU + .getConfiguration(), htd); + assertNotNull(region); + try { + OpenRegionHandler handler = new OpenRegionHandler(server, rss, hri, htd) { + HRegion openRegion() { + // Open region first, then remove znode as though it'd been hijacked. + HRegion region = super.openRegion(); + + // Don't actually open region BUT remove the znode as though it'd + // been hijacked on us. + ZooKeeperWatcher zkw = this.server.getZooKeeper(); + String node = ZKAssign.getNodeName(zkw, hri.getEncodedName()); + try { + ZKUtil.deleteNodeFailSilent(zkw, node); + } catch (KeeperException e) { + throw new RuntimeException("Ugh failed delete of " + node, e); + } + return region; + } + }; + // Call process without first creating OFFLINE region in zk, see if + // exception or just quiet return (expected). + handler.process(); + ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName()); + // Call process again but this time yank the zk znode out from under it + // post OPENING; again will expect it to come back w/o NPE or exception. + handler.process(); + } finally { + HRegion.closeHRegion(region); + } + } + + @Test + public void testFailedOpenRegion() throws Exception { + Server server = new MockServer(HTU); + RegionServerServices rsServices = new MockRegionServerServices(); + + // Create it OFFLINE, which is what it expects + ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName()); + ZKAssign.transitionNodeOpening(server.getZooKeeper(), TEST_HRI, server.getServerName()); + + // Create the handler + OpenRegionHandler handler = + new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD) { + @Override + HRegion openRegion() { + // Fake failure of opening a region due to an IOE, which is caught + return null; + } + }; + handler.process(); + + // Handler should have transitioned it to FAILED_OPEN + RegionTransitionData data = + ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName()); + assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, data.getEventType()); + } + + @Test + public void testFailedUpdateMeta() throws Exception { + Server server = new MockServer(HTU); + RegionServerServices rsServices = new MockRegionServerServices(); + + // Create it OFFLINE, which is what it expects + ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName()); + ZKAssign.transitionNodeOpening(server.getZooKeeper(), TEST_HRI, server.getServerName()); + // Create the handler + OpenRegionHandler handler = + new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD) { + @Override + boolean updateMeta(final HRegion r) { + // Fake failure of updating META + return false; + } + }; + handler.process(); + + // Handler should have transitioned it to FAILED_OPEN + RegionTransitionData data = + ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName()); + assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, data.getEventType()); + } + + public static class TestOpenRegionHandlerRegionServer extends HRegionServer { + public TestOpenRegionHandlerRegionServer(Configuration conf) + throws IOException, InterruptedException { + super(conf); + } + @Override + public boolean addRegionsInTransition(HRegionInfo region, + String currentAction) throws RegionAlreadyInTransitionException { + return super.addRegionsInTransition(region, currentAction); + } + } + + @Test + public void testTransitionToFailedOpenEvenIfCleanupFails() throws Exception { + MiniHBaseCluster cluster = HTU.getHBaseCluster(); + HRegionServer server = + cluster.getLiveRegionServerThreads().get(0).getRegionServer(); + // Create it OFFLINE, which is what it expects + ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName()); + ZKAssign.transitionNodeOpening(server.getZooKeeper(), TEST_HRI, server.getServerName()); + // Create the handler + OpenRegionHandler handler = new OpenRegionHandler(server, server, TEST_HRI, TEST_HTD) { + @Override + boolean updateMeta(HRegion r) { + return false; + }; + + @Override + void cleanupFailedOpen(HRegion region) throws IOException { + throw new IOException("FileSystem got closed."); + } + }; + ((TestOpenRegionHandlerRegionServer)server).addRegionsInTransition(TEST_HRI, "OPEN"); + try { + handler.process(); + } catch (Exception e) { + // Ignore the IOException that we have thrown from cleanupFailedOpen + } + RegionTransitionData data = + ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName()); + assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, data.getEventType()); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaConfigured.java b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaConfigured.java new file mode 100644 index 0000000..2a77d20 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaConfigured.java @@ -0,0 +1,246 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import static org.junit.Assert.*; + +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONStringer; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestSchemaConfigured { + private static final Log LOG = LogFactory.getLog(TestSchemaConfigured.class); + private final String TABLE_NAME = "myTable"; + private final String CF_NAME = "myColumnFamily"; + + private static final Path TMP_HFILE_PATH = new Path( + "/hbase/myTable/myRegion/" + HRegion.REGION_TEMP_SUBDIR + "/hfilename"); + + /** Test if toString generates real JSON */ + @Test + public void testToString() throws JSONException { + SchemaConfigured sc = new SchemaConfigured(null, TABLE_NAME, CF_NAME); + JSONStringer json = new JSONStringer(); + json.object(); + json.key("tableName"); + json.value(TABLE_NAME); + json.key("cfName"); + json.value(CF_NAME); + json.endObject(); + assertEquals(json.toString(), sc.schemaConfAsJSON()); + } + + /** Don't allow requesting metrics before setting table/CF name */ + @Test + public void testDelayedInitialization() { + SchemaConfigured unconfigured = new SchemaConfigured(); + try { + unconfigured.getSchemaMetrics(); + fail(IllegalStateException.class.getSimpleName() + " expected"); + } catch (IllegalStateException ex) { + assertTrue("Unexpected exception message: " + ex.getMessage(), + Pattern.matches(".* metrics requested before .* initialization.*", + ex.getMessage())); + LOG.debug("Expected exception: " + ex.getMessage()); + } + + SchemaMetrics.setUseTableNameInTest(false); + SchemaConfigured other = new SchemaConfigured(null, TABLE_NAME, CF_NAME); + other.passSchemaMetricsTo(unconfigured); + unconfigured.getSchemaMetrics(); // now this should succeed + } + + /** Don't allow setting table/CF name twice */ + @Test + public void testInitializingTwice() { + Configuration conf = HBaseConfiguration.create(); + for (int i = 0; i < 4; ++i) { + SchemaConfigured sc = new SchemaConfigured(conf, TABLE_NAME, CF_NAME); + SchemaConfigured target = + new SchemaConfigured(conf, TABLE_NAME + (i % 2 == 1 ? "1" : ""), + CF_NAME + ((i & 2) != 0 ? "1" : "")); + if (i == 0) { + sc.passSchemaMetricsTo(target); // No exception expected. + continue; + } + + String testDesc = + "Trying to re-configure " + target.schemaConfAsJSON() + " with " + + sc.schemaConfAsJSON(); + try { + sc.passSchemaMetricsTo(target); + fail(IllegalArgumentException.class.getSimpleName() + " expected"); + } catch (IllegalArgumentException ex) { + final String errorMsg = testDesc + ". Unexpected exception message: " + + ex.getMessage(); + final String exceptionRegex = "Trying to change table .* CF .*"; + assertTrue(errorMsg, Pattern.matches(exceptionRegex, ex.getMessage())); + LOG.debug("Expected exception: " + ex.getMessage()); + } + } + } + + @Test(expected=IllegalStateException.class) + public void testConfigureWithUnconfigured1() { + SchemaConfigured unconfigured = new SchemaConfigured(null, "t1", null); + SchemaConfigured target = new SchemaConfigured(); + unconfigured.passSchemaMetricsTo(target); + } + + @Test(expected=IllegalStateException.class) + public void testConfigureWithUnconfigured2() { + SchemaConfigured unconfigured = new SchemaConfigured(null, null, "cf1"); + SchemaConfigured target = new SchemaConfigured(); + unconfigured.passSchemaMetricsTo(target); + } + + /** + * Configuring with an uninitialized object is equivalent to re-setting + * schema metrics configuration. + */ + public void testConfigureWithNull() { + SchemaConfigured unconfigured = new SchemaConfigured(); + SchemaConfigured target = new SchemaConfigured(null, "t1", "cf1"); + unconfigured.passSchemaMetricsTo(target); + assertTrue(target.getTableName() == null); + assertTrue(target.getColumnFamilyName() == null); + } + + public void testConfigurePartiallyDefined() { + final SchemaConfigured sc = new SchemaConfigured(null, "t1", "cf1"); + final SchemaConfigured target1 = new SchemaConfigured(null, "t2", null); + sc.passSchemaMetricsTo(target1); + assertEquals("t2", target1.getColumnFamilyName()); + assertEquals("cf1", target1.getColumnFamilyName()); + + final SchemaConfigured target2 = new SchemaConfigured(null, null, "cf2"); + sc.passSchemaMetricsTo(target2); + assertEquals("t1", target2.getColumnFamilyName()); + assertEquals("cf2", target2.getColumnFamilyName()); + + final SchemaConfigured target3 = new SchemaConfigured(null, null, null); + sc.passSchemaMetricsTo(target3); + assertEquals("t1", target2.getColumnFamilyName()); + assertEquals("cf1", target2.getColumnFamilyName()); + } + + @Test(expected=IllegalArgumentException.class) + public void testConflictingConf() { + SchemaConfigured sc = new SchemaConfigured(null, "t1", "cf1"); + SchemaConfigured target = new SchemaConfigured(null, "t2", "cf1"); + sc.passSchemaMetricsTo(target); + } + + /** We allow setting CF to unknown and then reconfiguring it */ + public void testReconfigureUnknownCF() { + SchemaConfigured sc = new SchemaConfigured(null, "t1", "cf1"); + SchemaConfigured target = + new SchemaConfigured(null, "t1", SchemaMetrics.UNKNOWN); + sc.passSchemaMetricsTo(target); + } + + /** + * When the "column family" deduced from the path is ".tmp" (this happens + * for files written on compaction) we allow re-setting the CF to another + * value. + */ + @Test + public void testTmpPath() { + SchemaConfigured sc = new SchemaConfigured(null, "myTable", "myCF"); + SchemaConfigured target = new SchemaConfigured(TMP_HFILE_PATH); + sc.passSchemaMetricsTo(target); + } + + /** + * Even if CF is initially undefined (".tmp"), we don't allow to change + * table name. + */ + @Test(expected=IllegalArgumentException.class) + public void testTmpPathButInvalidTable() { + SchemaConfigured sc = new SchemaConfigured(null, "anotherTable", "myCF"); + SchemaConfigured target = new SchemaConfigured(TMP_HFILE_PATH); + sc.passSchemaMetricsTo(target); + } + + @Test + public void testSchemaConfigurationHook() { + SchemaConfigured sc = new SchemaConfigured(null, "myTable", "myCF"); + final StringBuilder newCF = new StringBuilder(); + final StringBuilder newTable = new StringBuilder(); + SchemaConfigured target = new SchemaConfigured() { + @Override + protected void schemaConfigurationChanged() { + newCF.append(getColumnFamilyName()); + newTable.append(getTableName()); + } + }; + sc.passSchemaMetricsTo(target); + assertEquals("myTable", newTable.toString()); + assertEquals("myCF", newCF.toString()); + } + + @Test + public void testResetSchemaMetricsConf() { + SchemaConfigured target = new SchemaConfigured(null, "t1", "cf1"); + SchemaConfigured.resetSchemaMetricsConf(target); + new SchemaConfigured(null, "t2", "cf2").passSchemaMetricsTo(target); + assertEquals("t2", target.getTableName()); + assertEquals("cf2", target.getColumnFamilyName()); + } + + @Test + public void testPathTooShort() { + // This has too few path components (four, the first one is empty). + SchemaConfigured sc1 = new SchemaConfigured(new Path("/a/b/c/d")); + assertEquals(SchemaMetrics.UNKNOWN, sc1.getTableName()); + assertEquals(SchemaMetrics.UNKNOWN, sc1.getColumnFamilyName()); + + SchemaConfigured sc2 = new SchemaConfigured(new Path("a/b/c/d")); + assertEquals(SchemaMetrics.UNKNOWN, sc2.getTableName()); + assertEquals(SchemaMetrics.UNKNOWN, sc2.getColumnFamilyName()); + + SchemaConfigured sc3 = new SchemaConfigured( + new Path("/hbase/tableName/regionId/cfName/hfileName")); + assertEquals("tableName", sc3.getTableName()); + assertEquals("cfName", sc3.getColumnFamilyName()); + + SchemaConfigured sc4 = new SchemaConfigured( + new Path("hbase/tableName/regionId/cfName/hfileName")); + assertEquals("tableName", sc4.getTableName()); + assertEquals("cfName", sc4.getColumnFamilyName()); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java new file mode 100644 index 0000000..dbe425f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java @@ -0,0 +1,252 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import static org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics. + BOOL_VALUES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.io.hfile.BlockType; +import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics. + BlockMetricType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@Category(MediumTests.class) +@RunWith(Parameterized.class) +public class TestSchemaMetrics { + + private final String TABLE_NAME = "myTable"; + private final String CF_NAME = "myColumnFamily"; + + private final boolean useTableName; + private Map startingMetrics; + + @Parameters + public static Collection parameters() { + return HBaseTestingUtility.BOOLEAN_PARAMETERIZED; + } + + public TestSchemaMetrics(boolean useTableName) { + this.useTableName = useTableName; + SchemaMetrics.setUseTableNameInTest(useTableName); + } + + @Before + public void setUp() { + startingMetrics = SchemaMetrics.getMetricsSnapshot(); + }; + + @Test + public void testNaming() { + final String metricPrefix = (useTableName ? "tbl." + + TABLE_NAME + "." : "") + "cf." + CF_NAME + "."; + SchemaMetrics schemaMetrics = SchemaMetrics.getInstance(TABLE_NAME, + CF_NAME); + SchemaMetrics ALL_CF_METRICS = SchemaMetrics.ALL_SCHEMA_METRICS; + + // fsReadTimeMetric + assertEquals(metricPrefix + "fsRead", schemaMetrics.getBlockMetricName( + BlockCategory.ALL_CATEGORIES, false, BlockMetricType.READ_TIME)); + + // compactionReadTimeMetric + assertEquals(metricPrefix + "compactionRead", + schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, true, + BlockMetricType.READ_TIME)); + + // fsBlockReadCntMetric + assertEquals(metricPrefix + "fsBlockReadCnt", + schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, false, + BlockMetricType.READ_COUNT)); + + // fsBlockReadCacheHitCntMetric + assertEquals(metricPrefix + "fsBlockReadCacheHitCnt", + schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, false, + BlockMetricType.CACHE_HIT)); + + // fsBlockReadCacheMissCntMetric + assertEquals(metricPrefix + "fsBlockReadCacheMissCnt", + schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, false, + BlockMetricType.CACHE_MISS)); + + // compactionBlockReadCntMetric + assertEquals(metricPrefix + "compactionBlockReadCnt", + schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, true, + BlockMetricType.READ_COUNT)); + + // compactionBlockReadCacheHitCntMetric + assertEquals(metricPrefix + "compactionBlockReadCacheHitCnt", + schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, true, + BlockMetricType.CACHE_HIT)); + + // compactionBlockReadCacheMissCntMetric + assertEquals(metricPrefix + "compactionBlockReadCacheMissCnt", + schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, true, + BlockMetricType.CACHE_MISS)); + + // fsMetaBlockReadCntMetric + assertEquals("fsMetaBlockReadCnt", ALL_CF_METRICS.getBlockMetricName( + BlockCategory.META, false, BlockMetricType.READ_COUNT)); + + // fsMetaBlockReadCacheHitCntMetric + assertEquals("fsMetaBlockReadCacheHitCnt", + ALL_CF_METRICS.getBlockMetricName(BlockCategory.META, false, + BlockMetricType.CACHE_HIT)); + + // fsMetaBlockReadCacheMissCntMetric + assertEquals("fsMetaBlockReadCacheMissCnt", + ALL_CF_METRICS.getBlockMetricName(BlockCategory.META, false, + BlockMetricType.CACHE_MISS)); + + // Per-(column family, block type) statistics. + assertEquals(metricPrefix + "bt.Index.fsBlockReadCnt", + schemaMetrics.getBlockMetricName(BlockCategory.INDEX, false, + BlockMetricType.READ_COUNT)); + + assertEquals(metricPrefix + "bt.Data.compactionBlockReadCacheHitCnt", + schemaMetrics.getBlockMetricName(BlockCategory.DATA, true, + BlockMetricType.CACHE_HIT)); + + // A special case for Meta blocks + assertEquals(metricPrefix + "compactionMetaBlockReadCacheHitCnt", + schemaMetrics.getBlockMetricName(BlockCategory.META, true, + BlockMetricType.CACHE_HIT)); + + // Cache metrics + assertEquals(metricPrefix + "blockCacheSize", + schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, false, + BlockMetricType.CACHE_SIZE)); + + assertEquals(metricPrefix + "bt.Index.blockCacheNumEvicted", + schemaMetrics.getBlockMetricName(BlockCategory.INDEX, false, + BlockMetricType.EVICTED)); + + assertEquals("bt.Data.blockCacheNumCached", + ALL_CF_METRICS.getBlockMetricName(BlockCategory.DATA, false, + BlockMetricType.CACHED)); + + assertEquals("blockCacheNumCached", ALL_CF_METRICS.getBlockMetricName( + BlockCategory.ALL_CATEGORIES, false, BlockMetricType.CACHED)); + + // "Non-compaction aware" metrics + try { + ALL_CF_METRICS.getBlockMetricName(BlockCategory.ALL_CATEGORIES, true, + BlockMetricType.CACHE_SIZE); + fail("Exception expected"); + } catch (IllegalArgumentException ex) { + } + + // Bloom metrics + assertEquals("keyMaybeInBloomCnt", ALL_CF_METRICS.getBloomMetricName(true)); + assertEquals(metricPrefix + "keyNotInBloomCnt", + schemaMetrics.getBloomMetricName(false)); + + schemaMetrics.printMetricNames(); + } + + public void checkMetrics() { + SchemaMetrics.validateMetricChanges(startingMetrics); + } + + @Test + public void testIncrements() { + Random rand = new Random(23982737L); + for (int i = 1; i <= 3; ++i) { + final String tableName = "table" + i; + for (int j = 1; j <= 3; ++j) { + final String cfName = "cf" + j; + SchemaMetrics sm = SchemaMetrics.getInstance(tableName, cfName); + for (boolean isInBloom : BOOL_VALUES) { + sm.updateBloomMetrics(isInBloom); + checkMetrics(); + } + + for (BlockCategory blockCat : BlockType.BlockCategory.values()) { + if (blockCat == BlockCategory.ALL_CATEGORIES) { + continue; + } + + for (boolean isCompaction : BOOL_VALUES) { + sm.updateOnCacheHit(blockCat, isCompaction); + checkMetrics(); + sm.updateOnCacheMiss(blockCat, isCompaction, rand.nextInt()); + checkMetrics(); + } + + for (boolean isEviction : BOOL_VALUES) { + sm.updateOnCachePutOrEvict(blockCat, (isEviction ? -1 : 1) + * rand.nextInt(1024 * 1024), isEviction); + } + } + } + } + } + + @Test + public void testGenerateSchemaMetricsPrefix() { + String tableName = "table1"; + int numCF = 3; + + StringBuilder expected = new StringBuilder(); + if (useTableName) { + expected.append("tbl."); + expected.append(tableName); + expected.append("."); + } + expected.append("cf."); + Set families = new HashSet(); + for (int i = 1; i <= numCF; i++) { + String cf = "cf" + i; + families.add(Bytes.toBytes(cf)); + expected.append(cf); + if (i == numCF) { + expected.append("."); + } else { + expected.append("~"); + } + } + + String result = SchemaMetrics.generateSchemaMetricsPrefix(tableName, + families); + assertEquals(expected.toString(), result); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java new file mode 100644 index 0000000..45c4727 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java @@ -0,0 +1,82 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.Queue; + +import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry; + +public class FaultySequenceFileLogReader extends SequenceFileLogReader { + + enum FailureType { + BEGINNING, MIDDLE, END, NONE + } + + Queue nextQueue = new LinkedList(); + int numberOfFileEntries = 0; + + FailureType getFailureType() { + return FailureType.valueOf(conf.get("faultysequencefilelogreader.failuretype", "NONE")); + } + + @Override + public HLog.Entry next(HLog.Entry reuse) throws IOException { + this.entryStart = this.reader.getPosition(); + boolean b = true; + + if (nextQueue.isEmpty()) { // Read the whole thing at once and fake reading + while (b == true) { + HLogKey key = HLog.newKey(conf); + WALEdit val = new WALEdit(); + HLog.Entry e = new HLog.Entry(key, val); + if (compressionContext != null) { + e.setCompressionContext(compressionContext); + } + b = this.reader.next(e.getKey(), e.getEdit()); + nextQueue.offer(e); + numberOfFileEntries++; + } + } + + if (nextQueue.size() == this.numberOfFileEntries + && getFailureType() == FailureType.BEGINNING) { + throw this.addFileInfoToException(new IOException("fake Exception")); + } else if (nextQueue.size() == this.numberOfFileEntries / 2 + && getFailureType() == FailureType.MIDDLE) { + throw this.addFileInfoToException(new IOException("fake Exception")); + } else if (nextQueue.size() == 1 && getFailureType() == FailureType.END) { + throw this.addFileInfoToException(new IOException("fake Exception")); + } + + if (nextQueue.peek() != null) { + edit++; + } + + Entry e = nextQueue.poll(); + + if (e.getEdit().isEmpty()) { + return null; + } + return e; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java new file mode 100644 index 0000000..e534cc2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java @@ -0,0 +1,361 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import java.util.Map; +import java.util.List; +import java.util.Random; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.conf.Configured; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry; + +/** + * This class runs performance benchmarks for {@link HLog}. + * See usage for this tool by running: + * $ hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation -h + */ +public final class HLogPerformanceEvaluation extends Configured implements Tool { + static final Log LOG = LogFactory.getLog(HLogPerformanceEvaluation.class.getName()); + + private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + static final String TABLE_NAME = "HLogPerformanceEvaluation"; + static final String QUALIFIER_PREFIX = "q"; + static final String FAMILY_PREFIX = "cf"; + + private int numQualifiers = 1; + private int valueSize = 512; + private int keySize = 16; + + /** + * Perform HLog.append() of Put object, for the number of iterations requested. + * Keys and Vaues are generated randomly, the number of column familes, + * qualifiers and key/value size is tunable by the user. + */ + class HLogPutBenchmark implements Runnable { + private final long numIterations; + private final int numFamilies; + private final boolean noSync; + private final HRegion region; + private final HTableDescriptor htd; + + HLogPutBenchmark(final HRegion region, final HTableDescriptor htd, + final long numIterations, final boolean noSync) { + this.numIterations = numIterations; + this.noSync = noSync; + this.numFamilies = htd.getColumnFamilies().length; + this.region = region; + this.htd = htd; + } + + public void run() { + byte[] key = new byte[keySize]; + byte[] value = new byte[valueSize]; + Random rand = new Random(Thread.currentThread().getId()); + HLog hlog = region.getLog(); + + try { + long startTime = System.currentTimeMillis(); + for (int i = 0; i < numIterations; ++i) { + Put put = setupPut(rand, key, value, numFamilies); + long now = System.currentTimeMillis(); + WALEdit walEdit = new WALEdit(); + addFamilyMapToWALEdit(put.getFamilyMap(), walEdit); + HRegionInfo hri = region.getRegionInfo(); + if (this.noSync) { + hlog.appendNoSync(hri, hri.getTableName(), walEdit, + HConstants.DEFAULT_CLUSTER_ID, now, htd); + } else { + hlog.append(hri, hri.getTableName(), walEdit, now, htd); + } + } + long totalTime = (System.currentTimeMillis() - startTime); + logBenchmarkResult(Thread.currentThread().getName(), numIterations, totalTime); + } catch (Exception e) { + LOG.error(getClass().getSimpleName() + " Thread failed", e); + } + } + } + + @Override + public int run(String[] args) throws Exception { + Path rootRegionDir = null; + int numThreads = 1; + long numIterations = 10000; + int numFamilies = 1; + boolean noSync = false; + boolean verify = false; + boolean verbose = false; + long roll = Long.MAX_VALUE; + // Process command line args + for (int i = 0; i < args.length; i++) { + String cmd = args[i]; + try { + if (cmd.equals("-threads")) { + numThreads = Integer.parseInt(args[++i]); + } else if (cmd.equals("-iterations")) { + numIterations = Long.parseLong(args[++i]); + } else if (cmd.equals("-path")) { + rootRegionDir = new Path(args[++i]); + } else if (cmd.equals("-families")) { + numFamilies = Integer.parseInt(args[++i]); + } else if (cmd.equals("-qualifiers")) { + numQualifiers = Integer.parseInt(args[++i]); + } else if (cmd.equals("-keySize")) { + keySize = Integer.parseInt(args[++i]); + } else if (cmd.equals("-valueSize")) { + valueSize = Integer.parseInt(args[++i]); + } else if (cmd.equals("-nosync")) { + noSync = true; + } else if (cmd.equals("-verify")) { + verify = true; + } else if (cmd.equals("-verbose")) { + verbose = true; + } else if (cmd.equals("-roll")) { + roll = Long.parseLong(args[++i]); + } else if (cmd.equals("-h")) { + printUsageAndExit(); + } else if (cmd.equals("--help")) { + printUsageAndExit(); + } else { + System.err.println("UNEXPECTED: " + cmd); + printUsageAndExit(); + } + } catch (Exception e) { + printUsageAndExit(); + } + } + + // Run HLog Performance Evaluation + FileSystem fs = FileSystem.get(getConf()); + LOG.info("" + fs); + try { + if (rootRegionDir == null) { + rootRegionDir = TEST_UTIL.getDataTestDir("HLogPerformanceEvaluation"); + } + rootRegionDir = rootRegionDir.makeQualified(fs); + cleanRegionRootDir(fs, rootRegionDir); + // Initialize Table Descriptor + HTableDescriptor htd = createHTableDescriptor(numFamilies); + final long whenToRoll = roll; + HLog hlog = new HLog(fs, new Path(rootRegionDir, "wals"), + new Path(rootRegionDir, "old.wals"), getConf()) { + int appends = 0; + protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit, + HTableDescriptor htd) + throws IOException { + this.appends++; + if (this.appends % whenToRoll == 0) { + LOG.info("Rolling after " + appends + " edits"); + rollWriter(); + } + super.doWrite(info, logKey, logEdit, htd); + }; + }; + hlog.rollWriter(); + HRegion region = null; + try { + region = openRegion(fs, rootRegionDir, htd, hlog); + long putTime = runBenchmark(new HLogPutBenchmark(region, htd, numIterations, noSync), numThreads); + logBenchmarkResult("Summary: threads=" + numThreads + ", iterations=" + numIterations, + numIterations * numThreads, putTime); + if (region != null) { + closeRegion(region); + region = null; + } + if (verify) { + Path dir = hlog.getDir(); + long editCount = 0; + for (FileStatus fss: fs.listStatus(dir)) { + editCount += verify(fss.getPath(), verbose); + } + long expected = numIterations * numThreads; + if (editCount != expected) { + throw new IllegalStateException("Counted=" + editCount + ", expected=" + expected); + } + } + } finally { + if (region != null) closeRegion(region); + // Remove the root dir for this test region + cleanRegionRootDir(fs, rootRegionDir); + } + } finally { + fs.close(); + } + + return(0); + } + + private static HTableDescriptor createHTableDescriptor(final int numFamilies) { + HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); + for (int i = 0; i < numFamilies; ++i) { + HColumnDescriptor colDef = new HColumnDescriptor(FAMILY_PREFIX + i); + htd.addFamily(colDef); + } + return htd; + } + + /** + * Verify the content of the WAL file. + * Verify that sequenceids are ascending and that the file has expected number + * of edits. + * @param wal + * @return Count of edits. + * @throws IOException + */ + private long verify(final Path wal, final boolean verbose) throws IOException { + HLog.Reader reader = HLog.getReader(wal.getFileSystem(getConf()), wal, getConf()); + long previousSeqid = -1; + long count = 0; + try { + while (true) { + Entry e = reader.next(); + if (e == null) break; + count++; + long seqid = e.getKey().getLogSeqNum(); + if (verbose) LOG.info("seqid=" + seqid); + if (previousSeqid >= seqid) { + throw new IllegalStateException("wal=" + wal.getName() + + ", previousSeqid=" + previousSeqid + ", seqid=" + seqid); + } + previousSeqid = seqid; + } + } finally { + reader.close(); + } + return count; + } + + private static void logBenchmarkResult(String testName, long numTests, long totalTime) { + float tsec = totalTime / 1000.0f; + LOG.info(String.format("%s took %.3fs %.3fops/s", testName, tsec, numTests / tsec)); + } + + private void printUsageAndExit() { + System.err.printf("Usage: bin/hbase %s [options]\n", getClass().getName()); + System.err.println(" where [options] are:"); + System.err.println(" -h|-help Show this help and exit."); + System.err.println(" -threads Number of threads writing on the WAL."); + System.err.println(" -iterations Number of iterations per thread."); + System.err.println(" -path Path where region's root directory is created."); + System.err.println(" -families Number of column families to write."); + System.err.println(" -qualifiers Number of qualifiers to write."); + System.err.println(" -keySize Row key size in byte."); + System.err.println(" -valueSize Row/Col value size in byte."); + System.err.println(" -nosync Append without syncing"); + System.err.println(" -verify Verify edits written in sequence"); + System.err.println(" -verbose Output extra info; e.g. all edit seq ids when verifying"); + System.err.println(" -roll Roll the way every N appends"); + System.err.println(""); + System.err.println("Examples:"); + System.err.println(""); + System.err.println(" To run 100 threads on hdfs with log rolling every 10k edits and verification afterward do:"); + System.err.println(" $ ./bin/hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation \\"); + System.err.println(" -conf ./core-site.xml -path hdfs://example.org:7000/tmp -threads 100 -roll 10000 -verify"); + System.exit(1); + } + + private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd, final HLog hlog) + throws IOException { + // Initialize HRegion + HRegionInfo regionInfo = new HRegionInfo(htd.getName()); + return HRegion.createHRegion(regionInfo, dir, getConf(), htd, hlog); + } + + private void closeRegion(final HRegion region) throws IOException { + if (region != null) { + region.close(); + HLog wal = region.getLog(); + if (wal != null) wal.close(); + } + } + + private void cleanRegionRootDir(final FileSystem fs, final Path dir) throws IOException { + if (fs.exists(dir)) { + fs.delete(dir, true); + } + } + + private Put setupPut(Random rand, byte[] key, byte[] value, final int numFamilies) { + rand.nextBytes(key); + Put put = new Put(key); + for (int cf = 0; cf < numFamilies; ++cf) { + for (int q = 0; q < numQualifiers; ++q) { + rand.nextBytes(value); + put.add(Bytes.toBytes(FAMILY_PREFIX + cf), Bytes.toBytes(QUALIFIER_PREFIX + q), value); + } + } + return put; + } + + private void addFamilyMapToWALEdit(Map> familyMap, WALEdit walEdit) { + for (List edits : familyMap.values()) { + for (KeyValue kv : edits) { + walEdit.add(kv); + } + } + } + + private long runBenchmark(Runnable runnable, final int numThreads) throws InterruptedException { + Thread[] threads = new Thread[numThreads]; + long startTime = System.currentTimeMillis(); + for (int i = 0; i < numThreads; ++i) { + threads[i] = new Thread(runnable); + threads[i].start(); + } + for (Thread t : threads) t.join(); + long endTime = System.currentTimeMillis(); + return(endTime - startTime); + } + + /** + * The guts of the {@link #main} method. + * Call this method to avoid the {@link #main(String[])} System.exit. + * @param args + * @return errCode + * @throws Exception + */ + static int innerMain(final String [] args) throws Exception { + return ToolRunner.run(HBaseConfiguration.create(), new HLogPerformanceEvaluation(), args); + } + + public static void main(String[] args) throws Exception { + System.exit(innerMain(args)); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java new file mode 100644 index 0000000..33a6b6b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtilsForTests.java @@ -0,0 +1,46 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; + +/** + * An Utility testcase that returns the number of log files that + * were rolled to be accessed from outside packages. + * + * This class makes available methods that are package protected. + * This is interesting for test only. + */ +public class HLogUtilsForTests { + + /** + * + * @param log + * @return + */ + public static int getNumLogFiles(HLog log) { + return log.getNumLogFiles(); + } + + public static int getNumEntries(HLog log) { + return log.getNumEntries(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java new file mode 100644 index 0000000..bf9bfc4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedSequenceFileLogWriter.java @@ -0,0 +1,41 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; + +import org.apache.hadoop.hbase.util.Bytes; + +public class InstrumentedSequenceFileLogWriter extends SequenceFileLogWriter { + + public InstrumentedSequenceFileLogWriter() { + super(HLogKey.class); + } + + public static boolean activateFailure = false; + @Override + public void append(HLog.Entry entry) throws IOException { + super.append(entry); + if (activateFailure && Bytes.equals(entry.getKey().getEncodedRegionName(), "break".getBytes())) { + System.out.println(getClass().getName() + ": I will throw an exception now..."); + throw(new IOException("This exception is instrumented and should only be thrown for testing")); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCompressor.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCompressor.java new file mode 100644 index 0000000..dad681d --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCompressor.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package org.apache.hadoop.hbase.regionserver.wal; + + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test our compressor class. + */ +@Category(SmallTests.class) +public class TestCompressor { + @BeforeClass + public static void setUpBeforeClass() throws Exception { + } + + @Test + public void testToShort() { + short s = 1; + assertEquals(s, Compressor.toShort((byte)0, (byte)1)); + s <<= 8; + assertEquals(s, Compressor.toShort((byte)1, (byte)0)); + } + + @Test (expected = IllegalArgumentException.class) + public void testNegativeToShort() { + Compressor.toShort((byte)0xff, (byte)0xff); + } + + @Test + public void testCompressingWithNullDictionaries() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + byte [] blahBytes = Bytes.toBytes("blah"); + Compressor.writeCompressed(blahBytes, 0, blahBytes.length, dos, null); + dos.close(); + byte [] dosbytes = baos.toByteArray(); + DataInputStream dis = + new DataInputStream(new ByteArrayInputStream(dosbytes)); + byte [] product = Compressor.readCompressed(dis, null); + assertTrue(Bytes.equals(blahBytes, product)); + } + + @Test + public void testCompressingWithClearDictionaries() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + Dictionary dictionary = new LRUDictionary(); + byte [] blahBytes = Bytes.toBytes("blah"); + Compressor.writeCompressed(blahBytes, 0, blahBytes.length, dos, dictionary); + dos.close(); + byte [] dosbytes = baos.toByteArray(); + DataInputStream dis = + new DataInputStream(new ByteArrayInputStream(dosbytes)); + dictionary = new LRUDictionary(); + byte [] product = Compressor.readCompressed(dis, dictionary); + assertTrue(Bytes.equals(blahBytes, product)); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java new file mode 100644 index 0000000..5308636 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests for HLog write durability + */ +@Category(MediumTests.class) +public class TestDurability { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static FileSystem FS; + private static MiniDFSCluster CLUSTER; + private static Configuration CONF; + private static final Path DIR = TEST_UTIL.getDataTestDir("TestDurability"); + + private static byte[] FAMILY = Bytes.toBytes("family"); + private static byte[] ROW = Bytes.toBytes("row"); + private static byte[] COL = Bytes.toBytes("col"); + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + CONF = TEST_UTIL.getConfiguration(); + CONF.setLong("hbase.regionserver.optionallogflushinterval", 500*1000); + TEST_UTIL.startMiniDFSCluster(1); + + CLUSTER = TEST_UTIL.getDFSCluster(); + FS = CLUSTER.getFileSystem(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testDurability() throws Exception { + HLog wal = new HLog(FS, new Path(DIR, "hlogdir"), + new Path(DIR, "hlogdir_archive"), CONF); + byte[] tableName = Bytes.toBytes("TestDurability"); + HRegion region = createHRegion(tableName, "region", wal, false); + HRegion deferredRegion = createHRegion(tableName, "deferredRegion", wal, true); + + region.put(newPut(null)); + + verifyHLogCount(wal, 1); + + // a put through the deferred table does not write to the wal immdiately + deferredRegion.put(newPut(null)); + verifyHLogCount(wal, 1); + // but will after we sync the wal + wal.sync(); + verifyHLogCount(wal, 2); + + // a put through a deferred table will be sync with the put sync'ed put + deferredRegion.put(newPut(null)); + verifyHLogCount(wal, 2); + region.put(newPut(null)); + verifyHLogCount(wal, 4); + + // a put through a deferred table will be sync with the put sync'ed put + deferredRegion.put(newPut(Durability.USE_DEFAULT)); + verifyHLogCount(wal, 4); + region.put(newPut(Durability.USE_DEFAULT)); + verifyHLogCount(wal, 6); + + // SKIP_WAL never writes to the wal + region.put(newPut(Durability.SKIP_WAL)); + deferredRegion.put(newPut(Durability.SKIP_WAL)); + verifyHLogCount(wal, 6); + wal.sync(); + verifyHLogCount(wal, 6); + + // async overrides sync table default + region.put(newPut(Durability.ASYNC_WAL)); + deferredRegion.put(newPut(Durability.ASYNC_WAL)); + verifyHLogCount(wal, 6); + wal.sync(); + verifyHLogCount(wal, 8); + + // sync overrides async table default + region.put(newPut(Durability.SYNC_WAL)); + deferredRegion.put(newPut(Durability.SYNC_WAL)); + verifyHLogCount(wal, 10); + + // fsync behaves like sync + region.put(newPut(Durability.FSYNC_WAL)); + deferredRegion.put(newPut(Durability.FSYNC_WAL)); + verifyHLogCount(wal, 12); + } + + private Put[] newPut(Durability durability) { + Put p = new Put(ROW); + p.add(FAMILY, COL, COL); + if (durability != null) { + p.setDurability(durability); + } + return new Put[]{p}; + } + + private void verifyHLogCount(HLog log, int expected) throws Exception { + Path walPath = log.computeFilename(); + HLog.Reader reader = HLog.getReader(FS, walPath, CONF); + int count = 0; + HLog.Entry entry = new HLog.Entry(); + while (reader.next(entry) != null) count++; + reader.close(); + assertEquals(expected, count); + } + + // lifted from TestAtomicOperation + private HRegion createHRegion (byte [] tableName, String callingMethod, HLog log, boolean isDeferredLogFlush) + throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.setDeferredLogFlush(isDeferredLogFlush); + HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); + htd.addFamily(hcd); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + Path path = new Path(DIR + callingMethod); + if (FS.exists(path)) { + if (!FS.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + return HRegion.createHRegion(info, path, HBaseConfiguration.create(), htd, log); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java new file mode 100644 index 0000000..c9bb3b2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java @@ -0,0 +1,821 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSHDFSUtils; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.SampleRegionWALObserver; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.FSConstants; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.namenode.LeaseManager; +import org.apache.hadoop.io.SequenceFile; +import org.apache.log4j.Level; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** JUnit test case for HLog */ +@Category(LargeTests.class) +public class TestHLog { + private static final Log LOG = LogFactory.getLog(TestHLog.class); + { + ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem")) + .getLogger().setLevel(Level.ALL); + ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL); + } + + private static Configuration conf; + private static FileSystem fs; + private static Path dir; + private static MiniDFSCluster cluster; + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static Path hbaseDir; + private static Path oldLogDir; + + @Before + public void setUp() throws Exception { + + FileStatus[] entries = fs.listStatus(new Path("/")); + for (FileStatus dir : entries) { + fs.delete(dir.getPath(), true); + } + + } + + @After + public void tearDown() throws Exception { + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Make block sizes small. + TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024); + // needed for testAppendClose() + TEST_UTIL.getConfiguration().setBoolean("dfs.support.broken.append", true); + TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); + // quicker heartbeat interval for faster DN death notification + TEST_UTIL.getConfiguration().setInt("heartbeat.recheck.interval", 5000); + TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1); + TEST_UTIL.getConfiguration().setInt("dfs.socket.timeout", 5000); + // faster failover with cluster.shutdown();fs.close() idiom + TEST_UTIL.getConfiguration() + .setInt("ipc.client.connect.max.retries", 1); + TEST_UTIL.getConfiguration().setInt( + "dfs.client.block.recovery.retries", 1); + TEST_UTIL.getConfiguration().setInt( + "ipc.client.connection.maxidletime", 500); + TEST_UTIL.getConfiguration().set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, + SampleRegionWALObserver.class.getName()); + TEST_UTIL.startMiniDFSCluster(3); + + conf = TEST_UTIL.getConfiguration(); + cluster = TEST_UTIL.getDFSCluster(); + fs = cluster.getFileSystem(); + + hbaseDir = TEST_UTIL.createRootDir(); + oldLogDir = new Path(hbaseDir, ".oldlogs"); + dir = new Path(hbaseDir, getName()); + } + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + private static String getName() { + // TODO Auto-generated method stub + return "TestHLog"; + } + + /** + * Test that with three concurrent threads we still write edits in sequence + * edit id order. + * @throws Exception + */ + @Test + public void testMaintainOrderWithConcurrentWrites() throws Exception { + // Run the HPE tool with three threads writing 3000 edits each concurrently. + // When done, verify that all edits were written and that the order in the + // WALs is of ascending edit sequence ids. + int errCode = + HLogPerformanceEvaluation.innerMain(new String [] {"-threads", "3", "-verify", "-iterations", "3000"}); + assertEquals(0, errCode); + } + + /** + * Just write multiple logs then split. Before fix for HADOOP-2283, this + * would fail. + * @throws IOException + */ + @Test + public void testSplit() throws IOException { + + final byte [] tableName = Bytes.toBytes(getName()); + final byte [] rowName = tableName; + Path logdir = new Path(hbaseDir, HConstants.HREGION_LOGDIR_NAME); + HLog log = new HLog(fs, logdir, oldLogDir, conf); + final int howmany = 3; + HRegionInfo[] infos = new HRegionInfo[3]; + Path tabledir = new Path(hbaseDir, getName()); + fs.mkdirs(tabledir); + for(int i = 0; i < howmany; i++) { + infos[i] = new HRegionInfo(tableName, + Bytes.toBytes("" + i), Bytes.toBytes("" + (i+1)), false); + fs.mkdirs(new Path(tabledir, infos[i].getEncodedName())); + LOG.info("allo " + new Path(tabledir, infos[i].getEncodedName()).toString()); + } + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor("column")); + + // Add edits for three regions. + try { + for (int ii = 0; ii < howmany; ii++) { + for (int i = 0; i < howmany; i++) { + + for (int j = 0; j < howmany; j++) { + WALEdit edit = new WALEdit(); + byte [] family = Bytes.toBytes("column"); + byte [] qualifier = Bytes.toBytes(Integer.toString(j)); + byte [] column = Bytes.toBytes("column:" + Integer.toString(j)); + edit.add(new KeyValue(rowName, family, qualifier, + System.currentTimeMillis(), column)); + LOG.info("Region " + i + ": " + edit); + log.append(infos[i], tableName, edit, + System.currentTimeMillis(), htd); + } + } + log.rollWriter(); + } + log.close(); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, logdir, this.oldLogDir, this.fs); + List splits = + logSplitter.splitLog(); + verifySplits(splits, howmany); + log = null; + } finally { + if (log != null) { + log.closeAndDelete(); + } + } + } + + /** + * Test new HDFS-265 sync. + * @throws Exception + */ + @Test + public void Broken_testSync() throws Exception { + byte [] bytes = Bytes.toBytes(getName()); + // First verify that using streams all works. + Path p = new Path(dir, getName() + ".fsdos"); + FSDataOutputStream out = fs.create(p); + out.write(bytes); + Method syncMethod = null; + try { + syncMethod = out.getClass().getMethod("hflush", new Class []{}); + } catch (NoSuchMethodException e) { + try { + syncMethod = out.getClass().getMethod("sync", new Class []{}); + } catch (NoSuchMethodException ex) { + fail("This version of Hadoop supports neither Syncable.sync() " + + "nor Syncable.hflush()."); + } + } + syncMethod.invoke(out, new Object[]{}); + FSDataInputStream in = fs.open(p); + assertTrue(in.available() > 0); + byte [] buffer = new byte [1024]; + int read = in.read(buffer); + assertEquals(bytes.length, read); + out.close(); + in.close(); + Path subdir = new Path(dir, "hlogdir"); + HLog wal = new HLog(fs, subdir, oldLogDir, conf); + final int total = 20; + HLog.Reader reader = null; + + try { + HRegionInfo info = new HRegionInfo(bytes, + null,null, false); + HTableDescriptor htd = new HTableDescriptor(); + htd.addFamily(new HColumnDescriptor(bytes)); + + for (int i = 0; i < total; i++) { + WALEdit kvs = new WALEdit(); + kvs.add(new KeyValue(Bytes.toBytes(i), bytes, bytes)); + wal.append(info, bytes, kvs, System.currentTimeMillis(), htd); + } + // Now call sync and try reading. Opening a Reader before you sync just + // gives you EOFE. + wal.sync(); + // Open a Reader. + Path walPath = wal.computeFilename(); + reader = HLog.getReader(fs, walPath, conf); + int count = 0; + HLog.Entry entry = new HLog.Entry(); + while ((entry = reader.next(entry)) != null) count++; + assertEquals(total, count); + reader.close(); + // Add test that checks to see that an open of a Reader works on a file + // that has had a sync done on it. + for (int i = 0; i < total; i++) { + WALEdit kvs = new WALEdit(); + kvs.add(new KeyValue(Bytes.toBytes(i), bytes, bytes)); + wal.append(info, bytes, kvs, System.currentTimeMillis(), htd); + } + reader = HLog.getReader(fs, walPath, conf); + count = 0; + while((entry = reader.next(entry)) != null) count++; + assertTrue(count >= total); + reader.close(); + // If I sync, should see double the edits. + wal.sync(); + reader = HLog.getReader(fs, walPath, conf); + count = 0; + while((entry = reader.next(entry)) != null) count++; + assertEquals(total * 2, count); + // Now do a test that ensures stuff works when we go over block boundary, + // especially that we return good length on file. + final byte [] value = new byte[1025 * 1024]; // Make a 1M value. + for (int i = 0; i < total; i++) { + WALEdit kvs = new WALEdit(); + kvs.add(new KeyValue(Bytes.toBytes(i), bytes, value)); + wal.append(info, bytes, kvs, System.currentTimeMillis(), htd); + } + // Now I should have written out lots of blocks. Sync then read. + wal.sync(); + reader = HLog.getReader(fs, walPath, conf); + count = 0; + while((entry = reader.next(entry)) != null) count++; + assertEquals(total * 3, count); + reader.close(); + // Close it and ensure that closed, Reader gets right length also. + wal.close(); + reader = HLog.getReader(fs, walPath, conf); + count = 0; + while((entry = reader.next(entry)) != null) count++; + assertEquals(total * 3, count); + reader.close(); + } finally { + if (wal != null) wal.closeAndDelete(); + if (reader != null) reader.close(); + } + } + + /** + * Test the findMemstoresWithEditsEqualOrOlderThan method. + * @throws IOException + */ + @Test + public void testFindMemstoresWithEditsEqualOrOlderThan() throws IOException { + Map regionsToSeqids = new HashMap(); + for (int i = 0; i < 10; i++) { + Long l = Long.valueOf(i); + regionsToSeqids.put(l.toString().getBytes(), l); + } + byte [][] regions = + HLog.findMemstoresWithEditsEqualOrOlderThan(1, regionsToSeqids); + assertEquals(2, regions.length); + assertTrue(Bytes.equals(regions[0], "0".getBytes()) || + Bytes.equals(regions[0], "1".getBytes())); + regions = HLog.findMemstoresWithEditsEqualOrOlderThan(3, regionsToSeqids); + int count = 4; + assertEquals(count, regions.length); + // Regions returned are not ordered. + for (int i = 0; i < count; i++) { + assertTrue(Bytes.equals(regions[i], "0".getBytes()) || + Bytes.equals(regions[i], "1".getBytes()) || + Bytes.equals(regions[i], "2".getBytes()) || + Bytes.equals(regions[i], "3".getBytes())); + } + } + + private void verifySplits(List splits, final int howmany) + throws IOException { + assertEquals(howmany, splits.size()); + for (int i = 0; i < splits.size(); i++) { + LOG.info("Verifying=" + splits.get(i)); + HLog.Reader reader = HLog.getReader(fs, splits.get(i), conf); + try { + int count = 0; + String previousRegion = null; + long seqno = -1; + HLog.Entry entry = new HLog.Entry(); + while((entry = reader.next(entry)) != null) { + HLogKey key = entry.getKey(); + String region = Bytes.toString(key.getEncodedRegionName()); + // Assert that all edits are for same region. + if (previousRegion != null) { + assertEquals(previousRegion, region); + } + LOG.info("oldseqno=" + seqno + ", newseqno=" + key.getLogSeqNum()); + assertTrue(seqno < key.getLogSeqNum()); + seqno = key.getLogSeqNum(); + previousRegion = region; + count++; + } + assertEquals(howmany * howmany, count); + } finally { + reader.close(); + } + } + } + + /* + * We pass different values to recoverFileLease() so that different code paths are covered + * + * For this test to pass, requires: + * 1. HDFS-200 (append support) + * 2. HDFS-988 (SafeMode should freeze file operations + * [FSNamesystem.nextGenerationStampForBlock]) + * 3. HDFS-142 (on restart, maintain pendingCreates) + */ + @Test + public void testAppendClose() throws Exception { + testAppendClose(true); + testAppendClose(false); + } + + /* + * @param triggerDirectAppend whether to trigger direct call of fs.append() + */ + public void testAppendClose(final boolean triggerDirectAppend) throws Exception { + byte [] tableName = Bytes.toBytes(getName()); + HRegionInfo regioninfo = new HRegionInfo(tableName, + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false); + Path subdir = new Path(dir, "hlogdir" + triggerDirectAppend); + Path archdir = new Path(dir, "hlogdir_archive"); + HLog wal = new HLog(fs, subdir, archdir, conf); + final int total = 20; + + HTableDescriptor htd = new HTableDescriptor(); + htd.addFamily(new HColumnDescriptor(tableName)); + + for (int i = 0; i < total; i++) { + WALEdit kvs = new WALEdit(); + kvs.add(new KeyValue(Bytes.toBytes(i), tableName, tableName)); + wal.append(regioninfo, tableName, kvs, System.currentTimeMillis(), htd); + } + // Now call sync to send the data to HDFS datanodes + wal.sync(); + int namenodePort = cluster.getNameNodePort(); + final Path walPath = wal.computeFilename(); + + + // Stop the cluster. (ensure restart since we're sharing MiniDFSCluster) + try { + DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); + dfs.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_ENTER); + cluster.shutdown(); + try { + // wal.writer.close() will throw an exception, + // but still call this since it closes the LogSyncer thread first + wal.close(); + } catch (IOException e) { + LOG.info(e); + } + fs.close(); // closing FS last so DFSOutputStream can't call close + LOG.info("STOPPED first instance of the cluster"); + } finally { + // Restart the cluster + while (cluster.isClusterUp()){ + LOG.error("Waiting for cluster to go down"); + Thread.sleep(1000); + } + + // Workaround a strange issue with Hadoop's RPC system - if we don't + // sleep here, the new datanodes will pick up a cached IPC connection to + // the old (dead) NN and fail to start. Sleeping 2 seconds goes past + // the idle time threshold configured in the conf above + Thread.sleep(2000); + + cluster = new MiniDFSCluster(namenodePort, conf, 5, false, true, true, null, null, null, null); + TEST_UTIL.setDFSCluster(cluster); + cluster.waitActive(); + fs = cluster.getFileSystem(); + LOG.info("START second instance."); + } + + // set the lease period to be 1 second so that the + // namenode triggers lease recovery upon append request + Method setLeasePeriod = cluster.getClass() + .getDeclaredMethod("setLeasePeriod", new Class[]{Long.TYPE, Long.TYPE}); + setLeasePeriod.setAccessible(true); + setLeasePeriod.invoke(cluster, + new Object[]{new Long(1000), new Long(1000)}); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + LOG.info(e); + } + + // Now try recovering the log, like the HMaster would do + final FileSystem recoveredFs = fs; + final Configuration rlConf = conf; + + class RecoverLogThread extends Thread { + public Exception exception = null; + public void run() { + try { + rlConf.setBoolean(FSHDFSUtils.TEST_TRIGGER_DFS_APPEND, triggerDirectAppend); + FSUtils.getInstance(fs, rlConf) + .recoverFileLease(recoveredFs, walPath, rlConf); + } catch (IOException e) { + exception = e; + } + } + } + + RecoverLogThread t = new RecoverLogThread(); + t.start(); + // Timeout after 60 sec. Without correct patches, would be an infinite loop + t.join(60 * 1000); + if(t.isAlive()) { + t.interrupt(); + throw new Exception("Timed out waiting for HLog.recoverLog()"); + } + + if (t.exception != null) + throw t.exception; + + // Make sure you can read all the content + HLog.Reader reader = HLog.getReader(this.fs, walPath, this.conf); + int count = 0; + HLog.Entry entry = new HLog.Entry(); + while (reader.next(entry) != null) { + count++; + assertTrue("Should be one KeyValue per WALEdit", + entry.getEdit().getKeyValues().size() == 1); + } + assertEquals(total, count); + reader.close(); + + // Reset the lease period + setLeasePeriod.invoke(cluster, new Object[]{new Long(60000), new Long(3600000)}); + } + + /** + * Tests that we can write out an edit, close, and then read it back in again. + * @throws IOException + */ + @Test + public void testEditAdd() throws IOException { + final int COL_COUNT = 10; + final byte [] tableName = Bytes.toBytes("tablename"); + final byte [] row = Bytes.toBytes("row"); + HLog.Reader reader = null; + HLog log = null; + try { + log = new HLog(fs, dir, oldLogDir, conf); + // Write columns named 1, 2, 3, etc. and then values of single byte + // 1, 2, 3... + long timestamp = System.currentTimeMillis(); + WALEdit cols = new WALEdit(); + for (int i = 0; i < COL_COUNT; i++) { + cols.add(new KeyValue(row, Bytes.toBytes("column"), + Bytes.toBytes(Integer.toString(i)), + timestamp, new byte[] { (byte)(i + '0') })); + } + HRegionInfo info = new HRegionInfo(tableName, + row,Bytes.toBytes(Bytes.toString(row) + "1"), false); + HTableDescriptor htd = new HTableDescriptor(); + htd.addFamily(new HColumnDescriptor("column")); + + log.append(info, tableName, cols, System.currentTimeMillis(), htd); + long logSeqId = log.startCacheFlush(info.getEncodedNameAsBytes()); + log.completeCacheFlush(info.getEncodedNameAsBytes(), tableName, logSeqId, + info.isMetaRegion()); + log.close(); + Path filename = log.computeFilename(); + log = null; + // Now open a reader on the log and assert append worked. + reader = HLog.getReader(fs, filename, conf); + // Above we added all columns on a single row so we only read one + // entry in the below... thats why we have '1'. + for (int i = 0; i < 1; i++) { + HLog.Entry entry = reader.next(null); + if (entry == null) break; + HLogKey key = entry.getKey(); + WALEdit val = entry.getEdit(); + assertTrue(Bytes.equals(info.getEncodedNameAsBytes(), key.getEncodedRegionName())); + assertTrue(Bytes.equals(tableName, key.getTablename())); + KeyValue kv = val.getKeyValues().get(0); + assertTrue(Bytes.equals(row, kv.getRow())); + assertEquals((byte)(i + '0'), kv.getValue()[0]); + System.out.println(key + " " + val); + } + HLog.Entry entry = null; + while ((entry = reader.next(null)) != null) { + HLogKey key = entry.getKey(); + WALEdit val = entry.getEdit(); + // Assert only one more row... the meta flushed row. + assertTrue(Bytes.equals(info.getEncodedNameAsBytes(), key.getEncodedRegionName())); + assertTrue(Bytes.equals(tableName, key.getTablename())); + KeyValue kv = val.getKeyValues().get(0); + assertTrue(Bytes.equals(HLog.METAROW, kv.getRow())); + assertTrue(Bytes.equals(HLog.METAFAMILY, kv.getFamily())); + assertEquals(0, Bytes.compareTo(HLog.COMPLETE_CACHE_FLUSH, + val.getKeyValues().get(0).getValue())); + System.out.println(key + " " + val); + } + } finally { + if (log != null) { + log.closeAndDelete(); + } + if (reader != null) { + reader.close(); + } + } + } + + /** + * @throws IOException + */ + @Test + public void testAppend() throws IOException { + final int COL_COUNT = 10; + final byte [] tableName = Bytes.toBytes("tablename"); + final byte [] row = Bytes.toBytes("row"); + Reader reader = null; + HLog log = new HLog(fs, dir, oldLogDir, conf); + try { + // Write columns named 1, 2, 3, etc. and then values of single byte + // 1, 2, 3... + long timestamp = System.currentTimeMillis(); + WALEdit cols = new WALEdit(); + for (int i = 0; i < COL_COUNT; i++) { + cols.add(new KeyValue(row, Bytes.toBytes("column"), + Bytes.toBytes(Integer.toString(i)), + timestamp, new byte[] { (byte)(i + '0') })); + } + HRegionInfo hri = new HRegionInfo(tableName, + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); + HTableDescriptor htd = new HTableDescriptor(); + htd.addFamily(new HColumnDescriptor("column")); + log.append(hri, tableName, cols, System.currentTimeMillis(), htd); + long logSeqId = log.startCacheFlush(hri.getEncodedNameAsBytes()); + log.completeCacheFlush(hri.getEncodedNameAsBytes(), tableName, logSeqId, false); + log.close(); + Path filename = log.computeFilename(); + log = null; + // Now open a reader on the log and assert append worked. + reader = HLog.getReader(fs, filename, conf); + HLog.Entry entry = reader.next(); + assertEquals(COL_COUNT, entry.getEdit().size()); + int idx = 0; + for (KeyValue val : entry.getEdit().getKeyValues()) { + assertTrue(Bytes.equals(hri.getEncodedNameAsBytes(), + entry.getKey().getEncodedRegionName())); + assertTrue(Bytes.equals(tableName, entry.getKey().getTablename())); + assertTrue(Bytes.equals(row, val.getRow())); + assertEquals((byte)(idx + '0'), val.getValue()[0]); + System.out.println(entry.getKey() + " " + val); + idx++; + } + + // Get next row... the meta flushed row. + entry = reader.next(); + assertEquals(1, entry.getEdit().size()); + for (KeyValue val : entry.getEdit().getKeyValues()) { + assertTrue(Bytes.equals(hri.getEncodedNameAsBytes(), + entry.getKey().getEncodedRegionName())); + assertTrue(Bytes.equals(tableName, entry.getKey().getTablename())); + assertTrue(Bytes.equals(HLog.METAROW, val.getRow())); + assertTrue(Bytes.equals(HLog.METAFAMILY, val.getFamily())); + assertEquals(0, Bytes.compareTo(HLog.COMPLETE_CACHE_FLUSH, + val.getValue())); + System.out.println(entry.getKey() + " " + val); + } + } finally { + if (log != null) { + log.closeAndDelete(); + } + if (reader != null) { + reader.close(); + } + } + } + + /** + * Test that we can visit entries before they are appended + * @throws Exception + */ + @Test + public void testVisitors() throws Exception { + final int COL_COUNT = 10; + final byte [] tableName = Bytes.toBytes("tablename"); + final byte [] row = Bytes.toBytes("row"); + HLog log = new HLog(fs, dir, oldLogDir, conf); + try { + DumbWALActionsListener visitor = new DumbWALActionsListener(); + log.registerWALActionsListener(visitor); + long timestamp = System.currentTimeMillis(); + HTableDescriptor htd = new HTableDescriptor(); + htd.addFamily(new HColumnDescriptor("column")); + + HRegionInfo hri = new HRegionInfo(tableName, + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); + for (int i = 0; i < COL_COUNT; i++) { + WALEdit cols = new WALEdit(); + cols.add(new KeyValue(row, Bytes.toBytes("column"), + Bytes.toBytes(Integer.toString(i)), + timestamp, new byte[]{(byte) (i + '0')})); + log.append(hri, tableName, cols, System.currentTimeMillis(), htd); + } + assertEquals(COL_COUNT, visitor.increments); + log.unregisterWALActionsListener(visitor); + WALEdit cols = new WALEdit(); + cols.add(new KeyValue(row, Bytes.toBytes("column"), + Bytes.toBytes(Integer.toString(11)), + timestamp, new byte[]{(byte) (11 + '0')})); + log.append(hri, tableName, cols, System.currentTimeMillis(), htd); + assertEquals(COL_COUNT, visitor.increments); + } finally { + if (log != null) log.closeAndDelete(); + } + } + + @Test + public void testLogCleaning() throws Exception { + LOG.info("testLogCleaning"); + final byte [] tableName = Bytes.toBytes("testLogCleaning"); + final byte [] tableName2 = Bytes.toBytes("testLogCleaning2"); + + HLog log = new HLog(fs, dir, oldLogDir, conf); + try { + HRegionInfo hri = new HRegionInfo(tableName, + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); + HRegionInfo hri2 = new HRegionInfo(tableName2, + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); + + // Add a single edit and make sure that rolling won't remove the file + // Before HBASE-3198 it used to delete it + addEdits(log, hri, tableName, 1); + log.rollWriter(); + assertEquals(1, log.getNumLogFiles()); + + // See if there's anything wrong with more than 1 edit + addEdits(log, hri, tableName, 2); + log.rollWriter(); + assertEquals(2, log.getNumLogFiles()); + + // Now mix edits from 2 regions, still no flushing + addEdits(log, hri, tableName, 1); + addEdits(log, hri2, tableName2, 1); + addEdits(log, hri, tableName, 1); + addEdits(log, hri2, tableName2, 1); + log.rollWriter(); + assertEquals(3, log.getNumLogFiles()); + + // Flush the first region, we expect to see the first two files getting + // archived + long seqId = log.startCacheFlush(hri.getEncodedNameAsBytes()); + log.completeCacheFlush(hri.getEncodedNameAsBytes(), tableName, seqId, false); + log.rollWriter(); + assertEquals(2, log.getNumLogFiles()); + + // Flush the second region, which removes all the remaining output files + // since the oldest was completely flushed and the two others only contain + // flush information + seqId = log.startCacheFlush(hri2.getEncodedNameAsBytes()); + log.completeCacheFlush(hri2.getEncodedNameAsBytes(), tableName2, seqId, false); + log.rollWriter(); + assertEquals(0, log.getNumLogFiles()); + } finally { + if (log != null) log.closeAndDelete(); + } + } + + /** + * A loaded WAL coprocessor won't break existing HLog test cases. + */ + @Test + public void testWALCoprocessorLoaded() throws Exception { + // test to see whether the coprocessor is loaded or not. + HLog log = new HLog(fs, dir, oldLogDir, conf); + try { + WALCoprocessorHost host = log.getCoprocessorHost(); + Coprocessor c = host.findCoprocessor(SampleRegionWALObserver.class.getName()); + assertNotNull(c); + } finally { + if (log != null) log.closeAndDelete(); + } + } + + private void addEdits(HLog log, HRegionInfo hri, byte [] tableName, + int times) throws IOException { + HTableDescriptor htd = new HTableDescriptor(); + htd.addFamily(new HColumnDescriptor("row")); + + final byte [] row = Bytes.toBytes("row"); + for (int i = 0; i < times; i++) { + long timestamp = System.currentTimeMillis(); + WALEdit cols = new WALEdit(); + cols.add(new KeyValue(row, row, row, timestamp, row)); + log.append(hri, tableName, cols, timestamp, htd); + } + } + + static class DumbWALActionsListener implements WALActionsListener { + int increments = 0; + + @Override + public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, + WALEdit logEdit) { + increments++; + } + + @Override + public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEdit logEdit) { + //To change body of implemented methods use File | Settings | File Templates. + increments++; + } + + @Override + public void preLogRoll(Path oldFile, Path newFile) { + // TODO Auto-generated method stub + } + + @Override + public void postLogRoll(Path oldFile, Path newFile) { + // TODO Auto-generated method stub + } + + @Override + public void preLogArchive(Path oldFile, Path newFile) { + // TODO Auto-generated method stub + } + + @Override + public void postLogArchive(Path oldFile, Path newFile) { + // TODO Auto-generated method stub + } + + @Override + public void logRollRequested() { + // TODO Auto-generated method stub + + } + + @Override + public void logCloseRequested() { + // not interested + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogBench.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogBench.java new file mode 100644 index 0000000..0b0a7ce --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogBench.java @@ -0,0 +1,346 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; +import java.util.Random; + +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.hbase.*; +import org.apache.log4j.Level; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.ipc.HBaseRPC; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestHLogBench extends Configured implements Tool { + + static final Log LOG = LogFactory.getLog(TestHLogBench.class); + private static final Random r = new Random(); + + private static final byte [] FAMILY = Bytes.toBytes("hlogbenchFamily"); + + // accumulate time here + private static int totalTime = 0; + private static Object lock = new Object(); + + // the file system where to create the Hlog file + protected FileSystem fs; + + // the number of threads and the number of iterations per thread + private int numThreads = 300; + private int numIterationsPerThread = 10000; + + private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private Path regionRootDir =TEST_UTIL.getDataTestDir("TestHLogBench") ; + + private boolean appendNoSync = false; + + public TestHLogBench() { + this(null); + } + + private TestHLogBench(Configuration conf) { + super(conf); + fs = null; + } + + /** + * Initialize file system object + */ + public void init() throws IOException { + getConf().setQuietMode(true); + if (this.fs == null) { + this.fs = FileSystem.get(getConf()); + } + } + + /** + * Close down file system + */ + public void close() throws IOException { + if (fs != null) { + fs.close(); + fs = null; + } + } + + /** + * The main run method of TestHLogBench + */ + public int run(String argv[]) throws Exception { + + int exitCode = -1; + int i = 0; + + // verify that we have enough command line parameters + if (argv.length < 4) { + printUsage(""); + return exitCode; + } + + // initialize LogBench + try { + init(); + } catch (HBaseRPC.VersionMismatch v) { + LOG.warn("Version Mismatch between client and server" + + "... command aborted."); + return exitCode; + } catch (IOException e) { + LOG.warn("Bad connection to FS. command aborted."); + return exitCode; + } + + try { + for (; i < argv.length; i++) { + if ("-numThreads".equals(argv[i])) { + i++; + this.numThreads = Integer.parseInt(argv[i]); + } else if ("-numIterationsPerThread".equals(argv[i])) { + i++; + this.numIterationsPerThread = Integer.parseInt(argv[i]); + } else if ("-path".equals(argv[i])) { + // get an absolute path using the default file system + i++; + this.regionRootDir = new Path(argv[i]); + this.regionRootDir = regionRootDir.makeQualified(this.fs); + } else if ("-nosync".equals(argv[i])) { + this.appendNoSync = true; + } else { + printUsage(argv[i]); + return exitCode; + } + } + } catch (NumberFormatException nfe) { + LOG.warn("Illegal numThreads or numIterationsPerThread, " + + " a positive integer expected"); + throw nfe; + } + go(); + return 0; + } + + private void go() throws IOException, InterruptedException { + + long start = System.currentTimeMillis(); + log("Running TestHLogBench with " + numThreads + " threads each doing " + + numIterationsPerThread + " HLog appends " + + (appendNoSync ? "nosync" : "sync") + + " at rootDir " + regionRootDir); + + // Mock an HRegion + byte [] tableName = Bytes.toBytes("table"); + byte [][] familyNames = new byte [][] { FAMILY }; + HTableDescriptor htd = new HTableDescriptor(); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f1"))); + HRegion region = mockRegion(tableName, familyNames, regionRootDir); + HLog hlog = region.getLog(); + + // Spin up N threads to each perform M log operations + LogWriter [] incrementors = new LogWriter[numThreads]; + for (int i=0; i] " + + " [-numIterationsPerThread ] " + + " [-path ]" + + " [-nosync]"); + } + + /** + * A thread that writes data to an HLog + */ + public static class LogWriter extends Thread { + + private final HRegion region; + private final int threadNumber; + private final int numIncrements; + private final HLog hlog; + private boolean appendNoSync; + private byte[] tableName; + + private int count; + + public LogWriter(HRegion region, byte[] tableName, + HLog log, int threadNumber, + int numIncrements, boolean appendNoSync) { + this.region = region; + this.threadNumber = threadNumber; + this.numIncrements = numIncrements; + this.hlog = log; + this.count = 0; + this.appendNoSync = appendNoSync; + this.tableName = tableName; + setDaemon(true); + //log("LogWriter[" + threadNumber + "] instantiated"); + } + + @Override + public void run() { + long now = System.currentTimeMillis(); + byte [] key = Bytes.toBytes("thisisakey"); + KeyValue kv = new KeyValue(key, now); + WALEdit walEdit = new WALEdit(); + walEdit.add(kv); + HRegionInfo hri = region.getRegionInfo(); + HTableDescriptor htd = new HTableDescriptor(); + htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f1"))); + boolean isMetaRegion = false; + long start = System.currentTimeMillis(); + for (int i=0; i files = HLog.getSplitEditFilesSorted(fs, regiondir); + assertEquals(7, files.size()); + assertEquals(files.pollFirst().getName(), first); + assertEquals(files.pollLast().getName(), last); + assertEquals(files.pollFirst().getName(), + HLogSplitter + .formatRecoveredEditsFileName(0)); + assertEquals(files.pollFirst().getName(), + HLogSplitter + .formatRecoveredEditsFileName(1)); + assertEquals(files.pollFirst().getName(), + HLogSplitter + .formatRecoveredEditsFileName(2)); + assertEquals(files.pollFirst().getName(), + HLogSplitter + .formatRecoveredEditsFileName(11)); + } + + private void createFile(final FileSystem fs, final Path testdir, + final String name) + throws IOException { + FSDataOutputStream fdos = fs.create(new Path(testdir, name), true); + fdos.close(); + } + + @Test + public void testRegionEntryBuffer() throws Exception { + HLogSplitter.RegionEntryBuffer reb = new HLogSplitter.RegionEntryBuffer( + TEST_TABLE, TEST_REGION); + assertEquals(0, reb.heapSize()); + + reb.appendEntry(createTestLogEntry(1)); + assertTrue(reb.heapSize() > 0); + } + + @Test + public void testEntrySink() throws Exception { + Configuration conf = new Configuration(); + HLogSplitter splitter = HLogSplitter.createLogSplitter( + conf, mock(Path.class), mock(Path.class), mock(Path.class), + mock(FileSystem.class)); + + EntryBuffers sink = splitter.new EntryBuffers(1*1024*1024); + for (int i = 0; i < 1000; i++) { + HLog.Entry entry = createTestLogEntry(i); + sink.appendEntry(entry); + } + + assertTrue(sink.totalBuffered > 0); + long amountInChunk = sink.totalBuffered; + // Get a chunk + RegionEntryBuffer chunk = sink.getChunkToWrite(); + assertEquals(chunk.heapSize(), amountInChunk); + + // Make sure it got marked that a thread is "working on this" + assertTrue(sink.isRegionCurrentlyWriting(TEST_REGION)); + + // Insert some more entries + for (int i = 0; i < 500; i++) { + HLog.Entry entry = createTestLogEntry(i); + sink.appendEntry(entry); + } + // Asking for another chunk shouldn't work since the first one + // is still writing + assertNull(sink.getChunkToWrite()); + + // If we say we're done writing the first chunk, then we should be able + // to get the second + sink.doneWriting(chunk); + + RegionEntryBuffer chunk2 = sink.getChunkToWrite(); + assertNotNull(chunk2); + assertNotSame(chunk, chunk2); + long amountInChunk2 = sink.totalBuffered; + // The second chunk had fewer rows than the first + assertTrue(amountInChunk2 < amountInChunk); + + sink.doneWriting(chunk2); + assertEquals(0, sink.totalBuffered); + } + + private HLog.Entry createTestLogEntry(int i) { + long seq = i; + long now = i * 1000; + + WALEdit edit = new WALEdit(); + edit.add(KeyValueTestUtil.create("row", "fam", "qual", 1234, "val")); + HLogKey key = new HLogKey(TEST_REGION, TEST_TABLE, seq, now, + HConstants.DEFAULT_CLUSTER_ID); + HLog.Entry entry = new HLog.Entry(key, edit); + return entry; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java new file mode 100644 index 0000000..d3e3a53 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java @@ -0,0 +1,1403 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry; +import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CancelableProgressable; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException; +import org.apache.hadoop.ipc.RemoteException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; + +/** + * Testing {@link HLog} splitting code. + */ +@Category(LargeTests.class) +public class TestHLogSplit { + + private final static Log LOG = LogFactory.getLog(TestHLogSplit.class); + + private Configuration conf; + private FileSystem fs; + + protected final static HBaseTestingUtility + TEST_UTIL = new HBaseTestingUtility(); + + + private static final Path hbaseDir = new Path("/hbase"); + private static final Path hlogDir = new Path(hbaseDir, "hlog"); + private static final Path oldLogDir = new Path(hbaseDir, "hlog.old"); + private static final Path corruptDir = new Path(hbaseDir, ".corrupt"); + + private static final int NUM_WRITERS = 10; + private static final int ENTRIES = 10; // entries per writer per region + + private HLog.Writer[] writer = new HLog.Writer[NUM_WRITERS]; + private long seq = 0; + private static final byte[] TABLE_NAME = "t1".getBytes(); + private static final byte[] FAMILY = "f1".getBytes(); + private static final byte[] QUALIFIER = "q1".getBytes(); + private static final byte[] VALUE = "v1".getBytes(); + private static final String HLOG_FILE_PREFIX = "hlog.dat."; + private static List regions; + private static final String HBASE_SKIP_ERRORS = "hbase.hlog.split.skip.errors"; + private static final Path tabledir = + new Path(hbaseDir, Bytes.toString(TABLE_NAME)); + + static enum Corruptions { + INSERT_GARBAGE_ON_FIRST_LINE, + INSERT_GARBAGE_IN_THE_MIDDLE, + APPEND_GARBAGE, + TRUNCATE, + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setStrings("hbase.rootdir", hbaseDir.toString()); + TEST_UTIL.getConfiguration().setClass("hbase.regionserver.hlog.writer.impl", + InstrumentedSequenceFileLogWriter.class, HLog.Writer.class); + TEST_UTIL.getConfiguration().setBoolean("dfs.support.broken.append", true); + TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); + TEST_UTIL.startMiniDFSCluster(2); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniDFSCluster(); + } + + @Before + public void setUp() throws Exception { + flushToConsole("Cleaning up cluster for new test\n" + + "--------------------------"); + conf = TEST_UTIL.getConfiguration(); + fs = TEST_UTIL.getDFSCluster().getFileSystem(); + FileStatus[] entries = fs.listStatus(new Path("/")); + flushToConsole("Num entries in /:" + entries.length); + for (FileStatus dir : entries){ + assertTrue("Deleting " + dir.getPath(), + fs.delete(dir.getPath(), true)); + } + // create the HLog directory because recursive log creates are not allowed + fs.mkdirs(hlogDir); + seq = 0; + regions = new ArrayList(); + Collections.addAll(regions, "bbb", "ccc"); + InstrumentedSequenceFileLogWriter.activateFailure = false; + } + + @After + public void tearDown() throws Exception { + } + + /** + * @throws IOException + * @see https://issues.apache.org/jira/browse/HBASE-3020 + */ + @Test + public void testRecoveredEditsPathForMeta() throws IOException { + FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); + byte [] encoded = HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); + Path tdir = new Path(hbaseDir, Bytes.toString(HConstants.META_TABLE_NAME)); + Path regiondir = new Path(tdir, + HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()); + fs.mkdirs(regiondir); + long now = System.currentTimeMillis(); + HLog.Entry entry = + new HLog.Entry(new HLogKey(encoded, + HConstants.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), + new WALEdit()); + Path p = HLogSplitter.getRegionSplitEditsPath(fs, entry, hbaseDir, true); + String parentOfParent = p.getParent().getParent().getName(); + assertEquals(parentOfParent, HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()); + } + + @Test(expected = OrphanHLogAfterSplitException.class) + public void testSplitFailsIfNewHLogGetsCreatedAfterSplitStarted() + throws IOException { + AtomicBoolean stop = new AtomicBoolean(false); + + assertFalse("Previous test should clean up table dir", + fs.exists(new Path("/hbase/t1"))); + + generateHLogs(-1); + + CountDownLatch latch = new CountDownLatch(1); + try { + (new ZombieNewLogWriterRegionServer(latch, stop)).start(); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, hbaseDir, hlogDir, oldLogDir, + fs); + logSplitter.splitLog(latch); + } finally { + stop.set(true); + } + } + + @Test + public void testSplitPreservesEdits() throws IOException{ + final String REGION = "region__1"; + regions.removeAll(regions); + regions.add(REGION); + + generateHLogs(1, 10, -1); + fs.initialize(fs.getUri(), conf); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + + Path originalLog = (fs.listStatus(oldLogDir))[0].getPath(); + Path splitLog = getLogForRegion(hbaseDir, TABLE_NAME, REGION); + + assertEquals("edits differ after split", true, logsAreEqual(originalLog, splitLog)); + } + + + @Test + public void testEmptyLogFiles() throws IOException { + + injectEmptyFile(".empty", true); + generateHLogs(Integer.MAX_VALUE); + injectEmptyFile("empty", true); + + // make fs act as a different client now + // initialize will create a new DFSClient with a new client ID + fs.initialize(fs.getUri(), conf); + + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + + + for (String region : regions) { + Path logfile = getLogForRegion(hbaseDir, TABLE_NAME, region); + assertEquals(NUM_WRITERS * ENTRIES, countHLog(logfile, fs, conf)); + } + + } + + + @Test + public void testEmptyOpenLogFiles() throws IOException { + injectEmptyFile(".empty", false); + generateHLogs(Integer.MAX_VALUE); + injectEmptyFile("empty", false); + + // make fs act as a different client now + // initialize will create a new DFSClient with a new client ID + fs.initialize(fs.getUri(), conf); + + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + + for (String region : regions) { + Path logfile = getLogForRegion(hbaseDir, TABLE_NAME, region); + assertEquals(NUM_WRITERS * ENTRIES, countHLog(logfile, fs, conf)); + } + } + + @Test + public void testOpenZeroLengthReportedFileButWithDataGetsSplit() throws IOException { + // generate logs but leave hlog.dat.5 open. + generateHLogs(5); + + fs.initialize(fs.getUri(), conf); + + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + + for (String region : regions) { + Path logfile = getLogForRegion(hbaseDir, TABLE_NAME, region); + assertEquals(NUM_WRITERS * ENTRIES, countHLog(logfile, fs, conf)); + } + + + } + + + @Test + public void testTralingGarbageCorruptionFileSkipErrorsPasses() throws IOException { + conf.setBoolean(HBASE_SKIP_ERRORS, true); + generateHLogs(Integer.MAX_VALUE); + corruptHLog(new Path(hlogDir, HLOG_FILE_PREFIX + "5"), + Corruptions.APPEND_GARBAGE, true, fs); + fs.initialize(fs.getUri(), conf); + + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + for (String region : regions) { + Path logfile = getLogForRegion(hbaseDir, TABLE_NAME, region); + assertEquals(NUM_WRITERS * ENTRIES, countHLog(logfile, fs, conf)); + } + + + } + + @Test + public void testFirstLineCorruptionLogFileSkipErrorsPasses() throws IOException { + conf.setBoolean(HBASE_SKIP_ERRORS, true); + generateHLogs(Integer.MAX_VALUE); + corruptHLog(new Path(hlogDir, HLOG_FILE_PREFIX + "5"), + Corruptions.INSERT_GARBAGE_ON_FIRST_LINE, true, fs); + fs.initialize(fs.getUri(), conf); + + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + for (String region : regions) { + Path logfile = getLogForRegion(hbaseDir, TABLE_NAME, region); + assertEquals((NUM_WRITERS - 1) * ENTRIES, countHLog(logfile, fs, conf)); + } + + + } + + + @Test + public void testMiddleGarbageCorruptionSkipErrorsReadsHalfOfFile() throws IOException { + conf.setBoolean(HBASE_SKIP_ERRORS, true); + generateHLogs(Integer.MAX_VALUE); + corruptHLog(new Path(hlogDir, HLOG_FILE_PREFIX + "5"), + Corruptions.INSERT_GARBAGE_IN_THE_MIDDLE, false, fs); + fs.initialize(fs.getUri(), conf); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + + for (String region : regions) { + Path logfile = getLogForRegion(hbaseDir, TABLE_NAME, region); + // the entries in the original logs are alternating regions + // considering the sequence file header, the middle corruption should + // affect at least half of the entries + int goodEntries = (NUM_WRITERS - 1) * ENTRIES; + int firstHalfEntries = (int) Math.ceil(ENTRIES / 2) - 1; + assertTrue("The file up to the corrupted area hasn't been parsed", + goodEntries + firstHalfEntries <= countHLog(logfile, fs, conf)); + } + } + + @Test + public void testCorruptedFileGetsArchivedIfSkipErrors() throws IOException { + conf.setBoolean(HBASE_SKIP_ERRORS, true); + Class backupClass = conf.getClass("hbase.regionserver.hlog.reader.impl", + Reader.class); + InstrumentedSequenceFileLogWriter.activateFailure = false; + HLog.resetLogReaderClass(); + + try { + Path c1 = new Path(hlogDir, HLOG_FILE_PREFIX + "0"); + conf.setClass("hbase.regionserver.hlog.reader.impl", + FaultySequenceFileLogReader.class, HLog.Reader.class); + for (FaultySequenceFileLogReader.FailureType failureType : FaultySequenceFileLogReader.FailureType.values()) { + conf.set("faultysequencefilelogreader.failuretype", failureType.name()); + generateHLogs(1, ENTRIES, -1); + fs.initialize(fs.getUri(), conf); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + FileStatus[] archivedLogs = fs.listStatus(corruptDir); + assertEquals("expected a different file", c1.getName(), archivedLogs[0] + .getPath().getName()); + assertEquals(archivedLogs.length, 1); + fs.delete(new Path(oldLogDir, HLOG_FILE_PREFIX + "0"), false); + } + } finally { + conf.setClass("hbase.regionserver.hlog.reader.impl", backupClass, + Reader.class); + HLog.resetLogReaderClass(); + } + } + + @Test(expected = IOException.class) + public void testTrailingGarbageCorruptionLogFileSkipErrorsFalseThrows() + throws IOException { + conf.setBoolean(HBASE_SKIP_ERRORS, false); + Class backupClass = conf.getClass("hbase.regionserver.hlog.reader.impl", + Reader.class); + InstrumentedSequenceFileLogWriter.activateFailure = false; + HLog.resetLogReaderClass(); + + try { + conf.setClass("hbase.regionserver.hlog.reader.impl", + FaultySequenceFileLogReader.class, HLog.Reader.class); + conf.set("faultysequencefilelogreader.failuretype", FaultySequenceFileLogReader.FailureType.BEGINNING.name()); + generateHLogs(Integer.MAX_VALUE); + fs.initialize(fs.getUri(), conf); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + } finally { + conf.setClass("hbase.regionserver.hlog.reader.impl", backupClass, + Reader.class); + HLog.resetLogReaderClass(); + } + + } + + @Test + public void testCorruptedLogFilesSkipErrorsFalseDoesNotTouchLogs() + throws IOException { + conf.setBoolean(HBASE_SKIP_ERRORS, false); + Class backupClass = conf.getClass("hbase.regionserver.hlog.reader.impl", + Reader.class); + InstrumentedSequenceFileLogWriter.activateFailure = false; + HLog.resetLogReaderClass(); + + try { + conf.setClass("hbase.regionserver.hlog.reader.impl", + FaultySequenceFileLogReader.class, HLog.Reader.class); + conf.set("faultysequencefilelogreader.failuretype", FaultySequenceFileLogReader.FailureType.BEGINNING.name()); + generateHLogs(-1); + fs.initialize(fs.getUri(), conf); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + try { + logSplitter.splitLog(); + } catch (IOException e) { + assertEquals( + "if skip.errors is false all files should remain in place", + NUM_WRITERS, fs.listStatus(hlogDir).length); + } + } finally { + conf.setClass("hbase.regionserver.hlog.reader.impl", backupClass, + Reader.class); + HLog.resetLogReaderClass(); + } + + } + + @Test + public void testEOFisIgnored() throws IOException { + conf.setBoolean(HBASE_SKIP_ERRORS, false); + + final String REGION = "region__1"; + regions.removeAll(regions); + regions.add(REGION); + + int entryCount = 10; + Path c1 = new Path(hlogDir, HLOG_FILE_PREFIX + "0"); + generateHLogs(1, entryCount, -1); + corruptHLog(c1, Corruptions.TRUNCATE, true, fs); + + fs.initialize(fs.getUri(), conf); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + + Path originalLog = (fs.listStatus(oldLogDir))[0].getPath(); + Path splitLog = getLogForRegion(hbaseDir, TABLE_NAME, REGION); + + int actualCount = 0; + HLog.Reader in = HLog.getReader(fs, splitLog, conf); + HLog.Entry entry; + while ((entry = in.next()) != null) ++actualCount; + assertEquals(entryCount-1, actualCount); + + // should not have stored the EOF files as corrupt + FileStatus[] archivedLogs = fs.listStatus(corruptDir); + assertEquals(archivedLogs.length, 0); + } + + @Test + public void testLogsGetArchivedAfterSplit() throws IOException { + conf.setBoolean(HBASE_SKIP_ERRORS, false); + + generateHLogs(-1); + + fs.initialize(fs.getUri(), conf); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + + FileStatus[] archivedLogs = fs.listStatus(oldLogDir); + + assertEquals("wrong number of files in the archive log", NUM_WRITERS, archivedLogs.length); + } + + @Test + public void testSplit() throws IOException { + generateHLogs(-1); + fs.initialize(fs.getUri(), conf); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + + for (String region : regions) { + Path logfile = getLogForRegion(hbaseDir, TABLE_NAME, region); + assertEquals(NUM_WRITERS * ENTRIES, countHLog(logfile, fs, conf)); + + } + } + + @Test + public void testLogDirectoryShouldBeDeletedAfterSuccessfulSplit() + throws IOException { + generateHLogs(-1); + fs.initialize(fs.getUri(), conf); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + FileStatus [] statuses = null; + try { + statuses = fs.listStatus(hlogDir); + if (statuses != null) { + Assert.fail("Files left in log dir: " + + Joiner.on(",").join(FileUtil.stat2Paths(statuses))); + } + } catch (FileNotFoundException e) { + // hadoop 0.21 throws FNFE whereas hadoop 0.20 returns null + } + } +/* DISABLED for now. TODO: HBASE-2645 + @Test + public void testLogCannotBeWrittenOnceParsed() throws IOException { + AtomicLong counter = new AtomicLong(0); + AtomicBoolean stop = new AtomicBoolean(false); + generateHLogs(9); + fs.initialize(fs.getUri(), conf); + + Thread zombie = new ZombieLastLogWriterRegionServer(writer[9], counter, stop); + + + + try { + zombie.start(); + + HLog.splitLog(hbaseDir, hlogDir, oldLogDir, fs, conf); + + Path logfile = getLogForRegion(hbaseDir, TABLE_NAME, "juliet"); + + // It's possible that the writer got an error while appending and didn't count it + // however the entry will in fact be written to file and split with the rest + long numberOfEditsInRegion = countHLog(logfile, fs, conf); + assertTrue("The log file could have at most 1 extra log entry, but " + + "can't have less. Zombie could write "+counter.get() +" and logfile had only"+ numberOfEditsInRegion+" " + logfile, counter.get() == numberOfEditsInRegion || + counter.get() + 1 == numberOfEditsInRegion); + } finally { + stop.set(true); + } + } +*/ + + @Test + public void testSplitWillNotTouchLogsIfNewHLogGetsCreatedAfterSplitStarted() + throws IOException { + AtomicBoolean stop = new AtomicBoolean(false); + generateHLogs(-1); + fs.initialize(fs.getUri(), conf); + CountDownLatch latch = new CountDownLatch(1); + Thread zombie = new ZombieNewLogWriterRegionServer(latch, stop); + + List splits = null; + try { + zombie.start(); + try { + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + splits = logSplitter.splitLog(latch); + } catch (IOException ex) { + /* expected */ + LOG.warn("testSplitWillNotTouchLogsIfNewHLogGetsCreatedAfterSplitStarted", ex); + } + FileStatus[] files = fs.listStatus(hlogDir); + if (files == null) fail("no files in " + hlogDir + " with splits " + splits); + int logFilesNumber = files.length; + + assertEquals("Log files should not be archived if there's an extra file after split", + NUM_WRITERS + 1, logFilesNumber); + } finally { + stop.set(true); + } + + } + + + + @Test(expected = IOException.class) + public void testSplitWillFailIfWritingToRegionFails() throws Exception { + //leave 5th log open so we could append the "trap" + generateHLogs(4); + + fs.initialize(fs.getUri(), conf); + + String region = "break"; + Path regiondir = new Path(tabledir, region); + fs.mkdirs(regiondir); + + InstrumentedSequenceFileLogWriter.activateFailure = false; + appendEntry(writer[4], TABLE_NAME, Bytes.toBytes(region), + ("r" + 999).getBytes(), FAMILY, QUALIFIER, VALUE, 0); + writer[4].close(); + + try { + InstrumentedSequenceFileLogWriter.activateFailure = true; + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + + } catch (IOException e) { + assertEquals("This exception is instrumented and should only be thrown for testing", e.getMessage()); + throw e; + } finally { + InstrumentedSequenceFileLogWriter.activateFailure = false; + } + } + + + // @Test TODO this test has been disabled since it was created! + // It currently fails because the second split doesn't output anything + // -- because there are no region dirs after we move aside the first + // split result + public void testSplittingLargeNumberOfRegionsConsistency() throws IOException { + + regions.removeAll(regions); + for (int i=0; i<100; i++) { + regions.add("region__"+i); + } + + generateHLogs(1, 100, -1); + fs.initialize(fs.getUri(), conf); + + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + fs.rename(oldLogDir, hlogDir); + Path firstSplitPath = new Path(hbaseDir, Bytes.toString(TABLE_NAME) + ".first"); + Path splitPath = new Path(hbaseDir, Bytes.toString(TABLE_NAME)); + fs.rename(splitPath, + firstSplitPath); + + + fs.initialize(fs.getUri(), conf); + logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + + assertEquals(0, compareHLogSplitDirs(firstSplitPath, splitPath)); + } + + @Test + public void testSplitDeletedRegion() throws IOException { + regions.removeAll(regions); + String region = "region_that_splits"; + regions.add(region); + + generateHLogs(1); + + fs.initialize(fs.getUri(), conf); + + Path regiondir = new Path(tabledir, region); + fs.delete(regiondir, true); + + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, hlogDir, oldLogDir, fs); + logSplitter.splitLog(); + + assertFalse(fs.exists(regiondir)); + } + + @Test + public void testIOEOnOutputThread() throws Exception { + conf.setBoolean(HBASE_SKIP_ERRORS, false); + + generateHLogs(-1); + + fs.initialize(fs.getUri(), conf); + // Set up a splitter that will throw an IOE on the output side + HLogSplitter logSplitter = new HLogSplitter( + conf, hbaseDir, hlogDir, oldLogDir, fs) { + protected HLog.Writer createWriter(FileSystem fs, Path logfile, Configuration conf) + throws IOException { + HLog.Writer mockWriter = Mockito.mock(HLog.Writer.class); + Mockito.doThrow(new IOException("Injected")).when(mockWriter).append(Mockito.any()); + return mockWriter; + + } + }; + try { + logSplitter.splitLog(); + fail("Didn't throw!"); + } catch (IOException ioe) { + assertTrue(ioe.toString().contains("Injected")); + } + } + + // Test for HBASE-3412 + @Test + public void testMovedHLogDuringRecovery() throws Exception { + generateHLogs(-1); + + fs.initialize(fs.getUri(), conf); + + // This partial mock will throw LEE for every file simulating + // files that were moved + FileSystem spiedFs = Mockito.spy(fs); + // The "File does not exist" part is very important, + // that's how it comes out of HDFS + Mockito.doThrow(new LeaseExpiredException("Injected: File does not exist")). + when(spiedFs).append(Mockito.any()); + + HLogSplitter logSplitter = new HLogSplitter( + conf, hbaseDir, hlogDir, oldLogDir, spiedFs); + + try { + logSplitter.splitLog(); + assertEquals(NUM_WRITERS, fs.listStatus(oldLogDir).length); + assertFalse(fs.exists(hlogDir)); + } catch (IOException e) { + fail("There shouldn't be any exception but: " + e.toString()); + } + } + + /** + * Test log split process with fake data and lots of edits to trigger threading + * issues. + */ + @Test + public void testThreading() throws Exception { + doTestThreading(20000, 128*1024*1024, 0); + } + + /** + * Test blocking behavior of the log split process if writers are writing slower + * than the reader is reading. + */ + @Test + public void testThreadingSlowWriterSmallBuffer() throws Exception { + doTestThreading(200, 1024, 50); + } + + /** + * Sets up a log splitter with a mock reader and writer. The mock reader generates + * a specified number of edits spread across 5 regions. The mock writer optionally + * sleeps for each edit it is fed. + * * + * After the split is complete, verifies that the statistics show the correct number + * of edits output into each region. + * + * @param numFakeEdits number of fake edits to push through pipeline + * @param bufferSize size of in-memory buffer + * @param writerSlowness writer threads will sleep this many ms per edit + */ + private void doTestThreading(final int numFakeEdits, + final int bufferSize, + final int writerSlowness) throws Exception { + + Configuration localConf = new Configuration(conf); + localConf.setInt("hbase.regionserver.hlog.splitlog.buffersize", bufferSize); + + // Create a fake log file (we'll override the reader to produce a stream of edits) + FSDataOutputStream out = fs.create(new Path(hlogDir, HLOG_FILE_PREFIX + ".fake")); + out.close(); + + // Make region dirs for our destination regions so the output doesn't get skipped + final List regions = ImmutableList.of("r0", "r1", "r2", "r3", "r4"); + makeRegionDirs(fs, regions); + + // Create a splitter that reads and writes the data without touching disk + HLogSplitter logSplitter = new HLogSplitter( + localConf, hbaseDir, hlogDir, oldLogDir, fs) { + + /* Produce a mock writer that doesn't write anywhere */ + protected HLog.Writer createWriter(FileSystem fs, Path logfile, Configuration conf) + throws IOException { + HLog.Writer mockWriter = Mockito.mock(HLog.Writer.class); + Mockito.doAnswer(new Answer() { + int expectedIndex = 0; + + @Override + public Void answer(InvocationOnMock invocation) { + if (writerSlowness > 0) { + try { + Thread.sleep(writerSlowness); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + } + HLog.Entry entry = (Entry) invocation.getArguments()[0]; + WALEdit edit = entry.getEdit(); + List keyValues = edit.getKeyValues(); + assertEquals(1, keyValues.size()); + KeyValue kv = keyValues.get(0); + + // Check that the edits come in the right order. + assertEquals(expectedIndex, Bytes.toInt(kv.getRow())); + expectedIndex++; + return null; + } + }).when(mockWriter).append(Mockito.any()); + return mockWriter; + } + + + /* Produce a mock reader that generates fake entries */ + protected Reader getReader(FileSystem fs, Path curLogFile, Configuration conf) + throws IOException { + Reader mockReader = Mockito.mock(Reader.class); + Mockito.doAnswer(new Answer() { + int index = 0; + + @Override + public HLog.Entry answer(InvocationOnMock invocation) throws Throwable { + if (index >= numFakeEdits) return null; + + // Generate r0 through r4 in round robin fashion + int regionIdx = index % regions.size(); + byte region[] = new byte[] {(byte)'r', (byte) (0x30 + regionIdx)}; + + HLog.Entry ret = createTestEntry(TABLE_NAME, region, + Bytes.toBytes((int)(index / regions.size())), + FAMILY, QUALIFIER, VALUE, index); + index++; + return ret; + } + }).when(mockReader).next(); + return mockReader; + } + }; + + logSplitter.splitLog(); + + // Verify number of written edits per region + + Map outputCounts = logSplitter.getOutputCounts(); + for (Map.Entry entry : outputCounts.entrySet()) { + LOG.info("Got " + entry.getValue() + " output edits for region " + + Bytes.toString(entry.getKey())); + + assertEquals((long)entry.getValue(), numFakeEdits / regions.size()); + } + assertEquals(regions.size(), outputCounts.size()); + } + + // HBASE-2312: tests the case where a RegionServer enters a GC pause, + // comes back online after the master declared it dead and started to split. + // Want log rolling after a master split to fail + @Test + @Ignore("Need HADOOP-6886, HADOOP-6840, & HDFS-617 for this. HDFS 0.20.205.1+ should have this") + public void testLogRollAfterSplitStart() throws IOException { + // set flush interval to a large number so it doesn't interrupt us + final String F_INTERVAL = "hbase.regionserver.optionallogflushinterval"; + long oldFlushInterval = conf.getLong(F_INTERVAL, 1000); + conf.setLong(F_INTERVAL, 1000*1000*100); + HLog log = null; + Path thisTestsDir = new Path(hbaseDir, "testLogRollAfterSplitStart"); + + try { + // put some entries in an HLog + byte [] tableName = Bytes.toBytes(this.getClass().getName()); + HRegionInfo regioninfo = new HRegionInfo(tableName, + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); + log = new HLog(fs, thisTestsDir, oldLogDir, conf); + final int total = 20; + for (int i = 0; i < total; i++) { + WALEdit kvs = new WALEdit(); + kvs.add(new KeyValue(Bytes.toBytes(i), tableName, tableName)); + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor("column")); + log.append(regioninfo, tableName, kvs, System.currentTimeMillis(), htd); + } + // Send the data to HDFS datanodes and close the HDFS writer + log.sync(); + log.cleanupCurrentWriter(log.getFilenum()); + + /* code taken from ProcessServerShutdown.process() + * handles RS shutdowns (as observed by the Master) + */ + // rename the directory so a rogue RS doesn't create more HLogs + Path rsSplitDir = new Path(thisTestsDir.getParent(), + thisTestsDir.getName() + "-splitting"); + fs.rename(thisTestsDir, rsSplitDir); + LOG.debug("Renamed region directory: " + rsSplitDir); + + // Process the old log files + HLogSplitter splitter = HLogSplitter.createLogSplitter(conf, + hbaseDir, rsSplitDir, oldLogDir, fs); + splitter.splitLog(); + + // Now, try to roll the HLog and verify failure + try { + log.rollWriter(); + Assert.fail("rollWriter() did not throw any exception."); + } catch (IOException ioe) { + if (ioe.getCause().getMessage().contains("FileNotFound")) { + LOG.info("Got the expected exception: ", ioe.getCause()); + } else { + Assert.fail("Unexpected exception: " + ioe); + } + } + } finally { + conf.setLong(F_INTERVAL, oldFlushInterval); + if (log != null) { + log.close(); + } + if (fs.exists(thisTestsDir)) { + fs.delete(thisTestsDir, true); + } + } + } + + /** + * This thread will keep writing to the file after the split process has started + * It simulates a region server that was considered dead but woke up and wrote + * some more to he last log entry + */ + class ZombieLastLogWriterRegionServer extends Thread { + AtomicLong editsCount; + AtomicBoolean stop; + Path log; + HLog.Writer lastLogWriter; + public ZombieLastLogWriterRegionServer(HLog.Writer writer, AtomicLong counter, AtomicBoolean stop) { + this.stop = stop; + this.editsCount = counter; + this.lastLogWriter = writer; + } + + @Override + public void run() { + if (stop.get()){ + return; + } + flushToConsole("starting"); + while (true) { + try { + String region = "juliet"; + + fs.mkdirs(new Path(new Path(hbaseDir, region), region)); + appendEntry(lastLogWriter, TABLE_NAME, region.getBytes(), + ("r" + editsCount).getBytes(), FAMILY, QUALIFIER, VALUE, 0); + lastLogWriter.sync(); + editsCount.incrementAndGet(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + // + } + + + } catch (IOException ex) { + if (ex instanceof RemoteException) { + flushToConsole("Juliet: got RemoteException " + + ex.getMessage() + " while writing " + (editsCount.get() + 1)); + break; + } else { + assertTrue("Failed to write " + editsCount.get(), false); + } + + } + } + + + } + } + + /** + * This thread will keep adding new log files + * It simulates a region server that was considered dead but woke up and wrote + * some more to a new hlog + */ + class ZombieNewLogWriterRegionServer extends Thread { + AtomicBoolean stop; + CountDownLatch latch; + public ZombieNewLogWriterRegionServer(CountDownLatch latch, AtomicBoolean stop) { + super("ZombieNewLogWriterRegionServer"); + this.latch = latch; + this.stop = stop; + } + + @Override + public void run() { + if (stop.get()) { + return; + } + Path tableDir = new Path(hbaseDir, new String(TABLE_NAME)); + Path regionDir = new Path(tableDir, regions.get(0)); + Path recoveredEdits = new Path(regionDir, HLogSplitter.RECOVERED_EDITS); + String region = "juliet"; + Path julietLog = new Path(hlogDir, HLOG_FILE_PREFIX + ".juliet"); + try { + + while (!fs.exists(recoveredEdits) && !stop.get()) { + LOG.info("Juliet: split not started, sleeping a bit..."); + Threads.sleep(10); + } + + fs.mkdirs(new Path(tableDir, region)); + HLog.Writer writer = HLog.createWriter(fs, + julietLog, conf); + appendEntry(writer, "juliet".getBytes(), ("juliet").getBytes(), + ("r").getBytes(), FAMILY, QUALIFIER, VALUE, 0); + writer.close(); + LOG.info("Juliet file creator: created file " + julietLog); + latch.countDown(); + } catch (IOException e1) { + LOG.error("Failed to create file " + julietLog, e1); + assertTrue("Failed to create file " + julietLog, false); + } + } + } + + private CancelableProgressable reporter = new CancelableProgressable() { + int count = 0; + + @Override + public boolean progress() { + count++; + LOG.debug("progress = " + count); + return true; + } + }; + + @Test + public void testSplitLogFileWithOneRegion() throws IOException { + LOG.info("testSplitLogFileWithOneRegion"); + final String REGION = "region__1"; + regions.removeAll(regions); + regions.add(REGION); + + + generateHLogs(1, 10, -1); + FileStatus logfile = fs.listStatus(hlogDir)[0]; + fs.initialize(fs.getUri(), conf); + HLogSplitter.splitLogFile(hbaseDir, logfile, fs, conf, reporter); + HLogSplitter.finishSplitLogFile(hbaseDir, oldLogDir, logfile.getPath() + .toString(), conf); + + + Path originalLog = (fs.listStatus(oldLogDir))[0].getPath(); + Path splitLog = getLogForRegion(hbaseDir, TABLE_NAME, REGION); + + + assertEquals(true, logsAreEqual(originalLog, splitLog)); + } + + @Test + public void testSplitLogFileDeletedRegionDir() + throws IOException { + LOG.info("testSplitLogFileDeletedRegionDir"); + final String REGION = "region__1"; + regions.removeAll(regions); + regions.add(REGION); + + + generateHLogs(1, 10, -1); + FileStatus logfile = fs.listStatus(hlogDir)[0]; + fs.initialize(fs.getUri(), conf); + + Path regiondir = new Path(tabledir, REGION); + LOG.info("Region directory is" + regiondir); + fs.delete(regiondir, true); + + HLogSplitter.splitLogFile(hbaseDir, logfile, fs, conf, reporter); + HLogSplitter.finishSplitLogFile(hbaseDir, oldLogDir, logfile.getPath() + .toString(), conf); + + assertTrue(!fs.exists(regiondir)); + assertTrue(true); + } + + + + @Test + public void testSplitLogFileEmpty() throws IOException { + LOG.info("testSplitLogFileEmpty"); + injectEmptyFile(".empty", true); + FileStatus logfile = fs.listStatus(hlogDir)[0]; + + fs.initialize(fs.getUri(), conf); + + HLogSplitter.splitLogFile(hbaseDir, logfile, fs, conf, reporter); + HLogSplitter.finishSplitLogFile(hbaseDir, oldLogDir, logfile.getPath() + .toString(), conf); + Path tdir = HTableDescriptor.getTableDir(hbaseDir, TABLE_NAME); + assertFalse(fs.exists(tdir)); + + assertEquals(0, countHLog(fs.listStatus(oldLogDir)[0].getPath(), fs, conf)); + } + + @Test + public void testSplitLogFileMultipleRegions() throws IOException { + LOG.info("testSplitLogFileMultipleRegions"); + generateHLogs(1, 10, -1); + FileStatus logfile = fs.listStatus(hlogDir)[0]; + fs.initialize(fs.getUri(), conf); + + HLogSplitter.splitLogFile(hbaseDir, logfile, fs, conf, reporter); + HLogSplitter.finishSplitLogFile(hbaseDir, oldLogDir, logfile.getPath() + .toString(), conf); + for (String region : regions) { + Path recovered = getLogForRegion(hbaseDir, TABLE_NAME, region); + assertEquals(10, countHLog(recovered, fs, conf)); + } + } + + @Test + public void testSplitLogFileFirstLineCorruptionLog() + throws IOException { + conf.setBoolean(HBASE_SKIP_ERRORS, true); + generateHLogs(1, 10, -1); + FileStatus logfile = fs.listStatus(hlogDir)[0]; + + corruptHLog(logfile.getPath(), + Corruptions.INSERT_GARBAGE_ON_FIRST_LINE, true, fs); + + fs.initialize(fs.getUri(), conf); + HLogSplitter.splitLogFile(hbaseDir, logfile, fs, conf, reporter); + HLogSplitter.finishSplitLogFile(hbaseDir, oldLogDir, logfile.getPath() + .toString(), conf); + + final Path corruptDir = new Path(conf.get(HConstants.HBASE_DIR), conf.get( + "hbase.regionserver.hlog.splitlog.corrupt.dir", ".corrupt")); + assertEquals(1, fs.listStatus(corruptDir).length); + } + + /** + * @throws IOException + * @see https://issues.apache.org/jira/browse/HBASE-4862 + */ + @Test + public void testConcurrentSplitLogAndReplayRecoverEdit() throws IOException { + LOG.info("testConcurrentSplitLogAndReplayRecoverEdit"); + // Generate hlogs for our destination region + String regionName = "r0"; + final Path regiondir = new Path(tabledir, regionName); + regions = new ArrayList(); + regions.add(regionName); + generateHLogs(-1); + + HLogSplitter logSplitter = new HLogSplitter( + conf, hbaseDir, hlogDir, oldLogDir, fs) { + protected HLog.Writer createWriter(FileSystem fs, Path logfile, Configuration conf) + throws IOException { + HLog.Writer writer = HLog.createWriter(fs, logfile, conf); + // After creating writer, simulate region's + // replayRecoveredEditsIfAny() which gets SplitEditFiles of this + // region and delete them, excluding files with '.temp' suffix. + NavigableSet files = HLog.getSplitEditFilesSorted(this.fs, + regiondir); + if (files != null && !files.isEmpty()) { + for (Path file : files) { + if (!this.fs.delete(file, false)) { + LOG.error("Failed delete of " + file); + } else { + LOG.debug("Deleted recovered.edits file=" + file); + } + } + } + return writer; + } + }; + try{ + logSplitter.splitLog(); + } catch (IOException e) { + LOG.info(e); + Assert.fail("Throws IOException when spliting " + + "log, it is most likely because writing file does not " + + "exist which is caused by concurrent replayRecoveredEditsIfAny()"); + } + if (fs.exists(corruptDir)) { + if (fs.listStatus(corruptDir).length > 0) { + Assert.fail("There are some corrupt logs, " + + "it is most likely caused by concurrent replayRecoveredEditsIfAny()"); + } + } + } + + private void flushToConsole(String s) { + System.out.println(s); + System.out.flush(); + } + + + private void generateHLogs(int leaveOpen) throws IOException { + generateHLogs(NUM_WRITERS, ENTRIES, leaveOpen); + } + + private void makeRegionDirs(FileSystem fs, List regions) throws IOException { + for (String region : regions) { + flushToConsole("Creating dir for region " + region); + fs.mkdirs(new Path(tabledir, region)); + } + } + + private void generateHLogs(int writers, int entries, int leaveOpen) throws IOException { + makeRegionDirs(fs, regions); + fs.mkdirs(hlogDir); + for (int i = 0; i < writers; i++) { + writer[i] = HLog.createWriter(fs, new Path(hlogDir, HLOG_FILE_PREFIX + i), conf); + for (int j = 0; j < entries; j++) { + int prefix = 0; + for (String region : regions) { + String row_key = region + prefix++ + i + j; + appendEntry(writer[i], TABLE_NAME, region.getBytes(), + row_key.getBytes(), FAMILY, QUALIFIER, VALUE, seq); + } + } + if (i != leaveOpen) { + writer[i].close(); + LOG.info("Closing writer " + i); + } + } + } + + private Path getLogForRegion(Path rootdir, byte[] table, String region) + throws IOException { + Path tdir = HTableDescriptor.getTableDir(rootdir, table); + Path editsdir = HLog.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, + Bytes.toString(region.getBytes()))); + FileStatus [] files = this.fs.listStatus(editsdir); + assertEquals(1, files.length); + return files[0].getPath(); + } + + private void corruptHLog(Path path, Corruptions corruption, boolean close, + FileSystem fs) throws IOException { + + FSDataOutputStream out; + int fileSize = (int) fs.listStatus(path)[0].getLen(); + + FSDataInputStream in = fs.open(path); + byte[] corrupted_bytes = new byte[fileSize]; + in.readFully(0, corrupted_bytes, 0, fileSize); + in.close(); + + switch (corruption) { + case APPEND_GARBAGE: + fs.delete(path, false); + out = fs.create(path); + out.write(corrupted_bytes); + out.write("-----".getBytes()); + closeOrFlush(close, out); + break; + + case INSERT_GARBAGE_ON_FIRST_LINE: + fs.delete(path, false); + out = fs.create(path); + out.write(0); + out.write(corrupted_bytes); + closeOrFlush(close, out); + break; + + case INSERT_GARBAGE_IN_THE_MIDDLE: + fs.delete(path, false); + out = fs.create(path); + int middle = (int) Math.floor(corrupted_bytes.length / 2); + out.write(corrupted_bytes, 0, middle); + out.write(0); + out.write(corrupted_bytes, middle, corrupted_bytes.length - middle); + closeOrFlush(close, out); + break; + + case TRUNCATE: + fs.delete(path, false); + out = fs.create(path); + out.write(corrupted_bytes, 0, fileSize-32); + closeOrFlush(close, out); + + break; + } + + + } + + private void closeOrFlush(boolean close, FSDataOutputStream out) + throws IOException { + if (close) { + out.close(); + } else { + Method syncMethod = null; + try { + syncMethod = out.getClass().getMethod("hflush", new Class []{}); + } catch (NoSuchMethodException e) { + try { + syncMethod = out.getClass().getMethod("sync", new Class []{}); + } catch (NoSuchMethodException ex) { + throw new IOException("This version of Hadoop supports " + + "neither Syncable.sync() nor Syncable.hflush()."); + } + } + try { + syncMethod.invoke(out, new Object[]{}); + } catch (Exception e) { + throw new IOException(e); + } + // Not in 0out.hflush(); + } + } + + @SuppressWarnings("unused") + private void dumpHLog(Path log, FileSystem fs, Configuration conf) throws IOException { + HLog.Entry entry; + HLog.Reader in = HLog.getReader(fs, log, conf); + while ((entry = in.next()) != null) { + System.out.println(entry); + } + } + + private int countHLog(Path log, FileSystem fs, Configuration conf) throws IOException { + int count = 0; + HLog.Reader in = HLog.getReader(fs, log, conf); + while (in.next() != null) { + count++; + } + return count; + } + + + public long appendEntry(HLog.Writer writer, byte[] table, byte[] region, + byte[] row, byte[] family, byte[] qualifier, + byte[] value, long seq) + throws IOException { + LOG.info(Thread.currentThread().getName() + " append"); + writer.append(createTestEntry(table, region, row, family, qualifier, value, seq)); + LOG.info(Thread.currentThread().getName() + " sync"); + writer.sync(); + return seq; + } + + private HLog.Entry createTestEntry( + byte[] table, byte[] region, + byte[] row, byte[] family, byte[] qualifier, + byte[] value, long seq) { + long time = System.nanoTime(); + WALEdit edit = new WALEdit(); + seq++; + edit.add(new KeyValue(row, family, qualifier, time, KeyValue.Type.Put, value)); + return new HLog.Entry(new HLogKey(region, table, seq, time, + HConstants.DEFAULT_CLUSTER_ID), edit); + } + + + private void injectEmptyFile(String suffix, boolean closeFile) + throws IOException { + HLog.Writer writer = HLog.createWriter( + fs, new Path(hlogDir, HLOG_FILE_PREFIX + suffix), conf); + if (closeFile) writer.close(); + } + + @SuppressWarnings("unused") + private void listLogs(FileSystem fs, Path dir) throws IOException { + for (FileStatus file : fs.listStatus(dir)) { + System.out.println(file.getPath()); + } + + } + + private int compareHLogSplitDirs(Path p1, Path p2) throws IOException { + FileStatus[] f1 = fs.listStatus(p1); + FileStatus[] f2 = fs.listStatus(p2); + assertNotNull("Path " + p1 + " doesn't exist", f1); + assertNotNull("Path " + p2 + " doesn't exist", f2); + + System.out.println("Files in " + p1 + ": " + + Joiner.on(",").join(FileUtil.stat2Paths(f1))); + System.out.println("Files in " + p2 + ": " + + Joiner.on(",").join(FileUtil.stat2Paths(f2))); + assertEquals(f1.length, f2.length); + + for (int i = 0; i < f1.length; i++) { + // Regions now have a directory named RECOVERED_EDITS_DIR and in here + // are split edit files. In below presume only 1. + Path rd1 = HLog.getRegionDirRecoveredEditsDir(f1[i].getPath()); + FileStatus[] rd1fs = fs.listStatus(rd1); + assertEquals(1, rd1fs.length); + Path rd2 = HLog.getRegionDirRecoveredEditsDir(f2[i].getPath()); + FileStatus[] rd2fs = fs.listStatus(rd2); + assertEquals(1, rd2fs.length); + if (!logsAreEqual(rd1fs[0].getPath(), rd2fs[0].getPath())) { + return -1; + } + } + return 0; + } + + private boolean logsAreEqual(Path p1, Path p2) throws IOException { + HLog.Reader in1, in2; + in1 = HLog.getReader(fs, p1, conf); + in2 = HLog.getReader(fs, p2, conf); + HLog.Entry entry1; + HLog.Entry entry2; + while ((entry1 = in1.next()) != null) { + entry2 = in2.next(); + if ((entry1.getKey().compareTo(entry2.getKey()) != 0) || + (!entry1.getEdit().toString().equals(entry2.getEdit().toString()))) { + return false; + } + } + return true; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java new file mode 100644 index 0000000..101678a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplitCompressed.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.LargeTests; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestHLogSplitCompressed extends TestHLogSplit { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TestHLogSplit.setUpBeforeClass(); + TEST_UTIL.getConfiguration().setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestKeyValueCompression.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestKeyValueCompression.java new file mode 100644 index 0000000..8fa7fe8 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestKeyValueCompression.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.util.List; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.DataOutputBuffer; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +import com.google.common.collect.Lists; + +@Category(SmallTests.class) +public class TestKeyValueCompression { + private static final byte[] VALUE = Bytes.toBytes("fake value"); + private static final int BUF_SIZE = 256*1024; + + @Test + public void testCountingKVs() throws Exception { + List kvs = Lists.newArrayList(); + for (int i = 0; i < 400; i++) { + byte[] row = Bytes.toBytes("row" + i); + byte[] fam = Bytes.toBytes("fam" + i); + byte[] qual = Bytes.toBytes("qual" + i); + kvs.add(new KeyValue(row, fam, qual, 12345L, VALUE)); + } + + runTestCycle(kvs); + } + + @Test + public void testRepeatingKVs() throws Exception { + List kvs = Lists.newArrayList(); + for (int i = 0; i < 400; i++) { + byte[] row = Bytes.toBytes("row" + (i % 10)); + byte[] fam = Bytes.toBytes("fam" + (i % 127)); + byte[] qual = Bytes.toBytes("qual" + (i % 128)); + kvs.add(new KeyValue(row, fam, qual, 12345L, VALUE)); + } + + runTestCycle(kvs); + } + + private void runTestCycle(List kvs) throws Exception { + CompressionContext ctx = new CompressionContext(LRUDictionary.class); + DataOutputBuffer buf = new DataOutputBuffer(BUF_SIZE); + for (KeyValue kv : kvs) { + KeyValueCompression.writeKV(buf, kv, ctx); + } + + ctx.clear(); + DataInputStream in = new DataInputStream(new ByteArrayInputStream( + buf.getData(), 0, buf.getLength())); + for (KeyValue kv : kvs) { + KeyValue readBack = KeyValueCompression.readKV(in, ctx); + assertEquals(kv, readBack); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLRUDictionary.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLRUDictionary.java new file mode 100644 index 0000000..99983a2 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLRUDictionary.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import static org.junit.Assert.*; + +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Random; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests LRUDictionary + */ +@Category(SmallTests.class) +public class TestLRUDictionary { + LRUDictionary testee; + + @Before + public void setUp() throws Exception { + testee = new LRUDictionary(); + } + + @Test + public void TestContainsNothing() { + assertTrue(isDictionaryEmpty(testee)); + } + + /** + * Assert can't add empty array. + */ + @Test + public void testPassingEmptyArrayToFindEntry() { + assertEquals(Dictionary.NOT_IN_DICTIONARY, + testee.findEntry(HConstants.EMPTY_BYTE_ARRAY, 0, 0)); + assertEquals(Dictionary.NOT_IN_DICTIONARY, + testee.addEntry(HConstants.EMPTY_BYTE_ARRAY, 0, 0)); + } + + @Test + public void testPassingSameArrayToAddEntry() { + // Add random predefined byte array, in this case a random byte array from + // HConstants. Assert that when we add, we get new index. Thats how it + // works. + int len = HConstants.CATALOG_FAMILY.length; + int index = testee.addEntry(HConstants.CATALOG_FAMILY, 0, len); + assertFalse(index == testee.addEntry(HConstants.CATALOG_FAMILY, 0, len)); + assertFalse(index == testee.addEntry(HConstants.CATALOG_FAMILY, 0, len)); + } + + @Test + public void testBasic() { + Random rand = new Random(); + byte[] testBytes = new byte[10]; + rand.nextBytes(testBytes); + + // Verify that our randomly generated array doesn't exist in the dictionary + assertEquals(testee.findEntry(testBytes, 0, testBytes.length), -1); + + // now since we looked up an entry, we should have added it to the + // dictionary, so it isn't empty + + assertFalse(isDictionaryEmpty(testee)); + + // Check if we can find it using findEntry + short t = testee.findEntry(testBytes, 0, testBytes.length); + + // Making sure we do find what we're looking for + assertTrue(t != -1); + + byte[] testBytesCopy = new byte[20]; + + Bytes.putBytes(testBytesCopy, 10, testBytes, 0, testBytes.length); + + // copy byte arrays, make sure that we check that equal byte arrays are + // equal without just checking the reference + assertEquals(testee.findEntry(testBytesCopy, 10, testBytes.length), t); + + // make sure the entry retrieved is the same as the one put in + assertTrue(Arrays.equals(testBytes, testee.getEntry(t))); + + testee.clear(); + + // making sure clear clears the dictionary + assertTrue(isDictionaryEmpty(testee)); + } + + @Test + public void TestLRUPolicy(){ + //start by filling the dictionary up with byte arrays + for (int i = 0; i < LRUDictionary.BidirectionalLRUMap.MAX_SIZE; i++) { + testee.findEntry((BigInteger.valueOf(i)).toByteArray(), 0, + (BigInteger.valueOf(i)).toByteArray().length); + } + + // check we have the first element added + assertTrue(testee.findEntry(BigInteger.ZERO.toByteArray(), 0, + BigInteger.ZERO.toByteArray().length) != -1); + + // check for an element we know isn't there + assertTrue(testee.findEntry(BigInteger.valueOf(Integer.MAX_VALUE).toByteArray(), 0, + BigInteger.valueOf(Integer.MAX_VALUE).toByteArray().length) == -1); + + // since we just checked for this element, it should be there now. + assertTrue(testee.findEntry(BigInteger.valueOf(Integer.MAX_VALUE).toByteArray(), 0, + BigInteger.valueOf(Integer.MAX_VALUE).toByteArray().length) != -1); + + // test eviction, that the least recently added or looked at element is + // evicted. We looked at ZERO so it should be in the dictionary still. + assertTrue(testee.findEntry(BigInteger.ZERO.toByteArray(), 0, + BigInteger.ZERO.toByteArray().length) != -1); + // Now go from beyond 1 to the end. + for(int i = 1; i < LRUDictionary.BidirectionalLRUMap.MAX_SIZE; i++) { + assertTrue(testee.findEntry(BigInteger.valueOf(i).toByteArray(), 0, + BigInteger.valueOf(i).toByteArray().length) == -1); + } + + // check we can find all of these. + for (int i = 0; i < LRUDictionary.BidirectionalLRUMap.MAX_SIZE; i++) { + assertTrue(testee.findEntry(BigInteger.valueOf(i).toByteArray(), 0, + BigInteger.valueOf(i).toByteArray().length) != -1); + } + } + + static private boolean isDictionaryEmpty(LRUDictionary dict) { + try { + dict.getEntry((short)0); + return false; + } catch (IndexOutOfBoundsException ioobe) { + return true; + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java new file mode 100644 index 0000000..3ea0a3b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.namenode.LeaseManager; +import org.apache.log4j.Level; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests for conditions that should trigger RegionServer aborts when + * rolling the current HLog fails. + */ +@Category(MediumTests.class) +public class TestLogRollAbort { + private static final Log LOG = LogFactory.getLog(TestLogRolling.class); + private static MiniDFSCluster dfsCluster; + private static HBaseAdmin admin; + private static MiniHBaseCluster cluster; + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + // verbose logging on classes that are touched in these tests + { + ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem")) + .getLogger().setLevel(Level.ALL); + ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)HRegionServer.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)HRegion.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL); + } + + // Need to override this setup so we can edit the config before it gets sent + // to the HDFS & HBase cluster startup. + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Tweak default timeout values down for faster recovery + TEST_UTIL.getConfiguration().setInt( + "hbase.regionserver.logroll.errors.tolerated", 2); + TEST_UTIL.getConfiguration().setInt("ipc.ping.interval", 10 * 1000); + TEST_UTIL.getConfiguration().setInt("ipc.socket.timeout", 10 * 1000); + TEST_UTIL.getConfiguration().setInt("hbase.rpc.timeout", 10 * 1000); + + // Increase the amount of time between client retries + TEST_UTIL.getConfiguration().setLong("hbase.client.pause", 5 * 1000); + + // make sure log.hflush() calls syncFs() to open a pipeline + TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); + // lower the namenode & datanode heartbeat so the namenode + // quickly detects datanode failures + TEST_UTIL.getConfiguration().setInt("heartbeat.recheck.interval", 5000); + TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1); + // the namenode might still try to choose the recently-dead datanode + // for a pipeline, so try to a new pipeline multiple times + TEST_UTIL.getConfiguration().setInt("dfs.client.block.write.retries", 10); + // set periodic sync to 2 min so it doesn't run during test + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.optionallogflushinterval", + 120 * 1000); + } + + @Before + public void setUp() throws Exception { + TEST_UTIL.startMiniCluster(2); + + cluster = TEST_UTIL.getHBaseCluster(); + dfsCluster = TEST_UTIL.getDFSCluster(); + admin = TEST_UTIL.getHBaseAdmin(); + + // disable region rebalancing (interferes with log watching) + cluster.getMaster().balanceSwitch(false); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Tests that RegionServer aborts if we hit an error closing the WAL when + * there are unsynced WAL edits. See HBASE-4282. + */ + @Test + public void testRSAbortWithUnflushedEdits() throws Exception { + LOG.info("Starting testRSAbortWithUnflushedEdits()"); + + // When the META table can be opened, the region servers are running + new HTable(TEST_UTIL.getConfiguration(), + HConstants.META_TABLE_NAME).close(); + + // Create the test table and open it + String tableName = this.getClass().getSimpleName(); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + desc.setDeferredLogFlush(true); + + admin.createTable(desc); + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + + HRegionServer server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName)); + HLog log = server.getWAL(); + + assertTrue("Need HDFS-826 for this test", log.canGetCurReplicas()); + // don't run this test without append support (HDFS-200 & HDFS-142) + assertTrue("Need append support for this test", + FSUtils.isAppendSupported(TEST_UTIL.getConfiguration())); + + Put p = new Put(Bytes.toBytes("row2001")); + p.add(HConstants.CATALOG_FAMILY, Bytes.toBytes("col"), Bytes.toBytes(2001)); + table.put(p); + + log.sync(); + + p = new Put(Bytes.toBytes("row2002")); + p.add(HConstants.CATALOG_FAMILY, Bytes.toBytes("col"), Bytes.toBytes(2002)); + table.put(p); + + dfsCluster.restartDataNodes(); + LOG.info("Restarted datanodes"); + + assertTrue("Should have an outstanding WAL edit", log.hasDeferredEntries()); + try { + log.rollWriter(true); + fail("Log roll should have triggered FailedLogCloseException"); + } catch (FailedLogCloseException flce) { + assertTrue("Should have deferred flush log edits outstanding", + log.hasDeferredEntries()); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java new file mode 100644 index 0000000..a42c801 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java @@ -0,0 +1,553 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.*; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.namenode.LeaseManager; +import org.apache.log4j.Level; +import org.junit.*; +import org.junit.experimental.categories.Category; + +import java.io.*; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.*; + +import static org.junit.Assert.*; + +/** + * Test log deletion as logs are rolled. + */ +@Category(LargeTests.class) +public class TestLogRolling { + private static final Log LOG = LogFactory.getLog(TestLogRolling.class); + private HRegionServer server; + private HLog log; + private String tableName; + private byte[] value; + private FileSystem fs; + private MiniDFSCluster dfsCluster; + private HBaseAdmin admin; + private MiniHBaseCluster cluster; + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + // verbose logging on classes that are touched in these tests + { + ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem")) + .getLogger().setLevel(Level.ALL); + ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)HRegionServer.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)HRegion.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL); + } + + /** + * constructor + * @throws Exception + */ + public TestLogRolling() { + this.server = null; + this.log = null; + this.tableName = null; + + String className = this.getClass().getName(); + StringBuilder v = new StringBuilder(className); + while (v.length() < 1000) { + v.append(className); + } + this.value = Bytes.toBytes(v.toString()); + } + + // Need to override this setup so we can edit the config before it gets sent + // to the HDFS & HBase cluster startup. + @BeforeClass + public static void setUpBeforeClass() throws Exception { + /**** configuration for testLogRolling ****/ + // Force a region split after every 768KB + TEST_UTIL.getConfiguration().setLong(HConstants.HREGION_MAX_FILESIZE, 768L * 1024L); + + // We roll the log after every 32 writes + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.maxlogentries", 32); + + TEST_UTIL.getConfiguration().setInt( + "hbase.regionserver.logroll.errors.tolerated", 2); + TEST_UTIL.getConfiguration().setInt("ipc.ping.interval", 10 * 1000); + TEST_UTIL.getConfiguration().setInt("ipc.socket.timeout", 10 * 1000); + TEST_UTIL.getConfiguration().setInt("hbase.rpc.timeout", 10 * 1000); + + // For less frequently updated regions flush after every 2 flushes + TEST_UTIL.getConfiguration().setInt("hbase.hregion.memstore.optionalflushcount", 2); + + // We flush the cache after every 8192 bytes + TEST_UTIL.getConfiguration().setInt( + HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 8192); + + // Increase the amount of time between client retries + TEST_UTIL.getConfiguration().setLong("hbase.client.pause", 10 * 1000); + + // Reduce thread wake frequency so that other threads can get + // a chance to run. + TEST_UTIL.getConfiguration().setInt(HConstants.THREAD_WAKE_FREQUENCY, 2 * 1000); + + /**** configuration for testLogRollOnDatanodeDeath ****/ + // make sure log.hflush() calls syncFs() to open a pipeline + TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); + // lower the namenode & datanode heartbeat so the namenode + // quickly detects datanode failures + TEST_UTIL.getConfiguration().setInt("heartbeat.recheck.interval", 5000); + TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1); + // the namenode might still try to choose the recently-dead datanode + // for a pipeline, so try to a new pipeline multiple times + TEST_UTIL.getConfiguration().setInt("dfs.client.block.write.retries", 30); + TEST_UTIL.getConfiguration().setInt( + "hbase.regionserver.hlog.tolerable.lowreplication", 2); + TEST_UTIL.getConfiguration().setInt( + "hbase.regionserver.hlog.lowreplication.rolllimit", 3); + } + + @Before + public void setUp() throws Exception { + TEST_UTIL.startMiniCluster(2); + + cluster = TEST_UTIL.getHBaseCluster(); + dfsCluster = TEST_UTIL.getDFSCluster(); + fs = TEST_UTIL.getTestFileSystem(); + admin = TEST_UTIL.getHBaseAdmin(); + + // disable region rebalancing (interferes with log watching) + cluster.getMaster().balanceSwitch(false); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + private void startAndWriteData() throws IOException { + // When the META table can be opened, the region servers are running + new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME); + this.server = cluster.getRegionServerThreads().get(0).getRegionServer(); + this.log = server.getWAL(); + + // Create the test table and open it + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc); + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + + server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName)); + this.log = server.getWAL(); + for (int i = 1; i <= 256; i++) { // 256 writes should cause 8 log rolls + Put put = new Put(Bytes.toBytes("row" + String.format("%1$04d", i))); + put.add(HConstants.CATALOG_FAMILY, null, value); + table.put(put); + if (i % 32 == 0) { + // After every 32 writes sleep to let the log roller run + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + // continue + } + } + } + } + + /** + * Tests that logs are deleted + * @throws IOException + * @throws FailedLogCloseException + */ + @Test + public void testLogRolling() throws FailedLogCloseException, IOException { + this.tableName = getName(); + startAndWriteData(); + LOG.info("after writing there are " + log.getNumLogFiles() + " log files"); + + // flush all regions + + List regions = + new ArrayList(server.getOnlineRegionsLocalContext()); + for (HRegion r: regions) { + r.flushcache(); + } + + // Now roll the log + log.rollWriter(); + + int count = log.getNumLogFiles(); + LOG.info("after flushing all regions and rolling logs there are " + + log.getNumLogFiles() + " log files"); + assertTrue(("actual count: " + count), count <= 2); + } + + private static String getName() { + return "TestLogRolling"; + } + + void writeData(HTable table, int rownum) throws IOException { + Put put = new Put(Bytes.toBytes("row" + String.format("%1$04d", rownum))); + put.add(HConstants.CATALOG_FAMILY, null, value); + table.put(put); + + // sleep to let the log roller run (if it needs to) + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + // continue + } + } + + void validateData(HTable table, int rownum) throws IOException { + String row = "row" + String.format("%1$04d", rownum); + Get get = new Get(Bytes.toBytes(row)); + get.addFamily(HConstants.CATALOG_FAMILY); + Result result = table.get(get); + assertTrue(result.size() == 1); + assertTrue(Bytes.equals(value, + result.getValue(HConstants.CATALOG_FAMILY, null))); + LOG.info("Validated row " + row); + } + + void batchWriteAndWait(HTable table, int start, boolean expect, int timeout) + throws IOException { + for (int i = 0; i < 10; i++) { + Put put = new Put(Bytes.toBytes("row" + + String.format("%1$04d", (start + i)))); + put.add(HConstants.CATALOG_FAMILY, null, value); + table.put(put); + } + long startTime = System.currentTimeMillis(); + long remaining = timeout; + while (remaining > 0) { + if (log.isLowReplicationRollEnabled() == expect) { + break; + } else { + try { + Thread.sleep(200); + } catch (InterruptedException e) { + // continue + } + remaining = timeout - (System.currentTimeMillis() - startTime); + } + } + } + + /** + * Give me the HDFS pipeline for this log file + */ + DatanodeInfo[] getPipeline(HLog log) throws IllegalArgumentException, + IllegalAccessException, InvocationTargetException { + OutputStream stm = log.getOutputStream(); + Method getPipeline = null; + for (Method m : stm.getClass().getDeclaredMethods()) { + if (m.getName().endsWith("getPipeline")) { + getPipeline = m; + getPipeline.setAccessible(true); + break; + } + } + + assertTrue("Need DFSOutputStream.getPipeline() for this test", + null != getPipeline); + Object repl = getPipeline.invoke(stm, new Object[] {} /* NO_ARGS */); + return (DatanodeInfo[]) repl; + } + + + /** + * Tests that logs are rolled upon detecting datanode death + * Requires an HDFS jar with HDFS-826 & syncFs() support (HDFS-200) + * @throws IOException + * @throws InterruptedException + * @throws InvocationTargetException + * @throws IllegalAccessException + * @throws IllegalArgumentException + */ + @Test + public void testLogRollOnDatanodeDeath() throws Exception { + assertTrue("This test requires HLog file replication set to 2.", + fs.getDefaultReplication() == 2); + LOG.info("Replication=" + fs.getDefaultReplication()); + + this.server = cluster.getRegionServer(0); + this.log = server.getWAL(); + + // Create the test table and open it + String tableName = getName(); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + + admin.createTable(desc); + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + assertTrue(table.isAutoFlush()); + + server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName)); + this.log = server.getWAL(); + + assertTrue("Need HDFS-826 for this test", log.canGetCurReplicas()); + // don't run this test without append support (HDFS-200 & HDFS-142) + assertTrue("Need append support for this test", FSUtils + .isAppendSupported(TEST_UTIL.getConfiguration())); + + // add up the datanode count, to ensure proper replication when we kill 1 + // This function is synchronous; when it returns, the dfs cluster is active + // We start 3 servers and then stop 2 to avoid a directory naming conflict + // when we stop/start a namenode later, as mentioned in HBASE-5163 + List existingNodes = dfsCluster.getDataNodes(); + int numDataNodes = 3; + dfsCluster.startDataNodes(TEST_UTIL.getConfiguration(), numDataNodes, true, + null, null); + List allNodes = dfsCluster.getDataNodes(); + for (int i = allNodes.size()-1; i >= 0; i--) { + if (existingNodes.contains(allNodes.get(i))) { + dfsCluster.stopDataNode( i ); + } + } + + assertTrue("DataNodes " + dfsCluster.getDataNodes().size() + + " default replication " + fs.getDefaultReplication(), + dfsCluster.getDataNodes().size() >= fs.getDefaultReplication() + 1); + + writeData(table, 2); + + long curTime = System.currentTimeMillis(); + long oldFilenum = log.getFilenum(); + assertTrue("Log should have a timestamp older than now", + curTime > oldFilenum && oldFilenum != -1); + + assertTrue("The log shouldn't have rolled yet", + oldFilenum == log.getFilenum()); + final DatanodeInfo[] pipeline = getPipeline(log); + assertTrue(pipeline.length == fs.getDefaultReplication()); + + // kill a datanode in the pipeline to force a log roll on the next sync() + // This function is synchronous, when it returns the node is killed. + assertTrue(dfsCluster.stopDataNode(pipeline[0].getName()) != null); + + // this write should succeed, but trigger a log roll + writeData(table, 2); + long newFilenum = log.getFilenum(); + + assertTrue("Missing datanode should've triggered a log roll", + newFilenum > oldFilenum && newFilenum > curTime); + + // write some more log data (this should use a new hdfs_out) + writeData(table, 3); + assertTrue("The log should not roll again.", + log.getFilenum() == newFilenum); + // kill another datanode in the pipeline, so the replicas will be lower than + // the configured value 2. + assertTrue(dfsCluster.stopDataNode(pipeline[1].getName()) != null); + + batchWriteAndWait(table, 3, false, 10000); + assertTrue("LowReplication Roller should've been disabled", + !log.isLowReplicationRollEnabled()); + + dfsCluster + .startDataNodes(TEST_UTIL.getConfiguration(), 1, true, null, null); + + // Force roll writer. The new log file will have the default replications, + // and the LowReplication Roller will be enabled. + log.rollWriter(true); + batchWriteAndWait(table, 13, true, 10000); + assertTrue("New log file should have the default replication instead of " + + log.getLogReplication(), + log.getLogReplication() == fs.getDefaultReplication()); + assertTrue("LowReplication Roller should've been enabled", + log.isLowReplicationRollEnabled()); + } + + /** + * Test that HLog is rolled when all data nodes in the pipeline have been + * restarted. + * @throws Exception + */ + //DISABLED BECAUSE FLAKEY @Test + public void testLogRollOnPipelineRestart() throws Exception { + LOG.info("Starting testLogRollOnPipelineRestart"); + assertTrue("This test requires HLog file replication.", + fs.getDefaultReplication() > 1); + LOG.info("Replication=" + fs.getDefaultReplication()); + // When the META table can be opened, the region servers are running + new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME); + + this.server = cluster.getRegionServer(0); + this.log = server.getWAL(); + + // Create the test table and open it + String tableName = getName(); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + + admin.createTable(desc); + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + + server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName)); + this.log = server.getWAL(); + final List paths = new ArrayList(); + final List preLogRolledCalled = new ArrayList(); + paths.add(log.computeFilename()); + log.registerWALActionsListener(new WALActionsListener() { + @Override + public void preLogRoll(Path oldFile, Path newFile) { + LOG.debug("preLogRoll: oldFile="+oldFile+" newFile="+newFile); + preLogRolledCalled.add(new Integer(1)); + } + @Override + public void postLogRoll(Path oldFile, Path newFile) { + paths.add(newFile); + } + @Override + public void preLogArchive(Path oldFile, Path newFile) {} + @Override + public void postLogArchive(Path oldFile, Path newFile) {} + @Override + public void logRollRequested() {} + @Override + public void logCloseRequested() {} + @Override + public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, + WALEdit logEdit) {} + @Override + public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, + WALEdit logEdit) {} + }); + + assertTrue("Need HDFS-826 for this test", log.canGetCurReplicas()); + // don't run this test without append support (HDFS-200 & HDFS-142) + assertTrue("Need append support for this test", FSUtils + .isAppendSupported(TEST_UTIL.getConfiguration())); + + writeData(table, 1002); + + table.setAutoFlush(true); + + long curTime = System.currentTimeMillis(); + long oldFilenum = log.getFilenum(); + assertTrue("Log should have a timestamp older than now", + curTime > oldFilenum && oldFilenum != -1); + + assertTrue("The log shouldn't have rolled yet", oldFilenum == log.getFilenum()); + + // roll all datanodes in the pipeline + dfsCluster.restartDataNodes(); + Thread.sleep(1000); + dfsCluster.waitActive(); + LOG.info("Data Nodes restarted"); + validateData(table, 1002); + + // this write should succeed, but trigger a log roll + writeData(table, 1003); + long newFilenum = log.getFilenum(); + + assertTrue("Missing datanode should've triggered a log roll", + newFilenum > oldFilenum && newFilenum > curTime); + validateData(table, 1003); + + writeData(table, 1004); + + // roll all datanode again + dfsCluster.restartDataNodes(); + Thread.sleep(1000); + dfsCluster.waitActive(); + LOG.info("Data Nodes restarted"); + validateData(table, 1004); + + // this write should succeed, but trigger a log roll + writeData(table, 1005); + + // force a log roll to read back and verify previously written logs + log.rollWriter(true); + assertTrue("preLogRolledCalled has size of " + preLogRolledCalled.size(), + preLogRolledCalled.size() >= 1); + + // read back the data written + Set loggedRows = new HashSet(); + for (Path p : paths) { + LOG.debug("Reading HLog "+FSUtils.getPath(p)); + HLog.Reader reader = null; + try { + reader = HLog.getReader(fs, p, TEST_UTIL.getConfiguration()); + HLog.Entry entry; + while ((entry = reader.next()) != null) { + LOG.debug("#"+entry.getKey().getLogSeqNum()+": "+entry.getEdit().getKeyValues()); + for (KeyValue kv : entry.getEdit().getKeyValues()) { + loggedRows.add(Bytes.toStringBinary(kv.getRow())); + } + } + } catch (EOFException e) { + LOG.debug("EOF reading file "+FSUtils.getPath(p)); + } finally { + if (reader != null) reader.close(); + } + } + + // verify the written rows are there + assertTrue(loggedRows.contains("row1002")); + assertTrue(loggedRows.contains("row1003")); + assertTrue(loggedRows.contains("row1004")); + assertTrue(loggedRows.contains("row1005")); + + // flush all regions + List regions = + new ArrayList(server.getOnlineRegionsLocalContext()); + for (HRegion r: regions) { + r.flushcache(); + } + + ResultScanner scanner = table.getScanner(new Scan()); + try { + for (int i=2; i<=5; i++) { + Result r = scanner.next(); + assertNotNull(r); + assertFalse(r.isEmpty()); + assertEquals("row100"+i, Bytes.toString(r.getRow())); + } + } finally { + scanner.close(); + } + + // verify that no region servers aborted + for (JVMClusterUtil.RegionServerThread rsThread: + TEST_UTIL.getHBaseCluster().getRegionServerThreads()) { + assertFalse(rsThread.getRegionServer().isAborted()); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java new file mode 100644 index 0000000..3f46736 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import static org.junit.Assert.assertFalse; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test many concurrent appenders to an {@link #HLog} while rolling the log. + */ +@Category(MediumTests.class) +public class TestLogRollingNoCluster { + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static byte [] EMPTY_1K_ARRAY = new byte[1024]; + private static final int THREAD_COUNT = 100; // Spin up this many threads + + /** + * Spin up a bunch of threads and have them all append to a WAL. Roll the + * WAL frequently to try and trigger NPE. + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testContendedLogRolling() throws IOException, InterruptedException { + FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); + Path dir = TEST_UTIL.getDataTestDir(); + HLog wal = new HLog(fs, new Path(dir, "logs"), new Path(dir, "oldlogs"), + TEST_UTIL.getConfiguration()); + Appender [] appenders = null; + + final int count = THREAD_COUNT; + appenders = new Appender[count]; + try { + for (int i = 0; i < count; i++) { + // Have each appending thread write 'count' entries + appenders[i] = new Appender(wal, i, count); + } + for (int i = 0; i < count; i++) { + appenders[i].start(); + } + for (int i = 0; i < count; i++) { + //ensure that all threads are joined before closing the wal + appenders[i].join(); + } + } finally { + wal.close(); + } + for (int i = 0; i < count; i++) { + assertFalse(appenders[i].isException()); + } + } + + /** + * Appender thread. Appends to passed wal file. + */ + static class Appender extends Thread { + private final Log log; + private final HLog wal; + private final int count; + private Exception e = null; + + Appender(final HLog wal, final int index, final int count) { + super("" + index); + this.wal = wal; + this.count = count; + this.log = LogFactory.getLog("Appender:" + getName()); + } + + /** + * @return Call when the thread is done. + */ + boolean isException() { + return !isAlive() && this.e != null; + } + + Exception getException() { + return this.e; + } + + @Override + public void run() { + this.log.info(getName() +" started"); + try { + for (int i = 0; i < this.count; i++) { + long now = System.currentTimeMillis(); + // Roll every ten edits if the log has anything in it. + if (i % 10 == 0 && this.wal.getNumEntries() > 0) { + this.wal.rollWriter(); + } + WALEdit edit = new WALEdit(); + byte[] bytes = Bytes.toBytes(i); + edit.add(new KeyValue(bytes, bytes, bytes, now, EMPTY_1K_ARRAY)); + + this.wal.append(HRegionInfo.FIRST_META_REGIONINFO, + HTableDescriptor.META_TABLEDESC.getName(), + edit, now, HTableDescriptor.META_TABLEDESC); + } + String msg = getName() + " finished"; + if (isException()) + this.log.info(msg, getException()); + else + this.log.info(msg); + } catch (Exception e) { + this.e = e; + log.info("Caught exception from Appender:" + getName(), e); + } + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java new file mode 100644 index 0000000..de81e91 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java @@ -0,0 +1,177 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +/** + * Test that the actions are called while playing with an HLog + */ +@Category(SmallTests.class) +public class TestWALActionsListener { + protected static final Log LOG = LogFactory.getLog(TestWALActionsListener.class); + + private final static HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + private final static byte[] SOME_BYTES = Bytes.toBytes("t"); + private static FileSystem fs; + private static Path oldLogDir; + private static Path logDir; + private static Configuration conf; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf = TEST_UTIL.getConfiguration(); + conf.setInt("hbase.regionserver.maxlogs", 5); + fs = FileSystem.get(conf); + oldLogDir = new Path(TEST_UTIL.getDataTestDir(), + HConstants.HREGION_OLDLOGDIR_NAME); + logDir = new Path(TEST_UTIL.getDataTestDir(), + HConstants.HREGION_LOGDIR_NAME); + } + + @Before + public void setUp() throws Exception { + fs.delete(logDir, true); + fs.delete(oldLogDir, true); + } + + @After + public void tearDown() throws Exception { + setUp(); + } + + /** + * Add a bunch of dummy data and roll the logs every two insert. We + * should end up with 10 rolled files (plus the roll called in + * the constructor). Also test adding a listener while it's running. + */ + @Test + public void testActionListener() throws Exception { + DummyWALActionsListener observer = new DummyWALActionsListener(); + List list = new ArrayList(); + list.add(observer); + DummyWALActionsListener laterobserver = new DummyWALActionsListener(); + HLog hlog = new HLog(fs, logDir, oldLogDir, conf, list, null); + HRegionInfo hri = new HRegionInfo(SOME_BYTES, + SOME_BYTES, SOME_BYTES, false); + + for (int i = 0; i < 20; i++) { + byte[] b = Bytes.toBytes(i+""); + KeyValue kv = new KeyValue(b,b,b); + WALEdit edit = new WALEdit(); + edit.add(kv); + HTableDescriptor htd = new HTableDescriptor(); + htd.addFamily(new HColumnDescriptor(b)); + + HLogKey key = new HLogKey(b,b, 0, 0, HConstants.DEFAULT_CLUSTER_ID); + hlog.append(hri, key, edit, htd, true); + if (i == 10) { + hlog.registerWALActionsListener(laterobserver); + } + if (i % 2 == 0) { + hlog.rollWriter(); + } + } + + hlog.close(); + hlog.closeAndDelete(); + + assertEquals(11, observer.preLogRollCounter); + assertEquals(11, observer.postLogRollCounter); + assertEquals(5, laterobserver.preLogRollCounter); + assertEquals(5, laterobserver.postLogRollCounter); + assertEquals(2, observer.closedCount); + } + + + /** + * Just counts when methods are called + */ + static class DummyWALActionsListener implements WALActionsListener { + public int preLogRollCounter = 0; + public int postLogRollCounter = 0; + public int closedCount = 0; + + @Override + public void preLogRoll(Path oldFile, Path newFile) { + preLogRollCounter++; + } + + @Override + public void postLogRoll(Path oldFile, Path newFile) { + postLogRollCounter++; + } + + @Override + public void preLogArchive(Path oldFile, Path newFile) { + // Not interested + } + + @Override + public void postLogArchive(Path oldFile, Path newFile) { + // Not interested + } + + @Override + public void logRollRequested() { + // Not interested + } + + @Override + public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, + WALEdit logEdit) { + // Not interested + + } + + @Override + public void logCloseRequested() { + closedCount++; + } + + public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEdit logEdit) { + //To change body of implemented methods use File | Settings | File Templates. + } + + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java new file mode 100644 index 0000000..f95fa1f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java @@ -0,0 +1,959 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; +import java.util.SortedSet; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.monitoring.MonitoredTask; +import org.apache.hadoop.hbase.regionserver.FlushRequester; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test replay of edits out of a WAL split. + */ +@Category(MediumTests.class) +public class TestWALReplay { + public static final Log LOG = LogFactory.getLog(TestWALReplay.class); + static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate(); + private Path hbaseRootDir = null; + private Path oldLogDir; + private Path logDir; + private FileSystem fs; + private Configuration conf; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setBoolean("dfs.support.append", true); + // The below config supported by 0.20-append and CDH3b2 + conf.setInt("dfs.client.block.recovery.retries", 2); + TEST_UTIL.startMiniCluster(3); + Path hbaseRootDir = + TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase")); + LOG.info("hbase.rootdir=" + hbaseRootDir); + conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); + this.fs = TEST_UTIL.getDFSCluster().getFileSystem(); + this.hbaseRootDir = new Path(this.conf.get(HConstants.HBASE_DIR)); + this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME); + this.logDir = new Path(this.hbaseRootDir, HConstants.HREGION_LOGDIR_NAME); + if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) { + TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true); + } + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true); + } + + /* + * @param p Directory to cleanup + */ + private void deleteDir(final Path p) throws IOException { + if (this.fs.exists(p)) { + if (!this.fs.delete(p, true)) { + throw new IOException("Failed remove of " + p); + } + } + } + + /** + * + * @throws Exception + */ + @Test + public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception { + final byte[] tableName = Bytes + .toBytes("testReplayEditsAfterRegionMovedWithMultiCF"); + byte[] family1 = Bytes.toBytes("cf1"); + byte[] family2 = Bytes.toBytes("cf2"); + byte[] qualifier = Bytes.toBytes("q"); + byte[] value = Bytes.toBytes("testV"); + byte[][] familys = { family1, family2 }; + TEST_UTIL.createTable(tableName, familys); + HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName); + Put put = new Put(Bytes.toBytes("r1")); + put.add(family1, qualifier, value); + htable.put(put); + ResultScanner resultScanner = htable.getScanner(new Scan()); + int count = 0; + while (resultScanner.next() != null) { + count++; + } + resultScanner.close(); + assertEquals(1, count); + + MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster(); + List regions = hbaseCluster.getRegions(tableName); + assertEquals(1, regions.size()); + + // move region to another regionserver + HRegion destRegion = regions.get(0); + int originServerNum = hbaseCluster + .getServerWith(destRegion.getRegionName()); + assertTrue("Please start more than 1 regionserver", hbaseCluster + .getRegionServerThreads().size() > 1); + int destServerNum = 0; + while (destServerNum == originServerNum) { + destServerNum++; + } + HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum); + HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum); + // move region to destination regionserver + moveRegionAndWait(destRegion, destServer); + + // delete the row + Delete del = new Delete(Bytes.toBytes("r1")); + htable.delete(del); + resultScanner = htable.getScanner(new Scan()); + count = 0; + while (resultScanner.next() != null) { + count++; + } + resultScanner.close(); + assertEquals(0, count); + + // flush region and make major compaction + destServer.getOnlineRegion(destRegion.getRegionName()).flushcache(); + // wait to complete major compaction + for (Store store : destServer.getOnlineRegion(destRegion.getRegionName()) + .getStores().values()) { + store.triggerMajorCompaction(); + } + destServer.getOnlineRegion(destRegion.getRegionName()).compactStores(); + + // move region to origin regionserver + moveRegionAndWait(destRegion, originServer); + // abort the origin regionserver + originServer.abort("testing"); + + // see what we get + Result result = htable.get(new Get(Bytes.toBytes("r1"))); + if (result != null) { + assertTrue("Row is deleted, but we get" + result.toString(), + (result == null) || result.isEmpty()); + } + resultScanner.close(); + } + + private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer) + throws InterruptedException, MasterNotRunningException, + ZooKeeperConnectionException, IOException { + HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); + TEST_UTIL.getHBaseAdmin().move( + destRegion.getRegionInfo().getEncodedNameAsBytes(), + Bytes.toBytes(destServer.getServerName().getServerName())); + while (true) { + ServerName serverName = master.getAssignmentManager() + .getRegionServerOfRegion(destRegion.getRegionInfo()); + if (serverName != null && serverName.equals(destServer.getServerName())) break; + Thread.sleep(10); + } + } + + /** + * Tests for hbase-2727. + * @throws Exception + * @see https://issues.apache.org/jira/browse/HBASE-2727 + */ + @Test + public void test2727() throws Exception { + // Test being able to have > 1 set of edits in the recovered.edits directory. + // Ensure edits are replayed properly. + final String tableNameStr = "test2727"; + HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr); + Path basedir = new Path(hbaseRootDir, tableNameStr); + deleteDir(basedir); + fs.mkdirs(new Path(basedir, hri.getEncodedName())); + + HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr); + HRegion region2 = HRegion.createHRegion(hri, + hbaseRootDir, this.conf, htd); + region2.close(); + region2.getLog().closeAndDelete(); + final byte [] tableName = Bytes.toBytes(tableNameStr); + final byte [] rowName = tableName; + + HLog wal1 = createWAL(this.conf); + // Add 1k to each family. + final int countPerFamily = 1000; + for (HColumnDescriptor hcd: htd.getFamilies()) { + addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee, + wal1, htd); + } + wal1.close(); + runWALSplit(this.conf); + + HLog wal2 = createWAL(this.conf); + // Up the sequenceid so that these edits are after the ones added above. + wal2.setSequenceNumber(wal1.getSequenceNumber()); + // Add 1k to each family. + for (HColumnDescriptor hcd: htd.getFamilies()) { + addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, + ee, wal2, htd); + } + wal2.close(); + runWALSplit(this.conf); + + HLog wal3 = createWAL(this.conf); + wal3.setSequenceNumber(wal2.getSequenceNumber()); + try { + final HRegion region = new HRegion(basedir, wal3, this.fs, this.conf, hri, + htd, null); + long seqid = region.initialize(); + assertTrue(seqid > wal3.getSequenceNumber()); + + // TODO: Scan all. + region.close(); + } finally { + wal3.closeAndDelete(); + } + } + + /** + * Test case of HRegion that is only made out of bulk loaded files. Assert + * that we don't 'crash'. + * @throws IOException + * @throws IllegalAccessException + * @throws NoSuchFieldException + * @throws IllegalArgumentException + * @throws SecurityException + */ + @Test + public void testRegionMadeOfBulkLoadedFilesOnly() + throws IOException, SecurityException, IllegalArgumentException, + NoSuchFieldException, IllegalAccessException, InterruptedException { + final String tableNameStr = "testReplayEditsWrittenViaHRegion"; + final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr); + final Path basedir = new Path(this.hbaseRootDir, tableNameStr); + deleteDir(basedir); + final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr); + HRegion region2 = HRegion.createHRegion(hri, + hbaseRootDir, this.conf, htd); + region2.close(); + region2.getLog().closeAndDelete(); + HLog wal = createWAL(this.conf); + HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf); + Path f = new Path(basedir, "hfile"); + HFile.Writer writer = + HFile.getWriterFactoryNoCache(conf).withPath(fs, f).create(); + byte [] family = htd.getFamilies().iterator().next().getName(); + byte [] row = Bytes.toBytes(tableNameStr); + writer.append(new KeyValue(row, family, family, row)); + writer.close(); + List > hfs= new ArrayList>(1); + hfs.add(Pair.newPair(family, f.toString())); + region.bulkLoadHFiles(hfs); + // Add an edit so something in the WAL + region.put((new Put(row)).add(family, family, family)); + wal.sync(); + + // Now 'crash' the region by stealing its wal + final Configuration newConf = HBaseConfiguration.create(this.conf); + User user = HBaseTestingUtility.getDifferentUser(newConf, + tableNameStr); + user.runAs(new PrivilegedExceptionAction() { + public Object run() throws Exception { + runWALSplit(newConf); + HLog wal2 = createWAL(newConf); + HRegion region2 = new HRegion(basedir, wal2, FileSystem.get(newConf), + newConf, hri, htd, null); + long seqid2 = region2.initialize(); + assertTrue(seqid2 > -1); + + // I can't close wal1. Its been appropriated when we split. + region2.close(); + wal2.closeAndDelete(); + return null; + } + }); + } + + /** + * Test writing edits into an HRegion, closing it, splitting logs, opening + * Region again. Verify seqids. + * @throws IOException + * @throws IllegalAccessException + * @throws NoSuchFieldException + * @throws IllegalArgumentException + * @throws SecurityException + */ + @Test + public void testReplayEditsWrittenViaHRegion() + throws IOException, SecurityException, IllegalArgumentException, + NoSuchFieldException, IllegalAccessException, InterruptedException { + final String tableNameStr = "testReplayEditsWrittenViaHRegion"; + final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr); + final Path basedir = new Path(this.hbaseRootDir, tableNameStr); + deleteDir(basedir); + final byte[] rowName = Bytes.toBytes(tableNameStr); + final int countPerFamily = 10; + final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr); + HRegion region3 = HRegion.createHRegion(hri, + hbaseRootDir, this.conf, htd); + region3.close(); + region3.getLog().closeAndDelete(); + // Write countPerFamily edits into the three families. Do a flush on one + // of the families during the load of edits so its seqid is not same as + // others to test we do right thing when different seqids. + HLog wal = createWAL(this.conf); + HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd, null); + long seqid = region.initialize(); + // HRegionServer usually does this. It knows the largest seqid across all regions. + wal.setSequenceNumber(seqid); + boolean first = true; + for (HColumnDescriptor hcd: htd.getFamilies()) { + addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x"); + if (first ) { + // If first, so we have at least one family w/ different seqid to rest. + region.flushcache(); + first = false; + } + } + // Now assert edits made it in. + final Get g = new Get(rowName); + Result result = region.get(g, null); + assertEquals(countPerFamily * htd.getFamilies().size(), + result.size()); + // Now close the region (without flush), split the log, reopen the region and assert that + // replay of log has the correct effect, that our seqids are calculated correctly so + // all edits in logs are seen as 'stale'/old. + region.close(true); + wal.close(); + runWALSplit(this.conf); + HLog wal2 = createWAL(this.conf); + HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd, null); + long seqid2 = region2.initialize(); + // HRegionServer usually does this. It knows the largest seqid across all regions. + wal2.setSequenceNumber(seqid2); + assertTrue(seqid + result.size() < seqid2); + final Result result1b = region2.get(g, null); + assertEquals(result.size(), result1b.size()); + + // Next test. Add more edits, then 'crash' this region by stealing its wal + // out from under it and assert that replay of the log adds the edits back + // correctly when region is opened again. + for (HColumnDescriptor hcd: htd.getFamilies()) { + addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y"); + } + // Get count of edits. + final Result result2 = region2.get(g, null); + assertEquals(2 * result.size(), result2.size()); + wal2.sync(); + // Set down maximum recovery so we dfsclient doesn't linger retrying something + // long gone. + HBaseTestingUtility.setMaxRecoveryErrorCount(wal2.getOutputStream(), 1); + final Configuration newConf = HBaseConfiguration.create(this.conf); + User user = HBaseTestingUtility.getDifferentUser(newConf, + tableNameStr); + user.runAs(new PrivilegedExceptionAction() { + public Object run() throws Exception { + runWALSplit(newConf); + FileSystem newFS = FileSystem.get(newConf); + // Make a new wal for new region open. + HLog wal3 = createWAL(newConf); + final AtomicInteger countOfRestoredEdits = new AtomicInteger(0); + HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) { + @Override + protected boolean restoreEdit(Store s, KeyValue kv) { + boolean b = super.restoreEdit(s, kv); + countOfRestoredEdits.incrementAndGet(); + return b; + } + }; + long seqid3 = region3.initialize(); + // HRegionServer usually does this. It knows the largest seqid across all regions. + wal3.setSequenceNumber(seqid3); + Result result3 = region3.get(g, null); + // Assert that count of cells is same as before crash. + assertEquals(result2.size(), result3.size()); + assertEquals(htd.getFamilies().size() * countPerFamily, + countOfRestoredEdits.get()); + + // I can't close wal1. Its been appropriated when we split. + region3.close(); + wal3.closeAndDelete(); + return null; + } + }); + } + + /** + * Test that we recover correctly when there is a failure in between the + * flushes. i.e. Some stores got flushed but others did not. + * + * Unfortunately, there is no easy hook to flush at a store level. The way + * we get around this is by flushing at the region level, and then deleting + * the recently flushed store file for one of the Stores. This would put us + * back in the situation where all but that store got flushed and the region + * died. + * + * We restart Region again, and verify that the edits were replayed. + * + * @throws IOException + * @throws IllegalAccessException + * @throws NoSuchFieldException + * @throws IllegalArgumentException + * @throws SecurityException + */ + @Test + public void testReplayEditsAfterPartialFlush() + throws IOException, SecurityException, IllegalArgumentException, + NoSuchFieldException, IllegalAccessException, InterruptedException { + final String tableNameStr = "testReplayEditsWrittenViaHRegion"; + final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr); + final Path basedir = new Path(this.hbaseRootDir, tableNameStr); + deleteDir(basedir); + final byte[] rowName = Bytes.toBytes(tableNameStr); + final int countPerFamily = 10; + final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr); + HRegion region3 = HRegion.createHRegion(hri, + hbaseRootDir, this.conf, htd); + region3.close(); + region3.getLog().closeAndDelete(); + // Write countPerFamily edits into the three families. Do a flush on one + // of the families during the load of edits so its seqid is not same as + // others to test we do right thing when different seqids. + HLog wal = createWAL(this.conf); + HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd, null); + long seqid = region.initialize(); + // HRegionServer usually does this. It knows the largest seqid across all regions. + wal.setSequenceNumber(seqid); + for (HColumnDescriptor hcd: htd.getFamilies()) { + addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x"); + } + + // Now assert edits made it in. + final Get g = new Get(rowName); + Result result = region.get(g, null); + assertEquals(countPerFamily * htd.getFamilies().size(), + result.size()); + + // Let us flush the region + region.flushcache(); + region.close(true); + wal.close(); + + // delete the store files in the second column family to simulate a failure + // in between the flushcache(); + // we have 3 families. killing the middle one ensures that taking the maximum + // will make us fail. + int cf_count = 0; + for (HColumnDescriptor hcd: htd.getFamilies()) { + cf_count++; + if (cf_count == 2) { + this.fs.delete(new Path(region.getRegionDir(), Bytes.toString(hcd.getName())) + , true); + } + } + + + // Let us try to split and recover + runWALSplit(this.conf); + HLog wal2 = createWAL(this.conf); + HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd, null); + long seqid2 = region2.initialize(); + // HRegionServer usually does this. It knows the largest seqid across all regions. + wal2.setSequenceNumber(seqid2); + assertTrue(seqid + result.size() < seqid2); + + final Result result1b = region2.get(g, null); + assertEquals(result.size(), result1b.size()); + } + + /** + * Test that we could recover the data correctly after aborting flush. In the + * test, first we abort flush after writing some data, then writing more data + * and flush again, at last verify the data. + * @throws IOException + */ + @Test + public void testReplayEditsAfterAbortingFlush() throws IOException { + final String tableNameStr = "testReplayEditsAfterAbortingFlush"; + final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr); + final Path basedir = new Path(this.hbaseRootDir, tableNameStr); + deleteDir(basedir); + final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr); + HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd); + region3.close(); + region3.getLog().closeAndDelete(); + // Write countPerFamily edits into the three families. Do a flush on one + // of the families during the load of edits so its seqid is not same as + // others to test we do right thing when different seqids. + HLog wal = createWAL(this.conf); + final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false); + RegionServerServices rsServices = Mockito.mock(RegionServerServices.class); + Mockito.doReturn(false).when(rsServices).isAborted(); + HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd, + rsServices) { + @Override + protected Store instantiateHStore(Path tableDir, HColumnDescriptor c) + throws IOException { + return new Store(tableDir, this, c, fs, conf) { + @Override + protected Path flushCache(final long logCacheFlushId, + SortedSet snapshot, + TimeRangeTracker snapshotTimeRangeTracker, + AtomicLong flushedSize, MonitoredTask status) throws IOException { + if (throwExceptionWhenFlushing.get()) { + throw new IOException("Simulated exception by tests"); + } + return super.flushCache(logCacheFlushId, snapshot, + snapshotTimeRangeTracker, flushedSize, status); + } + }; + } + }; + long seqid = region.initialize(); + // HRegionServer usually does this. It knows the largest seqid across all + // regions. + wal.setSequenceNumber(seqid); + + int writtenRowCount = 10; + List families = new ArrayList( + htd.getFamilies()); + for (int i = 0; i < writtenRowCount; i++) { + Put put = new Put(Bytes.toBytes(tableNameStr + Integer.toString(i))); + put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"), + Bytes.toBytes("val")); + region.put(put); + } + + // Now assert edits made it in. + RegionScanner scanner = region.getScanner(new Scan()); + assertEquals(writtenRowCount, getScannedCount(scanner)); + + // Let us flush the region + throwExceptionWhenFlushing.set(true); + try { + region.flushcache(); + fail("Injected exception hasn't been thrown"); + } catch (Throwable t) { + LOG.info("Expected simulated exception when flushing region," + + t.getMessage()); + // simulated to abort server + Mockito.doReturn(true).when(rsServices).isAborted(); + } + // writing more data + int moreRow = 10; + for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) { + Put put = new Put(Bytes.toBytes(tableNameStr + Integer.toString(i))); + put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"), + Bytes.toBytes("val")); + region.put(put); + } + writtenRowCount += moreRow; + // call flush again + throwExceptionWhenFlushing.set(false); + try { + region.flushcache(); + } catch (IOException t) { + LOG.info("Expected exception when flushing region because server is stopped," + + t.getMessage()); + } + + region.close(true); + wal.close(); + + // Let us try to split and recover + runWALSplit(this.conf); + HLog wal2 = createWAL(this.conf); + Mockito.doReturn(false).when(rsServices).isAborted(); + HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd, + rsServices); + long seqid2 = region2.initialize(); + // HRegionServer usually does this. It knows the largest seqid across all + // regions. + wal2.setSequenceNumber(seqid2); + + scanner = region2.getScanner(new Scan()); + assertEquals(writtenRowCount, getScannedCount(scanner)); + } + + private int getScannedCount(RegionScanner scanner) throws IOException { + int scannedCount = 0; + List results = new ArrayList(); + while (true) { + boolean existMore = scanner.next(results); + if (!results.isEmpty()) + scannedCount++; + if (!existMore) + break; + results.clear(); + } + return scannedCount; + } + + /** + * Create an HRegion with the result of a HLog split and test we only see the + * good edits + * @throws Exception + */ + @Test + public void testReplayEditsWrittenIntoWAL() throws Exception { + final String tableNameStr = "testReplayEditsWrittenIntoWAL"; + final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr); + final Path basedir = new Path(hbaseRootDir, tableNameStr); + deleteDir(basedir); + fs.mkdirs(new Path(basedir, hri.getEncodedName())); + final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr); + HRegion region2 = HRegion.createHRegion(hri, + hbaseRootDir, this.conf, htd); + region2.close(); + region2.getLog().closeAndDelete(); + final HLog wal = createWAL(this.conf); + final byte[] tableName = Bytes.toBytes(tableNameStr); + final byte[] rowName = tableName; + final byte[] regionName = hri.getEncodedNameAsBytes(); + + // Add 1k to each family. + final int countPerFamily = 1000; + for (HColumnDescriptor hcd: htd.getFamilies()) { + addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, + ee, wal, htd); + } + + // Add a cache flush, shouldn't have any effect + long logSeqId = wal.startCacheFlush(regionName); + wal.completeCacheFlush(regionName, tableName, logSeqId, hri.isMetaRegion()); + + // Add an edit to another family, should be skipped. + WALEdit edit = new WALEdit(); + long now = ee.currentTimeMillis(); + edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName, + now, rowName)); + wal.append(hri, tableName, edit, now, htd); + + // Delete the c family to verify deletes make it over. + edit = new WALEdit(); + now = ee.currentTimeMillis(); + edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now, + KeyValue.Type.DeleteFamily)); + wal.append(hri, tableName, edit, now, htd); + + // Sync. + wal.sync(); + // Set down maximum recovery so we dfsclient doesn't linger retrying something + // long gone. + HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1); + // Make a new conf and a new fs for the splitter to run on so we can take + // over old wal. + final Configuration newConf = HBaseConfiguration.create(this.conf); + User user = HBaseTestingUtility.getDifferentUser(newConf, + ".replay.wal.secondtime"); + user.runAs(new PrivilegedExceptionAction() { + public Object run() throws Exception { + runWALSplit(newConf); + FileSystem newFS = FileSystem.get(newConf); + // 100k seems to make for about 4 flushes during HRegion#initialize. + newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100); + // Make a new wal for new region. + HLog newWal = createWAL(newConf); + final AtomicInteger flushcount = new AtomicInteger(0); + try { + final HRegion region = + new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) { + protected boolean internalFlushcache( + final HLog wal, final long myseqid, MonitoredTask status) + throws IOException { + LOG.info("InternalFlushCache Invoked"); + boolean b = super.internalFlushcache(wal, myseqid, + Mockito.mock(MonitoredTask.class)); + flushcount.incrementAndGet(); + return b; + }; + }; + long seqid = region.initialize(); + // We flushed during init. + assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0); + assertTrue(seqid > wal.getSequenceNumber()); + + Get get = new Get(rowName); + Result result = region.get(get, -1); + // Make sure we only see the good edits + assertEquals(countPerFamily * (htd.getFamilies().size() - 1), + result.size()); + region.close(); + } finally { + newWal.closeAndDelete(); + } + return null; + } + }); + } + + @Test + public void testSequentialEditLogSeqNum() throws IOException { + final String tableNameStr = "testSequentialEditLogSeqNum"; + final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr); + final Path basedir = new Path(this.hbaseRootDir, tableNameStr); + deleteDir(basedir); + final byte[] rowName = Bytes.toBytes(tableNameStr); + final int countPerFamily = 10; + final HTableDescriptor htd = createBasic1FamilyHTD(tableNameStr); + + // Mock the HLog + MockHLog wal = createMockWAL(this.conf); + + HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd, null); + long seqid = region.initialize(); + // HRegionServer usually does this. It knows the largest seqid across all + // regions. + wal.setSequenceNumber(seqid); + for (HColumnDescriptor hcd : htd.getFamilies()) { + addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x"); + } + // get the seq no after first set of entries. + long sequenceNumber = wal.getSequenceNumber(); + + // Let us flush the region + // But this time completeflushcache is not yet done + region.flushcache(); + for (HColumnDescriptor hcd : htd.getFamilies()) { + addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x"); + } + long lastestSeqNumber = wal.getSequenceNumber(); + // get the current seq no + wal.doCompleteCacheFlush = true; + // allow complete cache flush with the previous seq number got after first + // set of edits. + wal.completeCacheFlush(hri.getEncodedNameAsBytes(), hri.getTableName(), sequenceNumber, false); + wal.close(); + FileStatus[] listStatus = this.fs.listStatus(wal.getDir()); + HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0], this.fs, this.conf, + null); + FileStatus[] listStatus1 = this.fs.listStatus(new Path(hbaseRootDir + "/" + + tableNameStr + "/" + hri.getEncodedName() + "/recovered.edits")); + int editCount = 0; + for (FileStatus fileStatus : listStatus1) { + editCount = Integer.parseInt(fileStatus.getPath().getName()); + } + // The sequence number should be same + assertEquals( + "The sequence number of the recoverd.edits and the current edit seq should be same", + lastestSeqNumber, editCount); + } + + static class MockHLog extends HLog { + boolean doCompleteCacheFlush = false; + + public MockHLog(FileSystem fs, Path dir, Path oldLogDir, Configuration conf) throws IOException { + super(fs, dir, oldLogDir, conf); + } + + @Override + public void completeCacheFlush(byte[] encodedRegionName, byte[] tableName, long logSeqId, + boolean isMetaRegion) throws IOException { + if (!doCompleteCacheFlush) { + return; + } + super.completeCacheFlush(encodedRegionName, tableName, logSeqId, isMetaRegion); + } + } + + private HTableDescriptor createBasic1FamilyHTD(final String tableName) { + HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a")); + htd.addFamily(a); + return htd; + } + + private MockHLog createMockWAL(Configuration conf) throws IOException { + MockHLog wal = new MockHLog(FileSystem.get(conf), logDir, oldLogDir, conf); + // Set down maximum recovery so we dfsclient doesn't linger retrying something + // long gone. + HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1); + return wal; + } + + + // Flusher used in this test. Keep count of how often we are called and + // actually run the flush inside here. + class TestFlusher implements FlushRequester { + private int count = 0; + private HRegion r; + + @Override + public void requestFlush(HRegion region) { + count++; + try { + r.flushcache(); + } catch (IOException e) { + throw new RuntimeException("Exception flushing", e); + } + } + + @Override + public void requestDelayedFlush(HRegion region, long when) { + // TODO Auto-generated method stub + + } + } + + private void addWALEdits (final byte [] tableName, final HRegionInfo hri, + final byte [] rowName, final byte [] family, + final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd) + throws IOException { + String familyStr = Bytes.toString(family); + for (int j = 0; j < count; j++) { + byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); + byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); + WALEdit edit = new WALEdit(); + edit.add(new KeyValue(rowName, family, qualifierBytes, + ee.currentTimeMillis(), columnBytes)); + wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd); + } + } + + private void addRegionEdits (final byte [] rowName, final byte [] family, + final int count, EnvironmentEdge ee, final HRegion r, + final String qualifierPrefix) + throws IOException { + for (int j = 0; j < count; j++) { + byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j)); + Put p = new Put(rowName); + p.add(family, qualifier, ee.currentTimeMillis(), rowName); + r.put(p); + } + } + + /* + * Creates an HRI around an HTD that has tableName and three + * column families named 'a','b', and 'c'. + * @param tableName Name of table to use when we create HTableDescriptor. + */ + private HRegionInfo createBasic3FamilyHRegionInfo(final String tableName) { + return new HRegionInfo(Bytes.toBytes(tableName), null, null, false); + } + + /* + * Run the split. Verify only single split file made. + * @param c + * @return The single split file made + * @throws IOException + */ + private Path runWALSplit(final Configuration c) throws IOException { + FileSystem fs = FileSystem.get(c); + HLogSplitter logSplitter = HLogSplitter.createLogSplitter(c, + this.hbaseRootDir, this.logDir, this.oldLogDir, fs); + List splits = logSplitter.splitLog(); + // Split should generate only 1 file since there's only 1 region + assertEquals("splits=" + splits, 1, splits.size()); + // Make sure the file exists + assertTrue(fs.exists(splits.get(0))); + LOG.info("Split file=" + splits.get(0)); + return splits.get(0); + } + + /* + * @param c + * @return WAL with retries set down from 5 to 1 only. + * @throws IOException + */ + private HLog createWAL(final Configuration c) throws IOException { + HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c); + // Set down maximum recovery so we dfsclient doesn't linger retrying something + // long gone. + HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1); + return wal; + } + + private HTableDescriptor createBasic3FamilyHTD(final String tableName) { + HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a")); + htd.addFamily(a); + HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b")); + htd.addFamily(b); + HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c")); + htd.addFamily(c); + return htd; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayCompressed.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayCompressed.java new file mode 100644 index 0000000..7e57359 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayCompressed.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +/** + * Enables compression and runs the TestWALReplay tests. + */ +@Category(MediumTests.class) +public class TestWALReplayCompressed extends TestWALReplay { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TestWALReplay.setUpBeforeClass(); + Configuration conf = TestWALReplay.TEST_UTIL.getConfiguration(); + conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java b/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java new file mode 100644 index 0000000..2daf643 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java @@ -0,0 +1,84 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceInterface; +import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager; + +/** + * Source that does nothing at all, helpful to test ReplicationSourceManager + */ +public class ReplicationSourceDummy implements ReplicationSourceInterface { + + ReplicationSourceManager manager; + String peerClusterId; + Path currentPath; + + @Override + public void init(Configuration conf, FileSystem fs, + ReplicationSourceManager manager, Stoppable stopper, + AtomicBoolean replicating, String peerClusterId) + throws IOException { + this.manager = manager; + this.peerClusterId = peerClusterId; + } + + @Override + public void enqueueLog(Path log) { + this.currentPath = log; + } + + @Override + public Path getCurrentPath() { + return this.currentPath; + } + + @Override + public void startup() { + + } + + @Override + public void terminate(String reason) { + + } + + @Override + public void terminate(String reason, Exception e) { + + } + + @Override + public String getPeerClusterZnode() { + return peerClusterId; + } + + @Override + public String getPeerClusterId() { + return peerClusterId; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java new file mode 100644 index 0000000..6a0c2cc --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java @@ -0,0 +1,330 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestMasterReplication { + + private static final Log LOG = LogFactory.getLog(TestReplicationBase.class); + + private Configuration conf1; + private Configuration conf2; + private Configuration conf3; + + private HBaseTestingUtility utility1; + private HBaseTestingUtility utility2; + private HBaseTestingUtility utility3; + + private MiniZooKeeperCluster miniZK; + + private static final long SLEEP_TIME = 500; + private static final int NB_RETRIES = 10; + + private static final byte[] tableName = Bytes.toBytes("test"); + private static final byte[] famName = Bytes.toBytes("f"); + private static final byte[] row = Bytes.toBytes("row"); + private static final byte[] row1 = Bytes.toBytes("row1"); + private static final byte[] row2 = Bytes.toBytes("row2"); + private static final byte[] noRepfamName = Bytes.toBytes("norep"); + + private static final byte[] count = Bytes.toBytes("count"); + private static final byte[] put = Bytes.toBytes("put"); + private static final byte[] delete = Bytes.toBytes("delete"); + + private HTableDescriptor table; + + @Before + public void setUp() throws Exception { + conf1 = HBaseConfiguration.create(); + conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); + // smaller block size and capacity to trigger more operations + // and test them + conf1.setInt("hbase.regionserver.hlog.blocksize", 1024*20); + conf1.setInt("replication.source.size.capacity", 1024); + conf1.setLong("replication.source.sleepforretries", 100); + conf1.setInt("hbase.regionserver.maxlogs", 10); + conf1.setLong("hbase.master.logcleaner.ttl", 10); + conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true); + conf1.setBoolean("dfs.support.append", true); + conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); + conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, + CoprocessorCounter.class.getName()); + + utility1 = new HBaseTestingUtility(conf1); + utility1.startMiniZKCluster(); + miniZK = utility1.getZkCluster(); + // By setting the mini ZK cluster through this method, even though this is + // already utility1's mini ZK cluster, we are telling utility1 not to shut + // the mini ZK cluster when we shut down the HBase cluster. + utility1.setZkCluster(miniZK); + new ZooKeeperWatcher(conf1, "cluster1", null, true); + + conf2 = new Configuration(conf1); + conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); + + utility2 = new HBaseTestingUtility(conf2); + utility2.setZkCluster(miniZK); + new ZooKeeperWatcher(conf2, "cluster2", null, true); + + conf3 = new Configuration(conf1); + conf3.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3"); + + utility3 = new HBaseTestingUtility(conf3); + utility3.setZkCluster(miniZK); + new ZooKeeperWatcher(conf3, "cluster3", null, true); + + table = new HTableDescriptor(tableName); + HColumnDescriptor fam = new HColumnDescriptor(famName); + fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); + table.addFamily(fam); + fam = new HColumnDescriptor(noRepfamName); + table.addFamily(fam); + } + + @After + public void tearDown() throws IOException { + miniZK.shutdown(); + } + + @Test(timeout=300000) + public void testCyclicReplication() throws Exception { + LOG.info("testCyclicReplication"); + utility1.startMiniCluster(); + utility2.startMiniCluster(); + utility3.startMiniCluster(); + ReplicationAdmin admin1 = new ReplicationAdmin(conf1); + ReplicationAdmin admin2 = new ReplicationAdmin(conf2); + ReplicationAdmin admin3 = new ReplicationAdmin(conf3); + + new HBaseAdmin(conf1).createTable(table); + new HBaseAdmin(conf2).createTable(table); + new HBaseAdmin(conf3).createTable(table); + HTable htable1 = new HTable(conf1, tableName); + htable1.setWriteBufferSize(1024); + HTable htable2 = new HTable(conf2, tableName); + htable2.setWriteBufferSize(1024); + HTable htable3 = new HTable(conf3, tableName); + htable3.setWriteBufferSize(1024); + + admin1.addPeer("1", utility2.getClusterKey()); + admin2.addPeer("1", utility3.getClusterKey()); + admin3.addPeer("1", utility1.getClusterKey()); + + // put "row" and wait 'til it got around + putAndWait(row, famName, htable1, htable3); + // it should have passed through table2 + check(row,famName,htable2); + + putAndWait(row1, famName, htable2, htable1); + check(row,famName,htable3); + putAndWait(row2, famName, htable3, htable2); + check(row,famName,htable1); + + deleteAndWait(row,htable1,htable3); + deleteAndWait(row1,htable2,htable1); + deleteAndWait(row2,htable3,htable2); + + assertEquals("Puts were replicated back ", 3, getCount(htable1, put)); + assertEquals("Puts were replicated back ", 3, getCount(htable2, put)); + assertEquals("Puts were replicated back ", 3, getCount(htable3, put)); + assertEquals("Deletes were replicated back ", 3, getCount(htable1, delete)); + assertEquals("Deletes were replicated back ", 3, getCount(htable2, delete)); + assertEquals("Deletes were replicated back ", 3, getCount(htable3, delete)); + utility3.shutdownMiniCluster(); + utility2.shutdownMiniCluster(); + utility1.shutdownMiniCluster(); + } + + /** + * Add a row to a table in each cluster, check it's replicated, + * delete it, check's gone + * Also check the puts and deletes are not replicated back to + * the originating cluster. + */ + @Test(timeout=300000) + public void testSimplePutDelete() throws Exception { + LOG.info("testSimplePutDelete"); + utility1.startMiniCluster(); + utility2.startMiniCluster(); + + ReplicationAdmin admin1 = new ReplicationAdmin(conf1); + ReplicationAdmin admin2 = new ReplicationAdmin(conf2); + + new HBaseAdmin(conf1).createTable(table); + new HBaseAdmin(conf2).createTable(table); + HTable htable1 = new HTable(conf1, tableName); + htable1.setWriteBufferSize(1024); + HTable htable2 = new HTable(conf2, tableName); + htable2.setWriteBufferSize(1024); + + // set M-M + admin1.addPeer("1", utility2.getClusterKey()); + admin2.addPeer("1", utility1.getClusterKey()); + + // add rows to both clusters, + // make sure they are both replication + putAndWait(row, famName, htable1, htable2); + putAndWait(row1, famName, htable2, htable1); + + // make sure "row" did not get replicated back. + assertEquals("Puts were replicated back ", 2, getCount(htable1, put)); + + // delete "row" and wait + deleteAndWait(row, htable1, htable2); + + // make the 2nd cluster replicated back + assertEquals("Puts were replicated back ", 2, getCount(htable2, put)); + + deleteAndWait(row1, htable2, htable1); + + assertEquals("Deletes were replicated back ", 2, getCount(htable1, delete)); + utility2.shutdownMiniCluster(); + utility1.shutdownMiniCluster(); + } + + private int getCount(HTable t, byte[] type) throws IOException { + Get test = new Get(row); + test.setAttribute("count", new byte[]{}); + Result res = t.get(test); + return Bytes.toInt(res.getValue(count, type)); + } + + private void deleteAndWait(byte[] row, HTable source, HTable target) + throws Exception { + Delete del = new Delete(row); + source.delete(del); + + Get get = new Get(row); + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for del replication"); + } + Result res = target.get(get); + if (res.size() >= 1) { + LOG.info("Row not deleted"); + Thread.sleep(SLEEP_TIME); + } else { + break; + } + } + } + + private void check(byte[] row, byte[] fam, HTable t) throws IOException { + Get get = new Get(row); + Result res = t.get(get); + if (res.size() == 0) { + fail("Row is missing"); + } + } + + private void putAndWait(byte[] row, byte[] fam, HTable source, HTable target) + throws Exception { + Put put = new Put(row); + put.add(fam, row, row); + source.put(put); + + Get get = new Get(row); + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for put replication"); + } + Result res = target.get(get); + if (res.size() == 0) { + LOG.info("Row not available"); + Thread.sleep(SLEEP_TIME); + } else { + assertArrayEquals(res.value(), row); + break; + } + } + } + + /** + * Use a coprocessor to count puts and deletes. + * as KVs would be replicated back with the same timestamp + * there is otherwise no way to count them. + */ + public static class CoprocessorCounter extends BaseRegionObserver { + private int nCount = 0; + private int nDelete = 0; + + @Override + public void prePut(final ObserverContext e, + final Put put, final WALEdit edit, + final boolean writeToWAL) + throws IOException { + nCount++; + } + @Override + public void postDelete(final ObserverContext c, + final Delete delete, final WALEdit edit, + final boolean writeToWAL) + throws IOException { + nDelete++; + } + @Override + public void preGet(final ObserverContext c, + final Get get, final List result) throws IOException { + if (get.getAttribute("count") != null) { + result.clear(); + // order is important! + result.add(new KeyValue(count, count, delete, Bytes.toBytes(nDelete))); + result.add(new KeyValue(count, count, put, Bytes.toBytes(nCount))); + c.bypass(); + } + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java b/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java new file mode 100644 index 0000000..1672d97 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java @@ -0,0 +1,281 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestMultiSlaveReplication { + + private static final Log LOG = LogFactory.getLog(TestReplicationBase.class); + + private static Configuration conf1; + private static Configuration conf2; + private static Configuration conf3; + + private static HBaseTestingUtility utility1; + private static HBaseTestingUtility utility2; + private static HBaseTestingUtility utility3; + private static final long SLEEP_TIME = 500; + private static final int NB_RETRIES = 10; + + private static final byte[] tableName = Bytes.toBytes("test"); + private static final byte[] famName = Bytes.toBytes("f"); + private static final byte[] row = Bytes.toBytes("row"); + private static final byte[] row1 = Bytes.toBytes("row1"); + private static final byte[] row2 = Bytes.toBytes("row2"); + private static final byte[] row3 = Bytes.toBytes("row3"); + private static final byte[] noRepfamName = Bytes.toBytes("norep"); + + private static HTableDescriptor table; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf1 = HBaseConfiguration.create(); + conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); + // smaller block size and capacity to trigger more operations + // and test them + conf1.setInt("hbase.regionserver.hlog.blocksize", 1024*20); + conf1.setInt("replication.source.size.capacity", 1024); + conf1.setLong("replication.source.sleepforretries", 100); + conf1.setInt("hbase.regionserver.maxlogs", 10); + conf1.setLong("hbase.master.logcleaner.ttl", 10); + conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true); + conf1.setBoolean("dfs.support.append", true); + conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); + conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, + "org.apache.hadoop.hbase.replication.TestMasterReplication$CoprocessorCounter"); + + utility1 = new HBaseTestingUtility(conf1); + utility1.startMiniZKCluster(); + MiniZooKeeperCluster miniZK = utility1.getZkCluster(); + new ZooKeeperWatcher(conf1, "cluster1", null, true); + + conf2 = new Configuration(conf1); + conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); + + conf3 = new Configuration(conf1); + conf3.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3"); + + utility2 = new HBaseTestingUtility(conf2); + utility2.setZkCluster(miniZK); + new ZooKeeperWatcher(conf2, "cluster3", null, true); + + utility3 = new HBaseTestingUtility(conf3); + utility3.setZkCluster(miniZK); + new ZooKeeperWatcher(conf3, "cluster3", null, true); + + table = new HTableDescriptor(tableName); + HColumnDescriptor fam = new HColumnDescriptor(famName); + fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); + table.addFamily(fam); + fam = new HColumnDescriptor(noRepfamName); + table.addFamily(fam); + } + + @Test(timeout=300000) + public void testMultiSlaveReplication() throws Exception { + LOG.info("testCyclicReplication"); + MiniHBaseCluster master = utility1.startMiniCluster(); + utility2.startMiniCluster(); + utility3.startMiniCluster(); + ReplicationAdmin admin1 = new ReplicationAdmin(conf1); + + new HBaseAdmin(conf1).createTable(table); + new HBaseAdmin(conf2).createTable(table); + new HBaseAdmin(conf3).createTable(table); + HTable htable1 = new HTable(conf1, tableName); + htable1.setWriteBufferSize(1024); + HTable htable2 = new HTable(conf2, tableName); + htable2.setWriteBufferSize(1024); + HTable htable3 = new HTable(conf3, tableName); + htable3.setWriteBufferSize(1024); + + admin1.addPeer("1", utility2.getClusterKey()); + + // put "row" and wait 'til it got around, then delete + putAndWait(row, famName, htable1, htable2); + deleteAndWait(row, htable1, htable2); + // check it wasn't replication to cluster 3 + checkRow(row,0,htable3); + + putAndWait(row2, famName, htable1, htable2); + + // now roll the region server's logs + new HBaseAdmin(conf1).rollHLogWriter(master.getRegionServer(0).getServerName().toString()); + // after the log was rolled put a new row + putAndWait(row3, famName, htable1, htable2); + + admin1.addPeer("2", utility3.getClusterKey()); + + // put a row, check it was replicated to all clusters + putAndWait(row1, famName, htable1, htable2, htable3); + // delete and verify + deleteAndWait(row1, htable1, htable2, htable3); + + // make sure row2 did not get replicated after + // cluster 3 was added + checkRow(row2,0,htable3); + + // row3 will get replicated, because it was in the + // latest log + checkRow(row3,1,htable3); + + Put p = new Put(row); + p.add(famName, row, row); + htable1.put(p); + // now roll the logs again + new HBaseAdmin(conf1).rollHLogWriter(master.getRegionServer(0) + .getServerName().toString()); + + // cleanup "row2", also conveniently use this to wait replication + // to finish + deleteAndWait(row2, htable1, htable2, htable3); + // Even if the log was rolled in the middle of the replication + // "row" is still replication. + checkRow(row, 1, htable2); + // Replication thread of cluster 2 may be sleeping, and since row2 is not there in it, + // we should wait before checking. + checkWithWait(row, 1, htable3); + + // cleanup the rest + deleteAndWait(row, htable1, htable2, htable3); + deleteAndWait(row3, htable1, htable2, htable3); + + utility3.shutdownMiniCluster(); + utility2.shutdownMiniCluster(); + utility1.shutdownMiniCluster(); + } + + private void checkWithWait(byte[] row, int count, HTable table) throws Exception { + Get get = new Get(row); + for (int i = 0; i < NB_RETRIES; i++) { + if (i == NB_RETRIES - 1) { + fail("Waited too much time while getting the row."); + } + boolean rowReplicated = false; + Result res = table.get(get); + if (res.size() >= 1) { + LOG.info("Row is replicated"); + rowReplicated = true; + assertEquals(count, res.size()); + break; + } + if (rowReplicated) { + break; + } else { + Thread.sleep(SLEEP_TIME); + } + } + } + + private void checkRow(byte[] row, int count, HTable... tables) throws IOException { + Get get = new Get(row); + for (HTable table : tables) { + Result res = table.get(get); + assertEquals(count, res.size()); + } + } + + private void deleteAndWait(byte[] row, HTable source, HTable... targets) + throws Exception { + Delete del = new Delete(row); + source.delete(del); + + Get get = new Get(row); + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for del replication"); + } + boolean removedFromAll = true; + for (HTable target : targets) { + Result res = target.get(get); + if (res.size() >= 1) { + LOG.info("Row not deleted"); + removedFromAll = false; + break; + } + } + if (removedFromAll) { + break; + } else { + Thread.sleep(SLEEP_TIME); + } + } + } + + private void putAndWait(byte[] row, byte[] fam, HTable source, HTable... targets) + throws Exception { + Put put = new Put(row); + put.add(fam, row, row); + source.put(put); + + Get get = new Get(row); + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for put replication"); + } + boolean replicatedToAll = true; + for (HTable target : targets) { + Result res = target.get(get); + if (res.size() == 0) { + LOG.info("Row not available"); + replicatedToAll = false; + break; + } else { + assertArrayEquals(res.value(), row); + } + } + if (replicatedToAll) { + break; + } else { + Thread.sleep(SLEEP_TIME); + } + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java new file mode 100644 index 0000000..aabaedc --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java @@ -0,0 +1,154 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** + * This class is only a base for other integration-level replication tests. + * Do not add tests here. + * TestReplicationSmallTests is where tests that don't require bring machines up/down should go + * All other tests should have their own classes and extend this one + */ +public class TestReplicationBase { + + private static final Log LOG = LogFactory.getLog(TestReplicationBase.class); + + protected static Configuration conf1 = HBaseConfiguration.create(); + protected static Configuration conf2; + protected static Configuration CONF_WITH_LOCALFS; + + protected static ZooKeeperWatcher zkw1; + protected static ZooKeeperWatcher zkw2; + + protected static ReplicationAdmin admin; + + protected static HTable htable1; + protected static HTable htable2; + + protected static HBaseTestingUtility utility1; + protected static HBaseTestingUtility utility2; + protected static final int NB_ROWS_IN_BATCH = 100; + protected static final int NB_ROWS_IN_BIG_BATCH = + NB_ROWS_IN_BATCH * 10; + protected static final long SLEEP_TIME = 1000; + protected static final int NB_RETRIES = 15; + protected static final int NB_RETRIES_FOR_BIG_BATCH = 30; + + protected static final byte[] tableName = Bytes.toBytes("test"); + protected static final byte[] famName = Bytes.toBytes("f"); + protected static final byte[] row = Bytes.toBytes("row"); + protected static final byte[] noRepfamName = Bytes.toBytes("norep"); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); + // smaller log roll size to trigger more events + conf1.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f); + conf1.setInt("replication.source.size.capacity", 10240); + conf1.setLong("replication.source.sleepforretries", 100); + conf1.setInt("hbase.regionserver.maxlogs", 10); + conf1.setLong("hbase.master.logcleaner.ttl", 10); + conf1.setInt("zookeeper.recovery.retry", 1); + conf1.setInt("zookeeper.recovery.retry.intervalmill", 10); + conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true); + conf1.setBoolean("dfs.support.append", true); + conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); + conf1.setInt("replication.stats.thread.period.seconds", 5); + + utility1 = new HBaseTestingUtility(conf1); + utility1.startMiniZKCluster(); + MiniZooKeeperCluster miniZK = utility1.getZkCluster(); + // Have to reget conf1 in case zk cluster location different + // than default + conf1 = utility1.getConfiguration(); + zkw1 = new ZooKeeperWatcher(conf1, "cluster1", null, true); + admin = new ReplicationAdmin(conf1); + LOG.info("Setup first Zk"); + + // Base conf2 on conf1 so it gets the right zk cluster. + conf2 = HBaseConfiguration.create(conf1); + conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); + conf2.setInt("hbase.client.retries.number", 6); + conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true); + conf2.setBoolean("dfs.support.append", true); + + utility2 = new HBaseTestingUtility(conf2); + utility2.setZkCluster(miniZK); + zkw2 = new ZooKeeperWatcher(conf2, "cluster2", null, true); + + admin.addPeer("2", utility2.getClusterKey()); + setIsReplication(true); + + LOG.info("Setup second Zk"); + CONF_WITH_LOCALFS = HBaseConfiguration.create(conf1); + utility1.startMiniCluster(2); + utility2.startMiniCluster(2); + + HTableDescriptor table = new HTableDescriptor(tableName); + HColumnDescriptor fam = new HColumnDescriptor(famName); + fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); + table.addFamily(fam); + fam = new HColumnDescriptor(noRepfamName); + table.addFamily(fam); + HBaseAdmin admin1 = new HBaseAdmin(conf1); + HBaseAdmin admin2 = new HBaseAdmin(conf2); + admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); + admin2.createTable(table); + htable1 = new HTable(conf1, tableName); + htable1.setWriteBufferSize(1024); + htable2 = new HTable(conf2, tableName); + } + + protected static void setIsReplication(boolean rep) throws Exception { + LOG.info("Set rep " + rep); + admin.setReplicating(rep); + Thread.sleep(SLEEP_TIME); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + utility2.shutdownMiniCluster(); + utility1.shutdownMiniCluster(); + } + + +} + diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java new file mode 100644 index 0000000..b089fbe --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java @@ -0,0 +1,92 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.fail; + +@Category(LargeTests.class) +public class TestReplicationDisableInactivePeer extends TestReplicationBase { + + private static final Log LOG = LogFactory.getLog(TestReplicationDisableInactivePeer.class); + + /** + * Test disabling an inactive peer. Add a peer which is inactive, trying to + * insert, disable the peer, then activate the peer and make sure nothing is + * replicated. In Addition, enable the peer and check the updates are + * replicated. + * + * @throws Exception + */ + @Test(timeout = 600000) + public void testDisableInactivePeer() throws Exception { + + // enabling and shutdown the peer + admin.enablePeer("2"); + utility2.shutdownMiniHBaseCluster(); + + byte[] rowkey = Bytes.toBytes("disable inactive peer"); + Put put = new Put(rowkey); + put.add(famName, row, row); + htable1.put(put); + + // wait for the sleep interval of the master cluster to become long + Thread.sleep(SLEEP_TIME * NB_RETRIES); + + // disable and start the peer + admin.disablePeer("2"); + utility2.startMiniHBaseCluster(1, 2); + Get get = new Get(rowkey); + for (int i = 0; i < NB_RETRIES; i++) { + Result res = htable2.get(get); + if (res.size() >= 1) { + fail("Replication wasn't disabled"); + } else { + LOG.info("Row not replicated, let's wait a bit more..."); + Thread.sleep(SLEEP_TIME); + } + } + + // Test enable replication + admin.enablePeer("2"); + // wait since the sleep interval would be long + Thread.sleep(SLEEP_TIME * NB_RETRIES); + for (int i = 0; i < NB_RETRIES; i++) { + Result res = htable2.get(get); + if (res.size() == 0) { + LOG.info("Row not available"); + Thread.sleep(SLEEP_TIME * NB_RETRIES); + } else { + assertArrayEquals(res.value(), row); + return; + } + } + fail("Waited too much time for put replication"); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailover.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailover.java new file mode 100644 index 0000000..9e7e0f5 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailover.java @@ -0,0 +1,133 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.UnknownScannerException; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.fail; + +@Category(LargeTests.class) +public class TestReplicationQueueFailover extends TestReplicationBase { + + private static final Log LOG = LogFactory.getLog(TestReplicationQueueFailover.class); + + /** + * Load up multiple tables over 2 region servers and kill a source during + * the upload. The failover happens internally. + * + * WARNING this test sometimes fails because of HBASE-3515 + * + * @throws Exception + */ + @Test(timeout=300000) + public void queueFailover() throws Exception { + // killing the RS with .META. can result into failed puts until we solve + // IO fencing + int rsToKill1 = + utility1.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0; + int rsToKill2 = + utility2.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0; + + // Takes about 20 secs to run the full loading, kill around the middle + Thread killer1 = killARegionServer(utility1, 7500, rsToKill1); + Thread killer2 = killARegionServer(utility2, 10000, rsToKill2); + + LOG.info("Start loading table"); + int initialCount = utility1.loadTable(htable1, famName); + LOG.info("Done loading table"); + killer1.join(5000); + killer2.join(5000); + LOG.info("Done waiting for threads"); + + Result[] res; + while (true) { + try { + Scan scan = new Scan(); + ResultScanner scanner = htable1.getScanner(scan); + res = scanner.next(initialCount); + scanner.close(); + break; + } catch (UnknownScannerException ex) { + LOG.info("Cluster wasn't ready yet, restarting scanner"); + } + } + // Test we actually have all the rows, we may miss some because we + // don't have IO fencing. + if (res.length != initialCount) { + LOG.warn("We lost some rows on the master cluster!"); + // We don't really expect the other cluster to have more rows + initialCount = res.length; + } + + int lastCount = 0; + + final long start = System.currentTimeMillis(); + int i = 0; + while (true) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for queueFailover replication. " + + "Waited "+(System.currentTimeMillis() - start)+"ms."); + } + Scan scan2 = new Scan(); + ResultScanner scanner2 = htable2.getScanner(scan2); + Result[] res2 = scanner2.next(initialCount * 2); + scanner2.close(); + if (res2.length < initialCount) { + if (lastCount < res2.length) { + i--; // Don't increment timeout if we make progress + } else { + i++; + } + lastCount = res2.length; + LOG.info("Only got " + lastCount + " rows instead of " + + initialCount + " current i=" + i); + Thread.sleep(SLEEP_TIME*2); + } else { + break; + } + } + } + + private static Thread killARegionServer(final HBaseTestingUtility utility, + final long timeout, final int rs) { + Thread killer = new Thread() { + public void run() { + try { + Thread.sleep(timeout); + utility.expireRegionServerSession(rs); + } catch (Exception e) { + LOG.error("Couldn't kill a region server", e); + } + } + }; + killer.setDaemon(true); + killer.start(); + return killer; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailoverCompressed.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailoverCompressed.java new file mode 100644 index 0000000..35c0715 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailoverCompressed.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.LargeTests; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +/** + * Run the same test as TestReplication but with HLog compression enabled + */ +@Category(LargeTests.class) +public class TestReplicationQueueFailoverCompressed extends TestReplicationQueueFailover { + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf1.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); + TestReplicationBase.setUpBeforeClass(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java new file mode 100644 index 0000000..7da487a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java @@ -0,0 +1,530 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.apache.hadoop.mapreduce.Job; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@Category(LargeTests.class) +public class TestReplicationSmallTests extends TestReplicationBase { + + private static final Log LOG = LogFactory.getLog(TestReplicationSmallTests.class); + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + htable1.setAutoFlush(true); + // Starting and stopping replication can make us miss new logs, + // rolling like this makes sure the most recent one gets added to the queue + for ( JVMClusterUtil.RegionServerThread r : + utility1.getHBaseCluster().getRegionServerThreads()) { + r.getRegionServer().getWAL().rollWriter(); + } + utility1.truncateTable(tableName); + // truncating the table will send one Delete per row to the slave cluster + // in an async fashion, which is why we cannot just call truncateTable on + // utility2 since late writes could make it to the slave in some way. + // Instead, we truncate the first table and wait for all the Deletes to + // make it to the slave. + Scan scan = new Scan(); + int lastCount = 0; + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for truncate"); + } + ResultScanner scanner = htable2.getScanner(scan); + Result[] res = scanner.next(NB_ROWS_IN_BIG_BATCH); + scanner.close(); + if (res.length != 0) { + if (res.length < lastCount) { + i--; // Don't increment timeout if we make progress + } + lastCount = res.length; + LOG.info("Still got " + res.length + " rows"); + Thread.sleep(SLEEP_TIME); + } else { + break; + } + } + } + + /** + * Verify that version and column delete marker types are replicated + * correctly. + * @throws Exception + */ + @Test(timeout=300000) + public void testDeleteTypes() throws Exception { + LOG.info("testDeleteTypes"); + final byte[] v1 = Bytes.toBytes("v1"); + final byte[] v2 = Bytes.toBytes("v2"); + final byte[] v3 = Bytes.toBytes("v3"); + htable1 = new HTable(conf1, tableName); + + long t = EnvironmentEdgeManager.currentTimeMillis(); + // create three versions for "row" + Put put = new Put(row); + put.add(famName, row, t, v1); + htable1.put(put); + + put = new Put(row); + put.add(famName, row, t+1, v2); + htable1.put(put); + + put = new Put(row); + put.add(famName, row, t+2, v3); + htable1.put(put); + + Get get = new Get(row); + get.setMaxVersions(); + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for put replication"); + } + Result res = htable2.get(get); + if (res.size() < 3) { + LOG.info("Rows not available"); + Thread.sleep(SLEEP_TIME); + } else { + assertArrayEquals(res.raw()[0].getValue(), v3); + assertArrayEquals(res.raw()[1].getValue(), v2); + assertArrayEquals(res.raw()[2].getValue(), v1); + break; + } + } + // place a version delete marker (delete last version) + Delete d = new Delete(row); + d.deleteColumn(famName, row, t); + htable1.delete(d); + + get = new Get(row); + get.setMaxVersions(); + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for put replication"); + } + Result res = htable2.get(get); + if (res.size() > 2) { + LOG.info("Version not deleted"); + Thread.sleep(SLEEP_TIME); + } else { + assertArrayEquals(res.raw()[0].getValue(), v3); + assertArrayEquals(res.raw()[1].getValue(), v2); + break; + } + } + + // place a column delete marker + d = new Delete(row); + d.deleteColumns(famName, row, t+2); + htable1.delete(d); + + // now *both* of the remaining version should be deleted + // at the replica + get = new Get(row); + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for del replication"); + } + Result res = htable2.get(get); + if (res.size() >= 1) { + LOG.info("Rows not deleted"); + Thread.sleep(SLEEP_TIME); + } else { + break; + } + } + } + + /** + * Add a row, check it's replicated, delete it, check's gone + * @throws Exception + */ + @Test(timeout=300000) + public void testSimplePutDelete() throws Exception { + LOG.info("testSimplePutDelete"); + Put put = new Put(row); + put.add(famName, row, row); + + htable1 = new HTable(conf1, tableName); + htable1.put(put); + + Get get = new Get(row); + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for put replication"); + } + Result res = htable2.get(get); + if (res.size() == 0) { + LOG.info("Row not available"); + Thread.sleep(SLEEP_TIME); + } else { + assertArrayEquals(res.value(), row); + break; + } + } + + Delete del = new Delete(row); + htable1.delete(del); + + get = new Get(row); + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for del replication"); + } + Result res = htable2.get(get); + if (res.size() >= 1) { + LOG.info("Row not deleted"); + Thread.sleep(SLEEP_TIME); + } else { + break; + } + } + } + + /** + * Try a small batch upload using the write buffer, check it's replicated + * @throws Exception + */ + @Test(timeout=300000) + public void testSmallBatch() throws Exception { + LOG.info("testSmallBatch"); + Put put; + // normal Batch tests + htable1.setAutoFlush(false); + for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { + put = new Put(Bytes.toBytes(i)); + put.add(famName, row, row); + htable1.put(put); + } + htable1.flushCommits(); + + Scan scan = new Scan(); + + ResultScanner scanner1 = htable1.getScanner(scan); + Result[] res1 = scanner1.next(NB_ROWS_IN_BATCH); + scanner1.close(); + assertEquals(NB_ROWS_IN_BATCH, res1.length); + + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for normal batch replication"); + } + ResultScanner scanner = htable2.getScanner(scan); + Result[] res = scanner.next(NB_ROWS_IN_BATCH); + scanner.close(); + if (res.length != NB_ROWS_IN_BATCH) { + LOG.info("Only got " + res.length + " rows"); + Thread.sleep(SLEEP_TIME); + } else { + break; + } + } + } + + /** + * Test stopping replication, trying to insert, make sure nothing's + * replicated, enable it, try replicating and it should work + * @throws Exception + */ + @Test(timeout=300000) + public void testStartStop() throws Exception { + + // Test stopping replication + setIsReplication(false); + + Put put = new Put(Bytes.toBytes("stop start")); + put.add(famName, row, row); + htable1.put(put); + + Get get = new Get(Bytes.toBytes("stop start")); + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + break; + } + Result res = htable2.get(get); + if(res.size() >= 1) { + fail("Replication wasn't stopped"); + + } else { + LOG.info("Row not replicated, let's wait a bit more..."); + Thread.sleep(SLEEP_TIME); + } + } + + // Test restart replication + setIsReplication(true); + + htable1.put(put); + + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for put replication"); + } + Result res = htable2.get(get); + if(res.size() == 0) { + LOG.info("Row not available"); + Thread.sleep(SLEEP_TIME); + } else { + assertArrayEquals(res.value(), row); + break; + } + } + + put = new Put(Bytes.toBytes("do not rep")); + put.add(noRepfamName, row, row); + htable1.put(put); + + get = new Get(Bytes.toBytes("do not rep")); + for (int i = 0; i < NB_RETRIES; i++) { + if (i == NB_RETRIES-1) { + break; + } + Result res = htable2.get(get); + if (res.size() >= 1) { + fail("Not supposed to be replicated"); + } else { + LOG.info("Row not replicated, let's wait a bit more..."); + Thread.sleep(SLEEP_TIME); + } + } + + } + + /** + * Test disable/enable replication, trying to insert, make sure nothing's + * replicated, enable it, the insert should be replicated + * + * @throws Exception + */ + @Test(timeout = 300000) + public void testDisableEnable() throws Exception { + + // Test disabling replication + admin.disablePeer("2"); + + byte[] rowkey = Bytes.toBytes("disable enable"); + Put put = new Put(rowkey); + put.add(famName, row, row); + htable1.put(put); + + Get get = new Get(rowkey); + for (int i = 0; i < NB_RETRIES; i++) { + Result res = htable2.get(get); + if (res.size() >= 1) { + fail("Replication wasn't disabled"); + } else { + LOG.info("Row not replicated, let's wait a bit more..."); + Thread.sleep(SLEEP_TIME); + } + } + + // Test enable replication + admin.enablePeer("2"); + + for (int i = 0; i < NB_RETRIES; i++) { + Result res = htable2.get(get); + if (res.size() == 0) { + LOG.info("Row not available"); + Thread.sleep(SLEEP_TIME); + } else { + assertArrayEquals(res.value(), row); + return; + } + } + fail("Waited too much time for put replication"); + } + + /** + * Integration test for TestReplicationAdmin, removes and re-add a peer + * cluster + * + * @throws Exception + */ + @Test(timeout=300000) + public void testAddAndRemoveClusters() throws Exception { + LOG.info("testAddAndRemoveClusters"); + admin.removePeer("2"); + Thread.sleep(SLEEP_TIME); + byte[] rowKey = Bytes.toBytes("Won't be replicated"); + Put put = new Put(rowKey); + put.add(famName, row, row); + htable1.put(put); + + Get get = new Get(rowKey); + for (int i = 0; i < NB_RETRIES; i++) { + if (i == NB_RETRIES-1) { + break; + } + Result res = htable2.get(get); + if (res.size() >= 1) { + fail("Not supposed to be replicated"); + } else { + LOG.info("Row not replicated, let's wait a bit more..."); + Thread.sleep(SLEEP_TIME); + } + } + + admin.addPeer("2", utility2.getClusterKey()); + Thread.sleep(SLEEP_TIME); + rowKey = Bytes.toBytes("do rep"); + put = new Put(rowKey); + put.add(famName, row, row); + LOG.info("Adding new row"); + htable1.put(put); + + get = new Get(rowKey); + for (int i = 0; i < NB_RETRIES; i++) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for put replication"); + } + Result res = htable2.get(get); + if (res.size() == 0) { + LOG.info("Row not available"); + Thread.sleep(SLEEP_TIME*i); + } else { + assertArrayEquals(res.value(), row); + break; + } + } + } + + + /** + * Do a more intense version testSmallBatch, one that will trigger + * hlog rolling and other non-trivial code paths + * @throws Exception + */ + @Test(timeout=300000) + public void loadTesting() throws Exception { + htable1.setWriteBufferSize(1024); + htable1.setAutoFlush(false); + for (int i = 0; i < NB_ROWS_IN_BIG_BATCH; i++) { + Put put = new Put(Bytes.toBytes(i)); + put.add(famName, row, row); + htable1.put(put); + } + htable1.flushCommits(); + + Scan scan = new Scan(); + + ResultScanner scanner = htable1.getScanner(scan); + Result[] res = scanner.next(NB_ROWS_IN_BIG_BATCH); + scanner.close(); + + assertEquals(NB_ROWS_IN_BIG_BATCH, res.length); + + scan = new Scan(); + + for (int i = 0; i < NB_RETRIES_FOR_BIG_BATCH; i++) { + + scanner = htable2.getScanner(scan); + res = scanner.next(NB_ROWS_IN_BIG_BATCH); + scanner.close(); + if (res.length != NB_ROWS_IN_BIG_BATCH) { + if (i == NB_RETRIES_FOR_BIG_BATCH-1) { + int lastRow = -1; + for (Result result : res) { + int currentRow = Bytes.toInt(result.getRow()); + for (int row = lastRow+1; row < currentRow; row++) { + LOG.error("Row missing: " + row); + } + lastRow = currentRow; + } + LOG.error("Last row: " + lastRow); + fail("Waited too much time for normal batch replication, " + + res.length + " instead of " + NB_ROWS_IN_BIG_BATCH); + } else { + LOG.info("Only got " + res.length + " rows"); + Thread.sleep(SLEEP_TIME); + } + } else { + break; + } + } + } + + /** + * Do a small loading into a table, make sure the data is really the same, + * then run the VerifyReplication job to check the results. Do a second + * comparison where all the cells are different. + * @throws Exception + */ + @Test(timeout=300000) + public void testVerifyRepJob() throws Exception { + // Populate the tables, at the same time it guarantees that the tables are + // identical since it does the check + testSmallBatch(); + + String[] args = new String[] {"2", Bytes.toString(tableName)}; + Job job = VerifyReplication.createSubmittableJob(CONF_WITH_LOCALFS, args); + if (job == null) { + fail("Job wasn't created, see the log"); + } + if (!job.waitForCompletion(true)) { + fail("Job failed, see the log"); + } + assertEquals(NB_ROWS_IN_BATCH, job.getCounters(). + findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue()); + assertEquals(0, job.getCounters(). + findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue()); + + Scan scan = new Scan(); + ResultScanner rs = htable2.getScanner(scan); + Put put = null; + for (Result result : rs) { + put = new Put(result.getRow()); + KeyValue firstVal = result.raw()[0]; + put.add(firstVal.getFamily(), + firstVal.getQualifier(), Bytes.toBytes("diff data")); + htable2.put(put); + } + Delete delete = new Delete(put.getRow()); + htable2.delete(delete); + job = VerifyReplication.createSubmittableJob(CONF_WITH_LOCALFS, args); + if (job == null) { + fail("Job wasn't created, see the log"); + } + if (!job.waitForCompletion(true)) { + fail("Job failed, see the log"); + } + assertEquals(0, job.getCounters(). + findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue()); + assertEquals(NB_ROWS_IN_BATCH, job.getCounters(). + findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue()); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java new file mode 100644 index 0000000..fbf38f1 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java @@ -0,0 +1,114 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestReplicationSource { + + private static final Log LOG = + LogFactory.getLog(TestReplicationSource.class); + private final static HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + private static FileSystem FS; + private static Path oldLogDir; + private static Path logDir; + private static Configuration conf = HBaseConfiguration.create(); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniDFSCluster(1); + FS = TEST_UTIL.getDFSCluster().getFileSystem(); + oldLogDir = new Path(FS.getHomeDirectory(), + HConstants.HREGION_OLDLOGDIR_NAME); + if (FS.exists(oldLogDir)) FS.delete(oldLogDir, true); + logDir = new Path(FS.getHomeDirectory(), + HConstants.HREGION_LOGDIR_NAME); + if (FS.exists(logDir)) FS.delete(logDir, true); + } + + /** + * Sanity check that we can move logs around while we are reading + * from them. Should this test fail, ReplicationSource would have a hard + * time reading logs that are being archived. + * @throws Exception + */ + @Test + public void testLogMoving() throws Exception{ + Path logPath = new Path(logDir, "log"); + if (!FS.exists(logDir)) FS.mkdirs(logDir); + if (!FS.exists(oldLogDir)) FS.mkdirs(oldLogDir); + HLog.Writer writer = HLog.createWriter(FS, logPath, conf); + for(int i = 0; i < 3; i++) { + byte[] b = Bytes.toBytes(Integer.toString(i)); + KeyValue kv = new KeyValue(b,b,b); + WALEdit edit = new WALEdit(); + edit.add(kv); + HLogKey key = new HLogKey(b, b, 0, 0, HConstants.DEFAULT_CLUSTER_ID); + writer.append(new HLog.Entry(key, edit)); + writer.sync(); + } + writer.close(); + + HLog.Reader reader = HLog.getReader(FS, logPath, conf); + HLog.Entry entry = reader.next(); + assertNotNull(entry); + + Path oldLogPath = new Path(oldLogDir, "log"); + FS.rename(logPath, oldLogPath); + + entry = reader.next(); + assertNotNull(entry); + + entry = reader.next(); + entry = reader.next(); + + assertNull(entry); + + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationZookeeper.java b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationZookeeper.java new file mode 100644 index 0000000..553b5cd --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationZookeeper.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; + +@Category(MediumTests.class) +public class TestReplicationZookeeper { + + private static Configuration conf; + + private static HBaseTestingUtility utility; + + private static ZooKeeperWatcher zkw; + + private static ReplicationZookeeper repZk; + + private static String slaveClusterKey; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + utility = new HBaseTestingUtility(); + utility.startMiniZKCluster(); + conf = utility.getConfiguration(); + zkw = HBaseTestingUtility.getZooKeeperWatcher(utility); + DummyServer server = new DummyServer(); + repZk = new ReplicationZookeeper(server, new AtomicBoolean()); + slaveClusterKey = conf.get(HConstants.ZOOKEEPER_QUORUM) + ":" + + conf.get("hbase.zookeeper.property.clientPort") + ":/1"; + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + utility.shutdownMiniZKCluster(); + } + + @Test + public void testGetAddressesMissingSlave() + throws IOException, KeeperException { + repZk.addPeer("1", slaveClusterKey); + // HBASE-5586 used to get an NPE + assertEquals(0, repZk.getSlavesAddresses("1").size()); + } + + static class DummyServer implements Server { + + @Override + public Configuration getConfiguration() { + return conf; + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + return zkw; + } + + @Override + public CatalogTracker getCatalogTracker() { + return null; + } + + @Override + public ServerName getServerName() { + return new ServerName("hostname.example.org", 1234, -1L); + } + + @Override + public void abort(String why, Throwable e) { + } + + @Override + public boolean isAborted() { + return false; + } + + @Override + public void stop(String why) { + } + + @Override + public boolean isStopped() { + return false; + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java new file mode 100644 index 0000000..18eb530 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java @@ -0,0 +1,257 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestReplicationSink { + private static final Log LOG = LogFactory.getLog(TestReplicationSink.class); + private static final int BATCH_SIZE = 10; + + private final static HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + private static ReplicationSink SINK; + + private static final byte[] TABLE_NAME1 = + Bytes.toBytes("table1"); + private static final byte[] TABLE_NAME2 = + Bytes.toBytes("table2"); + + private static final byte[] FAM_NAME1 = Bytes.toBytes("info1"); + private static final byte[] FAM_NAME2 = Bytes.toBytes("info2"); + + private static HTable table1; + private static Stoppable STOPPABLE = new Stoppable() { + final AtomicBoolean stop = new AtomicBoolean(false); + + @Override + public boolean isStopped() { + return this.stop.get(); + } + + @Override + public void stop(String why) { + LOG.info("STOPPING BECAUSE: " + why); + this.stop.set(true); + } + + }; + + private static HTable table2; + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); + TEST_UTIL.getConfiguration().setBoolean(HConstants.REPLICATION_ENABLE_KEY, true); + TEST_UTIL.startMiniCluster(3); + SINK = + new ReplicationSink(new Configuration(TEST_UTIL.getConfiguration()), STOPPABLE); + table1 = TEST_UTIL.createTable(TABLE_NAME1, FAM_NAME1); + table2 = TEST_UTIL.createTable(TABLE_NAME2, FAM_NAME2); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + STOPPABLE.stop("Shutting down"); + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + table1 = TEST_UTIL.truncateTable(TABLE_NAME1); + table2 = TEST_UTIL.truncateTable(TABLE_NAME2); + } + + /** + * Insert a whole batch of entries + * @throws Exception + */ + @Test + public void testBatchSink() throws Exception { + HLog.Entry[] entries = new HLog.Entry[BATCH_SIZE]; + for(int i = 0; i < BATCH_SIZE; i++) { + entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put); + } + SINK.replicateEntries(entries); + Scan scan = new Scan(); + ResultScanner scanRes = table1.getScanner(scan); + assertEquals(BATCH_SIZE, scanRes.next(BATCH_SIZE).length); + } + + /** + * Insert a mix of puts and deletes + * @throws Exception + */ + @Test + public void testMixedPutDelete() throws Exception { + HLog.Entry[] entries = new HLog.Entry[BATCH_SIZE/2]; + for(int i = 0; i < BATCH_SIZE/2; i++) { + entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put); + } + SINK.replicateEntries(entries); + + entries = new HLog.Entry[BATCH_SIZE]; + for(int i = 0; i < BATCH_SIZE; i++) { + entries[i] = createEntry(TABLE_NAME1, i, + i % 2 != 0 ? KeyValue.Type.Put: KeyValue.Type.DeleteColumn); + } + + SINK.replicateEntries(entries); + Scan scan = new Scan(); + ResultScanner scanRes = table1.getScanner(scan); + assertEquals(BATCH_SIZE/2, scanRes.next(BATCH_SIZE).length); + } + + /** + * Insert to 2 different tables + * @throws Exception + */ + @Test + public void testMixedPutTables() throws Exception { + HLog.Entry[] entries = new HLog.Entry[BATCH_SIZE]; + for(int i = 0; i < BATCH_SIZE; i++) { + entries[i] = + createEntry( i % 2 == 0 ? TABLE_NAME2 : TABLE_NAME1, + i, KeyValue.Type.Put); + } + + SINK.replicateEntries(entries); + Scan scan = new Scan(); + ResultScanner scanRes = table2.getScanner(scan); + for(Result res : scanRes) { + assertTrue(Bytes.toInt(res.getRow()) % 2 == 0); + } + } + + /** + * Insert then do different types of deletes + * @throws Exception + */ + @Test + public void testMixedDeletes() throws Exception { + HLog.Entry[] entries = new HLog.Entry[3]; + for(int i = 0; i < 3; i++) { + entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put); + } + SINK.replicateEntries(entries); + entries = new HLog.Entry[3]; + + entries[0] = createEntry(TABLE_NAME1, 0, KeyValue.Type.DeleteColumn); + entries[1] = createEntry(TABLE_NAME1, 1, KeyValue.Type.DeleteFamily); + entries[2] = createEntry(TABLE_NAME1, 2, KeyValue.Type.DeleteColumn); + + SINK.replicateEntries(entries); + + Scan scan = new Scan(); + ResultScanner scanRes = table1.getScanner(scan); + assertEquals(0, scanRes.next(3).length); + } + + /** + * Puts are buffered, but this tests when a delete (not-buffered) is applied + * before the actual Put that creates it. + * @throws Exception + */ + @Test + public void testApplyDeleteBeforePut() throws Exception { + HLog.Entry[] entries = new HLog.Entry[5]; + for(int i = 0; i < 2; i++) { + entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put); + } + entries[2] = createEntry(TABLE_NAME1, 1, KeyValue.Type.DeleteFamily); + for(int i = 3; i < 5; i++) { + entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put); + } + SINK.replicateEntries(entries); + Get get = new Get(Bytes.toBytes(1)); + Result res = table1.get(get); + assertEquals(0, res.size()); + } + + private HLog.Entry createEntry(byte [] table, int row, KeyValue.Type type) { + byte[] fam = Bytes.equals(table, TABLE_NAME1) ? FAM_NAME1 : FAM_NAME2; + byte[] rowBytes = Bytes.toBytes(row); + // Just make sure we don't get the same ts for two consecutive rows with + // same key + try { + Thread.sleep(1); + } catch (InterruptedException e) { + LOG.info("Was interrupted while sleep, meh", e); + } + final long now = System.currentTimeMillis(); + KeyValue kv = null; + if(type.getCode() == KeyValue.Type.Put.getCode()) { + kv = new KeyValue(rowBytes, fam, fam, now, + KeyValue.Type.Put, Bytes.toBytes(row)); + } else if (type.getCode() == KeyValue.Type.DeleteColumn.getCode()) { + kv = new KeyValue(rowBytes, fam, fam, + now, KeyValue.Type.DeleteColumn); + } else if (type.getCode() == KeyValue.Type.DeleteFamily.getCode()) { + kv = new KeyValue(rowBytes, fam, null, + now, KeyValue.Type.DeleteFamily); + } + + HLogKey key = new HLogKey(table, table, now, now, + HConstants.DEFAULT_CLUSTER_ID); + + WALEdit edit = new WALEdit(); + edit.add(kv); + + return new HLog.Entry(key, edit); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java new file mode 100644 index 0000000..f14fed9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java @@ -0,0 +1,410 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.replication.ReplicationSourceDummy; +import org.apache.hadoop.hbase.replication.ReplicationZookeeper; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestReplicationSourceManager { + + private static final Log LOG = + LogFactory.getLog(TestReplicationSourceManager.class); + + private static Configuration conf; + + private static HBaseTestingUtility utility; + + private static Replication replication; + + private static ReplicationSourceManager manager; + + private static ZooKeeperWatcher zkw; + + private static HTableDescriptor htd; + + private static HRegionInfo hri; + + private static final byte[] r1 = Bytes.toBytes("r1"); + + private static final byte[] r2 = Bytes.toBytes("r2"); + + private static final byte[] f1 = Bytes.toBytes("f1"); + + private static final byte[] test = Bytes.toBytes("test"); + + private static final String slaveId = "1"; + + private static FileSystem fs; + + private static Path oldLogDir; + + private static Path logDir; + + private static CountDownLatch latch; + + private static List files = new ArrayList(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + + conf = HBaseConfiguration.create(); + conf.set("replication.replicationsource.implementation", + ReplicationSourceDummy.class.getCanonicalName()); + conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true); + utility = new HBaseTestingUtility(conf); + utility.startMiniZKCluster(); + + zkw = new ZooKeeperWatcher(conf, "test", null); + ZKUtil.createWithParents(zkw, "/hbase/replication"); + ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1"); + ZKUtil.setData(zkw, "/hbase/replication/peers/1", + Bytes.toBytes(conf.get(HConstants.ZOOKEEPER_QUORUM) + ":" + + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":/1")); + ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/peer-state"); + ZKUtil.setData(zkw, "/hbase/replication/peers/1/peer-state", + Bytes.toBytes(ReplicationZookeeper.PeerState.ENABLED.name())); + ZKUtil.createWithParents(zkw, "/hbase/replication/state"); + ZKUtil.setData(zkw, "/hbase/replication/state", Bytes.toBytes("true")); + + replication = new Replication(new DummyServer(), fs, logDir, oldLogDir); + manager = replication.getReplicationManager(); + fs = FileSystem.get(conf); + oldLogDir = new Path(utility.getDataTestDir(), + HConstants.HREGION_OLDLOGDIR_NAME); + logDir = new Path(utility.getDataTestDir(), + HConstants.HREGION_LOGDIR_NAME); + + manager.addSource(slaveId); + + htd = new HTableDescriptor(test); + HColumnDescriptor col = new HColumnDescriptor("f1"); + col.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); + htd.addFamily(col); + col = new HColumnDescriptor("f2"); + col.setScope(HConstants.REPLICATION_SCOPE_LOCAL); + htd.addFamily(col); + + hri = new HRegionInfo(htd.getName(), r1, r2); + + + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + manager.join(); + utility.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + fs.delete(logDir, true); + fs.delete(oldLogDir, true); + } + + @After + public void tearDown() throws Exception { + setUp(); + } + + @Test + public void testLogRoll() throws Exception { + long seq = 0; + long baseline = 1000; + long time = baseline; + KeyValue kv = new KeyValue(r1, f1, r1); + WALEdit edit = new WALEdit(); + edit.add(kv); + + List listeners = new ArrayList(); + listeners.add(replication); + HLog hlog = new HLog(fs, logDir, oldLogDir, conf, listeners, + URLEncoder.encode("regionserver:60020", "UTF8")); + + manager.init(); + HTableDescriptor htd = new HTableDescriptor(); + htd.addFamily(new HColumnDescriptor(f1)); + // Testing normal log rolling every 20 + for(long i = 1; i < 101; i++) { + if(i > 1 && i % 20 == 0) { + hlog.rollWriter(); + } + LOG.info(i); + HLogKey key = new HLogKey(hri.getRegionName(), test, seq++, + System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID); + hlog.append(hri, key, edit, htd, true); + } + + // Simulate a rapid insert that's followed + // by a report that's still not totally complete (missing last one) + LOG.info(baseline + " and " + time); + baseline += 101; + time = baseline; + LOG.info(baseline + " and " + time); + + for (int i = 0; i < 3; i++) { + HLogKey key = new HLogKey(hri.getRegionName(), test, seq++, + System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID); + hlog.append(hri, key, edit, htd, true); + } + + assertEquals(6, manager.getHLogs().get(slaveId).size()); + + hlog.rollWriter(); + + manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(), + "1", 0, false, false); + + HLogKey key = new HLogKey(hri.getRegionName(), test, seq++, + System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID); + hlog.append(hri, key, edit, htd, true); + + assertEquals(1, manager.getHLogs().size()); + + + // TODO Need a case with only 2 HLogs and we only want to delete the first one + } + + @Test + public void testNodeFailoverWorkerCopyQueuesFromRSUsingMulti() throws Exception { + LOG.debug("testNodeFailoverWorkerCopyQueuesFromRSUsingMulti"); + conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, true); + final Server server = new DummyServer("hostname0.example.org"); + AtomicBoolean replicating = new AtomicBoolean(true); + ReplicationZookeeper rz = new ReplicationZookeeper(server, replicating); + // populate some znodes in the peer znode + files.add("log1"); + files.add("log2"); + for (String file : files) { + rz.addLogToList(file, "1"); + } + // create 3 DummyServers + Server s1 = new DummyServer("dummyserver1.example.org"); + Server s2 = new DummyServer("dummyserver2.example.org"); + Server s3 = new DummyServer("dummyserver3.example.org"); + + // create 3 DummyNodeFailoverWorkers + DummyNodeFailoverWorker w1 = new DummyNodeFailoverWorker( + server.getServerName().getServerName(), s1); + DummyNodeFailoverWorker w2 = new DummyNodeFailoverWorker( + server.getServerName().getServerName(), s2); + DummyNodeFailoverWorker w3 = new DummyNodeFailoverWorker( + server.getServerName().getServerName(), s3); + + latch = new CountDownLatch(3); + // start the threads + w1.start(); + w2.start(); + w3.start(); + // make sure only one is successful + int populatedMap = 0; + // wait for result now... till all the workers are done. + latch.await(); + populatedMap += w1.isLogZnodesMapPopulated() + w2.isLogZnodesMapPopulated() + + w3.isLogZnodesMapPopulated(); + assertEquals(1, populatedMap); + // close out the resources. + server.abort("", null); + } + + @Test + public void testNodeFailoverDeadServerParsing() throws Exception { + LOG.debug("testNodeFailoverDeadServerParsing"); + conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, true); + final Server server = new DummyServer("ec2-54-234-230-108.compute-1.amazonaws.com"); + AtomicBoolean replicating = new AtomicBoolean(true); + ReplicationZookeeper rz = new ReplicationZookeeper(server, replicating); + // populate some znodes in the peer znode + files.add("log1"); + files.add("log2"); + for (String file : files) { + rz.addLogToList(file, "1"); + } + // create 3 DummyServers + Server s1 = new DummyServer("ip-10-8-101-114.ec2.internal"); + Server s2 = new DummyServer("ec2-107-20-52-47.compute-1.amazonaws.com"); + Server s3 = new DummyServer("ec2-23-20-187-167.compute-1.amazonaws.com"); + + // simulate three server fail sequentially + ReplicationZookeeper rz1 = new ReplicationZookeeper(s1, new AtomicBoolean(true)); + SortedMap> testMap = + rz1.copyQueuesFromRSUsingMulti(server.getServerName().getServerName()); + ReplicationZookeeper rz2 = new ReplicationZookeeper(s2, new AtomicBoolean(true)); + testMap = rz2.copyQueuesFromRSUsingMulti(s1.getServerName().getServerName()); + ReplicationZookeeper rz3 = new ReplicationZookeeper(s3, new AtomicBoolean(true)); + testMap = rz3.copyQueuesFromRSUsingMulti(s2.getServerName().getServerName()); + + ReplicationSource s = new ReplicationSource(); + s.checkIfQueueRecovered(testMap.firstKey()); + List result = s.getDeadRegionServers(); + + // verify + assertTrue(result.contains(server.getServerName().getServerName())); + assertTrue(result.contains(s1.getServerName().getServerName())); + assertTrue(result.contains(s2.getServerName().getServerName())); + + server.abort("", null); + } + + static class DummyNodeFailoverWorker extends Thread { + private SortedMap> logZnodesMap; + Server server; + private String deadRsZnode; + ReplicationZookeeper rz; + + public DummyNodeFailoverWorker(String znode, Server s) throws Exception { + this.deadRsZnode = znode; + this.server = s; + rz = new ReplicationZookeeper(server, new AtomicBoolean(true)); + } + + @Override + public void run() { + try { + logZnodesMap = rz.copyQueuesFromRSUsingMulti(deadRsZnode); + server.abort("Done with testing", null); + } catch (Exception e) { + LOG.error("Got exception while running NodeFailoverWorker", e); + } finally { + latch.countDown(); + } + } + + /** + * @return 1 when the map is not empty. + */ + private int isLogZnodesMapPopulated() { + Collection> sets = logZnodesMap.values(); + if (sets.size() > 1) { + throw new RuntimeException("unexpected size of logZnodesMap: " + sets.size()); + } + if (sets.size() == 1) { + SortedSet s = sets.iterator().next(); + for (String file : files) { + // at least one file was missing + if (!s.contains(file)) { + return 0; + } + } + return 1; // we found all the files + } + return 0; + } + } + + static class DummyServer implements Server { + String hostname; + + DummyServer() { + hostname = "hostname.example.org"; + } + + DummyServer(String hostname) { + this.hostname = hostname; + } + + @Override + public Configuration getConfiguration() { + return conf; + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + return zkw; + } + + @Override + public CatalogTracker getCatalogTracker() { + return null; // To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public ServerName getServerName() { + return new ServerName(hostname, 1234, 1L); + } + + @Override + public void abort(String why, Throwable e) { + // To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public boolean isAborted() { + return false; + } + + @Override + public void stop(String why) { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public boolean isStopped() { + return false; //To change body of implemented methods use File | Settings | File Templates. + } + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java new file mode 100644 index 0000000..6b723be --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java @@ -0,0 +1,89 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rest; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.rest.filter.GzipFilter; +import org.apache.hadoop.util.StringUtils; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.servlet.Context; +import org.mortbay.jetty.servlet.ServletHolder; + +import com.sun.jersey.spi.container.servlet.ServletContainer; + +public class HBaseRESTTestingUtility { + + static final Log LOG = LogFactory.getLog(HBaseRESTTestingUtility.class); + + private int testServletPort; + private Server server; + + public int getServletPort() { + return testServletPort; + } + + public void startServletContainer(Configuration conf) throws Exception { + if (server != null) { + LOG.error("ServletContainer already running"); + return; + } + + // Inject the conf for the test by being first to make singleton + RESTServlet.getInstance(conf); + + // set up the Jersey servlet container for Jetty + ServletHolder sh = new ServletHolder(ServletContainer.class); + sh.setInitParameter( + "com.sun.jersey.config.property.resourceConfigClass", + ResourceConfig.class.getCanonicalName()); + sh.setInitParameter("com.sun.jersey.config.property.packages", + "jetty"); + + LOG.info("configured " + ServletContainer.class.getName()); + + // set up Jetty and run the embedded server + server = new Server(0); + server.setSendServerVersion(false); + server.setSendDateHeader(false); + // set up context + Context context = new Context(server, "/", Context.SESSIONS); + context.addServlet(sh, "/*"); + context.addFilter(GzipFilter.class, "/*", 0); + // start the server + server.start(); + // get the port + testServletPort = server.getConnectors()[0].getLocalPort(); + + LOG.info("started " + server.getClass().getName() + " on port " + + testServletPort); + } + + public void shutdownServletContainer() { + if (server != null) try { + server.stop(); + server = null; + RESTServlet.stop(); + } catch (Exception e) { + LOG.warn(StringUtils.stringifyException(e)); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/rest/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/rest/PerformanceEvaluation.java new file mode 100644 index 0000000..41893a4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/PerformanceEvaluation.java @@ -0,0 +1,1245 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rest; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.PrintStream; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; +import java.util.Arrays; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.lang.reflect.Constructor; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.filter.WhileMatchFilter; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.RemoteAdmin; +import org.apache.hadoop.hbase.rest.client.RemoteHTable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Hash; +import org.apache.hadoop.hbase.util.MurmurHash; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer; +import org.apache.hadoop.util.LineReader; + +/** + * Script used evaluating Stargate performance and scalability. Runs a SG + * client that steps through one of a set of hardcoded tests or 'experiments' + * (e.g. a random reads test, a random writes test, etc.). Pass on the + * command-line which test to run and how many clients are participating in + * this experiment. Run java PerformanceEvaluation --help to + * obtain usage. + * + *

    This class sets up and runs the evaluation programs described in + * Section 7, Performance Evaluation, of the Bigtable + * paper, pages 8-10. + * + *

    If number of clients > 1, we start up a MapReduce job. Each map task + * runs an individual client. Each client does about 1GB of data. + */ +public class PerformanceEvaluation { + protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName()); + + private static final int ROW_LENGTH = 1000; + private static final int ONE_GB = 1024 * 1024 * 1000; + private static final int ROWS_PER_GB = ONE_GB / ROW_LENGTH; + + public static final byte [] TABLE_NAME = Bytes.toBytes("TestTable"); + public static final byte [] FAMILY_NAME = Bytes.toBytes("info"); + public static final byte [] QUALIFIER_NAME = Bytes.toBytes("data"); + + protected static final HTableDescriptor TABLE_DESCRIPTOR; + static { + TABLE_DESCRIPTOR = new HTableDescriptor(TABLE_NAME); + TABLE_DESCRIPTOR.addFamily(new HColumnDescriptor(FAMILY_NAME)); + } + + protected Map commands = new TreeMap(); + protected static Cluster cluster = new Cluster(); + + volatile Configuration conf; + private boolean nomapred = false; + private int N = 1; + private int R = ROWS_PER_GB; + private int B = 100; + + private static final Path PERF_EVAL_DIR = new Path("performance_evaluation"); + /** + * Regex to parse lines in input file passed to mapreduce task. + */ + public static final Pattern LINE_PATTERN = + Pattern.compile("startRow=(\\d+),\\s+" + + "perClientRunRows=(\\d+),\\s+" + + "totalRows=(\\d+),\\s+" + + "clients=(\\d+),\\s+" + + "rowsPerPut=(\\d+)"); + + /** + * Enum for map metrics. Keep it out here rather than inside in the Map + * inner-class so we can find associated properties. + */ + protected static enum Counter { + /** elapsed time */ + ELAPSED_TIME, + /** number of rows */ + ROWS} + + /** + * Constructor + * @param c Configuration object + */ + public PerformanceEvaluation(final Configuration c) { + this.conf = c; + + addCommandDescriptor(RandomReadTest.class, "randomRead", + "Run random read test"); + addCommandDescriptor(RandomSeekScanTest.class, "randomSeekScan", + "Run random seek and scan 100 test"); + addCommandDescriptor(RandomScanWithRange10Test.class, "scanRange10", + "Run random seek scan with both start and stop row (max 10 rows)"); + addCommandDescriptor(RandomScanWithRange100Test.class, "scanRange100", + "Run random seek scan with both start and stop row (max 100 rows)"); + addCommandDescriptor(RandomScanWithRange1000Test.class, "scanRange1000", + "Run random seek scan with both start and stop row (max 1000 rows)"); + addCommandDescriptor(RandomScanWithRange10000Test.class, "scanRange10000", + "Run random seek scan with both start and stop row (max 10000 rows)"); + addCommandDescriptor(RandomWriteTest.class, "randomWrite", + "Run random write test"); + addCommandDescriptor(SequentialReadTest.class, "sequentialRead", + "Run sequential read test"); + addCommandDescriptor(SequentialWriteTest.class, "sequentialWrite", + "Run sequential write test"); + addCommandDescriptor(ScanTest.class, "scan", + "Run scan test (read every row)"); + addCommandDescriptor(FilteredScanTest.class, "filterScan", + "Run scan test using a filter to find a specific row based on it's value (make sure to use --rows=20)"); + } + + protected void addCommandDescriptor(Class cmdClass, + String name, String description) { + CmdDescriptor cmdDescriptor = + new CmdDescriptor(cmdClass, name, description); + commands.put(name, cmdDescriptor); + } + + /** + * Implementations can have their status set. + */ + static interface Status { + /** + * Sets status + * @param msg status message + * @throws IOException + */ + void setStatus(final String msg) throws IOException; + } + + /** + * This class works as the InputSplit of Performance Evaluation + * MapReduce InputFormat, and the Record Value of RecordReader. + * Each map task will only read one record from a PeInputSplit, + * the record value is the PeInputSplit itself. + */ + public static class PeInputSplit extends InputSplit implements Writable { + private int startRow = 0; + private int rows = 0; + private int totalRows = 0; + private int clients = 0; + private int rowsPerPut = 1; + + public PeInputSplit() { + this.startRow = 0; + this.rows = 0; + this.totalRows = 0; + this.clients = 0; + this.rowsPerPut = 1; + } + + public PeInputSplit(int startRow, int rows, int totalRows, int clients, + int rowsPerPut) { + this.startRow = startRow; + this.rows = rows; + this.totalRows = totalRows; + this.clients = clients; + this.rowsPerPut = 1; + } + + @Override + public void readFields(DataInput in) throws IOException { + this.startRow = in.readInt(); + this.rows = in.readInt(); + this.totalRows = in.readInt(); + this.clients = in.readInt(); + this.rowsPerPut = in.readInt(); + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(startRow); + out.writeInt(rows); + out.writeInt(totalRows); + out.writeInt(clients); + out.writeInt(rowsPerPut); + } + + @Override + public long getLength() throws IOException, InterruptedException { + return 0; + } + + @Override + public String[] getLocations() throws IOException, InterruptedException { + return new String[0]; + } + + public int getStartRow() { + return startRow; + } + + public int getRows() { + return rows; + } + + public int getTotalRows() { + return totalRows; + } + + public int getClients() { + return clients; + } + + public int getRowsPerPut() { + return rowsPerPut; + } + } + + /** + * InputFormat of Performance Evaluation MapReduce job. + * It extends from FileInputFormat, want to use it's methods such as setInputPaths(). + */ + public static class PeInputFormat extends FileInputFormat { + + @Override + public List getSplits(JobContext job) throws IOException { + // generate splits + List splitList = new ArrayList(); + + for (FileStatus file: listStatus(job)) { + Path path = file.getPath(); + FileSystem fs = path.getFileSystem(job.getConfiguration()); + FSDataInputStream fileIn = fs.open(path); + LineReader in = new LineReader(fileIn, job.getConfiguration()); + int lineLen = 0; + while(true) { + Text lineText = new Text(); + lineLen = in.readLine(lineText); + if(lineLen <= 0) { + break; + } + Matcher m = LINE_PATTERN.matcher(lineText.toString()); + if((m != null) && m.matches()) { + int startRow = Integer.parseInt(m.group(1)); + int rows = Integer.parseInt(m.group(2)); + int totalRows = Integer.parseInt(m.group(3)); + int clients = Integer.parseInt(m.group(4)); + int rowsPerPut = Integer.parseInt(m.group(5)); + + LOG.debug("split["+ splitList.size() + "] " + + " startRow=" + startRow + + " rows=" + rows + + " totalRows=" + totalRows + + " clients=" + clients + + " rowsPerPut=" + rowsPerPut); + + PeInputSplit newSplit = + new PeInputSplit(startRow, rows, totalRows, clients, rowsPerPut); + splitList.add(newSplit); + } + } + in.close(); + } + + LOG.info("Total # of splits: " + splitList.size()); + return splitList; + } + + @Override + public RecordReader createRecordReader(InputSplit split, + TaskAttemptContext context) { + return new PeRecordReader(); + } + + public static class PeRecordReader extends RecordReader { + private boolean readOver = false; + private PeInputSplit split = null; + private NullWritable key = null; + private PeInputSplit value = null; + + @Override + public void initialize(InputSplit split, TaskAttemptContext context) + throws IOException, InterruptedException { + this.readOver = false; + this.split = (PeInputSplit)split; + } + + @Override + public boolean nextKeyValue() throws IOException, InterruptedException { + if(readOver) { + return false; + } + + key = NullWritable.get(); + value = (PeInputSplit)split; + + readOver = true; + return true; + } + + @Override + public NullWritable getCurrentKey() throws IOException, InterruptedException { + return key; + } + + @Override + public PeInputSplit getCurrentValue() throws IOException, InterruptedException { + return value; + } + + @Override + public float getProgress() throws IOException, InterruptedException { + if(readOver) { + return 1.0f; + } else { + return 0.0f; + } + } + + @Override + public void close() throws IOException { + // do nothing + } + } + } + + /** + * MapReduce job that runs a performance evaluation client in each map task. + */ + public static class EvaluationMapTask + extends Mapper { + + /** configuration parameter name that contains the command */ + public final static String CMD_KEY = "EvaluationMapTask.command"; + /** configuration parameter name that contains the PE impl */ + public static final String PE_KEY = "EvaluationMapTask.performanceEvalImpl"; + + private Class cmd; + private PerformanceEvaluation pe; + + @Override + protected void setup(Context context) throws IOException, InterruptedException { + this.cmd = forName(context.getConfiguration().get(CMD_KEY), Test.class); + + // this is required so that extensions of PE are instantiated within the + // map reduce task... + Class peClass = + forName(context.getConfiguration().get(PE_KEY), PerformanceEvaluation.class); + try { + this.pe = peClass.getConstructor(Configuration.class) + .newInstance(context.getConfiguration()); + } catch (Exception e) { + throw new IllegalStateException("Could not instantiate PE instance", e); + } + } + + private Class forName(String className, Class type) { + Class clazz = null; + try { + clazz = Class.forName(className).asSubclass(type); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Could not find class for name: " + className, e); + } + return clazz; + } + + protected void map(NullWritable key, PeInputSplit value, final Context context) + throws IOException, InterruptedException { + + Status status = new Status() { + public void setStatus(String msg) { + context.setStatus(msg); + } + }; + + // Evaluation task + long elapsedTime = this.pe.runOneClient(this.cmd, value.getStartRow(), + value.getRows(), value.getTotalRows(), value.getRowsPerPut(), status); + // Collect how much time the thing took. Report as map output and + // to the ELAPSED_TIME counter. + context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime); + context.getCounter(Counter.ROWS).increment(value.rows); + context.write(new LongWritable(value.startRow), new LongWritable(elapsedTime)); + context.progress(); + } + } + + /* + * If table does not already exist, create. + * @param c Client to use checking. + * @return True if we created the table. + * @throws IOException + */ + private boolean checkTable() throws IOException { + HTableDescriptor tableDescriptor = getTableDescriptor(); + RemoteAdmin admin = new RemoteAdmin(new Client(cluster), conf); + if (!admin.isTableAvailable(tableDescriptor.getName())) { + admin.createTable(tableDescriptor); + return true; + } + return false; + } + + protected HTableDescriptor getTableDescriptor() { + return TABLE_DESCRIPTOR; + } + + /* + * We're to run multiple clients concurrently. Setup a mapreduce job. Run + * one map per client. Then run a single reduce to sum the elapsed times. + * @param cmd Command to run. + * @throws IOException + */ + private void runNIsMoreThanOne(final Class cmd) + throws IOException, InterruptedException, ClassNotFoundException { + checkTable(); + if (nomapred) { + doMultipleClients(cmd); + } else { + doMapReduce(cmd); + } + } + + /* + * Run all clients in this vm each to its own thread. + * @param cmd Command to run. + * @throws IOException + */ + private void doMultipleClients(final Class cmd) throws IOException { + final List threads = new ArrayList(N); + final int perClientRows = R/N; + for (int i = 0; i < N; i++) { + Thread t = new Thread (Integer.toString(i)) { + @Override + public void run() { + super.run(); + PerformanceEvaluation pe = new PerformanceEvaluation(conf); + int index = Integer.parseInt(getName()); + try { + long elapsedTime = pe.runOneClient(cmd, index * perClientRows, + perClientRows, R, B, new Status() { + public void setStatus(final String msg) throws IOException { + LOG.info("client-" + getName() + " " + msg); + } + }); + LOG.info("Finished " + getName() + " in " + elapsedTime + + "ms writing " + perClientRows + " rows"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + threads.add(t); + } + for (Thread t: threads) { + t.start(); + } + for (Thread t: threads) { + while(t.isAlive()) { + try { + t.join(); + } catch (InterruptedException e) { + LOG.debug("Interrupted, continuing" + e.toString()); + } + } + } + } + + /* + * Run a mapreduce job. Run as many maps as asked-for clients. + * Before we start up the job, write out an input file with instruction + * per client regards which row they are to start on. + * @param cmd Command to run. + * @throws IOException + */ + private void doMapReduce(final Class cmd) throws IOException, + InterruptedException, ClassNotFoundException { + Path inputDir = writeInputFile(this.conf); + this.conf.set(EvaluationMapTask.CMD_KEY, cmd.getName()); + this.conf.set(EvaluationMapTask.PE_KEY, getClass().getName()); + Job job = new Job(this.conf); + job.setJarByClass(PerformanceEvaluation.class); + job.setJobName("HBase Performance Evaluation"); + + job.setInputFormatClass(PeInputFormat.class); + PeInputFormat.setInputPaths(job, inputDir); + + job.setOutputKeyClass(LongWritable.class); + job.setOutputValueClass(LongWritable.class); + + job.setMapperClass(EvaluationMapTask.class); + job.setReducerClass(LongSumReducer.class); + + job.setNumReduceTasks(1); + + job.setOutputFormatClass(TextOutputFormat.class); + TextOutputFormat.setOutputPath(job, new Path(inputDir,"outputs")); + + job.waitForCompletion(true); + } + + /* + * Write input file of offsets-per-client for the mapreduce job. + * @param c Configuration + * @return Directory that contains file written. + * @throws IOException + */ + private Path writeInputFile(final Configuration c) throws IOException { + FileSystem fs = FileSystem.get(c); + if (!fs.exists(PERF_EVAL_DIR)) { + fs.mkdirs(PERF_EVAL_DIR); + } + SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss"); + Path subdir = new Path(PERF_EVAL_DIR, formatter.format(new Date())); + fs.mkdirs(subdir); + Path inputFile = new Path(subdir, "input.txt"); + PrintStream out = new PrintStream(fs.create(inputFile)); + // Make input random. + Map m = new TreeMap(); + Hash h = MurmurHash.getInstance(); + int perClientRows = (R / N); + try { + for (int i = 0; i < 10; i++) { + for (int j = 0; j < N; j++) { + String s = "startRow=" + ((j * perClientRows) + (i * (perClientRows/10))) + + ", perClientRunRows=" + (perClientRows / 10) + + ", totalRows=" + R + + ", clients=" + N + + ", rowsPerPut=" + B; + int hash = h.hash(Bytes.toBytes(s)); + m.put(hash, s); + } + } + for (Map.Entry e: m.entrySet()) { + out.println(e.getValue()); + } + } finally { + out.close(); + } + return subdir; + } + + /** + * Describes a command. + */ + static class CmdDescriptor { + private Class cmdClass; + private String name; + private String description; + + CmdDescriptor(Class cmdClass, String name, String description) { + this.cmdClass = cmdClass; + this.name = name; + this.description = description; + } + + public Class getCmdClass() { + return cmdClass; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + } + + /** + * Wraps up options passed to {@link org.apache.hadoop.hbase.PerformanceEvaluation.Test + * tests}. This makes the reflection logic a little easier to understand... + */ + static class TestOptions { + private int startRow; + private int perClientRunRows; + private int totalRows; + private byte[] tableName; + private int rowsPerPut; + + TestOptions() { + } + + TestOptions(int startRow, int perClientRunRows, int totalRows, byte[] tableName, int rowsPerPut) { + this.startRow = startRow; + this.perClientRunRows = perClientRunRows; + this.totalRows = totalRows; + this.tableName = tableName; + this.rowsPerPut = rowsPerPut; + } + + public int getStartRow() { + return startRow; + } + + public int getPerClientRunRows() { + return perClientRunRows; + } + + public int getTotalRows() { + return totalRows; + } + + public byte[] getTableName() { + return tableName; + } + + public int getRowsPerPut() { + return rowsPerPut; + } + } + + /* + * A test. + * Subclass to particularize what happens per row. + */ + static abstract class Test { + // Below is make it so when Tests are all running in the one + // jvm, that they each have a differently seeded Random. + private static final Random randomSeed = + new Random(System.currentTimeMillis()); + private static long nextRandomSeed() { + return randomSeed.nextLong(); + } + protected final Random rand = new Random(nextRandomSeed()); + + protected final int startRow; + protected final int perClientRunRows; + protected final int totalRows; + protected final Status status; + protected byte[] tableName; + protected RemoteHTable table; + protected volatile Configuration conf; + + /** + * Note that all subclasses of this class must provide a public contructor + * that has the exact same list of arguments. + */ + Test(final Configuration conf, final TestOptions options, final Status status) { + super(); + this.startRow = options.getStartRow(); + this.perClientRunRows = options.getPerClientRunRows(); + this.totalRows = options.getTotalRows(); + this.status = status; + this.tableName = options.getTableName(); + this.table = null; + this.conf = conf; + } + + protected String generateStatus(final int sr, final int i, final int lr) { + return sr + "/" + i + "/" + lr; + } + + protected int getReportingPeriod() { + int period = this.perClientRunRows / 10; + return period == 0? this.perClientRunRows: period; + } + + void testSetup() throws IOException { + this.table = new RemoteHTable(new Client(cluster), conf, tableName); + } + + void testTakedown() throws IOException { + this.table.close(); + } + + /* + * Run test + * @return Elapsed time. + * @throws IOException + */ + long test() throws IOException { + long elapsedTime; + testSetup(); + long startTime = System.currentTimeMillis(); + try { + testTimed(); + elapsedTime = System.currentTimeMillis() - startTime; + } finally { + testTakedown(); + } + return elapsedTime; + } + + /** + * Provides an extension point for tests that don't want a per row invocation. + */ + void testTimed() throws IOException { + int lastRow = this.startRow + this.perClientRunRows; + // Report on completion of 1/10th of total. + for (int i = this.startRow; i < lastRow; i++) { + testRow(i); + if (status != null && i > 0 && (i % getReportingPeriod()) == 0) { + status.setStatus(generateStatus(this.startRow, i, lastRow)); + } + } + } + + /* + * Test for individual row. + * @param i Row index. + */ + void testRow(final int i) throws IOException { + } + } + + @SuppressWarnings("unused") + static class RandomSeekScanTest extends Test { + RandomSeekScanTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testRow(final int i) throws IOException { + Scan scan = new Scan(getRandomRow(this.rand, this.totalRows)); + scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); + scan.setFilter(new WhileMatchFilter(new PageFilter(120))); + ResultScanner s = this.table.getScanner(scan); + //int count = 0; + for (Result rr = null; (rr = s.next()) != null;) { + // LOG.info("" + count++ + " " + rr.toString()); + } + s.close(); + } + + @Override + protected int getReportingPeriod() { + int period = this.perClientRunRows / 100; + return period == 0? this.perClientRunRows: period; + } + + } + + @SuppressWarnings("unused") + static abstract class RandomScanWithRangeTest extends Test { + RandomScanWithRangeTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testRow(final int i) throws IOException { + Pair startAndStopRow = getStartAndStopRow(); + Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond()); + scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); + ResultScanner s = this.table.getScanner(scan); + int count = 0; + for (Result rr = null; (rr = s.next()) != null;) { + count++; + } + + if (i % 100 == 0) { + LOG.info(String.format("Scan for key range %s - %s returned %s rows", + Bytes.toString(startAndStopRow.getFirst()), + Bytes.toString(startAndStopRow.getSecond()), count)); + } + + s.close(); + } + + protected abstract Pair getStartAndStopRow(); + + protected Pair generateStartAndStopRows(int maxRange) { + int start = this.rand.nextInt(Integer.MAX_VALUE) % totalRows; + int stop = start + maxRange; + return new Pair(format(start), format(stop)); + } + + @Override + protected int getReportingPeriod() { + int period = this.perClientRunRows / 100; + return period == 0? this.perClientRunRows: period; + } + } + + static class RandomScanWithRange10Test extends RandomScanWithRangeTest { + RandomScanWithRange10Test(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + protected Pair getStartAndStopRow() { + return generateStartAndStopRows(10); + } + } + + static class RandomScanWithRange100Test extends RandomScanWithRangeTest { + RandomScanWithRange100Test(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + protected Pair getStartAndStopRow() { + return generateStartAndStopRows(100); + } + } + + static class RandomScanWithRange1000Test extends RandomScanWithRangeTest { + RandomScanWithRange1000Test(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + protected Pair getStartAndStopRow() { + return generateStartAndStopRows(1000); + } + } + + static class RandomScanWithRange10000Test extends RandomScanWithRangeTest { + RandomScanWithRange10000Test(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + protected Pair getStartAndStopRow() { + return generateStartAndStopRows(10000); + } + } + + static class RandomReadTest extends Test { + RandomReadTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testRow(final int i) throws IOException { + Get get = new Get(getRandomRow(this.rand, this.totalRows)); + get.addColumn(FAMILY_NAME, QUALIFIER_NAME); + this.table.get(get); + } + + @Override + protected int getReportingPeriod() { + int period = this.perClientRunRows / 100; + return period == 0? this.perClientRunRows: period; + } + + } + + static class RandomWriteTest extends Test { + int rowsPerPut; + + RandomWriteTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + rowsPerPut = options.getRowsPerPut(); + } + + @Override + void testTimed() throws IOException { + int lastRow = this.startRow + this.perClientRunRows; + // Report on completion of 1/10th of total. + List puts = new ArrayList(); + for (int i = this.startRow; i < lastRow; i += rowsPerPut) { + for (int j = 0; j < rowsPerPut; j++) { + byte [] row = getRandomRow(this.rand, this.totalRows); + Put put = new Put(row); + byte[] value = generateValue(this.rand); + put.add(FAMILY_NAME, QUALIFIER_NAME, value); + puts.add(put); + if (status != null && i > 0 && (i % getReportingPeriod()) == 0) { + status.setStatus(generateStatus(this.startRow, i, lastRow)); + } + } + table.put(puts); + } + } + } + + static class ScanTest extends Test { + private ResultScanner testScanner; + + ScanTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testSetup() throws IOException { + super.testSetup(); + } + + @Override + void testTakedown() throws IOException { + if (this.testScanner != null) { + this.testScanner.close(); + } + super.testTakedown(); + } + + + @Override + void testRow(final int i) throws IOException { + if (this.testScanner == null) { + Scan scan = new Scan(format(this.startRow)); + scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); + this.testScanner = table.getScanner(scan); + } + testScanner.next(); + } + + } + + static class SequentialReadTest extends Test { + SequentialReadTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testRow(final int i) throws IOException { + Get get = new Get(format(i)); + get.addColumn(FAMILY_NAME, QUALIFIER_NAME); + table.get(get); + } + + } + + static class SequentialWriteTest extends Test { + int rowsPerPut; + + SequentialWriteTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + rowsPerPut = options.getRowsPerPut(); + } + + @Override + void testTimed() throws IOException { + int lastRow = this.startRow + this.perClientRunRows; + // Report on completion of 1/10th of total. + List puts = new ArrayList(); + for (int i = this.startRow; i < lastRow; i += rowsPerPut) { + for (int j = 0; j < rowsPerPut; j++) { + Put put = new Put(format(i + j)); + byte[] value = generateValue(this.rand); + put.add(FAMILY_NAME, QUALIFIER_NAME, value); + puts.add(put); + if (status != null && i > 0 && (i % getReportingPeriod()) == 0) { + status.setStatus(generateStatus(this.startRow, i, lastRow)); + } + } + table.put(puts); + } + } + } + + static class FilteredScanTest extends Test { + protected static final Log LOG = LogFactory.getLog(FilteredScanTest.class.getName()); + + FilteredScanTest(Configuration conf, TestOptions options, Status status) { + super(conf, options, status); + } + + @Override + void testRow(int i) throws IOException { + byte[] value = generateValue(this.rand); + Scan scan = constructScan(value); + ResultScanner scanner = null; + try { + scanner = this.table.getScanner(scan); + while (scanner.next() != null) { + } + } finally { + if (scanner != null) scanner.close(); + } + } + + protected Scan constructScan(byte[] valuePrefix) throws IOException { + Filter filter = new SingleColumnValueFilter( + FAMILY_NAME, QUALIFIER_NAME, CompareFilter.CompareOp.EQUAL, + new BinaryComparator(valuePrefix) + ); + Scan scan = new Scan(); + scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); + scan.setFilter(filter); + return scan; + } + } + + /* + * Format passed integer. + * @param number + * @return Returns zero-prefixed 10-byte wide decimal version of passed + * number (Does absolute in case number is negative). + */ + public static byte [] format(final int number) { + byte [] b = new byte[10]; + int d = Math.abs(number); + for (int i = b.length - 1; i >= 0; i--) { + b[i] = (byte)((d % 10) + '0'); + d /= 10; + } + return b; + } + + /* + * This method takes some time and is done inline uploading data. For + * example, doing the mapfile test, generation of the key and value + * consumes about 30% of CPU time. + * @return Generated random value to insert into a table cell. + */ + public static byte[] generateValue(final Random r) { + byte [] b = new byte [ROW_LENGTH]; + r.nextBytes(b); + return b; + } + + static byte [] getRandomRow(final Random random, final int totalRows) { + return format(random.nextInt(Integer.MAX_VALUE) % totalRows); + } + + long runOneClient(final Class cmd, final int startRow, + final int perClientRunRows, final int totalRows, + final int rowsPerPut, final Status status) + throws IOException { + status.setStatus("Start " + cmd + " at offset " + startRow + " for " + + perClientRunRows + " rows"); + long totalElapsedTime = 0; + + Test t = null; + TestOptions options = new TestOptions(startRow, perClientRunRows, + totalRows, getTableDescriptor().getName(), rowsPerPut); + try { + Constructor constructor = cmd.getDeclaredConstructor( + Configuration.class, TestOptions.class, Status.class); + t = constructor.newInstance(this.conf, options, status); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException("Invalid command class: " + + cmd.getName() + ". It does not provide a constructor as described by" + + "the javadoc comment. Available constructors are: " + + Arrays.toString(cmd.getConstructors())); + } catch (Exception e) { + throw new IllegalStateException("Failed to construct command class", e); + } + totalElapsedTime = t.test(); + + status.setStatus("Finished " + cmd + " in " + totalElapsedTime + + "ms at offset " + startRow + " for " + perClientRunRows + " rows"); + return totalElapsedTime; + } + + private void runNIsOne(final Class cmd) { + Status status = new Status() { + public void setStatus(String msg) throws IOException { + LOG.info(msg); + } + }; + + try { + checkTable(); + runOneClient(cmd, 0, R, R, B, status); + } catch (Exception e) { + LOG.error("Failed", e); + } + } + + private void runTest(final Class cmd) throws IOException, + InterruptedException, ClassNotFoundException { + if (N == 1) { + // If there is only one client and one HRegionServer, we assume nothing + // has been set up at all. + runNIsOne(cmd); + } else { + // Else, run + runNIsMoreThanOne(cmd); + } + } + + protected void printUsage() { + printUsage(null); + } + + protected void printUsage(final String message) { + if (message != null && message.length() > 0) { + System.err.println(message); + } + System.err.println("Usage: java " + this.getClass().getName() + " \\"); + System.err.println(" [--option] [--option=value] "); + System.err.println(); + System.err.println("Options:"); + System.err.println(" host String. Specify Stargate endpoint."); + System.err.println(" rows Integer. Rows each client runs. Default: One million"); + System.err.println(" rowsPerPut Integer. Rows each Stargate (multi)Put. Default: 100"); + System.err.println(" nomapred (Flag) Run multiple clients using threads " + + "(rather than use mapreduce)"); + System.err.println(); + System.err.println("Command:"); + for (CmdDescriptor command : commands.values()) { + System.err.println(String.format(" %-15s %s", command.getName(), command.getDescription())); + } + System.err.println(); + System.err.println("Args:"); + System.err.println(" nclients Integer. Required. Total number of " + + "clients (and HRegionServers)"); + System.err.println(" running: 1 <= value <= 500"); + System.err.println("Examples:"); + System.err.println(" To run a single evaluation client:"); + System.err.println(" $ bin/hbase " + this.getClass().getName() + + " sequentialWrite 1"); + } + + private void getArgs(final int start, final String[] args) { + if(start + 1 > args.length) { + throw new IllegalArgumentException("must supply the number of clients"); + } + N = Integer.parseInt(args[start]); + if (N < 1) { + throw new IllegalArgumentException("Number of clients must be > 1"); + } + // Set total number of rows to write. + R = R * N; + } + + public int doCommandLine(final String[] args) { + // Process command-line args. TODO: Better cmd-line processing + // (but hopefully something not as painful as cli options). + int errCode = -1; + if (args.length < 1) { + printUsage(); + return errCode; + } + + try { + for (int i = 0; i < args.length; i++) { + String cmd = args[i]; + if (cmd.equals("-h")) { + printUsage(); + errCode = 0; + break; + } + + final String nmr = "--nomapred"; + if (cmd.startsWith(nmr)) { + nomapred = true; + continue; + } + + final String rows = "--rows="; + if (cmd.startsWith(rows)) { + R = Integer.parseInt(cmd.substring(rows.length())); + continue; + } + + final String rowsPerPut = "--rowsPerPut="; + if (cmd.startsWith(rowsPerPut)) { + this.B = Integer.parseInt(cmd.substring(rowsPerPut.length())); + continue; + } + + final String host = "--host="; + if (cmd.startsWith(host)) { + cluster.add(cmd.substring(host.length())); + continue; + } + + Class cmdClass = determineCommandClass(cmd); + if (cmdClass != null) { + getArgs(i + 1, args); + if (cluster.isEmpty()) { + String s = conf.get("stargate.hostname", "localhost"); + if (s.contains(":")) { + cluster.add(s); + } else { + cluster.add(s, conf.getInt("stargate.port", 8080)); + } + } + runTest(cmdClass); + errCode = 0; + break; + } + + printUsage(); + break; + } + } catch (Exception e) { + e.printStackTrace(); + } + + return errCode; + } + + private Class determineCommandClass(String cmd) { + CmdDescriptor descriptor = commands.get(cmd); + return descriptor != null ? descriptor.getCmdClass() : null; + } + + /** + * @param args + */ + public static void main(final String[] args) { + Configuration c = HBaseConfiguration.create(); + System.exit(new PerformanceEvaluation(c).doCommandLine(args)); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java b/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java new file mode 100644 index 0000000..570addd --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java @@ -0,0 +1,157 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import org.apache.commons.httpclient.Header; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.util.Bytes; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestGzipFilter { + private static final String TABLE = "TestGzipFilter"; + private static final String CFA = "a"; + private static final String COLUMN_1 = CFA + ":1"; + private static final String COLUMN_2 = CFA + ":2"; + private static final String ROW_1 = "testrow1"; + private static final byte[] VALUE_1 = Bytes.toBytes("testvalue1"); + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static Client client; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); + client = new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(TABLE)) { + return; + } + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(new HColumnDescriptor(CFA)); + admin.createTable(htd); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testGzipFilter() throws Exception { + String path = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_1; + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + GZIPOutputStream os = new GZIPOutputStream(bos); + os.write(VALUE_1); + os.close(); + byte[] value_1_gzip = bos.toByteArray(); + + // input side filter + + Header[] headers = new Header[2]; + headers[0] = new Header("Content-Type", Constants.MIMETYPE_BINARY); + headers[1] = new Header("Content-Encoding", "gzip"); + Response response = client.put(path, headers, value_1_gzip); + assertEquals(response.getCode(), 200); + + HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE); + Get get = new Get(Bytes.toBytes(ROW_1)); + get.addColumn(Bytes.toBytes(CFA), Bytes.toBytes("1")); + Result result = table.get(get); + byte[] value = result.getValue(Bytes.toBytes(CFA), Bytes.toBytes("1")); + assertNotNull(value); + assertTrue(Bytes.equals(value, VALUE_1)); + + // output side filter + + headers[0] = new Header("Accept", Constants.MIMETYPE_BINARY); + headers[1] = new Header("Accept-Encoding", "gzip"); + response = client.get(path, headers); + assertEquals(response.getCode(), 200); + ByteArrayInputStream bis = new ByteArrayInputStream(response.getBody()); + GZIPInputStream is = new GZIPInputStream(bis); + value = new byte[VALUE_1.length]; + is.read(value, 0, VALUE_1.length); + assertTrue(Bytes.equals(value, VALUE_1)); + is.close(); + } + + @Test + public void testErrorNotGzipped() throws Exception { + Header[] headers = new Header[2]; + headers[0] = new Header("Accept", Constants.MIMETYPE_BINARY); + headers[1] = new Header("Accept-Encoding", "gzip"); + Response response = client.get("/" + TABLE + "/" + ROW_1 + "/" + COLUMN_2, headers); + assertEquals(response.getCode(), 404); + String contentEncoding = response.getHeader("Content-Encoding"); + assertTrue(contentEncoding == null || !contentEncoding.contains("gzip")); + response = client.get("/" + TABLE, headers); + assertEquals(response.getCode(), 405); + contentEncoding = response.getHeader("Content-Encoding"); + assertTrue(contentEncoding == null || !contentEncoding.contains("gzip")); + } + + @Test + public void testScannerResultCodes() throws Exception { + Header[] headers = new Header[3]; + headers[0] = new Header("Content-Type", Constants.MIMETYPE_XML); + headers[1] = new Header("Accept", Constants.MIMETYPE_JSON); + headers[2] = new Header("Accept-Encoding", "gzip"); + Response response = client.post("/" + TABLE + "/scanner", headers, + "".getBytes()); + assertEquals(response.getCode(), 201); + String scannerUrl = response.getLocation(); + assertNotNull(scannerUrl); + response = client.get(scannerUrl); + assertEquals(response.getCode(), 200); + response = client.get(scannerUrl); + assertEquals(response.getCode(), 204); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java new file mode 100644 index 0000000..e45ed2a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java @@ -0,0 +1,178 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.CellModel; +import org.apache.hadoop.hbase.rest.model.CellSetModel; +import org.apache.hadoop.hbase.rest.model.RowModel; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + + +@Category(MediumTests.class) +public class TestMultiRowResource { + + private static final String TABLE = "TestRowResource"; + private static final String CFA = "a"; + private static final String CFB = "b"; + private static final String COLUMN_1 = CFA + ":1"; + private static final String COLUMN_2 = CFB + ":2"; + private static final String ROW_1 = "testrow5"; + private static final String VALUE_1 = "testvalue5"; + private static final String ROW_2 = "testrow6"; + private static final String VALUE_2 = "testvalue6"; + + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); + + private static Client client; + private static JAXBContext context; + private static Marshaller marshaller; + private static Unmarshaller unmarshaller; + private static Configuration conf; + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf = TEST_UTIL.getConfiguration(); + TEST_UTIL.startMiniCluster(); + REST_TEST_UTIL.startServletContainer(conf); + context = JAXBContext.newInstance( + CellModel.class, + CellSetModel.class, + RowModel.class); + marshaller = context.createMarshaller(); + unmarshaller = context.createUnmarshaller(); + client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(TABLE)) { + return; + } + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(new HColumnDescriptor(CFA)); + htd.addFamily(new HColumnDescriptor(CFB)); + admin.createTable(htd); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + + @Test + public void testMultiCellGetJSON() throws IOException, JAXBException { + String row_5_url = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_1; + String row_6_url = "/" + TABLE + "/" + ROW_2 + "/" + COLUMN_2; + + + StringBuilder path = new StringBuilder(); + path.append("/"); + path.append(TABLE); + path.append("/multiget/?row="); + path.append(ROW_1); + path.append("&row="); + path.append(ROW_2); + + client.post(row_5_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_1)); + client.post(row_6_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_2)); + + + Response response = client.get(path.toString(), Constants.MIMETYPE_JSON); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type")); + + client.delete(row_5_url); + client.delete(row_6_url); + + } + + @Test + public void testMultiCellGetXML() throws IOException, JAXBException { + String row_5_url = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_1; + String row_6_url = "/" + TABLE + "/" + ROW_2 + "/" + COLUMN_2; + + + StringBuilder path = new StringBuilder(); + path.append("/"); + path.append(TABLE); + path.append("/multiget/?row="); + path.append(ROW_1); + path.append("&row="); + path.append(ROW_2); + + client.post(row_5_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_1)); + client.post(row_6_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_2)); + + + Response response = client.get(path.toString(), Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + + client.delete(row_5_url); + client.delete(row_6_url); + + } + + @Test + public void testMultiCellGetJSONNotFound() throws IOException { + String row_5_url = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_1; + + StringBuilder path = new StringBuilder(); + path.append("/"); + path.append(TABLE); + path.append("/multiget/?row="); + path.append(ROW_1); + path.append("&row="); + path.append(ROW_2); + + client.post(row_5_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_1)); + + Response response = client.get(path.toString(), Constants.MIMETYPE_JSON); + + assertEquals(response.getCode(), 404); + + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java new file mode 100644 index 0000000..2a0284e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/TestRowResource.java @@ -0,0 +1,674 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.net.URLEncoder; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; + +import org.apache.commons.httpclient.Header; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.CellModel; +import org.apache.hadoop.hbase.rest.model.CellSetModel; +import org.apache.hadoop.hbase.rest.model.RowModel; +import org.apache.hadoop.hbase.util.Bytes; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestRowResource { + private static final String TABLE = "TestRowResource"; + private static final String CFA = "a"; + private static final String CFB = "b"; + private static final String COLUMN_1 = CFA + ":1"; + private static final String COLUMN_2 = CFB + ":2"; + private static final String ROW_1 = "testrow1"; + private static final String VALUE_1 = "testvalue1"; + private static final String ROW_2 = "testrow2"; + private static final String VALUE_2 = "testvalue2"; + private static final String ROW_3 = "testrow3"; + private static final String VALUE_3 = "testvalue3"; + private static final String ROW_4 = "testrow4"; + private static final String VALUE_4 = "testvalue4"; + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static Client client; + private static JAXBContext context; + private static Marshaller marshaller; + private static Unmarshaller unmarshaller; + private static Configuration conf; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf = TEST_UTIL.getConfiguration(); + TEST_UTIL.startMiniCluster(3); + REST_TEST_UTIL.startServletContainer(conf); + context = JAXBContext.newInstance( + CellModel.class, + CellSetModel.class, + RowModel.class); + marshaller = context.createMarshaller(); + unmarshaller = context.createUnmarshaller(); + client = new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(TABLE)) { + return; + } + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(new HColumnDescriptor(CFA)); + htd.addFamily(new HColumnDescriptor(CFB)); + admin.createTable(htd); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + private static Response deleteRow(String table, String row) + throws IOException { + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(row); + Response response = client.delete(path.toString()); + Thread.yield(); + return response; + } + + private static Response deleteValue(String table, String row, String column) + throws IOException { + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(row); + path.append('/'); + path.append(column); + Response response = client.delete(path.toString()); + Thread.yield(); + return response; + } + + private static Response getValueXML(String table, String row, String column) + throws IOException { + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(row); + path.append('/'); + path.append(column); + return getValueXML(path.toString()); + } + + private static Response getValueXML(String table, String startRow, + String endRow, String column) throws IOException { + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(startRow); + path.append(","); + path.append(endRow); + path.append('/'); + path.append(column); + return getValueXML(path.toString()); + } + + private static Response getValueXML(String url) throws IOException { + Response response = client.get(url, Constants.MIMETYPE_XML); + return response; + } + + private static Response getValuePB(String table, String row, String column) + throws IOException { + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(row); + path.append('/'); + path.append(column); + return getValuePB(path.toString()); + } + + private static Response getValuePB(String url) throws IOException { + Response response = client.get(url, Constants.MIMETYPE_PROTOBUF); + return response; + } + + private static Response putValueXML(String table, String row, String column, + String value) throws IOException, JAXBException { + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(row); + path.append('/'); + path.append(column); + return putValueXML(path.toString(), table, row, column, value); + } + + private static Response putValueXML(String url, String table, String row, + String column, String value) throws IOException, JAXBException { + RowModel rowModel = new RowModel(row); + rowModel.addCell(new CellModel(Bytes.toBytes(column), + Bytes.toBytes(value))); + CellSetModel cellSetModel = new CellSetModel(); + cellSetModel.addRow(rowModel); + StringWriter writer = new StringWriter(); + marshaller.marshal(cellSetModel, writer); + Response response = client.put(url, Constants.MIMETYPE_XML, + Bytes.toBytes(writer.toString())); + Thread.yield(); + return response; + } + + private static void checkValueXML(String table, String row, String column, + String value) throws IOException, JAXBException { + Response response = getValueXML(table, row, column); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + CellSetModel cellSet = (CellSetModel) + unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); + RowModel rowModel = cellSet.getRows().get(0); + CellModel cell = rowModel.getCells().get(0); + assertEquals(Bytes.toString(cell.getColumn()), column); + assertEquals(Bytes.toString(cell.getValue()), value); + } + + private static void checkValueXML(String url, String table, String row, + String column, String value) throws IOException, JAXBException { + Response response = getValueXML(url); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + CellSetModel cellSet = (CellSetModel) + unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); + RowModel rowModel = cellSet.getRows().get(0); + CellModel cell = rowModel.getCells().get(0); + assertEquals(Bytes.toString(cell.getColumn()), column); + assertEquals(Bytes.toString(cell.getValue()), value); + } + + private static Response putValuePB(String table, String row, String column, + String value) throws IOException { + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(row); + path.append('/'); + path.append(column); + return putValuePB(path.toString(), table, row, column, value); + } + + private static Response putValuePB(String url, String table, String row, + String column, String value) throws IOException { + RowModel rowModel = new RowModel(row); + rowModel.addCell(new CellModel(Bytes.toBytes(column), + Bytes.toBytes(value))); + CellSetModel cellSetModel = new CellSetModel(); + cellSetModel.addRow(rowModel); + Response response = client.put(url, Constants.MIMETYPE_PROTOBUF, + cellSetModel.createProtobufOutput()); + Thread.yield(); + return response; + } + + private static void checkValuePB(String table, String row, String column, + String value) throws IOException { + Response response = getValuePB(table, row, column); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); + CellSetModel cellSet = new CellSetModel(); + cellSet.getObjectFromMessage(response.getBody()); + RowModel rowModel = cellSet.getRows().get(0); + CellModel cell = rowModel.getCells().get(0); + assertEquals(Bytes.toString(cell.getColumn()), column); + assertEquals(Bytes.toString(cell.getValue()), value); + } + + private static Response checkAndPutValuePB(String url, String table, + String row, String column, String valueToCheck, String valueToPut) + throws IOException { + RowModel rowModel = new RowModel(row); + rowModel.addCell(new CellModel(Bytes.toBytes(column), + Bytes.toBytes(valueToPut))); + rowModel.addCell(new CellModel(Bytes.toBytes(column), + Bytes.toBytes(valueToCheck))); + CellSetModel cellSetModel = new CellSetModel(); + cellSetModel.addRow(rowModel); + Response response = client.put(url, Constants.MIMETYPE_PROTOBUF, + cellSetModel.createProtobufOutput()); + Thread.yield(); + return response; + } + + private static Response checkAndPutValuePB(String table, String row, + String column, String valueToCheck, String valueToPut) throws IOException { + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(row); + path.append("?check=put"); + return checkAndPutValuePB(path.toString(), table, row, column, + valueToCheck, valueToPut); + } + + private static Response checkAndPutValueXML(String url, String table, + String row, String column, String valueToCheck, String valueToPut) + throws IOException, JAXBException { + RowModel rowModel = new RowModel(row); + rowModel.addCell(new CellModel(Bytes.toBytes(column), + Bytes.toBytes(valueToPut))); + rowModel.addCell(new CellModel(Bytes.toBytes(column), + Bytes.toBytes(valueToCheck))); + CellSetModel cellSetModel = new CellSetModel(); + cellSetModel.addRow(rowModel); + StringWriter writer = new StringWriter(); + marshaller.marshal(cellSetModel, writer); + Response response = client.put(url, Constants.MIMETYPE_XML, + Bytes.toBytes(writer.toString())); + Thread.yield(); + return response; + } + + private static Response checkAndPutValueXML(String table, String row, + String column, String valueToCheck, String valueToPut) + throws IOException, JAXBException { + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(row); + path.append("?check=put"); + return checkAndPutValueXML(path.toString(), table, row, column, + valueToCheck, valueToPut); + } + + private static Response checkAndDeleteXML(String url, String table, + String row, String column, String valueToCheck) + throws IOException, JAXBException { + RowModel rowModel = new RowModel(row); + rowModel.addCell(new CellModel(Bytes.toBytes(column), + Bytes.toBytes(valueToCheck))); + CellSetModel cellSetModel = new CellSetModel(); + cellSetModel.addRow(rowModel); + StringWriter writer = new StringWriter(); + marshaller.marshal(cellSetModel, writer); + Response response = client.put(url, Constants.MIMETYPE_XML, + Bytes.toBytes(writer.toString())); + Thread.yield(); + return response; + } + + private static Response checkAndDeleteXML(String table, String row, + String column, String valueToCheck) throws IOException, JAXBException { + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(row); + path.append("?check=delete"); + return checkAndDeleteXML(path.toString(), table, row, column, valueToCheck); + } + + private static Response checkAndDeletePB(String table, String row, + String column, String value) throws IOException { + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(row); + path.append("?check=delete"); + return checkAndDeleteValuePB(path.toString(), table, row, column, value); + } + + private static Response checkAndDeleteValuePB(String url, String table, + String row, String column, String valueToCheck) + throws IOException { + RowModel rowModel = new RowModel(row); + rowModel.addCell(new CellModel(Bytes.toBytes(column), Bytes + .toBytes(valueToCheck))); + CellSetModel cellSetModel = new CellSetModel(); + cellSetModel.addRow(rowModel); + Response response = client.put(url, Constants.MIMETYPE_PROTOBUF, + cellSetModel.createProtobufOutput()); + Thread.yield(); + return response; + } + + @Test + public void testDelete() throws IOException, JAXBException { + Response response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(response.getCode(), 200); + response = putValueXML(TABLE, ROW_1, COLUMN_2, VALUE_2); + assertEquals(response.getCode(), 200); + checkValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1); + checkValueXML(TABLE, ROW_1, COLUMN_2, VALUE_2); + + response = deleteValue(TABLE, ROW_1, COLUMN_1); + assertEquals(response.getCode(), 200); + response = getValueXML(TABLE, ROW_1, COLUMN_1); + assertEquals(response.getCode(), 404); + checkValueXML(TABLE, ROW_1, COLUMN_2, VALUE_2); + + response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(response.getCode(), 200); + response = checkAndDeletePB(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(response.getCode(), 200); + response = getValueXML(TABLE, ROW_1, COLUMN_1); + assertEquals(response.getCode(), 404); + + response = deleteRow(TABLE, ROW_1); + assertEquals(response.getCode(), 200); + response = getValueXML(TABLE, ROW_1, COLUMN_1); + assertEquals(response.getCode(), 404); + response = getValueXML(TABLE, ROW_1, COLUMN_2); + assertEquals(response.getCode(), 404); + } + + @Test + public void testForbidden() throws IOException, JAXBException { + conf.set("hbase.rest.readonly", "true"); + + Response response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(response.getCode(), 403); + response = putValuePB(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(response.getCode(), 403); + response = checkAndPutValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1, VALUE_2); + assertEquals(response.getCode(), 403); + response = checkAndPutValuePB(TABLE, ROW_1, COLUMN_1, VALUE_1, VALUE_2); + assertEquals(response.getCode(), 403); + response = deleteValue(TABLE, ROW_1, COLUMN_1); + assertEquals(response.getCode(), 403); + response = checkAndDeletePB(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(response.getCode(), 403); + response = deleteRow(TABLE, ROW_1); + assertEquals(response.getCode(), 403); + + conf.set("hbase.rest.readonly", "false"); + + response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(response.getCode(), 200); + response = putValuePB(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(response.getCode(), 200); + response = checkAndPutValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1, VALUE_2); + assertEquals(response.getCode(), 200); + response = checkAndPutValuePB(TABLE, ROW_1, COLUMN_1, VALUE_2, VALUE_3); + assertEquals(response.getCode(), 200); + response = deleteValue(TABLE, ROW_1, COLUMN_1); + assertEquals(response.getCode(), 200); + response = deleteRow(TABLE, ROW_1); + assertEquals(response.getCode(), 200); + } + + @Test + public void testSingleCellGetPutXML() throws IOException, JAXBException { + Response response = getValueXML(TABLE, ROW_1, COLUMN_1); + assertEquals(response.getCode(), 404); + + response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(response.getCode(), 200); + checkValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1); + response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_2); + assertEquals(response.getCode(), 200); + checkValueXML(TABLE, ROW_1, COLUMN_1, VALUE_2); + response = checkAndPutValueXML(TABLE, ROW_1, COLUMN_1, VALUE_2, VALUE_3); + assertEquals(response.getCode(), 200); + checkValueXML(TABLE, ROW_1, COLUMN_1, VALUE_3); + response = checkAndDeleteXML(TABLE, ROW_1, COLUMN_1, VALUE_3); + assertEquals(response.getCode(), 200); + + response = deleteRow(TABLE, ROW_1); + assertEquals(response.getCode(), 200); + } + + @Test + public void testSingleCellGetPutPB() throws IOException, JAXBException { + Response response = getValuePB(TABLE, ROW_1, COLUMN_1); + assertEquals(response.getCode(), 404); + + response = putValuePB(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(response.getCode(), 200); + checkValuePB(TABLE, ROW_1, COLUMN_1, VALUE_1); + + response = putValuePB(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(response.getCode(), 200); + checkValuePB(TABLE, ROW_1, COLUMN_1, VALUE_1); + response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_2); + assertEquals(response.getCode(), 200); + checkValuePB(TABLE, ROW_1, COLUMN_1, VALUE_2); + + response = checkAndPutValuePB(TABLE, ROW_1, COLUMN_1, VALUE_2, VALUE_3); + assertEquals(response.getCode(), 200); + checkValuePB(TABLE, ROW_1, COLUMN_1, VALUE_3); + response = checkAndPutValueXML(TABLE, ROW_1, COLUMN_1, VALUE_3, VALUE_4); + assertEquals(response.getCode(), 200); + checkValuePB(TABLE, ROW_1, COLUMN_1, VALUE_4); + + response = deleteRow(TABLE, ROW_1); + assertEquals(response.getCode(), 200); + } + + @Test + public void testSingleCellGetPutBinary() throws IOException { + final String path = "/" + TABLE + "/" + ROW_3 + "/" + COLUMN_1; + final byte[] body = Bytes.toBytes(VALUE_3); + Response response = client.put(path, Constants.MIMETYPE_BINARY, body); + assertEquals(response.getCode(), 200); + Thread.yield(); + + response = client.get(path, Constants.MIMETYPE_BINARY); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type")); + assertTrue(Bytes.equals(response.getBody(), body)); + boolean foundTimestampHeader = false; + for (Header header: response.getHeaders()) { + if (header.getName().equals("X-Timestamp")) { + foundTimestampHeader = true; + break; + } + } + assertTrue(foundTimestampHeader); + + response = deleteRow(TABLE, ROW_3); + assertEquals(response.getCode(), 200); + } + + @Test + public void testSingleCellGetJSON() throws IOException, JAXBException { + final String path = "/" + TABLE + "/" + ROW_4 + "/" + COLUMN_1; + Response response = client.put(path, Constants.MIMETYPE_BINARY, + Bytes.toBytes(VALUE_4)); + assertEquals(response.getCode(), 200); + Thread.yield(); + response = client.get(path, Constants.MIMETYPE_JSON); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type")); + response = deleteRow(TABLE, ROW_4); + assertEquals(response.getCode(), 200); + } + + @Test + public void testURLEncodedKey() throws IOException, JAXBException { + String urlKey = "http://example.com/foo"; + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(TABLE); + path.append('/'); + path.append(URLEncoder.encode(urlKey, HConstants.UTF8_ENCODING)); + path.append('/'); + path.append(COLUMN_1); + Response response; + response = putValueXML(path.toString(), TABLE, urlKey, COLUMN_1, + VALUE_1); + assertEquals(response.getCode(), 200); + checkValueXML(path.toString(), TABLE, urlKey, COLUMN_1, VALUE_1); + } + + @Test + public void testNoSuchCF() throws IOException, JAXBException { + final String goodPath = "/" + TABLE + "/" + ROW_1 + "/" + CFA+":"; + final String badPath = "/" + TABLE + "/" + ROW_1 + "/" + "BAD"; + Response response = client.post(goodPath, Constants.MIMETYPE_BINARY, + Bytes.toBytes(VALUE_1)); + assertEquals(response.getCode(), 200); + assertEquals(client.get(goodPath, Constants.MIMETYPE_BINARY).getCode(), + 200); + assertEquals(client.get(badPath, Constants.MIMETYPE_BINARY).getCode(), + 404); + assertEquals(client.get(goodPath, Constants.MIMETYPE_BINARY).getCode(), + 200); + } + + @Test + public void testMultiCellGetPutXML() throws IOException, JAXBException { + String path = "/" + TABLE + "/fakerow"; // deliberate nonexistent row + + CellSetModel cellSetModel = new CellSetModel(); + RowModel rowModel = new RowModel(ROW_1); + rowModel.addCell(new CellModel(Bytes.toBytes(COLUMN_1), + Bytes.toBytes(VALUE_1))); + rowModel.addCell(new CellModel(Bytes.toBytes(COLUMN_2), + Bytes.toBytes(VALUE_2))); + cellSetModel.addRow(rowModel); + rowModel = new RowModel(ROW_2); + rowModel.addCell(new CellModel(Bytes.toBytes(COLUMN_1), + Bytes.toBytes(VALUE_3))); + rowModel.addCell(new CellModel(Bytes.toBytes(COLUMN_2), + Bytes.toBytes(VALUE_4))); + cellSetModel.addRow(rowModel); + StringWriter writer = new StringWriter(); + marshaller.marshal(cellSetModel, writer); + Response response = client.put(path, Constants.MIMETYPE_XML, + Bytes.toBytes(writer.toString())); + Thread.yield(); + + // make sure the fake row was not actually created + response = client.get(path, Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 404); + + // check that all of the values were created + checkValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1); + checkValueXML(TABLE, ROW_1, COLUMN_2, VALUE_2); + checkValueXML(TABLE, ROW_2, COLUMN_1, VALUE_3); + checkValueXML(TABLE, ROW_2, COLUMN_2, VALUE_4); + + response = deleteRow(TABLE, ROW_1); + assertEquals(response.getCode(), 200); + response = deleteRow(TABLE, ROW_2); + assertEquals(response.getCode(), 200); + } + + @Test + public void testMultiCellGetPutPB() throws IOException { + String path = "/" + TABLE + "/fakerow"; // deliberate nonexistent row + + CellSetModel cellSetModel = new CellSetModel(); + RowModel rowModel = new RowModel(ROW_1); + rowModel.addCell(new CellModel(Bytes.toBytes(COLUMN_1), + Bytes.toBytes(VALUE_1))); + rowModel.addCell(new CellModel(Bytes.toBytes(COLUMN_2), + Bytes.toBytes(VALUE_2))); + cellSetModel.addRow(rowModel); + rowModel = new RowModel(ROW_2); + rowModel.addCell(new CellModel(Bytes.toBytes(COLUMN_1), + Bytes.toBytes(VALUE_3))); + rowModel.addCell(new CellModel(Bytes.toBytes(COLUMN_2), + Bytes.toBytes(VALUE_4))); + cellSetModel.addRow(rowModel); + Response response = client.put(path, Constants.MIMETYPE_PROTOBUF, + cellSetModel.createProtobufOutput()); + Thread.yield(); + + // make sure the fake row was not actually created + response = client.get(path, Constants.MIMETYPE_PROTOBUF); + assertEquals(response.getCode(), 404); + + // check that all of the values were created + checkValuePB(TABLE, ROW_1, COLUMN_1, VALUE_1); + checkValuePB(TABLE, ROW_1, COLUMN_2, VALUE_2); + checkValuePB(TABLE, ROW_2, COLUMN_1, VALUE_3); + checkValuePB(TABLE, ROW_2, COLUMN_2, VALUE_4); + + response = deleteRow(TABLE, ROW_1); + assertEquals(response.getCode(), 200); + response = deleteRow(TABLE, ROW_2); + assertEquals(response.getCode(), 200); + } + + @Test + public void testStartEndRowGetPutXML() throws IOException, JAXBException { + String[] rows = { ROW_1, ROW_2, ROW_3 }; + String[] values = { VALUE_1, VALUE_2, VALUE_3 }; + Response response = null; + for (int i = 0; i < rows.length; i++) { + response = putValueXML(TABLE, rows[i], COLUMN_1, values[i]); + assertEquals(200, response.getCode()); + checkValueXML(TABLE, rows[i], COLUMN_1, values[i]); + } + response = getValueXML(TABLE, rows[0], rows[2], COLUMN_1); + assertEquals(200, response.getCode()); + CellSetModel cellSet = (CellSetModel) + unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); + assertEquals(2, cellSet.getRows().size()); + for (int i = 0; i < cellSet.getRows().size()-1; i++) { + RowModel rowModel = cellSet.getRows().get(i); + for (CellModel cell: rowModel.getCells()) { + assertEquals(COLUMN_1, Bytes.toString(cell.getColumn())); + assertEquals(values[i], Bytes.toString(cell.getValue())); + } + } + for (String row : rows) { + response = deleteRow(TABLE, row); + assertEquals(200, response.getCode()); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java new file mode 100644 index 0000000..3c09c38 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java @@ -0,0 +1,355 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.util.Iterator; +import java.util.Random; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; + +import org.apache.commons.httpclient.Header; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.CellModel; +import org.apache.hadoop.hbase.rest.model.CellSetModel; +import org.apache.hadoop.hbase.rest.model.RowModel; +import org.apache.hadoop.hbase.rest.model.ScannerModel; +import org.apache.hadoop.hbase.util.Bytes; + +import static org.junit.Assert.*; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestScannerResource { + private static final String TABLE = "TestScannerResource"; + private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist"; + private static final String CFA = "a"; + private static final String CFB = "b"; + private static final String COLUMN_1 = CFA + ":1"; + private static final String COLUMN_2 = CFB + ":2"; + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static Client client; + private static JAXBContext context; + private static Marshaller marshaller; + private static Unmarshaller unmarshaller; + private static int expectedRows1; + private static int expectedRows2; + private static Configuration conf; + + private static int insertData(String tableName, String column, double prob) + throws IOException { + Random rng = new Random(); + int count = 0; + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + byte[] k = new byte[3]; + byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column)); + for (byte b1 = 'a'; b1 < 'z'; b1++) { + for (byte b2 = 'a'; b2 < 'z'; b2++) { + for (byte b3 = 'a'; b3 < 'z'; b3++) { + if (rng.nextDouble() < prob) { + k[0] = b1; + k[1] = b2; + k[2] = b3; + Put put = new Put(k); + put.setWriteToWAL(false); + put.add(famAndQf[0], famAndQf[1], k); + table.put(put); + count++; + } + } + } + } + table.flushCommits(); + return count; + } + + private static int countCellSet(CellSetModel model) { + int count = 0; + Iterator rows = model.getRows().iterator(); + while (rows.hasNext()) { + RowModel row = rows.next(); + Iterator cells = row.getCells().iterator(); + while (cells.hasNext()) { + cells.next(); + count++; + } + } + return count; + } + + private static int fullTableScan(ScannerModel model) throws IOException { + model.setBatch(100); + Response response = client.put("/" + TABLE + "/scanner", + Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); + assertEquals(response.getCode(), 201); + String scannerURI = response.getLocation(); + assertNotNull(scannerURI); + int count = 0; + while (true) { + response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF); + assertTrue(response.getCode() == 200 || response.getCode() == 204); + if (response.getCode() == 200) { + assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); + CellSetModel cellSet = new CellSetModel(); + cellSet.getObjectFromMessage(response.getBody()); + Iterator rows = cellSet.getRows().iterator(); + while (rows.hasNext()) { + RowModel row = rows.next(); + Iterator cells = row.getCells().iterator(); + while (cells.hasNext()) { + cells.next(); + count++; + } + } + } else { + break; + } + } + // delete the scanner + response = client.delete(scannerURI); + assertEquals(response.getCode(), 200); + return count; + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf = TEST_UTIL.getConfiguration(); + TEST_UTIL.startMiniCluster(); + REST_TEST_UTIL.startServletContainer(conf); + client = new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())); + context = JAXBContext.newInstance( + CellModel.class, + CellSetModel.class, + RowModel.class, + ScannerModel.class); + marshaller = context.createMarshaller(); + unmarshaller = context.createUnmarshaller(); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(TABLE)) { + return; + } + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(new HColumnDescriptor(CFA)); + htd.addFamily(new HColumnDescriptor(CFB)); + admin.createTable(htd); + expectedRows1 = insertData(TABLE, COLUMN_1, 1.0); + expectedRows2 = insertData(TABLE, COLUMN_2, 0.5); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testSimpleScannerXML() throws IOException, JAXBException { + final int BATCH_SIZE = 5; + // new scanner + ScannerModel model = new ScannerModel(); + model.setBatch(BATCH_SIZE); + model.addColumn(Bytes.toBytes(COLUMN_1)); + StringWriter writer = new StringWriter(); + marshaller.marshal(model, writer); + byte[] body = Bytes.toBytes(writer.toString()); + + // test put operation is forbidden in read-only mode + conf.set("hbase.rest.readonly", "true"); + Response response = client.put("/" + TABLE + "/scanner", + Constants.MIMETYPE_XML, body); + assertEquals(response.getCode(), 403); + String scannerURI = response.getLocation(); + assertNull(scannerURI); + + // recall previous put operation with read-only off + conf.set("hbase.rest.readonly", "false"); + response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, + body); + assertEquals(response.getCode(), 201); + scannerURI = response.getLocation(); + assertNotNull(scannerURI); + + // get a cell set + response = client.get(scannerURI, Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + CellSetModel cellSet = (CellSetModel) + unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); + // confirm batch size conformance + assertEquals(countCellSet(cellSet), BATCH_SIZE); + + // test delete scanner operation is forbidden in read-only mode + conf.set("hbase.rest.readonly", "true"); + response = client.delete(scannerURI); + assertEquals(response.getCode(), 403); + + // recall previous delete scanner operation with read-only off + conf.set("hbase.rest.readonly", "false"); + response = client.delete(scannerURI); + assertEquals(response.getCode(), 200); + } + + @Test + public void testSimpleScannerPB() throws IOException { + final int BATCH_SIZE = 10; + // new scanner + ScannerModel model = new ScannerModel(); + model.setBatch(BATCH_SIZE); + model.addColumn(Bytes.toBytes(COLUMN_1)); + + // test put operation is forbidden in read-only mode + conf.set("hbase.rest.readonly", "true"); + Response response = client.put("/" + TABLE + "/scanner", + Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); + assertEquals(response.getCode(), 403); + String scannerURI = response.getLocation(); + assertNull(scannerURI); + + // recall previous put operation with read-only off + conf.set("hbase.rest.readonly", "false"); + response = client.put("/" + TABLE + "/scanner", + Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); + assertEquals(response.getCode(), 201); + scannerURI = response.getLocation(); + assertNotNull(scannerURI); + + // get a cell set + response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); + CellSetModel cellSet = new CellSetModel(); + cellSet.getObjectFromMessage(response.getBody()); + // confirm batch size conformance + assertEquals(countCellSet(cellSet), BATCH_SIZE); + + // test delete scanner operation is forbidden in read-only mode + conf.set("hbase.rest.readonly", "true"); + response = client.delete(scannerURI); + assertEquals(response.getCode(), 403); + + // recall previous delete scanner operation with read-only off + conf.set("hbase.rest.readonly", "false"); + response = client.delete(scannerURI); + assertEquals(response.getCode(), 200); + } + + @Test + public void testSimpleScannerBinary() throws IOException { + // new scanner + ScannerModel model = new ScannerModel(); + model.setBatch(1); + model.addColumn(Bytes.toBytes(COLUMN_1)); + + // test put operation is forbidden in read-only mode + conf.set("hbase.rest.readonly", "true"); + Response response = client.put("/" + TABLE + "/scanner", + Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); + assertEquals(response.getCode(), 403); + String scannerURI = response.getLocation(); + assertNull(scannerURI); + + // recall previous put operation with read-only off + conf.set("hbase.rest.readonly", "false"); + response = client.put("/" + TABLE + "/scanner", + Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput()); + assertEquals(response.getCode(), 201); + scannerURI = response.getLocation(); + assertNotNull(scannerURI); + + // get a cell + response = client.get(scannerURI, Constants.MIMETYPE_BINARY); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type")); + // verify that data was returned + assertTrue(response.getBody().length > 0); + // verify that the expected X-headers are present + boolean foundRowHeader = false, foundColumnHeader = false, + foundTimestampHeader = false; + for (Header header: response.getHeaders()) { + if (header.getName().equals("X-Row")) { + foundRowHeader = true; + } else if (header.getName().equals("X-Column")) { + foundColumnHeader = true; + } else if (header.getName().equals("X-Timestamp")) { + foundTimestampHeader = true; + } + } + assertTrue(foundRowHeader); + assertTrue(foundColumnHeader); + assertTrue(foundTimestampHeader); + + // test delete scanner operation is forbidden in read-only mode + conf.set("hbase.rest.readonly", "true"); + response = client.delete(scannerURI); + assertEquals(response.getCode(), 403); + + // recall previous delete scanner operation with read-only off + conf.set("hbase.rest.readonly", "false"); + response = client.delete(scannerURI); + assertEquals(response.getCode(), 200); + } + + @Test + public void testFullTableScan() throws IOException { + ScannerModel model = new ScannerModel(); + model.addColumn(Bytes.toBytes(COLUMN_1)); + assertEquals(fullTableScan(model), expectedRows1); + + model = new ScannerModel(); + model.addColumn(Bytes.toBytes(COLUMN_2)); + assertEquals(fullTableScan(model), expectedRows2); + } + + @Test + public void testTableDoesNotExist() throws IOException, JAXBException { + ScannerModel model = new ScannerModel(); + StringWriter writer = new StringWriter(); + marshaller.marshal(model, writer); + byte[] body = Bytes.toBytes(writer.toString()); + Response response = client.put("/" + NONEXISTENT_TABLE + + "/scanner", Constants.MIMETYPE_XML, body); + assertEquals(response.getCode(), 404); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java b/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java new file mode 100644 index 0000000..d83e186 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java @@ -0,0 +1,1002 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.ByteArrayInputStream; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.InclusiveStopFilter; +import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.filter.RegexStringComparator; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SkipFilter; +import org.apache.hadoop.hbase.filter.SubstringComparator; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.CellModel; +import org.apache.hadoop.hbase.rest.model.CellSetModel; +import org.apache.hadoop.hbase.rest.model.RowModel; +import org.apache.hadoop.hbase.rest.model.ScannerModel; +import org.apache.hadoop.hbase.util.Bytes; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestScannersWithFilters { + + private static final Log LOG = LogFactory.getLog(TestScannersWithFilters.class); + + private static final String TABLE = "TestScannersWithFilters"; + + private static final byte [][] ROWS_ONE = { + Bytes.toBytes("testRowOne-0"), Bytes.toBytes("testRowOne-1"), + Bytes.toBytes("testRowOne-2"), Bytes.toBytes("testRowOne-3") + }; + + private static final byte [][] ROWS_TWO = { + Bytes.toBytes("testRowTwo-0"), Bytes.toBytes("testRowTwo-1"), + Bytes.toBytes("testRowTwo-2"), Bytes.toBytes("testRowTwo-3") + }; + + private static final byte [][] FAMILIES = { + Bytes.toBytes("testFamilyOne"), Bytes.toBytes("testFamilyTwo") + }; + + private static final byte [][] QUALIFIERS_ONE = { + Bytes.toBytes("testQualifierOne-0"), Bytes.toBytes("testQualifierOne-1"), + Bytes.toBytes("testQualifierOne-2"), Bytes.toBytes("testQualifierOne-3") + }; + + private static final byte [][] QUALIFIERS_TWO = { + Bytes.toBytes("testQualifierTwo-0"), Bytes.toBytes("testQualifierTwo-1"), + Bytes.toBytes("testQualifierTwo-2"), Bytes.toBytes("testQualifierTwo-3") + }; + + private static final byte [][] VALUES = { + Bytes.toBytes("testValueOne"), Bytes.toBytes("testValueTwo") + }; + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static Client client; + private static JAXBContext context; + private static Marshaller marshaller; + private static Unmarshaller unmarshaller; + private static long numRows = ROWS_ONE.length + ROWS_TWO.length; + private static long colsPerRow = FAMILIES.length * QUALIFIERS_ONE.length; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(3); + REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); + context = JAXBContext.newInstance( + CellModel.class, + CellSetModel.class, + RowModel.class, + ScannerModel.class); + marshaller = context.createMarshaller(); + unmarshaller = context.createUnmarshaller(); + client = new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + if (!admin.tableExists(TABLE)) { + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(new HColumnDescriptor(FAMILIES[0])); + htd.addFamily(new HColumnDescriptor(FAMILIES[1])); + admin.createTable(htd); + HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE); + // Insert first half + for(byte [] ROW : ROWS_ONE) { + Put p = new Put(ROW); + p.setWriteToWAL(false); + for(byte [] QUALIFIER : QUALIFIERS_ONE) { + p.add(FAMILIES[0], QUALIFIER, VALUES[0]); + } + table.put(p); + } + for(byte [] ROW : ROWS_TWO) { + Put p = new Put(ROW); + p.setWriteToWAL(false); + for(byte [] QUALIFIER : QUALIFIERS_TWO) { + p.add(FAMILIES[1], QUALIFIER, VALUES[1]); + } + table.put(p); + } + + // Insert second half (reverse families) + for(byte [] ROW : ROWS_ONE) { + Put p = new Put(ROW); + p.setWriteToWAL(false); + for(byte [] QUALIFIER : QUALIFIERS_ONE) { + p.add(FAMILIES[1], QUALIFIER, VALUES[0]); + } + table.put(p); + } + for(byte [] ROW : ROWS_TWO) { + Put p = new Put(ROW); + p.setWriteToWAL(false); + for(byte [] QUALIFIER : QUALIFIERS_TWO) { + p.add(FAMILIES[0], QUALIFIER, VALUES[1]); + } + table.put(p); + } + + // Delete the second qualifier from all rows and families + for(byte [] ROW : ROWS_ONE) { + Delete d = new Delete(ROW); + d.deleteColumns(FAMILIES[0], QUALIFIERS_ONE[1]); + d.deleteColumns(FAMILIES[1], QUALIFIERS_ONE[1]); + table.delete(d); + } + for(byte [] ROW : ROWS_TWO) { + Delete d = new Delete(ROW); + d.deleteColumns(FAMILIES[0], QUALIFIERS_TWO[1]); + d.deleteColumns(FAMILIES[1], QUALIFIERS_TWO[1]); + table.delete(d); + } + colsPerRow -= 2; + + // Delete the second rows from both groups, one column at a time + for(byte [] QUALIFIER : QUALIFIERS_ONE) { + Delete d = new Delete(ROWS_ONE[1]); + d.deleteColumns(FAMILIES[0], QUALIFIER); + d.deleteColumns(FAMILIES[1], QUALIFIER); + table.delete(d); + } + for(byte [] QUALIFIER : QUALIFIERS_TWO) { + Delete d = new Delete(ROWS_TWO[1]); + d.deleteColumns(FAMILIES[0], QUALIFIER); + d.deleteColumns(FAMILIES[1], QUALIFIER); + table.delete(d); + } + numRows -= 2; + table.close(); + } + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + private static void verifyScan(Scan s, long expectedRows, long expectedKeys) + throws Exception { + ScannerModel model = ScannerModel.fromScan(s); + model.setBatch(Integer.MAX_VALUE); // fetch it all at once + StringWriter writer = new StringWriter(); + marshaller.marshal(model, writer); + LOG.debug(writer.toString()); + byte[] body = Bytes.toBytes(writer.toString()); + Response response = client.put("/" + TABLE + "/scanner", + Constants.MIMETYPE_XML, body); + assertEquals(response.getCode(), 201); + String scannerURI = response.getLocation(); + assertNotNull(scannerURI); + + // get a cell set + response = client.get(scannerURI, Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + CellSetModel cells = (CellSetModel) + unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); + + int rows = cells.getRows().size(); + assertTrue("Scanned too many rows! Only expected " + expectedRows + + " total but scanned " + rows, expectedRows == rows); + for (RowModel row: cells.getRows()) { + int count = row.getCells().size(); + assertEquals("Expected " + expectedKeys + " keys per row but " + + "returned " + count, expectedKeys, count); + } + + // delete the scanner + response = client.delete(scannerURI); + assertEquals(response.getCode(), 200); + } + + private static void verifyScanFull(Scan s, KeyValue [] kvs) + throws Exception { + ScannerModel model = ScannerModel.fromScan(s); + model.setBatch(Integer.MAX_VALUE); // fetch it all at once + StringWriter writer = new StringWriter(); + marshaller.marshal(model, writer); + LOG.debug(writer.toString()); + byte[] body = Bytes.toBytes(writer.toString()); + Response response = client.put("/" + TABLE + "/scanner", + Constants.MIMETYPE_XML, body); + assertEquals(response.getCode(), 201); + String scannerURI = response.getLocation(); + assertNotNull(scannerURI); + + // get a cell set + response = client.get(scannerURI, Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + CellSetModel cellSet = (CellSetModel) + unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); + + // delete the scanner + response = client.delete(scannerURI); + assertEquals(response.getCode(), 200); + + int row = 0; + int idx = 0; + Iterator i = cellSet.getRows().iterator(); + for (boolean done = true; done; row++) { + done = i.hasNext(); + if (!done) break; + RowModel rowModel = i.next(); + List cells = rowModel.getCells(); + if (cells.isEmpty()) break; + assertTrue("Scanned too many keys! Only expected " + kvs.length + + " total but already scanned " + (cells.size() + idx), + kvs.length >= idx + cells.size()); + for (CellModel cell: cells) { + assertTrue("Row mismatch", + Bytes.equals(rowModel.getKey(), kvs[idx].getRow())); + byte[][] split = KeyValue.parseColumn(cell.getColumn()); + assertTrue("Family mismatch", + Bytes.equals(split[0], kvs[idx].getFamily())); + assertTrue("Qualifier mismatch", + Bytes.equals(split[1], kvs[idx].getQualifier())); + assertTrue("Value mismatch", + Bytes.equals(cell.getValue(), kvs[idx].getValue())); + idx++; + } + } + assertEquals("Expected " + kvs.length + " total keys but scanned " + idx, + kvs.length, idx); + } + + private static void verifyScanNoEarlyOut(Scan s, long expectedRows, + long expectedKeys) throws Exception { + ScannerModel model = ScannerModel.fromScan(s); + model.setBatch(Integer.MAX_VALUE); // fetch it all at once + StringWriter writer = new StringWriter(); + marshaller.marshal(model, writer); + LOG.debug(writer.toString()); + byte[] body = Bytes.toBytes(writer.toString()); + Response response = client.put("/" + TABLE + "/scanner", + Constants.MIMETYPE_XML, body); + assertEquals(response.getCode(), 201); + String scannerURI = response.getLocation(); + assertNotNull(scannerURI); + + // get a cell set + response = client.get(scannerURI, Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + CellSetModel cellSet = (CellSetModel) + unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); + + // delete the scanner + response = client.delete(scannerURI); + assertEquals(response.getCode(), 200); + + Iterator i = cellSet.getRows().iterator(); + int j = 0; + for (boolean done = true; done; j++) { + done = i.hasNext(); + if (!done) break; + RowModel rowModel = i.next(); + List cells = rowModel.getCells(); + if (cells.isEmpty()) break; + assertTrue("Scanned too many rows! Only expected " + expectedRows + + " total but already scanned " + (j+1), expectedRows > j); + assertEquals("Expected " + expectedKeys + " keys per row but " + + "returned " + cells.size(), expectedKeys, cells.size()); + } + assertEquals("Expected " + expectedRows + " rows but scanned " + j + + " rows", expectedRows, j); + } + + @Test + public void testNoFilter() throws Exception { + // No filter + long expectedRows = numRows; + long expectedKeys = colsPerRow; + + // Both families + Scan s = new Scan(); + verifyScan(s, expectedRows, expectedKeys); + + // One family + s = new Scan(); + s.addFamily(FAMILIES[0]); + verifyScan(s, expectedRows, expectedKeys/2); + } + + @Test + public void testPrefixFilter() throws Exception { + // Grab rows from group one (half of total) + long expectedRows = numRows / 2; + long expectedKeys = colsPerRow; + Scan s = new Scan(); + s.setFilter(new PrefixFilter(Bytes.toBytes("testRowOne"))); + verifyScan(s, expectedRows, expectedKeys); + } + + @Test + public void testPageFilter() throws Exception { + // KVs in first 6 rows + KeyValue [] expectedKVs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) + }; + + // Grab all 6 rows + long expectedRows = 6; + long expectedKeys = colsPerRow; + Scan s = new Scan(); + s.setFilter(new PageFilter(expectedRows)); + verifyScan(s, expectedRows, expectedKeys); + s.setFilter(new PageFilter(expectedRows)); + verifyScanFull(s, expectedKVs); + + // Grab first 4 rows (6 cols per row) + expectedRows = 4; + expectedKeys = colsPerRow; + s = new Scan(); + s.setFilter(new PageFilter(expectedRows)); + verifyScan(s, expectedRows, expectedKeys); + s.setFilter(new PageFilter(expectedRows)); + verifyScanFull(s, Arrays.copyOf(expectedKVs, 24)); + + // Grab first 2 rows + expectedRows = 2; + expectedKeys = colsPerRow; + s = new Scan(); + s.setFilter(new PageFilter(expectedRows)); + verifyScan(s, expectedRows, expectedKeys); + s.setFilter(new PageFilter(expectedRows)); + verifyScanFull(s, Arrays.copyOf(expectedKVs, 12)); + + // Grab first row + expectedRows = 1; + expectedKeys = colsPerRow; + s = new Scan(); + s.setFilter(new PageFilter(expectedRows)); + verifyScan(s, expectedRows, expectedKeys); + s.setFilter(new PageFilter(expectedRows)); + verifyScanFull(s, Arrays.copyOf(expectedKVs, 6)); + } + + @Test + public void testInclusiveStopFilter() throws Exception { + // Grab rows from group one + + // If we just use start/stop row, we get total/2 - 1 rows + long expectedRows = (numRows / 2) - 1; + long expectedKeys = colsPerRow; + Scan s = new Scan(Bytes.toBytes("testRowOne-0"), + Bytes.toBytes("testRowOne-3")); + verifyScan(s, expectedRows, expectedKeys); + + // Now use start row with inclusive stop filter + expectedRows = numRows / 2; + s = new Scan(Bytes.toBytes("testRowOne-0")); + s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowOne-3"))); + verifyScan(s, expectedRows, expectedKeys); + + // Grab rows from group two + + // If we just use start/stop row, we get total/2 - 1 rows + expectedRows = (numRows / 2) - 1; + expectedKeys = colsPerRow; + s = new Scan(Bytes.toBytes("testRowTwo-0"), + Bytes.toBytes("testRowTwo-3")); + verifyScan(s, expectedRows, expectedKeys); + + // Now use start row with inclusive stop filter + expectedRows = numRows / 2; + s = new Scan(Bytes.toBytes("testRowTwo-0")); + s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowTwo-3"))); + verifyScan(s, expectedRows, expectedKeys); + } + + @Test + public void testQualifierFilter() throws Exception { + // Match two keys (one from each family) in half the rows + long expectedRows = numRows / 2; + long expectedKeys = 2; + Filter f = new QualifierFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + Scan s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys less than same qualifier + // Expect only two keys (one from each family) in half the rows + expectedRows = numRows / 2; + expectedKeys = 2; + f = new QualifierFilter(CompareOp.LESS, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys less than or equal + // Expect four keys (two from each family) in half the rows + expectedRows = numRows / 2; + expectedKeys = 4; + f = new QualifierFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys not equal + // Expect four keys (two from each family) + // Only look in first group of rows + expectedRows = numRows / 2; + expectedKeys = 4; + f = new QualifierFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(HConstants.EMPTY_START_ROW, Bytes.toBytes("testRowTwo")); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys greater or equal + // Expect four keys (two from each family) + // Only look in first group of rows + expectedRows = numRows / 2; + expectedKeys = 4; + f = new QualifierFilter(CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(HConstants.EMPTY_START_ROW, Bytes.toBytes("testRowTwo")); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys greater + // Expect two keys (one from each family) + // Only look in first group of rows + expectedRows = numRows / 2; + expectedKeys = 2; + f = new QualifierFilter(CompareOp.GREATER, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(HConstants.EMPTY_START_ROW, Bytes.toBytes("testRowTwo")); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys not equal to + // Look across rows and fully validate the keys and ordering + // Expect varied numbers of keys, 4 per row in group one, 6 per row in + // group two + f = new QualifierFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(QUALIFIERS_ONE[2])); + s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + + // Test across rows and groups with a regex + // Filter out "test*-2" + // Expect 4 keys per row across both groups + f = new QualifierFilter(CompareOp.NOT_EQUAL, + new RegexStringComparator("test.+-2")); + s = new Scan(); + s.setFilter(f); + + kvs = new KeyValue [] { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + } + + @Test + public void testRowFilter() throws Exception { + // Match a single row, all keys + long expectedRows = 1; + long expectedKeys = colsPerRow; + Filter f = new RowFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + Scan s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match a two rows, one from each group, using regex + expectedRows = 2; + expectedKeys = colsPerRow; + f = new RowFilter(CompareOp.EQUAL, + new RegexStringComparator("testRow.+-2")); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match rows less than + // Expect all keys in one row + expectedRows = 1; + expectedKeys = colsPerRow; + f = new RowFilter(CompareOp.LESS, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match rows less than or equal + // Expect all keys in two rows + expectedRows = 2; + expectedKeys = colsPerRow; + f = new RowFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match rows not equal + // Expect all keys in all but one row + expectedRows = numRows - 1; + expectedKeys = colsPerRow; + f = new RowFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys greater or equal + // Expect all keys in all but one row + expectedRows = numRows - 1; + expectedKeys = colsPerRow; + f = new RowFilter(CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys greater + // Expect all keys in all but two rows + expectedRows = numRows - 2; + expectedKeys = colsPerRow; + f = new RowFilter(CompareOp.GREATER, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match rows not equal to testRowTwo-2 + // Look across rows and fully validate the keys and ordering + // Should see all keys in all rows but testRowTwo-2 + f = new RowFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + + // Test across rows and groups with a regex + // Filter out everything that doesn't match "*-2" + // Expect all keys in two rows + f = new RowFilter(CompareOp.EQUAL, + new RegexStringComparator(".+-2")); + s = new Scan(); + s.setFilter(f); + + kvs = new KeyValue [] { + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) + }; + verifyScanFull(s, kvs); + } + + @Test + public void testValueFilter() throws Exception { + // Match group one rows + long expectedRows = numRows / 2; + long expectedKeys = colsPerRow; + Filter f = new ValueFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + Scan s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match group two rows + expectedRows = numRows / 2; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testValueTwo"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match all values using regex + expectedRows = numRows; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.EQUAL, + new RegexStringComparator("testValue((One)|(Two))")); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values less than + // Expect group one rows + expectedRows = numRows / 2; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.LESS, + new BinaryComparator(Bytes.toBytes("testValueTwo"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values less than or equal + // Expect all rows + expectedRows = numRows; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueTwo"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values less than or equal + // Expect group one rows + expectedRows = numRows / 2; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values not equal + // Expect half the rows + expectedRows = numRows / 2; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values greater or equal + // Expect all rows + expectedRows = numRows; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values greater + // Expect half rows + expectedRows = numRows / 2; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.GREATER, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values not equal to testValueOne + // Look across rows and fully validate the keys and ordering + // Should see all keys in all group two rows + f = new ValueFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + } + + @Test + public void testSkipFilter() throws Exception { + // Test for qualifier regex: "testQualifierOne-2" + // Should only get rows from second group, and all keys + Filter f = new SkipFilter(new QualifierFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2")))); + Scan s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + } + + @Test + public void testFilterList() throws Exception { + // Test getting a single row, single key using Row, Qualifier, and Value + // regular expression and substring filters + // Use must pass all + List filters = new ArrayList(); + filters.add(new RowFilter(CompareOp.EQUAL, + new RegexStringComparator(".+-2"))); + filters.add(new QualifierFilter(CompareOp.EQUAL, + new RegexStringComparator(".+-2"))); + filters.add(new ValueFilter(CompareOp.EQUAL, + new SubstringComparator("One"))); + Filter f = new FilterList(Operator.MUST_PASS_ALL, filters); + Scan s = new Scan(); + s.addFamily(FAMILIES[0]); + s.setFilter(f); + KeyValue [] kvs = { + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]) + }; + verifyScanFull(s, kvs); + + // Test getting everything with a MUST_PASS_ONE filter including row, qf, + // val, regular expression and substring filters + filters.clear(); + filters.add(new RowFilter(CompareOp.EQUAL, + new RegexStringComparator(".+Two.+"))); + filters.add(new QualifierFilter(CompareOp.EQUAL, + new RegexStringComparator(".+-2"))); + filters.add(new ValueFilter(CompareOp.EQUAL, + new SubstringComparator("One"))); + f = new FilterList(Operator.MUST_PASS_ONE, filters); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, numRows, colsPerRow); + } + + @Test + public void testFirstKeyOnlyFilter() throws Exception { + Scan s = new Scan(); + s.setFilter(new FirstKeyOnlyFilter()); + // Expected KVs, the first KV from each of the remaining 6 rows + KeyValue [] kvs = { + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]) + }; + verifyScanFull(s, kvs); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java new file mode 100644 index 0000000..1d2ed5d --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java @@ -0,0 +1,180 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.StringWriter; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.ColumnSchemaModel; +import org.apache.hadoop.hbase.rest.model.TableSchemaModel; +import org.apache.hadoop.hbase.rest.model.TestTableSchemaModel; +import org.apache.hadoop.hbase.util.Bytes; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestSchemaResource { + private static String TABLE1 = "TestSchemaResource1"; + private static String TABLE2 = "TestSchemaResource2"; + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static Client client; + private static JAXBContext context; + private static Configuration conf; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf = TEST_UTIL.getConfiguration(); + TEST_UTIL.startMiniCluster(); + REST_TEST_UTIL.startServletContainer(conf); + client = new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())); + context = JAXBContext.newInstance( + ColumnSchemaModel.class, + TableSchemaModel.class); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + private static byte[] toXML(TableSchemaModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return Bytes.toBytes(writer.toString()); + } + + private static TableSchemaModel fromXML(byte[] content) + throws JAXBException { + return (TableSchemaModel) context.createUnmarshaller() + .unmarshal(new ByteArrayInputStream(content)); + } + + @Test + public void testTableCreateAndDeleteXML() throws IOException, JAXBException { + String schemaPath = "/" + TABLE1 + "/schema"; + TableSchemaModel model; + Response response; + + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + assertFalse(admin.tableExists(TABLE1)); + + // create the table + model = TestTableSchemaModel.buildTestModel(TABLE1); + TestTableSchemaModel.checkModel(model, TABLE1); + response = client.put(schemaPath, Constants.MIMETYPE_XML, toXML(model)); + assertEquals(response.getCode(), 201); + + // recall the same put operation but in read-only mode + conf.set("hbase.rest.readonly", "true"); + response = client.put(schemaPath, Constants.MIMETYPE_XML, toXML(model)); + assertEquals(response.getCode(), 403); + + // retrieve the schema and validate it + response = client.get(schemaPath, Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + model = fromXML(response.getBody()); + TestTableSchemaModel.checkModel(model, TABLE1); + + // delete the table + client.delete(schemaPath); + + // make sure HBase concurs + assertFalse(admin.tableExists(TABLE1)); + + // return read-only setting back to default + conf.set("hbase.rest.readonly", "false"); + } + + @Test + public void testTableCreateAndDeletePB() throws IOException, JAXBException { + String schemaPath = "/" + TABLE2 + "/schema"; + TableSchemaModel model; + Response response; + + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + assertFalse(admin.tableExists(TABLE2)); + + // create the table + model = TestTableSchemaModel.buildTestModel(TABLE2); + TestTableSchemaModel.checkModel(model, TABLE2); + response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, + model.createProtobufOutput()); + assertEquals(response.getCode(), 201); + + // recall the same put operation but in read-only mode + conf.set("hbase.rest.readonly", "true"); + response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, + model.createProtobufOutput()); + assertEquals(response.getCode(), 403); + + // retrieve the schema and validate it + response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); + model = new TableSchemaModel(); + model.getObjectFromMessage(response.getBody()); + TestTableSchemaModel.checkModel(model, TABLE2); + + // retrieve the schema and validate it with alternate pbuf type + response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF_IETF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type")); + model = new TableSchemaModel(); + model.getObjectFromMessage(response.getBody()); + TestTableSchemaModel.checkModel(model, TABLE2); + + // delete the table + client.delete(schemaPath); + + // make sure HBase concurs + assertFalse(admin.tableExists(TABLE2)); + + // return read-only setting back to default + conf.set("hbase.rest.readonly", "false"); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java new file mode 100644 index 0000000..ea61dc4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java @@ -0,0 +1,125 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.StorageClusterStatusModel; +import org.apache.hadoop.hbase.util.Bytes; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestStatusResource { + private static final byte[] ROOT_REGION_NAME = Bytes.toBytes("-ROOT-,,0"); + private static final byte[] META_REGION_NAME = Bytes.toBytes(".META.,,1"); + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static Client client; + private static JAXBContext context; + + private static void validate(StorageClusterStatusModel model) { + assertNotNull(model); + assertTrue(model.getRegions() >= 1); + assertTrue(model.getRequests() >= 0); + assertTrue(model.getAverageLoad() >= 0.0); + assertNotNull(model.getLiveNodes()); + assertNotNull(model.getDeadNodes()); + assertFalse(model.getLiveNodes().isEmpty()); + boolean foundRoot = false, foundMeta = false; + for (StorageClusterStatusModel.Node node: model.getLiveNodes()) { + assertNotNull(node.getName()); + assertTrue(node.getStartCode() > 0L); + assertTrue(node.getRequests() >= 0); + for (StorageClusterStatusModel.Node.Region region: node.getRegions()) { + if (Bytes.equals(region.getName(), ROOT_REGION_NAME)) { + foundRoot = true; + } else if (Bytes.equals(region.getName(), META_REGION_NAME)) { + foundMeta = true; + } + } + } + assertTrue(foundRoot); + assertTrue(foundMeta); + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); + client = new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())); + context = JAXBContext.newInstance(StorageClusterStatusModel.class); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testGetClusterStatusXML() throws IOException, JAXBException { + Response response = client.get("/status/cluster", Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + StorageClusterStatusModel model = (StorageClusterStatusModel) + context.createUnmarshaller().unmarshal( + new ByteArrayInputStream(response.getBody())); + validate(model); + } + + @Test + public void testGetClusterStatusPB() throws IOException { + Response response = client.get("/status/cluster", Constants.MIMETYPE_PROTOBUF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); + StorageClusterStatusModel model = new StorageClusterStatusModel(); + model.getObjectFromMessage(response.getBody()); + validate(model); + response = client.get("/status/cluster", Constants.MIMETYPE_PROTOBUF_IETF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type")); + model = new StorageClusterStatusModel(); + model.getObjectFromMessage(response.getBody()); + validate(model); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java new file mode 100644 index 0000000..4cf4636 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java @@ -0,0 +1,265 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Iterator; +import java.util.Map; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.TableModel; +import org.apache.hadoop.hbase.rest.model.TableInfoModel; +import org.apache.hadoop.hbase.rest.model.TableListModel; +import org.apache.hadoop.hbase.rest.model.TableRegionModel; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.util.StringUtils; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestTableResource { + private static final Log LOG = LogFactory.getLog(TestTableResource.class); + + private static String TABLE = "TestTableResource"; + private static String COLUMN_FAMILY = "test"; + private static String COLUMN = COLUMN_FAMILY + ":qualifier"; + private static Map regionMap; + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static Client client; + private static JAXBContext context; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(3); + REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); + client = new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())); + context = JAXBContext.newInstance( + TableModel.class, + TableInfoModel.class, + TableListModel.class, + TableRegionModel.class); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(TABLE)) { + return; + } + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(new HColumnDescriptor(COLUMN_FAMILY)); + admin.createTable(htd); + HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE); + byte[] k = new byte[3]; + byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(COLUMN)); + for (byte b1 = 'a'; b1 < 'z'; b1++) { + for (byte b2 = 'a'; b2 < 'z'; b2++) { + for (byte b3 = 'a'; b3 < 'z'; b3++) { + k[0] = b1; + k[1] = b2; + k[2] = b3; + Put put = new Put(k); + put.setWriteToWAL(false); + put.add(famAndQf[0], famAndQf[1], k); + table.put(put); + } + } + } + table.flushCommits(); + // get the initial layout (should just be one region) + Map m = table.getRegionsInfo(); + assertEquals(m.size(), 1); + // tell the master to split the table + admin.split(TABLE); + // give some time for the split to happen + + long timeout = System.currentTimeMillis() + (15 * 1000); + while (System.currentTimeMillis() < timeout && m.size()!=2){ + try { + Thread.sleep(250); + } catch (InterruptedException e) { + LOG.warn(StringUtils.stringifyException(e)); + } + // check again + m = table.getRegionsInfo(); + } + + // should have two regions now + assertEquals(m.size(), 2); + regionMap = m; + LOG.info("regions: " + regionMap); + table.close(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + private static void checkTableList(TableListModel model) { + boolean found = false; + Iterator tables = model.getTables().iterator(); + assertTrue(tables.hasNext()); + while (tables.hasNext()) { + TableModel table = tables.next(); + if (table.getName().equals(TABLE)) { + found = true; + break; + } + } + assertTrue(found); + } + + void checkTableInfo(TableInfoModel model) { + assertEquals(model.getName(), TABLE); + Iterator regions = model.getRegions().iterator(); + assertTrue(regions.hasNext()); + while (regions.hasNext()) { + TableRegionModel region = regions.next(); + boolean found = false; + for (Map.Entry e: regionMap.entrySet()) { + HRegionInfo hri = e.getKey(); + String hriRegionName = hri.getRegionNameAsString(); + String regionName = region.getName(); + if (hriRegionName.equals(regionName)) { + found = true; + byte[] startKey = hri.getStartKey(); + byte[] endKey = hri.getEndKey(); + InetSocketAddress sa = e.getValue().getInetSocketAddress(); + String location = sa.getHostName() + ":" + + Integer.valueOf(sa.getPort()); + assertEquals(hri.getRegionId(), region.getId()); + assertTrue(Bytes.equals(startKey, region.getStartKey())); + assertTrue(Bytes.equals(endKey, region.getEndKey())); + assertEquals(location, region.getLocation()); + break; + } + } + assertTrue(found); + } + } + + @Test + public void testTableListText() throws IOException { + Response response = client.get("/", Constants.MIMETYPE_TEXT); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_TEXT, response.getHeader("content-type")); + } + + @Test + public void testTableListXML() throws IOException, JAXBException { + Response response = client.get("/", Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + TableListModel model = (TableListModel) + context.createUnmarshaller() + .unmarshal(new ByteArrayInputStream(response.getBody())); + checkTableList(model); + } + + @Test + public void testTableListJSON() throws IOException { + Response response = client.get("/", Constants.MIMETYPE_JSON); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type")); + } + + @Test + public void testTableListPB() throws IOException, JAXBException { + Response response = client.get("/", Constants.MIMETYPE_PROTOBUF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); + TableListModel model = new TableListModel(); + model.getObjectFromMessage(response.getBody()); + checkTableList(model); + response = client.get("/", Constants.MIMETYPE_PROTOBUF_IETF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type")); + model = new TableListModel(); + model.getObjectFromMessage(response.getBody()); + checkTableList(model); + } + + @Test + public void testTableInfoText() throws IOException { + Response response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_TEXT); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_TEXT, response.getHeader("content-type")); + } + + @Test + public void testTableInfoXML() throws IOException, JAXBException { + Response response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + TableInfoModel model = (TableInfoModel) + context.createUnmarshaller() + .unmarshal(new ByteArrayInputStream(response.getBody())); + checkTableInfo(model); + } + + @Test + public void testTableInfoJSON() throws IOException { + Response response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_JSON); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type")); + } + + @Test + public void testTableInfoPB() throws IOException, JAXBException { + Response response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_PROTOBUF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); + TableInfoModel model = new TableInfoModel(); + model.getObjectFromMessage(response.getBody()); + checkTableInfo(model); + response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_PROTOBUF_IETF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type")); + model = new TableInfoModel(); + model.getObjectFromMessage(response.getBody()); + checkTableInfo(model); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java b/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java new file mode 100644 index 0000000..5e388a9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java @@ -0,0 +1,177 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.StorageClusterVersionModel; +import org.apache.hadoop.hbase.rest.model.VersionModel; +import org.apache.hadoop.hbase.util.Bytes; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.sun.jersey.spi.container.servlet.ServletContainer; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestVersionResource { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static Client client; + private static JAXBContext context; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); + client = new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())); + context = JAXBContext.newInstance( + VersionModel.class, + StorageClusterVersionModel.class); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + private static void validate(VersionModel model) { + assertNotNull(model); + assertNotNull(model.getRESTVersion()); + assertEquals(model.getRESTVersion(), RESTServlet.VERSION_STRING); + String osVersion = model.getOSVersion(); + assertNotNull(osVersion); + assertTrue(osVersion.contains(System.getProperty("os.name"))); + assertTrue(osVersion.contains(System.getProperty("os.version"))); + assertTrue(osVersion.contains(System.getProperty("os.arch"))); + String jvmVersion = model.getJVMVersion(); + assertNotNull(jvmVersion); + assertTrue(jvmVersion.contains(System.getProperty("java.vm.vendor"))); + assertTrue(jvmVersion.contains(System.getProperty("java.version"))); + assertTrue(jvmVersion.contains(System.getProperty("java.vm.version"))); + assertNotNull(model.getServerVersion()); + String jerseyVersion = model.getJerseyVersion(); + assertNotNull(jerseyVersion); + assertEquals(jerseyVersion, ServletContainer.class.getPackage() + .getImplementationVersion()); + } + + @Test + public void testGetStargateVersionText() throws IOException { + Response response = client.get("/version", Constants.MIMETYPE_TEXT); + assertTrue(response.getCode() == 200); + assertEquals(Constants.MIMETYPE_TEXT, response.getHeader("content-type")); + String body = Bytes.toString(response.getBody()); + assertTrue(body.length() > 0); + assertTrue(body.contains(RESTServlet.VERSION_STRING)); + assertTrue(body.contains(System.getProperty("java.vm.vendor"))); + assertTrue(body.contains(System.getProperty("java.version"))); + assertTrue(body.contains(System.getProperty("java.vm.version"))); + assertTrue(body.contains(System.getProperty("os.name"))); + assertTrue(body.contains(System.getProperty("os.version"))); + assertTrue(body.contains(System.getProperty("os.arch"))); + assertTrue(body.contains(ServletContainer.class.getPackage() + .getImplementationVersion())); + } + + @Test + public void testGetStargateVersionXML() throws IOException, JAXBException { + Response response = client.get("/version", Constants.MIMETYPE_XML); + assertTrue(response.getCode() == 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + VersionModel model = (VersionModel) + context.createUnmarshaller().unmarshal( + new ByteArrayInputStream(response.getBody())); + validate(model); + } + + @Test + public void testGetStargateVersionJSON() throws IOException { + Response response = client.get("/version", Constants.MIMETYPE_JSON); + assertTrue(response.getCode() == 200); + assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type")); + } + + @Test + public void testGetStargateVersionPB() throws IOException { + Response response = client.get("/version", Constants.MIMETYPE_PROTOBUF); + assertTrue(response.getCode() == 200); + assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); + VersionModel model = new VersionModel(); + model.getObjectFromMessage(response.getBody()); + validate(model); + response = client.get("/version", Constants.MIMETYPE_PROTOBUF_IETF); + assertTrue(response.getCode() == 200); + assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type")); + model = new VersionModel(); + model.getObjectFromMessage(response.getBody()); + validate(model); + } + + @Test + public void testGetStorageClusterVersionText() throws IOException { + Response response = client.get("/version/cluster", Constants.MIMETYPE_TEXT); + assertTrue(response.getCode() == 200); + assertEquals(Constants.MIMETYPE_TEXT, response.getHeader("content-type")); + } + + @Test + public void testGetStorageClusterVersionXML() throws IOException, + JAXBException { + Response response = client.get("/version/cluster",Constants.MIMETYPE_XML); + assertTrue(response.getCode() == 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + StorageClusterVersionModel clusterVersionModel = + (StorageClusterVersionModel) + context.createUnmarshaller().unmarshal( + new ByteArrayInputStream(response.getBody())); + assertNotNull(clusterVersionModel); + assertNotNull(clusterVersionModel.getVersion()); + } + + @Test + public void doTestGetStorageClusterVersionJSON() throws IOException { + Response response = client.get("/version/cluster", Constants.MIMETYPE_JSON); + assertTrue(response.getCode() == 200); + assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type")); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java new file mode 100644 index 0000000..630a9d8 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdmin.java @@ -0,0 +1,146 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.rest.HBaseRESTTestingUtility; +import org.apache.hadoop.hbase.rest.model.StorageClusterStatusModel; +import org.apache.hadoop.hbase.rest.model.TableModel; +import org.apache.hadoop.hbase.rest.model.VersionModel; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestRemoteAdmin { + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static final String TABLE_1 = "TestRemoteAdmin_Table_1"; + private static final String TABLE_2 = TABLE_1 + System.currentTimeMillis(); + private static final byte[] COLUMN_1 = Bytes.toBytes("a"); + static final HTableDescriptor DESC_1 = new HTableDescriptor(TABLE_1); + static final HTableDescriptor DESC_2 = new HTableDescriptor(TABLE_2); + private static RemoteAdmin remoteAdmin; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + DESC_1.addFamily(new HColumnDescriptor(COLUMN_1)); + + TEST_UTIL.startMiniCluster(); + REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); + + remoteAdmin = new RemoteAdmin(new Client( + new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())), + TEST_UTIL.getConfiguration()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testCreateAnDeleteTable() throws Exception { + assertFalse(remoteAdmin.isTableAvailable(TABLE_1)); + remoteAdmin.createTable(DESC_1); + assertTrue(remoteAdmin.isTableAvailable(TABLE_1)); + remoteAdmin.deleteTable(TABLE_1); + assertFalse(remoteAdmin.isTableAvailable(TABLE_1)); + } + + @Test + public void testGetRestVersion() throws Exception { + + VersionModel RETURNED_REST_VERSION = remoteAdmin.getRestVersion(); + System.out.print("Returned version is: " + RETURNED_REST_VERSION); + + // Assert that it contains info about rest version, OS, JVM + assertTrue("Returned REST version did not contain info about rest.", + RETURNED_REST_VERSION.toString().contains("rest")); + assertTrue("Returned REST version did not contain info about the JVM.", + RETURNED_REST_VERSION.toString().contains("JVM")); + assertTrue("Returned REST version did not contain info about OS.", + RETURNED_REST_VERSION.toString().contains("OS")); + } + + @Test + public void testClusterVersion() throws Exception { + // testing the /version/cluster endpoint + final String HBASE_VERSION = TEST_UTIL.getHBaseCluster().getClusterStatus() + .getHBaseVersion(); + assertEquals("Cluster status from REST API did not match. ", HBASE_VERSION, + remoteAdmin.getClusterVersion().getVersion()); + } + + @Test + public void testClusterStatus() throws Exception { + + ClusterStatus status = TEST_UTIL.getHBaseClusterInterface() + .getClusterStatus(); + StorageClusterStatusModel returnedStatus = remoteAdmin.getClusterStatus(); + assertEquals( + "Region count from cluster status and returned status did not match up. ", + status.getRegionsCount(), returnedStatus.getRegions()); + assertEquals( + "Dead server count from cluster status and returned status did not match up. ", + status.getDeadServers(), returnedStatus.getDeadNodes().size()); + assertEquals( + "Number of requests from cluster status and returned status did not match up. ", + status.getRequestsCount(), returnedStatus.getRequests()); + } + + @Test + public void testListTables() throws Exception { + + remoteAdmin.createTable(DESC_2); + List tableList = remoteAdmin.getTableList().getTables(); + System.out.println("List of tables is: "); + boolean found = false; + for (TableModel tm : tableList) { + + if (tm.getName().equals(TABLE_2)) { + found = true; + break; + } + } + assertTrue("Table " + TABLE_2 + " was not found by get request to '/'", + found); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java new file mode 100644 index 0000000..a3ab9ec --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java @@ -0,0 +1,390 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.rest.HBaseRESTTestingUtility; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.RemoteHTable; +import org.apache.hadoop.hbase.util.Bytes; + +import static org.junit.Assert.*; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestRemoteTable { + private static final Log LOG = LogFactory.getLog(TestRemoteTable.class); + private static final String TABLE = "TestRemoteTable"; + private static final byte[] ROW_1 = Bytes.toBytes("testrow1"); + private static final byte[] ROW_2 = Bytes.toBytes("testrow2"); + private static final byte[] ROW_3 = Bytes.toBytes("testrow3"); + private static final byte[] ROW_4 = Bytes.toBytes("testrow4"); + private static final byte[] COLUMN_1 = Bytes.toBytes("a"); + private static final byte[] COLUMN_2 = Bytes.toBytes("b"); + private static final byte[] COLUMN_3 = Bytes.toBytes("c"); + private static final byte[] QUALIFIER_1 = Bytes.toBytes("1"); + private static final byte[] QUALIFIER_2 = Bytes.toBytes("2"); + private static final byte[] VALUE_1 = Bytes.toBytes("testvalue1"); + private static final byte[] VALUE_2 = Bytes.toBytes("testvalue2"); + + private static final long ONE_HOUR = 60 * 60 * 1000; + private static final long TS_2 = System.currentTimeMillis(); + private static final long TS_1 = TS_2 - ONE_HOUR; + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static RemoteHTable remoteTable; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + LOG.info("Admin Connection=" + admin.getConnection() + ", " + + admin.getConnection().getZooKeeperWatcher()); + if (!admin.tableExists(TABLE)) { + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(new HColumnDescriptor(COLUMN_1)); + htd.addFamily(new HColumnDescriptor(COLUMN_2)); + htd.addFamily(new HColumnDescriptor(COLUMN_3)); + admin.createTable(htd); + HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE); + LOG.info("Table connection=" + table.getConnection() + ", " + + admin.getConnection().getZooKeeperWatcher()); + Put put = new Put(ROW_1); + put.add(COLUMN_1, QUALIFIER_1, TS_2, VALUE_1); + table.put(put); + put = new Put(ROW_2); + put.add(COLUMN_1, QUALIFIER_1, TS_1, VALUE_1); + put.add(COLUMN_1, QUALIFIER_1, TS_2, VALUE_2); + put.add(COLUMN_2, QUALIFIER_2, TS_2, VALUE_2); + table.put(put); + table.flushCommits(); + } + remoteTable = new RemoteHTable( + new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())), + TEST_UTIL.getConfiguration(), TABLE); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + remoteTable.close(); + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testGetTableDescriptor() throws IOException { + HTableDescriptor local = new HTable(TEST_UTIL.getConfiguration(), + TABLE).getTableDescriptor(); + assertEquals(remoteTable.getTableDescriptor(), local); + } + + @Test + public void testGet() throws IOException { + Get get = new Get(ROW_1); + Result result = remoteTable.get(get); + byte[] value1 = result.getValue(COLUMN_1, QUALIFIER_1); + byte[] value2 = result.getValue(COLUMN_2, QUALIFIER_2); + assertNotNull(value1); + assertTrue(Bytes.equals(VALUE_1, value1)); + assertNull(value2); + + get = new Get(ROW_1); + get.addFamily(COLUMN_3); + result = remoteTable.get(get); + value1 = result.getValue(COLUMN_1, QUALIFIER_1); + value2 = result.getValue(COLUMN_2, QUALIFIER_2); + assertNull(value1); + assertNull(value2); + + get = new Get(ROW_1); + get.addColumn(COLUMN_1, QUALIFIER_1); + get.addColumn(COLUMN_2, QUALIFIER_2); + result = remoteTable.get(get); + value1 = result.getValue(COLUMN_1, QUALIFIER_1); + value2 = result.getValue(COLUMN_2, QUALIFIER_2); + assertNotNull(value1); + assertTrue(Bytes.equals(VALUE_1, value1)); + assertNull(value2); + + get = new Get(ROW_2); + result = remoteTable.get(get); + value1 = result.getValue(COLUMN_1, QUALIFIER_1); + value2 = result.getValue(COLUMN_2, QUALIFIER_2); + assertNotNull(value1); + assertTrue(Bytes.equals(VALUE_2, value1)); // @TS_2 + assertNotNull(value2); + assertTrue(Bytes.equals(VALUE_2, value2)); + + get = new Get(ROW_2); + get.addFamily(COLUMN_1); + result = remoteTable.get(get); + value1 = result.getValue(COLUMN_1, QUALIFIER_1); + value2 = result.getValue(COLUMN_2, QUALIFIER_2); + assertNotNull(value1); + assertTrue(Bytes.equals(VALUE_2, value1)); // @TS_2 + assertNull(value2); + + get = new Get(ROW_2); + get.addColumn(COLUMN_1, QUALIFIER_1); + get.addColumn(COLUMN_2, QUALIFIER_2); + result = remoteTable.get(get); + value1 = result.getValue(COLUMN_1, QUALIFIER_1); + value2 = result.getValue(COLUMN_2, QUALIFIER_2); + assertNotNull(value1); + assertTrue(Bytes.equals(VALUE_2, value1)); // @TS_2 + assertNotNull(value2); + assertTrue(Bytes.equals(VALUE_2, value2)); + + // test timestamp + + get = new Get(ROW_2); + get.addFamily(COLUMN_1); + get.addFamily(COLUMN_2); + get.setTimeStamp(TS_1); + result = remoteTable.get(get); + value1 = result.getValue(COLUMN_1, QUALIFIER_1); + value2 = result.getValue(COLUMN_2, QUALIFIER_2); + assertNotNull(value1); + assertTrue(Bytes.equals(VALUE_1, value1)); // @TS_1 + assertNull(value2); + + // test timerange + + get = new Get(ROW_2); + get.addFamily(COLUMN_1); + get.addFamily(COLUMN_2); + get.setTimeRange(0, TS_1 + 1); + result = remoteTable.get(get); + value1 = result.getValue(COLUMN_1, QUALIFIER_1); + value2 = result.getValue(COLUMN_2, QUALIFIER_2); + assertNotNull(value1); + assertTrue(Bytes.equals(VALUE_1, value1)); // @TS_1 + assertNull(value2); + + // test maxVersions + + get = new Get(ROW_2); + get.addFamily(COLUMN_1); + get.setMaxVersions(2); + result = remoteTable.get(get); + int count = 0; + for (KeyValue kv: result.list()) { + if (Bytes.equals(COLUMN_1, kv.getFamily()) && TS_1 == kv.getTimestamp()) { + assertTrue(Bytes.equals(VALUE_1, kv.getValue())); // @TS_1 + count++; + } + if (Bytes.equals(COLUMN_1, kv.getFamily()) && TS_2 == kv.getTimestamp()) { + assertTrue(Bytes.equals(VALUE_2, kv.getValue())); // @TS_2 + count++; + } + } + assertEquals(2, count); + } + + @Test + public void testMultiGet() throws Exception { + ArrayList gets = new ArrayList(); + gets.add(new Get(ROW_1)); + gets.add(new Get(ROW_2)); + Result[] results = remoteTable.get(gets); + assertNotNull(results); + assertEquals(2, results.length); + assertEquals(1, results[0].size()); + assertEquals(2, results[1].size()); + + //Test Versions + gets = new ArrayList(); + Get g = new Get(ROW_1); + g.setMaxVersions(3); + gets.add(g); + gets.add(new Get(ROW_2)); + results = remoteTable.get(gets); + assertNotNull(results); + assertEquals(2, results.length); + assertEquals(1, results[0].size()); + assertEquals(3, results[1].size()); + + //404 + gets = new ArrayList(); + gets.add(new Get(Bytes.toBytes("RESALLYREALLYNOTTHERE"))); + results = remoteTable.get(gets); + assertNotNull(results); + assertEquals(0, results.length); + + gets = new ArrayList(); + gets.add(new Get(Bytes.toBytes("RESALLYREALLYNOTTHERE"))); + gets.add(new Get(ROW_1)); + gets.add(new Get(ROW_2)); + results = remoteTable.get(gets); + assertNotNull(results); + assertEquals(0, results.length); + } + + @Test + public void testPut() throws IOException { + Put put = new Put(ROW_3); + put.add(COLUMN_1, QUALIFIER_1, VALUE_1); + remoteTable.put(put); + + Get get = new Get(ROW_3); + get.addFamily(COLUMN_1); + Result result = remoteTable.get(get); + byte[] value = result.getValue(COLUMN_1, QUALIFIER_1); + assertNotNull(value); + assertTrue(Bytes.equals(VALUE_1, value)); + + // multiput + + List puts = new ArrayList(); + put = new Put(ROW_3); + put.add(COLUMN_2, QUALIFIER_2, VALUE_2); + puts.add(put); + put = new Put(ROW_4); + put.add(COLUMN_1, QUALIFIER_1, VALUE_1); + puts.add(put); + put = new Put(ROW_4); + put.add(COLUMN_2, QUALIFIER_2, VALUE_2); + puts.add(put); + remoteTable.put(puts); + + get = new Get(ROW_3); + get.addFamily(COLUMN_2); + result = remoteTable.get(get); + value = result.getValue(COLUMN_2, QUALIFIER_2); + assertNotNull(value); + assertTrue(Bytes.equals(VALUE_2, value)); + get = new Get(ROW_4); + result = remoteTable.get(get); + value = result.getValue(COLUMN_1, QUALIFIER_1); + assertNotNull(value); + assertTrue(Bytes.equals(VALUE_1, value)); + value = result.getValue(COLUMN_2, QUALIFIER_2); + assertNotNull(value); + assertTrue(Bytes.equals(VALUE_2, value)); + } + + public void testDelete() throws IOException { + Put put = new Put(ROW_3); + put.add(COLUMN_1, QUALIFIER_1, VALUE_1); + put.add(COLUMN_2, QUALIFIER_2, VALUE_2); + remoteTable.put(put); + + Get get = new Get(ROW_3); + get.addFamily(COLUMN_1); + get.addFamily(COLUMN_2); + Result result = remoteTable.get(get); + byte[] value1 = result.getValue(COLUMN_1, QUALIFIER_1); + byte[] value2 = result.getValue(COLUMN_2, QUALIFIER_2); + assertNotNull(value1); + assertTrue(Bytes.equals(VALUE_1, value1)); + assertNotNull(value2); + assertTrue(Bytes.equals(VALUE_2, value2)); + + Delete delete = new Delete(ROW_3); + delete.deleteColumn(COLUMN_2, QUALIFIER_2); + remoteTable.delete(delete); + + get = new Get(ROW_3); + get.addFamily(COLUMN_1); + get.addFamily(COLUMN_2); + result = remoteTable.get(get); + value1 = result.getValue(COLUMN_1, QUALIFIER_1); + value2 = result.getValue(COLUMN_2, QUALIFIER_2); + assertNotNull(value1); + assertTrue(Bytes.equals(VALUE_1, value1)); + assertNull(value2); + + delete = new Delete(ROW_3); + remoteTable.delete(delete); + + get = new Get(ROW_3); + get.addFamily(COLUMN_1); + get.addFamily(COLUMN_2); + result = remoteTable.get(get); + value1 = result.getValue(COLUMN_1, QUALIFIER_1); + value2 = result.getValue(COLUMN_2, QUALIFIER_2); + assertNull(value1); + assertNull(value2); + } + + public void testScanner() throws IOException { + List puts = new ArrayList(); + Put put = new Put(ROW_1); + put.add(COLUMN_1, QUALIFIER_1, VALUE_1); + puts.add(put); + put = new Put(ROW_2); + put.add(COLUMN_1, QUALIFIER_1, VALUE_1); + puts.add(put); + put = new Put(ROW_3); + put.add(COLUMN_1, QUALIFIER_1, VALUE_1); + puts.add(put); + put = new Put(ROW_4); + put.add(COLUMN_1, QUALIFIER_1, VALUE_1); + puts.add(put); + remoteTable.put(puts); + + ResultScanner scanner = remoteTable.getScanner(new Scan()); + + Result[] results = scanner.next(1); + assertNotNull(results); + assertEquals(1, results.length); + assertTrue(Bytes.equals(ROW_1, results[0].getRow())); + + results = scanner.next(3); + assertNotNull(results); + assertEquals(3, results.length); + assertTrue(Bytes.equals(ROW_2, results[0].getRow())); + assertTrue(Bytes.equals(ROW_3, results[1].getRow())); + assertTrue(Bytes.equals(ROW_4, results[2].getRow())); + + results = scanner.next(1); + assertNull(results); + + scanner.close(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellModel.java new file mode 100644 index 0000000..4e7a56b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellModel.java @@ -0,0 +1,112 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Base64; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestCellModel extends TestCase { + + private static final long TIMESTAMP = 1245219839331L; + private static final byte[] COLUMN = Bytes.toBytes("testcolumn"); + private static final byte[] VALUE = Bytes.toBytes("testvalue"); + + private static final String AS_XML = + "" + + "dGVzdHZhbHVl"; + + private static final String AS_PB = + "Egp0ZXN0Y29sdW1uGOO6i+eeJCIJdGVzdHZhbHVl"; + + private JAXBContext context; + + public TestCellModel() throws JAXBException { + super(); + context = JAXBContext.newInstance(CellModel.class); + } + + private CellModel buildTestModel() { + CellModel model = new CellModel(); + model.setColumn(COLUMN); + model.setTimestamp(TIMESTAMP); + model.setValue(VALUE); + return model; + } + + @SuppressWarnings("unused") + private String toXML(CellModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return writer.toString(); + } + + private CellModel fromXML(String xml) throws JAXBException { + return (CellModel) + context.createUnmarshaller().unmarshal(new StringReader(xml)); + } + + @SuppressWarnings("unused") + private byte[] toPB(CellModel model) { + return model.createProtobufOutput(); + } + + private CellModel fromPB(String pb) throws IOException { + return (CellModel) + new CellModel().getObjectFromMessage(Base64.decode(AS_PB)); + } + + private void checkModel(CellModel model) { + assertTrue(Bytes.equals(model.getColumn(), COLUMN)); + assertTrue(Bytes.equals(model.getValue(), VALUE)); + assertTrue(model.hasUserTimestamp()); + assertEquals(model.getTimestamp(), TIMESTAMP); + } + + public void testBuildModel() throws Exception { + checkModel(buildTestModel()); + } + + public void testFromXML() throws Exception { + checkModel(fromXML(AS_XML)); + } + + public void testFromPB() throws Exception { + checkModel(fromPB(AS_PB)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellSetModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellSetModel.java new file mode 100644 index 0000000..3d358c0 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellSetModel.java @@ -0,0 +1,162 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.Iterator; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Base64; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestCellSetModel extends TestCase { + + private static final byte[] ROW1 = Bytes.toBytes("testrow1"); + private static final byte[] COLUMN1 = Bytes.toBytes("testcolumn1"); + private static final byte[] VALUE1 = Bytes.toBytes("testvalue1"); + private static final long TIMESTAMP1 = 1245219839331L; + private static final byte[] ROW2 = Bytes.toBytes("testrow1"); + private static final byte[] COLUMN2 = Bytes.toBytes("testcolumn2"); + private static final byte[] VALUE2 = Bytes.toBytes("testvalue2"); + private static final long TIMESTAMP2 = 1245239813319L; + private static final byte[] COLUMN3 = Bytes.toBytes("testcolumn3"); + private static final byte[] VALUE3 = Bytes.toBytes("testvalue3"); + private static final long TIMESTAMP3 = 1245393318192L; + + private static final String AS_XML = + "" + + "" + + "" + + "dGVzdHZhbHVlMQ==" + + "" + + "" + + "" + + "dGVzdHZhbHVlMg==" + + "" + + "dGVzdHZhbHVlMw==" + + "" + + ""; + + private static final String AS_PB = + "CiwKCHRlc3Ryb3cxEiASC3Rlc3Rjb2x1bW4xGOO6i+eeJCIKdGVzdHZhbHVlMQpOCgh0ZXN0cm93" + + "MRIgEgt0ZXN0Y29sdW1uMhjHyc7wniQiCnRlc3R2YWx1ZTISIBILdGVzdGNvbHVtbjMYsOLnuZ8k" + + "Igp0ZXN0dmFsdWUz"; + + private JAXBContext context; + + public TestCellSetModel() throws JAXBException { + super(); + context = JAXBContext.newInstance( + CellModel.class, + CellSetModel.class, + RowModel.class); + } + + private CellSetModel buildTestModel() { + CellSetModel model = new CellSetModel(); + RowModel row; + row = new RowModel(); + row.setKey(ROW1); + row.addCell(new CellModel(COLUMN1, TIMESTAMP1, VALUE1)); + model.addRow(row); + row = new RowModel(); + row.setKey(ROW2); + row.addCell(new CellModel(COLUMN2, TIMESTAMP2, VALUE2)); + row.addCell(new CellModel(COLUMN3, TIMESTAMP3, VALUE3)); + model.addRow(row); + return model; + } + + @SuppressWarnings("unused") + private String toXML(CellSetModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return writer.toString(); + } + + private CellSetModel fromXML(String xml) throws JAXBException { + return (CellSetModel) + context.createUnmarshaller().unmarshal(new StringReader(xml)); + } + + @SuppressWarnings("unused") + private byte[] toPB(CellSetModel model) { + return model.createProtobufOutput(); + } + + private CellSetModel fromPB(String pb) throws IOException { + return (CellSetModel) + new CellSetModel().getObjectFromMessage(Base64.decode(AS_PB)); + } + + private void checkModel(CellSetModel model) { + Iterator rows = model.getRows().iterator(); + RowModel row = rows.next(); + assertTrue(Bytes.equals(ROW1, row.getKey())); + Iterator cells = row.getCells().iterator(); + CellModel cell = cells.next(); + assertTrue(Bytes.equals(COLUMN1, cell.getColumn())); + assertTrue(Bytes.equals(VALUE1, cell.getValue())); + assertTrue(cell.hasUserTimestamp()); + assertEquals(cell.getTimestamp(), TIMESTAMP1); + assertFalse(cells.hasNext()); + row = rows.next(); + assertTrue(Bytes.equals(ROW2, row.getKey())); + cells = row.getCells().iterator(); + cell = cells.next(); + assertTrue(Bytes.equals(COLUMN2, cell.getColumn())); + assertTrue(Bytes.equals(VALUE2, cell.getValue())); + assertTrue(cell.hasUserTimestamp()); + assertEquals(cell.getTimestamp(), TIMESTAMP2); + cell = cells.next(); + assertTrue(Bytes.equals(COLUMN3, cell.getColumn())); + assertTrue(Bytes.equals(VALUE3, cell.getValue())); + assertTrue(cell.hasUserTimestamp()); + assertEquals(cell.getTimestamp(), TIMESTAMP3); + assertFalse(cells.hasNext()); + } + + public void testBuildModel() throws Exception { + checkModel(buildTestModel()); + } + + public void testFromXML() throws Exception { + checkModel(fromXML(AS_XML)); + } + + public void testFromPB() throws Exception { + checkModel(fromPB(AS_PB)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java new file mode 100644 index 0000000..62014a3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java @@ -0,0 +1,110 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.StringReader; +import java.io.StringWriter; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestColumnSchemaModel extends TestCase { + + protected static final String COLUMN_NAME = "testcolumn"; + protected static final boolean BLOCKCACHE = true; + protected static final int BLOCKSIZE = 16384; + protected static final String BLOOMFILTER = "NONE"; + protected static final String COMPRESSION = "GZ"; + protected static final boolean IN_MEMORY = false; + protected static final int TTL = 86400; + protected static final int VERSIONS = 1; + + protected static final String AS_XML = + ""; + + private JAXBContext context; + + public TestColumnSchemaModel() throws JAXBException { + super(); + context = JAXBContext.newInstance(ColumnSchemaModel.class); + } + + protected static ColumnSchemaModel buildTestModel() { + ColumnSchemaModel model = new ColumnSchemaModel(); + model.setName(COLUMN_NAME); + model.__setBlockcache(BLOCKCACHE); + model.__setBlocksize(BLOCKSIZE); + model.__setBloomfilter(BLOOMFILTER); + model.__setCompression(COMPRESSION); + model.__setInMemory(IN_MEMORY); + model.__setTTL(TTL); + model.__setVersions(VERSIONS); + return model; + } + + @SuppressWarnings("unused") + private String toXML(ColumnSchemaModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return writer.toString(); + } + + private ColumnSchemaModel fromXML(String xml) throws JAXBException { + return (ColumnSchemaModel) + context.createUnmarshaller().unmarshal(new StringReader(xml)); + } + + protected static void checkModel(ColumnSchemaModel model) { + assertEquals(model.getName(), COLUMN_NAME); + assertEquals(model.__getBlockcache(), BLOCKCACHE); + assertEquals(model.__getBlocksize(), BLOCKSIZE); + assertEquals(model.__getBloomfilter(), BLOOMFILTER); + assertTrue(model.__getCompression().equalsIgnoreCase(COMPRESSION)); + assertEquals(model.__getInMemory(), IN_MEMORY); + assertEquals(model.__getTTL(), TTL); + assertEquals(model.__getVersions(), VERSIONS); + } + + public void testBuildModel() throws Exception { + checkModel(buildTestModel()); + } + + public void testFromXML() throws Exception { + checkModel(fromXML(AS_XML)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java new file mode 100644 index 0000000..8038645 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java @@ -0,0 +1,101 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.StringReader; +import java.io.StringWriter; +import java.util.Iterator; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestRowModel extends TestCase { + + private static final byte[] ROW1 = Bytes.toBytes("testrow1"); + private static final byte[] COLUMN1 = Bytes.toBytes("testcolumn1"); + private static final byte[] VALUE1 = Bytes.toBytes("testvalue1"); + private static final long TIMESTAMP1 = 1245219839331L; + + private static final String AS_XML = + "" + + "" + + "dGVzdHZhbHVlMQ==" + + ""; + + private JAXBContext context; + + public TestRowModel() throws JAXBException { + super(); + context = JAXBContext.newInstance( + CellModel.class, + RowModel.class); + } + + private RowModel buildTestModel() { + RowModel model = new RowModel(); + model.setKey(ROW1); + model.addCell(new CellModel(COLUMN1, TIMESTAMP1, VALUE1)); + return model; + } + + @SuppressWarnings("unused") + private String toXML(RowModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return writer.toString(); + } + + private RowModel fromXML(String xml) throws JAXBException { + return (RowModel) + context.createUnmarshaller().unmarshal(new StringReader(xml)); + } + + private void checkModel(RowModel model) { + assertTrue(Bytes.equals(ROW1, model.getKey())); + Iterator cells = model.getCells().iterator(); + CellModel cell = cells.next(); + assertTrue(Bytes.equals(COLUMN1, cell.getColumn())); + assertTrue(Bytes.equals(VALUE1, cell.getValue())); + assertTrue(cell.hasUserTimestamp()); + assertEquals(cell.getTimestamp(), TIMESTAMP1); + assertFalse(cells.hasNext()); + } + + public void testBuildModel() throws Exception { + checkModel(buildTestModel()); + } + + public void testFromXML() throws Exception { + checkModel(fromXML(AS_XML)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java new file mode 100644 index 0000000..dc55f3a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java @@ -0,0 +1,136 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Base64; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestScannerModel extends TestCase { + private static final byte[] START_ROW = Bytes.toBytes("abracadabra"); + private static final byte[] END_ROW = Bytes.toBytes("zzyzx"); + private static final byte[] COLUMN1 = Bytes.toBytes("column1"); + private static final byte[] COLUMN2 = Bytes.toBytes("column2:foo"); + private static final long START_TIME = 1245219839331L; + private static final long END_TIME = 1245393318192L; + private static final int BATCH = 100; + + private static final String AS_XML = + "" + + "Y29sdW1uMQ==" + + "Y29sdW1uMjpmb28=" + + ""; + + private static final String AS_PB = + "CgthYnJhY2FkYWJyYRIFenp5engaB2NvbHVtbjEaC2NvbHVtbjI6Zm9vIGQo47qL554kMLDi57mf" + + "JA=="; + + private JAXBContext context; + + public TestScannerModel() throws JAXBException { + super(); + context = JAXBContext.newInstance(ScannerModel.class); + } + + private ScannerModel buildTestModel() { + ScannerModel model = new ScannerModel(); + model.setStartRow(START_ROW); + model.setEndRow(END_ROW); + model.addColumn(COLUMN1); + model.addColumn(COLUMN2); + model.setStartTime(START_TIME); + model.setEndTime(END_TIME); + model.setBatch(BATCH); + return model; + } + + @SuppressWarnings("unused") + private String toXML(ScannerModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return writer.toString(); + } + + private ScannerModel fromXML(String xml) throws JAXBException { + return (ScannerModel) + context.createUnmarshaller().unmarshal(new StringReader(xml)); + } + + @SuppressWarnings("unused") + private byte[] toPB(ScannerModel model) { + return model.createProtobufOutput(); + } + + private ScannerModel fromPB(String pb) throws IOException { + return (ScannerModel) + new ScannerModel().getObjectFromMessage(Base64.decode(AS_PB)); + } + + private void checkModel(ScannerModel model) { + assertTrue(Bytes.equals(model.getStartRow(), START_ROW)); + assertTrue(Bytes.equals(model.getEndRow(), END_ROW)); + boolean foundCol1 = false, foundCol2 = false; + for (byte[] column: model.getColumns()) { + if (Bytes.equals(column, COLUMN1)) { + foundCol1 = true; + } else if (Bytes.equals(column, COLUMN2)) { + foundCol2 = true; + } + } + assertTrue(foundCol1); + assertTrue(foundCol2); + assertEquals(model.getStartTime(), START_TIME); + assertEquals(model.getEndTime(), END_TIME); + assertEquals(model.getBatch(), BATCH); + } + + public void testBuildModel() throws Exception { + checkModel(buildTestModel()); + } + + public void testFromXML() throws Exception { + checkModel(fromXML(AS_XML)); + } + + public void testFromPB() throws Exception { + checkModel(fromPB(AS_PB)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java new file mode 100644 index 0000000..c44f720 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java @@ -0,0 +1,178 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.Iterator; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Base64; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestStorageClusterStatusModel extends TestCase { + + private static final String AS_XML = + "" + + "" + + "" + + "" + + "" + + ""+ + ""; + + private static final String AS_PB = + "CjsKBXRlc3QxEOO6i+eeJBgAIIABKIAIMiMKCS1ST09ULSwsMBABGAEgACgAMAA4AUACSAFQAVgB" + + "YAFoAQpHCgV0ZXN0MhD+krHwniQYACCABCiACDIvChUuTUVUQS4sLDEyNDYwMDAwNDM3MjQQARgB" + + "IAAoADAAOAFAAkgBUAFYAWABaAEYAiAAKQAAAAAAAPA/"; + + private JAXBContext context; + + public TestStorageClusterStatusModel() throws JAXBException { + super(); + context = JAXBContext.newInstance(StorageClusterStatusModel.class); + } + + private StorageClusterStatusModel buildTestModel() { + StorageClusterStatusModel model = new StorageClusterStatusModel(); + model.setRegions(2); + model.setRequests(0); + model.setAverageLoad(1.0); + model.addLiveNode("test1", 1245219839331L, 128, 1024) + .addRegion(Bytes.toBytes("-ROOT-,,0"), 1, 1, 0, 0, 0, 1, 2, 1, 1, 1, 1, 1); + model.addLiveNode("test2", 1245239331198L, 512, 1024) + .addRegion(Bytes.toBytes(".META.,,1246000043724"),1, 1, 0, 0, 0, + 1, 2, 1, 1, 1, 1, 1); + return model; + } + + @SuppressWarnings("unused") + private String toXML(StorageClusterStatusModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return writer.toString(); + } + + private StorageClusterStatusModel fromXML(String xml) throws JAXBException { + return (StorageClusterStatusModel) + context.createUnmarshaller().unmarshal(new StringReader(xml)); + } + + @SuppressWarnings("unused") + private byte[] toPB(StorageClusterStatusModel model) { + return model.createProtobufOutput(); + } + + private StorageClusterStatusModel fromPB(String pb) throws IOException { + return (StorageClusterStatusModel) + new StorageClusterStatusModel().getObjectFromMessage(Base64.decode(AS_PB)); + } + + private void checkModel(StorageClusterStatusModel model) { + assertEquals(model.getRegions(), 2); + assertEquals(model.getRequests(), 0); + assertEquals(model.getAverageLoad(), 1.0); + Iterator nodes = + model.getLiveNodes().iterator(); + StorageClusterStatusModel.Node node = nodes.next(); + assertEquals(node.getName(), "test1"); + assertEquals(node.getStartCode(), 1245219839331L); + assertEquals(node.getHeapSizeMB(), 128); + assertEquals(node.getMaxHeapSizeMB(), 1024); + Iterator regions = + node.getRegions().iterator(); + StorageClusterStatusModel.Node.Region region = regions.next(); + assertTrue(Bytes.toString(region.getName()).equals("-ROOT-,,0")); + assertEquals(region.getStores(), 1); + assertEquals(region.getStorefiles(), 1); + assertEquals(region.getStorefileSizeMB(), 0); + assertEquals(region.getMemstoreSizeMB(), 0); + assertEquals(region.getStorefileIndexSizeMB(), 0); + assertEquals(region.getReadRequestsCount(), 1); + assertEquals(region.getWriteRequestsCount(), 2); + assertEquals(region.getRootIndexSizeKB(), 1); + assertEquals(region.getTotalStaticIndexSizeKB(), 1); + assertEquals(region.getTotalStaticBloomSizeKB(), 1); + assertEquals(region.getTotalCompactingKVs(), 1); + assertEquals(region.getCurrentCompactedKVs(), 1); + assertFalse(regions.hasNext()); + node = nodes.next(); + assertEquals(node.getName(), "test2"); + assertEquals(node.getStartCode(), 1245239331198L); + assertEquals(node.getHeapSizeMB(), 512); + assertEquals(node.getMaxHeapSizeMB(), 1024); + regions = node.getRegions().iterator(); + region = regions.next(); + assertEquals(Bytes.toString(region.getName()), ".META.,,1246000043724"); + assertEquals(region.getStores(), 1); + assertEquals(region.getStorefiles(), 1); + assertEquals(region.getStorefileSizeMB(), 0); + assertEquals(region.getMemstoreSizeMB(), 0); + assertEquals(region.getStorefileIndexSizeMB(), 0); + assertEquals(region.getReadRequestsCount(), 1); + assertEquals(region.getWriteRequestsCount(), 2); + assertEquals(region.getRootIndexSizeKB(), 1); + assertEquals(region.getTotalStaticIndexSizeKB(), 1); + assertEquals(region.getTotalStaticBloomSizeKB(), 1); + assertEquals(region.getTotalCompactingKVs(), 1); + assertEquals(region.getCurrentCompactedKVs(), 1); + + assertFalse(regions.hasNext()); + assertFalse(nodes.hasNext()); + } + + public void testBuildModel() throws Exception { + checkModel(buildTestModel()); + } + + public void testFromXML() throws Exception { + checkModel(fromXML(AS_XML)); + } + + public void testFromPB() throws Exception { + checkModel(fromPB(AS_PB)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterVersionModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterVersionModel.java new file mode 100644 index 0000000..3a01486 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterVersionModel.java @@ -0,0 +1,81 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.StringReader; +import java.io.StringWriter; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestStorageClusterVersionModel extends TestCase { + private static final String VERSION = "0.0.1-testing"; + + private static final String AS_XML = + "" + VERSION + ""; + + private JAXBContext context; + + public TestStorageClusterVersionModel() throws JAXBException { + super(); + context = JAXBContext.newInstance(StorageClusterVersionModel.class); + } + + private StorageClusterVersionModel buildTestModel() { + StorageClusterVersionModel model = new StorageClusterVersionModel(); + model.setVersion(VERSION); + return model; + } + + @SuppressWarnings("unused") + private String toXML(StorageClusterVersionModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return writer.toString(); + } + + private StorageClusterVersionModel fromXML(String xml) throws JAXBException { + return (StorageClusterVersionModel) + context.createUnmarshaller().unmarshal(new StringReader(xml)); + } + + private void checkModel(StorageClusterVersionModel model) { + assertEquals(model.getVersion(), VERSION); + } + + public void testBuildModel() throws Exception { + checkModel(buildTestModel()); + } + + public void testFromXML() throws Exception { + checkModel(fromXML(AS_XML)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableInfoModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableInfoModel.java new file mode 100644 index 0000000..7fb74a7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableInfoModel.java @@ -0,0 +1,124 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.Iterator; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Base64; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestTableInfoModel extends TestCase { + private static final String TABLE = "testtable"; + private static final byte[] START_KEY = Bytes.toBytes("abracadbra"); + private static final byte[] END_KEY = Bytes.toBytes("zzyzx"); + private static final long ID = 8731042424L; + private static final String LOCATION = "testhost:9876"; + + private static final String AS_XML = + "" + + "" + + ""; + + private static final String AS_PB = + "Cgl0ZXN0dGFibGUSSQofdGVzdHRhYmxlLGFicmFjYWRicmEsODczMTA0MjQyNBIKYWJyYWNhZGJy" + + "YRoFenp5engg+MSkwyAqDXRlc3Rob3N0Ojk4NzY="; + + private JAXBContext context; + + public TestTableInfoModel() throws JAXBException { + super(); + context = JAXBContext.newInstance( + TableInfoModel.class, + TableRegionModel.class); + } + + private TableInfoModel buildTestModel() { + TableInfoModel model = new TableInfoModel(); + model.setName(TABLE); + model.add(new TableRegionModel(TABLE, ID, START_KEY, END_KEY, LOCATION)); + return model; + } + + @SuppressWarnings("unused") + private String toXML(TableInfoModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return writer.toString(); + } + + private TableInfoModel fromXML(String xml) throws JAXBException { + return (TableInfoModel) + context.createUnmarshaller().unmarshal(new StringReader(xml)); + } + + @SuppressWarnings("unused") + private byte[] toPB(TableInfoModel model) { + return model.createProtobufOutput(); + } + + private TableInfoModel fromPB(String pb) throws IOException { + return (TableInfoModel) + new TableInfoModel().getObjectFromMessage(Base64.decode(AS_PB)); + } + + private void checkModel(TableInfoModel model) { + assertEquals(model.getName(), TABLE); + Iterator regions = model.getRegions().iterator(); + TableRegionModel region = regions.next(); + assertTrue(Bytes.equals(region.getStartKey(), START_KEY)); + assertTrue(Bytes.equals(region.getEndKey(), END_KEY)); + assertEquals(region.getId(), ID); + assertEquals(region.getLocation(), LOCATION); + assertFalse(regions.hasNext()); + } + + public void testBuildModel() throws Exception { + checkModel(buildTestModel()); + } + + public void testFromXML() throws Exception { + checkModel(fromXML(AS_XML)); + } + + public void testFromPB() throws Exception { + checkModel(fromPB(AS_PB)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableListModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableListModel.java new file mode 100644 index 0000000..a3d1bf9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableListModel.java @@ -0,0 +1,115 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.Iterator; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Base64; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestTableListModel extends TestCase { + private static final String TABLE1 = "table1"; + private static final String TABLE2 = "table2"; + private static final String TABLE3 = "table3"; + + private static final String AS_XML = + "

    " + + "
    "; + + private static final String AS_PB = "CgZ0YWJsZTEKBnRhYmxlMgoGdGFibGUz"; + + private JAXBContext context; + + public TestTableListModel() throws JAXBException { + super(); + context = JAXBContext.newInstance( + TableListModel.class, + TableModel.class); + } + + private TableListModel buildTestModel() { + TableListModel model = new TableListModel(); + model.add(new TableModel(TABLE1)); + model.add(new TableModel(TABLE2)); + model.add(new TableModel(TABLE3)); + return model; + } + + @SuppressWarnings("unused") + private String toXML(TableListModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return writer.toString(); + } + + private TableListModel fromXML(String xml) throws JAXBException { + return (TableListModel) + context.createUnmarshaller().unmarshal(new StringReader(xml)); + } + + @SuppressWarnings("unused") + private byte[] toPB(TableListModel model) { + return model.createProtobufOutput(); + } + + private TableListModel fromPB(String pb) throws IOException { + return (TableListModel) + new TableListModel().getObjectFromMessage(Base64.decode(AS_PB)); + } + + private void checkModel(TableListModel model) { + Iterator tables = model.getTables().iterator(); + TableModel table = tables.next(); + assertEquals(table.getName(), TABLE1); + table = tables.next(); + assertEquals(table.getName(), TABLE2); + table = tables.next(); + assertEquals(table.getName(), TABLE3); + assertFalse(tables.hasNext()); + } + + public void testBuildModel() throws Exception { + checkModel(buildTestModel()); + } + + public void testFromXML() throws Exception { + checkModel(fromXML(AS_XML)); + } + + public void testFromPB() throws Exception { + checkModel(fromPB(AS_PB)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableRegionModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableRegionModel.java new file mode 100644 index 0000000..b6f0ab5 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableRegionModel.java @@ -0,0 +1,112 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.StringReader; +import java.io.StringWriter; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestTableRegionModel extends TestCase { + private static final String TABLE = "testtable"; + private static final byte[] START_KEY = Bytes.toBytes("abracadbra"); + private static final byte[] END_KEY = Bytes.toBytes("zzyzx"); + private static final long ID = 8731042424L; + private static final String LOCATION = "testhost:9876"; + + private static final String AS_XML = + ""; + + private JAXBContext context; + + public TestTableRegionModel() throws JAXBException { + super(); + context = JAXBContext.newInstance(TableRegionModel.class); + } + + private TableRegionModel buildTestModel() { + TableRegionModel model = + new TableRegionModel(TABLE, ID, START_KEY, END_KEY, LOCATION); + return model; + } + + @SuppressWarnings("unused") + private String toXML(TableRegionModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return writer.toString(); + } + + private TableRegionModel fromXML(String xml) throws JAXBException { + return (TableRegionModel) + context.createUnmarshaller().unmarshal(new StringReader(xml)); + } + + private void checkModel(TableRegionModel model) { + assertTrue(Bytes.equals(model.getStartKey(), START_KEY)); + assertTrue(Bytes.equals(model.getEndKey(), END_KEY)); + assertEquals(model.getId(), ID); + assertEquals(model.getLocation(), LOCATION); + assertEquals(model.getName(), + TABLE + "," + Bytes.toString(START_KEY) + "," + Long.toString(ID) + + ".ad9860f031282c46ed431d7af8f94aca."); + } + + public void testBuildModel() throws Exception { + checkModel(buildTestModel()); + } + + public void testGetName() { + TableRegionModel model = buildTestModel(); + String modelName = model.getName(); + HRegionInfo hri = new HRegionInfo(Bytes.toBytes(TABLE), + START_KEY, END_KEY, false, ID); + assertEquals(modelName, hri.getRegionNameAsString()); + } + + public void testSetName() { + TableRegionModel model = buildTestModel(); + String name = model.getName(); + model.setName(name); + assertEquals(name, model.getName()); + } + + public void testFromXML() throws Exception { + checkModel(fromXML(AS_XML)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableSchemaModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableSchemaModel.java new file mode 100644 index 0000000..c9b2989 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableSchemaModel.java @@ -0,0 +1,136 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.Iterator; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Base64; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestTableSchemaModel extends TestCase { + + public static final String TABLE_NAME = "testTable"; + private static final boolean IS_META = false; + private static final boolean IS_ROOT = false; + private static final boolean READONLY = false; + + private static final String AS_XML = + "" + + TestColumnSchemaModel.AS_XML + + ""; + + private static final String AS_PB = + "Cgl0ZXN0VGFibGUSEAoHSVNfTUVUQRIFZmFsc2USEAoHSVNfUk9PVBIFZmFsc2USEQoIUkVBRE9O" + + "TFkSBWZhbHNlGpcBCgp0ZXN0Y29sdW1uEhIKCUJMT0NLU0laRRIFMTYzODQSEwoLQkxPT01GSUxU" + + "RVISBE5PTkUSEgoKQkxPQ0tDQUNIRRIEdHJ1ZRIRCgtDT01QUkVTU0lPThICR1oSDQoIVkVSU0lP" + + "TlMSATESDAoDVFRMEgU4NjQwMBISCglJTl9NRU1PUlkSBWZhbHNlGICjBSABKgJHWigA"; + + private JAXBContext context; + + public TestTableSchemaModel() throws JAXBException { + super(); + context = JAXBContext.newInstance( + ColumnSchemaModel.class, + TableSchemaModel.class); + } + + public static TableSchemaModel buildTestModel() { + return buildTestModel(TABLE_NAME); + } + + public static TableSchemaModel buildTestModel(String name) { + TableSchemaModel model = new TableSchemaModel(); + model.setName(name); + model.__setIsMeta(IS_META); + model.__setIsRoot(IS_ROOT); + model.__setReadOnly(READONLY); + model.addColumnFamily(TestColumnSchemaModel.buildTestModel()); + return model; + } + + @SuppressWarnings("unused") + private String toXML(TableSchemaModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return writer.toString(); + } + + private TableSchemaModel fromXML(String xml) throws JAXBException { + return (TableSchemaModel) + context.createUnmarshaller().unmarshal(new StringReader(xml)); + } + + @SuppressWarnings("unused") + private byte[] toPB(TableSchemaModel model) { + return model.createProtobufOutput(); + } + + private TableSchemaModel fromPB(String pb) throws IOException { + return (TableSchemaModel) + new TableSchemaModel().getObjectFromMessage(Base64.decode(AS_PB)); + } + + public static void checkModel(TableSchemaModel model) { + checkModel(model, TABLE_NAME); + } + + public static void checkModel(TableSchemaModel model, String tableName) { + assertEquals(model.getName(), tableName); + assertEquals(model.__getIsMeta(), IS_META); + assertEquals(model.__getIsRoot(), IS_ROOT); + assertEquals(model.__getReadOnly(), READONLY); + Iterator families = model.getColumns().iterator(); + assertTrue(families.hasNext()); + ColumnSchemaModel family = families.next(); + TestColumnSchemaModel.checkModel(family); + assertFalse(families.hasNext()); + } + + public void testBuildModel() throws Exception { + checkModel(buildTestModel()); + } + + public void testFromXML() throws Exception { + checkModel(fromXML(AS_XML)); + } + + public void testFromPB() throws Exception { + checkModel(fromPB(AS_PB)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/rest/model/TestVersionModel.java b/src/test/java/org/apache/hadoop/hbase/rest/model/TestVersionModel.java new file mode 100644 index 0000000..2ecb7d9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/rest/model/TestVersionModel.java @@ -0,0 +1,120 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest.model; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Base64; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestVersionModel extends TestCase { + private static final String REST_VERSION = "0.0.1"; + private static final String OS_VERSION = + "Linux 2.6.18-128.1.6.el5.centos.plusxen amd64"; + private static final String JVM_VERSION = + "Sun Microsystems Inc. 1.6.0_13-11.3-b02"; + private static final String JETTY_VERSION = "6.1.14"; + private static final String JERSEY_VERSION = "1.1.0-ea"; + + private static final String AS_XML = + ""; + + private static final String AS_PB = + "CgUwLjAuMRInU3VuIE1pY3Jvc3lzdGVtcyBJbmMuIDEuNi4wXzEzLTExLjMtYjAyGi1MaW51eCAy" + + "LjYuMTgtMTI4LjEuNi5lbDUuY2VudG9zLnBsdXN4ZW4gYW1kNjQiBjYuMS4xNCoIMS4xLjAtZWE="; + + private JAXBContext context; + + public TestVersionModel() throws JAXBException { + super(); + context = JAXBContext.newInstance(VersionModel.class); + } + + private VersionModel buildTestModel() { + VersionModel model = new VersionModel(); + model.setRESTVersion(REST_VERSION); + model.setOSVersion(OS_VERSION); + model.setJVMVersion(JVM_VERSION); + model.setServerVersion(JETTY_VERSION); + model.setJerseyVersion(JERSEY_VERSION); + return model; + } + + @SuppressWarnings("unused") + private String toXML(VersionModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return writer.toString(); + } + + private VersionModel fromXML(String xml) throws JAXBException { + return (VersionModel) + context.createUnmarshaller().unmarshal(new StringReader(xml)); + } + + @SuppressWarnings("unused") + private byte[] toPB(VersionModel model) { + return model.createProtobufOutput(); + } + + private VersionModel fromPB(String pb) throws IOException { + return (VersionModel) + new VersionModel().getObjectFromMessage(Base64.decode(AS_PB)); + } + + private void checkModel(VersionModel model) { + assertEquals(model.getRESTVersion(), REST_VERSION); + assertEquals(model.getOSVersion(), OS_VERSION); + assertEquals(model.getJVMVersion(), JVM_VERSION); + assertEquals(model.getServerVersion(), JETTY_VERSION); + assertEquals(model.getJerseyVersion(), JERSEY_VERSION); + } + + public void testBuildModel() throws Exception { + checkModel(buildTestModel()); + } + + public void testFromXML() throws Exception { + checkModel(fromXML(AS_XML)); + } + + public void testFromPB() throws Exception { + checkModel(fromPB(AS_PB)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/security/TestUser.java b/src/test/java/org/apache/hadoop/hbase/security/TestUser.java new file mode 100644 index 0000000..e90de1f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/security/TestUser.java @@ -0,0 +1,114 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security; + +import static org.junit.Assert.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; + +@Category(SmallTests.class) +public class TestUser { + private static Log LOG = LogFactory.getLog(TestUser.class); + + @Test + public void testBasicAttributes() throws Exception { + Configuration conf = HBaseConfiguration.create(); + User user = User.createUserForTesting(conf, "simple", new String[]{"foo"}); + assertEquals("Username should match", "simple", user.getName()); + assertEquals("Short username should match", "simple", user.getShortName()); + // don't test shortening of kerberos names because regular Hadoop doesn't support them + } + + @Test + public void testRunAs() throws Exception { + Configuration conf = HBaseConfiguration.create(); + final User user = User.createUserForTesting(conf, "testuser", new String[]{"foo"}); + final PrivilegedExceptionAction action = new PrivilegedExceptionAction(){ + public String run() throws IOException { + User u = User.getCurrent(); + return u.getName(); + } + }; + + String username = user.runAs(action); + assertEquals("Current user within runAs() should match", + "testuser", username); + + // ensure the next run is correctly set + User user2 = User.createUserForTesting(conf, "testuser2", new String[]{"foo"}); + String username2 = user2.runAs(action); + assertEquals("Second username should match second user", + "testuser2", username2); + + // check the exception version + username = user.runAs(new PrivilegedExceptionAction(){ + public String run() throws Exception { + return User.getCurrent().getName(); + } + }); + assertEquals("User name in runAs() should match", "testuser", username); + + // verify that nested contexts work + user2.runAs(new PrivilegedExceptionAction(){ + public Object run() throws IOException, InterruptedException{ + String nestedName = user.runAs(action); + assertEquals("Nest name should match nested user", "testuser", nestedName); + assertEquals("Current name should match current user", + "testuser2", User.getCurrent().getName()); + return null; + } + }); + } + + /** + * Make sure that we're returning a result for the current user. + * Previously getCurrent() was returning null if not initialized on + * non-secure Hadoop variants. + */ + @Test + public void testGetCurrent() throws Exception { + User user1 = User.getCurrent(); + assertNotNull(user1.ugi); + LOG.debug("User1 is "+user1.getName()); + + for (int i =0 ; i< 100; i++) { + User u = User.getCurrent(); + assertNotNull(u); + assertEquals(user1.getName(), u.getName()); + assertEquals(user1, u); + assertEquals(user1.hashCode(), u.hashCode()); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java new file mode 100644 index 0000000..b04fb55 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException; +import org.apache.hadoop.hbase.snapshot.HSnapshotDescription; +import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.Assert; + +/** + * Utilities class for snapshots + */ +public class SnapshotTestingUtils { + + private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class); + + /** + * Assert that we don't have any snapshots lists + * @throws IOException if the admin operation fails + */ + public static void assertNoSnapshots(HBaseAdmin admin) throws IOException { + assertEquals("Have some previous snapshots", 0, admin.listSnapshots().size()); + } + + /** + * Make sure that there is only one snapshot returned from the master and its name and table match + * the passed in parameters. + */ + public static void assertOneSnapshotThatMatches(HBaseAdmin admin, HSnapshotDescription snapshot) + throws IOException { + assertOneSnapshotThatMatches(admin, snapshot.getName(), snapshot.getTable()); + } + + /** + * Make sure that there is only one snapshot returned from the master and its name and table match + * the passed in parameters. + */ + public static void assertOneSnapshotThatMatches(HBaseAdmin admin, SnapshotDescription snapshot) + throws IOException { + assertOneSnapshotThatMatches(admin, snapshot.getName(), snapshot.getTable()); + } + + /** + * Make sure that there is only one snapshot returned from the master and its name and table match + * the passed in parameters. + */ + public static List assertOneSnapshotThatMatches(HBaseAdmin admin, + String snapshotName, String tableName) throws IOException { + // list the snapshot + List snapshots = admin.listSnapshots(); + + assertEquals("Should only have 1 snapshot", 1, snapshots.size()); + assertEquals(snapshotName, snapshots.get(0).getName()); + assertEquals(tableName, snapshots.get(0).getTable()); + + return snapshots; + } + + /** + * Make sure that there is only one snapshot returned from the master and its name and table match + * the passed in parameters. + */ + public static List assertOneSnapshotThatMatches(HBaseAdmin admin, + byte[] snapshot, byte[] tableName) throws IOException { + return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot), Bytes.toString(tableName)); + } + + /** + * Confirm that the snapshot contains references to all the files that should be in the snapshot + */ + public static void confirmSnapshotValid(SnapshotDescription snapshotDescriptor, + byte[] tableName, byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs, + boolean requireLogs, Path logsDir, Set snapshotServers) throws IOException { + Path snapshotDir = SnapshotDescriptionUtils + .getCompletedSnapshotDir(snapshotDescriptor, rootDir); + assertTrue(fs.exists(snapshotDir)); + Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE); + assertTrue(fs.exists(snapshotinfo)); + // check the logs dir + if (requireLogs) { + TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir, snapshotServers, + snapshotDescriptor, new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME)); + } + // check the table info + HTableDescriptor desc = FSTableDescriptors.getTableDescriptor(fs, rootDir, tableName); + HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir); + assertEquals(desc, snapshotDesc); + + // check the region snapshot for all the regions + List regions = admin.getTableRegions(tableName); + for (HRegionInfo info : regions) { + String regionName = info.getEncodedName(); + Path regionDir = new Path(snapshotDir, regionName); + HRegionInfo snapshotRegionInfo = HRegion.loadDotRegionInfoFileContent(fs, regionDir); + assertEquals(info, snapshotRegionInfo); + // check to make sure we have the family + Path familyDir = new Path(regionDir, Bytes.toString(testFamily)); + assertTrue("Expected to find: " + familyDir + ", but it doesn't exist", fs.exists(familyDir)); + // make sure we have some files references + assertTrue(fs.listStatus(familyDir).length > 0); + } + } + + /** + * Helper method for testing async snapshot operations. Just waits for the given snapshot to + * complete on the server by repeatedly checking the master. + * @param master running the snapshot + * @param snapshot to check + * @param sleep amount to sleep between checks to see if the snapshot is done + * @throws IOException if the snapshot fails + */ + public static void waitForSnapshotToComplete(HMaster master, HSnapshotDescription snapshot, + long sleep) throws IOException { + boolean done = false; + while (!done) { + done = master.isSnapshotDone(snapshot); + try { + Thread.sleep(sleep); + } catch (InterruptedException e) { + throw new IOException(e); + } + } + } + + public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName) throws IOException { + SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName)); + } + + public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName) throws IOException { + // delete the taken snapshot + admin.deleteSnapshot(snapshotName); + assertNoSnapshots(admin); + } + + /** + * Expect the snapshot to throw an error when checking if the snapshot is complete + * @param master master to check + * @param snapshot the {@link HSnapshotDescription} request to pass to the master + * @param clazz expected exception from the master + */ + public static void expectSnapshotDoneException(HMaster master, HSnapshotDescription snapshot, + Class clazz) { + try { + boolean res = master.isSnapshotDone(snapshot); + Assert.fail("didn't fail to lookup a snapshot: res=" + res); + } catch (HBaseSnapshotException e) { + assertEquals("Threw wrong snapshot exception!", clazz, e.getClass()); + } catch (Throwable t) { + Assert.fail("Threw an unexpected exception:" + t); + } + } + + /** + * List all the HFiles in the given table + * @param fs FileSystem where the table lives + * @param tableDir directory of the table + * @return array of the current HFiles in the table (could be a zero-length array) + * @throws IOException on unexecpted error reading the FS + */ + public static FileStatus[] listHFiles(final FileSystem fs, Path tableDir) throws IOException { + // setup the filters we will need based on the filesystem + PathFilter regionFilter = new FSUtils.RegionDirFilter(fs); + PathFilter familyFilter = new FSUtils.FamilyDirFilter(fs); + final PathFilter fileFilter = new PathFilter() { + @Override + public boolean accept(Path file) { + try { + return fs.isFile(file); + } catch (IOException e) { + return false; + } + } + }; + + FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, regionFilter); + // if no regions, then we are done + if (regionDirs == null || regionDirs.length == 0) return new FileStatus[0]; + + // go through each of the regions, and add al the hfiles under each family + List regionFiles = new ArrayList(regionDirs.length); + for (FileStatus regionDir : regionDirs) { + FileStatus[] fams = FSUtils.listStatus(fs, regionDir.getPath(), familyFilter); + // if no families, then we are done again + if (fams == null || fams.length == 0) continue; + // add all the hfiles under the family + regionFiles.addAll(SnapshotTestingUtils.getHFilesInRegion(fams, fs, fileFilter)); + } + FileStatus[] files = new FileStatus[regionFiles.size()]; + regionFiles.toArray(files); + return files; + } + + /** + * Get all the hfiles in the region, under the passed set of families + * @param families all the family directories under the region + * @param fs filesystem where the families live + * @param fileFilter filter to only include files + * @return collection of all the hfiles under all the passed in families (non-null) + * @throws IOException on unexecpted error reading the FS + */ + public static Collection getHFilesInRegion(FileStatus[] families, FileSystem fs, + PathFilter fileFilter) throws IOException { + Set files = new TreeSet(); + for (FileStatus family : families) { + // get all the hfiles in the family + FileStatus[] hfiles = FSUtils.listStatus(fs, family.getPath(), fileFilter); + // if no hfiles, then we are done with this family + if (hfiles == null || hfiles.length == 0) continue; + files.addAll(Arrays.asList(hfiles)); + } + return files; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestCopyRecoveredEditsTask.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestCopyRecoveredEditsTask.java new file mode 100644 index 0000000..b68c3b9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestCopyRecoveredEditsTask.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.errorhandling.ForeignException; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test that we correctly copy the recovered edits from a directory + */ +@Category(SmallTests.class) +public class TestCopyRecoveredEditsTask { + + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @Test + public void testCopyFiles() throws Exception { + + SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build(); + ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); + FileSystem fs = UTIL.getTestFileSystem(); + Path root = UTIL.getDataTestDir(); + String regionName = "regionA"; + Path regionDir = new Path(root, regionName); + Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root); + + try { + // doesn't really matter where the region's snapshot directory is, but this is pretty close + Path snapshotRegionDir = new Path(workingDir, regionName); + fs.mkdirs(snapshotRegionDir); + + // put some stuff in the recovered.edits directory + Path edits = HLog.getRegionDirRecoveredEditsDir(regionDir); + fs.mkdirs(edits); + // make a file with some data + Path file1 = new Path(edits, "0000000000000002352"); + FSDataOutputStream out = fs.create(file1); + byte[] data = new byte[] { 1, 2, 3, 4 }; + out.write(data); + out.close(); + // make an empty file + Path empty = new Path(edits, "empty"); + fs.createNewFile(empty); + + CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, + snapshotRegionDir); + CopyRecoveredEditsTask taskSpy = Mockito.spy(task); + taskSpy.call(); + + Path snapshotEdits = HLog.getRegionDirRecoveredEditsDir(snapshotRegionDir); + FileStatus[] snapshotEditFiles = FSUtils.listStatus(fs, snapshotEdits); + assertEquals("Got wrong number of files in the snapshot edits", 1, snapshotEditFiles.length); + FileStatus file = snapshotEditFiles[0]; + assertEquals("Didn't copy expected file", file1.getName(), file.getPath().getName()); + + Mockito.verify(monitor, Mockito.never()).receive(Mockito.any(ForeignException.class)); + Mockito.verify(taskSpy, Mockito.never()).snapshotFailure(Mockito.anyString(), + Mockito.any(Exception.class)); + } finally { + // cleanup the working directory + FSUtils.delete(fs, regionDir, true); + FSUtils.delete(fs, workingDir, true); + } + } + + /** + * Check that we don't get an exception if there is no recovered edits directory to copy + * @throws Exception on failure + */ + @Test + public void testNoEditsDir() throws Exception { + SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build(); + ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); + FileSystem fs = UTIL.getTestFileSystem(); + Path root = UTIL.getDataTestDir(); + String regionName = "regionA"; + Path regionDir = new Path(root, regionName); + Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root); + try { + // doesn't really matter where the region's snapshot directory is, but this is pretty close + Path snapshotRegionDir = new Path(workingDir, regionName); + fs.mkdirs(snapshotRegionDir); + Path regionEdits = HLog.getRegionDirRecoveredEditsDir(regionDir); + assertFalse("Edits dir exists already - it shouldn't", fs.exists(regionEdits)); + + CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, + snapshotRegionDir); + task.call(); + } finally { + // cleanup the working directory + FSUtils.delete(fs, regionDir, true); + FSUtils.delete(fs, workingDir, true); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java new file mode 100644 index 0000000..9dafa9a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java @@ -0,0 +1,335 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.snapshot.ExportSnapshot; +import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; +import org.apache.hadoop.mapreduce.Job; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test Export Snapshot Tool + */ +@Category(MediumTests.class) +public class TestExportSnapshot { + private final Log LOG = LogFactory.getLog(getClass()); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private final static byte[] FAMILY = Bytes.toBytes("cf"); + + private byte[] emptySnapshotName; + private byte[] snapshotName; + private byte[] tableName; + private HBaseAdmin admin; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); + TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); + TEST_UTIL.startMiniCluster(3); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Create a table and take a snapshot of the table used by the export test. + */ + @Before + public void setUp() throws Exception { + this.admin = TEST_UTIL.getHBaseAdmin(); + + long tid = System.currentTimeMillis(); + tableName = Bytes.toBytes("testtb-" + tid); + snapshotName = Bytes.toBytes("snaptb0-" + tid); + emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + tid); + + // create Table + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(FAMILY)); + admin.createTable(htd, null); + + // Take an empty snapshot + admin.snapshot(emptySnapshotName, tableName); + + // Add some rows + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + TEST_UTIL.loadTable(table, FAMILY); + + // take a snapshot + admin.snapshot(snapshotName, tableName); + } + + @After + public void tearDown() throws Exception { + admin.disableTable(tableName); + admin.deleteSnapshot(snapshotName); + admin.deleteSnapshot(emptySnapshotName); + admin.deleteTable(tableName); + admin.close(); + } + + /** + * Verfy the result of getBalanceSplits() method. + * The result are groups of files, used as input list for the "export" mappers. + * All the groups should have similar amount of data. + * + * The input list is a pair of file path and length. + * The getBalanceSplits() function sort it by length, + * and assign to each group a file, going back and forth through the groups. + */ + @Test + public void testBalanceSplit() throws Exception { + // Create a list of files + List> files = new ArrayList>(); + for (long i = 0; i <= 20; i++) { + files.add(new Pair(new Path("file-" + i), i)); + } + + // Create 5 groups (total size 210) + // group 0: 20, 11, 10, 1 (total size: 42) + // group 1: 19, 12, 9, 2 (total size: 42) + // group 2: 18, 13, 8, 3 (total size: 42) + // group 3: 17, 12, 7, 4 (total size: 42) + // group 4: 16, 11, 6, 5 (total size: 42) + List> splits = ExportSnapshot.getBalancedSplits(files, 5); + assertEquals(5, splits.size()); + assertEquals(Arrays.asList(new Path("file-20"), new Path("file-11"), + new Path("file-10"), new Path("file-1"), new Path("file-0")), splits.get(0)); + assertEquals(Arrays.asList(new Path("file-19"), new Path("file-12"), + new Path("file-9"), new Path("file-2")), splits.get(1)); + assertEquals(Arrays.asList(new Path("file-18"), new Path("file-13"), + new Path("file-8"), new Path("file-3")), splits.get(2)); + assertEquals(Arrays.asList(new Path("file-17"), new Path("file-14"), + new Path("file-7"), new Path("file-4")), splits.get(3)); + assertEquals(Arrays.asList(new Path("file-16"), new Path("file-15"), + new Path("file-6"), new Path("file-5")), splits.get(4)); + } + + /** + * Verify if exported snapshot and copied files matches the original one. + */ + @Test + public void testExportFileSystemState() throws Exception { + testExportFileSystemState(tableName, snapshotName, 2); + } + + @Test + public void testEmptyExportFileSystemState() throws Exception { + testExportFileSystemState(tableName, emptySnapshotName, 1); + } + + /** + * Mock a snapshot with files in the archive dir, + * two regions, and one reference file. + */ + @Test + public void testSnapshotWithRefsExportFileSystemState() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + + final byte[] tableWithRefsName = Bytes.toBytes("tableWithRefs"); + final String snapshotName = "tableWithRefs"; + final String TEST_FAMILY = Bytes.toString(FAMILY); + final String TEST_HFILE = "abc"; + + final SnapshotDescription sd = SnapshotDescription.newBuilder() + .setName(snapshotName).setTable(Bytes.toString(tableWithRefsName)).build(); + + FileSystem fs = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); + Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); + Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); + + HTableDescriptor htd = new HTableDescriptor(tableWithRefsName); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + + // First region, simple with one plain hfile. + HRegion r0 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir, + conf, htd, null, true, true); + Path storeFile = new Path(new Path(r0.getRegionDir(), TEST_FAMILY), TEST_HFILE); + FSDataOutputStream out = fs.create(storeFile); + out.write(Bytes.toBytes("Test Data")); + out.close(); + r0.close(); + + // Second region, used to test the split case. + // This region contains a reference to the hfile in the first region. + HRegion r1 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir, + conf, htd, null, true, true); + out = fs.create(new Path(new Path(r1.getRegionDir(), TEST_FAMILY), + storeFile.getName() + '.' + r0.getRegionInfo().getEncodedName())); + out.write(Bytes.toBytes("Test Data")); + out.close(); + r1.close(); + + Path tableDir = HTableDescriptor.getTableDir(archiveDir, tableWithRefsName); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf); + SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs); + + testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2); + } + + /** + * Test ExportSnapshot + */ + private void testExportFileSystemState(final byte[] tableName, final byte[] snapshotName, + int filesExpected) throws Exception { + Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis()); + URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri(); + FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); + copyDir = copyDir.makeQualified(fs); + + // Export Snapshot + int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(), new String[] { + "-snapshot", Bytes.toString(snapshotName), + "-copy-to", copyDir.toString() + }); + assertEquals(0, res); + + // Verify File-System state + FileStatus[] rootFiles = fs.listStatus(copyDir); + assertEquals(filesExpected, rootFiles.length); + for (FileStatus fileStatus: rootFiles) { + String name = fileStatus.getPath().getName(); + assertTrue(fileStatus.isDir()); + assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME) || name.equals(".archive")); + } + + // compare the snapshot metadata and verify the hfiles + final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration()); + final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName)); + verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir), + fs, new Path(copyDir, snapshotDir)); + verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName)); + FSUtils.logFileSystemState(hdfs, snapshotDir, LOG); + + // Remove the exported dir + fs.delete(copyDir, true); + } + + /* + * verify if the snapshot folder on file-system 1 match the one on file-system 2 + */ + private void verifySnapshot(final FileSystem fs1, final Path root1, + final FileSystem fs2, final Path root2) throws IOException { + Set s = new HashSet(); + assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2)); + } + + /* + * Verify if the files exists + */ + private void verifyArchive(final FileSystem fs, final Path rootDir, + final byte[] tableName, final String snapshotName) throws IOException { + final Path exportedSnapshot = new Path(rootDir, + new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName)); + final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); + LOG.debug(listFiles(fs, exportedArchive, exportedArchive)); + SnapshotReferenceUtil.visitReferencedFiles(fs, exportedSnapshot, + new SnapshotReferenceUtil.FileVisitor() { + public void storeFile (final String region, final String family, final String hfile) + throws IOException { + verifyNonEmptyFile(new Path(exportedArchive, + new Path(Bytes.toString(tableName), new Path(region, new Path(family, hfile))))); + } + + public void recoveredEdits (final String region, final String logfile) + throws IOException { + verifyNonEmptyFile(new Path(exportedSnapshot, + new Path(Bytes.toString(tableName), new Path(region, logfile)))); + } + + public void logFile (final String server, final String logfile) + throws IOException { + verifyNonEmptyFile(new Path(exportedSnapshot, new Path(server, logfile))); + } + + private void verifyNonEmptyFile(final Path path) throws IOException { + assertTrue(path + " should exist", fs.exists(path)); + assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0); + } + }); + } + + private Set listFiles(final FileSystem fs, final Path root, final Path dir) + throws IOException { + Set files = new HashSet(); + int rootPrefix = root.toString().length(); + FileStatus[] list = FSUtils.listStatus(fs, dir); + if (list != null) { + for (FileStatus fstat: list) { + LOG.debug(fstat.getPath()); + if (fstat.isDir()) { + files.addAll(listFiles(fs, root, fstat.getPath())); + } else { + files.add(fstat.getPath().toString().substring(rootPrefix)); + } + } + } + return files; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java new file mode 100644 index 0000000..ded6c66 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java @@ -0,0 +1,427 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test creating/using/deleting snapshots from the client + *

    + * This is an end-to-end test for the snapshot utility + * + * TODO This is essentially a clone of TestSnapshotFromClient. This is worth refactoring this + * because there will be a few more flavors of snapshots that need to run these tests. + */ +@Category(LargeTests.class) +public class TestFlushSnapshotFromClient { + private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final int NUM_RS = 2; + private static final String STRING_TABLE_NAME = "test"; + private static final byte[] TEST_FAM = Bytes.toBytes("fam"); + private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME); + + /** + * Setup the config for the cluster + * @throws Exception on failure + */ + @BeforeClass + public static void setupCluster() throws Exception { + setupConf(UTIL.getConfiguration()); + UTIL.startMiniCluster(NUM_RS); + } + + private static void setupConf(Configuration conf) { + // disable the ui + conf.setInt("hbase.regionsever.info.port", -1); + // change the flush size to a small amount, regulating number of store files + conf.setInt("hbase.hregion.memstore.flush.size", 25000); + // so make sure we get a compaction when doing a load, but keep around some + // files in the store + conf.setInt("hbase.hstore.compaction.min", 10); + conf.setInt("hbase.hstore.compactionThreshold", 10); + // block writes if we get to 12 store files + conf.setInt("hbase.hstore.blockingStoreFiles", 12); + // drop the number of attempts for the hbase admin + conf.setInt("hbase.client.retries.number", 1); + // Enable snapshot + conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + // prevent aggressive region split + conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, + ConstantSizeRegionSplitPolicy.class.getName()); + } + + @Before + public void setup() throws Exception { + UTIL.createTable(TABLE_NAME, TEST_FAM); + } + + @After + public void tearDown() throws Exception { + UTIL.deleteTable(TABLE_NAME); + // and cleanup the archive directory + try { + UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true); + } catch (IOException e) { + LOG.warn("Failure to delete archive directory", e); + } + } + + @AfterClass + public static void cleanupTest() throws Exception { + try { + UTIL.shutdownMiniCluster(); + } catch (Exception e) { + LOG.warn("failure shutting down cluster", e); + } + } + + /** + * Test simple flush snapshotting a table that is online + * @throws Exception + */ + @Test + public void testFlushTableSnapshot() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + // make sure we don't fail on listing snapshots + SnapshotTestingUtils.assertNoSnapshots(admin); + + // put some stuff in the table + HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME); + UTIL.loadTable(table, TEST_FAM); + + // get the name of all the regionservers hosting the snapshotted table + Set snapshotServers = new HashSet(); + List servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + for (RegionServerThread server : servers) { + if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) { + snapshotServers.add(server.getRegionServer().getServerName().toString()); + } + } + + LOG.debug("FS state before snapshot:"); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), + FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + + // take a snapshot of the enabled table + String snapshotString = "offlineTableSnapshot"; + byte[] snapshot = Bytes.toBytes(snapshotString); + admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH); + LOG.debug("Snapshot completed."); + + // make sure we have the snapshot + List snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, + snapshot, TABLE_NAME); + + // make sure its a valid snapshot + FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); + Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); + LOG.debug("FS state after snapshot:"); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), + FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + + SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, + admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers); + + admin.deleteSnapshot(snapshot); + snapshots = admin.listSnapshots(); + SnapshotTestingUtils.assertNoSnapshots(admin); + } + + @Test + public void testSnapshotFailsOnNonExistantTable() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + // make sure we don't fail on listing snapshots + SnapshotTestingUtils.assertNoSnapshots(admin); + String tableName = "_not_a_table"; + + // make sure the table doesn't exist + boolean fail = false; + do { + try { + admin.getTableDescriptor(Bytes.toBytes(tableName)); + fail = true; + LOG.error("Table:" + tableName + " already exists, checking a new name"); + tableName = tableName+"!"; + } catch (TableNotFoundException e) { + fail = false; + } + } while (fail); + + // snapshot the non-existant table + try { + admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH); + fail("Snapshot succeeded even though there is not table."); + } catch (SnapshotCreationException e) { + LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage()); + } + } + + @Test(timeout = 60000) + public void testAsyncFlushSnapshot() throws Exception { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot") + .setTable(STRING_TABLE_NAME).setType(SnapshotDescription.Type.FLUSH).build(); + + // take the snapshot async + admin.takeSnapshotAsync(snapshot); + + // constantly loop, looking for the snapshot to complete + HMaster master = UTIL.getMiniHBaseCluster().getMaster(); + SnapshotTestingUtils.waitForSnapshotToComplete(master, new HSnapshotDescription(snapshot), 200); + LOG.info(" === Async Snapshot Completed ==="); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), + FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + // make sure we get the snapshot + SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot); + + // test that we can delete the snapshot + admin.deleteSnapshot(snapshot.getName()); + LOG.info(" === Async Snapshot Deleted ==="); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), + FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + // make sure we don't have any snapshots + SnapshotTestingUtils.assertNoSnapshots(admin); + LOG.info(" === Async Snapshot Test Completed ==="); + + } + + /** + * Basic end-to-end test of simple-flush-based snapshots + */ + @Test + public void testFlushCreateListDestroy() throws Exception { + LOG.debug("------- Starting Snapshot test -------------"); + HBaseAdmin admin = UTIL.getHBaseAdmin(); + // make sure we don't fail on listing snapshots + SnapshotTestingUtils.assertNoSnapshots(admin); + // load the table so we have some data + UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM); + // and wait until everything stabilizes + waitForTableToBeOnline(TABLE_NAME); + + String snapshotName = "flushSnapshotCreateListDestroy"; + // test creating the snapshot + admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH); + logFSTree(new Path(UTIL.getConfiguration().get(HConstants.HBASE_DIR))); + + // make sure we only have 1 matching snapshot + List snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, + snapshotName, STRING_TABLE_NAME); + + // check the directory structure + FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); + Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshots.get(0), rootDir); + assertTrue(fs.exists(snapshotDir)); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), snapshotDir, LOG); + Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE); + assertTrue(fs.exists(snapshotinfo)); + + // check the table info + HTableDescriptor desc = FSTableDescriptors.getTableDescriptor(fs, rootDir, TABLE_NAME); + HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptor(fs, + SnapshotDescriptionUtils.getSnapshotsDir(rootDir), Bytes.toBytes(snapshotName)); + assertEquals(desc, snapshotDesc); + + // check the region snapshot for all the regions + List regions = admin.getTableRegions(TABLE_NAME); + for (HRegionInfo info : regions) { + String regionName = info.getEncodedName(); + Path regionDir = new Path(snapshotDir, regionName); + HRegionInfo snapshotRegionInfo = HRegion.loadDotRegionInfoFileContent(fs, regionDir); + assertEquals(info, snapshotRegionInfo); + // check to make sure we have the family + Path familyDir = new Path(regionDir, Bytes.toString(TEST_FAM)); + assertTrue(fs.exists(familyDir)); + // make sure we have some file references + assertTrue(fs.listStatus(familyDir).length > 0); + } + + // test that we can delete the snapshot + admin.deleteSnapshot(snapshotName); + FSUtils.logFileSystemState(UTIL.getTestFileSystem(), + FSUtils.getRootDir(UTIL.getConfiguration()), LOG); + + // make sure we don't have any snapshots + SnapshotTestingUtils.assertNoSnapshots(admin); + LOG.debug("------- Flush-Snapshot Create List Destroy-------------"); + } + + /** + * Demonstrate that we reject snapshot requests if there is a snapshot already running on the + * same table currently running and that concurrent snapshots on different tables can both + * succeed concurretly. + */ + @Test(timeout=60000) + public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException { + final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2"; + final byte[] TABLE2_NAME = Bytes.toBytes(STRING_TABLE2_NAME); + + int ssNum = 20; + HBaseAdmin admin = UTIL.getHBaseAdmin(); + // make sure we don't fail on listing snapshots + SnapshotTestingUtils.assertNoSnapshots(admin); + // create second testing table + UTIL.createTable(TABLE2_NAME, TEST_FAM); + // load the table so we have some data + UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM); + UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE2_NAME), TEST_FAM); + // and wait until everything stabilizes + waitForTableToBeOnline(TABLE_NAME); + waitForTableToBeOnline(TABLE2_NAME); + + final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum); + // We'll have one of these per thread + class SSRunnable implements Runnable { + SnapshotDescription ss; + SSRunnable(SnapshotDescription ss) { + this.ss = ss; + } + + @Override + public void run() { + try { + HBaseAdmin admin = UTIL.getHBaseAdmin(); + LOG.info("Submitting snapshot request: " + SnapshotDescriptionUtils.toString(ss)); + admin.takeSnapshotAsync(ss); + } catch (Exception e) { + LOG.info("Exception during snapshot request: " + SnapshotDescriptionUtils.toString(ss) + + ". This is ok, we expect some", e); + } + LOG.info("Submitted snapshot request: " + SnapshotDescriptionUtils.toString(ss)); + toBeSubmitted.countDown(); + } + }; + + // build descriptions + SnapshotDescription[] descs = new SnapshotDescription[ssNum]; + for (int i = 0; i < ssNum; i++) { + SnapshotDescription.Builder builder = SnapshotDescription.newBuilder(); + builder.setTable((i % 2) == 0 ? STRING_TABLE_NAME : STRING_TABLE2_NAME); + builder.setName("ss"+i); + builder.setType(SnapshotDescription.Type.FLUSH); + descs[i] = builder.build(); + } + + // kick each off its own thread + for (int i=0 ; i < ssNum; i++) { + new Thread(new SSRunnable(descs[i])).start(); + } + + // wait until all have been submitted + toBeSubmitted.await(); + + // loop until all are done. + while (true) { + int doneCount = 0; + for (SnapshotDescription ss : descs) { + try { + if (admin.isSnapshotFinished(ss)) { + doneCount++; + } + } catch (Exception e) { + LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e); + doneCount++; + } + } + if (doneCount == descs.length) { + break; + } + Thread.sleep(100); + } + + // dump for debugging + logFSTree(new Path(UTIL.getConfiguration().get(HConstants.HBASE_DIR))); + + List taken = admin.listSnapshots(); + int takenSize = taken.size(); + LOG.info("Taken " + takenSize + " snapshots: " + taken); + assertTrue("We expect at least 1 request to be rejected because of we concurrently" + + " issued many requests", takenSize < ssNum && takenSize > 0); + + // Verify that there's at least one snapshot per table + int t1SnapshotsCount = 0; + int t2SnapshotsCount = 0; + for (SnapshotDescription ss : taken) { + if (ss.getTable().equals(STRING_TABLE_NAME)) { + t1SnapshotsCount++; + } else if (ss.getTable().equals(STRING_TABLE2_NAME)) { + t2SnapshotsCount++; + } + } + assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0); + assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0); + + // delete snapshots so subsequent tests are clean. + for (SnapshotDescription ss : taken) { + admin.deleteSnapshot(ss.getName()); + } + UTIL.deleteTable(TABLE2_NAME); + } + + private void logFSTree(Path root) throws IOException { + FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG); + } + + private void waitForTableToBeOnline(final byte[] tableName) throws IOException { + HRegionServer rs = UTIL.getRSForFirstRegionInTable(tableName); + List onlineRegions = rs.getOnlineRegions(tableName); + for (HRegion region : onlineRegions) { + region.waitForFlushesAndCompactions(); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestReferenceRegionHFilesTask.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestReferenceRegionHFilesTask.java new file mode 100644 index 0000000..73c2aba --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestReferenceRegionHFilesTask.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.snapshot.ReferenceRegionHFilesTask; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +@Category(SmallTests.class) +public class TestReferenceRegionHFilesTask { + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @Test + public void testRun() throws IOException { + FileSystem fs = UTIL.getTestFileSystem(); + // setup the region internals + Path testdir = UTIL.getDataTestDir(); + Path regionDir = new Path(testdir, "region"); + Path family1 = new Path(regionDir, "fam1"); + // make an empty family + Path family2 = new Path(regionDir, "fam2"); + fs.mkdirs(family2); + + // add some files to family 1 + Path file1 = new Path(family1, "05f99689ae254693836613d1884c6b63"); + fs.createNewFile(file1); + Path file2 = new Path(family1, "7ac9898bf41d445aa0003e3d699d5d26"); + fs.createNewFile(file2); + + // create the snapshot directory + Path snapshotRegionDir = new Path(testdir, HConstants.SNAPSHOT_DIR_NAME); + fs.mkdirs(snapshotRegionDir); + + SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("name") + .setTable("table").build(); + ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); + ReferenceRegionHFilesTask task = new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, + fs, snapshotRegionDir); + ReferenceRegionHFilesTask taskSpy = Mockito.spy(task); + task.call(); + + // make sure we never get an error + Mockito.verify(taskSpy, Mockito.never()).snapshotFailure(Mockito.anyString(), + Mockito.any(Exception.class)); + + // verify that all the hfiles get referenced + List hfiles = new ArrayList(2); + FileStatus[] regions = FSUtils.listStatus(fs, snapshotRegionDir); + for (FileStatus region : regions) { + FileStatus[] fams = FSUtils.listStatus(fs, region.getPath()); + for (FileStatus fam : fams) { + FileStatus[] files = FSUtils.listStatus(fs, fam.getPath()); + for (FileStatus file : files) { + hfiles.add(file.getPath().getName()); + } + } + } + assertTrue("Didn't reference :" + file1, hfiles.contains(file1.getName())); + assertTrue("Didn't reference :" + file1, hfiles.contains(file2.getName())); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java new file mode 100644 index 0000000..498d508 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java @@ -0,0 +1,254 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.MD5Hash; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test clone/restore snapshots from the client + * + * TODO This is essentially a clone of TestRestoreSnapshotFromClient. This is worth refactoring + * this because there will be a few more flavors of snapshots that need to run these tests. + */ +@Category(LargeTests.class) +public class TestRestoreFlushSnapshotFromClient { + final Log LOG = LogFactory.getLog(getClass()); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private final byte[] FAMILY = Bytes.toBytes("cf"); + + private byte[] snapshotName0; + private byte[] snapshotName1; + private byte[] snapshotName2; + private int snapshot0Rows; + private int snapshot1Rows; + private byte[] tableName; + private HBaseAdmin admin; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true); + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); + TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6); + TEST_UTIL.getConfiguration().setBoolean( + "hbase.master.enabletable.roundrobin", true); + + // Enable snapshot + TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + + TEST_UTIL.startMiniCluster(3); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Initialize the tests with a table filled with some data + * and two snapshots (snapshotName0, snapshotName1) of different states. + * The tableName, snapshotNames and the number of rows in the snapshot are initialized. + */ + @Before + public void setup() throws Exception { + this.admin = TEST_UTIL.getHBaseAdmin(); + + long tid = System.currentTimeMillis(); + tableName = Bytes.toBytes("testtb-" + tid); + snapshotName0 = Bytes.toBytes("snaptb0-" + tid); + snapshotName1 = Bytes.toBytes("snaptb1-" + tid); + snapshotName2 = Bytes.toBytes("snaptb2-" + tid); + + // create Table and disable it + createTable(tableName, FAMILY); + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + try { + loadData(table, 500, FAMILY); + snapshot0Rows = TEST_UTIL.countRows(table); + LOG.info("=== before snapshot with 500 rows"); + logFSTree(); + + // take a snapshot + admin.snapshot(Bytes.toString(snapshotName0), Bytes.toString(tableName), + SnapshotDescription.Type.FLUSH); + + LOG.info("=== after snapshot with 500 rows"); + logFSTree(); + + // insert more data + loadData(table, 500, FAMILY); + snapshot1Rows = TEST_UTIL.countRows(table); + LOG.info("=== before snapshot with 1000 rows"); + logFSTree(); + + // take a snapshot of the updated table + admin.snapshot(Bytes.toString(snapshotName1), Bytes.toString(tableName), + SnapshotDescription.Type.FLUSH); + LOG.info("=== after snapshot with 1000 rows"); + logFSTree(); + } finally { + table.close(); + } + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.deleteTable(tableName); + admin.deleteSnapshot(snapshotName0); + admin.deleteSnapshot(snapshotName1); + + // Ensure the archiver to be empty + MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); + mfs.getFileSystem().delete( + new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true); + } + + @Test + public void testTakeFlushSnapshot() throws IOException { + // taking happens in setup. + } + + @Test + public void testRestoreSnapshot() throws IOException { + verifyRowCount(tableName, snapshot1Rows); + + // Restore from snapshot-0 + admin.disableTable(tableName); + admin.restoreSnapshot(snapshotName0); + logFSTree(); + admin.enableTable(tableName); + LOG.info("=== after restore with 500 row snapshot"); + logFSTree(); + verifyRowCount(tableName, snapshot0Rows); + + // Restore from snapshot-1 + admin.disableTable(tableName); + admin.restoreSnapshot(snapshotName1); + admin.enableTable(tableName); + verifyRowCount(tableName, snapshot1Rows); + } + + @Test(expected=SnapshotDoesNotExistException.class) + public void testCloneNonExistentSnapshot() throws IOException, InterruptedException { + String snapshotName = "random-snapshot-" + System.currentTimeMillis(); + String tableName = "random-table-" + System.currentTimeMillis(); + admin.cloneSnapshot(snapshotName, tableName); + } + + @Test + public void testCloneSnapshot() throws IOException, InterruptedException { + byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis()); + testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows); + testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows); + } + + private void testCloneSnapshot(final byte[] tableName, final byte[] snapshotName, + int snapshotRows) throws IOException, InterruptedException { + // create a new table from snapshot + admin.cloneSnapshot(snapshotName, tableName); + verifyRowCount(tableName, snapshotRows); + + TEST_UTIL.deleteTable(tableName); + } + + @Test + public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException { + byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis()); + admin.cloneSnapshot(snapshotName0, clonedTableName); + verifyRowCount(clonedTableName, snapshot0Rows); + admin.snapshot(Bytes.toString(snapshotName2), Bytes.toString(clonedTableName), SnapshotDescription.Type.FLUSH); + TEST_UTIL.deleteTable(clonedTableName); + + admin.cloneSnapshot(snapshotName2, clonedTableName); + verifyRowCount(clonedTableName, snapshot0Rows); + TEST_UTIL.deleteTable(clonedTableName); + } + + // ========================================================================== + // Helpers + // ========================================================================== + private void createTable(final byte[] tableName, final byte[]... families) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + for (byte[] family: families) { + HColumnDescriptor hcd = new HColumnDescriptor(family); + htd.addFamily(hcd); + } + byte[][] splitKeys = new byte[16][]; + byte[] hex = Bytes.toBytes("0123456789abcdef"); + for (int i = 0; i < 16; ++i) { + splitKeys[i] = new byte[] { hex[i] }; + } + admin.createTable(htd, splitKeys); + } + + public void loadData(final HTable table, int rows, byte[]... families) throws IOException { + byte[] qualifier = Bytes.toBytes("q"); + table.setAutoFlush(false); + while (rows-- > 0) { + byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows)); + byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value)); + Put put = new Put(key); + put.setWriteToWAL(false); + for (byte[] family: families) { + put.add(family, qualifier, value); + } + table.put(put); + } + table.flushCommits(); + } + + private void logFSTree() throws IOException { + MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); + FSUtils.logFileSystemState(mfs.getFileSystem(), mfs.getRootDir(), LOG); + } + + private void verifyRowCount(final byte[] tableName, long expectedRows) throws IOException { + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + assertEquals(expectedRows, TEST_UTIL.countRows(table)); + table.close(); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java new file mode 100644 index 0000000..b995a56 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java @@ -0,0 +1,202 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionTestingUtility; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.monitoring.MonitoredTask; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.StoreFile; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.MD5Hash; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test the restore/clone operation from a file-system point of view. + */ +@Category(SmallTests.class) +public class TestRestoreSnapshotHelper { + final Log LOG = LogFactory.getLog(getClass()); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static String TEST_FAMILY = "cf"; + private final static String TEST_HFILE = "abc"; + + private Configuration conf; + private Path archiveDir; + private FileSystem fs; + private Path rootDir; + + @Before + public void setup() throws Exception { + rootDir = TEST_UTIL.getDataTestDir("testRestore"); + archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); + fs = TEST_UTIL.getTestFileSystem(); + conf = TEST_UTIL.getConfiguration(); + FSUtils.setRootDir(conf, rootDir); + } + + @After + public void tearDown() throws Exception { + fs.delete(TEST_UTIL.getDataTestDir(), true); + } + + @Test + public void testRestore() throws IOException { + HTableDescriptor htd = createTableDescriptor("testtb"); + + Path snapshotDir = new Path(rootDir, "snapshot"); + createSnapshot(rootDir, snapshotDir, htd); + + // Test clone a snapshot + HTableDescriptor htdClone = createTableDescriptor("testtb-clone"); + testRestore(snapshotDir, htd.getNameAsString(), htdClone); + verifyRestore(rootDir, htd, htdClone); + + // Test clone a clone ("link to link") + Path cloneDir = HTableDescriptor.getTableDir(rootDir, htdClone.getName()); + HTableDescriptor htdClone2 = createTableDescriptor("testtb-clone2"); + testRestore(cloneDir, htdClone.getNameAsString(), htdClone2); + verifyRestore(rootDir, htd, htdClone2); + } + + private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd, + final HTableDescriptor htdClone) throws IOException { + String[] files = getHFiles(HTableDescriptor.getTableDir(rootDir, htdClone.getName())); + assertEquals(2, files.length); + assertTrue(files[0] + " should be a HFileLink", HFileLink.isHFileLink(files[0])); + assertTrue(files[1] + " should be a Referene", StoreFile.isReference(files[1])); + assertEquals(sourceHtd.getNameAsString(), HFileLink.getReferencedTableName(files[0])); + assertEquals(TEST_HFILE, HFileLink.getReferencedHFileName(files[0])); + Path refPath = getReferredToFile(files[1]); + assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName())); + assertEquals(files[0], refPath.getName()); + } + + /** + * Execute the restore operation + * @param snapshotDir The snapshot directory to use as "restore source" + * @param sourceTableName The name of the snapshotted table + * @param htdClone The HTableDescriptor of the table to restore/clone. + */ + public void testRestore(final Path snapshotDir, final String sourceTableName, + final HTableDescriptor htdClone) throws IOException { + LOG.debug("pre-restore table=" + htdClone.getNameAsString() + " snapshot=" + snapshotDir); + FSUtils.logFileSystemState(fs, rootDir, LOG); + + FSTableDescriptors.createTableDescriptor(htdClone, conf); + RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sourceTableName, htdClone); + helper.restoreHdfsRegions(); + + LOG.debug("post-restore table=" + htdClone.getNameAsString() + " snapshot=" + snapshotDir); + FSUtils.logFileSystemState(fs, rootDir, LOG); + } + + /** + * Initialize the restore helper, based on the snapshot and table information provided. + */ + private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir, + final String sourceTableName, final HTableDescriptor htdClone) throws IOException { + CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class); + HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class); + ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); + MonitoredTask status = Mockito.mock(MonitoredTask.class); + + SnapshotDescription sd = SnapshotDescription.newBuilder() + .setName("snapshot").setTable(sourceTableName).build(); + + return new RestoreSnapshotHelper(conf, fs, sd, snapshotDir, + htdClone, HTableDescriptor.getTableDir(rootDir, htdClone.getName()), monitor, status); + } + + private void createSnapshot(final Path rootDir, final Path snapshotDir, final HTableDescriptor htd) + throws IOException { + // First region, simple with one plain hfile. + HRegion r0 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir, + conf, htd, null, true, true); + Path storeFile = new Path(new Path(r0.getRegionDir(), TEST_FAMILY), TEST_HFILE); + fs.createNewFile(storeFile); + r0.close(); + + // Second region, used to test the split case. + // This region contains a reference to the hfile in the first region. + HRegion r1 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir, + conf, htd, null, true, true); + fs.createNewFile(new Path(new Path(r1.getRegionDir(), TEST_FAMILY), + storeFile.getName() + '.' + r0.getRegionInfo().getEncodedName())); + r1.close(); + + Path tableDir = HTableDescriptor.getTableDir(archiveDir, htd.getName()); + FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf); + } + + private HTableDescriptor createTableDescriptor(final String tableName) { + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + return htd; + } + + private Path getReferredToFile(final String referenceName) { + Path fakeBasePath = new Path(new Path("table", "region"), "cf"); + return StoreFile.getReferredToFile(new Path(fakeBasePath, referenceName)); + } + + private String[] getHFiles(final Path tableDir) throws IOException { + List files = new ArrayList(); + for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) { + for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { + for (FileStatus file: FSUtils.listStatus(fs, familyDir)) { + files.add(file.getPath().getName()); + } + } + } + Collections.sort(files); + return files.toArray(new String[files.size()]); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java new file mode 100644 index 0000000..0a32ca3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test that the {@link SnapshotDescription} helper is helping correctly. + */ +@Category(MediumTests.class) +public class TestSnapshotDescriptionUtils { + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static FileSystem fs; + private static Path root; + + @BeforeClass + public static void setupFS() throws Exception { + fs = UTIL.getTestFileSystem(); + root = new Path(UTIL.getDataTestDir(), "hbase"); + } + + @After + public void cleanupFS() throws Exception { + if (fs.exists(root)) { + if (!fs.delete(root, true)) { + throw new IOException("Failed to delete root test dir: " + root); + } + if (!fs.mkdirs(root)) { + throw new IOException("Failed to create root test dir: " + root); + } + } + EnvironmentEdgeManagerTestHelper.reset(); + } + + private static final Log LOG = LogFactory.getLog(TestSnapshotDescriptionUtils.class); + + @Test + public void testValidateMissingTableName() { + Configuration conf = new Configuration(false); + try { + SnapshotDescriptionUtils.validate(SnapshotDescription.newBuilder().setName("fail").build(), + conf); + fail("Snapshot was considered valid without a table name"); + } catch (IllegalArgumentException e) { + LOG.debug("Correctly failed when snapshot doesn't have a tablename"); + } + } + + /** + * Test that we throw an exception if there is no working snapshot directory when we attempt to + * 'complete' the snapshot + * @throws Exception on failure + */ + @Test + public void testCompleteSnapshotWithNoSnapshotDirectoryFailure() throws Exception { + Path snapshotDir = new Path(root, HConstants.SNAPSHOT_DIR_NAME); + Path tmpDir = new Path(snapshotDir, ".tmp"); + Path workingDir = new Path(tmpDir, "not_a_snapshot"); + assertFalse("Already have working snapshot dir: " + workingDir + + " but shouldn't. Test file leak?", fs.exists(workingDir)); + SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build(); + try { + SnapshotDescriptionUtils.completeSnapshot(snapshot, root, workingDir, fs); + fail("Shouldn't successfully complete move of a non-existent directory."); + } catch (IOException e) { + LOG.info("Correctly failed to move non-existant directory: " + e.getMessage()); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotLogSplitter.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotLogSplitter.java new file mode 100644 index 0000000..66b941a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotLogSplitter.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.*; +import org.junit.experimental.categories.Category; + +/** + * Test snapshot log splitter + */ +@Category(SmallTests.class) +public class TestSnapshotLogSplitter { + final Log LOG = LogFactory.getLog(getClass()); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private byte[] TEST_QUALIFIER = Bytes.toBytes("q"); + private byte[] TEST_FAMILY = Bytes.toBytes("f"); + + private Configuration conf; + private FileSystem fs; + private Path logFile; + + @Before + public void setup() throws Exception { + conf = TEST_UTIL.getConfiguration(); + fs = FileSystem.get(conf); + logFile = new Path(TEST_UTIL.getDataTestDir(), "test.log"); + writeTestLog(logFile); + } + + @After + public void tearDown() throws Exception { + fs.delete(logFile, false); + } + + @Test + public void testSplitLogs() throws IOException { + Map regionsMap = new TreeMap(Bytes.BYTES_COMPARATOR); + splitTestLogs(getTableName(5), regionsMap); + } + + @Test + public void testSplitLogsOnDifferentTable() throws IOException { + byte[] tableName = getTableName(1); + Map regionsMap = new TreeMap(Bytes.BYTES_COMPARATOR); + for (int j = 0; j < 10; ++j) { + byte[] regionName = getRegionName(tableName, j); + byte[] newRegionName = getNewRegionName(tableName, j); + regionsMap.put(regionName, newRegionName); + } + splitTestLogs(tableName, regionsMap); + } + + /* + * Split and verify test logs for the specified table + */ + private void splitTestLogs(final byte[] tableName, final Map regionsMap) + throws IOException { + Path tableDir = new Path(TEST_UTIL.getDataTestDir(), Bytes.toString(tableName)); + SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir, + tableName, regionsMap); + try { + logSplitter.splitLog(logFile); + } finally { + logSplitter.close(); + } + verifyRecoverEdits(tableDir, tableName, regionsMap); + } + + /* + * Verify that every logs in the table directory has just the specified table and regions. + */ + private void verifyRecoverEdits(final Path tableDir, final byte[] tableName, + final Map regionsMap) throws IOException { + for (FileStatus regionStatus: FSUtils.listStatus(fs, tableDir)) { + assertTrue(regionStatus.getPath().getName().startsWith(Bytes.toString(tableName))); + Path regionEdits = HLog.getRegionDirRecoveredEditsDir(regionStatus.getPath()); + byte[] regionName = Bytes.toBytes(regionStatus.getPath().getName()); + assertFalse(regionsMap.containsKey(regionName)); + for (FileStatus logStatus: FSUtils.listStatus(fs, regionEdits)) { + HLog.Reader reader = HLog.getReader(fs, logStatus.getPath(), conf); + try { + HLog.Entry entry; + while ((entry = reader.next()) != null) { + HLogKey key = entry.getKey(); + assertArrayEquals(tableName, key.getTablename()); + assertArrayEquals(regionName, key.getEncodedRegionName()); + } + } finally { + reader.close(); + } + } + } + } + + /* + * Write some entries in the log file. + * 7 different tables with name "testtb-%d" + * 10 region per table with name "tableName-region-%d" + * 50 entry with row key "row-%d" + */ + private void writeTestLog(final Path logFile) throws IOException { + fs.mkdirs(logFile.getParent()); + HLog.Writer writer = HLog.createWriter(fs, logFile, conf); + try { + for (int i = 0; i < 7; ++i) { + byte[] tableName = getTableName(i); + for (int j = 0; j < 10; ++j) { + byte[] regionName = getRegionName(tableName, j); + for (int k = 0; k < 50; ++k) { + byte[] rowkey = Bytes.toBytes("row-" + k); + HLogKey key = new HLogKey(regionName, tableName, (long)k, + System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID); + WALEdit edit = new WALEdit(); + edit.add(new KeyValue(rowkey, TEST_FAMILY, TEST_QUALIFIER, rowkey)); + writer.append(new HLog.Entry(key, edit)); + } + } + } + } finally { + writer.close(); + } + } + + private byte[] getTableName(int tableId) { + return Bytes.toBytes("testtb-" + tableId); + } + + private byte[] getRegionName(final byte[] tableName, int regionId) { + return Bytes.toBytes(Bytes.toString(tableName) + "-region-" + regionId); + } + + private byte[] getNewRegionName(final byte[] tableName, int regionId) { + return Bytes.toBytes(Bytes.toString(tableName) + "-new-region-" + regionId); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotTask.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotTask.java new file mode 100644 index 0000000..36b7050 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotTask.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.errorhandling.ForeignException; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.snapshot.SnapshotTask; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +@Category(SmallTests.class) +public class TestSnapshotTask { + + /** + * Check that errors from running the task get propagated back to the error listener. + */ + @Test + public void testErrorPropagation() throws Exception { + ForeignExceptionDispatcher error = mock(ForeignExceptionDispatcher.class); + SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot") + .setTable("table").build(); + final Exception thrown = new Exception("Failed!"); + SnapshotTask fail = new SnapshotTask(snapshot, error) { + @Override + public Void call() { + snapshotFailure("Injected failure", thrown); + return null; + } + }; + fail.call(); + + verify(error, Mockito.times(1)).receive(any(ForeignException.class)); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/snapshot/TestWALReferenceTask.java b/src/test/java/org/apache/hadoop/hbase/snapshot/TestWALReferenceTask.java new file mode 100644 index 0000000..a813ba7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/snapshot/TestWALReferenceTask.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.snapshot.ReferenceServerWALsTask; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test that the WAL reference task works as expected + */ +@Category(SmallTests.class) +public class TestWALReferenceTask { + + private static final Log LOG = LogFactory.getLog(TestWALReferenceTask.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @Test + public void testRun() throws IOException { + Configuration conf = UTIL.getConfiguration(); + FileSystem fs = UTIL.getTestFileSystem(); + // setup the log dir + Path testDir = UTIL.getDataTestDir(); + Set servers = new HashSet(); + Path logDir = new Path(testDir, ".logs"); + Path server1Dir = new Path(logDir, "Server1"); + servers.add(server1Dir.getName()); + Path server2Dir = new Path(logDir, "me.hbase.com,56073,1348618509968"); + servers.add(server2Dir.getName()); + // logs under server 1 + Path log1_1 = new Path(server1Dir, "me.hbase.com%2C56073%2C1348618509968.1348618520536"); + Path log1_2 = new Path(server1Dir, "me.hbase.com%2C56073%2C1348618509968.1234567890123"); + // logs under server 2 + Path log2_1 = new Path(server2Dir, "me.hbase.com%2C56074%2C1348618509998.1348618515589"); + Path log2_2 = new Path(server2Dir, "me.hbase.com%2C56073%2C1348618509968.1234567890123"); + + // create all the log files + fs.createNewFile(log1_1); + fs.createNewFile(log1_2); + fs.createNewFile(log2_1); + fs.createNewFile(log2_2); + + FSUtils.logFileSystemState(fs, testDir, LOG); + FSUtils.setRootDir(conf, testDir); + SnapshotDescription snapshot = SnapshotDescription.newBuilder() + .setName("testWALReferenceSnapshot").build(); + ForeignExceptionDispatcher listener = Mockito.mock(ForeignExceptionDispatcher.class); + + // reference all the files in the first server directory + ReferenceServerWALsTask task = new ReferenceServerWALsTask(snapshot, listener, server1Dir, + conf, fs); + task.call(); + + // reference all the files in the first server directory + task = new ReferenceServerWALsTask(snapshot, listener, server2Dir, conf, fs); + task.call(); + + // verify that we got everything + FSUtils.logFileSystemState(fs, testDir, LOG); + Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, testDir); + Path snapshotLogDir = new Path(workingDir, HConstants.HREGION_LOGDIR_NAME); + + // make sure we reference the all the wal files + TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logDir, servers, snapshot, snapshotLogDir); + + // make sure we never got an error + Mockito.verify(listener, Mockito.atLeastOnce()).rethrowException(); + Mockito.verifyNoMoreInteractions(listener); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java new file mode 100644 index 0000000..8f2f86e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -0,0 +1,1019 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.test; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.UUID; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.IntegrationTestingUtility; +import org.apache.hadoop.hbase.IntegrationTests; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.mapreduce.TableMapper; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.VLongWritable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.Counters; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * This is an integration test borrowed from goraci, written by Keith Turner, + * which is in turn inspired by the Accumulo test called continous ingest (ci). + * The original source code can be found here: + * https://github.com/keith-turner/goraci + * https://github.com/enis/goraci/ + * + * Apache Accumulo [0] has a simple test suite that verifies that data is not + * lost at scale. This test suite is called continuous ingest. This test runs + * many ingest clients that continually create linked lists containing 25 + * million nodes. At some point the clients are stopped and a map reduce job is + * run to ensure no linked list has a hole. A hole indicates data was lost.·· + * + * The nodes in the linked list are random. This causes each linked list to + * spread across the table. Therefore if one part of a table loses data, then it + * will be detected by references in another part of the table. + * + * THE ANATOMY OF THE TEST + * + * Below is rough sketch of how data is written. For specific details look at + * the Generator code. + * + * 1 Write out 1 million nodes· 2 Flush the client· 3 Write out 1 million that + * reference previous million· 4 If this is the 25th set of 1 million nodes, + * then update 1st set of million to point to last· 5 goto 1 + * + * The key is that nodes only reference flushed nodes. Therefore a node should + * never reference a missing node, even if the ingest client is killed at any + * point in time. + * + * When running this test suite w/ Accumulo there is a script running in + * parallel called the Aggitator that randomly and continuously kills server + * processes.·· The outcome was that many data loss bugs were found in Accumulo + * by doing this.· This test suite can also help find bugs that impact uptime + * and stability when· run for days or weeks.·· + * + * This test suite consists the following· - a few Java programs· - a little + * helper script to run the java programs - a maven script to build it.·· + * + * When generating data, its best to have each map task generate a multiple of + * 25 million. The reason for this is that circular linked list are generated + * every 25M. Not generating a multiple in 25M will result in some nodes in the + * linked list not having references. The loss of an unreferenced node can not + * be detected. + * + * + * Below is a description of the Java programs + * + * Generator - A map only job that generates data. As stated previously,· + * its best to generate data in multiples of 25M. + * + * Verify - A map reduce job that looks for holes. Look at the counts after running. REFERENCED and + * UNREFERENCED are· ok, any UNDEFINED counts are bad. Do not run at the· same + * time as the Generator. + * + * Walker - A standalong program that start following a linked list· and emits timing info.·· + * + * Print - A standalone program that prints nodes in the linked list + * + * Delete - A standalone program that deletes a single node + * + * This class can be run as a unit test, as an integration test, or from the command line + */ +@Category(IntegrationTests.class) +public class IntegrationTestBigLinkedList extends Configured implements Tool { + + private static final String TABLE_NAME_KEY = "IntegrationTestBigLinkedList.table"; + + private static final String DEFAULT_TABLE_NAME = "IntegrationTestBigLinkedList"; + + private static byte[] FAMILY_NAME = Bytes.toBytes("meta"); + + //link to the id of the prev node in the linked list + private static final byte[] COLUMN_PREV = Bytes.toBytes("prev"); + + //identifier of the mapred task that generated this row + private static final byte[] COLUMN_CLIENT = Bytes.toBytes("client"); + + //the id of the row within the same client. + private static final byte[] COLUMN_COUNT = Bytes.toBytes("count"); + + /** How many rows to write per map task. This has to be a multiple of 25M */ + private static final String GENERATOR_NUM_ROWS_PER_MAP_KEY + = "IntegrationTestBigLinkedList.generator.num_rows"; + + private static final String GENERATOR_NUM_MAPPERS_KEY + = "IntegrationTestBigLinkedList.generator.map.tasks"; + + static class CINode { + long key; + long prev; + String client; + long count; + } + + /** + * A Map only job that generates random linked list and stores them. + */ + static class Generator extends Configured implements Tool { + + private static final Log LOG = LogFactory.getLog(Generator.class); + + private static final int WIDTH = 1000000; + private static final int WRAP = WIDTH * 25; + + public static enum Counts { + UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT + } + + static class GeneratorInputFormat extends InputFormat { + static class GeneratorInputSplit extends InputSplit implements Writable { + @Override + public long getLength() throws IOException, InterruptedException { + return 1; + } + @Override + public String[] getLocations() throws IOException, InterruptedException { + return new String[0]; + } + @Override + public void readFields(DataInput arg0) throws IOException { + } + @Override + public void write(DataOutput arg0) throws IOException { + } + } + + static class GeneratorRecordReader extends RecordReader { + private long count; + private long numNodes; + private Random rand; + + @Override + public void close() throws IOException { + } + + @Override + public LongWritable getCurrentKey() throws IOException, InterruptedException { + return new LongWritable(Math.abs(rand.nextLong())); + } + + @Override + public NullWritable getCurrentValue() throws IOException, InterruptedException { + return NullWritable.get(); + } + + @Override + public float getProgress() throws IOException, InterruptedException { + return (float)(count / (double)numNodes); + } + + @Override + public void initialize(InputSplit arg0, TaskAttemptContext context) + throws IOException, InterruptedException { + numNodes = context.getConfiguration().getLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, 25000000); + rand = new Random(); + } + + @Override + public boolean nextKeyValue() throws IOException, InterruptedException { + return count++ < numNodes; + } + + } + + @Override + public RecordReader createRecordReader( + InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { + GeneratorRecordReader rr = new GeneratorRecordReader(); + rr.initialize(split, context); + return rr; + } + + @Override + public List getSplits(JobContext job) throws IOException, InterruptedException { + int numMappers = job.getConfiguration().getInt(GENERATOR_NUM_MAPPERS_KEY, 1); + + ArrayList splits = new ArrayList(numMappers); + + for (int i = 0; i < numMappers; i++) { + splits.add(new GeneratorInputSplit()); + } + + return splits; + } + } + + /** Ensure output files from prev-job go to map inputs for current job */ + static class OneFilePerMapperSFIF extends SequenceFileInputFormat { + @Override + protected boolean isSplitable(JobContext context, Path filename) { + return false; + } + } + + /** + * Some ASCII art time: + * [ . . . ] represents one batch of random longs of length WIDTH + * + * _________________________ + * | ______ | + * | | || + * __+_________________+_____ || + * v v v ||| + * first = [ . . . . . . . . . . . ] ||| + * ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ||| + * | | | | | | | | | | | ||| + * prev = [ . . . . . . . . . . . ] ||| + * ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ||| + * | | | | | | | | | | | ||| + * current = [ . . . . . . . . . . . ] ||| + * ||| + * ... ||| + * ||| + * last = [ . . . . . . . . . . . ] ||| + * | | | | | | | | | | |-----||| + * | |--------|| + * |___________________________| + */ + static class GeneratorMapper + extends Mapper { + Random rand = new Random(); + + long[] first = null; + long[] prev = null; + long[] current = new long[WIDTH]; + byte[] id; + long count = 0; + int i; + HTable table; + long numNodes; + long wrap = WRAP; + + protected void setup(Context context) throws IOException, InterruptedException { + id = Bytes.toBytes(UUID.randomUUID().toString()); + Configuration conf = context.getConfiguration(); + table = new HTable(conf, getTableName(conf)); + table.setAutoFlush(false); + table.setWriteBufferSize(4 * 1024 * 1024); + numNodes = context.getConfiguration().getLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, 25000000); + if (numNodes < 25000000) { + wrap = numNodes; + } + }; + + protected void cleanup(Context context) throws IOException ,InterruptedException { + table.close(); + }; + + @Override + protected void map(LongWritable key, NullWritable value, Context output) throws IOException { + current[i++] = Math.abs(key.get()); + + if (i == current.length) { + persist(output, count, prev, current, id); + i = 0; + + if (first == null) + first = current; + prev = current; + current = new long[WIDTH]; + + count += current.length; + output.setStatus("Count " + count); + + if (count % wrap == 0) { + // this block of code turns the 1 million linked list of length 25 into one giant + //circular linked list of 25 million + circularLeftShift(first); + + persist(output, -1, prev, first, null); + + first = null; + prev = null; + } + } + } + + private static void circularLeftShift(long[] first) { + long ez = first[0]; + for (int i = 0; i < first.length - 1; i++) + first[i] = first[i + 1]; + first[first.length - 1] = ez; + } + + private void persist(Context output, long count, long[] prev, long[] current, byte[] id) + throws IOException { + for (int i = 0; i < current.length; i++) { + Put put = new Put(Bytes.toBytes(current[i])); + put.add(FAMILY_NAME, COLUMN_PREV, Bytes.toBytes(prev == null ? -1 : prev[i])); + + if (count >= 0) { + put.add(FAMILY_NAME, COLUMN_COUNT, Bytes.toBytes(count + i)); + } + if (id != null) { + put.add(FAMILY_NAME, COLUMN_CLIENT, id); + } + table.put(put); + + if (i % 1000 == 0) { + // Tickle progress every so often else maprunner will think us hung + output.progress(); + } + } + + table.flushCommits(); + } + } + + @Override + public int run(String[] args) throws Exception { + if (args.length < 3) { + System.out.println("Usage : " + Generator.class.getSimpleName() + + " "); + System.out.println(" where should be a multiple of 25M"); + return 0; + } + + int numMappers = Integer.parseInt(args[0]); + long numNodes = Long.parseLong(args[1]); + Path tmpOutput = new Path(args[2]); + return run(numMappers, numNodes, tmpOutput); + } + + protected void createSchema() throws IOException { + HBaseAdmin admin = new HBaseAdmin(getConf()); + byte[] tableName = getTableName(getConf()); + if (!admin.tableExists(tableName)) { + HTableDescriptor htd = new HTableDescriptor(getTableName(getConf())); + htd.addFamily(new HColumnDescriptor(FAMILY_NAME)); + admin.createTable(htd); + } + admin.close(); + } + + public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput) + throws Exception { + LOG.info("Running RandomInputGenerator with numMappers=" + numMappers + + ", numNodes=" + numNodes); + Job job = new Job(getConf()); + + job.setJobName("Random Input Generator"); + job.setNumReduceTasks(0); + job.setJarByClass(getClass()); + + job.setInputFormatClass(GeneratorInputFormat.class); + job.setOutputKeyClass(LongWritable.class); + job.setOutputValueClass(NullWritable.class); + + job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers); + job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes); + + job.setMapperClass(Mapper.class); //identity mapper + + FileOutputFormat.setOutputPath(job, tmpOutput); + job.setOutputFormatClass(SequenceFileOutputFormat.class); + + boolean success = job.waitForCompletion(true); + + return success ? 0 : 1; + } + + public int runGenerator(int numMappers, long numNodes, Path tmpOutput) throws Exception { + LOG.info("Running Generator with numMappers=" + numMappers +", numNodes=" + numNodes); + createSchema(); + + Job job = new Job(getConf()); + + job.setJobName("Link Generator"); + job.setNumReduceTasks(0); + job.setJarByClass(getClass()); + + FileInputFormat.setInputPaths(job, tmpOutput); + job.setInputFormatClass(OneFilePerMapperSFIF.class); + job.setOutputKeyClass(NullWritable.class); + job.setOutputValueClass(NullWritable.class); + + job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers); + job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes); + + job.setMapperClass(GeneratorMapper.class); + + job.setOutputFormatClass(NullOutputFormat.class); + + job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false); + TableMapReduceUtil.addDependencyJars(job); + TableMapReduceUtil.initCredentials(job); + + boolean success = job.waitForCompletion(true); + + return success ? 0 : 1; + } + + public int run(int numMappers, long numNodes, Path tmpOutput) throws Exception { + int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput); + if (ret > 0) { + return ret; + } + + return runGenerator(numMappers, numNodes, tmpOutput); + } + } + + /** + * A Map Reduce job that verifies that the linked lists generated by + * {@link Generator} do not have any holes. + */ + static class Verify extends Configured implements Tool { + + private static final Log LOG = LogFactory.getLog(Verify.class); + private static final VLongWritable DEF = new VLongWritable(-1); + + private Job job; + + public static class VerifyMapper extends TableMapper { + private LongWritable row = new LongWritable(); + private LongWritable ref = new LongWritable(); + private VLongWritable vrow = new VLongWritable(); + + @Override + protected void map(ImmutableBytesWritable key, Result value, Context context) + throws IOException ,InterruptedException { + row.set(Bytes.toLong(key.get())); + context.write(row, DEF); + + long prev = Bytes.toLong(value.getValue(FAMILY_NAME, COLUMN_PREV)); + if (prev >= 0) { + ref.set(prev); + vrow.set(Bytes.toLong(key.get())); + context.write(ref, vrow); + } + } + } + + public static enum Counts { + UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT + } + + public static class VerifyReducer extends Reducer { + private ArrayList refs = new ArrayList(); + + public void reduce(LongWritable key, Iterable values, Context context) + throws IOException, InterruptedException { + + int defCount = 0; + + refs.clear(); + for (VLongWritable type : values) { + if (type.get() == -1) { + defCount++; + } else { + refs.add(type.get()); + } + } + + // TODO check for more than one def, should not happen + + if (defCount == 0 && refs.size() > 0) { + // this is bad, found a node that is referenced but not defined. It must have been + //lost, emit some info about this node for debugging purposes. + + StringBuilder sb = new StringBuilder(); + String comma = ""; + for (Long ref : refs) { + sb.append(comma); + comma = ","; + sb.append(String.format("%016x", ref)); + } + + context.write(new Text(String.format("%016x", key.get())), new Text(sb.toString())); + context.getCounter(Counts.UNDEFINED).increment(1); + + } else if (defCount > 0 && refs.size() == 0) { + // node is defined but not referenced + context.getCounter(Counts.UNREFERENCED).increment(1); + } else { + // node is defined and referenced + context.getCounter(Counts.REFERENCED).increment(1); + } + + } + } + + @Override + public int run(String[] args) throws Exception { + + if (args.length != 2) { + System.out.println("Usage : " + Verify.class.getSimpleName() + " "); + return 0; + } + + String outputDir = args[0]; + int numReducers = Integer.parseInt(args[1]); + + return run(outputDir, numReducers); + } + + public int run(String outputDir, int numReducers) throws Exception { + return run(new Path(outputDir), numReducers); + } + + public int run(Path outputDir, int numReducers) throws Exception { + LOG.info("Running Verify with outputDir=" + outputDir +", numReducers=" + numReducers); + + job = new Job(getConf()); + + job.setJobName("Link Verifier"); + job.setNumReduceTasks(numReducers); + job.setJarByClass(getClass()); + + Scan scan = new Scan(); + scan.addColumn(FAMILY_NAME, COLUMN_PREV); + scan.setCaching(10000); + scan.setCacheBlocks(false); + + TableMapReduceUtil.initTableMapperJob(getTableName(getConf()), scan, + VerifyMapper.class, LongWritable.class, VLongWritable.class, job); + + job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false); + + job.setReducerClass(VerifyReducer.class); + job.setOutputFormatClass(TextOutputFormat.class); + TextOutputFormat.setOutputPath(job, outputDir); + + boolean success = job.waitForCompletion(true); + + return success ? 0 : 1; + } + + public boolean verify(long expectedReferenced) throws Exception { + if (job == null) { + throw new IllegalStateException("You should call run() first"); + } + + Counters counters = job.getCounters(); + + Counter referenced = counters.findCounter(Counts.REFERENCED); + Counter unreferenced = counters.findCounter(Counts.UNREFERENCED); + Counter undefined = counters.findCounter(Counts.UNDEFINED); + + boolean success = true; + //assert + if (expectedReferenced != referenced.getValue()) { + LOG.error("Expected referenced count does not match with actual referenced count. " + + "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue()); + success = false; + } + + if (unreferenced.getValue() > 0) { + LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()); + success = false; + } + + if (undefined.getValue() > 0) { + LOG.error("Found an undefined node. Undefined count=" + undefined.getValue()); + success = false; + } + + return success; + } + } + + /** + * Executes Generate and Verify in a loop. Data is not cleaned between runs, so each iteration + * adds more data. + */ + private static class Loop extends Configured implements Tool { + + private static final Log LOG = LogFactory.getLog(Loop.class); + + protected void runGenerator(int numMappers, long numNodes, String outputDir) throws Exception { + Path outputPath = new Path(outputDir); + UUID uuid = UUID.randomUUID(); //create a random UUID. + Path generatorOutput = new Path(outputPath, uuid.toString()); + + Generator generator = new Generator(); + generator.setConf(getConf()); + int retCode = generator.run(numMappers, numNodes, generatorOutput); + + if (retCode > 0) { + throw new RuntimeException("Generator failed with return code: " + retCode); + } + } + + protected void runVerify(String outputDir, int numReducers, long expectedNumNodes) throws Exception { + Path outputPath = new Path(outputDir); + UUID uuid = UUID.randomUUID(); //create a random UUID. + Path iterationOutput = new Path(outputPath, uuid.toString()); + + Verify verify = new Verify(); + verify.setConf(getConf()); + int retCode = verify.run(iterationOutput, numReducers); + if (retCode > 0) { + throw new RuntimeException("Verify.run failed with return code: " + retCode); + } + + boolean verifySuccess = verify.verify(expectedNumNodes); + if (!verifySuccess) { + throw new RuntimeException("Verify.verify failed"); + } + + LOG.info("Verify finished with succees. Total nodes=" + expectedNumNodes); + } + + @Override + public int run(String[] args) throws Exception { + if (args.length < 5) { + System.err.println("Usage: Loop "); + return 1; + } + + LOG.info("Running Loop with args:" + Arrays.deepToString(args)); + + int numIterations = Integer.parseInt(args[0]); + int numMappers = Integer.parseInt(args[1]); + long numNodes = Long.parseLong(args[2]); + String outputDir = args[3]; + int numReducers = Integer.parseInt(args[4]); + + long expectedNumNodes = 0; + + if (numIterations < 0) { + numIterations = Integer.MAX_VALUE; //run indefinitely (kind of) + } + + for (int i=0; i < numIterations; i++) { + LOG.info("Starting iteration = " + i); + runGenerator(numMappers, numNodes, outputDir); + expectedNumNodes += numMappers * numNodes; + + runVerify(outputDir, numReducers, expectedNumNodes); + } + + return 0; + } + } + + /** + * A stand alone program that prints out portions of a list created by {@link Generator} + */ + private static class Print extends Configured implements Tool { + public int run(String[] args) throws Exception { + Options options = new Options(); + options.addOption("s", "start", true, "start key"); + options.addOption("e", "end", true, "end key"); + options.addOption("l", "limit", true, "number to print"); + + GnuParser parser = new GnuParser(); + CommandLine cmd = null; + try { + cmd = parser.parse(options, args); + if (cmd.getArgs().length != 0) { + throw new ParseException("Command takes no arguments"); + } + } catch (ParseException e) { + System.err.println("Failed to parse command line " + e.getMessage()); + System.err.println(); + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp(getClass().getSimpleName(), options); + System.exit(-1); + } + + HTable table = new HTable(getConf(), getTableName(getConf())); + + Scan scan = new Scan(); + scan.setBatch(10000); + + if (cmd.hasOption("s")) + scan.setStartRow(Bytes.toBytes(new BigInteger(cmd.getOptionValue("s"), 16).longValue())); + + if (cmd.hasOption("e")) + scan.setStopRow(Bytes.toBytes(new BigInteger(cmd.getOptionValue("e"), 16).longValue())); + + int limit = 0; + if (cmd.hasOption("l")) + limit = Integer.parseInt(cmd.getOptionValue("l")); + else + limit = 100; + + ResultScanner scanner = table.getScanner(scan); + + CINode node = new CINode(); + Result result = scanner.next(); + int count = 0; + while (result != null && count++ < limit) { + node = getCINode(result, node); + System.out.printf("%016x:%016x:%012d:%s\n", node.key, node.prev, node.count, node.client); + result = scanner.next(); + } + scanner.close(); + table.close(); + + return 0; + } + } + + /** + * A stand alone program that deletes a single node. + */ + private static class Delete extends Configured implements Tool { + public int run(String[] args) throws Exception { + if (args.length != 1) { + System.out.println("Usage : " + Delete.class.getSimpleName() + " "); + return 0; + } + long val = new BigInteger(args[0], 16).longValue(); + + org.apache.hadoop.hbase.client.Delete delete + = new org.apache.hadoop.hbase.client.Delete(Bytes.toBytes(val)); + + HTable table = new HTable(getConf(), getTableName(getConf())); + + table.delete(delete); + table.flushCommits(); + table.close(); + + System.out.println("Delete successful"); + return 0; + } + } + + /** + * A stand alone program that follows a linked list created by {@link Generator} and prints timing info. + */ + private static class Walker extends Configured implements Tool { + public int run(String[] args) throws IOException { + Options options = new Options(); + options.addOption("n", "num", true, "number of queries"); + + GnuParser parser = new GnuParser(); + CommandLine cmd = null; + try { + cmd = parser.parse(options, args); + if (cmd.getArgs().length != 0) { + throw new ParseException("Command takes no arguments"); + } + } catch (ParseException e) { + System.err.println("Failed to parse command line " + e.getMessage()); + System.err.println(); + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp(getClass().getSimpleName(), options); + System.exit(-1); + } + + long maxQueries = Long.MAX_VALUE; + if (cmd.hasOption('n')) { + maxQueries = Long.parseLong(cmd.getOptionValue("n")); + } + + HTable table = new HTable(getConf(), getTableName(getConf())); + + Random rand = new Random(); + + long numQueries = 0; + + while (numQueries < maxQueries) { + CINode node = findStartNode(rand, table); + numQueries++; + while (node != null && node.prev >= 0 && numQueries < maxQueries) { + long prev = node.prev; + + long t1 = System.currentTimeMillis(); + node = getNode(prev, table, node); + long t2 = System.currentTimeMillis(); + System.out.printf("CQ %d %016x \n", t2 - t1, prev); //cold cache + numQueries++; + + t1 = System.currentTimeMillis(); + node = getNode(prev, table, node); + t2 = System.currentTimeMillis(); + System.out.printf("HQ %d %016x \n", t2 - t1, prev); //hot cache + numQueries++; + } + } + + table.close(); + return 0; + } + + private static CINode findStartNode(Random rand, HTable table) throws IOException { + Scan scan = new Scan(); + scan.setStartRow(Bytes.toBytes(Math.abs(rand.nextLong()))); + scan.setBatch(1); + scan.addColumn(FAMILY_NAME, COLUMN_PREV); + + long t1 = System.currentTimeMillis(); + ResultScanner scanner = table.getScanner(scan); + Result result = scanner.next(); + long t2 = System.currentTimeMillis(); + scanner.close(); + + if ( result != null) { + CINode node = getCINode(result, new CINode()); + System.out.printf("FSR %d %016x\n", t2 - t1, node.key); + return node; + } + + System.out.println("FSR " + (t2 - t1)); + + return null; + } + + private CINode getNode(long row, HTable table, CINode node) throws IOException { + Get get = new Get(Bytes.toBytes(row)); + get.addColumn(FAMILY_NAME, COLUMN_PREV); + Result result = table.get(get); + return getCINode(result, node); + } + } + + private static byte[] getTableName(Configuration conf) { + return Bytes.toBytes(conf.get(TABLE_NAME_KEY, DEFAULT_TABLE_NAME)); + } + + private static CINode getCINode(Result result, CINode node) { + node.key = Bytes.toLong(result.getRow()); + if (result.containsColumn(FAMILY_NAME, COLUMN_PREV)) { + node.prev = Bytes.toLong(result.getValue(FAMILY_NAME, COLUMN_PREV)); + } else { + node.prev = -1; + } + if (result.containsColumn(FAMILY_NAME, COLUMN_COUNT)) { + node.count = Bytes.toLong(result.getValue(FAMILY_NAME, COLUMN_COUNT)); + } else { + node.count = -1; + } + if (result.containsColumn(FAMILY_NAME, COLUMN_CLIENT)) { + node.client = Bytes.toString(result.getValue(FAMILY_NAME, COLUMN_CLIENT)); + } else { + node.client = ""; + } + return node; + } + + private IntegrationTestingUtility util; + + @Before + public void setUp() throws Exception { + util = getTestingUtil(); + util.initializeCluster(3); + this.setConf(util.getConfiguration()); + } + + @After + public void tearDown() throws Exception { + util.restoreCluster(); + } + + @Test + public void testContinuousIngest() throws IOException, Exception { + //Loop + int ret = ToolRunner.run(getTestingUtil().getConfiguration(), new Loop(), + new String[] {"1", "1", "2000000", + getTestDir("IntegrationTestBigLinkedList", "testContinuousIngest").toString(), "1"}); + org.junit.Assert.assertEquals(0, ret); + } + + public Path getTestDir(String testName, String subdir) throws IOException { + //HBaseTestingUtility.getDataTestDirOnTestFs() has not been backported. + FileSystem fs = FileSystem.get(getConf()); + Path base = new Path(fs.getWorkingDirectory(), "test-data"); + String randomStr = UUID.randomUUID().toString(); + Path testDir = new Path(base, randomStr); + fs.deleteOnExit(testDir); + + return new Path(new Path(testDir, testName), subdir); + } + + private IntegrationTestingUtility getTestingUtil() { + if (this.util == null) { + if (getConf() == null) { + this.util = new IntegrationTestingUtility(); + } else { + this.util = new IntegrationTestingUtility(getConf()); + } + } + return util; + } + + private int printUsage() { + System.err.println("Usage: " + this.getClass().getSimpleName() + " COMMAND [COMMAND options]"); + System.err.println(" where COMMAND is one of:"); + System.err.println(""); + System.err.println(" Generator A map only job that generates data."); + System.err.println(" Verify A map reduce job that looks for holes"); + System.err.println(" Look at the counts after running"); + System.err.println(" REFERENCED and UNREFERENCED are ok"); + System.err.println(" any UNDEFINED counts are bad. Do not"); + System.err.println(" run at the same time as the Generator."); + System.err.println(" Walker A standalong program that starts "); + System.err.println(" following a linked list and emits"); + System.err.println(" timing info."); + System.err.println(" Print A standalone program that prints nodes"); + System.err.println(" in the linked list."); + System.err.println(" Delete A standalone program that deletes a·"); + System.err.println(" single node."); + System.err.println(" Loop A program to Loop through Generator and"); + System.err.println(" Verify steps"); + System.err.println("\t "); + return 1; + } + + @Override + public int run(String[] args) throws Exception { + //get the class, run with the conf + if (args.length < 1) { + return printUsage(); + } + Tool tool = null; + if (args[0].equals("Generator")) { + tool = new Generator(); + } else if (args[0].equals("Verify")) { + tool = new Verify(); + } else if (args[0].equals("Loop")) { + tool = new Loop(); + } else if (args[0].equals("Walker")) { + tool = new Walker(); + } else if (args[0].equals("Print")) { + tool = new Print(); + } else if (args[0].equals("Delete")) { + tool = new Delete(); + } else { + return printUsage(); + } + + args = Arrays.copyOfRange(args, 1, args.length); + return ToolRunner.run(getConf(), tool, args); + } + + public static void main(String[] args) throws Exception { + int ret = ToolRunner.run(HBaseConfiguration.create(), new IntegrationTestBigLinkedList(), args); + System.exit(ret); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java new file mode 100644 index 0000000..3bf7f24 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java @@ -0,0 +1,460 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Random; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.IntegrationTestingUtility; +import org.apache.hadoop.hbase.IntegrationTests; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.NMapInputFormat; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.mapreduce.TableMapper; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + +/** + * A large test which loads a lot of data that has internal references, and + * verifies the data. + * + * In load step, 200 map tasks are launched, which in turn write loadmapper.num_to_write + * (default 100K) rows to an hbase table. Rows are written in blocks, for a total of + * 100 blocks. Each row in a block, contains loadmapper.backrefs (default 50) references + * to random rows in the prev block. + * + * Verify step is scans the table, and verifies that for every referenced row, the row is + * actually there (no data loss). Failed rows are output from reduce to be saved in the + * job output dir in hdfs and inspected later. + * + * This class can be run as a unit test, as an integration test, or from the command line + * + * Originally taken from Apache Bigtop. + */ +@Category(IntegrationTests.class) +public class IntegrationTestLoadAndVerify extends Configured implements Tool { + private static final String TEST_NAME = "IntegrationTestLoadAndVerify"; + private static final byte[] TEST_FAMILY = Bytes.toBytes("f1"); + private static final byte[] TEST_QUALIFIER = Bytes.toBytes("q1"); + + private static final String NUM_TO_WRITE_KEY = + "loadmapper.num_to_write"; + private static final long NUM_TO_WRITE_DEFAULT = 100*1000; + + private static final String TABLE_NAME_KEY = "loadmapper.table"; + private static final String TABLE_NAME_DEFAULT = "table"; + + private static final String NUM_BACKREFS_KEY = "loadmapper.backrefs"; + private static final int NUM_BACKREFS_DEFAULT = 50; + + private static final String NUM_MAP_TASKS_KEY = "loadmapper.map.tasks"; + private static final String NUM_REDUCE_TASKS_KEY = "verify.reduce.tasks"; + private static final int NUM_MAP_TASKS_DEFAULT = 200; + private static final int NUM_REDUCE_TASKS_DEFAULT = 35; + + private static final int SCANNER_CACHING = 500; + + private IntegrationTestingUtility util; + + private enum Counters { + ROWS_WRITTEN, + REFERENCES_WRITTEN, + REFERENCES_CHECKED; + } + + @Before + public void setUp() throws Exception { + util = getTestingUtil(); + util.initializeCluster(3); + this.setConf(util.getConfiguration()); + getConf().setLong(NUM_TO_WRITE_KEY, NUM_TO_WRITE_DEFAULT / 100); + getConf().setInt(NUM_MAP_TASKS_KEY, NUM_MAP_TASKS_DEFAULT / 100); + getConf().setInt(NUM_REDUCE_TASKS_KEY, NUM_REDUCE_TASKS_DEFAULT / 10); + } + + @After + public void tearDown() throws Exception { + util.restoreCluster(); + } + + /** + * Converts a "long" value between endian systems. + * Borrowed from Apache Commons IO + * @param value value to convert + * @return the converted value + */ + public static long swapLong(long value) + { + return + ( ( ( value >> 0 ) & 0xff ) << 56 ) + + ( ( ( value >> 8 ) & 0xff ) << 48 ) + + ( ( ( value >> 16 ) & 0xff ) << 40 ) + + ( ( ( value >> 24 ) & 0xff ) << 32 ) + + ( ( ( value >> 32 ) & 0xff ) << 24 ) + + ( ( ( value >> 40 ) & 0xff ) << 16 ) + + ( ( ( value >> 48 ) & 0xff ) << 8 ) + + ( ( ( value >> 56 ) & 0xff ) << 0 ); + } + + public static class LoadMapper + extends Mapper + { + private long recordsToWrite; + private HTable table; + private Configuration conf; + private int numBackReferencesPerRow; + private String shortTaskId; + + private Random rand = new Random(); + + private Counter rowsWritten, refsWritten; + + @Override + public void setup(Context context) throws IOException { + conf = context.getConfiguration(); + recordsToWrite = conf.getLong(NUM_TO_WRITE_KEY, NUM_TO_WRITE_DEFAULT); + String tableName = conf.get(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); + numBackReferencesPerRow = conf.getInt(NUM_BACKREFS_KEY, NUM_BACKREFS_DEFAULT); + table = new HTable(conf, tableName); + table.setWriteBufferSize(4*1024*1024); + table.setAutoFlush(false); + + String taskId = conf.get("mapred.task.id"); + Matcher matcher = Pattern.compile(".+_m_(\\d+_\\d+)").matcher(taskId); + if (!matcher.matches()) { + throw new RuntimeException("Strange task ID: " + taskId); + } + shortTaskId = matcher.group(1); + + rowsWritten = context.getCounter(Counters.ROWS_WRITTEN); + refsWritten = context.getCounter(Counters.REFERENCES_WRITTEN); + } + + @Override + public void cleanup(Context context) throws IOException { + table.flushCommits(); + table.close(); + } + + @Override + protected void map(NullWritable key, NullWritable value, + Context context) throws IOException, InterruptedException { + + String suffix = "/" + shortTaskId; + byte[] row = Bytes.add(new byte[8], Bytes.toBytes(suffix)); + + int BLOCK_SIZE = (int)(recordsToWrite / 100); + + for (long i = 0; i < recordsToWrite;) { + long blockStart = i; + for (long idxInBlock = 0; + idxInBlock < BLOCK_SIZE && i < recordsToWrite; + idxInBlock++, i++) { + + long byteSwapped = swapLong(i); + Bytes.putLong(row, 0, byteSwapped); + + Put p = new Put(row); + p.add(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY); + if (blockStart > 0) { + for (int j = 0; j < numBackReferencesPerRow; j++) { + long referredRow = blockStart - BLOCK_SIZE + rand.nextInt(BLOCK_SIZE); + Bytes.putLong(row, 0, swapLong(referredRow)); + p.add(TEST_FAMILY, row, HConstants.EMPTY_BYTE_ARRAY); + } + refsWritten.increment(1); + } + rowsWritten.increment(1); + table.put(p); + + if (i % 100 == 0) { + context.setStatus("Written " + i + "/" + recordsToWrite + " records"); + context.progress(); + } + } + // End of block, flush all of them before we start writing anything + // pointing to these! + table.flushCommits(); + } + } + } + + public static class VerifyMapper extends TableMapper { + static final BytesWritable EMPTY = new BytesWritable(HConstants.EMPTY_BYTE_ARRAY); + + @Override + protected void map(ImmutableBytesWritable key, Result value, Context context) + throws IOException, InterruptedException { + BytesWritable bwKey = new BytesWritable(key.get()); + BytesWritable bwVal = new BytesWritable(); + for (KeyValue kv : value.list()) { + if (Bytes.compareTo(TEST_QUALIFIER, 0, TEST_QUALIFIER.length, + kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength()) == 0) { + context.write(bwKey, EMPTY); + } else { + bwVal.set(kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength()); + context.write(bwVal, bwKey); + } + } + } + } + + public static class VerifyReducer extends Reducer { + private Counter refsChecked; + private Counter rowsWritten; + + @Override + public void setup(Context context) throws IOException { + refsChecked = context.getCounter(Counters.REFERENCES_CHECKED); + rowsWritten = context.getCounter(Counters.ROWS_WRITTEN); + } + + @Override + protected void reduce(BytesWritable referredRow, Iterable referrers, + VerifyReducer.Context ctx) throws IOException, InterruptedException { + boolean gotOriginalRow = false; + int refCount = 0; + + for (BytesWritable ref : referrers) { + if (ref.getLength() == 0) { + assert !gotOriginalRow; + gotOriginalRow = true; + } else { + refCount++; + } + } + refsChecked.increment(refCount); + + if (!gotOriginalRow) { + String parsedRow = makeRowReadable(referredRow.getBytes(), referredRow.getLength()); + String binRow = Bytes.toStringBinary(referredRow.getBytes(), 0, referredRow.getLength()); + ctx.write(new Text(binRow), new Text(parsedRow)); + rowsWritten.increment(1); + } + } + + private String makeRowReadable(byte[] bytes, int length) { + long rowIdx = swapLong(Bytes.toLong(bytes, 0)); + String suffix = Bytes.toString(bytes, 8, length - 8); + + return "Row #" + rowIdx + " suffix " + suffix; + } + } + + private void doLoad(Configuration conf, HTableDescriptor htd) throws Exception { + Path outputDir = getTestDir(TEST_NAME, "load-output"); + + NMapInputFormat.setNumMapTasks(conf, conf.getInt(NUM_MAP_TASKS_KEY, NUM_MAP_TASKS_DEFAULT)); + conf.set(TABLE_NAME_KEY, htd.getNameAsString()); + + Job job = new Job(conf); + job.setJobName(TEST_NAME + " Load for " + htd.getNameAsString()); + job.setJarByClass(this.getClass()); + job.setMapperClass(LoadMapper.class); + job.setInputFormatClass(NMapInputFormat.class); + job.setNumReduceTasks(0); + FileOutputFormat.setOutputPath(job, outputDir); + + TableMapReduceUtil.addDependencyJars(job); + TableMapReduceUtil.addDependencyJars( + job.getConfiguration(), HTable.class, Lists.class); + TableMapReduceUtil.initCredentials(job); + assertTrue(job.waitForCompletion(true)); + } + + private void doVerify(Configuration conf, HTableDescriptor htd) throws Exception { + Path outputDir = getTestDir(TEST_NAME, "verify-output"); + + Job job = new Job(conf); + job.setJarByClass(this.getClass()); + job.setJobName(TEST_NAME + " Verification for " + htd.getNameAsString()); + + Scan scan = new Scan(); + + TableMapReduceUtil.initTableMapperJob( + htd.getNameAsString(), scan, VerifyMapper.class, + BytesWritable.class, BytesWritable.class, job); + int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING); + TableMapReduceUtil.setScannerCaching(job, scannerCaching); + + job.setReducerClass(VerifyReducer.class); + job.setNumReduceTasks(conf.getInt(NUM_REDUCE_TASKS_KEY, NUM_REDUCE_TASKS_DEFAULT)); + FileOutputFormat.setOutputPath(job, outputDir); + assertTrue(job.waitForCompletion(true)); + + long numOutputRecords = job.getCounters().findCounter(Counters.ROWS_WRITTEN).getValue(); + assertEquals(0, numOutputRecords); + } + + public Path getTestDir(String testName, String subdir) throws IOException { + //HBaseTestingUtility.getDataTestDirOnTestFs() has not been backported. + FileSystem fs = FileSystem.get(getConf()); + Path base = new Path(fs.getWorkingDirectory(), "test-data"); + String randomStr = UUID.randomUUID().toString(); + Path testDir = new Path(base, randomStr); + fs.deleteOnExit(testDir); + + return new Path(new Path(testDir, testName), subdir); + } + + @Test + public void testLoadAndVerify() throws Exception { + HTableDescriptor htd = new HTableDescriptor(TEST_NAME); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + + HBaseAdmin admin = getTestingUtil().getHBaseAdmin(); + int numPreCreate = 40; + admin.createTable(htd, Bytes.toBytes(0L), Bytes.toBytes(-1L), numPreCreate); + + doLoad(getConf(), htd); + doVerify(getConf(), htd); + + // Only disable and drop if we succeeded to verify - otherwise it's useful + // to leave it around for post-mortem + deleteTable(admin, htd); + } + + private void deleteTable(HBaseAdmin admin, HTableDescriptor htd) + throws IOException, InterruptedException { + // Use disableTestAsync because disable can take a long time to complete + System.out.print("Disabling table " + htd.getNameAsString() +" "); + admin.disableTableAsync(htd.getName()); + + long start = System.currentTimeMillis(); + // NOTE tables can be both admin.isTableEnabled=false and + // isTableDisabled=false, when disabling must use isTableDisabled! + while (!admin.isTableDisabled(htd.getName())) { + System.out.print("."); + Thread.sleep(1000); + } + long delta = System.currentTimeMillis() - start; + System.out.println(" " + delta +" ms"); + System.out.println("Deleting table " + htd.getNameAsString() +" "); + admin.deleteTable(htd.getName()); + } + + public void usage() { + System.err.println(this.getClass().getSimpleName() + " [-Doptions] "); + System.err.println(" Loads a table with row dependencies and verifies the dependency chains"); + System.err.println("Options"); + System.err.println(" -Dloadmapper.table= Table to write/verify (default autogen)"); + System.err.println(" -Dloadmapper.backrefs= Number of backreferences per row (default 50)"); + System.err.println(" -Dloadmapper.num_to_write= Number of rows per mapper (default 100,000 per mapper)"); + System.err.println(" -Dloadmapper.deleteAfter= Delete after a successful verify (default true)"); + System.err.println(" -Dloadmapper.numPresplits= Number of presplit regions to start with (default 40)"); + System.err.println(" -Dloadmapper.map.tasks= Number of map tasks for load (default 200)"); + System.err.println(" -Dverify.reduce.tasks= Number of reduce tasks for verify (default 35)"); + System.err.println(" -Dverify.scannercaching= Number hbase scanner caching rows to read (default 50)"); + } + + public int run(String argv[]) throws Exception { + if (argv.length < 1 || argv.length > 1) { + usage(); + return 1; + } + + IntegrationTestingUtility.setUseDistributedCluster(getConf()); + boolean doLoad = false; + boolean doVerify = false; + boolean doDelete = getConf().getBoolean("loadmapper.deleteAfter",true); + int numPresplits = getConf().getInt("loadmapper.numPresplits", 40); + + if (argv[0].equals("load")) { + doLoad = true; + } else if (argv[0].equals("verify")) { + doVerify= true; + } else if (argv[0].equals("loadAndVerify")) { + doLoad=true; + doVerify= true; + } else { + System.err.println("Invalid argument " + argv[0]); + usage(); + return 1; + } + + // create HTableDescriptor for specified table + String table = getConf().get(TABLE_NAME_KEY, TEST_NAME); + HTableDescriptor htd = new HTableDescriptor(table); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); + + HBaseAdmin admin = new HBaseAdmin(getConf()); + if (doLoad) { + admin.createTable(htd, Bytes.toBytes(0L), Bytes.toBytes(-1L), numPresplits); + doLoad(getConf(), htd); + } + if (doVerify) { + doVerify(getConf(), htd); + if (doDelete) { + deleteTable(admin, htd); + } + } + return 0; + } + + private IntegrationTestingUtility getTestingUtil() { + if (this.util == null) { + if (getConf() == null) { + this.util = new IntegrationTestingUtility(); + } else { + this.util = new IntegrationTestingUtility(getConf()); + } + } + return util; + } + + public static void main(String argv[]) throws Exception { + Configuration conf = HBaseConfiguration.create(); + int ret = ToolRunner.run(conf, new IntegrationTestLoadAndVerify(), argv); + System.exit(ret); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestCallQueue.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestCallQueue.java new file mode 100644 index 0000000..1965b05 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestCallQueue.java @@ -0,0 +1,144 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.thrift; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.LinkedBlockingQueue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.thrift.CallQueue.Call; +import org.apache.hadoop.hbase.thrift.generated.Hbase; +import org.apache.hadoop.metrics.ContextFactory; +import org.apache.hadoop.metrics.MetricsContext; +import org.apache.hadoop.metrics.MetricsUtil; +import org.apache.hadoop.metrics.spi.NoEmitMetricsContext; +import org.apache.hadoop.metrics.spi.OutputRecord; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import org.junit.Test; + +/** + * Unit testing for CallQueue, a part of the + * org.apache.hadoop.hbase.thrift package. + */ +@Category(SmallTests.class) +@RunWith(Parameterized.class) +public class TestCallQueue { + + public static final Log LOG = LogFactory.getLog(TestCallQueue.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private int elementsAdded; + private int elementsRemoved; + + @Parameters + public static Collection getParameters() { + Collection parameters = new ArrayList(); + for (int elementsAdded : new int[] {100, 200, 300}) { + for (int elementsRemoved : new int[] {0, 20, 100}) { + parameters.add(new Object[]{new Integer(elementsAdded), + new Integer(elementsRemoved)}); + } + } + return parameters; + } + + public TestCallQueue(int elementsAdded, int elementsRemoved) { + this.elementsAdded = elementsAdded; + this.elementsRemoved = elementsRemoved; + LOG.debug("elementsAdded:" + elementsAdded + + " elementsRemoved:" + elementsRemoved); + } + + @Test(timeout=60000) + public void testPutTake() throws Exception { + ThriftMetrics metrics = createMetrics(); + CallQueue callQueue = new CallQueue( + new LinkedBlockingQueue(), metrics); + for (int i = 0; i < elementsAdded; ++i) { + callQueue.put(createDummyRunnable()); + } + for (int i = 0; i < elementsRemoved; ++i) { + callQueue.take(); + } + verifyMetrics(metrics, "timeInQueue_num_ops", elementsRemoved); + } + + @Test(timeout=60000) + public void testOfferPoll() throws Exception { + ThriftMetrics metrics = createMetrics(); + CallQueue callQueue = new CallQueue( + new LinkedBlockingQueue(), metrics); + for (int i = 0; i < elementsAdded; ++i) { + callQueue.offer(createDummyRunnable()); + } + for (int i = 0; i < elementsRemoved; ++i) { + callQueue.poll(); + } + verifyMetrics(metrics, "timeInQueue_num_ops", elementsRemoved); + } + + private static ThriftMetrics createMetrics() throws Exception { + setupMetricsContext(); + Configuration conf = UTIL.getConfiguration(); + return new ThriftMetrics( + ThriftServerRunner.DEFAULT_LISTEN_PORT, conf, Hbase.Iface.class); + } + + private static void setupMetricsContext() throws Exception { + ContextFactory factory = ContextFactory.getFactory(); + factory.setAttribute(ThriftMetrics.CONTEXT_NAME + ".class", + NoEmitMetricsContext.class.getName()); + MetricsUtil.getContext(ThriftMetrics.CONTEXT_NAME) + .createRecord(ThriftMetrics.CONTEXT_NAME).remove(); + } + + private static void verifyMetrics(ThriftMetrics metrics, String name, int expectValue) + throws Exception { + MetricsContext context = MetricsUtil.getContext( + ThriftMetrics.CONTEXT_NAME); + metrics.doUpdates(context); + OutputRecord record = context.getAllRecords().get( + ThriftMetrics.CONTEXT_NAME).iterator().next(); + assertEquals(expectValue, record.getMetric(name).intValue()); + } + + private static Runnable createDummyRunnable() { + return new Runnable() { + @Override + public void run() { + } + }; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java new file mode 100644 index 0000000..81409c4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java @@ -0,0 +1,604 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.thrift; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.filter.ParseFilter; +import org.apache.hadoop.hbase.thrift.generated.BatchMutation; +import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor; +import org.apache.hadoop.hbase.thrift.generated.Hbase; +import org.apache.hadoop.hbase.thrift.generated.Mutation; +import org.apache.hadoop.hbase.thrift.generated.TCell; +import org.apache.hadoop.hbase.thrift.generated.TRegionInfo; +import org.apache.hadoop.hbase.thrift.generated.TRowResult; +import org.apache.hadoop.hbase.thrift.generated.TIncrement; +import org.apache.hadoop.hbase.thrift.ThriftServerRunner.HBaseHandler; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.metrics.ContextFactory; +import org.apache.hadoop.metrics.MetricsContext; +import org.apache.hadoop.metrics.MetricsUtil; +import org.apache.hadoop.metrics.spi.NoEmitMetricsContext; +import org.apache.hadoop.metrics.spi.OutputRecord; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Unit testing for ThriftServerRunner.HBaseHandler, a part of the + * org.apache.hadoop.hbase.thrift package. + */ +@Category(MediumTests.class) +public class TestThriftServer { + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final Log LOG = LogFactory.getLog(TestThriftServer.class); + protected static final int MAXVERSIONS = 3; + + private static ByteBuffer asByteBuffer(String i) { + return ByteBuffer.wrap(Bytes.toBytes(i)); + } + private static ByteBuffer asByteBuffer(long l) { + return ByteBuffer.wrap(Bytes.toBytes(l)); + } + + // Static names for tables, columns, rows, and values + private static ByteBuffer tableAname = asByteBuffer("tableA"); + private static ByteBuffer tableBname = asByteBuffer("tableB"); + private static ByteBuffer columnAname = asByteBuffer("columnA:"); + private static ByteBuffer columnAAname = asByteBuffer("columnA:A"); + private static ByteBuffer columnBname = asByteBuffer("columnB:"); + private static ByteBuffer rowAname = asByteBuffer("rowA"); + private static ByteBuffer rowBname = asByteBuffer("rowB"); + private static ByteBuffer valueAname = asByteBuffer("valueA"); + private static ByteBuffer valueBname = asByteBuffer("valueB"); + private static ByteBuffer valueCname = asByteBuffer("valueC"); + private static ByteBuffer valueDname = asByteBuffer("valueD"); + private static ByteBuffer valueEname = asByteBuffer(100l); + + @BeforeClass + public static void beforeClass() throws Exception { + UTIL.getConfiguration().setBoolean(ThriftServerRunner.COALESCE_INC_KEY, true); + UTIL.startMiniCluster(); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + /** + * Runs all of the tests under a single JUnit test method. We + * consolidate all testing to one method because HBaseClusterTestCase + * is prone to OutOfMemoryExceptions when there are three or more + * JUnit test methods. + * + * @throws Exception + */ + @Test + public void testAll() throws Exception { + // Run all tests + doTestTableCreateDrop(); + doTestThriftMetrics(); + doTestTableMutations(); + doTestTableTimestampsAndColumns(); + doTestTableScanners(); + doTestGetTableRegions(); + doTestFilterRegistration(); + } + + /** + * Tests for creating, enabling, disabling, and deleting tables. Also + * tests that creating a table with an invalid column name yields an + * IllegalArgument exception. + * + * @throws Exception + */ + public void doTestTableCreateDrop() throws Exception { + ThriftServerRunner.HBaseHandler handler = + new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration()); + doTestTableCreateDrop(handler); + } + public static void doTestTableCreateDrop(Hbase.Iface handler) throws Exception { + createTestTables(handler); + dropTestTables(handler); + } + + /** + * Tests if the metrics for thrift handler work correctly + */ + public void doTestThriftMetrics() throws Exception { + Configuration conf = UTIL.getConfiguration(); + ThriftMetrics metrics = getMetrics(conf); + Hbase.Iface handler = getHandler(metrics, conf); + createTestTables(handler); + dropTestTables(handler); + verifyMetrics(metrics, "createTable_num_ops", 2); + verifyMetrics(metrics, "deleteTable_num_ops", 2); + verifyMetrics(metrics, "disableTable_num_ops", 2); + } + + private static Hbase.Iface getHandler(ThriftMetrics metrics, Configuration conf) + throws Exception { + Hbase.Iface handler = + new ThriftServerRunner.HBaseHandler(conf); + return HbaseHandlerMetricsProxy.newInstance(handler, metrics, conf); + } + + private static ThriftMetrics getMetrics(Configuration conf) throws Exception { + setupMetricsContext(); + return new ThriftMetrics(ThriftServerRunner.DEFAULT_LISTEN_PORT, conf, Hbase.Iface.class); + } + + private static void setupMetricsContext() throws IOException { + ContextFactory factory = ContextFactory.getFactory(); + factory.setAttribute(ThriftMetrics.CONTEXT_NAME + ".class", + NoEmitMetricsContext.class.getName()); + MetricsUtil.getContext(ThriftMetrics.CONTEXT_NAME) + .createRecord(ThriftMetrics.CONTEXT_NAME).remove(); + } + + private static void verifyMetrics(ThriftMetrics metrics, String name, int expectValue) + throws Exception { + MetricsContext context = MetricsUtil.getContext( + ThriftMetrics.CONTEXT_NAME); + metrics.doUpdates(context); + OutputRecord record = context.getAllRecords().get( + ThriftMetrics.CONTEXT_NAME).iterator().next(); + assertEquals(expectValue, record.getMetric(name).intValue()); + } + + public static void createTestTables(Hbase.Iface handler) throws Exception { + // Create/enable/disable/delete tables, ensure methods act correctly + assertEquals(handler.getTableNames().size(), 0); + handler.createTable(tableAname, getColumnDescriptors()); + assertEquals(handler.getTableNames().size(), 1); + assertEquals(handler.getColumnDescriptors(tableAname).size(), 2); + assertTrue(handler.isTableEnabled(tableAname)); + handler.createTable(tableBname, new ArrayList()); + assertEquals(handler.getTableNames().size(), 2); + } + + public static void dropTestTables(Hbase.Iface handler) throws Exception { + handler.disableTable(tableBname); + assertFalse(handler.isTableEnabled(tableBname)); + handler.deleteTable(tableBname); + assertEquals(handler.getTableNames().size(), 1); + handler.disableTable(tableAname); + /* TODO Reenable. + assertFalse(handler.isTableEnabled(tableAname)); + handler.enableTable(tableAname); + assertTrue(handler.isTableEnabled(tableAname)); + handler.disableTable(tableAname);*/ + handler.deleteTable(tableAname); + } + + public void doTestIncrements() throws Exception { + ThriftServerRunner.HBaseHandler handler = + new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration()); + createTestTables(handler); + doTestIncrements(handler); + dropTestTables(handler); + } + + public static void doTestIncrements(HBaseHandler handler) throws Exception { + List mutations = new ArrayList(1); + mutations.add(new Mutation(false, columnAAname, valueEname, true)); + mutations.add(new Mutation(false, columnAname, valueEname, true)); + handler.mutateRow(tableAname, rowAname, mutations, null); + handler.mutateRow(tableAname, rowBname, mutations, null); + + List increments = new ArrayList(); + increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7)); + increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7)); + increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7)); + + int numIncrements = 60000; + for (int i = 0; i < numIncrements; i++) { + handler.increment(new TIncrement(tableAname, rowAname, columnAname, 2)); + handler.incrementRows(increments); + } + + Thread.sleep(1000); + long lv = handler.get(tableAname, rowAname, columnAname, null).get(0).value.getLong(); + assertEquals((100 + (2 * numIncrements)), lv ); + + + lv = handler.get(tableAname, rowBname, columnAAname, null).get(0).value.getLong(); + assertEquals((100 + (3 * 7 * numIncrements)), lv); + + assertTrue(handler.coalescer.getSuccessfulCoalescings() > 0); + + } + + /** + * Tests adding a series of Mutations and BatchMutations, including a + * delete mutation. Also tests data retrieval, and getting back multiple + * versions. + * + * @throws Exception + */ + public void doTestTableMutations() throws Exception { + ThriftServerRunner.HBaseHandler handler = + new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration()); + doTestTableMutations(handler); + } + + public static void doTestTableMutations(Hbase.Iface handler) throws Exception { + // Setup + handler.createTable(tableAname, getColumnDescriptors()); + + // Apply a few Mutations to rowA + // mutations.add(new Mutation(false, columnAname, valueAname)); + // mutations.add(new Mutation(false, columnBname, valueBname)); + handler.mutateRow(tableAname, rowAname, getMutations(), null); + + // Assert that the changes were made + assertEquals(valueAname, + handler.get(tableAname, rowAname, columnAname, null).get(0).value); + TRowResult rowResult1 = handler.getRow(tableAname, rowAname, null).get(0); + assertEquals(rowAname, rowResult1.row); + assertEquals(valueBname, + rowResult1.columns.get(columnBname).value); + + // Apply a few BatchMutations for rowA and rowB + // rowAmutations.add(new Mutation(true, columnAname, null)); + // rowAmutations.add(new Mutation(false, columnBname, valueCname)); + // batchMutations.add(new BatchMutation(rowAname, rowAmutations)); + // Mutations to rowB + // rowBmutations.add(new Mutation(false, columnAname, valueCname)); + // rowBmutations.add(new Mutation(false, columnBname, valueDname)); + // batchMutations.add(new BatchMutation(rowBname, rowBmutations)); + handler.mutateRows(tableAname, getBatchMutations(), null); + + // Assert that changes were made to rowA + List cells = handler.get(tableAname, rowAname, columnAname, null); + assertFalse(cells.size() > 0); + assertEquals(valueCname, handler.get(tableAname, rowAname, columnBname, null).get(0).value); + List versions = handler.getVer(tableAname, rowAname, columnBname, MAXVERSIONS, null); + assertEquals(valueCname, versions.get(0).value); + assertEquals(valueBname, versions.get(1).value); + + // Assert that changes were made to rowB + TRowResult rowResult2 = handler.getRow(tableAname, rowBname, null).get(0); + assertEquals(rowBname, rowResult2.row); + assertEquals(valueCname, rowResult2.columns.get(columnAname).value); + assertEquals(valueDname, rowResult2.columns.get(columnBname).value); + + // Apply some deletes + handler.deleteAll(tableAname, rowAname, columnBname, null); + handler.deleteAllRow(tableAname, rowBname, null); + + // Assert that the deletes were applied + int size = handler.get(tableAname, rowAname, columnBname, null).size(); + assertEquals(0, size); + size = handler.getRow(tableAname, rowBname, null).size(); + assertEquals(0, size); + + // Try null mutation + List mutations = new ArrayList(); + mutations.add(new Mutation(false, columnAname, null, true)); + handler.mutateRow(tableAname, rowAname, mutations, null); + TRowResult rowResult3 = handler.getRow(tableAname, rowAname, null).get(0); + assertEquals(rowAname, rowResult3.row); + assertEquals(0, rowResult3.columns.get(columnAname).value.remaining()); + + // Teardown + handler.disableTable(tableAname); + handler.deleteTable(tableAname); + } + + /** + * Similar to testTableMutations(), except Mutations are applied with + * specific timestamps and data retrieval uses these timestamps to + * extract specific versions of data. + * + * @throws Exception + */ + public void doTestTableTimestampsAndColumns() throws Exception { + // Setup + ThriftServerRunner.HBaseHandler handler = + new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration()); + handler.createTable(tableAname, getColumnDescriptors()); + + // Apply timestamped Mutations to rowA + long time1 = System.currentTimeMillis(); + handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null); + + Thread.sleep(1000); + + // Apply timestamped BatchMutations for rowA and rowB + long time2 = System.currentTimeMillis(); + handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null); + + // Apply an overlapping timestamped mutation to rowB + handler.mutateRowTs(tableAname, rowBname, getMutations(), time2, null); + + // the getVerTs is [inf, ts) so you need to increment one. + time1 += 1; + time2 += 2; + + // Assert that the timestamp-related methods retrieve the correct data + assertEquals(2, handler.getVerTs(tableAname, rowAname, columnBname, time2, + MAXVERSIONS, null).size()); + assertEquals(1, handler.getVerTs(tableAname, rowAname, columnBname, time1, + MAXVERSIONS, null).size()); + + TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1, null).get(0); + TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2, null).get(0); + // columnA was completely deleted + //assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname)); + assertEquals(rowResult1.columns.get(columnBname).value, valueBname); + assertEquals(rowResult2.columns.get(columnBname).value, valueCname); + + // ColumnAname has been deleted, and will never be visible even with a getRowTs() + assertFalse(rowResult2.columns.containsKey(columnAname)); + + List columns = new ArrayList(); + columns.add(columnBname); + + rowResult1 = handler.getRowWithColumns(tableAname, rowAname, columns, null).get(0); + assertEquals(rowResult1.columns.get(columnBname).value, valueCname); + assertFalse(rowResult1.columns.containsKey(columnAname)); + + rowResult1 = handler.getRowWithColumnsTs(tableAname, rowAname, columns, time1, null).get(0); + assertEquals(rowResult1.columns.get(columnBname).value, valueBname); + assertFalse(rowResult1.columns.containsKey(columnAname)); + + // Apply some timestamped deletes + // this actually deletes _everything_. + // nukes everything in columnB: forever. + handler.deleteAllTs(tableAname, rowAname, columnBname, time1, null); + handler.deleteAllRowTs(tableAname, rowBname, time2, null); + + // Assert that the timestamp-related methods retrieve the correct data + int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS, null).size(); + assertEquals(0, size); + + size = handler.getVerTs(tableAname, rowAname, columnBname, time2, MAXVERSIONS, null).size(); + assertEquals(1, size); + + // should be available.... + assertEquals(handler.get(tableAname, rowAname, columnBname, null).get(0).value, valueCname); + + assertEquals(0, handler.getRow(tableAname, rowBname, null).size()); + + // Teardown + handler.disableTable(tableAname); + handler.deleteTable(tableAname); + } + + /** + * Tests the four different scanner-opening methods (with and without + * a stoprow, with and without a timestamp). + * + * @throws Exception + */ + public void doTestTableScanners() throws Exception { + // Setup + ThriftServerRunner.HBaseHandler handler = + new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration()); + handler.createTable(tableAname, getColumnDescriptors()); + + // Apply timestamped Mutations to rowA + long time1 = System.currentTimeMillis(); + handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null); + + // Sleep to assure that 'time1' and 'time2' will be different even with a + // coarse grained system timer. + Thread.sleep(1000); + + // Apply timestamped BatchMutations for rowA and rowB + long time2 = System.currentTimeMillis(); + handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null); + + time1 += 1; + + // Test a scanner on all rows and all columns, no timestamp + int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true), null); + TRowResult rowResult1a = handler.scannerGet(scanner1).get(0); + assertEquals(rowResult1a.row, rowAname); + // This used to be '1'. I don't know why when we are asking for two columns + // and when the mutations above would seem to add two columns to the row. + // -- St.Ack 05/12/2009 + assertEquals(rowResult1a.columns.size(), 1); + assertEquals(rowResult1a.columns.get(columnBname).value, valueCname); + + TRowResult rowResult1b = handler.scannerGet(scanner1).get(0); + assertEquals(rowResult1b.row, rowBname); + assertEquals(rowResult1b.columns.size(), 2); + assertEquals(rowResult1b.columns.get(columnAname).value, valueCname); + assertEquals(rowResult1b.columns.get(columnBname).value, valueDname); + closeScanner(scanner1, handler); + + // Test a scanner on all rows and all columns, with timestamp + int scanner2 = handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1, null); + TRowResult rowResult2a = handler.scannerGet(scanner2).get(0); + assertEquals(rowResult2a.columns.size(), 1); + // column A deleted, does not exist. + //assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname)); + assertEquals(rowResult2a.columns.get(columnBname).value, valueBname); + closeScanner(scanner2, handler); + + // Test a scanner on the first row and first column only, no timestamp + int scanner3 = handler.scannerOpenWithStop(tableAname, rowAname, rowBname, + getColumnList(true, false), null); + closeScanner(scanner3, handler); + + // Test a scanner on the first row and second column only, with timestamp + int scanner4 = handler.scannerOpenWithStopTs(tableAname, rowAname, rowBname, + getColumnList(false, true), time1, null); + TRowResult rowResult4a = handler.scannerGet(scanner4).get(0); + assertEquals(rowResult4a.columns.size(), 1); + assertEquals(rowResult4a.columns.get(columnBname).value, valueBname); + + // Teardown + handler.disableTable(tableAname); + handler.deleteTable(tableAname); + } + + /** + * For HBASE-2556 + * Tests for GetTableRegions + * + * @throws Exception + */ + public void doTestGetTableRegions() throws Exception { + ThriftServerRunner.HBaseHandler handler = + new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration()); + doTestGetTableRegions(handler); + } + + public static void doTestGetTableRegions(Hbase.Iface handler) + throws Exception { + assertEquals(handler.getTableNames().size(), 0); + handler.createTable(tableAname, getColumnDescriptors()); + assertEquals(handler.getTableNames().size(), 1); + List regions = handler.getTableRegions(tableAname); + int regionCount = regions.size(); + assertEquals("empty table should have only 1 region, " + + "but found " + regionCount, regionCount, 1); + LOG.info("Region found:" + regions.get(0)); + handler.disableTable(tableAname); + handler.deleteTable(tableAname); + regionCount = handler.getTableRegions(tableAname).size(); + assertEquals("non-existing table should have 0 region, " + + "but found " + regionCount, regionCount, 0); + } + + public void doTestFilterRegistration() throws Exception { + Configuration conf = UTIL.getConfiguration(); + + conf.set("hbase.thrift.filters", "MyFilter:filterclass"); + + ThriftServerRunner.registerFilters(conf); + + Map registeredFilters = ParseFilter.getAllFilters(); + + assertEquals("filterclass", registeredFilters.get("MyFilter")); + } + + /** + * + * @return a List of ColumnDescriptors for use in creating a table. Has one + * default ColumnDescriptor and one ColumnDescriptor with fewer versions + */ + private static List getColumnDescriptors() { + ArrayList cDescriptors = new ArrayList(); + + // A default ColumnDescriptor + ColumnDescriptor cDescA = new ColumnDescriptor(); + cDescA.name = columnAname; + cDescriptors.add(cDescA); + + // A slightly customized ColumnDescriptor (only 2 versions) + ColumnDescriptor cDescB = new ColumnDescriptor(columnBname, 2, "NONE", + false, "NONE", 0, 0, false, -1); + cDescriptors.add(cDescB); + + return cDescriptors; + } + + /** + * + * @param includeA whether or not to include columnA + * @param includeB whether or not to include columnB + * @return a List of column names for use in retrieving a scanner + */ + private List getColumnList(boolean includeA, boolean includeB) { + List columnList = new ArrayList(); + if (includeA) columnList.add(columnAname); + if (includeB) columnList.add(columnBname); + return columnList; + } + + /** + * + * @return a List of Mutations for a row, with columnA having valueA + * and columnB having valueB + */ + private static List getMutations() { + List mutations = new ArrayList(); + mutations.add(new Mutation(false, columnAname, valueAname, true)); + mutations.add(new Mutation(false, columnBname, valueBname, true)); + return mutations; + } + + /** + * + * @return a List of BatchMutations with the following effects: + * (rowA, columnA): delete + * (rowA, columnB): place valueC + * (rowB, columnA): place valueC + * (rowB, columnB): place valueD + */ + private static List getBatchMutations() { + List batchMutations = new ArrayList(); + + // Mutations to rowA. You can't mix delete and put anymore. + List rowAmutations = new ArrayList(); + rowAmutations.add(new Mutation(true, columnAname, null, true)); + batchMutations.add(new BatchMutation(rowAname, rowAmutations)); + + rowAmutations = new ArrayList(); + rowAmutations.add(new Mutation(false, columnBname, valueCname, true)); + batchMutations.add(new BatchMutation(rowAname, rowAmutations)); + + // Mutations to rowB + List rowBmutations = new ArrayList(); + rowBmutations.add(new Mutation(false, columnAname, valueCname, true)); + rowBmutations.add(new Mutation(false, columnBname, valueDname, true)); + batchMutations.add(new BatchMutation(rowBname, rowBmutations)); + + return batchMutations; + } + + /** + * Asserts that the passed scanner is exhausted, and then closes + * the scanner. + * + * @param scannerId the scanner to close + * @param handler the HBaseHandler interfacing to HBase + * @throws Exception + */ + private void closeScanner( + int scannerId, ThriftServerRunner.HBaseHandler handler) throws Exception { + handler.scannerGet(scannerId); + handler.scannerClose(scannerId); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java new file mode 100644 index 0000000..516f321 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServerCmdLine.java @@ -0,0 +1,229 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.thrift; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.net.InetAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.thrift.ThriftServerRunner.ImplType; +import org.apache.hadoop.hbase.thrift.generated.Hbase; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.server.TServer; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import com.google.common.base.Joiner; + +/** + * Start the HBase Thrift server on a random port through the command-line + * interface and talk to it from client side. + */ +@Category(LargeTests.class) +@RunWith(Parameterized.class) +public class TestThriftServerCmdLine { + + public static final Log LOG = + LogFactory.getLog(TestThriftServerCmdLine.class); + + private final ImplType implType; + private boolean specifyFramed; + private boolean specifyBindIP; + private boolean specifyCompact; + + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + private Thread cmdLineThread; + private volatile Exception cmdLineException; + + private Exception clientSideException; + + private ThriftServer thriftServer; + private int port; + + @Parameters + public static Collection getParameters() { + Collection parameters = new ArrayList(); + for (ImplType implType : ImplType.values()) { + for (boolean specifyFramed : new boolean[] {false, true}) { + for (boolean specifyBindIP : new boolean[] {false, true}) { + if (specifyBindIP && !implType.canSpecifyBindIP) { + continue; + } + for (boolean specifyCompact : new boolean[] {false, true}) { + parameters.add(new Object[]{implType, new Boolean(specifyFramed), + new Boolean(specifyBindIP), new Boolean(specifyCompact)}); + } + } + } + } + return parameters; + } + + public TestThriftServerCmdLine(ImplType implType, boolean specifyFramed, + boolean specifyBindIP, boolean specifyCompact) { + this.implType = implType; + this.specifyFramed = specifyFramed; + this.specifyBindIP = specifyBindIP; + this.specifyCompact = specifyCompact; + LOG.debug("implType=" + implType + ", " + + "specifyFramed=" + specifyFramed + ", " + + "specifyBindIP=" + specifyBindIP + ", " + + "specifyCompact=" + specifyCompact); + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + private void startCmdLineThread(final String[] args) { + LOG.info("Starting HBase Thrift server with command line: " + + Joiner.on(" ").join(args)); + + cmdLineException = null; + cmdLineThread = new Thread(new Runnable() { + @Override + public void run() { + try { + thriftServer.doMain(args); + } catch (Exception e) { + cmdLineException = e; + } + } + }); + cmdLineThread.setName(ThriftServer.class.getSimpleName() + + "-cmdline"); + cmdLineThread.start(); + } + + @Test(timeout=120 * 1000) + public void testRunThriftServer() throws Exception { + List args = new ArrayList(); + if (implType != null) { + String serverTypeOption = implType.toString(); + assertTrue(serverTypeOption.startsWith("-")); + args.add(serverTypeOption); + } + port = HBaseTestingUtility.randomFreePort(); + args.add("-" + ThriftServer.PORT_OPTION); + args.add(String.valueOf(port)); + if (specifyFramed) { + args.add("-" + ThriftServer.FRAMED_OPTION); + } + if (specifyBindIP) { + args.add("-" + ThriftServer.BIND_OPTION); + args.add(InetAddress.getLocalHost().getHostName()); + } + if (specifyCompact) { + args.add("-" + ThriftServer.COMPACT_OPTION); + } + args.add("start"); + + thriftServer = new ThriftServer(TEST_UTIL.getConfiguration()); + startCmdLineThread(args.toArray(new String[0])); + Threads.sleepWithoutInterrupt(2000); + + Class expectedClass = implType != null ? + implType.serverClass : TBoundedThreadPoolServer.class; + assertEquals(expectedClass, + thriftServer.serverRunner.tserver.getClass()); + + try { + talkToThriftServer(); + } catch (Exception ex) { + clientSideException = ex; + } finally { + stopCmdLineThread(); + } + + if (clientSideException != null) { + LOG.error("Thrift client threw an exception", clientSideException); + throw new Exception(clientSideException); + } + } + + private void talkToThriftServer() throws Exception { + TSocket sock = new TSocket(InetAddress.getLocalHost().getHostName(), + port); + TTransport transport = sock; + if (specifyFramed || implType.isAlwaysFramed) { + transport = new TFramedTransport(transport); + } + + sock.open(); + try { + TProtocol prot; + if (specifyCompact) { + prot = new TCompactProtocol(transport); + } else { + prot = new TBinaryProtocol(transport); + } + Hbase.Client client = new Hbase.Client(prot); + TestThriftServer.doTestTableCreateDrop(client); + TestThriftServer.doTestGetTableRegions(client); + TestThriftServer.doTestTableMutations(client); + } finally { + sock.close(); + } + } + + private void stopCmdLineThread() throws Exception { + LOG.debug("Stopping " + implType.simpleClassName() + " Thrift server"); + thriftServer.stop(); + cmdLineThread.join(); + if (cmdLineException != null) { + LOG.error("Command-line invocation of HBase Thrift server threw an " + + "exception", cmdLineException); + throw new Exception(cmdLineException); + } + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java new file mode 100644 index 0000000..b848ac7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java @@ -0,0 +1,611 @@ +/* + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.thrift2; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.thrift.ThriftMetrics; +import org.apache.hadoop.hbase.thrift2.generated.TColumn; +import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement; +import org.apache.hadoop.hbase.thrift2.generated.TColumnValue; +import org.apache.hadoop.hbase.thrift2.generated.TDelete; +import org.apache.hadoop.hbase.thrift2.generated.TDeleteType; +import org.apache.hadoop.hbase.thrift2.generated.TGet; +import org.apache.hadoop.hbase.thrift2.generated.THBaseService; +import org.apache.hadoop.hbase.thrift2.generated.TIOError; +import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument; +import org.apache.hadoop.hbase.thrift2.generated.TIncrement; +import org.apache.hadoop.hbase.thrift2.generated.TPut; +import org.apache.hadoop.hbase.thrift2.generated.TResult; +import org.apache.hadoop.hbase.thrift2.generated.TScan; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.metrics.ContextFactory; +import org.apache.hadoop.metrics.MetricsContext; +import org.apache.hadoop.metrics.MetricsUtil; +import org.apache.hadoop.metrics.spi.NoEmitMetricsContext; +import org.apache.hadoop.metrics.spi.OutputRecord; +import org.apache.thrift.TException; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Unit testing for ThriftServer.HBaseHandler, a part of the org.apache.hadoop.hbase.thrift2 package. + */ +@Category(MediumTests.class) +public class TestThriftHBaseServiceHandler { + + public static final Log LOG = LogFactory.getLog(TestThriftHBaseServiceHandler.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + // Static names for tables, columns, rows, and values + private static byte[] tableAname = Bytes.toBytes("tableA"); + private static byte[] familyAname = Bytes.toBytes("familyA"); + private static byte[] familyBname = Bytes.toBytes("familyB"); + private static byte[] qualifierAname = Bytes.toBytes("qualifierA"); + private static byte[] qualifierBname = Bytes.toBytes("qualifierB"); + private static byte[] valueAname = Bytes.toBytes("valueA"); + private static byte[] valueBname = Bytes.toBytes("valueB"); + private static HColumnDescriptor[] families = new HColumnDescriptor[] { + new HColumnDescriptor(familyAname), + new HColumnDescriptor(familyBname) + .setMaxVersions(2) + }; + + public void assertTColumnValuesEqual(List columnValuesA, List columnValuesB) { + assertEquals(columnValuesA.size(), columnValuesB.size()); + Comparator comparator = new Comparator() { + @Override + public int compare(TColumnValue o1, TColumnValue o2) { + return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()), + Bytes.add(o2.getFamily(), o2.getQualifier())); + } + }; + Collections.sort(columnValuesA, comparator); + Collections.sort(columnValuesB, comparator); + + for (int i = 0; i < columnValuesA.size(); i++) { + TColumnValue a = columnValuesA.get(i); + TColumnValue b = columnValuesB.get(i); + assertArrayEquals(a.getFamily(), b.getFamily()); + assertArrayEquals(a.getQualifier(), b.getQualifier()); + assertArrayEquals(a.getValue(), b.getValue()); + } + } + + @BeforeClass + public static void beforeClass() throws Exception { + UTIL.startMiniCluster(); + HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); + HTableDescriptor tableDescriptor = new HTableDescriptor(tableAname); + for (HColumnDescriptor family : families) { + tableDescriptor.addFamily(family); + } + admin.createTable(tableDescriptor); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Before + public void setup() throws Exception { + + } + + private ThriftHBaseServiceHandler createHandler() { + return new ThriftHBaseServiceHandler(UTIL.getConfiguration()); + } + + @Test + public void testExists() throws TIOError, TException { + ThriftHBaseServiceHandler handler = createHandler(); + byte[] rowName = "testExists".getBytes(); + ByteBuffer table = ByteBuffer.wrap(tableAname); + + TGet get = new TGet(ByteBuffer.wrap(rowName)); + assertFalse(handler.exists(table, get)); + + List columnValues = new ArrayList(); + columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), ByteBuffer + .wrap(valueAname))); + columnValues.add(new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname), ByteBuffer + .wrap(valueBname))); + TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues); + put.setColumnValues(columnValues); + + handler.put(table, put); + + assertTrue(handler.exists(table, get)); + } + + @Test + public void testPutGet() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + byte[] rowName = "testPutGet".getBytes(); + ByteBuffer table = ByteBuffer.wrap(tableAname); + + List columnValues = new ArrayList(); + columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), ByteBuffer + .wrap(valueAname))); + columnValues.add(new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname), ByteBuffer + .wrap(valueBname))); + TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues); + + put.setColumnValues(columnValues); + + handler.put(table, put); + + TGet get = new TGet(ByteBuffer.wrap(rowName)); + + TResult result = handler.get(table, get); + assertArrayEquals(rowName, result.getRow()); + List returnedColumnValues = result.getColumnValues(); + assertTColumnValuesEqual(columnValues, returnedColumnValues); + } + + @Test + public void testPutGetMultiple() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + ByteBuffer table = ByteBuffer.wrap(tableAname); + byte[] rowName1 = "testPutGetMultiple1".getBytes(); + byte[] rowName2 = "testPutGetMultiple2".getBytes(); + + List columnValues = new ArrayList(); + columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), ByteBuffer + .wrap(valueAname))); + columnValues.add(new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname), ByteBuffer + .wrap(valueBname))); + List puts = new ArrayList(); + puts.add(new TPut(ByteBuffer.wrap(rowName1), columnValues)); + puts.add(new TPut(ByteBuffer.wrap(rowName2), columnValues)); + + handler.putMultiple(table, puts); + + List gets = new ArrayList(); + gets.add(new TGet(ByteBuffer.wrap(rowName1))); + gets.add(new TGet(ByteBuffer.wrap(rowName2))); + + List results = handler.getMultiple(table, gets); + assertEquals(2, results.size()); + + assertArrayEquals(rowName1, results.get(0).getRow()); + assertTColumnValuesEqual(columnValues, results.get(0).getColumnValues()); + + assertArrayEquals(rowName2, results.get(1).getRow()); + assertTColumnValuesEqual(columnValues, results.get(1).getColumnValues()); + } + + @Test + public void testDeleteMultiple() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + ByteBuffer table = ByteBuffer.wrap(tableAname); + byte[] rowName1 = "testDeleteMultiple1".getBytes(); + byte[] rowName2 = "testDeleteMultiple2".getBytes(); + + List columnValues = new ArrayList(); + columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), ByteBuffer + .wrap(valueAname))); + columnValues.add(new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname), ByteBuffer + .wrap(valueBname))); + List puts = new ArrayList(); + puts.add(new TPut(ByteBuffer.wrap(rowName1), columnValues)); + puts.add(new TPut(ByteBuffer.wrap(rowName2), columnValues)); + + handler.putMultiple(table, puts); + + List deletes = new ArrayList(); + deletes.add(new TDelete(ByteBuffer.wrap(rowName1))); + deletes.add(new TDelete(ByteBuffer.wrap(rowName2))); + + List deleteResults = handler.deleteMultiple(table, deletes); + // 0 means they were all successfully applies + assertEquals(0, deleteResults.size()); + + assertFalse(handler.exists(table, new TGet(ByteBuffer.wrap(rowName1)))); + assertFalse(handler.exists(table, new TGet(ByteBuffer.wrap(rowName2)))); + } + + @Test + public void testDelete() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + byte[] rowName = "testDelete".getBytes(); + ByteBuffer table = ByteBuffer.wrap(tableAname); + + List columnValues = new ArrayList(); + TColumnValue columnValueA = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), + ByteBuffer.wrap(valueAname)); + TColumnValue columnValueB = new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname), + ByteBuffer.wrap(valueBname)); + columnValues.add(columnValueA); + columnValues.add(columnValueB); + TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues); + + put.setColumnValues(columnValues); + + handler.put(table, put); + + TDelete delete = new TDelete(ByteBuffer.wrap(rowName)); + List deleteColumns = new ArrayList(); + TColumn deleteColumn = new TColumn(ByteBuffer.wrap(familyAname)); + deleteColumn.setQualifier(qualifierAname); + deleteColumns.add(deleteColumn); + delete.setColumns(deleteColumns); + + handler.deleteSingle(table, delete); + + TGet get = new TGet(ByteBuffer.wrap(rowName)); + TResult result = handler.get(table, get); + assertArrayEquals(rowName, result.getRow()); + List returnedColumnValues = result.getColumnValues(); + List expectedColumnValues = new ArrayList(); + expectedColumnValues.add(columnValueB); + assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues); + } + + @Test + public void testDeleteAllTimestamps() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + byte[] rowName = "testDeleteAllTimestamps".getBytes(); + ByteBuffer table = ByteBuffer.wrap(tableAname); + + List columnValues = new ArrayList(); + TColumnValue columnValueA = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), + ByteBuffer.wrap(valueAname)); + columnValueA.setTimestamp(System.currentTimeMillis() - 10); + columnValues.add(columnValueA); + TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues); + + put.setColumnValues(columnValues); + + handler.put(table, put); + columnValueA.setTimestamp(System.currentTimeMillis()); + handler.put(table, put); + + TGet get = new TGet(ByteBuffer.wrap(rowName)); + get.setMaxVersions(2); + TResult result = handler.get(table, get); + assertEquals(2, result.getColumnValuesSize()); + + TDelete delete = new TDelete(ByteBuffer.wrap(rowName)); + List deleteColumns = new ArrayList(); + TColumn deleteColumn = new TColumn(ByteBuffer.wrap(familyAname)); + deleteColumn.setQualifier(qualifierAname); + deleteColumns.add(deleteColumn); + delete.setColumns(deleteColumns); + delete.setDeleteType(TDeleteType.DELETE_COLUMNS); // This is the default anyway. + + handler.deleteSingle(table, delete); + + get = new TGet(ByteBuffer.wrap(rowName)); + result = handler.get(table, get); + assertNull(result.getRow()); + assertEquals(0, result.getColumnValuesSize()); + } + + @Test + public void testDeleteSingleTimestamp() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + byte[] rowName = "testDeleteSingleTimestamp".getBytes(); + ByteBuffer table = ByteBuffer.wrap(tableAname); + + long timestamp1 = System.currentTimeMillis() - 10; + long timestamp2 = System.currentTimeMillis(); + + List columnValues = new ArrayList(); + TColumnValue columnValueA = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), + ByteBuffer.wrap(valueAname)); + columnValueA.setTimestamp(timestamp1); + columnValues.add(columnValueA); + TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues); + + put.setColumnValues(columnValues); + + handler.put(table, put); + columnValueA.setTimestamp(timestamp2); + handler.put(table, put); + + TGet get = new TGet(ByteBuffer.wrap(rowName)); + get.setMaxVersions(2); + TResult result = handler.get(table, get); + assertEquals(2, result.getColumnValuesSize()); + + TDelete delete = new TDelete(ByteBuffer.wrap(rowName)); + List deleteColumns = new ArrayList(); + TColumn deleteColumn = new TColumn(ByteBuffer.wrap(familyAname)); + deleteColumn.setQualifier(qualifierAname); + deleteColumns.add(deleteColumn); + delete.setColumns(deleteColumns); + delete.setDeleteType(TDeleteType.DELETE_COLUMN); + + handler.deleteSingle(table, delete); + + get = new TGet(ByteBuffer.wrap(rowName)); + result = handler.get(table, get); + assertArrayEquals(rowName, result.getRow()); + assertEquals(1, result.getColumnValuesSize()); + // the older timestamp should remain. + assertEquals(timestamp1, result.getColumnValues().get(0).getTimestamp()); + } + + @Test + public void testIncrement() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + byte[] rowName = "testIncrement".getBytes(); + ByteBuffer table = ByteBuffer.wrap(tableAname); + + List columnValues = new ArrayList(); + columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), ByteBuffer + .wrap(Bytes.toBytes(1L)))); + TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues); + put.setColumnValues(columnValues); + handler.put(table, put); + + List incrementColumns = new ArrayList(); + incrementColumns.add(new TColumnIncrement(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname))); + TIncrement increment = new TIncrement(ByteBuffer.wrap(rowName), incrementColumns); + handler.increment(table, increment); + + TGet get = new TGet(ByteBuffer.wrap(rowName)); + TResult result = handler.get(table, get); + + assertArrayEquals(rowName, result.getRow()); + assertEquals(1, result.getColumnValuesSize()); + TColumnValue columnValue = result.getColumnValues().get(0); + assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue()); + } + + /** + * check that checkAndPut fails if the cell does not exist, then put in the cell, then check that the checkAndPut + * succeeds. + * + * @throws Exception + */ + @Test + public void testCheckAndPut() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + byte[] rowName = "testCheckAndPut".getBytes(); + ByteBuffer table = ByteBuffer.wrap(tableAname); + + List columnValuesA = new ArrayList(); + TColumnValue columnValueA = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), + ByteBuffer.wrap(valueAname)); + columnValuesA.add(columnValueA); + TPut putA = new TPut(ByteBuffer.wrap(rowName), columnValuesA); + putA.setColumnValues(columnValuesA); + + List columnValuesB = new ArrayList(); + TColumnValue columnValueB = new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname), + ByteBuffer.wrap(valueBname)); + columnValuesB.add(columnValueB); + TPut putB = new TPut(ByteBuffer.wrap(rowName), columnValuesB); + putB.setColumnValues(columnValuesB); + + assertFalse(handler.checkAndPut(table, ByteBuffer.wrap(rowName), ByteBuffer.wrap(familyAname), + ByteBuffer.wrap(qualifierAname), ByteBuffer.wrap(valueAname), putB)); + + TGet get = new TGet(ByteBuffer.wrap(rowName)); + TResult result = handler.get(table, get); + assertEquals(0, result.getColumnValuesSize()); + + handler.put(table, putA); + + assertTrue(handler.checkAndPut(table, ByteBuffer.wrap(rowName), ByteBuffer.wrap(familyAname), + ByteBuffer.wrap(qualifierAname), ByteBuffer.wrap(valueAname), putB)); + + result = handler.get(table, get); + assertArrayEquals(rowName, result.getRow()); + List returnedColumnValues = result.getColumnValues(); + List expectedColumnValues = new ArrayList(); + expectedColumnValues.add(columnValueA); + expectedColumnValues.add(columnValueB); + assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues); + } + + /** + * check that checkAndDelete fails if the cell does not exist, then put in the cell, then check that the + * checkAndDelete succeeds. + * + * @throws Exception + */ + @Test + public void testCheckAndDelete() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + byte[] rowName = "testCheckAndDelete".getBytes(); + ByteBuffer table = ByteBuffer.wrap(tableAname); + + List columnValuesA = new ArrayList(); + TColumnValue columnValueA = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), + ByteBuffer.wrap(valueAname)); + columnValuesA.add(columnValueA); + TPut putA = new TPut(ByteBuffer.wrap(rowName), columnValuesA); + putA.setColumnValues(columnValuesA); + + List columnValuesB = new ArrayList(); + TColumnValue columnValueB = new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname), + ByteBuffer.wrap(valueBname)); + columnValuesB.add(columnValueB); + TPut putB = new TPut(ByteBuffer.wrap(rowName), columnValuesB); + putB.setColumnValues(columnValuesB); + + // put putB so that we know whether the row has been deleted or not + handler.put(table, putB); + + TDelete delete = new TDelete(ByteBuffer.wrap(rowName)); + + assertFalse(handler.checkAndDelete(table, ByteBuffer.wrap(rowName), ByteBuffer.wrap(familyAname), + ByteBuffer.wrap(qualifierAname), ByteBuffer.wrap(valueAname), delete)); + + TGet get = new TGet(ByteBuffer.wrap(rowName)); + TResult result = handler.get(table, get); + assertArrayEquals(rowName, result.getRow()); + assertTColumnValuesEqual(columnValuesB, result.getColumnValues()); + + handler.put(table, putA); + + assertTrue(handler.checkAndDelete(table, ByteBuffer.wrap(rowName), ByteBuffer.wrap(familyAname), + ByteBuffer.wrap(qualifierAname), ByteBuffer.wrap(valueAname), delete)); + + result = handler.get(table, get); + assertFalse(result.isSetRow()); + assertEquals(0, result.getColumnValuesSize()); + } + + @Test + public void testScan() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + ByteBuffer table = ByteBuffer.wrap(tableAname); + + TScan scan = new TScan(); + List columns = new ArrayList(); + TColumn column = new TColumn(); + column.setFamily(familyAname); + column.setQualifier(qualifierAname); + columns.add(column); + scan.setColumns(columns); + scan.setStartRow("testScan".getBytes()); + + TColumnValue columnValue = new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), + ByteBuffer.wrap(valueAname)); + List columnValues = new ArrayList(); + columnValues.add(columnValue); + for (int i = 0; i < 10; i++) { + TPut put = new TPut(ByteBuffer.wrap(("testScan" + i).getBytes()), columnValues); + handler.put(table, put); + } + + int scanId = handler.openScanner(table, scan); + List results = handler.getScannerRows(scanId, 10); + assertEquals(10, results.size()); + for (int i = 0; i < 10; i++) { + assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow()); + } + + results = handler.getScannerRows(scanId, 10); + assertEquals(0, results.size()); + + handler.closeScanner(scanId); + + try { + handler.getScannerRows(scanId, 10); + fail("Scanner id should be invalid"); + } catch (TIllegalArgument e) { + } + } + + @Test + public void testMetrics() throws Exception { + Configuration conf = UTIL.getConfiguration(); + ThriftMetrics metrics = getMetrics(conf); + THBaseService.Iface handler = + ThriftHBaseServiceHandler.newInstance(conf, metrics); + byte[] rowName = "testMetrics".getBytes(); + ByteBuffer table = ByteBuffer.wrap(tableAname); + + TGet get = new TGet(ByteBuffer.wrap(rowName)); + assertFalse(handler.exists(table, get)); + + List columnValues = new ArrayList(); + columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname), + ByteBuffer.wrap(qualifierAname), + ByteBuffer.wrap(valueAname))); + columnValues.add(new TColumnValue(ByteBuffer.wrap(familyBname), + ByteBuffer.wrap(qualifierBname), + ByteBuffer.wrap(valueBname))); + TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues); + put.setColumnValues(columnValues); + + handler.put(table, put); + + assertTrue(handler.exists(table, get)); + logMetrics(metrics); + verifyMetrics(metrics, "put_num_ops", 1); + verifyMetrics(metrics, "exists_num_ops", 2); + } + + private static ThriftMetrics getMetrics(Configuration conf) throws Exception { + setupMetricsContext(); + return new ThriftMetrics(Integer.parseInt(ThriftServer.DEFAULT_LISTEN_PORT), + conf, THBaseService.Iface.class); + } + + private static void setupMetricsContext() throws IOException { + ContextFactory factory = ContextFactory.getFactory(); + factory.setAttribute(ThriftMetrics.CONTEXT_NAME + ".class", + NoEmitMetricsContext.class.getName()); + MetricsUtil.getContext(ThriftMetrics.CONTEXT_NAME) + .createRecord(ThriftMetrics.CONTEXT_NAME).remove(); + } + + private static void logMetrics(ThriftMetrics metrics) throws Exception { + if (LOG.isDebugEnabled()) { + return; + } + MetricsContext context = MetricsUtil.getContext( + ThriftMetrics.CONTEXT_NAME); + metrics.doUpdates(context); + for (String key : context.getAllRecords().keySet()) { + for (OutputRecord record : context.getAllRecords().get(key)) { + for (String name : record.getMetricNames()) { + LOG.debug("metrics:" + name + " value:" + + record.getMetric(name).intValue()); + } + } + } + } + + private static void verifyMetrics(ThriftMetrics metrics, String name, int expectValue) + throws Exception { + MetricsContext context = MetricsUtil.getContext( + ThriftMetrics.CONTEXT_NAME); + metrics.doUpdates(context); + OutputRecord record = context.getAllRecords().get( + ThriftMetrics.CONTEXT_NAME).iterator().next(); + assertEquals(expectValue, record.getMetric(name).intValue()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java new file mode 100644 index 0000000..266586f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java @@ -0,0 +1,747 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Random; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseCluster; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HServerLoad; +import org.apache.hadoop.hbase.IntegrationTestingUtility; +import org.apache.hadoop.hbase.IntegrationTestDataIngestWithChaosMonkey; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolRunner; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.protobuf.ServiceException; + +/** + * A utility to injects faults in a running cluster. + *

    + * ChaosMonkey defines Action's and Policy's. Actions are sequences of events, like + * - Select a random server to kill + * - Sleep for 5 sec + * - Start the server on the same host + * Actions can also be complex events, like rolling restart of all of the servers. + *

    + * Policies on the other hand are responsible for executing the actions based on a strategy. + * The default policy is to execute a random action every minute based on predefined action + * weights. ChaosMonkey executes predefined named policies until it is stopped. More than one + * policy can be active at any time. + *

    + * Chaos monkey can be run from the command line, or can be invoked from integration tests. + * See {@link IntegrationTestDataIngestWithChaosMonkey} or other integration tests that use + * chaos monkey for code examples. + *

    + * ChaosMonkey class is indeed inspired by the Netflix's same-named tool: + * http://techblog.netflix.com/2012/07/chaos-monkey-released-into-wild.html + */ +public class ChaosMonkey extends AbstractHBaseTool implements Stoppable { + + private static final Log LOG = LogFactory.getLog(ChaosMonkey.class); + + private static final long ONE_SEC = 1000; + private static final long FIVE_SEC = 5 * ONE_SEC; + private static final long ONE_MIN = 60 * ONE_SEC; + private static final long TIMEOUT = ONE_MIN; + + final IntegrationTestingUtility util; + + /** + * Construct a new ChaosMonkey + * @param util the HBaseIntegrationTestingUtility already configured + * @param policies names of pre-defined policies to use + */ + public ChaosMonkey(IntegrationTestingUtility util, String... policies) { + this.util = util; + setPoliciesByName(policies); + } + + /** + * Construct a new ChaosMonkey + * @param util the HBaseIntegrationTestingUtility already configured + * @param policies custom policies to use + */ + public ChaosMonkey(IntegrationTestingUtility util, Policy... policies) { + this.util = util; + this.policies = policies; + } + + private void setPoliciesByName(String... policies) { + this.policies = new Policy[policies.length]; + for (int i=0; i < policies.length; i++) { + this.policies[i] = NAMED_POLICIES.get(policies[i]); + } + } + + /** + * Context for Action's + */ + private static class ActionContext { + private IntegrationTestingUtility util; + + ActionContext(IntegrationTestingUtility util) { + this.util = util; + } + + IntegrationTestingUtility getHaseIntegrationTestingUtility() { + return util; + } + + HBaseCluster getHBaseCluster() { + return util.getHBaseClusterInterface(); + } + } + + /** + * A (possibly mischievous) action that the ChaosMonkey can perform. + */ + public static class Action { + // TODO: interesting question - should actions be implemented inside + // ChaosMonkey, or outside? If they are inside (initial), the class becomes + // huge and all-encompassing; if they are outside ChaosMonkey becomes just + // a random task scheduler. For now, keep inside. + + protected ActionContext context; + protected HBaseCluster cluster; + protected ClusterStatus initialStatus; + protected ServerName[] initialServers; + + void init(ActionContext context) throws Exception { + this.context = context; + cluster = context.getHBaseCluster(); + initialStatus = cluster.getInitialClusterStatus(); + Collection regionServers = initialStatus.getServers(); + initialServers = regionServers.toArray(new ServerName[regionServers.size()]); + } + + void perform() throws Exception { }; + + // TODO: perhaps these methods should be elsewhere? + /** Returns current region servers */ + protected ServerName[] getCurrentServers() throws IOException { + Collection regionServers = cluster.getClusterStatus().getServers(); + return regionServers.toArray(new ServerName[regionServers.size()]); + } + + protected void killMaster(ServerName server) throws IOException { + LOG.info("Killing master:" + server); + cluster.killMaster(server); + cluster.waitForMasterToStop(server, TIMEOUT); + LOG.info("Killed master server:" + server); + } + + protected void startMaster(ServerName server) throws IOException { + LOG.info("Starting master:" + server.getHostname()); + cluster.startMaster(server.getHostname()); + cluster.waitForActiveAndReadyMaster(TIMEOUT); + LOG.info("Started master: " + server); + } + + protected void killRs(ServerName server) throws IOException { + LOG.info("Killing region server:" + server); + cluster.killRegionServer(server); + cluster.waitForRegionServerToStop(server, TIMEOUT); + LOG.info("Killed region server:" + server + ". Reported num of rs:" + + cluster.getClusterStatus().getServersSize()); + } + + protected void startRs(ServerName server) throws IOException { + LOG.info("Starting region server:" + server.getHostname()); + cluster.startRegionServer(server.getHostname()); + cluster.waitForRegionServerToStart(server.getHostname(), TIMEOUT); + LOG.info("Started region server:" + server + ". Reported num of rs:" + + cluster.getClusterStatus().getServersSize()); + } + } + + private static class RestartActionBase extends Action { + long sleepTime; // how long should we sleep + + public RestartActionBase(long sleepTime) { + this.sleepTime = sleepTime; + } + + void sleep(long sleepTime) { + LOG.info("Sleeping for:" + sleepTime); + Threads.sleep(sleepTime); + } + + void restartMaster(ServerName server, long sleepTime) throws IOException { + killMaster(server); + sleep(sleepTime); + startMaster(server); + } + + void restartRs(ServerName server, long sleepTime) throws IOException { + killRs(server); + sleep(sleepTime); + startRs(server); + } + } + + public static class RestartActiveMaster extends RestartActionBase { + public RestartActiveMaster(long sleepTime) { + super(sleepTime); + } + @Override + void perform() throws Exception { + LOG.info("Performing action: Restart active master"); + + ServerName master = cluster.getClusterStatus().getMaster(); + restartMaster(master, sleepTime); + } + } + + public static class RestartRandomRs extends RestartActionBase { + public RestartRandomRs(long sleepTime) { + super(sleepTime); + } + + @Override + void perform() throws Exception { + LOG.info("Performing action: Restart random region server"); + ServerName server = selectRandomItem(getCurrentServers()); + + restartRs(server, sleepTime); + } + } + + public static class RestartRsHoldingMeta extends RestartRandomRs { + public RestartRsHoldingMeta(long sleepTime) { + super(sleepTime); + } + @Override + void perform() throws Exception { + LOG.info("Performing action: Restart region server holding META"); + ServerName server = cluster.getServerHoldingMeta(); + if (server == null) { + LOG.warn("No server is holding .META. right now."); + return; + } + restartRs(server, sleepTime); + } + } + + public static class RestartRsHoldingRoot extends RestartRandomRs { + public RestartRsHoldingRoot(long sleepTime) { + super(sleepTime); + } + @Override + void perform() throws Exception { + LOG.info("Performing action: Restart region server holding ROOT"); + ServerName server = cluster.getServerHoldingMeta(); + if (server == null) { + LOG.warn("No server is holding -ROOT- right now."); + return; + } + restartRs(server, sleepTime); + } + } + + /** + * Restarts a ratio of the running regionservers at the same time + */ + public static class BatchRestartRs extends RestartActionBase { + float ratio; //ratio of regionservers to restart + + public BatchRestartRs(long sleepTime, float ratio) { + super(sleepTime); + this.ratio = ratio; + } + + @Override + void perform() throws Exception { + LOG.info(String.format("Performing action: Batch restarting %d%% of region servers", + (int)(ratio * 100))); + List selectedServers = selectRandomItems(getCurrentServers(), ratio); + + for (ServerName server : selectedServers) { + LOG.info("Killing region server:" + server); + cluster.killRegionServer(server); + } + + for (ServerName server : selectedServers) { + cluster.waitForRegionServerToStop(server, TIMEOUT); + } + + LOG.info("Killed " + selectedServers.size() + " region servers. Reported num of rs:" + + cluster.getClusterStatus().getServersSize()); + + sleep(sleepTime); + + for (ServerName server : selectedServers) { + LOG.info("Starting region server:" + server.getHostname()); + cluster.startRegionServer(server.getHostname()); + + } + for (ServerName server : selectedServers) { + cluster.waitForRegionServerToStart(server.getHostname(), TIMEOUT); + } + LOG.info("Started " + selectedServers.size() +" region servers. Reported num of rs:" + + cluster.getClusterStatus().getServersSize()); + } + } + + /** + * Restarts a ratio of the regionservers in a rolling fashion. At each step, either kills a + * server, or starts one, sleeping randomly (0-sleepTime) in between steps. + */ + public static class RollingBatchRestartRs extends BatchRestartRs { + public RollingBatchRestartRs(long sleepTime, float ratio) { + super(sleepTime, ratio); + } + + @Override + void perform() throws Exception { + LOG.info(String.format("Performing action: Rolling batch restarting %d%% of region servers", + (int)(ratio * 100))); + Random random = new Random(); + List selectedServers = selectRandomItems(getCurrentServers(), ratio); + + Queue serversToBeKilled = new LinkedList(selectedServers); + Queue deadServers = new LinkedList(); + + // + while (!serversToBeKilled.isEmpty() || !deadServers.isEmpty()) { + boolean action = true; //action true = kill server, false = start server + + if (serversToBeKilled.isEmpty() || deadServers.isEmpty()) { + action = deadServers.isEmpty(); + } else { + action = random.nextBoolean(); + } + + if (action) { + ServerName server = serversToBeKilled.remove(); + killRs(server); + deadServers.add(server); + } else { + ServerName server = deadServers.remove(); + startRs(server); + } + + sleep(random.nextInt((int)sleepTime)); + } + } + } + + public static class UnbalanceRegionsAction extends Action { + private double fractionOfRegions; + private double fractionOfServers; + private Random random = new Random(); + + /** + * Unbalances the regions on the cluster by choosing "target" servers, and moving + * some regions from each of the non-target servers to random target servers. + * @param fractionOfRegions Fraction of regions to move from each server. + * @param fractionOfServers Fraction of servers to be chosen as targets. + */ + public UnbalanceRegionsAction(double fractionOfRegions, double fractionOfServers) { + this.fractionOfRegions = fractionOfRegions; + this.fractionOfServers = fractionOfServers; + } + + @Override + void perform() throws Exception { + LOG.info("Unbalancing regions"); + ClusterStatus status = this.cluster.getClusterStatus(); + List victimServers = new LinkedList(status.getServers()); + int targetServerCount = (int)Math.ceil(fractionOfServers * victimServers.size()); + List targetServers = new ArrayList(targetServerCount); + for (int i = 0; i < targetServerCount; ++i) { + int victimIx = random.nextInt(victimServers.size()); + String serverName = victimServers.remove(victimIx).getServerName(); + targetServers.add(Bytes.toBytes(serverName)); + } + + List victimRegions = new LinkedList(); + for (ServerName server : victimServers) { + HServerLoad serverLoad = status.getLoad(server); + // Ugh. + List regions = new LinkedList(serverLoad.getRegionsLoad().keySet()); + int victimRegionCount = (int)Math.ceil(fractionOfRegions * regions.size()); + LOG.debug("Removing " + victimRegionCount + " regions from " + server.getServerName()); + for (int i = 0; i < victimRegionCount; ++i) { + int victimIx = random.nextInt(regions.size()); + String regionId = HRegionInfo.encodeRegionName(regions.remove(victimIx)); + victimRegions.add(Bytes.toBytes(regionId)); + } + } + + LOG.info("Moving " + victimRegions.size() + " regions from " + victimServers.size() + + " servers to " + targetServers.size() + " different servers"); + HBaseAdmin admin = this.context.getHaseIntegrationTestingUtility().getHBaseAdmin(); + for (byte[] victimRegion : victimRegions) { + int targetIx = random.nextInt(targetServers.size()); + admin.move(victimRegion, targetServers.get(targetIx)); + } + } + } + + public static class ForceBalancerAction extends Action { + @Override + void perform() throws Exception { + LOG.info("Balancing regions"); + HBaseAdmin admin = this.context.getHaseIntegrationTestingUtility().getHBaseAdmin(); + boolean result = admin.balancer(); + if (!result) { + LOG.error("Balancer didn't succeed"); + } + } + } + + /** + * A context for a Policy + */ + private static class PolicyContext extends ActionContext { + PolicyContext(IntegrationTestingUtility util) { + super(util); + } + } + + /** + * A policy to introduce chaos to the cluster + */ + public static abstract class Policy extends StoppableImplementation implements Runnable { + PolicyContext context; + public void init(PolicyContext context) throws Exception { + this.context = context; + } + } + + /** A policy that runs multiple other policies one after the other */ + public static class CompositeSequentialPolicy extends Policy { + private List policies; + public CompositeSequentialPolicy(Policy... policies) { + this.policies = Arrays.asList(policies); + } + + @Override + public void stop(String why) { + super.stop(why); + for (Policy p : policies) { + p.stop(why); + } + } + + @Override + public void run() { + for (Policy p : policies) { + p.run(); + } + } + + @Override + public void init(PolicyContext context) throws Exception { + super.init(context); + for (Policy p : policies) { + p.init(context); + } + } + } + + /** A policy which does stuff every time interval. */ + public static abstract class PeriodicPolicy extends Policy { + private long periodMs; + + public PeriodicPolicy(long periodMs) { + this.periodMs = periodMs; + } + + @Override + public void run() { + // Add some jitter. + int jitter = new Random().nextInt((int)periodMs); + LOG.info("Sleeping for " + jitter + " to add jitter"); + Threads.sleep(jitter); + + while (!isStopped()) { + long start = System.currentTimeMillis(); + runOneIteration(); + + if (isStopped()) return; + long sleepTime = periodMs - (System.currentTimeMillis() - start); + if (sleepTime > 0) { + LOG.info("Sleeping for: " + sleepTime); + Threads.sleep(sleepTime); + } + } + } + + protected abstract void runOneIteration(); + + @Override + public void init(PolicyContext context) throws Exception { + super.init(context); + LOG.info("Using ChaosMonkey Policy: " + this.getClass() + ", period: " + periodMs); + } + } + + + /** A policy which performs a sequence of actions deterministically. */ + public static class DoActionsOncePolicy extends PeriodicPolicy { + private List actions; + + public DoActionsOncePolicy(long periodMs, List actions) { + super(periodMs); + this.actions = new ArrayList(actions); + } + + public DoActionsOncePolicy(long periodMs, Action... actions) { + this(periodMs, Arrays.asList(actions)); + } + + @Override + protected void runOneIteration() { + if (actions.isEmpty()) { + this.stop("done"); + return; + } + Action action = actions.remove(0); + + try { + action.perform(); + } catch (Exception ex) { + LOG.warn("Exception occured during performing action: " + + StringUtils.stringifyException(ex)); + } + } + + @Override + public void init(PolicyContext context) throws Exception { + super.init(context); + for (Action action : actions) { + action.init(this.context); + } + } + } + + /** + * A policy, which picks a random action according to the given weights, + * and performs it every configurable period. + */ + public static class PeriodicRandomActionPolicy extends PeriodicPolicy { + private List> actions; + + public PeriodicRandomActionPolicy(long periodMs, List> actions) { + super(periodMs); + this.actions = actions; + } + + public PeriodicRandomActionPolicy(long periodMs, Pair... actions) { + // We don't expect it to be modified. + this(periodMs, Arrays.asList(actions)); + } + + public PeriodicRandomActionPolicy(long periodMs, Action... actions) { + super(periodMs); + this.actions = new ArrayList>(actions.length); + for (Action action : actions) { + this.actions.add(new Pair(action, 1)); + } + } + + @Override + protected void runOneIteration() { + Action action = selectWeightedRandomItem(actions); + try { + action.perform(); + } catch (Exception ex) { + LOG.warn("Exception occured during performing action: " + + StringUtils.stringifyException(ex)); + } + } + + @Override + public void init(PolicyContext context) throws Exception { + super.init(context); + for (Pair action : actions) { + action.getFirst().init(this.context); + } + } + } + + /** Selects a random item from the given items */ + static T selectRandomItem(T[] items) { + Random random = new Random(); + return items[random.nextInt(items.length)]; + } + + /** Selects a random item from the given items with weights*/ + static T selectWeightedRandomItem(List> items) { + Random random = new Random(); + int totalWeight = 0; + for (Pair pair : items) { + totalWeight += pair.getSecond(); + } + + int cutoff = random.nextInt(totalWeight); + int cummulative = 0; + T item = null; + + //warn: O(n) + for (int i=0; i List selectRandomItems(T[] items, float ratio) { + Random random = new Random(); + int remaining = (int)Math.ceil(items.length * ratio); + + List selectedItems = new ArrayList(remaining); + + for (int i=0; i 0; i++) { + if (random.nextFloat() < ((float)remaining/(items.length-i))) { + selectedItems.add(items[i]); + remaining--; + } + } + + return selectedItems; + } + + /** + * All actions that deal with RS's with the following weights (relative probabilities): + * - Restart active master (sleep 5 sec) : 2 + * - Restart random regionserver (sleep 5 sec) : 2 + * - Restart random regionserver (sleep 60 sec) : 2 + * - Restart META regionserver (sleep 5 sec) : 1 + * - Restart ROOT regionserver (sleep 5 sec) : 1 + * - Batch restart of 50% of regionservers (sleep 5 sec) : 2 + * - Rolling restart of 100% of regionservers (sleep 5 sec) : 2 + */ + @SuppressWarnings("unchecked") + private static final List> ALL_ACTIONS = Lists.newArrayList( + new Pair(new RestartActiveMaster(FIVE_SEC), 2), + new Pair(new RestartRandomRs(FIVE_SEC), 2), + new Pair(new RestartRandomRs(ONE_MIN), 2), + new Pair(new RestartRsHoldingMeta(FIVE_SEC), 1), + new Pair(new RestartRsHoldingRoot(FIVE_SEC), 1), + new Pair(new BatchRestartRs(FIVE_SEC, 0.5f), 2), + new Pair(new RollingBatchRestartRs(FIVE_SEC, 1.0f), 2) + ); + + public static final String EVERY_MINUTE_RANDOM_ACTION_POLICY = "EVERY_MINUTE_RANDOM_ACTION_POLICY"; + + private Policy[] policies; + private Thread[] monkeyThreads; + + public void start() throws Exception { + monkeyThreads = new Thread[policies.length]; + + for (int i=0; i NAMED_POLICIES = Maps.newHashMap(); + static { + NAMED_POLICIES.put(EVERY_MINUTE_RANDOM_ACTION_POLICY, + new PeriodicRandomActionPolicy(ONE_MIN, ALL_ACTIONS)); + } + + @Override + protected void addOptions() { + addOptWithArg("policy", "a named policy defined in ChaosMonkey.java. Possible values: " + + NAMED_POLICIES.keySet()); + //we can add more options, and make policies more configurable + } + + @Override + protected void processOptions(CommandLine cmd) { + String[] policies = cmd.getOptionValues("policy"); + if (policies != null) { + setPoliciesByName(policies); + } + } + + @Override + protected int doWork() throws Exception { + start(); + waitForStop(); + return 0; + } + + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + IntegrationTestingUtility.setUseDistributedCluster(conf); + IntegrationTestingUtility util = new IntegrationTestingUtility(conf); + util.initializeCluster(1); + + ChaosMonkey monkey = new ChaosMonkey(util, EVERY_MINUTE_RANDOM_ACTION_POLICY); + int ret = ToolRunner.run(conf, monkey, args); + System.exit(ret); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java b/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java new file mode 100644 index 0000000..55a3feb --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertTrue; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; + +import javax.tools.JavaCompiler; +import javax.tools.JavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; + +/** + * Some utilities to help class loader testing + */ +public class ClassLoaderTestHelper { + private static final Log LOG = LogFactory.getLog(ClassLoaderTestHelper.class); + + /** + * Jar a list of files into a jar archive. + * + * @param archiveFile the target jar archive + * @param tobejared a list of files to be jared + */ + private static boolean createJarArchive(File archiveFile, File[] tobeJared) { + try { + byte buffer[] = new byte[4096]; + // Open archive file + FileOutputStream stream = new FileOutputStream(archiveFile); + JarOutputStream out = new JarOutputStream(stream, new Manifest()); + + for (int i = 0; i < tobeJared.length; i++) { + if (tobeJared[i] == null || !tobeJared[i].exists() + || tobeJared[i].isDirectory()) { + continue; + } + + // Add archive entry + JarEntry jarAdd = new JarEntry(tobeJared[i].getName()); + jarAdd.setTime(tobeJared[i].lastModified()); + out.putNextEntry(jarAdd); + + // Write file to archive + FileInputStream in = new FileInputStream(tobeJared[i]); + while (true) { + int nRead = in.read(buffer, 0, buffer.length); + if (nRead <= 0) + break; + out.write(buffer, 0, nRead); + } + in.close(); + } + out.close(); + stream.close(); + LOG.info("Adding classes to jar file completed"); + return true; + } catch (Exception ex) { + LOG.error("Error: " + ex.getMessage()); + return false; + } + } + + /** + * Create a test jar for testing purpose for a given class + * name with specified code string: save the class to a file, + * compile it, and jar it up. If the code string passed in is + * null, a bare empty class will be created and used. + * + * @param testDir the folder under which to store the test class and jar + * @param className the test class name + * @param code the optional test class code, which can be null. + * If null, a bare empty class will be used + * @return the test jar file generated + */ + public static File buildJar(String testDir, + String className, String code) throws Exception { + return buildJar(testDir, className, code, testDir); + } + + /** + * Create a test jar for testing purpose for a given class + * name with specified code string. + * + * @param testDir the folder under which to store the test class + * @param className the test class name + * @param code the optional test class code, which can be null. + * If null, an empty class will be used + * @param folder the folder under which to store the generated jar + * @return the test jar file generated + */ + public static File buildJar(String testDir, + String className, String code, String folder) throws Exception { + String javaCode = code != null ? code : "public class " + className + " {}"; + Path srcDir = new Path(testDir, "src"); + File srcDirPath = new File(srcDir.toString()); + srcDirPath.mkdirs(); + File sourceCodeFile = new File(srcDir.toString(), className + ".java"); + BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile)); + bw.write(javaCode); + bw.close(); + + // compile it by JavaCompiler + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + ArrayList srcFileNames = new ArrayList(); + srcFileNames.add(sourceCodeFile.toString()); + StandardJavaFileManager fm = compiler.getStandardFileManager(null, null, + null); + Iterable cu = + fm.getJavaFileObjects(sourceCodeFile); + List options = new ArrayList(); + options.add("-classpath"); + // only add hbase classes to classpath. This is a little bit tricky: assume + // the classpath is {hbaseSrc}/target/classes. + String currentDir = new File(".").getAbsolutePath(); + String classpath = currentDir + File.separator + "target"+ File.separator + + "classes" + System.getProperty("path.separator") + + System.getProperty("java.class.path") + System.getProperty("path.separator") + + System.getProperty("surefire.test.class.path"); + options.add(classpath); + LOG.debug("Setting classpath to: " + classpath); + + JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null, + options, null, cu); + assertTrue("Compile file " + sourceCodeFile + " failed.", task.call()); + + // build a jar file by the classes files + String jarFileName = className + ".jar"; + File jarFile = new File(folder, jarFileName); + if (!createJarArchive(jarFile, + new File[]{new File(srcDir.toString(), className + ".class")})){ + assertTrue("Build jar file failed.", false); + } + return jarFile; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java b/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java new file mode 100644 index 0000000..730f4e3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java @@ -0,0 +1,36 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +/** + * Used by tests to inject an edge into the manager. The intent is to minimise + * the use of the injectEdge method giving it default permissions, but in + * testing we may need to use this functionality elsewhere. + */ +public class EnvironmentEdgeManagerTestHelper { + + public static void reset() { + EnvironmentEdgeManager.reset(); + } + + public static void injectEdge(EnvironmentEdge edge) { + EnvironmentEdgeManager.injectEdge(edge); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java b/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java new file mode 100644 index 0000000..7609e0e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.Store; + +/** + * Test helper for testing archiving of HFiles + */ +public class HFileArchiveTestingUtil { + + private static final Log LOG = LogFactory.getLog(HFileArchiveTestingUtil.class); + + private HFileArchiveTestingUtil() { + // NOOP private ctor since this is just a utility class + } + + public static boolean compareArchiveToOriginal(FileStatus[] previous, FileStatus[] archived, + FileSystem fs, boolean hasTimedBackup) { + + List> lists = getFileLists(previous, archived); + List original = lists.get(0); + Collections.sort(original); + + List currentFiles = lists.get(1); + Collections.sort(currentFiles); + + List backedup = lists.get(2); + Collections.sort(backedup); + + // check the backed up files versus the current (should match up, less the + // backup time in the name) + if (!hasTimedBackup == (backedup.size() > 0)) { + LOG.debug("backedup files doesn't match expected."); + return false; + } + String msg = null; + if (hasTimedBackup) { + msg = assertArchiveEquality(original, backedup); + if (msg != null) { + LOG.debug(msg); + return false; + } + } + msg = assertArchiveEquality(original, currentFiles); + if (msg != null) { + LOG.debug(msg); + return false; + } + return true; + } + + /** + * Compare the archived files to the files in the original directory + * @param previous original files that should have been archived + * @param archived files that were archived + * @param fs filessystem on which the archiving took place + * @throws IOException + */ + public static void assertArchiveEqualToOriginal(FileStatus[] previous, FileStatus[] archived, + FileSystem fs) throws IOException { + assertArchiveEqualToOriginal(previous, archived, fs, false); + } + + /** + * Compare the archived files to the files in the original directory + * @param previous original files that should have been archived + * @param archived files that were archived + * @param fs {@link FileSystem} on which the archiving took place + * @param hasTimedBackup true if we expect to find an archive backup directory with a + * copy of the files in the archive directory (and the original files). + * @throws IOException + */ + public static void assertArchiveEqualToOriginal(FileStatus[] previous, FileStatus[] archived, + FileSystem fs, boolean hasTimedBackup) throws IOException { + + List> lists = getFileLists(previous, archived); + List original = lists.get(0); + Collections.sort(original); + + List currentFiles = lists.get(1); + Collections.sort(currentFiles); + + List backedup = lists.get(2); + Collections.sort(backedup); + + // check the backed up files versus the current (should match up, less the + // backup time in the name) + assertEquals("Didn't expect any backup files, but got: " + backedup, hasTimedBackup, + backedup.size() > 0); + String msg = null; + if (hasTimedBackup) { + assertArchiveEquality(original, backedup); + assertNull(msg, msg); + } + + // do the rest of the comparison + msg = assertArchiveEquality(original, currentFiles); + assertNull(msg, msg); + } + + private static String assertArchiveEquality(List expected, List archived) { + String compare = compareFileLists(expected, archived); + if (!(expected.size() == archived.size())) return "Not the same number of current files\n" + + compare; + if (!expected.equals(archived)) return "Different backup files, but same amount\n" + compare; + return null; + } + + /** + * @return , where each is sorted + */ + private static List> getFileLists(FileStatus[] previous, FileStatus[] archived) { + List> files = new ArrayList>(); + + // copy over the original files + List originalFileNames = convertToString(previous); + files.add(originalFileNames); + + List currentFiles = new ArrayList(previous.length); + List backedupFiles = new ArrayList(previous.length); + for (FileStatus f : archived) { + String name = f.getPath().getName(); + // if the file has been backed up + if (name.contains(".")) { + Path parent = f.getPath().getParent(); + String shortName = name.split("[.]")[0]; + Path modPath = new Path(parent, shortName); + FileStatus file = new FileStatus(f.getLen(), f.isDir(), f.getReplication(), + f.getBlockSize(), f.getModificationTime(), modPath); + backedupFiles.add(file); + } else { + // otherwise, add it to the list to compare to the original store files + currentFiles.add(name); + } + } + + files.add(currentFiles); + files.add(convertToString(backedupFiles)); + return files; + } + + private static List convertToString(FileStatus[] files) { + return convertToString(Arrays.asList(files)); + } + + private static List convertToString(List files) { + List originalFileNames = new ArrayList(files.size()); + for (FileStatus f : files) { + originalFileNames.add(f.getPath().getName()); + } + return originalFileNames; + } + + /* Get a pretty representation of the differences */ + private static String compareFileLists(List expected, List gotten) { + StringBuilder sb = new StringBuilder("Expected (" + expected.size() + "): \t\t Gotten (" + + gotten.size() + "):\n"); + List notFound = new ArrayList(); + for (String s : expected) { + if (gotten.contains(s)) sb.append(s + "\t\t" + s + "\n"); + else notFound.add(s); + } + sb.append("Not Found:\n"); + for (String s : notFound) { + sb.append(s + "\n"); + } + sb.append("\nExtra:\n"); + for (String s : gotten) { + if (!expected.contains(s)) sb.append(s + "\n"); + } + return sb.toString(); + } + + /** + * Helper method to get the archive directory for the specified region + * @param conf {@link Configuration} to check for the name of the archive directory + * @param region region that is being archived + * @return {@link Path} to the archive directory for the given region + */ + public static Path getRegionArchiveDir(Configuration conf, HRegion region) { + return HFileArchiveUtil.getRegionArchiveDir(conf, region.getTableDir(), region.getRegionDir()); + } + + /** + * Helper method to get the store archive directory for the specified region + * @param conf {@link Configuration} to check for the name of the archive directory + * @param region region that is being archived + * @param store store that is archiving files + * @return {@link Path} to the store archive directory for the given region + */ + public static Path getStoreArchivePath(Configuration conf, HRegion region, Store store) { + return HFileArchiveUtil.getStoreArchivePath(conf, region, store.getFamily().getName()); + } + + public static Path getStoreArchivePath(HBaseTestingUtility util, String tableName, + byte[] storeName) throws IOException { + byte[] table = Bytes.toBytes(tableName); + // get the RS and region serving our table + List servingRegions = util.getHBaseCluster().getRegions(table); + HRegion region = servingRegions.get(0); + + // check that we actually have some store files that were archived + Store store = region.getStore(storeName); + return HFileArchiveTestingUtil.getStoreArchivePath(util.getConfiguration(), region, store); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGenerator.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGenerator.java new file mode 100644 index 0000000..c7f6dd6 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGenerator.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.util.Set; + +/** + * A generator of random data (keys/cfs/columns/values) for load testing. + * Contains LoadTestKVGenerator as a matter of convenience... + */ +public abstract class LoadTestDataGenerator { + protected final LoadTestKVGenerator kvGenerator; + + /** + * Initializes the object. + * @param minValueSize minimum size of the value generated by + * {@link #generateValue(byte[], byte[], byte[])}. + * @param maxValueSize maximum size of the value generated by + * {@link #generateValue(byte[], byte[], byte[])}. + */ + public LoadTestDataGenerator(int minValueSize, int maxValueSize) { + this.kvGenerator = new LoadTestKVGenerator(minValueSize, maxValueSize); + } + /** + * Generates a deterministic, unique hashed row key from a number. That way, the user can + * keep track of numbers, without messing with byte array and ensuring key distribution. + * @param keyBase Base number for a key, such as a loop counter. + */ + public abstract byte[] getDeterministicUniqueKey(long keyBase); + + /** + * Gets column families for the load test table. + * @return The array of byte[]s representing column family names. + */ + public abstract byte[][] getColumnFamilies(); + + /** + * Generates an applicable set of columns to be used for a particular key and family. + * @param rowKey The row key to generate for. + * @param cf The column family name to generate for. + * @return The array of byte[]s representing column names. + */ + public abstract byte[][] generateColumnsForCf(byte[] rowKey, byte[] cf); + + /** + * Generates a value to be used for a particular row/cf/column. + * @param rowKey The row key to generate for. + * @param cf The column family name to generate for. + * @param column The column name to generate for. + * @return The value to use. + */ + public abstract byte[] generateValue(byte[] rowKey, byte[] cf, byte[] column); + + /** + * Checks that columns for a rowKey and cf are valid if generated via + * {@link #generateColumnsForCf(byte[], byte[])} + * @param rowKey The row key to verify for. + * @param cf The column family name to verify for. + * @param columnSet The column set (for example, encountered by read). + * @return True iff valid. + */ + public abstract boolean verify(byte[] rowKey, byte[] cf, Set columnSet); + + /** + * Checks that value for a rowKey/cf/column is valid if generated via + * {@link #generateValue(byte[], byte[], byte[])} + * @param rowKey The row key to verify for. + * @param cf The column family name to verify for. + * @param column The column name to verify for. + * @param value The value (for example, encountered by read). + * @return True iff valid. + */ + public abstract boolean verify(byte[] rowKey, byte[] cf, byte[] column, byte[] value); +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java new file mode 100644 index 0000000..77f5420 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.Random; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.MD5Hash; + +/** + * A generator of random keys and values for load testing. Keys are generated + * by converting numeric indexes to strings and prefixing them with an MD5 + * hash. Values are generated by selecting value size in the configured range + * and generating a pseudo-random sequence of bytes seeded by key, column + * qualifier, and value size. + */ +public class LoadTestKVGenerator { + + /** A random number generator for determining value size */ + private Random randomForValueSize = new Random(); + + private final int minValueSize; + private final int maxValueSize; + + public LoadTestKVGenerator(int minValueSize, int maxValueSize) { + if (minValueSize <= 0 || maxValueSize <= 0) { + throw new IllegalArgumentException("Invalid min/max value sizes: " + + minValueSize + ", " + maxValueSize); + } + this.minValueSize = minValueSize; + this.maxValueSize = maxValueSize; + } + + /** + * Verifies that the given byte array is the same as what would be generated + * for the given seed strings (row/cf/column/...). We are assuming that the + * value size is correct, and only verify the actual bytes. However, if the + * min/max value sizes are set sufficiently high, an accidental match should be + * extremely improbable. + */ + public static boolean verify(byte[] value, byte[]... seedStrings) { + byte[] expectedData = getValueForRowColumn(value.length, seedStrings); + return Bytes.equals(expectedData, value); + } + + /** + * Converts the given key to string, and prefixes it with the MD5 hash of + * the index's string representation. + */ + public static String md5PrefixedKey(long key) { + String stringKey = Long.toString(key); + String md5hash = MD5Hash.getMD5AsHex(Bytes.toBytes(stringKey)); + + // flip the key to randomize + return md5hash + "-" + stringKey; + } + + /** + * Generates a value for the given key index and column qualifier. Size is + * selected randomly in the configured range. The generated value depends + * only on the combination of the strings passed (key/cf/column/...) and the selected + * value size. This allows to verify the actual value bytes when reading, as done + * in {#verify(byte[], byte[]...)} + * This method is as thread-safe as Random class. It appears that the worst bug ever + * found with the latter is that multiple threads will get some duplicate values, which + * we don't care about. + */ + public byte[] generateRandomSizeValue(byte[]... seedStrings) { + int dataSize = minValueSize; + if (minValueSize != maxValueSize) { + dataSize = minValueSize + randomForValueSize.nextInt(Math.abs(maxValueSize - minValueSize)); + } + return getValueForRowColumn(dataSize, seedStrings); + } + + /** + * Generates random bytes of the given size for the given row and column + * qualifier. The random seed is fully determined by these parameters. + */ + private static byte[] getValueForRowColumn(int dataSize, byte[]... seedStrings) { + long seed = dataSize; + for (byte[] str : seedStrings) { + seed += Bytes.toString(str).hashCode(); + } + Random seededRandom = new Random(seed); + byte[] randomBytes = new byte[dataSize]; + seededRandom.nextBytes(randomBytes); + return randomBytes; + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java new file mode 100644 index 0000000..3761242 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java @@ -0,0 +1,397 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.PerformanceEvaluation; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.regionserver.StoreFile; + +/** + * A command-line utility that reads, writes, and verifies data. Unlike + * {@link PerformanceEvaluation}, this tool validates the data written, + * and supports simultaneously writing and reading the same set of keys. + */ +public class LoadTestTool extends AbstractHBaseTool { + + private static final Log LOG = LogFactory.getLog(LoadTestTool.class); + + /** Table name for the test */ + private byte[] tableName; + + /** Table name to use of not overridden on the command line */ + private static final String DEFAULT_TABLE_NAME = "cluster_test"; + + /** Column family used by the test */ + static byte[] COLUMN_FAMILY = Bytes.toBytes("test_cf"); + + /** Column families used by the test */ + static final byte[][] COLUMN_FAMILIES = { COLUMN_FAMILY }; + + /** The number of reader/writer threads if not specified */ + private static final int DEFAULT_NUM_THREADS = 20; + + /** Usage string for the load option */ + private static final String OPT_USAGE_LOAD = + ":" + + "[:<#threads=" + DEFAULT_NUM_THREADS + ">]"; + + /** Usa\ge string for the read option */ + private static final String OPT_USAGE_READ = + "[:<#threads=" + DEFAULT_NUM_THREADS + ">]"; + + private static final String OPT_USAGE_BLOOM = "Bloom filter type, one of " + + Arrays.toString(StoreFile.BloomType.values()); + + private static final String OPT_USAGE_COMPRESSION = "Compression type, " + + "one of " + Arrays.toString(Compression.Algorithm.values()); + + public static final String OPT_DATA_BLOCK_ENCODING_USAGE = + "Encoding algorithm (e.g. prefix " + + "compression) to use for data blocks in the test column family, " + + "one of " + Arrays.toString(DataBlockEncoding.values()) + "."; + + private static final String OPT_BLOOM = "bloom"; + private static final String OPT_COMPRESSION = "compression"; + public static final String OPT_DATA_BLOCK_ENCODING = + HColumnDescriptor.DATA_BLOCK_ENCODING.toLowerCase(); + public static final String OPT_ENCODE_IN_CACHE_ONLY = + "encode_in_cache_only"; + public static final String OPT_ENCODE_IN_CACHE_ONLY_USAGE = + "If this is specified, data blocks will only be encoded in block " + + "cache but not on disk"; + + private static final String OPT_KEY_WINDOW = "key_window"; + private static final String OPT_WRITE = "write"; + private static final String OPT_MAX_READ_ERRORS = "max_read_errors"; + private static final String OPT_MULTIPUT = "multiput"; + private static final String OPT_NUM_KEYS = "num_keys"; + private static final String OPT_READ = "read"; + private static final String OPT_START_KEY = "start_key"; + private static final String OPT_TABLE_NAME = "tn"; + private static final String OPT_ZK_QUORUM = "zk"; + private static final String OPT_SKIP_INIT = "skip_init"; + private static final String OPT_INIT_ONLY = "init_only"; + + private static final long DEFAULT_START_KEY = 0; + + /** This will be removed as we factor out the dependency on command line */ + private CommandLine cmd; + + private MultiThreadedWriter writerThreads = null; + private MultiThreadedReader readerThreads = null; + + private long startKey, endKey; + + private boolean isWrite, isRead; + + // Column family options + private DataBlockEncoding dataBlockEncodingAlgo; + private boolean encodeInCacheOnly; + private Compression.Algorithm compressAlgo; + private StoreFile.BloomType bloomType; + + // Writer options + private int numWriterThreads = DEFAULT_NUM_THREADS; + private int minColsPerKey, maxColsPerKey; + private int minColDataSize, maxColDataSize; + private boolean isMultiPut; + + // Reader options + private int numReaderThreads = DEFAULT_NUM_THREADS; + private int keyWindow = MultiThreadedReader.DEFAULT_KEY_WINDOW; + private int maxReadErrors = MultiThreadedReader.DEFAULT_MAX_ERRORS; + private int verifyPercent; + + // TODO: refactor LoadTestToolImpl somewhere to make the usage from tests less bad, + // console tool itself should only be used from console. + private boolean isSkipInit = false; + private boolean isInitOnly = false; + + private String[] splitColonSeparated(String option, + int minNumCols, int maxNumCols) { + String optVal = cmd.getOptionValue(option); + String[] cols = optVal.split(":"); + if (cols.length < minNumCols || cols.length > maxNumCols) { + throw new IllegalArgumentException("Expected at least " + + minNumCols + " columns but no more than " + maxNumCols + + " in the colon-separated value '" + optVal + "' of the " + + "-" + option + " option"); + } + return cols; + } + + private int getNumThreads(String numThreadsStr) { + return parseInt(numThreadsStr, 1, Short.MAX_VALUE); + } + + /** + * Apply column family options such as Bloom filters, compression, and data + * block encoding. + */ + private void applyColumnFamilyOptions(byte[] tableName, + byte[][] columnFamilies) throws IOException { + HBaseAdmin admin = new HBaseAdmin(conf); + HTableDescriptor tableDesc = admin.getTableDescriptor(tableName); + LOG.info("Disabling table " + Bytes.toString(tableName)); + admin.disableTable(tableName); + for (byte[] cf : columnFamilies) { + HColumnDescriptor columnDesc = tableDesc.getFamily(cf); + boolean isNewCf = columnDesc == null; + if (isNewCf) { + columnDesc = new HColumnDescriptor(cf); + } + if (bloomType != null) { + columnDesc.setBloomFilterType(bloomType); + } + if (compressAlgo != null) { + columnDesc.setCompressionType(compressAlgo); + } + if (dataBlockEncodingAlgo != null) { + columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo); + columnDesc.setEncodeOnDisk(!encodeInCacheOnly); + } + if (isNewCf) { + admin.addColumn(tableName, columnDesc); + } else { + admin.modifyColumn(tableName, columnDesc); + } + } + LOG.info("Enabling table " + Bytes.toString(tableName)); + admin.enableTable(tableName); + } + + @Override + protected void addOptions() { + addOptWithArg(OPT_ZK_QUORUM, "ZK quorum as comma-separated host names " + + "without port numbers"); + addOptWithArg(OPT_TABLE_NAME, "The name of the table to read or write"); + addOptWithArg(OPT_WRITE, OPT_USAGE_LOAD); + addOptWithArg(OPT_READ, OPT_USAGE_READ); + addOptNoArg(OPT_INIT_ONLY, "Initialize the test table only, don't do any loading"); + addOptWithArg(OPT_BLOOM, OPT_USAGE_BLOOM); + addOptWithArg(OPT_COMPRESSION, OPT_USAGE_COMPRESSION); + addOptWithArg(OPT_DATA_BLOCK_ENCODING, OPT_DATA_BLOCK_ENCODING_USAGE); + addOptWithArg(OPT_MAX_READ_ERRORS, "The maximum number of read errors " + + "to tolerate before terminating all reader threads. The default is " + + MultiThreadedReader.DEFAULT_MAX_ERRORS + "."); + addOptWithArg(OPT_KEY_WINDOW, "The 'key window' to maintain between " + + "reads and writes for concurrent write/read workload. The default " + + "is " + MultiThreadedReader.DEFAULT_KEY_WINDOW + "."); + + addOptNoArg(OPT_MULTIPUT, "Whether to use multi-puts as opposed to " + + "separate puts for every column in a row"); + addOptNoArg(OPT_ENCODE_IN_CACHE_ONLY, OPT_ENCODE_IN_CACHE_ONLY_USAGE); + + addOptWithArg(OPT_NUM_KEYS, "The number of keys to read/write"); + addOptWithArg(OPT_START_KEY, "The first key to read/write " + + "(a 0-based index). The default value is " + + DEFAULT_START_KEY + "."); + addOptNoArg(OPT_SKIP_INIT, "Skip the initialization; assume test table " + + "already exists"); + } + + @Override + protected void processOptions(CommandLine cmd) { + this.cmd = cmd; + + tableName = Bytes.toBytes(cmd.getOptionValue(OPT_TABLE_NAME, + DEFAULT_TABLE_NAME)); + + isWrite = cmd.hasOption(OPT_WRITE); + isRead = cmd.hasOption(OPT_READ); + isInitOnly = cmd.hasOption(OPT_INIT_ONLY); + + if (!isWrite && !isRead && !isInitOnly) { + throw new IllegalArgumentException("Either -" + OPT_WRITE + " or " + + "-" + OPT_READ + " has to be specified"); + } + + if (isInitOnly && (isRead || isWrite)) { + throw new IllegalArgumentException(OPT_INIT_ONLY + " cannot be specified with" + + " either -" + OPT_WRITE + " or -" + OPT_READ); + } + + if (!isInitOnly) { + if (!cmd.hasOption(OPT_NUM_KEYS)) { + throw new IllegalArgumentException(OPT_NUM_KEYS + " must be specified in " + + "read or write mode"); + } + startKey = parseLong(cmd.getOptionValue(OPT_START_KEY, + String.valueOf(DEFAULT_START_KEY)), 0, Long.MAX_VALUE); + long numKeys = parseLong(cmd.getOptionValue(OPT_NUM_KEYS), 1, + Long.MAX_VALUE - startKey); + endKey = startKey + numKeys; + isSkipInit = cmd.hasOption(OPT_SKIP_INIT); + System.out.println("Key range: [" + startKey + ".." + (endKey - 1) + "]"); + } + + encodeInCacheOnly = cmd.hasOption(OPT_ENCODE_IN_CACHE_ONLY); + parseColumnFamilyOptions(cmd); + + if (isWrite) { + String[] writeOpts = splitColonSeparated(OPT_WRITE, 2, 3); + + int colIndex = 0; + minColsPerKey = 1; + maxColsPerKey = 2 * Integer.parseInt(writeOpts[colIndex++]); + int avgColDataSize = + parseInt(writeOpts[colIndex++], 1, Integer.MAX_VALUE); + minColDataSize = avgColDataSize / 2; + maxColDataSize = avgColDataSize * 3 / 2; + + if (colIndex < writeOpts.length) { + numWriterThreads = getNumThreads(writeOpts[colIndex++]); + } + + isMultiPut = cmd.hasOption(OPT_MULTIPUT); + + System.out.println("Multi-puts: " + isMultiPut); + System.out.println("Columns per key: " + minColsPerKey + ".." + + maxColsPerKey); + System.out.println("Data size per column: " + minColDataSize + ".." + + maxColDataSize); + } + + if (isRead) { + String[] readOpts = splitColonSeparated(OPT_READ, 1, 2); + int colIndex = 0; + verifyPercent = parseInt(readOpts[colIndex++], 0, 100); + if (colIndex < readOpts.length) { + numReaderThreads = getNumThreads(readOpts[colIndex++]); + } + + if (cmd.hasOption(OPT_MAX_READ_ERRORS)) { + maxReadErrors = parseInt(cmd.getOptionValue(OPT_MAX_READ_ERRORS), + 0, Integer.MAX_VALUE); + } + + if (cmd.hasOption(OPT_KEY_WINDOW)) { + keyWindow = parseInt(cmd.getOptionValue(OPT_KEY_WINDOW), + 0, Integer.MAX_VALUE); + } + + System.out.println("Percent of keys to verify: " + verifyPercent); + System.out.println("Reader threads: " + numReaderThreads); + } + } + + private void parseColumnFamilyOptions(CommandLine cmd) { + String dataBlockEncodingStr = cmd.getOptionValue(OPT_DATA_BLOCK_ENCODING); + dataBlockEncodingAlgo = dataBlockEncodingStr == null ? null : + DataBlockEncoding.valueOf(dataBlockEncodingStr); + if (dataBlockEncodingAlgo == DataBlockEncoding.NONE && encodeInCacheOnly) { + throw new IllegalArgumentException("-" + OPT_ENCODE_IN_CACHE_ONLY + " " + + "does not make sense when data block encoding is not used"); + } + + String compressStr = cmd.getOptionValue(OPT_COMPRESSION); + compressAlgo = compressStr == null ? Compression.Algorithm.NONE : + Compression.Algorithm.valueOf(compressStr); + + String bloomStr = cmd.getOptionValue(OPT_BLOOM); + bloomType = bloomStr == null ? null : + StoreFile.BloomType.valueOf(bloomStr); + } + + public void initTestTable() throws IOException { + HBaseTestingUtility.createPreSplitLoadTestTable(conf, tableName, + COLUMN_FAMILY, compressAlgo, dataBlockEncodingAlgo); + applyColumnFamilyOptions(tableName, COLUMN_FAMILIES); + } + + @Override + protected int doWork() throws IOException { + if (cmd.hasOption(OPT_ZK_QUORUM)) { + conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue(OPT_ZK_QUORUM)); + } + + if (isInitOnly) { + LOG.info("Initializing only; no reads or writes"); + initTestTable(); + return 0; + } + + if (!isSkipInit) { + initTestTable(); + } + + LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator( + minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, COLUMN_FAMILY); + + if (isWrite) { + writerThreads = new MultiThreadedWriter(dataGen, conf, tableName); + writerThreads.setMultiPut(isMultiPut); + } + + if (isRead) { + readerThreads = new MultiThreadedReader(dataGen, conf, tableName, verifyPercent); + readerThreads.setMaxErrors(maxReadErrors); + readerThreads.setKeyWindow(keyWindow); + } + + if (isRead && isWrite) { + LOG.info("Concurrent read/write workload: making readers aware of the " + + "write point"); + readerThreads.linkToWriter(writerThreads); + } + + if (isWrite) { + System.out.println("Starting to write data..."); + writerThreads.start(startKey, endKey, numWriterThreads); + } + + if (isRead) { + System.out.println("Starting to read data..."); + readerThreads.start(startKey, endKey, numReaderThreads); + } + + if (isWrite) { + writerThreads.waitForFinish(); + } + + if (isRead) { + readerThreads.waitForFinish(); + } + + boolean success = true; + if (isWrite) { + success = success && writerThreads.getNumWriteFailures() == 0; + } + if (isRead) { + success = success && readerThreads.getNumReadErrors() == 0 + && readerThreads.getNumReadFailures() == 0; + } + return success ? 0 : 1; + } + + public static void main(String[] args) { + new LoadTestTool().doStaticMain(args); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java new file mode 100644 index 0000000..09c9549 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentSkipListMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.ipc.RpcServer; +import org.apache.hadoop.hbase.regionserver.CompactionRequestor; +import org.apache.hadoop.hbase.regionserver.FlushRequester; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.Leases; +import org.apache.hadoop.hbase.regionserver.RegionServerAccounting; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +/** + * Basic mock region server services. + */ +public class MockRegionServerServices implements RegionServerServices { + private final Map regions = new HashMap(); + private boolean stopping = false; + private final ConcurrentSkipListMap rit = + new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR); + private HFileSystem hfs = null; + + @Override + public boolean removeFromOnlineRegions(String encodedRegionName) { + return this.regions.remove(encodedRegionName) != null; + } + + @Override + public HRegion getFromOnlineRegions(String encodedRegionName) { + return this.regions.get(encodedRegionName); + } + + public List getOnlineRegions(byte[] tableName) throws IOException { + return null; + } + + @Override + public void addToOnlineRegions(HRegion r) { + this.regions.put(r.getRegionInfo().getEncodedName(), r); + } + + @Override + public void postOpenDeployTasks(HRegion r, CatalogTracker ct, boolean daughter) + throws KeeperException, IOException { + addToOnlineRegions(r); + } + + @Override + public boolean isStopping() { + return this.stopping; + } + + @Override + public HLog getWAL(HRegionInfo regionInfo) throws IOException { + return null; + } + + @Override + public RpcServer getRpcServer() { + return null; + } + + @Override + public FlushRequester getFlushRequester() { + return null; + } + + @Override + public CompactionRequestor getCompactionRequester() { + return null; + } + + @Override + public CatalogTracker getCatalogTracker() { + return null; + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + return null; + } + + public RegionServerAccounting getRegionServerAccounting() { + return null; + } + + @Override + public ServerName getServerName() { + return null; + } + + @Override + public Configuration getConfiguration() { + return null; + } + + @Override + public void abort(String why, Throwable e) { + //no-op + } + + @Override + public void stop(String why) { + //no-op + } + + @Override + public boolean isStopped() { + return false; + } + + @Override + public boolean isAborted() { + return false; + } + + @Override + public HFileSystem getFileSystem() { + return this.hfs; + } + + public void setFileSystem(FileSystem hfs) { + this.hfs = (HFileSystem)hfs; + } + + @Override + public Leases getLeases() { + return null; + } + + @Override + public boolean removeFromRegionsInTransition(HRegionInfo hri) { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean containsKeyInRegionsInTransition(HRegionInfo hri) { + // TODO Auto-generated method stub + return false; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/MockServer.java b/src/test/java/org/apache/hadoop/hbase/util/MockServer.java new file mode 100644 index 0000000..50cef63 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/MockServer.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; + +/** + * Basic mock Server for handler tests. + */ +public class MockServer implements Server { + static final Log LOG = LogFactory.getLog(MockServer.class); + final static ServerName NAME = new ServerName("MockServer", 123, -1); + + boolean stopped; + boolean aborted; + final ZooKeeperWatcher zk; + final HBaseTestingUtility htu; + + @SuppressWarnings("unused") + public MockServer() throws ZooKeeperConnectionException, IOException { + // Shutdown default constructor by making it private. + this(null); + } + + public MockServer(final HBaseTestingUtility htu) + throws ZooKeeperConnectionException, IOException { + this(htu, true); + } + + /** + * @param htu Testing utility to use + * @param zkw If true, create a zkw. + * @throws ZooKeeperConnectionException + * @throws IOException + */ + public MockServer(final HBaseTestingUtility htu, final boolean zkw) + throws ZooKeeperConnectionException, IOException { + this.htu = htu; + this.zk = zkw? + new ZooKeeperWatcher(htu.getConfiguration(), NAME.toString(), this, true): + null; + } + + @Override + public void abort(String why, Throwable e) { + LOG.fatal("Abort why=" + why, e); + stop(why); + this.aborted = true; + } + + @Override + public void stop(String why) { + LOG.debug("Stop why=" + why); + this.stopped = true; + } + + @Override + public boolean isStopped() { + return this.stopped; + } + + @Override + public Configuration getConfiguration() { + return this.htu.getConfiguration(); + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + return this.zk; + } + + @Override + public CatalogTracker getCatalogTracker() { + return null; + } + + @Override + public ServerName getServerName() { + return NAME; + } + + @Override + public boolean isAborted() { + // TODO Auto-generated method stub + return this.aborted; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java new file mode 100644 index 0000000..92eb11c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.util.StringUtils; + +/** + * Common base class for reader and writer parts of multi-thread HBase load + * test ({@link LoadTestTool}). + */ +public abstract class MultiThreadedAction { + private static final Log LOG = LogFactory.getLog(MultiThreadedAction.class); + + protected final byte[] tableName; + protected final Configuration conf; + + protected int numThreads = 1; + + /** The start key of the key range, inclusive */ + protected long startKey = 0; + + /** The end key of the key range, exclusive */ + protected long endKey = 1; + + protected AtomicInteger numThreadsWorking = new AtomicInteger(); + protected AtomicLong numKeys = new AtomicLong(); + protected AtomicLong numCols = new AtomicLong(); + protected AtomicLong totalOpTimeMs = new AtomicLong(); + protected boolean verbose = false; + + protected LoadTestDataGenerator dataGenerator = null; + + /** + * Default implementation of LoadTestDataGenerator that uses LoadTestKVGenerator, fixed + * set of column families, and random number of columns in range. The table for it can + * be created manually or, for example, via + * {@link HBaseTestingUtility#createPreSplitLoadTestTable( + * org.apache.hadoop.hbase.Configuration, byte[], byte[], Algorithm, DataBlockEncoding)} + */ + public static class DefaultDataGenerator extends LoadTestDataGenerator { + private byte[][] columnFamilies = null; + private int minColumnsPerKey; + private int maxColumnsPerKey; + private final Random random = new Random(); + + public DefaultDataGenerator(int minValueSize, int maxValueSize, + int minColumnsPerKey, int maxColumnsPerKey, byte[]... columnFamilies) { + super(minValueSize, maxValueSize); + this.columnFamilies = columnFamilies; + this.minColumnsPerKey = minColumnsPerKey; + this.maxColumnsPerKey = maxColumnsPerKey; + } + + public DefaultDataGenerator(byte[]... columnFamilies) { + // Default values for tests that didn't care to provide theirs. + this(256, 1024, 1, 10, columnFamilies); + } + + @Override + public byte[] getDeterministicUniqueKey(long keyBase) { + return LoadTestKVGenerator.md5PrefixedKey(keyBase).getBytes(); + } + + @Override + public byte[][] getColumnFamilies() { + return columnFamilies; + } + + @Override + public byte[][] generateColumnsForCf(byte[] rowKey, byte[] cf) { + int numColumns = minColumnsPerKey + random.nextInt(maxColumnsPerKey - minColumnsPerKey + 1); + byte[][] columns = new byte[numColumns][]; + for (int i = 0; i < numColumns; ++i) { + columns[i] = Integer.toString(i).getBytes(); + } + return columns; + } + + @Override + public byte[] generateValue(byte[] rowKey, byte[] cf, byte[] column) { + return kvGenerator.generateRandomSizeValue(rowKey, cf, column); + } + + @Override + public boolean verify(byte[] rowKey, byte[] cf, byte[] column, byte[] value) { + return LoadTestKVGenerator.verify(value, rowKey, cf, column); + } + + @Override + public boolean verify(byte[] rowKey, byte[] cf, Set columnSet) { + return (columnSet.size() >= minColumnsPerKey) && (columnSet.size() <= maxColumnsPerKey); + } + } + + /** "R" or "W" */ + private String actionLetter; + + /** Whether we need to print out Hadoop Streaming-style counters */ + private boolean streamingCounters; + + public static final int REPORTING_INTERVAL_MS = 5000; + + public MultiThreadedAction(LoadTestDataGenerator dataGen, Configuration conf, byte[] tableName, + String actionLetter) { + this.conf = conf; + this.tableName = tableName; + this.dataGenerator = dataGen; + this.actionLetter = actionLetter; + } + + public void start(long startKey, long endKey, int numThreads) + throws IOException { + this.startKey = startKey; + this.endKey = endKey; + this.numThreads = numThreads; + (new Thread(new ProgressReporter(actionLetter))).start(); + } + + private static String formatTime(long elapsedTime) { + String format = String.format("%%0%dd", 2); + elapsedTime = elapsedTime / 1000; + String seconds = String.format(format, elapsedTime % 60); + String minutes = String.format(format, (elapsedTime % 3600) / 60); + String hours = String.format(format, elapsedTime / 3600); + String time = hours + ":" + minutes + ":" + seconds; + return time; + } + + /** Asynchronously reports progress */ + private class ProgressReporter implements Runnable { + + private String reporterId = ""; + + public ProgressReporter(String id) { + this.reporterId = id; + } + + @Override + public void run() { + long startTime = System.currentTimeMillis(); + long priorNumKeys = 0; + long priorCumulativeOpTime = 0; + int priorAverageKeysPerSecond = 0; + + // Give other threads time to start. + Threads.sleep(REPORTING_INTERVAL_MS); + + while (numThreadsWorking.get() != 0) { + String threadsLeft = + "[" + reporterId + ":" + numThreadsWorking.get() + "] "; + if (numKeys.get() == 0) { + LOG.info(threadsLeft + "Number of keys = 0"); + } else { + long numKeys = MultiThreadedAction.this.numKeys.get(); + long time = System.currentTimeMillis() - startTime; + long totalOpTime = totalOpTimeMs.get(); + + long numKeysDelta = numKeys - priorNumKeys; + long totalOpTimeDelta = totalOpTime - priorCumulativeOpTime; + + double averageKeysPerSecond = + (time > 0) ? (numKeys * 1000 / time) : 0; + + LOG.info(threadsLeft + + "Keys=" + + numKeys + + ", cols=" + + StringUtils.humanReadableInt(numCols.get()) + + ", time=" + + formatTime(time) + + ((numKeys > 0 && time > 0) ? (" Overall: [" + "keys/s= " + + numKeys * 1000 / time + ", latency=" + totalOpTime + / numKeys + " ms]") : "") + + ((numKeysDelta > 0) ? (" Current: [" + "keys/s=" + + numKeysDelta * 1000 / REPORTING_INTERVAL_MS + ", latency=" + + totalOpTimeDelta / numKeysDelta + " ms]") : "") + + progressInfo()); + + if (streamingCounters) { + printStreamingCounters(numKeysDelta, + averageKeysPerSecond - priorAverageKeysPerSecond); + } + + priorNumKeys = numKeys; + priorCumulativeOpTime = totalOpTime; + priorAverageKeysPerSecond = (int) averageKeysPerSecond; + } + + Threads.sleep(REPORTING_INTERVAL_MS); + } + } + + private void printStreamingCounters(long numKeysDelta, + double avgKeysPerSecondDelta) { + // Write stats in a format that can be interpreted as counters by + // streaming map-reduce jobs. + System.err.println("reporter:counter:numKeys," + reporterId + "," + + numKeysDelta); + System.err.println("reporter:counter:numCols," + reporterId + "," + + numCols.get()); + System.err.println("reporter:counter:avgKeysPerSecond," + reporterId + + "," + (long) (avgKeysPerSecondDelta)); + } + } + + public void waitForFinish() { + while (numThreadsWorking.get() != 0) { + Threads.sleepWithoutInterrupt(1000); + } + } + + public boolean isDone() { + return (numThreadsWorking.get() == 0); + } + + protected void startThreads(Collection threads) { + numThreadsWorking.addAndGet(threads.size()); + for (Thread thread : threads) { + thread.start(); + } + } + + /** @return the end key of the key range, exclusive */ + public long getEndKey() { + return endKey; + } + + /** Returns a task-specific progress string */ + protected abstract String progressInfo(); + + protected static void appendToStatus(StringBuilder sb, String desc, + long v) { + if (v == 0) { + return; + } + sb.append(", "); + sb.append(desc); + sb.append("="); + sb.append(v); + } + + protected static void appendToStatus(StringBuilder sb, String desc, + String v) { + sb.append(", "); + sb.append(desc); + sb.append("="); + sb.append(v); + } + + /** + * See {@link #verifyResultAgainstDataGenerator(Result, boolean, boolean)}. + * Does not verify cf/column integrity. + */ + public boolean verifyResultAgainstDataGenerator(Result result, boolean verifyValues) { + return verifyResultAgainstDataGenerator(result, verifyValues, false); + } + + /** + * Verifies the result from get or scan using the dataGenerator (that was presumably + * also used to generate said result). + * @param verifyValues verify that values in the result make sense for row/cf/column combination + * @param verifyCfAndColumnIntegrity verify that cf/column set in the result is complete. Note + * that to use this multiPut should be used, or verification + * has to happen after writes, otherwise there can be races. + * @return + */ + public boolean verifyResultAgainstDataGenerator(Result result, boolean verifyValues, + boolean verifyCfAndColumnIntegrity) { + String rowKeyStr = Bytes.toString(result.getRow()); + + // See if we have any data at all. + if (result.isEmpty()) { + LOG.error("No data returned for key = [" + rowKeyStr + "]"); + return false; + } + + if (!verifyValues && !verifyCfAndColumnIntegrity) { + return true; // as long as we have something, we are good. + } + + // See if we have all the CFs. + byte[][] expectedCfs = dataGenerator.getColumnFamilies(); + if (verifyCfAndColumnIntegrity && (expectedCfs.length != result.getMap().size())) { + LOG.error("Bad family count for [" + rowKeyStr + "]: " + result.getMap().size()); + return false; + } + + // Verify each column family from get in the result. + for (byte[] cf : result.getMap().keySet()) { + String cfStr = Bytes.toString(cf); + Map columnValues = result.getFamilyMap(cf); + if (columnValues == null) { + LOG.error("No data for family [" + cfStr + "] for [" + rowKeyStr + "]"); + return false; + } + // See if we have correct columns. + if (verifyCfAndColumnIntegrity + && !dataGenerator.verify(result.getRow(), cf, columnValues.keySet())) { + String colsStr = ""; + for (byte[] col : columnValues.keySet()) { + if (colsStr.length() > 0) { + colsStr += ", "; + } + colsStr += "[" + Bytes.toString(col) + "]"; + } + LOG.error("Bad columns for family [" + cfStr + "] for [" + rowKeyStr + "]: " + colsStr); + return false; + } + // See if values check out. + if (verifyValues) { + for (Map.Entry kv : columnValues.entrySet()) { + if (!dataGenerator.verify(result.getRow(), cf, kv.getKey(), kv.getValue())) { + LOG.error("Error checking data for key [" + rowKeyStr + "], column family [" + + cfStr + "], column [" + Bytes.toString(kv.getKey()) + "]; value of length " + + + kv.getValue().length); + return false; + } + } + } + } + return true; + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReader.java b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReader.java new file mode 100644 index 0000000..e42152f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReader.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; + +/** Creates multiple threads that read and verify previously written data */ +public class MultiThreadedReader extends MultiThreadedAction { + + private static final Log LOG = LogFactory.getLog(MultiThreadedReader.class); + + private Set readers = new HashSet(); + private final double verifyPercent; + private volatile boolean aborted; + + private MultiThreadedWriter writer = null; + + /** + * The number of keys verified in a sequence. This will never be larger than + * the total number of keys in the range. The reader might also verify + * random keys when it catches up with the writer. + */ + private final AtomicLong numUniqueKeysVerified = new AtomicLong(); + + /** + * Default maximum number of read errors to tolerate before shutting down all + * readers. + */ + public static final int DEFAULT_MAX_ERRORS = 10; + + /** + * Default "window" size between the last key written by the writer and the + * key that we attempt to read. The lower this number, the stricter our + * testing is. If this is zero, we always attempt to read the highest key + * in the contiguous sequence of keys written by the writers. + */ + public static final int DEFAULT_KEY_WINDOW = 0; + + protected AtomicLong numKeysVerified = new AtomicLong(0); + private AtomicLong numReadErrors = new AtomicLong(0); + private AtomicLong numReadFailures = new AtomicLong(0); + + private int maxErrors = DEFAULT_MAX_ERRORS; + private int keyWindow = DEFAULT_KEY_WINDOW; + + public MultiThreadedReader(LoadTestDataGenerator dataGen, Configuration conf, + byte[] tableName, double verifyPercent) { + super(dataGen, conf, tableName, "R"); + this.verifyPercent = verifyPercent; + } + + public void linkToWriter(MultiThreadedWriter writer) { + this.writer = writer; + writer.setTrackInsertedKeys(true); + } + + public void setMaxErrors(int maxErrors) { + this.maxErrors = maxErrors; + } + + public void setKeyWindow(int keyWindow) { + this.keyWindow = keyWindow; + } + + @Override + public void start(long startKey, long endKey, int numThreads) + throws IOException { + super.start(startKey, endKey, numThreads); + if (verbose) { + LOG.debug("Reading keys [" + startKey + ", " + endKey + ")"); + } + + for (int i = 0; i < numThreads; ++i) { + HBaseReaderThread reader = new HBaseReaderThread(i); + readers.add(reader); + } + startThreads(readers); + } + + public class HBaseReaderThread extends Thread { + private final int readerId; + private final HTable table; + private final Random random = new Random(); + + /** The "current" key being read. Increases from startKey to endKey. */ + private long curKey; + + /** Time when the thread started */ + private long startTimeMs; + + /** If we are ahead of the writer and reading a random key. */ + private boolean readingRandomKey; + + /** + * @param readerId only the keys with this remainder from division by + * {@link #numThreads} will be read by this thread + */ + public HBaseReaderThread(int readerId) throws IOException { + this.readerId = readerId; + table = new HTable(conf, tableName); + setName(getClass().getSimpleName() + "_" + readerId); + } + + @Override + public void run() { + try { + runReader(); + } finally { + try { + table.close(); + } catch (IOException e) { + LOG.error("Error closing table", e); + } + numThreadsWorking.decrementAndGet(); + } + } + + private void runReader() { + if (verbose) { + LOG.info("Started thread #" + readerId + " for reads..."); + } + + startTimeMs = System.currentTimeMillis(); + curKey = startKey; + while (curKey < endKey && !aborted) { + long k = getNextKeyToRead(); + + // A sanity check for the key range. + if (k < startKey || k >= endKey) { + numReadErrors.incrementAndGet(); + throw new AssertionError("Load tester logic error: proposed key " + + "to read " + k + " is out of range (startKey=" + startKey + + ", endKey=" + endKey + ")"); + } + + if (k % numThreads != readerId || + writer != null && writer.failedToWriteKey(k)) { + // Skip keys that this thread should not read, as well as the keys + // that we know the writer failed to write. + continue; + } + + readKey(k); + if (k == curKey - 1 && !readingRandomKey) { + // We have verified another unique key. + numUniqueKeysVerified.incrementAndGet(); + } + } + } + + /** + * Should only be used for the concurrent writer/reader workload. The + * maximum key we are allowed to read, subject to the "key window" + * constraint. + */ + private long maxKeyWeCanRead() { + long insertedUpToKey = writer.insertedUpToKey(); + if (insertedUpToKey >= endKey - 1) { + // The writer has finished writing our range, so we can read any + // key in the range. + return endKey - 1; + } + return Math.min(endKey - 1, writer.insertedUpToKey() - keyWindow); + } + + private long getNextKeyToRead() { + readingRandomKey = false; + if (writer == null || curKey <= maxKeyWeCanRead()) { + return curKey++; + } + + // We caught up with the writer. See if we can read any keys at all. + long maxKeyToRead; + while ((maxKeyToRead = maxKeyWeCanRead()) < startKey) { + // The writer has not written sufficient keys for us to be able to read + // anything at all. Sleep a bit. This should only happen in the + // beginning of a load test run. + Threads.sleepWithoutInterrupt(50); + } + + if (curKey <= maxKeyToRead) { + // The writer wrote some keys, and we are now allowed to read our + // current key. + return curKey++; + } + + // startKey <= maxKeyToRead <= curKey - 1. Read one of the previous keys. + // Don't increment the current key -- we still have to try reading it + // later. Set a flag to make sure that we don't count this key towards + // the set of unique keys we have verified. + readingRandomKey = true; + return startKey + Math.abs(random.nextLong()) + % (maxKeyToRead - startKey + 1); + } + + private Get readKey(long keyToRead) { + Get get = new Get(dataGenerator.getDeterministicUniqueKey(keyToRead)); + String cfsString = ""; + byte[][] columnFamilies = dataGenerator.getColumnFamilies(); + for (byte[] cf : columnFamilies) { + get.addFamily(cf); + if (verbose) { + if (cfsString.length() > 0) { + cfsString += ", "; + } + cfsString += "[" + Bytes.toStringBinary(cf) + "]"; + } + } + + try { + if (verbose) { + LOG.info("[" + readerId + "] " + "Querying key " + keyToRead + ", cfs " + cfsString); + } + queryKey(get, random.nextInt(100) < verifyPercent); + } catch (IOException e) { + numReadFailures.addAndGet(1); + LOG.debug("[" + readerId + "] FAILED read, key = " + (keyToRead + "") + + ", time from start: " + + (System.currentTimeMillis() - startTimeMs) + " ms"); + } + return get; + } + + public void queryKey(Get get, boolean verify) throws IOException { + String rowKey = Bytes.toString(get.getRow()); + + // read the data + long start = System.currentTimeMillis(); + Result result = table.get(get); + totalOpTimeMs.addAndGet(System.currentTimeMillis() - start); + numKeys.addAndGet(1); + + // if we got no data report error + if (!result.isEmpty()) { + if (verify) { + numKeysVerified.incrementAndGet(); + } + } else { + HRegionLocation hloc = table.getRegionLocation(Bytes.toBytes(rowKey)); + LOG.info("Key = " + rowKey + ", RegionServer: " + + hloc.getHostname()); + } + + boolean isOk = verifyResultAgainstDataGenerator(result, verify); + long numErrorsAfterThis = 0; + if (isOk) { + long cols = 0; + // Count the columns for reporting purposes. + for (byte[] cf : result.getMap().keySet()) { + cols += result.getFamilyMap(cf).size(); + } + numCols.addAndGet(cols); + } else { + if (writer != null) { + LOG.error("At the time of failure, writer inserted " + writer.numKeys.get() + " keys"); + } + numErrorsAfterThis = numReadErrors.incrementAndGet(); + } + + if (numErrorsAfterThis > maxErrors) { + LOG.error("Aborting readers -- found more than " + maxErrors + " errors"); + aborted = true; + } + } + } + + public long getNumReadFailures() { + return numReadFailures.get(); + } + + public long getNumReadErrors() { + return numReadErrors.get(); + } + + public long getNumKeysVerified() { + return numKeysVerified.get(); + } + + public long getNumUniqueKeysVerified() { + return numUniqueKeysVerified.get(); + } + + @Override + protected String progressInfo() { + StringBuilder sb = new StringBuilder(); + appendToStatus(sb, "verified", numKeysVerified.get()); + appendToStatus(sb, "READ FAILURES", numReadFailures.get()); + appendToStatus(sb, "READ ERRORS", numReadErrors.get()); + return sb.toString(); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java new file mode 100644 index 0000000..ea372cc --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.util.HashSet; +import java.util.PriorityQueue; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; + +/** Creates multiple threads that write key/values into the */ +public class MultiThreadedWriter extends MultiThreadedAction { + private static final Log LOG = LogFactory.getLog(MultiThreadedWriter.class); + + private Set writers = new HashSet(); + + private boolean isMultiPut = false; + + /** + * A temporary place to keep track of inserted keys. This is written to by + * all writers and is drained on a separate thread that populates + * {@link #insertedUpToKey}, the maximum key in the contiguous range of keys + * being inserted. This queue is supposed to stay small. + */ + private BlockingQueue insertedKeys = new ArrayBlockingQueue(10000); + + /** + * This is the current key to be inserted by any thread. Each thread does an + * atomic get and increment operation and inserts the current value. + */ + private AtomicLong nextKeyToInsert = new AtomicLong(); + + /** + * The highest key in the contiguous range of keys . + */ + private AtomicLong insertedUpToKey = new AtomicLong(); + + /** The sorted set of keys NOT inserted by the writers */ + private Set failedKeySet = new ConcurrentSkipListSet(); + + /** + * The total size of the temporary inserted key set that have not yet lined + * up in a our contiguous sequence starting from startKey. Supposed to stay + * small. + */ + private AtomicLong insertedKeyQueueSize = new AtomicLong(); + + /** Enable this if used in conjunction with a concurrent reader. */ + private boolean trackInsertedKeys; + + public MultiThreadedWriter(LoadTestDataGenerator dataGen, Configuration conf, + byte[] tableName) { + super(dataGen, conf, tableName, "W"); + } + + /** Use multi-puts vs. separate puts for every column in a row */ + public void setMultiPut(boolean isMultiPut) { + this.isMultiPut = isMultiPut; + } + + @Override + public void start(long startKey, long endKey, int numThreads) + throws IOException { + super.start(startKey, endKey, numThreads); + + if (verbose) { + LOG.debug("Inserting keys [" + startKey + ", " + endKey + ")"); + } + + nextKeyToInsert.set(startKey); + insertedUpToKey.set(startKey - 1); + + for (int i = 0; i < numThreads; ++i) { + HBaseWriterThread writer = new HBaseWriterThread(i); + writers.add(writer); + } + + if (trackInsertedKeys) { + new Thread(new InsertedKeysTracker()).start(); + numThreadsWorking.incrementAndGet(); + } + + startThreads(writers); + } + + private class HBaseWriterThread extends Thread { + private final HTable table; + + public HBaseWriterThread(int writerId) throws IOException { + setName(getClass().getSimpleName() + "_" + writerId); + table = new HTable(conf, tableName); + } + + public void run() { + try { + long rowKeyBase; + byte[][] columnFamilies = dataGenerator.getColumnFamilies(); + while ((rowKeyBase = nextKeyToInsert.getAndIncrement()) < endKey) { + byte[] rowKey = dataGenerator.getDeterministicUniqueKey(rowKeyBase); + Put put = new Put(rowKey); + numKeys.addAndGet(1); + int columnCount = 0; + for (byte[] cf : columnFamilies) { + byte[][] columns = dataGenerator.generateColumnsForCf(rowKey, cf); + for (byte[] column : columns) { + byte[] value = dataGenerator.generateValue(rowKey, cf, column); + put.add(cf, column, value); + ++columnCount; + if (!isMultiPut) { + insert(table, put, rowKeyBase); + numCols.addAndGet(1); + put = new Put(rowKey); + } + } + } + if (isMultiPut) { + if (verbose) { + LOG.debug("Preparing put for key = [" + rowKey + "], " + columnCount + " columns"); + } + insert(table, put, rowKeyBase); + numCols.addAndGet(columnCount); + } + if (trackInsertedKeys) { + insertedKeys.add(rowKeyBase); + } + } + } finally { + try { + table.close(); + } catch (IOException e) { + LOG.error("Error closing table", e); + } + numThreadsWorking.decrementAndGet(); + } + } + } + + public void insert(HTable table, Put put, long keyBase) { + try { + long start = System.currentTimeMillis(); + table.put(put); + totalOpTimeMs.addAndGet(System.currentTimeMillis() - start); + } catch (IOException e) { + failedKeySet.add(keyBase); + LOG.error("Failed to insert: " + keyBase); + e.printStackTrace(); + } + } + + /** + * A thread that keeps track of the highest key in the contiguous range of + * inserted keys. + */ + private class InsertedKeysTracker implements Runnable { + + @Override + public void run() { + Thread.currentThread().setName(getClass().getSimpleName()); + try { + long expectedKey = startKey; + Queue sortedKeys = new PriorityQueue(); + while (expectedKey < endKey) { + // Block until a new element is available. + Long k; + try { + k = insertedKeys.poll(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOG.info("Inserted key tracker thread interrupted", e); + break; + } + if (k == null) { + continue; + } + if (k == expectedKey) { + // Skip the "sorted key" queue and consume this key. + insertedUpToKey.set(k); + ++expectedKey; + } else { + sortedKeys.add(k); + } + + // See if we have a sequence of contiguous keys lined up. + while (!sortedKeys.isEmpty() + && ((k = sortedKeys.peek()) == expectedKey)) { + sortedKeys.poll(); + insertedUpToKey.set(k); + ++expectedKey; + } + + insertedKeyQueueSize.set(insertedKeys.size() + sortedKeys.size()); + } + } catch (Exception ex) { + LOG.error("Error in inserted key tracker", ex); + } finally { + numThreadsWorking.decrementAndGet(); + } + } + + } + + @Override + public void waitForFinish() { + super.waitForFinish(); + System.out.println("Failed to write keys: " + failedKeySet.size()); + for (Long key : failedKeySet) { + System.out.println("Failed to write key: " + key); + } + } + + public int getNumWriteFailures() { + return failedKeySet.size(); + } + + /** + * The max key until which all keys have been inserted (successfully or not). + * @return the last key that we have inserted all keys up to (inclusive) + */ + public long insertedUpToKey() { + return insertedUpToKey.get(); + } + + public boolean failedToWriteKey(long k) { + return failedKeySet.contains(k); + } + + @Override + protected String progressInfo() { + StringBuilder sb = new StringBuilder(); + appendToStatus(sb, "insertedUpTo", insertedUpToKey.get()); + appendToStatus(sb, "insertedQSize", insertedKeyQueueSize.get()); + return sb.toString(); + } + + /** + * Used for a joint write/read workload. Enables tracking the last inserted + * key, which requires a blocking queue and a consumer thread. + * @param enable whether to enable tracking the last inserted key + */ + public void setTrackInsertedKeys(boolean enable) { + trackInsertedKeys = enable; + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/ProcessBasedLocalHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/util/ProcessBasedLocalHBaseCluster.java new file mode 100644 index 0000000..05c97f0 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/ProcessBasedLocalHBaseCluster.java @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; +import java.util.TreeMap; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; + +/** + * A helper class for process-based mini-cluster tests. Unlike + * {@link MiniHBaseCluster}, starts daemons as separate processes, allowing to + * do real kill testing. + */ +public class ProcessBasedLocalHBaseCluster { + + private static final String DEFAULT_WORKDIR = + "/tmp/hbase-" + System.getenv("USER"); + + private final String hbaseHome; + private final String workDir; + + private int numRegionServers; + private final int zkClientPort; + private final int masterPort; + + private final Configuration conf; + + private static final int MAX_FILE_SIZE_OVERRIDE = 10 * 1000 * 1000; + + private static final Log LOG = LogFactory.getLog( + ProcessBasedLocalHBaseCluster.class); + + private List daemonPidFiles = + Collections.synchronizedList(new ArrayList());; + + private boolean shutdownHookInstalled; + + private String hbaseDaemonScript; + + /** + * Constructor. Modifies the passed configuration. + * @param hbaseHome the top directory of the HBase source tree + */ + public ProcessBasedLocalHBaseCluster(Configuration conf, String hbaseHome, + int numRegionServers) { + this.conf = conf; + this.hbaseHome = hbaseHome; + this.numRegionServers = numRegionServers; + this.workDir = DEFAULT_WORKDIR; + + hbaseDaemonScript = hbaseHome + "/bin/hbase-daemon.sh"; + zkClientPort = HBaseTestingUtility.randomFreePort(); + masterPort = HBaseTestingUtility.randomFreePort(); + + conf.set(HConstants.ZOOKEEPER_QUORUM, HConstants.LOCALHOST); + conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort); + } + + public void start() throws IOException { + cleanupOldState(); + + // start ZK + LOG.info("Starting ZooKeeper"); + startZK(); + + HBaseTestingUtility.waitForHostPort(HConstants.LOCALHOST, zkClientPort); + + startMaster(); + ZKUtil.waitForBaseZNode(conf); + + for (int idx = 0; idx < numRegionServers; idx++) { + startRegionServer(HBaseTestingUtility.randomFreePort()); + } + + LOG.info("Waiting for HBase startup by scanning META"); + int attemptsLeft = 10; + while (attemptsLeft-- > 0) { + try { + new HTable(conf, HConstants.META_TABLE_NAME); + } catch (Exception e) { + LOG.info("Waiting for HBase to startup. Retries left: " + attemptsLeft, + e); + Threads.sleep(1000); + } + } + + LOG.info("Process-based HBase Cluster with " + numRegionServers + + " region servers up and running... \n\n"); + } + + public void startRegionServer(int port) { + startServer("regionserver", port); + } + + public void startMaster() { + startServer("master", 0); + } + + public void killRegionServer(int port) throws IOException { + killServer("regionserver", port); + } + + public void killMaster() throws IOException { + killServer("master", 0); + } + + public void startZK() { + startServer("zookeeper", 0); + } + + private void executeCommand(String command) { + ensureShutdownHookInstalled(); + executeCommand(command, null); + } + + private void executeCommand(String command, Map envOverrides) { + LOG.debug("Command : " + command); + + try { + String [] envp = null; + if (envOverrides != null) { + Map map = new HashMap( + System.getenv()); + map.putAll(envOverrides); + envp = new String[map.size()]; + int idx = 0; + for (Map.Entry e: map.entrySet()) { + envp[idx++] = e.getKey() + "=" + e.getValue(); + } + } + + Process p = Runtime.getRuntime().exec(command, envp); + + BufferedReader stdInput = new BufferedReader( + new InputStreamReader(p.getInputStream())); + BufferedReader stdError = new BufferedReader( + new InputStreamReader(p.getErrorStream())); + + // read the output from the command + String s = null; + while ((s = stdInput.readLine()) != null) { + System.out.println(s); + } + + // read any errors from the attempted command + while ((s = stdError.readLine()) != null) { + System.out.println(s); + } + } catch (IOException e) { + LOG.error("Error running: " + command, e); + } + } + + private void shutdownAllProcesses() { + LOG.info("Killing daemons using pid files"); + final List pidFiles = new ArrayList(daemonPidFiles); + for (String pidFile : pidFiles) { + int pid = 0; + try { + pid = readPidFromFile(pidFile); + } catch (IOException ex) { + LOG.error("Could not kill process with pid from " + pidFile); + } + + if (pid > 0) { + LOG.info("Killing pid " + pid + " (" + pidFile + ")"); + killProcess(pid); + } + } + + LOG.info("Waiting a bit to let processes terminate"); + Threads.sleep(5000); + } + + private void ensureShutdownHookInstalled() { + if (shutdownHookInstalled) { + return; + } + + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + @Override + public void run() { + shutdownAllProcesses(); + } + })); + + shutdownHookInstalled = true; + } + + private void cleanupOldState() { + executeCommand("rm -rf " + workDir); + } + + private void writeStringToFile(String s, String fileName) { + try { + BufferedWriter out = new BufferedWriter(new FileWriter(fileName)); + out.write(s); + out.close(); + } catch (IOException e) { + LOG.error("Error writing to: " + fileName, e); + } + } + + private String serverWorkingDir(String serverName, int port) { + String dir; + if (serverName.equals("regionserver")) { + dir = workDir + "/" + serverName + "-" + port; + } else { + dir = workDir + "/" + serverName; + } + return dir; + } + + private int getServerPID(String serverName, int port) throws IOException { + String pidFile = pidFilePath(serverName, port); + return readPidFromFile(pidFile); + } + + private static int readPidFromFile(String pidFile) throws IOException { + Scanner scanner = new Scanner(new File(pidFile)); + try { + return scanner.nextInt(); + } finally { + scanner.close(); + } + } + + private String pidFilePath(String serverName, int port) { + String dir = serverWorkingDir(serverName, port); + String user = System.getenv("USER"); + String pidFile = String.format("%s/hbase-%s-%s.pid", + dir, user, serverName); + return pidFile; + } + + private void killServer(String serverName, int port) throws IOException { + int pid = getServerPID(serverName, port); + if (pid > 0) { + LOG.info("Killing " + serverName + "; pid=" + pid); + killProcess(pid); + } + } + + private void killProcess(int pid) { + String cmd = "kill -s KILL " + pid; + executeCommand(cmd); + } + + private void startServer(String serverName, int rsPort) { + String conf = generateConfig(rsPort); + + // create working directory for this region server. + String dir = serverWorkingDir(serverName, rsPort); + executeCommand("mkdir -p " + dir); + + writeStringToFile(conf, dir + "/" + "hbase-site.xml"); + + Map envOverrides = new HashMap(); + envOverrides.put("HBASE_LOG_DIR", dir); + envOverrides.put("HBASE_PID_DIR", dir); + try { + FileUtils.copyFile( + new File(hbaseHome, "conf/log4j.properties"), + new File(dir, "log4j.properties")); + } catch (IOException ex) { + LOG.error("Could not install log4j.properties into " + dir); + } + + executeCommand(hbaseDaemonScript + " --config " + dir + + " start " + serverName, envOverrides); + daemonPidFiles.add(pidFilePath(serverName, rsPort)); + } + + private final String generateConfig(int rsPort) { + StringBuilder sb = new StringBuilder(); + Map confMap = new TreeMap(); + confMap.put(HConstants.CLUSTER_DISTRIBUTED, true); + if (rsPort > 0) { + confMap.put(HConstants.REGIONSERVER_PORT, rsPort); + confMap.put(HConstants.REGIONSERVER_INFO_PORT_AUTO, true); + } + + confMap.put(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort); + confMap.put(HConstants.MASTER_PORT, masterPort); + confMap.put(HConstants.HREGION_MAX_FILESIZE, MAX_FILE_SIZE_OVERRIDE); + confMap.put("fs.file.impl", RawLocalFileSystem.class.getName()); + + sb.append("\n"); + for (Map.Entry entry : confMap.entrySet()) { + sb.append(" \n"); + sb.append(" " + entry.getKey() + "\n"); + sb.append(" " + entry.getValue() + "\n"); + sb.append(" \n"); + } + sb.append("\n"); + return sb.toString(); + } + + public Configuration getConf() { + return conf; + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java b/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java new file mode 100644 index 0000000..3b4b66c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.Compression; + +/** + * A command-line tool that spins up a local process-based cluster, loads + * some data, restarts the regionserver holding .META., and verifies that the + * cluster recovers. + */ +public class RestartMetaTest extends AbstractHBaseTool { + + private static final Log LOG = LogFactory.getLog(RestartMetaTest.class); + + /** The number of region servers used if not specified */ + private static final int DEFAULT_NUM_RS = 2; + + /** Table name for the test */ + private static byte[] TABLE_NAME = Bytes.toBytes("load_test"); + + /** The number of seconds to sleep after loading the data */ + private static final int SLEEP_SEC_AFTER_DATA_LOAD = 5; + + /** The actual number of region servers */ + private int numRegionServers; + + /** HBase home source tree home directory */ + private String hbaseHome; + + private static final String OPT_HBASE_HOME = "hbase_home"; + private static final String OPT_NUM_RS = "num_rs"; + + /** Loads data into the table using the multi-threaded writer. */ + private void loadData() throws IOException { + long startKey = 0; + long endKey = 100000; + int minColsPerKey = 5; + int maxColsPerKey = 15; + int minColDataSize = 256; + int maxColDataSize = 256 * 3; + int numThreads = 10; + + // print out the arguments + System.out.printf("Key range %d .. %d\n", startKey, endKey); + System.out.printf("Number of Columns/Key: %d..%d\n", minColsPerKey, + maxColsPerKey); + System.out.printf("Data Size/Column: %d..%d bytes\n", minColDataSize, + maxColDataSize); + System.out.printf("Client Threads: %d\n", numThreads); + + // start the writers + LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator( + minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, LoadTestTool.COLUMN_FAMILY); + MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, TABLE_NAME); + writer.setMultiPut(true); + writer.start(startKey, endKey, numThreads); + System.out.printf("Started loading data..."); + writer.waitForFinish(); + System.out.printf("Finished loading data..."); + } + + @Override + protected int doWork() throws Exception { + ProcessBasedLocalHBaseCluster hbaseCluster = + new ProcessBasedLocalHBaseCluster(conf, hbaseHome, numRegionServers); + + // start the process based HBase cluster + hbaseCluster.start(); + + // create tables if needed + HBaseTestingUtility.createPreSplitLoadTestTable(conf, TABLE_NAME, + LoadTestTool.COLUMN_FAMILY, Compression.Algorithm.NONE, + DataBlockEncoding.NONE); + + LOG.debug("Loading data....\n\n"); + loadData(); + + LOG.debug("Sleeping for " + SLEEP_SEC_AFTER_DATA_LOAD + + " seconds....\n\n"); + Threads.sleep(5 * SLEEP_SEC_AFTER_DATA_LOAD); + + int metaRSPort = HBaseTestingUtility.getMetaRSPort(conf); + + LOG.debug("Killing META region server running on port " + metaRSPort); + hbaseCluster.killRegionServer(metaRSPort); + Threads.sleep(2000); + + LOG.debug("Restarting region server running on port metaRSPort"); + hbaseCluster.startRegionServer(metaRSPort); + Threads.sleep(2000); + + LOG.debug("Trying to scan meta"); + + HTable metaTable = new HTable(conf, HConstants.META_TABLE_NAME); + ResultScanner scanner = metaTable.getScanner(new Scan()); + Result result; + while ((result = scanner.next()) != null) { + LOG.info("Region assignment from META: " + + Bytes.toStringBinary(result.getRow()) + + " => " + + Bytes.toStringBinary(result.getFamilyMap(HConstants.CATALOG_FAMILY) + .get(HConstants.SERVER_QUALIFIER))); + } + return 0; + } + + @Override + protected void addOptions() { + addRequiredOptWithArg(OPT_HBASE_HOME, "HBase home directory"); + addOptWithArg(OPT_NUM_RS, "Number of Region Servers"); + addOptWithArg(LoadTestTool.OPT_DATA_BLOCK_ENCODING, + LoadTestTool.OPT_DATA_BLOCK_ENCODING_USAGE); + } + + @Override + protected void processOptions(CommandLine cmd) { + hbaseHome = cmd.getOptionValue(OPT_HBASE_HOME); + if (hbaseHome == null || !new File(hbaseHome).isDirectory()) { + throw new IllegalArgumentException("Invalid HBase home directory: " + + hbaseHome); + } + + LOG.info("Using HBase home directory " + hbaseHome); + numRegionServers = Integer.parseInt(cmd.getOptionValue(OPT_NUM_RS, + String.valueOf(DEFAULT_NUM_RS))); + } + + public static void main(String[] args) { + new RestartMetaTest().doStaticMain(args); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/StoppableImplementation.java b/src/test/java/org/apache/hadoop/hbase/util/StoppableImplementation.java new file mode 100644 index 0000000..51a22f3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/StoppableImplementation.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Stoppable; + +/** + * A base implementation for a Stoppable service + */ +@InterfaceAudience.Private +public class StoppableImplementation implements Stoppable { + volatile boolean stopped = false; + + @Override + public void stop(String why) { + this.stopped = true; + } + + @Override + public boolean isStopped() { + return stopped; + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestBase64.java b/src/test/java/org/apache/hadoop/hbase/util/TestBase64.java new file mode 100644 index 0000000..c55c4d4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestBase64.java @@ -0,0 +1,75 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.UnsupportedEncodingException; +import java.util.Map; +import java.util.TreeMap; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.experimental.categories.Category; + +/** + * Test order preservation characteristics of ordered Base64 dialect + */ +@Category(SmallTests.class) +public class TestBase64 extends TestCase { + // Note: uris is sorted. We need to prove that the ordered Base64 + // preserves that ordering + private String[] uris = { + "dns://dns.powerset.com/www.powerset.com", + "dns:www.powerset.com", + "file:///usr/bin/java", + "filename", + "ftp://one.two.three/index.html", + "http://one.two.three/index.html", + "https://one.two.three:9443/index.html", + "r:dns://com.powerset.dns/www.powerset.com", + "r:ftp://three.two.one/index.html", + "r:http://three.two.one/index.html", + "r:https://three.two.one:9443/index.html" + }; + + /** + * the test + * @throws UnsupportedEncodingException + */ + public void testBase64() throws UnsupportedEncodingException { + TreeMap sorted = new TreeMap(); + + for (int i = 0; i < uris.length; i++) { + byte[] bytes = uris[i].getBytes("UTF-8"); + sorted.put(Base64.encodeBytes(bytes, Base64.ORDERED), uris[i]); + } + System.out.println(); + + int i = 0; + for (Map.Entry e: sorted.entrySet()) { + assertTrue(uris[i++].compareTo(e.getValue()) == 0); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestByteBloomFilter.java b/src/test/java/org/apache/hadoop/hbase/util/TestByteBloomFilter.java new file mode 100644 index 0000000..2240018 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestByteBloomFilter.java @@ -0,0 +1,172 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.nio.ByteBuffer; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestByteBloomFilter extends TestCase { + + public void testBasicBloom() throws Exception { + ByteBloomFilter bf1 = new ByteBloomFilter(1000, (float)0.01, Hash.MURMUR_HASH, 0); + ByteBloomFilter bf2 = new ByteBloomFilter(1000, (float)0.01, Hash.MURMUR_HASH, 0); + bf1.allocBloom(); + bf2.allocBloom(); + + // test 1: verify no fundamental false negatives or positives + byte[] key1 = {1,2,3,4,5,6,7,8,9}; + byte[] key2 = {1,2,3,4,5,6,7,8,7}; + + bf1.add(key1); + bf2.add(key2); + + assertTrue(bf1.contains(key1)); + assertFalse(bf1.contains(key2)); + assertFalse(bf2.contains(key1)); + assertTrue(bf2.contains(key2)); + + byte [] bkey = {1,2,3,4}; + byte [] bval = "this is a much larger byte array".getBytes(); + + bf1.add(bkey); + bf1.add(bval, 1, bval.length-1); + + assertTrue( bf1.contains(bkey) ); + assertTrue( bf1.contains(bval, 1, bval.length-1) ); + assertFalse( bf1.contains(bval) ); + assertFalse( bf1.contains(bval) ); + + // test 2: serialization & deserialization. + // (convert bloom to byte array & read byte array back in as input) + ByteArrayOutputStream bOut = new ByteArrayOutputStream(); + bf1.writeBloom(new DataOutputStream(bOut)); + ByteBuffer bb = ByteBuffer.wrap(bOut.toByteArray()); + ByteBloomFilter newBf1 = new ByteBloomFilter(1000, (float)0.01, + Hash.MURMUR_HASH, 0); + assertTrue(newBf1.contains(key1, bb)); + assertFalse(newBf1.contains(key2, bb)); + assertTrue( newBf1.contains(bkey, bb) ); + assertTrue( newBf1.contains(bval, 1, bval.length-1, bb) ); + assertFalse( newBf1.contains(bval, bb) ); + assertFalse( newBf1.contains(bval, bb) ); + + System.out.println("Serialized as " + bOut.size() + " bytes"); + assertTrue(bOut.size() - bf1.byteSize < 10); //... allow small padding + } + + public void testBloomFold() throws Exception { + // test: foldFactor < log(max/actual) + ByteBloomFilter b = new ByteBloomFilter(1003, (float) 0.01, + Hash.MURMUR_HASH, 2); + b.allocBloom(); + long origSize = b.getByteSize(); + assertEquals(1204, origSize); + for (int i = 0; i < 12; ++i) { + b.add(Bytes.toBytes(i)); + } + b.compactBloom(); + assertEquals(origSize>>2, b.getByteSize()); + int falsePositives = 0; + for (int i = 0; i < 25; ++i) { + if (b.contains(Bytes.toBytes(i))) { + if(i >= 12) falsePositives++; + } else { + assertFalse(i < 12); + } + } + assertTrue(falsePositives <= 1); + + // test: foldFactor > log(max/actual) + } + + public void testBloomPerf() throws Exception { + // add + float err = (float)0.01; + ByteBloomFilter b = new ByteBloomFilter(10*1000*1000, (float)err, Hash.MURMUR_HASH, 3); + b.allocBloom(); + long startTime = System.currentTimeMillis(); + long origSize = b.getByteSize(); + for (int i = 0; i < 1*1000*1000; ++i) { + b.add(Bytes.toBytes(i)); + } + long endTime = System.currentTimeMillis(); + System.out.println("Total Add time = " + (endTime - startTime) + "ms"); + + // fold + startTime = System.currentTimeMillis(); + b.compactBloom(); + endTime = System.currentTimeMillis(); + System.out.println("Total Fold time = " + (endTime - startTime) + "ms"); + assertTrue(origSize >= b.getByteSize()<<3); + + // test + startTime = System.currentTimeMillis(); + int falsePositives = 0; + for (int i = 0; i < 2*1000*1000; ++i) { + + if (b.contains(Bytes.toBytes(i))) { + if(i >= 1*1000*1000) falsePositives++; + } else { + assertFalse(i < 1*1000*1000); + } + } + endTime = System.currentTimeMillis(); + System.out.println("Total Contains time = " + (endTime - startTime) + "ms"); + System.out.println("False Positive = " + falsePositives); + assertTrue(falsePositives <= (1*1000*1000)*err); + + // test: foldFactor > log(max/actual) + } + + public void testSizing() { + int bitSize = 8 * 128 * 1024; // 128 KB + double errorRate = 0.025; // target false positive rate + + // How many keys can we store in a Bloom filter of this size maintaining + // the given false positive rate, not taking into account that the n + long maxKeys = ByteBloomFilter.idealMaxKeys(bitSize, errorRate); + assertEquals(136570, maxKeys); + + // A reverse operation: how many bits would we need to store this many keys + // and keep the same low false positive rate? + long bitSize2 = ByteBloomFilter.computeBitSize(maxKeys, errorRate); + + // The bit size comes out a little different due to rounding. + assertTrue(Math.abs(bitSize2 - bitSize) * 1.0 / bitSize < 1e-5); + } + + public void testFoldableByteSize() { + assertEquals(128, ByteBloomFilter.computeFoldableByteSize(1000, 5)); + assertEquals(640, ByteBloomFilter.computeFoldableByteSize(5001, 4)); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java b/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java new file mode 100644 index 0000000..c31eec0 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.io.WritableUtils; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestByteBufferUtils { + + private byte[] array; + + /** + * Create an array with sample data. + */ + @Before + public void setUp() { + array = new byte[8]; + for (int i = 0; i < array.length; ++i) { + array[i] = (byte) ('a' + i); + } + } + + private static final int MAX_VLONG_LENGTH = 9; + private static final Collection testNumbers; + + private static void addNumber(Set a, long l) { + if (l != Long.MIN_VALUE) { + a.add(l - 1); + } + a.add(l); + if (l != Long.MAX_VALUE) { + a.add(l + 1); + } + for (long divisor = 3; divisor <= 10; ++divisor) { + for (long delta = -1; delta <= 1; ++delta) { + a.add(l / divisor + delta); + } + } + } + + static { + SortedSet a = new TreeSet(); + for (int i = 0; i <= 63; ++i) { + long v = (-1L) << i; + assertTrue(v < 0); + addNumber(a, v); + v = (1L << i) - 1; + assertTrue(v >= 0); + addNumber(a, v); + } + + testNumbers = Collections.unmodifiableSet(a); + System.err.println("Testing variable-length long serialization using: " + + testNumbers + " (count: " + testNumbers.size() + ")"); + assertEquals(1753, testNumbers.size()); + assertEquals(Long.MIN_VALUE, a.first().longValue()); + assertEquals(Long.MAX_VALUE, a.last().longValue()); + } + + @Test + public void testReadWriteVLong() { + for (long l : testNumbers) { + ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH); + ByteBufferUtils.writeVLong(b, l); + b.flip(); + assertEquals(l, ByteBufferUtils.readVLong(b)); + } + } + + @Test + public void testConsistencyWithHadoopVLong() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + for (long l : testNumbers) { + baos.reset(); + ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH); + ByteBufferUtils.writeVLong(b, l); + String bufStr = Bytes.toStringBinary(b.array(), + b.arrayOffset(), b.position()); + WritableUtils.writeVLong(dos, l); + String baosStr = Bytes.toStringBinary(baos.toByteArray()); + assertEquals(baosStr, bufStr); + } + } + + /** + * Test copying to stream from buffer. + */ + @Test + public void testMoveBufferToStream() { + final int arrayOffset = 7; + final int initialPosition = 10; + final int endPadding = 5; + byte[] arrayWrapper = + new byte[arrayOffset + initialPosition + array.length + endPadding]; + System.arraycopy(array, 0, arrayWrapper, + arrayOffset + initialPosition, array.length); + ByteBuffer buffer = ByteBuffer.wrap(arrayWrapper, arrayOffset, + initialPosition + array.length).slice(); + assertEquals(initialPosition + array.length, buffer.limit()); + assertEquals(0, buffer.position()); + buffer.position(initialPosition); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try { + ByteBufferUtils.moveBufferToStream(bos, buffer, array.length); + } catch (IOException e) { + fail("IOException in testCopyToStream()"); + } + assertArrayEquals(array, bos.toByteArray()); + assertEquals(initialPosition + array.length, buffer.position()); + } + + /** + * Test copying to stream from buffer with offset. + * @throws IOException On test failure. + */ + @Test + public void testCopyToStreamWithOffset() throws IOException { + ByteBuffer buffer = ByteBuffer.wrap(array); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + + ByteBufferUtils.copyBufferToStream(bos, buffer, array.length / 2, + array.length / 2); + + byte[] returnedArray = bos.toByteArray(); + for (int i = 0; i < array.length / 2; ++i) { + int pos = array.length / 2 + i; + assertEquals(returnedArray[i], array[pos]); + } + } + + /** + * Test copying data from stream. + * @throws IOException On test failure. + */ + @Test + public void testCopyFromStream() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(array.length); + ByteArrayInputStream bis = new ByteArrayInputStream(array); + DataInputStream dis = new DataInputStream(bis); + + ByteBufferUtils.copyFromStreamToBuffer(buffer, dis, array.length / 2); + ByteBufferUtils.copyFromStreamToBuffer(buffer, dis, + array.length - array.length / 2); + for (int i = 0; i < array.length; ++i) { + assertEquals(array[i], buffer.get(i)); + } + } + + /** + * Test copying from buffer. + */ + @Test + public void testCopyFromBuffer() { + ByteBuffer srcBuffer = ByteBuffer.allocate(array.length); + ByteBuffer dstBuffer = ByteBuffer.allocate(array.length); + srcBuffer.put(array); + + ByteBufferUtils.copyFromBufferToBuffer(dstBuffer, srcBuffer, + array.length / 2, array.length / 4); + for (int i = 0; i < array.length / 4; ++i) { + assertEquals(srcBuffer.get(i + array.length / 2), + dstBuffer.get(i)); + } + } + + /** + * Test 7-bit encoding of integers. + * @throws IOException On test failure. + */ + @Test + public void testCompressedInt() throws IOException { + testCompressedInt(0); + testCompressedInt(Integer.MAX_VALUE); + testCompressedInt(Integer.MIN_VALUE); + + for (int i = 0; i < 3; i++) { + testCompressedInt((128 << i) - 1); + } + + for (int i = 0; i < 3; i++) { + testCompressedInt((128 << i)); + } + } + + /** + * Test how much bytes we need to store integer. + */ + @Test + public void testIntFitsIn() { + assertEquals(1, ByteBufferUtils.intFitsIn(0)); + assertEquals(1, ByteBufferUtils.intFitsIn(1)); + assertEquals(2, ByteBufferUtils.intFitsIn(1 << 8)); + assertEquals(3, ByteBufferUtils.intFitsIn(1 << 16)); + assertEquals(4, ByteBufferUtils.intFitsIn(-1)); + assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MAX_VALUE)); + assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MIN_VALUE)); + } + + /** + * Test how much bytes we need to store long. + */ + @Test + public void testLongFitsIn() { + assertEquals(1, ByteBufferUtils.longFitsIn(0)); + assertEquals(1, ByteBufferUtils.longFitsIn(1)); + assertEquals(3, ByteBufferUtils.longFitsIn(1l << 16)); + assertEquals(5, ByteBufferUtils.longFitsIn(1l << 32)); + assertEquals(8, ByteBufferUtils.longFitsIn(-1)); + assertEquals(8, ByteBufferUtils.longFitsIn(Long.MIN_VALUE)); + assertEquals(8, ByteBufferUtils.longFitsIn(Long.MAX_VALUE)); + } + + /** + * Test if we are comparing equal bytes. + */ + @Test + public void testArePartEqual() { + byte[] array = new byte[] { 1, 2, 3, 4, 5, 1, 2, 3, 4 }; + ByteBuffer buffer = ByteBuffer.wrap(array); + assertTrue(ByteBufferUtils.arePartsEqual(buffer, 0, 4, 5, 4)); + assertTrue(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 2)); + assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 3)); + assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 3, 6, 2)); + assertFalse(ByteBufferUtils.arePartsEqual(buffer, 0, 3, 6, 3)); + } + + /** + * Test serializing int to bytes + */ + @Test + public void testPutInt() { + testPutInt(0); + testPutInt(Integer.MAX_VALUE); + + for (int i = 0; i < 3; i++) { + testPutInt((128 << i) - 1); + } + + for (int i = 0; i < 3; i++) { + testPutInt((128 << i)); + } + } + + // Utility methods invoked from test methods + + private void testCompressedInt(int value) throws IOException { + int parsedValue = 0; + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ByteBufferUtils.putCompressedInt(bos, value); + + ByteArrayInputStream bis = new ByteArrayInputStream( + bos.toByteArray()); + parsedValue = ByteBufferUtils.readCompressedInt(bis); + + assertEquals(value, parsedValue); + } + + private void testPutInt(int value) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + ByteBufferUtils.putInt(baos, value); + } catch (IOException e) { + throw new RuntimeException("Bug in putIn()", e); + } + + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + DataInputStream dis = new DataInputStream(bais); + try { + assertEquals(dis.readInt(), value); + } catch (IOException e) { + throw new RuntimeException("Bug in test!", e); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java b/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java new file mode 100644 index 0000000..70e707a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java @@ -0,0 +1,331 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Random; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestBytes extends TestCase { + public void testNullHashCode() { + byte [] b = null; + Exception ee = null; + try { + Bytes.hashCode(b); + } catch (Exception e) { + ee = e; + } + assertNotNull(ee); + } + + public void testSplit() throws Exception { + byte [] lowest = Bytes.toBytes("AAA"); + byte [] middle = Bytes.toBytes("CCC"); + byte [] highest = Bytes.toBytes("EEE"); + byte [][] parts = Bytes.split(lowest, highest, 1); + for (int i = 0; i < parts.length; i++) { + System.out.println(Bytes.toString(parts[i])); + } + assertEquals(3, parts.length); + assertTrue(Bytes.equals(parts[1], middle)); + // Now divide into three parts. Change highest so split is even. + highest = Bytes.toBytes("DDD"); + parts = Bytes.split(lowest, highest, 2); + for (int i = 0; i < parts.length; i++) { + System.out.println(Bytes.toString(parts[i])); + } + assertEquals(4, parts.length); + // Assert that 3rd part is 'CCC'. + assertTrue(Bytes.equals(parts[2], middle)); + } + + public void testSplit2() throws Exception { + // More split tests. + byte [] lowest = Bytes.toBytes("http://A"); + byte [] highest = Bytes.toBytes("http://z"); + byte [] middle = Bytes.toBytes("http://]"); + byte [][] parts = Bytes.split(lowest, highest, 1); + for (int i = 0; i < parts.length; i++) { + System.out.println(Bytes.toString(parts[i])); + } + assertEquals(3, parts.length); + assertTrue(Bytes.equals(parts[1], middle)); + } + + public void testSplit3() throws Exception { + // Test invalid split cases + byte [] low = { 1, 1, 1 }; + byte [] high = { 1, 1, 3 }; + + // If swapped, should throw IAE + try { + Bytes.split(high, low, 1); + assertTrue("Should not be able to split if low > high", false); + } catch(IllegalArgumentException iae) { + // Correct + } + + // Single split should work + byte [][] parts = Bytes.split(low, high, 1); + for (int i = 0; i < parts.length; i++) { + System.out.println("" + i + " -> " + Bytes.toStringBinary(parts[i])); + } + assertTrue("Returned split should have 3 parts but has " + parts.length, parts.length == 3); + + // If split more than once, this should fail + parts = Bytes.split(low, high, 2); + assertTrue("Returned split but should have failed", parts == null); + + // Split 0 times should throw IAE + try { + parts = Bytes.split(low, high, 0); + assertTrue("Should not be able to split 0 times", false); + } catch(IllegalArgumentException iae) { + // Correct + } + } + + public void testToInt() throws Exception { + int [] ints = {-1, 123, Integer.MIN_VALUE, Integer.MAX_VALUE}; + for (int i = 0; i < ints.length; i++) { + byte [] b = Bytes.toBytes(ints[i]); + assertEquals(ints[i], Bytes.toInt(b)); + byte [] b2 = bytesWithOffset(b); + assertEquals(ints[i], Bytes.toInt(b2, 1)); + assertEquals(ints[i], Bytes.toInt(b2, 1, Bytes.SIZEOF_INT)); + } + } + + public void testToLong() throws Exception { + long [] longs = {-1l, 123l, Long.MIN_VALUE, Long.MAX_VALUE}; + for (int i = 0; i < longs.length; i++) { + byte [] b = Bytes.toBytes(longs[i]); + assertEquals(longs[i], Bytes.toLong(b)); + byte [] b2 = bytesWithOffset(b); + assertEquals(longs[i], Bytes.toLong(b2, 1)); + assertEquals(longs[i], Bytes.toLong(b2, 1, Bytes.SIZEOF_LONG)); + } + } + + public void testToFloat() throws Exception { + float [] floats = {-1f, 123.123f, Float.MAX_VALUE}; + for (int i = 0; i < floats.length; i++) { + byte [] b = Bytes.toBytes(floats[i]); + assertEquals(floats[i], Bytes.toFloat(b)); + byte [] b2 = bytesWithOffset(b); + assertEquals(floats[i], Bytes.toFloat(b2, 1)); + } + } + + public void testToDouble() throws Exception { + double [] doubles = {Double.MIN_VALUE, Double.MAX_VALUE}; + for (int i = 0; i < doubles.length; i++) { + byte [] b = Bytes.toBytes(doubles[i]); + assertEquals(doubles[i], Bytes.toDouble(b)); + byte [] b2 = bytesWithOffset(b); + assertEquals(doubles[i], Bytes.toDouble(b2, 1)); + } + } + + public void testToBigDecimal() throws Exception { + BigDecimal [] decimals = {new BigDecimal("-1"), new BigDecimal("123.123"), + new BigDecimal("123123123123")}; + for (int i = 0; i < decimals.length; i++) { + byte [] b = Bytes.toBytes(decimals[i]); + assertEquals(decimals[i], Bytes.toBigDecimal(b)); + byte [] b2 = bytesWithOffset(b); + assertEquals(decimals[i], Bytes.toBigDecimal(b2, 1, b.length)); + } + } + + private byte [] bytesWithOffset(byte [] src) { + // add one byte in front to test offset + byte [] result = new byte[src.length + 1]; + result[0] = (byte) 0xAA; + System.arraycopy(src, 0, result, 1, src.length); + return result; + } + + public void testBinarySearch() throws Exception { + byte [][] arr = { + {1}, + {3}, + {5}, + {7}, + {9}, + {11}, + {13}, + {15}, + }; + byte [] key1 = {3,1}; + byte [] key2 = {4,9}; + byte [] key2_2 = {4}; + byte [] key3 = {5,11}; + byte [] key4 = {0}; + byte [] key5 = {2}; + + assertEquals(1, Bytes.binarySearch(arr, key1, 0, 1, + Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(0, Bytes.binarySearch(arr, key1, 1, 1, + Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(-(2+1), Arrays.binarySearch(arr, key2_2, + Bytes.BYTES_COMPARATOR)); + assertEquals(-(2+1), Bytes.binarySearch(arr, key2, 0, 1, + Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(4, Bytes.binarySearch(arr, key2, 1, 1, + Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(2, Bytes.binarySearch(arr, key3, 0, 1, + Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(5, Bytes.binarySearch(arr, key3, 1, 1, + Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(-1, + Bytes.binarySearch(arr, key4, 0, 1, Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(-2, + Bytes.binarySearch(arr, key5, 0, 1, Bytes.BYTES_RAWCOMPARATOR)); + + // Search for values to the left and to the right of each item in the array. + for (int i = 0; i < arr.length; ++i) { + assertEquals(-(i + 1), Bytes.binarySearch(arr, + new byte[] { (byte) (arr[i][0] - 1) }, 0, 1, + Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(-(i + 2), Bytes.binarySearch(arr, + new byte[] { (byte) (arr[i][0] + 1) }, 0, 1, + Bytes.BYTES_RAWCOMPARATOR)); + } + } + + public void testToStringBytesBinaryReversible() { + // let's run test with 1000 randomly generated byte arrays + Random rand = new Random(System.currentTimeMillis()); + byte[] randomBytes = new byte[1000]; + for (int i = 0; i < 1000; i++) { + rand.nextBytes(randomBytes); + verifyReversibleForBytes(randomBytes); + } + + + // some specific cases + verifyReversibleForBytes(new byte[] {}); + verifyReversibleForBytes(new byte[] {'\\', 'x', 'A', 'D'}); + verifyReversibleForBytes(new byte[] {'\\', 'x', 'A', 'D', '\\'}); + } + + private void verifyReversibleForBytes(byte[] originalBytes) { + String convertedString = Bytes.toStringBinary(originalBytes); + byte[] convertedBytes = Bytes.toBytesBinary(convertedString); + if (Bytes.compareTo(originalBytes, convertedBytes) != 0) { + fail("Not reversible for\nbyte[]: " + Arrays.toString(originalBytes) + + ",\nStringBinary: " + convertedString); + } + } + + public void testStartsWith() { + assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("h"))); + assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes(""))); + assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("hello"))); + assertFalse(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("helloworld"))); + assertFalse(Bytes.startsWith(Bytes.toBytes(""), Bytes.toBytes("hello"))); + } + + public void testIncrementBytes() throws IOException { + + assertTrue(checkTestIncrementBytes(10, 1)); + assertTrue(checkTestIncrementBytes(12, 123435445)); + assertTrue(checkTestIncrementBytes(124634654, 1)); + assertTrue(checkTestIncrementBytes(10005460, 5005645)); + assertTrue(checkTestIncrementBytes(1, -1)); + assertTrue(checkTestIncrementBytes(10, -1)); + assertTrue(checkTestIncrementBytes(10, -5)); + assertTrue(checkTestIncrementBytes(1005435000, -5)); + assertTrue(checkTestIncrementBytes(10, -43657655)); + assertTrue(checkTestIncrementBytes(-1, 1)); + assertTrue(checkTestIncrementBytes(-26, 5034520)); + assertTrue(checkTestIncrementBytes(-10657200, 5)); + assertTrue(checkTestIncrementBytes(-12343250, 45376475)); + assertTrue(checkTestIncrementBytes(-10, -5)); + assertTrue(checkTestIncrementBytes(-12343250, -5)); + assertTrue(checkTestIncrementBytes(-12, -34565445)); + assertTrue(checkTestIncrementBytes(-1546543452, -34565445)); + } + + private static boolean checkTestIncrementBytes(long val, long amount) + throws IOException { + byte[] value = Bytes.toBytes(val); + byte [] testValue = {-1, -1, -1, -1, -1, -1, -1, -1}; + if (value[0] > 0) { + testValue = new byte[Bytes.SIZEOF_LONG]; + } + System.arraycopy(value, 0, testValue, testValue.length - value.length, + value.length); + + long incrementResult = Bytes.toLong(Bytes.incrementBytes(value, amount)); + + return (Bytes.toLong(testValue) + amount) == incrementResult; + } + + public void testFixedSizeString() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + Bytes.writeStringFixedSize(dos, "Hello", 5); + Bytes.writeStringFixedSize(dos, "World", 18); + Bytes.writeStringFixedSize(dos, "", 9); + + try { + // Use a long dash which is three bytes in UTF-8. If encoding happens + // using ISO-8859-1, this will fail. + Bytes.writeStringFixedSize(dos, "Too\u2013Long", 9); + fail("Exception expected"); + } catch (IOException ex) { + assertEquals( + "Trying to write 10 bytes (Too\\xE2\\x80\\x93Long) into a field of " + + "length 9", ex.getMessage()); + } + + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + DataInputStream dis = new DataInputStream(bais); + assertEquals("Hello", Bytes.readStringFixedSize(dis, 5)); + assertEquals("World", Bytes.readStringFixedSize(dis, 18)); + assertEquals("", Bytes.readStringFixedSize(dis, 9)); + } + + public void testToBytesBinaryTrailingBackslashes() throws Exception { + try { + Bytes.toBytesBinary("abc\\x00\\x01\\"); + } catch (StringIndexOutOfBoundsException ex) { + fail("Illegal string access: " + ex.getMessage()); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java b/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java new file mode 100644 index 0000000..df78395 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java @@ -0,0 +1,136 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.io.compress.CompressionOutputStream; +import org.apache.hadoop.util.NativeCodeLoader; +import org.apache.hadoop.util.ReflectionUtils; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.BufferedOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +import static org.junit.Assert.*; + +@Category(SmallTests.class) +public class TestCompressionTest { + + @Test + public void testTestCompression() { + + // This test will fail if you run the tests with LZO compression available. + try { + CompressionTest.testCompression(Compression.Algorithm.LZO); + fail(); // always throws + } catch (IOException e) { + // there should be a 'cause'. + assertNotNull(e.getCause()); + } + + // this is testing the caching of the test results. + try { + CompressionTest.testCompression(Compression.Algorithm.LZO); + fail(); // always throws + } catch (IOException e) { + // there should be NO cause because it's a direct exception not wrapped + assertNull(e.getCause()); + } + + + assertFalse(CompressionTest.testCompression("LZO")); + assertTrue(CompressionTest.testCompression("NONE")); + assertTrue(CompressionTest.testCompression("GZ")); + + if (isCompressionAvailable("org.apache.hadoop.io.compress.SnappyCodec")) { + if (NativeCodeLoader.isNativeCodeLoaded()) { + try { + System.loadLibrary("snappy"); + + try { + Configuration conf = new Configuration(); + CompressionCodec codec = (CompressionCodec) + ReflectionUtils.newInstance( + conf.getClassByName("org.apache.hadoop.io.compress.SnappyCodec"), conf); + + DataOutputBuffer compressedDataBuffer = new DataOutputBuffer(); + CompressionOutputStream deflateFilter = + codec.createOutputStream(compressedDataBuffer); + + byte[] data = new byte[1024]; + DataOutputStream deflateOut = new DataOutputStream( + new BufferedOutputStream(deflateFilter)); + deflateOut.write(data, 0, data.length); + deflateOut.flush(); + deflateFilter.finish(); + + // Snappy Codec class, Snappy nativelib and Hadoop nativelib with + // Snappy JNIs are present + assertTrue(CompressionTest.testCompression("SNAPPY")); + } + catch (UnsatisfiedLinkError ex) { + // Hadoop nativelib does not have Snappy JNIs + + // cannot assert the codec here because the current logic of + // CompressionTest checks only classloading, not the codec + // usage. + } + catch (Exception ex) { + } + } + catch (UnsatisfiedLinkError ex) { + // Snappy nativelib is not available + assertFalse(CompressionTest.testCompression("SNAPPY")); + } + } + else { + // Hadoop nativelib is not available + assertFalse(CompressionTest.testCompression("SNAPPY")); + } + } + else { + // Snappy Codec class is not available + assertFalse(CompressionTest.testCompression("SNAPPY")); + } + } + + private boolean isCompressionAvailable(String codecClassName) { + try { + Thread.currentThread().getContextClassLoader().loadClass(codecClassName); + return true; + } + catch (Exception ex) { + return false; + } + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java b/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java new file mode 100644 index 0000000..15533a6 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; +// this is deliberately not in the o.a.h.h.regionserver package +// in order to make sure all required classes/method are available + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreScanner; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +@Category(MediumTests.class) +public class TestCoprocessorScanPolicy { + final Log LOG = LogFactory.getLog(getClass()); + protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final byte[] F = Bytes.toBytes("fam"); + private static final byte[] Q = Bytes.toBytes("qual"); + private static final byte[] R = Bytes.toBytes("row"); + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + ScanObserver.class.getName()); + TEST_UTIL.startMiniCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testBaseCases() throws Exception { + byte[] tableName = Bytes.toBytes("baseCases"); + HTable t = TEST_UTIL.createTable(tableName, F, 1); + // set the version override to 2 + Put p = new Put(R); + p.setAttribute("versions", new byte[]{}); + p.add(F, tableName, Bytes.toBytes(2)); + t.put(p); + + long now = EnvironmentEdgeManager.currentTimeMillis(); + + // insert 2 versions + p = new Put(R); + p.add(F, Q, now, Q); + t.put(p); + p = new Put(R); + p.add(F, Q, now+1, Q); + t.put(p); + Get g = new Get(R); + g.setMaxVersions(10); + Result r = t.get(g); + assertEquals(2, r.size()); + + TEST_UTIL.flush(tableName); + TEST_UTIL.compact(tableName, true); + + // both version are still visible even after a flush/compaction + g = new Get(R); + g.setMaxVersions(10); + r = t.get(g); + assertEquals(2, r.size()); + + // insert a 3rd version + p = new Put(R); + p.add(F, Q, now+2, Q); + t.put(p); + g = new Get(R); + g.setMaxVersions(10); + r = t.get(g); + // still only two version visible + assertEquals(2, r.size()); + + t.close(); + } + + @Test + public void testTTL() throws Exception { + byte[] tableName = Bytes.toBytes("testTTL"); + HTableDescriptor desc = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor(F) + .setMaxVersions(10) + .setTimeToLive(1); + desc.addFamily(hcd); + TEST_UTIL.getHBaseAdmin().createTable(desc); + HTable t = new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName); + long now = EnvironmentEdgeManager.currentTimeMillis(); + ManualEnvironmentEdge me = new ManualEnvironmentEdge(); + me.setValue(now); + EnvironmentEdgeManagerTestHelper.injectEdge(me); + // 2s in the past + long ts = now - 2000; + // Set the TTL override to 3s + Put p = new Put(R); + p.setAttribute("ttl", new byte[]{}); + p.add(F, tableName, Bytes.toBytes(3000L)); + t.put(p); + + p = new Put(R); + p.add(F, Q, ts, Q); + t.put(p); + p = new Put(R); + p.add(F, Q, ts+1, Q); + t.put(p); + + // these two should be expired but for the override + // (their ts was 2s in the past) + Get g = new Get(R); + g.setMaxVersions(10); + Result r = t.get(g); + // still there? + assertEquals(2, r.size()); + + TEST_UTIL.flush(tableName); + TEST_UTIL.compact(tableName, true); + + g = new Get(R); + g.setMaxVersions(10); + r = t.get(g); + // still there? + assertEquals(2, r.size()); + + // roll time forward 2s. + me.setValue(now + 2000); + // now verify that data eventually does expire + g = new Get(R); + g.setMaxVersions(10); + r = t.get(g); + // should be gone now + assertEquals(0, r.size()); + t.close(); + } + + public static class ScanObserver extends BaseRegionObserver { + private Map ttls = new HashMap(); + private Map versions = new HashMap(); + + // lame way to communicate with the coprocessor, + // since it is loaded by a different class loader + @Override + public void prePut(final ObserverContext c, final Put put, + final WALEdit edit, final boolean writeToWAL) throws IOException { + if (put.getAttribute("ttl") != null) { + KeyValue kv = put.getFamilyMap().values().iterator().next().get(0); + ttls.put(Bytes.toString(kv.getQualifier()), Bytes.toLong(kv.getValue())); + c.bypass(); + } else if (put.getAttribute("versions") != null) { + KeyValue kv = put.getFamilyMap().values().iterator().next().get(0); + versions.put(Bytes.toString(kv.getQualifier()), Bytes.toInt(kv.getValue())); + c.bypass(); + } + } + + @Override + public InternalScanner preFlushScannerOpen(final ObserverContext c, + Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException { + Long newTtl = ttls.get(store.getTableName()); + if (newTtl != null) { + System.out.println("PreFlush:" + newTtl); + } + Integer newVersions = versions.get(store.getTableName()); + Store.ScanInfo oldSI = store.getScanInfo(); + HColumnDescriptor family = store.getFamily(); + Store.ScanInfo scanInfo = new Store.ScanInfo(family.getName(), family.getMinVersions(), + newVersions == null ? family.getMaxVersions() : newVersions, + newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(), + oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + Scan scan = new Scan(); + scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions); + return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner), + ScanType.MINOR_COMPACT, store.getHRegion().getSmallestReadPoint(), + HConstants.OLDEST_TIMESTAMP); + } + + @Override + public InternalScanner preCompactScannerOpen(final ObserverContext c, + Store store, List scanners, ScanType scanType, + long earliestPutTs, InternalScanner s) throws IOException { + Long newTtl = ttls.get(store.getTableName()); + Integer newVersions = versions.get(store.getTableName()); + Store.ScanInfo oldSI = store.getScanInfo(); + HColumnDescriptor family = store.getFamily(); + Store.ScanInfo scanInfo = new Store.ScanInfo(family.getName(), family.getMinVersions(), + newVersions == null ? family.getMaxVersions() : newVersions, + newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(), + oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + Scan scan = new Scan(); + scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions); + return new StoreScanner(store, scanInfo, scan, scanners, scanType, store.getHRegion() + .getSmallestReadPoint(), earliestPutTs); + } + + @Override + public KeyValueScanner preStoreScannerOpen( + final ObserverContext c, Store store, final Scan scan, + final NavigableSet targetCols, KeyValueScanner s) throws IOException { + Long newTtl = ttls.get(store.getTableName()); + Integer newVersions = versions.get(store.getTableName()); + Store.ScanInfo oldSI = store.getScanInfo(); + HColumnDescriptor family = store.getFamily(); + Store.ScanInfo scanInfo = new Store.ScanInfo(family.getName(), family.getMinVersions(), + newVersions == null ? family.getMaxVersions() : newVersions, + newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(), + oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + return new StoreScanner(store, scanInfo, scan, targetCols); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestDefaultEnvironmentEdge.java b/src/test/java/org/apache/hadoop/hbase/util/TestDefaultEnvironmentEdge.java new file mode 100644 index 0000000..ea6903e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestDefaultEnvironmentEdge.java @@ -0,0 +1,58 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.hbase.MediumTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static junit.framework.Assert.assertTrue; +import static junit.framework.Assert.fail; + +/** + * Tests to make sure that the default environment edge conforms to appropriate + * behaviour. + */ +@Category(MediumTests.class) +public class TestDefaultEnvironmentEdge { + + @Test + public void testGetCurrentTimeUsesSystemClock() { + DefaultEnvironmentEdge edge = new DefaultEnvironmentEdge(); + long systemTime = System.currentTimeMillis(); + long edgeTime = edge.currentTimeMillis(); + assertTrue("System time must be either the same or less than the edge time", + systemTime < edgeTime || systemTime == edgeTime); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + fail(e.getMessage()); + } + long secondEdgeTime = edge.currentTimeMillis(); + assertTrue("Second time must be greater than the first", + secondEdgeTime > edgeTime); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java b/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java new file mode 100644 index 0000000..de2f77e --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java @@ -0,0 +1,121 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +import java.io.File; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test TestDynamicClassLoader + */ +@Category(SmallTests.class) +public class TestDynamicClassLoader { + private static final Log LOG = LogFactory.getLog(TestDynamicClassLoader.class); + + private static final Configuration conf = HBaseConfiguration.create(); + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + static { + conf.set("hbase.dynamic.jars.dir", TEST_UTIL.getDataTestDir().toString()); + } + + @Test + public void testLoadClassFromLocalPath() throws Exception { + ClassLoader parent = TestDynamicClassLoader.class.getClassLoader(); + DynamicClassLoader classLoader = new DynamicClassLoader(conf, parent); + + String className = "TestLoadClassFromLocalPath"; + deleteClass(className); + try { + classLoader.loadClass(className); + fail("Should not be able to load class " + className); + } catch (ClassNotFoundException cnfe) { + // expected, move on + } + + try { + String folder = TEST_UTIL.getDataTestDir().toString(); + ClassLoaderTestHelper.buildJar(folder, className, null, localDirPath()); + classLoader.loadClass(className); + } catch (ClassNotFoundException cnfe) { + LOG.error("Should be able to load class " + className, cnfe); + fail(cnfe.getMessage()); + } + } + + @Test + public void testLoadClassFromAnotherPath() throws Exception { + ClassLoader parent = TestDynamicClassLoader.class.getClassLoader(); + DynamicClassLoader classLoader = new DynamicClassLoader(conf, parent); + + String className = "TestLoadClassFromAnotherPath"; + deleteClass(className); + try { + classLoader.loadClass(className); + fail("Should not be able to load class " + className); + } catch (ClassNotFoundException cnfe) { + // expected, move on + } + + try { + String folder = TEST_UTIL.getDataTestDir().toString(); + ClassLoaderTestHelper.buildJar(folder, className, null); + classLoader.loadClass(className); + } catch (ClassNotFoundException cnfe) { + LOG.error("Should be able to load class " + className, cnfe); + fail(cnfe.getMessage()); + } + } + + private String localDirPath() { + return conf.get("hbase.local.dir") + + File.separator + "jars" + File.separator; + } + + private void deleteClass(String className) throws Exception { + String jarFileName = className + ".jar"; + File file = new File(TEST_UTIL.getDataTestDir().toString(), jarFileName); + file.delete(); + assertFalse("Should be deleted: " + file.getPath(), file.exists()); + + file = new File(conf.get("hbase.dynamic.jars.dir"), jarFileName); + file.delete(); + assertFalse("Should be deleted: " + file.getPath(), file.exists()); + + file = new File(localDirPath(), jarFileName); + file.delete(); + assertFalse("Should be deleted: " + file.getPath(), file.exists()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java b/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java new file mode 100644 index 0000000..59b2c2b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java @@ -0,0 +1,71 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.hbase.MediumTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +@Category(MediumTests.class) +public class TestEnvironmentEdgeManager { + + @Test + public void testManageSingleton() { + EnvironmentEdge edge = EnvironmentEdgeManager.getDelegate(); + assertNotNull(edge); + assertTrue(edge instanceof DefaultEnvironmentEdge); + EnvironmentEdgeManager.reset(); + EnvironmentEdge edge2 = EnvironmentEdgeManager.getDelegate(); + assertFalse(edge == edge2); + IncrementingEnvironmentEdge newEdge = new IncrementingEnvironmentEdge(); + EnvironmentEdgeManager.injectEdge(newEdge); + assertEquals(newEdge, EnvironmentEdgeManager.getDelegate()); + + //injecting null will result in default being assigned. + EnvironmentEdgeManager.injectEdge(null); + EnvironmentEdge nullResult = EnvironmentEdgeManager.getDelegate(); + assertTrue(nullResult instanceof DefaultEnvironmentEdge); + } + + @Test + public void testCurrentTimeInMillis() { + EnvironmentEdge mock = mock(EnvironmentEdge.class); + EnvironmentEdgeManager.injectEdge(mock); + long expectation = 3456; + when(mock.currentTimeMillis()).thenReturn(expectation); + long result = EnvironmentEdgeManager.currentTimeMillis(); + verify(mock).currentTimeMillis(); + assertEquals(expectation, result); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java new file mode 100644 index 0000000..c5dda2f --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.*; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.junit.Test; +import org.junit.experimental.categories.Category; + + +/** + * Tests for {@link FSTableDescriptors}. + */ +// Do not support to be executed in he same JVM as other tests +@Category(MediumTests.class) +public class TestFSTableDescriptors { + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final Log LOG = LogFactory.getLog(TestFSTableDescriptors.class); + + @Test (expected=IllegalArgumentException.class) + public void testRegexAgainstOldStyleTableInfo() { + Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_NAME); + int i = FSTableDescriptors.getTableInfoSequenceid(p); + assertEquals(0, i); + // Assert it won't eat garbage -- that it fails + p = new Path("/tmp", "abc"); + FSTableDescriptors.getTableInfoSequenceid(p); + } + + @Test + public void testCreateAndUpdate() throws IOException { + Path testdir = UTIL.getDataTestDir("testCreate"); + HTableDescriptor htd = new HTableDescriptor("testCreate"); + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + assertTrue(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); + assertFalse(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); + FileStatus [] statuses = fs.listStatus(testdir); + assertTrue("statuses.length="+statuses.length, statuses.length == 1); + for (int i = 0; i < 10; i++) { + FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); + } + statuses = fs.listStatus(testdir); + assertTrue(statuses.length == 1); + Path tmpTableDir = new Path(FSUtils.getTablePath(testdir, htd.getName()), ".tmp"); + statuses = fs.listStatus(tmpTableDir); + assertTrue(statuses.length == 0); + } + + @Test + public void testSequenceidAdvancesOnTableInfo() throws IOException { + Path testdir = UTIL.getDataTestDir("testSequenceidAdvancesOnTableInfo"); + HTableDescriptor htd = new HTableDescriptor("testSequenceidAdvancesOnTableInfo"); + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + Path p0 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); + int i0 = FSTableDescriptors.getTableInfoSequenceid(p0); + Path p1 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); + // Assert we cleaned up the old file. + assertTrue(!fs.exists(p0)); + int i1 = FSTableDescriptors.getTableInfoSequenceid(p1); + assertTrue(i1 == i0 + 1); + Path p2 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); + // Assert we cleaned up the old file. + assertTrue(!fs.exists(p1)); + int i2 = FSTableDescriptors.getTableInfoSequenceid(p2); + assertTrue(i2 == i1 + 1); + } + + @Test + public void testFormatTableInfoSequenceId() { + Path p0 = assertWriteAndReadSequenceid(0); + // Assert p0 has format we expect. + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) { + sb.append("0"); + } + assertEquals(FSTableDescriptors.TABLEINFO_NAME + "." + sb.toString(), + p0.getName()); + // Check a few more. + Path p2 = assertWriteAndReadSequenceid(2); + Path p10000 = assertWriteAndReadSequenceid(10000); + // Get a .tablinfo that has no sequenceid suffix. + Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_NAME); + FileStatus fs = new FileStatus(0, false, 0, 0, 0, p); + FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0); + FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2); + FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000); + FSTableDescriptors.FileStatusFileNameComparator comparator = + new FSTableDescriptors.FileStatusFileNameComparator(); + assertTrue(comparator.compare(fs, fs0) > 0); + assertTrue(comparator.compare(fs0, fs2) > 0); + assertTrue(comparator.compare(fs2, fs10000) > 0); + } + + private Path assertWriteAndReadSequenceid(final int i) { + Path p = FSTableDescriptors.getTableInfoFileName(new Path("/tmp"), i); + int ii = FSTableDescriptors.getTableInfoSequenceid(p); + assertEquals(i, ii); + return p; + } + + @Test + public void testRemoves() throws IOException { + final String name = "testRemoves"; + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + // Cleanup old tests if any detrius laying around. + Path rootdir = new Path(UTIL.getDataTestDir(), name); + TableDescriptors htds = new FSTableDescriptors(fs, rootdir); + HTableDescriptor htd = new HTableDescriptor(name); + htds.add(htd); + assertNotNull(htds.remove(htd.getNameAsString())); + assertNull(htds.remove(htd.getNameAsString())); + } + + @Test public void testReadingHTDFromFS() throws IOException { + final String name = "testReadingHTDFromFS"; + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + HTableDescriptor htd = new HTableDescriptor(name); + Path rootdir = UTIL.getDataTestDir(name); + createHTDInFS(fs, rootdir, htd); + HTableDescriptor htd2 = + FSTableDescriptors.getTableDescriptor(fs, rootdir, htd.getNameAsString()); + assertTrue(htd.equals(htd2)); + } + + private void createHTDInFS(final FileSystem fs, Path rootdir, + final HTableDescriptor htd) + throws IOException { + FSTableDescriptors.createTableDescriptor(fs, rootdir, htd); + } + + @Test public void testHTableDescriptors() + throws IOException, InterruptedException { + final String name = "testHTableDescriptors"; + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + // Cleanup old tests if any debris laying around. + Path rootdir = new Path(UTIL.getDataTestDir(), name); + final int count = 10; + // Write out table infos. + for (int i = 0; i < count; i++) { + HTableDescriptor htd = new HTableDescriptor(name + i); + createHTDInFS(fs, rootdir, htd); + } + FSTableDescriptors htds = new FSTableDescriptors(fs, rootdir) { + @Override + public HTableDescriptor get(byte[] tablename) + throws TableExistsException, FileNotFoundException, IOException { + LOG.info(Bytes.toString(tablename) + ", cachehits=" + this.cachehits); + return super.get(tablename); + } + }; + for (int i = 0; i < count; i++) { + assertTrue(htds.get(Bytes.toBytes(name + i)) != null); + } + for (int i = 0; i < count; i++) { + assertTrue(htds.get(Bytes.toBytes(name + i)) != null); + } + // Update the table infos + for (int i = 0; i < count; i++) { + HTableDescriptor htd = new HTableDescriptor(name + i); + htd.addFamily(new HColumnDescriptor("" + i)); + FSTableDescriptors.updateHTableDescriptor(fs, rootdir, htd); + } + // Wait a while so mod time we write is for sure different. + Thread.sleep(100); + for (int i = 0; i < count; i++) { + assertTrue(htds.get(Bytes.toBytes(name + i)) != null); + } + for (int i = 0; i < count; i++) { + assertTrue(htds.get(Bytes.toBytes(name + i)) != null); + } + assertEquals(count * 4, htds.invocations); + assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits, + htds.cachehits >= (count * 2)); + assertTrue(htds.get(HConstants.ROOT_TABLE_NAME) != null); + assertEquals(htds.invocations, count * 4 + 1); + assertTrue("expected=" + ((count * 2) + 1) + ", actual=" + htds.cachehits, + htds.cachehits >= ((count * 2) + 1)); + } + + @Test + public void testNoSuchTable() throws IOException { + final String name = "testNoSuchTable"; + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + // Cleanup old tests if any detrius laying around. + Path rootdir = new Path(UTIL.getDataTestDir(), name); + TableDescriptors htds = new FSTableDescriptors(fs, rootdir); + assertNull("There shouldn't be any HTD for this table", + htds.get("NoSuchTable")); + } + + @Test + public void testUpdates() throws IOException { + final String name = "testUpdates"; + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + // Cleanup old tests if any detrius laying around. + Path rootdir = new Path(UTIL.getDataTestDir(), name); + TableDescriptors htds = new FSTableDescriptors(fs, rootdir); + HTableDescriptor htd = new HTableDescriptor(name); + htds.add(htd); + htds.add(htd); + htds.add(htd); + } + + @Test + public void testTableInfoFileStatusComparator() { + FileStatus bare = + new FileStatus(0, false, 0, 0, -1, new Path("/tmp", FSTableDescriptors.TABLEINFO_NAME)); + FileStatus future = + new FileStatus(0, false, 0, 0, -1, + new Path("/tmp/tablinfo." + System.currentTimeMillis())); + FileStatus farFuture = + new FileStatus(0, false, 0, 0, -1, + new Path("/tmp/tablinfo." + System.currentTimeMillis() + 1000)); + FileStatus [] alist = {bare, future, farFuture}; + FileStatus [] blist = {bare, farFuture, future}; + FileStatus [] clist = {farFuture, bare, future}; + FSTableDescriptors.FileStatusFileNameComparator c = + new FSTableDescriptors.FileStatusFileNameComparator(); + Arrays.sort(alist, c); + Arrays.sort(blist, c); + Arrays.sort(clist, c); + // Now assert all sorted same in way we want. + for (int i = 0; i < alist.length; i++) { + assertTrue(alist[i].equals(blist[i])); + assertTrue(blist[i].equals(clist[i])); + assertTrue(clist[i].equals(i == 0? farFuture: i == 1? future: bare)); + } + } + + @Test + public void testReadingArchiveDirectoryFromFS() throws IOException { + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + try { + new FSTableDescriptors(fs, FSUtils.getRootDir(UTIL.getConfiguration())) + .get(HConstants.HFILE_ARCHIVE_DIRECTORY); + fail("Shouldn't be able to read a table descriptor for the archive directory."); + } catch (IOException e) { + LOG.debug("Correctly got error when reading a table descriptor from the archive directory: " + + e.getMessage()); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java new file mode 100644 index 0000000..21ac529 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java @@ -0,0 +1,239 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.util.UUID; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HDFSBlocksDistribution; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test {@link FSUtils}. + */ +@Category(MediumTests.class) +public class TestFSUtils { + @Test public void testIsHDFS() throws Exception { + HBaseTestingUtility htu = new HBaseTestingUtility(); + htu.getConfiguration().setBoolean("dfs.support.append", false); + assertFalse(FSUtils.isHDFS(htu.getConfiguration())); + htu.getConfiguration().setBoolean("dfs.support.append", true); + MiniDFSCluster cluster = null; + try { + cluster = htu.startMiniDFSCluster(1); + assertTrue(FSUtils.isHDFS(htu.getConfiguration())); + assertTrue(FSUtils.isAppendSupported(htu.getConfiguration())); + } finally { + if (cluster != null) cluster.shutdown(); + } + } + + private void WriteDataToHDFS(FileSystem fs, Path file, int dataSize) + throws Exception { + FSDataOutputStream out = fs.create(file); + byte [] data = new byte[dataSize]; + out.write(data, 0, dataSize); + out.close(); + } + + @Test public void testcomputeHDFSBlocksDistribution() throws Exception { + HBaseTestingUtility htu = new HBaseTestingUtility(); + final int DEFAULT_BLOCK_SIZE = 1024; + htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE); + MiniDFSCluster cluster = null; + Path testFile = null; + + try { + // set up a cluster with 3 nodes + String hosts[] = new String[] { "host1", "host2", "host3" }; + cluster = htu.startMiniDFSCluster(hosts); + cluster.waitActive(); + FileSystem fs = cluster.getFileSystem(); + + // create a file with two blocks + testFile = new Path("/test1.txt"); + WriteDataToHDFS(fs, testFile, 2*DEFAULT_BLOCK_SIZE); + + // given the default replication factor is 3, the same as the number of + // datanodes; the locality index for each host should be 100%, + // or getWeight for each host should be the same as getUniqueBlocksWeights + final long maxTime = System.currentTimeMillis() + 2000; + boolean ok; + do { + ok = true; + FileStatus status = fs.getFileStatus(testFile); + HDFSBlocksDistribution blocksDistribution = + FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen()); + long uniqueBlocksTotalWeight = + blocksDistribution.getUniqueBlocksTotalWeight(); + for (String host : hosts) { + long weight = blocksDistribution.getWeight(host); + ok = (ok && uniqueBlocksTotalWeight == weight); + } + } while (!ok && System.currentTimeMillis() < maxTime); + assertTrue(ok); + } finally { + htu.shutdownMiniDFSCluster(); + } + + + try { + // set up a cluster with 4 nodes + String hosts[] = new String[] { "host1", "host2", "host3", "host4" }; + cluster = htu.startMiniDFSCluster(hosts); + cluster.waitActive(); + FileSystem fs = cluster.getFileSystem(); + + // create a file with three blocks + testFile = new Path("/test2.txt"); + WriteDataToHDFS(fs, testFile, 3*DEFAULT_BLOCK_SIZE); + + // given the default replication factor is 3, we will have total of 9 + // replica of blocks; thus the host with the highest weight should have + // weight == 3 * DEFAULT_BLOCK_SIZE + final long maxTime = System.currentTimeMillis() + 2000; + long weight; + long uniqueBlocksTotalWeight; + do { + FileStatus status = fs.getFileStatus(testFile); + HDFSBlocksDistribution blocksDistribution = + FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen()); + uniqueBlocksTotalWeight = blocksDistribution.getUniqueBlocksTotalWeight(); + + String tophost = blocksDistribution.getTopHosts().get(0); + weight = blocksDistribution.getWeight(tophost); + + // NameNode is informed asynchronously, so we may have a delay. See HBASE-6175 + } while (uniqueBlocksTotalWeight != weight && System.currentTimeMillis() < maxTime); + assertTrue(uniqueBlocksTotalWeight == weight); + + } finally { + htu.shutdownMiniDFSCluster(); + } + + + try { + // set up a cluster with 4 nodes + String hosts[] = new String[] { "host1", "host2", "host3", "host4" }; + cluster = htu.startMiniDFSCluster(hosts); + cluster.waitActive(); + FileSystem fs = cluster.getFileSystem(); + + // create a file with one block + testFile = new Path("/test3.txt"); + WriteDataToHDFS(fs, testFile, DEFAULT_BLOCK_SIZE); + + // given the default replication factor is 3, we will have total of 3 + // replica of blocks; thus there is one host without weight + final long maxTime = System.currentTimeMillis() + 2000; + HDFSBlocksDistribution blocksDistribution; + do { + FileStatus status = fs.getFileStatus(testFile); + blocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen()); + // NameNode is informed asynchronously, so we may have a delay. See HBASE-6175 + } + while (blocksDistribution.getTopHosts().size() != 3 && System.currentTimeMillis() < maxTime); + assertEquals("Wrong number of hosts distributing blocks.", 3, + blocksDistribution.getTopHosts().size()); + } finally { + htu.shutdownMiniDFSCluster(); + } + } + + @Test + public void testPermMask() throws Exception { + + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(HConstants.ENABLE_DATA_FILE_UMASK, true); + FileSystem fs = FileSystem.get(conf); + // first check that we don't crash if we don't have perms set + FsPermission defaultPerms = FSUtils.getFilePermissions(fs, conf, + HConstants.DATA_FILE_UMASK_KEY); + assertEquals(FsPermission.getDefault(), defaultPerms); + + conf.setStrings(HConstants.DATA_FILE_UMASK_KEY, "077"); + // now check that we get the right perms + FsPermission filePerm = FSUtils.getFilePermissions(fs, conf, + HConstants.DATA_FILE_UMASK_KEY); + assertEquals(new FsPermission("700"), filePerm); + + // then that the correct file is created + Path p = new Path("target" + File.separator + UUID.randomUUID().toString()); + try { + FSDataOutputStream out = FSUtils.create(fs, p, filePerm); + out.close(); + FileStatus stat = fs.getFileStatus(p); + assertEquals(new FsPermission("700"), stat.getPermission()); + // and then cleanup + } finally { + fs.delete(p, true); + } + } + + @Test + public void testDeleteAndExists() throws Exception { + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(HConstants.ENABLE_DATA_FILE_UMASK, true); + FileSystem fs = FileSystem.get(conf); + FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY); + // then that the correct file is created + String file = UUID.randomUUID().toString(); + Path p = new Path("temptarget" + File.separator + file); + Path p1 = new Path("temppath" + File.separator + file); + try { + FSDataOutputStream out = FSUtils.create(fs, p, perms); + out.close(); + assertTrue("The created file should be present", FSUtils.isExists(fs, p)); + // delete the file with recursion as false. Only the file will be deleted. + FSUtils.delete(fs, p, false); + // Create another file + FSDataOutputStream out1 = FSUtils.create(fs, p1, perms); + out1.close(); + // delete the file with recursion as false. Still the file only will be deleted + FSUtils.delete(fs, p1, true); + assertFalse("The created file should be present", FSUtils.isExists(fs, p1)); + // and then cleanup + } finally { + FSUtils.delete(fs, p, true); + FSUtils.delete(fs, p1, true); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java b/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java new file mode 100644 index 0000000..c2c95b1 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java @@ -0,0 +1,225 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.UUID; +import java.util.Set; +import java.util.HashSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HDFSBlocksDistribution; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.util.MD5Hash; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.*; +import org.junit.experimental.categories.Category; + +/** + * Test {@link FSUtils}. + */ +@Category(MediumTests.class) +public class TestFSVisitor { + final Log LOG = LogFactory.getLog(getClass()); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private final String TABLE_NAME = "testtb"; + + private Set tableFamilies; + private Set tableRegions; + private Set recoveredEdits; + private Set tableHFiles; + private Set regionServers; + private Set serverLogs; + + private FileSystem fs; + private Path tableDir; + private Path logsDir; + private Path rootDir; + + @Before + public void setUp() throws Exception { + fs = FileSystem.get(TEST_UTIL.getConfiguration()); + rootDir = TEST_UTIL.getDataTestDir("hbase"); + logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); + + tableFamilies = new HashSet(); + tableRegions = new HashSet(); + recoveredEdits = new HashSet(); + tableHFiles = new HashSet(); + regionServers = new HashSet(); + serverLogs = new HashSet(); + tableDir = createTableFiles(rootDir, TABLE_NAME, tableRegions, tableFamilies, tableHFiles); + createRecoverEdits(tableDir, tableRegions, recoveredEdits); + createLogs(logsDir, regionServers, serverLogs); + FSUtils.logFileSystemState(fs, rootDir, LOG); + } + + @After + public void tearDown() throws Exception { + fs.delete(rootDir); + } + + @Test + public void testVisitStoreFiles() throws IOException { + final Set regions = new HashSet(); + final Set families = new HashSet(); + final Set hfiles = new HashSet(); + FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() { + public void storeFile(final String region, final String family, final String hfileName) + throws IOException { + regions.add(region); + families.add(family); + hfiles.add(hfileName); + } + }); + assertEquals(tableRegions, regions); + assertEquals(tableFamilies, families); + assertEquals(tableHFiles, hfiles); + } + + @Test + public void testVisitRecoveredEdits() throws IOException { + final Set regions = new HashSet(); + final Set edits = new HashSet(); + FSVisitor.visitTableRecoveredEdits(fs, tableDir, new FSVisitor.RecoveredEditsVisitor() { + public void recoveredEdits (final String region, final String logfile) + throws IOException { + regions.add(region); + edits.add(logfile); + } + }); + assertEquals(tableRegions, regions); + assertEquals(recoveredEdits, edits); + } + + @Test + public void testVisitLogFiles() throws IOException { + final Set servers = new HashSet(); + final Set logs = new HashSet(); + FSVisitor.visitLogFiles(fs, rootDir, new FSVisitor.LogFileVisitor() { + public void logFile (final String server, final String logfile) throws IOException { + servers.add(server); + logs.add(logfile); + } + }); + assertEquals(regionServers, servers); + assertEquals(serverLogs, logs); + } + + + /* + * |-testtb/ + * |----f1d3ff8443297732862df21dc4e57262/ + * |-------f1/ + * |----------d0be84935ba84b66b1e866752ec5d663 + * |----------9fc9d481718f4878b29aad0a597ecb94 + * |-------f2/ + * |----------4b0fe6068c564737946bcf4fd4ab8ae1 + */ + private Path createTableFiles(final Path rootDir, final String tableName, + final Set tableRegions, final Set tableFamilies, + final Set tableHFiles) throws IOException { + Path tableDir = new Path(rootDir, tableName); + for (int r = 0; r < 10; ++r) { + String regionName = MD5Hash.getMD5AsHex(Bytes.toBytes(r)); + tableRegions.add(regionName); + Path regionDir = new Path(tableDir, regionName); + for (int f = 0; f < 3; ++f) { + String familyName = "f" + f; + tableFamilies.add(familyName); + Path familyDir = new Path(regionDir, familyName); + fs.mkdirs(familyDir); + for (int h = 0; h < 5; ++h) { + String hfileName = UUID.randomUUID().toString().replaceAll("-", ""); + tableHFiles.add(hfileName); + fs.createNewFile(new Path(familyDir, hfileName)); + } + } + } + return tableDir; + } + + /* + * |-testtb/ + * |----f1d3ff8443297732862df21dc4e57262/ + * |-------recovered.edits/ + * |----------0000001351969633479 + * |----------0000001351969633481 + */ + private void createRecoverEdits(final Path tableDir, final Set tableRegions, + final Set recoverEdits) throws IOException { + for (String region: tableRegions) { + Path regionEditsDir = HLog.getRegionDirRecoveredEditsDir(new Path(tableDir, region)); + long seqId = System.currentTimeMillis(); + for (int i = 0; i < 3; ++i) { + String editName = String.format("%019d", seqId + i); + recoverEdits.add(editName); + FSDataOutputStream stream = fs.create(new Path(regionEditsDir, editName)); + stream.write(Bytes.toBytes("test")); + stream.close(); + } + } + } + + /* + * |-.logs/ + * |----server5,5,1351969633508/ + * |-------server5,5,1351969633508.0 + * |----server6,6,1351969633512/ + * |-------server6,6,1351969633512.0 + * |-------server6,6,1351969633512.3 + */ + private void createLogs(final Path logDir, final Set servers, + final Set logs) throws IOException { + for (int s = 0; s < 7; ++s) { + String server = String.format("server%d,%d,%d", s, s, System.currentTimeMillis()); + servers.add(server); + Path serverLogDir = new Path(logDir, server); + fs.mkdirs(serverLogDir); + for (int i = 0; i < 5; ++i) { + String logfile = server + '.' + i; + logs.add(logfile); + FSDataOutputStream stream = fs.create(new Path(serverLogDir, logfile)); + stream.write(Bytes.toBytes("test")); + stream.close(); + } + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java new file mode 100644 index 0000000..84d8db8 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -0,0 +1,1897 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors; +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.executor.RegionTransitionData; +import org.apache.hadoop.hbase.io.hfile.TestHFile; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction; +import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter; +import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; +import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo; +import org.apache.hadoop.hbase.util.HBaseFsck.PrintingErrorReporter; +import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo; +import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; +import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +import com.google.common.collect.Multimap; + +/** + * This tests HBaseFsck's ability to detect reasons for inconsistent tables. + */ +@Category(LargeTests.class) +public class TestHBaseFsck { + final static Log LOG = LogFactory.getLog(TestHBaseFsck.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static Configuration conf = TEST_UTIL.getConfiguration(); + private final static String FAM_STR = "fam"; + private final static byte[] FAM = Bytes.toBytes(FAM_STR); + private final static int REGION_ONLINE_TIMEOUT = 800; + + // for the instance, reset every test run + private HTable tbl; + private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"), + Bytes.toBytes("B"), Bytes.toBytes("C") }; + // one row per region. + private final static byte[][] ROWKEYS= new byte[][] { + Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"), + Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") }; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean("hbase.master.distributed.log.splitting", false); + TEST_UTIL.startMiniCluster(3); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testHBaseFsck() throws Exception { + assertNoErrors(doFsck(conf, false)); + String table = "tableBadMetaAssign"; + TEST_UTIL.createTable(Bytes.toBytes(table), FAM); + + // We created 1 table, should be fine + assertNoErrors(doFsck(conf, false)); + + // Now let's mess it up and change the assignment in .META. to + // point to a different region server + HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName()); + ResultScanner scanner = meta.getScanner(new Scan()); + + resforloop: + for (Result res : scanner) { + long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY, + HConstants.STARTCODE_QUALIFIER)); + + for (JVMClusterUtil.RegionServerThread rs : + TEST_UTIL.getHBaseCluster().getRegionServerThreads()) { + + ServerName sn = rs.getRegionServer().getServerName(); + + // When we find a diff RS, change the assignment and break + if (startCode != sn.getStartcode()) { + Put put = new Put(res.getRow()); + put.setWriteToWAL(false); + put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, + Bytes.toBytes(sn.getHostAndPort())); + put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, + Bytes.toBytes(sn.getStartcode())); + meta.put(put); + break resforloop; + } + } + } + + // Try to fix the data + assertErrors(doFsck(conf, true), new ERROR_CODE[]{ + ERROR_CODE.SERVER_DOES_NOT_MATCH_META}); + + // fixing assignments require opening regions is not synchronous. To make + // the test pass consistently so for now we bake in some sleep to let it + // finish. 1s seems sufficient. + Thread.sleep(1000); + + // Should be fixed now + assertNoErrors(doFsck(conf, false)); + + // comment needed - what is the purpose of this line + HTable t = new HTable(conf, Bytes.toBytes(table)); + ResultScanner s = t.getScanner(new Scan()); + s.close(); + t.close(); + + scanner.close(); + meta.close(); + } + + /** + * Create a new region in META. + */ + private HRegionInfo createRegion(Configuration conf, final HTableDescriptor + htd, byte[] startKey, byte[] endKey) + throws IOException { + HTable meta = new HTable(conf, HConstants.META_TABLE_NAME); + HRegionInfo hri = new HRegionInfo(htd.getName(), startKey, endKey); + Put put = new Put(hri.getRegionName()); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + meta.put(put); + return hri; + } + + /** + * Debugging method to dump the contents of meta. + */ + private void dumpMeta(byte[] tableName) throws IOException { + List metaRows = TEST_UTIL.getMetaTableRows(tableName); + for (byte[] row : metaRows) { + LOG.info(Bytes.toString(row)); + } + } + + /** + * This method is used to undeploy a region -- close it and attempt to + * remove its state from the Master. + */ + private void undeployRegion(HBaseAdmin admin, ServerName sn, + HRegionInfo hri) throws IOException, InterruptedException { + try { + HBaseFsckRepair.closeRegionSilentlyAndWait(admin, sn, hri); + admin.getMaster().offline(hri.getRegionName()); + } catch (IOException ioe) { + LOG.warn("Got exception when attempting to offline region " + + Bytes.toString(hri.getRegionName()), ioe); + } + } + /** + * Delete a region from assignments, meta, or completely from hdfs. + * @param unassign if true unassign region if assigned + * @param metaRow if true remove region's row from META + * @param hdfs if true remove region's dir in HDFS + */ + private void deleteRegion(Configuration conf, final HTableDescriptor htd, + byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow, + boolean hdfs) throws IOException, InterruptedException { + deleteRegion(conf, htd, startKey, endKey, unassign, metaRow, hdfs, false); + } + + /** + * Delete a region from assignments, meta, or completely from hdfs. + * @param unassign if true unassign region if assigned + * @param metaRow if true remove region's row from META + * @param hdfs if true remove region's dir in HDFS + * @param regionInfoOnly if true remove a region dir's .regioninfo file + */ + private void deleteRegion(Configuration conf, final HTableDescriptor htd, + byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow, + boolean hdfs, boolean regionInfoOnly) throws IOException, InterruptedException { + LOG.info("** Before delete:"); + dumpMeta(htd.getName()); + + Map hris = tbl.getRegionLocations(); + for (Entry e: hris.entrySet()) { + HRegionInfo hri = e.getKey(); + ServerName hsa = e.getValue(); + if (Bytes.compareTo(hri.getStartKey(), startKey) == 0 + && Bytes.compareTo(hri.getEndKey(), endKey) == 0) { + + LOG.info("RegionName: " +hri.getRegionNameAsString()); + byte[] deleteRow = hri.getRegionName(); + + if (unassign) { + LOG.info("Undeploying region " + hri + " from server " + hsa); + undeployRegion(new HBaseAdmin(conf), hsa, new HRegionInfo(hri)); + } + + if (regionInfoOnly) { + LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString()); + Path rootDir = new Path(conf.get(HConstants.HBASE_DIR)); + FileSystem fs = rootDir.getFileSystem(conf); + Path p = new Path(rootDir + "/" + htd.getNameAsString(), hri.getEncodedName()); + Path hriPath = new Path(p, HRegion.REGIONINFO_FILE); + fs.delete(hriPath, true); + } + + if (hdfs) { + LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString()); + Path rootDir = new Path(conf.get(HConstants.HBASE_DIR)); + FileSystem fs = rootDir.getFileSystem(conf); + Path p = new Path(rootDir + "/" + htd.getNameAsString(), hri.getEncodedName()); + HBaseFsck.debugLsr(conf, p); + boolean success = fs.delete(p, true); + LOG.info("Deleted " + p + " sucessfully? " + success); + HBaseFsck.debugLsr(conf, p); + } + + if (metaRow) { + HTable meta = new HTable(conf, HConstants.META_TABLE_NAME); + Delete delete = new Delete(deleteRow); + meta.delete(delete); + } + } + LOG.info(hri.toString() + hsa.toString()); + } + + TEST_UTIL.getMetaTableRows(htd.getName()); + LOG.info("*** After delete:"); + dumpMeta(htd.getName()); + } + + /** + * Setup a clean table before we start mucking with it. + * + * @throws IOException + * @throws InterruptedException + * @throws KeeperException + */ + HTable setupTable(String tablename) throws Exception { + HTableDescriptor desc = new HTableDescriptor(tablename); + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM)); + desc.addFamily(hcd); // If a table has no CF's it doesn't get checked + TEST_UTIL.getHBaseAdmin().createTable(desc, SPLITS); + tbl = new HTable(TEST_UTIL.getConfiguration(), tablename); + + List puts = new ArrayList(); + for (byte[] row : ROWKEYS) { + Put p = new Put(row); + p.add(FAM, Bytes.toBytes("val"), row); + puts.add(p); + } + tbl.put(puts); + tbl.flushCommits(); + long endTime = System.currentTimeMillis() + 60000; + while (!TEST_UTIL.getHBaseAdmin().isTableEnabled(tablename)) { + try { + if (System.currentTimeMillis() > endTime) { + fail("Failed to enable table " + tablename + " after waiting for 60 sec"); + } + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + fail("Interrupted when waiting table " + tablename + " to be enabled"); + } + } + return tbl; + } + + /** + * Counts the number of row to verify data loss or non-dataloss. + */ + int countRows() throws IOException { + Scan s = new Scan(); + ResultScanner rs = tbl.getScanner(s); + int i = 0; + while(rs.next() !=null) { + i++; + } + return i; + } + + /** + * delete table in preparation for next test + * + * @param tablename + * @throws IOException + */ + void deleteTable(String tablename) throws IOException { + HBaseAdmin admin = new HBaseAdmin(conf); + admin.getConnection().clearRegionCache(); + byte[] tbytes = Bytes.toBytes(tablename); + admin.disableTableAsync(tbytes); + while (!admin.isTableDisabled(tbytes)) { + try { + Thread.sleep(250); + } catch (InterruptedException e) { + e.printStackTrace(); + fail("Interrupted when trying to disable table " + tablename); + } + } + admin.deleteTable(tbytes); + } + + /** + * This creates a clean table and confirms that the table is clean. + */ + @Test + public void testHBaseFsckClean() throws Exception { + assertNoErrors(doFsck(conf, false)); + String table = "tableClean"; + try { + HBaseFsck hbck = doFsck(conf, false); + assertNoErrors(hbck); + + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // We created 1 table, should be fine + hbck = doFsck(conf, false); + assertNoErrors(hbck); + assertEquals(0, hbck.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * Test thread pooling in the case where there are more regions than threads + */ + @Test + public void testHbckThreadpooling() throws Exception { + String table = "tableDupeStartKey"; + try { + // Create table with 4 regions + setupTable(table); + + // limit number of threads to 1. + Configuration newconf = new Configuration(conf); + newconf.setInt("hbasefsck.numthreads", 1); + assertNoErrors(doFsck(newconf, false)); + + // We should pass without triggering a RejectedExecutionException + } finally { + deleteTable(table); + } + } + + @Test + public void testHbckFixOrphanTable() throws Exception { + String table = "tableInfo"; + FileSystem fs = null; + Path tableinfo = null; + try { + setupTable(table); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + + Path hbaseTableDir = new Path(conf.get(HConstants.HBASE_DIR) + "/" + table ); + fs = hbaseTableDir.getFileSystem(conf); + FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir); + tableinfo = status.getPath(); + fs.rename(tableinfo, new Path("/.tableinfo")); + + //to report error if .tableinfo is missing. + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLEINFO_FILE }); + + // fix OrphanTable with default .tableinfo (htd not yet cached on master) + hbck = doFsck(conf, true); + assertNoErrors(hbck); + status = null; + status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir); + assertNotNull(status); + + HTableDescriptor htd = admin.getTableDescriptor(table.getBytes()); + htd.setValue("NOT_DEFAULT", "true"); + admin.disableTable(table); + admin.modifyTable(table.getBytes(), htd); + admin.enableTable(table); + fs.delete(status.getPath(), true); + + // fix OrphanTable with cache + htd = admin.getTableDescriptor(table.getBytes()); // warms up cached htd on master + hbck = doFsck(conf, true); + assertNoErrors(hbck); + status = null; + status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir); + assertNotNull(status); + htd = admin.getTableDescriptor(table.getBytes()); + assertEquals(htd.getValue("NOT_DEFAULT"), "true"); + } finally { + fs.rename(new Path("/.tableinfo"), tableinfo); + deleteTable(table); + } + } + + /** + * This create and fixes a bad table with regions that have a duplicate + * start key + */ + @Test + public void testDupeStartKey() throws Exception { + String table = "tableDupeStartKey"; + try { + setupTable(table); + assertNoErrors(doFsck(conf, false)); + assertEquals(ROWKEYS.length, countRows()); + + // Now let's mess it up, by adding a region with a duplicate startkey + HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(), + Bytes.toBytes("A"), Bytes.toBytes("A2")); + TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe); + TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .waitForAssignment(hriDupe); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS, + ERROR_CODE.DUPE_STARTKEYS}); + assertEquals(2, hbck.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); // seems like the "bigger" region won. + + // fix the degenerate region. + doFsck(conf,true); + + // check that the degenerate region is gone and no data loss + HBaseFsck hbck2 = doFsck(conf,false); + assertNoErrors(hbck2); + assertEquals(0, hbck2.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * Get region info from local cluster. + */ + Map> getDeployedHRIs(HBaseAdmin admin) + throws IOException { + ClusterStatus status = admin.getMaster().getClusterStatus(); + Collection regionServers = status.getServers(); + Map> mm = + new HashMap>(); + HConnection connection = admin.getConnection(); + for (ServerName hsi : regionServers) { + HRegionInterface server = + connection.getHRegionConnection(hsi.getHostname(), hsi.getPort()); + + // list all online regions from this region server + List regions = server.getOnlineRegions(); + List regionNames = new ArrayList(); + for (HRegionInfo hri : regions) { + regionNames.add(hri.getRegionNameAsString()); + } + mm.put(hsi, regionNames); + } + return mm; + } + + /** + * Returns the HSI a region info is on. + */ + ServerName findDeployedHSI(Map> mm, HRegionInfo hri) { + for (Map.Entry> e : mm.entrySet()) { + if (e.getValue().contains(hri.getRegionNameAsString())) { + return e.getKey(); + } + } + return null; + } + + /** + * This create and fixes a bad table with regions that have a duplicate + * start key + */ + @Test + public void testDupeRegion() throws Exception { + String table = "tableDupeRegion"; + try { + setupTable(table); + assertNoErrors(doFsck(conf, false)); + assertEquals(ROWKEYS.length, countRows()); + + // Now let's mess it up, by adding a region with a duplicate startkey + HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(), + Bytes.toBytes("A"), Bytes.toBytes("B")); + + TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe); + TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .waitForAssignment(hriDupe); + + // Yikes! The assignment manager can't tell between diff between two + // different regions with the same start/endkeys since it doesn't + // differentiate on ts/regionId! We actually need to recheck + // deployments! + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + while (findDeployedHSI(getDeployedHRIs(admin), hriDupe) == null) { + Thread.sleep(250); + } + + LOG.debug("Finished assignment of dupe region"); + + // TODO why is dupe region different from dupe start keys? + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS, + ERROR_CODE.DUPE_STARTKEYS}); + assertEquals(2, hbck.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); // seems like the "bigger" region won. + + // fix the degenerate region. + doFsck(conf,true); + + // check that the degenerate region is gone and no data loss + HBaseFsck hbck2 = doFsck(conf,false); + assertNoErrors(hbck2); + assertEquals(0, hbck2.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table with regions that has startkey == endkey + */ + @Test + public void testDegenerateRegions() throws Exception { + String table = "tableDegenerateRegions"; + try { + setupTable(table); + assertNoErrors(doFsck(conf,false)); + assertEquals(ROWKEYS.length, countRows()); + + // Now let's mess it up, by adding a region with a duplicate startkey + HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(), + Bytes.toBytes("B"), Bytes.toBytes("B")); + TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe); + TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .waitForAssignment(hriDupe); + + HBaseFsck hbck = doFsck(conf,false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DEGENERATE_REGION, + ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.DUPE_STARTKEYS}); + assertEquals(2, hbck.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); + + // fix the degenerate region. + doFsck(conf,true); + + // check that the degenerate region is gone and no data loss + HBaseFsck hbck2 = doFsck(conf,false); + assertNoErrors(hbck2); + assertEquals(0, hbck2.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table where a region is completely contained + * by another region. + */ + @Test + public void testContainedRegionOverlap() throws Exception { + String table = "tableContainedRegionOverlap"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by creating an overlap in the metadata + HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(), + Bytes.toBytes("A2"), Bytes.toBytes("B")); + TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap); + TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .waitForAssignment(hriOverlap); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.OVERLAP_IN_REGION_CHAIN }); + assertEquals(2, hbck.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); + + // fix the problem. + doFsck(conf, true); + + // verify that overlaps are fixed + HBaseFsck hbck2 = doFsck(conf,false); + assertNoErrors(hbck2); + assertEquals(0, hbck2.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table where an overlap group of + * 3 regions. Set HBaseFsck.maxMerge to 2 to trigger sideline overlapped + * region. Mess around the meta data so that closeRegion/offlineRegion + * throws exceptions. + */ + @Test + public void testSidelineOverlapRegion() throws Exception { + String table = "testSidelineOverlapRegion"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by creating an overlap + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + HRegionInfo hriOverlap1 = createRegion(conf, tbl.getTableDescriptor(), + Bytes.toBytes("A"), Bytes.toBytes("AB")); + master.assignRegion(hriOverlap1); + master.getAssignmentManager().waitForAssignment(hriOverlap1); + HRegionInfo hriOverlap2 = createRegion(conf, tbl.getTableDescriptor(), + Bytes.toBytes("AB"), Bytes.toBytes("B")); + master.assignRegion(hriOverlap2); + master.getAssignmentManager().waitForAssignment(hriOverlap2); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.DUPE_STARTKEYS, + ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.OVERLAP_IN_REGION_CHAIN}); + assertEquals(3, hbck.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); + + // mess around the overlapped regions, to trigger NotServingRegionException + Multimap overlapGroups = hbck.getOverlapGroups(table); + ServerName serverName = null; + byte[] regionName = null; + for (HbckInfo hbi: overlapGroups.values()) { + if ("A".equals(Bytes.toString(hbi.getStartKey())) + && "B".equals(Bytes.toString(hbi.getEndKey()))) { + regionName = hbi.getRegionName(); + + // get an RS not serving the region to force bad assignment info in to META. + int k = cluster.getServerWith(regionName); + for (int i = 0; i < 3; i++) { + if (i != k) { + HRegionServer rs = cluster.getRegionServer(i); + serverName = rs.getServerName(); + break; + } + } + + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + HBaseFsckRepair.closeRegionSilentlyAndWait(admin, + cluster.getRegionServer(k).getServerName(), hbi.getHdfsHRI()); + admin.unassign(regionName, true); + break; + } + } + + assertNotNull(regionName); + assertNotNull(serverName); + HTable meta = new HTable(conf, HConstants.META_TABLE_NAME); + Put put = new Put(regionName); + put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, + Bytes.toBytes(serverName.getHostAndPort())); + meta.put(put); + + // fix the problem. + HBaseFsck fsck = new HBaseFsck(conf); + fsck.connect(); + fsck.setDisplayFullReport(); // i.e. -details + fsck.setTimeLag(0); + fsck.setFixAssignments(true); + fsck.setFixMeta(true); + fsck.setFixHdfsHoles(true); + fsck.setFixHdfsOverlaps(true); + fsck.setFixHdfsOrphans(true); + fsck.setFixVersionFile(true); + fsck.setSidelineBigOverlaps(true); + fsck.setMaxMerge(2); + fsck.onlineHbck(); + + // verify that overlaps are fixed, and there are less rows + // since one region is sidelined. + HBaseFsck hbck2 = doFsck(conf,false); + assertNoErrors(hbck2); + assertEquals(0, hbck2.getOverlapGroups(table).size()); + assertTrue(ROWKEYS.length > countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table where a region is completely contained + * by another region, and there is a hole (sort of like a bad split) + */ + @Test + public void testOverlapAndOrphan() throws Exception { + String table = "tableOverlapAndOrphan"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by creating an overlap in the metadata + TEST_UTIL.getHBaseAdmin().disableTable(table); + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"), + Bytes.toBytes("B"), true, true, false, true); + TEST_UTIL.getHBaseAdmin().enableTable(table); + + HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(), + Bytes.toBytes("A2"), Bytes.toBytes("B")); + TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap); + TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .waitForAssignment(hriOverlap); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // fix the problem. + doFsck(conf, true); + + // verify that overlaps are fixed + HBaseFsck hbck2 = doFsck(conf,false); + assertNoErrors(hbck2); + assertEquals(0, hbck2.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table where a region overlaps two regions -- + * a start key contained in another region and its end key is contained in + * yet another region. + */ + @Test + public void testCoveredStartKey() throws Exception { + String table = "tableCoveredStartKey"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by creating an overlap in the metadata + HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(), + Bytes.toBytes("A2"), Bytes.toBytes("B2")); + TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap); + TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .waitForAssignment(hriOverlap); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.OVERLAP_IN_REGION_CHAIN, + ERROR_CODE.OVERLAP_IN_REGION_CHAIN }); + assertEquals(3, hbck.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); + + // fix the problem. + doFsck(conf, true); + + // verify that overlaps are fixed + HBaseFsck hbck2 = doFsck(conf, false); + assertErrors(hbck2, new ERROR_CODE[0]); + assertEquals(0, hbck2.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table with a missing region -- hole in meta + * and data missing in the fs. + */ + @Test + public void testRegionHole() throws Exception { + String table = "tableRegionHole"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by leaving a hole in the assignment, meta, and hdfs data + TEST_UTIL.getHBaseAdmin().disableTable(table); + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), + Bytes.toBytes("C"), true, true, true); + TEST_UTIL.getHBaseAdmin().enableTable(table); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.HOLE_IN_REGION_CHAIN}); + // holes are separate from overlap groups + assertEquals(0, hbck.getOverlapGroups(table).size()); + + // fix hole + doFsck(conf, true); + + // check that hole fixed + assertNoErrors(doFsck(conf,false)); + assertEquals(ROWKEYS.length - 2 , countRows()); // lost a region so lost a row + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table with a missing region -- hole in meta + * and data present but .regioinfino missing (an orphan hdfs region)in the fs. + */ + @Test + public void testHDFSRegioninfoMissing() throws Exception { + String table = "tableHDFSRegioininfoMissing"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by leaving a hole in the meta data + TEST_UTIL.getHBaseAdmin().disableTable(table); + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), + Bytes.toBytes("C"), true, true, false, true); + TEST_UTIL.getHBaseAdmin().enableTable(table); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.ORPHAN_HDFS_REGION, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.HOLE_IN_REGION_CHAIN}); + // holes are separate from overlap groups + assertEquals(0, hbck.getOverlapGroups(table).size()); + + // fix hole + doFsck(conf, true); + + // check that hole fixed + assertNoErrors(doFsck(conf, false)); + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table with a region that is missing meta and + * not assigned to a region server. + */ + @Test + public void testNotInMetaOrDeployedHole() throws Exception { + String table = "tableNotInMetaOrDeployedHole"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by leaving a hole in the meta data + TEST_UTIL.getHBaseAdmin().disableTable(table); + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), + Bytes.toBytes("C"), true, true, false); // don't rm from fs + TEST_UTIL.getHBaseAdmin().enableTable(table); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + // holes are separate from overlap groups + assertEquals(0, hbck.getOverlapGroups(table).size()); + + // fix hole + assertErrors(doFsck(conf, true) , new ERROR_CODE[] { + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // check that hole fixed + assertNoErrors(doFsck(conf,false)); + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates fixes a bad table with a hole in meta. + */ + @Test + public void testNotInMetaHole() throws Exception { + String table = "tableNotInMetaHole"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by leaving a hole in the meta data + TEST_UTIL.getHBaseAdmin().disableTable(table); + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), + Bytes.toBytes("C"), false, true, false); // don't rm from fs + TEST_UTIL.getHBaseAdmin().enableTable(table); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + // holes are separate from overlap groups + assertEquals(0, hbck.getOverlapGroups(table).size()); + + // fix hole + assertErrors(doFsck(conf, true) , new ERROR_CODE[] { + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // check that hole fixed + assertNoErrors(doFsck(conf,false)); + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table with a region that is in meta but has + * no deployment or data hdfs + */ + @Test + public void testNotInHdfs() throws Exception { + String table = "tableNotInHdfs"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // make sure data in regions, if in hlog only there is no data loss + TEST_UTIL.getHBaseAdmin().flush(table); + + // Mess it up by leaving a hole in the hdfs data + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), + Bytes.toBytes("C"), false, false, true); // don't rm meta + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS}); + // holes are separate from overlap groups + assertEquals(0, hbck.getOverlapGroups(table).size()); + + // fix hole + doFsck(conf, true); + + // check that hole fixed + assertNoErrors(doFsck(conf,false)); + assertEquals(ROWKEYS.length - 2, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates entries in META with no hdfs data. This should cleanly + * remove the table. + */ + @Test + public void testNoHdfsTable() throws Exception { + String table = "NoHdfsTable"; + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // make sure data in regions, if in hlog only there is no data loss + TEST_UTIL.getHBaseAdmin().flush(table); + + // Mess it up by leaving a giant hole in meta + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), + Bytes.toBytes("A"), false, false, true); // don't rm meta + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"), + Bytes.toBytes("B"), false, false, true); // don't rm meta + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), + Bytes.toBytes("C"), false, false, true); // don't rm meta + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), + Bytes.toBytes(""), false, false, true); // don't rm meta + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS, + ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS, + ERROR_CODE.NOT_IN_HDFS,}); + // holes are separate from overlap groups + assertEquals(0, hbck.getOverlapGroups(table).size()); + + // fix hole + doFsck(conf, true); // in 0.92+, meta entries auto create regiondirs + + // check that hole fixed + assertNoErrors(doFsck(conf,false)); + assertFalse("Table "+ table + " should have been deleted", + TEST_UTIL.getHBaseAdmin().tableExists(table)); + } + + /** + * when the hbase.version file missing, It is fix the fault. + */ + @Test + public void testNoVersionFile() throws Exception { + // delete the hbase.version file + Path rootDir = new Path(conf.get(HConstants.HBASE_DIR)); + FileSystem fs = rootDir.getFileSystem(conf); + Path versionFile = new Path(rootDir, HConstants.VERSION_FILE_NAME); + fs.delete(versionFile, true); + + // test + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_VERSION_FILE }); + // fix hbase.version missing + doFsck(conf, true); + + // no version file fixed + assertNoErrors(doFsck(conf, false)); + } + + /** + * the region is not deployed when the table is disabled. + */ + @Test + public void testRegionShouldNotBeDeployed() throws Exception { + String table = "tableRegionShouldNotBeDeployed"; + try { + LOG.info("Starting testRegionShouldNotBeDeployed."); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + assertTrue(cluster.waitForActiveAndReadyMaster()); + + // Create a ZKW to use in the test + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL); + + FileSystem filesystem = FileSystem.get(conf); + Path rootdir = filesystem.makeQualified(new Path(conf + .get(HConstants.HBASE_DIR))); + + byte[][] SPLIT_KEYS = new byte[][] { new byte[0], Bytes.toBytes("aaa"), + Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), Bytes.toBytes("ddd") }; + HTableDescriptor htdDisabled = new HTableDescriptor(Bytes.toBytes(table)); + htdDisabled.addFamily(new HColumnDescriptor(FAM)); + + // Write the .tableinfo + FSTableDescriptors + .createTableDescriptor(filesystem, rootdir, htdDisabled); + List disabledRegions = TEST_UTIL.createMultiRegionsInMeta( + TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS); + + // Let's just assign everything to first RS + HRegionServer hrs = cluster.getRegionServer(0); + ServerName serverName = hrs.getServerName(); + + // create region files. + TEST_UTIL.getHBaseAdmin().disableTable(table); + TEST_UTIL.getHBaseAdmin().enableTable(table); + + // Region of disable table was opened on RS + TEST_UTIL.getHBaseAdmin().disableTable(table); + HRegionInfo region = disabledRegions.remove(0); + ZKAssign.createNodeOffline(zkw, region, serverName); + hrs.openRegion(region); + + int iTimes = 0; + while (true) { + RegionTransitionData rtd = ZKAssign.getData(zkw, + region.getEncodedName()); + if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) { + break; + } + Thread.sleep(100); + iTimes++; + if (iTimes >= REGION_ONLINE_TIMEOUT) { + break; + } + } + assertTrue(iTimes < REGION_ONLINE_TIMEOUT); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.SHOULD_NOT_BE_DEPLOYED }); + + // fix this fault + doFsck(conf, true); + + // check result + assertNoErrors(doFsck(conf, false)); + } finally { + TEST_UTIL.getHBaseAdmin().enableTable(table); + deleteTable(table); + } + } + + /** + * This creates two tables and mess both of them and fix them one by one + */ + @Test + public void testFixByTable() throws Exception { + String table1 = "testFixByTable1"; + String table2 = "testFixByTable2"; + try { + setupTable(table1); + // make sure data in regions, if in hlog only there is no data loss + TEST_UTIL.getHBaseAdmin().flush(table1); + // Mess them up by leaving a hole in the hdfs data + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), + Bytes.toBytes("C"), false, false, true); // don't rm meta + + setupTable(table2); + // make sure data in regions, if in hlog only there is no data loss + TEST_UTIL.getHBaseAdmin().flush(table2); + // Mess them up by leaving a hole in the hdfs data + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), + Bytes.toBytes("C"), false, false, true); // don't rm meta + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS}); + + // fix hole in table 1 + doFsck(conf, true, table1); + // check that hole in table 1 fixed + assertNoErrors(doFsck(conf, false, table1)); + // check that hole in table 2 still there + assertErrors(doFsck(conf, false, table2), + new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS}); + + // fix hole in table 2 + doFsck(conf, true, table2); + // check that hole in both tables fixed + assertNoErrors(doFsck(conf, false)); + assertEquals(ROWKEYS.length - 2, countRows()); + } finally { + deleteTable(table1); + deleteTable(table2); + } + } + /** + * A split parent in meta, in hdfs, and not deployed + */ + @Test + public void testLingeringSplitParent() throws Exception { + String table = "testLingeringSplitParent"; + HTable meta = null; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // make sure data in regions, if in hlog only there is no data loss + TEST_UTIL.getHBaseAdmin().flush(table); + HRegionLocation location = tbl.getRegionLocation("B"); + + // Delete one region from meta, but not hdfs, unassign it. + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), + Bytes.toBytes("C"), true, true, false); + + // Create a new meta entry to fake it as a split parent. + meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName()); + HRegionInfo hri = location.getRegionInfo(); + + HRegionInfo a = new HRegionInfo(tbl.getTableName(), + Bytes.toBytes("B"), Bytes.toBytes("BM")); + HRegionInfo b = new HRegionInfo(tbl.getTableName(), + Bytes.toBytes("BM"), Bytes.toBytes("C")); + Put p = new Put(hri.getRegionName()); + hri.setOffline(true); + hri.setSplit(true); + p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + p.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER, + Writables.getBytes(a)); + p.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER, + Writables.getBytes(b)); + meta.put(p); + meta.flushCommits(); + TEST_UTIL.getHBaseAdmin().flush(HConstants.META_TABLE_NAME); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // regular repair cannot fix lingering split parent + hbck = doFsck(conf, true); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + assertFalse(hbck.shouldRerun()); + hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // fix lingering split parent + hbck = new HBaseFsck(conf); + hbck.connect(); + hbck.setDisplayFullReport(); // i.e. -details + hbck.setTimeLag(0); + hbck.setFixSplitParents(true); + hbck.onlineHbck(); + assertTrue(hbck.shouldRerun()); + + Get get = new Get(hri.getRegionName()); + Result result = meta.get(get); + assertTrue(result.getColumn(HConstants.CATALOG_FAMILY, + HConstants.SPLITA_QUALIFIER).isEmpty()); + assertTrue(result.getColumn(HConstants.CATALOG_FAMILY, + HConstants.SPLITB_QUALIFIER).isEmpty()); + TEST_UTIL.getHBaseAdmin().flush(HConstants.META_TABLE_NAME); + + // fix other issues + doFsck(conf, true); + + // check that all are fixed + assertNoErrors(doFsck(conf, false)); + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + IOUtils.closeQuietly(meta); + } + } + + /** + * Tests that LINGERING_SPLIT_PARENT is not erroneously reported for + * valid cases where the daughters are there. + */ + @Test + public void testValidLingeringSplitParent() throws Exception { + String table = "testLingeringSplitParent"; + HTable meta = null; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // make sure data in regions, if in hlog only there is no data loss + TEST_UTIL.getHBaseAdmin().flush(table); + HRegionLocation location = tbl.getRegionLocation("B"); + + meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName()); + HRegionInfo hri = location.getRegionInfo(); + + // do a regular split + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + byte[] regionName = location.getRegionInfo().getRegionName(); + admin.split(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM")); + TestEndToEndSplitTransaction.blockUntilRegionSplit( + TEST_UTIL.getConfiguration(), 60000, regionName, true); + + // TODO: fixHdfsHoles does not work against splits, since the parent dir lingers on + // for some time until children references are deleted. HBCK erroneously sees this as + // overlapping regions + HBaseFsck hbck = doFsck(conf, true, true, false, false, false, true, true, true, null); + assertErrors(hbck, new ERROR_CODE[] {}); //no LINGERING_SPLIT_PARENT reported + + // assert that the split META entry is still there. + Get get = new Get(hri.getRegionName()); + Result result = meta.get(get); + assertNotNull(result); + assertNotNull(MetaReader.parseCatalogResult(result).getFirst()); + + assertEquals(ROWKEYS.length, countRows()); + + // assert that we still have the split regions + assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1); //SPLITS + 1 is # regions pre-split. + assertNoErrors(doFsck(conf, false)); + } finally { + deleteTable(table); + IOUtils.closeQuietly(meta); + } + } + + /** + * Split crashed after write to META finished for the parent region, but + * failed to write daughters (pre HBASE-7721 codebase) + */ + @Test + public void testSplitDaughtersNotInMeta() throws Exception { + String table = "testSplitdaughtersNotInMeta"; + HTable meta = null; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // make sure data in regions, if in hlog only there is no data loss + TEST_UTIL.getHBaseAdmin().flush(table); + HRegionLocation location = tbl.getRegionLocation("B"); + + meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName()); + HRegionInfo hri = location.getRegionInfo(); + + // do a regular split + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + byte[] regionName = location.getRegionInfo().getRegionName(); + admin.split(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM")); + TestEndToEndSplitTransaction.blockUntilRegionSplit( + TEST_UTIL.getConfiguration(), 60000, regionName, true); + + PairOfSameType daughters = MetaReader.getDaughterRegions(meta.get(new Get(regionName))); + + // Delete daughter regions from meta, but not hdfs, unassign it. + Map hris = tbl.getRegionLocations(); + undeployRegion(admin, hris.get(daughters.getFirst()), daughters.getFirst()); + undeployRegion(admin, hris.get(daughters.getSecond()), daughters.getSecond()); + + meta.delete(new Delete(daughters.getFirst().getRegionName())); + meta.delete(new Delete(daughters.getSecond().getRegionName())); + meta.flushCommits(); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN}); //no LINGERING_SPLIT_PARENT + + // now fix it. The fix should not revert the region split, but add daughters to META + hbck = doFsck(conf, true, true, false, false, false, false, false, false, null); + assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // assert that the split META entry is still there. + Get get = new Get(hri.getRegionName()); + Result result = meta.get(get); + assertNotNull(result); + assertNotNull(MetaReader.parseCatalogResult(result).getFirst()); + + assertEquals(ROWKEYS.length, countRows()); + + // assert that we still have the split regions + assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1); //SPLITS + 1 is # regions pre-split. + assertNoErrors(doFsck(conf, false)); //should be fixed by now + } finally { + deleteTable(table); + IOUtils.closeQuietly(meta); + } + } + + /** + * This creates and fixes a bad table with a missing region which is the 1st region -- hole in + * meta and data missing in the fs. + */ + @Test + public void testMissingFirstRegion() throws Exception { + String table = "testMissingFirstRegion"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by leaving a hole in the assignment, meta, and hdfs data + TEST_UTIL.getHBaseAdmin().disableTable(table); + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes("A"), true, + true, true); + TEST_UTIL.getHBaseAdmin().enableTable(table); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY }); + // fix hole + doFsck(conf, true); + // check that hole fixed + assertNoErrors(doFsck(conf, false)); + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table with missing last region -- hole in meta and data missing in + * the fs. + */ + @Test + public void testMissingLastRegion() throws Exception { + String table = "testMissingLastRegion"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by leaving a hole in the assignment, meta, and hdfs data + TEST_UTIL.getHBaseAdmin().disableTable(table); + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""), true, + true, true); + TEST_UTIL.getHBaseAdmin().enableTable(table); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY }); + // fix hole + doFsck(conf, true); + // check that hole fixed + assertNoErrors(doFsck(conf, false)); + } finally { + deleteTable(table); + } + } + + /** + * Test -noHdfsChecking option can detect and fix assignments issue. + */ + @Test + public void testFixAssignmentsAndNoHdfsChecking() throws Exception { + String table = "testFixAssignmentsAndNoHdfsChecking"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by closing a region + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"), + Bytes.toBytes("B"), true, false, false, false); + + // verify there is no other errors + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // verify that noHdfsChecking report the same errors + HBaseFsck fsck = new HBaseFsck(conf); + fsck.connect(); + fsck.setDisplayFullReport(); // i.e. -details + fsck.setTimeLag(0); + fsck.setCheckHdfs(false); + fsck.onlineHbck(); + assertErrors(fsck, new ERROR_CODE[] { + ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // verify that fixAssignments works fine with noHdfsChecking + fsck = new HBaseFsck(conf); + fsck.connect(); + fsck.setDisplayFullReport(); // i.e. -details + fsck.setTimeLag(0); + fsck.setCheckHdfs(false); + fsck.setFixAssignments(true); + fsck.onlineHbck(); + assertTrue(fsck.shouldRerun()); + fsck.onlineHbck(); + assertNoErrors(fsck); + + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * Test -noHdfsChecking option can detect region is not in meta but deployed. + * However, it can not fix it without checking Hdfs because we need to get + * the region info from Hdfs in this case, then to patch the meta. + */ + @Test + public void testFixMetaNotWorkingWithNoHdfsChecking() throws Exception { + String table = "testFixMetaNotWorkingWithNoHdfsChecking"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by deleting a region from the metadata + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"), + Bytes.toBytes("B"), false, true, false, false); + + // verify there is no other errors + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // verify that noHdfsChecking report the same errors + HBaseFsck fsck = new HBaseFsck(conf); + fsck.connect(); + fsck.setDisplayFullReport(); // i.e. -details + fsck.setTimeLag(0); + fsck.setCheckHdfs(false); + fsck.onlineHbck(); + assertErrors(fsck, new ERROR_CODE[] { + ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // verify that fixMeta doesn't work with noHdfsChecking + fsck = new HBaseFsck(conf); + fsck.connect(); + fsck.setDisplayFullReport(); // i.e. -details + fsck.setTimeLag(0); + fsck.setCheckHdfs(false); + fsck.setFixAssignments(true); + fsck.setFixMeta(true); + fsck.onlineHbck(); + assertFalse(fsck.shouldRerun()); + assertErrors(fsck, new ERROR_CODE[] { + ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN}); + } finally { + deleteTable(table); + } + } + + /** + * Test -fixHdfsHoles doesn't work with -noHdfsChecking option, + * and -noHdfsChecking can't detect orphan Hdfs region. + */ + @Test + public void testFixHdfsHolesNotWorkingWithNoHdfsChecking() throws Exception { + String table = "testFixHdfsHolesNotWorkingWithNoHdfsChecking"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by creating an overlap in the metadata + TEST_UTIL.getHBaseAdmin().disableTable(table); + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"), + Bytes.toBytes("B"), true, true, false, true); + TEST_UTIL.getHBaseAdmin().enableTable(table); + + HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(), + Bytes.toBytes("A2"), Bytes.toBytes("B")); + TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap); + TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .waitForAssignment(hriOverlap); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { + ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // verify that noHdfsChecking can't detect ORPHAN_HDFS_REGION + HBaseFsck fsck = new HBaseFsck(conf); + fsck.connect(); + fsck.setDisplayFullReport(); // i.e. -details + fsck.setTimeLag(0); + fsck.setCheckHdfs(false); + fsck.onlineHbck(); + assertErrors(fsck, new ERROR_CODE[] { + ERROR_CODE.HOLE_IN_REGION_CHAIN}); + + // verify that fixHdfsHoles doesn't work with noHdfsChecking + fsck = new HBaseFsck(conf); + fsck.connect(); + fsck.setDisplayFullReport(); // i.e. -details + fsck.setTimeLag(0); + fsck.setCheckHdfs(false); + fsck.setFixHdfsHoles(true); + fsck.setFixHdfsOverlaps(true); + fsck.setFixHdfsOrphans(true); + fsck.onlineHbck(); + assertFalse(fsck.shouldRerun()); + assertErrors(fsck, new ERROR_CODE[] { + ERROR_CODE.HOLE_IN_REGION_CHAIN}); + } finally { + if (TEST_UTIL.getHBaseAdmin().isTableDisabled(table)) { + TEST_UTIL.getHBaseAdmin().enableTable(table); + } + deleteTable(table); + } + } + + /** + * We don't have an easy way to verify that a flush completed, so we loop until we find a + * legitimate hfile and return it. + * @param fs + * @param table + * @return Path of a flushed hfile. + * @throws IOException + */ + Path getFlushedHFile(FileSystem fs, String table) throws IOException { + Path tableDir= FSUtils.getTablePath(FSUtils.getRootDir(conf), table); + Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0); + Path famDir = new Path(regionDir, FAM_STR); + + // keep doing this until we get a legit hfile + while (true) { + FileStatus[] hfFss = fs.listStatus(famDir); + if (hfFss.length == 0) { + continue; + } + for (FileStatus hfs : hfFss) { + if (!hfs.isDir()) { + return hfs.getPath(); + } + } + } + } + + /** + * This creates a table and then corrupts an hfile. Hbck should quarantine the file. + */ + @Test(timeout=120000) + public void testQuarantineCorruptHFile() throws Exception { + String table = name.getMethodName(); + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + TEST_UTIL.getHBaseAdmin().flush(table); // flush is async. + + FileSystem fs = FileSystem.get(conf); + Path hfile = getFlushedHFile(fs, table); + + // Mess it up by leaving a hole in the assignment, meta, and hdfs data + TEST_UTIL.getHBaseAdmin().disableTable(table); + + // create new corrupt file called deadbeef (valid hfile name) + Path corrupt = new Path(hfile.getParent(), "deadbeef"); + TestHFile.truncateFile(fs, hfile, corrupt); + LOG.info("Created corrupted file " + corrupt); + HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf)); + + // we cannot enable here because enable never finished due to the corrupt region. + HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table); + assertEquals(res.getRetCode(), 0); + HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker(); + assertEquals(hfcc.getHFilesChecked(), 5); + assertEquals(hfcc.getCorrupted().size(), 1); + assertEquals(hfcc.getFailures().size(), 0); + assertEquals(hfcc.getQuarantined().size(), 1); + assertEquals(hfcc.getMissing().size(), 0); + + // Its been fixed, verify that we can enable. + TEST_UTIL.getHBaseAdmin().enableTable(table); + } finally { + deleteTable(table); + } + } + + /** + * Test that use this should have a timeout, because this method could potentially wait forever. + */ + private void doQuarantineTest(String table, HBaseFsck hbck, int check, int corrupt, int fail, + int quar, int missing) throws Exception { + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + TEST_UTIL.getHBaseAdmin().flush(table); // flush is async. + + // Mess it up by leaving a hole in the assignment, meta, and hdfs data + TEST_UTIL.getHBaseAdmin().disableTable(table); + + String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission", table}; + ExecutorService exec = new ScheduledThreadPoolExecutor(10); + HBaseFsck res = hbck.exec(exec, args); + + HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker(); + assertEquals(hfcc.getHFilesChecked(), check); + assertEquals(hfcc.getCorrupted().size(), corrupt); + assertEquals(hfcc.getFailures().size(), fail); + assertEquals(hfcc.getQuarantined().size(), quar); + assertEquals(hfcc.getMissing().size(), missing); + + // its been fixed, verify that we can enable + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + admin.enableTableAsync(table); + while (!admin.isTableEnabled(table)) { + try { + Thread.sleep(250); + } catch (InterruptedException e) { + e.printStackTrace(); + fail("Interrupted when trying to enable table " + table); + } + } + } finally { + deleteTable(table); + } + } + + /** + * This creates a table and simulates the race situation where a concurrent compaction or split + * has removed an hfile after the corruption checker learned about it. + */ + @Test(timeout=120000) + public void testQuarantineMissingHFile() throws Exception { + String table = name.getMethodName(); + ExecutorService exec = new ScheduledThreadPoolExecutor(10); + // inject a fault in the hfcc created. + final FileSystem fs = FileSystem.get(conf); + HBaseFsck hbck = new HBaseFsck(conf, exec) { + public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException { + return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) { + boolean attemptedFirstHFile = false; + protected void checkHFile(Path p) throws IOException { + if (!attemptedFirstHFile) { + attemptedFirstHFile = true; + assertTrue(fs.delete(p, true)); // make sure delete happened. + } + super.checkHFile(p); + } + }; + } + }; + doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing. + } + + /** + * This creates a table and simulates the race situation where a concurrent compaction or split + * has removed an colfam dir before the corruption checker got to it. + */ + @Test(timeout=120000) + public void testQuarantineMissingFamdir() throws Exception { + String table = name.getMethodName(); + ExecutorService exec = new ScheduledThreadPoolExecutor(10); + // inject a fault in the hfcc created. + final FileSystem fs = FileSystem.get(conf); + HBaseFsck hbck = new HBaseFsck(conf, exec) { + public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException { + return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) { + boolean attemptedFirstFamDir = false; + protected void checkColFamDir(Path p) throws IOException { + if (!attemptedFirstFamDir) { + attemptedFirstFamDir = true; + assertTrue(fs.delete(p, true)); // make sure delete happened. + } + super.checkColFamDir(p); + } + }; + } + }; + doQuarantineTest(table, hbck, 3, 0, 0, 0, 1); + } + + /** + * This creates a table and simulates the race situation where a concurrent compaction or split + * has removed a region dir before the corruption checker got to it. + */ + @Test(timeout=120000) + public void testQuarantineMissingRegionDir() throws Exception { + String table = name.getMethodName(); + ExecutorService exec = new ScheduledThreadPoolExecutor(10); + // inject a fault in the hfcc created. + final FileSystem fs = FileSystem.get(conf); + HBaseFsck hbck = new HBaseFsck(conf, exec) { + public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException { + return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) { + boolean attemptedFirstRegionDir = false; + protected void checkRegionDir(Path p) throws IOException { + if (!attemptedFirstRegionDir) { + attemptedFirstRegionDir = true; + assertTrue(fs.delete(p, true)); // make sure delete happened. + } + super.checkRegionDir(p); + } + }; + } + }; + doQuarantineTest(table, hbck, 3, 0, 0, 0, 1); + } + + /** + * Test fixing lingering reference file. + */ + @Test + public void testLingeringReferenceFile() throws Exception { + String table = "testLingeringReferenceFile"; + try { + setupTable(table); + assertEquals(ROWKEYS.length, countRows()); + + // Mess it up by creating a fake reference file + FileSystem fs = FileSystem.get(conf); + Path tableDir= FSUtils.getTablePath(FSUtils.getRootDir(conf), table); + Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0); + Path famDir = new Path(regionDir, FAM_STR); + Path fakeReferenceFile = new Path(famDir, "fbce357483ceea.12144538"); + fs.create(fakeReferenceFile); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LINGERING_REFERENCE_HFILE }); + // fix reference file + doFsck(conf, true); + // check that reference file fixed + assertNoErrors(doFsck(conf, false)); + } finally { + deleteTable(table); + } + } + + /** + * Test pluggable error reporter. It can be plugged in + * from system property or configuration. + */ + @Test + public void testErrorReporter() throws Exception { + try { + MockErrorReporter.calledCount = 0; + doFsck(conf, false); + assertEquals(MockErrorReporter.calledCount, 0); + + conf.set("hbasefsck.errorreporter", MockErrorReporter.class.getName()); + doFsck(conf, false); + assertTrue(MockErrorReporter.calledCount > 20); + } finally { + conf.set("hbasefsck.errorreporter", + PrintingErrorReporter.class.getName()); + MockErrorReporter.calledCount = 0; + } + } + + static class MockErrorReporter implements ErrorReporter { + static int calledCount = 0; + + public void clear() { + calledCount++; + } + + public void report(String message) { + calledCount++; + } + + public void reportError(String message) { + calledCount++; + } + + public void reportError(ERROR_CODE errorCode, String message) { + calledCount++; + } + + public void reportError(ERROR_CODE errorCode, String message, TableInfo table) { + calledCount++; + } + + public void reportError(ERROR_CODE errorCode, + String message, TableInfo table, HbckInfo info) { + calledCount++; + } + + public void reportError(ERROR_CODE errorCode, String message, + TableInfo table, HbckInfo info1, HbckInfo info2) { + calledCount++; + } + + public int summarize() { + return ++calledCount; + } + + public void detail(String details) { + calledCount++; + } + + public ArrayList getErrorList() { + calledCount++; + return new ArrayList(); + } + + public void progress() { + calledCount++; + } + + public void print(String message) { + calledCount++; + } + + public void resetErrors() { + calledCount++; + } + + public boolean tableHasErrors(TableInfo table) { + calledCount++; + return false; + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); + + @org.junit.Rule + public TestName name = new TestName(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java new file mode 100644 index 0000000..00d2440 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java @@ -0,0 +1,103 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo; +import org.apache.hadoop.hbase.util.HBaseFsck.MetaEntry; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test the comparator used by Hbck. + */ +@Category(SmallTests.class) +public class TestHBaseFsckComparator { + + byte[] table = Bytes.toBytes("table1"); + byte[] table2 = Bytes.toBytes("table2"); + byte[] keyStart = Bytes.toBytes(""); + byte[] keyA = Bytes.toBytes("A"); + byte[] keyB = Bytes.toBytes("B"); + byte[] keyC = Bytes.toBytes("C"); + byte[] keyEnd = Bytes.toBytes(""); + + static HbckInfo genHbckInfo(byte[] table, byte[] start, byte[] end, int time) { + return new HbckInfo(new MetaEntry(new HRegionInfo(table, start, end), null, + time)); + } + + @Test + public void testEquals() { + HbckInfo hi1 = genHbckInfo(table, keyA, keyB, 0); + HbckInfo hi2 = genHbckInfo(table, keyA, keyB, 0); + assertEquals(0, HBaseFsck.cmp.compare(hi1, hi2)); + assertEquals(0, HBaseFsck.cmp.compare(hi2, hi1)); + } + + @Test + public void testEqualsInstance() { + HbckInfo hi1 = genHbckInfo(table, keyA, keyB, 0); + HbckInfo hi2 = hi1; + assertEquals(0, HBaseFsck.cmp.compare(hi1, hi2)); + assertEquals(0, HBaseFsck.cmp.compare(hi2, hi1)); + } + + @Test + public void testDiffTable() { + HbckInfo hi1 = genHbckInfo(table, keyA, keyC, 0); + HbckInfo hi2 = genHbckInfo(table2, keyA, keyC, 0); + assertTrue(HBaseFsck.cmp.compare(hi1, hi2) < 0); + assertTrue(HBaseFsck.cmp.compare(hi2, hi1) > 0); + } + + @Test + public void testDiffStartKey() { + HbckInfo hi1 = genHbckInfo(table, keyStart, keyC, 0); + HbckInfo hi2 = genHbckInfo(table, keyA, keyC, 0); + assertTrue(HBaseFsck.cmp.compare(hi1, hi2) < 0); + assertTrue(HBaseFsck.cmp.compare(hi2, hi1) > 0); + } + + @Test + public void testDiffEndKey() { + HbckInfo hi1 = genHbckInfo(table, keyA, keyB, 0); + HbckInfo hi2 = genHbckInfo(table, keyA, keyC, 0); + assertTrue(HBaseFsck.cmp.compare(hi1, hi2) < 0); + assertTrue(HBaseFsck.cmp.compare(hi2, hi1) > 0); + } + + @Test + public void testAbsEndKey() { + HbckInfo hi1 = genHbckInfo(table, keyA, keyC, 0); + HbckInfo hi2 = genHbckInfo(table, keyA, keyEnd, 0); + assertTrue(HBaseFsck.cmp.compare(hi1, hi2) < 0); + assertTrue(HBaseFsck.cmp.compare(hi2, hi1) > 0); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestHFileArchiveUtil.java b/src/test/java/org/apache/hadoop/hbase/util/TestHFileArchiveUtil.java new file mode 100644 index 0000000..82f96d9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestHFileArchiveUtil.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.*; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +/** + * Test that the utility works as expected + */ +@Category(SmallTests.class) +public class TestHFileArchiveUtil { + + @Test + public void testGetTableArchivePath() { + assertNotNull(HFileArchiveUtil.getTableArchivePath(new Path("table"))); + assertNotNull(HFileArchiveUtil.getTableArchivePath(new Path("root", new Path("table")))); + } + + @Test + public void testGetArchivePath() throws Exception { + Configuration conf = new Configuration(); + FSUtils.setRootDir(conf, new Path("root")); + assertNotNull(HFileArchiveUtil.getArchivePath(conf)); + } + + @Test + public void testRegionArchiveDir() { + Configuration conf = null; + Path tableDir = new Path("table"); + Path regionDir = new Path("region"); + assertNotNull(HFileArchiveUtil.getRegionArchiveDir(conf, tableDir, regionDir)); + } + + @Test + public void testGetStoreArchivePath(){ + byte[] family = Bytes.toBytes("Family"); + Path tabledir = new Path("table"); + HRegionInfo region = new HRegionInfo(Bytes.toBytes("table")); + Configuration conf = null; + assertNotNull(HFileArchiveUtil.getStoreArchivePath(conf, region, tabledir, family)); + conf = new Configuration(); + assertNotNull(HFileArchiveUtil.getStoreArchivePath(conf, region, tabledir, family)); + + // do a little mocking of a region to get the same results + HRegion mockRegion = Mockito.mock(HRegion.class); + Mockito.when(mockRegion.getRegionInfo()).thenReturn(region); + Mockito.when(mockRegion.getTableDir()).thenReturn(tabledir); + + assertNotNull(HFileArchiveUtil.getStoreArchivePath(null, mockRegion, family)); + conf = new Configuration(); + assertNotNull(HFileArchiveUtil.getStoreArchivePath(conf, mockRegion, family)); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestIdLock.java b/src/test/java/org/apache/hadoop/hbase/util/TestIdLock.java new file mode 100644 index 0000000..4038974 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestIdLock.java @@ -0,0 +1,118 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertTrue; + +import java.util.Map; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.MediumTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +// Medium as it creates 100 threads; seems better to run it isolated +public class TestIdLock { + + private static final Log LOG = LogFactory.getLog(TestIdLock.class); + + private static final int NUM_IDS = 16; + private static final int NUM_THREADS = 128; + private static final int NUM_SECONDS = 15; + + private IdLock idLock = new IdLock(); + + private Map idOwner = new ConcurrentHashMap(); + + private class IdLockTestThread implements Callable { + + private String clientId; + + public IdLockTestThread(String clientId) { + this.clientId = clientId; + } + + @Override + public Boolean call() throws Exception { + Thread.currentThread().setName(clientId); + Random rand = new Random(); + long endTime = System.currentTimeMillis() + NUM_SECONDS * 1000; + while (System.currentTimeMillis() < endTime) { + long id = rand.nextInt(NUM_IDS); + + IdLock.Entry lockEntry = idLock.getLockEntry(id); + try { + int sleepMs = 1 + rand.nextInt(4); + String owner = idOwner.get(id); + if (owner != null) { + LOG.error("Id " + id + " already taken by " + owner + ", " + + clientId + " failed"); + return false; + } + + idOwner.put(id, clientId); + Thread.sleep(sleepMs); + idOwner.remove(id); + + } finally { + idLock.releaseLockEntry(lockEntry); + } + } + return true; + } + + } + + @Test + public void testMultipleClients() throws Exception { + ExecutorService exec = Executors.newFixedThreadPool(NUM_THREADS); + try { + ExecutorCompletionService ecs = + new ExecutorCompletionService(exec); + for (int i = 0; i < NUM_THREADS; ++i) + ecs.submit(new IdLockTestThread("client_" + i)); + for (int i = 0; i < NUM_THREADS; ++i) { + Future result = ecs.take(); + assertTrue(result.get()); + } + idLock.assertMapEmpty(); + } finally { + exec.shutdown(); + exec.awaitTermination(5000, TimeUnit.MILLISECONDS); + } + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java b/src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java new file mode 100644 index 0000000..f4fbce9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java @@ -0,0 +1,49 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static junit.framework.Assert.assertEquals; + +/** + * Tests that the incrementing environment edge increments time instead of using + * the default. + */ +@Category(SmallTests.class) +public class TestIncrementingEnvironmentEdge { + + @Test + public void testGetCurrentTimeUsesSystemClock() { + IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(); + assertEquals(1, edge.currentTimeMillis()); + assertEquals(2, edge.currentTimeMillis()); + assertEquals(3, edge.currentTimeMillis()); + assertEquals(4, edge.currentTimeMillis()); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestKeying.java b/src/test/java/org/apache/hadoop/hbase/util/TestKeying.java new file mode 100644 index 0000000..88f5821 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestKeying.java @@ -0,0 +1,70 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import junit.framework.TestCase; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.experimental.categories.Category; + +/** + * Tests url transformations + */ +@Category(SmallTests.class) +public class TestKeying extends TestCase { + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + } + + /** + * Test url transformations + * @throws Exception + */ + public void testURI() throws Exception { + checkTransform("http://abc:bcd@www.example.com/index.html" + + "?query=something#middle"); + checkTransform("file:///usr/bin/java"); + checkTransform("dns:www.powerset.com"); + checkTransform("dns://dns.powerset.com/www.powerset.com"); + checkTransform("http://one.two.three/index.html"); + checkTransform("https://one.two.three:9443/index.html"); + checkTransform("ftp://one.two.three/index.html"); + + checkTransform("filename"); + } + + private void checkTransform(final String u) { + String k = Keying.createKey(u); + String uri = Keying.keyToUri(k); + System.out.println("Original url " + u + ", Transformed url " + k); + assertEquals(u, uri); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java b/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java new file mode 100644 index 0000000..f6408b8 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.HashSet; +import java.util.Random; +import java.util.Set; + +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestLoadTestKVGenerator { + + private static final int MIN_LEN = 10; + private static final int MAX_LEN = 20; + + private Random rand = new Random(28937293L); + private LoadTestKVGenerator gen = new LoadTestKVGenerator(MIN_LEN, MAX_LEN); + + @Test + public void testValueLength() { + for (int i = 0; i < 1000; ++i) { + byte[] v = gen.generateRandomSizeValue(Integer.toString(i).getBytes(), + String.valueOf(rand.nextInt()).getBytes()); + assertTrue(MIN_LEN <= v.length); + assertTrue(v.length <= MAX_LEN); + } + } + + @Test + public void testVerification() { + for (int i = 0; i < 1000; ++i) { + for (int qualIndex = 0; qualIndex < 20; ++qualIndex) { + byte[] qual = String.valueOf(qualIndex).getBytes(); + byte[] rowKey = LoadTestKVGenerator.md5PrefixedKey(i).getBytes(); + byte[] v = gen.generateRandomSizeValue(rowKey, qual); + assertTrue(LoadTestKVGenerator.verify(v, rowKey, qual)); + v[0]++; + assertFalse(LoadTestKVGenerator.verify(v, rowKey, qual)); + } + } + } + + @Test + public void testCorrectAndUniqueKeys() { + Set keys = new HashSet(); + for (int i = 0; i < 1000; ++i) { + String k = gen.md5PrefixedKey(i); + assertFalse(keys.contains(k)); + assertTrue(k.endsWith("-" + i)); + keys.add(k); + } + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java new file mode 100644 index 0000000..a9948c7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java @@ -0,0 +1,180 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests merging a normal table's regions + */ +@Category(MediumTests.class) +public class TestMergeTable { + private static final Log LOG = LogFactory.getLog(TestMergeTable.class); + private final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final byte [] COLUMN_NAME = Bytes.toBytes("contents"); + private static final byte [] VALUE; + static { + // We will use the same value for the rows as that is not really important here + String partialValue = String.valueOf(System.currentTimeMillis()); + StringBuilder val = new StringBuilder(); + while (val.length() < 1024) { + val.append(partialValue); + } + VALUE = Bytes.toBytes(val.toString()); + } + + /** + * Test merge. + * Hand-makes regions of a mergeable size and adds the hand-made regions to + * hand-made meta. The hand-made regions are created offline. We then start + * up mini cluster, disables the hand-made table and starts in on merging. + * @throws Exception + */ + @Test (timeout=300000) public void testMergeTable() throws Exception { + // Table we are manually creating offline. + HTableDescriptor desc = new HTableDescriptor(Bytes.toBytes("test")); + desc.addFamily(new HColumnDescriptor(COLUMN_NAME)); + + // Set maximum regionsize down. + UTIL.getConfiguration().setLong(HConstants.HREGION_MAX_FILESIZE, 64L * 1024L * 1024L); + // Make it so we don't split. + UTIL.getConfiguration().setInt("hbase.regionserver.regionSplitLimit", 0); + // Startup hdfs. Its in here we'll be putting our manually made regions. + UTIL.startMiniDFSCluster(1); + // Create hdfs hbase rootdir. + Path rootdir = UTIL.createRootDir(); + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + if (fs.exists(rootdir)) { + if (fs.delete(rootdir, true)) { + LOG.info("Cleaned up existing " + rootdir); + } + } + + // Now create three data regions: The first is too large to merge since it + // will be > 64 MB in size. The second two will be smaller and will be + // selected for merging. + + // To ensure that the first region is larger than 64MB we need to write at + // least 65536 rows. We will make certain by writing 70000 + byte [] row_70001 = Bytes.toBytes("row_70001"); + byte [] row_80001 = Bytes.toBytes("row_80001"); + + // Create regions and populate them at same time. Create the tabledir + // for them first. + FSTableDescriptors.createTableDescriptor(fs, rootdir, desc); + HRegion [] regions = { + createRegion(desc, null, row_70001, 1, 70000, rootdir), + createRegion(desc, row_70001, row_80001, 70001, 10000, rootdir), + createRegion(desc, row_80001, null, 80001, 11000, rootdir) + }; + + // Now create the root and meta regions and insert the data regions + // created above into .META. + setupROOTAndMeta(rootdir, regions); + try { + LOG.info("Starting mini zk cluster"); + UTIL.startMiniZKCluster(); + LOG.info("Starting mini hbase cluster"); + UTIL.startMiniHBaseCluster(1, 1); + Configuration c = new Configuration(UTIL.getConfiguration()); + CatalogTracker ct = new CatalogTracker(c); + ct.start(); + List originalTableRegions = + MetaReader.getTableRegions(ct, desc.getName()); + LOG.info("originalTableRegions size=" + originalTableRegions.size() + + "; " + originalTableRegions); + HBaseAdmin admin = new HBaseAdmin(new Configuration(c)); + admin.disableTable(desc.getName()); + HMerge.merge(c, FileSystem.get(c), desc.getName()); + List postMergeTableRegions = + MetaReader.getTableRegions(ct, desc.getName()); + LOG.info("postMergeTableRegions size=" + postMergeTableRegions.size() + + "; " + postMergeTableRegions); + assertTrue("originalTableRegions=" + originalTableRegions.size() + + ", postMergeTableRegions=" + postMergeTableRegions.size(), + postMergeTableRegions.size() < originalTableRegions.size()); + LOG.info("Done with merge"); + } finally { + UTIL.shutdownMiniCluster(); + LOG.info("After cluster shutdown"); + } + } + + private HRegion createRegion(final HTableDescriptor desc, + byte [] startKey, byte [] endKey, int firstRow, int nrows, Path rootdir) + throws IOException { + HRegionInfo hri = new HRegionInfo(desc.getName(), startKey, endKey); + HRegion region = HRegion.createHRegion(hri, rootdir, UTIL.getConfiguration(), desc); + LOG.info("Created region " + region.getRegionNameAsString()); + for(int i = firstRow; i < firstRow + nrows; i++) { + Put put = new Put(Bytes.toBytes("row_" + String.format("%1$05d", i))); + put.setWriteToWAL(false); + put.add(COLUMN_NAME, null, VALUE); + region.put(put); + if (i % 10000 == 0) { + LOG.info("Flushing write #" + i); + region.flushcache(); + } + } + region.close(); + region.getLog().closeAndDelete(); + return region; + } + + protected void setupROOTAndMeta(Path rootdir, final HRegion [] regions) + throws IOException { + HRegion root = + HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, rootdir, + UTIL.getConfiguration(), HTableDescriptor.ROOT_TABLEDESC); + HRegion meta = + HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, rootdir, + UTIL.getConfiguration(), HTableDescriptor.META_TABLEDESC); + HRegion.addRegionToMETA(root, meta); + for (HRegion r: regions) { + HRegion.addRegionToMETA(meta, r); + } + meta.close(); + meta.getLog().closeAndDelete(); + root.close(); + root.getLog().closeAndDelete(); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java new file mode 100644 index 0000000..f5f0ac7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java @@ -0,0 +1,302 @@ +/** + * Copyright 2008 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.wal.HLog; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.util.ToolRunner; +import org.junit.experimental.categories.Category; + +/** Test stand alone merge tool that can merge arbitrary regions */ +@Category(LargeTests.class) +public class TestMergeTool extends HBaseTestCase { + static final Log LOG = LogFactory.getLog(TestMergeTool.class); + HBaseTestingUtility TEST_UTIL; +// static final byte [] COLUMN_NAME = Bytes.toBytes("contents:"); + static final byte [] FAMILY = Bytes.toBytes("contents"); + static final byte [] QUALIFIER = Bytes.toBytes("dc"); + + private final HRegionInfo[] sourceRegions = new HRegionInfo[5]; + private final HRegion[] regions = new HRegion[5]; + private HTableDescriptor desc; + private byte [][][] rows; + private MiniDFSCluster dfsCluster = null; + + @Override + public void setUp() throws Exception { + // Set the timeout down else this test will take a while to complete. + this.conf.setLong("hbase.zookeeper.recoverable.waittime", 1000); + // Make it so we try and connect to a zk that is not there (else we might + // find a zk ensemble put up by another concurrent test and this will + // mess up this test. Choose unlikely port. Default test port is 21818. + // Default zk port is 2181. + this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, 10001); + + this.conf.set("hbase.hstore.compactionThreshold", "2"); + + // Create table description + this.desc = new HTableDescriptor("TestMergeTool"); + this.desc.addFamily(new HColumnDescriptor(FAMILY)); + + /* + * Create the HRegionInfos for the regions. + */ + // Region 0 will contain the key range [row_0200,row_0300) + sourceRegions[0] = new HRegionInfo(this.desc.getName(), + Bytes.toBytes("row_0200"), + Bytes.toBytes("row_0300")); + + // Region 1 will contain the key range [row_0250,row_0400) and overlaps + // with Region 0 + sourceRegions[1] = + new HRegionInfo(this.desc.getName(), + Bytes.toBytes("row_0250"), + Bytes.toBytes("row_0400")); + + // Region 2 will contain the key range [row_0100,row_0200) and is adjacent + // to Region 0 or the region resulting from the merge of Regions 0 and 1 + sourceRegions[2] = + new HRegionInfo(this.desc.getName(), + Bytes.toBytes("row_0100"), + Bytes.toBytes("row_0200")); + + // Region 3 will contain the key range [row_0500,row_0600) and is not + // adjacent to any of Regions 0, 1, 2 or the merged result of any or all + // of those regions + sourceRegions[3] = + new HRegionInfo(this.desc.getName(), + Bytes.toBytes("row_0500"), + Bytes.toBytes("row_0600")); + + // Region 4 will have empty start and end keys and overlaps all regions. + sourceRegions[4] = + new HRegionInfo(this.desc.getName(), + HConstants.EMPTY_BYTE_ARRAY, + HConstants.EMPTY_BYTE_ARRAY); + + /* + * Now create some row keys + */ + this.rows = new byte [5][][]; + this.rows[0] = Bytes.toByteArrays(new String[] { "row_0210", "row_0280" }); + this.rows[1] = Bytes.toByteArrays(new String[] { "row_0260", "row_0350", + "row_035" }); + this.rows[2] = Bytes.toByteArrays(new String[] { "row_0110", "row_0175", + "row_0175", "row_0175"}); + this.rows[3] = Bytes.toByteArrays(new String[] { "row_0525", "row_0560", + "row_0560", "row_0560", "row_0560"}); + this.rows[4] = Bytes.toByteArrays(new String[] { "row_0050", "row_1000", + "row_1000", "row_1000", "row_1000", "row_1000" }); + + // Start up dfs + TEST_UTIL = new HBaseTestingUtility(conf); + this.dfsCluster = TEST_UTIL.startMiniDFSCluster(2); + this.fs = this.dfsCluster.getFileSystem(); + System.out.println("fs=" + this.fs); + this.conf.set("fs.defaultFS", fs.getUri().toString()); + Path parentdir = fs.getHomeDirectory(); + conf.set(HConstants.HBASE_DIR, parentdir.toString()); + fs.mkdirs(parentdir); + FSUtils.setVersion(fs, parentdir); + + // Note: we must call super.setUp after starting the mini cluster or + // we will end up with a local file system + + super.setUp(); + try { + // Create root and meta regions + createRootAndMetaRegions(); + FSTableDescriptors.createTableDescriptor(this.fs, this.testDir, this.desc); + /* + * Create the regions we will merge + */ + for (int i = 0; i < sourceRegions.length; i++) { + regions[i] = + HRegion.createHRegion(this.sourceRegions[i], this.testDir, this.conf, + this.desc); + /* + * Insert data + */ + for (int j = 0; j < rows[i].length; j++) { + byte [] row = rows[i][j]; + Put put = new Put(row); + put.add(FAMILY, QUALIFIER, row); + regions[i].put(put); + } + HRegion.addRegionToMETA(meta, regions[i]); + } + // Close root and meta regions + closeRootAndMeta(); + + } catch (Exception e) { + TEST_UTIL.shutdownMiniCluster(); + throw e; + } + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + for (int i = 0; i < sourceRegions.length; i++) { + HRegion r = regions[i]; + if (r != null) { + r.close(); + r.getLog().closeAndDelete(); + } + } + TEST_UTIL.shutdownMiniCluster(); + } + + /* + * @param msg Message that describes this merge + * @param regionName1 + * @param regionName2 + * @param log Log to use merging. + * @param upperbound Verifying, how high up in this.rows to go. + * @return Merged region. + * @throws Exception + */ + private HRegion mergeAndVerify(final String msg, final String regionName1, + final String regionName2, final HLog log, final int upperbound) + throws Exception { + Merge merger = new Merge(this.conf); + LOG.info(msg); + System.out.println("fs2=" + this.conf.get("fs.defaultFS")); + int errCode = ToolRunner.run(this.conf, merger, + new String[] {this.desc.getNameAsString(), regionName1, regionName2} + ); + assertTrue("'" + msg + "' failed with errCode " + errCode, errCode == 0); + HRegionInfo mergedInfo = merger.getMergedHRegionInfo(); + + // Now verify that we can read all the rows from regions 0, 1 + // in the new merged region. + HRegion merged = HRegion.openHRegion(mergedInfo, this.desc, log, this.conf); + verifyMerge(merged, upperbound); + merged.close(); + LOG.info("Verified " + msg); + return merged; + } + + private void verifyMerge(final HRegion merged, final int upperbound) + throws IOException { + //Test + Scan scan = new Scan(); + scan.addFamily(FAMILY); + InternalScanner scanner = merged.getScanner(scan); + try { + List testRes = null; + while (true) { + testRes = new ArrayList(); + boolean hasNext = scanner.next(testRes); + if (!hasNext) { + break; + } + } + } finally { + scanner.close(); + } + + //!Test + + for (int i = 0; i < upperbound; i++) { + for (int j = 0; j < rows[i].length; j++) { + Get get = new Get(rows[i][j]); + get.addFamily(FAMILY); + Result result = merged.get(get, null); + assertEquals(1, result.size()); + byte [] bytes = result.raw()[0].getValue(); + assertNotNull(Bytes.toStringBinary(rows[i][j]), bytes); + assertTrue(Bytes.equals(bytes, rows[i][j])); + } + } + } + + /** + * Test merge tool. + * @throws Exception + */ + public void testMergeTool() throws Exception { + // First verify we can read the rows from the source regions and that they + // contain the right data. + for (int i = 0; i < regions.length; i++) { + for (int j = 0; j < rows[i].length; j++) { + Get get = new Get(rows[i][j]); + get.addFamily(FAMILY); + Result result = regions[i].get(get, null); + byte [] bytes = result.raw()[0].getValue(); + assertNotNull(bytes); + assertTrue(Bytes.equals(bytes, rows[i][j])); + } + // Close the region and delete the log + regions[i].close(); + regions[i].getLog().closeAndDelete(); + } + + // Create a log that we can reuse when we need to open regions + Path logPath = new Path("/tmp", HConstants.HREGION_LOGDIR_NAME + "_" + + System.currentTimeMillis()); + LOG.info("Creating log " + logPath.toString()); + Path oldLogDir = new Path("/tmp", HConstants.HREGION_OLDLOGDIR_NAME); + HLog log = new HLog(this.fs, logPath, oldLogDir, this.conf); + try { + // Merge Region 0 and Region 1 + HRegion merged = mergeAndVerify("merging regions 0 and 1", + this.sourceRegions[0].getRegionNameAsString(), + this.sourceRegions[1].getRegionNameAsString(), log, 2); + + // Merge the result of merging regions 0 and 1 with region 2 + merged = mergeAndVerify("merging regions 0+1 and 2", + merged.getRegionInfo().getRegionNameAsString(), + this.sourceRegions[2].getRegionNameAsString(), log, 3); + + // Merge the result of merging regions 0, 1 and 2 with region 3 + merged = mergeAndVerify("merging regions 0+1+2 and 3", + merged.getRegionInfo().getRegionNameAsString(), + this.sourceRegions[3].getRegionNameAsString(), log, 4); + + // Merge the result of merging regions 0, 1, 2 and 3 with region 4 + merged = mergeAndVerify("merging regions 0+1+2+3 and 4", + merged.getRegionInfo().getRegionNameAsString(), + this.sourceRegions[4].getRegionNameAsString(), log, rows.length); + } finally { + log.closeAndDelete(); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadEncoded.java b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadEncoded.java new file mode 100644 index 0000000..c0326aa --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadEncoded.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized.Parameters; + +/** + * Runs a load test on a mini HBase cluster with data block encoding turned on. + * Compared to other load-test-style unit tests, this one writes a smaller + * amount of data, but goes through all available data block encoding + * algorithms. + */ +@Category(LargeTests.class) +public class TestMiniClusterLoadEncoded extends TestMiniClusterLoadParallel { + + /** We do not alternate the multi-put flag in this test. */ + private static final boolean USE_MULTI_PUT = true; + + @Parameters + public static Collection parameters() { + List parameters = new ArrayList(); + for (DataBlockEncoding dataBlockEncoding : DataBlockEncoding.values() ) { + parameters.add(new Object[]{dataBlockEncoding}); + } + return parameters; + } + + public TestMiniClusterLoadEncoded(DataBlockEncoding encoding) { + super(USE_MULTI_PUT, encoding); + } + + /** + * Use a smaller number of keys in in this test. + */ + @Override + protected int numKeys() { + return 3000; + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadParallel.java b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadParallel.java new file mode 100644 index 0000000..eab23d5 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadParallel.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * A write/read/verify load test on a mini HBase cluster. Tests reading + * and writing at the same time. + */ +@Category(LargeTests.class) +@RunWith(Parameterized.class) +public class TestMiniClusterLoadParallel + extends TestMiniClusterLoadSequential { + + public TestMiniClusterLoadParallel(boolean isMultiPut, + DataBlockEncoding encoding) { + super(isMultiPut, encoding); + } + + @Test(timeout=TIMEOUT_MS) + public void loadTest() throws Exception { + prepareForLoadTest(); + + readerThreads.linkToWriter(writerThreads); + + writerThreads.start(0, numKeys, NUM_THREADS); + readerThreads.start(0, numKeys, NUM_THREADS); + + writerThreads.waitForFinish(); + readerThreads.waitForFinish(); + + assertEquals(0, writerThreads.getNumWriteFailures()); + assertEquals(0, readerThreads.getNumReadFailures()); + assertEquals(0, readerThreads.getNumReadErrors()); + assertEquals(numKeys, readerThreads.getNumUniqueKeysVerified()); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java new file mode 100644 index 0000000..64c67d3 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * A write/read/verify load test on a mini HBase cluster. Tests reading + * and then writing. + */ +@Category(LargeTests.class) +@RunWith(Parameterized.class) +public class TestMiniClusterLoadSequential { + + private static final Log LOG = LogFactory.getLog( + TestMiniClusterLoadSequential.class); + + protected static final byte[] TABLE = Bytes.toBytes("load_test_tbl"); + protected static final byte[] CF = Bytes.toBytes("load_test_cf"); + protected static final int NUM_THREADS = 8; + protected static final int NUM_RS = 2; + protected static final int TIMEOUT_MS = 180000; + protected static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + protected final Configuration conf = TEST_UTIL.getConfiguration(); + protected final boolean isMultiPut; + protected final DataBlockEncoding dataBlockEncoding; + + protected MultiThreadedWriter writerThreads; + protected MultiThreadedReader readerThreads; + protected int numKeys; + + protected Compression.Algorithm compression = Compression.Algorithm.NONE; + + public TestMiniClusterLoadSequential(boolean isMultiPut, + DataBlockEncoding dataBlockEncoding) { + this.isMultiPut = isMultiPut; + this.dataBlockEncoding = dataBlockEncoding; + conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024); + } + + @Parameters + public static Collection parameters() { + List parameters = new ArrayList(); + for (boolean multiPut : new boolean[]{false, true}) { + for (DataBlockEncoding dataBlockEncoding : new DataBlockEncoding[] { + DataBlockEncoding.NONE, DataBlockEncoding.PREFIX }) { + parameters.add(new Object[]{multiPut, dataBlockEncoding}); + } + } + return parameters; + } + + @Before + public void setUp() throws Exception { + LOG.debug("Test setup: isMultiPut=" + isMultiPut); + TEST_UTIL.startMiniCluster(1, NUM_RS); + } + + @After + public void tearDown() throws Exception { + LOG.debug("Test teardown: isMultiPut=" + isMultiPut); + TEST_UTIL.shutdownMiniCluster(); + } + + protected MultiThreadedReader prepareReaderThreads(LoadTestDataGenerator dataGen, + Configuration conf, byte[] tableName, double verifyPercent) { + MultiThreadedReader reader = new MultiThreadedReader(dataGen, conf, tableName, verifyPercent); + return reader; + } + + protected MultiThreadedWriter prepareWriterThreads(LoadTestDataGenerator dataGen, + Configuration conf, byte[] tableName) { + MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, tableName); + writer.setMultiPut(isMultiPut); + return writer; + } + + @Test(timeout=TIMEOUT_MS) + public void loadTest() throws Exception { + prepareForLoadTest(); + runLoadTestOnExistingTable(); + } + + protected void runLoadTestOnExistingTable() throws IOException { + writerThreads.start(0, numKeys, NUM_THREADS); + writerThreads.waitForFinish(); + assertEquals(0, writerThreads.getNumWriteFailures()); + + readerThreads.start(0, numKeys, NUM_THREADS); + readerThreads.waitForFinish(); + assertEquals(0, readerThreads.getNumReadFailures()); + assertEquals(0, readerThreads.getNumReadErrors()); + assertEquals(numKeys, readerThreads.getNumKeysVerified()); + } + + protected void createPreSplitLoadTestTable(HTableDescriptor htd, HColumnDescriptor hcd) + throws IOException { + HBaseTestingUtility.createPreSplitLoadTestTable(conf, htd, hcd); + TEST_UTIL.waitUntilAllRegionsAssigned(htd.getName()); + } + + protected void prepareForLoadTest() throws IOException { + LOG.info("Starting load test: dataBlockEncoding=" + dataBlockEncoding + + ", isMultiPut=" + isMultiPut); + numKeys = numKeys(); + HBaseAdmin admin = new HBaseAdmin(conf); + while (admin.getClusterStatus().getServers().size() < NUM_RS) { + LOG.info("Sleeping until " + NUM_RS + " RSs are online"); + Threads.sleepWithoutInterrupt(1000); + } + admin.close(); + + HTableDescriptor htd = new HTableDescriptor(TABLE); + HColumnDescriptor hcd = new HColumnDescriptor(CF) + .setCompressionType(compression) + .setDataBlockEncoding(dataBlockEncoding); + createPreSplitLoadTestTable(htd, hcd); + + LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator(CF); + writerThreads = prepareWriterThreads(dataGen, conf, TABLE); + readerThreads = prepareReaderThreads(dataGen, conf, TABLE, 100); + } + + protected int numKeys() { + return 10000; + } + + protected HColumnDescriptor getColumnDesc(HBaseAdmin admin) + throws TableNotFoundException, IOException { + return admin.getTableDescriptor(TABLE).getFamily(CF); + } + +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestPoolMap.java b/src/test/java/org/apache/hadoop/hbase/util/TestPoolMap.java new file mode 100644 index 0000000..bb4304c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestPoolMap.java @@ -0,0 +1,238 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +import junit.framework.Test; +import junit.framework.TestCase; + +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.PoolMap.PoolType; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({TestPoolMap.TestRoundRobinPoolType.class, TestPoolMap.TestThreadLocalPoolType.class, TestPoolMap.TestReusablePoolType.class}) +@Category(MediumTests.class) +public class TestPoolMap { + public abstract static class TestPoolType extends TestCase { + protected PoolMap poolMap; + protected Random random = new Random(); + + protected static final int POOL_SIZE = 3; + + @Override + protected void setUp() throws Exception { + this.poolMap = new PoolMap(getPoolType(), POOL_SIZE); + } + + protected abstract PoolType getPoolType(); + + @Override + protected void tearDown() throws Exception { + this.poolMap.clear(); + } + + protected void runThread(final String randomKey, final String randomValue, + final String expectedValue) throws InterruptedException { + final AtomicBoolean matchFound = new AtomicBoolean(false); + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + poolMap.put(randomKey, randomValue); + String actualValue = poolMap.get(randomKey); + matchFound.set(expectedValue == null ? actualValue == null + : expectedValue.equals(actualValue)); + } + }); + thread.start(); + thread.join(); + assertTrue(matchFound.get()); + } + } + + @Category(MediumTests.class) + public static class TestRoundRobinPoolType extends TestPoolType { + @Override + protected PoolType getPoolType() { + return PoolType.RoundRobin; + } + + public void testSingleThreadedClient() throws InterruptedException, + ExecutionException { + String randomKey = String.valueOf(random.nextInt()); + String randomValue = String.valueOf(random.nextInt()); + // As long as the pool is not full, we'll get null back. + // This forces the user to create new values that can be used to populate + // the pool. + runThread(randomKey, randomValue, null); + assertEquals(1, poolMap.size(randomKey)); + } + + public void testMultiThreadedClients() throws InterruptedException, + ExecutionException { + for (int i = 0; i < POOL_SIZE; i++) { + String randomKey = String.valueOf(random.nextInt()); + String randomValue = String.valueOf(random.nextInt()); + // As long as the pool is not full, we'll get null back + runThread(randomKey, randomValue, null); + // As long as we use distinct keys, each pool will have one value + assertEquals(1, poolMap.size(randomKey)); + } + poolMap.clear(); + String randomKey = String.valueOf(random.nextInt()); + for (int i = 0; i < POOL_SIZE - 1; i++) { + String randomValue = String.valueOf(random.nextInt()); + // As long as the pool is not full, we'll get null back + runThread(randomKey, randomValue, null); + // since we use the same key, the pool size should grow + assertEquals(i + 1, poolMap.size(randomKey)); + } + // at the end of the day, there should be as many values as we put + assertEquals(POOL_SIZE - 1, poolMap.size(randomKey)); + } + + public void testPoolCap() throws InterruptedException, ExecutionException { + String randomKey = String.valueOf(random.nextInt()); + List randomValues = new ArrayList(); + for (int i = 0; i < POOL_SIZE * 2; i++) { + String randomValue = String.valueOf(random.nextInt()); + randomValues.add(randomValue); + if (i < POOL_SIZE - 1) { + // As long as the pool is not full, we'll get null back + runThread(randomKey, randomValue, null); + } else { + // when the pool becomes full, we expect the value we get back to be + // what we put earlier, in round-robin order + runThread(randomKey, randomValue, + randomValues.get((i - POOL_SIZE + 1) % POOL_SIZE)); + } + } + assertEquals(POOL_SIZE, poolMap.size(randomKey)); + } + + } + + @Category(MediumTests.class) + public static class TestThreadLocalPoolType extends TestPoolType { + @Override + protected PoolType getPoolType() { + return PoolType.ThreadLocal; + } + + public void testSingleThreadedClient() throws InterruptedException, + ExecutionException { + String randomKey = String.valueOf(random.nextInt()); + String randomValue = String.valueOf(random.nextInt()); + // As long as the pool is not full, we should get back what we put + runThread(randomKey, randomValue, randomValue); + assertEquals(1, poolMap.size(randomKey)); + } + + public void testMultiThreadedClients() throws InterruptedException, + ExecutionException { + // As long as the pool is not full, we should get back what we put + for (int i = 0; i < POOL_SIZE; i++) { + String randomKey = String.valueOf(random.nextInt()); + String randomValue = String.valueOf(random.nextInt()); + runThread(randomKey, randomValue, randomValue); + assertEquals(1, poolMap.size(randomKey)); + } + String randomKey = String.valueOf(random.nextInt()); + for (int i = 0; i < POOL_SIZE; i++) { + String randomValue = String.valueOf(random.nextInt()); + runThread(randomKey, randomValue, randomValue); + assertEquals(i + 1, poolMap.size(randomKey)); + } + } + + public void testPoolCap() throws InterruptedException, ExecutionException { + String randomKey = String.valueOf(random.nextInt()); + for (int i = 0; i < POOL_SIZE * 2; i++) { + String randomValue = String.valueOf(random.nextInt()); + // as of HBASE-4150, pool limit is no longer used with ThreadLocalPool + runThread(randomKey, randomValue, randomValue); + } + assertEquals(POOL_SIZE * 2, poolMap.size(randomKey)); + } + + } + + @Category(MediumTests.class) + public static class TestReusablePoolType extends TestPoolType { + @Override + protected PoolType getPoolType() { + return PoolType.Reusable; + } + + public void testSingleThreadedClient() throws InterruptedException, + ExecutionException { + String randomKey = String.valueOf(random.nextInt()); + String randomValue = String.valueOf(random.nextInt()); + // As long as we poll values we put, the pool size should remain zero + runThread(randomKey, randomValue, randomValue); + assertEquals(0, poolMap.size(randomKey)); + } + + public void testMultiThreadedClients() throws InterruptedException, + ExecutionException { + // As long as we poll values we put, the pool size should remain zero + for (int i = 0; i < POOL_SIZE; i++) { + String randomKey = String.valueOf(random.nextInt()); + String randomValue = String.valueOf(random.nextInt()); + runThread(randomKey, randomValue, randomValue); + assertEquals(0, poolMap.size(randomKey)); + } + poolMap.clear(); + String randomKey = String.valueOf(random.nextInt()); + for (int i = 0; i < POOL_SIZE - 1; i++) { + String randomValue = String.valueOf(random.nextInt()); + runThread(randomKey, randomValue, randomValue); + assertEquals(0, poolMap.size(randomKey)); + } + assertEquals(0, poolMap.size(randomKey)); + } + + public void testPoolCap() throws InterruptedException, ExecutionException { + // As long as we poll values we put, the pool size should remain zero + String randomKey = String.valueOf(random.nextInt()); + List randomValues = new ArrayList(); + for (int i = 0; i < POOL_SIZE * 2; i++) { + String randomValue = String.valueOf(random.nextInt()); + randomValues.add(randomValue); + runThread(randomKey, randomValue, randomValue); + } + assertEquals(0, poolMap.size(randomKey)); + } + + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java b/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java new file mode 100644 index 0000000..a2419f9 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java @@ -0,0 +1,401 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.SortedSet; +import java.util.UUID; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; + +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.Multimap; + +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestRegionSplitCalculator { + private static final Log LOG = LogFactory.getLog(TestRegionSplitCalculator.class); + + /** + * This is range uses a user specified start and end keys. It also has an + * extra tiebreaker so that different ranges with the same start/end key pair + * count as different regions. + */ + static class SimpleRange implements KeyRange { + byte[] start, end; + UUID tiebreaker; + + SimpleRange(byte[] start, byte[] end) { + this.start = start; + this.end = end; + this.tiebreaker = UUID.randomUUID(); + } + + @Override + public byte[] getStartKey() { + return start; + } + + @Override + public byte[] getEndKey() { + return end; + } + + public String toString() { + return "[" + Bytes.toString(start) + ", " + Bytes.toString(end) + "]"; + } + } + + Comparator cmp = new Comparator() { + @Override + public int compare(SimpleRange sr1, SimpleRange sr2) { + ComparisonChain cc = ComparisonChain.start(); + cc = cc.compare(sr1.getStartKey(), sr2.getStartKey(), + Bytes.BYTES_COMPARATOR); + cc = cc.compare(sr1.getEndKey(), sr2.getEndKey(), + RegionSplitCalculator.BYTES_COMPARATOR); + cc = cc.compare(sr1.tiebreaker, sr2.tiebreaker); + return cc.result(); + } + }; + + /** + * Check the "depth" (number of regions included at a split) of a generated + * split calculation + */ + void checkDepths(SortedSet splits, + Multimap regions, Integer... depths) { + assertEquals(splits.size(), depths.length); + int i = 0; + for (byte[] k : splits) { + Collection rs = regions.get(k); + int sz = rs == null ? 0 : rs.size(); + assertEquals((int) depths[i], sz); + i++; + } + } + + /** + * This dumps data in a visually reasonable way for visual debugging. It has + * the basic iteration structure. + */ + String dump(SortedSet splits, Multimap regions) { + // we display this way because the last end key should be displayed as well. + StringBuilder sb = new StringBuilder(); + for (byte[] k : splits) { + sb.append(Bytes.toString(k)).append(":\t"); + for (SimpleRange r : regions.get(k)) { + sb.append(r.toString()).append("\t"); + } + sb.append("\n"); + } + String s = sb.toString(); + LOG.info("\n" + s); + return s; + } + + @Test + public void testSplitCalculator() { + SimpleRange a = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("B")); + SimpleRange b = new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes("C")); + SimpleRange c = new SimpleRange(Bytes.toBytes("C"), Bytes.toBytes("D")); + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(a); + sc.add(b); + sc.add(c); + + Multimap regions = sc.calcCoverage(); + LOG.info("Standard"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions, 1, 1, 1, 0); + assertEquals(res, "A:\t[A, B]\t\n" + "B:\t[B, C]\t\n" + "C:\t[C, D]\t\n" + + "D:\t\n"); + } + + @Test + public void testSplitCalculatorNoEdge() { + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + + Multimap regions = sc.calcCoverage(); + LOG.info("Empty"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions); + assertEquals("", res); + } + + @Test + public void testSplitCalculatorSingleEdge() { + SimpleRange a = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("B")); + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(a); + + Multimap regions = sc.calcCoverage(); + LOG.info("Single edge"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions, 1, 0); + assertEquals("A:\t[A, B]\t\n" + "B:\t\n", res); + } + + @Test + public void testSplitCalculatorDegenerateEdge() { + SimpleRange a = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("A")); + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(a); + + Multimap regions = sc.calcCoverage(); + LOG.info("Single empty edge"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions, 1); + assertEquals("A:\t[A, A]\t\n", res); + } + + @Test + public void testSplitCalculatorCoverSplit() { + SimpleRange a = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("B")); + SimpleRange b = new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes("C")); + SimpleRange c = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C")); + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(a); + sc.add(b); + sc.add(c); + + Multimap regions = sc.calcCoverage(); + LOG.info("AC covers AB, BC"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions, 2, 2, 0); + assertEquals("A:\t[A, B]\t[A, C]\t\n" + "B:\t[A, C]\t[B, C]\t\n" + + "C:\t\n", res); + } + + @Test + public void testSplitCalculatorOverEndpoint() { + SimpleRange a = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("B")); + SimpleRange b = new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes("C")); + SimpleRange c = new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes("D")); + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(a); + sc.add(b); + sc.add(c); + + Multimap regions = sc.calcCoverage(); + LOG.info("AB, BD covers BC"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions, 1, 2, 1, 0); + assertEquals("A:\t[A, B]\t\n" + "B:\t[B, C]\t[B, D]\t\n" + + "C:\t[B, D]\t\n" + "D:\t\n", res); + } + + @Test + public void testSplitCalculatorHoles() { + SimpleRange a = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("B")); + SimpleRange b = new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes("C")); + SimpleRange c = new SimpleRange(Bytes.toBytes("E"), Bytes.toBytes("F")); + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(a); + sc.add(b); + sc.add(c); + + Multimap regions = sc.calcCoverage(); + LOG.info("Hole between C and E"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions, 1, 1, 0, 1, 0); + assertEquals("A:\t[A, B]\t\n" + "B:\t[B, C]\t\n" + "C:\t\n" + + "E:\t[E, F]\t\n" + "F:\t\n", res); + } + + @Test + public void testSplitCalculatorOverreach() { + SimpleRange a = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C")); + SimpleRange b = new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes("D")); + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(a); + sc.add(b); + + Multimap regions = sc.calcCoverage(); + LOG.info("AC and BD overlap but share no start/end keys"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions, 1, 2, 1, 0); + assertEquals("A:\t[A, C]\t\n" + "B:\t[A, C]\t[B, D]\t\n" + + "C:\t[B, D]\t\n" + "D:\t\n", res); + } + + @Test + public void testSplitCalculatorFloor() { + SimpleRange a = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C")); + SimpleRange b = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("B")); + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(a); + sc.add(b); + + Multimap regions = sc.calcCoverage(); + LOG.info("AC and AB overlap in the beginning"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions, 2, 1, 0); + assertEquals("A:\t[A, B]\t[A, C]\t\n" + "B:\t[A, C]\t\n" + "C:\t\n", res); + } + + @Test + public void testSplitCalculatorCeil() { + SimpleRange a = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C")); + SimpleRange b = new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes("C")); + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(a); + sc.add(b); + + Multimap regions = sc.calcCoverage(); + LOG.info("AC and BC overlap in the end"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions, 1, 2, 0); + assertEquals("A:\t[A, C]\t\n" + "B:\t[A, C]\t[B, C]\t\n" + "C:\t\n", res); + } + + @Test + public void testSplitCalculatorEq() { + SimpleRange a = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C")); + SimpleRange b = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C")); + + LOG.info(a.tiebreaker + " - " + b.tiebreaker); + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(a); + sc.add(b); + + Multimap regions = sc.calcCoverage(); + LOG.info("AC and AC overlap completely"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions, 2, 0); + assertEquals("A:\t[A, C]\t[A, C]\t\n" + "C:\t\n", res); + } + + @Test + public void testSplitCalculatorBackwards() { + SimpleRange a = new SimpleRange(Bytes.toBytes("C"), Bytes.toBytes("A")); + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(a); + + Multimap regions = sc.calcCoverage(); + LOG.info("CA is backwards"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions); // expect nothing + assertEquals("", res); + } + + @Test + public void testComplex() { + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("Am"))); + sc.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C"))); + sc.add(new SimpleRange(Bytes.toBytes("Am"), Bytes.toBytes("C"))); + sc.add(new SimpleRange(Bytes.toBytes("D"), Bytes.toBytes("E"))); + sc.add(new SimpleRange(Bytes.toBytes("F"), Bytes.toBytes("G"))); + sc.add(new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes("E"))); + sc.add(new SimpleRange(Bytes.toBytes("H"), Bytes.toBytes("I"))); + sc.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("B"))); + + Multimap regions = sc.calcCoverage(); + LOG.info("Something fairly complex"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions, 3, 3, 3, 1, 2, 0, 1, 0, 1, 0); + assertEquals("A:\t[A, Am]\t[A, B]\t[A, C]\t\n" + + "Am:\t[A, B]\t[A, C]\t[Am, C]\t\n" + + "B:\t[A, C]\t[Am, C]\t[B, E]\t\n" + "C:\t[B, E]\t\n" + + "D:\t[B, E]\t[D, E]\t\n" + "E:\t\n" + "F:\t[F, G]\t\n" + "G:\t\n" + + "H:\t[H, I]\t\n" + "I:\t\n", res); + } + + @Test + public void testBeginEndMarker() { + RegionSplitCalculator sc = new RegionSplitCalculator( + cmp); + sc.add(new SimpleRange(Bytes.toBytes(""), Bytes.toBytes("A"))); + sc.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("B"))); + sc.add(new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes(""))); + + Multimap regions = sc.calcCoverage(); + LOG.info("Special cases -- empty"); + String res = dump(sc.getSplits(), regions); + checkDepths(sc.getSplits(), regions, 1, 1, 1, 0); + assertEquals(":\t[, A]\t\n" + "A:\t[A, B]\t\n" + "B:\t[B, ]\t\n" + + "null:\t\n", res); + } + + @Test + public void testBigRanges() { + SimpleRange ai = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("I")); + SimpleRange ae = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("E")); + SimpleRange ac = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C")); + + Collection bigOverlap = new ArrayList(); + bigOverlap.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("E"))); + bigOverlap.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C"))); + bigOverlap.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("B"))); + bigOverlap.add(new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes("C"))); + bigOverlap.add(new SimpleRange(Bytes.toBytes("E"), Bytes.toBytes("H"))); + bigOverlap.add(ai); + bigOverlap.add(ae); + bigOverlap.add(ac); + + // Expect 1 range to be returned: ai + List bigRanges = RegionSplitCalculator.findBigRanges(bigOverlap, 1); + assertEquals(1, bigRanges.size()); + assertEquals(ai, bigRanges.get(0)); + + // Expect 3 ranges to be returned: ai, ae and ac + bigRanges = RegionSplitCalculator.findBigRanges(bigOverlap, 3); + assertEquals(3, bigRanges.size()); + assertEquals(ai, bigRanges.get(0)); + + SimpleRange r1 = bigRanges.get(1); + SimpleRange r2 = bigRanges.get(2); + assertEquals("A", Bytes.toString(r1.start)); + assertEquals("A", Bytes.toString(r2.start)); + String r1e = Bytes.toString(r1.end); + String r2e = Bytes.toString(r2.end); + assertTrue((r1e.equals("C") && r2e.equals("E")) + || (r1e.equals("E") && r2e.equals("C"))); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java b/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java new file mode 100644 index 0000000..2a72c71 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java @@ -0,0 +1,344 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.RegionSplitter.HexStringSplit; +import org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm; +import org.apache.hadoop.hbase.util.RegionSplitter.UniformSplit; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests for {@link RegionSplitter}, which can create a pre-split table or do a + * rolling split of an existing table. + */ +@Category(MediumTests.class) +public class TestRegionSplitter { + private final static Log LOG = LogFactory.getLog(TestRegionSplitter.class); + private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private final static String CF_NAME = "SPLIT_TEST_CF"; + private final static byte xFF = (byte) 0xff; + + @BeforeClass + public static void setup() throws Exception { + UTIL.startMiniCluster(); + } + + @AfterClass + public static void teardown() throws Exception { + UTIL.shutdownMiniCluster(); + } + + /** + * Test creating a pre-split table using the HexStringSplit algorithm. + */ + @Test + public void testCreatePresplitTableHex() throws Exception { + final List expectedBounds = new ArrayList(); + expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY); + expectedBounds.add("10000000".getBytes()); + expectedBounds.add("20000000".getBytes()); + expectedBounds.add("30000000".getBytes()); + expectedBounds.add("40000000".getBytes()); + expectedBounds.add("50000000".getBytes()); + expectedBounds.add("60000000".getBytes()); + expectedBounds.add("70000000".getBytes()); + expectedBounds.add("80000000".getBytes()); + expectedBounds.add("90000000".getBytes()); + expectedBounds.add("a0000000".getBytes()); + expectedBounds.add("b0000000".getBytes()); + expectedBounds.add("c0000000".getBytes()); + expectedBounds.add("d0000000".getBytes()); + expectedBounds.add("e0000000".getBytes()); + expectedBounds.add("f0000000".getBytes()); + expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY); + + // Do table creation/pre-splitting and verification of region boundaries + preSplitTableAndVerify(expectedBounds, + HexStringSplit.class.getSimpleName(), "NewHexPresplitTable"); + } + + /** + * Test creating a pre-split table using the UniformSplit algorithm. + */ + @Test + public void testCreatePresplitTableUniform() throws Exception { + List expectedBounds = new ArrayList(); + expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY); + expectedBounds.add(new byte[] { 0x10, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] { 0x20, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] { 0x30, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] { 0x40, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] { 0x50, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] { 0x60, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] { 0x70, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] {(byte)0x80, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] {(byte)0x90, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] {(byte)0xa0, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] {(byte)0xb0, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] {(byte)0xc0, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] {(byte)0xd0, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] {(byte)0xe0, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(new byte[] {(byte)0xf0, 0, 0, 0, 0, 0, 0, 0}); + expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY); + + // Do table creation/pre-splitting and verification of region boundaries + preSplitTableAndVerify(expectedBounds, UniformSplit.class.getSimpleName(), + "NewUniformPresplitTable"); + } + + /** + * Unit tests for the HexStringSplit algorithm. Makes sure it divides up the + * space of keys in the way that we expect. + */ + @Test + public void unitTestHexStringSplit() { + HexStringSplit splitter = new HexStringSplit(); + // Check splitting while starting from scratch + + byte[][] twoRegionsSplits = splitter.split(2); + assertEquals(1, twoRegionsSplits.length); + assertArrayEquals(twoRegionsSplits[0], "80000000".getBytes()); + + byte[][] threeRegionsSplits = splitter.split(3); + assertEquals(2, threeRegionsSplits.length); + byte[] expectedSplit0 = "55555555".getBytes(); + assertArrayEquals(expectedSplit0, threeRegionsSplits[0]); + byte[] expectedSplit1 = "aaaaaaaa".getBytes(); + assertArrayEquals(expectedSplit1, threeRegionsSplits[1]); + + // Check splitting existing regions that have start and end points + byte[] splitPoint = splitter.split("10000000".getBytes(), "30000000".getBytes()); + assertArrayEquals("20000000".getBytes(), splitPoint); + + byte[] lastRow = "ffffffff".getBytes(); + assertArrayEquals(lastRow, splitter.lastRow()); + byte[] firstRow = "00000000".getBytes(); + assertArrayEquals(firstRow, splitter.firstRow()); + + // Halfway between 00... and 20... should be 10... + splitPoint = splitter.split(firstRow, "20000000".getBytes()); + assertArrayEquals(splitPoint, "10000000".getBytes()); + + // Halfway between df... and ff... should be ef.... + splitPoint = splitter.split("dfffffff".getBytes(), lastRow); + assertArrayEquals(splitPoint,"efffffff".getBytes()); + } + + /** + * Unit tests for the UniformSplit algorithm. Makes sure it divides up the space of + * keys in the way that we expect. + */ + @Test + public void unitTestUniformSplit() { + UniformSplit splitter = new UniformSplit(); + + // Check splitting while starting from scratch + try { + splitter.split(1); + throw new AssertionError("Splitting into <2 regions should have thrown exception"); + } catch (IllegalArgumentException e) { } + + byte[][] twoRegionsSplits = splitter.split(2); + assertEquals(1, twoRegionsSplits.length); + assertArrayEquals(twoRegionsSplits[0], + new byte[] { (byte) 0x80, 0, 0, 0, 0, 0, 0, 0 }); + + byte[][] threeRegionsSplits = splitter.split(3); + assertEquals(2, threeRegionsSplits.length); + byte[] expectedSplit0 = new byte[] {0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55}; + assertArrayEquals(expectedSplit0, threeRegionsSplits[0]); + byte[] expectedSplit1 = new byte[] {(byte)0xAA, (byte)0xAA, (byte)0xAA, (byte)0xAA, + (byte)0xAA, (byte)0xAA, (byte)0xAA, (byte)0xAA}; + assertArrayEquals(expectedSplit1, threeRegionsSplits[1]); + + // Check splitting existing regions that have start and end points + byte[] splitPoint = splitter.split(new byte[] {0x10}, new byte[] {0x30}); + assertArrayEquals(new byte[] {0x20}, splitPoint); + + byte[] lastRow = new byte[] {xFF, xFF, xFF, xFF, xFF, xFF, xFF, xFF}; + assertArrayEquals(lastRow, splitter.lastRow()); + byte[] firstRow = ArrayUtils.EMPTY_BYTE_ARRAY; + assertArrayEquals(firstRow, splitter.firstRow()); + + splitPoint = splitter.split(firstRow, new byte[] {0x20}); + assertArrayEquals(splitPoint, new byte[] {0x10}); + + splitPoint = splitter.split(new byte[] {(byte)0xdf, xFF, xFF, xFF, xFF, + xFF, xFF, xFF}, lastRow); + assertArrayEquals(splitPoint, + new byte[] {(byte)0xef, xFF, xFF, xFF, xFF, xFF, xFF, xFF}); + } + + @Test + public void testUserInput() { + SplitAlgorithm algo = new HexStringSplit(); + assertFalse(splitFailsPrecondition(algo)); // default settings are fine + assertFalse(splitFailsPrecondition(algo, "00", "AA")); // custom is fine + assertTrue(splitFailsPrecondition(algo, "AA", "00")); // range error + assertTrue(splitFailsPrecondition(algo, "AA", "AA")); // range error + assertFalse(splitFailsPrecondition(algo, "0", "2", 3)); // should be fine + assertFalse(splitFailsPrecondition(algo, "0", "A", 11)); // should be fine + assertTrue(splitFailsPrecondition(algo, "0", "A", 12)); // too granular + + algo = new UniformSplit(); + assertFalse(splitFailsPrecondition(algo)); // default settings are fine + assertFalse(splitFailsPrecondition(algo, "\\x00", "\\xAA")); // custom is fine + assertTrue(splitFailsPrecondition(algo, "\\xAA", "\\x00")); // range error + assertTrue(splitFailsPrecondition(algo, "\\xAA", "\\xAA")); // range error + assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x02", 3)); // should be fine + assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x0A", 11)); // should be fine + assertTrue(splitFailsPrecondition(algo, "\\x00", "\\x0A", 12)); // too granular + } + + private boolean splitFailsPrecondition(SplitAlgorithm algo) { + return splitFailsPrecondition(algo, 100); + } + + private boolean splitFailsPrecondition(SplitAlgorithm algo, String firstRow, + String lastRow) { + return splitFailsPrecondition(algo, firstRow, lastRow, 100); + } + + private boolean splitFailsPrecondition(SplitAlgorithm algo, String firstRow, + String lastRow, int numRegions) { + algo.setFirstRow(firstRow); + algo.setLastRow(lastRow); + return splitFailsPrecondition(algo, numRegions); + } + + private boolean splitFailsPrecondition(SplitAlgorithm algo, int numRegions) { + try { + byte[][] s = algo.split(numRegions); + LOG.debug("split algo = " + algo); + if (s != null) { + StringBuilder sb = new StringBuilder(); + for (byte[] b : s) { + sb.append(Bytes.toStringBinary(b) + " "); + } + LOG.debug(sb.toString()); + } + return false; + } catch (IllegalArgumentException e) { + return true; + } catch (IllegalStateException e) { + return true; + } catch (IndexOutOfBoundsException e) { + return true; + } + } + + /** + * Creates a pre-split table with expectedBounds.size()+1 regions, then + * verifies that the region boundaries are the same as the expected + * region boundaries in expectedBounds. + * @throws Various junit assertions + */ + private void preSplitTableAndVerify(List expectedBounds, + String splitClass, String tableName) throws Exception { + final int numRegions = expectedBounds.size()-1; + final Configuration conf = UTIL.getConfiguration(); + conf.setInt("split.count", numRegions); + SplitAlgorithm splitAlgo = RegionSplitter.newSplitAlgoInstance(conf, splitClass); + RegionSplitter.createPresplitTable(tableName, splitAlgo, + new String[] {CF_NAME}, conf); + verifyBounds(expectedBounds, tableName); + } + + private void rollingSplitAndVerify(String tableName, String splitClass, + List expectedBounds) throws Exception { + final Configuration conf = UTIL.getConfiguration(); + + // Set this larger than the number of splits so RegionSplitter won't block + conf.setInt("split.outstanding", 5); + SplitAlgorithm splitAlgo = RegionSplitter.newSplitAlgoInstance(conf, splitClass); + RegionSplitter.rollingSplit(tableName, splitAlgo, conf); + verifyBounds(expectedBounds, tableName); + } + + private void verifyBounds(List expectedBounds, String tableName) + throws Exception { + // Get region boundaries from the cluster and verify their endpoints + final Configuration conf = UTIL.getConfiguration(); + final int numRegions = expectedBounds.size()-1; + final HTable hTable = new HTable(conf, tableName.getBytes()); + final Map regionInfoMap = + hTable.getRegionsInfo(); + assertEquals(numRegions, regionInfoMap.size()); + for (Map.Entry entry: + regionInfoMap.entrySet()) { + final HRegionInfo regionInfo = entry.getKey(); + byte[] regionStart = regionInfo.getStartKey(); + byte[] regionEnd = regionInfo.getEndKey(); + + // This region's start key should be one of the region boundaries + int startBoundaryIndex = indexOfBytes(expectedBounds, regionStart); + assertNotSame(-1, startBoundaryIndex); + + // This region's end key should be the region boundary that comes + // after the starting boundary. + byte[] expectedRegionEnd = expectedBounds.get( + startBoundaryIndex+1); + assertEquals(0, Bytes.compareTo(regionEnd, expectedRegionEnd)); + } + } + + /** + * List.indexOf() doesn't really work for a List, because byte[] + * doesn't override equals(). This method checks whether a list contains + * a given element by checking each element using the byte array + * comparator. + * @return the index of the first element that equals compareTo, or -1 + * if no elements are equal. + */ + static private int indexOfBytes(List list, byte[] compareTo) { + int listIndex = 0; + for(byte[] elem: list) { + if(Bytes.BYTES_COMPARATOR.compare(elem, compareTo) == 0) { + return listIndex; + } + listIndex++; + } + return -1; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestRootPath.java b/src/test/java/org/apache/hadoop/hbase/util/TestRootPath.java new file mode 100644 index 0000000..c596018 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestRootPath.java @@ -0,0 +1,71 @@ +/** + * Copyright 2008 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import junit.framework.TestCase; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.experimental.categories.Category; + +/** + * Test requirement that root directory must be a URI + */ +@Category(SmallTests.class) +public class TestRootPath extends TestCase { + private static final Log LOG = LogFactory.getLog(TestRootPath.class); + + /** The test */ + public void testRootPath() { + try { + // Try good path + FSUtils.validateRootPath(new Path("file:///tmp/hbase/hbase")); + } catch (IOException e) { + LOG.fatal("Unexpected exception checking valid path:", e); + fail(); + } + try { + // Try good path + FSUtils.validateRootPath(new Path("hdfs://a:9000/hbase")); + } catch (IOException e) { + LOG.fatal("Unexpected exception checking valid path:", e); + fail(); + } + try { + // bad path + FSUtils.validateRootPath(new Path("/hbase")); + fail(); + } catch (IOException e) { + // Expected. + LOG.info("Got expected exception when checking invalid path:", e); + } + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java b/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java new file mode 100644 index 0000000..e974b9a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestSizeBasedThrottler.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.hbase.MediumTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * This tests some race conditions that can happen + * occasionally, but not every time. + */ +@Category(MediumTests.class) +public class TestSizeBasedThrottler { + + private static final int REPEATS = 100; + + private Thread makeThread(final SizeBasedThrottler throttler, + final AtomicBoolean failed, final int delta, + final int limit, final CountDownLatch latch) { + + Thread ret = new Thread(new Runnable() { + + @Override + public void run() { + try { + latch.await(); + if (throttler.increase(delta) > limit) { + failed.set(true); + } + throttler.decrease(delta); + } catch (Exception e) { + failed.set(true); + } + } + }); + + ret.start(); + return ret; + } + + private void runGenericTest(int threshold, int delta, int maxValueAllowed, + int numberOfThreads, long timeout) { + SizeBasedThrottler throttler = new SizeBasedThrottler(threshold); + AtomicBoolean failed = new AtomicBoolean(false); + + ArrayList threads = new ArrayList(numberOfThreads); + CountDownLatch latch = new CountDownLatch(1); + long timeElapsed = 0; + + for (int i = 0; i < numberOfThreads; ++i) { + threads.add(makeThread(throttler, failed, delta, maxValueAllowed, latch)); + } + + latch.countDown(); + for (Thread t : threads) { + try { + long beforeJoin = System.currentTimeMillis(); + t.join(timeout - timeElapsed); + timeElapsed += System.currentTimeMillis() - beforeJoin; + if (t.isAlive() || timeElapsed >= timeout) { + fail("Timeout reached."); + } + } catch (InterruptedException e) { + fail("Got InterruptedException"); + } + } + + assertFalse(failed.get()); + } + + @Test + public void testSmallIncreases(){ + for (int i = 0; i < REPEATS; ++i) { + runGenericTest( + 10, // threshold + 1, // delta + 15, // fail if throttler's value + // exceeds 15 + 1000, // use 1000 threads + 500 // wait for 500ms + ); + } + } + + @Test + public void testBigIncreases() { + for (int i = 0; i < REPEATS; ++i) { + runGenericTest( + 1, // threshold + 2, // delta + 4, // fail if throttler's value + // exceeds 4 + 1000, // use 1000 threads + 500 // wait for 500ms + ); + } + } + + @Test + public void testIncreasesEqualToThreshold(){ + for (int i = 0; i < REPEATS; ++i) { + runGenericTest( + 1, // threshold + 1, // delta + 2, // fail if throttler's value + // exceeds 2 + 1000, // use 1000 threads + 500 // wait for 500ms + ); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestSortedCopyOnWriteSet.java b/src/test/java/org/apache/hadoop/hbase/util/TestSortedCopyOnWriteSet.java new file mode 100644 index 0000000..29e025a --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestSortedCopyOnWriteSet.java @@ -0,0 +1,111 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.*; + +import java.util.Arrays; +import java.util.Iterator; + +import com.google.common.collect.Lists; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestSortedCopyOnWriteSet { + + @Test + public void testSorting() throws Exception { + SortedCopyOnWriteSet set = new SortedCopyOnWriteSet(); + set.add("c"); + set.add("d"); + set.add("a"); + set.add("b"); + + String[] expected = new String[]{"a", "b", "c", "d"}; + String[] stored = set.toArray(new String[4]); + assertArrayEquals(expected, stored); + + set.add("c"); + assertEquals(4, set.size()); + stored = set.toArray(new String[4]); + assertArrayEquals(expected, stored); + } + + @Test + public void testIteratorIsolation() throws Exception { + SortedCopyOnWriteSet set = new SortedCopyOnWriteSet( + Lists.newArrayList("a", "b", "c", "d", "e")); + + // isolation of remove() + Iterator iter = set.iterator(); + set.remove("c"); + boolean found = false; + while (iter.hasNext() && !found) { + found = "c".equals(iter.next()); + } + assertTrue(found); + + iter = set.iterator(); + found = false; + while (iter.hasNext() && !found) { + found = "c".equals(iter.next()); + } + assertFalse(found); + + // isolation of add() + iter = set.iterator(); + set.add("f"); + found = false; + while (iter.hasNext() && !found) { + String next = iter.next(); + found = "f".equals(next); + } + assertFalse(found); + + // isolation of addAll() + iter = set.iterator(); + set.addAll(Lists.newArrayList("g", "h", "i")); + found = false; + while (iter.hasNext() && !found) { + String next = iter.next(); + found = "g".equals(next) || "h".equals(next) || "i".equals(next); + } + assertFalse(found); + + // isolation of clear() + iter = set.iterator(); + set.clear(); + assertEquals(0, set.size()); + int size = 0; + while (iter.hasNext()) { + iter.next(); + size++; + } + assertTrue(size > 0); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestThreads.java b/src/test/java/org/apache/hadoop/hbase/util/TestThreads.java new file mode 100644 index 0000000..6c249c7 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/TestThreads.java @@ -0,0 +1,83 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertTrue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestThreads { + private static final Log LOG = LogFactory.getLog(TestThreads.class); + + private static final int SLEEP_TIME_MS = 5000; + private static final int TOLERANCE_MS = (int) (0.05 * SLEEP_TIME_MS); + + private volatile boolean wasInterrupted; + + @Test(timeout=60000) + public void testSleepWithoutInterrupt() throws InterruptedException { + Thread sleeper = new Thread(new Runnable() { + @Override + public void run() { + LOG.debug("Sleeper thread: sleeping for " + SLEEP_TIME_MS); + Threads.sleepWithoutInterrupt(SLEEP_TIME_MS); + LOG.debug("Sleeper thread: finished sleeping"); + wasInterrupted = Thread.currentThread().isInterrupted(); + } + }); + LOG.debug("Starting sleeper thread (" + SLEEP_TIME_MS + " ms)"); + sleeper.start(); + long startTime = System.currentTimeMillis(); + LOG.debug("Main thread: sleeping for 500 ms"); + Threads.sleep(500); + + LOG.debug("Interrupting the sleeper thread and sleeping for 2000 ms"); + sleeper.interrupt(); + Threads.sleep(2000); + + LOG.debug("Interrupting the sleeper thread and sleeping for 1000 ms"); + sleeper.interrupt(); + Threads.sleep(1000); + + LOG.debug("Interrupting the sleeper thread again"); + sleeper.interrupt(); + sleeper.join(); + + assertTrue("sleepWithoutInterrupt did not preserve the thread's " + + "interrupted status", wasInterrupted); + + long timeElapsed = System.currentTimeMillis() - startTime; + assertTrue("Elapsed time " + timeElapsed + " ms is out of the expected " + + "range of the sleep time " + SLEEP_TIME_MS, + Math.abs(timeElapsed - SLEEP_TIME_MS) < TOLERANCE_MS); + LOG.debug("Target sleep time: " + SLEEP_TIME_MS + ", time elapsed: " + + timeElapsed); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java new file mode 100644 index 0000000..99f4f9b --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util.hbck; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.HBaseFsck; +import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; + +public class HbckTestingUtil { + private static ExecutorService exec = new ScheduledThreadPoolExecutor(10); + public static HBaseFsck doFsck( + Configuration conf, boolean fix) throws Exception { + return doFsck(conf, fix, null); + } + + public static HBaseFsck doFsck( + Configuration conf, boolean fix, String table) throws Exception { + return doFsck(conf, fix, fix, fix, fix,fix, fix, fix, fix, table); + } + + public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments, + boolean fixMeta, boolean fixHdfsHoles, boolean fixHdfsOverlaps, + boolean fixHdfsOrphans, boolean fixTableOrphans, boolean fixVersionFile, + boolean fixReferenceFiles, String table) throws Exception { + HBaseFsck fsck = new HBaseFsck(conf, exec); + fsck.connect(); + fsck.setDisplayFullReport(); // i.e. -details + fsck.setTimeLag(0); + fsck.setFixAssignments(fixAssignments); + fsck.setFixMeta(fixMeta); + fsck.setFixHdfsHoles(fixHdfsHoles); + fsck.setFixHdfsOverlaps(fixHdfsOverlaps); + fsck.setFixHdfsOrphans(fixHdfsOrphans); + fsck.setFixTableOrphans(fixTableOrphans); + fsck.setFixVersionFile(fixVersionFile); + fsck.setFixReferenceFiles(fixReferenceFiles); + if (table != null) { + fsck.includeTable(table); + } + fsck.onlineHbck(); + return fsck; + } + + /** + * Runs hbck with the -sidelineCorruptHFiles option + * @param conf + * @param table table constraint + * @return + * @throws Exception + */ + public static HBaseFsck doHFileQuarantine(Configuration conf, String table) throws Exception { + String[] args = {"-sidelineCorruptHFiles", "-ignorePreCheckPermission", table}; + HBaseFsck hbck = new HBaseFsck(conf, exec); + hbck.exec(exec, args); + return hbck; + } + + public static void assertNoErrors(HBaseFsck fsck) throws Exception { + List errs = fsck.getErrors().getErrorList(); + assertEquals(new ArrayList(), errs); + } + + public static void assertErrors(HBaseFsck fsck, ERROR_CODE[] expectedErrors) { + List errs = fsck.getErrors().getErrorList(); + assertEquals(Arrays.asList(expectedErrors), errs); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java new file mode 100644 index 0000000..078f1f6 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java @@ -0,0 +1,276 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util.hbck; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.Before; +import org.junit.experimental.categories.Category; + +/** + * This testing base class creates a minicluster and testing table table + * and shuts down the cluster afterwards. It also provides methods wipes out + * meta and to inject errors into meta and the file system. + * + * Tests should generally break stuff, then attempt to rebuild the meta table + * offline, then restart hbase, and finally perform checks. + * + * NOTE: This is a slow set of tests which takes ~30s each needs to run on a + * relatively beefy machine. It seems necessary to have each test in a new jvm + * since minicluster startup and tear downs seem to leak file handles and + * eventually cause out of file handle exceptions. + */ +@Category(LargeTests.class) +public class OfflineMetaRebuildTestCore { + protected final static Log LOG = LogFactory + .getLog(OfflineMetaRebuildTestCore.class); + protected HBaseTestingUtility TEST_UTIL; + protected Configuration conf; + private final static byte[] FAM = Bytes.toBytes("fam"); + + // for the instance, reset every test run + protected HTable htbl; + protected final static byte[][] splits = new byte[][] { Bytes.toBytes("A"), + Bytes.toBytes("B"), Bytes.toBytes("C") }; + + private final static String TABLE_BASE = "tableMetaRebuild"; + private static int tableIdx = 0; + protected String table = "tableMetaRebuild"; + + @Before + public void setUpBefore() throws Exception { + TEST_UTIL = new HBaseTestingUtility(); + TEST_UTIL.getConfiguration().setInt("dfs.datanode.max.xceivers", 9192); + TEST_UTIL.startMiniCluster(3); + conf = TEST_UTIL.getConfiguration(); + assertEquals(0, TEST_UTIL.getHBaseAdmin().listTables().length); + + // setup the table + table = TABLE_BASE + "-" + tableIdx; + tableIdx++; + htbl = setupTable(table); + populateTable(htbl); + assertEquals(4, scanMeta()); + LOG.info("Table " + table + " has " + tableRowCount(conf, table) + + " entries."); + assertEquals(16, tableRowCount(conf, table)); + TEST_UTIL.getHBaseAdmin().disableTable(table); + assertEquals(1, TEST_UTIL.getHBaseAdmin().listTables().length); + } + + @After + public void tearDownAfter() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + HConnectionManager.deleteConnection(conf); + } + + /** + * Setup a clean table before we start mucking with it. + * + * @throws IOException + * @throws InterruptedException + * @throws KeeperException + */ + private HTable setupTable(String tablename) throws Exception { + HTableDescriptor desc = new HTableDescriptor(tablename); + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM)); + desc.addFamily(hcd); // If a table has no CF's it doesn't get checked + TEST_UTIL.getHBaseAdmin().createTable(desc, splits); + return new HTable(TEST_UTIL.getConfiguration(), tablename); + } + + private void dumpMeta(HTableDescriptor htd) throws IOException { + List metaRows = TEST_UTIL.getMetaTableRows(htd.getName()); + for (byte[] row : metaRows) { + LOG.info(Bytes.toString(row)); + } + } + + private void populateTable(HTable tbl) throws IOException { + byte[] values = { 'A', 'B', 'C', 'D' }; + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < values.length; j++) { + Put put = new Put(new byte[] { values[i], values[j] }); + put.add(Bytes.toBytes("fam"), new byte[] {}, new byte[] { values[i], + values[j] }); + tbl.put(put); + } + } + tbl.flushCommits(); + } + + /** + * delete table in preparation for next test + * + * @param tablename + * @throws IOException + */ + void deleteTable(HBaseAdmin admin, String tablename) throws IOException { + try { + byte[] tbytes = Bytes.toBytes(tablename); + admin.disableTable(tbytes); + admin.deleteTable(tbytes); + } catch (Exception e) { + // Do nothing. + } + } + + protected void deleteRegion(Configuration conf, final HTable tbl, + byte[] startKey, byte[] endKey) throws IOException { + + LOG.info("Before delete:"); + HTableDescriptor htd = tbl.getTableDescriptor(); + dumpMeta(htd); + + Map hris = tbl.getRegionsInfo(); + for (Entry e : hris.entrySet()) { + HRegionInfo hri = e.getKey(); + HServerAddress hsa = e.getValue(); + if (Bytes.compareTo(hri.getStartKey(), startKey) == 0 + && Bytes.compareTo(hri.getEndKey(), endKey) == 0) { + + LOG.info("RegionName: " + hri.getRegionNameAsString()); + byte[] deleteRow = hri.getRegionName(); + TEST_UTIL.getHBaseAdmin().unassign(deleteRow, true); + + LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString()); + Path rootDir = new Path(conf.get(HConstants.HBASE_DIR)); + FileSystem fs = rootDir.getFileSystem(conf); + Path p = new Path(rootDir + "/" + htd.getNameAsString(), + hri.getEncodedName()); + fs.delete(p, true); + + HTable meta = new HTable(conf, HConstants.META_TABLE_NAME); + Delete delete = new Delete(deleteRow); + meta.delete(delete); + } + LOG.info(hri.toString() + hsa.toString()); + } + + TEST_UTIL.getMetaTableRows(htd.getName()); + LOG.info("After delete:"); + dumpMeta(htd); + } + + protected HRegionInfo createRegion(Configuration conf, final HTable htbl, + byte[] startKey, byte[] endKey) throws IOException { + HTable meta = new HTable(conf, HConstants.META_TABLE_NAME); + HTableDescriptor htd = htbl.getTableDescriptor(); + HRegionInfo hri = new HRegionInfo(htbl.getTableName(), startKey, endKey); + + LOG.info("manually adding regioninfo and hdfs data: " + hri.toString()); + Path rootDir = new Path(conf.get(HConstants.HBASE_DIR)); + FileSystem fs = rootDir.getFileSystem(conf); + Path p = new Path(rootDir + "/" + htd.getNameAsString(), + hri.getEncodedName()); + fs.mkdirs(p); + Path riPath = new Path(p, HRegion.REGIONINFO_FILE); + FSDataOutputStream out = fs.create(riPath); + hri.write(out); + out.close(); + + // add to meta. + Put put = new Put(hri.getRegionName()); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + meta.put(put); + meta.flushCommits(); + return hri; + } + + protected void wipeOutMeta() throws IOException { + // Mess it up by blowing up meta. + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + Scan s = new Scan(); + HTable meta = new HTable(conf, HConstants.META_TABLE_NAME); + ResultScanner scanner = meta.getScanner(s); + List dels = new ArrayList(); + for (Result r : scanner) { + Delete d = new Delete(r.getRow()); + dels.add(d); + admin.unassign(r.getRow(), true); + } + meta.delete(dels); + meta.flushCommits(); + scanner.close(); + meta.close(); + } + + /** + * Returns the number of rows in a given table. HBase must be up and the table + * should be present (will wait for timeout for a while otherwise) + * + * @return # of rows in the specified table + */ + protected int tableRowCount(Configuration conf, String table) + throws IOException { + HTable t = new HTable(conf, table); + Scan st = new Scan(); + + ResultScanner rst = t.getScanner(st); + int count = 0; + for (@SuppressWarnings("unused") + Result rt : rst) { + count++; + } + return count; + } + + /** + * Dumps .META. table info + * + * @return # of entries in meta. + */ + protected int scanMeta() throws IOException { + int count = 0; + HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName()); + ResultScanner scanner = meta.getScanner(new Scan()); + LOG.info("Table: " + Bytes.toString(meta.getTableName())); + for (Result res : scanner) { + LOG.info(Bytes.toString(res.getRow())); + count++; + } + return count; + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java new file mode 100644 index 0000000..4bdc645 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util.hbck; + +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; + +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.util.HBaseFsck; +import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.HBaseTestingUtility; +/** + * This builds a table, removes info from meta, and then rebuilds meta. + */ +@Category(MediumTests.class) +public class TestOfflineMetaRebuildBase extends OfflineMetaRebuildTestCore { + + @Test(timeout = 120000) + public void testMetaRebuild() throws Exception { + wipeOutMeta(); + + // is meta really messed up? + assertEquals(0, scanMeta()); + assertErrors(doFsck(conf, false), + new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, }); + // Note, would like to check # of tables, but this takes a while to time + // out. + + // shutdown the minicluster + TEST_UTIL.shutdownMiniHBaseCluster(); + TEST_UTIL.shutdownMiniZKCluster(); + HConnectionManager.deleteConnection(conf); + + // rebuild meta table from scratch + HBaseFsck fsck = new HBaseFsck(conf); + assertTrue(fsck.rebuildMeta(false)); + + // bring up the minicluster + TEST_UTIL.startMiniZKCluster(); // tables seem enabled by default + TEST_UTIL.restartHBaseCluster(3); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL); + + LOG.info("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + LOG.info("No more RIT in ZK, now doing final test verification"); + + // everything is good again. + assertEquals(4, scanMeta()); + HTableDescriptor[] htbls = TEST_UTIL.getHBaseAdmin().listTables(); + LOG.info("Tables present after restart: " + Arrays.toString(htbls)); + + assertEquals(1, htbls.length); + assertErrors(doFsck(conf, false), new ERROR_CODE[] {}); + LOG.info("Table " + table + " has " + tableRowCount(conf, table) + + " entries."); + assertEquals(16, tableRowCount(conf, table)); + } + + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java new file mode 100644 index 0000000..d527498 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util.hbck; + +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.util.HBaseFsck; +import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * This builds a table, removes info from meta, and then fails when attempting + * to rebuild meta. + */ +@Category(MediumTests.class) +public class TestOfflineMetaRebuildHole extends OfflineMetaRebuildTestCore { + + @Test(timeout = 120000) + public void testMetaRebuildHoleFail() throws Exception { + // Fully remove a meta entry and hdfs region + byte[] startKey = splits[1]; + byte[] endKey = splits[2]; + deleteRegion(conf, htbl, startKey, endKey); + + wipeOutMeta(); + + // is meta really messed up? + assertEquals(0, scanMeta()); + assertErrors(doFsck(conf, false), new ERROR_CODE[] { + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, }); + // Note, would like to check # of tables, but this takes a while to time + // out. + + // shutdown the minicluster + TEST_UTIL.shutdownMiniHBaseCluster(); + TEST_UTIL.shutdownMiniZKCluster(); + + // attempt to rebuild meta table from scratch + HBaseFsck fsck = new HBaseFsck(conf); + assertFalse(fsck.rebuildMeta(false)); + + // bring up the minicluster + TEST_UTIL.startMiniZKCluster(); // tables seem enabled by default + TEST_UTIL.restartHBaseCluster(3); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL); + + LOG.info("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + LOG.info("No more RIT in ZK, now doing final test verification"); + + // Meta still messed up. + assertEquals(0, scanMeta()); + HTableDescriptor[] htbls = TEST_UTIL.getHBaseAdmin().listTables(); + LOG.info("Tables present after restart: " + Arrays.toString(htbls)); + + // After HBASE-451 HBaseAdmin.listTables() gets table descriptors from FS, + // so the table is still present and this should be 1. + assertEquals(1, htbls.length); + assertErrors(doFsck(conf, false), new ERROR_CODE[] { + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, }); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java new file mode 100644 index 0000000..67d7c83 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util.hbck; + +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.util.Arrays; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.util.HBaseFsck; +import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; +import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Multimap; + +/** + * This builds a table, builds an overlap, and then fails when attempting to + * rebuild meta. + */ +@Category(MediumTests.class) +public class TestOfflineMetaRebuildOverlap extends OfflineMetaRebuildTestCore { + + @Test(timeout = 120000) + public void testMetaRebuildOverlapFail() throws Exception { + // Add a new .regioninfo meta entry in hdfs + byte[] startKey = splits[0]; + byte[] endKey = splits[2]; + createRegion(conf, htbl, startKey, endKey); + + wipeOutMeta(); + + // is meta really messed up? + assertEquals(0, scanMeta()); + assertErrors(doFsck(conf, false), + new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, }); + // Note, would like to check # of tables, but this takes a while to time + // out. + + // shutdown the minicluster + TEST_UTIL.shutdownMiniHBaseCluster(); + TEST_UTIL.shutdownMiniZKCluster(); + + // attempt to rebuild meta table from scratch + HBaseFsck fsck = new HBaseFsck(conf); + assertFalse(fsck.rebuildMeta(false)); + + Multimap problems = fsck.getOverlapGroups(table); + assertEquals(1, problems.keySet().size()); + assertEquals(3, problems.size()); + + // bring up the minicluster + TEST_UTIL.startMiniZKCluster(); // tables seem enabled by default + TEST_UTIL.restartHBaseCluster(3); + + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL); + + LOG.info("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + LOG.info("No more RIT in ZK, now doing final test verification"); + + // Meta still messed up. + assertEquals(0, scanMeta()); + HTableDescriptor[] htbls = TEST_UTIL.getHBaseAdmin().listTables(); + LOG.info("Tables present after restart: " + Arrays.toString(htbls)); + + // After HBASE-451 HBaseAdmin.listTables() gets table descriptors from FS, + // so the table is still present and this should be 1. + assertEquals(1, htbls.length); + assertErrors(doFsck(conf, false), + new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + ERROR_CODE.NOT_IN_META_OR_DEPLOYED, }); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java new file mode 100644 index 0000000..764ac19 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestHQuorumPeer.java @@ -0,0 +1,147 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; +import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.*; + +/** + * Test for HQuorumPeer. + */ +@Category(SmallTests.class) +public class TestHQuorumPeer { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static int PORT_NO = 21818; + private Path dataDir; + + + @Before public void setup() throws IOException { + // Set it to a non-standard port. + TEST_UTIL.getConfiguration().setInt(HConstants.ZOOKEEPER_CLIENT_PORT, + PORT_NO); + this.dataDir = TEST_UTIL.getDataTestDir(this.getClass().getName()); + FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); + if (fs.exists(this.dataDir)) { + if (!fs.delete(this.dataDir, true)) { + throw new IOException("Failed cleanup of " + this.dataDir); + } + } + if (!fs.mkdirs(this.dataDir)) { + throw new IOException("Failed create of " + this.dataDir); + } + } + + @Test public void testMakeZKProps() { + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + conf.set(HConstants.ZOOKEEPER_DATA_DIR, this.dataDir.toString()); + Properties properties = ZKConfig.makeZKProps(conf); + assertEquals(dataDir.toString(), (String)properties.get("dataDir")); + assertEquals(Integer.valueOf(PORT_NO), + Integer.valueOf(properties.getProperty("clientPort"))); + assertEquals("localhost:2888:3888", properties.get("server.0")); + assertEquals(null, properties.get("server.1")); + + String oldValue = conf.get(HConstants.ZOOKEEPER_QUORUM); + conf.set(HConstants.ZOOKEEPER_QUORUM, "a.foo.bar,b.foo.bar,c.foo.bar"); + properties = ZKConfig.makeZKProps(conf); + assertEquals(dataDir.toString(), properties.get("dataDir")); + assertEquals(Integer.valueOf(PORT_NO), + Integer.valueOf(properties.getProperty("clientPort"))); + assertEquals("a.foo.bar:2888:3888", properties.get("server.0")); + assertEquals("b.foo.bar:2888:3888", properties.get("server.1")); + assertEquals("c.foo.bar:2888:3888", properties.get("server.2")); + assertEquals(null, properties.get("server.3")); + conf.set(HConstants.ZOOKEEPER_QUORUM, oldValue); + } + + @Test public void testConfigInjection() throws Exception { + String s = + "dataDir=" + this.dataDir.toString() + "\n" + + "clientPort=2181\n" + + "initLimit=2\n" + + "syncLimit=2\n" + + "server.0=${hbase.master.hostname}:2888:3888\n" + + "server.1=server1:2888:3888\n" + + "server.2=server2:2888:3888\n"; + + System.setProperty("hbase.master.hostname", "localhost"); + InputStream is = new ByteArrayInputStream(s.getBytes()); + Configuration conf = TEST_UTIL.getConfiguration(); + Properties properties = ZKConfig.parseZooCfg(conf, is); + + assertEquals(this.dataDir.toString(), properties.get("dataDir")); + assertEquals(Integer.valueOf(2181), + Integer.valueOf(properties.getProperty("clientPort"))); + assertEquals("localhost:2888:3888", properties.get("server.0")); + + HQuorumPeer.writeMyID(properties); + QuorumPeerConfig config = new QuorumPeerConfig(); + config.parseProperties(properties); + + assertEquals(this.dataDir.toString(), config.getDataDir()); + assertEquals(2181, config.getClientPortAddress().getPort()); + Map servers = config.getServers(); + assertEquals(3, servers.size()); + assertTrue(servers.containsKey(Long.valueOf(0))); + QuorumServer server = servers.get(Long.valueOf(0)); + assertEquals("localhost", server.addr.getHostName()); + + // Override with system property. + System.setProperty("hbase.master.hostname", "foo.bar"); + is = new ByteArrayInputStream(s.getBytes()); + properties = ZKConfig.parseZooCfg(conf, is); + assertEquals("foo.bar:2888:3888", properties.get("server.0")); + + config.parseProperties(properties); + + servers = config.getServers(); + server = servers.get(Long.valueOf(0)); + assertEquals("foo.bar", server.addr.getHostName()); + } + + @Test public void testShouldAssignDefaultZookeeperClientPort() { + Configuration config = HBaseConfiguration.create(); + config.clear(); + Properties p = ZKConfig.makeZKProps(config); + assertNotNull(p); + assertEquals(2181, p.get("clientPort")); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java new file mode 100644 index 0000000..a693884 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.Stat; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestRecoverableZooKeeper { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + Abortable abortable = new Abortable() { + @Override + public void abort(String why, Throwable e) { + + } + + @Override + public boolean isAborted() { + return false; + } + }; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + @Test + public void testSetDataVersionMismatchInLoop() throws Exception { + String znode = "/hbase/unassigned/9af7cfc9b15910a0b3d714bf40a3248f"; + Configuration conf = TEST_UTIL.getConfiguration(); + Properties properties = ZKConfig.makeZKProps(conf); + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "testSetDataVersionMismatchInLoop", + abortable, true); + String ensemble = ZKConfig.getZKQuorumServersString(properties); + RecoverableZooKeeper rzk = ZKUtil.connect(conf, ensemble, zkw); + rzk.create(znode, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + rzk.setData(znode, "OPENING".getBytes(), 0); + Field zkField = RecoverableZooKeeper.class.getDeclaredField("zk"); + zkField.setAccessible(true); + int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT); + ZookeeperStub zkStub = new ZookeeperStub(ensemble, timeout, zkw); + zkStub.setThrowExceptionInNumOperations(1); + zkField.set(rzk, zkStub); + byte[] opened = "OPENED".getBytes(); + rzk.setData(znode, opened, 1); + byte[] data = rzk.getData(znode, false, new Stat()); + assertTrue(Bytes.equals(opened, data)); + } + + class ZookeeperStub extends ZooKeeper { + + private int throwExceptionInNumOperations; + + public ZookeeperStub(String connectString, int sessionTimeout, Watcher watcher) + throws IOException { + super(connectString, sessionTimeout, watcher); + } + + public void setThrowExceptionInNumOperations(int throwExceptionInNumOperations) { + this.throwExceptionInNumOperations = throwExceptionInNumOperations; + } + + private void checkThrowKeeperException() throws KeeperException { + if (throwExceptionInNumOperations == 1) { + throwExceptionInNumOperations = 0; + throw new KeeperException.ConnectionLossException(); + } + if (throwExceptionInNumOperations > 0) + throwExceptionInNumOperations--; + } + + @Override + public Stat setData(String path, byte[] data, int version) throws KeeperException, + InterruptedException { + Stat stat = super.setData(path, data, version); + checkThrowKeeperException(); + return stat; + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java new file mode 100644 index 0000000..4304680 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKLeaderManager.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.zookeeper; + +import static org.junit.Assert.*; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + */ +@Category(MediumTests.class) +public class TestZKLeaderManager { + private static Log LOG = LogFactory.getLog(TestZKLeaderManager.class); + + private static final String LEADER_ZNODE = + "/test/" + TestZKLeaderManager.class.getSimpleName(); + + private static class MockAbortable implements Abortable { + private boolean aborted; + + @Override + public void abort(String why, Throwable e) { + aborted = true; + LOG.fatal("Aborting during test: "+why, e); + fail("Aborted during test: " + why); + } + + @Override + public boolean isAborted() { + return aborted; + } + } + + private static class MockLeader extends Thread implements Stoppable { + private boolean stopped; + private ZooKeeperWatcher watcher; + private ZKLeaderManager zkLeader; + private AtomicBoolean master = new AtomicBoolean(false); + private int index; + + public MockLeader(ZooKeeperWatcher watcher, int index) { + setDaemon(true); + setName("TestZKLeaderManager-leader-" + index); + this.index = index; + this.watcher = watcher; + this.zkLeader = new ZKLeaderManager(watcher, LEADER_ZNODE, + Bytes.toBytes(index), this); + } + + public boolean isMaster() { + return master.get(); + } + + public int getIndex() { + return index; + } + + public ZooKeeperWatcher getWatcher() { + return watcher; + } + + public void run() { + while (!stopped) { + zkLeader.start(); + zkLeader.waitToBecomeLeader(); + master.set(true); + + while (master.get() && !stopped) { + try { + Thread.sleep(200); + } catch (InterruptedException ignored) {} + } + } + } + + public void abdicate() { + zkLeader.stepDownAsLeader(); + master.set(false); + } + + @Override + public void stop(String why) { + stopped = true; + abdicate(); + watcher.close(); + } + + @Override + public boolean isStopped() { + return stopped; + } + } + + private static HBaseTestingUtility TEST_UTIL; + private static MockLeader[] CANDIDATES; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + TEST_UTIL = new HBaseTestingUtility(); + TEST_UTIL.startMiniZKCluster(); + Configuration conf = TEST_UTIL.getConfiguration(); + + // use an abortable to fail the test in the case of any KeeperExceptions + MockAbortable abortable = new MockAbortable(); + CANDIDATES = new MockLeader[3]; + for (int i = 0; i < 3; i++) { + ZooKeeperWatcher watcher = newZK(conf, "server"+i, abortable); + CANDIDATES[i] = new MockLeader(watcher, i); + CANDIDATES[i].start(); + } + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + @Test + public void testLeaderSelection() throws Exception { + MockLeader currentLeader = getCurrentLeader(); + // one leader should have been found + assertNotNull("Leader should exist", currentLeader); + LOG.debug("Current leader index is "+currentLeader.getIndex()); + + byte[] znodeData = ZKUtil.getData(currentLeader.getWatcher(), LEADER_ZNODE); + assertNotNull("Leader znode should contain leader index", znodeData); + assertTrue("Leader znode should not be empty", znodeData.length > 0); + int storedIndex = Bytes.toInt(znodeData); + LOG.debug("Stored leader index in ZK is "+storedIndex); + assertEquals("Leader znode should match leader index", + currentLeader.getIndex(), storedIndex); + + // force a leader transition + currentLeader.abdicate(); + assertFalse(currentLeader.isMaster()); + + // check for new leader + currentLeader = getCurrentLeader(); + // one leader should have been found + assertNotNull("New leader should exist after abdication", currentLeader); + LOG.debug("New leader index is "+currentLeader.getIndex()); + + znodeData = ZKUtil.getData(currentLeader.getWatcher(), LEADER_ZNODE); + assertNotNull("Leader znode should contain leader index", znodeData); + assertTrue("Leader znode should not be empty", znodeData.length > 0); + storedIndex = Bytes.toInt(znodeData); + LOG.debug("Stored leader index in ZK is "+storedIndex); + assertEquals("Leader znode should match leader index", + currentLeader.getIndex(), storedIndex); + + // force another transition by stopping the current + currentLeader.stop("Stopping for test"); + assertFalse(currentLeader.isMaster()); + + // check for new leader + currentLeader = getCurrentLeader(); + // one leader should have been found + assertNotNull("New leader should exist after stop", currentLeader); + LOG.debug("New leader index is "+currentLeader.getIndex()); + + znodeData = ZKUtil.getData(currentLeader.getWatcher(), LEADER_ZNODE); + assertNotNull("Leader znode should contain leader index", znodeData); + assertTrue("Leader znode should not be empty", znodeData.length > 0); + storedIndex = Bytes.toInt(znodeData); + LOG.debug("Stored leader index in ZK is "+storedIndex); + assertEquals("Leader znode should match leader index", + currentLeader.getIndex(), storedIndex); + + // with a second stop we can guarantee that a previous leader has resumed leading + currentLeader.stop("Stopping for test"); + assertFalse(currentLeader.isMaster()); + + // check for new + currentLeader = getCurrentLeader(); + assertNotNull("New leader should exist", currentLeader); + } + + private MockLeader getCurrentLeader() throws Exception { + MockLeader currentLeader = null; + outer: + // wait up to 2 secs for initial leader + for (int i = 0; i < 20; i++) { + for (int j = 0; j < CANDIDATES.length; j++) { + if (CANDIDATES[j].isMaster()) { + // should only be one leader + if (currentLeader != null) { + fail("Both candidate "+currentLeader.getIndex()+" and "+j+" claim to be leader!"); + } + currentLeader = CANDIDATES[j]; + } + } + if (currentLeader != null) { + break outer; + } + Thread.sleep(100); + } + return currentLeader; + } + + private static ZooKeeperWatcher newZK(Configuration conf, String name, + Abortable abort) throws Exception { + Configuration copy = HBaseConfiguration.create(conf); + ZooKeeperWatcher zk = new ZooKeeperWatcher(copy, name, abort); + return zk; + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMulti.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMulti.java new file mode 100644 index 0000000..dd00372 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMulti.java @@ -0,0 +1,291 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.zookeeper; + + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.LinkedList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test ZooKeeper multi-update functionality + */ +@Category(MediumTests.class) +public class TestZKMulti { + private static final Log LOG = LogFactory.getLog(TestZKMulti.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static ZooKeeperWatcher zkw = null; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setBoolean("hbase.zookeeper.useMulti", true); + Abortable abortable = new Abortable() { + @Override + public void abort(String why, Throwable e) { + LOG.info(why, e); + } + + @Override + public boolean isAborted() { + return false; + } + }; + zkw = new ZooKeeperWatcher(conf, + "TestZKMulti", abortable, true); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + @Test + public void testSimpleMulti() throws Exception { + // null multi + ZKUtil.multiOrSequential(zkw, null, false); + + // empty multi + ZKUtil.multiOrSequential(zkw, new LinkedList(), false); + + // single create + String path = ZKUtil.joinZNode(zkw.baseZNode, "testSimpleMulti"); + LinkedList singleCreate = new LinkedList(); + singleCreate.add(ZKUtilOp.createAndFailSilent(path, new byte[0])); + ZKUtil.multiOrSequential(zkw, singleCreate, false); + assertTrue(ZKUtil.checkExists(zkw, path) != -1); + + // single setdata + LinkedList singleSetData = new LinkedList(); + byte [] data = Bytes.toBytes("foobar"); + singleSetData.add(ZKUtilOp.setData(path, data)); + ZKUtil.multiOrSequential(zkw, singleSetData, false); + assertTrue(Bytes.equals(ZKUtil.getData(zkw, path), data)); + + // single delete + LinkedList singleDelete = new LinkedList(); + singleDelete.add(ZKUtilOp.deleteNodeFailSilent(path)); + ZKUtil.multiOrSequential(zkw, singleDelete, false); + assertTrue(ZKUtil.checkExists(zkw, path) == -1); + } + + @Test + public void testComplexMulti() throws Exception { + String path1 = ZKUtil.joinZNode(zkw.baseZNode, "testComplexMulti1"); + String path2 = ZKUtil.joinZNode(zkw.baseZNode, "testComplexMulti2"); + String path3 = ZKUtil.joinZNode(zkw.baseZNode, "testComplexMulti3"); + String path4 = ZKUtil.joinZNode(zkw.baseZNode, "testComplexMulti4"); + String path5 = ZKUtil.joinZNode(zkw.baseZNode, "testComplexMulti5"); + String path6 = ZKUtil.joinZNode(zkw.baseZNode, "testComplexMulti6"); + // create 4 nodes that we'll setData on or delete later + LinkedList create4Nodes = new LinkedList(); + create4Nodes.add(ZKUtilOp.createAndFailSilent(path1, Bytes.toBytes(path1))); + create4Nodes.add(ZKUtilOp.createAndFailSilent(path2, Bytes.toBytes(path2))); + create4Nodes.add(ZKUtilOp.createAndFailSilent(path3, Bytes.toBytes(path3))); + create4Nodes.add(ZKUtilOp.createAndFailSilent(path4, Bytes.toBytes(path4))); + ZKUtil.multiOrSequential(zkw, create4Nodes, false); + assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1), Bytes.toBytes(path1))); + assertTrue(Bytes.equals(ZKUtil.getData(zkw, path2), Bytes.toBytes(path2))); + assertTrue(Bytes.equals(ZKUtil.getData(zkw, path3), Bytes.toBytes(path3))); + assertTrue(Bytes.equals(ZKUtil.getData(zkw, path4), Bytes.toBytes(path4))); + + // do multiple of each operation (setData, delete, create) + LinkedList ops = new LinkedList(); + // setData + ops.add(ZKUtilOp.setData(path1, Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1)))); + ops.add(ZKUtilOp.setData(path2, Bytes.add(Bytes.toBytes(path2), Bytes.toBytes(path2)))); + // delete + ops.add(ZKUtilOp.deleteNodeFailSilent(path3)); + ops.add(ZKUtilOp.deleteNodeFailSilent(path4)); + // create + ops.add(ZKUtilOp.createAndFailSilent(path5, Bytes.toBytes(path5))); + ops.add(ZKUtilOp.createAndFailSilent(path6, Bytes.toBytes(path6))); + ZKUtil.multiOrSequential(zkw, ops, false); + assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1), + Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1)))); + assertTrue(Bytes.equals(ZKUtil.getData(zkw, path2), + Bytes.add(Bytes.toBytes(path2), Bytes.toBytes(path2)))); + assertTrue(ZKUtil.checkExists(zkw, path3) == -1); + assertTrue(ZKUtil.checkExists(zkw, path4) == -1); + assertTrue(Bytes.equals(ZKUtil.getData(zkw, path5), Bytes.toBytes(path5))); + assertTrue(Bytes.equals(ZKUtil.getData(zkw, path6), Bytes.toBytes(path6))); + } + + @Test + public void testSingleFailure() throws Exception { + // try to delete a node that doesn't exist + boolean caughtNoNode = false; + String path = ZKUtil.joinZNode(zkw.baseZNode, "testSingleFailureZ"); + LinkedList ops = new LinkedList(); + ops.add(ZKUtilOp.deleteNodeFailSilent(path)); + try { + ZKUtil.multiOrSequential(zkw, ops, false); + } catch (KeeperException.NoNodeException nne) { + caughtNoNode = true; + } + assertTrue(caughtNoNode); + + // try to setData on a node that doesn't exist + caughtNoNode = false; + ops = new LinkedList(); + ops.add(ZKUtilOp.setData(path, Bytes.toBytes(path))); + try { + ZKUtil.multiOrSequential(zkw, ops, false); + } catch (KeeperException.NoNodeException nne) { + caughtNoNode = true; + } + assertTrue(caughtNoNode); + + // try to create on a node that already exists + boolean caughtNodeExists = false; + ops = new LinkedList(); + ops.add(ZKUtilOp.createAndFailSilent(path, Bytes.toBytes(path))); + ZKUtil.multiOrSequential(zkw, ops, false); + try { + ZKUtil.multiOrSequential(zkw, ops, false); + } catch (KeeperException.NodeExistsException nee) { + caughtNodeExists = true; + } + assertTrue(caughtNodeExists); + } + + @Test + public void testSingleFailureInMulti() throws Exception { + // try a multi where all but one operation succeeds + String pathA = ZKUtil.joinZNode(zkw.baseZNode, "testSingleFailureInMultiA"); + String pathB = ZKUtil.joinZNode(zkw.baseZNode, "testSingleFailureInMultiB"); + String pathC = ZKUtil.joinZNode(zkw.baseZNode, "testSingleFailureInMultiC"); + LinkedList ops = new LinkedList(); + ops.add(ZKUtilOp.createAndFailSilent(pathA, Bytes.toBytes(pathA))); + ops.add(ZKUtilOp.createAndFailSilent(pathB, Bytes.toBytes(pathB))); + ops.add(ZKUtilOp.deleteNodeFailSilent(pathC)); + boolean caughtNoNode = false; + try { + ZKUtil.multiOrSequential(zkw, ops, false); + } catch (KeeperException.NoNodeException nne) { + caughtNoNode = true; + } + assertTrue(caughtNoNode); + // assert that none of the operations succeeded + assertTrue(ZKUtil.checkExists(zkw, pathA) == -1); + assertTrue(ZKUtil.checkExists(zkw, pathB) == -1); + assertTrue(ZKUtil.checkExists(zkw, pathC) == -1); + } + + @Test + public void testMultiFailure() throws Exception { + String pathX = ZKUtil.joinZNode(zkw.baseZNode, "testMultiFailureX"); + String pathY = ZKUtil.joinZNode(zkw.baseZNode, "testMultiFailureY"); + String pathZ = ZKUtil.joinZNode(zkw.baseZNode, "testMultiFailureZ"); + // create X that we will use to fail create later + LinkedList ops = new LinkedList(); + ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathX))); + ZKUtil.multiOrSequential(zkw, ops, false); + + // fail one of each create ,setData, delete + String pathV = ZKUtil.joinZNode(zkw.baseZNode, "testMultiFailureV"); + String pathW = ZKUtil.joinZNode(zkw.baseZNode, "testMultiFailureW"); + ops = new LinkedList(); + ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathX))); // fail -- already exists + ops.add(ZKUtilOp.setData(pathY, Bytes.toBytes(pathY))); // fail -- doesn't exist + ops.add(ZKUtilOp.deleteNodeFailSilent(pathZ)); // fail -- doesn't exist + ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathV))); // pass + ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathW))); // pass + boolean caughtNodeExists = false; + try { + ZKUtil.multiOrSequential(zkw, ops, false); + } catch (KeeperException.NodeExistsException nee) { + // check first operation that fails throws exception + caughtNodeExists = true; + } + assertTrue(caughtNodeExists); + // check that no modifications were made + assertFalse(ZKUtil.checkExists(zkw, pathX) == -1); + assertTrue(ZKUtil.checkExists(zkw, pathY) == -1); + assertTrue(ZKUtil.checkExists(zkw, pathZ) == -1); + assertTrue(ZKUtil.checkExists(zkw, pathW) == -1); + assertTrue(ZKUtil.checkExists(zkw, pathV) == -1); + + // test that with multiple failures, throws an exception corresponding to first failure in list + ops = new LinkedList(); + ops.add(ZKUtilOp.setData(pathY, Bytes.toBytes(pathY))); // fail -- doesn't exist + ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathX))); // fail -- exists + boolean caughtNoNode = false; + try { + ZKUtil.multiOrSequential(zkw, ops, false); + } catch (KeeperException.NoNodeException nne) { + // check first operation that fails throws exception + caughtNoNode = true; + } + assertTrue(caughtNoNode); + // check that no modifications were made + assertFalse(ZKUtil.checkExists(zkw, pathX) == -1); + assertTrue(ZKUtil.checkExists(zkw, pathY) == -1); + assertTrue(ZKUtil.checkExists(zkw, pathZ) == -1); + assertTrue(ZKUtil.checkExists(zkw, pathW) == -1); + assertTrue(ZKUtil.checkExists(zkw, pathV) == -1); + } + + @Test + public void testRunSequentialOnMultiFailure() throws Exception { + String path1 = ZKUtil.joinZNode(zkw.baseZNode, "runSequential1"); + String path2 = ZKUtil.joinZNode(zkw.baseZNode, "runSequential2"); + String path3 = ZKUtil.joinZNode(zkw.baseZNode, "runSequential3"); + String path4 = ZKUtil.joinZNode(zkw.baseZNode, "runSequential4"); + + // create some nodes that we will use later + LinkedList ops = new LinkedList(); + ops.add(ZKUtilOp.createAndFailSilent(path1, Bytes.toBytes(path1))); + ops.add(ZKUtilOp.createAndFailSilent(path2, Bytes.toBytes(path2))); + ZKUtil.multiOrSequential(zkw, ops, false); + + // test that, even with operations that fail, the ones that would pass will pass + // with runSequentialOnMultiFailure + ops = new LinkedList(); + ops.add(ZKUtilOp.setData(path1, Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1)))); // pass + ops.add(ZKUtilOp.deleteNodeFailSilent(path2)); // pass + ops.add(ZKUtilOp.deleteNodeFailSilent(path3)); // fail -- node doesn't exist + ops.add(ZKUtilOp.createAndFailSilent(path4, + Bytes.add(Bytes.toBytes(path4), Bytes.toBytes(path4)))); // pass + ZKUtil.multiOrSequential(zkw, ops, true); + assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1), + Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1)))); + assertTrue(ZKUtil.checkExists(zkw, path2) == -1); + assertTrue(ZKUtil.checkExists(zkw, path3) == -1); + assertFalse(ZKUtil.checkExists(zkw, path4) == -1); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java new file mode 100644 index 0000000..643c583 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTable.java @@ -0,0 +1,299 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.data.Stat; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +@Category(MediumTests.class) +public class TestZKTable { + private static final Log LOG = LogFactory.getLog(TestZKTable.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + Abortable abortable = new Abortable() { + @Override + public void abort(String why, Throwable e) { + LOG.info(why, e); + } + + @Override + public boolean isAborted() { + return false; + } + }; + + @Test + public void testTableStates() + throws ZooKeeperConnectionException, IOException, KeeperException { + final String name = "testDisabled"; + + ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + name, abortable, true); + ZKTable zkt = new ZKTable(zkw); + assertFalse(zkt.isEnabledTable(name)); + assertFalse(zkt.isDisablingTable(name)); + assertFalse(zkt.isDisabledTable(name)); + assertFalse(zkt.isEnablingTable(name)); + assertFalse(zkt.isDisablingOrDisabledTable(name)); + assertFalse(zkt.isDisabledOrEnablingTable(name)); + assertFalse(zkt.isTablePresent(name)); + zkt.setDisablingTable(name); + assertTrue(zkt.isDisablingTable(name)); + assertTrue(zkt.isDisablingOrDisabledTable(name)); + assertFalse(zkt.getDisabledTables().contains(name)); + assertTrue(zkt.isTablePresent(name)); + zkt.setDisabledTable(name); + assertTrue(zkt.isDisabledTable(name)); + assertTrue(zkt.isDisablingOrDisabledTable(name)); + assertFalse(zkt.isDisablingTable(name)); + assertTrue(zkt.getDisabledTables().contains(name)); + assertTrue(zkt.isTablePresent(name)); + zkt.setEnablingTable(name); + assertTrue(zkt.isEnablingTable(name)); + assertTrue(zkt.isDisabledOrEnablingTable(name)); + assertFalse(zkt.isDisabledTable(name)); + assertFalse(zkt.getDisabledTables().contains(name)); + assertTrue(zkt.isTablePresent(name)); + zkt.setEnabledTable(name); + assertTrue(zkt.isEnabledTable(name)); + assertFalse(zkt.isEnablingTable(name)); + assertTrue(zkt.isTablePresent(name)); + zkt.setDeletedTable(name); + assertFalse(zkt.isEnabledTable(name)); + assertFalse(zkt.isDisablingTable(name)); + assertFalse(zkt.isDisabledTable(name)); + assertFalse(zkt.isEnablingTable(name)); + assertFalse(zkt.isDisablingOrDisabledTable(name)); + assertFalse(zkt.isDisabledOrEnablingTable(name)); + assertFalse(zkt.isTablePresent(name)); + } + + private void runTest9294CompatibilityTest(String tableName, Configuration conf) + throws Exception { + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, + tableName, abortable, true); + ZKTable zkt = new ZKTable(zkw); + zkt.setEnabledTable(tableName); + // check that current/0.94 format table has proper ENABLED format + assertTrue( + ZKTableReadOnly.getTableState(zkw, zkw.masterTableZNode, tableName) == TableState.ENABLED); + // check that 0.92 format table is null, as expected by 0.92.0/0.92.1 clients + assertTrue(ZKTableReadOnly.getTableState(zkw, zkw.masterTableZNode92, tableName) == null); + } + + /** + * Test that ZK table writes table state in formats expected by 0.92 and 0.94 clients + */ + @Test + public void test9294Compatibility() throws Exception { + // without useMulti + String tableName = "test9294Compatibility"; + runTest9294CompatibilityTest(tableName, TEST_UTIL.getConfiguration()); + + // with useMulti + tableName = "test9294CompatibilityWithMulti"; + Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); + conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, true); + runTest9294CompatibilityTest(tableName, conf); + } + + /** + * RecoverableZookeeper that throws a KeeperException after throwExceptionInNumOperations + */ + class ThrowingRecoverableZookeeper extends RecoverableZooKeeper { + private ZooKeeperWatcher zkw; + private int throwExceptionInNumOperations; + + public ThrowingRecoverableZookeeper(ZooKeeperWatcher zkw) throws Exception { + super(ZKConfig.getZKQuorumServersString(TEST_UTIL.getConfiguration()), + HConstants.DEFAULT_ZK_SESSION_TIMEOUT, zkw, 3, 1000); + this.zkw = zkw; + this.throwExceptionInNumOperations = 0; // indicate not to throw an exception + } + + public void setThrowExceptionInNumOperations(int throwExceptionInNumOperations) { + this.throwExceptionInNumOperations = throwExceptionInNumOperations; + } + + private void checkThrowKeeperException() throws KeeperException { + if (throwExceptionInNumOperations == 1) { + throwExceptionInNumOperations = 0; + throw new KeeperException.DataInconsistencyException(); + } + if(throwExceptionInNumOperations > 0) throwExceptionInNumOperations--; + } + + public Stat setData(String path, byte[] data, int version) + throws KeeperException, InterruptedException { + checkThrowKeeperException(); + return zkw.getRecoverableZooKeeper().setData(path, data, version); + } + + public void delete(String path, int version) + throws InterruptedException, KeeperException { + checkThrowKeeperException(); + zkw.getRecoverableZooKeeper().delete(path, version); + } + } + /** + * Because two ZooKeeper nodes are written for each table state transition + * {@link ZooKeeperWatcher#masterTableZNode} and {@link ZooKeeperWatcher#masterTableZNode92} + * it is possible that we fail in between the two operations and are left with + * inconsistent state (when hbase.zookeeper.useMulti is false). + * Check that we can get back to a consistent state by retrying the operation. + */ + @Test + public void testDisableTableRetry() throws Exception { + final String tableName = "testDisableTableRetry"; + + Configuration conf = TEST_UTIL.getConfiguration(); + // test only relevant if useMulti is false + conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, false); + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, + tableName, abortable, true); + ThrowingRecoverableZookeeper throwing = new ThrowingRecoverableZookeeper(zkw); + ZooKeeperWatcher spyZookeeperWatcher = Mockito.spy(zkw); + Mockito.doReturn(throwing).when(spyZookeeperWatcher).getRecoverableZooKeeper(); + + ZKTable zkt = new ZKTable(spyZookeeperWatcher); + zkt.setEnabledTable(tableName); + assertTrue(zkt.isEnabledOrDisablingTable(tableName)); + boolean caughtExpectedException = false; + try { + // throw an exception on the second ZK operation, which means the first will succeed. + throwing.setThrowExceptionInNumOperations(2); + zkt.setDisabledTable(tableName); + } catch (KeeperException ke) { + caughtExpectedException = true; + } + assertTrue(caughtExpectedException); + assertFalse(zkt.isDisabledTable(tableName)); + // try again, ensure table is disabled + zkt.setDisabledTable(tableName); + // ensure disabled from master perspective + assertTrue(zkt.isDisabledTable(tableName)); + // ensure disabled from client perspective + assertTrue(ZKTableReadOnly.isDisabledTable(zkw, tableName)); + } + + /** + * Same as above, but with enableTable + */ + @Test + public void testEnableTableRetry() throws Exception { + final String tableName = "testEnableTableRetry"; + + Configuration conf = TEST_UTIL.getConfiguration(); + // test only relevant if useMulti is false + conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, false); + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, + tableName, abortable, true); + ThrowingRecoverableZookeeper throwing = new ThrowingRecoverableZookeeper(zkw); + ZooKeeperWatcher spyZookeeperWatcher = Mockito.spy(zkw); + Mockito.doReturn(throwing).when(spyZookeeperWatcher).getRecoverableZooKeeper(); + + ZKTable zkt = new ZKTable(spyZookeeperWatcher); + zkt.setDisabledTable(tableName); + assertTrue(zkt.isDisabledTable(tableName)); + boolean caughtExpectedException = false; + try { + // throw an exception on the second ZK operation, which means the first will succeed. + throwing.throwExceptionInNumOperations = 2; + zkt.setEnabledTable(tableName); + } catch (KeeperException ke) { + caughtExpectedException = true; + } + assertTrue(caughtExpectedException); + assertFalse(zkt.isEnabledTable(tableName)); + // try again, ensure table is enabled + zkt.setEnabledTable(tableName); + // ensure enabled from master perspective + assertTrue(zkt.isEnabledTable(tableName)); + // ensure enabled from client perspective + assertTrue(ZKTableReadOnly.isEnabledTable(zkw, tableName)); + } + + /** + * Same as above, but with deleteTable + */ + @Test + public void testDeleteTableRetry() throws Exception { + final String tableName = "testEnableTableRetry"; + + Configuration conf = TEST_UTIL.getConfiguration(); + // test only relevant if useMulti is false + conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, false); + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, + tableName, abortable, true); + ThrowingRecoverableZookeeper throwing = new ThrowingRecoverableZookeeper(zkw); + ZooKeeperWatcher spyZookeeperWatcher = Mockito.spy(zkw); + Mockito.doReturn(throwing).when(spyZookeeperWatcher).getRecoverableZooKeeper(); + + ZKTable zkt = new ZKTable(spyZookeeperWatcher); + zkt.setDisabledTable(tableName); + assertTrue(zkt.isDisabledTable(tableName)); + boolean caughtExpectedException = false; + try { + // throw an exception on the second ZK operation, which means the first will succeed. + throwing.setThrowExceptionInNumOperations(2); + zkt.setDeletedTable(tableName); + } catch (KeeperException ke) { + caughtExpectedException = true; + } + assertTrue(caughtExpectedException); + assertTrue(zkt.isTablePresent(tableName)); + // try again, ensure table is deleted + zkt.setDeletedTable(tableName); + // ensure deleted from master perspective + assertFalse(zkt.isTablePresent(tableName)); + // ensure deleted from client perspective + assertFalse(ZKTableReadOnly.getDisabledTables(zkw).contains(tableName)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTableReadOnly.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTableReadOnly.java new file mode 100644 index 0000000..36baf6c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKTableReadOnly.java @@ -0,0 +1,123 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestZKTableReadOnly { + private static final Log LOG = LogFactory.getLog(TestZooKeeperNodeTracker.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + Abortable abortable = new Abortable() { + @Override + public void abort(String why, Throwable e) { + LOG.info(why, e); + } + + @Override + public boolean isAborted() { + return false; + } + }; + + private boolean enableAndCheckEnabled(ZooKeeperWatcher zkw, String tableName) throws Exception { + // set the table to enabled, as that is the only state that differs + // between the two formats + ZKTable zkt = new ZKTable(zkw); + zkt.setEnabledTable(tableName); + return ZKTableReadOnly.isEnabledTable(zkw, tableName); + } + + private void runClientCompatiblityWith92ZNodeTest(String tableName, Configuration conf) + throws Exception { + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, + tableName, abortable, true); + assertTrue(enableAndCheckEnabled(zkw, tableName)); + } + /** + * Test that client ZK reader can handle the 0.92 table format znode. + */ + @Test + public void testClientCompatibilityWith92ZNode() throws Exception { + // test without useMulti + String tableName = "testClientCompatibilityWith92ZNode"; + // Set the client to read from the 0.92 table znode format + Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); + String znode92 = conf.get("zookeeper.znode.masterTableEnableDisable92", "table92"); + conf.set("zookeeper.znode.clientTableEnableDisable", znode92); + runClientCompatiblityWith92ZNodeTest(tableName, conf); + + // test with useMulti + tableName = "testClientCompatibilityWith92ZNodeUseMulti"; + conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, true); + runClientCompatiblityWith92ZNodeTest(tableName, conf); + } + + private void runClientCompatibilityWith94ZNodeTest(String tableName, Configuration conf) + throws Exception { + ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + tableName, abortable, true); + assertTrue(enableAndCheckEnabled(zkw, tableName)); + } + + /** + * Test that client ZK reader can handle the current (0.94) table format znode. + */ + @Test + public void testClientCompatibilityWith94ZNode() throws Exception { + String tableName = "testClientCompatibilityWith94ZNode"; + + // without useMulti + runClientCompatibilityWith94ZNodeTest(tableName, TEST_UTIL.getConfiguration()); + + // with useMulti + tableName = "testClientCompatiblityWith94ZNodeUseMulti"; + Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); + conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, true); + runClientCompatibilityWith94ZNodeTest(tableName, conf); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java new file mode 100644 index 0000000..fba1caf --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java @@ -0,0 +1,271 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestZooKeeperACL { + private final static Log LOG = LogFactory.getLog(TestZooKeeperACL.class); + private final static HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + private static ZooKeeperWatcher zkw; + private static boolean secureZKAvailable; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + File saslConfFile = File.createTempFile("tmp", "jaas.conf"); + FileWriter fwriter = new FileWriter(saslConfFile); + + fwriter.write("" + + "Server {\n" + + "org.apache.zookeeper.server.auth.DigestLoginModule required\n" + + "user_hbase=\"secret\";\n" + + "};\n" + + "Client {\n" + + "org.apache.zookeeper.server.auth.DigestLoginModule required\n" + + "username=\"hbase\"\n" + + "password=\"secret\";\n" + + "};" + "\n"); + fwriter.close(); + System.setProperty("java.security.auth.login.config", + saslConfFile.getAbsolutePath()); + System.setProperty("zookeeper.authProvider.1", + "org.apache.zookeeper.server.auth.SASLAuthenticationProvider"); + + TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); + TEST_UTIL.getConfiguration().setInt("hbase.zookeeper.property.maxClientCnxns", 1000); + + // If Hadoop is missing HADOOP-7070 the cluster will fail to start due to + // the JAAS configuration required by ZK being clobbered by Hadoop + try { + TEST_UTIL.startMiniCluster(); + } catch (IOException e) { + LOG.warn("Hadoop is missing HADOOP-7070", e); + secureZKAvailable = false; + return; + } + zkw = new ZooKeeperWatcher( + new Configuration(TEST_UTIL.getConfiguration()), + TestZooKeeper.class.getName(), null); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + if (!secureZKAvailable) { + return; + } + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + if (!secureZKAvailable) { + return; + } + TEST_UTIL.ensureSomeRegionServersAvailable(2); + } + + /** + * Create a node and check its ACL. When authentication is enabled on + * Zookeeper, all nodes (except /hbase/root-region-server, /hbase/master + * and /hbase/hbaseid) should be created so that only the hbase server user + * (master or region server user) that created them can access them, and + * this user should have all permissions on this node. For + * /hbase/root-region-server, /hbase/master, and /hbase/hbaseid the + * permissions should be as above, but should also be world-readable. First + * we check the general case of /hbase nodes in the following test, and + * then check the subset of world-readable nodes in the three tests after + * that. + */ + @Test (timeout=30000) + public void testHBaseRootZNodeACL() throws Exception { + if (!secureZKAvailable) { + return; + } + + List acls = zkw.getRecoverableZooKeeper().getZooKeeper() + .getACL("/hbase", new Stat()); + assertEquals(acls.size(),1); + assertEquals(acls.get(0).getId().getScheme(),"sasl"); + assertEquals(acls.get(0).getId().getId(),"hbase"); + assertEquals(acls.get(0).getPerms(), ZooDefs.Perms.ALL); + } + + /** + * When authentication is enabled on Zookeeper, /hbase/root-region-server + * should be created with 2 ACLs: one specifies that the hbase user has + * full access to the node; the other, that it is world-readable. + */ + @Test (timeout=30000) + public void testHBaseRootRegionServerZNodeACL() throws Exception { + if (!secureZKAvailable) { + return; + } + + List acls = zkw.getRecoverableZooKeeper().getZooKeeper() + .getACL("/hbase/root-region-server", new Stat()); + assertEquals(acls.size(),2); + + boolean foundWorldReadableAcl = false; + boolean foundHBaseOwnerAcl = false; + for(int i = 0; i < 2; i++) { + if (acls.get(i).getId().getScheme().equals("world") == true) { + assertEquals(acls.get(0).getId().getId(),"anyone"); + assertEquals(acls.get(0).getPerms(), ZooDefs.Perms.READ); + foundWorldReadableAcl = true; + } + else { + if (acls.get(i).getId().getScheme().equals("sasl") == true) { + assertEquals(acls.get(1).getId().getId(),"hbase"); + assertEquals(acls.get(1).getId().getScheme(),"sasl"); + foundHBaseOwnerAcl = true; + } else { // error: should not get here: test fails. + assertTrue(false); + } + } + } + assertTrue(foundWorldReadableAcl); + assertTrue(foundHBaseOwnerAcl); + } + + /** + * When authentication is enabled on Zookeeper, /hbase/master should be + * created with 2 ACLs: one specifies that the hbase user has full access + * to the node; the other, that it is world-readable. + */ + @Test (timeout=30000) + public void testHBaseMasterServerZNodeACL() throws Exception { + if (!secureZKAvailable) { + return; + } + + List acls = zkw.getRecoverableZooKeeper().getZooKeeper() + .getACL("/hbase/master", new Stat()); + assertEquals(acls.size(),2); + + boolean foundWorldReadableAcl = false; + boolean foundHBaseOwnerAcl = false; + for(int i = 0; i < 2; i++) { + if (acls.get(i).getId().getScheme().equals("world") == true) { + assertEquals(acls.get(0).getId().getId(),"anyone"); + assertEquals(acls.get(0).getPerms(), ZooDefs.Perms.READ); + foundWorldReadableAcl = true; + } else { + if (acls.get(i).getId().getScheme().equals("sasl") == true) { + assertEquals(acls.get(1).getId().getId(),"hbase"); + assertEquals(acls.get(1).getId().getScheme(),"sasl"); + foundHBaseOwnerAcl = true; + } else { // error: should not get here: test fails. + assertTrue(false); + } + } + } + assertTrue(foundWorldReadableAcl); + assertTrue(foundHBaseOwnerAcl); + } + + /** + * When authentication is enabled on Zookeeper, /hbase/hbaseid should be + * created with 2 ACLs: one specifies that the hbase user has full access + * to the node; the other, that it is world-readable. + */ + @Test (timeout=30000) + public void testHBaseIDZNodeACL() throws Exception { + if (!secureZKAvailable) { + return; + } + + List acls = zkw.getRecoverableZooKeeper().getZooKeeper() + .getACL("/hbase/hbaseid", new Stat()); + assertEquals(acls.size(),2); + + boolean foundWorldReadableAcl = false; + boolean foundHBaseOwnerAcl = false; + for(int i = 0; i < 2; i++) { + if (acls.get(i).getId().getScheme().equals("world") == true) { + assertEquals(acls.get(0).getId().getId(),"anyone"); + assertEquals(acls.get(0).getPerms(), ZooDefs.Perms.READ); + foundWorldReadableAcl = true; + } else { + if (acls.get(i).getId().getScheme().equals("sasl") == true) { + assertEquals(acls.get(1).getId().getId(),"hbase"); + assertEquals(acls.get(1).getId().getScheme(),"sasl"); + foundHBaseOwnerAcl = true; + } else { // error: should not get here: test fails. + assertTrue(false); + } + } + } + assertTrue(foundWorldReadableAcl); + assertTrue(foundHBaseOwnerAcl); + } + + /** + * Finally, we check the ACLs of a node outside of the /hbase hierarchy and + * verify that its ACL is simply 'hbase:Perms.ALL'. + */ + @Test + public void testOutsideHBaseNodeACL() throws Exception { + if (!secureZKAvailable) { + return; + } + + ZKUtil.createWithParents(zkw, "/testACLNode"); + List acls = zkw.getRecoverableZooKeeper().getZooKeeper() + .getACL("/testACLNode", new Stat()); + assertEquals(acls.size(),1); + assertEquals(acls.get(0).getId().getScheme(),"sasl"); + assertEquals(acls.get(0).getId().getId(),"hbase"); + assertEquals(acls.get(0).getPerms(), ZooDefs.Perms.ALL); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperMainServerArg.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperMainServerArg.java new file mode 100644 index 0000000..ddc8dc4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperMainServerArg.java @@ -0,0 +1,51 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestZooKeeperMainServerArg { + private final ZooKeeperMainServerArg parser = new ZooKeeperMainServerArg(); + + @Test public void test() { + Configuration c = HBaseConfiguration.create(); + assertEquals("localhost:" + c.get(HConstants.ZOOKEEPER_CLIENT_PORT), + parser.parse(c)); + final String port = "1234"; + c.set(HConstants.ZOOKEEPER_CLIENT_PORT, port); + c.set("hbase.zookeeper.quorum", "example.com"); + assertEquals("example.com:" + port, parser.parse(c)); + c.set("hbase.zookeeper.quorum", "example1.com,example2.com,example3.com"); + assertTrue(port, + parser.parse(c).matches("(example[1-3]\\.com,){2}example[1-3]\\.com:" + port)); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperNodeTracker.java b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperNodeTracker.java new file mode 100644 index 0000000..3707389 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperNodeTracker.java @@ -0,0 +1,320 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Random; +import java.util.concurrent.Semaphore; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.master.TestActiveMasterManager.NodeDeletionListener; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooKeeper; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestZooKeeperNodeTracker { + private static final Log LOG = LogFactory.getLog(TestZooKeeperNodeTracker.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private final static Random rand = new Random(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + /** + * Test that we can interrupt a node that is blocked on a wait. + * @throws IOException + * @throws InterruptedException + */ + @Test public void testInterruptible() throws IOException, InterruptedException { + Abortable abortable = new StubAbortable(); + ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + "testInterruptible", abortable); + final TestTracker tracker = new TestTracker(zk, "/xyz", abortable); + tracker.start(); + Thread t = new Thread() { + @Override + public void run() { + try { + tracker.blockUntilAvailable(); + } catch (InterruptedException e) { + throw new RuntimeException("Interrupted", e); + } + } + }; + t.start(); + while (!t.isAlive()) Threads.sleep(1); + tracker.stop(); + t.join(); + // If it wasn't interruptible, we'd never get to here. + } + + @Test + public void testNodeTracker() throws Exception { + Abortable abortable = new StubAbortable(); + ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), + "testNodeTracker", abortable); + ZKUtil.createAndFailSilent(zk, zk.baseZNode); + + final String node = + ZKUtil.joinZNode(zk.baseZNode, new Long(rand.nextLong()).toString()); + + final byte [] dataOne = Bytes.toBytes("dataOne"); + final byte [] dataTwo = Bytes.toBytes("dataTwo"); + + // Start a ZKNT with no node currently available + TestTracker localTracker = new TestTracker(zk, node, abortable); + localTracker.start(); + zk.registerListener(localTracker); + + // Make sure we don't have a node + assertNull(localTracker.getData(false)); + + // Spin up a thread with another ZKNT and have it block + WaitToGetDataThread thread = new WaitToGetDataThread(zk, node); + thread.start(); + + // Verify the thread doesn't have a node + assertFalse(thread.hasData); + + // Now, start a new ZKNT with the node already available + TestTracker secondTracker = new TestTracker(zk, node, null); + secondTracker.start(); + zk.registerListener(secondTracker); + + // Put up an additional zk listener so we know when zk event is done + TestingZKListener zkListener = new TestingZKListener(zk, node); + zk.registerListener(zkListener); + assertEquals(0, zkListener.createdLock.availablePermits()); + + // Create a completely separate zk connection for test triggers and avoid + // any weird watcher interactions from the test + final ZooKeeper zkconn = new ZooKeeper( + ZKConfig.getZKQuorumServersString(TEST_UTIL.getConfiguration()), 60000, + new StubWatcher()); + + // Add the node with data one + zkconn.create(node, dataOne, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + + // Wait for the zk event to be processed + zkListener.waitForCreation(); + thread.join(); + + // Both trackers should have the node available with data one + assertNotNull(localTracker.getData(false)); + assertNotNull(localTracker.blockUntilAvailable()); + assertTrue(Bytes.equals(localTracker.getData(false), dataOne)); + assertTrue(thread.hasData); + assertTrue(Bytes.equals(thread.tracker.getData(false), dataOne)); + LOG.info("Successfully got data one"); + + // Make sure it's available and with the expected data + assertNotNull(secondTracker.getData(false)); + assertNotNull(secondTracker.blockUntilAvailable()); + assertTrue(Bytes.equals(secondTracker.getData(false), dataOne)); + LOG.info("Successfully got data one with the second tracker"); + + // Drop the node + zkconn.delete(node, -1); + zkListener.waitForDeletion(); + + // Create a new thread but with the existing thread's tracker to wait + TestTracker threadTracker = thread.tracker; + thread = new WaitToGetDataThread(zk, node, threadTracker); + thread.start(); + + // Verify other guys don't have data + assertFalse(thread.hasData); + assertNull(secondTracker.getData(false)); + assertNull(localTracker.getData(false)); + LOG.info("Successfully made unavailable"); + + // Create with second data + zkconn.create(node, dataTwo, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + + // Wait for the zk event to be processed + zkListener.waitForCreation(); + thread.join(); + + // All trackers should have the node available with data two + assertNotNull(localTracker.getData(false)); + assertNotNull(localTracker.blockUntilAvailable()); + assertTrue(Bytes.equals(localTracker.getData(false), dataTwo)); + assertNotNull(secondTracker.getData(false)); + assertNotNull(secondTracker.blockUntilAvailable()); + assertTrue(Bytes.equals(secondTracker.getData(false), dataTwo)); + assertTrue(thread.hasData); + assertTrue(Bytes.equals(thread.tracker.getData(false), dataTwo)); + LOG.info("Successfully got data two on all trackers and threads"); + + // Change the data back to data one + zkconn.setData(node, dataOne, -1); + + // Wait for zk event to be processed + zkListener.waitForDataChange(); + + // All trackers should have the node available with data one + assertNotNull(localTracker.getData(false)); + assertNotNull(localTracker.blockUntilAvailable()); + assertTrue(Bytes.equals(localTracker.getData(false), dataOne)); + assertNotNull(secondTracker.getData(false)); + assertNotNull(secondTracker.blockUntilAvailable()); + assertTrue(Bytes.equals(secondTracker.getData(false), dataOne)); + assertTrue(thread.hasData); + assertTrue(Bytes.equals(thread.tracker.getData(false), dataOne)); + LOG.info("Successfully got data one following a data change on all trackers and threads"); + } + + public static class WaitToGetDataThread extends Thread { + + TestTracker tracker; + boolean hasData; + + public WaitToGetDataThread(ZooKeeperWatcher zk, String node) { + tracker = new TestTracker(zk, node, null); + tracker.start(); + zk.registerListener(tracker); + hasData = false; + } + + public WaitToGetDataThread(ZooKeeperWatcher zk, String node, + TestTracker tracker) { + this.tracker = tracker; + hasData = false; + } + + @Override + public void run() { + LOG.info("Waiting for data to be available in WaitToGetDataThread"); + try { + tracker.blockUntilAvailable(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + LOG.info("Data now available in tracker from WaitToGetDataThread"); + hasData = true; + } + } + + public static class TestTracker extends ZooKeeperNodeTracker { + public TestTracker(ZooKeeperWatcher watcher, String node, + Abortable abortable) { + super(watcher, node, abortable); + } + } + + public static class TestingZKListener extends ZooKeeperListener { + private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class); + + private Semaphore deletedLock; + private Semaphore createdLock; + private Semaphore changedLock; + private String node; + + public TestingZKListener(ZooKeeperWatcher watcher, String node) { + super(watcher); + deletedLock = new Semaphore(0); + createdLock = new Semaphore(0); + changedLock = new Semaphore(0); + this.node = node; + } + + @Override + public void nodeDeleted(String path) { + if(path.equals(node)) { + LOG.debug("nodeDeleted(" + path + ")"); + deletedLock.release(); + } + } + + @Override + public void nodeCreated(String path) { + if(path.equals(node)) { + LOG.debug("nodeCreated(" + path + ")"); + createdLock.release(); + } + } + + @Override + public void nodeDataChanged(String path) { + if(path.equals(node)) { + LOG.debug("nodeDataChanged(" + path + ")"); + changedLock.release(); + } + } + + public void waitForDeletion() throws InterruptedException { + deletedLock.acquire(); + } + + public void waitForCreation() throws InterruptedException { + createdLock.acquire(); + } + + public void waitForDataChange() throws InterruptedException { + changedLock.acquire(); + } + } + + public static class StubAbortable implements Abortable { + @Override + public void abort(final String msg, final Throwable t) {} + + @Override + public boolean isAborted() { + return false; + } + + } + + public static class StubWatcher implements Watcher { + @Override + public void process(WatchedEvent event) {} + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); +} + diff --git a/src/test/resources/hbase-site.xml b/src/test/resources/hbase-site.xml new file mode 100644 index 0000000..b9a4468 --- /dev/null +++ b/src/test/resources/hbase-site.xml @@ -0,0 +1,137 @@ + + + + + + hbase.regionserver.msginterval + 1000 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.client.pause + 1000 + General client pause value. Used mostly as value to wait + before running a retry of a failed get, region lookup, etc. + + + hbase.client.retries.number + 10 + Maximum retries. Used as maximum for all retryable + operations such as fetching of the root region from root region + server, getting a cell's value, starting a row update, etc. + Default: 10. + + + + hbase.server.thread.wakefrequency + 1000 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as META scanner and log roller. + + + + hbase.master.event.waiting.time + 50 + Time to sleep between checks to see if a table event took place. + + + + hbase.regionserver.handler.count + 5 + Count of RPC Server instances spun up on RegionServers + Same property is used by the HMaster for count of master handlers. + Default is 10. + + + + hbase.master.info.port + -1 + The port for the hbase master web UI + Set to -1 if you do not want the info server to run. + + + + hbase.regionserver.info.port + -1 + The port for the hbase regionserver web UI + Set to -1 if you do not want the info server to run. + + + + hbase.regionserver.info.port.auto + true + Info server auto port bind. Enables automatic port + search if hbase.regionserver.info.port is already in use. + Enabled for testing to run multiple tests on one machine. + + + + hbase.master.lease.thread.wakefrequency + 3000 + The interval between checks for expired region server leases. + This value has been reduced due to the other reduced values above so that + the master will notice a dead region server sooner. The default is 15 seconds. + + + + hbase.regionserver.safemode + false + + Turn on/off safe mode in region server. Always on for production, always off + for tests. + + + + hbase.hregion.max.filesize + 67108864 + + Maximum desired file size for an HRegion. If filesize exceeds + value + (value / 2), the HRegion is split in two. Default: 256M. + + Keep the maximum filesize small so we split more often in tests. + + + + hbase.zookeeper.property.clientPort + 21818 + Property from ZooKeeper's config zoo.cfg. + The port at which the clients will connect. + + + + hbase.defaults.for.version.skip + true + + Set to true to skip the 'hbase.defaults.for.version'. + Setting this to true can be useful in contexts other than + the other side of a maven generation; i.e. running in an + ide. You'll want to set this boolean to true to avoid + seeing the RuntimException complaint: "hbase-default.xml file + seems to be for and old version of HBase (@@@VERSION@@@), this + version is X.X.X-SNAPSHOT" + + + diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties new file mode 100644 index 0000000..2ca347d --- /dev/null +++ b/src/test/resources/log4j.properties @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshhold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# Debugging Pattern format +log4j.appender.DRFA.layout.ConversionPattern=%d %-5p [%t] %C{2}(%L): %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d %-5p [%t] %C{2}(%L): %m%n + +# Custom Logging levels + +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG + +log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.apache.zookeeper=ERROR +log4j.logger.org.apache.hadoop.hbase=DEBUG diff --git a/src/test/resources/mapred-queues.xml b/src/test/resources/mapred-queues.xml new file mode 100644 index 0000000..43f3e2a --- /dev/null +++ b/src/test/resources/mapred-queues.xml @@ -0,0 +1,75 @@ + + + + + + + + + + default + + + + + + + running + + + * + + + * + + + + diff --git a/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties b/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties new file mode 100644 index 0000000..28493ff --- /dev/null +++ b/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties @@ -0,0 +1,30 @@ +# ResourceBundle properties file for Map-Reduce counters + +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +CounterGroupName= HBase Performance Evaluation +ELAPSED_TIME.name= Elapsed time in milliseconds +ROWS.name= Row count +# ResourceBundle properties file for Map-Reduce counters + +CounterGroupName= HBase Performance Evaluation +ELAPSED_TIME.name= Elapsed time in milliseconds +ROWS.name= Row count diff --git a/src/test/resources/org/apache/hadoop/hbase/io/hfile/8e8ab58dcf39412da19833fcd8f687ac b/src/test/resources/org/apache/hadoop/hbase/io/hfile/8e8ab58dcf39412da19833fcd8f687ac new file mode 100644 index 0000000..cc260fb Binary files /dev/null and b/src/test/resources/org/apache/hadoop/hbase/io/hfile/8e8ab58dcf39412da19833fcd8f687ac differ diff --git a/src/test/ruby/hbase/admin_test.rb b/src/test/ruby/hbase/admin_test.rb new file mode 100644 index 0000000..97bf443 --- /dev/null +++ b/src/test/ruby/hbase/admin_test.rb @@ -0,0 +1,303 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'hbase' + +include HBaseConstants + +module Hbase + class AdminHelpersTest < Test::Unit::TestCase + include TestHelpers + + def setup + setup_hbase + # Create test table if it does not exist + @test_name = "hbase_shell_tests_table" + create_test_table(@test_name) + end + + define_test "exists? should return true when a table exists" do + assert(admin.exists?('.META.')) + end + + define_test "exists? should return false when a table exists" do + assert(!admin.exists?('.NOT.EXISTS.')) + end + + define_test "enabled? should return true for enabled tables" do + admin.enable(@test_name) + assert(admin.enabled?(@test_name)) + end + + define_test "enabled? should return false for disabled tables" do + admin.disable(@test_name) + assert(!admin.enabled?(@test_name)) + end + end + + # Simple administration methods tests + class AdminMethodsTest < Test::Unit::TestCase + include TestHelpers + + def setup + setup_hbase + # Create test table if it does not exist + @test_name = "hbase_shell_tests_table" + create_test_table(@test_name) + + # Create table test table name + @create_test_name = 'hbase_create_table_test_table' + end + + define_test "list should return a list of tables" do + assert(admin.list.member?(@test_name)) + end + + define_test "list should not return meta tables" do + assert(!admin.list.member?('.META.')) + assert(!admin.list.member?('-ROOT-')) + end + + #------------------------------------------------------------------------------- + + define_test "flush should work" do + admin.flush('.META.') + end + + #------------------------------------------------------------------------------- + + define_test "compact should work" do + admin.compact('.META.') + end + + #------------------------------------------------------------------------------- + + define_test "major_compact should work" do + admin.major_compact('.META.') + end + + #------------------------------------------------------------------------------- + + define_test "split should work" do + admin.split('.META.', nil) + end + + #------------------------------------------------------------------------------- + + define_test "drop should fail on non-existent tables" do + assert_raise(ArgumentError) do + admin.drop('.NOT.EXISTS.') + end + end + + define_test "drop should fail on enabled tables" do + assert_raise(ArgumentError) do + admin.drop(@test_name) + end + end + + define_test "drop should drop tables" do + admin.disable(@test_name) + admin.drop(@test_name) + assert(!admin.exists?(@test_name)) + end + + #------------------------------------------------------------------------------- + + define_test "zk_dump should work" do + assert_not_nil(admin.zk_dump) + end + + #------------------------------------------------------------------------------- + + define_test "create should fail with non-string table names" do + assert_raise(ArgumentError) do + admin.create(123, 'xxx') + end + end + + define_test "create should fail with non-string/non-hash column args" do + assert_raise(ArgumentError) do + admin.create(@create_test_name, 123) + end + end + + define_test "create should fail without columns" do + drop_test_table(@create_test_name) + assert_raise(ArgumentError) do + admin.create(@create_test_name) + end + end + + define_test "create should work with string column args" do + drop_test_table(@create_test_name) + admin.create(@create_test_name, 'a', 'b') + assert_equal(['a:', 'b:'], table(@create_test_name).get_all_columns.sort) + end + + define_test "create should work with hash column args" do + drop_test_table(@create_test_name) + admin.create(@create_test_name, { NAME => 'a'}, { NAME => 'b'}) + assert_equal(['a:', 'b:'], table(@create_test_name).get_all_columns.sort) + end + + #------------------------------------------------------------------------------- + + define_test "describe should fail for non-existent tables" do + assert_raise(ArgumentError) do + admin.describe('.NOT.EXISTS.') + end + end + + define_test "describe should return a description" do + assert_not_nil admin.describe(@test_name) + end + + #------------------------------------------------------------------------------- + + define_test "truncate should empty a table" do + table(@test_name).put(1, "x:a", 1) + table(@test_name).put(2, "x:a", 2) + assert_equal(2, table(@test_name).count) + # This is hacky. Need to get the configuration into admin instance + admin.truncate(@test_name, $TEST_CLUSTER.getConfiguration) + assert_equal(0, table(@test_name).count) + end + + define_test "truncate should yield log records" do + logs = [] + admin.truncate(@test_name, $TEST_CLUSTER.getConfiguration) do |log| + assert_kind_of(String, log) + logs << log + end + assert(!logs.empty?) + end + end + + # Simple administration methods tests + class AdminAlterTableTest < Test::Unit::TestCase + include TestHelpers + + def setup + setup_hbase + # Create test table if it does not exist + @test_name = "hbase_shell_tests_table" + drop_test_table(@test_name) + create_test_table(@test_name) + end + + #------------------------------------------------------------------------------- + + define_test "alter should fail with non-string table names" do + assert_raise(ArgumentError) do + admin.alter(123, true, METHOD => 'delete', NAME => 'y') + end + end + + define_test "alter should fail with non-existing tables" do + assert_raise(ArgumentError) do + admin.alter('.NOT.EXISTS.', true, METHOD => 'delete', NAME => 'y') + end + end + + define_test "alter should not fail with enabled tables" do + admin.enable(@test_name) + admin.alter(@test_name, true, METHOD => 'delete', NAME => 'y') + end + + define_test "alter should be able to delete column families" do + assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) + admin.alter(@test_name, true, METHOD => 'delete', NAME => 'y') + admin.enable(@test_name) + assert_equal(['x:'], table(@test_name).get_all_columns.sort) + end + + define_test "alter should be able to add column families" do + assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) + admin.alter(@test_name, true, NAME => 'z') + admin.enable(@test_name) + assert_equal(['x:', 'y:', 'z:'], table(@test_name).get_all_columns.sort) + end + + define_test "alter should be able to add column families (name-only alter spec)" do + assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) + admin.alter(@test_name, true, 'z') + admin.enable(@test_name) + assert_equal(['x:', 'y:', 'z:'], table(@test_name).get_all_columns.sort) + end + + define_test "alter should support more than one alteration in one call" do + assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) + admin.alter(@test_name, true, { NAME => 'z' }, { METHOD => 'delete', NAME => 'y' }) + admin.enable(@test_name) + assert_equal(['x:', 'z:'], table(@test_name).get_all_columns.sort) + end + + define_test 'alter should support shortcut DELETE alter specs' do + assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) + admin.alter(@test_name, true, 'delete' => 'y') + assert_equal(['x:'], table(@test_name).get_all_columns.sort) + end + + define_test "alter should be able to change table options" do + admin.alter(@test_name, true, METHOD => 'table_att', 'MAX_FILESIZE' => 12345678) + assert_match(/12345678/, admin.describe(@test_name)) + end + + define_test "alter should be able to change coprocessor attributes" do + drop_test_table(@test_name) + create_test_table(@test_name) + + cp_key = "coprocessor" + class_name = "SimpleRegionObserver" + + cp_value = "hdfs:///foo.jar|" + class_name + "|12|arg1=1,arg2=2" + + # eval() is used to convert a string to regex + assert_no_match(eval("/" + class_name + "/"), admin.describe(@test_name)) + assert_no_match(eval("/" + cp_key + "/"), admin.describe(@test_name)) + admin.alter(@test_name, true, 'METHOD' => 'table_att', cp_key => cp_value) + assert_match(eval("/" + class_name + "/"), admin.describe(@test_name)) + assert_match(eval("/" + cp_key + "\\$(\\d+)/"), admin.describe(@test_name)) + end + + define_test "alter should be able to remove a table attribute" do + drop_test_table(@test_name) + create_test_table(@test_name) + + key1 = "coprocessor" + key2 = "MAX_FILESIZE" + admin.alter(@test_name, true, 'METHOD' => 'table_att', key1 => "|TestCP||") + admin.alter(@test_name, true, 'METHOD' => 'table_att', key2 => 12345678) + + # eval() is used to convert a string to regex + assert_match(eval("/" + key1 + "\\$(\\d+)/"), admin.describe(@test_name)) + assert_match(eval("/" + key2 + "/"), admin.describe(@test_name)) + + # get the cp key + cp_keys = admin.describe(@test_name).scan(/(coprocessor\$\d+)/i) + + admin.alter(@test_name, true, 'METHOD' => 'table_att_unset', 'NAME' => cp_keys[0][0]) + admin.alter(@test_name, true, 'METHOD' => 'table_att_unset', 'NAME' => key2) + assert_no_match(eval("/" + key1 + "\\$(\\d+)/"), admin.describe(@test_name)) + assert_no_match(eval("/" + key2 + "/"), admin.describe(@test_name)) + end + end +end diff --git a/src/test/ruby/hbase/hbase_test.rb b/src/test/ruby/hbase/hbase_test.rb new file mode 100644 index 0000000..4e3fae3 --- /dev/null +++ b/src/test/ruby/hbase/hbase_test.rb @@ -0,0 +1,50 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'hbase' + +module Hbase + class HbaseTest < Test::Unit::TestCase + def setup + @formatter = Shell::Formatter::Console.new() + @hbase = ::Hbase::Hbase.new($TEST_CLUSTER.getConfiguration) + end + + define_test "Hbase::Hbase constructor should initialize hbase configuration object" do + assert_kind_of(org.apache.hadoop.conf.Configuration, @hbase.configuration) + end + + define_test "Hbase::Hbase#admin should create a new admin object when called the first time" do + assert_kind_of(::Hbase::Admin, @hbase.admin(@formatter)) + end + + define_test "Hbase::Hbase#admin should create a new admin object every call" do + assert_not_same(@hbase.admin(@formatter), @hbase.admin(@formatter)) + end + + define_test "Hbase::Hbase#table should create a new table object when called the first time" do + assert_kind_of(::Hbase::Table, @hbase.table('.META.', @formatter)) + end + + define_test "Hbase::Hbase#table should create a new table object every call" do + assert_not_same(@hbase.table('.META.', @formatter), @hbase.table('.META.', @formatter)) + end + end +end diff --git a/src/test/ruby/hbase/table_test.rb b/src/test/ruby/hbase/table_test.rb new file mode 100644 index 0000000..067058d --- /dev/null +++ b/src/test/ruby/hbase/table_test.rb @@ -0,0 +1,483 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'hbase' + +include HBaseConstants + +module Hbase + # Constructor tests + class TableConstructorTest < Test::Unit::TestCase + include TestHelpers + def setup + setup_hbase + end + + define_test "Hbase::Table constructor should fail for non-existent tables" do + assert_raise(NativeException) do + table('non-existent-table-name') + end + end + + define_test "Hbase::Table constructor should not fail for existent tables" do + assert_nothing_raised do + table('.META.') + end + end + end + + # Helper methods tests + class TableHelpersTest < Test::Unit::TestCase + include TestHelpers + + def setup + setup_hbase + # Create test table if it does not exist + @test_name = "hbase_shell_tests_table" + create_test_table(@test_name) + @test_table = table(@test_name) + end + + define_test "is_meta_table? method should return true for the meta table" do + assert(table('.META.').is_meta_table?) + end + + define_test "is_meta_table? method should return true for the root table" do + assert(table('-ROOT-').is_meta_table?) + end + + define_test "is_meta_table? method should return false for a normal table" do + assert(!@test_table.is_meta_table?) + end + + #------------------------------------------------------------------------------- + + define_test "get_all_columns should return columns list" do + cols = table('.META.').get_all_columns + assert_kind_of(Array, cols) + assert(cols.length > 0) + end + + #------------------------------------------------------------------------------- + + define_test "parse_column_name should not return a qualifier for name-only column specifiers" do + col, qual = table('.META.').parse_column_name('foo') + assert_not_nil(col) + assert_nil(qual) + end + + define_test "parse_column_name should not return a qualifier for family-only column specifiers" do + col, qual = table('.META.').parse_column_name('foo:') + assert_not_nil(col) + assert_nil(qual) + end + + define_test "parse_column_name should return a qualifier for family:qualifier column specifiers" do + col, qual = table('.META.').parse_column_name('foo:bar') + assert_not_nil(col) + assert_not_nil(qual) + end + end + + # Simple data management methods tests + class TableSimpleMethodsTest < Test::Unit::TestCase + include TestHelpers + + def setup + setup_hbase + # Create test table if it does not exist + @test_name = "hbase_shell_tests_table" + create_test_table(@test_name) + @test_table = table(@test_name) + + # Insert data to perform delete operations + @test_table.put("101", "x:a", "1") + @test_table.put("101", "x:a", "2", Time.now.to_i) + + @test_table.put("102", "x:a", "1",1212) + @test_table.put("102", "x:a", "2", 1213) + + @test_table.put(103, "x:a", "3") + @test_table.put(103, "x:a", "4") + + @test_table.put("104", "x:a", 5) + @test_table.put("104", "x:b", 6) + + @test_table.put(105, "x:a", "3") + @test_table.put(105, "x:a", "4") + end + + define_test "put should work without timestamp" do + @test_table.put("123", "x:a", "1") + end + + define_test "put should work with timestamp" do + @test_table.put("123", "x:a", "2", Time.now.to_i) + end + + define_test "put should work with integer keys" do + @test_table.put(123, "x:a", "3") + end + + define_test "put should work with integer values" do + @test_table.put("123", "x:a", 4) + end + + #------------------------------------------------------------------------------- + + define_test "delete should work without timestamp" do + @test_table.delete("101", "x:a") + res = @test_table.get('101', 'x:a') + assert_nil(res) + end + + define_test "delete should work with timestamp" do + @test_table.delete("102", "x:a", 1214) + res = @test_table.get('102', 'x:a') + assert_nil(res) + end + + define_test "delete should work with integer keys" do + @test_table.delete(103, "x:a") + res = @test_table.get('103', 'x:a') + assert_nil(res) + end + + #------------------------------------------------------------------------------- + + define_test "deleteall should work w/o columns and timestamps" do + @test_table.deleteall("104") + res = @test_table.get('104', 'x:a', 'x:b') + assert_nil(res) + end + + define_test "deleteall should work with integer keys" do + @test_table.deleteall(105) + res = @test_table.get('105', 'x:a') + assert_nil(res) + end + + #------------------------------------------------------------------------------- + + define_test "incr should work w/o value" do + @test_table.incr("123", 'x:cnt1') + end + + define_test "incr should work with value" do + @test_table.incr("123", 'x:cnt2', 10) + end + + define_test "incr should work with integer keys" do + @test_table.incr(123, 'x:cnt3') + end + + #------------------------------------------------------------------------------- + + define_test "get_counter should work with integer keys" do + @test_table.incr(12345, 'x:cnt') + assert_kind_of(Fixnum, @test_table.get_counter(12345, 'x:cnt')) + end + + define_test "get_counter should return nil for non-existent counters" do + assert_nil(@test_table.get_counter(12345, 'x:qqqq')) + end + end + + # Complex data management methods tests + class TableComplexMethodsTest < Test::Unit::TestCase + include TestHelpers + + def setup + setup_hbase + # Create test table if it does not exist + @test_name = "hbase_shell_tests_table" + create_test_table(@test_name) + @test_table = table(@test_name) + + # Test data + @test_ts = 12345678 + @test_table.put(1, "x:a", 1) + @test_table.put(1, "x:b", 2, @test_ts) + + @test_table.put(2, "x:a", 11) + @test_table.put(2, "x:b", 12, @test_ts) + end + + define_test "count should work w/o a block passed" do + assert(@test_table.count > 0) + end + + define_test "count should work with a block passed (and yield)" do + rows = [] + cnt = @test_table.count(1) do |cnt, row| + rows << row + end + assert(cnt > 0) + assert(!rows.empty?) + end + + #------------------------------------------------------------------------------- + + define_test "get should work w/o columns specification" do + res = @test_table.get('1') + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['x:a']) + assert_not_nil(res['x:b']) + end + + define_test "get should work with integer keys" do + res = @test_table.get(1) + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['x:a']) + assert_not_nil(res['x:b']) + end + + define_test "get should work with hash columns spec and a single string COLUMN parameter" do + res = @test_table.get('1', COLUMN => 'x:a') + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['x:a']) + assert_nil(res['x:b']) + end + + define_test "get should work with hash columns spec and a single string COLUMNS parameter" do + res = @test_table.get('1', COLUMNS => 'x:a') + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['x:a']) + assert_nil(res['x:b']) + end + + define_test "get should work with hash columns spec and an array of strings COLUMN parameter" do + res = @test_table.get('1', COLUMN => [ 'x:a', 'x:b' ]) + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['x:a']) + assert_not_nil(res['x:b']) + end + + define_test "get should work with hash columns spec and an array of strings COLUMNS parameter" do + res = @test_table.get('1', COLUMNS => [ 'x:a', 'x:b' ]) + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['x:a']) + assert_not_nil(res['x:b']) + end + + define_test "get should work with hash columns spec and TIMESTAMP only" do + res = @test_table.get('1', TIMESTAMP => @test_ts) + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_nil(res['x:a']) + assert_not_nil(res['x:b']) + end + + define_test "get should fail with hash columns spec and strange COLUMN value" do + assert_raise(ArgumentError) do + @test_table.get('1', COLUMN => {}) + end + end + + define_test "get should fail with hash columns spec and strange COLUMNS value" do + assert_raise(ArgumentError) do + @test_table.get('1', COLUMN => {}) + end + end + + define_test "get should fail with hash columns spec and no TIMESTAMP or COLUMN[S]" do + assert_raise(ArgumentError) do + @test_table.get('1', { :foo => :bar }) + end + end + + define_test "get should work with a string column spec" do + res = @test_table.get('1', 'x:b') + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_nil(res['x:a']) + assert_not_nil(res['x:b']) + end + + define_test "get should work with an array columns spec" do + res = @test_table.get('1', 'x:a', 'x:b') + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['x:a']) + assert_not_nil(res['x:b']) + end + + define_test "get should work with an array or arrays columns spec (yeah, crazy)" do + res = @test_table.get('1', ['x:a'], ['x:b']) + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['x:a']) + assert_not_nil(res['x:b']) + end + + define_test "get with a block should yield (column, value) pairs" do + res = {} + @test_table.get('1') { |col, val| res[col] = val } + assert_equal(res.keys.sort, [ 'x:a', 'x:b' ]) + end + + define_test "get should support FILTER" do + @test_table.put(1, "x:v", "thisvalue") + begin + res = @test_table.get('1', FILTER => "ValueFilter(=, 'binary:thisvalue')") + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['x:v']) + assert_nil(res['x:a']) + res = @test_table.get('1', FILTER => "ValueFilter(=, 'binary:thatvalue')") + assert_nil(res) + ensure + # clean up newly added columns for this test only. + @test_table.delete(1, "x:v") + end + end + + #------------------------------------------------------------------------------- + + define_test "scan should work w/o any params" do + res = @test_table.scan + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['1']) + assert_not_nil(res['1']['x:a']) + assert_not_nil(res['1']['x:b']) + assert_not_nil(res['2']) + assert_not_nil(res['2']['x:a']) + assert_not_nil(res['2']['x:b']) + end + + define_test "scan should support STARTROW parameter" do + res = @test_table.scan STARTROW => '2' + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_nil(res['1']) + assert_not_nil(res['2']) + assert_not_nil(res['2']['x:a']) + assert_not_nil(res['2']['x:b']) + end + + define_test "scan should support STOPROW parameter" do + res = @test_table.scan STOPROW => '2' + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['1']) + assert_not_nil(res['1']['x:a']) + assert_not_nil(res['1']['x:b']) + assert_nil(res['2']) + end + + define_test "scan should support LIMIT parameter" do + res = @test_table.scan LIMIT => 1 + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['1']) + assert_not_nil(res['1']['x:a']) + assert_not_nil(res['1']['x:b']) + assert_nil(res['2']) + end + + define_test "scan should support TIMESTAMP parameter" do + res = @test_table.scan TIMESTAMP => @test_ts + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['1']) + assert_nil(res['1']['x:a']) + assert_not_nil(res['1']['x:b']) + assert_not_nil(res['2']) + assert_nil(res['2']['x:a']) + assert_not_nil(res['2']['x:b']) + end + + define_test "scan should support TIMERANGE parameter" do + res = @test_table.scan TIMERANGE => [0, 1] + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_nil(res['1']) + assert_nil(res['2']) + end + + define_test "scan should support COLUMNS parameter with an array of columns" do + res = @test_table.scan COLUMNS => [ 'x:a', 'x:b' ] + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['1']) + assert_not_nil(res['1']['x:a']) + assert_not_nil(res['1']['x:b']) + assert_not_nil(res['2']) + assert_not_nil(res['2']['x:a']) + assert_not_nil(res['2']['x:b']) + end + + define_test "scan should support COLUMNS parameter with a single column name" do + res = @test_table.scan COLUMNS => 'x:a' + assert_not_nil(res) + assert_kind_of(Hash, res) + assert_not_nil(res['1']) + assert_not_nil(res['1']['x:a']) + assert_nil(res['1']['x:b']) + assert_not_nil(res['2']) + assert_not_nil(res['2']['x:a']) + assert_nil(res['2']['x:b']) + end + + define_test "scan should fail on invalid COLUMNS parameter types" do + assert_raise(ArgumentError) do + @test_table.scan COLUMNS => {} + end + end + + define_test "scan should fail on non-hash params" do + assert_raise(ArgumentError) do + @test_table.scan 123 + end + end + + define_test "scan with a block should yield rows and return rows counter" do + rows = {} + res = @test_table.scan { |row, cells| rows[row] = cells } + assert_equal(rows.keys.size, res) + end + + define_test "scan should support FILTER" do + @test_table.put(1, "x:v", "thisvalue") + begin + res = @test_table.scan FILTER => "ValueFilter(=, 'binary:thisvalue')" + assert_not_equal(res, {}, "Result is empty") + assert_kind_of(Hash, res) + assert_not_nil(res['1']) + assert_not_nil(res['1']['x:v']) + assert_nil(res['1']['x:a']) + assert_nil(res['2']) + res = @test_table.scan FILTER => "ValueFilter(=, 'binary:thatvalue')" + assert_equal(res, {}, "Result is not empty") + ensure + # clean up newly added columns for this test only. + @test_table.delete(1, "x:v") + end + end + + end +end diff --git a/src/test/ruby/shell/commands_test.rb b/src/test/ruby/shell/commands_test.rb new file mode 100644 index 0000000..1a315a7 --- /dev/null +++ b/src/test/ruby/shell/commands_test.rb @@ -0,0 +1,34 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'shell' +require 'shell/formatter' + +class ShellCommandsTest < Test::Unit::TestCase + Shell.commands.each do |name, klass| + define_test "#{name} command class #{klass} should respond to help" do + assert_respond_to(klass.new(nil), :help) + end + + define_test "#{name} command class #{klass} should respond to :command" do + assert_respond_to(klass.new(nil), :command) + end + end +end diff --git a/src/test/ruby/shell/formatter_test.rb b/src/test/ruby/shell/formatter_test.rb new file mode 100644 index 0000000..5b5c636 --- /dev/null +++ b/src/test/ruby/shell/formatter_test.rb @@ -0,0 +1,69 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'shell/formatter' + +class ShellFormatterTest < Test::Unit::TestCase + # Helper method to construct a null formatter + def formatter + Shell::Formatter::Base.new(:output_stream => STDOUT) + end + + # + # Constructor tests + # + define_test "Formatter constructor should not raise error valid IO streams" do + assert_nothing_raised do + Shell::Formatter::Base.new(:output_stream => STDOUT) + end + end + + define_test "Formatter constructor should not raise error when no IO stream passed" do + assert_nothing_raised do + Shell::Formatter::Base.new() + end + end + + define_test "Formatter constructor should raise error on non-IO streams" do + assert_raise TypeError do + Shell::Formatter::Base.new(:output_stream => 'foostring') + end + end + + #------------------------------------------------------------------------------------------------------- + # Printing methods tests + # FIXME: The tests are just checking that the code has no typos, try to figure out a better way to test + # + define_test "Formatter#header should work" do + formatter.header(['a', 'b']) + formatter.header(['a', 'b'], [10, 20]) + end + + define_test "Formatter#row should work" do + formatter.row(['a', 'b']) + formatter.row(['xxxxxxxxx xxxxxxxxxxx xxxxxxxxxxx xxxxxxxxxxxx xxxxxxxxx xxxxxxxxxxxx xxxxxxxxxxxxxxx xxxxxxxxx xxxxxxxxxxxxxx']) + formatter.row(['yyyyyy yyyyyy yyyyy yyy', 'xxxxxxxxx xxxxxxxxxxx xxxxxxxxxxx xxxxxxxxxxxx xxxxxxxxx xxxxxxxxxxxx xxxxxxxxxxxxxxx xxxxxxxxx xxxxxxxxxxxxxx xxx xx x xx xxx xx xx xx x xx x x xxx x x xxx x x xx x x x x x x xx ']) + formatter.row(["NAME => 'table1', FAMILIES => [{NAME => 'fam2', VERSIONS => 3, COMPRESSION => 'NONE', IN_MEMORY => false, BLOCKCACHE => false, LENGTH => 2147483647, TTL => FOREVER, BLOOMFILTER => NONE}, {NAME => 'fam1', VERSIONS => 3, COMPRESSION => 'NONE', IN_MEMORY => false, BLOCKCACHE => false, LENGTH => 2147483647, TTL => FOREVER, BLOOMFILTER => NONE}]"]) + end + + define_test "Froematter#footer should work" do + formatter.footer(Time.now - 5) + end +end diff --git a/src/test/ruby/shell/shell_test.rb b/src/test/ruby/shell/shell_test.rb new file mode 100644 index 0000000..bc3000c --- /dev/null +++ b/src/test/ruby/shell/shell_test.rb @@ -0,0 +1,70 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'hbase' +require 'shell' +require 'shell/formatter' + +class ShellTest < Test::Unit::TestCase + def setup + @formatter = ::Shell::Formatter::Console.new() + @hbase = ::Hbase::Hbase.new($TEST_CLUSTER.getConfiguration) + @shell = Shell::Shell.new(@hbase, @formatter) + end + + define_test "Shell::Shell#hbase_admin should return an admin instance" do + assert_kind_of(Hbase::Admin, @shell.hbase_admin) + end + + define_test "Shell::Shell#hbase_admin should cache admin instances" do + assert_same(@shell.hbase_admin, @shell.hbase_admin) + end + + #------------------------------------------------------------------------------- + + define_test "Shell::Shell#hbase_table should return a table instance" do + assert_kind_of(Hbase::Table, @shell.hbase_table('.META.')) + end + + define_test "Shell::Shell#hbase_table should not cache table instances" do + assert_not_same(@shell.hbase_table('.META.'), @shell.hbase_table('.META.')) + end + + #------------------------------------------------------------------------------- + + define_test "Shell::Shell#export_commands should export command methods to specified object" do + module Foo; end + assert(!Foo.respond_to?(:version)) + @shell.export_commands(Foo) + assert(Foo.respond_to?(:version)) + end + + #------------------------------------------------------------------------------- + + define_test "Shell::Shell#command_instance should return a command class" do + assert_kind_of(Shell::Commands::Command, @shell.command_instance('version')) + end + + #------------------------------------------------------------------------------- + + define_test "Shell::Shell#command should execute a command" do + @shell.command('version') + end +end diff --git a/src/test/ruby/test_helper.rb b/src/test/ruby/test_helper.rb new file mode 100644 index 0000000..14eeaf8 --- /dev/null +++ b/src/test/ruby/test_helper.rb @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require 'test/unit' + +module Testing + module Declarative + # define_test "should do something" do + # ... + # end + def define_test(name, &block) + test_name = "test_#{name.gsub(/\s+/,'_')}".to_sym + defined = instance_method(test_name) rescue false + raise "#{test_name} is already defined in #{self}" if defined + if block_given? + define_method(test_name, &block) + else + define_method(test_name) do + flunk "No implementation provided for #{name}" + end + end + end + end +end + +module Hbase + module TestHelpers + def setup_hbase + @formatter = Shell::Formatter::Console.new() + @hbase = ::Hbase::Hbase.new($TEST_CLUSTER.getConfiguration) + end + + def table(table) + @hbase.table(table, @formatter) + end + + def admin + @hbase.admin(@formatter) + end + + def create_test_table(name) + # Create the table if needed + unless admin.exists?(name) + admin.create name, [{'NAME' => 'x', 'VERSIONS' => 5}, 'y'] + return + end + + # Enable the table if needed + unless admin.enabled?(name) + admin.enable(name) + end + end + + def drop_test_table(name) + return unless admin.exists?(name) + begin + admin.disable(name) if admin.enabled?(name) + rescue => e + puts "IGNORING DISABLE TABLE ERROR: #{e}" + end + begin + admin.drop(name) + rescue => e + puts "IGNORING DROP TABLE ERROR: #{e}" + end + end + end +end + +# Extend standard unit tests with our helpers +Test::Unit::TestCase.extend(Testing::Declarative) + +# Add the $HBASE_HOME/lib/ruby directory to the ruby +# load path so I can load up my HBase ruby modules +$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "..", "main", "ruby") diff --git a/src/test/ruby/tests_runner.rb b/src/test/ruby/tests_runner.rb new file mode 100644 index 0000000..251473f --- /dev/null +++ b/src/test/ruby/tests_runner.rb @@ -0,0 +1,66 @@ +# +# Copyright 2010 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'rubygems' +require 'rake' + +unless defined?($TEST_CLUSTER) + include Java + + # Set logging level to avoid verboseness + org.apache.log4j.Logger.getRootLogger.setLevel(org.apache.log4j.Level::OFF) + org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(org.apache.log4j.Level::OFF) + org.apache.log4j.Logger.getLogger("org.apache.hadoop.hdfs").setLevel(org.apache.log4j.Level::OFF) + org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(org.apache.log4j.Level::OFF) + org.apache.log4j.Logger.getLogger("org.apache.hadoop.ipc.HBaseServer").setLevel(org.apache.log4j.Level::OFF) + + java_import org.apache.hadoop.hbase.HBaseTestingUtility + + $TEST_CLUSTER = HBaseTestingUtility.new + $TEST_CLUSTER.configuration.setInt("hbase.regionserver.msginterval", 100) + $TEST_CLUSTER.configuration.setInt("hbase.client.pause", 250) + $TEST_CLUSTER.configuration.setInt("hbase.client.retries.number", 6) + $TEST_CLUSTER.startMiniCluster + @own_cluster = true +end + +require 'test_helper' + +puts "Running tests..." + +files = Dir[ File.dirname(__FILE__) + "/**/*_test.rb" ] +files.each do |file| + begin + load(file) + rescue => e + puts "ERROR: #{e}" + raise + end +end + +if !Test::Unit::AutoRunner.run + raise "Shell unit tests failed. Check output file for details." +end + +puts "Done with tests! Shutting down the cluster..." +if @own_cluster + $TEST_CLUSTER.shutdownMiniCluster + java.lang.System.exit(0) +end